From eca505acaae7001c64a3bcb7923b65b0a9623590 Mon Sep 17 00:00:00 2001
From: Greg Cocks <greg.cocks@noaa.gov>
Date: Tue, 29 Dec 2020 13:56:07 -0600
Subject: [PATCH 001/359] Fixed Typo

Fixed small typo in add_crosswalk.py
---
 lib/add_crosswalk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/add_crosswalk.py b/lib/add_crosswalk.py
index 6ae1c6980..5d66da70b 100755
--- a/lib/add_crosswalk.py
+++ b/lib/add_crosswalk.py
@@ -18,7 +18,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     if extent == 'FR':
         ## crosswalk using majority catchment method
 
-        # calculate majority catchemnts
+        # calculate majority catchments
         majority_calc = zonal_stats(input_catchments, input_nwmcatras_fileName, stats=['majority'], geojson_out=True)
         input_majorities = gpd.GeoDataFrame.from_features(majority_calc)
         input_majorities = input_majorities.rename(columns={'majority' : 'feature_id'})

From 2ce8b6a0421c244e41072cdae790e7a37269930c Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matthew.luck@gmail.com>
Date: Thu, 31 Dec 2020 14:36:33 -0500
Subject: [PATCH 002/359] Modifications to build and run Docker image more
 reliably

Modifications to build and run Docker image more reliably. Closes #178.

Changes
Changed to noninteractive install of grass. Fixes #182.

Notes
aggregate_vector_inputs.py doesn't work yet. Need to externally download required data to run fim_run.sh
---
 .gitignore                           |  1 +
 CHANGELOG.md                         | 13 ++++++-
 Dockerfile.dev                       |  3 +-
 install_grass.exp                    | 52 ----------------------------
 lib/acquire_and_preprocess_inputs.py |  1 +
 lib/aggregate_vector_inputs.py       |  8 ++---
 lib/derive_headwaters.py             |  2 +-
 7 files changed, 20 insertions(+), 60 deletions(-)
 delete mode 100755 install_grass.exp

diff --git a/.gitignore b/.gitignore
index 370f4f97d..ec0085323 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ config/**
 !config/*default*
 !config/*calibrated*
 !config/symbology/
+.vscode/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 867435f65..bc1459cb2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.0.1 - 2020-12-31 - [PR #184](https://github.com/NOAA-OWP/cahaba/pull/184)
 
-## v3.0.0.0 - 2020-12-22
+Modifications to build and run Docker image more reliably. Cleanup on some pre-processing scripts.
+
+### Changes
+
+ - Changed to noninteractive install of GRASS.
+ - Changed some paths from relative to absolute and cleaned up some python shebang lines.
+
+### Notes
+ - `aggregate_vector_inputs.py` doesn't work yet. Need to externally download required data to run fim_run.sh
+ 
+## v3.0.0.0 - 2020-12-22 - [PR #181](https://github.com/NOAA-OWP/cahaba/pull/181)
 
 The software released here builds on the flood inundation mapping capabilities demonstrated as part of the National Flood Interoperability Experiment, the Office of Water Prediction's Innovators Program and the National Water Center Summer Institute. The flood inundation mapping software implements the Height Above Nearest Drainage (HAND) algorithm and incorporates community feedback and lessons learned over several years. The software has been designed to meet the requirements set by stakeholders interested in flood prediction and has been developed in partnership with several entities across the water enterprise.
diff --git a/Dockerfile.dev b/Dockerfile.dev
index a57389da8..626ef94ef 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -74,8 +74,7 @@ COPY --from=builder $depDir $depDir
 RUN apt update --fix-missing
 RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
 
-COPY install_grass.exp .
-RUN ./install_grass.exp
+RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
 
 RUN apt auto-remove
 
diff --git a/install_grass.exp b/install_grass.exp
deleted file mode 100755
index e1320ffb9..000000000
--- a/install_grass.exp
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/expect -f
-#
-# This Expect script was generated by autoexpect on Thu Oct 22 20:27:42 2020
-# Expect and autoexpect were both written by Don Libes, NIST.
-#
-# Note that autoexpect does not guarantee a working script.  It
-# necessarily has to guess about certain things.  Two reasons a script
-# might fail are:
-#
-# 1) timing - A surprising number of programs (rn, ksh, zsh, telnet,
-# etc.) and devices discard or ignore keystrokes that arrive "too
-# quickly" after prompts.  If you find your new script hanging up at
-# one spot, try adding a short sleep just before the previous send.
-# Setting "force_conservative" to 1 (see below) makes Expect do this
-# automatically - pausing briefly before sending each character.  This
-# pacifies every program I know of.  The -c flag makes the script do
-# this in the first place.  The -C flag allows you to define a
-# character to toggle this mode off and on.
-
-set force_conservative 0  ;# set to 1 to force conservative mode even if
-                          ;# script wasn't run conservatively originally
-if {$force_conservative} {
-        set send_slow {1 .1}
-        proc send {ignore arg} {
-                sleep .1
-                exp_send -s -- $arg
-        }
-}
-
-#
-# 2) differing output - Some programs produce different output each time
-# they run.  The "date" command is an obvious example.  Another is
-# ftp, if it produces throughput statistics at the end of a file
-# transfer.  If this causes a problem, delete these patterns or replace
-# them with wildcards.  An alternative is to use the -p flag (for
-# "prompt") which makes Expect only look for the last line of output
-# (i.e., the prompt).  The -P flag allows you to define a character to
-# toggle this mode off and on.
-#
-# Read the man page for more info.
-#
-# -Don
-
-
-set timeout -1
-spawn apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
-match_max 100000
-expect -exact "Country of origin for the keyboard: "
-send -- "31\r"
-expect -exact "Keyboard layout: "
-send -- "1\r"
-expect eof
diff --git a/lib/acquire_and_preprocess_inputs.py b/lib/acquire_and_preprocess_inputs.py
index f3b200e94..446e6955e 100755
--- a/lib/acquire_and_preprocess_inputs.py
+++ b/lib/acquire_and_preprocess_inputs.py
@@ -8,6 +8,7 @@
 from multiprocessing import Pool
 import geopandas as gp
 from urllib.error import HTTPError
+from tqdm import tqdm
 
 from utils.shared_variables import (NHD_URL_PARENT,
                                     NHD_URL_PREFIX,
diff --git a/lib/aggregate_vector_inputs.py b/lib/aggregate_vector_inputs.py
index 6d9b2abc9..48dadecbf 100755
--- a/lib/aggregate_vector_inputs.py
+++ b/lib/aggregate_vector_inputs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env·python3
+#!/usr/bin/env python3
 
 import os
 import geopandas as gpd
@@ -8,9 +8,9 @@
 from derive_headwaters import findHeadWaterPoints
 from tqdm import tqdm
 
-in_dir ='data/inputs/nhdplus_vectors'
-nhd_dir ='data/inputs/nhdplus_vectors_aggregate'
-nwm_dir = 'data/inputs/nwm_hydrofabric'
+in_dir ='/data/inputs/nhdplus_vectors'
+nhd_dir ='/data/inputs/nhdplus_vectors_aggregate'
+nwm_dir = '/data/inputs/nwm_hydrofabric'
 
 ## NWM Headwaters
 print ('deriving NWM headwater points')
diff --git a/lib/derive_headwaters.py b/lib/derive_headwaters.py
index e0c39f6b5..d29381c60 100644
--- a/lib/derive_headwaters.py
+++ b/lib/derive_headwaters.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env·python3
+#!/usr/bin/env python3
 
 import geopandas as gpd
 from shapely.geometry import Point

From 68b1969bc28b123c7bc1044f19e5a258e54815be Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Wed, 6 Jan 2021 14:14:08 -0600
Subject: [PATCH 003/359] hotfix to address AHPSs mapping errors (#200)

* handle dytpe and leading zeros for hydrotable

* exit inundation.py when all stream segments are lakes
---
 lib/add_crosswalk.py | 5 +++++
 lib/run_by_unit.sh   | 2 +-
 tests/inundation.py  | 7 ++++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/lib/add_crosswalk.py b/lib/add_crosswalk.py
index 5d66da70b..1f51efb4d 100755
--- a/lib/add_crosswalk.py
+++ b/lib/add_crosswalk.py
@@ -135,8 +135,13 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_flows['HydroID'] = output_flows.HydroID.str.zfill(8)
     output_hydro_table = output_hydro_table.merge(output_flows.loc[:,['HydroID','LakeID']],how='left',on='HydroID')
     output_hydro_table['LakeID'] = output_hydro_table['LakeID'].astype(int)
+
     output_hydro_table = output_hydro_table.rename(columns={'HUC8':'HUC'})
+    if output_hydro_table.HUC.dtype != 'str': output_hydro_table.HUC = output_hydro_table.HUC.astype(str)
+    output_hydro_table.HUC = output_hydro_table.HUC.str.zfill(8)
+
     output_hydro_table.drop(columns='fossid',inplace=True)
+    if output_hydro_table.feature_id.dtype != 'int': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(int)
     if output_hydro_table.feature_id.dtype != 'str': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(str)
 
     # write out based on mode
diff --git a/lib/run_by_unit.sh b/lib/run_by_unit.sh
index d2c63a208..7a19cd88b 100755
--- a/lib/run_by_unit.sh
+++ b/lib/run_by_unit.sh
@@ -59,7 +59,7 @@ echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-$libDir/snap_and_clip_to_nhd.py -d $hucNumber -w $input_NWM_Flows -f $input_NWM_Headwaters -s $input_NHD_Flowlines -l $input_NWM_Lakes -r $input_NLD -u $outputHucDataDir/wbd.gpkg -g $outputHucDataDir/wbd_buffered.gpkg -y $inputDataDir/ahp_sites/ahps_sites.gpkg -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -t $outputHucDataDir/nwm_headwaters_proj_subset.gpkg -m $input_NWM_Catchments -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -p $extent
+$libDir/snap_and_clip_to_nhd.py -d $hucNumber -w $input_NWM_Flows -f $input_NWM_Headwaters -s $input_NHD_Flowlines -l $input_NWM_Lakes -r $input_NLD -u $outputHucDataDir/wbd.gpkg -g $outputHucDataDir/wbd_buffered.gpkg -y $inputDataDir/ahp_sites/nws_lid.gpkg -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -t $outputHucDataDir/nwm_headwaters_proj_subset.gpkg -m $input_NWM_Catchments -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -p $extent
 Tcount
 
 if [ "$extent" = "MS" ]; then
diff --git a/tests/inundation.py b/tests/inundation.py
index a07f8d96f..0496d8d1b 100755
--- a/tests/inundation.py
+++ b/tests/inundation.py
@@ -400,7 +400,7 @@ def __return_huc_in_hucSet(hucCode,hucSet):
                     rem_array,window_transform = mask(rem,[shape(huc['geometry'])],crop=True,indexes=1)
                     catchments_array,_ = mask(catchments,[shape(huc['geometry'])],crop=True,indexes=1)
                 elif mask_type == "filter":
-                    
+
                     # input catchments polygon
                     if isinstance(catchment_poly,str):
                         catchment_poly=gpd.read_file(catchment_poly)
@@ -458,6 +458,11 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
                                 )
         hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
         hydroTable = hydroTable[hydroTable["LakeID"] == -999]  # Subset hydroTable to include only non-lake catchments.
+
+        if hydroTable.empty:
+            print ("All stream segments in this HUC are within lake boundaries.")
+            sys.exit(0)
+
     elif isinstance(hydroTable,pd.DataFrame):
         pass #consider checking for correct dtypes, indices, and columns
     else:

From 9bffb885f32dfcd95978c7ccd2639f9df56ff829 Mon Sep 17 00:00:00 2001
From: FernandoSalas-NOAA <fernando.salas@noaa.gov>
Date: Thu, 7 Jan 2021 22:35:27 -0600
Subject: [PATCH 004/359] Update README.md

Added to reference list
---
 README.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 63c2a27f2..69a815b6d 100644
--- a/README.md
+++ b/README.md
@@ -112,8 +112,9 @@ NOAA's National Water Center welcomes anyone to contribute to the Cahaba reposit
 2. National Flood Interoperability Experiment [(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)
 3. Garousi‐Nejad, I., Tarboton, D. G.,Aboutalebi, M., & Torres‐Rua, A.(2019). Terrain analysis enhancements to the Height Above Nearest Drainage flood inundation mapping method. Water Resources Research, 55 , 7983–8009. https://doi.org/10.1029/2019WR0248375.
 4. Zheng, X., D.G. Tarboton, D.R. Maidment, Y.Y. Liu, and P. Passalacqua. 2018. “River Channel Geometry and Rating Curve Estimation Using Height above the Nearest Drainage.” Journal of the American Water Resources Association 54 (4): 785–806. https://doi.org/10.1111/1752-1688.12661.
-5. Barnes, Richard. 2016. RichDEM: Terrain Analysis Software. http://github.com/r-barnes/richdem
-6. [TauDEM](https://github.com/dtarb/TauDEM)
-7. Federal Emergency Management Agency (FEMA) Base Level Engineering [(BLE)](https://webapps.usgs.gov/infrm/estBFE/)
-8. Verdin, James; Verdin, Kristine; Mathis, Melissa; Magadzire, Tamuka; Kabuchanga, Eric; Woodbury, Mark; and Gadain, Hussein, 2016, A software tool for rapid flood inundation mapping: U.S. Geological Survey Open-File Report 2016–1038, 26 p., http://dx.doi.org/10.3133/ofr20161038.
-9. United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR). https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution
+5. Liu, Y. Y., D. R. Maidment, D. G. Tarboton, X. Zheng and S. Wang, (2018), "A CyberGIS Integration and Computation Framework for High-Resolution Continental-Scale Flood Inundation Mapping," JAWRA Journal of the American Water Resources Association, 54(4): 770-784, https://doi.org/10.1111/1752-1688.12660.
+6. Barnes, Richard. 2016. RichDEM: Terrain Analysis Software. http://github.com/r-barnes/richdem
+7. [TauDEM](https://github.com/dtarb/TauDEM)
+8. Federal Emergency Management Agency (FEMA) Base Level Engineering [(BLE)](https://webapps.usgs.gov/infrm/estBFE/)
+9. Verdin, James; Verdin, Kristine; Mathis, Melissa; Magadzire, Tamuka; Kabuchanga, Eric; Woodbury, Mark; and Gadain, Hussein, 2016, A software tool for rapid flood inundation mapping: U.S. Geological Survey Open-File Report 2016–1038, 26 p., http://dx.doi.org/10.3133/ofr20161038.
+10. United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR). https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution

From 474c209751942940e8943bc9d858c875b102c0a0 Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Fri, 8 Jan 2021 12:18:15 -0600
Subject: [PATCH 005/359] Update CHANGELOG.md

---
 CHANGELOG.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bc1459cb2..e17cbd453 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.0.2 - 2021-01-06 - [PR #200](https://github.com/NOAA-OWP/cahaba/pull/200)
+
+Patch to address AHPSs mapping errors.
+
+### Changes
+
+ - Checks `dtype` of `hydroTable.csv` columns to resolve errors caused in `inundation.py` when joining to flow forecast.
+ - Exits `inundation.py` when all hydrotable HydroIDs are lake features.
+ - Updates path to latest AHPs site layer.
+ - Updated [readme](https://github.com/NOAA-OWP/cahaba/commit/9bffb885f32dfcd95978c7ccd2639f9df56ff829)
+
 ## v3.0.0.1 - 2020-12-31 - [PR #184](https://github.com/NOAA-OWP/cahaba/pull/184)
 
 Modifications to build and run Docker image more reliably. Cleanup on some pre-processing scripts.

From 0993052ebc51530af136e6dfc9c393aa090400bd Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Thu, 14 Jan 2021 09:08:45 -0600
Subject: [PATCH 006/359] Hotfix for handling nodata value in rasterized levee
 lines

- Resolves bug for HUCs where $ndv > 0 (Great Lakes region)
- Initialize the nld_rasterized_elev.tif using a value of -9999 instead of $ndv
---
 CHANGELOG.md       | 9 +++++++++
 lib/run_by_unit.sh | 4 ++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e17cbd453..608c93979 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.0.3 - 2021-01-14 - [PR #210](https://github.com/NOAA-OWP/cahaba/pull/210)
+
+Hotfix for handling nodata value in rasterized levee lines.
+
+### Changes
+
+ - Resolves bug for HUCs where `$ndv > 0` (Great Lakes region).
+ - Initialize the `nld_rasterized_elev.tif` using a value of `-9999` instead of `$ndv`.
+ 
 ## v3.0.0.2 - 2021-01-06 - [PR #200](https://github.com/NOAA-OWP/cahaba/pull/200)
 
 Patch to address AHPSs mapping errors.
diff --git a/lib/run_by_unit.sh b/lib/run_by_unit.sh
index 7a19cd88b..81bc676da 100755
--- a/lib/run_by_unit.sh
+++ b/lib/run_by_unit.sh
@@ -96,7 +96,7 @@ echo -e $startDiv"Rasterize all NLD multilines using zelev vertices"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/nld_rasterized_elev.tif ] && [ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
-gdal_rasterize -l nld_subset_levees -3d -at -init $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputHucDataDir/nld_rasterized_elev.tif
+gdal_rasterize -l nld_subset_levees -3d -at -init -9999 -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputHucDataDir/nld_rasterized_elev.tif
 Tcount
 
 ## CONVERT TO METERS ##
@@ -138,7 +138,7 @@ echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwr
 date -u
 Tstart
 [ -f $outputHucDataDir/nld_rasterized_elev.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --NoDataValue $ndv --co "BLOCKXSIZE=512" --co "BLOCKYSIZE=512" --co "TILED=YES" --co "COMPRESS=LZW" --co "BIGTIFF=YES" -A $outputHucDataDir/dem_meters.tif -B $outputHucDataDir/nld_rasterized_elev.tif --outfile="$outputHucDataDir/dem_meters.tif" --calc="maximum(A,(B*0.3048))" --NoDataValue=$ndv
+gdal_calc.py --quiet --type=Float32 --overwrite --NoDataValue $ndv --co "BLOCKXSIZE=512" --co "BLOCKYSIZE=512" --co "TILED=YES" --co "COMPRESS=LZW" --co "BIGTIFF=YES" -A $outputHucDataDir/dem_meters.tif -B $outputHucDataDir/nld_rasterized_elev.tif --outfile="$outputHucDataDir/dem_meters.tif" --calc="maximum(A,((B>-9999)*0.3048))" --NoDataValue=$ndv
 Tcount
 
 ## DEM Reconditioning ##

From d9c6a2afb517b39259d1df6279d96cdb114f8b14 Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Wed, 20 Jan 2021 12:01:15 -0600
Subject: [PATCH 007/359] Changed the directory where the included_huc*.lst
 files are being read from.

Changed the directory where the `included_huc*.lst` files are being read from.
---
 CHANGELOG.md            | 8 ++++++++
 lib/check_huc_inputs.py | 8 ++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 608c93979..5133f88e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.0.4 - 2021-01-20 - [PR #230](https://github.com/NOAA-OWP/cahaba/pull/230)
+
+Changed the directory where the `included_huc*.lst` files are being read from.
+
+### Changes
+
+ - Changed the directory where the `included_huc*.lst` files are being read from.
+
 ## v3.0.0.3 - 2021-01-14 - [PR #210](https://github.com/NOAA-OWP/cahaba/pull/210)
 
 Hotfix for handling nodata value in rasterized levee lines.
diff --git a/lib/check_huc_inputs.py b/lib/check_huc_inputs.py
index b1b83e7c0..1302e18ad 100755
--- a/lib/check_huc_inputs.py
+++ b/lib/check_huc_inputs.py
@@ -5,7 +5,7 @@
 from glob import glob
 
 def __read_included_files(parent_dir_path):
-    
+
     filename_patterns = glob(os.path.join(parent_dir_path,'included_huc*.lst'))
 
     accepted_hucs_set = set()
@@ -23,7 +23,7 @@ def __read_included_files(parent_dir_path):
 
 
 def __read_input_hucs(hucs):
-    
+
     hucs = [h.split() for h in hucs][0]
     if os.path.isfile(hucs[0]):
         with open(hucs[0],'r') as hucs_file:
@@ -42,7 +42,7 @@ def __check_for_membership(hucs,accepted_hucs_set):
 
 def check_hucs(hucs):
 
-    accepted_hucs = __read_included_files(os.environ['inputDataDir'])
+    accepted_hucs = __read_included_files(os.path.join(os.environ['inputDataDir'],'huc_lists'))
     hucs = __read_input_hucs(hucs)
     __check_for_membership(hucs,accepted_hucs)
 
@@ -54,6 +54,6 @@ def check_hucs(hucs):
 
     # extract to dictionary
     args = vars(parser.parse_args())
-    
+
     # call function
     check_hucs(**args)

From 8b6eb18f75665b5b2bd06297a373078f1dca5825 Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Thu, 21 Jan 2021 10:38:15 -0600
Subject: [PATCH 008/359] Preprocess MS and FR stream networks

* headwater stream segments geometries are adjusted to align with with NWM streams
* incoming streams are selected using intersection points between NWM streams and HUC4 boundaries
* clip_vectors_to_wbd.py handles local headwaters
* removes NHDPlus features categorized as coastline and underground conduit
* added streams layer to production whitelist
* fixed progress bar in lib/acquire_and_preprocess_inputs.py
* added getDriver to shared functions.py
* cleaned up variable names and types

This addresses Github issues #58, #12, #118, #73, and partial resolution of #4
---
 CHANGELOG.md                         |  15 +
 config/params_calibrated.env         |   8 +-
 config/params_template.env           |   8 +-
 fim_run.sh                           |   5 +-
 lib/acquire_and_preprocess_inputs.py |   2 +-
 lib/adjust_headwater_streams.py      | 142 +++++++++
 lib/aggregate_vector_inputs.py       | 445 ++++++++++++++++++++++++---
 lib/buildstreamtraversal.py          |  93 +++---
 lib/clip_vectors_to_wbd.py           | 145 +++++++++
 lib/output_cleanup.py                |  16 +-
 lib/reduce_nhd_stream_density.py     | 165 ++++++++++
 lib/run_by_unit.sh                   |  26 +-
 lib/snap_and_clip_to_nhd.py          | 298 ------------------
 lib/split_flows.py                   |   3 +
 lib/utils/shared_functions.py        |  43 +--
 15 files changed, 978 insertions(+), 436 deletions(-)
 create mode 100644 lib/adjust_headwater_streams.py
 create mode 100755 lib/clip_vectors_to_wbd.py
 create mode 100644 lib/reduce_nhd_stream_density.py
 delete mode 100755 lib/snap_and_clip_to_nhd.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5133f88e8..d028b692d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.1.0 - 2021-01-21 - [PR #206](https://github.com/NOAA-OWP/cahaba/pull/206)
+
+Preprocess MS and FR stream networks
+
+### Changes
+
+ - Headwater stream segments geometries are adjusted to align with with NWM streams.
+ - Incoming streams are selected using intersection points between NWM streams and HUC4 boundaries.
+ - `clip_vectors_to_wbd.py` handles local headwaters.
+ - Removes NHDPlus features categorized as coastline and underground conduit.  
+ - Added streams layer to production whitelist.
+ - Fixed progress bar in `lib/acquire_and_preprocess_inputs.py`.
+ - Added `getDriver` to shared `functions.py`.
+ - Cleaned up variable names and types.
+
 ## v3.0.0.4 - 2021-01-20 - [PR #230](https://github.com/NOAA-OWP/cahaba/pull/230)
 
 Changed the directory where the `included_huc*.lst` files are being read from.
diff --git a/config/params_calibrated.env b/config/params_calibrated.env
index 09e7f0167..f150d0428 100644
--- a/config/params_calibrated.env
+++ b/config/params_calibrated.env
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 #### geospatial parameters ####
-export negativeBurnValue=1000
-export buffer=70
-export maxSplitDistance_meters=1500
+export negative_burn_value=1000
+export agree_DEM_buffer=70
+export max_split_distance_meters=1500
 export manning_n="/foss_fim/config/mannings_calibrated.json"
 export stage_min_meters=0
 export stage_interval_meters=0.3048
@@ -15,7 +15,7 @@ export lakes_buffer_dist_meters=20
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
-export defaultMaxJobs=1 # default number of max concurrent jobs to run
+export default_max_jobs=1 # default number of max concurrent jobs to run
 export memfree=0G # min free memory required to start a new job or keep youngest job alive
 
 #### logging parameters ####
diff --git a/config/params_template.env b/config/params_template.env
index f87b4c15a..04af26828 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 #### geospatial parameters ####
-export negativeBurnValue=1000
-export buffer=70
-export maxSplitDistance_meters=1500
+export negative_burn_value=1000
+export agree_DEM_buffer=70
+export max_split_distance_meters=1500
 export manning_n="/foss_fim/config/mannings_default.json"
 export stage_min_meters=0
 export stage_interval_meters=0.3048
@@ -15,7 +15,7 @@ export lakes_buffer_dist_meters=20
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
-export defaultMaxJobs=1 # default number of max concurrent jobs to run
+export default_max_jobs=1 # default number of max concurrent jobs to run
 export memfree=0G # min free memory required to start a new job or keep youngest job alive
 
 #### logging parameters ####
diff --git a/fim_run.sh b/fim_run.sh
index fb22bc8c2..a8763da60 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -115,7 +115,10 @@ export input_NWM_Lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
 export input_NWM_Catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
 export input_NWM_Flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
 export input_NWM_Headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
-export input_NHD_Flowlines=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_wVAA.gpkg
+export input_nhd_flowlines_fr=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_fr_adjusted.gpkg
+export input_nhd_flowlines_ms=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_ms_adjusted.gpkg
+export input_nhd_headwaters_fr=$inputDataDir/nhdplus_vectors_aggregate/nhd_headwaters_adjusted_fr.gpkg
+export input_nhd_headwaters_ms=$inputDataDir/nhdplus_vectors_aggregate/nhd_headwaters_adjusted_ms.gpkg
 
 ## Input handling ##
 $libDir/check_huc_inputs.py -u "$hucList"
diff --git a/lib/acquire_and_preprocess_inputs.py b/lib/acquire_and_preprocess_inputs.py
index 446e6955e..01270f235 100755
--- a/lib/acquire_and_preprocess_inputs.py
+++ b/lib/acquire_and_preprocess_inputs.py
@@ -249,7 +249,7 @@ def build_huc_list_files(path_to_saved_data_parent_dir, wbd_directory):
     wbd = gp.read_file(full_huc_gpkg, layer=huc_gpkg)
 
     # Loop through entries and compare against the huc4_list to get available HUCs within the geopackage domain.
-    for index, row in tqdm(wbd.iterrows()):
+    for index, row in tqdm(wbd.iterrows(),total=len(wbd)):
         huc = row["HUC" + huc_gpkg[-1]]
         huc_mask = wbd.loc[wbd[str("HUC" + huc_gpkg[-1])]==huc].geometry
         burnline = os.path.join(nhd_plus_vector_dir, huc[0:4], 'NHDPlusBurnLineEvent' + huc[0:4] + '.gpkg')
diff --git a/lib/adjust_headwater_streams.py b/lib/adjust_headwater_streams.py
new file mode 100644
index 000000000..e08bf3352
--- /dev/null
+++ b/lib/adjust_headwater_streams.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+from os.path import splitext
+from tqdm import tqdm
+import argparse
+import pygeos
+from shapely.geometry import Point,LineString
+from shapely.ops import split
+from shapely.wkb import dumps, loads
+from utils.shared_variables import PREP_PROJECTION
+from utils.shared_functions import getDriver
+
+def adjust_headwaters(huc,nhd_streams,headwaters,headwater_id):
+
+    # identify true headwater segments
+    if nhd_streams['headwaters_id'].dtype=='int':
+        nhd_streams_adj = nhd_streams.loc[(nhd_streams.headwaters_id > 0) & (nhd_streams.downstream_of_headwater == False),:].copy()
+        if headwaters[headwater_id].dtype != 'int': headwaters[headwater_id] = headwaters[headwater_id].astype(int)
+    else:
+        nhd_streams_adj = nhd_streams.loc[(nhd_streams.headwaters_id.notna()) & (nhd_streams.downstream_of_headwater == False),:].copy()
+
+    nhd_streams_adj = nhd_streams_adj.explode()
+    nhd_streams_adj = nhd_streams_adj.reset_index(drop=True)
+
+    headwater_limited = headwaters.merge(nhd_streams_adj["headwaters_id"],left_on=headwater_id, right_on="headwaters_id",how='right')
+
+    headwaterstreams = []
+    referencedpoints = []
+
+    for index, point in headwater_limited.iterrows():
+
+        # convert headwaterpoint geometries to WKB representation
+        wkb_points = dumps(point.geometry)
+
+        # create pygeos headwaterpoint geometries from WKB representation
+        pointbin_geom = pygeos.io.from_wkb(wkb_points)
+
+        # Closest segment to headwater
+        closest_stream = nhd_streams_adj.loc[nhd_streams_adj["headwaters_id"]==point[headwater_id]]
+
+        try: # seeing inconsistent geometry objects even after exploding nhd_streams_adj; not sure why this is
+            closest_stream =closest_stream.explode()
+        except:
+            pass
+        try:
+            wkb_closest_stream = dumps(closest_stream.geometry[0])
+        except:
+            wkb_closest_stream = dumps(closest_stream.geometry[0][0])
+
+        streambin_geom = pygeos.io.from_wkb(wkb_closest_stream)
+
+        # Linear reference headwater to closest stream segment
+        pointdistancetoline = pygeos.linear.line_locate_point(streambin_geom, pointbin_geom)
+        referencedpoint = pygeos.linear.line_interpolate_point(streambin_geom, pointdistancetoline)
+
+        # convert geometries to wkb representation
+        bin_referencedpoint = pygeos.io.to_wkb(referencedpoint)
+
+        # convert to shapely geometries
+        shply_referencedpoint = loads(bin_referencedpoint)
+        shply_linestring = loads(wkb_closest_stream)
+        headpoint = Point(shply_referencedpoint.coords)
+        cumulative_line = []
+        relativedistlst = []
+
+        # collect all nhd stream segment linestring verticies
+        for point in zip(*shply_linestring.coords.xy):
+            cumulative_line = cumulative_line + [point]
+            relativedist = shply_linestring.project(Point(point))
+            relativedistlst = relativedistlst + [relativedist]
+
+        # add linear referenced headwater point to closest nhd stream segment
+        if not headpoint in cumulative_line:
+            cumulative_line = cumulative_line + [headpoint]
+            relativedist = shply_linestring.project(headpoint)
+            relativedistlst = relativedistlst + [relativedist]
+
+        # sort by relative line distance to place headwater point in linestring
+        sortline = pd.DataFrame({'geom' : cumulative_line, 'dist' : relativedistlst}).sort_values('dist')
+        shply_linestring = LineString(sortline.geom.tolist())
+        referencedpoints = referencedpoints + [headpoint]
+
+        # split the new linestring at the new headwater point
+        try:
+            line1,line2 = split(shply_linestring, headpoint)
+            headwaterstreams = headwaterstreams + [LineString(line1)]
+            nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1)
+        except:
+            line1 = split(shply_linestring, headpoint)
+            headwaterstreams = headwaterstreams + [LineString(line1[0])]
+            nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1[0])
+
+    nhd_streams = nhd_streams.drop(columns=['is_relevant_stream', 'headwaters_id', 'downstream_of_headwater'])
+
+    try:
+        del nhd_streams_adj, headwaters, headwater_limited, headwaterstreams, referencedpoints, cumulative_line, relativedistlst
+    except:
+        print ('issue deleting adjusted stream variables for huc ' + str(huc))
+
+    ## identify ajusted nhd headwaters
+    # print('Identify NHD headwater points',flush=True)
+    nhd_headwater_streams_adj = nhd_streams.loc[nhd_streams['is_headwater'],:]
+    nhd_headwater_streams_adj = nhd_headwater_streams_adj.explode()
+
+    hw_points = np.zeros(len(nhd_headwater_streams_adj),dtype=object)
+    for index,lineString in enumerate(nhd_headwater_streams_adj.geometry):
+        hw_point = [point for point in zip(*lineString.coords.xy)][-1]
+        hw_points[index] = Point(*hw_point)
+
+    nhd_headwater_points_adj = gpd.GeoDataFrame({'NHDPlusID' : nhd_headwater_streams_adj['NHDPlusID'],
+                                            'geometry' : hw_points},geometry='geometry',crs=PREP_PROJECTION)
+
+    del nhd_headwater_streams_adj
+
+    return(nhd_streams, nhd_headwater_points_adj)
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='adjust headwater stream geometery based on headwater start points')
+    parser.add_argument('-f','--huc',help='huc number',required=True)
+    parser.add_argument('-l','--nhd-streams',help='NHDPlus HR geodataframe',required=True)
+    parser.add_argument('-p','--headwaters',help='Headwater points layer',required=True,type=str)
+    parser.add_argument('-s','--subset-nhd-streams-fileName',help='Output streams layer name',required=False,type=str,default=None)
+    parser.add_argument('-s','--adj-headwater-points-fileName',help='Output adj headwater points layer name',required=False,type=str,default=None)
+    parser.add_argument('-g','--headwater-points-fileName',help='Output headwater points layer name',required=False,type=str,default=None)
+    parser.add_argument('-i','--headwater-id',help='Output headwaters points',required=True)
+
+    args = vars(parser.parse_args())
+
+    adj_streams_gdf,adj_headwaters_gdf = adjust_headwaters(huc,nhd_streams,headwaters,headwater_id)
+
+    if subset_nhd_streams_fileName is not None:
+        adj_streams_gdf.to_file(args['subset_nhd_streams_fileName'],driver=getDriver(args['subset_nhd_streams_fileName']),index=False)
+
+    if headwater_points_fileName is not None:
+        headwater_points_fileName.to_file(args['headwater_points_fileName'],driver=getDriver(args['headwater_points_fileName']),index=False)
+
+    if adj_headwater_points_fileName is not None:
+        adj_headwaters_gdf.to_file(args['adj_headwater_points_fileName'],driver=getDriver(args['adj_headwater_points_fileName']),index=False)
diff --git a/lib/aggregate_vector_inputs.py b/lib/aggregate_vector_inputs.py
index 48dadecbf..d60cdc2f0 100755
--- a/lib/aggregate_vector_inputs.py
+++ b/lib/aggregate_vector_inputs.py
@@ -2,51 +2,410 @@
 
 import os
 import geopandas as gpd
-import pandas as pd
-from os.path import splitext
 from utils.shared_variables import PREP_PROJECTION
+from utils.shared_functions import getDriver
 from derive_headwaters import findHeadWaterPoints
+from reduce_nhd_stream_density import subset_nhd_network
+from adjust_headwater_streams import adjust_headwaters
 from tqdm import tqdm
+from os.path import splitext
+from shapely.geometry import Point
+from concurrent.futures import ProcessPoolExecutor,as_completed
+from collections import deque
+import numpy as np
+from shapely.wkb import dumps, loads
+import pygeos
+
+in_dir ='data/inputs/nhdplus_vectors'
+nwm_dir = 'data/inputs/nwm_hydrofabric'
+wbd_dir = 'data/inputs/wbd'
+ahps_dir = 'data/inputs/ahp_sites'
+agg_dir = 'data/inputs/nhdplus_vectors_aggregate'
+
+wbd_filename = os.path.join(wbd_dir, 'WBD_National.gpkg')
+nwm_streams_fr_filename = os.path.join(nwm_dir,'nwm_flows.gpkg')
+nwm_streams_ms_filename = os.path.join(nwm_dir,'nwm_flows_ms.gpkg')
+nwm_headwaters_filename = os.path.join(nwm_dir,'nwm_headwaters.gpkg')
+nwm_huc4_intersections_ms_filename = os.path.join(nwm_dir,'nwm_ms_huc4_intersections.gpkg')
+nwm_huc4_intersections_fr_filename = os.path.join(nwm_dir,'nwm_fr_huc4_intersections.gpkg')
+
+def subset_nwm_ms_streams(args):
+    nwm_streams_filename    = args[0]
+    in_dir                  = args[1]
+    ahps_dir                = args[2]
+    output_filename         = args[3]
+
+    # subset nwm network to ms
+    ahps_headwaters_filename = os.path.join(ahps_dir,'bed_lids.gpkg')
+    ahps_headwaters = gpd.read_file(ahps_headwaters_filename)
+
+    nwm_streams = gpd.read_file(nwm_streams_filename)
+
+    nwm_streams['is_headwater'] = False
+    nwm_streams['downstream_of_headwater'] = False
+
+    nwm_streams.loc[nwm_streams.ID.isin(list(ahps_headwaters.nwm_featur)),'is_headwater'] = True
+
+    ## subset NHDPlus HR
+    nwm_streams['is_relevant_stream'] = nwm_streams['is_headwater'].copy()
+
+    nwm_streams = nwm_streams.explode()
+
+    # trace down from headwaters
+    nwm_streams.set_index('ID',inplace=True,drop=False)
+
+    Q = deque(nwm_streams.loc[nwm_streams['is_headwater'],'ID'].tolist())
+    visited = set()
+
+    while Q:
+        q = Q.popleft()
+        if q in visited:
+            continue
+        #
+        visited.add(q)
+        toNode = nwm_streams.loc[q,'to']
+        #
+        if not toNode == 0:
+        #
+            nwm_streams.loc[nwm_streams.ID==toNode,'is_relevant_stream'] = True
+            #
+            if toNode not in visited:
+                Q.append(toNode)
+
+    nwm_streams = nwm_streams.loc[nwm_streams['is_relevant_stream'],:]
+
+    nwm_streams.reset_index(drop=True,inplace=True)
+
+    nwm_streams.to_file(output_filename,getDriver(output_filename),index=False)
+
+def find_nwm_incoming_streams(args):
+
+    nwm_streams_filename    = args[0]
+    wbd_filename            = args[1]
+    in_dir                  = args[2]
+    output_filename         = args[3]
+
+    wbd = gpd.read_file(wbd_filename, layer='WBDHU4')
+
+    intersecting_points = []
+    for index, row in tqdm(wbd.iterrows(),total=len(wbd)):
+        col_name = 'HUC4'
+        huc = row[col_name]
+
+        huc_mask = wbd.loc[wbd[col_name]==str(huc)]
+        huc_mask = huc_mask.explode()
+        huc_mask = huc_mask.reset_index(drop=True)
+
+        nwm_streams = gpd.read_file(nwm_streams_filename, mask=huc_mask)
+        nwm_streams = nwm_streams.explode()
+        nwm_streams = nwm_streams.reset_index(drop=True)
+
+        for index, polygon in enumerate(huc_mask.geometry):
+            crosses=nwm_streams.crosses(polygon.exterior)
+            nwm_streams_subset =nwm_streams[crosses]
+            nwm_streams_subset = nwm_streams_subset.reset_index(drop=True)
+
+            for index, linestring in enumerate(nwm_streams_subset.geometry):
+                distances = []
+                # distance to each stream segment
+                for point in zip(*linestring.coords.xy):
+                    distance = Point(point).distance(polygon.exterior)
+                    distances = distances + [distance]
+
+                # find minimum distance
+                min_index = np.argmin(distances)
+
+                # Closest segment to headwater
+                closest_point = list(linestring.coords)[min_index]
+                last_node = Point(closest_point)
+
+                # convert geometries to WKB representation
+                wkb_point = dumps(last_node)
+                wkb_poly = dumps(polygon.exterior)
+
+                # create pygeos geometries from WKB representation
+                stream_point_geom = pygeos.io.from_wkb(wkb_point)
+                polybin_geom = pygeos.io.from_wkb(wkb_poly)
+
+                # Linear reference end node to huc boundary
+                pointdistancetoline = pygeos.linear.line_locate_point(polybin_geom,stream_point_geom)
+                referencedpoint = pygeos.linear.line_interpolate_point(polybin_geom, pointdistancetoline)
+
+                # convert geometries to wkb representation
+                bin_referencedpoint = pygeos.io.to_wkb(referencedpoint)
+
+                # convert to shapely geometries
+                shply_referencedpoint = loads(bin_referencedpoint)
+
+                # collect all nhd stream segment linestring verticies
+                intersecting_points = intersecting_points + [shply_referencedpoint]
+
+    huc_intersection = gpd.GeoDataFrame({'geometry' : intersecting_points},crs=nwm_streams.crs,geometry='geometry')
+    huc_intersection = huc_intersection.drop_duplicates()
+    huc_intersection.to_file(output_filename,getDriver(output_filename))
+
+def collect_stream_attributes(args, huc):
+    print ('Starting huc: ' + str(huc))
+    in_dir = args[0]
+    nwm_dir = args[1]
+    ahps_dir = args[2]
+
+    print ('Collecting NHDPlus HR attributes')
+    burnline_filename = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')
+    vaa_filename = os.path.join(in_dir,huc,'NHDPlusFlowLineVAA' + str(huc) + '.gpkg')
+    flowline_filename = os.path.join(in_dir,huc,'NHDFlowline' + str(huc) + '.gpkg')
+
+    if os.path.exists(os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')):
+
+        burnline = gpd.read_file(burnline_filename)
+        burnline = burnline[['NHDPlusID','ReachCode','geometry']]
+
+        flowline = gpd.read_file(flowline_filename)
+        flowline = flowline[['NHDPlusID','FType','FCode']]
+        # flowline = flowline.loc[flowline["FType"].isin([334,420,428,460,558])]
+        flowline = flowline.loc[~flowline["FType"].isin([566,420])]
+
+        nhd_streams_vaa = gpd.read_file(vaa_filename)
+        nhd_streams_vaa = nhd_streams_vaa[['FromNode','ToNode','NHDPlusID','StreamOrde','DnLevelPat','LevelPathI']]
+        nhd_streams = burnline.merge(nhd_streams_vaa,on='NHDPlusID',how='inner')
+        nhd_streams = nhd_streams.merge(flowline,on='NHDPlusID',how='inner')
+
+        del burnline, flowline, nhd_streams_vaa
+
+        nhd_streams = nhd_streams.to_crs(PREP_PROJECTION)
+        nhd_streams = nhd_streams.loc[nhd_streams.geometry!=None,:] # special case: remove segments without geometries
+        nhd_streams['HUC4'] = str(huc)
+
+        # write out NHDPlus HR aggregated
+        nhd_streams_agg_fileName = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
+        nhd_streams.to_file(nhd_streams_agg_fileName,driver=getDriver(nhd_streams_agg_fileName),index=False)
+        del nhd_streams
+
+        print ('finished huc: ' + str(huc))
 
-in_dir ='/data/inputs/nhdplus_vectors'
-nhd_dir ='/data/inputs/nhdplus_vectors_aggregate'
-nwm_dir = '/data/inputs/nwm_hydrofabric'
-
-## NWM Headwaters
-print ('deriving NWM headwater points')
-nwm_streams = gpd.read_file(os.path.join(nwm_dir,'nwm_flows.gpkg'))
-nwm_headwaters = findHeadWaterPoints(nwm_streams)
-nwm_headwaters.to_file(os.path.join(nwm_dir,'nwm_headwaters.gpkg'),driver='GPKG',index=False)
-
-## NHDPlus HR
-print ('aggregating NHDPlus HR burnline layers')
-nhd_streams_wVAA_fileName_pre=os.path.join(nhd_dir,'NHDPlusBurnLineEvent_wVAA.gpkg')
-
-schema = {'geometry': 'MultiLineString','properties': {'NHDPlusID': 'str','ReachCode': 'str',
-                                                  'FromNode': 'str','ToNode': 'str',
-                                                  'StreamOrde': 'str','DnLevelPat': 'str',
-                                                  'LevelPathI': 'str'}}
-
-for huc in tqdm(os.listdir(in_dir)):
-    if not huc[0]=='#':
-        burnline_filename = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')
-        vaa_filename = os.path.join(in_dir,huc,'NHDPlusFlowLineVAA' + str(huc) + '.gpkg')
-        flowline_filename = os.path.join(in_dir,huc,'NHDFlowline' + str(huc) + '.gpkg')
-        if os.path.exists(os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')):
-            burnline = gpd.read_file(burnline_filename)
-            nhd_streams_vaa = gpd.read_file(vaa_filename)
-            flowline = gpd.read_file(flowline_filename)
-            burnline = burnline[['NHDPlusID','ReachCode','geometry']]
-            flowline = flowline[['NHDPlusID','FCode']]
-            nhd_streams_vaa = nhd_streams_vaa[['FromNode','ToNode','NHDPlusID','StreamOrde','DnLevelPat','LevelPathI']]
-            nhd_streams_withVAA = burnline.merge(nhd_streams_vaa,on='NHDPlusID',how='inner')
-            nhd_streams_fcode = nhd_streams_withVAA.merge(flowline,on='NHDPlusID',how='inner')
-            nhd_streams = nhd_streams_fcode.to_crs(PREP_PROJECTION)
-            if os.path.isfile(nhd_streams_wVAA_fileName_pre):
-                nhd_streams.to_file(nhd_streams_wVAA_fileName_pre,driver='GPKG',index=False, mode='a')
-            else:
-                nhd_streams.to_file(nhd_streams_wVAA_fileName_pre,driver='GPKG',index=False)
-        else:
-            print ('missing data for huc ' + str(huc))
     else:
-        print ('skipping huc ' + str(huc))
+        print ('missing data for huc ' + str(huc))
+
+def subset_stream_networks(args, huc):
+
+    nwm_dir                            = args[0]
+    ahps_dir                           = args[1]
+    wbd4                               = args[2]
+    wbd8                               = args[3]
+    in_dir                             = args[4]
+    nwm_huc4_intersect_fr_filename     = args[5]
+    nwm_huc4_intersect_ms_filename     = args[6]
+
+    print("starting HUC " + str(huc),flush=True)
+    nwm_headwater_id = 'ID'
+    nwm_headwaters_filename = os.path.join(nwm_dir,'nwm_headwaters.gpkg')
+    ahps_headwater_id = 'nws_lid'
+    ahps_headwaters_filename = os.path.join(ahps_dir,'nws_lid.gpkg')
+    nhd_streams_filename = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
+
+    # subset to reduce footprint
+    selected_wbd4 = wbd4.loc[wbd4.HUC4.str.startswith(str(huc))]
+    del wbd4
+    selected_wbd8 = wbd8.loc[wbd8.HUC8.str.startswith(huc)]
+    del wbd8
+
+    huc_mask = selected_wbd4.loc[selected_wbd4.HUC4.str.startswith(str(huc))]
+    huc_mask = huc_mask.explode()
+    huc_mask = huc_mask.reset_index(drop=True)
+
+    if len(selected_wbd8.HUC8) > 0:
+        selected_wbd8 = selected_wbd8.reset_index(drop=True)
+
+        # identify FR/NWM headwaters
+        nhd_streams_fr = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,nwm_headwaters_filename,nwm_headwater_id,nwm_huc4_intersect_fr_filename)
+
+        ## adjust FR/NWM headwater segments
+        nwm_headwaters = gpd.read_file(nwm_headwaters_filename, mask=huc_mask)
+
+        if len(nwm_headwaters) > 0:
+
+            adj_nhd_streams_fr, adj_nhd_headwater_points_fr = adjust_headwaters(str(huc),nhd_streams_fr,nwm_headwaters,nwm_headwater_id)
+
+            nhd_streams_fr_adjusted_fileName=os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr_adjusted.gpkg')
+            adj_nhd_headwaters_fr_fileName=os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_fr.gpkg')
+
+            # write out FR adjusted
+            adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,getDriver(nhd_streams_fr_adjusted_fileName),index=False)
+            adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,getDriver(adj_nhd_headwaters_fr_fileName),index=False)
+
+            del adj_nhd_streams_fr, adj_nhd_headwater_points_fr
+        else:
+            print ('skipping FR headwater adjustments for HUC: ' + str(huc))
+
+        del nhd_streams_fr
+
+        ## identify MS/AHPs headwaters
+        nhd_streams_ms = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,ahps_headwaters_filename,ahps_headwater_id,nwm_huc4_intersect_ms_filename)
+
+        ## adjust MS/AHPs headwater segments
+        ahps_headwaters = gpd.read_file(ahps_headwaters_filename, mask=huc_mask)
+
+        if len(ahps_headwaters) > 0:
+
+            adj_nhd_streams_ms, adj_nhd_headwater_points_ms = adjust_headwaters(str(huc),nhd_streams_ms,ahps_headwaters,ahps_headwater_id)
+
+            nhd_streams_ms_adjusted_fileName=os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms_adjusted.gpkg')
+            adj_nhd_headwaters_ms_fileName=os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_ms.gpkg')
+
+            # write out MS adjusted
+            adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,getDriver(nhd_streams_ms_adjusted_fileName),index=False)
+            adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,getDriver(adj_nhd_headwaters_ms_fileName),index=False)
+
+            del adj_nhd_streams_ms, adj_nhd_headwater_points_ms
+
+        else:
+            print ('skipping MS headwater adjustments for HUC: ' + str(huc))
+            del nhd_streams_ms
+
+def aggregate_stream_networks(in_dir,agg_dir, huc_list):
+
+    for huc in huc_list:
+
+        ## FR adjusted
+        adj_nhd_headwaters_fr_fileName=os.path.join(agg_dir,'nhd_headwaters_adjusted_fr.gpkg')
+        nhd_fr_adj_huc_subset = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr_adjusted.gpkg')
+        nhd_streams_fr_adjusted_fileName=os.path.join(agg_dir,'NHDPlusBurnLineEvent_fr_adjusted.gpkg')
+        nhd_fr_adj_headwaters_subset = os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_fr.gpkg')
+
+        if os.path.isfile(nhd_fr_adj_huc_subset):
+            adj_nhd_streams_fr = gpd.read_file(nhd_fr_adj_huc_subset)
+
+            # write out FR adjusted
+            if os.path.isfile(nhd_streams_fr_adjusted_fileName):
+                adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,driver=getDriver(nhd_streams_fr_adjusted_fileName),index=False, mode='a')
+            else:
+                adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,driver=getDriver(nhd_streams_fr_adjusted_fileName),index=False)
+
+            del adj_nhd_streams_fr
+
+        if os.path.isfile(nhd_fr_adj_headwaters_subset):
+            adj_nhd_headwater_points_fr = gpd.read_file(nhd_fr_adj_headwaters_subset)
+
+            # write out FR adjusted
+            if os.path.isfile(adj_nhd_headwaters_fr_fileName):
+                adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,driver=getDriver(adj_nhd_headwaters_fr_fileName),index=False, mode='a')
+            else:
+                adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,driver=getDriver(adj_nhd_headwaters_fr_fileName),index=False)
+
+            del adj_nhd_headwater_points_fr
+
+        ## MS adjusted
+        adj_nhd_headwaters_ms_fileName=os.path.join(agg_dir,'nhd_headwaters_adjusted_ms.gpkg')
+        nhd_ms_adj_huc_subset = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms_adjusted.gpkg')
+        nhd_streams_ms_adjusted_fileName=os.path.join(agg_dir,'NHDPlusBurnLineEvent_ms_adjusted.gpkg')
+        nhd_ms_adj_headwater_subset = os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_ms.gpkg')
+
+        if os.path.isfile(nhd_ms_adj_huc_subset):
+            adj_nhd_streams_ms = gpd.read_file(nhd_ms_adj_huc_subset)
+
+            # write out ms adjusted
+            if os.path.isfile(nhd_streams_ms_adjusted_fileName):
+                adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,driver=getDriver(nhd_streams_ms_adjusted_fileName),index=False, mode='a')
+            else:
+                adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,driver=getDriver(nhd_streams_ms_adjusted_fileName),index=False)
+
+            del adj_nhd_streams_ms
+
+        if os.path.isfile(nhd_ms_adj_headwater_subset):
+            adj_nhd_headwater_points_ms = gpd.read_file(nhd_ms_adj_headwater_subset)
+
+            # write out ms adjusted
+            if os.path.isfile(adj_nhd_headwaters_ms_fileName):
+                adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,driver=getDriver(adj_nhd_headwaters_ms_fileName),index=False, mode='a')
+            else:
+                adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,driver=getDriver(adj_nhd_headwaters_ms_fileName),index=False)
+
+            del adj_nhd_headwater_points_ms
+
+
+def clean_up_intermediate_files(in_dir):
+
+    for huc in os.listdir(in_dir):
+        agg_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
+        fr_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr.gpkg')
+        fr_adj_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr_adjusted.gpkg')
+        ms_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms.gpkg')
+        ms_adj_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms_adjusted.gpkg')
+        ms_headwater_adj_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_ms.gpkg')
+        fr_headwater_adj_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_fr.gpkg')
+        ms_headwater_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_ms.gpkg')
+        fr_headwater_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_fr.gpkg')
+
+        if os.path.exists(agg_path):
+            os.remove(agg_path)
+
+        if os.path.exists(fr_path):
+            os.remove(fr_path)
+
+        if os.path.exists(fr_adj_path):
+            os.remove(fr_adj_path)
+
+        if os.path.exists(ms_path):
+            os.remove(ms_path)
+
+        if os.path.exists(ms_adj_path):
+            os.remove(ms_adj_path)
+
+        if os.path.exists(ms_headwater_adj_path):
+            os.remove(ms_headwater_adj_path)
+
+        if os.path.exists(fr_headwater_adj_path):
+            os.remove(fr_headwater_adj_path)
+
+        if os.path.exists(ms_headwater_path):
+            os.remove(ms_headwater_path)
+
+        if os.path.exists(fr_headwater_path):
+            os.remove(fr_headwater_path)
+
+
+
+if(__name__=='__main__'):
+
+    ## generate NWM Headwaters
+    # print ('deriving nwm headwater points')
+    # nwm_headwaters = findHeadWaterPoints(nwm_streams_fr_filename)
+    # nwm_headwaters['ID'] = nwm_headwaters.index + 1
+    # nwm_headwaters.to_file(nwm_headwaters_filename,getDriver(nwm_headwaters_filename),index=False)
+    # del nwm_headwaters, nwm_streams
+
+    ## subset NWM MS Streams
+    # nwm_subset_ms_args = (nwm_streams_fr_filename,in_dir,ahps_dir,nwm_streams_ms_filename)
+    # print ('deriving nwm ms streams')
+    # subset_nwm_ms_streams(nwm_subset_ms_args)
+
+    ## generate NWM intersection points with WBD4 boundaries
+    # ms_nwm_intersect_args = (nwm_streams_ms_filename,wbd_filename,in_dir,nwm_huc4_intersections_ms_filename)
+    # fr_nwm_intersect_args = (nwm_streams_fr_filename,wbd_filename,in_dir,nwm_huc4_intersections_fr_filename)
+    # print ('deriving nwm ms intersection points')
+    # find_nwm_incoming_streams(ms_nwm_intersect_args)
+    # print ('deriving nwm fr intersection points')
+    # find_nwm_incoming_streams(fr_nwm_intersect_args)
+
+    print ('loading wb4')
+    wbd4 = gpd.read_file(wbd_filename, layer='WBDHU4')
+    print ('loading wb8')
+    wbd8 = gpd.read_file(wbd_filename, layer='WBDHU8')
+
+    subset_arg_list = (nwm_dir,ahps_dir,wbd4,wbd8,in_dir,nwm_huc4_intersections_fr_filename,nwm_huc4_intersections_ms_filename)
+    collect_arg_list = (in_dir,nwm_dir,ahps_dir)
+
+    num_workers=9
+
+    with ProcessPoolExecutor(max_workers=num_workers) as executor:
+        ## preprocess nhd hr and add attributes
+        collect_attributes = [executor.submit(collect_stream_attributes, collect_arg_list, str(huc)) for huc in os.listdir(in_dir)]
+        ## subset nhd hr network
+        subset_results = [executor.submit(subset_stream_networks, subset_arg_list, str(huc)) for huc in os.listdir(in_dir)]
+
+
+    ## aggregate fr and ms nhd netowrks for entire nwm domain
+    aggregate_stream_networks(in_dir,agg_dir, os.listdir(in_dir))
+
+    ## remove intermediate files
+    # clean_up_intermediate_files(in_dir)
diff --git a/lib/buildstreamtraversal.py b/lib/buildstreamtraversal.py
index 1fcb00d65..f7b2bcbe4 100644
--- a/lib/buildstreamtraversal.py
+++ b/lib/buildstreamtraversal.py
@@ -2,9 +2,9 @@
 Description:
         This tool creates unique IDs for each segment and builds the To_Node, From_Node, and NextDownID columns to traverse the network
 Required Arguments:
-        modelstream   = stream network
-        WBD8          = HUC8 boundary dataset
-        HYDROID       = name of ID column (string)
+        streams   = stream network
+        wbd8          = HUC8 boundary dataset
+        hydro_id       = name of ID column (string)
 '''
 import sys
 import datetime
@@ -22,9 +22,9 @@ def trace():
     synerror = traceback.format_exc().splitlines()[-1]
     return line, filename, synerror
 
-FN_FROMNODE = "From_Node"
-FN_TONODE = "To_Node"
-FN_NEXTDOWNID = "NextDownID"
+from_node = "From_Node"
+to_node = "To_Node"
+next_down_id = "NextDownID"
 
 class BuildStreamTraversalColumns(object):
     '''Tool class for updating the next down IDs of stream features.'''
@@ -33,60 +33,60 @@ def __init__(self):
         self.label = 'Find Next Downstream Line'
         self.description = '''Finds next downstream line, retrieves its HydroID and stores it in the NextDownID field.'''
 
-    def execute(self, modelstream, WBD8, HYDROID):
+    def execute(self, streams, wbd8, hydro_id):
         try:
             split_code = 1
             sOK = 'OK'
 
             # check for HydroID; Assign if it doesn't exist
-            if not HYDROID in modelstream.columns:
-                print ("Required field " + HYDROID + " does not exist in input. Generating..")
+            if not hydro_id in streams.columns:
+                print ("Required field " + hydro_id + " does not exist in input. Generating..")
 
-                stream_centroid = gpd.GeoDataFrame({'geometry':modelstream.geometry.centroid}, crs=modelstream.crs, geometry='geometry')
-                stream_wbdjoin = gpd.sjoin(stream_centroid, WBD8, how='left', op='within')
+                stream_centroid = gpd.GeoDataFrame({'geometry':streams.geometry.centroid}, crs=streams.crs, geometry='geometry')
+                stream_wbdjoin = gpd.sjoin(stream_centroid, wbd8, how='left', op='within')
                 stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "centroid", "index_right": "HUC8id"})
-                modelstream = modelstream.join(stream_wbdjoin).drop(columns=['centroid'])
+                streams = streams.join(stream_wbdjoin).drop(columns=['centroid'])
 
-                modelstream['seqID'] = (modelstream.groupby('HUC8id').cumcount(ascending=True)+1).astype('str').str.zfill(4)
-                modelstream = modelstream.loc[modelstream['HUC8id'].notna(),:]
-                modelstream = modelstream.assign(HYDROID= lambda x: x.HUC8id + x.seqID)
-                modelstream = modelstream.rename(columns={"HYDROID": HYDROID}).sort_values(HYDROID)
-                modelstream = modelstream.drop(columns=['HUC8id', 'seqID'])
-                modelstream[HYDROID] = modelstream[HYDROID].astype(int)
-                print ('Generated ' + HYDROID)
+                streams['seqID'] = (streams.groupby('HUC8id').cumcount(ascending=True)+1).astype('str').str.zfill(4)
+                streams = streams.loc[streams['HUC8id'].notna(),:]
+                streams = streams.assign(hydro_id= lambda x: x.HUC8id + x.seqID)
+                streams = streams.rename(columns={"hydro_id": hydro_id}).sort_values(hydro_id)
+                streams = streams.drop(columns=['HUC8id', 'seqID'])
+                streams[hydro_id] = streams[hydro_id].astype(int)
+                print ('Generated ' + hydro_id)
 
             # Check for TO/From Nodes; Assign if doesnt exist
             bOK = True
-            if not FN_FROMNODE in modelstream.columns:
-                print ("Field " + FN_FROMNODE + " does not exist in input ")
+            if not from_node in streams.columns:
+                print ("Field " + from_node + " does not exist in input ")
                 bOK = False
-            if not FN_TONODE in modelstream.columns:
-                print ("Field " + FN_TONODE + " does not exist in input. Generating..")
+            if not to_node in streams.columns:
+                print ("Field " + to_node + " does not exist in input. Generating..")
                 bOK = False
 
             if(bOK==False):
                 # Add fields if not they do not exist.
-                if not FN_FROMNODE in modelstream.columns:
-                    modelstream[FN_FROMNODE] = ''
+                if not from_node in streams.columns:
+                    streams[from_node] = ''
 
-                if not FN_TONODE in modelstream.columns:
-                    modelstream[FN_TONODE] = ''
+                if not to_node in streams.columns:
+                    streams[to_node] = ''
 
-                modelstream = modelstream.sort_values(by=[HYDROID], ascending=True).copy()
+                streams = streams.sort_values(by=[hydro_id], ascending=True).copy()
 
                 xy_dict = {}
                 bhasnullshape=False
-                for rows in modelstream[['geometry', FN_FROMNODE, FN_TONODE]].iterrows():
+                for rows in streams[['geometry', from_node, to_node]].iterrows():
                     if rows[1][0]:
                         # From Node
                         firstx = round(rows[1][0].coords.xy[0][0], 7)
                         firsty = round(rows[1][0].coords.xy[1][0], 7)
                         from_key = '{},{}'.format(firstx, firsty)
                         if from_key in xy_dict:
-                            modelstream.at[rows[0], FN_FROMNODE,] = xy_dict[from_key]
+                            streams.at[rows[0], from_node,] = xy_dict[from_key]
                         else:
                             xy_dict[from_key] = len(xy_dict) + 1
-                            modelstream.at[rows[0], FN_FROMNODE,] = xy_dict[from_key]
+                            streams.at[rows[0], from_node,] = xy_dict[from_key]
 
                         # To Node
                         lastx = round(rows[1][0].coords.xy[0][-1], 7)
@@ -94,27 +94,27 @@ def execute(self, modelstream, WBD8, HYDROID):
                         to_key = '{},{}'.format(lastx, lasty)
                         #if xy_dict.has_key(to_key):
                         if to_key in xy_dict:
-                            modelstream.at[rows[0], FN_TONODE] = xy_dict[to_key]
+                            streams.at[rows[0], to_node] = xy_dict[to_key]
                         else:
                             xy_dict[to_key] = len(xy_dict) + 1
-                            modelstream.at[rows[0], FN_TONODE] = xy_dict[to_key]
+                            streams.at[rows[0], to_node] = xy_dict[to_key]
                     else:
                          bhasnullshape=True
 
                 if bhasnullshape==True:
                     print ("Some of the input features have a null shape.")
-                    print (FN_FROMNODE + " and " + FN_TONODE + " fields cannot be populated for those features.")
+                    print (from_node + " and " + to_node + " fields cannot be populated for those features.")
                 else:
                     print ('Generated To/From Nodes')
 
             # Create NextDownID field
-            if not FN_NEXTDOWNID in modelstream.columns:
-                modelstream[FN_NEXTDOWNID] = ''
+            if not next_down_id in streams.columns:
+                streams[next_down_id] = ''
 
-            # Create dict to store FN_FROMNODE values for each HydroID
+            # Create dict to store from_node values for each HydroID
             dnodes=dict()
             lstHydroIDs=[]
-            for row in modelstream[[FN_FROMNODE,HYDROID]].iterrows():
+            for row in streams[[from_node,hydro_id]].iterrows():
 
                 if (row[1][0] in dnodes)==False:
                     lstHydroIDs=[row[1][1]]
@@ -124,7 +124,7 @@ def execute(self, modelstream, WBD8, HYDROID):
                     lstHydroIDs.append(row[1][1])
 
             # for each stream segment, search dict for HydroID downstream and
-            for urow in modelstream[[FN_NEXTDOWNID, FN_TONODE, FN_FROMNODE, HYDROID]].iterrows():
+            for urow in streams[[next_down_id, to_node, from_node, hydro_id]].iterrows():
                 tonodecol = urow[1][1]
                 nextdownIDcol = urow[1][0]
                 hydroIDcol = urow[1][3]
@@ -152,9 +152,9 @@ def execute(self, modelstream, WBD8, HYDROID):
                     if next_down_ids:del next_down_ids
                 except:
                     pass
-                modelstream.loc[modelstream[HYDROID]== hydroIDcol,[FN_NEXTDOWNID]] = nextdownIDcol
+                streams.loc[streams[hydro_id]== hydroIDcol,[next_down_id]] = nextdownIDcol
 
-            tReturns = (sOK, modelstream)
+            tReturns = (sOK, streams)
         except Exception:
             sOK = "{}".format(trace())
             tReturns = (sOK, )
@@ -166,18 +166,15 @@ def execute(self, modelstream, WBD8, HYDROID):
         ap.add_argument("-p", "--parameters", nargs='+', default=[], required=True,
                     help="list of parameters")
         args = ap.parse_args()
-        modelstream    = args.parameters[0]
-        WBD8           = args.parameters[1]
-        HYDROID        = args.parameters[2]
+        streams    = args.parameters[0]
+        wbd8           = args.parameters[1]
+        hydro_id        = args.parameters[2]
 
         oProcessor = BuildStreamTraversalColumns()
-        params = (modelstream, WBD8, HYDROID)
+        params = (streams, wbd8, hydro_id)
         tResults=None
         tResults = oProcessor.execute(params)
 
         del oProcessor
     except:
         print (str(trace()))
-    finally:
-        dt = datetime.datetime.now()
-        print  ('Finished at ' + dt.strftime("%Y-%m-%d %H:%M:%S"))
diff --git a/lib/clip_vectors_to_wbd.py b/lib/clip_vectors_to_wbd.py
new file mode 100755
index 000000000..92ece11f8
--- /dev/null
+++ b/lib/clip_vectors_to_wbd.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+
+import sys
+import geopandas as gpd
+import argparse
+from os.path import splitext
+from shapely.geometry import MultiPolygon,Polygon,Point
+
+def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks=False,extent='FR'):
+
+    hucUnitLength = len(str(hucCode))
+
+    # Get wbd buffer
+    wbd = gpd.read_file(wbd_filename)
+    wbd_buffer = gpd.read_file(wbd_buffer_filename)
+    projection = wbd_buffer.crs
+
+    # Clip ocean water polygon for future masking ocean areas (where applicable)
+    landsea = gpd.read_file(landsea_filename, mask = wbd_buffer)
+    if not landsea.empty:
+        landsea.to_file(subset_landsea_filename,driver=getDriver(subset_landsea_filename),index=False)
+    del landsea
+
+    # find intersecting lakes and writeout
+    print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+    nwm_lakes = gpd.read_file(nwm_lakes_filename, mask = wbd_buffer)
+
+    if not nwm_lakes.empty:
+        # perform fill process to remove holes/islands in the NWM lake polygons
+        nwm_lakes = nwm_lakes.explode()
+        nwm_lakes_fill_holes=MultiPolygon(Polygon(p.exterior) for p in nwm_lakes['geometry']) # remove donut hole geometries
+        # loop through the filled polygons and insert the new geometry
+        for i in range(len(nwm_lakes_fill_holes)):
+            nwm_lakes.loc[i,'geometry'] = nwm_lakes_fill_holes[i]
+        nwm_lakes.to_file(subset_nwm_lakes_filename,driver=getDriver(subset_nwm_lakes_filename),index=False)
+    del nwm_lakes
+
+    # find intersecting levee lines
+    print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+    nld_lines = gpd.read_file(nld_lines_filename, mask = wbd_buffer)
+    if not nld_lines.empty:
+        nld_lines.to_file(subset_nld_lines_filename,driver=getDriver(subset_nld_lines_filename),index=False)
+    del nld_lines
+
+    # find intersecting nwm_catchments
+    print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+    nwm_catchments = gpd.read_file(nwm_catchments_filename, mask = wbd_buffer)
+    nwm_catchments.to_file(subset_nwm_catchments_filename,driver=getDriver(subset_nwm_catchments_filename),index=False)
+    del nwm_catchments
+
+    # subset nhd headwaters
+    print("Subsetting NHD Headwater Points for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+    nhd_headwaters = gpd.read_file(nhd_headwaters_filename, mask = wbd_buffer)
+
+    # subset nhd streams
+    print("Querying NHD Streams for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+    nhd_streams = gpd.read_file(nhd_streams_filename, mask = wbd_buffer)
+
+    ## identify local headwater stream segments
+    nhd_streams_subset = gpd.read_file(nhd_streams_filename, mask = wbd)
+    nhd_streams_subset = nhd_streams_subset.loc[~nhd_streams_subset.FromNode.isin(list(set(nhd_streams_subset.ToNode) & set(nhd_streams_subset.FromNode)))]
+    nhd_streams_subset = nhd_streams_subset[~nhd_streams_subset['is_headwater']]
+
+    if not nhd_streams_subset.empty:
+        nhd_streams_subset = nhd_streams_subset.reset_index(drop=True)
+        start_coords = []
+        NHDPlusIDs = []
+        for index, linestring in enumerate(nhd_streams_subset.geometry):
+            start_coords = start_coords + [linestring.coords[-1]]
+            NHDPlusIDs = NHDPlusIDs + [nhd_streams_subset.iloc[index].NHDPlusID]
+
+        start_geoms = [Point(point) for point in start_coords]
+        local_headwaters = gpd.GeoDataFrame({'NHDPlusID': NHDPlusIDs,'geometry': start_geoms}, crs=projection, geometry='geometry')
+        nhd_headwaters = nhd_headwaters.append(local_headwaters)
+
+        # nhd_streams = nhd_streams.loc[~nhd_streams.NHDPlusID.isin(NHDPlusIDs)]
+
+    nhd_streams.to_file(subset_nhd_streams_filename,driver=getDriver(subset_nhd_streams_filename),index=False)
+
+    if len(nhd_headwaters) > 0:
+        nhd_headwaters.to_file(subset_nhd_headwaters_filename,driver=getDriver(subset_nhd_headwaters_filename),index=False)
+        del nhd_headwaters, nhd_streams
+    else:
+        print ("No headwater point(s) within HUC " + str(hucCode) +  " boundaries.")
+        sys.exit(0)
+
+    # subset nwm streams
+    print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+    nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd_buffer)
+    nwm_streams.to_file(subset_nwm_streams_filename,driver=getDriver(subset_nwm_streams_filename),index=False)
+    del nwm_streams
+
+def getDriver(filename):
+
+    driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
+    driver = driverDictionary[splitext(filename)[1]]
+
+    return(driver)
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Subset vector layers')
+    parser.add_argument('-d','--hucCode', help='HUC boundary ID', required=True,type=str)
+    parser.add_argument('-w','--nwm-streams', help='NWM flowlines', required=True)
+    parser.add_argument('-s','--nhd-streams',help='NHDPlus HR burnline',required=True)
+    parser.add_argument('-l','--nwm-lakes', help='NWM Lakes', required=True)
+    parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path', required=True)
+    parser.add_argument('-g','--wbd',help='HUC boundary',required=True)
+    parser.add_argument('-f','--wbd-buffer',help='Buffered HUC boundary',required=True)
+    parser.add_argument('-m','--nwm-catchments', help='NWM catchments', required=True)
+    parser.add_argument('-y','--nhd-headwaters',help='NHD headwaters',required=True)
+    parser.add_argument('-v','--landsea',help='LandSea - land boundary',required=True)
+    parser.add_argument('-c','--subset-nhd-streams',help='NHD streams subset',required=True)
+    parser.add_argument('-z','--subset-nld-lines',help='Subset of NLD levee vectors for HUC',required=True)
+    parser.add_argument('-a','--subset-lakes',help='NWM lake subset',required=True)
+    parser.add_argument('-n','--subset-catchments',help='NWM catchments subset',required=True)
+    parser.add_argument('-e','--subset-nhd-headwaters',help='NHD headwaters subset',required=True,default=None)
+    parser.add_argument('-b','--subset-nwm-streams',help='NWM streams subset',required=True)
+    parser.add_argument('-x','--subset-landsea',help='LandSea subset',required=True)
+    parser.add_argument('-o','--dissolve-links',help='remove multi-line strings',action="store_true",default=False)
+    parser.add_argument('-p','--extent',help='MS or FR extent',required=True)
+
+    args = vars(parser.parse_args())
+
+    hucCode = args['hucCode']
+    nwm_streams_filename = args['nwm_streams']
+    nhd_streams_filename = args['nhd_streams']
+    nwm_lakes_filename = args['nwm_lakes']
+    nld_lines_filename = args['nld_lines']
+    wbd_filename = args['wbd']
+    wbd_buffer_filename = args['wbd_buffer']
+    nwm_catchments_filename = args['nwm_catchments']
+    nhd_headwaters_filename = args['nhd_headwaters']
+    landsea_filename = args['landsea']
+    subset_nhd_streams_filename = args['subset_nhd_streams']
+    subset_nld_lines_filename = args['subset_nld_lines']
+    subset_nwm_lakes_filename = args['subset_lakes']
+    subset_nwm_catchments_filename = args['subset_catchments']
+    subset_nhd_headwaters_filename = args['subset_nhd_headwaters']
+    subset_nwm_streams_filename = args['subset_nwm_streams']
+    subset_landsea_filename = args['subset_landsea']
+    dissolveLinks = args['dissolve_links']
+    extent = args['extent']
+
+    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks,extent)
diff --git a/lib/output_cleanup.py b/lib/output_cleanup.py
index 98dcb9044..f4ea3ea19 100755
--- a/lib/output_cleanup.py
+++ b/lib/output_cleanup.py
@@ -27,6 +27,7 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
     production_whitelist = [
         'rem_zeroed_masked.tif',
         'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg',
+        'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg',
         'gw_catchments_reaches_filtered_addedAttributes.tif',
         'hydroTable.csv',
         'src.json'
@@ -54,15 +55,15 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         src_data = {}
         with open(os.path.join(output_folder_path, 'src.json')) as jsonf:
             src_data = json.load(jsonf)
-        
-        with open(os.path.join(output_folder_path, 'hydroTable.csv')) as csvf: 
-            csvReader = csv.DictReader(csvf) 
-            for row in csvReader: 
+
+        with open(os.path.join(output_folder_path, 'hydroTable.csv')) as csvf:
+            csvReader = csv.DictReader(csvf)
+            for row in csvReader:
                 if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
                     src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
 
         # Write src_data to JSON file
-        with open(os.path.join(output_folder_path, f'rating_curves_{huc_number}.json'), 'w') as jsonf: 
+        with open(os.path.join(output_folder_path, f'rating_curves_{huc_number}.json'), 'w') as jsonf:
             json.dump(src_data, jsonf)
 
         # Step 3, copy files to desired names
@@ -78,7 +79,7 @@ def whitelist_directory(directory_path, whitelist, additional_whitelist):
     directory = os.fsencode(directory_path)
     for file in os.listdir(directory_path):
         filename = os.fsdecode(file)
-        if filename not in whitelist: 
+        if filename not in whitelist:
             os.remove(os.path.join(directory_path, filename))
 
 
@@ -100,7 +101,6 @@ def whitelist_directory(directory_path, whitelist, additional_whitelist):
     additional_whitelist = args['additional_whitelist']
     is_production = args['is_production']
     is_viz_post_processing = args['is_viz_post_processing']
-    
+
     # Run output_cleanup
     output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, is_viz_post_processing)
-    
\ No newline at end of file
diff --git a/lib/reduce_nhd_stream_density.py b/lib/reduce_nhd_stream_density.py
new file mode 100644
index 000000000..cce2fa7ca
--- /dev/null
+++ b/lib/reduce_nhd_stream_density.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+from os.path import splitext
+from tqdm import tqdm
+from collections import deque
+import argparse
+import pygeos
+from shapely.wkb import dumps
+from utils.shared_functions import getDriver
+
+def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_filename,headwaters_filename,headwater_id,nwm_intersections_filename):
+
+    headwater_streams = pd.DataFrame()
+
+    nhd_streams = gpd.read_file(nhd_streams_filename)
+
+    for index, row in selected_wbd8.iterrows():
+        huc = row["HUC8"]
+
+        if huc.startswith(str(huc4)):
+            huc8_mask = selected_wbd8.loc[selected_wbd8.HUC8.str.startswith(huc)]
+            huc8_mask = huc8_mask.reset_index(drop=True)
+
+            headwaters_mask = gpd.read_file(headwaters_filename, mask = huc8_mask)
+            headwaters_mask = headwaters_mask.reset_index(drop=True)
+
+            streams_subset = gpd.read_file(nhd_streams_filename, mask = huc8_mask)
+
+            if not streams_subset.empty:
+                streams_subset.loc[:,'is_headwater'] = False
+                streams_subset = streams_subset.reset_index(drop=True)
+
+                streams_subset['b_geom'] = None
+                for index, linestring in enumerate(streams_subset.geometry):
+                    streams_subset.at[index, 'b_geom'] = dumps(linestring)
+
+                # create pygeos nhd stream geometries from WKB representation
+                streambin_geom = pygeos.io.from_wkb(streams_subset['b_geom'])
+
+                streams_subset.loc[:,'HUC8'] = str(huc)
+
+                if headwaters_mask[headwater_id].dtype=='int':
+                    n = -1
+                else:
+                    n = ''
+
+                streams_subset.loc[:,'headwaters_id'] = n
+
+                # find stream segment closest to headwater point
+                for index, point in headwaters_mask.iterrows():
+
+                    # convert headwaterpoint geometries to WKB representation
+                    wkb_points = dumps(point.geometry)
+
+                    # create pygeos headwaterpoint geometries from WKB representation
+                    pointbin_geom = pygeos.io.from_wkb(wkb_points)
+
+                    # distance to each stream segment
+                    distances = pygeos.measurement.distance(streambin_geom, pointbin_geom)
+
+                    # find minimum distance
+                    min_index = np.argmin(distances)
+
+                    # Closest segment to headwater
+                    streams_subset.loc[min_index,'is_headwater'] = True
+                    streams_subset.loc[min_index,'headwaters_id'] = point[headwater_id]
+
+                headwater_streams = headwater_streams.append(streams_subset[['NHDPlusID','is_headwater','headwaters_id','HUC8']])
+
+    headwater_streams = headwater_streams.sort_values('is_headwater', ascending=False).drop_duplicates('NHDPlusID') # keeps headwater=True for conflicting duplicates
+    nhd_streams = nhd_streams.merge(headwater_streams,on='NHDPlusID',how='inner')
+
+    del selected_wbd8, streams_subset, headwater_streams
+
+    huc4_mask_buffer = huc4_mask.buffer(10)
+
+    # identify inflowing streams
+    nwm_intersections = gpd.read_file(nwm_intersections_filename, mask=huc4_mask_buffer)
+
+    nhd_streams['downstream_of_headwater'] = False
+    nhd_streams = nhd_streams.explode()
+    nhd_streams = nhd_streams.reset_index(drop=True)
+
+    # find stream segment closest to nwm intersection point
+    for index, point in nwm_intersections.iterrows():
+
+        # distance to each stream segment
+        distances = nhd_streams.distance(point.geometry)
+
+        # find minimum distance
+        min_index = np.argmin(distances)
+
+        # update attributes for incoming stream
+        nhd_streams.loc[min_index,'is_headwater'] = True
+        nhd_streams.loc[min_index,'downstream_of_headwater'] = True
+
+    ## subset NHDPlus HR
+    nhd_streams['is_relevant_stream'] = nhd_streams['is_headwater'].copy()
+
+    # trace down from headwaters
+    nhd_streams.set_index('NHDPlusID',inplace=True,drop=False)
+
+    nhd_streams = get_downstream_segments(nhd_streams, 'is_headwater')
+
+    nhd_streams = nhd_streams.loc[nhd_streams['is_relevant_stream'],:]
+    nhd_streams.reset_index(drop=True,inplace=True)
+
+    return(nhd_streams)
+
+def get_downstream_segments(streams, attribute):
+
+    Q = deque(streams.loc[streams[attribute],'NHDPlusID'].tolist())
+    visited = set()
+
+    while Q:
+        q = Q.popleft()
+        if q in visited:
+            continue
+
+        visited.add(q)
+        toNode,DnLevelPat = streams.loc[q,['ToNode','DnLevelPat']]
+
+        try:
+            downstream_ids = streams.loc[streams['FromNode'] == toNode,:].index.tolist()
+        except ValueError: # 18050002 has duplicate nhd stream feature
+            if len(toNode.unique()) == 1:
+                toNode = toNode.iloc[0]
+                downstream_ids = streams.loc[streams['FromNode'] == toNode,:].index.tolist()
+
+        # If multiple downstream_ids are returned select the ids that are along the main flow path (i.e. exclude segments that are diversions)
+        if len(set(downstream_ids))>1: # special case: remove duplicate NHDPlusIDs
+            relevant_ids = [segment for segment in downstream_ids if DnLevelPat == streams.loc[segment,'LevelPathI']]
+        else:
+            relevant_ids = downstream_ids
+
+        streams.loc[relevant_ids,'is_relevant_stream'] = True
+        streams.loc[relevant_ids,'downstream_of_headwater'] = True
+
+        for i in relevant_ids:
+            if i not in visited:
+                Q.append(i)
+
+    return(streams)
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Reduce NHDPlus HR network based on headwater points')
+    parser.add_argument('-n','--huc-number',help='HUC number',required=True,type=str)
+    parser.add_argument('-b','--huc4-mask',help='HUC4 mask',required=True)
+    parser.add_argument('-w','--selected-wbd8',help='WBD8 layer',required=True)
+    parser.add_argument('-t','--nhd-streams',help='NHDPlus HR geodataframe',required=True)
+    parser.add_argument('-a','--headwaters-filename',help='Headwaters points layer name',required=True,type=str)
+    parser.add_argument('-s','--subset-nhd-streams-fileName',help='Output streams layer name',required=False,type=str,default=None)
+    parser.add_argument('-i','--headwater-id',help='Headwater points ID column',required=True)
+    parser.add_argument('-i','--nwm-intersections-filename',help='NWM HUC4 intersection points',required=True)
+
+    args = vars(parser.parse_args())
+
+    subset_streams_gdf = subset_nhd_network(huc_number,huc4_mask,selected_wbd8,nhd_streams,headwaters_filename,headwater_id)
+
+    if subset_nhd_streams_fileName is not None:
+        subset_streams_gdf.to_file(args['subset_nhd_streams_fileName'],driver=getDriver(args['subset_nhd_streams_fileName']),index=False)
diff --git a/lib/run_by_unit.sh b/lib/run_by_unit.sh
index 81bc676da..d3733dd6f 100755
--- a/lib/run_by_unit.sh
+++ b/lib/run_by_unit.sh
@@ -5,8 +5,8 @@ T_total_start
 
 echo -e $startDiv"Parameter Values"
 echo -e "extent=$extent"
-echo -e "negativeBurnValue=$negativeBurnValue"
-echo -e "maxSplitDistance_meters=$maxSplitDistance_meters"
+echo -e "negative_burn_value=$negative_burn_value"
+echo -e "max_split_distance_meters=$max_split_distance_meters"
 echo -e "mannings_n=$manning_n"
 echo -e "stage_min_meters=$stage_min_meters"
 echo -e "stage_interval_meters=$stage_interval_meters"
@@ -15,7 +15,7 @@ echo -e "slope_min=$slope_min"
 echo -e "ms_buffer_dist=$ms_buffer_dist"
 echo -e "ncores_gw=$ncores_gw"
 echo -e "ncores_fd=$ncores_fd"
-echo -e "defaultMaxJobs=$defaultMaxJobs"
+echo -e "default_max_jobs=$default_max_jobs"
 echo -e "memfree=$memfree"$stopDiv
 
 ## SET OUTPUT DIRECTORY FOR UNIT ##
@@ -38,6 +38,14 @@ else
   input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
 fi
 
+if [ "$extent" = "MS" ]; then
+  input_nhd_flowlines=$input_nhd_flowlines_ms
+  input_nhd_headwaters=$input_nhd_headwaters_ms
+else
+  input_nhd_flowlines=$input_nhd_flowlines_fr
+  input_nhd_headwaters=$input_nhd_headwaters_fr
+fi
+
 ## GET WBD ##
 echo -e $startDiv"Get WBD $hucNumber"$stopDiv
 date -u
@@ -54,16 +62,16 @@ Tstart
 ogr2ogr -f GPKG -dialect sqlite -sql "select ST_buffer(geom, 5000) from 'WBDHU$hucUnitLength'" $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd.gpkg
 Tcount
 
-## GET STREAMS ##
+## Subset Vector Layers ##
 echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-$libDir/snap_and_clip_to_nhd.py -d $hucNumber -w $input_NWM_Flows -f $input_NWM_Headwaters -s $input_NHD_Flowlines -l $input_NWM_Lakes -r $input_NLD -u $outputHucDataDir/wbd.gpkg -g $outputHucDataDir/wbd_buffered.gpkg -y $inputDataDir/ahp_sites/nws_lid.gpkg -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -t $outputHucDataDir/nwm_headwaters_proj_subset.gpkg -m $input_NWM_Catchments -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -p $extent
+$libDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_NWM_Flows -s $input_nhd_flowlines -l $input_NWM_Lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_NWM_Catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -p $extent
 Tcount
 
 if [ "$extent" = "MS" ]; then
-  if [[ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ]] ; then
+  if [[ ! -f $outputHucDataDir/nhd_headwater_points_subset.gpkg ]] ; then
     echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
     rm -rf $outputHucDataDir
     exit 0
@@ -145,11 +153,11 @@ Tcount
 # Using AGREE methodology, hydroenforce the DEM so that it is consistent
 # with the supplied stream network. This allows for more realistic catchment
 # delineation which is ultimately reflected in the output FIM mapping.
-echo -e $startDiv"Creating AGREE DEM using $buffer meter buffer"$stopDiv
+echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/dem_burned.tif ] && \
-$libDir/agreedem.py -r $outputHucDataDir/flows_grid_boolean.tif -d $outputHucDataDir/dem_meters.tif -w $outputHucDataDir -g $outputHucDataDir/temp_work -o $outputHucDataDir/dem_burned.tif -b $buffer -sm 10 -sh 1000
+$libDir/agreedem.py -r $outputHucDataDir/flows_grid_boolean.tif -d $outputHucDataDir/dem_meters.tif -w $outputHucDataDir -g $outputHucDataDir/temp_work -o $outputHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
 Tcount
 
 ## PIT REMOVE BURNED DEM ##
@@ -240,7 +248,7 @@ echo -e $startDiv"Split Derived Reaches $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ] && \
-$libDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $maxSplitDistance_meters $slope_min $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg $lakes_buffer_dist_meters
+$libDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $max_split_distance_meters $slope_min $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg $lakes_buffer_dist_meters
 Tcount
 
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
diff --git a/lib/snap_and_clip_to_nhd.py b/lib/snap_and_clip_to_nhd.py
deleted file mode 100755
index 7c715e06f..000000000
--- a/lib/snap_and_clip_to_nhd.py
+++ /dev/null
@@ -1,298 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import geopandas as gpd
-import pandas as pd
-from collections import deque,Counter
-import numpy as np
-from tqdm import tqdm
-import argparse
-from os.path import splitext,isfile
-from shapely.strtree import STRtree
-from shapely.geometry import Point,MultiLineString,LineString,mapping,MultiPolygon,Polygon
-
-def subset_vector_layers(hucCode,nwm_streams_fileName,nwm_headwaters_fileName,nhd_streams_fileName,nwm_lakes_fileName,nld_lines_fileName,nwm_catchments_fileName,wbd_fileName,wbd_buffer_fileName,ahps_sites_fileName,landsea_filename,subset_nhd_streams_fileName,subset_nwm_lakes_fileName,subset_nld_lines_fileName,subset_nwm_headwaters_fileName,subset_nwm_catchments_fileName,subset_nwm_streams_fileName,subset_landsea_filename,subset_nhd_headwaters_fileName=None,dissolveLinks=False,extent='FR'):
-
-    hucUnitLength = len(str(hucCode))
-
-    wbd = gpd.read_file(wbd_fileName)
-    wbd_buffer = gpd.read_file(wbd_buffer_fileName)
-    projection = wbd.crs
-    
-    # Clip ocean water polygon for future masking ocean areas (where applicable)
-    landsea_read = gpd.read_file(landsea_filename, mask = wbd_buffer)
-    landsea = gpd.clip(landsea_read, wbd_buffer)
-    if not landsea.empty:
-        landsea.to_file(subset_landsea_filename,driver=getDriver(subset_landsea_filename),index=False)
-    del landsea
-
-    # find intersecting lakes and writeout
-    print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_lakes = gpd.read_file(nwm_lakes_fileName, mask = wbd_buffer)
-
-    if not nwm_lakes.empty:
-        # perform fill process to remove holes/islands in the NWM lake polygons
-        nwm_lakes = nwm_lakes.explode()
-        nwm_lakes_fill_holes=MultiPolygon(Polygon(p.exterior) for p in nwm_lakes['geometry']) # remove donut hole geometries
-        # loop through the filled polygons and insert the new geometry
-        for i in range(len(nwm_lakes_fill_holes)):
-            nwm_lakes.loc[i,'geometry'] = nwm_lakes_fill_holes[i]
-        nwm_lakes.to_file(subset_nwm_lakes_fileName,driver=getDriver(subset_nwm_lakes_fileName),index=False)
-    del nwm_lakes
-
-    # find intersecting levee lines
-    print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nld_lines = gpd.read_file(nld_lines_fileName, mask = wbd)
-    if not nld_lines.empty:
-        nld_lines.to_file(subset_nld_lines_fileName,driver=getDriver(subset_nld_lines_fileName),index=False)
-    del nld_lines
-
-    # find intersecting nwm_catchments
-    print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_catchments = gpd.read_file(nwm_catchments_fileName, mask = wbd)
-    nwm_catchments.to_file(subset_nwm_catchments_fileName,driver=getDriver(subset_nwm_catchments_fileName),index=False)
-    del nwm_catchments
-
-    # query nhd+HR streams for HUC code
-    print("Querying NHD Streams for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nhd_streams = gpd.read_file(nhd_streams_fileName, mask = wbd_buffer)
-    nhd_streams = nhd_streams.explode()
-
-    # find intersecting nwm_headwaters
-    print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_streams = gpd.read_file(nwm_streams_fileName, mask = wbd_buffer)
-    nwm_streams.to_file(subset_nwm_streams_fileName,driver=getDriver(subset_nwm_streams_fileName),index=False)
-    del nwm_streams
-
-    # get nhd headwaters closest to nwm headwater points
-    print('Identify NHD Headwater streams nearest to NWM Headwater points',flush=True)
-    nhd_streams.loc[:,'is_nwm_headwater'] = False
-    # nhd_streams_tree = STRtree(nhd_streams.geometry)
-
-    if extent == 'FR':
-        nwm_headwaters = gpd.read_file(nwm_headwaters_fileName, mask = wbd_buffer)
-    elif extent == 'MS':
-        nwm_headwaters = gpd.read_file(ahps_sites_fileName, mask = wbd)
-
-        # check for incoming MS streams and convert to points
-        intersecting = nhd_streams.crosses(wbd.geometry[0])
-        incoming_flows = nhd_streams.loc[intersecting,:]
-        incoming_points_list = []
-
-        if len(incoming_flows) > 0:
-            for i,linesting in enumerate(incoming_flows.geometry):
-                incoming_points_list = incoming_points_list + [linesting.coords[-1]]
-
-        geometry = [Point(xy) for xy in zip(incoming_points_list)]
-        incoming_points = gpd.GeoDataFrame({'feature_id' : 0 ,'nwsid' : 'huc8_incoming' ,'geometry':geometry}, crs=nhd_streams.crs, geometry='geometry')
-
-        if (len(nwm_headwaters) > 0) or (len(incoming_points) > 0):
-
-            if len(nwm_headwaters) > 0:
-                    print ("Snapping forecasting points to nhd stream network")
-                    streamlines_union = nhd_streams.geometry.unary_union
-                    snapped_geoms = []
-                    snappedpoints_df = pd.DataFrame(nwm_headwaters).drop(columns=['geometry'])
-
-                    # snap lines to streams
-                    for i in range(len(nwm_headwaters)):
-                        snapped_geoms.append(streamlines_union.interpolate(streamlines_union.project(nwm_headwaters.geometry[i])))
-
-                    snappedpoints_df['geometry'] = snapped_geoms
-                    snapped_points = gpd.GeoDataFrame(snappedpoints_df,crs=nhd_streams.crs)
-
-            if (len(incoming_points) > 0) and (len(nwm_headwaters) > 0):
-                nwm_headwaters = snapped_points.append(incoming_points).reset_index(drop=True)
-            elif len(incoming_points) > 0:
-                nwm_headwaters = incoming_points.copy()
-        else:
-            print ("No AHPs point(s) within HUC " + str(hucCode) +  " boundaries.")
-            sys.exit(0)
-
-    for index, row in tqdm(nwm_headwaters.iterrows(),total=len(nwm_headwaters)):
-        distances = nhd_streams.distance(row['geometry'])
-        # nearestGeom = nhd_streams_tree.nearest(row['geometry'])
-        min_index = np.argmin(distances)
-        nhd_streams.loc[min_index,'is_nwm_headwater'] = True
-
-    nhd_streams = nhd_streams.loc[nhd_streams.geometry!=None,:] # special case: remove segments without geometries
-
-    # writeout nwm headwaters
-    if not nwm_headwaters.empty:
-        nwm_headwaters.to_file(subset_nwm_headwaters_fileName,driver=getDriver(subset_nwm_headwaters_fileName),index=False)
-    del nwm_headwaters
-
-    # copy over headwater features to nwm streams
-    nhd_streams['is_nwm_stream'] = nhd_streams['is_nwm_headwater'].copy()
-
-    # trace down from NWM Headwaters
-    print('Identify NHD streams downstream of relevant NHD Headwater streams',flush=True)
-    nhd_streams.set_index('NHDPlusID',inplace=True,drop=False)
-
-    Q = deque(nhd_streams.loc[nhd_streams['is_nwm_headwater'],'NHDPlusID'].tolist())
-    visited = set()
-
-    while Q:
-        q = Q.popleft()
-        if q in visited:
-            continue
-        visited.add(q)
-        toNode,DnLevelPat = nhd_streams.loc[q,['ToNode','DnLevelPat']]
-        try:
-            downstream_ids = nhd_streams.loc[nhd_streams['FromNode'] == toNode,:].index.tolist()
-        except ValueError: # 18050002 has duplicate nhd stream feature
-            if len(toNode.unique()) == 1:
-                toNode = toNode.iloc[0]
-                downstream_ids = nhd_streams.loc[nhd_streams['FromNode'] == toNode,:].index.tolist()
-        #If multiple downstream_ids are returned select the ids that are along the main flow path (i.e. exclude segments that are diversions)
-        if len(set(downstream_ids))>1: # special case: remove duplicate NHDPlusIDs
-            relevant_ids = [segment for segment in downstream_ids if DnLevelPat == nhd_streams.loc[segment,'LevelPathI']]
-        else:
-            relevant_ids = downstream_ids
-        nhd_streams.loc[relevant_ids,'is_nwm_stream'] = True
-        for i in relevant_ids:
-            if i not in visited:
-                Q.append(i)
-
-    nhd_streams = nhd_streams.loc[nhd_streams['is_nwm_stream'],:]
-
-    if dissolveLinks:
-        # remove multi-line strings
-        print("Dissolving NHD reaches to Links (reaches constrained to stream intersections)",flush=True)
-
-        nhd_streams.set_index('NHDPlusID',inplace=True,drop=False)
-        nhd_streams['before_confluence'] = nhd_streams.duplicated(subset='ToNode',keep=False)
-
-        nhd_streams.loc[nhd_streams['is_nwm_headwater'],'linkNo'] = np.arange(1,nhd_streams['is_nwm_headwater'].sum()+1)
-
-        Q = deque(nhd_streams.loc[nhd_streams['is_nwm_headwater'],'NHDPlusID'].tolist())
-        visited = set()
-        linkNo = np.max(nhd_streams.loc[nhd_streams['is_nwm_headwater'],'linkNo']) + 1
-        link_geometries = dict()
-
-        # adds all headwaters to link_geometries
-        for q in Q:
-            link_geometries[nhd_streams.loc[q,'linkNo']] = [p for p in zip(*nhd_streams.loc[q,'geometry'].coords.xy)][::-1]
-
-        # Do BFS
-        while Q:
-            q = Q.popleft()
-
-            if q in visited:
-                continue
-
-            visited.add(q)
-
-            toNode = nhd_streams.loc[q,'ToNode']
-
-            downstream_ids = nhd_streams.loc[nhd_streams['FromNode'] == toNode,:].index.tolist()
-            numberOfDownstreamIDs = len(downstream_ids)
-
-            for i in downstream_ids:
-                if i not in visited:
-                    Q.append(i)
-
-                    if nhd_streams.loc[q,'before_confluence'] or (numberOfDownstreamIDs > 1):
-                        # do not dissolve
-                        linkNo += 1
-                        nhd_streams.loc[i,'linkNo'] = linkNo
-
-                        next_stream_geometry = [p for p in zip(*nhd_streams.loc[i,'geometry'].coords.xy)][::-1]
-
-                        link_geometries[linkNo] = next_stream_geometry
-
-                    else:
-                        nhd_streams.loc[i,'linkNo'] = nhd_streams.loc[q,'linkNo']
-
-                        next_stream_geometry = [p for p in zip(*nhd_streams.loc[i,'geometry'].coords.xy)][::-1]
-
-                        link_geometries[nhd_streams.loc[i,'linkNo']] = link_geometries[nhd_streams.loc[i,'linkNo']] + next_stream_geometry
-
-
-        # convert dictionary to lists for keys (linkNos) and values (geometry linestrings)
-        output_links = [] ; output_geometries = []
-        for ln_no, ln_geom in link_geometries.items():
-            output_links = output_links + [ln_no]
-            output_geometries = output_geometries + [LineString(ln_geom)]
-
-        nhd_streams = gpd.GeoDataFrame({'linkNO' : output_links,'geometry': output_geometries},geometry='geometry',crs=projection)
-
-    # write to files
-    nhd_streams.reset_index(drop=True,inplace=True)
-    nhd_streams.to_file(subset_nhd_streams_fileName,driver=getDriver(subset_nhd_streams_fileName),index=False)
-
-    if subset_nhd_headwaters_fileName is not None:
-        # identify all nhd headwaters
-        print('Identify NHD headwater points',flush=True)
-        nhd_headwater_streams = nhd_streams.loc[nhd_streams['is_nwm_headwater'],:]
-        nhd_headwater_streams = nhd_headwater_streams.explode()
-
-        hw_points = np.zeros(len(nhd_headwater_streams),dtype=object)
-        for index,lineString in enumerate(nhd_headwater_streams.geometry):
-            hw_point = [point for point in zip(*lineString.coords.xy)][-1]
-            hw_points[index] = Point(*hw_point)
-
-        nhd_headwater_points = gpd.GeoDataFrame({'NHDPlusID' : nhd_headwater_streams['NHDPlusID'],
-                                                'geometry' : hw_points},geometry='geometry',crs=projection)
-
-        nhd_headwater_points.to_file(subset_nhd_headwaters_fileName,driver=getDriver(subset_nhd_headwaters_fileName),index=False)
-        del nhd_headwater_streams, nhd_headwater_points
-
-def getDriver(fileName):
-
-    driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
-    driver = driverDictionary[splitext(fileName)[1]]
-
-    return(driver)
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Subset vector layers')
-    parser.add_argument('-d','--hucCode', help='HUC boundary ID', required=True,type=str)
-    parser.add_argument('-w','--nwm-streams', help='NWM flowlines', required=True)
-    parser.add_argument('-f','--nwm-headwaters', help='NWM headwater points', required=True)
-    parser.add_argument('-s','--nhd-streams',help='NHDPlus HR burnline',required=True)
-    parser.add_argument('-l','--nwm-lakes', help='NWM Lakes', required=True)
-    parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path', required=True)
-    parser.add_argument('-m','--nwm-catchments', help='NWM catchments', required=True)
-    parser.add_argument('-u','--wbd',help='HUC boundary',required=True)
-    parser.add_argument('-g','--wbd-buffer',help='Buffered HUC boundary',required=True)
-    parser.add_argument('-y','--ahps-sites',help='Buffered HUC boundary',required=True)
-    parser.add_argument('-v','--landsea',help='LandSea - land boundary',required=True)
-    parser.add_argument('-c','--subset-nhd-streams',help='NHD streams subset',required=True)
-    parser.add_argument('-a','--subset-lakes',help='NWM lake subset',required=True)
-    parser.add_argument('-t','--subset-nwm-headwaters',help='NWM headwaters subset',required=True)
-    parser.add_argument('-z','--subset-nld-lines',help='Subset of NLD levee vectors for HUC',required=True)
-    parser.add_argument('-e','--subset-nhd-headwaters',help='NHD headwaters subset',required=True,default=None)
-    parser.add_argument('-n','--subset-catchments',help='NWM catchments subset',required=True)
-    parser.add_argument('-b','--subset-nwm-streams',help='NWM streams subset',required=True)
-    parser.add_argument('-x','--subset-landsea',help='LandSea subset',required=True)
-    parser.add_argument('-o','--dissolve-links',help='remove multi-line strings',action="store_true",default=False)
-    parser.add_argument('-p','--extent',help='MS or FR extent',required=True)
-
-    args = vars(parser.parse_args())
-
-    hucCode = args['hucCode']
-    nwm_streams_fileName = args['nwm_streams']
-    nwm_headwaters_fileName = args['nwm_headwaters']
-    nhd_streams_fileName = args['nhd_streams']
-    nwm_lakes_fileName = args['nwm_lakes']
-    nld_lines_fileName = args['nld_lines']
-    nwm_catchments_fileName = args['nwm_catchments']
-    wbd_fileName = args['wbd']
-    wbd_buffer_fileName = args['wbd_buffer']
-    ahps_sites_fileName = args['ahps_sites']
-    landsea_fileName = args['landsea']
-    subset_nhd_streams_fileName = args['subset_nhd_streams']
-    subset_nwm_lakes_fileName = args['subset_lakes']
-    subset_nwm_headwaters_fileName = args['subset_nwm_headwaters']
-    subset_nld_lines_fileName = args['subset_nld_lines']
-    subset_nwm_catchments_fileName = args['subset_catchments']
-    subset_nhd_headwaters_fileName = args['subset_nhd_headwaters']
-    subset_nwm_streams_fileName = args['subset_nwm_streams']
-    subset_landsea_filename = args['subset_landsea']
-    dissolveLinks = args['dissolve_links']
-    extent = args['extent']
-
-    subset_vector_layers(hucCode,nwm_streams_fileName,nwm_headwaters_fileName,nhd_streams_fileName,nwm_lakes_fileName,nld_lines_fileName,nwm_catchments_fileName,wbd_fileName,wbd_buffer_fileName,ahps_sites_fileName,landsea_fileName,subset_nhd_streams_fileName,subset_nwm_lakes_fileName,subset_nld_lines_fileName,subset_nwm_headwaters_fileName,subset_nwm_catchments_fileName,subset_nwm_streams_fileName,subset_landsea_filename,subset_nhd_headwaters_fileName,dissolveLinks,extent)
diff --git a/lib/split_flows.py b/lib/split_flows.py
index 0687ea0cd..db11d6d4a 100755
--- a/lib/split_flows.py
+++ b/lib/split_flows.py
@@ -161,6 +161,9 @@
 else:
     split_flows_gdf['LakeID'] = -999
 
+# need to figure out why so many duplicate stream segments for 04010101 FR
+split_flows_gdf = split_flows_gdf.drop_duplicates()
+
 # Create Ids and Network Traversal Columns
 addattributes = buildstreamtraversal.BuildStreamTraversalColumns()
 tResults=None
diff --git a/lib/utils/shared_functions.py b/lib/utils/shared_functions.py
index fa643d7fe..6ea7b0a74 100644
--- a/lib/utils/shared_functions.py
+++ b/lib/utils/shared_functions.py
@@ -1,62 +1,69 @@
 #!/usr/bin/env python3
 
 import os
+from os.path import splitext
 
+def getDriver(fileName):
+
+    driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
+    driver = driverDictionary[splitext(fileName)[1]]
+
+    return(driver)
 
 def pull_file(url, full_pulled_filepath):
     """
     This helper function pulls a file and saves it to a specified path.
-    
+
     Args:
         url (str): The full URL to the file to download.
         full_pulled_filepath (str): The full system path where the downloaded file will be saved.
     """
     import urllib.request
-    
+
     print("Pulling " + url)
     urllib.request.urlretrieve(url, full_pulled_filepath)
 
-        
+
 def delete_file(file_path):
     """
     This helper function deletes a file.
-    
+
     Args:
         file_path (str): System path to a file to be deleted.
     """
-    
+
     try:
         os.remove(file_path)
     except FileNotFoundError:
         pass
-    
-    
+
+
 def run_system_command(args):
     """
     This helper function takes a system command and runs it. This function is designed for use
     in multiprocessing.
-    
+
     Args:
         args (list): A single-item list, the first and only item being a system command string.
     """
-    
+
     # Parse system command.
     command = args[0]
-    
+
     # Run system command.
     os.system(command)
-    
-    
+
+
 def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
-    
+
     import geopandas as gp
     from utils.shared_variables import CONUS_STATE_LIST, PREP_PROJECTION
-    
+
     print("Subsetting " + wbd_gpkg + "...")
     # Read geopackage into dataframe.
     wbd = gp.read_file(wbd_gpkg)
     gdf = gp.GeoDataFrame(wbd)
-        
+
     for index, row in gdf.iterrows():
         state = row["STATES"]
         if state != None:  # Some polygons are empty in the STATES field.
@@ -75,8 +82,4 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
     # Overwrite geopackage.
     layer_name = os.path.split(wbd_gpkg)[1].strip('.gpkg')
     gdf.crs = PREP_PROJECTION
-    gdf.to_file(multilayer_wbd_geopackage, layer=layer_name, driver='GPKG')
-
-
-    
-    
\ No newline at end of file
+    gdf.to_file(multilayer_wbd_geopackage, layer=layer_name,driver='GPKG',index=False)

From b017e3e086bef9da42ae22e909a541430231ffcb Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Mon, 25 Jan 2021 08:54:05 -0600
Subject: [PATCH 009/359] Initial Version of the FIM API

Addition of an API service to schedule, run and manage fim_run jobs through a user-friendly web interface.

* Added api folder that contains all the codebase for the new service.
---
 CHANGELOG.md                                  |   8 +
 Dockerfile.prod                               |   5 +-
 api/.gitignore                                |   1 +
 api/README.md                                 |   0
 api/frontend/.env-template                    |   2 +
 api/frontend/docker-compose-dev.yml           |  52 +++
 api/frontend/docker-compose-prod.yml          |  47 +++
 api/frontend/gui/Dockerfile                   |  14 +
 api/frontend/gui/entrypoint.sh                |   5 +
 api/frontend/gui/gui.py                       |  16 +
 api/frontend/gui/requirements.txt             |   4 +
 api/frontend/gui/templates/index.html         | 395 ++++++++++++++++++
 api/frontend/gui/wsgi.py                      |   4 +
 api/frontend/nginx-dev.conf                   |  72 ++++
 api/frontend/nginx-prod.conf                  |  55 +++
 api/frontend/output_handler/Dockerfile        |  12 +
 api/frontend/output_handler/entrypoint.sh     |   6 +
 api/frontend/output_handler/output_handler.py |  94 +++++
 api/frontend/output_handler/requirements.txt  |   2 +
 api/node/.env-template                        |   5 +
 api/node/connector/Dockerfile                 |  16 +
 api/node/connector/connector.py               | 169 ++++++++
 api/node/connector/entrypoint.sh              |   5 +
 api/node/connector/requirements.txt           |   3 +
 api/node/docker-compose-dev.yml               |  38 ++
 api/node/docker-compose-prod.yml              |  51 +++
 api/node/nginx.conf                           |  64 +++
 api/node/updater/Dockerfile                   |  14 +
 api/node/updater/entrypoint.sh                |   5 +
 api/node/updater/requirements.txt             |   2 +
 api/node/updater/updater.py                   | 147 +++++++
 31 files changed, 1311 insertions(+), 2 deletions(-)
 create mode 100644 api/.gitignore
 create mode 100644 api/README.md
 create mode 100644 api/frontend/.env-template
 create mode 100644 api/frontend/docker-compose-dev.yml
 create mode 100644 api/frontend/docker-compose-prod.yml
 create mode 100644 api/frontend/gui/Dockerfile
 create mode 100755 api/frontend/gui/entrypoint.sh
 create mode 100644 api/frontend/gui/gui.py
 create mode 100644 api/frontend/gui/requirements.txt
 create mode 100644 api/frontend/gui/templates/index.html
 create mode 100644 api/frontend/gui/wsgi.py
 create mode 100644 api/frontend/nginx-dev.conf
 create mode 100644 api/frontend/nginx-prod.conf
 create mode 100644 api/frontend/output_handler/Dockerfile
 create mode 100755 api/frontend/output_handler/entrypoint.sh
 create mode 100644 api/frontend/output_handler/output_handler.py
 create mode 100644 api/frontend/output_handler/requirements.txt
 create mode 100644 api/node/.env-template
 create mode 100644 api/node/connector/Dockerfile
 create mode 100644 api/node/connector/connector.py
 create mode 100755 api/node/connector/entrypoint.sh
 create mode 100644 api/node/connector/requirements.txt
 create mode 100644 api/node/docker-compose-dev.yml
 create mode 100644 api/node/docker-compose-prod.yml
 create mode 100644 api/node/nginx.conf
 create mode 100644 api/node/updater/Dockerfile
 create mode 100755 api/node/updater/entrypoint.sh
 create mode 100644 api/node/updater/requirements.txt
 create mode 100644 api/node/updater/updater.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d028b692d..eed3cd415 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.2.0 - 2021-01-25 - [PR #218](https://github.com/NOAA-OWP/cahaba/pull/218)
+
+Addition of an API service to schedule, run and manage `fim_run` jobs through a user-friendly web interface.
+
+### Additions
+
+ - `api` folder that contains all the codebase for the new service.
+
 ## v3.0.1.0 - 2021-01-21 - [PR #206](https://github.com/NOAA-OWP/cahaba/pull/206)
 
 Preprocess MS and FR stream networks
diff --git a/Dockerfile.prod b/Dockerfile.prod
index 86fd4ec31..79c50bdaf 100644
--- a/Dockerfile.prod
+++ b/Dockerfile.prod
@@ -84,8 +84,9 @@ ENV PYTHONUNBUFFERED=TRUE
 ENV PATH="$projectDir:${PATH}"
 
 ## install python 3 modules ##
-COPY requirements.txt .
-RUN pip3 install --no-binary shapely --no-binary pygeos -r requirements.txt --no-cache-dir
+COPY Pipfile .
+COPY Pipfile.lock .
+RUN pip3 install pipenv && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
 
 ## Copy the source code to the image
 COPY . $projectDir/
diff --git a/api/.gitignore b/api/.gitignore
new file mode 100644
index 000000000..2eea525d8
--- /dev/null
+++ b/api/.gitignore
@@ -0,0 +1 @@
+.env
\ No newline at end of file
diff --git a/api/README.md b/api/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/api/frontend/.env-template b/api/frontend/.env-template
new file mode 100644
index 000000000..afd955443
--- /dev/null
+++ b/api/frontend/.env-template
@@ -0,0 +1,2 @@
+DATA_PATH=
+SOCKET_URL=
\ No newline at end of file
diff --git a/api/frontend/docker-compose-dev.yml b/api/frontend/docker-compose-dev.yml
new file mode 100644
index 000000000..1f7b0ca2d
--- /dev/null
+++ b/api/frontend/docker-compose-dev.yml
@@ -0,0 +1,52 @@
+version: '3.5'
+services:
+  fim_frontend_gui:
+    image: fim_frontend_gui
+    build:
+      context: ./gui
+    container_name: fim_frontend_gui
+    restart: always
+    env_file:
+      - .env
+    expose:
+      - "5000"
+    networks:
+      - fim
+    volumes:
+      - ./gui/templates/:/opt/gui/templates/
+      - ./gui/gui.py:/opt/gui/gui.py
+  fim_frontend_output_handler:
+    image: fim_frontend_output_handler
+    build:
+      context: ./output_handler
+    container_name: fim_frontend_output_handler
+    restart: always
+    external_links: 
+      - fim_node_connector
+    env_file:
+      - .env
+    networks:
+      - fim
+    volumes:
+      - ${DATA_PATH}:/data/
+      - ./output_handler/output_handler.py:/opt/output_handler/output_handler.py
+  fim_nginx:
+    image: nginx
+    container_name: fim_nginx
+    restart: always
+    depends_on:
+      - fim_frontend_gui
+    external_links: 
+      - fim_node_connector
+    ports:
+      - "80:80"
+    volumes:
+      - ./nginx-dev.conf:/etc/nginx/nginx.conf:ro
+    networks:
+      - fim
+    command: [nginx]
+
+networks:
+  fim:
+    name: fim
+          
\ No newline at end of file
diff --git a/api/frontend/docker-compose-prod.yml b/api/frontend/docker-compose-prod.yml
new file mode 100644
index 000000000..b408fe054
--- /dev/null
+++ b/api/frontend/docker-compose-prod.yml
@@ -0,0 +1,47 @@
+version: '3.5'
+services:
+  fim_frontend_gui:
+    image: fim_frontend_gui
+    build:
+      context: ./gui
+    container_name: fim_frontend_gui
+    restart: always
+    env_file:
+      - .env
+    expose:
+      - "5000"
+    networks:
+      - fim
+  fim_frontend_output_handler:
+    image: fim_frontend_output_handler
+    build:
+      context: ./output_handler
+    container_name: fim_frontend_output_handler
+    restart: always
+    external_links: 
+      - fim_node_connector
+    env_file:
+      - .env
+    networks:
+      - fim
+    volumes:
+      - ${DATA_PATH}:/data/
+  fim_nginx:
+    image: nginx
+    container_name: fim_nginx
+    restart: always
+    depends_on:
+      - fim_frontend_gui
+    external_links: 
+      - fim_node_connector
+    ports:
+      - "80:80"
+    volumes:
+      - ./nginx-prod.conf:/etc/nginx/nginx.conf:ro
+    networks:
+      - fim
+    command: [nginx]
+
+networks:
+  fim:
+    name: fim
\ No newline at end of file
diff --git a/api/frontend/gui/Dockerfile b/api/frontend/gui/Dockerfile
new file mode 100644
index 000000000..fb5d0750e
--- /dev/null
+++ b/api/frontend/gui/Dockerfile
@@ -0,0 +1,14 @@
+FROM python:3.8.5-slim-buster
+
+ENV PYTHONUNBUFFERED 1
+RUN mkdir -p /opt/gui
+WORKDIR /opt/gui
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt --no-cache-dir
+COPY . /opt/gui
+
+EXPOSE 5000
+
+RUN chmod +x /opt/gui/entrypoint.sh
+ENTRYPOINT ["/opt/gui/entrypoint.sh"]
diff --git a/api/frontend/gui/entrypoint.sh b/api/frontend/gui/entrypoint.sh
new file mode 100755
index 000000000..fbf7d3145
--- /dev/null
+++ b/api/frontend/gui/entrypoint.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+cd /opt/gui/
+echo "Starting Gunicorn"
+exec gunicorn --bind 0.0.0.0:5000 --reload wsgi:app
\ No newline at end of file
diff --git a/api/frontend/gui/gui.py b/api/frontend/gui/gui.py
new file mode 100644
index 000000000..456622d72
--- /dev/null
+++ b/api/frontend/gui/gui.py
@@ -0,0 +1,16 @@
+import os
+from gevent import monkey
+monkey.patch_all()
+
+from flask import Flask, render_template, request
+
+SOCKET_URL = os.environ.get('SOCKET_URL')
+
+app = Flask(__name__)
+
+@app.route('/')
+def main():
+    return render_template('index.html', socket_url=SOCKET_URL)
+
+if __name__ == '__main__':
+    app.run("0.0.0.0", port=5000)
\ No newline at end of file
diff --git a/api/frontend/gui/requirements.txt b/api/frontend/gui/requirements.txt
new file mode 100644
index 000000000..a78a884a3
--- /dev/null
+++ b/api/frontend/gui/requirements.txt
@@ -0,0 +1,4 @@
+flask==1.1.2
+flask-socketio==5.0.0
+gevent==20.9.0
+gunicorn==20.0.4
\ No newline at end of file
diff --git a/api/frontend/gui/templates/index.html b/api/frontend/gui/templates/index.html
new file mode 100644
index 000000000..abccdd7cc
--- /dev/null
+++ b/api/frontend/gui/templates/index.html
@@ -0,0 +1,395 @@
+<html>
+    <head>
+        <title>Cahaba API</title>
+
+        <link rel="preconnect" href="https://fonts.gstatic.com">
+        <link href="https://fonts.googleapis.com/css2?family=Roboto&display=swap" rel="stylesheet">
+        <style>
+            html, body {
+                display: flex;
+                width: 100%;
+                height: 100%;
+
+                flex-direction: column;
+                align-items: center;
+                justify-content: center;
+            }
+
+            p, span, h1, h2, h3, h4, h5, h6, div, form, input, label, table {
+                font-family: 'Roboto', sans-serif;
+            }
+
+            #request-container {
+                display: flex;
+                width: 25rem;
+
+                flex-direction: column;
+
+                cursor: inherit;
+                opacity: inherit;
+            }
+
+            #request-container.loading {
+                cursor: progress;
+                opacity: 0.3;
+            }
+
+            #request-container.loading > div {
+                pointer-events: none;
+            }
+
+            #request-tabs-container {
+                display: flex;
+                width: 100%;
+                height: 2rem;
+
+                justify-content: space-evenly;
+
+                border: 2px solid black;
+            }
+
+            .request-tab {
+                display: flex;
+                flex: 1;
+                height: 100%;
+
+                align-items: center;
+                justify-content: center;
+
+                color: black;
+                background-color: white;
+
+                transition: all 0.1s linear;
+
+                cursor: pointer;
+            }
+
+            .request-tab:hover {
+                color: white;
+                background-color: rgb(99, 99, 99);
+            }
+
+            .request-tab.active {
+                color: white;
+                background-color: rgb(56, 56, 56);
+            }
+
+            .request-form {
+                display: flex;
+                width: 100%;
+                min-height: 20rem;
+                flex-direction: column;
+
+                justify-content: space-between;
+
+                border: 2px solid black;
+                border-top: none;
+                border-radius: 0 0 8px 8px;
+
+                margin: 0;
+                overflow: hidden;
+            }
+
+            .error-list {
+                display: flex;
+                width: 100%;
+                min-height: 4rem;
+                flex-direction: column;
+
+                align-items: center;
+                justify-content: center;
+
+                color: red;
+            }
+
+            .form-content {
+                display: flex;
+                flex-direction: column;
+                margin-bottom: 1rem;
+
+                padding: 1rem;
+            }
+
+            .start-button {
+                display: flex;
+                width: 100%;
+                height: 4rem;
+
+                align-items: center;
+                justify-content: center;
+
+                font-size: 2rem;
+                color: white;
+                background-color: #286ebe;
+                cursor: pointer;
+
+                transition: all 0.5s linear;
+                pointer-events: inherit;
+            }
+            
+            .start-button.not-connected {
+                background-color: #d11111;
+                pointer-events: none;
+            }
+
+            .start-button:hover {
+                background-color: #093568;
+            }
+
+            .form-field {
+                display: flex;
+                width: 0;
+
+                flex: 1;
+            }
+
+            .container-horizontal {
+                display: flex;
+
+                margin-bottom: 1rem;
+            }
+
+            .input-with-label {
+                display: flex;
+
+                margin-right: 2rem;
+            }
+
+            .styled-table-wrapper {
+                display: none;
+                height: 12rem;
+
+                margin-top: 2rem;
+
+                overflow-y: auto;
+            }
+
+            .styled-table {
+                border-collapse: collapse;
+                min-width: 10rem;
+                box-shadow: 0 0 20px rgba(0, 0, 0, 0.15);
+            }
+
+            .styled-table th {
+                position: sticky;
+                top: 0;
+
+                background-color: #286ebe;
+                color: #ffffff;
+                font-weight: normal;
+                text-align: left;
+            }
+
+            .styled-table th,
+            .styled-table td {
+                padding: 1rem;
+            }
+
+            .styled-table tbody tr {
+                border-bottom: thin solid #dddddd;
+            }
+
+            .styled-table tbody tr:nth-of-type(even) {
+                background-color: #f3f3f3;
+            }
+
+            .styled-table tbody tr:last-of-type {
+                border-bottom: 2px solid #286ebe;
+            }
+
+            .styled-table tbody tr.job-completed {
+                font-weight: bold;
+                color: #2fd107;
+            }
+        </style>
+    </head>
+    <body>
+        <!-- <h1>FIM Run API</h1> -->
+        <div id="request-container">
+            <div id="request-tabs-container">
+                <div class="request-tab active" onclick="selectForm(event, 'fim-run')">FIM Run</div>
+                <div class="request-tab" onclick="selectForm(event, 'calibration')">Calibration</div>
+                <div class="request-tab" onclick="selectForm(event, 'preprocessing')">Pre-processing</div>
+            </div>
+            <div id="fim-run" class="request-form">
+                <div class="form-content">
+                    <h3 style="margin: 0;">Basic</h3>
+                    <div class="container-horizontal">
+                        <input type="text" id="job-name" class="form-field" placeholder="Job Name"></input>
+                        <input type="text" id="hucs" class="form-field" placeholder="HUC(s)"></input>
+                        <input type="text" id="git-branch" class="form-field" placeholder="Git Branch"></input>
+                    </div>
+                    <h3 style="margin: 0;">Configuration</h3>
+                    <div class="container-horizontal">
+                        <div class="input-with-label">
+                            <input type="radio" name="configuration" value="default" checked>
+                            <label for="default">Default</label>
+                        </div>
+                        <div class="input-with-label">
+                            <input type="radio" name="configuration" value="calibrated">
+                            <label for="calibrated">Calibrated</label>
+                        </div>
+                    </div>
+                    <h3 style="margin: 0;">Extent</h3>
+                    <div class="container-horizontal">
+                        <div class="input-with-label">
+                            <input type="radio" name="extent" value="FR" checked>
+                            <label for="FR">Full Resolution</label>
+                        </div>
+                        <div class="input-with-label">
+                            <input type="radio" name="extent" value="MS">
+                            <label for="MS">Main Stem</label>
+                        </div>
+                    </div>
+                    <div class="input-with-label">
+                        <input type="checkbox" name="dev-run" value="dev-run">
+                        <label for="dev-run"> Include all intermediate output files?</label>
+                    </div>
+                    <div class="error-list" id="fim-run-errors">
+    
+                    </div>
+                </div>
+                <div class="start-button not-connected" onclick="submitRequest('fim-run')">Not Connected</div>
+            </div>
+            <div id="calibration" class="request-form" style="display: None;">
+                Calibration stuff goes here
+            </div>
+            <div id="preprocessing" class="request-form" style="display: None;">
+                Pre-processing stuff goes here
+            </div>
+        </div>
+        <div id="job-list-wrapper" class="styled-table-wrapper">
+            <table class="styled-table">
+                <thead>
+                    <tr>
+                        <th>Name</th>
+                        <th>Time Elapsed</th>
+                        <th>Status</th>
+                        <th>Outputs Saved</th>
+                    </tr>
+                </thead>
+                <tbody id="job-list">
+                </tbody>
+            </table>
+        </div>
+
+        <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/socket.io/3.0.4/socket.io.min.js"></script>
+
+        <script type="text/javascript">
+            const socket = io.connect('{{socket_url}}')
+
+            document.addEventListener("DOMContentLoaded", () => {
+                // Socket.io related code
+                socket.on('connect', () => {
+                    const buttons = document.getElementsByClassName('start-button')
+                    for (let i = 0; i < buttons.length; i++) {
+                        buttons[i].classList.remove('not-connected')
+                        buttons[i].innerHTML = 'Start'
+                    }
+                })
+
+                socket.on('disconnect', () => {
+                    const buttons = document.getElementsByClassName('start-button')
+                    for (let i = 0; i < buttons.length; i++) {
+                        buttons[i].classList.add('not-connected')
+                        buttons[i].innerHTML = 'Not Connected'
+                    }
+                })
+
+                socket.on('client_update', jobs => {
+                    let jobsTableBody = ''
+
+                    if (Object.keys(jobs).length === 0)
+                        document.querySelector('#job-list-wrapper').style.display = 'none'
+                    else
+                        document.querySelector('#job-list-wrapper').style.display = 'block'
+
+                    Object.values(jobs).forEach(j => {
+                        let outputs_saved = "N/A"
+                        if (Object.keys(j['output_files_saved']).length > 0) {
+                            const total = Object.keys(j['output_files_saved']).length
+                            const current =  Object.values(j['output_files_saved']).filter(v => v === true).length
+                            outputs_saved = `${current}/${total}`
+                        }
+
+                        jobsTableBody += `<tr ${j['status'] === 'Completed' && 'class="job-completed"'}>
+                                            <td>${j['nice_name'].replace(/_/g, ' ')}</td>
+                                            <td>${j['time_elapsed']} Seconds</td>
+                                            <td>${j['status']}</td>
+                                            <td>${outputs_saved}</td>
+                                        </tr>`
+                    })
+                    document.querySelector('#job-list').innerHTML = jobsTableBody
+                })
+
+                socket.on('validation_errors', validation_errors => {
+                    document.querySelector('#request-container').classList.remove('loading')
+                    document.querySelector('#fim-run-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
+                })
+
+                socket.on('job_started', job_type => {
+                    document.querySelector('#request-container').classList.remove('loading')
+                    if (job_type === 'fim_run') {
+                        document.querySelector('#job-name').value = ''
+                        document.querySelector('#hucs').value = ''
+                        document.querySelector('#git-branch').value = ''
+                        document.querySelector("input[name='dev-run']").checked = false
+                        document.querySelector("input[name='configuration'][value='default']").checked = true
+                        document.querySelector("input[name='extent'][value='FR']").checked = true
+                        document.querySelector('#fim-run-errors').innerHTML = ''
+                    }
+                })
+            })
+
+            // Fim_run related code
+            const submitRequest = requestName => {
+                let validation_errors = []
+                const job_name = document.querySelector('#job-name').value
+                const hucs = document.querySelector('#hucs').value
+                const git_branch = document.querySelector('#git-branch').value
+                const dev_run = document.querySelector("input[name='dev-run']").checked
+                const configuration =  Array.from(document.querySelectorAll("input[name='configuration']")).filter(c => c.checked)[0].value
+                const extent =  Array.from(document.querySelectorAll("input[name='extent']")).filter(c => c.checked)[0].value
+
+                if (job_name === '') validation_errors.push('Job Name Cannot Be Empty')
+                if (hucs === '') validation_errors.push('Huc(s) Cannot Be Empty')
+                if (git_branch === '') validation_errors.push('Git Branch Cannot Be Empty')
+
+                if (validation_errors.length > 0){
+                    document.querySelector('#fim-run-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
+                    return
+                }
+
+                document.querySelector('#request-container').classList.add('loading')
+                
+                socket.emit('new_job', {
+                    job_name,
+                    hucs,
+                    git_branch,
+                    dev_run,
+                    configuration,
+                    extent
+                })
+            }
+
+            const selectForm = (e, formName) => {
+                // Close all forms and remove active from all tabs
+                const forms = document.getElementsByClassName('request-form')
+                for (let i = 0; i < forms.length; i++) {
+                    forms[i].style.display = 'none'
+                }
+
+                const tabs = document.getElementsByClassName('request-tab')
+                for (let i = 0; i < tabs.length; i++) {
+                    tabs[i].classList.remove('active')
+                }
+                
+                // Add active class to newly selected tab
+                e.target.classList.add('active')
+
+                // Make newly selected form visible
+                document.getElementById(formName).style.display = 'flex'
+            }
+        </script>
+    </body>
+</html>
\ No newline at end of file
diff --git a/api/frontend/gui/wsgi.py b/api/frontend/gui/wsgi.py
new file mode 100644
index 000000000..b9303d13e
--- /dev/null
+++ b/api/frontend/gui/wsgi.py
@@ -0,0 +1,4 @@
+from gui import app
+
+if __name__ == "__main__":
+    app.run()
\ No newline at end of file
diff --git a/api/frontend/nginx-dev.conf b/api/frontend/nginx-dev.conf
new file mode 100644
index 000000000..97e37c1f0
--- /dev/null
+++ b/api/frontend/nginx-dev.conf
@@ -0,0 +1,72 @@
+user nginx;
+worker_processes 1;
+pid /var/run/nginx.pid;
+daemon off;
+
+events {
+  worker_connections 512;
+  # multi_accept on;
+}
+
+http {
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  proxy_connect_timeout 300;
+  proxy_send_timeout 300;
+  proxy_read_timeout 90m;
+  send_timeout 300;
+  keepalive_timeout 65;
+  types_hash_max_size 2048;
+
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+
+  log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                             '"$request" $status $body_bytes_sent '
+                             '"$http_referer" "$http_user_agent"'
+                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+
+  access_log /var/log/nginx/access.log upstream_time;
+  error_log /var/log/nginx/error.log warn;
+
+  server {
+    listen 80;
+    client_header_buffer_size 64k;
+    large_client_header_buffers 4 64k;
+    server_name _;
+    root /var/www/;
+
+    gzip on;
+    gzip_types application/json;
+    proxy_http_version 1.1;
+
+    location /stats/nginx {
+      stub_status on;
+    }
+
+    # Node side (these should only be used if the frontend is on the same machine as the connector)
+    location / {
+      proxy_pass http://fim_node_connector:6000/;
+      proxy_set_header Host            $host;
+      proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    location /socket.io {
+      proxy_http_version 1.1;
+      proxy_buffering off;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "Upgrade";
+      proxy_pass http://fim_node_connector:6000/socket.io;
+      proxy_set_header Host            $host;
+      proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # Frontend Side
+    location /api {
+      proxy_pass http://fim_frontend_gui:5000/;
+      proxy_set_header Host            $host;
+      proxy_set_header X-Forwarded-Proto $scheme;
+    }
+  }
+}
diff --git a/api/frontend/nginx-prod.conf b/api/frontend/nginx-prod.conf
new file mode 100644
index 000000000..878d423fc
--- /dev/null
+++ b/api/frontend/nginx-prod.conf
@@ -0,0 +1,55 @@
+user nginx;
+worker_processes 1;
+pid /var/run/nginx.pid;
+daemon off;
+
+events {
+  worker_connections 512;
+  # multi_accept on;
+}
+
+http {
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  proxy_connect_timeout 300;
+  proxy_send_timeout 300;
+  proxy_read_timeout 90m;
+  send_timeout 300;
+  keepalive_timeout 65;
+  types_hash_max_size 2048;
+
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+
+  log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                             '"$request" $status $body_bytes_sent '
+                             '"$http_referer" "$http_user_agent"'
+                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+
+  access_log /var/log/nginx/access.log upstream_time;
+  error_log /var/log/nginx/error.log warn;
+
+  server {
+    listen 80;
+    client_header_buffer_size 64k;
+    large_client_header_buffers 4 64k;
+    server_name _;
+    root /var/www/;
+
+    gzip on;
+    gzip_types application/json;
+    proxy_http_version 1.1;
+
+    location /stats/nginx {
+      stub_status on;
+    }
+    
+    # Frontend Side
+    location /api {
+      proxy_pass http://fim_frontend_gui:5000/;
+      proxy_set_header Host            $host;
+      proxy_set_header X-Forwarded-Proto $scheme;
+    }
+  }
+}
diff --git a/api/frontend/output_handler/Dockerfile b/api/frontend/output_handler/Dockerfile
new file mode 100644
index 000000000..68498a6d3
--- /dev/null
+++ b/api/frontend/output_handler/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3.8.5-slim-buster
+
+ENV PYTHONUNBUFFERED 1
+RUN mkdir -p /opt/output_handler
+WORKDIR /opt/output_handler
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt --no-cache-dir
+COPY . /opt/output_handler
+
+RUN chmod +x /opt/output_handler/entrypoint.sh
+ENTRYPOINT ["/opt/output_handler/entrypoint.sh"]
diff --git a/api/frontend/output_handler/entrypoint.sh b/api/frontend/output_handler/entrypoint.sh
new file mode 100755
index 000000000..248541b09
--- /dev/null
+++ b/api/frontend/output_handler/entrypoint.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+umask 002
+cd /opt/output_handler/
+echo "Starting Output Handler"
+python ./output_handler.py
\ No newline at end of file
diff --git a/api/frontend/output_handler/output_handler.py b/api/frontend/output_handler/output_handler.py
new file mode 100644
index 000000000..1d7de4a88
--- /dev/null
+++ b/api/frontend/output_handler/output_handler.py
@@ -0,0 +1,94 @@
+import os
+import time
+
+import socketio
+
+SOCKET_URL = os.environ.get('SOCKET_URL')
+
+pending_files = {}
+
+def handle_outputs(data):
+    name = f"{data['job_name']}_{data['file_name']}"
+    if name not in pending_files:
+        pending_files[name] = {
+            'locked': False,
+            'current_index': 0,
+            'nice_name': data['nice_name'],
+            'job_name': data['job_name'],
+            'directory_path': data['directory_path'],
+            'file_name': data['file_name']
+        }
+
+    pending_files[name][data['chunk_index']] = data['file_chunk']
+
+
+    work_to_do = True
+    while work_to_do:
+        work_to_do = False
+
+        nice_name = pending_files[name]['nice_name']
+        job_name = pending_files[name]['job_name']
+        directory_path = pending_files[name]['directory_path']
+        file_name = pending_files[name]['file_name']
+
+        # If the last chunk just got added, waiting to write any potentially missing data to file
+        if data['file_chunk'] == None and pending_files[name]['locked']:
+            while name in pending_files and pending_files[name]['locked']:
+                print("EOF, waiting till not locked")
+                sio.sleep(0.5)
+            if not name in pending_files:
+                return
+
+        # To ensure that the files are being written in the correct order, use current_index
+        # to write the correct file chunk.
+        if not pending_files[name]['locked'] and pending_files[name]['current_index'] in pending_files[name]:
+            pending_files[name]['locked'] = True
+            file_chunk = pending_files[name].pop(pending_files[name]['current_index'])
+
+            # End of file
+            if file_chunk == None:
+                if sio.connected:
+                    sio.emit('output_handler_finished_file', {'job_name': job_name, 'file_path': f"{directory_path}/{file_name}"})
+                    print("finished with file", name, directory_path, file_name)
+                    # files_to_delete.append(name)
+                    pending_files.pop(name)
+                continue
+            else:
+                # Not end of file, keep looping till you can't do more work
+                work_to_do = True
+
+            # Create folder if it doesn't yet exist and set writing mode
+            mode = 'ab'
+            if pending_files[name]['current_index'] == 0:
+                mode = 'wb'
+                try:
+                    os.makedirs(f"/data/outputs/{nice_name}/{directory_path}")
+                except:
+                    pass
+                
+            # Write binary data to file
+            with open(f"/data/outputs/{nice_name}/{directory_path}/{file_name}", mode) as binary_file:
+                print(f"Writing chunk {pending_files[name]['current_index']} for file {directory_path}/{file_name}")
+                binary_file.write(file_chunk)
+
+            # Remove current chunk from list
+            pending_files[name]['current_index'] += 1
+            pending_files[name]['locked'] = False
+    
+sio = socketio.Client()
+
+@sio.event
+def connect():
+    print("Output Handler Connected!")
+    sio.emit('output_handler_connected')
+    
+
+@sio.event
+def disconnect():
+    print('disconnected from server')
+
+@sio.on('new_job_outputs')
+def ws_new_job_outputs(data):
+    handle_outputs(data)
+
+sio.connect(SOCKET_URL)
\ No newline at end of file
diff --git a/api/frontend/output_handler/requirements.txt b/api/frontend/output_handler/requirements.txt
new file mode 100644
index 000000000..a2217b4e9
--- /dev/null
+++ b/api/frontend/output_handler/requirements.txt
@@ -0,0 +1,2 @@
+python-engineio[client]==4.0.0
+python-socketio[client]==5.0.3
\ No newline at end of file
diff --git a/api/node/.env-template b/api/node/.env-template
new file mode 100644
index 000000000..6df26095b
--- /dev/null
+++ b/api/node/.env-template
@@ -0,0 +1,5 @@
+DATA_PATH=
+DOCKER_IMAGE_PATH=
+SOCKET_URL=
+FRONTEND_URL=
+GITHUB_REPO=https://github.com/NOAA-OWP/cahaba.git
\ No newline at end of file
diff --git a/api/node/connector/Dockerfile b/api/node/connector/Dockerfile
new file mode 100644
index 000000000..ffdb1581e
--- /dev/null
+++ b/api/node/connector/Dockerfile
@@ -0,0 +1,16 @@
+FROM docker:19.03.14-dind
+
+RUN apk add --no-cache python3 python3-dev py3-pip build-base openssl-dev libffi-dev git
+
+ENV PYTHONUNBUFFERED 1
+RUN mkdir -p /opt/connector
+WORKDIR /opt/connector
+
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt --no-cache-dir
+COPY . /opt/connector
+
+EXPOSE 6000
+
+RUN chmod +x /opt/connector/entrypoint.sh
+ENTRYPOINT ["/opt/connector/entrypoint.sh"]
diff --git a/api/node/connector/connector.py b/api/node/connector/connector.py
new file mode 100644
index 000000000..0148947c2
--- /dev/null
+++ b/api/node/connector/connector.py
@@ -0,0 +1,169 @@
+import eventlet
+eventlet.monkey_patch()
+
+import os
+import re
+import random
+import logging
+import subprocess
+from datetime import date
+
+from flask import Flask, request
+from flask_socketio import SocketIO, emit
+
+DATA_PATH = os.environ.get('DATA_PATH')
+DOCKER_IMAGE_PATH = os.environ.get('DOCKER_IMAGE_PATH')
+SOCKET_URL = os.environ.get('SOCKET_URL')
+FRONTEND_URL = os.environ.get('FRONTEND_URL')
+GITHUB_REPO = os.environ.get('GITHUB_REPO')
+
+app = Flask(__name__)
+socketio = SocketIO(app, cors_allowed_origins=[SOCKET_URL, FRONTEND_URL, "http://fim_node_connector:6000"])
+
+shared_data = {
+    'handler_sid': None,
+    'updater_sid': None
+}
+
+@app.route('/')
+def main():
+    return '<h1>Nothing to see here....</h1>'
+
+@socketio.on('connect')
+def ws_conn():
+    print('user connected!')
+    emit('is_connected', True)
+
+@socketio.on('disconnect')
+def ws_disconn():
+    print('user disconnected!')
+    emit('is_connected', False)
+
+@socketio.on('update')
+def ws_update(current_jobs):
+    emit('client_update', current_jobs, broadcast=True)
+
+@socketio.on('output_handler_connected')
+def ws_output_handler_connected():
+    print('handler_sid: ', request.sid)
+    shared_data['handler_sid'] = request.sid
+    emit('retry_saving_files', room=shared_data['updater_sid'])
+
+@socketio.on('updater_connected')
+def ws_updater_connected():
+    print('updater_sid: ', request.sid)
+    shared_data['updater_sid'] = request.sid
+    emit('retry_saving_files', room=shared_data['updater_sid'])
+
+@socketio.on('ready_for_output_handler')
+def ws_ready_for_output_handler(data):
+    nice_name = data['nice_name']
+    job_name = data['job_name']
+    path = data['path']
+
+    print(f"handler_sid: {shared_data['handler_sid']}")
+
+    if shared_data['handler_sid'] == None:
+        print("output handler not connected!")
+        emit('retry_saving_files')
+        return
+
+    # Split up path into parts for the output handler
+    path_parts = re.search(rf"/data/outputs/{job_name}/(.+)/(.+)", path)
+    directory_path = path_parts.group(1)
+    file_name = path_parts.group(2)
+
+    with open(path, "rb") as binary_file:
+        print("Sending to output handler", path)
+        
+        # Read and emit file chunk by chunk (50MB at a time)
+        chunk_index = 0
+        file_chunk = binary_file.read(52428800)
+        # file_chunk = binary_file.read(104857600)
+        while file_chunk:
+            print("Sending to output handler", path, "Chunk:", chunk_index)
+            emit('new_job_outputs', {
+                'nice_name': nice_name,
+                'job_name': job_name,
+                'directory_path': directory_path,
+                'file_name': file_name,
+                'file_chunk': file_chunk,
+                'chunk_index': chunk_index
+            }, room=shared_data['handler_sid'])
+
+            chunk_index += 1
+            file_chunk = binary_file.read(52428800)
+            # file_chunk = binary_file.read(104857600)
+
+    # Send None to indicate end of file
+    print("Sending to output handler", path, "Chunk:", chunk_index, "EOF")
+    emit('new_job_outputs', {
+        'nice_name': nice_name,
+        'job_name': job_name,
+        'directory_path': directory_path,
+        'file_name': file_name,
+        'file_chunk': None,
+        'chunk_index': chunk_index
+    }, room=shared_data['handler_sid'])
+
+@socketio.on('output_handler_finished_file')
+def ws_output_handler_finished_file(data):
+    job_name = data['job_name']
+    file_path = data['file_path']
+
+    print('done saving', job_name, file_path)
+    emit('file_saved', {
+        'job_name': job_name,
+        'file_path': f"/data/outputs/{job_name}/{file_path}"
+    }, room=shared_data['updater_sid'])
+
+@socketio.on('new_job')
+def ws_new_job(job_params):
+    validation_errors = []
+
+    # Validate Hucs Name Option
+    hucs = ' '.join(job_params['hucs'].replace(',', ' ').split())
+    invalid_hucs = re.search('[a-zA-Z]', hucs)
+    if invalid_hucs: validation_errors.append('Invalid Huc(s)')
+
+    # Validate Git Branch Option
+    branch = ''
+    branch_exists = subprocess.run(['git', 'ls-remote', '--heads', GITHUB_REPO, job_params['git_branch'].replace(' ', '_')], stdout=subprocess.PIPE).stdout.decode('utf-8')
+    if branch_exists: branch = job_params['git_branch'].replace(' ', '_')
+    else: validation_errors.append('Git Branch Does Not Exist')
+
+    # Validate Job Name Option
+    job_name = f"apijob_{job_params['job_name'].replace(' ', '_')[0:50]}_apijob_{branch}_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
+
+    # Validate Extent Option
+    extent = ''
+    if job_params['extent'] == 'FR': extent = 'FR'
+    elif job_params['extent'] == 'MS': extent = 'MS'
+    else: validation_errors.append('Invalid Extent Option')
+
+    # Validate Configuration Option
+    config_path = ''
+    if job_params['configuration'] == 'default': config_path = './foss_fim/config/params_template.env'
+    elif job_params['configuration'] == 'calibrated': config_path = './foss_fim/config/params_calibrated.env'
+    else: validation_errors.append('Invalid Configuration Option')
+    
+    # Validate Dev Run Option
+    if job_params['dev_run'] : dev_run = True
+    else: dev_run = False
+
+    if len(validation_errors) == 0:
+        # Clone github repo, with specific branch, to a temp folder
+        print(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}')
+        subprocess.call(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}', shell=True)
+
+        # TODO: instead of starting the job right away, add it to a queue until there are enough resources to run it. Also track things like huc count and huc type (6 or 8)
+
+        # Kick off the new job as a docker container with the new cloned repo as the volume
+        print(f"docker run -d --rm --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'}")
+        subprocess.call(f"docker run -d --rm --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'}", shell=True)
+        emit('job_started', 'fim_run')
+    else:
+        emit('validation_errors', validation_errors)
+
+if __name__ == '__main__':
+    socketio.run(app, host="0.0.0.0", port="6000")
diff --git a/api/node/connector/entrypoint.sh b/api/node/connector/entrypoint.sh
new file mode 100755
index 000000000..d6d853d6a
--- /dev/null
+++ b/api/node/connector/entrypoint.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+cd /opt/connector/
+echo "Starting Connector"
+python3 ./connector.py
diff --git a/api/node/connector/requirements.txt b/api/node/connector/requirements.txt
new file mode 100644
index 000000000..9d8c9d17f
--- /dev/null
+++ b/api/node/connector/requirements.txt
@@ -0,0 +1,3 @@
+flask==1.1.2
+flask-socketio==5.0.0
+eventlet==0.30.0
\ No newline at end of file
diff --git a/api/node/docker-compose-dev.yml b/api/node/docker-compose-dev.yml
new file mode 100644
index 000000000..9823afad9
--- /dev/null
+++ b/api/node/docker-compose-dev.yml
@@ -0,0 +1,38 @@
+version: '3.5'
+services:
+  fim_node_connector:
+    image: fim_node_connector
+    build:
+      context: ./connector
+    container_name: fim_node_connector
+    env_file:
+      - .env
+    restart: always
+    expose:
+      - "6000"
+    networks:
+      fim:
+        aliases: 
+          - fimnodeconnector
+    volumes:
+      - ${DATA_PATH}:/data/
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./connector/connector.py:/opt/connector/connector.py
+  fim_node_updater:
+    image: fim_node_updater
+    build:
+      context: ./updater
+    container_name: fim_node_updater
+    restart: always
+    depends_on: 
+      - fim_node_connector
+    networks:
+      - fim
+    volumes:
+      - ${DATA_PATH}:/data/
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./updater/updater.py:/opt/updater/updater.py
+networks:
+  fim:
+    name: fim
+          
\ No newline at end of file
diff --git a/api/node/docker-compose-prod.yml b/api/node/docker-compose-prod.yml
new file mode 100644
index 000000000..3e1ee654a
--- /dev/null
+++ b/api/node/docker-compose-prod.yml
@@ -0,0 +1,51 @@
+version: '3.5'
+services:
+  fim_node_connector:
+    image: fim_node_connector
+    build:
+      context: ./connector
+    container_name: fim_node_connector
+    env_file:
+      - .env
+    restart: always
+    expose:
+      - "6000"
+    networks:
+      fim:
+        aliases: 
+          - fimnodeconnector
+    volumes:
+      - ${DATA_PATH}:/data/
+      - /var/run/docker.sock:/var/run/docker.sock
+  fim_node_updater:
+    image: fim_node_updater
+    build:
+      context: ./updater
+    container_name: fim_node_updater
+    restart: always
+    depends_on: 
+      - fim_node_connector
+    networks:
+      - fim
+    volumes:
+      - ${DATA_PATH}:/data/
+      - /var/run/docker.sock:/var/run/docker.sock
+  fim_nginx:
+    image: nginx
+    container_name: fim_nginx
+    restart: always
+    depends_on:
+      - fim_node_connector
+    ports:
+      - "80:80"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    networks:
+      - fim
+    command: [nginx]
+
+networks:
+  fim:
+    name: fim
+          
+      
\ No newline at end of file
diff --git a/api/node/nginx.conf b/api/node/nginx.conf
new file mode 100644
index 000000000..c2fc935bd
--- /dev/null
+++ b/api/node/nginx.conf
@@ -0,0 +1,64 @@
+user nginx;
+worker_processes 1;
+pid /var/run/nginx.pid;
+daemon off;
+
+events {
+  worker_connections 512;
+  # multi_accept on;
+}
+
+http {
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  proxy_connect_timeout 300;
+  proxy_send_timeout 300;
+  proxy_read_timeout 90m;
+  send_timeout 300;
+  keepalive_timeout 65;
+  types_hash_max_size 2048;
+
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+
+  log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                             '"$request" $status $body_bytes_sent '
+                             '"$http_referer" "$http_user_agent"'
+                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+
+  access_log /var/log/nginx/access.log upstream_time;
+  error_log /var/log/nginx/error.log warn;
+
+  server {
+    listen 80;
+    client_header_buffer_size 64k;
+    large_client_header_buffers 4 64k;
+    server_name _;
+    root /var/www/;
+
+    gzip on;
+    gzip_types application/json;
+    proxy_http_version 1.1;
+
+    location /stats/nginx {
+      stub_status on;
+    }
+
+    location / {
+      proxy_pass http://fim_node_connector:6000/;
+      proxy_set_header Host            $host;
+      proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    location /socket.io {
+      proxy_http_version 1.1;
+      proxy_buffering off;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "Upgrade";
+      proxy_pass http://fim_node_connector:6000/socket.io;
+      proxy_set_header Host            $host;
+      proxy_set_header X-Forwarded-Proto $scheme;
+    }
+  }
+}
\ No newline at end of file
diff --git a/api/node/updater/Dockerfile b/api/node/updater/Dockerfile
new file mode 100644
index 000000000..e179f3a42
--- /dev/null
+++ b/api/node/updater/Dockerfile
@@ -0,0 +1,14 @@
+FROM docker:19.03.14-dind
+
+RUN apk add --no-cache python3 python3-dev py3-pip
+
+ENV PYTHONUNBUFFERED 1
+RUN mkdir -p /opt/updater
+WORKDIR /opt/updater
+
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt --no-cache-dir
+COPY . /opt/updater
+
+RUN chmod +x /opt/updater/entrypoint.sh
+ENTRYPOINT ["/opt/updater/entrypoint.sh"]
diff --git a/api/node/updater/entrypoint.sh b/api/node/updater/entrypoint.sh
new file mode 100755
index 000000000..0f74f2541
--- /dev/null
+++ b/api/node/updater/entrypoint.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+cd /opt/updater/
+echo "Starting Update Loop"
+python3 ./updater.py
diff --git a/api/node/updater/requirements.txt b/api/node/updater/requirements.txt
new file mode 100644
index 000000000..a2217b4e9
--- /dev/null
+++ b/api/node/updater/requirements.txt
@@ -0,0 +1,2 @@
+python-engineio[client]==4.0.0
+python-socketio[client]==5.0.3
\ No newline at end of file
diff --git a/api/node/updater/updater.py b/api/node/updater/updater.py
new file mode 100644
index 000000000..7bc65836c
--- /dev/null
+++ b/api/node/updater/updater.py
@@ -0,0 +1,147 @@
+import os
+import re
+import math
+import time
+import json
+import shutil
+import logging
+
+import socketio
+
+DATA_PATH = os.environ.get('DATA_PATH')
+
+connected = False
+shared_data = {
+    'connected': False
+}
+
+current_jobs = {}
+if os.path.exists('/data/outputs/current_jobs.json'):
+    with open('/data/outputs/current_jobs.json') as f:
+        current_jobs = json.load(f)
+
+# Get all the current running jobs from the list of docker containers, store that data in a dictionary
+# along with any other needed metadata (like if it's still running, doing post processing, copying outputs
+# to its destination, etc), and then update the websocket server of the status of the jobs.
+def update_loop():
+    while True:
+        # Get list of current docker containers that are fim run jobs
+        job_names = os.popen("docker container ls --filter=name=apijob --format '{{.Names}}'").read().splitlines()
+        for job_name in job_names:
+            if job_name not in current_jobs:
+                # If it's a new job, add it to the dictionary
+                current_jobs[job_name] = {
+                    'job_name': job_name,
+                    'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
+                    'status': 'In Progress',
+                    'time_started': time.time(),
+                    'time_elapsed': 0,
+                    'output_files_saved': {}
+                }
+
+        jobs_to_delete = []
+        for job_name in current_jobs.keys():
+            # Update the time elapsed for all jobs that are currently in progress or saving outputs
+            if current_jobs[job_name]['status'] == 'In Progress' or current_jobs[job_name]['status'] == 'Ready to Save File'\
+                or current_jobs[job_name]['status'] == 'Saving File':
+                current_jobs[job_name]['time_elapsed'] = math.ceil(time.time() - current_jobs[job_name]['time_started'])
+
+            # TODO: While job is in progress, keep track of how many hucs are done and overall progress %
+
+            # Once the Docker container is done, set the job as ready to save output
+            if current_jobs[job_name]['status'] == 'In Progress' and job_name not in job_names:
+                for path, folders, files in os.walk(f"/data/outputs/{job_name}"):
+                    for file in files:
+                        current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = False
+
+                current_jobs[job_name]['status'] = 'Ready to Save File'
+                # TODO: Possible check the completed job's log for its exit code
+    
+            # Trigger connector to transmit the outputs to the output_handler
+            # If the output_handler is offline, it will keep retrying until the output_handler is online
+            if current_jobs[job_name]['status'] == 'Ready to Save File':
+                print(f"{job_name} ready for output handler")
+                outputs_to_save = []
+                for path in current_jobs[job_name]['output_files_saved']:
+                    if current_jobs[job_name]['output_files_saved'][path] == False:
+                        outputs_to_save.append(path)
+
+                if len(outputs_to_save) > 0:
+                    if shared_data['connected']: 
+                        sio.emit('ready_for_output_handler', {
+                            'nice_name': current_jobs[job_name]['nice_name'],
+                            'job_name': job_name,
+                            'path': outputs_to_save[0]
+                        })
+                current_jobs[job_name]['status'] = 'Saving File'
+            
+            # Once the output_handler is done getting the outputs and the connector deletes the temp repo source,
+            # mark as completed
+            if current_jobs[job_name]['status'] == 'Saving File':
+                is_done = True
+                for path in current_jobs[job_name]['output_files_saved']:
+                    if current_jobs[job_name]['output_files_saved'][path] == False:
+                        is_done = False
+                        break
+
+                if is_done:
+                    print("output_handler finished, deleted temp source files and output files")
+                    temp_path = f"/data/temp/{job_name}"
+                    if os.path.isdir(temp_path):
+                        shutil.rmtree(temp_path)
+
+                    outputs_path = f"/data/outputs/{job_name}"
+                    if os.path.isdir(outputs_path):
+                        shutil.rmtree(outputs_path)
+
+                    current_jobs[job_name]['status'] = 'Completed'
+                    print(f"{job_name} completed")
+                    # TODO: Insert Slack notification here for finished job
+
+            # Remove job from list after it's been completed for more than 15 minutes
+            if current_jobs[job_name]['status'] == 'Completed' and \
+                time.time() >= current_jobs[job_name]['time_started'] + current_jobs[job_name]['time_elapsed'] + 900:
+                print(f"{job_name} removed from job list")
+                jobs_to_delete.append(job_name)
+
+        for job in jobs_to_delete:
+            del current_jobs[job]
+
+        # Send updates to the connector and write job progress to file
+        if shared_data['connected']: sio.emit('update', current_jobs)
+        with open('/data/outputs/current_jobs.json', 'w') as f:
+            json.dump(current_jobs, f)
+
+        time.sleep(1)
+
+sio = socketio.Client()
+
+@sio.event
+def connect():
+    print("Update Loop Connected!")
+    sio.emit('updater_connected')
+    shared_data['connected'] = True
+
+@sio.event
+def disconnect():
+    print('disconnected from server')
+    shared_data['connected'] = False
+
+# If the output_handler is offline, try the saving process again
+@sio.on('retry_saving_files')
+def ws_retry_saving_files():
+    print('saving files failed, retrying')
+    for job_name in current_jobs:
+        if current_jobs[job_name]['status'] == "Saving File":
+            current_jobs[job_name]['status'] = 'Ready to Save File'
+
+@sio.on('file_saved')
+def ws_file_saved(data):
+    job_name = data['job_name']
+    file_path = data['file_path']
+
+    current_jobs[job_name]['output_files_saved'][file_path] = True
+    current_jobs[job_name]['status'] = 'Ready to Save File'
+
+sio.connect('http://fim_node_connector:6000/')
+update_loop()
\ No newline at end of file

From 2ded12aa2ada7da5599f21193fa6c31f694fbfdf Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 28 Jan 2021 10:22:57 -0600
Subject: [PATCH 010/359] Updated point of contact on README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 69a815b6d..5b2e3c781 100644
--- a/README.md
+++ b/README.md
@@ -97,7 +97,7 @@ Please see the issue tracker on GitHub for known issues and for getting help.
 
 ## Getting Involved
 
-NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Fernando Aristizabal (fernando.aristizabal@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
+NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
 ----
 

From dda9e22e927816dc359e70382543c249314de462 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 2 Feb 2021 13:12:47 -0600
Subject: [PATCH 011/359] Updated README.md credits

Updated README.md with Esri Arc Hydro credit to acknowledge the knowledge gained and discoveries made through collaboration with Esri.
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 5b2e3c781..47d3561ab 100644
--- a/README.md
+++ b/README.md
@@ -118,3 +118,4 @@ NOAA's National Water Center welcomes anyone to contribute to the Cahaba reposit
 8. Federal Emergency Management Agency (FEMA) Base Level Engineering [(BLE)](https://webapps.usgs.gov/infrm/estBFE/)
 9. Verdin, James; Verdin, Kristine; Mathis, Melissa; Magadzire, Tamuka; Kabuchanga, Eric; Woodbury, Mark; and Gadain, Hussein, 2016, A software tool for rapid flood inundation mapping: U.S. Geological Survey Open-File Report 2016–1038, 26 p., http://dx.doi.org/10.3133/ofr20161038.
 10. United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR). https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution
+11. Esri Arc Hydro. https://www.esri.com/library/fliers/pdfs/archydro.pdf

From d657b985b4442e31ba8a4f730a59774f2f0717e7 Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Thu, 4 Feb 2021 09:50:31 -0600
Subject: [PATCH 012/359] Post-process to aggregate FIM outputs to HUC6 scale

Additions

* viz outputs aggregated to HUC6 scale; saves outputs to aggregate_fim_outputs folder

Changes

* split_flows.py now splits streams at HUC8 boundaries to ensure consistent catchment boundaries along edges
* aggregate_fim_outputs.sh has been depreciated but remains in the repo for potential FIM 4 development
* replaced geopandas driver arg with getDriver throughout repo
* organized parameters in environment files by group
* cleaned up variable names in split_flows.py and build_stream_traversal.py
* build_stream_traversal.py is now assigning HydroID by midpoint instead centroid
* cleanup of clip_vectors_to_wbd.py

resolves #161 and #4
---
 CHANGELOG.md                                  |  18 +++
 config/params_calibrated.env                  |  11 +-
 config/params_template.env                    |  11 +-
 fim_run.sh                                    |   7 +-
 lib/acquire_and_preprocess_inputs.py          |   7 +-
 lib/add_crosswalk.py                          |   5 +-
 lib/aggregate_fim_outputs.py                  | 106 ++++++++++++++++++
 lib/aggregate_vector_inputs.py                |  17 +--
 ...traversal.py => build_stream_traversal.py} |  20 +++-
 lib/clip_vectors_to_wbd.py                    |  20 ++--
 lib/derive_headwaters.py                      |   7 +-
 lib/reachID_grid_to_vector_points.py          |   4 +-
 lib/run_by_unit.sh                            |  10 +-
 lib/split_flows.py                            |  36 +++---
 14 files changed, 215 insertions(+), 64 deletions(-)
 create mode 100644 lib/aggregate_fim_outputs.py
 rename lib/{buildstreamtraversal.py => build_stream_traversal.py} (90%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eed3cd415..fd90d4713 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.3.0 - 2021-02-04 - [PR #227](https://github.com/NOAA-OWP/cahaba/pull/227)
+
+Post-process to aggregate FIM outputs to HUC6 scale.
+
+### Additions
+
+ - Viz outputs aggregated to HUC6 scale; saves outputs to `aggregate_fim_outputs` folder.
+ 
+### Changes
+
+ - `split_flows.py` now splits streams at HUC8 boundaries to ensure consistent catchment boundaries along edges.
+ - `aggregate_fim_outputs.sh` has been depreciated but remains in the repo for potential FIM 4 development.
+ - Replaced geopandas driver arg with getDriver throughout repo.
+ - Organized parameters in environment files by group.
+ - Cleaned up variable names in `split_flows.py` and `build_stream_traversal.py`.
+ - `build_stream_traversal.py` is now assigning HydroID by midpoint instead centroid.
+ - Cleanup of `clip_vectors_to_wbd.py`.
+ 
 ## v3.0.2.0 - 2021-01-25 - [PR #218](https://github.com/NOAA-OWP/cahaba/pull/218)
 
 Addition of an API service to schedule, run and manage `fim_run` jobs through a user-friendly web interface.
diff --git a/config/params_calibrated.env b/config/params_calibrated.env
index f150d0428..ff0da17b0 100644
--- a/config/params_calibrated.env
+++ b/config/params_calibrated.env
@@ -1,16 +1,21 @@
 #!/bin/bash
 
-#### geospatial parameters ####
+#### hydroconditioning parameters ####
 export negative_burn_value=1000
 export agree_DEM_buffer=70
+export wbd_buffer=5000
+
+#### geospatial parameters ####
 export max_split_distance_meters=1500
+export ms_buffer_dist=7000
+export lakes_buffer_dist_meters=20
+
+#### rating curve parameters ####
 export manning_n="/foss_fim/config/mannings_calibrated.json"
 export stage_min_meters=0
 export stage_interval_meters=0.3048
 export stage_max_meters=25
 export slope_min=0.001
-export ms_buffer_dist=7000
-export lakes_buffer_dist_meters=20
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/config/params_template.env b/config/params_template.env
index 04af26828..b250de4bf 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -1,16 +1,21 @@
 #!/bin/bash
 
-#### geospatial parameters ####
+#### hydroconditioning parameters ####
 export negative_burn_value=1000
 export agree_DEM_buffer=70
+export wbd_buffer=5000
+
+#### geospatial parameters ####
 export max_split_distance_meters=1500
+export ms_buffer_dist=7000
+export lakes_buffer_dist_meters=20
+
+#### rating curve parameters ####
 export manning_n="/foss_fim/config/mannings_default.json"
 export stage_min_meters=0
 export stage_interval_meters=0.3048
 export stage_max_meters=25
 export slope_min=0.001
-export ms_buffer_dist=7000
-export lakes_buffer_dist_meters=20
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/fim_run.sh b/fim_run.sh
index a8763da60..b2efa0ec5 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -147,5 +147,8 @@ else
     fi
 fi
 
-# aggregate outputs
-bash /foss_fim/lib/aggregate_fim_outputs.sh $outputRunDataDir
+echo "$viz"
+if [[ "$viz" -eq 1 ]]; then
+    # aggregate outputs
+    python3 /foss_fim/lib/aggregate_fim_outputs.py -d $outputRunDataDir
+fi
diff --git a/lib/acquire_and_preprocess_inputs.py b/lib/acquire_and_preprocess_inputs.py
index 01270f235..bd1eda194 100755
--- a/lib/acquire_and_preprocess_inputs.py
+++ b/lib/acquire_and_preprocess_inputs.py
@@ -23,7 +23,7 @@
                                     OVERWRITE_NHD,
                                     OVERWRITE_ALL)
 
-from utils.shared_functions import pull_file, run_system_command, subset_wbd_gpkg, delete_file
+from utils.shared_functions import pull_file, run_system_command, subset_wbd_gpkg, delete_file, getDriver
 
 NHDPLUS_VECTORS_DIRNAME = 'nhdplus_vectors'
 NHDPLUS_RASTERS_DIRNAME = 'nhdplus_rasters'
@@ -82,10 +82,9 @@ def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_
         fossids = [str(item).zfill(4) for item in list(range(1, 1 + len(wbd_hu8)))]
         wbd_hu8[FOSS_ID] = fossids
         wbd_hu8 = wbd_hu8.to_crs(PREP_PROJECTION)  # Project.
-        #wbd_hu8.to_file(os.path.join(wbd_directory, 'WBDHU8.gpkg'), driver='GPKG')  # Save.
         wbd_hu8 = subset_wbd_to_nwm_domain(wbd_hu8,nwm_file_to_use)
         wbd_hu8.geometry = wbd_hu8.buffer(0)
-        wbd_hu8.to_file(multilayer_wbd_geopackage, driver='GPKG',layer='WBDHU8')  # Save.
+        wbd_hu8.to_file(multilayer_wbd_geopackage,layer='WBDHU8',driver=getDriver(multilayer_wbd_geopackage),index=False)  # Save.
         wbd_hu8.HUC8.to_csv(nwm_huc_list_file_template.format('8'),index=False,header=False)
         #wbd_gpkg_list.append(os.path.join(wbd_directory, 'WBDHU8.gpkg'))  # Append to wbd_gpkg_list for subsetting later.
         del wbd_hu8
@@ -99,7 +98,7 @@ def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_
             wbd = wbd.rename(columns={'huc'+wbd_layer_num : 'HUC' + wbd_layer_num})
             wbd = subset_wbd_to_nwm_domain(wbd,nwm_file_to_use)
             wbd.geometry = wbd.buffer(0)
-            wbd.to_file(multilayer_wbd_geopackage,driver="GPKG",layer=wbd_layer)
+            wbd.to_file(multilayer_wbd_geopackage,layer=wbd_layer,driver=getDriver(multilayer_wbd_geopackage),index=False)
             wbd['HUC{}'.format(wbd_layer_num)].to_csv(nwm_huc_list_file_template.format(wbd_layer_num),index=False,header=False)
             #output_gpkg = os.path.join(wbd_directory, wbd_layer + '.gpkg')
             #wbd_gpkg_list.append(output_gpkg)
diff --git a/lib/add_crosswalk.py b/lib/add_crosswalk.py
index 1f51efb4d..c5ed0dd5a 100755
--- a/lib/add_crosswalk.py
+++ b/lib/add_crosswalk.py
@@ -7,6 +7,7 @@
 import json
 import argparse
 import sys
+from utils.shared_functions import getDriver
 
 def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,calibration_mode=False):
 
@@ -163,8 +164,8 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
             output_src_json[str(hid)] = { 'q_list' : q_list , 'stage_list' : stage_list }
 
         # write out
-        output_catchments.to_file(output_catchments_fileName, driver="GPKG",index=False)
-        output_flows.to_file(output_flows_fileName, driver="GPKG", index=False)
+        output_catchments.to_file(output_catchments_fileName,driver=getDriver(output_catchments_fileName),index=False)
+        output_flows.to_file(output_flows_fileName,driver=getDriver(output_flows_fileName),index=False)
         output_src.to_csv(output_src_fileName,index=False)
         output_crosswalk.to_csv(output_crosswalk_fileName,index=False)
         output_hydro_table.to_csv(output_hydro_table_fileName,index=False)
diff --git a/lib/aggregate_fim_outputs.py b/lib/aggregate_fim_outputs.py
new file mode 100644
index 000000000..1296fb8cc
--- /dev/null
+++ b/lib/aggregate_fim_outputs.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import pandas as pd
+import json
+import rasterio
+from rasterio.merge import merge
+from utils.shared_variables import PREP_PROJECTION
+
+
+def aggregate_fim_outputs(fim_out_dir):
+
+    print ("aggregating outputs to HUC6 scale")
+    drop_folders = ['logs']
+    huc_list = [huc for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
+    huc6_list = [str(huc[0:6]) for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
+    huc6_list = list(set(huc6_list))
+
+    for huc in huc_list:
+
+        os.makedirs(os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6])), exist_ok=True)
+
+        # merge hydrotable
+        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),'hydroTable.csv')
+        hydrotable = pd.read_csv(os.path.join(fim_out_dir,huc,'hydroTable.csv'))
+
+        # write out hydrotable
+        if os.path.isfile(aggregate_hydrotable):
+            hydrotable.to_csv(aggregate_hydrotable,index=False, mode='a',header=False)
+        else:
+            hydrotable.to_csv(aggregate_hydrotable,index=False)
+
+        del hydrotable
+
+        # merge src
+        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),'src.json')
+        src = open(os.path.join(fim_out_dir,huc,'src.json'))
+        src = json.load(src)
+
+        # write out src
+        if os.path.isfile(aggregate_src):
+            with open(aggregate_src, 'a') as outfile:
+                    json.dump(src, outfile)
+        else:
+            with open(aggregate_src, 'w') as outfile:
+                    json.dump(src, outfile)
+
+        del src
+
+    for huc6 in huc6_list:
+        huc6_dir = os.path.join(fim_out_dir,'aggregate_fim_outputs',huc6)
+
+        huc6_filter = [path.startswith(huc6) for path in huc_list]
+        subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
+
+        # aggregate and mosaic rem
+        rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
+
+        rem_files_to_mosaic = []
+
+        for rem in rem_list:
+            rem_src = rasterio.open(rem)
+            rem_files_to_mosaic.append(rem_src)
+
+        mosaic, out_trans = merge(rem_files_to_mosaic)
+        out_meta = rem_src.meta.copy()
+        out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION})
+
+        rem_mosaic = os.path.join(huc6_dir,'rem_zeroed_masked.tif')
+        with rasterio.open(rem_mosaic, "w", **out_meta) as dest:
+            dest.write(mosaic)
+
+        del rem_files_to_mosaic,rem_src,out_meta,mosaic
+
+        # aggregate and mosaic catchments
+        catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
+
+        cat_files_to_mosaic = []
+
+        for cat in catchment_list:
+            cat_src = rasterio.open(cat)
+            cat_files_to_mosaic.append(cat_src)
+
+        mosaic, out_trans = merge(cat_files_to_mosaic)
+        out_meta = cat_src.meta.copy()
+        out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION})
+
+        catchment_mosaic = os.path.join(huc6_dir,'gw_catchments_reaches_filtered_addedAttributes.tif')
+        with rasterio.open(catchment_mosaic, "w", **out_meta) as dest:
+            dest.write(mosaic)
+
+        del cat_files_to_mosaic,cat_src,out_meta,mosaic
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Aggregate layers buy HUC6')
+    parser.add_argument('-d','--fim-outputs-directory', help='FIM outputs directory', required=True)
+
+
+    args = vars(parser.parse_args())
+
+    fim_outputs_directory = args['fim_outputs_directory']
+
+    aggregate_fim_outputs(fim_outputs_directory)
diff --git a/lib/aggregate_vector_inputs.py b/lib/aggregate_vector_inputs.py
index d60cdc2f0..eb4d3e4f5 100755
--- a/lib/aggregate_vector_inputs.py
+++ b/lib/aggregate_vector_inputs.py
@@ -76,7 +76,7 @@ def subset_nwm_ms_streams(args):
 
     nwm_streams.reset_index(drop=True,inplace=True)
 
-    nwm_streams.to_file(output_filename,getDriver(output_filename),index=False)
+    nwm_streams.to_file(output_filename,driver=getDriver(output_filename),index=False)
 
 def find_nwm_incoming_streams(args):
 
@@ -142,7 +142,8 @@ def find_nwm_incoming_streams(args):
 
     huc_intersection = gpd.GeoDataFrame({'geometry' : intersecting_points},crs=nwm_streams.crs,geometry='geometry')
     huc_intersection = huc_intersection.drop_duplicates()
-    huc_intersection.to_file(output_filename,getDriver(output_filename))
+    huc_intersection.to_file(output_filename,driver=getDriver(output_filename))
+
 
 def collect_stream_attributes(args, huc):
     print ('Starting huc: ' + str(huc))
@@ -230,8 +231,8 @@ def subset_stream_networks(args, huc):
             adj_nhd_headwaters_fr_fileName=os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_fr.gpkg')
 
             # write out FR adjusted
-            adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,getDriver(nhd_streams_fr_adjusted_fileName),index=False)
-            adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,getDriver(adj_nhd_headwaters_fr_fileName),index=False)
+            adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,driver=getDriver(nhd_streams_fr_adjusted_fileName),index=False)
+            adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,driver=getDriver(adj_nhd_headwaters_fr_fileName),index=False)
 
             del adj_nhd_streams_fr, adj_nhd_headwater_points_fr
         else:
@@ -253,8 +254,8 @@ def subset_stream_networks(args, huc):
             adj_nhd_headwaters_ms_fileName=os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_ms.gpkg')
 
             # write out MS adjusted
-            adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,getDriver(nhd_streams_ms_adjusted_fileName),index=False)
-            adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,getDriver(adj_nhd_headwaters_ms_fileName),index=False)
+            adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,driver=getDriver(nhd_streams_ms_adjusted_fileName),index=False)
+            adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,driver=getDriver(adj_nhd_headwaters_ms_fileName),index=False)
 
             del adj_nhd_streams_ms, adj_nhd_headwater_points_ms
 
@@ -322,7 +323,6 @@ def aggregate_stream_networks(in_dir,agg_dir, huc_list):
 
             del adj_nhd_headwater_points_ms
 
-
 def clean_up_intermediate_files(in_dir):
 
     for huc in os.listdir(in_dir):
@@ -371,7 +371,8 @@ def clean_up_intermediate_files(in_dir):
     # print ('deriving nwm headwater points')
     # nwm_headwaters = findHeadWaterPoints(nwm_streams_fr_filename)
     # nwm_headwaters['ID'] = nwm_headwaters.index + 1
-    # nwm_headwaters.to_file(nwm_headwaters_filename,getDriver(nwm_headwaters_filename),index=False)
+    # nwm_headwaters.to_file(nwm_headwaters_filename,driver=getDriver(nwm_headwaters_filename),index=False)
+    
     # del nwm_headwaters, nwm_streams
 
     ## subset NWM MS Streams
diff --git a/lib/buildstreamtraversal.py b/lib/build_stream_traversal.py
similarity index 90%
rename from lib/buildstreamtraversal.py
rename to lib/build_stream_traversal.py
index f7b2bcbe4..a30fe04ed 100644
--- a/lib/buildstreamtraversal.py
+++ b/lib/build_stream_traversal.py
@@ -26,7 +26,7 @@ def trace():
 to_node = "To_Node"
 next_down_id = "NextDownID"
 
-class BuildStreamTraversalColumns(object):
+class build_stream_traversal_columns(object):
     '''Tool class for updating the next down IDs of stream features.'''
     def __init__(self):
         '''Define tool properties (tool name is the class name).'''
@@ -42,13 +42,21 @@ def execute(self, streams, wbd8, hydro_id):
             if not hydro_id in streams.columns:
                 print ("Required field " + hydro_id + " does not exist in input. Generating..")
 
-                stream_centroid = gpd.GeoDataFrame({'geometry':streams.geometry.centroid}, crs=streams.crs, geometry='geometry')
-                stream_wbdjoin = gpd.sjoin(stream_centroid, wbd8, how='left', op='within')
-                stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "centroid", "index_right": "HUC8id"})
-                streams = streams.join(stream_wbdjoin).drop(columns=['centroid'])
+                # Get stream midpoint
+                stream_midpoint = []
+                for i,lineString in enumerate(streams.geometry):
+                    stream_midpoint = stream_midpoint + [lineString.interpolate(0.05,normalized=True)]
+
+                stream_md_gpd = gpd.GeoDataFrame({'geometry':stream_midpoint}, crs=streams.crs, geometry='geometry')
+                stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', op='within')
+                stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "midpoint", "index_right": "HUC8id"})
+                streams = streams.join(stream_wbdjoin).drop(columns=['midpoint'])
 
                 streams['seqID'] = (streams.groupby('HUC8id').cumcount(ascending=True)+1).astype('str').str.zfill(4)
                 streams = streams.loc[streams['HUC8id'].notna(),:]
+                if streams.HUC8id.dtype != 'str': streams.HUC8id = streams.HUC8id.astype(str)
+                if streams.seqID.dtype != 'str': streams.seqID = streams.seqID.astype(str)
+
                 streams = streams.assign(hydro_id= lambda x: x.HUC8id + x.seqID)
                 streams = streams.rename(columns={"hydro_id": hydro_id}).sort_values(hydro_id)
                 streams = streams.drop(columns=['HUC8id', 'seqID'])
@@ -170,7 +178,7 @@ def execute(self, streams, wbd8, hydro_id):
         wbd8           = args.parameters[1]
         hydro_id        = args.parameters[2]
 
-        oProcessor = BuildStreamTraversalColumns()
+        oProcessor = build_stream_traversal_columns()
         params = (streams, wbd8, hydro_id)
         tResults=None
         tResults = oProcessor.execute(params)
diff --git a/lib/clip_vectors_to_wbd.py b/lib/clip_vectors_to_wbd.py
index 92ece11f8..654fe6e4f 100755
--- a/lib/clip_vectors_to_wbd.py
+++ b/lib/clip_vectors_to_wbd.py
@@ -5,8 +5,9 @@
 import argparse
 from os.path import splitext
 from shapely.geometry import MultiPolygon,Polygon,Point
+from utils.shared_functions import getDriver
 
-def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks=False,extent='FR'):
+def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks=False):
 
     hucUnitLength = len(str(hucCode))
 
@@ -75,7 +76,11 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
 
         # nhd_streams = nhd_streams.loc[~nhd_streams.NHDPlusID.isin(NHDPlusIDs)]
 
-    nhd_streams.to_file(subset_nhd_streams_filename,driver=getDriver(subset_nhd_streams_filename),index=False)
+    if len(nhd_streams) > 0:
+        nhd_streams.to_file(subset_nhd_streams_filename,driver=getDriver(subset_nhd_streams_filename),index=False)
+    else:
+        print ("No NHD streams within HUC " + str(hucCode) +  " boundaries.")
+        sys.exit(0)
 
     if len(nhd_headwaters) > 0:
         nhd_headwaters.to_file(subset_nhd_headwaters_filename,driver=getDriver(subset_nhd_headwaters_filename),index=False)
@@ -90,13 +95,6 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     nwm_streams.to_file(subset_nwm_streams_filename,driver=getDriver(subset_nwm_streams_filename),index=False)
     del nwm_streams
 
-def getDriver(filename):
-
-    driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
-    driver = driverDictionary[splitext(filename)[1]]
-
-    return(driver)
-
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Subset vector layers')
@@ -118,7 +116,6 @@ def getDriver(filename):
     parser.add_argument('-b','--subset-nwm-streams',help='NWM streams subset',required=True)
     parser.add_argument('-x','--subset-landsea',help='LandSea subset',required=True)
     parser.add_argument('-o','--dissolve-links',help='remove multi-line strings',action="store_true",default=False)
-    parser.add_argument('-p','--extent',help='MS or FR extent',required=True)
 
     args = vars(parser.parse_args())
 
@@ -140,6 +137,5 @@ def getDriver(filename):
     subset_nwm_streams_filename = args['subset_nwm_streams']
     subset_landsea_filename = args['subset_landsea']
     dissolveLinks = args['dissolve_links']
-    extent = args['extent']
 
-    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks,extent)
+    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks)
diff --git a/lib/derive_headwaters.py b/lib/derive_headwaters.py
index d29381c60..9fb7c1020 100644
--- a/lib/derive_headwaters.py
+++ b/lib/derive_headwaters.py
@@ -2,6 +2,7 @@
 
 import geopandas as gpd
 from shapely.geometry import Point
+from utils.shared_functions import getDriver
 
 
 def findHeadWaterPoints(flows):
@@ -32,12 +33,6 @@ def findHeadWaterPoints(flows):
 
     return(hw_gdf)
 
-def getDriver(fileName):
-
-    driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
-    driver = driverDictionary[splitext(fileName)[1]]
-
-    return(driver)
 
 if __name__ == '__main__':
 
diff --git a/lib/reachID_grid_to_vector_points.py b/lib/reachID_grid_to_vector_points.py
index c4a54b2ec..5dadd43c1 100755
--- a/lib/reachID_grid_to_vector_points.py
+++ b/lib/reachID_grid_to_vector_points.py
@@ -12,6 +12,7 @@
 import geopandas as gpd
 from shapely.geometry import Point
 from raster import Raster
+from utils.shared_functions import getDriver
 
 """
 USAGE:
@@ -78,8 +79,7 @@
     i += 1
 
 pointGDF = gpd.GeoDataFrame({'id' : id, 'geometry' : points},crs=boolean.proj,geometry='geometry')
-pointGDF.to_file(outputFileName,driver='GPKG',index=False)
+pointGDF.to_file(outputFileName,driver=getDriver(outputFileName),index=False)
 
 print("Complete")
 #shapeData.Destroy()
-
diff --git a/lib/run_by_unit.sh b/lib/run_by_unit.sh
index d3733dd6f..7715304e3 100755
--- a/lib/run_by_unit.sh
+++ b/lib/run_by_unit.sh
@@ -5,6 +5,10 @@ T_total_start
 
 echo -e $startDiv"Parameter Values"
 echo -e "extent=$extent"
+echo -e "agree_DEM_buffer=$agree_DEM_buffer"
+echo -e "wbd_buffer=$wbd_buffer"
+echo -e "ms_buffer_dist=$ms_buffer_dist"
+echo -e "lakes_buffer_dist_meters=$lakes_buffer_dist_meters"
 echo -e "negative_burn_value=$negative_burn_value"
 echo -e "max_split_distance_meters=$max_split_distance_meters"
 echo -e "mannings_n=$manning_n"
@@ -30,6 +34,7 @@ huc2Identifier=${hucNumber:0:2}
 input_NHD_WBHD_layer=WBDHU$hucUnitLength
 input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_cm.tif
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
+
 # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
 if [[ $huc2Identifier == "04" ]] ; then
   input_LANDSEA=$inputDataDir/landsea/gl_water_polygons.gpkg
@@ -38,6 +43,7 @@ else
   input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
 fi
 
+# Define streams and headwaters based on extent #
 if [ "$extent" = "MS" ]; then
   input_nhd_flowlines=$input_nhd_flowlines_ms
   input_nhd_headwaters=$input_nhd_headwaters_ms
@@ -59,7 +65,7 @@ echo -e $startDiv"Buffer WBD $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/wbd_buffered.gpkg ] && \
-ogr2ogr -f GPKG -dialect sqlite -sql "select ST_buffer(geom, 5000) from 'WBDHU$hucUnitLength'" $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd.gpkg
+ogr2ogr -f GPKG -dialect sqlite -sql "select ST_buffer(geom, $wbd_buffer) from 'WBDHU$hucUnitLength'" $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd.gpkg
 Tcount
 
 ## Subset Vector Layers ##
@@ -67,7 +73,7 @@ echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-$libDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_NWM_Flows -s $input_nhd_flowlines -l $input_NWM_Lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_NWM_Catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -p $extent
+$libDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_NWM_Flows -s $input_nhd_flowlines -l $input_NWM_Lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_NWM_Catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg
 Tcount
 
 if [ "$extent" = "MS" ]; then
diff --git a/lib/split_flows.py b/lib/split_flows.py
index db11d6d4a..c8de19b34 100755
--- a/lib/split_flows.py
+++ b/lib/split_flows.py
@@ -21,7 +21,8 @@
 from os.path import isfile
 from os import remove
 from collections import OrderedDict
-import buildstreamtraversal
+import build_stream_traversal
+from utils.shared_functions import getDriver
 
 flows_fileName         = sys.argv[1]
 dem_fileName           = sys.argv[2]
@@ -29,10 +30,12 @@
 split_points_fileName  = sys.argv[4]
 maxLength              = float(sys.argv[5])
 slope_min              = float(sys.argv[6])
-huc8_filename          = sys.argv[7]
+wbd8_clp_filename      = sys.argv[7]
 lakes_filename         = sys.argv[8]
 lakes_buffer_input     = float(sys.argv[9])
 
+wbd = gpd.read_file(wbd8_clp_filename)
+
 toMetersConversion = 1e-3
 
 print('Loading data ...')
@@ -42,7 +45,7 @@
     print ("No relevant streams within HUC boundaries.")
     sys.exit(0)
 
-WBD8 = gpd.read_file(huc8_filename)
+wbd8 = gpd.read_file(wbd8_clp_filename)
 #dem = Raster(dem_fileName)
 dem = rasterio.open(dem_fileName,'r')
 if isfile(lakes_filename):
@@ -50,17 +53,21 @@
 else:
     lakes = None
 
-WBD8 = WBD8.filter(items=['fossid', 'geometry'])
-WBD8 = WBD8.set_index('fossid')
+wbd8 = wbd8.filter(items=['fossid', 'geometry'])
+wbd8 = wbd8.set_index('fossid')
 flows = flows.explode()
 
 # temp
-flows = flows.to_crs(WBD8.crs)
+flows = flows.to_crs(wbd8.crs)
 
 split_flows = []
 slopes = []
-HYDROID = 'HydroID'
-split_endpoints = OrderedDict()
+hydro_id = 'HydroID'
+
+# split at HUC8 boundaries
+print ('splitting stream segments at HUC8 boundaries')
+flows = gpd.overlay(flows, wbd8, how='union').explode().reset_index(drop=True)
+
 # check for lake features
 if lakes is not None:
     if len(lakes) > 0:
@@ -165,9 +172,9 @@
 split_flows_gdf = split_flows_gdf.drop_duplicates()
 
 # Create Ids and Network Traversal Columns
-addattributes = buildstreamtraversal.BuildStreamTraversalColumns()
+addattributes = build_stream_traversal.build_stream_traversal_columns()
 tResults=None
-tResults = addattributes.execute(split_flows_gdf, WBD8, HYDROID)
+tResults = addattributes.execute(split_flows_gdf, wbd8, hydro_id)
 if tResults[0] == 'OK':
     split_flows_gdf = tResults[1]
 else:
@@ -176,14 +183,14 @@
 # Get Outlet Point Only
 #outlet = OrderedDict()
 #for i,segment in split_flows_gdf.iterrows():
-#    outlet[segment.geometry.coords[-1]] = segment[HYDROID]
+#    outlet[segment.geometry.coords[-1]] = segment[hydro_id]
 
 #hydroIDs_points = [hidp for hidp in outlet.values()]
 #split_points = [Point(*point) for point in outlet]
 
 # Get all vertices
 split_points = OrderedDict()
-for row in split_flows_gdf[['geometry',HYDROID, 'NextDownID']].iterrows():
+for row in split_flows_gdf[['geometry',hydro_id, 'NextDownID']].iterrows():
     lineString = row[1][0]
 
     for point in zip(*lineString.coords.xy):
@@ -199,12 +206,13 @@
 split_points = [Point(*point) for point in split_points]
 
 split_points_gdf = gpd.GeoDataFrame({'id': hydroIDs_points , 'geometry':split_points}, crs=flows.crs, geometry='geometry')
+
 print('Writing outputs ...')
 
 if isfile(split_flows_fileName):
     remove(split_flows_fileName)
-split_flows_gdf.to_file(split_flows_fileName,driver='GPKG',index=False)
+split_flows_gdf.to_file(split_flows_fileName,driver=getDriver(split_flows_fileName),index=False)
 
 if isfile(split_points_fileName):
     remove(split_points_fileName)
-split_points_gdf.to_file(split_points_fileName,driver='GPKG',index=False)
+split_points_gdf.to_file(split_points_fileName,driver=getDriver(split_points_fileName),index=False)

From abbe0bd4bf3b03c5000476d0dfdcd70ec17b20eb Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Thu, 4 Feb 2021 11:07:24 -0500
Subject: [PATCH 013/359] Minor bug fixes

Changes
- corrected variable name in fim_run.sh
- acquire_and_preprocess_inputs.py now creates huc_lists folder and updates file file path

Fixes #252
---
 CHANGELOG.md                         | 11 ++++++++++-
 fim_run.sh                           |  2 +-
 lib/acquire_and_preprocess_inputs.py |  9 ++++++---
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd90d4713..14b2a0fe1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.3.1 - 2021-02-04 - [PR #253](https://github.com/NOAA-OWP/cahaba/pull/253)
+
+Bug fixes to correct mismatched variable name and file path.
+
+### Changes
+
+ - Corrected variable name in `fim_run.sh`.
+ - `acquire_and_preprocess_inputs.py` now creates `huc_lists` folder and updates file path.
+
 ## v3.0.3.0 - 2021-02-04 - [PR #227](https://github.com/NOAA-OWP/cahaba/pull/227)
 
 Post-process to aggregate FIM outputs to HUC6 scale.
@@ -18,7 +27,7 @@ Post-process to aggregate FIM outputs to HUC6 scale.
  - Cleaned up variable names in `split_flows.py` and `build_stream_traversal.py`.
  - `build_stream_traversal.py` is now assigning HydroID by midpoint instead centroid.
  - Cleanup of `clip_vectors_to_wbd.py`.
- 
+
 ## v3.0.2.0 - 2021-01-25 - [PR #218](https://github.com/NOAA-OWP/cahaba/pull/218)
 
 Addition of an API service to schedule, run and manage `fim_run` jobs through a user-friendly web interface.
diff --git a/fim_run.sh b/fim_run.sh
index b2efa0ec5..a66ecb312 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -98,7 +98,7 @@ source $libDir/bash_functions.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
-    jobLimit=$defaultMaxJobs
+    jobLimit=$default_max_jobs
 fi
 
 ## Define Outputs Data Dir & Log File##
diff --git a/lib/acquire_and_preprocess_inputs.py b/lib/acquire_and_preprocess_inputs.py
index bd1eda194..bace41e19 100755
--- a/lib/acquire_and_preprocess_inputs.py
+++ b/lib/acquire_and_preprocess_inputs.py
@@ -262,9 +262,12 @@ def build_huc_list_files(path_to_saved_data_parent_dir, wbd_directory):
     huc6_list = set(huc6_list)
 
     # Write huc lists to appropriate .lst files.
-    included_huc4_file = os.path.join(path_to_saved_data_parent_dir, 'included_huc4.lst')
-    included_huc6_file = os.path.join(path_to_saved_data_parent_dir, 'included_huc6.lst')
-    included_huc8_file = os.path.join(path_to_saved_data_parent_dir, 'included_huc8.lst')
+    huc_lists_dir = os.path.join(path_to_saved_data_parent_dir, 'huc_lists')
+    if not os.path.exists(huc_lists_dir):
+        os.mkdir(huc_lists_dir)
+    included_huc4_file = os.path.join(huc_lists_dir, 'included_huc4.lst')
+    included_huc6_file = os.path.join(huc_lists_dir, 'included_huc6.lst')
+    included_huc8_file = os.path.join(huc_lists_dir, 'included_huc8.lst')
 
     # Overly verbose file writing loops. Doing this in a pinch.
     with open(included_huc4_file, 'w') as f:

From 2ffd97b7e9749c984697d8778859467afa26bfb6 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Wed, 10 Feb 2021 09:46:29 -0600
Subject: [PATCH 014/359] New python script "wrappers" for using inundation.py.

- Created inundation_wrapper_nwm_flows.py to produce inundation outputs using NWM recurrence flows: 1.5 year, 5 year, 10 year.
- Created inundation_wrapper_custom_flow.py to produce inundation outputs with user-created flow file.
- Created new tools parent directory to store inundation_wrapper_nwm_flows.py and inundation_wrapper_custom_flow.py.

This resolves #220.
---
 CHANGELOG.md                            |  19 +++
 tools/inundation_wrapper_custom_flow.py | 136 ++++++++++++++++++++
 tools/inundation_wrapper_nwm_flows.py   | 160 ++++++++++++++++++++++++
 3 files changed, 315 insertions(+)
 create mode 100644 tools/inundation_wrapper_custom_flow.py
 create mode 100755 tools/inundation_wrapper_nwm_flows.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 14b2a0fe1..4002936b9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.4.0 - 2021-02-10 - [PR #256](https://github.com/NOAA-OWP/cahaba/pull/256)
+
+New python script "wrappers" for using `inundation.py`.
+
+### Additions
+
+ - Created `inundation_wrapper_nwm_flows.py` to produce inundation outputs using NWM recurrence flows: 1.5 year, 5 year, 10 year.
+ - Created `inundation_wrapper_custom_flow.py` to produce inundation outputs with user-created flow file.
+ - Created new `tools` parent directory to store `inundation_wrapper_nwm_flows.py` and  `inundation_wrapper_custom_flow.py`.
+ 
+<br/><br/>
 ## v3.0.3.1 - 2021-02-04 - [PR #253](https://github.com/NOAA-OWP/cahaba/pull/253)
 
 Bug fixes to correct mismatched variable name and file path.
@@ -10,6 +21,7 @@ Bug fixes to correct mismatched variable name and file path.
  - Corrected variable name in `fim_run.sh`.
  - `acquire_and_preprocess_inputs.py` now creates `huc_lists` folder and updates file path.
 
+<br/><br/>
 ## v3.0.3.0 - 2021-02-04 - [PR #227](https://github.com/NOAA-OWP/cahaba/pull/227)
 
 Post-process to aggregate FIM outputs to HUC6 scale.
@@ -28,6 +40,7 @@ Post-process to aggregate FIM outputs to HUC6 scale.
  - `build_stream_traversal.py` is now assigning HydroID by midpoint instead centroid.
  - Cleanup of `clip_vectors_to_wbd.py`.
 
+<br/><br/>
 ## v3.0.2.0 - 2021-01-25 - [PR #218](https://github.com/NOAA-OWP/cahaba/pull/218)
 
 Addition of an API service to schedule, run and manage `fim_run` jobs through a user-friendly web interface.
@@ -36,6 +49,7 @@ Addition of an API service to schedule, run and manage `fim_run` jobs through a
 
  - `api` folder that contains all the codebase for the new service.
 
+<br/><br/>
 ## v3.0.1.0 - 2021-01-21 - [PR #206](https://github.com/NOAA-OWP/cahaba/pull/206)
 
 Preprocess MS and FR stream networks
@@ -51,6 +65,7 @@ Preprocess MS and FR stream networks
  - Added `getDriver` to shared `functions.py`.
  - Cleaned up variable names and types.
 
+<br/><br/>
 ## v3.0.0.4 - 2021-01-20 - [PR #230](https://github.com/NOAA-OWP/cahaba/pull/230)
 
 Changed the directory where the `included_huc*.lst` files are being read from.
@@ -59,6 +74,7 @@ Changed the directory where the `included_huc*.lst` files are being read from.
 
  - Changed the directory where the `included_huc*.lst` files are being read from.
 
+<br/><br/>
 ## v3.0.0.3 - 2021-01-14 - [PR #210](https://github.com/NOAA-OWP/cahaba/pull/210)
 
 Hotfix for handling nodata value in rasterized levee lines.
@@ -68,6 +84,7 @@ Hotfix for handling nodata value in rasterized levee lines.
  - Resolves bug for HUCs where `$ndv > 0` (Great Lakes region).
  - Initialize the `nld_rasterized_elev.tif` using a value of `-9999` instead of `$ndv`.
  
+ <br/><br/>
 ## v3.0.0.2 - 2021-01-06 - [PR #200](https://github.com/NOAA-OWP/cahaba/pull/200)
 
 Patch to address AHPSs mapping errors.
@@ -79,6 +96,7 @@ Patch to address AHPSs mapping errors.
  - Updates path to latest AHPs site layer.
  - Updated [readme](https://github.com/NOAA-OWP/cahaba/commit/9bffb885f32dfcd95978c7ccd2639f9df56ff829)
 
+<br/><br/>
 ## v3.0.0.1 - 2020-12-31 - [PR #184](https://github.com/NOAA-OWP/cahaba/pull/184)
 
 Modifications to build and run Docker image more reliably. Cleanup on some pre-processing scripts.
@@ -91,6 +109,7 @@ Modifications to build and run Docker image more reliably. Cleanup on some pre-p
 ### Notes
  - `aggregate_vector_inputs.py` doesn't work yet. Need to externally download required data to run fim_run.sh
  
+ <br/><br/>
 ## v3.0.0.0 - 2020-12-22 - [PR #181](https://github.com/NOAA-OWP/cahaba/pull/181)
 
 The software released here builds on the flood inundation mapping capabilities demonstrated as part of the National Flood Interoperability Experiment, the Office of Water Prediction's Innovators Program and the National Water Center Summer Institute. The flood inundation mapping software implements the Height Above Nearest Drainage (HAND) algorithm and incorporates community feedback and lessons learned over several years. The software has been designed to meet the requirements set by stakeholders interested in flood prediction and has been developed in partnership with several entities across the water enterprise.
diff --git a/tools/inundation_wrapper_custom_flow.py b/tools/inundation_wrapper_custom_flow.py
new file mode 100644
index 000000000..8602bb008
--- /dev/null
+++ b/tools/inundation_wrapper_custom_flow.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+# Created: 1/11/2021
+# Primary developer(s): ryan.spies@noaa.gov
+# Purpose: This script provides the user to input a customized flow entry to produce
+# inundation outputs using outputs from fim_run. Note that the flow csv must be
+# formatted with "feature_id" & "discharge" columns. Flow must be in cubic m/s
+
+import os
+import sys
+import pandas as pd
+import geopandas as gpd
+import rasterio
+import json
+import csv
+import argparse
+import shutil
+
+# insert python path at runtime for accessing scripts in foss_fim/tests dir (e.g. inundation.py)
+sys.path.insert(1, 'foss_fim/tests')
+from utils.shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
+from inundation import inundate
+
+TEST_CASES_DIR = r'/data/inundation_review/inundation_custom_flow/'  # Will update.
+INPUTS_DIR = r'/data/inputs'
+OUTPUTS_DIR = os.environ['outputDataDir']
+
+ENDC = '\033[m'
+TGREEN_BOLD = '\033[32;1m'
+TGREEN = '\033[32m'
+TRED_BOLD = '\033[31;1m'
+TWHITE = '\033[37m'
+WHITE_BOLD = '\033[37;1m'
+CYAN_BOLD = '\033[36;1m'
+
+def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type='huc'):
+
+    # Construct paths to development test results if not existent.
+    huc_id_dir_parent = os.path.join(TEST_CASES_DIR, huc_id)
+    if not os.path.exists(huc_id_dir_parent):
+        os.mkdir(huc_id_dir_parent)
+    branch_test_case_dir_parent = os.path.join(TEST_CASES_DIR, huc_id, branch_name)
+
+    # Delete the entire directory if it already exists.
+    if os.path.exists(branch_test_case_dir_parent):
+        shutil.rmtree(branch_test_case_dir_parent)
+
+    print("Running the NWM recurrence intervals for HUC: " + huc_id + ", " + branch_name + "...")
+
+    assert os.path.exists(fim_run_dir), "Cannot locate " + fim_run_dir
+
+    # Create paths to fim_run outputs for use in inundate().
+    if "previous_fim" in fim_run_dir and "fim_2" in fim_run_dir:
+        rem = os.path.join(fim_run_dir, 'rem_clipped_zeroed_masked.tif')
+        catchments = os.path.join(fim_run_dir, 'gw_catchments_reaches_clipped_addedAttributes.tif')
+    else:
+        rem = os.path.join(fim_run_dir, 'rem_zeroed_masked.tif')
+        catchments = os.path.join(fim_run_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+    if mask_type == 'huc':
+        catchment_poly = ''
+    else:
+        catchment_poly = os.path.join(fim_run_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+    hydro_table = os.path.join(fim_run_dir, 'hydroTable.csv')
+
+    # Map necessary inputs for inundation().
+    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+
+    #benchmark_category = huc_id.split('_')[1]
+    current_huc = huc_id.split('_')[0]  # Break off HUC ID and assign to variable.
+
+    if not os.path.exists(branch_test_case_dir_parent):
+        os.mkdir(branch_test_case_dir_parent)
+
+
+    #branch_test_case_dir = os.path.join(branch_test_case_dir_parent)
+
+    #os.makedirs(branch_test_case_dir)  # Make output directory for branch.
+
+    # Define paths to inundation_raster and forecast file.
+    inundation_raster = os.path.join(branch_test_case_dir_parent, branch_name + '_inund_extent.tif')
+    forecast = os.path.join(TEST_CASES_DIR,"_input_flow_files", input_flow_csv)
+
+    # Copy forecast flow file into the outputs directory to all viewer to reference the flows used to create inundation_raster
+    shutil.copyfile(forecast,os.path.join(branch_test_case_dir_parent,input_flow_csv))
+
+    # Run inundate.
+    print("-----> Running inundate() to produce modeled inundation extent for the " + input_flow_csv)
+    inundate(
+             rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+             subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
+             depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
+            )
+
+    print("-----> Inundation mapping complete.")
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using a user supplied flow data file. Inundation outputs are stored in the /inundation_review/inundation_custom_flow/ directory.')
+    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev/12345678)',required=True)
+    parser.add_argument('-b', '--branch-name',help='The name of the working branch in which features are being tested (used to name the output inundation directory) -> type=str',required=True,default="")
+    parser.add_argument('-t', '--huc-id',help='The huc id to use (single huc). Format as: xxxxxxxx, e.g. 12345678',required=True,default="")
+    parser.add_argument('-m', '--mask-type', help='Optional: specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
+    parser.add_argument('-y', '--input-flow-csv',help='Filename of the user generated (customized) csv. Must contain nwm feature ids and flow value(s) (units: cms) --> put this file in the "_input_flow_files" directory',required=True, default="")
+
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    valid_huc_id_list = ['nwm_recurr']
+
+    exit_flag = False  # Default to False.
+    print()
+
+    # Ensure fim_run_dir exists.
+    if not os.path.exists(args['fim_run_dir']):
+        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
+        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. data/outputs/123456/123456." + ENDC)
+        print()
+        exit_flag = True
+
+
+    if args['input_flow_csv'] == '':
+        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided input_flow_csv (-y) " + CYAN_BOLD + args['input_flow_csv'] + WHITE_BOLD + " is not provided. Please provide a csv file with nwm featureid and flow values" + ENDC)
+        exit_flag = True
+
+
+    if exit_flag:
+        print()
+        sys.exit()
+
+
+    else:
+
+        run_recurr_test(**args)
diff --git a/tools/inundation_wrapper_nwm_flows.py b/tools/inundation_wrapper_nwm_flows.py
new file mode 100755
index 000000000..1854bb5c3
--- /dev/null
+++ b/tools/inundation_wrapper_nwm_flows.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+
+# Created: 1/10/2021
+# Primary developer(s): ryan.spies@noaa.gov
+# Purpose: This script provides the user to generate inundation outputs using
+# the NWM Recurrence Interval flow data for 1.5yr, 5yr, & 10yr events.
+
+import os
+import sys
+import pandas as pd
+import geopandas as gpd
+import rasterio
+import json
+import csv
+import argparse
+import shutil
+
+# insert python path at runtime for accessing scripts in foss_fim/tests dir (e.g. inundation.py)
+sys.path.insert(1, 'foss_fim/tests')
+from utils.shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
+from inundation import inundate
+
+TEST_CASES_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
+INPUTS_DIR = r'/data/inputs'
+OUTPUTS_DIR = os.environ['outputDataDir']
+
+ENDC = '\033[m'
+TGREEN_BOLD = '\033[32;1m'
+TGREEN = '\033[32m'
+TRED_BOLD = '\033[31;1m'
+TWHITE = '\033[37m'
+WHITE_BOLD = '\033[37;1m'
+CYAN_BOLD = '\033[36;1m'
+
+def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'):
+
+    # Construct paths to development test results if not existent.
+    huc_id_dir_parent = os.path.join(TEST_CASES_DIR, huc_id)
+    if not os.path.exists(huc_id_dir_parent):
+        os.mkdir(huc_id_dir_parent)
+    branch_test_case_dir_parent = os.path.join(TEST_CASES_DIR, huc_id, branch_name)
+
+    # Delete the entire directory if it already exists.
+    if os.path.exists(branch_test_case_dir_parent):
+        shutil.rmtree(branch_test_case_dir_parent)
+
+    print("Running the NWM recurrence intervals for huc_id: " + huc_id + ", " + branch_name + "...")
+
+    fim_run_parent = os.path.join(fim_run_dir)
+    assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
+
+    # Create paths to fim_run outputs for use in inundate().
+    if "previous_fim" in fim_run_parent and "fim_2" in fim_run_parent:
+        rem = os.path.join(fim_run_parent, 'rem_clipped_zeroed_masked.tif')
+        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_clipped_addedAttributes.tif')
+    else:
+        rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
+        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+    if mask_type == 'huc':
+        catchment_poly = ''
+    else:
+        catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
+
+    # Map necessary inputs for inundation().
+    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+
+    #benchmark_category = huc_id.split('_')[1]
+    current_huc = huc_id.split('_')[0]  # Break off HUC ID and assign to variable.
+
+    if not os.path.exists(branch_test_case_dir_parent):
+        os.mkdir(branch_test_case_dir_parent)
+
+    # Check if magnitude is list of magnitudes or single value.
+    magnitude_list = magnitude
+    if type(magnitude_list) != list:
+        magnitude_list = [magnitude_list]
+
+    for magnitude in magnitude_list:
+        # Construct path to validation raster and forecast file.
+
+        branch_test_case_dir = os.path.join(branch_test_case_dir_parent, magnitude)
+
+        os.makedirs(branch_test_case_dir)  # Make output directory for branch.
+
+        # Define paths to inundation_raster and forecast file.
+        inundation_raster = os.path.join(branch_test_case_dir, branch_name + '_inund_extent.tif')
+        forecast = os.path.join(TEST_CASES_DIR, 'nwm_recurr_flow_data', 'recurr_' + magnitude + '_cms.csv')
+
+        # Run inundate.
+        print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
+        inundate(
+                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                 subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
+                 depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
+                )
+
+        print("-----> Inundation mapping complete.")
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow recurrence interflow data. Inundation outputs are stored in the /inundation_review/inundation_nwm_recurr/ directory.')
+    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)',required=True)
+    parser.add_argument('-b', '--branch-name',help='The name of the working branch in which features are being tested (used to name the output inundation directory) -> type=str',required=True,default="")
+    parser.add_argument('-t', '--huc-id',help='Provide either a single hucid (Format as: xxxxxxxx, e.g. 12345678) or a filepath to a list of hucids',required=True,default="")
+    parser.add_argument('-m', '--mask-type', help='Optional: specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
+    parser.add_argument('-y', '--magnitude',help='The magnitude (reccur interval) to run. Leave blank to use default intervals (options: 1_5, 5_0, 10_0).',required=False, default="")
+
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    valid_test_id_list = ['nwm_recurr']
+
+    exit_flag = False  # Default to False.
+    print()
+
+    # check if user provided a single huc_id or a file path to a list of huc ids
+    if args['huc_id'].isdigit():
+        huc_list = [args['huc_id']]
+    elif os.path.exists(args['huc_id']): # check if provided str is a valid path
+        with open(args['huc_id'],newline='') as list_file:
+            read_list = csv.reader(list_file)
+            huc_list=[i for row in read_list for i in row]
+    else:
+        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "Invalid huc-id entry: " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " --> check that huc_id number or list file is valid")
+        exit_flag = True
+    print(huc_list)
+    if exit_flag:
+        print()
+        sys.exit()
+
+    for huc_id in huc_list:
+        args['huc_id'] = huc_id
+        # Ensure fim_run_dir exists.
+        fim_run_dir = args['fim_run_dir'] + os.sep + huc_id
+        if not os.path.exists(fim_run_dir):
+            print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + fim_run_dir + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
+            print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. data/outputs/123456/123456." + ENDC)
+            print()
+            exit_flag = True
+
+        # Ensure valid flow recurr intervals
+        default_flow_intervals = ['1_5','5_0','10_0']
+        if args['magnitude'] == '':
+            args['magnitude'] = default_flow_intervals
+            print(TRED_BOLD + "Using default flow reccurence intervals: " + WHITE_BOLD + str(default_flow_intervals)[1:-1])
+        else:
+            if set(default_flow_intervals).issuperset(set(args['magnitude'])) == False:
+                print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. NWM Recurrence Interval options include: " + str(default_flow_intervals)[1:-1] + ENDC)
+                exit_flag = True
+
+        if exit_flag:
+            print()
+            sys.exit()
+
+        else:
+            run_recurr_test(fim_run_dir,args['branch_name'],huc_id,args['magnitude'],args['mask_type'])

From b147f45c9b88343361a6dc10d5922442f666090f Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 10 Feb 2021 14:00:16 -0600
Subject: [PATCH 015/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 47d3561ab..c2fca221e 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 ### Cahaba: Flood Inundation Mapping for U.S. National Water Model
 
-Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Weather Service (NWS). Software enables inundation mapping capability by generating Relative Elevation Models (REMs) and Synthetic Rating Curves (SRCs). Included are tests to evaluate skill and computational efficiency as well as functions to generate inundation maps.
+Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Weather Service (NWS). This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs) and Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIM). This repository also includes functionality to generate FIMs as well as tests to evaluate FIM prediction skill.
 
 ## Dependencies
 

From 57cb283c8d194893a0b96af1b6c95b333b651fe1 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 10 Feb 2021 14:01:24 -0600
Subject: [PATCH 016/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c2fca221e..5cf6f92ca 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 ### Cahaba: Flood Inundation Mapping for U.S. National Water Model
 
-Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Weather Service (NWS). This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs) and Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIM). This repository also includes functionality to generate FIMs as well as tests to evaluate FIM prediction skill.
+Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Weather Service (NWS). This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIM). This repository also includes functionality to generate FIMs as well as tests to evaluate FIM prediction skill.
 
 ## Dependencies
 

From ad89e6d36691f8fa9a4a743e15faee8e8e10c20c Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Fri, 12 Feb 2021 08:57:03 -0600
Subject: [PATCH 017/359] Updated MS Crosswalk method to address gaps in FIM.

- Fixed typo in stream midpoint calculation in split_flows.py and add_crosswalk.py.
- add_crosswalk.py now restricts the MS crosswalk to NWM MS catchments.
- add_crosswalk.py now performs a secondary MS crosswalk selection by nearest NWM MS catchment.

This resolves #232.
---
 CHANGELOG.md                  | 11 +++++++++++
 fim_run.sh                    |  6 ++++--
 lib/add_crosswalk.py          | 18 +++++++++++++++++-
 lib/build_stream_traversal.py |  2 +-
 lib/run_by_unit.sh            |  4 ++++
 5 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4002936b9..d89843748 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.4.1 - 2021-02-12 - [PR #261](https://github.com/NOAA-OWP/cahaba/pull/261)
+
+Updated MS Crosswalk method to address gaps in FIM.
+
+### Changes
+
+- Fixed typo in stream midpoint calculation in `split_flows.py` and `add_crosswalk.py`.
+- `add_crosswalk.py` now restricts the MS crosswalk to NWM MS catchments.
+- `add_crosswalk.py` now performs a secondary MS crosswalk selection by nearest NWM MS catchment.
+
+<br/><br/>
 ## v3.0.4.0 - 2021-02-10 - [PR #256](https://github.com/NOAA-OWP/cahaba/pull/256)
 
 New python script "wrappers" for using `inundation.py`.
diff --git a/fim_run.sh b/fim_run.sh
index a66ecb312..8c6862dae 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -112,8 +112,10 @@ logFile=$outputRunDataDir/logs/summary.log
 ## Define inputs
 export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
 export input_NWM_Lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
-export input_NWM_Catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
-export input_NWM_Flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
+export input_NWM_Catchments_fr=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
+export input_NWM_Catchments_ms=$inputDataDir/nwm_hydrofabric/nwm_catchments_ms.gpkg
+export input_NWM_Flows_fr=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
+export input_NWM_Flows_ms=$inputDataDir/nwm_hydrofabric/nwm_flows_ms.gpkg
 export input_NWM_Headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
 export input_nhd_flowlines_fr=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_fr_adjusted.gpkg
 export input_nhd_flowlines_ms=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_ms_adjusted.gpkg
diff --git a/lib/add_crosswalk.py b/lib/add_crosswalk.py
index c5ed0dd5a..d3f052c3a 100755
--- a/lib/add_crosswalk.py
+++ b/lib/add_crosswalk.py
@@ -58,7 +58,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
         hydroID = []
         for i,lineString in enumerate(input_flows.geometry):
             hydroID = hydroID + [input_flows.loc[i,'HydroID']]
-            stream_midpoint = stream_midpoint + [lineString.interpolate(0.05,normalized=True)]
+            stream_midpoint = stream_midpoint + [lineString.interpolate(0.5,normalized=True)]
 
         input_flows_midpoint = gpd.GeoDataFrame({'HydroID':hydroID, 'geometry':stream_midpoint}, crs=input_flows.crs, geometry='geometry')
         input_flows_midpoint = input_flows_midpoint.set_index('HydroID')
@@ -66,6 +66,22 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
         # Create crosswalk
         crosswalk = gpd.sjoin(input_flows_midpoint, input_nwmcat, how='left', op='within').reset_index()
         crosswalk = crosswalk.rename(columns={"index_right": "feature_id"})
+
+        # fill in missing ms
+        crosswalk_missing = crosswalk.loc[crosswalk.feature_id.isna()]
+        for index, stream in crosswalk_missing.iterrows():
+
+            # find closest nwm catchment by distance
+            distances = [stream.geometry.distance(poly) for poly in input_nwmcat.geometry]
+            min_dist = min(distances)
+            nwmcat_index=distances.index(min_dist)
+
+            # update crosswalk
+            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'feature_id'] = input_nwmcat.iloc[nwmcat_index].name
+            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'AreaSqKM'] = input_nwmcat.iloc[nwmcat_index].AreaSqKM
+            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'Shape_Length'] = input_nwmcat.iloc[nwmcat_index].Shape_Length
+            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'Shape_Area'] = input_nwmcat.iloc[nwmcat_index].Shape_Area
+
         crosswalk = crosswalk.filter(items=['HydroID', 'feature_id'])
         crosswalk = crosswalk.merge(input_nwmflows[['feature_id','order_']],on='feature_id')
 
diff --git a/lib/build_stream_traversal.py b/lib/build_stream_traversal.py
index a30fe04ed..daca019f2 100644
--- a/lib/build_stream_traversal.py
+++ b/lib/build_stream_traversal.py
@@ -45,7 +45,7 @@ def execute(self, streams, wbd8, hydro_id):
                 # Get stream midpoint
                 stream_midpoint = []
                 for i,lineString in enumerate(streams.geometry):
-                    stream_midpoint = stream_midpoint + [lineString.interpolate(0.05,normalized=True)]
+                    stream_midpoint = stream_midpoint + [lineString.interpolate(0.5,normalized=True)]
 
                 stream_md_gpd = gpd.GeoDataFrame({'geometry':stream_midpoint}, crs=streams.crs, geometry='geometry')
                 stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', op='within')
diff --git a/lib/run_by_unit.sh b/lib/run_by_unit.sh
index 7715304e3..9f34bc222 100755
--- a/lib/run_by_unit.sh
+++ b/lib/run_by_unit.sh
@@ -47,9 +47,13 @@ fi
 if [ "$extent" = "MS" ]; then
   input_nhd_flowlines=$input_nhd_flowlines_ms
   input_nhd_headwaters=$input_nhd_headwaters_ms
+  input_NWM_Flows=$input_NWM_Flows_ms
+  input_NWM_Catchments=$input_NWM_Catchments_ms
 else
   input_nhd_flowlines=$input_nhd_flowlines_fr
   input_nhd_headwaters=$input_nhd_headwaters_fr
+  input_NWM_Flows=$input_NWM_Flows_fr
+  input_NWM_Catchments=$input_NWM_Catchments_fr
 fi
 
 ## GET WBD ##

From f474736240f6d27c7a1d5637efd88134214db87b Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Fri, 12 Feb 2021 15:35:11 -0600
Subject: [PATCH 018/359] Addresses issue when running on HUC6 scale

* src should be fixed and slightly smaller by removing whitespace
* rasters are about the same size as running fim as huc6 (compressed and tiled; aggregated are slightly larger)
* naming convention and feature id attribute are only added to the aggregated hucs (this bullet and the next one assume that Viz will not use the huc8 layer outputs)
* HydroIDs are different for huc6 vs aggregated huc8s mostly due to forced split at huc boundaries (so long we use consistent workflow it shouldn't matter)
* known issue where sometimes an incoming stream is not included in the final selection (resolves #238) will affect aggregate outputs
---
 CHANGELOG.md                 |  19 +++-
 lib/aggregate_fim_outputs.py | 164 +++++++++++++++++++++++++----------
 lib/output_cleanup.py        |  20 -----
 3 files changed, 132 insertions(+), 71 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d89843748..65cdeb4f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,28 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.4.2 - 2021-02-12 - [PR #255](https://github.com/NOAA-OWP/cahaba/pull/255)
+
+Addresses issue when running on HUC6 scale.
+
+### Changes
+
+ - `src.json` should be fixed and slightly smaller by removing whitespace.
+ - Rasters are about the same size as running fim as huc6 (compressed and tiled; aggregated are slightly larger).
+ - Naming convention and feature id attribute are only added to the aggregated hucs.
+ - HydroIDs are different for huc6 vs aggregated huc8s mostly due to forced split at huc boundaries (so long we use consistent workflow it shouldn't matter).
+ - Fixed known issue where sometimes an incoming stream is not included in the final selection will affect aggregate outputs.
+
+<br/><br/>
 ## v3.0.4.1 - 2021-02-12 - [PR #261](https://github.com/NOAA-OWP/cahaba/pull/261)
 
 Updated MS Crosswalk method to address gaps in FIM.
 
 ### Changes
 
-- Fixed typo in stream midpoint calculation in `split_flows.py` and `add_crosswalk.py`.
-- `add_crosswalk.py` now restricts the MS crosswalk to NWM MS catchments.
-- `add_crosswalk.py` now performs a secondary MS crosswalk selection by nearest NWM MS catchment.
+ - Fixed typo in stream midpoint calculation in `split_flows.py` and `add_crosswalk.py`.
+ - `add_crosswalk.py` now restricts the MS crosswalk to NWM MS catchments.
+ - `add_crosswalk.py` now performs a secondary MS crosswalk selection by nearest NWM MS catchment.
 
 <br/><br/>
 ## v3.0.4.0 - 2021-02-10 - [PR #256](https://github.com/NOAA-OWP/cahaba/pull/256)
diff --git a/lib/aggregate_fim_outputs.py b/lib/aggregate_fim_outputs.py
index 1296fb8cc..edafd93a3 100644
--- a/lib/aggregate_fim_outputs.py
+++ b/lib/aggregate_fim_outputs.py
@@ -6,12 +6,15 @@
 import json
 import rasterio
 from rasterio.merge import merge
+import shutil
+import csv
 from utils.shared_variables import PREP_PROJECTION
 
 
 def aggregate_fim_outputs(fim_out_dir):
 
     print ("aggregating outputs to HUC6 scale")
+
     drop_folders = ['logs']
     huc_list = [huc for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
     huc6_list = [str(huc[0:6]) for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
@@ -21,76 +24,141 @@ def aggregate_fim_outputs(fim_out_dir):
 
         os.makedirs(os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6])), exist_ok=True)
 
-        # merge hydrotable
+        # original file paths
+        hydrotable_filename = os.path.join(fim_out_dir,huc,'hydroTable.csv')
+        src_filename = os.path.join(fim_out_dir,huc,'src.json')
+
+        # aggregate file name paths
         aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),'hydroTable.csv')
-        hydrotable = pd.read_csv(os.path.join(fim_out_dir,huc,'hydroTable.csv'))
+        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),f'rating_curves_{huc[0:6]}.json')
 
-        # write out hydrotable
-        if os.path.isfile(aggregate_hydrotable):
-            hydrotable.to_csv(aggregate_hydrotable,index=False, mode='a',header=False)
-        else:
-            hydrotable.to_csv(aggregate_hydrotable,index=False)
+        if len(huc)> 6:
 
-        del hydrotable
+            # open hydrotable
+            hydrotable = pd.read_csv(hydrotable_filename)
 
-        # merge src
-        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),'src.json')
-        src = open(os.path.join(fim_out_dir,huc,'src.json'))
-        src = json.load(src)
+            # write/append aggregate hydrotable
+            if os.path.isfile(aggregate_hydrotable):
+                hydrotable.to_csv(aggregate_hydrotable,index=False, mode='a',header=False)
+            else:
+                hydrotable.to_csv(aggregate_hydrotable,index=False)
 
-        # write out src
-        if os.path.isfile(aggregate_src):
-            with open(aggregate_src, 'a') as outfile:
-                    json.dump(src, outfile)
-        else:
-            with open(aggregate_src, 'w') as outfile:
-                    json.dump(src, outfile)
+            del hydrotable
+
+            # open src
+            src = open(src_filename)
+            src = json.load(src)
+
+            # write/append aggregate src
+            if os.path.isfile(aggregate_src):
+
+                with open(aggregate_src, "r+") as file:
+                    data = json.load(file)
+                    data.update(src)
+
+                with open(aggregate_src, 'w') as outfile:
+                    json.dump(data, outfile)
+            else:
+                with open(aggregate_src, 'w') as outfile:
+                        json.dump(src, outfile)
 
-        del src
+            del src
+
+        else:
+            shutil.copy(hydrotable_filename, aggregate_hydrotable)
+            shutil.copy(src_filename, aggregate_src)
 
     for huc6 in huc6_list:
+
+        ## add feature_id to aggregate src
+        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),'hydroTable.csv')
+        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),f'rating_curves_{huc6}.json')
+
+        # Open aggregate src for writing feature_ids to
+        src_data = {}
+        with open(aggregate_src) as jsonf:
+            src_data = json.load(jsonf)
+
+        with open(aggregate_hydrotable) as csvf:
+            csvReader = csv.DictReader(csvf)
+
+            for row in csvReader:
+                if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
+                    src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
+
+        # Write src_data to JSON file
+        with open(aggregate_src, 'w') as jsonf:
+            json.dump(src_data, jsonf)
+
+        ## aggregate rasters
         huc6_dir = os.path.join(fim_out_dir,'aggregate_fim_outputs',huc6)
 
-        huc6_filter = [path.startswith(huc6) for path in huc_list]
-        subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
+        # aggregate file paths
+        rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}.tif')
+        catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}.tif')
+
+        if huc6 not in huc_list:
+
+            huc6_filter = [path.startswith(huc6) for path in huc_list]
+            subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
+
+            # aggregate and mosaic rem
+            rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
+
+            if len(rem_list) > 1:
+
+                rem_files_to_mosaic = []
+
+                for rem in rem_list:
 
-        # aggregate and mosaic rem
-        rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
+                    rem_src = rasterio.open(rem)
+                    rem_files_to_mosaic.append(rem_src)
 
-        rem_files_to_mosaic = []
+                mosaic, out_trans = merge(rem_files_to_mosaic)
+                out_meta = rem_src.meta.copy()
+                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
 
-        for rem in rem_list:
-            rem_src = rasterio.open(rem)
-            rem_files_to_mosaic.append(rem_src)
+                with rasterio.open(rem_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
+                    dest.write(mosaic)
 
-        mosaic, out_trans = merge(rem_files_to_mosaic)
-        out_meta = rem_src.meta.copy()
-        out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION})
+                del rem_files_to_mosaic,rem_src,out_meta,mosaic
 
-        rem_mosaic = os.path.join(huc6_dir,'rem_zeroed_masked.tif')
-        with rasterio.open(rem_mosaic, "w", **out_meta) as dest:
-            dest.write(mosaic)
+            elif len(rem_list)==1:
 
-        del rem_files_to_mosaic,rem_src,out_meta,mosaic
+                shutil.copy(rem_list[0], rem_mosaic)
 
-        # aggregate and mosaic catchments
-        catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
+            # aggregate and mosaic catchments
+            catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
 
-        cat_files_to_mosaic = []
+            if len(catchment_list) > 1:
 
-        for cat in catchment_list:
-            cat_src = rasterio.open(cat)
-            cat_files_to_mosaic.append(cat_src)
+                cat_files_to_mosaic = []
 
-        mosaic, out_trans = merge(cat_files_to_mosaic)
-        out_meta = cat_src.meta.copy()
-        out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION})
+                for cat in catchment_list:
+                    cat_src = rasterio.open(cat)
+                    cat_files_to_mosaic.append(cat_src)
 
-        catchment_mosaic = os.path.join(huc6_dir,'gw_catchments_reaches_filtered_addedAttributes.tif')
-        with rasterio.open(catchment_mosaic, "w", **out_meta) as dest:
-            dest.write(mosaic)
+                mosaic, out_trans = merge(cat_files_to_mosaic)
+                out_meta = cat_src.meta.copy()
+
+                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
+
+                with rasterio.open(catchment_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
+                    dest.write(mosaic)
+
+                del cat_files_to_mosaic,cat_src,out_meta,mosaic
+
+            elif len(catchment_list)==1:
+
+                shutil.copy(catchment_list[0], catchment_mosaic)
+
+        else:
+            # original file paths
+            rem_filename = os.path.join(fim_out_dir,huc6,'rem_zeroed_masked.tif')
+            catchment_filename = os.path.join(fim_out_dir,huc6,'gw_catchments_reaches_filtered_addedAttributes.tif')
 
-        del cat_files_to_mosaic,cat_src,out_meta,mosaic
+            shutil.copy(rem_filename, rem_mosaic)
+            shutil.copy(catchment_filename, catchment_mosaic)
 
 
 if __name__ == '__main__':
diff --git a/lib/output_cleanup.py b/lib/output_cleanup.py
index f4ea3ea19..b253598cf 100755
--- a/lib/output_cleanup.py
+++ b/lib/output_cleanup.py
@@ -50,26 +50,6 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         # Step 1, keep only files that Viz needs
         whitelist_directory(output_folder_path, viz_whitelist, additional_whitelist)
 
-        # Step 2, add feature_id to src.json and rename file
-        # Open src.json for writing feature_ids to
-        src_data = {}
-        with open(os.path.join(output_folder_path, 'src.json')) as jsonf:
-            src_data = json.load(jsonf)
-
-        with open(os.path.join(output_folder_path, 'hydroTable.csv')) as csvf:
-            csvReader = csv.DictReader(csvf)
-            for row in csvReader:
-                if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
-                    src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
-
-        # Write src_data to JSON file
-        with open(os.path.join(output_folder_path, f'rating_curves_{huc_number}.json'), 'w') as jsonf:
-            json.dump(src_data, jsonf)
-
-        # Step 3, copy files to desired names
-        shutil.copy(os.path.join(output_folder_path, 'rem_zeroed_masked.tif'), os.path.join(output_folder_path, f'hand_grid_{huc_number}.tif'))
-        shutil.copy(os.path.join(output_folder_path, 'gw_catchments_reaches_filtered_addedAttributes.tif'), os.path.join(output_folder_path, f'catchments_{huc_number}.tif'))
-
 def whitelist_directory(directory_path, whitelist, additional_whitelist):
     # Add any additional files to the whitelist that the user wanted to keep
     if additional_whitelist:

From d615a6ca459e5de6d26c17bfa83b234c01dab8a9 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 12 Feb 2021 15:59:00 -0600
Subject: [PATCH 019/359]  Modified rem.py with a new function to output HAND
 reference elev
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Function "make_catchment_hydroid_dict" creates a df of pixel catchment ids and overlapping hydroids
- Merge hydroid df and thalweg minimum elevation df
- Produces new output containing all catchment ids and min thalweg elevation value → “hand_ref_elev_table.csv”
- Overwrites the demDerived_reaches_split.gpk layer --> adding additional attribute: “Min_Thal_Elev_meters” to view the elevation value for each hydroid
---
 CHANGELOG.md       |  12 +++++
 lib/rem.py         | 106 ++++++++++++++++++++++++++++++++++++---------
 lib/run_by_unit.sh |   2 +-
 3 files changed, 99 insertions(+), 21 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 65cdeb4f9..7b469c5c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.4.3 - 2021-02-12 - [PR #254](https://github.com/NOAA-OWP/cahaba/pull/254)
+
+Modified `rem.py` with a new function to output HAND reference elev.
+
+### Changes
+
+ - Function `make_catchment_hydroid_dict` creates a df of pixel catchment ids and overlapping hydroids.
+ - Merge hydroid df and thalweg minimum elevation df.
+ - Produces new output containing all catchment ids and min thalweg elevation value named `hand_ref_elev_table.csv`.
+ - Overwrites the `demDerived_reaches_split.gpk` layer by adding additional attribute `Min_Thal_Elev_meters` to view the elevation value for each hydroid.
+
+<br/><br/>
 ## v3.0.4.2 - 2021-02-12 - [PR #255](https://github.com/NOAA-OWP/cahaba/pull/255)
 
 Addresses issue when running on HUC6 scale.
diff --git a/lib/rem.py b/lib/rem.py
index aa7faeafa..411380070 100755
--- a/lib/rem.py
+++ b/lib/rem.py
@@ -5,13 +5,16 @@
 import numpy as np
 import argparse
 import os
+import pandas as pd
 from osgeo import ogr, gdal
+import geopandas as gpd
+from utils.shared_functions import getDriver
 
 
-def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster):
+def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, hand_ref_elev_fileName, dem_reaches_filename):
     """
         Calculates REM/HAND/Detrended DEM
-        
+
         Parameters
         ----------
         dem_fileName : str
@@ -20,56 +23,112 @@ def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raste
             File name of stream pixel watersheds raster.
         rem_fileName : str
             File name of output relative elevation raster.
+        hydroid_fileName : str
+            File name of the hydroid raster (i.e. gw_catchments_reaches.tif)
+        hand_ref_elev_fileName
+            File name of the output csv containing list of hydroid values and HAND zero/reference elev
+        dem_reaches_filename
+            File name of the reaches layer to populate HAND elevation attribute values and overwrite as output
 
     """
 
-    
+    # ------------------------------------------- Get catchment_hydroid_dict --------------------------------------------------- #
+    # The following creates a dictionary of the catchment ids (key) and their hydroid along the thalweg (value).
+    # This is needed to produce a HAND zero reference elevation by hydroid dataframe (helpful for evaluating rating curves & bathy properties)
+    @njit
+    def make_catchment_hydroid_dict(flat_value_raster, catchment_hydroid_dict, flat_catchments, thalweg_window):
+
+        for i,cm in enumerate(flat_catchments):
+            if thalweg_window[i] == 1:  # Only allow reference hydroid to be within thalweg.
+                catchment_hydroid_dict[cm] = flat_value_raster[i]
+        return(catchment_hydroid_dict)
+
+    # Open the masked gw_catchments_pixels_masked, hydroid_raster, and dem_thalwegCond_masked.
+    gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
+    hydroid_pixels_object = rasterio.open(hydroid_fileName)
+    thalweg_raster_object = rasterio.open(thalweg_raster)
+
+    # Specify raster object metadata.
+    meta = hydroid_pixels_object.meta.copy()
+    meta['tiled'], meta['compress'] = True, 'lzw'
+
+    # -- Create catchment_hydroid_dict -- #
+    catchment_hydroid_dict = typed.Dict.empty(types.int64,types.int64)  # Initialize an empty dictionary to store the catchment hydroid.
+    # Update catchment_hydroid_dict with each pixel sheds hydroid.
+    # Creating dictionary containing catchment ids (key) and corresponding hydroid within the thalweg...
+    for ji, window in hydroid_pixels_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
+        hydroid_window = hydroid_pixels_object.read(1,window=window).ravel()  # Define hydroid_window.
+        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window.
+        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window.
+
+        # Call numba-optimized function to update catchment_hydroid_dict with pixel sheds overlapping hydroid.
+        catchment_hydroid_dict = make_catchment_hydroid_dict(hydroid_window, catchment_hydroid_dict, catchments_window, thalweg_window)
+
+    hydroid_pixels_object.close()
+    gw_catchments_pixels_masked_object.close()
+    thalweg_raster_object.close()
     # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
     # The following creates a dictionary of the catchment ids (key) and their elevation along the thalweg (value).
     @njit
     def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalweg_window):
-  
+
         for i,cm in enumerate(flat_catchments):
             if thalweg_window[i] == 1:  # Only allow reference elevation to be within thalweg.
                 # If the catchment really exists in the dictionary, compare elevation values.
                 if (cm in catchment_min_dict):
                     if (flat_dem[i] < catchment_min_dict[cm]):
                         # If the flat_dem's elevation value is less than the catchment_min_dict min, update the catchment_min_dict min.
-                        catchment_min_dict[cm] = flat_dem[i]                                                
+                        catchment_min_dict[cm] = flat_dem[i]
                 else:
-                    catchment_min_dict[cm] = flat_dem[i]                
+                    catchment_min_dict[cm] = flat_dem[i]
         return(catchment_min_dict)
 
     # Open the masked gw_catchments_pixels_masked and dem_thalwegCond_masked.
     gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
     dem_thalwegCond_masked_object = rasterio.open(dem_fileName)
     thalweg_raster_object = rasterio.open(thalweg_raster)
-    
+
     # Specify raster object metadata.
     meta = dem_thalwegCond_masked_object.meta.copy()
     meta['tiled'], meta['compress'] = True, 'lzw'
-    
+
     # -- Create catchment_min_dict -- #
-    catchment_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums.
+    catchment_min_dict = typed.Dict.empty(types.int64,types.float32)  # Initialize an empty dictionary to store the catchment minimums.
     # Update catchment_min_dict with pixel sheds minimum.
+    # Creating dictionary containing catchment ids (key) and corresponding elevation within the thalweg (value)...
     for ji, window in dem_thalwegCond_masked_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
         dem_window = dem_thalwegCond_masked_object.read(1,window=window).ravel()  # Define dem_window.
         catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window.
         thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window.
-        
+
         # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
         catchment_min_dict = make_catchment_min_dict(dem_window, catchment_min_dict, catchments_window, thalweg_window)
 
     dem_thalwegCond_masked_object.close()
     gw_catchments_pixels_masked_object.close()
     thalweg_raster_object.close()
+
+###############################################
+    # Merge and export dictionary to to_csv
+    catchment_min_dict_df = pd.DataFrame.from_dict(catchment_min_dict, orient='index') # convert dict to dataframe
+    catchment_min_dict_df.columns = ['Min_Thal_Elev_meters']
+    catchment_hydroid_dict_df = pd.DataFrame.from_dict(catchment_hydroid_dict, orient='index') # convert dict to dataframe
+    catchment_hydroid_dict_df.columns = ['HydroID']
+    merge_df = catchment_hydroid_dict_df.merge(catchment_min_dict_df, left_index=True, right_index=True)
+    merge_df.index.name = 'pixelcatch_id'
+    merge_df.to_csv(hand_ref_elev_fileName,index=True) # export dataframe to csv file
+
+    # Merge the HAND reference elvation by HydroID dataframe with the demDerived_reaches layer (add new layer attribute)
+    merge_df = merge_df.groupby(['HydroID']).median() # median value of all Min_Thal_Elev_meters for pixel catchments in each HydroID reach
+    input_reaches = gpd.read_file(dem_reaches_filename)
+    input_reaches = input_reaches.merge(merge_df, on='HydroID') # merge dataframes by HydroID variable
+    input_reaches.to_file(dem_reaches_filename,driver=getDriver(dem_reaches_filename),index=False)
     # ------------------------------------------------------------------------------------------------------------------------ #
-    
-    
+
+
     # ------------------------------------------- Produce relative elevation model ------------------------------------------- #
     @njit
     def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
-
         rem_window = np.zeros(len(flat_dem),dtype=np.float32)
         for i,cm in enumerate(flat_catchments):
             if cm in catchmentMinDict:
@@ -83,19 +142,20 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
     rem_rasterio_object = rasterio.open(rem_fileName,'w',**meta)  # Open rem_rasterio_object for writing to rem_fileName.
     pixel_catchments_rasterio_object = rasterio.open(pixel_watersheds_fileName)  # Open pixel_catchments_rasterio_object
     dem_rasterio_object = rasterio.open(dem_fileName)
-        
+
+    # Producing relative elevation model raster
     for ji, window in dem_rasterio_object.block_windows(1):
         dem_window = dem_rasterio_object.read(1,window=window)
         window_shape = dem_window.shape
 
         dem_window = dem_window.ravel()
         catchments_window = pixel_catchments_rasterio_object.read(1,window=window).ravel()
-        
+
         rem_window = calculate_rem(dem_window, catchment_min_dict, catchments_window, meta['nodata'])
         rem_window = rem_window.reshape(window_shape).astype(np.float32)
-        
+
         rem_rasterio_object.write(rem_window, window=window, indexes=1)
-        
+
     dem_rasterio_object.close()
     pixel_catchments_rasterio_object.close()
     rem_rasterio_object.close()
@@ -110,8 +170,11 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
     parser.add_argument('-w','--watersheds',help='Pixel based watersheds raster to use within project path',required=True)
     parser.add_argument('-t','--thalweg-raster',help='A binary raster representing the thalweg. 1 for thalweg, 0 for non-thalweg.',required=True)
     parser.add_argument('-o','--rem',help='Output REM raster',required=True)
-    
-    
+    parser.add_argument('-i','--hydroid', help='HydroID raster to use within project path', required=True)
+    parser.add_argument('-r','--hand_ref_elev_table',help='Output table of HAND reference elev by catchment',required=True)
+    parser.add_argument('-s','--dem_reaches_in_out',help='DEM derived reach layer to join HAND reference elevation attribute',required=True)
+
+
     # extract to dictionary
     args = vars(parser.parse_args())
 
@@ -120,5 +183,8 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
     pixel_watersheds_fileName = args['watersheds']
     rem_fileName = args['rem']
     thalweg_raster = args['thalweg_raster']
+    hydroid_fileName = args['hydroid']
+    hand_ref_elev_fileName = args['hand_ref_elev_table']
+    dem_reaches_filename = args['dem_reaches_in_out']
 
-    rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster)
+    rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, hand_ref_elev_fileName, dem_reaches_filename)
diff --git a/lib/run_by_unit.sh b/lib/run_by_unit.sh
index 9f34bc222..3501bfdbd 100755
--- a/lib/run_by_unit.sh
+++ b/lib/run_by_unit.sh
@@ -321,7 +321,7 @@ echo -e $startDiv"D8 REM $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/rem.tif ] && \
-$libDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels
+$libDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -r $outputHucDataDir/hand_ref_elev_table.csv -s $outputHucDataDir/demDerived_reaches_split.gpkg
 Tcount
 
 ## DINF DISTANCE DOWN ##

From 0412e412c096edb8e773b8a9be18866ecae56496 Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Fri, 19 Feb 2021 12:29:17 -0600
Subject: [PATCH 020/359] Rating curves for short stream segments are replaced
 with rating curves from upstream/downstream segments.

 - Short stream segments are identified and are reassigned the channel geometry from upstream/downstream segment.
 - fossid renamed to fimid and the attribute's starting value is now 1000 to avoid HydroIDs with leading zeroes.
 - Addresses issue where HydroIDs were not included in final hydrotable.
 - Added import sys to inundation.py (missing from previous feature branch).
 - Variable names and general workflow are cleaned up.

This resolves #100.
---
 CHANGELOG.md                                |  13 +++
 config/params_calibrated.env                |   2 +
 config/params_template.env                  |   2 +
 lib/acquire_and_preprocess_inputs.py        |  26 ++---
 lib/add_crosswalk.py                        | 103 +++++++++++++++++---
 lib/build_stream_traversal.py               |   1 -
 lib/filter_catchments_and_add_attributes.py |  13 ++-
 lib/fr_to_ms_raster_mask.py                 |  48 ++++-----
 lib/output_cleanup.py                       |   3 +-
 lib/run_by_unit.sh                          |   6 +-
 lib/split_flows.py                          |  29 +++---
 lib/utils/shared_variables.py               |  10 +-
 tests/inundation.py                         |   5 +-
 13 files changed, 176 insertions(+), 85 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b469c5c2..cdcc09070 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.4.4 - 2021-02-19 - [PR #266](https://github.com/NOAA-OWP/cahaba/pull/266)
+
+Rating curves for short stream segments are replaced with rating curves from upstream/downstream segments.
+
+### Changes
+
+ - Short stream segments are identified and are reassigned the channel geometry from upstream/downstream segment.
+ - `fossid` renamed to `fimid` and the attribute's starting value is now 1000 to avoid HydroIDs with leading zeroes.
+ - Addresses issue where HydroIDs were not included in final hydrotable.
+ - Added `import sys` to `inundation.py` (missing from previous feature branch).
+ - Variable names and general workflow are cleaned up.
+
+<br/><br/>
 ## v3.0.4.3 - 2021-02-12 - [PR #254](https://github.com/NOAA-OWP/cahaba/pull/254)
 
 Modified `rem.py` with a new function to output HAND reference elev.
diff --git a/config/params_calibrated.env b/config/params_calibrated.env
index ff0da17b0..c5b040215 100644
--- a/config/params_calibrated.env
+++ b/config/params_calibrated.env
@@ -16,6 +16,8 @@ export stage_min_meters=0
 export stage_interval_meters=0.3048
 export stage_max_meters=25
 export slope_min=0.001
+export min_catchment_area=0.25
+export min_stream_length=0.5
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/config/params_template.env b/config/params_template.env
index b250de4bf..d6c9e2865 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -16,6 +16,8 @@ export stage_min_meters=0
 export stage_interval_meters=0.3048
 export stage_max_meters=25
 export slope_min=0.001
+export min_catchment_area=0.25
+export min_stream_length=0.5
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/lib/acquire_and_preprocess_inputs.py b/lib/acquire_and_preprocess_inputs.py
index bace41e19..30ce7974a 100755
--- a/lib/acquire_and_preprocess_inputs.py
+++ b/lib/acquire_and_preprocess_inputs.py
@@ -6,7 +6,7 @@
 import sys
 import shutil
 from multiprocessing import Pool
-import geopandas as gp
+import geopandas as gpd
 from urllib.error import HTTPError
 from tqdm import tqdm
 
@@ -18,7 +18,7 @@
                                     NHD_VECTOR_EXTRACTION_SUFFIX,
                                     PREP_PROJECTION,
                                     WBD_NATIONAL_URL,
-                                    FOSS_ID,
+                                    FIM_ID,
                                     OVERWRITE_WBD,
                                     OVERWRITE_NHD,
                                     OVERWRITE_ALL)
@@ -32,7 +32,7 @@
 
 def subset_wbd_to_nwm_domain(wbd,nwm_file_to_use):
 
-    intersecting_indices = [not (gp.read_file(nwm_file_to_use,mask=b).empty) for b in wbd.geometry]
+    intersecting_indices = [not (gpd.read_file(nwm_file_to_use,mask=b).empty) for b in wbd.geometry]
 
     return(wbd[intersecting_indices])
 
@@ -71,16 +71,16 @@ def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_
 
         procs_list, wbd_gpkg_list = [], []
         multilayer_wbd_geopackage = os.path.join(wbd_directory, 'WBD_National.gpkg')
-        # Add fossid to HU8, project, and convert to geopackage. Code block from Brian Avant.
+        # Add fimid to HU8, project, and convert to geopackage.
         if os.path.isfile(multilayer_wbd_geopackage):
             os.remove(multilayer_wbd_geopackage)
         print("Making National WBD GPKG...")
         print("\tWBDHU8")
-        wbd_hu8 = gp.read_file(wbd_gdb_path, layer='WBDHU8')
+        wbd_hu8 = gpd.read_file(wbd_gdb_path, layer='WBDHU8')
         wbd_hu8 = wbd_hu8.rename(columns={'huc8':'HUC8'}) # rename column to caps
         wbd_hu8 = wbd_hu8.sort_values('HUC8')
-        fossids = [str(item).zfill(4) for item in list(range(1, 1 + len(wbd_hu8)))]
-        wbd_hu8[FOSS_ID] = fossids
+        fimids = [str(item).zfill(4) for item in list(range(1000, 1000 + len(wbd_hu8)))]
+        wbd_hu8[FIM_ID] = fimids
         wbd_hu8 = wbd_hu8.to_crs(PREP_PROJECTION)  # Project.
         wbd_hu8 = subset_wbd_to_nwm_domain(wbd_hu8,nwm_file_to_use)
         wbd_hu8.geometry = wbd_hu8.buffer(0)
@@ -93,7 +93,7 @@ def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_
         for wbd_layer_num in ['4', '6']:
             wbd_layer = 'WBDHU' + wbd_layer_num
             print("\t{}".format(wbd_layer))
-            wbd = gp.read_file(wbd_gdb_path,layer=wbd_layer)
+            wbd = gpd.read_file(wbd_gdb_path,layer=wbd_layer)
             wbd = wbd.to_crs(PREP_PROJECTION)
             wbd = wbd.rename(columns={'huc'+wbd_layer_num : 'HUC' + wbd_layer_num})
             wbd = subset_wbd_to_nwm_domain(wbd,nwm_file_to_use)
@@ -204,15 +204,15 @@ def pull_and_prepare_nhd_data(args):
         huc = os.path.split(nhd_vector_extraction_parent)[1]  # Parse HUC.
         os.system("7za x {nhd_vector_extraction_path} -o{nhd_vector_extraction_parent}".format(nhd_vector_extraction_path=nhd_vector_extraction_path, nhd_vector_extraction_parent=nhd_vector_extraction_parent))
         # extract input stream network
-        nhd = gp.read_file(nhd_gdb,layer='NHDPlusBurnLineEvent')
+        nhd = gpd.read_file(nhd_gdb,layer='NHDPlusBurnLineEvent')
         nhd = nhd.to_crs(PREP_PROJECTION)
         nhd.to_file(os.path.join(nhd_vector_extraction_parent, 'NHDPlusBurnLineEvent' + huc + '.gpkg'),driver='GPKG')
         # extract flowlines for FType attributes
-        nhd = gp.read_file(nhd_gdb,layer='NHDFlowline')
+        nhd = gpd.read_file(nhd_gdb,layer='NHDFlowline')
         nhd = nhd.to_crs(PREP_PROJECTION)
         nhd.to_file(os.path.join(nhd_vector_extraction_parent, 'NHDFlowline' + huc + '.gpkg'),driver='GPKG')
         # extract attributes
-        nhd = gp.read_file(nhd_gdb,layer='NHDPlusFlowLineVAA')
+        nhd = gpd.read_file(nhd_gdb,layer='NHDPlusFlowLineVAA')
         nhd.to_file(os.path.join(nhd_vector_extraction_parent, 'NHDPlusFlowLineVAA' + huc + '.gpkg'),driver='GPKG')
         # -- Project and convert NHDPlusBurnLineEvent and NHDPlusFlowLineVAA vectors to geopackage -- #
         #for nhd_layer in ['NHDPlusBurnLineEvent', 'NHDPlusFlowlineVAA']:
@@ -245,7 +245,7 @@ def build_huc_list_files(path_to_saved_data_parent_dir, wbd_directory):
     huc_gpkg = 'WBDHU8' # The WBDHU4 are handled by the nhd_plus_raster_dir name.
 
     # Open geopackage.
-    wbd = gp.read_file(full_huc_gpkg, layer=huc_gpkg)
+    wbd = gpd.read_file(full_huc_gpkg, layer=huc_gpkg)
 
     # Loop through entries and compare against the huc4_list to get available HUCs within the geopackage domain.
     for index, row in tqdm(wbd.iterrows(),total=len(wbd)):
@@ -253,7 +253,7 @@ def build_huc_list_files(path_to_saved_data_parent_dir, wbd_directory):
         huc_mask = wbd.loc[wbd[str("HUC" + huc_gpkg[-1])]==huc].geometry
         burnline = os.path.join(nhd_plus_vector_dir, huc[0:4], 'NHDPlusBurnLineEvent' + huc[0:4] + '.gpkg')
         if os.path.exists(burnline):
-            nhd_test = len(gp.read_file(burnline, mask = huc_mask)) # this is slow, iterates through 2000+ HUC8s
+            nhd_test = len(gpd.read_file(burnline, mask = huc_mask)) # this is slow, iterates through 2000+ HUC8s
             # Append huc to huc8 list.
             if (str(huc[:4]) in huc4_list) & (nhd_test>0):
                 huc8_list.append(huc)
diff --git a/lib/add_crosswalk.py b/lib/add_crosswalk.py
index d3f052c3a..a0e57c7de 100755
--- a/lib/add_crosswalk.py
+++ b/lib/add_crosswalk.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import os
 import geopandas as gpd
 import pandas as pd
 from numpy import unique
@@ -8,13 +9,16 @@
 import argparse
 import sys
 from utils.shared_functions import getDriver
+from utils.shared_variables import FIM_ID
 
-def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,calibration_mode=False):
+def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
 
     input_catchments = gpd.read_file(input_catchments_fileName)
     input_flows = gpd.read_file(input_flows_fileName)
     input_huc = gpd.read_file(input_huc_fileName)
     input_nwmflows = gpd.read_file(input_nwmflows_fileName)
+    min_catchment_area = float(os.environ['min_catchment_area'])
+    min_stream_length = float(os.environ['min_stream_length'])
 
     if extent == 'FR':
         ## crosswalk using majority catchment method
@@ -33,15 +37,15 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
         relevant_input_nwmflows = input_nwmflows[input_nwmflows['feature_id'].isin(input_majorities['feature_id'])]
         relevant_input_nwmflows = relevant_input_nwmflows.filter(items=['feature_id','order_'])
 
-        if calibration_mode == False:
-            if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
-            output_catchments = input_catchments.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
-            output_catchments = output_catchments.merge(relevant_input_nwmflows[['order_','feature_id']],on='feature_id')
+        if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
+        output_catchments = input_catchments.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
+        output_catchments = output_catchments.merge(relevant_input_nwmflows[['order_','feature_id']],on='feature_id')
 
         if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
         output_flows = input_flows.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
         if output_flows.HydroID.dtype != 'int': output_flows.HydroID = output_flows.HydroID.astype(int)
         output_flows = output_flows.merge(relevant_input_nwmflows[['order_','feature_id']],on='feature_id')
+        output_flows = output_flows.merge(output_catchments.filter(items=['HydroID','areasqkm']),on='HydroID')
 
     elif extent == 'MS':
         ## crosswalk using stream segment midpoint method
@@ -89,12 +93,12 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
             print ("No relevant streams within HUC boundaries.")
             sys.exit(0)
 
-        if calibration_mode == False:
-            if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
-            output_catchments = input_catchments.merge(crosswalk,on='HydroID')
+        if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
+        output_catchments = input_catchments.merge(crosswalk,on='HydroID')
 
         if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
         output_flows = input_flows.merge(crosswalk,on='HydroID')
+        output_flows = output_flows.merge(output_catchments.filter(items=['HydroID','areasqkm']),on='HydroID')
 
     # read in manning's n values
     if calibration_mode == False:
@@ -108,6 +112,57 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
     output_flows['ManningN'] = output_flows['order_'].astype(str).map(mannings_dict)
 
+    if output_flows.NextDownID.dtype != 'int': output_flows.NextDownID = output_flows.NextDownID.astype(int)
+
+    # Adjust short model reach rating curves
+    print("Adjusting model reach rating curves")
+    sml_segs = pd.DataFrame()
+
+    # replace small segment geometry with neighboring stream
+    for stream_index in output_flows.index:
+
+        if output_flows["areasqkm"][stream_index] < min_catchment_area and output_flows["LengthKm"][stream_index] < min_stream_length and output_flows["LakeID"][stream_index] < 0:
+
+            short_id = output_flows['HydroID'][stream_index]
+            to_node = output_flows['To_Node'][stream_index]
+            from_node = output_flows['From_Node'][stream_index]
+
+            # multiple upstream segments
+            if len(output_flows.loc[output_flows['NextDownID'] == short_id]['HydroID']) > 1:
+                max_order = max(output_flows.loc[output_flows['NextDownID'] == short_id]['order_']) # drainage area would be better than stream order but we would need to calculate
+
+                if len(output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID']) == 1:
+                    update_id = output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID'].item()
+
+                else:
+                    update_id = output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+
+            # single upstream segments
+            elif len(output_flows.loc[output_flows['NextDownID'] == short_id]['HydroID']) == 1:
+                update_id = output_flows.loc[output_flows.To_Node==from_node]['HydroID'].item()
+
+            # no upstream segments; multiple downstream segments
+            elif len(output_flows.loc[output_flows.From_Node==to_node]['HydroID']) > 1:
+                max_order = max(output_flows.loc[output_flows.From_Node==to_node]['HydroID']['order_']) # drainage area would be better than stream order but we would need to calculate
+
+                if len(output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID']) == 1:
+                    update_id = output_flows.loc[(output_flows.From_Node==to_node) & (output_flows['order_'] == max_order)]['HydroID'].item()
+
+                else:
+                    update_id = output_flows.loc[(output_flows.From_Node==to_node) & (output_flows['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+
+            # no upstream segments; single downstream segment
+            elif len(output_flows.loc[output_flows.From_Node==to_node]['HydroID']) == 1:
+                    update_id = output_flows.loc[output_flows.From_Node==to_node]['HydroID'].item()
+
+            else:
+                update_id = output_flows.loc[output_flows.HydroID==short_id]['HydroID'].item()
+
+            str_order = output_flows.loc[output_flows.HydroID==short_id]['order_'].item()
+            sml_segs = sml_segs.append({'short_id':short_id, 'update_id':update_id, 'str_order':str_order}, ignore_index=True)
+
+    print("Number of short reaches [{} < {} and {} < {}] = {}".format("areasqkm", min_catchment_area, "LengthKm", min_stream_length, len(sml_segs)))
+
     # calculate src_full
     input_src_base = pd.read_csv(input_srcbase_fileName, dtype= object)
     if input_src_base.CatchId.dtype != 'int': input_src_base.CatchId = input_src_base.CatchId.astype(int)
@@ -131,6 +186,21 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_src = input_src_base.drop(columns=['CatchId'])
     if output_src.HydroID.dtype != 'int': output_src.HydroID = output_src.HydroID.astype(int)
 
+    # update rating curves
+    if len(sml_segs) > 0:
+
+        sml_segs.to_csv(small_segments_filename,index=False)
+        print("Update rating curves for short reaches.")
+
+        for index, segment in sml_segs.iterrows():
+
+            short_id = segment[0]
+            update_id= segment[1]
+            new_values = output_src.loc[output_src['HydroID'] == update_id][['Stage', 'Discharge (m3s-1)']]
+
+            for src_index, src_stage in new_values.iterrows():
+                output_src.loc[(output_src['HydroID']== short_id) & (output_src['Stage']== src_stage[0]),['Discharge (m3s-1)']] = src_stage[1]
+
     if extent == 'FR':
         output_src = output_src.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
     elif extent == 'MS':
@@ -142,22 +212,21 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     # make hydroTable
     output_hydro_table = output_src.loc[:,['HydroID','feature_id','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
+
     if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
-    output_hydro_table['HydroID'] = output_hydro_table.HydroID.str.zfill(8)
-    output_hydro_table['fossid'] = output_hydro_table.loc[:,'HydroID'].apply(lambda x : str(x)[0:4])
-    if input_huc.fossid.dtype != 'str': input_huc.fossid = input_huc.fossid.astype(str)
+    output_hydro_table[FIM_ID] = output_hydro_table.loc[:,'HydroID'].apply(lambda x : str(x)[0:4])
+
+    if input_huc[FIM_ID].dtype != 'str': input_huc[FIM_ID] = input_huc[FIM_ID].astype(str)
+    output_hydro_table = output_hydro_table.merge(input_huc.loc[:,[FIM_ID,'HUC8']],how='left',on=FIM_ID)
 
-    output_hydro_table = output_hydro_table.merge(input_huc.loc[:,['fossid','HUC8']],how='left',on='fossid')
     if output_flows.HydroID.dtype != 'str': output_flows.HydroID = output_flows.HydroID.astype(str)
-    output_flows['HydroID'] = output_flows.HydroID.str.zfill(8)
     output_hydro_table = output_hydro_table.merge(output_flows.loc[:,['HydroID','LakeID']],how='left',on='HydroID')
     output_hydro_table['LakeID'] = output_hydro_table['LakeID'].astype(int)
 
     output_hydro_table = output_hydro_table.rename(columns={'HUC8':'HUC'})
     if output_hydro_table.HUC.dtype != 'str': output_hydro_table.HUC = output_hydro_table.HUC.astype(str)
-    output_hydro_table.HUC = output_hydro_table.HUC.str.zfill(8)
 
-    output_hydro_table.drop(columns='fossid',inplace=True)
+    output_hydro_table.drop(columns=FIM_ID,inplace=True)
     if output_hydro_table.feature_id.dtype != 'int': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(int)
     if output_hydro_table.feature_id.dtype != 'str': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(str)
 
@@ -207,6 +276,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     parser.add_argument('-m','--mannings-n',help='Mannings n. Accepts single parameter set or list of parameter set in calibration mode. Currently input as csv.',required=True)
     parser.add_argument('-z','--input-nwmcat-fileName',help='NWM catchment polygon',required=True)
     parser.add_argument('-p','--extent',help='MS or FR extent',required=True)
+    parser.add_argument('-k','--small-segments-filename',help='output list of short segments',required=True)
     parser.add_argument('-c','--calibration-mode',help='Mannings calibration flag',required=False,action='store_true')
 
     args = vars(parser.parse_args())
@@ -226,6 +296,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     mannings_n = args['mannings_n']
     input_nwmcat_fileName = args['input_nwmcat_fileName']
     extent = args['extent']
+    small_segments_filename = args['small_segments_filename']
     calibration_mode = args['calibration_mode']
 
-    add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,calibration_mode)
+    add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode)
diff --git a/lib/build_stream_traversal.py b/lib/build_stream_traversal.py
index daca019f2..672fe8a01 100644
--- a/lib/build_stream_traversal.py
+++ b/lib/build_stream_traversal.py
@@ -7,7 +7,6 @@
         hydro_id       = name of ID column (string)
 '''
 import sys
-import datetime
 import pandas as pd
 import argparse
 import geopandas as gpd
diff --git a/lib/filter_catchments_and_add_attributes.py b/lib/filter_catchments_and_add_attributes.py
index ad9e6e543..104bc4275 100755
--- a/lib/filter_catchments_and_add_attributes.py
+++ b/lib/filter_catchments_and_add_attributes.py
@@ -5,6 +5,7 @@
 import numpy as np
 import argparse
 import sys
+from utils.shared_variables import FIM_ID
 
 input_catchments_fileName = sys.argv[1]
 input_flows_fileName = sys.argv[2]
@@ -16,30 +17,28 @@
 input_catchments = gpd.read_file(input_catchments_fileName)
 wbd = gpd.read_file(wbd_fileName)
 input_flows = gpd.read_file(input_flows_fileName)
-# must drop leading zeroes
-select_flows = tuple(map(str,map(int,wbd[wbd.HUC8.str.contains(hucCode)].fossid)))
+
+# filter segments within huc boundary
+select_flows = tuple(map(str,map(int,wbd[wbd.HUC8.str.contains(hucCode)][FIM_ID])))
 
 if input_flows.HydroID.dtype != 'str': input_flows.HydroID = input_flows.HydroID.astype(str)
 output_flows = input_flows[input_flows.HydroID.str.startswith(select_flows)].copy()
 if output_flows.HydroID.dtype != 'int': output_flows.HydroID = output_flows.HydroID.astype(int)
 
 if len(output_flows) > 0:
+    
     # merges input flows attributes and filters hydroids
     if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
     output_catchments = input_catchments.merge(output_flows.drop(['geometry'],axis=1),on='HydroID')
 
     # filter out smaller duplicate features
     duplicateFeatures = np.where(np.bincount(output_catchments['HydroID'])>1)[0]
-    # print(duplicateFeatures)
 
     for dp in duplicateFeatures:
-        # print(dp)
+
         indices_of_duplicate = np.where(output_catchments['HydroID'] == dp)[0]
-        # print(indices_of_duplicate)
         areas = output_catchments.iloc[indices_of_duplicate,:].geometry.area
-        # print(areas)
         indices_of_smaller_duplicates = indices_of_duplicate[np.where(areas != np.amax(areas))[0]]
-        # print(indices_of_smaller_duplicates)
         output_catchments = output_catchments.drop(output_catchments.index[indices_of_smaller_duplicates])
 
     # add geometry column
diff --git a/lib/fr_to_ms_raster_mask.py b/lib/fr_to_ms_raster_mask.py
index 3f4d6e9b2..567b5cf19 100755
--- a/lib/fr_to_ms_raster_mask.py
+++ b/lib/fr_to_ms_raster_mask.py
@@ -9,30 +9,30 @@
 import geopandas as gpd
 import rasterio.mask
 
-split_flows_fileName      = sys.argv[1]
-fdrFR                     = sys.argv[2]
-nhddemFR                  = sys.argv[3]
-slpFR                     = sys.argv[4]
-fdrMSname                 = sys.argv[5]
-nhddemMSname              = sys.argv[6]
-slpMSname                 = sys.argv[7]
-strpixelFR                = sys.argv[8]
-strpixelMSname            = sys.argv[9]
-floodAOIbuf               = sys.argv[10]
+split_flows_fileName    = sys.argv[1]
+fdr_fr                  = sys.argv[2]
+dem_fr                  = sys.argv[3]
+slope_fr                = sys.argv[4]
+fdr_ms_filename         = sys.argv[5]
+dem_ms_filename         = sys.argv[6]
+slope_ms_filename       = sys.argv[7]
+str_pixel_fr            = sys.argv[8]
+str_pixel_ms_filename   = sys.argv[9]
+ms_buffer_dist          = int(os.environ['ms_buffer_dist'])
 
 # create output layer names
 split_flows = gpd.read_file(split_flows_fileName)
 
 # Limit the rasters to the buffer distance around the draft streams.
-print ("Limiting rasters to buffer area ({} meters) around model streams".format(str(floodAOIbuf)))
+print ("Limiting rasters to buffer area ({} meters) around model streams".format(str(ms_buffer_dist)))
 
-MSsplit_flows_gdf_buffered = split_flows.unary_union.buffer(int(floodAOIbuf))
+split_flows_ms_buffer = split_flows.unary_union.buffer(ms_buffer_dist)
 
 print('Writing raster outputs ...')
 
 # Mask nhddem
-with rasterio.open(nhddemFR) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [MSsplit_flows_gdf_buffered], crop=True)
+with rasterio.open(dem_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
     out_meta = src.meta
 
 out_meta.update({"driver": "GTiff",
@@ -40,12 +40,12 @@
      "width": out_image.shape[2],
      "transform": out_transform})
 
-with rasterio.open(os.path.join(os.path.dirname(nhddemFR), nhddemMSname), "w", **out_meta) as dest:
+with rasterio.open(os.path.join(os.path.dirname(dem_fr), dem_ms_filename), "w", **out_meta) as dest:
     dest.write(out_image)
 
 # Mask fdr
-with rasterio.open(fdrFR) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [MSsplit_flows_gdf_buffered], crop=True)
+with rasterio.open(fdr_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
     out_meta = src.meta
 
 out_meta.update({"driver": "GTiff",
@@ -53,12 +53,12 @@
      "width": out_image.shape[2],
      "transform": out_transform})
 
-with rasterio.open(os.path.join(os.path.dirname(fdrFR), fdrMSname), "w", **out_meta) as dest:
+with rasterio.open(os.path.join(os.path.dirname(fdr_fr), fdr_ms_filename), "w", **out_meta) as dest:
     dest.write(out_image)
 
 # Mask slope
-with rasterio.open(slpFR) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [MSsplit_flows_gdf_buffered], crop=True)
+with rasterio.open(slope_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
     out_meta = src.meta
 
 out_meta.update({"driver": "GTiff",
@@ -66,12 +66,12 @@
      "width": out_image.shape[2],
      "transform": out_transform})
 
-with rasterio.open(os.path.join(os.path.dirname(slpFR), slpMSname), "w", **out_meta) as dest:
+with rasterio.open(os.path.join(os.path.dirname(slope_fr), slope_ms_filename), "w", **out_meta) as dest:
     dest.write(out_image)
 
 # Mask stream pixels
-with rasterio.open(strpixelFR) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [MSsplit_flows_gdf_buffered], crop=True)
+with rasterio.open(str_pixel_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
     out_meta = src.meta
 
 out_meta.update({"driver": "GTiff",
@@ -79,5 +79,5 @@
      "width": out_image.shape[2],
      "transform": out_transform})
 
-with rasterio.open(os.path.join(os.path.dirname(strpixelFR), strpixelMSname), "w", **out_meta) as dest:
+with rasterio.open(os.path.join(os.path.dirname(str_pixel_fr), str_pixel_ms_filename), "w", **out_meta) as dest:
     dest.write(out_image)
diff --git a/lib/output_cleanup.py b/lib/output_cleanup.py
index b253598cf..7e211bdc5 100755
--- a/lib/output_cleanup.py
+++ b/lib/output_cleanup.py
@@ -30,7 +30,8 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg',
         'gw_catchments_reaches_filtered_addedAttributes.tif',
         'hydroTable.csv',
-        'src.json'
+        'src.json',
+        'small_segments.csv'
     ]
 
     # List of files that will be saved during a viz run
diff --git a/lib/run_by_unit.sh b/lib/run_by_unit.sh
index 3501bfdbd..2ba820a32 100755
--- a/lib/run_by_unit.sh
+++ b/lib/run_by_unit.sh
@@ -258,7 +258,7 @@ echo -e $startDiv"Split Derived Reaches $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ] && \
-$libDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $max_split_distance_meters $slope_min $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg $lakes_buffer_dist_meters
+$libDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 Tcount
 
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
@@ -272,7 +272,7 @@ if [ "$extent" = "MS" ]; then
   echo -e $startDiv"Mask Rasters with Stream Buffer $hucNumber"$stopDiv
   date -u
   Tstart
-  $libDir/fr_to_ms_raster_mask.py $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/flowdir_d8_burned_filled.tif $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/slopes_d8_dem_meters.tif $outputHucDataDir/flowdir_d8_MS.tif $outputHucDataDir/dem_thalwegCond_MS.tif $outputHucDataDir/slopes_d8_dem_metersMS.tif $outputHucDataDir/demDerived_streamPixels.tif $outputHucDataDir/demDerived_streamPixelsMS.tif $ms_buffer_dist
+  $libDir/fr_to_ms_raster_mask.py $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/flowdir_d8_burned_filled.tif $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/slopes_d8_dem_meters.tif $outputHucDataDir/flowdir_d8_MS.tif $outputHucDataDir/dem_thalwegCond_MS.tif $outputHucDataDir/slopes_d8_dem_metersMS.tif $outputHucDataDir/demDerived_streamPixels.tif $outputHucDataDir/demDerived_streamPixelsMS.tif
   Tcount
 
   if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
@@ -429,7 +429,7 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-$libDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent
+$libDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
 ## CLEANUP OUTPUTS ##
diff --git a/lib/split_flows.py b/lib/split_flows.py
index c8de19b34..67b69f7e9 100755
--- a/lib/split_flows.py
+++ b/lib/split_flows.py
@@ -4,7 +4,7 @@
 Description:
     1) split stream segments based on lake boundaries and input threshold distance
     2) calculate channel slope, manning's n, and LengthKm for each segment
-    3) create unique ids using HUC8 boundaries (and unique 'fossid' column)
+    3) create unique ids using HUC8 boundaries (and unique FIM_ID column)
     4) create network traversal attribute columns (To_Node, From_Node, NextDownID)
     5) create points layer with segment verticies encoded with HydroID's (used for catchment delineation in next step)
 '''
@@ -19,20 +19,21 @@
 from tqdm import tqdm
 import time
 from os.path import isfile
-from os import remove
+from os import remove,environ
 from collections import OrderedDict
 import build_stream_traversal
 from utils.shared_functions import getDriver
+from utils.shared_variables import FIM_ID
 
 flows_fileName         = sys.argv[1]
 dem_fileName           = sys.argv[2]
 split_flows_fileName   = sys.argv[3]
 split_points_fileName  = sys.argv[4]
-maxLength              = float(sys.argv[5])
-slope_min              = float(sys.argv[6])
-wbd8_clp_filename      = sys.argv[7]
-lakes_filename         = sys.argv[8]
-lakes_buffer_input     = float(sys.argv[9])
+wbd8_clp_filename      = sys.argv[5]
+lakes_filename         = sys.argv[6]
+max_length             = float(environ['max_split_distance_meters'])
+slope_min              = float(environ['slope_min'])
+lakes_buffer_input     = float(environ['lakes_buffer_dist_meters'])
 
 wbd = gpd.read_file(wbd8_clp_filename)
 
@@ -46,15 +47,15 @@
     sys.exit(0)
 
 wbd8 = gpd.read_file(wbd8_clp_filename)
-#dem = Raster(dem_fileName)
 dem = rasterio.open(dem_fileName,'r')
+
 if isfile(lakes_filename):
     lakes = gpd.read_file(lakes_filename)
 else:
     lakes = None
 
-wbd8 = wbd8.filter(items=['fossid', 'geometry'])
-wbd8 = wbd8.set_index('fossid')
+wbd8 = wbd8.filter(items=[FIM_ID, 'geometry'])
+wbd8 = wbd8.set_index(FIM_ID)
 flows = flows.explode()
 
 # temp
@@ -79,7 +80,7 @@
       lakes_buffer = lakes.copy()
       lakes_buffer['geometry'] = lakes.buffer(lakes_buffer_input) # adding X meter buffer for spatial join comparison (currently using 20meters)
 
-print ('splitting ' + str(len(flows)) + ' stream segments based on ' + str(maxLength) + ' m max length')
+print ('splitting ' + str(len(flows)) + ' stream segments based on ' + str(max_length) + ' m max length')
 
 # remove empty geometries
 flows = flows.loc[~flows.is_empty,:]
@@ -92,8 +93,8 @@
     if lineString.length == 0:
         continue
 
-    # existing reaches of less than maxLength
-    if lineString.length < maxLength:
+    # existing reaches of less than max_length
+    if lineString.length < max_length:
         split_flows = split_flows + [lineString]
         line_points = [point for point in zip(*lineString.coords.xy)]
 
@@ -106,7 +107,7 @@
         slopes = slopes + [slope]
         continue
 
-    splitLength = lineString.length / np.ceil(lineString.length / maxLength)
+    splitLength = lineString.length / np.ceil(lineString.length / max_length)
 
     cumulative_line = []
     line_points = []
diff --git a/lib/utils/shared_variables.py b/lib/utils/shared_variables.py
index cf75c733c..40a8feacb 100644
--- a/lib/utils/shared_variables.py
+++ b/lib/utils/shared_variables.py
@@ -21,13 +21,13 @@
 NHD_VECTOR_EXTRACTION_SUFFIX = '_HU4_GDB.zip'
 
 # -- Field Names -- #
-FOSS_ID = 'fossid'
+FIM_ID = 'fimid'
 
 # -- Other -- #
-CONUS_STATE_LIST = {"AL", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA", 
-                    "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", 
-                    "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", 
-                    "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "PR", "RI", "SC", 
+CONUS_STATE_LIST = {"AL", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA",
+                    "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
+                    "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
+                    "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "PR", "RI", "SC",
                     "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY"}
 
 OVERWRITE_WBD = 'OVERWRITE_WBD'
diff --git a/tests/inundation.py b/tests/inundation.py
index 0496d8d1b..679102432 100755
--- a/tests/inundation.py
+++ b/tests/inundation.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import sys
 import numpy as np
 import pandas as pd
 from numba import njit, typeof, typed, types
@@ -456,11 +457,13 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
                                          'HydroID':str,'stage':float,
                                          'discharge_cms':float,'LakeID' : int}
                                 )
+
         hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
+
         hydroTable = hydroTable[hydroTable["LakeID"] == -999]  # Subset hydroTable to include only non-lake catchments.
 
         if hydroTable.empty:
-            print ("All stream segments in this HUC are within lake boundaries.")
+            print ("All stream segments in HUC are within lake boundaries.")
             sys.exit(0)
 
     elif isinstance(hydroTable,pd.DataFrame):

From 584f4a36d2b39fe6b62b46b830a3de4263a10f43 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 22 Feb 2021 09:22:43 -0600
Subject: [PATCH 021/359] Enhancements to allow for evaluation at AHPS sites,
 the generation of a query-optimized metrics CSV, and the generation of
 categorical FIM.

Enhancements to allow for evaluation at AHPS sites, the generation of a query-optimized metrics CSV, and the generation of categorical FIM. This merge requires that the `/test_cases` directory be updated for all machines performing evaluation.

 - `generate_categorical_fim.py` was added to allow production of NWS Flood Categorical HAND FIM (CatFIM) source data. More changes on this script are to follow in subsequent branches.
 - `ble_autoeval.sh` and `all_ble_stats_comparison.py` were deleted because `synthesize_test_cases.py` now handles the merging of metrics.
 - The code block in `run_test_case.py` that was responsible for printing the colored metrics to screen has been commented out because of the new scale of evaluations (formerly in `run_test_case.py`, now in `shared_functions.py`)
 - Remove unused imports from inundation wrappers in `/tools`.
 - Updated `synthesize_test_cases.py` to allow for AHPS site evaluations.
 - Reorganized `run_test_case.py` by moving more functions into `shared_functions.py`.
 - Created more shared variables in `shared_variables.py` and updated import statements in relevant scripts.

This resolves #258 and #259.
---
 CHANGELOG.md                            |  22 ++
 tests/aggregate_metrics.py              | 167 ++++++++-
 tests/all_ble_stats_comparison.py       |  54 ---
 tests/ble_autoeval.sh                   |  89 -----
 tests/cache_metrics.py                  |  88 +++--
 tests/run_test_case.py                  | 452 +++++++-----------------
 tests/synthesize_test_cases.py          | 297 +++++++++++++---
 tests/utils/shared_functions.py         | 234 ++++++++++++
 tests/utils/shared_variables.py         |  20 ++
 tools/generate_categorical_fim.py       | 155 ++++++++
 tools/inundation_wrapper_custom_flow.py |   6 -
 tools/inundation_wrapper_nwm_flows.py   |   5 -
 12 files changed, 1004 insertions(+), 585 deletions(-)
 delete mode 100755 tests/all_ble_stats_comparison.py
 delete mode 100755 tests/ble_autoeval.sh
 create mode 100644 tests/utils/shared_variables.py
 create mode 100644 tools/generate_categorical_fim.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cdcc09070..f7f97b7dd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,28 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.5.0 - 2021-02-22 - [PR #267](https://github.com/NOAA-OWP/cahaba/pull/267)
+
+Enhancements to allow for evaluation at AHPS sites, the generation of a query-optimized metrics CSV, and the generation of categorical FIM. This merge requires that the `/test_cases` directory be updated for all machines performing evaluation.
+
+### Additions
+
+ - `generate_categorical_fim.py` was added to allow production of NWS Flood Categorical HAND FIM (CatFIM) source data. More changes on this script are to follow in subsequent branches.
+
+### Removals
+
+ - `ble_autoeval.sh` and `all_ble_stats_comparison.py` were deleted because `synthesize_test_cases.py` now handles the merging of metrics.
+ - The code block in `run_test_case.py` that was responsible for printing the colored metrics to screen has been commented out because of the new scale of evaluations (formerly in `run_test_case.py`, now in `shared_functions.py`)
+ - Remove unused imports from inundation wrappers in `/tools`.
+
+### Changes
+
+ - Updated `synthesize_test_cases.py` to allow for AHPS site evaluations.
+ - Reorganized `run_test_case.py` by moving more functions into `shared_functions.py`.
+ - Created more shared variables in `shared_variables.py` and updated import statements in relevant scripts.
+
+<br/><br/>
+
 ## v3.0.4.4 - 2021-02-19 - [PR #266](https://github.com/NOAA-OWP/cahaba/pull/266)
 
 Rating curves for short stream segments are replaced with rating curves from upstream/downstream segments.
diff --git a/tests/aggregate_metrics.py b/tests/aggregate_metrics.py
index 98134da59..d8a462d5b 100644
--- a/tests/aggregate_metrics.py
+++ b/tests/aggregate_metrics.py
@@ -3,15 +3,178 @@
 import json
 import os
 import csv
-
+    
 import argparse
 
-TEST_CASES_DIR = r'/data/test_cases/'
+TEST_CASES_DIR = r'/data/test_cases_new/'
 # TEMP = r'/data/temp'
 
 # Search through all previous_versions in test_cases
 from utils.shared_functions import compute_stats_from_contingency_table
 
+def create_master_metrics_csv():
+            
+    # Construct header
+    metrics_to_write = ['true_negatives_count',
+                        'false_negatives_count',
+                        'true_positives_count',
+                        'false_positives_count',
+                        'contingency_tot_count',
+                        'cell_area_m2',
+                        'TP_area_km2',
+                        'FP_area_km2',
+                        'TN_area_km2',
+                        'FN_area_km2',
+                        'contingency_tot_area_km2',
+                        'predPositive_area_km2',
+                        'predNegative_area_km2',
+                        'obsPositive_area_km2',
+                        'obsNegative_area_km2',
+                        'positiveDiff_area_km2',
+                        'CSI',
+                        'FAR',
+                        'TPR',
+                        'TNR',
+                        'PPV',
+                        'NPV',
+                        'ACC',
+                        'Bal_ACC',
+                        'MCC',
+                        'EQUITABLE_THREAT_SCORE',
+                        'PREVALENCE',
+                        'BIAS',
+                        'F1_SCORE',
+                        'TP_perc',
+                        'FP_perc',
+                        'TN_perc',
+                        'FN_perc',
+                        'predPositive_perc',
+                        'predNegative_perc',
+                        'obsPositive_perc',
+                        'obsNegative_perc',
+                        'positiveDiff_perc',
+                        'masked_count',
+                        'masked_perc',
+                        'masked_area_km2'
+                        ]
+    
+    additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
+    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source']]
+    
+    for benchmark_type in ['ble', 'ahps']:
+        
+        if benchmark_type == 'ble':
+        
+            test_cases = r'/data/test_cases'
+            test_cases_list = os.listdir(test_cases)
+            # AHPS test_ids
+            versions_to_aggregate = ['fim_1_0_0', 'fim_2_3_3', 'fim_3_0_0_3_fr_c']
+                            
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
+                    
+                    for magnitude in ['100yr', '500yr']:
+                        for version in versions_to_aggregate:
+                            version_dir = os.path.join(previous_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+
+                            if os.path.exists(magnitude_dir):
+                                
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f:
+                                        flow = 'NA'
+                                        nws_lid = "NA"
+                                        benchmark_source = 'ble'
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        if os.path.exists(full_json_path):
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            
+                                            list_to_write.append(sub_list_to_append)
+                                                                                    
+                except ValueError:
+                    pass
+                
+        if benchmark_type == 'ahps':
+    
+            test_cases = r'/data/test_cases_ahps_testing'
+            test_cases_list = os.listdir(test_cases)
+            # AHPS test_ids
+            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021', 
+                                     'fim_2_x_ms_nws_1_21_2021', 'fim_2_x_ms_usgs_1_21_2021',
+                                     'fim_3_0_0_3_ms_c_nws_1_21_2021', 'fim_3_0_0_3_ms_c_usgs_1_21_2021',
+                                     'ms_xwalk_fill_missing_cal_nws', 'ms_xwalk_fill_missing_cal_usgs']
+            
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
+                    
+                    for magnitude in ['action', 'minor', 'moderate', 'major']:
+                        for version in versions_to_aggregate:
+                            
+                            if 'nws' in version:
+                                benchmark_source = 'ahps_nws'
+                            if 'usgs' in version:
+                                benchmark_source = 'ahps_usgs'
+                            
+                            version_dir = os.path.join(previous_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+                            
+                            if os.path.exists(magnitude_dir):
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f and 'total_area' not in f:
+                                        nws_lid = f[:5]
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        flow = ''
+                                        if os.path.exists(full_json_path):
+                                            # Get flow used to map.
+                                            if 'usgs' in version:
+                                                parent_dir = 'usgs_1_21_2021'
+                                            if 'nws' in version:
+                                                parent_dir = 'nws_1_21_2021'
+                                                
+                                            flow_file = os.path.join(test_cases, parent_dir, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                                            if os.path.exists(flow_file):
+                                                with open(flow_file, newline='') as csv_file:
+                                                    reader = csv.reader(csv_file)
+                                                    next(reader)
+                                                    for row in reader:
+                                                        flow = row[1]
+                                                    if nws_lid == 'mcc01':
+                                                        print(flow)
+                                            
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            list_to_write.append(sub_list_to_append)
+                                        
+                except ValueError:
+                    pass
+        
+    with open(output_csv, 'w', newline='') as csvfile:
+        csv_writer = csv.writer(csvfile)
+        csv_writer.writerows(list_to_write)
+
+
+
 def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfolder=""):
 
     # Read hucs into list.
diff --git a/tests/all_ble_stats_comparison.py b/tests/all_ble_stats_comparison.py
deleted file mode 100755
index c4683c4e1..000000000
--- a/tests/all_ble_stats_comparison.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import pandas as pd
-import argparse
-
-
-def subset_vector_layers(huclist, branch, current_dev, outfolder):
-
-    test_cases= r'data/test_cases'
-    ble_sitelist = [str(line.rstrip('\n')) for line in open(huclist)]
-    stat_list = ['fim_1_0_0', 'fim_2_3_3',str(current_dev), 'new_feature','eval']
-    eval_all = pd.DataFrame([])
-        
-    # stat_list = stat_list + [branch]
-    for site in ble_sitelist:
-        eval_100_path=os.path.join(test_cases,str(site) + '_ble', 'performance_archive', 'development_versions', branch, '100yr','stats_summary.csv')
-        eval_500_path=os.path.join(test_cases,str(site) + '_ble', 'performance_archive', 'development_versions', branch, '500yr','stats_summary.csv')
-        
-        if os.path.exists(eval_100_path) and os.path.exists(eval_500_path):
-            eval_100 = pd.read_csv(eval_100_path,index_col=0)
-            eval_100['eval'] = '100yr'
-
-            eval_500 = pd.read_csv(eval_500_path,index_col=0)
-            eval_500['eval'] = '500yr'
-
-            eval_combined = eval_100.append(eval_500)
-            eval_combined.columns = ['new_feature' if x==str(branch) else x for x in eval_combined.columns]
-            eval_combined = eval_combined.filter(items=stat_list)
-            eval_combined = eval_combined.reindex(columns=stat_list)
-            eval_combined['site'] = str(site)
-            eval_combined['branch'] = str(branch)
-            eval_all = eval_all.append(eval_combined)
-
-    if not os.path.exists(outfolder):
-        os.makedirs(outfolder)
-    eval_all.to_csv(os.path.join(outfolder,'ble_stats_comparison.csv'))
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Collect eval stats for BLE sites')
-    parser.add_argument('-b','--huclist', help='list of ble sites to test', required=True,type=str)
-    parser.add_argument('-e','--branch', help='list of outfolder(s)', required=False,type=str)
-    parser.add_argument('-d','--current-dev',help='name of current dev stat column',required=True,type=str)
-    parser.add_argument('-f','--outfolder',help='output folder',required=True,type=str)
-
-    args = vars(parser.parse_args())
-
-    huclist = args['huclist']
-    branch = args['branch']
-    current_dev = args['current_dev']
-    outfolder = args['outfolder']
-
-    subset_vector_layers(huclist,branch,current_dev,outfolder)
diff --git a/tests/ble_autoeval.sh b/tests/ble_autoeval.sh
deleted file mode 100755
index ea8af2d94..000000000
--- a/tests/ble_autoeval.sh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/bin/bash -e
-:
-usage ()
-{
-    echo 'Automate runs of run_test_case.py and the aggregation of metrics for list of BLE sites.'
-    echo 'Usage : ble_auto_eval.sh [REQ: -f <fim outfolder(s)> -b <ble list> -d <current dev> -s <outfolder> -v <version>] [OPT: -h -j <job limit>]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -f/--fim-outfolder     : fim output directory(s)'
-    echo '  -b/--ble-list          : list of ble sites to evaluate'
-    echo '  -d/--current-dev       : current archived dev stats column name'
-    echo '  -s/--outfolder         : outfolder name'
-    echo '  -v/--version           : version eval results. options are Options: "DEV" or "PREV"'
-    echo ''
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
-    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    exit
-}
-
-if [ "$#" -lt 7 ]
-then
-  usage
-fi
-
-while [ "$1" != "" ]; do
-case $1
-in
-    -f|--fim_outfolder)
-        shift
-        fim_outfolder="$1"
-        ;;
-    -b|--ble_list)
-        shift
-        ble_list="$1"
-        ;;
-    -d|--current_dev)
-        shift
-        current_dev="$1"
-        ;;
-    -s|--outfolder)
-        shift
-        outfolder="$1"
-        ;;
-    -v|--version)
-        shift
-        version="$1"
-        ;;
-    -j|--jobLimit)
-        shift
-        jobLimit=$1
-        ;;
-    -h|--help)
-        shift
-        usage
-        ;;
-    esac
-    shift
-done
-
-export testDir='foss_fim/tests'
-
-
-for branch in $fim_outfolder
-do
-  echo "processing feature branch: $branch"
-
-  while read p; do
-      # Run Eval
-      if [ -d "/data/outputs/$branch/$p" ]
-      then
-        echo "processing ble for $branch/$p"
-        python3 /$testDir/run_test_case.py -r $branch/$p -t $p"_ble" -b $branch -c
-      fi
-
-      if [ -d "/data/outputs/$branch/$(echo $p| cut -b 1-6)" ]
-      then
-        echo "processing ble for $branch/$(echo $p| cut -b 1-6)"
-        python3 /$testDir/run_test_case.py -r $branch/$(echo $p| cut -b 1-6) -t $p"_ble" -b $branch -c
-      fi
-  done <$ble_list
-done
-
-echo "combining ble metrics"
-python3 /$testDir/all_ble_stats_comparison.py -b $ble_list -e "$fim_outfolder" -d $current_dev -f $outfolder
-
-echo "calculating aggregate metrics"
-python3 /$testDir/aggregate_metrics.py -c $version -b "$fim_outfolder" -u $ble_list -f $outfolder
diff --git a/tests/cache_metrics.py b/tests/cache_metrics.py
index 63571977f..3f601cc5c 100644
--- a/tests/cache_metrics.py
+++ b/tests/cache_metrics.py
@@ -2,11 +2,12 @@
 
 import os
 import argparse
+import traceback
 
 from run_test_case import run_alpha_test
 from multiprocessing import Pool
 
-TEST_CASES_DIR = r'/data/test_cases/'
+TEST_CASES_DIR = r'/data/test_cases_new/'  #TODO remove "_new"
 PREVIOUS_FIM_DIR = r'/data/previous_fim'
 OUTPUTS_DIR = r'/data/outputs'
 
@@ -14,7 +15,7 @@
 def process_alpha_test(args):
     
     fim_run_dir = args[0]
-    branch_name = args[1]
+    version = args[1]
     test_id = args[2]
     magnitude = args[3]
     archive_results = args[4]
@@ -27,9 +28,9 @@ def process_alpha_test(args):
         compare_to_previous = False
 
     try:
-        run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type)
-    except Exception as e:
-        print(e)
+        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type)
+    except Exception:
+        traceback.print_exc()
 
 
 if __name__ == '__main__':
@@ -40,7 +41,7 @@ def process_alpha_test(args):
     parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
     parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
     parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
-    parser.add_argument('-b','--benchmark-category',help='Options include ble or ahps. Defaults to process both.',required=False, default=['ble', 'ahps'])
+    parser.add_argument('-b','--benchmark-category',help='Options include ble or ahps. Defaults to process both.',required=False, default=None)
         
     test_cases_dir_list = os.listdir(TEST_CASES_DIR)
     
@@ -64,44 +65,53 @@ def process_alpha_test(args):
     else:
         print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
     
-    if type(benchmark_category) != list:
-        benchmark_category = [benchmark_category]
+    benchmark_category_list = []
     
+    if benchmark_category == None:
+        for d in test_cases_dir_list:
+            if 'test_cases' in d:
+                benchmark_category_list.append(d.replace('_test_cases', ''))
+    else:
+        benchmark_category_list = [benchmark_category]
+
     procs_list = []
-    for test_id in test_cases_dir_list:
-        if 'validation' and 'other' not in test_id:
-                        
-            current_huc = test_id.split('_')[0]
-            
-            if test_id.split('_')[1] in benchmark_category:
-            
-                for branch_name in previous_fim_list:
-                    
-                    if config == 'DEV':
-                        fim_run_dir = os.path.join(OUTPUTS_DIR, branch_name, current_huc)
-                    elif config == 'PREV':
-                        fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, branch_name, current_huc)
-                        
-                    if os.path.exists(fim_run_dir):    
+    for bench_cat in benchmark_category_list:
+        bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
+        
+        bench_cat_test_case_list = os.listdir(bench_cat_test_case_dir)
+    
+        for test_id in bench_cat_test_case_list:
+            if 'validation' and 'other' not in test_id:
+                            
+                current_huc = test_id.split('_')[0]
+                if test_id.split('_')[1] in bench_cat:
+                
+                    for version in previous_fim_list:
                         
-                        if special_string != "":
-                            branch_name = branch_name + '_' + special_string
+                        if config == 'DEV':
+                            fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
+                        elif config == 'PREV':
+                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
+                                                
+                        if not os.path.exists(fim_run_dir):
+                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  # For previous versions of HAND computed at HUC6 scale
                         
-                        if 'ble' in test_id:
-                            magnitude = ['100yr', '500yr']
-                        elif 'ahps' in test_id:
-                            magnitude = ['action', 'minor', 'moderate', 'major']
-                        else:
-                            continue
-                    
-                        print("Adding " + test_id + " to list of test_ids to process...")
-                        if job_number > 1:
-                            procs_list.append([fim_run_dir, branch_name, test_id, magnitude, archive_results])
-                        else:
-                            process_alpha_test([fim_run_dir, branch_name, test_id, magnitude, archive_results])
+                        if os.path.exists(fim_run_dir):
+                            if special_string != "":
+                                version = version + '_' + special_string
                             
-            else:
-                print("No test_ids were found for the provided benchmark category: " + str(benchmark_category))
+                            if 'ble' in test_id:
+                                magnitude = ['100yr', '500yr']
+                            elif 'usgs' or 'nws' in test_id:
+                                magnitude = ['action', 'minor', 'moderate', 'major']
+                            else:
+                                continue
+                        
+                            print("Adding " + test_id + " to list of test_ids to process...")
+                            if job_number > 1:
+                                procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results])
+                            else:                            
+                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results])
 
     if job_number > 1:
         pool = Pool(job_number)
diff --git a/tests/run_test_case.py b/tests/run_test_case.py
index 16f50b882..a11fa9ab5 100755
--- a/tests/run_test_case.py
+++ b/tests/run_test_case.py
@@ -2,152 +2,36 @@
 
 import os
 import sys
-import pandas as pd
-import geopandas as gpd
-import rasterio
-import json
-import csv
-import argparse
 import shutil
+import argparse
 
-from utils.shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
+from utils.shared_functions import compute_contingency_stats_from_rasters
+from utils.shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
 from inundation import inundate
 
-TEST_CASES_DIR = r'/data/test_cases/'  # Will update.
-INPUTS_DIR = r'/data/inputs'
-PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
-GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
-GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
-OUTPUTS_DIR = os.environ['outputDataDir']
-
-ENDC = '\033[m'
-TGREEN_BOLD = '\033[32;1m'
-TGREEN = '\033[32m'
-TRED_BOLD = '\033[31;1m'
-TWHITE = '\033[37m'
-WHITE_BOLD = '\033[37;1m'
-CYAN_BOLD = '\033[36;1m'
-
-
-def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
-    """
-    This function searches multiple directories and locates previously produced performance statistics.
-
-    Args:
-        archive_to_check (str): The directory path to search.
-        magnitude (str): Because a benchmark dataset may have multiple magnitudes, this argument defines
-                               which magnitude is to be used when searching for previous statistics.
-    Returns:
-        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain and magnitude.
-                                  {version: {agreement_raster: agreement_raster_path, stats_csv: stats_csv_path, stats_json: stats_json_path}}
-                                  *Will only add the paths to files that exist.
-
-    """
-
-    archive_dictionary = {}
-
-    # List through previous version and check for available stats and maps. If available, add to dictionary.
-    available_versions_list = os.listdir(archive_to_check)
-
-    if len(available_versions_list) == 0:
-        print("Cannot compare with -c flag because there are no data in the previous_versions directory.")
-        return
-
-    for version in available_versions_list:
-        version_magnitude_dir = os.path.join(archive_to_check, version, magnitude)
-        stats_json = os.path.join(version_magnitude_dir, stats_mode + '_stats.json')
-
-        if os.path.exists(stats_json):
-            archive_dictionary.update({version: {'stats_json': stats_json}})
-
-    return archive_dictionary
-
-
-def compute_contingency_stats_from_rasters(predicted_raster_path, benchmark_raster_path, agreement_raster=None, stats_csv=None, stats_json=None, mask_values=None, stats_modes_list=['total_area'], test_id='', mask_dict={}):
-    """
-    This function contains FIM-specific logic to prepare raster datasets for use in the generic get_contingency_table_from_binary_rasters() function.
-    This function also calls the generic compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
-
-    Args:
-        predicted_raster_path (str): The path to the predicted, or modeled, FIM extent raster.
-        benchmark_raster_path (str): The path to the benchmark, or truth, FIM extent raster.
-        agreement_raster (str): Optional. An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive, 3: True Positive.
-        stats_csv (str): Optional. Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
-        stats_json (str): Optional. Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary in other processes.
-
-    Returns:
-        stats_dictionary (dict): A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and statistic values are the values.
-    """
-
-    # Get cell size of benchmark raster.
-    raster = rasterio.open(predicted_raster_path)
-    t = raster.transform
-    cell_x = t[0]
-    cell_y = t[4]
-    cell_area = abs(cell_x*cell_y)
-
-    # Get contingency table from two rasters.
-    contingency_table_dictionary = get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster, mask_values=mask_values, mask_dict=mask_dict)
-
-    stats_dictionary = {}
-
-    for stats_mode in contingency_table_dictionary:
-        true_negatives = contingency_table_dictionary[stats_mode]['true_negatives']
-        false_negatives = contingency_table_dictionary[stats_mode]['false_negatives']
-        false_positives = contingency_table_dictionary[stats_mode]['false_positives']
-        true_positives = contingency_table_dictionary[stats_mode]['true_positives']
-        masked_count = contingency_table_dictionary[stats_mode]['masked_count']
-        file_handle = contingency_table_dictionary[stats_mode]['file_handle']
-
-        # Produce statistics from continency table and assign to dictionary. cell_area argument optional (defaults to None).
-        mode_stats_dictionary = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area, masked_count)
-
-        # Write the mode_stats_dictionary to the stats_csv.
-        if stats_csv != None:
-            stats_csv = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.csv')
-            df = pd.DataFrame.from_dict(mode_stats_dictionary, orient="index", columns=['value'])
-            df.to_csv(stats_csv)
-
-        # Write the mode_stats_dictionary to the stats_json.
-        if stats_json != None:
-            stats_json = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.json')
-            with open(stats_json, "w") as outfile:
-                json.dump(mode_stats_dictionary, outfile)
-
-        stats_dictionary.update({stats_mode: mode_stats_dictionary})
-
-    return stats_dictionary
-
-
-def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
-
-    difference_dict = {}
-
-    # Compare stats_csv to previous_version_stats_file
-    stats_dict_to_test = json.load(open(stats_json_to_test))
-    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
-
-    for stat, value in stats_dict_to_test.items():
-        previous_version_value = previous_version_stats_dict[stat]
-        stat_value_diff = value - previous_version_value
-        difference_dict.update({stat + '_diff': stat_value_diff})
-
-    return difference_dict
-
-
-def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False):
 
+def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True):
+    
+    benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
+    current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
+    
     # Construct paths to development test results if not existent.
     if archive_results:
-        branch_test_case_dir_parent = os.path.join(TEST_CASES_DIR, test_id, 'performance_archive', 'previous_versions', branch_name)
+        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'official_versions', version)
     else:
-        branch_test_case_dir_parent = os.path.join(TEST_CASES_DIR, test_id, 'performance_archive', 'development_versions', branch_name)
+        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'testing_versions', version)
 
     # Delete the entire directory if it already exists.
-    if os.path.exists(branch_test_case_dir_parent):
-        shutil.rmtree(branch_test_case_dir_parent)
+    if os.path.exists(version_test_case_dir_parent):
+        if overwrite == True:
+            shutil.rmtree(version_test_case_dir_parent)
+        else:
+            print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
+            return
+        
+    os.mkdir(version_test_case_dir_parent)
 
-    print("Running the alpha test for test_id: " + test_id + ", " + branch_name + "...")
+    print("Running the alpha test for test_id: " + test_id + ", " + version + "...")
     stats_modes_list = ['total_area']
 
     fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
@@ -155,7 +39,11 @@ def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_prev
 
     # Create paths to fim_run outputs for use in inundate().
     rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
+    if not os.path.exists(rem):
+        rem = os.path.join(fim_run_parent, 'rem_clipped_zeroed_masked.tif')
     catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+    if not os.path.exists(catchments):
+        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_clipped_addedAttributes.tif')
     if mask_type == 'huc':
         catchment_poly = ''
     else:
@@ -164,9 +52,6 @@ def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_prev
         
     # Map necessary inputs for inundation().
     hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-    
-    benchmark_category = test_id.split('_')[1]
-    current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
 
     # Create list of shapefile paths to use as exclusion areas.
     zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
@@ -182,41 +67,6 @@ def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_prev
                      },
                 }
             
-    if not os.path.exists(branch_test_case_dir_parent):
-        os.mkdir(branch_test_case_dir_parent)
-            
-    # If the test_id is AHPS, then identify possible inclusion zones in the HUC.
-    if benchmark_category == 'ahps':
-        
-        ahps_inclusion_zones_dir = os.path.join(branch_test_case_dir_parent, 'ahps_domains')
-        print(ahps_inclusion_zones_dir)
-        if not os.path.exists(ahps_inclusion_zones_dir):
-            os.mkdir(ahps_inclusion_zones_dir)
-        
-        ahps_domain_shapefile = os.path.join(TEST_CASES_DIR, 'other', 'zones', 'ahps_domains.shp')
-        
-        # Open shapefile, determine the polys in the huc, create a different shapefile for each poly--name according to AHPS.
-        ahps_domain_obj = gpd.read_file(ahps_domain_shapefile)
-        ahps_domain_gdf = gpd.GeoDataFrame(ahps_domain_obj)
-            
-        # Loop through entries and compare against the huc4_list to get available HUCs within the geopackage domain.
-        for index, row in ahps_domain_gdf.iterrows():
-            huc8_code = row['huc8_code']
-            ahps = row['ahps_code']
-            
-            if huc8_code == current_huc:
-                ahps_domain_subset = ahps_domain_obj[ahps_domain_obj.ahps_code == ahps]
-                
-                #.query("ahps_code=='{ahps_code}'".format(ahps_code=ahps_code))
-                ahps_domain_subset_output = os.path.join(ahps_inclusion_zones_dir, ahps + '.shp')
-                ahps_domain_subset.to_file(ahps_domain_subset_output,driver='ESRI Shapefile')
-                
-                mask_dict.update({ahps:
-                    {'path': ahps_domain_subset_output,
-                     'buffer': None,
-                     'operation': 'include'}
-                        })
-                
     if inclusion_area != '':
         inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
         mask_dict.update({inclusion_area_name: {'path': inclusion_area,
@@ -232,167 +82,107 @@ def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_prev
     if type(magnitude_list) != list:
         magnitude_list = [magnitude_list]
 
+    # Get path to validation_data_{benchmark} directory and huc_dir.
+    validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
+    
     for magnitude in magnitude_list:
-        # Construct path to validation raster and forecast file.
-        
-        benchmark_raster_path = os.path.join(TEST_CASES_DIR, 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_depth_' + magnitude + '.tif')
-        if not os.path.exists(benchmark_raster_path):  # Skip loop instance if the benchmark raster doesn't exist.
-            continue
-
-        branch_test_case_dir = os.path.join(branch_test_case_dir_parent, magnitude)
-
-        os.makedirs(branch_test_case_dir)  # Make output directory for branch.
-
-        # Define paths to inundation_raster and forecast file.
-        inundation_raster = os.path.join(branch_test_case_dir, 'inundation_extent.tif')
-        forecast = os.path.join(TEST_CASES_DIR, 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
-
-        # Run inundate.
-        print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
-        inundate(
-                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-                 depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
-                )
-
-        print("-----> Inundation mapping complete.")
-        predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
-
-        # Define outputs for agreement_raster, stats_json, and stats_csv.
-
-        agreement_raster, stats_json, stats_csv = os.path.join(branch_test_case_dir, 'total_area_agreement.tif'), os.path.join(branch_test_case_dir, 'stats.json'), os.path.join(branch_test_case_dir, 'stats.csv')
-
-        test_version_dictionary = compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                                         benchmark_raster_path,
-                                                                         agreement_raster,
-                                                                         stats_csv=stats_csv,
-                                                                         stats_json=stats_json,
-                                                                         mask_values=[],
-                                                                         stats_modes_list=stats_modes_list,
-                                                                         test_id=test_id,
-                                                                         mask_dict=mask_dict,
-                                                                         )
-        print(" ")
-        print("Evaluation complete. All metrics for " + test_id + ", " + branch_name + ", " + magnitude + " are available at " + CYAN_BOLD + branch_test_case_dir + ENDC)
-        print(" ")
-
-        if compare_to_previous:
-            text_block = []
-            # Compare to previous stats files that are available.
-            archive_to_check = os.path.join(TEST_CASES_DIR, test_id, 'performance_archive', 'previous_versions')
-            for stats_mode in stats_modes_list:
-                archive_dictionary = profile_test_case_archive(archive_to_check, magnitude, stats_mode)
-
-                if archive_dictionary == {}:
-                    break
-
-                # Create header for section.
-                header = [stats_mode]
-                for previous_version, paths in archive_dictionary.items():
-                    header.append(previous_version)
-                header.append(branch_name)
-                text_block.append(header)
-
-                # Loop through stats in PRINTWORTHY_STATS for left.
-                for stat in PRINTWORTHY_STATS:
-                    stat_line = [stat]
-                    for previous_version, paths in archive_dictionary.items():
-                        # Load stats for previous version.
-                        previous_version_stats_json_path = paths['stats_json']
-                        if os.path.exists(previous_version_stats_json_path):
-                            previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
+        version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
+        if not os.path.exists(version_test_case_dir):
+            os.mkdir(version_test_case_dir)
     
-                            # Append stat for the version to state_line.
-                            stat_line.append(previous_version_stats_dict[stat])
-
-
-                    # Append stat for the current version to stat_line.
-                    stat_line.append(test_version_dictionary[stats_mode][stat])
-
-                    text_block.append(stat_line)
-
-                text_block.append([" "])
-
-            regression_report_csv = os.path.join(branch_test_case_dir, 'stats_summary.csv')
-            with open(regression_report_csv, 'w', newline='') as csvfile:
-                csv_writer = csv.writer(csvfile)
-                csv_writer.writerows(text_block)
+        # Construct path to validation raster and forecast file.
+        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+            benchmark_raster_path_list, forecast_list = [], []
+            lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
+            lid_list, inundation_raster_list, extent_file_list = [], [], []
+            
+            for lid in lid_dir_list:
+                lid_dir = os.path.join(validation_data_path, current_huc, lid)
+                benchmark_raster_path_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_depth_' + magnitude + '.tif'))  # TEMP
+                forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
+                lid_list.append(lid)
+                inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
+                extent_file_list.append(os.path.join(lid_dir, lid + '_extent.shp'))
+                    
+            ahps_inclusion_zones_dir = os.path.join(version_test_case_dir_parent, 'ahps_domains')
+            
+            if not os.path.exists(ahps_inclusion_zones_dir):
+                os.mkdir(ahps_inclusion_zones_dir)
 
-            print()
-            print("--------------------------------------------------------------------------------------------------")
+        else:
+            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_depth_' + magnitude + '.tif')
+            benchmark_raster_path_list = [benchmark_raster_file]
+            forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
+            forecast_list = [forecast_path]
+            inundation_raster_list = [os.path.join(version_test_case_dir, 'inundation_extent.tif')]
+            
+        for index in range(0, len(benchmark_raster_path_list)):
+            benchmark_raster_path = benchmark_raster_path_list[index]
+            forecast = forecast_list[index]
+            inundation_raster = inundation_raster_list[index]
             
-            stats_mode = stats_modes_list[0]
+            # Only need to define ahps_lid and ahps_extent_file for AHPS_BENCHMARK_CATEGORIES.
+            if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                ahps_lid = lid_list[index]
+                ahps_extent_file = extent_file_list[index]
+                mask_dict.update({ahps_lid:
+                    {'path': ahps_extent_file,
+                     'buffer': None,
+                     'operation': 'include'}
+                        })
+        
+                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_extent_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
+                    continue
+            else:  # If not in AHPS_BENCHMARK_CATEGORIES.
+                if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
+                    continue
+    
+            # Run inundate.
+            print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
             try:
-                last_version_index = text_block[0].index('dev_latest')
-            except ValueError:
-                try:
-                    last_version_index = text_block[0].index('fim_2_3_3')
-                except ValueError:
-                    try:
-                        last_version_index = text_block[0].index('fim_1_0_0')
-                    except ValueError:
-                        print(TRED_BOLD + "Warning: " + ENDC + "Cannot compare " + branch_name + " to a previous version because no authoritative versions were found in previous_versions directory. Future version of run_test_case may allow for comparisons between dev branches.")
-                        print()
-                        continue
-
+                inundate(
+                         rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                         subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
+                         depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
+                        )
             
-
-            for line in text_block:
-                first_item = line[0]
-                if first_item in stats_modes_list:
-                    current_version_index = line.index(branch_name)
-                    if first_item != stats_mode:  # Update the stats_mode and print a separator.
-                        print()
-                        print()
-                        print("--------------------------------------------------------------------------------------------------")
-                    print()
-                    stats_mode = first_item
-                    print(CYAN_BOLD + current_huc + ": " + magnitude.upper(), ENDC)
-                    print(CYAN_BOLD + stats_mode.upper().replace('_', ' ') + " METRICS" + ENDC)
-                    print()
-
-                    color = WHITE_BOLD
-                    metric_name = '      '.center(len(max(PRINTWORTHY_STATS, key=len)))
-                    percent_change_header = '% CHG'
-                    difference_header = 'DIFF'
-                    current_version_header = line[current_version_index].upper()
-                    last_version_header = line[last_version_index].upper()
-                    # Print Header.
-                    print(color + metric_name + "      " + percent_change_header.center((7)) + "       " + difference_header.center((15))  + "    " + current_version_header.center(18) + " " + last_version_header.center(18), ENDC)
-                # Format and print stat row.
-                elif first_item in PRINTWORTHY_STATS:
-                    stat_name = first_item.upper().center(len(max(PRINTWORTHY_STATS, key=len))).replace('_', ' ')
-                    current_version = round((line[current_version_index]), 3)
-                    last_version = round((line[last_version_index]) + 0.000, 3)
-                    difference = round(current_version - last_version, 3)
-                    if difference > 0:
-                        symbol = '+'
-                        if first_item in GO_UP_STATS:
-                            color = TGREEN_BOLD
-                        elif first_item in GO_DOWN_STATS:
-                            color = TRED_BOLD
-                        else:
-                            color = TWHITE
-                    if difference < 0:
-                        symbol = '-'
-                        if first_item in GO_UP_STATS:
-                            color = TRED_BOLD
-                        elif first_item in GO_DOWN_STATS:
-                            color = TGREEN_BOLD
-                        else:
-                            color = TWHITE
-
-                    if difference == 0 :
-                        symbol, color = '+', TGREEN
-                    percent_change = round((difference / last_version)*100,2)
-
-                    print(WHITE_BOLD + stat_name + ENDC + "     " + color + (symbol + " {:5.2f}".format(abs(percent_change)) + " %").rjust(len(percent_change_header)), ENDC + "    " + color + ("{:12.3f}".format((difference))).rjust(len(difference_header)), ENDC + "    " + "{:15.3f}".format(current_version).rjust(len(current_version_header)) + "   " + "{:15.3f}".format(last_version).rjust(len(last_version_header)) + "  ")
-
-            print()
-            print()
-            print()
-            print("--------------------------------------------------------------------------------------------------")
-            print()
+                print("-----> Inundation mapping complete.")
+                predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
+        
+                # Define outputs for agreement_raster, stats_json, and stats_csv.
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+                else:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+         
+                compute_contingency_stats_from_rasters(predicted_raster_path,
+                                                       benchmark_raster_path,
+                                                       agreement_raster,
+                                                       stats_csv=stats_csv,
+                                                       stats_json=stats_json,
+                                                       mask_values=[],
+                                                       stats_modes_list=stats_modes_list,
+                                                       test_id=test_id,
+                                                       mask_dict=mask_dict,
+                                                       )
+        
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    del mask_dict[ahps_lid]
+                
+                print(" ")
+                print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
+                print(" ")
+            except Exception as e:
+                print(e)      
+        
+        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+            # -- Delete temp files -- #
+            # List all files in the output directory.
+            output_file_list = os.listdir(version_test_case_dir)
+            for output_file in output_file_list:
+                if "total_area" in output_file:
+                    full_output_file_path = os.path.join(version_test_case_dir, output_file)
+                    os.remove(full_output_file_path)
 
 
 if __name__ == '__main__':
@@ -400,7 +190,7 @@ def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_prev
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
     parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-b', '--branch-name',help='The name of the working branch in which features are being tested',required=True,default="")
+    parser.add_argument('-b', '--version-name',help='The name of the working version in which features are being tested',required=True,default="")
     parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
     parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
     parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
@@ -409,6 +199,7 @@ def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_prev
     parser.add_argument('-i', '--inclusion-area', help='Path to shapefile. Contingency metrics will be produced from pixels inside of shapefile extent.', required=False, default="")
     parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
     parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
+    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
@@ -454,12 +245,9 @@ def run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_prev
             print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
             exit_flag = True     
             
-            
     if exit_flag:
         print()
         sys.exit()
-        
 
     else:
-
         run_alpha_test(**args)
diff --git a/tests/synthesize_test_cases.py b/tests/synthesize_test_cases.py
index 913ffd6da..65c241336 100644
--- a/tests/synthesize_test_cases.py
+++ b/tests/synthesize_test_cases.py
@@ -1,26 +1,182 @@
 #!/usr/bin/env python3
 
-
 import os
 import argparse
 from multiprocessing import Pool
+import json
+import csv
 
 from run_test_case import run_alpha_test
-from all_ble_stats_comparison import subset_vector_layers
-from aggregate_metrics import aggregate_metrics
+from utils.shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR
 
-TEST_CASES_DIR = r'/data/test_cases/'
-PREVIOUS_FIM_DIR = r'/data/previous_fim'
-OUTPUTS_DIR = r'/data/outputs'
+
+def create_master_metrics_csv(master_metrics_csv_output):
+            
+    # Construct header
+    metrics_to_write = ['true_negatives_count',
+                        'false_negatives_count',
+                        'true_positives_count',
+                        'false_positives_count',
+                        'contingency_tot_count',
+                        'cell_area_m2',
+                        'TP_area_km2',
+                        'FP_area_km2',
+                        'TN_area_km2',
+                        'FN_area_km2',
+                        'contingency_tot_area_km2',
+                        'predPositive_area_km2',
+                        'predNegative_area_km2',
+                        'obsPositive_area_km2',
+                        'obsNegative_area_km2',
+                        'positiveDiff_area_km2',
+                        'CSI',
+                        'FAR',
+                        'TPR',
+                        'TNR',
+                        'PPV',
+                        'NPV',
+                        'ACC',
+                        'Bal_ACC',
+                        'MCC',
+                        'EQUITABLE_THREAT_SCORE',
+                        'PREVALENCE',
+                        'BIAS',
+                        'F1_SCORE',
+                        'TP_perc',
+                        'FP_perc',
+                        'TN_perc',
+                        'FN_perc',
+                        'predPositive_perc',
+                        'predNegative_perc',
+                        'obsPositive_perc',
+                        'obsNegative_perc',
+                        'positiveDiff_perc',
+                        'masked_count',
+                        'masked_perc',
+                        'masked_area_km2'
+                        ]
+    
+    additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
+    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config']]
+    
+    versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
+    
+    for benchmark_source in ['ble', 'nws', 'usgs']:
+        
+        benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
+        
+        if benchmark_source == 'ble':
+            test_cases_list = os.listdir(benchmark_test_case_dir)
+                            
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
+                    
+                    for magnitude in ['100yr', '500yr']:
+                        for version in versions_to_aggregate:
+                            if '_fr_' in version:
+                                extent_config = 'FR'
+                            if '_ms_' in version:
+                                extent_config = 'MS'
+                            if '_fr_' or '_ms_' not in version:
+                                extent_config = 'FR'
+                            version_dir = os.path.join(official_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+
+                            if os.path.exists(magnitude_dir):
+                                
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f:
+                                        flow = 'NA'
+                                        nws_lid = "NA"
+                                        benchmark_source = 'ble'
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        if os.path.exists(full_json_path):
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            sub_list_to_append.append(extent_config)
+                                            
+                                            list_to_write.append(sub_list_to_append)
+                except ValueError:
+                    pass
+                
+        if benchmark_source in ['nws', 'usgs']:
+            test_cases_list = os.listdir(TEST_CASES_DIR)
+
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'performance_archive', 'previous_versions')
+                    
+                    for magnitude in ['action', 'minor', 'moderate', 'major']:
+                        for version in versions_to_aggregate:
+                            if '_fr_' in version:
+                                extent_config = 'FR'
+                            if '_ms_' in version:
+                                extent_config = 'MS'
+                            if '_fr_' or '_ms_' not in version:
+                                extent_config = 'FR'
+                                
+                            version_dir = os.path.join(official_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+                            
+                            if os.path.exists(magnitude_dir):
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f and 'total_area' not in f:
+                                        nws_lid = f[:5]
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        flow = ''
+                                        if os.path.exists(full_json_path):
+                                            
+                                            # Get flow used to map.                                                
+                                            flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                                            if os.path.exists(flow_file):
+                                                with open(flow_file, newline='') as csv_file:
+                                                    reader = csv.reader(csv_file)
+                                                    next(reader)
+                                                    for row in reader:
+                                                        flow = row[1]
+                                                    if nws_lid == 'mcc01':
+                                                        print(flow)
+                                            
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            sub_list_to_append.append(extent_config)
+                                            
+                                            list_to_write.append(sub_list_to_append)
+                except ValueError:
+                    pass
+        
+    with open(master_metrics_csv_output, 'w', newline='') as csvfile:
+        csv_writer = csv.writer(csvfile)
+        csv_writer.writerows(list_to_write)
 
 
 def process_alpha_test(args):
     
     fim_run_dir = args[0]
-    branch_name = args[1]
+    version = args[1]
     test_id = args[2]
     magnitude = args[3]
     archive_results = args[4]
+    overwrite = args[5]
     
     mask_type = 'huc'    
     
@@ -30,7 +186,7 @@ def process_alpha_test(args):
         compare_to_previous = False
 
     try:
-        run_alpha_test(fim_run_dir, branch_name, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type)
+        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type, overwrite=overwrite)
     except Exception as e:
         print(e)
 
@@ -40,80 +196,105 @@ def process_alpha_test(args):
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
     parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
-    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=True)
+    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
     parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
     parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
-    parser.add_argument('-b','--benchmark-category',help='Options include ble or ahps. Defaults to process both.',required=False, default=['ble', 'ahps'])
-    parser.add_argument('-l','--huc8-list',help='A list of HUC8s to synthesize.',required=True)
-    parser.add_argument('-d','--current-dev',help='The current dev id.',required=True)
-    parser.add_argument('-o','--output-folder',help='The directory where synthesis outputs will be written.',required=True)
+    parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
+    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
+    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=True)
         
-    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
-    
+    # Assign variables from arguments.
     args = vars(parser.parse_args())
-
     config = args['config']
-    branch_name = args['fim_version']
+    fim_version = args['fim_version']
     job_number = int(args['job_number'])
     special_string = args['special_string']
     benchmark_category = args['benchmark_category']
+    overwrite = args['overwrite']
+    master_metrics_csv = args['master_metrics_csv']
+        
+    # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
+    if fim_version != "all":
+        previous_fim_list = [fim_version]
+    else:
+        if config == 'PREV':
+            previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)
+        elif config == 'DEV':
+            previous_fim_list = os.listdir(OUTPUTS_DIR)
     
-    
+    # Define whether or not to archive metrics in "official_versions" or "testing_versions" for each test_id.
     if config == 'PREV':
         archive_results = True
     elif config == 'DEV':
         archive_results = False
     else:
         print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
-    
-    if type(benchmark_category) != list:
-        benchmark_category = [benchmark_category]
+     
+    # List all available benchmark categories and test_cases.
+    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
+    benchmark_category_list = []
+    if benchmark_category == "all":
+        for d in test_cases_dir_list:
+            if 'test_cases' in d:
+                benchmark_category_list.append(d.replace('_test_cases', ''))
+    else:
+        benchmark_category_list = [benchmark_category]
         
+    # Loop through benchmark categories.
     procs_list = []
-    for test_id in test_cases_dir_list:
-        if not any(x in test_id for x in ['validation','other','.lst']):#if 'validation' and 'other' not in test_id:
-                        
-            current_huc = test_id.split('_')[0]
-            print(current_huc)
-            if test_id.split('_')[1] in benchmark_category:
-            
-                
-                if config == 'DEV':
-                    fim_run_dir = os.path.join(OUTPUTS_DIR, branch_name, current_huc)
-                elif config == 'PREV':
-                    fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, branch_name, current_huc)
-                    
-                if os.path.exists(fim_run_dir):    
-                    
-                    if special_string != "":
-                        branch_name = branch_name + '_' + special_string
-                    
-                    if 'ble' in test_id:
-                        magnitude = ['100yr', '500yr']
-                    elif 'ahps' in test_id:
-                        magnitude = ['action', 'minor', 'moderate', 'major']
-                    else:
-                        continue
+    for bench_cat in benchmark_category_list:
+        
+        # Map path to appropriate test_cases folder and list test_ids into bench_cat_id_list.
+        bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
+        bench_cat_id_list = os.listdir(bench_cat_test_case_dir)
+    
+        # Loop through test_ids in bench_cat_id_list.
+        for test_id in bench_cat_id_list:
+            if 'validation' and 'other' not in test_id:
+                current_huc = test_id.split('_')[0]
+                if test_id.split('_')[1] in bench_cat:
                 
-                    print("Adding " + test_id + " to list of test_ids to process...")
-                    if job_number > 1:
-                        procs_list.append([fim_run_dir, branch_name, test_id, magnitude, archive_results])
-                    else:
-                        process_alpha_test([fim_run_dir, branch_name, test_id, magnitude, archive_results])
+                    # Loop through versions.
+                    for version in previous_fim_list:
+                        if config == 'DEV':
+                            fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
+                        elif config == 'PREV':
+                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
+                                                
+                        # For previous versions of HAND computed at HUC6 scale
+                        if not os.path.exists(fim_run_dir):
+                            if config == 'DEV':
+                                fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
+                                print(fim_run_dir)
+                            elif config == 'PREV':
+                                fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  
                         
-            else:
-                print("No test_ids were found for the provided benchmark category: " + str(test_id.split('_')[1]))
+                        if os.path.exists(fim_run_dir):
+                            
+                            # If a user supplies a specia_string (-s), then add it to the end of the created dirs.
+                            if special_string != "":
+                                version = version + '_' + special_string
+                            
+                            # Define the magnitude lists to use, depending on test_id.
+                            if 'ble' in test_id:
+                                magnitude = ['100yr', '500yr']
+                            elif 'usgs' or 'nws' in test_id:
+                                magnitude = ['action', 'minor', 'moderate', 'major']
+                            else:
+                                continue
+                        
+                            # Either add to list to multiprocess or process serially, depending on user specification.
+                            if job_number > 1:
+                                procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
+                            else:                            
+                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
 
     # Multiprocess alpha test runs.
     if job_number > 1:
         pool = Pool(job_number)
         pool.map(process_alpha_test, procs_list)
-        
-    # Do all_ble_stats_comparison.
-    subset_vector_layers(args['huc8_list'], branch_name, args['current_dev'], args['output_folder'])
     
     # Do aggregate_metrics.
-    aggregate_metrics(config=config, branch=branch_name, hucs=args['huc8_list'], special_string=args['special_string'], outfolder=args['output_folder'])
-    
-        
+    print("Creating master metrics CSV...")
+    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv)
     
\ No newline at end of file
diff --git a/tests/utils/shared_functions.py b/tests/utils/shared_functions.py
index 777575f02..d36c22814 100644
--- a/tests/utils/shared_functions.py
+++ b/tests/utils/shared_functions.py
@@ -1,5 +1,239 @@
 #!/usr/bin/env python3
 
+import os
+import json
+import csv
+import rasterio
+import pandas as pd
+from utils.shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
+                                    ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
+
+def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
+
+    difference_dict = {}
+
+    # Compare stats_csv to previous_version_stats_file
+    stats_dict_to_test = json.load(open(stats_json_to_test))
+    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
+
+    for stat, value in stats_dict_to_test.items():
+        previous_version_value = previous_version_stats_dict[stat]
+        stat_value_diff = value - previous_version_value
+        difference_dict.update({stat + '_diff': stat_value_diff})
+
+    return difference_dict
+
+
+def compute_contingency_stats_from_rasters(predicted_raster_path, benchmark_raster_path, agreement_raster=None, stats_csv=None, stats_json=None, mask_values=None, stats_modes_list=['total_area'], test_id='', mask_dict={}):
+    """
+    This function contains FIM-specific logic to prepare raster datasets for use in the generic get_contingency_table_from_binary_rasters() function.
+    This function also calls the generic compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
+
+    Args:
+        predicted_raster_path (str): The path to the predicted, or modeled, FIM extent raster.
+        benchmark_raster_path (str): The path to the benchmark, or truth, FIM extent raster.
+        agreement_raster (str): Optional. An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive, 3: True Positive.
+        stats_csv (str): Optional. Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
+        stats_json (str): Optional. Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary in other processes.
+
+    Returns:
+        stats_dictionary (dict): A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and statistic values are the values.
+    """
+
+    # Get cell size of benchmark raster.
+    raster = rasterio.open(predicted_raster_path)
+    t = raster.transform
+    cell_x = t[0]
+    cell_y = t[4]
+    cell_area = abs(cell_x*cell_y)
+
+    # Get contingency table from two rasters.
+    contingency_table_dictionary = get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster, mask_values=mask_values, mask_dict=mask_dict)
+
+    stats_dictionary = {}
+
+    for stats_mode in contingency_table_dictionary:
+        true_negatives = contingency_table_dictionary[stats_mode]['true_negatives']
+        false_negatives = contingency_table_dictionary[stats_mode]['false_negatives']
+        false_positives = contingency_table_dictionary[stats_mode]['false_positives']
+        true_positives = contingency_table_dictionary[stats_mode]['true_positives']
+        masked_count = contingency_table_dictionary[stats_mode]['masked_count']
+        file_handle = contingency_table_dictionary[stats_mode]['file_handle']
+
+        # Produce statistics from continency table and assign to dictionary. cell_area argument optional (defaults to None).
+        mode_stats_dictionary = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area, masked_count)
+
+        # Write the mode_stats_dictionary to the stats_csv.
+        if stats_csv != None:
+            stats_csv = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.csv')
+            df = pd.DataFrame.from_dict(mode_stats_dictionary, orient="index", columns=['value'])
+            df.to_csv(stats_csv)
+
+        # Write the mode_stats_dictionary to the stats_json.
+        if stats_json != None:
+            stats_json = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.json')
+            with open(stats_json, "w") as outfile:
+                json.dump(mode_stats_dictionary, outfile)
+
+        stats_dictionary.update({stats_mode: mode_stats_dictionary})
+
+    return stats_dictionary
+
+
+def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
+    """
+    This function searches multiple directories and locates previously produced performance statistics.
+
+    Args:
+        archive_to_check (str): The directory path to search.
+        magnitude (str): Because a benchmark dataset may have multiple magnitudes, this argument defines
+                               which magnitude is to be used when searching for previous statistics.
+    Returns:
+        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain and magnitude.
+                                  {version: {agreement_raster: agreement_raster_path, stats_csv: stats_csv_path, stats_json: stats_json_path}}
+                                  *Will only add the paths to files that exist.
+
+    """
+
+    archive_dictionary = {}
+
+    # List through previous version and check for available stats and maps. If available, add to dictionary.
+    available_versions_list = os.listdir(archive_to_check)
+
+    if len(available_versions_list) == 0:
+        print("Cannot compare with -c flag because there are no data in the previous_versions directory.")
+        return
+
+    for version in available_versions_list:
+        version_magnitude_dir = os.path.join(archive_to_check, version, magnitude)
+        stats_json = os.path.join(version_magnitude_dir, stats_mode + '_stats.json')
+
+        if os.path.exists(stats_json):
+            archive_dictionary.update({version: {'stats_json': stats_json}})
+
+    return archive_dictionary
+
+
+#def compare_to_previous(benchmark_category, test_id, stats_modes_list, magnitude, version, test_version_dictionary, version_test_case_dir):
+#    text_block = []
+#    # Compare to previous stats files that are available.
+#    archive_to_check = os.path.join(TEST_CASES_DIR, benchmark_category + 'test_cases', test_id, 'official_versions')
+#    for stats_mode in stats_modes_list:
+#        archive_dictionary = profile_test_case_archive(archive_to_check, magnitude, stats_mode)
+#
+#        if archive_dictionary == {}:
+#            break
+#
+#        # Create header for section.
+#        header = [stats_mode]
+#        for previous_version, paths in archive_dictionary.items():
+#            header.append(previous_version)
+#        header.append(version)
+#        text_block.append(header)
+#
+#        # Loop through stats in PRINTWORTHY_STATS for left.
+#        for stat in PRINTWORTHY_STATS:
+#            stat_line = [stat]
+#            for previous_version, paths in archive_dictionary.items():
+#                # Load stats for previous version.
+#                previous_version_stats_json_path = paths['stats_json']
+#                if os.path.exists(previous_version_stats_json_path):
+#                    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
+#
+#                    # Append stat for the version to state_line.
+#                    stat_line.append(previous_version_stats_dict[stat])
+#
+#
+#            # Append stat for the current version to stat_line.
+#            stat_line.append(test_version_dictionary[stats_mode][stat])
+#
+#            text_block.append(stat_line)
+#
+#        text_block.append([" "])
+#
+#    regression_report_csv = os.path.join(version_test_case_dir, 'stats_summary.csv')
+#    with open(regression_report_csv, 'w', newline='') as csvfile:
+#        csv_writer = csv.writer(csvfile)
+#        csv_writer.writerows(text_block)
+#
+#    print()
+#    print("--------------------------------------------------------------------------------------------------")
+#    
+#    stats_mode = stats_modes_list[0]
+#    try:
+#        last_version_index = text_block[0].index('dev_latest')
+#    except ValueError:
+#        try:
+#            last_version_index = text_block[0].index('fim_2_3_3')
+#        except ValueError:
+#            try:
+#                last_version_index = text_block[0].index('fim_1_0_0')
+#            except ValueError:
+#                print(TRED_BOLD + "Warning: " + ENDC + "Cannot compare " + version + " to a previous version because no authoritative versions were found in previous_versions directory. Future version of run_test_case may allow for comparisons between dev versions.")
+#                print()
+#                continue
+#
+#    
+#
+#    for line in text_block:
+#        first_item = line[0]
+#        if first_item in stats_modes_list:
+#            current_version_index = line.index(version)
+#            if first_item != stats_mode:  # Update the stats_mode and print a separator.
+#                print()
+#                print()
+#                print("--------------------------------------------------------------------------------------------------")
+#            print()
+#            stats_mode = first_item
+#            print(CYAN_BOLD + current_huc + ": " + magnitude.upper(), ENDC)
+#            print(CYAN_BOLD + stats_mode.upper().replace('_', ' ') + " METRICS" + ENDC)
+#            print()
+#
+#            color = WHITE_BOLD
+#            metric_name = '      '.center(len(max(PRINTWORTHY_STATS, key=len)))
+#            percent_change_header = '% CHG'
+#            difference_header = 'DIFF'
+#            current_version_header = line[current_version_index].upper()
+#            last_version_header = line[last_version_index].upper()
+#            # Print Header.
+#            print(color + metric_name + "      " + percent_change_header.center((7)) + "       " + difference_header.center((15))  + "    " + current_version_header.center(18) + " " + last_version_header.center(18), ENDC)
+#        # Format and print stat row.
+#        elif first_item in PRINTWORTHY_STATS:
+#            stat_name = first_item.upper().center(len(max(PRINTWORTHY_STATS, key=len))).replace('_', ' ')
+#            current_version = round((line[current_version_index]), 3)
+#            last_version = round((line[last_version_index]) + 0.000, 3)
+#            difference = round(current_version - last_version, 3)
+#            if difference > 0:
+#                symbol = '+'
+#                if first_item in GO_UP_STATS:
+#                    color = TGREEN_BOLD
+#                elif first_item in GO_DOWN_STATS:
+#                    color = TRED_BOLD
+#                else:
+#                    color = TWHITE
+#            if difference < 0:
+#                symbol = '-'
+#                if first_item in GO_UP_STATS:
+#                    color = TRED_BOLD
+#                elif first_item in GO_DOWN_STATS:
+#                    color = TGREEN_BOLD
+#                else:
+#                    color = TWHITE
+#
+#            if difference == 0 :
+#                symbol, color = '+', TGREEN
+#            percent_change = round((difference / last_version)*100,2)
+#
+#            print(WHITE_BOLD + stat_name + ENDC + "     " + color + (symbol + " {:5.2f}".format(abs(percent_change)) + " %").rjust(len(percent_change_header)), ENDC + "    " + color + ("{:12.3f}".format((difference))).rjust(len(difference_header)), ENDC + "    " + "{:15.3f}".format(current_version).rjust(len(current_version_header)) + "   " + "{:15.3f}".format(last_version).rjust(len(last_version_header)) + "  ")
+#
+#    print()
+#    print()
+#    print()
+#    print("--------------------------------------------------------------------------------------------------")
+#    print()
+#
+
+
 def compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=None, masked_count=None):
     """
     This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.
diff --git a/tests/utils/shared_variables.py b/tests/utils/shared_variables.py
new file mode 100644
index 000000000..4f844e3e9
--- /dev/null
+++ b/tests/utils/shared_variables.py
@@ -0,0 +1,20 @@
+import os
+
+TEST_CASES_DIR = r'/data/test_cases_new/'  # TODO remove "_new"
+PREVIOUS_FIM_DIR = r'/data/previous_fim'
+OUTPUTS_DIR = os.environ['outputDataDir']
+INPUTS_DIR = r'/data/inputs'
+PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
+GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
+GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
+AHPS_BENCHMARK_CATEGORIES = ['usgs', 'ble']
+
+
+
+ENDC = '\033[m'
+TGREEN_BOLD = '\033[32;1m'
+TGREEN = '\033[32m'
+TRED_BOLD = '\033[31;1m'
+TWHITE = '\033[37m'
+WHITE_BOLD = '\033[37;1m'
+CYAN_BOLD = '\033[36;1m'
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
new file mode 100644
index 000000000..9423f6c81
--- /dev/null
+++ b/tools/generate_categorical_fim.py
@@ -0,0 +1,155 @@
+import os
+from multiprocessing import Pool
+import argparse
+import traceback
+import sys
+
+sys.path.insert(1, 'foss_fim/tests')
+from inundation import inundate
+
+INPUTS_DIR = r'/data/inputs'
+
+# Define necessary variables for inundation().
+hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+mask_type, catchment_poly = 'huc', ''
+    
+
+def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, job_number, gpkg, extif, depthtif):
+    
+    # Create output directory and log directory.
+    if not os.path.exists(output_cat_fim_dir):
+        os.mkdir(output_cat_fim_dir)
+    log_dir = os.path.join(output_cat_fim_dir, 'logs')
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+    
+    no_data_list = []
+    procs_list = []
+       
+    # Loop through huc directories in the source_flow directory.
+    source_flow_dir_list = os.listdir(source_flow_dir)
+    for huc in source_flow_dir_list:
+        if "." not in huc:
+            
+            # Get list of AHPS site directories.
+            ahps_site_dir = os.path.join(source_flow_dir, huc)
+            ahps_site_dir_list = os.listdir(ahps_site_dir)
+            
+            # Map paths to HAND files needed for inundation().
+            fim_run_huc_dir = os.path.join(fim_run_dir, huc)
+            rem = os.path.join(fim_run_huc_dir, 'rem_zeroed_masked.tif')
+            catchments = os.path.join(fim_run_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+            hydroTable =  os.path.join(fim_run_huc_dir, 'hydroTable.csv')
+            
+            exit_flag = False  # Default to False.
+            
+            # Check if necessary data exist; set exit_flag to True if they don't exist.
+            for f in [rem, catchments, hydroTable]:
+                if not os.path.exists(f):
+                    print(f)
+                    no_data_list.append(f)
+                    exit_flag = True
+                    
+            # Log "Missing data" if missing TODO improve this.
+            if exit_flag == True:
+                f = open(os.path.join(log_dir, huc + '.txt'), 'w')
+                f.write("Missing data")
+                continue
+            
+            # Map path to huc directory inside out output_cat_fim_dir.
+            cat_fim_huc_dir = os.path.join(output_cat_fim_dir, huc)
+            if not os.path.exists(cat_fim_huc_dir):
+                os.mkdir(cat_fim_huc_dir)
+            
+            # Loop through AHPS sites.
+            for ahps_site in ahps_site_dir_list:
+                # Map parent directory for AHPS source data dir and list AHPS thresholds (act, min, mod, maj).
+                ahps_site_parent = os.path.join(ahps_site_dir, ahps_site)
+                thresholds_dir_list = os.listdir(ahps_site_parent)
+                
+                # Map parent directory for all inundation output filesoutput files.
+                cat_fim_huc_ahps_dir = os.path.join(cat_fim_huc_dir, ahps_site)
+                if not os.path.exists(cat_fim_huc_ahps_dir):
+                    os.mkdir(cat_fim_huc_ahps_dir)
+                        
+                # Loop through thresholds/magnitudes and define inundation output files paths
+                for magnitude in thresholds_dir_list:
+                    if "." not in magnitude:
+                        magnitude_flows_csv = os.path.join(ahps_site_parent, magnitude, 'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                        if os.path.exists(magnitude_flows_csv):
+                            if gpkg:
+                                output_extent_gpkg = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.gpkg')
+                            else:
+                                output_extent_gpkg = None
+                            if extif:
+                                output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
+                            else:
+                                output_extent_grid = None
+                            if depthtif:
+                                output_depth_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_depth.tif')
+                            else:
+                                output_depth_grid = None
+                            
+                            # Append necessary variables to list for multiprocessing.
+                            procs_list.append([rem, catchments, catchment_poly, magnitude_flows_csv, huc, hydroTable, output_extent_gpkg, output_extent_grid, output_depth_grid, ahps_site, magnitude, log_dir])
+    # Initiate multiprocessing.                                    
+    pool = Pool(job_number)
+    pool.map(run_inundation, procs_list)
+
+
+def run_inundation(args):
+    
+    # Parse args.
+    rem = args[0]
+    catchments = args[1]
+    catchment_poly = args[2]
+    magnitude_flows_csv = args[3]
+    huc = args[4]
+    hydroTable = args[5]
+    output_extent_gpkg = args[6]
+    output_extent_grid = args[7]
+    output_depth_grid = args[8]
+    ahps_site = args[9]
+    magnitude = args[10]
+    log_dir = args[11]
+    
+    print("Running inundation for " + str(os.path.split(os.path.split(output_extent_gpkg)[0])[0]))
+    try:
+        inundate(
+                 rem,catchments,catchment_poly,hydroTable,magnitude_flows_csv,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=output_extent_grid,inundation_polygon=output_extent_gpkg,
+                 depths=output_depth_grid,out_raster_profile=None,out_vector_profile=None,quiet=True
+                )
+    except Exception:
+        # Log errors and their tracebacks.
+        f = open(os.path.join(log_dir, huc + "_" + ahps_site + "_" + magnitude + '.txt'), 'w')
+        f.write(traceback.format_exc())
+        f.close()
+        
+        
+if __name__ == '__main__':
+    
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
+    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
+    parser.add_argument('-s', '--source-flow-dir',help='Path to directory containing flow CSVs to use to generate categorical FIM.',required=True, default="")
+    parser.add_argument('-o', '--output-cat-fim-dir',help='Path to directory where categorical FIM outputs will be written.',required=True, default="")
+    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
+    parser.add_argument('-gpkg','--write-geopackage',help='Using this option will write a geopackage.',required=False, action='store_true')
+    parser.add_argument('-extif','--write-extent-tiff',help='Using this option will write extent TIFFs. This is the default.',required=False, action='store_true')
+    parser.add_argument('-depthtif','--write-depth-tiff',help='Using this option will write depth TIFFs.',required=False, action='store_true')
+    
+    args = vars(parser.parse_args())
+    
+    fim_run_dir = args['fim_run_dir']
+    source_flow_dir = args['source_flow_dir']
+    output_cat_fim_dir = args['output_cat_fim_dir']
+    job_number = int(args['job_number'])
+    gpkg = args['write_geopackage']
+    extif = args['write_extent_tiff']
+    depthtif = args['write_depth_tiff']
+    
+    generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, job_number, gpkg, extif, depthtif)
+    
+    
+    
diff --git a/tools/inundation_wrapper_custom_flow.py b/tools/inundation_wrapper_custom_flow.py
index 8602bb008..e82a474e6 100644
--- a/tools/inundation_wrapper_custom_flow.py
+++ b/tools/inundation_wrapper_custom_flow.py
@@ -8,17 +8,11 @@
 
 import os
 import sys
-import pandas as pd
-import geopandas as gpd
-import rasterio
-import json
-import csv
 import argparse
 import shutil
 
 # insert python path at runtime for accessing scripts in foss_fim/tests dir (e.g. inundation.py)
 sys.path.insert(1, 'foss_fim/tests')
-from utils.shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
 from inundation import inundate
 
 TEST_CASES_DIR = r'/data/inundation_review/inundation_custom_flow/'  # Will update.
diff --git a/tools/inundation_wrapper_nwm_flows.py b/tools/inundation_wrapper_nwm_flows.py
index 1854bb5c3..f6d158a79 100755
--- a/tools/inundation_wrapper_nwm_flows.py
+++ b/tools/inundation_wrapper_nwm_flows.py
@@ -7,17 +7,12 @@
 
 import os
 import sys
-import pandas as pd
-import geopandas as gpd
-import rasterio
-import json
 import csv
 import argparse
 import shutil
 
 # insert python path at runtime for accessing scripts in foss_fim/tests dir (e.g. inundation.py)
 sys.path.insert(1, 'foss_fim/tests')
-from utils.shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
 from inundation import inundate
 
 TEST_CASES_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.

From 7db12841dd579ff7b9e2e3f59c49422c3d49c7c5 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 22 Feb 2021 09:27:01 -0600
Subject: [PATCH 022/359] Removed "_new" from test_cases dir

---
 tests/utils/shared_variables.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/utils/shared_variables.py b/tests/utils/shared_variables.py
index 4f844e3e9..ffcf4bfd0 100644
--- a/tests/utils/shared_variables.py
+++ b/tests/utils/shared_variables.py
@@ -1,6 +1,6 @@
 import os
 
-TEST_CASES_DIR = r'/data/test_cases_new/'  # TODO remove "_new"
+TEST_CASES_DIR = r'/data/test_cases/'
 PREVIOUS_FIM_DIR = r'/data/previous_fim'
 OUTPUTS_DIR = os.environ['outputDataDir']
 INPUTS_DIR = r'/data/inputs'

From 8e2ccc21c7f07cf69d13117ba959e4af35841c56 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 22 Feb 2021 10:29:23 -0600
Subject: [PATCH 023/359] Update CHANGELOG.md

---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f7f97b7dd..0b7aa15cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.5.1 - 2021-02-22
+
+Fixed `TEST_CASES_DIR` path in `tests/utils/shared_variables.py`.
+
+### Changes
+
+ - Removed `"_new"` from `TEST_CASES_DIR` variable.
+
 ## v3.0.5.0 - 2021-02-22 - [PR #267](https://github.com/NOAA-OWP/cahaba/pull/267)
 
 Enhancements to allow for evaluation at AHPS sites, the generation of a query-optimized metrics CSV, and the generation of categorical FIM. This merge requires that the `/test_cases` directory be updated for all machines performing evaluation.

From 6df54a5c787e3ed6b484412d83ef09aece699778 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 22 Feb 2021 17:08:00 -0600
Subject: [PATCH 024/359] Update README.md

---
 README.md | 60 ++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 5cf6f92ca..e4fc1bf6f 100644
--- a/README.md
+++ b/README.md
@@ -2,10 +2,12 @@
 
 Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Weather Service (NWS). This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIM). This repository also includes functionality to generate FIMs as well as tests to evaluate FIM prediction skill.
 
+<br/><br/>
 ## Dependencies
 
 [Docker](https://docs.docker.com/get-docker/)
 
+<br/><br/>
 ## Installation
 
 1. Install Docker : [Docker](https://docs.docker.com/get-docker/)
@@ -15,35 +17,42 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
     - Linux: `chgrp -R fim <path/to/repository>`
 
+<br/><br/>
 ## Configuration
 
 Software is configurable via parameters found in config directory. Copy files before editing and remove "template" pattern from the filename.
 Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
-- params_template.env
-- mannings_default.json
-    - must change filepath in params_template.env under "manning_n" variable name
-- params_calibrated.env
-    - runs calibrated mannings parameters from mannings_calibrated.json
+- `params_template.env`
+- `mannings_default.json`
+    - must change filepath in `params_template.env` under "manning_n" variable name
+- `params_calibrated.env`
+    - runs calibrated mannings parameters from `mannings_calibrated.json`
 
+<br/><br/>
 ## Input Data
 
 The following input data sources should be downloaded and preprocessed prior to executing the preprocessing & hydrofabric generation code:
-USACE National Levee Database:
--Access here: https://levees.sec.usace.army.mil/
--Recommend downloading the “Full GeoJSON” file for the area of interest
--Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
-AHPs site locations for MS extent (currently not available to public)
-NHDPlus HR datasets
--Acquire_and_preprocess_inputs.py
--aggregate_nhd_hr_streams.py
-NWM Hydrofabric
--nwm_flows.gpkg (currently not available to public)
--nwm_catchments.gpkg (currently not available to public)
--nwm_lakes.gpkg (currently not available to public)
--nwm_headwaters.gpkg - derived
-
-NOTE: Some of the input data is not easy to acquire and will need to be shared with outside users. We are currently working on providing this functionality and should be available soon.
+### USACE National Levee Database:
+- Access here: https://levees.sec.usace.army.mil/
+- Recommend downloading the “Full GeoJSON” file for the area of interest
+- Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
+- Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
 
+### NHDPlus HR datasets
+- `acquire_and_preprocess_inputs.py`
+- `aggregate_nhd_hr_streams.py`
+
+### AHPs Site Locations for MS Extent (See Note Below)
+
+### NWM Hydrofabric (See Note Below)
+- `nwm_flows.gpkg`
+- `nwm_catchments.gpkg`
+- `nwm_lakes.gpkg`
+- `nwm_headwaters.gpkg`
+
+**NOTE:** We are currently working on a long-term data sharing solution. Until then, please fill out this [Google form](https://docs.google.com/forms/d/e/1FAIpQLSf4jkg3Fcfgl-zTCeuTzKleiM_5tE5qwwUvVUQrjC9DBa7Ulg/viewform) to notify us that you would like to obtain the AHPS Site Locations NWM Hydrofabric.
+
+<br/><br/>
 ## Usage
 
 1. Run Docker Container : `docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <image_name>:<tag>`
@@ -57,8 +66,9 @@ NOTE: Some of the input data is not easy to acquire and will need to be shared w
         i. To run entire domain of available data use one of the `/data/inputs/included_huc[4,6,8].lst` files
     - Outputs can be found under `/data/outputs/<name_your_run>`
 
-## Evaluate FIM output to a Benchmark Dataset
-Once the hydrofabric has been generated from fim_run.sh for, evaluation against a benchmark dataset can be performed using binary contingency statistics. One benchmark dataset that can be used for evaluations are Base Level Engineering studies available on the FEMA Base Flood Elevation Viewer. To acquire FEMA datasets go to the FEMA Base Flood Elevation Viewer (https://webapps.usgs.gov/infrm/estbfe/) and download the file geodatabase and depth grids for a HUC. To perform an evaluation a flow forecast file is required and benchmark grids are preprocessed prior to running run_test_case.py.
+<br/><br/>
+## Evaluate Inundation Map Performance
+Once the hydrofabric has been generated from fim_run.sh for, evaluation against a benchmark dataset can be performed using binary contingency statistics. One benchmark dataset that can be used for evaluations are Base Level Engineering studies available on the FEMA Base Flood Elevation Viewer. To acquire FEMA datasets go to the FEMA Base Flood Elevation Viewer (https://webapps.usgs.gov/infrm/estbfe/) and download the file geodatabase and depth grids for a HUC. To perform an evaluation a flow forecast file is required and benchmark grids are preprocessed prior to running `run_test_case.py`.
 
 1. Flow Forecast File Creation
 `/foss_fim/tests/preprocess/create_flow_forecast_file.py -b <path to BLE geodatabase> -n <path to NWM geodatabase> -o <output directory> -xs <Cross Section layer name in BLE geodatabase> -hu <HUC layer name in BLE geodatabase> -huid <HUC ID field in HUC layer> -l <Stream layer name in NWM geodatabase> -f <feature id field in stream layer of NWM geodatabase>`
@@ -72,6 +82,7 @@ For HUC 12090301, the benchmark datasets (-b) are the 100 year (“BLE_DEP01PCT
 3. Run hydrologic evaluation (from inside Docker container): `/foss_fim/tests/run_test_case.py -r <fim_run_name/hucID> -b <name_of_test_instance_to_use> -t <test_case_id>`
     - More information can be found by running `/foss_fim/tests/run_test_case.py --help`
 
+<br/><br/>
 ## Dependencies
 
 Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/). To add new dependencies, from the projects's top-level directory:
@@ -90,23 +101,26 @@ and include both `Pipfile` and `Pipfile.lock` in your commits. The docker image
 
 If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
 
-
+<br/><br/>
 ## Known Issues & Getting Help
 
 Please see the issue tracker on GitHub for known issues and for getting help.
 
+<br/><br/>
 ## Getting Involved
 
 NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
 ----
 
+<br/><br/>
 ## Open Source Licensing Info
 1. [TERMS](TERMS.md)
 2. [LICENSE](LICENSE)
 
 ----
 
+<br/><br/>
 ## Credits and References
 1. Office of Water Prediction [(OWP)](https://water.noaa.gov/)
 2. National Flood Interoperability Experiment [(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)

From 08a4b9868e014ed1d84ceaeca428bc13a8473f19 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 22 Feb 2021 17:09:36 -0600
Subject: [PATCH 025/359] Update README.md

---
 README.md | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/README.md b/README.md
index e4fc1bf6f..1327ab3e4 100644
--- a/README.md
+++ b/README.md
@@ -2,12 +2,10 @@
 
 Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Weather Service (NWS). This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIM). This repository also includes functionality to generate FIMs as well as tests to evaluate FIM prediction skill.
 
-<br/><br/>
 ## Dependencies
 
 [Docker](https://docs.docker.com/get-docker/)
 
-<br/><br/>
 ## Installation
 
 1. Install Docker : [Docker](https://docs.docker.com/get-docker/)
@@ -17,7 +15,6 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
     - Linux: `chgrp -R fim <path/to/repository>`
 
-<br/><br/>
 ## Configuration
 
 Software is configurable via parameters found in config directory. Copy files before editing and remove "template" pattern from the filename.
@@ -28,7 +25,6 @@ Make sure to set the config folder group to 'fim' recursively using the chown co
 - `params_calibrated.env`
     - runs calibrated mannings parameters from `mannings_calibrated.json`
 
-<br/><br/>
 ## Input Data
 
 The following input data sources should be downloaded and preprocessed prior to executing the preprocessing & hydrofabric generation code:
@@ -52,7 +48,6 @@ The following input data sources should be downloaded and preprocessed prior to
 
 **NOTE:** We are currently working on a long-term data sharing solution. Until then, please fill out this [Google form](https://docs.google.com/forms/d/e/1FAIpQLSf4jkg3Fcfgl-zTCeuTzKleiM_5tE5qwwUvVUQrjC9DBa7Ulg/viewform) to notify us that you would like to obtain the AHPS Site Locations NWM Hydrofabric.
 
-<br/><br/>
 ## Usage
 
 1. Run Docker Container : `docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <image_name>:<tag>`
@@ -66,7 +61,6 @@ The following input data sources should be downloaded and preprocessed prior to
         i. To run entire domain of available data use one of the `/data/inputs/included_huc[4,6,8].lst` files
     - Outputs can be found under `/data/outputs/<name_your_run>`
 
-<br/><br/>
 ## Evaluate Inundation Map Performance
 Once the hydrofabric has been generated from fim_run.sh for, evaluation against a benchmark dataset can be performed using binary contingency statistics. One benchmark dataset that can be used for evaluations are Base Level Engineering studies available on the FEMA Base Flood Elevation Viewer. To acquire FEMA datasets go to the FEMA Base Flood Elevation Viewer (https://webapps.usgs.gov/infrm/estbfe/) and download the file geodatabase and depth grids for a HUC. To perform an evaluation a flow forecast file is required and benchmark grids are preprocessed prior to running `run_test_case.py`.
 
@@ -82,7 +76,6 @@ For HUC 12090301, the benchmark datasets (-b) are the 100 year (“BLE_DEP01PCT
 3. Run hydrologic evaluation (from inside Docker container): `/foss_fim/tests/run_test_case.py -r <fim_run_name/hucID> -b <name_of_test_instance_to_use> -t <test_case_id>`
     - More information can be found by running `/foss_fim/tests/run_test_case.py --help`
 
-<br/><br/>
 ## Dependencies
 
 Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/). To add new dependencies, from the projects's top-level directory:
@@ -101,26 +94,22 @@ and include both `Pipfile` and `Pipfile.lock` in your commits. The docker image
 
 If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
 
-<br/><br/>
 ## Known Issues & Getting Help
 
 Please see the issue tracker on GitHub for known issues and for getting help.
 
-<br/><br/>
 ## Getting Involved
 
 NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
 ----
 
-<br/><br/>
 ## Open Source Licensing Info
 1. [TERMS](TERMS.md)
 2. [LICENSE](LICENSE)
 
 ----
 
-<br/><br/>
 ## Credits and References
 1. Office of Water Prediction [(OWP)](https://water.noaa.gov/)
 2. National Flood Interoperability Experiment [(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)

From 6d17e9a9f78adbd983d73402ebd96e34cee009a2 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 22 Feb 2021 18:33:34 -0600
Subject: [PATCH 026/359] Update README.md

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1327ab3e4..d71154b0f 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
 ### Cahaba: Flood Inundation Mapping for U.S. National Water Model
 
-Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Weather Service (NWS). This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIM). This repository also includes functionality to generate FIMs as well as tests to evaluate FIM prediction skill.
+Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
+
+This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIMs). This repository also includes functionality to generate FIMs and tests to evaluate FIM prediction skill.
 
 ## Dependencies
 

From 21d05c602130ae2b689cb7a8f076ac15292d60b6 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 22 Feb 2021 18:52:45 -0600
Subject: [PATCH 027/359] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d71154b0f..fad922c31 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-### Cahaba: Flood Inundation Mapping for U.S. National Water Model
+## Cahaba: Flood Inundation Mapping for U.S. National Water Model
 
 Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
 
@@ -48,7 +48,7 @@ The following input data sources should be downloaded and preprocessed prior to
 - `nwm_lakes.gpkg`
 - `nwm_headwaters.gpkg`
 
-**NOTE:** We are currently working on a long-term data sharing solution. Until then, please fill out this [Google form](https://docs.google.com/forms/d/e/1FAIpQLSf4jkg3Fcfgl-zTCeuTzKleiM_5tE5qwwUvVUQrjC9DBa7Ulg/viewform) to notify us that you would like to obtain the AHPS Site Locations NWM Hydrofabric.
+**NOTE:** We are currently working on a long-term data sharing solution. Until then, please fill out this [Google form](https://docs.google.com/forms/d/e/1FAIpQLSf4jkg3Fcfgl-zTCeuTzKleiM_5tE5qwwUvVUQrjC9DBa7Ulg/viewform) to notify us that you would like to obtain the AHPS Site Locations NWM Hydrofabric data.
 
 ## Usage
 

From 0347e63bfa3e694a48c22b06bd16751a4c312cf8 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 07:52:17 -0600
Subject: [PATCH 028/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fad922c31..53ac82607 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ The following input data sources should be downloaded and preprocessed prior to
 - `nwm_lakes.gpkg`
 - `nwm_headwaters.gpkg`
 
-**NOTE:** We are currently working on a long-term data sharing solution. Until then, please fill out this [Google form](https://docs.google.com/forms/d/e/1FAIpQLSf4jkg3Fcfgl-zTCeuTzKleiM_5tE5qwwUvVUQrjC9DBa7Ulg/viewform) to notify us that you would like to obtain the AHPS Site Locations NWM Hydrofabric data.
+**NOTE:** We are currently working on a long-term data sharing solution. Until then, please contact Brad Bates (bradford.bates@noaa.gov) for data access.
 
 ## Usage
 

From 90dc6951c69d83e0efdb0aec223e87d5b39141a1 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:11:44 -0600
Subject: [PATCH 029/359] Update README.md

---
 README.md | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 53ac82607..9c8b4a340 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 
 ## Configuration
 
-Software is configurable via parameters found in config directory. Copy files before editing and remove "template" pattern from the filename.
+This software is configurable via parameters found in `config` directory. Copy files before editing and remove "template" pattern from the filename.
 Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
 - `params_template.env`
 - `mannings_default.json`
@@ -63,20 +63,11 @@ The following input data sources should be downloaded and preprocessed prior to
         i. To run entire domain of available data use one of the `/data/inputs/included_huc[4,6,8].lst` files
     - Outputs can be found under `/data/outputs/<name_your_run>`
 
-## Evaluate Inundation Map Performance
-Once the hydrofabric has been generated from fim_run.sh for, evaluation against a benchmark dataset can be performed using binary contingency statistics. One benchmark dataset that can be used for evaluations are Base Level Engineering studies available on the FEMA Base Flood Elevation Viewer. To acquire FEMA datasets go to the FEMA Base Flood Elevation Viewer (https://webapps.usgs.gov/infrm/estbfe/) and download the file geodatabase and depth grids for a HUC. To perform an evaluation a flow forecast file is required and benchmark grids are preprocessed prior to running `run_test_case.py`.
+## Evaluating Inundation Map Performance
+After `fim_run.sh` completes, you are ready to evaluate the model's skill. The following information can be used to preprocess the appropriate benchmark data against which model comparisons can be performed. Please note that you will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates). (A longer term data sharing solution is in the works).
 
-1. Flow Forecast File Creation
-`/foss_fim/tests/preprocess/create_flow_forecast_file.py -b <path to BLE geodatabase> -n <path to NWM geodatabase> -o <output directory> -xs <Cross Section layer name in BLE geodatabase> -hu <HUC layer name in BLE geodatabase> -huid <HUC ID field in HUC layer> -l <Stream layer name in NWM geodatabase> -f <feature id field in stream layer of NWM geodatabase>`
-For example, if HUC 12090301 were downloaded from the FEMA BFE viewer the geodatabase, “BLE_LowColoradoCummins.gdb”, contains a HUC Layer “S_HUC_Ar” (-hu) and a cross section layer “XS” (-xs). The HUC ID corresponds to the “HUC_CODE” field (-huid) within the “S_HUC_AR” layer.  Additionally, the National Water Model geodatabase (-n) will be required with the stream layer (-l) along with the ID field (-f) in the attribute table. Instructions on how to obtain the National Water Model GIS layers will be forthcoming.
-
-2. Process benchmark grid data
-`/foss_fim/tests/preprocess/preprocess_benchmark.py -b <path to ble grid (in geotiff format)> -r <path to a reference dataset> -o <path to output raster>`
-For HUC 12090301, the benchmark datasets (-b) are the 100 year (“BLE_DEP01PCT”) and 500 year (“BLE_DEP0_2PCT”) depth grids converted to Geotiff format. An example of a reference dataset (-r) is the “rem_zeroed_masked.tif” produced as part of the hydrofabric from fim_run.sh. The output raster name (if doing ble data) should be `ble_huc_<huc 08 code>_depth_<event>.tif` where event is '100yr' or '500yr'. Once the flow file and benchmark grids are processed, the output files are then placed in this folder (from inside a Docker container):
-`/foss_fim/tests_cases/validation_data_ble/<huc 08 code>/<event>/` where event is ‘100yr’ or ‘500yr’
-
-3. Run hydrologic evaluation (from inside Docker container): `/foss_fim/tests/run_test_case.py -r <fim_run_name/hucID> -b <name_of_test_instance_to_use> -t <test_case_id>`
-    - More information can be found by running `/foss_fim/tests/run_test_case.py --help`
+`/foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path_to_output_metrics_csv>-j [num_of_jobs]`
+    - More information can be found by running `/foss_fim/tests/synthesize_test_cases.py --help`
 
 ## Dependencies
 

From 83f9a3b21828cedc4abbb12ee3d17d8292720160 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:12:46 -0600
Subject: [PATCH 030/359] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9c8b4a340..d1600180f 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ The following input data sources should be downloaded and preprocessed prior to
 - `nwm_lakes.gpkg`
 - `nwm_headwaters.gpkg`
 
-**NOTE:** We are currently working on a long-term data sharing solution. Until then, please contact Brad Bates (bradford.bates@noaa.gov) for data access.
+**Please note:** We are currently working on a long-term data sharing solution. Until then, please contact Brad Bates (bradford.bates@noaa.gov) for data access.
 
 ## Usage
 
@@ -64,7 +64,7 @@ The following input data sources should be downloaded and preprocessed prior to
     - Outputs can be found under `/data/outputs/<name_your_run>`
 
 ## Evaluating Inundation Map Performance
-After `fim_run.sh` completes, you are ready to evaluate the model's skill. The following information can be used to preprocess the appropriate benchmark data against which model comparisons can be performed. Please note that you will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates). (A longer term data sharing solution is in the works).
+After `fim_run.sh` completes, you are ready to evaluate the model's skill. The following information can be used to preprocess the appropriate benchmark data against which model comparisons can be performed. **Please note:** You will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates). (A longer term data sharing solution is in the works).
 
 `/foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path_to_output_metrics_csv>-j [num_of_jobs]`
     - More information can be found by running `/foss_fim/tests/synthesize_test_cases.py --help`

From 747deaf6fba191403c49ad46f440e98b4e84eed4 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:19:36 -0600
Subject: [PATCH 031/359] Update README.md

---
 README.md | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index d1600180f..5f4393b80 100644
--- a/README.md
+++ b/README.md
@@ -40,15 +40,18 @@ The following input data sources should be downloaded and preprocessed prior to
 - `acquire_and_preprocess_inputs.py`
 - `aggregate_nhd_hr_streams.py`
 
-### AHPs Site Locations for MS Extent (See Note Below)
-
-### NWM Hydrofabric (See Note Below)
-- `nwm_flows.gpkg`
-- `nwm_catchments.gpkg`
-- `nwm_lakes.gpkg`
-- `nwm_headwaters.gpkg`
-
-**Please note:** We are currently working on a long-term data sharing solution. Until then, please contact Brad Bates (bradford.bates@noaa.gov) for data access.
+<br/><br/>
+**Please note:** For the following two datasets, please contact Brad Bates (bradford.bates@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
+
+- **NWM Hydrofabric**
+   - `nwm_flows.gpkg`
+   - `nwm_catchments.gpkg`
+   - `nwm_lakes.gpkg`
+   - `nwm_headwaters.gpkg`
+
+- **AHPS Site Locations (For Mainstem Configuration)**
+   - `nws_lid.gpkg`
+   - `ms_segs.gpkg`
 
 ## Usage
 

From 6c1e6a36282d55b456166e6e4df6bfc98aa0774a Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:21:43 -0600
Subject: [PATCH 032/359] Update README.md

---
 README.md | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 5f4393b80..cdac59744 100644
--- a/README.md
+++ b/README.md
@@ -39,19 +39,18 @@ The following input data sources should be downloaded and preprocessed prior to
 ### NHDPlus HR datasets
 - `acquire_and_preprocess_inputs.py`
 - `aggregate_nhd_hr_streams.py`
-
-<br/><br/>
+----
 **Please note:** For the following two datasets, please contact Brad Bates (bradford.bates@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
 
-- **NWM Hydrofabric**
-   - `nwm_flows.gpkg`
-   - `nwm_catchments.gpkg`
-   - `nwm_lakes.gpkg`
-   - `nwm_headwaters.gpkg`
+### NWM Hydrofabric
+- `nwm_flows.gpkg`
+- `nwm_catchments.gpkg`
+- `nwm_lakes.gpkg`
+- `nwm_headwaters.gpkg`
 
-- **AHPS Site Locations (For Mainstem Configuration)**
-   - `nws_lid.gpkg`
-   - `ms_segs.gpkg`
+### AHPS Site Locations (For Mainstem Configuration)
+- `nws_lid.gpkg`
+- `ms_segs.gpkg`
 
 ## Usage
 

From 1640e26add539395af8db7f53719838b8db89626 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:23:50 -0600
Subject: [PATCH 033/359] Update README.md

---
 README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index cdac59744..d964d6595 100644
--- a/README.md
+++ b/README.md
@@ -66,10 +66,11 @@ The following input data sources should be downloaded and preprocessed prior to
     - Outputs can be found under `/data/outputs/<name_your_run>`
 
 ## Evaluating Inundation Map Performance
-After `fim_run.sh` completes, you are ready to evaluate the model's skill. The following information can be used to preprocess the appropriate benchmark data against which model comparisons can be performed. **Please note:** You will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates). (A longer term data sharing solution is in the works).
+After `fim_run.sh` completes, you are ready to evaluate the model's skill. **Please note:** You will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates). (A longer term data sharing solution is in the works).
 
-`/foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path_to_output_metrics_csv>-j [num_of_jobs]`
-    - More information can be found by running `/foss_fim/tests/synthesize_test_cases.py --help`
+`/foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path_to_output_metrics_csv> -j [num_of_jobs]`
+
+More information can be found by running `/foss_fim/tests/synthesize_test_cases.py --help`
 
 ## Dependencies
 

From 4d42ccc92552e5781eddb8721f130d24f2450151 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:26:01 -0600
Subject: [PATCH 034/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d964d6595..6bbbb1cad 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 
 ## Configuration
 
-This software is configurable via parameters found in `config` directory. Copy files before editing and remove "template" pattern from the filename.
+This software is configurable via parameters found in the `config` directory. Copy files before editing and remove "template" pattern from the filename.
 Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
 - `params_template.env`
 - `mannings_default.json`

From ad97abb5970ec8cb88833d80271cafb84a574ed9 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:50:21 -0600
Subject: [PATCH 035/359] Update README.md

---
 README.md | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 6bbbb1cad..a9c2204c8 100644
--- a/README.md
+++ b/README.md
@@ -66,11 +66,14 @@ The following input data sources should be downloaded and preprocessed prior to
     - Outputs can be found under `/data/outputs/<name_your_run>`
 
 ## Evaluating Inundation Map Performance
-After `fim_run.sh` completes, you are ready to evaluate the model's skill. **Please note:** You will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates). (A longer term data sharing solution is in the works).
+After `fim_run.sh` completes, you are ready to evaluate the model's skill.
 
-`/foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path_to_output_metrics_csv> -j [num_of_jobs]`
+**Please note:** You will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates@noaa.gov). (A longer term data sharing solution is in the works).
 
-More information can be found by running `/foss_fim/tests/synthesize_test_cases.py --help`
+To evaluate model skill, run the following:
+`python /foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs]`
+
+More information can be found by running `python /foss_fim/tests/synthesize_test_cases.py --help`
 
 ## Dependencies
 

From a683d8b33c5a62fcd0bd4d03edb1c23b5a84283f Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 08:59:04 -0600
Subject: [PATCH 036/359] Update README.md

---
 README.md | 47 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index a9c2204c8..bc3ea9964 100644
--- a/README.md
+++ b/README.md
@@ -54,26 +54,47 @@ The following input data sources should be downloaded and preprocessed prior to
 
 ## Usage
 
-1. Run Docker Container : `docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <image_name>:<tag>`
-2. Acquire and Prepare Data : `/foss_fim/lib/acquire_and_preprocess_inputs.py -u <huc4s_to_process>`
-    - `-u` can be a single HUC4, series of HUC4s (e.g. 1209 1210), path to line-delimited file with HUC4s.
-    - Please run `/foss_fim/lib/acquire_and_preprocess_inputs.py --help` for more information.
-    - See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
-3. Aggregate NHD HR streams and create NWM headwater points : /foss_fim/lib/aggregate_vector_inputs.py
-4. Produce Hydrofabric : `fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>`
-    - `-u` can be a single huc, a series passed in quotes, or a line-delimited file
-        i. To run entire domain of available data use one of the `/data/inputs/included_huc[4,6,8].lst` files
-    - Outputs can be found under `/data/outputs/<name_your_run>`
+### Run Docker Container
+```
+docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <image_name>:<tag>
+```
+
+### Acquire and Prepare Data
+```
+/foss_fim/lib/acquire_and_preprocess_inputs.py -u <huc4s_to_process>
+```
+- `-u` can be a single HUC4, series of HUC4s (e.g. 1209 1210), path to line-delimited file with HUC4s.
+- Please run `/foss_fim/lib/acquire_and_preprocess_inputs.py --help` for more information.
+- See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
+
+### Aggregate NHD HR streams and create NWM headwater points 
+```
+/foss_fim/lib/aggregate_vector_inputs.py
+```
+### Produce Hydrofabric 
+```
+fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
+```
+- `-u` can be a single huc, a series passed in quotes, or a line-delimited file
+    i. To run entire domain of available data use one of the ```/data/inputs/included_huc[4,6,8].lst``` files
+- Outputs can be found under ```/data/outputs/<name_your_run>```
+
+----
 
 ## Evaluating Inundation Map Performance
 After `fim_run.sh` completes, you are ready to evaluate the model's skill.
 
-**Please note:** You will need access to the test_cases benchmark data. Similar to the other non-publicly available datasets, you can acquire the benchmark data from Brad Bates (bradford.bates@noaa.gov). (A longer term data sharing solution is in the works).
+**Please note:** You will need access to the test_cases benchmark data. You can acquire the benchmark data from Brad Bates (bradford.bates@noaa.gov). As mentioned before, a long term data sharing solution is still in the works.
 
 To evaluate model skill, run the following:
-`python /foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs]`
+```
+python /foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs]
+```
 
-More information can be found by running `python /foss_fim/tests/synthesize_test_cases.py --help`
+More information can be found by running:
+```
+python /foss_fim/tests/synthesize_test_cases.py --help
+```
 
 ## Dependencies
 

From e0276ad9f3f9b3703b6d563bef195eda65291b9a Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 09:00:23 -0600
Subject: [PATCH 037/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index bc3ea9964..f78721b15 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <i
 - Please run `/foss_fim/lib/acquire_and_preprocess_inputs.py --help` for more information.
 - See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
 
-### Aggregate NHD HR streams and create NWM headwater points 
+### Aggregate NHDHR Streams and Create NWM Headwater Points 
 ```
 /foss_fim/lib/aggregate_vector_inputs.py
 ```

From 9782453f3a6085b06bc0a67b9016ae93dffb2dc9 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 09:02:23 -0600
Subject: [PATCH 038/359] Update README.md

---
 README.md | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index f78721b15..1c3470d54 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,7 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 
 This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIMs). This repository also includes functionality to generate FIMs and tests to evaluate FIM prediction skill.
 
+----
 ## Dependencies
 
 [Docker](https://docs.docker.com/get-docker/)
@@ -27,6 +28,7 @@ Make sure to set the config folder group to 'fim' recursively using the chown co
 - `params_calibrated.env`
     - runs calibrated mannings parameters from `mannings_calibrated.json`
 
+----
 ## Input Data
 
 The following input data sources should be downloaded and preprocessed prior to executing the preprocessing & hydrofabric generation code:
@@ -36,10 +38,11 @@ The following input data sources should be downloaded and preprocessed prior to
 - Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
 - Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
 
+----
 ### NHDPlus HR datasets
 - `acquire_and_preprocess_inputs.py`
 - `aggregate_nhd_hr_streams.py`
-----
+
 **Please note:** For the following two datasets, please contact Brad Bates (bradford.bates@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
 
 ### NWM Hydrofabric
@@ -52,6 +55,7 @@ The following input data sources should be downloaded and preprocessed prior to
 - `nws_lid.gpkg`
 - `ms_segs.gpkg`
 
+----
 ## Usage
 
 ### Run Docker Container
@@ -80,7 +84,6 @@ fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_
 - Outputs can be found under ```/data/outputs/<name_your_run>```
 
 ----
-
 ## Evaluating Inundation Map Performance
 After `fim_run.sh` completes, you are ready to evaluate the model's skill.
 
@@ -96,6 +99,7 @@ More information can be found by running:
 python /foss_fim/tests/synthesize_test_cases.py --help
 ```
 
+----
 ## Dependencies
 
 Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/). To add new dependencies, from the projects's top-level directory:
@@ -114,22 +118,22 @@ and include both `Pipfile` and `Pipfile.lock` in your commits. The docker image
 
 If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
 
+----
 ## Known Issues & Getting Help
 
 Please see the issue tracker on GitHub for known issues and for getting help.
 
+----
 ## Getting Involved
 
 NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
 ----
-
 ## Open Source Licensing Info
 1. [TERMS](TERMS.md)
 2. [LICENSE](LICENSE)
 
 ----
-
 ## Credits and References
 1. Office of Water Prediction [(OWP)](https://water.noaa.gov/)
 2. National Flood Interoperability Experiment [(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)

From 48e23597713190e429e021e31a3df341aa8c9f0d Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 09:03:31 -0600
Subject: [PATCH 039/359] Update README.md

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1c3470d54..0d352a787 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 
 [Docker](https://docs.docker.com/get-docker/)
 
+----
 ## Installation
 
 1. Install Docker : [Docker](https://docs.docker.com/get-docker/)
@@ -18,6 +19,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
     - Linux: `chgrp -R fim <path/to/repository>`
 
+----
 ## Configuration
 
 This software is configurable via parameters found in the `config` directory. Copy files before editing and remove "template" pattern from the filename.
@@ -38,7 +40,6 @@ The following input data sources should be downloaded and preprocessed prior to
 - Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
 - Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
 
-----
 ### NHDPlus HR datasets
 - `acquire_and_preprocess_inputs.py`
 - `aggregate_nhd_hr_streams.py`

From 31af057baa8dcb367e041f6994fb70260e02967d Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 09:04:19 -0600
Subject: [PATCH 040/359] Update README.md

---
 README.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.md b/README.md
index 0d352a787..49e09b95a 100644
--- a/README.md
+++ b/README.md
@@ -4,12 +4,10 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 
 This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIMs). This repository also includes functionality to generate FIMs and tests to evaluate FIM prediction skill.
 
-----
 ## Dependencies
 
 [Docker](https://docs.docker.com/get-docker/)
 
-----
 ## Installation
 
 1. Install Docker : [Docker](https://docs.docker.com/get-docker/)
@@ -19,7 +17,6 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
     - Linux: `chgrp -R fim <path/to/repository>`
 
-----
 ## Configuration
 
 This software is configurable via parameters found in the `config` directory. Copy files before editing and remove "template" pattern from the filename.

From 1898e934dcc6d464d9b5796dd7df04a7109deb3a Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 23 Feb 2021 09:05:15 -0600
Subject: [PATCH 041/359] Update README.md

---
 README.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.md b/README.md
index 49e09b95a..a19932c17 100644
--- a/README.md
+++ b/README.md
@@ -121,17 +121,14 @@ If you are on a machine that has a particularly slow internet connection, you ma
 
 Please see the issue tracker on GitHub for known issues and for getting help.
 
-----
 ## Getting Involved
 
 NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
-----
 ## Open Source Licensing Info
 1. [TERMS](TERMS.md)
 2. [LICENSE](LICENSE)
 
-----
 ## Credits and References
 1. Office of Water Prediction [(OWP)](https://water.noaa.gov/)
 2. National Flood Interoperability Experiment [(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)

From ffa0a006b3247cccca8f566252bee8f3f7ec9a27 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Tue, 23 Feb 2021 13:59:17 -0600
Subject: [PATCH 042/359] Adding HAND SRC datum elev values to hydroTable.csv
 output (#272)

- Adding HAND SRC datum elev values to hydroTable.csv output
- Renamed hydroid attribute to "Median_Thal_Elev_m"
---
 CHANGELOG.md         | 9 +++++++++
 lib/add_crosswalk.py | 4 ++--
 lib/rem.py           | 4 ++--
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b7aa15cf..ac7cf3ae1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.5.2 - 2021-02-23
+
+Adding HAND SRC datum elev values to `hydroTable.csv` output
+
+### Changes
+
+ - Updated `add_crosswalk.py` to included "Median_Thal_Elev_m" variable outputs in hydroTable.csv
+ - Renamed hydroid attribute in `rem.py` to "Median" in case we want to include other statistics in the future (e.g. min, max, range etc.)
+
 ## v3.0.5.1 - 2021-02-22
 
 Fixed `TEST_CASES_DIR` path in `tests/utils/shared_variables.py`.
diff --git a/lib/add_crosswalk.py b/lib/add_crosswalk.py
index a0e57c7de..eb4198cb3 100755
--- a/lib/add_crosswalk.py
+++ b/lib/add_crosswalk.py
@@ -220,9 +220,9 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_hydro_table = output_hydro_table.merge(input_huc.loc[:,[FIM_ID,'HUC8']],how='left',on=FIM_ID)
 
     if output_flows.HydroID.dtype != 'str': output_flows.HydroID = output_flows.HydroID.astype(str)
-    output_hydro_table = output_hydro_table.merge(output_flows.loc[:,['HydroID','LakeID']],how='left',on='HydroID')
+    output_hydro_table = output_hydro_table.merge(output_flows.loc[:,['HydroID','LakeID','Median_Thal_Elev_m']],how='left',on='HydroID')
     output_hydro_table['LakeID'] = output_hydro_table['LakeID'].astype(int)
-
+    output_hydro_table['Median_Thal_Elev_m'] = output_hydro_table['Median_Thal_Elev_m'].astype(float).round(2)
     output_hydro_table = output_hydro_table.rename(columns={'HUC8':'HUC'})
     if output_hydro_table.HUC.dtype != 'str': output_hydro_table.HUC = output_hydro_table.HUC.astype(str)
 
diff --git a/lib/rem.py b/lib/rem.py
index 411380070..403edf9db 100755
--- a/lib/rem.py
+++ b/lib/rem.py
@@ -111,7 +111,7 @@ def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalw
 ###############################################
     # Merge and export dictionary to to_csv
     catchment_min_dict_df = pd.DataFrame.from_dict(catchment_min_dict, orient='index') # convert dict to dataframe
-    catchment_min_dict_df.columns = ['Min_Thal_Elev_meters']
+    catchment_min_dict_df.columns = ['Median_Thal_Elev_m']
     catchment_hydroid_dict_df = pd.DataFrame.from_dict(catchment_hydroid_dict, orient='index') # convert dict to dataframe
     catchment_hydroid_dict_df.columns = ['HydroID']
     merge_df = catchment_hydroid_dict_df.merge(catchment_min_dict_df, left_index=True, right_index=True)
@@ -119,7 +119,7 @@ def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalw
     merge_df.to_csv(hand_ref_elev_fileName,index=True) # export dataframe to csv file
 
     # Merge the HAND reference elvation by HydroID dataframe with the demDerived_reaches layer (add new layer attribute)
-    merge_df = merge_df.groupby(['HydroID']).median() # median value of all Min_Thal_Elev_meters for pixel catchments in each HydroID reach
+    merge_df = merge_df.groupby(['HydroID']).median() # median value of all Median_Thal_Elev_m for pixel catchments in each HydroID reach
     input_reaches = gpd.read_file(dem_reaches_filename)
     input_reaches = input_reaches.merge(merge_df, on='HydroID') # merge dataframes by HydroID variable
     input_reaches.to_file(dem_reaches_filename,driver=getDriver(dem_reaches_filename),index=False)

From e2ae250c2a0867d8e74a8c2c00be2b7df6c45627 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 24 Feb 2021 10:58:31 -0600
Subject: [PATCH 043/359] Bug fixes to new evaluation code and README.md
 cleanup

- Fixed a bug in synthesize_test_cases.py where the extent (MS/FR) was not being written to merged metrics file properly.
- Fixed a bug in synthesize_test_cases.py where only BLE test cases were being written to merged metrics file.
- Removed unused imports from inundation.py.
- Updated README.md

This resolves #270.
---
 CHANGELOG.md                    | 22 ++++++++++++++----
 README.md                       |  5 ++--
 tests/inundation.py             | 12 ++++------
 tests/run_test_case.py          | 23 ++++++++----------
 tests/synthesize_test_cases.py  | 41 +++++++++++++++++++++------------
 tests/utils/shared_variables.py |  6 ++---
 6 files changed, 62 insertions(+), 47 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac7cf3ae1..dc768902a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,28 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-## v3.0.5.2 - 2021-02-23
+## v3.0.5.3 - 2021-02-23 - [PR #275](https://github.com/NOAA-OWP/cahaba/pull/275)
 
-Adding HAND SRC datum elev values to `hydroTable.csv` output
+Bug fixes to new evaluation code.
 
 ### Changes
 
- - Updated `add_crosswalk.py` to included "Median_Thal_Elev_m" variable outputs in hydroTable.csv
- - Renamed hydroid attribute in `rem.py` to "Median" in case we want to include other statistics in the future (e.g. min, max, range etc.)
+ - Fixed a bug in `synthesize_test_cases.py` where the extent (MS/FR) was not being written to merged metrics file properly.
+ - Fixed a bug in `synthesize_test_cases.py` where only BLE test cases were being written to merged metrics file.
+ - Removed unused imports from `inundation.py`.
+ - Updated README.md
 
+<br/><br/>
+## v3.0.5.2 - 2021-02-23 - [PR #272](https://github.com/NOAA-OWP/cahaba/pull/272)
+
+Adds HAND synthetic rating curve (SRC) datum elevation values to `hydroTable.csv` output.
+
+### Changes
+
+ - Updated `add_crosswalk.py` to included "Median_Thal_Elev_m" variable outputs in `hydroTable.cs`v.
+ - Renamed hydroid attribute in `rem.py` to "Median" in case we want to include other statistics in the future (e.g. min, max, range etc.).
+
+<br/><br/>
 ## v3.0.5.1 - 2021-02-22
 
 Fixed `TEST_CASES_DIR` path in `tests/utils/shared_variables.py`.
@@ -18,6 +31,7 @@ Fixed `TEST_CASES_DIR` path in `tests/utils/shared_variables.py`.
 
  - Removed `"_new"` from `TEST_CASES_DIR` variable.
 
+<br/><br/>
 ## v3.0.5.0 - 2021-02-22 - [PR #267](https://github.com/NOAA-OWP/cahaba/pull/267)
 
 Enhancements to allow for evaluation at AHPS sites, the generation of a query-optimized metrics CSV, and the generation of categorical FIM. This merge requires that the `/test_cases` directory be updated for all machines performing evaluation.
diff --git a/README.md b/README.md
index a19932c17..cb091529a 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
 
-This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids, which together are used to produce flood inundation maps (FIMs). This repository also includes functionality to generate FIMs and tests to evaluate FIM prediction skill.
+This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids. This repository also includes functionality to generate flood inundation maps (FIMs) and evaluate FIM accuracy.
 
 ## Dependencies
 
@@ -33,8 +33,7 @@ Make sure to set the config folder group to 'fim' recursively using the chown co
 The following input data sources should be downloaded and preprocessed prior to executing the preprocessing & hydrofabric generation code:
 ### USACE National Levee Database:
 - Access here: https://levees.sec.usace.army.mil/
-- Recommend downloading the “Full GeoJSON” file for the area of interest
-- Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
+- Download the “Full GeoJSON” file for the area of interest
 - Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
 
 ### NHDPlus HR datasets
diff --git a/tests/inundation.py b/tests/inundation.py
index 679102432..b4db4fa49 100755
--- a/tests/inundation.py
+++ b/tests/inundation.py
@@ -1,27 +1,23 @@
 #!/usr/bin/env python3
 
-import sys
 import numpy as np
 import pandas as pd
-from numba import njit, typeof, typed, types
+from numba import njit, typed, types
 from concurrent.futures import ThreadPoolExecutor,as_completed
 from subprocess import run
 from os.path import splitext
 import rasterio
 import fiona
-import shapely
 from shapely.geometry import shape
-from fiona.crs import to_string
-from rasterio.errors import WindowError
 from rasterio.mask import mask
 from rasterio.io import DatasetReader,DatasetWriter
-from rasterio.features import shapes,geometry_window,dataset_features
-from rasterio.windows import transform,Window
 from collections import OrderedDict
 import argparse
 from warnings import warn
 from gdal import BuildVRT
 import geopandas as gpd
+
+
 def inundate(
              rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=None,hucs_layerName=None,
              subset_hucs=None,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
@@ -464,7 +460,7 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
 
         if hydroTable.empty:
             print ("All stream segments in HUC are within lake boundaries.")
-            sys.exit(0)
+            return
 
     elif isinstance(hydroTable,pd.DataFrame):
         pass #consider checking for correct dtypes, indices, and columns
diff --git a/tests/run_test_case.py b/tests/run_test_case.py
index a11fa9ab5..5beac4f16 100755
--- a/tests/run_test_case.py
+++ b/tests/run_test_case.py
@@ -103,11 +103,6 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                 lid_list.append(lid)
                 inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
                 extent_file_list.append(os.path.join(lid_dir, lid + '_extent.shp'))
-                    
-            ahps_inclusion_zones_dir = os.path.join(version_test_case_dir_parent, 'ahps_domains')
-            
-            if not os.path.exists(ahps_inclusion_zones_dir):
-                os.mkdir(ahps_inclusion_zones_dir)
 
         else:
             benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_depth_' + magnitude + '.tif')
@@ -190,7 +185,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
     parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-b', '--version-name',help='The name of the working version in which features are being tested',required=True,default="")
+    parser.add_argument('-b', '--version',help='The name of the working version in which features are being tested',required=True,default="")
     parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
     parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
     parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
@@ -210,14 +205,14 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     print()
 
     # Ensure test_id is valid.
-    if args['test_id'] not in valid_test_id_list:
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
-        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
-        for test_id in valid_test_id_list:
-          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
-              print(CYAN_BOLD + test_id + ENDC)
-        print()
-        exit_flag = True
+#    if args['test_id'] not in valid_test_id_list:
+#        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
+#        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
+#        for test_id in valid_test_id_list:
+#          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
+#              print(CYAN_BOLD + test_id + ENDC)
+#        print()
+#        exit_flag = True
 
     # Ensure fim_run_dir exists.
     if not os.path.exists(os.path.join(os.environ['outputDataDir'], args['fim_run_dir'])):
diff --git a/tests/synthesize_test_cases.py b/tests/synthesize_test_cases.py
index 65c241336..1fdb0a4dc 100644
--- a/tests/synthesize_test_cases.py
+++ b/tests/synthesize_test_cases.py
@@ -7,7 +7,7 @@
 import csv
 
 from run_test_case import run_alpha_test
-from utils.shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR
+from utils.shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
 
 
 def create_master_metrics_csv(master_metrics_csv_output):
@@ -57,7 +57,7 @@ def create_master_metrics_csv(master_metrics_csv_output):
                         ]
     
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
-    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config']]
+    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
     
     versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
     
@@ -77,17 +77,20 @@ def create_master_metrics_csv(master_metrics_csv_output):
                     
                     for magnitude in ['100yr', '500yr']:
                         for version in versions_to_aggregate:
-                            if '_fr_' in version:
+                            if '_fr' in version:
                                 extent_config = 'FR'
-                            if '_ms_' in version:
+                            elif '_ms' in version:
                                 extent_config = 'MS'
-                            if '_fr_' or '_ms_' not in version:
+                            else:
                                 extent_config = 'FR'
+                            if "_c" in version and version.split('_c')[1] == "":
+                                calibrated = "yes"
+                            else:
+                                calibrated = "no"
                             version_dir = os.path.join(official_versions, version)
                             magnitude_dir = os.path.join(version_dir, magnitude)
 
                             if os.path.exists(magnitude_dir):
-                                
                                 magnitude_dir_list = os.listdir(magnitude_dir)
                                 for f in magnitude_dir_list:
                                     if '.json' in f:
@@ -104,33 +107,37 @@ def create_master_metrics_csv(master_metrics_csv_output):
                                             sub_list_to_append.append(flow)
                                             sub_list_to_append.append(benchmark_source)
                                             sub_list_to_append.append(extent_config)
+                                            sub_list_to_append.append(calibrated)
                                             
                                             list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
                 
-        if benchmark_source in ['nws', 'usgs']:
-            test_cases_list = os.listdir(TEST_CASES_DIR)
+        if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
+            test_cases_list = os.listdir(benchmark_test_case_dir)
 
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
                     
                     huc = test_case.split('_')[0]
-                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'performance_archive', 'previous_versions')
+                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
                     
                     for magnitude in ['action', 'minor', 'moderate', 'major']:
                         for version in versions_to_aggregate:
-                            if '_fr_' in version:
+                            if '_fr' in version:
                                 extent_config = 'FR'
-                            if '_ms_' in version:
+                            elif '_ms' in version:
                                 extent_config = 'MS'
-                            if '_fr_' or '_ms_' not in version:
+                            else:
                                 extent_config = 'FR'
+                            if "_c" in version and version.split('_c')[1] == "":
+                                calibrated = "yes"
+                            else:
+                                calibrated = "no"
                                 
                             version_dir = os.path.join(official_versions, version)
                             magnitude_dir = os.path.join(version_dir, magnitude)
-                            
                             if os.path.exists(magnitude_dir):
                                 magnitude_dir_list = os.listdir(magnitude_dir)
                                 for f in magnitude_dir_list:
@@ -159,6 +166,7 @@ def create_master_metrics_csv(master_metrics_csv_output):
                                             sub_list_to_append.append(flow)
                                             sub_list_to_append.append(benchmark_source)
                                             sub_list_to_append.append(extent_config)
+                                            sub_list_to_append.append(calibrated)
                                             
                                             list_to_write.append(sub_list_to_append)
                 except ValueError:
@@ -202,7 +210,7 @@ def process_alpha_test(args):
     parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
     parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
     parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=True)
-        
+    
     # Assign variables from arguments.
     args = vars(parser.parse_args())
     config = args['config']
@@ -212,6 +220,10 @@ def process_alpha_test(args):
     benchmark_category = args['benchmark_category']
     overwrite = args['overwrite']
     master_metrics_csv = args['master_metrics_csv']
+    
+    if overwrite:
+        if input("Are you sure you want to overwrite metrics? y/n: ") == "n":
+            quit
         
     # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
     if fim_version != "all":
@@ -265,7 +277,6 @@ def process_alpha_test(args):
                         if not os.path.exists(fim_run_dir):
                             if config == 'DEV':
                                 fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
-                                print(fim_run_dir)
                             elif config == 'PREV':
                                 fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  
                         
diff --git a/tests/utils/shared_variables.py b/tests/utils/shared_variables.py
index ffcf4bfd0..292f55de1 100644
--- a/tests/utils/shared_variables.py
+++ b/tests/utils/shared_variables.py
@@ -1,16 +1,16 @@
 import os
 
+# Environmental variables and constants.
 TEST_CASES_DIR = r'/data/test_cases/'
 PREVIOUS_FIM_DIR = r'/data/previous_fim'
 OUTPUTS_DIR = os.environ['outputDataDir']
 INPUTS_DIR = r'/data/inputs'
+AHPS_BENCHMARK_CATEGORIES = ['usgs', 'nws']
 PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
 GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
 GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
-AHPS_BENCHMARK_CATEGORIES = ['usgs', 'ble']
-
-
 
+# Colors.
 ENDC = '\033[m'
 TGREEN_BOLD = '\033[32;1m'
 TGREEN = '\033[32m'

From ca68f2b492e3e643cc4a254a09770789858d5e0e Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 25 Feb 2021 08:37:21 -0600
Subject: [PATCH 044/359] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dc768902a..48dc68648 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,7 +19,7 @@ Adds HAND synthetic rating curve (SRC) datum elevation values to `hydroTable.csv
 
 ### Changes
 
- - Updated `add_crosswalk.py` to included "Median_Thal_Elev_m" variable outputs in `hydroTable.cs`v.
+ - Updated `add_crosswalk.py` to included "Median_Thal_Elev_m" variable outputs in `hydroTable.csv`.
  - Renamed hydroid attribute in `rem.py` to "Median" in case we want to include other statistics in the future (e.g. min, max, range etc.).
 
 <br/><br/>

From a92212db5612c8c5b0bec0227be7f6de20c5f302 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Thu, 25 Feb 2021 12:45:26 -0600
Subject: [PATCH 045/359] Enhancement that creates metric plots and summary
 statistics.

Enhancement that creates metric plots and summary statistics using metrics compiled by synthesize_test_cases.py.

Additions
- Added eval_plots.py, which produces:
- Boxplots of CSI, FAR, and POD/TPR
- Barplot of aggregated CSI scores
- Scatterplot of CSI comparing two FIM versions
- CSV of aggregated statistics (CSI, FAR, POD/TPR)
- CSV of analyzed data and analyzed sites

This resolves #70.
---
 CHANGELOG.md                          |  13 +
 tests/plots/eval_plots.py             | 343 ++++++++++++++++++++++++++
 tests/plots/utils/__init__.py         |   0
 tests/plots/utils/shared_functions.py | 321 ++++++++++++++++++++++++
 4 files changed, 677 insertions(+)
 create mode 100644 tests/plots/eval_plots.py
 create mode 100644 tests/plots/utils/__init__.py
 create mode 100644 tests/plots/utils/shared_functions.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 48dc68648..8e3d7b31f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.6.0 - 2021-02-25 - [PR #276](https://github.com/NOAA-OWP/cahaba/pull/276)
+
+Enhancement that creates metric plots and summary statistics using metrics compiled by `synthesize_test_cases.py`. 
 
+### Additions
+ - Added `eval_plots.py`, which produces:
+    - Boxplots of CSI, FAR, and POD/TPR
+    - Barplot of aggregated CSI scores
+    - Scatterplot of CSI comparing two FIM versions
+    - CSV of aggregated statistics (CSI, FAR, POD/TPR)
+    - CSV of analyzed data and analyzed sites
+
+<br/><br/>
 ## v3.0.5.3 - 2021-02-23 - [PR #275](https://github.com/NOAA-OWP/cahaba/pull/275)
 
 Bug fixes to new evaluation code.
diff --git a/tests/plots/eval_plots.py b/tests/plots/eval_plots.py
new file mode 100644
index 000000000..9c29087e1
--- /dev/null
+++ b/tests/plots/eval_plots.py
@@ -0,0 +1,343 @@
+#!/usr/bin/env python3
+import pandas as pd
+from pathlib import Path
+import argparse
+from natsort import natsorted
+import geopandas as gpd
+from utils.shared_functions import filter_dataframe, boxplot, scatterplot, barplot
+def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial_ahps = False, fim_1_ms = False):
+
+    '''
+    Creates plots and summary statistics using metrics compiled from 
+    synthesize_test_cases. Required inputs are metrics_csv and workspace. 
+    Outputs include:
+        aggregate_<benchmark source>_<configuration>.csv: this csv 
+            contains the aggregated total statistics (i.e. CSI, FAR, POD)
+            using the summed area_sq_km fields
+        <benchmark source>_<configuration>_common_sites.csv: this csv 
+            contains the unique sites (e.g usgs/nws: nws_lid; ble: huc08) 
+            considered for aggregation/plots for each magnitude. The selected
+            sites occur in all versions analyzed. For example, if FIM 1,
+            FIM 2, FIM 3.0.0.3 were versions analyzed, the common sites 
+            would be those that had data for ALL versions. This 
+            analysis is then redone for each magnitude. As such, the number
+            of sites may vary with magnitude. The number of sites for each
+            magnitude is annotated on generated plots.
+        <benchmark source>_<configuration>_analyzed_data.csv: this is the 
+            dataset used to create plots and aggregate statistics. It is 
+            a subset of the input metrics file and consists of the common
+            sites.
+        csi_aggr_<benchmark source>_<configuration>.png: bar plot of the 
+            aggregated CSI scores. Number of common sites is annotated
+            (see list of sites listed in *_*_common_sites.csv).
+        csi_<benchmark source>_<configuration>.png: box plot of CSI scores 
+            (sites weighted equally). Number of common sites is annotated 
+            (see list of sites listed in *_*_common_sites.csv).
+        far_<benchmark source>_<configuration>*.png: box plot of FAR scores
+            (sites weighted equally). Number of common sites is annotated 
+            (see list of sites listed in *_*_common_sites.csv).
+        tpr_<benchmark source>_<configuration>*.png: box plot of TPR/POD 
+            scores (sites weighted equally). Number of common sites is 
+            annotated (see list of sites listed in *_*_common_sites.csv).
+        csi_scatter_<magnitude>_<configuration>*.png: scatter plot comparing 
+            two versions for a given magnitude. This is only generated if
+            there are exactly two versions analyzed.
+
+    Parameters
+    ----------
+    metrics_csv : STRING
+        Path to csv produced as part of synthesize_test_cases containing
+        all metrics across all versions.
+    workspace : STRING
+        Path to the output workspace. Subdirectories will be created 
+        reflecting the evaluation datasets.
+    versions: LIST
+        A list of versions to be aggregated/plotted. Uses the "startswith" 
+        approach. Versions should be supplied in the order they are to 
+        be plotted. For example: ['fim_', 'fb']; This will evaluate all 
+        versions that start with fim_ (e.g. fim_1, fim_2, fim_3) and any
+        feature branch that starts with "fb". To esbalish version order,
+        the fim versions are naturally sorted and then fb versions 
+        (naturally sorted) are appended. These versions are also used to 
+        filter the input metric csv as only these versions are retained 
+        for analysis. 
+    stats: LIST
+        A list of statistics to be plotted. Must be identical to column 
+        field in metrics_csv. CSI, POD, TPR are currently calculated, if 
+        additional statistics are desired formulas would need to be coded.
+    alternate_ahps_query : STRING, optional
+        The default is false. Currently the default ahps query is same 
+        as done for apg goals. If a different query is desired it can be 
+        supplied and it will supercede the default query. 
+    spatial_ahps : DICTIONARY, optional
+        The default is false. A dictionary with keys as follows: 
+            'static': Path to AHPS point file created during creation of
+                FIM 3 static libraries.
+            'evaluated': Path to extent file created during the creation
+                of the NWS/USGS AHPS preprocessing.
+            'metadata': Path to previously created file that contains 
+                metadata about each site (feature_id, wfo, rfc and etc).
+        No spatial layers will be created if set to False, if a dictionary
+        is supplied then a spatial layer is produced.
+    fim_1_ms: BOOL
+        Default is false. If True then fim_1 rows are duplicated with 
+        extent_config set to MS. This allows for FIM 1 to be included 
+        in MS plots/stats (helpful for nws/usgs ahps comparisons).
+
+    Returns
+    -------
+    all_datasets : DICT
+        Dictionary containing all datasets generated. 
+        Keys: (benchmark_source, extent_config), 
+        Values: (filtered dataframe, common sites)
+
+    '''
+    
+    #Import metrics csv as DataFrame and initialize all_datasets dictionary
+    csv_df = pd.read_csv(metrics_csv)
+
+    #fim_1_ms flag enables FIM 1 to be shown on MS plots/stats
+    if fim_1_ms:
+        #Query FIM 1 rows based on version beginning with "fim_1"
+        fim_1_rows = csv_df.query('version.str.startswith("fim_1")').copy()
+        #Set extent configuration to MS (instead of FR)
+        fim_1_rows['extent_config'] = 'MS'
+        #Append duplicate FIM 1 rows to original dataframe
+        csv_df = csv_df.append(fim_1_rows, ignore_index = True)
+        
+    #If versions are supplied then filter out    
+    if versions:
+        #Filter out versions based on supplied version list
+        metrics = csv_df.query('version.str.startswith(tuple(@versions))')
+    else:
+        metrics = csv_df
+       
+    #Group by benchmark source
+    benchmark_by_source = metrics.groupby(['benchmark_source', 'extent_config'])
+
+    #Iterate through benchmark_by_source. Pre-filter metrics dataframe 
+    #as needed (e.g. usgs/nws filter query). Then further filtering to 
+    #discard all hucs/nws_lid that are not present across all analyzed 
+    #versions for a given magnitude. The final filtered dataset is written 
+    #to a dictionary with the key (benchmark source, extent config) 
+    #and values (filtered dataframe, common sites).
+    all_datasets = {}
+    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:        
+                
+        #If source is usgs/nws define the base resolution and query 
+        #(use alternate query if passed). Append filtered datasets to 
+        #all_datasets dictionary.
+        if benchmark_source in ['usgs','nws']:
+            
+            #Set the base processing unit for the ahps runs.
+            base_resolution = 'nws_lid'
+            
+            #Default query (used for APG) it could be that bad_sites should be modified. If so pass an alternate query using the "alternate_ahps_query"
+            bad_sites = ['grfi2','ksdm7','hohn4','rwdn4']
+            query = "not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites"
+
+            #If alternate ahps evaluation query argument is passed, use that.
+            if alternate_ahps_query:
+                query = alternate_ahps_query
+
+            #Filter the dataset based on query
+            ahps_metrics = benchmark_metrics.query(query)
+            
+            #Filter out all instances where the base_resolution doesn't 
+            #exist across all desired fim versions for a given magnitude.
+            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
+                     
+        #If source is 'ble', set base_resolution and append ble dataset 
+        #to all_datasets dictionary
+        elif benchmark_source == 'ble':
+            
+            #Set the base processing unit for ble runs
+            base_resolution = 'huc'
+            
+            #Filter out all instances where base_resolution doesn't exist 
+            #across all desired fim versions for a given magnitude.
+            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(benchmark_metrics, base_resolution)
+            
+    #For each dataset in all_datasets, generate plots and aggregate statistics.
+    for (dataset_name,configuration), (dataset, sites) in all_datasets.items():
+        
+        #Define and create the output workspace as a subfolder within 
+        #the supplied workspace
+        output_workspace = Path(workspace) / dataset_name / configuration.lower()
+        output_workspace.mkdir(parents = True, exist_ok = True)         
+                
+        #Write out the filtered dataset and common sites to file
+        dataset.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_analyzed_data.csv'), index = False)
+        sites_pd = pd.DataFrame.from_dict(sites, orient = 'index').transpose()
+        sites_pd.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_common_sites.csv'), index = False)
+        
+        #set the order of the magnitudes and define base resolution.
+        if dataset_name == 'ble':
+            magnitude_order = ['100yr', '500yr']
+            base_resolution = 'huc'
+        elif dataset_name in ['usgs','nws']:
+            magnitude_order = ['action','minor','moderate','major']
+            base_resolution = 'nws_lid'
+
+        #Calculate aggregated metrics based on total_sq_km fields.
+        dataset_sums = dataset.groupby(['version', 'magnitude'])[['TP_area_km2','FP_area_km2','FN_area_km2']].sum()
+        dataset_sums['csi'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'] + dataset_sums['FN_area_km2'])
+        dataset_sums['far'] = dataset_sums['FP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'])
+        dataset_sums['pod'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FN_area_km2'])
+        dataset_sums = dataset_sums.reset_index()
+        
+        #Write aggregated metrics to file.
+        dataset_sums.to_csv(output_workspace / f'aggregate_{dataset_name}_{configuration.lower()}.csv', index = False )
+
+        #This section naturally orders analyzed versions which defines 
+        #the hue order for the generated plots.
+        #Get all versions in dataset
+        all_versions = list(dataset.version.unique())        
+        version_order = []        
+        #If versions are not specified then use all available versions 
+        #and assign to versions_list
+        if not versions:
+            versions_list = all_versions
+        #if versions are supplied assign to versions_list
+        else:
+            versions_list = versions        
+        #For each version supplied by the user
+        for version in versions_list:
+            #Select all the versions that start with the supplied version.
+            selected_versions = [sel_version for sel_version in all_versions if sel_version.startswith(version)]
+            #Naturally sort selected_versions
+            selected_versions = natsorted(selected_versions)
+            #Populate version order based on the sorted subsets.
+            version_order.extend(selected_versions)
+            
+        #Define textbox which will contain the counts of each magnitude.
+        textbox = []
+        for magnitude in sites:
+            count = len(sites[magnitude])
+            line_text = f'{magnitude.title()} Sites = {count}'
+            textbox.append(line_text)
+        textbox = '\n'.join(textbox)
+
+        #Create aggregate barplot
+        aggregate_file = output_workspace / (f'csi_aggr_{dataset_name}_{configuration.lower()}.png')        
+        barplot(dataframe = dataset_sums, x_field = 'magnitude', x_order = magnitude_order, y_field = 'csi', hue_field = 'version', ordered_hue = version_order, title_text = f'Aggregate {dataset_name.upper()} FIM Scores', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = aggregate_file)
+        
+        #Create box plots for each metric in supplied stats.
+        for stat in stats:
+            output_file = output_workspace / (f'{stat.lower()}_{dataset_name}_{configuration.lower()}.png')    
+            boxplot(dataframe = dataset, x_field = 'magnitude', x_order = magnitude_order, y_field = stat, hue_field = 'version', ordered_hue = version_order, title_text = f'{dataset_name.upper()} FIM Sites', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = output_file)
+        
+        #Get the last 2 versions from the version order for scatter plot.
+        if len(version_order) == 2:            
+            x_version, y_version = version_order
+            for magnitude in magnitude_order:
+                #Scatterplot comparison between last 2 versions.
+                x_csi = dataset.query(f'version == "{x_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
+                y_csi = dataset.query(f'version == "{y_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
+                plotdf = pd.merge(x_csi, y_csi, on = base_resolution, suffixes = (f"_{x_version}",f"_{y_version}"))
+                #Define arguments for scatterplot function.
+                title_text = f'CSI {magnitude}'
+                dest_file = output_workspace / f'csi_scatter_{magnitude}_{configuration.lower()}.png'
+                scatterplot(dataframe = plotdf, x_field = f'CSI_{x_version}', y_field = f'CSI_{y_version}', title_text = title_text, annotate = False, dest_file = dest_file)
+    
+
+    #######################################################################
+    #Create spatial layers with threshold and mapping information
+    ########################################################################
+    if spatial_ahps:
+
+        #Read in supplied shapefile layers
+        #Layer containing metadata for each site (feature_id, wfo, etc). 
+        #Convert nws_lid to lower case.
+        ahps_metadata = gpd.read_file(spatial_ahps['metadata'])
+        ahps_metadata['nws_lid'] = ahps_metadata['nws_lid'].str.lower()
+        metadata_crs = ahps_metadata.crs
+        
+        #Extent layer generated from preprocessing NWS/USGS datasets
+        evaluated_ahps_extent = gpd.read_file(spatial_ahps['evaluated'])
+        
+        #Extent layer generated from static ahps library preprocessing
+        static_library = gpd.read_file(spatial_ahps['static'])
+        
+        #Fields to keep
+        #Get list of fields to keep in merge
+        preserved_static_library_fields = ['nws_lid'] + [i for i in static_library.columns if i.startswith(('Q','S'))]                
+        #Get list of fields to keep in merge.
+        preserved_evaluated_ahps_fields = ['nws_lid', 'source', 'geometry'] + [i for i in evaluated_ahps_extent.columns if i.startswith(('action','minor','moderate','major'))]
+
+        #Join tables to evaluated_ahps_extent
+        evaluated_ahps_extent = evaluated_ahps_extent[preserved_evaluated_ahps_fields]
+        evaluated_ahps_extent = evaluated_ahps_extent.merge(ahps_metadata, on = 'nws_lid')
+        evaluated_ahps_extent['geometry'] = evaluated_ahps_extent['geometry_y']
+        evaluated_ahps_extent.drop(columns = ['geometry_y','geometry_x'], inplace = True)
+        evaluated_ahps_extent = evaluated_ahps_extent.merge(static_library[preserved_static_library_fields], on = 'nws_lid')
+        
+        #Join dataset metrics to evaluated_ahps_extent data. 
+        final_join = pd.DataFrame()
+        for (dataset_name, configuration), (dataset, sites) in all_datasets.items():
+            #Only select ahps from dataset if config is MS
+            if dataset_name in ['usgs','nws'] and configuration == 'MS':
+                #Select records from evaluated_ahps_extent that match the dataset name
+                subset = evaluated_ahps_extent.query(f'source == "{dataset_name}"')                        
+                #Join to dataset
+                dataset_with_subset = dataset.merge(subset, on = 'nws_lid')
+                #Append rows to final_join dataframe
+                final_join = final_join.append(dataset_with_subset)
+        
+        #Modify version field
+        final_join['version'] = final_join.version.str.split('_nws|_usgs').str[0]
+        
+        #Write geodataframe to file
+        gdf = gpd.GeoDataFrame(final_join, geometry = final_join['geometry'], crs = metadata_crs)
+        output_shapefile = Path(workspace) / 'nws_usgs_site_info.shp'
+        gdf.to_file(output_shapefile) 
+                    
+
+
+#######################################################################
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
+    parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
+    parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
+    parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
+    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR'], required = False)
+    parser.add_argument('-q', '--alternate_ahps_query',help = 'Alternate filter query for AHPS. Default is: "not nws_lid.isnull() & not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites" where bad_sites are (grfi2,ksdm7,hohn4,rwdn4)', default = False, required = False)
+    parser.add_argument('-sp', '--spatial_ahps', help = 'If spatial point layer is desired, supply a csv with 3 lines of the following format: metadata, path/to/metadata/shapefile\nevaluated, path/to/evaluated/shapefile\nstatic, path/to/static/shapefile.', default = False, required = False)
+    parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
+    
+    #Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+    
+    #If errors occur reassign error to True
+    error = False
+    #Create dictionary if file specified for spatial_ahps
+    if args['spatial_ahps']:
+        #Create dictionary
+        spatial_dict = {}
+        with open(args['spatial_ahps']) as file:
+            for line in file:
+                key, value = line.strip('\n').split(',')
+                spatial_dict[key] = Path(value)
+        args['spatial_ahps'] = spatial_dict
+        #Check that all required keys are present and overwrite args with spatial_dict
+        required_keys = set(['metadata', 'evaluated', 'static'])
+        if required_keys - spatial_dict.keys():
+          print('\n Required keys are: metadata, evaluated, static')
+          error = True
+        else:
+            args['spatial_ahps'] = spatial_dict
+
+
+    #Finalize Variables
+    m = args['metrics_csv']
+    w = args['workspace']
+    v = args['versions']
+    s = args['stats']
+    q = args['alternate_ahps_query']
+    sp= args['spatial_ahps']
+    f = args['fim_1_ms']
+
+    #Run eval_plots function
+    if not error:        
+        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f)
\ No newline at end of file
diff --git a/tests/plots/utils/__init__.py b/tests/plots/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/plots/utils/shared_functions.py b/tests/plots/utils/shared_functions.py
new file mode 100644
index 000000000..60342059e
--- /dev/null
+++ b/tests/plots/utils/shared_functions.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import re
+#########################################################################
+#Create boxplot
+#########################################################################
+def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):      
+    '''
+    Create boxplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis
+    x_order : List
+        Order to arrange the x-axis.
+    y_field : STR
+        Field to use for the y-axis
+    hue_field : STR
+        Field to use for hue (typically FIM version)
+    title_text : STR
+        Text for plot title.
+    fim_configuration: STR
+        Configuration of FIM (FR or MS or Composite).
+    simplify_legend : BOOL, optional
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        The default is False.
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. The default is False.
+
+    Returns
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    #Use seaborn to plot the boxplot
+    axes=sns.boxplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
+    #set title of plot
+    axes.set_title(f'{title_text} ({y_field})',fontsize=20, weight = 'bold')
+    #Set yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    for index,ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel('',fontsize=0,weight = 'bold')
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+    axes.legend(markerscale = 2, fontsize =20, loc = 'lower left')
+    
+    #If simple legend desired
+    if simplify_legend:
+        #trim labels to FIM 1, FIM 2, and the FIM 3 version    
+        handles, org_labels = axes.get_legend_handles_labels()
+        label_dict = {}
+        for label in org_labels:
+            if 'fim_1' in label:
+                label_dict[label] = 'FIM 1'
+            elif 'fim_2' in label:
+                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()                  
+            elif 'fim_3' in label:
+                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                if label.endswith('_c'):
+                    label_dict[label] = label_dict[label] + ' c'
+            else:
+                label_dict[label] = label + ' ' + fim_configuration.lower()
+        #Define simplified labels as a list.
+        new_labels = [label_dict[label] for label in org_labels]
+        #Define legend location. FAR needs to be in different location than CSI/POD.
+        if y_field == 'FAR':
+            legend_location = 'upper right'
+        else:
+            legend_location = 'lower left' 
+        #rename legend labels to the simplified labels.
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
+    #Print textbox if supplied
+    if textbox_str:
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=14, verticalalignment='top', bbox=box_props)
+
+    #If figure to be saved to disk, then do so, otherwise return figure
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+
+#########################################################################
+#Create scatter plot
+#########################################################################
+def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):      
+    '''
+    Create boxplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis (Assumes FIM 2)
+    y_field : STR
+        Field to use for the y-axis (Assumes FIM 3)
+    title_text : STR
+        Text for plot title.    
+    stats_text : STR or BOOL
+        Text for stats to place on chart. Default is false (no stats printed)
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. The default is False.
+
+    Returnsy
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    
+    #Use seaborn to plot the boxplot
+    axes=sns.scatterplot(data=dataframe, x=x_field, y=y_field, color = 'black', s = 150)
+    
+    #Set xticks and yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
+    axes.grid(b=True, which='major', axis='both')    
+    
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel(f'{x_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
+
+    #Plot diagonal line
+    diag_range = [0,1]
+    axes.plot(diag_range, diag_range, color='gray', transform=axes.transAxes)
+
+
+    #set title of plot
+    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
+   
+    if annotate:
+        #Set text for labels
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        textbox_str = 'Target Better'
+        axes.text(0.3, 0.6, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+        textbox_str = 'Baseline Better'
+        axes.text(0.5, 0.2, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+
+    if stats_text:
+        #Add statistics textbox
+        axes.text(0.01, 0.80, stats_text, transform=axes.transAxes, fontsize=24, verticalalignment='top', bbox=box_props)
+
+    #If figure to be saved to disk, then do so, otherwise return fig
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+#########################################################################
+#Create barplot
+#########################################################################
+def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):      
+    '''
+    Create barplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis
+    x_order : List
+        Order to arrange the x-axis.
+    y_field : STR
+        Field to use for the y-axis
+    hue_field : STR
+        Field to use for hue (typically FIM version)
+    title_text : STR
+        Text for plot title.
+    fim_configuration: STR
+        Configuration of FIM (FR or MS or Composite).
+    simplify_legend : BOOL, optional
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        Default is False.
+    display_values : BOOL, optional
+        If True, Y values will be displayed above bars. 
+        Default is False.
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. Default is False.
+
+    Returns
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    #Use seaborn to plot the boxplot
+    axes=sns.barplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
+    #set title of plot
+    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
+    #Set yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    for index,ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.upper()}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel('',fontsize=0,weight = 'bold')
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+    axes.legend(markerscale = 2, fontsize =20, loc = 'upper right')
+    #If simple legend desired
+    if simplify_legend:
+        #trim labels to FIM 1, FIM 2, FIM 3    
+        handles, org_labels = axes.get_legend_handles_labels()
+        label_dict = {}
+        for label in org_labels:
+            if 'fim_1' in label:
+                label_dict[label] = 'FIM 1'
+            elif 'fim_2' in label:
+                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
+            elif 'fim_3' in label:
+                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                if label.endswith('_c'):
+                    label_dict[label] = label_dict[label] + ' c'
+            else:
+                label_dict[label] = label + ' ' + fim_configuration.lower()
+        #Define simplified labels as a list.
+        new_labels = [label_dict[label] for label in org_labels]
+        #rename legend labels to the simplified labels.
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
+    #Add Textbox
+    if textbox_str:
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=18, verticalalignment='top', bbox=box_props)
+
+    #Display Y values above bars
+    if display_values:
+        #Add values of bars directly above bar.
+        for patch in axes.patches:
+            value = round(patch.get_height(),3)
+            axes.text(patch.get_x()+patch.get_width()/2.,
+                    patch.get_height(),
+                    '{:1.3f}'.format(value),
+                    ha="center", fontsize=18)  
+    
+    #If figure to be saved to disk, then do so, otherwise return fig
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+#######################################################################
+#Filter dataframe generated from csv file from run_test_case aggregation
+########################################################################
+def filter_dataframe(dataframe, unique_field):
+    '''
+
+    This script will filter out the sites (or hucs) which are not consistently 
+    found for all versions for a given magnitude. For example, an AHPS 
+    lid site must have output for all 3 versions (fim1, fim2, fim3) for 
+    a given magnitude (eg action) otherwise that lid is filtered out. 
+    Likewise for a BLE a huc must have output for all 3 versions 
+    (fim1, fim2, fim3) for a given magnitude (eg 100yr) otherwise it is 
+    filtered out.
+
+    Parameters
+    ----------
+    dataframe : Pandas DataFrame
+        Containing the input metrics originating from synthesize_test_cases
+    unique_field : STR
+        base resolution for each benchmark source: 'nws'/'usgs' (nws_lid)
+        ble (huc).
+
+    Returns
+    -------
+    final_filtered_dataframe : Pandas Dataframe
+        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude. 
+    unique_sites: DICT
+        The sites that were included in the dataframe for each magnitude.
+
+    '''
+    
+    #Get lists of sites for each magnitude/version
+    unique_sites = dataframe.groupby(['magnitude','version'])[unique_field].agg('unique')
+    #Get unique magnitudes
+    magnitudes = dataframe.magnitude.unique()
+    #Create new dataframe to hold metrics for the common sites as well as the actual lists of common sites.
+    final_filtered_dataframe = pd.DataFrame()
+    all_unique_sites = {}
+    #Cycle through each magnitude
+    for magnitude in magnitudes:
+        #Compile a list of sets containing unique lids pertaining to each threshold. List contains 3 unique sets [{fim1:unique lids},{fim2: unique lids},{fim3: unique lids}]
+        sites_per_magnitude=[set(a) for a in unique_sites[magnitude]]
+        #Intersect the sets to get the common lids per threshold then convert to list.
+        common_sites_per_magnitude = list(set.intersection(*sites_per_magnitude))
+        #Write common sites to dataframe
+        all_unique_sites[magnitude] = common_sites_per_magnitude
+        #Query filtered dataframe and only include data associated with the common sites for that magnitude
+        filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
+        #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
+        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)            
+    
+    return final_filtered_dataframe, all_unique_sites
+

From 71a57b76a806a20f059ad4a9e84a2d6ee329c167 Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Mon, 1 Mar 2021 14:19:09 -0600
Subject: [PATCH 046/359] Restructured the repository

Restructured the repository. This has no impact on hydrological code and simply moves files and renaming directories.

- Moved the contents of the lib folder to a new folder called src.
- Moved the contents of the tests folder to the tools folder.
- Changed any instance of lib or libDir to src or srcDir

This resolves #283.
---
 .gitignore                                    |  2 ++
 CHANGELOG.md                                  | 11 ++++++++
 Dockerfile.dev                                |  4 +--
 Dockerfile.prod                               |  4 +--
 README.md                                     |  6 ++--
 INSTALL.md => docs/INSTALL.md                 |  0
 fim_run.sh                                    | 14 +++++-----
 lib/time_and_tee_run_by_unit.sh               |  5 ----
 {lib => src}/__init__.py                      |  0
 {lib => src}/acquire_and_preprocess_inputs.py |  0
 {lib => src}/add_crosswalk.py                 |  0
 {lib => src}/adjust_headwater_streams.py      |  0
 {lib => src}/adjust_thalweg_lateral.py        |  0
 {lib => src}/aggregate_fim_outputs.py         |  0
 {lib => src}/aggregate_fim_outputs.sh         |  0
 {lib => src}/aggregate_vector_inputs.py       |  0
 {lib => src}/agreedem.py                      |  0
 {lib => src}/bash_functions.env               |  0
 {lib => src}/build_stream_traversal.py        |  0
 {lib => src}/check_huc_inputs.py              |  0
 {lib => src}/clip_vectors_to_wbd.py           |  0
 {lib => src}/derive_headwaters.py             |  0
 {lib => src}/entrypoint.sh                    |  0
 .../filter_catchments_and_add_attributes.py   |  0
 {lib => src}/fr_to_ms_raster_mask.py          |  0
 {lib => src}/getRasterInfoNative.py           |  0
 {lib => src}/get_all_huc_in_inputs.py         |  0
 {lib => src}/make_stages_and_catchlist.py     |  0
 {lib => src}/output_cleanup.py                |  0
 {lib => src}/r_grow_distance.py               |  0
 {lib => src}/raster.py                        |  0
 {lib => src}/reachID_grid_to_vector_points.py |  0
 {lib => src}/reduce_nhd_stream_density.py     |  0
 {lib => src}/rem.py                           |  0
 {lib => src}/run_by_unit.sh                   | 28 +++++++++----------
 {lib => src}/split_flows.py                   |  0
 src/time_and_tee_run_by_unit.sh               |  5 ++++
 {lib => src}/unique_pixel_and_allocation.py   |  0
 {lib => src}/utils/__init__.py                |  0
 {lib => src}/utils/archive_cleanup.py         |  0
 {lib => src}/utils/shared_functions.py        |  0
 {lib => src}/utils/shared_variables.py        |  0
 {tests => tools}/__init__.py                  |  0
 .../aggregate_mannings_calibration.py         |  0
 {tests => tools}/aggregate_metrics.py         |  0
 {tests => tools}/cache_metrics.py             |  0
 {tests => tools}/comparing_src.py             |  0
 {tests => tools}/inundation.py                |  0
 {tests => tools}/mannings_calibration_run.sh  |  0
 {tests => tools}/mannings_run_by_set.sh       |  2 +-
 {tests => tools}/plots/eval_plots.py          |  0
 {tests => tools}/plots/utils/__init__.py      |  0
 .../plots/utils/shared_functions.py           |  0
 .../preprocess/create_flow_forecast_file.py   |  2 ++
 .../preprocess/preprocess_benchmark.py        |  2 ++
 .../preprocess/preprocess_fimx.py             |  0
 {tests => tools}/run_test_case.py             |  0
 {tests => tools}/run_test_case_calibration.py |  0
 {tests => tools}/synthesize_test_cases.py     |  0
 .../time_and_tee_mannings_calibration.sh      |  0
 {tests => tools}/utils/__init__.py            |  0
 {tests => tools}/utils/shapefile_to_raster.py |  0
 {tests => tools}/utils/shared_functions.py    |  0
 {tests => tools}/utils/shared_variables.py    |  0
 64 files changed, 51 insertions(+), 34 deletions(-)
 rename INSTALL.md => docs/INSTALL.md (100%)
 delete mode 100755 lib/time_and_tee_run_by_unit.sh
 rename {lib => src}/__init__.py (100%)
 rename {lib => src}/acquire_and_preprocess_inputs.py (100%)
 rename {lib => src}/add_crosswalk.py (100%)
 rename {lib => src}/adjust_headwater_streams.py (100%)
 rename {lib => src}/adjust_thalweg_lateral.py (100%)
 rename {lib => src}/aggregate_fim_outputs.py (100%)
 rename {lib => src}/aggregate_fim_outputs.sh (100%)
 rename {lib => src}/aggregate_vector_inputs.py (100%)
 rename {lib => src}/agreedem.py (100%)
 rename {lib => src}/bash_functions.env (100%)
 rename {lib => src}/build_stream_traversal.py (100%)
 rename {lib => src}/check_huc_inputs.py (100%)
 rename {lib => src}/clip_vectors_to_wbd.py (100%)
 rename {lib => src}/derive_headwaters.py (100%)
 rename {lib => src}/entrypoint.sh (100%)
 rename {lib => src}/filter_catchments_and_add_attributes.py (100%)
 rename {lib => src}/fr_to_ms_raster_mask.py (100%)
 rename {lib => src}/getRasterInfoNative.py (100%)
 rename {lib => src}/get_all_huc_in_inputs.py (100%)
 rename {lib => src}/make_stages_and_catchlist.py (100%)
 rename {lib => src}/output_cleanup.py (100%)
 rename {lib => src}/r_grow_distance.py (100%)
 rename {lib => src}/raster.py (100%)
 rename {lib => src}/reachID_grid_to_vector_points.py (100%)
 rename {lib => src}/reduce_nhd_stream_density.py (100%)
 rename {lib => src}/rem.py (100%)
 rename {lib => src}/run_by_unit.sh (95%)
 rename {lib => src}/split_flows.py (100%)
 create mode 100755 src/time_and_tee_run_by_unit.sh
 rename {lib => src}/unique_pixel_and_allocation.py (100%)
 rename {lib => src}/utils/__init__.py (100%)
 rename {lib => src}/utils/archive_cleanup.py (100%)
 rename {lib => src}/utils/shared_functions.py (100%)
 rename {lib => src}/utils/shared_variables.py (100%)
 rename {tests => tools}/__init__.py (100%)
 rename {tests => tools}/aggregate_mannings_calibration.py (100%)
 rename {tests => tools}/aggregate_metrics.py (100%)
 rename {tests => tools}/cache_metrics.py (100%)
 rename {tests => tools}/comparing_src.py (100%)
 rename {tests => tools}/inundation.py (100%)
 rename {tests => tools}/mannings_calibration_run.sh (100%)
 rename {tests => tools}/mannings_run_by_set.sh (93%)
 rename {tests => tools}/plots/eval_plots.py (100%)
 rename {tests => tools}/plots/utils/__init__.py (100%)
 rename {tests => tools}/plots/utils/shared_functions.py (100%)
 rename {tests => tools}/preprocess/create_flow_forecast_file.py (99%)
 rename {tests => tools}/preprocess/preprocess_benchmark.py (99%)
 rename {tests => tools}/preprocess/preprocess_fimx.py (100%)
 rename {tests => tools}/run_test_case.py (100%)
 rename {tests => tools}/run_test_case_calibration.py (100%)
 rename {tests => tools}/synthesize_test_cases.py (100%)
 rename {tests => tools}/time_and_tee_mannings_calibration.sh (100%)
 rename {tests => tools}/utils/__init__.py (100%)
 rename {tests => tools}/utils/shapefile_to_raster.py (100%)
 rename {tests => tools}/utils/shared_functions.py (100%)
 rename {tests => tools}/utils/shared_variables.py (100%)

diff --git a/.gitignore b/.gitignore
index ec0085323..54bcfb6a6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ config/**
 !config/*calibrated*
 !config/symbology/
 .vscode/
+**/.DS_Store
+**/*_pytest.py
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e3d7b31f..64bc07129 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
+
+## v3.0.7.0 - 2021-03-01 - [PR #288](https://github.com/NOAA-OWP/cahaba/pull/288)
+
+Restructured the repository. This has no impact on hydrological work done in the codebase and is simply moving files and renaming directories.
+
+### Changes
+ - Moved the contents of the `lib` folder to a new folder called `src`.
+ - Moved the contents of the `tests` folder to the `tools` folder.
+ - Changed any instance of `lib` or `libDir` to `src` or `srcDir`.
+
+<br/><br/>
 ## v3.0.6.0 - 2021-02-25 - [PR #276](https://github.com/NOAA-OWP/cahaba/pull/276)
 
 Enhancement that creates metric plots and summary statistics using metrics compiled by `synthesize_test_cases.py`. 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index 626ef94ef..dea61c348 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -54,7 +54,7 @@ ARG projectDir=/foss_fim
 ARG depDir=/dependencies
 ENV inputDataDir=$dataDir/inputs
 ENV outputDataDir=$dataDir/outputs 
-ENV libDir=$projectDir/lib
+ENV srcDir=$projectDir/src
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
@@ -92,5 +92,5 @@ COPY Pipfile.lock .
 RUN pip3 install pipenv && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
-ADD ./lib/entrypoint.sh /
+ADD ./src/entrypoint.sh /
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
diff --git a/Dockerfile.prod b/Dockerfile.prod
index 79c50bdaf..23d0980ec 100644
--- a/Dockerfile.prod
+++ b/Dockerfile.prod
@@ -54,7 +54,7 @@ ARG projectDir=/foss_fim
 ARG depDir=/dependencies
 ENV inputDataDir=$dataDir/inputs
 ENV outputDataDir=$dataDir/outputs 
-ENV libDir=$projectDir/lib
+ENV srcDir=$projectDir/src
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
@@ -95,6 +95,6 @@ COPY . $projectDir/
 USER root:$GroupName
 
 # RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
-ADD ./lib/entrypoint.sh /
+ADD ./src/entrypoint.sh /
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 
diff --git a/README.md b/README.md
index cb091529a..311c29cde 100644
--- a/README.md
+++ b/README.md
@@ -62,15 +62,15 @@ docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <i
 
 ### Acquire and Prepare Data
 ```
-/foss_fim/lib/acquire_and_preprocess_inputs.py -u <huc4s_to_process>
+/foss_fim/src/acquire_and_preprocess_inputs.py -u <huc4s_to_process>
 ```
 - `-u` can be a single HUC4, series of HUC4s (e.g. 1209 1210), path to line-delimited file with HUC4s.
-- Please run `/foss_fim/lib/acquire_and_preprocess_inputs.py --help` for more information.
+- Please run `/foss_fim/src/acquire_and_preprocess_inputs.py --help` for more information.
 - See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
 
 ### Aggregate NHDHR Streams and Create NWM Headwater Points 
 ```
-/foss_fim/lib/aggregate_vector_inputs.py
+/foss_fim/src/aggregate_vector_inputs.py
 ```
 ### Produce Hydrofabric 
 ```
diff --git a/INSTALL.md b/docs/INSTALL.md
similarity index 100%
rename from INSTALL.md
rename to docs/INSTALL.md
diff --git a/fim_run.sh b/fim_run.sh
index 8c6862dae..5acdeff71 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -94,7 +94,7 @@ fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
-source $libDir/bash_functions.env
+source $srcDir/bash_functions.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
@@ -123,7 +123,7 @@ export input_nhd_headwaters_fr=$inputDataDir/nhdplus_vectors_aggregate/nhd_headw
 export input_nhd_headwaters_ms=$inputDataDir/nhdplus_vectors_aggregate/nhd_headwaters_adjusted_ms.gpkg
 
 ## Input handling ##
-$libDir/check_huc_inputs.py -u "$hucList"
+$srcDir/check_huc_inputs.py -u "$hucList"
 
 ## Make output and data directories ##
 if [ -d "$outputRunDataDir" ] && [  "$overwrite" -eq 1 ]; then
@@ -137,20 +137,20 @@ mkdir -p $outputRunDataDir/logs
 ## RUN ##
 if [ -f "$hucList" ]; then
     if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $libDir/time_and_tee_run_by_unit.sh :::: $hucList
+        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh :::: $hucList
     else
-        parallel --eta -j $jobLimit --joblog $logFile -- $libDir/time_and_tee_run_by_unit.sh :::: $hucList
+        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh :::: $hucList
     fi
 else
     if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb -j $jobLimit --joblog $logFile -- $libDir/time_and_tee_run_by_unit.sh ::: $hucList
+        parallel --verbose --lb -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh ::: $hucList
     else
-        parallel --eta -j $jobLimit --joblog $logFile -- $libDir/time_and_tee_run_by_unit.sh ::: $hucList
+        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh ::: $hucList
     fi
 fi
 
 echo "$viz"
 if [[ "$viz" -eq 1 ]]; then
     # aggregate outputs
-    python3 /foss_fim/lib/aggregate_fim_outputs.py -d $outputRunDataDir
+    python3 /foss_fim/src/aggregate_fim_outputs.py -d $outputRunDataDir
 fi
diff --git a/lib/time_and_tee_run_by_unit.sh b/lib/time_and_tee_run_by_unit.sh
deleted file mode 100755
index c1eca97fc..000000000
--- a/lib/time_and_tee_run_by_unit.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash -e
-
-/usr/bin/time -v $libDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log
-exit ${PIPESTATUS[0]}
-
diff --git a/lib/__init__.py b/src/__init__.py
similarity index 100%
rename from lib/__init__.py
rename to src/__init__.py
diff --git a/lib/acquire_and_preprocess_inputs.py b/src/acquire_and_preprocess_inputs.py
similarity index 100%
rename from lib/acquire_and_preprocess_inputs.py
rename to src/acquire_and_preprocess_inputs.py
diff --git a/lib/add_crosswalk.py b/src/add_crosswalk.py
similarity index 100%
rename from lib/add_crosswalk.py
rename to src/add_crosswalk.py
diff --git a/lib/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
similarity index 100%
rename from lib/adjust_headwater_streams.py
rename to src/adjust_headwater_streams.py
diff --git a/lib/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
similarity index 100%
rename from lib/adjust_thalweg_lateral.py
rename to src/adjust_thalweg_lateral.py
diff --git a/lib/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py
similarity index 100%
rename from lib/aggregate_fim_outputs.py
rename to src/aggregate_fim_outputs.py
diff --git a/lib/aggregate_fim_outputs.sh b/src/aggregate_fim_outputs.sh
similarity index 100%
rename from lib/aggregate_fim_outputs.sh
rename to src/aggregate_fim_outputs.sh
diff --git a/lib/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
similarity index 100%
rename from lib/aggregate_vector_inputs.py
rename to src/aggregate_vector_inputs.py
diff --git a/lib/agreedem.py b/src/agreedem.py
similarity index 100%
rename from lib/agreedem.py
rename to src/agreedem.py
diff --git a/lib/bash_functions.env b/src/bash_functions.env
similarity index 100%
rename from lib/bash_functions.env
rename to src/bash_functions.env
diff --git a/lib/build_stream_traversal.py b/src/build_stream_traversal.py
similarity index 100%
rename from lib/build_stream_traversal.py
rename to src/build_stream_traversal.py
diff --git a/lib/check_huc_inputs.py b/src/check_huc_inputs.py
similarity index 100%
rename from lib/check_huc_inputs.py
rename to src/check_huc_inputs.py
diff --git a/lib/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
similarity index 100%
rename from lib/clip_vectors_to_wbd.py
rename to src/clip_vectors_to_wbd.py
diff --git a/lib/derive_headwaters.py b/src/derive_headwaters.py
similarity index 100%
rename from lib/derive_headwaters.py
rename to src/derive_headwaters.py
diff --git a/lib/entrypoint.sh b/src/entrypoint.sh
similarity index 100%
rename from lib/entrypoint.sh
rename to src/entrypoint.sh
diff --git a/lib/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
similarity index 100%
rename from lib/filter_catchments_and_add_attributes.py
rename to src/filter_catchments_and_add_attributes.py
diff --git a/lib/fr_to_ms_raster_mask.py b/src/fr_to_ms_raster_mask.py
similarity index 100%
rename from lib/fr_to_ms_raster_mask.py
rename to src/fr_to_ms_raster_mask.py
diff --git a/lib/getRasterInfoNative.py b/src/getRasterInfoNative.py
similarity index 100%
rename from lib/getRasterInfoNative.py
rename to src/getRasterInfoNative.py
diff --git a/lib/get_all_huc_in_inputs.py b/src/get_all_huc_in_inputs.py
similarity index 100%
rename from lib/get_all_huc_in_inputs.py
rename to src/get_all_huc_in_inputs.py
diff --git a/lib/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
similarity index 100%
rename from lib/make_stages_and_catchlist.py
rename to src/make_stages_and_catchlist.py
diff --git a/lib/output_cleanup.py b/src/output_cleanup.py
similarity index 100%
rename from lib/output_cleanup.py
rename to src/output_cleanup.py
diff --git a/lib/r_grow_distance.py b/src/r_grow_distance.py
similarity index 100%
rename from lib/r_grow_distance.py
rename to src/r_grow_distance.py
diff --git a/lib/raster.py b/src/raster.py
similarity index 100%
rename from lib/raster.py
rename to src/raster.py
diff --git a/lib/reachID_grid_to_vector_points.py b/src/reachID_grid_to_vector_points.py
similarity index 100%
rename from lib/reachID_grid_to_vector_points.py
rename to src/reachID_grid_to_vector_points.py
diff --git a/lib/reduce_nhd_stream_density.py b/src/reduce_nhd_stream_density.py
similarity index 100%
rename from lib/reduce_nhd_stream_density.py
rename to src/reduce_nhd_stream_density.py
diff --git a/lib/rem.py b/src/rem.py
similarity index 100%
rename from lib/rem.py
rename to src/rem.py
diff --git a/lib/run_by_unit.sh b/src/run_by_unit.sh
similarity index 95%
rename from lib/run_by_unit.sh
rename to src/run_by_unit.sh
index 2ba820a32..68866729d 100755
--- a/lib/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -77,7 +77,7 @@ echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-$libDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_NWM_Flows -s $input_nhd_flowlines -l $input_NWM_Lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_NWM_Catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg
+$srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_NWM_Flows -s $input_nhd_flowlines -l $input_NWM_Lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_NWM_Catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg
 Tcount
 
 if [ "$extent" = "MS" ]; then
@@ -107,7 +107,7 @@ Tcount
 echo -e $startDiv"Get DEM Metadata $hucNumber"$stopDiv
 date -u
 Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($libDir/getRasterInfoNative.py $outputHucDataDir/dem.tif)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/dem.tif)
 
 ## RASTERIZE NLD MULTILINES ##
 echo -e $startDiv"Rasterize all NLD multilines using zelev vertices"$stopDiv
@@ -167,7 +167,7 @@ echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer"$stopD
 date -u
 Tstart
 [ ! -f $outputHucDataDir/dem_burned.tif ] && \
-$libDir/agreedem.py -r $outputHucDataDir/flows_grid_boolean.tif -d $outputHucDataDir/dem_meters.tif -w $outputHucDataDir -g $outputHucDataDir/temp_work -o $outputHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+$srcDir/agreedem.py -r $outputHucDataDir/flows_grid_boolean.tif -d $outputHucDataDir/dem_meters.tif -w $outputHucDataDir -g $outputHucDataDir/temp_work -o $outputHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
 Tcount
 
 ## PIT REMOVE BURNED DEM ##
@@ -212,14 +212,14 @@ Tcount
 echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber"$stopDiv
 date -u
 Tstart
-$libDir/unique_pixel_and_allocation.py -s $outputHucDataDir/demDerived_streamPixels.tif -o $outputHucDataDir/demDerived_streamPixels_ids.tif -g $outputHucDataDir/temp_grass
+$srcDir/unique_pixel_and_allocation.py -s $outputHucDataDir/demDerived_streamPixels.tif -o $outputHucDataDir/demDerived_streamPixels_ids.tif -g $outputHucDataDir/temp_grass
 Tcount
 
 ## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
 echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber"$stopDiv
 date -u
 Tstart
-$libDir/adjust_thalweg_lateral.py -e $outputHucDataDir/dem_meters.tif -s $outputHucDataDir/demDerived_streamPixels.tif -a $outputHucDataDir/demDerived_streamPixels_ids_allo.tif -d $outputHucDataDir/demDerived_streamPixels_ids_dist.tif -t 50 -o $outputHucDataDir/dem_lateral_thalweg_adj.tif
+$srcDir/adjust_thalweg_lateral.py -e $outputHucDataDir/dem_meters.tif -s $outputHucDataDir/demDerived_streamPixels.tif -a $outputHucDataDir/demDerived_streamPixels_ids_allo.tif -d $outputHucDataDir/demDerived_streamPixels_ids_dist.tif -t 50 -o $outputHucDataDir/dem_lateral_thalweg_adj.tif
 Tcount
 
 ## MASK BURNED DEM FOR STREAMS ONLY ###
@@ -258,7 +258,7 @@ echo -e $startDiv"Split Derived Reaches $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ] && \
-$libDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+$srcDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 Tcount
 
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
@@ -272,7 +272,7 @@ if [ "$extent" = "MS" ]; then
   echo -e $startDiv"Mask Rasters with Stream Buffer $hucNumber"$stopDiv
   date -u
   Tstart
-  $libDir/fr_to_ms_raster_mask.py $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/flowdir_d8_burned_filled.tif $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/slopes_d8_dem_meters.tif $outputHucDataDir/flowdir_d8_MS.tif $outputHucDataDir/dem_thalwegCond_MS.tif $outputHucDataDir/slopes_d8_dem_metersMS.tif $outputHucDataDir/demDerived_streamPixels.tif $outputHucDataDir/demDerived_streamPixelsMS.tif
+  $srcDir/fr_to_ms_raster_mask.py $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/flowdir_d8_burned_filled.tif $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/slopes_d8_dem_meters.tif $outputHucDataDir/flowdir_d8_MS.tif $outputHucDataDir/dem_thalwegCond_MS.tif $outputHucDataDir/slopes_d8_dem_metersMS.tif $outputHucDataDir/demDerived_streamPixels.tif $outputHucDataDir/demDerived_streamPixelsMS.tif
   Tcount
 
   if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
@@ -305,7 +305,7 @@ echo -e $startDiv"Vectorize Pixel Centroids $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/flows_points_pixels.gpkg ] && \
-$libDir/reachID_grid_to_vector_points.py $demDerived_streamPixels $outputHucDataDir/flows_points_pixels.gpkg featureID
+$srcDir/reachID_grid_to_vector_points.py $demDerived_streamPixels $outputHucDataDir/flows_points_pixels.gpkg featureID
 Tcount
 
 ## GAGE WATERSHED FOR PIXELS ##
@@ -321,7 +321,7 @@ echo -e $startDiv"D8 REM $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/rem.tif ] && \
-$libDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -r $outputHucDataDir/hand_ref_elev_table.csv -s $outputHucDataDir/demDerived_reaches_split.gpkg
+$srcDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -r $outputHucDataDir/hand_ref_elev_table.csv -s $outputHucDataDir/demDerived_reaches_split.gpkg
 Tcount
 
 ## DINF DISTANCE DOWN ##
@@ -353,7 +353,7 @@ echo -e $startDiv"Process catchments and model streams step 1 $hucNumber"$stopDi
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ] && \
-$libDir/filter_catchments_and_add_attributes.py $outputHucDataDir/gw_catchments_reaches.gpkg $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/wbd8_clp.gpkg $hucNumber
+$srcDir/filter_catchments_and_add_attributes.py $outputHucDataDir/gw_catchments_reaches.gpkg $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/wbd8_clp.gpkg $hucNumber
 
 if [[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ]] ; then
   echo "No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
@@ -366,7 +366,7 @@ Tcount
 echo -e $startDiv"Get Clipped Raster Metadata $hucNumber"$stopDiv
 date -u
 Tstart
-read fsize ncols nrows ndv_clipped xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($libDir/getRasterInfoNative.py $outputHucDataDir/gw_catchments_reaches.tif)
+read fsize ncols nrows ndv_clipped xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/gw_catchments_reaches.tif)
 Tcount
 
 ## RASTERIZE NEW CATCHMENTS AGAIN ##
@@ -413,7 +413,7 @@ Tcount
 echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber"$stopDiv
 date -u
 Tstart
-$libDir/make_stages_and_catchlist.py $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/stage.txt $outputHucDataDir/catchment_list.txt $stage_min_meters $stage_interval_meters $stage_max_meters
+$srcDir/make_stages_and_catchlist.py $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/stage.txt $outputHucDataDir/catchment_list.txt $stage_min_meters $stage_interval_meters $stage_max_meters
 Tcount
 
 ## HYDRAULIC PROPERTIES ##
@@ -429,7 +429,7 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-$libDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent -k $outputHucDataDir/small_segments.csv
+$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
 ## CLEANUP OUTPUTS ##
@@ -440,5 +440,5 @@ args=()
 (( viz == 1 )) && args+=( '-v' )
 date -u
 Tstart
-$libDir/output_cleanup.py $hucNumber $outputHucDataDir "${args[@]}"
+$srcDir/output_cleanup.py $hucNumber $outputHucDataDir "${args[@]}"
 Tcount
diff --git a/lib/split_flows.py b/src/split_flows.py
similarity index 100%
rename from lib/split_flows.py
rename to src/split_flows.py
diff --git a/src/time_and_tee_run_by_unit.sh b/src/time_and_tee_run_by_unit.sh
new file mode 100755
index 000000000..25dffd967
--- /dev/null
+++ b/src/time_and_tee_run_by_unit.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -e
+
+/usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log
+exit ${PIPESTATUS[0]}
+
diff --git a/lib/unique_pixel_and_allocation.py b/src/unique_pixel_and_allocation.py
similarity index 100%
rename from lib/unique_pixel_and_allocation.py
rename to src/unique_pixel_and_allocation.py
diff --git a/lib/utils/__init__.py b/src/utils/__init__.py
similarity index 100%
rename from lib/utils/__init__.py
rename to src/utils/__init__.py
diff --git a/lib/utils/archive_cleanup.py b/src/utils/archive_cleanup.py
similarity index 100%
rename from lib/utils/archive_cleanup.py
rename to src/utils/archive_cleanup.py
diff --git a/lib/utils/shared_functions.py b/src/utils/shared_functions.py
similarity index 100%
rename from lib/utils/shared_functions.py
rename to src/utils/shared_functions.py
diff --git a/lib/utils/shared_variables.py b/src/utils/shared_variables.py
similarity index 100%
rename from lib/utils/shared_variables.py
rename to src/utils/shared_variables.py
diff --git a/tests/__init__.py b/tools/__init__.py
similarity index 100%
rename from tests/__init__.py
rename to tools/__init__.py
diff --git a/tests/aggregate_mannings_calibration.py b/tools/aggregate_mannings_calibration.py
similarity index 100%
rename from tests/aggregate_mannings_calibration.py
rename to tools/aggregate_mannings_calibration.py
diff --git a/tests/aggregate_metrics.py b/tools/aggregate_metrics.py
similarity index 100%
rename from tests/aggregate_metrics.py
rename to tools/aggregate_metrics.py
diff --git a/tests/cache_metrics.py b/tools/cache_metrics.py
similarity index 100%
rename from tests/cache_metrics.py
rename to tools/cache_metrics.py
diff --git a/tests/comparing_src.py b/tools/comparing_src.py
similarity index 100%
rename from tests/comparing_src.py
rename to tools/comparing_src.py
diff --git a/tests/inundation.py b/tools/inundation.py
similarity index 100%
rename from tests/inundation.py
rename to tools/inundation.py
diff --git a/tests/mannings_calibration_run.sh b/tools/mannings_calibration_run.sh
similarity index 100%
rename from tests/mannings_calibration_run.sh
rename to tools/mannings_calibration_run.sh
diff --git a/tests/mannings_run_by_set.sh b/tools/mannings_run_by_set.sh
similarity index 93%
rename from tests/mannings_run_by_set.sh
rename to tools/mannings_run_by_set.sh
index 9ee875e2a..917672a63 100755
--- a/tests/mannings_run_by_set.sh
+++ b/tools/mannings_run_by_set.sh
@@ -10,6 +10,6 @@ mannings_value="${array[$mannings_row]}"
 subdir=$outdir/$huc"_"$strorder"_"$mannings_value
 mkdir -p $subdir
 
-$libDir/add_crosswalk.py -d $hucdir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $hucdir/demDerived_reaches_split_filtered.gpkg -s $hucdir/src_base.csv -l $subdir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $subdir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $subdir/src_full_crosswalked.csv -j $subdir/src.json -x $subdir/crosswalk_table.csv -t $subdir/hydroTable.csv -w $hucdir/wbd8_clp.gpkg -b $hucdir/nwm_subset_streams.gpkg -y $hucdir/nwm_catchments_proj_subset.tif -m $param_set -z $input_NWM_Catchments -p FR -c
+$srcDir/add_crosswalk.py -d $hucdir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $hucdir/demDerived_reaches_split_filtered.gpkg -s $hucdir/src_base.csv -l $subdir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $subdir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $subdir/src_full_crosswalked.csv -j $subdir/src.json -x $subdir/crosswalk_table.csv -t $subdir/hydroTable.csv -w $hucdir/wbd8_clp.gpkg -b $hucdir/nwm_subset_streams.gpkg -y $hucdir/nwm_catchments_proj_subset.tif -m $param_set -z $input_NWM_Catchments -p FR -c
 
 python3 foss_fim/tests/run_test_case_calibration.py -r $fimdir/$huc -d $subdir -t $huc"_ble" -b "mannings_calibration"/$strorder/$mannings_value
diff --git a/tests/plots/eval_plots.py b/tools/plots/eval_plots.py
similarity index 100%
rename from tests/plots/eval_plots.py
rename to tools/plots/eval_plots.py
diff --git a/tests/plots/utils/__init__.py b/tools/plots/utils/__init__.py
similarity index 100%
rename from tests/plots/utils/__init__.py
rename to tools/plots/utils/__init__.py
diff --git a/tests/plots/utils/shared_functions.py b/tools/plots/utils/shared_functions.py
similarity index 100%
rename from tests/plots/utils/shared_functions.py
rename to tools/plots/utils/shared_functions.py
diff --git a/tests/preprocess/create_flow_forecast_file.py b/tools/preprocess/create_flow_forecast_file.py
similarity index 99%
rename from tests/preprocess/create_flow_forecast_file.py
rename to tools/preprocess/create_flow_forecast_file.py
index e7df52e15..9de7abfe5 100644
--- a/tests/preprocess/create_flow_forecast_file.py
+++ b/tools/preprocess/create_flow_forecast_file.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # -*- coding: utf-8 -*-
 """
 Created on Wed Jul 29 11:48:37 2020
diff --git a/tests/preprocess/preprocess_benchmark.py b/tools/preprocess/preprocess_benchmark.py
similarity index 99%
rename from tests/preprocess/preprocess_benchmark.py
rename to tools/preprocess/preprocess_benchmark.py
index 4d6d860b0..02f8e5ea8 100644
--- a/tests/preprocess/preprocess_benchmark.py
+++ b/tools/preprocess/preprocess_benchmark.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # -*- coding: utf-8 -*-
 """
 Created on Thu Jul 23 15:17:04 2020
diff --git a/tests/preprocess/preprocess_fimx.py b/tools/preprocess/preprocess_fimx.py
similarity index 100%
rename from tests/preprocess/preprocess_fimx.py
rename to tools/preprocess/preprocess_fimx.py
diff --git a/tests/run_test_case.py b/tools/run_test_case.py
similarity index 100%
rename from tests/run_test_case.py
rename to tools/run_test_case.py
diff --git a/tests/run_test_case_calibration.py b/tools/run_test_case_calibration.py
similarity index 100%
rename from tests/run_test_case_calibration.py
rename to tools/run_test_case_calibration.py
diff --git a/tests/synthesize_test_cases.py b/tools/synthesize_test_cases.py
similarity index 100%
rename from tests/synthesize_test_cases.py
rename to tools/synthesize_test_cases.py
diff --git a/tests/time_and_tee_mannings_calibration.sh b/tools/time_and_tee_mannings_calibration.sh
similarity index 100%
rename from tests/time_and_tee_mannings_calibration.sh
rename to tools/time_and_tee_mannings_calibration.sh
diff --git a/tests/utils/__init__.py b/tools/utils/__init__.py
similarity index 100%
rename from tests/utils/__init__.py
rename to tools/utils/__init__.py
diff --git a/tests/utils/shapefile_to_raster.py b/tools/utils/shapefile_to_raster.py
similarity index 100%
rename from tests/utils/shapefile_to_raster.py
rename to tools/utils/shapefile_to_raster.py
diff --git a/tests/utils/shared_functions.py b/tools/utils/shared_functions.py
similarity index 100%
rename from tests/utils/shared_functions.py
rename to tools/utils/shared_functions.py
diff --git a/tests/utils/shared_variables.py b/tools/utils/shared_variables.py
similarity index 100%
rename from tests/utils/shared_variables.py
rename to tools/utils/shared_variables.py

From 7f65a0ab6f040775674f1fecc3f1948d243a8d48 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 2 Mar 2021 10:58:38 -0600
Subject: [PATCH 047/359] Renaming of benchmark data and associated variables
 in evaluation scripts

Renamed benchmark layers in test_cases and updated variable names in evaluation scripts.

- Updated run_test_case.py with new benchmark layer names.
- Updated run_test_case_calibration.py with new benchmark layer names.

This resolves #284.
---
 CHANGELOG.md                       |  9 +++++++++
 tools/run_test_case.py             | 21 +++++++++------------
 tools/run_test_case_calibration.py |  2 +-
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 64bc07129..3f26caaa9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.7.1 - 2021-03-02 - [PR #290](https://github.com/NOAA-OWP/cahaba/pull/290)
+
+Renamed benchmark layers in `test_cases` and updated variable names in evaluation scripts.
+
+### Changes
+ - Updated `run_test_case.py` with new benchmark layer names.
+ - Updated `run_test_case_calibration.py` with new benchmark layer names.
+ 
+<br/><br/>
 ## v3.0.7.0 - 2021-03-01 - [PR #288](https://github.com/NOAA-OWP/cahaba/pull/288)
 
 Restructured the repository. This has no impact on hydrological work done in the codebase and is simply moving files and renaming directories.
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 5beac4f16..2a0a279c5 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -84,7 +84,6 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
 
     # Get path to validation_data_{benchmark} directory and huc_dir.
     validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
-    
     for magnitude in magnitude_list:
         version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
         if not os.path.exists(version_test_case_dir):
@@ -94,18 +93,18 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
         if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
             benchmark_raster_path_list, forecast_list = [], []
             lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
-            lid_list, inundation_raster_list, extent_file_list = [], [], []
+            lid_list, inundation_raster_list, domain_file_list = [], [], []
             
             for lid in lid_dir_list:
                 lid_dir = os.path.join(validation_data_path, current_huc, lid)
-                benchmark_raster_path_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_depth_' + magnitude + '.tif'))  # TEMP
+                benchmark_raster_path_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif'))  # TEMP
                 forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
                 lid_list.append(lid)
                 inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
-                extent_file_list.append(os.path.join(lid_dir, lid + '_extent.shp'))
+                domain_file_list.append(os.path.join(lid_dir, lid + '_domain.shp'))
 
         else:
-            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_depth_' + magnitude + '.tif')
+            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
             benchmark_raster_path_list = [benchmark_raster_file]
             forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
             forecast_list = [forecast_path]
@@ -115,23 +114,21 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
             benchmark_raster_path = benchmark_raster_path_list[index]
             forecast = forecast_list[index]
             inundation_raster = inundation_raster_list[index]
-            
             # Only need to define ahps_lid and ahps_extent_file for AHPS_BENCHMARK_CATEGORIES.
             if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
                 ahps_lid = lid_list[index]
-                ahps_extent_file = extent_file_list[index]
+                ahps_domain_file = domain_file_list[index]
                 mask_dict.update({ahps_lid:
-                    {'path': ahps_extent_file,
+                    {'path': ahps_domain_file,
                      'buffer': None,
                      'operation': 'include'}
                         })
         
-                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_extent_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
+                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_domain_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
                     continue
             else:  # If not in AHPS_BENCHMARK_CATEGORIES.
                 if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
                     continue
-    
             # Run inundate.
             print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
             try:
@@ -194,7 +191,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     parser.add_argument('-i', '--inclusion-area', help='Path to shapefile. Contingency metrics will be produced from pixels inside of shapefile extent.', required=False, default="")
     parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
     parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
-    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False)
+    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False, action='store_true')
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
@@ -234,7 +231,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     if args['magnitude'] == '':
         if 'ble' in args['test_id'].split('_'):
             args['magnitude'] = ['100yr', '500yr']
-        elif 'ahps' in args['test_id'].split('_'):
+        elif 'nws' or 'usgs' in args['test_id'].split('_'):
             args['magnitude'] = ['action', 'minor', 'moderate', 'major']
         else:
             print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
diff --git a/tools/run_test_case_calibration.py b/tools/run_test_case_calibration.py
index cefe80594..f630360b0 100755
--- a/tools/run_test_case_calibration.py
+++ b/tools/run_test_case_calibration.py
@@ -232,7 +232,7 @@ def run_alpha_test(fim_run_dir, calib_dir, branch_name, test_id, mask_type, retu
     for return_interval in return_interval_list:
         # Construct path to validation raster and forecast file.
         benchmark_category = test_id.split('_')[1]
-        benchmark_raster_path = os.path.join(TEST_CASES_DIR, 'validation_data_' + benchmark_category, current_huc, return_interval, benchmark_category + '_huc_' + current_huc + '_depth_' + return_interval + '.tif')
+        benchmark_raster_path = os.path.join(TEST_CASES_DIR, 'validation_data_' + benchmark_category, current_huc, return_interval, benchmark_category + '_huc_' + current_huc + '_extent_' + return_interval + '.tif')
         if not os.path.exists(benchmark_raster_path):  # Skip loop instance if the benchmark raster doesn't exist.
             continue
 

From f909134fe570eb1a6ab361013cc1c8dbf675eac4 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 3 Mar 2021 13:28:24 -0500
Subject: [PATCH 048/359] Update README.md

Updates the move from tests/ to tools/synthesize_test_cases.py
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 311c29cde..9f44d94ad 100644
--- a/README.md
+++ b/README.md
@@ -88,12 +88,12 @@ After `fim_run.sh` completes, you are ready to evaluate the model's skill.
 
 To evaluate model skill, run the following:
 ```
-python /foss_fim/tests/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs]
+python /foss_fim/tools/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs]
 ```
 
 More information can be found by running:
 ```
-python /foss_fim/tests/synthesize_test_cases.py --help
+python /foss_fim/tools/synthesize_test_cases.py --help
 ```
 
 ----

From 30f401a8066291aec65cdb9629d7551a059995c3 Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Tue, 9 Mar 2021 10:30:11 -0600
Subject: [PATCH 049/359] Convert CatFIM pipeline to open source

Refactored CatFIM pipeline to open source.

- Added VIZ_PROJECTION to shared_variables.py.
- Added missing library referenced in inundation.py.
- Cleaned up and converted evaluation scripts in generate_categorical_fim.py to open source.
- Removed util folders under tools directory.

This resolves #279.
---
 CHANGELOG.md                                  |  11 +
 src/utils/shared_variables.py                 |   2 +-
 tools/__init__.py                             |   0
 tools/aggregate_mannings_calibration.py       |   1 -
 tools/aggregate_metrics.py                    |  64 ++--
 tools/cache_metrics.py                        |   0
 tools/comparing_src.py                        |   2 +
 tools/generate_categorical_fim.py             | 336 +++++++++++++-----
 tools/inundation.py                           |   1 +
 tools/inundation_wrapper_custom_flow.py       |   3 -
 tools/inundation_wrapper_nwm_flows.py         |   3 -
 tools/mannings_calibration_run.sh             |  18 +-
 tools/mannings_run_by_set.sh                  |   2 +-
 tools/plots/{utils => }/__init__.py           |   0
 tools/plots/eval_plots.py                     | 288 +++++++--------
 .../shared_functions.py => plot_functions.py} |   0
 tools/preprocess/create_flow_forecast_file.py |  67 ++--
 tools/preprocess/preprocess_benchmark.py      |  51 ++-
 tools/preprocess/preprocess_fimx.py           |  74 ++--
 tools/run_test_case.py                        |  48 +--
 tools/run_test_case_calibration.py            |   2 +-
 tools/{utils => }/shapefile_to_raster.py      |  11 +-
 tools/synthesize_test_cases.py                |  83 +++--
 tools/time_and_tee_mannings_calibration.sh    |   2 +-
 ...functions.py => tools_shared_functions.py} |   0
 ...variables.py => tools_shared_variables.py} |   0
 tools/utils/__init__.py                       |   0
 27 files changed, 606 insertions(+), 463 deletions(-)
 mode change 100644 => 100755 tools/__init__.py
 mode change 100644 => 100755 tools/aggregate_metrics.py
 mode change 100644 => 100755 tools/cache_metrics.py
 mode change 100644 => 100755 tools/generate_categorical_fim.py
 mode change 100644 => 100755 tools/inundation_wrapper_custom_flow.py
 rename tools/plots/{utils => }/__init__.py (100%)
 mode change 100644 => 100755
 mode change 100644 => 100755 tools/plots/eval_plots.py
 rename tools/plots/{utils/shared_functions.py => plot_functions.py} (100%)
 mode change 100644 => 100755
 mode change 100644 => 100755 tools/preprocess/create_flow_forecast_file.py
 mode change 100644 => 100755 tools/preprocess/preprocess_benchmark.py
 mode change 100644 => 100755 tools/preprocess/preprocess_fimx.py
 rename tools/{utils => }/shapefile_to_raster.py (88%)
 mode change 100644 => 100755
 mode change 100644 => 100755 tools/synthesize_test_cases.py
 rename tools/{utils/shared_functions.py => tools_shared_functions.py} (100%)
 mode change 100644 => 100755
 rename tools/{utils/shared_variables.py => tools_shared_variables.py} (100%)
 mode change 100644 => 100755
 delete mode 100644 tools/utils/__init__.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f26caaa9..da0e1cb32 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.8.0 - 2021-03-09 - [PR #279](https://github.com/NOAA-OWP/cahaba/pull/279)
+
+Refactored NWS Flood Categorical HAND FIM (CatFIM) pipeline to open source.
+
+### Changes
+ - Added `VIZ_PROJECTION` to `shared_variables.py`.
+ - Added missing library referenced in `inundation.py`.
+ - Cleaned up and converted evaluation scripts in `generate_categorical_fim.py` to open source.
+ - Removed `util` folders under `tools` directory.
+
+<br/><br/>
 ## v3.0.7.1 - 2021-03-02 - [PR #290](https://github.com/NOAA-OWP/cahaba/pull/290)
 
 Renamed benchmark layers in `test_cases` and updated variable names in evaluation scripts.
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 40a8feacb..244a12d2b 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -3,7 +3,7 @@
 # Projections.
 #PREP_PROJECTION = "+proj=aea +datum=NAD83 +x_0=0.0 +y_0=0.0 +lon_0=96dW +lat_0=23dN +lat_1=29d30'N +lat_2=45d30'N +towgs84=-0.9956000824677655,1.901299877314078,0.5215002840524426,0.02591500053005733,0.009425998542707753,0.01159900118427752,-0.00062000005129903 +no_defs +units=m"
 PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'
-
+VIZ_PROJECTION ='PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
 # -- Data URLs-- #
 NHD_URL_PARENT = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/'
 NWM_HYDROFABRIC_URL = r'http://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_channel_hydrofabric.tar.gz'  # Temporary
diff --git a/tools/__init__.py b/tools/__init__.py
old mode 100644
new mode 100755
diff --git a/tools/aggregate_mannings_calibration.py b/tools/aggregate_mannings_calibration.py
index f94b1d025..c57b17776 100755
--- a/tools/aggregate_mannings_calibration.py
+++ b/tools/aggregate_mannings_calibration.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-
 import os
 import pandas as pd
 import csv
diff --git a/tools/aggregate_metrics.py b/tools/aggregate_metrics.py
old mode 100644
new mode 100755
index d8a462d5b..7cc5951b5
--- a/tools/aggregate_metrics.py
+++ b/tools/aggregate_metrics.py
@@ -3,17 +3,17 @@
 import json
 import os
 import csv
-    
+
 import argparse
 
 TEST_CASES_DIR = r'/data/test_cases_new/'
 # TEMP = r'/data/temp'
 
 # Search through all previous_versions in test_cases
-from utils.shared_functions import compute_stats_from_contingency_table
+from tools_shared_functions import compute_stats_from_contingency_table
 
 def create_master_metrics_csv():
-            
+
     # Construct header
     metrics_to_write = ['true_negatives_count',
                         'false_negatives_count',
@@ -57,33 +57,33 @@ def create_master_metrics_csv():
                         'masked_perc',
                         'masked_area_km2'
                         ]
-    
+
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
     list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source']]
-    
+
     for benchmark_type in ['ble', 'ahps']:
-        
+
         if benchmark_type == 'ble':
-        
+
             test_cases = r'/data/test_cases'
             test_cases_list = os.listdir(test_cases)
             # AHPS test_ids
             versions_to_aggregate = ['fim_1_0_0', 'fim_2_3_3', 'fim_3_0_0_3_fr_c']
-                            
+
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
-                    
+
                     huc = test_case.split('_')[0]
                     previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
-                    
+
                     for magnitude in ['100yr', '500yr']:
                         for version in versions_to_aggregate:
                             version_dir = os.path.join(previous_versions, version)
                             magnitude_dir = os.path.join(version_dir, magnitude)
 
                             if os.path.exists(magnitude_dir):
-                                
+
                                 magnitude_dir_list = os.listdir(magnitude_dir)
                                 for f in magnitude_dir_list:
                                     if '.json' in f:
@@ -99,40 +99,40 @@ def create_master_metrics_csv():
                                             sub_list_to_append.append(full_json_path)
                                             sub_list_to_append.append(flow)
                                             sub_list_to_append.append(benchmark_source)
-                                            
+
                                             list_to_write.append(sub_list_to_append)
-                                                                                    
+
                 except ValueError:
                     pass
-                
+
         if benchmark_type == 'ahps':
-    
+
             test_cases = r'/data/test_cases_ahps_testing'
             test_cases_list = os.listdir(test_cases)
             # AHPS test_ids
-            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021', 
+            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021',
                                      'fim_2_x_ms_nws_1_21_2021', 'fim_2_x_ms_usgs_1_21_2021',
                                      'fim_3_0_0_3_ms_c_nws_1_21_2021', 'fim_3_0_0_3_ms_c_usgs_1_21_2021',
                                      'ms_xwalk_fill_missing_cal_nws', 'ms_xwalk_fill_missing_cal_usgs']
-            
+
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
-                    
+
                     huc = test_case.split('_')[0]
                     previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
-                    
+
                     for magnitude in ['action', 'minor', 'moderate', 'major']:
                         for version in versions_to_aggregate:
-                            
+
                             if 'nws' in version:
                                 benchmark_source = 'ahps_nws'
                             if 'usgs' in version:
                                 benchmark_source = 'ahps_usgs'
-                            
+
                             version_dir = os.path.join(previous_versions, version)
                             magnitude_dir = os.path.join(version_dir, magnitude)
-                            
+
                             if os.path.exists(magnitude_dir):
                                 magnitude_dir_list = os.listdir(magnitude_dir)
                                 for f in magnitude_dir_list:
@@ -147,7 +147,7 @@ def create_master_metrics_csv():
                                                 parent_dir = 'usgs_1_21_2021'
                                             if 'nws' in version:
                                                 parent_dir = 'nws_1_21_2021'
-                                                
+
                                             flow_file = os.path.join(test_cases, parent_dir, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
                                             if os.path.exists(flow_file):
                                                 with open(flow_file, newline='') as csv_file:
@@ -157,7 +157,7 @@ def create_master_metrics_csv():
                                                         flow = row[1]
                                                     if nws_lid == 'mcc01':
                                                         print(flow)
-                                            
+
                                             stats_dict = json.load(open(full_json_path))
                                             for metric in metrics_to_write:
                                                 sub_list_to_append.append(stats_dict[metric])
@@ -165,10 +165,10 @@ def create_master_metrics_csv():
                                             sub_list_to_append.append(flow)
                                             sub_list_to_append.append(benchmark_source)
                                             list_to_write.append(sub_list_to_append)
-                                        
+
                 except ValueError:
                     pass
-        
+
     with open(output_csv, 'w', newline='') as csvfile:
         csv_writer = csv.writer(csvfile)
         csv_writer.writerows(list_to_write)
@@ -201,7 +201,7 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
     for magnitude in ['100yr', '500yr', 'action', 'minor', 'moderate', 'major']:
         huc_path_list = [['huc', 'path']]
         true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = 0, 0, 0, 0, 0, 0
-        
+
         for test_case in test_cases_dir_list:
 
             if test_case not in ['other', 'validation_data_ble', 'validation_data_legacy', 'validation_data_ahps']:
@@ -227,11 +227,11 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
                     cell_area = json_dict['cell_area_m2']
 
                     huc_path_list.append([huc, stats_json_path])
-                
-                    
+
+
             if cell_area == 0:
                 continue
-            
+
             # Pass all sums to shared function to calculate metrics.
             stats_dict = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=cell_area, masked_count=masked_count)
 
@@ -239,7 +239,7 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
 
             for stat in stats_dict:
                 list_to_write.append([stat, stats_dict[stat]])
-                
+
             # Map path to output directory for aggregate metrics.
             output_file = os.path.join(aggregate_output_dir, branch + '_aggregate_metrics_' + magnitude + special_string + '.csv')
 
@@ -249,7 +249,7 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
                 csv_writer.writerows(list_to_write)
                 csv_writer.writerow([])
                 csv_writer.writerows(huc_path_list)
-    
+
             print()
             print("Finished aggregating for the '" + magnitude + "' magnitude. Aggregated metrics over " + str(len(huc_path_list)-1) + " test cases.")
             print()
diff --git a/tools/cache_metrics.py b/tools/cache_metrics.py
old mode 100644
new mode 100755
diff --git a/tools/comparing_src.py b/tools/comparing_src.py
index a9c8a1c8a..977b05794 100755
--- a/tools/comparing_src.py
+++ b/tools/comparing_src.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import matplotlib.pyplot as plt
 import numpy as np
 import json
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
old mode 100644
new mode 100755
index 9423f6c81..290a44704
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -1,155 +1,315 @@
+#!/usr/bin/env python3
+
+import sys
 import os
 from multiprocessing import Pool
 import argparse
 import traceback
-import sys
-
-sys.path.insert(1, 'foss_fim/tests')
+import rasterio
+import geopandas as gpd
+import pandas as pd
+import shutil
+from rasterio.features import shapes
+from shapely.geometry.polygon import Polygon
+from shapely.geometry.multipolygon import MultiPolygon
 from inundation import inundate
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+from utils.shared_functions import getDriver
 
 INPUTS_DIR = r'/data/inputs'
+magnitude_list = ['action', 'minor', 'moderate','major']
 
-# Define necessary variables for inundation().
+# Map path to points with attributes
+all_mapped_ahps_conus_hipr = os.path.join(INPUTS_DIR, 'ahp_sites', 'all_mapped_ahps_reformatted.csv')
+
+# Define necessary variables for inundation()
 hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
 mask_type, catchment_poly = 'huc', ''
-    
 
-def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, job_number, gpkg, extif, depthtif):
-    
-    # Create output directory and log directory.
-    if not os.path.exists(output_cat_fim_dir):
-        os.mkdir(output_cat_fim_dir)
-    log_dir = os.path.join(output_cat_fim_dir, 'logs')
-    if not os.path.exists(log_dir):
-        os.mkdir(log_dir)
-    
+
+def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif, log_file):
+
     no_data_list = []
     procs_list = []
-       
-    # Loop through huc directories in the source_flow directory.
+
     source_flow_dir_list = os.listdir(source_flow_dir)
-    for huc in source_flow_dir_list:
+    output_flow_dir_list = os.listdir(fim_run_dir)
+
+    # Log missing hucs
+    missing_hucs = list(set(source_flow_dir_list) - set(output_flow_dir_list))
+    missing_hucs = [huc for huc in missing_hucs if "." not in huc]
+    if len(missing_hucs) > 0:
+        f = open(log_file, 'a+')
+        f.write(f"Missing hucs from output directory: {', '.join(missing_hucs)}\n")
+        f.close()
+
+    # Loop through matching huc directories in the source_flow directory
+    matching_hucs = list(set(output_flow_dir_list) & set(source_flow_dir_list))
+    for huc in matching_hucs:
+
         if "." not in huc:
-            
-            # Get list of AHPS site directories.
+
+            # Get list of AHPS site directories
             ahps_site_dir = os.path.join(source_flow_dir, huc)
             ahps_site_dir_list = os.listdir(ahps_site_dir)
-            
-            # Map paths to HAND files needed for inundation().
+
+            # Map paths to HAND files needed for inundation()
             fim_run_huc_dir = os.path.join(fim_run_dir, huc)
             rem = os.path.join(fim_run_huc_dir, 'rem_zeroed_masked.tif')
             catchments = os.path.join(fim_run_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
             hydroTable =  os.path.join(fim_run_huc_dir, 'hydroTable.csv')
-            
+
             exit_flag = False  # Default to False.
-            
-            # Check if necessary data exist; set exit_flag to True if they don't exist.
+
+            # Check if necessary data exist; set exit_flag to True if they don't exist
             for f in [rem, catchments, hydroTable]:
                 if not os.path.exists(f):
-                    print(f)
                     no_data_list.append(f)
                     exit_flag = True
-                    
-            # Log "Missing data" if missing TODO improve this.
+
+            # Log missing data
             if exit_flag == True:
-                f = open(os.path.join(log_dir, huc + '.txt'), 'w')
-                f.write("Missing data")
-                continue
-            
-            # Map path to huc directory inside out output_cat_fim_dir.
+                f = open(log_file, 'a+')
+                f.write(f"Missing data for: {fim_run_huc_dir}\n")
+                f.close()
+
+            # Map path to huc directory inside out output_cat_fim_dir
             cat_fim_huc_dir = os.path.join(output_cat_fim_dir, huc)
             if not os.path.exists(cat_fim_huc_dir):
                 os.mkdir(cat_fim_huc_dir)
-            
-            # Loop through AHPS sites.
+
+            # Loop through AHPS sites
             for ahps_site in ahps_site_dir_list:
-                # Map parent directory for AHPS source data dir and list AHPS thresholds (act, min, mod, maj).
+                # map parent directory for AHPS source data dir and list AHPS thresholds (act, min, mod, maj)
                 ahps_site_parent = os.path.join(ahps_site_dir, ahps_site)
                 thresholds_dir_list = os.listdir(ahps_site_parent)
-                
+
                 # Map parent directory for all inundation output filesoutput files.
                 cat_fim_huc_ahps_dir = os.path.join(cat_fim_huc_dir, ahps_site)
                 if not os.path.exists(cat_fim_huc_ahps_dir):
                     os.mkdir(cat_fim_huc_ahps_dir)
-                        
+
                 # Loop through thresholds/magnitudes and define inundation output files paths
                 for magnitude in thresholds_dir_list:
+
                     if "." not in magnitude:
+
                         magnitude_flows_csv = os.path.join(ahps_site_parent, magnitude, 'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+
                         if os.path.exists(magnitude_flows_csv):
-                            if gpkg:
-                                output_extent_gpkg = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.gpkg')
-                            else:
-                                output_extent_gpkg = None
-                            if extif:
-                                output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
-                            else:
-                                output_extent_grid = None
+
+                            output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
+
                             if depthtif:
                                 output_depth_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_depth.tif')
                             else:
                                 output_depth_grid = None
-                            
+
                             # Append necessary variables to list for multiprocessing.
-                            procs_list.append([rem, catchments, catchment_poly, magnitude_flows_csv, huc, hydroTable, output_extent_gpkg, output_extent_grid, output_depth_grid, ahps_site, magnitude, log_dir])
-    # Initiate multiprocessing.                                    
-    pool = Pool(job_number)
+                            procs_list.append([rem, catchments, catchment_poly, magnitude_flows_csv, huc, hydroTable, output_extent_grid, output_depth_grid, ahps_site, magnitude, log_file])
+
+    # Initiate multiprocessing
+    print(f"Running inundation for {len(procs_list)} sites using {number_of_jobs} jobs")
+    pool = Pool(number_of_jobs)
     pool.map(run_inundation, procs_list)
 
 
 def run_inundation(args):
-    
-    # Parse args.
-    rem = args[0]
-    catchments = args[1]
-    catchment_poly = args[2]
+
+    rem                 = args[0]
+    catchments          = args[1]
+    catchment_poly      = args[2]
     magnitude_flows_csv = args[3]
-    huc = args[4]
-    hydroTable = args[5]
-    output_extent_gpkg = args[6]
-    output_extent_grid = args[7]
-    output_depth_grid = args[8]
-    ahps_site = args[9]
-    magnitude = args[10]
-    log_dir = args[11]
-    
-    print("Running inundation for " + str(os.path.split(os.path.split(output_extent_gpkg)[0])[0]))
+    huc                 = args[4]
+    hydroTable          = args[5]
+    output_extent_grid  = args[6]
+    output_depth_grid   = args[7]
+    ahps_site           = args[8]
+    magnitude           = args[9]
+    log_file            = args[10]
+
     try:
-        inundate(
-                 rem,catchments,catchment_poly,hydroTable,magnitude_flows_csv,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=output_extent_grid,inundation_polygon=output_extent_gpkg,
+        inundate(rem,catchments,catchment_poly,hydroTable,magnitude_flows_csv,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=output_extent_grid,inundation_polygon=None,
                  depths=output_depth_grid,out_raster_profile=None,out_vector_profile=None,quiet=True
                 )
+
     except Exception:
-        # Log errors and their tracebacks.
-        f = open(os.path.join(log_dir, huc + "_" + ahps_site + "_" + magnitude + '.txt'), 'w')
-        f.write(traceback.format_exc())
+        # Log errors and their tracebacks
+        f = open(log_file, 'a+')
+        f.write(f"{output_extent_grid} - inundation error: {traceback.format_exc()}\n")
         f.close()
-        
-        
+
+
+def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, log_file):
+
+    # Create workspace
+    gpkg_dir = os.path.join(output_cat_fim_dir, 'gpkg')
+    if not os.path.exists(gpkg_dir):
+        os.mkdir(gpkg_dir)
+
+    fim_version  = os.path.basename(output_cat_fim_dir)
+    merged_layer = os.path.join(output_cat_fim_dir, 'catfim_library.gpkg')
+
+    if not os.path.exists(merged_layer): # prevents appending to existing output
+
+        huc_ahps_dir_list = os.listdir(output_cat_fim_dir)
+        skip_list=['errors','logs','gpkg',merged_layer]
+
+        for magnitude in magnitude_list:
+
+            procs_list = []
+
+            # Loop through all categories
+            for huc in huc_ahps_dir_list:
+
+                if huc not in skip_list:
+
+                    huc_dir = os.path.join(output_cat_fim_dir, huc)
+                    ahps_dir_list = os.listdir(huc_dir)
+
+                    # Loop through ahps sites
+                    for ahps_lid in ahps_dir_list:
+                        ahps_lid_dir = os.path.join(huc_dir, ahps_lid)
+
+                        extent_grid = os.path.join(ahps_lid_dir, ahps_lid + '_' + magnitude + '_extent_' + huc + '.tif')
+
+                        if os.path.exists(extent_grid):
+                            procs_list.append([ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude])
+
+                        else:
+                            try:
+                                f = open(log_file, 'a+')
+                                f.write(f"Missing layers: {extent_gpkg}\n")
+                                f.close()
+                            except:
+                                pass
+
+            # Multiprocess with instructions
+            pool = Pool(number_of_jobs)
+            pool.map(reformat_inundation_maps, procs_list)
+
+        # Merge all layers
+        print(f"Merging {len(os.listdir(gpkg_dir))} layers...")
+
+        for layer in os.listdir(gpkg_dir):
+
+            diss_extent_filename = os.path.join(gpkg_dir, layer)
+
+            # Open diss_extent
+            diss_extent = gpd.read_file(diss_extent_filename)
+
+            # Write/append aggregate diss_extent
+            if os.path.isfile(merged_layer):
+                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False, mode='a')
+            else:
+                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False)
+
+            del diss_extent
+
+        shutil.rmtree(gpkg_dir)
+
+    else:
+        print(f"{merged_layer} already exists.")
+
+
+def reformat_inundation_maps(args):
+
+    try:
+        lid          = args[0]
+        grid_path    = args[1]
+        gpkg_dir     = args[2]
+        fim_version  = args[3]
+        huc          = args[4]
+        magnitude    = args[5]
+
+        # Convert raster to to shapes
+        with rasterio.open(grid_path) as src:
+            image = src.read(1)
+            mask = image > 0
+
+        # Aggregate shapes
+        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
+
+        # convert list of shapes to polygon
+        extent_poly  = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
+
+        # Dissolve polygons
+        extent_poly_diss = extent_poly.dissolve(by='extent')
+
+        # Update attributes
+        extent_poly_diss = extent_poly_diss.reset_index(drop=True)
+        extent_poly_diss['ahps_lid'] = lid
+        extent_poly_diss['magnitude'] = magnitude
+        extent_poly_diss['version'] = fim_version
+        extent_poly_diss['huc'] = huc
+
+        # Project to Web Mercator
+        extent_poly = extent_poly.to_crs(VIZ_PROJECTION)
+
+        # Join attributes
+        all_mapped_ahps_conus_hipr_fl = pd.read_table(all_mapped_ahps_conus_hipr, sep=",")
+        all_mapped_ahps_conus_hipr_fl = all_mapped_ahps_conus_hipr_fl.loc[(all_mapped_ahps_conus_hipr_fl.magnitude==magnitude) & (all_mapped_ahps_conus_hipr_fl.nws_lid==lid)]
+
+        extent_poly_diss = extent_poly_diss.merge(all_mapped_ahps_conus_hipr_fl, left_on=['ahps_lid','magnitude'], right_on=['nws_lid','magnitude'])
+
+        extent_poly_diss = extent_poly_diss.drop(columns='nws_lid')
+
+        # Save dissolved multipolygon
+        handle = os.path.split(grid_path)[1].replace('.tif', '')
+
+        diss_extent_filename = os.path.join(gpkg_dir, handle + "_dissolved.gpkg")
+
+        extent_poly_diss["geometry"] = [MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in extent_poly_diss["geometry"]]
+
+        if not extent_poly_diss.empty:
+
+            extent_poly_diss.to_file(diss_extent_filename,driver=getDriver(diss_extent_filename),index=False)
+
+    except Exception as e:
+        # Log and clean out the gdb so it's not merged in later
+        try:
+            f = open(log_file, 'a+')
+            f.write(str(diss_extent_filename) + " - dissolve error: " + str(e))
+            f.close()
+        except:
+            pass
+
+
 if __name__ == '__main__':
-    
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Categorical inundation mapping for FOSS FIM.')
     parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
     parser.add_argument('-s', '--source-flow-dir',help='Path to directory containing flow CSVs to use to generate categorical FIM.',required=True, default="")
     parser.add_argument('-o', '--output-cat-fim-dir',help='Path to directory where categorical FIM outputs will be written.',required=True, default="")
-    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
-    parser.add_argument('-gpkg','--write-geopackage',help='Using this option will write a geopackage.',required=False, action='store_true')
-    parser.add_argument('-extif','--write-extent-tiff',help='Using this option will write extent TIFFs. This is the default.',required=False, action='store_true')
+    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
     parser.add_argument('-depthtif','--write-depth-tiff',help='Using this option will write depth TIFFs.',required=False, action='store_true')
-    
+
     args = vars(parser.parse_args())
-    
+
     fim_run_dir = args['fim_run_dir']
     source_flow_dir = args['source_flow_dir']
     output_cat_fim_dir = args['output_cat_fim_dir']
-    job_number = int(args['job_number'])
-    gpkg = args['write_geopackage']
-    extif = args['write_extent_tiff']
+    number_of_jobs = int(args['number_of_jobs'])
     depthtif = args['write_depth_tiff']
-    
-    generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, job_number, gpkg, extif, depthtif)
-    
-    
-    
+
+
+    # Create output directory
+    if not os.path.exists(output_cat_fim_dir):
+        os.mkdir(output_cat_fim_dir)
+
+    # Create log directory
+    log_dir = os.path.join(output_cat_fim_dir, 'logs')
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+
+    # Create error log path
+    log_file = os.path.join(log_dir, 'errors.log')
+
+    print("Generating Categorical FIM")
+    generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif,log_file)
+
+    print("Aggregating Categorical FIM")
+    post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir,log_file)
diff --git a/tools/inundation.py b/tools/inundation.py
index b4db4fa49..e7c600510 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -11,6 +11,7 @@
 from shapely.geometry import shape
 from rasterio.mask import mask
 from rasterio.io import DatasetReader,DatasetWriter
+from rasterio.features import shapes
 from collections import OrderedDict
 import argparse
 from warnings import warn
diff --git a/tools/inundation_wrapper_custom_flow.py b/tools/inundation_wrapper_custom_flow.py
old mode 100644
new mode 100755
index e82a474e6..6867bea5f
--- a/tools/inundation_wrapper_custom_flow.py
+++ b/tools/inundation_wrapper_custom_flow.py
@@ -10,9 +10,6 @@
 import sys
 import argparse
 import shutil
-
-# insert python path at runtime for accessing scripts in foss_fim/tests dir (e.g. inundation.py)
-sys.path.insert(1, 'foss_fim/tests')
 from inundation import inundate
 
 TEST_CASES_DIR = r'/data/inundation_review/inundation_custom_flow/'  # Will update.
diff --git a/tools/inundation_wrapper_nwm_flows.py b/tools/inundation_wrapper_nwm_flows.py
index f6d158a79..f2b641e83 100755
--- a/tools/inundation_wrapper_nwm_flows.py
+++ b/tools/inundation_wrapper_nwm_flows.py
@@ -10,9 +10,6 @@
 import csv
 import argparse
 import shutil
-
-# insert python path at runtime for accessing scripts in foss_fim/tests dir (e.g. inundation.py)
-sys.path.insert(1, 'foss_fim/tests')
 from inundation import inundate
 
 TEST_CASES_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
diff --git a/tools/mannings_calibration_run.sh b/tools/mannings_calibration_run.sh
index 8dc737ee4..89d54cd17 100755
--- a/tools/mannings_calibration_run.sh
+++ b/tools/mannings_calibration_run.sh
@@ -71,7 +71,7 @@ fi
 
 export input_NWM_Catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
 export outdir=$outdir
-export testdir="/foss_fim/tests"
+export toolsdir="/foss_fim/tools"
 
 if [ -f "$huclist" ]; then
 
@@ -84,15 +84,15 @@ if [ -f "$huclist" ]; then
     ## RUN ##
     if [ -f "$paramfile" ]; then
         if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb  -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh :::: $paramfile
+            parallel --verbose --lb  -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
         else
-            parallel --eta -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh :::: $paramfile
+            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
         fi
     else
         if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh ::: $paramfile
+            parallel --verbose --lb -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
         else
-            parallel --eta -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh ::: $paramfile
+            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
         fi
     fi
   done <$huclist
@@ -108,15 +108,15 @@ else
     ## RUN ##
     if [ -f "$paramfile" ]; then
         if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb  -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh :::: $paramfile
+            parallel --verbose --lb  -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
         else
-            parallel --eta -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh :::: $paramfile
+            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
         fi
     else
         if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh ::: $paramfile
+            parallel --verbose --lb -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
         else
-            parallel --eta -j $jobLimit -- $testdir/time_and_tee_mannings_calibration.sh ::: $paramfile
+            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
         fi
     fi
   done
diff --git a/tools/mannings_run_by_set.sh b/tools/mannings_run_by_set.sh
index 917672a63..8394b3a5f 100755
--- a/tools/mannings_run_by_set.sh
+++ b/tools/mannings_run_by_set.sh
@@ -12,4 +12,4 @@ mkdir -p $subdir
 
 $srcDir/add_crosswalk.py -d $hucdir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $hucdir/demDerived_reaches_split_filtered.gpkg -s $hucdir/src_base.csv -l $subdir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $subdir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $subdir/src_full_crosswalked.csv -j $subdir/src.json -x $subdir/crosswalk_table.csv -t $subdir/hydroTable.csv -w $hucdir/wbd8_clp.gpkg -b $hucdir/nwm_subset_streams.gpkg -y $hucdir/nwm_catchments_proj_subset.tif -m $param_set -z $input_NWM_Catchments -p FR -c
 
-python3 foss_fim/tests/run_test_case_calibration.py -r $fimdir/$huc -d $subdir -t $huc"_ble" -b "mannings_calibration"/$strorder/$mannings_value
+python3 foss_fim/tools/run_test_case_calibration.py -r $fimdir/$huc -d $subdir -t $huc"_ble" -b "mannings_calibration"/$strorder/$mannings_value
diff --git a/tools/plots/utils/__init__.py b/tools/plots/__init__.py
old mode 100644
new mode 100755
similarity index 100%
rename from tools/plots/utils/__init__.py
rename to tools/plots/__init__.py
diff --git a/tools/plots/eval_plots.py b/tools/plots/eval_plots.py
old mode 100644
new mode 100755
index 9c29087e1..f18390f5e
--- a/tools/plots/eval_plots.py
+++ b/tools/plots/eval_plots.py
@@ -1,45 +1,48 @@
 #!/usr/bin/env python3
+
 import pandas as pd
 from pathlib import Path
 import argparse
 from natsort import natsorted
 import geopandas as gpd
-from utils.shared_functions import filter_dataframe, boxplot, scatterplot, barplot
+from plot_functions import filter_dataframe, boxplot, scatterplot, barplot
+
+
 def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial_ahps = False, fim_1_ms = False):
 
     '''
-    Creates plots and summary statistics using metrics compiled from 
-    synthesize_test_cases. Required inputs are metrics_csv and workspace. 
+    Creates plots and summary statistics using metrics compiled from
+    synthesize_test_cases. Required inputs are metrics_csv and workspace.
     Outputs include:
-        aggregate_<benchmark source>_<configuration>.csv: this csv 
+        aggregate_<benchmark source>_<configuration>.csv: this csv
             contains the aggregated total statistics (i.e. CSI, FAR, POD)
             using the summed area_sq_km fields
-        <benchmark source>_<configuration>_common_sites.csv: this csv 
-            contains the unique sites (e.g usgs/nws: nws_lid; ble: huc08) 
+        <benchmark source>_<configuration>_common_sites.csv: this csv
+            contains the unique sites (e.g usgs/nws: nws_lid; ble: huc08)
             considered for aggregation/plots for each magnitude. The selected
             sites occur in all versions analyzed. For example, if FIM 1,
-            FIM 2, FIM 3.0.0.3 were versions analyzed, the common sites 
-            would be those that had data for ALL versions. This 
+            FIM 2, FIM 3.0.0.3 were versions analyzed, the common sites
+            would be those that had data for ALL versions. This
             analysis is then redone for each magnitude. As such, the number
             of sites may vary with magnitude. The number of sites for each
             magnitude is annotated on generated plots.
-        <benchmark source>_<configuration>_analyzed_data.csv: this is the 
-            dataset used to create plots and aggregate statistics. It is 
+        <benchmark source>_<configuration>_analyzed_data.csv: this is the
+            dataset used to create plots and aggregate statistics. It is
             a subset of the input metrics file and consists of the common
             sites.
-        csi_aggr_<benchmark source>_<configuration>.png: bar plot of the 
+        csi_aggr_<benchmark source>_<configuration>.png: bar plot of the
             aggregated CSI scores. Number of common sites is annotated
             (see list of sites listed in *_*_common_sites.csv).
-        csi_<benchmark source>_<configuration>.png: box plot of CSI scores 
-            (sites weighted equally). Number of common sites is annotated 
+        csi_<benchmark source>_<configuration>.png: box plot of CSI scores
+            (sites weighted equally). Number of common sites is annotated
             (see list of sites listed in *_*_common_sites.csv).
         far_<benchmark source>_<configuration>*.png: box plot of FAR scores
-            (sites weighted equally). Number of common sites is annotated 
+            (sites weighted equally). Number of common sites is annotated
             (see list of sites listed in *_*_common_sites.csv).
-        tpr_<benchmark source>_<configuration>*.png: box plot of TPR/POD 
-            scores (sites weighted equally). Number of common sites is 
+        tpr_<benchmark source>_<configuration>*.png: box plot of TPR/POD
+            scores (sites weighted equally). Number of common sites is
             annotated (see list of sites listed in *_*_common_sites.csv).
-        csi_scatter_<magnitude>_<configuration>*.png: scatter plot comparing 
+        csi_scatter_<magnitude>_<configuration>*.png: scatter plot comparing
             two versions for a given magnitude. This is only generated if
             there are exactly two versions analyzed.
 
@@ -49,54 +52,54 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         Path to csv produced as part of synthesize_test_cases containing
         all metrics across all versions.
     workspace : STRING
-        Path to the output workspace. Subdirectories will be created 
+        Path to the output workspace. Subdirectories will be created
         reflecting the evaluation datasets.
     versions: LIST
-        A list of versions to be aggregated/plotted. Uses the "startswith" 
-        approach. Versions should be supplied in the order they are to 
-        be plotted. For example: ['fim_', 'fb']; This will evaluate all 
+        A list of versions to be aggregated/plotted. Uses the "startswith"
+        approach. Versions should be supplied in the order they are to
+        be plotted. For example: ['fim_', 'fb']; This will evaluate all
         versions that start with fim_ (e.g. fim_1, fim_2, fim_3) and any
         feature branch that starts with "fb". To esbalish version order,
-        the fim versions are naturally sorted and then fb versions 
-        (naturally sorted) are appended. These versions are also used to 
-        filter the input metric csv as only these versions are retained 
-        for analysis. 
+        the fim versions are naturally sorted and then fb versions
+        (naturally sorted) are appended. These versions are also used to
+        filter the input metric csv as only these versions are retained
+        for analysis.
     stats: LIST
-        A list of statistics to be plotted. Must be identical to column 
-        field in metrics_csv. CSI, POD, TPR are currently calculated, if 
+        A list of statistics to be plotted. Must be identical to column
+        field in metrics_csv. CSI, POD, TPR are currently calculated, if
         additional statistics are desired formulas would need to be coded.
     alternate_ahps_query : STRING, optional
-        The default is false. Currently the default ahps query is same 
-        as done for apg goals. If a different query is desired it can be 
-        supplied and it will supercede the default query. 
+        The default is false. Currently the default ahps query is same
+        as done for apg goals. If a different query is desired it can be
+        supplied and it will supercede the default query.
     spatial_ahps : DICTIONARY, optional
-        The default is false. A dictionary with keys as follows: 
+        The default is false. A dictionary with keys as follows:
             'static': Path to AHPS point file created during creation of
                 FIM 3 static libraries.
             'evaluated': Path to extent file created during the creation
                 of the NWS/USGS AHPS preprocessing.
-            'metadata': Path to previously created file that contains 
+            'metadata': Path to previously created file that contains
                 metadata about each site (feature_id, wfo, rfc and etc).
         No spatial layers will be created if set to False, if a dictionary
         is supplied then a spatial layer is produced.
     fim_1_ms: BOOL
-        Default is false. If True then fim_1 rows are duplicated with 
-        extent_config set to MS. This allows for FIM 1 to be included 
+        Default is false. If True then fim_1 rows are duplicated with
+        extent_config set to MS. This allows for FIM 1 to be included
         in MS plots/stats (helpful for nws/usgs ahps comparisons).
 
     Returns
     -------
     all_datasets : DICT
-        Dictionary containing all datasets generated. 
-        Keys: (benchmark_source, extent_config), 
+        Dictionary containing all datasets generated.
+        Keys: (benchmark_source, extent_config),
         Values: (filtered dataframe, common sites)
 
     '''
-    
-    #Import metrics csv as DataFrame and initialize all_datasets dictionary
+
+    # Import metrics csv as DataFrame and initialize all_datasets dictionary
     csv_df = pd.read_csv(metrics_csv)
 
-    #fim_1_ms flag enables FIM 1 to be shown on MS plots/stats
+    # fim_1_ms flag enables FIM 1 to be shown on MS plots/stats
     if fim_1_ms:
         #Query FIM 1 rows based on version beginning with "fim_1"
         fim_1_rows = csv_df.query('version.str.startswith("fim_1")').copy()
@@ -104,74 +107,72 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         fim_1_rows['extent_config'] = 'MS'
         #Append duplicate FIM 1 rows to original dataframe
         csv_df = csv_df.append(fim_1_rows, ignore_index = True)
-        
-    #If versions are supplied then filter out    
+
+    # If versions are supplied then filter out
     if versions:
         #Filter out versions based on supplied version list
         metrics = csv_df.query('version.str.startswith(tuple(@versions))')
     else:
         metrics = csv_df
-       
-    #Group by benchmark source
+
+    # Group by benchmark source
     benchmark_by_source = metrics.groupby(['benchmark_source', 'extent_config'])
 
-    #Iterate through benchmark_by_source. Pre-filter metrics dataframe 
-    #as needed (e.g. usgs/nws filter query). Then further filtering to 
-    #discard all hucs/nws_lid that are not present across all analyzed 
-    #versions for a given magnitude. The final filtered dataset is written 
-    #to a dictionary with the key (benchmark source, extent config) 
-    #and values (filtered dataframe, common sites).
+    ''' Iterate through benchmark_by_source. Pre-filter metrics dataframe
+    as needed (e.g. usgs/nws filter query). Then further filtering to
+    discard all hucs/nws_lid that are not present across all analyzed
+    versions for a given magnitude. The final filtered dataset is written
+    to a dictionary with the key (benchmark source, extent config)
+    and values (filtered dataframe, common sites). '''
+    
     all_datasets = {}
-    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:        
-                
-        #If source is usgs/nws define the base resolution and query 
-        #(use alternate query if passed). Append filtered datasets to 
-        #all_datasets dictionary.
+    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:
+
+        '''If source is usgs/nws define the base resolution and query
+        (use alternate query if passed). Append filtered datasets to
+        all_datasets dictionary.'''
+        
         if benchmark_source in ['usgs','nws']:
-            
-            #Set the base processing unit for the ahps runs.
+
+            # Set the base processing unit for the ahps runs.
             base_resolution = 'nws_lid'
-            
+
             #Default query (used for APG) it could be that bad_sites should be modified. If so pass an alternate query using the "alternate_ahps_query"
             bad_sites = ['grfi2','ksdm7','hohn4','rwdn4']
             query = "not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites"
 
-            #If alternate ahps evaluation query argument is passed, use that.
+            # If alternate ahps evaluation query argument is passed, use that.
             if alternate_ahps_query:
                 query = alternate_ahps_query
 
-            #Filter the dataset based on query
+            # Filter the dataset based on query
             ahps_metrics = benchmark_metrics.query(query)
-            
-            #Filter out all instances where the base_resolution doesn't 
-            #exist across all desired fim versions for a given magnitude.
+
+            # Filter out all instances where the base_resolution doesn't exist across all desired fim versions for a given magnitude
             all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
-                     
-        #If source is 'ble', set base_resolution and append ble dataset 
-        #to all_datasets dictionary
+
+        # If source is 'ble', set base_resolution and append ble dataset to all_datasets dictionary
         elif benchmark_source == 'ble':
-            
-            #Set the base processing unit for ble runs
+
+            # Set the base processing unit for ble runs
             base_resolution = 'huc'
-            
-            #Filter out all instances where base_resolution doesn't exist 
-            #across all desired fim versions for a given magnitude.
+
+            # Filter out all instances where base_resolution doesn't exist across all desired fim versions for a given magnitude
             all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(benchmark_metrics, base_resolution)
-            
-    #For each dataset in all_datasets, generate plots and aggregate statistics.
+
+    # For each dataset in all_datasets, generate plots and aggregate statistics
     for (dataset_name,configuration), (dataset, sites) in all_datasets.items():
-        
-        #Define and create the output workspace as a subfolder within 
-        #the supplied workspace
+
+        # Define and create the output workspace as a subfolder within the supplied workspace
         output_workspace = Path(workspace) / dataset_name / configuration.lower()
-        output_workspace.mkdir(parents = True, exist_ok = True)         
-                
-        #Write out the filtered dataset and common sites to file
+        output_workspace.mkdir(parents = True, exist_ok = True)
+
+        # Write out the filtered dataset and common sites to file
         dataset.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_analyzed_data.csv'), index = False)
         sites_pd = pd.DataFrame.from_dict(sites, orient = 'index').transpose()
         sites_pd.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_common_sites.csv'), index = False)
-        
-        #set the order of the magnitudes and define base resolution.
+
+        # Set the order of the magnitudes and define base resolution
         if dataset_name == 'ble':
             magnitude_order = ['100yr', '500yr']
             base_resolution = 'huc'
@@ -179,29 +180,28 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             magnitude_order = ['action','minor','moderate','major']
             base_resolution = 'nws_lid'
 
-        #Calculate aggregated metrics based on total_sq_km fields.
+        # Calculate aggregated metrics based on total_sq_km fields
         dataset_sums = dataset.groupby(['version', 'magnitude'])[['TP_area_km2','FP_area_km2','FN_area_km2']].sum()
         dataset_sums['csi'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'] + dataset_sums['FN_area_km2'])
         dataset_sums['far'] = dataset_sums['FP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'])
         dataset_sums['pod'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FN_area_km2'])
         dataset_sums = dataset_sums.reset_index()
-        
-        #Write aggregated metrics to file.
+
+        # Write aggregated metrics to file
         dataset_sums.to_csv(output_workspace / f'aggregate_{dataset_name}_{configuration.lower()}.csv', index = False )
 
-        #This section naturally orders analyzed versions which defines 
-        #the hue order for the generated plots.
-        #Get all versions in dataset
-        all_versions = list(dataset.version.unique())        
-        version_order = []        
-        #If versions are not specified then use all available versions 
-        #and assign to versions_list
+        ## This section naturally orders analyzed versions which defines the hue order for the generated plots
+        # Get all versions in dataset
+        all_versions = list(dataset.version.unique())
+        version_order = []
+        
+        # If versions are not specified then use all available versions and assign to versions_list
         if not versions:
             versions_list = all_versions
-        #if versions are supplied assign to versions_list
+        # If versions are supplied assign to versions_list
         else:
-            versions_list = versions        
-        #For each version supplied by the user
+            versions_list = versions
+        # For each version supplied by the user
         for version in versions_list:
             #Select all the versions that start with the supplied version.
             selected_versions = [sel_version for sel_version in all_versions if sel_version.startswith(version)]
@@ -209,8 +209,8 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             selected_versions = natsorted(selected_versions)
             #Populate version order based on the sorted subsets.
             version_order.extend(selected_versions)
-            
-        #Define textbox which will contain the counts of each magnitude.
+
+        # Define textbox which will contain the counts of each magnitude
         textbox = []
         for magnitude in sites:
             count = len(sites[magnitude])
@@ -218,85 +218,85 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             textbox.append(line_text)
         textbox = '\n'.join(textbox)
 
-        #Create aggregate barplot
-        aggregate_file = output_workspace / (f'csi_aggr_{dataset_name}_{configuration.lower()}.png')        
+        # Create aggregate barplot
+        aggregate_file = output_workspace / (f'csi_aggr_{dataset_name}_{configuration.lower()}.png')
         barplot(dataframe = dataset_sums, x_field = 'magnitude', x_order = magnitude_order, y_field = 'csi', hue_field = 'version', ordered_hue = version_order, title_text = f'Aggregate {dataset_name.upper()} FIM Scores', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = aggregate_file)
-        
-        #Create box plots for each metric in supplied stats.
+
+        # Create box plots for each metric in supplied stats
         for stat in stats:
-            output_file = output_workspace / (f'{stat.lower()}_{dataset_name}_{configuration.lower()}.png')    
+            output_file = output_workspace / (f'{stat.lower()}_{dataset_name}_{configuration.lower()}.png')
             boxplot(dataframe = dataset, x_field = 'magnitude', x_order = magnitude_order, y_field = stat, hue_field = 'version', ordered_hue = version_order, title_text = f'{dataset_name.upper()} FIM Sites', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = output_file)
-        
-        #Get the last 2 versions from the version order for scatter plot.
-        if len(version_order) == 2:            
+
+        # Get the last 2 versions from the version order for scatter plot
+        if len(version_order) == 2:
             x_version, y_version = version_order
             for magnitude in magnitude_order:
-                #Scatterplot comparison between last 2 versions.
+                # Scatterplot comparison between last 2 versions
                 x_csi = dataset.query(f'version == "{x_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
                 y_csi = dataset.query(f'version == "{y_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
                 plotdf = pd.merge(x_csi, y_csi, on = base_resolution, suffixes = (f"_{x_version}",f"_{y_version}"))
-                #Define arguments for scatterplot function.
+                # Define arguments for scatterplot function
                 title_text = f'CSI {magnitude}'
                 dest_file = output_workspace / f'csi_scatter_{magnitude}_{configuration.lower()}.png'
                 scatterplot(dataframe = plotdf, x_field = f'CSI_{x_version}', y_field = f'CSI_{y_version}', title_text = title_text, annotate = False, dest_file = dest_file)
-    
+
 
     #######################################################################
     #Create spatial layers with threshold and mapping information
     ########################################################################
     if spatial_ahps:
 
-        #Read in supplied shapefile layers
-        #Layer containing metadata for each site (feature_id, wfo, etc). 
-        #Convert nws_lid to lower case.
+        # Read in supplied shapefile layers
+        # Layer containing metadata for each site (feature_id, wfo, etc)
+        # Convert nws_lid to lower case
         ahps_metadata = gpd.read_file(spatial_ahps['metadata'])
         ahps_metadata['nws_lid'] = ahps_metadata['nws_lid'].str.lower()
         metadata_crs = ahps_metadata.crs
-        
-        #Extent layer generated from preprocessing NWS/USGS datasets
+
+        # Extent layer generated from preprocessing NWS/USGS datasets
         evaluated_ahps_extent = gpd.read_file(spatial_ahps['evaluated'])
-        
-        #Extent layer generated from static ahps library preprocessing
+
+        # Extent layer generated from static ahps library preprocessing
         static_library = gpd.read_file(spatial_ahps['static'])
-        
-        #Fields to keep
-        #Get list of fields to keep in merge
-        preserved_static_library_fields = ['nws_lid'] + [i for i in static_library.columns if i.startswith(('Q','S'))]                
-        #Get list of fields to keep in merge.
+
+        # Fields to keep
+        # Get list of fields to keep in merge
+        preserved_static_library_fields = ['nws_lid'] + [i for i in static_library.columns if i.startswith(('Q','S'))]
+        # Get list of fields to keep in merge
         preserved_evaluated_ahps_fields = ['nws_lid', 'source', 'geometry'] + [i for i in evaluated_ahps_extent.columns if i.startswith(('action','minor','moderate','major'))]
 
-        #Join tables to evaluated_ahps_extent
+        # Join tables to evaluated_ahps_extent
         evaluated_ahps_extent = evaluated_ahps_extent[preserved_evaluated_ahps_fields]
         evaluated_ahps_extent = evaluated_ahps_extent.merge(ahps_metadata, on = 'nws_lid')
         evaluated_ahps_extent['geometry'] = evaluated_ahps_extent['geometry_y']
         evaluated_ahps_extent.drop(columns = ['geometry_y','geometry_x'], inplace = True)
         evaluated_ahps_extent = evaluated_ahps_extent.merge(static_library[preserved_static_library_fields], on = 'nws_lid')
-        
-        #Join dataset metrics to evaluated_ahps_extent data. 
+
+        # Join dataset metrics to evaluated_ahps_extent data
         final_join = pd.DataFrame()
         for (dataset_name, configuration), (dataset, sites) in all_datasets.items():
-            #Only select ahps from dataset if config is MS
+            # Only select ahps from dataset if config is MS
             if dataset_name in ['usgs','nws'] and configuration == 'MS':
-                #Select records from evaluated_ahps_extent that match the dataset name
-                subset = evaluated_ahps_extent.query(f'source == "{dataset_name}"')                        
-                #Join to dataset
+                # Select records from evaluated_ahps_extent that match the dataset name
+                subset = evaluated_ahps_extent.query(f'source == "{dataset_name}"')
+                # Join to dataset
                 dataset_with_subset = dataset.merge(subset, on = 'nws_lid')
-                #Append rows to final_join dataframe
+                # Append rows to final_join dataframe
                 final_join = final_join.append(dataset_with_subset)
-        
-        #Modify version field
+
+        # Modify version field
         final_join['version'] = final_join.version.str.split('_nws|_usgs').str[0]
-        
-        #Write geodataframe to file
+
+        # Write geodataframe to file
         gdf = gpd.GeoDataFrame(final_join, geometry = final_join['geometry'], crs = metadata_crs)
         output_shapefile = Path(workspace) / 'nws_usgs_site_info.shp'
-        gdf.to_file(output_shapefile) 
-                    
+        gdf.to_file(output_shapefile)
+
 
 
 #######################################################################
 if __name__ == '__main__':
-    #Parse arguments
+    # Parse arguments
     parser = argparse.ArgumentParser(description = 'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
     parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
     parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
@@ -305,22 +305,22 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     parser.add_argument('-q', '--alternate_ahps_query',help = 'Alternate filter query for AHPS. Default is: "not nws_lid.isnull() & not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites" where bad_sites are (grfi2,ksdm7,hohn4,rwdn4)', default = False, required = False)
     parser.add_argument('-sp', '--spatial_ahps', help = 'If spatial point layer is desired, supply a csv with 3 lines of the following format: metadata, path/to/metadata/shapefile\nevaluated, path/to/evaluated/shapefile\nstatic, path/to/static/shapefile.', default = False, required = False)
     parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
-    
-    #Extract to dictionary and assign to variables.
+
+    # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
-    
-    #If errors occur reassign error to True
+
+    # If errors occur reassign error to True
     error = False
-    #Create dictionary if file specified for spatial_ahps
+    # Create dictionary if file specified for spatial_ahps
     if args['spatial_ahps']:
-        #Create dictionary
+        # Create dictionary
         spatial_dict = {}
         with open(args['spatial_ahps']) as file:
             for line in file:
                 key, value = line.strip('\n').split(',')
                 spatial_dict[key] = Path(value)
         args['spatial_ahps'] = spatial_dict
-        #Check that all required keys are present and overwrite args with spatial_dict
+        # Check that all required keys are present and overwrite args with spatial_dict
         required_keys = set(['metadata', 'evaluated', 'static'])
         if required_keys - spatial_dict.keys():
           print('\n Required keys are: metadata, evaluated, static')
@@ -329,7 +329,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             args['spatial_ahps'] = spatial_dict
 
 
-    #Finalize Variables
+    # Finalize Variables
     m = args['metrics_csv']
     w = args['workspace']
     v = args['versions']
@@ -338,6 +338,6 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     sp= args['spatial_ahps']
     f = args['fim_1_ms']
 
-    #Run eval_plots function
-    if not error:        
-        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f)
\ No newline at end of file
+    # Run eval_plots function
+    if not error:
+        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f)
diff --git a/tools/plots/utils/shared_functions.py b/tools/plots/plot_functions.py
old mode 100644
new mode 100755
similarity index 100%
rename from tools/plots/utils/shared_functions.py
rename to tools/plots/plot_functions.py
diff --git a/tools/preprocess/create_flow_forecast_file.py b/tools/preprocess/create_flow_forecast_file.py
old mode 100644
new mode 100755
index 9de7abfe5..bb8833343
--- a/tools/preprocess/create_flow_forecast_file.py
+++ b/tools/preprocess/create_flow_forecast_file.py
@@ -1,10 +1,5 @@
 #!/usr/bin/env python3
 
-# -*- coding: utf-8 -*-
-"""
-Created on Wed Jul 29 11:48:37 2020
-@author: Fernando Aristizabal with edits by Trevor Grout
-"""
 import os
 import geopandas as gpd
 import argparse
@@ -21,54 +16,54 @@ def create_flow_forecast_file(ble_geodatabase, nwm_geodatabase, output_parent_di
         Path to nwm geodatabase.
     output_parent_dir : STRING
         Output parent directory of output. Flow files will be output to subdirectories within parent directory.
-    ble_xs_layer_name : STRING 
-       The cross section layer in the ble geodatabase to be imported. Default is 'XS' (sometimes it is 'XS_1D') 
-    ble_huc_layer_name : STRING 
+    ble_xs_layer_name : STRING
+       The cross section layer in the ble geodatabase to be imported. Default is 'XS' (sometimes it is 'XS_1D')
+    ble_huc_layer_name : STRING
        The huc layer in the ble geodatabase.  Default is 'S_HUC_Ar' (sometimes it is 'S_HUC_ar' )
-    ble_huc_id_field : STRING 
+    ble_huc_id_field : STRING
        The attribute field within the ble_huc_layer_name containing the huc code. Default is 'HUC_CODE'. Assumes only 1 unique code.
-    nwm_stream_layer_name : STRING 
+    nwm_stream_layer_name : STRING
        The stream centerline layer name (or partial layer name) for the NWM geodatabase.  Default is 'RouteLink_FL_2020_04_07'.
-    nwm_feature_id_field : STRING 
+    nwm_feature_id_field : STRING
        The feature id of the nwm segments.  Default is 'ID' (applicable if nwmv2.1 is used)
     Returns
     -------
     None.
 
     '''
-    #Read the ble xs layer into a geopandas dataframe.
+    # Read the ble xs layer into a geopandas dataframe.
     xs_layer = gpd.read_file(ble_geodatabase,layer = ble_xs_layer_name)
 
-    #Read ble huc layer into a geopandas dataframe and extract the huc code. By default it assumes only one HUC in the layer (typically always the case).
+    # Read ble huc layer into a geopandas dataframe and extract the huc code. By default it assumes only one HUC in the layer (typically always the case).
     huc_layer = gpd.read_file(ble_geodatabase, layer = ble_huc_layer_name)
     [huc] = huc_layer[ble_huc_id_field].unique()
-    
-    #Read in the NWM stream layer into a geopandas dataframe using the bounding box option based on the extents of the BLE XS layer.
+
+    # Read in the NWM stream layer into a geopandas dataframe using the bounding box option based on the extents of the BLE XS layer.
     nwm_river_layer = gpd.read_file(nwm_geodatabase, bbox = xs_layer, layer = nwm_stream_layer_name)
-    
-    #Make sure xs_layer is in same projection as nwm_river_layer.
+
+    # Make sure xs_layer is in same projection as nwm_river_layer.
     xs_layer_proj = xs_layer.to_crs(nwm_river_layer.crs)
-    
-    #Perform an intersection of the BLE layers and the NWM layers, using the keep_geom_type set to False produces a point output.
+
+    # Perform an intersection of the BLE layers and the NWM layers, using the keep_geom_type set to False produces a point output.
     intersection = gpd.overlay(xs_layer_proj, nwm_river_layer, how = 'intersection', keep_geom_type = False)
 
-    #Create the flow forecast files
-    #define fields containing flow (typically these won't change for BLE)
+    ## Create the flow forecast files
+    # Define fields containing flow (typically these won't change for BLE)
     flow_fields = ['E_Q_01PCT','E_Q_0_2PCT']
 
-    #define return period associated with flow_fields (in same order as flow_fields). These will also serve as subdirectory names.
+    # Define return period associated with flow_fields (in same order as flow_fields). These will also serve as subdirectory names.
     return_period = ['100yr','500yr']
 
-    #Conversion factor from CFS to CMS
-    dischargeMultiplier = 0.3048 ** 3 
-        
-    #Write individual flow csv files
+    # Conversion factor from CFS to CMS
+    dischargeMultiplier = 0.3048 ** 3
+
+    # Write individual flow csv files
     for i,flow in enumerate(flow_fields):
 
-        #Write dataframe with just ID and single flow event
+        # Write dataframe with just ID and single flow event
         forecast = intersection[[nwm_feature_id_field,flow]]
 
-        #Rename field names and re-define datatypes
+        # Rename field names and re-define datatypes
         forecast = forecast.rename(columns={nwm_feature_id_field :'feature_id',flow : 'discharge'})
         forecast = forecast.astype({'feature_id' : int , 'discharge' : float})
 
@@ -76,18 +71,18 @@ def create_flow_forecast_file(ble_geodatabase, nwm_geodatabase, output_parent_di
         forecast = forecast.groupby('feature_id').median()
         forecast = forecast.reset_index(level=0)
 
-        #Convert CFS to CMS
+        # Convert CFS to CMS
         forecast['discharge'] = forecast['discharge'] * dischargeMultiplier
 
-        #Set paths and write file
+        # Set paths and write file
         output_dir = os.path.join(output_parent_dir, huc)
         dir_of_csv = os.path.join(output_dir,return_period[i])
         os.makedirs(dir_of_csv,exist_ok = True)
         path_to_csv = os.path.join(dir_of_csv,"ble_huc_{}_flows_{}.csv".format(huc,return_period[i]))
-        forecast.to_csv(path_to_csv,index=False)   
-   
+        forecast.to_csv(path_to_csv,index=False)
+
 if __name__ == '__main__':
-    #Parse arguments
+    # Parse arguments
     parser = argparse.ArgumentParser(description = 'Produce forecast flow files from BLE datasets')
     parser.add_argument('-b', '--ble-geodatabase', help = 'BLE geodatabase (.gdb file extension). Will look for layer with "XS" in name. It is assumed the 100 year flow field is "E_Q_01PCT" and the 500 year flow field is "E_Q_0_2_PCT" as these are the default field names.', required = True)
     parser.add_argument('-n', '--nwm-geodatabase',  help = 'NWM geodatabase (.gdb file extension).', required = True)
@@ -97,9 +92,7 @@ def create_flow_forecast_file(ble_geodatabase, nwm_geodatabase, output_parent_di
     parser.add_argument('-huid', '--ble-huc-id-field', help = 'BLE id field in the ble-huc-layer-name. Default field is "HUC_CODE".', required = False, default = 'HUC_CODE')
     parser.add_argument('-l', '--nwm-stream-layer-name', help = 'NWM streams layer. Default layer is "RouteLink_FL_2020_04_07")', required = False, default = 'RouteLink_FL_2020_04_07')
     parser.add_argument('-f', '--nwm-feature-id-field', help = 'id field for nwm streams. Not required if NWM v2.1 is used (default id field is "ID")', required = False, default = 'ID')
-    #Extract to dictionary and assign to variables.
+    # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
-    #Run create_flow_forecast_file
+    # Run create_flow_forecast_file
     create_flow_forecast_file(**args)
-    
-    
diff --git a/tools/preprocess/preprocess_benchmark.py b/tools/preprocess/preprocess_benchmark.py
old mode 100644
new mode 100755
index 02f8e5ea8..81a65db2d
--- a/tools/preprocess/preprocess_benchmark.py
+++ b/tools/preprocess/preprocess_benchmark.py
@@ -1,12 +1,5 @@
 #!/usr/bin/env python3
 
-# -*- coding: utf-8 -*-
-"""
-Created on Thu Jul 23 15:17:04 2020
-
-@author: trevor.grout
-"""
-
 import rasterio
 from rasterio.warp import calculate_default_transform, reproject, Resampling
 import rasterio.mask
@@ -15,7 +8,7 @@
 
 def preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_path = None):
     '''
-    This function will preprocess a benchmark dataset for purposes of evaluating FIM output. A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference dataset. The benchmark raster will also be converted to a boolean (True/False) raster with inundated areas (True or 1) and dry areas (False or 0). 
+    This function will preprocess a benchmark dataset for purposes of evaluating FIM output. A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference dataset. The benchmark raster will also be converted to a boolean (True/False) raster with inundated areas (True or 1) and dry areas (False or 0).
 
     Parameters
     ----------
@@ -34,59 +27,59 @@ def preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_p
         Raster profile information for the preprocessed benchmark array (required for writing to output dataset).
 
     '''
-    #Open and read raster and benchmark rasters
+    # Open and read raster and benchmark rasters
     reference = rasterio.open(reference_raster)
     benchmark = rasterio.open(benchmark_raster)
-    benchmark_arr = benchmark.read(1)    
+    benchmark_arr = benchmark.read(1)
 
-    #Set arbitrary no data value that is not possible value of the benchmark dataset. This will be reassigned later.
+    # Set arbitrary no data value that is not possible value of the benchmark dataset. This will be reassigned later
     nodata_value = -2147483648
-    
-    #Determine the new transform and dimensions of reprojected/resampled raster.
+
+    # Determine the new transform and dimensions of reprojected/resampled raster
     new_transform, new_width, new_height = calculate_default_transform(benchmark.crs, reference.crs, benchmark.width, benchmark.height, *benchmark.bounds, resolution = reference.res)
 
-    #Define an empty array that is same dimensions as output by the "calculate_default_transform" command. 
+    # Define an empty array that is same dimensions as output by the "calculate_default_transform" command
     benchmark_projected = np.empty((new_height,new_width), dtype=np.int32)
 
-    #Reproject and resample the benchmark dataset. Bilinear resampling due to continuous depth data.
-    reproject(benchmark_arr, 
+    # Reproject and resample the benchmark dataset. Bilinear resampling due to continuous depth data
+    reproject(benchmark_arr,
               destination = benchmark_projected,
-              src_transform = benchmark.transform, 
+              src_transform = benchmark.transform,
               src_crs = benchmark.crs,
               src_nodata = benchmark.nodata,
-              dst_transform = new_transform, 
+              dst_transform = new_transform,
               dst_crs = reference.crs,
               dst_nodata = nodata_value,
               dst_resolution = reference.res,
               resampling = Resampling.bilinear)
 
-    #Convert entire depth grid to boolean (1 = Flood, 0 = No Flood)
+    # Convert entire depth grid to boolean (1 = Flood, 0 = No Flood)
     boolean_benchmark = np.where(benchmark_projected != nodata_value, 1, 0)
 
-    #Update profile (data type, NODATA, transform, width/height).
+    #Update profile (data type, NODATA, transform, width/height)
     profile = reference.profile
     profile.update(transform = new_transform)
     profile.update(dtype = rasterio.int8)
-    profile.update(nodata = 2) #Update NODATA to some integer so we can keep int8 datatype. There are no NODATA in the raster dataset.
+    profile.update(nodata = 2) #Update NODATA to some integer so we can keep int8 datatype. There are no NODATA in the raster dataset
     profile.update (width = new_width)
     profile.update(height = new_height)
 
-    #Write out preprocessed benchmark array to raster if path is supplied
-    if out_raster_path is not None:    
-        with rasterio.Env():    
-            #Write out reassigned values to raster dataset.
+    # Write out preprocessed benchmark array to raster if path is supplied
+    if out_raster_path is not None:
+        with rasterio.Env():
+            # Write out reassigned values to raster dataset
             with rasterio.open(out_raster_path, 'w', **profile) as dst:
-                dst.write(boolean_benchmark.astype('int8'),1)   
+                dst.write(boolean_benchmark.astype('int8'),1)
     return boolean_benchmark.astype('int8'), profile
 
 if __name__ == '__main__':
-    #Parse arguments
+    # Parse arguments
     parser = argparse.ArgumentParser(description = 'Preprocess BLE grids (in tiff format) for use in run_test_cast.py. Preprocessing includes reprojecting and converting to boolean raster (1 = Flooding, 0 = No Flooding)')
     parser.add_argument('-b','--benchmark-raster', help = 'BLE depth or water surface elevation grid (in GTiff format).', required = True)
     parser.add_argument('-r', '--reference-raster', help = 'Benchmark will use reference raster to set CRS and resolution to reference raster CRS.', required = True)
     parser.add_argument('-o', '--out-raster-path', help = 'Output raster path (include name and extension).', required = True)
 
-    #Extract to dictionary and assign to variables.
+    # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
-    #Run preprocess benchmark function
+    # Run preprocess benchmark function
     preprocess_benchmark_static(**args)
diff --git a/tools/preprocess/preprocess_fimx.py b/tools/preprocess/preprocess_fimx.py
old mode 100644
new mode 100755
index 344fecf7d..cad6058d0
--- a/tools/preprocess/preprocess_fimx.py
+++ b/tools/preprocess/preprocess_fimx.py
@@ -1,9 +1,5 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Jul 24 13:50:59 2020
+#!/usr/bin/env python3
 
-@author: trevor.grout
-"""
 import rasterio
 from rasterio.warp import calculate_default_transform, reproject, Resampling
 from rasterio import features
@@ -47,74 +43,75 @@ def fimx_to_fim3(catchments_path, raster_value_field, hand_raster_path, template
         Preprocessed catchment raster profile.
 
     '''
-    
-    
-    #Read in template raster as band object.
+
+
+    # Read in template raster as band object
     reference = rasterio.open(template_raster)
-    
-    #Step 1: Convert HAND grid
-    #Read in the hand raster     
+
+    ## Step 1: Convert HAND grid
+    # Read in the hand raster
     hand = rasterio.open(hand_raster_path)
     hand_arr = hand.read(1)
-    #Determine the new transform and dimensions of reprojected raster (CRS = reference raster).
+    #Determine the new transform and dimensions of reprojected raster (CRS = reference raster)
     new_transform, new_width, new_height = calculate_default_transform(hand.crs, reference.crs, hand.width, hand.height, *hand.bounds)
-    #Define an empty array that is same dimensions as output by the "calculate_default_transform" command. 
-    hand_proj = np.empty((new_height,new_width), dtype=np.float)    
-    #Reproject to target dataset (resample method is bilinear due to elevation type data).
+    # Define an empty array that is same dimensions as output by the "calculate_default_transform" command
+    hand_proj = np.empty((new_height,new_width), dtype=np.float)
+    # Reproject to target dataset (resample method is bilinear due to elevation type data)
     hand_nodata_value = -2147483648
-    reproject(hand_arr, 
+    reproject(hand_arr,
               destination = hand_proj,
-              src_transform = hand.transform, 
+              src_transform = hand.transform,
               src_crs = hand.crs,
               src_nodata = hand.nodata,
-              dst_transform = new_transform, 
+              dst_transform = new_transform,
               dst_crs = reference.crs,
               dst_nodata = hand_nodata_value,
               dst_resolution = hand.res,
               resampling = Resampling.bilinear)
-    #Update profile data type and no data value.
+
+    # Update profile data type and no data value
     hand_profile = reference.profile
     hand_profile.update(dtype = rasterio.float32)
     hand_profile.update(nodata = hand_nodata_value)
     hand_profile.update(width = new_width)
     hand_profile.update(height = new_height)
     hand_profile.update(transform = new_transform)
-    
-    #Step 2: Catchments to Polygons (same extent as the HAND raster)
-    #Read in the catchment layer to geopandas dataframe and convert to same CRS as reference raster.
+
+    ## Step 2: Catchments to Polygons (same extent as the HAND raster)
+    # Read in the catchment layer to geopandas dataframe and convert to same CRS as reference raster
     gdbpath, layername = os.path.split(catchments_path)
     gdb_layer=gpd.read_file(gdbpath, driver='FileGDB', layer=layername)
     proj_gdb_layer = gdb_layer.to_crs(reference.crs)
-    #Prepare vector data to be written to raster.
-    shapes = list(zip(proj_gdb_layer['geometry'],proj_gdb_layer[raster_value_field].astype('int32')))   
-    #Write vector data to raster image. Fill raster with zeros for areas that do not have data. We will set nodata to be zero later.
-    catchment_proj = features.rasterize(((geometry, value) for geometry, value in shapes), fill = 0, out_shape=hand_proj.shape, transform=hand_profile['transform'], dtype = 'int32' )    
-    #Save raster image to in-memory dataset. Reset dtype and nodata values.
+    # Prepare vector data to be written to raster
+    shapes = list(zip(proj_gdb_layer['geometry'],proj_gdb_layer[raster_value_field].astype('int32')))
+    # Write vector data to raster image. Fill raster with zeros for areas that do not have data. We will set nodata to be zero later
+    catchment_proj = features.rasterize(((geometry, value) for geometry, value in shapes), fill = 0, out_shape=hand_proj.shape, transform=hand_profile['transform'], dtype = 'int32' )
+    # Save raster image to in-memory dataset. Reset dtype and nodata values.
     catchment_profile = hand_profile.copy()
     catchment_profile.update(dtype = 'int32')
     catchment_profile.update(nodata=0)
-  
-    #Step 3: Union of NODATA locations applied to both HAND and Catchment grids. 
+
+    ## Step 3: Union of NODATA locations applied to both HAND and Catchment grids
     catchment_masked = np.where(np.logical_or(hand_proj == hand_profile['nodata'], catchment_proj == catchment_profile['nodata']), catchment_profile['nodata'],catchment_proj)
-        #Assign NODATA to hand where both catchment and hand have NODATA else assign hand values.
+        # Assign NODATA to hand where both catchment and hand have NODATA else assign hand values.
     hand_masked = np.where(np.logical_or(hand_proj == hand_profile['nodata'], catchment_proj == catchment_profile['nodata']), hand_profile['nodata'],hand_proj)
 
-    #Step 4: Write out hand and catchment rasters to file if path is specified
+    ## Step 4: Write out hand and catchment rasters to file if path is specified
     if out_hand_path is not None:
-        os.makedirs(os.path.split(out_hand_path)[0], exist_ok = True)        
+        os.makedirs(os.path.split(out_hand_path)[0], exist_ok = True)
         with rasterio.Env():
             with rasterio.open(out_hand_path, 'w', **hand_profile) as hnd_dst:
                 hnd_dst.write(hand_masked.astype('float32'),1)
     if out_catchment_path is not None:
-        os.makedirs(os.path.split(out_catchment_path)[0], exist_ok = True)        
+        os.makedirs(os.path.split(out_catchment_path)[0], exist_ok = True)
         with rasterio.Env():
             with rasterio.open(out_catchment_path, 'w', **catchment_profile) as cat_dst:
-                cat_dst.write(catchment_masked.astype('int32'),1)   
-    
+                cat_dst.write(catchment_masked.astype('int32'),1)
+
     return hand_masked, hand_profile, catchment_masked, catchment_profile
 
 if __name__ == '__main__':
-    #Parse arguments
+    # Parse arguments
     parser = argparse.ArgumentParser(description = 'Preprocess FIM 1 and FIM 2 HAND and Catchment grids to be compatible with FIM 3.')
     parser.add_argument('-c','--catchments-path', help = 'Path to catchments vector file', required = True)
     parser.add_argument('-f', '--raster-value-field', help = 'Attribute ID field from which raster values will be assigned. Typically this will be "HydroID" for FIM2 and "feature_ID" for fim 1.', required = True)
@@ -122,8 +119,7 @@ def fimx_to_fim3(catchments_path, raster_value_field, hand_raster_path, template
     parser.add_argument('-t', '--template-raster', help = 'Path to a template raster. Properties (CRS, resolution) of the template raster will be used to preprocess HAND and Catchments grids', required = True)
     parser.add_argument('-oh', '--out-hand-path', help = 'Path to the output HAND raster. Raster must be named "rem_clipped_zeroed_masked.tif', required = True)
     parser.add_argument('-oc', '--out-catchment-path', help = 'Path to the output Catchment raster. Raster must be named "gw_catchments_reaches_clipped_addedAttributes.tif"', required = True)
-    #Extract to dictionary and assign to variables.
+    # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
-    #Run fimx to fim3 function.
+    # Run fimx to fim3 function
     fimx_to_fim3(**args)
-
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 2a0a279c5..e3168a422 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -5,16 +5,16 @@
 import shutil
 import argparse
 
-from utils.shared_functions import compute_contingency_stats_from_rasters
-from utils.shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
+from tools_shared_functions import compute_contingency_stats_from_rasters
+from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
 from inundation import inundate
 
 
 def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True):
-    
+
     benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
     current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
-    
+
     # Construct paths to development test results if not existent.
     if archive_results:
         version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'official_versions', version)
@@ -28,7 +28,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
         else:
             print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
             return
-        
+
     os.mkdir(version_test_case_dir_parent)
 
     print("Running the alpha test for test_id: " + test_id + ", " + version + "...")
@@ -49,13 +49,13 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     else:
         catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
     hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
-        
+
     # Map necessary inputs for inundation().
     hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
 
     # Create list of shapefile paths to use as exclusion areas.
     zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
-    mask_dict = {'levees': 
+    mask_dict = {'levees':
                     {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
                      'buffer': None,
                      'operation': 'exclude'
@@ -66,7 +66,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                      'operation': 'exclude',
                      },
                 }
-            
+
     if inclusion_area != '':
         inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
         mask_dict.update({inclusion_area_name: {'path': inclusion_area,
@@ -75,7 +75,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
         # Append the concatenated inclusion_area_name and buffer.
         if inclusion_area_buffer == None:
             inclusion_area_buffer = 0
-        stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm') 
+        stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm')
 
     # Check if magnitude is list of magnitudes or single value.
     magnitude_list = magnitude
@@ -88,13 +88,13 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
         version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
         if not os.path.exists(version_test_case_dir):
             os.mkdir(version_test_case_dir)
-    
+
         # Construct path to validation raster and forecast file.
         if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
             benchmark_raster_path_list, forecast_list = [], []
             lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
             lid_list, inundation_raster_list, domain_file_list = [], [], []
-            
+
             for lid in lid_dir_list:
                 lid_dir = os.path.join(validation_data_path, current_huc, lid)
                 benchmark_raster_path_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif'))  # TEMP
@@ -109,7 +109,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
             forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
             forecast_list = [forecast_path]
             inundation_raster_list = [os.path.join(version_test_case_dir, 'inundation_extent.tif')]
-            
+
         for index in range(0, len(benchmark_raster_path_list)):
             benchmark_raster_path = benchmark_raster_path_list[index]
             forecast = forecast_list[index]
@@ -123,7 +123,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                      'buffer': None,
                      'operation': 'include'}
                         })
-        
+
                 if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_domain_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
                     continue
             else:  # If not in AHPS_BENCHMARK_CATEGORIES.
@@ -137,16 +137,16 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                          subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
                          depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
                         )
-            
+
                 print("-----> Inundation mapping complete.")
                 predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
-        
+
                 # Define outputs for agreement_raster, stats_json, and stats_csv.
                 if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
                     agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
                 else:
                     agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-         
+
                 compute_contingency_stats_from_rasters(predicted_raster_path,
                                                        benchmark_raster_path,
                                                        agreement_raster,
@@ -157,16 +157,16 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                                                        test_id=test_id,
                                                        mask_dict=mask_dict,
                                                        )
-        
+
                 if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
                     del mask_dict[ahps_lid]
-                
+
                 print(" ")
                 print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
                 print(" ")
             except Exception as e:
-                print(e)      
-        
+                print(e)
+
         if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
             # -- Delete temp files -- #
             # List all files in the output directory.
@@ -217,12 +217,12 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
         print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
         print()
         exit_flag = True
-    
+
     # Ensure inclusion_area path exists.
     if args['inclusion_area'] != "" and not os.path.exists(args['inclusion_area']):
         print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area (-i) " + CYAN_BOLD + args['inclusion_area'] + WHITE_BOLD + " could not be located." + ENDC)
         exit_flag = True
-        
+
     try:
         inclusion_buffer = int(args['inclusion_area_buffer'])
     except ValueError:
@@ -235,8 +235,8 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
             args['magnitude'] = ['action', 'minor', 'moderate', 'major']
         else:
             print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
-            exit_flag = True     
-            
+            exit_flag = True
+
     if exit_flag:
         print()
         sys.exit()
diff --git a/tools/run_test_case_calibration.py b/tools/run_test_case_calibration.py
index f630360b0..728b87abe 100755
--- a/tools/run_test_case_calibration.py
+++ b/tools/run_test_case_calibration.py
@@ -9,7 +9,7 @@
 import argparse
 import shutil
 
-from utils.shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
+from tools_shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
 from inundation import inundate
 
 TEST_CASES_DIR = r'/data/test_cases/'  # Will update.
diff --git a/tools/utils/shapefile_to_raster.py b/tools/shapefile_to_raster.py
old mode 100644
new mode 100755
similarity index 88%
rename from tools/utils/shapefile_to_raster.py
rename to tools/shapefile_to_raster.py
index 4d1a61ed9..fc1689954
--- a/tools/utils/shapefile_to_raster.py
+++ b/tools/shapefile_to_raster.py
@@ -1,9 +1,4 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Jul 14 16:19:26 2020
-
-@author: bradford.bates
-"""
+#!/usr/bin/env python3
 
 # A script to rasterise a shapefile to the same projection & pixel resolution as a reference image.
 from osgeo import ogr, gdal
@@ -28,7 +23,7 @@
 print("Rasterising shapefile...")
 Output = gdal.GetDriverByName(gdalformat).Create(OutputImage, Image.RasterXSize, Image.RasterYSize, 1, datatype, options=['COMPRESS=DEFLATE'])
 Output.SetProjection(Image.GetProjectionRef())
-Output.SetGeoTransform(Image.GetGeoTransform()) 
+Output.SetGeoTransform(Image.GetGeoTransform())
 
 # Write data to band 1
 Band = Output.GetRasterBand(1)
@@ -43,4 +38,4 @@
 
 # Build image overviews
 subprocess.call("gdaladdo --config COMPRESS_OVERVIEW DEFLATE "+OutputImage+" 2 4 8 16 32 64", shell=True)
-print("Done.")
\ No newline at end of file
+print("Done.")
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
old mode 100644
new mode 100755
index 1fdb0a4dc..f3d02192c
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -7,11 +7,11 @@
 import csv
 
 from run_test_case import run_alpha_test
-from utils.shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
+from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
 
 
 def create_master_metrics_csv(master_metrics_csv_output):
-            
+
     # Construct header
     metrics_to_write = ['true_negatives_count',
                         'false_negatives_count',
@@ -55,26 +55,26 @@ def create_master_metrics_csv(master_metrics_csv_output):
                         'masked_perc',
                         'masked_area_km2'
                         ]
-    
+
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
     list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
-    
+
     versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
-    
+
     for benchmark_source in ['ble', 'nws', 'usgs']:
-        
+
         benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
-        
+
         if benchmark_source == 'ble':
             test_cases_list = os.listdir(benchmark_test_case_dir)
-                            
+
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
-                    
+
                     huc = test_case.split('_')[0]
                     official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-                    
+
                     for magnitude in ['100yr', '500yr']:
                         for version in versions_to_aggregate:
                             if '_fr' in version:
@@ -108,21 +108,21 @@ def create_master_metrics_csv(master_metrics_csv_output):
                                             sub_list_to_append.append(benchmark_source)
                                             sub_list_to_append.append(extent_config)
                                             sub_list_to_append.append(calibrated)
-                                            
+
                                             list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
-                
+
         if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
             test_cases_list = os.listdir(benchmark_test_case_dir)
 
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
-                    
+
                     huc = test_case.split('_')[0]
                     official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-                    
+
                     for magnitude in ['action', 'minor', 'moderate', 'major']:
                         for version in versions_to_aggregate:
                             if '_fr' in version:
@@ -135,7 +135,7 @@ def create_master_metrics_csv(master_metrics_csv_output):
                                 calibrated = "yes"
                             else:
                                 calibrated = "no"
-                                
+
                             version_dir = os.path.join(official_versions, version)
                             magnitude_dir = os.path.join(version_dir, magnitude)
                             if os.path.exists(magnitude_dir):
@@ -147,8 +147,8 @@ def create_master_metrics_csv(master_metrics_csv_output):
                                         full_json_path = os.path.join(magnitude_dir, f)
                                         flow = ''
                                         if os.path.exists(full_json_path):
-                                            
-                                            # Get flow used to map.                                                
+
+                                            # Get flow used to map.
                                             flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
                                             if os.path.exists(flow_file):
                                                 with open(flow_file, newline='') as csv_file:
@@ -158,7 +158,7 @@ def create_master_metrics_csv(master_metrics_csv_output):
                                                         flow = row[1]
                                                     if nws_lid == 'mcc01':
                                                         print(flow)
-                                            
+
                                             stats_dict = json.load(open(full_json_path))
                                             for metric in metrics_to_write:
                                                 sub_list_to_append.append(stats_dict[metric])
@@ -167,27 +167,27 @@ def create_master_metrics_csv(master_metrics_csv_output):
                                             sub_list_to_append.append(benchmark_source)
                                             sub_list_to_append.append(extent_config)
                                             sub_list_to_append.append(calibrated)
-                                            
+
                                             list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
-        
+
     with open(master_metrics_csv_output, 'w', newline='') as csvfile:
         csv_writer = csv.writer(csvfile)
         csv_writer.writerows(list_to_write)
 
 
 def process_alpha_test(args):
-    
+
     fim_run_dir = args[0]
     version = args[1]
     test_id = args[2]
     magnitude = args[3]
     archive_results = args[4]
     overwrite = args[5]
-    
-    mask_type = 'huc'    
-    
+
+    mask_type = 'huc'
+
     if archive_results == False:
         compare_to_previous = True
     else:
@@ -210,7 +210,7 @@ def process_alpha_test(args):
     parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
     parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
     parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=True)
-    
+
     # Assign variables from arguments.
     args = vars(parser.parse_args())
     config = args['config']
@@ -220,11 +220,11 @@ def process_alpha_test(args):
     benchmark_category = args['benchmark_category']
     overwrite = args['overwrite']
     master_metrics_csv = args['master_metrics_csv']
-    
+
     if overwrite:
         if input("Are you sure you want to overwrite metrics? y/n: ") == "n":
             quit
-        
+
     # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
     if fim_version != "all":
         previous_fim_list = [fim_version]
@@ -233,7 +233,7 @@ def process_alpha_test(args):
             previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)
         elif config == 'DEV':
             previous_fim_list = os.listdir(OUTPUTS_DIR)
-    
+
     # Define whether or not to archive metrics in "official_versions" or "testing_versions" for each test_id.
     if config == 'PREV':
         archive_results = True
@@ -241,7 +241,7 @@ def process_alpha_test(args):
         archive_results = False
     else:
         print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
-     
+
     # List all available benchmark categories and test_cases.
     test_cases_dir_list = os.listdir(TEST_CASES_DIR)
     benchmark_category_list = []
@@ -251,41 +251,41 @@ def process_alpha_test(args):
                 benchmark_category_list.append(d.replace('_test_cases', ''))
     else:
         benchmark_category_list = [benchmark_category]
-        
+
     # Loop through benchmark categories.
     procs_list = []
     for bench_cat in benchmark_category_list:
-        
+
         # Map path to appropriate test_cases folder and list test_ids into bench_cat_id_list.
         bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
         bench_cat_id_list = os.listdir(bench_cat_test_case_dir)
-    
+
         # Loop through test_ids in bench_cat_id_list.
         for test_id in bench_cat_id_list:
             if 'validation' and 'other' not in test_id:
                 current_huc = test_id.split('_')[0]
                 if test_id.split('_')[1] in bench_cat:
-                
+
                     # Loop through versions.
                     for version in previous_fim_list:
                         if config == 'DEV':
                             fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
                         elif config == 'PREV':
                             fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
-                                                
+
                         # For previous versions of HAND computed at HUC6 scale
                         if not os.path.exists(fim_run_dir):
                             if config == 'DEV':
                                 fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
                             elif config == 'PREV':
-                                fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  
-                        
+                                fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])
+
                         if os.path.exists(fim_run_dir):
-                            
+
                             # If a user supplies a specia_string (-s), then add it to the end of the created dirs.
                             if special_string != "":
                                 version = version + '_' + special_string
-                            
+
                             # Define the magnitude lists to use, depending on test_id.
                             if 'ble' in test_id:
                                 magnitude = ['100yr', '500yr']
@@ -293,19 +293,18 @@ def process_alpha_test(args):
                                 magnitude = ['action', 'minor', 'moderate', 'major']
                             else:
                                 continue
-                        
+
                             # Either add to list to multiprocess or process serially, depending on user specification.
                             if job_number > 1:
                                 procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
-                            else:                            
+                            else:
                                 process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
 
     # Multiprocess alpha test runs.
     if job_number > 1:
         pool = Pool(job_number)
         pool.map(process_alpha_test, procs_list)
-    
+
     # Do aggregate_metrics.
     print("Creating master metrics CSV...")
     create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv)
-    
\ No newline at end of file
diff --git a/tools/time_and_tee_mannings_calibration.sh b/tools/time_and_tee_mannings_calibration.sh
index d45976cc8..7a1c06cea 100755
--- a/tools/time_and_tee_mannings_calibration.sh
+++ b/tools/time_and_tee_mannings_calibration.sh
@@ -1,4 +1,4 @@
 #!/bin/bash -e
 
-/usr/bin/time -v $testdir/mannings_run_by_set.sh $1 |& tee
+/usr/bin/time -v $toolsdir/mannings_run_by_set.sh $1 |& tee
 exit ${PIPESTATUS[0]}
diff --git a/tools/utils/shared_functions.py b/tools/tools_shared_functions.py
old mode 100644
new mode 100755
similarity index 100%
rename from tools/utils/shared_functions.py
rename to tools/tools_shared_functions.py
diff --git a/tools/utils/shared_variables.py b/tools/tools_shared_variables.py
old mode 100644
new mode 100755
similarity index 100%
rename from tools/utils/shared_variables.py
rename to tools/tools_shared_variables.py
diff --git a/tools/utils/__init__.py b/tools/utils/__init__.py
deleted file mode 100644
index e69de29bb..000000000

From 18afaefeaa01b1212f131c4826d9669dcd647aa3 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 10 Mar 2021 12:18:15 -0600
Subject: [PATCH 050/359] [1pt] PR: Patch import in tools_shared_functions.py
 (#302)

Minor patch to tools_shared_functions.py where tools_shared_functions.py was not imported correctly.
---
 CHANGELOG.md                    | 8 ++++++++
 tools/tools_shared_functions.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index da0e1cb32..7b9bce771 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.8.1 - 2021-03-10 - [PR #302](https://github.com/NOAA-OWP/cahaba/pull/302)
+
+Patched import issue in `tools_shared_functions.py`.
+
+### Changes
+ - Changed `utils.` to `tools_` in `tools_shared_functions.py` after recent structural change to `tools` directory.
+ 
+<br/><br/>
 ## v3.0.8.0 - 2021-03-09 - [PR #279](https://github.com/NOAA-OWP/cahaba/pull/279)
 
 Refactored NWS Flood Categorical HAND FIM (CatFIM) pipeline to open source.
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index d36c22814..094f5e2a5 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -5,7 +5,7 @@
 import csv
 import rasterio
 import pandas as pd
-from utils.shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
+from tools_shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
                                     ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
 
 def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):

From 4364dc4899fee694e19e2960ce4dc1c537b97a3b Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Thu, 11 Mar 2021 11:36:42 -0600
Subject: [PATCH 051/359] Enhancements to post-processing for Viz-related
 use-cases.

- Aggregate grids are projected to Web Mercator during -v runs in fim_run.sh.
- HUC6 aggregation is parallelized.
- Aggregate grid blocksize is changed from 256 to 1024 for faster postprocessing.

This resolves #294 and resolves #295.
---
 CHANGELOG.md                 |  10 ++
 fim_run.sh                   |   2 +-
 src/aggregate_fim_outputs.py | 206 +++++++++++++++++++++--------------
 3 files changed, 138 insertions(+), 80 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b9bce771..38b9f4980 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.8.2 - 2021-03-11 - [PR #296](https://github.com/NOAA-OWP/cahaba/pull/296)
+
+Enhancements to post-processing for Viz-related use-cases.
+
+### Changes
+ - Aggregate grids are projected to Web Mercator during `-v` runs in `fim_run.sh`.
+ - HUC6 aggregation is parallelized.
+ - Aggregate grid blocksize is changed from 256 to 1024 for faster postprocessing.
+ 
+<br/><br/>
 ## v3.0.8.1 - 2021-03-10 - [PR #302](https://github.com/NOAA-OWP/cahaba/pull/302)
 
 Patched import issue in `tools_shared_functions.py`.
diff --git a/fim_run.sh b/fim_run.sh
index 5acdeff71..42a5d022e 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -152,5 +152,5 @@ fi
 echo "$viz"
 if [[ "$viz" -eq 1 ]]; then
     # aggregate outputs
-    python3 /foss_fim/src/aggregate_fim_outputs.py -d $outputRunDataDir
+    python3 /foss_fim/src/aggregate_fim_outputs.py -d $outputRunDataDir -j 4
 fi
diff --git a/src/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py
index edafd93a3..9d8676364 100644
--- a/src/aggregate_fim_outputs.py
+++ b/src/aggregate_fim_outputs.py
@@ -2,36 +2,37 @@
 
 import os
 import argparse
+from multiprocessing import Pool
 import pandas as pd
 import json
 import rasterio
 from rasterio.merge import merge
+from rasterio.warp import calculate_default_transform, reproject, Resampling
 import shutil
 import csv
-from utils.shared_variables import PREP_PROJECTION
+from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
 
+def aggregate_fim_outputs(args):
 
-def aggregate_fim_outputs(fim_out_dir):
+    fim_out_dir   = args[0]
+    huc6          = args[1]
+    huc_list      = args[2]
 
-    print ("aggregating outputs to HUC6 scale")
+    print(f"aggregating {huc6}")
 
-    drop_folders = ['logs']
-    huc_list = [huc for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
-    huc6_list = [str(huc[0:6]) for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
-    huc6_list = list(set(huc6_list))
+    huc6_dir = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6))
+    os.makedirs(huc6_dir, exist_ok=True)
 
-    for huc in huc_list:
+    # aggregate file name paths
+    aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),'hydroTable.csv')
+    aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),f'rating_curves_{huc6}.json')
 
-        os.makedirs(os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6])), exist_ok=True)
+    for huc in huc_list:
 
         # original file paths
         hydrotable_filename = os.path.join(fim_out_dir,huc,'hydroTable.csv')
         src_filename = os.path.join(fim_out_dir,huc,'src.json')
 
-        # aggregate file name paths
-        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),'hydroTable.csv')
-        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),f'rating_curves_{huc[0:6]}.json')
-
         if len(huc)> 6:
 
             # open hydrotable
@@ -68,107 +69,154 @@ def aggregate_fim_outputs(fim_out_dir):
             shutil.copy(hydrotable_filename, aggregate_hydrotable)
             shutil.copy(src_filename, aggregate_src)
 
-    for huc6 in huc6_list:
-
-        ## add feature_id to aggregate src
-        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),'hydroTable.csv')
-        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),f'rating_curves_{huc6}.json')
-
-        # Open aggregate src for writing feature_ids to
-        src_data = {}
-        with open(aggregate_src) as jsonf:
-            src_data = json.load(jsonf)
-
-        with open(aggregate_hydrotable) as csvf:
-            csvReader = csv.DictReader(csvf)
-
-            for row in csvReader:
-                if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
-                    src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
-
-        # Write src_data to JSON file
-        with open(aggregate_src, 'w') as jsonf:
-            json.dump(src_data, jsonf)
+    ## add feature_id to aggregate src
+    # Open aggregate src for writing feature_ids to
+    src_data = {}
+    with open(aggregate_src) as jsonf:
+        src_data = json.load(jsonf)
 
-        ## aggregate rasters
-        huc6_dir = os.path.join(fim_out_dir,'aggregate_fim_outputs',huc6)
+    with open(aggregate_hydrotable) as csvf:
+        csvReader = csv.DictReader(csvf)
 
-        # aggregate file paths
-        rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}.tif')
-        catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}.tif')
+        for row in csvReader:
+            if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
+                src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
 
-        if huc6 not in huc_list:
+    # Write src_data to JSON file
+    with open(aggregate_src, 'w') as jsonf:
+        json.dump(src_data, jsonf)
 
-            huc6_filter = [path.startswith(huc6) for path in huc_list]
-            subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
+    ## aggregate rasters
+    # aggregate file paths
+    rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}_unprj.tif')
+    catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}_unprj.tif')
 
-            # aggregate and mosaic rem
-            rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
+    if huc6 not in huc_list:
 
-            if len(rem_list) > 1:
+        huc6_filter = [path.startswith(huc6) for path in huc_list]
+        subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
 
-                rem_files_to_mosaic = []
+        # aggregate and mosaic rem
+        rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
 
-                for rem in rem_list:
+        if len(rem_list) > 1:
 
-                    rem_src = rasterio.open(rem)
-                    rem_files_to_mosaic.append(rem_src)
+            rem_files_to_mosaic = []
 
-                mosaic, out_trans = merge(rem_files_to_mosaic)
-                out_meta = rem_src.meta.copy()
-                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
+            for rem in rem_list:
 
-                with rasterio.open(rem_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
-                    dest.write(mosaic)
+                rem_src = rasterio.open(rem)
+                rem_files_to_mosaic.append(rem_src)
 
-                del rem_files_to_mosaic,rem_src,out_meta,mosaic
+            mosaic, out_trans = merge(rem_files_to_mosaic)
+            out_meta = rem_src.meta.copy()
+            out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
 
-            elif len(rem_list)==1:
+            with rasterio.open(rem_mosaic, "w", **out_meta, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES') as dest:
+                dest.write(mosaic)
 
-                shutil.copy(rem_list[0], rem_mosaic)
+            del rem_files_to_mosaic,rem_src,out_meta,mosaic
 
-            # aggregate and mosaic catchments
-            catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
+        elif len(rem_list)==1:
 
-            if len(catchment_list) > 1:
+            shutil.copy(rem_list[0], rem_mosaic)
 
-                cat_files_to_mosaic = []
+        # aggregate and mosaic catchments
+        catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
 
-                for cat in catchment_list:
-                    cat_src = rasterio.open(cat)
-                    cat_files_to_mosaic.append(cat_src)
+        if len(catchment_list) > 1:
 
-                mosaic, out_trans = merge(cat_files_to_mosaic)
-                out_meta = cat_src.meta.copy()
+            cat_files_to_mosaic = []
 
-                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
+            for cat in catchment_list:
+                cat_src = rasterio.open(cat)
+                cat_files_to_mosaic.append(cat_src)
 
-                with rasterio.open(catchment_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
-                    dest.write(mosaic)
+            mosaic, out_trans = merge(cat_files_to_mosaic)
+            out_meta = cat_src.meta.copy()
 
-                del cat_files_to_mosaic,cat_src,out_meta,mosaic
+            out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
 
-            elif len(catchment_list)==1:
+            with rasterio.open(catchment_mosaic, "w", **out_meta, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES') as dest:
+                dest.write(mosaic)
 
-                shutil.copy(catchment_list[0], catchment_mosaic)
+            del cat_files_to_mosaic,cat_src,out_meta,mosaic
 
-        else:
-            # original file paths
-            rem_filename = os.path.join(fim_out_dir,huc6,'rem_zeroed_masked.tif')
-            catchment_filename = os.path.join(fim_out_dir,huc6,'gw_catchments_reaches_filtered_addedAttributes.tif')
+        elif len(catchment_list)==1:
 
-            shutil.copy(rem_filename, rem_mosaic)
-            shutil.copy(catchment_filename, catchment_mosaic)
+            shutil.copy(catchment_list[0], catchment_mosaic)
 
+    else:
+        # original file paths
+        rem_filename = os.path.join(fim_out_dir,huc6,'rem_zeroed_masked.tif')
+        catchment_filename = os.path.join(fim_out_dir,huc6,'gw_catchments_reaches_filtered_addedAttributes.tif')
+
+        shutil.copy(rem_filename, rem_mosaic)
+        shutil.copy(catchment_filename, catchment_mosaic)
+
+    ## reproject rasters
+    reproject_raster(rem_mosaic)
+    os.remove(rem_mosaic)
+
+    reproject_raster(catchment_mosaic)
+    os.remove(catchment_mosaic)
+
+
+def reproject_raster(raster_name):
+
+    with rasterio.open(raster_name) as src:
+        transform, width, height = calculate_default_transform(
+            src.crs, VIZ_PROJECTION, src.width, src.height, *src.bounds)
+        kwargs = src.meta.copy()
+        kwargs.update({
+            'crs': VIZ_PROJECTION,
+            'transform': transform,
+            'width': width,
+            'height': height,
+            'compress': 'lzw'
+        })
+
+        raster_proj_rename = os.path.split(raster_name)[1].replace('_unprj.tif', '.tif')
+        raster_proj_dir = os.path.join(os.path.dirname(raster_name), raster_proj_rename)
+
+        with rasterio.open(raster_proj_dir, 'w', **kwargs, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES') as dst:
+            # for i in range(1, src.count + 1):
+            reproject(
+                source=rasterio.band(src, 1),
+                destination=rasterio.band(dst, 1),
+                src_transform=src.transform,
+                src_crs=src.crs,
+                dst_transform=transform,
+                dst_crs=VIZ_PROJECTION,
+                resampling=Resampling.nearest)
+    del src, dst
 
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Aggregate layers buy HUC6')
     parser.add_argument('-d','--fim-outputs-directory', help='FIM outputs directory', required=True)
+    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
 
 
     args = vars(parser.parse_args())
 
     fim_outputs_directory = args['fim_outputs_directory']
+    number_of_jobs = int(args['number_of_jobs'])
+
+    drop_folders = ['logs']
+    huc_list = [huc for huc in os.listdir(fim_outputs_directory) if huc not in drop_folders]
+    huc6_list = [str(huc[0:6]) for huc in os.listdir(fim_outputs_directory) if huc not in drop_folders]
+    huc6_list = list(set(huc6_list))
+
+
+    procs_list = []
+
+    for huc6 in huc6_list:
+
+        limited_huc_list = [huc for huc in huc_list if huc.startswith(huc6)]
+
+        procs_list.append([fim_outputs_directory,huc6,limited_huc_list])
 
-    aggregate_fim_outputs(fim_outputs_directory)
+    print(f"aggregating {len(huc_list)} hucs to HUC6 scale using {number_of_jobs} jobs")
+    pool = Pool(number_of_jobs)
+    pool.map(aggregate_fim_outputs, procs_list)

From 5fa77f1e33230e1480d0aef1b1feb4411ceaa7a4 Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Fri, 12 Mar 2021 11:02:25 -0600
Subject: [PATCH 052/359] Additional Enhancements for the FIM API

Enhancements to FIM API.

 - fim_run.sh can now be run with jobs in parallel.
 - Viz post-processing can now be selected in API interface.
 - Jobs table shows jobs that end with errors.
 - HUC preset lists can now be selected in interface.
 - Better output_handler file writing.
 - Overall better restart and retry handlers for networking problems.
 - Jobs can now be canceled in API interface.
 - Both FR and MS configs can be selected for a single job.

Resolves issues #231 and #265
---
 CHANGELOG.md                                  |  15 ++
 Dockerfile.dev                                |   3 +-
 api/frontend/gui/templates/index.html         | 113 ++++++--
 api/frontend/output_handler/output_handler.py |  92 ++-----
 api/node/.env-template                        |   3 +-
 api/node/connector/Dockerfile                 |   2 +-
 api/node/connector/connector.py               | 139 +++++-----
 api/node/docker-compose-prod.yml              |   2 +
 api/node/updater/Dockerfile                   |   4 +-
 api/node/updater/updater.py                   | 243 +++++++++++++++---
 10 files changed, 434 insertions(+), 182 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 38b9f4980..177bcfc03 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)
+
+Enhancements to FIM API.
+
+### Changes
+ - `fim_run.sh` can now be run with jobs in parallel.
+ - Viz post-processing can now be selected in API interface.
+ - Jobs table shows jobs that end with errors.
+ - HUC preset lists can now be selected in interface.
+ - Better `output_handler` file writing.
+ - Overall better restart and retry handlers for networking problems.
+ - Jobs can now be canceled in API interface.
+ - Both FR and MS configs can be selected for a single job.
+ 
+<br/><br/>
 ## v3.0.8.2 - 2021-03-11 - [PR #296](https://github.com/NOAA-OWP/cahaba/pull/296)
 
 Enhancements to post-processing for Viz-related use-cases.
diff --git a/Dockerfile.dev b/Dockerfile.dev
index dea61c348..d48d3f0d0 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -71,8 +71,7 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 
-RUN apt update --fix-missing
-RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
+RUN apt update --fix-missing && apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
 
 RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
 
diff --git a/api/frontend/gui/templates/index.html b/api/frontend/gui/templates/index.html
index abccdd7cc..4dd2c2fc3 100644
--- a/api/frontend/gui/templates/index.html
+++ b/api/frontend/gui/templates/index.html
@@ -2,6 +2,7 @@
     <head>
         <title>Cahaba API</title>
 
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.2/css/all.min.css" integrity="sha512-HK5fgLBL+xu6dm/Ii3z4xhlSUyZgTT9tuc/hSrtw6uzJOvgRr2a9jyxxT1ely+B+xFAmJKVSTbpM/CuL7qxO8w==" crossorigin="anonymous" />
         <link rel="preconnect" href="https://fonts.gstatic.com">
         <link href="https://fonts.googleapis.com/css2?family=Roboto&display=swap" rel="stylesheet">
         <style>
@@ -201,6 +202,22 @@
                 font-weight: bold;
                 color: #2fd107;
             }
+            .styled-table tbody tr.job-error {
+                font-weight: bold;
+                color: #d10707;
+            }
+
+            .cancel-button {
+                color: red;
+                cursor: pointer;
+                font-size: 1.5rem;
+            }
+
+            .info-button {
+                color: #286ebe;
+                cursor: pointer;
+                font-size: 1.5rem;
+            }
         </style>
     </head>
     <body>
@@ -216,7 +233,8 @@
                     <h3 style="margin: 0;">Basic</h3>
                     <div class="container-horizontal">
                         <input type="text" id="job-name" class="form-field" placeholder="Job Name"></input>
-                        <input type="text" id="hucs" class="form-field" placeholder="HUC(s)"></input>
+                        <select id="presets-list" class="form-field" onchange="presetListSelected(event)"></select>
+                        <input type="text" id="hucs" class="form-field" placeholder="HUC(s)" style="display: none;"></input>
                         <input type="text" id="git-branch" class="form-field" placeholder="Git Branch"></input>
                     </div>
                     <h3 style="margin: 0;">Configuration</h3>
@@ -233,17 +251,21 @@ <h3 style="margin: 0;">Configuration</h3>
                     <h3 style="margin: 0;">Extent</h3>
                     <div class="container-horizontal">
                         <div class="input-with-label">
-                            <input type="radio" name="extent" value="FR" checked>
+                            <input type="checkbox" name="extent" value="FR" checked>
                             <label for="FR">Full Resolution</label>
                         </div>
                         <div class="input-with-label">
-                            <input type="radio" name="extent" value="MS">
+                            <input type="checkbox" name="extent" value="MS">
                             <label for="MS">Main Stem</label>
                         </div>
                     </div>
                     <div class="input-with-label">
                         <input type="checkbox" name="dev-run" value="dev-run">
-                        <label for="dev-run"> Include all intermediate output files?</label>
+                        <label for="dev-run">Include all intermediate output files?</label>
+                    </div>
+                    <div class="input-with-label">
+                        <input type="checkbox" name="viz-run" value="viz-run" onclick="disableDevRun(event)">
+                        <label for="viz-run">Include Viz Post-Processing?</label>
                     </div>
                     <div class="error-list" id="fim-run-errors">
     
@@ -266,6 +288,8 @@ <h3 style="margin: 0;">Extent</h3>
                         <th>Time Elapsed</th>
                         <th>Status</th>
                         <th>Outputs Saved</th>
+                        <th></th>
+                        <th></th>
                     </tr>
                 </thead>
                 <tbody id="job-list">
@@ -296,27 +320,43 @@ <h3 style="margin: 0;">Extent</h3>
                     }
                 })
 
-                socket.on('client_update', jobs => {
-                    let jobsTableBody = ''
+                socket.on('client_update', ({jobUpdates, presetsList}) => {
+                    if(document.querySelector('#presets-list').innerHTML === '') {
+                        let presetsSelectBody = ''
+                        if (Object.keys(presetsList).length === 0)
+                            document.querySelector('#presets-list').style.display = 'none'
+                        else
+                            document.querySelector('#presets-list').style.display = 'block'
+                        Object.values(presetsList).forEach(p => {
+                            presetsSelectBody += `<option value="${p}">${p}</option>`
+                        })
+                        presetsSelectBody += `<option value="custom">Custom</option>`
+                        document.querySelector('#presets-list').innerHTML = presetsSelectBody
+                    }
+
 
-                    if (Object.keys(jobs).length === 0)
+                    let jobsTableBody = ''
+                    if (jobUpdates.length === 0)
                         document.querySelector('#job-list-wrapper').style.display = 'none'
                     else
                         document.querySelector('#job-list-wrapper').style.display = 'block'
-
-                    Object.values(jobs).forEach(j => {
+                    jobUpdates.forEach(j => {
                         let outputs_saved = "N/A"
-                        if (Object.keys(j['output_files_saved']).length > 0) {
-                            const total = Object.keys(j['output_files_saved']).length
-                            const current =  Object.values(j['output_files_saved']).filter(v => v === true).length
-                            outputs_saved = `${current}/${total}`
+                        if (j['total_output_files_length']) {
+                            outputs_saved = `${j['current_output_files_saved_length']}/${j['total_output_files_length']}`
                         }
 
-                        jobsTableBody += `<tr ${j['status'] === 'Completed' && 'class="job-completed"'}>
+                        statusClass = ''
+                        if (j['status'] === 'Completed') statusClass = 'class="job-completed"'
+                        if (j['status'] === 'Error' || j['exit_code'] !== 0) statusClass = 'class="job-error"'
+
+                        jobsTableBody += `<tr ${statusClass}>
                                             <td>${j['nice_name'].replace(/_/g, ' ')}</td>
-                                            <td>${j['time_elapsed']} Seconds</td>
+                                            <td>${new Date(j['time_elapsed'] * 1000).toISOString().substr(11, 8)}</td>
                                             <td>${j['status']}</td>
                                             <td>${outputs_saved}</td>
+                                            <td><i class="fas fa-info info-button" onclick="getJobInfo('${j['job_name']}')"></i></td>
+                                            <td><i class="fas fa-times cancel-button" onclick="cancelJob('${j['job_name']}')"></i></td>
                                         </tr>`
                     })
                     document.querySelector('#job-list').innerHTML = jobsTableBody
@@ -327,32 +367,40 @@ <h3 style="margin: 0;">Extent</h3>
                     document.querySelector('#fim-run-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
                 })
 
-                socket.on('job_started', job_type => {
+                socket.on('job_added', job_type => {
                     document.querySelector('#request-container').classList.remove('loading')
                     if (job_type === 'fim_run') {
                         document.querySelector('#job-name').value = ''
                         document.querySelector('#hucs').value = ''
                         document.querySelector('#git-branch').value = ''
                         document.querySelector("input[name='dev-run']").checked = false
+                        document.querySelector("input[name='dev-run']").disabled = false
+                        document.querySelector("input[name='viz-run']").checked = false
                         document.querySelector("input[name='configuration'][value='default']").checked = true
                         document.querySelector("input[name='extent'][value='FR']").checked = true
+                        document.querySelector("input[name='extent'][value='MS']").checked = false
                         document.querySelector('#fim-run-errors').innerHTML = ''
                     }
                 })
+                socket.on('job_canceled', () => {
+                    console.log("Job has been cancelled")
+                })
             })
 
             // Fim_run related code
             const submitRequest = requestName => {
                 let validation_errors = []
                 const job_name = document.querySelector('#job-name').value
+                const preset = document.querySelector('#presets-list').value
                 const hucs = document.querySelector('#hucs').value
                 const git_branch = document.querySelector('#git-branch').value
                 const dev_run = document.querySelector("input[name='dev-run']").checked
+                const viz_run = document.querySelector("input[name='viz-run']").checked
                 const configuration =  Array.from(document.querySelectorAll("input[name='configuration']")).filter(c => c.checked)[0].value
-                const extent =  Array.from(document.querySelectorAll("input[name='extent']")).filter(c => c.checked)[0].value
+                const extents =  Array.from(document.querySelectorAll("input[name='extent']")).filter(c => c.checked).map(c => c.value)
 
                 if (job_name === '') validation_errors.push('Job Name Cannot Be Empty')
-                if (hucs === '') validation_errors.push('Huc(s) Cannot Be Empty')
+                if (preset === 'custom' && hucs === '') validation_errors.push('Huc(s) Cannot Be Empty')
                 if (git_branch === '') validation_errors.push('Git Branch Cannot Be Empty')
 
                 if (validation_errors.length > 0){
@@ -364,14 +412,41 @@ <h3 style="margin: 0;">Extent</h3>
                 
                 socket.emit('new_job', {
                     job_name,
+                    preset,
                     hucs,
                     git_branch,
                     dev_run,
+                    viz_run,
                     configuration,
-                    extent
+                    extents
                 })
             }
 
+            const disableDevRun = e => {
+                if (e.target.checked){
+                    document.querySelector("input[name='dev-run']").checked = false
+                    document.querySelector("input[name='dev-run']").disabled = true
+                } else {
+                    document.querySelector("input[name='dev-run']").disabled = false
+                }
+            }
+
+            const presetListSelected = e => {
+                if (e.target.value === 'custom')
+                    document.querySelector('#hucs').style.display = 'flex'
+                else
+                    document.querySelector('#hucs').style.display = 'none'
+            }
+
+            const getJobInfo = jobName => {
+                console.log(jobName)
+            }
+
+            const cancelJob = jobName => {
+                console.log(jobName)
+                socket.emit('cancel_job', {job_name: jobName})
+            }
+
             const selectForm = (e, formName) => {
                 // Close all forms and remove active from all tabs
                 const forms = document.getElementsByClassName('request-form')
diff --git a/api/frontend/output_handler/output_handler.py b/api/frontend/output_handler/output_handler.py
index 1d7de4a88..f665c5669 100644
--- a/api/frontend/output_handler/output_handler.py
+++ b/api/frontend/output_handler/output_handler.py
@@ -5,83 +5,37 @@
 
 SOCKET_URL = os.environ.get('SOCKET_URL')
 
-pending_files = {}
-
 def handle_outputs(data):
-    name = f"{data['job_name']}_{data['file_name']}"
-    if name not in pending_files:
-        pending_files[name] = {
-            'locked': False,
-            'current_index': 0,
-            'nice_name': data['nice_name'],
-            'job_name': data['job_name'],
-            'directory_path': data['directory_path'],
-            'file_name': data['file_name']
-        }
-
-    pending_files[name][data['chunk_index']] = data['file_chunk']
-
-
-    work_to_do = True
-    while work_to_do:
-        work_to_do = False
-
-        nice_name = pending_files[name]['nice_name']
-        job_name = pending_files[name]['job_name']
-        directory_path = pending_files[name]['directory_path']
-        file_name = pending_files[name]['file_name']
-
-        # If the last chunk just got added, waiting to write any potentially missing data to file
-        if data['file_chunk'] == None and pending_files[name]['locked']:
-            while name in pending_files and pending_files[name]['locked']:
-                print("EOF, waiting till not locked")
-                sio.sleep(0.5)
-            if not name in pending_files:
-                return
-
-        # To ensure that the files are being written in the correct order, use current_index
-        # to write the correct file chunk.
-        if not pending_files[name]['locked'] and pending_files[name]['current_index'] in pending_files[name]:
-            pending_files[name]['locked'] = True
-            file_chunk = pending_files[name].pop(pending_files[name]['current_index'])
-
-            # End of file
-            if file_chunk == None:
-                if sio.connected:
-                    sio.emit('output_handler_finished_file', {'job_name': job_name, 'file_path': f"{directory_path}/{file_name}"})
-                    print("finished with file", name, directory_path, file_name)
-                    # files_to_delete.append(name)
-                    pending_files.pop(name)
-                continue
-            else:
-                # Not end of file, keep looping till you can't do more work
-                work_to_do = True
-
-            # Create folder if it doesn't yet exist and set writing mode
-            mode = 'ab'
-            if pending_files[name]['current_index'] == 0:
-                mode = 'wb'
-                try:
-                    os.makedirs(f"/data/outputs/{nice_name}/{directory_path}")
-                except:
-                    pass
-                
-            # Write binary data to file
-            with open(f"/data/outputs/{nice_name}/{directory_path}/{file_name}", mode) as binary_file:
-                print(f"Writing chunk {pending_files[name]['current_index']} for file {directory_path}/{file_name}")
-                binary_file.write(file_chunk)
+    nice_name = data['nice_name']
+    job_name = data['job_name']
+    directory_path = data['directory_path']
+    file_name = data['file_name']
+    file_chunk = data['file_chunk']
+    chunk_index = data['chunk_index']
+
+    # Create folder if it doesn't yet exist and set writing mode
+    mode = 'ab'
+    if chunk_index == 0:
+        mode = 'wb'
+        try:
+            os.makedirs(f"/data/outputs/{nice_name}/{directory_path}")
+        except:
+            pass
+        
+    # Write binary data to file
+    with open(f"/data/outputs/{nice_name}/{directory_path}/{file_name}", mode) as binary_file:
+        print(f"Writing chunk {chunk_index} for file {directory_path}/{file_name}")
+        binary_file.write(file_chunk)
+
+    # TODO: Write an emit that will trigger the next chunk to be sent.
+    sio.emit('output_handler_finished_file_chunk', {'job_name': job_name, 'file_path': f"{directory_path}/{file_name}"})
 
-            # Remove current chunk from list
-            pending_files[name]['current_index'] += 1
-            pending_files[name]['locked'] = False
-    
 sio = socketio.Client()
 
 @sio.event
 def connect():
     print("Output Handler Connected!")
     sio.emit('output_handler_connected')
-    
 
 @sio.event
 def disconnect():
diff --git a/api/node/.env-template b/api/node/.env-template
index 6df26095b..352f2d2ca 100644
--- a/api/node/.env-template
+++ b/api/node/.env-template
@@ -2,4 +2,5 @@ DATA_PATH=
 DOCKER_IMAGE_PATH=
 SOCKET_URL=
 FRONTEND_URL=
-GITHUB_REPO=https://github.com/NOAA-OWP/cahaba.git
\ No newline at end of file
+GITHUB_REPO=https://github.com/NOAA-OWP/cahaba.git
+MAX_ALLOWED_CPU_CORES=
diff --git a/api/node/connector/Dockerfile b/api/node/connector/Dockerfile
index ffdb1581e..091fdb364 100644
--- a/api/node/connector/Dockerfile
+++ b/api/node/connector/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker:19.03.14-dind
+FROM docker:20.10.2-dind
 
 RUN apk add --no-cache python3 python3-dev py3-pip build-base openssl-dev libffi-dev git
 
diff --git a/api/node/connector/connector.py b/api/node/connector/connector.py
index 0148947c2..992f0f0f5 100644
--- a/api/node/connector/connector.py
+++ b/api/node/connector/connector.py
@@ -3,6 +3,7 @@
 
 import os
 import re
+import time
 import random
 import logging
 import subprocess
@@ -40,8 +41,8 @@ def ws_disconn():
     emit('is_connected', False)
 
 @socketio.on('update')
-def ws_update(current_jobs):
-    emit('client_update', current_jobs, broadcast=True)
+def ws_update(data):
+    emit('client_update', data, broadcast=True)
 
 @socketio.on('output_handler_connected')
 def ws_output_handler_connected():
@@ -60,8 +61,7 @@ def ws_ready_for_output_handler(data):
     nice_name = data['nice_name']
     job_name = data['job_name']
     path = data['path']
-
-    print(f"handler_sid: {shared_data['handler_sid']}")
+    chunk_index = data['chunk_index']
 
     if shared_data['handler_sid'] == None:
         print("output handler not connected!")
@@ -73,58 +73,60 @@ def ws_ready_for_output_handler(data):
     directory_path = path_parts.group(1)
     file_name = path_parts.group(2)
 
+    file_read_start = time.time()
     with open(path, "rb") as binary_file:
-        print("Sending to output handler", path)
-        
         # Read and emit file chunk by chunk (50MB at a time)
-        chunk_index = 0
+        binary_file.seek(chunk_index * 52428800)
         file_chunk = binary_file.read(52428800)
-        # file_chunk = binary_file.read(104857600)
-        while file_chunk:
-            print("Sending to output handler", path, "Chunk:", chunk_index)
-            emit('new_job_outputs', {
-                'nice_name': nice_name,
+
+        if len(file_chunk) == 0:
+            print('End of File')
+            emit('file_saved', {
                 'job_name': job_name,
-                'directory_path': directory_path,
-                'file_name': file_name,
-                'file_chunk': file_chunk,
-                'chunk_index': chunk_index
-            }, room=shared_data['handler_sid'])
-
-            chunk_index += 1
-            file_chunk = binary_file.read(52428800)
-            # file_chunk = binary_file.read(104857600)
-
-    # Send None to indicate end of file
-    print("Sending to output handler", path, "Chunk:", chunk_index, "EOF")
-    emit('new_job_outputs', {
-        'nice_name': nice_name,
-        'job_name': job_name,
-        'directory_path': directory_path,
-        'file_name': file_name,
-        'file_chunk': None,
-        'chunk_index': chunk_index
-    }, room=shared_data['handler_sid'])
-
-@socketio.on('output_handler_finished_file')
-def ws_output_handler_finished_file(data):
+                'file_path': path
+            }, room=shared_data['updater_sid'])
+            return
+
+        print("Sending to output handler", path, "Chunk:", chunk_index)
+        emit('new_job_outputs', {
+            'nice_name': nice_name,
+            'job_name': job_name,
+            'directory_path': directory_path,
+            'file_name': file_name,
+            'file_chunk': file_chunk,
+            'chunk_index': chunk_index
+        }, room=shared_data['handler_sid'])
+
+@socketio.on('output_handler_finished_file_chunk')
+def output_handler_finished_file_chunk(data):
     job_name = data['job_name']
     file_path = data['file_path']
 
-    print('done saving', job_name, file_path)
-    emit('file_saved', {
+    print('done saving chunk', job_name, file_path)
+    emit('file_chunk_saved', {
         'job_name': job_name,
-        'file_path': f"/data/outputs/{job_name}/{file_path}"
+        'file_path': f"/data/outputs/{job_name}/{file_path}",
     }, room=shared_data['updater_sid'])
 
 @socketio.on('new_job')
 def ws_new_job(job_params):
     validation_errors = []
 
+    # Get Preset Option
+    preset = job_params['preset']
+
     # Validate Hucs Name Option
-    hucs = ' '.join(job_params['hucs'].replace(',', ' ').split())
-    invalid_hucs = re.search('[a-zA-Z]', hucs)
-    if invalid_hucs: validation_errors.append('Invalid Huc(s)')
+    if preset == 'custom':
+        hucs_raw = job_params['hucs'].replace(',', ' ').split()
+        parallel_jobs = len(hucs_raw)
+        hucs_type = len(hucs_raw[0])
+        hucs = ' '.join(hucs_raw)
+        invalid_hucs = re.search('[^0-9 ]', hucs)
+        if invalid_hucs: validation_errors.append('Invalid Huc(s)')
+    else:
+        hucs = f"/data/inputs/huc_lists/{preset}"
+        parallel_jobs = 0
+        hucs_type = 0
 
     # Validate Git Branch Option
     branch = ''
@@ -132,14 +134,14 @@ def ws_new_job(job_params):
     if branch_exists: branch = job_params['git_branch'].replace(' ', '_')
     else: validation_errors.append('Git Branch Does Not Exist')
 
-    # Validate Job Name Option
-    job_name = f"apijob_{job_params['job_name'].replace(' ', '_')[0:50]}_apijob_{branch}_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
-
     # Validate Extent Option
-    extent = ''
-    if job_params['extent'] == 'FR': extent = 'FR'
-    elif job_params['extent'] == 'MS': extent = 'MS'
-    else: validation_errors.append('Invalid Extent Option')
+    valid_extents = ['FR', 'MS']
+    extents = []
+    for extent in job_params['extents']:
+        if extent in valid_extents:
+            extents.append(extent)
+        else:
+            validation_errors.append('Invalid Extent Option')
 
     # Validate Configuration Option
     config_path = ''
@@ -151,19 +153,40 @@ def ws_new_job(job_params):
     if job_params['dev_run'] : dev_run = True
     else: dev_run = False
 
-    if len(validation_errors) == 0:
-        # Clone github repo, with specific branch, to a temp folder
-        print(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}')
-        subprocess.call(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}', shell=True)
-
-        # TODO: instead of starting the job right away, add it to a queue until there are enough resources to run it. Also track things like huc count and huc type (6 or 8)
+    # Validate Viz Run Option
+    if job_params['viz_run'] : viz_run = True
+    else: viz_run = False
 
-        # Kick off the new job as a docker container with the new cloned repo as the volume
-        print(f"docker run -d --rm --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'}")
-        subprocess.call(f"docker run -d --rm --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'}", shell=True)
-        emit('job_started', 'fim_run')
+    if len(validation_errors) == 0:
+        for extent in extents:
+            # Validate Job Name Option
+            job_name = f"apijob_{job_params['job_name'].replace(' ', '_')[0:50]}_{extent}_apijob_{branch}_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
+            print(f"adding job {job_name} {branch} {preset} {hucs} {parallel_jobs} {hucs_type} {extent} {config_path} {dev_run} {viz_run}")
+            emit('add_job_to_queue', {
+                'job_name': job_name,
+                'branch': branch,
+                'hucs': hucs,
+                'parallel_jobs': parallel_jobs,
+                'hucs_type': hucs_type,
+                'extent': extent,
+                'config_path': config_path,
+                'dev_run': dev_run,
+                'viz_run': viz_run,
+            }, room=shared_data['updater_sid'])
+            print('job added')
+            emit('job_added', 'fim_run')
     else:
         emit('validation_errors', validation_errors)
+    
+    @socketio.on('cancel_job')
+    def ws_cancel_job(job_params):
+        # Validate Job Name Option
+        job_name = job_params['job_name']
+
+        emit('remove_job_from_queue', {'job_name': job_name}, room=shared_data['updater_sid'])
+        print('job canceled')
+        emit('job_canceled', 'fim_run')
+   
 
 if __name__ == '__main__':
     socketio.run(app, host="0.0.0.0", port="6000")
diff --git a/api/node/docker-compose-prod.yml b/api/node/docker-compose-prod.yml
index 3e1ee654a..f9787ab59 100644
--- a/api/node/docker-compose-prod.yml
+++ b/api/node/docker-compose-prod.yml
@@ -22,6 +22,8 @@ services:
     build:
       context: ./updater
     container_name: fim_node_updater
+    env_file:
+      - .env
     restart: always
     depends_on: 
       - fim_node_connector
diff --git a/api/node/updater/Dockerfile b/api/node/updater/Dockerfile
index e179f3a42..d62a77652 100644
--- a/api/node/updater/Dockerfile
+++ b/api/node/updater/Dockerfile
@@ -1,6 +1,6 @@
-FROM docker:19.03.14-dind
+FROM docker:20.10.2-dind
 
-RUN apk add --no-cache python3 python3-dev py3-pip
+RUN apk add --no-cache python3 python3-dev py3-pip build-base openssl-dev libffi-dev git
 
 ENV PYTHONUNBUFFERED 1
 RUN mkdir -p /opt/updater
diff --git a/api/node/updater/updater.py b/api/node/updater/updater.py
index 7bc65836c..ea162f50d 100644
--- a/api/node/updater/updater.py
+++ b/api/node/updater/updater.py
@@ -5,42 +5,78 @@
 import json
 import shutil
 import logging
+import subprocess
 
 import socketio
 
 DATA_PATH = os.environ.get('DATA_PATH')
+DOCKER_IMAGE_PATH = os.environ.get('DOCKER_IMAGE_PATH')
+GITHUB_REPO = os.environ.get('GITHUB_REPO')
+MAX_ALLOWED_CPU_CORES = int(os.environ.get('MAX_ALLOWED_CPU_CORES'))
 
-connected = False
 shared_data = {
-    'connected': False
+    'connected': False,
+    'current_saving_job': ''
 }
 
+buffer_jobs = []
+buffer_remove_jobs = []
 current_jobs = {}
 if os.path.exists('/data/outputs/current_jobs.json'):
     with open('/data/outputs/current_jobs.json') as f:
         current_jobs = json.load(f)
+        for job_name in current_jobs.keys():
+            if 'is_actively_saving' in current_jobs[job_name] and current_jobs[job_name]['is_actively_saving'] == True:
+                shared_data['current_saving_job'] = current_jobs[job_name]
+
 
 # Get all the current running jobs from the list of docker containers, store that data in a dictionary
 # along with any other needed metadata (like if it's still running, doing post processing, copying outputs
 # to its destination, etc), and then update the websocket server of the status of the jobs.
 def update_loop():
     while True:
+        # If there are no current jobs, just check every 10 seconds till there is
+        if len(current_jobs.keys()) == 0: sio.sleep(10)
+
+        while len(buffer_jobs) > 0:
+            new_job = buffer_jobs.pop()
+            current_jobs[new_job['job_name']] = new_job
+
+        while len(buffer_remove_jobs) > 0:
+            job_to_remove = buffer_remove_jobs.pop()
+            current_jobs[job_to_remove['job_name']]['status'] = 'Cancelled'
+
         # Get list of current docker containers that are fim run jobs
-        job_names = os.popen("docker container ls --filter=name=apijob --format '{{.Names}}'").read().splitlines()
-        for job_name in job_names:
-            if job_name not in current_jobs:
-                # If it's a new job, add it to the dictionary
-                current_jobs[job_name] = {
-                    'job_name': job_name,
-                    'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
-                    'status': 'In Progress',
-                    'time_started': time.time(),
-                    'time_elapsed': 0,
-                    'output_files_saved': {}
-                }
+                                #    docker ps --all --filter=name=apijob --format='{{.Names}} {{.State}}'
+        containers_raw = os.popen("docker ps --all --filter=name=apijob --format='{{.Names}} {{.State}}'").read().splitlines()
+        containers_split = [ line.split() for line in containers_raw ]
+        container_states = { name: state for (name, state) in containers_split }
 
         jobs_to_delete = []
         for job_name in current_jobs.keys():
+            sio.sleep(0)
+            if job_name in container_states:
+                current_jobs[job_name]['container_state'] = container_states[job_name]
+
+            # If the user chooses to cancel the job early
+            if current_jobs[job_name]['status'] == 'Cancelled':
+                # If the docker container is running, stop and remove it
+                if current_jobs[job_name]['time_elapsed'] > 0 and current_jobs[job_name]['container_state'] != 'exited':
+                    subprocess.call(f"docker container stop {job_name}", shell=True)
+                    subprocess.call(f"docker container rm {job_name}", shell=True)
+
+                print("output_handler finished, deleted temp source files and output files")
+                temp_path = f"/data/temp/{job_name}"
+                if os.path.isdir(temp_path):
+                    shutil.rmtree(temp_path)
+
+                outputs_path = f"/data/outputs/{job_name}"
+                if os.path.isdir(outputs_path):
+                    shutil.rmtree(outputs_path)
+                
+                jobs_to_delete.append(job_name)
+
+
             # Update the time elapsed for all jobs that are currently in progress or saving outputs
             if current_jobs[job_name]['status'] == 'In Progress' or current_jobs[job_name]['status'] == 'Ready to Save File'\
                 or current_jobs[job_name]['status'] == 'Saving File':
@@ -48,30 +84,96 @@ def update_loop():
 
             # TODO: While job is in progress, keep track of how many hucs are done and overall progress %
 
+            # Once resources recome available, start a new job that is in queue
+            if current_jobs[job_name]['status'] == 'In Queue':
+                # TODO: Start Dcoker containers here and set start time.
+                current_jobs[job_name]['time_started'] = time.time()
+
+                total_active_cores = 0
+                for j in current_jobs.keys():
+                    if current_jobs[j]['status'] == 'In Progress':
+                        # This is to account for the fact that HUC6's take a lot more resources to run.
+                        # (not necessarily cpu cores but rather RAM, so this artificially reduces how many jobs can run when HUC6's
+                        #  are running)
+                        # HACK: this is more of a temporary solution until we no longer need to run HUC6's
+                        if current_jobs[j]['hucs_type'] == '6':
+                            total_active_cores += current_jobs[j]['parallel_jobs'] * 5
+                        else:
+                            total_active_cores += current_jobs[j]['parallel_jobs']
+                
+                # Machine has enough resources to run a new job
+                potential_active_cores = 0
+                if current_jobs[job_name]['hucs_type'] == '6':
+                    potential_active_cores = current_jobs[job_name]['parallel_jobs'] * 5 + total_active_cores
+                else:
+                    potential_active_cores = current_jobs[job_name]['parallel_jobs'] + total_active_cores
+
+                # print(f"Checking whether a new job can start {potential_active_cores} <= {MAX_ALLOWED_CPU_CORES}")
+                # print(potential_active_cores <= MAX_ALLOWED_CPU_CORES)
+                if potential_active_cores <= MAX_ALLOWED_CPU_CORES:
+                    job_name = current_jobs[job_name]['job_name']
+                    branch = current_jobs[job_name]['branch']
+                    hucs = current_jobs[job_name]['hucs']
+                    parallel_jobs = current_jobs[job_name]['parallel_jobs']
+                    extent = current_jobs[job_name]['extent']
+                    config_path = current_jobs[job_name]['config_path']
+                    dev_run = current_jobs[job_name]['dev_run']
+                    viz_run = current_jobs[job_name]['viz_run']
+
+                    # Clone github repo, with specific branch, to a temp folder
+                    print(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}')
+                    subprocess.call(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}', shell=True)
+
+                    # Kick off the new job as a docker container with the new cloned repo as the volume
+                    print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}")
+                    subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}", shell=True)
+                    current_jobs[job_name]['status'] = 'In Progress'
+
             # Once the Docker container is done, set the job as ready to save output
-            if current_jobs[job_name]['status'] == 'In Progress' and job_name not in job_names:
+            if current_jobs[job_name]['status'] == 'In Progress' and current_jobs[job_name]['container_state'] == 'exited':
+
+                # Get container exit code, get the docker log, and then remove container
+                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
+
+                print("Exit code")
+                print(exit_code_raw)
+                print(exit_code_raw[0])
+                try:
+                    print(int(exit_code_raw[0]))
+                except:
+                    pass
+
+                exit_code = int(exit_code_raw[0])
+                current_jobs[job_name]['exit_code'] = exit_code
+                subprocess.call(f"docker logs {job_name} >& /data/outputs/{job_name}/logs/docker.log", shell=True)
+                subprocess.call(f"docker container rm {job_name}", shell=True)
+
                 for path, folders, files in os.walk(f"/data/outputs/{job_name}"):
                     for file in files:
-                        current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = False
+                        current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = 0
 
+                current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
                 current_jobs[job_name]['status'] = 'Ready to Save File'
-                # TODO: Possible check the completed job's log for its exit code
     
             # Trigger connector to transmit the outputs to the output_handler
             # If the output_handler is offline, it will keep retrying until the output_handler is online
-            if current_jobs[job_name]['status'] == 'Ready to Save File':
+            if current_jobs[job_name]['status'] == 'Ready to Save File' and (shared_data['current_saving_job'] == '' or shared_data['current_saving_job'] == current_jobs[job_name]):
                 print(f"{job_name} ready for output handler")
-                outputs_to_save = []
+
+                shared_data['current_saving_job'] = current_jobs[job_name]
+                current_jobs[job_name]['is_actively_saving'] = True
+                output_to_save = {}
                 for path in current_jobs[job_name]['output_files_saved']:
-                    if current_jobs[job_name]['output_files_saved'][path] == False:
-                        outputs_to_save.append(path)
+                    if current_jobs[job_name]['output_files_saved'][path] != -1:
+                        output_to_save = {'path': path, 'chunk_index': current_jobs[job_name]['output_files_saved'][path]}
 
-                if len(outputs_to_save) > 0:
+                if output_to_save != {}:
                     if shared_data['connected']: 
                         sio.emit('ready_for_output_handler', {
                             'nice_name': current_jobs[job_name]['nice_name'],
                             'job_name': job_name,
-                            'path': outputs_to_save[0]
+                            'path': output_to_save['path'],
+                            'chunk_index': output_to_save['chunk_index']
                         })
                 current_jobs[job_name]['status'] = 'Saving File'
             
@@ -80,7 +182,7 @@ def update_loop():
             if current_jobs[job_name]['status'] == 'Saving File':
                 is_done = True
                 for path in current_jobs[job_name]['output_files_saved']:
-                    if current_jobs[job_name]['output_files_saved'][path] == False:
+                    if current_jobs[job_name]['output_files_saved'][path] != -1:
                         is_done = False
                         break
 
@@ -94,12 +196,14 @@ def update_loop():
                     if os.path.isdir(outputs_path):
                         shutil.rmtree(outputs_path)
 
-                    current_jobs[job_name]['status'] = 'Completed'
+                    current_jobs[job_name]['status'] = 'Completed' if current_jobs[job_name]['exit_code'] == 0 else 'Error'
+                    shared_data['current_saving_job'] = ''
+                    current_jobs[job_name]['is_actively_saving'] = False
                     print(f"{job_name} completed")
                     # TODO: Insert Slack notification here for finished job
 
             # Remove job from list after it's been completed for more than 15 minutes
-            if current_jobs[job_name]['status'] == 'Completed' and \
+            if (current_jobs[job_name]['status'] == 'Completed' or current_jobs[job_name]['status'] == 'Error') and \
                 time.time() >= current_jobs[job_name]['time_started'] + current_jobs[job_name]['time_elapsed'] + 900:
                 print(f"{job_name} removed from job list")
                 jobs_to_delete.append(job_name)
@@ -107,13 +211,27 @@ def update_loop():
         for job in jobs_to_delete:
             del current_jobs[job]
 
+        # TODO: Get Preset Lists here
+        presets_list = []
+        for path, folders, files in os.walk(f"/data/inputs/huc_lists"):
+            for file in files:
+                presets_list.append(file)
+
         # Send updates to the connector and write job progress to file
-        if shared_data['connected']: sio.emit('update', current_jobs)
+        job_updates = [ {
+            'job_name': job['job_name'],
+            'nice_name': job['nice_name'],
+            'status': job['status'],
+            'exit_code': job['exit_code'],
+            'time_elapsed': job['time_elapsed'],
+            'total_output_files_length': job['total_output_files_length'],
+            'current_output_files_saved_length': job['current_output_files_saved_length'],
+        } for job in current_jobs.values()]
+
+        if shared_data['connected']: sio.emit('update', {'jobUpdates': job_updates, 'presetsList': presets_list})
         with open('/data/outputs/current_jobs.json', 'w') as f:
             json.dump(current_jobs, f)
 
-        time.sleep(1)
-
 sio = socketio.Client()
 
 @sio.event
@@ -127,20 +245,85 @@ def disconnect():
     print('disconnected from server')
     shared_data['connected'] = False
 
+@sio.on('add_job_to_queue')
+def ws_add_job_to_queue(data):
+    job_name = data['job_name']
+    branch = data['branch']
+    hucs = data['hucs']
+    parallel_jobs = data['parallel_jobs']
+    hucs_type = data['hucs_type']
+    extent = data['extent']
+    config_path = data['config_path']
+    dev_run = data['dev_run']
+    viz_run = data['viz_run']
+
+    # This is a preset list instead of a custom list of hucs
+    if hucs_type == 0:
+        if os.path.exists(hucs):
+            with open(hucs, "r") as preset_file:
+                hucs_raw = preset_file.read().splitlines()
+                parallel_jobs = len(hucs_raw)
+                hucs_type = len(hucs_raw[0])
+    print(f"{hucs} {parallel_jobs} {hucs_type}")
+
+
+    parallel_jobs = parallel_jobs if parallel_jobs <= MAX_ALLOWED_CPU_CORES else MAX_ALLOWED_CPU_CORES
+
+    buffer_jobs.append({
+        'job_name': job_name,
+        'branch': branch,
+        'hucs': hucs,
+        'parallel_jobs': parallel_jobs,
+        'hucs_type': hucs_type,
+        'extent': extent,
+        'config_path': config_path,
+        'dev_run': dev_run,
+        'viz_run': viz_run,
+        'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
+        'status': 'In Queue',
+        'time_started': 0,
+        'time_elapsed': 0,
+        'output_files_saved': {},
+        'total_output_files_length': 0,
+        'current_output_files_saved_length': 0,
+        'output_files_saved': {},
+        'container_state': 'running',
+        'exit_code': 0,
+        'is_actively_saving': False
+    })
+
+@sio.on('remove_job_from_queue')
+def ws_remove_job_from_queue(data):
+    job_name = data['job_name']
+    buffer_remove_jobs.append({'job_name': job_name})
+
 # If the output_handler is offline, try the saving process again
 @sio.on('retry_saving_files')
 def ws_retry_saving_files():
     print('saving files failed, retrying')
     for job_name in current_jobs:
         if current_jobs[job_name]['status'] == "Saving File":
+            for path in current_jobs[job_name]['output_files_saved']:
+                if current_jobs[job_name]['output_files_saved'][path] != -1:
+                    current_jobs[job_name]['output_files_saved'][path] = 0
+
             current_jobs[job_name]['status'] = 'Ready to Save File'
 
+@sio.on('file_chunk_saved')
+def ws_file_chunk_saved(data):
+    job_name = data['job_name']
+    file_path = data['file_path']
+
+    current_jobs[job_name]['output_files_saved'][file_path] += 1
+    current_jobs[job_name]['status'] = 'Ready to Save File'
+
 @sio.on('file_saved')
 def ws_file_saved(data):
     job_name = data['job_name']
     file_path = data['file_path']
 
-    current_jobs[job_name]['output_files_saved'][file_path] = True
+    current_jobs[job_name]['output_files_saved'][file_path] = -1
+    current_jobs[job_name]['current_output_files_saved_length'] += 1
     current_jobs[job_name]['status'] = 'Ready to Save File'
 
 sio.connect('http://fim_node_connector:6000/')

From 665f53488432ac598f0b4142228843dcb05cc81f Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Fri, 12 Mar 2021 14:24:40 -0600
Subject: [PATCH 053/359] Preprocessing of flow files for Categorical FIM.

- Generate Categorical FIM flow files for each category (action, minor, moderate, major).
- Generate point shapefile of Categorical FIM sites.
- Generate csv of attribute data in shapefile.
- Aggregate all shapefiles and csv files into one file in parent directory.
- Add flood of record category.
- Stability fixes to generate_categorical_fim.py.

This resolves #243 and resolves #274.
---
 .gitignore                              |   3 +-
 CHANGELOG.md                            |  13 +
 Dockerfile.dev                          |   2 +-
 Pipfile                                 |   3 +
 Pipfile.lock                            | 393 +++++++++--------
 src/add_crosswalk.py                    |   4 +-
 tools/.env.template                     |   3 +
 tools/.gitignore                        |   1 +
 tools/generate_categorical_fim.py       |  34 +-
 tools/generate_categorical_fim_flows.py | 236 ++++++++++
 tools/tools_shared_functions.py         | 545 ++++++++++++++++++++----
 11 files changed, 951 insertions(+), 286 deletions(-)
 mode change 100644 => 100755 Pipfile
 create mode 100644 tools/.env.template
 create mode 100644 tools/.gitignore
 create mode 100755 tools/generate_categorical_fim_flows.py

diff --git a/.gitignore b/.gitignore
index 54bcfb6a6..e18bb6432 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,5 @@ config/**
 !config/symbology/
 .vscode/
 **/.DS_Store
-**/*_pytest.py
\ No newline at end of file
+**/*_pytest.py
+
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 177bcfc03..5601ba729 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
+## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
+
+ Preprocessing of flow files for Categorical FIM.
+ 
+### Additions
+ - Generate Categorical FIM flow files for each category (action, minor, moderate, major).
+ - Generate point shapefile of Categorical FIM sites.
+ - Generate csv of attribute data in shapefile.
+ - Aggregate all shapefiles and csv files into one file in parent directory.
+ - Add flood of record category.
+ ### Changes
+ - Stability fixes to `generate_categorical_fim.py`.
+<br/><br/>
 
 ## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index d48d3f0d0..638ab46d6 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/*
 RUN git clone https://github.com/dtarb/taudem.git
 RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
 
-RUN apt-get update && apt-get install -y cmake mpich \
+RUN apt-get update --fix-missing && apt-get install -y cmake mpich \
     libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/* 
 
 ## Compile Main taudem repo ##
diff --git a/Pipfile b/Pipfile
old mode 100644
new mode 100755
index c8fbfc0f6..219f69ba2
--- a/Pipfile
+++ b/Pipfile
@@ -7,6 +7,7 @@ verify_ssl = true
 ipython = "*"
 
 [packages]
+fiona = "==1.8.17"
 geopandas = "==0.8.1"
 numba = "==0.50.1"
 pandas = "==1.0.5"
@@ -18,6 +19,8 @@ tqdm = "==4.48.0"
 Shapely = "==1.7.0"
 grass-session = "==0.5"
 seaborn = "==0.11.0"
+python-dotenv = "*"
+natsort = "*"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 5de3d9856..1250dcc4b 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "5e7e798bab631b1e9012e79e84031f1b4852178692adc10bb8a071c9994c9d56"
+            "sha256": "f74c6d08c7c900c6cea86a5ea4661791356aac573ed738c9c3a1405303a4200c"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -25,18 +25,18 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594",
-                "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.2.0"
+            "version": "==20.3.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
-                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
             ],
-            "version": "==2020.6.20"
+            "version": "==2020.12.5"
         },
         "click": {
             "hashes": [
@@ -55,12 +55,11 @@
         },
         "cligj": {
             "hashes": [
-                "sha256:2bf2042a81be581d707f726aef5efbbd935a62af85d5521305026dabeb798f5d",
-                "sha256:394a0905fe6f36821b82f086bf8cc12fef20d99d0a3c26a8a92a9207a18b70c6",
-                "sha256:9881e3b71ff450a83412fadee026347ca15b99c623b9485593da4929cf884ca9"
+                "sha256:07171c1e287f45511f97df4ea071abc5d19924153413d5683a8e4866369bc676",
+                "sha256:b2f1f7247d59a5387bd3013a08b9ed6829e96fafa4a6e6292341efdb46fe6220"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
-            "version": "==0.7.0"
+            "version": "==0.7.1"
         },
         "cycler": {
             "hashes": [
@@ -82,6 +81,7 @@
                 "sha256:d38a6ef59087b5a20ad7298608c5392e37705ff14d27b44435e04072bbf6632c",
                 "sha256:fcfd8b67403de9b1cc53c045c72542e9f30cb15e617c89c41b928046a9b27daa"
             ],
+            "index": "pypi",
             "version": "==1.8.17"
         },
         "geopandas": {
@@ -102,28 +102,41 @@
         },
         "kiwisolver": {
             "hashes": [
-                "sha256:03662cbd3e6729f341a97dd2690b271e51a67a68322affab12a5b011344b973c",
-                "sha256:18d749f3e56c0480dccd1714230da0f328e6e4accf188dd4e6884bdd06bf02dd",
-                "sha256:247800260cd38160c362d211dcaf4ed0f7816afb5efe56544748b21d6ad6d17f",
-                "sha256:38d05c9ecb24eee1246391820ed7137ac42a50209c203c908154782fced90e44",
-                "sha256:443c2320520eda0a5b930b2725b26f6175ca4453c61f739fef7a5847bd262f74",
-                "sha256:4eadb361baf3069f278b055e3bb53fa189cea2fd02cb2c353b7a99ebb4477ef1",
-                "sha256:556da0a5f60f6486ec4969abbc1dd83cf9b5c2deadc8288508e55c0f5f87d29c",
-                "sha256:603162139684ee56bcd57acc74035fceed7dd8d732f38c0959c8bd157f913fec",
-                "sha256:60a78858580761fe611d22127868f3dc9f98871e6fdf0a15cc4203ed9ba6179b",
-                "sha256:63f55f490b958b6299e4e5bdac66ac988c3d11b7fafa522800359075d4fa56d1",
-                "sha256:7cc095a4661bdd8a5742aaf7c10ea9fac142d76ff1770a0f84394038126d8fc7",
-                "sha256:be046da49fbc3aa9491cc7296db7e8d27bcf0c3d5d1a40259c10471b014e4e0c",
-                "sha256:c31bc3c8e903d60a1ea31a754c72559398d91b5929fcb329b1c3a3d3f6e72113",
-                "sha256:c955791d80e464da3b471ab41eb65cf5a40c15ce9b001fdc5bbc241170de58ec",
-                "sha256:d069ef4b20b1e6b19f790d00097a5d5d2c50871b66d10075dab78938dc2ee2cf",
-                "sha256:d52b989dc23cdaa92582ceb4af8d5bcc94d74b2c3e64cd6785558ec6a879793e",
-                "sha256:e586b28354d7b6584d8973656a7954b1c69c93f708c0c07b77884f91640b7657",
-                "sha256:efcf3397ae1e3c3a4a0a0636542bcad5adad3b1dd3e8e629d0b6e201347176c8",
-                "sha256:fccefc0d36a38c57b7bd233a9b485e2f1eb71903ca7ad7adacad6c28a56d62d2"
+                "sha256:0cd53f403202159b44528498de18f9285b04482bab2a6fc3f5dd8dbb9352e30d",
+                "sha256:1e1bc12fb773a7b2ffdeb8380609f4f8064777877b2225dec3da711b421fda31",
+                "sha256:225e2e18f271e0ed8157d7f4518ffbf99b9450fca398d561eb5c4a87d0986dd9",
+                "sha256:232c9e11fd7ac3a470d65cd67e4359eee155ec57e822e5220322d7b2ac84fbf0",
+                "sha256:31dfd2ac56edc0ff9ac295193eeaea1c0c923c0355bf948fbd99ed6018010b72",
+                "sha256:33449715e0101e4d34f64990352bce4095c8bf13bed1b390773fc0a7295967b3",
+                "sha256:401a2e9afa8588589775fe34fc22d918ae839aaaf0c0e96441c0fdbce6d8ebe6",
+                "sha256:44a62e24d9b01ba94ae7a4a6c3fb215dc4af1dde817e7498d901e229aaf50e4e",
+                "sha256:50af681a36b2a1dee1d3c169ade9fdc59207d3c31e522519181e12f1b3ba7000",
+                "sha256:563c649cfdef27d081c84e72a03b48ea9408c16657500c312575ae9d9f7bc1c3",
+                "sha256:5989db3b3b34b76c09253deeaf7fbc2707616f130e166996606c284395da3f18",
+                "sha256:5a7a7dbff17e66fac9142ae2ecafb719393aaee6a3768c9de2fd425c63b53e21",
+                "sha256:5c3e6455341008a054cccee8c5d24481bcfe1acdbc9add30aa95798e95c65621",
+                "sha256:5f6ccd3dd0b9739edcf407514016108e2280769c73a85b9e59aa390046dbf08b",
+                "sha256:72c99e39d005b793fb7d3d4e660aed6b6281b502e8c1eaf8ee8346023c8e03bc",
+                "sha256:78751b33595f7f9511952e7e60ce858c6d64db2e062afb325985ddbd34b5c131",
+                "sha256:834ee27348c4aefc20b479335fd422a2c69db55f7d9ab61721ac8cd83eb78882",
+                "sha256:8be8d84b7d4f2ba4ffff3665bcd0211318aa632395a1a41553250484a871d454",
+                "sha256:950a199911a8d94683a6b10321f9345d5a3a8433ec58b217ace979e18f16e248",
+                "sha256:a357fd4f15ee49b4a98b44ec23a34a95f1e00292a139d6015c11f55774ef10de",
+                "sha256:a53d27d0c2a0ebd07e395e56a1fbdf75ffedc4a05943daf472af163413ce9598",
+                "sha256:acef3d59d47dd85ecf909c359d0fd2c81ed33bdff70216d3956b463e12c38a54",
+                "sha256:b38694dcdac990a743aa654037ff1188c7a9801ac3ccc548d3341014bc5ca278",
+                "sha256:b9edd0110a77fc321ab090aaa1cfcaba1d8499850a12848b81be2222eab648f6",
+                "sha256:c08e95114951dc2090c4a630c2385bef681cacf12636fb0241accdc6b303fd81",
+                "sha256:c5518d51a0735b1e6cee1fdce66359f8d2b59c3ca85dc2b0813a8aa86818a030",
+                "sha256:c8fd0f1ae9d92b42854b2979024d7597685ce4ada367172ed7c09edf2cef9cb8",
+                "sha256:ca3820eb7f7faf7f0aa88de0e54681bddcb46e485beb844fcecbcd1c8bd01689",
+                "sha256:cf8b574c7b9aa060c62116d4181f3a1a4e821b2ec5cbfe3775809474113748d4",
+                "sha256:d3155d828dec1d43283bd24d3d3e0d9c7c350cdfcc0bd06c0ad1209c1bbc36d0",
+                "sha256:f8d6f8db88049a699817fd9178782867bf22283e3813064302ac59f61d95be05",
+                "sha256:fd34fbbfbc40628200730bc1febe30631347103fc8d3d4fa012c21ab9c11eca9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==1.2.0"
+            "version": "==1.3.1"
         },
         "llvmlite": {
             "hashes": [
@@ -149,27 +162,34 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:06866c138d81a593b535d037b2727bec9b0818cadfe6a81f6ec5715b8dd38a89",
-                "sha256:16b241c3d17be786966495229714de37de04472da472277869b8d5b456a8df00",
-                "sha256:27f9de4784ae6fb97679556c5542cf36c0751dccb4d6407f7c62517fa2078868",
-                "sha256:2f5eefc17dc2a71318d5a3496313be5c351c0731e8c4c6182c9ac3782cfc4076",
-                "sha256:371518c769d84af8ec9b7dcb871ac44f7a67ef126dd3a15c88c25458e6b6d205",
-                "sha256:3d2edbf59367f03cd9daf42939ca06383a7d7803e3993eb5ff1bee8e8a3fbb6b",
-                "sha256:3fb0409754b26f48045bacd6818e44e38ca9338089f8ba689e2f9344ff2847c7",
-                "sha256:548cfe81476dbac44db96e9c0b074b6fb333b4d1f12b1ae68dbed47e45166384",
-                "sha256:57be9e21073fc367237b03ecac0d9e4b8ddbe38e86ec4a316857d8d93ac9286c",
-                "sha256:5ccecb5f78b51b885f0028b646786889f49c54883e554fca41a2a05998063f23",
-                "sha256:69cf76d673682140f46c6cb5e073332c1f1b2853c748dc1cb04f7d00023567f7",
-                "sha256:793e061054662aa27acaff9201cdd510a698541c6e8659eeceb31d66c16facc6",
-                "sha256:799c421bc245a0749c1515b6dea6dc02db0a8c1f42446a0f03b3b82a60a900dc",
-                "sha256:8bc1d3284dee001f41ec98f59675f4d723683e1cc082830b440b5f081d8e0ade",
-                "sha256:a522de31e07ed7d6f954cda3fbd5ca4b8edbfc592a821a7b00291be6f843292e",
-                "sha256:be2f0ec62e0939a9dcfd3638c140c5a74fc929ee3fd1f31408ab8633db6e1523",
-                "sha256:c5d0c2ae3e3ed4e9f46b7c03b40d443601012ffe8eb8dfbb2bd6b2d00509f797",
-                "sha256:f0268613073df055bcc6a490de733012f2cf4fe191c1adb74e41cec8add1a165"
+                "sha256:1de0bb6cbfe460725f0e97b88daa8643bcf9571c18ba90bb8e41432aaeca91d6",
+                "sha256:1e850163579a8936eede29fad41e202b25923a0a8d5ffd08ce50fc0a97dcdc93",
+                "sha256:215e2a30a2090221a9481db58b770ce56b8ef46f13224ae33afe221b14b24dc1",
+                "sha256:348e6032f666ffd151b323342f9278b16b95d4a75dfacae84a11d2829a7816ae",
+                "sha256:3d2eb9c1cc254d0ffa90bc96fde4b6005d09c2228f99dfd493a4219c1af99644",
+                "sha256:3e477db76c22929e4c6876c44f88d790aacdf3c3f8f3a90cb1975c0bf37825b0",
+                "sha256:451cc89cb33d6652c509fc6b588dc51c41d7246afdcc29b8624e256b7663ed1f",
+                "sha256:46b1a60a04e6d884f0250d5cc8dc7bd21a9a96c584a7acdaab44698a44710bab",
+                "sha256:5f571b92a536206f7958f7cb2d367ff6c9a1fa8229dc35020006e4cdd1ca0acd",
+                "sha256:672960dd114e342b7c610bf32fb99d14227f29919894388b41553217457ba7ef",
+                "sha256:7310e353a4a35477c7f032409966920197d7df3e757c7624fd842f3eeb307d3d",
+                "sha256:746a1df55749629e26af7f977ea426817ca9370ad1569436608dc48d1069b87c",
+                "sha256:7c155437ae4fd366e2700e2716564d1787700687443de46bcb895fe0f84b761d",
+                "sha256:9265ae0fb35e29f9b8cc86c2ab0a2e3dcddc4dd9de4b85bf26c0f63fe5c1c2ca",
+                "sha256:94bdd1d55c20e764d8aea9d471d2ae7a7b2c84445e0fa463f02e20f9730783e1",
+                "sha256:9a79e5dd7bb797aa611048f5b70588b23c5be05b63eefd8a0d152ac77c4243db",
+                "sha256:a17f0a10604fac7627ec82820439e7db611722e80c408a726cd00d8c974c2fb3",
+                "sha256:a1acb72f095f1d58ecc2538ed1b8bca0b57df313b13db36ed34b8cdf1868e674",
+                "sha256:aa49571d8030ad0b9ac39708ee77bd2a22f87815e12bdee52ecaffece9313ed8",
+                "sha256:c24c05f645aef776e8b8931cb81e0f1632d229b42b6d216e30836e2e145a2b40",
+                "sha256:cf3a7e54eff792f0815dbbe9b85df2f13d739289c93d346925554f71d484be78",
+                "sha256:d738acfdfb65da34c91acbdb56abed46803db39af259b7f194dc96920360dbe4",
+                "sha256:e15fa23d844d54e7b3b7243afd53b7567ee71c721f592deb0727ee85e668f96a",
+                "sha256:ed4a9e6dcacba56b17a0a9ac22ae2c72a35b7f0ef0693aa68574f0b2df607a89",
+                "sha256:f44149a0ef5b4991aaef12a93b8e8d66d6412e762745fea1faa61d98524e0ba9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==3.3.2"
+            "version": "==3.3.4"
         },
         "munch": {
             "hashes": [
@@ -178,6 +198,14 @@
             ],
             "version": "==2.5.0"
         },
+        "natsort": {
+            "hashes": [
+                "sha256:00c603a42365830c4722a2eb7663a25919551217ec09a243d3399fa8dd4ac403",
+                "sha256:d0f4fc06ca163fa4a5ef638d9bf111c67f65eedcc7920f98dec08e489045b67e"
+            ],
+            "index": "pypi",
+            "version": "==7.1.1"
+        },
         "numba": {
             "hashes": [
                 "sha256:24852c21fbf7edf9e000eeec9fbd1b24d1ca17c86ae449b06a3707bcdec95479",
@@ -205,35 +233,33 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:04c7d4ebc5ff93d9822075ddb1751ff392a4375e5885299445fcebf877f179d5",
-                "sha256:0bfd85053d1e9f60234f28f63d4a5147ada7f432943c113a11afcf3e65d9d4c8",
-                "sha256:0c66da1d202c52051625e55a249da35b31f65a81cb56e4c69af0dfb8fb0125bf",
-                "sha256:0d310730e1e793527065ad7dde736197b705d0e4c9999775f212b03c44a8484c",
-                "sha256:1669ec8e42f169ff715a904c9b2105b6640f3f2a4c4c2cb4920ae8b2785dac65",
-                "sha256:2117536e968abb7357d34d754e3733b0d7113d4c9f1d921f21a3d96dec5ff716",
-                "sha256:3733640466733441295b0d6d3dcbf8e1ffa7e897d4d82903169529fd3386919a",
-                "sha256:4339741994c775396e1a274dba3609c69ab0f16056c1077f18979bec2a2c2e6e",
-                "sha256:51ee93e1fac3fe08ef54ff1c7f329db64d8a9c5557e6c8e908be9497ac76374b",
-                "sha256:54045b198aebf41bf6bf4088012777c1d11703bf74461d70cd350c0af2182e45",
-                "sha256:58d66a6b3b55178a1f8a5fe98df26ace76260a70de694d99577ddeab7eaa9a9d",
-                "sha256:59f3d687faea7a4f7f93bd9665e5b102f32f3fa28514f15b126f099b7997203d",
-                "sha256:62139af94728d22350a571b7c82795b9d59be77fc162414ada6c8b6a10ef5d02",
-                "sha256:7118f0a9f2f617f921ec7d278d981244ba83c85eea197be7c5a4f84af80a9c3c",
-                "sha256:7c6646314291d8f5ea900a7ea9c4261f834b5b62159ba2abe3836f4fa6705526",
-                "sha256:967c92435f0b3ba37a4257c48b8715b76741410467e2bdb1097e8391fccfae15",
-                "sha256:9a3001248b9231ed73894c773142658bab914645261275f675d86c290c37f66d",
-                "sha256:aba1d5daf1144b956bc87ffb87966791f5e9f3e1f6fab3d7f581db1f5b598f7a",
-                "sha256:addaa551b298052c16885fc70408d3848d4e2e7352de4e7a1e13e691abc734c1",
-                "sha256:b594f76771bc7fc8a044c5ba303427ee67c17a09b36e1fa32bde82f5c419d17a",
-                "sha256:c35a01777f81e7333bcf276b605f39c872e28295441c265cd0c860f4b40148c1",
-                "sha256:cebd4f4e64cfe87f2039e4725781f6326a61f095bc77b3716502bed812b385a9",
-                "sha256:d526fa58ae4aead839161535d59ea9565863bb0b0bdb3cc63214613fb16aced4",
-                "sha256:d7ac33585e1f09e7345aa902c281bd777fdb792432d27fca857f39b70e5dd31c",
-                "sha256:e6ddbdc5113628f15de7e4911c02aed74a4ccff531842c583e5032f6e5a179bd",
-                "sha256:eb25c381d168daf351147713f49c626030dcff7a393d5caa62515d415a6071d8"
+                "sha256:032be656d89bbf786d743fee11d01ef318b0781281241997558fa7950028dd29",
+                "sha256:104f5e90b143dbf298361a99ac1af4cf59131218a045ebf4ee5990b83cff5fab",
+                "sha256:125a0e10ddd99a874fd357bfa1b636cd58deb78ba4a30b5ddb09f645c3512e04",
+                "sha256:12e4ba5c6420917571f1a5becc9338abbde71dd811ce40b37ba62dec7b39af6d",
+                "sha256:13adf545732bb23a796914fe5f891a12bd74cf3d2986eed7b7eba2941eea1590",
+                "sha256:2d7e27442599104ee08f4faed56bb87c55f8b10a5494ac2ead5c98a4b289e61f",
+                "sha256:3bc63486a870294683980d76ec1e3efc786295ae00128f9ea38e2c6e74d5a60a",
+                "sha256:3d3087e24e354c18fb35c454026af3ed8997cfd4997765266897c68d724e4845",
+                "sha256:4ed8e96dc146e12c1c5cdd6fb9fd0757f2ba66048bf94c5126b7efebd12d0090",
+                "sha256:60759ab15c94dd0e1ed88241fd4fa3312db4e91d2c8f5a2d4cf3863fad83d65b",
+                "sha256:65410c7f4398a0047eea5cca9b74009ea61178efd78d1be9847fac1d6716ec1e",
+                "sha256:66b467adfcf628f66ea4ac6430ded0614f5cc06ba530d09571ea404789064adc",
+                "sha256:7199109fa46277be503393be9250b983f325880766f847885607d9b13848f257",
+                "sha256:72251e43ac426ff98ea802a931922c79b8d7596480300eb9f1b1e45e0543571e",
+                "sha256:89e5336f2bec0c726ac7e7cdae181b325a9c0ee24e604704ed830d241c5e47ff",
+                "sha256:89f937b13b8dd17b0099c7c2e22066883c86ca1575a975f754babc8fbf8d69a9",
+                "sha256:9c94cab5054bad82a70b2e77741271790304651d584e2cdfe2041488e753863b",
+                "sha256:9eb551d122fadca7774b97db8a112b77231dcccda8e91a5bc99e79890797175e",
+                "sha256:a1d7995d1023335e67fb070b2fae6f5968f5be3802b15ad6d79d81ecaa014fe0",
+                "sha256:ae61f02b84a0211abb56462a3b6cd1e7ec39d466d3160eb4e1da8bf6717cdbeb",
+                "sha256:b9410c0b6fed4a22554f072a86c361e417f0258838957b78bd063bde2c7f841f",
+                "sha256:c26287dfc888cf1e65181f39ea75e11f42ffc4f4529e5bd19add57ad458996e2",
+                "sha256:c91ec9569facd4757ade0888371eced2ecf49e7982ce5634cc2cf4e7331a4b14",
+                "sha256:ecb5b74c702358cdc21268ff4c37f7466357871f53a30e6f84c686952bef16a9"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.19.2"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.20.1"
         },
         "pandas": {
             "hashes": [
@@ -259,37 +285,42 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a",
-                "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae",
-                "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce",
-                "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e",
-                "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140",
-                "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb",
-                "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021",
-                "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6",
-                "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302",
-                "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c",
-                "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271",
-                "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09",
-                "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3",
-                "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015",
-                "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3",
-                "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544",
-                "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8",
-                "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792",
-                "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0",
-                "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3",
-                "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8",
-                "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11",
-                "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7",
-                "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11",
-                "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e",
-                "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039",
-                "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5",
-                "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"
+                "sha256:15306d71a1e96d7e271fd2a0737038b5a92ca2978d2e38b6ced7966583e3d5af",
+                "sha256:1940fc4d361f9cc7e558d6f56ff38d7351b53052fd7911f4b60cd7bc091ea3b1",
+                "sha256:1f93f2fe211f1ef75e6f589327f4d4f8545d5c8e826231b042b483d8383e8a7c",
+                "sha256:30d33a1a6400132e6f521640dd3f64578ac9bfb79a619416d7e8802b4ce1dd55",
+                "sha256:328240f7dddf77783e72d5ed79899a6b48bc6681f8d1f6001f55933cb4905060",
+                "sha256:46c2bcf8e1e75d154e78417b3e3c64e96def738c2a25435e74909e127a8cba5e",
+                "sha256:5762ebb4436f46b566fc6351d67a9b5386b5e5de4e58fdaa18a1c83e0e20f1a8",
+                "sha256:5a2d957eb4aba9d48170b8fe6538ec1fbc2119ffe6373782c03d8acad3323f2e",
+                "sha256:5cf03b9534aca63b192856aa601c68d0764810857786ea5da652581f3a44c2b0",
+                "sha256:5daba2b40782c1c5157a788ec4454067c6616f5a0c1b70e26ac326a880c2d328",
+                "sha256:63cd413ac52ee3f67057223d363f4f82ce966e64906aea046daf46695e3c8238",
+                "sha256:6efac40344d8f668b6c4533ae02a48d52fd852ef0654cc6f19f6ac146399c733",
+                "sha256:71b01ee69e7df527439d7752a2ce8fb89e19a32df484a308eca3e81f673d3a03",
+                "sha256:71f31ee4df3d5e0b366dd362007740106d3210fb6a56ec4b581a5324ba254f06",
+                "sha256:72027ebf682abc9bafd93b43edc44279f641e8996fb2945104471419113cfc71",
+                "sha256:74cd9aa648ed6dd25e572453eb09b08817a1e3d9f8d1bd4d8403d99e42ea790b",
+                "sha256:81b3716cc9744ffdf76b39afb6247eae754186838cedad0b0ac63b2571253fe6",
+                "sha256:8565355a29655b28fdc2c666fd9a3890fe5edc6639d128814fafecfae2d70910",
+                "sha256:87f42c976f91ca2fc21a3293e25bd3cd895918597db1b95b93cbd949f7d019ce",
+                "sha256:89e4c757a91b8c55d97c91fa09c69b3677c227b942fa749e9a66eef602f59c28",
+                "sha256:8c4e32218c764bc27fe49b7328195579581aa419920edcc321c4cb877c65258d",
+                "sha256:903293320efe2466c1ab3509a33d6b866dc850cfd0c5d9cc92632014cec185fb",
+                "sha256:90882c6f084ef68b71bba190209a734bf90abb82ab5e8f64444c71d5974008c6",
+                "sha256:98afcac3205d31ab6a10c5006b0cf040d0026a68ec051edd3517b776c1d78b09",
+                "sha256:a01da2c266d9868c4f91a9c6faf47a251f23b9a862dce81d2ff583135206f5be",
+                "sha256:aeab4cd016e11e7aa5cfc49dcff8e51561fa64818a0be86efa82c7038e9369d0",
+                "sha256:b07c660e014852d98a00a91adfbe25033898a9d90a8f39beb2437d22a203fc44",
+                "sha256:bead24c0ae3f1f6afcb915a057943ccf65fc755d11a1410a909c1fefb6c06ad1",
+                "sha256:d1d6bca39bb6dd94fba23cdb3eeaea5e30c7717c5343004d900e2a63b132c341",
+                "sha256:e2cd8ac157c1e5ae88b6dd790648ee5d2777e76f1e5c7d184eaddb2938594f34",
+                "sha256:e5739ae63636a52b706a0facec77b2b58e485637e1638202556156e424a02dc2",
+                "sha256:f36c3ff63d6fc509ce599a2f5b0d0732189eed653420e7294c039d342c6e204a",
+                "sha256:f91b50ad88048d795c0ad004abbe1390aa1882073b1dca10bfd55d0b8cf18ec5"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==8.0.1"
+            "version": "==8.1.2"
         },
         "pygeos": {
             "hashes": [
@@ -323,32 +354,34 @@
         },
         "pyproj": {
             "hashes": [
-                "sha256:2518d1606e2229b82318e704b40290e02a2a52d77b40cdcb2978973d6fc27b20",
-                "sha256:33a5d1cfbb40a019422eb80709a0e270704390ecde7278fdc0b88f3647c56a39",
-                "sha256:33c1c2968a4f4f87d517c4275a18b557e5c13907cf2609371fadea8463c3ba05",
-                "sha256:3fef83a01c1e86dd9fa99d8214f749837cfafc34d9d6230b4b0a998fa7a68a1a",
-                "sha256:451a3d1c563b672458029ebc04acbb3266cd8b3025268eb871a9176dc3638911",
-                "sha256:457ad3856014ac26af1d86def6dc8cf69c1fa377b6e2fd6e97912d51cf66bdbe",
-                "sha256:4f5b02b4abbd41610397c635b275a8ee4a2b5bc72a75572b98ac6ae7befa471e",
-                "sha256:6a212d0e5c7efa33d039f0c8b0a489e2204fcd28b56206567852ad7f5f2a653e",
-                "sha256:6f3f36440ea61f5f6da4e6beb365dddcbe159815450001d9fb753545affa45ff",
-                "sha256:93cbad7b699e8e80def7de80c350617f35e6a0b82862f8ce3c014657c25fdb3c",
-                "sha256:9f097e8f341a162438918e908be86d105a28194ff6224633b2e9616c5031153f",
-                "sha256:a13e5731b3a360ee7fbd1e9199ec9203fafcece8ebd0b1351f16d0a90cad6828",
-                "sha256:a6ac4861979cd05a0f5400fefa41d26c0269a5fb8237618aef7c998907db39e1",
-                "sha256:a8b7c8accdc61dac8e91acab7c1f7b4590d1e102f2ee9b1f1e6399fad225958e",
-                "sha256:adacb67a9f71fb54ca1b887a6ab20f32dd536fcdf2acec84a19e25ad768f7965",
-                "sha256:bc2f3a15d065e206d63edd2cc4739aa0a35c05338ee276ab1dc72f56f1944bda",
-                "sha256:cbf6ccf990860b06c5262ff97c4b78e1d07883981635cd53a6aa438a68d92945",
-                "sha256:d87836be6b720fb4d9c112136aa47621b6ca09a554e645c1081561eb8e2fa1f4",
-                "sha256:d90a5d1fdd066b0e9b22409b0f5e81933469918fa04c2cf7f9a76ce84cb29dad",
-                "sha256:daf2998e3f5bcdd579a18faf009f37f53538e9b7d0a252581a610297d31e8536",
-                "sha256:e015f900b4b84e908f8035ab16ebf02d67389c1c216c17a2196fc2e515c00762",
-                "sha256:e50d5d20b87758acf8f13f39a3b3eb21d5ef32339d2bc8cdeb8092416e0051df",
-                "sha256:f5a8015c74ec8f6508aebf493b58ba20ccb4da8168bf05f0c2a37faccb518da9"
+                "sha256:09bead60769e69b592e8cb3ac51b5215f75e9bb9c213ce575031961deb48d6da",
+                "sha256:09db64a8088b23f001e574d92bcc3080bf7de44ddca152d0282a2b50c918a64a",
+                "sha256:1be7d54900eb7e2d1e637319080b3a047c70d1fb2f3c12d3400c0fa8a90cf440",
+                "sha256:36ba436675f9dea4ab3db7d9a32d3ff11c2fbb4d6690a83454d2f3c5c0b54041",
+                "sha256:3e7e851e6d58c16ac2cd920a1bacb7fbb24758a6fcd7f234d594a88ebae04ec9",
+                "sha256:489a96da87d8846c34c90da90e637544e4f4f50a13589b5aac54297f5ee1b01d",
+                "sha256:4a333f3e46fe8b2eb4647a3daa3a2cec52ddc6c107c653b45880526114942ee8",
+                "sha256:708d6e01b9ff3d6dc62a5ad2d2ba1264a863eaa657c1a9bf713a10cc35d34553",
+                "sha256:7ae8e7052f18fde1884574da449010e94fa205ad27aeeaa34a097f49a1ed6a2b",
+                "sha256:7bfaa34e8bb0510d4380310374deecd9e4328b9cf556925cfb45b5a94d5bbdbe",
+                "sha256:81c06df20d09d621e52791c19ce3c880695fb430061e59c2472fa5467e890391",
+                "sha256:86ef2fcd584a3222bf73e2befc24b2badd139b3371f4a1e88649978ef7649540",
+                "sha256:9666d01faf4e758ac68f2c16695c90de49c3170e3760988bf76a34aae11f4e15",
+                "sha256:9b845510255f9580d7e226dd3321a51c468cefb7be24e46415caf67caa4287c4",
+                "sha256:9e2ef75401f17062166d3fe53c555cd62c9577697a2f5ded916b23c54e5db497",
+                "sha256:a3805e026a5547be205a5e322c08e3069f0a48c63bbd53dbc7a8e3499bc66d58",
+                "sha256:a3a8ab19232bf4f4bb2590536538881b7bd0c07df23e0c2a792402ca2476c197",
+                "sha256:aa0a2981b25145523ca17a643c5be077fe13e514fdca9b6d1c412a95d723a5a5",
+                "sha256:bc70b6adcfa713d89bc561673cb57af5fb3a1718cd7d57ec537430cd1007a864",
+                "sha256:bfbac35490dd17f706700673506eeb8170f8a2a63fb5878171d4e6eef242d141",
+                "sha256:c658afc8a6115b58b02aa53d27bf2a67c1b00b55067edb1b7711c6c7391cfaa9",
+                "sha256:cba99e171d744969e13a865ad28fa9c949c4400b0e9c431a802cdd804f52f632",
+                "sha256:d27d40ec541ef69a5107bfcd85f40170e9e122ceb6315ce508cd44d199983d41",
+                "sha256:f942a976ea3de6a519cf48be30a12f465e44d0ac0c38a0d820ab3acfcc0a48a6",
+                "sha256:fee7517bd389a1db7b8bebb18838d04dedca9eaacda01d353d98f5ee421f263e"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.6.1.post1"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.1"
         },
         "python-dateutil": {
             "hashes": [
@@ -358,12 +391,20 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.1"
         },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e",
+                "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"
+            ],
+            "index": "pypi",
+            "version": "==0.15.0"
+        },
         "pytz": {
             "hashes": [
-                "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed",
-                "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"
+                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
+                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
             ],
-            "version": "==2020.1"
+            "version": "==2021.1"
         },
         "rasterio": {
             "hashes": [
@@ -410,28 +451,28 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:07b083128beae040f1129bd8a82b01804f5e716a7fd2962c1053fa683433e4ab",
-                "sha256:0edd67e8a00903aaf7a29c968555a2e27c5a69fea9d1dcfffda80614281a884f",
-                "sha256:12fdcbfa56cac926a0a9364a30cbf4ad03c2c7b59f75b14234656a5e4fd52bf3",
-                "sha256:1fee28b6641ecbff6e80fe7788e50f50c5576157d278fa40f36c851940eb0aff",
-                "sha256:33e6a7439f43f37d4c1135bc95bcd490ffeac6ef4b374892c7005ce2c729cf4a",
-                "sha256:5163200ab14fd2b83aba8f0c4ddcc1fa982a43192867264ab0f4c8065fd10d17",
-                "sha256:66ec29348444ed6e8a14c9adc2de65e74a8fc526dc2c770741725464488ede1f",
-                "sha256:8cc5c39ed287a8b52a5509cd6680af078a40b0e010e2657eca01ffbfec929468",
-                "sha256:a1a13858b10d41beb0413c4378462b43eafef88a1948d286cb357eadc0aec024",
-                "sha256:a3db1fe7c6cb29ca02b14c9141151ebafd11e06ffb6da8ecd330eee5c8283a8a",
-                "sha256:aebb69bcdec209d874fc4b0c7ac36f509d50418a431c1422465fa34c2c0143ea",
-                "sha256:b9751b39c52a3fa59312bd2e1f40144ee26b51404db5d2f0d5259c511ff6f614",
-                "sha256:bc0e63daf43bf052aefbbd6c5424bc03f629d115ece828e87303a0bcc04a37e4",
-                "sha256:d5e3cc60868f396b78fc881d2c76460febccfe90f6d2f082b9952265c79a8788",
-                "sha256:ddae76784574cc4c172f3d5edd7308be16078dd3b977e8746860c76c195fa707",
-                "sha256:e2602f79c85924e4486f684aa9bbab74afff90606100db88d0785a0088be7edb",
-                "sha256:e527c9221b6494bcd06a17f9f16874406b32121385f9ab353b8a9545be458f0b",
-                "sha256:f574558f1b774864516f3c3fe072ebc90a29186f49b720f60ed339294b7f32ac",
-                "sha256:ffcbd331f1ffa82e22f1d408e93c37463c9a83088243158635baec61983aaacf"
+                "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67",
+                "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d",
+                "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d",
+                "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced",
+                "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07",
+                "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf",
+                "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c",
+                "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78",
+                "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4",
+                "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0",
+                "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627",
+                "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4",
+                "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76",
+                "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261",
+                "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f",
+                "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519",
+                "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11",
+                "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092",
+                "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.5.3"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.6.1"
         },
         "seaborn": {
             "hashes": [
@@ -558,11 +599,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:2e22c1f74477b5106a6fb301c342ab8c64bb75d702e350f05a649e8cb40a0fb8",
-                "sha256:a331e78086001931de9424940699691ad49dfb457cea31f5471eae7b78222d5e"
+                "sha256:04323f72d5b85b606330b6d7e2dc8d2683ad46c3905e955aa96ecc7a99388e70",
+                "sha256:34207ffb2f653bced2bc8e3756c1db86e7d93e44ed049daae9814fed66d408ec"
             ],
             "index": "pypi",
-            "version": "==7.18.1"
+            "version": "==7.21.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -573,19 +614,19 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:86ed7d9b750603e4ba582ea8edc678657fb4007894a12bcf6f4bb97892f31d20",
-                "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5"
+                "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93",
+                "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==0.17.2"
+            "markers": "python_version >= '3.6'",
+            "version": "==0.18.0"
         },
         "parso": {
             "hashes": [
-                "sha256:97218d9159b2520ff45eb78028ba8b50d2bc61dcc062a9682666f2dc4bd331ea",
-                "sha256:caba44724b994a8a5e086460bb212abc5a8bc46951bf4a9a1210745953622eb9"
+                "sha256:15b00182f472319383252c18d5913b69269590616c947747bc50bf4ac768f410",
+                "sha256:8519430ad07087d4c997fda3a7918f7cfa27cb58972a8c89c2a0295a1c940e9e"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.7.1"
+            "markers": "python_version >= '3.6'",
+            "version": "==0.8.1"
         },
         "pexpect": {
             "hashes": [
@@ -604,26 +645,26 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:25c95d2ac813909f813c93fde734b6e44406d1477a9faef7c915ff37d39c0a8c",
-                "sha256:7debb9a521e0b1ee7d2fe96ee4bd60ef03c6492784de0547337ca4433e46aa63"
+                "sha256:0fa02fa80363844a4ab4b8d6891f62dd0645ba672723130423ca4037b80c1974",
+                "sha256:62c811e46bd09130fb11ab759012a4ae385ce4fb2073442d1898867a824183bd"
             ],
             "markers": "python_full_version >= '3.6.1'",
-            "version": "==3.0.8"
+            "version": "==3.0.16"
         },
         "ptyprocess": {
             "hashes": [
-                "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0",
-                "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"
+                "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35",
+                "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"
             ],
-            "version": "==0.6.0"
+            "version": "==0.7.0"
         },
         "pygments": {
             "hashes": [
-                "sha256:307543fe65c0947b126e83dd5a61bd8acbd84abec11f43caebaf5534cbc17998",
-                "sha256:926c3f319eda178d1bd90851e4317e6d8cdb5e292a3386aac9bd75eca29cf9c7"
+                "sha256:2656e1a6edcdabf4275f9a3640db59fd5de107d88e8663c5d4e9a0fa62f77f94",
+                "sha256:534ef71d539ae97d4c3a4cf7d6f110f214b0e687e92f9cb9d2a3b0d3101289c8"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==2.7.1"
+            "version": "==2.8.1"
         },
         "traitlets": {
             "hashes": [
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index eb4198cb3..2958c2882 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -260,12 +260,12 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
 if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description='Subset vector layers')
+    parser = argparse.ArgumentParser(description='Crosswalk for MS/FR networks; calculate synthetic rating curves; update short rating curves')
     parser.add_argument('-d','--input-catchments-fileName', help='DEM derived catchments', required=True)
     parser.add_argument('-a','--input-flows-fileName', help='DEM derived streams', required=True)
     parser.add_argument('-s','--input-srcbase-fileName', help='Base synthetic rating curve table', required=True)
     parser.add_argument('-l','--output-catchments-fileName', help='Subset crosswalked catchments', required=True)
-    parser.add_argument('-f','--output-flows-fileName', help='Subset crosswalked  streams', required=True)
+    parser.add_argument('-f','--output-flows-fileName', help='Subset crosswalked streams', required=True)
     parser.add_argument('-r','--output-src-fileName', help='Output crosswalked synthetic rating curve table', required=True)
     parser.add_argument('-j','--output-src-json-fileName',help='Output synthetic rating curve json',required=True)
     parser.add_argument('-x','--output-crosswalk-fileName',help='Crosswalk table',required=True)
diff --git a/tools/.env.template b/tools/.env.template
new file mode 100644
index 000000000..462d9c556
--- /dev/null
+++ b/tools/.env.template
@@ -0,0 +1,3 @@
+API_BASE_URL=
+EVALUATED_SITES_CSV=
+WBD_LAYER=
diff --git a/tools/.gitignore b/tools/.gitignore
new file mode 100644
index 000000000..4c49bd78f
--- /dev/null
+++ b/tools/.gitignore
@@ -0,0 +1 @@
+.env
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index 290a44704..bec059960 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -18,10 +18,7 @@
 from utils.shared_functions import getDriver
 
 INPUTS_DIR = r'/data/inputs'
-magnitude_list = ['action', 'minor', 'moderate','major']
-
-# Map path to points with attributes
-all_mapped_ahps_conus_hipr = os.path.join(INPUTS_DIR, 'ahp_sites', 'all_mapped_ahps_reformatted.csv')
+magnitude_list = ['action', 'minor', 'moderate','major', 'record']
 
 # Define necessary variables for inundation()
 hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
@@ -142,7 +139,7 @@ def run_inundation(args):
         f.close()
 
 
-def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, log_file):
+def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_attributes_filename, log_file):
 
     # Create workspace
     gpkg_dir = os.path.join(output_cat_fim_dir, 'gpkg')
@@ -176,7 +173,7 @@ def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, log_file):
                         extent_grid = os.path.join(ahps_lid_dir, ahps_lid + '_' + magnitude + '_extent_' + huc + '.tif')
 
                         if os.path.exists(extent_grid):
-                            procs_list.append([ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude])
+                            procs_list.append([ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename])
 
                         else:
                             try:
@@ -217,12 +214,13 @@ def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, log_file):
 def reformat_inundation_maps(args):
 
     try:
-        lid          = args[0]
-        grid_path    = args[1]
-        gpkg_dir     = args[2]
-        fim_version  = args[3]
-        huc          = args[4]
-        magnitude    = args[5]
+        lid                         = args[0]
+        grid_path                   = args[1]
+        gpkg_dir                    = args[2]
+        fim_version                 = args[3]
+        huc                         = args[4]
+        magnitude                   = args[5]
+        nws_lid_attributes_filename = args[6]
 
         # Convert raster to to shapes
         with rasterio.open(grid_path) as src:
@@ -249,10 +247,11 @@ def reformat_inundation_maps(args):
         extent_poly = extent_poly.to_crs(VIZ_PROJECTION)
 
         # Join attributes
-        all_mapped_ahps_conus_hipr_fl = pd.read_table(all_mapped_ahps_conus_hipr, sep=",")
-        all_mapped_ahps_conus_hipr_fl = all_mapped_ahps_conus_hipr_fl.loc[(all_mapped_ahps_conus_hipr_fl.magnitude==magnitude) & (all_mapped_ahps_conus_hipr_fl.nws_lid==lid)]
+        nws_lid_attributes_table = pd.read_csv(nws_lid_attributes_filename, dtype={'huc':str})
+        nws_lid_attributes_table = nws_lid_attributes_table.loc[(nws_lid_attributes_table.magnitude==magnitude) & (nws_lid_attributes_table.nws_lid==lid)]
+
 
-        extent_poly_diss = extent_poly_diss.merge(all_mapped_ahps_conus_hipr_fl, left_on=['ahps_lid','magnitude'], right_on=['nws_lid','magnitude'])
+        extent_poly_diss = extent_poly_diss.merge(nws_lid_attributes_table, left_on=['ahps_lid','magnitude','huc'], right_on=['nws_lid','magnitude','huc'])
 
         extent_poly_diss = extent_poly_diss.drop(columns='nws_lid')
 
@@ -308,8 +307,11 @@ def reformat_inundation_maps(args):
     # Create error log path
     log_file = os.path.join(log_dir, 'errors.log')
 
+    # Map path to points with attributes
+    nws_lid_attributes_filename = os.path.join(source_flow_dir, 'nws_lid_attributes.csv')
+
     print("Generating Categorical FIM")
     generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif,log_file)
 
     print("Aggregating Categorical FIM")
-    post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir,log_file)
+    post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir,nws_lid_attributes_filename,log_file)
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
new file mode 100755
index 000000000..562b3e8e5
--- /dev/null
+++ b/tools/generate_categorical_fim_flows.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import geopandas as gpd
+import pandas as pd
+import time
+from tools_shared_functions import aggregate_wbd_hucs, mainstem_nwm_segs, get_thresholds, flow_data, get_metadata, get_nwm_segs, flow_data
+import argparse
+from dotenv import load_dotenv
+import os
+import sys
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+
+load_dotenv()
+#import variables from .env file
+API_BASE_URL = os.getenv("API_BASE_URL")
+EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+WBD_LAYER = os.getenv("WBD_LAYER")
+
+def static_flow_lids(workspace, nwm_us_search, nwm_ds_search):
+    '''
+    This will create static flow files for all nws_lids and save to the 
+    workspace directory with the following format:
+    huc code
+        nws_lid_code
+            threshold (action/minor/moderate/major if they exist/are defined by WRDS)
+                flow file (ahps_{lid code}_huc_{huc 8 code}_flows_{threshold}.csv)
+    
+    This will use the WRDS API to get the nwm segments as well as the flow 
+    values for each threshold at each nws_lid and then create the necessary 
+    flow file to use for inundation mapping.
+
+    Parameters
+    ----------
+    workspace : STR
+        Location where output flow files will exist.
+    nwm_us_search : STR
+        Upstream distance (in miles) for walking up NWM network.
+    nwm_ds_search : STR
+        Downstream distance (in miles) for walking down NWM network.
+    wbd_path : STR
+        Location of HUC geospatial data (geopackage).
+        
+    Returns
+    -------
+    None.
+
+    '''    
+    all_start = time.time()
+    #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
+    workspace = Path(workspace)
+    nwm_us_search = int(nwm_us_search)
+    nwm_ds_search = int(nwm_ds_search)
+    metadata_url = f'{API_BASE_URL}/metadata'
+    threshold_url = f'{API_BASE_URL}/threshold'
+    ###################################################################
+    #Create workspace
+    workspace.mkdir(exist_ok = True)
+
+    #Return dictionary of huc (key) and sublist of ahps(value) as well as geodataframe of sites.
+    print('Retrieving metadata...')
+    #Get metadata for 'CONUS'
+    conus_list, conus_dataframe = get_metadata(metadata_url, select_by = 'nws_lid', selector = ['all'], must_include = 'nws_data.rfc_forecast_point', upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search )
+
+    #Get metadata for Islands
+    islands_list, islands_dataframe = get_metadata(metadata_url, select_by = 'state', selector = ['HI','PR'] , must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+    
+    #Append the dataframes and lists
+    all_lists = conus_list + islands_list
+    all_dataframe = conus_dataframe.append(islands_dataframe)
+    
+    print('Determining HUC using WBD layer...')
+    #Assign FIM HUC to GeoDataFrame and export to shapefile all candidate sites.
+    agg_start = time.time()
+    huc_dictionary, out_gdf = aggregate_wbd_hucs(metadata_list = all_lists, wbd_huc8_path = WBD_LAYER)
+    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)
+    viz_out_gdf.to_file(workspace / f'candidate_sites.shp')
+    agg_end = time.time()
+    print(f'agg time is {(agg_end - agg_start)/60} minutes')
+    #Get all possible mainstem segments
+    print('Getting list of mainstem segments')
+    #Import list of evaluated sites
+    list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
+    #The entire routine to get mainstems is harcoded in this function.
+    ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
+    
+    #Loop through each huc unit
+    all_messages = []
+    for huc in huc_dictionary:
+        print(f'Iterating through {huc}')
+        #Get list of nws_lids
+        nws_lids = huc_dictionary[huc]
+        #Loop through each lid in list to create flow file
+        for lid in nws_lids:
+            #In some instances the lid is not assigned a name, skip over these.
+            if not isinstance(lid,str):
+                print(f'{lid} is {type(lid)}')
+                continue
+            #Convert lid to lower case
+            lid = lid.lower()
+            #Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
+            stages, flows = get_thresholds(threshold_url = threshold_url, location_ids = lid, physical_element = 'all', threshold = 'all', bypass_source_flag = False)
+            #If stages/flows don't exist write message and exit out.
+            if not (stages and flows):
+                message = f'{lid} no thresholds'
+                all_messages.append(message)
+                continue
+
+            #find lid metadata from master list of metadata dictionaries (line 66).
+            metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
+        
+            #Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
+            segments = get_nwm_segs(metadata)        
+            site_ms_segs = set(segments).intersection(ms_segs)
+            segments = list(site_ms_segs)       
+            #if no segments, write message and exit out
+            if not segments:
+                print(f'{lid} no segments')
+                message = f'{lid} no segments'
+                all_messages.append(message)
+                continue
+            #For each flood category
+            for category in ['action', 'minor', 'moderate', 'major', 'record']:
+                #Get the flow
+                flow = flows[category]
+                #If there is a valid flow value, write a flow file.
+                if flow:
+                    #round flow to nearest hundredth
+                    flow = round(flow,2)
+                    #Create the guts of the flow file.
+                    flow_info = flow_data(segments,flow)
+                    #Define destination path and create folders
+                    output_file = workspace / huc / lid / category / (f'ahps_{lid}_huc_{huc}_flows_{category}.csv') 
+                    output_file.parent.mkdir(parents = True, exist_ok = True)
+                    #Write flow file to file
+                    flow_info.to_csv(output_file, index = False)
+                else:
+                    message = f'{lid}_{category}_no flow'
+                    all_messages.append(message)
+            #This section will produce a point file of the LID location
+            #Get various attributes of the site.
+            lat = float(metadata['usgs_data']['latitude'])
+            lon = float(metadata['usgs_data']['longitude'])
+            wfo = metadata['nws_data']['wfo']
+            rfc = metadata['nws_data']['rfc']
+            state = metadata['nws_data']['state']
+            county = metadata['nws_data']['county']
+            name = metadata['nws_data']['name']
+            q_act = flows['action']
+            q_min = flows['minor']
+            q_mod = flows['moderate']
+            q_maj = flows['major']
+            q_rec = flows['record']
+            flow_units = flows['units']
+            flow_source = flows['source']
+            s_act = stages['action']
+            s_min = stages['minor']
+            s_mod = stages['moderate']
+            s_maj = stages['major']
+            s_rec = stages['record']
+            stage_units = stages['units']
+            stage_source = stages['source']
+            wrds_timestamp = stages['wrds_timestamp']
+            #Create a DataFrame using the collected attributes
+            df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'q_act':q_act, 'q_min':q_min, 'q_mod':q_mod, 'q_maj':q_maj, 'q_rec':q_rec, 'q_uni':flow_units, 'q_src':flow_source, 'stage_act':s_act, 'stage_min':s_min, 'stage_mod':s_mod, 'stage_maj':s_maj, 'stage_rec':s_rec, 'stage_uni':stage_units, 's_src':stage_source, 'wrds_time':wrds_timestamp, 'lat':[lat], 'lon':[lon]})
+            #Round stages and flows to nearest hundredth
+            df = df.round({'q_act':2,'q_min':2,'q_mod':2,'q_maj':2,'q_rec':2,'stage_act':2,'stage_min':2,'stage_mod':2,'stage_maj':2,'stage_rec':2})
+            
+            #Create a geodataframe using usgs lat/lon property from WRDS then reproject to WGS84.
+            #Define EPSG codes for possible usgs latlon datum names (NAD83WGS84 assigned NAD83)
+            crs_lookup ={'NAD27':'EPSG:4267', 'NAD83':'EPSG:4269', 'NAD83WGS84': 'EPSG:4269'}
+            #Get horizontal datum (from dataframe) and assign appropriate EPSG code, assume NAD83 if not assigned.
+            h_datum = metadata['usgs_data']['latlon_datum_name']
+            src_crs = crs_lookup.get(h_datum, 'EPSG:4269')            
+            gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df['lon'], df['lat']), crs =  src_crs) 
+            #Reproject to VIZ_PROJECTION
+            viz_gdf = gdf.to_csv(VIZ_PROJECTION)
+            
+            #Create a csv with same info as shapefile
+            csv_df = pd.DataFrame()
+            for threshold in ['action', 'minor', 'moderate', 'major', 'record']:
+                line_df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'magnitude': threshold, 'q':flows[threshold], 'q_uni':flows['units'], 'q_src':flow_source, 'stage':stages[threshold], 'stage_uni':stages['units'], 's_src':stage_source, 'wrds_time':wrds_timestamp, 'lat':[lat], 'lon':[lon]})
+                csv_df = csv_df.append(line_df)
+            #Round flow and stage columns to 2 decimal places.
+            csv = csv_df.round({'q':2,'stage':2})
+
+            #If a site folder exists (ie a flow file was written) save files containing site attributes.
+            try:
+                #Save GeoDataFrame to shapefile format and export csv containing attributes
+                output_dir = workspace / huc / lid
+                viz_gdf.to_file(output_dir / f'{lid}_location.shp' )
+                csv_df.to_csv(output_dir / f'{lid}_attributes.csv', index = False)
+            except:
+                print(f'{lid} missing all flows')
+                message = f'{lid} missing all flows'
+                all_messages.append(message)
+    #Write out messages to file
+    messages_df  = pd.DataFrame(all_messages, columns = ['message'])
+    messages_df.to_csv(workspace / f'all_messages.csv', index = False)
+
+    #Recursively find all location shapefiles
+    locations_files = list(workspace.rglob('*_location.shp'))    
+    spatial_layers = gpd.GeoDataFrame()
+    #Append all shapefile info to a geodataframe
+    for location in locations_files:
+        location_gdf = gpd.read_file(location)
+        spatial_layers = spatial_layers.append(location_gdf)
+    #Write appended spatial data to disk.
+    output_file = workspace /'all_mapped_ahps.shp'
+    spatial_layers.to_file(output_file)
+    
+    #Recursively find all *_info csv files and append
+    csv_files = list(workspace.rglob('*_attributes.csv'))
+    all_csv_df = pd.DataFrame()
+    for csv in csv_files:
+        temp_df = pd.read_csv(csv, dtype={'huc':str})
+        all_csv_df = all_csv_df.append(temp_df, ignore_index = True)
+    #Write appended _info.csvs to file
+    all_info_csv = workspace / 'nws_lid_attributes.csv'
+    all_csv_df.to_csv(all_info_csv, index = False)
+    all_end = time.time()
+    print(f'total time is {(all_end - all_start)/60} minutes')
+    
+
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Create forecast files for all nws_lid sites')
+    parser.add_argument('-w', '--workspace', help = 'Workspace where all data will be stored.', required = True)
+    parser.add_argument('-u', '--nwm_us_search',  help = 'Walk upstream on NWM network this many miles', required = True)
+    parser.add_argument('-d', '--nwm_ds_search', help = 'Walk downstream on NWM network this many miles', required = True)
+    #Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+    
+
+    #Run create_flow_forecast_file
+    static_flow_lids(**args)
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 094f5e2a5..aed515f5c 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -5,6 +5,9 @@
 import csv
 import rasterio
 import pandas as pd
+import geopandas as gpd
+import requests
+
 from tools_shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
                                     ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
 
@@ -158,7 +161,7 @@ def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
 #
 #    print()
 #    print("--------------------------------------------------------------------------------------------------")
-#    
+#
 #    stats_mode = stats_modes_list[0]
 #    try:
 #        last_version_index = text_block[0].index('dev_latest')
@@ -173,7 +176,7 @@ def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
 #                print()
 #                continue
 #
-#    
+#
 #
 #    for line in text_block:
 #        first_item = line[0]
@@ -238,7 +241,7 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
     """
     This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.
     Much of the calculations below were taken from older Python files. This is evident in the inconsistent use of case.
-    
+
     Args:
         true_negatives (int): The true negatives from a contingency table.
         false_negatives (int): The false negatives from a contingency table.
@@ -246,37 +249,37 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
         true_positives (int): The true positives from a contingency table.
         cell_area (float or None): This optional argument allows for area-based statistics to be calculated, in the case that
                                    contingency table metrics were derived from areal analysis.
-    
+
     Returns:
         stats_dictionary (dict): A dictionary of statistics. Statistic names are keys and statistic values are the values.
                                  Refer to dictionary definition in bottom of function for statistic names.
-    
+
     """
-    
+
     import numpy as np
-    
+
     total_population = true_negatives + false_negatives + false_positives + true_positives
-        
+
     # Basic stats.
 #    Percent_correct = ((true_positives + true_negatives) / total_population) * 100
 #    pod             = true_positives / (true_positives + false_negatives)
-    
+
     try:
         FAR = false_positives / (true_positives + false_positives)
     except ZeroDivisionError:
         FAR = "NA"
-        
+
     try:
         CSI = true_positives / (true_positives + false_positives + false_negatives)
     except ZeroDivisionError:
         CSI = "NA"
-    
+
     try:
         BIAS = (true_positives + false_positives) / (true_positives + false_negatives)
     except ZeroDivisionError:
         BIAS = "NA"
-    
-    # Compute equitable threat score (ETS) / Gilbert Score. 
+
+    # Compute equitable threat score (ETS) / Gilbert Score.
     try:
         a_ref = ((true_positives + false_positives)*(true_positives + false_negatives)) / total_population
         EQUITABLE_THREAT_SCORE = (true_positives - a_ref) / (true_positives - a_ref + false_positives + false_negatives)
@@ -290,12 +293,12 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
         FP_perc = (false_positives / total_population) * 100
         TN_perc = (true_negatives / total_population) * 100
         FN_perc = (false_negatives / total_population) * 100
-    
+
     predPositive = true_positives + false_positives
     predNegative = true_negatives + false_negatives
     obsPositive = true_positives + false_negatives
     obsNegative = true_negatives + false_positives
-    
+
     TP = float(true_positives)
     TN = float(true_negatives)
     FN = float(false_negatives)
@@ -304,7 +307,7 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
         MCC = (TP*TN - FP*FN)/ np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))
     except ZeroDivisionError:
         MCC = "NA"
-    
+
     if masked_count != None:
         total_pop_and_mask_pop = total_population + masked_count
         if total_pop_and_mask_pop == 0:
@@ -313,23 +316,23 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
             masked_perc = (masked_count / total_pop_and_mask_pop) * 100
     else:
         masked_perc = None
-    
+
     # This checks if a cell_area has been provided, thus making areal calculations possible.
     sq_km_converter = 1000000
-        
+
     if cell_area != None:
         TP_area = (true_positives * cell_area) / sq_km_converter
         FP_area = (false_positives * cell_area) / sq_km_converter
         TN_area = (true_negatives * cell_area) / sq_km_converter
         FN_area = (false_negatives * cell_area) / sq_km_converter
         area = (total_population * cell_area) / sq_km_converter
-        
+
         predPositive_area = (predPositive * cell_area) / sq_km_converter
         predNegative_area = (predNegative * cell_area) / sq_km_converter
         obsPositive_area =  (obsPositive * cell_area) / sq_km_converter
         obsNegative_area =  (obsNegative * cell_area) / sq_km_converter
         positiveDiff_area = predPositive_area - obsPositive_area
-        
+
         if masked_count != None:
             masked_area = (masked_count * cell_area) / sq_km_converter
         else:
@@ -342,14 +345,14 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
         TN_area = None
         FN_area = None
         area = None
-        
+
         predPositive_area = None
         predNegative_area = None
         obsPositive_area =  None
         obsNegative_area =  None
         positiveDiff_area = None
         MCC = None
-        
+
     if total_population == 0:
         predPositive_perc, predNegative_perc, obsPositive_perc, obsNegative_perc , positiveDiff_perc = "NA", "NA", "NA", "NA", "NA"
     else:
@@ -357,32 +360,32 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
         predNegative_perc = (predNegative / total_population) * 100
         obsPositive_perc = (obsPositive / total_population) * 100
         obsNegative_perc = (obsNegative / total_population) * 100
-        
+
         positiveDiff_perc = predPositive_perc - obsPositive_perc
-        
+
     if total_population == 0:
         prevalence = "NA"
     else:
         prevalence = (true_positives + false_negatives) / total_population
-    
+
     try:
         PPV = true_positives / predPositive
     except ZeroDivisionError:
         PPV = "NA"
-    
+
     try:
         NPV = true_negatives / predNegative
     except ZeroDivisionError:
         NPV = "NA"
-    
+
     try:
         TNR = true_negatives / obsNegative
     except ZeroDivisionError:
         TNR = "NA"
-        
+
     try:
         TPR = true_positives / obsPositive
-        
+
     except ZeroDivisionError:
         TPR = "NA"
 
@@ -390,12 +393,12 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
         Bal_ACC = np.mean([TPR,TNR])
     except TypeError:
         Bal_ACC = "NA"
-    
+
     if total_population == 0:
         ACC = "NA"
     else:
         ACC = (true_positives + true_negatives) / total_population
-    
+
     try:
         F1_score = (2*true_positives) / (2*true_positives + false_positives + false_negatives)
     except ZeroDivisionError:
@@ -407,7 +410,7 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
                         'false_positives_count': int(false_positives),
                         'contingency_tot_count': int(total_population),
                         'cell_area_m2': cell_area,
-                        
+
                         'TP_area_km2': TP_area,
                         'FP_area_km2': FP_area,
                         'TN_area_km2': TN_area,
@@ -422,15 +425,15 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
 
                         'CSI': CSI,
                         'FAR': FAR,
-                        'TPR': TPR,                        
-                        'TNR': TNR,                        
-                        
+                        'TPR': TPR,
+                        'TNR': TNR,
+
                         'PPV': PPV,
                         'NPV': NPV,
                         'ACC': ACC,
                         'Bal_ACC': Bal_ACC,
                         'MCC': MCC,
-                        'EQUITABLE_THREAT_SCORE': EQUITABLE_THREAT_SCORE, 
+                        'EQUITABLE_THREAT_SCORE': EQUITABLE_THREAT_SCORE,
                         'PREVALENCE': prevalence,
                         'BIAS': BIAS,
                         'F1_SCORE': F1_score,
@@ -444,11 +447,11 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
                         'obsPositive_perc': obsPositive_perc,
                         'obsNegative_perc': obsNegative_perc,
                         'positiveDiff_perc': positiveDiff_perc,
-  
+
                         'masked_count': int(masked_count),
                         'masked_perc': masked_perc,
                         'masked_area_km2': masked_area,
-                        
+
                         }
 
     return stats_dictionary
@@ -461,15 +464,15 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
         1: False Negative
         2: False Positive
         3: True Positive
-        
+
     Args:
         benchmark_raster_path (str): Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
         predicted_raster_path (str): Path to the predicted raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
-    
+
     Returns:
         contingency_table_dictionary (dict): A Python dictionary of a contingency table. Key/value pair formatted as:
                                             {true_negatives: int, false_negatives: int, false_positives: int, true_positives: int}
-    
+
     """
     from rasterio.warp import reproject, Resampling
     import rasterio
@@ -478,72 +481,72 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
     import rasterio.mask
     import geopandas as gpd
     from shapely.geometry import box
-            
+
     print("-----> Evaluating performance across the total area...")
     # Load rasters.
     benchmark_src = rasterio.open(benchmark_raster_path)
     predicted_src = rasterio.open(predicted_raster_path)
     predicted_array = predicted_src.read(1)
-    
+
     benchmark_array_original = benchmark_src.read(1)
-    
+
     if benchmark_array_original.shape != predicted_array.shape:
         benchmark_array = np.empty(predicted_array.shape, dtype=np.int8)
-        
-        reproject(benchmark_array_original, 
+
+        reproject(benchmark_array_original,
               destination = benchmark_array,
-              src_transform = benchmark_src.transform, 
+              src_transform = benchmark_src.transform,
               src_crs = benchmark_src.crs,
               src_nodata = benchmark_src.nodata,
-              dst_transform = predicted_src.transform, 
+              dst_transform = predicted_src.transform,
               dst_crs = predicted_src.crs,
               dst_nodata = benchmark_src.nodata,
               dst_resolution = predicted_src.res,
               resampling = Resampling.nearest)
-    
+
     predicted_array_raw = predicted_src.read(1)
-    
+
     # Align the benchmark domain to the modeled domain.
     benchmark_array = np.where(predicted_array==predicted_src.nodata, 10, benchmark_array)
-            
-    # Ensure zeros and ones for binary comparison. Assume that positive values mean flooding and 0 or negative values mean dry. 
+
+    # Ensure zeros and ones for binary comparison. Assume that positive values mean flooding and 0 or negative values mean dry.
     predicted_array = np.where(predicted_array==predicted_src.nodata, 10, predicted_array)  # Reclassify NoData to 10
     predicted_array = np.where(predicted_array<0, 0, predicted_array)
     predicted_array = np.where(predicted_array>0, 1, predicted_array)
-    
-    benchmark_array = np.where(benchmark_array==benchmark_src.nodata, 10, benchmark_array)  # Reclassify NoData to 10        
+
+    benchmark_array = np.where(benchmark_array==benchmark_src.nodata, 10, benchmark_array)  # Reclassify NoData to 10
 
     agreement_array = np.add(benchmark_array, 2*predicted_array)
     agreement_array = np.where(agreement_array>4, 10, agreement_array)
-    
+
     del benchmark_src, benchmark_array, predicted_array, predicted_array_raw
 
     # Loop through exclusion masks and mask the agreement_array.
     if mask_dict != {}:
         for poly_layer in mask_dict:
-            
+
             operation = mask_dict[poly_layer]['operation']
-            
+
             if operation == 'exclude':
-                
+
                 poly_path = mask_dict[poly_layer]['path']
                 buffer_val = mask_dict[poly_layer]['buffer']
-                
+
                 reference = predicted_src
-                            
+
                 bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
-                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).  
+                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).
                 poly_all = gpd.read_file(poly_path, bbox = bounding_box)
-                
+
                 # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
                 if poly_all.empty:
                     continue
-                
+
                 print("-----> Masking at " + poly_layer + "...")
                 #Project layer to reference crs.
                 poly_all_proj = poly_all.to_crs(reference.crs)
                 # check if there are any lakes within our reference raster extent.
-                if poly_all_proj.empty: 
+                if poly_all_proj.empty:
                     #If no features within reference raster extent, create a zero array of same shape as reference raster.
                     poly_mask = np.zeros(reference.shape)
                 else:
@@ -554,7 +557,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                     else:
                         #If  features are present and a buffer is passed, assign buffered geometry to variable.
                         geometry = poly_all_proj.buffer(buffer_val)
-                        
+
                     #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
                     in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
                     #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
@@ -562,12 +565,12 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
 
                     # Perform mask.
                     masked_agreement_array = np.where(poly_mask == 1, 4, agreement_array)
-                    
+
                     # Get rid of masked values outside of the modeled domain.
                     agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
-                    
+
     contingency_table_dictionary = {}  # Initialize empty dictionary.
-    
+
     # Only write the agreement raster if user-specified.
     if agreement_raster != None:
         with rasterio.Env():
@@ -575,15 +578,15 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
             profile.update(nodata=10)
             with rasterio.open(agreement_raster, 'w', **profile) as dst:
                 dst.write(agreement_array, 1)
-         
+
         # Write legend text file
         legend_txt = os.path.join(os.path.split(agreement_raster)[0], 'read_me.txt')
-        
+
         from datetime import datetime
-        
+
         now = datetime.now()
         current_time = now.strftime("%m/%d/%Y %H:%M:%S")
-                
+
         with open(legend_txt, 'w') as f:
             f.write("%s\n" % '0: True Negative')
             f.write("%s\n" % '1: False Negative')
@@ -591,7 +594,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
             f.write("%s\n" % '3: True Positive')
             f.write("%s\n" % '4: Masked area (excluded from contingency table analysis). Mask layers: {mask_dict}'.format(mask_dict=mask_dict))
             f.write("%s\n" % 'Results produced at: {current_time}'.format(current_time=current_time))
-                          
+
     # Store summed pixel counts in dictionary.
     contingency_table_dictionary.update({'total_area':{'true_negatives': int((agreement_array == 0).sum()),
                                                       'false_negatives': int((agreement_array == 1).sum()),
@@ -599,34 +602,34 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                                                       'true_positives': int((agreement_array == 3).sum()),
                                                       'masked_count': int((agreement_array == 4).sum()),
                                                       'file_handle': 'total_area'
-                                                     
-                                                      }})                               
-        
+
+                                                      }})
+
     # After agreement_array is masked with default mask layers, check for inclusion masks in mask_dict.
     if mask_dict != {}:
         for poly_layer in mask_dict:
-            
+
             operation = mask_dict[poly_layer]['operation']
-            
+
             if operation == 'include':
                 poly_path = mask_dict[poly_layer]['path']
                 buffer_val = mask_dict[poly_layer]['buffer']
 
                 reference = predicted_src
-                            
+
                 bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
-                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).  
+                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).
                 poly_all = gpd.read_file(poly_path, bbox = bounding_box)
-                
+
                 # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
                 if poly_all.empty:
                     continue
-                
+
                 print("-----> Evaluating performance at " + poly_layer + "...")
                 #Project layer to reference crs.
                 poly_all_proj = poly_all.to_crs(reference.crs)
                 # check if there are any lakes within our reference raster extent.
-                if poly_all_proj.empty: 
+                if poly_all_proj.empty:
                     #If no features within reference raster extent, create a zero array of same shape as reference raster.
                     poly_mask = np.zeros(reference.shape)
                 else:
@@ -637,7 +640,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                     else:
                         #If  features are present and a buffer is passed, assign buffered geometry to variable.
                         geometry = poly_all_proj.buffer(buffer_val)
-                        
+
                     #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
                     in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
                     #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
@@ -645,15 +648,15 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
 
                     # Perform mask.
                     masked_agreement_array = np.where(poly_mask == 0, 4, agreement_array)  # Changed to poly_mask == 0
-                    
+
                     # Get rid of masked values outside of the modeled domain.
                     temp_agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
-                    
+
                     if buffer_val == None:  # The buffer used is added to filename, and 0 is easier to read than None.
                         buffer_val = 0
-                    
+
                     poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
-                    
+
                     # Write the layer_agreement_raster.
                     layer_agreement_raster = os.path.join(os.path.split(agreement_raster)[0], poly_handle + '_agreement.tif')
                     with rasterio.Env():
@@ -661,7 +664,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                         profile.update(nodata=10)
                         with rasterio.open(layer_agreement_raster, 'w', **profile) as dst:
                             dst.write(temp_agreement_array, 1)
-                    
+
 
                     # Store summed pixel counts in dictionary.
                     contingency_table_dictionary.update({poly_handle:{'true_negatives': int((temp_agreement_array == 0).sum()),
@@ -673,4 +676,366 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                                                                       }})
 
     return contingency_table_dictionary
-    
+########################################################################
+########################################################################
+#Functions related to categorical fim and ahps evaluation
+########################################################################
+def get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None ):
+    '''
+    Retrieve metadata for a site or list of sites.
+
+    Parameters
+    ----------
+    metadata_url : STR
+        metadata base URL.
+    select_by : STR
+        Location search option.
+    selector : LIST
+        Value to match location data against. Supplied as a LIST.
+    must_include : STR, optional
+        What attributes are required to be valid response. The default is None.
+    upstream_trace_distance : INT, optional
+        Distance in miles upstream of site to trace NWM network. The default is None.
+    downstream_trace_distance : INT, optional
+        Distance in miles downstream of site to trace NWM network. The default is None.
+
+    Returns
+    -------
+    metadata_list : LIST
+        Dictionary or list of dictionaries containing metadata at each site.
+    metadata_dataframe : Pandas DataFrame
+        Dataframe of metadata for each site.
+
+    '''
+
+    #Format selector variable in case multiple selectors supplied
+    format_selector = '%2C'.join(selector)
+    #Define the url
+    url = f'{metadata_url}/{select_by}/{format_selector}/'
+    #Assign optional parameters to a dictionary
+    params = {}
+    params['must_include'] = must_include
+    params['upstream_trace_distance'] = upstream_trace_distance
+    params['downstream_trace_distance'] = downstream_trace_distance
+    #Request data from url
+    response = requests.get(url, params = params)
+    if response.ok:
+        #Convert data response to a json
+        metadata_json = response.json()
+        #Get the count of returned records
+        location_count = metadata_json['_metrics']['location_count']
+        #Get metadata
+        metadata_list = metadata_json['locations']
+        #Add timestamp of WRDS retrieval
+        timestamp = response.headers['Date']
+        #get crosswalk info (always last dictionary in list)
+        *metadata_list, crosswalk_info = metadata_list
+        #Update each dictionary with timestamp and crosswalk info
+        for metadata in metadata_list:
+            metadata.update({"wrds_timestamp": timestamp})
+            metadata.update(crosswalk_info)
+        #If count is 1
+        if location_count == 1:
+            metadata_list = metadata_json['locations'][0]
+        metadata_dataframe = pd.json_normalize(metadata_list)
+        #Replace all periods with underscores in column names
+        metadata_dataframe.columns = metadata_dataframe.columns.str.replace('.','_')
+    else:
+        #if request was not succesful, print error message.
+        print(f'Code: {response.status_code}\nMessage: {response.reason}\nURL: {response.url}')
+        #Return empty outputs
+        metadata_list = []
+        metadata_dataframe = pd.DataFrame()
+    return metadata_list, metadata_dataframe
+
+########################################################################
+#Function to assign HUC code using the WBD spatial layer using a spatial join
+########################################################################
+def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
+    '''
+    Assigns the proper FIM HUC 08 code to each site in the input DataFrame.
+    Converts input DataFrame to a GeoDataFrame using the lat/lon attributes
+    with sites containing null lat/lon removed. Reprojects GeoDataFrame
+    to same CRS as the HUC 08 layer. Performs a spatial join to assign the
+    HUC 08 layer to the GeoDataFrame. Sites that are not assigned a HUC
+    code removed as well as sites in Alaska and Canada.
+
+    Parameters
+    ----------
+    metadata_list: List of Dictionaries
+        Output list from get_metadata
+    wbd_huc8_path : pathlib Path
+        Path to HUC8 wbd layer (assumed to be geopackage format)
+    retain_attributes ; Bool OR List
+        Flag to define attributes of output GeoDataBase. If True, retain
+        all attributes. If False, the site metadata will be trimmed to a
+        default list. If a list of desired attributes is supplied these
+        will serve as the retained attributes.
+    Returns
+    -------
+    dictionary : DICT
+        Dictionary with HUC (key) and corresponding AHPS codes (values).
+    all_gdf: GeoDataFrame
+        GeoDataFrame of all NWS_LID sites.
+
+    '''
+    #Import huc8 layer as geodataframe and retain necessary columns
+    huc8 = gpd.read_file(wbd_huc8_path, layer = 'WBDHU8')
+    huc8 = huc8[['HUC8','name','states', 'geometry']]
+    #Define EPSG codes for possible usgs latlon datum names (NAD83WGS84 assigned NAD83)
+    crs_lookup ={'NAD27':'EPSG:4267', 'NAD83':'EPSG:4269', 'NAD83WGS84': 'EPSG:4269'}
+    #Create empty geodataframe and define CRS for potential horizontal datums
+    metadata_gdf = gpd.GeoDataFrame()
+    #Iterate through each site
+    for metadata in metadata_list:
+        #Convert metadata to json
+        df = pd.json_normalize(metadata)
+        #Columns have periods due to nested dictionaries
+        df.columns = df.columns.str.replace('.', '_')
+        #Drop any metadata sites that don't have lat/lon populated
+        df.dropna(subset = ['identifiers_nws_lid','usgs_data_latitude','usgs_data_longitude'], inplace = True)
+        #If dataframe still has data
+        if not df.empty:
+            #Get horizontal datum (use usgs) and assign appropriate EPSG code
+            h_datum = df.usgs_data_latlon_datum_name.item()
+            src_crs = crs_lookup[h_datum]
+            #Convert dataframe to geodataframe using lat/lon (USGS)
+            site_gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.usgs_data_longitude, df.usgs_data_latitude), crs =  src_crs)
+            #Reproject to huc 8 crs
+            site_gdf = site_gdf.to_crs(huc8.crs)
+            #Append site geodataframe to metadata geodataframe
+            metadata_gdf = metadata_gdf.append(site_gdf, ignore_index = True)
+
+    #Trim metadata to only have certain fields.
+    if not retain_attributes:
+        metadata_gdf = metadata_gdf[['identifiers_nwm_feature_id', 'identifiers_nws_lid', 'geometry']]
+    #If a list of attributes is supplied then use that list.
+    elif isinstance(retain_attributes,list):
+        metadata_gdf = metadata_gdf[retain_attributes]
+
+    #Perform a spatial join to get the WBD HUC 8 assigned to each AHPS
+    joined_gdf = gpd.sjoin(metadata_gdf, huc8, how = 'inner', op = 'intersects', lsuffix = 'ahps', rsuffix = 'wbd')
+    joined_gdf = joined_gdf.drop(columns = 'index_wbd')
+
+    #Remove all Alaska HUCS (Not in NWM v2.0 domain)
+    joined_gdf = joined_gdf[~joined_gdf.states.str.contains('AK')]
+
+    #Create a dictionary of huc [key] and nws_lid[value]
+    dictionary = joined_gdf.groupby('HUC8')['identifiers_nws_lid'].apply(list).to_dict()
+
+    return dictionary, joined_gdf
+
+########################################################################
+def mainstem_nwm_segs(metadata_url, list_of_sites):
+    '''
+    Define the mainstems network. Currently a 4 pass approach that probably needs refined.
+    Once a final method is decided the code can be shortened. Passes are:
+        1) Search downstream of gages designated as upstream. This is done to hopefully reduce the issue of mapping starting at the nws_lid. 91038 segments
+        2) Search downstream of all LID that are rfc_forecast_point = True. Additional 48,402 segments
+        3) Search downstream of all evaluated sites (sites with detailed FIM maps) Additional 222 segments
+        4) Search downstream of all sites in HI/PR (locations have no rfc_forecast_point = True) Additional 408 segments
+
+    Parameters
+    ----------
+    metadata_url : STR
+        URL of API.
+    list_of_sites : LIST
+        List of evaluated sites.
+
+    Returns
+    -------
+    ms_nwm_segs_set : SET
+        Mainstems network segments as a set.
+
+    '''
+
+    #Define the downstream trace distance
+    downstream_trace_distance = 'all'
+
+    #Trace downstream from all 'headwater' usgs gages
+    select_by = 'tag'
+    selector = ['usgs_gages_ii_ref_headwater']
+    must_include = None
+    gages_list, gages_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
+
+    #Trace downstream from all rfc_forecast_point.
+    select_by = 'nws_lid'
+    selector = ['all']
+    must_include = 'nws_data.rfc_forecast_point'
+    fcst_list, fcst_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
+
+    #Trace downstream from all evaluated ahps sites.
+    select_by = 'nws_lid'
+    selector = list_of_sites
+    must_include = None
+    eval_list, eval_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
+
+    #Trace downstream from all sites in HI/PR.
+    select_by = 'state'
+    selector = ['HI','PR']
+    must_include = None
+    islands_list, islands_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
+
+    #Combine all lists of metadata dictionaries into a single list.
+    combined_lists = gages_list + fcst_list + eval_list + islands_list
+    #Define list that will contain all segments listed in metadata.
+    all_nwm_segments = []
+    #For each lid metadata dictionary in list
+    for lid in combined_lists:
+        #get all downstream segments
+        downstream_nwm_segs = lid.get('downstream_nwm_features')
+        #Append downstream segments
+        if downstream_nwm_segs:
+            all_nwm_segments.extend(downstream_nwm_segs)
+        #Get the nwm feature id associated with the location
+        location_nwm_seg = lid.get('identifiers').get('nwm_feature_id')
+        if location_nwm_seg:
+            #Append nwm segment (conver to list)
+            all_nwm_segments.extend([location_nwm_seg])
+    #Remove duplicates by assigning to a set.
+    ms_nwm_segs_set = set(all_nwm_segments)
+
+    return ms_nwm_segs_set
+
+##############################################################################
+#Function to create list of NWM segments
+###############################################################################
+def get_nwm_segs(metadata):
+    '''
+    Using the metadata output from "get_metadata", output the NWM segments.
+
+    Parameters
+    ----------
+    metadata : DICT
+        Dictionary output from "get_metadata" function.
+
+    Returns
+    -------
+    all_segments : LIST
+        List of all NWM segments.
+
+    '''
+
+    nwm_feature_id = metadata.get('identifiers').get('nwm_feature_id')
+    upstream_nwm_features = metadata.get('upstream_nwm_features')
+    downstream_nwm_features = metadata.get('downstream_nwm_features')
+
+    all_segments = []
+    #Convert NWM feature id segment to a list (this is always a string or empty)
+    if nwm_feature_id:
+        nwm_feature_id = [nwm_feature_id]
+        all_segments.extend(nwm_feature_id)
+    #Add all upstream segments (always a list or empty)
+    if upstream_nwm_features:
+        all_segments.extend(upstream_nwm_features)
+    #Add all downstream segments (always a list or empty)
+    if downstream_nwm_features:
+        all_segments.extend(downstream_nwm_features)
+
+    return all_segments
+
+#######################################################################
+#Thresholds
+#######################################################################
+def get_thresholds(threshold_url, location_ids, physical_element = 'all', threshold = 'all', bypass_source_flag = False):
+    '''
+    Get nws_lid threshold stages and flows (i.e. bankfull, action, minor,
+    moderate, major). Returns a dictionary for stages and one for flows.
+
+    Parameters
+    ----------
+    threshold_url : STR
+        WRDS threshold API.
+    location_ids : STR
+        nws_lid code (only a single code).
+    physical_element : STR, optional
+        Physical element option. The default is 'all'.
+    threshold : STR, optional
+        Threshold option. The default is 'all'.
+    bypass_source_flag : BOOL, optional
+        Special case if calculated values are not available (e.g. no rating
+        curve is available) then this allows for just a stage to be returned.
+        Used in case a flow is already known from another source, such as
+        a model. The default is False.
+
+    Returns
+    -------
+    stages : DICT
+        Dictionary of stages at each threshold.
+    flows : DICT
+        Dictionary of flows at each threshold.
+
+    '''
+
+    url = f'{threshold_url}/{physical_element}/{threshold}/{location_ids}'
+    response = requests.get(url)
+    if response.ok:
+        thresholds_json = response.json()
+        #Get metadata
+        thresholds_info = thresholds_json['stream_thresholds']
+        #Initialize stages/flows dictionaries
+        stages = {}
+        flows = {}
+        #Check if thresholds information is populated. If site is non-existent thresholds info is blank
+        if thresholds_info:
+            #Get all rating sources and corresponding indexes in a dictionary
+            rating_sources = {i.get('calc_flow_values').get('rating_curve').get('source'): index for index, i in enumerate(thresholds_info)}
+            #Get threshold data use USGS Rating Depot (priority) otherwise NRLDB.
+            if 'USGS Rating Depot' in rating_sources:
+                threshold_data = thresholds_info[rating_sources['USGS Rating Depot']]
+            elif 'NRLDB' in rating_sources:
+                threshold_data = thresholds_info[rating_sources['NRLDB']]
+            #If neither USGS or NRLDB is available
+            else:
+                #A flag option for cases where only a stage is needed for USGS scenario where a rating curve source is not available yet stages are available for the site. If flag is enabled, then stages are retrieved from the first record in thresholds_info. Typically the flows will not be populated as no rating curve is available. Flag should only be enabled when flows are already supplied by source (e.g. USGS) and threshold stages are needed.
+                if bypass_source_flag:
+                    threshold_data = thresholds_info[0]
+                else:
+                    threshold_data = []
+            #Get stages and flows for each threshold
+            if threshold_data:
+                stages = threshold_data['stage_values']
+                flows = threshold_data['calc_flow_values']
+                #Add source information to stages and flows. Flows source inside a nested dictionary. Remove key once source assigned to flows.
+                stages['source'] = threshold_data['metadata']['threshold_source']
+                flows['source'] = flows['rating_curve']['source']
+                flows.pop('rating_curve', None)
+                #Add timestamp WRDS data was retrieved.
+                stages['wrds_timestamp'] = response.headers['Date']
+                flows['wrds_timestamp'] = response.headers['Date']
+    return stages, flows
+
+########################################################################
+# Function to write flow file
+########################################################################
+def flow_data(segments, flows, convert_to_cms = True):
+    '''
+    Given a list of NWM segments and a flow value in cfs, convert flow to
+    cms and return a DataFrame that is set up for export to a flow file.
+
+    Parameters
+    ----------
+    segments : LIST
+        List of NWM segments.
+    flows : FLOAT
+        Flow in CFS.
+    convert_to_cms : BOOL
+        Flag to indicate if supplied flows should be converted to metric.
+        Default value is True (assume input flows are CFS).
+
+    Returns
+    -------
+    flow_data : DataFrame
+        Dataframe ready for export to a flow file.
+
+    '''
+    if convert_to_cms:
+        #Convert cfs to cms
+        cfs_to_cms = 0.3048**3
+        flows_cms = round(flows * cfs_to_cms,2)
+    else:
+        flows_cms = round(flows,2)
+
+    flow_data = pd.DataFrame({'feature_id':segments, 'discharge':flows_cms})
+    flow_data = flow_data.astype({'feature_id' : int , 'discharge' : float})
+    return flow_data

From d43bd8b6ab433f68729fb9337bcb7cb6132461e2 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 24 Mar 2021 10:21:04 -0500
Subject: [PATCH 054/359] Patch to synthesize_test_cases.py

Bug fix to synthesize_test_cases.py to allow comparison between testing version and official versions.
---
 CHANGELOG.md                   |   9 ++
 tools/synthesize_test_cases.py | 202 ++++++++++++++++++---------------
 2 files changed, 122 insertions(+), 89 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5601ba729..07be1fdb7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
+
+## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
+
+ Patch to synthesize_test_cases.py.
+ 
+ ### Changes
+ - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
+<br/><br/>
+
 ## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
 
  Preprocessing of flow files for Categorical FIM.
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index f3d02192c..06f55b4a0 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -10,7 +10,7 @@
 from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
 
 
-def create_master_metrics_csv(master_metrics_csv_output):
+def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
 
     # Construct header
     metrics_to_write = ['true_negatives_count',
@@ -61,10 +61,13 @@ def create_master_metrics_csv(master_metrics_csv_output):
 
     versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
 
-    for benchmark_source in ['ble', 'nws', 'usgs']:
+    if dev_comparison != None:
+        iteration_list = ['official', 'comparison']
+    if dev_comparison == None:
+        iteration_list = ['official']
 
+    for benchmark_source in ['ble', 'nws', 'usgs']:
         benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
-
         if benchmark_source == 'ble':
             test_cases_list = os.listdir(benchmark_test_case_dir)
 
@@ -73,43 +76,51 @@ def create_master_metrics_csv(master_metrics_csv_output):
                     int(test_case.split('_')[0])
 
                     huc = test_case.split('_')[0]
-                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-
-                    for magnitude in ['100yr', '500yr']:
-                        for version in versions_to_aggregate:
-                            if '_fr' in version:
-                                extent_config = 'FR'
-                            elif '_ms' in version:
-                                extent_config = 'MS'
-                            else:
-                                extent_config = 'FR'
-                            if "_c" in version and version.split('_c')[1] == "":
-                                calibrated = "yes"
-                            else:
-                                calibrated = "no"
-                            version_dir = os.path.join(official_versions, version)
-                            magnitude_dir = os.path.join(version_dir, magnitude)
-
-                            if os.path.exists(magnitude_dir):
-                                magnitude_dir_list = os.listdir(magnitude_dir)
-                                for f in magnitude_dir_list:
-                                    if '.json' in f:
-                                        flow = 'NA'
-                                        nws_lid = "NA"
-                                        benchmark_source = 'ble'
-                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
-                                        full_json_path = os.path.join(magnitude_dir, f)
-                                        if os.path.exists(full_json_path):
-                                            stats_dict = json.load(open(full_json_path))
-                                            for metric in metrics_to_write:
-                                                sub_list_to_append.append(stats_dict[metric])
-                                            sub_list_to_append.append(full_json_path)
-                                            sub_list_to_append.append(flow)
-                                            sub_list_to_append.append(benchmark_source)
-                                            sub_list_to_append.append(extent_config)
-                                            sub_list_to_append.append(calibrated)
-
-                                            list_to_write.append(sub_list_to_append)
+                    
+                    for iteration in iteration_list:
+                        
+                        if iteration == "official":
+                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
+                            versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
+                        if iteration == "comparison":
+                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
+                            versions_to_aggregate = [dev_comparison]
+        
+                        for magnitude in ['100yr', '500yr']:
+                            for version in versions_to_aggregate:
+                                if '_fr' in version:
+                                    extent_config = 'FR'
+                                elif '_ms' in version:
+                                    extent_config = 'MS'
+                                else:
+                                    extent_config = 'FR'
+                                if "_c" in version and version.split('_c')[1] == "":
+                                    calibrated = "yes"
+                                else:
+                                    calibrated = "no"
+                                version_dir = os.path.join(versions_to_crawl, version)
+                                magnitude_dir = os.path.join(version_dir, magnitude)
+    
+                                if os.path.exists(magnitude_dir):
+                                    magnitude_dir_list = os.listdir(magnitude_dir)
+                                    for f in magnitude_dir_list:
+                                        if '.json' in f:
+                                            flow = 'NA'
+                                            nws_lid = "NA"
+                                            benchmark_source = 'ble'
+                                            sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                            full_json_path = os.path.join(magnitude_dir, f)
+                                            if os.path.exists(full_json_path):
+                                                stats_dict = json.load(open(full_json_path))
+                                                for metric in metrics_to_write:
+                                                    sub_list_to_append.append(stats_dict[metric])
+                                                sub_list_to_append.append(full_json_path)
+                                                sub_list_to_append.append(flow)
+                                                sub_list_to_append.append(benchmark_source)
+                                                sub_list_to_append.append(extent_config)
+                                                sub_list_to_append.append(calibrated)
+    
+                                                list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
 
@@ -121,54 +132,62 @@ def create_master_metrics_csv(master_metrics_csv_output):
                     int(test_case.split('_')[0])
 
                     huc = test_case.split('_')[0]
-                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-
-                    for magnitude in ['action', 'minor', 'moderate', 'major']:
-                        for version in versions_to_aggregate:
-                            if '_fr' in version:
-                                extent_config = 'FR'
-                            elif '_ms' in version:
-                                extent_config = 'MS'
-                            else:
-                                extent_config = 'FR'
-                            if "_c" in version and version.split('_c')[1] == "":
-                                calibrated = "yes"
-                            else:
-                                calibrated = "no"
-
-                            version_dir = os.path.join(official_versions, version)
-                            magnitude_dir = os.path.join(version_dir, magnitude)
-                            if os.path.exists(magnitude_dir):
-                                magnitude_dir_list = os.listdir(magnitude_dir)
-                                for f in magnitude_dir_list:
-                                    if '.json' in f and 'total_area' not in f:
-                                        nws_lid = f[:5]
-                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
-                                        full_json_path = os.path.join(magnitude_dir, f)
-                                        flow = ''
-                                        if os.path.exists(full_json_path):
-
-                                            # Get flow used to map.
-                                            flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
-                                            if os.path.exists(flow_file):
-                                                with open(flow_file, newline='') as csv_file:
-                                                    reader = csv.reader(csv_file)
-                                                    next(reader)
-                                                    for row in reader:
-                                                        flow = row[1]
-                                                    if nws_lid == 'mcc01':
-                                                        print(flow)
-
-                                            stats_dict = json.load(open(full_json_path))
-                                            for metric in metrics_to_write:
-                                                sub_list_to_append.append(stats_dict[metric])
-                                            sub_list_to_append.append(full_json_path)
-                                            sub_list_to_append.append(flow)
-                                            sub_list_to_append.append(benchmark_source)
-                                            sub_list_to_append.append(extent_config)
-                                            sub_list_to_append.append(calibrated)
-
-                                            list_to_write.append(sub_list_to_append)
+                    
+                    for iteration in iteration_list:
+                        
+                        if iteration == "official":
+                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
+                            versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
+                        if iteration == "comparison":
+                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
+                            versions_to_aggregate = [dev_comparison]
+
+                        for magnitude in ['action', 'minor', 'moderate', 'major']:
+                            for version in versions_to_aggregate:
+                                if '_fr' in version:
+                                    extent_config = 'FR'
+                                elif '_ms' in version:
+                                    extent_config = 'MS'
+                                else:
+                                    extent_config = 'FR'
+                                if "_c" in version and version.split('_c')[1] == "":
+                                    calibrated = "yes"
+                                else:
+                                    calibrated = "no"
+    
+                                version_dir = os.path.join(versions_to_crawl, version)
+                                magnitude_dir = os.path.join(version_dir, magnitude)
+                                if os.path.exists(magnitude_dir):
+                                    magnitude_dir_list = os.listdir(magnitude_dir)
+                                    for f in magnitude_dir_list:
+                                        if '.json' in f and 'total_area' not in f:
+                                            nws_lid = f[:5]
+                                            sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                            full_json_path = os.path.join(magnitude_dir, f)
+                                            flow = ''
+                                            if os.path.exists(full_json_path):
+    
+                                                # Get flow used to map.
+                                                flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                                                if os.path.exists(flow_file):
+                                                    with open(flow_file, newline='') as csv_file:
+                                                        reader = csv.reader(csv_file)
+                                                        next(reader)
+                                                        for row in reader:
+                                                            flow = row[1]
+                                                        if nws_lid == 'mcc01':
+                                                            print(flow)
+    
+                                                stats_dict = json.load(open(full_json_path))
+                                                for metric in metrics_to_write:
+                                                    sub_list_to_append.append(stats_dict[metric])
+                                                sub_list_to_append.append(full_json_path)
+                                                sub_list_to_append.append(flow)
+                                                sub_list_to_append.append(benchmark_source)
+                                                sub_list_to_append.append(extent_config)
+                                                sub_list_to_append.append(calibrated)
+    
+                                                list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
 
@@ -307,4 +326,9 @@ def process_alpha_test(args):
 
     # Do aggregate_metrics.
     print("Creating master metrics CSV...")
-    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv)
+    
+    if config == 'DEV':
+        dev_comparison = fim_version + "_" + special_string
+    else:
+        dev_comparison = None
+    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, dev_comparison=dev_comparison)

From 34123ddb37d6b6c44caf3572d8683d6b433ebca1 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Thu, 25 Mar 2021 08:51:04 -0500
Subject: [PATCH 055/359] Improvements to CatFIM service source data generation

Improvements to CatFIM service source data generation.

- Renamed generate_categorical_fim.py to generate_categorical_fim_mapping.py.
- Updated the status outputs of the nws_lid_sites layer and saved it in the same directory as the merged catfim_library layer.
- Additional stability fixes (such as improved compatability with WRDS updates).
- Added generate_categorical_fim.py to wrap generate_categorical_fim_flows.py and generate_categorical_fim_mapping.py.
- Create new nws_lid_sites shapefile located in same directory as the catfim_library shapefile.

This resolves #305, resolves #321, and resolves #322.
---
 CHANGELOG.md                              |  13 +
 tools/generate_categorical_fim.py         | 417 ++++++----------------
 tools/generate_categorical_fim_flows.py   | 183 +++++-----
 tools/generate_categorical_fim_mapping.py | 329 +++++++++++++++++
 tools/inundation.py                       |  13 +-
 tools/tools_shared_functions.py           |  82 +++--
 6 files changed, 592 insertions(+), 445 deletions(-)
 create mode 100755 tools/generate_categorical_fim_mapping.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 07be1fdb7..164ce1a44 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+
+## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
+
+ Improvements to CatFIM service source data generation.
+ 
+ ### Changes
+ - Renamed `generate_categorical_fim.py` to `generate_categorical_fim_mapping.py`.
+ - Updated the status outputs of the `nws_lid_sites layer` and saved it in the same directory as the `merged catfim_library layer`.
+ - Additional stability fixes (such as improved compatability with WRDS updates).
+### Additions
+ - Added `generate_categorical_fim.py` to wrap `generate_categorical_fim_flows.py` and `generate_categorical_fim_mapping.py`.
+ - Create new `nws_lid_sites` shapefile located in same directory as the `catfim_library` shapefile.
+ 
 <br/><br/>
 
 ## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index bec059960..f51bf5aa8 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -1,317 +1,112 @@
 #!/usr/bin/env python3
-
-import sys
-import os
-from multiprocessing import Pool
+import subprocess
 import argparse
-import traceback
-import rasterio
+import time
+from pathlib import Path
 import geopandas as gpd
 import pandas as pd
-import shutil
-from rasterio.features import shapes
-from shapely.geometry.polygon import Polygon
-from shapely.geometry.multipolygon import MultiPolygon
-from inundation import inundate
-sys.path.append('/foss_fim/src')
-from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
-from utils.shared_functions import getDriver
-
-INPUTS_DIR = r'/data/inputs'
-magnitude_list = ['action', 'minor', 'moderate','major', 'record']
-
-# Define necessary variables for inundation()
-hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-mask_type, catchment_poly = 'huc', ''
-
-
-def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif, log_file):
-
-    no_data_list = []
-    procs_list = []
-
-    source_flow_dir_list = os.listdir(source_flow_dir)
-    output_flow_dir_list = os.listdir(fim_run_dir)
-
-    # Log missing hucs
-    missing_hucs = list(set(source_flow_dir_list) - set(output_flow_dir_list))
-    missing_hucs = [huc for huc in missing_hucs if "." not in huc]
-    if len(missing_hucs) > 0:
-        f = open(log_file, 'a+')
-        f.write(f"Missing hucs from output directory: {', '.join(missing_hucs)}\n")
-        f.close()
-
-    # Loop through matching huc directories in the source_flow directory
-    matching_hucs = list(set(output_flow_dir_list) & set(source_flow_dir_list))
-    for huc in matching_hucs:
-
-        if "." not in huc:
-
-            # Get list of AHPS site directories
-            ahps_site_dir = os.path.join(source_flow_dir, huc)
-            ahps_site_dir_list = os.listdir(ahps_site_dir)
-
-            # Map paths to HAND files needed for inundation()
-            fim_run_huc_dir = os.path.join(fim_run_dir, huc)
-            rem = os.path.join(fim_run_huc_dir, 'rem_zeroed_masked.tif')
-            catchments = os.path.join(fim_run_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-            hydroTable =  os.path.join(fim_run_huc_dir, 'hydroTable.csv')
-
-            exit_flag = False  # Default to False.
-
-            # Check if necessary data exist; set exit_flag to True if they don't exist
-            for f in [rem, catchments, hydroTable]:
-                if not os.path.exists(f):
-                    no_data_list.append(f)
-                    exit_flag = True
-
-            # Log missing data
-            if exit_flag == True:
-                f = open(log_file, 'a+')
-                f.write(f"Missing data for: {fim_run_huc_dir}\n")
-                f.close()
-
-            # Map path to huc directory inside out output_cat_fim_dir
-            cat_fim_huc_dir = os.path.join(output_cat_fim_dir, huc)
-            if not os.path.exists(cat_fim_huc_dir):
-                os.mkdir(cat_fim_huc_dir)
-
-            # Loop through AHPS sites
-            for ahps_site in ahps_site_dir_list:
-                # map parent directory for AHPS source data dir and list AHPS thresholds (act, min, mod, maj)
-                ahps_site_parent = os.path.join(ahps_site_dir, ahps_site)
-                thresholds_dir_list = os.listdir(ahps_site_parent)
-
-                # Map parent directory for all inundation output filesoutput files.
-                cat_fim_huc_ahps_dir = os.path.join(cat_fim_huc_dir, ahps_site)
-                if not os.path.exists(cat_fim_huc_ahps_dir):
-                    os.mkdir(cat_fim_huc_ahps_dir)
-
-                # Loop through thresholds/magnitudes and define inundation output files paths
-                for magnitude in thresholds_dir_list:
-
-                    if "." not in magnitude:
-
-                        magnitude_flows_csv = os.path.join(ahps_site_parent, magnitude, 'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv')
-
-                        if os.path.exists(magnitude_flows_csv):
-
-                            output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
-
-                            if depthtif:
-                                output_depth_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_depth.tif')
-                            else:
-                                output_depth_grid = None
-
-                            # Append necessary variables to list for multiprocessing.
-                            procs_list.append([rem, catchments, catchment_poly, magnitude_flows_csv, huc, hydroTable, output_extent_grid, output_depth_grid, ahps_site, magnitude, log_file])
-
-    # Initiate multiprocessing
-    print(f"Running inundation for {len(procs_list)} sites using {number_of_jobs} jobs")
-    pool = Pool(number_of_jobs)
-    pool.map(run_inundation, procs_list)
-
-
-def run_inundation(args):
-
-    rem                 = args[0]
-    catchments          = args[1]
-    catchment_poly      = args[2]
-    magnitude_flows_csv = args[3]
-    huc                 = args[4]
-    hydroTable          = args[5]
-    output_extent_grid  = args[6]
-    output_depth_grid   = args[7]
-    ahps_site           = args[8]
-    magnitude           = args[9]
-    log_file            = args[10]
-
-    try:
-        inundate(rem,catchments,catchment_poly,hydroTable,magnitude_flows_csv,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=output_extent_grid,inundation_polygon=None,
-                 depths=output_depth_grid,out_raster_profile=None,out_vector_profile=None,quiet=True
-                )
-
-    except Exception:
-        # Log errors and their tracebacks
-        f = open(log_file, 'a+')
-        f.write(f"{output_extent_grid} - inundation error: {traceback.format_exc()}\n")
-        f.close()
-
-
-def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_attributes_filename, log_file):
-
-    # Create workspace
-    gpkg_dir = os.path.join(output_cat_fim_dir, 'gpkg')
-    if not os.path.exists(gpkg_dir):
-        os.mkdir(gpkg_dir)
-
-    fim_version  = os.path.basename(output_cat_fim_dir)
-    merged_layer = os.path.join(output_cat_fim_dir, 'catfim_library.gpkg')
-
-    if not os.path.exists(merged_layer): # prevents appending to existing output
-
-        huc_ahps_dir_list = os.listdir(output_cat_fim_dir)
-        skip_list=['errors','logs','gpkg',merged_layer]
-
-        for magnitude in magnitude_list:
-
-            procs_list = []
-
-            # Loop through all categories
-            for huc in huc_ahps_dir_list:
-
-                if huc not in skip_list:
-
-                    huc_dir = os.path.join(output_cat_fim_dir, huc)
-                    ahps_dir_list = os.listdir(huc_dir)
-
-                    # Loop through ahps sites
-                    for ahps_lid in ahps_dir_list:
-                        ahps_lid_dir = os.path.join(huc_dir, ahps_lid)
-
-                        extent_grid = os.path.join(ahps_lid_dir, ahps_lid + '_' + magnitude + '_extent_' + huc + '.tif')
-
-                        if os.path.exists(extent_grid):
-                            procs_list.append([ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename])
-
-                        else:
-                            try:
-                                f = open(log_file, 'a+')
-                                f.write(f"Missing layers: {extent_gpkg}\n")
-                                f.close()
-                            except:
-                                pass
-
-            # Multiprocess with instructions
-            pool = Pool(number_of_jobs)
-            pool.map(reformat_inundation_maps, procs_list)
-
-        # Merge all layers
-        print(f"Merging {len(os.listdir(gpkg_dir))} layers...")
-
-        for layer in os.listdir(gpkg_dir):
-
-            diss_extent_filename = os.path.join(gpkg_dir, layer)
-
-            # Open diss_extent
-            diss_extent = gpd.read_file(diss_extent_filename)
-
-            # Write/append aggregate diss_extent
-            if os.path.isfile(merged_layer):
-                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False, mode='a')
-            else:
-                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False)
-
-            del diss_extent
-
-        shutil.rmtree(gpkg_dir)
-
-    else:
-        print(f"{merged_layer} already exists.")
-
-
-def reformat_inundation_maps(args):
-
-    try:
-        lid                         = args[0]
-        grid_path                   = args[1]
-        gpkg_dir                    = args[2]
-        fim_version                 = args[3]
-        huc                         = args[4]
-        magnitude                   = args[5]
-        nws_lid_attributes_filename = args[6]
-
-        # Convert raster to to shapes
-        with rasterio.open(grid_path) as src:
-            image = src.read(1)
-            mask = image > 0
-
-        # Aggregate shapes
-        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
-
-        # convert list of shapes to polygon
-        extent_poly  = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
-
-        # Dissolve polygons
-        extent_poly_diss = extent_poly.dissolve(by='extent')
-
-        # Update attributes
-        extent_poly_diss = extent_poly_diss.reset_index(drop=True)
-        extent_poly_diss['ahps_lid'] = lid
-        extent_poly_diss['magnitude'] = magnitude
-        extent_poly_diss['version'] = fim_version
-        extent_poly_diss['huc'] = huc
-
-        # Project to Web Mercator
-        extent_poly = extent_poly.to_crs(VIZ_PROJECTION)
-
-        # Join attributes
-        nws_lid_attributes_table = pd.read_csv(nws_lid_attributes_filename, dtype={'huc':str})
-        nws_lid_attributes_table = nws_lid_attributes_table.loc[(nws_lid_attributes_table.magnitude==magnitude) & (nws_lid_attributes_table.nws_lid==lid)]
-
-
-        extent_poly_diss = extent_poly_diss.merge(nws_lid_attributes_table, left_on=['ahps_lid','magnitude','huc'], right_on=['nws_lid','magnitude','huc'])
-
-        extent_poly_diss = extent_poly_diss.drop(columns='nws_lid')
-
-        # Save dissolved multipolygon
-        handle = os.path.split(grid_path)[1].replace('.tif', '')
-
-        diss_extent_filename = os.path.join(gpkg_dir, handle + "_dissolved.gpkg")
-
-        extent_poly_diss["geometry"] = [MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in extent_poly_diss["geometry"]]
-
-        if not extent_poly_diss.empty:
-
-            extent_poly_diss.to_file(diss_extent_filename,driver=getDriver(diss_extent_filename),index=False)
-
-    except Exception as e:
-        # Log and clean out the gdb so it's not merged in later
-        try:
-            f = open(log_file, 'a+')
-            f.write(str(diss_extent_filename) + " - dissolve error: " + str(e))
-            f.close()
-        except:
-            pass
-
-
+from datetime import date
+
+def update_mapping_status(output_mapping_dir, output_flows_dir):
+    '''
+    Updates the status for nws_lids from the flows subdirectory. Status
+    is updated for sites where the inundation.py routine was not able to
+    produce inundation for the supplied flow files. It is assumed that if 
+    an error occured in inundation.py that all flow files for a given site
+    experienced the error as they all would have the same nwm segments.
+
+    Parameters
+    ----------
+    output_mapping_dir : STR
+        Path to the output directory of all inundation maps.
+    output_flows_dir : STR
+        Path to the directory containing all flows.
+
+    Returns
+    -------
+    None.
+
+    '''
+    #Find all LIDs with empty mapping output folders
+    subdirs = [str(i) for i in Path(output_mapping_dir).rglob('**/*') if i.is_dir()]
+    empty_nws_lids = [Path(directory).name for directory in subdirs if not list(Path(directory).iterdir())]
+    
+    #Write list of empty nws_lids to DataFrame, these are sites that failed in inundation.py
+    mapping_df = pd.DataFrame({'nws_lid':empty_nws_lids})
+    mapping_df['did_it_map'] = 'no'
+    mapping_df['map_status'] = ' and all categories failed to map'
+    
+    #Import shapefile output from flows creation 
+    shapefile = Path(output_flows_dir)/'nws_lid_flows_sites.shp'
+    flows_df = gpd.read_file(shapefile)
+    
+    #Join failed sites to flows df    
+    flows_df = flows_df.merge(mapping_df, how = 'left', on = 'nws_lid')
+    
+    #Switch mapped column to no for failed sites and update status
+    flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
+    flows_df.loc[flows_df['did_it_map']=='no','status'] = flows_df['status'] + flows_df['map_status']
+    
+    #Perform pass for HUCs where mapping was skipped due to missing data.
+    flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
+    mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
+    missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
+    #Update status for nws_lid in missing hucs and change mapped attribute to 'no'
+    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
+    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
+            
+    #Clean up GeoDataFrame and rename columns for consistency.
+    flows_df = flows_df.drop(columns = ['did_it_map','map_status'])
+    flows_df = flows_df.rename(columns = {'nws_lid':'ahps_lid'})
+    
+    #Write out to file
+    nws_lid_path = Path(output_mapping_dir) / 'nws_lid_sites.shp'
+    flows_df.to_file(nws_lid_path)
+    
 if __name__ == '__main__':
-
-    # Parse arguments
-    parser = argparse.ArgumentParser(description='Categorical inundation mapping for FOSS FIM.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-s', '--source-flow-dir',help='Path to directory containing flow CSVs to use to generate categorical FIM.',required=True, default="")
-    parser.add_argument('-o', '--output-cat-fim-dir',help='Path to directory where categorical FIM outputs will be written.',required=True, default="")
-    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
-    parser.add_argument('-depthtif','--write-depth-tiff',help='Using this option will write depth TIFFs.',required=False, action='store_true')
-
+    
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Run Categorical FIM')
+    parser.add_argument('-f','--fim_version',help='Name of directory containing outputs of fim_run.sh',required=True)
+    parser.add_argument('-j','--number_of_jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
     args = vars(parser.parse_args())
-
-    fim_run_dir = args['fim_run_dir']
-    source_flow_dir = args['source_flow_dir']
-    output_cat_fim_dir = args['output_cat_fim_dir']
-    number_of_jobs = int(args['number_of_jobs'])
-    depthtif = args['write_depth_tiff']
-
-
-    # Create output directory
-    if not os.path.exists(output_cat_fim_dir):
-        os.mkdir(output_cat_fim_dir)
-
-    # Create log directory
-    log_dir = os.path.join(output_cat_fim_dir, 'logs')
-    if not os.path.exists(log_dir):
-        os.mkdir(log_dir)
-
-    # Create error log path
-    log_file = os.path.join(log_dir, 'errors.log')
-
-    # Map path to points with attributes
-    nws_lid_attributes_filename = os.path.join(source_flow_dir, 'nws_lid_attributes.csv')
-
-    print("Generating Categorical FIM")
-    generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif,log_file)
-
-    print("Aggregating Categorical FIM")
-    post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir,nws_lid_attributes_filename,log_file)
+    
+    #Get arguments
+    fim_version = args['fim_version']
+    number_of_jobs = args['number_of_jobs']
+    
+    ####################################################################
+    #Define default arguments. Modify these if necessary. 
+    today = date.today().strftime('%m%d%Y')
+    fim_run_dir = Path(f'/data/previous_fim/{fim_version}')
+    output_flows_dir = Path(f'/data/catfim/{fim_version}/{today}/flows')
+    output_mapping_dir = Path(f'/data/catfim/{fim_version}/{today}/mapping')
+    nwm_us_search = '10'
+    nwm_ds_search = '10'        
+    write_depth_tiff = False
+    ####################################################################
+    
+    ####################################################################
+    #Run CatFIM scripts in sequence
+    ####################################################################
+    #Generate CatFIM flow files.
+    print('Creating flow files')
+    start = time.time()
+    subprocess.call(['python3','generate_categorical_fim_flows.py', '-w' , str(output_flows_dir), '-u', nwm_us_search, '-d', nwm_ds_search])
+    end = time.time()
+    elapsed_time = (end-start)/60
+    print(f'Finished creating flow files in {elapsed_time} minutes')
+    
+    #Generate CatFIM mapping.
+    print('Begin mapping')
+    start = time.time()
+    subprocess.call(['python3','generate_categorical_fim_mapping.py', '-r' , str(fim_run_dir), '-s', str(output_flows_dir), '-o', str(output_mapping_dir), '-j', str(number_of_jobs)]) 
+    end = time.time()
+    elapsed_time = (end-start)/60
+    print(f'Finished mapping in {elapsed_time} minutes')
+    
+    #Updating Mapping Status
+    print('Updating mapping status')
+    update_mapping_status(str(output_mapping_dir), str(output_flows_dir))
+
+   
\ No newline at end of file
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index 562b3e8e5..ca57e089c 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -11,13 +11,15 @@
 sys.path.append('/foss_fim/src')
 from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
 
-load_dotenv()
-#import variables from .env file
-API_BASE_URL = os.getenv("API_BASE_URL")
-EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
-WBD_LAYER = os.getenv("WBD_LAYER")
+def get_env_paths():
+    load_dotenv()
+    #import variables from .env file
+    API_BASE_URL = os.getenv("API_BASE_URL")
+    EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+    WBD_LAYER = os.getenv("WBD_LAYER")
+    return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER
 
-def static_flow_lids(workspace, nwm_us_search, nwm_ds_search):
+def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     '''
     This will create static flow files for all nws_lids and save to the 
     workspace directory with the following format:
@@ -45,19 +47,20 @@ def static_flow_lids(workspace, nwm_us_search, nwm_ds_search):
     -------
     None.
 
-    '''    
+    '''
+    
     all_start = time.time()
     #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
     workspace = Path(workspace)
     nwm_us_search = int(nwm_us_search)
     nwm_ds_search = int(nwm_ds_search)
     metadata_url = f'{API_BASE_URL}/metadata'
-    threshold_url = f'{API_BASE_URL}/threshold'
+    threshold_url = f'{API_BASE_URL}/nws_threshold'
     ###################################################################
+    
     #Create workspace
-    workspace.mkdir(exist_ok = True)
+    workspace.mkdir(parents=True,exist_ok = True)
 
-    #Return dictionary of huc (key) and sublist of ahps(value) as well as geodataframe of sites.
     print('Retrieving metadata...')
     #Get metadata for 'CONUS'
     conus_list, conus_dataframe = get_metadata(metadata_url, select_by = 'nws_lid', selector = ['all'], must_include = 'nws_data.rfc_forecast_point', upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search )
@@ -67,42 +70,41 @@ def static_flow_lids(workspace, nwm_us_search, nwm_ds_search):
     
     #Append the dataframes and lists
     all_lists = conus_list + islands_list
-    all_dataframe = conus_dataframe.append(islands_dataframe)
     
     print('Determining HUC using WBD layer...')
-    #Assign FIM HUC to GeoDataFrame and export to shapefile all candidate sites.
-    agg_start = time.time()
+    #Assign HUCs to all sites using a spatial join of the FIM 3 HUC layer. 
+    #Get a dictionary of hucs (key) and sites (values) as well as a GeoDataFrame
+    #of all sites used later in script.
     huc_dictionary, out_gdf = aggregate_wbd_hucs(metadata_list = all_lists, wbd_huc8_path = WBD_LAYER)
-    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)
-    viz_out_gdf.to_file(workspace / f'candidate_sites.shp')
-    agg_end = time.time()
-    print(f'agg time is {(agg_end - agg_start)/60} minutes')
+
     #Get all possible mainstem segments
     print('Getting list of mainstem segments')
     #Import list of evaluated sites
     list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
-    #The entire routine to get mainstems is harcoded in this function.
+    #The entire routine to get mainstems is hardcoded in this function.
     ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
     
-    #Loop through each huc unit
+    #Loop through each huc unit, first define message variable and flood categories.
     all_messages = []
+    flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
     for huc in huc_dictionary:
         print(f'Iterating through {huc}')
         #Get list of nws_lids
         nws_lids = huc_dictionary[huc]
         #Loop through each lid in list to create flow file
         for lid in nws_lids:
-            #In some instances the lid is not assigned a name, skip over these.
-            if not isinstance(lid,str):
-                print(f'{lid} is {type(lid)}')
-                continue
             #Convert lid to lower case
             lid = lid.lower()
             #Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
-            stages, flows = get_thresholds(threshold_url = threshold_url, location_ids = lid, physical_element = 'all', threshold = 'all', bypass_source_flag = False)
-            #If stages/flows don't exist write message and exit out.
-            if not (stages and flows):
-                message = f'{lid} no thresholds'
+            stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
+            #Check if stages are supplied, if not write message and exit. 
+            if all(stages.get(category, None)==None for category in flood_categories):
+                message = f'{lid}:missing threshold stages'
+                all_messages.append(message)
+                continue
+            #Check if calculated flows are supplied, if not write message and exit.
+            if all(flows.get(category, None) == None for category in flood_categories):
+                message = f'{lid}:missing calculated flows'
                 all_messages.append(message)
                 continue
 
@@ -116,11 +118,11 @@ def static_flow_lids(workspace, nwm_us_search, nwm_ds_search):
             #if no segments, write message and exit out
             if not segments:
                 print(f'{lid} no segments')
-                message = f'{lid} no segments'
+                message = f'{lid}:missing nwm segments'
                 all_messages.append(message)
                 continue
             #For each flood category
-            for category in ['action', 'minor', 'moderate', 'major', 'record']:
+            for category in flood_categories:
                 #Get the flow
                 flow = flows[category]
                 #If there is a valid flow value, write a flow file.
@@ -135,102 +137,99 @@ def static_flow_lids(workspace, nwm_us_search, nwm_ds_search):
                     #Write flow file to file
                     flow_info.to_csv(output_file, index = False)
                 else:
-                    message = f'{lid}_{category}_no flow'
+                    message = f'{lid}:{category} is missing calculated flow'
                     all_messages.append(message)
-            #This section will produce a point file of the LID location
+
             #Get various attributes of the site.
-            lat = float(metadata['usgs_data']['latitude'])
-            lon = float(metadata['usgs_data']['longitude'])
+            lat = float(metadata['usgs_preferred']['latitude'])
+            lon = float(metadata['usgs_preferred']['longitude'])
             wfo = metadata['nws_data']['wfo']
             rfc = metadata['nws_data']['rfc']
             state = metadata['nws_data']['state']
             county = metadata['nws_data']['county']
             name = metadata['nws_data']['name']
-            q_act = flows['action']
-            q_min = flows['minor']
-            q_mod = flows['moderate']
-            q_maj = flows['major']
-            q_rec = flows['record']
             flow_units = flows['units']
             flow_source = flows['source']
-            s_act = stages['action']
-            s_min = stages['minor']
-            s_mod = stages['moderate']
-            s_maj = stages['major']
-            s_rec = stages['record']
             stage_units = stages['units']
             stage_source = stages['source']
             wrds_timestamp = stages['wrds_timestamp']
-            #Create a DataFrame using the collected attributes
-            df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'q_act':q_act, 'q_min':q_min, 'q_mod':q_mod, 'q_maj':q_maj, 'q_rec':q_rec, 'q_uni':flow_units, 'q_src':flow_source, 'stage_act':s_act, 'stage_min':s_min, 'stage_mod':s_mod, 'stage_maj':s_maj, 'stage_rec':s_rec, 'stage_uni':stage_units, 's_src':stage_source, 'wrds_time':wrds_timestamp, 'lat':[lat], 'lon':[lon]})
-            #Round stages and flows to nearest hundredth
-            df = df.round({'q_act':2,'q_min':2,'q_mod':2,'q_maj':2,'q_rec':2,'stage_act':2,'stage_min':2,'stage_mod':2,'stage_maj':2,'stage_rec':2})
-            
-            #Create a geodataframe using usgs lat/lon property from WRDS then reproject to WGS84.
-            #Define EPSG codes for possible usgs latlon datum names (NAD83WGS84 assigned NAD83)
-            crs_lookup ={'NAD27':'EPSG:4267', 'NAD83':'EPSG:4269', 'NAD83WGS84': 'EPSG:4269'}
-            #Get horizontal datum (from dataframe) and assign appropriate EPSG code, assume NAD83 if not assigned.
-            h_datum = metadata['usgs_data']['latlon_datum_name']
-            src_crs = crs_lookup.get(h_datum, 'EPSG:4269')            
-            gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df['lon'], df['lat']), crs =  src_crs) 
-            #Reproject to VIZ_PROJECTION
-            viz_gdf = gdf.to_csv(VIZ_PROJECTION)
+            nrldb_timestamp = metadata['nrldb_timestamp']
+            nwis_timestamp = metadata['nwis_timestamp']
             
-            #Create a csv with same info as shapefile
+            #Create a csv with same information as shapefile but with each threshold as new record.
             csv_df = pd.DataFrame()
-            for threshold in ['action', 'minor', 'moderate', 'major', 'record']:
-                line_df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'magnitude': threshold, 'q':flows[threshold], 'q_uni':flows['units'], 'q_src':flow_source, 'stage':stages[threshold], 'stage_uni':stages['units'], 's_src':stage_source, 'wrds_time':wrds_timestamp, 'lat':[lat], 'lon':[lon]})
+            for threshold in flood_categories:
+                line_df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'magnitude': threshold, 'q':flows[threshold], 'q_uni':flows['units'], 'q_src':flow_source, 'stage':stages[threshold], 'stage_uni':stages['units'], 's_src':stage_source, 'wrds_time':wrds_timestamp, 'nrldb_time':nrldb_timestamp,'nwis_time':nwis_timestamp, 'lat':[lat], 'lon':[lon]})
                 csv_df = csv_df.append(line_df)
             #Round flow and stage columns to 2 decimal places.
-            csv = csv_df.round({'q':2,'stage':2})
+            csv_df = csv_df.round({'q':2,'stage':2})
 
             #If a site folder exists (ie a flow file was written) save files containing site attributes.
-            try:
-                #Save GeoDataFrame to shapefile format and export csv containing attributes
-                output_dir = workspace / huc / lid
-                viz_gdf.to_file(output_dir / f'{lid}_location.shp' )
+            output_dir = workspace / huc / lid
+            if output_dir.exists():
+                #Export DataFrame to csv containing attributes
                 csv_df.to_csv(output_dir / f'{lid}_attributes.csv', index = False)
-            except:
-                print(f'{lid} missing all flows')
-                message = f'{lid} missing all flows'
+            else:
+                message = f'{lid}:missing all calculated flows'
                 all_messages.append(message)
-    #Write out messages to file
-    messages_df  = pd.DataFrame(all_messages, columns = ['message'])
-    messages_df.to_csv(workspace / f'all_messages.csv', index = False)
-
-    #Recursively find all location shapefiles
-    locations_files = list(workspace.rglob('*_location.shp'))    
-    spatial_layers = gpd.GeoDataFrame()
-    #Append all shapefile info to a geodataframe
-    for location in locations_files:
-        location_gdf = gpd.read_file(location)
-        spatial_layers = spatial_layers.append(location_gdf)
-    #Write appended spatial data to disk.
-    output_file = workspace /'all_mapped_ahps.shp'
-    spatial_layers.to_file(output_file)
-    
-    #Recursively find all *_info csv files and append
+        
+    print('wrapping up...')
+    #Recursively find all *_attributes csv files and append
     csv_files = list(workspace.rglob('*_attributes.csv'))
     all_csv_df = pd.DataFrame()
     for csv in csv_files:
+        #Huc has to be read in as string to preserve leading zeros.
         temp_df = pd.read_csv(csv, dtype={'huc':str})
         all_csv_df = all_csv_df.append(temp_df, ignore_index = True)
-    #Write appended _info.csvs to file
-    all_info_csv = workspace / 'nws_lid_attributes.csv'
-    all_csv_df.to_csv(all_info_csv, index = False)
+    #Write to file
+    all_csv_df.to_csv(workspace / 'nws_lid_attributes.csv', index = False)
+   
+    #This section populates a shapefile of all potential sites and details
+    #whether it was mapped or not (mapped field) and if not, why (status field).
+    
+    #Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
+    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)    
+    viz_out_gdf.rename(columns = {'identifiers_nwm_feature_id': 'nwm_seg', 'identifiers_nws_lid':'nws_lid', 'identifiers_usgs_site_code':'usgs_gage'}, inplace = True)
+    viz_out_gdf['nws_lid'] = viz_out_gdf['nws_lid'].str.lower()
+    
+    #Using list of csv_files, populate DataFrame of all nws_lids that had
+    #a flow file produced and denote with "mapped" column.
+    nws_lids = [file.stem.split('_attributes')[0] for file in csv_files]
+    lids_df = pd.DataFrame(nws_lids, columns = ['nws_lid'])
+    lids_df['mapped'] = 'yes'
+    
+    #Identify what lids were mapped by merging with lids_df. Populate 
+    #'mapped' column with 'No' if sites did not map.
+    viz_out_gdf = viz_out_gdf.merge(lids_df, how = 'left', on = 'nws_lid')    
+    viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
+    
+    #Write messages to DataFrame, split into columns, aggregate messages.
+    messages_df  = pd.DataFrame(all_messages, columns = ['message'])
+    messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
+    status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
+    
+    #Join messages to populate status field to candidate sites. Assign 
+    #status for null fields.
+    viz_out_gdf = viz_out_gdf.merge(status_df, how = 'left', on = 'nws_lid')
+    viz_out_gdf['status'] = viz_out_gdf['status'].fillna('all calculated flows available')
+    
+    #Filter out columns and write out to file
+    viz_out_gdf = viz_out_gdf.filter(['nws_lid','usgs_gage','nwm_seg','HUC8','mapped','status','geometry'])
+    viz_out_gdf.to_file(workspace /'nws_lid_flows_sites.shp')
+    
+    #time operation
     all_end = time.time()
-    print(f'total time is {(all_end - all_start)/60} minutes')
+    print(f'total time is {round((all_end - all_start)/60),1} minutes')
     
-
 if __name__ == '__main__':
     #Parse arguments
     parser = argparse.ArgumentParser(description = 'Create forecast files for all nws_lid sites')
     parser.add_argument('-w', '--workspace', help = 'Workspace where all data will be stored.', required = True)
     parser.add_argument('-u', '--nwm_us_search',  help = 'Walk upstream on NWM network this many miles', required = True)
     parser.add_argument('-d', '--nwm_ds_search', help = 'Walk downstream on NWM network this many miles', required = True)
-    #Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
     
 
-    #Run create_flow_forecast_file
-    static_flow_lids(**args)
+    #Run get_env_paths and static_flow_lids
+    API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER = get_env_paths()
+    generate_catfim_flows(**args)
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
new file mode 100755
index 000000000..3d591b989
--- /dev/null
+++ b/tools/generate_categorical_fim_mapping.py
@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+from multiprocessing import Pool
+import argparse
+import traceback
+import rasterio
+import geopandas as gpd
+import pandas as pd
+import shutil
+from rasterio.features import shapes
+from shapely.geometry.polygon import Polygon
+from shapely.geometry.multipolygon import MultiPolygon
+from inundation import inundate
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+from utils.shared_functions import getDriver
+
+INPUTS_DIR = r'/data/inputs'
+magnitude_list = ['action', 'minor', 'moderate','major', 'record']
+
+# Define necessary variables for inundation()
+hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+mask_type, catchment_poly = 'huc', ''
+
+
+def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif, log_file):
+
+    no_data_list = []
+    procs_list = []
+
+    source_flow_dir_list = os.listdir(source_flow_dir)
+    output_flow_dir_list = os.listdir(fim_run_dir)
+
+    # Log missing hucs
+    missing_hucs = list(set(source_flow_dir_list) - set(output_flow_dir_list))
+    missing_hucs = [huc for huc in missing_hucs if "." not in huc]
+    if len(missing_hucs) > 0:
+        f = open(log_file, 'a+')
+        f.write(f"Missing hucs from output directory: {', '.join(missing_hucs)}\n")
+        f.close()
+
+    # Loop through matching huc directories in the source_flow directory
+    matching_hucs = list(set(output_flow_dir_list) & set(source_flow_dir_list))
+    for huc in matching_hucs:
+
+        if "." not in huc:
+
+            # Get list of AHPS site directories
+            ahps_site_dir = os.path.join(source_flow_dir, huc)
+            ahps_site_dir_list = os.listdir(ahps_site_dir)
+
+            # Map paths to HAND files needed for inundation()
+            fim_run_huc_dir = os.path.join(fim_run_dir, huc)
+            rem = os.path.join(fim_run_huc_dir, 'rem_zeroed_masked.tif')
+            catchments = os.path.join(fim_run_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+            hydroTable =  os.path.join(fim_run_huc_dir, 'hydroTable.csv')
+
+            exit_flag = False  # Default to False.
+
+            # Check if necessary data exist; set exit_flag to True if they don't exist
+            for f in [rem, catchments, hydroTable]:
+                if not os.path.exists(f):
+                    no_data_list.append(f)
+                    exit_flag = True
+
+            # Log missing data
+            if exit_flag == True:
+                f = open(log_file, 'a+')
+                f.write(f"Missing data for: {fim_run_huc_dir}\n")
+                f.close()
+
+            # Map path to huc directory inside out output_cat_fim_dir
+            cat_fim_huc_dir = os.path.join(output_cat_fim_dir, huc)
+            if not os.path.exists(cat_fim_huc_dir):
+                os.mkdir(cat_fim_huc_dir)
+
+            # Loop through AHPS sites
+            for ahps_site in ahps_site_dir_list:
+                # map parent directory for AHPS source data dir and list AHPS thresholds (act, min, mod, maj)
+                ahps_site_parent = os.path.join(ahps_site_dir, ahps_site)
+                thresholds_dir_list = os.listdir(ahps_site_parent)
+
+                # Map parent directory for all inundation output filesoutput files.
+                cat_fim_huc_ahps_dir = os.path.join(cat_fim_huc_dir, ahps_site)
+                if not os.path.exists(cat_fim_huc_ahps_dir):
+                    os.mkdir(cat_fim_huc_ahps_dir)
+
+                # Loop through thresholds/magnitudes and define inundation output files paths
+                for magnitude in thresholds_dir_list:
+
+                    if "." not in magnitude:
+
+                        magnitude_flows_csv = os.path.join(ahps_site_parent, magnitude, 'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+
+                        if os.path.exists(magnitude_flows_csv):
+
+                            output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
+
+                            if depthtif:
+                                output_depth_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_depth.tif')
+                            else:
+                                output_depth_grid = None
+
+                            # Append necessary variables to list for multiprocessing.
+                            procs_list.append([rem, catchments, magnitude_flows_csv, huc, hydroTable, output_extent_grid, output_depth_grid, ahps_site, magnitude, log_file])
+
+    # Initiate multiprocessing
+    print(f"Running inundation for {len(procs_list)} sites using {number_of_jobs} jobs")
+    pool = Pool(number_of_jobs)
+    pool.map(run_inundation, procs_list)
+
+
+def run_inundation(args):
+
+    rem                 = args[0]
+    catchments          = args[1]
+    magnitude_flows_csv = args[2]
+    huc                 = args[3]
+    hydroTable          = args[4]
+    output_extent_grid  = args[5]
+    output_depth_grid   = args[6]
+    ahps_site           = args[7]
+    magnitude           = args[8]
+    log_file            = args[9]
+
+    try:
+        inundate(rem,catchments,catchment_poly,hydroTable,magnitude_flows_csv,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=output_extent_grid,inundation_polygon=None,
+                 depths=output_depth_grid,out_raster_profile=None,out_vector_profile=None,quiet=True
+                )
+    
+    except:
+        # Log errors and their tracebacks
+        f = open(log_file, 'a+')
+        f.write(f"{output_extent_grid} - inundation error: {traceback.format_exc()}\n")
+        f.close()
+
+    #Inundation.py appends the huc code to the supplied output_extent_grid.
+    #Modify output_extent_grid to match inundation.py saved filename. 
+    #Search for this file, if it didn't create, send message to log file.
+    base_file_path,extension = os.path.splitext(output_extent_grid)
+    saved_extent_grid_filename = "{}_{}{}".format(base_file_path,huc,extension)
+    if not os.path.exists(saved_extent_grid_filename):
+        with open(log_file, 'a+') as f:
+            f.write('FAILURE_huc_{}:{}:{} map failed to create\n'.format(huc,ahps_site,magnitude))
+        
+def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_attributes_filename, log_file):
+
+    # Create workspace
+    gpkg_dir = os.path.join(output_cat_fim_dir, 'gpkg')
+    if not os.path.exists(gpkg_dir):
+        os.mkdir(gpkg_dir)
+
+    
+    #Find the FIM version
+    norm_path = os.path.normpath(output_cat_fim_dir)
+    cat_fim_dir_parts = norm_path.split(os.sep)
+    [fim_version] = [part for part in cat_fim_dir_parts if part.startswith('fim_3')]
+    merged_layer = os.path.join(output_cat_fim_dir, 'catfim_library.shp')
+
+    if not os.path.exists(merged_layer): # prevents appending to existing output
+
+        huc_ahps_dir_list = os.listdir(output_cat_fim_dir)
+        skip_list=['errors','logs','gpkg',merged_layer]
+
+        for magnitude in magnitude_list:
+
+            procs_list = []
+
+            # Loop through all categories
+            for huc in huc_ahps_dir_list:
+
+                if huc not in skip_list:
+
+                    huc_dir = os.path.join(output_cat_fim_dir, huc)
+                    ahps_dir_list = os.listdir(huc_dir)
+
+                    # Loop through ahps sites
+                    for ahps_lid in ahps_dir_list:
+                        ahps_lid_dir = os.path.join(huc_dir, ahps_lid)
+
+                        extent_grid = os.path.join(ahps_lid_dir, ahps_lid + '_' + magnitude + '_extent_' + huc + '.tif')
+
+                        if os.path.exists(extent_grid):
+                            procs_list.append([ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename])
+
+                        else:
+                            try:
+                                f = open(log_file, 'a+')
+                                f.write(f"Missing layers: {extent_gpkg}\n")
+                                f.close()
+                            except:
+                                pass
+
+            # Multiprocess with instructions
+            pool = Pool(number_of_jobs)
+            pool.map(reformat_inundation_maps, procs_list)
+
+        # Merge all layers
+        print(f"Merging {len(os.listdir(gpkg_dir))} layers...")
+
+        for layer in os.listdir(gpkg_dir):
+
+            diss_extent_filename = os.path.join(gpkg_dir, layer)
+
+            # Open diss_extent
+            diss_extent = gpd.read_file(diss_extent_filename)
+            diss_extent['viz'] = 'yes'
+
+            # Write/append aggregate diss_extent
+            if os.path.isfile(merged_layer):
+                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False, mode='a')
+            else:
+                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False)
+
+            del diss_extent
+
+        shutil.rmtree(gpkg_dir)
+
+    else:
+        print(f"{merged_layer} already exists.")
+
+
+def reformat_inundation_maps(args):
+
+    try:
+        lid                         = args[0]
+        grid_path                   = args[1]
+        gpkg_dir                    = args[2]
+        fim_version                 = args[3]
+        huc                         = args[4]
+        magnitude                   = args[5]
+        nws_lid_attributes_filename = args[6]
+
+        # Convert raster to to shapes
+        with rasterio.open(grid_path) as src:
+            image = src.read(1)
+            mask = image > 0
+
+        # Aggregate shapes
+        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
+                       
+        # convert list of shapes to polygon
+        extent_poly  = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
+
+        # Dissolve polygons
+        extent_poly_diss = extent_poly.dissolve(by='extent')
+
+        # Update attributes
+        extent_poly_diss = extent_poly_diss.reset_index(drop=True)
+        extent_poly_diss['ahps_lid'] = lid
+        extent_poly_diss['magnitude'] = magnitude
+        extent_poly_diss['version'] = fim_version
+        extent_poly_diss['huc'] = huc
+            
+        # Project to Web Mercator
+        extent_poly_diss = extent_poly_diss.to_crs(VIZ_PROJECTION)
+
+        # Join attributes
+        nws_lid_attributes_table = pd.read_csv(nws_lid_attributes_filename, dtype={'huc':str})
+        nws_lid_attributes_table = nws_lid_attributes_table.loc[(nws_lid_attributes_table.magnitude==magnitude) & (nws_lid_attributes_table.nws_lid==lid)]
+
+
+        extent_poly_diss = extent_poly_diss.merge(nws_lid_attributes_table, left_on=['ahps_lid','magnitude','huc'], right_on=['nws_lid','magnitude','huc'])
+
+        extent_poly_diss = extent_poly_diss.drop(columns='nws_lid')
+
+        # Save dissolved multipolygon
+        handle = os.path.split(grid_path)[1].replace('.tif', '')
+
+        diss_extent_filename = os.path.join(gpkg_dir, handle + "_dissolved.gpkg")
+
+        extent_poly_diss["geometry"] = [MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in extent_poly_diss["geometry"]]
+
+        if not extent_poly_diss.empty:
+
+            extent_poly_diss.to_file(diss_extent_filename,driver=getDriver(diss_extent_filename),index=False)
+
+    except Exception as e:
+        # Log and clean out the gdb so it's not merged in later
+        try:
+            f = open(log_file, 'a+')
+            f.write(str(diss_extent_filename) + " - dissolve error: " + str(e))
+            f.close()
+        except:
+            pass
+
+
+if __name__ == '__main__':
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Categorical inundation mapping for FOSS FIM.')
+    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
+    parser.add_argument('-s', '--source-flow-dir',help='Path to directory containing flow CSVs to use to generate categorical FIM.',required=True, default="")
+    parser.add_argument('-o', '--output-cat-fim-dir',help='Path to directory where categorical FIM outputs will be written.',required=True, default="")
+    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
+    parser.add_argument('-depthtif','--write-depth-tiff',help='Using this option will write depth TIFFs.',required=False, action='store_true')
+
+    args = vars(parser.parse_args())
+
+    fim_run_dir = args['fim_run_dir']
+    source_flow_dir = args['source_flow_dir']
+    output_cat_fim_dir = args['output_cat_fim_dir']
+    number_of_jobs = int(args['number_of_jobs'])
+    depthtif = args['write_depth_tiff']
+
+
+    # Create output directory
+    if not os.path.exists(output_cat_fim_dir):
+        os.mkdir(output_cat_fim_dir)
+
+    # Create log directory
+    log_dir = os.path.join(output_cat_fim_dir, 'logs')
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+
+    # Create error log path
+    log_file = os.path.join(log_dir, 'errors.log')
+
+    # Map path to points with attributes
+    nws_lid_attributes_filename = os.path.join(source_flow_dir, 'nws_lid_attributes.csv')
+
+    print("Generating Categorical FIM")
+    generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif,log_file)
+
+    print("Aggregating Categorical FIM")
+    post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir,nws_lid_attributes_filename,log_file)
diff --git a/tools/inundation.py b/tools/inundation.py
index e7c600510..d093385b8 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -17,6 +17,7 @@
 from warnings import warn
 from gdal import BuildVRT
 import geopandas as gpd
+import sys
 
 
 def inundate(
@@ -454,14 +455,14 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
                                          'HydroID':str,'stage':float,
                                          'discharge_cms':float,'LakeID' : int}
                                 )
-
+        huc_error = hydroTable.HUC.unique()
         hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
 
         hydroTable = hydroTable[hydroTable["LakeID"] == -999]  # Subset hydroTable to include only non-lake catchments.
 
         if hydroTable.empty:
-            print ("All stream segments in HUC are within lake boundaries.")
-            return
+            print(f"All stream segments in HUC(s): {huc_error} are within lake boundaries.")
+            sys.exit(0)
 
     elif isinstance(hydroTable,pd.DataFrame):
         pass #consider checking for correct dtypes, indices, and columns
@@ -504,7 +505,11 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
         hydroTable = hydroTable[np.in1d(hydroTable.index.get_level_values('HUC'), subset_hucs)]
 
     # join tables
-    hydroTable = hydroTable.join(forecast,on=['feature_id'],how='inner')
+    try:
+        hydroTable = hydroTable.join(forecast,on=['feature_id'],how='inner')
+    except AttributeError:
+        print (f"No matching feature IDs between forecast and hydrotable for HUC(s): {subset_hucs}")
+        sys.exit(0)
 
     # initialize dictionary
     catchmentStagesDict = typed.Dict.empty(types.int32,types.float64)
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index aed515f5c..13534f87b 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -728,15 +728,16 @@ def get_metadata(metadata_url, select_by, selector, must_include = None, upstrea
         metadata_list = metadata_json['locations']
         #Add timestamp of WRDS retrieval
         timestamp = response.headers['Date']
+        #Add timestamp of sources retrieval
+        nrldb_timestamp, nwis_timestamp = metadata_json['data_sources']['metadata_sources']        
         #get crosswalk info (always last dictionary in list)
-        *metadata_list, crosswalk_info = metadata_list
-        #Update each dictionary with timestamp and crosswalk info
+        crosswalk_info = metadata_json['data_sources']
+        #Update each dictionary with timestamp and crosswalk info also save to DataFrame.
         for metadata in metadata_list:
             metadata.update({"wrds_timestamp": timestamp})
+            metadata.update({"nrldb_timestamp":nrldb_timestamp})
+            metadata.update({"nwis_timestamp":nwis_timestamp})
             metadata.update(crosswalk_info)
-        #If count is 1
-        if location_count == 1:
-            metadata_list = metadata_json['locations'][0]
         metadata_dataframe = pd.json_normalize(metadata_list)
         #Replace all periods with underscores in column names
         metadata_dataframe.columns = metadata_dataframe.columns.str.replace('.','_')
@@ -754,8 +755,8 @@ def get_metadata(metadata_url, select_by, selector, must_include = None, upstrea
 def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
     '''
     Assigns the proper FIM HUC 08 code to each site in the input DataFrame.
-    Converts input DataFrame to a GeoDataFrame using the lat/lon attributes
-    with sites containing null lat/lon removed. Reprojects GeoDataFrame
+    Converts input DataFrame to a GeoDataFrame using lat/lon attributes
+    with sites containing null nws_lid/lat/lon removed. Reprojects GeoDataFrame
     to same CRS as the HUC 08 layer. Performs a spatial join to assign the
     HUC 08 layer to the GeoDataFrame. Sites that are not assigned a HUC
     code removed as well as sites in Alaska and Canada.
@@ -782,8 +783,8 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
     #Import huc8 layer as geodataframe and retain necessary columns
     huc8 = gpd.read_file(wbd_huc8_path, layer = 'WBDHU8')
     huc8 = huc8[['HUC8','name','states', 'geometry']]
-    #Define EPSG codes for possible usgs latlon datum names (NAD83WGS84 assigned NAD83)
-    crs_lookup ={'NAD27':'EPSG:4267', 'NAD83':'EPSG:4269', 'NAD83WGS84': 'EPSG:4269'}
+    #Define EPSG codes for possible latlon datum names (default of NAD83 if unassigned)
+    crs_lookup ={'NAD27':'EPSG:4267', 'NAD83':'EPSG:4269', 'WGS84': 'EPSG:4326'}
     #Create empty geodataframe and define CRS for potential horizontal datums
     metadata_gdf = gpd.GeoDataFrame()
     #Iterate through each site
@@ -793,14 +794,20 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
         #Columns have periods due to nested dictionaries
         df.columns = df.columns.str.replace('.', '_')
         #Drop any metadata sites that don't have lat/lon populated
-        df.dropna(subset = ['identifiers_nws_lid','usgs_data_latitude','usgs_data_longitude'], inplace = True)
+        df.dropna(subset = ['identifiers_nws_lid','usgs_preferred_latitude', 'usgs_preferred_longitude'], inplace = True)
         #If dataframe still has data
         if not df.empty:
-            #Get horizontal datum (use usgs) and assign appropriate EPSG code
-            h_datum = df.usgs_data_latlon_datum_name.item()
-            src_crs = crs_lookup[h_datum]
-            #Convert dataframe to geodataframe using lat/lon (USGS)
-            site_gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.usgs_data_longitude, df.usgs_data_latitude), crs =  src_crs)
+            #Get horizontal datum
+            h_datum = df['usgs_preferred_latlon_datum_name'].item()
+            #Look up EPSG code, if not returned Assume NAD83 as default. 
+            dict_crs = crs_lookup.get(h_datum,'EPSG:4269_ Assumed')
+            #We want to know what sites were assumed, hence the split.
+            src_crs, *message = dict_crs.split('_')
+            #Convert dataframe to geodataframe using lat/lon (USGS). Add attribute of assigned crs (label ones that are assumed)
+            site_gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df['usgs_preferred_longitude'], df['usgs_preferred_latitude']), crs =  src_crs)
+            #Field to indicate if a latlon datum was assumed
+            site_gdf['assigned_crs'] = src_crs + ''.join(message)
+            
             #Reproject to huc 8 crs
             site_gdf = site_gdf.to_crs(huc8.crs)
             #Append site geodataframe to metadata geodataframe
@@ -808,7 +815,7 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
 
     #Trim metadata to only have certain fields.
     if not retain_attributes:
-        metadata_gdf = metadata_gdf[['identifiers_nwm_feature_id', 'identifiers_nws_lid', 'geometry']]
+        metadata_gdf = metadata_gdf[['identifiers_nwm_feature_id', 'identifiers_nws_lid', 'identifiers_usgs_site_code', 'geometry']]
     #If a list of attributes is supplied then use that list.
     elif isinstance(retain_attributes,list):
         metadata_gdf = metadata_gdf[retain_attributes]
@@ -937,7 +944,7 @@ def get_nwm_segs(metadata):
 #######################################################################
 #Thresholds
 #######################################################################
-def get_thresholds(threshold_url, location_ids, physical_element = 'all', threshold = 'all', bypass_source_flag = False):
+def get_thresholds(threshold_url, select_by, selector, threshold = 'all'):
     '''
     Get nws_lid threshold stages and flows (i.e. bankfull, action, minor,
     moderate, major). Returns a dictionary for stages and one for flows.
@@ -946,17 +953,12 @@ def get_thresholds(threshold_url, location_ids, physical_element = 'all', thresh
     ----------
     threshold_url : STR
         WRDS threshold API.
-    location_ids : STR
-        nws_lid code (only a single code).
-    physical_element : STR, optional
-        Physical element option. The default is 'all'.
+    select_by : STR
+        Type of site (nws_lid, usgs_site_code etc).
+    selector : STR
+        Site for selection. Must be a single site.
     threshold : STR, optional
         Threshold option. The default is 'all'.
-    bypass_source_flag : BOOL, optional
-        Special case if calculated values are not available (e.g. no rating
-        curve is available) then this allows for just a stage to be returned.
-        Used in case a flow is already known from another source, such as
-        a model. The default is False.
 
     Returns
     -------
@@ -966,13 +968,14 @@ def get_thresholds(threshold_url, location_ids, physical_element = 'all', thresh
         Dictionary of flows at each threshold.
 
     '''
-
-    url = f'{threshold_url}/{physical_element}/{threshold}/{location_ids}'
-    response = requests.get(url)
+    params = {}
+    params['threshold'] = threshold
+    url = f'{threshold_url}/{select_by}/{selector}'
+    response = requests.get(url, params = params)
     if response.ok:
         thresholds_json = response.json()
         #Get metadata
-        thresholds_info = thresholds_json['stream_thresholds']
+        thresholds_info = thresholds_json['value_set']
         #Initialize stages/flows dictionaries
         stages = {}
         flows = {}
@@ -985,24 +988,27 @@ def get_thresholds(threshold_url, location_ids, physical_element = 'all', thresh
                 threshold_data = thresholds_info[rating_sources['USGS Rating Depot']]
             elif 'NRLDB' in rating_sources:
                 threshold_data = thresholds_info[rating_sources['NRLDB']]
-            #If neither USGS or NRLDB is available
+            #If neither USGS or NRLDB is available use first dictionary to get stage values.
             else:
-                #A flag option for cases where only a stage is needed for USGS scenario where a rating curve source is not available yet stages are available for the site. If flag is enabled, then stages are retrieved from the first record in thresholds_info. Typically the flows will not be populated as no rating curve is available. Flag should only be enabled when flows are already supplied by source (e.g. USGS) and threshold stages are needed.
-                if bypass_source_flag:
-                    threshold_data = thresholds_info[0]
-                else:
-                    threshold_data = []
+                threshold_data = thresholds_info[0]
             #Get stages and flows for each threshold
             if threshold_data:
                 stages = threshold_data['stage_values']
                 flows = threshold_data['calc_flow_values']
                 #Add source information to stages and flows. Flows source inside a nested dictionary. Remove key once source assigned to flows.
-                stages['source'] = threshold_data['metadata']['threshold_source']
-                flows['source'] = flows['rating_curve']['source']
+                stages['source'] = threshold_data.get('metadata').get('threshold_source')
+                flows['source'] = flows.get('rating_curve', {}).get('source')
                 flows.pop('rating_curve', None)
                 #Add timestamp WRDS data was retrieved.
                 stages['wrds_timestamp'] = response.headers['Date']
                 flows['wrds_timestamp'] = response.headers['Date']
+                #Add Site information
+                stages['nws_lid'] = threshold_data.get('metadata').get('nws_lid')
+                flows['nws_lid'] = threshold_data.get('metadata').get('nws_lid')
+                stages['usgs_site_code'] = threshold_data.get('metadata').get('usgs_site_code')
+                flows['usgs_site_code'] = threshold_data.get('metadata').get('usgs_site_code')
+                stages['units'] = threshold_data.get('metadata').get('stage_units')
+                flows['units'] = threshold_data.get('metadata').get('calc_flow_units')
     return stages, flows
 
 ########################################################################

From 2b260655cd018d6ffb44f2868f55efdd1d98585d Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 25 Mar 2021 09:08:06 -0500
Subject: [PATCH 056/359] Update CHANGELOG.md

---
 CHANGELOG.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 164ce1a44..c32f8a555 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,9 +18,10 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 ## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
 
  Patch to synthesize_test_cases.py.
- 
+  
  ### Changes
  - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
+
 <br/><br/>
 
 ## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
@@ -35,6 +36,7 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
  - Add flood of record category.
  ### Changes
  - Stability fixes to `generate_categorical_fim.py`.
+
 <br/><br/>
 
 ## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)

From a60e6eee97391416a3c8f44242f181cdf02abac1 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Tue, 30 Mar 2021 16:17:02 -0500
Subject: [PATCH 057/359] Adds detail and more information to plotting
 capabilities

Add detail/information to plotting capabilities.

- Merge plot_functions.py into eval_plots.py and move eval_plots.py into the tools directory.
- Remove plots subdirectory.
- Optional argument to create barplots of CSI for each individual site.
- Create a csv containing the data used to create the scatterplots.

This resolves #325.
---
 CHANGELOG.md                    |  11 +
 tools/{plots => }/eval_plots.py | 354 +++++++++++++++++++++++++++++++-
 tools/plots/__init__.py         |   0
 tools/plots/plot_functions.py   | 321 -----------------------------
 4 files changed, 359 insertions(+), 327 deletions(-)
 rename tools/{plots => }/eval_plots.py (54%)
 mode change 100755 => 100644
 delete mode 100755 tools/plots/__init__.py
 delete mode 100755 tools/plots/plot_functions.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c32f8a555..5f11e2d3d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.12.0 - 2021-03-26 - [PR #327](https://github.com/NOAA-OWP/cahaba/pull/237)
+
+ Add more detail/information to plotting capabilities.
+ ### Changes
+ - Merge `plot_functions.py` into `eval_plots.py` and move `eval_plots.py` into the tools directory.
+ - Remove `plots` subdirectory.
+### Additions
+ - Optional argument to create barplots of CSI for each individual site.
+ - Create a csv containing the data used to create the scatterplots. 
+<br/><br/>
+
 ## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
 
  Improvements to CatFIM service source data generation.
diff --git a/tools/plots/eval_plots.py b/tools/eval_plots.py
old mode 100755
new mode 100644
similarity index 54%
rename from tools/plots/eval_plots.py
rename to tools/eval_plots.py
index f18390f5e..b22af66ec
--- a/tools/plots/eval_plots.py
+++ b/tools/eval_plots.py
@@ -1,14 +1,332 @@
 #!/usr/bin/env python3
-
 import pandas as pd
 from pathlib import Path
 import argparse
 from natsort import natsorted
 import geopandas as gpd
-from plot_functions import filter_dataframe, boxplot, scatterplot, barplot
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import re
+#########################################################################
+#Create boxplot
+#########################################################################
+def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):      
+    '''
+    Create boxplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis
+    x_order : List
+        Order to arrange the x-axis.
+    y_field : STR
+        Field to use for the y-axis
+    hue_field : STR
+        Field to use for hue (typically FIM version)
+    title_text : STR
+        Text for plot title.
+    fim_configuration: STR
+        Configuration of FIM (FR or MS or Composite).
+    simplify_legend : BOOL, optional
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        The default is False.
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. The default is False.
+
+    Returns
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    #Use seaborn to plot the boxplot
+    axes=sns.boxplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
+    #set title of plot
+    axes.set_title(f'{title_text} ({y_field})',fontsize=20, weight = 'bold')
+    #Set yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    for index,ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel('',fontsize=0,weight = 'bold')
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+    axes.legend(markerscale = 2, fontsize =20, loc = 'lower left')
+    
+    #If simple legend desired
+    if simplify_legend:
+        #trim labels to FIM 1, FIM 2, and the FIM 3 version    
+        handles, org_labels = axes.get_legend_handles_labels()
+        label_dict = {}
+        for label in org_labels:
+            if 'fim_1' in label:
+                label_dict[label] = 'FIM 1'
+            elif 'fim_2' in label:
+                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()                  
+            elif 'fim_3' in label:
+                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                if label.endswith('_c'):
+                    label_dict[label] = label_dict[label] + ' c'
+            else:
+                label_dict[label] = label + ' ' + fim_configuration.lower()
+        #Define simplified labels as a list.
+        new_labels = [label_dict[label] for label in org_labels]
+        #Define legend location. FAR needs to be in different location than CSI/POD.
+        if y_field == 'FAR':
+            legend_location = 'upper right'
+        else:
+            legend_location = 'lower left' 
+        #rename legend labels to the simplified labels.
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
+    #Print textbox if supplied
+    if textbox_str:
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=14, verticalalignment='top', bbox=box_props)
+
+    #If figure to be saved to disk, then do so, otherwise return figure
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
 
+#########################################################################
+#Create scatter plot
+#########################################################################
+def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):      
+    '''
+    Create boxplots. 
 
-def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial_ahps = False, fim_1_ms = False):
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis (Assumes FIM 2)
+    y_field : STR
+        Field to use for the y-axis (Assumes FIM 3)
+    title_text : STR
+        Text for plot title.    
+    stats_text : STR or BOOL
+        Text for stats to place on chart. Default is false (no stats printed)
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. The default is False.
+
+    Returnsy
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    
+    #Use seaborn to plot the boxplot
+    axes=sns.scatterplot(data=dataframe, x=x_field, y=y_field, color = 'black', s = 150)
+    
+    #Set xticks and yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
+    axes.grid(b=True, which='major', axis='both')    
+    
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel(f'{x_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
+
+    #Plot diagonal line
+    diag_range = [0,1]
+    axes.plot(diag_range, diag_range, color='gray', transform=axes.transAxes)
+
+
+    #set title of plot
+    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
+   
+    if annotate:
+        #Set text for labels
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        textbox_str = 'Target Better'
+        axes.text(0.3, 0.6, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+        textbox_str = 'Baseline Better'
+        axes.text(0.5, 0.2, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+
+    if stats_text:
+        #Add statistics textbox
+        axes.text(0.01, 0.80, stats_text, transform=axes.transAxes, fontsize=24, verticalalignment='top', bbox=box_props)
+
+    #If figure to be saved to disk, then do so, otherwise return fig
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+#########################################################################
+#Create barplot
+#########################################################################
+def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):      
+    '''
+    Create barplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis
+    x_order : List
+        Order to arrange the x-axis.
+    y_field : STR
+        Field to use for the y-axis
+    hue_field : STR
+        Field to use for hue (typically FIM version)
+    title_text : STR
+        Text for plot title.
+    fim_configuration: STR
+        Configuration of FIM (FR or MS or Composite).
+    simplify_legend : BOOL, optional
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        Default is False.
+    display_values : BOOL, optional
+        If True, Y values will be displayed above bars. 
+        Default is False.
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. Default is False.
+
+    Returns
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    #Use seaborn to plot the boxplot
+    axes=sns.barplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
+    #set title of plot
+    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
+    #Set yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    for index,ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.upper()}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel('',fontsize=0,weight = 'bold')
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+    axes.legend(markerscale = 2, fontsize =20, loc = 'upper right')
+    #If simple legend desired
+    if simplify_legend:
+        #trim labels to FIM 1, FIM 2, FIM 3    
+        handles, org_labels = axes.get_legend_handles_labels()
+        label_dict = {}
+        for label in org_labels:
+            if 'fim_1' in label:
+                label_dict[label] = 'FIM 1'
+            elif 'fim_2' in label:
+                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
+            elif 'fim_3' in label:
+                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                if label.endswith('_c'):
+                    label_dict[label] = label_dict[label] + ' c'
+            else:
+                label_dict[label] = label + ' ' + fim_configuration.lower()
+        #Define simplified labels as a list.
+        new_labels = [label_dict[label] for label in org_labels]
+        #rename legend labels to the simplified labels.
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
+    #Add Textbox
+    if textbox_str:
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=18, verticalalignment='top', bbox=box_props)
+
+    #Display Y values above bars
+    if display_values:
+        #Add values of bars directly above bar.
+        for patch in axes.patches:
+            value = round(patch.get_height(),3)
+            axes.text(patch.get_x()+patch.get_width()/2.,
+                    patch.get_height(),
+                    '{:1.3f}'.format(value),
+                    ha="center", fontsize=18)  
+    
+    #If figure to be saved to disk, then do so, otherwise return fig
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+#######################################################################
+#Filter dataframe generated from csv file from run_test_case aggregation
+########################################################################
+def filter_dataframe(dataframe, unique_field):
+    '''
+
+    This script will filter out the sites (or hucs) which are not consistently 
+    found for all versions for a given magnitude. For example, an AHPS 
+    lid site must have output for all 3 versions (fim1, fim2, fim3) for 
+    a given magnitude (eg action) otherwise that lid is filtered out. 
+    Likewise for a BLE a huc must have output for all 3 versions 
+    (fim1, fim2, fim3) for a given magnitude (eg 100yr) otherwise it is 
+    filtered out.
+
+    Parameters
+    ----------
+    dataframe : Pandas DataFrame
+        Containing the input metrics originating from synthesize_test_cases
+    unique_field : STR
+        base resolution for each benchmark source: 'nws'/'usgs' (nws_lid)
+        ble (huc).
+
+    Returns
+    -------
+    final_filtered_dataframe : Pandas Dataframe
+        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude. 
+    unique_sites: DICT
+        The sites that were included in the dataframe for each magnitude.
+
+    '''
+    
+    #Get lists of sites for each magnitude/version
+    unique_sites = dataframe.groupby(['magnitude','version'])[unique_field].agg('unique')
+    #Get unique magnitudes
+    magnitudes = dataframe.magnitude.unique()
+    #Create new dataframe to hold metrics for the common sites as well as the actual lists of common sites.
+    final_filtered_dataframe = pd.DataFrame()
+    all_unique_sites = {}
+    #Cycle through each magnitude
+    for magnitude in magnitudes:
+        #Compile a list of sets containing unique lids pertaining to each threshold. List contains 3 unique sets [{fim1:unique lids},{fim2: unique lids},{fim3: unique lids}]
+        sites_per_magnitude=[set(a) for a in unique_sites[magnitude]]
+        #Intersect the sets to get the common lids per threshold then convert to list.
+        common_sites_per_magnitude = list(set.intersection(*sites_per_magnitude))
+        #Write common sites to dataframe
+        all_unique_sites[magnitude] = common_sites_per_magnitude
+        #Query filtered dataframe and only include data associated with the common sites for that magnitude
+        filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
+        #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
+        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)            
+    
+    return final_filtered_dataframe, all_unique_sites
+##############################################################################
+##############################################################################
+#Main function to analyze metric csv.
+##############################################################################
+def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial_ahps = False, fim_1_ms = False, site_barplots = False):
 
     '''
     Creates plots and summary statistics using metrics compiled from
@@ -45,6 +363,13 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         csi_scatter_<magnitude>_<configuration>*.png: scatter plot comparing
             two versions for a given magnitude. This is only generated if
             there are exactly two versions analyzed.
+        csi_scatter_<magnitude>_<configuration>_data.csv: data used to create the
+            csi_scatter_plot
+        Optional: 'individual' directory with subfolders for each site in analysis. In these
+            site subdirectories are the following files:
+                csi_<site_name>_<benchmark_source>_<configuration>.png: A barplot
+                    of CSI for each version for all magnitudes for the site.
+
 
     Parameters
     ----------
@@ -86,6 +411,10 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         Default is false. If True then fim_1 rows are duplicated with
         extent_config set to MS. This allows for FIM 1 to be included
         in MS plots/stats (helpful for nws/usgs ahps comparisons).
+    site_barplots: BOOL
+        Default is false. If True then barplots for each individual site are 
+        created. An 'individual' directory with subdirectories of each site
+        are created and the plot is located in each site subdirectory.
 
     Returns
     -------
@@ -222,6 +551,15 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         aggregate_file = output_workspace / (f'csi_aggr_{dataset_name}_{configuration.lower()}.png')
         barplot(dataframe = dataset_sums, x_field = 'magnitude', x_order = magnitude_order, y_field = 'csi', hue_field = 'version', ordered_hue = version_order, title_text = f'Aggregate {dataset_name.upper()} FIM Scores', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = aggregate_file)
 
+        #If enabled, write out barplots of CSI for individual sites.
+        if site_barplots:
+            subset = dataset.groupby(base_resolution)
+            for site_name, site_data in subset:
+                individual_dirs = output_workspace / 'individual' / str(site_name)
+                individual_dirs.mkdir(parents = True, exist_ok = True)
+                site_file = individual_dirs / f'csi_{str(site_name)}_{dataset_name}_{configuration.lower()}.png'
+                barplot(dataframe = site_data, x_field = 'magnitude', x_order = magnitude_order, y_field = 'CSI', hue_field = 'version', ordered_hue = version_order, title_text = f'{str(site_name).upper()} FIM Scores', fim_configuration = configuration, textbox_str = False, simplify_legend = True, dest_file = site_file)
+
         # Create box plots for each metric in supplied stats
         for stat in stats:
             output_file = output_workspace / (f'{stat.lower()}_{dataset_name}_{configuration.lower()}.png')
@@ -239,7 +577,9 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
                 title_text = f'CSI {magnitude}'
                 dest_file = output_workspace / f'csi_scatter_{magnitude}_{configuration.lower()}.png'
                 scatterplot(dataframe = plotdf, x_field = f'CSI_{x_version}', y_field = f'CSI_{y_version}', title_text = title_text, annotate = False, dest_file = dest_file)
-
+                #Write out dataframe used to create scatter plots
+                plotdf['Diff (C-B)'] = plotdf[f'CSI_{y_version}'] - plotdf[f'CSI_{x_version}']
+                plotdf.to_csv(output_workspace /  f'csi_scatter_{magnitude}_{configuration.lower()}_data.csv', index = False)
 
     #######################################################################
     #Create spatial layers with threshold and mapping information
@@ -305,7 +645,8 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     parser.add_argument('-q', '--alternate_ahps_query',help = 'Alternate filter query for AHPS. Default is: "not nws_lid.isnull() & not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites" where bad_sites are (grfi2,ksdm7,hohn4,rwdn4)', default = False, required = False)
     parser.add_argument('-sp', '--spatial_ahps', help = 'If spatial point layer is desired, supply a csv with 3 lines of the following format: metadata, path/to/metadata/shapefile\nevaluated, path/to/evaluated/shapefile\nstatic, path/to/static/shapefile.', default = False, required = False)
     parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
-
+    parser.add_argument('-i', '--site_plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
+    
     # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
 
@@ -337,7 +678,8 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     q = args['alternate_ahps_query']
     sp= args['spatial_ahps']
     f = args['fim_1_ms']
+    i = args['site_plots']
 
     # Run eval_plots function
     if not error:
-        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f)
+        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f, site_barplots = i)
diff --git a/tools/plots/__init__.py b/tools/plots/__init__.py
deleted file mode 100755
index e69de29bb..000000000
diff --git a/tools/plots/plot_functions.py b/tools/plots/plot_functions.py
deleted file mode 100755
index 60342059e..000000000
--- a/tools/plots/plot_functions.py
+++ /dev/null
@@ -1,321 +0,0 @@
-#!/usr/bin/env python3
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-import re
-#########################################################################
-#Create boxplot
-#########################################################################
-def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):      
-    '''
-    Create boxplots. 
-
-    Parameters
-    ----------
-    dataframe : DataFrame
-        Pandas dataframe data to be plotted.
-    x_field : STR
-        Field to use for x-axis
-    x_order : List
-        Order to arrange the x-axis.
-    y_field : STR
-        Field to use for the y-axis
-    hue_field : STR
-        Field to use for hue (typically FIM version)
-    title_text : STR
-        Text for plot title.
-    fim_configuration: STR
-        Configuration of FIM (FR or MS or Composite).
-    simplify_legend : BOOL, optional
-        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
-        The default is False.
-    dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
-        no plot is saved to disk. The default is False.
-
-    Returns
-    -------
-    fig : MATPLOTLIB
-        Plot.
-
-    '''
-
-    #initialize plot
-    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    #Use seaborn to plot the boxplot
-    axes=sns.boxplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
-    #set title of plot
-    axes.set_title(f'{title_text} ({y_field})',fontsize=20, weight = 'bold')
-    #Set yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    for index,ytick in enumerate(axes.get_yticks()):
-        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel('',fontsize=0,weight = 'bold')
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-    axes.legend(markerscale = 2, fontsize =20, loc = 'lower left')
-    
-    #If simple legend desired
-    if simplify_legend:
-        #trim labels to FIM 1, FIM 2, and the FIM 3 version    
-        handles, org_labels = axes.get_legend_handles_labels()
-        label_dict = {}
-        for label in org_labels:
-            if 'fim_1' in label:
-                label_dict[label] = 'FIM 1'
-            elif 'fim_2' in label:
-                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()                  
-            elif 'fim_3' in label:
-                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
-                if label.endswith('_c'):
-                    label_dict[label] = label_dict[label] + ' c'
-            else:
-                label_dict[label] = label + ' ' + fim_configuration.lower()
-        #Define simplified labels as a list.
-        new_labels = [label_dict[label] for label in org_labels]
-        #Define legend location. FAR needs to be in different location than CSI/POD.
-        if y_field == 'FAR':
-            legend_location = 'upper right'
-        else:
-            legend_location = 'lower left' 
-        #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
-    #Print textbox if supplied
-    if textbox_str:
-        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=14, verticalalignment='top', bbox=box_props)
-
-    #If figure to be saved to disk, then do so, otherwise return figure
-    if dest_file:
-        fig.savefig(dest_file)
-        plt.close(fig)
-    else:
-        return fig
-
-#########################################################################
-#Create scatter plot
-#########################################################################
-def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):      
-    '''
-    Create boxplots. 
-
-    Parameters
-    ----------
-    dataframe : DataFrame
-        Pandas dataframe data to be plotted.
-    x_field : STR
-        Field to use for x-axis (Assumes FIM 2)
-    y_field : STR
-        Field to use for the y-axis (Assumes FIM 3)
-    title_text : STR
-        Text for plot title.    
-    stats_text : STR or BOOL
-        Text for stats to place on chart. Default is false (no stats printed)
-    dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
-        no plot is saved to disk. The default is False.
-
-    Returnsy
-    -------
-    fig : MATPLOTLIB
-        Plot.
-
-    '''
-
-    #initialize plot
-    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    
-    #Use seaborn to plot the boxplot
-    axes=sns.scatterplot(data=dataframe, x=x_field, y=y_field, color = 'black', s = 150)
-    
-    #Set xticks and yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
-    axes.grid(b=True, which='major', axis='both')    
-    
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel(f'{x_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
-
-    #Plot diagonal line
-    diag_range = [0,1]
-    axes.plot(diag_range, diag_range, color='gray', transform=axes.transAxes)
-
-
-    #set title of plot
-    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
-   
-    if annotate:
-        #Set text for labels
-        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        textbox_str = 'Target Better'
-        axes.text(0.3, 0.6, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
-        textbox_str = 'Baseline Better'
-        axes.text(0.5, 0.2, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
-
-    if stats_text:
-        #Add statistics textbox
-        axes.text(0.01, 0.80, stats_text, transform=axes.transAxes, fontsize=24, verticalalignment='top', bbox=box_props)
-
-    #If figure to be saved to disk, then do so, otherwise return fig
-    if dest_file:
-        fig.savefig(dest_file)
-        plt.close(fig)
-    else:
-        return fig
-#########################################################################
-#Create barplot
-#########################################################################
-def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):      
-    '''
-    Create barplots. 
-
-    Parameters
-    ----------
-    dataframe : DataFrame
-        Pandas dataframe data to be plotted.
-    x_field : STR
-        Field to use for x-axis
-    x_order : List
-        Order to arrange the x-axis.
-    y_field : STR
-        Field to use for the y-axis
-    hue_field : STR
-        Field to use for hue (typically FIM version)
-    title_text : STR
-        Text for plot title.
-    fim_configuration: STR
-        Configuration of FIM (FR or MS or Composite).
-    simplify_legend : BOOL, optional
-        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
-        Default is False.
-    display_values : BOOL, optional
-        If True, Y values will be displayed above bars. 
-        Default is False.
-    dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
-        no plot is saved to disk. Default is False.
-
-    Returns
-    -------
-    fig : MATPLOTLIB
-        Plot.
-
-    '''
-
-    #initialize plot
-    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    #Use seaborn to plot the boxplot
-    axes=sns.barplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
-    #set title of plot
-    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
-    #Set yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    for index,ytick in enumerate(axes.get_yticks()):
-        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field.upper()}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel('',fontsize=0,weight = 'bold')
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-    axes.legend(markerscale = 2, fontsize =20, loc = 'upper right')
-    #If simple legend desired
-    if simplify_legend:
-        #trim labels to FIM 1, FIM 2, FIM 3    
-        handles, org_labels = axes.get_legend_handles_labels()
-        label_dict = {}
-        for label in org_labels:
-            if 'fim_1' in label:
-                label_dict[label] = 'FIM 1'
-            elif 'fim_2' in label:
-                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
-            elif 'fim_3' in label:
-                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
-                if label.endswith('_c'):
-                    label_dict[label] = label_dict[label] + ' c'
-            else:
-                label_dict[label] = label + ' ' + fim_configuration.lower()
-        #Define simplified labels as a list.
-        new_labels = [label_dict[label] for label in org_labels]
-        #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
-    #Add Textbox
-    if textbox_str:
-        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=18, verticalalignment='top', bbox=box_props)
-
-    #Display Y values above bars
-    if display_values:
-        #Add values of bars directly above bar.
-        for patch in axes.patches:
-            value = round(patch.get_height(),3)
-            axes.text(patch.get_x()+patch.get_width()/2.,
-                    patch.get_height(),
-                    '{:1.3f}'.format(value),
-                    ha="center", fontsize=18)  
-    
-    #If figure to be saved to disk, then do so, otherwise return fig
-    if dest_file:
-        fig.savefig(dest_file)
-        plt.close(fig)
-    else:
-        return fig
-#######################################################################
-#Filter dataframe generated from csv file from run_test_case aggregation
-########################################################################
-def filter_dataframe(dataframe, unique_field):
-    '''
-
-    This script will filter out the sites (or hucs) which are not consistently 
-    found for all versions for a given magnitude. For example, an AHPS 
-    lid site must have output for all 3 versions (fim1, fim2, fim3) for 
-    a given magnitude (eg action) otherwise that lid is filtered out. 
-    Likewise for a BLE a huc must have output for all 3 versions 
-    (fim1, fim2, fim3) for a given magnitude (eg 100yr) otherwise it is 
-    filtered out.
-
-    Parameters
-    ----------
-    dataframe : Pandas DataFrame
-        Containing the input metrics originating from synthesize_test_cases
-    unique_field : STR
-        base resolution for each benchmark source: 'nws'/'usgs' (nws_lid)
-        ble (huc).
-
-    Returns
-    -------
-    final_filtered_dataframe : Pandas Dataframe
-        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude. 
-    unique_sites: DICT
-        The sites that were included in the dataframe for each magnitude.
-
-    '''
-    
-    #Get lists of sites for each magnitude/version
-    unique_sites = dataframe.groupby(['magnitude','version'])[unique_field].agg('unique')
-    #Get unique magnitudes
-    magnitudes = dataframe.magnitude.unique()
-    #Create new dataframe to hold metrics for the common sites as well as the actual lists of common sites.
-    final_filtered_dataframe = pd.DataFrame()
-    all_unique_sites = {}
-    #Cycle through each magnitude
-    for magnitude in magnitudes:
-        #Compile a list of sets containing unique lids pertaining to each threshold. List contains 3 unique sets [{fim1:unique lids},{fim2: unique lids},{fim3: unique lids}]
-        sites_per_magnitude=[set(a) for a in unique_sites[magnitude]]
-        #Intersect the sets to get the common lids per threshold then convert to list.
-        common_sites_per_magnitude = list(set.intersection(*sites_per_magnitude))
-        #Write common sites to dataframe
-        all_unique_sites[magnitude] = common_sites_per_magnitude
-        #Query filtered dataframe and only include data associated with the common sites for that magnitude
-        filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
-        #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
-        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)            
-    
-    return final_filtered_dataframe, all_unique_sites
-

From 391b547ab9bec87eb26671a73190474c9c6c0a5d Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 31 Mar 2021 10:00:34 -0500
Subject: [PATCH 058/359] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9f44d94ad..c067ea29d 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ The following input data sources should be downloaded and preprocessed prior to
 - `acquire_and_preprocess_inputs.py`
 - `aggregate_nhd_hr_streams.py`
 
-**Please note:** For the following two datasets, please contact Brad Bates (bradford.bates@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
+**Please note:** For the following two datasets, please contact Mark Glaudemans (mark.glaudemans@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
 
 ### NWM Hydrofabric
 - `nwm_flows.gpkg`
@@ -84,7 +84,7 @@ fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_
 ## Evaluating Inundation Map Performance
 After `fim_run.sh` completes, you are ready to evaluate the model's skill.
 
-**Please note:** You will need access to the test_cases benchmark data. You can acquire the benchmark data from Brad Bates (bradford.bates@noaa.gov). As mentioned before, a long term data sharing solution is still in the works.
+**Please note:** You will need access to the test_cases benchmark data. Please contact Mark Glaudemans for access to these data (mark.glaudemans@noaa.gov). As mentioned before, a long term data sharing solution is still in the works.
 
 To evaluate model skill, run the following:
 ```

From 385ae0f5e5e96c6f07dd9796c0056b276e03cb41 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Thu, 1 Apr 2021 12:24:05 -0500
Subject: [PATCH 059/359] Update spatial option when performing eval plots

Removes file dependencies from spatial option. Does require the WBD layer which should be specified in .env file.

- Produces outputs in a format consistent with requirements needed for publishing.
- Preserves leading zeros in huc information for all outputs from eval_plots.py.
- Creates fim_performance_points.shp: this layer consists of all evaluated ahps points (with metrics). Spatial data retrieved from WRDS on the fly.
- Creates fim_performance_polys.shp: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.

This resolves #325.
---
 CHANGELOG.md        |  15 ++++
 tools/eval_plots.py | 188 ++++++++++++++++++++++++--------------------
 2 files changed, 118 insertions(+), 85 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f11e2d3d..843e37282 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v3.0.12.1 - 2021-03-26 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
+
+ Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
+ 
+ ### Changes
+ - Removes file dependencies from spatial option. Does require the WBD layer which should be specified in `.env` file. 
+ - Produces outputs in a format consistent with requirements needed for publishing.
+ - Preserves leading zeros in huc information for all outputs from `eval_plots.py`.
+
+### Additions
+- Creates `fim_performance_points.shp`: this layer consists of all evaluated ahps points (with metrics). Spatial data retrieved from WRDS on the fly.
+- Creates `fim_performance_polys.shp`: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.
+<br/><br/>
+
 ## v3.0.12.0 - 2021-03-26 - [PR #327](https://github.com/NOAA-OWP/cahaba/pull/237)
 
  Add more detail/information to plotting capabilities.
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index b22af66ec..0af2ae9ad 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -8,6 +8,18 @@
 import matplotlib.pyplot as plt
 import seaborn as sns
 import re
+import os
+import sys
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import VIZ_PROJECTION
+from dotenv import load_dotenv
+from tools_shared_functions import aggregate_wbd_hucs, get_metadata
+
+#Get variables from .env file.
+load_dotenv()
+WBD_LAYER = os.getenv("WBD_LAYER")
+API_BASE_URL = os.getenv("API_BASE_URL")
+
 #########################################################################
 #Create boxplot
 #########################################################################
@@ -326,7 +338,7 @@ def filter_dataframe(dataframe, unique_field):
 ##############################################################################
 #Main function to analyze metric csv.
 ##############################################################################
-def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial_ahps = False, fim_1_ms = False, site_barplots = False):
+def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial = False, fim_1_ms = False, site_barplots = False):
 
     '''
     Creates plots and summary statistics using metrics compiled from
@@ -369,6 +381,12 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             site subdirectories are the following files:
                 csi_<site_name>_<benchmark_source>_<configuration>.png: A barplot
                     of CSI for each version for all magnitudes for the site.
+        Optional (if spatial argument supplied): 
+            fim_performance_points.shp -- A shapefile of ahps points with 
+            metrics contained in attribute table.
+            fim_performance_polys.shp -- A shapefile of huc8 polygons with
+            metrics contained in attribute table.
+            
 
 
     Parameters
@@ -397,16 +415,15 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         The default is false. Currently the default ahps query is same
         as done for apg goals. If a different query is desired it can be
         supplied and it will supercede the default query.
-    spatial_ahps : DICTIONARY, optional
-        The default is false. A dictionary with keys as follows:
-            'static': Path to AHPS point file created during creation of
-                FIM 3 static libraries.
-            'evaluated': Path to extent file created during the creation
-                of the NWS/USGS AHPS preprocessing.
-            'metadata': Path to previously created file that contains
-                metadata about each site (feature_id, wfo, rfc and etc).
-        No spatial layers will be created if set to False, if a dictionary
-        is supplied then a spatial layer is produced.
+    spatial : BOOL, optional
+        Creates spatial datasets of the base unit (ble: huc polygon, ahps: point) 
+        with metrics contained in attribute tables. The geospatial data is 
+        either supplied in the .env file (WBD Huc layer) or from WRDS (ahps).
+        The outputs are consistent with requirements set forth by the vizualization team.
+        Additionally, there is a commented out section where if the user
+        passes the extent files generated during creation of nws/usgs ahps
+        preprocessing, the actual maps and flows used for evaluation are
+        appended to the ahps shapefile output. 
     fim_1_ms: BOOL
         Default is false. If True then fim_1 rows are duplicated with
         extent_config set to MS. This allows for FIM 1 to be included
@@ -426,7 +443,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     '''
 
     # Import metrics csv as DataFrame and initialize all_datasets dictionary
-    csv_df = pd.read_csv(metrics_csv)
+    csv_df = pd.read_csv(metrics_csv, dtype = {'huc':str})
 
     # fim_1_ms flag enables FIM 1 to be shown on MS plots/stats
     if fim_1_ms:
@@ -584,55 +601,77 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     #######################################################################
     #Create spatial layers with threshold and mapping information
     ########################################################################
-    if spatial_ahps:
-
-        # Read in supplied shapefile layers
-        # Layer containing metadata for each site (feature_id, wfo, etc)
-        # Convert nws_lid to lower case
-        ahps_metadata = gpd.read_file(spatial_ahps['metadata'])
-        ahps_metadata['nws_lid'] = ahps_metadata['nws_lid'].str.lower()
-        metadata_crs = ahps_metadata.crs
-
-        # Extent layer generated from preprocessing NWS/USGS datasets
-        evaluated_ahps_extent = gpd.read_file(spatial_ahps['evaluated'])
-
-        # Extent layer generated from static ahps library preprocessing
-        static_library = gpd.read_file(spatial_ahps['static'])
-
-        # Fields to keep
-        # Get list of fields to keep in merge
-        preserved_static_library_fields = ['nws_lid'] + [i for i in static_library.columns if i.startswith(('Q','S'))]
-        # Get list of fields to keep in merge
-        preserved_evaluated_ahps_fields = ['nws_lid', 'source', 'geometry'] + [i for i in evaluated_ahps_extent.columns if i.startswith(('action','minor','moderate','major'))]
-
-        # Join tables to evaluated_ahps_extent
-        evaluated_ahps_extent = evaluated_ahps_extent[preserved_evaluated_ahps_fields]
-        evaluated_ahps_extent = evaluated_ahps_extent.merge(ahps_metadata, on = 'nws_lid')
-        evaluated_ahps_extent['geometry'] = evaluated_ahps_extent['geometry_y']
-        evaluated_ahps_extent.drop(columns = ['geometry_y','geometry_x'], inplace = True)
-        evaluated_ahps_extent = evaluated_ahps_extent.merge(static_library[preserved_static_library_fields], on = 'nws_lid')
-
-        # Join dataset metrics to evaluated_ahps_extent data
-        final_join = pd.DataFrame()
-        for (dataset_name, configuration), (dataset, sites) in all_datasets.items():
-            # Only select ahps from dataset if config is MS
-            if dataset_name in ['usgs','nws'] and configuration == 'MS':
-                # Select records from evaluated_ahps_extent that match the dataset name
-                subset = evaluated_ahps_extent.query(f'source == "{dataset_name}"')
-                # Join to dataset
-                dataset_with_subset = dataset.merge(subset, on = 'nws_lid')
-                # Append rows to final_join dataframe
-                final_join = final_join.append(dataset_with_subset)
-
-        # Modify version field
-        final_join['version'] = final_join.version.str.split('_nws|_usgs').str[0]
-
-        # Write geodataframe to file
-        gdf = gpd.GeoDataFrame(final_join, geometry = final_join['geometry'], crs = metadata_crs)
-        output_shapefile = Path(workspace) / 'nws_usgs_site_info.shp'
-        gdf.to_file(output_shapefile)
-
-
+    if spatial:
+        ###############################################################
+        #This section will join ahps metrics to a spatial point layer
+        ###############################################################
+        
+        #Get point data for ahps sites
+        #Get metrics for usgs and nws benchmark sources
+        usgs_dataset,sites = all_datasets.get(('usgs','MS'))
+        nws_dataset, sites = all_datasets.get(('nws','MS'))
+        #Append usgs/nws dataframes and filter unnecessary columns and rename remaining.
+        all_ahps_datasets = usgs_dataset.append(nws_dataset)
+        all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source'])
+        all_ahps_datasets.rename(columns = {'benchmark_source':'source'}, inplace = True)
+        
+        #Get spatial data from WRDS
+        #Get metadata from WRDS API
+        select_by = 'nws_lid'
+        selector = list(all_ahps_datasets.nws_lid.unique())
+        metadata_url = f'{API_BASE_URL}/metadata'
+        metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector)
+        #Create geospatial data from WRDS output
+        dictionary, gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = True)
+        #Trim out unecessary columns and rename remaining columns
+        gdf = gdf.filter(['identifiers_nws_lid', 'nws_data_name', 'identifiers_nwm_feature_id','nws_data_wfo','nws_data_state','nws_data_county','geometry'])
+        gdf.rename(columns = {'identifiers_nws_lid':'nws_lid', 'nws_data_name':'lid_name','identifiers_nwm_feature_id':'feature_id','nws_data_wfo':'wfo','nws_data_state':'state','nws_data_county':'county','HUC8':'huc8'}, inplace = True)
+        
+        #Join spatial data to metric data        
+        gdf['nws_lid'] = gdf['nws_lid'].str.lower()        
+        joined = gdf.merge(all_ahps_datasets, on = 'nws_lid')
+        #Project to VIZ projection and write to file
+        joined = joined.to_crs(VIZ_PROJECTION)
+        joined.to_file(Path(workspace) / 'fim_performance_points.shp')
+        
+        '''
+        ###############################################################
+        #If user wants to append information such as what maps or flows were used for evaluation. This is already tested.
+        #User must supply the extent layer generated from preprocessing NWS/USGS datasets.
+        ###############################################################
+        #Read extent layer to GeoDataFrame and drop the geometry column
+        evaluated_ahps_extent = gpd.read_file(/Path/to/extent/layer/generated/during/preprocessing)        
+        evaluated_ahps_extent.drop(columns = ['geometry'], inplace = True)
+        #Re-arrange dataset to get flows used for evaluation
+        flows = pd.melt(evaluated_ahps_extent, id_vars = ['nws_lid','source'], value_vars = ['action_Q','minor_Q','moderate_Q','major_Q'], var_name = 'magnitude', value_name = 'eval_Q')
+        flows['magnitude'] = flows['magnitude'].str.split('_', 1, expand = True)
+        #Re-arrange dataset to get maps used for evaluation
+        maps = pd.melt(evaluated_ahps_extent, id_vars = ['nws_lid','source'], value_vars = ['action','minor','moderate','major'], var_name = 'magnitude', value_name = 'eval_maps')
+        maps['eval_maps'] = maps['eval_maps'].str.split('\\').str[-1]
+        #Merge flows and maps into single DataFrame
+        flows_maps = pd.merge(flows,maps, how = 'left', left_on = ['nws_lid','source','magnitude'], right_on = ['nws_lid','source','magnitude'])        
+        # combine flows_maps to spatial layer (gdf)
+        joined = joined.merge(flows_maps, left_on = ['nws_lid','magnitude','source'], right_on = ['nws_lid','magnitude','source'])
+        #Write to file
+        joined.to_file(Path(workspace)/'fim_performance_points.shp')
+        '''
+        ################################################################
+        #This section joins ble (FR) metrics to a spatial layer of HUCs.
+        ################################################################
+        #Read in HUC spatial layer
+        wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')
+        #Select BLE, FR dataset.
+        ble_dataset, sites = all_datasets.get(('ble','FR'))              
+        #Join metrics to HUC spatial layer
+        wbd_with_metrics = wbd_gdf.merge(ble_dataset, how = 'inner', left_on = 'HUC8', right_on = 'huc')
+        #Filter out unnecessary columns
+        wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source','geometry'])
+        wbd_with_metrics.rename(columns = {'benchmark_source':'source'}, inplace = True )
+        #Project to VIZ projection
+        wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION)
+        #Write out to file
+        wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
+        
 
 #######################################################################
 if __name__ == '__main__':
@@ -643,43 +682,22 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
     parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR'], required = False)
     parser.add_argument('-q', '--alternate_ahps_query',help = 'Alternate filter query for AHPS. Default is: "not nws_lid.isnull() & not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites" where bad_sites are (grfi2,ksdm7,hohn4,rwdn4)', default = False, required = False)
-    parser.add_argument('-sp', '--spatial_ahps', help = 'If spatial point layer is desired, supply a csv with 3 lines of the following format: metadata, path/to/metadata/shapefile\nevaluated, path/to/evaluated/shapefile\nstatic, path/to/static/shapefile.', default = False, required = False)
+    parser.add_argument('-sp', '--spatial', help = 'If enabled, creates spatial layers with metrics populated in attribute table.', action = 'store_true', required = False)
     parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
     parser.add_argument('-i', '--site_plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
     
     # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
 
-    # If errors occur reassign error to True
-    error = False
-    # Create dictionary if file specified for spatial_ahps
-    if args['spatial_ahps']:
-        # Create dictionary
-        spatial_dict = {}
-        with open(args['spatial_ahps']) as file:
-            for line in file:
-                key, value = line.strip('\n').split(',')
-                spatial_dict[key] = Path(value)
-        args['spatial_ahps'] = spatial_dict
-        # Check that all required keys are present and overwrite args with spatial_dict
-        required_keys = set(['metadata', 'evaluated', 'static'])
-        if required_keys - spatial_dict.keys():
-          print('\n Required keys are: metadata, evaluated, static')
-          error = True
-        else:
-            args['spatial_ahps'] = spatial_dict
-
-
     # Finalize Variables
     m = args['metrics_csv']
     w = args['workspace']
     v = args['versions']
     s = args['stats']
     q = args['alternate_ahps_query']
-    sp= args['spatial_ahps']
+    sp= args['spatial']
     f = args['fim_1_ms']
     i = args['site_plots']
 
     # Run eval_plots function
-    if not error:
-        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f, site_barplots = i)
+    eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial = sp, fim_1_ms = f, site_barplots = i)

From 4df6d1d7a5935310a873415f046da6c84ddc4f0c Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 1 Apr 2021 12:27:10 -0500
Subject: [PATCH 060/359] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 843e37282..af8181826 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
-## v3.0.12.1 - 2021-03-26 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
+## v3.0.12.1 - 2021-03-31 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
 
  Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
  

From 44b633381d37eb7f0a636ba56914850a33acdd42 Mon Sep 17 00:00:00 2001
From: "brian.avant" <brian.avant@noaa.gov>
Date: Thu, 1 Apr 2021 13:40:49 -0500
Subject: [PATCH 061/359] [1pt] Tool to compare synthetic rating curve with
 benchmark rating curve (sierra test) (#332)

* `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in run_by_unit.py with elevation at USGS gages
* `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between FIM and USGS rating curve data
* updates `aggregate_fim_outputs.py` call argument in `fim_run.sh` from 4 jobs to 6 jobs (optimizing API performance)
* reroutes median elevation data from `add_crosswalk.py` and `rem.py` to new file (depreciating `hand_ref_elev_table.csv`)
*- adds new files to `viz_whitelist` in `output_cleanup.py`
---
 CHANGELOG.md                     |  42 ++--
 fim_run.sh                       |   2 +-
 src/add_crosswalk.py             |   3 +-
 src/output_cleanup.py            |   9 +-
 src/rem.py                       |  23 +-
 src/run_by_unit.sh               |  10 +-
 src/usgs_gage_crosswalk.py       | 124 ++++++++++
 tools/rating_curve_comparison.py | 408 +++++++++++++++++++++++++++++++
 8 files changed, 590 insertions(+), 31 deletions(-)
 create mode 100755 src/usgs_gage_crosswalk.py
 create mode 100755 tools/rating_curve_comparison.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index af8181826..8845241ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,27 +1,42 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.12.2 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
+
+Created tool to compare synthetic rating curve with benchmark rating curve (sierra test). resolves issue #293; resolves issue #308
+ 
+### Changes
+ - update `aggregate_fim_outputs.py` call argument in `fim_run.sh` from 4 jobs to 6 jobs (optimizing API performance)
+ - reroutes median elevation data from `add_crosswalk.py` and `rem.py` to new file (depreciating `hand_ref_elev_table.csv`)
+ - adding new files to `viz_whitelist` in `output_cleanup.py`
+
+### Additions
+ - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in run_by_unit.py with elevation and additional attributes at USGS gages. 
+ - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data. 
+<br/><br/>
 
 ## v3.0.12.1 - 2021-03-31 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
 
- Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
+Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
  
- ### Changes
+### Changes
  - Removes file dependencies from spatial option. Does require the WBD layer which should be specified in `.env` file. 
  - Produces outputs in a format consistent with requirements needed for publishing.
  - Preserves leading zeros in huc information for all outputs from `eval_plots.py`.
 
 ### Additions
-- Creates `fim_performance_points.shp`: this layer consists of all evaluated ahps points (with metrics). Spatial data retrieved from WRDS on the fly.
-- Creates `fim_performance_polys.shp`: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.
+ - Creates `fim_performance_points.shp`: this layer consists of all evaluated ahps points (with metrics). Spatial data retrieved from WRDS on the fly.
+ - Creates `fim_performance_polys.shp`: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.
 <br/><br/>
 
 ## v3.0.12.0 - 2021-03-26 - [PR #327](https://github.com/NOAA-OWP/cahaba/pull/237)
 
- Add more detail/information to plotting capabilities.
- ### Changes
+Add more detail/information to plotting capabilities.
+
+### Changes
  - Merge `plot_functions.py` into `eval_plots.py` and move `eval_plots.py` into the tools directory.
  - Remove `plots` subdirectory.
+ - 
 ### Additions
  - Optional argument to create barplots of CSI for each individual site.
  - Create a csv containing the data used to create the scatterplots. 
@@ -29,30 +44,29 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 ## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
 
- Improvements to CatFIM service source data generation.
+Improvements to CatFIM service source data generation.
  
- ### Changes
+### Changes
  - Renamed `generate_categorical_fim.py` to `generate_categorical_fim_mapping.py`.
  - Updated the status outputs of the `nws_lid_sites layer` and saved it in the same directory as the `merged catfim_library layer`.
  - Additional stability fixes (such as improved compatability with WRDS updates).
+ 
 ### Additions
  - Added `generate_categorical_fim.py` to wrap `generate_categorical_fim_flows.py` and `generate_categorical_fim_mapping.py`.
  - Create new `nws_lid_sites` shapefile located in same directory as the `catfim_library` shapefile.
- 
 <br/><br/>
 
 ## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
 
- Patch to synthesize_test_cases.py.
+Patch to synthesize_test_cases.py.
   
- ### Changes
+### Changes
  - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
-
 <br/><br/>
 
 ## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
 
- Preprocessing of flow files for Categorical FIM.
+Preprocessing of flow files for Categorical FIM.
  
 ### Additions
  - Generate Categorical FIM flow files for each category (action, minor, moderate, major).
@@ -60,9 +74,9 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
  - Generate csv of attribute data in shapefile.
  - Aggregate all shapefiles and csv files into one file in parent directory.
  - Add flood of record category.
+ 
  ### Changes
  - Stability fixes to `generate_categorical_fim.py`.
-
 <br/><br/>
 
 ## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)
diff --git a/fim_run.sh b/fim_run.sh
index 42a5d022e..8d1875e5f 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -152,5 +152,5 @@ fi
 echo "$viz"
 if [[ "$viz" -eq 1 ]]; then
     # aggregate outputs
-    python3 /foss_fim/src/aggregate_fim_outputs.py -d $outputRunDataDir -j 4
+    time python3 /foss_fim/src/aggregate_fim_outputs.py -d $outputRunDataDir -j 6
 fi
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 2958c2882..96f2805c0 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -220,9 +220,8 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_hydro_table = output_hydro_table.merge(input_huc.loc[:,[FIM_ID,'HUC8']],how='left',on=FIM_ID)
 
     if output_flows.HydroID.dtype != 'str': output_flows.HydroID = output_flows.HydroID.astype(str)
-    output_hydro_table = output_hydro_table.merge(output_flows.loc[:,['HydroID','LakeID','Median_Thal_Elev_m']],how='left',on='HydroID')
+    output_hydro_table = output_hydro_table.merge(output_flows.loc[:,['HydroID','LakeID']],how='left',on='HydroID')
     output_hydro_table['LakeID'] = output_hydro_table['LakeID'].astype(int)
-    output_hydro_table['Median_Thal_Elev_m'] = output_hydro_table['Median_Thal_Elev_m'].astype(float).round(2)
     output_hydro_table = output_hydro_table.rename(columns={'HUC8':'HUC'})
     if output_hydro_table.HUC.dtype != 'str': output_hydro_table.HUC = output_hydro_table.HUC.astype(str)
 
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index 7e211bdc5..e74e26b17 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -31,15 +31,20 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         'gw_catchments_reaches_filtered_addedAttributes.tif',
         'hydroTable.csv',
         'src.json',
-        'small_segments.csv'
+        'small_segments.csv',
+        'usgs_elev_table.csv',
+        'hand_ref_elev_table.csv'
     ]
 
     # List of files that will be saved during a viz run
     viz_whitelist = [
         'rem_zeroed_masked.tif',
+        'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg',
+        'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg',
         'gw_catchments_reaches_filtered_addedAttributes.tif',
         'hydroTable.csv',
-        'src.json'
+        'src.json',
+        'small_segments.csv'
     ]
 
     # If "production" run, only keep whitelisted files
diff --git a/src/rem.py b/src/rem.py
index 403edf9db..f0cd8fad3 100755
--- a/src/rem.py
+++ b/src/rem.py
@@ -11,7 +11,7 @@
 from utils.shared_functions import getDriver
 
 
-def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, hand_ref_elev_fileName, dem_reaches_filename):
+def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, dem_reaches_filename):
     """
         Calculates REM/HAND/Detrended DEM
 
@@ -25,8 +25,6 @@ def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raste
             File name of output relative elevation raster.
         hydroid_fileName : str
             File name of the hydroid raster (i.e. gw_catchments_reaches.tif)
-        hand_ref_elev_fileName
-            File name of the output csv containing list of hydroid values and HAND zero/reference elev
         dem_reaches_filename
             File name of the reaches layer to populate HAND elevation attribute values and overwrite as output
 
@@ -108,7 +106,6 @@ def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalw
     gw_catchments_pixels_masked_object.close()
     thalweg_raster_object.close()
 
-###############################################
     # Merge and export dictionary to to_csv
     catchment_min_dict_df = pd.DataFrame.from_dict(catchment_min_dict, orient='index') # convert dict to dataframe
     catchment_min_dict_df.columns = ['Median_Thal_Elev_m']
@@ -116,12 +113,18 @@ def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalw
     catchment_hydroid_dict_df.columns = ['HydroID']
     merge_df = catchment_hydroid_dict_df.merge(catchment_min_dict_df, left_index=True, right_index=True)
     merge_df.index.name = 'pixelcatch_id'
-    merge_df.to_csv(hand_ref_elev_fileName,index=True) # export dataframe to csv file
 
-    # Merge the HAND reference elvation by HydroID dataframe with the demDerived_reaches layer (add new layer attribute)
-    merge_df = merge_df.groupby(['HydroID']).median() # median value of all Median_Thal_Elev_m for pixel catchments in each HydroID reach
+    # Merge the HAND reference elevation by HydroID dataframe with the demDerived_reaches layer (add new layer attribute)
+    min_by_hydroid = merge_df.groupby(['HydroID']).min() # min value of all med_thal_elev for pixel catchments in each HydroID reach
+    min_by_hydroid.columns = ['min_thal_elev']
+    med_by_hydroid = merge_df.groupby(['HydroID']).median() # median value of all med_thal_elev for pixel catchments in each HydroID reach
+    med_by_hydroid.columns = ['med_thal_elev']
+    max_by_hydroid = merge_df.groupby(['HydroID']).max() # max value of all med_thal_elev for pixel catchments in each HydroID reach
+    max_by_hydroid.columns = ['max_thal_elev']
     input_reaches = gpd.read_file(dem_reaches_filename)
-    input_reaches = input_reaches.merge(merge_df, on='HydroID') # merge dataframes by HydroID variable
+    input_reaches = input_reaches.merge(min_by_hydroid, on='HydroID') # merge dataframes by HydroID variable
+    input_reaches = input_reaches.merge(med_by_hydroid, on='HydroID') # merge dataframes by HydroID variable
+    input_reaches = input_reaches.merge(max_by_hydroid, on='HydroID') # merge dataframes by HydroID variable
     input_reaches.to_file(dem_reaches_filename,driver=getDriver(dem_reaches_filename),index=False)
     # ------------------------------------------------------------------------------------------------------------------------ #
 
@@ -171,7 +174,6 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
     parser.add_argument('-t','--thalweg-raster',help='A binary raster representing the thalweg. 1 for thalweg, 0 for non-thalweg.',required=True)
     parser.add_argument('-o','--rem',help='Output REM raster',required=True)
     parser.add_argument('-i','--hydroid', help='HydroID raster to use within project path', required=True)
-    parser.add_argument('-r','--hand_ref_elev_table',help='Output table of HAND reference elev by catchment',required=True)
     parser.add_argument('-s','--dem_reaches_in_out',help='DEM derived reach layer to join HAND reference elevation attribute',required=True)
 
 
@@ -184,7 +186,6 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
     rem_fileName = args['rem']
     thalweg_raster = args['thalweg_raster']
     hydroid_fileName = args['hydroid']
-    hand_ref_elev_fileName = args['hand_ref_elev_table']
     dem_reaches_filename = args['dem_reaches_in_out']
 
-    rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, hand_ref_elev_fileName, dem_reaches_filename)
+    rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, dem_reaches_filename)
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 68866729d..1242768dc 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -321,7 +321,7 @@ echo -e $startDiv"D8 REM $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/rem.tif ] && \
-$srcDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -r $outputHucDataDir/hand_ref_elev_table.csv -s $outputHucDataDir/demDerived_reaches_split.gpkg
+$srcDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -s $outputHucDataDir/demDerived_reaches_split.gpkg
 Tcount
 
 ## DINF DISTANCE DOWN ##
@@ -432,6 +432,14 @@ Tstart
 $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
+
+## USGS CROSSWALK ##
+echo -e $startDiv"USGS Crosswalk $hucNumber"$stopDiv
+date -u
+Tstart
+$srcDir/usgs_gage_crosswalk.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -dem $outputHucDataDir/dem_meters.tif -flows $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -cat $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -wbd $outputHucDataDir/wbd_buffered.gpkg -dem_adj $dem_thalwegCond -outtable $outputHucDataDir/usgs_elev_table.csv
+Tcount
+
 ## CLEANUP OUTPUTS ##
 echo -e $startDiv"Cleaning up outputs $hucNumber"$stopDiv
 args=()
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
new file mode 100755
index 000000000..29ef7b592
--- /dev/null
+++ b/src/usgs_gage_crosswalk.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+import os
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+import rasterio
+import argparse
+import pygeos
+from shapely.wkb import dumps, loads
+import warnings
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+''' Get elevation at adjusted USGS gages locations
+
+    Parameters
+    ----------
+    usgs_gages_filename : str
+        File name of USGS stations layer.
+    dem_filename : str
+        File name of original DEM.
+    input_flows_filename : str
+        File name of FIM streams layer.
+    input_catchment_filename : str
+        File name of FIM catchment layer.
+    wbd_buffer_filename : str
+        File name of buffered wbd.
+    dem_adj_filename : str
+        File name of thalweg adjusted DEM.
+    output_table_filename : str
+        File name of output table.
+'''
+
+
+def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename,dem_adj_filename,output_table_filename):
+
+    wbd_buffer = gpd.read_file(wbd_buffer_filename)
+    usgs_gages = gpd.read_file(usgs_gages_filename, mask=wbd_buffer)
+    dem_m = rasterio.open(dem_filename,'r')
+    input_flows = gpd.read_file(input_flows_filename)
+    input_catchment = gpd.read_file(input_catchment_filename)
+    dem_adj = rasterio.open(dem_adj_filename,'r')
+
+    if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
+
+    # Identify closest HydroID
+    closest_catchment = gpd.sjoin(usgs_gages, input_catchment, how='left', op='within').reset_index(drop=True)
+    closest_hydro_id = closest_catchment.filter(items=['site_no','HydroID','min_thal_elev','med_thal_elev','max_thal_elev', 'order_'])
+    closest_hydro_id = closest_hydro_id.dropna()
+
+    # Get USGS gages that are within catchment boundaries
+    usgs_gages = usgs_gages.loc[usgs_gages.site_no.isin(list(closest_hydro_id.site_no))]
+
+    columns = ['location_id','HydroID','dem_elevation','dem_adj_elevation','min_thal_elev', 'med_thal_elev','max_thal_elev','str_order']
+    gage_data = []
+
+    # Move USGS gage to stream
+    for index, gage in usgs_gages.iterrows():
+
+        # Get stream attributes
+        hydro_id = closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].HydroID.item()
+        str_order = str(int(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].order_.item()))
+        min_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].min_thal_elev.item(),2)
+        med_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].med_thal_elev.item(),2)
+        max_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].max_thal_elev.item(),2)
+
+        # Convert headwater point geometries to WKB representation
+        wkb_gages = dumps(gage.geometry)
+
+        # Create pygeos headwater point geometries from WKB representation
+        gage_bin_geom = pygeos.io.from_wkb(wkb_gages)
+
+        # Closest segment to headwater
+        closest_stream = input_flows.loc[input_flows.HydroID==hydro_id]
+        wkb_closest_stream = dumps(closest_stream.geometry.item())
+        stream_bin_geom = pygeos.io.from_wkb(wkb_closest_stream)
+
+        # Linear reference headwater to closest stream segment
+        gage_distance_to_line = pygeos.linear.line_locate_point(stream_bin_geom, gage_bin_geom)
+        referenced_gage = pygeos.linear.line_interpolate_point(stream_bin_geom, gage_distance_to_line)
+
+        # Convert geometries to wkb representation
+        bin_referenced_gage = pygeos.io.to_wkb(referenced_gage)
+
+        # Convert to shapely geometries
+        shply_referenced_gage = loads(bin_referenced_gage)
+
+        # Sample rasters at adjusted gage
+        dem_m_elev = round(list(rasterio.sample.sample_gen(dem_m,shply_referenced_gage.coords))[0].item(),2)
+        dem_adj_elev = round(list(rasterio.sample.sample_gen(dem_adj,shply_referenced_gage.coords))[0].item(),2)
+
+        # Append dem_m_elev, dem_adj_elev, hydro_id, and gage number to table
+        site_elevations = [str(gage.site_no), str(hydro_id), dem_m_elev, dem_adj_elev, min_thal_elev, med_thal_elev, max_thal_elev,str(str_order)]
+        gage_data.append(site_elevations)
+
+
+    elev_table = pd.DataFrame(gage_data, columns=columns)
+
+    if not elev_table.empty:
+        elev_table.to_csv(output_table_filename,index=False)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Crosswalk USGS sites to HydroID and get elevations')
+    parser.add_argument('-gages','--usgs-gages-filename', help='USGS gages', required=True)
+    parser.add_argument('-dem','--dem-filename',help='DEM',required=True)
+    parser.add_argument('-flows','--input-flows-filename', help='DEM derived streams', required=True)
+    parser.add_argument('-cat','--input-catchment-filename', help='DEM derived catchments', required=True)
+    parser.add_argument('-wbd','--wbd-buffer-filename', help='WBD buffer', required=True)
+    parser.add_argument('-dem_adj','--dem-adj-filename', help='Thalweg adjusted DEM', required=True)
+    parser.add_argument('-outtable','--output-table-filename', help='Table to append data', required=True)
+
+    args = vars(parser.parse_args())
+
+    usgs_gages_filename = args['usgs_gages_filename']
+    dem_filename = args['dem_filename']
+    input_flows_filename = args['input_flows_filename']
+    input_catchment_filename = args['input_catchment_filename']
+    wbd_buffer_filename = args['wbd_buffer_filename']
+    dem_adj_filename = args['dem_adj_filename']
+    output_table_filename = args['output_table_filename']
+
+    crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename, dem_adj_filename,output_table_filename)
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
new file mode 100755
index 000000000..d8df892fb
--- /dev/null
+++ b/tools/rating_curve_comparison.py
@@ -0,0 +1,408 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+import argparse
+import matplotlib.pyplot as plt
+import seaborn as sns
+from functools import reduce
+from multiprocessing import Pool
+from os.path import isfile, join, dirname
+import shutil
+import warnings
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+"""
+    Plot Rating Curves and Compare to USGS Gages
+
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders.
+    output_dir : str
+        Directory containing rating curve plots and tables.
+    usgs_gages_filename : str
+        File name of USGS rating curves.
+    nwm_flow_dir : str
+        Directory containing NWM recurrence flows files.
+    number_of_jobs : str
+        Number of jobs.
+    stat_groups : str
+        string of columns to group eval metrics.
+"""
+
+# recurr_intervals = ['recurr_1_5_cms.csv','recurr_5_0_cms.csv','recurr_10_0_cms.csv']
+
+def generate_rating_curve_metrics(args):
+
+    elev_table_filename         = args[0]
+    hydrotable_filename         = args[1]
+    usgs_gages_filename         = args[2]
+    usgs_recurr_stats_filename  = args[3]
+    nwm_recurr_data_filename    = args[4]
+    rc_comparison_plot_filename = args[5]
+    nwm_flow_dir                = args[6]
+    huc                         = args[7]
+
+    elev_table = pd.read_csv(elev_table_filename,dtype={'location_id': str})
+    hydrotable = pd.read_csv(hydrotable_filename,dtype={'HUC': str,'feature_id': str})
+    usgs_gages = pd.read_csv(usgs_gages_filename,dtype={'location_id': str})
+
+    # Join rating curves with elevation data
+    hydrotable = hydrotable.merge(elev_table, on="HydroID")
+    relevant_gages = list(hydrotable.location_id.unique())
+    usgs_gages = usgs_gages[usgs_gages['location_id'].isin(relevant_gages)]
+    usgs_gages = usgs_gages.reset_index(drop=True)
+
+    if len(usgs_gages) > 0:
+
+        # Adjust rating curve to elevation
+        hydrotable['elevation_ft'] = (hydrotable.stage + hydrotable.dem_adj_elevation) * 3.28084 # convert from m to ft
+        # hydrotable['raw_elevation_ft'] = (hydrotable.stage + hydrotable.dem_elevation) * 3.28084 # convert from m to ft
+        hydrotable['discharge_cfs'] = hydrotable.discharge_cms * 35.3147
+        usgs_gages = usgs_gages.rename(columns={"flow": "discharge_cfs", "elevation_navd88": "elevation_ft"})
+
+        hydrotable['source'] = "FIM"
+        usgs_gages['source'] = "USGS"
+        limited_hydrotable = hydrotable.filter(items=['location_id','elevation_ft','discharge_cfs','source'])
+        select_usgs_gages = usgs_gages.filter(items=['location_id', 'elevation_ft', 'discharge_cfs','source'])
+
+        rating_curves = limited_hydrotable.append(select_usgs_gages)
+
+        # Add stream order
+        stream_orders = hydrotable.filter(items=['location_id','str_order']).drop_duplicates()
+        rating_curves = rating_curves.merge(stream_orders, on='location_id')
+        rating_curves['str_order'] = rating_curves['str_order'].astype('int')
+
+        # plot rating curves
+        generate_facet_plot(rating_curves, rc_comparison_plot_filename)
+
+        # NWM recurr intervals
+        recurr_1_5_yr_filename = join(nwm_flow_dir,'recurr_1_5_cms.csv')
+        recurr_5_yr_filename = join(nwm_flow_dir,'recurr_5_0_cms.csv')
+        recurr_10_yr_filename = join(nwm_flow_dir,'recurr_10_0_cms.csv')
+
+        # Update column names
+        recurr_1_5_yr = pd.read_csv(recurr_1_5_yr_filename,dtype={'feature_id': str})
+        recurr_1_5_yr = recurr_1_5_yr.rename(columns={"discharge": "1.5"})
+        recurr_5_yr = pd.read_csv(recurr_5_yr_filename,dtype={'feature_id': str})
+        recurr_5_yr = recurr_5_yr.rename(columns={"discharge": "5.0"})
+        recurr_10_yr = pd.read_csv(recurr_10_yr_filename,dtype={'feature_id': str})
+        recurr_10_yr = recurr_10_yr.rename(columns={"discharge": "10.0"})
+
+        # Merge NWM recurr intervals into a single layer
+        nwm_recurr_intervals_all = reduce(lambda x,y: pd.merge(x,y, on='feature_id', how='outer'), [recurr_1_5_yr, recurr_5_yr, recurr_10_yr])
+        nwm_recurr_intervals_all = pd.melt(nwm_recurr_intervals_all, id_vars=['feature_id'], value_vars=['1.5','5.0','10.0'], var_name='recurr_interval', value_name='discharge_cms')
+        nwm_recurr_intervals_all['discharge_cfs'] = nwm_recurr_intervals_all.discharge_cms * 35.3147
+        nwm_recurr_intervals_all = nwm_recurr_intervals_all.filter(items=['discharge_cfs', 'recurr_interval','feature_id']).drop_duplicates()
+
+
+        # Identify unique gages
+        usgs_crosswalk = hydrotable.filter(items=['location_id', 'feature_id']).drop_duplicates()
+
+        nwm_recurr_data_table = pd.DataFrame()
+        usgs_recurr_data = pd.DataFrame()
+
+        # Interpolate USGS/FIM elevation at each gage
+        for index, gage in usgs_crosswalk.iterrows():
+
+            # Interpolate USGS elevation at NWM recurrence intervals
+            usgs_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="USGS")]
+
+            if len(usgs_rc) <1:
+                print(f"missing USGS rating curve data for usgs station {gage.location_id} in huc {huc}")
+                continue
+
+            str_order = np.unique(usgs_rc.str_order).item()
+            feature_id = str(gage.feature_id)
+
+            usgs_pred_elev = get_reccur_intervals(usgs_rc, usgs_crosswalk,nwm_recurr_intervals_all)
+
+            # Handle sites missing data
+            if len(usgs_pred_elev) <1:
+                print(f"missing USGS elevation data for usgs station {gage.location_id} in huc {huc}")
+                continue
+
+            # Clean up data
+            usgs_pred_elev['location_id'] = gage.location_id
+            usgs_pred_elev = usgs_pred_elev.filter(items=['location_id','recurr_interval', 'discharge_cfs','pred_elev'])
+            usgs_pred_elev = usgs_pred_elev.rename(columns={"pred_elev": "USGS"})
+
+            # Interpolate FIM elevation at NWM recurrence intervals
+            fim_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="FIM")]
+
+            if len(fim_rc) <1:
+                print(f"missing FIM rating curve data for usgs station {gage.location_id} in huc {huc}")
+                continue
+
+            fim_pred_elev = get_reccur_intervals(fim_rc, usgs_crosswalk,nwm_recurr_intervals_all)
+
+            # Handle sites missing data
+            if len(fim_pred_elev) <1:
+                print(f"missing FIM elevation data for usgs station {gage.location_id} in huc {huc}")
+                continue
+
+            # Clean up data
+            fim_pred_elev = fim_pred_elev.rename(columns={"pred_elev": "FIM"})
+            fim_pred_elev = fim_pred_elev.filter(items=['recurr_interval', 'discharge_cfs','FIM'])
+            usgs_pred_elev = usgs_pred_elev.merge(fim_pred_elev, on=['recurr_interval','discharge_cfs'])
+
+            # Add attributes
+            usgs_pred_elev['HUC'] = huc
+            usgs_pred_elev['HUC4'] = huc[0:4]
+            usgs_pred_elev['str_order'] = str_order
+            usgs_pred_elev['feature_id'] = feature_id
+
+            # Melt dataframe
+            usgs_pred_elev = pd.melt(usgs_pred_elev, id_vars=['location_id','feature_id','recurr_interval','discharge_cfs','HUC','HUC4','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
+            nwm_recurr_data_table = nwm_recurr_data_table.append(usgs_pred_elev)
+
+            # Interpolate FIM elevation at USGS observations
+            # fim_rc = fim_rc.merge(usgs_crosswalk, on="location_id")
+            # usgs_rc = usgs_rc.rename(columns={"elevation_ft": "USGS"})
+            #
+            # # Sort stage in ascending order
+            # usgs_rc = usgs_rc.sort_values('USGS',ascending=True)
+            #
+            # # Interpolate FIM elevation at USGS observations
+            # usgs_rc['FIM'] = np.interp(usgs_rc.discharge_cfs.values, fim_rc['discharge_cfs'], fim_rc['elevation_ft'], left = np.nan, right = np.nan)
+            # usgs_rc = usgs_rc[usgs_rc['FIM'].notna()]
+            # usgs_rc = usgs_rc.drop(columns=["source"])
+            #
+            # # Melt dataframe
+            # usgs_rc = pd.melt(usgs_rc, id_vars=['location_id','discharge_cfs','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
+            #
+            # if not usgs_rc.empty:
+            #     usgs_recurr_data = usgs_recurr_data.append(usgs_rc)
+
+        # Generate stats for all sites in huc
+        # if not usgs_recurr_data.empty:
+        #     usgs_recurr_stats_table = calculate_rc_stats_elev(usgs_recurr_data)
+        #     usgs_recurr_stats_table.to_csv(usgs_recurr_stats_filename,index=False)
+
+        # # Generate plots (not currently being used)
+        # fim_elev_at_USGS_rc_plot_filename = join(dirname(rc_comparison_plot_filename),'FIM_elevations_at_USGS_rc_' + str(huc) +'.png')
+        # generate_facet_plot(usgs_recurr_data, fim_elev_at_USGS_rc_plot_filename)
+
+        if not nwm_recurr_data_table.empty:
+            nwm_recurr_data_table.discharge_cfs = np.round(nwm_recurr_data_table.discharge_cfs,2)
+            nwm_recurr_data_table.elevation_ft = np.round(nwm_recurr_data_table.elevation_ft,2)
+            nwm_recurr_data_table.to_csv(nwm_recurr_data_filename,index=False)
+
+    else:
+        print(f"no USGS data for gage(s): {relevant_gages} in huc {huc}")
+
+def aggregate_metrics(output_dir,procs_list,stat_groups):
+
+    # agg_usgs_interp_elev_stats = join(output_dir,'agg_usgs_interp_elev_stats.csv')
+    agg_nwm_recurr_flow_elev = join(output_dir,'agg_nwm_recurr_flow_elevations.csv')
+    agg_nwm_recurr_flow_elev_stats = join(output_dir,f"agg_nwm_recurr_flow_elev_stats_{'_'.join(stat_groups)}.csv")
+
+    # if os.path.isfile(agg_usgs_interp_elev_stats):
+    #     os.remove(agg_usgs_interp_elev_stats)
+    if os.path.isfile(agg_nwm_recurr_flow_elev):
+        os.remove(agg_nwm_recurr_flow_elev)
+    if os.path.isfile(agg_nwm_recurr_flow_elev_stats):
+        os.remove(agg_nwm_recurr_flow_elev_stats)
+
+    for huc in procs_list:
+        # if os.path.isfile(huc[3]):
+        #     usgs_recurr_stats = pd.read_csv(huc[3])
+        #
+        #     # Write/append usgs_recurr_stats
+        #     if os.path.isfile(agg_usgs_interp_elev_stats):
+        #         usgs_recurr_stats.to_csv(agg_usgs_interp_elev_stats,index=False, mode='a',header=False)
+        #     else:
+        #         usgs_recurr_stats.to_csv(agg_usgs_interp_elev_stats,index=False)
+
+        if os.path.isfile(huc[4]):
+            nwm_recurr_data = pd.read_csv(huc[4],dtype={'location_id': str,
+                                                        'feature_id': str})
+
+            # Write/append nwm_recurr_data
+            if os.path.isfile(agg_nwm_recurr_flow_elev):
+                nwm_recurr_data.to_csv(agg_nwm_recurr_flow_elev,index=False, mode='a',header=False)
+            else:
+                nwm_recurr_data.to_csv(agg_nwm_recurr_flow_elev,index=False)
+
+    agg_stats = pd.read_csv(agg_nwm_recurr_flow_elev,dtype={'location_id': str,
+                                                            'feature_id': str})
+
+    agg_recurr_stats_table = calculate_rc_stats_elev(agg_stats,stat_groups)
+
+    agg_recurr_stats_table.to_csv(agg_nwm_recurr_flow_elev_stats,index=False)
+
+
+def generate_facet_plot(rc, plot_filename):
+
+    # Filter FIM elevation based on USGS data
+    for gage in rc.location_id.unique():
+
+        min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
+        max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
+
+        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft > (max_elev + 2))].index)
+        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2)].index)
+
+    rc = rc.rename(columns={"location_id": "USGS Gage"})
+
+    ## Generate rating curve plots
+    num_plots = len(rc["USGS Gage"].unique())
+    if num_plots > 3:
+        columns = num_plots // 3
+    else:
+        columns = 1
+
+    sns.set(style="ticks")
+    g = sns.FacetGrid(rc, col="USGS Gage", hue="source",sharex=False, sharey=False,col_wrap=columns)
+    g.map(sns.scatterplot, "discharge_cfs", "elevation_ft", palette="tab20c", marker="o")
+    g.set_axis_labels(x_var="Discharge (cfs)", y_var="Elevation (ft)")
+
+    # Adjust the arrangement of the plots
+    g.fig.tight_layout(w_pad=1)
+    g.add_legend()
+
+    plt.savefig(plot_filename)
+    plt.close()
+
+
+def get_reccur_intervals(site_rc, usgs_crosswalk,nwm_recurr_intervals):
+
+    usgs_site = site_rc.merge(usgs_crosswalk, on="location_id")
+    nwm_ids = len(usgs_site.feature_id.drop_duplicates())
+
+    if nwm_ids > 0:
+
+        nwm_recurr_intervals = nwm_recurr_intervals.copy().loc[nwm_recurr_intervals.feature_id==usgs_site.feature_id.drop_duplicates().item()]
+        nwm_recurr_intervals['pred_elev'] = np.interp(nwm_recurr_intervals.discharge_cfs.values, usgs_site['discharge_cfs'], usgs_site['elevation_ft'], left = np.nan, right = np.nan)
+
+        return nwm_recurr_intervals
+
+    else:
+        return []
+
+
+def calculate_rc_stats_elev(rc,stat_groups=None):
+
+    usgs_elev = "USGS"
+    src_elev = "FIM"
+
+    # Collect any extra columns not associated with melt
+    col_index = list(rc.columns)
+    pivot_vars = ['source','elevation_ft']
+    col_index = [col for col in col_index if col not in pivot_vars]
+
+    # Unmelt elevation/source
+    rc_unmelt = (rc.set_index(col_index)
+        .pivot(columns="source")['elevation_ft']
+        .reset_index()
+        .rename_axis(None, axis=1)
+     )
+
+    if stat_groups is None:
+        stat_groups = ['location_id']
+
+    # Calculate variables for NRMSE
+    rc_unmelt["yhat_minus_y"] = rc_unmelt[src_elev] - rc_unmelt[usgs_elev]
+    rc_unmelt["yhat_minus_y_squared"] = rc_unmelt["yhat_minus_y"] ** 2
+
+    # Calculate metrics by group
+    station_rc = rc_unmelt.groupby(stat_groups)
+
+    # Calculate variables for NRMSE
+    sum_y_diff = station_rc.apply(lambda x: x["yhat_minus_y_squared"].sum())\
+        .reset_index(stat_groups, drop = False).rename({0: "sum_y_diff"}, axis=1)
+
+    # Determine number of events that are modeled
+    n = station_rc.apply(lambda x: x[usgs_elev].count())\
+        .reset_index(stat_groups, drop = False).rename({0: "n"}, axis=1)
+
+    # Determine the maximum/minimum USGS elevation
+    y_max = station_rc.apply(lambda x: x[usgs_elev].max())\
+        .reset_index(stat_groups, drop = False).rename({0: "y_max"}, axis=1)
+    y_min = station_rc.apply(lambda x: x[usgs_elev].min())\
+        .reset_index(stat_groups, drop = False).rename({0: "y_min"}, axis=1)
+
+    # Collect variables for NRMSE
+    nrmse_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [sum_y_diff, n, y_max, y_min])
+    nrmse_table_group = nrmse_table.groupby(stat_groups)
+
+    # Calculate nrmse
+    nrmse = nrmse_table_group.apply(lambda x: ((x['sum_y_diff'] / x['n']) ** 0.5) / (x['y_max'] - x['y_min']))\
+        .reset_index(stat_groups, drop = False).rename({0: "nrmse"}, axis=1)
+
+    # Calculate Mean Absolute Depth Difference
+    mean_abs_y_diff = station_rc.apply(lambda x: (abs(x["yhat_minus_y"]).mean()))\
+        .reset_index(stat_groups, drop = False).rename({0: "mean_abs_y_diff_ft"}, axis=1)
+
+    # Calculate Percent Bias
+    percent_bias = station_rc.apply(lambda x: 100 * (x["yhat_minus_y"].sum() / x[usgs_elev].sum()))\
+        .reset_index(stat_groups, drop = False).rename({0: "percent_bias"}, axis=1)
+
+    rc_stat_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [nrmse, mean_abs_y_diff, percent_bias])
+
+    return rc_stat_table
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='generate rating curve plots and tables for FIM and USGS gages')
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True)
+    parser.add_argument('-output_dir','--output-dir', help='rating curves output folder', required=True)
+    parser.add_argument('-gages','--usgs-gages-filename',help='USGS rating curves',required=True)
+    parser.add_argument('-flows','--nwm-flow-dir',help='NWM recurrence flows dir',required=True)
+    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
+    parser.add_argument('-group','--stat-groups',help='column(s) to group stats',required=False)
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    output_dir = args['output_dir']
+    usgs_gages_filename = args['usgs_gages_filename']
+    nwm_flow_dir = args['nwm_flow_dir']
+    number_of_jobs = args['number_of_jobs']
+    stat_groups = args['stat_groups']
+
+    stat_groups = stat_groups.split()
+    procs_list = []
+
+    plots_dir = join(output_dir,'plots')
+    os.makedirs(plots_dir, exist_ok=True)
+    tables_dir = join(output_dir,'tables')
+    os.makedirs(tables_dir, exist_ok=True)
+
+    # Open log file
+    sys.__stdout__ = sys.stdout
+    log_file = open(join(output_dir,'rating_curve_comparison.log'),"w")
+    sys.stdout = log_file
+
+    huc_list  = os.listdir(fim_dir)
+    for huc in huc_list:
+
+        if huc != 'logs':
+            elev_table_filename = join(fim_dir,huc,'usgs_elev_table.csv')
+            hydrotable_filename = join(fim_dir,huc,'hydroTable.csv')
+            usgs_recurr_stats_filename = join(tables_dir,f"usgs_interpolated_elevation_stats_{huc}.csv")
+            nwm_recurr_data_filename = join(tables_dir,f"nwm_recurrence_flow_elevations_{huc}.csv")
+            rc_comparison_plot_filename = join(plots_dir,f"FIM-USGS_rating_curve_comparison_{huc}.png")
+
+            if isfile(elev_table_filename):
+                procs_list.append([elev_table_filename, hydrotable_filename, usgs_gages_filename, usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,nwm_flow_dir,huc])
+
+    # Initiate multiprocessing
+    print(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    pool = Pool(number_of_jobs)
+    pool.map(generate_rating_curve_metrics, procs_list)
+
+    print(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
+    aggregate_metrics(output_dir,procs_list,stat_groups)
+
+    print('Delete intermediate tables')
+    shutil.rmtree(tables_dir, ignore_errors=True)
+
+    # Close log file
+    sys.stdout = sys.__stdout__
+    log_file.close()

From 40631f42e5f9ac97204c9a54d4bd9cb7204efffa Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Mon, 5 Apr 2021 13:45:09 -0500
Subject: [PATCH 062/359] Create tool to retrieve rating curves from USGS sites
 and convert to elevation (NAVD88). Intended to be used as part of the Sierra
 Test.

- Modify usgs_gage_crosswalk.py to:
    - Look for location_id instead of site_no attribute field in usgs_gages.gpkg file.
    - Filter out gages that do not have rating curves included in the usgs_rating_curves.csv
- Modify rating_curve_comparison.py to perform a check on the age of the user specified usgs_rating_curves.csv and alert user to the age of the file and recommend updating if file is older the 30 days.
- Add rating_curve_get_usgs_curves.py. This script will generate the following files:
    - usgs_rating_curves.csv: A csv file that contains rating curves (including converted to NAVD88 elevation) for USGS gages in a format that is compatible with rating_curve_comparisons.py. As it is is currently configured, only gages within CONUS will have rating curve data.
    - log.csv: A log file that records status for each gage and includes error messages.
    - usgs_gages.gpkg: A geospatial layer (in FIM projection) of all active USGS gages that meet a predefined criteria. Additionally, the curve attribute indicates whether a rating curve is found in the usgs_rating_curves.csv. This spatial file is only generated if the all option is passed with the -l argument.

This resolves #289.
---
 CHANGELOG.md                          |  29 ++-
 src/usgs_gage_crosswalk.py            |  19 +-
 tools/rating_curve_comparison.py      |  25 +++
 tools/rating_curve_get_usgs_curves.py | 288 ++++++++++++++++++++++++++
 tools/tools_shared_functions.py       | 210 +++++++++++++++++++
 5 files changed, 556 insertions(+), 15 deletions(-)
 create mode 100644 tools/rating_curve_get_usgs_curves.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8845241ae..ba1d25901 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,17 +1,32 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v3.0.14.0 - 2021-04-05 - [PR #338](https://github.com/NOAA-OWP/cahaba/pull/338)
 
-## v3.0.12.2 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
+Create tool to retrieve rating curves from USGS sites and convert to elevation (NAVD88). Intended to be used as part of the Sierra Test.
+ 
+### Changes
+ - Modify `usgs_gage_crosswalk.py` to:
+    1) Look for `location_id` instead of `site_no` attribute field in `usgs_gages.gpkg` file.
+    2) Filter out gages that do not have rating curves included in the `usgs_rating_curves.csv`.
+ - Modify `rating_curve_comparison.py` to perform a check on the age of the user specified `usgs_rating_curves.csv` and alert user to the age of the file and recommend updating if file is older the 30 days.
+
+### Additions
+ - Add `rating_curve_get_usgs_curves.py`. This script will generate the following files:
+     1) `usgs_rating_curves.csv`: A csv file that contains rating curves (including converted to NAVD88 elevation) for USGS gages in a format that is compatible with  `rating_curve_comparisons.py`. As it is is currently configured, only gages within CONUS will have rating curve data.
+     2) `log.csv`: A log file that records status for each gage and includes error messages.
+     3) `usgs_gages.gpkg`: A geospatial layer (in FIM projection) of all active USGS gages that meet a predefined criteria. Additionally, the `curve` attribute indicates whether a rating curve is found in the `usgs_rating_curves.csv`. This spatial file is only generated if the `all` option is passed with the `-l` argument.
+<br/><br/>
+## v3.0.13.0 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
 
-Created tool to compare synthetic rating curve with benchmark rating curve (sierra test). resolves issue #293; resolves issue #308
+Created tool to compare synthetic rating curve with benchmark rating curve (Sierra Test).
  
 ### Changes
- - update `aggregate_fim_outputs.py` call argument in `fim_run.sh` from 4 jobs to 6 jobs (optimizing API performance)
- - reroutes median elevation data from `add_crosswalk.py` and `rem.py` to new file (depreciating `hand_ref_elev_table.csv`)
- - adding new files to `viz_whitelist` in `output_cleanup.py`
+ - Update `aggregate_fim_outputs.py` call argument in `fim_run.sh` from 4 jobs to 6 jobs, to optimize API performance.
+ - Reroutes median elevation data from `add_crosswalk.py` and `rem.py` to new file (depreciating `hand_ref_elev_table.csv`).
+ - Adds new files to `viz_whitelist` in `output_cleanup.py`.
 
 ### Additions
- - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in run_by_unit.py with elevation and additional attributes at USGS gages. 
+ - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in `run_by_unit.py` with elevation and additional attributes at USGS gages. 
  - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data. 
 <br/><br/>
 
@@ -36,7 +51,7 @@ Add more detail/information to plotting capabilities.
 ### Changes
  - Merge `plot_functions.py` into `eval_plots.py` and move `eval_plots.py` into the tools directory.
  - Remove `plots` subdirectory.
- - 
+
 ### Additions
  - Optional argument to create barplots of CSI for each individual site.
  - Create a csv containing the data used to create the scatterplots. 
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 29ef7b592..fb4b9533d 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -41,15 +41,18 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     input_catchment = gpd.read_file(input_catchment_filename)
     dem_adj = rasterio.open(dem_adj_filename,'r')
 
+    #Query out usgs_gages that don't have rating curve data
+    usgs_gages = usgs_gages.query('curve == "yes"')
+
     if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
 
     # Identify closest HydroID
     closest_catchment = gpd.sjoin(usgs_gages, input_catchment, how='left', op='within').reset_index(drop=True)
-    closest_hydro_id = closest_catchment.filter(items=['site_no','HydroID','min_thal_elev','med_thal_elev','max_thal_elev', 'order_'])
+    closest_hydro_id = closest_catchment.filter(items=['location_id','HydroID','min_thal_elev','med_thal_elev','max_thal_elev', 'order_'])
     closest_hydro_id = closest_hydro_id.dropna()
 
     # Get USGS gages that are within catchment boundaries
-    usgs_gages = usgs_gages.loc[usgs_gages.site_no.isin(list(closest_hydro_id.site_no))]
+    usgs_gages = usgs_gages.loc[usgs_gages.location_id.isin(list(closest_hydro_id.location_id))]
 
     columns = ['location_id','HydroID','dem_elevation','dem_adj_elevation','min_thal_elev', 'med_thal_elev','max_thal_elev','str_order']
     gage_data = []
@@ -58,11 +61,11 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     for index, gage in usgs_gages.iterrows():
 
         # Get stream attributes
-        hydro_id = closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].HydroID.item()
-        str_order = str(int(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].order_.item()))
-        min_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].min_thal_elev.item(),2)
-        med_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].med_thal_elev.item(),2)
-        max_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.site_no==gage.site_no].max_thal_elev.item(),2)
+        hydro_id = closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].HydroID.item()
+        str_order = str(int(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].order_.item()))
+        min_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].min_thal_elev.item(),2)
+        med_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].med_thal_elev.item(),2)
+        max_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].max_thal_elev.item(),2)
 
         # Convert headwater point geometries to WKB representation
         wkb_gages = dumps(gage.geometry)
@@ -90,7 +93,7 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
         dem_adj_elev = round(list(rasterio.sample.sample_gen(dem_adj,shply_referenced_gage.coords))[0].item(),2)
 
         # Append dem_m_elev, dem_adj_elev, hydro_id, and gage number to table
-        site_elevations = [str(gage.site_no), str(hydro_id), dem_m_elev, dem_adj_elev, min_thal_elev, med_thal_elev, max_thal_elev,str(str_order)]
+        site_elevations = [str(gage.location_id), str(hydro_id), dem_m_elev, dem_adj_elev, min_thal_elev, med_thal_elev, max_thal_elev,str(str_order)]
         gage_data.append(site_elevations)
 
 
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index d8df892fb..77e6b91ba 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -13,6 +13,8 @@
 from os.path import isfile, join, dirname
 import shutil
 import warnings
+from pathlib import Path
+import time
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
 """
@@ -33,6 +35,26 @@
     stat_groups : str
         string of columns to group eval metrics.
 """
+def check_file_age(file):
+    '''
+    Checks if file exists, determines the file age, and recommends
+    updating if older than 1 month. 
+
+    Returns
+    -------
+    None.
+
+    '''    
+    file = Path(file)
+    if file.is_file():
+        modification_time = file.stat().st_mtime
+        current_time = time.time()
+        file_age_days = (current_time - modification_time)/86400
+        if file_age_days > 30:
+            check = f'{file.name} is {int(file_age_days)} days old, consider updating.\nUpdate with rating_curve_get_usgs_curves.py'
+        else:
+            check = f'{file.name} is {int(file_age_days)} days old.'
+    return check
 
 # recurr_intervals = ['recurr_1_5_cms.csv','recurr_5_0_cms.csv','recurr_10_0_cms.csv']
 
@@ -374,6 +396,9 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     tables_dir = join(output_dir,'tables')
     os.makedirs(tables_dir, exist_ok=True)
 
+    #Check age of gages csv and recommend updating if older than 30 days.
+    print(check_file_age(usgs_gages_filename))
+
     # Open log file
     sys.__stdout__ = sys.stdout
     log_file = open(join(output_dir,'rating_curve_comparison.log'),"w")
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
new file mode 100644
index 000000000..eb43bab3e
--- /dev/null
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+import time
+import pandas as pd
+import geopandas as gpd
+from pathlib import Path
+from tools_shared_functions import get_metadata, get_datum, ngvd_to_navd_ft, get_rating_curve, aggregate_wbd_hucs
+from dotenv import load_dotenv
+import os
+import argparse
+import sys
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+
+'''
+This script calls the NOAA Tidal API for datum conversions. Experience shows that
+running script outside of business hours seems to be most consistent way
+to avoid API errors. Currently configured to get rating curve data within
+CONUS. Tidal API call may need to be modified to get datum conversions for 
+AK, HI, PR/VI.
+'''
+
+#import variables from .env file
+load_dotenv()
+API_BASE_URL = os.getenv("API_BASE_URL")
+WBD_LAYER = os.getenv("WBD_LAYER")
+
+def get_all_active_usgs_sites():
+    '''
+    Compile a list of all active usgs gage sites that meet certain criteria. 
+    Return a GeoDataFrame of all sites.
+
+    Returns
+    -------
+    None.
+
+    '''
+    #Get metadata for all usgs_site_codes that are active in the U.S.
+    metadata_url = f'{API_BASE_URL}/metadata' 
+    #Define arguments to retrieve metadata and then get metadata from WRDS
+    select_by = 'usgs_site_code'
+    selector = ['all']
+    must_include = 'usgs_data.active'
+    metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = None )
+
+    #Filter out sites based quality of site. These acceptable codes were initially
+    #decided upon and may need fine tuning. A link where more information
+    #regarding the USGS attributes is provided. 
+    
+    #https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html
+    acceptable_coord_acc_code = ['H','1','5','S','R','B','C','D','E']
+    #https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html
+    acceptable_coord_method_code = ['C','D','W','X','Y','Z','N','M','L','G','R','F','S']
+    #https://help.waterdata.usgs.gov/codes-and-parameters/codes#SI
+    acceptable_alt_acc_thresh = 1
+    #https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html
+    acceptable_alt_meth_code = ['A','D','F','I','J','L','N','R','W','X','Y','Z']
+    #https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html
+    acceptable_site_type = ['ST']
+    
+    #Cycle through each site and filter out if site doesn't meet criteria.
+    acceptable_sites_metadata = []
+    for metadata in metadata_list:
+        #Get the usgs info from each site
+        usgs_data = metadata['usgs_data']
+                
+        #Get site quality attributes      
+        coord_accuracy_code = usgs_data.get('coord_accuracy_code')        
+        coord_method_code =   usgs_data.get('coord_method_code')
+        alt_accuracy_code =   usgs_data.get('alt_accuracy_code')
+        alt_method_code =     usgs_data.get('alt_method_code')
+        site_type =           usgs_data.get('site_type')
+        
+        #Check to make sure that none of the codes were null, if null values are found, skip to next.
+        if not all([coord_accuracy_code, coord_method_code, alt_accuracy_code, alt_method_code, site_type]):
+            continue
+        
+        #Test if site meets criteria.
+        if (coord_accuracy_code in acceptable_coord_acc_code and 
+            coord_method_code in acceptable_coord_method_code and
+            alt_accuracy_code <= acceptable_alt_acc_thresh and 
+            alt_method_code in acceptable_alt_meth_code and
+            site_type in acceptable_site_type):
+            
+            #If nws_lid is not populated then add a dummy ID so that 'aggregate_wbd_hucs' works correctly.
+            if not metadata.get('identifiers').get('nws_lid'):
+                metadata['identifiers']['nws_lid'] = 'Bogus_ID' 
+            
+            #Append metadata of acceptable site to acceptable_sites list.
+            acceptable_sites_metadata.append(metadata)  
+        
+    #Get a geospatial layer (gdf) for all acceptable sites
+    dictionary, gdf = aggregate_wbd_hucs(acceptable_sites_metadata, Path(WBD_LAYER), retain_attributes = False)
+    #Get a list of all sites in gdf
+    list_of_sites = gdf['identifiers_usgs_site_code'].to_list()
+    #Rename gdf fields
+    gdf.columns = gdf.columns.str.replace('identifiers_','')
+
+    return gdf, list_of_sites, acceptable_sites_metadata
+            
+            
+def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
+    '''
+
+    Returns rating curves, for a set of sites, adjusted to elevation NAVD.
+    Currently configured to get rating curve data within CONUS. Tidal API 
+    call may need to be modified to get datum conversions for AK, HI, PR/VI.
+    Workflow as follows:
+        1a. If 'all' option passed, get metadata for all acceptable USGS sites in CONUS.
+        1b. If a list of sites passed, get metadata for all sites supplied by user.
+        2.  Extract datum information for each site.
+        3.  If site is not in contiguous US skip (due to issue with datum conversions)
+        4.  Convert datum if NGVD
+        5.  Get rating curve for each site individually
+        6.  Convert rating curve to absolute elevation (NAVD) and store in DataFrame
+        7.  Append all rating curves to a master DataFrame.
+
+    
+    Outputs, if a workspace is specified, are:
+        usgs_rating_curves.csv -- A csv containing USGS rating curve as well
+        as datum adjustment and rating curve expressed as an elevation (NAVD88).
+        ONLY SITES IN CONUS ARE CURRENTLY LISTED IN THIS CSV. To get 
+        additional sites, the Tidal API will need to be reconfigured and tested.
+        
+        log.csv -- A csv containing runtime messages.
+        
+        (if all option passed) usgs_gages.gpkg -- a point layer containing ALL USGS gage sites that meet
+        certain criteria. In the attribute table is a 'curve' column that will indicate if a rating
+        curve is provided in "usgs_rating_curves.csv"
+       
+    Parameters
+    ----------
+    list_of_gage_sites : LIST
+        List of all gage site IDs. If all acceptable sites in CONUS are desired
+        list_of_gage_sites can be passed 'all' and it will use the get_all_active_usgs_sites
+        function to filter out sites that meet certain requirements across CONUS.
+        
+    workspace : STR
+        Directory, if specified, where output csv is saved. OPTIONAL, Default is False.
+    
+    sleep_time: FLOAT
+        Amount of time to rest between API calls. The Tidal API appears to 
+        error out more during business hours. Increasing sleep_time may help.
+        
+
+    Returns
+    -------
+    all_rating_curves : Pandas DataFrame
+        DataFrame containing USGS rating curves adjusted to elevation for 
+        all input sites. Additional metadata also contained in DataFrame
+
+    '''
+    #Define URLs for metadata and rating curve
+    metadata_url = f'{API_BASE_URL}/metadata'
+    rating_curve_url = f'{API_BASE_URL}/rating_curve'
+
+    #If 'all' option passed to list of gages sites, it retrieves all acceptable sites within CONUS.
+    print('getting metadata for all sites')
+    if list_of_gage_sites == ['all']:
+        acceptable_sites_gdf, acceptable_sites_list, metadata_list = get_all_active_usgs_sites()
+    #Otherwise, if a list of sites is passed, retrieve sites from WRDS.
+    else:        
+        #Define arguments to retrieve metadata and then get metadata from WRDS
+        select_by = 'usgs_site_code'
+        selector = list_of_gage_sites        
+        #Since there is a limit to number characters in url, split up selector if too many sites.
+        max_sites = 150
+        if len(selector)>max_sites:
+            chunks = [selector[i:i+max_sites] for i in range(0,len(selector),max_sites)]
+            #Get metadata for each chunk
+            metadata_list = []
+            metadata_df = pd.DataFrame()
+            for chunk in chunks:
+                chunk_list, chunk_df = get_metadata(metadata_url, select_by, chunk, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None )
+                #Append chunk data to metadata_list/df
+                metadata_list.extend(chunk_list)
+                metadata_df = metadata_df.append(chunk_df)
+        else:
+            #If selector has less than max sites, then get metadata.
+            metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None )
+    
+    #Create DataFrame to store all appended rating curves
+    print('processing metadata')
+    all_rating_curves = pd.DataFrame()
+    regular_messages = []    
+    api_failure_messages=[]
+    #For each site in metadata_list
+    for metadata in metadata_list:
+        
+        #Get datum information for site (only need usgs_data)
+        nws, usgs = get_datum(metadata)        
+        
+        #Filter out sites that are not in contiguous US. If this section is removed be sure to test with datum adjustment section (region will need changed)
+        if usgs['state'] in ['Alaska', 'Puerto Rico', 'Virgin Islands', 'Hawaii']:
+            continue        
+        
+        #Get rating curve for site
+        location_ids = usgs['usgs_site_code']
+        curve = get_rating_curve(rating_curve_url, location_ids = [location_ids])
+        #If no rating curve was returned, skip site.
+        if curve.empty:
+            message = f'{location_ids}: has no rating curve'
+            regular_messages.append(message)
+            continue
+
+        #Adjust datum to NAVD88 if needed. If datum unknown, skip site.
+        if usgs['vcs'] == 'NGVD29':
+            #To prevent time-out errors
+            time.sleep(sleep_time)
+            #Get the datum adjustment to convert NGVD to NAVD. Region needs changed if not in CONUS.
+            datum_adj_ft = ngvd_to_navd_ft(datum_info = usgs, region = 'contiguous')
+
+            #If datum API failed, print message and skip site.
+            if datum_adj_ft is None:
+                api_message = f"{location_ids}: datum adjustment failed!!"
+                api_failure_messages.append(api_message)
+                print(api_message)
+                continue
+
+            #If datum adjustment succeeded, calculate datum in NAVD88            
+            navd88_datum = round(usgs['datum'] + datum_adj_ft, 2)
+            message = f'{location_ids}:succesfully converted NGVD29 to NAVD88'
+            regular_messages.append(message)
+
+        elif usgs['vcs'] == 'NAVD88':
+            navd88_datum = usgs['datum']
+            message = f'{location_ids}: already NAVD88'
+            regular_messages.append(message)
+
+        else:
+            message = f"{location_ids}: datum unknown"
+            regular_messages.append(message)
+            continue
+
+        #Populate rating curve with metadata and use navd88 datum to convert stage to elevation.
+        curve['active'] = usgs['active']
+        curve['datum'] = usgs['datum']
+        curve['datum_vcs'] = usgs['vcs']
+        curve['navd88_datum'] = navd88_datum
+        curve['elevation_navd88'] = curve['stage'] + navd88_datum
+        #Append all rating curves to a dataframe
+        all_rating_curves = all_rating_curves.append(curve)        
+
+    #Rename columns and add attribute indicating if rating curve exists
+    acceptable_sites_gdf.rename(columns = {'nwm_feature_id':'feature_id','usgs_site_code':'location_id'}, inplace = True)
+    sites_with_data = pd.DataFrame({'location_id':all_rating_curves['location_id'].unique(),'curve':'yes'})
+    acceptable_sites_gdf = acceptable_sites_gdf.merge(sites_with_data, on = 'location_id', how = 'left')
+    acceptable_sites_gdf.fillna({'curve':'no'},inplace = True)    
+   
+    #If workspace is specified, write data to file.
+    if workspace:
+        #Write rating curve dataframe to file
+        Path(workspace).mkdir(parents = True, exist_ok = True)
+        all_rating_curves.to_csv(Path(workspace) / 'usgs_rating_curves.csv', index = False)
+        #Save out messages to file.
+        first_line = [f'THERE WERE {len(api_failure_messages)} SITES THAT EXPERIENCED DATUM CONVERSION ISSUES']
+        api_failure_messages = first_line + api_failure_messages
+        regular_messages = api_failure_messages + regular_messages                
+        all_messages = pd.DataFrame({'Messages':regular_messages})
+        all_messages.to_csv(Path(workspace) / 'log.csv', index = False)
+        #If 'all' option specified, reproject then write out shapefile of acceptable sites.
+        if list_of_gage_sites == ['all']:            
+            acceptable_sites_gdf = acceptable_sites_gdf.to_crs(PREP_PROJECTION)
+            acceptable_sites_gdf.to_file(Path(workspace) / 'usgs_gages.gpkg', layer = 'usgs_gages', driver = 'GPKG')
+    
+    return all_rating_curves
+
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Retrieve USGS rating curves adjusted to elevation (NAVD88).\nCurrently configured to get rating curves within CONUS.\nRecommend running outside of business hours to reduce API related errors.\nIf error occurs try increasing sleep time (from default of 1).')
+    parser.add_argument('-l', '--list_of_gage_sites',  help = '"all" for all active usgs sites, specify individual sites separated by space, or provide a csv of sites (one per line).', nargs = '+', required = True)
+    parser.add_argument('-w', '--workspace', help = 'Directory where all outputs will be stored.', default = False, required = False)
+    parser.add_argument('-t', '--sleep_timer', help = 'How long to rest between datum API calls', default = 1.0, required = False)
+       
+    #Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    #Check if csv is supplied       
+    if args['list_of_gage_sites'][0].endswith('.csv'):        
+        #Convert csv list to python list
+        with open(args['list_of_gage_sites']) as f:
+            sites = f.read().splitlines()
+        args['list_of_gage_sites'] = sites
+
+    l = args['list_of_gage_sites']
+    w = args['workspace'] 
+    t = float(args['sleep_timer'])           
+    #Run create_flow_forecast_file
+    usgs_rating_to_elev(list_of_gage_sites = l, workspace=w, sleep_time = t)
\ No newline at end of file
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 13534f87b..8ea3d5d59 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -7,6 +7,8 @@
 import pandas as pd
 import geopandas as gpd
 import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from tools_shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
                                     ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
@@ -1045,3 +1047,211 @@ def flow_data(segments, flows, convert_to_cms = True):
     flow_data = pd.DataFrame({'feature_id':segments, 'discharge':flows_cms})
     flow_data = flow_data.astype({'feature_id' : int , 'discharge' : float})
     return flow_data
+#######################################################################
+#Function to get datum information
+#######################################################################
+def get_datum(metadata):
+    '''
+    Given a record from the metadata endpoint, retrieve important information
+    related to the datum and site from both NWS and USGS sources. This information
+    is saved to a dictionary with common keys. USGS has more data available so
+    it has more keys. 
+
+    Parameters
+    ----------
+    metadata : DICT
+        Single record from the get_metadata function. Must iterate through
+        the get_metadata output list. 
+
+    Returns
+    -------
+    nws_datums : DICT
+        Dictionary of NWS data.
+    usgs_datums : DICT
+        Dictionary of USGS Data.
+
+    '''
+    #Get site and datum information from nws sub-dictionary. Use consistent naming between USGS and NWS sources.
+    nws_datums = {}
+    nws_datums['nws_lid'] = metadata['identifiers']['nws_lid']
+    nws_datums['usgs_site_code'] = metadata['identifiers']['usgs_site_code']
+    nws_datums['state'] = metadata['nws_data']['state']
+    nws_datums['datum'] = metadata['nws_data']['zero_datum']
+    nws_datums['vcs'] = metadata['nws_data']['vertical_datum_name']
+    nws_datums['lat'] = metadata['nws_data']['latitude']
+    nws_datums['lon'] = metadata['nws_data']['longitude']
+    nws_datums['crs'] = metadata['nws_data']['horizontal_datum_name']
+    nws_datums['source'] = 'nws_data'
+    
+    #Get site and datum information from usgs_data sub-dictionary. Use consistent naming between USGS and NWS sources.
+    usgs_datums = {}
+    usgs_datums['nws_lid'] = metadata['identifiers']['nws_lid']
+    usgs_datums['usgs_site_code'] = metadata['identifiers']['usgs_site_code']
+    usgs_datums['active'] = metadata['usgs_data']['active']
+    usgs_datums['state'] = metadata['usgs_data']['state']
+    usgs_datums['datum'] = metadata['usgs_data']['altitude']
+    usgs_datums['vcs'] = metadata['usgs_data']['alt_datum_code']
+    usgs_datums['datum_acy'] = metadata['usgs_data']['alt_accuracy_code']
+    usgs_datums['datum_meth'] = metadata['usgs_data']['alt_method_code']
+    usgs_datums['lat'] = metadata['usgs_data']['latitude']
+    usgs_datums['lon'] = metadata['usgs_data']['longitude']
+    usgs_datums['crs'] = metadata['usgs_data']['latlon_datum_name']
+    usgs_datums['source'] = 'usgs_data' 
+    
+    return nws_datums, usgs_datums
+########################################################################
+#Function to convert horizontal datums
+########################################################################
+def convert_latlon_datum(lat,lon,src_crs,dest_crs):
+    '''
+    Converts latitude and longitude datum from a source CRS to a dest CRS 
+    using geopandas and returns the projected latitude and longitude coordinates.
+
+    Parameters
+    ----------
+    lat : FLOAT
+        Input Latitude.
+    lon : FLOAT
+        Input Longitude.
+    src_crs : STR
+        CRS associated with input lat/lon. Geopandas must recognize code.
+    dest_crs : STR
+        Target CRS that lat/lon will be projected to. Geopandas must recognize code.
+
+    Returns
+    -------
+    new_lat : FLOAT
+        Reprojected latitude coordinate in dest_crs.
+    new_lon : FLOAT
+        Reprojected longitude coordinate in dest_crs.
+
+    '''
+    
+    #Create a temporary DataFrame containing the input lat/lon.
+    temp_df = pd.DataFrame({'lat':[lat],'lon':[lon]})
+    #Convert dataframe to a GeoDataFrame using the lat/lon coords. Input CRS is assigned.
+    temp_gdf = gpd.GeoDataFrame(temp_df, geometry=gpd.points_from_xy(temp_df.lon, temp_df.lat), crs =  src_crs)
+    #Reproject GeoDataFrame to destination CRS.
+    reproject = temp_gdf.to_crs(dest_crs)
+    #Get new Lat/Lon coordinates from the geometry data.
+    new_lat,new_lon = [reproject.geometry.y.item(), reproject.geometry.x.item()]
+    return new_lat, new_lon
+#######################################################################
+#Function to get conversion adjustment NGVD to NAVD in FEET
+#######################################################################
+def ngvd_to_navd_ft(datum_info, region = 'contiguous'):
+    '''
+    Given the lat/lon, retrieve the adjustment from NGVD29 to NAVD88 in feet. 
+    Uses NOAA tidal API to get conversion factor. Requires that lat/lon is
+    in NAD27 crs. If input lat/lon are not NAD27 then these coords are 
+    reprojected to NAD27 and the reproject coords are used to get adjustment.
+    There appears to be an issue when region is not in contiguous US.
+
+    Parameters
+    ----------
+    lat : FLOAT
+        Latitude.
+    lon : FLOAT
+        Longitude.
+
+    Returns
+    -------
+    datum_adj_ft : FLOAT
+        Vertical adjustment in feet, from NGVD29 to NAVD88, and rounded to nearest hundredth.
+
+    '''
+    #If crs is not NAD 27, convert crs to NAD27 and get adjusted lat lon
+    if datum_info['crs'] != 'NAD27':
+        lat, lon = convert_latlon_datum(datum_info['lat'],datum_info['lon'],datum_info['crs'],'NAD27')
+    else:
+        #Otherwise assume lat/lon is in NAD27.
+        lat = datum_info['lat']
+        lon = datum_info['lon']
+    
+    #Define url for datum API
+    datum_url = 'https://vdatum.noaa.gov/vdatumweb/api/tidal'     
+    
+    #Define parameters. Hard code most parameters to convert NGVD to NAVD.    
+    params = {}
+    params['lat'] = lat
+    params['lon'] = lon
+    params['region'] = region
+    params['s_h_frame'] = 'NAD27'     #Source CRS
+    params['s_v_frame'] = 'NGVD29'    #Source vertical coord datum
+    params['s_vertical_unit'] = 'm'   #Source vertical units
+    params['src_height'] = 0.0        #Source vertical height
+    params['t_v_frame'] = 'NAVD88'    #Target vertical datum
+    params['tar_vertical_unit'] = 'm' #Target vertical height
+    
+    #Call the API
+    response = requests.get(datum_url, params = params)
+    #If succesful get the navd adjustment
+    if response:
+        results = response.json()
+        #Get adjustment in meters (NGVD29 to NAVD88)
+        adjustment = results['tar_height']
+        #convert meters to feet
+        adjustment_ft = round(float(adjustment) * 3.28084,2)                
+    else:
+        adjustment_ft = None
+    return adjustment_ft       
+#######################################################################
+#Function to download rating curve from API
+#######################################################################
+def get_rating_curve(rating_curve_url, location_ids):
+    '''
+    Given list of location_ids (nws_lids, usgs_site_codes, etc) get the 
+    rating curve from WRDS API and export as a DataFrame.
+
+    Parameters
+    ----------
+    rating_curve_url : STR
+        URL to retrieve rating curve
+    location_ids : LIST
+        List of location ids. Can be nws_lid or usgs_site_codes.
+
+    Returns
+    -------
+    all_curves : pandas DataFrame
+        Rating curves from input list as well as other site information.
+
+    '''
+    #Define DataFrame to contain all returned curves.
+    all_curves = pd.DataFrame()
+    
+    #Define call to retrieve all rating curve information from WRDS.
+    joined_location_ids = '%2C'.join(location_ids)
+    url = f'{rating_curve_url}/{joined_location_ids}'
+    
+    #Call the API 
+    response = requests.get(url)
+
+    #If successful
+    if response.ok:
+
+        #Write return to json and extract the rating curves
+        site_json = response.json()
+        rating_curves_list = site_json['rating_curves']
+
+        #For each curve returned
+        for curve in rating_curves_list:
+            #Check if a curve was populated (e.g wasn't blank)
+            if curve:
+
+                #Write rating curve to pandas dataframe as well as site attributes
+                curve_df = pd.DataFrame(curve['rating_curve'],dtype=float)
+
+                #Add other information such as site, site type, source, units, and timestamp.
+                curve_df['location_id'] = curve['metadata']['location_id']
+                curve_df['location_type'] = curve['metadata']['id_type']
+                curve_df['source'] = curve['metadata']['source']
+                curve_df['flow_units'] = curve['metadata']['flow_unit']
+                curve_df['stage_units'] = curve['metadata']['stage_unit']
+                curve_df['wrds_timestamp'] = response.headers['Date']
+
+                #Append rating curve to DataFrame containing all curves
+                all_curves = all_curves.append(curve_df)
+            else:
+                continue
+
+    return all_curves
\ No newline at end of file

From c9161b81bf124a16f714486cb36a0e8d44067cf0 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Thu, 8 Apr 2021 11:11:56 -0500
Subject: [PATCH 063/359] Implementing a prototype technique to estimate the
 missing bathymetric component in the HAND-derived synthetic rating curves.

Implementing a prototype technique to estimate the missing bathymetric component in the HAND-derived synthetic rating curves. The new Bathymetric Adjusted Rating Curve (BARC) function is built within the fim_run.sh workflow and will ingest bankfull geometry estimates provided by the user to modify the cross section area used in the synthetic rating curve generation.

- add_crosswalk.py outputs the stream order variables to src_full_crosswalked.csv and calls the new bathy_rc_adjust.py if bathy env variable set to True and extent=MS.
- run_by_unit.sh includes a new csv outputs for reviewing BARC calculations.
- params_template.env & params_calibrated.env contain new BARC function input variables and on/off toggle variable.
eval_plots.py now includes additional AHPS eval sites in the list of "bad_sites" (flagged issues with MS flowlines).
- Adds bathy_rc_adjust.py
    - Imports the existing synthetic rating curve table and the bankfull geometry input data (topwidth and cross section area per COMID).
    - Performs new synthetic rating curve calculations with bathymetry estimation modifications.
    - Flags issues with the thalweg-notch artifact.

This resolves #306, resolves #328, resolves #309, resolves #311, and resolves #330.
---
 CHANGELOG.md                 |  58 ++++++++-----
 config/params_calibrated.env |   8 ++
 config/params_template.env   |   8 ++
 src/add_crosswalk.py         |  28 +++++--
 src/bathy_rc_adjust.py       | 156 +++++++++++++++++++++++++++++++++++
 src/output_cleanup.py        |   5 ++
 src/run_by_unit.sh           |   5 +-
 tools/eval_plots.py          |  82 +++++++++---------
 8 files changed, 280 insertions(+), 70 deletions(-)
 create mode 100755 src/bathy_rc_adjust.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba1d25901..b5e9662f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,25 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v3.0.15.0 - 2021-04-08 - [PR #340](https://github.com/NOAA-OWP/cahaba/pull/340)
+
+Implementing a prototype technique to estimate the missing bathymetric component in the HAND-derived synthetic rating curves. The new Bathymetric Adjusted Rating Curve (BARC) function is built within the `fim_run.sh` workflow and will ingest bankfull geometry estimates provided by the user to modify the cross section area used in the synthetic rating curve generation.
+
+### Changes
+ - `add_crosswalk.py` outputs the stream order variables to `src_full_crosswalked.csv` and calls the new `bathy_rc_adjust.py` if bathy env variable set to True and `extent=MS`.
+ - `run_by_unit.sh` includes a new csv outputs for reviewing BARC calculations.
+ - `params_template.env` & `params_calibrated.env` contain new BARC function input variables and on/off toggle variable.
+ - `eval_plots.py` now includes additional AHPS eval sites in the list of "bad_sites" (flagged issues with MS flowlines).
+
+### Additions
+ - `bathy_rc_adjust.py`:
+    - Imports the existing synthetic rating curve table and the bankfull geometry input data (topwidth and cross section area per COMID).
+    - Performs new synthetic rating curve calculations with bathymetry estimation modifications.
+    - Flags issues with the thalweg-notch artifact.
+<br/><br/>
 ## v3.0.14.0 - 2021-04-05 - [PR #338](https://github.com/NOAA-OWP/cahaba/pull/338)
 
 Create tool to retrieve rating curves from USGS sites and convert to elevation (NAVD88). Intended to be used as part of the Sierra Test.
- 
+
 ### Changes
  - Modify `usgs_gage_crosswalk.py` to:
     1) Look for `location_id` instead of `site_no` attribute field in `usgs_gages.gpkg` file.
@@ -19,23 +35,23 @@ Create tool to retrieve rating curves from USGS sites and convert to elevation (
 ## v3.0.13.0 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
 
 Created tool to compare synthetic rating curve with benchmark rating curve (Sierra Test).
- 
+
 ### Changes
  - Update `aggregate_fim_outputs.py` call argument in `fim_run.sh` from 4 jobs to 6 jobs, to optimize API performance.
  - Reroutes median elevation data from `add_crosswalk.py` and `rem.py` to new file (depreciating `hand_ref_elev_table.csv`).
  - Adds new files to `viz_whitelist` in `output_cleanup.py`.
 
 ### Additions
- - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in `run_by_unit.py` with elevation and additional attributes at USGS gages. 
- - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data. 
+ - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in `run_by_unit.py` with elevation and additional attributes at USGS gages.
+ - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data.
 <br/><br/>
 
 ## v3.0.12.1 - 2021-03-31 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
 
 Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
- 
+
 ### Changes
- - Removes file dependencies from spatial option. Does require the WBD layer which should be specified in `.env` file. 
+ - Removes file dependencies from spatial option. Does require the WBD layer which should be specified in `.env` file.
  - Produces outputs in a format consistent with requirements needed for publishing.
  - Preserves leading zeros in huc information for all outputs from `eval_plots.py`.
 
@@ -54,18 +70,18 @@ Add more detail/information to plotting capabilities.
 
 ### Additions
  - Optional argument to create barplots of CSI for each individual site.
- - Create a csv containing the data used to create the scatterplots. 
+ - Create a csv containing the data used to create the scatterplots.
 <br/><br/>
 
 ## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
 
 Improvements to CatFIM service source data generation.
- 
+
 ### Changes
  - Renamed `generate_categorical_fim.py` to `generate_categorical_fim_mapping.py`.
  - Updated the status outputs of the `nws_lid_sites layer` and saved it in the same directory as the `merged catfim_library layer`.
  - Additional stability fixes (such as improved compatability with WRDS updates).
- 
+
 ### Additions
  - Added `generate_categorical_fim.py` to wrap `generate_categorical_fim_flows.py` and `generate_categorical_fim_mapping.py`.
  - Create new `nws_lid_sites` shapefile located in same directory as the `catfim_library` shapefile.
@@ -74,7 +90,7 @@ Improvements to CatFIM service source data generation.
 ## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
 
 Patch to synthesize_test_cases.py.
-  
+
 ### Changes
  - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
 <br/><br/>
@@ -82,14 +98,14 @@ Patch to synthesize_test_cases.py.
 ## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
 
 Preprocessing of flow files for Categorical FIM.
- 
+
 ### Additions
  - Generate Categorical FIM flow files for each category (action, minor, moderate, major).
  - Generate point shapefile of Categorical FIM sites.
  - Generate csv of attribute data in shapefile.
  - Aggregate all shapefiles and csv files into one file in parent directory.
  - Add flood of record category.
- 
+
  ### Changes
  - Stability fixes to `generate_categorical_fim.py`.
 <br/><br/>
@@ -107,7 +123,7 @@ Enhancements to FIM API.
  - Overall better restart and retry handlers for networking problems.
  - Jobs can now be canceled in API interface.
  - Both FR and MS configs can be selected for a single job.
- 
+
 <br/><br/>
 ## v3.0.8.2 - 2021-03-11 - [PR #296](https://github.com/NOAA-OWP/cahaba/pull/296)
 
@@ -117,7 +133,7 @@ Enhancements to post-processing for Viz-related use-cases.
  - Aggregate grids are projected to Web Mercator during `-v` runs in `fim_run.sh`.
  - HUC6 aggregation is parallelized.
  - Aggregate grid blocksize is changed from 256 to 1024 for faster postprocessing.
- 
+
 <br/><br/>
 ## v3.0.8.1 - 2021-03-10 - [PR #302](https://github.com/NOAA-OWP/cahaba/pull/302)
 
@@ -125,7 +141,7 @@ Patched import issue in `tools_shared_functions.py`.
 
 ### Changes
  - Changed `utils.` to `tools_` in `tools_shared_functions.py` after recent structural change to `tools` directory.
- 
+
 <br/><br/>
 ## v3.0.8.0 - 2021-03-09 - [PR #279](https://github.com/NOAA-OWP/cahaba/pull/279)
 
@@ -145,7 +161,7 @@ Renamed benchmark layers in `test_cases` and updated variable names in evaluatio
 ### Changes
  - Updated `run_test_case.py` with new benchmark layer names.
  - Updated `run_test_case_calibration.py` with new benchmark layer names.
- 
+
 <br/><br/>
 ## v3.0.7.0 - 2021-03-01 - [PR #288](https://github.com/NOAA-OWP/cahaba/pull/288)
 
@@ -159,7 +175,7 @@ Restructured the repository. This has no impact on hydrological work done in the
 <br/><br/>
 ## v3.0.6.0 - 2021-02-25 - [PR #276](https://github.com/NOAA-OWP/cahaba/pull/276)
 
-Enhancement that creates metric plots and summary statistics using metrics compiled by `synthesize_test_cases.py`. 
+Enhancement that creates metric plots and summary statistics using metrics compiled by `synthesize_test_cases.py`.
 
 ### Additions
  - Added `eval_plots.py`, which produces:
@@ -281,7 +297,7 @@ New python script "wrappers" for using `inundation.py`.
  - Created `inundation_wrapper_nwm_flows.py` to produce inundation outputs using NWM recurrence flows: 1.5 year, 5 year, 10 year.
  - Created `inundation_wrapper_custom_flow.py` to produce inundation outputs with user-created flow file.
  - Created new `tools` parent directory to store `inundation_wrapper_nwm_flows.py` and  `inundation_wrapper_custom_flow.py`.
- 
+
 <br/><br/>
 ## v3.0.3.1 - 2021-02-04 - [PR #253](https://github.com/NOAA-OWP/cahaba/pull/253)
 
@@ -300,7 +316,7 @@ Post-process to aggregate FIM outputs to HUC6 scale.
 ### Additions
 
  - Viz outputs aggregated to HUC6 scale; saves outputs to `aggregate_fim_outputs` folder.
- 
+
 ### Changes
 
  - `split_flows.py` now splits streams at HUC8 boundaries to ensure consistent catchment boundaries along edges.
@@ -354,7 +370,7 @@ Hotfix for handling nodata value in rasterized levee lines.
 
  - Resolves bug for HUCs where `$ndv > 0` (Great Lakes region).
  - Initialize the `nld_rasterized_elev.tif` using a value of `-9999` instead of `$ndv`.
- 
+
  <br/><br/>
 ## v3.0.0.2 - 2021-01-06 - [PR #200](https://github.com/NOAA-OWP/cahaba/pull/200)
 
@@ -379,7 +395,7 @@ Modifications to build and run Docker image more reliably. Cleanup on some pre-p
 
 ### Notes
  - `aggregate_vector_inputs.py` doesn't work yet. Need to externally download required data to run fim_run.sh
- 
+
  <br/><br/>
 ## v3.0.0.0 - 2020-12-22 - [PR #181](https://github.com/NOAA-OWP/cahaba/pull/181)
 
diff --git a/config/params_calibrated.env b/config/params_calibrated.env
index c5b040215..899c8a4cb 100644
--- a/config/params_calibrated.env
+++ b/config/params_calibrated.env
@@ -19,6 +19,14 @@ export slope_min=0.001
 export min_catchment_area=0.25
 export min_stream_length=0.5
 
+#### bathy SRC estimation parameters ####
+export bathy_src_modification=True
+export surf_area_thalweg_ratio_flag=10
+export thalweg_stg_search_max_limit=3
+export bathy_xs_area_chg_flag=5
+export bankful_xs_area_ratio_flag=10
+export thalweg_hyd_radius_flag=10
+
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
diff --git a/config/params_template.env b/config/params_template.env
index d6c9e2865..21d31d1de 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -19,6 +19,14 @@ export slope_min=0.001
 export min_catchment_area=0.25
 export min_stream_length=0.5
 
+#### bathy SRC estimation parameters ####
+export bathy_src_modification=True
+export surf_area_thalweg_ratio_flag=10
+export thalweg_stg_search_max_limit=3
+export bathy_xs_area_chg_flag=5
+export bankful_xs_area_ratio_flag=10
+export thalweg_hyd_radius_flag=10
+
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 96f2805c0..2e7fbccbd 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -8,17 +8,19 @@
 import json
 import argparse
 import sys
+from bathy_rc_adjust import bathy_rc_lookup
 from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
 
-def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
+def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
 
     input_catchments = gpd.read_file(input_catchments_fileName)
     input_flows = gpd.read_file(input_flows_fileName)
     input_huc = gpd.read_file(input_huc_fileName)
     input_nwmflows = gpd.read_file(input_nwmflows_fileName)
-    min_catchment_area = float(os.environ['min_catchment_area'])
-    min_stream_length = float(os.environ['min_stream_length'])
+    min_catchment_area = float(os.environ['min_catchment_area']) #0.25#
+    min_stream_length = float(os.environ['min_stream_length']) #0.5#
+    bathy_src_calc = os.environ['bathy_src_modification'] == "True" # env variable to toggle on/off the bathy calc and src modifications
 
     if extent == 'FR':
         ## crosswalk using majority catchment method
@@ -167,7 +169,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     input_src_base = pd.read_csv(input_srcbase_fileName, dtype= object)
     if input_src_base.CatchId.dtype != 'int': input_src_base.CatchId = input_src_base.CatchId.astype(int)
 
-    input_src_base = input_src_base.merge(output_flows[['ManningN','HydroID']],left_on='CatchId',right_on='HydroID')
+    input_src_base = input_src_base.merge(output_flows[['ManningN','HydroID','order_']],left_on='CatchId',right_on='HydroID')
 
     input_src_base = input_src_base.rename(columns=lambda x: x.strip(" "))
     input_src_base = input_src_base.apply(pd.to_numeric,**{'errors' : 'coerce'})
@@ -209,6 +211,12 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_crosswalk = output_src[['HydroID','feature_id']]
     output_crosswalk = output_crosswalk.drop_duplicates(ignore_index=True)
 
+    ## bathy estimation integration in synthetic rating curve calculations
+    if (bathy_src_calc == True and extent == 'MS'):
+        output_src = bathy_rc_lookup(output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName)
+    else:
+        print('Note: NOT using bathy estimation approach to modify the SRC...')
+
     # make hydroTable
     output_hydro_table = output_src.loc[:,['HydroID','feature_id','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
@@ -263,6 +271,11 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     parser.add_argument('-d','--input-catchments-fileName', help='DEM derived catchments', required=True)
     parser.add_argument('-a','--input-flows-fileName', help='DEM derived streams', required=True)
     parser.add_argument('-s','--input-srcbase-fileName', help='Base synthetic rating curve table', required=True)
+    parser.add_argument('-u','--input-bathy-fileName', help='Text file with bankfull bathy variables', required=True)
+    parser.add_argument('-v','--output-bathy-fileName', help='Output bathy/bankfull crosswalk and calculated bathy variables', required=True)
+    parser.add_argument('-e','--output-bathy-order-fileName', help='Output bathy/bankfull stream order avg calculated bathy variables', required=True)
+    parser.add_argument('-g','--output-bathy-thalweg-fileName', help='Output bathy thalweg calculation', required=True)
+    parser.add_argument('-i','--output-bathy-xs-lookup-fileName', help='Output bathy XS Area lookup calculation', required=True)
     parser.add_argument('-l','--output-catchments-fileName', help='Subset crosswalked catchments', required=True)
     parser.add_argument('-f','--output-flows-fileName', help='Subset crosswalked streams', required=True)
     parser.add_argument('-r','--output-src-fileName', help='Output crosswalked synthetic rating curve table', required=True)
@@ -283,6 +296,11 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     input_catchments_fileName = args['input_catchments_fileName']
     input_flows_fileName = args['input_flows_fileName']
     input_srcbase_fileName = args['input_srcbase_fileName']
+    input_bathy_fileName = args['input_bathy_fileName']
+    output_bathy_fileName = args['output_bathy_fileName']
+    output_bathy_streamorder_fileName = args['output_bathy_order_fileName']
+    output_bathy_thalweg_fileName = args['output_bathy_thalweg_fileName']
+    output_bathy_xs_lookup_fileName = args['output_bathy_xs_lookup_fileName']
     output_catchments_fileName = args['output_catchments_fileName']
     output_flows_fileName = args['output_flows_fileName']
     output_src_fileName = args['output_src_fileName']
@@ -298,4 +316,4 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     small_segments_filename = args['small_segments_filename']
     calibration_mode = args['calibration_mode']
 
-    add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode)
+    add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode)
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
new file mode 100755
index 000000000..15a0055b5
--- /dev/null
+++ b/src/bathy_rc_adjust.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+
+from os import environ
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+
+sa_ratio_flag             = float(environ['surf_area_thalweg_ratio_flag']) #10x
+thal_stg_limit            = float(environ['thalweg_stg_search_max_limit']) #3m
+bankful_xs_ratio_flag     = float(environ['bankful_xs_area_ratio_flag']) #10x
+bathy_xsarea_flag         = float(environ['bathy_xs_area_chg_flag']) #5x
+thal_hyd_radius_flag      = float(environ['thalweg_hyd_radius_flag']) #10x
+
+def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,):
+    ## Convert input_src_base featureid to integer
+    if input_src_base.feature_id.dtype != 'int': input_src_base.feature_id = input_src_base.feature_id.astype(int)
+
+    ## Read in the bankfull channel geometry text file
+    input_bathy = pd.read_csv(input_bathy_fileName, dtype= {'COMID': int})
+
+    ## Merge input_bathy and modified_src_base df using feature_id/COMID attributes
+    input_bathy = input_bathy.rename(columns={'COMID':'feature_id','BANKFULL_WIDTH':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA':'BANKFULL_XSEC_AREA (m2)'})
+    modified_src_base = input_src_base.merge(input_bathy.loc[:,['feature_id','BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA (m2)']],how='left',on='feature_id')
+
+    ## Check that the merge process returned matching feature_id entries
+    if modified_src_base['BANKFULL_WIDTH (m)'].count() == 0:
+        print('No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC!')
+        return(input_src_base)
+    else:
+        ## Use SurfaceArea variable to identify thalweg-restricted stage values for each hydroid
+        ## Calculate the interrow SurfaceArea ratio n/(n-1)
+        modified_src_base['SA_div'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
+        ## Mask SA_div when Stage = 0 or when the SA_div value (n / n-1) is > threshold value (i.e. 10x)
+        modified_src_base['SA_div'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
+        ## Create new df to filter and groupby HydroID
+        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div']]
+        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['Stage']<thal_stg_limit] # assuming thalweg burn-in is less than 3 meters
+        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div'].notnull()]
+        find_thalweg_notch = find_thalweg_notch.loc[find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()].reset_index(drop=True)
+        ## Assign thalweg_burn_elev variable to the stage value found in previous step
+        find_thalweg_notch['Thalweg_burn_elev'] = find_thalweg_notch['Stage']
+        ## Merge the Thalweg_burn_elev value back into the modified SRC --> this is used to mask the discharge after Manning's equation
+        modified_src_base = modified_src_base.merge(find_thalweg_notch.loc[:,['HydroID','Thalweg_burn_elev']],how='left',on='HydroID')
+
+        ## Calculate bankfull vs top width difference for each feature_id
+        modified_src_base['Top Width Diff (m)'] = (modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']).abs()
+        ## Calculate XS Area field (Channel Volume / Stream Length)
+        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (modified_src_base['LENGTHKM'] * 1000)
+
+        ## Groupby HydroID and find min of Top Width Diff (m)
+        output_bathy = modified_src_base[['feature_id','HydroID','order_','Stage','SurfaceArea (m2)','Thalweg_burn_elev','BANKFULL_WIDTH (m)','TopWidth (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','Top Width Diff (m)']]
+        ## filter out stage = 0 rows in SRC (assuming geom at stage 0 is not a valid channel geom)
+        output_bathy = output_bathy[output_bathy['Stage'] > 0]
+        ## filter SRC rows identified as Thalweg burned
+        output_bathy['Top Width Diff (m)'].mask(output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'],inplace=True)
+        ## ignore hydroid/featureid that did not have a valid Bankfull lookup (areas outside CONUS - i.e. Canada)
+        output_bathy = output_bathy[output_bathy['BANKFULL_XSEC_AREA (m2)'].notnull()]
+        ## ignore SRC entries with 0 surface area --> handles input SRC artifacts/errors in Great Lakes region
+        output_bathy = output_bathy[output_bathy['SurfaceArea (m2)'] > 0]
+        ## find index of minimum top width difference --> this will be used as the SRC "bankfull" row for future calcs
+        output_bathy = output_bathy.loc[output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()].reset_index(drop=True)
+        print('Average: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].mean()))
+        print('Minimum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].min()))
+        print('Maximum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].max()))
+        print('STD: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].std()) +'\n' + '#################')
+
+        ## Calculate XS Area difference between SRC and Bankfull database
+        output_bathy['XS Area Diff (m2)'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)'])
+        output_bathy['XS Bankfull Area Ratio'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']).round(2)
+        ## masking negative XS Area Diff and XS Area = 0
+        output_bathy['XS Bankfull Area Ratio'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        ## masking negative XS Area Diff and XS Area = 0
+        output_bathy['XS Area Diff (m2)'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
+        output_bathy['XS Area Diff (m2)'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
+        output_bathy['XS Bankfull Area Ratio'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        ## Print XS Area Diff statistics
+        print('Average: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].mean()))
+        print('Minimum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].min()))
+        print('Maximum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].max()))
+        print('STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std()))
+
+        ## Bin XS Bankfull Area Ratio by stream order
+        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']]
+        ## mask stage values when XS Bankfull Area Ratio is null (need to filter to calculate the median for valid values below)
+        stream_order_bathy_ratio['Stage'].mask(stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(),inplace=True)
+        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(count=('XS Bankfull Area Ratio','count'),mean_xs_area_ratio=('XS Bankfull Area Ratio','mean'),median_stage_bankfull=('Stage','median'))
+        ## fill XS Bankfull Area Ratio and Stage values if no values were found in the grouby calcs
+        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill()+stream_order_bathy_ratio.bfill())/2
+        ## fill first and last stream order values if needed
+        stream_order_bathy_ratio = stream_order_bathy_ratio.bfill().ffill()
+        ## Get count_total tally of the total number of stream order hydroids in the HUC (not filtering anything out)
+        stream_order_bathy_ratio_count = output_bathy[['order_','Stage']]
+        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage','count'))
+        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(stream_order_bathy_ratio_count,how='left',on='order_')
+        ## Fill any remaining null values: mean_xs_area_ratio --> 1 median_stage_bankfull --> 0
+        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(),1,inplace=True)
+        stream_order_bathy_ratio['median_stage_bankfull'].mask(stream_order_bathy_ratio['median_stage_bankfull'].isnull(),0,inplace=True)
+        print(stream_order_bathy_ratio.head)
+
+        ## Combine SRC df and df of XS Area for each hydroid and matching stage and order from bins above
+        output_bathy = output_bathy.merge(stream_order_bathy_ratio,how='left',on='order_')
+        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio,how='left',on='order_')
+
+        ## Calculate stage vs median_stage_bankfull difference for bankfull lookup
+        modified_src_base['lookup_stage_diff'] = (modified_src_base[['median_stage_bankfull','Thalweg_burn_elev']].max(axis=1) - modified_src_base['Stage']).abs()
+
+        ## If median_stage_bankfull is null then set lookup_stage_diff to 999 at stage 0 (handles errors for channels outside CONUS)
+        modified_src_base['lookup_stage_diff'].mask((modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),999,inplace=True)
+
+        ## Groupby HydroID again and find min of lookup_stage_diff
+        xs_area_hydroid_lookup = modified_src_base[['HydroID','BANKFULL_XSEC_AREA (m2)','XS Area (m2)','Stage','Thalweg_burn_elev','median_stage_bankfull','lookup_stage_diff','mean_xs_area_ratio']]
+        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()].reset_index(drop=True)
+
+        ## Calculate bathy adjusted XS Area ('XS Area (m2)' mutliplied by mean_xs_area_ratio)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']) - xs_area_hydroid_lookup['XS Area (m2)']
+
+        ## Calculate the ratio btw the lookup SRC XS_Area and the Bankfull_XSEC_AREA --> use this as a flag for potentially bad XS data
+        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'])
+        ## Set bath_cal_xs_area to 0 if the bankfull_XS_ratio_flag is > threshold --> 5x (assuming too large of difference to be a reliable bankfull calculation)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask((xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag) | (xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull()),0,inplace=True)
+
+        ## Merge bathy_calc_xs_area to the modified_src_base
+        modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area']],how='left',on='HydroID')
+
+        ## Calculate new bathy adjusted channel geometry variables
+        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'Discharge (m3s-1)_nobathy'})
+        modified_src_base['XS Area (m2)_bathy_adj'] = modified_src_base['XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
+        modified_src_base['Volume (m3)_bathy_adj'] = modified_src_base['XS Area (m2)_bathy_adj'] * modified_src_base['LENGTHKM'] * 1000
+        modified_src_base['WetArea (m2)_bathy_adj'] = modified_src_base['Volume (m3)_bathy_adj']/modified_src_base['LENGTHKM']/1000
+        modified_src_base['HydraulicRadius (m)_bathy_adj'] = modified_src_base['WetArea (m2)_bathy_adj']/modified_src_base['WettedPerimeter (m)']
+        modified_src_base['HydraulicRadius (m)_bathy_adj'].fillna(0, inplace=True)
+        ## mask out negative top width differences (avoid thalweg burn notch)
+        modified_src_base['HydraulicRadius (m)_bathy_adj'].mask((modified_src_base['HydraulicRadius (m)_bathy_adj']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
+
+        ## Calculate Q using Manning's equation
+        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)_bathy_adj']* \
+        pow(modified_src_base['HydraulicRadius (m)_bathy_adj'],2.0/3)* \
+        pow(modified_src_base['SLOPE'],0.5)/modified_src_base['ManningN']
+        ## mask discharge values for stage = 0 rows in SRC (replace with 0) --> do we need SRC to start at 0??
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0,0,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'],0,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'],-999,inplace=True)
+
+        ## Organize bathy calc output variables for csv
+        output_bathy = output_bathy[['HydroID','order_','Stage','SurfaceArea (m2)','TopWidth (m)','BANKFULL_WIDTH (m)','Top Width Diff (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','XS Area Diff (m2)','XS Bankfull Area Ratio','count','median_stage_bankfull','mean_xs_area_ratio']]
+
+        ## Export bathy/bankful calculation tables for easy viewing
+        output_bathy.to_csv(output_bathy_fileName,index=False)
+        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName,index=True)
+        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName,index=True)
+        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName,index=True)
+
+        print('Completed Bathy Calculations...')
+        return(modified_src_base)
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index e74e26b17..def5a286c 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -32,6 +32,11 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         'hydroTable.csv',
         'src.json',
         'small_segments.csv',
+        'bathy_crosswalk_calcs.csv',
+        'bathy_stream_order_calcs.csv',
+        'bathy_thalweg_flag.csv',
+        'bathy_xs_area_hydroid_lookup.csv'
+        'src_full_crosswalked.csv'
         'usgs_elev_table.csv',
         'hand_ref_elev_table.csv'
     ]
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 1242768dc..de460fb1f 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -425,14 +425,13 @@ $taudemDir/catchhydrogeo -hand $outputHucDataDir/rem_zeroed_masked.tif -catch $o
 Tcount
 
 ## FINALIZE CATCHMENTS AND MODEL STREAMS ##
-echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv
+echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent -k $outputHucDataDir/small_segments.csv
+$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $inputDataDir/bathymetry/BANKFULL_CONUS.txt -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
-
 ## USGS CROSSWALK ##
 echo -e $startDiv"USGS Crosswalk $hucNumber"$stopDiv
 date -u
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 0af2ae9ad..0327319cb 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -23,9 +23,9 @@
 #########################################################################
 #Create boxplot
 #########################################################################
-def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):      
+def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):
     '''
-    Create boxplots. 
+    Create boxplots.
 
     Parameters
     ----------
@@ -44,10 +44,10 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
     fim_configuration: STR
         Configuration of FIM (FR or MS or Composite).
     simplify_legend : BOOL, optional
-        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3.
         The default is False.
     dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
+        If STR provide the full path to the figure to be saved. If False
         no plot is saved to disk. The default is False.
 
     Returns
@@ -73,17 +73,17 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
     #Set sizes of ticks and legend.
     axes.tick_params(labelsize = 'xx-large')
     axes.legend(markerscale = 2, fontsize =20, loc = 'lower left')
-    
+
     #If simple legend desired
     if simplify_legend:
-        #trim labels to FIM 1, FIM 2, and the FIM 3 version    
+        #trim labels to FIM 1, FIM 2, and the FIM 3 version
         handles, org_labels = axes.get_legend_handles_labels()
         label_dict = {}
         for label in org_labels:
             if 'fim_1' in label:
                 label_dict[label] = 'FIM 1'
             elif 'fim_2' in label:
-                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()                  
+                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
             elif 'fim_3' in label:
                 label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
                 if label.endswith('_c'):
@@ -96,7 +96,7 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
         if y_field == 'FAR':
             legend_location = 'upper right'
         else:
-            legend_location = 'lower left' 
+            legend_location = 'lower left'
         #rename legend labels to the simplified labels.
         axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
     #Print textbox if supplied
@@ -114,9 +114,9 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
 #########################################################################
 #Create scatter plot
 #########################################################################
-def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):      
+def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):
     '''
-    Create boxplots. 
+    Create boxplots.
 
     Parameters
     ----------
@@ -127,11 +127,11 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
     y_field : STR
         Field to use for the y-axis (Assumes FIM 3)
     title_text : STR
-        Text for plot title.    
+        Text for plot title.
     stats_text : STR or BOOL
         Text for stats to place on chart. Default is false (no stats printed)
     dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
+        If STR provide the full path to the figure to be saved. If False
         no plot is saved to disk. The default is False.
 
     Returnsy
@@ -143,15 +143,15 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
 
     #initialize plot
     fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    
+
     #Use seaborn to plot the boxplot
     axes=sns.scatterplot(data=dataframe, x=x_field, y=y_field, color = 'black', s = 150)
-    
+
     #Set xticks and yticks and background horizontal line.
     axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
     axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
-    axes.grid(b=True, which='major', axis='both')    
-    
+    axes.grid(b=True, which='major', axis='both')
+
     #Set sizes of ticks and legend.
     axes.tick_params(labelsize = 'xx-large')
 
@@ -166,7 +166,7 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
 
     #set title of plot
     axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
-   
+
     if annotate:
         #Set text for labels
         box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
@@ -188,9 +188,9 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
 #########################################################################
 #Create barplot
 #########################################################################
-def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):      
+def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):
     '''
-    Create barplots. 
+    Create barplots.
 
     Parameters
     ----------
@@ -209,13 +209,13 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
     fim_configuration: STR
         Configuration of FIM (FR or MS or Composite).
     simplify_legend : BOOL, optional
-        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3.
         Default is False.
     display_values : BOOL, optional
-        If True, Y values will be displayed above bars. 
+        If True, Y values will be displayed above bars.
         Default is False.
     dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
+        If STR provide the full path to the figure to be saved. If False
         no plot is saved to disk. Default is False.
 
     Returns
@@ -243,7 +243,7 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
     axes.legend(markerscale = 2, fontsize =20, loc = 'upper right')
     #If simple legend desired
     if simplify_legend:
-        #trim labels to FIM 1, FIM 2, FIM 3    
+        #trim labels to FIM 1, FIM 2, FIM 3
         handles, org_labels = axes.get_legend_handles_labels()
         label_dict = {}
         for label in org_labels:
@@ -274,8 +274,8 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
             axes.text(patch.get_x()+patch.get_width()/2.,
                     patch.get_height(),
                     '{:1.3f}'.format(value),
-                    ha="center", fontsize=18)  
-    
+                    ha="center", fontsize=18)
+
     #If figure to be saved to disk, then do so, otherwise return fig
     if dest_file:
         fig.savefig(dest_file)
@@ -288,12 +288,12 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
 def filter_dataframe(dataframe, unique_field):
     '''
 
-    This script will filter out the sites (or hucs) which are not consistently 
-    found for all versions for a given magnitude. For example, an AHPS 
-    lid site must have output for all 3 versions (fim1, fim2, fim3) for 
-    a given magnitude (eg action) otherwise that lid is filtered out. 
-    Likewise for a BLE a huc must have output for all 3 versions 
-    (fim1, fim2, fim3) for a given magnitude (eg 100yr) otherwise it is 
+    This script will filter out the sites (or hucs) which are not consistently
+    found for all versions for a given magnitude. For example, an AHPS
+    lid site must have output for all 3 versions (fim1, fim2, fim3) for
+    a given magnitude (eg action) otherwise that lid is filtered out.
+    Likewise for a BLE a huc must have output for all 3 versions
+    (fim1, fim2, fim3) for a given magnitude (eg 100yr) otherwise it is
     filtered out.
 
     Parameters
@@ -307,12 +307,12 @@ def filter_dataframe(dataframe, unique_field):
     Returns
     -------
     final_filtered_dataframe : Pandas Dataframe
-        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude. 
+        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude.
     unique_sites: DICT
         The sites that were included in the dataframe for each magnitude.
 
     '''
-    
+
     #Get lists of sites for each magnitude/version
     unique_sites = dataframe.groupby(['magnitude','version'])[unique_field].agg('unique')
     #Get unique magnitudes
@@ -331,8 +331,8 @@ def filter_dataframe(dataframe, unique_field):
         #Query filtered dataframe and only include data associated with the common sites for that magnitude
         filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
         #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
-        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)            
-    
+        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)
+
     return final_filtered_dataframe, all_unique_sites
 ##############################################################################
 ##############################################################################
@@ -429,7 +429,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         extent_config set to MS. This allows for FIM 1 to be included
         in MS plots/stats (helpful for nws/usgs ahps comparisons).
     site_barplots: BOOL
-        Default is false. If True then barplots for each individual site are 
+        Default is false. If True then barplots for each individual site are
         created. An 'individual' directory with subdirectories of each site
         are created and the plot is located in each site subdirectory.
 
@@ -470,21 +470,21 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     versions for a given magnitude. The final filtered dataset is written
     to a dictionary with the key (benchmark source, extent config)
     and values (filtered dataframe, common sites). '''
-    
+
     all_datasets = {}
     for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:
 
         '''If source is usgs/nws define the base resolution and query
         (use alternate query if passed). Append filtered datasets to
         all_datasets dictionary.'''
-        
+
         if benchmark_source in ['usgs','nws']:
 
             # Set the base processing unit for the ahps runs.
             base_resolution = 'nws_lid'
 
             #Default query (used for APG) it could be that bad_sites should be modified. If so pass an alternate query using the "alternate_ahps_query"
-            bad_sites = ['grfi2','ksdm7','hohn4','rwdn4']
+            bad_sites = ['grfi2','ksdm7','hohn4','rwdn4','efdn7','kilo1','chin7','segt2','eagi1','levk1','trbf1']
             query = "not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites"
 
             # If alternate ahps evaluation query argument is passed, use that.
@@ -540,7 +540,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         # Get all versions in dataset
         all_versions = list(dataset.version.unique())
         version_order = []
-        
+
         # If versions are not specified then use all available versions and assign to versions_list
         if not versions:
             versions_list = all_versions
@@ -685,7 +685,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     parser.add_argument('-sp', '--spatial', help = 'If enabled, creates spatial layers with metrics populated in attribute table.', action = 'store_true', required = False)
     parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
     parser.add_argument('-i', '--site_plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
-    
+
     # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
 

From 5a8ebcd1e8c054c0a6e82a724142a78a469bffe2 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Wed, 14 Apr 2021 08:47:58 -0500
Subject: [PATCH 064/359] Add mainstems attribute for sierra test

Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field.

- Modifies usgs_gage_crosswalk.py to filter out gages from the usgs_gages.gpkg layer such that for a "MS" run, only consider gages that contain rating curve information (via curve attribute) and are also mainstems gages (via mainstems attribute).
- Modifies usgs_gage_crosswalk.py to filter out gages from the usgs_gages.gpkg layer such that for a "FR" run, only consider gages that contain rating curve information (via curve attribute) and are not mainstems gages (via mainstems attribute).
- Modifies how mainstems segments are determined by using the nwm_flows_ms.gpkg as a lookup to determine if the NWM segment specified by WRDS for a gage site is a mainstems gage.
- Adds a mainstem attribute field to usgs_gages.gpkg that indicates whether a gage is located on a mainstems river.
- Adds NWM_FLOWS_MS variable to the .env and .env.template files.
- Adds the extent argument specified by user when running fim_run.sh to usgs_gage_crosswalk.py.
---
 CHANGELOG.md                          | 18 ++++++++++++++++++
 src/run_by_unit.sh                    |  2 +-
 src/usgs_gage_crosswalk.py            | 15 ++++++++++-----
 tools/.env.template                   |  1 +
 tools/rating_curve_get_usgs_curves.py | 13 ++++++++++++-
 5 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b5e9662f1..baa3ba19f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+
+## v3.0.15.1 - 2021-04-13 - [PR #355](https://github.com/NOAA-OWP/cahaba/pull/355)
+
+Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field. 
+## Changes
+
+- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "MS" run, only consider gages that contain rating curve information (via `curve` attribute) and are also mainstems gages (via `mainstems` attribute). 
+- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "FR" run, only consider gages that contain rating curve information (via `curve` attribute) and are not mainstems gages (via `mainstems` attribute). 
+- Modifies how mainstems segments are determined by using the `nwm_flows_ms.gpkg` as a lookup to determine if the NWM segment specified by WRDS for a gage site is a mainstems gage. 
+
+## Additions
+
+- Adds a `mainstem` attribute field to `usgs_gages.gpkg` that indicates whether a gage is located on a mainstems river. 
+- Adds `NWM_FLOWS_MS` variable to the `.env` and `.env.template` files.
+- Adds the `extent` argument specified by user when running `fim_run.sh` to `usgs_gage_crosswalk.py`.
+
+<br/><br/>
+
 ## v3.0.15.0 - 2021-04-08 - [PR #340](https://github.com/NOAA-OWP/cahaba/pull/340)
 
 Implementing a prototype technique to estimate the missing bathymetric component in the HAND-derived synthetic rating curves. The new Bathymetric Adjusted Rating Curve (BARC) function is built within the `fim_run.sh` workflow and will ingest bankfull geometry estimates provided by the user to modify the cross section area used in the synthetic rating curve generation.
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index de460fb1f..841f07522 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -436,7 +436,7 @@ Tcount
 echo -e $startDiv"USGS Crosswalk $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/usgs_gage_crosswalk.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -dem $outputHucDataDir/dem_meters.tif -flows $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -cat $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -wbd $outputHucDataDir/wbd_buffered.gpkg -dem_adj $dem_thalwegCond -outtable $outputHucDataDir/usgs_elev_table.csv
+$srcDir/usgs_gage_crosswalk.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -dem $outputHucDataDir/dem_meters.tif -flows $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -cat $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -wbd $outputHucDataDir/wbd_buffered.gpkg -dem_adj $dem_thalwegCond -outtable $outputHucDataDir/usgs_elev_table.csv -e $extent
 Tcount
 
 ## CLEANUP OUTPUTS ##
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index fb4b9533d..ee61c3d11 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -32,7 +32,7 @@
 '''
 
 
-def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename,dem_adj_filename,output_table_filename):
+def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename,dem_adj_filename,output_table_filename,extent):
 
     wbd_buffer = gpd.read_file(wbd_buffer_filename)
     usgs_gages = gpd.read_file(usgs_gages_filename, mask=wbd_buffer)
@@ -41,8 +41,12 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     input_catchment = gpd.read_file(input_catchment_filename)
     dem_adj = rasterio.open(dem_adj_filename,'r')
 
-    #Query out usgs_gages that don't have rating curve data
-    usgs_gages = usgs_gages.query('curve == "yes"')
+    #MS extent use gages that are mainstem
+    if extent == "MS":
+        usgs_gages = usgs_gages.query('curve == "yes" & mainstem == "yes"')
+    #FR extent use gages that are not mainstem
+    if extent == "FR":
+        usgs_gages = usgs_gages.query('curve == "yes" & mainstem == "no"')
 
     if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
 
@@ -113,7 +117,7 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     parser.add_argument('-wbd','--wbd-buffer-filename', help='WBD buffer', required=True)
     parser.add_argument('-dem_adj','--dem-adj-filename', help='Thalweg adjusted DEM', required=True)
     parser.add_argument('-outtable','--output-table-filename', help='Table to append data', required=True)
-
+    parser.add_argument('-e', '--extent', help="extent configuration entered by user when running fim_run.sh", required = True)
     args = vars(parser.parse_args())
 
     usgs_gages_filename = args['usgs_gages_filename']
@@ -123,5 +127,6 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     wbd_buffer_filename = args['wbd_buffer_filename']
     dem_adj_filename = args['dem_adj_filename']
     output_table_filename = args['output_table_filename']
+    extent = args['extent']
 
-    crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename, dem_adj_filename,output_table_filename)
+    crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename, dem_adj_filename,output_table_filename, extent)
diff --git a/tools/.env.template b/tools/.env.template
index 462d9c556..ed83c8113 100644
--- a/tools/.env.template
+++ b/tools/.env.template
@@ -1,3 +1,4 @@
 API_BASE_URL=
 EVALUATED_SITES_CSV=
 WBD_LAYER=
+NWM_FLOWS_MS=
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
index eb43bab3e..6676c27b3 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -23,6 +23,8 @@
 load_dotenv()
 API_BASE_URL = os.getenv("API_BASE_URL")
 WBD_LAYER = os.getenv("WBD_LAYER")
+EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+NWM_FLOWS_MS = os.getenv("NWM_FLOWS_MS")
 
 def get_all_active_usgs_sites():
     '''
@@ -245,7 +247,16 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
     sites_with_data = pd.DataFrame({'location_id':all_rating_curves['location_id'].unique(),'curve':'yes'})
     acceptable_sites_gdf = acceptable_sites_gdf.merge(sites_with_data, on = 'location_id', how = 'left')
     acceptable_sites_gdf.fillna({'curve':'no'},inplace = True)    
-   
+    #Add mainstems attribute to acceptable sites
+    print('Attributing mainstems sites')
+    #Import mainstems segments used in run_by_unit.sh
+    ms_df = gpd.read_file(NWM_FLOWS_MS)
+    ms_segs = ms_df.ID.astype(str).to_list()
+    #Populate mainstems attribute field
+    acceptable_sites_gdf['mainstem'] = 'no'
+    acceptable_sites_gdf.loc[acceptable_sites_gdf.eval('feature_id in @ms_segs'),'mainstem'] = 'yes' 
+    
+    
     #If workspace is specified, write data to file.
     if workspace:
         #Write rating curve dataframe to file

From 1a15cd822d249744c74f8a2c43ac7ffb588dc346 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Fri, 16 Apr 2021 14:09:21 -0500
Subject: [PATCH 065/359] Fixes production list in output_cleanup.py.

Fixes production list in output_cleanup.py.
---
 CHANGELOG.md          | 7 +++++++
 src/output_cleanup.py | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index baa3ba19f..c883cc7ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v3.0.15.2 - 2021-04-16 - [PR #359](https://github.com/NOAA-OWP/cahaba/pull/359)
 
+Hotfix to preserve desired files when production flag used in `fim_run.sh`. 
+## Changes
+
+- Fixed production whitelisted files.
+
+<br/><br/>
 ## v3.0.15.1 - 2021-04-13 - [PR #355](https://github.com/NOAA-OWP/cahaba/pull/355)
 
 Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field. 
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index def5a286c..879103ad6 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -35,8 +35,8 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         'bathy_crosswalk_calcs.csv',
         'bathy_stream_order_calcs.csv',
         'bathy_thalweg_flag.csv',
-        'bathy_xs_area_hydroid_lookup.csv'
-        'src_full_crosswalked.csv'
+        'bathy_xs_area_hydroid_lookup.csv',
+        'src_full_crosswalked.csv',
         'usgs_elev_table.csv',
         'hand_ref_elev_table.csv'
     ]

From a304cf386b6c196d58d6bf7d794672ccea2a500d Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Mon, 19 Apr 2021 12:14:47 -0500
Subject: [PATCH 066/359] Preprocess NHDPlus HR rasters for consistent
 projections, nodata values, and convert from cm to meters.

Preprocess NHDPlus HR rasters for consistent projections, nodata values, and convert from cm to meters.

- preprocess_rasters.py reprojects raster, converts to meters, and updates nodata value to -9999.
- Cleaned up log messages from bathy_rc_adjust.py and usgs_gage_crosswalk.py.
- Outputs paths updated in generate_categorical_fim_mapping.py and generate_categorical_fim.py.
- update_raster_profile cleans up raster crs, blocksize, nodata values, and converts elevation grids from cm to meters.
- reproject_dem.py imports gdal to reproject elevation rasters because an error was occurring when using rasterio.
- burn_in_levees.py replaces the gdal_calc.py command to resolve inconsistent outputs with burned in levee values.

This resolves #300.
---
 CHANGELOG.md                              |  15 +
 README.md                                 |   5 +-
 src/acquire_and_preprocess_inputs.py      |   9 +-
 src/bathy_rc_adjust.py                    |   4 +-
 src/burn_in_levees.py                     |  42 +++
 src/preprocess_rasters.py                 |  56 +++
 src/run_by_unit.sh                        |  20 +-
 src/usgs_gage_crosswalk.py                |   2 +-
 src/utils/reproject_dem.py                |  50 +++
 src/utils/shared_functions.py             | 124 +++++++
 src/utils/shared_variables.py             |   1 +
 tools/comparing_src.py                    | 393 ----------------------
 tools/generate_categorical_fim.py         |  84 +++--
 tools/generate_categorical_fim_mapping.py |  20 +-
 14 files changed, 353 insertions(+), 472 deletions(-)
 create mode 100755 src/burn_in_levees.py
 create mode 100755 src/preprocess_rasters.py
 create mode 100755 src/utils/reproject_dem.py
 delete mode 100755 tools/comparing_src.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c883cc7ba..2d9969a11 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v3.0.15.3 - 2021-04-19 - [PR #358](https://github.com/NOAA-OWP/cahaba/pull/358)
+
+Preprocess NHDPlus HR rasters for consistent projections, nodata values, and convert from cm to meters.
+
+## Additions
+- `preprocess_rasters.py` reprojects raster, converts to meters, and updates nodata value to -9999.
+- Cleaned up log messages from `bathy_rc_adjust.py` and `usgs_gage_crosswalk.py`.
+- Outputs paths updated in `generate_categorical_fim_mapping.py` and `generate_categorical_fim.py`.
+- `update_raster_profile` cleans up raster crs, blocksize, nodata values, and converts elevation grids from cm to meters.
+- `reproject_dem.py` imports gdal to reproject elevation rasters because an error was occurring when using rasterio.
+
+## Changes
+- `burn_in_levees.py` replaces the `gdal_calc.py` command to resolve inconsistent outputs with burned in levee values.
+
+<br/><br/>
 ## v3.0.15.2 - 2021-04-16 - [PR #359](https://github.com/NOAA-OWP/cahaba/pull/359)
 
 Hotfix to preserve desired files when production flag used in `fim_run.sh`. 
diff --git a/README.md b/README.md
index c067ea29d..8bbeaeb96 100644
--- a/README.md
+++ b/README.md
@@ -38,6 +38,7 @@ The following input data sources should be downloaded and preprocessed prior to
 
 ### NHDPlus HR datasets
 - `acquire_and_preprocess_inputs.py`
+- `preprocess_rasters.py`
 - `aggregate_nhd_hr_streams.py`
 
 **Please note:** For the following two datasets, please contact Mark Glaudemans (mark.glaudemans@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
@@ -68,11 +69,11 @@ docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <i
 - Please run `/foss_fim/src/acquire_and_preprocess_inputs.py --help` for more information.
 - See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
 
-### Aggregate NHDHR Streams and Create NWM Headwater Points 
+### Aggregate NHDHR Streams and Create NWM Headwater Points
 ```
 /foss_fim/src/aggregate_vector_inputs.py
 ```
-### Produce Hydrofabric 
+### Produce Hydrofabric
 ```
 fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
 ```
diff --git a/src/acquire_and_preprocess_inputs.py b/src/acquire_and_preprocess_inputs.py
index 30ce7974a..852942ff3 100755
--- a/src/acquire_and_preprocess_inputs.py
+++ b/src/acquire_and_preprocess_inputs.py
@@ -4,6 +4,7 @@
 import argparse
 import csv
 import sys
+sys.path.append('/foss_fim/src')
 import shutil
 from multiprocessing import Pool
 import geopandas as gpd
@@ -23,7 +24,9 @@
                                     OVERWRITE_NHD,
                                     OVERWRITE_ALL)
 
-from utils.shared_functions import pull_file, run_system_command, subset_wbd_gpkg, delete_file, getDriver
+from utils.shared_functions import (pull_file, run_system_command,
+                                    subset_wbd_gpkg, delete_file,
+                                    getDriver)
 
 NHDPLUS_VECTORS_DIRNAME = 'nhdplus_vectors'
 NHDPLUS_RASTERS_DIRNAME = 'nhdplus_rasters'
@@ -180,9 +183,7 @@ def pull_and_prepare_nhd_data(args):
     if not os.path.exists(elev_cm_tif) or overwrite_nhd:
         pull_file(nhd_raster_download_url, nhd_raster_extraction_path)
         os.system("7za e {nhd_raster_extraction_path} -o{nhd_raster_parent_dir} elev_cm.tif -r ".format(nhd_raster_extraction_path=nhd_raster_extraction_path, nhd_raster_parent_dir=nhd_raster_parent_dir))
-        # Change projection for elev_cm.tif.
-        #print("Projecting elev_cm...")
-        #run_system_command(['gdal_edit.py -a_srs "{projection}" {elev_cm_tif}'.format(projection=PREP_PROJECTION, elev_cm_tif=elev_cm_tif)])
+
         file_list = os.listdir(nhd_raster_parent_dir)
         for f in file_list:
             full_path = os.path.join(nhd_raster_parent_dir, f)
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
index 15a0055b5..41683c905 100755
--- a/src/bathy_rc_adjust.py
+++ b/src/bathy_rc_adjust.py
@@ -82,7 +82,7 @@ def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,ou
         print('STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std()))
 
         ## Bin XS Bankfull Area Ratio by stream order
-        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']]
+        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']].copy()
         ## mask stage values when XS Bankfull Area Ratio is null (need to filter to calculate the median for valid values below)
         stream_order_bathy_ratio['Stage'].mask(stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(),inplace=True)
         stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(count=('XS Bankfull Area Ratio','count'),mean_xs_area_ratio=('XS Bankfull Area Ratio','mean'),median_stage_bankfull=('Stage','median'))
@@ -91,13 +91,11 @@ def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,ou
         ## fill first and last stream order values if needed
         stream_order_bathy_ratio = stream_order_bathy_ratio.bfill().ffill()
         ## Get count_total tally of the total number of stream order hydroids in the HUC (not filtering anything out)
-        stream_order_bathy_ratio_count = output_bathy[['order_','Stage']]
         stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage','count'))
         stream_order_bathy_ratio = stream_order_bathy_ratio.merge(stream_order_bathy_ratio_count,how='left',on='order_')
         ## Fill any remaining null values: mean_xs_area_ratio --> 1 median_stage_bankfull --> 0
         stream_order_bathy_ratio['mean_xs_area_ratio'].mask(stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(),1,inplace=True)
         stream_order_bathy_ratio['median_stage_bankfull'].mask(stream_order_bathy_ratio['median_stage_bankfull'].isnull(),0,inplace=True)
-        print(stream_order_bathy_ratio.head)
 
         ## Combine SRC df and df of XS Area for each hydroid and matching stage and order from bins above
         output_bathy = output_bathy.merge(stream_order_bathy_ratio,how='left',on='order_')
diff --git a/src/burn_in_levees.py b/src/burn_in_levees.py
new file mode 100755
index 000000000..89d10a0b0
--- /dev/null
+++ b/src/burn_in_levees.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import rasterio
+import numpy as np
+import argparse
+
+
+def burn_in_levees(dem_filename,nld_filename,out_dem_filename):
+
+    dem = rasterio.open(dem_filename)
+    nld = rasterio.open(nld_filename)
+
+    dem_data = dem.read(1)
+    nld_data = nld.read(1)
+
+    no_data = nld.nodata
+
+    nld_m = np.where(nld_data == int(no_data), -9999.0, (nld_data*0.3048).astype(rasterio.float32))
+
+    dem_profile = dem.profile.copy()
+
+    dem_nld_burn = np.maximum(dem_data, nld_m)
+
+
+    with rasterio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
+        dest.write(dem_nld_burn, indexes = 1)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Burn in NLD elevations')
+    parser.add_argument('-dem','--dem-filename', help='DEM filename', required=True,type=str)
+    parser.add_argument('-nld','--nld-filename', help='NLD filename', required=True,type=str)
+    parser.add_argument('-out','--out-dem-filename', help='out DEM filename', required=True,type=str)
+
+    args = vars(parser.parse_args())
+
+    dem_filename = args['dem_filename']
+    nld_filename = args['nld_filename']
+    out_dem_filename = args['out_dem_filename']
+
+    burn_in_levees(dem_filename,nld_filename,out_dem_filename)
diff --git a/src/preprocess_rasters.py b/src/preprocess_rasters.py
new file mode 100755
index 000000000..7026f4cb2
--- /dev/null
+++ b/src/preprocess_rasters.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import os
+from osgeo import gdal
+import sys
+sys.path.append('/foss_fim/src')
+from multiprocessing import Pool
+import argparse
+from utils.reproject_dem import reproject_dem
+from utils.shared_functions import update_raster_profile
+from utils.shared_variables import PREP_PROJECTION, PREP_PROJECTION_CM
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Reproject Elevation rasters and update profile')
+    parser.add_argument('-dem_dir','--dem-dir', help='DEM filename', required=True,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
+    parser.add_argument('-nodata','--nodata-val', help='DEM nodata value', required=False,type=float,default=-9999.0)
+    parser.add_argument('-block','--blocksize', help='DEM blocksize', required=False,type=int,default=512)
+    parser.add_argument('-keep','--keep-intermediate', help='keep intermediate files', required=False,type=bool,default=True)
+
+    args = vars(parser.parse_args())
+
+    dem_dir            = args['dem_dir']
+    number_of_jobs     = args['number_of_jobs']
+    nodata_val         = args['nodata_val']
+    blocksize          = args['blocksize']
+    keep_intermediate  = args['keep_intermediate']
+
+    reproject_procs_list = []
+
+    for huc in os.listdir(dem_dir):
+        raster_dir = os.path.join(dem_dir,huc)
+        elev_cm = os.path.join(raster_dir, 'elev_cm.tif')
+        elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
+        reproject_procs_list.append([raster_dir, elev_cm, elev_cm_proj, PREP_PROJECTION_CM])
+
+    # Multiprocess reprojection
+    with Pool(processes=number_of_jobs) as pool:
+        pool.map(reproject_dem, reproject_procs_list)
+
+    profile_procs_list = []
+
+    for huc in os.listdir(dem_dir):
+        elev_m_tif = os.path.join(dem_dir,huc, 'elev_m.tif')
+        if not os.path.exists(elev_m_tif):
+            raster_dir = os.path.join(dem_dir,huc)
+            elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
+            elev_m = os.path.join(raster_dir, 'elev_m.tif')
+            profile_procs_list.append([elev_cm_proj, elev_m,PREP_PROJECTION,nodata_val,blocksize,keep_intermediate])
+
+    # Multiprocess update profile
+    with Pool(processes=2) as pool:
+        # TODO read in windows becasue gdal rasters are massive
+        pool.map(update_raster_profile, profile_procs_list)
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 841f07522..c8f490696 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -32,7 +32,7 @@ hucUnitLength=${#hucNumber}
 huc4Identifier=${hucNumber:0:4}
 huc2Identifier=${hucNumber:0:2}
 input_NHD_WBHD_layer=WBDHU$hucUnitLength
-input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_cm.tif
+input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m.tif
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
 
 # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
@@ -99,30 +99,22 @@ Tcount
 echo -e $startDiv"Clip DEM $hucNumber"$stopDiv
 date -u
 Tstart
-[ ! -f $outputHucDataDir/dem.tif ] && \
-gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Int32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" $input_DEM $outputHucDataDir/dem.tif
+[ ! -f $outputHucDataDir/dem_meters.tif ] && \
+gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" $input_DEM $outputHucDataDir/dem_meters.tif
 Tcount
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber"$stopDiv
 date -u
 Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/dem.tif)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/dem_meters.tif)
 
 ## RASTERIZE NLD MULTILINES ##
 echo -e $startDiv"Rasterize all NLD multilines using zelev vertices"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/nld_rasterized_elev.tif ] && [ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
-gdal_rasterize -l nld_subset_levees -3d -at -init -9999 -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputHucDataDir/nld_rasterized_elev.tif
-Tcount
-
-## CONVERT TO METERS ##
-echo -e $startDiv"Convert DEM to Meters $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/dem_meters.tif ] && \
-gdal_calc.py --quiet --type=Float32 --co "BLOCKXSIZE=512" --co "BLOCKYSIZE=512" --co "TILED=YES" --co "COMPRESS=LZW" --co "BIGTIFF=YES" -A $outputHucDataDir/dem.tif --outfile="$outputHucDataDir/dem_meters.tif" --calc="A/100" --NoDataValue=$ndv
+gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputHucDataDir/nld_rasterized_elev.tif
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
@@ -156,7 +148,7 @@ echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwr
 date -u
 Tstart
 [ -f $outputHucDataDir/nld_rasterized_elev.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --NoDataValue $ndv --co "BLOCKXSIZE=512" --co "BLOCKYSIZE=512" --co "TILED=YES" --co "COMPRESS=LZW" --co "BIGTIFF=YES" -A $outputHucDataDir/dem_meters.tif -B $outputHucDataDir/nld_rasterized_elev.tif --outfile="$outputHucDataDir/dem_meters.tif" --calc="maximum(A,((B>-9999)*0.3048))" --NoDataValue=$ndv
+$srcDir/burn_in_levees.py -dem $outputHucDataDir/dem_meters.tif -nld $outputHucDataDir/nld_rasterized_elev.tif -out $outputHucDataDir/dem_meters.tif
 Tcount
 
 ## DEM Reconditioning ##
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index ee61c3d11..c29a92acf 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -9,7 +9,7 @@
 import pygeos
 from shapely.wkb import dumps, loads
 import warnings
-warnings.simplefilter(action='ignore', category=FutureWarning)
+warnings.simplefilter("ignore")
 
 ''' Get elevation at adjusted USGS gages locations
 
diff --git a/src/utils/reproject_dem.py b/src/utils/reproject_dem.py
new file mode 100755
index 000000000..dba8f65de
--- /dev/null
+++ b/src/utils/reproject_dem.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import os
+from osgeo import gdal
+import sys
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION_CM
+import shutil
+from multiprocessing import Pool
+import argparse
+
+
+def reproject_dem(args):
+
+    raster_dir            = args[0]
+    elev_cm               = args[1]
+    elev_cm_proj          = args[2]
+    reprojection          = args[3]
+
+    if os.path.exists(elev_cm_proj):
+        os.remove(elev_cm_proj)
+
+    shutil.copy(elev_cm, elev_cm_proj)
+
+    print(f"Reprojecting {elev_cm_proj}")
+    gdal.Warp(elev_cm_proj,elev_cm_proj,dstSRS=reprojection)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Burn in NLD elevations')
+    parser.add_argument('-dem_dir','--dem-dir', help='DEM filename', required=True,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
+
+    args = vars(parser.parse_args())
+
+    dem_dir            = args['dem_dir']
+    number_of_jobs     = args['number_of_jobs']
+
+    reproject_procs_list = []
+
+    for huc in os.listdir(dem_dir):
+        raster_dir = os.path.join(dem_dir,huc)
+        elev_cm = os.path.join(raster_dir, 'elev_cm.tif')
+        elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
+        reproject_procs_list.append([raster_dir, elev_cm, elev_cm_proj, PREP_PROJECTION_CM])
+
+    # Multiprocess reprojection
+    with Pool(processes=number_of_jobs) as pool:
+        pool.map(reproject_dem, reproject_procs_list)
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 6ea7b0a74..876c1769e 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -2,6 +2,10 @@
 
 import os
 from os.path import splitext
+import rasterio
+import numpy as np
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+from pyproj.crs import CRS
 
 def getDriver(fileName):
 
@@ -83,3 +87,123 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
     layer_name = os.path.split(wbd_gpkg)[1].strip('.gpkg')
     gdf.crs = PREP_PROJECTION
     gdf.to_file(multilayer_wbd_geopackage, layer=layer_name,driver='GPKG',index=False)
+
+
+def update_raster_profile(args):
+
+    elev_cm_filename   = args[0]
+    elev_m_filename    = args[1]
+    projection         = args[2]
+    nodata_val         = args[3]
+    blocksize          = args[4]
+    keep_intermediate  = args[5]
+
+    if isinstance(blocksize, int):
+        pass
+    elif isinstance(blocksize,str):
+        blocksize = int(blocksize)
+    elif isinstance(blocksize,float):
+        blocksize = int(blocksize)
+    else:
+        raise TypeError("Pass integer for blocksize")
+
+    assert elev_cm_filename.endswith('.tif'), "input raster needs to be a tif"
+
+    # Update nodata value and convert from cm to meters
+    dem_cm = rasterio.open(elev_cm_filename)
+
+    no_data = dem_cm.nodata
+    data = dem_cm.read(1)
+
+    dem_m = np.where(data == int(no_data), nodata_val, (data/100).astype(rasterio.float32))
+
+    del data
+
+    dem_m_profile = dem_cm.profile.copy()
+
+    dem_m_profile.update(driver='GTiff',tiled=True,nodata=nodata_val,
+                         blockxsize=blocksize, blockysize=blocksize,
+                         dtype='float32',crs=projection,compress='lzw',interleave='band')
+
+    with rasterio.open(elev_m_filename, "w", **dem_m_profile, BIGTIFF='YES') as dest:
+        dest.write(dem_m, indexes = 1)
+
+    if keep_intermediate == False:
+        os.remove(elev_cm_filename)
+
+    del dem_m
+    dem_cm.close()
+
+
+'''
+This function isn't currently used but is the preferred method for
+reprojecting elevation grids.
+
+Several USGS elev_cm.tifs have the crs value in their profile stored as the string "CRS.from_epsg(26904)"
+instead of the actual output of that command.
+
+Rasterio fails to properly read the crs but using gdal retrieves the correct projection.
+Until this issue is resolved use the reproject_dem function in reproject_dem.py instead.
+reproject_dem is not stored in the shared_functions.py because rasterio and
+gdal bindings are not entirely compatible: https://rasterio.readthedocs.io/en/latest/topics/switch.html
+
+'''
+def reproject_raster(input_raster_name,reprojection,blocksize=None,reprojected_raster_name=None):
+
+    if blocksize is not None:
+        if isinstance(blocksize, int):
+            pass
+        elif isinstance(blocksize,str):
+            blocksize = int(blocksize)
+        elif isinstance(blocksize,float):
+            blocksize = int(blocksize)
+        else:
+            raise TypeError("Pass integer for blocksize")
+    else:
+        blocksize = 256
+
+    assert input_raster_name.endswith('.tif'), "input raster needs to be a tif"
+
+    reprojection = rasterio.crs.CRS.from_string(reprojection)
+
+    with rasterio.open(input_raster_name) as src:
+
+        # Check projection
+        if src.crs.to_string() != reprojection:
+            if src.crs.to_string().startswith('EPSG'):
+                epsg = src.crs.to_epsg()
+                proj_crs = CRS.from_epsg(epsg)
+                rio_crs = rasterio.crs.CRS.from_user_input(proj_crs).to_string()
+            else:
+                rio_crs = src.crs.to_string()
+
+            print(f"{input_raster_name} not projected")
+            print(f"Reprojecting from {rio_crs} to {reprojection}")
+
+            transform, width, height = calculate_default_transform(
+                src.crs, reprojection, src.width, src.height, *src.bounds)
+            kwargs = src.meta.copy()
+            kwargs.update({
+                'crs': reprojection,
+                'transform': transform,
+                'width': width,
+                'height': height,
+                'compress': 'lzw'
+            })
+
+            if reprojected_raster_name is None:
+                reprojected_raster_name = input_raster_name
+
+            assert reprojected_raster_name.endswith('.tif'), "output raster needs to be a tif"
+
+            with rasterio.open(reprojected_raster_name, 'w', **kwargs, tiled=True, blockxsize=blocksize, blockysize=blocksize, BIGTIFF='YES') as dst:
+                reproject(
+                    source=rasterio.band(src, 1),
+                    destination=rasterio.band(dst, 1),
+                    src_transform=src.transform,
+                    src_crs=rio_crs,
+                    dst_transform=transform,
+                    dst_crs=reprojection.to_string(),
+                    resampling=Resampling.nearest)
+                del dst
+        del src
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 244a12d2b..37a98b424 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -2,6 +2,7 @@
 
 # Projections.
 #PREP_PROJECTION = "+proj=aea +datum=NAD83 +x_0=0.0 +y_0=0.0 +lon_0=96dW +lat_0=23dN +lat_1=29d30'N +lat_2=45d30'N +towgs84=-0.9956000824677655,1.901299877314078,0.5215002840524426,0.02591500053005733,0.009425998542707753,0.01159900118427752,-0.00062000005129903 +no_defs +units=m"
+PREP_PROJECTION_CM = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Albers"],PARAMETER["false_easting",0.0],PARAMETER["false_northing",0.0],PARAMETER["central_meridian",-96.0],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_origin",23.0],UNIT["Meter",1.0],VERTCS["NAVD_1988",VDATUM["North_American_Vertical_Datum_1988"],PARAMETER["Vertical_Shift",0.0],PARAMETER["Direction",1.0],UNIT["Centimeter",0.01]]]'
 PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'
 VIZ_PROJECTION ='PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
 # -- Data URLs-- #
diff --git a/tools/comparing_src.py b/tools/comparing_src.py
deleted file mode 100755
index 977b05794..000000000
--- a/tools/comparing_src.py
+++ /dev/null
@@ -1,393 +0,0 @@
-#!/usr/bin/env python3
-
-import matplotlib.pyplot as plt
-import numpy as np
-import json
-import geopandas as gpd
-import pandas as pd
-from raster import Raster
-import os
-from shapely.geometry import Point
-
-projectDirectory = os.path.join(os.path.expanduser('~'),'projects','foss_fim')
-dataDirectory = os.path.join(projectDirectory,'data')
-
-# nwm_catchments_fileName = os.path.join(dataDirectory,'nwm','NWMCatchment.shp')
-# nwm_flows_fileName = os.path.join(dataDirectory,'test2','inputs','nwm_flows_proj_120903_v2.gpkg')
-#
-# esri_catchments_fileName = os.path.join(projectDirectory,'tests','CatchmentH.shp')
-esri_flows_fileName = os.path.join(projectDirectory,'tests','eval_1','final_esri_hand_outputs','FPRiver.gpkg')
-
-# foss_catchments_fileName = os.path.join(dataDirectory,'test2','outputs','gw_catchments_reaches_clipped_addedAttributes_crosswalked.gpkg')
-# foss_raster_catchments_fileName = os.path.join(dataDirectory,'test2','outputs','gw_catchments_reaches_clipped_addedAttributes.tif')
-foss_flows_fileName = os.path.join(dataDirectory,'test2','outputs','demDerived_reaches_split_clipped_addedAttributes_crosswalked.gpkg')
-foss_flows_fileName = os.path.join(dataDirectory,'test2','outputs','NHDPlusBurnLineEvent_subset_split_clipped_addedAttributes_crosswalked.gpkg')
-# foss_flows_fileName = os.path.join(dataDirectory,'test2','outputs_v32','demDerived_reaches_split_clipped_addedAttributes_crosswalked.gpkg')
-
-esri_src_fileName = os.path.join(projectDirectory,'tests','eval_1','final_esri_hand_outputs','120903_channel_properties.json')
-foss_src_fileName = os.path.join(dataDirectory,'test2','outputs','src.json')
-# foss_src_fileName = os.path.join(dataDirectory,'test2','outputs_v32','src.json')
-
-esri_src_table_fileName = os.path.join(projectDirectory,'tests','eval_1','final_esri_hand_outputs','pf_ModelStream.csv')
-foss_src_table_fileName = os.path.join(dataDirectory,'test2','outputs','src_full_crosswalked.csv')
-# foss_src_table_fileName = os.path.join(dataDirectory,'test2','outputs_v32','src_full_crosswalked.csv')
-
-esri_cw_fileName = os.path.join(projectDirectory,'tests','eval_1','final_esri_hand_outputs','cross_walk_table_esri_120903.csv')
-foss_cw_fileName = os.path.join(dataDirectory,'test2','outputs','crosswalk_table.csv')
-# foss_cw_fileName = os.path.join(dataDirectory,'test2','outputs_v32','crosswalk_table.csv')
-
-esri_rem_fileName = os.path.join(projectDirectory,'tests','eval_1','final_esri_hand_outputs','hand_120903.tif')
-foss_rem_fileName = os.path.join(dataDirectory,'test2','outputs','rem_clipped_zeroed_masked.tif')
-
-forecast_100_fileName = os.path.join(projectDirectory,'tests','eval_1','validation_data','forecast_120903_100yr.csv')
-
-# catchments
-# esri_catchments = gpd.read_file(esri_catchments_fileName)
-# foss_catchments = gpd.read_file(foss_catchments_fileName)
-# # foss_raster_catchments = Raster(foss_raster_catchments_fileName)
-# nwm_catchments = gpd.read_file(nwm_catchments_fileName)
-#
-# # flows
-esri_flows = gpd.read_file(esri_flows_fileName)
-foss_flows = gpd.read_file(foss_flows_fileName)
-# nwm_flows = gpd.read_file(nwm_flows_fileName)
-
-# slopes
-# esri_slope=Raster('eval_1/final_esri_hand_outputs/unitrun/hdr.adf')
-# foss_slope=Raster('../data/test2/outputs/slopes_d8_thalwegCond_filled_clipped_masked.tif')
-#
-# foss_cell_area = abs(foss_rem.gt[1]*foss_rem.gt[5])
-# esri_cell_area = abs(esri_rem.gt[1] * esri_rem.gt[5])
-#
-# foss_dv_bool = foss_rem.array!=foss_rem.ndv
-# esri_dv_bool = esri_rem.array!=esri_rem.ndv
-#
-# esri_slope.array = esri_slope.array[esri_dv_bool] - 1
-#
-# foss_slopes_trans = np.sqrt(1+(foss_slope.array[foss_dv_bool])**2)
-# esri_slopes_trans = np.sqrt(1+(esri_slope.array[esri_dv_bool])**2)
-# for d in np.array(range(0,30))*.3048:
-#     foss_area = np.sum(np.logical_and(foss_dv_bool,foss_rem.array<=d) * foss_cell_area * foss_slopes_trans)
-#     esri_area = np.sum(np.logical_and(esri_dv_bool,esri_rem.array<=d) * esri_cell_area * esri_slopes_trans)
-
-
-
-
-
-# sinuosity
-
-def sinuosity(flows_geometry):
-
-    numberOfGeoms = len(flows_geometry)
-    arc_lengths = [-1] * numberOfGeoms ; straight_lengths = [-1] * numberOfGeoms
-
-    for i,geom in enumerate(flows_geometry):
-        arc_lengths[i] = geom.length
-
-        point_1 = Point(*geom.bounds[0:2])
-        point_2 = Point(*geom.bounds[2:4])
-
-        straight_lengths[i] = point_1.distance(point_2)
-
-    sinuosity_table = pd.DataFrame({'arc_lengths' : arc_lengths , 'straight_lengths' : straight_lengths})
-
-    return(sinuosity_table)
-
-esri_sinuosity = sinuosity(esri_flows.geometry)
-foss_sinuosity = sinuosity(foss_flows.geometry)
-
-avg_esri_si = (esri_sinuosity['arc_lengths']/esri_sinuosity['straight_lengths']).mean()
-avg_foss_si = (foss_sinuosity['arc_lengths']/foss_sinuosity['straight_lengths']).mean()
-
-print(avg_esri_si,avg_foss_si,avg_esri_si/avg_foss_si)
-
-
-# SRCS's
-with open(esri_src_fileName,'r') as f:
-    esri_src = json.load(f)
-
-with open(foss_src_fileName,'r') as f:
-    foss_src = json.load(f)
-
-esri_cw = pd.read_csv(esri_cw_fileName,dtype=int)
-foss_cw = pd.read_csv(foss_cw_fileName,dtype=int)
-
-esri_rem = Raster(esri_rem_fileName)
-foss_rem = Raster(foss_rem_fileName)
-
-esri_src_table = pd.read_csv(esri_src_table_fileName,dtype={'A':float, 'B':float, 'H':float, 'Length_m':float, 'P':float, 'R':float, 'HydroID':int, 'Q':float})
-foss_src_table = pd.read_csv(foss_src_table_fileName,dtype={'HydroID':int, 'Stage':float, 'Number of Cells':int, 'SurfaceArea (m2)':float,
-       'BedArea (m2)':float, 'Volume (m3)':float, 'SLOPE':float, 'LENGTHKM':float, 'AREASQKM':float,
-       'Roughness':float, 'TopWidth (m)':float, 'WettedPerimeter (m)':float, 'WetArea (m2)':float,
-       'HydraulicRadius (m)':float, 'Discharge (m3s-1)':float, 'feature_id':int})
-
-forecast_100 = pd.read_csv(forecast_100_fileName,dtype={'feature_id' : int , 'discharge' : float})
-
-intersection_of_feature_id = list(set(esri_cw['feature_id'].unique()) & set(foss_cw['feature_id'].unique()) & set(forecast_100['feature_id'].unique())  )
-
-
-max_q = np.max(forecast_100['discharge'])
-# print(max_q)
-
-esri_src_table['BA'] = esri_src_table['P'] * esri_src_table['Length_m']
-esri_src_table['V'] = esri_src_table['A'] * esri_src_table['Length_m']
-
-esri_src_table = esri_src_table[:][esri_src_table['H']<=10]
-foss_src_table = foss_src_table[:][foss_src_table['Stage']<=10]
-# print(esri_src_table.sort_values(by=['HydroID','H']))
-
-esri_cw = esri_cw[:][esri_cw['feature_id'].isin(intersection_of_feature_id)]
-foss_cw = foss_cw[:][foss_cw['feature_id'].isin(intersection_of_feature_id)]
-
-esri_src_table = esri_src_table.merge(esri_cw,on='HydroID',how='inner')
-foss_src_table = foss_src_table.merge(foss_cw,on='HydroID',how='inner')
-
-foss_src_table.drop(columns='feature_id_y',inplace=True)
-foss_src_table.rename(columns={'feature_id_x':'feature_id'},inplace=True)
-# esri_hids = esri_cw['HydroID'][esri_cw['feature_id'].isin(intersection_of_feature_id)]
-# foss_hids = foss_cw['HydroID'][foss_cw['feature_id'].isin(intersection_of_feature_id)]
-
-# esri_src_table = esri_src_table[:][esri_src_table['HydroID'].isin(esri_hids)]
-# foss_src_table = foss_src_table[:][foss_src_table['HydroID'].isin(foss_hids)]
-
-# esri_src_table = esri_src_table[:][esri_src_table['HydroID'].isin(esri_hids)]
-# foss_src_table = foss_src_table[:][foss_src_table['HydroID'].isin(foss_hids)]
-
-foss_src_table['Length_m'] = foss_src_table['LENGTHKM'] *1000
-esri_src_table = esri_src_table.merge(esri_flows[['HydroID','S0']],on='HydroID',how='left')
-
-foss_src_table.rename(columns={'Stage' : 'H' , 'BedArea (m2)' : 'BA','Volume (m3)' : 'V' ,
-                                'SLOPE' : 'S0' , 'WettedPerimeter (m)': 'P', 'WetArea (m2)' : 'A',
-                                'HydraulicRadius (m)':'R', 'Discharge (m3s-1)': 'Q'},inplace=True)
-
-foss_src_table = foss_src_table[['H' , 'BA','V' ,'S0' ,'P','A','R','Q','feature_id','HydroID','Length_m']]
-
-foss_src_table['n'] = 0.06
-esri_src_table['n'] = 0.06
-
-# esri_src_table.sort_values(by=['HydroID','H'],inplace=True)
-# foss_src_table.sort_values(by=['HydroID','H'],inplace=True)
-
-esri_src_table.drop(columns='HydroID',inplace=True)
-foss_src_table.drop(columns='HydroID',inplace=True)
-
-esri_src_table = esri_src_table.astype({'H' : str})
-foss_src_table = foss_src_table.astype({'H' : str})
-# esri_src_table = esri_src_table.groupby(['feature_id','H']).mean()
-# foss_src_table = foss_src_table.groupby(['feature_id','H']).mean()
-# esri_src_table = esri_src_table.astype({'H' :float})
-# foss_src_table = foss_src_table.astype({'H' :float})
-
-# esri_src_table.reset_index(drop=True)
-# foss_src_table.reset_index(drop=True)
-
-src_table = foss_src_table.merge(esri_src_table,suffixes=('_foss','_esri'),on=['feature_id','H'])
-# esri_src_table.sort_values(by=['HydroID','H'],inplace=True)
-
-# src_table.sort_values(by=['feature_id','H'],inplace=True)
-# src_table.reset_index(drop=False,inplace=True)
-
-src_table = src_table.groupby('H').mean()
-src_table.reset_index(drop=False,inplace=True)
-src_table = src_table.astype({'H' :float})
-src_table.sort_values(by=['H'],inplace=True)
-# print(src_table.index)
-
-pd.set_option('display.max_rows', 2000)
-# print(src_table[['feature_id','H','V_esri','V_foss']].iloc[0:200,:])
-# print(src_table)
-percent_error_V = 100 * (src_table['V_foss'].iloc[1:]-src_table['V_esri'].iloc[1:])/src_table['V_esri'].iloc[1:]
-percent_error_BA = 100 * (src_table['BA_foss'].iloc[1:]-src_table['BA_esri'].iloc[1:])/src_table['BA_esri'].iloc[1:]
-percent_error_L = 100 * (src_table['Length_m_foss']-src_table['Length_m_esri'])/src_table['Length_m_esri']
-percent_error_S = 100 * (src_table['S0_foss']-src_table['S0_esri'])/src_table['S0_esri']
-percent_error_Q = 100 * (src_table['Q_foss'].iloc[1:]-src_table['Q_esri'].iloc[1:])/src_table['Q_esri'].iloc[1:]
-
-multiplied_error_V = (src_table['V_foss'].iloc[1:]/src_table['V_esri'].iloc[1:])**(5/3)
-multiplied_error_BA = (src_table['BA_foss'].iloc[1:]/src_table['BA_esri'].iloc[1:])**(2/3)
-multiplied_error_L = (src_table['Length_m_foss']/src_table['Length_m_esri'])
-multiplied_error_S = (src_table['S0_foss']/src_table['S0_esri'])**(1/2)
-multiplied_error_Q = (src_table['Q_foss'].iloc[1:]/src_table['Q_esri'].iloc[1:])
-
-print(percent_error_V.mean(),percent_error_BA.mean(),percent_error_L.mean(),percent_error_S.mean(),percent_error_Q.mean())
-print(multiplied_error_V.mean(),multiplied_error_BA.mean(),multiplied_error_L.mean(),multiplied_error_S.mean(),multiplied_error_Q.mean())
-print((multiplied_error_V.mean()*multiplied_error_S.mean())/(multiplied_error_BA.mean()*multiplied_error_L.mean()))
-# print(percent_error_V,percent_error_BA,percent_error_L,percent_error_S,percent_error_Q)
-# exit()
-#
-# tot_V_esri = [] ; tot_V_foss = []
-# foss_dv_bool = foss_rem.array!=foss_rem.ndv
-# esri_dv_bool = esri_rem.array!=esri_rem.ndv
-# for d in np.array(range(0,30))*.3048:
-#     foss_cell_area = abs(foss_rem.gt[1]*foss_rem.gt[5])
-#     esri_cell_area = abs(esri_rem.gt[1] * esri_rem.gt[5])
-#     foss_volume = np.sum(d-foss_rem.array[np.logical_and(foss_dv_bool,foss_rem.array<=d)]) * foss_cell_area
-#     esri_volume = np.sum(d-esri_rem.array[np.logical_and(esri_dv_bool,esri_rem.array<=d)]) * esri_cell_area
-#     tot_V_esri = tot_V_esri + [esri_volume] ; tot_V_foss = tot_V_foss + [foss_volume]
-#
-# print(np.array(tot_V_foss).mean()/np.array(tot_V_esri).mean())
-# print((foss_dv_bool.sum() * foss_cell_area) / (esri_dv_bool.sum() * esri_cell_area))
-
-
-
-
-
-# print(esri_src_table[['feature_id','H','V']].iloc[0:20,:])
-# print(foss_src_table)
-# print(esri_src_table)
-
-
-
-
-# foss_src_table['HydroID']
-
-stage_list = foss_src[str(500)]['stage_list']
-maxLength = len(stage_list)
-
-overall_esri = None
-for fid in intersection_of_feature_id:
-    esri_hid = esri_cw['HydroID'][esri_cw['feature_id'] == fid].to_numpy()
-    foss_hid = foss_cw['HydroID'][foss_cw['feature_id'] == fid].to_numpy()
-
-    # all_esri_q = np.zeros(len(esri_src[str(esri_hid[0])]['stage_list']),dtype=np.float32)
-    all_esri_q = None
-    for hid in esri_hid:
-        current_esri_q = np.array(esri_src[str(hid)]['q_list'])
-
-        if len(current_esri_q) < maxLength:
-            nan_array = np.repeat(np.nan, maxLength - len(current_esri_q))
-            # print(nan_array)
-            current_esri_q = np.hstack((current_esri_q,nan_array))
-
-        if len(current_esri_q) > maxLength:
-            current_esri_q = current_esri_q[0:maxLength]
-
-        if all_esri_q is None:
-            all_esri_q = current_esri_q
-        else:
-            all_esri_q = np.vstack((all_esri_q,current_esri_q))
-
-    all_foss_q = None
-    for hid in foss_hid:
-
-        current_foss_q = np.array(foss_src[str(hid)]['q_list'])
-
-        if all_foss_q is None:
-            all_foss_q = current_foss_q
-        else:
-            all_foss_q = np.vstack((all_foss_q,current_foss_q))
-
-    # print(all_esri_q.shape,all_foss_q.shape)
-    # print(all_esri_q)
-
-    if len(all_esri_q.shape) == 2:
-        mean_esri_q = np.nanmean(all_esri_q,axis=0)
-
-    if len(all_foss_q.shape) == 2:
-        mean_foss_q = np.nanmean(all_foss_q,axis=0)
-
-    # mean_error = mean_foss_q-mean_esri_q
-
-    # print(mean_esri_q.shape,mean_foss_q.shape,mean_error.shape)
-
-    # mean_abs_error = np.absolute(mean_error)
-
-    if overall_esri is None:
-        # overall_error = mean_error
-        overall_esri = mean_esri_q
-        overall_foss = mean_foss_q
-        # overall_abs_error = mean_abs_error
-    else:
-        # print(mean_error,overall_error.shape)
-        # overall_error = np.vstack((overall_error,mean_error))
-        overall_esri = np.vstack((overall_esri,mean_esri_q))
-        overall_foss = np.vstack((overall_foss,mean_foss_q))
-        # overall_abs_error = np.vstack((overall_abs_error,mean_abs_error))
-
-# print(overall_error)
-# print(list(overall_error))
-# overall_error_q_list = list(np.nanmean(overall_error,axis=0))
-overall_esri_q_list = list(np.nanmean(overall_esri,axis=0))
-overall_foss_q_list = list(np.nanmean(overall_foss,axis=0))
-
-plt.plot(overall_esri_q_list,stage_list,'r')
-plt.plot(overall_foss_q_list,stage_list,'b')
-# plt.axis([0,max_q*1.1,0,10])
-plt.show()
-
-exit()
-
-
-
-
-
-
-
-
-# print(np.mean(overall_abs_error,axis=0))
-
-# foss_src = pd.read_csv(foss_src_fileName,skip_blank_lines=True,dtype=object)
-
-# print('\nFeature IDs')
-# print("ESRI # of unique catchments: {}".format(len(np.unique(esri_catchments['feature_id']))))
-# print("FOSS # of unique catchments: {}".format(len(np.unique(foss_catchments['feature_id']))))
-# print("NWM # of unique catchments: {}".format(len(np.unique(nwm_catchments['feature_id']))))
-# print("ESRI # of unique flows: {}".format(len(np.unique(esri_flows['feature_id']))))
-# print("FOSS # of unique flows: {}".format(len(np.unique(foss_flows['feature_id']))))
-# print("NWM # of unique flows: {}".format(len(np.unique(nwm_flows['ID']))))
-# print("FOSS # of unique SRC Feature ID: {}".format(len(np.unique(foss_src['feature_id']))))
-#
-# print('\nHydroID')
-# print("ESRI # of unique catchments: {}".format(len(np.unique(esri_catchments['HydroID']))))
-# print("FOSS # of unique catchments: {}".format(len(np.unique(foss_catchments['HydroID']))))
-# # print("FOSS # of unique catchments in raster: {}".format(len(np.unique(foss_raster_catchments.array[foss_raster_catchments.array!=foss_raster_catchments.ndv]))))
-# print("ESRI # of unique flows: {}".format(len(np.unique(esri_flows['HydroID']))))
-# print("FOSS # of unique flows: {}".format(len(np.unique(foss_flows['HydroID']))))
-# print("ESRI # of unique SRC HydroID: {}".format(len(np.unique(list(esri_src.keys())))))
-# print("FOSS # of unique HydroID's: {}".format(len(np.unique(foss_src['HydroID']))))
-#
-# print(foss_flows['LengthKm'].max())
-# print(foss_flows['LengthKm'].mean())
-
-# print(list(esri_src.keys()))
-
-# print(len(foss_src))
-# plots src's
-# unique_feature_ids_in_foss_src = np.unique(foss_src['feature_id'])
-
-# featID = 5791828
-
-# indices_of_feature = np.where(foss_src['feature_id'] == featID)
-
-# unique_hydro_ids = np.unique(foss_src['HydroID'][indices_of_feature])
-
-# hydroID = '822'
-# esri_hydroID = '9975'
-
-# hydroID = '1279'
-# esri_hydroID = '10349'
-
-hydroID = '1268'
-esri_hydroID = '10743'
-
-hydroID = '1269'
-esri_hydroID = '10742'
-
-# indices_of_hydroid = np.where(foss_src['HydroID'] == hydroID)[0]
-
-foss_stages = foss_src[hydroID]['stage_list']
-foss_discharge = foss_src[hydroID]['q_list']
-
-# feature_id = foss_src['feature_id'][indices_of_hydroid[0]]
-esri_stages = esri_src[esri_hydroID]['stage_list']
-esri_flows = esri_src[esri_hydroID]['q_list']
-
-
-plt.plot(foss_discharge,foss_stages,'b')
-plt.plot(esri_flows,esri_stages,'r')
-plt.show()
-
-# for hid in unique_hydro_ids:
-
-
-
-# for featID in unique_feature_ids_in_foss_src:
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index f51bf5aa8..cea992df1 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+
+import os
 import subprocess
 import argparse
 import time
@@ -11,7 +13,7 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     '''
     Updates the status for nws_lids from the flows subdirectory. Status
     is updated for sites where the inundation.py routine was not able to
-    produce inundation for the supplied flow files. It is assumed that if 
+    produce inundation for the supplied flow files. It is assumed that if
     an error occured in inundation.py that all flow files for a given site
     experienced the error as they all would have the same nwm segments.
 
@@ -27,86 +29,80 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     None.
 
     '''
-    #Find all LIDs with empty mapping output folders
+    # Find all LIDs with empty mapping output folders
     subdirs = [str(i) for i in Path(output_mapping_dir).rglob('**/*') if i.is_dir()]
     empty_nws_lids = [Path(directory).name for directory in subdirs if not list(Path(directory).iterdir())]
-    
-    #Write list of empty nws_lids to DataFrame, these are sites that failed in inundation.py
+
+    # Write list of empty nws_lids to DataFrame, these are sites that failed in inundation.py
     mapping_df = pd.DataFrame({'nws_lid':empty_nws_lids})
     mapping_df['did_it_map'] = 'no'
     mapping_df['map_status'] = ' and all categories failed to map'
-    
-    #Import shapefile output from flows creation 
+
+    # Import shapefile output from flows creation
     shapefile = Path(output_flows_dir)/'nws_lid_flows_sites.shp'
     flows_df = gpd.read_file(shapefile)
-    
-    #Join failed sites to flows df    
+
+    # Join failed sites to flows df
     flows_df = flows_df.merge(mapping_df, how = 'left', on = 'nws_lid')
-    
-    #Switch mapped column to no for failed sites and update status
+
+    # Switch mapped column to no for failed sites and update status
     flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
     flows_df.loc[flows_df['did_it_map']=='no','status'] = flows_df['status'] + flows_df['map_status']
-    
-    #Perform pass for HUCs where mapping was skipped due to missing data.
+
+    # Perform pass for HUCs where mapping was skipped due to missing data
     flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
     mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
     missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
-    #Update status for nws_lid in missing hucs and change mapped attribute to 'no'
+    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
     flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
     flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
-            
-    #Clean up GeoDataFrame and rename columns for consistency.
+
+    # Clean up GeoDataFrame and rename columns for consistency
     flows_df = flows_df.drop(columns = ['did_it_map','map_status'])
     flows_df = flows_df.rename(columns = {'nws_lid':'ahps_lid'})
-    
-    #Write out to file
+
+    # Write out to file
     nws_lid_path = Path(output_mapping_dir) / 'nws_lid_sites.shp'
     flows_df.to_file(nws_lid_path)
-    
+
 if __name__ == '__main__':
-    
-    #Parse arguments
+
+    # Parse arguments
     parser = argparse.ArgumentParser(description = 'Run Categorical FIM')
     parser.add_argument('-f','--fim_version',help='Name of directory containing outputs of fim_run.sh',required=True)
     parser.add_argument('-j','--number_of_jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
     args = vars(parser.parse_args())
-    
-    #Get arguments
+
+    # Get arguments
     fim_version = args['fim_version']
     number_of_jobs = args['number_of_jobs']
-    
-    ####################################################################
-    #Define default arguments. Modify these if necessary. 
-    today = date.today().strftime('%m%d%Y')
-    fim_run_dir = Path(f'/data/previous_fim/{fim_version}')
-    output_flows_dir = Path(f'/data/catfim/{fim_version}/{today}/flows')
-    output_mapping_dir = Path(f'/data/catfim/{fim_version}/{today}/mapping')
+
+    # Define default arguments. Modify these if necessary
+    fim_run_dir = Path(f'{fim_version}')
+    fim_version_folder = os.path.basename(fim_version)
+    output_flows_dir = Path(f'/data/catfim/{fim_version_folder}/flows')
+    output_mapping_dir = Path(f'/data/catfim/{fim_version_folder}/mapping')
     nwm_us_search = '10'
-    nwm_ds_search = '10'        
+    nwm_ds_search = '10'
     write_depth_tiff = False
-    ####################################################################
-    
-    ####################################################################
-    #Run CatFIM scripts in sequence
-    ####################################################################
-    #Generate CatFIM flow files.
+
+    ## Run CatFIM scripts in sequence
+    # Generate CatFIM flow files
     print('Creating flow files')
     start = time.time()
-    subprocess.call(['python3','generate_categorical_fim_flows.py', '-w' , str(output_flows_dir), '-u', nwm_us_search, '-d', nwm_ds_search])
+    subprocess.call(['python3','foss_fim/tools/generate_categorical_fim_flows.py', '-w' , str(output_flows_dir), '-u', nwm_us_search, '-d', nwm_ds_search])
     end = time.time()
     elapsed_time = (end-start)/60
     print(f'Finished creating flow files in {elapsed_time} minutes')
-    
-    #Generate CatFIM mapping.
+
+    # Generate CatFIM mapping
     print('Begin mapping')
     start = time.time()
-    subprocess.call(['python3','generate_categorical_fim_mapping.py', '-r' , str(fim_run_dir), '-s', str(output_flows_dir), '-o', str(output_mapping_dir), '-j', str(number_of_jobs)]) 
+    subprocess.call(['python3','foss_fim/tools/generate_categorical_fim_mapping.py', '-r' , str(fim_run_dir), '-s', str(output_flows_dir), '-o', str(output_mapping_dir), '-j', str(number_of_jobs)])
     end = time.time()
     elapsed_time = (end-start)/60
     print(f'Finished mapping in {elapsed_time} minutes')
-    
-    #Updating Mapping Status
+
+    # Updating mapping status
     print('Updating mapping status')
     update_mapping_status(str(output_mapping_dir), str(output_flows_dir))
-
-   
\ No newline at end of file
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index 3d591b989..9924d9305 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -130,7 +130,7 @@ def run_inundation(args):
                  subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=output_extent_grid,inundation_polygon=None,
                  depths=output_depth_grid,out_raster_profile=None,out_vector_profile=None,quiet=True
                 )
-    
+
     except:
         # Log errors and their tracebacks
         f = open(log_file, 'a+')
@@ -138,14 +138,14 @@ def run_inundation(args):
         f.close()
 
     #Inundation.py appends the huc code to the supplied output_extent_grid.
-    #Modify output_extent_grid to match inundation.py saved filename. 
+    #Modify output_extent_grid to match inundation.py saved filename.
     #Search for this file, if it didn't create, send message to log file.
     base_file_path,extension = os.path.splitext(output_extent_grid)
     saved_extent_grid_filename = "{}_{}{}".format(base_file_path,huc,extension)
     if not os.path.exists(saved_extent_grid_filename):
         with open(log_file, 'a+') as f:
             f.write('FAILURE_huc_{}:{}:{} map failed to create\n'.format(huc,ahps_site,magnitude))
-        
+
 def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_attributes_filename, log_file):
 
     # Create workspace
@@ -153,11 +153,9 @@ def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_att
     if not os.path.exists(gpkg_dir):
         os.mkdir(gpkg_dir)
 
-    
-    #Find the FIM version
-    norm_path = os.path.normpath(output_cat_fim_dir)
-    cat_fim_dir_parts = norm_path.split(os.sep)
-    [fim_version] = [part for part in cat_fim_dir_parts if part.startswith('fim_3')]
+
+    # Find the FIM version
+    fim_version  = os.path.basename(output_cat_fim_dir)
     merged_layer = os.path.join(output_cat_fim_dir, 'catfim_library.shp')
 
     if not os.path.exists(merged_layer): # prevents appending to existing output
@@ -241,8 +239,8 @@ def reformat_inundation_maps(args):
 
         # Aggregate shapes
         results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
-                       
-        # convert list of shapes to polygon
+
+        # Convert list of shapes to polygon
         extent_poly  = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
 
         # Dissolve polygons
@@ -254,7 +252,7 @@ def reformat_inundation_maps(args):
         extent_poly_diss['magnitude'] = magnitude
         extent_poly_diss['version'] = fim_version
         extent_poly_diss['huc'] = huc
-            
+
         # Project to Web Mercator
         extent_poly_diss = extent_poly_diss.to_crs(VIZ_PROJECTION)
 

From 6387384b62523b601790a04e1f5dd2ca09578c29 Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Tue, 20 Apr 2021 09:54:37 -0500
Subject: [PATCH 067/359] Closing multiprocessing pool objects

Updating repo to close all multiprocessing Pool objects to fix memory issues
---
 CHANGELOG.md                              |  4 ++
 src/acquire_and_preprocess_inputs.py      | 14 +++----
 src/aggregate_fim_outputs.py              |  4 +-
 tools/cache_metrics.py                    | 46 +++++++++++------------
 tools/generate_categorical_fim_mapping.py |  9 ++---
 tools/rating_curve_comparison.py          |  8 ++--
 tools/synthesize_test_cases.py            | 28 +++++++-------
 7 files changed, 58 insertions(+), 55 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2d9969a11..f6143d388 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v3.0.15.4 - 2021-04-20 - [PR #356](https://github.com/NOAA-OWP/cahaba/pull/356)
+
+Closing all multiprocessing Pool objects in repo.
+<br/><br/>
 ## v3.0.15.3 - 2021-04-19 - [PR #358](https://github.com/NOAA-OWP/cahaba/pull/358)
 
 Preprocess NHDPlus HR rasters for consistent projections, nodata values, and convert from cm to meters.
diff --git a/src/acquire_and_preprocess_inputs.py b/src/acquire_and_preprocess_inputs.py
index 852942ff3..89e432966 100755
--- a/src/acquire_and_preprocess_inputs.py
+++ b/src/acquire_and_preprocess_inputs.py
@@ -107,8 +107,8 @@ def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_
             #wbd_gpkg_list.append(output_gpkg)
             #procs_list.append(['ogr2ogr -overwrite -progress -f GPKG -t_srs "{projection}" {output_gpkg} {wbd_gdb_path} {wbd_layer}'.format(output_gpkg=output_gpkg, wbd_gdb_path=wbd_gdb_path, wbd_layer=wbd_layer, projection=PREP_PROJECTION)])
 
-        #pool = Pool(num_workers)
-        #pool.map(run_system_command, procs_list)
+        # with Pool(processes=num_workers) as pool:
+            # pool.map(run_system_command, procs_list)
 
         # Subset WBD layers to CONUS and add to single geopackage.
         #print("Subsetting WBD layers to CONUS...")
@@ -150,9 +150,8 @@ def pull_and_prepare_nwm_hydrofabric(path_to_saved_data_parent_dir, path_to_prei
         output_gpkg = os.path.join(nwm_hydrofabric_directory, nwm_layer + '_proj.gpkg')
         procs_list.append(['ogr2ogr -overwrite -progress -f GPKG -t_srs "{projection}" {output_gpkg} {nwm_hydrofabric_gdb} {nwm_layer}'.format(projection=PREP_PROJECTION, output_gpkg=output_gpkg, nwm_hydrofabric_gdb=nwm_hydrofabric_gdb, nwm_layer=nwm_layer)])
 
-    pool = Pool(num_workers)
-    pool.map(run_system_command, procs_list)
-    pool.close()
+    with Pool(processes=num_workers) as pool:
+        pool.map(run_system_command, procs_list)
 
 
 def pull_and_prepare_nhd_data(args):
@@ -349,8 +348,9 @@ def manage_preprocessing(hucs_of_interest, num_workers=1,overwrite_nhd=False, ov
         nhd_procs_list.append([nhd_raster_download_url, nhd_raster_extraction_path, nhd_vector_download_url, nhd_vector_extraction_path, overwrite_nhd])
 
     # Pull and prepare NHD data.
-    #pool = Pool(num_workers)
-    #pool.map(pull_and_prepare_nhd_data, nhd_procs_list)
+    # with Pool(processes=num_workers) as pool:
+        # pool.map(pull_and_prepare_nhd_data, nhd_procs_list)
+
     for huc in nhd_procs_list:
         try:
             pull_and_prepare_nhd_data(huc)
diff --git a/src/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py
index 9d8676364..d6fdcf698 100644
--- a/src/aggregate_fim_outputs.py
+++ b/src/aggregate_fim_outputs.py
@@ -218,5 +218,5 @@ def reproject_raster(raster_name):
         procs_list.append([fim_outputs_directory,huc6,limited_huc_list])
 
     print(f"aggregating {len(huc_list)} hucs to HUC6 scale using {number_of_jobs} jobs")
-    pool = Pool(number_of_jobs)
-    pool.map(aggregate_fim_outputs, procs_list)
+    with Pool(processes=number_of_jobs) as pool:
+        pool.map(aggregate_fim_outputs, procs_list)
diff --git a/tools/cache_metrics.py b/tools/cache_metrics.py
index 3f601cc5c..0d02fb217 100755
--- a/tools/cache_metrics.py
+++ b/tools/cache_metrics.py
@@ -13,15 +13,15 @@
 
 
 def process_alpha_test(args):
-    
+
     fim_run_dir = args[0]
     version = args[1]
     test_id = args[2]
     magnitude = args[3]
     archive_results = args[4]
-    
-    mask_type = 'huc'    
-    
+
+    mask_type = 'huc'
+
     if archive_results == False:
         compare_to_previous = True
     else:
@@ -42,9 +42,9 @@ def process_alpha_test(args):
     parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
     parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
     parser.add_argument('-b','--benchmark-category',help='Options include ble or ahps. Defaults to process both.',required=False, default=None)
-        
+
     test_cases_dir_list = os.listdir(TEST_CASES_DIR)
-    
+
     args = vars(parser.parse_args())
 
     config = args['config']
@@ -52,21 +52,21 @@ def process_alpha_test(args):
     job_number = int(args['job_number'])
     special_string = args['special_string']
     benchmark_category = args['benchmark_category']
-    
+
     if fim_version != "all":
         previous_fim_list = [fim_version]
     else:
-        previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)    
-    
+        previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)
+
     if config == 'PREV':
         archive_results = True
     elif config == 'DEV':
         archive_results = False
     else:
         print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
-    
+
     benchmark_category_list = []
-    
+
     if benchmark_category == None:
         for d in test_cases_dir_list:
             if 'test_cases' in d:
@@ -77,42 +77,42 @@ def process_alpha_test(args):
     procs_list = []
     for bench_cat in benchmark_category_list:
         bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
-        
+
         bench_cat_test_case_list = os.listdir(bench_cat_test_case_dir)
-    
+
         for test_id in bench_cat_test_case_list:
             if 'validation' and 'other' not in test_id:
-                            
+
                 current_huc = test_id.split('_')[0]
                 if test_id.split('_')[1] in bench_cat:
-                
+
                     for version in previous_fim_list:
-                        
+
                         if config == 'DEV':
                             fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
                         elif config == 'PREV':
                             fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
-                                                
+
                         if not os.path.exists(fim_run_dir):
                             fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  # For previous versions of HAND computed at HUC6 scale
-                        
+
                         if os.path.exists(fim_run_dir):
                             if special_string != "":
                                 version = version + '_' + special_string
-                            
+
                             if 'ble' in test_id:
                                 magnitude = ['100yr', '500yr']
                             elif 'usgs' or 'nws' in test_id:
                                 magnitude = ['action', 'minor', 'moderate', 'major']
                             else:
                                 continue
-                        
+
                             print("Adding " + test_id + " to list of test_ids to process...")
                             if job_number > 1:
                                 procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results])
-                            else:                            
+                            else:
                                 process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results])
 
     if job_number > 1:
-        pool = Pool(job_number)
-        pool.map(process_alpha_test, procs_list)
\ No newline at end of file
+        with Pool(processes=job_number) as pool:
+            pool.map(process_alpha_test, procs_list)
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index 9924d9305..ebe76cd61 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -108,8 +108,8 @@ def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, n
 
     # Initiate multiprocessing
     print(f"Running inundation for {len(procs_list)} sites using {number_of_jobs} jobs")
-    pool = Pool(number_of_jobs)
-    pool.map(run_inundation, procs_list)
+    with Pool(processes=number_of_jobs) as pool:
+        pool.map(run_inundation, procs_list)
 
 
 def run_inundation(args):
@@ -153,7 +153,6 @@ def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_att
     if not os.path.exists(gpkg_dir):
         os.mkdir(gpkg_dir)
 
-
     # Find the FIM version
     fim_version  = os.path.basename(output_cat_fim_dir)
     merged_layer = os.path.join(output_cat_fim_dir, 'catfim_library.shp')
@@ -193,8 +192,8 @@ def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_att
                                 pass
 
             # Multiprocess with instructions
-            pool = Pool(number_of_jobs)
-            pool.map(reformat_inundation_maps, procs_list)
+            with Pool(processes=number_of_jobs) as pool:
+                pool.map(reformat_inundation_maps, procs_list)
 
         # Merge all layers
         print(f"Merging {len(os.listdir(gpkg_dir))} layers...")
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 77e6b91ba..714f2c535 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -38,13 +38,13 @@
 def check_file_age(file):
     '''
     Checks if file exists, determines the file age, and recommends
-    updating if older than 1 month. 
+    updating if older than 1 month.
 
     Returns
     -------
     None.
 
-    '''    
+    '''
     file = Path(file)
     if file.is_file():
         modification_time = file.stat().st_mtime
@@ -419,8 +419,8 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
 
     # Initiate multiprocessing
     print(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")
-    pool = Pool(number_of_jobs)
-    pool.map(generate_rating_curve_metrics, procs_list)
+    with Pool(processes=number_of_jobs) as pool:
+        pool.map(generate_rating_curve_metrics, procs_list)
 
     print(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
     aggregate_metrics(output_dir,procs_list,stat_groups)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 06f55b4a0..142f0d387 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -76,16 +76,16 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                     int(test_case.split('_')[0])
 
                     huc = test_case.split('_')[0]
-                    
+
                     for iteration in iteration_list:
-                        
+
                         if iteration == "official":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
                         if iteration == "comparison":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = [dev_comparison]
-        
+
                         for magnitude in ['100yr', '500yr']:
                             for version in versions_to_aggregate:
                                 if '_fr' in version:
@@ -100,7 +100,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                                     calibrated = "no"
                                 version_dir = os.path.join(versions_to_crawl, version)
                                 magnitude_dir = os.path.join(version_dir, magnitude)
-    
+
                                 if os.path.exists(magnitude_dir):
                                     magnitude_dir_list = os.listdir(magnitude_dir)
                                     for f in magnitude_dir_list:
@@ -119,7 +119,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                                                 sub_list_to_append.append(benchmark_source)
                                                 sub_list_to_append.append(extent_config)
                                                 sub_list_to_append.append(calibrated)
-    
+
                                                 list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
@@ -132,9 +132,9 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                     int(test_case.split('_')[0])
 
                     huc = test_case.split('_')[0]
-                    
+
                     for iteration in iteration_list:
-                        
+
                         if iteration == "official":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
@@ -154,7 +154,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                                     calibrated = "yes"
                                 else:
                                     calibrated = "no"
-    
+
                                 version_dir = os.path.join(versions_to_crawl, version)
                                 magnitude_dir = os.path.join(version_dir, magnitude)
                                 if os.path.exists(magnitude_dir):
@@ -166,7 +166,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                                             full_json_path = os.path.join(magnitude_dir, f)
                                             flow = ''
                                             if os.path.exists(full_json_path):
-    
+
                                                 # Get flow used to map.
                                                 flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
                                                 if os.path.exists(flow_file):
@@ -177,7 +177,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                                                             flow = row[1]
                                                         if nws_lid == 'mcc01':
                                                             print(flow)
-    
+
                                                 stats_dict = json.load(open(full_json_path))
                                                 for metric in metrics_to_write:
                                                     sub_list_to_append.append(stats_dict[metric])
@@ -186,7 +186,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                                                 sub_list_to_append.append(benchmark_source)
                                                 sub_list_to_append.append(extent_config)
                                                 sub_list_to_append.append(calibrated)
-    
+
                                                 list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
@@ -321,12 +321,12 @@ def process_alpha_test(args):
 
     # Multiprocess alpha test runs.
     if job_number > 1:
-        pool = Pool(job_number)
-        pool.map(process_alpha_test, procs_list)
+        with Pool(processes=job_number) as pool:
+            pool.map(process_alpha_test, procs_list)
 
     # Do aggregate_metrics.
     print("Creating master metrics CSV...")
-    
+
     if config == 'DEV':
         dev_comparison = fim_version + "_" + special_string
     else:

From c31131716224eccfd937343deb67e226e24dcef3 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Thu, 22 Apr 2021 15:32:10 -0500
Subject: [PATCH 068/359] Update plotting script to not error out if FR or MS
 are not analyzed when using spatial argument

update eval_plots.py to correctly produce spatial datasets if version is not analyzed. For example, if no FR versions are analyzed, instead of throwing an error, it will check to make sure there exists a FR analyzed datasets, if one exists then create spatial data for FR datasets (ble).
---
 CHANGELOG.md                      |   7 ++
 tools/eval_plots.py               | 111 +++++++++++++-----------------
 tools/generate_categorical_fim.py |   4 +-
 3 files changed, 56 insertions(+), 66 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f6143d388..1aa4e5883 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,12 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v3.0.15.5 - 2021-04-20 - [PR #363](https://github.com/NOAA-OWP/cahaba/pull/363)
+
+Prevent eval_plots.py from erroring out when spatial argument enabled if certain datasets not analyzed.
+
+## Changes
+- Add check to make sure analyzed dataset is available prior to creating spatial dataset.
+<br/><br/>
 ## v3.0.15.4 - 2021-04-20 - [PR #356](https://github.com/NOAA-OWP/cahaba/pull/356)
 
 Closing all multiprocessing Pool objects in repo.
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 0327319cb..5ea839d8e 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -605,73 +605,56 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         ###############################################################
         #This section will join ahps metrics to a spatial point layer
         ###############################################################
-        
-        #Get point data for ahps sites
-        #Get metrics for usgs and nws benchmark sources
-        usgs_dataset,sites = all_datasets.get(('usgs','MS'))
-        nws_dataset, sites = all_datasets.get(('nws','MS'))
-        #Append usgs/nws dataframes and filter unnecessary columns and rename remaining.
-        all_ahps_datasets = usgs_dataset.append(nws_dataset)
-        all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source'])
-        all_ahps_datasets.rename(columns = {'benchmark_source':'source'}, inplace = True)
-        
-        #Get spatial data from WRDS
-        #Get metadata from WRDS API
-        select_by = 'nws_lid'
-        selector = list(all_ahps_datasets.nws_lid.unique())
-        metadata_url = f'{API_BASE_URL}/metadata'
-        metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector)
-        #Create geospatial data from WRDS output
-        dictionary, gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = True)
-        #Trim out unecessary columns and rename remaining columns
-        gdf = gdf.filter(['identifiers_nws_lid', 'nws_data_name', 'identifiers_nwm_feature_id','nws_data_wfo','nws_data_state','nws_data_county','geometry'])
-        gdf.rename(columns = {'identifiers_nws_lid':'nws_lid', 'nws_data_name':'lid_name','identifiers_nwm_feature_id':'feature_id','nws_data_wfo':'wfo','nws_data_state':'state','nws_data_county':'county','HUC8':'huc8'}, inplace = True)
-        
-        #Join spatial data to metric data        
-        gdf['nws_lid'] = gdf['nws_lid'].str.lower()        
-        joined = gdf.merge(all_ahps_datasets, on = 'nws_lid')
-        #Project to VIZ projection and write to file
-        joined = joined.to_crs(VIZ_PROJECTION)
-        joined.to_file(Path(workspace) / 'fim_performance_points.shp')
-        
-        '''
-        ###############################################################
-        #If user wants to append information such as what maps or flows were used for evaluation. This is already tested.
-        #User must supply the extent layer generated from preprocessing NWS/USGS datasets.
-        ###############################################################
-        #Read extent layer to GeoDataFrame and drop the geometry column
-        evaluated_ahps_extent = gpd.read_file(/Path/to/extent/layer/generated/during/preprocessing)        
-        evaluated_ahps_extent.drop(columns = ['geometry'], inplace = True)
-        #Re-arrange dataset to get flows used for evaluation
-        flows = pd.melt(evaluated_ahps_extent, id_vars = ['nws_lid','source'], value_vars = ['action_Q','minor_Q','moderate_Q','major_Q'], var_name = 'magnitude', value_name = 'eval_Q')
-        flows['magnitude'] = flows['magnitude'].str.split('_', 1, expand = True)
-        #Re-arrange dataset to get maps used for evaluation
-        maps = pd.melt(evaluated_ahps_extent, id_vars = ['nws_lid','source'], value_vars = ['action','minor','moderate','major'], var_name = 'magnitude', value_name = 'eval_maps')
-        maps['eval_maps'] = maps['eval_maps'].str.split('\\').str[-1]
-        #Merge flows and maps into single DataFrame
-        flows_maps = pd.merge(flows,maps, how = 'left', left_on = ['nws_lid','source','magnitude'], right_on = ['nws_lid','source','magnitude'])        
-        # combine flows_maps to spatial layer (gdf)
-        joined = joined.merge(flows_maps, left_on = ['nws_lid','magnitude','source'], right_on = ['nws_lid','magnitude','source'])
-        #Write to file
-        joined.to_file(Path(workspace)/'fim_performance_points.shp')
-        '''
+        if all_datasets.get(('nws','MS')) and all_datasets.get(('usgs','MS')):
+            #Get point data for ahps sites
+            #Get metrics for usgs and nws benchmark sources
+            usgs_dataset,sites = all_datasets.get(('usgs','MS'))
+            nws_dataset, sites = all_datasets.get(('nws','MS'))
+            #Append usgs/nws dataframes and filter unnecessary columns and rename remaining.
+            all_ahps_datasets = usgs_dataset.append(nws_dataset)
+            all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source'])
+            all_ahps_datasets.rename(columns = {'benchmark_source':'source'}, inplace = True)
+            
+            #Get spatial data from WRDS
+            #Get metadata from WRDS API
+            select_by = 'nws_lid'
+            selector = list(all_ahps_datasets.nws_lid.unique())
+            metadata_url = f'{API_BASE_URL}/metadata'
+            metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector)
+            #Create geospatial data from WRDS output
+            dictionary, gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = True)
+            #Trim out unecessary columns and rename remaining columns
+            gdf = gdf.filter(['identifiers_nws_lid', 'nws_data_name', 'identifiers_nwm_feature_id','nws_data_wfo','nws_data_state','nws_data_county','geometry'])
+            gdf.rename(columns = {'identifiers_nws_lid':'nws_lid', 'nws_data_name':'lid_name','identifiers_nwm_feature_id':'feature_id','nws_data_wfo':'wfo','nws_data_state':'state','nws_data_county':'county','HUC8':'huc8'}, inplace = True)
+            
+            #Join spatial data to metric data        
+            gdf['nws_lid'] = gdf['nws_lid'].str.lower()        
+            joined = gdf.merge(all_ahps_datasets, on = 'nws_lid')
+            #Project to VIZ projection and write to file
+            joined = joined.to_crs(VIZ_PROJECTION)
+            joined.to_file(Path(workspace) / 'fim_performance_points.shp')
+        else:
+            print('NWS/USGS MS datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a MS version.')
+            
         ################################################################
         #This section joins ble (FR) metrics to a spatial layer of HUCs.
         ################################################################
-        #Read in HUC spatial layer
-        wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')
-        #Select BLE, FR dataset.
-        ble_dataset, sites = all_datasets.get(('ble','FR'))              
-        #Join metrics to HUC spatial layer
-        wbd_with_metrics = wbd_gdf.merge(ble_dataset, how = 'inner', left_on = 'HUC8', right_on = 'huc')
-        #Filter out unnecessary columns
-        wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source','geometry'])
-        wbd_with_metrics.rename(columns = {'benchmark_source':'source'}, inplace = True )
-        #Project to VIZ projection
-        wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION)
-        #Write out to file
-        wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
-        
+        if all_datasets.get(('ble','FR')):            
+            #Select BLE, FR dataset.
+            ble_dataset, sites = all_datasets.get(('ble','FR'))         
+            #Read in HUC spatial layer
+            wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')             
+            #Join metrics to HUC spatial layer
+            wbd_with_metrics = wbd_gdf.merge(ble_dataset, how = 'inner', left_on = 'HUC8', right_on = 'huc')
+            #Filter out unnecessary columns
+            wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source','geometry'])
+            wbd_with_metrics.rename(columns = {'benchmark_source':'source'}, inplace = True )
+            #Project to VIZ projection
+            wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION)
+            #Write out to file
+            wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
+        else:
+            print('BLE FR datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a FR version')
 
 #######################################################################
 if __name__ == '__main__':
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index cea992df1..c572ee6ee 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -90,7 +90,7 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     # Generate CatFIM flow files
     print('Creating flow files')
     start = time.time()
-    subprocess.call(['python3','foss_fim/tools/generate_categorical_fim_flows.py', '-w' , str(output_flows_dir), '-u', nwm_us_search, '-d', nwm_ds_search])
+    subprocess.call(['python3','/foss_fim/tools/generate_categorical_fim_flows.py', '-w' , str(output_flows_dir), '-u', nwm_us_search, '-d', nwm_ds_search])
     end = time.time()
     elapsed_time = (end-start)/60
     print(f'Finished creating flow files in {elapsed_time} minutes')
@@ -98,7 +98,7 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     # Generate CatFIM mapping
     print('Begin mapping')
     start = time.time()
-    subprocess.call(['python3','foss_fim/tools/generate_categorical_fim_mapping.py', '-r' , str(fim_run_dir), '-s', str(output_flows_dir), '-o', str(output_mapping_dir), '-j', str(number_of_jobs)])
+    subprocess.call(['python3','/foss_fim/tools/generate_categorical_fim_mapping.py', '-r' , str(fim_run_dir), '-s', str(output_flows_dir), '-o', str(output_mapping_dir), '-j', str(number_of_jobs)])
     end = time.time()
     elapsed_time = (end-start)/60
     print(f'Finished mapping in {elapsed_time} minutes')

From 99c197b5ec6cbddb69d58795b0103980e8075c18 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Tue, 27 Apr 2021 12:42:59 -0500
Subject: [PATCH 069/359] Implement CatFIM threshold flows to Sierra test and
 add AHPS benchmark preprocessing scripts.

Implement CatFIM threshold flows to Sierra test and add AHPS benchmark preprocessing scripts.

- Produce CatFIM flows file when running rating_curve_get_usgs_gages.py.
- Several scripts to preprocess AHPS benchmark data. Requires numerous file dependencies not available through Cahaba.
- Modify rating_curve_comparison.py to ingest CatFIM threshold flows in calculations.
- Modify eval_plots.py to save all site specific bar plots in same parent directory instead of in subdirectories.
- Add variables to env.template for AHPS benchmark preprocessing.

This resolves #341, resolves #361, and resolves #357.
---
 CHANGELOG.md                            |  15 +
 tools/.env.template                     |   2 +
 tools/eval_plots.py                     |   4 +-
 tools/preprocess_ahps_nws.py            | 355 ++++++++++++++++++
 tools/preprocess_ahps_usgs.py           | 471 ++++++++++++++++++++++++
 tools/preprocess_download_usgs_grids.py | 162 ++++++++
 tools/rating_curve_comparison.py        |  27 +-
 tools/rating_curve_get_usgs_curves.py   |  79 +++-
 tools/tools_shared_functions.py         | 315 +++++++++++++++-
 9 files changed, 1411 insertions(+), 19 deletions(-)
 create mode 100644 tools/preprocess_ahps_nws.py
 create mode 100644 tools/preprocess_ahps_usgs.py
 create mode 100644 tools/preprocess_download_usgs_grids.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1aa4e5883..1cc4cc499 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v3.0.15.6 - 2021-04-23 - [PR #365](https://github.com/NOAA-OWP/cahaba/pull/365)
+
+Implement CatFIM threshold flows to Sierra test and add AHPS benchmark preprocessing scripts.
+
+## Additions
+- Produce CatFIM flows file when running `rating_curve_get_usgs_gages.py`.
+- Several scripts to preprocess AHPS benchmark data. Requires numerous file dependencies not available through Cahaba.
+
+## Changes
+- Modify `rating_curve_comparison.py` to ingest CatFIM threshold flows in calculations.
+- Modify `eval_plots.py` to save all site specific bar plots in same parent directory instead of in subdirectories.
+- Add variables to `env.template` for AHPS benchmark preprocessing.
+
+<br/><br/>
 ## v3.0.15.5 - 2021-04-20 - [PR #363](https://github.com/NOAA-OWP/cahaba/pull/363)
 
 Prevent eval_plots.py from erroring out when spatial argument enabled if certain datasets not analyzed.
@@ -7,6 +21,7 @@ Prevent eval_plots.py from erroring out when spatial argument enabled if certain
 ## Changes
 - Add check to make sure analyzed dataset is available prior to creating spatial dataset.
 <br/><br/>
+
 ## v3.0.15.4 - 2021-04-20 - [PR #356](https://github.com/NOAA-OWP/cahaba/pull/356)
 
 Closing all multiprocessing Pool objects in repo.
diff --git a/tools/.env.template b/tools/.env.template
index ed83c8113..048c2283c 100644
--- a/tools/.env.template
+++ b/tools/.env.template
@@ -2,3 +2,5 @@ API_BASE_URL=
 EVALUATED_SITES_CSV=
 WBD_LAYER=
 NWM_FLOWS_MS=
+USGS_METADATA_URL=
+USGS_DOWNLOAD_URL=
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 5ea839d8e..860c38382 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -570,10 +570,10 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
 
         #If enabled, write out barplots of CSI for individual sites.
         if site_barplots:
+            individual_dirs = output_workspace / 'individual'
+            individual_dirs.mkdir(parents = True, exist_ok = True)
             subset = dataset.groupby(base_resolution)
             for site_name, site_data in subset:
-                individual_dirs = output_workspace / 'individual' / str(site_name)
-                individual_dirs.mkdir(parents = True, exist_ok = True)
                 site_file = individual_dirs / f'csi_{str(site_name)}_{dataset_name}_{configuration.lower()}.png'
                 barplot(dataframe = site_data, x_field = 'magnitude', x_order = magnitude_order, y_field = 'CSI', hue_field = 'version', ordered_hue = version_order, title_text = f'{str(site_name).upper()} FIM Scores', fim_configuration = configuration, textbox_str = False, simplify_legend = True, dest_file = site_file)
 
diff --git a/tools/preprocess_ahps_nws.py b/tools/preprocess_ahps_nws.py
new file mode 100644
index 000000000..e2db2b47d
--- /dev/null
+++ b/tools/preprocess_ahps_nws.py
@@ -0,0 +1,355 @@
+#!/usr/bin/env python3
+import numpy as np
+import pathlib
+from pathlib import Path
+import pandas as pd
+import geopandas as gpd
+import rasterio
+from collections import defaultdict
+from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
+import argparse
+from dotenv import load_dotenv
+import os
+import traceback
+import sys
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+
+def get_env_paths():
+    load_dotenv()
+    #import variables from .env file
+    API_BASE_URL = os.getenv("API_BASE_URL")
+    EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+    WBD_LAYER = os.getenv("WBD_LAYER")
+    return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER
+
+########################################################
+#Preprocess AHPS NWS
+#This script will work on NWS AHPS fim data (some assumptions made about the data structure). 
+#Provide a source directory path (source_dir) where all NWS AHPS FIM data is located. NWS source data was previously downloaded and extracted. Some data is buried through several layers of subfolders in the source data. In general, the downloaded datasets were unzipped and starting from where the folder name was the AHPS code, this was copied and pasted into a new directory which is the source_dir. 
+#Provide a destination directory path (destination) which is where all outputs are located.
+#Provide a reference raster path.
+########################################################
+# source_dir = Path(r'path/to/nws/downloads')
+# destination = Path(r'path/to/preprocessed/nws/data')
+# reference_raster= Path(r'path/to/reference raster') 
+
+def preprocess_nws(source_dir, destination, reference_raster):
+    source_dir = Path(source_dir)
+    destination = Path(destination)
+    reference_raster = Path(reference_raster)
+    metadata_url = f'{API_BASE_URL}/metadata' 
+    threshold_url = f'{API_BASE_URL}/nws_threshold'
+    rating_curve_url = f'{API_BASE_URL}/rating_curve'
+    log_file = destination / 'log.txt'
+    
+    #Write a run-time log file
+    destination.mkdir(parents = True, exist_ok = True)
+    log_file = destination / 'log.txt'
+    f = open(log_file, 'a+')
+    
+    #Define distance (in miles) to search for nwm segments
+    nwm_ds_search = 10
+    nwm_us_search = 10
+    #The NWS data was downloaded and unzipped. The ahps folder (with 5 digit code as folder name) was cut and pasted into a separate directory. So the ahps_codes iterates through that parent directory to get all of the AHPS codes that have data. 
+    ahps_codes = [i.name for i in source_dir.glob('*') if i.is_dir() and len(i.name) == 5]
+    #Get mainstems NWM segments
+    #Workaround for sites in 02030103 and 02030104, many are not rfc_forecast_point = True
+    list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
+    ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
+    
+    #Find depth grid subfolder
+    for code in ahps_codes:
+        f.write(f'{code} : Processing\n')
+        print(f'processing {code}')
+        #'mnda2' is in Alaska outside of NWM domain.
+        if code in ['mnda2']:
+            f.write(f'{code} : skipping because outside of NWM domain\n')
+            continue
+           
+        #Get metadata of site and search for NWM segments x miles upstream/x miles downstream
+        select_by = 'nws_lid'
+        selector = [code]    
+        metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        metadata = metadata_list[0]  
+    
+        #Assign huc to site using FIM huc layer.
+        dictionary, out_gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = False)
+        [huc] = list(dictionary.keys())
+      
+        #Get thresholds for action, minor, moderate, major. If no threshold data present, exit.
+        #The threshold flows source will dictate what rating curve (and datum) to use as it uses a decision tree (USGS priority then NRLDB)
+        #In multiple instances a USGS ID is given but then no USGS rating curve or in some cases no USGS datum is supplied.
+        select_by = 'nws_lid'
+        selector = code
+        stages, flows =get_thresholds(threshold_url, select_by, selector, threshold = 'all')
+    
+        #Make sure at least one valid threshold is supplied from WRDS.
+        threshold_categories = ['action','minor','moderate','major'] 
+        if not any([stages[threshold] for threshold in threshold_categories]):
+            f.write(f'{code} : skipping because no threshold stages available\n')
+            continue
+        
+        #determine source of interpolated threshold flows, this will be the rating curve that will be used.
+        rating_curve_source = flows.get('source')
+        if rating_curve_source is None:
+            f.write(f'{code} : skipping because no rating curve source\n')
+            continue
+        
+        #Get the datum and adjust to NAVD if necessary.
+        nws, usgs = get_datum(metadata)
+        datum_data = {}
+        if rating_curve_source == 'USGS Rating Depot':
+            datum_data = usgs
+        elif rating_curve_source == 'NRLDB':
+            datum_data = nws
+    
+        #If datum not supplied, skip to new site
+        datum = datum_data.get('datum', None)
+        if datum is None:
+            f.write(f'{code} : skipping because site is missing datum\n')
+            continue        
+
+        #Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29 conversion to NAVD88
+        # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7, stkn4, wlln7 
+        # Assumed to be NAD83 (no info from USGS or NWS data): dlrt2, eagi1, eppt2, jffw3, ldot2, rgdt2
+        if code in ['bgwn7', 'dlrt2','eagi1','eppt2','fatw3','jffw3','ldot2','mnvn4','nhpp1','pinn4','rgdt2','rgln4','rssk1','sign4','smfn7','stkn4','wlln7' ]:
+            datum_data.update(crs = 'NAD83')
+        
+        #Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure datum reported in NAVD88. If NGVD29 it is converted to NAVD88.
+        #bgwn7, eagi1 vertical datum unknown, assume navd88
+        #fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
+        #wlln7 USGS data indicates vcs is NGVD29 (USGS and NWS info agree on datum value).
+        if code in ['bgwn7','eagi1','fatw3']:
+            datum_data.update(vcs = 'NAVD88')
+        elif code == 'wlln7':
+            datum_data.update(vcs = 'NGVD29')
+        
+        #Adjust datum to NAVD88 if needed
+        if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929', 'NGVD,1929']:
+            #Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
+            datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
+            datum88 = round(datum + datum_adj_ft, 2)
+        else:
+            datum88 = datum
+    
+        #get entire rating curve, same source as interpolated threshold flows (USGS Rating Depot first then NRLDB rating curve).
+        if rating_curve_source == 'NRLDB':
+            site = [code]
+        elif rating_curve_source == 'USGS Rating Depot':
+            site = [metadata.get('identifiers').get('usgs_site_code')]
+            
+        rating_curve = get_rating_curve(rating_curve_url, site)
+    
+        #Add elevation fields to rating curve
+        #Add field with vertical coordinate system
+        vcs = datum_data['vcs']
+        if not vcs:
+            vcs = 'Unspecified, Assumed NAVD88'
+        rating_curve['vcs'] = vcs
+    
+        #Add field with original datum
+        rating_curve['datum'] = datum
+        
+        #If VCS is NGVD29 add rating curve elevation (in NGVD) as well as the NAVD88 datum
+        if vcs in ['NGVD29', 'NGVD 1929']:        
+            #Add field with raw elevation conversion (datum + stage)
+            rating_curve['elevation_ngvd29'] = rating_curve['stage'] + datum
+            #Add field with adjusted NAVD88 datum
+            rating_curve['datum_navd88'] = datum88
+    
+        #Add field with NAVD88 elevation
+        rating_curve['elevation_navd88'] = rating_curve['stage'] + datum88
+    
+        
+        #Search through ahps directory find depth grid folder
+        parent_path = source_dir / code    
+          
+        #Work around for bgwn7 and smit2 where grids were custom created from polygons (bgwn7-no grids, smit2 - no projection and applying projection from polygons had errors)
+        if code in ['bgwn7', 'smit2']:
+            [grids_dir] = [directory for directory in parent_path.glob('*custom*') if directory.is_dir()]
+        else:
+            #Find the directory containing depth grids. Assumes only one directory will be returned.
+            [grids_dir] = [directory for directory in parent_path.glob('*depth_grid*') if directory.is_dir()]        
+            
+        #Get grids (all NWS ESRI grids were converted to Geotiff)
+        grid_paths = [grids for grids in grids_dir.glob('*.tif*') if grids.suffix in ['.tif', '.tiff']]
+        grid_names = [name.stem for name in grid_paths]
+        #If grids are present, interpolate a flow for the grid.
+        if grid_paths:
+            #Construct Dataframe containing grid paths, names, datum, code
+            df = pd.DataFrame({'code': code, 'path':grid_paths, 'name': grid_names, 'datum88': datum88})
+            #Determine elevation from the grid name. All elevations are assumed to be in NAVD88 based on random inspection of AHPS inundation website layers.
+            df['elevation'] = df['name'].str.replace('elev_', '', case = False).str.replace('_','.').astype(float)             
+            # Add a stage column using the datum (in NAVD88). Stage is rounded to the nearest 0.1 ft.
+            df['stage'] = round(df['elevation'] - df['datum88'],1)                                
+            #Sort stage in ascending order
+            df.sort_values(by = 'elevation', ascending = True, inplace = True)
+            #Interpolate flow from the rating curve using the elevation_navd88 values, if value is above or below the rating curve assign nan.
+            df['flow'] = np.interp(df['elevation'], rating_curve['elevation_navd88'], rating_curve['flow'], left = np.nan, right = np.nan)        
+            #Assign flow source to reflect interpolation from rc
+            df['flow_source'] = f'interpolated from {rating_curve_source} rating curve'
+
+        else: 
+            f.write(f'{code} : Site has no benchmark grids\n')
+       
+        #Select the appropriate threshold grid for evaluation. Using the supplied threshold stages and the calculated map stages. 
+        grids,grid_flows = select_grids(df, stages, datum88, 1.1)
+    
+        #workaroud for bigi1 and eag1 which have gridnames based on flows (not elevations)
+        if code in ['eagi1', 'bigi1']:
+            #Elevation is really flows (due to file names), assign this to stage
+            df['flow'] = df['elevation']
+            df['stage'] = df['elevation']
+            #Select grids using flows
+            grids, grid_flows = select_grids(df, flows, datum88, 500)
+            f.write(f'{code} : Site workaround grids names based on flows not elevation\n')
+    
+        #Obtain NWM segments that are on ms to apply flows
+        segments = get_nwm_segs(metadata)
+        site_ms_segs = set(segments).intersection(ms_segs)
+        segments = list(site_ms_segs)
+    
+        #Write out boolean benchmark raster and flow file
+        try:
+            #for each threshold
+            for i in ['action', 'minor', 'moderate', 'major']:
+                #Obtain the flow and grid associated with threshold.
+                flow = grid_flows[i]
+                grid = grids[i]
+                extent = grids['extent']
+                #Make sure that flow and flow grid are valid
+                if not grid in ['No Map', 'No Threshold', 'No Flow']:
+                    #define output directory (to be created later)
+                    outputdir = destination / huc / code / i
+                                  
+                    #Create Binary Grids, first create domain of analysis, then create binary grid
+                    
+                    #Domain extent is largest floodmap in the static library WITH holes filled
+                    filled_domain_raster = outputdir.parent / f'{code}_extent.tif'
+    
+                    #Open benchmark data as a rasterio object.
+                    benchmark = rasterio.open(grid)
+                    benchmark_profile = benchmark.profile   
+    
+                    #Open extent data as rasterio object
+                    domain = rasterio.open(extent)
+                    domain_profile = domain.profile
+    
+                    #if grid doesn't have CRS, then assign CRS using a polygon from the ahps inundation library
+                    if not benchmark.crs:
+                        #Obtain crs of the first polygon inundation layer associated with ahps code. Assumes only one polygon* subdirectory and assumes the polygon directory has at least 1 inundation shapefile.
+                        [ahps_polygons_directory] = [directory for directory in parent_path.glob('*polygon*') if directory.is_dir()]
+                        shapefile_path = list(ahps_polygons_directory.glob('*.shp'))[0]
+                        shapefile = gpd.read_file(shapefile_path)
+                        #Update benchmark and domain profiles with crs from shapefile. Assumed that benchmark/extent have same crs.
+                        benchmark_profile.update(crs = shapefile.crs)  
+                        domain_profile.update(crs = shapefile.crs)
+    
+                    #Create a domain raster if it does not exist.
+                    if not filled_domain_raster.exists():
+                        #Domain should have donut holes removed
+                        process_extent(domain, domain_profile, output_raster = filled_domain_raster)
+    
+    
+                    #Open domain raster as rasterio object
+                    filled_domain = rasterio.open(filled_domain_raster)
+                    filled_domain_profile = filled_domain.profile
+    
+                    #Create the binary benchmark raster                        
+                    boolean_benchmark, boolean_profile = process_grid(benchmark, benchmark_profile, filled_domain, filled_domain_profile, reference_raster)    
+                    
+                    #Output binary benchmark grid and flow file to destination
+                    outputdir.mkdir(parents = True, exist_ok = True)  
+                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_depth_{i}.tif')
+                    
+                    with rasterio.Env():
+                        with rasterio.open(output_raster, 'w', **boolean_profile) as dst:
+                            dst.write(boolean_benchmark,1) 
+                    
+                    #Close datasets
+                    domain.close()
+                    filled_domain.close()
+                    benchmark.close()
+                    
+                    #Create the guts of the flow file.
+                    flow_info = flow_data(segments,flow)
+                    #Write out the flow file to csv
+                    output_flow_file = outputdir / (f'ahps_{code}_huc_{huc}_flows_{i}.csv')
+                    flow_info.to_csv(output_flow_file, index = False)
+                    
+        except Exception as e:
+            f.write(f'{code} : Error preprocessing benchmark\n{repr(e)}\n')
+            f.write(traceback.format_exc())
+            f.write('\n')
+            print(traceback.format_exc())               
+        #Process extents, only create extent if ahps code subfolder is present in destination directory.
+        ahps_directory = destination / huc / code
+        if ahps_directory.exists():
+            #Delete extent raster
+            filled_extent = ahps_directory / f'{code}_extent.tif' 
+            if filled_extent.exists:
+                filled_extent.unlink()              
+    
+            #Populate attribute information for site
+            grids_attributes = pd.DataFrame(data=grids.items(), columns = ['magnitude','path'])
+            flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude','grid_flow_cfs'])
+            threshold_attributes = pd.DataFrame(data=stages.items(), columns = ['magnitude','magnitude_stage'])       
+            #merge dataframes
+            attributes = grids_attributes.merge(flows_attributes, on = 'magnitude')
+            attributes = attributes.merge(threshold_attributes, on = 'magnitude')
+            attributes = attributes.merge(df[['path','stage','elevation', 'flow_source']], on = 'path')        
+            #Strip out sensitive paths and convert magnitude stage to elevation
+            attributes['path'] = attributes['path'].apply(lambda x :Path(x).name)
+            attributes['magnitude_elev_navd88']=(datum88 + attributes['magnitude_stage']).astype(float).round(1)
+            #Add general site information
+            attributes['nws_lid'] = code
+            attributes['wfo'] = metadata['nws_data']['wfo']
+            attributes['rfc'] = metadata['nws_data']['rfc']
+            attributes['state'] = metadata['nws_data']['state']
+            attributes['huc'] = huc
+            #Rename and Reorder columns
+            attributes.rename(columns = {'path':'grid_name', 'flow_source':'grid_flow_source','stage':'grid_stage','elevation':'grid_elev_navd88'}, inplace = True)        
+            attributes = attributes[['nws_lid','wfo','rfc','state','huc','magnitude','magnitude_stage','magnitude_elev_navd88','grid_name','grid_stage','grid_elev_navd88', 'grid_flow_cfs','grid_flow_source']]        
+            #Save attributes to csv
+            attributes.to_csv(ahps_directory / f'{code}_attributes.csv', index = False) 
+    
+            #Write the rating curve to a file
+            rating_curve_output = ahps_directory / (f'{code}_rating_curve.csv')
+            rating_curve['lat'] = datum_data['lat']
+            rating_curve['lon'] = datum_data['lon']
+            rating_curve.to_csv(rating_curve_output, index = False)
+            
+            #Write the interpolated flows to file
+            df_output = ahps_directory / (f'{code}_interpolated_flows.csv')
+            df.to_csv(df_output, index = False)
+        
+        else: 
+            f.write(f'{code} : Unable to evaluate site, missing all flows\n')
+    
+    #Close log file.
+    f.close()
+    
+    #Combine all attribute files
+    attribute_files = list(destination.rglob('*_attributes.csv'))
+    all_attributes = pd.DataFrame()
+    for i in attribute_files:
+        attribute_df = pd.read_csv(i, dtype={'huc':str})
+        all_attributes = all_attributes.append(attribute_df)
+    
+    if not all_attributes.empty:
+        all_attributes.to_csv(destination / 'attributes.csv', index = False)    
+    return
+
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
+    parser.add_argument('-s', '--source_dir', help = 'Workspace where all source data is located.', required = True)
+    parser.add_argument('-d', '--destination',  help = 'Directory where outputs are to be stored', required = True)
+    parser.add_argument('-r', '--reference_raster', help = 'reference raster used for benchmark raster creation', required = True)
+    args = vars(parser.parse_args())
+    
+
+    #Run get_env_paths and static_flow_lids
+    API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER = get_env_paths()
+    preprocess_nws(**args)
\ No newline at end of file
diff --git a/tools/preprocess_ahps_usgs.py b/tools/preprocess_ahps_usgs.py
new file mode 100644
index 000000000..f465ba682
--- /dev/null
+++ b/tools/preprocess_ahps_usgs.py
@@ -0,0 +1,471 @@
+#!/usr/bin/env python3
+
+import numpy as np
+import pathlib
+from pathlib import Path
+import pandas as pd
+import geopandas as gpd
+import rasterio
+import requests
+from collections import defaultdict
+from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
+import argparse
+from dotenv import load_dotenv
+import os
+import sys
+sys.path.append('/foss_fim/src')
+import traceback
+from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+
+
+def get_env_paths():
+    load_dotenv()
+    #import variables from .env file
+    API_BASE_URL = os.getenv("API_BASE_URL")
+    EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+    WBD_LAYER = os.getenv("WBD_LAYER")
+    USGS_METADATA_URL = os.getenv("USGS_METADATA_URL")   
+    return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER, USGS_METADATA_URL
+###############################################################################
+#Get USGS Site metadata
+###############################################################################
+def usgs_site_metadata(code):
+    '''
+    Retrieves site metadata from USGS API and saves output as dictionary. Information used includes shortname and site number.
+    
+    Parameters
+    ----------
+    code : STR
+        AHPS code.
+    USGS_METADATA_URL : STR
+        URL for USGS datasets.
+
+    Returns
+    -------
+    site_metadata : DICT
+        Output metadata for an AHPS site.
+    '''
+    # Make sure code is lower case
+    code = code.lower()
+    # Get site metadata from USGS API using ahps code
+    site_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/sites/MapServer/0/query?where=AHPS_ID+%3D+%27{code}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
+    #Get data from API
+    response = requests.get(site_url)
+    #If response is valid, then get metadata and save to dictionary
+    if response.ok:
+        response_json = response.json()
+        site_metadata = response_json['features'][0]['attributes']
+    return site_metadata
+
+###############################################################################
+#Get USGS grid metadata
+###############################################################################
+def usgs_grid_metadata(code, has_grid_override = False):    
+    '''
+    Given an ahps code, retrieve the site metadata (using usgs_site_metadata) and then use that information to obtain metadata about available grids. Information includes elevation, stage, and flow for each grid. 
+
+    Parameters
+    ----------
+    code : STR
+        AHPS code.
+
+    Returns
+    -------
+    appended_dictionary : DICT
+        Dictionary of metadata for each available inundation grid including grid id, flows, elevations, grid name for each inundation grid.
+    '''
+    #Make sure code is in lower case
+    code = code.lower()   
+    # Get site_metadata
+    site_metadata = usgs_site_metadata(code)    
+    #From site metadata get the SHORT_NAME, SITE_NO, and 'MULTI_SITE', 'HAS_GRIDS' key values
+    short_name = site_metadata['SHORT_NAME']
+    site_no = site_metadata['SITE_NO']
+    has_grids = site_metadata['HAS_GRIDS']
+    #There is at least one site (kilo1) that doesn't have grids but polygons are available which have been converted grids.
+    if has_grid_override:
+        has_grids = 1
+    multi_site = site_metadata['MULTI_SITE']    
+    #Grid metadata located at one of three URLs
+    if multi_site == 0 and has_grids == 1:
+        grids_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/floodExtents/MapServer/0/query?where=USGSID+%3D+%27{site_no}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
+    elif multi_site > 0 and multi_site < 3 and has_grids == 1:
+        grids_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/floodExtentsMulti/MapServer/0/query?where=USGSID_1+%3D+%27{site_no}%27+OR+USGSID_2+%3D+%27{site_no}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
+    elif multi_site == 3 and has_grids == 1:
+        grids_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/floodExtentsThreeSites/MapServer/0/query?where=USGSID_1+%3D+%27{site_no}%27+OR+USGSID_2+%3D+%27{site_no}%27+OR+USGSID_3+%3D+%27{site_no}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'            
+    #Only get metadata on grids if site has grids available
+    if has_grids == 1:
+        #Get data from API
+        response = requests.get(grids_url)
+        #If response is valid then combine metadata on all grids into a single dictionary and write out to DataFrame.
+        if response.ok:
+            response_json =response.json()
+            metadata = response_json['features']
+            appended_dictionary = {}
+            for i in metadata:
+                dictionary = i['attributes']
+                gridname = short_name + '_' + str(dictionary['GRIDID']).zfill(4)
+                appended_dictionary[gridname] = dictionary
+    else: 
+        appended_dictionary = {}
+    return appended_dictionary
+
+
+########################################################
+#Preprocess USGS FIM
+#This script will work on USGS FIM datasets. 
+#Provide source directory path (source_dir) where all USGS FIM data is located. This data was previously downloaded from USGS urls.
+#Provide a destination directory path (destination) where all outputs are located.
+#Provide a reference raster path.
+########################################################
+#source_dir = Path(r'path/to/usgs/downloads')
+#destination = Path(r'path/to/preprocessed/usgs/data')
+#reference_raster= Path(r'path/to/reference raster') 
+def preprocess_usgs(source_dir, destination, reference_raster):
+    '''
+    Preprocess USGS AHPS datasets.
+
+    Parameters
+    ----------
+    source_dir : str
+        Path to USGS Benchmark Datasets (AHPS)
+    destination : str
+        Path to output directory of preprocessed datasets.
+    reference_raster : str
+        Path to reference raster for benchmark binary raster creation.
+
+    Returns
+    -------
+    None.
+
+    '''
+    
+    source_dir = Path(source_dir)
+    destination = Path(destination)
+    reference_raster = Path(reference_raster)
+    metadata_url = f'{API_BASE_URL}/metadata' 
+    threshold_url = f'{API_BASE_URL}/nws_threshold'
+    rating_curve_url = f'{API_BASE_URL}/rating_curve'
+    
+    #Write log file
+    destination.mkdir(parents=True, exist_ok = True)
+    log_file = destination / 'log.txt'
+    f = open(log_file, 'a+')
+    
+    #Define distance (in miles) to search for nwm segments
+    nwm_ds_search = 10
+    nwm_us_search = 10
+    #Need a list of AHPS codes. See "ahps_dictionaries" for method of getting this list.
+    ahps_codes = [folder.name for folder in source_dir.glob('*') if len(folder.name) == 5]
+    
+    #Get mainstems NWM segments
+    #Workaround for sites in 02030103 and 02030104, many are not rfc_forecast_point = True
+    #Import list of evaluated sites
+    list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
+    ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
+    
+    for code in ahps_codes:
+        f.write(f'{code} : Processing\n')
+        print(f'processing {code}')
+        #For a given code, find all inundation grids under that code.
+        code = code.lower()
+          
+        #Get metadata of site and search for NWM segments x miles upstream/x miles downstream
+        select_by = 'nws_lid'
+        selector = [code]    
+        metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        metadata = metadata_list[0]
+        
+        #Assign huc to site using FIM huc layer.
+        dictionary, out_gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = False)
+        [huc] = list(dictionary.keys())
+    
+        #There are 12 sites with special issues such as these don't have any crs coordinates and grid/polygon data don't align or missing grid data but polygons are available.
+        #Sites with no grid data but polygon data --> cfmm8, kilo1
+        #Sites with no projection assigned to grid and polygon/grid don't align --> stak1, nmso1, nori3, sasi3
+        #Sites with reprojection issues using rasterio (manually reprojected with ESRI) --> kcdm7, knym7, mcri2, ptvn6, tmai4
+        #Sites with incomplete grids (used polys to convert to grids) --> 'roun6'
+        ahps_dir = source_dir / code / 'depth_grids'
+        if code in ['cfmm8','kilo1','stak1', 'sasi3', 'nori3', 'nmso1', 'kcdm7', 'knym7', 'mcri2','ptvn6','tmai4', 'roun6']:        
+            f.write(f'{code} : Custom workaround related to benchmark data (mismatch crs, no grid data, etc)\n')
+            ahps_dir = source_dir / code / 'custom'
+    
+        #Get thresholds (action/minor/moderate/major flows and stages), if not available exit.
+        #For USGS many sites may not have rating curves but the threshold stages are available.
+    
+        select_by = 'nws_lid'
+        selector = code
+        stages, flows =get_thresholds(threshold_url, select_by, selector, threshold = 'all')
+       
+        #Make sure at least one valid threshold is supplied from WRDS.
+        threshold_categories = ['action','minor','moderate','major'] 
+        if not any([stages[threshold] for threshold in threshold_categories]):
+            f.write(f'{code} : Skipping because no threshold stages available\n')
+            continue
+    
+        #We need to adjust stages to elevations using the datum adjustment. This next section finds the datum adjustment.
+        #determine primary source for interpolated threshold flows (USGS first then NRLDB). This will dictate what rating curve to pull.
+        rating_curve_source = flows['source']
+        #Workaround for sites that don't have rating curve but do have flows specified (USGS only). Assign rating_curve_source to 'USGS Rating Depot' manually inspected all of these sites and USGS datum is available and will be used.
+        if code in ['bdxt1','ccti3', 'fnnm7', 'mtao1', 'nfsi3', 'omot1' , 'sbrn1', 'vron4', 'watv1']:
+            rating_curve_source = 'USGS Rating Depot'         
+    
+        #Get the datum and adjust to NAVD if necessary.
+        nws, usgs = get_datum(metadata)
+        datum_data = {}
+        if rating_curve_source == 'USGS Rating Depot':
+            datum_data = usgs
+        elif rating_curve_source == 'NRLDB':
+            datum_data = nws
+        
+        #If datum not supplied, skip to new site
+        datum = datum_data.get('datum', None)
+        if datum is None:
+            f.write(f'{code}: Skipping because of missing datum\n')
+            continue        
+
+        #Custom workaround, some sites have poorly defined crs. CRS requuired for ngvd to navd conversions
+        # Assumed NAVD88 (no info from USGS or NWS metadata): kynm7, ksvm7, yesg1
+        # Assigned NAVD88 because USGS metadata indicates NAD83: arnm7, grfi2, kctm7, nast1, nhri3, roun6, vllm7
+        # Assigned NAVD88 (reported NAVD 1988): cmtl1
+        if code in ['arnm7', 'cmtl1','grfi2','kctm7','knym7','ksvm7','nast1','nhri3','roun6','vllm7','yesg1']:
+            #Update crs to NAD83 (some are assumed, others have USGS info indicating NAD83 crs)
+            datum_data.update(crs = 'NAD83')
+        
+        #Adjust datum to NAVD88 if needed (Assumes that if vcs not NGVD29 or NGVD 1929 it is in NAVD88)
+        if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929']:
+            #Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
+            datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
+            datum88 = round(datum + datum_adj_ft, 2)
+        else:
+            datum88 = datum
+    
+    
+        #Set Grid override flag, if set to True then the 'has_grids' property is ignored. Allows for custom workaround.
+        #Special exception for kilo1, where it has attribute (has_grids == 0) yet there is grid metadata and polygons were converted to grids.
+        if code == 'kilo1':
+            grid_override = True
+            f.write(f'{code} : Custom workaround related to "has_grids" attribute')
+        else: 
+            grid_override = False        
+        #get grid metadata (metadata includes, elevation/stage/flow and etc for each site). If empty exit.
+        grid_metadata = usgs_grid_metadata(code, has_grid_override=grid_override)
+        if not grid_metadata:
+            f.write(f'{code} : Skipping because no grid metadata available\n')
+            continue
+         
+        #Get paths of all grids that have been downloaded, if no grids available for site then exit.
+        grid_paths = [grids for grids in ahps_dir.glob('*.tif*') if grids.suffix in ['.tif', '.tiff']]
+        if not grid_paths:
+            f.write(f'{code} : Skipping because no benchmark grids available\n')
+            continue
+        
+        # Iterate through grid_metadata and add the path to the dictionary as well as an indicator of whether the path exists.
+        for key in grid_metadata:        
+            #When USGS grid data was downloaded, grid was saved with the 'key' name. Update the grid_metadata to include the path.
+            path = ahps_dir / (key + '.tif')
+            grid_metadata[key]['path'] = path                
+            #Verify that the path exists (in some instances the grid should be available but it isn't) and add as subkey
+            if path.is_file():
+                grid_metadata[key]['path_exist'] = True
+            else:
+                grid_metadata[key]['path_exist'] = False
+        
+        #Convert grid metadata information to a DataFrame
+        df = pd.DataFrame.from_dict(grid_metadata, orient = 'index')
+        #Filter out rows where grids do not exist
+        df = df.query('path_exist == True')
+        #Prior to renaming columns do a check to make sure single site (will add functionality for multi-sites later)
+        if not 'QCFS' in df.columns:
+            f.write(f'{code} : Skipping because multisite\n')
+            continue
+        #Rename columns to match NWS AHPS data structure, this only applies to single USGS sites, if a multisite the columns are different from QCFS.
+        df.rename(columns = {'QCFS':'flow', 'STAGE':'stage', 'ELEV':'elevation'}, inplace=True)
+        #Many USGS maps have elevations to numerous decimal places. Round to nearest tenth. 
+        #NWS has maps to nearest tenth, for example HARP1 is both USGS and NWS, the USGS maps are to the hundredth of foot and NWS are to tenth.
+        df['elevation'] = round(df['elevation'],1)
+        #Assume flow source is supplied, if it is interpolated, this column will be changed later on.
+        df['flow_source'] = 'supplied by USGS'
+        #Accomodate for vdsg1 (upon inspection WRDS API reports thresholds in elevation instead of stage for this site)
+        if code == 'vdsg1':
+            df['stage'] = df['elevation']
+            f.write(f'{code} : Custom workaround because thresholds are reported as elevations\n')
+        
+        #Define rating curve as empty dataframe, populate if needed.
+        rating_curve = pd.DataFrame()        
+        #If flows are missing from the grid metadata, then interpolate flows using NWS or USGS rating curve
+        if df['flow'].isnull().all():        
+            #get entire rating curve, same source as interpolated threshold flows (USGS Rating Depot first then NRLDB rating curve).
+            if rating_curve_source == 'NRLDB':
+                site = [code]
+            elif rating_curve_source == 'USGS Rating Depot':
+                site = [metadata.get('identifiers').get('usgs_site_code')]
+            
+            rating_curve = get_rating_curve(rating_curve_url, site)
+    
+            #If rating curve is not present, skip site
+            if rating_curve.empty:
+                f.write(f'{code} : Skipping because no rating curve\n')
+                continue
+            #Add elevation fields to rating curve
+            #Add field with vertical coordinate system
+            vcs = datum_data['vcs']
+            if not vcs:
+                vcs = 'Unspecified, Assumed NAVD88'
+            rating_curve['vcs'] = vcs
+    
+            #Add field with original datum
+            rating_curve['datum'] = datum
+            
+            #If VCS is NGVD29 add rating curve elevation (in NGVD) as well as the NAVD88 datum
+            if vcs in ['NGVD29', 'NGVD 1929']:        
+                #Add field with raw elevation conversion (datum + stage)
+                rating_curve['elevation_ngvd29'] = rating_curve['stage'] + datum
+                #Add field with adjusted NAVD88 datum
+                rating_curve['datum_navd88'] = datum88
+            #Add field with NAVD88 elevation
+            rating_curve['elevation_navd88'] = rating_curve['stage'] + datum88
+            # sort inundation grids in ascending order based on stage
+            df.sort_values(by = 'elevation', ascending = True, inplace = True)        
+            #interpolate based on stage (don't need elevation because we have stage of floodgrid)
+            df['flow'] = np.interp(df['elevation'], rating_curve['elevation_navd88'], rating_curve['flow'], left = np.nan, right = np.nan)
+            #Overwrite flow source to reflect interpolation from rc
+            df['flow_source'] = f'interpolated from {rating_curve_source} rating curve'
+            
+        #Select the appropriate threshold grid for evaluation. Using the supplied threshold stages and the calculated map stages. 
+        grids,grid_flows = select_grids(df, stages, datum88, 1.1)
+    
+        #Obtain NWM segments that are on ms to apply flows
+        segments = get_nwm_segs(metadata)
+        site_ms_segs = set(segments).intersection(ms_segs)
+        segments = list(site_ms_segs)
+        #Preprocess grids and export to file and create flow file.
+        try:
+            #for each threshold
+            for i in threshold_categories:
+                #Obtain the flow and grid associated with threshold as well as extent grid which serves as the domain.
+                flow = grid_flows[i]
+                grid = grids[i]
+                extent = grids['extent']
+                #Make sure that flow and flow grid are valid
+                if not grid in ['No Map', 'No Threshold', 'No Flow']:
+                    #Define output directory (to be created later)
+                    outputdir = destination / huc / code / i                    
+                    
+                    #Create Binary Grids, first create domain of analysis, then create binary grid
+                    
+                    #Domain extent is largest floodmap in the static library WITH holes filled
+                    filled_domain_raster = outputdir.parent / f'{code}_extent.tif'
+                    #Create a domain raster if it does not exist.
+                    if not filled_domain_raster.exists():
+                        #Open extent data as rasterio object
+                        domain = rasterio.open(extent)
+                        domain_profile = domain.profile
+                        #Domain should have donut holes removed
+                        process_extent(domain, domain_profile, output_raster = filled_domain_raster)
+    
+                    #Open domain raster as rasterio object
+                    filled_domain = rasterio.open(filled_domain_raster)
+                    filled_domain_profile = filled_domain.profile
+    
+                    #Open benchmark data as a rasterio object.
+                    benchmark = rasterio.open(grid)
+                    benchmark_profile = benchmark.profile                
+    
+                    #Create the binary benchmark raster                        
+                    boolean_benchmark, boolean_profile = process_grid(benchmark, benchmark_profile, filled_domain, filled_domain_profile, reference_raster)    
+                                    
+                    #Output binary benchmark grid and flow file to destination
+                    outputdir.mkdir(parents = True, exist_ok = True)
+                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_depth_{i}.tif')
+                    with rasterio.Env():
+                        with rasterio.open(output_raster, 'w', **boolean_profile) as dst:
+                            dst.write(boolean_benchmark,1)
+                    
+                    #Close datasets
+                    domain.close()
+                    filled_domain.close()
+                    benchmark.close()
+                    
+                    #Create the guts of the flow file.
+                    flow_info = flow_data(segments,flow) 
+                    #Write out the flow file to csv
+                    output_flow_file = outputdir / (f'ahps_{code}_huc_{huc}_flows_{i}.csv')
+                    flow_info.to_csv(output_flow_file, index = False)
+                    
+        except Exception as e:
+            f.write(f'{code} : Error preprocessing benchmark\n{repr(e)}\n')
+            f.write(traceback.format_exc())
+            f.write('\n')
+            print(traceback.format_exc())
+        #Create extent if ahps code subfolder is present in destination directory.
+        ahps_directory = destination / huc / code
+        if ahps_directory.exists():
+            #Delete extent raster
+            filled_extent = ahps_directory / f'{code}_extent.tif' 
+            if filled_extent.exists:
+                filled_extent.unlink()            
+    
+            #Populate attribute information for site
+            grids_attributes = pd.DataFrame(data=grids.items(), columns = ['magnitude','path'])
+            flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude','grid_flow_cfs'])
+            threshold_attributes = pd.DataFrame(data=stages.items(), columns = ['magnitude','magnitude_stage'])       
+            #merge dataframes
+            attributes = grids_attributes.merge(flows_attributes, on = 'magnitude')
+            attributes = attributes.merge(threshold_attributes, on = 'magnitude')
+            attributes = attributes.merge(df[['path','stage','elevation', 'flow_source']], on = 'path')        
+            #Strip out sensitive paths and convert magnitude stage to elevation
+            attributes['path'] = attributes['path'].apply(lambda x :Path(x).name)
+            attributes['magnitude_elev_navd88']=(datum88 + attributes['magnitude_stage']).astype(float).round(1)
+            #Add general site information
+            attributes['nws_lid'] = code
+            attributes['wfo'] = metadata['nws_data']['wfo']
+            attributes['rfc'] = metadata['nws_data']['rfc']
+            attributes['state'] = metadata['nws_data']['state']
+            attributes['huc'] = huc
+            #Rename and Reorder columns
+            attributes.rename(columns = {'path':'grid_name', 'flow_source':'grid_flow_source','stage':'grid_stage','elevation':'grid_elev_navd88'}, inplace = True)        
+            attributes = attributes[['nws_lid','wfo','rfc','state','huc','magnitude','magnitude_stage','magnitude_elev_navd88','grid_name','grid_stage','grid_elev_navd88', 'grid_flow_cfs','grid_flow_source']]        
+            #Save attributes to csv
+            attributes.to_csv(ahps_directory / f'{code}_attributes.csv', index = False) 
+                
+            #if rating_curve generated, write the rating curve to a file
+            if not rating_curve.empty:
+                rating_curve_output = ahps_directory / (f'{code}_rating_curve.csv')
+                rating_curve['lat'] = datum_data['lat']
+                rating_curve['lon'] = datum_data['lon']
+                rating_curve.to_csv(rating_curve_output, index = False)
+                f.write(f'{code} : Rating curve needed to interpolate flow\n')
+            
+            #Write the interpolated flows to file
+            df_output = ahps_directory / (f'{code}_flows.csv')
+            df.to_csv(df_output, index = False)
+            
+        else: 
+            f.write(f'{code} : Unable to evaluate site, missing all flows\n')
+    
+    f.close()
+    
+    #Combine all attribute files
+    attribute_files = list(destination.rglob('*_attributes.csv'))
+    all_attributes = pd.DataFrame()
+    for i in attribute_files:
+        attribute_df = pd.read_csv(i, dtype={'huc':str})
+        all_attributes = all_attributes.append(attribute_df)
+    if not all_attributes.empty:
+        all_attributes.to_csv(destination / 'attributes.csv', index = False)
+            
+    return 
+
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
+    parser.add_argument('-s', '--source_dir', help = 'Workspace where all source data is located.', required = True)
+    parser.add_argument('-d', '--destination',  help = 'Directory where outputs are to be stored', required = True)
+    parser.add_argument('-r', '--reference_raster', help = 'reference raster used for benchmark raster creation', required = True)
+    args = vars(parser.parse_args())
+    
+
+    #Run get_env_paths and static_flow_lids
+    API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER, USGS_METADATA_URL = get_env_paths()
+    preprocess_usgs(**args)
\ No newline at end of file
diff --git a/tools/preprocess_download_usgs_grids.py b/tools/preprocess_download_usgs_grids.py
new file mode 100644
index 000000000..332a30ed1
--- /dev/null
+++ b/tools/preprocess_download_usgs_grids.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+import urllib.request 
+from pathlib import Path
+from dotenv import load_dotenv
+import os
+import argparse
+import requests
+from collections import defaultdict
+import urllib
+import pandas as pd
+
+load_dotenv()
+USGS_DOWNLOAD_URL = os.getenv("USGS_DOWNLOAD_URL")
+USGS_METADATA_URL = os.getenv("USGS_METADATA_URL")   
+EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+###############################################################################
+#Get all usgs grids available for download. This step is required because the grid metadata API returns gridID as an integer and truncates leading zeros found in grid names.
+###############################################################################            
+def get_all_usgs_gridnames():
+    '''
+    Retrieve all the available grids for download from USGS. This is necessary as the grid metadata available from USGS API doesn't preserve leading zeros.
+
+    Returns
+    -------
+    grid_lookup : collections.defaultdict
+        Dictionary with shortname as the key and a list of gridnames associated with a given shortname as values.
+    '''
+    
+    #Grid names are split between 4 websites
+    sites = ['grids_1', 'grids_2', 'grids_3', 'grids_4']
+    #Append all grid names to this variable
+    grid_names = []
+    #loop through each site and append the grid name to a list.
+    for i in sites:
+        #Get gridnames
+        url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/{i}/MapServer?f=pjson'
+        response = requests.get(url)
+        site_json = response.json()
+        info = site_json['layers']
+        #Loop through all grid info and extract the grid name.
+        for i in info:
+            grid_name = i['name']
+            grid_names.append(grid_name)
+    #Create dictionary with key of shortname and values being list of grids available.    
+    grid_lookup = defaultdict(list)
+    for i in grid_names:
+        #define key (shortname) and value (gridname)
+        key = i.split('_')[0]
+        value = i   
+        grid_lookup[key].append(value)
+    return grid_lookup
+###############################################################################
+#Get USGS Site metadata
+###############################################################################
+def usgs_site_metadata(code):
+    '''
+    Retrieves site metadata from USGS API and saves output as dictionary. Information used includes shortname and site number.
+    
+    Parameters
+    ----------
+    code : STR
+        AHPS code.
+    USGS_METADATA_URL : STR
+        URL for USGS datasets.
+
+    Returns
+    -------
+    site_metadata : DICT
+        Output metadata for an AHPS site.
+    '''
+    # Make sure code is lower case
+    code = code.lower()
+    # Get site metadata from USGS API using ahps code
+    site_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/sites/MapServer/0/query?where=AHPS_ID+%3D+%27{code}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
+    #Get data from API
+    response = requests.get(site_url)
+    #If response is valid, then get metadata and save to dictionary
+    if response.ok:
+        response_json = response.json()
+        site_metadata = response_json['features'][0]['attributes']
+    return site_metadata
+########################################################################
+#Get USGS Benchmark Grids
+########################################################################
+def obtain_usgs_data(workspace):
+    '''
+    Download GRIDS from USGS FIM studies
+
+    Parameters
+    ----------
+    workspace : STR
+        Output directory where grids are placed.
+
+    Returns
+    -------
+    None.
+
+    '''
+    
+    
+    #Define workspace where output data is downloaded to
+    workspace = Path(workspace)
+    #Get all names of grids available for download from USGS website.
+    grid_lookup = get_all_usgs_gridnames()
+    #List of target ahps codes. In "ahps_dictionary.py" we defined a dictionary (ahps_lookup) that contains all ahps codes and their sources.
+    target_ahps_codes = pd.read_csv(EVALUATED_SITES_CSV)
+    target_ahps_codes = target_ahps_codes.query('Source in ["Both","USGS"]')['Total_List'].to_list()
+    #Loop through all codes in the target_ahps_codes list.
+    all_messages = []
+    for code in target_ahps_codes:
+        #Get metadata information related to ahps code from USGS API.
+        code_metadata = usgs_site_metadata(code)
+        #From code_metadata get the shortname and site_no associated with the code.
+        shortname = code_metadata['SHORT_NAME']
+        site_no = code_metadata['SITE_NO']     
+        #Define the output location for all grids and create if it doesn't exist.
+        dest_dir = workspace / code.lower() / 'depth_grids'
+        dest_dir.mkdir(parents = True, exist_ok = True)    
+        #Get list of all available grids for download using the grid_lookup dictionary
+        gridnames = grid_lookup[shortname]    
+        #Loop through all available grids for download, download them, and save to defined location.
+        for gridname in gridnames:
+            print(f'working on {gridname}')
+            gridid = gridname.split('_')[1]
+            #Define a filled gridID that has leading zeros out to 4 digits.
+            filled_gridid = gridid.zfill(4)       
+            #Download gridded data from the USGS s3 website. The files will be copied specified directory and the GRIDID will have 4 digits with leading zeros.
+            base_url = f'{USGS_DOWNLOAD_URL}/FIM/tmp1/fimgrids2iwrss/'       
+            #Each grid dataset has these file extensions. Download each file
+            extensions = ['.tif', '.tfw', '.tif.aux.xml', '.tif.ovr', '.tif.xml']
+            #Loop through each extension type and download.
+            for gridext in extensions:
+                #Define the url associated with each grid
+                url = base_url + gridname + gridext       
+                #Define the output file path of the grid. The grid filename uses the filled gridID. This resolves issues down the road of USGS grid metadata information storing the gridid as a number and truncating leading zeros from the gridname.
+                saved_grid_path = dest_dir / (f'{shortname}_{filled_gridid}{gridext}')
+                #Check to see if file has already been downloaded
+                if not saved_grid_path.is_file():
+                    #If file hasn't been downloaded, download it. If there was an error downloading, make note.
+                    try:
+                        urllib.request.urlretrieve(url, saved_grid_path)
+                        message = f'{gridname} downloaded'
+                        all_messages.append(message)
+                    except: 
+                        message = f'{gridname} error downloading'
+                        all_messages.append(message)
+                #If file exists make note of it.
+                else:
+                    message = f'skipping {gridname}, exists on file'
+                    all_messages.append(message)
+    return
+
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Download Grid data associated with USGS FIM studies.')
+    parser.add_argument('-w', '--workspace', help = 'Workspace where all outputs will be saved.', required = True)
+    args = vars(parser.parse_args())
+    
+    #Download datasets
+    obtain_usgs_data(**args)
+
+
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 714f2c535..c5bf9884d 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -54,7 +54,8 @@ def check_file_age(file):
             check = f'{file.name} is {int(file_age_days)} days old, consider updating.\nUpdate with rating_curve_get_usgs_curves.py'
         else:
             check = f'{file.name} is {int(file_age_days)} days old.'
-    return check
+
+        return check
 
 # recurr_intervals = ['recurr_1_5_cms.csv','recurr_5_0_cms.csv','recurr_10_0_cms.csv']
 
@@ -67,7 +68,8 @@ def generate_rating_curve_metrics(args):
     nwm_recurr_data_filename    = args[4]
     rc_comparison_plot_filename = args[5]
     nwm_flow_dir                = args[6]
-    huc                         = args[7]
+    catfim_flows_filename       = args[7]
+    huc                         = args[8]
 
     elev_table = pd.read_csv(elev_table_filename,dtype={'location_id': str})
     hydrotable = pd.read_csv(hydrotable_filename,dtype={'HUC': str,'feature_id': str})
@@ -118,10 +120,15 @@ def generate_rating_curve_metrics(args):
         # Merge NWM recurr intervals into a single layer
         nwm_recurr_intervals_all = reduce(lambda x,y: pd.merge(x,y, on='feature_id', how='outer'), [recurr_1_5_yr, recurr_5_yr, recurr_10_yr])
         nwm_recurr_intervals_all = pd.melt(nwm_recurr_intervals_all, id_vars=['feature_id'], value_vars=['1.5','5.0','10.0'], var_name='recurr_interval', value_name='discharge_cms')
+
+        # Append catfim data (already set up in format similar to nwm_recurr_intervals_all)
+        cat_fim = pd.read_csv(catfim_flows_filename, dtype={'feature_id':str})
+        nwm_recurr_intervals_all = nwm_recurr_intervals_all.append(cat_fim)
+
+        # Convert discharge to cfs and filter
         nwm_recurr_intervals_all['discharge_cfs'] = nwm_recurr_intervals_all.discharge_cms * 35.3147
         nwm_recurr_intervals_all = nwm_recurr_intervals_all.filter(items=['discharge_cfs', 'recurr_interval','feature_id']).drop_duplicates()
 
-
         # Identify unique gages
         usgs_crosswalk = hydrotable.filter(items=['location_id', 'feature_id']).drop_duplicates()
 
@@ -372,12 +379,13 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='generate rating curve plots and tables for FIM and USGS gages')
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True)
-    parser.add_argument('-output_dir','--output-dir', help='rating curves output folder', required=True)
-    parser.add_argument('-gages','--usgs-gages-filename',help='USGS rating curves',required=True)
-    parser.add_argument('-flows','--nwm-flow-dir',help='NWM recurrence flows dir',required=True)
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-output_dir','--output-dir', help='rating curves output folder', required=True,type=str)
+    parser.add_argument('-gages','--usgs-gages-filename',help='USGS rating curves',required=True,type=str)
+    parser.add_argument('-flows','--nwm-flow-dir',help='NWM recurrence flows dir',required=True,type=str)
+    parser.add_argument('-catfim', '--catfim-flows-filename', help='Categorical FIM flows file',required = True,type=str)
     parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-    parser.add_argument('-group','--stat-groups',help='column(s) to group stats',required=False)
+    parser.add_argument('-group','--stat-groups',help='column(s) to group stats',required=False,type=str)
 
     args = vars(parser.parse_args())
 
@@ -385,6 +393,7 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     output_dir = args['output_dir']
     usgs_gages_filename = args['usgs_gages_filename']
     nwm_flow_dir = args['nwm_flow_dir']
+    catfim_flows_filename = args['catfim_flows_filename']
     number_of_jobs = args['number_of_jobs']
     stat_groups = args['stat_groups']
 
@@ -415,7 +424,7 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
             rc_comparison_plot_filename = join(plots_dir,f"FIM-USGS_rating_curve_comparison_{huc}.png")
 
             if isfile(elev_table_filename):
-                procs_list.append([elev_table_filename, hydrotable_filename, usgs_gages_filename, usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,nwm_flow_dir,huc])
+                procs_list.append([elev_table_filename, hydrotable_filename, usgs_gages_filename, usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,nwm_flow_dir, catfim_flows_filename, huc])
 
     # Initiate multiprocessing
     print(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
index 6676c27b3..cea2a0dbf 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -3,7 +3,7 @@
 import pandas as pd
 import geopandas as gpd
 from pathlib import Path
-from tools_shared_functions import get_metadata, get_datum, ngvd_to_navd_ft, get_rating_curve, aggregate_wbd_hucs
+from tools_shared_functions import get_metadata, get_datum, ngvd_to_navd_ft, get_rating_curve, aggregate_wbd_hucs, get_thresholds, flow_data
 from dotenv import load_dotenv
 import os
 import argparse
@@ -99,7 +99,73 @@ def get_all_active_usgs_sites():
 
     return gdf, list_of_sites, acceptable_sites_metadata
             
-            
+##############################################################################
+#Generate categorical flows for each category across all sites.
+##############################################################################
+def write_categorical_flow_files(metadata, workspace):
+    '''
+    Writes flow files of each category for every feature_id in the input metadata.
+    Written to supply input flow files of all gage sites for each flood category.
+
+    Parameters
+    ----------
+    metadata : DICT
+        Dictionary of metadata from WRDS (e.g. output from get_all_active_usgs_sites).
+    workspace : STR
+        Path to workspace where flow files will be saved.
+
+    Returns
+    -------
+    None.
+
+    '''
+    
+    threshold_url = f'{API_BASE_URL}/nws_threshold'
+    workspace = Path(workspace)
+    workspace.mkdir(parents = True, exist_ok = True)
+    #For each site in metadata 
+    all_data = pd.DataFrame()
+    
+    for site in metadata:
+        #Get the feature_id and usgs_site_code
+        feature_id = site.get('identifiers').get('nwm_feature_id')
+        usgs_code = site.get('identifiers').get('usgs_site_code')
+        nws_lid = site.get('identifiers').get('nws_lid')
+        
+        #thresholds only provided for valid nws_lid.
+        if nws_lid == 'Bogus_ID':
+            continue
+        
+        #if invalid feature_id skip to next site
+        if feature_id is None:
+            continue
+        
+        #Get the stages and flows
+        stages, flows = get_thresholds(threshold_url, select_by = 'nws_lid', selector = nws_lid, threshold = 'all')
+        
+        #For each flood category
+        for category in ['action','minor','moderate','major']:
+            #Get flow
+            flow = flows.get(category, None)
+            #If flow or feature id are not valid, skip to next site
+            if flow is None:
+                continue
+            #Otherwise, write 'guts' of a flow file and append to a master DataFrame.
+            else:
+                data = flow_data([feature_id], flow, convert_to_cms = True)
+                data['recurr_interval'] = category            
+                data['nws_lid'] = nws_lid
+                data['location_id'] = usgs_code
+                data = data.rename(columns = {'discharge':'discharge_cms'})
+                #Append site data to master DataFrame
+                all_data = all_data.append(data, ignore_index = True)
+    
+    #Write CatFIM flows to file
+    final_data = all_data[['feature_id','discharge_cms', 'recurr_interval']]
+    final_data.to_csv(workspace / f'catfim_flows_cms.csv', index = False)
+    return all_data
+###############################################################################
+           
 def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
     '''
 
@@ -272,6 +338,9 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
         if list_of_gage_sites == ['all']:            
             acceptable_sites_gdf = acceptable_sites_gdf.to_crs(PREP_PROJECTION)
             acceptable_sites_gdf.to_file(Path(workspace) / 'usgs_gages.gpkg', layer = 'usgs_gages', driver = 'GPKG')
+        
+        #Write out flow files for each threshold across all sites
+        all_data = write_categorical_flow_files(metadata_list, workspace)
     
     return all_rating_curves
 
@@ -295,5 +364,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
     l = args['list_of_gage_sites']
     w = args['workspace'] 
     t = float(args['sleep_timer'])           
-    #Run create_flow_forecast_file
-    usgs_rating_to_elev(list_of_gage_sites = l, workspace=w, sleep_time = t)
\ No newline at end of file
+    
+    #Generate USGS rating curves
+    usgs_rating_to_elev(list_of_gage_sites = l, workspace=w, sleep_time = t)
+    
\ No newline at end of file
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 8ea3d5d59..fb6204dd5 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -7,9 +7,18 @@
 import pandas as pd
 import geopandas as gpd
 import requests
-from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util.retry import Retry
-
+import numpy as np
+import pathlib
+from pathlib import Path
+import rasterio.shutil
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+import rasterio.crs
+from rasterio.merge import merge
+from rasterio import features
+from shapely.geometry import shape
+from shapely.geometry import Polygon
+from shapely.geometry import MultiPolygon
+import re
 from tools_shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
                                     ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
 
@@ -1254,4 +1263,302 @@ def get_rating_curve(rating_curve_url, location_ids):
             else:
                 continue
 
-    return all_curves
\ No newline at end of file
+    return all_curves
+#######################################################################
+#Following Functions used for preprocesing of AHPS sites (NWS and USGS)
+########################################################################
+    
+#######################################################################
+#Function to return a correct maps.
+########################################################################
+def select_grids(dataframe, stages, datum88, buffer):
+    '''
+    Given a DataFrame (in a specific format), and a dictionary of stages, and the datum (in navd88). 
+    loop through the available inundation datasets and find the datasets that are equal to or immediately above the thresholds and only return 1 dataset per threshold (if any).
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        DataFrame that has to be in a specific format and contains the stages and paths to the inundation datasets.
+    stages : DICT
+        Dictionary of thresholds (key) and stages (values)
+    datum88: FLOAT
+        The datum associated with the LID that is pre-converted to NAVD88 (if needed)
+    buffer: Float
+        Interval which the uppder bound can be assigned. For example, Threshold + buffer = upper bound. Recommended to make buffer 0.1 greater than desired interval as code selects maps < and not <=
+
+    Returns
+    -------
+    maps : DICT
+        Dictionary of threshold (key) and inundation dataset path (value)
+    map_flows: DICT
+        Dictionary of threshold (key) and flows in CFS rounded to the nearest whole number associated with the selected maps (value)
+
+    '''
+    #Define threshold categories
+    thresholds = ['action', 'minor', 'moderate', 'major']
+    maps = {}
+    map_flows={}
+    #For each threshold, pick the appropriate map for analysis.
+    for i,threshold in enumerate(thresholds):
+        #Check if stage is None
+        if not stages[threshold] is None:
+            #Define the threshold floor elevation (navd88).
+            lower_bound = round((stages[threshold] + datum88),1)
+            #Define the threshold ceiling (navd88)
+            upper_bound = round((stages[threshold] + datum88 + buffer),1)
+            #For thresholds that are action, minor, moderate
+            if threshold in ['action', 'minor', 'moderate']:
+                #Make sure the next threshold has a valid stage
+                if stages[thresholds[i+1]] is None:
+                    next_threshold = upper_bound
+                else:
+                    #Determine what the next threshold elevation is.
+                    next_threshold = round((stages[thresholds[i+1]] + datum88),1)
+               #Make sure the upper_bound is not greater than the next threshold, if it is then reassign upper_bound.
+                if upper_bound > next_threshold:
+                    upper_bound = next_threshold                
+                #Get the single map which meets the criteria.
+                value = dataframe.query(f'({lower_bound}<=elevation) & (elevation<{upper_bound})')['elevation'].min()
+            #For major threshold
+            else:
+                #Get the single map which meets criteria.
+                value = dataframe.query(f'({lower_bound}<=elevation) & (elevation<{upper_bound})')['elevation'].min()
+    
+            #If the selected value is a number
+            if np.isfinite(value):
+                #Get the map path and the flow associated with the map (rounded to nearest whole number)
+                map_path = dataframe.query(f'elevation == {value}')['path'].item()
+                map_flow = round(dataframe.query(f'elevation == {value}')['flow'].item(),0)
+                #Check to see if map_flow is valid (if beyond rating_curve it is nan)
+                if not np.isfinite(map_flow):
+                    map_path = 'No Flow'
+                    map_flow = 'No Flow'
+                                    
+            #If the selected value is not a number (or interpolated flow is nan caused by elevation of map which is beyond rating curve range), then map_path and map_flows are both set to 'No Map'. 
+            else: 
+                map_path = 'No Map' 
+                map_flow = 'No Map' 
+        else:
+            map_path = 'No Threshold'
+            map_flow = 'No Threshold'
+        
+        #Write map paths and flows to dictionary        
+        maps[threshold] = map_path
+        map_flows[threshold] = map_flow
+    
+    #Get the maximum inundation map (using elevation) and this will be the domain extent
+    max_value = dataframe['elevation'].max()
+    map_path = dataframe.query(f'elevation == {max_value}')['path'].item()
+    map_flow = 'Not Used'
+    maps['extent'] = map_path
+    map_flows['extent'] = map_flow
+    
+    return maps,map_flows
+
+#######################################################################
+#Process AHPS Extent Grid (Fill Holes)
+#######################################################################
+def process_extent(extent, profile, output_raster = False):
+    '''
+    Convert raster to feature (using raster_to_feature), the footprint is used so all raster values are set to 1 where there is data.
+    fill all donut holes in resulting feature. 
+    Filled geometry is then converted back to raster using same raster properties as input profile. 
+    Output raster will have be encoded as follows:
+        filled footprint (wet) = 1
+        remaining area in raster domain (dry) = 0
+        NoData = 3
+
+    Parameters
+    ----------
+    extent : Rasterio Dataset Reader
+        Path to extent raster
+    extent_profile: Rasterio Profile
+        profile related to the extent argument
+    output_raster: STR
+        Path to output raster. If no path supplied, then no raster is written to disk. default = False
+
+    Returns (If no output raster specified)
+    -------
+    extent_filled_raster : rasterio dataset
+        Extent raster with filled donut holes
+    profile : rasterio profile
+        Profile associated with extent_filled_raster
+
+    '''
+        
+    #Convert extent to feature and explode geometry
+    poly_extent = raster_to_feature(extent, profile, footprint_only = True)
+    poly_extent = poly_extent.explode()
+    
+    #Fill holes in extent
+    poly_extent_fill_holes=MultiPolygon(Polygon(p.exterior) for p in poly_extent['geometry'])
+    # loop through the filled polygons and insert the new geometry
+    for i in range(len(poly_extent_fill_holes)):
+        poly_extent.loc[i,'geometry'] = poly_extent_fill_holes[i]
+    
+    #Dissolve filled holes with main map and explode
+    poly_extent['dissolve_field'] = 1
+    poly_extent = poly_extent.dissolve(by = 'dissolve_field')
+    poly_extent = poly_extent.explode()
+    poly_extent = poly_extent.reset_index()
+    
+    #Convert filled polygon back to raster
+    extent_filled_raster = features.rasterize(((geometry, 1) for geometry in poly_extent['geometry']), fill = 0, dtype = 'int32',transform = profile['transform'], out_shape = (profile['height'], profile['width']))
+    
+    #Update profile properties (dtype and no data)
+    profile.update(dtype = rasterio.int32)
+    profile.update(nodata=0)
+    
+    #Check if output raster is specified. If so, the write extent filled raster to disk. 
+    if output_raster:
+        #Create directory
+        Path(output_raster).parent.mkdir(parents = True, exist_ok = True)
+        with rasterio.Env():
+            with rasterio.open(output_raster, 'w', **profile) as dst:
+                dst.write(extent_filled_raster, 1)
+    #If no output raster is supplied the return the rasterio array and profile.
+    else:
+        return extent_filled_raster, profile
+########################################################################
+#Convert raster to polygon
+########################################################################
+def raster_to_feature(grid, profile_override = False, footprint_only = False):
+    '''
+    Given a grid path, convert to vector, dissolved by grid value, in GeoDataFrame format.
+
+    Parameters
+    ----------
+    grid_path : pathlib path OR rasterio Dataset Reader
+        Path to grid or a rasterio Dataset Reader
+    profile_override: rasterio Profile
+        Default is False, If a rasterio Profile is supplied, it will dictate the transform and crs.
+    footprint_only: BOOL
+        If true, dataset will be divided by itself to remove all unique values. If False, all values in grid will be carried through on raster to feature conversion. default = False
+
+    Returns
+    -------
+    dissolve_geodatabase : GeoDataFrame
+        Dissolved (by gridvalue) vector data in GeoDataFrame.
+
+    '''
+    #Determine what format input grid is:
+    #If a pathlib path, open with rasterio
+    if isinstance(grid, pathlib.PurePath):
+        dataset = rasterio.open(grid)
+    #If a rasterio dataset object, assign to dataset
+    elif isinstance(grid, rasterio.DatasetReader):
+        dataset = grid
+
+    #Get data/mask and profile properties from dataset
+    data = dataset.read(1)
+    msk = dataset.read_masks(1)   
+    data_transform = dataset.transform
+    coord_sys = dataset.crs
+
+    #If a profile override was supplied, use it to get the transform and coordinate system.
+    if profile_override:
+        data_transform = profile_override['transform']
+        coord_sys = profile_override['crs']
+    
+    #If a footprint of the raster is desired, convert all data values to 1
+    if footprint_only:
+        data[msk == 255] = 1   
+    
+    #Convert grid to feature
+    spatial = []
+    values = []
+    for geom, val in rasterio.features.shapes(data, mask = msk, transform = data_transform):
+        spatial.append(shape(geom))
+        values.append(val)        
+    spatial_geodataframe = gpd.GeoDataFrame({'values': values,'geometry':spatial }, crs = coord_sys)
+    dissolve_geodataframe = spatial_geodataframe.dissolve(by = 'values')
+    return dissolve_geodataframe
+########################################################################
+#Create AHPS Benchmark Grid
+########################################################################
+def process_grid(benchmark, benchmark_profile, domain, domain_profile, reference_raster):
+    '''
+    Given a benchmark grid and profile, a domain rasterio dataset and profile, and a reference raster, 
+    Match the benchmark dataset to the domain extent and create a classified grid convert to:
+        0 (no data footprint of domain)
+        1 (data footprint of domain)
+        2 (data footprint of benchmark)
+    Then reproject classified benchmark grid to match reference grid resolution and crs. 
+    Output is an array of values and a profile.
+
+    Parameters
+    ----------
+    benchmark : rasterio dataset
+        Rasterio dataset of the benchmark dataset for a given threshold
+    benchmark_profile : rasterio profile
+        A potentially modified profile to the benchmark dataset.
+    domain: rasterio dataset
+        Rasterio dataset of the domain grid (the maximum available grid for a given site)
+    domain_profile: rasterio profile
+        A potentially modified profile of the domain dataset.
+    reference_raster : pathlib Path
+        Path to reference dataset.
+
+    Returns
+    -------
+    boolean_benchmark : numpy Array
+        Array of values for the benchmark_boolean grid.
+    profile : rasterio profile
+        Updated, final profile of the boolean_benchmark grid. 
+
+    '''
+    
+    #Make benchmark have same dimensions as domain (Assume domain has same CRS as benchmark)
+    #Get source CRS (benchmark and domain assumed to be same CRS)
+    source_crs = benchmark_profile['crs'].to_wkt()
+    #Get domain data
+    domain_arr = domain.read(1)
+    #Get benchmark data
+    benchmark_arr = benchmark.read(1)
+    #Create empty array with same dimensions as domain
+    benchmark_fit_to_domain = np.empty(domain_arr.shape)
+    #Make benchmark have same footprint as domain (Assume domain has same CRS as benchmark)
+    reproject(benchmark_arr, 
+              destination = benchmark_fit_to_domain,
+              src_transform = benchmark.transform, 
+              src_crs = source_crs,
+              src_nodata = benchmark.nodata,
+              dst_transform = domain.transform, 
+              dst_crs = source_crs,
+              dst_nodata = benchmark.nodata,
+              dst_resolution = source_crs,
+              resampling = Resampling.bilinear)    
+    #Convert fitted benchmark dataset to boolean. 0 = NODATA Regions and 1 = Data Regions
+    benchmark_fit_to_domain_bool = np.where(benchmark_fit_to_domain == benchmark.nodata,0,1)
+    #Merge domain datamask and benchmark data mask. New_nodata_value (2) = Domain NO DATA footprint, 0 = NO DATA for benchmark (within data region of domain), 1 = DATA region of benchmark.
+    new_nodata_value = 2
+    classified_benchmark = np.where(domain_arr == domain.nodata, new_nodata_value, benchmark_fit_to_domain_bool)
+    
+    ##Reproject classified benchmark to reference raster crs and resolution.
+    #Read in reference raster
+    reference = rasterio.open(reference_raster)
+    #Determine the new transform and dimensions of reprojected/resampled classified benchmark dataset whos width, height, and bounds are same as domain dataset.
+    new_benchmark_transform, new_benchmark_width, new_benchmark_height = calculate_default_transform(source_crs, reference.crs, domain.width, domain.height, *domain.bounds, resolution = reference.res)   
+    #Define an empty array that is same dimensions as output by the "calculate_default_transform" command. 
+    classified_benchmark_projected = np.empty((new_benchmark_height,new_benchmark_width), dtype=np.uint8) 
+    #Reproject and resample the classified benchmark dataset. Nearest Neighbor resampling due to integer values of classified benchmark.
+    reproject(classified_benchmark, 
+              destination = classified_benchmark_projected,
+              src_transform = domain.transform, 
+              src_crs = source_crs,
+              src_nodata = new_nodata_value,
+              dst_transform = new_benchmark_transform, 
+              dst_crs = reference.crs,
+              dst_nodata = new_nodata_value,
+              dst_resolution = reference.res,
+              resampling = Resampling.nearest)
+    
+    #Update profile using reference profile as base (data type, NODATA, transform, width/height).
+    profile = reference.profile
+    profile.update(transform = new_benchmark_transform)
+    profile.update(dtype = rasterio.uint8)
+    profile.update(nodata = new_nodata_value) 
+    profile.update (width = new_benchmark_width)
+    profile.update(height = new_benchmark_height)   
+    return classified_benchmark_projected, profile
\ No newline at end of file

From 199ab6e787b1742f0558a69247e08d411add2505 Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Wed, 28 Apr 2021 11:32:11 -0500
Subject: [PATCH 070/359] Refactor synthesize_test_case.py to handle exceptions
 during multiprocessing

Refactor synthesize_test_case.py to handle exceptions during multiprocessing. Resolves issue #351

Changes
 - refactored inundation.py and run_test_case.py to handle exceptions without using sys.exit(). This isn't the ideal way to handle exceptions when using multiprocessing because the child process is expecting a return.
---
 CHANGELOG.md           |  29 ++++--
 tools/inundation.py    | 217 +++++++++++++++++++++--------------------
 tools/run_test_case.py |  59 +++++------
 3 files changed, 164 insertions(+), 141 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1cc4cc499..82b0f0cd3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.15.7 - 2021-04-28 - [PR #367](https://github.com/NOAA-OWP/cahaba/pull/367)
+
+Refactor synthesize_test_case.py to handle exceptions during multiprocessing. Resolves issue #351
+
+## Changes
+- refactored `inundation.py` and `run_test_case.py` to handle exceptions without using `sys.exit()`.
+
+<br/><br/>
 ## v3.0.15.6 - 2021-04-23 - [PR #365](https://github.com/NOAA-OWP/cahaba/pull/365)
 
 Implement CatFIM threshold flows to Sierra test and add AHPS benchmark preprocessing scripts.
@@ -20,11 +29,12 @@ Prevent eval_plots.py from erroring out when spatial argument enabled if certain
 
 ## Changes
 - Add check to make sure analyzed dataset is available prior to creating spatial dataset.
-<br/><br/>
 
+<br/><br/>
 ## v3.0.15.4 - 2021-04-20 - [PR #356](https://github.com/NOAA-OWP/cahaba/pull/356)
 
 Closing all multiprocessing Pool objects in repo.
+
 <br/><br/>
 ## v3.0.15.3 - 2021-04-19 - [PR #358](https://github.com/NOAA-OWP/cahaba/pull/358)
 
@@ -44,6 +54,7 @@ Preprocess NHDPlus HR rasters for consistent projections, nodata values, and con
 ## v3.0.15.2 - 2021-04-16 - [PR #359](https://github.com/NOAA-OWP/cahaba/pull/359)
 
 Hotfix to preserve desired files when production flag used in `fim_run.sh`. 
+
 ## Changes
 
 - Fixed production whitelisted files.
@@ -52,6 +63,7 @@ Hotfix to preserve desired files when production flag used in `fim_run.sh`.
 ## v3.0.15.1 - 2021-04-13 - [PR #355](https://github.com/NOAA-OWP/cahaba/pull/355)
 
 Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field. 
+
 ## Changes
 
 - Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "MS" run, only consider gages that contain rating curve information (via `curve` attribute) and are also mainstems gages (via `mainstems` attribute). 
@@ -65,7 +77,6 @@ Sierra test considered all USGS gage locations to be mainstems even though many
 - Adds the `extent` argument specified by user when running `fim_run.sh` to `usgs_gage_crosswalk.py`.
 
 <br/><br/>
-
 ## v3.0.15.0 - 2021-04-08 - [PR #340](https://github.com/NOAA-OWP/cahaba/pull/340)
 
 Implementing a prototype technique to estimate the missing bathymetric component in the HAND-derived synthetic rating curves. The new Bathymetric Adjusted Rating Curve (BARC) function is built within the `fim_run.sh` workflow and will ingest bankfull geometry estimates provided by the user to modify the cross section area used in the synthetic rating curve generation.
@@ -81,6 +92,7 @@ Implementing a prototype technique to estimate the missing bathymetric component
     - Imports the existing synthetic rating curve table and the bankfull geometry input data (topwidth and cross section area per COMID).
     - Performs new synthetic rating curve calculations with bathymetry estimation modifications.
     - Flags issues with the thalweg-notch artifact.
+
 <br/><br/>
 ## v3.0.14.0 - 2021-04-05 - [PR #338](https://github.com/NOAA-OWP/cahaba/pull/338)
 
@@ -97,6 +109,7 @@ Create tool to retrieve rating curves from USGS sites and convert to elevation (
      1) `usgs_rating_curves.csv`: A csv file that contains rating curves (including converted to NAVD88 elevation) for USGS gages in a format that is compatible with  `rating_curve_comparisons.py`. As it is is currently configured, only gages within CONUS will have rating curve data.
      2) `log.csv`: A log file that records status for each gage and includes error messages.
      3) `usgs_gages.gpkg`: A geospatial layer (in FIM projection) of all active USGS gages that meet a predefined criteria. Additionally, the `curve` attribute indicates whether a rating curve is found in the `usgs_rating_curves.csv`. This spatial file is only generated if the `all` option is passed with the `-l` argument.
+
 <br/><br/>
 ## v3.0.13.0 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
 
@@ -110,8 +123,8 @@ Created tool to compare synthetic rating curve with benchmark rating curve (Sier
 ### Additions
  - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in `run_by_unit.py` with elevation and additional attributes at USGS gages.
  - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data.
-<br/><br/>
 
+<br/><br/>
 ## v3.0.12.1 - 2021-03-31 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
 
 Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
@@ -124,8 +137,8 @@ Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
 ### Additions
  - Creates `fim_performance_points.shp`: this layer consists of all evaluated ahps points (with metrics). Spatial data retrieved from WRDS on the fly.
  - Creates `fim_performance_polys.shp`: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.
-<br/><br/>
 
+<br/><br/>
 ## v3.0.12.0 - 2021-03-26 - [PR #327](https://github.com/NOAA-OWP/cahaba/pull/237)
 
 Add more detail/information to plotting capabilities.
@@ -137,8 +150,8 @@ Add more detail/information to plotting capabilities.
 ### Additions
  - Optional argument to create barplots of CSI for each individual site.
  - Create a csv containing the data used to create the scatterplots.
-<br/><br/>
 
+<br/><br/>
 ## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
 
 Improvements to CatFIM service source data generation.
@@ -151,16 +164,16 @@ Improvements to CatFIM service source data generation.
 ### Additions
  - Added `generate_categorical_fim.py` to wrap `generate_categorical_fim_flows.py` and `generate_categorical_fim_mapping.py`.
  - Create new `nws_lid_sites` shapefile located in same directory as the `catfim_library` shapefile.
-<br/><br/>
 
+<br/><br/>
 ## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
 
 Patch to synthesize_test_cases.py.
 
 ### Changes
  - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
-<br/><br/>
 
+<br/><br/>
 ## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
 
 Preprocessing of flow files for Categorical FIM.
@@ -174,8 +187,8 @@ Preprocessing of flow files for Categorical FIM.
 
  ### Changes
  - Stability fixes to `generate_categorical_fim.py`.
-<br/><br/>
 
+<br/><br/>
 ## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)
 
 Enhancements to FIM API.
diff --git a/tools/inundation.py b/tools/inundation.py
index d093385b8..d105ea52c 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -156,59 +156,63 @@ def inundate(
     else:
         raise TypeError("Pass hydro table csv")
 
-    # make windows generator
-    window_gen = __make_windows_generator(rem,catchments,catchment_poly,mask_type,catchmentStagesDict,inundation_raster,inundation_polygon,
-                                          depths,out_raster_profile,out_vector_profile,quiet,hucs=hucs,hucSet=hucSet)
+    if catchmentStagesDict is not None:
 
-    # start up thread pool
-    executor = ThreadPoolExecutor(max_workers=num_workers)
+        # make windows generator
+        window_gen = __make_windows_generator(rem,catchments,catchment_poly,mask_type,catchmentStagesDict,inundation_raster,inundation_polygon,
+                                              depths,out_raster_profile,out_vector_profile,quiet,hucs=hucs,hucSet=hucSet)
 
-    # submit jobs
-    results = {executor.submit(__inundate_in_huc,*wg) : wg[6] for wg in window_gen}
+        # start up thread pool
+        executor = ThreadPoolExecutor(max_workers=num_workers)
 
-    inundation_rasters = [] ; depth_rasters = [] ; inundation_polys = []
-    for future in as_completed(results):
-        try:
-            future.result()
-        except Exception as exc:
-            __vprint("Exception {} for {}".format(exc,results[future]),not quiet)
-        else:
+        # submit jobs
+        results = {executor.submit(__inundate_in_huc,*wg) : wg[6] for wg in window_gen}
 
-            if results[future] is not None:
-                __vprint("... {} complete".format(results[future]),not quiet)
+        inundation_rasters = [] ; depth_rasters = [] ; inundation_polys = []
+        for future in as_completed(results):
+            try:
+                future.result()
+            except Exception as exc:
+                __vprint("Exception {} for {}".format(exc,results[future]),not quiet)
             else:
-                __vprint("... complete",not quiet)
-
-            inundation_rasters += [future.result()[0]]
-            depth_rasters += [future.result()[1]]
-            inundation_polys += [future.result()[2]]
-
-    # power down pool
-    executor.shutdown(wait=True)
-
-    # optional aggregation
-    if (aggregate) & (hucs is not None):
-        # inun grid vrt
-        if inundation_raster is not None:
-            inun_vrt = BuildVRT(splitext(inundation_raster)[0]+'.vrt',inundation_rasters)
-            inun_vrt = None
-            #_ = run('gdalbuildvrt -q -overwrite {} {}'.format(splitext(inundation_raster)[0]+'.vrt'," ".join(inundation_rasters)),shell=True)
-        # depths vrt
-        if depths is not None:
-            depths_vrt = BuildVRT(splitext(depths)[0]+'.vrt',depth_rasters,resampleAlg='bilinear')
-            depths_vrt = None
-            #_ = run('gdalbuildvrt -q -overwrite -r bilinear {} {}'.format(splitext(depths)[0]+'.vrt'," ".join(depth_rasters)),shell=True)
 
-        # concat inun poly
-        if inundation_polygon is not None:
-            _ = run('ogrmerge.py -o {} {} -f GPKG -single -overwrite_ds'.format(inundation_polygon," ".join(inundation_polys)),shell=True)
-
-    # close datasets
-    rem.close()
-    catchments.close()
-
-    return(0)
+                if results[future] is not None:
+                    __vprint("... {} complete".format(results[future]),not quiet)
+                else:
+                    __vprint("... complete",not quiet)
+
+                inundation_rasters += [future.result()[0]]
+                depth_rasters += [future.result()[1]]
+                inundation_polys += [future.result()[2]]
+
+        # power down pool
+        executor.shutdown(wait=True)
+
+        # optional aggregation
+        if (aggregate) & (hucs is not None):
+            # inun grid vrt
+            if inundation_raster is not None:
+                inun_vrt = BuildVRT(splitext(inundation_raster)[0]+'.vrt',inundation_rasters)
+                inun_vrt = None
+                #_ = run('gdalbuildvrt -q -overwrite {} {}'.format(splitext(inundation_raster)[0]+'.vrt'," ".join(inundation_rasters)),shell=True)
+            # depths vrt
+            if depths is not None:
+                depths_vrt = BuildVRT(splitext(depths)[0]+'.vrt',depth_rasters,resampleAlg='bilinear')
+                depths_vrt = None
+                #_ = run('gdalbuildvrt -q -overwrite -r bilinear {} {}'.format(splitext(depths)[0]+'.vrt'," ".join(depth_rasters)),shell=True)
+
+            # concat inun poly
+            if inundation_polygon is not None:
+                _ = run('ogrmerge.py -o {} {} -f GPKG -single -overwrite_ds'.format(inundation_polygon," ".join(inundation_polys)),shell=True)
+
+        # close datasets
+        rem.close()
+        catchments.close()
+
+        return(0)
 
+    else:
+        return(1)
 
 def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profile,catchments_profile,hucCode,
                       catchmentStagesDict,depths,inundation_raster,inundation_polygon,
@@ -328,6 +332,7 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
     if isinstance(depths,DatasetWriter): depths.close()
     if isinstance(inundation_raster,DatasetWriter): inundation_raster.close()
     if isinstance(inundation_polygon,fiona.Collection): inundation_polygon.close()
+    if isinstance(hucs,fiona.Collection): inundation_polygon.close()
 
     # return file names of outputs for aggregation. Handle Nones
     try:
@@ -414,6 +419,7 @@ def __return_huc_in_hucSet(hucCode,hucSet):
 
                     rem_array,window_transform = mask(rem,catchment_poly['geometry'],crop=True,indexes=1)
                     catchments_array,_ = mask(catchments,catchment_poly['geometry'],crop=True,indexes=1)
+                    del catchment_poly
                 else:
                     print ("invalid mask type. Options are 'huc' or 'filter'")
             except ValueError: # shape doesn't overlap raster
@@ -458,78 +464,79 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
         huc_error = hydroTable.HUC.unique()
         hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
 
-        hydroTable = hydroTable[hydroTable["LakeID"] == -999]  # Subset hydroTable to include only non-lake catchments.
-
-        if hydroTable.empty:
-            print(f"All stream segments in HUC(s): {huc_error} are within lake boundaries.")
-            sys.exit(0)
-
     elif isinstance(hydroTable,pd.DataFrame):
         pass #consider checking for correct dtypes, indices, and columns
     else:
         raise TypeError("Pass path to hydro-table csv or Pandas DataFrame")
 
-    if isinstance(forecast,str):
-        forecast = pd.read_csv(
-                               forecast,
-                               dtype={'feature_id' : str , 'discharge' : float}
-                              )
-        forecast.set_index('feature_id',inplace=True)
-    elif isinstance(forecast,pd.DataFrame):
-        pass # consider checking for dtypes, indices, and columns
-    else:
-        raise TypeError("Pass path to forecast file csv or Pandas DataFrame")
-
-
-    # susbset hucs if passed
-    if subset_hucs is not None:
-        if isinstance(subset_hucs,list):
-            if len(subset_hucs) == 1:
-                try:
-                    subset_hucs = open(subset_hucs[0]).read().split('\n')
-                except FileNotFoundError:
-                    pass
-        elif isinstance(subset_hucs,str):
-                try:
-                    subset_hucs = open(subset_hucs).read().split('\n')
-                except FileNotFoundError:
-                    subset_hucs = [subset_hucs]
-
-        # subsets HUCS
-        subset_hucs_orig = subset_hucs.copy() ; subset_hucs = []
-        for huc in np.unique(hydroTable.index.get_level_values('HUC')):
-            for sh in subset_hucs_orig:
-                if huc.startswith(sh):
-                    subset_hucs += [huc]
-
-        hydroTable = hydroTable[np.in1d(hydroTable.index.get_level_values('HUC'), subset_hucs)]
-
-    # join tables
-    try:
-        hydroTable = hydroTable.join(forecast,on=['feature_id'],how='inner')
-    except AttributeError:
-        print (f"No matching feature IDs between forecast and hydrotable for HUC(s): {subset_hucs}")
-        sys.exit(0)
+    hydroTable = hydroTable[hydroTable["LakeID"] == -999]  # Subset hydroTable to include only non-lake catchments.
 
-    # initialize dictionary
-    catchmentStagesDict = typed.Dict.empty(types.int32,types.float64)
+    if not hydroTable.empty:
 
-    # interpolate stages
-    for hid,sub_table in hydroTable.groupby(level='HydroID'):
+        if isinstance(forecast,str):
+            forecast = pd.read_csv(
+                                   forecast,
+                                   dtype={'feature_id' : str , 'discharge' : float}
+                                  )
+            forecast.set_index('feature_id',inplace=True)
+        elif isinstance(forecast,pd.DataFrame):
+            pass # consider checking for dtypes, indices, and columns
+        else:
+            raise TypeError("Pass path to forecast file csv or Pandas DataFrame")
+
+        # susbset hucs if passed
+        if subset_hucs is not None:
+            if isinstance(subset_hucs,list):
+                if len(subset_hucs) == 1:
+                    try:
+                        subset_hucs = open(subset_hucs[0]).read().split('\n')
+                    except FileNotFoundError:
+                        pass
+            elif isinstance(subset_hucs,str):
+                    try:
+                        subset_hucs = open(subset_hucs).read().split('\n')
+                    except FileNotFoundError:
+                        subset_hucs = [subset_hucs]
+
+            # subsets HUCS
+            subset_hucs_orig = subset_hucs.copy() ; subset_hucs = []
+            for huc in np.unique(hydroTable.index.get_level_values('HUC')):
+                for sh in subset_hucs_orig:
+                    if huc.startswith(sh):
+                        subset_hucs += [huc]
+
+            hydroTable = hydroTable[np.in1d(hydroTable.index.get_level_values('HUC'), subset_hucs)]
+
+        # join tables
+        try:
+            hydroTable = hydroTable.join(forecast,on=['feature_id'],how='inner')
+
+
+            # initialize dictionary
+            catchmentStagesDict = typed.Dict.empty(types.int32,types.float64)
 
-        interpolated_stage = np.interp(sub_table.loc[:,'discharge'].unique(),sub_table.loc[:,'discharge_cms'],sub_table.loc[:,'stage'])
+            # interpolate stages
+            for hid,sub_table in hydroTable.groupby(level='HydroID'):
 
-        # add this interpolated stage to catchment stages dict
-        h = round(interpolated_stage[0],4)
+                interpolated_stage = np.interp(sub_table.loc[:,'discharge'].unique(),sub_table.loc[:,'discharge_cms'],sub_table.loc[:,'stage'])
 
-        hid = types.int32(hid) ; h = types.float32(h)
-        catchmentStagesDict[hid] = h
+                # add this interpolated stage to catchment stages dict
+                h = round(interpolated_stage[0],4)
 
-    # huc set
-    hucSet = [str(i) for i in hydroTable.index.get_level_values('HUC').unique().to_list()]
+                hid = types.int32(hid) ; h = types.float32(h)
+                catchmentStagesDict[hid] = h
 
-    return(catchmentStagesDict,hucSet)
+            # huc set
+            hucSet = [str(i) for i in hydroTable.index.get_level_values('HUC').unique().to_list()]
 
+            return(catchmentStagesDict,hucSet)
+
+        except AttributeError:
+            print (f"No matching feature IDs between forecast and hydrotable for HUC(s): {subset_hucs}")
+            return(None,None)
+    else:
+        print(f"All stream segments in HUC(s): {huc_error} are within lake boundaries.")
+        return(None,None)
 
 def __vprint(message,verbose):
     if verbose:
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index e3168a422..3b0f2ff1f 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -132,38 +132,41 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
             # Run inundate.
             print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
             try:
-                inundate(
+                inundate_test = inundate(
                          rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
                          subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
                          depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
                         )
-
-                print("-----> Inundation mapping complete.")
-                predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
-
-                # Define outputs for agreement_raster, stats_json, and stats_csv.
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-                else:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-
-                compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                       benchmark_raster_path,
-                                                       agreement_raster,
-                                                       stats_csv=stats_csv,
-                                                       stats_json=stats_json,
-                                                       mask_values=[],
-                                                       stats_modes_list=stats_modes_list,
-                                                       test_id=test_id,
-                                                       mask_dict=mask_dict,
-                                                       )
-
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    del mask_dict[ahps_lid]
-
-                print(" ")
-                print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
-                print(" ")
+                if inundate_test == 0:
+                    print("-----> Inundation mapping complete.")
+                    predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
+
+                    # Define outputs for agreement_raster, stats_json, and stats_csv.
+                    if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                        agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+                    else:
+                        agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+
+                    compute_contingency_stats_from_rasters(predicted_raster_path,
+                                                           benchmark_raster_path,
+                                                           agreement_raster,
+                                                           stats_csv=stats_csv,
+                                                           stats_json=stats_json,
+                                                           mask_values=[],
+                                                           stats_modes_list=stats_modes_list,
+                                                           test_id=test_id,
+                                                           mask_dict=mask_dict,
+                                                           )
+
+                    if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                        del mask_dict[ahps_lid]
+
+                    print(" ")
+                    print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
+                    print(" ")
+                elif inundate_test == 1:
+                    print (f"No matching feature IDs between forecast and hydrotable for magnitude: {magnitude}")
+                    return
             except Exception as e:
                 print(e)
 

From 9518cbe20383ebbff4d2dbfbf7e658ed112835e3 Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Fri, 30 Apr 2021 15:41:29 -0500
Subject: [PATCH 071/359] Refactor NHDPlus HR preprocessing workflow

Refactor NHDPlus HR preprocessing workflow.

- Consolidate NHD streams, NWM catchments, and headwaters MS and FR layers with mainstem column.
- HUC8 intersections are included in the input headwaters layer.
- clip_vectors_to_wbd.py removes incoming stream segment from the selected layers.

This resolves #238.
---
 CHANGELOG.md                     |  10 +
 fim_run.sh                       |  15 +-
 src/add_crosswalk.py             |   1 +
 src/adjust_headwater_streams.py  | 237 ++++++++++-------
 src/aggregate_fim_outputs.py     |  18 +-
 src/aggregate_vector_inputs.py   | 435 +++++++++++++++----------------
 src/agreedem.py                  |   2 -
 src/clip_vectors_to_wbd.py       |  81 +++---
 src/reduce_nhd_stream_density.py | 129 +++++----
 src/run_by_unit.sh               |  17 +-
 src/usgs_gage_crosswalk.py       |   2 +
 src/utils/shared_functions.py    |   1 +
 src/utils/shared_variables.py    |  27 ++
 tools/rating_curve_comparison.py |   1 +
 14 files changed, 539 insertions(+), 437 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 82b0f0cd3..4c6e3ade4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.15.8 - 2021-04-29 - [PR #371](https://github.com/NOAA-OWP/cahaba/pull/371)
+
+Refactor NHDPlus HR preprocessing workflow. Resolves issue #238
+
+## Changes
+- Consolidate NHD streams, NWM catchments, and headwaters MS and FR layers with `mainstem` column.
+- HUC8 intersections are included in the input headwaters layer.
+- `clip_vectors_to_wbd.py` removes incoming stream segment from the selected layers. 
+
 <br/><br/>
 ## v3.0.15.7 - 2021-04-28 - [PR #367](https://github.com/NOAA-OWP/cahaba/pull/367)
 
diff --git a/fim_run.sh b/fim_run.sh
index 8d1875e5f..2cfc744e2 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -111,16 +111,11 @@ logFile=$outputRunDataDir/logs/summary.log
 
 ## Define inputs
 export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
-export input_NWM_Lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
-export input_NWM_Catchments_fr=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
-export input_NWM_Catchments_ms=$inputDataDir/nwm_hydrofabric/nwm_catchments_ms.gpkg
-export input_NWM_Flows_fr=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_NWM_Flows_ms=$inputDataDir/nwm_hydrofabric/nwm_flows_ms.gpkg
-export input_NWM_Headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
-export input_nhd_flowlines_fr=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_fr_adjusted.gpkg
-export input_nhd_flowlines_ms=$inputDataDir/nhdplus_vectors_aggregate/NHDPlusBurnLineEvent_ms_adjusted.gpkg
-export input_nhd_headwaters_fr=$inputDataDir/nhdplus_vectors_aggregate/nhd_headwaters_adjusted_fr.gpkg
-export input_nhd_headwaters_ms=$inputDataDir/nhdplus_vectors_aggregate/nhd_headwaters_adjusted_ms.gpkg
+export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
+export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
+export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
+export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
+export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
 
 ## Input handling ##
 $srcDir/check_huc_inputs.py -u "$hucList"
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 2e7fbccbd..ef21ea23a 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -52,6 +52,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     elif extent == 'MS':
         ## crosswalk using stream segment midpoint method
         input_nwmcat = gpd.read_file(input_nwmcat_fileName, mask=input_huc)
+        input_nwmcat = input_nwmcat.loc[input_nwmcat.mainstem==1]
         input_nwmcat = input_nwmcat.rename(columns={'ID':'feature_id'})
         if input_nwmcat.feature_id.dtype != 'int': input_nwmcat.feature_id = input_nwmcat.feature_id.astype(int)
         input_nwmcat=input_nwmcat.set_index('feature_id')
diff --git a/src/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
index e08bf3352..71f73186e 100644
--- a/src/adjust_headwater_streams.py
+++ b/src/adjust_headwater_streams.py
@@ -3,8 +3,6 @@
 import geopandas as gpd
 import pandas as pd
 import numpy as np
-from os.path import splitext
-from tqdm import tqdm
 import argparse
 import pygeos
 from shapely.geometry import Point,LineString
@@ -12,131 +10,176 @@
 from shapely.wkb import dumps, loads
 from utils.shared_variables import PREP_PROJECTION
 from utils.shared_functions import getDriver
+import warnings
+warnings.simplefilter("ignore")
 
-def adjust_headwaters(huc,nhd_streams,headwaters,headwater_id):
 
-    # identify true headwater segments
-    if nhd_streams['headwaters_id'].dtype=='int':
-        nhd_streams_adj = nhd_streams.loc[(nhd_streams.headwaters_id > 0) & (nhd_streams.downstream_of_headwater == False),:].copy()
-        if headwaters[headwater_id].dtype != 'int': headwaters[headwater_id] = headwaters[headwater_id].astype(int)
-    else:
-        nhd_streams_adj = nhd_streams.loc[(nhd_streams.headwaters_id.notna()) & (nhd_streams.downstream_of_headwater == False),:].copy()
+def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
+    # Identify true headwater segments
+    nhd_streams_adj = nhd_streams.loc[(nhd_streams.headwaters_id > 0) & (nhd_streams.downstream_of_headwater == False),:].copy()
     nhd_streams_adj = nhd_streams_adj.explode()
     nhd_streams_adj = nhd_streams_adj.reset_index(drop=True)
 
-    headwater_limited = headwaters.merge(nhd_streams_adj["headwaters_id"],left_on=headwater_id, right_on="headwaters_id",how='right')
+    if nwm_headwaters["site_id"].dtype != 'int': nwm_headwaters["site_id"] = nwm_headwaters["site_id"].astype(int)
+    headwater_limited = nwm_headwaters.merge(nhd_streams_adj[["headwaters_id","mainstem"]],left_on="site_id", right_on="headwaters_id",how='right')
+    headwater_limited = headwater_limited.drop(columns=['headwaters_id'])
+
+    nws_lid_limited = nws_lids.merge(nhd_streams[["nws_lid"]],left_on="site_id", right_on="nws_lid",how='right')
+    nws_lid_limited = nws_lid_limited.loc[nws_lid_limited.nws_lid!='']
+    nws_lid_limited = nws_lid_limited.drop(columns=['nws_lid'])
+
+    # Check for issues in nws_lid layer
+    if len(nws_lid_limited) < len(nws_lids):
+        missing_nws_lids = list(set(nws_lids.site_id) - set(nws_lid_limited.site_id))
+        print (f"nws lid(s) {missing_nws_lids} missing from aggregate dataset in huc {huc}")
+
+    # Combine NWM headwaters and AHPS sites to be snapped to NHDPlus HR segments
+    headwater_pts = headwater_limited.append(nws_lid_limited)
+    headwater_pts = headwater_pts.reset_index(drop=True)
+
+    if headwater_pts is not None:
+
+        headwaterstreams = []
+        referencedpoints = []
+        snapped_ahps = []
+        nws_lid = []
+        for index, point in headwater_pts.iterrows():
+
+            # Convert headwaterpoint geometries to WKB representation
+            wkb_points = dumps(point.geometry)
+
+            # Create pygeos headwaterpoint geometries from WKB representation
+            pointbin_geom = pygeos.io.from_wkb(wkb_points)
+
+            if point.pt_type == 'nwm_headwater':
+                # Closest segment to headwater
+                closest_stream = nhd_streams_adj.loc[nhd_streams_adj["headwaters_id"]==point[headwater_id]]
+            else:
+                # Closest segment to ahps site
+                closest_stream = nhd_streams.loc[nhd_streams["nws_lid"]==point[headwater_id]]
+
+            try: # Seeing inconsistent geometry objects even after exploding nhd_streams_adj; not sure why this is
+                closest_stream =closest_stream.explode()
+            except:
+                pass
+
+            try:
+                wkb_closest_stream = dumps(closest_stream.geometry[0])
+            except:
+                wkb_closest_stream = dumps(closest_stream.geometry[0][0])
+
+            streambin_geom = pygeos.io.from_wkb(wkb_closest_stream)
+
+            # Linear reference headwater to closest stream segment
+            pointdistancetoline = pygeos.linear.line_locate_point(streambin_geom, pointbin_geom)
+            referencedpoint = pygeos.linear.line_interpolate_point(streambin_geom, pointdistancetoline)
+
+            # Convert geometries to wkb representation
+            bin_referencedpoint = pygeos.io.to_wkb(referencedpoint)
+
+            # Convert to shapely geometries
+            shply_referencedpoint = loads(bin_referencedpoint)
+            shply_linestring = loads(wkb_closest_stream)
+            headpoint = Point(shply_referencedpoint.coords)
+
+            if point.pt_type == 'nwm_headwater':
+                cumulative_line = []
+                relativedistlst = []
+
+                # Collect all nhd stream segment linestring verticies
+                for point in zip(*shply_linestring.coords.xy):
+                    cumulative_line = cumulative_line + [point]
+                    relativedist = shply_linestring.project(Point(point))
+                    relativedistlst = relativedistlst + [relativedist]
+
+                # Add linear referenced headwater point to closest nhd stream segment
+                if not headpoint in cumulative_line:
+                    cumulative_line = cumulative_line + [headpoint]
+                    relativedist = shply_linestring.project(headpoint)
+                    relativedistlst = relativedistlst + [relativedist]
+
+                # Sort by relative line distance to place headwater point in linestring
+                sortline = pd.DataFrame({'geom' : cumulative_line, 'dist' : relativedistlst}).sort_values('dist')
+                shply_linestring = LineString(sortline.geom.tolist())
+                referencedpoints = referencedpoints + [headpoint]
+
+                # Split the new linestring at the new headwater point
+                try:
+                    line1,line2 = split(shply_linestring, headpoint)
+                    headwaterstreams = headwaterstreams + [LineString(line1)]
+                    nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1)
+                except:
+                    line1 = split(shply_linestring, headpoint)
+                    headwaterstreams = headwaterstreams + [LineString(line1[0])]
+                    nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1[0])
+
+                try:
+                    del cumulative_line, relativedistlst
+                except:
+                    print (f"issue deleting adjusted stream variables for huc {huc}")
+
+            else:
+                snapped_ahps = snapped_ahps + [headpoint]
+                nws_lid = nws_lid + [point[headwater_id]]
+
+        nhd_streams = nhd_streams.drop(columns=['is_relevant_stream', 'headwaters_id', 'downstream_of_headwater'])
 
-    headwaterstreams = []
-    referencedpoints = []
-
-    for index, point in headwater_limited.iterrows():
-
-        # convert headwaterpoint geometries to WKB representation
-        wkb_points = dumps(point.geometry)
-
-        # create pygeos headwaterpoint geometries from WKB representation
-        pointbin_geom = pygeos.io.from_wkb(wkb_points)
-
-        # Closest segment to headwater
-        closest_stream = nhd_streams_adj.loc[nhd_streams_adj["headwaters_id"]==point[headwater_id]]
-
-        try: # seeing inconsistent geometry objects even after exploding nhd_streams_adj; not sure why this is
-            closest_stream =closest_stream.explode()
-        except:
-            pass
         try:
-            wkb_closest_stream = dumps(closest_stream.geometry[0])
+            del nhd_streams_adj, headwater_limited, referencedpoints, headwaterstreams
         except:
-            wkb_closest_stream = dumps(closest_stream.geometry[0][0])
-
-        streambin_geom = pygeos.io.from_wkb(wkb_closest_stream)
-
-        # Linear reference headwater to closest stream segment
-        pointdistancetoline = pygeos.linear.line_locate_point(streambin_geom, pointbin_geom)
-        referencedpoint = pygeos.linear.line_interpolate_point(streambin_geom, pointdistancetoline)
-
-        # convert geometries to wkb representation
-        bin_referencedpoint = pygeos.io.to_wkb(referencedpoint)
-
-        # convert to shapely geometries
-        shply_referencedpoint = loads(bin_referencedpoint)
-        shply_linestring = loads(wkb_closest_stream)
-        headpoint = Point(shply_referencedpoint.coords)
-        cumulative_line = []
-        relativedistlst = []
-
-        # collect all nhd stream segment linestring verticies
-        for point in zip(*shply_linestring.coords.xy):
-            cumulative_line = cumulative_line + [point]
-            relativedist = shply_linestring.project(Point(point))
-            relativedistlst = relativedistlst + [relativedist]
-
-        # add linear referenced headwater point to closest nhd stream segment
-        if not headpoint in cumulative_line:
-            cumulative_line = cumulative_line + [headpoint]
-            relativedist = shply_linestring.project(headpoint)
-            relativedistlst = relativedistlst + [relativedist]
-
-        # sort by relative line distance to place headwater point in linestring
-        sortline = pd.DataFrame({'geom' : cumulative_line, 'dist' : relativedistlst}).sort_values('dist')
-        shply_linestring = LineString(sortline.geom.tolist())
-        referencedpoints = referencedpoints + [headpoint]
-
-        # split the new linestring at the new headwater point
-        try:
-            line1,line2 = split(shply_linestring, headpoint)
-            headwaterstreams = headwaterstreams + [LineString(line1)]
-            nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1)
-        except:
-            line1 = split(shply_linestring, headpoint)
-            headwaterstreams = headwaterstreams + [LineString(line1[0])]
-            nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1[0])
+            print (f"issue deleting adjusted stream variables for huc {huc}")
+
+        # Create snapped ahps sites
+        if len(snapped_ahps) > 0:
+            snapped_ahps_points = gpd.GeoDataFrame({'pt_type': 'nws_lid', headwater_id: nws_lid, 'mainstem': True,
+                                                    'geometry': snapped_ahps},geometry='geometry',crs=PREP_PROJECTION)
 
-    nhd_streams = nhd_streams.drop(columns=['is_relevant_stream', 'headwaters_id', 'downstream_of_headwater'])
+        # Identify ajusted nhd headwaters
+        nhd_headwater_streams_adj = nhd_streams.loc[nhd_streams['is_headwater'],:]
+        nhd_headwater_streams_adj = nhd_headwater_streams_adj.explode()
 
-    try:
-        del nhd_streams_adj, headwaters, headwater_limited, headwaterstreams, referencedpoints, cumulative_line, relativedistlst
-    except:
-        print ('issue deleting adjusted stream variables for huc ' + str(huc))
+        hw_points = np.zeros(len(nhd_headwater_streams_adj),dtype=object)
+        for index,lineString in enumerate(nhd_headwater_streams_adj.geometry):
+            hw_point = [point for point in zip(*lineString.coords.xy)][-1]
+            hw_points[index] = Point(*hw_point)
 
-    ## identify ajusted nhd headwaters
-    # print('Identify NHD headwater points',flush=True)
-    nhd_headwater_streams_adj = nhd_streams.loc[nhd_streams['is_headwater'],:]
-    nhd_headwater_streams_adj = nhd_headwater_streams_adj.explode()
 
-    hw_points = np.zeros(len(nhd_headwater_streams_adj),dtype=object)
-    for index,lineString in enumerate(nhd_headwater_streams_adj.geometry):
-        hw_point = [point for point in zip(*lineString.coords.xy)][-1]
-        hw_points[index] = Point(*hw_point)
+        nhd_headwater_points_adj = gpd.GeoDataFrame({'pt_type': 'NHDPlusID', headwater_id: nhd_headwater_streams_adj['NHDPlusID'],
+                                                 'mainstem': False, 'geometry': hw_points},geometry='geometry',crs=PREP_PROJECTION)
 
-    nhd_headwater_points_adj = gpd.GeoDataFrame({'NHDPlusID' : nhd_headwater_streams_adj['NHDPlusID'],
-                                            'geometry' : hw_points},geometry='geometry',crs=PREP_PROJECTION)
+        nhd_headwater_points_adj = nhd_headwater_points_adj.reset_index(drop=True)
 
-    del nhd_headwater_streams_adj
+        del nhd_headwater_streams_adj
+
+        try:
+            combined_pts = snapped_ahps_points.append(nhd_headwater_points_adj)
+        except:
+            combined_pts = nhd_headwater_points_adj.copy()
 
-    return(nhd_streams, nhd_headwater_points_adj)
+        return nhd_streams, combined_pts
 
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='adjust headwater stream geometery based on headwater start points')
     parser.add_argument('-f','--huc',help='huc number',required=True)
     parser.add_argument('-l','--nhd-streams',help='NHDPlus HR geodataframe',required=True)
-    parser.add_argument('-p','--headwaters',help='Headwater points layer',required=True,type=str)
+    parser.add_argument('-p','--nwm-headwaters',help='Headwater points layer',required=True,type=str)
     parser.add_argument('-s','--subset-nhd-streams-fileName',help='Output streams layer name',required=False,type=str,default=None)
-    parser.add_argument('-s','--adj-headwater-points-fileName',help='Output adj headwater points layer name',required=False,type=str,default=None)
+    parser.add_argument('-a','--adj-headwater-points-fileName',help='Output adj headwater points layer name',required=False,type=str,default=None)
     parser.add_argument('-g','--headwater-points-fileName',help='Output headwater points layer name',required=False,type=str,default=None)
-    parser.add_argument('-i','--headwater-id',help='Output headwaters points',required=True)
+    parser.add_argument('-b','--nws-lids',help='NWS lid points',required=True)
+    parser.add_argument('-i','--headwater-id',help='Headwater id column name',required=True)
 
     args = vars(parser.parse_args())
 
-    adj_streams_gdf,adj_headwaters_gdf = adjust_headwaters(huc,nhd_streams,headwaters,headwater_id)
+    adj_streams_gdf, adj_headwaters_gdf = adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id)
 
     if subset_nhd_streams_fileName is not None:
-        adj_streams_gdf.to_file(args['subset_nhd_streams_fileName'],driver=getDriver(args['subset_nhd_streams_fileName']),index=False)
+        adj_streams_gdf.to_file(args['subset_nhd_streams_fileName'],driver=getDriver(args['subset_nhd_streams_fileName']))
 
     if headwater_points_fileName is not None:
-        headwater_points_fileName.to_file(args['headwater_points_fileName'],driver=getDriver(args['headwater_points_fileName']),index=False)
+        headwater_points_fileName.to_file(args['headwater_points_fileName'],driver=getDriver(args['headwater_points_fileName']))
 
     if adj_headwater_points_fileName is not None:
-        adj_headwaters_gdf.to_file(args['adj_headwater_points_fileName'],driver=getDriver(args['adj_headwater_points_fileName']),index=False)
+        adj_headwaters_gdf.to_file(args['adj_headwater_points_fileName'],driver=getDriver(args['adj_headwater_points_fileName']))
diff --git a/src/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py
index d6fdcf698..bf1a8d09c 100644
--- a/src/aggregate_fim_outputs.py
+++ b/src/aggregate_fim_outputs.py
@@ -88,8 +88,8 @@ def aggregate_fim_outputs(args):
 
     ## aggregate rasters
     # aggregate file paths
-    rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}_unprj.tif')
-    catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}_unprj.tif')
+    rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}_prepprj.tif')
+    catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}_prepprj.tif')
 
     if huc6 not in huc_list:
 
@@ -155,28 +155,28 @@ def aggregate_fim_outputs(args):
         shutil.copy(catchment_filename, catchment_mosaic)
 
     ## reproject rasters
-    reproject_raster(rem_mosaic)
+    reproject_raster(rem_mosaic,VIZ_PROJECTION)
     os.remove(rem_mosaic)
 
-    reproject_raster(catchment_mosaic)
+    reproject_raster(catchment_mosaic,VIZ_PROJECTION)
     os.remove(catchment_mosaic)
 
 
-def reproject_raster(raster_name):
+def reproject_raster(raster_name,reprojection):
 
     with rasterio.open(raster_name) as src:
         transform, width, height = calculate_default_transform(
-            src.crs, VIZ_PROJECTION, src.width, src.height, *src.bounds)
+            src.crs, reprojection, src.width, src.height, *src.bounds)
         kwargs = src.meta.copy()
         kwargs.update({
-            'crs': VIZ_PROJECTION,
+            'crs': reprojection,
             'transform': transform,
             'width': width,
             'height': height,
             'compress': 'lzw'
         })
 
-        raster_proj_rename = os.path.split(raster_name)[1].replace('_unprj.tif', '.tif')
+        raster_proj_rename = os.path.split(raster_name)[1].replace('_prepprj.tif', '.tif')
         raster_proj_dir = os.path.join(os.path.dirname(raster_name), raster_proj_rename)
 
         with rasterio.open(raster_proj_dir, 'w', **kwargs, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES') as dst:
@@ -187,7 +187,7 @@ def reproject_raster(raster_name):
                 src_transform=src.transform,
                 src_crs=src.crs,
                 dst_transform=transform,
-                dst_crs=VIZ_PROJECTION,
+                dst_crs=reprojection,
                 resampling=Resampling.nearest)
     del src, dst
 
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
index eb4d3e4f5..2c1081989 100755
--- a/src/aggregate_vector_inputs.py
+++ b/src/aggregate_vector_inputs.py
@@ -1,57 +1,55 @@
 #!/usr/bin/env python3
 
 import os
+import sys
+# sys.path.append('/foss_fim/src')
 import geopandas as gpd
 from utils.shared_variables import PREP_PROJECTION
 from utils.shared_functions import getDriver
 from derive_headwaters import findHeadWaterPoints
 from reduce_nhd_stream_density import subset_nhd_network
 from adjust_headwater_streams import adjust_headwaters
-from tqdm import tqdm
-from os.path import splitext
 from shapely.geometry import Point
-from concurrent.futures import ProcessPoolExecutor,as_completed
+from concurrent.futures import ProcessPoolExecutor
 from collections import deque
 import numpy as np
 from shapely.wkb import dumps, loads
 import pygeos
 
-in_dir ='data/inputs/nhdplus_vectors'
-nwm_dir = 'data/inputs/nwm_hydrofabric'
-wbd_dir = 'data/inputs/wbd'
-ahps_dir = 'data/inputs/ahp_sites'
-agg_dir = 'data/inputs/nhdplus_vectors_aggregate'
-
-wbd_filename = os.path.join(wbd_dir, 'WBD_National.gpkg')
-nwm_streams_fr_filename = os.path.join(nwm_dir,'nwm_flows.gpkg')
-nwm_streams_ms_filename = os.path.join(nwm_dir,'nwm_flows_ms.gpkg')
-nwm_headwaters_filename = os.path.join(nwm_dir,'nwm_headwaters.gpkg')
-nwm_huc4_intersections_ms_filename = os.path.join(nwm_dir,'nwm_ms_huc4_intersections.gpkg')
-nwm_huc4_intersections_fr_filename = os.path.join(nwm_dir,'nwm_fr_huc4_intersections.gpkg')
-
-def subset_nwm_ms_streams(args):
-    nwm_streams_filename    = args[0]
-    in_dir                  = args[1]
-    ahps_dir                = args[2]
-    output_filename         = args[3]
-
-    # subset nwm network to ms
-    ahps_headwaters_filename = os.path.join(ahps_dir,'bed_lids.gpkg')
-    ahps_headwaters = gpd.read_file(ahps_headwaters_filename)
+nhdplus_vectors_dir = os.environ.get('nhdplus_vectors_dir')
+wbd_filename = os.environ.get('wbd_filename')
+nwm_streams_orig_filename = os.environ.get('nwm_streams_orig_filename')
+nwm_streams_all_filename = os.environ.get('nwm_streams_all_filename')
+nwm_headwaters_filename = os.environ.get('nwm_headwaters_filename')
+nwm_catchments_orig_filename = os.environ.get('nwm_catchments_orig_filename')
+nwm_catchments_all_filename = os.environ.get('nwm_catchments_all_filename')
+ahps_filename = os.environ.get('ahps_filename')
+nwm_huc4_intersections_filename = os.environ.get('nwm_huc4_intersections_filename')
+nhd_huc8_intersections_filename = os.environ.get('nhd_huc8_intersections_filename')
+agg_nhd_headwaters_adj_fileName = os.environ['agg_nhd_headwaters_adj_fileName']
+agg_nhd_streams_adj_fileName = os.environ['agg_nhd_streams_adj_fileName']
+
+
+def identify_nwm_ms_streams(nwm_streams_filename,ahps_filename,nwm_streams_all_filename):
+
+    # Subset nwm network to ms
+    ahps_headwaters = gpd.read_file(ahps_filename)
 
     nwm_streams = gpd.read_file(nwm_streams_filename)
 
+    # Remove mainstem column if it already exists
+    nwm_streams = nwm_streams.drop(['mainstem'], axis=1, errors='ignore')
+
     nwm_streams['is_headwater'] = False
-    nwm_streams['downstream_of_headwater'] = False
 
     nwm_streams.loc[nwm_streams.ID.isin(list(ahps_headwaters.nwm_featur)),'is_headwater'] = True
 
-    ## subset NHDPlus HR
+    # Subset NHDPlus HR
     nwm_streams['is_relevant_stream'] = nwm_streams['is_headwater'].copy()
 
     nwm_streams = nwm_streams.explode()
 
-    # trace down from headwaters
+    # Trace down from headwaters
     nwm_streams.set_index('ID',inplace=True,drop=False)
 
     Q = deque(nwm_streams.loc[nwm_streams['is_headwater'],'ID'].tolist())
@@ -61,69 +59,100 @@ def subset_nwm_ms_streams(args):
         q = Q.popleft()
         if q in visited:
             continue
-        #
+
         visited.add(q)
         toNode = nwm_streams.loc[q,'to']
-        #
+
         if not toNode == 0:
-        #
+
             nwm_streams.loc[nwm_streams.ID==toNode,'is_relevant_stream'] = True
-            #
+
             if toNode not in visited:
                 Q.append(toNode)
 
-    nwm_streams = nwm_streams.loc[nwm_streams['is_relevant_stream'],:]
+    nwm_streams_ms = nwm_streams.loc[nwm_streams['is_relevant_stream'],:]
+    ms_segments = nwm_streams_ms.ID.to_list()
 
     nwm_streams.reset_index(drop=True,inplace=True)
 
-    nwm_streams.to_file(output_filename,driver=getDriver(output_filename),index=False)
+    # Add column to FR nwm layer to indicate MS segments
+    nwm_streams['mainstem'] = np.where(nwm_streams.ID.isin(ms_segments), 1, 0)
+
+    nwm_streams = nwm_streams.drop(['is_relevant_stream','is_headwater'], axis=1, errors='ignore')
 
-def find_nwm_incoming_streams(args):
+    nwm_streams.to_file(nwm_streams_all_filename,driver=getDriver(nwm_streams_all_filename),index=False,layer='nwm_streams')
 
-    nwm_streams_filename    = args[0]
-    wbd_filename            = args[1]
-    in_dir                  = args[2]
-    output_filename         = args[3]
+    return ms_segments
 
-    wbd = gpd.read_file(wbd_filename, layer='WBDHU4')
+
+def find_nwm_incoming_streams(nwm_streams_,wbd,huc_unit):
+
+    # Input wbd
+    if isinstance(wbd,str):
+        layer = f"WBDHU{huc_unit}"
+        wbd = gpd.read_file(wbd, layer=layer)
+    elif isinstance(wbd,gpd.GeoDataFrame):
+        pass
+    else:
+        raise TypeError("Pass dataframe or filepath for wbd")
 
     intersecting_points = []
-    for index, row in tqdm(wbd.iterrows(),total=len(wbd)):
-        col_name = 'HUC4'
-        huc = row[col_name]
+    nhdplus_ids = []
+    mainstem_flag = []
+    print (f"iterating through {len(wbd)} hucs")
+    for index, row in wbd.iterrows():
 
+        col_name = f"HUC{huc_unit}"
+        huc = row[col_name]
         huc_mask = wbd.loc[wbd[col_name]==str(huc)]
         huc_mask = huc_mask.explode()
         huc_mask = huc_mask.reset_index(drop=True)
 
-        nwm_streams = gpd.read_file(nwm_streams_filename, mask=huc_mask)
+        # Input nwm streams
+        if isinstance(nwm_streams_,str):
+            nwm_streams = gpd.read_file(nwm_streams_, mask=huc_mask)
+        elif isinstance(nwm_streams_,gpd.GeoDataFrame):
+            nwm_streams = nwm_streams_.copy()
+        else:
+            raise TypeError("Pass dataframe or filepath for nwm streams")
+
         nwm_streams = nwm_streams.explode()
         nwm_streams = nwm_streams.reset_index(drop=True)
 
         for index, polygon in enumerate(huc_mask.geometry):
+
             crosses=nwm_streams.crosses(polygon.exterior)
             nwm_streams_subset =nwm_streams[crosses]
             nwm_streams_subset = nwm_streams_subset.reset_index(drop=True)
 
-            for index, linestring in enumerate(nwm_streams_subset.geometry):
+            for index, segment in nwm_streams_subset.iterrows():
                 distances = []
-                # distance to each stream segment
+
+                try:
+                    nhdplus_id = segment.ID
+                except:
+                    nhdplus_id = segment.NHDPlusID
+
+                linestring = segment.geometry
+                mainstem = segment.mainstem
+
+                # Distance to each stream segment
                 for point in zip(*linestring.coords.xy):
                     distance = Point(point).distance(polygon.exterior)
                     distances = distances + [distance]
 
-                # find minimum distance
+                # Find minimum distance
                 min_index = np.argmin(distances)
 
                 # Closest segment to headwater
                 closest_point = list(linestring.coords)[min_index]
                 last_node = Point(closest_point)
 
-                # convert geometries to WKB representation
+                # Convert geometries to WKB representation
                 wkb_point = dumps(last_node)
                 wkb_poly = dumps(polygon.exterior)
 
-                # create pygeos geometries from WKB representation
+                # Create pygeos geometries from WKB representation
                 stream_point_geom = pygeos.io.from_wkb(wkb_point)
                 polybin_geom = pygeos.io.from_wkb(wkb_poly)
 
@@ -131,32 +160,35 @@ def find_nwm_incoming_streams(args):
                 pointdistancetoline = pygeos.linear.line_locate_point(polybin_geom,stream_point_geom)
                 referencedpoint = pygeos.linear.line_interpolate_point(polybin_geom, pointdistancetoline)
 
-                # convert geometries to wkb representation
+                # Convert geometries to wkb representation
                 bin_referencedpoint = pygeos.io.to_wkb(referencedpoint)
 
-                # convert to shapely geometries
+                # Convert to shapely geometries
                 shply_referencedpoint = loads(bin_referencedpoint)
 
-                # collect all nhd stream segment linestring verticies
+                # Collect all nhd stream segment linestring verticies
                 intersecting_points = intersecting_points + [shply_referencedpoint]
+                nhdplus_ids = nhdplus_ids + [nhdplus_id]
+                mainstem_flag = mainstem_flag + [mainstem]
+
+        del huc_mask
 
-    huc_intersection = gpd.GeoDataFrame({'geometry' : intersecting_points},crs=nwm_streams.crs,geometry='geometry')
+    huc_intersection = gpd.GeoDataFrame({'geometry': intersecting_points, 'NHDPlusID': nhdplus_ids,'mainstem': mainstem_flag},crs=nwm_streams.crs,geometry='geometry')
     huc_intersection = huc_intersection.drop_duplicates()
-    huc_intersection.to_file(output_filename,driver=getDriver(output_filename))
+    del nwm_streams,wbd
 
+    return huc_intersection
 
-def collect_stream_attributes(args, huc):
-    print ('Starting huc: ' + str(huc))
-    in_dir = args[0]
-    nwm_dir = args[1]
-    ahps_dir = args[2]
 
-    print ('Collecting NHDPlus HR attributes')
-    burnline_filename = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')
-    vaa_filename = os.path.join(in_dir,huc,'NHDPlusFlowLineVAA' + str(huc) + '.gpkg')
-    flowline_filename = os.path.join(in_dir,huc,'NHDFlowline' + str(huc) + '.gpkg')
+def collect_stream_attributes(nhdplus_vectors_dir, huc):
+
+    print ('Starting huc: ' + str(huc))
+    # Collecting NHDPlus HR attributes
+    burnline_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')
+    vaa_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusFlowLineVAA' + str(huc) + '.gpkg')
+    flowline_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDFlowline' + str(huc) + '.gpkg')
 
-    if os.path.exists(os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')):
+    if os.path.exists(os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')):
 
         burnline = gpd.read_file(burnline_filename)
         burnline = burnline[['NHDPlusID','ReachCode','geometry']]
@@ -177,8 +209,8 @@ def collect_stream_attributes(args, huc):
         nhd_streams = nhd_streams.loc[nhd_streams.geometry!=None,:] # special case: remove segments without geometries
         nhd_streams['HUC4'] = str(huc)
 
-        # write out NHDPlus HR aggregated
-        nhd_streams_agg_fileName = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
+        # Write out NHDPlus HR aggregated
+        nhd_streams_agg_fileName = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
         nhd_streams.to_file(nhd_streams_agg_fileName,driver=getDriver(nhd_streams_agg_fileName),index=False)
         del nhd_streams
 
@@ -187,226 +219,185 @@ def collect_stream_attributes(args, huc):
     else:
         print ('missing data for huc ' + str(huc))
 
+
 def subset_stream_networks(args, huc):
 
-    nwm_dir                            = args[0]
-    ahps_dir                           = args[1]
+    nwm_headwaters_filename            = args[0]
+    ahps_filename                      = args[1]
     wbd4                               = args[2]
     wbd8                               = args[3]
-    in_dir                             = args[4]
-    nwm_huc4_intersect_fr_filename     = args[5]
-    nwm_huc4_intersect_ms_filename     = args[6]
+    nhdplus_vectors_dir                = args[4]
+    nwm_huc4_intersections_filename    = args[5]
 
     print("starting HUC " + str(huc),flush=True)
     nwm_headwater_id = 'ID'
-    nwm_headwaters_filename = os.path.join(nwm_dir,'nwm_headwaters.gpkg')
     ahps_headwater_id = 'nws_lid'
-    ahps_headwaters_filename = os.path.join(ahps_dir,'nws_lid.gpkg')
-    nhd_streams_filename = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
+    headwater_pts_id = 'site_id'
+    column_order = ['pt_type', headwater_pts_id, 'mainstem', 'geometry']
+    nhd_streams_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
 
-    # subset to reduce footprint
-    selected_wbd4 = wbd4.loc[wbd4.HUC4.str.startswith(str(huc))]
+    # Subset to reduce footprint
+    selected_wbd4 = wbd4.loc[wbd4.HUC4.str.startswith(huc)]
     del wbd4
     selected_wbd8 = wbd8.loc[wbd8.HUC8.str.startswith(huc)]
     del wbd8
 
-    huc_mask = selected_wbd4.loc[selected_wbd4.HUC4.str.startswith(str(huc))]
+    huc_mask = selected_wbd4.loc[selected_wbd4.HUC4.str.startswith(huc)]
     huc_mask = huc_mask.explode()
     huc_mask = huc_mask.reset_index(drop=True)
 
     if len(selected_wbd8.HUC8) > 0:
         selected_wbd8 = selected_wbd8.reset_index(drop=True)
 
-        # identify FR/NWM headwaters
-        nhd_streams_fr = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,nwm_headwaters_filename,nwm_headwater_id,nwm_huc4_intersect_fr_filename)
-
-        ## adjust FR/NWM headwater segments
-        nwm_headwaters = gpd.read_file(nwm_headwaters_filename, mask=huc_mask)
-
-        if len(nwm_headwaters) > 0:
-
-            adj_nhd_streams_fr, adj_nhd_headwater_points_fr = adjust_headwaters(str(huc),nhd_streams_fr,nwm_headwaters,nwm_headwater_id)
+        # Identify FR/NWM headwaters and subset HR network
+        nhd_streams_fr = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,nwm_headwaters_filename,nwm_headwater_id,nwm_huc4_intersections_filename)
 
-            nhd_streams_fr_adjusted_fileName=os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr_adjusted.gpkg')
-            adj_nhd_headwaters_fr_fileName=os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_fr.gpkg')
+        # Identify nhd mainstem streams
+        nhd_streams_all = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_fr,ahps_filename,ahps_headwater_id,nwm_huc4_intersections_filename,True)
 
-            # write out FR adjusted
-            adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,driver=getDriver(nhd_streams_fr_adjusted_fileName),index=False)
-            adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,driver=getDriver(adj_nhd_headwaters_fr_fileName),index=False)
-
-            del adj_nhd_streams_fr, adj_nhd_headwater_points_fr
-        else:
-            print ('skipping FR headwater adjustments for HUC: ' + str(huc))
+        # Identify HUC8 intersection points
+        nhd_huc8_intersections = find_nwm_incoming_streams(nhd_streams_all,selected_wbd8,8)
 
-        del nhd_streams_fr
-
-        ## identify MS/AHPs headwaters
-        nhd_streams_ms = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,ahps_headwaters_filename,ahps_headwater_id,nwm_huc4_intersect_ms_filename)
+        # Load nwm headwaters
+        nwm_headwaters = gpd.read_file(nwm_headwaters_filename, mask=huc_mask)
+        nwm_headwaters['pt_type'] = 'nwm_headwater'
+        nwm_headwaters = nwm_headwaters.rename(columns={"ID": headwater_pts_id})
 
-        ## adjust MS/AHPs headwater segments
-        ahps_headwaters = gpd.read_file(ahps_headwaters_filename, mask=huc_mask)
+        # Load nws lids
+        nws_lids = gpd.read_file(ahps_filename, mask=huc_mask)
+        nws_lids = nws_lids.drop(columns=['name','nwm_featur'])
+        nws_lids = nws_lids.rename(columns={"nws_lid": headwater_pts_id})
+        nws_lids['pt_type'] = 'nws_lid'
+        nws_lids['mainstem'] = True
 
-        if len(ahps_headwaters) > 0:
+        if (len(nwm_headwaters) > 0) or (len(nws_lids) > 0):
+            # Adjust FR/NWM headwater segments
+            adj_nhd_streams_all, adj_nhd_headwater_points = adjust_headwaters(huc,nhd_streams_all,nwm_headwaters,nws_lids,headwater_pts_id)
 
-            adj_nhd_streams_ms, adj_nhd_headwater_points_ms = adjust_headwaters(str(huc),nhd_streams_ms,ahps_headwaters,ahps_headwater_id)
+            adj_nhd_headwater_points = adj_nhd_headwater_points[column_order]
 
-            nhd_streams_ms_adjusted_fileName=os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms_adjusted.gpkg')
-            adj_nhd_headwaters_ms_fileName=os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_ms.gpkg')
+            nhd_huc8_intersections['pt_type'] = 'nhd_huc8_intersections'
+            nhd_huc8_intersections = nhd_huc8_intersections.rename(columns={"NHDPlusID": headwater_pts_id})
+            nhd_huc8_intersections = nhd_huc8_intersections[column_order]
+            adj_nhd_headwater_points_all = adj_nhd_headwater_points.append(nhd_huc8_intersections)
+            adj_nhd_headwater_points_all = adj_nhd_headwater_points_all.reset_index(drop=True)
 
-            # write out MS adjusted
-            adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,driver=getDriver(nhd_streams_ms_adjusted_fileName),index=False)
-            adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,driver=getDriver(adj_nhd_headwaters_ms_fileName),index=False)
+            adj_nhd_streams_all_fileName = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
+            adj_nhd_headwaters_all_fileName = os.path.join(nhdplus_vectors_dir,huc,'nhd' + str(huc) + '_headwaters_adj.gpkg')
 
-            del adj_nhd_streams_ms, adj_nhd_headwater_points_ms
+            # Write out FR adjusted
+            adj_nhd_streams_all.to_file(adj_nhd_streams_all_fileName,driver=getDriver(adj_nhd_streams_all_fileName),index=False)
+            adj_nhd_headwater_points_all.to_file(adj_nhd_headwaters_all_fileName,driver=getDriver(adj_nhd_headwaters_all_fileName),index=False)
 
+            del adj_nhd_streams_all, adj_nhd_headwater_points_all
         else:
-            print ('skipping MS headwater adjustments for HUC: ' + str(huc))
-            del nhd_streams_ms
+            print (f"skipping headwater adjustments for HUC: {huc}")
 
-def aggregate_stream_networks(in_dir,agg_dir, huc_list):
-
-    for huc in huc_list:
+        del nhd_streams_fr
 
-        ## FR adjusted
-        adj_nhd_headwaters_fr_fileName=os.path.join(agg_dir,'nhd_headwaters_adjusted_fr.gpkg')
-        nhd_fr_adj_huc_subset = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr_adjusted.gpkg')
-        nhd_streams_fr_adjusted_fileName=os.path.join(agg_dir,'NHDPlusBurnLineEvent_fr_adjusted.gpkg')
-        nhd_fr_adj_headwaters_subset = os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_fr.gpkg')
 
-        if os.path.isfile(nhd_fr_adj_huc_subset):
-            adj_nhd_streams_fr = gpd.read_file(nhd_fr_adj_huc_subset)
+def aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,huc_list):
 
-            # write out FR adjusted
-            if os.path.isfile(nhd_streams_fr_adjusted_fileName):
-                adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,driver=getDriver(nhd_streams_fr_adjusted_fileName),index=False, mode='a')
-            else:
-                adj_nhd_streams_fr.to_file(nhd_streams_fr_adjusted_fileName,driver=getDriver(nhd_streams_fr_adjusted_fileName),index=False)
+    for huc in huc_list:
 
-            del adj_nhd_streams_fr
+        # aggregated final filenames
+        nhd_agg_adj_huc_subset = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
+        nhd_agg_adj_headwaters_subset = os.path.join(nhdplus_vectors_dir,huc,'nhd' + str(huc) + '_headwaters_adj.gpkg')
 
-        if os.path.isfile(nhd_fr_adj_headwaters_subset):
-            adj_nhd_headwater_points_fr = gpd.read_file(nhd_fr_adj_headwaters_subset)
+        if os.path.isfile(nhd_agg_adj_huc_subset):
+            adj_nhd_streams_all = gpd.read_file(nhd_agg_adj_huc_subset)
 
-            # write out FR adjusted
-            if os.path.isfile(adj_nhd_headwaters_fr_fileName):
-                adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,driver=getDriver(adj_nhd_headwaters_fr_fileName),index=False, mode='a')
+            # Write out FR adjusted
+            if os.path.isfile(agg_nhd_streams_adj_fileName):
+                adj_nhd_streams_all.to_file(agg_nhd_streams_adj_fileName,driver=getDriver(agg_nhd_streams_adj_fileName),index=False, mode='a')
             else:
-                adj_nhd_headwater_points_fr.to_file(adj_nhd_headwaters_fr_fileName,driver=getDriver(adj_nhd_headwaters_fr_fileName),index=False)
-
-            del adj_nhd_headwater_points_fr
+                adj_nhd_streams_all.to_file(agg_nhd_streams_adj_fileName,driver=getDriver(agg_nhd_streams_adj_fileName),index=False)
 
-        ## MS adjusted
-        adj_nhd_headwaters_ms_fileName=os.path.join(agg_dir,'nhd_headwaters_adjusted_ms.gpkg')
-        nhd_ms_adj_huc_subset = os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms_adjusted.gpkg')
-        nhd_streams_ms_adjusted_fileName=os.path.join(agg_dir,'NHDPlusBurnLineEvent_ms_adjusted.gpkg')
-        nhd_ms_adj_headwater_subset = os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_ms.gpkg')
+            del adj_nhd_streams_all
 
-        if os.path.isfile(nhd_ms_adj_huc_subset):
-            adj_nhd_streams_ms = gpd.read_file(nhd_ms_adj_huc_subset)
+        if os.path.isfile(nhd_agg_adj_headwaters_subset):
+            adj_nhd_headwater_points_all = gpd.read_file(nhd_agg_adj_headwaters_subset)
 
-            # write out ms adjusted
-            if os.path.isfile(nhd_streams_ms_adjusted_fileName):
-                adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,driver=getDriver(nhd_streams_ms_adjusted_fileName),index=False, mode='a')
+            # Write out FR adjusted
+            if os.path.isfile(agg_nhd_headwaters_adj_fileName):
+                adj_nhd_headwater_points_all.to_file(agg_nhd_headwaters_adj_fileName,driver=getDriver(agg_nhd_headwaters_adj_fileName),index=False, mode='a')
             else:
-                adj_nhd_streams_ms.to_file(nhd_streams_ms_adjusted_fileName,driver=getDriver(nhd_streams_ms_adjusted_fileName),index=False)
+                adj_nhd_headwater_points_all.to_file(agg_nhd_headwaters_adj_fileName,driver=getDriver(agg_nhd_headwaters_adj_fileName),index=False)
 
-            del adj_nhd_streams_ms
+            del adj_nhd_headwater_points_all
 
-        if os.path.isfile(nhd_ms_adj_headwater_subset):
-            adj_nhd_headwater_points_ms = gpd.read_file(nhd_ms_adj_headwater_subset)
 
-            # write out ms adjusted
-            if os.path.isfile(adj_nhd_headwaters_ms_fileName):
-                adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,driver=getDriver(adj_nhd_headwaters_ms_fileName),index=False, mode='a')
-            else:
-                adj_nhd_headwater_points_ms.to_file(adj_nhd_headwaters_ms_fileName,driver=getDriver(adj_nhd_headwaters_ms_fileName),index=False)
+def clean_up_intermediate_files(nhdplus_vectors_dir):
 
-            del adj_nhd_headwater_points_ms
-
-def clean_up_intermediate_files(in_dir):
-
-    for huc in os.listdir(in_dir):
-        agg_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
-        fr_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr.gpkg')
-        fr_adj_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_fr_adjusted.gpkg')
-        ms_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms.gpkg')
-        ms_adj_path= os.path.join(in_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_ms_adjusted.gpkg')
-        ms_headwater_adj_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_ms.gpkg')
-        fr_headwater_adj_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_adjusted_fr.gpkg')
-        ms_headwater_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_ms.gpkg')
-        fr_headwater_path= os.path.join(in_dir,huc,'nhd' + str(huc) + '_headwaters_fr.gpkg')
+    for huc in os.listdir(nhdplus_vectors_dir):
+        agg_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
+        streams_adj_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
+        headwater_adj_path= os.path.join(nhdplus_vectors_dir,huc,'nhd' + str(huc) + '_headwaters_adj.gpkg')
 
         if os.path.exists(agg_path):
             os.remove(agg_path)
 
-        if os.path.exists(fr_path):
-            os.remove(fr_path)
-
-        if os.path.exists(fr_adj_path):
-            os.remove(fr_adj_path)
-
-        if os.path.exists(ms_path):
-            os.remove(ms_path)
-
-        if os.path.exists(ms_adj_path):
-            os.remove(ms_adj_path)
-
-        if os.path.exists(ms_headwater_adj_path):
-            os.remove(ms_headwater_adj_path)
-
-        if os.path.exists(fr_headwater_adj_path):
-            os.remove(fr_headwater_adj_path)
-
-        if os.path.exists(ms_headwater_path):
-            os.remove(ms_headwater_path)
-
-        if os.path.exists(fr_headwater_path):
-            os.remove(fr_headwater_path)
+        if os.path.exists(streams_adj_path):
+            os.remove(streams_adj_path)
 
+        if os.path.exists(headwater_adj_path):
+            os.remove(headwater_adj_path)
 
 
 if(__name__=='__main__'):
 
-    ## generate NWM Headwaters
+    # # Generate NWM Headwaters
     # print ('deriving nwm headwater points')
-    # nwm_headwaters = findHeadWaterPoints(nwm_streams_fr_filename)
+    # nwm_headwaters = findHeadWaterPoints(nwm_streams_orig_filename)
     # nwm_headwaters['ID'] = nwm_headwaters.index + 1
-    # nwm_headwaters.to_file(nwm_headwaters_filename,driver=getDriver(nwm_headwaters_filename),index=False)
-    
+    # nwm_headwaters.to_file(nwm_headwaters_filename,driver=getDriver(nwm_headwaters_filename),index=False,layer='nwm_headwaters')
     # del nwm_headwaters, nwm_streams
-
-    ## subset NWM MS Streams
-    # nwm_subset_ms_args = (nwm_streams_fr_filename,in_dir,ahps_dir,nwm_streams_ms_filename)
-    # print ('deriving nwm ms streams')
-    # subset_nwm_ms_streams(nwm_subset_ms_args)
-
-    ## generate NWM intersection points with WBD4 boundaries
-    # ms_nwm_intersect_args = (nwm_streams_ms_filename,wbd_filename,in_dir,nwm_huc4_intersections_ms_filename)
-    # fr_nwm_intersect_args = (nwm_streams_fr_filename,wbd_filename,in_dir,nwm_huc4_intersections_fr_filename)
-    # print ('deriving nwm ms intersection points')
-    # find_nwm_incoming_streams(ms_nwm_intersect_args)
-    # print ('deriving nwm fr intersection points')
-    # find_nwm_incoming_streams(fr_nwm_intersect_args)
-
-    print ('loading wb4')
+    #
+    # # Identify NWM MS Streams
+    # print ('identifing nwm ms streams')
+    # ms_segments = identify_nwm_ms_streams(nwm_streams_orig_filename,ahps_filename,nwm_streams_all_filename)
+    #
+    # # Identify NWM MS Catchments
+    # print ('identifing nwm ms catchments')
+    # nwm_catchments = gpd.read_file(nwm_catchments_orig_filename)
+    # # Add column to FR nwm layer to indicate MS segments
+    # nwm_catchments['mainstem'] = np.where(nwm_catchments.ID.isin(ms_segments), 1, 0)
+    # nwm_catchments.to_file(nwm_catchments_all_filename,driver=getDriver(nwm_catchments_all_filename),index=False,layer='nwm_catchments')
+    # del nwm_catchments, ms_segments
+
+    # # Generate NWM intersection points with WBD4 boundaries
+    # print ('deriving NWM fr/ms intersection points')
+    # huc4_intersection = find_nwm_incoming_streams(nwm_streams_all_filename,wbd_filename,4)
+    # huc4_intersection.to_file(nwm_huc4_intersections_filename,driver=getDriver(nwm_huc4_intersections_filename),layer='huc4_intersection')
+    # del huc4_intersection
+
+    print ('loading HUC4s')
     wbd4 = gpd.read_file(wbd_filename, layer='WBDHU4')
-    print ('loading wb8')
+    print ('loading HUC8s')
     wbd8 = gpd.read_file(wbd_filename, layer='WBDHU8')
 
-    subset_arg_list = (nwm_dir,ahps_dir,wbd4,wbd8,in_dir,nwm_huc4_intersections_fr_filename,nwm_huc4_intersections_ms_filename)
-    collect_arg_list = (in_dir,nwm_dir,ahps_dir)
+    subset_arg_list = (nwm_headwaters_filename,ahps_filename,wbd4,wbd8,nhdplus_vectors_dir,nwm_huc4_intersections_filename)
+    huc_list = os.listdir(nhdplus_vectors_dir)
+
+    missing_subsets = []
+    for huc in os.listdir(nhdplus_vectors_dir):
+        streams_adj_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
+        if not os.path.isfile(streams_adj_path):
+            missing_subsets = missing_subsets + [huc]
 
-    num_workers=9
+    print (f"running subset_results on {len(missing_subsets)} HUC4s")
+    num_workers=11
 
     with ProcessPoolExecutor(max_workers=num_workers) as executor:
-        ## preprocess nhd hr and add attributes
-        collect_attributes = [executor.submit(collect_stream_attributes, collect_arg_list, str(huc)) for huc in os.listdir(in_dir)]
-        ## subset nhd hr network
-        subset_results = [executor.submit(subset_stream_networks, subset_arg_list, str(huc)) for huc in os.listdir(in_dir)]
+        # Preprocess nhd hr and add attributes
+        # collect_attributes = [executor.submit(collect_stream_attributes, nhdplus_vectors_dir, str(huc)) for huc in huc_list]
+        # Subset nhd hr network
+        subset_results = [executor.submit(subset_stream_networks, subset_arg_list, str(huc)) for huc in missing_subsets]
 
+    # del wbd4,wbd8
 
-    ## aggregate fr and ms nhd netowrks for entire nwm domain
-    aggregate_stream_networks(in_dir,agg_dir, os.listdir(in_dir))
+    # Aggregate fr and ms nhd netowrks for entire nwm domain
+    aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,huc_list)
 
-    ## remove intermediate files
-    # clean_up_intermediate_files(in_dir)
+    # Remove intermediate files
+    # clean_up_intermediate_files(nhdplus_vectors_dir)
diff --git a/src/agreedem.py b/src/agreedem.py
index 15ae40c4c..dbff2d2d4 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -45,8 +45,6 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
     # Import dem layer and river layer and get dem profile.
     elev = rasterio.open(dem)
     dem_profile = elev.profile
-    if elev.nodata == 0.0:
-        dem_profile.update(nodata = -999)
 
     rivers = rasterio.open(rivers_raster)
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 654fe6e4f..f29217e76 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -7,7 +7,7 @@
 from shapely.geometry import MultiPolygon,Polygon,Point
 from utils.shared_functions import getDriver
 
-def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks=False):
+def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,dissolveLinks=False):
 
     hucUnitLength = len(str(hucCode))
 
@@ -22,76 +22,80 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
         landsea.to_file(subset_landsea_filename,driver=getDriver(subset_landsea_filename),index=False)
     del landsea
 
-    # find intersecting lakes and writeout
+    # Find intersecting lakes and writeout
     print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
     nwm_lakes = gpd.read_file(nwm_lakes_filename, mask = wbd_buffer)
 
     if not nwm_lakes.empty:
-        # perform fill process to remove holes/islands in the NWM lake polygons
+        # Perform fill process to remove holes/islands in the NWM lake polygons
         nwm_lakes = nwm_lakes.explode()
         nwm_lakes_fill_holes=MultiPolygon(Polygon(p.exterior) for p in nwm_lakes['geometry']) # remove donut hole geometries
-        # loop through the filled polygons and insert the new geometry
+        # Loop through the filled polygons and insert the new geometry
         for i in range(len(nwm_lakes_fill_holes)):
             nwm_lakes.loc[i,'geometry'] = nwm_lakes_fill_holes[i]
         nwm_lakes.to_file(subset_nwm_lakes_filename,driver=getDriver(subset_nwm_lakes_filename),index=False)
     del nwm_lakes
 
-    # find intersecting levee lines
+    # Find intersecting levee lines
     print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
     nld_lines = gpd.read_file(nld_lines_filename, mask = wbd_buffer)
     if not nld_lines.empty:
         nld_lines.to_file(subset_nld_lines_filename,driver=getDriver(subset_nld_lines_filename),index=False)
     del nld_lines
 
-    # find intersecting nwm_catchments
-    print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_catchments = gpd.read_file(nwm_catchments_filename, mask = wbd_buffer)
-    nwm_catchments.to_file(subset_nwm_catchments_filename,driver=getDriver(subset_nwm_catchments_filename),index=False)
-    del nwm_catchments
-
-    # subset nhd headwaters
+    # Subset nhd headwaters
     print("Subsetting NHD Headwater Points for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
     nhd_headwaters = gpd.read_file(nhd_headwaters_filename, mask = wbd_buffer)
+    if extent == 'MS':
+        nhd_headwaters = nhd_headwaters.loc[nhd_headwaters.mainstem==1]
 
-    # subset nhd streams
+    if len(nhd_headwaters) > 0:
+        nhd_headwaters.to_file(subset_nhd_headwaters_filename,driver=getDriver(subset_nhd_headwaters_filename),index=False)
+    else:
+        print ("No headwater point(s) within HUC " + str(hucCode) +  " boundaries.")
+        sys.exit(0)
+    del nhd_headwaters
+
+    # Subset nhd streams
     print("Querying NHD Streams for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
     nhd_streams = gpd.read_file(nhd_streams_filename, mask = wbd_buffer)
+    if extent == 'MS':
+        nhd_streams = nhd_streams.loc[nhd_streams.mainstem==1]
 
-    ## identify local headwater stream segments
-    nhd_streams_subset = gpd.read_file(nhd_streams_filename, mask = wbd)
-    nhd_streams_subset = nhd_streams_subset.loc[~nhd_streams_subset.FromNode.isin(list(set(nhd_streams_subset.ToNode) & set(nhd_streams_subset.FromNode)))]
-    nhd_streams_subset = nhd_streams_subset[~nhd_streams_subset['is_headwater']]
-
-    if not nhd_streams_subset.empty:
-        nhd_streams_subset = nhd_streams_subset.reset_index(drop=True)
-        start_coords = []
-        NHDPlusIDs = []
-        for index, linestring in enumerate(nhd_streams_subset.geometry):
-            start_coords = start_coords + [linestring.coords[-1]]
-            NHDPlusIDs = NHDPlusIDs + [nhd_streams_subset.iloc[index].NHDPlusID]
-
-        start_geoms = [Point(point) for point in start_coords]
-        local_headwaters = gpd.GeoDataFrame({'NHDPlusID': NHDPlusIDs,'geometry': start_geoms}, crs=projection, geometry='geometry')
-        nhd_headwaters = nhd_headwaters.append(local_headwaters)
+    if len(nhd_streams) > 0:
+        # Find incoming stream segments (to WBD buffer) and identify which are upstream
+        threshold_segments = gpd.overlay(nhd_streams, wbd_buffer, how='symmetric_difference')
+        from_list = threshold_segments.FromNode.to_list()
+        to_list = nhd_streams.ToNode.to_list()
+        missing_segments = list(set(from_list) - set(to_list))
 
-        # nhd_streams = nhd_streams.loc[~nhd_streams.NHDPlusID.isin(NHDPlusIDs)]
+        # Remove incoming stream segment so it won't be routed as outflow during hydroconditioning
+        nhd_streams = nhd_streams.loc[~nhd_streams.FromNode.isin(missing_segments)]
 
-    if len(nhd_streams) > 0:
         nhd_streams.to_file(subset_nhd_streams_filename,driver=getDriver(subset_nhd_streams_filename),index=False)
     else:
         print ("No NHD streams within HUC " + str(hucCode) +  " boundaries.")
         sys.exit(0)
+    del nhd_streams
 
-    if len(nhd_headwaters) > 0:
-        nhd_headwaters.to_file(subset_nhd_headwaters_filename,driver=getDriver(subset_nhd_headwaters_filename),index=False)
-        del nhd_headwaters, nhd_streams
+    # Find intersecting nwm_catchments
+    print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+    nwm_catchments = gpd.read_file(nwm_catchments_filename, mask = wbd_buffer)
+    if extent == 'MS':
+        nwm_catchments = nwm_catchments.loc[nwm_catchments.mainstem==1]
+
+    if len(nwm_catchments) > 0:
+        nwm_catchments.to_file(subset_nwm_catchments_filename,driver=getDriver(subset_nwm_catchments_filename),index=False)
     else:
-        print ("No headwater point(s) within HUC " + str(hucCode) +  " boundaries.")
+        print ("No NHD catchments within HUC " + str(hucCode) +  " boundaries.")
         sys.exit(0)
+    del nwm_catchments
 
-    # subset nwm streams
+    # Subset nwm streams
     print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
     nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd_buffer)
+    if extent == 'MS':
+        nwm_streams = nwm_streams.loc[nwm_streams.mainstem==1]
     nwm_streams.to_file(subset_nwm_streams_filename,driver=getDriver(subset_nwm_streams_filename),index=False)
     del nwm_streams
 
@@ -115,8 +119,10 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     parser.add_argument('-e','--subset-nhd-headwaters',help='NHD headwaters subset',required=True,default=None)
     parser.add_argument('-b','--subset-nwm-streams',help='NWM streams subset',required=True)
     parser.add_argument('-x','--subset-landsea',help='LandSea subset',required=True)
+    parser.add_argument('-extent','--extent',help='FIM extent',required=True)
     parser.add_argument('-o','--dissolve-links',help='remove multi-line strings',action="store_true",default=False)
 
+
     args = vars(parser.parse_args())
 
     hucCode = args['hucCode']
@@ -136,6 +142,7 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     subset_nhd_headwaters_filename = args['subset_nhd_headwaters']
     subset_nwm_streams_filename = args['subset_nwm_streams']
     subset_landsea_filename = args['subset_landsea']
+    extent = args['extent']
     dissolveLinks = args['dissolve_links']
 
-    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,dissolveLinks)
+    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,dissolveLinks)
diff --git a/src/reduce_nhd_stream_density.py b/src/reduce_nhd_stream_density.py
index cce2fa7ca..9614bfe32 100644
--- a/src/reduce_nhd_stream_density.py
+++ b/src/reduce_nhd_stream_density.py
@@ -3,114 +3,139 @@
 import geopandas as gpd
 import pandas as pd
 import numpy as np
-from os.path import splitext
-from tqdm import tqdm
 from collections import deque
 import argparse
 import pygeos
 from shapely.wkb import dumps
 from utils.shared_functions import getDriver
 
-def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_filename,headwaters_filename,headwater_id,nwm_intersections_filename):
 
-    headwater_streams = pd.DataFrame()
+def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_filename,headwater_id,nwm_intersections_filename,mainstem_flag=False):
 
-    nhd_streams = gpd.read_file(nhd_streams_filename)
+    headwater_streams = pd.DataFrame()
 
+    if mainstem_flag == False:
+        nhd_streams = gpd.read_file(nhd_streams_)
+        headwater_col = 'is_headwater'
+        id_col = 'headwaters_id'
+        n = -1
+    else:
+        nhd_streams = nhd_streams_.copy()
+        headwater_col = 'mainstem'
+        id_col = 'nws_lid'
+        n = ''
+
+    # Locate the closest NHDPlus HR stream segment to NWM headwater points. Done by HUC8 to reduce processing time and to contain NWM headwater in the same HUC
     for index, row in selected_wbd8.iterrows():
         huc = row["HUC8"]
 
+        # Double check that this is a nested HUC (probably overkill)
         if huc.startswith(str(huc4)):
-            huc8_mask = selected_wbd8.loc[selected_wbd8.HUC8.str.startswith(huc)]
+
+            huc8_mask = selected_wbd8.loc[selected_wbd8.HUC8==huc]
             huc8_mask = huc8_mask.reset_index(drop=True)
 
+            # Masking headwaters by HUC8
             headwaters_mask = gpd.read_file(headwaters_filename, mask = huc8_mask)
             headwaters_mask = headwaters_mask.reset_index(drop=True)
 
-            streams_subset = gpd.read_file(nhd_streams_filename, mask = huc8_mask)
+            # Masking subset streams by HUC8
+            if mainstem_flag == False:
+                streams_subset = gpd.read_file(nhd_streams_, mask = huc8_mask)
+            else:
+                streams_subset = nhd_streams.loc[nhd_streams.HUC8==huc].copy()
 
             if not streams_subset.empty:
-                streams_subset.loc[:,'is_headwater'] = False
+                streams_subset[headwater_col] = False
                 streams_subset = streams_subset.reset_index(drop=True)
 
+                # Create WKB geometry column
                 streams_subset['b_geom'] = None
                 for index, linestring in enumerate(streams_subset.geometry):
                     streams_subset.at[index, 'b_geom'] = dumps(linestring)
 
-                # create pygeos nhd stream geometries from WKB representation
+                # Create pygeos nhd stream geometries from WKB representation
                 streambin_geom = pygeos.io.from_wkb(streams_subset['b_geom'])
 
-                streams_subset.loc[:,'HUC8'] = str(huc)
-
-                if headwaters_mask[headwater_id].dtype=='int':
-                    n = -1
-                else:
-                    n = ''
+                # Add HUC8 column
+                streams_subset['HUC8'] = str(huc)
 
-                streams_subset.loc[:,'headwaters_id'] = n
+                # Add headwaters_id column
+                streams_subset[id_col] = n
 
-                # find stream segment closest to headwater point
+                # Find stream segment closest to headwater point
                 for index, point in headwaters_mask.iterrows():
 
-                    # convert headwaterpoint geometries to WKB representation
+                    # Convert headwaterpoint geometries to WKB representation
                     wkb_points = dumps(point.geometry)
 
-                    # create pygeos headwaterpoint geometries from WKB representation
+                    # Create pygeos headwaterpoint geometries from WKB representation
                     pointbin_geom = pygeos.io.from_wkb(wkb_points)
 
-                    # distance to each stream segment
+                    # Distance to each stream segment
                     distances = pygeos.measurement.distance(streambin_geom, pointbin_geom)
 
-                    # find minimum distance
+                    # Find minimum distance
                     min_index = np.argmin(distances)
 
                     # Closest segment to headwater
-                    streams_subset.loc[min_index,'is_headwater'] = True
-                    streams_subset.loc[min_index,'headwaters_id'] = point[headwater_id]
+                    streams_subset.loc[min_index,headwater_col] = True
+                    streams_subset.loc[min_index,id_col] = point[headwater_id]
 
-                headwater_streams = headwater_streams.append(streams_subset[['NHDPlusID','is_headwater','headwaters_id','HUC8']])
+                headwater_streams = headwater_streams.append(streams_subset[['NHDPlusID',headwater_col,id_col,'HUC8']])
 
-    headwater_streams = headwater_streams.sort_values('is_headwater', ascending=False).drop_duplicates('NHDPlusID') # keeps headwater=True for conflicting duplicates
-    nhd_streams = nhd_streams.merge(headwater_streams,on='NHDPlusID',how='inner')
+    headwater_streams = headwater_streams.sort_values(headwater_col, ascending=False).drop_duplicates('NHDPlusID') # keeps headwater=True for conflicting duplicates
+
+    if mainstem_flag == False:
+        nhd_streams = nhd_streams.merge(headwater_streams,on='NHDPlusID',how='inner')
+    else:
+        headwater_streams = headwater_streams.drop(columns=['HUC8'])
+        nhd_streams = nhd_streams.merge(headwater_streams,on='NHDPlusID',how='outer')
+        nhd_streams[id_col] = nhd_streams[id_col].fillna(n)
+        nhd_streams[headwater_col] = nhd_streams[headwater_col].fillna(0)
 
     del selected_wbd8, streams_subset, headwater_streams
 
     huc4_mask_buffer = huc4_mask.buffer(10)
 
-    # identify inflowing streams
+    # Identify inflowing streams
     nwm_intersections = gpd.read_file(nwm_intersections_filename, mask=huc4_mask_buffer)
 
-    nhd_streams['downstream_of_headwater'] = False
+    if mainstem_flag == False:
+        nhd_streams['downstream_of_headwater'] = False
+    else:
+        nwm_intersections = nwm_intersections.loc[nwm_intersections.mainstem==1]
+
     nhd_streams = nhd_streams.explode()
     nhd_streams = nhd_streams.reset_index(drop=True)
 
-    # find stream segment closest to nwm intersection point
+    # Find stream segment closest to nwm intersection point
     for index, point in nwm_intersections.iterrows():
 
-        # distance to each stream segment
+        # Distance to each stream segment
         distances = nhd_streams.distance(point.geometry)
 
-        # find minimum distance
+        # Find minimum distance
         min_index = np.argmin(distances)
 
-        # update attributes for incoming stream
-        nhd_streams.loc[min_index,'is_headwater'] = True
-        nhd_streams.loc[min_index,'downstream_of_headwater'] = True
+        # Update attributes for incoming stream
+        nhd_streams.loc[min_index,headwater_col] = True
 
-    ## subset NHDPlus HR
-    nhd_streams['is_relevant_stream'] = nhd_streams['is_headwater'].copy()
+        if mainstem_flag == False:
+            nhd_streams.loc[min_index,'downstream_of_headwater'] = True
+            nhd_streams['is_relevant_stream'] = nhd_streams[headwater_col].copy()
 
-    # trace down from headwaters
+    # Trace down from headwaters
     nhd_streams.set_index('NHDPlusID',inplace=True,drop=False)
 
-    nhd_streams = get_downstream_segments(nhd_streams, 'is_headwater')
+    nhd_streams = get_downstream_segments(nhd_streams,headwater_col,mainstem_flag)
 
     nhd_streams = nhd_streams.loc[nhd_streams['is_relevant_stream'],:]
     nhd_streams.reset_index(drop=True,inplace=True)
 
-    return(nhd_streams)
+    return nhd_streams
 
-def get_downstream_segments(streams, attribute):
+def get_downstream_segments(streams, attribute,mainstem_flag):
 
     Q = deque(streams.loc[streams[attribute],'NHDPlusID'].tolist())
     visited = set()
@@ -136,8 +161,11 @@ def get_downstream_segments(streams, attribute):
         else:
             relevant_ids = downstream_ids
 
-        streams.loc[relevant_ids,'is_relevant_stream'] = True
-        streams.loc[relevant_ids,'downstream_of_headwater'] = True
+        if mainstem_flag == False:
+            streams.loc[relevant_ids,'is_relevant_stream'] = True
+            streams.loc[relevant_ids,'downstream_of_headwater'] = True
+        else:
+            streams.loc[relevant_ids,'mainstem'] = True
 
         for i in relevant_ids:
             if i not in visited:
@@ -155,11 +183,22 @@ def get_downstream_segments(streams, attribute):
     parser.add_argument('-a','--headwaters-filename',help='Headwaters points layer name',required=True,type=str)
     parser.add_argument('-s','--subset-nhd-streams-fileName',help='Output streams layer name',required=False,type=str,default=None)
     parser.add_argument('-i','--headwater-id',help='Headwater points ID column',required=True)
-    parser.add_argument('-i','--nwm-intersections-filename',help='NWM HUC4 intersection points',required=True)
+    parser.add_argument('-c','--nwm-intersections-filename',help='NWM HUC4 intersection points',required=True)
+    parser.add_argument('-d','--mainstem-flag',help='flag for mainstems network',required=False,default=False)
 
     args = vars(parser.parse_args())
 
-    subset_streams_gdf = subset_nhd_network(huc_number,huc4_mask,selected_wbd8,nhd_streams,headwaters_filename,headwater_id)
+    huc_number = args['huc_number']
+    huc4_mask = args['huc4_mask']
+    selected_wbd8 = args['selected_wbd8']
+    nhd_streams = args['nhd_streams']
+    headwaters_filename = args['headwaters_filename']
+    subset_nhd_streams_fileName = args['subset_nhd_streams_fileName']
+    headwater_id = args['headwater_id']
+    nwm_intersections_filename = args['nwm_intersections_filename']
+    mainstem_flag = args['mainstem_flag']
+
+    subset_streams_gdf = subset_nhd_network(huc_number,huc4_mask,selected_wbd8,nhd_streams,headwaters_filename,headwater_id,nwm_intersections_filename,mainstem_flag=False)
 
     if subset_nhd_streams_fileName is not None:
-        subset_streams_gdf.to_file(args['subset_nhd_streams_fileName'],driver=getDriver(args['subset_nhd_streams_fileName']),index=False)
+        subset_streams_gdf.to_file(subset_nhd_streams_fileName,driver=getDriver(subset_nhd_streams_fileName),index=False)
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index c8f490696..0c5e65cf5 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -43,19 +43,6 @@ else
   input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
 fi
 
-# Define streams and headwaters based on extent #
-if [ "$extent" = "MS" ]; then
-  input_nhd_flowlines=$input_nhd_flowlines_ms
-  input_nhd_headwaters=$input_nhd_headwaters_ms
-  input_NWM_Flows=$input_NWM_Flows_ms
-  input_NWM_Catchments=$input_NWM_Catchments_ms
-else
-  input_nhd_flowlines=$input_nhd_flowlines_fr
-  input_nhd_headwaters=$input_nhd_headwaters_fr
-  input_NWM_Flows=$input_NWM_Flows_fr
-  input_NWM_Catchments=$input_NWM_Catchments_fr
-fi
-
 ## GET WBD ##
 echo -e $startDiv"Get WBD $hucNumber"$stopDiv
 date -u
@@ -77,7 +64,7 @@ echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-$srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_NWM_Flows -s $input_nhd_flowlines -l $input_NWM_Lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_NWM_Catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg
+$srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent
 Tcount
 
 if [ "$extent" = "MS" ]; then
@@ -421,7 +408,7 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv outp
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $inputDataDir/bathymetry/BANKFULL_CONUS.txt -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_NWM_Catchments -p $extent -k $outputHucDataDir/small_segments.csv
+$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $inputDataDir/bathymetry/BANKFULL_CONUS.txt -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
 ## USGS CROSSWALK ##
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index c29a92acf..c85cdbe32 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -41,6 +41,7 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     input_catchment = gpd.read_file(input_catchment_filename)
     dem_adj = rasterio.open(dem_adj_filename,'r')
 
+
     #MS extent use gages that are mainstem
     if extent == "MS":
         usgs_gages = usgs_gages.query('curve == "yes" & mainstem == "yes"')
@@ -118,6 +119,7 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     parser.add_argument('-dem_adj','--dem-adj-filename', help='Thalweg adjusted DEM', required=True)
     parser.add_argument('-outtable','--output-table-filename', help='Table to append data', required=True)
     parser.add_argument('-e', '--extent', help="extent configuration entered by user when running fim_run.sh", required = True)
+
     args = vars(parser.parse_args())
 
     usgs_gages_filename = args['usgs_gages_filename']
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 876c1769e..78d884eea 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -148,6 +148,7 @@ def update_raster_profile(args):
 gdal bindings are not entirely compatible: https://rasterio.readthedocs.io/en/latest/topics/switch.html
 
 '''
+
 def reproject_raster(input_raster_name,reprojection,blocksize=None,reprojected_raster_name=None):
 
     if blocksize is not None:
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 37a98b424..fefad3cfa 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+import os
+
 # Projections.
 #PREP_PROJECTION = "+proj=aea +datum=NAD83 +x_0=0.0 +y_0=0.0 +lon_0=96dW +lat_0=23dN +lat_1=29d30'N +lat_2=45d30'N +towgs84=-0.9956000824677655,1.901299877314078,0.5215002840524426,0.02591500053005733,0.009425998542707753,0.01159900118427752,-0.00062000005129903 +no_defs +units=m"
 PREP_PROJECTION_CM = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Albers"],PARAMETER["false_easting",0.0],PARAMETER["false_northing",0.0],PARAMETER["central_meridian",-96.0],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_origin",23.0],UNIT["Meter",1.0],VERTCS["NAVD_1988",VDATUM["North_American_Vertical_Datum_1988"],PARAMETER["Vertical_Shift",0.0],PARAMETER["Direction",1.0],UNIT["Centimeter",0.01]]]'
@@ -34,3 +36,28 @@
 OVERWRITE_WBD = 'OVERWRITE_WBD'
 OVERWRITE_NHD = 'OVERWRITE_NHD'
 OVERWRITE_ALL = 'OVERWRITE_ALL'
+
+## Input Paths and Directories
+# Directories
+os.environ['src_dir'] = '/foss_fim/src'
+os.environ['input_dir'] = 'data/inputs'
+
+os.environ['nhdplus_rasters_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_rasters')
+os.environ['nhdplus_vectors_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors')
+os.environ['nwm_dir'] = os.path.join(os.environ.get('input_dir'),'nwm_hydrofabric')
+os.environ['wbd_dir'] = os.path.join(os.environ.get('input_dir'),'wbd')
+os.environ['ahps_dir'] = os.path.join(os.environ.get('input_dir'),'ahp_sites')
+os.environ['nhdplus_aggregate_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors_aggregate')
+
+# File Paths
+os.environ['wbd_filename'] = os.path.join(os.environ.get('wbd_dir'),'WBD_National.gpkg')
+os.environ['nwm_streams_orig_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_flows_original.gpkg')
+os.environ['nwm_streams_all_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_flows.gpkg')
+os.environ['nwm_headwaters_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_headwaters.gpkg')
+os.environ['nwm_huc4_intersections_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_huc4_intersections.gpkg')
+os.environ['nhd_huc8_intersections_filename'] = os.path.join(os.environ.get('nwm_dir'),'nhd_huc8_intersections.gpkg')
+os.environ['ahps_filename'] = os.path.join(os.environ.get('ahps_dir'),'nws_lid.gpkg')
+os.environ['agg_nhd_headwaters_adj_fileName'] = os.path.join(os.environ.get('nhdplus_aggregate_dir'),'agg_nhd_headwaters_adj.gpkg')
+os.environ['agg_nhd_streams_adj_fileName'] = os.path.join(os.environ.get('nhdplus_aggregate_dir'),'agg_nhd_streams_adj.gpkg')
+os.environ['nwm_catchments_orig_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_catchments_original.gpkg')
+os.environ['nwm_catchments_all_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_catchments.gpkg')
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index c5bf9884d..1f3b138d6 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -35,6 +35,7 @@
     stat_groups : str
         string of columns to group eval metrics.
 """
+
 def check_file_age(file):
     '''
     Checks if file exists, determines the file age, and recommends

From 25c3ec155cb4439f527bcd6dcdfb1415056f0975 Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Mon, 3 May 2021 10:07:19 -0500
Subject: [PATCH 072/359] Generate nws_lid.gpkg

Generate nws_lid.gpkg.

- Generate nws_lid.gpkg with attributes indicating if site is a headwater nws_lid as well as if it is co-located with another nws_lid which is referenced to the same nwm_feature_id segment.

This resolves #368.
---
 CHANGELOG.md              |   8 ++
 tools/generate_nws_lid.py | 180 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 tools/generate_nws_lid.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c6e3ade4..e10fd0822 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.15.9 - 2021-05-03 - [PR #372](https://github.com/NOAA-OWP/cahaba/pull/372)
+
+Generate `nws_lid.gpkg`.
+
+## Additions
+- Generate `nws_lid.gpkg` with attributes indicating if site is a headwater `nws_lid` as well as if it is co-located with another `nws_lid` which is referenced to the same `nwm_feature_id` segment. 
+
 <br/><br/>
 ## v3.0.15.8 - 2021-04-29 - [PR #371](https://github.com/NOAA-OWP/cahaba/pull/371)
 
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
new file mode 100644
index 000000000..a2110c57d
--- /dev/null
+++ b/tools/generate_nws_lid.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import pandas as pd
+import geopandas as gpd
+from collections import defaultdict
+from tools_shared_functions import aggregate_wbd_hucs, get_metadata
+import argparse
+from dotenv import load_dotenv
+import os
+import sys
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION
+
+
+load_dotenv()
+#import variables from .env file
+API_BASE_URL = os.getenv("API_BASE_URL")
+EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+WBD_LAYER = os.getenv("WBD_LAYER")
+#Define path to NWM stream layer
+NWM_FILE='/data/inputs/nwm_hydrofabric/nwm_flows.gpkg'
+
+
+def generate_nws_lid(workspace):
+    '''
+    Generate the nws_lid layer containing all nws_lid points attributed whether site is mainstems and co-located
+
+    Parameters
+    ----------
+    workspace : STR
+        Directory where outputs will be saved.
+
+    Returns
+    -------
+    None.
+
+    '''
+    
+    
+    ##############################################################################
+    #Get all nws_lid points
+    print('Retrieving metadata ..')
+    
+    metadata_url = f'{API_BASE_URL}/metadata/'
+    #Trace downstream from all rfc_forecast_point.
+    select_by = 'nws_lid'
+    selector = ['all']
+    must_include = 'nws_data.rfc_forecast_point'
+    downstream_trace_distance = 'all'
+    fcst_list, fcst_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
+    
+    #Get list of all evaluated sites not in fcst_list
+    fcst_list_sites = [record.get('identifiers').get('nws_lid').lower() for record in fcst_list]
+    evaluated_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].str.lower().to_list()
+    evaluated_sites= list(set(evaluated_sites) - set(fcst_list_sites))
+    
+    #Trace downstream from all evaluated sites not in fcst_list
+    select_by = 'nws_lid'
+    selector = evaluated_sites
+    must_include = None
+    downstream_trace_distance = 'all'
+    eval_list, eval_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
+    
+    #Trace downstream from all sites in HI/PR.
+    select_by = 'state'
+    selector = ['HI','PR']
+    must_include = None
+    downstream_trace_distance = 'all'
+    islands_list, islands_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
+    
+    #Append all lists
+    all_lists = fcst_list + eval_list + islands_list
+    
+    ###############################################################################
+    #Compile NWM segments from all_lists
+    
+    #Get dictionary of downstream segment (key) and target segments (values)
+    #Get dictionary of target segment (key) and site code (value) 
+    downstream = defaultdict(list)
+    target = defaultdict(list)
+    #For each lid metadata dictionary in list
+    for lid in all_lists:
+        site = lid.get('identifiers').get('nws_lid')
+        #Get the nwm feature id associated with the location
+        location_nwm_seg = lid.get('identifiers').get('nwm_feature_id')
+        #get all downstream segments
+        downstream_nwm_segs = lid.get('downstream_nwm_features')
+        #If valid location_nwm_segs construct two dictionaries.
+        if location_nwm_seg: 
+            #Dictionary with target segment and site
+            target[int(location_nwm_seg)].append(site)         
+           #Dictionary of key (2nd to last element) and value (target segment)
+           #2nd to last element used because last element is always 0 (ocean) and the 2nd to last allows for us to get the river 'tree' (Mississippi, Colorado, etc)
+            value = location_nwm_seg
+            if not downstream_nwm_segs:
+                #Special case, no downstream nwm segments are returned (PR/VI/HI).
+                key = location_nwm_seg
+            elif len(downstream_nwm_segs) == 1:
+                #Special case, the nws_lid is within 1 segment of the ocean (0)
+                key = location_nwm_seg
+            elif len(downstream_nwm_segs)>1:
+                #Otherwise, 2nd to last element used to identify proper river system.
+                key = downstream_nwm_segs[-2]            
+            #Dictionary with key of 2nd to last downstream segment and value of site nwm segment 
+            downstream[int(key)].append(int(value))
+    ###############################################################################
+    #Walk downstream the network and identify headwater points
+    print('Traversing network..')
+    
+    #Import NWM file and create dictionary of network and create the NWM network dictionary.
+    nwm_gdf = gpd.read_file(NWM_FILE)
+    network = nwm_gdf.groupby('ID')['to'].apply(list).to_dict()
+    
+    #Walk through network and find headwater points
+    all_dicts = {}
+    for tree, targets in downstream.items():    
+        #All targets are assigned headwaters
+        sub_dict = {i:'is_headwater' for i in targets}
+        #Walk downstream of each target
+        for i in targets:
+            #Check to see element is not a headwater
+            if sub_dict[i] == 'not_headwater':
+                continue
+            #Get from_node and to_node.
+            from_node = i
+            [to_node] = network[from_node]
+            #Walk downstream from target
+            while to_node>0:
+                [to_node] = network[to_node]
+                #Check if to_node is in targets list
+                if to_node in targets:
+                    sub_dict[to_node] = 'not_headwater'    
+        #Append status to master dictionary
+        all_dicts.update(sub_dict)
+    
+    #Create dictionaries of nws_lid (key) and headwater status (value) and nws_lid (key) and co-located with same feature_id(value)
+    final_dict = {}
+    duplicate_dict = {}
+    for key,status in all_dicts.items():
+        site_list = target[key]       
+        for site in site_list:
+            final_dict[site] = status
+            if len(site_list) > 1:
+                duplicate_dict[site] = 'is_colocated'
+            else:
+                duplicate_dict[site] = 'not_colocated'
+    
+    ##############################################################################
+    #Get Spatial data and populate headwater/duplicate attributes
+    print('Attributing nws_lid layer..')
+            
+    #Geodataframe from all_lists, reproject, and reset index.
+    trash, nws_lid_gdf = aggregate_wbd_hucs(all_lists, WBD_LAYER, retain_attributes = False)
+    nws_lid_gdf.columns = [name.replace('identifiers_','') for name in nws_lid_gdf.columns]
+    nws_lid_gdf.to_crs(PREP_PROJECTION, inplace = True)
+    nws_lid_gdf.reset_index(drop = True)
+    
+    #Create DataFrames of headwater and duplicates and join.
+    final_dict_pd = pd.DataFrame(list(final_dict.items()), columns = ['nws_lid','is_headwater'])
+    duplicate_dict_pd = pd.DataFrame(list(duplicate_dict.items()),columns = ['nws_lid','is_colocated'])
+    attributes = final_dict_pd.merge(duplicate_dict_pd, on = 'nws_lid')
+    attributes.replace({'is_headwater': True,'is_colocated': True,'not_headwater': False,'not_colocated':False}, inplace = True)
+    
+    #Join attributes, remove sites with no assigned nwm_feature_id and write to file
+    joined = nws_lid_gdf.merge(attributes, on='nws_lid', how = 'left')
+    joined.dropna(subset =['nwm_feature_id'], inplace = True)
+    Path(workspace).mkdir(parents = True, exist_ok = True)
+    joined.to_file(Path(workspace) / 'nws_lid.gpkg', driver = 'GPKG')
+
+
+
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Create spatial data of nws_lid points attributed with mainstems and colocated.')
+    parser.add_argument('-w', '--workspace', help = 'Workspace where all data will be stored.', required = True)    
+    args = vars(parser.parse_args())
+    
+    #Run get_env_paths and static_flow_lids
+    generate_nws_lid(**args)

From 22ba0df00e403681e12d8f247e35da124536a74a Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Fri, 7 May 2021 12:51:19 -0500
Subject: [PATCH 073/359] Remove Great Lakes from wbd buffer

- The gl_water_polygons.gpkg layer is used to mask out Great Lakes boundaries and remove NHDPlus HR coastline segments. These segments are causing issues later in run_by_unit.sh and unnecessarily increasing total processing time.

Resolves issue #374
---
 CHANGELOG.md               |  8 ++++++++
 fim_run.sh                 |  2 +-
 src/clip_vectors_to_wbd.py | 37 ++++++++++++++++++++++++++++++++++---
 src/run_by_unit.sh         | 12 ++----------
 4 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e10fd0822..e24396a88 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.15.10 - 2021-05-06 - [PR #375](https://github.com/NOAA-OWP/cahaba/pull/375)
+
+Remove Great Lakes coastlines from WBD buffer.
+
+## Changes
+- `gl_water_polygons.gpkg` layer is used to mask out Great Lakes boundaries and remove NHDPlus HR coastline segments.
+
 <br/><br/>
 ## v3.0.15.9 - 2021-05-03 - [PR #372](https://github.com/NOAA-OWP/cahaba/pull/372)
 
diff --git a/fim_run.sh b/fim_run.sh
index 2cfc744e2..cf5de36da 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -116,7 +116,7 @@ export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
 export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
 export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
 export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
-
+export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
 ## Input handling ##
 $srcDir/check_huc_inputs.py -u "$hucList"
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index f29217e76..9c3d73470 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -7,15 +7,38 @@
 from shapely.geometry import MultiPolygon,Polygon,Point
 from utils.shared_functions import getDriver
 
-def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,dissolveLinks=False):
+def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance,dissolveLinks=False):
 
     hucUnitLength = len(str(hucCode))
 
     # Get wbd buffer
     wbd = gpd.read_file(wbd_filename)
-    wbd_buffer = gpd.read_file(wbd_buffer_filename)
+    wbd_buffer = wbd.copy()
+    wbd_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance,resolution=32)
     projection = wbd_buffer.crs
 
+    great_lakes = gpd.read_file(great_lakes_filename, mask = wbd_buffer).reset_index(drop=True)
+
+    if not great_lakes.empty:
+        print("Masking Great Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+
+        # Clip excess lake area
+        great_lakes = gpd.clip(great_lakes, wbd_buffer)
+
+        # Buffer remaining lake area
+        great_lakes.geometry = great_lakes.buffer(lake_buffer_distance)
+
+        # Removed buffered GL from WBD buffer
+        wbd_buffer = gpd.overlay(wbd_buffer, great_lakes, how='difference')
+        wbd_buffer = wbd_buffer[['geometry']]
+        wbd_buffer.to_file(wbd_buffer_filename,driver=getDriver(wbd_buffer_filename),index=False)
+
+    else:
+        wbd_buffer = wbd_buffer[['geometry']]
+        wbd_buffer.to_file(wbd_buffer_filename,driver=getDriver(wbd_buffer_filename),index=False)
+
+    del great_lakes
+
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea_filename, mask = wbd_buffer)
     if not landsea.empty:
@@ -25,6 +48,7 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     # Find intersecting lakes and writeout
     print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
     nwm_lakes = gpd.read_file(nwm_lakes_filename, mask = wbd_buffer)
+    nwm_lakes = nwm_lakes.loc[nwm_lakes.Shape_Area < 18990454000.0]
 
     if not nwm_lakes.empty:
         # Perform fill process to remove holes/islands in the NWM lake polygons
@@ -59,6 +83,7 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     # Subset nhd streams
     print("Querying NHD Streams for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
     nhd_streams = gpd.read_file(nhd_streams_filename, mask = wbd_buffer)
+
     if extent == 'MS':
         nhd_streams = nhd_streams.loc[nhd_streams.mainstem==1]
 
@@ -120,6 +145,9 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     parser.add_argument('-b','--subset-nwm-streams',help='NWM streams subset',required=True)
     parser.add_argument('-x','--subset-landsea',help='LandSea subset',required=True)
     parser.add_argument('-extent','--extent',help='FIM extent',required=True)
+    parser.add_argument('-gl','--great-lakes-filename',help='Great Lakes layer',required=True)
+    parser.add_argument('-wb','--wbd-buffer-distance',help='WBD Mask buffer distance',required=True,type=int)
+    parser.add_argument('-lb','--lake-buffer-distance',help='Great Lakes Mask buffer distance',required=True,type=int)
     parser.add_argument('-o','--dissolve-links',help='remove multi-line strings',action="store_true",default=False)
 
 
@@ -143,6 +171,9 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     subset_nwm_streams_filename = args['subset_nwm_streams']
     subset_landsea_filename = args['subset_landsea']
     extent = args['extent']
+    great_lakes_filename = args['great_lakes_filename']
+    wbd_buffer_distance = args['wbd_buffer_distance']
+    lake_buffer_distance  = args['lake_buffer_distance']
     dissolveLinks = args['dissolve_links']
 
-    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,dissolveLinks)
+    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance,dissolveLinks)
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 0c5e65cf5..01cc2c98a 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -37,7 +37,7 @@ input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Iden
 
 # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
 if [[ $huc2Identifier == "04" ]] ; then
-  input_LANDSEA=$inputDataDir/landsea/gl_water_polygons.gpkg
+  input_LANDSEA=$input_GL_boundaries
   echo -e "Using $input_LANDSEA for water body mask (Great Lakes)"
 else
   input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
@@ -51,20 +51,12 @@ Tstart
 ogr2ogr -f GPKG $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
 Tcount
 
-## BUFFER WBD ##
-echo -e $startDiv"Buffer WBD $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/wbd_buffered.gpkg ] && \
-ogr2ogr -f GPKG -dialect sqlite -sql "select ST_buffer(geom, $wbd_buffer) from 'WBDHU$hucUnitLength'" $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd.gpkg
-Tcount
-
 ## Subset Vector Layers ##
 echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-$srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent
+$srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
 Tcount
 
 if [ "$extent" = "MS" ]; then

From b5da4b4869ccff2d87b2326c29d378990daf1c3e Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Fri, 7 May 2021 14:45:39 -0500
Subject: [PATCH 074/359] Added fuctionality for new "Release" feature in FIM
 API

New "Release" feature added to the FIM API. This feature will allow for automated FIM, CatFIM, and relevant metrics to be generated when a new FIM Version is released. See #373 for more detailed steps that take place in this feature.

Additions
- Added new window to the UI in api/frontend/gui/templates/index.html.
- Added new job type to api/node/connector/connector.py to allow these release jobs to run.
- Added additional logic in api/node/updater/updater.py to run the new eval and CatFIM scripts used in the release feature.

Changes
- Updated api/frontend/output_handler/output_handler.py to allow for copying more broad ranges of file paths instead of only the /data/outputs directory.

Resolves Issues #264 #278 #307 and #373
---
 CHANGELOG.md                                  |  13 +
 api/frontend/gui/templates/index.html         | 158 +++++++--
 api/frontend/output_handler/output_handler.py |   6 +-
 api/node/connector/connector.py               | 157 +++++----
 api/node/updater/updater.py                   | 306 ++++++++++++++----
 5 files changed, 486 insertions(+), 154 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e24396a88..fd4928cf9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.16.0 - 2021-05-07 - [PR #378](https://github.com/NOAA-OWP/cahaba/pull/378)
+
+New "Release" feature added to the FIM API. This feature will allow for automated FIM, CatFIM, and relevant metrics to be generated when a new FIM Version is released. See [#373](https://github.com/NOAA-OWP/cahaba/issues/373) for more detailed steps that take place in this feature.
+
+## Additions
+- Added new window to the UI in `api/frontend/gui/templates/index.html`.
+- Added new job type to `api/node/connector/connector.py` to allow these release jobs to run.
+- Added additional logic in `api/node/updater/updater.py` to run the new eval and CatFIM scripts used in the release feature.
+
+## Changes
+- Updated `api/frontend/output_handler/output_handler.py` to allow for copying more broad ranges of file paths instead of only the `/data/outputs` directory.
+
 <br/><br/>
 ## v3.0.15.10 - 2021-05-06 - [PR #375](https://github.com/NOAA-OWP/cahaba/pull/375)
 
diff --git a/api/frontend/gui/templates/index.html b/api/frontend/gui/templates/index.html
index 4dd2c2fc3..421050dbd 100644
--- a/api/frontend/gui/templates/index.html
+++ b/api/frontend/gui/templates/index.html
@@ -59,6 +59,7 @@
 
                 color: black;
                 background-color: white;
+                font-size: 0.9rem;
 
                 transition: all 0.1s linear;
 
@@ -137,6 +138,27 @@
                 background-color: #093568;
             }
 
+            #release-button {
+                display: flex;
+                width: 100%;
+                height: 100%;
+
+                align-items: center;
+                justify-content: center;
+
+                font-size: 1.2rem;
+                color: white;
+                background-color: #28be99;
+                cursor: pointer;
+
+                transition: all 0.5s linear;
+                pointer-events: inherit;
+            }
+
+            #release-button:hover, #release-button.active {
+                background-color: #09685b;
+            }
+
             .form-field {
                 display: flex;
                 width: 0;
@@ -224,15 +246,40 @@
         <!-- <h1>FIM Run API</h1> -->
         <div id="request-container">
             <div id="request-tabs-container">
+                <div class="request-tab" onclick="selectForm(event, 'release')">Release</div>
                 <div class="request-tab active" onclick="selectForm(event, 'fim-run')">FIM Run</div>
                 <div class="request-tab" onclick="selectForm(event, 'calibration')">Calibration</div>
                 <div class="request-tab" onclick="selectForm(event, 'preprocessing')">Pre-processing</div>
             </div>
+            <div id="release" class="request-form" style="display: None;">
+                <div class="form-content">
+                    <h3 style="margin: 0;">FIM Version</h3>
+                    <div class="container-horizontal" style="width: 10rem;">
+                        <span>FIM_3_</span>
+                        <input type="text" id="release-job-version-major" class="form-field" placeholder="0"></input>
+                        <span>_</span>
+                        <input type="text" id="release-job-version-minor" class="form-field" placeholder="0"></input>
+                        <span>_</span>
+                        <input type="text" id="release-job-version-patch" class="form-field" placeholder="0"></input>
+                    </div>
+                    <h3 style="margin: 0;">Previous Major FIM Version</h3>
+                    <div class="container-horizontal" style="width: 10rem;">
+                        <span>FIM_3_</span>
+                        <input type="text" id="prev-release-job-version-major" class="form-field" placeholder="0"></input>
+                        <span>_</span>
+                        <input type="text" id="prev-release-job-version-minor" class="form-field" placeholder="0"></input>
+                        <span>_</span>
+                        <input type="text" id="prev-release-job-version-patch" class="form-field" placeholder="0"></input>
+                    </div>
+                    <div class="error-list" id="release-errors"></div>
+                </div>
+                <div class="start-button not-connected" onclick="submitRequest('release')">Not Connected</div>
+            </div>
             <div id="fim-run" class="request-form">
                 <div class="form-content">
                     <h3 style="margin: 0;">Basic</h3>
                     <div class="container-horizontal">
-                        <input type="text" id="job-name" class="form-field" placeholder="Job Name"></input>
+                        <input type="text" id="fim-run-job-name" class="form-field" placeholder="Job Name"></input>
                         <select id="presets-list" class="form-field" onchange="presetListSelected(event)"></select>
                         <input type="text" id="hucs" class="form-field" placeholder="HUC(s)" style="display: none;"></input>
                         <input type="text" id="git-branch" class="form-field" placeholder="Git Branch"></input>
@@ -271,7 +318,7 @@ <h3 style="margin: 0;">Extent</h3>
     
                     </div>
                 </div>
-                <div class="start-button not-connected" onclick="submitRequest('fim-run')">Not Connected</div>
+                <div class="start-button not-connected" onclick="submitRequest('fim_run')">Not Connected</div>
             </div>
             <div id="calibration" class="request-form" style="display: None;">
                 Calibration stuff goes here
@@ -370,7 +417,7 @@ <h3 style="margin: 0;">Extent</h3>
                 socket.on('job_added', job_type => {
                     document.querySelector('#request-container').classList.remove('loading')
                     if (job_type === 'fim_run') {
-                        document.querySelector('#job-name').value = ''
+                        document.querySelector('#fim-run-job-name').value = ''
                         document.querySelector('#hucs').value = ''
                         document.querySelector('#git-branch').value = ''
                         document.querySelector("input[name='dev-run']").checked = false
@@ -380,6 +427,15 @@ <h3 style="margin: 0;">Extent</h3>
                         document.querySelector("input[name='extent'][value='FR']").checked = true
                         document.querySelector("input[name='extent'][value='MS']").checked = false
                         document.querySelector('#fim-run-errors').innerHTML = ''
+                    } else if (job_type === 'release') {
+                        console.log('release job has been added')
+                        document.querySelector('#release-job-version-major').value = ''
+                        document.querySelector('#release-job-version-minor').value = ''
+                        document.querySelector('#release-job-version-patch').value = ''
+                        document.querySelector('#prev-release-job-version-major').value = ''
+                        document.querySelector('#prev-release-job-version-minor').value = ''
+                        document.querySelector('#prev-release-job-version-patch').value = ''
+                        document.querySelector('#release-errors').innerHTML = ''
                     }
                 })
                 socket.on('job_canceled', () => {
@@ -387,39 +443,73 @@ <h3 style="margin: 0;">Extent</h3>
                 })
             })
 
-            // Fim_run related code
             const submitRequest = requestName => {
-                let validation_errors = []
-                const job_name = document.querySelector('#job-name').value
-                const preset = document.querySelector('#presets-list').value
-                const hucs = document.querySelector('#hucs').value
-                const git_branch = document.querySelector('#git-branch').value
-                const dev_run = document.querySelector("input[name='dev-run']").checked
-                const viz_run = document.querySelector("input[name='viz-run']").checked
-                const configuration =  Array.from(document.querySelectorAll("input[name='configuration']")).filter(c => c.checked)[0].value
-                const extents =  Array.from(document.querySelectorAll("input[name='extent']")).filter(c => c.checked).map(c => c.value)
-
-                if (job_name === '') validation_errors.push('Job Name Cannot Be Empty')
-                if (preset === 'custom' && hucs === '') validation_errors.push('Huc(s) Cannot Be Empty')
-                if (git_branch === '') validation_errors.push('Git Branch Cannot Be Empty')
-
-                if (validation_errors.length > 0){
-                    document.querySelector('#fim-run-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
-                    return
-                }
+                if (requestName === 'fim_run'){
+                    // Fim_run related code
+                    let validation_errors = []
+                    const job_name = document.querySelector('#fim-run-job-name').value
+                    const preset = document.querySelector('#presets-list').value
+                    const hucs = document.querySelector('#hucs').value
+                    const git_branch = document.querySelector('#git-branch').value
+                    const dev_run = document.querySelector("input[name='dev-run']").checked
+                    const viz_run = document.querySelector("input[name='viz-run']").checked
+                    const configuration =  Array.from(document.querySelectorAll("input[name='configuration']")).filter(c => c.checked)[0].value
+                    const extents =  Array.from(document.querySelectorAll("input[name='extent']")).filter(c => c.checked).map(c => c.value)
+
+                    if (job_name === '') validation_errors.push('Job Name Cannot Be Empty')
+                    if (preset === 'custom' && hucs === '') validation_errors.push('Huc(s) Cannot Be Empty')
+                    if (git_branch === '') validation_errors.push('Git Branch Cannot Be Empty')
+
+                    if (validation_errors.length > 0){
+                        document.querySelector('#fim-run-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
+                        return
+                    }
 
-                document.querySelector('#request-container').classList.add('loading')
-                
-                socket.emit('new_job', {
-                    job_name,
-                    preset,
-                    hucs,
-                    git_branch,
-                    dev_run,
-                    viz_run,
-                    configuration,
-                    extents
-                })
+                    document.querySelector('#request-container').classList.add('loading')
+                    
+                    socket.emit('new_job', {
+                        job_type: 'fim_run',
+                        job_name,
+                        preset,
+                        hucs,
+                        git_branch,
+                        dev_run,
+                        viz_run,
+                        configuration,
+                        extents
+                    })
+                } else if (requestName === 'release'){
+                    // Release related code
+                    let validation_errors = []
+                    const job_version_major = document.querySelector('#release-job-version-major').value
+                    const job_version_minor = document.querySelector('#release-job-version-minor').value
+                    const job_version_patch = document.querySelector('#release-job-version-patch').value
+                    const prev_job_version_major = document.querySelector('#prev-release-job-version-major').value
+                    const prev_job_version_minor = document.querySelector('#prev-release-job-version-minor').value
+                    const prev_job_version_patch = document.querySelector('#prev-release-job-version-patch').value
+
+                    if (job_version_major === '' || job_version_minor === '' || job_version_patch === '')
+                        validation_errors.push('Fim Version must be completely filled out')
+                    if (prev_job_version_major === '' || prev_job_version_minor === '' || prev_job_version_patch === '')
+                        validation_errors.push('Previous Major Fim Version must be completely filled out')
+
+                    if (validation_errors.length > 0){
+                        document.querySelector('#release-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
+                        return
+                    }
+
+                    document.querySelector('#request-container').classList.add('loading')
+
+                    socket.emit('new_job', {
+                        job_type: 'release',
+                        job_version_major,
+                        job_version_minor,
+                        job_version_patch,
+                        prev_job_version_major,
+                        prev_job_version_minor,
+                        prev_job_version_patch
+                    })
+                }
             }
 
             const disableDevRun = e => {
diff --git a/api/frontend/output_handler/output_handler.py b/api/frontend/output_handler/output_handler.py
index f665c5669..1d803a696 100644
--- a/api/frontend/output_handler/output_handler.py
+++ b/api/frontend/output_handler/output_handler.py
@@ -6,7 +6,6 @@
 SOCKET_URL = os.environ.get('SOCKET_URL')
 
 def handle_outputs(data):
-    nice_name = data['nice_name']
     job_name = data['job_name']
     directory_path = data['directory_path']
     file_name = data['file_name']
@@ -18,16 +17,15 @@ def handle_outputs(data):
     if chunk_index == 0:
         mode = 'wb'
         try:
-            os.makedirs(f"/data/outputs/{nice_name}/{directory_path}")
+            os.makedirs(directory_path)
         except:
             pass
         
     # Write binary data to file
-    with open(f"/data/outputs/{nice_name}/{directory_path}/{file_name}", mode) as binary_file:
+    with open(f"{directory_path}/{file_name}", mode) as binary_file:
         print(f"Writing chunk {chunk_index} for file {directory_path}/{file_name}")
         binary_file.write(file_chunk)
 
-    # TODO: Write an emit that will trigger the next chunk to be sent.
     sio.emit('output_handler_finished_file_chunk', {'job_name': job_name, 'file_path': f"{directory_path}/{file_name}"})
 
 sio = socketio.Client()
diff --git a/api/node/connector/connector.py b/api/node/connector/connector.py
index 992f0f0f5..117448d4b 100644
--- a/api/node/connector/connector.py
+++ b/api/node/connector/connector.py
@@ -58,7 +58,6 @@ def ws_updater_connected():
 
 @socketio.on('ready_for_output_handler')
 def ws_ready_for_output_handler(data):
-    nice_name = data['nice_name']
     job_name = data['job_name']
     path = data['path']
     chunk_index = data['chunk_index']
@@ -69,7 +68,7 @@ def ws_ready_for_output_handler(data):
         return
 
     # Split up path into parts for the output handler
-    path_parts = re.search(rf"/data/outputs/{job_name}/(.+)/(.+)", path)
+    path_parts = re.search(rf"(.+)/(.+)", path)
     directory_path = path_parts.group(1)
     file_name = path_parts.group(2)
 
@@ -89,7 +88,6 @@ def ws_ready_for_output_handler(data):
 
         print("Sending to output handler", path, "Chunk:", chunk_index)
         emit('new_job_outputs', {
-            'nice_name': nice_name,
             'job_name': job_name,
             'directory_path': directory_path,
             'file_name': file_name,
@@ -105,78 +103,115 @@ def output_handler_finished_file_chunk(data):
     print('done saving chunk', job_name, file_path)
     emit('file_chunk_saved', {
         'job_name': job_name,
-        'file_path': f"/data/outputs/{job_name}/{file_path}",
+        'file_path': file_path,
     }, room=shared_data['updater_sid'])
 
 @socketio.on('new_job')
 def ws_new_job(job_params):
-    validation_errors = []
-
-    # Get Preset Option
-    preset = job_params['preset']
-
-    # Validate Hucs Name Option
-    if preset == 'custom':
-        hucs_raw = job_params['hucs'].replace(',', ' ').split()
-        parallel_jobs = len(hucs_raw)
-        hucs_type = len(hucs_raw[0])
-        hucs = ' '.join(hucs_raw)
-        invalid_hucs = re.search('[^0-9 ]', hucs)
-        if invalid_hucs: validation_errors.append('Invalid Huc(s)')
-    else:
-        hucs = f"/data/inputs/huc_lists/{preset}"
-        parallel_jobs = 0
-        hucs_type = 0
-
-    # Validate Git Branch Option
-    branch = ''
-    branch_exists = subprocess.run(['git', 'ls-remote', '--heads', GITHUB_REPO, job_params['git_branch'].replace(' ', '_')], stdout=subprocess.PIPE).stdout.decode('utf-8')
-    if branch_exists: branch = job_params['git_branch'].replace(' ', '_')
-    else: validation_errors.append('Git Branch Does Not Exist')
-
-    # Validate Extent Option
-    valid_extents = ['FR', 'MS']
-    extents = []
-    for extent in job_params['extents']:
-        if extent in valid_extents:
-            extents.append(extent)
+    job_type = job_params['job_type']
+
+    if job_type == 'fim_run':
+        validation_errors = []
+
+        # Get Preset Option
+        preset = job_params['preset']
+
+        # Validate Hucs Name Option
+        if preset == 'custom':
+            hucs_raw = job_params['hucs'].replace(',', ' ').split()
+            parallel_jobs = len(hucs_raw)
+            hucs_type = len(hucs_raw[0])
+            hucs = ' '.join(hucs_raw)
+            invalid_hucs = re.search('[^0-9 ]', hucs)
+            if invalid_hucs: validation_errors.append('Invalid Huc(s)')
         else:
-            validation_errors.append('Invalid Extent Option')
+            hucs = f"/data/inputs/huc_lists/{preset}"
+            parallel_jobs = 0
+            hucs_type = 0
+
+        # Validate Git Branch Option
+        branch = ''
+        branch_exists = subprocess.run(['git', 'ls-remote', '--heads', GITHUB_REPO, job_params['git_branch'].replace(' ', '_')], stdout=subprocess.PIPE).stdout.decode('utf-8')
+        if branch_exists: branch = job_params['git_branch'].replace(' ', '_')
+        else: validation_errors.append('Git Branch Does Not Exist')
+
+        # Validate Extent Option
+        valid_extents = ['FR', 'MS']
+        extents = []
+        for extent in job_params['extents']:
+            if extent in valid_extents:
+                extents.append(extent)
+            else:
+                validation_errors.append('Invalid Extent Option')
+
+        # Validate Configuration Option
+        config_path = ''
+        if job_params['configuration'] == 'default': config_path = './foss_fim/config/params_template.env'
+        elif job_params['configuration'] == 'calibrated': config_path = './foss_fim/config/params_calibrated.env'
+        else: validation_errors.append('Invalid Configuration Option')
+        
+        # Validate Dev Run Option
+        if job_params['dev_run'] : dev_run = True
+        else: dev_run = False
+
+        # Validate Viz Run Option
+        if job_params['viz_run'] : viz_run = True
+        else: viz_run = False
+
+        if len(validation_errors) == 0:
+            for extent in extents:
+                # Validate Job Name Option
+                job_name = f"apijob_{job_params['job_name'].replace(' ', '_')[0:50]}_fim_run_{extent.lower()}{'_c' if job_params['configuration'] == 'calibrated' else ''}{'_v' if viz_run == True else ''}_apijob_{branch}_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
+                print(f"adding job {job_name} {branch} {preset} {hucs} {parallel_jobs} {hucs_type} {extent.lower()} {config_path} {dev_run} {viz_run}")
+                emit('add_job_to_queue', {
+                    'job_type': 'fim_run',
+                    'job_name': job_name,
+                    'branch': branch,
+                    'hucs': hucs,
+                    'parallel_jobs': parallel_jobs,
+                    'hucs_type': hucs_type,
+                    'extent': extent,
+                    'config_path': config_path,
+                    'dev_run': dev_run,
+                    'viz_run': viz_run,
+                }, room=shared_data['updater_sid'])
+                print('fim_run job added')
+                emit('job_added', 'fim_run')
+        else:
+            emit('validation_errors', validation_errors)
 
-    # Validate Configuration Option
-    config_path = ''
-    if job_params['configuration'] == 'default': config_path = './foss_fim/config/params_template.env'
-    elif job_params['configuration'] == 'calibrated': config_path = './foss_fim/config/params_calibrated.env'
-    else: validation_errors.append('Invalid Configuration Option')
-    
-    # Validate Dev Run Option
-    if job_params['dev_run'] : dev_run = True
-    else: dev_run = False
+    elif job_type == 'release':
+        job_version_major = job_params['job_version_major']
+        job_version_minor = job_params['job_version_minor']
+        job_version_patch = job_params['job_version_patch']
+
+        # TODO: validate version number
+
+        job_name_base = f"fim_3_{job_version_major}_{job_version_minor}_{job_version_patch}"
+
+        prev_job_version_major = job_params['prev_job_version_major']
+        prev_job_version_minor = job_params['prev_job_version_minor']
+        prev_job_version_patch = job_params['prev_job_version_patch']
+
+        prev_version_base = f"fim_3_{prev_job_version_major}_{prev_job_version_minor}_{prev_job_version_patch}"
 
-    # Validate Viz Run Option
-    if job_params['viz_run'] : viz_run = True
-    else: viz_run = False
+        huc_lists = ['/data/inputs/huc_lists/included_huc8.lst', '/data/inputs/huc_lists/included_huc8_ms.lst']
+        extents = ['FR', 'MS']
 
-    if len(validation_errors) == 0:
-        for extent in extents:
+        for hucs, extent in zip(huc_lists, extents):
             # Validate Job Name Option
-            job_name = f"apijob_{job_params['job_name'].replace(' ', '_')[0:50]}_{extent}_apijob_{branch}_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
-            print(f"adding job {job_name} {branch} {preset} {hucs} {parallel_jobs} {hucs_type} {extent} {config_path} {dev_run} {viz_run}")
+            prev_version = f"{prev_version_base}_{extent.lower()}_c"
+            job_name = f"apijob_{job_name_base}_{extent.lower()}_c_apijob_dev_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
+            print(f"adding job {job_name} {hucs} {extent.lower()}")
             emit('add_job_to_queue', {
+                'job_type': 'release',
                 'job_name': job_name,
-                'branch': branch,
                 'hucs': hucs,
-                'parallel_jobs': parallel_jobs,
-                'hucs_type': hucs_type,
                 'extent': extent,
-                'config_path': config_path,
-                'dev_run': dev_run,
-                'viz_run': viz_run,
+                'previous_major_fim_version': prev_version
             }, room=shared_data['updater_sid'])
-            print('job added')
-            emit('job_added', 'fim_run')
-    else:
-        emit('validation_errors', validation_errors)
+            print('release job added')
+            emit('job_added', 'release')
     
     @socketio.on('cancel_job')
     def ws_cancel_job(job_params):
diff --git a/api/node/updater/updater.py b/api/node/updater/updater.py
index ea162f50d..c0bdd40e6 100644
--- a/api/node/updater/updater.py
+++ b/api/node/updater/updater.py
@@ -1,5 +1,6 @@
 import os
 import re
+import glob
 import math
 import time
 import json
@@ -44,7 +45,8 @@ def update_loop():
 
         while len(buffer_remove_jobs) > 0:
             job_to_remove = buffer_remove_jobs.pop()
-            current_jobs[job_to_remove['job_name']]['status'] = 'Cancelled'
+            if job_to_remove['job_name'] in current_jobs:
+                current_jobs[job_to_remove['job_name']]['status'] = 'Cancelled'
 
         # Get list of current docker containers that are fim run jobs
                                 #    docker ps --all --filter=name=apijob --format='{{.Names}} {{.State}}'
@@ -70,28 +72,39 @@ def update_loop():
                 if os.path.isdir(temp_path):
                     shutil.rmtree(temp_path)
 
-                outputs_path = f"/data/outputs/{job_name}"
+                outputs_path = f"/data/outputs/{current_jobs[job_name]['nice_name']}"
                 if os.path.isdir(outputs_path):
                     shutil.rmtree(outputs_path)
                 
                 jobs_to_delete.append(job_name)
 
+            active_statuses = [
+                'In Progress',
+                'Ready for Synthesize Test Cases',
+                'Running Synthesize Test Cases',
+                'Ready for Eval Plots',
+                'Running Eval Plots',
+                'Ready for Generate Categorical FIM',
+                'Running Generate Categorical FIM',
+                'Ready to Save File',
+                'Saving File'
+            ]
+            # TODO: separate list for queuing so that one job can save and another run
 
             # Update the time elapsed for all jobs that are currently in progress or saving outputs
-            if current_jobs[job_name]['status'] == 'In Progress' or current_jobs[job_name]['status'] == 'Ready to Save File'\
-                or current_jobs[job_name]['status'] == 'Saving File':
+            if current_jobs[job_name]['status'] in active_statuses:
                 current_jobs[job_name]['time_elapsed'] = math.ceil(time.time() - current_jobs[job_name]['time_started'])
 
             # TODO: While job is in progress, keep track of how many hucs are done and overall progress %
 
             # Once resources recome available, start a new job that is in queue
+
             if current_jobs[job_name]['status'] == 'In Queue':
-                # TODO: Start Dcoker containers here and set start time.
                 current_jobs[job_name]['time_started'] = time.time()
 
                 total_active_cores = 0
                 for j in current_jobs.keys():
-                    if current_jobs[j]['status'] == 'In Progress':
+                    if current_jobs[j]['status'] in active_statuses:
                         # This is to account for the fact that HUC6's take a lot more resources to run.
                         # (not necessarily cpu cores but rather RAM, so this artificially reduces how many jobs can run when HUC6's
                         #  are running)
@@ -112,6 +125,7 @@ def update_loop():
                 # print(potential_active_cores <= MAX_ALLOWED_CPU_CORES)
                 if potential_active_cores <= MAX_ALLOWED_CPU_CORES:
                     job_name = current_jobs[job_name]['job_name']
+                    nice_name = current_jobs[job_name]['nice_name']
                     branch = current_jobs[job_name]['branch']
                     hucs = current_jobs[job_name]['hucs']
                     parallel_jobs = current_jobs[job_name]['parallel_jobs']
@@ -121,15 +135,15 @@ def update_loop():
                     viz_run = current_jobs[job_name]['viz_run']
 
                     # Clone github repo, with specific branch, to a temp folder
-                    print(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}')
-                    subprocess.call(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name}', shell=True)
+                    print(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name} && chmod -R 777 {job_name} && cp .env {job_name}/tools/.env')
+                    subprocess.call(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name} && chmod -R 777 {job_name} && cp .env {job_name}/tools/.env', shell=True)
 
                     # Kick off the new job as a docker container with the new cloned repo as the volume
-                    print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}")
-                    subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {job_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}", shell=True)
+                    print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {nice_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}")
+                    subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {nice_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}", shell=True)
                     current_jobs[job_name]['status'] = 'In Progress'
 
-            # Once the Docker container is done, set the job as ready to save output
+            # Once the Docker container is done, either save outputs or run release
             if current_jobs[job_name]['status'] == 'In Progress' and current_jobs[job_name]['container_state'] == 'exited':
 
                 # Get container exit code, get the docker log, and then remove container
@@ -145,13 +159,134 @@ def update_loop():
 
                 exit_code = int(exit_code_raw[0])
                 current_jobs[job_name]['exit_code'] = exit_code
-                subprocess.call(f"docker logs {job_name} >& /data/outputs/{job_name}/logs/docker.log", shell=True)
+                subprocess.call(f"docker logs {job_name} >& /data/outputs/{current_jobs[job_name]['nice_name']}/logs/docker.log", shell=True)
+                subprocess.call(f"docker container rm {job_name}", shell=True)
+
+                if current_jobs[job_name]['job_type'] == 'fim_run':
+                    for path, folders, files in os.walk(f"/data/outputs/{current_jobs[job_name]['nice_name']}"):
+                        for file in files:
+                            current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = 0
+
+                    current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
+                    current_jobs[job_name]['status'] = 'Ready to Save File'
+                elif current_jobs[job_name]['job_type'] == 'release':
+                    current_jobs[job_name]['status'] = 'Ready for Synthesize Test Cases'
+
+            if current_jobs[job_name]['status'] == 'Ready for Synthesize Test Cases':
+                job_name = current_jobs[job_name]['job_name']
+                nice_name = current_jobs[job_name]['nice_name']
+                parallel_jobs = current_jobs[job_name]['parallel_jobs']
+
+                if os.path.isdir(f"/data/previous_fim/{nice_name}"):
+                    shutil.rmtree(f"/data/previous_fim/{nice_name}")
+                if os.path.isdir(f"/data/outputs/{nice_name}"): shutil.move(f"/data/outputs/{nice_name}", '/data/previous_fim')
+                # Kick off the new job as a docker container to run eval metrics
+                print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/synthesize_test_cases.py -c PREV --fim-version {nice_name} --job-number {parallel_jobs} -m /data/test_cases/metrics_library/all_official_versions.csv")
+                subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/synthesize_test_cases.py -c PREV --fim-version {nice_name} --job-number {parallel_jobs} -m /data/test_cases/metrics_library/all_official_versions.csv", shell=True)
+                current_jobs[job_name]['container_state'] = 'running'
+                current_jobs[job_name]['status'] = 'Running Synthesize Test Cases'
+
+            # Once the Docker container is done, save outputs
+            if current_jobs[job_name]['status'] == 'Running Synthesize Test Cases' and current_jobs[job_name]['container_state'] == 'exited':
+                # Get container exit code, get the docker log, and then remove container
+                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
+
+                print("Exit code")
+                print(exit_code_raw)
+                print(exit_code_raw[0])
+                try:
+                    print(int(exit_code_raw[0]))
+                except:
+                    pass
+
+                exit_code = int(exit_code_raw[0])
+                current_jobs[job_name]['exit_code'] = exit_code
+                subprocess.call(f"docker logs {job_name} >& /data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/synthesize_test_cases_docker.log", shell=True)
+                subprocess.call(f"docker container rm {job_name}", shell=True)
+
+                current_jobs[job_name]['output_files_saved']['/data/test_cases/metrics_library/all_official_versions.csv'] = 0
+                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/docker.log"] = 0
+                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/synthesize_test_cases_docker.log"] = 0
+                current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
+                current_jobs[job_name]['status'] = 'Ready for Eval Plots'
+
+            if current_jobs[job_name]['status'] == 'Ready for Eval Plots':
+                job_name = current_jobs[job_name]['job_name']
+                nice_name = current_jobs[job_name]['nice_name']
+                previous_major_fim_version = current_jobs[job_name]['previous_major_fim_version']
+
+                # Kick off the new job as a docker container to run eval plots
+                print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/eval_plots.py -m /data/test_cases/metrics_library/all_official_versions.csv -w /data/test_cases/metrics_library/all_official_versions_viz -v {previous_major_fim_version} {nice_name} -sp")
+                subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/eval_plots.py -m /data/test_cases/metrics_library/all_official_versions.csv -w /data/test_cases/metrics_library/all_official_versions_viz -v {previous_major_fim_version} {nice_name} -sp", shell=True)
+                current_jobs[job_name]['container_state'] = 'running'
+                current_jobs[job_name]['status'] = 'Running Eval Plots'
+
+            # Once the Docker container is done, save outputs
+            if current_jobs[job_name]['status'] == 'Running Eval Plots' and current_jobs[job_name]['container_state'] == 'exited':
+                # Get container exit code, get the docker log, and then remove container
+                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
+
+                print("Exit code")
+                print(exit_code_raw)
+                print(exit_code_raw[0])
+                try:
+                    print(int(exit_code_raw[0]))
+                except:
+                    pass
+
+                exit_code = int(exit_code_raw[0])
+                current_jobs[job_name]['exit_code'] = exit_code
+                subprocess.call(f"docker logs {job_name} >& /data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/eval_plots_docker.log", shell=True)
                 subprocess.call(f"docker container rm {job_name}", shell=True)
 
-                for path, folders, files in os.walk(f"/data/outputs/{job_name}"):
+                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/eval_plots_docker.log"] = 0
+                for path, folders, files in os.walk('/data/test_cases/metrics_library/all_official_versions_viz'):
                     for file in files:
                         current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = 0
+                current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
+
+                current_jobs[job_name]['status'] = 'Ready for Generate Categorical FIM'
+
+            if current_jobs[job_name]['status'] == 'Ready for Generate Categorical FIM':
+                job_name = current_jobs[job_name]['job_name']
+                nice_name = current_jobs[job_name]['nice_name']
+                parallel_jobs = current_jobs[job_name]['parallel_jobs']
+
+                # Kick off the new job as a docker container to run CatFIM
+                print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/generate_categorical_fim.py -f {nice_name} -j {parallel_jobs}")
+                subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/generate_categorical_fim.py -f /data/previous_fim/{nice_name} -j {parallel_jobs}", shell=True)
+                current_jobs[job_name]['container_state'] = 'running'
+                current_jobs[job_name]['status'] = 'Running Generate Categorical FIM'
+
+            # Once the Docker container is done, save outputs
+            if current_jobs[job_name]['status'] == 'Running Generate Categorical FIM' and current_jobs[job_name]['container_state'] == 'exited':
+                # Get container exit code, get the docker log, and then remove container
+                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
+
+                print("Exit code")
+                print(exit_code_raw)
+                print(exit_code_raw[0])
+                try:
+                    print(int(exit_code_raw[0]))
+                except:
+                    pass
 
+                exit_code = int(exit_code_raw[0])
+                current_jobs[job_name]['exit_code'] = exit_code
+                subprocess.call(f"docker logs {job_name} >& /data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/generate_categorical_fim.log", shell=True)
+                subprocess.call(f"docker container rm {job_name}", shell=True)
+
+                os.makedirs(f"/data/catfim/{current_jobs[job_name]['nice_name']}_temp")
+                if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"):
+                    for path in glob.glob(f"/data/catfim/{current_jobs[job_name]['nice_name']}/**/mapping/*", recursive=True):
+                        if not os.path.isdir(path):
+                            shutil.move(path, f"/data/catfim/{current_jobs[job_name]['nice_name']}_temp")
+                            filename = os.path.basename(path)
+                            current_jobs[job_name]['output_files_saved'][os.path.join(f"/data/catfim/{current_jobs[job_name]['nice_name']}", filename)] = 0
+                    shutil.rmtree(f"/data/catfim/{current_jobs[job_name]['nice_name']}")
+                shutil.move(f"/data/catfim/{current_jobs[job_name]['nice_name']}_temp", f"/data/catfim/{current_jobs[job_name]['nice_name']}")
+
+                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/generate_categorical_fim.log"] = 0
                 current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
                 current_jobs[job_name]['status'] = 'Ready to Save File'
     
@@ -192,11 +327,29 @@ def update_loop():
                     if os.path.isdir(temp_path):
                         shutil.rmtree(temp_path)
 
-                    outputs_path = f"/data/outputs/{job_name}"
+                    outputs_path = f"/data/outputs/{current_jobs[job_name]['nice_name']}"
+                    if current_jobs[job_name]['job_type'] == 'release':
+                        outputs_path = f"/data/previous_fim/{current_jobs[job_name]['nice_name']}"
+                        destination = f"/data/viz/{current_jobs[job_name]['nice_name']}"
+
+                        if os.path.isdir(destination):
+                            shutil.rmtree(destination)
+                        if os.path.isdir(f"{outputs_path}/aggregate_fim_outputs"): shutil.move(f"{outputs_path}/aggregate_fim_outputs", destination)
+                        if os.path.isdir(f"{outputs_path}/logs"): shutil.move(f"{outputs_path}/logs", f"{destination}/logs") 
+                        if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"): shutil.move(f"/data/catfim/{current_jobs[job_name]['nice_name']}", f"{destination}/catfim") 
+
                     if os.path.isdir(outputs_path):
                         shutil.rmtree(outputs_path)
+                    if current_jobs[job_name]['job_type'] == 'release':
+                        if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"):
+                            shutil.rmtree(f"/data/catfim/{current_jobs[job_name]['nice_name']}")
+                        try:
+                            os.makedirs(outputs_path)
+                        except:
+                            pass
 
                     current_jobs[job_name]['status'] = 'Completed' if current_jobs[job_name]['exit_code'] == 0 else 'Error'
+                
                     shared_data['current_saving_job'] = ''
                     current_jobs[job_name]['is_actively_saving'] = False
                     print(f"{job_name} completed")
@@ -211,7 +364,6 @@ def update_loop():
         for job in jobs_to_delete:
             del current_jobs[job]
 
-        # TODO: Get Preset Lists here
         presets_list = []
         for path, folders, files in os.walk(f"/data/inputs/huc_lists"):
             for file in files:
@@ -229,8 +381,9 @@ def update_loop():
         } for job in current_jobs.values()]
 
         if shared_data['connected']: sio.emit('update', {'jobUpdates': job_updates, 'presetsList': presets_list})
-        with open('/data/outputs/current_jobs.json', 'w') as f:
+        with open('/data/outputs/current_jobs.json.temp', 'w') as f:
             json.dump(current_jobs, f)
+            shutil.move('/data/outputs/current_jobs.json.temp', '/data/outputs/current_jobs.json') 
 
 sio = socketio.Client()
 
@@ -247,50 +400,93 @@ def disconnect():
 
 @sio.on('add_job_to_queue')
 def ws_add_job_to_queue(data):
-    job_name = data['job_name']
-    branch = data['branch']
-    hucs = data['hucs']
-    parallel_jobs = data['parallel_jobs']
-    hucs_type = data['hucs_type']
-    extent = data['extent']
-    config_path = data['config_path']
-    dev_run = data['dev_run']
-    viz_run = data['viz_run']
-
-    # This is a preset list instead of a custom list of hucs
-    if hucs_type == 0:
+    job_type = data['job_type']
+    if job_type == 'fim_run':
+        job_name = data['job_name']
+        branch = data['branch']
+        hucs = data['hucs']
+        parallel_jobs = data['parallel_jobs']
+        hucs_type = data['hucs_type']
+        extent = data['extent']
+        config_path = data['config_path']
+        dev_run = data['dev_run']
+        viz_run = data['viz_run']
+
+        # This is a preset list instead of a custom list of hucs
+        if hucs_type == 0:
+            if os.path.exists(hucs):
+                with open(hucs, "r") as preset_file:
+                    hucs_raw = preset_file.read().splitlines()
+                    parallel_jobs = len(hucs_raw)
+                    hucs_type = len(hucs_raw[0])
+
+        parallel_jobs = parallel_jobs if parallel_jobs <= MAX_ALLOWED_CPU_CORES else MAX_ALLOWED_CPU_CORES
+
+        buffer_jobs.append({
+            'job_type': job_type,
+            'job_name': job_name,
+            'branch': branch,
+            'hucs': hucs,
+            'parallel_jobs': parallel_jobs,
+            'hucs_type': hucs_type,
+            'extent': extent,
+            'config_path': config_path,
+            'dev_run': dev_run,
+            'viz_run': viz_run,
+            'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
+            'status': 'In Queue',
+            'time_started': 0,
+            'time_elapsed': 0,
+            'output_files_saved': {},
+            'total_output_files_length': 0,
+            'current_output_files_saved_length': 0,
+            'output_files_saved': {},
+            'container_state': 'running',
+            'exit_code': 0,
+            'is_actively_saving': False
+        })
+    elif job_type == 'release':
+        job_name = data['job_name']
+        hucs = data['hucs']
+        extent = data['extent']
+        branch = 'dev'
+        config_path = './foss_fim/config/params_calibrated.env'
+        dev_run = False
+        viz_run = True
+        previous_major_fim_version = data['previous_major_fim_version']
+
         if os.path.exists(hucs):
             with open(hucs, "r") as preset_file:
                 hucs_raw = preset_file.read().splitlines()
                 parallel_jobs = len(hucs_raw)
                 hucs_type = len(hucs_raw[0])
-    print(f"{hucs} {parallel_jobs} {hucs_type}")
-
-
-    parallel_jobs = parallel_jobs if parallel_jobs <= MAX_ALLOWED_CPU_CORES else MAX_ALLOWED_CPU_CORES
-
-    buffer_jobs.append({
-        'job_name': job_name,
-        'branch': branch,
-        'hucs': hucs,
-        'parallel_jobs': parallel_jobs,
-        'hucs_type': hucs_type,
-        'extent': extent,
-        'config_path': config_path,
-        'dev_run': dev_run,
-        'viz_run': viz_run,
-        'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
-        'status': 'In Queue',
-        'time_started': 0,
-        'time_elapsed': 0,
-        'output_files_saved': {},
-        'total_output_files_length': 0,
-        'current_output_files_saved_length': 0,
-        'output_files_saved': {},
-        'container_state': 'running',
-        'exit_code': 0,
-        'is_actively_saving': False
-    })
+
+        parallel_jobs = parallel_jobs if parallel_jobs <= MAX_ALLOWED_CPU_CORES else MAX_ALLOWED_CPU_CORES
+
+        buffer_jobs.append({
+            'job_type': job_type,
+            'job_name': job_name,
+            'branch': branch,
+            'hucs': hucs,
+            'parallel_jobs': parallel_jobs,
+            'hucs_type': hucs_type,
+            'extent': extent,
+            'config_path': config_path,
+            'dev_run': dev_run,
+            'viz_run': viz_run,
+            'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
+            'status': 'In Queue',
+            'time_started': 0,
+            'time_elapsed': 0,
+            'output_files_saved': {},
+            'total_output_files_length': 0,
+            'current_output_files_saved_length': 0,
+            'output_files_saved': {},
+            'container_state': 'running',
+            'exit_code': 0,
+            'is_actively_saving': False,
+            'previous_major_fim_version': previous_major_fim_version
+        })
 
 @sio.on('remove_job_from_queue')
 def ws_remove_job_from_queue(data):

From cbc360f1f27ea15e87510024ae340b5f53fe1638 Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Tue, 11 May 2021 10:42:27 -0500
Subject: [PATCH 075/359] Updated Eventlet package to 0.31.0 to patch a
 vulnerability

The current version of Eventlet used in the Connector module of the FIM API is outdated and vulnerable. This update bumps the version to the patched version.

Using new nhd inputs.
---
 CHANGELOG.md                        | 8 ++++++++
 api/node/connector/requirements.txt | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd4928cf9..aef36d239 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.16.1 - 2021-05-11 - [PR #380](https://github.com/NOAA-OWP/cahaba/pull/380)
+
+The current version of Eventlet used in the Connector module of the FIM API is outdated and vulnerable. This update bumps the version to the patched version.
+
+## Changes
+- Updated `api/node/connector/requirements.txt` to have the Eventlet version as 0.31.0
+
 <br/><br/>
 ## v3.0.16.0 - 2021-05-07 - [PR #378](https://github.com/NOAA-OWP/cahaba/pull/378)
 
diff --git a/api/node/connector/requirements.txt b/api/node/connector/requirements.txt
index 9d8c9d17f..e0b0b19de 100644
--- a/api/node/connector/requirements.txt
+++ b/api/node/connector/requirements.txt
@@ -1,3 +1,3 @@
 flask==1.1.2
 flask-socketio==5.0.0
-eventlet==0.30.0
\ No newline at end of file
+eventlet==0.31.0

From 8fbf4b666efdba9ab7706d6a93b77baf07dede6f Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Tue, 18 May 2021 13:26:56 -0500
Subject: [PATCH 076/359] Bug fixes in preprocessing and run_test_case.py and
 update eval_plots

Modifications and fixes to run_test_case.py, eval_plots.py, and AHPS preprocessing scripts.

- Comment out return statement causing run_test_case.py to skip over sites/hucs when calculating contingency rasters.
- Move bad sites list and query statement used to filter out bad sites to the tools_shared_variables.py.
- Add print statements in eval_plots.py detailing the bad sites used and the query used to filter out bad sites.
- Update AHPS preprocessing scripts to produce a domain shapefile.
- Change output filenames produced in ahps preprocessing scripts.
- Update workarounds for some sites in ahps preprocessing scripts.

This resolves #381, resolves #369, resolves #326, resolves #385, resolves #377.
---
 CHANGELOG.md                    | 13 +++++++++++++
 tools/eval_plots.py             | 27 ++++++++-------------------
 tools/preprocess_ahps_nws.py    | 27 +++++++++++++++++++--------
 tools/preprocess_ahps_usgs.py   | 20 +++++++++++---------
 tools/run_test_case.py          |  2 +-
 tools/tools_shared_variables.py | 14 ++++++++++++++
 6 files changed, 66 insertions(+), 37 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aef36d239..a23b0194e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+## v3.0.16.2 - 2021-05-18 - [PR #384] (https://github.com/NOAA-OWP/cahaba/pull/384)
+
+Modifications and fixes to `run_test_case.py`, `eval_plots.py`, and AHPS preprocessing scripts.
+
+## Changes
+- Comment out return statement causing `run_test_case.py` to skip over sites/hucs when calculating contingency rasters.
+- Move bad sites list and query statement used to filter out bad sites to the `tools_shared_variables.py`.
+- Add print statements in `eval_plots.py` detailing the bad sites used and the query used to filter out bad sites.
+- Update AHPS preprocessing scripts to produce a domain shapefile.
+- Change output filenames produced in ahps preprocessing scripts.
+- Update workarounds for some sites in ahps preprocessing scripts.
+
 <br/><br/>
 ## v3.0.16.1 - 2021-05-11 - [PR #380](https://github.com/NOAA-OWP/cahaba/pull/380)
 
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 860c38382..b2d891c18 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -14,6 +14,7 @@
 from utils.shared_variables import VIZ_PROJECTION
 from dotenv import load_dotenv
 from tools_shared_functions import aggregate_wbd_hucs, get_metadata
+from tools_shared_variables import BAD_SITES, DISCARD_AHPS_QUERY
 
 #Get variables from .env file.
 load_dotenv()
@@ -338,7 +339,7 @@ def filter_dataframe(dataframe, unique_field):
 ##############################################################################
 #Main function to analyze metric csv.
 ##############################################################################
-def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial = False, fim_1_ms = False, site_barplots = False):
+def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , spatial = False, fim_1_ms = False, site_barplots = False):
 
     '''
     Creates plots and summary statistics using metrics compiled from
@@ -411,10 +412,6 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         A list of statistics to be plotted. Must be identical to column
         field in metrics_csv. CSI, POD, TPR are currently calculated, if
         additional statistics are desired formulas would need to be coded.
-    alternate_ahps_query : STRING, optional
-        The default is false. Currently the default ahps query is same
-        as done for apg goals. If a different query is desired it can be
-        supplied and it will supercede the default query.
     spatial : BOOL, optional
         Creates spatial datasets of the base unit (ble: huc polygon, ahps: point) 
         with metrics contained in attribute tables. The geospatial data is 
@@ -483,16 +480,8 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             # Set the base processing unit for the ahps runs.
             base_resolution = 'nws_lid'
 
-            #Default query (used for APG) it could be that bad_sites should be modified. If so pass an alternate query using the "alternate_ahps_query"
-            bad_sites = ['grfi2','ksdm7','hohn4','rwdn4','efdn7','kilo1','chin7','segt2','eagi1','levk1','trbf1']
-            query = "not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites"
-
-            # If alternate ahps evaluation query argument is passed, use that.
-            if alternate_ahps_query:
-                query = alternate_ahps_query
-
-            # Filter the dataset based on query
-            ahps_metrics = benchmark_metrics.query(query)
+            # Filter the dataset based on query (IMPORTED FROM TOOLS_SHARED_VARIABLES.py)
+            ahps_metrics = benchmark_metrics.query(DISCARD_AHPS_QUERY)
 
             # Filter out all instances where the base_resolution doesn't exist across all desired fim versions for a given magnitude
             all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
@@ -659,12 +648,11 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
 #######################################################################
 if __name__ == '__main__':
     # Parse arguments
-    parser = argparse.ArgumentParser(description = 'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
+    parser = argparse.ArgumentParser(description = f'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
     parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
     parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
     parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
     parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR'], required = False)
-    parser.add_argument('-q', '--alternate_ahps_query',help = 'Alternate filter query for AHPS. Default is: "not nws_lid.isnull() & not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites" where bad_sites are (grfi2,ksdm7,hohn4,rwdn4)', default = False, required = False)
     parser.add_argument('-sp', '--spatial', help = 'If enabled, creates spatial layers with metrics populated in attribute table.', action = 'store_true', required = False)
     parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
     parser.add_argument('-i', '--site_plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
@@ -677,10 +665,11 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     w = args['workspace']
     v = args['versions']
     s = args['stats']
-    q = args['alternate_ahps_query']
     sp= args['spatial']
     f = args['fim_1_ms']
     i = args['site_plots']
 
     # Run eval_plots function
-    eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial = sp, fim_1_ms = f, site_barplots = i)
+    print('The following AHPS sites are considered "BAD_SITES":  ' + ', '.join(BAD_SITES))
+    print('The following query is used to filter AHPS:  ' + DISCARD_AHPS_QUERY)
+    eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, spatial = sp, fim_1_ms = f, site_barplots = i)
diff --git a/tools/preprocess_ahps_nws.py b/tools/preprocess_ahps_nws.py
index e2db2b47d..c8efd464a 100644
--- a/tools/preprocess_ahps_nws.py
+++ b/tools/preprocess_ahps_nws.py
@@ -96,6 +96,10 @@ def preprocess_nws(source_dir, destination, reference_raster):
             f.write(f'{code} : skipping because no rating curve source\n')
             continue
         
+        #Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null). Modifying rating curve source will influence the rating curve and datum retrieved for benchmark determinations.
+        if code == 'bmbp1':
+            rating_curve_source = 'NRLDB'
+        
         #Get the datum and adjust to NAVD if necessary.
         nws, usgs = get_datum(metadata)
         datum_data = {}
@@ -116,6 +120,10 @@ def preprocess_nws(source_dir, destination, reference_raster):
         if code in ['bgwn7', 'dlrt2','eagi1','eppt2','fatw3','jffw3','ldot2','mnvn4','nhpp1','pinn4','rgdt2','rgln4','rssk1','sign4','smfn7','stkn4','wlln7' ]:
             datum_data.update(crs = 'NAD83')
         
+        #Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and is actually NAD27. This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and it matches NRLDB coordinates.
+        if code == 'bmbp1':
+            datum_data.update(crs = 'NAD27')
+        
         #Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure datum reported in NAVD88. If NGVD29 it is converted to NAVD88.
         #bgwn7, eagi1 vertical datum unknown, assume navd88
         #fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
@@ -226,7 +234,7 @@ def preprocess_nws(source_dir, destination, reference_raster):
                     #Create Binary Grids, first create domain of analysis, then create binary grid
                     
                     #Domain extent is largest floodmap in the static library WITH holes filled
-                    filled_domain_raster = outputdir.parent / f'{code}_extent.tif'
+                    filled_domain_raster = outputdir.parent / f'{code}_filled_orig_domain.tif'
     
                     #Open benchmark data as a rasterio object.
                     benchmark = rasterio.open(grid)
@@ -261,7 +269,7 @@ def preprocess_nws(source_dir, destination, reference_raster):
                     
                     #Output binary benchmark grid and flow file to destination
                     outputdir.mkdir(parents = True, exist_ok = True)  
-                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_depth_{i}.tif')
+                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_extent_{i}.tif')
                     
                     with rasterio.Env():
                         with rasterio.open(output_raster, 'w', **boolean_profile) as dst:
@@ -283,14 +291,17 @@ def preprocess_nws(source_dir, destination, reference_raster):
             f.write(traceback.format_exc())
             f.write('\n')
             print(traceback.format_exc())               
-        #Process extents, only create extent if ahps code subfolder is present in destination directory.
+        #Wrapup for ahps sites that were processed.
         ahps_directory = destination / huc / code
         if ahps_directory.exists():
-            #Delete extent raster
-            filled_extent = ahps_directory / f'{code}_extent.tif' 
-            if filled_extent.exists:
-                filled_extent.unlink()              
-    
+            #Delete original filled domain raster (it is an intermediate file to create benchmark data)
+            orig_domain_grid = ahps_directory / f'{code}_filled_orig_domain.tif'
+            orig_domain_grid.unlink() 
+            #Create domain shapefile from any benchmark grid for site (each benchmark has domain footprint, value = 0).
+            filled_extent = list(ahps_directory.rglob('*_extent_*.tif'))[0]
+            domain_gpd = raster_to_feature(grid = filled_extent, profile_override = False, footprint_only = True)           
+            domain_gpd['nws_lid'] = code
+            domain_gpd.to_file(ahps_directory / f'{code}_domain.shp')
             #Populate attribute information for site
             grids_attributes = pd.DataFrame(data=grids.items(), columns = ['magnitude','path'])
             flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude','grid_flow_cfs'])
diff --git a/tools/preprocess_ahps_usgs.py b/tools/preprocess_ahps_usgs.py
index f465ba682..8fb7a8147 100644
--- a/tools/preprocess_ahps_usgs.py
+++ b/tools/preprocess_ahps_usgs.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 import numpy as np
 import pathlib
 from pathlib import Path
@@ -355,7 +354,7 @@ def preprocess_usgs(source_dir, destination, reference_raster):
                     #Create Binary Grids, first create domain of analysis, then create binary grid
                     
                     #Domain extent is largest floodmap in the static library WITH holes filled
-                    filled_domain_raster = outputdir.parent / f'{code}_extent.tif'
+                    filled_domain_raster = outputdir.parent / f'{code}_filled_orig_domain.tif'
                     #Create a domain raster if it does not exist.
                     if not filled_domain_raster.exists():
                         #Open extent data as rasterio object
@@ -377,7 +376,7 @@ def preprocess_usgs(source_dir, destination, reference_raster):
                                     
                     #Output binary benchmark grid and flow file to destination
                     outputdir.mkdir(parents = True, exist_ok = True)
-                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_depth_{i}.tif')
+                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_extent_{i}.tif')
                     with rasterio.Env():
                         with rasterio.open(output_raster, 'w', **boolean_profile) as dst:
                             dst.write(boolean_benchmark,1)
@@ -398,14 +397,17 @@ def preprocess_usgs(source_dir, destination, reference_raster):
             f.write(traceback.format_exc())
             f.write('\n')
             print(traceback.format_exc())
-        #Create extent if ahps code subfolder is present in destination directory.
+        #Wrapup for ahps sites that were processed.
         ahps_directory = destination / huc / code
         if ahps_directory.exists():
-            #Delete extent raster
-            filled_extent = ahps_directory / f'{code}_extent.tif' 
-            if filled_extent.exists:
-                filled_extent.unlink()            
-    
+            #Delete original filled domain raster (it is an intermediate file to create benchmark data)
+            orig_domain_grid = ahps_directory / f'{code}_filled_orig_domain.tif'
+            orig_domain_grid.unlink()            
+            #Create domain shapefile from any benchmark grid for site (each benchmark has domain footprint, value = 0).
+            filled_extent = list(ahps_directory.rglob('*_extent_*.tif'))[0]
+            domain_gpd = raster_to_feature(grid = filled_extent, profile_override = False, footprint_only = True)           
+            domain_gpd['nws_lid'] = code
+            domain_gpd.to_file(ahps_directory / f'{code}_domain.shp')   
             #Populate attribute information for site
             grids_attributes = pd.DataFrame(data=grids.items(), columns = ['magnitude','path'])
             flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude','grid_flow_cfs'])
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 3b0f2ff1f..0b8d3183d 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -166,7 +166,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                     print(" ")
                 elif inundate_test == 1:
                     print (f"No matching feature IDs between forecast and hydrotable for magnitude: {magnitude}")
-                    return
+                    #return
             except Exception as e:
                 print(e)
 
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index 292f55de1..7746725af 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -10,6 +10,20 @@
 GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
 GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
 
+# Variables for eval_plots.py
+BAD_SITES = ['grfi2',# Bad crosswalk from USGS to AHPS
+             'ksdm7',# Large areas entirely within levee protected area (CSI of 0)
+             'hohn4',# Mainstems did not extend far enough upstream
+             'rwdn4',# Mainstems did not extend far enough upstream.
+             'efdn7',# Limited upstream mapping
+             'kilo1',# Limited upstream mapping
+             'chin7',# Limited upstream mapping
+             'segt2',# Limited upstream mapping
+             'eagi1',# Missing large portions of FIM
+             'levk1',# Missing large portions of FIM
+             'trbf1']# Limited upstream mapping
+DISCARD_AHPS_QUERY = "not flow.isnull() & masked_perc<97 & not nws_lid in @BAD_SITES"
+
 # Colors.
 ENDC = '\033[m'
 TGREEN_BOLD = '\033[32;1m'

From f2d65eed0166d3d5818091318e2b6487e93209de Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 18 May 2021 13:34:26 -0500
Subject: [PATCH 077/359] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a23b0194e..cae9bec53 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,7 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
-## v3.0.16.2 - 2021-05-18 - [PR #384] (https://github.com/NOAA-OWP/cahaba/pull/384)
+## v3.0.16.2 - 2021-05-18 - [PR #384](https://github.com/NOAA-OWP/cahaba/pull/384)
 
 Modifications and fixes to `run_test_case.py`, `eval_plots.py`, and AHPS preprocessing scripts.
 

From ac25186becf5438be9a1d6ecd4c9af99fef043aa Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 21 May 2021 13:31:07 -0500
Subject: [PATCH 078/359] Alpha test bug fixes and improvements

Enhancements and bug fixes to synthesize_test_cases.py.

- Addresses a bug where AHPS sites without benchmark data were receiving a CSI of 0 in the master metrics CSV produced by synthesize_test_cases.py.
- Includes a feature enhancement to synthesize_test_cases.py that allows for the inclusion of user-specified testing versions in the master metrics CSV.
- Removes some of the print statements used by synthesize_test_cases.py.

This resolves #354 and resolves #386.
---
 CHANGELOG.md                    | 11 +++++++++++
 tools/run_test_case.py          | 26 ++++++++++++++++----------
 tools/synthesize_test_cases.py  | 29 +++++++++++++++++------------
 tools/tools_shared_functions.py |  6 +++---
 4 files changed, 47 insertions(+), 25 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cae9bec53..04221a1e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+
+## v3.0.16.3 - 2021-05-21 - [PR #388](https://github.com/NOAA-OWP/cahaba/pull/388)
+
+Enhancement and bug fixes to `synthesize_test_cases.py`.
+
+## Changes
+- Addresses a bug where AHPS sites without benchmark data were receiving a CSI of 0 in the master metrics CSV produced by `synthesize_test_cases.py`.
+- Includes a feature enhancement to `synthesize_test_cases.py` that allows for the inclusion of user-specified testing versions in the master metrics CSV.
+- Removes some of the print statements used by `synthesize_test_cases.py`.
+
 <br/><br/>
 ## v3.0.16.2 - 2021-05-18 - [PR #384](https://github.com/NOAA-OWP/cahaba/pull/384)
 
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 0b8d3183d..59d90f15f 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -97,12 +97,16 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
 
             for lid in lid_dir_list:
                 lid_dir = os.path.join(validation_data_path, current_huc, lid)
-                benchmark_raster_path_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif'))  # TEMP
-                forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
-                lid_list.append(lid)
-                inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
-                domain_file_list.append(os.path.join(lid_dir, lid + '_domain.shp'))
-
+                benchmark_lid_raster_path = os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
+                
+                # Only compare if the benchmark data exist.
+                if os.path.exists(benchmark_lid_raster_path):
+                    benchmark_raster_path_list.append(benchmark_lid_raster_path)  # TEMP
+                    forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
+                    lid_list.append(lid)
+                    inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
+                    domain_file_list.append(os.path.join(lid_dir, lid + '_domain.shp'))
+                    
         else:
             benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
             benchmark_raster_path_list = [benchmark_raster_file]
@@ -124,13 +128,14 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                      'operation': 'include'}
                         })
 
+                
                 if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_domain_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
                     continue
             else:  # If not in AHPS_BENCHMARK_CATEGORIES.
                 if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
                     continue
             # Run inundate.
-            print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
+#            print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
             try:
                 inundate_test = inundate(
                          rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
@@ -138,7 +143,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                          depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
                         )
                 if inundate_test == 0:
-                    print("-----> Inundation mapping complete.")
+#                    print("-----> Inundation mapping complete.")
                     predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
 
                     # Define outputs for agreement_raster, stats_json, and stats_csv.
@@ -162,10 +167,11 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                         del mask_dict[ahps_lid]
 
                     print(" ")
-                    print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
+                    print("Evaluation metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
                     print(" ")
                 elif inundate_test == 1:
-                    print (f"No matching feature IDs between forecast and hydrotable for magnitude: {magnitude}")
+                    pass
+#                    print (f"No matching feature IDs between forecast and hydrotable for magnitude: {magnitude}")
                     #return
             except Exception as e:
                 print(e)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 142f0d387..de035359c 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -10,7 +10,7 @@
 from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
 
 
-def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
+def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list):
 
     # Construct header
     metrics_to_write = ['true_negatives_count',
@@ -61,9 +61,9 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
 
     versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
 
-    if dev_comparison != None:
+    if len(dev_versions_to_include_list) > 0:
         iteration_list = ['official', 'comparison']
-    if dev_comparison == None:
+    else:
         iteration_list = ['official']
 
     for benchmark_source in ['ble', 'nws', 'usgs']:
@@ -84,7 +84,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
                         if iteration == "comparison":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
-                            versions_to_aggregate = [dev_comparison]
+                            versions_to_aggregate = dev_versions_to_include_list
 
                         for magnitude in ['100yr', '500yr']:
                             for version in versions_to_aggregate:
@@ -140,7 +140,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_comparison):
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
                         if iteration == "comparison":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
-                            versions_to_aggregate = [dev_comparison]
+                            versions_to_aggregate = dev_versions_to_include_list
 
                         for magnitude in ['action', 'minor', 'moderate', 'major']:
                             for version in versions_to_aggregate:
@@ -228,6 +228,7 @@ def process_alpha_test(args):
     parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
     parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
     parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
+    parser.add_argument('-dc', '--dev-version-to-compare', nargs='+', help='Specify the name(s) of a dev (testing) version to include in master metrics CSV. Pass a space-delimited list.',required=False)
     parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=True)
 
     # Assign variables from arguments.
@@ -238,6 +239,7 @@ def process_alpha_test(args):
     special_string = args['special_string']
     benchmark_category = args['benchmark_category']
     overwrite = args['overwrite']
+    dev_versions_to_compare = args['dev_version_to_compare']
     master_metrics_csv = args['master_metrics_csv']
 
     if overwrite:
@@ -301,7 +303,7 @@ def process_alpha_test(args):
 
                         if os.path.exists(fim_run_dir):
 
-                            # If a user supplies a specia_string (-s), then add it to the end of the created dirs.
+                            # If a user supplies a special_string (-s), then add it to the end of the created dirs.
                             if special_string != "":
                                 version = version + '_' + special_string
 
@@ -323,12 +325,15 @@ def process_alpha_test(args):
     if job_number > 1:
         with Pool(processes=job_number) as pool:
             pool.map(process_alpha_test, procs_list)
+            
+    if config == 'DEV':
+        if dev_versions_to_compare != None:
+            dev_versions_to_include_list = dev_versions_to_compare + [version]
+        else:
+            dev_versions_to_include_list = [version]
+    if config == 'PREV':
+        dev_versions_to_include_list = []
 
     # Do aggregate_metrics.
     print("Creating master metrics CSV...")
-
-    if config == 'DEV':
-        dev_comparison = fim_version + "_" + special_string
-    else:
-        dev_comparison = None
-    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, dev_comparison=dev_comparison)
+    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, dev_versions_to_include_list=dev_versions_to_include_list)
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index fb6204dd5..2c4bdc339 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -493,7 +493,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
     import geopandas as gpd
     from shapely.geometry import box
 
-    print("-----> Evaluating performance across the total area...")
+#    print("-----> Evaluating performance across the total area...")
     # Load rasters.
     benchmark_src = rasterio.open(benchmark_raster_path)
     predicted_src = rasterio.open(predicted_raster_path)
@@ -553,7 +553,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                 if poly_all.empty:
                     continue
 
-                print("-----> Masking at " + poly_layer + "...")
+#                print("-----> Masking at " + poly_layer + "...")
                 #Project layer to reference crs.
                 poly_all_proj = poly_all.to_crs(reference.crs)
                 # check if there are any lakes within our reference raster extent.
@@ -636,7 +636,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                 if poly_all.empty:
                     continue
 
-                print("-----> Evaluating performance at " + poly_layer + "...")
+#                print("-----> Evaluating performance at " + poly_layer + "...")
                 #Project layer to reference crs.
                 poly_all_proj = poly_all.to_crs(reference.crs)
                 # check if there are any lakes within our reference raster extent.

From b9167d52a8bb6260d6bae066e9a51e39ae93c11b Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 4 Jun 2021 14:00:06 -0500
Subject: [PATCH 079/359] BARC updates to cap xsec area and allow user to
 choose input bankfull geometry

BARC updates to cap the bathy calculated xsec area in bathy_rc_adjust.py and allow user to choose input bankfull geometry.

- Added new env variable to control which input file is used for the bankfull geometry input to bathy estimation workflow.
- Modified the bathymetry cross section area calculation to cap the additional area value so that it cannot exceed the bankfull cross section area value for each stream segment (bankfull value obtained from regression equation dataset).
- Modified the rating_curve_comparison.py plot output to always put the FIM rating curve on top of the USGS rating curve (avoids USGS points covering FIM).
- Created a new aggregate csv file (aggregates for all hucs) for all of the usgs_elev_table.csv files (one per huc).
- Evaluate the FIM Bathymetry Adjusted Rating Curve (BARC) tool performance using the estimated bankfull geometry dataset derived for the NWM route link dataset.

This resolves #342, resolves #343, resolves #313, resolves #401, and resolves #402.
---
 CHANGELOG.md                     | 28 ++++++++++++++++++++--------
 config/params_calibrated.env     |  4 +++-
 config/params_template.env       |  4 +++-
 src/bathy_rc_adjust.py           |  5 +++--
 src/run_by_unit.sh               |  3 ++-
 tools/rating_curve_comparison.py | 14 +++++++++++++-
 6 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04221a1e2..ed5ea2850 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.17.0 - 2021-06-04 - PR #393
+BARC updates to cap the bathy calculated xsec area in `bathy_rc_adjust.py` and allow user to choose input bankfull geometry.
+
+## Changes
+
+- Added new env variable to control which input file is used for the bankfull geometry input to bathy estimation workflow.
+- Modified the bathymetry cross section area calculation to cap the additional area value so that it cannot exceed the bankfull cross section area value for each stream segment (bankfull value obtained from regression equation dataset).
+- Modified the `rating_curve_comparison.py` plot output to always put the FIM rating curve on top of the USGS rating curve (avoids USGS points covering FIM).
+- Created a new aggregate csv file (aggregates for all hucs) for all of the `usgs_elev_table.csv` files (one per huc).
+- Evaluate the FIM Bathymetry Adjusted Rating Curve (BARC) tool performance using the estimated bankfull geometry dataset derived for the NWM route link dataset.
+
+<br/><br/>
 ## v3.0.16.3 - 2021-05-21 - [PR #388](https://github.com/NOAA-OWP/cahaba/pull/388)
 
 Enhancement and bug fixes to `synthesize_test_cases.py`.
@@ -59,7 +71,7 @@ Remove Great Lakes coastlines from WBD buffer.
 Generate `nws_lid.gpkg`.
 
 ## Additions
-- Generate `nws_lid.gpkg` with attributes indicating if site is a headwater `nws_lid` as well as if it is co-located with another `nws_lid` which is referenced to the same `nwm_feature_id` segment. 
+- Generate `nws_lid.gpkg` with attributes indicating if site is a headwater `nws_lid` as well as if it is co-located with another `nws_lid` which is referenced to the same `nwm_feature_id` segment.
 
 <br/><br/>
 ## v3.0.15.8 - 2021-04-29 - [PR #371](https://github.com/NOAA-OWP/cahaba/pull/371)
@@ -69,7 +81,7 @@ Refactor NHDPlus HR preprocessing workflow. Resolves issue #238
 ## Changes
 - Consolidate NHD streams, NWM catchments, and headwaters MS and FR layers with `mainstem` column.
 - HUC8 intersections are included in the input headwaters layer.
-- `clip_vectors_to_wbd.py` removes incoming stream segment from the selected layers. 
+- `clip_vectors_to_wbd.py` removes incoming stream segment from the selected layers.
 
 <br/><br/>
 ## v3.0.15.7 - 2021-04-28 - [PR #367](https://github.com/NOAA-OWP/cahaba/pull/367)
@@ -124,7 +136,7 @@ Preprocess NHDPlus HR rasters for consistent projections, nodata values, and con
 <br/><br/>
 ## v3.0.15.2 - 2021-04-16 - [PR #359](https://github.com/NOAA-OWP/cahaba/pull/359)
 
-Hotfix to preserve desired files when production flag used in `fim_run.sh`. 
+Hotfix to preserve desired files when production flag used in `fim_run.sh`.
 
 ## Changes
 
@@ -133,17 +145,17 @@ Hotfix to preserve desired files when production flag used in `fim_run.sh`.
 <br/><br/>
 ## v3.0.15.1 - 2021-04-13 - [PR #355](https://github.com/NOAA-OWP/cahaba/pull/355)
 
-Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field. 
+Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field.
 
 ## Changes
 
-- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "MS" run, only consider gages that contain rating curve information (via `curve` attribute) and are also mainstems gages (via `mainstems` attribute). 
-- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "FR" run, only consider gages that contain rating curve information (via `curve` attribute) and are not mainstems gages (via `mainstems` attribute). 
-- Modifies how mainstems segments are determined by using the `nwm_flows_ms.gpkg` as a lookup to determine if the NWM segment specified by WRDS for a gage site is a mainstems gage. 
+- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "MS" run, only consider gages that contain rating curve information (via `curve` attribute) and are also mainstems gages (via `mainstems` attribute).
+- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "FR" run, only consider gages that contain rating curve information (via `curve` attribute) and are not mainstems gages (via `mainstems` attribute).
+- Modifies how mainstems segments are determined by using the `nwm_flows_ms.gpkg` as a lookup to determine if the NWM segment specified by WRDS for a gage site is a mainstems gage.
 
 ## Additions
 
-- Adds a `mainstem` attribute field to `usgs_gages.gpkg` that indicates whether a gage is located on a mainstems river. 
+- Adds a `mainstem` attribute field to `usgs_gages.gpkg` that indicates whether a gage is located on a mainstems river.
 - Adds `NWM_FLOWS_MS` variable to the `.env` and `.env.template` files.
 - Adds the `extent` argument specified by user when running `fim_run.sh` to `usgs_gage_crosswalk.py`.
 
diff --git a/config/params_calibrated.env b/config/params_calibrated.env
index 899c8a4cb..aa7aba1b0 100644
--- a/config/params_calibrated.env
+++ b/config/params_calibrated.env
@@ -20,10 +20,12 @@ export min_catchment_area=0.25
 export min_stream_length=0.5
 
 #### bathy SRC estimation parameters ####
+export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
+#export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
 export bathy_src_modification=True
 export surf_area_thalweg_ratio_flag=10
 export thalweg_stg_search_max_limit=3
-export bathy_xs_area_chg_flag=5
+export bathy_xs_area_chg_flag=1
 export bankful_xs_area_ratio_flag=10
 export thalweg_hyd_radius_flag=10
 
diff --git a/config/params_template.env b/config/params_template.env
index 21d31d1de..a998dc675 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -20,10 +20,12 @@ export min_catchment_area=0.25
 export min_stream_length=0.5
 
 #### bathy SRC estimation parameters ####
+export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
+#export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
 export bathy_src_modification=True
 export surf_area_thalweg_ratio_flag=10
 export thalweg_stg_search_max_limit=3
-export bathy_xs_area_chg_flag=5
+export bathy_xs_area_chg_flag=1
 export bankful_xs_area_ratio_flag=10
 export thalweg_hyd_radius_flag=10
 
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
index 41683c905..ebda61044 100755
--- a/src/bathy_rc_adjust.py
+++ b/src/bathy_rc_adjust.py
@@ -8,7 +8,7 @@
 sa_ratio_flag             = float(environ['surf_area_thalweg_ratio_flag']) #10x
 thal_stg_limit            = float(environ['thalweg_stg_search_max_limit']) #3m
 bankful_xs_ratio_flag     = float(environ['bankful_xs_area_ratio_flag']) #10x
-bathy_xsarea_flag         = float(environ['bathy_xs_area_chg_flag']) #5x
+bathy_xsarea_flag         = float(environ['bathy_xs_area_chg_flag']) #1x
 thal_hyd_radius_flag      = float(environ['thalweg_hyd_radius_flag']) #10x
 
 def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,):
@@ -117,7 +117,8 @@ def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,ou
         ## Calculate the ratio btw the lookup SRC XS_Area and the Bankfull_XSEC_AREA --> use this as a flag for potentially bad XS data
         xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'])
         ## Set bath_cal_xs_area to 0 if the bankfull_XS_ratio_flag is > threshold --> 5x (assuming too large of difference to be a reliable bankfull calculation)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask((xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag) | (xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull()),0,inplace=True)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag,xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],inplace=True)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(),0,inplace=True)
 
         ## Merge bathy_calc_xs_area to the modified_src_base
         modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area']],how='left',on='HydroID')
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 01cc2c98a..3da0983ee 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -34,6 +34,7 @@ huc2Identifier=${hucNumber:0:2}
 input_NHD_WBHD_layer=WBDHU$hucUnitLength
 input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m.tif
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
+input_bathy_bankfull=$inputDataDir/$bankfull_input_table
 
 # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
 if [[ $huc2Identifier == "04" ]] ; then
@@ -400,7 +401,7 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv outp
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $inputDataDir/bathymetry/BANKFULL_CONUS.txt -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
+$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
 ## USGS CROSSWALK ##
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 1f3b138d6..6c2d3b3ba 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -287,7 +287,7 @@ def generate_facet_plot(rc, plot_filename):
         columns = 1
 
     sns.set(style="ticks")
-    g = sns.FacetGrid(rc, col="USGS Gage", hue="source",sharex=False, sharey=False,col_wrap=columns)
+    g = sns.FacetGrid(rc, col="USGS Gage", hue="source", hue_order=['USGS','FIM'], sharex=False, sharey=False,col_wrap=columns)
     g.map(sns.scatterplot, "discharge_cfs", "elevation_ft", palette="tab20c", marker="o")
     g.set_axis_labels(x_var="Discharge (cfs)", y_var="Elevation (ft)")
 
@@ -414,6 +414,7 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     log_file = open(join(output_dir,'rating_curve_comparison.log'),"w")
     sys.stdout = log_file
 
+    merged_elev_table = []
     huc_list  = os.listdir(fim_dir)
     for huc in huc_list:
 
@@ -426,6 +427,17 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
 
             if isfile(elev_table_filename):
                 procs_list.append([elev_table_filename, hydrotable_filename, usgs_gages_filename, usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,nwm_flow_dir, catfim_flows_filename, huc])
+                # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
+                read_elev_table = pd.read_csv(elev_table_filename)
+                read_elev_table['huc'] = huc
+                merged_elev_table.append(read_elev_table)
+
+    # Output a concatenated elev_table to_csv
+    if merged_elev_table:
+        print(f"Creating aggregate elev table csv")
+        concat_elev_table = pd.concat(merged_elev_table)
+        concat_elev_table['thal_burn_depth_meters'] = concat_elev_table['dem_elevation'] - concat_elev_table['dem_adj_elevation']
+        concat_elev_table.to_csv(join(output_dir,'agg_usgs_elev_table.csv'),index=False)
 
     # Initiate multiprocessing
     print(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")

From cca68e14a337714ac28727da5de944e3b7c615d0 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 4 Jun 2021 14:03:05 -0500
Subject: [PATCH 080/359] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ed5ea2850..63b7c0916 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
-## v3.0.17.0 - 2021-06-04 - PR #393
+## v3.0.17.0 - 2021-06-04 - [PR #393](https://github.com/NOAA-OWP/cahaba/pull/393)
 BARC updates to cap the bathy calculated xsec area in `bathy_rc_adjust.py` and allow user to choose input bankfull geometry.
 
 ## Changes

From a001d75fd7905f7232423c9f9ada0b148291e66c Mon Sep 17 00:00:00 2001
From: TrevorGrout-NOAA <69653333+TrevorGrout-NOAA@users.noreply.github.com>
Date: Fri, 4 Jun 2021 15:25:56 -0500
Subject: [PATCH 081/359] Correct headwater assignment nws_lid layer

Bug fix to the generate_nws_lid.py script

- Fixes incorrectly assigned attribute field "is_headwater" for some sites in the nws_lid.gpkg layer.
- Updated agg_nhd_headwaters_adj.gpkg, agg_nhd_streams_adj.gpkg, nwm_flows.gpkg, and nwm_catchments.gpkg input layers using latest NWS LIDs.

This resolves #383 and resolves #397.
---
 CHANGELOG.md              | 9 +++++++++
 tools/generate_nws_lid.py | 6 ++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 63b7c0916..025211419 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.17.1 - 2021-06-04 - [PR #395](https://github.com/NOAA-OWP/cahaba/pull/395)
+
+Bug fix to the `generate_nws_lid.py` script
+
+## Changes
+- Fixes incorrectly assigned attribute field "is_headwater" for some sites in the `nws_lid.gpkg` layer.
+- Updated `agg_nhd_headwaters_adj.gpkg`, `agg_nhd_streams_adj.gpkg`, `nwm_flows.gpkg`, and `nwm_catchments.gpkg` input layers using latest NWS LIDs.
+
+<br/><br/>
 ## v3.0.17.0 - 2021-06-04 - [PR #393](https://github.com/NOAA-OWP/cahaba/pull/393)
 BARC updates to cap the bathy calculated xsec area in `bathy_rc_adjust.py` and allow user to choose input bankfull geometry.
 
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
index a2110c57d..8fda24316 100644
--- a/tools/generate_nws_lid.py
+++ b/tools/generate_nws_lid.py
@@ -127,10 +127,12 @@ def generate_nws_lid(workspace):
             [to_node] = network[from_node]
             #Walk downstream from target
             while to_node>0:
-                [to_node] = network[to_node]
                 #Check if to_node is in targets list
                 if to_node in targets:
-                    sub_dict[to_node] = 'not_headwater'    
+                    sub_dict[to_node] = 'not_headwater'  
+                #Assign downstream ID as to_node
+                [to_node] = network[to_node]
+  
         #Append status to master dictionary
         all_dicts.update(sub_dict)
     

From 129f2d567f2932ad6f3f2b15f8cdc92d8c1d0645 Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Wed, 9 Jun 2021 10:38:19 -0500
Subject: [PATCH 082/359] Fim Run Memory Profiling

To help analyze the memory consumption of the Fim Run process, the python module memory-profiler has been added to give insights into where peak memory usage is with in the codebase.

In addition, the Dockerfile was previously broken due to the Taudem dependency removing the version that was previously being used by FIM. To fix this, and allow new docker images to be built, the Taudem version has been updated to the newest version on the Github repo and thus needs to be thoroughly tested to determine if this new version has affected the overall FIM outputs.

Resolves #392 and #403

Additions
- Added memory-profiler to Pipfile and Pipfile.lock.
- Added mprof (memory-profiler cli utility) call to the time_and_tee_run_by_unitsh to create overall memory usage graph location in the /logs/{HUC}_memory.png of the outputs directory.
- Added @profile decorator to all functions within scripts used in the run_by_unit.sh script to allow for memory usage tracking, which is then recorded in the /logs/{HUC}.log file of the outputs directory.

Changes
- Changed the Taudem version in Dockerfile.dev to 98137bb6541a0d0077a9c95becfed4e56d0aa0ac.
- Changed all calls of python scripts in run_by_unit.sh to be called with the -m memory-profiler argument to allow scripts to also track memory usage.
---
 CHANGELOG.md                       |  16 ++
 Dockerfile.dev                     |   2 +-
 Pipfile                            |   1 +
 Pipfile.lock                       | 373 ++++++++++++++++-------------
 src/add_crosswalk.py               |   1 +
 src/adjust_thalweg_lateral.py      |   2 +-
 src/agreedem.py                    |   2 +-
 src/burn_in_levees.py              |   2 +-
 src/clip_vectors_to_wbd.py         |   4 +-
 src/output_cleanup.py              |   3 +
 src/rem.py                         |   2 +-
 src/run_by_unit.sh                 |  26 +-
 src/time_and_tee_run_by_unit.sh    |   3 +-
 src/unique_pixel_and_allocation.py |   1 +
 14 files changed, 250 insertions(+), 188 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 025211419..81f2e3962 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.18.0 - 2021-06-09 - [PR #404](https://github.com/NOAA-OWP/cahaba/pull/404)
+
+To help analyze the memory consumption of the Fim Run process, the python module `memory-profiler` has been added to give insights into where peak memory usage is with in the codebase.
+
+In addition, the Dockerfile was previously broken due to the Taudem dependency removing the version that was previously being used by FIM. To fix this, and allow new docker images to be built, the Taudem version has been updated to the newest version on the Github repo and thus needs to be thoroughly tested to determine if this new version has affected the overall FIM outputs.
+
+## Additions
+- Added `memory-profiler` to `Pipfile` and `Pipfile.lock`.
+- Added `mprof` (memory-profiler cli utility) call to the `time_and_tee_run_by_unit.sh` to create overall memory usage graph location in the `/logs/{HUC}_memory.png` of the outputs directory.
+- Added `@profile` decorator to all functions within scripts used in the `run_by_unit.sh` script to allow for memory usage tracking, which is then recorded in the `/logs/{HUC}.log` file of the outputs directory.
+
+## Changes
+- Changed the Taudem version in `Dockerfile.dev` to `98137bb6541a0d0077a9c95becfed4e56d0aa0ac`.
+- Changed all calls of python scripts in `run_by_unit.s` to be called with the `-m memory-profiler` argument to allow scripts to also track memory usage.
+
+<br/><br/>
 ## v3.0.17.1 - 2021-06-04 - [PR #395](https://github.com/NOAA-OWP/cahaba/pull/395)
 
 Bug fix to the `generate_nws_lid.py` script
diff --git a/Dockerfile.dev b/Dockerfile.dev
index 638ab46d6..aa646a3f1 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -4,7 +4,7 @@ WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
 ARG depDir=/dependencies
-ARG taudemVersion=bf9417172225a9ce2462f11138c72c569c253a1a
+ARG taudemVersion=98137bb6541a0d0077a9c95becfed4e56d0aa0ac
 ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
 ENV DEBIAN_FRONTEND noninteractive
 ENV taudemDir=$depDir/taudem/bin
diff --git a/Pipfile b/Pipfile
index 219f69ba2..48f8da27d 100755
--- a/Pipfile
+++ b/Pipfile
@@ -21,6 +21,7 @@ grass-session = "==0.5"
 seaborn = "==0.11.0"
 python-dotenv = "*"
 natsort = "*"
+memory-profiler = "*"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 1250dcc4b..e1db9681c 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f74c6d08c7c900c6cea86a5ea4661791356aac573ed738c9c3a1405303a4200c"
+            "sha256": "dbdc717b9aea8c37d6aecaa1b57bf669689c63758542bea1d47bb4e02fddc663"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -25,18 +25,18 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
-                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
+                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.3.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==21.2.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
-                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
+                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
             ],
-            "version": "==2020.12.5"
+            "version": "==2021.5.30"
         },
         "click": {
             "hashes": [
@@ -55,11 +55,11 @@
         },
         "cligj": {
             "hashes": [
-                "sha256:07171c1e287f45511f97df4ea071abc5d19924153413d5683a8e4866369bc676",
-                "sha256:b2f1f7247d59a5387bd3013a08b9ed6829e96fafa4a6e6292341efdb46fe6220"
+                "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
+                "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
-            "version": "==0.7.1"
+            "version": "==0.7.2"
         },
         "cycler": {
             "hashes": [
@@ -162,34 +162,35 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:1de0bb6cbfe460725f0e97b88daa8643bcf9571c18ba90bb8e41432aaeca91d6",
-                "sha256:1e850163579a8936eede29fad41e202b25923a0a8d5ffd08ce50fc0a97dcdc93",
-                "sha256:215e2a30a2090221a9481db58b770ce56b8ef46f13224ae33afe221b14b24dc1",
-                "sha256:348e6032f666ffd151b323342f9278b16b95d4a75dfacae84a11d2829a7816ae",
-                "sha256:3d2eb9c1cc254d0ffa90bc96fde4b6005d09c2228f99dfd493a4219c1af99644",
-                "sha256:3e477db76c22929e4c6876c44f88d790aacdf3c3f8f3a90cb1975c0bf37825b0",
-                "sha256:451cc89cb33d6652c509fc6b588dc51c41d7246afdcc29b8624e256b7663ed1f",
-                "sha256:46b1a60a04e6d884f0250d5cc8dc7bd21a9a96c584a7acdaab44698a44710bab",
-                "sha256:5f571b92a536206f7958f7cb2d367ff6c9a1fa8229dc35020006e4cdd1ca0acd",
-                "sha256:672960dd114e342b7c610bf32fb99d14227f29919894388b41553217457ba7ef",
-                "sha256:7310e353a4a35477c7f032409966920197d7df3e757c7624fd842f3eeb307d3d",
-                "sha256:746a1df55749629e26af7f977ea426817ca9370ad1569436608dc48d1069b87c",
-                "sha256:7c155437ae4fd366e2700e2716564d1787700687443de46bcb895fe0f84b761d",
-                "sha256:9265ae0fb35e29f9b8cc86c2ab0a2e3dcddc4dd9de4b85bf26c0f63fe5c1c2ca",
-                "sha256:94bdd1d55c20e764d8aea9d471d2ae7a7b2c84445e0fa463f02e20f9730783e1",
-                "sha256:9a79e5dd7bb797aa611048f5b70588b23c5be05b63eefd8a0d152ac77c4243db",
-                "sha256:a17f0a10604fac7627ec82820439e7db611722e80c408a726cd00d8c974c2fb3",
-                "sha256:a1acb72f095f1d58ecc2538ed1b8bca0b57df313b13db36ed34b8cdf1868e674",
-                "sha256:aa49571d8030ad0b9ac39708ee77bd2a22f87815e12bdee52ecaffece9313ed8",
-                "sha256:c24c05f645aef776e8b8931cb81e0f1632d229b42b6d216e30836e2e145a2b40",
-                "sha256:cf3a7e54eff792f0815dbbe9b85df2f13d739289c93d346925554f71d484be78",
-                "sha256:d738acfdfb65da34c91acbdb56abed46803db39af259b7f194dc96920360dbe4",
-                "sha256:e15fa23d844d54e7b3b7243afd53b7567ee71c721f592deb0727ee85e668f96a",
-                "sha256:ed4a9e6dcacba56b17a0a9ac22ae2c72a35b7f0ef0693aa68574f0b2df607a89",
-                "sha256:f44149a0ef5b4991aaef12a93b8e8d66d6412e762745fea1faa61d98524e0ba9"
+                "sha256:0bea5ec5c28d49020e5d7923c2725b837e60bc8be99d3164af410eb4b4c827da",
+                "sha256:1c1779f7ab7d8bdb7d4c605e6ffaa0614b3e80f1e3c8ccf7b9269a22dbc5986b",
+                "sha256:21b31057bbc5e75b08e70a43cefc4c0b2c2f1b1a850f4a0f7af044eb4163086c",
+                "sha256:32fa638cc10886885d1ca3d409d4473d6a22f7ceecd11322150961a70fab66dd",
+                "sha256:3a5c18dbd2c7c366da26a4ad1462fe3e03a577b39e3b503bbcf482b9cdac093c",
+                "sha256:5826f56055b9b1c80fef82e326097e34dc4af8c7249226b7dd63095a686177d1",
+                "sha256:6382bc6e2d7e481bcd977eb131c31dee96e0fb4f9177d15ec6fb976d3b9ace1a",
+                "sha256:6475d0209024a77f869163ec3657c47fed35d9b6ed8bccba8aa0f0099fbbdaa8",
+                "sha256:6a6a44f27aabe720ec4fd485061e8a35784c2b9ffa6363ad546316dfc9cea04e",
+                "sha256:7a58f3d8fe8fac3be522c79d921c9b86e090a59637cb88e3bc51298d7a2c862a",
+                "sha256:7ad19f3fb6145b9eb41c08e7cbb9f8e10b91291396bee21e9ce761bb78df63ec",
+                "sha256:85f191bb03cb1a7b04b5c2cca4792bef94df06ef473bc49e2818105671766fee",
+                "sha256:956c8849b134b4a343598305a3ca1bdd3094f01f5efc8afccdebeffe6b315247",
+                "sha256:a9d8cb5329df13e0cdaa14b3b43f47b5e593ec637f13f14db75bb16e46178b05",
+                "sha256:b1d5a2cedf5de05567c441b3a8c2651fbde56df08b82640e7f06c8cd91e201f6",
+                "sha256:b26535b9de85326e6958cdef720ecd10bcf74a3f4371bf9a7e5b2e659c17e153",
+                "sha256:c541ee5a3287efe066bbe358320853cf4916bc14c00c38f8f3d8d75275a405a9",
+                "sha256:d8d994cefdff9aaba45166eb3de4f5211adb4accac85cbf97137e98f26ea0219",
+                "sha256:df815378a754a7edd4559f8c51fc7064f779a74013644a7f5ac7a0c31f875866"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.3.4"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.4.2"
+        },
+        "memory-profiler": {
+            "hashes": [
+                "sha256:01385ac0fec944fcf7969814ec4406c6d8a9c66c079d09276723c5a7680f44e5"
+            ],
+            "index": "pypi",
+            "version": "==0.58.0"
         },
         "munch": {
             "hashes": [
@@ -233,33 +234,33 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:032be656d89bbf786d743fee11d01ef318b0781281241997558fa7950028dd29",
-                "sha256:104f5e90b143dbf298361a99ac1af4cf59131218a045ebf4ee5990b83cff5fab",
-                "sha256:125a0e10ddd99a874fd357bfa1b636cd58deb78ba4a30b5ddb09f645c3512e04",
-                "sha256:12e4ba5c6420917571f1a5becc9338abbde71dd811ce40b37ba62dec7b39af6d",
-                "sha256:13adf545732bb23a796914fe5f891a12bd74cf3d2986eed7b7eba2941eea1590",
-                "sha256:2d7e27442599104ee08f4faed56bb87c55f8b10a5494ac2ead5c98a4b289e61f",
-                "sha256:3bc63486a870294683980d76ec1e3efc786295ae00128f9ea38e2c6e74d5a60a",
-                "sha256:3d3087e24e354c18fb35c454026af3ed8997cfd4997765266897c68d724e4845",
-                "sha256:4ed8e96dc146e12c1c5cdd6fb9fd0757f2ba66048bf94c5126b7efebd12d0090",
-                "sha256:60759ab15c94dd0e1ed88241fd4fa3312db4e91d2c8f5a2d4cf3863fad83d65b",
-                "sha256:65410c7f4398a0047eea5cca9b74009ea61178efd78d1be9847fac1d6716ec1e",
-                "sha256:66b467adfcf628f66ea4ac6430ded0614f5cc06ba530d09571ea404789064adc",
-                "sha256:7199109fa46277be503393be9250b983f325880766f847885607d9b13848f257",
-                "sha256:72251e43ac426ff98ea802a931922c79b8d7596480300eb9f1b1e45e0543571e",
-                "sha256:89e5336f2bec0c726ac7e7cdae181b325a9c0ee24e604704ed830d241c5e47ff",
-                "sha256:89f937b13b8dd17b0099c7c2e22066883c86ca1575a975f754babc8fbf8d69a9",
-                "sha256:9c94cab5054bad82a70b2e77741271790304651d584e2cdfe2041488e753863b",
-                "sha256:9eb551d122fadca7774b97db8a112b77231dcccda8e91a5bc99e79890797175e",
-                "sha256:a1d7995d1023335e67fb070b2fae6f5968f5be3802b15ad6d79d81ecaa014fe0",
-                "sha256:ae61f02b84a0211abb56462a3b6cd1e7ec39d466d3160eb4e1da8bf6717cdbeb",
-                "sha256:b9410c0b6fed4a22554f072a86c361e417f0258838957b78bd063bde2c7f841f",
-                "sha256:c26287dfc888cf1e65181f39ea75e11f42ffc4f4529e5bd19add57ad458996e2",
-                "sha256:c91ec9569facd4757ade0888371eced2ecf49e7982ce5634cc2cf4e7331a4b14",
-                "sha256:ecb5b74c702358cdc21268ff4c37f7466357871f53a30e6f84c686952bef16a9"
+                "sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010",
+                "sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd",
+                "sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43",
+                "sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9",
+                "sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df",
+                "sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400",
+                "sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2",
+                "sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4",
+                "sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a",
+                "sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6",
+                "sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8",
+                "sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b",
+                "sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8",
+                "sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb",
+                "sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2",
+                "sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f",
+                "sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4",
+                "sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a",
+                "sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16",
+                "sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f",
+                "sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69",
+                "sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65",
+                "sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17",
+                "sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.20.1"
+            "version": "==1.20.3"
         },
         "pandas": {
             "hashes": [
@@ -285,42 +286,77 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:15306d71a1e96d7e271fd2a0737038b5a92ca2978d2e38b6ced7966583e3d5af",
-                "sha256:1940fc4d361f9cc7e558d6f56ff38d7351b53052fd7911f4b60cd7bc091ea3b1",
-                "sha256:1f93f2fe211f1ef75e6f589327f4d4f8545d5c8e826231b042b483d8383e8a7c",
-                "sha256:30d33a1a6400132e6f521640dd3f64578ac9bfb79a619416d7e8802b4ce1dd55",
-                "sha256:328240f7dddf77783e72d5ed79899a6b48bc6681f8d1f6001f55933cb4905060",
-                "sha256:46c2bcf8e1e75d154e78417b3e3c64e96def738c2a25435e74909e127a8cba5e",
-                "sha256:5762ebb4436f46b566fc6351d67a9b5386b5e5de4e58fdaa18a1c83e0e20f1a8",
-                "sha256:5a2d957eb4aba9d48170b8fe6538ec1fbc2119ffe6373782c03d8acad3323f2e",
-                "sha256:5cf03b9534aca63b192856aa601c68d0764810857786ea5da652581f3a44c2b0",
-                "sha256:5daba2b40782c1c5157a788ec4454067c6616f5a0c1b70e26ac326a880c2d328",
-                "sha256:63cd413ac52ee3f67057223d363f4f82ce966e64906aea046daf46695e3c8238",
-                "sha256:6efac40344d8f668b6c4533ae02a48d52fd852ef0654cc6f19f6ac146399c733",
-                "sha256:71b01ee69e7df527439d7752a2ce8fb89e19a32df484a308eca3e81f673d3a03",
-                "sha256:71f31ee4df3d5e0b366dd362007740106d3210fb6a56ec4b581a5324ba254f06",
-                "sha256:72027ebf682abc9bafd93b43edc44279f641e8996fb2945104471419113cfc71",
-                "sha256:74cd9aa648ed6dd25e572453eb09b08817a1e3d9f8d1bd4d8403d99e42ea790b",
-                "sha256:81b3716cc9744ffdf76b39afb6247eae754186838cedad0b0ac63b2571253fe6",
-                "sha256:8565355a29655b28fdc2c666fd9a3890fe5edc6639d128814fafecfae2d70910",
-                "sha256:87f42c976f91ca2fc21a3293e25bd3cd895918597db1b95b93cbd949f7d019ce",
-                "sha256:89e4c757a91b8c55d97c91fa09c69b3677c227b942fa749e9a66eef602f59c28",
-                "sha256:8c4e32218c764bc27fe49b7328195579581aa419920edcc321c4cb877c65258d",
-                "sha256:903293320efe2466c1ab3509a33d6b866dc850cfd0c5d9cc92632014cec185fb",
-                "sha256:90882c6f084ef68b71bba190209a734bf90abb82ab5e8f64444c71d5974008c6",
-                "sha256:98afcac3205d31ab6a10c5006b0cf040d0026a68ec051edd3517b776c1d78b09",
-                "sha256:a01da2c266d9868c4f91a9c6faf47a251f23b9a862dce81d2ff583135206f5be",
-                "sha256:aeab4cd016e11e7aa5cfc49dcff8e51561fa64818a0be86efa82c7038e9369d0",
-                "sha256:b07c660e014852d98a00a91adfbe25033898a9d90a8f39beb2437d22a203fc44",
-                "sha256:bead24c0ae3f1f6afcb915a057943ccf65fc755d11a1410a909c1fefb6c06ad1",
-                "sha256:d1d6bca39bb6dd94fba23cdb3eeaea5e30c7717c5343004d900e2a63b132c341",
-                "sha256:e2cd8ac157c1e5ae88b6dd790648ee5d2777e76f1e5c7d184eaddb2938594f34",
-                "sha256:e5739ae63636a52b706a0facec77b2b58e485637e1638202556156e424a02dc2",
-                "sha256:f36c3ff63d6fc509ce599a2f5b0d0732189eed653420e7294c039d342c6e204a",
-                "sha256:f91b50ad88048d795c0ad004abbe1390aa1882073b1dca10bfd55d0b8cf18ec5"
+                "sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5",
+                "sha256:031a6c88c77d08aab84fecc05c3cde8414cd6f8406f4d2b16fed1e97634cc8a4",
+                "sha256:083781abd261bdabf090ad07bb69f8f5599943ddb539d64497ed021b2a67e5a9",
+                "sha256:0d19d70ee7c2ba97631bae1e7d4725cdb2ecf238178096e8c82ee481e189168a",
+                "sha256:0e04d61f0064b545b989126197930807c86bcbd4534d39168f4aa5fda39bb8f9",
+                "sha256:12e5e7471f9b637762453da74e390e56cc43e486a88289995c1f4c1dc0bfe727",
+                "sha256:22fd0f42ad15dfdde6c581347eaa4adb9a6fc4b865f90b23378aa7914895e120",
+                "sha256:238c197fc275b475e87c1453b05b467d2d02c2915fdfdd4af126145ff2e4610c",
+                "sha256:3b570f84a6161cf8865c4e08adf629441f56e32f180f7aa4ccbd2e0a5a02cba2",
+                "sha256:463822e2f0d81459e113372a168f2ff59723e78528f91f0bd25680ac185cf797",
+                "sha256:4d98abdd6b1e3bf1a1cbb14c3895226816e666749ac040c4e2554231068c639b",
+                "sha256:5afe6b237a0b81bd54b53f835a153770802f164c5570bab5e005aad693dab87f",
+                "sha256:5b70110acb39f3aff6b74cf09bb4169b167e2660dabc304c1e25b6555fa781ef",
+                "sha256:5cbf3e3b1014dddc45496e8cf38b9f099c95a326275885199f427825c6522232",
+                "sha256:624b977355cde8b065f6d51b98497d6cd5fbdd4f36405f7a8790e3376125e2bb",
+                "sha256:63728564c1410d99e6d1ae8e3b810fe012bc440952168af0a2877e8ff5ab96b9",
+                "sha256:66cc56579fd91f517290ab02c51e3a80f581aba45fd924fcdee01fa06e635812",
+                "sha256:6c32cc3145928c4305d142ebec682419a6c0a8ce9e33db900027ddca1ec39178",
+                "sha256:8b56553c0345ad6dcb2e9b433ae47d67f95fc23fe28a0bde15a120f25257e291",
+                "sha256:8bb1e155a74e1bfbacd84555ea62fa21c58e0b4e7e6b20e4447b8d07990ac78b",
+                "sha256:95d5ef984eff897850f3a83883363da64aae1000e79cb3c321915468e8c6add5",
+                "sha256:a013cbe25d20c2e0c4e85a9daf438f85121a4d0344ddc76e33fd7e3965d9af4b",
+                "sha256:a787ab10d7bb5494e5f76536ac460741788f1fbce851068d73a87ca7c35fc3e1",
+                "sha256:a7d5e9fad90eff8f6f6106d3b98b553a88b6f976e51fce287192a5d2d5363713",
+                "sha256:aac00e4bc94d1b7813fe882c28990c1bc2f9d0e1aa765a5f2b516e8a6a16a9e4",
+                "sha256:b91c36492a4bbb1ee855b7d16fe51379e5f96b85692dc8210831fbb24c43e484",
+                "sha256:c03c07ed32c5324939b19e36ae5f75c660c81461e312a41aea30acdd46f93a7c",
+                "sha256:c5236606e8570542ed424849f7852a0ff0bce2c4c8d0ba05cc202a5a9c97dee9",
+                "sha256:c6b39294464b03457f9064e98c124e09008b35a62e3189d3513e5148611c9388",
+                "sha256:cb7a09e173903541fa888ba010c345893cd9fc1b5891aaf060f6ca77b6a3722d",
+                "sha256:d68cb92c408261f806b15923834203f024110a2e2872ecb0bd2a110f89d3c602",
+                "sha256:dc38f57d8f20f06dd7c3161c59ca2c86893632623f33a42d592f097b00f720a9",
+                "sha256:e98eca29a05913e82177b3ba3d198b1728e164869c613d76d0de4bde6768a50e",
+                "sha256:f217c3954ce5fd88303fc0c317af55d5e0204106d86dea17eb8205700d47dec2"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==8.1.2"
+            "version": "==8.2.0"
+        },
+        "psutil": {
+            "hashes": [
+                "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64",
+                "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131",
+                "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c",
+                "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6",
+                "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023",
+                "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df",
+                "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394",
+                "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4",
+                "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b",
+                "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2",
+                "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d",
+                "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65",
+                "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d",
+                "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef",
+                "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7",
+                "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60",
+                "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6",
+                "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8",
+                "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b",
+                "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d",
+                "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac",
+                "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935",
+                "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d",
+                "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28",
+                "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876",
+                "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0",
+                "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3",
+                "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==5.8.0"
         },
         "pygeos": {
             "hashes": [
@@ -354,34 +390,28 @@
         },
         "pyproj": {
             "hashes": [
-                "sha256:09bead60769e69b592e8cb3ac51b5215f75e9bb9c213ce575031961deb48d6da",
-                "sha256:09db64a8088b23f001e574d92bcc3080bf7de44ddca152d0282a2b50c918a64a",
-                "sha256:1be7d54900eb7e2d1e637319080b3a047c70d1fb2f3c12d3400c0fa8a90cf440",
-                "sha256:36ba436675f9dea4ab3db7d9a32d3ff11c2fbb4d6690a83454d2f3c5c0b54041",
-                "sha256:3e7e851e6d58c16ac2cd920a1bacb7fbb24758a6fcd7f234d594a88ebae04ec9",
-                "sha256:489a96da87d8846c34c90da90e637544e4f4f50a13589b5aac54297f5ee1b01d",
-                "sha256:4a333f3e46fe8b2eb4647a3daa3a2cec52ddc6c107c653b45880526114942ee8",
-                "sha256:708d6e01b9ff3d6dc62a5ad2d2ba1264a863eaa657c1a9bf713a10cc35d34553",
-                "sha256:7ae8e7052f18fde1884574da449010e94fa205ad27aeeaa34a097f49a1ed6a2b",
-                "sha256:7bfaa34e8bb0510d4380310374deecd9e4328b9cf556925cfb45b5a94d5bbdbe",
-                "sha256:81c06df20d09d621e52791c19ce3c880695fb430061e59c2472fa5467e890391",
-                "sha256:86ef2fcd584a3222bf73e2befc24b2badd139b3371f4a1e88649978ef7649540",
-                "sha256:9666d01faf4e758ac68f2c16695c90de49c3170e3760988bf76a34aae11f4e15",
-                "sha256:9b845510255f9580d7e226dd3321a51c468cefb7be24e46415caf67caa4287c4",
-                "sha256:9e2ef75401f17062166d3fe53c555cd62c9577697a2f5ded916b23c54e5db497",
-                "sha256:a3805e026a5547be205a5e322c08e3069f0a48c63bbd53dbc7a8e3499bc66d58",
-                "sha256:a3a8ab19232bf4f4bb2590536538881b7bd0c07df23e0c2a792402ca2476c197",
-                "sha256:aa0a2981b25145523ca17a643c5be077fe13e514fdca9b6d1c412a95d723a5a5",
-                "sha256:bc70b6adcfa713d89bc561673cb57af5fb3a1718cd7d57ec537430cd1007a864",
-                "sha256:bfbac35490dd17f706700673506eeb8170f8a2a63fb5878171d4e6eef242d141",
-                "sha256:c658afc8a6115b58b02aa53d27bf2a67c1b00b55067edb1b7711c6c7391cfaa9",
-                "sha256:cba99e171d744969e13a865ad28fa9c949c4400b0e9c431a802cdd804f52f632",
-                "sha256:d27d40ec541ef69a5107bfcd85f40170e9e122ceb6315ce508cd44d199983d41",
-                "sha256:f942a976ea3de6a519cf48be30a12f465e44d0ac0c38a0d820ab3acfcc0a48a6",
-                "sha256:fee7517bd389a1db7b8bebb18838d04dedca9eaacda01d353d98f5ee421f263e"
+                "sha256:04c185102e659439c5bd428ac5473d36ef795fca8e225bbbe78e20643d804ec0",
+                "sha256:10dad599b9f7ce2194996dc25f1000e0aa15754ecef9db46b624713959c67957",
+                "sha256:1e88ebc4e08e661e9011b5c1ebfb32f0d311963a9824a6effb4168c7e07918b1",
+                "sha256:4f3ad09cf3352bf5664794042b28d98781362ec8d9774ad73f28a1a0101a27f1",
+                "sha256:5f8a8d982bde211e65dc2de1f8f36cf162f9cc7fcd8a7625046ea265284e5e65",
+                "sha256:67b94f4e694ae33fc90dfb7da0e6b5ed5f671dd0acc2f6cf46e9c39d56e16e1a",
+                "sha256:808f5992320e9631b2e45444028a65cd6ba3ee40229292934178ef07020a5ffd",
+                "sha256:8eda240225971b5cd0bac2d399ed6222068f0598ee92d5f6e847bd2019d2c8b0",
+                "sha256:911d773da9fa4d4f3f7580173858c391e3ee0b61acaf0be303baab323d2eae78",
+                "sha256:9cc464a1c51baad28ffb7a233116e8d4ce4c560b32039fa986d0f992ac3c431f",
+                "sha256:a162ed199cd2ec392cffe20b2fa3381b68e7a166d55f3f060eceb8d517e4f46d",
+                "sha256:aa87df0982aa0f4477478899d9c930cc0f97cd6d8a4ce84c43ac88ccf86d1da7",
+                "sha256:ae237492767e0225f99b53a0fd7110fde2b7e7cabc105bbc243c151a7497de88",
+                "sha256:ae5534fa7a3b74f20534694d297fce6f7483890ff6ca404394ecf372f3c589d4",
+                "sha256:b635e7e21fea5af74e90fc9e54d1a4c27078efdce6f214101c98dd93afae599a",
+                "sha256:b6c74bbec679199746a3e02c0e0fad093c3652df96dd63e086a2fbf2afe9dc0e",
+                "sha256:c4193e1069d165476b2d0f7d882b7712b3eab6e2e6fe2a0a78ef40de825a1f28",
+                "sha256:da88abc5e2f6a8fb07533855a57ca2a31845f58901a87f821b68b0db6b023978",
+                "sha256:ebbba7707fe83a01e54bce8e3e7342feb0b3e0d74ff8c28df12f8bc59b76827c"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.0.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.1.0"
         },
         "python-dateutil": {
             "hashes": [
@@ -393,11 +423,11 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e",
-                "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"
+                "sha256:00aa34e92d992e9f8383730816359647f358f4a3be1ba45e5a5cefd27ee91544",
+                "sha256:b1ae5e9643d5ed987fc57cc2583021e38db531946518130777734f9589b3141f"
             ],
             "index": "pypi",
-            "version": "==0.15.0"
+            "version": "==0.17.1"
         },
         "pytz": {
             "hashes": [
@@ -451,28 +481,28 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67",
-                "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d",
-                "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d",
-                "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced",
-                "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07",
-                "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf",
-                "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c",
-                "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78",
-                "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4",
-                "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0",
-                "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627",
-                "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4",
-                "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76",
-                "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261",
-                "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f",
-                "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519",
-                "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11",
-                "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092",
-                "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.1"
+                "sha256:01b38dec7e9f897d4db04f8de4e20f0f5be3feac98468188a0f47a991b796055",
+                "sha256:10dbcc7de03b8d635a1031cb18fd3eaa997969b64fdf78f99f19ac163a825445",
+                "sha256:19aeac1ad3e57338723f4657ac8520f41714804568f2e30bd547d684d72c392e",
+                "sha256:1b21c6e0dc97b1762590b70dee0daddb291271be0580384d39f02c480b78290a",
+                "sha256:1caade0ede6967cc675e235c41451f9fb89ae34319ddf4740194094ab736b88d",
+                "sha256:23995dfcf269ec3735e5a8c80cfceaf384369a47699df111a6246b83a55da582",
+                "sha256:2a799714bf1f791fb2650d73222b248d18d53fd40d6af2df2c898db048189606",
+                "sha256:3274ce145b5dc416c49c0cf8b6119f787f0965cd35e22058fe1932c09fe15d77",
+                "sha256:33d1677d46111cfa1c84b87472a0274dde9ef4a7ef2e1f155f012f5f1e995d8f",
+                "sha256:44d452850f77e65e25b1eb1ac01e25770323a782bfe3a1a3e43847ad4266d93d",
+                "sha256:9e3302149a369697c6aaea18b430b216e3c88f9a61b62869f6104881e5f9ef85",
+                "sha256:a75b014d3294fce26852a9d04ea27b5671d86736beb34acdfc05859246260707",
+                "sha256:ad7269254de06743fb4768f658753de47d8b54e4672c5ebe8612a007a088bd48",
+                "sha256:b30280fbc1fd8082ac822994a98632111810311a9ece71a0e48f739df3c555a2",
+                "sha256:b79104878003487e2b4639a20b9092b02e1bad07fc4cf924b495cf413748a777",
+                "sha256:d449d40e830366b4c612692ad19fbebb722b6b847f78a7b701b1e0d6cda3cc13",
+                "sha256:d647757373985207af3343301d89fe738d5a294435a4f2aafb04c13b4388c896",
+                "sha256:f68eb46b86b2c246af99fcaa6f6e37c7a7a413e1084a794990b877f2ff71f7b6",
+                "sha256:fdf606341cd798530b05705c87779606fcdfaf768a8129c348ea94441da15b04"
+            ],
+            "markers": "python_version < '3.10' and python_version >= '3.7'",
+            "version": "==1.6.3"
         },
         "seaborn": {
             "hashes": [
@@ -560,11 +590,11 @@
         },
         "six": {
             "hashes": [
-                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
-                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.15.0"
+            "version": "==1.16.0"
         },
         "snuggs": {
             "hashes": [
@@ -592,18 +622,19 @@
         },
         "decorator": {
             "hashes": [
-                "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760",
-                "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"
+                "sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323",
+                "sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5"
             ],
-            "version": "==4.4.2"
+            "markers": "python_version >= '3.5'",
+            "version": "==5.0.9"
         },
         "ipython": {
             "hashes": [
-                "sha256:04323f72d5b85b606330b6d7e2dc8d2683ad46c3905e955aa96ecc7a99388e70",
-                "sha256:34207ffb2f653bced2bc8e3756c1db86e7d93e44ed049daae9814fed66d408ec"
+                "sha256:a171caa3d3d4c819a1c0742e3abecfd5a2b8ab525ca1c9f114b40b76b0679ab1",
+                "sha256:f86788eef439891438af3498525094cc2acbdbea4f2aa2f8895782d4ff471341"
             ],
             "index": "pypi",
-            "version": "==7.21.0"
+            "version": "==7.24.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -620,13 +651,21 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.18.0"
         },
+        "matplotlib-inline": {
+            "hashes": [
+                "sha256:5cf1176f554abb4fa98cb362aa2b55c500147e4bdbb07e3fda359143e1da0811",
+                "sha256:f41d5ff73c9f5385775d5c0bc13b424535c8402fe70ea8210f93e11f3683993e"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==0.1.2"
+        },
         "parso": {
             "hashes": [
-                "sha256:15b00182f472319383252c18d5913b69269590616c947747bc50bf4ac768f410",
-                "sha256:8519430ad07087d4c997fda3a7918f7cfa27cb58972a8c89c2a0295a1c940e9e"
+                "sha256:12b83492c6239ce32ff5eed6d3639d6a536170723c6f3f1506869f1ace413398",
+                "sha256:a8c4922db71e4fdb90e0d0bc6e50f9b273d3397925e5e60a717e719201778d22"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.8.1"
+            "version": "==0.8.2"
         },
         "pexpect": {
             "hashes": [
@@ -645,11 +684,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:0fa02fa80363844a4ab4b8d6891f62dd0645ba672723130423ca4037b80c1974",
-                "sha256:62c811e46bd09130fb11ab759012a4ae385ce4fb2073442d1898867a824183bd"
+                "sha256:bf00f22079f5fadc949f42ae8ff7f05702826a97059ffcc6281036ad40ac6f04",
+                "sha256:e1b4f11b9336a28fa11810bc623c357420f69dfdb6d2dac41ca2c21a55c033bc"
             ],
             "markers": "python_full_version >= '3.6.1'",
-            "version": "==3.0.16"
+            "version": "==3.0.18"
         },
         "ptyprocess": {
             "hashes": [
@@ -660,11 +699,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:2656e1a6edcdabf4275f9a3640db59fd5de107d88e8663c5d4e9a0fa62f77f94",
-                "sha256:534ef71d539ae97d4c3a4cf7d6f110f214b0e687e92f9cb9d2a3b0d3101289c8"
+                "sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f",
+                "sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==2.8.1"
+            "version": "==2.9.0"
         },
         "traitlets": {
             "hashes": [
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index ef21ea23a..d92a0dc13 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -12,6 +12,7 @@
 from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
 
+@profile
 def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
 
     input_catchments = gpd.read_file(input_catchments_fileName)
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index 24b0222e2..90c8cecbf 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -6,7 +6,7 @@
 import rasterio
 import numpy as np
 
-
+@profile
 def adjust_thalweg_laterally(elevation_raster, stream_raster, allocation_raster, cost_distance_raster, cost_distance_tolerance, dem_lateral_thalweg_adj):
     
     # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
diff --git a/src/agreedem.py b/src/agreedem.py
index dbff2d2d4..3d92a2d42 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -5,7 +5,7 @@
 import argparse
 from r_grow_distance import r_grow_distance
 
-
+@profile
 def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data):
     '''
     Produces a hydroconditioned raster using the AGREE DEM methodology as described by Ferdi Hellweger (https://www.caee.utexas.edu/prof/maidment/gishydro/ferdi/research/agree/agree.html). The GRASS gis tool r.grow.distance is used to calculate intermediate allocation and proximity rasters.
diff --git a/src/burn_in_levees.py b/src/burn_in_levees.py
index 89d10a0b0..010c41e0c 100755
--- a/src/burn_in_levees.py
+++ b/src/burn_in_levees.py
@@ -4,7 +4,7 @@
 import numpy as np
 import argparse
 
-
+@profile
 def burn_in_levees(dem_filename,nld_filename,out_dem_filename):
 
     dem = rasterio.open(dem_filename)
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 9c3d73470..ef7ff4837 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -7,8 +7,8 @@
 from shapely.geometry import MultiPolygon,Polygon,Point
 from utils.shared_functions import getDriver
 
+@profile
 def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance,dissolveLinks=False):
-
     hucUnitLength = len(str(hucCode))
 
     # Get wbd buffer
@@ -176,4 +176,4 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     lake_buffer_distance  = args['lake_buffer_distance']
     dissolveLinks = args['dissolve_links']
 
-    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance,dissolveLinks)
+    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance,dissolveLinks)
\ No newline at end of file
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index 879103ad6..19d74abc8 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -5,6 +5,7 @@
 import shutil
 import argparse
 
+@profile
 def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, viz_post_processing):
     '''
     Processes all the final output files to cleanup and add post-processing
@@ -61,6 +62,8 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         # Step 1, keep only files that Viz needs
         whitelist_directory(output_folder_path, viz_whitelist, additional_whitelist)
 
+
+@profile
 def whitelist_directory(directory_path, whitelist, additional_whitelist):
     # Add any additional files to the whitelist that the user wanted to keep
     if additional_whitelist:
diff --git a/src/rem.py b/src/rem.py
index f0cd8fad3..43c75b0f0 100755
--- a/src/rem.py
+++ b/src/rem.py
@@ -10,7 +10,7 @@
 import geopandas as gpd
 from utils.shared_functions import getDriver
 
-
+@profile
 def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, dem_reaches_filename):
     """
         Calculates REM/HAND/Detrended DEM
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 3da0983ee..6e9cfca7b 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -57,7 +57,7 @@ echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-$srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
+python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
 Tcount
 
 if [ "$extent" = "MS" ]; then
@@ -128,7 +128,7 @@ echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwr
 date -u
 Tstart
 [ -f $outputHucDataDir/nld_rasterized_elev.tif ] && \
-$srcDir/burn_in_levees.py -dem $outputHucDataDir/dem_meters.tif -nld $outputHucDataDir/nld_rasterized_elev.tif -out $outputHucDataDir/dem_meters.tif
+python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputHucDataDir/dem_meters.tif -nld $outputHucDataDir/nld_rasterized_elev.tif -out $outputHucDataDir/dem_meters.tif
 Tcount
 
 ## DEM Reconditioning ##
@@ -139,7 +139,7 @@ echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer"$stopD
 date -u
 Tstart
 [ ! -f $outputHucDataDir/dem_burned.tif ] && \
-$srcDir/agreedem.py -r $outputHucDataDir/flows_grid_boolean.tif -d $outputHucDataDir/dem_meters.tif -w $outputHucDataDir -g $outputHucDataDir/temp_work -o $outputHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+python3 -m memory_profiler $srcDir/agreedem.py -r $outputHucDataDir/flows_grid_boolean.tif -d $outputHucDataDir/dem_meters.tif -w $outputHucDataDir -g $outputHucDataDir/temp_work -o $outputHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
 Tcount
 
 ## PIT REMOVE BURNED DEM ##
@@ -184,14 +184,14 @@ Tcount
 echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/unique_pixel_and_allocation.py -s $outputHucDataDir/demDerived_streamPixels.tif -o $outputHucDataDir/demDerived_streamPixels_ids.tif -g $outputHucDataDir/temp_grass
+python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputHucDataDir/demDerived_streamPixels.tif -o $outputHucDataDir/demDerived_streamPixels_ids.tif -g $outputHucDataDir/temp_grass
 Tcount
 
 ## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
 echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/adjust_thalweg_lateral.py -e $outputHucDataDir/dem_meters.tif -s $outputHucDataDir/demDerived_streamPixels.tif -a $outputHucDataDir/demDerived_streamPixels_ids_allo.tif -d $outputHucDataDir/demDerived_streamPixels_ids_dist.tif -t 50 -o $outputHucDataDir/dem_lateral_thalweg_adj.tif
+python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputHucDataDir/dem_meters.tif -s $outputHucDataDir/demDerived_streamPixels.tif -a $outputHucDataDir/demDerived_streamPixels_ids_allo.tif -d $outputHucDataDir/demDerived_streamPixels_ids_dist.tif -t 50 -o $outputHucDataDir/dem_lateral_thalweg_adj.tif
 Tcount
 
 ## MASK BURNED DEM FOR STREAMS ONLY ###
@@ -230,7 +230,7 @@ echo -e $startDiv"Split Derived Reaches $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ] && \
-$srcDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+python3 -m memory_profiler $srcDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 Tcount
 
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
@@ -244,7 +244,7 @@ if [ "$extent" = "MS" ]; then
   echo -e $startDiv"Mask Rasters with Stream Buffer $hucNumber"$stopDiv
   date -u
   Tstart
-  $srcDir/fr_to_ms_raster_mask.py $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/flowdir_d8_burned_filled.tif $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/slopes_d8_dem_meters.tif $outputHucDataDir/flowdir_d8_MS.tif $outputHucDataDir/dem_thalwegCond_MS.tif $outputHucDataDir/slopes_d8_dem_metersMS.tif $outputHucDataDir/demDerived_streamPixels.tif $outputHucDataDir/demDerived_streamPixelsMS.tif
+  python3 -m memory_profiler $srcDir/fr_to_ms_raster_mask.py $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/flowdir_d8_burned_filled.tif $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/slopes_d8_dem_meters.tif $outputHucDataDir/flowdir_d8_MS.tif $outputHucDataDir/dem_thalwegCond_MS.tif $outputHucDataDir/slopes_d8_dem_metersMS.tif $outputHucDataDir/demDerived_streamPixels.tif $outputHucDataDir/demDerived_streamPixelsMS.tif
   Tcount
 
   if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
@@ -293,7 +293,7 @@ echo -e $startDiv"D8 REM $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/rem.tif ] && \
-$srcDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -s $outputHucDataDir/demDerived_reaches_split.gpkg
+python3 -m memory_profiler $srcDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -s $outputHucDataDir/demDerived_reaches_split.gpkg
 Tcount
 
 ## DINF DISTANCE DOWN ##
@@ -325,7 +325,7 @@ echo -e $startDiv"Process catchments and model streams step 1 $hucNumber"$stopDi
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ] && \
-$srcDir/filter_catchments_and_add_attributes.py $outputHucDataDir/gw_catchments_reaches.gpkg $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/wbd8_clp.gpkg $hucNumber
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py $outputHucDataDir/gw_catchments_reaches.gpkg $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/wbd8_clp.gpkg $hucNumber
 
 if [[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ]] ; then
   echo "No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
@@ -385,7 +385,7 @@ Tcount
 echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/make_stages_and_catchlist.py $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/stage.txt $outputHucDataDir/catchment_list.txt $stage_min_meters $stage_interval_meters $stage_max_meters
+python3 -m memory_profiler $srcDir/make_stages_and_catchlist.py $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/stage.txt $outputHucDataDir/catchment_list.txt $stage_min_meters $stage_interval_meters $stage_max_meters
 Tcount
 
 ## HYDRAULIC PROPERTIES ##
@@ -401,14 +401,14 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv outp
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-$srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
+python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
 ## USGS CROSSWALK ##
 echo -e $startDiv"USGS Crosswalk $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/usgs_gage_crosswalk.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -dem $outputHucDataDir/dem_meters.tif -flows $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -cat $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -wbd $outputHucDataDir/wbd_buffered.gpkg -dem_adj $dem_thalwegCond -outtable $outputHucDataDir/usgs_elev_table.csv -e $extent
+python3 -m memory_profiler $srcDir/usgs_gage_crosswalk.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -dem $outputHucDataDir/dem_meters.tif -flows $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -cat $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -wbd $outputHucDataDir/wbd_buffered.gpkg -dem_adj $dem_thalwegCond -outtable $outputHucDataDir/usgs_elev_table.csv -e $extent
 Tcount
 
 ## CLEANUP OUTPUTS ##
@@ -419,5 +419,5 @@ args=()
 (( viz == 1 )) && args+=( '-v' )
 date -u
 Tstart
-$srcDir/output_cleanup.py $hucNumber $outputHucDataDir "${args[@]}"
+python3 -m memory_profiler $srcDir/output_cleanup.py $hucNumber $outputHucDataDir "${args[@]}"
 Tcount
diff --git a/src/time_and_tee_run_by_unit.sh b/src/time_and_tee_run_by_unit.sh
index 25dffd967..c3c83f426 100755
--- a/src/time_and_tee_run_by_unit.sh
+++ b/src/time_and_tee_run_by_unit.sh
@@ -1,5 +1,6 @@
 #!/bin/bash -e
 
-/usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log
+mprof run --include-children /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log
+mprof plot -o $outputRunDataDir/logs/$1_memory
 exit ${PIPESTATUS[0]}
 
diff --git a/src/unique_pixel_and_allocation.py b/src/unique_pixel_and_allocation.py
index 8f5bce600..da7efc92e 100755
--- a/src/unique_pixel_and_allocation.py
+++ b/src/unique_pixel_and_allocation.py
@@ -9,6 +9,7 @@
 import argparse
 from r_grow_distance import r_grow_distance
 
+@profile
 def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
     '''
     This function will assign a unique ID for each stream pixel and writes to file. It then uses this raster to run GRASS r.grow.distance tool to create the allocation and proximity rasters required to complete the lateral thalweg conditioning. 

From ca8cb7e8000befe1b05495277f0b2d362565a860 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 10 Jun 2021 10:54:32 -0500
Subject: [PATCH 083/359] Feature to evaluate alternative CatFIM technique
 performance

Added eval_catfim_alt.py to evaluate performance of alternative CatFIM techniques.

This resolves #414.
---
 CHANGELOG.md                   |   8 ++
 tools/eval_alt_catfim.py       | 221 +++++++++++++++++++++++++++++++++
 tools/synthesize_test_cases.py |   2 -
 3 files changed, 229 insertions(+), 2 deletions(-)
 create mode 100644 tools/eval_alt_catfim.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 81f2e3962..a12d1911c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.19.0 - 2021-06-10 - [PR #415](https://github.com/NOAA-OWP/cahaba/pull/415)
+
+Feature to evaluate performance of alternative CatFIM techniques.
+
+## Additions
+- Added `eval_catfim_alt.py` to evaluate performance of alternative CatFIM techniques.
+
+<br/><br/>
 ## v3.0.18.0 - 2021-06-09 - [PR #404](https://github.com/NOAA-OWP/cahaba/pull/404)
 
 To help analyze the memory consumption of the Fim Run process, the python module `memory-profiler` has been added to give insights into where peak memory usage is with in the codebase.
diff --git a/tools/eval_alt_catfim.py b/tools/eval_alt_catfim.py
new file mode 100644
index 000000000..ffe86cf0e
--- /dev/null
+++ b/tools/eval_alt_catfim.py
@@ -0,0 +1,221 @@
+
+import os
+import argparse
+from multiprocessing import Pool
+import csv
+import json
+
+from tools_shared_variables import TEST_CASES_DIR
+from tools_shared_functions import compute_contingency_stats_from_rasters
+
+
+def create_master_metrics_csv_alt(master_metrics_csv_output, json_list, version):
+
+    # Construct header
+    metrics_to_write = ['true_negatives_count',
+                        'false_negatives_count',
+                        'true_positives_count',
+                        'false_positives_count',
+                        'contingency_tot_count',
+                        'cell_area_m2',
+                        'TP_area_km2',
+                        'FP_area_km2',
+                        'TN_area_km2',
+                        'FN_area_km2',
+                        'contingency_tot_area_km2',
+                        'predPositive_area_km2',
+                        'predNegative_area_km2',
+                        'obsPositive_area_km2',
+                        'obsNegative_area_km2',
+                        'positiveDiff_area_km2',
+                        'CSI',
+                        'FAR',
+                        'TPR',
+                        'TNR',
+                        'PPV',
+                        'NPV',
+                        'ACC',
+                        'Bal_ACC',
+                        'MCC',
+                        'EQUITABLE_THREAT_SCORE',
+                        'PREVALENCE',
+                        'BIAS',
+                        'F1_SCORE',
+                        'TP_perc',
+                        'FP_perc',
+                        'TN_perc',
+                        'FN_perc',
+                        'predPositive_perc',
+                        'predNegative_perc',
+                        'obsPositive_perc',
+                        'obsNegative_perc',
+                        'positiveDiff_perc',
+                        'masked_count',
+                        'masked_perc',
+                        'masked_area_km2'
+                        ]
+
+    additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
+    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
+
+
+
+    for full_json_path in json_list:
+         
+         # Parse variables from json path.
+        split_json_handle = os.path.split(full_json_path)[1].split('_')
+        
+        benchmark_source = split_json_handle[2]
+        huc = split_json_handle[1]
+        nws_lid = split_json_handle[0]
+        magnitude = split_json_handle[3].replace('.json', '')
+                  
+        real_json_path = os.path.join(os.path.split(full_json_path)[0], nws_lid + '_b0m_stats.json')
+        
+        sub_list_to_append = [version, nws_lid, magnitude, huc]
+        
+        stats_dict = json.load(open(real_json_path))
+        for metric in metrics_to_write:
+            sub_list_to_append.append(stats_dict[metric])
+        sub_list_to_append.append(real_json_path)
+        sub_list_to_append.append('NA')
+        sub_list_to_append.append(benchmark_source)
+        sub_list_to_append.append('MS')
+        sub_list_to_append.append('yes')
+        
+        list_to_write.append(sub_list_to_append)
+
+
+    with open(master_metrics_csv_output, 'w', newline='') as csvfile:
+        csv_writer = csv.writer(csvfile)
+        csv_writer.writerows(list_to_write)
+
+
+def process_alt_comparison(args):
+    
+    predicted_raster_path = args[0]
+    benchmark_raster_path = args[1]
+    agreement_raster = args[2]
+    stats_csv = args[3]
+    stats_json = args[4]
+    mask_values = args[5]
+    stats_modes_list = args[6]
+    test_id = args[7]
+    mask_dict = args[8]
+
+    compute_contingency_stats_from_rasters(predicted_raster_path,
+                                           benchmark_raster_path,
+                                           agreement_raster,
+                                           stats_csv=stats_csv,
+                                           stats_json=stats_json,
+                                           mask_values=[],
+                                           stats_modes_list=stats_modes_list,
+                                           test_id=test_id,
+                                           mask_dict=mask_dict,
+                                           )
+
+    print("Finished processing " + agreement_raster)
+
+
+if __name__ == '__main__':
+    
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Produces metrics for alternative CatFIM.')
+    parser.add_argument('-d','--catfim-directory',help='Path to directory storing CatFIM outputs. This is the most parent dir, usually named by a version.',required=True)
+    parser.add_argument('-w','--output-workspace',help='Add a special name to the end of the branch.',required=True, default="")
+    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=False,default=None)
+    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
+    
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+    catfim_dir = args['catfim_directory']
+    output_workspace = args['output_workspace']
+    job_number = int(args['job_number'])
+    master_metrics_csv = args['master_metrics_csv']
+    
+    if master_metrics_csv == None:
+        master_metrics_csv = os.path.join(output_workspace, 'master_metrics.csv')
+    
+    if not os.path.exists(catfim_dir):
+        print("CatFIM directory: " + catfim_dir + " does not exist.")
+        quit
+    
+    if not os.path.exists(output_workspace):
+        os.mkdir(output_workspace)
+    
+    catfim_dir_list = os.listdir(catfim_dir)
+    
+    procs_list = []
+    json_list = []
+    
+    for huc in catfim_dir_list:
+        if len(huc) == 8:
+            
+            huc_workspace = os.path.join(output_workspace, huc)
+            if not os.path.exists(huc_workspace):
+                os.mkdir(huc_workspace)
+
+            huc_dir_path = os.path.join(catfim_dir, huc)
+            
+            # List AHPS sites.
+            site_list = os.listdir(huc_dir_path)
+            
+            # Loop through AHPS sites.
+            for site in site_list:
+                site_dir = os.path.join(huc_dir_path, site)
+                
+                site_workspace = os.path.join(huc_workspace, site)
+                if not os.path.exists(site_workspace):
+                    os.mkdir(site_workspace)
+                
+                for category in ['action', 'minor', 'moderate', 'major']:
+                    # Presumptiously define inundation grid path.
+                    category_grid_path = os.path.join(site_dir, site + '_' + category + '_extent_' + huc + '.tif')
+                
+                    if os.path.exists(category_grid_path):
+
+                        site_category_workspace = os.path.join(site_workspace, category)
+                        if not os.path.exists(site_category_workspace):
+                            os.mkdir(site_category_workspace)
+                            
+                        # Map path to benchmark data, both NWS and USGS.
+                        for benchmark_type in ['nws', 'usgs']:
+                            benchmark_grid = os.path.join(TEST_CASES_DIR, benchmark_type + '_test_cases', 'validation_data_' + benchmark_type, huc, site, category, 'ahps_' + site + '_huc_' + huc + '_extent_' + category + '.tif')
+ 
+                            if os.path.exists(benchmark_grid):
+                                
+                                # Create dir in output workspace for results.                                
+                                file_handle = site + '_' + huc + '_' + benchmark_type + '_' + category 
+                                
+                                predicted_raster_path = category_grid_path
+                                benchmark_raster_path = benchmark_grid
+                                agreement_raster = os.path.join(site_category_workspace, file_handle + '.tif')
+                                stats_csv = os.path.join(site_category_workspace, file_handle + '.csv')
+                                stats_json = os.path.join(site_category_workspace, file_handle + '.json')
+                                mask_values=None
+                                stats_modes_list=['total_area']
+                                test_id=''
+                                mask_dict={'levees': {'path': '/data/test_cases/other/zones/leveed_areas_conus.shp', 'buffer': None, 'operation': 'exclude'}, 
+                                            'waterbodies': {'path': '/data/test_cases/other/zones/nwm_v2_reservoirs.shp', 'buffer': None, 'operation': 'exclude'},
+                                            site: {'path': '/data/test_cases/{benchmark_type}_test_cases/validation_data_{benchmark_type}/{huc}/{site}/{site}_domain.shp'.format(benchmark_type=benchmark_type, site=site, category=category, huc=huc), 'buffer': None, 'operation': 'include'}}
+                                
+                                json_list.append(stats_json)
+                                
+                                # Either add to list to multiprocess or process serially, depending on user specification.
+                                if job_number > 1:
+                                    procs_list.append([predicted_raster_path, benchmark_raster_path, agreement_raster,stats_csv,stats_json,mask_values,stats_modes_list,test_id, mask_dict])
+                                else:
+                                    process_alt_comparison([predicted_raster_path, benchmark_raster_path, agreement_raster,stats_csv,stats_json, mask_values,stats_modes_list,test_id, mask_dict])
+
+    # Multiprocess.
+    if job_number > 1:
+        with Pool(processes=job_number) as pool:
+            pool.map(process_alt_comparison, procs_list)
+            
+    # Merge stats into single file.
+    version = os.path.split(output_workspace)[1]
+    create_master_metrics_csv_alt(master_metrics_csv, json_list, version)
+
+            
+    
+    
\ No newline at end of file
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index de035359c..9fef7841d 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -175,8 +175,6 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                         next(reader)
                                                         for row in reader:
                                                             flow = row[1]
-                                                        if nws_lid == 'mcc01':
-                                                            print(flow)
 
                                                 stats_dict = json.load(open(full_json_path))
                                                 for metric in metrics_to_write:

From ac4ae06febed568d15e349fd3206b5e534f89fd0 Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Mon, 21 Jun 2021 07:46:35 -0500
Subject: [PATCH 084/359] Thalweg Profile Tool and lateral Thalweg Adjustment
 Threshold

Adding a thalweg profile tool to identify significant drops in thalweg elevation. Also setting lateral thalweg adjustment threshold in hydroconditioning.

- thalweg_drop_check.py checks the elevation along the thalweg for each stream path downstream of MS headwaters within a HUC.
- Removing dissolveLinks arg from clip_vectors_to_wbd.py.
- Cleaned up code in split_flows.py to make it more readable.
- Refactored reduce_nhd_stream_density.py and adjust_headwater_streams.py to limit MS headwater points in agg_nhd_headwaters_adj.gpkg.
- Fixed a bug in adjust_thalweg_lateral.py lateral elevation replacement threshold; changed threshold to 3 meters.
- Updated aggregate_vector_inputs.py to log intermediate processes.

This resolves #408, resolves #18, and resolves #409.
---
 CHANGELOG.md                         |  19 ++
 config/params_calibrated.env         |   1 +
 config/params_template.env           |   1 +
 src/adjust_headwater_streams.py      |   8 +-
 src/adjust_thalweg_lateral.py        |  88 +++--
 src/aggregate_vector_inputs.py       |  42 ++-
 src/clip_vectors_to_wbd.py           |   9 +-
 src/output_cleanup.py                |   2 +-
 src/raster.py                        | 462 ---------------------------
 src/reachID_grid_to_vector_points.py |  98 +++---
 src/reduce_nhd_stream_density.py     |  35 +-
 src/run_by_unit.sh                   |   9 +-
 src/split_flows.py                   |  18 +-
 tools/thalweg_drop_check.py          | 385 ++++++++++++++++++++++
 14 files changed, 567 insertions(+), 610 deletions(-)
 delete mode 100644 src/raster.py
 create mode 100644 tools/thalweg_drop_check.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a12d1911c..808effc01 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.19.1 - 2021-06-17 - [PR #417](https://github.com/NOAA-OWP/cahaba/pull/417)
+
+Adding a thalweg profile tool to identify significant drops in thalweg elevation. Also setting lateral thalweg adjustment threshold in hydroconditioning.
+
+## Additions
+- `thalweg_drop_check.py` checks the elevation along the thalweg for each stream path downstream of MS headwaters within a HUC.
+
+## Removals
+- Removing `dissolveLinks` arg from `clip_vectors_to_wbd.py`.
+
+
+## Changes
+- Cleaned up code in `split_flows.py` to make it more readable.
+- Refactored `reduce_nhd_stream_density.py` and `adjust_headwater_streams.py` to limit MS headwater points in `agg_nhd_headwaters_adj.gpkg`.
+- Fixed a bug in `adjust_thalweg_lateral.py` lateral elevation replacement threshold; changed threshold to 3 meters.
+- Updated `aggregate_vector_inputs.py` to log intermediate processes.
+
+<br/><br/>
+
 ## v3.0.19.0 - 2021-06-10 - [PR #415](https://github.com/NOAA-OWP/cahaba/pull/415)
 
 Feature to evaluate performance of alternative CatFIM techniques.
diff --git a/config/params_calibrated.env b/config/params_calibrated.env
index aa7aba1b0..3ca30650e 100644
--- a/config/params_calibrated.env
+++ b/config/params_calibrated.env
@@ -4,6 +4,7 @@
 export negative_burn_value=1000
 export agree_DEM_buffer=70
 export wbd_buffer=5000
+export thalweg_lateral_elev_threshold=3
 
 #### geospatial parameters ####
 export max_split_distance_meters=1500
diff --git a/config/params_template.env b/config/params_template.env
index a998dc675..87270a669 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -4,6 +4,7 @@
 export negative_burn_value=1000
 export agree_DEM_buffer=70
 export wbd_buffer=5000
+export thalweg_lateral_elev_threshold=3
 
 #### geospatial parameters ####
 export max_split_distance_meters=1500
diff --git a/src/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
index 71f73186e..961d39d01 100644
--- a/src/adjust_headwater_streams.py
+++ b/src/adjust_headwater_streams.py
@@ -29,10 +29,10 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
     nws_lid_limited = nws_lid_limited.loc[nws_lid_limited.nws_lid!='']
     nws_lid_limited = nws_lid_limited.drop(columns=['nws_lid'])
 
-    # Check for issues in nws_lid layer
-    if len(nws_lid_limited) < len(nws_lids):
-        missing_nws_lids = list(set(nws_lids.site_id) - set(nws_lid_limited.site_id))
-        print (f"nws lid(s) {missing_nws_lids} missing from aggregate dataset in huc {huc}")
+    # Check for issues in nws_lid layer (now this reports back non-headwater nws_lids)
+    # if len(nws_lid_limited) < len(nws_lids):
+    #     missing_nws_lids = list(set(nws_lids.site_id) - set(nws_lid_limited.site_id))
+    #     print (f"nws lid(s) {missing_nws_lids} missing from aggregate dataset in huc {huc}")
 
     # Combine NWM headwaters and AHPS sites to be snapped to NHDPlus HR segments
     headwater_pts = headwater_limited.append(nws_lid_limited)
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index 90c8cecbf..47cd2e209 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -7,42 +7,43 @@
 import numpy as np
 
 @profile
-def adjust_thalweg_laterally(elevation_raster, stream_raster, allocation_raster, cost_distance_raster, cost_distance_tolerance, dem_lateral_thalweg_adj):
-    
+def adjust_thalweg_laterally(elevation_raster, stream_raster, allocation_raster, cost_distance_raster, cost_distance_tolerance, dem_lateral_thalweg_adj,lateral_elevation_threshold):
+
     # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
     # The following algorithm searches for the zonal minimum elevation in each pixel catchment
     # It updates the catchment_min_dict with this zonal minimum elevation value.
     @njit
     def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, cost_tolerance, ndv):
-        for i,cm in enumerate(zone_window):
+        for i,elev_m in enumerate(zone_window):
             # If the zone really exists in the dictionary, compare elevation values.
             i = int(i)
-            cm = int(cm)
-            
+            elev_m = int(elev_m)
+
             if (cost_window[i] <= cost_tolerance):
                 if elevation_window[i] > 0:  # Don't allow bad elevation values
-                    if (cm in zone_min_dict):                     
-                                
-                        if (elevation_window[i] < zone_min_dict[cm]):
+                    if (elev_m in zone_min_dict):
+
+                        if (elevation_window[i] < zone_min_dict[elev_m]):
                             # If the elevation_window's elevation value is less than the zone_min_dict min, update the zone_min_dict min.
-                            zone_min_dict[cm] = elevation_window[i]                                                
+                            zone_min_dict[elev_m] = elevation_window[i]
                     else:
-                        zone_min_dict[cm] = elevation_window[i]
+                        zone_min_dict[elev_m] = elevation_window[i]
+
         return(zone_min_dict)
-    
+
     # Open the masked gw_catchments_pixels_masked and dem_thalwegCond_masked.
     elevation_raster_object = rasterio.open(elevation_raster)
     allocation_zone_raster_object = rasterio.open(allocation_raster)
     cost_distance_raster_object = rasterio.open(cost_distance_raster)
-    
+
     meta = elevation_raster_object.meta.copy()
     meta['tiled'], meta['compress'] = True, 'lzw'
-    
+
     # -- Create zone_min_dict -- #
     print("Create zone_min_dict")
     zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums.
     # Update catchment_min_dict with pixel sheds minimum.
-    
+
     for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template.
         elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window.
         zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window.
@@ -50,72 +51,69 @@ def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window
 
         # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
         zone_min_dict = make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, int(cost_distance_tolerance), meta['nodata'])
-         
-    # ------------------------------------------------------------------------------------------------------------------------ # 
-    
+
+    # ------------------------------------------------------------------------------------------------------------------------ #
+
     elevation_raster_object.close()
     allocation_zone_raster_object.close()
     cost_distance_raster_object.close()
 
-    
+
     # ------------------------------------------- Assign zonal min to thalweg ------------------------------------------------ #
     @njit
     def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window):
-                
+
         # Copy elevation values into new array that will store the minimized elevation values.
         dem_window_to_return = np.empty_like (dem_window)
         dem_window_to_return[:] = dem_window
-        
 
-        for i,cm in enumerate(zone_window):
+
+        for i,elev_m in enumerate(zone_window):
             i = int(i)
-            cm = int(cm)
+            elev_m = int(elev_m)
             thalweg_cell = thalweg_window[i]  # From flows_grid_boolean.tif (0s and 1s)
             if thalweg_cell == 1:  # Make sure thalweg cells are checked.
-                if cm in zone_min_dict:
-                    zone_min_elevation = zone_min_dict[cm]
+                if elev_m in zone_min_dict:
+                    zone_min_elevation = zone_min_dict[elev_m]
                     dem_thalweg_elevation = dem_window[i]
-                    
-                    elevation_difference = zone_min_elevation - dem_thalweg_elevation
-    
-                    if zone_min_elevation < dem_thalweg_elevation and elevation_difference <= 5:
+
+                    elevation_difference = dem_thalweg_elevation - zone_min_elevation
+
+                    if (zone_min_elevation < dem_thalweg_elevation) and (elevation_difference <= lateral_elevation_threshold):
                         dem_window_to_return[i] = zone_min_elevation
 
         return(dem_window_to_return)
-        
+
     # Specify raster object metadata.
     elevation_raster_object = rasterio.open(elevation_raster)
     allocation_zone_raster_object = rasterio.open(allocation_raster)
     thalweg_object = rasterio.open(stream_raster)
 
-    
+
     dem_lateral_thalweg_adj_object = rasterio.open(dem_lateral_thalweg_adj, 'w', **meta)
-    
+
     for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
         dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window.
         window_shape = dem_window.shape
         dem_window = dem_window.ravel()
-        
+
         zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window.
         thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window.
-        
+
         # Call numba-optimized function to reassign thalweg cell values to catchment minimum value.
         minimized_dem_window = minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window)
         minimized_dem_window = minimized_dem_window.reshape(window_shape).astype(np.float32)
 
 
-        dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)    
-    
+        dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)
+
     elevation_raster_object.close()
     allocation_zone_raster_object.close()
     cost_distance_raster_object.close()
-    
-    # Delete allocation_raster and distance_raster.
-    
-    
-    
+
+
 if __name__ == '__main__':
-    
+
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Adjusts the elevation of the thalweg to the lateral zonal minimum.')
     parser.add_argument('-e','--elevation_raster',help='Raster of elevation.',required=True)
@@ -124,11 +122,9 @@ def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_w
     parser.add_argument('-d','--cost_distance_raster',help='Raster of cost distances for the allocation raster.',required=True)
     parser.add_argument('-t','--cost_distance_tolerance',help='Tolerance in meters to use when searching for zonal minimum.',required=True)
     parser.add_argument('-o','--dem_lateral_thalweg_adj',help='Output elevation raster with adjusted thalweg.',required=True)
-    
+    parser.add_argument('-th','--lateral_elevation_threshold',help='Maximum difference between current thalweg elevation and lowest lateral elevation in meters.',required=True,type=int)
+
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
-    
+
     adjust_thalweg_laterally(**args)
-    
-    
-    
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
index 2c1081989..3675b88b8 100755
--- a/src/aggregate_vector_inputs.py
+++ b/src/aggregate_vector_inputs.py
@@ -89,6 +89,7 @@ def find_nwm_incoming_streams(nwm_streams_,wbd,huc_unit):
 
     # Input wbd
     if isinstance(wbd,str):
+
         layer = f"WBDHU{huc_unit}"
         wbd = gpd.read_file(wbd, layer=layer)
     elif isinstance(wbd,gpd.GeoDataFrame):
@@ -175,6 +176,7 @@ def find_nwm_incoming_streams(nwm_streams_,wbd,huc_unit):
 
     huc_intersection = gpd.GeoDataFrame({'geometry': intersecting_points, 'NHDPlusID': nhdplus_ids,'mainstem': mainstem_flag},crs=nwm_streams.crs,geometry='geometry')
     huc_intersection = huc_intersection.drop_duplicates()
+
     del nwm_streams,wbd
 
     return huc_intersection
@@ -182,7 +184,8 @@ def find_nwm_incoming_streams(nwm_streams_,wbd,huc_unit):
 
 def collect_stream_attributes(nhdplus_vectors_dir, huc):
 
-    print ('Starting huc: ' + str(huc))
+    print (f"Starting attribute collection for HUC {huc}",flush=True)
+
     # Collecting NHDPlus HR attributes
     burnline_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')
     vaa_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusFlowLineVAA' + str(huc) + '.gpkg')
@@ -214,10 +217,10 @@ def collect_stream_attributes(nhdplus_vectors_dir, huc):
         nhd_streams.to_file(nhd_streams_agg_fileName,driver=getDriver(nhd_streams_agg_fileName),index=False)
         del nhd_streams
 
-        print ('finished huc: ' + str(huc))
+        print (f"finished attribute collection for HUC {huc}",flush=True)
 
     else:
-        print ('missing data for huc ' + str(huc))
+        print (f"missing data for HUC {huc}",flush=True)
 
 
 def subset_stream_networks(args, huc):
@@ -229,10 +232,11 @@ def subset_stream_networks(args, huc):
     nhdplus_vectors_dir                = args[4]
     nwm_huc4_intersections_filename    = args[5]
 
-    print("starting HUC " + str(huc),flush=True)
+    print(f"starting stream subset for HUC {huc}",flush=True)
     nwm_headwater_id = 'ID'
     ahps_headwater_id = 'nws_lid'
     headwater_pts_id = 'site_id'
+
     column_order = ['pt_type', headwater_pts_id, 'mainstem', 'geometry']
     nhd_streams_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
 
@@ -247,13 +251,20 @@ def subset_stream_networks(args, huc):
     huc_mask = huc_mask.reset_index(drop=True)
 
     if len(selected_wbd8.HUC8) > 0:
+
         selected_wbd8 = selected_wbd8.reset_index(drop=True)
 
         # Identify FR/NWM headwaters and subset HR network
-        nhd_streams_fr = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,nwm_headwaters_filename,nwm_headwater_id,nwm_huc4_intersections_filename)
+        try:
+            nhd_streams_fr = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,nwm_headwaters_filename,nwm_headwater_id,nwm_huc4_intersections_filename)
+        except:
+            print (f"Error subsetting NHD HR network for HUC {huc}",flush=True)
 
         # Identify nhd mainstem streams
-        nhd_streams_all = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_fr,ahps_filename,ahps_headwater_id,nwm_huc4_intersections_filename,True)
+        try:
+            nhd_streams_all = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_fr,ahps_filename,ahps_headwater_id,nwm_huc4_intersections_filename,True)
+        except:
+            print (f"Error identifing MS network for HUC {huc}",flush=True)
 
         # Identify HUC8 intersection points
         nhd_huc8_intersections = find_nwm_incoming_streams(nhd_streams_all,selected_wbd8,8)
@@ -265,17 +276,17 @@ def subset_stream_networks(args, huc):
 
         # Load nws lids
         nws_lids = gpd.read_file(ahps_filename, mask=huc_mask)
-        nws_lids = nws_lids.drop(columns=['name','nwm_featur'])
+        nws_lids = nws_lids.drop(columns=['name','nwm_feature_id','usgs_site_code','states','HUC8','is_headwater', 'is_colocated'])
         nws_lids = nws_lids.rename(columns={"nws_lid": headwater_pts_id})
         nws_lids['pt_type'] = 'nws_lid'
         nws_lids['mainstem'] = True
 
         if (len(nwm_headwaters) > 0) or (len(nws_lids) > 0):
+
             # Adjust FR/NWM headwater segments
             adj_nhd_streams_all, adj_nhd_headwater_points = adjust_headwaters(huc,nhd_streams_all,nwm_headwaters,nws_lids,headwater_pts_id)
 
             adj_nhd_headwater_points = adj_nhd_headwater_points[column_order]
-
             nhd_huc8_intersections['pt_type'] = 'nhd_huc8_intersections'
             nhd_huc8_intersections = nhd_huc8_intersections.rename(columns={"NHDPlusID": headwater_pts_id})
             nhd_huc8_intersections = nhd_huc8_intersections[column_order]
@@ -290,11 +301,14 @@ def subset_stream_networks(args, huc):
             adj_nhd_headwater_points_all.to_file(adj_nhd_headwaters_all_fileName,driver=getDriver(adj_nhd_headwaters_all_fileName),index=False)
 
             del adj_nhd_streams_all, adj_nhd_headwater_points_all
+
         else:
-            print (f"skipping headwater adjustments for HUC: {huc}")
+
+            print (f"skipping headwater adjustments for HUC {huc}")
 
         del nhd_streams_fr
 
+    print(f"finished stream subset for HUC {huc}",flush=True)
 
 def aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,huc_list):
 
@@ -330,6 +344,7 @@ def aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileNam
 def clean_up_intermediate_files(nhdplus_vectors_dir):
 
     for huc in os.listdir(nhdplus_vectors_dir):
+
         agg_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
         streams_adj_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
         headwater_adj_path= os.path.join(nhdplus_vectors_dir,huc,'nhd' + str(huc) + '_headwaters_adj.gpkg')
@@ -385,18 +400,19 @@ def clean_up_intermediate_files(nhdplus_vectors_dir):
         if not os.path.isfile(streams_adj_path):
             missing_subsets = missing_subsets + [huc]
 
-    print (f"running subset_results on {len(missing_subsets)} HUC4s")
+    print (f"Subsetting stream network for {len(missing_subsets)} HUC4s")
     num_workers=11
 
     with ProcessPoolExecutor(max_workers=num_workers) as executor:
         # Preprocess nhd hr and add attributes
-        # collect_attributes = [executor.submit(collect_stream_attributes, nhdplus_vectors_dir, str(huc)) for huc in huc_list]
+        collect_attributes = [executor.submit(collect_stream_attributes, nhdplus_vectors_dir, str(huc)) for huc in huc_list]
         # Subset nhd hr network
         subset_results = [executor.submit(subset_stream_networks, subset_arg_list, str(huc)) for huc in missing_subsets]
 
-    # del wbd4,wbd8
+    del wbd4,wbd8
 
-    # Aggregate fr and ms nhd netowrks for entire nwm domain
+    # Aggregate subset nhd networks for entire nwm domain
+    print ('Aggregating subset NHD networks for entire NWM domain')
     aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,huc_list)
 
     # Remove intermediate files
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index ef7ff4837..65fd72d20 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -8,7 +8,8 @@
 from utils.shared_functions import getDriver
 
 @profile
-def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance,dissolveLinks=False):
+def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance):
+
     hucUnitLength = len(str(hucCode))
 
     # Get wbd buffer
@@ -88,6 +89,7 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
         nhd_streams = nhd_streams.loc[nhd_streams.mainstem==1]
 
     if len(nhd_streams) > 0:
+
         # Find incoming stream segments (to WBD buffer) and identify which are upstream
         threshold_segments = gpd.overlay(nhd_streams, wbd_buffer, how='symmetric_difference')
         from_list = threshold_segments.FromNode.to_list()
@@ -148,8 +150,6 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     parser.add_argument('-gl','--great-lakes-filename',help='Great Lakes layer',required=True)
     parser.add_argument('-wb','--wbd-buffer-distance',help='WBD Mask buffer distance',required=True,type=int)
     parser.add_argument('-lb','--lake-buffer-distance',help='Great Lakes Mask buffer distance',required=True,type=int)
-    parser.add_argument('-o','--dissolve-links',help='remove multi-line strings',action="store_true",default=False)
-
 
     args = vars(parser.parse_args())
 
@@ -174,6 +174,5 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     great_lakes_filename = args['great_lakes_filename']
     wbd_buffer_distance = args['wbd_buffer_distance']
     lake_buffer_distance  = args['lake_buffer_distance']
-    dissolveLinks = args['dissolve_links']
 
-    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance,dissolveLinks)
\ No newline at end of file
+    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance)
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index 19d74abc8..d73a2deee 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -39,7 +39,7 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         'bathy_xs_area_hydroid_lookup.csv',
         'src_full_crosswalked.csv',
         'usgs_elev_table.csv',
-        'hand_ref_elev_table.csv'
+        'hand_ref_elev_table.csv',
     ]
 
     # List of files that will be saved during a viz run
diff --git a/src/raster.py b/src/raster.py
deleted file mode 100644
index a10a02430..000000000
--- a/src/raster.py
+++ /dev/null
@@ -1,462 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from osgeo import gdal, ogr, osr
-import numpy as np
-from os.path import isfile
-from os import remove
-from copy import deepcopy
-from subprocess import call
-
-class Raster:
-
-	"""
-	Raster object from single band rasters
-
-	...
-
-	Attributes
-	----------
-	array : numpy array
-		raster data in numpy array
-	gt : list
-		geotransform. see gdal docs for more info.
-	proj : str
-		Projection string
-	ndv : number
-		No data value
-	des : str
-		band description
-	ct : gdal.colorTable
-		color table
-	dt : int
-		GDAL GDT data type. See notes.
-	dim : tuple
-		raster dimensions (bands, rows, columns) for multi-bands and (row, columns) for single band
-	nbands : int
-		number of bands.
-	nrows : int
-		number of rows
-	ncols : int
-		number of columns
-
-	Methods
-	-------
-	writeRaster(fileName,dtype=None,driverName='GTiff',verbose=False)
-		Write out raster file as geotiff
-	copy()
-		Copy method. Uses deepcopy since array data is present
-	clipToVector(raster_fileName,vector_fileName,verbose=False,output_fileType='GTiff',output_fileName=None,loadOutput=True)
-		Clips to vector using gdalwarp command line utility
-
-	Raises
-	------
-	OSError
-		If fileName does not exist
-	ValueError
-		Raises if input raster
-
-	See Also
-	--------
-
-	Notes
-	-----
-	Currently only accepts single band rasters.
-
-	Multiple datatypes are used. The table below shows which numpy datatypes correspond to the the GDAL types and their integer codes.
-
-	#  ## Integer Code ##   ## Global Descriptor Table ##      ## Numpy ##
-  	#         0                     GDT_Unknown                   NA
-  	#         1  				    GDT_Byte                      np.bool, np.int ,np.int8, np.long, np.byte, np.uint8
-  	#         2      				GDT_UInt16                    np.uint16, np.ushort
-  	#         3     				GDT_Int16                     np.int16, np.short
-  	#         4       				GDT_UInt32                    np.uint32 , np.uintc
-  	#         5       				GDT_Int32                     np.int32, np.intc
-  	#         6       				GDT_Float32                   np.float32, np.single
-  	#         7       				GDT_Float64                   np.float64, np.double
-  	#         8       				GDT_CInt16                    np.complex64
-  	#         9       				GDT_CInt32                    np.complex64
-  	#         10       				GDT_CFloat32                  np.complex64
-  	#         11       				GDT_CFloat64                  np.complex128
-  	#         12       				GDT_TypeCount                 NA
-
-	Examples
-	--------
-	Load Raster
-	>>> rasterData = fldpln.Raster('path/to/raster')
-
-	"""
-
-	# converts numpy datatypes and gdal GDT variables to integer codes
-	dataTypeConversion_name_to_integer = { np.int8 : 1 , np.bool : 1 , np.int : 1 , np.long : 1 , np.byte : 1, np.uint8 : 1,
-										   np.uint16 : 2 , np.int16 : 3 ,
-										   np.ushort : 2 , np.short : 3 ,
-										   np.uint32 : 4 , np.uintc : 4 , np.int32 : 5 , np.intc : 5 ,
-										   np.float32 : 6 , np.single : 6 ,
-										   np.float64 : 7 , np.double : 7 ,
-										   np.complex64 : 10 , np.complex128 : 11 ,
-										   0:0,1:1,2:2,3:3,4:4,5:5,6:6,7:7,8:8,9:9,10:10,11:11,12:12 }
-
-	# converts integer codes and gdal GDT variables to numpy datatypes
-	dataTypeConversion_integer_to_name = {0 : np.complex128 , 1 : np.int8 , 2 : np.uint16 , 3 : np.int16 ,
-										  4 : np.uint32 , 5 : np.int32 , 6 : np.float32 , 7 : np.float64 ,
-										  8 : np.complex64 , 9 : np.complex64 , 10 : np.complex64 , 11 : np.complex128 }
-
-
-	def __init__(self,fileName,loadArray=True,dtype=None):
-
-		"""
-		Initializes Raster Instance from single band raster
-
-		...
-
-		Parameters
-		----------
-		fileName : str
-			File path to single band raster
-		dtype : numpy datatype or int, optional
-			Numpy, GDT, or integer code data type used to override the data type on the file when imported to array (Default Value = None, None sets to the numpy array data type to the one in the raster file)
-
-		Returns
-		-------
-		raster
-			Instance of raster object
-
-		"""
-
-		if not isfile(fileName):
-			raise OSError("File \'{}\' does not exist".format(fileName))
-
-		stream = gdal.Open(fileName,gdal.GA_ReadOnly)
-
-		self.nrows,self.ncols = stream.RasterYSize , stream.RasterXSize
-		self.nbands = stream.RasterCount
-
-		if loadArray:
-			self.array = stream.ReadAsArray()
-
-		self.gt = stream.GetGeoTransform()
-		self.proj = stream.GetProjection()
-
-		# if self.nbands > 1:
-			# raise ValueError('Raster class only accepts single band rasters for now')
-
-		band = stream.GetRasterBand(1)
-
-		self.ndv = band.GetNoDataValue()
-
-		# set data type
-		if dtype is not None: # override raster file type
-
-			# sets dt to dtype integer code
-			try:
-				self.dt = self.dataTypeConversion_name_to_integer[dtype]
-			except KeyError:
-				raise ValueError('{} dtype parameter not accepted. check docs for valid input or set to None to use data type from raster'.format(dtype))
-
-			# sets array data type
-			if isinstance(dtype,type): # if dtype is a numpy data tpe
-
-				self.array = self.array.astype(dtype)
-
-			else: # if dtype is an integer code of GDAL GDT variable
-
-				try:
-					self.array = self.array.astype(self.dataTypeConversion_integer_to_name[dtype])
-				except KeyError:
-					raise ValueError('{} dtype parameter not accepted. check docs for valid input or set to None to use data type from raster'.format(dtype))
-
-		else: # sets to default data type in raster file
-
-			self.dt = band.DataType
-
-			try:
-				self.array.astype(self.dataTypeConversion_integer_to_name[self.dt])
-			except KeyError:
-				raise ValueError('{} dtype parameter not accepted. check docs for valid input or set to None to use data type from raster'.format(self.dt))
-
-		try:
-			self.des = band.GetDescription()
-		except AttributeError:
-			pass
-
-		try:
-			self.ct = stream.GetRasterColorTable()
-		except AttributeError:
-			pass
-
-		# self.dim = self.array.shape
-		self.fileName = fileName
-
-		stream,band = None,None
-
-
-	@property
-	def dim(self):
-		""" Property method for number of dimensions """
-
-		if self.nbands == 1:
-			DIMS = self.nrows,self.ncols
-		if self.nbands > 1:
-			DIMS = self.nbands,self.nrows,self.ncols
-
-		return(DIMS)
-
-
-	def copy(self):
-		""" Copy method. Uses deepcopy since array data is present """
-		return(deepcopy(self))
-
-
-	def writeRaster(self,fileName,dtype=None,driverName='GTiff',verbose=False):
-
-		"""
-		Write out raster file as geotiff
-
-		Parameters
-		----------
-		fileName : str
-			File path to output raster to
-		dtype : numpy datatype or int, optional
-			Numpy, GDT, or integer code data type (Default Value = self.dt attribute value, otherwise uses data type from the numpy array)
-		driverName : str, optional
-			GDAL driver type. See gdal docs for more details. Only tested for GTiff. (Default Value = 'GTiff')
-		verbose : Boolean, optional
-			Verbose output (Default Value = False)
-
-		Returns
-		-------
-		None
-
-		Raises
-		------
-		ValueError
-			Raises ValueError when the data type parameter is not recognized. See the help docs for raster class to see which numpy, gdal, or encoded values are accepted.
-
-		Examples
-		--------
-		Write Geotiff raster
-		>>> rasterData = fldpln.Raster('path/to/raster')
-		>>> rasterData.writeRaster('/different/path/to/raster',dtype=np.int8)
-
-		"""
-
-		driver = gdal.GetDriverByName(driverName)
-
-		if dtype is None:
-			try:
-				dtype = self.dt
-			except AttributeError:
-				# dtype = gdal.GDT_Float64
-				try:
-					dtype = self.dataTypeConversion_name_to_integer[self.array.dtype]
-				except KeyError:
-					raise ValueError('{} dtype parameter not accepted. check docs for valid input or set to None to use data type from numpy array'.format(self.array.dtype))
-		else:
-			try:
-				dtype = self.dataTypeConversion_name_to_integer[dtype]
-			except KeyError:
-				raise ValueError('{} dtype parameter not accepted. check docs for valid input or set to None to use data type from numpy array'.format(self.array.dtype))
-
-		dataset = driver.Create(fileName, self.ncols, self.nrows, 1, dtype)
-		dataset.SetGeoTransform(self.gt)
-		dataset.SetProjection(self.proj)
-		band = dataset.GetRasterBand(1)
-
-		# set color table and color interpretation
-		#print(band.__dict__)
-		try:
-			band.SetRasterColorTable(self.ct)
-			#band.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
-		except AttributeError:
-			pass
-
-		try:
-			band.SetDescription(self.des)
-		except AttributeError:
-			pass
-
-		band.SetNoDataValue(self.ndv)
-		band.WriteArray(self.array)
-		band, dataset = None,None  # Close the file
-
-		if verbose:
-			print("Successfully wrote out raster to {}".format(fileName))
-
-	def polygonize(self,vector_fileName,vector_driver,layer_name,verbose):
-
-		gdal.UseExceptions()
-
-		#  get raster datasource
-		#
-		src_ds = gdal.Open( self.fileName )
-		srcband = src_ds.GetRasterBand(1)
-
-		#
-		#  create output datasource
-		driver_ext_dict = {'ESRI Shapefile' : 'shp' , 'GPKG' : 'gpkg'}
-
-		if vector_driver not in driver_ext_dict:
-			raise ValueError('Driver not found in {}'.format(driver_ext_dict))
-
-		drv = ogr.GetDriverByName(vector_driver)
-		dst_ds = drv.CreateDataSource( vector_fileName)
-
-		srs = osr.SpatialReference()
-		srs.ImportFromWkt(self.proj)
-
-		dst_layer = dst_ds.CreateLayer(layer_name, srs = srs, geom_type = ogr.wkbPolygon )
-
-		if verbose:
-			prog_func = gdal.TermProgress_nocb
-		else:
-			prog_func = None
-
-		gdal.Polygonize( srcband, None, dst_layer, -1, ['8CONNECTED=8'], callback=prog_func )
-
-	@classmethod
-	def clipToVector(cls,raster_fileName,vector_fileName,output_fileName=None,output_fileType='GTiff',verbose=False):
-		"""
-		Clips to vector using gdalwarp command line utility
-
-		...
-
-		Parameters
-		----------
-		raster_fileName : str
-			File path to raster to clip
-		vector_fileName : str
-			File path to vector layer to clip with
-		output_fileName : str
-			Set file path to output clipped raster (Default Value = None)
-		output_fileType : str
-			Set file type of output from GDAL drivers list (Default Value = 'GTiff')
-		verbose : Boolean
-			Verbose output (Default Value = False)
-
-		Returns
-		-------
-		raster : raster
-			Clipped raster layer
-
-		Notes
-		-----
-		gdalwarp utility must be installed and callable via a subprocess
-
-		Examples
-		--------
-		clip raster and don't return
-		>>> fldpln.raster.clipToVector('path/to/raster','path/to/clipping/vector','path/to/write/output/raster/to')
-		Clip raster and return but don't write
-		>>> clippedRaster = fldpln.raster.clipToVector('path/to/raster','path/to/clipping/vector')
-
-
-		"""
-
-		# create temp output if none is desired
-		if output_fileName is None:
-			output_fileName = 'temp.tif'
-
-		# generate command
-		command = ['gdalwarp','-overwrite','-of',output_fileType,'-cutline',vector_fileName,'-crop_to_cutline',raster_fileName,output_fileName]
-
-		# insert quiet flag if not verbose
-		if not verbose:
-			command = command.insert(1,'-q')
-
-		# call command
-		call(command)
-
-		# remove temp file
-		if output_fileName is None:
-			remove(output_fileName)
-
-		return(cls(output_fileName))
-
-	def getCoordinatesFromIndex(self,row,col):
-		"""
-		Returns coordinates in the rasters projection from a given multi-index
-
-		"""
-
-		# extract variables for readability
-		x_upper_limit, y_upper_limit = self.gt[0], self.gt[3]
-		x_resolution, y_resolution = self.gt[1], self.gt[5]
-		nrows, ncols = self.nrows, self.ncols
-
-		x = x_upper_limit + (col * x_resolution)
-		y = y_upper_limit + (row * y_resolution)
-
-		return(x,y)
-
-
-	def sampleFromCoordinates(self,x,y,returns='value'):
-		"""
-		Sample raster value from coordinates
-		...
-
-		Parameters
-		----------
-		raster_fileName : str
-			File path to raster to clip
-		vector_fileName : str
-			File path to vector layer to clip with
-		output_fileName : str
-			Set file path to output clipped raster (Default Value = None)
-		output_fileType : str
-			Set file type of output from GDAL drivers list (Default Value = 'GTiff')
-		verbose : Boolean
-			Verbose output (Default Value = False)
-
-		Returns
-		-------
-		raster : raster
-			Clipped raster layer
-
-		Notes
-		-----
-		gdalwarp utility must be installed and callable via a subprocess
-
-		Examples
-		--------
-		clip raster and don't return
-		>>> fldpln.raster.clipToVector('path/to/raster','path/to/clipping/vector','path/to/write/output/raster/to')
-		Clip raster and return but don't write
-		>>> clippedRaster = fldpln.raster.clipToVector('path/to/raster','path/to/clipping/vector')
-
-
-		"""
-
-		# extract variables for readability
-		x_upper_limit, y_upper_limit = self.gt[0], self.gt[3]
-		x_resolution, y_resolution = self.gt[1], self.gt[5]
-		nrows, ncols = self.nrows, self.ncols
-
-		# get upper left hand corner coordinates from the centroid coordinates of the upper left pixel
-		x_upper_limit =  x_upper_limit - (x_resolution/2)
-		y_upper_limit = y_upper_limit - (y_resolution/2)
-
-		# get indices
-		columnIndex = int( ( x - x_upper_limit) / x_resolution)
-		rowIndex = int( ( y - y_upper_limit) / y_resolution)
-
-		# check indices lie within raster limits
-		columnIndexInRange = ncols > columnIndex >= 0
-		rowIndexInRange = nrows > rowIndex >= 0
-
-		if (not columnIndexInRange) | (not rowIndexInRange):
-			raise ValueError("Row Index {} or column index {} not in raster range ({},{})".format(rowIndex,columnIndex,nrows,ncols))
-
-		# check value is not ndv
-		if self.array[rowIndex,columnIndex] == self.ndv:
-			raise ValueError("Sample value is no data at ({},{})".format(nrows,ncols))
-
-		# return if statements
-		if returns == 'value':
-			return(self.array[rowIndex,columnIndex])
-		elif returns == 'multi-index':
-			return(rowIndex,columnIndex)
-		elif returns == 'ravel-index':
-			return(np.ravel_multi_index((rowIndex,columnIndex),(nrows,ncols)))
-		else:
-			raise ValueError('Enter valid returns argument')
diff --git a/src/reachID_grid_to_vector_points.py b/src/reachID_grid_to_vector_points.py
index 5dadd43c1..c77bcc732 100755
--- a/src/reachID_grid_to_vector_points.py
+++ b/src/reachID_grid_to_vector_points.py
@@ -1,17 +1,15 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8
 
-from osgeo import gdal
 import numpy as np
 import osgeo.ogr
 import osgeo.osr
 import sys
-
-import cProfile
+import argparse
 from tqdm import tqdm
 import geopandas as gpd
+from utils.shared_variables import PREP_PROJECTION
 from shapely.geometry import Point
-from raster import Raster
+import rasterio
 from utils.shared_functions import getDriver
 
 """
@@ -19,67 +17,55 @@
 ./reachID_grid_to_vector_points.py <flows_grid_IDs raster file> <flows_points vector file> <reachID or featureID>
 
 """
+def convert_grid_cells_to_points(raster,index_option,output_points_filename=False):
 
-path = sys.argv[1]
-outputFileName = sys.argv[2]
-writeOption = sys.argv[3]
-
-#r = gdal.Open(path)
-#band = r.GetRasterBand(1)
-boolean=Raster(path)
-
-#(upper_left_x, x_size, x_rotation, upper_left_y, y_rotation, y_size) = r.GetGeoTransform()
-(upper_left_x, x_size, x_rotation, upper_left_y, y_rotation, y_size) = boolean.gt
-
-#a = band.ReadAsArray().astype(np.float)
-
-# indices = np.nonzero(a != band.GetNoDataValue())
-indices = np.nonzero(boolean.array >= 1)
+    # Input raster
+    if isinstance(raster,str):
+        raster = rasterio.open(raster,'r')
 
-# Init the shapefile stuff..
-#srs = osgeo.osr.SpatialReference()
-#srs.ImportFromWkt(r.GetProjection())
+    elif isinstance(raster,rasterio.io.DatasetReader):
+        pass
 
-#driver = osgeo.ogr.GetDriverByName('GPKG')
-#shapeData = driver.CreateDataSource(outputFileName)
+    else:
+        raise TypeError("Pass raster dataset or filepath for raster")
 
-#layer = shapeData.CreateLayer('ogr_pts', srs, osgeo.ogr.wkbPoint)
-#layerDefinition = layer.GetLayerDefn()
+    (upper_left_x, x_size, x_rotation, upper_left_y, y_rotation, y_size) = raster.get_transform()
+    indices = np.nonzero(raster.read(1) >= 1)
 
-#idField = osgeo.ogr.FieldDefn("id", osgeo.ogr.OFTInteger)
-#layer.CreateField(idField)
+    id =[None] * len(indices[0]);points = [None]*len(indices[0])
 
-id =[None] * len(indices[0]);points = [None]*len(indices[0])
+    # Iterate over the Numpy points..
+    i = 1
+    for y_index,x_index in zip(*indices):
+        x = x_index * x_size + upper_left_x + (x_size / 2) # add half the cell size
+        y = y_index * y_size + upper_left_y + (y_size / 2) # to center the point
+        points[i-1] = Point(x,y)
+        if index_option == 'reachID':
+            reachID = np.array(list(raster.sample((Point(x,y).coords), indexes=1))).item() # check this; needs to add raster cell value + index
+            id[i-1] = reachID*10000 + i #reachID + i/100
+        elif (index_option == 'featureID') |(index_option == 'pixelID'):
+            id[i-1] = i
+        i += 1
 
-# Iterate over the Numpy points..
-i = 1
-for y_index,x_index in tqdm(zip(*indices),total=len(indices[0])):
-    x = x_index * x_size + upper_left_x + (x_size / 2) #add half the cell size
-    y = y_index * y_size + upper_left_y + (y_size / 2) #to centre the point
+    pointGDF = gpd.GeoDataFrame({'id' : id, 'geometry' : points},crs=PREP_PROJECTION,geometry='geometry')
 
-    # get raster value
-    #reachID = a[y_index,x_index]
+    if output_points_filename == False:
+        return pointGDF
+    else:
+        pointGDF.to_file(output_points_filename,driver=getDriver(output_points_filename),index=False)
 
-    #point = osgeo.ogr.Geometry(osgeo.ogr.wkbPoint)
-    #point.SetPoint(0, x, y)
-    points[i-1] = Point(x,y)
+if __name__ == '__main__':
 
-    #feature = osgeo.ogr.Feature(layerDefinition)
-    #feature.SetGeometry(point)
-    #feature.SetFID(i)
-    if writeOption == 'reachID':
-        reachID = a[y_index,x_index]
-        id[i-1] = reachID
-	#feature.SetField("id",reachID)
-    elif (writeOption == 'featureID') |( writeOption == 'pixelID'):
-        #feature.SetField("id",i)
-        id[i-1] = i
-    #layer.CreateFeature(feature)
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Converts a raster to points')
+    parser.add_argument('-r','--raster',help='Raster to be converted to points',required=True,type=str)
+    parser.add_argument('-i', '--index-option',help='Indexing option',required=True,type=str,choices=['reachID','featureID','pixelID'])
+    parser.add_argument('-p', '--output-points-filename',help='Output points layer filename',required=False,type=str,default=False)
 
-    i += 1
+    args = vars(parser.parse_args())
 
-pointGDF = gpd.GeoDataFrame({'id' : id, 'geometry' : points},crs=boolean.proj,geometry='geometry')
-pointGDF.to_file(outputFileName,driver=getDriver(outputFileName),index=False)
+    raster = args['raster']
+    index_option = args['index_option']
+    output_points_filename = args['output_points_filename']
 
-print("Complete")
-#shapeData.Destroy()
+    convert_grid_cells_to_points(raster,index_option,output_points_filename)
diff --git a/src/reduce_nhd_stream_density.py b/src/reduce_nhd_stream_density.py
index 9614bfe32..c32afd990 100644
--- a/src/reduce_nhd_stream_density.py
+++ b/src/reduce_nhd_stream_density.py
@@ -7,6 +7,7 @@
 import argparse
 import pygeos
 from shapely.wkb import dumps
+from shapely.geometry import Point
 from utils.shared_functions import getDriver
 
 
@@ -29,7 +30,7 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
     for index, row in selected_wbd8.iterrows():
         huc = row["HUC8"]
 
-        # Double check that this is a nested HUC (probably overkill)
+        # Double check that this is a nested HUC
         if huc.startswith(str(huc4)):
 
             huc8_mask = selected_wbd8.loc[selected_wbd8.HUC8==huc]
@@ -44,6 +45,9 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
                 streams_subset = gpd.read_file(nhd_streams_, mask = huc8_mask)
             else:
                 streams_subset = nhd_streams.loc[nhd_streams.HUC8==huc].copy()
+                if headwaters_mask.is_headwater.dtype != 'int': headwaters_mask.is_headwater = headwaters_mask.is_headwater.astype('int')
+                if headwaters_mask.is_colocated.dtype != 'int': headwaters_mask.is_colocated = headwaters_mask.is_colocated.astype('int')
+                headwaters_mask = headwaters_mask.loc[headwaters_mask.is_headwater==True]
 
             if not streams_subset.empty:
                 streams_subset[headwater_col] = False
@@ -63,14 +67,14 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
                 # Add headwaters_id column
                 streams_subset[id_col] = n
 
-                # Find stream segment closest to headwater point
+                distance_from_upstream = {}
                 for index, point in headwaters_mask.iterrows():
 
                     # Convert headwaterpoint geometries to WKB representation
-                    wkb_points = dumps(point.geometry)
+                    wkb_point = dumps(point.geometry)
 
                     # Create pygeos headwaterpoint geometries from WKB representation
-                    pointbin_geom = pygeos.io.from_wkb(wkb_points)
+                    pointbin_geom = pygeos.io.from_wkb(wkb_point)
 
                     # Distance to each stream segment
                     distances = pygeos.measurement.distance(streambin_geom, pointbin_geom)
@@ -78,9 +82,30 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
                     # Find minimum distance
                     min_index = np.argmin(distances)
 
+                    headwater_point_name = point[headwater_id]
+
+                    # Find stream segment closest to headwater point
+                    if mainstem_flag==True:
+
+                        if point.is_colocated==True:
+
+                            closest_stream = streams_subset.iloc[min_index]
+                            distance_to_line = point.geometry.distance(Point(closest_stream.geometry.coords[-1]))
+
+                            print(f"{point.nws_lid} distance on line {closest_stream.NHDPlusID}:  {np.round(distance_to_line,1)}")
+                            if not closest_stream.NHDPlusID in distance_from_upstream.keys():
+
+                                distance_from_upstream[closest_stream.NHDPlusID] = [point.nws_lid,distance_to_line]
+
+                            elif distance_from_upstream[closest_stream.NHDPlusID][1] > distance_to_line:
+
+                                distance_from_upstream[closest_stream.NHDPlusID] = [point.nws_lid,distance_to_line]
+
+                            headwater_point_name = distance_from_upstream[closest_stream.NHDPlusID][0]
+
                     # Closest segment to headwater
                     streams_subset.loc[min_index,headwater_col] = True
-                    streams_subset.loc[min_index,id_col] = point[headwater_id]
+                    streams_subset.loc[min_index,id_col] = headwater_point_name
 
                 headwater_streams = headwater_streams.append(streams_subset[['NHDPlusID',headwater_col,id_col,'HUC8']])
 
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 6e9cfca7b..cab360898 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -132,9 +132,8 @@ python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputHucDataDir/dem_
 Tcount
 
 ## DEM Reconditioning ##
-# Using AGREE methodology, hydroenforce the DEM so that it is consistent
-# with the supplied stream network. This allows for more realistic catchment
-# delineation which is ultimately reflected in the output FIM mapping.
+# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
+# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer"$stopDiv
 date -u
 Tstart
@@ -191,7 +190,7 @@ Tcount
 echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputHucDataDir/dem_meters.tif -s $outputHucDataDir/demDerived_streamPixels.tif -a $outputHucDataDir/demDerived_streamPixels_ids_allo.tif -d $outputHucDataDir/demDerived_streamPixels_ids_dist.tif -t 50 -o $outputHucDataDir/dem_lateral_thalweg_adj.tif
+python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputHucDataDir/dem_meters.tif -s $outputHucDataDir/demDerived_streamPixels.tif -a $outputHucDataDir/demDerived_streamPixels_ids_allo.tif -d $outputHucDataDir/demDerived_streamPixels_ids_dist.tif -t 50 -o $outputHucDataDir/dem_lateral_thalweg_adj.tif -th $thalweg_lateral_elev_threshold
 Tcount
 
 ## MASK BURNED DEM FOR STREAMS ONLY ###
@@ -277,7 +276,7 @@ echo -e $startDiv"Vectorize Pixel Centroids $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/flows_points_pixels.gpkg ] && \
-$srcDir/reachID_grid_to_vector_points.py $demDerived_streamPixels $outputHucDataDir/flows_points_pixels.gpkg featureID
+$srcDir/reachID_grid_to_vector_points.py -r $demDerived_streamPixels -i featureID -p $outputHucDataDir/flows_points_pixels.gpkg
 Tcount
 
 ## GAGE WATERSHED FOR PIXELS ##
diff --git a/src/split_flows.py b/src/split_flows.py
index 67b69f7e9..9d51065dd 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -181,27 +181,19 @@
 else:
     print ('Error: Could not add network attributes to stream segments')
 
-# Get Outlet Point Only
-#outlet = OrderedDict()
-#for i,segment in split_flows_gdf.iterrows():
-#    outlet[segment.geometry.coords[-1]] = segment[hydro_id]
-
-#hydroIDs_points = [hidp for hidp in outlet.values()]
-#split_points = [Point(*point) for point in outlet]
-
 # Get all vertices
 split_points = OrderedDict()
-for row in split_flows_gdf[['geometry',hydro_id, 'NextDownID']].iterrows():
-    lineString = row[1][0]
+for index, segment in split_flows_gdf.iterrows():
+    lineString = segment.geometry
 
     for point in zip(*lineString.coords.xy):
         if point in split_points:
-            if row[1][2] == split_points[point]:
+            if segment.NextDownID == split_points[point]:
                 pass
             else:
-                split_points[point] = row[1][1]
+                split_points[point] = segment[hydro_id]
         else:
-            split_points[point] = row[1][1]
+            split_points[point] = segment[hydro_id]
 
 hydroIDs_points = [hidp for hidp in split_points.values()]
 split_points = [Point(*point) for point in split_points]
diff --git a/tools/thalweg_drop_check.py b/tools/thalweg_drop_check.py
new file mode 100644
index 000000000..c56d743e5
--- /dev/null
+++ b/tools/thalweg_drop_check.py
@@ -0,0 +1,385 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import geopandas as gpd
+sys.path.append('/foss_fim/src')
+from utils.shared_variables import PREP_PROJECTION
+from shapely.geometry import Point, LineString
+import rasterio
+import pandas as pd
+import numpy as np
+import argparse
+import matplotlib.pyplot as plt
+import seaborn as sns
+from collections import deque
+from os.path import join
+from multiprocessing import Pool
+from utils.shared_functions import getDriver
+from rasterio import features
+from reachID_grid_to_vector_points import convert_grid_cells_to_points
+import warnings
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+"""
+    Plot Rating Curves and Compare to USGS Gages
+
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders.
+    output_dir : str
+        Stream layer to be evaluated.
+    stream_type : str
+        File name of USGS rating curves.
+    point_density : str
+        Elevation sampling density.
+    number_of_jobs : str
+        Number of jobs.
+"""
+
+
+def compare_thalweg(args):
+
+    huc_dir                             = args[0]
+    stream_type                         = args[1]
+    point_density                       = args[2]
+    huc                                 = args[3]
+    dem_meters_filename                 = args[4]
+    dem_lateral_thalweg_adj_filename    = args[5]
+    dem_thalwegCond_filename            = args[6]
+    profile_plots_filename              = args[7]
+    profile_gpkg_filename               = args[8]
+    profile_table_filename              = args[9]
+    flows_grid_boolean_filename         = args[10]
+
+    if stream_type == 'derived':
+
+        dem_derived_reaches_filename = os.path.join(huc_dir,'demDerived_reaches_split.gpkg')
+        streams = gpd.read_file(dem_derived_reaches_filename)
+        nhd_headwater_filename = os.path.join(huc_dir,'nhd_headwater_points_subset.gpkg')
+        wbd_filename = os.path.join(huc_dir,'wbd.gpkg')
+        wbd = gpd.read_file(wbd_filename)
+        headwaters_layer = gpd.read_file(nhd_headwater_filename,mask=wbd)
+        headwater_list = headwaters_layer.loc[headwaters_layer.pt_type == 'nws_lid']
+        stream_id = 'HydroID'
+
+    elif stream_type == 'burnline':
+
+        nhd_reaches_filename = os.path.join(huc_dir,'NHDPlusBurnLineEvent_subset.gpkg')
+        nhd_reaches = gpd.read_file(nhd_reaches_filename)
+        streams = nhd_reaches.copy()
+        headwaters_layer = None
+
+        # Get lists of all complete reaches using headwater attributes
+        headwater_list = streams.loc[streams.nws_lid!=''].nws_lid
+        stream_id = 'NHDPlusID'
+
+    headwater_col = 'is_headwater'
+    streams[headwater_col] = False
+    headwater_list = headwater_list.reset_index(drop=True)
+
+    if stream_type == 'derived':
+        streams['nws_lid'] = ''
+
+        if streams.NextDownID.dtype != 'int': streams.NextDownID = streams.NextDownID.astype(int)
+
+        min_dist = np.empty(len(headwater_list))
+        streams['min_dist'] = 1000
+
+        for i, point in headwater_list.iterrows():
+            streams['min_dist'] = [point.geometry.distance(line) for line in streams.geometry]
+            streams.loc[streams.min_dist==np.min(streams.min_dist),'nws_lid'] = point.site_id
+
+        headwater_list = headwater_list.site_id
+
+    streams.set_index(stream_id,inplace=True,drop=False)
+
+    # Collect headwater streams
+    single_stream_paths = []
+    dem_meters = rasterio.open(dem_meters_filename,'r')
+    index_option = 'reachID'
+    for index, headwater_site in enumerate(headwater_list):
+        stream_path = get_downstream_segments(streams.copy(),'nws_lid', headwater_site,'downstream',stream_id,stream_type)
+        stream_path = stream_path.reset_index(drop=True)
+        stream_path = stream_path.sort_values(by=['downstream_count'])
+        stream_path = stream_path.loc[stream_path.downstream==True]
+        if stream_type == 'burnline':
+            geom_value = []
+            for index, segment in stream_path.iterrows():
+                lineString = LineString(segment.geometry.coords[::-1])
+                geom_value = geom_value + [(lineString, segment.downstream_count)]
+            nhd_reaches_raster = features.rasterize(shapes=geom_value , out_shape=[dem_meters.height, dem_meters.width],fill=dem_meters.nodata,transform=dem_meters.transform, all_touched=True, dtype=np.float32)
+            flow_bool = rasterio.open(flows_grid_boolean_filename)
+            flow_bool_data = flow_bool.read(1)
+            nhd_reaches_raster = np.where(flow_bool_data == int(0), -9999.0, (nhd_reaches_raster).astype(rasterio.float32))
+            out_dem_filename = os.path.join(huc_dir,'NHDPlusBurnLineEvent_raster.tif')
+            with rasterio.open(out_dem_filename, "w", **dem_meters.profile, BIGTIFF='YES') as dest:
+                dest.write(nhd_reaches_raster, indexes = 1)
+            stream_path = convert_grid_cells_to_points(out_dem_filename,index_option)
+        stream_path["headwater_path"] = headwater_site
+        single_stream_paths = single_stream_paths + [stream_path]
+        print(f"length of {headwater_site} path: {len(stream_path)}")
+
+    # Collect elevation values from multiple grids along each individual reach point
+    dem_lateral_thalweg_adj = rasterio.open(dem_lateral_thalweg_adj_filename,'r')
+    dem_thalwegCond = rasterio.open(dem_thalwegCond_filename,'r')
+    thalweg_points = gpd.GeoDataFrame()
+    for path in single_stream_paths:
+        split_points = []
+        stream_ids = []
+        dem_m_elev = []
+        dem_burned_filled_elev = []
+        dem_lat_thal_adj_elev = []
+        dem_thal_adj_elev = []
+        headwater_path = []
+        index_count = []
+        for index, segment in path.iterrows():
+            if stream_type == 'derived':
+                linestring = segment.geometry
+                if point_density == 'midpoints':
+                    midpoint = linestring.interpolate(0.5,normalized=True)
+                    stream_ids = stream_ids + [segment[stream_id]]
+                    split_points = split_points + [midpoint]
+                    index_count = index_count + [segment.downstream_count]
+                    dem_m_elev = dem_m_elev + [np.array(list(dem_meters.sample((Point(midpoint).coords), indexes=1))).item()]
+                    dem_lat_thal_adj_elev = dem_lat_thal_adj_elev + [np.array(list(dem_lateral_thalweg_adj.sample((Point(midpoint).coords), indexes=1))).item()]
+                    dem_thal_adj_elev = dem_thal_adj_elev + [np.array(list(dem_thalwegCond.sample((Point(midpoint).coords), indexes=1))).item()]
+                    headwater_path = headwater_path + [segment.headwater_path]
+                elif point_density == 'all_points':
+                    count=0
+                    for point in zip(*linestring.coords.xy):
+                        stream_ids = stream_ids + [segment[stream_id]]
+                        split_points = split_points + [Point(point)]
+                        count = count + 1
+                        index_count = index_count + [segment.downstream_count*1000 + count]
+                        dem_m_elev = dem_m_elev + [np.array(list(dem_meters.sample((Point(point).coords), indexes=1))).item()]
+                        dem_lat_thal_adj_elev = dem_lat_thal_adj_elev + [np.array(list(dem_lateral_thalweg_adj.sample((Point(point).coords), indexes=1))).item()]
+                        dem_thal_adj_elev = dem_thal_adj_elev + [np.array(list(dem_thalwegCond.sample((Point(point).coords), indexes=1))).item()]
+                        headwater_path = headwater_path + [segment.headwater_path]
+            elif stream_type == 'burnline':
+                stream_ids = stream_ids + [segment['id']]
+                split_points = split_points + [Point(segment.geometry)]
+                index_count = index_count + [segment['id']]
+                dem_m_elev = dem_m_elev + [np.array(list(dem_meters.sample((Point(segment.geometry).coords), indexes=1))).item()]
+                dem_lat_thal_adj_elev = dem_lat_thal_adj_elev + [np.array(list(dem_lateral_thalweg_adj.sample((Point(segment.geometry).coords), indexes=1))).item()]
+                dem_thal_adj_elev = dem_thal_adj_elev + [np.array(list(dem_thalwegCond.sample((Point(segment.geometry).coords), indexes=1))).item()]
+                headwater_path = headwater_path + [segment.headwater_path]
+        # gpd.GeoDataFrame({**data, "source": "dem_m"})
+        dem_m_pts = gpd.GeoDataFrame({'stream_id': stream_ids, 'source': 'dem_m', 'elevation_m': dem_m_elev, 'headwater_path': headwater_path, 'index_count': index_count, 'geometry': split_points}, crs=path.crs, geometry='geometry')
+        dem_lat_thal_adj_pts = gpd.GeoDataFrame({'stream_id': stream_ids, 'source': 'dem_lat_thal_adj', 'elevation_m': dem_lat_thal_adj_elev, 'headwater_path': headwater_path, 'index_count': index_count, 'geometry': split_points}, crs=path.crs, geometry='geometry')
+        dem_thal_adj_pts = gpd.GeoDataFrame({'stream_id': stream_ids, 'source': 'thal_adj_dem', 'elevation_m': dem_thal_adj_elev, 'headwater_path': headwater_path, 'index_count': index_count, 'geometry': split_points}, crs=path.crs, geometry='geometry')
+        for raster in [dem_m_pts,dem_lat_thal_adj_pts,dem_thal_adj_pts]:
+            raster = raster.sort_values(by=['index_count'])
+            raster.set_index('index_count',inplace=True,drop=True)
+            raster = raster.reset_index(drop=True)
+            raster.index.names = ['index_count']
+            raster = raster.reset_index(drop=False)
+            thalweg_points = thalweg_points.append(raster,ignore_index = True)
+            del raster
+        del dem_m_pts,dem_lat_thal_adj_pts,dem_thal_adj_pts
+
+    del dem_lateral_thalweg_adj,dem_thalwegCond,dem_meters
+
+    try:
+        # Remove nodata_pts and convert elevation to ft
+        thalweg_points = thalweg_points.loc[thalweg_points.elevation_m > 0.0]
+        thalweg_points.elevation_m =  np.round(thalweg_points.elevation_m,3)
+        thalweg_points['elevation_ft'] =  np.round(thalweg_points.elevation_m*3.28084,3)
+
+        # Plot thalweg profile
+        plot_profile(thalweg_points, profile_plots_filename)
+
+        # Filter final thalweg ajdusted layer
+        thal_adj_points = thalweg_points.loc[thalweg_points.source=='thal_adj_dem'].copy()
+        # thal_adj_points.to_file(profile_gpkg_filename,driver=getDriver(profile_gpkg_filename))
+
+        # Identify significant rises/drops in elevation
+        thal_adj_points['elev_change'] = thal_adj_points.groupby(['headwater_path', 'source'])['elevation_m'].apply(lambda x: x - x.shift())
+        elev_changes = thal_adj_points.loc[(thal_adj_points.elev_change<=-lateral_elevation_threshold) | (thal_adj_points.elev_change>0.0)]
+
+        if not elev_changes.empty:
+            # elev_changes.to_csv(profile_table_filename,index=False)
+            elev_changes.to_file(profile_gpkg_filename,index=False,driver=getDriver(profile_gpkg_filename))
+
+
+        # Zoom in to plot only areas with steep elevation changes
+        # select_streams = elev_changes.stream_id.to_list()
+        # downstream_segments = [index + 1 for index in select_streams]
+        # upstream_segments = [index - 1 for index in select_streams]
+        # select_streams = list(set(upstream_segments + downstream_segments + select_streams))
+        # thal_adj_points_select = thal_adj_points.loc[thal_adj_points.stream_id.isin(select_streams)]
+        # plot_profile(thal_adj_points_select, profile_plots_filename_zoom)
+
+    except:
+        print(f"huc {huc} has {len(thalweg_points)} thalweg points")
+
+def get_downstream_segments(streams, headwater_col,headwater_id,flag_column,stream_id,stream_type):
+
+    streams[flag_column] = False
+    streams['downstream_count'] = -9
+    streams.loc[streams[headwater_col]==headwater_id,flag_column] = True
+    streams.loc[streams[headwater_col]==headwater_id,'downstream_count'] = 0
+    count = 0
+
+    Q = deque(streams.loc[streams[headwater_col]==headwater_id,stream_id].tolist())
+    visited = set()
+
+    while Q:
+        q = Q.popleft()
+
+        if q in visited:
+            continue
+
+        visited.add(q)
+
+        count = count + 1
+        if stream_type == 'burnline':
+
+            toNode,DnLevelPat = streams.loc[q,['ToNode','DnLevelPat']]
+            downstream_ids = streams.loc[streams['FromNode'] == toNode,:].index.tolist()
+
+            # If multiple downstream_ids are returned select the ids that are along the main flow path (i.e. exclude segments that are diversions)
+            if len(set(downstream_ids)) > 1: # special case: remove duplicate NHDPlusIDs
+
+                relevant_ids = [segment for segment in downstream_ids if DnLevelPat == streams.loc[segment,'LevelPathI']]
+
+            else:
+
+                relevant_ids = downstream_ids
+
+        elif stream_type == 'derived':
+
+            toNode = streams.loc[q,['NextDownID']].item()
+            relevant_ids = streams.loc[streams[stream_id] == toNode,:].index.tolist()
+
+        streams.loc[relevant_ids,flag_column] = True
+        streams.loc[relevant_ids,'downstream_count'] = count
+
+        for i in relevant_ids:
+
+            if i not in visited:
+                Q.append(i)
+
+    streams = streams.loc[streams[flag_column],:]
+
+    return streams
+
+
+def plot_profile(elevation_table,profile_plots_filename):
+    num_plots = len(elevation_table.headwater_path.unique())
+    unique_rasters = elevation_table.source.unique()
+    if num_plots > 3:
+        columns = int(np.ceil(num_plots / 3))
+    else:
+        columns = 1
+    # palette = dict(zip(unique_rasters, sns.color_palette(n_colors=len(unique_rasters))))
+    # palette.update({'dem_m':'gray'})
+    sns.set(style="ticks")
+    if len(unique_rasters) > 1:
+        g = sns.FacetGrid(elevation_table, col="headwater_path", hue="source", hue_order=['dem_m', 'dem_lat_thal_adj', 'thal_adj_dem'], sharex=False, sharey=False,col_wrap=columns)
+    else:
+        g = sns.FacetGrid(elevation_table, col="headwater_path", hue="source", sharex=False, sharey=False,col_wrap=columns)
+    g.map(sns.lineplot, "index_count", "elevation_ft", palette="tab20c")
+    g.set_axis_labels(x_var="Longitudinal Profile (index)", y_var="Elevation (ft)")
+    # Iterate thorugh each axis to get individual y-axis bounds
+    for ax in g.axes.flat:
+        mins = []
+        maxes = []
+        for line in ax.lines:
+            mins = mins + [min(line.get_ydata())]
+            maxes = maxes + [max(line.get_ydata())]
+        min_y = min(mins) - (max(maxes) - min(mins))/10
+        max_y = max(maxes) + (max(maxes) - min(mins))/10
+        ax.set_ylim(min_y,max_y)
+    # if len(unique_rasters) > 1:
+    #     ax.lines[0].set_linestyle("--")
+    #     ax.lines[0].set_color('gray')
+    # box = ax.get_position()
+    # ax.set_position([box.x0, box.y0 + box.height * 0.1,box.width, box.height * 0.9])
+    # Adjust the arrangement of the plots
+    # g.fig.tight_layout(w_pad=5) #w_pad=2
+    g.add_legend()
+    # plt.legend(bbox_to_anchor=(1.04,0.5), loc="center left", borderaxespad=0)
+    plt.subplots_adjust(bottom=0.25)
+    plt.savefig(profile_plots_filename)
+    plt.close()
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='generate rating curve plots and tables for FIM and USGS gages')
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-output_dir','--output-dir', help='rating curves output folder', required=True,type=str)
+    # parser.add_argument('-rasters','--raster-list',help='list of rasters to be evaluated',required=True,type=str)
+    parser.add_argument('-stream_type','--stream-type',help='stream layer to be evaluated',required=True,type=str,choices=['derived','burnline'])
+    parser.add_argument('-point_density','--point-density',help='elevation sampling density',required=True,type=str,choices=['midpoints','all_points'])
+    parser.add_argument('-th','--elevation_threshold',help='significant elevation drop threshold in meters.',required=True)
+    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    output_dir = args['output_dir']
+    # raster_list = args['raster_list']
+    stream_type = args['stream_type']
+    point_density = args['point_density']
+    number_of_jobs = args['number_of_jobs']
+
+    # dem_meters_dir = os.environ.get('dem_meters')
+
+    plots_dir = join(output_dir,'plots')
+    os.makedirs(plots_dir, exist_ok=True)
+    spatial_dir = os.path.join(output_dir,'spatial_layers')
+    os.makedirs(spatial_dir, exist_ok=True)
+
+    # Open log file
+    sys.__stdout__ = sys.stdout
+    log_file = open(join(output_dir,'thalweg_profile_comparison.log'),"w")
+    sys.stdout = log_file
+
+    procs_list = []
+    huc_list  = os.listdir(fim_dir)
+    for huc in huc_list:
+        if huc != 'logs':
+
+            huc_dir = os.path.join(fim_dir,huc)
+            flows_grid_boolean_filename = os.path.join(huc_dir,'flows_grid_boolean.tif')
+            dem_meters_filename = os.path.join(huc_dir,'dem_meters.tif')
+            dem_lateral_thalweg_adj_filename = os.path.join(huc_dir,'dem_lateral_thalweg_adj.tif')
+            dem_thalwegCond_filename = os.path.join(huc_dir,'dem_thalwegCond.tif')
+            profile_plots_filename = os.path.join(plots_dir,f"profile_drop_plots_{huc}_{point_density}_{stream_type}.png")
+            profile_gpkg_filename = os.path.join(spatial_dir,f"thalweg_elevation_changes_{huc}_{point_density}_{stream_type}.gpkg")
+            profile_table_filename = os.path.join(spatial_dir,f"thalweg_elevation_changes_{huc}_{point_density}_{stream_type}.csv")
+
+            procs_list.append([huc_dir,stream_type,point_density,huc,dem_meters_filename,dem_lateral_thalweg_adj_filename,dem_thalwegCond_filename,profile_plots_filename,profile_gpkg_filename,profile_table_filename,flows_grid_boolean_filename])
+
+    # Initiate multiprocessing
+    print(f"Generating thalweg elevation profiles for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    with Pool(processes=number_of_jobs) as pool:
+        # Get elevation values along thalweg for each headwater stream path
+        pool.map(compare_thalweg, procs_list)
+
+    # Append all elevation change spatial layers to a single gpkg
+    spatial_list  = os.listdir(spatial_dir)
+    agg_thalweg_elevations_gpkg_fileName = os.path.join(output_dir, f"agg_thalweg_elevation_changes_{point_density}_{stream_type}.gpkg")
+    agg_thalweg_elevation_table_fileName = os.path.join(output_dir, f"agg_thalweg_elevation_changes_{point_density}_{stream_type}.csv")
+    for layer in spatial_list:
+
+        huc_gpd = gpd.read_file(os.path.join(spatial_dir,layer))
+
+        # Write aggregate layer
+        if os.path.isfile(agg_thalweg_elevations_gpkg_fileName):
+            huc_gpd.to_file(agg_thalweg_elevations_gpkg_fileName,driver=getDriver(agg_thalweg_elevations_gpkg_fileName),index=False, mode='a')
+        else:
+            huc_gpd.to_file(agg_thalweg_elevations_gpkg_fileName,driver=getDriver(agg_thalweg_elevations_gpkg_fileName),index=False)
+
+        del huc_gpd
+
+    # Create csv of elevation table
+    huc_table = gpd.read_file(agg_thalweg_elevations_gpkg_fileName)
+    huc_table.to_csv(agg_thalweg_elevation_table_fileName,index=False)
+
+    # Close log file
+    sys.stdout = sys.__stdout__
+    log_file.close()

From 7befb3929d573de1ee6128b05a33a84c7f77a6da Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 1 Jul 2021 10:16:49 -0500
Subject: [PATCH 085/359] Allowance of evaluation at IFC sites and updating of
 bad eval sites list

Updates to evaluation scripts to allow for Alpha testing at Iowa Flood Center (IFC) sites. Also, BAD_SITES variable updates to omit sites not suitable for evaluation from metric calculations.

- The BAD_SITES list in tools_shared_variables.py was updated and reasons for site omission are documented.
- Refactored run_test_case.py, synthesize_test_cases.py, tools_shared_variables.py, and eval_plots.py to allow for IFC comparisons.

This resolves #424, resolves #412, and resolves #428.
---
 CHANGELOG.md                    | 10 ++++++++
 tools/eval_plots.py             | 14 ++++++----
 tools/run_test_case.py          |  4 +--
 tools/synthesize_test_cases.py  | 21 +++++++++------
 tools/tools_shared_variables.py | 45 +++++++++++++++++++++++++--------
 5 files changed, 68 insertions(+), 26 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 808effc01..ebe838c85 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.19.2 - 2021-07-01 - [PR #429](https://github.com/NOAA-OWP/cahaba/pull/429)
+
+Updates to evaluation scripts to allow for Alpha testing at Iowa Flood Center (IFC) sites. Also, `BAD_SITES` variable updates to omit sites not suitable for evaluation from metric calculations.
+
+## Changes
+- The `BAD_SITES` list in `tools_shared_variables.py` was updated and reasons for site omission are documented.
+- Refactored `run_test_case.py`, `synthesize_test_cases.py`, `tools_shared_variables.py`, and `eval_plots.py` to allow for IFC comparisons.
+
+<br/><br/>
+
 ## v3.0.19.1 - 2021-06-17 - [PR #417](https://github.com/NOAA-OWP/cahaba/pull/417)
 
 Adding a thalweg profile tool to identify significant drops in thalweg elevation. Also setting lateral thalweg adjustment threshold in hydroconditioning.
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index b2d891c18..ea339867e 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -487,7 +487,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
 
         # If source is 'ble', set base_resolution and append ble dataset to all_datasets dictionary
-        elif benchmark_source == 'ble':
+        elif benchmark_source in ['ble', 'ifc']:
 
             # Set the base processing unit for ble runs
             base_resolution = 'huc'
@@ -511,6 +511,9 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         if dataset_name == 'ble':
             magnitude_order = ['100yr', '500yr']
             base_resolution = 'huc'
+        elif dataset_name == 'ifc':
+            magnitude_order = ['2yr','5yr','10yr','25yr','50yr','100yr','200yr','500yr']
+            base_resolution = 'huc'
         elif dataset_name in ['usgs','nws']:
             magnitude_order = ['action','minor','moderate','major']
             base_resolution = 'nws_lid'
@@ -628,13 +631,15 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         ################################################################
         #This section joins ble (FR) metrics to a spatial layer of HUCs.
         ################################################################
-        if all_datasets.get(('ble','FR')):            
+        if all_datasets.get(('ble','FR')) and all_datasets.get(('ifc','FR')):            
             #Select BLE, FR dataset.
             ble_dataset, sites = all_datasets.get(('ble','FR'))         
+            ifc_dataset, sites = all_datasets.get(('ifc','FR'))               
+            huc_datasets = ble_dataset.append(ifc_dataset)
             #Read in HUC spatial layer
             wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')             
             #Join metrics to HUC spatial layer
-            wbd_with_metrics = wbd_gdf.merge(ble_dataset, how = 'inner', left_on = 'HUC8', right_on = 'huc')
+            wbd_with_metrics = wbd_gdf.merge(huc_datasets, how = 'inner', left_on = 'HUC8', right_on = 'huc')
             #Filter out unnecessary columns
             wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source','geometry'])
             wbd_with_metrics.rename(columns = {'benchmark_source':'source'}, inplace = True )
@@ -643,8 +648,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             #Write out to file
             wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
         else:
-            print('BLE FR datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a FR version')
-
+            print('BLE/IFC FR datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a FR version')
 #######################################################################
 if __name__ == '__main__':
     # Parse arguments
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 59d90f15f..e455cf472 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -81,6 +81,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     magnitude_list = magnitude
     if type(magnitude_list) != list:
         magnitude_list = [magnitude_list]
+        
 
     # Get path to validation_data_{benchmark} directory and huc_dir.
     validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
@@ -88,7 +89,6 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
         version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
         if not os.path.exists(version_test_case_dir):
             os.mkdir(version_test_case_dir)
-
         # Construct path to validation raster and forecast file.
         if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
             benchmark_raster_path_list, forecast_list = [], []
@@ -171,7 +171,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                     print(" ")
                 elif inundate_test == 1:
                     pass
-#                    print (f"No matching feature IDs between forecast and hydrotable for magnitude: {magnitude}")
+                    print (f"No matching feature IDs between forecast and hydrotable for magnitude: {magnitude}")
                     #return
             except Exception as e:
                 print(e)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 9fef7841d..01e29be05 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -7,7 +7,7 @@
 import csv
 
 from run_test_case import run_alpha_test
-from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
+from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, BLE_MAGNITUDE_LIST, IFC_MAGNITUDE_LIST
 
 
 def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list):
@@ -66,9 +66,14 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
     else:
         iteration_list = ['official']
 
-    for benchmark_source in ['ble', 'nws', 'usgs']:
+    for benchmark_source in ['ble', 'nws', 'usgs', 'ifc']:
         benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
-        if benchmark_source == 'ble':
+        if benchmark_source in ['ble', 'ifc']:
+            
+            if benchmark_source == 'ble':
+                magnitude_list = BLE_MAGNITUDE_LIST
+            if benchmark_source == 'ifc':
+                magnitude_list = IFC_MAGNITUDE_LIST
             test_cases_list = os.listdir(benchmark_test_case_dir)
 
             for test_case in test_cases_list:
@@ -86,7 +91,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = dev_versions_to_include_list
 
-                        for magnitude in ['100yr', '500yr']:
+                        for magnitude in magnitude_list:
                             for version in versions_to_aggregate:
                                 if '_fr' in version:
                                     extent_config = 'FR'
@@ -107,7 +112,6 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                         if '.json' in f:
                                             flow = 'NA'
                                             nws_lid = "NA"
-                                            benchmark_source = 'ble'
                                             sub_list_to_append = [version, nws_lid, magnitude, huc]
                                             full_json_path = os.path.join(magnitude_dir, f)
                                             if os.path.exists(full_json_path):
@@ -284,7 +288,6 @@ def process_alpha_test(args):
             if 'validation' and 'other' not in test_id:
                 current_huc = test_id.split('_')[0]
                 if test_id.split('_')[1] in bench_cat:
-
                     # Loop through versions.
                     for version in previous_fim_list:
                         if config == 'DEV':
@@ -307,9 +310,11 @@ def process_alpha_test(args):
 
                             # Define the magnitude lists to use, depending on test_id.
                             if 'ble' in test_id:
-                                magnitude = ['100yr', '500yr']
-                            elif 'usgs' or 'nws' in test_id:
+                                magnitude = BLE_MAGNITUDE_LIST
+                            if 'usgs' or 'nws' in test_id:
                                 magnitude = ['action', 'minor', 'moderate', 'major']
+                            if 'ifc' in test_id:
+                                magnitude = IFC_MAGNITUDE_LIST
                             else:
                                 continue
 
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index 7746725af..b6cfcafcc 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -6,22 +6,45 @@
 OUTPUTS_DIR = os.environ['outputDataDir']
 INPUTS_DIR = r'/data/inputs'
 AHPS_BENCHMARK_CATEGORIES = ['usgs', 'nws']
+FR_BENCHMARK_CATEGORIES = ['ble', 'ifc']
+BLE_MAGNITUDE_LIST = ['100yr', '500yr']
+IFC_MAGNITUDE_LIST = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr', '200yr', '500yr']
 PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
 GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
 GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
 
 # Variables for eval_plots.py
-BAD_SITES = ['grfi2',# Bad crosswalk from USGS to AHPS
-             'ksdm7',# Large areas entirely within levee protected area (CSI of 0)
-             'hohn4',# Mainstems did not extend far enough upstream
-             'rwdn4',# Mainstems did not extend far enough upstream.
-             'efdn7',# Limited upstream mapping
-             'kilo1',# Limited upstream mapping
-             'chin7',# Limited upstream mapping
-             'segt2',# Limited upstream mapping
-             'eagi1',# Missing large portions of FIM
-             'levk1',# Missing large portions of FIM
-             'trbf1']# Limited upstream mapping
+BAD_SITES = [
+            'baki3', #USGS: ratio of evaluated vs domain is very low
+            'cpei3', #USGS: mainstems does not extend sufficiently upstream (~35%), significant masking upstream
+            'eagi1', #NWS:  ratio of evaluated vs domain is very low
+            'efdn7', #NWS:  mainstems does not extend sufficiently upstream (~30%)
+            'erwn6', #NWS:  ratio of evaluated vs domain is very low
+            'grfi2', #USGS: incorrect location
+            'hohn4', #Both: no mainstems in vicinity
+            'kcdm7', #USGS: incorrect location
+            'kilo1', #Both: mainstems does not extend sufficiently upstream (~20%)
+            'ksdm7', #USGS: masked
+            'levk1', #NWS:  Incorrect feature_id assigned from WRDS, this has been corrected
+            'lkcm7', #NWS:  masked 
+            'loun7', #NWS:  benchmark is not consistent between thresholds
+            'lrlm7', #NWS:  masked
+            'mcri2', #USGS: incorrect location
+            'monv1', #NWS:  mainstems does not extend sufficiently upstream (~30%)
+            'mtao1', #USGS: ratio of evaluated vs domain is very low
+            'nhso1', #USGS: mainstems does not extend sufficiently upstream (45%)
+            'nmso1', #Both: mainstems does not extend sufficiently upstream
+            'pori3', #USGS: mainstems does not extend sufficiently upstream
+            'ptvn6', #Both: mainstems does not extend sufficiently upstream (50%)
+            'roun6', #USGS: ratio of evaluated vs domain is very low
+            'rwdn4', #Both: no mainstems in vicinity
+            'selt2', #NWS:  mainstems does not extend sufficiently upstream (~30%)
+            'sweg1', #Both: mainstems does not extend sufficiently upstream (~30%)
+            'vcni3', #USGS: ratio of evaluated vs domain is very low
+            'watw3', #NWS:  mainstems does not extend sufficiently upstream (~30%)
+            'weat2', #NWS:  mainstems does not extend sufficiently upstream (~50%)
+            'wkew3' #NWS:  mainstems does not extend sufficiently upstream (~45%)    
+            ]
 DISCARD_AHPS_QUERY = "not flow.isnull() & masked_perc<97 & not nws_lid in @BAD_SITES"
 
 # Colors.

From bf2c3ecce663ffc2ccf5a22bf23fdc2c02741634 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 9 Jul 2021 09:11:42 -0500
Subject: [PATCH 086/359] Update synthesize_test_cases.py

Bug fix to if/else statement block
---
 tools/synthesize_test_cases.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 01e29be05..983bb501c 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -311,9 +311,9 @@ def process_alpha_test(args):
                             # Define the magnitude lists to use, depending on test_id.
                             if 'ble' in test_id:
                                 magnitude = BLE_MAGNITUDE_LIST
-                            if 'usgs' or 'nws' in test_id:
+                            elif 'usgs' or 'nws' in test_id:
                                 magnitude = ['action', 'minor', 'moderate', 'major']
-                            if 'ifc' in test_id:
+                            elif 'ifc' in test_id:
                                 magnitude = IFC_MAGNITUDE_LIST
                             else:
                                 continue

From 95f1011a14b45aba54c87b4231aa6c4c43345f40 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 9 Jul 2021 09:54:20 -0500
Subject: [PATCH 087/359] Update CHANGELOG.md

---
 CHANGELOG.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ebe838c85..af6f11b61 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.19.3 - 2021-07-09
+
+Hot fix to `synthesize_test_cases`.
+
+## Changes
+- Fixed if/elif/else statement in `synthesize_test_cases.py` that resulted in only IFC data being evaluated.
+
+<br/><br/>
+
 ## v3.0.19.2 - 2021-07-01 - [PR #429](https://github.com/NOAA-OWP/cahaba/pull/429)
 
 Updates to evaluation scripts to allow for Alpha testing at Iowa Flood Center (IFC) sites. Also, `BAD_SITES` variable updates to omit sites not suitable for evaluation from metric calculations.

From cb00fecb0060843fb54d606f3ec394787c5efbc2 Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Tue, 13 Jul 2021 10:01:25 -0500
Subject: [PATCH 088/359] Updating logging and fixing bug in vector
 preprocessing

Updating logging and fixing bug in vector preprocessing.

- fim_completion_check.py adds message to docker log to log any HUCs that were requested but did not finish run_by_unit.sh.
- Adds input_data_edits_changelog.txt to the inputs folder to track any manual or version/location specific changes that were made to data used in FIM 3.
- Provides unique exit codes to relevant domain checkpoints within run_by_unit.sh.
- Bug fixes in reduce_nhd_stream_density.py, 'mprof plot` call.
- Improved error handling in add_crosswalk.py.

This resolves #416 and resolves #427.
---
 CHANGELOG.md                     | 16 +++++++++++++-
 fim_run.sh                       |  7 +++++--
 src/add_crosswalk.py             | 13 +++++++++---
 src/aggregate_vector_inputs.py   | 13 +++++++++---
 src/clip_vectors_to_wbd.py       |  4 ++++
 src/reduce_nhd_stream_density.py | 19 +++++++++--------
 src/run_by_unit.sh               |  8 +++----
 src/split_flows.py               |  3 +++
 src/time_and_tee_run_by_unit.sh  |  5 ++---
 src/utils/shared_variables.py    |  2 +-
 tools/fim_completion_check.py    | 36 ++++++++++++++++++++++++++++++++
 11 files changed, 100 insertions(+), 26 deletions(-)
 create mode 100755 tools/fim_completion_check.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index af6f11b61..6b3cd955c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.19.4 - 2021-07-13 - [PR #431](https://github.com/NOAA-OWP/cahaba/pull/431)
+
+Updating logging and fixing bug in vector preprocessing.
+
+## Additions
+- `fim_completion_check.py` adds message to docker log to log any HUCs that were requested but did not finish `run_by_unit.sh`.
+- Adds `input_data_edits_changelog.txt` to the inputs folder to track any manual or version/location specific changes that were made to data used in FIM 3.
+
+## Changes
+- Provides unique exit codes to relevant domain checkpoints within `run_by_unit.sh`.
+- Bug fixes in `reduce_nhd_stream_density.py`, 'mprof plot` call.
+- Improved error handling in `add_crosswalk.py`.
+
+<br/><br/>
+
 ## v3.0.19.3 - 2021-07-09
 
 Hot fix to `synthesize_test_cases`.
@@ -31,7 +46,6 @@ Adding a thalweg profile tool to identify significant drops in thalweg elevation
 ## Removals
 - Removing `dissolveLinks` arg from `clip_vectors_to_wbd.py`.
 
-
 ## Changes
 - Cleaned up code in `split_flows.py` to make it more readable.
 - Refactored `reduce_nhd_stream_density.py` and `adjust_headwater_streams.py` to limit MS headwater points in `agg_nhd_headwaters_adj.gpkg`.
diff --git a/fim_run.sh b/fim_run.sh
index cf5de36da..b62c20ecb 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -114,8 +114,8 @@ export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
 export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
 export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
 export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
-export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
+export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj_fix.gpkg
+export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj_fix.gpkg
 export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
 ## Input handling ##
 $srcDir/check_huc_inputs.py -u "$hucList"
@@ -144,6 +144,9 @@ else
     fi
 fi
 
+# identify missing HUCs
+time python3 /foss_fim/tools/fim_completion_check.py -i $hucList -o $outputRunDataDir
+
 echo "$viz"
 if [[ "$viz" -eq 1 ]]; then
     # aggregate outputs
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index d92a0dc13..0496b8602 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -8,6 +8,8 @@
 import json
 import argparse
 import sys
+# sys.path.append('/foss_fim/src')
+# sys.path.append('/foss_fim/config')
 from bathy_rc_adjust import bathy_rc_lookup
 from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
@@ -124,7 +126,6 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
     # replace small segment geometry with neighboring stream
     for stream_index in output_flows.index:
-
         if output_flows["areasqkm"][stream_index] < min_catchment_area and output_flows["LengthKm"][stream_index] < min_stream_length and output_flows["LakeID"][stream_index] < 0:
 
             short_id = output_flows['HydroID'][stream_index]
@@ -133,7 +134,10 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
             # multiple upstream segments
             if len(output_flows.loc[output_flows['NextDownID'] == short_id]['HydroID']) > 1:
-                max_order = max(output_flows.loc[output_flows['NextDownID'] == short_id]['order_']) # drainage area would be better than stream order but we would need to calculate
+                try:
+                    max_order = max(output_flows.loc[output_flows['NextDownID'] == short_id]['order_']) # drainage area would be better than stream order but we would need to calculate
+                except:
+                    print(f"short_id {short_id} cannot calculate max stream order for multiple upstream segments scenario")
 
                 if len(output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID']) == 1:
                     update_id = output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID'].item()
@@ -147,7 +151,10 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
             # no upstream segments; multiple downstream segments
             elif len(output_flows.loc[output_flows.From_Node==to_node]['HydroID']) > 1:
-                max_order = max(output_flows.loc[output_flows.From_Node==to_node]['HydroID']['order_']) # drainage area would be better than stream order but we would need to calculate
+                try:
+                    max_order = max(output_flows.loc[output_flows.From_Node==to_node]['HydroID']['order_']) # drainage area would be better than stream order but we would need to calculate
+                except:
+                    print(f"To Node {to_node} cannot calculate max stream order for no upstream segments; multiple downstream segments scenario")
 
                 if len(output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID']) == 1:
                     update_id = output_flows.loc[(output_flows.From_Node==to_node) & (output_flows['order_'] == max_order)]['HydroID'].item()
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
index 3675b88b8..c05e62bb1 100755
--- a/src/aggregate_vector_inputs.py
+++ b/src/aggregate_vector_inputs.py
@@ -195,7 +195,6 @@ def collect_stream_attributes(nhdplus_vectors_dir, huc):
 
         burnline = gpd.read_file(burnline_filename)
         burnline = burnline[['NHDPlusID','ReachCode','geometry']]
-
         flowline = gpd.read_file(flowline_filename)
         flowline = flowline[['NHDPlusID','FType','FCode']]
         # flowline = flowline.loc[flowline["FType"].isin([334,420,428,460,558])]
@@ -212,9 +211,15 @@ def collect_stream_attributes(nhdplus_vectors_dir, huc):
         nhd_streams = nhd_streams.loc[nhd_streams.geometry!=None,:] # special case: remove segments without geometries
         nhd_streams['HUC4'] = str(huc)
 
+        # special case; breach in network at Tiber Dam
+        if huc == '1003' and nhd_streams.loc[nhd_streams.NHDPlusID==23001300078682.0,'DnLevelPat'] == 23001300001574.0:
+            nhd_streams = nhd_streams.loc[nhd_streams.NHDPlusID!=23001300009084.0]
+            nhd_streams.loc[nhd_streams.NHDPlusID==23001300078682.0,'DnLevelPat'] = 23001300001566.0
+
         # Write out NHDPlus HR aggregated
         nhd_streams_agg_fileName = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
         nhd_streams.to_file(nhd_streams_agg_fileName,driver=getDriver(nhd_streams_agg_fileName),index=False)
+
         del nhd_streams
 
         print (f"finished attribute collection for HUC {huc}",flush=True)
@@ -310,6 +315,7 @@ def subset_stream_networks(args, huc):
 
     print(f"finished stream subset for HUC {huc}",flush=True)
 
+
 def aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,huc_list):
 
     for huc in huc_list:
@@ -400,12 +406,13 @@ def clean_up_intermediate_files(nhdplus_vectors_dir):
         if not os.path.isfile(streams_adj_path):
             missing_subsets = missing_subsets + [huc]
 
+
     print (f"Subsetting stream network for {len(missing_subsets)} HUC4s")
     num_workers=11
 
     with ProcessPoolExecutor(max_workers=num_workers) as executor:
         # Preprocess nhd hr and add attributes
-        collect_attributes = [executor.submit(collect_stream_attributes, nhdplus_vectors_dir, str(huc)) for huc in huc_list]
+        # collect_attributes = [executor.submit(collect_stream_attributes, nhdplus_vectors_dir, str(huc)) for huc in huc_list]
         # Subset nhd hr network
         subset_results = [executor.submit(subset_stream_networks, subset_arg_list, str(huc)) for huc in missing_subsets]
 
@@ -413,7 +420,7 @@ def clean_up_intermediate_files(nhdplus_vectors_dir):
 
     # Aggregate subset nhd networks for entire nwm domain
     print ('Aggregating subset NHD networks for entire NWM domain')
-    aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,huc_list)
+    aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,missing_subsets)
 
     # Remove intermediate files
     # clean_up_intermediate_files(nhdplus_vectors_dir)
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 65fd72d20..3242f9d47 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -96,6 +96,10 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
         to_list = nhd_streams.ToNode.to_list()
         missing_segments = list(set(from_list) - set(to_list))
 
+        # special case: stream meanders in and out of WBD buffer boundary
+        if str(hucCode) == '10030203':
+            missing_segments = missing_segments + [23001300001840.0, 23001300016571.0]
+
         # Remove incoming stream segment so it won't be routed as outflow during hydroconditioning
         nhd_streams = nhd_streams.loc[~nhd_streams.FromNode.isin(missing_segments)]
 
diff --git a/src/reduce_nhd_stream_density.py b/src/reduce_nhd_stream_density.py
index c32afd990..afd775d93 100644
--- a/src/reduce_nhd_stream_density.py
+++ b/src/reduce_nhd_stream_density.py
@@ -66,14 +66,13 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
 
                 # Add headwaters_id column
                 streams_subset[id_col] = n
-
                 distance_from_upstream = {}
                 for index, point in headwaters_mask.iterrows():
 
-                    # Convert headwaterpoint geometries to WKB representation
+                    # Convert headwater point geometries to WKB representation
                     wkb_point = dumps(point.geometry)
 
-                    # Create pygeos headwaterpoint geometries from WKB representation
+                    # Create pygeos headwater point geometries from WKB representation
                     pointbin_geom = pygeos.io.from_wkb(wkb_point)
 
                     # Distance to each stream segment
@@ -81,7 +80,6 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
 
                     # Find minimum distance
                     min_index = np.argmin(distances)
-
                     headwater_point_name = point[headwater_id]
 
                     # Find stream segment closest to headwater point
@@ -91,14 +89,12 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
 
                             closest_stream = streams_subset.iloc[min_index]
                             distance_to_line = point.geometry.distance(Point(closest_stream.geometry.coords[-1]))
-
                             print(f"{point.nws_lid} distance on line {closest_stream.NHDPlusID}:  {np.round(distance_to_line,1)}")
-                            if not closest_stream.NHDPlusID in distance_from_upstream.keys():
 
+                            if not closest_stream.NHDPlusID in distance_from_upstream.keys():
                                 distance_from_upstream[closest_stream.NHDPlusID] = [point.nws_lid,distance_to_line]
 
                             elif distance_from_upstream[closest_stream.NHDPlusID][1] > distance_to_line:
-
                                 distance_from_upstream[closest_stream.NHDPlusID] = [point.nws_lid,distance_to_line]
 
                             headwater_point_name = distance_from_upstream[closest_stream.NHDPlusID][0]
@@ -128,12 +124,15 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
 
     if mainstem_flag == False:
         nhd_streams['downstream_of_headwater'] = False
+        nhd_streams['is_relevant_stream'] = nhd_streams['is_headwater'].copy()
     else:
         nwm_intersections = nwm_intersections.loc[nwm_intersections.mainstem==1]
 
     nhd_streams = nhd_streams.explode()
     nhd_streams = nhd_streams.reset_index(drop=True)
 
+
+
     # Find stream segment closest to nwm intersection point
     for index, point in nwm_intersections.iterrows():
 
@@ -153,8 +152,9 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
     # Trace down from headwaters
     nhd_streams.set_index('NHDPlusID',inplace=True,drop=False)
 
-    nhd_streams = get_downstream_segments(nhd_streams,headwater_col,mainstem_flag)
+    nhd_streams = get_downstream_segments(nhd_streams.copy(),headwater_col,mainstem_flag)
 
+    # nhd_streams.fillna(value = {"is_relevant_stream": False}, inplace=True)
     nhd_streams = nhd_streams.loc[nhd_streams['is_relevant_stream'],:]
     nhd_streams.reset_index(drop=True,inplace=True)
 
@@ -187,6 +187,7 @@ def get_downstream_segments(streams, attribute,mainstem_flag):
             relevant_ids = downstream_ids
 
         if mainstem_flag == False:
+
             streams.loc[relevant_ids,'is_relevant_stream'] = True
             streams.loc[relevant_ids,'downstream_of_headwater'] = True
         else:
@@ -196,7 +197,7 @@ def get_downstream_segments(streams, attribute,mainstem_flag):
             if i not in visited:
                 Q.append(i)
 
-    return(streams)
+    return streams
 
 if __name__ == '__main__':
 
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index cab360898..9c399f96d 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -64,7 +64,7 @@ if [ "$extent" = "MS" ]; then
   if [[ ! -f $outputHucDataDir/nhd_headwater_points_subset.gpkg ]] ; then
     echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
     rm -rf $outputHucDataDir
-    exit 0
+    exit 55
   fi
 fi
 
@@ -235,7 +235,7 @@ Tcount
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
   echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
   rm -rf $outputHucDataDir
-  exit 0
+  exit 56
 fi
 
 if [ "$extent" = "MS" ]; then
@@ -249,7 +249,7 @@ if [ "$extent" = "MS" ]; then
   if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
     echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
     rm -rf $outputHucDataDir
-    exit 0
+    exit 57
   fi
 
   dem_thalwegCond=$outputHucDataDir/dem_thalwegCond_MS.tif
@@ -329,7 +329,7 @@ python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py $outp
 if [[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ]] ; then
   echo "No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
   rm -rf $outputHucDataDir
-  exit 0
+  exit 65
 fi
 Tcount
 
diff --git a/src/split_flows.py b/src/split_flows.py
index 9d51065dd..f75405533 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -181,6 +181,9 @@
 else:
     print ('Error: Could not add network attributes to stream segments')
 
+# remove single node segments
+split_flows_gdf = split_flows_gdf.query("From_Node != To_Node")
+
 # Get all vertices
 split_points = OrderedDict()
 for index, segment in split_flows_gdf.iterrows():
diff --git a/src/time_and_tee_run_by_unit.sh b/src/time_and_tee_run_by_unit.sh
index c3c83f426..926762761 100755
--- a/src/time_and_tee_run_by_unit.sh
+++ b/src/time_and_tee_run_by_unit.sh
@@ -1,6 +1,5 @@
 #!/bin/bash -e
 
-mprof run --include-children /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log
-mprof plot -o $outputRunDataDir/logs/$1_memory
+mprof run -o $1.dat --include-children /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log 
+mprof plot -o $outputRunDataDir/logs/$1_memory $1.dat
 exit ${PIPESTATUS[0]}
-
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index fefad3cfa..44bc27f8b 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -46,7 +46,7 @@
 os.environ['nhdplus_vectors_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors')
 os.environ['nwm_dir'] = os.path.join(os.environ.get('input_dir'),'nwm_hydrofabric')
 os.environ['wbd_dir'] = os.path.join(os.environ.get('input_dir'),'wbd')
-os.environ['ahps_dir'] = os.path.join(os.environ.get('input_dir'),'ahp_sites')
+os.environ['ahps_dir'] = os.path.join(os.environ.get('input_dir'),'ahps_sites')
 os.environ['nhdplus_aggregate_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors_aggregate')
 
 # File Paths
diff --git a/tools/fim_completion_check.py b/tools/fim_completion_check.py
new file mode 100755
index 000000000..6623e4f1c
--- /dev/null
+++ b/tools/fim_completion_check.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Checks final FIM outputs to identify missing HUCs')
+    parser.add_argument('-i','--huc-list-dir', help='list of HUCs to run', required=True)
+    parser.add_argument('-o','--output-folder', help='directory of HUCs completed', required=True)
+
+    args = vars(parser.parse_args())
+
+    huc_list_dir = args['huc_list_dir']
+    output_folder = args['output_folder']
+
+    if not os.path.isfile(huc_list_dir):
+        huc_list = huc_list_dir.split()
+    else:
+
+        with open(huc_list_dir) as f:
+            huc_list = f.read().splitlines()
+
+    output_huc_list = os.listdir(output_folder)
+
+    if 'logs' in output_huc_list:
+        output_huc_list.remove('logs')
+
+    if 'aggregate_fim_outputs' in output_huc_list:
+        output_huc_list.remove('aggregate_fim_outputs')
+
+    missing_hucs = list(set(huc_list) - set(output_huc_list))
+
+    if len(missing_hucs) > 0:
+        print (f"MISSING {len(missing_hucs)} HUCS from outputs: {missing_hucs}")

From 493dc273ee42717e05271b7743db4abdfc24f34a Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Mon, 19 Jul 2021 14:46:35 -0500
Subject: [PATCH 089/359] Updating Readme (#435)

---
 CHANGELOG.md |  6 ++++++
 README.md    | 14 ++++++++++----
 fim_run.sh   |  4 ++--
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6b3cd955c..b9371d15e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.19.5 - 2021-07-19
+
+Updating `README.md`.
+
+<br/><br/>
+
 ## v3.0.19.4 - 2021-07-13 - [PR #431](https://github.com/NOAA-OWP/cahaba/pull/431)
 
 Updating logging and fixing bug in vector preprocessing.
diff --git a/README.md b/README.md
index 8bbeaeb96..e9953112c 100644
--- a/README.md
+++ b/README.md
@@ -37,11 +37,11 @@ The following input data sources should be downloaded and preprocessed prior to
 - Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
 
 ### NHDPlus HR datasets
-- `acquire_and_preprocess_inputs.py`
-- `preprocess_rasters.py`
-- `aggregate_nhd_hr_streams.py`
+- `agg_nhd_streams_adj.gpkg`
+- `agg_nhd_headwaters_adj.gpkg`
+- `elev_m.tif`
 
-**Please note:** For the following two datasets, please contact Mark Glaudemans (mark.glaudemans@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
+**Please note:** For the following datasets, please contact Mark Glaudemans (mark.glaudemans@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
 
 ### NWM Hydrofabric
 - `nwm_flows.gpkg`
@@ -73,6 +73,12 @@ docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <i
 ```
 /foss_fim/src/aggregate_vector_inputs.py
 ```
+
+### Reproject NHDHR Rasters and Convert to Meters.
+```
+/foss_fim/src/preprocess_rasters.py
+```
+
 ### Produce Hydrofabric
 ```
 fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
diff --git a/fim_run.sh b/fim_run.sh
index b62c20ecb..c3700ba30 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -114,8 +114,8 @@ export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
 export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
 export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
 export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj_fix.gpkg
-export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj_fix.gpkg
+export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
+export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
 export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
 ## Input handling ##
 $srcDir/check_huc_inputs.py -u "$hucList"

From f67a35dfe678dd0056d6139acb2b46d5931a5cb3 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 11 Aug 2021 11:44:40 -0500
Subject: [PATCH 090/359] Adds two QAQC scripts to tools directory

This merge adds two new scripts into /tools/ for use in QAQC.

Adds inundate_nation.py to produce inundation maps for the entire country for use in QAQC.
Adds check_deep_flooding.py to check for depths of inundation greater than a user-supplied threshold at specific areas defined by a user-supplied shapefile.

This resolves #439 and resolves #438.
---
 CHANGELOG.md                          |  10 +++
 tools/check_deep_flooding.py          | 110 ++++++++++++++++++++++++++
 tools/inundate_nation.py              |  97 +++++++++++++++++++++++
 tools/inundation_wrapper_nwm_flows.py |  14 ++--
 4 files changed, 226 insertions(+), 5 deletions(-)
 create mode 100644 tools/check_deep_flooding.py
 create mode 100644 tools/inundate_nation.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b9371d15e..f50953dc9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.20.0 - 2021-08-11 - [PR #440](https://github.com/NOAA-OWP/cahaba/pull/440)
+
+This merge adds two new scripts into `/tools/` for use in QAQC.
+
+## Additions
+- `inundate_nation.py` to produce inundation maps for the entire country for use in QAQC.
+- `check_deep_flooding.py` to check for depths of inundation greater than a user-supplied threshold at specific areas defined by a user-supplied shapefile.
+
+<br/><br/>
+
 ## v3.0.19.5 - 2021-07-19
 
 Updating `README.md`.
diff --git a/tools/check_deep_flooding.py b/tools/check_deep_flooding.py
new file mode 100644
index 000000000..e9bfd746c
--- /dev/null
+++ b/tools/check_deep_flooding.py
@@ -0,0 +1,110 @@
+import argparse
+import os
+from multiprocessing import Pool
+import numpy as np
+import rasterio.shutil
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+import rasterio.crs
+import rasterio
+import rasterio.mask
+import geopandas as gpd
+from shapely.geometry import box
+
+
+def check_deep_flooding(args):
+    
+    depth_grid_path = args[0]
+    shapefile_path = args[1]
+    depth_threshold = args[2]
+    output_dir = args[3]
+    
+    print("Checking " + depth_grid_path + "...")
+    
+    # Open depth_grid_path and shapefile_path and perform np.wheres
+    depth_src = rasterio.open(depth_grid_path)
+    depth_array = depth_src.read(1)
+    reference = depth_src
+    
+    #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).
+    bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
+    poly_all = gpd.read_file(shapefile_path, bbox = bounding_box)
+
+    # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
+    if poly_all.empty:
+        return
+
+    #Project layer to reference crs.
+    poly_all_proj = poly_all.to_crs(reference.crs)
+    # check if there are any lakes within our reference raster extent.
+    if poly_all_proj.empty:
+        #If no features within reference raster extent, create a zero array of same shape as reference raster.
+        poly_mask = np.zeros(reference.shape)
+    else:
+        #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
+        geometry = poly_all_proj.geometry
+        in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
+        #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
+        poly_mask = np.where(in_poly == True, 1,0)
+
+        # Filter depth_array by depth_threshold
+        filtered_depth_array = np.where(depth_array > depth_threshold, depth_array, -1)
+
+        # Perform mask.
+        masked_depth_array = np.where(poly_mask == 1, filtered_depth_array, -1)
+
+        if np.amax(masked_depth_array) > 0:
+
+            file_handle = os.path.split(depth_grid_path)[1]
+    
+            checked_depth_raster = os.path.join(output_dir, "checked_" + str(depth_threshold) + "_" + file_handle)
+    
+            print("Writing " + checked_depth_raster + "...")
+            # Write output.
+            with rasterio.Env():
+                profile = depth_src.profile
+                profile.update(nodata=-1)
+                with rasterio.open(checked_depth_raster, 'w', **profile) as dst:
+                    dst.write(masked_depth_array, 1)
+
+
+if __name__ == '__main__':
+    
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Checks for deep flooding in a specified shapefile. Requires a directory of depth grids and a shapefile.')
+    parser.add_argument('-d','--depth-grid-dir',help='Name of directory containing outputs of depth outputs of inundation.py',required=True)
+    parser.add_argument('-s','--shapefile-path',help='Path to shapefile to be used as the overlay.',required=True)
+    parser.add_argument('-t','--depth-threshold',help='Depth in meters to use as checking threshold.',required=True)
+    parser.add_argument('-o', '--output-dir',help='The path to a directory to write the outputs. If not used, the inundation_review directory is used by default -> type=str',required=True, default="")
+    parser.add_argument('-j', '--job-number',help='The number of jobs',required=False,default=1)
+        
+    args = vars(parser.parse_args())
+    
+    depth_grid_dir = args['depth_grid_dir']
+    shapefile_path = args['shapefile_path']
+    depth_threshold = int(args['depth_threshold'])
+    output_dir = args['output_dir']
+    job_number = int(args['job_number'])
+    
+    # Get list of files in depth_grid_dir.
+    # Loop through files and determine which ones are depth grids, adding them to a list.
+    
+    depth_grid_dir_list = os.listdir(depth_grid_dir)
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+    
+    procs_list = []
+    
+    for f in depth_grid_dir_list:
+        if 'depth' in f:
+            full_f_path = os.path.join(depth_grid_dir, f)
+            
+#            check_deep_flooding([full_f_path, shapefile_path, depth_threshold, output_dir])
+            procs_list.append([full_f_path, shapefile_path, depth_threshold, output_dir])
+            
+    # Multiprocess.
+    with Pool(processes=job_number) as pool:
+        pool.map(check_deep_flooding, procs_list)
+
+            
+            
+            
\ No newline at end of file
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
new file mode 100644
index 000000000..68ce9895d
--- /dev/null
+++ b/tools/inundate_nation.py
@@ -0,0 +1,97 @@
+import argparse
+import os
+
+from inundation import inundate
+from multiprocessing import Pool
+
+INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
+INPUTS_DIR = r'/data/inputs'
+
+
+def run_inundation(args):
+    
+    fim_run_dir = args[0]
+    huc = args[1]
+    magnitude = args[2]
+    magnitude_output_dir = args[3]
+    config = args[4]
+    
+    fim_run_parent = os.path.join(fim_run_dir, huc)
+    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
+    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+    mask_type = 'huc'
+    catchment_poly = ''
+    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
+    catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+    inundation_raster = os.path.join(magnitude_output_dir, magnitude + '_' + config + '_inund_extent.tif')
+    depth_raster = os.path.join(magnitude_output_dir, magnitude + '_' + config + '_inund_depth.tif')
+    forecast = os.path.join(INUN_REVIEW_DIR, 'nwm_recurr_flow_data', 'recurr_' + magnitude + '_cms.csv')
+    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+
+    print(depth_raster)
+
+    if not os.path.exists(depth_raster):
+        print("Running the NWM recurrence intervals for HUC: " + huc + ", " + magnitude + "...")
+        inundate(
+                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
+                 depths=depth_raster,out_raster_profile=None,out_vector_profile=None,quiet=True
+                )
+    else:
+        print(inundation_raster + " exists. Moving on...")
+        
+
+if __name__ == '__main__':
+    
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow recurrence interflow data. Inundation outputs are stored in the /inundation_review/inundation_nwm_recurr/ directory.')
+    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)',required=True)
+    parser.add_argument('-o', '--output-dir',help='The path to a directory to write the outputs. If not used, the inundation_review directory is used by default -> type=str',required=False, default="")
+    parser.add_argument('-j', '--job-number',help='The number of jobs',required=False,default=1)
+        
+    args = vars(parser.parse_args())
+
+    fim_run_dir = args['fim_run_dir']
+    output_dir = args['output_dir']
+    magnitude_list = ['1_5']
+    
+    job_number = int(args['job_number'])
+
+    huc_list = os.listdir(fim_run_dir)
+        
+    print(fim_run_dir)
+    fim_version = os.path.split(fim_run_dir)[1]
+    
+    print(fim_version)
+    
+    print(output_dir)
+    
+    if output_dir == "":
+        output_dir = os.path.join(INUN_REVIEW_DIR, fim_version)
+        
+    print(output_dir)
+    
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+    
+    if 'ms' in fim_version:
+        config = 'ms'
+    if 'fr' in fim_version:
+        config = 'fr'
+        
+    procs_list = []
+    
+    for huc in huc_list:
+        if huc != 'logs':
+            for magnitude in magnitude_list:
+                magnitude_output_dir = os.path.join(output_dir, magnitude + '_' + config)
+                if not os.path.exists(magnitude_output_dir):
+                    os.mkdir(magnitude_output_dir)
+                    print(magnitude_output_dir)
+                procs_list.append([fim_run_dir, huc, magnitude, magnitude_output_dir, config])
+            
+    # Multiprocess.
+    if job_number > 1:
+        with Pool(processes=job_number) as pool:
+            pool.map(run_inundation, procs_list)
+
diff --git a/tools/inundation_wrapper_nwm_flows.py b/tools/inundation_wrapper_nwm_flows.py
index f2b641e83..8af5462f4 100755
--- a/tools/inundation_wrapper_nwm_flows.py
+++ b/tools/inundation_wrapper_nwm_flows.py
@@ -12,7 +12,7 @@
 import shutil
 from inundation import inundate
 
-TEST_CASES_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
+INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
 INPUTS_DIR = r'/data/inputs'
 OUTPUTS_DIR = os.environ['outputDataDir']
 
@@ -24,13 +24,17 @@
 WHITE_BOLD = '\033[37;1m'
 CYAN_BOLD = '\033[36;1m'
 
-def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'):
+def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc', output_dir=None):
 
     # Construct paths to development test results if not existent.
-    huc_id_dir_parent = os.path.join(TEST_CASES_DIR, huc_id)
+    huc_id_dir_parent = os.path.join(INUN_REVIEW_DIR, huc_id)
     if not os.path.exists(huc_id_dir_parent):
         os.mkdir(huc_id_dir_parent)
-    branch_test_case_dir_parent = os.path.join(TEST_CASES_DIR, huc_id, branch_name)
+        
+    if output_dir == None:
+        branch_test_case_dir_parent = os.path.join(INUN_REVIEW_DIR, huc_id, branch_name)
+    else:
+        branch_test_case_dir_parent = os.path.join(output_dir, huc_id, branch_name)
 
     # Delete the entire directory if it already exists.
     if os.path.exists(branch_test_case_dir_parent):
@@ -77,7 +81,7 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'
 
         # Define paths to inundation_raster and forecast file.
         inundation_raster = os.path.join(branch_test_case_dir, branch_name + '_inund_extent.tif')
-        forecast = os.path.join(TEST_CASES_DIR, 'nwm_recurr_flow_data', 'recurr_' + magnitude + '_cms.csv')
+        forecast = os.path.join(INUN_REVIEW_DIR, 'nwm_recurr_flow_data', 'recurr_' + magnitude + '_cms.csv')
 
         # Run inundate.
         print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")

From 40a5a4c3de8674a30b7e2ff2d5f9ee88486b91d4 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 12 Aug 2021 16:31:13 -0500
Subject: [PATCH 091/359] Documentation of various scripts

This merge better documents the following scripts:

- inundate_nation.py
- synthesize_test_cases.py
- adjust_thalweg_lateral.py
- rem.py

This resolves #441.
---
 CHANGELOG.md                   | 14 +++++++++++++
 src/adjust_thalweg_lateral.py  | 26 +++++++++--------------
 src/rem.py                     | 38 +++++++++++++++++-----------------
 tools/inundate_nation.py       | 31 +++++++++++++++------------
 tools/synthesize_test_cases.py | 24 +++++++++++++++++++--
 5 files changed, 83 insertions(+), 50 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f50953dc9..ee13820c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,20 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.20.1 - 2021-08-12 - [PR #442](https://github.com/NOAA-OWP/cahaba/pull/442)
+
+This merge improves documentation in various scripts.
+
+## Changes
+This PR better documents the following:
+
+- `inundate_nation.py`
+- `synthesize_test_cases.py`
+- `adjust_thalweg_lateral.py`
+- `rem.py`
+
+<br/><br/>
+
 ## v3.0.20.0 - 2021-08-11 - [PR #440](https://github.com/NOAA-OWP/cahaba/pull/440)
 
 This merge adds two new scripts into `/tools/` for use in QAQC.
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index 47cd2e209..b4bbd8eb2 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -11,7 +11,6 @@ def adjust_thalweg_laterally(elevation_raster, stream_raster, allocation_raster,
 
     # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
     # The following algorithm searches for the zonal minimum elevation in each pixel catchment
-    # It updates the catchment_min_dict with this zonal minimum elevation value.
     @njit
     def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, cost_tolerance, ndv):
         for i,elev_m in enumerate(zone_window):
@@ -31,7 +30,7 @@ def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window
 
         return(zone_min_dict)
 
-    # Open the masked gw_catchments_pixels_masked and dem_thalwegCond_masked.
+    # Open files.
     elevation_raster_object = rasterio.open(elevation_raster)
     allocation_zone_raster_object = rasterio.open(allocation_raster)
     cost_distance_raster_object = rasterio.open(cost_distance_raster)
@@ -40,14 +39,12 @@ def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window
     meta['tiled'], meta['compress'] = True, 'lzw'
 
     # -- Create zone_min_dict -- #
-    print("Create zone_min_dict")
-    zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums.
+    zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums
     # Update catchment_min_dict with pixel sheds minimum.
-
-    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template.
-        elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window.
-        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window.
-        cost_window = cost_distance_raster_object.read(1, window=window).ravel()  # Define cost_window.
+    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template
+        elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window
+        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window
+        cost_window = cost_distance_raster_object.read(1, window=window).ravel()  # Define cost_window
 
         # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
         zone_min_dict = make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, int(cost_distance_tolerance), meta['nodata'])
@@ -67,7 +64,6 @@ def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_w
         dem_window_to_return = np.empty_like (dem_window)
         dem_window_to_return[:] = dem_window
 
-
         for i,elev_m in enumerate(zone_window):
             i = int(i)
             elev_m = int(elev_m)
@@ -89,22 +85,20 @@ def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_w
     allocation_zone_raster_object = rasterio.open(allocation_raster)
     thalweg_object = rasterio.open(stream_raster)
 
-
     dem_lateral_thalweg_adj_object = rasterio.open(dem_lateral_thalweg_adj, 'w', **meta)
 
-    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
-        dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window.
+    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
+        dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window
         window_shape = dem_window.shape
         dem_window = dem_window.ravel()
 
-        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window.
-        thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window.
+        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window
+        thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window
 
         # Call numba-optimized function to reassign thalweg cell values to catchment minimum value.
         minimized_dem_window = minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window)
         minimized_dem_window = minimized_dem_window.reshape(window_shape).astype(np.float32)
 
-
         dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)
 
     elevation_raster_object.close()
diff --git a/src/rem.py b/src/rem.py
index 43c75b0f0..9e0c78361 100755
--- a/src/rem.py
+++ b/src/rem.py
@@ -10,6 +10,7 @@
 import geopandas as gpd
 from utils.shared_functions import getDriver
 
+
 @profile
 def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, dem_reaches_filename):
     """
@@ -41,7 +42,7 @@ def make_catchment_hydroid_dict(flat_value_raster, catchment_hydroid_dict, flat_
                 catchment_hydroid_dict[cm] = flat_value_raster[i]
         return(catchment_hydroid_dict)
 
-    # Open the masked gw_catchments_pixels_masked, hydroid_raster, and dem_thalwegCond_masked.
+    # Open files.
     gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
     hydroid_pixels_object = rasterio.open(hydroid_fileName)
     thalweg_raster_object = rasterio.open(thalweg_raster)
@@ -54,10 +55,10 @@ def make_catchment_hydroid_dict(flat_value_raster, catchment_hydroid_dict, flat_
     catchment_hydroid_dict = typed.Dict.empty(types.int64,types.int64)  # Initialize an empty dictionary to store the catchment hydroid.
     # Update catchment_hydroid_dict with each pixel sheds hydroid.
     # Creating dictionary containing catchment ids (key) and corresponding hydroid within the thalweg...
-    for ji, window in hydroid_pixels_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
-        hydroid_window = hydroid_pixels_object.read(1,window=window).ravel()  # Define hydroid_window.
-        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window.
-        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window.
+    for ji, window in hydroid_pixels_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
+        hydroid_window = hydroid_pixels_object.read(1,window=window).ravel()  # Define hydroid_window
+        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window
+        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window
 
         # Call numba-optimized function to update catchment_hydroid_dict with pixel sheds overlapping hydroid.
         catchment_hydroid_dict = make_catchment_hydroid_dict(hydroid_window, catchment_hydroid_dict, catchments_window, thalweg_window)
@@ -71,17 +72,17 @@ def make_catchment_hydroid_dict(flat_value_raster, catchment_hydroid_dict, flat_
     def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalweg_window):
 
         for i,cm in enumerate(flat_catchments):
-            if thalweg_window[i] == 1:  # Only allow reference elevation to be within thalweg.
-                # If the catchment really exists in the dictionary, compare elevation values.
+            if thalweg_window[i] == 1:  # Only allow reference elevation to be within thalweg
+                # If the catchment really exists in the dictionary, compare elevation values
                 if (cm in catchment_min_dict):
                     if (flat_dem[i] < catchment_min_dict[cm]):
-                        # If the flat_dem's elevation value is less than the catchment_min_dict min, update the catchment_min_dict min.
+                        # If the flat_dem's elevation value is less than the catchment_min_dict min, update the catchment_min_dict min
                         catchment_min_dict[cm] = flat_dem[i]
                 else:
                     catchment_min_dict[cm] = flat_dem[i]
         return(catchment_min_dict)
 
-    # Open the masked gw_catchments_pixels_masked and dem_thalwegCond_masked.
+    # Open files.
     gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
     dem_thalwegCond_masked_object = rasterio.open(dem_fileName)
     thalweg_raster_object = rasterio.open(thalweg_raster)
@@ -91,13 +92,13 @@ def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalw
     meta['tiled'], meta['compress'] = True, 'lzw'
 
     # -- Create catchment_min_dict -- #
-    catchment_min_dict = typed.Dict.empty(types.int64,types.float32)  # Initialize an empty dictionary to store the catchment minimums.
+    catchment_min_dict = typed.Dict.empty(types.int64,types.float32)  # Initialize an empty dictionary to store the catchment minimums
     # Update catchment_min_dict with pixel sheds minimum.
     # Creating dictionary containing catchment ids (key) and corresponding elevation within the thalweg (value)...
-    for ji, window in dem_thalwegCond_masked_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
-        dem_window = dem_thalwegCond_masked_object.read(1,window=window).ravel()  # Define dem_window.
-        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window.
-        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window.
+    for ji, window in dem_thalwegCond_masked_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
+        dem_window = dem_thalwegCond_masked_object.read(1,window=window).ravel()  # Define dem_window
+        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window
+        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define thalweg_window
 
         # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
         catchment_min_dict = make_catchment_min_dict(dem_window, catchment_min_dict, catchments_window, thalweg_window)
@@ -106,7 +107,7 @@ def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalw
     gw_catchments_pixels_masked_object.close()
     thalweg_raster_object.close()
 
-    # Merge and export dictionary to to_csv
+    # Merge and export dictionary to csv.
     catchment_min_dict_df = pd.DataFrame.from_dict(catchment_min_dict, orient='index') # convert dict to dataframe
     catchment_min_dict_df.columns = ['Median_Thal_Elev_m']
     catchment_hydroid_dict_df = pd.DataFrame.from_dict(catchment_hydroid_dict, orient='index') # convert dict to dataframe
@@ -167,7 +168,7 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
 
 if __name__ == '__main__':
 
-    # parse arguments
+    # Parse arguments.
     parser = argparse.ArgumentParser(description='Relative elevation from pixel based watersheds')
     parser.add_argument('-d','--dem', help='DEM to use within project path', required=True)
     parser.add_argument('-w','--watersheds',help='Pixel based watersheds raster to use within project path',required=True)
@@ -176,11 +177,10 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
     parser.add_argument('-i','--hydroid', help='HydroID raster to use within project path', required=True)
     parser.add_argument('-s','--dem_reaches_in_out',help='DEM derived reach layer to join HAND reference elevation attribute',required=True)
 
-
-    # extract to dictionary
+    # Extract arguments to dictionary.
     args = vars(parser.parse_args())
 
-    # rename variable inputs
+    # Define variable inputs.
     dem_fileName = args['dem']
     pixel_watersheds_fileName = args['watersheds']
     rem_fileName = args['rem']
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index 68ce9895d..6fbcdf146 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -4,11 +4,18 @@
 from inundation import inundate
 from multiprocessing import Pool
 
-INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
+INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
 INPUTS_DIR = r'/data/inputs'
 
 
 def run_inundation(args):
+    """
+    This script is basically a wrapper for the inundate function and is designed for multiprocessing.
+    
+    Args:
+        args (list): [fim_run_dir (str), huc (str), magnitude (str), magnitude_output_dir (str), config (str)]
+    
+    """
     
     fim_run_dir = args[0]
     huc = args[1]
@@ -16,6 +23,7 @@ def run_inundation(args):
     magnitude_output_dir = args[3]
     config = args[4]
     
+    # Define file paths for use in inundate().
     fim_run_parent = os.path.join(fim_run_dir, huc)
     rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
     catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
@@ -28,8 +36,7 @@ def run_inundation(args):
     forecast = os.path.join(INUN_REVIEW_DIR, 'nwm_recurr_flow_data', 'recurr_' + magnitude + '_cms.csv')
     hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
 
-    print(depth_raster)
-
+    # Run inundate() once for depth and once for extent.
     if not os.path.exists(depth_raster):
         print("Running the NWM recurrence intervals for HUC: " + huc + ", " + magnitude + "...")
         inundate(
@@ -37,8 +44,13 @@ def run_inundation(args):
                  subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
                  depths=depth_raster,out_raster_profile=None,out_vector_profile=None,quiet=True
                 )
-    else:
-        print(inundation_raster + " exists. Moving on...")
+        
+    if not os.path.exists(inundation_raster):
+        inundate(
+                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
+                 depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
+                )
         
 
 if __name__ == '__main__':
@@ -59,18 +71,11 @@ def run_inundation(args):
 
     huc_list = os.listdir(fim_run_dir)
         
-    print(fim_run_dir)
     fim_version = os.path.split(fim_run_dir)[1]
     
-    print(fim_version)
-    
-    print(output_dir)
-    
     if output_dir == "":
         output_dir = os.path.join(INUN_REVIEW_DIR, fim_version)
-        
-    print(output_dir)
-    
+            
     if not os.path.exists(output_dir):
         os.mkdir(output_dir)
     
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 983bb501c..98a42e9c6 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -11,7 +11,19 @@
 
 
 def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list):
-
+    """
+    This function searches for and collates metrics into a single CSV file that can queried database-style. The
+    CSV is an input to eval_plots.py. This function automatically looks for metrics produced for official versions
+    and loads them into memory to be written to the output CSV.
+    
+    Args:
+        master_metrics_csv_output (str): Full path to CSV output. If a file already exists at this path, it will be overwritten.
+        dev_versions_to_include_list (list): A list of non-official FIM version names. If a user supplied information on the command
+                                            line using the -dc flag, then this function will search for metrics in the "testing_versions"
+                                            library of metrics and include them in the CSV output.
+    
+    """
+    
     # Construct header
     metrics_to_write = ['true_negatives_count',
                         'false_negatives_count',
@@ -199,7 +211,15 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
 
 def process_alpha_test(args):
-
+    """
+    This function is designed to be used in multiprocessing. It handles the calling of the run_alpha_test function.
+    
+    Args:
+        args (list): Formatted [fim_run_dir (str), version (str), test_id (str), magnitude (str), archive_results (bool), overwrite (bool)]
+    
+    """
+    
+    
     fim_run_dir = args[0]
     version = args[1]
     test_id = args[2]

From 7aa4ab9f11e76bacc995d5b27914378ca29fcfed Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Wed, 18 Aug 2021 10:34:07 -0500
Subject: [PATCH 092/359] Adding check for NWM stream segments within HUC
 boundaries

This merge modifies clip_vectors_to_wbd.py to check for relevant input data.

- clip_vectors_to_wbd.py now checks that there are NWM stream segments within the buffered HUC boundary.
- included_huc8_ms.lst has several additional HUC8s.
---
 CHANGELOG.md               | 10 ++++++++++
 src/clip_vectors_to_wbd.py |  6 +++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ee13820c2..3a925ce7a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.20.2 - 2021-08-13 - [PR #443](https://github.com/NOAA-OWP/cahaba/pull/443)
+
+This merge modifies `clip_vectors_to_wbd.py` to check for relevant input data.
+
+## Changes
+- `clip_vectors_to_wbd.py` now checks that there are NWM stream segments within the buffered HUC boundary.
+- `included_huc8_ms.lst` has several additional HUC8s.
+
+<br/><br/>
+
 ## v3.0.20.1 - 2021-08-12 - [PR #442](https://github.com/NOAA-OWP/cahaba/pull/442)
 
 This merge improves documentation in various scripts.
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 3242f9d47..b1c9c542e 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -127,7 +127,11 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd_buffer)
     if extent == 'MS':
         nwm_streams = nwm_streams.loc[nwm_streams.mainstem==1]
-    nwm_streams.to_file(subset_nwm_streams_filename,driver=getDriver(subset_nwm_streams_filename),index=False)
+    if len(nwm_streams) > 0:
+        nwm_streams.to_file(subset_nwm_streams_filename,driver=getDriver(subset_nwm_streams_filename),index=False)
+    else:
+        print ("No NWM stream segments within HUC " + str(hucCode) +  " boundaries.")
+        sys.exit(0)
     del nwm_streams
 
 if __name__ == '__main__':

From 8949dc319b96b24af1a4415c19cb8984f5043798 Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Wed, 18 Aug 2021 11:52:05 -0500
Subject: [PATCH 093/359] Cahaba Codebase Cleanup - Small Items

General repository cleanup, made memory-profiling an optional flag, API's release feature now saves outputs.

- Remove Dockerfile.prod, rename Dockerfile.dev to just Dockerfile, and remove ``.dockerignore`.
- Clean up Dockerfile and remove any unused* packages or variables.
- Remove any unused* Python packages from the Pipfile.
- Move the CHANGELOG.md, SECURITY.md, and TERMS.md files to the /docs folder.
- Remove any unused* scripts in the /tools and /src folders.
- Move tools/preprocess scripts into tools/.
- Ensure all scripts in the /src folder have their code in functions and are being called via a __main__ function (This will help with implementing memory profiling fully).
- Changed memory-profiling to be an option flag -m for fim_run.sh.
- Updated FIM API to save all outputs during a "release" job.

This resolves #432, resolves #426, and resolves #434.
---
 .dockerignore                                 |   1 -
 Dockerfile.dev => Dockerfile                  |   7 +-
 Dockerfile.prod                               | 100 ----
 Pipfile.lock                                  | 182 +++----
 README.md                                     |   2 +-
 api/node/updater/updater.py                   |  32 +-
 config/params_template.env                    |   2 +-
 CHANGELOG.md => docs/CHANGELOG.md             |  17 +
 SECURITY.md => docs/SECURITY.md               |   0
 TERMS.md => docs/TERMS.md                     |   0
 fim_run.sh                                    |   5 +
 src/acquire_and_preprocess_inputs.py          |  14 +-
 src/add_crosswalk.py                          |   6 +-
 src/adjust_headwater_streams.py               |   3 +
 src/adjust_thalweg_lateral.py                 |   6 +-
 src/aggregate_fim_outputs.py                  |   3 +-
 src/aggregate_vector_inputs.py                |  27 +-
 src/agreedem.py                               |   4 +-
 src/bathy_rc_adjust.py                        |  13 +-
 src/build_stream_traversal.py                 |   6 +-
 src/burn_in_levees.py                         |   5 +-
 src/check_huc_inputs.py                       |   2 +
 src/clip_vectors_to_wbd.py                    |   9 +-
 src/derive_headwaters.py                      |   2 +
 src/filter_catchments_and_add_attributes.py   |  84 ++--
 src/fr_to_ms_raster_mask.py                   | 159 +++---
 src/getRasterInfoNative.py                    |   4 +
 src/get_all_huc_in_inputs.py                  |   2 +-
 src/make_stages_and_catchlist.py              |  65 +--
 src/output_cleanup.py                         |   9 +-
 src/preprocess_rasters.py                     |   1 -
 src/r_grow_distance.py                        |   2 +
 src/reachID_grid_to_vector_points.py          |   9 +-
 src/reduce_nhd_stream_density.py              |   2 +
 src/rem.py                                    |   8 +-
 src/run_by_unit.sh                            |  10 +-
 src/split_flows.py                            | 314 ++++++------
 src/time_and_tee_run_by_unit.sh               |   9 +-
 src/unique_pixel_and_allocation.py            |  26 +-
 src/usgs_gage_crosswalk.py                    |   7 +-
 src/utils/shared_functions.py                 |   9 +
 .../create_flow_forecast_file.py              |   0
 tools/generate_categorical_fim.py             |   6 +-
 tools/generate_categorical_fim_flows.py       |   9 +-
 tools/generate_nws_lid.py                     |   3 -
 tools/inundation.py                           |   1 -
 tools/mannings_calibration_run.sh             | 123 -----
 tools/mannings_run_by_set.sh                  |  15 -
 tools/preprocess_ahps_nws.py                  |   6 +-
 tools/preprocess_ahps_usgs.py                 |   4 -
 .../{preprocess => }/preprocess_benchmark.py  |   0
 tools/{preprocess => }/preprocess_fimx.py     |   0
 tools/rating_curve_comparison.py              |   3 +-
 tools/rating_curve_get_usgs_curves.py         |   2 +-
 tools/run_test_case_calibration.py            | 456 ------------------
 tools/shapefile_to_raster.py                  |  41 --
 tools/thalweg_drop_check.py                   |   2 -
 tools/time_and_tee_mannings_calibration.sh    |   4 -
 tools/tools_shared_functions.py               | 126 +----
 59 files changed, 598 insertions(+), 1371 deletions(-)
 delete mode 100644 .dockerignore
 rename Dockerfile.dev => Dockerfile (96%)
 delete mode 100644 Dockerfile.prod
 rename CHANGELOG.md => docs/CHANGELOG.md (97%)
 rename SECURITY.md => docs/SECURITY.md (100%)
 rename TERMS.md => docs/TERMS.md (100%)
 rename tools/{preprocess => }/create_flow_forecast_file.py (100%)
 delete mode 100755 tools/mannings_calibration_run.sh
 delete mode 100755 tools/mannings_run_by_set.sh
 rename tools/{preprocess => }/preprocess_benchmark.py (100%)
 rename tools/{preprocess => }/preprocess_fimx.py (100%)
 delete mode 100755 tools/run_test_case_calibration.py
 delete mode 100755 tools/shapefile_to_raster.py
 delete mode 100755 tools/time_and_tee_mannings_calibration.sh

diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index 8b1378917..000000000
--- a/.dockerignore
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/Dockerfile.dev b/Dockerfile
similarity index 96%
rename from Dockerfile.dev
rename to Dockerfile
index aa646a3f1..ef56576b8 100644
--- a/Dockerfile.dev
+++ b/Dockerfile
@@ -6,17 +6,16 @@ ARG projectDir=/foss_fim
 ARG depDir=/dependencies
 ARG taudemVersion=98137bb6541a0d0077a9c95becfed4e56d0aa0ac
 ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
-ENV DEBIAN_FRONTEND noninteractive
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
-RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/* 
+RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/*
 
 RUN git clone https://github.com/dtarb/taudem.git
 RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
 
 RUN apt-get update --fix-missing && apt-get install -y cmake mpich \
-    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/* 
+    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/*
 
 ## Compile Main taudem repo ##
 RUN mkdir -p taudem/bin
@@ -53,7 +52,7 @@ ARG dataDir=/data
 ARG projectDir=/foss_fim
 ARG depDir=/dependencies
 ENV inputDataDir=$dataDir/inputs
-ENV outputDataDir=$dataDir/outputs 
+ENV outputDataDir=$dataDir/outputs
 ENV srcDir=$projectDir/src
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
diff --git a/Dockerfile.prod b/Dockerfile.prod
deleted file mode 100644
index 23d0980ec..000000000
--- a/Dockerfile.prod
+++ /dev/null
@@ -1,100 +0,0 @@
-## Temporary image to build the libraries and only save the needed artifacts
-FROM osgeo/gdal:ubuntu-full-3.1.2 AS builder
-WORKDIR /opt/builder
-ARG dataDir=/data
-ARG projectDir=/foss_fim
-ARG depDir=/dependencies
-ARG taudemVersion=bf9417172225a9ce2462f11138c72c569c253a1a
-ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
-ENV DEBIAN_FRONTEND noninteractive
-ENV taudemDir=$depDir/taudem/bin
-ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
-
-RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/* 
-
-RUN git clone https://github.com/dtarb/taudem.git
-RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
-
-RUN apt-get update && apt-get install -y cmake mpich \
-    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/* 
-
-## Compile Main taudem repo ##
-RUN mkdir -p taudem/bin
-RUN cd taudem \
-    && git checkout $taudemVersion \
-    && cd src \
-    && make
-
-## Compile taudem repo with accelerated flow directions ##
-RUN cd taudem_accelerated_flowDirections/taudem \
-    && git checkout $taudemVersion2 \
-    && mkdir build \
-    && cd build \
-    && cmake .. \
-    && make
-
-RUN mkdir -p $taudemDir
-RUN mkdir -p $taudemDir2
-
-## Move needed binaries to the next stage of the image
-RUN cd taudem/bin && mv -t $taudemDir flowdircond aread8 threshold streamnet gagewatershed catchhydrogeo dinfdistdown
-RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d8flowdir dinfflowdir
-
-
-
-
-###############################################################################################
-
-
-
-# Base Image that has GDAL, PROJ, etc
-FROM osgeo/gdal:ubuntu-full-3.1.2
-ARG dataDir=/data
-ARG projectDir=/foss_fim
-ARG depDir=/dependencies
-ENV inputDataDir=$dataDir/inputs
-ENV outputDataDir=$dataDir/outputs 
-ENV srcDir=$projectDir/src
-ENV taudemDir=$depDir/taudem/bin
-ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
-
-ARG GroupID=1370800235
-ARG GroupName=fim
-RUN addgroup --gid $GroupID $GroupName
-ENV GID=$GroupID
-ENV GN=$GroupName
-
-RUN mkdir -p $depDir
-COPY --from=builder $depDir $depDir
-
-RUN apt update --fix-missing
-RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
-
-COPY install_grass.exp .
-RUN ./install_grass.exp
-
-RUN apt auto-remove
-
-## adding environment variables for numba and python ##
-ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
-ENV PYTHONUNBUFFERED=TRUE
-
-## ADD TO PATHS ##
-ENV PATH="$projectDir:${PATH}"
-
-## install python 3 modules ##
-COPY Pipfile .
-COPY Pipfile.lock .
-RUN pip3 install pipenv && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
-
-## Copy the source code to the image
-COPY . $projectDir/
-
-## Set user:group for running docker in detached mode
-USER root:$GroupName
-
-# RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
-ADD ./src/entrypoint.sh /
-ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
-
diff --git a/Pipfile.lock b/Pipfile.lock
index e1db9681c..746158f4d 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -234,33 +234,37 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010",
-                "sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd",
-                "sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43",
-                "sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9",
-                "sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df",
-                "sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400",
-                "sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2",
-                "sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4",
-                "sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a",
-                "sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6",
-                "sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8",
-                "sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b",
-                "sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8",
-                "sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb",
-                "sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2",
-                "sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f",
-                "sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4",
-                "sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a",
-                "sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16",
-                "sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f",
-                "sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69",
-                "sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65",
-                "sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17",
-                "sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48"
+                "sha256:1a784e8ff7ea2a32e393cc53eb0003eca1597c7ca628227e34ce34eb11645a0e",
+                "sha256:2ba579dde0563f47021dcd652253103d6fd66165b18011dce1a0609215b2791e",
+                "sha256:3537b967b350ad17633b35c2f4b1a1bbd258c018910b518c30b48c8e41272717",
+                "sha256:3c40e6b860220ed862e8097b8f81c9af6d7405b723f4a7af24a267b46f90e461",
+                "sha256:598fe100b2948465cf3ed64b1a326424b5e4be2670552066e17dfaa67246011d",
+                "sha256:620732f42259eb2c4642761bd324462a01cdd13dd111740ce3d344992dd8492f",
+                "sha256:709884863def34d72b183d074d8ba5cfe042bc3ff8898f1ffad0209161caaa99",
+                "sha256:75579acbadbf74e3afd1153da6177f846212ea2a0cc77de53523ae02c9256513",
+                "sha256:7c55407f739f0bfcec67d0df49103f9333edc870061358ac8a8c9e37ea02fcd2",
+                "sha256:a1f2fb2da242568af0271455b89aee0f71e4e032086ee2b4c5098945d0e11cf6",
+                "sha256:a290989cd671cd0605e9c91a70e6df660f73ae87484218e8285c6522d29f6e38",
+                "sha256:ac4fd578322842dbda8d968e3962e9f22e862b6ec6e3378e7415625915e2da4d",
+                "sha256:ad09f55cc95ed8d80d8ab2052f78cc21cb231764de73e229140d81ff49d8145e",
+                "sha256:b9205711e5440954f861ceeea8f1b415d7dd15214add2e878b4d1cf2bcb1a914",
+                "sha256:bba474a87496d96e61461f7306fba2ebba127bed7836212c360f144d1e72ac54",
+                "sha256:bebab3eaf0641bba26039fb0b2c5bf9b99407924b53b1ea86e03c32c64ef5aef",
+                "sha256:cc367c86eb87e5b7c9592935620f22d13b090c609f1b27e49600cd033b529f54",
+                "sha256:ccc6c650f8700ce1e3a77668bb7c43e45c20ac06ae00d22bdf6760b38958c883",
+                "sha256:cf680682ad0a3bef56dae200dbcbac2d57294a73e5b0f9864955e7dd7c2c2491",
+                "sha256:d2910d0a075caed95de1a605df00ee03b599de5419d0b95d55342e9a33ad1fb3",
+                "sha256:d5caa946a9f55511e76446e170bdad1d12d6b54e17a2afe7b189112ed4412bb8",
+                "sha256:d89b0dc7f005090e32bb4f9bf796e1dcca6b52243caf1803fdd2b748d8561f63",
+                "sha256:d95d16204cd51ff1a1c8d5f9958ce90ae190be81d348b514f9be39f878b8044a",
+                "sha256:e4d5a86a5257843a18fb1220c5f1c199532bc5d24e849ed4b0289fb59fbd4d8f",
+                "sha256:e58ddb53a7b4959932f5582ac455ff90dcb05fac3f8dcc8079498d43afbbde6c",
+                "sha256:e80fe25cba41c124d04c662f33f6364909b985f2eb5998aaa5ae4b9587242cce",
+                "sha256:eda2829af498946c59d8585a9fd74da3f810866e05f8df03a86f70079c7531dd",
+                "sha256:fd0a359c1c17f00cb37de2969984a74320970e0ceef4808c32e00773b06649d9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.20.3"
+            "version": "==1.21.0"
         },
         "pandas": {
             "hashes": [
@@ -286,43 +290,43 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5",
-                "sha256:031a6c88c77d08aab84fecc05c3cde8414cd6f8406f4d2b16fed1e97634cc8a4",
-                "sha256:083781abd261bdabf090ad07bb69f8f5599943ddb539d64497ed021b2a67e5a9",
-                "sha256:0d19d70ee7c2ba97631bae1e7d4725cdb2ecf238178096e8c82ee481e189168a",
-                "sha256:0e04d61f0064b545b989126197930807c86bcbd4534d39168f4aa5fda39bb8f9",
-                "sha256:12e5e7471f9b637762453da74e390e56cc43e486a88289995c1f4c1dc0bfe727",
-                "sha256:22fd0f42ad15dfdde6c581347eaa4adb9a6fc4b865f90b23378aa7914895e120",
-                "sha256:238c197fc275b475e87c1453b05b467d2d02c2915fdfdd4af126145ff2e4610c",
-                "sha256:3b570f84a6161cf8865c4e08adf629441f56e32f180f7aa4ccbd2e0a5a02cba2",
-                "sha256:463822e2f0d81459e113372a168f2ff59723e78528f91f0bd25680ac185cf797",
-                "sha256:4d98abdd6b1e3bf1a1cbb14c3895226816e666749ac040c4e2554231068c639b",
-                "sha256:5afe6b237a0b81bd54b53f835a153770802f164c5570bab5e005aad693dab87f",
-                "sha256:5b70110acb39f3aff6b74cf09bb4169b167e2660dabc304c1e25b6555fa781ef",
-                "sha256:5cbf3e3b1014dddc45496e8cf38b9f099c95a326275885199f427825c6522232",
-                "sha256:624b977355cde8b065f6d51b98497d6cd5fbdd4f36405f7a8790e3376125e2bb",
-                "sha256:63728564c1410d99e6d1ae8e3b810fe012bc440952168af0a2877e8ff5ab96b9",
-                "sha256:66cc56579fd91f517290ab02c51e3a80f581aba45fd924fcdee01fa06e635812",
-                "sha256:6c32cc3145928c4305d142ebec682419a6c0a8ce9e33db900027ddca1ec39178",
-                "sha256:8b56553c0345ad6dcb2e9b433ae47d67f95fc23fe28a0bde15a120f25257e291",
-                "sha256:8bb1e155a74e1bfbacd84555ea62fa21c58e0b4e7e6b20e4447b8d07990ac78b",
-                "sha256:95d5ef984eff897850f3a83883363da64aae1000e79cb3c321915468e8c6add5",
-                "sha256:a013cbe25d20c2e0c4e85a9daf438f85121a4d0344ddc76e33fd7e3965d9af4b",
-                "sha256:a787ab10d7bb5494e5f76536ac460741788f1fbce851068d73a87ca7c35fc3e1",
-                "sha256:a7d5e9fad90eff8f6f6106d3b98b553a88b6f976e51fce287192a5d2d5363713",
-                "sha256:aac00e4bc94d1b7813fe882c28990c1bc2f9d0e1aa765a5f2b516e8a6a16a9e4",
-                "sha256:b91c36492a4bbb1ee855b7d16fe51379e5f96b85692dc8210831fbb24c43e484",
-                "sha256:c03c07ed32c5324939b19e36ae5f75c660c81461e312a41aea30acdd46f93a7c",
-                "sha256:c5236606e8570542ed424849f7852a0ff0bce2c4c8d0ba05cc202a5a9c97dee9",
-                "sha256:c6b39294464b03457f9064e98c124e09008b35a62e3189d3513e5148611c9388",
-                "sha256:cb7a09e173903541fa888ba010c345893cd9fc1b5891aaf060f6ca77b6a3722d",
-                "sha256:d68cb92c408261f806b15923834203f024110a2e2872ecb0bd2a110f89d3c602",
-                "sha256:dc38f57d8f20f06dd7c3161c59ca2c86893632623f33a42d592f097b00f720a9",
-                "sha256:e98eca29a05913e82177b3ba3d198b1728e164869c613d76d0de4bde6768a50e",
-                "sha256:f217c3954ce5fd88303fc0c317af55d5e0204106d86dea17eb8205700d47dec2"
+                "sha256:063d17a02a0170c2f880fbd373b2738b089c6adcbd1f7418667bc9e97524c11b",
+                "sha256:1037288a22cc8ec9d2918a24ded733a1cc4342fd7f21d15d37e6bbe5fb4a7306",
+                "sha256:25f6564df21d15bcac142b4ed92b6c02e53557539f535f31c1f3bcc985484753",
+                "sha256:28f184c0a65be098af412f78b0b6f3bbafd1614e1dc896e810d8357342a794b7",
+                "sha256:3251557c53c1ed0c345559afc02d2b0a0aa5788042e161366ed90b27bc322d3d",
+                "sha256:331f8321418682386e4f0d0e6369f732053f95abddd2af4e1b1ef74a9537ef37",
+                "sha256:333313bcc53a8a7359e98d5458dfe37bfa301da2fd0e0dc41f585ae0cede9181",
+                "sha256:34ce3d993cb4ca840b1e31165b38cb19c64f64f822a8bc5565bde084baff3bdb",
+                "sha256:490c9236ef4762733b6c2e1f1fcb37793cb9c57d860aa84d6994c990461882e5",
+                "sha256:519b3b24dedc81876d893475bade1b92c4ce7c24b9b82224f0bd8daae682e039",
+                "sha256:53f6e4b73b3899015ac4aa95d99da0f48ea18a6d7c8db672e8bead3fb9570ef5",
+                "sha256:561339ed7c324bbcb29b5e4f4705c97df950785394b3ac181f5bf6a08088a672",
+                "sha256:6f7517a220aca8b822e25b08b0df9546701a606a328da5bc057e5f32a3f9b07c",
+                "sha256:713b762892efa8cd5d8dac24d16ac2d2dbf981963ed1b3297e79755f03f8cbb8",
+                "sha256:72858a27dd7bd1c40f91c4f85db3b9f121c8412fd66573121febb00d074d0530",
+                "sha256:778a819c2d194e08d39d67ddb15ef0d32eba17bf7d0c2773e97bd221b2613a3e",
+                "sha256:803606e206f3e366eea46b1e7ab4dac74cfac770d04de9c35319814e11e47c46",
+                "sha256:856fcbc3201a6cabf0478daa0c0a1a8a175af7e5173e2084ddb91cc707a09dd1",
+                "sha256:8f65d2a98f198e904dbe89ecb10862d5f0511367d823689039e17c4d011de11e",
+                "sha256:94db5ea640330de0945b41dc77fb4847b4ab6e87149126c71b36b112e8400898",
+                "sha256:950e873ceefbd283cbe7bc5b648b832d1dcf89eeded6726ebec42bc7d67966c0",
+                "sha256:a7beda44f177ee602aa27e0a297da1657d9572679522c8fb8b336b734653516e",
+                "sha256:aef0838f28328523e9e5f2c1852dd96fb85768deb0eb8f908c54dad0f44d2f6f",
+                "sha256:b42ea77f4e7374a67e1f27aaa9c62627dff681f67890e5b8f0c1e21b1500d9d2",
+                "sha256:bccd0d604d814e9494f3bf3f077a23835580ed1743c5175581882e7dd1f178c3",
+                "sha256:c2d78c8230bda5fc9f6b1d457c7f8f3432f4fe85bed86f80ba3ed73d59775a88",
+                "sha256:c3529fb98a40f89269175442c5ff4ef81d22e91b2bdcbd33833a350709b5130c",
+                "sha256:cc8e926d6ffa65d0dddb871b7afe117f17bc045951e66afde60eb0eba923db9e",
+                "sha256:ce90aad0a3dc0f13a9ff0ab1f43bcbea436089b83c3fadbe37c6f1733b938bf1",
+                "sha256:cec702974f162026bf8de47f6f4b7ce9584a63c50002b38f195ee797165fea77",
+                "sha256:d9ef8119ce44f90d2f8ac7c58f7da480ada5151f217dc8da03681b73fc91dec3",
+                "sha256:eccaefbd646022b5313ca4b0c5f1ae6e0d3a52ef66de64970ecf3f9b2a1be751",
+                "sha256:fb91deb5121b6dde88599bcb3db3fdad9cf33ff3d4ccc5329ee1fe9655a2f7ff",
+                "sha256:fc25d59ecf23ea19571065306806a29c43c67f830f0e8a121303916ba257f484"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==8.2.0"
+            "version": "==8.3.0"
         },
         "psutil": {
             "hashes": [
@@ -423,11 +427,11 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:00aa34e92d992e9f8383730816359647f358f4a3be1ba45e5a5cefd27ee91544",
-                "sha256:b1ae5e9643d5ed987fc57cc2583021e38db531946518130777734f9589b3141f"
+                "sha256:dd8fe852847f4fbfadabf6183ddd4c824a9651f02d51714fa075c95561959c7d",
+                "sha256:effaac3c1e58d89b3ccb4d04a40dc7ad6e0275fda25fd75ae9d323e2465e202d"
             ],
             "index": "pypi",
-            "version": "==0.17.1"
+            "version": "==0.18.0"
         },
         "pytz": {
             "hashes": [
@@ -481,28 +485,28 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:01b38dec7e9f897d4db04f8de4e20f0f5be3feac98468188a0f47a991b796055",
-                "sha256:10dbcc7de03b8d635a1031cb18fd3eaa997969b64fdf78f99f19ac163a825445",
-                "sha256:19aeac1ad3e57338723f4657ac8520f41714804568f2e30bd547d684d72c392e",
-                "sha256:1b21c6e0dc97b1762590b70dee0daddb291271be0580384d39f02c480b78290a",
-                "sha256:1caade0ede6967cc675e235c41451f9fb89ae34319ddf4740194094ab736b88d",
-                "sha256:23995dfcf269ec3735e5a8c80cfceaf384369a47699df111a6246b83a55da582",
-                "sha256:2a799714bf1f791fb2650d73222b248d18d53fd40d6af2df2c898db048189606",
-                "sha256:3274ce145b5dc416c49c0cf8b6119f787f0965cd35e22058fe1932c09fe15d77",
-                "sha256:33d1677d46111cfa1c84b87472a0274dde9ef4a7ef2e1f155f012f5f1e995d8f",
-                "sha256:44d452850f77e65e25b1eb1ac01e25770323a782bfe3a1a3e43847ad4266d93d",
-                "sha256:9e3302149a369697c6aaea18b430b216e3c88f9a61b62869f6104881e5f9ef85",
-                "sha256:a75b014d3294fce26852a9d04ea27b5671d86736beb34acdfc05859246260707",
-                "sha256:ad7269254de06743fb4768f658753de47d8b54e4672c5ebe8612a007a088bd48",
-                "sha256:b30280fbc1fd8082ac822994a98632111810311a9ece71a0e48f739df3c555a2",
-                "sha256:b79104878003487e2b4639a20b9092b02e1bad07fc4cf924b495cf413748a777",
-                "sha256:d449d40e830366b4c612692ad19fbebb722b6b847f78a7b701b1e0d6cda3cc13",
-                "sha256:d647757373985207af3343301d89fe738d5a294435a4f2aafb04c13b4388c896",
-                "sha256:f68eb46b86b2c246af99fcaa6f6e37c7a7a413e1084a794990b877f2ff71f7b6",
-                "sha256:fdf606341cd798530b05705c87779606fcdfaf768a8129c348ea94441da15b04"
+                "sha256:0572256c10ddd058e3d315c555538671ddb2737f27eb56189bfbc3483391403f",
+                "sha256:2e685fdbfa5b989af4338b29c408b9157ea6addec15d661104c437980c292be5",
+                "sha256:3595c8b64970c9e5a3f137fa1a9eb64da417e78fb7991d0b098b18a00b776d88",
+                "sha256:3e7df79b42c3015058a5554bfeab6fd4c9906c46560c9ddebb5c652840f3e182",
+                "sha256:4ef3d4df8af40cb6f4d4eaf7b02780109ebabeec334cda26a7899ec9d8de9176",
+                "sha256:53116abd5060a5b4a58489cf689bee259b779e6b7ecd4ce366e7147aa7c9626e",
+                "sha256:5a983d3cebc27294897951a494cebd78af2eae37facf75d9e4ad4f1f62229860",
+                "sha256:5eb8f054eebb351af7490bbb57465ba9662c4e16e1786655c6c7ed530eb9a74e",
+                "sha256:6130e22bf6ee506f7cddde7e0515296d97eb6c6c94f7ef5103c2b77aec5833a7",
+                "sha256:7f4b89c223bd09460b52b669e2e642cab73c28855b540e6ed029692546a86f8d",
+                "sha256:80df8af7039bce92fb4cd1ceb056258631b11b3c627384e2d29bb48d44c0cae7",
+                "sha256:821e75f5c16cd7b0ab0ffe7eb9917e5af7b48c25306b4777287de8d792a5f7f3",
+                "sha256:97ca4552ace1c313707058e774609af59644321e278c3a539322fab2fb09b943",
+                "sha256:998c5e6ea649489302de2c0bc026ed34284f531df89d2bdc8df3a0d44d165739",
+                "sha256:aef6e922aea6f2e6bbb539b413c85210a9ee32757535b84204ebd22723e69704",
+                "sha256:b77ee5e3a9507622e7f98b16122242a3903397f98d1fe3bc269d904a9025e2bc",
+                "sha256:bd4399d4388ca0239a4825e312b3e61b60f743dd6daf49e5870837716502a92a",
+                "sha256:c5d012cb82cc1dcfa72609abaabb4a4ed8113e3e8ac43464508a418c146be57d",
+                "sha256:e7b733d4d98e604109715e11f2ab9340eb45d53f803634ed730039070fc3bc11"
             ],
             "markers": "python_version < '3.10' and python_version >= '3.7'",
-            "version": "==1.6.3"
+            "version": "==1.7.0"
         },
         "seaborn": {
             "hashes": [
@@ -630,11 +634,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:a171caa3d3d4c819a1c0742e3abecfd5a2b8ab525ca1c9f114b40b76b0679ab1",
-                "sha256:f86788eef439891438af3498525094cc2acbdbea4f2aa2f8895782d4ff471341"
+                "sha256:54bbd1fe3882457aaf28ae060a5ccdef97f212a741754e420028d4ec5c2291dc",
+                "sha256:aa21412f2b04ad1a652e30564fff6b4de04726ce875eab222c8430edc6db383a"
             ],
             "index": "pypi",
-            "version": "==7.24.0"
+            "version": "==7.25.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -684,11 +688,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:bf00f22079f5fadc949f42ae8ff7f05702826a97059ffcc6281036ad40ac6f04",
-                "sha256:e1b4f11b9336a28fa11810bc623c357420f69dfdb6d2dac41ca2c21a55c033bc"
+                "sha256:08360ee3a3148bdb5163621709ee322ec34fc4375099afa4bbf751e9b7b7fa4f",
+                "sha256:7089d8d2938043508aa9420ec18ce0922885304cddae87fb96eebca942299f88"
             ],
             "markers": "python_full_version >= '3.6.1'",
-            "version": "==3.0.18"
+            "version": "==3.0.19"
         },
         "ptyprocess": {
             "hashes": [
diff --git a/README.md b/README.md
index e9953112c..d0e868ec2 100644
--- a/README.md
+++ b/README.md
@@ -132,7 +132,7 @@ Please see the issue tracker on GitHub for known issues and for getting help.
 NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
 ## Open Source Licensing Info
-1. [TERMS](TERMS.md)
+1. [TERMS](docs/TERMS.md)
 2. [LICENSE](LICENSE)
 
 ## Credits and References
diff --git a/api/node/updater/updater.py b/api/node/updater/updater.py
index c0bdd40e6..33fa0e08b 100644
--- a/api/node/updater/updater.py
+++ b/api/node/updater/updater.py
@@ -75,7 +75,7 @@ def update_loop():
                 outputs_path = f"/data/outputs/{current_jobs[job_name]['nice_name']}"
                 if os.path.isdir(outputs_path):
                     shutil.rmtree(outputs_path)
-                
+
                 jobs_to_delete.append(job_name)
 
             active_statuses = [
@@ -113,7 +113,7 @@ def update_loop():
                             total_active_cores += current_jobs[j]['parallel_jobs'] * 5
                         else:
                             total_active_cores += current_jobs[j]['parallel_jobs']
-                
+
                 # Machine has enough resources to run a new job
                 potential_active_cores = 0
                 if current_jobs[job_name]['hucs_type'] == '6':
@@ -170,6 +170,14 @@ def update_loop():
                     current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
                     current_jobs[job_name]['status'] = 'Ready to Save File'
                 elif current_jobs[job_name]['job_type'] == 'release':
+                    # Move outputs to previous_fim and set them to be copied to the dev machine
+                    if os.path.isdir(f"/data/previous_fim/{current_jobs[job_name]['nice_name']}"):
+                        shutil.rmtree(f"/data/previous_fim/{current_jobs[job_name]['nice_name']}")
+                    if os.path.isdir(f"/data/outputs/{current_jobs[job_name]['nice_name']}"): shutil.move(f"/data/outputs/{current_jobs[job_name]['nice_name']}", '/data/previous_fim')
+                    for path, folders, files in os.walk(f"/data/previous_fim/{current_jobs[job_name]['nice_name']}"):
+                        for file in files:
+                            current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = 0
+                    current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
                     current_jobs[job_name]['status'] = 'Ready for Synthesize Test Cases'
 
             if current_jobs[job_name]['status'] == 'Ready for Synthesize Test Cases':
@@ -177,9 +185,6 @@ def update_loop():
                 nice_name = current_jobs[job_name]['nice_name']
                 parallel_jobs = current_jobs[job_name]['parallel_jobs']
 
-                if os.path.isdir(f"/data/previous_fim/{nice_name}"):
-                    shutil.rmtree(f"/data/previous_fim/{nice_name}")
-                if os.path.isdir(f"/data/outputs/{nice_name}"): shutil.move(f"/data/outputs/{nice_name}", '/data/previous_fim')
                 # Kick off the new job as a docker container to run eval metrics
                 print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/synthesize_test_cases.py -c PREV --fim-version {nice_name} --job-number {parallel_jobs} -m /data/test_cases/metrics_library/all_official_versions.csv")
                 subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/synthesize_test_cases.py -c PREV --fim-version {nice_name} --job-number {parallel_jobs} -m /data/test_cases/metrics_library/all_official_versions.csv", shell=True)
@@ -205,7 +210,6 @@ def update_loop():
                 subprocess.call(f"docker container rm {job_name}", shell=True)
 
                 current_jobs[job_name]['output_files_saved']['/data/test_cases/metrics_library/all_official_versions.csv'] = 0
-                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/docker.log"] = 0
                 current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/synthesize_test_cases_docker.log"] = 0
                 current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
                 current_jobs[job_name]['status'] = 'Ready for Eval Plots'
@@ -289,7 +293,7 @@ def update_loop():
                 current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/generate_categorical_fim.log"] = 0
                 current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
                 current_jobs[job_name]['status'] = 'Ready to Save File'
-    
+
             # Trigger connector to transmit the outputs to the output_handler
             # If the output_handler is offline, it will keep retrying until the output_handler is online
             if current_jobs[job_name]['status'] == 'Ready to Save File' and (shared_data['current_saving_job'] == '' or shared_data['current_saving_job'] == current_jobs[job_name]):
@@ -303,7 +307,7 @@ def update_loop():
                         output_to_save = {'path': path, 'chunk_index': current_jobs[job_name]['output_files_saved'][path]}
 
                 if output_to_save != {}:
-                    if shared_data['connected']: 
+                    if shared_data['connected']:
                         sio.emit('ready_for_output_handler', {
                             'nice_name': current_jobs[job_name]['nice_name'],
                             'job_name': job_name,
@@ -311,7 +315,7 @@ def update_loop():
                             'chunk_index': output_to_save['chunk_index']
                         })
                 current_jobs[job_name]['status'] = 'Saving File'
-            
+
             # Once the output_handler is done getting the outputs and the connector deletes the temp repo source,
             # mark as completed
             if current_jobs[job_name]['status'] == 'Saving File':
@@ -335,8 +339,8 @@ def update_loop():
                         if os.path.isdir(destination):
                             shutil.rmtree(destination)
                         if os.path.isdir(f"{outputs_path}/aggregate_fim_outputs"): shutil.move(f"{outputs_path}/aggregate_fim_outputs", destination)
-                        if os.path.isdir(f"{outputs_path}/logs"): shutil.move(f"{outputs_path}/logs", f"{destination}/logs") 
-                        if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"): shutil.move(f"/data/catfim/{current_jobs[job_name]['nice_name']}", f"{destination}/catfim") 
+                        if os.path.isdir(f"{outputs_path}/logs"): shutil.move(f"{outputs_path}/logs", f"{destination}/logs")
+                        if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"): shutil.move(f"/data/catfim/{current_jobs[job_name]['nice_name']}", f"{destination}/catfim")
 
                     if os.path.isdir(outputs_path):
                         shutil.rmtree(outputs_path)
@@ -349,7 +353,7 @@ def update_loop():
                             pass
 
                     current_jobs[job_name]['status'] = 'Completed' if current_jobs[job_name]['exit_code'] == 0 else 'Error'
-                
+
                     shared_data['current_saving_job'] = ''
                     current_jobs[job_name]['is_actively_saving'] = False
                     print(f"{job_name} completed")
@@ -383,7 +387,7 @@ def update_loop():
         if shared_data['connected']: sio.emit('update', {'jobUpdates': job_updates, 'presetsList': presets_list})
         with open('/data/outputs/current_jobs.json.temp', 'w') as f:
             json.dump(current_jobs, f)
-            shutil.move('/data/outputs/current_jobs.json.temp', '/data/outputs/current_jobs.json') 
+            shutil.move('/data/outputs/current_jobs.json.temp', '/data/outputs/current_jobs.json')
 
 sio = socketio.Client()
 
@@ -523,4 +527,4 @@ def ws_file_saved(data):
     current_jobs[job_name]['status'] = 'Ready to Save File'
 
 sio.connect('http://fim_node_connector:6000/')
-update_loop()
\ No newline at end of file
+update_loop()
diff --git a/config/params_template.env b/config/params_template.env
index 87270a669..b85c92e9d 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -7,7 +7,7 @@ export wbd_buffer=5000
 export thalweg_lateral_elev_threshold=3
 
 #### geospatial parameters ####
-export max_split_distance_meters=1500
+export max_split_distance_meters=2000
 export ms_buffer_dist=7000
 export lakes_buffer_dist_meters=20
 
diff --git a/CHANGELOG.md b/docs/CHANGELOG.md
similarity index 97%
rename from CHANGELOG.md
rename to docs/CHANGELOG.md
index 3a925ce7a..05c3d149a 100644
--- a/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.21.0 - 2021-08-18 - [PR #433](https://github.com/NOAA-OWP/cahaba/pull/433)
+
+General repository cleanup, made memory-profiling an optional flag, API's release feature now saves outputs.
+
+## Changes
+- Remove `Dockerfile.prod`, rename `Dockerfile.dev` to just `Dockerfile`, and remove ``.dockerignore`.
+- Clean up `Dockerfile` and remove any unused* packages or variables.
+- Remove any unused* Python packages from the `Pipfile`.
+- Move the `CHANGELOG.md`, `SECURITY.md`, and `TERMS.md` files to the `/docs` folder.
+- Remove any unused* scripts in the `/tools` and `/src` folders.
+- Move `tools/preprocess` scripts into `tools/`.
+- Ensure all scripts in the `/src` folder have their code in functions and are being called via a `__main__` function (This will help with implementing memory profiling fully).
+- Changed memory-profiling to be an option flag `-m` for `fim_run.sh`.
+- Updated FIM API to save all outputs during a "release" job.
+
+<br/><br/>
+
 ## v3.0.20.2 - 2021-08-13 - [PR #443](https://github.com/NOAA-OWP/cahaba/pull/443)
 
 This merge modifies `clip_vectors_to_wbd.py` to check for relevant input data.
diff --git a/SECURITY.md b/docs/SECURITY.md
similarity index 100%
rename from SECURITY.md
rename to docs/SECURITY.md
diff --git a/TERMS.md b/docs/TERMS.md
similarity index 100%
rename from TERMS.md
rename to docs/TERMS.md
diff --git a/fim_run.sh b/fim_run.sh
index c3700ba30..1978004d1 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -21,6 +21,7 @@ usage ()
     echo '  -w/--whitelist  : list of files to save in a production run in addition to final inundation outputs'
     echo '                     ex: file1.tif,file2.json,file3.csv'
     echo '  -v/--viz        : compute post-processing on outputs to be used in viz'
+    echo '  -m/--mem        : enable memory profiling'
     exit
 }
 
@@ -69,6 +70,9 @@ in
     -v|--viz)
         viz=1
         ;;
+    -m|--mem)
+        mem=1
+        ;;
     *) ;;
     esac
     shift
@@ -107,6 +111,7 @@ export extent=$extent
 export production=$production
 export whitelist=$whitelist
 export viz=$viz
+export mem=$mem
 logFile=$outputRunDataDir/logs/summary.log
 
 ## Define inputs
diff --git a/src/acquire_and_preprocess_inputs.py b/src/acquire_and_preprocess_inputs.py
index 89e432966..417cc4d93 100755
--- a/src/acquire_and_preprocess_inputs.py
+++ b/src/acquire_and_preprocess_inputs.py
@@ -19,13 +19,12 @@
                                     NHD_VECTOR_EXTRACTION_SUFFIX,
                                     PREP_PROJECTION,
                                     WBD_NATIONAL_URL,
-                                    FIM_ID,
-                                    OVERWRITE_WBD,
-                                    OVERWRITE_NHD,
-                                    OVERWRITE_ALL)
+                                    FIM_ID
+                                    )
 
-from utils.shared_functions import (pull_file, run_system_command,
-                                    subset_wbd_gpkg, delete_file,
+from utils.shared_functions import (pull_file, 
+                                    run_system_command,
+                                    delete_file,
                                     getDriver)
 
 NHDPLUS_VECTORS_DIRNAME = 'nhdplus_vectors'
@@ -33,12 +32,14 @@
 NWM_HYDROFABRIC_DIRNAME = 'nwm_hydrofabric'
 NWM_FILE_TO_SUBSET_WITH = 'nwm_flows.gpkg'
 
+
 def subset_wbd_to_nwm_domain(wbd,nwm_file_to_use):
 
     intersecting_indices = [not (gpd.read_file(nwm_file_to_use,mask=b).empty) for b in wbd.geometry]
 
     return(wbd[intersecting_indices])
 
+
 def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_use,overwrite_wbd,num_workers):
     """
     This helper function pulls and unzips Watershed Boundary Dataset (WBD) data. It uses the WBD URL defined by WBD_NATIONAL_URL.
@@ -125,6 +126,7 @@ def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_
 
     return(wbd_directory)
 
+
 def pull_and_prepare_nwm_hydrofabric(path_to_saved_data_parent_dir, path_to_preinputs_dir,num_workers):
     """
     This helper function pulls and unzips NWM hydrofabric data. It uses the NWM hydrofabric URL defined by NWM_HYDROFABRIC_URL.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 0496b8602..a8107ca5b 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -11,10 +11,11 @@
 # sys.path.append('/foss_fim/src')
 # sys.path.append('/foss_fim/config')
 from bathy_rc_adjust import bathy_rc_lookup
-from utils.shared_functions import getDriver
+from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
 
-@profile
+
+@mem_profile
 def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
 
     input_catchments = gpd.read_file(input_catchments_fileName)
@@ -274,6 +275,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
         with open(output_src_json_fileName,'w') as f:
             json.dump(output_src_json,f,sort_keys=True,indent=2)
 
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Crosswalk for MS/FR networks; calculate synthetic rating curves; update short rating curves')
diff --git a/src/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
index 961d39d01..8350311c5 100644
--- a/src/adjust_headwater_streams.py
+++ b/src/adjust_headwater_streams.py
@@ -159,6 +159,7 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
         return nhd_streams, combined_pts
 
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='adjust headwater stream geometery based on headwater start points')
@@ -173,6 +174,8 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
     args = vars(parser.parse_args())
 
+    #TODO variables below are not defined
+
     adj_streams_gdf, adj_headwaters_gdf = adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id)
 
     if subset_nhd_streams_fileName is not None:
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index b4bbd8eb2..c9930f480 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -2,11 +2,13 @@
 
 
 import argparse
-from numba import njit, typeof, typed, types
+from numba import njit, typed, types
 import rasterio
 import numpy as np
+from utils.shared_functions import mem_profile
 
-@profile
+
+@mem_profile
 def adjust_thalweg_laterally(elevation_raster, stream_raster, allocation_raster, cost_distance_raster, cost_distance_tolerance, dem_lateral_thalweg_adj,lateral_elevation_threshold):
 
     # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
diff --git a/src/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py
index bf1a8d09c..7725abc4f 100644
--- a/src/aggregate_fim_outputs.py
+++ b/src/aggregate_fim_outputs.py
@@ -12,6 +12,7 @@
 import csv
 from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
 
+
 def aggregate_fim_outputs(args):
 
     fim_out_dir   = args[0]
@@ -191,6 +192,7 @@ def reproject_raster(raster_name,reprojection):
                 resampling=Resampling.nearest)
     del src, dst
 
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Aggregate layers buy HUC6')
@@ -208,7 +210,6 @@ def reproject_raster(raster_name,reprojection):
     huc6_list = [str(huc[0:6]) for huc in os.listdir(fim_outputs_directory) if huc not in drop_folders]
     huc6_list = list(set(huc6_list))
 
-
     procs_list = []
 
     for huc6 in huc6_list:
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
index c05e62bb1..aa47342b8 100755
--- a/src/aggregate_vector_inputs.py
+++ b/src/aggregate_vector_inputs.py
@@ -1,12 +1,10 @@
 #!/usr/bin/env python3
 
 import os
-import sys
 # sys.path.append('/foss_fim/src')
 import geopandas as gpd
 from utils.shared_variables import PREP_PROJECTION
 from utils.shared_functions import getDriver
-from derive_headwaters import findHeadWaterPoints
 from reduce_nhd_stream_density import subset_nhd_network
 from adjust_headwater_streams import adjust_headwaters
 from shapely.geometry import Point
@@ -365,32 +363,9 @@ def clean_up_intermediate_files(nhdplus_vectors_dir):
             os.remove(headwater_adj_path)
 
 
-if(__name__=='__main__'):
+if __name__ == '__main__':
 
     # # Generate NWM Headwaters
-    # print ('deriving nwm headwater points')
-    # nwm_headwaters = findHeadWaterPoints(nwm_streams_orig_filename)
-    # nwm_headwaters['ID'] = nwm_headwaters.index + 1
-    # nwm_headwaters.to_file(nwm_headwaters_filename,driver=getDriver(nwm_headwaters_filename),index=False,layer='nwm_headwaters')
-    # del nwm_headwaters, nwm_streams
-    #
-    # # Identify NWM MS Streams
-    # print ('identifing nwm ms streams')
-    # ms_segments = identify_nwm_ms_streams(nwm_streams_orig_filename,ahps_filename,nwm_streams_all_filename)
-    #
-    # # Identify NWM MS Catchments
-    # print ('identifing nwm ms catchments')
-    # nwm_catchments = gpd.read_file(nwm_catchments_orig_filename)
-    # # Add column to FR nwm layer to indicate MS segments
-    # nwm_catchments['mainstem'] = np.where(nwm_catchments.ID.isin(ms_segments), 1, 0)
-    # nwm_catchments.to_file(nwm_catchments_all_filename,driver=getDriver(nwm_catchments_all_filename),index=False,layer='nwm_catchments')
-    # del nwm_catchments, ms_segments
-
-    # # Generate NWM intersection points with WBD4 boundaries
-    # print ('deriving NWM fr/ms intersection points')
-    # huc4_intersection = find_nwm_incoming_streams(nwm_streams_all_filename,wbd_filename,4)
-    # huc4_intersection.to_file(nwm_huc4_intersections_filename,driver=getDriver(nwm_huc4_intersections_filename),layer='huc4_intersection')
-    # del huc4_intersection
 
     print ('loading HUC4s')
     wbd4 = gpd.read_file(wbd_filename, layer='WBDHU4')
diff --git a/src/agreedem.py b/src/agreedem.py
index 3d92a2d42..7d15b7f80 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -4,8 +4,10 @@
 import os
 import argparse
 from r_grow_distance import r_grow_distance
+from utils.shared_functions import mem_profile
 
-@profile
+
+@mem_profile
 def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data):
     '''
     Produces a hydroconditioned raster using the AGREE DEM methodology as described by Ferdi Hellweger (https://www.caee.utexas.edu/prof/maidment/gishydro/ferdi/research/agree/agree.html). The GRASS gis tool r.grow.distance is used to calculate intermediate allocation and proximity rasters.
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
index ebda61044..524e75c46 100755
--- a/src/bathy_rc_adjust.py
+++ b/src/bathy_rc_adjust.py
@@ -1,15 +1,14 @@
 #!/usr/bin/env python3
 
 from os import environ
-import geopandas as gpd
 import pandas as pd
-import numpy as np
 
-sa_ratio_flag             = float(environ['surf_area_thalweg_ratio_flag']) #10x
-thal_stg_limit            = float(environ['thalweg_stg_search_max_limit']) #3m
-bankful_xs_ratio_flag     = float(environ['bankful_xs_area_ratio_flag']) #10x
-bathy_xsarea_flag         = float(environ['bathy_xs_area_chg_flag']) #1x
-thal_hyd_radius_flag      = float(environ['thalweg_hyd_radius_flag']) #10x
+sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag']) #10x
+thal_stg_limit = float(environ['thalweg_stg_search_max_limit']) #3m
+bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag']) #10x
+bathy_xsarea_flag = float(environ['bathy_xs_area_chg_flag']) #1x
+thal_hyd_radius_flag = float(environ['thalweg_hyd_radius_flag']) #10x
+
 
 def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,):
     ## Convert input_src_base featureid to integer
diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py
index 672fe8a01..b3639ac95 100644
--- a/src/build_stream_traversal.py
+++ b/src/build_stream_traversal.py
@@ -7,10 +7,10 @@
         hydro_id       = name of ID column (string)
 '''
 import sys
-import pandas as pd
 import argparse
 import geopandas as gpd
 
+
 def trace():
     import traceback, inspect
     tb = sys.exc_info()[2]
@@ -25,6 +25,7 @@ def trace():
 to_node = "To_Node"
 next_down_id = "NextDownID"
 
+
 class build_stream_traversal_columns(object):
     '''Tool class for updating the next down IDs of stream features.'''
     def __init__(self):
@@ -167,7 +168,8 @@ def execute(self, streams, wbd8, hydro_id):
             tReturns = (sOK, )
         return tReturns
 
-if(__name__=='__main__'):
+
+if __name__ == '__main__':
     try:
         ap = argparse.ArgumentParser()
         ap.add_argument("-p", "--parameters", nargs='+', default=[], required=True,
diff --git a/src/burn_in_levees.py b/src/burn_in_levees.py
index 010c41e0c..93acbc1e4 100755
--- a/src/burn_in_levees.py
+++ b/src/burn_in_levees.py
@@ -3,10 +3,13 @@
 import rasterio
 import numpy as np
 import argparse
+from utils.shared_functions import mem_profile
 
-@profile
+
+@mem_profile
 def burn_in_levees(dem_filename,nld_filename,out_dem_filename):
 
+    #TODO Document this code
     dem = rasterio.open(dem_filename)
     nld = rasterio.open(nld_filename)
 
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index 1302e18ad..cd471cd88 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -4,6 +4,7 @@
 import argparse
 from glob import glob
 
+
 def __read_included_files(parent_dir_path):
 
     filename_patterns = glob(os.path.join(parent_dir_path,'included_huc*.lst'))
@@ -46,6 +47,7 @@ def check_hucs(hucs):
     hucs = __read_input_hucs(hucs)
     __check_for_membership(hucs,accepted_hucs)
 
+
 if __name__ == '__main__':
 
     # parse arguments
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index b1c9c542e..c70c8766c 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -3,11 +3,11 @@
 import sys
 import geopandas as gpd
 import argparse
-from os.path import splitext
-from shapely.geometry import MultiPolygon,Polygon,Point
-from utils.shared_functions import getDriver
+from shapely.geometry import MultiPolygon,Polygon
+from utils.shared_functions import getDriver, mem_profile
 
-@profile
+
+@mem_profile
 def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance):
 
     hucUnitLength = len(str(hucCode))
@@ -134,6 +134,7 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
         sys.exit(0)
     del nwm_streams
 
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Subset vector layers')
diff --git a/src/derive_headwaters.py b/src/derive_headwaters.py
index 9fb7c1020..1bae7209e 100644
--- a/src/derive_headwaters.py
+++ b/src/derive_headwaters.py
@@ -3,6 +3,7 @@
 import geopandas as gpd
 from shapely.geometry import Point
 from utils.shared_functions import getDriver
+import argparse
 
 
 def findHeadWaterPoints(flows):
@@ -47,5 +48,6 @@ def findHeadWaterPoints(flows):
 
     hw_gdf = findHeadWaterPoints(flows)
 
+    #TODO check output_headwaters variable, not defined
     if output_headwaters is not None:
         hw_gdf.to_file(args['output_headwaters'],driver=getDriver(args['output_headwaters']))
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index 104bc4275..17aa5c611 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -1,48 +1,68 @@
 #!/usr/bin/env python3
 
+import argparse
 import geopandas as gpd
-import pandas as pd
 import numpy as np
-import argparse
 import sys
 from utils.shared_variables import FIM_ID
+from utils.shared_functions import mem_profile
+
+
+@mem_profile
+def filter_catchments_and_add_attributes(input_catchments_filename, input_flows_filename, output_catchments_filename, output_flows_filename, wbd_filename, huc_code):
+    input_catchments = gpd.read_file(input_catchments_filename)
+    wbd = gpd.read_file(wbd_filename)
+    input_flows = gpd.read_file(input_flows_filename)
+
+    # filter segments within huc boundary
+    select_flows = tuple(map(str,map(int,wbd[wbd.HUC8.str.contains(huc_code)][FIM_ID])))
+
+    if input_flows.HydroID.dtype != 'str': input_flows.HydroID = input_flows.HydroID.astype(str)
+    output_flows = input_flows[input_flows.HydroID.str.startswith(select_flows)].copy()
+    if output_flows.HydroID.dtype != 'int': output_flows.HydroID = output_flows.HydroID.astype(int)
+
+    if len(output_flows) > 0:
+
+        # merges input flows attributes and filters hydroids
+        if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
+        output_catchments = input_catchments.merge(output_flows.drop(['geometry'],axis=1),on='HydroID')
+
+        # filter out smaller duplicate features
+        duplicateFeatures = np.where(np.bincount(output_catchments['HydroID'])>1)[0]
 
-input_catchments_fileName = sys.argv[1]
-input_flows_fileName = sys.argv[2]
-output_catchments_fileName = sys.argv[3]
-output_flows_fileName = sys.argv[4]
-wbd_fileName = sys.argv[5]
-hucCode = str(sys.argv[6])
+        for dp in duplicateFeatures:
 
-input_catchments = gpd.read_file(input_catchments_fileName)
-wbd = gpd.read_file(wbd_fileName)
-input_flows = gpd.read_file(input_flows_fileName)
+            indices_of_duplicate = np.where(output_catchments['HydroID'] == dp)[0]
+            areas = output_catchments.iloc[indices_of_duplicate,:].geometry.area
+            indices_of_smaller_duplicates = indices_of_duplicate[np.where(areas != np.amax(areas))[0]]
+            output_catchments = output_catchments.drop(output_catchments.index[indices_of_smaller_duplicates])
 
-# filter segments within huc boundary
-select_flows = tuple(map(str,map(int,wbd[wbd.HUC8.str.contains(hucCode)][FIM_ID])))
+        # add geometry column
+        output_catchments['areasqkm'] = output_catchments.geometry.area/(1000**2)
 
-if input_flows.HydroID.dtype != 'str': input_flows.HydroID = input_flows.HydroID.astype(str)
-output_flows = input_flows[input_flows.HydroID.str.startswith(select_flows)].copy()
-if output_flows.HydroID.dtype != 'int': output_flows.HydroID = output_flows.HydroID.astype(int)
+        output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
+        output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
 
-if len(output_flows) > 0:
-    
-    # merges input flows attributes and filters hydroids
-    if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
-    output_catchments = input_catchments.merge(output_flows.drop(['geometry'],axis=1),on='HydroID')
 
-    # filter out smaller duplicate features
-    duplicateFeatures = np.where(np.bincount(output_catchments['HydroID'])>1)[0]
+if __name__ == '__main__':
 
-    for dp in duplicateFeatures:
+    input_catchments_filename = sys.argv[1]
+    input_flows_filename = sys.argv[2]
+    output_catchments_filename = sys.argv[3]
+    output_flows_filename = sys.argv[4]
+    wbd_filename = sys.argv[5]
+    huc_code = str(sys.argv[6])
 
-        indices_of_duplicate = np.where(output_catchments['HydroID'] == dp)[0]
-        areas = output_catchments.iloc[indices_of_duplicate,:].geometry.area
-        indices_of_smaller_duplicates = indices_of_duplicate[np.where(areas != np.amax(areas))[0]]
-        output_catchments = output_catchments.drop(output_catchments.index[indices_of_smaller_duplicates])
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='filter_catchments_and_add_attributes.py')
+    parser.add_argument('-i', '--input-catchments-filename', help='input-catchments-filename', required=True)
+    parser.add_argument('-f', '--input-flows-filename', help='input-flows-filename', required=True)
+    parser.add_argument('-c', '--output-catchments-filename', help='output-catchments-filename', required=True)
+    parser.add_argument('-o', '--output-flows-filename', help='output-flows-filename', required=True)
+    parser.add_argument('-w', '--wbd-filename', help='wbd-filename', required=True)
+    parser.add_argument('-u', '--huc-code', help='huc-code', required=True)
 
-    # add geometry column
-    output_catchments['areasqkm'] = output_catchments.geometry.area/(1000**2)
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
 
-    output_catchments.to_file(output_catchments_fileName, driver="GPKG",index=False)
-    output_flows.to_file(output_flows_fileName, driver="GPKG", index=False)
+    filter_catchments_and_add_attributes(**args)
diff --git a/src/fr_to_ms_raster_mask.py b/src/fr_to_ms_raster_mask.py
index 567b5cf19..2ebc713f0 100755
--- a/src/fr_to_ms_raster_mask.py
+++ b/src/fr_to_ms_raster_mask.py
@@ -6,78 +6,93 @@
 
 import sys
 import os
+import argparse
 import geopandas as gpd
 import rasterio.mask
+from utils.shared_functions import mem_profile
 
-split_flows_fileName    = sys.argv[1]
-fdr_fr                  = sys.argv[2]
-dem_fr                  = sys.argv[3]
-slope_fr                = sys.argv[4]
-fdr_ms_filename         = sys.argv[5]
-dem_ms_filename         = sys.argv[6]
-slope_ms_filename       = sys.argv[7]
-str_pixel_fr            = sys.argv[8]
-str_pixel_ms_filename   = sys.argv[9]
-ms_buffer_dist          = int(os.environ['ms_buffer_dist'])
-
-# create output layer names
-split_flows = gpd.read_file(split_flows_fileName)
-
-# Limit the rasters to the buffer distance around the draft streams.
-print ("Limiting rasters to buffer area ({} meters) around model streams".format(str(ms_buffer_dist)))
-
-split_flows_ms_buffer = split_flows.unary_union.buffer(ms_buffer_dist)
-
-print('Writing raster outputs ...')
-
-# Mask nhddem
-with rasterio.open(dem_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(dem_fr), dem_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
-
-# Mask fdr
-with rasterio.open(fdr_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(fdr_fr), fdr_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
-
-# Mask slope
-with rasterio.open(slope_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(slope_fr), slope_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
-
-# Mask stream pixels
-with rasterio.open(str_pixel_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(str_pixel_fr), str_pixel_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
+
+@mem_profile
+def fr_to_ms_raster_mask(ms_buffer_dist, split_flows_filename, fdr_fr, dem_fr, slope_fr, fdr_ms_filename, dem_ms_filename, slope_ms_filename, str_pixel_fr, str_pixel_ms_filename):
+    # create output layer names
+    split_flows = gpd.read_file(split_flows_filename)
+
+    # Limit the rasters to the buffer distance around the draft streams.
+    print ("Limiting rasters to buffer area ({} meters) around model streams".format(str(ms_buffer_dist)))
+
+    split_flows_ms_buffer = split_flows.unary_union.buffer(ms_buffer_dist)
+
+    print('Writing raster outputs ...')
+
+    # Mask nhddem
+    with rasterio.open(dem_fr) as src:
+        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+        out_meta = src.meta
+
+    out_meta.update({"driver": "GTiff",
+         "height": out_image.shape[1],
+         "width": out_image.shape[2],
+         "transform": out_transform})
+
+    with rasterio.open(os.path.join(os.path.dirname(dem_fr), dem_ms_filename), "w", **out_meta) as dest:
+        dest.write(out_image)
+
+    # Mask fdr
+    with rasterio.open(fdr_fr) as src:
+        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+        out_meta = src.meta
+
+    out_meta.update({"driver": "GTiff",
+         "height": out_image.shape[1],
+         "width": out_image.shape[2],
+         "transform": out_transform})
+
+    with rasterio.open(os.path.join(os.path.dirname(fdr_fr), fdr_ms_filename), "w", **out_meta) as dest:
+        dest.write(out_image)
+
+    # Mask slope
+    with rasterio.open(slope_fr) as src:
+        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+        out_meta = src.meta
+
+    out_meta.update({"driver": "GTiff",
+         "height": out_image.shape[1],
+         "width": out_image.shape[2],
+         "transform": out_transform})
+
+    with rasterio.open(os.path.join(os.path.dirname(slope_fr), slope_ms_filename), "w", **out_meta) as dest:
+        dest.write(out_image)
+
+    # Mask stream pixels
+    with rasterio.open(str_pixel_fr) as src:
+        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+        out_meta = src.meta
+
+    out_meta.update({"driver": "GTiff",
+         "height": out_image.shape[1],
+         "width": out_image.shape[2],
+         "transform": out_transform})
+
+    with rasterio.open(os.path.join(os.path.dirname(str_pixel_fr), str_pixel_ms_filename), "w", **out_meta) as dest:
+        dest.write(out_image)
+
+
+if __name__ == '__main__':
+    ms_buffer_dist = int(os.environ['ms_buffer_dist'])
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='fr_to_ms_raster_mask.py')
+    parser.add_argument('-s', '--split-flows-filename', help='split-flows-filename', required=True)
+    parser.add_argument('-f', '--fdr-fr', help='fdr-fr', required=True)
+    parser.add_argument('-d', '--dem-fr', help='dem-fr', required=True)
+    parser.add_argument('-r', '--slope-fr', help='slope-fr', required=True)
+    parser.add_argument('-m', '--fdr-ms-filename', help='fdr-ms-filename', required=True)
+    parser.add_argument('-n', '--dem-ms-filename', help='dem-ms-filename', required=True)
+    parser.add_argument('-o', '--slope-ms-filename', help='slope-ms-filename', required=True)
+    parser.add_argument('-p', '--str-pixel-fr', help='str-pixel-fr', required=True)
+    parser.add_argument('-q', '--str-pixel-ms-filename', help='str-pixel-ms-filename', required=True)
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    fr_to_ms_raster_mask(ms_buffer_dist, **args)
diff --git a/src/getRasterInfoNative.py b/src/getRasterInfoNative.py
index 27c1a3e73..11e93e322 100755
--- a/src/getRasterInfoNative.py
+++ b/src/getRasterInfoNative.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8
 
+#TODO standardize this script
+
 import gdal
 import os.path
 import sys
@@ -10,6 +12,7 @@
 read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy <<< $(./getRasterInfoNative.py <raster.tif>)
 """
 
+
 def GetExtent(gt,cols,rows):
 	''' Return list of corner coordinates from a geotransform
 
@@ -35,6 +38,7 @@ def GetExtent(gt,cols,rows):
 		yarr.reverse()
 	return ext
 
+
 def ReprojectCoords(coords,src_srs,tgt_srs):
 	''' Reproject a list of x,y coordinates.
 
diff --git a/src/get_all_huc_in_inputs.py b/src/get_all_huc_in_inputs.py
index 78be2ce7b..af507b10b 100755
--- a/src/get_all_huc_in_inputs.py
+++ b/src/get_all_huc_in_inputs.py
@@ -8,6 +8,7 @@
 from os.path import join
 from tqdm import tqdm
 
+
 def find_unique_hucs(inputsDir,hucLength):
 
     # get file list with glob
@@ -24,7 +25,6 @@ def find_unique_hucs(inputsDir,hucLength):
     unique_hucs.to_csv(join(inputsDir,'included_huc{}.lst'.format(hucLength)),header=False,index=False)
 
 
-
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Get unique HUCs in results data dir')
     parser.add_argument('-i','--inputs-directory',help='Basins polygons to use within project path',required=True)
diff --git a/src/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
index 35627377c..524360915 100755
--- a/src/make_stages_and_catchlist.py
+++ b/src/make_stages_and_catchlist.py
@@ -1,40 +1,51 @@
 #!/usr/bin/env python3
 
+import argparse
 import geopandas as gpd
-import pandas as pd
 import numpy as np
-import argparse
 import sys
-import decimal
+from utils.shared_functions import mem_profile
+
+
+@mem_profile
+def make_stages_and_catchlist(flows_filename, catchments_filename, stages_filename, catchlist_filename, stages_min, stages_interval, stages_max):
+    flows = gpd.read_file(flows_filename)
+    catchments = gpd.read_file(catchments_filename)
+
 
-flows_fileName = sys.argv[1]
-catchments_fileName = sys.argv[2]
-stages_fileName = sys.argv[3]
-catchlist_fileName = sys.argv[4]
-stages_min = float(sys.argv[5])
-stages_interval = float(sys.argv[6])
-stages_max = float(sys.argv[7])
+    hydroIDs = flows['HydroID'].tolist()
+    len_of_hydroIDs = len(hydroIDs)
+    slopes = flows['S0'].tolist()
+    lengthkm = flows['LengthKm'].tolist()
+    areasqkm = catchments['areasqkm'].tolist()
 
-flows = gpd.read_file(flows_fileName)
-catchments = gpd.read_file(catchments_fileName)
 
+    stages_max = stages_max + stages_interval
+    stages = np.round(np.arange(stages_min,stages_max,stages_interval),4)
 
-hydroIDs = flows['HydroID'].tolist()
-len_of_hydroIDs = len(hydroIDs)
-slopes = flows['S0'].tolist()
-lengthkm = flows['LengthKm'].tolist()
-areasqkm = catchments['areasqkm'].tolist()
+    with open(stages_filename,'w') as f:
+        f.write("Stage\n")
+        for stage in stages:
+            f.write("{}\n".format(stage))
 
+    with open(catchlist_filename,'w') as f:
+        f.write("{}\n".format(len_of_hydroIDs))
+        for h,s,l,a in zip(hydroIDs,slopes,lengthkm,areasqkm):
+            f.write("{} {} {} {}\n".format(h,s,l,a))
 
-stages_max = stages_max + stages_interval
-stages = np.round(np.arange(stages_min,stages_max,stages_interval),4)
+    #TODO we need a main block
+if __name__ == '__main__':
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='make_stages_and_catchlist.py')
+    parser.add_argument('-f', '--flows-filename', help='flows-filename',required=True)
+    parser.add_argument('-c', '--catchments-filename', help='catchments-filename',required=True)
+    parser.add_argument('-s', '--stages-filename', help='stages-filename',required=True)
+    parser.add_argument('-a', '--catchlist-filename', help='catchlist-filename',required=True)
+    parser.add_argument('-m', '--stages-min', help='stages-min',required=True,type=float)
+    parser.add_argument('-i', '--stages-interval', help='stages-interval',required=True,type=float)
+    parser.add_argument('-t', '--stages-max', help='stages-max',required=True,type=float)
 
-with open(stages_fileName,'w') as f:
-    f.write("Stage\n")
-    for stage in stages:
-        f.write("{}\n".format(stage))
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
 
-with open(catchlist_fileName,'w') as f:
-    f.write("{}\n".format(len_of_hydroIDs))
-    for h,s,l,a in zip(hydroIDs,slopes,lengthkm,areasqkm):
-        f.write("{} {} {} {}\n".format(h,s,l,a))
+    make_stages_and_catchlist(**args)
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index d73a2deee..e65dfeaee 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
 import os
-import csv
-import json
-import shutil
 import argparse
+from utils.shared_functions import mem_profile
 
-@profile
+
+@mem_profile
 def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, viz_post_processing):
     '''
     Processes all the final output files to cleanup and add post-processing
@@ -63,7 +62,7 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         whitelist_directory(output_folder_path, viz_whitelist, additional_whitelist)
 
 
-@profile
+@mem_profile
 def whitelist_directory(directory_path, whitelist, additional_whitelist):
     # Add any additional files to the whitelist that the user wanted to keep
     if additional_whitelist:
diff --git a/src/preprocess_rasters.py b/src/preprocess_rasters.py
index 7026f4cb2..635e1f227 100755
--- a/src/preprocess_rasters.py
+++ b/src/preprocess_rasters.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-from osgeo import gdal
 import sys
 sys.path.append('/foss_fim/src')
 from multiprocessing import Pool
diff --git a/src/r_grow_distance.py b/src/r_grow_distance.py
index 793ff8600..d42e88c0d 100755
--- a/src/r_grow_distance.py
+++ b/src/r_grow_distance.py
@@ -5,6 +5,7 @@
 import grass.script as gscript
 import argparse
 
+
 def r_grow_distance(input_raster, grass_workspace, proximity_dtype, allocation_dtype):
     '''
     Runs the r.grow.distance GRASS gis tool which given an input raster will produce an output proximity (or distance) and euclidian allocation tool.
@@ -76,6 +77,7 @@ def r_grow_distance(input_raster, grass_workspace, proximity_dtype, allocation_d
     
     return output_proximity_path,output_allocation_path
 
+
 if __name__ == '__main__':
 
     #Parse arguments
diff --git a/src/reachID_grid_to_vector_points.py b/src/reachID_grid_to_vector_points.py
index c77bcc732..849256953 100755
--- a/src/reachID_grid_to_vector_points.py
+++ b/src/reachID_grid_to_vector_points.py
@@ -1,22 +1,20 @@
 #!/usr/bin/env python3
 
 import numpy as np
-import osgeo.ogr
-import osgeo.osr
-import sys
 import argparse
-from tqdm import tqdm
 import geopandas as gpd
 from utils.shared_variables import PREP_PROJECTION
 from shapely.geometry import Point
 import rasterio
-from utils.shared_functions import getDriver
+from utils.shared_functions import getDriver, mem_profile
 
 """
 USAGE:
 ./reachID_grid_to_vector_points.py <flows_grid_IDs raster file> <flows_points vector file> <reachID or featureID>
 
 """
+
+@mem_profile
 def convert_grid_cells_to_points(raster,index_option,output_points_filename=False):
 
     # Input raster
@@ -54,6 +52,7 @@ def convert_grid_cells_to_points(raster,index_option,output_points_filename=Fals
     else:
         pointGDF.to_file(output_points_filename,driver=getDriver(output_points_filename),index=False)
 
+
 if __name__ == '__main__':
 
     # Parse arguments
diff --git a/src/reduce_nhd_stream_density.py b/src/reduce_nhd_stream_density.py
index afd775d93..e11472746 100644
--- a/src/reduce_nhd_stream_density.py
+++ b/src/reduce_nhd_stream_density.py
@@ -160,6 +160,7 @@ def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_file
 
     return nhd_streams
 
+
 def get_downstream_segments(streams, attribute,mainstem_flag):
 
     Q = deque(streams.loc[streams[attribute],'NHDPlusID'].tolist())
@@ -199,6 +200,7 @@ def get_downstream_segments(streams, attribute,mainstem_flag):
 
     return streams
 
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Reduce NHDPlus HR network based on headwater points')
diff --git a/src/rem.py b/src/rem.py
index 9e0c78361..c064710ed 100755
--- a/src/rem.py
+++ b/src/rem.py
@@ -1,17 +1,15 @@
 #!/usr/bin/env python3
 
-from numba import njit, typeof, typed, types
+from numba import njit, typed, types
 import rasterio
 import numpy as np
 import argparse
-import os
 import pandas as pd
-from osgeo import ogr, gdal
 import geopandas as gpd
-from utils.shared_functions import getDriver
+from utils.shared_functions import getDriver, mem_profile
 
 
-@profile
+@mem_profile
 def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, dem_reaches_filename):
     """
         Calculates REM/HAND/Detrended DEM
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 9c399f96d..0c732e970 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -229,7 +229,7 @@ echo -e $startDiv"Split Derived Reaches $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ] && \
-python3 -m memory_profiler $srcDir/split_flows.py $outputHucDataDir/demDerived_reaches.shp $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/demDerived_reaches_split_points.gpkg $outputHucDataDir/wbd8_clp.gpkg $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+python3 -m memory_profiler $srcDir/split_flows.py -f $outputHucDataDir/demDerived_reaches.shp -d $outputHucDataDir/dem_thalwegCond.tif -s $outputHucDataDir/demDerived_reaches_split.gpkg -p $outputHucDataDir/demDerived_reaches_split_points.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 Tcount
 
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
@@ -243,7 +243,7 @@ if [ "$extent" = "MS" ]; then
   echo -e $startDiv"Mask Rasters with Stream Buffer $hucNumber"$stopDiv
   date -u
   Tstart
-  python3 -m memory_profiler $srcDir/fr_to_ms_raster_mask.py $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/flowdir_d8_burned_filled.tif $outputHucDataDir/dem_thalwegCond.tif $outputHucDataDir/slopes_d8_dem_meters.tif $outputHucDataDir/flowdir_d8_MS.tif $outputHucDataDir/dem_thalwegCond_MS.tif $outputHucDataDir/slopes_d8_dem_metersMS.tif $outputHucDataDir/demDerived_streamPixels.tif $outputHucDataDir/demDerived_streamPixelsMS.tif
+  python3 -m memory_profiler $srcDir/fr_to_ms_raster_mask.py -s $outputHucDataDir/demDerived_reaches_split.gpkg -f $outputHucDataDir/flowdir_d8_burned_filled.tif -d $outputHucDataDir/dem_thalwegCond.tif -r $outputHucDataDir/slopes_d8_dem_meters.tif -m $outputHucDataDir/flowdir_d8_MS.tif -n $outputHucDataDir/dem_thalwegCond_MS.tif -o $outputHucDataDir/slopes_d8_dem_metersMS.tif -p $outputHucDataDir/demDerived_streamPixels.tif -q $outputHucDataDir/demDerived_streamPixelsMS.tif
   Tcount
 
   if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
@@ -276,7 +276,7 @@ echo -e $startDiv"Vectorize Pixel Centroids $hucNumber"$stopDiv
 date -u
 Tstart
 [ ! -f $outputHucDataDir/flows_points_pixels.gpkg ] && \
-$srcDir/reachID_grid_to_vector_points.py -r $demDerived_streamPixels -i featureID -p $outputHucDataDir/flows_points_pixels.gpkg
+python3 -m memory_profiler $srcDir/reachID_grid_to_vector_points.py -r $demDerived_streamPixels -i featureID -p $outputHucDataDir/flows_points_pixels.gpkg
 Tcount
 
 ## GAGE WATERSHED FOR PIXELS ##
@@ -324,7 +324,7 @@ echo -e $startDiv"Process catchments and model streams step 1 $hucNumber"$stopDi
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ] && \
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py $outputHucDataDir/gw_catchments_reaches.gpkg $outputHucDataDir/demDerived_reaches_split.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/wbd8_clp.gpkg $hucNumber
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputHucDataDir/gw_catchments_reaches.gpkg -f $outputHucDataDir/demDerived_reaches_split.gpkg -c $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -o $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
 
 if [[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ]] ; then
   echo "No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
@@ -384,7 +384,7 @@ Tcount
 echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/make_stages_and_catchlist.py $outputHucDataDir/demDerived_reaches_split_filtered.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/stage.txt $outputHucDataDir/catchment_list.txt $stage_min_meters $stage_interval_meters $stage_max_meters
+python3 -m memory_profiler $srcDir/make_stages_and_catchlist.py -f $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -c $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -s $outputHucDataDir/stage.txt -a $outputHucDataDir/catchment_list.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
 Tcount
 
 ## HYDRAULIC PROPERTIES ##
diff --git a/src/split_flows.py b/src/split_flows.py
index f75405533..4dff74fe0 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -22,193 +22,205 @@
 from os import remove,environ
 from collections import OrderedDict
 import build_stream_traversal
-from utils.shared_functions import getDriver
+from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
 
-flows_fileName         = sys.argv[1]
-dem_fileName           = sys.argv[2]
-split_flows_fileName   = sys.argv[3]
-split_points_fileName  = sys.argv[4]
-wbd8_clp_filename      = sys.argv[5]
-lakes_filename         = sys.argv[6]
-max_length             = float(environ['max_split_distance_meters'])
-slope_min              = float(environ['slope_min'])
-lakes_buffer_input     = float(environ['lakes_buffer_dist_meters'])
+@mem_profile
+def split_flows(max_length, slope_min, lakes_buffer_input, flows_filename, dem_filename, split_flows_filename, split_points_filename, wbd8_clp_filename, lakes_filename):
+    wbd = gpd.read_file(wbd8_clp_filename)
 
-wbd = gpd.read_file(wbd8_clp_filename)
+    toMetersConversion = 1e-3
 
-toMetersConversion = 1e-3
+    print('Loading data ...')
+    flows = gpd.read_file(flows_filename)
 
-print('Loading data ...')
-flows = gpd.read_file(flows_fileName)
+    if not len(flows) > 0:
+        print ("No relevant streams within HUC boundaries.")
+        sys.exit(0)
 
-if not len(flows) > 0:
-    print ("No relevant streams within HUC boundaries.")
-    sys.exit(0)
+    wbd8 = gpd.read_file(wbd8_clp_filename)
+    dem = rasterio.open(dem_filename,'r')
 
-wbd8 = gpd.read_file(wbd8_clp_filename)
-dem = rasterio.open(dem_fileName,'r')
+    if isfile(lakes_filename):
+        lakes = gpd.read_file(lakes_filename)
+    else:
+        lakes = None
 
-if isfile(lakes_filename):
-    lakes = gpd.read_file(lakes_filename)
-else:
-    lakes = None
+    wbd8 = wbd8.filter(items=[FIM_ID, 'geometry'])
+    wbd8 = wbd8.set_index(FIM_ID)
+    flows = flows.explode()
 
-wbd8 = wbd8.filter(items=[FIM_ID, 'geometry'])
-wbd8 = wbd8.set_index(FIM_ID)
-flows = flows.explode()
+    # temp
+    flows = flows.to_crs(wbd8.crs)
 
-# temp
-flows = flows.to_crs(wbd8.crs)
+    split_flows = []
+    slopes = []
+    hydro_id = 'HydroID'
 
-split_flows = []
-slopes = []
-hydro_id = 'HydroID'
+    # split at HUC8 boundaries
+    print ('splitting stream segments at HUC8 boundaries')
+    flows = gpd.overlay(flows, wbd8, how='union').explode().reset_index(drop=True)
 
-# split at HUC8 boundaries
-print ('splitting stream segments at HUC8 boundaries')
-flows = gpd.overlay(flows, wbd8, how='union').explode().reset_index(drop=True)
+    # check for lake features
+    if lakes is not None:
+        if len(lakes) > 0:
+          print ('splitting stream segments at ' + str(len(lakes)) + ' waterbodies')
+          #create splits at lake boundaries
+          lakes = lakes.filter(items=['newID', 'geometry'])
+          lakes = lakes.set_index('newID')
+          flows = gpd.overlay(flows, lakes, how='union').explode().reset_index(drop=True)
+          lakes_buffer = lakes.copy()
+          lakes_buffer['geometry'] = lakes.buffer(lakes_buffer_input) # adding X meter buffer for spatial join comparison (currently using 20meters)
 
-# check for lake features
-if lakes is not None:
-    if len(lakes) > 0:
-      print ('splitting stream segments at ' + str(len(lakes)) + ' waterbodies')
-      #create splits at lake boundaries
-      lakes = lakes.filter(items=['newID', 'geometry'])
-      lakes = lakes.set_index('newID')
-      flows = gpd.overlay(flows, lakes, how='union').explode().reset_index(drop=True)
-      lakes_buffer = lakes.copy()
-      lakes_buffer['geometry'] = lakes.buffer(lakes_buffer_input) # adding X meter buffer for spatial join comparison (currently using 20meters)
+    print ('splitting ' + str(len(flows)) + ' stream segments based on ' + str(max_length) + ' m max length')
 
-print ('splitting ' + str(len(flows)) + ' stream segments based on ' + str(max_length) + ' m max length')
+    # remove empty geometries
+    flows = flows.loc[~flows.is_empty,:]
 
-# remove empty geometries
-flows = flows.loc[~flows.is_empty,:]
+    for i,lineString in tqdm(enumerate(flows.geometry),total=len(flows.geometry)):
+        # Reverse geometry order (necessary for BurnLines)
+        lineString = LineString(lineString.coords[::-1])
 
-for i,lineString in tqdm(enumerate(flows.geometry),total=len(flows.geometry)):
-    # Reverse geometry order (necessary for BurnLines)
-    lineString = LineString(lineString.coords[::-1])
+        # skip lines of zero length
+        if lineString.length == 0:
+            continue
 
-    # skip lines of zero length
-    if lineString.length == 0:
-        continue
+        # existing reaches of less than max_length
+        if lineString.length < max_length:
+            split_flows = split_flows + [lineString]
+            line_points = [point for point in zip(*lineString.coords.xy)]
 
-    # existing reaches of less than max_length
-    if lineString.length < max_length:
-        split_flows = split_flows + [lineString]
-        line_points = [point for point in zip(*lineString.coords.xy)]
+            # Calculate channel slope
+            start_point = line_points[0]; end_point = line_points[-1]
+            start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
+            slope = float(abs(start_elev - end_elev) / lineString.length)
+            if slope < slope_min:
+                slope = slope_min
+            slopes = slopes + [slope]
+            continue
 
-        # Calculate channel slope
-        start_point = line_points[0]; end_point = line_points[-1]
-        start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
-        slope = float(abs(start_elev - end_elev) / lineString.length)
-        if slope < slope_min:
-            slope = slope_min
-        slopes = slopes + [slope]
-        continue
+        splitLength = lineString.length / np.ceil(lineString.length / max_length)
 
-    splitLength = lineString.length / np.ceil(lineString.length / max_length)
+        cumulative_line = []
+        line_points = []
+        last_point = []
 
-    cumulative_line = []
-    line_points = []
-    last_point = []
+        last_point_in_entire_lineString = list(zip(*lineString.coords.xy))[-1]
 
-    last_point_in_entire_lineString = list(zip(*lineString.coords.xy))[-1]
+        for point in zip(*lineString.coords.xy):
 
-    for point in zip(*lineString.coords.xy):
+            cumulative_line = cumulative_line + [point]
+            line_points = line_points + [point]
+            numberOfPoints_in_cumulative_line = len(cumulative_line)
 
-        cumulative_line = cumulative_line + [point]
-        line_points = line_points + [point]
-        numberOfPoints_in_cumulative_line = len(cumulative_line)
+            if last_point:
+                cumulative_line = [last_point] + cumulative_line
+                numberOfPoints_in_cumulative_line = len(cumulative_line)
+            elif numberOfPoints_in_cumulative_line == 1:
+                continue
 
-        if last_point:
-            cumulative_line = [last_point] + cumulative_line
-            numberOfPoints_in_cumulative_line = len(cumulative_line)
-        elif numberOfPoints_in_cumulative_line == 1:
-            continue
+            cumulative_length = LineString(cumulative_line).length
 
-        cumulative_length = LineString(cumulative_line).length
+            if cumulative_length >= splitLength:
 
-        if cumulative_length >= splitLength:
+                splitLineString = LineString(cumulative_line)
+                split_flows = split_flows + [splitLineString]
 
-            splitLineString = LineString(cumulative_line)
-            split_flows = split_flows + [splitLineString]
+                # Calculate channel slope
+                start_point = cumulative_line[0]; end_point = cumulative_line[-1]
+                start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
+                slope = float(abs(start_elev - end_elev) / splitLineString.length)
+                if slope < slope_min:
+                    slope = slope_min
+                slopes = slopes + [slope]
 
-            # Calculate channel slope
-            start_point = cumulative_line[0]; end_point = cumulative_line[-1]
-            start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
-            slope = float(abs(start_elev - end_elev) / splitLineString.length)
-            if slope < slope_min:
-                slope = slope_min
-            slopes = slopes + [slope]
+                last_point = end_point
 
-            last_point = end_point
+                if (last_point == last_point_in_entire_lineString):
+                    continue
 
-            if (last_point == last_point_in_entire_lineString):
-                continue
+                cumulative_line = []
+                line_points = []
 
-            cumulative_line = []
-            line_points = []
-
-    splitLineString = LineString(cumulative_line)
-    split_flows = split_flows + [splitLineString]
-
-    # Calculate channel slope
-    start_point = cumulative_line[0]; end_point = cumulative_line[-1]
-    start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
-    slope = float(abs(start_elev - end_elev) / splitLineString.length)
-    if slope < slope_min:
-        slope = slope_min
-    slopes = slopes + [slope]
-
-split_flows_gdf = gpd.GeoDataFrame({'S0' : slopes ,'geometry':split_flows}, crs=flows.crs, geometry='geometry')
-split_flows_gdf['LengthKm'] = split_flows_gdf.geometry.length * toMetersConversion
-if lakes is not None:
-    split_flows_gdf = gpd.sjoin(split_flows_gdf, lakes_buffer, how='left', op='within') #options: intersects, within, contains, crosses
-    split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
-else:
-    split_flows_gdf['LakeID'] = -999
-
-# need to figure out why so many duplicate stream segments for 04010101 FR
-split_flows_gdf = split_flows_gdf.drop_duplicates()
-
-# Create Ids and Network Traversal Columns
-addattributes = build_stream_traversal.build_stream_traversal_columns()
-tResults=None
-tResults = addattributes.execute(split_flows_gdf, wbd8, hydro_id)
-if tResults[0] == 'OK':
-    split_flows_gdf = tResults[1]
-else:
-    print ('Error: Could not add network attributes to stream segments')
-
-# remove single node segments
-split_flows_gdf = split_flows_gdf.query("From_Node != To_Node")
-
-# Get all vertices
-split_points = OrderedDict()
-for index, segment in split_flows_gdf.iterrows():
-    lineString = segment.geometry
-
-    for point in zip(*lineString.coords.xy):
-        if point in split_points:
-            if segment.NextDownID == split_points[point]:
-                pass
+        splitLineString = LineString(cumulative_line)
+        split_flows = split_flows + [splitLineString]
+
+        # Calculate channel slope
+        start_point = cumulative_line[0]; end_point = cumulative_line[-1]
+        start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
+        slope = float(abs(start_elev - end_elev) / splitLineString.length)
+        if slope < slope_min:
+            slope = slope_min
+        slopes = slopes + [slope]
+
+    split_flows_gdf = gpd.GeoDataFrame({'S0' : slopes ,'geometry':split_flows}, crs=flows.crs, geometry='geometry')
+    split_flows_gdf['LengthKm'] = split_flows_gdf.geometry.length * toMetersConversion
+    if lakes is not None:
+        split_flows_gdf = gpd.sjoin(split_flows_gdf, lakes_buffer, how='left', op='within') #options: intersects, within, contains, crosses
+        split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
+    else:
+        split_flows_gdf['LakeID'] = -999
+
+    # need to figure out why so many duplicate stream segments for 04010101 FR
+    split_flows_gdf = split_flows_gdf.drop_duplicates()
+
+    # Create Ids and Network Traversal Columns
+    addattributes = build_stream_traversal.build_stream_traversal_columns()
+    tResults=None
+    tResults = addattributes.execute(split_flows_gdf, wbd8, hydro_id)
+    if tResults[0] == 'OK':
+        split_flows_gdf = tResults[1]
+    else:
+        print ('Error: Could not add network attributes to stream segments')
+
+    # remove single node segments
+    split_flows_gdf = split_flows_gdf.query("From_Node != To_Node")
+
+    # Get all vertices
+    split_points = OrderedDict()
+    for index, segment in split_flows_gdf.iterrows():
+        lineString = segment.geometry
+
+        for point in zip(*lineString.coords.xy):
+            if point in split_points:
+                if segment.NextDownID == split_points[point]:
+                    pass
+                else:
+                    split_points[point] = segment[hydro_id]
             else:
                 split_points[point] = segment[hydro_id]
-        else:
-            split_points[point] = segment[hydro_id]
 
-hydroIDs_points = [hidp for hidp in split_points.values()]
-split_points = [Point(*point) for point in split_points]
+    hydroIDs_points = [hidp for hidp in split_points.values()]
+    split_points = [Point(*point) for point in split_points]
+
+    split_points_gdf = gpd.GeoDataFrame({'id': hydroIDs_points , 'geometry':split_points}, crs=flows.crs, geometry='geometry')
+
+    print('Writing outputs ...')
+
+    if isfile(split_flows_filename):
+        remove(split_flows_filename)
+    split_flows_gdf.to_file(split_flows_filename,driver=getDriver(split_flows_filename),index=False)
+
+    if isfile(split_points_filename):
+        remove(split_points_filename)
+    split_points_gdf.to_file(split_points_filename,driver=getDriver(split_points_filename),index=False)
+
 
-split_points_gdf = gpd.GeoDataFrame({'id': hydroIDs_points , 'geometry':split_points}, crs=flows.crs, geometry='geometry')
+if __name__ == '__main__':
+    max_length             = float(environ['max_split_distance_meters'])
+    slope_min              = float(environ['slope_min'])
+    lakes_buffer_input     = float(environ['lakes_buffer_dist_meters'])
 
-print('Writing outputs ...')
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='splitflows.py')
+    parser.add_argument('-f', '--flows-filename', help='flows-filename',required=True)
+    parser.add_argument('-d', '--dem-filename', help='dem-filename',required=True)
+    parser.add_argument('-s', '--split-flows-filename', help='split-flows-filename',required=True)
+    parser.add_argument('-p', '--split-points-filename', help='split-points-filename',required=True)
+    parser.add_argument('-w', '--wbd8-clp-filename', help='wbd8-clp-filename',required=True)
+    parser.add_argument('-l', '--lakes-filename', help='lakes-filename',required=True)
 
-if isfile(split_flows_fileName):
-    remove(split_flows_fileName)
-split_flows_gdf.to_file(split_flows_fileName,driver=getDriver(split_flows_fileName),index=False)
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
 
-if isfile(split_points_fileName):
-    remove(split_points_fileName)
-split_points_gdf.to_file(split_points_fileName,driver=getDriver(split_points_fileName),index=False)
+    split_flows(max_length, slope_min, lakes_buffer_input, **args)
diff --git a/src/time_and_tee_run_by_unit.sh b/src/time_and_tee_run_by_unit.sh
index 926762761..431beef97 100755
--- a/src/time_and_tee_run_by_unit.sh
+++ b/src/time_and_tee_run_by_unit.sh
@@ -1,5 +1,10 @@
 #!/bin/bash -e
 
-mprof run -o $1.dat --include-children /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log 
-mprof plot -o $outputRunDataDir/logs/$1_memory $1.dat
+if [[ "$mem" == "1" ]] ; then
+  mprof run -o $1.dat --include-children /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log
+  mprof plot -o $outputRunDataDir/logs/$1_memory $1.dat
+else
+  /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log 
+fi
+
 exit ${PIPESTATUS[0]}
diff --git a/src/unique_pixel_and_allocation.py b/src/unique_pixel_and_allocation.py
index da7efc92e..0df89560c 100755
--- a/src/unique_pixel_and_allocation.py
+++ b/src/unique_pixel_and_allocation.py
@@ -8,12 +8,14 @@
 import numpy as np
 import argparse
 from r_grow_distance import r_grow_distance
+from utils.shared_functions import mem_profile
 
-@profile
+
+@mem_profile
 def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
     '''
-    This function will assign a unique ID for each stream pixel and writes to file. It then uses this raster to run GRASS r.grow.distance tool to create the allocation and proximity rasters required to complete the lateral thalweg conditioning. 
-    
+    This function will assign a unique ID for each stream pixel and writes to file. It then uses this raster to run GRASS r.grow.distance tool to create the allocation and proximity rasters required to complete the lateral thalweg conditioning.
+
     Parameters
     ----------
     stream_pixels : STR
@@ -35,27 +37,28 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
     # Import stream pixel raster
     with rasterio.open(stream_pixels) as temp:
         streams_profile = temp.profile
-        streams = temp.read(1)  
-    
+        streams = temp.read(1)
+
     # Create array that matches shape of streams raster with unique values for each cell. Dataype is float64.
     unique_vals = np.arange(streams.size, dtype = 'float64').reshape(*streams.shape)
 
     # At streams return the unique array value otherwise return NODATA value from input streams layer. NODATA value for demDerived_streamPixels.tif is -32768.
     stream_pixel_values = np.where(streams == 1, unique_vals, streams_profile['nodata'])
-    
+
     # Reassign dtype to be float64 (needs to be float64)
     streams_profile.update(dtype = 'float64')
-    
-    # Output to raster 
+
+    # Output to raster
     with rasterio.Env():
         with rasterio.open(unique_stream_pixels, 'w', **streams_profile) as raster:
             raster.write(stream_pixel_values,1)
-    
+
     # Compute allocation and proximity grid using r.grow.distance. Output distance grid in meters. Set datatype for output allocation (needs to be float64) and proximity grids (float32).
     distance_grid, allocation_grid = r_grow_distance(unique_stream_pixels, grass_workspace, 'Float32', 'Float64')
 
     return distance_grid, allocation_grid
 
+
 if __name__ == '__main__':
 
     #Parse arguments
@@ -71,9 +74,6 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
     stream_pixels = args['stream']
     unique_stream_pixels = args['out']
     grass_workspace = args['grass_workspace']
-    
+
     # Run stream_pixel_zones
     stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace)
-
-
-
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index c85cdbe32..3b7b54256 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 
-import os
 import geopandas as gpd
 import pandas as pd
-import numpy as np
 import rasterio
 import argparse
 import pygeos
 from shapely.wkb import dumps, loads
 import warnings
+from utils.shared_functions import mem_profile
 warnings.simplefilter("ignore")
 
+
 ''' Get elevation at adjusted USGS gages locations
 
     Parameters
@@ -32,6 +32,7 @@
 '''
 
 
+@mem_profile
 def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename,dem_adj_filename,output_table_filename,extent):
 
     wbd_buffer = gpd.read_file(wbd_buffer_filename)
@@ -41,7 +42,6 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
     input_catchment = gpd.read_file(input_catchment_filename)
     dem_adj = rasterio.open(dem_adj_filename,'r')
 
-
     #MS extent use gages that are mainstem
     if extent == "MS":
         usgs_gages = usgs_gages.query('curve == "yes" & mainstem == "yes"')
@@ -101,7 +101,6 @@ def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,in
         site_elevations = [str(gage.location_id), str(hydro_id), dem_m_elev, dem_adj_elev, min_thal_elev, med_thal_elev, max_thal_elev,str(str_order)]
         gage_data.append(site_elevations)
 
-
     elev_table = pd.DataFrame(gage_data, columns=columns)
 
     if not elev_table.empty:
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 78d884eea..b01533c7d 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -208,3 +208,12 @@ def reproject_raster(input_raster_name,reprojection,blocksize=None,reprojected_r
                     resampling=Resampling.nearest)
                 del dst
         del src
+
+
+def mem_profile(func):
+    def wrapper(*args, **kwargs):
+        if (os.environ.get('mem') == "1"):
+            profile(func)(*args, **kwargs)
+        else:
+            func(*args, **kwargs)
+    return wrapper
diff --git a/tools/preprocess/create_flow_forecast_file.py b/tools/create_flow_forecast_file.py
similarity index 100%
rename from tools/preprocess/create_flow_forecast_file.py
rename to tools/create_flow_forecast_file.py
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index c572ee6ee..f1ea5ecab 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -7,7 +7,7 @@
 from pathlib import Path
 import geopandas as gpd
 import pandas as pd
-from datetime import date
+
 
 def update_mapping_status(output_mapping_dir, output_flows_dir):
     '''
@@ -49,10 +49,11 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
     flows_df.loc[flows_df['did_it_map']=='no','status'] = flows_df['status'] + flows_df['map_status']
 
-    # Perform pass for HUCs where mapping was skipped due to missing data
+    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
     flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
     mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
     missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
+    
     # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
     flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
     flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
@@ -65,6 +66,7 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     nws_lid_path = Path(output_mapping_dir) / 'nws_lid_sites.shp'
     flows_df.to_file(nws_lid_path)
 
+
 if __name__ == '__main__':
 
     # Parse arguments
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index ca57e089c..1648f073a 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -1,15 +1,15 @@
 #!/usr/bin/env python3
 from pathlib import Path
-import geopandas as gpd
 import pandas as pd
 import time
-from tools_shared_functions import aggregate_wbd_hucs, mainstem_nwm_segs, get_thresholds, flow_data, get_metadata, get_nwm_segs, flow_data
+from tools_shared_functions import aggregate_wbd_hucs, mainstem_nwm_segs, get_thresholds, flow_data, get_metadata, get_nwm_segs
 import argparse
 from dotenv import load_dotenv
 import os
 import sys
 sys.path.append('/foss_fim/src')
-from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+from utils.shared_variables import VIZ_PROJECTION
+
 
 def get_env_paths():
     load_dotenv()
@@ -19,6 +19,7 @@ def get_env_paths():
     WBD_LAYER = os.getenv("WBD_LAYER")
     return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER
 
+
 def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     '''
     This will create static flow files for all nws_lids and save to the 
@@ -221,6 +222,7 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     all_end = time.time()
     print(f'total time is {round((all_end - all_start)/60),1} minutes')
     
+    
 if __name__ == '__main__':
     #Parse arguments
     parser = argparse.ArgumentParser(description = 'Create forecast files for all nws_lid sites')
@@ -228,7 +230,6 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     parser.add_argument('-u', '--nwm_us_search',  help = 'Walk upstream on NWM network this many miles', required = True)
     parser.add_argument('-d', '--nwm_ds_search', help = 'Walk downstream on NWM network this many miles', required = True)
     args = vars(parser.parse_args())
-    
 
     #Run get_env_paths and static_flow_lids
     API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER = get_env_paths()
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
index 8fda24316..005befebe 100644
--- a/tools/generate_nws_lid.py
+++ b/tools/generate_nws_lid.py
@@ -12,7 +12,6 @@
 sys.path.append('/foss_fim/src')
 from utils.shared_variables import PREP_PROJECTION
 
-
 load_dotenv()
 #import variables from .env file
 API_BASE_URL = os.getenv("API_BASE_URL")
@@ -37,7 +36,6 @@ def generate_nws_lid(workspace):
 
     '''
     
-    
     ##############################################################################
     #Get all nws_lid points
     print('Retrieving metadata ..')
@@ -171,7 +169,6 @@ def generate_nws_lid(workspace):
     joined.to_file(Path(workspace) / 'nws_lid.gpkg', driver = 'GPKG')
 
 
-
 if __name__ == '__main__':
     #Parse arguments
     parser = argparse.ArgumentParser(description = 'Create spatial data of nws_lid points attributed with mainstems and colocated.')
diff --git a/tools/inundation.py b/tools/inundation.py
index d105ea52c..667895cd0 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -17,7 +17,6 @@
 from warnings import warn
 from gdal import BuildVRT
 import geopandas as gpd
-import sys
 
 
 def inundate(
diff --git a/tools/mannings_calibration_run.sh b/tools/mannings_calibration_run.sh
deleted file mode 100755
index 89d54cd17..000000000
--- a/tools/mannings_calibration_run.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/bash -e
-:
-usage ()
-{
-    echo "Calibrate FIM based on Manning's n values"
-    echo 'Usage : fim_run.sh [REQ: -d <fimdir> -t <huclist> -g <outdir> -n <paramfile>] [OPT: -h]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -d/--fimdir    : initial run directory with default mannings values'
-    echo '  -t/--huclist    : huc or list of hucs'
-    echo '  -g/--outdir     : output directory for mannings parameter adjustment files'
-    echo '  -n/--paramfile    : parameter set file'
-    echo ''
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
-    exit
-}
-
-if [ "$#" -lt 7 ]
-then
-  usage
-fi
-
-while [ "$1" != "" ]; do
-case $1
-in
-    -d|--fimdir)
-        shift
-        fimdir="$1"
-        ;;
-    -t|--huclist )
-        shift
-        huclist=$1
-        ;;
-    -g|--outdir )
-        shift
-        outdir=$1
-        ;;
-    -n|--paramfile)
-        shift
-        paramfile=$1
-        ;;
-    -h|--help)
-        shift
-        usage
-        ;;
-    -o|--overwrite)
-        overwrite=1
-        ;;
-    -j|--jobLimit)
-        shift
-        jobLimit=$1
-        ;;
-    *) ;;
-    esac
-    shift
-done
-
-# print usage if arguments empty
-if [ "$fimdir" = "" ]
-then
-    usage
-fi
-
-# default values
-if [ "$jobLimit" = "" ] ; then
-    jobLimit=1
-fi
-
-export input_NWM_Catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
-export outdir=$outdir
-export toolsdir="/foss_fim/tools"
-
-if [ -f "$huclist" ]; then
-
-  while read huc; do
-
-    export huc=$huc
-    export fimdir=$fimdir
-    export hucdir="/data/outputs/"$fimdir/$huc
-
-    ## RUN ##
-    if [ -f "$paramfile" ]; then
-        if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb  -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
-        else
-            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
-        fi
-    else
-        if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
-        else
-            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
-        fi
-    fi
-  done <$huclist
-
-else
-
-  for huc in $huclist
-  do
-    export huc=$huc
-    export fimdir=$fimdir
-    export hucdir="/data/outputs/"$fimdir/$huc
-
-    ## RUN ##
-    if [ -f "$paramfile" ]; then
-        if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb  -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
-        else
-            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh :::: $paramfile
-        fi
-    else
-        if [ "$jobLimit" -eq 1 ]; then
-            parallel --verbose --lb -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
-        else
-            parallel --eta -j $jobLimit -- $toolsdir/time_and_tee_mannings_calibration.sh ::: $paramfile
-        fi
-    fi
-  done
-fi
diff --git a/tools/mannings_run_by_set.sh b/tools/mannings_run_by_set.sh
deleted file mode 100755
index 8394b3a5f..000000000
--- a/tools/mannings_run_by_set.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash -e
-
-param_set="$1"
-IFS=',' read -r -a array <<< $param_set
-
-strorder="${array[1]}"
-mannings_row=1+"$strorder"
-mannings_value="${array[$mannings_row]}"
-
-subdir=$outdir/$huc"_"$strorder"_"$mannings_value
-mkdir -p $subdir
-
-$srcDir/add_crosswalk.py -d $hucdir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $hucdir/demDerived_reaches_split_filtered.gpkg -s $hucdir/src_base.csv -l $subdir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $subdir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $subdir/src_full_crosswalked.csv -j $subdir/src.json -x $subdir/crosswalk_table.csv -t $subdir/hydroTable.csv -w $hucdir/wbd8_clp.gpkg -b $hucdir/nwm_subset_streams.gpkg -y $hucdir/nwm_catchments_proj_subset.tif -m $param_set -z $input_NWM_Catchments -p FR -c
-
-python3 foss_fim/tools/run_test_case_calibration.py -r $fimdir/$huc -d $subdir -t $huc"_ble" -b "mannings_calibration"/$strorder/$mannings_value
diff --git a/tools/preprocess_ahps_nws.py b/tools/preprocess_ahps_nws.py
index c8efd464a..8c5a5f5ec 100644
--- a/tools/preprocess_ahps_nws.py
+++ b/tools/preprocess_ahps_nws.py
@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
 import numpy as np
-import pathlib
 from pathlib import Path
 import pandas as pd
 import geopandas as gpd
 import rasterio
-from collections import defaultdict
 from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 import argparse
 from dotenv import load_dotenv
@@ -13,7 +11,7 @@
 import traceback
 import sys
 sys.path.append('/foss_fim/src')
-from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+
 
 def get_env_paths():
     load_dotenv()
@@ -34,6 +32,7 @@ def get_env_paths():
 # destination = Path(r'path/to/preprocessed/nws/data')
 # reference_raster= Path(r'path/to/reference raster') 
 
+
 def preprocess_nws(source_dir, destination, reference_raster):
     source_dir = Path(source_dir)
     destination = Path(destination)
@@ -352,6 +351,7 @@ def preprocess_nws(source_dir, destination, reference_raster):
         all_attributes.to_csv(destination / 'attributes.csv', index = False)    
     return
 
+
 if __name__ == '__main__':
     #Parse arguments
     parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
diff --git a/tools/preprocess_ahps_usgs.py b/tools/preprocess_ahps_usgs.py
index 8fb7a8147..e34725b4d 100644
--- a/tools/preprocess_ahps_usgs.py
+++ b/tools/preprocess_ahps_usgs.py
@@ -1,12 +1,9 @@
 #!/usr/bin/env python3
 import numpy as np
-import pathlib
 from pathlib import Path
 import pandas as pd
-import geopandas as gpd
 import rasterio
 import requests
-from collections import defaultdict
 from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 import argparse
 from dotenv import load_dotenv
@@ -14,7 +11,6 @@
 import sys
 sys.path.append('/foss_fim/src')
 import traceback
-from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
 
 
 def get_env_paths():
diff --git a/tools/preprocess/preprocess_benchmark.py b/tools/preprocess_benchmark.py
similarity index 100%
rename from tools/preprocess/preprocess_benchmark.py
rename to tools/preprocess_benchmark.py
diff --git a/tools/preprocess/preprocess_fimx.py b/tools/preprocess_fimx.py
similarity index 100%
rename from tools/preprocess/preprocess_fimx.py
rename to tools/preprocess_fimx.py
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 6c2d3b3ba..d2adeba08 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -2,7 +2,6 @@
 
 import os
 import sys
-import geopandas as gpd
 import pandas as pd
 import numpy as np
 import argparse
@@ -10,7 +9,7 @@
 import seaborn as sns
 from functools import reduce
 from multiprocessing import Pool
-from os.path import isfile, join, dirname
+from os.path import isfile, join
 import shutil
 import warnings
 from pathlib import Path
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
index cea2a0dbf..cb8a33f56 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -9,7 +9,7 @@
 import argparse
 import sys
 sys.path.append('/foss_fim/src')
-from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+from utils.shared_variables import PREP_PROJECTION
 
 '''
 This script calls the NOAA Tidal API for datum conversions. Experience shows that
diff --git a/tools/run_test_case_calibration.py b/tools/run_test_case_calibration.py
deleted file mode 100755
index 728b87abe..000000000
--- a/tools/run_test_case_calibration.py
+++ /dev/null
@@ -1,456 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import pandas as pd
-import rasterio
-import json
-import csv
-import argparse
-import shutil
-
-from tools_shared_functions import get_contingency_table_from_binary_rasters, compute_stats_from_contingency_table
-from inundation import inundate
-
-TEST_CASES_DIR = r'/data/test_cases/'  # Will update.
-INPUTS_DIR = r'/data/inputs'
-PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
-GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
-GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
-OUTPUTS_DIR = os.environ['outputDataDir']
-
-ENDC = '\033[m'
-TGREEN_BOLD = '\033[32;1m'
-TGREEN = '\033[32m'
-TRED_BOLD = '\033[31;1m'
-TWHITE = '\033[37m'
-WHITE_BOLD = '\033[37;1m'
-CYAN_BOLD = '\033[36;1m'
-
-
-def profile_test_case_archive(archive_to_check, return_interval, stats_mode):
-    """
-    This function searches multiple directories and locates previously produced performance statistics.
-
-    Args:
-        archive_to_check (str): The directory path to search.
-        return_interval (str): Because a benchmark dataset may have multiple return intervals, this argument defines
-                               which return interval is to be used when searching for previous statistics.
-    Returns:
-        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain and return interval.
-                                  {version: {agreement_raster: agreement_raster_path, stats_csv: stats_csv_path, stats_json: stats_json_path}}
-                                  *Will only add the paths to files that exist.
-
-    """
-
-    archive_dictionary = {}
-
-    # List through previous version and check for available stats and maps. If available, add to dictionary.
-    available_versions_list = os.listdir(archive_to_check)
-
-    if len(available_versions_list) == 0:
-        print("Cannot compare with -c flag because there are no data in the previous_versions directory.")
-        return
-
-    for version in available_versions_list:
-        version_return_interval_dir = os.path.join(archive_to_check, version, return_interval)
-        # Initialize dictionary for version and set paths to None by default.
-        archive_dictionary.update({version: {'agreement_raster': None,
-                                             'stats_csv': None,
-                                             'stats_json': None}})
-        # Find stats files and raster files and add to dictionary.
-        agreement_raster = os.path.join(version_return_interval_dir, stats_mode + '_agreement.tif')
-        stats_csv = os.path.join(version_return_interval_dir, stats_mode + '_stats.csv')
-        stats_json = os.path.join(version_return_interval_dir, stats_mode + '_stats.json')
-
-        if os.path.exists(agreement_raster):
-            archive_dictionary[version]['agreement_raster'] = agreement_raster
-        if os.path.exists(stats_csv):
-            archive_dictionary[version]['stats_csv'] = stats_csv
-        if os.path.exists(stats_json):
-            archive_dictionary[version]['stats_json'] = stats_json
-
-    return archive_dictionary
-
-
-def compute_contingency_stats_from_rasters(predicted_raster_path, benchmark_raster_path, agreement_raster=None, stats_csv=None, stats_json=None, mask_values=None, stats_modes_list=['total_area'], test_id='', exclusion_mask_dict={}):
-    """
-    This function contains FIM-specific logic to prepare raster datasets for use in the generic get_contingency_table_from_binary_rasters() function.
-    This function also calls the generic compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
-
-    Args:
-        predicted_raster_path (str): The path to the predicted, or modeled, FIM extent raster.
-        benchmark_raster_path (str): The path to the benchmark, or truth, FIM extent raster.
-        agreement_raster (str): Optional. An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive, 3: True Positive.
-        stats_csv (str): Optional. Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
-        stats_json (str): Optional. Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary in other processes.
-
-    Returns:
-        stats_dictionary (dict): A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and statistic values are the values.
-    """
-
-    # Get cell size of benchmark raster.
-    raster = rasterio.open(predicted_raster_path)
-    t = raster.transform
-    cell_x = t[0]
-    cell_y = t[4]
-    cell_area = abs(cell_x*cell_y)
-
-    additional_layers_dict = {}
-    # Create path to additional_layer. Could put conditionals here to create path according to some version. Simply use stats_mode for now. Must be raster.
-    if len(stats_modes_list) > 1:
-        additional_layers_dict = {}
-        for stats_mode in stats_modes_list:
-            if stats_mode != 'total_area':
-                additional_layer_path = os.path.join(TEST_CASES_DIR, test_id, 'additional_layers', 'inclusion_areas', stats_mode + '.tif')
-                if os.path.exists(additional_layer_path):
-                    additional_layers_dict.update({stats_mode: additional_layer_path})
-                else:
-                    print("No " + stats_mode + " inclusion area found for " + test_id + ". Moving on with processing...")
-
-    # Get contingency table from two rasters.
-    contingency_table_dictionary = get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster=None, mask_values=mask_values, additional_layers_dict=additional_layers_dict, exclusion_mask_dict=exclusion_mask_dict)
-
-    stats_dictionary = {}
-
-    for stats_mode in contingency_table_dictionary:
-        true_negatives = contingency_table_dictionary[stats_mode]['true_negatives']
-        false_negatives = contingency_table_dictionary[stats_mode]['false_negatives']
-        false_positives = contingency_table_dictionary[stats_mode]['false_positives']
-        true_positives = contingency_table_dictionary[stats_mode]['true_positives']
-        masked_count = contingency_table_dictionary[stats_mode]['masked_count']
-
-        # Produce statistics from continency table and assign to dictionary. cell_area argument optional (defaults to None).
-        mode_stats_dictionary = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area, masked_count)
-
-        # Write the mode_stats_dictionary to the stats_csv.
-        if stats_csv != None:
-            stats_csv = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.csv')
-            df = pd.DataFrame.from_dict(mode_stats_dictionary, orient="index", columns=['value'])
-            df.to_csv(stats_csv)
-
-        # Write the mode_stats_dictionary to the stats_json.
-        if stats_json != None:
-            stats_json = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.json')
-            with open(stats_json, "w") as outfile:
-                json.dump(mode_stats_dictionary, outfile)
-
-        stats_dictionary.update({stats_mode: mode_stats_dictionary})
-
-    return stats_dictionary
-
-
-def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
-
-    difference_dict = {}
-
-    # Compare stats_csv to previous_version_stats_file
-    stats_dict_to_test = json.load(open(stats_json_to_test))
-    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
-
-    for stat, value in stats_dict_to_test.items():
-        previous_version_value = previous_version_stats_dict[stat]
-        stat_value_diff = value - previous_version_value
-        difference_dict.update({stat + '_diff': stat_value_diff})
-
-    return difference_dict
-
-def run_alpha_test(fim_run_dir, calib_dir, branch_name, test_id, mask_type, return_interval, compare_to_previous=False, run_structure_stats=False, run_levee_stats=False, archive_results=False):
-
-    # Construct paths to development test results if not existent.
-    if archive_results:
-        branch_test_case_dir_parent = os.path.join(TEST_CASES_DIR, test_id, 'performance_archive', 'previous_versions', branch_name)
-    else:
-        branch_test_case_dir_parent = os.path.join(TEST_CASES_DIR, test_id, 'performance_archive', 'development_versions', branch_name)
-
-    # Delete the entire directory if it already exists.
-    if os.path.exists(branch_test_case_dir_parent):
-        shutil.rmtree(branch_test_case_dir_parent)
-
-    print("Running the alpha test for test_id: " + test_id + ", " + branch_name + "...")
-    stats_modes_list = ['total_area']
-    if run_structure_stats: stats_modes_list.append('structures')
-    if run_levee_stats: stats_modes_list.append('levees')
-
-    fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
-    fim_run_calib = str(calib_dir)
-
-    assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
-    assert os.path.exists(fim_run_calib), "Cannot locate " + fim_run_calib
-
-    # Create paths to fim_run outputs for use in inundate().
-    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
-
-    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-    catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-    current_huc = test_id.split('_')[0]
-    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-    hydro_table = os.path.join(fim_run_calib, 'hydroTable.csv')
-
-    # Create list of shapefile paths to use as exclusion areas.
-    zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
-    exclusion_mask_dict = {'levees': {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
-                                      'buffer': None
-                                      },
-                            'waterbodies': {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
-                                            'buffer': None,
-                                            }
-                            }
-
-#    # Crosswalk feature_ids to hydroids.
-#    hydro_table_data = pd.read_csv(hydro_table, header=0)
-#    ht_feature_id_list = list(hydro_table_data.feature_id)
-#    ht_hydro_id_list = list(hydro_table_data.HydroID)
-#    lake_id_list = list(hydro_table_data.LakeID)
-#
-#    # Get list of feature_ids_to_mask.
-#    feature_ids_to_mask = []
-#    for f in range(0, len(lake_id_list)):
-#        if lake_id_list[f] != -999:
-#            lake_feature_id = ht_feature_id_list[f]
-#            if lake_feature_id not in feature_ids_to_mask:
-#                feature_ids_to_mask.append(lake_feature_id)
-
-    # Remove duplicates and create list of hydro_ids to use as waterbody mask.
-#    reduced_ht_feature_id_list, reduced_ht_hydro_id_list, hydro_ids_to_mask = [], [], []
-#
-#    for i in range(0, len(ht_hydro_id_list)):
-#        if ht_hydro_id_list[i] not in reduced_ht_hydro_id_list:
-#            reduced_ht_hydro_id_list.append(ht_hydro_id_list[i])
-#            reduced_ht_feature_id_list.append(ht_feature_id_list[i])
-#    for i in range(0, len(reduced_ht_feature_id_list)):
-#        ht_feature_id = reduced_ht_feature_id_list[i]
-#        ht_hydro_id = reduced_ht_hydro_id_list[i]
-#        if ht_feature_id in feature_ids_to_mask:
-#            hydro_ids_to_mask.append(ht_hydro_id)
-
-    # Check if return interval is list of return intervals or single value.
-    return_interval_list = return_interval
-    if type(return_interval_list) != list:
-        return_interval_list = [return_interval_list]
-
-    for return_interval in return_interval_list:
-        # Construct path to validation raster and forecast file.
-        benchmark_category = test_id.split('_')[1]
-        benchmark_raster_path = os.path.join(TEST_CASES_DIR, 'validation_data_' + benchmark_category, current_huc, return_interval, benchmark_category + '_huc_' + current_huc + '_extent_' + return_interval + '.tif')
-        if not os.path.exists(benchmark_raster_path):  # Skip loop instance if the benchmark raster doesn't exist.
-            continue
-
-        branch_test_case_dir = os.path.join(branch_test_case_dir_parent, return_interval)
-
-        os.makedirs(branch_test_case_dir)
-
-
-        # Define paths to inundation_raster and forecast file.
-        inundation_raster = os.path.join(branch_test_case_dir, 'inundation_extent.tif')
-        forecast = os.path.join(TEST_CASES_DIR, 'validation_data_' + benchmark_category, current_huc, return_interval, benchmark_category + '_huc_' + current_huc + '_flows_' + return_interval + '.csv')
-
-        # Run inundate.
-        print("-----> Running inundate() to produce modeled inundation extent for the " + return_interval + " return period...")
-        inundate(
-                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-                 depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
-                )
-
-        print("-----> Inundation mapping complete.")
-        predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
-
-        # Define outputs for agreement_raster, stats_json, and stats_csv.
-
-        agreement_raster, stats_json, stats_csv = os.path.join(branch_test_case_dir, 'total_area_agreement.tif'), os.path.join(branch_test_case_dir, 'stats.json'), os.path.join(branch_test_case_dir, 'stats.csv')
-
-        test_version_dictionary = compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                                         benchmark_raster_path,
-                                                                         agreement_raster=None,
-                                                                         stats_csv=stats_csv,
-                                                                         stats_json=stats_json,
-                                                                         mask_values=[],
-                                                                         stats_modes_list=stats_modes_list,
-                                                                         test_id=test_id,
-                                                                         exclusion_mask_dict=exclusion_mask_dict
-                                                                         )
-        print(" ")
-        print("Evaluation complete. All metrics for " + test_id + ", " + branch_name + ", " + return_interval + " are available at " + CYAN_BOLD + branch_test_case_dir + ENDC)
-        print(" ")
-
-        if compare_to_previous:
-            text_block = []
-            # Compare to previous stats files that are available.
-            archive_to_check = os.path.join(TEST_CASES_DIR, test_id, 'performance_archive', 'previous_versions')
-            for stats_mode in stats_modes_list:
-                archive_dictionary = profile_test_case_archive(archive_to_check, return_interval, stats_mode)
-
-                if archive_dictionary == {}:
-                    break
-
-                # Create header for section.
-                header = [stats_mode]
-                for previous_version, paths in archive_dictionary.items():
-                    header.append(previous_version)
-                header.append(branch_name)
-                text_block.append(header)
-
-                # Loop through stats in PRINTWORTHY_STATS for left.
-                for stat in PRINTWORTHY_STATS:
-                    stat_line = [stat]
-                    for previous_version, paths in archive_dictionary.items():
-                        # Load stats for previous version.
-                        previous_version_stats_json_path = paths['stats_json']
-                        previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
-
-                        # Append stat for the version to state_line.
-                        stat_line.append(previous_version_stats_dict[stat])
-
-                    # Append stat for the current version to stat_line.
-                    stat_line.append(test_version_dictionary[stats_mode][stat])
-
-                    text_block.append(stat_line)
-
-                text_block.append([" "])
-
-            regression_report_csv = os.path.join(branch_test_case_dir, 'stats_summary.csv')
-            with open(regression_report_csv, 'w', newline='') as csvfile:
-                csv_writer = csv.writer(csvfile)
-                csv_writer.writerows(text_block)
-
-            print()
-            print("--------------------------------------------------------------------------------------------------")
-
-            stats_mode = stats_modes_list[0]
-
-            try:
-                last_version_index = text_block[0].index('dev_latest')
-            except ValueError:
-                try:
-                    last_version_index = text_block[0].index('fim_2_3_3')
-                except ValueError:
-                    try:
-                        last_version_index = text_block[0].index('fim_1_0_0')
-                    except ValueError:
-                        print(TRED_BOLD + "Warning: " + ENDC + "Cannot compare " + branch_name + " to a previous version because no authoritative versions were found in previous_versions directory. Future version of run_test_case may allow for comparisons between dev branches.")
-                        print()
-                        continue
-
-            current_version_index = text_block[0].index(branch_name)
-
-            for line in text_block:
-                first_item = line[0]
-                if first_item in stats_modes_list:
-                    if first_item != stats_mode:  # Update the stats_mode and print a separator.
-                        print()
-                        print()
-                        print("--------------------------------------------------------------------------------------------------")
-                    print()
-                    stats_mode = first_item
-                    print(CYAN_BOLD + current_huc + ": " + return_interval.upper(), ENDC)
-                    print(CYAN_BOLD + stats_mode.upper().replace('_', ' ') + " METRICS" + ENDC)
-                    print()
-
-                    color = WHITE_BOLD
-                    metric_name = '      '.center(len(max(PRINTWORTHY_STATS, key=len)))
-                    percent_change_header = '% CHG'
-                    difference_header = 'DIFF'
-                    current_version_header = line[current_version_index].upper()
-                    last_version_header = line[last_version_index].upper()
-                    # Print Header.
-                    print(color + metric_name + "      " + percent_change_header.center((7)) + "       " + difference_header.center((15))  + "    " + current_version_header.center(18) + " " + last_version_header.center(18), ENDC)
-                # Format and print stat row.
-                elif first_item in PRINTWORTHY_STATS:
-                    stat_name = first_item.upper().center(len(max(PRINTWORTHY_STATS, key=len))).replace('_', ' ')
-                    current_version = round((line[current_version_index]), 3)
-                    last_version = round((line[last_version_index]) + 0.000, 3)
-                    difference = round(current_version - last_version, 3)
-                    if difference > 0:
-                        symbol = '+'
-                        if first_item in GO_UP_STATS:
-                            color = TGREEN_BOLD
-                        elif first_item in GO_DOWN_STATS:
-                            color = TRED_BOLD
-                        else:
-                            color = TWHITE
-                    if difference < 0:
-                        symbol = '-'
-                        if first_item in GO_UP_STATS:
-                            color = TRED_BOLD
-                        elif first_item in GO_DOWN_STATS:
-                            color = TGREEN_BOLD
-                        else:
-                            color = TWHITE
-
-                    if difference == 0 :
-                        symbol, color = '+', TGREEN
-                    percent_change = round((difference / last_version)*100,2)
-
-                    print(WHITE_BOLD + stat_name + ENDC + "     " + color + (symbol + " {:5.2f}".format(abs(percent_change)) + " %").rjust(len(percent_change_header)), ENDC + "    " + color + ("{:12.3f}".format((difference))).rjust(len(difference_header)), ENDC + "    " + "{:15.3f}".format(current_version).rjust(len(current_version_header)) + "   " + "{:15.3f}".format(last_version).rjust(len(last_version_header)) + "  ")
-
-            print()
-
-            print()
-            print()
-            print("--------------------------------------------------------------------------------------------------")
-            print()
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-d','--calib-dir',help='Name of directory containing parameter adjustment outputs',required=True)
-    parser.add_argument('-b', '--branch-name',help='The name of the working branch in which features are being tested',required=True,default="")
-    parser.add_argument('-t','--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
-    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
-    parser.add_argument('-y', '--return-interval',help='The return interval to check. Options include: 100yr, 500yr',required=False,default=['10yr', '100yr', '500yr'])
-    parser.add_argument('-c', '--compare-to-previous', help='Compare to previous versions of HAND.', required=False,action='store_true')
-    parser.add_argument('-s', '--run-structure-stats', help='Create contingency stats at structures.', required=False,action='store_true')
-    parser.add_argument('-a', '--archive-results', help='Automatically copy results to the "previous_version" archive for test_id. For admin use only.', required=False,action='store_true')
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    valid_test_id_list = os.listdir(TEST_CASES_DIR)
-
-    exit_flag = False  # Default to False.
-    print()
-
-    if args['run_structure_stats']:
-        print("Run structure stats (-c) not yet supported.")
-        run_structure_stats = False
-
-    # Ensure test_id is valid.
-    if args['test_id'] not in valid_test_id_list:
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
-        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
-        for test_id in valid_test_id_list:
-          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
-              print(CYAN_BOLD + test_id + ENDC)
-        print()
-        exit_flag = True
-
-    # Ensure fim_run_dir exists.
-    if not os.path.exists(os.path.join(os.environ['outputDataDir'], args['fim_run_dir'])):
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
-        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
-        print()
-        exit_flag = True
-
-    # Ensure calib_dir exists.
-    if not os.path.exists(args['calib_dir']):
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided calib_dir (-r) " + CYAN_BOLD + args['calib_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
-        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
-        print()
-        exit_flag = True
-
-    # Ensure return_interval available.
-    if args['return_interval'] == '10yr':
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided return interval (-y) " + CYAN_BOLD + args['return_interval'] + WHITE_BOLD + " is not available." + ENDC)
-        print()
-
-        exit_flag = True
-
-    if exit_flag:
-        print()
-        sys.exit()
-    else:
-        run_alpha_test(**args)
diff --git a/tools/shapefile_to_raster.py b/tools/shapefile_to_raster.py
deleted file mode 100755
index fc1689954..000000000
--- a/tools/shapefile_to_raster.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python3
-
-# A script to rasterise a shapefile to the same projection & pixel resolution as a reference image.
-from osgeo import ogr, gdal
-import subprocess
-
-InputVector = r'/data/misc/msft_building_footprints/MSFP_12090301.shp'
-OutputImage = r'/data/misc/msft_building_footprints/Result.tif'
-
-RefImage = r'/data/test_cases/12090301_ble/validation_data/100yr/ble_huc_12090301_inundation_extent_100yr.tif'
-
-gdalformat = 'GTiff'
-datatype = gdal.GDT_Byte
-burnVal = 1 #value for the output image pixels
-# Get projection info from reference image
-Image = gdal.Open(RefImage, gdal.GA_ReadOnly)
-
-# Open Shapefile
-Shapefile = ogr.Open(InputVector)
-Shapefile_layer = Shapefile.GetLayer()
-
-# Rasterise
-print("Rasterising shapefile...")
-Output = gdal.GetDriverByName(gdalformat).Create(OutputImage, Image.RasterXSize, Image.RasterYSize, 1, datatype, options=['COMPRESS=DEFLATE'])
-Output.SetProjection(Image.GetProjectionRef())
-Output.SetGeoTransform(Image.GetGeoTransform())
-
-# Write data to band 1
-Band = Output.GetRasterBand(1)
-Band.SetNoDataValue(0)
-gdal.RasterizeLayer(Output, [1], Shapefile_layer, burn_values=[burnVal])
-
-# Close datasets
-Band = None
-Output = None
-Image = None
-Shapefile = None
-
-# Build image overviews
-subprocess.call("gdaladdo --config COMPRESS_OVERVIEW DEFLATE "+OutputImage+" 2 4 8 16 32 64", shell=True)
-print("Done.")
diff --git a/tools/thalweg_drop_check.py b/tools/thalweg_drop_check.py
index c56d743e5..319c8bcd5 100644
--- a/tools/thalweg_drop_check.py
+++ b/tools/thalweg_drop_check.py
@@ -4,10 +4,8 @@
 import sys
 import geopandas as gpd
 sys.path.append('/foss_fim/src')
-from utils.shared_variables import PREP_PROJECTION
 from shapely.geometry import Point, LineString
 import rasterio
-import pandas as pd
 import numpy as np
 import argparse
 import matplotlib.pyplot as plt
diff --git a/tools/time_and_tee_mannings_calibration.sh b/tools/time_and_tee_mannings_calibration.sh
deleted file mode 100755
index 7a1c06cea..000000000
--- a/tools/time_and_tee_mannings_calibration.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash -e
-
-/usr/bin/time -v $toolsdir/mannings_run_by_set.sh $1 |& tee
-exit ${PIPESTATUS[0]}
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 2c4bdc339..9525ed9f8 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -2,7 +2,6 @@
 
 import os
 import json
-import csv
 import rasterio
 import pandas as pd
 import geopandas as gpd
@@ -13,14 +12,11 @@
 import rasterio.shutil
 from rasterio.warp import calculate_default_transform, reproject, Resampling
 import rasterio.crs
-from rasterio.merge import merge
 from rasterio import features
 from shapely.geometry import shape
 from shapely.geometry import Polygon
 from shapely.geometry import MultiPolygon
-import re
-from tools_shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
-                                    ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
+
 
 def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
 
@@ -128,126 +124,6 @@ def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
     return archive_dictionary
 
 
-#def compare_to_previous(benchmark_category, test_id, stats_modes_list, magnitude, version, test_version_dictionary, version_test_case_dir):
-#    text_block = []
-#    # Compare to previous stats files that are available.
-#    archive_to_check = os.path.join(TEST_CASES_DIR, benchmark_category + 'test_cases', test_id, 'official_versions')
-#    for stats_mode in stats_modes_list:
-#        archive_dictionary = profile_test_case_archive(archive_to_check, magnitude, stats_mode)
-#
-#        if archive_dictionary == {}:
-#            break
-#
-#        # Create header for section.
-#        header = [stats_mode]
-#        for previous_version, paths in archive_dictionary.items():
-#            header.append(previous_version)
-#        header.append(version)
-#        text_block.append(header)
-#
-#        # Loop through stats in PRINTWORTHY_STATS for left.
-#        for stat in PRINTWORTHY_STATS:
-#            stat_line = [stat]
-#            for previous_version, paths in archive_dictionary.items():
-#                # Load stats for previous version.
-#                previous_version_stats_json_path = paths['stats_json']
-#                if os.path.exists(previous_version_stats_json_path):
-#                    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
-#
-#                    # Append stat for the version to state_line.
-#                    stat_line.append(previous_version_stats_dict[stat])
-#
-#
-#            # Append stat for the current version to stat_line.
-#            stat_line.append(test_version_dictionary[stats_mode][stat])
-#
-#            text_block.append(stat_line)
-#
-#        text_block.append([" "])
-#
-#    regression_report_csv = os.path.join(version_test_case_dir, 'stats_summary.csv')
-#    with open(regression_report_csv, 'w', newline='') as csvfile:
-#        csv_writer = csv.writer(csvfile)
-#        csv_writer.writerows(text_block)
-#
-#    print()
-#    print("--------------------------------------------------------------------------------------------------")
-#
-#    stats_mode = stats_modes_list[0]
-#    try:
-#        last_version_index = text_block[0].index('dev_latest')
-#    except ValueError:
-#        try:
-#            last_version_index = text_block[0].index('fim_2_3_3')
-#        except ValueError:
-#            try:
-#                last_version_index = text_block[0].index('fim_1_0_0')
-#            except ValueError:
-#                print(TRED_BOLD + "Warning: " + ENDC + "Cannot compare " + version + " to a previous version because no authoritative versions were found in previous_versions directory. Future version of run_test_case may allow for comparisons between dev versions.")
-#                print()
-#                continue
-#
-#
-#
-#    for line in text_block:
-#        first_item = line[0]
-#        if first_item in stats_modes_list:
-#            current_version_index = line.index(version)
-#            if first_item != stats_mode:  # Update the stats_mode and print a separator.
-#                print()
-#                print()
-#                print("--------------------------------------------------------------------------------------------------")
-#            print()
-#            stats_mode = first_item
-#            print(CYAN_BOLD + current_huc + ": " + magnitude.upper(), ENDC)
-#            print(CYAN_BOLD + stats_mode.upper().replace('_', ' ') + " METRICS" + ENDC)
-#            print()
-#
-#            color = WHITE_BOLD
-#            metric_name = '      '.center(len(max(PRINTWORTHY_STATS, key=len)))
-#            percent_change_header = '% CHG'
-#            difference_header = 'DIFF'
-#            current_version_header = line[current_version_index].upper()
-#            last_version_header = line[last_version_index].upper()
-#            # Print Header.
-#            print(color + metric_name + "      " + percent_change_header.center((7)) + "       " + difference_header.center((15))  + "    " + current_version_header.center(18) + " " + last_version_header.center(18), ENDC)
-#        # Format and print stat row.
-#        elif first_item in PRINTWORTHY_STATS:
-#            stat_name = first_item.upper().center(len(max(PRINTWORTHY_STATS, key=len))).replace('_', ' ')
-#            current_version = round((line[current_version_index]), 3)
-#            last_version = round((line[last_version_index]) + 0.000, 3)
-#            difference = round(current_version - last_version, 3)
-#            if difference > 0:
-#                symbol = '+'
-#                if first_item in GO_UP_STATS:
-#                    color = TGREEN_BOLD
-#                elif first_item in GO_DOWN_STATS:
-#                    color = TRED_BOLD
-#                else:
-#                    color = TWHITE
-#            if difference < 0:
-#                symbol = '-'
-#                if first_item in GO_UP_STATS:
-#                    color = TRED_BOLD
-#                elif first_item in GO_DOWN_STATS:
-#                    color = TGREEN_BOLD
-#                else:
-#                    color = TWHITE
-#
-#            if difference == 0 :
-#                symbol, color = '+', TGREEN
-#            percent_change = round((difference / last_version)*100,2)
-#
-#            print(WHITE_BOLD + stat_name + ENDC + "     " + color + (symbol + " {:5.2f}".format(abs(percent_change)) + " %").rjust(len(percent_change_header)), ENDC + "    " + color + ("{:12.3f}".format((difference))).rjust(len(difference_header)), ENDC + "    " + "{:15.3f}".format(current_version).rjust(len(current_version_header)) + "   " + "{:15.3f}".format(last_version).rjust(len(last_version_header)) + "  ")
-#
-#    print()
-#    print()
-#    print()
-#    print("--------------------------------------------------------------------------------------------------")
-#    print()
-#
-
-
 def compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=None, masked_count=None):
     """
     This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.

From 74722385a5c5ca55df735394573ac68b2f0e2aae Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 19 Aug 2021 08:48:45 -0500
Subject: [PATCH 094/359] [1pt] PR: Function to extract HAND and hydroid values
 at point locations for SRC adjustment tool (#444)

This merges in the script, adjust_rc_with_feedback.py, that will be expanded in future issues. The primary function that performs the HAND value and hydroid extraction is ingest_points_layer() but this may change as the overall process evolves.

- Added adjust_rc_with_feedback.py with ingest_points_layer(), a function to extract HAND and hydroid values for use in an automatic synthetic rating curve updating mechanism.

This resolves #437.
---
 docs/CHANGELOG.md                |  21 ++++++
 fim_run.sh                       |   2 +-
 src/add_crosswalk.py             |   2 +-
 src/run_by_unit.sh               |   1 +
 tools/adjust_rc_with_feedback.py | 112 +++++++++++++++++++++++++++++++
 5 files changed, 136 insertions(+), 2 deletions(-)
 create mode 100644 tools/adjust_rc_with_feedback.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 05c3d149a..9efecd7c7 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,27 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.0 - 2021-08-19 - [PR #444](https://github.com/NOAA-OWP/cahaba/pull/444)
+
+This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in future issues. The primary function that performs the HAND value and hydroid extraction is ingest_points_layer() but this may change as the overall synthetic rating curve automatic update machanism evolves.
+
+## Additions
+- Added `adjust_rc_with_feedback.py` with `ingest_points_layer()`, a function to extract HAND and hydroid values for use in an automatic synthetic rating curve updating mechanism.
+
+<br/><br/>
+
+## Changes
+- Remove `Dockerfile.prod`, rename `Dockerfile.dev` to just `Dockerfile`, and remove ``.dockerignore`.
+- Clean up `Dockerfile` and remove any unused* packages or variables.
+- Remove any unused* Python packages from the `Pipfile`.
+- Move the `CHANGELOG.md`, `SECURITY.md`, and `TERMS.md` files to the `/docs` folder.
+- Remove any unused* scripts in the `/tools` and `/src` folders.
+- Move `tools/preprocess` scripts into `tools/`.
+- Ensure all scripts in the `/src` folder have their code in functions and are being called via a `__main__` function (This will help with implementing memory profiling fully).
+- Changed memory-profiling to be an option flag `-m` for `fim_run.sh`.
+- Updated FIM API to save all outputs during a "release" job.
+
+
 ## v3.0.21.0 - 2021-08-18 - [PR #433](https://github.com/NOAA-OWP/cahaba/pull/433)
 
 General repository cleanup, made memory-profiling an optional flag, API's release feature now saves outputs.
diff --git a/fim_run.sh b/fim_run.sh
index 1978004d1..2d1f8bd68 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -150,7 +150,7 @@ else
 fi
 
 # identify missing HUCs
-time python3 /foss_fim/tools/fim_completion_check.py -i $hucList -o $outputRunDataDir
+# time python3 /foss_fim/tools/fim_completion_check.py -i $hucList -o $outputRunDataDir
 
 echo "$viz"
 if [[ "$viz" -eq 1 ]]; then
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index a8107ca5b..2b352b6cd 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -228,7 +228,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
         print('Note: NOT using bathy estimation approach to modify the SRC...')
 
     # make hydroTable
-    output_hydro_table = output_src.loc[:,['HydroID','feature_id','Stage','Discharge (m3s-1)']]
+    output_hydro_table = output_src.loc[:,['HydroID','feature_id','Stage','Discharge (m3s-1)', 'HydraulicRadius (m)', 'WetArea (m2)', 'SLOPE', 'ManningN']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
 
     if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 0c732e970..07771ad71 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -400,6 +400,7 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv outp
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
+echo python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
diff --git a/tools/adjust_rc_with_feedback.py b/tools/adjust_rc_with_feedback.py
new file mode 100644
index 000000000..16e00b40a
--- /dev/null
+++ b/tools/adjust_rc_with_feedback.py
@@ -0,0 +1,112 @@
+import argparse
+import geopandas as gpd
+from geopandas.tools import sjoin
+import os
+import rasterio
+
+temp_workspace = r''
+HAND_CRS = 'EPSG:3857'
+
+def update_rating_curve(grouped_median, huc6):
+    pass
+
+
+def ingest_points_layer(points_layer, fim_directory, wbd_path):
+    
+    # Read wbd_path and points_layer to determine which HUC6 each point is in.
+    wbd_huc8_read = gpd.read_file(wbd_path, layer='WBDHU6')    
+    points_layer_read = gpd.read_file(points_layer)
+    
+    # Update CRS of points_layer_read.
+    points_layer_read = points_layer_read.to_crs(HAND_CRS)
+    wbd_huc8_read = wbd_huc8_read.to_crs(HAND_CRS)
+    
+    # Spatial join the two layers.
+    water_edge_df = sjoin(points_layer_read, wbd_huc8_read)
+        
+    # Convert to GeoDataFrame.
+    gdf = gpd.GeoDataFrame(water_edge_df)
+    
+    # Add two columns for X and Y.
+    gdf['X'] = gdf['geometry'].x
+    gdf['Y'] = gdf['geometry'].y
+            
+    # Extract information into dictionary.
+    huc6_list = []
+    for index, row in gdf.iterrows():
+        huc6 = row['HUC6']
+        if huc6 not in huc6_list:
+            huc6_list.append(huc6)
+
+    # Define coords variable to be used in point raster value attribution.
+    coords = [(x,y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
+        
+    # Define paths to relevant HUC6 HAND data.
+    for huc6 in huc6_list:
+        print(huc6)
+        
+        # Define paths to relevant HUC6 HAND data and get necessary metadata for point rasterization.
+        hand_path = os.path.join(fim_directory, huc6, 'hand_grid_' + huc6 + '.tif')
+        if not os.path.exists(hand_path):
+            print("HAND grid for " + huc6 + " does not exist.")
+            continue
+        catchments_path = os.path.join(fim_directory, huc6, 'catchments_' + huc6 + '.tif')
+        if not os.path.exists(catchments_path):
+            print("Catchments grid for " + huc6 + " does not exist.")
+            continue
+        
+#        water_edge_df = water_edge_df[water_edge_df['HUC6'] == huc6]
+        
+        # Use point geometry to determine pixel values at catchment and HAND grids.
+        hand_src = rasterio.open(hand_path)
+        water_edge_df['hand'] = [h[0] for h in hand_src.sample(coords)]
+        hand_src.close()
+        catchments_src = rasterio.open(catchments_path)
+        water_edge_df['hydroid'] = [c[0] for c in catchments_src.sample(coords)]
+                
+        # Get median HAND value for appropriate groups.
+        water_edge_median_ds = water_edge_df.groupby(["hydroid", "flow", "submitter", "coll_time", "flow_unit"])['hand'].median()
+               
+        output_csv = os.path.join(fim_directory, huc6, 'user_supplied_n_vals_' + huc6 + '.csv')
+        
+        water_edge_median_ds.to_csv(output_csv)
+                
+        # 1. Loop and find the corresponding hydroids in the Hydrotable
+        # 2. Grab slope, wetted area, hydraulic radius, and feature_id that correspond with the matching hydroids and HAND value for the nearest stage
+        # 3. Calculate new column for new roughness using the above info
+        #    3b. If multiple flows exist per hydroid, aggregate the resulting Manning Ns
+        #    3c. If range of resulting Manning Ns is high, notify human
+        # 4. Update Hydrotable
+        #    4a. Copy default flow and N columns to new columns with "_default" in the field name
+        #    4b. Overwrite the official flow and N columns with the new calculated values
+        #    4c. Add last_updated column with timestamp where values were changed, also add "submitter" column
+        # 5. What do we do in catchments that match the feature_id?
+        #    5a. If these catchments already have known data, then let it use those. If not, use new calculated Ns.
+        
+        update_rating_curve(water_edge_median_ds, huc6)
+
+
+
+if __name__ == '__main__':
+    
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Adjusts rating curve given a shapefile containing points of known water boundary.')
+    parser.add_argument('-p','--points-layer',help='Path to points layer containing known water boundary locations',required=True)
+    parser.add_argument('-d','--fim-directory',help='Parent directory of FIM-required datasets.',required=True)
+    parser.add_argument('-w','--wbd-path', help='Path to national HUC6 layer.',required=True)
+    
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+    points_layer = args['points_layer']
+    fim_directory = args['fim_directory']
+    wbd_path = args['wbd_path']
+    
+    ingest_points_layer(points_layer, fim_directory, wbd_path)
+    
+    # Open catchment, HAND, and point grids and determine pixel values for Hydroid, HAND value, and discharge value, respectively.
+    
+    # Open rating curve file(s).
+    
+    # Use three values to determine the hydroid rating curve(s) to update, then update them using a variation of Manning's Equation.
+    
+    # Ensure the JSON rating curve is updated and saved (overwitten). Consider adding attributes to document what was performed.
\ No newline at end of file

From 350424471ca6d27f1ccf7f3397cf1163695676b3 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 20 Aug 2021 12:37:02 -0500
Subject: [PATCH 095/359] Patch params_template.env

Changes max_split_distance_meters in params_template.env to 1500.

This resolves #446.
---
 config/params_template.env | 2 +-
 docs/CHANGELOG.md          | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/config/params_template.env b/config/params_template.env
index b85c92e9d..87270a669 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -7,7 +7,7 @@ export wbd_buffer=5000
 export thalweg_lateral_elev_threshold=3
 
 #### geospatial parameters ####
-export max_split_distance_meters=2000
+export max_split_distance_meters=1500
 export ms_buffer_dist=7000
 export lakes_buffer_dist_meters=20
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9efecd7c7..7899467f7 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.1 - 2021-08-20 - [PR #447](https://github.com/NOAA-OWP/cahaba/pull/447)
+
+Patches the minimum stream length in the template parameters file.
+
+## Changes
+- Changes `max_split_distance_meters` in `params_template.env` to 1500.
+
+<br/><br/>
+
 ## v3.0.22.0 - 2021-08-19 - [PR #444](https://github.com/NOAA-OWP/cahaba/pull/444)
 
 This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in future issues. The primary function that performs the HAND value and hydroid extraction is ingest_points_layer() but this may change as the overall synthetic rating curve automatic update machanism evolves.

From 3f22be0e333c03fd371c23868806bf7ab1c40c08 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Mon, 23 Aug 2021 11:52:09 -0500
Subject: [PATCH 096/359] Update CHANGELOG.md

---
 docs/CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7899467f7..0ef5aad21 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -18,7 +18,6 @@ This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in futu
 ## Additions
 - Added `adjust_rc_with_feedback.py` with `ingest_points_layer()`, a function to extract HAND and hydroid values for use in an automatic synthetic rating curve updating mechanism.
 
-<br/><br/>
 
 ## Changes
 - Remove `Dockerfile.prod`, rename `Dockerfile.dev` to just `Dockerfile`, and remove ``.dockerignore`.
@@ -31,6 +30,7 @@ This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in futu
 - Changed memory-profiling to be an option flag `-m` for `fim_run.sh`.
 - Updated FIM API to save all outputs during a "release" job.
 
+<br/><br/>
 
 ## v3.0.21.0 - 2021-08-18 - [PR #433](https://github.com/NOAA-OWP/cahaba/pull/433)
 

From 7e86e11a70678e3fe90a6aed60005e2f3a428e2d Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 25 Aug 2021 08:55:26 -0500
Subject: [PATCH 097/359] Updated README with ESIP/AWS information

---
 README.md | 115 +++++++++++++++++++++++++-----------------------------
 1 file changed, 54 insertions(+), 61 deletions(-)

diff --git a/README.md b/README.md
index d0e868ec2..880ac3f4e 100644
--- a/README.md
+++ b/README.md
@@ -4,12 +4,45 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 
 This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids. This repository also includes functionality to generate flood inundation maps (FIMs) and evaluate FIM accuracy.
 
-## Dependencies
+#### For more information, see the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki/Cahaba-Wiki-Home).
 
-[Docker](https://docs.docker.com/get-docker/)
+## Accessing Data through ESIP S3 Bucket
+The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.
+
+### Configuring the AWS CLI
+
+1. [Install AWS CLI tools](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html)
+
+2. [Configure AWS CLI tools](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html)
+
+### Accessing Data using the AWS CLI
+
+AWS Region: `US East (N. Virginia) us-east-1`
+
+AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
+
+This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If, however, you are using compute resources in the same region as the S3 Bucket, then there is no cost.
+
+#### Examples
+
+List bucket folder structure:
+```
+aws s3 ls s3://noaa-nws-owp-fim/ --request-payer requester
+```
+
+Download a directory of outputs for a HUC8:
+```
+aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/fim_3_0_21_0/outputs/fr/12090301 12090301 --request-payer requester
+```
+
+## Running the Code
+### Input Data
+Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP S3 Bucket" section above). All necessary non-publicly available files are in this S3 bucket, as well as sample input data for HUCs 1204 and 1209.
 
-## Installation
+### Dependencies
+[Docker](https://docs.docker.com/get-docker/)
 
+### Installation
 1. Install Docker : [Docker](https://docs.docker.com/get-docker/)
 2. Build Docker Image : `docker build -f Dockerfile.dev -t <image_name>:<tag> <path/to/repository>`
 3. Create FIM group on host machine:
@@ -17,8 +50,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
     - Linux: `chgrp -R fim <path/to/repository>`
 
-## Configuration
-
+### Configuration
 This software is configurable via parameters found in the `config` directory. Copy files before editing and remove "template" pattern from the filename.
 Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
 - `params_template.env`
@@ -27,41 +59,17 @@ Make sure to set the config folder group to 'fim' recursively using the chown co
 - `params_calibrated.env`
     - runs calibrated mannings parameters from `mannings_calibrated.json`
 
-----
-## Input Data
-
-The following input data sources should be downloaded and preprocessed prior to executing the preprocessing & hydrofabric generation code:
-### USACE National Levee Database:
-- Access here: https://levees.sec.usace.army.mil/
-- Download the “Full GeoJSON” file for the area of interest
-- Unzip data and then use the preprocessing scripts to filter data and fix geometries where needed
-
-### NHDPlus HR datasets
-- `agg_nhd_streams_adj.gpkg`
-- `agg_nhd_headwaters_adj.gpkg`
-- `elev_m.tif`
-
-**Please note:** For the following datasets, please contact Mark Glaudemans (mark.glaudemans@noaa.gov). We are currently working on a long-term data sharing solution for the in-house NOAA data.
-
-### NWM Hydrofabric
-- `nwm_flows.gpkg`
-- `nwm_catchments.gpkg`
-- `nwm_lakes.gpkg`
-- `nwm_headwaters.gpkg`
-
-### AHPS Site Locations (For Mainstem Configuration)
-- `nws_lid.gpkg`
-- `ms_segs.gpkg`
-
-----
-## Usage
-
-### Run Docker Container
+### Produce HAND Hydrofabric
 ```
-docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <image_name>:<tag>
+fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
 ```
+- `-u` can be a single huc, a series passed in quotes, or a line-delimited file
+    i. To run entire domain of available data use one of the ```/data/inputs/included_huc[4,6,8].lst``` files
+- Outputs can be found under ```/data/outputs/<name_your_run>```
+
+### Testing in Other HUCs
+To test in HUCs other than the provided HUCs, the following processes can be followed to acquire and preprocess additional NHDPlus rasters and vectors. After these steps are run, the "Produce HAND Hydrofabric" step can be run for the new HUCs.
 
-### Acquire and Prepare Data
 ```
 /foss_fim/src/acquire_and_preprocess_inputs.py -u <huc4s_to_process>
 ```
@@ -69,29 +77,14 @@ docker run --rm -it -v <path/to/data>:/data -v <path/to/repository>:/foss_fim <i
 - Please run `/foss_fim/src/acquire_and_preprocess_inputs.py --help` for more information.
 - See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
 
-### Aggregate NHDHR Streams and Create NWM Headwater Points
-```
-/foss_fim/src/aggregate_vector_inputs.py
-```
-
-### Reproject NHDHR Rasters and Convert to Meters.
+#### Reproject NHDHR Rasters and Convert to Meters.
 ```
 /foss_fim/src/preprocess_rasters.py
 ```
 
-### Produce Hydrofabric
-```
-fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
-```
-- `-u` can be a single huc, a series passed in quotes, or a line-delimited file
-    i. To run entire domain of available data use one of the ```/data/inputs/included_huc[4,6,8].lst``` files
-- Outputs can be found under ```/data/outputs/<name_your_run>```
-
 ----
-## Evaluating Inundation Map Performance
-After `fim_run.sh` completes, you are ready to evaluate the model's skill.
-
-**Please note:** You will need access to the test_cases benchmark data. Please contact Mark Glaudemans for access to these data (mark.glaudemans@noaa.gov). As mentioned before, a long term data sharing solution is still in the works.
+### Evaluating Inundation Map Performance
+After `fim_run.sh` completes, you can evaluate the model's skill. The evaluation benchmark datasets are available through ESIP in the `test_cases` directory.
 
 To evaluate model skill, run the following:
 ```
@@ -104,7 +97,7 @@ python /foss_fim/tools/synthesize_test_cases.py --help
 ```
 
 ----
-## Dependencies
+### Managing Dependencies
 
 Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/). To add new dependencies, from the projects's top-level directory:
 
@@ -123,19 +116,19 @@ and include both `Pipfile` and `Pipfile.lock` in your commits. The docker image
 If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
 
 ----
-## Known Issues & Getting Help
+### Known Issues & Getting Help
 
-Please see the issue tracker on GitHub for known issues and for getting help.
+Please see the issue tracker on GitHub and the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki/Cahaba-Wiki-Home) for known issues and getting help.
 
-## Getting Involved
+### Getting Involved
 
 NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
-## Open Source Licensing Info
+### Open Source Licensing Info
 1. [TERMS](docs/TERMS.md)
 2. [LICENSE](LICENSE)
 
-## Credits and References
+### Credits and References
 1. Office of Water Prediction [(OWP)](https://water.noaa.gov/)
 2. National Flood Interoperability Experiment [(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)
 3. Garousi‐Nejad, I., Tarboton, D. G.,Aboutalebi, M., & Torres‐Rua, A.(2019). Terrain analysis enhancements to the Height Above Nearest Drainage flood inundation mapping method. Water Resources Research, 55 , 7983–8009. https://doi.org/10.1029/2019WR0248375.

From a75eee66f1bbb4bcd7699ef0a9c2d5950f475be1 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 25 Aug 2021 08:59:46 -0500
Subject: [PATCH 098/359] Update README.md

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 880ac3f4e..eac8b5bcd 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,10 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 ## Accessing Data through ESIP S3 Bucket
 The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.
 
+AWS Region: `US East (N. Virginia) us-east-1`
+
+AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
+
 ### Configuring the AWS CLI
 
 1. [Install AWS CLI tools](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html)
@@ -17,10 +21,6 @@ The latest national generated HAND data and a subset of the inputs can be found
 
 ### Accessing Data using the AWS CLI
 
-AWS Region: `US East (N. Virginia) us-east-1`
-
-AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
-
 This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If, however, you are using compute resources in the same region as the S3 Bucket, then there is no cost.
 
 #### Examples

From 6578709c01e67360f9dafce83b27a4bd1adfabe6 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 25 Aug 2021 09:00:11 -0500
Subject: [PATCH 099/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index eac8b5bcd..3c34b2d60 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
 
 ### Accessing Data using the AWS CLI
 
-This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If, however, you are using compute resources in the same region as the S3 Bucket, then there is no cost.
+This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If you are using compute resources in the same region as the S3 Bucket, then there is no cost.
 
 #### Examples
 

From dc2a9d4837eb7e8b7abfcbd5877616a32cbdc1d6 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 25 Aug 2021 14:27:19 -0500
Subject: [PATCH 100/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3c34b2d60..2faddad01 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,7 @@ Make sure to set the config folder group to 'fim' recursively using the chown co
 
 ### Produce HAND Hydrofabric
 ```
-fim_run.sh -u <huc4,6,or8s> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
+fim_run.sh -u <huc8> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
 ```
 - `-u` can be a single huc, a series passed in quotes, or a line-delimited file
     i. To run entire domain of available data use one of the ```/data/inputs/included_huc[4,6,8].lst``` files

From eebfa79c88701ce1094e09ac5d2a2af347dc2a51 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 25 Aug 2021 14:28:06 -0500
Subject: [PATCH 101/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2faddad01..28c1a3a94 100644
--- a/README.md
+++ b/README.md
@@ -77,7 +77,7 @@ To test in HUCs other than the provided HUCs, the following processes can be fol
 - Please run `/foss_fim/src/acquire_and_preprocess_inputs.py --help` for more information.
 - See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
 
-#### Reproject NHDHR Rasters and Convert to Meters.
+#### Reproject NHDPlus High-Res Rasters and Convert to Meters.
 ```
 /foss_fim/src/preprocess_rasters.py
 ```

From aaed3cc415ece64fba47e7387d2f2b281d5a812e Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Thu, 26 Aug 2021 13:10:00 -0500
Subject: [PATCH 102/359] Disable BARC for specified stream order

This merge addresses an issues with the bathymetry adjusted rating curve (BARC) calculations exacerbating single-pixel inundation issues for the lower Mississippi River. This fix allows the user to specify a stream order value that will be ignored in BARC calculations (reverts to using the original/default rating curve). If/when the "thalweg notch" issue is addressed, this change may be unmade.

- Added new env variable ignore_streamorders set to 10.
- Added new BARC code to set the bathymetry adjusted cross-section area to 0 (reverts to using the default SRC values) based on the streamorder env variable.

This resolves #454.
---
 config/params_calibrated.env | 13 +++++++------
 config/params_template.env   | 13 +++++++------
 docs/CHANGELOG.md            | 22 ++++++++++------------
 src/bathy_rc_adjust.py       | 14 +++++++++-----
 4 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/config/params_calibrated.env b/config/params_calibrated.env
index 3ca30650e..f622949ec 100644
--- a/config/params_calibrated.env
+++ b/config/params_calibrated.env
@@ -23,12 +23,13 @@ export min_stream_length=0.5
 #### bathy SRC estimation parameters ####
 export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
 #export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
-export bathy_src_modification=True
-export surf_area_thalweg_ratio_flag=10
-export thalweg_stg_search_max_limit=3
-export bathy_xs_area_chg_flag=1
-export bankful_xs_area_ratio_flag=10
-export thalweg_hyd_radius_flag=10
+export bathy_src_modification=True # Toggle to turn BARC on/off (True=on; False=off)
+export surf_area_thalweg_ratio_flag=10 # Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
+export thalweg_stg_search_max_limit=3 # Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
+export bathy_xs_area_chg_flag=1 # Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
+export bankful_xs_area_ratio_flag=10 # Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
+export thalweg_hyd_radius_flag=10 # Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
+export ignore_streamorders=10 # Ignore BARC calculations for streamorders >= this value (10 is Mississippi R)
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/config/params_template.env b/config/params_template.env
index 87270a669..90a4ae305 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -23,12 +23,13 @@ export min_stream_length=0.5
 #### bathy SRC estimation parameters ####
 export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
 #export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
-export bathy_src_modification=True
-export surf_area_thalweg_ratio_flag=10
-export thalweg_stg_search_max_limit=3
-export bathy_xs_area_chg_flag=1
-export bankful_xs_area_ratio_flag=10
-export thalweg_hyd_radius_flag=10
+export bathy_src_modification=True # Toggle to turn BARC on/off (True=on; False=off)
+export surf_area_thalweg_ratio_flag=10 # Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
+export thalweg_stg_search_max_limit=3 # Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
+export bathy_xs_area_chg_flag=1 # Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
+export bankful_xs_area_ratio_flag=10 # Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
+export thalweg_hyd_radius_flag=10 # Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
+export ignore_streamorders=10 # Ignore BARC calculations for streamorders >= this value (10 is Mississippi R)
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0ef5aad21..e1d1486c0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.2 - 2021-08-26 - [PR #455](https://github.com/NOAA-OWP/cahaba/pull/455)
+
+This merge addresses an issues with the bathymetry adjusted rating curve (BARC) calculations exacerbating single-pixel inundation issues for the lower Mississippi River. This fix allows the user to specify a stream order value that will be ignored in BARC calculations (reverts to using the original/default rating curve). If/when the "thalweg notch" issue is addressed, this change may be unmade.
+
+## Changes
+- Added new env variable `ignore_streamorders` set to 10.
+- Added new BARC code to set the bathymetry adjusted cross-section area to 0 (reverts to using the default SRC values) based on the streamorder env variable.
+
+<br/><br/>
+
 ## v3.0.22.1 - 2021-08-20 - [PR #447](https://github.com/NOAA-OWP/cahaba/pull/447)
 
 Patches the minimum stream length in the template parameters file.
@@ -18,18 +28,6 @@ This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in futu
 ## Additions
 - Added `adjust_rc_with_feedback.py` with `ingest_points_layer()`, a function to extract HAND and hydroid values for use in an automatic synthetic rating curve updating mechanism.
 
-
-## Changes
-- Remove `Dockerfile.prod`, rename `Dockerfile.dev` to just `Dockerfile`, and remove ``.dockerignore`.
-- Clean up `Dockerfile` and remove any unused* packages or variables.
-- Remove any unused* Python packages from the `Pipfile`.
-- Move the `CHANGELOG.md`, `SECURITY.md`, and `TERMS.md` files to the `/docs` folder.
-- Remove any unused* scripts in the `/tools` and `/src` folders.
-- Move `tools/preprocess` scripts into `tools/`.
-- Ensure all scripts in the `/src` folder have their code in functions and are being called via a `__main__` function (This will help with implementing memory profiling fully).
-- Changed memory-profiling to be an option flag `-m` for `fim_run.sh`.
-- Updated FIM API to save all outputs during a "release" job.
-
 <br/><br/>
 
 ## v3.0.21.0 - 2021-08-18 - [PR #433](https://github.com/NOAA-OWP/cahaba/pull/433)
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
index 524e75c46..4a4932207 100755
--- a/src/bathy_rc_adjust.py
+++ b/src/bathy_rc_adjust.py
@@ -3,11 +3,12 @@
 from os import environ
 import pandas as pd
 
-sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag']) #10x
-thal_stg_limit = float(environ['thalweg_stg_search_max_limit']) #3m
-bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag']) #10x
-bathy_xsarea_flag = float(environ['bathy_xs_area_chg_flag']) #1x
-thal_hyd_radius_flag = float(environ['thalweg_hyd_radius_flag']) #10x
+sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag']) #10x --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
+thal_stg_limit = float(environ['thalweg_stg_search_max_limit']) #3m --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
+bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag']) #10x --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
+bathy_xsarea_flag = float(environ['bathy_xs_area_chg_flag']) #1x --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
+thal_hyd_radius_flag = float(environ['thalweg_hyd_radius_flag']) #10x --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
+ignore_streamorder = int(environ['ignore_streamorders']) #10 --> Do not perform BARC for streamorders >= provided value
 
 
 def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,):
@@ -122,6 +123,9 @@ def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,ou
         ## Merge bathy_calc_xs_area to the modified_src_base
         modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area']],how='left',on='HydroID')
 
+        ## Mask/null the bathy calculated area for streamorders that the user wants to ignore (set bathy_cals_xs_area = 0 for streamorder = 10)
+        modified_src_base['bathy_calc_xs_area'].mask(modified_src_base['order_'] >= ignore_streamorder,0.0,inplace=True)
+
         ## Calculate new bathy adjusted channel geometry variables
         modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'Discharge (m3s-1)_nobathy'})
         modified_src_base['XS Area (m2)_bathy_adj'] = modified_src_base['XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']

From f386a166db4d0fd9d1f2c8bf63e78494351860f5 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 27 Aug 2021 08:46:30 -0500
Subject: [PATCH 103/359] Fix two CatFIM bugs

This fixes a bug in the get_metadata() function in /tools/tools_shared_functions.py that arose because of a WRDS update. Previously the metadata_source response was returned as independent variables, but now it is are returned a list of strings. Another issue was observed where the EVALUATED_SITES_CSV variable was being misdefined (at least on the development VM) through the OS environmental variable setting.

- Changed parsing of WRDS metadata_sources to account for new list type.
- Changed the way the EVALUATED_SITES_CSVpath is defined from OS environmental setting to a relative path that will work within Docker container.

This resolves #445.
---
 docs/CHANGELOG.md                         | 10 +++++++
 tools/generate_categorical_fim.py         |  4 +--
 tools/generate_categorical_fim_flows.py   |  9 ++++---
 tools/generate_categorical_fim_mapping.py | 32 +++++++++++------------
 tools/tools_shared_functions.py           | 12 ++++++++-
 5 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e1d1486c0..c68ecc579 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.3 - 2021-08-27 - [PR #457](https://github.com/NOAA-OWP/cahaba/pull/457)
+
+This fixes a bug in the `get_metadata()` function in `/tools/tools_shared_functions.py` that arose because of a WRDS update. Previously the `metadata_source` response was returned as independent variables, but now it is are returned a list of strings. Another issue was observed where the `EVALUATED_SITES_CSV` variable was being misdefined (at least on the development VM) through the OS environmental variable setting.
+
+## Changes
+- Changed parsing of WRDS `metadata_sources` to account for new list type.
+- Changed the way the `EVALUATED_SITES_CSV`path is defined from OS environmental setting to a relative path that will work within Docker container.
+
+<br/><br/>
+
 ## v3.0.22.2 - 2021-08-26 - [PR #455](https://github.com/NOAA-OWP/cahaba/pull/455)
 
 This merge addresses an issues with the bathymetry adjusted rating curve (BARC) calculations exacerbating single-pixel inundation issues for the lower Mississippi River. This fix allows the user to specify a stream order value that will be ignored in BARC calculations (reverts to using the original/default rating curve). If/when the "thalweg notch" issue is addressed, this change may be unmade.
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index f1ea5ecab..bcd6fc14d 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -84,8 +84,8 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     fim_version_folder = os.path.basename(fim_version)
     output_flows_dir = Path(f'/data/catfim/{fim_version_folder}/flows')
     output_mapping_dir = Path(f'/data/catfim/{fim_version_folder}/mapping')
-    nwm_us_search = '10'
-    nwm_ds_search = '10'
+    nwm_us_search = '5'
+    nwm_ds_search = '5'
     write_depth_tiff = False
 
     ## Run CatFIM scripts in sequence
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index 1648f073a..d2f5f0501 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -10,14 +10,15 @@
 sys.path.append('/foss_fim/src')
 from utils.shared_variables import VIZ_PROJECTION
 
+EVALUATED_SITES_CSV = r'/data/inputs/ahps_sites/evaluated_ahps_sites.csv'
+
 
 def get_env_paths():
     load_dotenv()
     #import variables from .env file
     API_BASE_URL = os.getenv("API_BASE_URL")
-    EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
     WBD_LAYER = os.getenv("WBD_LAYER")
-    return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER
+    return API_BASE_URL, WBD_LAYER
 
 
 def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
@@ -81,6 +82,8 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     #Get all possible mainstem segments
     print('Getting list of mainstem segments')
     #Import list of evaluated sites
+    print(EVALUATED_SITES_CSV)
+    print(os.path.exists(EVALUATED_SITES_CSV))
     list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
     #The entire routine to get mainstems is hardcoded in this function.
     ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
@@ -232,5 +235,5 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     args = vars(parser.parse_args())
 
     #Run get_env_paths and static_flow_lids
-    API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER = get_env_paths()
+    API_BASE_URL, WBD_LAYER = get_env_paths()
     generate_catfim_flows(**args)
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index ebe76cd61..0827d3f08 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -114,16 +114,16 @@ def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, n
 
 def run_inundation(args):
 
-    rem                 = args[0]
-    catchments          = args[1]
+    rem = args[0]
+    catchments = args[1]
     magnitude_flows_csv = args[2]
-    huc                 = args[3]
-    hydroTable          = args[4]
-    output_extent_grid  = args[5]
-    output_depth_grid   = args[6]
-    ahps_site           = args[7]
-    magnitude           = args[8]
-    log_file            = args[9]
+    huc = args[3]
+    hydroTable = args[4]
+    output_extent_grid = args[5]
+    output_depth_grid = args[6]
+    ahps_site = args[7]
+    magnitude = args[8]
+    log_file = args[9]
 
     try:
         inundate(rem,catchments,catchment_poly,hydroTable,magnitude_flows_csv,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
@@ -146,6 +146,7 @@ def run_inundation(args):
         with open(log_file, 'a+') as f:
             f.write('FAILURE_huc_{}:{}:{} map failed to create\n'.format(huc,ahps_site,magnitude))
 
+
 def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_attributes_filename, log_file):
 
     # Create workspace
@@ -223,12 +224,12 @@ def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_att
 def reformat_inundation_maps(args):
 
     try:
-        lid                         = args[0]
-        grid_path                   = args[1]
-        gpkg_dir                    = args[2]
-        fim_version                 = args[3]
-        huc                         = args[4]
-        magnitude                   = args[5]
+        lid = args[0]
+        grid_path = args[1]
+        gpkg_dir = args[2]
+        fim_version = args[3]
+        huc = args[4]
+        magnitude = args[5]
         nws_lid_attributes_filename = args[6]
 
         # Convert raster to to shapes
@@ -303,7 +304,6 @@ def reformat_inundation_maps(args):
     number_of_jobs = int(args['number_of_jobs'])
     depthtif = args['write_depth_tiff']
 
-
     # Create output directory
     if not os.path.exists(output_cat_fim_dir):
         os.mkdir(output_cat_fim_dir)
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 9525ed9f8..6cf0e7685 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -616,7 +616,17 @@ def get_metadata(metadata_url, select_by, selector, must_include = None, upstrea
         #Add timestamp of WRDS retrieval
         timestamp = response.headers['Date']
         #Add timestamp of sources retrieval
-        nrldb_timestamp, nwis_timestamp = metadata_json['data_sources']['metadata_sources']        
+        timestamp_list = metadata_json['data_sources']['metadata_sources']
+        
+        # Default timestamps to "Not available" and overwrite with real values if possible.
+        nwis_timestamp, nrldb_timestamp = "Not available", "Not available"
+        for timestamp in timestamp_list:
+            if "NWIS" in timestamp:
+                nwis_timestamp = timestamp
+            if "NRLDB" in timestamp:
+                nrldb_timestamp = timestamp
+            
+#        nrldb_timestamp, nwis_timestamp = metadata_json['data_sources']['metadata_sources']        
         #get crosswalk info (always last dictionary in list)
         crosswalk_info = metadata_json['data_sources']
         #Update each dictionary with timestamp and crosswalk info also save to DataFrame.

From 6f6f8f9c96ad0134d1267076f060e95ed845c6cc Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 27 Aug 2021 08:50:50 -0500
Subject: [PATCH 104/359] Update CHANGELOG.md

---
 docs/CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c68ecc579..d307628ce 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -4,7 +4,7 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 ## v3.0.22.3 - 2021-08-27 - [PR #457](https://github.com/NOAA-OWP/cahaba/pull/457)
 
-This fixes a bug in the `get_metadata()` function in `/tools/tools_shared_functions.py` that arose because of a WRDS update. Previously the `metadata_source` response was returned as independent variables, but now it is are returned a list of strings. Another issue was observed where the `EVALUATED_SITES_CSV` variable was being misdefined (at least on the development VM) through the OS environmental variable setting.
+This fixes a bug in the `get_metadata()` function in `/tools/tools_shared_functions.py` that arose because of a WRDS update. Previously the `metadata_source` response was returned as independent variables, but now it is returned a list of strings. Another issue was observed where the `EVALUATED_SITES_CSV` variable was being misdefined (at least on the development VM) through the OS environmental variable setting.
 
 ## Changes
 - Changed parsing of WRDS `metadata_sources` to account for new list type.

From eb8b33f36a4733bc09b700739d3982cf824618dc Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 27 Aug 2021 08:52:10 -0500
Subject: [PATCH 105/359] Update CHANGELOG.md

---
 docs/CHANGELOG.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d307628ce..dff559c8e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -7,8 +7,8 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 This fixes a bug in the `get_metadata()` function in `/tools/tools_shared_functions.py` that arose because of a WRDS update. Previously the `metadata_source` response was returned as independent variables, but now it is returned a list of strings. Another issue was observed where the `EVALUATED_SITES_CSV` variable was being misdefined (at least on the development VM) through the OS environmental variable setting.
 
 ## Changes
-- Changed parsing of WRDS `metadata_sources` to account for new list type.
-- Changed the way the `EVALUATED_SITES_CSV`path is defined from OS environmental setting to a relative path that will work within Docker container.
+- In `tools_shared_functions.py`, changed parsing of WRDS `metadata_sources` to account for new list type.
+- In `generate_categorical_fim_flows.py`, changed the way the `EVALUATED_SITES_CSV` path is defined from OS environmental setting to a relative path that will work within Docker container.
 
 <br/><br/>
 

From 63878219efe11bba08a2e1efff6c3736f25ec360 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Mon, 30 Aug 2021 08:41:32 -0500
Subject: [PATCH 106/359] Rename BARC variables

Renames the BARC modified variables that are exported to src_full_crosswalked.csv to replace the original variables. The default/original variables are renamed with orig_ prefix. This change is needed to ensure downstream uses of the src_full_crosswalked.csv are able to reference the authoritative version of the channel geometry variables (i.e. BARC-adjust where available).

- In src_full_crosswalked.csv, default/original variables are renamed with orig_ prefix and SA_div is renamed to SA_div_flag.

This resolves #418.
---
 docs/CHANGELOG.md      |  9 +++++++++
 src/bathy_rc_adjust.py | 28 ++++++++++++++--------------
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index dff559c8e..1a7923116 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.4 - 2021-08-30 - [PR #456](https://github.com/NOAA-OWP/cahaba/pull/456)
+
+Renames the BARC modified variables that are exported to `src_full_crosswalked.csv` to replace the original variables. The default/original variables are renamed with `orig_` prefix. This change is needed to ensure downstream uses of the `src_full_crosswalked.csv` are able to reference the authoritative version of the channel geometry variables (i.e. BARC-adjust where available).
+
+## Changes
+- In `src_full_crosswalked.csv`, default/original variables are renamed with `orig_` prefix and `SA_div` is renamed to `SA_div_flag`.
+
+<br/><br/>
+
 ## v3.0.22.3 - 2021-08-27 - [PR #457](https://github.com/NOAA-OWP/cahaba/pull/457)
 
 This fixes a bug in the `get_metadata()` function in `/tools/tools_shared_functions.py` that arose because of a WRDS update. Previously the `metadata_source` response was returned as independent variables, but now it is returned a list of strings. Another issue was observed where the `EVALUATED_SITES_CSV` variable was being misdefined (at least on the development VM) through the OS environmental variable setting.
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
index 4a4932207..7b942929e 100755
--- a/src/bathy_rc_adjust.py
+++ b/src/bathy_rc_adjust.py
@@ -29,13 +29,13 @@ def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,ou
     else:
         ## Use SurfaceArea variable to identify thalweg-restricted stage values for each hydroid
         ## Calculate the interrow SurfaceArea ratio n/(n-1)
-        modified_src_base['SA_div'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
-        ## Mask SA_div when Stage = 0 or when the SA_div value (n / n-1) is > threshold value (i.e. 10x)
-        modified_src_base['SA_div'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
+        modified_src_base['SA_div_flag'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
+        ## Mask SA_div_flag when Stage = 0 or when the SA_div_flag value (n / n-1) is > threshold value (i.e. 10x)
+        modified_src_base['SA_div_flag'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div_flag']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
         ## Create new df to filter and groupby HydroID
-        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div']]
+        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div_flag']]
         find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['Stage']<thal_stg_limit] # assuming thalweg burn-in is less than 3 meters
-        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div'].notnull()]
+        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div_flag'].notnull()]
         find_thalweg_notch = find_thalweg_notch.loc[find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()].reset_index(drop=True)
         ## Assign thalweg_burn_elev variable to the stage value found in previous step
         find_thalweg_notch['Thalweg_burn_elev'] = find_thalweg_notch['Stage']
@@ -127,18 +127,18 @@ def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,ou
         modified_src_base['bathy_calc_xs_area'].mask(modified_src_base['order_'] >= ignore_streamorder,0.0,inplace=True)
 
         ## Calculate new bathy adjusted channel geometry variables
-        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'Discharge (m3s-1)_nobathy'})
-        modified_src_base['XS Area (m2)_bathy_adj'] = modified_src_base['XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
-        modified_src_base['Volume (m3)_bathy_adj'] = modified_src_base['XS Area (m2)_bathy_adj'] * modified_src_base['LENGTHKM'] * 1000
-        modified_src_base['WetArea (m2)_bathy_adj'] = modified_src_base['Volume (m3)_bathy_adj']/modified_src_base['LENGTHKM']/1000
-        modified_src_base['HydraulicRadius (m)_bathy_adj'] = modified_src_base['WetArea (m2)_bathy_adj']/modified_src_base['WettedPerimeter (m)']
-        modified_src_base['HydraulicRadius (m)_bathy_adj'].fillna(0, inplace=True)
+        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'orig_Discharge (m3s-1)','XS Area (m2)':'orig_XS Area (m2)','Volume (m3)':'orig_Volume (m3)','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'})
+        modified_src_base['XS Area (m2)'] = modified_src_base['orig_XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
+        modified_src_base['Volume (m3)'] = modified_src_base['XS Area (m2)'] * modified_src_base['LENGTHKM'] * 1000
+        modified_src_base['WetArea (m2)'] = modified_src_base['Volume (m3)']/modified_src_base['LENGTHKM']/1000
+        modified_src_base['HydraulicRadius (m)'] = modified_src_base['WetArea (m2)']/modified_src_base['WettedPerimeter (m)']
+        modified_src_base['HydraulicRadius (m)'].fillna(0, inplace=True)
         ## mask out negative top width differences (avoid thalweg burn notch)
-        modified_src_base['HydraulicRadius (m)_bathy_adj'].mask((modified_src_base['HydraulicRadius (m)_bathy_adj']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
+        modified_src_base['HydraulicRadius (m)'].mask((modified_src_base['HydraulicRadius (m)']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
 
         ## Calculate Q using Manning's equation
-        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)_bathy_adj']* \
-        pow(modified_src_base['HydraulicRadius (m)_bathy_adj'],2.0/3)* \
+        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)']* \
+        pow(modified_src_base['HydraulicRadius (m)'],2.0/3)* \
         pow(modified_src_base['SLOPE'],0.5)/modified_src_base['ManningN']
         ## mask discharge values for stage = 0 rows in SRC (replace with 0) --> do we need SRC to start at 0??
         modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0,0,inplace=True)

From aaca7d24690ed4947d4fbe2f3b795a9a8ed938d9 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 1 Sep 2021 08:41:34 -0500
Subject: [PATCH 107/359] Update README.md

---
 README.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 28c1a3a94..2e1bcf1d8 100644
--- a/README.md
+++ b/README.md
@@ -129,14 +129,14 @@ NOAA's National Water Center welcomes anyone to contribute to the Cahaba reposit
 2. [LICENSE](LICENSE)
 
 ### Credits and References
-1. Office of Water Prediction [(OWP)](https://water.noaa.gov/)
-2. National Flood Interoperability Experiment [(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)
-3. Garousi‐Nejad, I., Tarboton, D. G.,Aboutalebi, M., & Torres‐Rua, A.(2019). Terrain analysis enhancements to the Height Above Nearest Drainage flood inundation mapping method. Water Resources Research, 55 , 7983–8009. https://doi.org/10.1029/2019WR0248375.
-4. Zheng, X., D.G. Tarboton, D.R. Maidment, Y.Y. Liu, and P. Passalacqua. 2018. “River Channel Geometry and Rating Curve Estimation Using Height above the Nearest Drainage.” Journal of the American Water Resources Association 54 (4): 785–806. https://doi.org/10.1111/1752-1688.12661.
-5. Liu, Y. Y., D. R. Maidment, D. G. Tarboton, X. Zheng and S. Wang, (2018), "A CyberGIS Integration and Computation Framework for High-Resolution Continental-Scale Flood Inundation Mapping," JAWRA Journal of the American Water Resources Association, 54(4): 770-784, https://doi.org/10.1111/1752-1688.12660.
-6. Barnes, Richard. 2016. RichDEM: Terrain Analysis Software. http://github.com/r-barnes/richdem
+1. [Office of Water Prediction (OWP)](https://water.noaa.gov/)
+2. [National Flood Interoperability Experiment(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)
+3. Garousi‐Nejad, I., Tarboton, D. G.,Aboutalebi, M., & Torres‐Rua, A.(2019). Terrain analysis enhancements to the Height Above Nearest Drainage flood inundation mapping method. Water Resources Research, 55 , 7983–8009.
+4. [Zheng, X., D.G. Tarboton, D.R. Maidment, Y.Y. Liu, and P. Passalacqua. 2018. “River Channel Geometry and Rating Curve Estimation Using Height above the Nearest Drainage.” Journal of the American Water Resources Association 54 (4): 785–806.](https://doi.org/10.1111/1752-1688.12661)
+5. [Liu, Y. Y., D. R. Maidment, D. G. Tarboton, X. Zheng and S. Wang, (2018), "A CyberGIS Integration and Computation Framework for High-Resolution Continental-Scale Flood Inundation Mapping," JAWRA Journal of the American Water Resources Association, 54(4): 770-784.](https://doi.org/10.1111/1752-1688.12660)
+6. [Barnes, Richard. 2016. RichDEM: Terrain Analysis Software](http://github.com/r-barnes/richdem)
 7. [TauDEM](https://github.com/dtarb/TauDEM)
-8. Federal Emergency Management Agency (FEMA) Base Level Engineering [(BLE)](https://webapps.usgs.gov/infrm/estBFE/)
-9. Verdin, James; Verdin, Kristine; Mathis, Melissa; Magadzire, Tamuka; Kabuchanga, Eric; Woodbury, Mark; and Gadain, Hussein, 2016, A software tool for rapid flood inundation mapping: U.S. Geological Survey Open-File Report 2016–1038, 26 p., http://dx.doi.org/10.3133/ofr20161038.
-10. United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR). https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution
-11. Esri Arc Hydro. https://www.esri.com/library/fliers/pdfs/archydro.pdf
+8. [Federal Emergency Management Agency (FEMA) Base Level Engineering (BLE)](https://webapps.usgs.gov/infrm/estBFE/)
+9. [Verdin, James; Verdin, Kristine; Mathis, Melissa; Magadzire, Tamuka; Kabuchanga, Eric; Woodbury, Mark; and Gadain, Hussein, 2016, A software tool for rapid flood inundation mapping: U.S. Geological Survey Open-File Report 2016–1038, 26](http://dx.doi.org/10.3133/ofr20161038)
+10. [United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR)](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution)
+11. [Esri Arc Hydro](https://www.esri.com/library/fliers/pdfs/archydro.pdf)

From cf3ddcc63544d0175e40bb674edcd2419a627f8e Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 1 Sep 2021 08:42:58 -0500
Subject: [PATCH 108/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2e1bcf1d8..727134262 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ This software is configurable via parameters found in the `config` directory. Co
 Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
 - `params_template.env`
 - `mannings_default.json`
-    - must change filepath in `params_template.env` under "manning_n" variable name
+    - must change filepath in `params_template.env` in `manning_n` variable name
 - `params_calibrated.env`
     - runs calibrated mannings parameters from `mannings_calibrated.json`
 

From ae63c20b04189c3fb618be1a94311ae0549344bc Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 1 Sep 2021 14:46:50 -0500
Subject: [PATCH 109/359] Update CHANGELOG.md

---
 docs/CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1a7923116..21e9c7134 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -116,7 +116,7 @@ Updating logging and fixing bug in vector preprocessing.
 
 ## Changes
 - Provides unique exit codes to relevant domain checkpoints within `run_by_unit.sh`.
-- Bug fixes in `reduce_nhd_stream_density.py`, 'mprof plot` call.
+- Bug fixes in `reduce_nhd_stream_density.py`, `mprof plot` call.
 - Improved error handling in `add_crosswalk.py`.
 
 <br/><br/>

From 390e9cd4fbdf690209ab0d8c653876bea32e9d32 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 8 Sep 2021 14:11:04 -0500
Subject: [PATCH 110/359] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 727134262..24cee3ce7 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 
 This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids. This repository also includes functionality to generate flood inundation maps (FIMs) and evaluate FIM accuracy.
 
-#### For more information, see the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki/Cahaba-Wiki-Home).
+#### For more information, see the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki).
 
 ## Accessing Data through ESIP S3 Bucket
 The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.

From 09fe961023c004c0874ddc5037865b0ceb8297a4 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 8 Sep 2021 15:28:36 -0500
Subject: [PATCH 111/359] Evaluation patch

Patches an issue where only certain benchmark categories were being used in evaluation.

- In tools/tools_shared_variables.py, created a variable MAGNITUDE_DICT to store benchmark category magnitudes.
- synthesize_test_cases.py imports MAGNITUDE_DICT and uses it to assign magnitudes.
---
 docs/CHANGELOG.md               | 10 ++++++++++
 tools/synthesize_test_cases.py  | 25 ++++++++++---------------
 tools/tools_shared_variables.py |  3 +++
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 21e9c7134..433d8b82f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.5 - 2021-09-08 - [PR #460](https://github.com/NOAA-OWP/cahaba/pull/460)
+
+Patches an issue where only certain benchmark categories were being used in evaluation.
+
+## Changes
+- In `tools/tools_shared_variables.py`, created a variable `MAGNITUDE_DICT` to store benchmark category magnitudes.
+- `synthesize_test_cases.py` imports `MAGNITUDE_DICT` and uses it to assign magnitudes.
+
+<br/><br/>
+
 ## v3.0.22.4 - 2021-08-30 - [PR #456](https://github.com/NOAA-OWP/cahaba/pull/456)
 
 Renames the BARC modified variables that are exported to `src_full_crosswalked.csv` to replace the original variables. The default/original variables are renamed with `orig_` prefix. This change is needed to ensure downstream uses of the `src_full_crosswalked.csv` are able to reference the authoritative version of the channel geometry variables (i.e. BARC-adjust where available).
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 98a42e9c6..e49b1519d 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -7,7 +7,7 @@
 import csv
 
 from run_test_case import run_alpha_test
-from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, BLE_MAGNITUDE_LIST, IFC_MAGNITUDE_LIST
+from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT
 
 
 def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list):
@@ -83,9 +83,9 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         if benchmark_source in ['ble', 'ifc']:
             
             if benchmark_source == 'ble':
-                magnitude_list = BLE_MAGNITUDE_LIST
+                magnitude_list = MAGNITUDE_DICT['ble']
             if benchmark_source == 'ifc':
-                magnitude_list = IFC_MAGNITUDE_LIST
+                magnitude_list = MAGNITUDE_DICT['ifc']
             test_cases_list = os.listdir(benchmark_test_case_dir)
 
             for test_case in test_cases_list:
@@ -298,7 +298,7 @@ def process_alpha_test(args):
     # Loop through benchmark categories.
     procs_list = []
     for bench_cat in benchmark_category_list:
-
+        
         # Map path to appropriate test_cases folder and list test_ids into bench_cat_id_list.
         bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
         bench_cat_id_list = os.listdir(bench_cat_test_case_dir)
@@ -317,27 +317,22 @@ def process_alpha_test(args):
 
                         # For previous versions of HAND computed at HUC6 scale
                         if not os.path.exists(fim_run_dir):
+                            print(fim_run_dir)
                             if config == 'DEV':
                                 fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
                             elif config == 'PREV':
                                 fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])
-
+                                    
                         if os.path.exists(fim_run_dir):
-
                             # If a user supplies a special_string (-s), then add it to the end of the created dirs.
                             if special_string != "":
                                 version = version + '_' + special_string
 
+            
                             # Define the magnitude lists to use, depending on test_id.
-                            if 'ble' in test_id:
-                                magnitude = BLE_MAGNITUDE_LIST
-                            elif 'usgs' or 'nws' in test_id:
-                                magnitude = ['action', 'minor', 'moderate', 'major']
-                            elif 'ifc' in test_id:
-                                magnitude = IFC_MAGNITUDE_LIST
-                            else:
-                                continue
-
+                            benchmark_type = test_id.split('_')[1]
+                            magnitude = MAGNITUDE_DICT[benchmark_type]
+                            
                             # Either add to list to multiprocess or process serially, depending on user specification.
                             if job_number > 1:
                                 procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index b6cfcafcc..04356c3e7 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -9,6 +9,9 @@
 FR_BENCHMARK_CATEGORIES = ['ble', 'ifc']
 BLE_MAGNITUDE_LIST = ['100yr', '500yr']
 IFC_MAGNITUDE_LIST = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr', '200yr', '500yr']
+AHPS_MAGNITUDE_LIST = ['action', 'minor', 'moderate', 'major']
+
+MAGNITUDE_DICT = {'ble': BLE_MAGNITUDE_LIST, 'ifc': IFC_MAGNITUDE_LIST, 'usgs': AHPS_MAGNITUDE_LIST, 'nws': AHPS_MAGNITUDE_LIST}
 PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
 GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
 GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']

From dcf414f5655ecafbf8bb62cd219aef405e55f0a2 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Tue, 14 Sep 2021 09:17:16 -0500
Subject: [PATCH 112/359] Adjust SRC with user observed feedback points

See Changelog 3.0.22.6 for details.

This resolves #450 and resolves #436.
---
 docs/CHANGELOG.md                       |  13 +-
 src/add_crosswalk.py                    |   4 +-
 tools/adjust_rc_with_feedback.py        | 190 ++++++++++++++++++++----
 tools/inundation_wrapper_custom_flow.py |  26 +++-
 4 files changed, 199 insertions(+), 34 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 433d8b82f..2a3df5c29 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.6 - 2021-09-13 - [PR #462](https://github.com/NOAA-OWP/cahaba/pull/462)
+
+This new workflow ingests FIM point observations from users and “corrects” the synthetic rating curves to produce the desired FIM extent at locations where feedback is available (locally calibrate FIM).
+
+## Changes
+- `add_crosswalk.py`: added `NextDownID` and `order_` attributes to the exported `hydroTable.csv`. This will potentially be used in future enhancements to extend SRC changes to upstream/downstream catchments.
+- `adjust_rc_with_feedback.py`: added a new workflow to perform the SRC modifications (revised discharge) using the existing HAND geometry variables combined with the user provided point location flow and stage data.
+- `inundation_wrapper_custom_flow.py`: updated code to allow for huc6 processing to generate custom inundation outputs.
+
+<br/><br/>
+
 ## v3.0.22.5 - 2021-09-08 - [PR #460](https://github.com/NOAA-OWP/cahaba/pull/460)
 
 Patches an issue where only certain benchmark categories were being used in evaluation.
@@ -64,7 +75,7 @@ This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in futu
 General repository cleanup, made memory-profiling an optional flag, API's release feature now saves outputs.
 
 ## Changes
-- Remove `Dockerfile.prod`, rename `Dockerfile.dev` to just `Dockerfile`, and remove ``.dockerignore`.
+- Remove `Dockerfile.prod`, rename `Dockerfile.dev` to just `Dockerfile`, and remove `.dockerignore`.
 - Clean up `Dockerfile` and remove any unused* packages or variables.
 - Remove any unused* Python packages from the `Pipfile`.
 - Move the `CHANGELOG.md`, `SECURITY.md`, and `TERMS.md` files to the `/docs` folder.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 2b352b6cd..73bf3d4d1 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -179,7 +179,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     input_src_base = pd.read_csv(input_srcbase_fileName, dtype= object)
     if input_src_base.CatchId.dtype != 'int': input_src_base.CatchId = input_src_base.CatchId.astype(int)
 
-    input_src_base = input_src_base.merge(output_flows[['ManningN','HydroID','order_']],left_on='CatchId',right_on='HydroID')
+    input_src_base = input_src_base.merge(output_flows[['ManningN','HydroID','NextDownID','order_']],left_on='CatchId',right_on='HydroID')
 
     input_src_base = input_src_base.rename(columns=lambda x: x.strip(" "))
     input_src_base = input_src_base.apply(pd.to_numeric,**{'errors' : 'coerce'})
@@ -228,7 +228,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
         print('Note: NOT using bathy estimation approach to modify the SRC...')
 
     # make hydroTable
-    output_hydro_table = output_src.loc[:,['HydroID','feature_id','Stage','Discharge (m3s-1)', 'HydraulicRadius (m)', 'WetArea (m2)', 'SLOPE', 'ManningN']]
+    output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Stage','Discharge (m3s-1)', 'HydraulicRadius (m)', 'WetArea (m2)', 'SLOPE', 'ManningN']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
 
     if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
diff --git a/tools/adjust_rc_with_feedback.py b/tools/adjust_rc_with_feedback.py
index 16e00b40a..35a924572 100644
--- a/tools/adjust_rc_with_feedback.py
+++ b/tools/adjust_rc_with_feedback.py
@@ -3,48 +3,167 @@
 from geopandas.tools import sjoin
 import os
 import rasterio
+import pandas as pd
+import numpy as np
+import sys
+import json
 
 temp_workspace = r''
 HAND_CRS = 'EPSG:3857'
 
-def update_rating_curve(grouped_median, huc6):
-    pass
+def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_file, huc6):
+    print("Processing huc --> " + str(huc6))
+    log_file.write("\nProcessing huc --> " + str(huc6) + '\n')
+    df_gmed = pd.read_csv(output_csv) # read csv to import as a dataframe
+    df_gmed = df_gmed[df_gmed.hydroid != 0] # remove entries that do not have a valid hydroid
 
+    # Read in the hydroTable.csv and check wether it has previously been updated (rename orig columns if needed)
+    df_htable = pd.read_csv(htable_path)
+    if 'orig_discharge_cms' in df_htable.columns:
+        df_htable = df_htable[['HydroID','feature_id','stage','orig_discharge_cms','HydraulicRadius (m)','WetArea (m2)','SLOPE','default_ManningN','HUC','LakeID']]
+        df_htable.rename(columns={'orig_discharge_cms':'discharge_cms','default_ManningN':'ManningN'}, inplace=True)
+    else:
+        df_htable = df_htable[['HydroID','feature_id','stage','discharge_cms','HydraulicRadius (m)','WetArea (m2)','SLOPE','ManningN','HUC','LakeID']]
+
+    # loop through the user provided point data --> stage/flow dataframe row by row
+    for index, row in df_gmed.iterrows():
+        df_htable_hydroid = df_htable[df_htable.HydroID == row.hydroid] # filter htable for entries with matching hydroid
+        find_src_stage = df_htable_hydroid.loc[df_htable_hydroid['stage'].sub(row.hand).abs().idxmin()] # find closest matching stage to the user provided HAND value
+        # copy the corresponding htable values for the matching stage->HAND lookup
+        df_gmed.loc[index,'src_stage'] = find_src_stage.stage
+        df_gmed.loc[index,'ManningN'] = find_src_stage.ManningN
+        df_gmed.loc[index,'SLOPE'] = find_src_stage.SLOPE
+        df_gmed.loc[index,'HydraulicRadius_m'] = find_src_stage['HydraulicRadius (m)']
+        df_gmed.loc[index,'WetArea_m2'] = find_src_stage['WetArea (m2)']
+        df_gmed.loc[index,'discharge_cms'] = find_src_stage.discharge_cms
+
+    ## Create a df of hydroids and featureids
+    df_hydro_feat = df_htable.groupby(["HydroID"])[["feature_id"]].median()
+    #print(df_hydro_feat.to_string())
+
+    ## Calculate roughness using Manning's equation
+    df_gmed.rename(columns={'ManningN':'ManningN_default','hydroid':'HydroID'}, inplace=True) # rename the previous ManningN column
+    df_gmed['hydroid_ManningN'] = df_gmed['WetArea_m2']* \
+    pow(df_gmed['HydraulicRadius_m'],2.0/3)* \
+    pow(df_gmed['SLOPE'],0.5)/df_gmed['flow']
+    print('Adjusted Mannings N Calculations -->')
+    print(df_gmed)
+
+    # Create dataframe to check for erroneous Manning's n values (>0.6 or <0.001)
+    df_gmed['Mann_flag'] = np.where((df_gmed['hydroid_ManningN'] >= 0.6) | (df_gmed['hydroid_ManningN'] <= 0.001),'Fail','Pass')
+    df_mann_flag = df_gmed[(df_gmed['hydroid_ManningN'] >= 0.6) | (df_gmed['hydroid_ManningN'] <= 0.001)][['HydroID','hydroid_ManningN']]
+    print('Here is the df with mann_flag filter:')
+    print(df_mann_flag.to_string())
+    if not df_mann_flag.empty:
+        log_file.write('!!! Flaged Mannings Roughness values below !!!' +'\n')
+        log_file.write(df_mann_flag.to_string() + '\n')
+
+    # Export csv with the newly calculated Manning's N values
+    output_calc_n_csv = os.path.join(fim_directory, huc6, 'calc_src_n_vals_' + huc6 + '.csv')
+    df_gmed.to_csv(output_calc_n_csv,index=False)
+
+    # filter the modified Manning's n dataframe for values out side allowable range
+    df_gmed = df_gmed[df_gmed['Mann_flag'] == 'Pass']
+
+    # Merge df with hydroid and featureid crosswalked
+    df_gmed = df_gmed.merge(df_hydro_feat, how='left', on='HydroID')
+
+    # Create df with the most recent collection time entry
+    df_updated = df_gmed.groupby(["HydroID"])[['coll_time']].max()
+    df_updated.rename(columns={'coll_time':'last_updated'}, inplace=True)
+
+    # cacluate median ManningN to handle cases with multiple hydroid entries
+    df_mann = df_gmed.groupby(["HydroID"])[['hydroid_ManningN']].median()
+    print('df_mann:')
+    print(df_mann)
+
+    # Create a df with the median hydroid_ManningN value per feature_id
+    df_mann_featid = df_gmed.groupby(["feature_id"])[['hydroid_ManningN']].median()
+    df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
+
+    # Rename the original hydrotable variables to allow new calculations to use the primary var name
+    df_htable.rename(columns={'ManningN':'default_ManningN','discharge_cms':'orig_discharge_cms'}, inplace=True)
+
+    ## Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a singel hydroid)
+    df_nrange = df_gmed.groupby('HydroID').agg({'hydroid_ManningN': ['median', 'min', 'max','count']})
+    log_file.write('Statistics for Modified Roughness Calcs -->' +'\n')
+    log_file.write(df_nrange.to_string() + '\n')
+    log_file.write('----------------------------------------\n\n')
+
+    # Merge the newly caluclated ManningN dataframe with the original hydroTable
+    df_htable = df_htable.merge(df_mann, how='left', on='HydroID')
+    df_htable = df_htable.merge(df_mann_featid, how='left', on='feature_id')
+    df_htable = df_htable.merge(df_updated, how='left', on='HydroID')
+
+    # Create the modify_ManningN column by combining the hydroid_ManningN with the featid_ManningN (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
+    df_htable['modify_ManningN'] = np.where(df_htable['hydroid_ManningN'].isnull(),df_htable['featid_ManningN'],df_htable['hydroid_ManningN'])
+
+    # Create the ManningN column by combining the hydroid_ManningN with the default_ManningN (use modified where available)
+    df_htable['ManningN'] = np.where(df_htable['modify_ManningN'].isnull(),df_htable['default_ManningN'],df_htable['modify_ManningN'])
+
+    # Calculate new discharge_cms with new ManningN
+    df_htable['discharge_cms'] = df_htable['WetArea (m2)']* \
+    pow(df_htable['HydraulicRadius (m)'],2.0/3)* \
+    pow(df_htable['SLOPE'],0.5)/df_htable['ManningN']
+
+    # Replace discharge_cms with 0 or -999 if present in the original discharge
+    df_htable['discharge_cms'].mask(df_htable['orig_discharge_cms']==0.0,0.0,inplace=True)
+    df_htable['discharge_cms'].mask(df_htable['orig_discharge_cms']==-999,-999,inplace=True)
+
+    # Export a new hydroTable.csv and overwrite the previous version
+    out_htable = os.path.join(fim_directory, huc6, 'hydroTable.csv')
+    df_htable.to_csv(out_htable,index=False)
+
+    # output new src json (overwrite previous)
+    output_src_json = dict()
+    hydroID_list = np.unique(df_htable['HydroID'])
+
+    for hid in hydroID_list:
+        indices_of_hid = df_htable['HydroID'] == hid
+        stage_list = df_htable['stage'][indices_of_hid].astype(float)
+        q_list = df_htable['discharge_cms'][indices_of_hid].astype(float)
+        stage_list = stage_list.tolist()
+        q_list = q_list.tolist()
+        output_src_json[str(hid)] = { 'q_list' : q_list , 'stage_list' : stage_list }
+
+    with open(output_src_json_file,'w') as f:
+        json.dump(output_src_json,f,sort_keys=True)
 
 def ingest_points_layer(points_layer, fim_directory, wbd_path):
-    
+
     # Read wbd_path and points_layer to determine which HUC6 each point is in.
-    wbd_huc8_read = gpd.read_file(wbd_path, layer='WBDHU6')    
+    wbd_huc8_read = gpd.read_file(wbd_path, layer='WBDHU6')
     points_layer_read = gpd.read_file(points_layer)
-    
+
     # Update CRS of points_layer_read.
     points_layer_read = points_layer_read.to_crs(HAND_CRS)
     wbd_huc8_read = wbd_huc8_read.to_crs(HAND_CRS)
-    
+
     # Spatial join the two layers.
     water_edge_df = sjoin(points_layer_read, wbd_huc8_read)
-        
+
     # Convert to GeoDataFrame.
     gdf = gpd.GeoDataFrame(water_edge_df)
-    
+
     # Add two columns for X and Y.
     gdf['X'] = gdf['geometry'].x
     gdf['Y'] = gdf['geometry'].y
-            
+
     # Extract information into dictionary.
     huc6_list = []
     for index, row in gdf.iterrows():
         huc6 = row['HUC6']
         if huc6 not in huc6_list:
             huc6_list.append(huc6)
+            log_file.write(str(huc6) + '\n')
 
     # Define coords variable to be used in point raster value attribution.
     coords = [(x,y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
-        
+
     # Define paths to relevant HUC6 HAND data.
     for huc6 in huc6_list:
         print(huc6)
-        
+
         # Define paths to relevant HUC6 HAND data and get necessary metadata for point rasterization.
         hand_path = os.path.join(fim_directory, huc6, 'hand_grid_' + huc6 + '.tif')
         if not os.path.exists(hand_path):
@@ -54,23 +173,32 @@ def ingest_points_layer(points_layer, fim_directory, wbd_path):
         if not os.path.exists(catchments_path):
             print("Catchments grid for " + huc6 + " does not exist.")
             continue
-        
+        htable_path = os.path.join(fim_directory, huc6, 'hydroTable.csv')
+        if not os.path.exists(htable_path):
+            print("hydroTable for " + huc6 + " does not exist.")
+            continue
+        output_src_json_file = os.path.join(fim_directory, huc6, 'rating_curves_' + huc6 + '.json')
+        if not os.path.isfile(output_src_json_file):
+            print("Rating Curve JSON file for " + huc6 + " does not exist.")
+            continue
+
 #        water_edge_df = water_edge_df[water_edge_df['HUC6'] == huc6]
-        
+
         # Use point geometry to determine pixel values at catchment and HAND grids.
         hand_src = rasterio.open(hand_path)
         water_edge_df['hand'] = [h[0] for h in hand_src.sample(coords)]
         hand_src.close()
         catchments_src = rasterio.open(catchments_path)
         water_edge_df['hydroid'] = [c[0] for c in catchments_src.sample(coords)]
-                
+        #print(water_edge_df)
+
         # Get median HAND value for appropriate groups.
         water_edge_median_ds = water_edge_df.groupby(["hydroid", "flow", "submitter", "coll_time", "flow_unit"])['hand'].median()
-               
+
         output_csv = os.path.join(fim_directory, huc6, 'user_supplied_n_vals_' + huc6 + '.csv')
-        
+
         water_edge_median_ds.to_csv(output_csv)
-                
+
         # 1. Loop and find the corresponding hydroids in the Hydrotable
         # 2. Grab slope, wetted area, hydraulic radius, and feature_id that correspond with the matching hydroids and HAND value for the nearest stage
         # 3. Calculate new column for new roughness using the above info
@@ -82,31 +210,39 @@ def ingest_points_layer(points_layer, fim_directory, wbd_path):
         #    4c. Add last_updated column with timestamp where values were changed, also add "submitter" column
         # 5. What do we do in catchments that match the feature_id?
         #    5a. If these catchments already have known data, then let it use those. If not, use new calculated Ns.
-        
-        update_rating_curve(water_edge_median_ds, huc6)
+
+        update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_file, huc6)
 
 
 
 if __name__ == '__main__':
-    
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Adjusts rating curve given a shapefile containing points of known water boundary.')
     parser.add_argument('-p','--points-layer',help='Path to points layer containing known water boundary locations',required=True)
     parser.add_argument('-d','--fim-directory',help='Parent directory of FIM-required datasets.',required=True)
     parser.add_argument('-w','--wbd-path', help='Path to national HUC6 layer.',required=True)
-    
+
     # Assign variables from arguments.
     args = vars(parser.parse_args())
     points_layer = args['points_layer']
     fim_directory = args['fim_directory']
     wbd_path = args['wbd_path']
-    
+
+    # Create log file for processing records
+    print('This may take a few minutes...')
+    sys.__stdout__ = sys.stdout
+    log_file = open(os.path.join(fim_directory,'log_rating_curve_adjust.log'),"w")
+
     ingest_points_layer(points_layer, fim_directory, wbd_path)
-    
+
     # Open catchment, HAND, and point grids and determine pixel values for Hydroid, HAND value, and discharge value, respectively.
-    
+
     # Open rating curve file(s).
-    
+
     # Use three values to determine the hydroid rating curve(s) to update, then update them using a variation of Manning's Equation.
-    
-    # Ensure the JSON rating curve is updated and saved (overwitten). Consider adding attributes to document what was performed.
\ No newline at end of file
+
+    # Ensure the JSON rating curve is updated and saved (overwitten). Consider adding attributes to document what was performed.
+
+    # Close log file
+    sys.stdout = sys.__stdout__
+    log_file.close()
diff --git a/tools/inundation_wrapper_custom_flow.py b/tools/inundation_wrapper_custom_flow.py
index 6867bea5f..72ce20bf0 100755
--- a/tools/inundation_wrapper_custom_flow.py
+++ b/tools/inundation_wrapper_custom_flow.py
@@ -44,20 +44,34 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type=
     if "previous_fim" in fim_run_dir and "fim_2" in fim_run_dir:
         rem = os.path.join(fim_run_dir, 'rem_clipped_zeroed_masked.tif')
         catchments = os.path.join(fim_run_dir, 'gw_catchments_reaches_clipped_addedAttributes.tif')
+        if not os.path.isfile(rem):
+            print('Can not find REM file: ' + str(rem))
+        if not os.path.isfile(catchments):
+            print('Can not find catchments file: ' + str(catchments))
     else:
         rem = os.path.join(fim_run_dir, 'rem_zeroed_masked.tif')
         catchments = os.path.join(fim_run_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+        if not os.path.isfile(rem):
+            print('Can not find REM file: ' + str(rem))
+        if not os.path.isfile(catchments):
+            print('Can not find catchments file: ' + str(catchments))
     if mask_type == 'huc':
         catchment_poly = ''
+        print('Not using the catchment polygon input layer -- FIM version < 3??')
     else:
         catchment_poly = os.path.join(fim_run_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+        if not os.path.isfile(catchment_poly):
+            print('Can not find catchments polygon file: ' + str(catchments))
     hydro_table = os.path.join(fim_run_dir, 'hydroTable.csv')
-
-    # Map necessary inputs for inundation().
-    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+    if not os.path.isfile(hydro_table):
+        print('Can not find hydro_table file: ' + str(hydro_table))
 
     #benchmark_category = huc_id.split('_')[1]
     current_huc = huc_id.split('_')[0]  # Break off HUC ID and assign to variable.
+    wbd_huc = 'WBDHU' + str(len(huc_id)) # check if the input huc is 2,4,6,8 etc
+
+    # Map necessary inputs for inundation().
+    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), wbd_huc
 
     if not os.path.exists(branch_test_case_dir_parent):
         os.mkdir(branch_test_case_dir_parent)
@@ -70,6 +84,8 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type=
     # Define paths to inundation_raster and forecast file.
     inundation_raster = os.path.join(branch_test_case_dir_parent, branch_name + '_inund_extent.tif')
     forecast = os.path.join(TEST_CASES_DIR,"_input_flow_files", input_flow_csv)
+    if not os.path.isfile(forecast):
+        print('Can not find input flow file: ' + str(forecast))
 
     # Copy forecast flow file into the outputs directory to all viewer to reference the flows used to create inundation_raster
     shutil.copyfile(forecast,os.path.join(branch_test_case_dir_parent,input_flow_csv))
@@ -79,10 +95,12 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type=
     inundate(
              rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
              subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-             depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
+             depths=None,out_raster_profile=None,out_vector_profile=None,quiet=False
             )
 
     print("-----> Inundation mapping complete.")
+    if not os.path.isfile(inundation_raster):
+        print('Warning!! Inundation Raster not produced: ' + str(inundation_raster))
 
 
 if __name__ == '__main__':

From fb487ca400e83a56b875dcc3381b4c2afb9be231 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 16 Sep 2021 10:20:26 -0500
Subject: [PATCH 113/359] Bump pillow from 8.3.0 to 8.3.2

Bumps [pillow](https://github.com/python-pillow/Pillow) from 8.3.0 to 8.3.2.
- [Release notes](https://github.com/python-pillow/Pillow/releases)
- [Changelog](https://github.com/python-pillow/Pillow/blob/master/CHANGES.rst)
- [Commits](https://github.com/python-pillow/Pillow/compare/8.3.0...8.3.2)

---
updated-dependencies:
- dependency-name: pillow
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Pipfile.lock | 493 +++++++++++++++++++++++++++------------------------
 1 file changed, 263 insertions(+), 230 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 746158f4d..11f8a16a5 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -102,41 +102,53 @@
         },
         "kiwisolver": {
             "hashes": [
-                "sha256:0cd53f403202159b44528498de18f9285b04482bab2a6fc3f5dd8dbb9352e30d",
-                "sha256:1e1bc12fb773a7b2ffdeb8380609f4f8064777877b2225dec3da711b421fda31",
-                "sha256:225e2e18f271e0ed8157d7f4518ffbf99b9450fca398d561eb5c4a87d0986dd9",
-                "sha256:232c9e11fd7ac3a470d65cd67e4359eee155ec57e822e5220322d7b2ac84fbf0",
-                "sha256:31dfd2ac56edc0ff9ac295193eeaea1c0c923c0355bf948fbd99ed6018010b72",
-                "sha256:33449715e0101e4d34f64990352bce4095c8bf13bed1b390773fc0a7295967b3",
-                "sha256:401a2e9afa8588589775fe34fc22d918ae839aaaf0c0e96441c0fdbce6d8ebe6",
-                "sha256:44a62e24d9b01ba94ae7a4a6c3fb215dc4af1dde817e7498d901e229aaf50e4e",
-                "sha256:50af681a36b2a1dee1d3c169ade9fdc59207d3c31e522519181e12f1b3ba7000",
-                "sha256:563c649cfdef27d081c84e72a03b48ea9408c16657500c312575ae9d9f7bc1c3",
-                "sha256:5989db3b3b34b76c09253deeaf7fbc2707616f130e166996606c284395da3f18",
-                "sha256:5a7a7dbff17e66fac9142ae2ecafb719393aaee6a3768c9de2fd425c63b53e21",
-                "sha256:5c3e6455341008a054cccee8c5d24481bcfe1acdbc9add30aa95798e95c65621",
-                "sha256:5f6ccd3dd0b9739edcf407514016108e2280769c73a85b9e59aa390046dbf08b",
-                "sha256:72c99e39d005b793fb7d3d4e660aed6b6281b502e8c1eaf8ee8346023c8e03bc",
-                "sha256:78751b33595f7f9511952e7e60ce858c6d64db2e062afb325985ddbd34b5c131",
-                "sha256:834ee27348c4aefc20b479335fd422a2c69db55f7d9ab61721ac8cd83eb78882",
-                "sha256:8be8d84b7d4f2ba4ffff3665bcd0211318aa632395a1a41553250484a871d454",
-                "sha256:950a199911a8d94683a6b10321f9345d5a3a8433ec58b217ace979e18f16e248",
-                "sha256:a357fd4f15ee49b4a98b44ec23a34a95f1e00292a139d6015c11f55774ef10de",
-                "sha256:a53d27d0c2a0ebd07e395e56a1fbdf75ffedc4a05943daf472af163413ce9598",
-                "sha256:acef3d59d47dd85ecf909c359d0fd2c81ed33bdff70216d3956b463e12c38a54",
-                "sha256:b38694dcdac990a743aa654037ff1188c7a9801ac3ccc548d3341014bc5ca278",
-                "sha256:b9edd0110a77fc321ab090aaa1cfcaba1d8499850a12848b81be2222eab648f6",
-                "sha256:c08e95114951dc2090c4a630c2385bef681cacf12636fb0241accdc6b303fd81",
-                "sha256:c5518d51a0735b1e6cee1fdce66359f8d2b59c3ca85dc2b0813a8aa86818a030",
-                "sha256:c8fd0f1ae9d92b42854b2979024d7597685ce4ada367172ed7c09edf2cef9cb8",
-                "sha256:ca3820eb7f7faf7f0aa88de0e54681bddcb46e485beb844fcecbcd1c8bd01689",
-                "sha256:cf8b574c7b9aa060c62116d4181f3a1a4e821b2ec5cbfe3775809474113748d4",
-                "sha256:d3155d828dec1d43283bd24d3d3e0d9c7c350cdfcc0bd06c0ad1209c1bbc36d0",
-                "sha256:f8d6f8db88049a699817fd9178782867bf22283e3813064302ac59f61d95be05",
-                "sha256:fd34fbbfbc40628200730bc1febe30631347103fc8d3d4fa012c21ab9c11eca9"
+                "sha256:0007840186bacfaa0aba4466d5890334ea5938e0bb7e28078a0eb0e63b5b59d5",
+                "sha256:19554bd8d54cf41139f376753af1a644b63c9ca93f8f72009d50a2080f870f77",
+                "sha256:1d45d1c74f88b9f41062716c727f78f2a59a5476ecbe74956fafb423c5c87a76",
+                "sha256:1d819553730d3c2724582124aee8a03c846ec4362ded1034c16fb3ef309264e6",
+                "sha256:2210f28778c7d2ee13f3c2a20a3a22db889e75f4ec13a21072eabb5693801e84",
+                "sha256:22521219ca739654a296eea6d4367703558fba16f98688bd8ce65abff36eaa84",
+                "sha256:25405f88a37c5f5bcba01c6e350086d65e7465fd1caaf986333d2a045045a223",
+                "sha256:2b65bd35f3e06a47b5c30ea99e0c2b88f72c6476eedaf8cfbc8e66adb5479dcf",
+                "sha256:2ddb500a2808c100e72c075cbb00bf32e62763c82b6a882d403f01a119e3f402",
+                "sha256:2f8f6c8f4f1cff93ca5058d6ec5f0efda922ecb3f4c5fb76181f327decff98b8",
+                "sha256:30fa008c172355c7768159983a7270cb23838c4d7db73d6c0f6b60dde0d432c6",
+                "sha256:3dbb3cea20b4af4f49f84cffaf45dd5f88e8594d18568e0225e6ad9dec0e7967",
+                "sha256:4116ba9a58109ed5e4cb315bdcbff9838f3159d099ba5259c7c7fb77f8537492",
+                "sha256:44e6adf67577dbdfa2d9f06db9fbc5639afefdb5bf2b4dfec25c3a7fbc619536",
+                "sha256:5326ddfacbe51abf9469fe668944bc2e399181a2158cb5d45e1d40856b2a0589",
+                "sha256:70adc3658138bc77a36ce769f5f183169bc0a2906a4f61f09673f7181255ac9b",
+                "sha256:72be6ebb4e92520b9726d7146bc9c9b277513a57a38efcf66db0620aec0097e0",
+                "sha256:7843b1624d6ccca403a610d1277f7c28ad184c5aa88a1750c1a999754e65b439",
+                "sha256:7ba5a1041480c6e0a8b11a9544d53562abc2d19220bfa14133e0cdd9967e97af",
+                "sha256:80efd202108c3a4150e042b269f7c78643420cc232a0a771743bb96b742f838f",
+                "sha256:82f49c5a79d3839bc8f38cb5f4bfc87e15f04cbafa5fbd12fb32c941cb529cfb",
+                "sha256:83d2c9db5dfc537d0171e32de160461230eb14663299b7e6d18ca6dca21e4977",
+                "sha256:8d93a1095f83e908fc253f2fb569c2711414c0bfd451cab580466465b235b470",
+                "sha256:8dc3d842fa41a33fe83d9f5c66c0cc1f28756530cd89944b63b072281e852031",
+                "sha256:9661a04ca3c950a8ac8c47f53cbc0b530bce1b52f516a1e87b7736fec24bfff0",
+                "sha256:a498bcd005e8a3fedd0022bb30ee0ad92728154a8798b703f394484452550507",
+                "sha256:a7a4cf5bbdc861987a7745aed7a536c6405256853c94abc9f3287c3fa401b174",
+                "sha256:b5074fb09429f2b7bc82b6fb4be8645dcbac14e592128beeff5461dcde0af09f",
+                "sha256:b6a5431940f28b6de123de42f0eb47b84a073ee3c3345dc109ad550a3307dd28",
+                "sha256:ba677bcaff9429fd1bf01648ad0901cea56c0d068df383d5f5856d88221fe75b",
+                "sha256:bcadb05c3d4794eb9eee1dddf1c24215c92fb7b55a80beae7a60530a91060560",
+                "sha256:bf7eb45d14fc036514c09554bf983f2a72323254912ed0c3c8e697b62c4c158f",
+                "sha256:c358721aebd40c243894298f685a19eb0491a5c3e0b923b9f887ef1193ddf829",
+                "sha256:c4550a359c5157aaf8507e6820d98682872b9100ce7607f8aa070b4b8af6c298",
+                "sha256:c6572c2dab23c86a14e82c245473d45b4c515314f1f859e92608dcafbd2f19b8",
+                "sha256:cba430db673c29376135e695c6e2501c44c256a81495da849e85d1793ee975ad",
+                "sha256:dedc71c8eb9c5096037766390172c34fb86ef048b8e8958b4e484b9e505d66bc",
+                "sha256:e6f5eb2f53fac7d408a45fbcdeda7224b1cfff64919d0f95473420a931347ae9",
+                "sha256:ec2eba188c1906b05b9b49ae55aae4efd8150c61ba450e6721f64620c50b59eb",
+                "sha256:ee040a7de8d295dbd261ef2d6d3192f13e2b08ec4a954de34a6fb8ff6422e24c",
+                "sha256:eedd3b59190885d1ebdf6c5e0ca56828beb1949b4dfe6e5d0256a461429ac386",
+                "sha256:f441422bb313ab25de7b3dbfd388e790eceb76ce01a18199ec4944b369017009",
+                "sha256:f8eb7b6716f5b50e9c06207a14172cf2de201e41912ebe732846c02c830455b9",
+                "sha256:fc4453705b81d03568d5b808ad8f09c77c47534f6ac2e72e733f9ca4714aa75c"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.3.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.2"
         },
         "llvmlite": {
             "hashes": [
@@ -162,28 +174,30 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:0bea5ec5c28d49020e5d7923c2725b837e60bc8be99d3164af410eb4b4c827da",
-                "sha256:1c1779f7ab7d8bdb7d4c605e6ffaa0614b3e80f1e3c8ccf7b9269a22dbc5986b",
-                "sha256:21b31057bbc5e75b08e70a43cefc4c0b2c2f1b1a850f4a0f7af044eb4163086c",
-                "sha256:32fa638cc10886885d1ca3d409d4473d6a22f7ceecd11322150961a70fab66dd",
-                "sha256:3a5c18dbd2c7c366da26a4ad1462fe3e03a577b39e3b503bbcf482b9cdac093c",
-                "sha256:5826f56055b9b1c80fef82e326097e34dc4af8c7249226b7dd63095a686177d1",
-                "sha256:6382bc6e2d7e481bcd977eb131c31dee96e0fb4f9177d15ec6fb976d3b9ace1a",
-                "sha256:6475d0209024a77f869163ec3657c47fed35d9b6ed8bccba8aa0f0099fbbdaa8",
-                "sha256:6a6a44f27aabe720ec4fd485061e8a35784c2b9ffa6363ad546316dfc9cea04e",
-                "sha256:7a58f3d8fe8fac3be522c79d921c9b86e090a59637cb88e3bc51298d7a2c862a",
-                "sha256:7ad19f3fb6145b9eb41c08e7cbb9f8e10b91291396bee21e9ce761bb78df63ec",
-                "sha256:85f191bb03cb1a7b04b5c2cca4792bef94df06ef473bc49e2818105671766fee",
-                "sha256:956c8849b134b4a343598305a3ca1bdd3094f01f5efc8afccdebeffe6b315247",
-                "sha256:a9d8cb5329df13e0cdaa14b3b43f47b5e593ec637f13f14db75bb16e46178b05",
-                "sha256:b1d5a2cedf5de05567c441b3a8c2651fbde56df08b82640e7f06c8cd91e201f6",
-                "sha256:b26535b9de85326e6958cdef720ecd10bcf74a3f4371bf9a7e5b2e659c17e153",
-                "sha256:c541ee5a3287efe066bbe358320853cf4916bc14c00c38f8f3d8d75275a405a9",
-                "sha256:d8d994cefdff9aaba45166eb3de4f5211adb4accac85cbf97137e98f26ea0219",
-                "sha256:df815378a754a7edd4559f8c51fc7064f779a74013644a7f5ac7a0c31f875866"
+                "sha256:01c9de93a2ca0d128c9064f23709362e7fefb34910c7c9e0b8ab0de8258d5eda",
+                "sha256:41b6e307458988891fcdea2d8ecf84a8c92d53f84190aa32da65f9505546e684",
+                "sha256:48e1e0859b54d5f2e29bb78ca179fd59b971c6ceb29977fb52735bfd280eb0f5",
+                "sha256:54a026055d5f8614f184e588f6e29064019a0aa8448450214c0b60926d62d919",
+                "sha256:556965514b259204637c360d213de28d43a1f4aed1eca15596ce83f768c5a56f",
+                "sha256:5c988bb43414c7c2b0a31bd5187b4d27fd625c080371b463a6d422047df78913",
+                "sha256:6a724e3a48a54b8b6e7c4ae38cd3d07084508fa47c410c8757e9db9791421838",
+                "sha256:6be8df61b1626e1a142c57e065405e869e9429b4a6dab4a324757d0dc4d42235",
+                "sha256:844a7b0233e4ff7fba57e90b8799edaa40b9e31e300b8d5efc350937fa8b1bea",
+                "sha256:85f0c9cf724715e75243a7b3087cf4a3de056b55e05d4d76cc58d610d62894f3",
+                "sha256:a78a3b51f29448c7f4d4575e561f6b0dbb8d01c13c2046ab6c5220eb25c06506",
+                "sha256:b884715a59fec9ad3b6048ecf3860f3b2ce965e676ef52593d6fa29abcf7d330",
+                "sha256:b8b53f336a4688cfce615887505d7e41fd79b3594bf21dd300531a4f5b4f746a",
+                "sha256:c70b6311dda3e27672f1bf48851a0de816d1ca6aaf3d49365fbdd8e959b33d2b",
+                "sha256:ebfb01a65c3f5d53a8c2a8133fec2b5221281c053d944ae81ff5822a68266617",
+                "sha256:eeb1859efe7754b1460e1d4991bbd4a60a56f366bc422ef3a9c5ae05f0bc70b5",
+                "sha256:f15edcb0629a0801738925fe27070480f446fcaa15de65946ff946ad99a59a40",
+                "sha256:f1c5efc278d996af8a251b2ce0b07bbeccb821f25c8c9846bdcb00ffc7f158aa",
+                "sha256:f72657f1596199dc1e4e7a10f52a4784ead8a711f4e5b59bea95bdb97cf0e4fd",
+                "sha256:fc4f526dfdb31c9bd6b8ca06bf9fab663ca12f3ec9cdf4496fb44bc680140318",
+                "sha256:fcd6f1954943c0c192bfbebbac263f839d7055409f1173f80d8b11a224d236da"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.4.2"
+            "version": "==3.4.3"
         },
         "memory-profiler": {
             "hashes": [
@@ -234,37 +248,39 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:1a784e8ff7ea2a32e393cc53eb0003eca1597c7ca628227e34ce34eb11645a0e",
-                "sha256:2ba579dde0563f47021dcd652253103d6fd66165b18011dce1a0609215b2791e",
-                "sha256:3537b967b350ad17633b35c2f4b1a1bbd258c018910b518c30b48c8e41272717",
-                "sha256:3c40e6b860220ed862e8097b8f81c9af6d7405b723f4a7af24a267b46f90e461",
-                "sha256:598fe100b2948465cf3ed64b1a326424b5e4be2670552066e17dfaa67246011d",
-                "sha256:620732f42259eb2c4642761bd324462a01cdd13dd111740ce3d344992dd8492f",
-                "sha256:709884863def34d72b183d074d8ba5cfe042bc3ff8898f1ffad0209161caaa99",
-                "sha256:75579acbadbf74e3afd1153da6177f846212ea2a0cc77de53523ae02c9256513",
-                "sha256:7c55407f739f0bfcec67d0df49103f9333edc870061358ac8a8c9e37ea02fcd2",
-                "sha256:a1f2fb2da242568af0271455b89aee0f71e4e032086ee2b4c5098945d0e11cf6",
-                "sha256:a290989cd671cd0605e9c91a70e6df660f73ae87484218e8285c6522d29f6e38",
-                "sha256:ac4fd578322842dbda8d968e3962e9f22e862b6ec6e3378e7415625915e2da4d",
-                "sha256:ad09f55cc95ed8d80d8ab2052f78cc21cb231764de73e229140d81ff49d8145e",
-                "sha256:b9205711e5440954f861ceeea8f1b415d7dd15214add2e878b4d1cf2bcb1a914",
-                "sha256:bba474a87496d96e61461f7306fba2ebba127bed7836212c360f144d1e72ac54",
-                "sha256:bebab3eaf0641bba26039fb0b2c5bf9b99407924b53b1ea86e03c32c64ef5aef",
-                "sha256:cc367c86eb87e5b7c9592935620f22d13b090c609f1b27e49600cd033b529f54",
-                "sha256:ccc6c650f8700ce1e3a77668bb7c43e45c20ac06ae00d22bdf6760b38958c883",
-                "sha256:cf680682ad0a3bef56dae200dbcbac2d57294a73e5b0f9864955e7dd7c2c2491",
-                "sha256:d2910d0a075caed95de1a605df00ee03b599de5419d0b95d55342e9a33ad1fb3",
-                "sha256:d5caa946a9f55511e76446e170bdad1d12d6b54e17a2afe7b189112ed4412bb8",
-                "sha256:d89b0dc7f005090e32bb4f9bf796e1dcca6b52243caf1803fdd2b748d8561f63",
-                "sha256:d95d16204cd51ff1a1c8d5f9958ce90ae190be81d348b514f9be39f878b8044a",
-                "sha256:e4d5a86a5257843a18fb1220c5f1c199532bc5d24e849ed4b0289fb59fbd4d8f",
-                "sha256:e58ddb53a7b4959932f5582ac455ff90dcb05fac3f8dcc8079498d43afbbde6c",
-                "sha256:e80fe25cba41c124d04c662f33f6364909b985f2eb5998aaa5ae4b9587242cce",
-                "sha256:eda2829af498946c59d8585a9fd74da3f810866e05f8df03a86f70079c7531dd",
-                "sha256:fd0a359c1c17f00cb37de2969984a74320970e0ceef4808c32e00773b06649d9"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.21.0"
+                "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf",
+                "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2",
+                "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6",
+                "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad",
+                "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc",
+                "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254",
+                "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0",
+                "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218",
+                "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13",
+                "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef",
+                "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737",
+                "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723",
+                "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3",
+                "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1",
+                "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d",
+                "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52",
+                "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3",
+                "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5",
+                "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f",
+                "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496",
+                "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f",
+                "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724",
+                "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931",
+                "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f",
+                "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7",
+                "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38",
+                "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557",
+                "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57",
+                "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310",
+                "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419"
+            ],
+            "markers": "python_version < '3.11' and python_version >= '3.7'",
+            "version": "==1.21.2"
         },
         "pandas": {
             "hashes": [
@@ -290,43 +306,62 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:063d17a02a0170c2f880fbd373b2738b089c6adcbd1f7418667bc9e97524c11b",
-                "sha256:1037288a22cc8ec9d2918a24ded733a1cc4342fd7f21d15d37e6bbe5fb4a7306",
-                "sha256:25f6564df21d15bcac142b4ed92b6c02e53557539f535f31c1f3bcc985484753",
-                "sha256:28f184c0a65be098af412f78b0b6f3bbafd1614e1dc896e810d8357342a794b7",
-                "sha256:3251557c53c1ed0c345559afc02d2b0a0aa5788042e161366ed90b27bc322d3d",
-                "sha256:331f8321418682386e4f0d0e6369f732053f95abddd2af4e1b1ef74a9537ef37",
-                "sha256:333313bcc53a8a7359e98d5458dfe37bfa301da2fd0e0dc41f585ae0cede9181",
-                "sha256:34ce3d993cb4ca840b1e31165b38cb19c64f64f822a8bc5565bde084baff3bdb",
-                "sha256:490c9236ef4762733b6c2e1f1fcb37793cb9c57d860aa84d6994c990461882e5",
-                "sha256:519b3b24dedc81876d893475bade1b92c4ce7c24b9b82224f0bd8daae682e039",
-                "sha256:53f6e4b73b3899015ac4aa95d99da0f48ea18a6d7c8db672e8bead3fb9570ef5",
-                "sha256:561339ed7c324bbcb29b5e4f4705c97df950785394b3ac181f5bf6a08088a672",
-                "sha256:6f7517a220aca8b822e25b08b0df9546701a606a328da5bc057e5f32a3f9b07c",
-                "sha256:713b762892efa8cd5d8dac24d16ac2d2dbf981963ed1b3297e79755f03f8cbb8",
-                "sha256:72858a27dd7bd1c40f91c4f85db3b9f121c8412fd66573121febb00d074d0530",
-                "sha256:778a819c2d194e08d39d67ddb15ef0d32eba17bf7d0c2773e97bd221b2613a3e",
-                "sha256:803606e206f3e366eea46b1e7ab4dac74cfac770d04de9c35319814e11e47c46",
-                "sha256:856fcbc3201a6cabf0478daa0c0a1a8a175af7e5173e2084ddb91cc707a09dd1",
-                "sha256:8f65d2a98f198e904dbe89ecb10862d5f0511367d823689039e17c4d011de11e",
-                "sha256:94db5ea640330de0945b41dc77fb4847b4ab6e87149126c71b36b112e8400898",
-                "sha256:950e873ceefbd283cbe7bc5b648b832d1dcf89eeded6726ebec42bc7d67966c0",
-                "sha256:a7beda44f177ee602aa27e0a297da1657d9572679522c8fb8b336b734653516e",
-                "sha256:aef0838f28328523e9e5f2c1852dd96fb85768deb0eb8f908c54dad0f44d2f6f",
-                "sha256:b42ea77f4e7374a67e1f27aaa9c62627dff681f67890e5b8f0c1e21b1500d9d2",
-                "sha256:bccd0d604d814e9494f3bf3f077a23835580ed1743c5175581882e7dd1f178c3",
-                "sha256:c2d78c8230bda5fc9f6b1d457c7f8f3432f4fe85bed86f80ba3ed73d59775a88",
-                "sha256:c3529fb98a40f89269175442c5ff4ef81d22e91b2bdcbd33833a350709b5130c",
-                "sha256:cc8e926d6ffa65d0dddb871b7afe117f17bc045951e66afde60eb0eba923db9e",
-                "sha256:ce90aad0a3dc0f13a9ff0ab1f43bcbea436089b83c3fadbe37c6f1733b938bf1",
-                "sha256:cec702974f162026bf8de47f6f4b7ce9584a63c50002b38f195ee797165fea77",
-                "sha256:d9ef8119ce44f90d2f8ac7c58f7da480ada5151f217dc8da03681b73fc91dec3",
-                "sha256:eccaefbd646022b5313ca4b0c5f1ae6e0d3a52ef66de64970ecf3f9b2a1be751",
-                "sha256:fb91deb5121b6dde88599bcb3db3fdad9cf33ff3d4ccc5329ee1fe9655a2f7ff",
-                "sha256:fc25d59ecf23ea19571065306806a29c43c67f830f0e8a121303916ba257f484"
+                "sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30",
+                "sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9",
+                "sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71",
+                "sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9",
+                "sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b",
+                "sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630",
+                "sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875",
+                "sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2",
+                "sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1",
+                "sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7",
+                "sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3",
+                "sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b",
+                "sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6",
+                "sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba",
+                "sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4",
+                "sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864",
+                "sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056",
+                "sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228",
+                "sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8",
+                "sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb",
+                "sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d",
+                "sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da",
+                "sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073",
+                "sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3",
+                "sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616",
+                "sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa",
+                "sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979",
+                "sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a",
+                "sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b",
+                "sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6",
+                "sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441",
+                "sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624",
+                "sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd",
+                "sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550",
+                "sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09",
+                "sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196",
+                "sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b",
+                "sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1",
+                "sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6",
+                "sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83",
+                "sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f",
+                "sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4",
+                "sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19",
+                "sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341",
+                "sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96",
+                "sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355",
+                "sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c",
+                "sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c",
+                "sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629",
+                "sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2",
+                "sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87",
+                "sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5",
+                "sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==8.3.0"
+            "index": "pypi",
+            "version": "==8.3.2"
         },
         "psutil": {
             "hashes": [
@@ -394,36 +429,40 @@
         },
         "pyproj": {
             "hashes": [
-                "sha256:04c185102e659439c5bd428ac5473d36ef795fca8e225bbbe78e20643d804ec0",
-                "sha256:10dad599b9f7ce2194996dc25f1000e0aa15754ecef9db46b624713959c67957",
-                "sha256:1e88ebc4e08e661e9011b5c1ebfb32f0d311963a9824a6effb4168c7e07918b1",
-                "sha256:4f3ad09cf3352bf5664794042b28d98781362ec8d9774ad73f28a1a0101a27f1",
-                "sha256:5f8a8d982bde211e65dc2de1f8f36cf162f9cc7fcd8a7625046ea265284e5e65",
-                "sha256:67b94f4e694ae33fc90dfb7da0e6b5ed5f671dd0acc2f6cf46e9c39d56e16e1a",
-                "sha256:808f5992320e9631b2e45444028a65cd6ba3ee40229292934178ef07020a5ffd",
-                "sha256:8eda240225971b5cd0bac2d399ed6222068f0598ee92d5f6e847bd2019d2c8b0",
-                "sha256:911d773da9fa4d4f3f7580173858c391e3ee0b61acaf0be303baab323d2eae78",
-                "sha256:9cc464a1c51baad28ffb7a233116e8d4ce4c560b32039fa986d0f992ac3c431f",
-                "sha256:a162ed199cd2ec392cffe20b2fa3381b68e7a166d55f3f060eceb8d517e4f46d",
-                "sha256:aa87df0982aa0f4477478899d9c930cc0f97cd6d8a4ce84c43ac88ccf86d1da7",
-                "sha256:ae237492767e0225f99b53a0fd7110fde2b7e7cabc105bbc243c151a7497de88",
-                "sha256:ae5534fa7a3b74f20534694d297fce6f7483890ff6ca404394ecf372f3c589d4",
-                "sha256:b635e7e21fea5af74e90fc9e54d1a4c27078efdce6f214101c98dd93afae599a",
-                "sha256:b6c74bbec679199746a3e02c0e0fad093c3652df96dd63e086a2fbf2afe9dc0e",
-                "sha256:c4193e1069d165476b2d0f7d882b7712b3eab6e2e6fe2a0a78ef40de825a1f28",
-                "sha256:da88abc5e2f6a8fb07533855a57ca2a31845f58901a87f821b68b0db6b023978",
-                "sha256:ebbba7707fe83a01e54bce8e3e7342feb0b3e0d74ff8c28df12f8bc59b76827c"
+                "sha256:1563a854eff7b6ed430fb968f8f711d13fc02d587afb1dc93a88e5cda6d36462",
+                "sha256:1a0a55bbc2d903a6dada943ade11360a46d057ca42d305128c113273bae7cb58",
+                "sha256:2cc06ea53f8c6c97ff37a47a614b32adc4c4b667def08b528ea7f6a351fa8628",
+                "sha256:2e8e51e55bc6305b538c0c6576dbc37a45993b7f3e68699237411fbde9220aab",
+                "sha256:413cb8dfd358f3b89cb9756a3380295d34b3ea5059d47f327262e700f1c8baab",
+                "sha256:48df0d5ab085bd2dc6db3bca79e20bf15b08ffca4f4e42df6d87b566633b800c",
+                "sha256:582efc0cf0bae6ac5a4730280995fad6c20d844b56b70f41f4f04fdb1fdbd5d7",
+                "sha256:5974c1acbeb1d4f34bba111be547191f967ef182f77fc361c34984f8adbe6f66",
+                "sha256:5fb4887893d221a82082e95ac6dbf75cbbc739991fcaa80ba94fc16070321354",
+                "sha256:72e0c4409a0c2f83ba448ecdf6accc9615d3e4069b8cad2a2a37a464225d0582",
+                "sha256:8206748e1ef61151f19dd54378312e272812cb248226fb7909fda5052498f9b4",
+                "sha256:85b7f67a3606b8a846691effd80c187597ac4e2733ae818f3e3e8be33edcb582",
+                "sha256:87a2b36d7847149d1309c850d022da71de9398140a62e1496cfe64cf3e74040a",
+                "sha256:89c023b1f7c956343e87bc7165fe67986b5db0b840fd5adf0ca65c0b5dcb5007",
+                "sha256:8c0cd49eed818c9a02ea05cb9dc45d7209d1688fc4c9e08a37bc48a9ec72e852",
+                "sha256:911d711601d53208eb4dcca95df85661bee13592da6035c7f12ec0e8c7bbfea8",
+                "sha256:9a7007dea9d739a03afcc214c851dd43230db319876dd4bf3787031b67a57b77",
+                "sha256:b00e05f374d419be9689c53887149855e277b8fa6196f005b748c2c869ce260f",
+                "sha256:ca0838c2d979db4d71ae131214d336ad206b02b648a51e023e2ce2f5f8c2db11",
+                "sha256:cf52413347479035d361e5f760d1c29876b8941a763bb35e5f1505b3421707ae",
+                "sha256:e3415554fd1905c14632254f087f484e704b39865313aea8fe3e4d13017979c7",
+                "sha256:e95ad0255610b719e257150c4f989da51e025dea9a4d686b072c68f99ded538d",
+                "sha256:ff41af4b3bb33a1cdd0bfc1efe9eb0f0392ae946aeca5323309f66568d073efa"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.0"
+            "version": "==3.2.0"
         },
         "python-dateutil": {
             "hashes": [
-                "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
-                "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
+                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.8.1"
+            "version": "==2.8.2"
         },
         "python-dotenv": {
             "hashes": [
@@ -485,28 +524,28 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:0572256c10ddd058e3d315c555538671ddb2737f27eb56189bfbc3483391403f",
-                "sha256:2e685fdbfa5b989af4338b29c408b9157ea6addec15d661104c437980c292be5",
-                "sha256:3595c8b64970c9e5a3f137fa1a9eb64da417e78fb7991d0b098b18a00b776d88",
-                "sha256:3e7df79b42c3015058a5554bfeab6fd4c9906c46560c9ddebb5c652840f3e182",
-                "sha256:4ef3d4df8af40cb6f4d4eaf7b02780109ebabeec334cda26a7899ec9d8de9176",
-                "sha256:53116abd5060a5b4a58489cf689bee259b779e6b7ecd4ce366e7147aa7c9626e",
-                "sha256:5a983d3cebc27294897951a494cebd78af2eae37facf75d9e4ad4f1f62229860",
-                "sha256:5eb8f054eebb351af7490bbb57465ba9662c4e16e1786655c6c7ed530eb9a74e",
-                "sha256:6130e22bf6ee506f7cddde7e0515296d97eb6c6c94f7ef5103c2b77aec5833a7",
-                "sha256:7f4b89c223bd09460b52b669e2e642cab73c28855b540e6ed029692546a86f8d",
-                "sha256:80df8af7039bce92fb4cd1ceb056258631b11b3c627384e2d29bb48d44c0cae7",
-                "sha256:821e75f5c16cd7b0ab0ffe7eb9917e5af7b48c25306b4777287de8d792a5f7f3",
-                "sha256:97ca4552ace1c313707058e774609af59644321e278c3a539322fab2fb09b943",
-                "sha256:998c5e6ea649489302de2c0bc026ed34284f531df89d2bdc8df3a0d44d165739",
-                "sha256:aef6e922aea6f2e6bbb539b413c85210a9ee32757535b84204ebd22723e69704",
-                "sha256:b77ee5e3a9507622e7f98b16122242a3903397f98d1fe3bc269d904a9025e2bc",
-                "sha256:bd4399d4388ca0239a4825e312b3e61b60f743dd6daf49e5870837716502a92a",
-                "sha256:c5d012cb82cc1dcfa72609abaabb4a4ed8113e3e8ac43464508a418c146be57d",
-                "sha256:e7b733d4d98e604109715e11f2ab9340eb45d53f803634ed730039070fc3bc11"
+                "sha256:2a0eeaab01258e0870c4022a6cd329aef3b7c6c2b606bd7cf7bb2ba9820ae561",
+                "sha256:3304bd5bc32e00954ac4b3f4cc382ca8824719bf348aacbec6347337d6b125fe",
+                "sha256:3f52470e0548cdb74fb8ddf06773ffdcca7c97550f903b1c51312ec19243a7f7",
+                "sha256:4729b41a4cdaf4cd011aeac816b532f990bdf97710cef59149d3e293115cf467",
+                "sha256:4ee952f39a4a4c7ba775a32b664b1f4b74818548b65f765987adc14bb78f5802",
+                "sha256:611f9cb459d0707dd8e4de0c96f86e93f61aac7475fcb225e9ec71fecdc5cebf",
+                "sha256:6b47d5fa7ea651054362561a28b1ccc8da9368a39514c1bbf6c0977a1c376764",
+                "sha256:71cfc96297617eab911e22216e8a8597703202e95636d9406df9af5c2ac99a2b",
+                "sha256:787749110a23502031fb1643c55a2236c99c6b989cca703ea2114d65e21728ef",
+                "sha256:90c07ba5f34f33299a428b0d4fa24c30d2ceba44d63f8385b2b05be460819fcb",
+                "sha256:a496b42dbcd04ea9924f5e92be63af3d8e0f43a274b769bfaca0a297327d54ee",
+                "sha256:bc61e3e5ff92d2f32bb263621d54a9cff5e3f7c420af3d1fa122ce2529de2bd9",
+                "sha256:c9951e3746b68974125e5e3445008a4163dd6d20ae0bbdae22b38cb8951dc11b",
+                "sha256:d1388fbac9dd591ea630da75c455f4cc637a7ca5ecb31a6b6cef430914749cde",
+                "sha256:d13f31457f2216e5705304d9f28e2826edf75487410a57aa99263fa4ffd792c2",
+                "sha256:d648aa85dd5074b1ed83008ae987c3fbb53d68af619fce1dee231f4d8bd40e2f",
+                "sha256:da9c6b336e540def0b7fd65603da8abeb306c5fc9a5f4238665cbbb5ff95cf58",
+                "sha256:e101bceeb9e65a90dadbc5ca31283403a2d4667b9c178db29109750568e8d112",
+                "sha256:efdd3825d54c58df2cc394366ca4b9166cf940a0ebddeb87b6c10053deb625ea"
             ],
             "markers": "python_version < '3.10' and python_version >= '3.7'",
-            "version": "==1.7.0"
+            "version": "==1.7.1"
         },
         "seaborn": {
             "hashes": [
@@ -543,54 +582,55 @@
         },
         "simplejson": {
             "hashes": [
-                "sha256:034550078a11664d77bc1a8364c90bb7eef0e44c2dbb1fd0a4d92e3997088667",
-                "sha256:05b43d568300c1cd43f95ff4bfcff984bc658aa001be91efb3bb21df9d6288d3",
-                "sha256:0dd9d9c738cb008bfc0862c9b8fa6743495c03a0ed543884bf92fb7d30f8d043",
-                "sha256:10fc250c3edea4abc15d930d77274ddb8df4803453dde7ad50c2f5565a18a4bb",
-                "sha256:2862beabfb9097a745a961426fe7daf66e1714151da8bb9a0c430dde3d59c7c0",
-                "sha256:292c2e3f53be314cc59853bd20a35bf1f965f3bc121e007ab6fd526ed412a85d",
-                "sha256:2d3eab2c3fe52007d703a26f71cf649a8c771fcdd949a3ae73041ba6797cfcf8",
-                "sha256:2e7b57c2c146f8e4dadf84977a83f7ee50da17c8861fd7faf694d55e3274784f",
-                "sha256:311f5dc2af07361725033b13cc3d0351de3da8bede3397d45650784c3f21fbcf",
-                "sha256:344e2d920a7f27b4023c087ab539877a1e39ce8e3e90b867e0bfa97829824748",
-                "sha256:3fabde09af43e0cbdee407555383063f8b45bfb52c361bc5da83fcffdb4fd278",
-                "sha256:42b8b8dd0799f78e067e2aaae97e60d58a8f63582939af60abce4c48631a0aa4",
-                "sha256:4b3442249d5e3893b90cb9f72c7d6ce4d2ea144d2c0d9f75b9ae1e5460f3121a",
-                "sha256:55d65f9cc1b733d85ef95ab11f559cce55c7649a2160da2ac7a078534da676c8",
-                "sha256:5c659a0efc80aaaba57fcd878855c8534ecb655a28ac8508885c50648e6e659d",
-                "sha256:72d8a3ffca19a901002d6b068cf746be85747571c6a7ba12cbcf427bfb4ed971",
-                "sha256:75ecc79f26d99222a084fbdd1ce5aad3ac3a8bd535cd9059528452da38b68841",
-                "sha256:76ac9605bf2f6d9b56abf6f9da9047a8782574ad3531c82eae774947ae99cc3f",
-                "sha256:7d276f69bfc8c7ba6c717ba8deaf28f9d3c8450ff0aa8713f5a3280e232be16b",
-                "sha256:7f10f8ba9c1b1430addc7dd385fc322e221559d3ae49b812aebf57470ce8de45",
-                "sha256:8042040af86a494a23c189b5aa0ea9433769cc029707833f261a79c98e3375f9",
-                "sha256:813846738277729d7db71b82176204abc7fdae2f566e2d9fcf874f9b6472e3e6",
-                "sha256:845a14f6deb124a3bcb98a62def067a67462a000e0508f256f9c18eff5847efc",
-                "sha256:869a183c8e44bc03be1b2bbcc9ec4338e37fa8557fc506bf6115887c1d3bb956",
-                "sha256:8acf76443cfb5c949b6e781c154278c059b09ac717d2757a830c869ba000cf8d",
-                "sha256:8f713ea65958ef40049b6c45c40c206ab363db9591ff5a49d89b448933fa5746",
-                "sha256:934115642c8ba9659b402c8bdbdedb48651fb94b576e3b3efd1ccb079609b04a",
-                "sha256:9551f23e09300a9a528f7af20e35c9f79686d46d646152a0c8fc41d2d074d9b0",
-                "sha256:9a2b7543559f8a1c9ed72724b549d8cc3515da7daf3e79813a15bdc4a769de25",
-                "sha256:a55c76254d7cf8d4494bc508e7abb993a82a192d0db4552421e5139235604625",
-                "sha256:ad8f41c2357b73bc9e8606d2fa226233bf4d55d85a8982ecdfd55823a6959995",
-                "sha256:af4868da7dd53296cd7630687161d53a7ebe2e63814234631445697bd7c29f46",
-                "sha256:afebfc3dd3520d37056f641969ce320b071bc7a0800639c71877b90d053e087f",
-                "sha256:b59aa298137ca74a744c1e6e22cfc0bf9dca3a2f41f51bc92eb05695155d905a",
-                "sha256:bc00d1210567a4cdd215ac6e17dc00cb9893ee521cee701adfd0fa43f7c73139",
-                "sha256:c1cb29b1fced01f97e6d5631c3edc2dadb424d1f4421dad079cb13fc97acb42f",
-                "sha256:c94dc64b1a389a416fc4218cd4799aa3756f25940cae33530a4f7f2f54f166da",
-                "sha256:ceaa28a5bce8a46a130cd223e895080e258a88d51bf6e8de2fc54a6ef7e38c34",
-                "sha256:cff6453e25204d3369c47b97dd34783ca820611bd334779d22192da23784194b",
-                "sha256:d0b64409df09edb4c365d95004775c988259efe9be39697d7315c42b7a5e7e94",
-                "sha256:d4813b30cb62d3b63ccc60dd12f2121780c7a3068db692daeb90f989877aaf04",
-                "sha256:da3c55cdc66cfc3fffb607db49a42448785ea2732f055ac1549b69dcb392663b",
-                "sha256:e058c7656c44fb494a11443191e381355388443d543f6fc1a245d5d238544396",
-                "sha256:fed0f22bf1313ff79c7fc318f7199d6c2f96d4de3234b2f12a1eab350e597c06",
-                "sha256:ffd4e4877a78c84d693e491b223385e0271278f5f4e1476a4962dca6824ecfeb"
+                "sha256:065230b9659ac38c8021fa512802562d122afb0cf8d4b89e257014dcddb5730a",
+                "sha256:07707ba69324eaf58f0c6f59d289acc3e0ed9ec528dae5b0d4219c0d6da27dc5",
+                "sha256:10defa88dd10a0a4763f16c1b5504e96ae6dc68953cfe5fc572b4a8fcaf9409b",
+                "sha256:140eb58809f24d843736edb8080b220417e22c82ac07a3dfa473f57e78216b5f",
+                "sha256:188f2c78a8ac1eb7a70a4b2b7b9ad11f52181044957bf981fb3e399c719e30ee",
+                "sha256:1c2688365743b0f190392e674af5e313ebe9d621813d15f9332e874b7c1f2d04",
+                "sha256:24e413bd845bd17d4d72063d64e053898543fb7abc81afeae13e5c43cef9c171",
+                "sha256:2b59acd09b02da97728d0bae8ff48876d7efcbbb08e569c55e2d0c2e018324f5",
+                "sha256:2df15814529a4625ea6f7b354a083609b3944c269b954ece0d0e7455872e1b2a",
+                "sha256:352c11582aa1e49a2f0f7f7d8fd5ec5311da890d1354287e83c63ab6af857cf5",
+                "sha256:36b08b886027eac67e7a0e822e3a5bf419429efad7612e69501669d6252a21f2",
+                "sha256:376023f51edaf7290332dacfb055bc00ce864cb013c0338d0dea48731f37e42f",
+                "sha256:3ba82f8b421886f4a2311c43fb98faaf36c581976192349fef2a89ed0fcdbdef",
+                "sha256:3d72aa9e73134dacd049a2d6f9bd219f7be9c004d03d52395831611d66cedb71",
+                "sha256:40ece8fa730d1a947bff792bcc7824bd02d3ce6105432798e9a04a360c8c07b0",
+                "sha256:417b7e119d66085dc45bdd563dcb2c575ee10a3b1c492dd3502a029448d4be1c",
+                "sha256:42b7c7264229860fe879be961877f7466d9f7173bd6427b3ba98144a031d49fb",
+                "sha256:457d9cfe7ece1571770381edccdad7fc255b12cd7b5b813219441146d4f47595",
+                "sha256:4a6943816e10028eeed512ea03be52b54ea83108b408d1049b999f58a760089b",
+                "sha256:5b94df70bd34a3b946c0eb272022fb0f8a9eb27cad76e7f313fedbee2ebe4317",
+                "sha256:5f5051a13e7d53430a990604b532c9124253c5f348857e2d5106d45fc8533860",
+                "sha256:5f7f53b1edd4b23fb112b89208377480c0bcee45d43a03ffacf30f3290e0ed85",
+                "sha256:5fe8c6dcb9e6f7066bdc07d3c410a2fca78c0d0b4e0e72510ffd20a60a20eb8e",
+                "sha256:71a54815ec0212b0cba23adc1b2a731bdd2df7b9e4432718b2ed20e8aaf7f01a",
+                "sha256:7332f7b06d42153255f7bfeb10266141c08d48cc1a022a35473c95238ff2aebc",
+                "sha256:78c6f0ed72b440ebe1892d273c1e5f91e55e6861bea611d3b904e673152a7a4c",
+                "sha256:7c9b30a2524ae6983b708f12741a31fbc2fb8d6fecd0b6c8584a62fd59f59e09",
+                "sha256:86fcffc06f1125cb443e2bed812805739d64ceb78597ac3c1b2d439471a09717",
+                "sha256:87572213965fd8a4fb7a97f837221e01d8fddcfb558363c671b8aa93477fb6a2",
+                "sha256:8e595de17178dd3bbeb2c5b8ea97536341c63b7278639cb8ee2681a84c0ef037",
+                "sha256:917f01db71d5e720b731effa3ff4a2c702a1b6dacad9bcdc580d86a018dfc3ca",
+                "sha256:91cfb43fb91ff6d1e4258be04eee84b51a4ef40a28d899679b9ea2556322fb50",
+                "sha256:aa86cfdeb118795875855589934013e32895715ec2d9e8eb7a59be3e7e07a7e1",
+                "sha256:ade09aa3c284d11f39640aebdcbb748e1996f0c60504f8c4a0c5a9fec821e67a",
+                "sha256:b2a5688606dffbe95e1347a05b77eb90489fe337edde888e23bbb7fd81b0d93b",
+                "sha256:b92fbc2bc549c5045c8233d954f3260ccf99e0f3ec9edfd2372b74b350917752",
+                "sha256:c2d5334d935af711f6d6dfeec2d34e071cdf73ec0df8e8bd35ac435b26d8da97",
+                "sha256:cb0afc3bad49eb89a579103616574a54b523856d20fc539a4f7a513a0a8ba4b2",
+                "sha256:ce66f730031b9b3683b2fc6ad4160a18db86557c004c3d490a29bf8d450d7ab9",
+                "sha256:e29b9cea4216ec130df85d8c36efb9985fda1c9039e4706fb30e0fb6a67602ff",
+                "sha256:e2cc4b68e59319e3de778325e34fbff487bfdb2225530e89995402989898d681",
+                "sha256:e90d2e219c3dce1500dda95f5b893c293c4d53c4e330c968afbd4e7a90ff4a5b",
+                "sha256:f13c48cc4363829bdfecc0c181b6ddf28008931de54908a492dc8ccd0066cd60",
+                "sha256:f550730d18edec4ff9d4252784b62adfe885d4542946b6d5a54c8a6521b56afd",
+                "sha256:fa843ee0d34c7193f5a816e79df8142faff851549cab31e84b526f04878ac778",
+                "sha256:fe1c33f78d2060719d52ea9459d97d7ae3a5b707ec02548575c4fbed1d1d345b"
             ],
             "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.17.2"
+            "version": "==3.17.5"
         },
         "six": {
             "hashes": [
@@ -640,13 +680,6 @@
             "index": "pypi",
             "version": "==7.25.0"
         },
-        "ipython-genutils": {
-            "hashes": [
-                "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8",
-                "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"
-            ],
-            "version": "==0.2.0"
-        },
         "jedi": {
             "hashes": [
                 "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93",
@@ -657,11 +690,11 @@
         },
         "matplotlib-inline": {
             "hashes": [
-                "sha256:5cf1176f554abb4fa98cb362aa2b55c500147e4bdbb07e3fda359143e1da0811",
-                "sha256:f41d5ff73c9f5385775d5c0bc13b424535c8402fe70ea8210f93e11f3683993e"
+                "sha256:a04bfba22e0d1395479f866853ec1ee28eea1485c1d69a6faf00dc3e24ff34ee",
+                "sha256:aed605ba3b72462d64d475a21a9296f400a19c4f74a31b59103d2a99ffd5aa5c"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.1.2"
+            "version": "==0.1.3"
         },
         "parso": {
             "hashes": [
@@ -688,11 +721,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:08360ee3a3148bdb5163621709ee322ec34fc4375099afa4bbf751e9b7b7fa4f",
-                "sha256:7089d8d2938043508aa9420ec18ce0922885304cddae87fb96eebca942299f88"
+                "sha256:6076e46efae19b1e0ca1ec003ed37a933dc94b4d20f486235d436e64771dcd5c",
+                "sha256:eb71d5a6b72ce6db177af4a7d4d7085b99756bf656d98ffcc4fecd36850eea6c"
             ],
-            "markers": "python_full_version >= '3.6.1'",
-            "version": "==3.0.19"
+            "markers": "python_full_version >= '3.6.2'",
+            "version": "==3.0.20"
         },
         "ptyprocess": {
             "hashes": [
@@ -703,19 +736,19 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f",
-                "sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e"
+                "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380",
+                "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==2.9.0"
+            "version": "==2.10.0"
         },
         "traitlets": {
             "hashes": [
-                "sha256:178f4ce988f69189f7e523337a3e11d91c786ded9360174a3d9ca83e79bc5396",
-                "sha256:69ff3f9d5351f31a7ad80443c2674b7099df13cc41fc5fa6e2f6d3b0330b0426"
+                "sha256:03f172516916220b58c9f19d7f854734136dd9528103d04e9bf139a92c9f54c4",
+                "sha256:bd382d7ea181fbbcce157c133db9a829ce06edffe097bcf3ab945b435452b46d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.0.5"
+            "version": "==5.1.0"
         },
         "wcwidth": {
             "hashes": [

From 4e27d501867a02d7c0ddab7d456723bfd0380ae3 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Tue, 12 Oct 2021 07:09:14 -0500
Subject: [PATCH 114/359] Approx bankfull stage and apply composite roughness
 in synthetic rating curves

See Changelog for 3.0.22.7
---
 docs/CHANGELOG.md                  |  14 ++
 tools/eval_plots.py                |  42 +++---
 tools/identify_src_bankfull.py     | 211 ++++++++++++++++++++++++++
 tools/inundation.py                |  15 +-
 tools/vary_mannings_n_composite.py | 231 +++++++++++++++++++++++++++++
 5 files changed, 491 insertions(+), 22 deletions(-)
 create mode 100755 tools/identify_src_bankfull.py
 create mode 100755 tools/vary_mannings_n_composite.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2a3df5c29..fa84cd339 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,20 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.7 - 2021-10-08 - [PR #467](https://github.com/NOAA-OWP/cahaba/pull/467)
+
+These "tool" enhancements 1) delineate in-channel vs. out-of-channel geometry to allow more targeted development of key physical drivers influencing the SRC calculations (e.g. bathymetry & Manning’s n) #418 and 2) applies a variable/composite Manning’s roughness (n) using user provided csv with in-channel vs. overbank roughness values #419 & #410.
+
+## Additions
+- `identify_src_bankfull.p`: new post-processing tool that ingests a flow csv (e.g. NWM 1.5yr recurr flow) to approximate the bankfull STG and then calculate the channel vs. overbank proportions using the volume and hydraulic radius variables
+- `vary_mannings_n_composite.py`: new post-processing tool that ingests a csv containing feature_id, channel roughness, and overbank roughness and then generates composite n values via the channel ratio variable
+
+## Changes
+- `eval_plots.py`: modified the plot legend text to display full label for development tests
+- `inundation.py`: added new optional argument (-n) and corresponding function to produce a csv containing the stage value (and SRC variables) calculated from the flow to stage interpolation.
+
+<br/><br/>
+
 ## v3.0.22.6 - 2021-09-13 - [PR #462](https://github.com/NOAA-OWP/cahaba/pull/462)
 
 This new workflow ingests FIM point observations from users and “corrects” the synthetic rating curves to produce the desired FIM extent at locations where feedback is available (locally calibrate FIM).
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index ea339867e..6c2dcf00f 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -85,12 +85,12 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
                 label_dict[label] = 'FIM 1'
             elif 'fim_2' in label:
                 label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
-            elif 'fim_3' in label:
+            elif 'fim_3' in label and len(label) < 20:
                 label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
             else:
-                label_dict[label] = label + ' ' + fim_configuration.lower()
+                label_dict[label] = label
         #Define simplified labels as a list.
         new_labels = [label_dict[label] for label in org_labels]
         #Define legend location. FAR needs to be in different location than CSI/POD.
@@ -99,7 +99,7 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
         else:
             legend_location = 'lower left'
         #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 14, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
     #Print textbox if supplied
     if textbox_str:
         box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
@@ -252,16 +252,16 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
                 label_dict[label] = 'FIM 1'
             elif 'fim_2' in label:
                 label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
-            elif 'fim_3' in label:
+            elif 'fim_3' in label and len(label) < 20:
                 label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
             else:
-                label_dict[label] = label + ' ' + fim_configuration.lower()
+                label_dict[label] = label
         #Define simplified labels as a list.
         new_labels = [label_dict[label] for label in org_labels]
         #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 14, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
     #Add Textbox
     if textbox_str:
         box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
@@ -382,12 +382,12 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             site subdirectories are the following files:
                 csi_<site_name>_<benchmark_source>_<configuration>.png: A barplot
                     of CSI for each version for all magnitudes for the site.
-        Optional (if spatial argument supplied): 
-            fim_performance_points.shp -- A shapefile of ahps points with 
+        Optional (if spatial argument supplied):
+            fim_performance_points.shp -- A shapefile of ahps points with
             metrics contained in attribute table.
             fim_performance_polys.shp -- A shapefile of huc8 polygons with
             metrics contained in attribute table.
-            
+
 
 
     Parameters
@@ -413,14 +413,14 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         field in metrics_csv. CSI, POD, TPR are currently calculated, if
         additional statistics are desired formulas would need to be coded.
     spatial : BOOL, optional
-        Creates spatial datasets of the base unit (ble: huc polygon, ahps: point) 
-        with metrics contained in attribute tables. The geospatial data is 
+        Creates spatial datasets of the base unit (ble: huc polygon, ahps: point)
+        with metrics contained in attribute tables. The geospatial data is
         either supplied in the .env file (WBD Huc layer) or from WRDS (ahps).
         The outputs are consistent with requirements set forth by the vizualization team.
         Additionally, there is a commented out section where if the user
         passes the extent files generated during creation of nws/usgs ahps
         preprocessing, the actual maps and flows used for evaluation are
-        appended to the ahps shapefile output. 
+        appended to the ahps shapefile output.
     fim_1_ms: BOOL
         Default is false. If True then fim_1 rows are duplicated with
         extent_config set to MS. This allows for FIM 1 to be included
@@ -606,7 +606,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             all_ahps_datasets = usgs_dataset.append(nws_dataset)
             all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source'])
             all_ahps_datasets.rename(columns = {'benchmark_source':'source'}, inplace = True)
-            
+
             #Get spatial data from WRDS
             #Get metadata from WRDS API
             select_by = 'nws_lid'
@@ -618,26 +618,26 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             #Trim out unecessary columns and rename remaining columns
             gdf = gdf.filter(['identifiers_nws_lid', 'nws_data_name', 'identifiers_nwm_feature_id','nws_data_wfo','nws_data_state','nws_data_county','geometry'])
             gdf.rename(columns = {'identifiers_nws_lid':'nws_lid', 'nws_data_name':'lid_name','identifiers_nwm_feature_id':'feature_id','nws_data_wfo':'wfo','nws_data_state':'state','nws_data_county':'county','HUC8':'huc8'}, inplace = True)
-            
-            #Join spatial data to metric data        
-            gdf['nws_lid'] = gdf['nws_lid'].str.lower()        
+
+            #Join spatial data to metric data
+            gdf['nws_lid'] = gdf['nws_lid'].str.lower()
             joined = gdf.merge(all_ahps_datasets, on = 'nws_lid')
             #Project to VIZ projection and write to file
             joined = joined.to_crs(VIZ_PROJECTION)
             joined.to_file(Path(workspace) / 'fim_performance_points.shp')
         else:
             print('NWS/USGS MS datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a MS version.')
-            
+
         ################################################################
         #This section joins ble (FR) metrics to a spatial layer of HUCs.
         ################################################################
-        if all_datasets.get(('ble','FR')) and all_datasets.get(('ifc','FR')):            
+        if all_datasets.get(('ble','FR')) and all_datasets.get(('ifc','FR')):
             #Select BLE, FR dataset.
-            ble_dataset, sites = all_datasets.get(('ble','FR'))         
-            ifc_dataset, sites = all_datasets.get(('ifc','FR'))               
+            ble_dataset, sites = all_datasets.get(('ble','FR'))
+            ifc_dataset, sites = all_datasets.get(('ifc','FR'))
             huc_datasets = ble_dataset.append(ifc_dataset)
             #Read in HUC spatial layer
-            wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')             
+            wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')
             #Join metrics to HUC spatial layer
             wbd_with_metrics = wbd_gdf.merge(huc_datasets, how = 'inner', left_on = 'HUC8', right_on = 'huc')
             #Filter out unnecessary columns
diff --git a/tools/identify_src_bankfull.py b/tools/identify_src_bankfull.py
new file mode 100755
index 000000000..b011bc31e
--- /dev/null
+++ b/tools/identify_src_bankfull.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import pandas as pd
+import argparse
+import matplotlib.pyplot as plt
+import seaborn as sns
+from functools import reduce
+from multiprocessing import Pool
+from os.path import isfile, join, dirname, isdir
+import shutil
+import warnings
+from pathlib import Path
+import datetime as dt
+sns.set_theme(style="whitegrid")
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+"""
+    Identify the SRC bankfull stage values using the NWM 1.5yr flows
+
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders.
+    nwm_flow_dir : str
+        Directory containing NWM recurrence flows files.
+    number_of_jobs : str
+        Number of jobs.
+    plots : str
+        Flag to create SRC plots for all hydroids (True/False)
+"""
+
+def nwm_1_5_bankfull_lookup(args):
+
+    src_full_filename           = args[0]
+    src_modify_filename         = args[1]
+    df_nwm1_5                   = args[2]
+    huc                         = args[3]
+    src_plot_option             = args[4]
+    huc_output_dir              = args[5]
+
+    ## Read the src_full_crosswalked.csv
+    #print('Processing: ' + str(huc))
+    log_text = 'Calculating: ' + str(huc) + '\n'
+    df_src = pd.read_csv(src_full_filename,dtype={'HydroID': int,'feature_id': int})
+
+    ## NWM recurr rename discharge var
+    df_nwm1_5 = df_nwm1_5.rename(columns={'discharge':'discharge_1_5'})
+
+    ## Combine the nwm 1.5yr flows into the SRC via feature_id
+    df_src = df_src.merge(df_nwm1_5,how='left',on='feature_id')
+
+    ## Check if there are any missing data, negative or zero flow values in the discharge_1_5
+    check_null = df_src['discharge_1_5'].isnull().sum()
+    if check_null > 0:
+        log_text += 'Missing feature_id in crosswalk for huc: ' + str(huc) + ' --> these featureids will be ignored in bankfull calcs (~' + str(check_null/84) +  ' features) \n'
+        ## Fill missing/nan nwm discharge_1_5 values with -999 to handle later
+        df_src['discharge_1_5'] = df_src['discharge_1_5'].fillna(-999)
+    negative_flows = len(df_src.loc[(df_src.discharge_1_5 <= 0) & (df_src.discharge_1_5 != -999)])
+    if negative_flows > 0:
+        log_text += 'HUC: ' + str(huc) + ' --> Negative or zero flow values found (likely lakeid loc)\n'
+
+    ## Check which channel geometry parameters exist in df (use bathy adjusted vars by default) --> this is needed to handle differences btw BARC & no-BARC outputs
+    if 'HydraulicRadius (m)_bathy_adj' in df_src:
+        hradius_var = 'HydraulicRadius (m)_bathy_adj'
+    else:
+        hradius_var = 'HydraulicRadius (m)'
+
+    if 'Volume (m3)_bathy_adj' in df_src:
+        volume_var = 'Volume (m3)_bathy_adj'
+    else:
+        volume_var = 'Volume (m3)'
+
+    ## Locate the closest SRC discharge value to the NWM 1.5yr flow
+    df_src['Q_1_5_find'] = (df_src['discharge_1_5'] - df_src['Discharge (m3s-1)']).abs()
+
+    ## Check for any missing/null entries in the input SRC
+    if df_src['Q_1_5_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
+        log_text += 'HUC: ' + str(huc) + ' --> Null values found in "Q_1_5_find" calc. These will be filled with 999999 () \n'
+        ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
+        df_src['Q_1_5_find'] = df_src['Q_1_5_find'].fillna(999999)
+    if df_src['HydroID'].isnull().values.any():
+        log_text += 'HUC: ' + str(huc) + ' --> Null values found in "HydroID"... \n'
+
+    df_1_5 = df_src[['Stage','HydroID',volume_var,hradius_var,'Q_1_5_find']] # create new subset df to perform the Q_1_5 lookup
+    df_1_5 = df_1_5[df_1_5['Stage'] > 0.0] # Ensure bankfull stage is greater than stage=0
+    df_1_5.reset_index(drop=True, inplace=True)
+    df_1_5 = df_1_5.loc[df_1_5.groupby('HydroID')['Q_1_5_find'].idxmin()].reset_index(drop=True) # find the index of the Q_1_5_find (closest matching flow)
+    df_1_5 = df_1_5.rename(columns={'Stage':'Stage_1_5',volume_var:'Volume_bankfull',hradius_var:'HRadius_bankfull'}) # rename volume to use later for channel portion calc
+    df_src = df_src.merge(df_1_5[['Stage_1_5','HydroID','Volume_bankfull','HRadius_bankfull']],how='left',on='HydroID')
+    df_src.drop(['Q_1_5_find'], axis=1, inplace=True)
+
+    ## Calculate the channel portion of bankfull Volume
+    df_src['chann_volume_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
+    df_src['chann_volume_ratio'].where(df_src['Stage'] == 0, df_src['Volume_bankfull'] / (df_src[volume_var]),inplace=True)
+    #df_src['chann_volume_ratio'] = df_src['chann_volume_ratio'].clip_upper(1.0)
+    df_src['chann_volume_ratio'].where(df_src['chann_volume_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
+    df_src['chann_volume_ratio'].where(df_src['discharge_1_5'] > 0.0, 0.0, inplace=True) # if the discharge_1_5 value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+    #df_src.drop(['Volume_bankfull'], axis=1, inplace=True)
+
+    ## Calculate the channel portion of bankfull Hydraulic Radius
+    df_src['chann_hradius_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
+    df_src['chann_hradius_ratio'].where(df_src['Stage'] == 0, df_src['HRadius_bankfull'] / (df_src[hradius_var]),inplace=True)
+    #df_src['chann_hradius_ratio'] = df_src['HRadius_bankfull'] / (df_src[hradius_var]+.0001) # old adding 0.01 to avoid dividing by 0 at stage=0
+    df_src['chann_hradius_ratio'].where(df_src['chann_hradius_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
+    df_src['chann_hradius_ratio'].where(df_src['discharge_1_5'] > 0.0, 0.0, inplace=True) # if the discharge_1_5 value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+    #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
+
+    ## mask bankfull variables when the 1.5yr flow value is <= 0
+    df_src['Stage_1_5'].mask(df_src['discharge_1_5'] <= 0.0,inplace=True)
+
+    ## Create a new column to identify channel/floodplain via the bankfull stage value
+    df_src.loc[df_src['Stage'] <= df_src['Stage_1_5'], 'channel_fplain_1_5'] = 'channel'
+    df_src.loc[df_src['Stage'] > df_src['Stage_1_5'], 'channel_fplain_1_5'] = 'floodplain'
+    df_src['channel_fplain_1_5'] = df_src['channel_fplain_1_5'].fillna('channel')
+
+    ## Output new SRC with bankfull column
+    df_src.to_csv(src_modify_filename,index=False)
+    log_text += 'Completed: ' + str(huc) + '\n'
+    log_file.write(log_text)
+
+    ## plot rating curves (optional arg)
+    if src_plot_option == 'True':
+        if isdir(huc_output_dir) == False:
+            os.mkdir(huc_output_dir)
+        generate_src_plot(df_src, huc_output_dir)
+
+
+def generate_src_plot(df_src, plt_out_dir):
+
+    ## create list of unique hydroids
+    hydroids = df_src.HydroID.unique().tolist()
+    #hydroids = [17820017]
+
+    for hydroid in hydroids:
+        print("Creating SRC plot: " + str(hydroid))
+        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+
+        f, ax = plt.subplots(figsize=(6.5, 6.5))
+        ax.set_title(str(hydroid))
+        sns.despine(f, left=True, bottom=True)
+        #sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, ax=ax)
+        #sns.lineplot(x='Discharge (m3s-1)', y='Stage_1_5', data=plot_df, color='green', ax=ax)
+        #plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_1_5'],alpha=0.5)
+        #plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+        sns.scatterplot(x='chann_volume_ratio', y='Stage', data=plot_df, ax=ax, label="chann_volume_ratio", s=38)
+        sns.scatterplot(x='chann_hradius_ratio', y='Stage', data=plot_df, ax=ax, label="chann_hradius_ratio", s=12)
+        ax.legend()
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
+        plt.close()
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Identify bankfull stage for each hydroid synthetic rating curve')
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-flows','--bankfull-flow-input',help='NWM recurrence flows dir',required=True,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
+    parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    bankfull_flow_filepath = args['bankfull_flow_input']
+    number_of_jobs = args['number_of_jobs']
+    src_plot_option = args['src_plot_option']
+    procs_list = []
+
+    ## Print message to user and initiate run clock
+    print('Writing progress to log file here: ' + str(join(fim_dir,'bankfull_detect.log')))
+    print('This may take a few minutes...')
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+
+    ## Check that the bankfull flow filepath exists and read to dataframe
+    if not isfile(bankfull_flow_filepath):
+        print('!!! Can not find the input recurr flow file: ' + str(bankfull_flow_filepath))
+    else:
+        df_nwm1_5 = pd.read_csv(bankfull_flow_filepath,dtype={'feature_id': int})
+
+        ## initiate log file
+        sys.__stdout__ = sys.stdout
+        log_file = open(join(fim_dir,'log_bankfull_indentify.log'),"w")
+        sys.stdout = log_file
+        log_file.write('START TIME: ' + str(begin_time) + '\n')
+
+        huc_list  = os.listdir(fim_dir)
+        for huc in huc_list:
+            if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+                src_full_filename = join(fim_dir,huc,'src_full_crosswalked.csv')
+                src_modify_filename = join(fim_dir,huc,'src_full_crosswalked_bankfull.csv')
+                huc_output_dir = join(fim_dir,huc,'src_plots')
+
+                if isfile(src_full_filename):
+                    procs_list.append([src_full_filename, src_modify_filename, df_nwm1_5, huc, src_plot_option, huc_output_dir])
+                else:
+                    log_file.write(str(huc) + ' --> can not find the src_full_crosswalked.csv in the fim output dir: ' + str(join(fim_dir,huc)) + '\n')
+
+        ## Initiate multiprocessing
+        print(f"Identifying bankfull thresholds for {len(procs_list)} hucs using {number_of_jobs} jobs")
+        with Pool(processes=number_of_jobs) as pool:
+            pool.map(nwm_1_5_bankfull_lookup, procs_list)
+
+        ## Record run time and close log file
+        end_time = dt.datetime.now()
+        log_file.write('END TIME: ' + str(end_time) + '\n')
+        tot_run_time = end_time - begin_time
+        log_file.write(str(tot_run_time))
+        sys.stdout = sys.__stdout__
+        log_file.close()
diff --git a/tools/inundation.py b/tools/inundation.py
index 667895cd0..fec6e4cfc 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -22,7 +22,7 @@
 def inundate(
              rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=None,hucs_layerName=None,
              subset_hucs=None,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
-             depths=None,out_raster_profile=None,out_vector_profile=None,quiet=False
+             depths=None,out_raster_profile=None,out_vector_profile=None,src_table=None,quiet=False
             ):
     """
 
@@ -156,6 +156,8 @@ def inundate(
         raise TypeError("Pass hydro table csv")
 
     if catchmentStagesDict is not None:
+        if src_table is not None:
+            create_src_subset_csv(hydro_table,catchmentStagesDict,src_table)
 
         # make windows generator
         window_gen = __make_windows_generator(rem,catchments,catchment_poly,mask_type,catchmentStagesDict,inundation_raster,inundation_polygon,
@@ -541,6 +543,16 @@ def __vprint(message,verbose):
     if verbose:
         print(message)
 
+def create_src_subset_csv(hydro_table,catchmentStagesDict,src_table):
+    src_df = pd.DataFrame.from_dict(catchmentStagesDict, orient='index')
+    src_df.reset_index(inplace=True)
+    src_df.columns = ['HydroID','stage_inund']
+    df_htable = pd.read_csv(hydro_table,dtype={'HydroID': int})
+    df_htable = df_htable.merge(src_df,how='left',on='HydroID')
+    df_htable['find_match'] = (df_htable['stage'] - df_htable['stage_inund']).abs()
+    df_htable = df_htable.loc[df_htable.groupby('HydroID')['find_match'].idxmin()].reset_index(drop=True)
+    df_htable.to_csv(src_table,index=False)
+
 
 if __name__ == '__main__':
 
@@ -560,6 +572,7 @@ def __vprint(message,verbose):
     parser.add_argument('-i','--inundation-raster',help='Inundation Raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
     parser.add_argument('-p','--inundation-polygon',help='Inundation polygon output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
     parser.add_argument('-d','--depths',help='Depths raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
+    parser.add_argument('-n','--src-table',help='Output table with the SRC lookup/interpolation. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
     parser.add_argument('-q','--quiet',help='Quiet terminal output',required=False,default=False,action='store_true')
 
     # extract to dictionary
diff --git a/tools/vary_mannings_n_composite.py b/tools/vary_mannings_n_composite.py
new file mode 100755
index 000000000..4d48ff69e
--- /dev/null
+++ b/tools/vary_mannings_n_composite.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import pandas as pd
+import argparse
+import matplotlib.pyplot as plt
+import seaborn as sns
+from functools import reduce
+from multiprocessing import Pool
+from os.path import isfile, join, dirname, isdir
+import shutil
+import warnings
+from pathlib import Path
+import datetime as dt
+sns.set_theme(style="whitegrid")
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+"""
+    Vary the Manning's n values for in-channel vs. floodplain
+
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders.
+    channel_ratio_src_column : str
+        SRC attribute containing the channel vs. floodplain attribute
+    mann_n_table : str
+        Path to a csv file containing Manning's n values by feature_id
+    hydrotable_suffix : str
+        Suffix to append to the new hydroTable csv file
+    number_of_jobs : str
+        Number of jobs.
+    src_plot_option : str
+        Optional (True or False): use this flag to crate src plots for all hydroids
+"""
+
+def variable_mannings_calc(args):
+
+    in_src_bankfull_filename    = args[0]
+    channel_ratio_src_column    = args[1]
+    df_mann                     = args[2]
+    huc                         = args[3]
+    out_src_bankfull_filename   = args[4]
+    htable_filename             = args[5]
+    new_htable_filename         = args[6]
+    src_plot_option             = args[7]
+    huc_output_dir              = args[8]
+
+    ## Read the src_full_crosswalked.csv
+    log_text = 'Calculating: ' + str(huc) + '\n'
+    df_src = pd.read_csv(in_src_bankfull_filename,dtype={'feature_id': 'int64'})
+
+    ## Check that the channel ratio column the user specified exists in the def
+    if channel_ratio_src_column not in df_src.columns:
+        log_text += str(huc) + ' --> ' + out_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
+
+    ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the channel_ratio_src_column)
+    df_src = df_src.merge(df_mann,  how='left', on='feature_id')
+    check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
+    if check_null > 0:
+        log_text += str(huc) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
+
+    ## Calculate composite Manning's n using the channel geometry ratio attribute given by user (e.g. chann_hradius_ratio or chann_vol_ratio)
+    df_src['comp_ManningN'] = (df_src[channel_ratio_src_column]*df_src['channel_n']) + ((1.0 - df_src[channel_ratio_src_column])*df_src['overbank_n'])
+    #print('Done calculating composite Manning n (' + channel_ratio_src_column + '): ' + str(huc))
+
+    ## Check if there are any missing data in the composite ManningN column
+    check_null_comp = df_src['comp_ManningN'].isnull().sum()
+    if check_null_comp > 0:
+        log_text += str(huc) + ' --> ' + 'Missing values in the comp_ManningN calculation' + ' --> missing entries= ' + str(check_null_comp/84)  + '\n'
+
+    ## Check which channel geometry parameters exist in the src_df (use bathy adjusted vars by default) --> this is needed to handle differences btw BARC & no-BARC outputs
+    if 'HydraulicRadius (m)_bathy_adj' in df_src:
+        hydr_radius = 'HydraulicRadius (m)_bathy_adj'
+    else:
+        hydr_radius = 'HydraulicRadius (m)'
+
+    if 'WetArea (m2)_bathy_adj' in df_src:
+        wet_area = 'WetArea (m2)_bathy_adj'
+    else:
+        wet_area = 'WetArea (m2)'
+
+    ## Calculate Q using Manning's equation
+    #df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True) # rename the previous Discharge column
+    df_src['Discharge (m3s-1)_varMann'] = df_src[wet_area]* \
+    pow(df_src[hydr_radius],2.0/3)* \
+    pow(df_src['SLOPE'],0.5)/df_src['comp_ManningN']
+
+    ## Set Q values to 0 and -999 for specified criteria
+    df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == 0,0,inplace=True)
+    if 'Thalweg_burn_elev' in df_src:
+        df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == df_src['Thalweg_burn_elev'],0,inplace=True)
+        df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] < df_src['Thalweg_burn_elev'],-999,inplace=True)
+
+    ## Output new SRC with bankfull column
+    df_src.to_csv(out_src_bankfull_filename,index=False)
+
+    ## Output new hydroTable with updated discharge and ManningN column
+    df_src_trim = df_src[['HydroID','Stage','Discharge (m3s-1)_varMann','comp_ManningN']]
+    df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_varMann': 'discharge_cms','comp_ManningN':'ManningN'})
+    df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
+    df_htable.drop(['discharge_cms','ManningN'], axis=1, inplace=True) # drop the original discharge column to be replaced with updated version
+    df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
+    #extra_cols = ['HydraulicRadius (m)','WetArea (m2)','SLOPE','ManningN']
+    #if df_htable.columns.isin(cols).all()):
+    #    df_htable = df_htable[['HydroID','feature_id','stage','discharge_cms','HydraulicRadius (m)','WetArea (m2)','SLOPE','ManningN','HUC','LakeID']]
+    #else:
+    #    df_htable = df_htable[['HydroID','feature_id','stage','discharge_cms','HUC','LakeID']] # set column order for hydroTable output
+    df_htable.to_csv(new_htable_filename,index=False)
+    log_text += 'Completed: ' + str(huc) + '\n'
+    log_file.write(str(log_text))
+
+    ## plot rating curves
+    if src_plot_option == 'True':
+        if isdir(huc_output_dir) == False:
+            os.mkdir(huc_output_dir)
+        generate_src_plot(df_src, huc_output_dir)
+
+
+def generate_src_plot(df_src, plt_out_dir):
+
+    ## create list of unique hydroids
+    hydroids = df_src.HydroID.unique().tolist()
+
+    ## plot each hydroid SRC in the huc
+    for hydroid in hydroids:
+        print("Creating SRC plot: " + str(hydroid))
+        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+
+        f, ax = plt.subplots(figsize=(6.5, 6.5))
+        ax.set_title(str(hydroid))
+        sns.despine(f, left=True, bottom=True)
+        sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
+        sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
+        sns.lineplot(x='Discharge (m3s-1)', y='Stage_1_5', data=plot_df, color='green', ax=ax)
+        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_1_5'],alpha=0.5)
+        plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+        ax.legend()
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
+        plt.close()
+
+#    for hydroid in hydroids:
+#        print("Creating SRC plot: " + str(hydroid))
+#        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+#
+#        f, ax = plt.subplots(figsize=(6.5, 6.5))
+#        ax.set_title(str(hydroid))
+#        sns.despine(f, left=True, bottom=True)
+#        sns.scatterplot(x='comp_ManningN', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
+#        #sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
+#        sns.lineplot(x='comp_ManningN', y='Stage_1_5', data=plot_df, color='green', ax=ax)
+#        plt.fill_between(plot_df['comp_ManningN'], plot_df['Stage_1_5'],alpha=0.5)
+#        plt.text(plot_df['comp_ManningN'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+#        ax.legend()
+#        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
+#        plt.close()
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description="Vary the Manning's n values for in-channel vs. floodplain (recalculate Manning's eq for Discharge)")
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-bc','--channel-ratio-src-column',help='SRC attribute containing the channel vs. overbank geometry ratio (for composite calc)',required=False,type=str,default='chann_hradius_ratio')
+    parser.add_argument('-mann','--mann-n-table',help="Path to a csv file containing Manning's n values by featureid",required=True,type=str)
+    parser.add_argument('-suff','--hydrotable-suffix',help="Suffix to append to the new hydroTable csv file (e.g. '_global_06_011')",required=True,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
+    parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    channel_ratio_src_column = args['channel_ratio_src_column']
+    mann_n_table = args['mann_n_table']
+    hydrotable_suffix = args['hydrotable_suffix']
+    number_of_jobs = args['number_of_jobs']
+    src_plot_option = args['src_plot_option']
+    procs_list = []
+
+    print('Writing progress to log file here: ' + str(join(fim_dir,'log_composite_n' + hydrotable_suffix + '.log')))
+    print('This may take a few minutes...')
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+
+    ## Check that the bankfull flow filepath exists and read to dataframe
+    if not isfile(mann_n_table):
+        print('!!! Can not find the input roughness/feature_id file: ' + str(mann_n_table))
+    else:
+        ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
+        print('Importing the Manning roughness data file: ' + mann_n_table)
+        df_mann = pd.read_csv(mann_n_table,dtype={'feature_id': 'int64'})
+        if 'channel_n' not in df_mann.columns or 'overbank_n' not in df_mann.columns or 'feature_id' not in df_mann.columns:
+            print('Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann)
+        else:
+            print('Running the variable_mannings_calc function...')
+
+            # initiate log file
+            sys.__stdout__ = sys.stdout
+            log_file = open(join(fim_dir,'log_composite_n' + hydrotable_suffix + '.log'),"w")
+            sys.stdout = log_file
+            log_file.write('START TIME: ' + str(begin_time) + '\n')
+
+            ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
+            huc_list  = os.listdir(fim_dir)
+            skip_hucs_log = ""
+            for huc in huc_list:
+                if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+                    in_src_bankfull_filename = join(fim_dir,huc,'src_full_crosswalked_bankfull.csv')
+                    out_src_bankfull_filename = join(fim_dir,huc,'src_full_crosswalked_vmann.csv')
+                    htable_filename = join(fim_dir,huc,'hydroTable.csv')
+                    new_htable_filename = join(fim_dir,huc,'hydroTable' + hydrotable_suffix + '.csv')
+                    huc_plot_output_dir = join(fim_dir,huc,'src_plots')
+
+                    if isfile(in_src_bankfull_filename):
+                        print(str(huc))
+                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_bankfull_filename, htable_filename, new_htable_filename, src_plot_option, huc_plot_output_dir])
+                    else:
+                        print(str(huc) + ' --> can not find the src_full_crosswalked_bankfull.csv in the fim output dir: ' + str(join(fim_dir,huc)))
+
+            ## Initiate multiprocessing
+            print(f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
+            with Pool(processes=number_of_jobs) as pool:
+                pool.map(variable_mannings_calc, procs_list)
+
+            ## Record run time and close log file
+            end_time = dt.datetime.now()
+            log_file.write('END TIME: ' + str(end_time) + '\n')
+            tot_run_time = end_time - begin_time
+            log_file.write(str(tot_run_time))
+            sys.stdout = sys.__stdout__
+            log_file.close()

From 41fdf7be2870ca8de1ef5ec8825a0b959207f5cd Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Fri, 22 Oct 2021 10:20:09 -0500
Subject: [PATCH 115/359] adding tool that allows the codebase to be linted and
 standardized (#470)

---
 .isort.cfg                               |  3 +++
 tools/code_standardizer/Dockerfile       | 12 ++++++++++++
 tools/code_standardizer/entrypoint.sh    | 17 +++++++++++++++++
 tools/code_standardizer/requirements.txt |  3 +++
 4 files changed, 35 insertions(+)
 create mode 100755 .isort.cfg
 create mode 100755 tools/code_standardizer/Dockerfile
 create mode 100755 tools/code_standardizer/entrypoint.sh
 create mode 100755 tools/code_standardizer/requirements.txt

diff --git a/.isort.cfg b/.isort.cfg
new file mode 100755
index 000000000..aa7e7666d
--- /dev/null
+++ b/.isort.cfg
@@ -0,0 +1,3 @@
+[tool.isort]
+profile = "black"
+multi_line_output = 3
diff --git a/tools/code_standardizer/Dockerfile b/tools/code_standardizer/Dockerfile
new file mode 100755
index 000000000..0c58dd0a9
--- /dev/null
+++ b/tools/code_standardizer/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3.8.5-slim-buster
+
+ENV PYTHONUNBUFFERED 1
+RUN mkdir -p /opt/code_standardizer
+WORKDIR /opt/code_standardizer
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt --no-cache-dir
+COPY . /opt/code_standardizer
+
+RUN chmod +x /opt/code_standardizer/entrypoint.sh
+ENTRYPOINT ["/opt/code_standardizer/entrypoint.sh"]
diff --git a/tools/code_standardizer/entrypoint.sh b/tools/code_standardizer/entrypoint.sh
new file mode 100755
index 000000000..417a9a08d
--- /dev/null
+++ b/tools/code_standardizer/entrypoint.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+umask 002
+cd /cahaba
+
+echo "Starting Code Standardizer"
+
+echo "Running Python Black..."
+black .
+
+echo "Running iSort..."
+isort --profile black .
+
+echo "Running Flake8..."
+flake8 .
+
+echo " ALL DONE!"
diff --git a/tools/code_standardizer/requirements.txt b/tools/code_standardizer/requirements.txt
new file mode 100755
index 000000000..c8fd24694
--- /dev/null
+++ b/tools/code_standardizer/requirements.txt
@@ -0,0 +1,3 @@
+black==21.9b0
+flake8==3.9.2
+isort==5.9.3

From e00d322704b9d8498a524a1ea149674287ad2c94 Mon Sep 17 00:00:00 2001
From: Brian Avant <brian.avant@noaa.gov>
Date: Tue, 26 Oct 2021 12:19:52 -0500
Subject: [PATCH 116/359] Fixing reversed flow on MS River

See Changelog for 3.0.22.8 for more details.
---
 docs/CHANGELOG.md          | 9 +++++++++
 src/clip_vectors_to_wbd.py | 3 +++
 2 files changed, 12 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index fa84cd339..6721215b5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.22.8 - 2021-10-26 - [PR #471](https://github.com/NOAA-OWP/cahaba/pull/471)
+
+Manually filtering segments from stream input layer to fix flow reversal of the MS River (HUC 08030100).
+
+## Changes
+- `clip_vectors_to_wbd.py`: Fixes bug where flow direction is reversed for HUC 08030100. The issue is resolved by filtering incoming stream segments that intersect with the elevation grid boundary.
+
+<br/><br/>
+
 ## v3.0.22.7 - 2021-10-08 - [PR #467](https://github.com/NOAA-OWP/cahaba/pull/467)
 
 These "tool" enhancements 1) delineate in-channel vs. out-of-channel geometry to allow more targeted development of key physical drivers influencing the SRC calculations (e.g. bathymetry & Manning’s n) #418 and 2) applies a variable/composite Manning’s roughness (n) using user provided csv with in-channel vs. overbank roughness values #419 & #410.
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index c70c8766c..e5053627a 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -99,6 +99,9 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
         # special case: stream meanders in and out of WBD buffer boundary
         if str(hucCode) == '10030203':
             missing_segments = missing_segments + [23001300001840.0, 23001300016571.0]
+            
+        if str(hucCode) == '08030100':
+            missing_segments = missing_segments + [20000600011559.0, 20000600045761.0, 20000600002821.0]
 
         # Remove incoming stream segment so it won't be routed as outflow during hydroconditioning
         nhd_streams = nhd_streams.loc[~nhd_streams.FromNode.isin(missing_segments)]

From 8b6bc74cb81f3f3b54b4003a58230487d705103d Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Thu, 4 Nov 2021 07:04:14 -0500
Subject: [PATCH 117/359] Combine SRC modification modules in fim_run.sh
 post-processing step

See CHANGELOG.md, version 3.0.23.0 for details.

This resolves #466.
---
 config/params_calibrated.env                |  42 ---
 config/params_template.env                  |  21 +-
 docs/CHANGELOG.md                           |  26 ++
 fim_run.sh                                  |  19 ++
 src/add_crosswalk.py                        |   8 -
 src/bathy_rc_adjust.py                      | 158 ---------
 src/bathy_src_adjust_topwidth.py            | 339 ++++++++++++++++++++
 {tools => src}/identify_src_bankfull.py     | 106 +++---
 src/run_by_unit.sh                          |   9 +-
 {tools => src}/vary_mannings_n_composite.py | 160 +++++----
 tools/run_test_case.py                      |   8 +-
 11 files changed, 547 insertions(+), 349 deletions(-)
 delete mode 100644 config/params_calibrated.env
 delete mode 100755 src/bathy_rc_adjust.py
 create mode 100644 src/bathy_src_adjust_topwidth.py
 rename {tools => src}/identify_src_bankfull.py (70%)
 rename {tools => src}/vary_mannings_n_composite.py (58%)

diff --git a/config/params_calibrated.env b/config/params_calibrated.env
deleted file mode 100644
index f622949ec..000000000
--- a/config/params_calibrated.env
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-#### hydroconditioning parameters ####
-export negative_burn_value=1000
-export agree_DEM_buffer=70
-export wbd_buffer=5000
-export thalweg_lateral_elev_threshold=3
-
-#### geospatial parameters ####
-export max_split_distance_meters=1500
-export ms_buffer_dist=7000
-export lakes_buffer_dist_meters=20
-
-#### rating curve parameters ####
-export manning_n="/foss_fim/config/mannings_calibrated.json"
-export stage_min_meters=0
-export stage_interval_meters=0.3048
-export stage_max_meters=25
-export slope_min=0.001
-export min_catchment_area=0.25
-export min_stream_length=0.5
-
-#### bathy SRC estimation parameters ####
-export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
-#export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
-export bathy_src_modification=True # Toggle to turn BARC on/off (True=on; False=off)
-export surf_area_thalweg_ratio_flag=10 # Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
-export thalweg_stg_search_max_limit=3 # Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
-export bathy_xs_area_chg_flag=1 # Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
-export bankful_xs_area_ratio_flag=10 # Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
-export thalweg_hyd_radius_flag=10 # Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
-export ignore_streamorders=10 # Ignore BARC calculations for streamorders >= this value (10 is Mississippi R)
-
-#### computational parameters ####
-export ncores_gw=1 # mpi number of cores for gagewatershed
-export ncores_fd=1 # mpi number of cores for flow directions
-export default_max_jobs=1 # default number of max concurrent jobs to run
-export memfree=0G # min free memory required to start a new job or keep youngest job alive
-
-#### logging parameters ####
-export startDiv="\n##########################################################################\n"
-export stopDiv="\n##########################################################################"
diff --git a/config/params_template.env b/config/params_template.env
index 90a4ae305..41df712c6 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -21,9 +21,12 @@ export min_catchment_area=0.25
 export min_stream_length=0.5
 
 #### bathy SRC estimation parameters ####
-export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
-#export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
-export bathy_src_modification=True # Toggle to turn BARC on/off (True=on; False=off)
+export bathy_src_toggle=True # Toggle to run BARC routine (True=on; False=off)
+export bankfull_input_table="data/inputs/bathymetry/nwm_flow_bieger_qreg.csv" # input file location with feature_id and channel geometry attributes
+# Option 1: Bieger et al. 2015 discharge regression --> nwm_flow_bieger_qreg.csv
+# Option 2: Bieger et al. 2015 drainage area regression (obtained from Wieczorek (2018) database) --> BANKFULL_CONUS.txt
+# Option 3: NWM Route Link bankfull geometry (Blackburn-Lynch regression) --> nwm_route_link_geom_BED.csv
+export src_plot_option="False" # optional toggle to create SRC comparison plots for each hydroid (Warning: longer run times)
 export surf_area_thalweg_ratio_flag=10 # Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
 export thalweg_stg_search_max_limit=3 # Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
 export bathy_xs_area_chg_flag=1 # Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
@@ -31,6 +34,18 @@ export bankful_xs_area_ratio_flag=10 # Flag: Identify bogus BARC adjusted values
 export thalweg_hyd_radius_flag=10 # Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
 export ignore_streamorders=10 # Ignore BARC calculations for streamorders >= this value (10 is Mississippi R)
 
+#### estimating bankfull stage in SRCs ####
+export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
+export src_bankfull_plot_option="False" # optional toggle to create SRC comparison plots for each hydroid (Warning: longer run times)
+export bankfull_flows_file="data/inputs/rating_curve/bankfull_flows/nwm_v2_0_recurr_1_5_cms.csv" # input file location with nwm feature_id and recurrence flow values
+
+#### applying variable/composite roughness curve to SRCs ####
+export src_vrough_toggle="True" # Toggle to run composite roughness src routine (True=on; False=off)
+export src_vrough_plot_option="False" # optional toggle to create SRC comparison plots for each hydroid (Warning: longer run times)
+export vrough_suffix="_vmann" # text to append to output log and src_full_crosswalked file names
+export vmann_input_file="data/inputs/rating_curve/variable_roughness/mannings_global_06_011.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
+export bankfull_attribute="chann_volume_ratio" # src_full_crosswalked_bankfull.csv attribute (column id) containing the channel vs overbank ratio values (generated in the identify_src_bankfull.py)
+
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 6721215b5..e627a0d8c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,32 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.23.0 - 2021-10-31 - [PR #475](https://github.com/NOAA-OWP/cahaba/pull/475)
+
+Moved the synthetic rating curve (SRC) processes from the `\tools` directory to `\src` directory to support post-processing in `fim_run.sh`. These SRC post-processing modules will now run as part of the default `fim_run.sh` workflow. Reconfigured bathymetry adjusted rating curve (BARC) module to use the 1.5yr flow from NWM v2 recurrence flow data in combination with the Bieger et al. (2015) regression equations with bankfull discharge predictor variable input.
+
+## Additions
+- `src/bathy_src_adjust_topwidth.py` --> New version of bathymetry adjusted rating curve (BARC) module that is configured to use the Bieger et al. (2015) regression equation with input bankfull discharge as the predictor variable (previous version used the drainage area version of the regression equations). Also added log output capability, added reconfigured output content in `src_full_crosswalked_BARC.csv` and `hydroTable.csv`, and included modifications to allow BARC to run as a post-processing step in `fim_run.sh`. Reminder: BARC is only configured for MS extent.
+
+## Removals
+- `config/params_calibrated.env` --> deprecated the calibrated roughness values by stream order with the new introduction of variable/composite roughness module
+- `src/bathy_rc_adjust.py` --> deprecated the previous BARC version
+
+## Changes
+- `src/identify_src_bankfull.py` --> Moved this script from /tools to /src, added more doc strings, cleaned up output log, and reconfigured to allow execution from fim_run.sh post-processing.
+- `src/vary_mannings_n_composite.py` --> Moved this script from /tools to /src, added more doc strings, cleaned up output log, added/reconfigured output content in src_full_crosswalked_vmann.csv and hydroTable.csv, and reconfigured to allow execution from fim_run.sh post-processing.
+- `config/params_template.env` --> Added additional parameter/variables for input to `identify_src_bankfull.py`, `vary_mannings_n_composite.py`, and `bathy_src_adjust_topwidth.py`.
+      - default BARC input: bankfull channel geometry derived from the Bieger et al. (2015) bankfull discharge regression equations
+      - default bankfull flow input: NWM v2 1.5-year recurrence flows
+      - default variable roughness input: global (all NWM feature_ids) roughness values of 0.06 for in-channel and 0.11 for max overbank
+- `fim_run.sh` --> Added SRC post-processing calls after the `run_by_unit.sh` workflow
+- `src/add_crosswalk.py` --> Removed BARC module call (moved to post-processing)
+- `src/run_by_unit.sh` --> Removed old/unnecessary print statement.
+      - **Note: reset exit codes to 0 for unnecessary processing flags.** Non-zero error codes in `run_by_unit.sh` prevent the `fim_run.sh` post-processing steps from running. This error handling issue will be more appropriately handled in a soon to be release enhancement.
+- `tools/run_test_case.py` --> Reverted changes used during development process
+
+<br/><br/>
+
 ## v3.0.22.8 - 2021-10-26 - [PR #471](https://github.com/NOAA-OWP/cahaba/pull/471)
 
 Manually filtering segments from stream input layer to fix flow reversal of the MS River (HUC 08030100).
diff --git a/fim_run.sh b/fim_run.sh
index 2d1f8bd68..39565dc98 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -151,6 +151,25 @@ fi
 
 # identify missing HUCs
 # time python3 /foss_fim/tools/fim_completion_check.py -i $hucList -o $outputRunDataDir
+if [ "$extent" = "MS" ] && [ "$bathy_src_toggle" = "True" ]; then
+    # Run BARC routine
+    echo -e $startDiv"Performing Bathy Adjusted Rating Curve routine"$stopDiv
+    time python3 /foss_fim/src/bathy_src_adjust_topwidth.py -fim_dir $outputRunDataDir -bfull_geom $bankfull_input_table -j $jobLimit -plots $src_plot_option
+else
+    echo -e $startDiv"SKIPPING Bathy Adjusted Rating Curve routine"$stopDiv
+fi
+
+echo -e $startDiv"Estimating bankfull stage in SRCs"$stopDiv
+if [ "$src_bankfull_toggle" = "True" ]; then
+    # Run BARC routine
+    time python3 /foss_fim/src/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit -plots $src_bankfull_plot_option
+fi
+
+echo -e $startDiv"Applying variable roughness in SRCs"$stopDiv
+if [ "$src_vrough_toggle" = "True" ]; then
+    # Run BARC routine
+    time python3 /foss_fim/src/vary_mannings_n_composite.py -fim_dir $outputRunDataDir -mann $vmann_input_file -bc $bankfull_attribute -suff $vrough_suffix -j $jobLimit -plots $src_vrough_plot_option
+fi
 
 echo "$viz"
 if [[ "$viz" -eq 1 ]]; then
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 73bf3d4d1..82458a9c4 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -10,7 +10,6 @@
 import sys
 # sys.path.append('/foss_fim/src')
 # sys.path.append('/foss_fim/config')
-from bathy_rc_adjust import bathy_rc_lookup
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
 
@@ -24,7 +23,6 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     input_nwmflows = gpd.read_file(input_nwmflows_fileName)
     min_catchment_area = float(os.environ['min_catchment_area']) #0.25#
     min_stream_length = float(os.environ['min_stream_length']) #0.5#
-    bathy_src_calc = os.environ['bathy_src_modification'] == "True" # env variable to toggle on/off the bathy calc and src modifications
 
     if extent == 'FR':
         ## crosswalk using majority catchment method
@@ -221,12 +219,6 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_crosswalk = output_src[['HydroID','feature_id']]
     output_crosswalk = output_crosswalk.drop_duplicates(ignore_index=True)
 
-    ## bathy estimation integration in synthetic rating curve calculations
-    if (bathy_src_calc == True and extent == 'MS'):
-        output_src = bathy_rc_lookup(output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName)
-    else:
-        print('Note: NOT using bathy estimation approach to modify the SRC...')
-
     # make hydroTable
     output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Stage','Discharge (m3s-1)', 'HydraulicRadius (m)', 'WetArea (m2)', 'SLOPE', 'ManningN']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
deleted file mode 100755
index 7b942929e..000000000
--- a/src/bathy_rc_adjust.py
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/usr/bin/env python3
-
-from os import environ
-import pandas as pd
-
-sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag']) #10x --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
-thal_stg_limit = float(environ['thalweg_stg_search_max_limit']) #3m --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
-bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag']) #10x --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
-bathy_xsarea_flag = float(environ['bathy_xs_area_chg_flag']) #1x --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
-thal_hyd_radius_flag = float(environ['thalweg_hyd_radius_flag']) #10x --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
-ignore_streamorder = int(environ['ignore_streamorders']) #10 --> Do not perform BARC for streamorders >= provided value
-
-
-def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,):
-    ## Convert input_src_base featureid to integer
-    if input_src_base.feature_id.dtype != 'int': input_src_base.feature_id = input_src_base.feature_id.astype(int)
-
-    ## Read in the bankfull channel geometry text file
-    input_bathy = pd.read_csv(input_bathy_fileName, dtype= {'COMID': int})
-
-    ## Merge input_bathy and modified_src_base df using feature_id/COMID attributes
-    input_bathy = input_bathy.rename(columns={'COMID':'feature_id','BANKFULL_WIDTH':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA':'BANKFULL_XSEC_AREA (m2)'})
-    modified_src_base = input_src_base.merge(input_bathy.loc[:,['feature_id','BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA (m2)']],how='left',on='feature_id')
-
-    ## Check that the merge process returned matching feature_id entries
-    if modified_src_base['BANKFULL_WIDTH (m)'].count() == 0:
-        print('No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC!')
-        return(input_src_base)
-    else:
-        ## Use SurfaceArea variable to identify thalweg-restricted stage values for each hydroid
-        ## Calculate the interrow SurfaceArea ratio n/(n-1)
-        modified_src_base['SA_div_flag'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
-        ## Mask SA_div_flag when Stage = 0 or when the SA_div_flag value (n / n-1) is > threshold value (i.e. 10x)
-        modified_src_base['SA_div_flag'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div_flag']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
-        ## Create new df to filter and groupby HydroID
-        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div_flag']]
-        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['Stage']<thal_stg_limit] # assuming thalweg burn-in is less than 3 meters
-        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div_flag'].notnull()]
-        find_thalweg_notch = find_thalweg_notch.loc[find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()].reset_index(drop=True)
-        ## Assign thalweg_burn_elev variable to the stage value found in previous step
-        find_thalweg_notch['Thalweg_burn_elev'] = find_thalweg_notch['Stage']
-        ## Merge the Thalweg_burn_elev value back into the modified SRC --> this is used to mask the discharge after Manning's equation
-        modified_src_base = modified_src_base.merge(find_thalweg_notch.loc[:,['HydroID','Thalweg_burn_elev']],how='left',on='HydroID')
-
-        ## Calculate bankfull vs top width difference for each feature_id
-        modified_src_base['Top Width Diff (m)'] = (modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']).abs()
-        ## Calculate XS Area field (Channel Volume / Stream Length)
-        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (modified_src_base['LENGTHKM'] * 1000)
-
-        ## Groupby HydroID and find min of Top Width Diff (m)
-        output_bathy = modified_src_base[['feature_id','HydroID','order_','Stage','SurfaceArea (m2)','Thalweg_burn_elev','BANKFULL_WIDTH (m)','TopWidth (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','Top Width Diff (m)']]
-        ## filter out stage = 0 rows in SRC (assuming geom at stage 0 is not a valid channel geom)
-        output_bathy = output_bathy[output_bathy['Stage'] > 0]
-        ## filter SRC rows identified as Thalweg burned
-        output_bathy['Top Width Diff (m)'].mask(output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'],inplace=True)
-        ## ignore hydroid/featureid that did not have a valid Bankfull lookup (areas outside CONUS - i.e. Canada)
-        output_bathy = output_bathy[output_bathy['BANKFULL_XSEC_AREA (m2)'].notnull()]
-        ## ignore SRC entries with 0 surface area --> handles input SRC artifacts/errors in Great Lakes region
-        output_bathy = output_bathy[output_bathy['SurfaceArea (m2)'] > 0]
-        ## find index of minimum top width difference --> this will be used as the SRC "bankfull" row for future calcs
-        output_bathy = output_bathy.loc[output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()].reset_index(drop=True)
-        print('Average: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].mean()))
-        print('Minimum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].min()))
-        print('Maximum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].max()))
-        print('STD: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].std()) +'\n' + '#################')
-
-        ## Calculate XS Area difference between SRC and Bankfull database
-        output_bathy['XS Area Diff (m2)'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)'])
-        output_bathy['XS Bankfull Area Ratio'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']).round(2)
-        ## masking negative XS Area Diff and XS Area = 0
-        output_bathy['XS Bankfull Area Ratio'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
-        ## masking negative XS Area Diff and XS Area = 0
-        output_bathy['XS Area Diff (m2)'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
-        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
-        output_bathy['XS Area Diff (m2)'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
-        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
-        output_bathy['XS Bankfull Area Ratio'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
-        ## Print XS Area Diff statistics
-        print('Average: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].mean()))
-        print('Minimum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].min()))
-        print('Maximum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].max()))
-        print('STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std()))
-
-        ## Bin XS Bankfull Area Ratio by stream order
-        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']].copy()
-        ## mask stage values when XS Bankfull Area Ratio is null (need to filter to calculate the median for valid values below)
-        stream_order_bathy_ratio['Stage'].mask(stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(),inplace=True)
-        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(count=('XS Bankfull Area Ratio','count'),mean_xs_area_ratio=('XS Bankfull Area Ratio','mean'),median_stage_bankfull=('Stage','median'))
-        ## fill XS Bankfull Area Ratio and Stage values if no values were found in the grouby calcs
-        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill()+stream_order_bathy_ratio.bfill())/2
-        ## fill first and last stream order values if needed
-        stream_order_bathy_ratio = stream_order_bathy_ratio.bfill().ffill()
-        ## Get count_total tally of the total number of stream order hydroids in the HUC (not filtering anything out)
-        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage','count'))
-        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(stream_order_bathy_ratio_count,how='left',on='order_')
-        ## Fill any remaining null values: mean_xs_area_ratio --> 1 median_stage_bankfull --> 0
-        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(),1,inplace=True)
-        stream_order_bathy_ratio['median_stage_bankfull'].mask(stream_order_bathy_ratio['median_stage_bankfull'].isnull(),0,inplace=True)
-
-        ## Combine SRC df and df of XS Area for each hydroid and matching stage and order from bins above
-        output_bathy = output_bathy.merge(stream_order_bathy_ratio,how='left',on='order_')
-        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio,how='left',on='order_')
-
-        ## Calculate stage vs median_stage_bankfull difference for bankfull lookup
-        modified_src_base['lookup_stage_diff'] = (modified_src_base[['median_stage_bankfull','Thalweg_burn_elev']].max(axis=1) - modified_src_base['Stage']).abs()
-
-        ## If median_stage_bankfull is null then set lookup_stage_diff to 999 at stage 0 (handles errors for channels outside CONUS)
-        modified_src_base['lookup_stage_diff'].mask((modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),999,inplace=True)
-
-        ## Groupby HydroID again and find min of lookup_stage_diff
-        xs_area_hydroid_lookup = modified_src_base[['HydroID','BANKFULL_XSEC_AREA (m2)','XS Area (m2)','Stage','Thalweg_burn_elev','median_stage_bankfull','lookup_stage_diff','mean_xs_area_ratio']]
-        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()].reset_index(drop=True)
-
-        ## Calculate bathy adjusted XS Area ('XS Area (m2)' mutliplied by mean_xs_area_ratio)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']) - xs_area_hydroid_lookup['XS Area (m2)']
-
-        ## Calculate the ratio btw the lookup SRC XS_Area and the Bankfull_XSEC_AREA --> use this as a flag for potentially bad XS data
-        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'])
-        ## Set bath_cal_xs_area to 0 if the bankfull_XS_ratio_flag is > threshold --> 5x (assuming too large of difference to be a reliable bankfull calculation)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag,xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],inplace=True)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(),0,inplace=True)
-
-        ## Merge bathy_calc_xs_area to the modified_src_base
-        modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area']],how='left',on='HydroID')
-
-        ## Mask/null the bathy calculated area for streamorders that the user wants to ignore (set bathy_cals_xs_area = 0 for streamorder = 10)
-        modified_src_base['bathy_calc_xs_area'].mask(modified_src_base['order_'] >= ignore_streamorder,0.0,inplace=True)
-
-        ## Calculate new bathy adjusted channel geometry variables
-        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'orig_Discharge (m3s-1)','XS Area (m2)':'orig_XS Area (m2)','Volume (m3)':'orig_Volume (m3)','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'})
-        modified_src_base['XS Area (m2)'] = modified_src_base['orig_XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
-        modified_src_base['Volume (m3)'] = modified_src_base['XS Area (m2)'] * modified_src_base['LENGTHKM'] * 1000
-        modified_src_base['WetArea (m2)'] = modified_src_base['Volume (m3)']/modified_src_base['LENGTHKM']/1000
-        modified_src_base['HydraulicRadius (m)'] = modified_src_base['WetArea (m2)']/modified_src_base['WettedPerimeter (m)']
-        modified_src_base['HydraulicRadius (m)'].fillna(0, inplace=True)
-        ## mask out negative top width differences (avoid thalweg burn notch)
-        modified_src_base['HydraulicRadius (m)'].mask((modified_src_base['HydraulicRadius (m)']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
-
-        ## Calculate Q using Manning's equation
-        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)']* \
-        pow(modified_src_base['HydraulicRadius (m)'],2.0/3)* \
-        pow(modified_src_base['SLOPE'],0.5)/modified_src_base['ManningN']
-        ## mask discharge values for stage = 0 rows in SRC (replace with 0) --> do we need SRC to start at 0??
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0,0,inplace=True)
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'],0,inplace=True)
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'],-999,inplace=True)
-
-        ## Organize bathy calc output variables for csv
-        output_bathy = output_bathy[['HydroID','order_','Stage','SurfaceArea (m2)','TopWidth (m)','BANKFULL_WIDTH (m)','Top Width Diff (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','XS Area Diff (m2)','XS Bankfull Area Ratio','count','median_stage_bankfull','mean_xs_area_ratio']]
-
-        ## Export bathy/bankful calculation tables for easy viewing
-        output_bathy.to_csv(output_bathy_fileName,index=False)
-        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName,index=True)
-        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName,index=True)
-        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName,index=True)
-
-        print('Completed Bathy Calculations...')
-        return(modified_src_base)
diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py
new file mode 100644
index 000000000..31fba91fa
--- /dev/null
+++ b/src/bathy_src_adjust_topwidth.py
@@ -0,0 +1,339 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import pandas as pd
+import numpy as np
+import argparse
+import matplotlib.pyplot as plt
+import seaborn as sns
+from multiprocessing import Pool
+from os.path import isfile, join, dirname, isdir
+import datetime as dt
+import sys
+from os import environ
+sns.set_theme(style="whitegrid")
+# from utils.shared_functions import mem_profile
+
+"""
+    Estimate feature_id missing bathymetry in the raw channel geometry using input bankfull regression geometry
+
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders.
+    bankfull_geom : str
+        Input regression dataset w/ bankfull geometry by featureid (topwidth & xsec area)
+    number_of_jobs : int
+        Number of jobs.
+    src_plot_option : str
+        Optional (True or False): use this flag to crate src plots for all hydroids
+"""
+
+# sa_ratio_flag = 10
+# thal_stg_limit = 3
+# bankful_xs_ratio_flag = 10
+# bathy_xsarea_flag = 1
+# thal_hyd_radius_flag = 10
+# ignore_streamorder = 10
+sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag']) #10x --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
+thal_stg_limit = float(environ['thalweg_stg_search_max_limit']) #3m --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
+bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag']) #10x --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
+bathy_xsarea_flag = float(environ['bathy_xs_area_chg_flag']) #1x --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
+thal_hyd_radius_flag = float(environ['thalweg_hyd_radius_flag']) #10x --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
+ignore_streamorder = int(environ['ignore_streamorders']) #10 --> Do not perform BARC for streamorders >= provided value
+
+def bathy_rc_lookup(args):
+    input_src_fileName                  = args[0]
+    df_bfull_geom                       = args[1]
+    output_bathy_fileName               = args[2]
+    output_bathy_streamorder_fileName   = args[3]
+    output_bathy_thalweg_fileName       = args[4]
+    output_bathy_xs_lookup_fileName     = args[5]
+    input_htable_fileName               = args[6]
+    out_src_filename                    = args[7]
+    huc                                 = args[8]
+    src_plot_option                     = args[9]
+    huc_plot_output_dir                 = args[10]
+
+    log_text = 'Calculating: ' + str(huc) + '\n'
+
+    ## Read in the default src_full_crosswalked.csv
+    input_src_base = pd.read_csv(input_src_fileName, dtype= {'feature_id': int})
+
+    ## Rename input bankfull_geom data columns for consistant referencing
+    df_bfull_geom = df_bfull_geom.rename(columns={'COMID':'feature_id','BANKFULL_WIDTH':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA':'BANKFULL_XSEC_AREA (m2)'})
+    df_bfull_geom = df_bfull_geom.rename(columns={'BANKFULL_TOPWIDTH_q':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA_q':'BANKFULL_XSEC_AREA (m2)'})
+    ## Merge input_bathy and modified_src_base df using feature_id/COMID attributes
+    modified_src_base = input_src_base.merge(df_bfull_geom.loc[:,['feature_id','BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA (m2)']],how='left',on='feature_id')
+
+    ## Check that the merge process returned matching feature_id entries
+    if modified_src_base['BANKFULL_WIDTH (m)'].count() == 0:
+        log_text += 'WARNING: No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC for huc ' + str(huc) + '\n'
+    else:
+        ## Use SurfaceArea variable to identify thalweg-restricted stage values for each hydroid
+        ## Calculate the interrow SurfaceArea ratio n/(n-1)
+        modified_src_base['SA_div_flag'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
+        ## Mask SA_div_flag when Stage = 0 or when the SA_div_flag value (n / n-1) is > threshold value (i.e. 10x)
+        modified_src_base['SA_div_flag'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div_flag']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
+        ## Create new df to filter and groupby HydroID
+        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div_flag']]
+        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['Stage']<thal_stg_limit] # assuming thalweg burn-in is less than 3 meters
+        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div_flag'].notnull()]
+        find_thalweg_notch = find_thalweg_notch.loc[find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()].reset_index(drop=True)
+        ## Assign thalweg_burn_elev variable to the stage value found in previous step
+        find_thalweg_notch['Thalweg_burn_elev'] = find_thalweg_notch['Stage']
+        ## Merge the Thalweg_burn_elev value back into the modified SRC --> this is used to mask the discharge after Manning's equation
+        modified_src_base = modified_src_base.merge(find_thalweg_notch.loc[:,['HydroID','Thalweg_burn_elev']],how='left',on='HydroID')
+
+        ## Calculate bankfull vs top width difference for each feature_id
+        modified_src_base['Top Width Diff (m)'] = (modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']).abs()
+        ## Calculate XS Area field (Channel Volume / Stream Length)
+        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (modified_src_base['LENGTHKM'] * 1000)
+
+        ## Groupby HydroID and find min of Top Width Diff (m)
+        output_bathy = modified_src_base[['feature_id','HydroID','order_','Stage','SurfaceArea (m2)','Thalweg_burn_elev','BANKFULL_WIDTH (m)','TopWidth (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','Top Width Diff (m)']]
+        ## filter out stage = 0 rows in SRC (assuming geom at stage 0 is not a valid channel geom)
+        output_bathy = output_bathy[output_bathy['Stage'] > 0]
+        ## filter SRC rows identified as Thalweg burned
+        output_bathy['Top Width Diff (m)'].mask(output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'],inplace=True)
+        ## ignore hydroid/featureid that did not have a valid Bankfull lookup (areas outside CONUS - i.e. Canada)
+        output_bathy = output_bathy[output_bathy['BANKFULL_XSEC_AREA (m2)'].notnull()]
+        ## ignore SRC entries with 0 surface area --> handles input SRC artifacts/errors in Great Lakes region
+        output_bathy = output_bathy[output_bathy['SurfaceArea (m2)'] > 0]
+        ## find index of minimum top width difference --> this will be used as the SRC "bankfull" row for future calcs
+        output_bathy = output_bathy.loc[output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()].reset_index(drop=True)
+        log_text += ('Average: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].mean())) + '\n'
+        log_text += ('Minimum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].min())) + '\n'
+        log_text += ('Maximum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].max())) + '\n'
+        log_text += ('STD: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].std())) +'\n'
+
+        ## Calculate XS Area difference between SRC and Bankfull database
+        output_bathy['XS Area Diff (m2)'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)'])
+        output_bathy['XS Bankfull Area Ratio'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']).round(2)
+        ## masking negative XS Area Diff and XS Area = 0
+        output_bathy['XS Bankfull Area Ratio'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        ## masking negative XS Area Diff and XS Area = 0
+        output_bathy['XS Area Diff (m2)'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
+        output_bathy['XS Area Diff (m2)'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
+        output_bathy['XS Bankfull Area Ratio'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        ## Print XS Area Diff statistics
+        log_text += ('Average: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].mean())) + '\n'
+        log_text += ('Minimum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].min())) + '\n'
+        log_text += ('Maximum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].max())) + '\n'
+        log_text += ('STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std())) + '\n'
+
+        ## Bin XS Bankfull Area Ratio by stream order
+        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']].copy()
+        ## mask stage values when XS Bankfull Area Ratio is null (need to filter to calculate the median for valid values below)
+        stream_order_bathy_ratio['Stage'].mask(stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(),inplace=True)
+        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(count=('XS Bankfull Area Ratio','count'),mean_xs_area_ratio=('XS Bankfull Area Ratio','mean'),median_stage_bankfull=('Stage','median'))
+        ## fill XS Bankfull Area Ratio and Stage values if no values were found in the grouby calcs
+        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill()+stream_order_bathy_ratio.bfill())/2
+        ## fill first and last stream order values if needed
+        stream_order_bathy_ratio = stream_order_bathy_ratio.bfill().ffill()
+        ## Get count_total tally of the total number of stream order hydroids in the HUC (not filtering anything out)
+        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage','count'))
+        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(stream_order_bathy_ratio_count,how='left',on='order_')
+        ## Fill any remaining null values: mean_xs_area_ratio --> 1 median_stage_bankfull --> 0
+        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(),1,inplace=True)
+        stream_order_bathy_ratio['median_stage_bankfull'].mask(stream_order_bathy_ratio['median_stage_bankfull'].isnull(),0,inplace=True)
+
+        ## Combine SRC df and df of XS Area for each hydroid and matching stage and order from bins above
+        output_bathy = output_bathy.merge(stream_order_bathy_ratio,how='left',on='order_')
+        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio,how='left',on='order_')
+
+        ## Calculate stage vs median_stage_bankfull difference for bankfull lookup
+        modified_src_base['lookup_stage_diff'] = (modified_src_base[['median_stage_bankfull','Thalweg_burn_elev']].max(axis=1) - modified_src_base['Stage']).abs()
+
+        ## If median_stage_bankfull is null then set lookup_stage_diff to 999 at stage 0 (handles errors for channels outside CONUS)
+        modified_src_base['lookup_stage_diff'].mask((modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),999,inplace=True)
+
+        ## Groupby HydroID again and find min of lookup_stage_diff
+        xs_area_hydroid_lookup = modified_src_base[['HydroID','BANKFULL_XSEC_AREA (m2)','XS Area (m2)','Stage','Thalweg_burn_elev','median_stage_bankfull','lookup_stage_diff','mean_xs_area_ratio']]
+        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()].reset_index(drop=True)
+
+        ## Calculate bathy adjusted XS Area ('XS Area (m2)' mutliplied by mean_xs_area_ratio)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']) - xs_area_hydroid_lookup['XS Area (m2)']
+
+        ## Calculate the ratio btw the lookup SRC XS_Area and the Bankfull_XSEC_AREA --> use this as a flag for potentially bad XS data
+        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'])
+        ## Set bath_cal_xs_area to 0 if the bankfull_XS_ratio_flag is > threshold --> 5x (assuming too large of difference to be a reliable bankfull calculation)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag,xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],inplace=True)
+        xs_area_hydroid_lookup['barc_on'] = np.where(xs_area_hydroid_lookup['bathy_calc_xs_area'].isnull(), False, True) # field to identify where vmann is on/off
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(),0,inplace=True)
+
+        ## Merge bathy_calc_xs_area to the modified_src_base
+        modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area','barc_on']],how='left',on='HydroID')
+
+        ## Mask/null the bathy calculated area for streamorders that the user wants to ignore (set bathy_cals_xs_area = 0 for streamorder = 10)
+        modified_src_base['bathy_calc_xs_area'].mask(modified_src_base['order_'] >= ignore_streamorder,0.0,inplace=True)
+
+        ## Calculate new bathy adjusted channel geometry variables
+        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'orig_Discharge (m3s-1)','XS Area (m2)':'orig_XS Area (m2)','Volume (m3)':'orig_Volume (m3)','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'})
+        modified_src_base['XS Area (m2)'] = modified_src_base['orig_XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
+        modified_src_base['Volume (m3)'] = modified_src_base['XS Area (m2)'] * modified_src_base['LENGTHKM'] * 1000
+        modified_src_base['WetArea (m2)'] = modified_src_base['Volume (m3)']/modified_src_base['LENGTHKM']/1000
+        modified_src_base['HydraulicRadius (m)'] = modified_src_base['WetArea (m2)']/modified_src_base['WettedPerimeter (m)']
+        modified_src_base['HydraulicRadius (m)'].fillna(0, inplace=True)
+        ## mask out negative top width differences (avoid thalweg burn notch)
+        modified_src_base['HydraulicRadius (m)'].mask((modified_src_base['HydraulicRadius (m)']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
+
+        ## Calculate Q using Manning's equation
+        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)']* \
+        pow(modified_src_base['HydraulicRadius (m)'],2.0/3)* \
+        pow(modified_src_base['SLOPE'],0.5)/modified_src_base['ManningN']
+        ## mask discharge values for stage = 0 rows in SRC (replace with 0) --> do we need SRC to start at 0??
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0,0,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'],0,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'],-999,inplace=True)
+
+        ## Organize bathy calc output variables for csv
+        output_bathy = output_bathy[['HydroID','order_','Stage','SurfaceArea (m2)','TopWidth (m)','BANKFULL_WIDTH (m)','Top Width Diff (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','XS Area Diff (m2)','XS Bankfull Area Ratio','count','median_stage_bankfull','mean_xs_area_ratio']]
+
+        ## Export bathy/bankful calculation tables for easy viewing
+        output_bathy.to_csv(output_bathy_fileName,index=False)
+        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName,index=True)
+        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName,index=True)
+        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName,index=True)
+
+        ## Output new src_full_crosswalked
+        modified_src_base.to_csv(out_src_filename,index=False)
+        ## Update the hydroTable
+        modified_hydro_table = modified_src_base.loc[:,['HydroID','Stage','barc_on','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)']]
+        modified_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
+        df_htable = pd.read_csv(input_htable_fileName,dtype={'HUC': str})
+        if not set(['orig_discharge_cms','orig_WetArea (m2)','orig_HydraulicRadius (m)']).issubset(df_htable.columns):
+            df_htable.rename(columns={'discharge_cms':'orig_discharge_cms','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'},inplace=True)
+        else:
+            df_htable.drop(['discharge_cms','WetArea (m2)','HydraulicRadius (m)'], axis=1, inplace=True) # drop the previously modified columns - to be replaced with updated version
+        df_htable = df_htable.merge(modified_hydro_table, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
+        df_htable.to_csv(input_htable_fileName,index=False)
+        log_text += ('Output new hydroTable and src_full_crosswalked: ') + '\n'
+        log_text += ('Completed Bathy Calculations: ') + str(huc) +'\n#################\n'
+
+        ## plot rating curves (optional arg)
+        if src_plot_option == 'True':
+            if isdir(huc_plot_output_dir) == False:
+                os.mkdir(huc_plot_output_dir)
+            generate_src_plot(df_htable, huc_plot_output_dir)
+
+        return(log_text)
+
+
+def generate_src_plot(df_src, plt_out_dir):
+
+    ## create list of unique hydroids
+    hydroids = df_src.HydroID.unique().tolist()
+
+    ## plot each hydroid SRC in the huc
+    for hydroid in hydroids:
+        print("Creating SRC plot: " + str(hydroid))
+        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+
+        f, ax = plt.subplots(figsize=(6.5, 6.5))
+        ax.set_title(str(hydroid))
+        sns.despine(f, left=True, bottom=True)
+        sns.scatterplot(x='orig_discharge_cms', y='stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
+        sns.scatterplot(x='discharge_cms', y='stage', data=plot_df, label="SRC w/ BARC", ax=ax, color='orange')
+        #sns.lineplot(x='discharge_1_5', y='Stage_1_5', data=plot_df, color='green', ax=ax)
+        #plt.fill_between(plot_df['discharge_1_5'], plot_df['Stage_1_5'],alpha=0.5)
+        #plt.text(plot_df['discharge_1_5'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+        ax.legend()
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_barc.png',dpi=175, bbox_inches='tight')
+        plt.close()
+
+def multi_process(bathy_rc_lookup, procs_list):
+    print(f"Applying bathy adjustment calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    with Pool(processes=number_of_jobs) as pool:
+        map_output = pool.map(bathy_rc_lookup, procs_list)
+        #log_file.write(str(map_output))
+    log_file.writelines(["%s\n" % item  for item in map_output])
+
+if __name__ == '__main__':
+    #output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName
+    parser = argparse.ArgumentParser(description="Estimate the unaccounted for channel bathymetry using a regression-based estimate of channel XSec Area")
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-bfull_geom','--bankfull-xsec-input',help='Regression dataset w/ bankfull geometry by featureid (topwidth & xsec area)',required=True,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
+    parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    bankfull_regres_filepath = args['bankfull_xsec_input']
+    number_of_jobs = args['number_of_jobs']
+    src_plot_option = args['src_plot_option']
+    procs_list = []
+
+    print('STARTING Bathy Adjusted Rating Curve routine...')
+    ## Check that the input bankfull geom filepath exists and then read it to dataframe
+    if not isfile(bankfull_regres_filepath):
+        print('!!! Can not find the input bankfull geometry regression file: ' + str(bankfull_regres_filepath))
+    else:
+        ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
+        print('Importing the bankfull regression data file: ' + bankfull_regres_filepath)
+        df_bfull_geom = pd.read_csv(bankfull_regres_filepath,dtype= {'COMID': int})
+        if 'COMID' not in df_bfull_geom.columns and 'feature_id' not in df_bfull_geom.columns:
+            print('Missing required data column ("feature_id" or "COMID")!!! --> ' + df_bfull_geom)
+        else:
+            print('Running Bathy Adjust Rating Curve (BARC)...')
+
+            ## Print message to user and initiate run clock
+            print('Writing progress to log file here: ' + str(join(fim_dir,'log_BARC.log')))
+            print('This may take a few minutes...')
+            ## Create a time var to log run time
+            begin_time = dt.datetime.now()
+
+            ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
+            huc_list  = os.listdir(fim_dir)
+            huc_pass_list = []
+            for huc in huc_list:
+                if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+                    #output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName
+                    in_src_filename = join(fim_dir,huc,'src_full_crosswalked.csv')
+                    out_src_filename = join(fim_dir,huc,'src_full_crosswalked_BARC.csv')
+                    htable_filename = join(fim_dir,huc,'hydroTable.csv')
+                    output_bath_filename = join(fim_dir,huc,'bathy_crosswalk_calcs.csv')
+                    output_bathy_thalweg_fileName = join(fim_dir,huc,'bathy_thalweg_flag.csv')
+                    output_bathy_streamorder_fileName = join(fim_dir,huc,'bathy_stream_order_calcs.csv')
+                    output_bathy_thalweg_fileName = join(fim_dir,huc,'bathy_thalweg_flag.csv')
+                    output_bathy_xs_lookup_fileName = join(fim_dir,huc,'bathy_xs_area_hydroid_lookup.csv')
+                    huc_plot_output_dir = join(fim_dir,huc,'src_plots')
+
+                    if isfile(in_src_filename):
+                        print(str(huc))
+                        huc_pass_list.append(str(huc))
+                        procs_list.append([in_src_filename,df_bfull_geom,output_bath_filename,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,htable_filename,out_src_filename,huc,src_plot_option,huc_plot_output_dir])
+                    else:
+                        print(str(huc) + ' --> can not find the src_full_crosswalked.csv in the fim output dir: ' + str(join(fim_dir,huc)))
+
+            ## initiate log file
+            print(f"Applying bathy adjustment calcs for {len(procs_list)} hucs using {number_of_jobs} jobs...")
+            sys.__stdout__ = sys.stdout
+            log_file = open(join(fim_dir,'logs','log_barc.log'),"w")
+            sys.stdout = log_file
+            log_file.write('START TIME: ' + str(begin_time) + '\n')
+            log_file.writelines(["%s\n" % item  for item in huc_pass_list])
+
+            ## Write env variables to log files
+            log_file.write('sa_ratio_flag = ' + str(sa_ratio_flag) + ' --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)' + '\n')
+            log_file.write('thal_stg_limit = ' + str(thal_stg_limit) + ' --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)' + '\n')
+            log_file.write('bankful_xs_ratio_flag = ' + str(bankful_xs_ratio_flag) + ' --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)' + '\n')
+            log_file.write('bathy_xsarea_flag = ' + str(bathy_xsarea_flag) + ' --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold' + '\n')
+            log_file.write('thal_hyd_radius_flag = ' + str(thal_hyd_radius_flag) + ' --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0' + '\n')
+            log_file.write('ignore_streamorder = ' + str(ignore_streamorder) + ' --> Do not perform BARC for streamorders >= provided value' + '\n')
+            log_file.write('#########################################################\n\n')
+
+            ## Pass huc procs_list to multiprocessing function
+            multi_process(bathy_rc_lookup, procs_list)
+
+            ## Record run time and close log file
+            end_time = dt.datetime.now()
+            log_file.write('END TIME: ' + str(end_time) + '\n')
+            tot_run_time = end_time - begin_time
+            log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+            sys.stdout = sys.__stdout__
+            log_file.close()
diff --git a/tools/identify_src_bankfull.py b/src/identify_src_bankfull.py
similarity index 70%
rename from tools/identify_src_bankfull.py
rename to src/identify_src_bankfull.py
index b011bc31e..3be19fae1 100755
--- a/tools/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -23,19 +23,19 @@
     ----------
     fim_dir : str
         Directory containing FIM output folders.
-    nwm_flow_dir : str
-        Directory containing NWM recurrence flows files.
+    bankfull_flow_dir : str
+        Directory containing "bankfull" flows files (e.g. NWM 1.5yr recurr).
     number_of_jobs : str
         Number of jobs.
     plots : str
         Flag to create SRC plots for all hydroids (True/False)
 """
 
-def nwm_1_5_bankfull_lookup(args):
+def src_bankfull_lookup(args):
 
     src_full_filename           = args[0]
     src_modify_filename         = args[1]
-    df_nwm1_5                   = args[2]
+    df_bflows                   = args[2]
     huc                         = args[3]
     src_plot_option             = args[4]
     huc_output_dir              = args[5]
@@ -46,10 +46,10 @@ def nwm_1_5_bankfull_lookup(args):
     df_src = pd.read_csv(src_full_filename,dtype={'HydroID': int,'feature_id': int})
 
     ## NWM recurr rename discharge var
-    df_nwm1_5 = df_nwm1_5.rename(columns={'discharge':'discharge_1_5'})
+    df_bflows = df_bflows.rename(columns={'discharge':'discharge_1_5'})
 
     ## Combine the nwm 1.5yr flows into the SRC via feature_id
-    df_src = df_src.merge(df_nwm1_5,how='left',on='feature_id')
+    df_src = df_src.merge(df_bflows,how='left',on='feature_id')
 
     ## Check if there are any missing data, negative or zero flow values in the discharge_1_5
     check_null = df_src['discharge_1_5'].isnull().sum()
@@ -61,16 +61,9 @@ def nwm_1_5_bankfull_lookup(args):
     if negative_flows > 0:
         log_text += 'HUC: ' + str(huc) + ' --> Negative or zero flow values found (likely lakeid loc)\n'
 
-    ## Check which channel geometry parameters exist in df (use bathy adjusted vars by default) --> this is needed to handle differences btw BARC & no-BARC outputs
-    if 'HydraulicRadius (m)_bathy_adj' in df_src:
-        hradius_var = 'HydraulicRadius (m)_bathy_adj'
-    else:
-        hradius_var = 'HydraulicRadius (m)'
-
-    if 'Volume (m3)_bathy_adj' in df_src:
-        volume_var = 'Volume (m3)_bathy_adj'
-    else:
-        volume_var = 'Volume (m3)'
+    ## Define the channel geometry variable names to use from the src
+    hradius_var = 'HydraulicRadius (m)'
+    volume_var = 'Volume (m3)'
 
     ## Locate the closest SRC discharge value to the NWM 1.5yr flow
     df_src['Q_1_5_find'] = (df_src['discharge_1_5'] - df_src['Discharge (m3s-1)']).abs()
@@ -117,8 +110,7 @@ def nwm_1_5_bankfull_lookup(args):
 
     ## Output new SRC with bankfull column
     df_src.to_csv(src_modify_filename,index=False)
-    log_text += 'Completed: ' + str(huc) + '\n'
-    log_file.write(log_text)
+    log_text += 'Completed: ' + str(huc)
 
     ## plot rating curves (optional arg)
     if src_plot_option == 'True':
@@ -126,6 +118,7 @@ def nwm_1_5_bankfull_lookup(args):
             os.mkdir(huc_output_dir)
         generate_src_plot(df_src, huc_output_dir)
 
+    return(log_text)
 
 def generate_src_plot(df_src, plt_out_dir):
 
@@ -137,19 +130,28 @@ def generate_src_plot(df_src, plt_out_dir):
         print("Creating SRC plot: " + str(hydroid))
         plot_df = df_src.loc[df_src['HydroID'] == hydroid]
 
-        f, ax = plt.subplots(figsize=(6.5, 6.5))
-        ax.set_title(str(hydroid))
-        sns.despine(f, left=True, bottom=True)
-        #sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, ax=ax)
-        #sns.lineplot(x='Discharge (m3s-1)', y='Stage_1_5', data=plot_df, color='green', ax=ax)
-        #plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_1_5'],alpha=0.5)
-        #plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
-        sns.scatterplot(x='chann_volume_ratio', y='Stage', data=plot_df, ax=ax, label="chann_volume_ratio", s=38)
-        sns.scatterplot(x='chann_hradius_ratio', y='Stage', data=plot_df, ax=ax, label="chann_hradius_ratio", s=12)
-        ax.legend()
-        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
+        fig, axes = plt.subplots(1,2,figsize=(12, 6))
+        fig.suptitle(str(hydroid))
+        axes[0].set_title('Rating Curve w/ Bankfull')
+        axes[1].set_title('Channel Volume vs. HRadius Ratio')
+        sns.despine(fig, left=True, bottom=True)
+        sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, ax=axes[0])
+        sns.lineplot(x='Discharge (m3s-1)', y='Stage_1_5', data=plot_df, color='green', ax=axes[0])
+        axes[0].fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_1_5'],alpha=0.5)
+        axes[0].text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+        sns.scatterplot(x='chann_volume_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_volume_ratio", s=38)
+        sns.scatterplot(x='chann_hradius_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_hradius_ratio", s=12)
+        axes[1].legend()
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_bankfull.png',dpi=100, bbox_inches='tight')
         plt.close()
 
+def multi_process(src_bankfull_lookup, procs_list):
+    ## Initiate multiprocessing
+    print(f"Identifying bankfull stage for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    with Pool(processes=number_of_jobs) as pool:
+        map_output = pool.map(src_bankfull_lookup, procs_list)
+    log_file.writelines(["%s\n" % item  for item in map_output])
+
 
 if __name__ == '__main__':
 
@@ -175,37 +177,45 @@ def generate_src_plot(df_src, plt_out_dir):
 
     ## Check that the bankfull flow filepath exists and read to dataframe
     if not isfile(bankfull_flow_filepath):
-        print('!!! Can not find the input recurr flow file: ' + str(bankfull_flow_filepath))
+        print('!!!ERROR: Can not find the input bankfull flow file: ' + str(bankfull_flow_filepath))
     else:
-        df_nwm1_5 = pd.read_csv(bankfull_flow_filepath,dtype={'feature_id': int})
-
-        ## initiate log file
-        sys.__stdout__ = sys.stdout
-        log_file = open(join(fim_dir,'log_bankfull_indentify.log'),"w")
-        sys.stdout = log_file
-        log_file.write('START TIME: ' + str(begin_time) + '\n')
-
+        df_bflows = pd.read_csv(bankfull_flow_filepath,dtype={'feature_id': int})
         huc_list  = os.listdir(fim_dir)
+        huc_pass_list = []
         for huc in huc_list:
             if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
-                src_full_filename = join(fim_dir,huc,'src_full_crosswalked.csv')
+                src_barc_full_filename = join(fim_dir,huc,'src_full_crosswalked_BARC.csv')
+                src_orig_full_filename = join(fim_dir,huc,'src_full_crosswalked.csv')
                 src_modify_filename = join(fim_dir,huc,'src_full_crosswalked_bankfull.csv')
                 huc_output_dir = join(fim_dir,huc,'src_plots')
-
-                if isfile(src_full_filename):
-                    procs_list.append([src_full_filename, src_modify_filename, df_nwm1_5, huc, src_plot_option, huc_output_dir])
+                ## check if BARC modified src_full_crosswalked_BARC.csv exists otherwise use the orginial src_full_crosswalked.csv
+                if isfile(src_barc_full_filename):
+                    print(str(huc))
+                    huc_pass_list.append(str(huc) + " --> src_full_crosswalked_BARC.csv")
+                    procs_list.append([src_barc_full_filename, src_modify_filename, df_bflows, huc, src_plot_option, huc_output_dir])
+                elif isfile(src_orig_full_filename):
+                    print(str(huc))
+                    huc_pass_list.append(str(huc) + " --> src_full_crosswalked.csv")
+                    procs_list.append([src_orig_full_filename, src_modify_filename, df_bflows, huc, src_plot_option, huc_output_dir])
                 else:
-                    log_file.write(str(huc) + ' --> can not find the src_full_crosswalked.csv in the fim output dir: ' + str(join(fim_dir,huc)) + '\n')
+                    print(str(huc) + 'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: ' + str(join(fim_dir,huc)) + ' - skipping this HUC!!!\n')
+
+        ## initiate log file
+        print(f"Identifying bankfull stage for {len(procs_list)} hucs using {number_of_jobs} jobs")
+        sys.__stdout__ = sys.stdout
+        log_file = open(join(fim_dir,'logs','log_bankfull_indentify.log'),"w")
+        sys.stdout = log_file
+        log_file.write('START TIME: ' + str(begin_time) + '\n')
+        log_file.writelines(["%s\n" % item  for item in huc_pass_list])
+        log_file.write('#########################################################\n\n')
 
-        ## Initiate multiprocessing
-        print(f"Identifying bankfull thresholds for {len(procs_list)} hucs using {number_of_jobs} jobs")
-        with Pool(processes=number_of_jobs) as pool:
-            pool.map(nwm_1_5_bankfull_lookup, procs_list)
+        ## Pass huc procs_list to multiprocessing function
+        multi_process(src_bankfull_lookup, procs_list)
 
         ## Record run time and close log file
         end_time = dt.datetime.now()
         log_file.write('END TIME: ' + str(end_time) + '\n')
         tot_run_time = end_time - begin_time
-        log_file.write(str(tot_run_time))
+        log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
         sys.stdout = sys.__stdout__
         log_file.close()
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 07771ad71..ec456cefb 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -64,7 +64,7 @@ if [ "$extent" = "MS" ]; then
   if [[ ! -f $outputHucDataDir/nhd_headwater_points_subset.gpkg ]] ; then
     echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
     rm -rf $outputHucDataDir
-    exit 55
+    exit 0
   fi
 fi
 
@@ -235,7 +235,7 @@ Tcount
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
   echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
   rm -rf $outputHucDataDir
-  exit 56
+  exit 0
 fi
 
 if [ "$extent" = "MS" ]; then
@@ -249,7 +249,7 @@ if [ "$extent" = "MS" ]; then
   if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
     echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
     rm -rf $outputHucDataDir
-    exit 57
+    exit 0
   fi
 
   dem_thalwegCond=$outputHucDataDir/dem_thalwegCond_MS.tif
@@ -329,7 +329,7 @@ python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $o
 if [[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ]] ; then
   echo "No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
   rm -rf $outputHucDataDir
-  exit 65
+  exit 0
 fi
 Tcount
 
@@ -400,7 +400,6 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv outp
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-echo python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
diff --git a/tools/vary_mannings_n_composite.py b/src/vary_mannings_n_composite.py
similarity index 58%
rename from tools/vary_mannings_n_composite.py
rename to src/vary_mannings_n_composite.py
index 4d48ff69e..1e563e0f1 100755
--- a/tools/vary_mannings_n_composite.py
+++ b/src/vary_mannings_n_composite.py
@@ -3,6 +3,7 @@
 import os
 import sys
 import pandas as pd
+import numpy as np
 import argparse
 import matplotlib.pyplot as plt
 import seaborn as sns
@@ -27,8 +28,8 @@
         SRC attribute containing the channel vs. floodplain attribute
     mann_n_table : str
         Path to a csv file containing Manning's n values by feature_id
-    hydrotable_suffix : str
-        Suffix to append to the new hydroTable csv file
+    file_suffix : str
+        Suffix to append to the output log file
     number_of_jobs : str
         Number of jobs.
     src_plot_option : str
@@ -43,9 +44,8 @@ def variable_mannings_calc(args):
     huc                         = args[3]
     out_src_bankfull_filename   = args[4]
     htable_filename             = args[5]
-    new_htable_filename         = args[6]
-    src_plot_option             = args[7]
-    huc_output_dir              = args[8]
+    src_plot_option             = args[6]
+    huc_output_dir              = args[7]
 
     ## Read the src_full_crosswalked.csv
     log_text = 'Calculating: ' + str(huc) + '\n'
@@ -53,70 +53,63 @@ def variable_mannings_calc(args):
 
     ## Check that the channel ratio column the user specified exists in the def
     if channel_ratio_src_column not in df_src.columns:
-        log_text += str(huc) + ' --> ' + out_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
-
-    ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the channel_ratio_src_column)
-    df_src = df_src.merge(df_mann,  how='left', on='feature_id')
-    check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
-    if check_null > 0:
-        log_text += str(huc) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
-
-    ## Calculate composite Manning's n using the channel geometry ratio attribute given by user (e.g. chann_hradius_ratio or chann_vol_ratio)
-    df_src['comp_ManningN'] = (df_src[channel_ratio_src_column]*df_src['channel_n']) + ((1.0 - df_src[channel_ratio_src_column])*df_src['overbank_n'])
-    #print('Done calculating composite Manning n (' + channel_ratio_src_column + '): ' + str(huc))
-
-    ## Check if there are any missing data in the composite ManningN column
-    check_null_comp = df_src['comp_ManningN'].isnull().sum()
-    if check_null_comp > 0:
-        log_text += str(huc) + ' --> ' + 'Missing values in the comp_ManningN calculation' + ' --> missing entries= ' + str(check_null_comp/84)  + '\n'
-
-    ## Check which channel geometry parameters exist in the src_df (use bathy adjusted vars by default) --> this is needed to handle differences btw BARC & no-BARC outputs
-    if 'HydraulicRadius (m)_bathy_adj' in df_src:
-        hydr_radius = 'HydraulicRadius (m)_bathy_adj'
+        log_text += 'WARNING --> ' + str(huc) + out_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
     else:
+        ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the channel_ratio_src_column)
+        df_src = df_src.merge(df_mann,  how='left', on='feature_id')
+        check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
+        if check_null > 0:
+            log_text += str(huc) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
+
+        ## Calculate composite Manning's n using the channel geometry ratio attribute given by user (e.g. chann_hradius_ratio or chann_vol_ratio)
+        df_src['comp_ManningN'] = (df_src[channel_ratio_src_column]*df_src['channel_n']) + ((1.0 - df_src[channel_ratio_src_column])*df_src['overbank_n'])
+        #print('Done calculating composite Manning n (' + channel_ratio_src_column + '): ' + str(huc))
+
+        ## Check if there are any missing data in the composite ManningN column
+        check_null_comp = df_src['comp_ManningN'].isnull().sum()
+        if check_null_comp > 0:
+            log_text += str(huc) + ' --> ' + 'Missing values in the comp_ManningN calculation' + ' --> missing entries= ' + str(check_null_comp/84)  + '\n'
+        df_src['vmann_on'] = np.where(df_src['comp_ManningN'].isnull(), False, True) # field to identify where vmann is on/off
+
+        ## Define the channel geometry variable names to use from the src
         hydr_radius = 'HydraulicRadius (m)'
-
-    if 'WetArea (m2)_bathy_adj' in df_src:
-        wet_area = 'WetArea (m2)_bathy_adj'
-    else:
         wet_area = 'WetArea (m2)'
 
-    ## Calculate Q using Manning's equation
-    #df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True) # rename the previous Discharge column
-    df_src['Discharge (m3s-1)_varMann'] = df_src[wet_area]* \
-    pow(df_src[hydr_radius],2.0/3)* \
-    pow(df_src['SLOPE'],0.5)/df_src['comp_ManningN']
-
-    ## Set Q values to 0 and -999 for specified criteria
-    df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == 0,0,inplace=True)
-    if 'Thalweg_burn_elev' in df_src:
-        df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == df_src['Thalweg_burn_elev'],0,inplace=True)
-        df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] < df_src['Thalweg_burn_elev'],-999,inplace=True)
-
-    ## Output new SRC with bankfull column
-    df_src.to_csv(out_src_bankfull_filename,index=False)
-
-    ## Output new hydroTable with updated discharge and ManningN column
-    df_src_trim = df_src[['HydroID','Stage','Discharge (m3s-1)_varMann','comp_ManningN']]
-    df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_varMann': 'discharge_cms','comp_ManningN':'ManningN'})
-    df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
-    df_htable.drop(['discharge_cms','ManningN'], axis=1, inplace=True) # drop the original discharge column to be replaced with updated version
-    df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
-    #extra_cols = ['HydraulicRadius (m)','WetArea (m2)','SLOPE','ManningN']
-    #if df_htable.columns.isin(cols).all()):
-    #    df_htable = df_htable[['HydroID','feature_id','stage','discharge_cms','HydraulicRadius (m)','WetArea (m2)','SLOPE','ManningN','HUC','LakeID']]
-    #else:
-    #    df_htable = df_htable[['HydroID','feature_id','stage','discharge_cms','HUC','LakeID']] # set column order for hydroTable output
-    df_htable.to_csv(new_htable_filename,index=False)
-    log_text += 'Completed: ' + str(huc) + '\n'
-    log_file.write(str(log_text))
-
-    ## plot rating curves
-    if src_plot_option == 'True':
-        if isdir(huc_output_dir) == False:
-            os.mkdir(huc_output_dir)
-        generate_src_plot(df_src, huc_output_dir)
+        ## Calculate Q using Manning's equation
+        #df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True) # rename the previous Discharge column
+        df_src['Discharge (m3s-1)_varMann'] = df_src[wet_area]* \
+        pow(df_src[hydr_radius],2.0/3)* \
+        pow(df_src['SLOPE'],0.5)/df_src['comp_ManningN']
+
+        ## Set Q values to 0 and -999 for specified criteria
+        df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == 0,0,inplace=True)
+        if 'Thalweg_burn_elev' in df_src:
+            df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == df_src['Thalweg_burn_elev'],0,inplace=True)
+            df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] < df_src['Thalweg_burn_elev'],-999,inplace=True)
+
+        ## Output new SRC with bankfull column
+        df_src.to_csv(out_src_bankfull_filename,index=False)
+
+        ## Output new hydroTable with updated discharge and ManningN column
+        df_src_trim = df_src[['HydroID','Stage','vmann_on','Discharge (m3s-1)_varMann','comp_ManningN']]
+        df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_varMann': 'discharge_cms','comp_ManningN':'ManningN'})
+        df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
+        df_htable.rename(columns={'ManningN':'orig_ManningN'},inplace=True)
+        if not set(['orig_discharge_cms']).issubset(df_htable.columns):
+            df_htable.rename(columns={'discharge_cms':'orig_discharge_cms'},inplace=True)
+        else:
+            df_htable.drop(['discharge_cms'], axis=1, inplace=True) # drop the previously modified discharge column to be replaced with updated version
+        df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
+        df_htable.to_csv(htable_filename,index=False)
+        log_text += 'Completed: ' + str(huc)
+
+        ## plot rating curves
+        if src_plot_option == 'True':
+            if isdir(huc_output_dir) == False:
+                os.mkdir(huc_output_dir)
+            generate_src_plot(df_src, huc_output_dir)
 
+    return(log_text)
 
 def generate_src_plot(df_src, plt_out_dir):
 
@@ -137,7 +130,7 @@ def generate_src_plot(df_src, plt_out_dir):
         plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_1_5'],alpha=0.5)
         plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
         ax.legend()
-        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png',dpi=175, bbox_inches='tight')
         plt.close()
 
 #    for hydroid in hydroids:
@@ -156,6 +149,12 @@ def generate_src_plot(df_src, plt_out_dir):
 #        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
 #        plt.close()
 
+def multi_process(variable_mannings_calc, procs_list):
+    ## Initiate multiprocessing
+    print(f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    with Pool(processes=number_of_jobs) as pool:
+        map_output = pool.map(variable_mannings_calc, procs_list)
+    log_file.writelines(["%s\n" % item  for item in map_output])
 
 if __name__ == '__main__':
 
@@ -163,7 +162,7 @@ def generate_src_plot(df_src, plt_out_dir):
     parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
     parser.add_argument('-bc','--channel-ratio-src-column',help='SRC attribute containing the channel vs. overbank geometry ratio (for composite calc)',required=False,type=str,default='chann_hradius_ratio')
     parser.add_argument('-mann','--mann-n-table',help="Path to a csv file containing Manning's n values by featureid",required=True,type=str)
-    parser.add_argument('-suff','--hydrotable-suffix',help="Suffix to append to the new hydroTable csv file (e.g. '_global_06_011')",required=True,type=str)
+    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",required=True,type=str)
     parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
     parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
 
@@ -172,12 +171,12 @@ def generate_src_plot(df_src, plt_out_dir):
     fim_dir = args['fim_dir']
     channel_ratio_src_column = args['channel_ratio_src_column']
     mann_n_table = args['mann_n_table']
-    hydrotable_suffix = args['hydrotable_suffix']
+    output_suffix = args['output_suffix']
     number_of_jobs = args['number_of_jobs']
     src_plot_option = args['src_plot_option']
     procs_list = []
 
-    print('Writing progress to log file here: ' + str(join(fim_dir,'log_composite_n' + hydrotable_suffix + '.log')))
+    print('Writing progress to log file here: ' + str(join(fim_dir,'log_composite_n' + output_suffix + '.log')))
     print('This may take a few minutes...')
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
@@ -194,12 +193,6 @@ def generate_src_plot(df_src, plt_out_dir):
         else:
             print('Running the variable_mannings_calc function...')
 
-            # initiate log file
-            sys.__stdout__ = sys.stdout
-            log_file = open(join(fim_dir,'log_composite_n' + hydrotable_suffix + '.log'),"w")
-            sys.stdout = log_file
-            log_file.write('START TIME: ' + str(begin_time) + '\n')
-
             ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
             huc_list  = os.listdir(fim_dir)
             skip_hucs_log = ""
@@ -208,24 +201,29 @@ def generate_src_plot(df_src, plt_out_dir):
                     in_src_bankfull_filename = join(fim_dir,huc,'src_full_crosswalked_bankfull.csv')
                     out_src_bankfull_filename = join(fim_dir,huc,'src_full_crosswalked_vmann.csv')
                     htable_filename = join(fim_dir,huc,'hydroTable.csv')
-                    new_htable_filename = join(fim_dir,huc,'hydroTable' + hydrotable_suffix + '.csv')
                     huc_plot_output_dir = join(fim_dir,huc,'src_plots')
 
                     if isfile(in_src_bankfull_filename):
                         print(str(huc))
-                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_bankfull_filename, htable_filename, new_htable_filename, src_plot_option, huc_plot_output_dir])
+                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_bankfull_filename, htable_filename, src_plot_option, huc_plot_output_dir])
                     else:
-                        print(str(huc) + ' --> can not find the src_full_crosswalked_bankfull.csv in the fim output dir: ' + str(join(fim_dir,huc)))
+                        print(str(huc) + '\nWARNING --> can not find the src_full_crosswalked_bankfull.csv in the fim output dir: ' + str(join(fim_dir,huc)) + ' - skipping this HUC!!!\n')
 
-            ## Initiate multiprocessing
+            ## initiate log file
             print(f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
-            with Pool(processes=number_of_jobs) as pool:
-                pool.map(variable_mannings_calc, procs_list)
+            sys.__stdout__ = sys.stdout
+            log_file = open(join(fim_dir,'logs','log_composite_n' + output_suffix + '.log'),"w")
+            sys.stdout = log_file
+            log_file.write('START TIME: ' + str(begin_time) + '\n')
+            log_file.write('#########################################################\n\n')
+
+            ## Pass huc procs_list to multiprocessing function
+            multi_process(variable_mannings_calc, procs_list)
 
             ## Record run time and close log file
             end_time = dt.datetime.now()
             log_file.write('END TIME: ' + str(end_time) + '\n')
             tot_run_time = end_time - begin_time
-            log_file.write(str(tot_run_time))
+            log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
             sys.stdout = sys.__stdout__
             log_file.close()
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index e455cf472..f398a7a17 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -81,7 +81,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     magnitude_list = magnitude
     if type(magnitude_list) != list:
         magnitude_list = [magnitude_list]
-        
+
 
     # Get path to validation_data_{benchmark} directory and huc_dir.
     validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
@@ -98,7 +98,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
             for lid in lid_dir_list:
                 lid_dir = os.path.join(validation_data_path, current_huc, lid)
                 benchmark_lid_raster_path = os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
-                
+
                 # Only compare if the benchmark data exist.
                 if os.path.exists(benchmark_lid_raster_path):
                     benchmark_raster_path_list.append(benchmark_lid_raster_path)  # TEMP
@@ -106,7 +106,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                     lid_list.append(lid)
                     inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
                     domain_file_list.append(os.path.join(lid_dir, lid + '_domain.shp'))
-                    
+
         else:
             benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
             benchmark_raster_path_list = [benchmark_raster_file]
@@ -128,7 +128,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                      'operation': 'include'}
                         })
 
-                
+
                 if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_domain_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
                     continue
             else:  # If not in AHPS_BENCHMARK_CATEGORIES.

From dbefb75d69153da6e7bbcd0d718a88f1b94611f0 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 4 Nov 2021 08:58:28 -0500
Subject: [PATCH 118/359] Patch updater.py after BARC merge

Change API for release runs to use params_template instead of params_calibrated
---
 api/node/updater/updater.py | 2 +-
 docs/CHANGELOG.md           | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/api/node/updater/updater.py b/api/node/updater/updater.py
index 33fa0e08b..90fc0da71 100644
--- a/api/node/updater/updater.py
+++ b/api/node/updater/updater.py
@@ -454,7 +454,7 @@ def ws_add_job_to_queue(data):
         hucs = data['hucs']
         extent = data['extent']
         branch = 'dev'
-        config_path = './foss_fim/config/params_calibrated.env'
+        config_path = './foss_fim/config/params_template.env'
         dev_run = False
         viz_run = True
         previous_major_fim_version = data['previous_major_fim_version']
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e627a0d8c..53f751507 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.23.1 - 2021-11-03 - [PR #479](https://github.com/NOAA-OWP/cahaba/pull/479)
+Patches the API updater. The `params_calibrated.env` is replaced with `params_template.env` because the BARC and Multi-N modules supplant the calibrated values.
+
+## Changes
+- `api/node/updater/updater.py`: Changed `params_calibrated.env` to `params_template.env`
+
+<br/><br/>
+
 ## v3.0.23.0 - 2021-10-31 - [PR #475](https://github.com/NOAA-OWP/cahaba/pull/475)
 
 Moved the synthetic rating curve (SRC) processes from the `\tools` directory to `\src` directory to support post-processing in `fim_run.sh`. These SRC post-processing modules will now run as part of the default `fim_run.sh` workflow. Reconfigured bathymetry adjusted rating curve (BARC) module to use the 1.5yr flow from NWM v2 recurrence flow data in combination with the Bieger et al. (2015) regression equations with bankfull discharge predictor variable input.

From f903a809d99cb60c2af2612b25e5be8a82c83b1d Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Thu, 4 Nov 2021 14:24:38 -0500
Subject: [PATCH 119/359] Fix for null discharge values from composite
 roughness calcs

See Changelog for 3.0.32.2 for more details.
---
 docs/CHANGELOG.md                |  9 +++++++++
 src/vary_mannings_n_composite.py | 18 ++++++++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 53f751507..091b742ab 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,5 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+
+<br/><br/>
+
+## v3.0.23.2 - 2021-11-03 - [PR #480](https://github.com/NOAA-OWP/cahaba/pull/480)
+Hotfix for `vary_manning_n_composite.py` to address null discharge values for non-CONUS hucs.
+
+## Changes
+- `vary_manning_n_composite.py`: Add numpy where clause to set final discharge value to the original value if `vmann=False`.
+
 <br/><br/>
 
 ## v3.0.23.1 - 2021-11-03 - [PR #479](https://github.com/NOAA-OWP/cahaba/pull/479)
diff --git a/src/vary_mannings_n_composite.py b/src/vary_mannings_n_composite.py
index 1e563e0f1..900c5e281 100755
--- a/src/vary_mannings_n_composite.py
+++ b/src/vary_mannings_n_composite.py
@@ -42,7 +42,7 @@ def variable_mannings_calc(args):
     channel_ratio_src_column    = args[1]
     df_mann                     = args[2]
     huc                         = args[3]
-    out_src_bankfull_filename   = args[4]
+    out_src_vmann_filename   = args[4]
     htable_filename             = args[5]
     src_plot_option             = args[6]
     huc_output_dir              = args[7]
@@ -53,8 +53,10 @@ def variable_mannings_calc(args):
 
     ## Check that the channel ratio column the user specified exists in the def
     if channel_ratio_src_column not in df_src.columns:
-        log_text += 'WARNING --> ' + str(huc) + out_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
+        log_text += 'WARNING --> ' + str(huc) + in_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
     else:
+        ## Raname the current discharge & ManningN columns
+        df_src = df_src.rename(columns={'Discharge (m3s-1)':'default_Discharge (m3s-1)','ManningN':'default_ManningN'})
         ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the channel_ratio_src_column)
         df_src = df_src.merge(df_mann,  how='left', on='feature_id')
         check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
@@ -69,7 +71,7 @@ def variable_mannings_calc(args):
         check_null_comp = df_src['comp_ManningN'].isnull().sum()
         if check_null_comp > 0:
             log_text += str(huc) + ' --> ' + 'Missing values in the comp_ManningN calculation' + ' --> missing entries= ' + str(check_null_comp/84)  + '\n'
-        df_src['vmann_on'] = np.where(df_src['comp_ManningN'].isnull(), False, True) # field to identify where vmann is on/off
+        df_src['vmann_on'] = np.where(df_src['comp_ManningN'].isnull(), False, True) # create field to identify where vmann is applied (True=yes; False=no)
 
         ## Define the channel geometry variable names to use from the src
         hydr_radius = 'HydraulicRadius (m)'
@@ -87,8 +89,12 @@ def variable_mannings_calc(args):
             df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == df_src['Thalweg_burn_elev'],0,inplace=True)
             df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] < df_src['Thalweg_burn_elev'],-999,inplace=True)
 
+        ## Use the default discharge column when vmann is not being applied
+        df_src['Discharge (m3s-1)_varMann'] = np.where(df_src['vmann_on']==False, df_src['default_Discharge (m3s-1)'], df_src['Discharge (m3s-1)_varMann']) # reset the discharge value back to the original if vmann=false
+        df_src['comp_ManningN'] = np.where(df_src['vmann_on']==False, df_src['default_ManningN'], df_src['comp_ManningN']) # reset the ManningN value back to the original if vmann=false
+
         ## Output new SRC with bankfull column
-        df_src.to_csv(out_src_bankfull_filename,index=False)
+        df_src.to_csv(out_src_vmann_filename,index=False)
 
         ## Output new hydroTable with updated discharge and ManningN column
         df_src_trim = df_src[['HydroID','Stage','vmann_on','Discharge (m3s-1)_varMann','comp_ManningN']]
@@ -199,13 +205,13 @@ def multi_process(variable_mannings_calc, procs_list):
             for huc in huc_list:
                 if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
                     in_src_bankfull_filename = join(fim_dir,huc,'src_full_crosswalked_bankfull.csv')
-                    out_src_bankfull_filename = join(fim_dir,huc,'src_full_crosswalked_vmann.csv')
+                    out_src_vmann_filename = join(fim_dir,huc,'src_full_crosswalked_vmann.csv')
                     htable_filename = join(fim_dir,huc,'hydroTable.csv')
                     huc_plot_output_dir = join(fim_dir,huc,'src_plots')
 
                     if isfile(in_src_bankfull_filename):
                         print(str(huc))
-                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_bankfull_filename, htable_filename, src_plot_option, huc_plot_output_dir])
+                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_vmann_filename, htable_filename, src_plot_option, huc_plot_output_dir])
                     else:
                         print(str(huc) + '\nWARNING --> can not find the src_full_crosswalked_bankfull.csv in the fim output dir: ' + str(join(fim_dir,huc)) + ' - skipping this HUC!!!\n')
 

From 69f5fb36a0149a031ffb4b1044c382829c0e834f Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 5 Nov 2021 13:07:56 -0500
Subject: [PATCH 120/359] Additional hydraulic properties to hydroTable and
 additional files in viz whitelist

See CHANGELOG v3.0.23.3 for more details.
---
 docs/CHANGELOG.md                | 14 ++++++++++++--
 src/add_crosswalk.py             |  5 ++++-
 src/bathy_src_adjust_topwidth.py |  9 +++++----
 src/output_cleanup.py            |  6 +++++-
 src/run_by_unit.sh               |  8 ++++----
 src/vary_mannings_n_composite.py | 14 +++++++++++++-
 6 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 091b742ab..f3a7cb036 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,13 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.23.3 - 2021-11-04 - [PR #481](https://github.com/NOAA-OWP/cahaba/pull/481)
+Includes additional hydraulic properties to the `hydroTable.csv`: `Number of Cells`, `SurfaceArea (m2)`, `BedArea (m2)`, `Volume (m3)`, `SLOPE`, `LENGTHKM`, `AREASQKM`, `Roughness`, `TopWidth (m)`, `WettedPerimeter (m)`. Also adds `demDerived_reaches_split_points.gpkg`, `flowdir_d8_burned_filled.tif`, and `dem_thalwegCond.tif` to `-v` whitelist.
+
+## Changes
+- `run_by_unit.sh`: Added `EXIT FLAG` tag and previous non-zero exit code tag to the print statement to allow log lookup.
+- `add_crosswalk.py`: Added extra attributes to the hydroTable.csv. Includes a default `barc_on` and `vmann_on` (=False) attribute that is overwritten (=True) if SRC post-processing modules are run.
+- `bathy_src_adjust_topwidth.py`: Overwrites the `barc_on` attribute where applicable and includes the BARC-modified Volume property.
+- `vary_mannings_n_composite.py`: Overwrites the `vmann_on` attribute where applicable.
+- `output_cleanup.py`: Adds new files to the `-v` whitelist.
+
 <br/><br/>
 
-## v3.0.23.2 - 2021-11-03 - [PR #480](https://github.com/NOAA-OWP/cahaba/pull/480)
+## v3.0.23.2 - 2021-11-04 - [PR #480](https://github.com/NOAA-OWP/cahaba/pull/480)
 Hotfix for `vary_manning_n_composite.py` to address null discharge values for non-CONUS hucs.
 
 ## Changes
-- `vary_manning_n_composite.py`: Add numpy where clause to set final discharge value to the original value if `vmann=False`.
+- `vary_manning_n_composite.py`: Add numpy where clause to set final discharge value to the original value if `vmann=False`
 
 <br/><br/>
 
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 82458a9c4..49caf74e0 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -220,8 +220,11 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_crosswalk = output_crosswalk.drop_duplicates(ignore_index=True)
 
     # make hydroTable
-    output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Stage','Discharge (m3s-1)', 'HydraulicRadius (m)', 'WetArea (m2)', 'SLOPE', 'ManningN']]
+    output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)','BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)','Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
+    output_hydro_table['barc_on'] = False # set barc_on attribute to Fasle (default) --> will be overwritten if BARC module runs
+    output_hydro_table['vmann_on'] = False # set vmann_on attribute to Fasle (default) --> will be overwritten if variable roughness module runs
+
 
     if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
     output_hydro_table[FIM_ID] = output_hydro_table.loc[:,'HydroID'].apply(lambda x : str(x)[0:4])
diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py
index 31fba91fa..dc2c94dd9 100644
--- a/src/bathy_src_adjust_topwidth.py
+++ b/src/bathy_src_adjust_topwidth.py
@@ -202,13 +202,14 @@ def bathy_rc_lookup(args):
         ## Output new src_full_crosswalked
         modified_src_base.to_csv(out_src_filename,index=False)
         ## Update the hydroTable
-        modified_hydro_table = modified_src_base.loc[:,['HydroID','Stage','barc_on','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)']]
+        modified_hydro_table = modified_src_base.loc[:,['HydroID','Stage','barc_on','Volume (m3)','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)']]
         modified_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
         df_htable = pd.read_csv(input_htable_fileName,dtype={'HUC': str})
-        if not set(['orig_discharge_cms','orig_WetArea (m2)','orig_HydraulicRadius (m)']).issubset(df_htable.columns):
-            df_htable.rename(columns={'discharge_cms':'orig_discharge_cms','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'},inplace=True)
+        df_htable.drop(['barc_on'], axis=1, inplace=True) # drop the default "barc_on" variable from add_crosswalk.py
+        if not set(['orig_discharge_cms','orig_Volume (m3)','orig_WetArea (m2)','orig_HydraulicRadius (m)']).issubset(df_htable.columns): # check if "orig_" attributes do NOT already exist (likely generated from previous BARC run)
+            df_htable.rename(columns={'discharge_cms':'orig_discharge_cms','Volume (m3)':'orig_Volume (m3)','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'},inplace=True)
         else:
-            df_htable.drop(['discharge_cms','WetArea (m2)','HydraulicRadius (m)'], axis=1, inplace=True) # drop the previously modified columns - to be replaced with updated version
+            df_htable.drop(['discharge_cms','Volume (m3)','WetArea (m2)','HydraulicRadius (m)'], axis=1, inplace=True) # drop the previously modified columns - to be replaced with updated version
         df_htable = df_htable.merge(modified_hydro_table, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
         df_htable.to_csv(input_htable_fileName,index=False)
         log_text += ('Output new hydroTable and src_full_crosswalked: ') + '\n'
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index e65dfeaee..2491f05a5 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -49,7 +49,11 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         'gw_catchments_reaches_filtered_addedAttributes.tif',
         'hydroTable.csv',
         'src.json',
-        'small_segments.csv'
+        'small_segments.csv',
+        'src_full_crosswalked.csv',
+        'demDerived_reaches_split_points.gpkg',
+        'flowdir_d8_burned_filled.tif',
+        'dem_thalwegCond.tif'
     ]
 
     # If "production" run, only keep whitelisted files
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index ec456cefb..c5d5b4e38 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -62,7 +62,7 @@ Tcount
 
 if [ "$extent" = "MS" ]; then
   if [[ ! -f $outputHucDataDir/nhd_headwater_points_subset.gpkg ]] ; then
-    echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
+    echo "EXIT FLAG!! (exit 55): No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
     rm -rf $outputHucDataDir
     exit 0
   fi
@@ -233,7 +233,7 @@ python3 -m memory_profiler $srcDir/split_flows.py -f $outputHucDataDir/demDerive
 Tcount
 
 if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
-  echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
+  echo "EXIT FLAG!! (exit 56): No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
   rm -rf $outputHucDataDir
   exit 0
 fi
@@ -247,7 +247,7 @@ if [ "$extent" = "MS" ]; then
   Tcount
 
   if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
-    echo "No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
+    echo "EXIT FLAG!! (exit 57): No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
     rm -rf $outputHucDataDir
     exit 0
   fi
@@ -327,7 +327,7 @@ Tstart
 python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputHucDataDir/gw_catchments_reaches.gpkg -f $outputHucDataDir/demDerived_reaches_split.gpkg -c $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -o $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
 
 if [[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ]] ; then
-  echo "No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
+  echo "EXIT FLAG!! (exit 65): No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
   rm -rf $outputHucDataDir
   exit 0
 fi
diff --git a/src/vary_mannings_n_composite.py b/src/vary_mannings_n_composite.py
index 900c5e281..46445efa0 100755
--- a/src/vary_mannings_n_composite.py
+++ b/src/vary_mannings_n_composite.py
@@ -97,16 +97,28 @@ def variable_mannings_calc(args):
         df_src.to_csv(out_src_vmann_filename,index=False)
 
         ## Output new hydroTable with updated discharge and ManningN column
-        df_src_trim = df_src[['HydroID','Stage','vmann_on','Discharge (m3s-1)_varMann','comp_ManningN']]
+        df_src_trim = df_src[['HydroID','Stage','vmann_on',channel_ratio_src_column,'Discharge (m3s-1)_varMann','comp_ManningN']]
         df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_varMann': 'discharge_cms','comp_ManningN':'ManningN'})
         df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
         df_htable.rename(columns={'ManningN':'orig_ManningN'},inplace=True)
+        df_htable.drop(['vmann_on'], axis=1, inplace=True) # drop the default "vmann_on" variable from add_crosswalk.py
         if not set(['orig_discharge_cms']).issubset(df_htable.columns):
             df_htable.rename(columns={'discharge_cms':'orig_discharge_cms'},inplace=True)
         else:
             df_htable.drop(['discharge_cms'], axis=1, inplace=True) # drop the previously modified discharge column to be replaced with updated version
         df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
+        
+        # Delete intermediate CSVs outputs. Todo delete this block later.
+        htable_parent_dir = os.path.split(htable_filename)[0]
+        # List all CSVs.
+        file_list = os.listdir(htable_parent_dir)
+        for f in file_list:
+            if '.csv' in f:
+                if f != 'hydroTable.csv':
+                    os.remove(os.path.join(htable_parent_dir, f))
+        
         df_htable.to_csv(htable_filename,index=False)
+        
         log_text += 'Completed: ' + str(huc)
 
         ## plot rating curves

From da2421111d06511d178ba3396a3dd6d77a18f428 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Mon, 15 Nov 2021 20:21:38 -0600
Subject: [PATCH 121/359] Adds full resolution and mainstem inundation
 composite capability

This resolves #476. See Changelog 3.0.24.0 for full details.
---
 docs/CHANGELOG.md                         |  13 +
 tools/composite_ms_fr_inundation.py       | 207 +++++++++
 tools/gms_tools/inundate_gms.py           | 261 +++++++++++
 tools/gms_tools/mosaic_inundation.py      | 145 +++++++
 tools/gms_tools/overlapping_inundation.py | 505 ++++++++++++++++++++++
 tools/inundation.py                       |  10 +-
 tools/run_test_case.py                    |  13 +-
 tools/run_test_case_gms.py                | 376 ++++++++++++++++
 tools/tools_shared_variables.py           |   2 +
 9 files changed, 1528 insertions(+), 4 deletions(-)
 create mode 100644 tools/composite_ms_fr_inundation.py
 create mode 100644 tools/gms_tools/inundate_gms.py
 create mode 100644 tools/gms_tools/mosaic_inundation.py
 create mode 100644 tools/gms_tools/overlapping_inundation.py
 create mode 100644 tools/run_test_case_gms.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f3a7cb036..8a0de7039 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.24.0 - 2021-11-08 - [PR #482](https://github.com/NOAA-OWP/cahaba/pull/482)
+
+Adds `composite_ms_fr_inundation.py` to allow for the generation of an inundation map given a "flow file" CSV and full-resolution (FR) and mainstem (MS) relative elevation models, synthetic rating curves, and catchments rasters created by the `fim_run.sh` script.
+
+## Additions
+- `composite_ms_fr_inundation.py`: New module that is used to inundate both MS and FR FIM and composite the two inundation rasters.
+- `/tools/gms_tools/`: Three modules (`inundate_gms.py`, `mosaic_inundation.py`, `overlapping_inundation.py`) ported from the GMS branch used to composite inundation rasters.
+
+## Changes
+- `inundation.py`: Added 2 exception classes ported from the GMS branch.
+
+<br/><br/>
+
 ## v3.0.23.3 - 2021-11-04 - [PR #481](https://github.com/NOAA-OWP/cahaba/pull/481)
 Includes additional hydraulic properties to the `hydroTable.csv`: `Number of Cells`, `SurfaceArea (m2)`, `BedArea (m2)`, `Volume (m3)`, `SLOPE`, `LENGTHKM`, `AREASQKM`, `Roughness`, `TopWidth (m)`, `WettedPerimeter (m)`. Also adds `demDerived_reaches_split_points.gpkg`, `flowdir_d8_burned_filled.tif`, and `dem_thalwegCond.tif` to `-v` whitelist.
 
diff --git a/tools/composite_ms_fr_inundation.py b/tools/composite_ms_fr_inundation.py
new file mode 100644
index 000000000..1b975a8b9
--- /dev/null
+++ b/tools/composite_ms_fr_inundation.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+import os, argparse, rasterio
+import numpy as np
+import pandas as pd
+from multiprocessing import Pool
+
+from inundation import inundate
+from gms_tools.mosaic_inundation import Mosaic_inundation, __append_id_to_file_name
+
+
+def composite_inundation(fim_dir_ms, fim_dir_fr, huc, flows, composite_output_dir, ouput_name='',
+                         bin_rast_flag=False, depth_rast_flag=False, clean=True, quiet=True):
+    """
+    Run `inundate()` on FIM 3.X mainstem (MS) and full-resolution (FR) outputs and composite results. Assumes that all `fim_run` products 
+    necessary for `inundate()` are in each huc8 folder.
+
+    Parameters
+    ----------
+    fim_dir_ms : str
+        Path to MS FIM directory. This should be an output directory from `fim_run.sh`.
+    fim_dir_fr : str
+        Path to FR FIM directory. This should be an output directory from `fim_run.sh`.
+    huc : str
+        HUC8 to run `inundate()`. This should be a folder within both `fim_dir_ms` and `fim_dir_fr`.
+    flows : str or pandas.DataFrame, can be a single file or a comma-separated list of files
+        File path to forecast csv or Pandas DataFrame with correct column names.
+    composite_output_dir : str
+        Folder path to write outputs. It will be created if it does not exist.
+    ouput_name : str, optional
+        Name for output raster. If not specified, by default the raster will be named 'inundation_composite_{flows_root}.tif'.
+    bin_rast_flag : bool, optional
+        Flag to create binary raster as output. If no raster flags are passed, this is the default behavior.
+    depth_rast_flag : bool, optional
+        Flag to create depth raster as output.
+    clean : bool, optional
+        If True, intermediate files are deleted.
+    quiet : bool, optional
+        Quiet output.
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    TypeError
+        Wrong input data types
+    AssertionError
+        Wrong input data types
+
+    Notes
+    -----
+    - Specifying a subset of the domain in rem or catchments to inundate on is achieved by the HUCs file or the forecast file.
+
+    Examples
+    --------
+    >>> import composite_ms_fr_inundation
+    >>> composite_ms_fr_inundation.composite_inundation(
+            '/home/user/fim_ouput_mainstem',
+            '/home/user/fim_ouput_fullres',
+            '12090301',
+            '/home/user/forecast_file.csv',
+            '/home/user/fim_inundation_composite',
+            True,
+            False)
+    """
+    # Set inundation raster to True if no output type flags are passed
+    if not (bin_rast_flag or depth_rast_flag):
+        bin_rast_flag = True
+    assert not (bin_rast_flag and depth_rast_flag), 'Output can only be binary or depth grid, not both'
+    assert os.path.isdir(fim_dir_ms), f'{fim_dir_ms} is not a directory. Please specify an existing MS FIM directory.'
+    assert os.path.isdir(fim_dir_fr), f'{fim_dir_fr} is not a directory. Please specify an existing FR FIM directory.'
+    assert os.path.exists(flows), f'{flows} does not exist. Please specify a flow file.'
+
+    # Instantiate output variables
+    var_keeper = {
+        'ms': {
+            'dir': fim_dir_ms,
+            'outputs': {
+                'inundation_rast': os.path.join(composite_output_dir, f'inundation_ms_{huc}.tif') if bin_rast_flag else None,
+                'depth_rast':      os.path.join(composite_output_dir, f'depth_ms_{huc}.tif') if depth_rast_flag else None
+            }
+        },
+        'fr': {
+            'dir': fim_dir_fr,
+            'outputs': {
+                'inundation_rast': os.path.join(composite_output_dir, f'inundation_fr_{huc}.tif') if bin_rast_flag else None,
+                'depth_rast':      os.path.join(composite_output_dir, f'depth_fr_{huc}.tif') if depth_rast_flag else None
+            }
+        }
+    }
+    # Build inputs to inundate() based on the input folders and huc
+    if not quiet: print(f"HUC {huc}")
+    for extent in var_keeper:
+        rem = os.path.join(var_keeper[extent]['dir'], huc, 'rem_zeroed_masked.tif')
+        catchments = os.path.join(var_keeper[extent]['dir'], huc, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+        catchment_poly = os.path.join(var_keeper[extent]['dir'], huc, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+        hydro_table = os.path.join(var_keeper[extent]['dir'], huc, 'hydroTable.csv')
+
+        # Ensure that all of the required files exist in the huc directory
+        for file in (rem, catchments, catchment_poly, hydro_table):
+            if not os.path.exists(file):
+                raise Exception(f"The following file does not exist within the supplied FIM directory:\n{file}")
+
+        # Run inundation()
+        extent_friendly = "mainstem (MS)" if extent=="ms" else "full-resolution (FR)"
+        if not quiet: print(f"  Creating an inundation map for the {extent_friendly} configuration...")
+        result = inundate(rem,catchments,catchment_poly,hydro_table,flows,mask_type=None,
+                inundation_raster=  var_keeper[extent]['outputs']['inundation_rast'],
+                depths=             var_keeper[extent]['outputs']['depth_rast'],
+                quiet=              quiet)
+        if result != 0:
+            raise Exception(f"Failed to inundate {rem} using the provided flows.")
+
+    # If no output name supplied, create one using the flows file name 
+    if not ouput_name:
+        flows_root = os.path.splitext(os.path.basename(flows))[0]
+        ouput_name = os.path.join(composite_output_dir, f'inundation_composite_{flows_root}.tif')
+    else:
+        ouput_name = os.path.join(composite_output_dir, ouput_name)
+    
+    # Composite MS and FR
+    inundation_map_file = { 
+                    'huc8' : [huc] * 2,
+                    'branchID' : [None] * 2,
+                    'inundation_rasters':  [var_keeper['fr']['outputs']['inundation_rast'], 
+                                            var_keeper['ms']['outputs']['inundation_rast']],
+                    'depths_rasters':      [var_keeper['fr']['outputs']['depth_rast'], 
+                                            var_keeper['ms']['outputs']['depth_rast']]
+                    }
+    inundation_map_file = pd.DataFrame(inundation_map_file)
+    Mosaic_inundation(
+                    inundation_map_file,
+                    mosaic_attribute='depths_rasters' if depth_rast_flag else 'inundation_rasters',
+                    mosaic_output=ouput_name,
+                    mask=catchment_poly,
+                    unit_attribute_name='huc8',
+                    nodata=-9999,
+                    workers=1,
+                    remove_inputs=clean,
+                    subset=None,verbose=not quiet
+                    )
+    if bin_rast_flag:
+        hydroid_to_binary(__append_id_to_file_name(ouput_name, huc))
+
+def hydroid_to_binary(hydroid_raster_filename):
+    '''Converts hydroid positive/negative grid to 1/0'''
+
+    #to_bin = lambda x: np.where(x > 0, 1, np.where(x == 0, -9999, 0))
+    to_bin = lambda x: np.where(x > 0, 1, np.where(x != -9999, 0, -9999))
+    hydroid_raster = rasterio.open(hydroid_raster_filename)
+    profile = hydroid_raster.profile # get profile for new raster creation later on
+    profile['nodata'] = -9999
+    bin_raster = to_bin(hydroid_raster.read(1)) # converts neg/pos to 0/1
+    # Overwrite inundation raster
+    with rasterio.open(hydroid_raster_filename, "w", **profile) as out_raster:
+        out_raster.write(bin_raster.astype(hydroid_raster.profile['dtype']), 1)
+    del hydroid_raster,profile,bin_raster
+
+
+if __name__ == '__main__':
+    
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Inundate FIM 3 full resolution and mainstem outputs using a flow file and composite the results.')
+    parser.add_argument('-ms','--fim-dir-ms',help='Directory that contains MS FIM outputs.',required=True)
+    parser.add_argument('-fr','--fim-dir-fr',help='Directory that contains FR FIM outputs.',required=True)
+    parser.add_argument('-u','--huc',help='HUC within FIM directories to inunundate. Can be a comma-separated list.',required=True)
+    parser.add_argument('-f','--flows-file',help='File path of flows csv or comma-separated list of paths if running multiple HUCs',required=True)
+    parser.add_argument('-o','--ouput-dir',help='Folder to write Composite Raster output.',required=True)
+    parser.add_argument('-n','--ouput-name',help='File name for output(s).',default=None,required=False)
+    parser.add_argument('-b','--bin-raster',help='Output raster is a binary wet/dry grid. This is the default if no raster flags are passed.',required=False,default=False,action='store_true')
+    parser.add_argument('-d','--depth-raster',help='Output raster is a depth grid.',required=False,default=False,action='store_true')
+    parser.add_argument('-j','--num-workers',help='Number of concurrent processesto run.',required=False,default=1,type=int)
+    parser.add_argument('-c','--clean',help='If flag used, intermediate rasters are NOT cleaned up.',required=False,default=True,action='store_false')
+    parser.add_argument('-q','--quiet',help='Quiet terminal output.',required=False,default=False,action='store_true')
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+    fim_dir_ms    = args['fim_dir_ms']
+    fim_dir_fr    = args['fim_dir_fr']
+    hucs          = args['huc'].replace(' ', '').split(',')
+    flows_files   = args['flows_file'].replace(' ', '').split(',')
+    num_workers   = int(args['num_workers'])
+    output_dir    = args['ouput_dir']
+    ouput_name    = args['ouput_name']
+    bin_raster    = bool(args['bin_raster'])
+    depth_raster  = bool(args['depth_raster'])
+    clean         = bool(args['clean'])
+    quiet         = bool(args['quiet'])
+
+    assert num_workers >= 1, "Number of workers should be 1 or greater"
+    assert len(flows_files) == len(hucs), "Number of hucs must be equal to the number of forecasts provided"
+    assert not (bin_raster and depth_raster), "Cannot use both -b and -d flags"
+
+    # Create output directory if it does not exist
+    if not os.path.isdir(output_dir):
+        os.mkdir(output_dir)
+
+    # Create nested list for input into multi-threading
+    arg_list = []
+    for huc, flows_file in zip(hucs, flows_files):
+        arg_list.append((fim_dir_ms, fim_dir_fr, huc, flows_file, output_dir, ouput_name, bin_raster, depth_raster, clean, quiet))
+
+    # Multi-thread for each huc in input hucs
+    with Pool(processes=num_workers) as pool:
+        # Run composite_inundation()
+        pool.starmap(composite_inundation, arg_list)
diff --git a/tools/gms_tools/inundate_gms.py b/tools/gms_tools/inundate_gms.py
new file mode 100644
index 000000000..0148da336
--- /dev/null
+++ b/tools/gms_tools/inundate_gms.py
@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+
+import numpy as np
+import pandas as pd
+from inundation import inundate
+import os
+from tqdm import tqdm
+import argparse
+from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor,as_completed
+from inundation import hydroTableHasOnlyLakes, NoForecastFound
+import traceback
+import logging
+
+
+def Inundate_gms(
+                  hydrofabric_dir, forecast, num_workers=1,
+                  hucs=None,
+                  inundation_raster=None,
+                  inundation_polygon=None, depths_raster=None,
+                  verbose=False,
+                  log_file=None,
+                  output_fileNames=None
+                 ):
+
+    # input handling
+    if hucs is not None:
+        try:
+            _ = (i for i in hucs)
+        except TypeError:
+            raise ValueError("hucs argument must be an iterable")
+    
+    if isinstance(hucs,str):
+        hucs = [hucs]
+
+    num_workers = int(num_workers)
+
+    # log file
+    if log_file is not None:
+        if os.path.exists(log_file):
+            os.remove(log_file)
+        
+        print('HUC8,BranchID,Exception',file=open(log_file,'w'))
+    #if log_file:
+        #logging.basicConfig(filename=log_file, level=logging.INFO)
+        #logging.info('HUC8,BranchID,Exception')
+
+    # load gms inputs
+    hucs_branches = pd.read_csv( os.path.join(hydrofabric_dir,'gms_inputs.csv'),
+                                 header=None,
+                                 dtype= {0:str,1:str}
+                               )
+    
+    if hucs is not None:
+        hucs = set(hucs)
+        huc_indices = hucs_branches.loc[:,0].isin(hucs)
+        hucs_branches = hucs_branches.loc[huc_indices,:]
+
+    # get number of branches
+    number_of_branches = len(hucs_branches)
+    
+    # make inundate generator
+    inundate_input_generator = __inundate_gms_generator(
+                                                          hucs_branches,number_of_branches,
+                                                          hydrofabric_dir,
+                                                          inundation_raster,
+                                                          inundation_polygon,
+                                                          depths_raster,
+                                                          forecast,
+                                                          verbose=False
+                                                        )
+
+    # start up process pool
+    # better results with Process pool
+    executor = ProcessPoolExecutor(max_workers=num_workers)
+
+    # collect output filenames
+    inundation_raster_fileNames = [None] * number_of_branches
+    inundation_polygon_fileNames = [None] * number_of_branches
+    depths_raster_fileNames = [None] * number_of_branches
+    hucCodes = [None] * number_of_branches
+    branch_ids = [None] * number_of_branches
+       
+
+    executor_generator = { 
+                executor.submit(inundate,**inp) : ids for inp,ids in inundate_input_generator 
+                }
+
+    idx = 0
+    for future in tqdm(as_completed(executor_generator),
+                       total=len(executor_generator),
+                       disable=(not verbose),
+                       desc="Inundating branches with {} workers".format(num_workers)
+                      ):
+        
+        hucCode, branch_id = executor_generator[future]
+
+        try:
+            future.result()
+        
+        except NoForecastFound as exc:
+            if log_file is not None:
+                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
+                      file=open(log_file,'a'))
+            elif verbose:
+                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
+
+        except hydroTableHasOnlyLakes as exc:
+            if log_file is not None:
+                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
+                      file=open(log_file,'a'))
+            elif verbose:
+                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
+        
+        except Exception as exc:
+            if log_file is not None:
+                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
+                      file=open(log_file,'a'))
+            else:
+                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
+        else:
+            
+            hucCodes[idx] = hucCode
+            branch_ids[idx] = branch_id
+
+            try:
+                #print(hucCode,branch_id,future.result()[0][0])
+                inundation_raster_fileNames[idx] = future.result()[0][0]
+            except TypeError:
+                pass
+
+            try:
+                depths_raster_fileNames[idx] = future.result()[1][0]
+            except TypeError:
+                pass
+
+            try:
+                inundation_polygon_fileNames[idx] = future.result()[2][0]
+            except TypeError:
+                pass
+
+            idx += 1 
+    
+    # power down pool
+    executor.shutdown(wait=True)
+
+    # make filename dataframe
+    output_fileNames_df = pd.DataFrame( { 
+                                          'huc8' : hucCodes,
+                                          'branchID' : branch_ids,
+                                          'inundation_rasters' : inundation_raster_fileNames,
+                                          'depths_rasters' : depths_raster_fileNames,
+                                          'inundation_polygons' : inundation_polygon_fileNames } 
+                                       )
+
+    if output_fileNames is not None:
+        output_fileNames_df.to_csv(output_fileNames,index=False)
+
+    return(output_fileNames_df)
+
+
+
+
+def __inundate_gms_generator( 
+                              hucs_branches,number_of_branches,
+                              hydrofabric_dir,
+                              inundation_raster,
+                              inundation_polygon,
+                              depths_raster,
+                              forecast,verbose=False
+                            ):
+
+    # iterate over branches
+    for idx,row in hucs_branches.iterrows():
+        
+        huc = str(row[0])
+        branch_id = str(row[1])
+
+        gms_dir = os.path.join(hydrofabric_dir,huc,'branches')
+
+        rem_branch = os.path.join( gms_dir,branch_id,'rem_zeroed_masked_{}.tif'.format(branch_id) )
+        catchments_branch = os.path.join( gms_dir,branch_id,
+                                          f'gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif' )
+        hydroTable_branch = os.path.join( gms_dir,branch_id,'hydroTable_{}.csv'.format(branch_id) )
+        catchment_poly = os.path.join( gms_dir, branch_id,
+                                       f'gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg' )
+        
+    
+        # branch output
+        inundation_branch_raster = __append_id_to_file_name(inundation_raster,[huc,branch_id])
+        inundation_branch_polygon = __append_id_to_file_name(inundation_polygon,[huc,branch_id])
+        depths_branch_raster = __append_id_to_file_name(depths_raster,[huc,branch_id])
+
+        # identifiers
+        identifiers = (huc,branch_id)
+
+        # inundate input
+        inundate_input = { 
+                            'rem' : rem_branch, 'catchments' : catchments_branch, 'catchment_poly' : catchment_poly,
+                            'hydro_table' : hydroTable_branch,'forecast' : forecast,
+                            'mask_type' : None,
+                            'hucs' : None,
+                            'hucs_layerName' : None,
+                            'subset_hucs' : None, 'num_workers' : 1,
+                            'aggregate' : False,
+                            'inundation_raster' : inundation_branch_raster,
+                            'inundation_polygon' : inundation_branch_polygon,
+                            'depths' : depths_branch_raster,
+                            'out_raster_profile' : None,
+                            'out_vector_profile' : None,
+                            'quiet' : not verbose
+                          }
+
+        yield (inundate_input,identifiers)
+
+
+
+def __append_id_to_file_name(file_name,identifier):
+
+
+    if file_name is not None:
+
+        root,extension = os.path.splitext(file_name)
+
+        if isinstance(identifier,list):
+            for i in identifier:
+                out_file_name = root + "_{}".format(i)
+            out_file_name += extension
+        else:
+            out_file_name = root + "_{}".format(identifier) + extension
+
+    else:
+        out_file_name = None
+
+    return(out_file_name)
+
+
+def __vprint(message,verbose):
+    if verbose:
+        print(message)
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Inundate GMS')
+    parser.add_argument('-y','--hydrofabric_dir', help='Directory path to FIM hydrofabric by processing unit', required=True)
+    parser.add_argument('-u','--hucs',help='List of HUCS to run',required=False,default=None,type=str,nargs='+')
+    parser.add_argument('-f','--forecast',help='Forecast discharges in CMS as CSV file',required=True)
+    parser.add_argument('-i','--inundation-raster',help='Inundation Raster output. Only writes if designated.',required=False,default=None)
+    parser.add_argument('-p','--inundation-polygon',help='Inundation polygon output. Only writes if designated.',required=False,default=None)
+    parser.add_argument('-d','--depths-raster',help='Depths raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
+    parser.add_argument('-l','--log-file',help='Log-file to store level-path exceptions',required=False,default=None)
+    parser.add_argument('-o','--output-fileNames',help='Output CSV file with filenames for inundation rasters, inundation polygons, and depth rasters',required=False,default=None)
+    parser.add_argument('-w','--num-workers', help='Number of Workers', required=False,default=1)
+    parser.add_argument('-v','--verbose',help='Verbose printing',required=False,default=None,action='store_true')
+    
+    
+    # extract to dictionary and run
+    Inundate_gms( **vars(parser.parse_args()) )
+    
+    
\ No newline at end of file
diff --git a/tools/gms_tools/mosaic_inundation.py b/tools/gms_tools/mosaic_inundation.py
new file mode 100644
index 000000000..ea24cb75f
--- /dev/null
+++ b/tools/gms_tools/mosaic_inundation.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from glob import glob
+from gms_tools.overlapping_inundation import OverlapWindowMerge
+import argparse
+import os
+import pandas as pd
+from tqdm import tqdm
+from tools_shared_variables import elev_raster_ndv
+
+def Mosaic_inundation(
+                      map_file,mosaic_attribute='inundation_rasters',mosaic_output=None,
+                      mask=None,unit_attribute_name='huc8',
+                      nodata=elev_raster_ndv,workers=4,
+                      remove_inputs=False,
+                      subset=None,verbose=True
+                      ):
+    
+    # check input
+    if mosaic_attribute not in ('inundation_rasters','depths_rasters'):
+        raise ValueError('Pass inundation or depths for mosaic_attribute argument') 
+
+    # load file
+    if isinstance(map_file,pd.DataFrame):
+        inundation_maps_df = map_file
+        del map_file
+    elif isinstance(map_file,str):
+        inundation_maps_df = pd.read_csv(map_file,
+                                         dtype={unit_attribute_name:str,'branchID':str}
+                                        )
+    else:
+        raise TypeError('Pass Pandas Dataframe or file path string to csv for map_file argument')
+
+    # remove NaNs
+    inundation_maps_df.dropna(axis=0,how='all',inplace=True)
+
+    # subset
+    if subset is not None:
+        subset_mask = inundation_maps_df.loc[:,unit_attribute_name].isin(subset)
+        inundation_maps_df = inundation_maps_df.loc[subset_mask,:]
+    
+    # unique aggregation units
+    aggregation_units = inundation_maps_df.loc[:,unit_attribute_name].unique()
+
+    inundation_maps_df.set_index(unit_attribute_name,drop=True,inplace=True)
+
+    # decide upon wheter to display 
+    if verbose & len(aggregation_units) == 1:
+        tqdm_disable = False
+    elif verbose:
+        tqdm_disable = False
+    else:
+        tqdm_disable = True
+
+    for ag in tqdm(aggregation_units,disable=tqdm_disable,desc='Compositing MS and FR maps'):
+
+        try:
+            inundation_maps_list = inundation_maps_df.loc[ag,mosaic_attribute].tolist()
+        except AttributeError:
+            inundation_maps_list = [ inundation_maps_df.loc[ag,mosaic_attribute] ]
+
+        ag_mosaic_output = __append_id_to_file_name(mosaic_output,ag)
+        #try:
+        mosaic_by_unit(inundation_maps_list,ag_mosaic_output,nodata,
+                       workers=1,remove_inputs=remove_inputs,mask=mask,verbose=verbose)
+        #except Exception as exc:
+        #    print(ag,exc)
+    
+
+    # inundation maps
+    inundation_maps_df.reset_index(drop=True)
+
+
+
+def mosaic_by_unit(inundation_maps_list,mosaic_output,nodata=elev_raster_ndv,
+                   workers=1,remove_inputs=False,mask=None,verbose=False):
+
+
+    # overlap object instance
+    overlap = OverlapWindowMerge( inundation_maps_list, (30, 30) )
+
+    # mosaic
+    #if verbose:
+    #    print("Mosaicing ...")
+
+    if mosaic_output is not None:
+        if workers > 1:
+            threaded = True
+        else:
+            threaded= False
+        
+        overlap.merge_rasters(mosaic_output, threaded=threaded, workers=workers,nodata=nodata)
+
+        if mask:
+            #if verbose:
+            #    print("Masking ...")
+            overlap.mask_mosaic(mosaic_output,mask,outfile=mosaic_output)
+    
+    if remove_inputs:
+        #if verbose:
+        #    print("Removing inputs ...")
+
+        for inun_map in inundation_maps_list:
+            if inun_map is not None:
+                if os.path.isfile(inun_map):
+                    os.remove(inun_map)
+
+
+def __append_id_to_file_name(file_name,identifier):
+
+
+    if file_name is not None:
+
+        root,extension = os.path.splitext(file_name)
+
+        if isinstance(identifier,list):
+            for i in identifier:
+                out_file_name = root + "_{}".format(i)
+            out_file_name += extension
+        else:
+            out_file_name = root + "_{}".format(identifier) + extension
+
+    else:
+        out_file_name = None
+
+    return(out_file_name)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Mosaic GMS Inundation Rasters')
+    parser.add_argument('-i','--map-file', help='List of file paths to inundation/depth maps to mosaic', required=True)
+    parser.add_argument('-a','--mask', help='File path to vector polygon mask to clip mosaic too', required=False,default=None)
+    parser.add_argument('-s','--subset', help='Subset units', required=False,default=None,type=str,nargs='+')
+    parser.add_argument('-n','--nodata', help='Inundation Maps', required=False,default=elev_raster_ndv)
+    parser.add_argument('-w','--workers', help='Number of Workers', required=False,default=4,type=int)
+    parser.add_argument('-t','--mosaic-attribute', help='Mosaiced inundation Maps', required=False,default=None)
+    parser.add_argument('-m','--mosaic-output', help='Mosaiced inundation Maps', required=False,default=None)
+    parser.add_argument('-r','--remove-inputs', help='Remove original input inundation Maps', required=False,default=False,action='store_true')
+    parser.add_argument('-v','--verbose', help='Remove original input inundation Maps', required=False,default=False,action='store_true')
+
+    args = vars(parser.parse_args())
+    
+    Mosaic_inundation(**args)
\ No newline at end of file
diff --git a/tools/gms_tools/overlapping_inundation.py b/tools/gms_tools/overlapping_inundation.py
new file mode 100644
index 000000000..5dab3da96
--- /dev/null
+++ b/tools/gms_tools/overlapping_inundation.py
@@ -0,0 +1,505 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import rasterio
+from rasterio.windows import from_bounds
+import numpy as np
+from functools import partial
+from affine import Affine
+from scipy.optimize import newton
+from threading import Lock
+import concurrent.futures
+from numba import njit
+import geopandas as gpd
+from rasterio.mask import mask
+import sys
+import warnings
+
+class OverlapWindowMerge:
+
+    def __init__(self,
+                 inundation_rsts,
+                 num_partitions=None,
+                 window_xy_size=None):
+        """
+        Initialize the object
+
+        :param inundation_rsts: list of inundation paths or datasets
+        :param num_partitions: tuple of integers representing num windows in x and y space
+        :param window_xy_size: tuple of integers represeting num of pixels in windows in x an y space
+        """
+
+        # sort for largest spanning dataset (todo: handle mismatched resolutions)
+        size_func = lambda x: np.abs(x.bounds.left - x.bounds.right) * \
+                              np.abs(x.bounds.top - x.bounds.bottom)
+        key_sort_func = lambda x: x['size']
+        datasets = [rasterio.open(ds) for ds in inundation_rsts]
+        ds_dict = [{'dataset': ds, 'size': size_func(ds)} for ds in datasets]
+        ds_dict.sort(key=key_sort_func, reverse=True)
+
+        # load sample overlapping inundation depth rasters
+        self.depth_rsts = [x['dataset'] for x in ds_dict]
+        del ds_dict
+
+        self.rst_dims = [[x.height, x.width] for x in self.depth_rsts]
+
+        self.res = self.depth_rsts[0].meta['transform'][0]
+        self.depth_bounds = np.array([[[x.bounds.top,
+                                        x.bounds.left],
+                                       [x.bounds.bottom,
+                                        x.bounds.right]] for x in self.depth_rsts]) / self.res
+
+
+        # get transform, width, height and bounds
+        self.proc_unit_transform, self.proc_unit_width, \
+        self.proc_unit_height, final_bounds = \
+        self.get_final_dims()
+
+        self.proc_unit_bounds = np.array([[final_bounds['top'],
+                                           final_bounds['left']],
+                                          [final_bounds['bottom'],
+                                           final_bounds['right']]])
+
+        self.proc_unit_bounds = self.proc_unit_bounds / self.res
+
+        self.lat_lon_sign = [np.sign(self.proc_unit_bounds[1, 0] - self.proc_unit_bounds[0, 0]),
+                             np.sign(self.proc_unit_bounds[1, 1] - self.proc_unit_bounds[0, 1])]
+
+        self.partitions = num_partitions
+        self.window_sizes = window_xy_size
+
+    @staticmethod
+    @njit
+    def get_res_bbox_min(x, v, z, y):
+        """
+        Optimize for bounds that fit the final resolution
+
+        :param x: float of compare
+        :param v: float representing min bound
+        :param z: float representing max bound
+        :param y: float representing resolution
+        """
+        return np.abs(z - x) - np.round(np.abs(z - v) / y) * y
+
+    def get_final_dims(self):
+        """
+        Get transform, width, height, and bbox of final dataset
+
+        :return: Affine transform, int width, int height, dict bounds
+        """
+
+        left = np.min([d.bounds.left for d in self.depth_rsts])
+        top = np.max([d.bounds.top for d in self.depth_rsts])
+        right = np.max([d.bounds.right for d in self.depth_rsts])
+        bottom = np.min([d.bounds.bottom for d in self.depth_rsts])
+
+        left = newton(self.get_res_bbox_min, left, args=(left, right, self.res))
+        bottom = newton(self.get_res_bbox_min, bottom, args=(bottom, top, self.res))
+
+        transform = self.depth_rsts[0].meta['transform']
+
+        width = int(np.abs(right - left) / self.res)
+        height = int(np.abs(top - bottom) / self.res)
+        new_transform = Affine(transform[0],
+                               transform[1],
+                               left,
+                               transform[3],
+                               transform[4],
+                               top)
+
+        return new_transform, width, height, {'left': left,
+                                              'top': top,
+                                              'right': right,
+                                              'bottom': bottom}
+
+    def get_window_coords(self):
+        """
+        Return ul/br bounds of window and its respective window idx
+
+        :param partitions: tuple or list of partition sizes for x and y
+        :param sizes: tuple or list of pixel sizes for x and y
+        :return: list of ul/br bounds of window, int of respective window idx
+        """
+
+        # Set up desired number of partitions (can also be set pixel size)
+        if self.partitions is not None:
+            x_res, y_res = self.partitions
+        elif self.window_sizes is not None:
+            x_res, y_res = self.window_sizes
+        else:
+            raise('in bran crunch')
+
+        # Get window widths (both normal and edge windows)
+        window_width1 = np.repeat(int(self.proc_unit_width / x_res), x_res) * self.lat_lon_sign[1]
+        window_width2 = window_width1.copy()
+        window_width2[-1] += self.proc_unit_width - window_width1[0] * x_res * self.lat_lon_sign[1]
+
+        # Get window heights (both normal and edge windows)
+        window_height1 = np.repeat(int(self.proc_unit_height / y_res), y_res) * self.lat_lon_sign[0]
+        window_height2 = window_height1.copy()
+        window_height2[-1] += self.proc_unit_height - window_height1[0] * y_res * self.lat_lon_sign[0]
+
+        # Get window sizes (both normal and edge windows)
+        window_bounds1 = np.flip(np.array(np.meshgrid(window_width1,
+                                                      window_height1)).T.reshape(-1, 2),
+                               axis=1).astype(np.int)
+        window_bounds2 = np.flip(np.array(np.meshgrid(window_width2,
+                                                      window_height2)).T.reshape(-1, 2),
+                               axis=1).astype(np.int)
+
+        window_idx = np.array(np.unravel_index(np.arange(y_res * x_res), (y_res, x_res), order='F'))
+
+        return [window_bounds1, window_bounds2], window_idx
+
+    def create_lat_lons(self,
+                        window_bounds,
+                        window_idx):
+        """
+        Return bbox of window and list of latitudes and longitudes
+
+        :param window_bounds: tuple or list of partition sizes for x and y
+        :param window_idx: int representing index of window
+        :return: list of float latitudes, list of float longitudes, list of window bbox, list of ul/br coords for window
+        """
+
+        upper_left = (window_idx.T * window_bounds[0])
+        lower_right = upper_left + window_bounds[1]
+
+        # Merge point arrays, convert back to original units, and get drawable path for each window
+        bbox = np.hstack([upper_left, lower_right])
+        scaled_path_points = [np.array(np.meshgrid([st[0], st[2]], [st[1], st[3]])).T.reshape(-1, 2) for st in bbox]
+        path_points = (scaled_path_points + self.proc_unit_bounds[0]) * self.res
+
+        # Create arange of latitudes and longitudes and add half of window size
+        latitudes = np.arange(self.proc_unit_bounds[0, 0],
+                              self.proc_unit_bounds[1, 0] + self.lat_lon_sign[0],
+                              window_bounds[1][0][0])[:-1] + (window_bounds[1][0][0] / 2)
+        longitudes = np.arange(self.proc_unit_bounds[0, 1],
+                               self.proc_unit_bounds[1, 1] + self.lat_lon_sign[1],
+                               window_bounds[1][0][1])[:-1] + (window_bounds[1][0][1] / 2)
+
+        return latitudes, longitudes, path_points, bbox
+
+    @staticmethod
+    def get_window_idx(latitudes,
+                       longitudes,
+                       coords,
+                       partitions):
+        """
+        Return raveled window indices
+
+        :param latitudes: list of latitudes within bounds
+        :param longitudes: list of longitudes within bounds
+
+        :return: ndarray of raveled multi indexes
+        """
+        # Get difference of upper-left and lower-right boundaries and computed lat lons
+        lat_dif = [np.abs(latitudes - coords[0, 0]), np.abs(latitudes - coords[1, 0])]
+        lon_dif = [np.abs(longitudes - coords[0, 1]), np.abs(longitudes - coords[1, 1])]
+
+        # Create range between the closest idx for both lats and lons
+        lon_range = np.arange(np.argmin(lon_dif[0]), np.argmin(lon_dif[1]) + 1)
+        lat_range = np.arange(np.argmin(lat_dif[0]), np.argmin(lat_dif[1]) + 1)
+
+        # Create mesh grid for each possible set of coords and ravel to get window idx
+        grid = np.array(np.meshgrid(lat_range, lon_range)).T.reshape(-1, 2)
+        del lon_range, lat_range, lat_dif, lon_dif
+        return np.ravel_multi_index([grid[:, 0], grid[:, 1]], partitions, order='F')
+
+    def read_rst_data(self,
+                      win_idx,
+                      datasets,
+                      path_points,
+                      bbox,
+                      meta):
+        """
+        Return data windows and final bounds of window
+
+        :param win_idx: int window index
+        :param datasets: list of int representing dataset inx
+        :param path_points: list of bbox for windows
+        :param bbox: list of ul/br coords of windows
+        :param meta: metadata for final dataset
+
+        :return: rasterio window object for final window, rasterio window of data window bounds,
+        data for each raster in window,
+        """
+        # Get window bounding box and get final array output dimensions
+        window = path_points[win_idx]
+        window_height, window_width = np.array([np.abs(bbox[win_idx][2] - bbox[win_idx][0]),
+                                                np.abs(bbox[win_idx][3] - bbox[win_idx][1])]).astype(np.int)
+
+        bnds = []
+        data = []
+        for ds in datasets:
+            # Get rasterio window for each pair of window bounds and depth dataset
+
+            bnd = from_bounds(window[0][1],
+                              window[-1][0],
+                              window[-1][1],
+                              window[0][0],
+                              transform=self.depth_rsts[ds].transform,
+                              height=window_height,
+                              width=window_width)
+
+            bnds.append(bnd)
+
+            # Read raster data with window
+            read_data = self.depth_rsts[ds].read(1, window=bnd).astype(np.float32)
+            # Convert all no data to nan values
+            read_data[read_data == np.float32(self.depth_rsts[ds].meta['nodata'])] = np.nan
+            data.append(read_data)
+            del bnd
+
+        final_bnds = from_bounds(window[0][1],
+                                 window[-1][0],
+                                 window[-1][1],
+                                 window[0][0],
+                                 transform=meta['transform'],
+                                 height=window_height,
+                                 width=window_width)
+
+        return [final_bnds, bnds, data]
+
+
+    def merge_rasters(self, out_fname, nodata=-9999, threaded=False, workers=4):
+        """
+        Merge multiple raster datasets
+
+        :param out_fname: str path for final merged dataset
+        :param nodata: int/float representing no data value
+        """
+
+        window_bounds, window_idx = self.get_window_coords()
+        latitudes, longitudes, path_points, bbox = self.create_lat_lons(window_bounds,
+                                                                        window_idx)
+        
+        windows = [self.get_window_idx(latitudes,
+                                       longitudes,
+                                       coords,
+                                       self.partitions)
+                   for coords in self.depth_bounds]
+
+        # Create dict with window idx key and dataset idx vals
+        data_dict = {}
+        for idx, win in enumerate(windows):
+            for win_idx in win:
+                if win_idx in data_dict:
+                    data_dict[win_idx].append(idx)
+                else:
+                    data_dict[win_idx] = [idx]
+
+        agg_function = partial(np.nanmax, axis=0)
+
+        meta = self.depth_rsts[0].meta
+
+        meta.update(transform=self.proc_unit_transform,
+                    width=self.proc_unit_width,
+                    height=self.proc_unit_height,
+                    nodata=nodata,blockxsize=256,
+                    blockysize=256, tiled=True,
+                    compress='lzw')
+
+        final_windows, data_windows, data = [], [], []
+
+        def __data_generator(data_dict,path_points,bbox,meta):
+
+            for key, val in data_dict.items():
+
+                f_window, window, dat = self.read_rst_data(key,
+                                                           val,
+                                                           path_points,
+                                                           bbox,
+                                                           meta
+                                                          )
+                yield(dat, window, f_window, val)
+                #final_windows.append(f_window)
+                #data_windows.append(window)
+                #data.append(dat)
+                #del f_window, window, dat
+
+        # create data generator
+        dgen = __data_generator(data_dict,path_points,bbox,meta)
+        
+        lock = Lock()
+
+        with rasterio.open(out_fname, 'w', **meta) as rst:
+
+            merge_partial = partial(merge_data,
+                             rst=rst,
+                             lock=lock,
+                             dtype=meta['dtype'],
+                             agg_function=agg_function,
+                             nodata=meta['nodata'],
+                             rst_dims=self.rst_dims)
+            
+            if not threaded:
+                #for d, dw, fw, ddict in zip(data,
+                #                            data_windows,
+                #                            final_windows,
+                #                            data_dict.values()):
+                for d, dw, fw, ddict in dgen:
+                    merge_partial(d, dw, fw, ddict)
+            else:
+                with concurrent.futures.ThreadPoolExecutor(
+                        max_workers=workers
+                ) as executor:
+                    executor.map(merge_partial,
+                                 data,
+                                 data_windows,
+                                 final_windows,
+                                 data_dict.values()
+                                 )
+
+    def mask_mosaic(self,mosaic,polys,polys_layer=None,outfile=None):
+        
+        #rem_array,window_transform = mask(rem,[shape(huc['geometry'])],crop=True,indexes=1)
+        
+        # input rem
+        if isinstance(mosaic,str):
+            mosaic = rasterio.open(mosaic)
+        elif isinstance(mosaic,rasterio.DatasetReader):
+            pass
+        else:
+            raise TypeError("Pass rasterio dataset or filepath for mosaic")
+
+        if isinstance(polys,str):
+            polys=gpd.read_file(polys,layer=polys_layer)
+        elif isinstance(polys,gpd.GeoDataFrame):
+            pass
+        else:
+            raise TypeError("Pass geopandas dataset or filepath for catchment polygons")
+
+        #fossid = huc['properties']['fossid']
+        #if polys.HydroID.dtype != 'str': polys.HydroID = polys.HydroID.astype(str)
+        #polys=polys[polys.HydroID.str.startswith(fossid)]
+        mosaic_array, window_transform = mask(mosaic,polys['geometry'],crop=True,indexes=1)
+        
+        if outfile:
+            out_profile = mosaic.profile
+            out_profile.update(height=mosaic_array.shape[0],width=mosaic_array.shape[1],
+                                      transform = window_transform, driver= 'GTiff',
+                                      blockxsize=256, blockysize=256, tiled=True, compress='lzw')
+
+            with rasterio.open(outfile,'w',**out_profile) as otfi:
+                otfi.write(mosaic_array,indexes=1)
+         
+        return(mosaic_array,out_profile)
+
+# Quasi multi write
+# Throughput achieved assuming processing time is not identical between windows
+# and queued datasets, preferably approx N/2 threads for 9 windows
+# @njit
+def merge_data(rst_data,
+               window_bnds,
+               final_window,
+               datasets,
+               dtype,
+               rst,
+               lock,
+               agg_function,
+               nodata,
+               rst_dims
+               ):
+    """
+    Merge data in to final dataset (multi threaded)
+
+    :param rst_data: list of rst data from window
+    :param window_bnds: list rasterio windows representing data window bounds
+    :param final_window: rasterio window representing final window bounds
+    :param datasets: list of int representing dataset idx
+    :param dtype: data type of final output
+    :param rst: rasterio writer for final dataset
+    :param lock: thread concurrency lock
+    :param agg_function: function to aggregate datasets
+    :param nodata: nodata of final output
+    :param rst_dims: dimensions of overlapping rasters
+    """
+
+    nan_tile = np.array([np.nan]).astype(dtype)[0]
+    window_data = np.tile(float(nan_tile), [int(final_window.height), int(final_window.width)])
+
+    for data, bnds, idx in zip(rst_data, window_bnds, datasets):
+        # Get indices to apply to base
+
+        col_slice = slice(int(np.max([0,
+                                      np.ceil(bnds.col_off * -1)])),
+                          int(np.min([bnds.width,
+                                      rst_dims[idx][1] - bnds.col_off])))
+
+        row_slice = slice(int(np.max([0,
+                                      np.ceil(bnds.row_off * -1)])),
+                          int(np.min([bnds.height,
+                                      rst_dims[idx][0] - bnds.row_off])))
+
+        win_shape = window_data[row_slice, col_slice].shape
+
+        if not np.all(np.sign(np.array(win_shape) - np.array(data.shape)) > 0):
+            data = data[:win_shape[0], :win_shape[1]]
+        # Assign the data to the base array with aggregate function
+        merge = [window_data[row_slice,
+                             col_slice],
+                 data]
+
+        del data
+        with warnings.catch_warnings():
+            # This `with` block supresses the RuntimeWarning thrown by numpy when aggregating nan values
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            window_data[row_slice, col_slice] = agg_function(merge)
+        window_data[np.isnan(window_data)] = nodata
+        del merge
+
+    del rst_data, window_bnds, datasets
+
+    window_data[(window_data == nan_tile) | (np.isnan(window_data))] = nodata
+
+    with lock:
+        rst.write_band(1, window_data.astype(dtype), window=final_window)
+    del window_data
+
+
+if __name__ == '__main__':
+    import time
+    # import tracemalloc
+    import glob
+
+    # print('start', time.localtime())
+    # project_path = r'../documentation/data'
+    # overlap = OverlapWindowMerge([project_path + '/overlap1.tif',
+    #                               project_path + '/overlap2.tif',
+    #                               project_path + '/overlap3.tif',
+    #                              ],
+    #                              (3, 3))
+    # overlap.merge_rasters(project_path + '/merged_overlap.tif', nodata=0)
+    # print('end', time.localtime())
+
+    # tracemalloc.start()
+    print('start', time.localtime())
+    # project_path = r'../documentation/data'
+    # project_path = '*/mosaicing_data/1_fr_ms_composite'
+    # overlap = OverlapWindowMerge([project_path + '/inundation_extent_12090301_FR.tif',
+    #                               project_path + '/inundation_extent_12090301_MS.tif'
+    #                              ],
+    #                              (30, 30))
+    # overlap.merge_rasters(project_path + '/merged_final5.tif', threaded=True, workers=4, nodata=0)
+
+    # tracemalloc.start()
+    print('start', time.localtime())
+    # project_path = r'../documentation/data'
+    # project_path = '*/mosaicing_data/2_gms'
+    # a = glob.glob(project_path + '/inundation*.tif')
+    # overlap = OverlapWindowMerge(a,
+    #                              (30, 30))
+    # overlap.merge_rasters(project_path + '/merged_final5.tif', threaded=True, workers=4, nodata=-2e9)
+    # current, peak = tracemalloc.get_traced_memory()
+    # print(f"Current memory usage is {current / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB")
+    # tracemalloc.stop()
+
+    project_path = '*'
+    overlap = OverlapWindowMerge([project_path + '/nwm_resampled.tif',
+                                  project_path + '/rnr_inundation_031403_2020092000.tif'
+                                 ],
+                                 (1, 1))
+    overlap.merge_rasters(project_path + '/merged_final5.tif', threaded=False, workers=4)
+
+    print('end', time.localtime())
\ No newline at end of file
diff --git a/tools/inundation.py b/tools/inundation.py
index fec6e4cfc..2b5a4599c 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -18,6 +18,14 @@
 from gdal import BuildVRT
 import geopandas as gpd
 
+class hydroTableHasOnlyLakes(Exception): 
+    """ Raised when a Hydro-Table only has lakes """
+    pass
+
+
+class NoForecastFound(Exception):
+    """ Raised when no forecast is available for a given Hydro-Table """
+    pass
 
 def inundate(
              rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=None,hucs_layerName=None,
@@ -333,7 +341,7 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
     if isinstance(depths,DatasetWriter): depths.close()
     if isinstance(inundation_raster,DatasetWriter): inundation_raster.close()
     if isinstance(inundation_polygon,fiona.Collection): inundation_polygon.close()
-    if isinstance(hucs,fiona.Collection): inundation_polygon.close()
+    #if isinstance(hucs,fiona.Collection): inundation_polygon.close()
 
     # return file names of outputs for aggregation. Handle Nones
     try:
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index f398a7a17..6b7046770 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -8,9 +8,16 @@
 from tools_shared_functions import compute_contingency_stats_from_rasters
 from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
 from inundation import inundate
-
-
-def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True):
+from gms_tools.inundate_gms import Inundate_gms
+from gms_tools.mosaic_inundation import Mosaic_inundation
+from gms_tools.overlapping_inundation import OverlapWindowMerge
+
+def run_alpha_test(fim_run_dir, version, test_id, magnitude, 
+                   #
+                   compare_to_previous=False, archive_results=False, 
+                   mask_type='huc', inclusion_area='', 
+                   inclusion_area_buffer=0, light_run=False, 
+                   overwrite=True):
 
     benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
     current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
diff --git a/tools/run_test_case_gms.py b/tools/run_test_case_gms.py
new file mode 100644
index 000000000..0c5d5210f
--- /dev/null
+++ b/tools/run_test_case_gms.py
@@ -0,0 +1,376 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import shutil
+import argparse
+import traceback
+from pathlib import Path
+import json
+import ast
+import pandas as pd
+
+from tools_shared_functions import compute_contingency_stats_from_rasters
+from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES, IFC_MAGNITUDE_LIST, BLE_MAGNITUDE_LIST )
+from inundation import inundate
+from gms_tools.inundate_gms import Inundate_gms
+from gms_tools.mosaic_inundation import Mosaic_inundation
+from gms_tools.overlapping_inundation import OverlapWindowMerge
+from glob import glob
+from utils.shared_variables import elev_raster_ndv
+
+def run_alpha_test( fim_run_dir, version, test_id, magnitude, 
+                    calibrated, model,
+                    compare_to_previous=False, archive_results=False, 
+                    mask_type='filter', inclusion_area='', 
+                    inclusion_area_buffer=0, light_run=False, 
+                    overwrite=True, fr_run_dir=None, 
+                    gms_workers=1,verbose=False,
+                    gms_verbose=False
+                  ):
+
+    # check eval_meta input
+    if model not in {None,'FR','MS','GMS'}:
+        raise ValueError("Model argument needs to be \'FR\', \'MS\', or \'GMS.\'")
+
+   # make bool
+    calibrated = bool( calibrated )
+
+    if (model == "MS") & (fr_run_dir is None):
+        raise ValueError("fr_run_dir argument needs to be specified with MS model")
+
+    benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
+    current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
+    
+    # Construct paths to development test results if not existent.
+    if archive_results:
+        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'official_versions', version)
+    else:
+        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'testing_versions', version)
+
+    # Delete the entire directory if it already exists.
+    if os.path.exists(version_test_case_dir_parent):
+        if overwrite == True:
+            shutil.rmtree(version_test_case_dir_parent)
+        elif model == 'MS':
+            pass
+        else:
+            print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
+            return
+
+    os.makedirs(version_test_case_dir_parent,exist_ok=True)
+
+    __vprint("Running the alpha test for test_id: " + test_id + ", " + version + "...",verbose)
+    stats_modes_list = ['total_area']
+
+    fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
+    assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
+
+    # get hydrofabric directory
+    hydrofabric_dir = Path(fim_run_parent).parent.absolute()
+
+    # Create paths to fim_run outputs for use in inundate().
+    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
+    if not os.path.exists(rem):
+        rem = os.path.join(fim_run_parent, 'rem_clipped_zeroed_masked.tif')
+    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+    if not os.path.exists(catchments):
+        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_clipped_addedAttributes.tif')
+    if mask_type == 'huc':
+        catchment_poly = ''
+    else:
+        catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
+
+    # Map necessary inputs for inundation().
+    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+    
+    # Create list of shapefile paths to use as exclusion areas.
+    zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
+    mask_dict = {'levees':
+                    {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
+                     'buffer': None,
+                     'operation': 'exclude'
+                     },
+                'waterbodies':
+                    {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
+                     'buffer': None,
+                     'operation': 'exclude',
+                     },
+                }
+
+    if inclusion_area != '':
+        inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
+        mask_dict.update({inclusion_area_name: {'path': inclusion_area,
+                                                'buffer': int(inclusion_area_buffer),
+                                                'operation': 'include'}})
+        # Append the concatenated inclusion_area_name and buffer.
+        if inclusion_area_buffer == None:
+            inclusion_area_buffer = 0
+        stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm')
+
+    # Check if magnitude is list of magnitudes or single value.
+    magnitude_list = magnitude
+    if type(magnitude_list) != list:
+        magnitude_list = [magnitude_list]
+        
+
+    # Get path to validation_data_{benchmark} directory and huc_dir.
+    validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
+    for magnitude in magnitude_list:
+        version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
+        if not os.path.exists(version_test_case_dir):
+            os.mkdir(version_test_case_dir)
+        # Construct path to validation raster and forecast file.
+        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+            benchmark_raster_path_list, forecast_list = [], []
+            lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
+            lid_list, inundation_raster_list, domain_file_list = [], [], []
+
+            for lid in lid_dir_list:
+                lid_dir = os.path.join(validation_data_path, current_huc, lid)
+                benchmark_lid_raster_path = os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
+                
+                # Only compare if the benchmark data exist.
+                if os.path.exists(benchmark_lid_raster_path):
+                    benchmark_raster_path_list.append(benchmark_lid_raster_path)  # TEMP
+                    forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
+                    lid_list.append(lid)
+                    inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
+                    domain_file_list.append(os.path.join(lid_dir, lid + '_domain.shp'))
+                    
+        else:
+            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
+            benchmark_raster_path_list = [benchmark_raster_file]
+            forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
+            forecast_list = [forecast_path]
+            inundation_raster_list = [os.path.join(version_test_case_dir, 'inundation_extent.tif')]
+
+        for index in range(0, len(benchmark_raster_path_list)):
+            benchmark_raster_path = benchmark_raster_path_list[index]
+            forecast = forecast_list[index]
+            inundation_raster = inundation_raster_list[index]
+            # Only need to define ahps_lid and ahps_extent_file for AHPS_BENCHMARK_CATEGORIES.
+            if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                ahps_lid = lid_list[index]
+                ahps_domain_file = domain_file_list[index]
+                mask_dict.update({ahps_lid:
+                    {'path': ahps_domain_file,
+                     'buffer': None,
+                     'operation': 'include'}
+                        })
+
+                
+                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_domain_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
+                    continue
+            else:  # If not in AHPS_BENCHMARK_CATEGORIES.
+                if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
+                    continue
+            # Run inundate.
+            __vprint("-----> Running inundate() to produce inundation extent for the " + magnitude + " magnitude...",verbose)
+            # The inundate adds the huc to the name so I account for that here.
+            predicted_raster_path = os.path.join(
+                                        os.path.split(inundation_raster)[0], 
+                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
+                                                )  
+            try:
+                if model == 'GMS':
+                    
+                    map_file = Inundate_gms(
+                                             hydrofabric_dir=hydrofabric_dir, 
+                                             forecast=forecast, 
+                                             num_workers=gms_workers,
+                                             hucs=current_huc,
+                                             inundation_raster=inundation_raster,
+                                             inundation_polygon=None, depths_raster=None,
+                                             verbose=gms_verbose,
+                                             log_file=None,
+                                             output_fileNames=None
+                                            )
+                    
+                    mask_path_gms = os.path.join(fim_run_parent, 'wbd.gpkg')
+
+                    Mosaic_inundation(
+                                        map_file,mosaic_attribute='inundation_rasters',
+                                        mosaic_output=inundation_raster,
+                                        mask=mask_path_gms,unit_attribute_name='huc8',
+                                        nodata=elev_raster_ndv,workers=1,
+                                        remove_inputs=True,
+                                        subset=None,verbose=verbose
+                                      )
+                
+                else:
+                    inundate(
+                             rem,catchments,catchment_poly,hydro_table,forecast,
+                             mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                             subset_hucs=current_huc,num_workers=1,aggregate=False,
+                             inundation_raster=inundation_raster,inundation_polygon=None,
+                             depths=None,out_raster_profile=None,out_vector_profile=None,
+                             quiet=True
+                        )
+
+                if model =='MS':
+                    
+                    # Mainstems inundation
+                    #fr_run_parent = os.path.join(os.environ['outputDataDir'], fr_run_dir,current_huc)
+                    #assert os.path.exists(fr_run_parent), "Cannot locate " + fr_run_parent
+                    
+                    inundation_raster_ms = os.path.join(
+                                        os.path.split(inundation_raster)[0], 
+                                        os.path.split(inundation_raster)[1].replace('.tif', '_{}_MS.tif'.format(current_huc))
+                                           )  
+                    inundation_raster_fr = os.path.join(
+                                        os.path.split(version_test_case_dir_parent)[0],
+                                        fr_run_dir,
+                                        magnitude,
+                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
+                                           )  
+                    
+                    os.rename(predicted_raster_path,inundation_raster_ms)
+
+                    ms_inundation_map_file = { 
+                                               'huc8' : [current_huc] * 2,
+                                               'branchID' : [None] * 2,
+                                               'inundation_rasters' : [inundation_raster_fr,inundation_raster_ms],
+                                               'depths_rasters' : [None] * 2,
+                                               'inundation_polygons' : [None] * 2
+                                             }
+                    ms_inundation_map_file = pd.DataFrame(ms_inundation_map_file)
+                    
+                    Mosaic_inundation(
+                                        ms_inundation_map_file,mosaic_attribute='inundation_rasters',
+                                        mosaic_output=inundation_raster,
+                                        mask=catchment_poly,unit_attribute_name='huc8',
+                                        nodata=elev_raster_ndv,workers=1,
+                                        remove_inputs=False,
+                                        subset=None,verbose=verbose
+                                      )
+
+                __vprint("-----> Inundation mapping complete.",verbose)
+
+                # Define outputs for agreement_raster, stats_json, and stats_csv.
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+                else:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+                
+                compute_contingency_stats_from_rasters(predicted_raster_path,
+                                                       benchmark_raster_path,
+                                                       agreement_raster,
+                                                       stats_csv=stats_csv,
+                                                       stats_json=stats_json,
+                                                       mask_values=[],
+                                                       stats_modes_list=stats_modes_list,
+                                                       test_id=test_id,
+                                                       mask_dict=mask_dict,
+                                                       )
+
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    del mask_dict[ahps_lid]
+
+                __vprint(" ",verbose)
+                # print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC) # GMS
+                __vprint("Evaluation metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC,verbose) # cahaba/dev
+                __vprint(" ",verbose)
+
+            except Exception as e:
+                print(traceback.print_exc())
+                #print(e)
+
+        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+            # -- Delete temp files -- #
+            # List all files in the output directory.
+            output_file_list = os.listdir(version_test_case_dir)
+            for output_file in output_file_list:
+                if "total_area" in output_file:
+                    full_output_file_path = os.path.join(version_test_case_dir, output_file)
+                    os.remove(full_output_file_path)
+
+    # write out evaluation meta-data
+    with open(os.path.join(version_test_case_dir_parent,'eval_metadata.json'),'w') as meta:
+        eval_meta = { 'calibrated' : calibrated , 'model' : model }
+        meta.write( 
+                    json.dumps(eval_meta,indent=2) 
+                   )
+
+def __vprint(message,verbose):
+    if verbose:
+        print(message)
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
+    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
+    parser.add_argument('-b', '--version',help='The name of the working version in which features are being tested',required=True,default="")
+    parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
+    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method. MS and GMS are currently on supporting huc', required=False,default="filter")
+    parser.add_argument('-n','--calibrated',help='Denotes use of calibrated n values',required=False, default=False,action='store_true')
+    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed',required=True)
+    parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
+    parser.add_argument('-c', '--compare-to-previous', help='Compare to previous versions of HAND.', required=False,action='store_true')
+    parser.add_argument('-a', '--archive-results', help='Automatically copy results to the "previous_version" archive for test_id. For admin use only.', required=False,action='store_true')
+    parser.add_argument('-i', '--inclusion-area', help='Path to shapefile. Contingency metrics will be produced from pixels inside of shapefile extent.', required=False, default="")
+    parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
+    parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
+    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False, action='store_true')
+    parser.add_argument('-w','--gms-workers', help='Number of workers to use for GMS Branch Inundation', required=False, default=1)
+    parser.add_argument('-d','--fr-run-dir',help='Name of test case directory containing inundation for FR configuration',required=False,default=None)
+    parser.add_argument('-v', '--verbose', help='Verbose operation', required=False, action='store_true', default=False)
+    parser.add_argument('-vg', '--gms-verbose', help='Prints progress bar for GMS', required=False, action='store_true', default=False)
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    valid_test_id_list = os.listdir(TEST_CASES_DIR)
+
+    exit_flag = False  # Default to False.
+    __vprint("",args['verbose'])
+
+    # Ensure test_id is valid.
+#    if args['test_id'] not in valid_test_id_list:
+#        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
+#        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
+#        for test_id in valid_test_id_list:
+#          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
+#              print(CYAN_BOLD + test_id + ENDC)
+#        print()
+#        exit_flag = True
+
+    # Ensure fim_run_dir exists.
+    if not os.path.exists(os.path.join(os.environ['outputDataDir'], args['fim_run_dir'])):
+        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
+        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
+        print()
+        exit_flag = True
+
+    # Ensure inclusion_area path exists.
+    if args['inclusion_area'] != "" and not os.path.exists(args['inclusion_area']):
+        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area (-i) " + CYAN_BOLD + args['inclusion_area'] + WHITE_BOLD + " could not be located." + ENDC)
+        exit_flag = True
+
+    try:
+        inclusion_buffer = int(args['inclusion_area_buffer'])
+    except ValueError:
+        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area_buffer (-ib) " + CYAN_BOLD + args['inclusion_area_buffer'] + WHITE_BOLD + " is not a round number." + ENDC)
+
+    benchmark_category = args['test_id'].split('_')[1]
+
+    if args['magnitude'] == '':
+        if 'ble' == benchmark_category:
+            args['magnitude'] = BLE_MAGNITUDE_LIST
+        elif ('nws' == benchmark_category) | ('usgs' == benchmark_category):
+            args['magnitude'] = ['action', 'minor', 'moderate', 'major']
+        elif 'ifc' == current_benchmark_category:
+            args['magnitude'] = IFC_MAGNITUDE_LIST
+        else:
+            print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
+            exit_flag = True
+
+    if exit_flag:
+        print()
+        sys.exit()
+
+    else:
+        run_alpha_test(**args)
\ No newline at end of file
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index 04356c3e7..0c31a6fb8 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -50,6 +50,8 @@
             ]
 DISCARD_AHPS_QUERY = "not flow.isnull() & masked_perc<97 & not nws_lid in @BAD_SITES"
 
+elev_raster_ndv = -9999.0
+
 # Colors.
 ENDC = '\033[m'
 TGREEN_BOLD = '\033[32;1m'

From b4e8682624bdedf58c2bd4dbcccd03bb9b7ed1ac Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Thu, 18 Nov 2021 09:11:00 -0600
Subject: [PATCH 122/359] Patch for inundation composite

See Changelog for version 3.0.24.1 for full details.
---
 docs/CHANGELOG.md                   |  15 ++
 tools/composite_ms_fr_inundation.py |  26 +-
 tools/run_test_case.py              |  12 +-
 tools/run_test_case_gms.py          | 376 ----------------------------
 4 files changed, 33 insertions(+), 396 deletions(-)
 delete mode 100644 tools/run_test_case_gms.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8a0de7039..c187a71a0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.24.1 - 2021-11-17 - [PR #484](https://github.com/NOAA-OWP/cahaba/pull/484)
+
+Patch to clean up unnecessary files and create better names for intermediate raster files.
+
+## Removals
+
+- `tools/run_test_case_gms.py`: Unnecessary file.
+
+## Changes
+
+- `tools/composite_ms_fr_inundation.py`: Clean up documentation and intermediate file names.
+- `tools/run_test_case.py`: Remove unnecessary imports.
+
+<br/><br/>
+
 ## v3.0.24.0 - 2021-11-08 - [PR #482](https://github.com/NOAA-OWP/cahaba/pull/482)
 
 Adds `composite_ms_fr_inundation.py` to allow for the generation of an inundation map given a "flow file" CSV and full-resolution (FR) and mainstem (MS) relative elevation models, synthetic rating curves, and catchments rasters created by the `fim_run.sh` script.
diff --git a/tools/composite_ms_fr_inundation.py b/tools/composite_ms_fr_inundation.py
index 1b975a8b9..e0153e143 100644
--- a/tools/composite_ms_fr_inundation.py
+++ b/tools/composite_ms_fr_inundation.py
@@ -2,7 +2,6 @@
 import os, argparse, rasterio
 import numpy as np
 import pandas as pd
-from multiprocessing import Pool
 
 from inundation import inundate
 from gms_tools.mosaic_inundation import Mosaic_inundation, __append_id_to_file_name
@@ -11,7 +10,7 @@
 def composite_inundation(fim_dir_ms, fim_dir_fr, huc, flows, composite_output_dir, ouput_name='',
                          bin_rast_flag=False, depth_rast_flag=False, clean=True, quiet=True):
     """
-    Run `inundate()` on FIM 3.X mainstem (MS) and full-resolution (FR) outputs and composite results. Assumes that all `fim_run` products 
+    Runs `inundate()` on FIM 3.X mainstem (MS) and full-resolution (FR) outputs and composites results. Assumes that all `fim_run` products 
     necessary for `inundate()` are in each huc8 folder.
 
     Parameters
@@ -61,6 +60,7 @@ def composite_inundation(fim_dir_ms, fim_dir_fr, huc, flows, composite_output_di
             '12090301',
             '/home/user/forecast_file.csv',
             '/home/user/fim_inundation_composite',
+            'inundation_composite.tif',
             True,
             False)
     """
@@ -77,15 +77,15 @@ def composite_inundation(fim_dir_ms, fim_dir_fr, huc, flows, composite_output_di
         'ms': {
             'dir': fim_dir_ms,
             'outputs': {
-                'inundation_rast': os.path.join(composite_output_dir, f'inundation_ms_{huc}.tif') if bin_rast_flag else None,
-                'depth_rast':      os.path.join(composite_output_dir, f'depth_ms_{huc}.tif') if depth_rast_flag else None
+                'inundation_rast': os.path.join(composite_output_dir, f'{huc}_inundation_ms.tif') if bin_rast_flag else None,
+                'depth_rast':      os.path.join(composite_output_dir, f'{huc}_depth_ms.tif') if depth_rast_flag else None
             }
         },
         'fr': {
             'dir': fim_dir_fr,
             'outputs': {
-                'inundation_rast': os.path.join(composite_output_dir, f'inundation_fr_{huc}.tif') if bin_rast_flag else None,
-                'depth_rast':      os.path.join(composite_output_dir, f'depth_fr_{huc}.tif') if depth_rast_flag else None
+                'inundation_rast': os.path.join(composite_output_dir, f'{huc}_inundation_fr.tif') if bin_rast_flag else None,
+                'depth_rast':      os.path.join(composite_output_dir, f'{huc}_depth_fr.tif') if depth_rast_flag else None
             }
         }
     }
@@ -104,7 +104,8 @@ def composite_inundation(fim_dir_ms, fim_dir_fr, huc, flows, composite_output_di
 
         # Run inundation()
         extent_friendly = "mainstem (MS)" if extent=="ms" else "full-resolution (FR)"
-        if not quiet: print(f"  Creating an inundation map for the {extent_friendly} configuration...")
+        grid_type = "an inundation" if bin_rast_flag else "a depth"
+        if not quiet: print(f"  Creating {grid_type} map for the {extent_friendly} configuration...")
         result = inundate(rem,catchments,catchment_poly,hydro_table,flows,mask_type=None,
                 inundation_raster=  var_keeper[extent]['outputs']['inundation_rast'],
                 depths=             var_keeper[extent]['outputs']['depth_rast'],
@@ -202,6 +203,11 @@ def hydroid_to_binary(hydroid_raster_filename):
         arg_list.append((fim_dir_ms, fim_dir_fr, huc, flows_file, output_dir, ouput_name, bin_raster, depth_raster, clean, quiet))
 
     # Multi-thread for each huc in input hucs
-    with Pool(processes=num_workers) as pool:
-        # Run composite_inundation()
-        pool.starmap(composite_inundation, arg_list)
+    if num_workers > 1:
+        from multiprocessing import Pool
+        with Pool(processes=num_workers) as pool:
+            # Run composite_inundation()
+            pool.starmap(composite_inundation, arg_list)
+    else: # run linear if jobs == 1
+        for arg in arg_list:
+            composite_inundation(*arg)
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 6b7046770..9d4870565 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -8,16 +8,8 @@
 from tools_shared_functions import compute_contingency_stats_from_rasters
 from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
 from inundation import inundate
-from gms_tools.inundate_gms import Inundate_gms
-from gms_tools.mosaic_inundation import Mosaic_inundation
-from gms_tools.overlapping_inundation import OverlapWindowMerge
-
-def run_alpha_test(fim_run_dir, version, test_id, magnitude, 
-                   #
-                   compare_to_previous=False, archive_results=False, 
-                   mask_type='huc', inclusion_area='', 
-                   inclusion_area_buffer=0, light_run=False, 
-                   overwrite=True):
+
+def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True):
 
     benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
     current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
diff --git a/tools/run_test_case_gms.py b/tools/run_test_case_gms.py
deleted file mode 100644
index 0c5d5210f..000000000
--- a/tools/run_test_case_gms.py
+++ /dev/null
@@ -1,376 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import shutil
-import argparse
-import traceback
-from pathlib import Path
-import json
-import ast
-import pandas as pd
-
-from tools_shared_functions import compute_contingency_stats_from_rasters
-from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES, IFC_MAGNITUDE_LIST, BLE_MAGNITUDE_LIST )
-from inundation import inundate
-from gms_tools.inundate_gms import Inundate_gms
-from gms_tools.mosaic_inundation import Mosaic_inundation
-from gms_tools.overlapping_inundation import OverlapWindowMerge
-from glob import glob
-from utils.shared_variables import elev_raster_ndv
-
-def run_alpha_test( fim_run_dir, version, test_id, magnitude, 
-                    calibrated, model,
-                    compare_to_previous=False, archive_results=False, 
-                    mask_type='filter', inclusion_area='', 
-                    inclusion_area_buffer=0, light_run=False, 
-                    overwrite=True, fr_run_dir=None, 
-                    gms_workers=1,verbose=False,
-                    gms_verbose=False
-                  ):
-
-    # check eval_meta input
-    if model not in {None,'FR','MS','GMS'}:
-        raise ValueError("Model argument needs to be \'FR\', \'MS\', or \'GMS.\'")
-
-   # make bool
-    calibrated = bool( calibrated )
-
-    if (model == "MS") & (fr_run_dir is None):
-        raise ValueError("fr_run_dir argument needs to be specified with MS model")
-
-    benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
-    current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
-    
-    # Construct paths to development test results if not existent.
-    if archive_results:
-        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'official_versions', version)
-    else:
-        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'testing_versions', version)
-
-    # Delete the entire directory if it already exists.
-    if os.path.exists(version_test_case_dir_parent):
-        if overwrite == True:
-            shutil.rmtree(version_test_case_dir_parent)
-        elif model == 'MS':
-            pass
-        else:
-            print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
-            return
-
-    os.makedirs(version_test_case_dir_parent,exist_ok=True)
-
-    __vprint("Running the alpha test for test_id: " + test_id + ", " + version + "...",verbose)
-    stats_modes_list = ['total_area']
-
-    fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
-    assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
-
-    # get hydrofabric directory
-    hydrofabric_dir = Path(fim_run_parent).parent.absolute()
-
-    # Create paths to fim_run outputs for use in inundate().
-    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
-    if not os.path.exists(rem):
-        rem = os.path.join(fim_run_parent, 'rem_clipped_zeroed_masked.tif')
-    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-    if not os.path.exists(catchments):
-        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_clipped_addedAttributes.tif')
-    if mask_type == 'huc':
-        catchment_poly = ''
-    else:
-        catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
-
-    # Map necessary inputs for inundation().
-    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-    
-    # Create list of shapefile paths to use as exclusion areas.
-    zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
-    mask_dict = {'levees':
-                    {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
-                     'buffer': None,
-                     'operation': 'exclude'
-                     },
-                'waterbodies':
-                    {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
-                     'buffer': None,
-                     'operation': 'exclude',
-                     },
-                }
-
-    if inclusion_area != '':
-        inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
-        mask_dict.update({inclusion_area_name: {'path': inclusion_area,
-                                                'buffer': int(inclusion_area_buffer),
-                                                'operation': 'include'}})
-        # Append the concatenated inclusion_area_name and buffer.
-        if inclusion_area_buffer == None:
-            inclusion_area_buffer = 0
-        stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm')
-
-    # Check if magnitude is list of magnitudes or single value.
-    magnitude_list = magnitude
-    if type(magnitude_list) != list:
-        magnitude_list = [magnitude_list]
-        
-
-    # Get path to validation_data_{benchmark} directory and huc_dir.
-    validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
-    for magnitude in magnitude_list:
-        version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
-        if not os.path.exists(version_test_case_dir):
-            os.mkdir(version_test_case_dir)
-        # Construct path to validation raster and forecast file.
-        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-            benchmark_raster_path_list, forecast_list = [], []
-            lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
-            lid_list, inundation_raster_list, domain_file_list = [], [], []
-
-            for lid in lid_dir_list:
-                lid_dir = os.path.join(validation_data_path, current_huc, lid)
-                benchmark_lid_raster_path = os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
-                
-                # Only compare if the benchmark data exist.
-                if os.path.exists(benchmark_lid_raster_path):
-                    benchmark_raster_path_list.append(benchmark_lid_raster_path)  # TEMP
-                    forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
-                    lid_list.append(lid)
-                    inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
-                    domain_file_list.append(os.path.join(lid_dir, lid + '_domain.shp'))
-                    
-        else:
-            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
-            benchmark_raster_path_list = [benchmark_raster_file]
-            forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
-            forecast_list = [forecast_path]
-            inundation_raster_list = [os.path.join(version_test_case_dir, 'inundation_extent.tif')]
-
-        for index in range(0, len(benchmark_raster_path_list)):
-            benchmark_raster_path = benchmark_raster_path_list[index]
-            forecast = forecast_list[index]
-            inundation_raster = inundation_raster_list[index]
-            # Only need to define ahps_lid and ahps_extent_file for AHPS_BENCHMARK_CATEGORIES.
-            if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                ahps_lid = lid_list[index]
-                ahps_domain_file = domain_file_list[index]
-                mask_dict.update({ahps_lid:
-                    {'path': ahps_domain_file,
-                     'buffer': None,
-                     'operation': 'include'}
-                        })
-
-                
-                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_domain_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
-                    continue
-            else:  # If not in AHPS_BENCHMARK_CATEGORIES.
-                if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
-                    continue
-            # Run inundate.
-            __vprint("-----> Running inundate() to produce inundation extent for the " + magnitude + " magnitude...",verbose)
-            # The inundate adds the huc to the name so I account for that here.
-            predicted_raster_path = os.path.join(
-                                        os.path.split(inundation_raster)[0], 
-                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
-                                                )  
-            try:
-                if model == 'GMS':
-                    
-                    map_file = Inundate_gms(
-                                             hydrofabric_dir=hydrofabric_dir, 
-                                             forecast=forecast, 
-                                             num_workers=gms_workers,
-                                             hucs=current_huc,
-                                             inundation_raster=inundation_raster,
-                                             inundation_polygon=None, depths_raster=None,
-                                             verbose=gms_verbose,
-                                             log_file=None,
-                                             output_fileNames=None
-                                            )
-                    
-                    mask_path_gms = os.path.join(fim_run_parent, 'wbd.gpkg')
-
-                    Mosaic_inundation(
-                                        map_file,mosaic_attribute='inundation_rasters',
-                                        mosaic_output=inundation_raster,
-                                        mask=mask_path_gms,unit_attribute_name='huc8',
-                                        nodata=elev_raster_ndv,workers=1,
-                                        remove_inputs=True,
-                                        subset=None,verbose=verbose
-                                      )
-                
-                else:
-                    inundate(
-                             rem,catchments,catchment_poly,hydro_table,forecast,
-                             mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                             subset_hucs=current_huc,num_workers=1,aggregate=False,
-                             inundation_raster=inundation_raster,inundation_polygon=None,
-                             depths=None,out_raster_profile=None,out_vector_profile=None,
-                             quiet=True
-                        )
-
-                if model =='MS':
-                    
-                    # Mainstems inundation
-                    #fr_run_parent = os.path.join(os.environ['outputDataDir'], fr_run_dir,current_huc)
-                    #assert os.path.exists(fr_run_parent), "Cannot locate " + fr_run_parent
-                    
-                    inundation_raster_ms = os.path.join(
-                                        os.path.split(inundation_raster)[0], 
-                                        os.path.split(inundation_raster)[1].replace('.tif', '_{}_MS.tif'.format(current_huc))
-                                           )  
-                    inundation_raster_fr = os.path.join(
-                                        os.path.split(version_test_case_dir_parent)[0],
-                                        fr_run_dir,
-                                        magnitude,
-                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
-                                           )  
-                    
-                    os.rename(predicted_raster_path,inundation_raster_ms)
-
-                    ms_inundation_map_file = { 
-                                               'huc8' : [current_huc] * 2,
-                                               'branchID' : [None] * 2,
-                                               'inundation_rasters' : [inundation_raster_fr,inundation_raster_ms],
-                                               'depths_rasters' : [None] * 2,
-                                               'inundation_polygons' : [None] * 2
-                                             }
-                    ms_inundation_map_file = pd.DataFrame(ms_inundation_map_file)
-                    
-                    Mosaic_inundation(
-                                        ms_inundation_map_file,mosaic_attribute='inundation_rasters',
-                                        mosaic_output=inundation_raster,
-                                        mask=catchment_poly,unit_attribute_name='huc8',
-                                        nodata=elev_raster_ndv,workers=1,
-                                        remove_inputs=False,
-                                        subset=None,verbose=verbose
-                                      )
-
-                __vprint("-----> Inundation mapping complete.",verbose)
-
-                # Define outputs for agreement_raster, stats_json, and stats_csv.
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-                else:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-                
-                compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                       benchmark_raster_path,
-                                                       agreement_raster,
-                                                       stats_csv=stats_csv,
-                                                       stats_json=stats_json,
-                                                       mask_values=[],
-                                                       stats_modes_list=stats_modes_list,
-                                                       test_id=test_id,
-                                                       mask_dict=mask_dict,
-                                                       )
-
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    del mask_dict[ahps_lid]
-
-                __vprint(" ",verbose)
-                # print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC) # GMS
-                __vprint("Evaluation metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC,verbose) # cahaba/dev
-                __vprint(" ",verbose)
-
-            except Exception as e:
-                print(traceback.print_exc())
-                #print(e)
-
-        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-            # -- Delete temp files -- #
-            # List all files in the output directory.
-            output_file_list = os.listdir(version_test_case_dir)
-            for output_file in output_file_list:
-                if "total_area" in output_file:
-                    full_output_file_path = os.path.join(version_test_case_dir, output_file)
-                    os.remove(full_output_file_path)
-
-    # write out evaluation meta-data
-    with open(os.path.join(version_test_case_dir_parent,'eval_metadata.json'),'w') as meta:
-        eval_meta = { 'calibrated' : calibrated , 'model' : model }
-        meta.write( 
-                    json.dumps(eval_meta,indent=2) 
-                   )
-
-def __vprint(message,verbose):
-    if verbose:
-        print(message)
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-b', '--version',help='The name of the working version in which features are being tested',required=True,default="")
-    parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
-    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method. MS and GMS are currently on supporting huc', required=False,default="filter")
-    parser.add_argument('-n','--calibrated',help='Denotes use of calibrated n values',required=False, default=False,action='store_true')
-    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed',required=True)
-    parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
-    parser.add_argument('-c', '--compare-to-previous', help='Compare to previous versions of HAND.', required=False,action='store_true')
-    parser.add_argument('-a', '--archive-results', help='Automatically copy results to the "previous_version" archive for test_id. For admin use only.', required=False,action='store_true')
-    parser.add_argument('-i', '--inclusion-area', help='Path to shapefile. Contingency metrics will be produced from pixels inside of shapefile extent.', required=False, default="")
-    parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
-    parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
-    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False, action='store_true')
-    parser.add_argument('-w','--gms-workers', help='Number of workers to use for GMS Branch Inundation', required=False, default=1)
-    parser.add_argument('-d','--fr-run-dir',help='Name of test case directory containing inundation for FR configuration',required=False,default=None)
-    parser.add_argument('-v', '--verbose', help='Verbose operation', required=False, action='store_true', default=False)
-    parser.add_argument('-vg', '--gms-verbose', help='Prints progress bar for GMS', required=False, action='store_true', default=False)
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    valid_test_id_list = os.listdir(TEST_CASES_DIR)
-
-    exit_flag = False  # Default to False.
-    __vprint("",args['verbose'])
-
-    # Ensure test_id is valid.
-#    if args['test_id'] not in valid_test_id_list:
-#        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
-#        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
-#        for test_id in valid_test_id_list:
-#          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
-#              print(CYAN_BOLD + test_id + ENDC)
-#        print()
-#        exit_flag = True
-
-    # Ensure fim_run_dir exists.
-    if not os.path.exists(os.path.join(os.environ['outputDataDir'], args['fim_run_dir'])):
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
-        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
-        print()
-        exit_flag = True
-
-    # Ensure inclusion_area path exists.
-    if args['inclusion_area'] != "" and not os.path.exists(args['inclusion_area']):
-        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area (-i) " + CYAN_BOLD + args['inclusion_area'] + WHITE_BOLD + " could not be located." + ENDC)
-        exit_flag = True
-
-    try:
-        inclusion_buffer = int(args['inclusion_area_buffer'])
-    except ValueError:
-        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area_buffer (-ib) " + CYAN_BOLD + args['inclusion_area_buffer'] + WHITE_BOLD + " is not a round number." + ENDC)
-
-    benchmark_category = args['test_id'].split('_')[1]
-
-    if args['magnitude'] == '':
-        if 'ble' == benchmark_category:
-            args['magnitude'] = BLE_MAGNITUDE_LIST
-        elif ('nws' == benchmark_category) | ('usgs' == benchmark_category):
-            args['magnitude'] = ['action', 'minor', 'moderate', 'major']
-        elif 'ifc' == current_benchmark_category:
-            args['magnitude'] = IFC_MAGNITUDE_LIST
-        else:
-            print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
-            exit_flag = True
-
-    if exit_flag:
-        print()
-        sys.exit()
-
-    else:
-        run_alpha_test(**args)
\ No newline at end of file

From f58af3e75012964cc0bd73dc010ba6b675603699 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Thu, 18 Nov 2021 11:20:19 -0600
Subject: [PATCH 123/359] [1pt] PR: Hotfix for .csv file delete in SRC
 post-processing (#486)

Full details in Changelog for version 3.0.24.2. This resolves #485.
---
 docs/CHANGELOG.md                | 13 +++++++++++++
 fim_run.sh                       |  5 ++++-
 src/add_crosswalk.py             | 14 ++------------
 src/run_by_unit.sh               |  2 +-
 src/vary_mannings_n_composite.py | 23 ++++++++++++++++-------
 5 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c187a71a0..2ee0fa8b4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.24.2 - 2021-11-18 - [PR #486](https://github.com/NOAA-OWP/cahaba/pull/486)
+
+Adding a new check to keep `usgs_elev_table.csv`, `src_base.csv`, `small_segments.csv` for runs not using the `-viz` flag. We unintentionally deleted some .csv files in `vary_mannings_n_composite.py` but need to maintain some of these for non `-viz` runs (e.g. `usgs_elev_table.csv` is used for sierra test input).
+
+## Changes
+
+- `fim_run.sh`: passing `-v` flag to `vary_mannings_n_composite.py` to determine which csv files to delete. Setting `$viz` = 0 for non `-v` runs.
+- `src/vary_mannings_n_composite.py`: added `-v` input arg and if statement to check which .csv files to delete.
+- `src/add_crosswalk.py`: removed deprecated barc variables from input args.
+- `src/run_by_unit.sh`: removed deprecated barc variables from input args to `add_crosswalk.py`.
+
+<br/><br/>
+
 ## v3.0.24.1 - 2021-11-17 - [PR #484](https://github.com/NOAA-OWP/cahaba/pull/484)
 
 Patch to clean up unnecessary files and create better names for intermediate raster files.
diff --git a/fim_run.sh b/fim_run.sh
index 39565dc98..6d4d58a7a 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -104,6 +104,9 @@ source $srcDir/bash_functions.env
 if [ "$jobLimit" = "" ] ; then
     jobLimit=$default_max_jobs
 fi
+if [ "$viz" = "" ] ; then
+    viz=0
+fi
 
 ## Define Outputs Data Dir & Log File##
 export outputRunDataDir=$outputDataDir/$runName
@@ -168,7 +171,7 @@ fi
 echo -e $startDiv"Applying variable roughness in SRCs"$stopDiv
 if [ "$src_vrough_toggle" = "True" ]; then
     # Run BARC routine
-    time python3 /foss_fim/src/vary_mannings_n_composite.py -fim_dir $outputRunDataDir -mann $vmann_input_file -bc $bankfull_attribute -suff $vrough_suffix -j $jobLimit -plots $src_vrough_plot_option
+    time python3 /foss_fim/src/vary_mannings_n_composite.py -fim_dir $outputRunDataDir -mann $vmann_input_file -bc $bankfull_attribute -suff $vrough_suffix -j $jobLimit -plots $src_vrough_plot_option -viz_clean $viz
 fi
 
 echo "$viz"
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 49caf74e0..9b38418b2 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -15,7 +15,7 @@
 
 
 @mem_profile
-def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
+def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
 
     input_catchments = gpd.read_file(input_catchments_fileName)
     input_flows = gpd.read_file(input_flows_fileName)
@@ -277,11 +277,6 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     parser.add_argument('-d','--input-catchments-fileName', help='DEM derived catchments', required=True)
     parser.add_argument('-a','--input-flows-fileName', help='DEM derived streams', required=True)
     parser.add_argument('-s','--input-srcbase-fileName', help='Base synthetic rating curve table', required=True)
-    parser.add_argument('-u','--input-bathy-fileName', help='Text file with bankfull bathy variables', required=True)
-    parser.add_argument('-v','--output-bathy-fileName', help='Output bathy/bankfull crosswalk and calculated bathy variables', required=True)
-    parser.add_argument('-e','--output-bathy-order-fileName', help='Output bathy/bankfull stream order avg calculated bathy variables', required=True)
-    parser.add_argument('-g','--output-bathy-thalweg-fileName', help='Output bathy thalweg calculation', required=True)
-    parser.add_argument('-i','--output-bathy-xs-lookup-fileName', help='Output bathy XS Area lookup calculation', required=True)
     parser.add_argument('-l','--output-catchments-fileName', help='Subset crosswalked catchments', required=True)
     parser.add_argument('-f','--output-flows-fileName', help='Subset crosswalked streams', required=True)
     parser.add_argument('-r','--output-src-fileName', help='Output crosswalked synthetic rating curve table', required=True)
@@ -302,11 +297,6 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     input_catchments_fileName = args['input_catchments_fileName']
     input_flows_fileName = args['input_flows_fileName']
     input_srcbase_fileName = args['input_srcbase_fileName']
-    input_bathy_fileName = args['input_bathy_fileName']
-    output_bathy_fileName = args['output_bathy_fileName']
-    output_bathy_streamorder_fileName = args['output_bathy_order_fileName']
-    output_bathy_thalweg_fileName = args['output_bathy_thalweg_fileName']
-    output_bathy_xs_lookup_fileName = args['output_bathy_xs_lookup_fileName']
     output_catchments_fileName = args['output_catchments_fileName']
     output_flows_fileName = args['output_flows_fileName']
     output_src_fileName = args['output_src_fileName']
@@ -322,4 +312,4 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     small_segments_filename = args['small_segments_filename']
     calibration_mode = args['calibration_mode']
 
-    add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode)
+    add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode)
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index c5d5b4e38..b46f7ce9a 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -400,7 +400,7 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv outp
 date -u
 Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -u $input_bathy_bankfull -v $outputHucDataDir/bathy_crosswalk_calcs.csv -e $outputHucDataDir/bathy_stream_order_calcs.csv -g $outputHucDataDir/bathy_thalweg_flag.csv -i $outputHucDataDir/bathy_xs_area_hydroid_lookup.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
+python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
 Tcount
 
 ## USGS CROSSWALK ##
diff --git a/src/vary_mannings_n_composite.py b/src/vary_mannings_n_composite.py
index 46445efa0..0490cc368 100755
--- a/src/vary_mannings_n_composite.py
+++ b/src/vary_mannings_n_composite.py
@@ -46,6 +46,7 @@ def variable_mannings_calc(args):
     htable_filename             = args[5]
     src_plot_option             = args[6]
     huc_output_dir              = args[7]
+    viz_clean_flag              = args[8]
 
     ## Read the src_full_crosswalked.csv
     log_text = 'Calculating: ' + str(huc) + '\n'
@@ -107,18 +108,24 @@ def variable_mannings_calc(args):
         else:
             df_htable.drop(['discharge_cms'], axis=1, inplace=True) # drop the previously modified discharge column to be replaced with updated version
         df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
-        
+
         # Delete intermediate CSVs outputs. Todo delete this block later.
         htable_parent_dir = os.path.split(htable_filename)[0]
         # List all CSVs.
         file_list = os.listdir(htable_parent_dir)
         for f in file_list:
-            if '.csv' in f:
-                if f != 'hydroTable.csv':
-                    os.remove(os.path.join(htable_parent_dir, f))
-        
+            if viz_clean_flag == 1: # if using the viz flag then delete all intermediate csv files
+                if '.csv' in f:
+                    if f != 'hydroTable.csv':
+                        os.remove(os.path.join(htable_parent_dir, f))
+            else:
+                keep_files = ['usgs_elev_table.csv', 'src_base.csv', 'small_segments.csv']
+                if '.csv' in f:
+                    if f not in keep_files:
+                        os.remove(os.path.join(htable_parent_dir, f))
+
         df_htable.to_csv(htable_filename,index=False)
-        
+
         log_text += 'Completed: ' + str(huc)
 
         ## plot rating curves
@@ -183,6 +190,7 @@ def multi_process(variable_mannings_calc, procs_list):
     parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",required=True,type=str)
     parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
     parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
+    parser.add_argument('-viz_clean','--viz-clean',help='Optional (Viz flag): pass the viz flag (0 or 1) to delete intermediate csv files',required=False,default=0,type=int)
 
     args = vars(parser.parse_args())
 
@@ -192,6 +200,7 @@ def multi_process(variable_mannings_calc, procs_list):
     output_suffix = args['output_suffix']
     number_of_jobs = args['number_of_jobs']
     src_plot_option = args['src_plot_option']
+    viz_clean_flag = args['viz_clean']
     procs_list = []
 
     print('Writing progress to log file here: ' + str(join(fim_dir,'log_composite_n' + output_suffix + '.log')))
@@ -223,7 +232,7 @@ def multi_process(variable_mannings_calc, procs_list):
 
                     if isfile(in_src_bankfull_filename):
                         print(str(huc))
-                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_vmann_filename, htable_filename, src_plot_option, huc_plot_output_dir])
+                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_vmann_filename, htable_filename, src_plot_option, huc_plot_output_dir,viz_clean_flag])
                     else:
                         print(str(huc) + '\nWARNING --> can not find the src_full_crosswalked_bankfull.csv in the fim output dir: ' + str(join(fim_dir,huc)) + ' - skipping this HUC!!!\n')
 

From 3dabefc9b1bc5a9ceba7f4ac6519191398239c02 Mon Sep 17 00:00:00 2001
From: NickChadwick-NOAA <nick.chadwick@noaa.gov>
Date: Mon, 29 Nov 2021 13:07:51 -0600
Subject: [PATCH 124/359] Update pip package to fix projection issue

See Changelog for version 3.0.23.3. This resolves #483.
---
 Pipfile           |   1 +
 Pipfile.lock      | 525 +++++++++++++++++++++++++---------------------
 docs/CHANGELOG.md |  10 +
 3 files changed, 299 insertions(+), 237 deletions(-)

diff --git a/Pipfile b/Pipfile
index 48f8da27d..b54b7daf7 100755
--- a/Pipfile
+++ b/Pipfile
@@ -22,6 +22,7 @@ seaborn = "==0.11.0"
 python-dotenv = "*"
 natsort = "*"
 memory-profiler = "*"
+pyproj = "==3.1.0"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 11f8a16a5..4307ca71f 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "dbdc717b9aea8c37d6aecaa1b57bf669689c63758542bea1d47bb4e02fddc663"
+            "sha256": "f1296d064e5178d77fdf23130a571d8169821f2eeed04b37c1b2cf60fe6928ba"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -33,10 +33,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
-                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
+                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
+                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
             ],
-            "version": "==2021.5.30"
+            "version": "==2021.10.8"
         },
         "click": {
             "hashes": [
@@ -63,10 +63,11 @@
         },
         "cycler": {
             "hashes": [
-                "sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d",
-                "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8"
+                "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3",
+                "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"
             ],
-            "version": "==0.10.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==0.11.0"
         },
         "fiona": {
             "hashes": [
@@ -84,6 +85,14 @@
             "index": "pypi",
             "version": "==1.8.17"
         },
+        "fonttools": {
+            "hashes": [
+                "sha256:68071406009e7ef6a5fdcd85d95975cd6963867bb226f2b786bfffe15d1959ef",
+                "sha256:8c8f84131bf04f3b1dcf99b9763cec35c347164ab6ad006e18d2f99fcab05529"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==4.28.1"
+        },
         "geopandas": {
             "hashes": [
                 "sha256:e28a729e44ac53c1891b54b1aca60e3bc0bb9e88ad0f2be8e301a03b9510f6e2",
@@ -174,30 +183,44 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:01c9de93a2ca0d128c9064f23709362e7fefb34910c7c9e0b8ab0de8258d5eda",
-                "sha256:41b6e307458988891fcdea2d8ecf84a8c92d53f84190aa32da65f9505546e684",
-                "sha256:48e1e0859b54d5f2e29bb78ca179fd59b971c6ceb29977fb52735bfd280eb0f5",
-                "sha256:54a026055d5f8614f184e588f6e29064019a0aa8448450214c0b60926d62d919",
-                "sha256:556965514b259204637c360d213de28d43a1f4aed1eca15596ce83f768c5a56f",
-                "sha256:5c988bb43414c7c2b0a31bd5187b4d27fd625c080371b463a6d422047df78913",
-                "sha256:6a724e3a48a54b8b6e7c4ae38cd3d07084508fa47c410c8757e9db9791421838",
-                "sha256:6be8df61b1626e1a142c57e065405e869e9429b4a6dab4a324757d0dc4d42235",
-                "sha256:844a7b0233e4ff7fba57e90b8799edaa40b9e31e300b8d5efc350937fa8b1bea",
-                "sha256:85f0c9cf724715e75243a7b3087cf4a3de056b55e05d4d76cc58d610d62894f3",
-                "sha256:a78a3b51f29448c7f4d4575e561f6b0dbb8d01c13c2046ab6c5220eb25c06506",
-                "sha256:b884715a59fec9ad3b6048ecf3860f3b2ce965e676ef52593d6fa29abcf7d330",
-                "sha256:b8b53f336a4688cfce615887505d7e41fd79b3594bf21dd300531a4f5b4f746a",
-                "sha256:c70b6311dda3e27672f1bf48851a0de816d1ca6aaf3d49365fbdd8e959b33d2b",
-                "sha256:ebfb01a65c3f5d53a8c2a8133fec2b5221281c053d944ae81ff5822a68266617",
-                "sha256:eeb1859efe7754b1460e1d4991bbd4a60a56f366bc422ef3a9c5ae05f0bc70b5",
-                "sha256:f15edcb0629a0801738925fe27070480f446fcaa15de65946ff946ad99a59a40",
-                "sha256:f1c5efc278d996af8a251b2ce0b07bbeccb821f25c8c9846bdcb00ffc7f158aa",
-                "sha256:f72657f1596199dc1e4e7a10f52a4784ead8a711f4e5b59bea95bdb97cf0e4fd",
-                "sha256:fc4f526dfdb31c9bd6b8ca06bf9fab663ca12f3ec9cdf4496fb44bc680140318",
-                "sha256:fcd6f1954943c0c192bfbebbac263f839d7055409f1173f80d8b11a224d236da"
+                "sha256:0abf8b51cc6d3ba34d1b15b26e329f23879848a0cf1216954c1f432ffc7e1af7",
+                "sha256:0e020a42f3338823a393dd2f80e39a2c07b9f941dfe2c778eb104eeb33d60bb5",
+                "sha256:13930a0c9bec0fd25f43c448b047a21af1353328b946f044a8fc3be077c6b1a8",
+                "sha256:153a0cf6a6ff4f406a0600d2034710c49988bacc6313d193b32716f98a697580",
+                "sha256:18f6e52386300db5cc4d1e9019ad9da2e80658bab018834d963ebb0aa5355095",
+                "sha256:2089b9014792dcc87bb1d620cde847913338abf7d957ef05587382b0cb76d44e",
+                "sha256:2eea16883aa7724c95eea0eb473ab585c6cf66f0e28f7f13e63deb38f4fd6d0f",
+                "sha256:38892a254420d95594285077276162a5e9e9c30b6da08bdc2a4d53331ad9a6fa",
+                "sha256:4b018ea6f26424a0852eb60eb406420d9f0d34f65736ea7bbfbb104946a66d86",
+                "sha256:65f877882b7ddede7090c7d87be27a0f4720fe7fc6fddd4409c06e1aa0f1ae8d",
+                "sha256:666d717a4798eb9c5d3ae83fe80c7bc6ed696b93e879cb01cb24a74155c73612",
+                "sha256:66b172610db0ececebebb09d146f54205f87c7b841454e408fba854764f91bdd",
+                "sha256:6db02c5605f063b67780f4d5753476b6a4944343284aa4e93c5e8ff6e9ec7f76",
+                "sha256:6e0e6b2111165522ad336705499b1f968c34a9e84d05d498ee5af0b5697d1efe",
+                "sha256:71a1851111f23f82fc43d2b6b2bfdd3f760579a664ebc939576fe21cc6133d01",
+                "sha256:7a7cb59ebd63a8ac4542ec1c61dd08724f82ec3aa7bb6b4b9e212d43c611ce3d",
+                "sha256:7baf23adb698d8c6ca7339c9dde00931bc47b2dd82fa912827fef9f93db77f5e",
+                "sha256:970aa97297537540369d05fe0fd1bb952593f9ab696c9b427c06990a83e2418b",
+                "sha256:9bac8eb1eccef540d7f4e844b6313d9f7722efd48c07e1b4bfec1056132127fd",
+                "sha256:a07ff2565da72a7b384a9e000b15b6b8270d81370af8a3531a16f6fbcee023cc",
+                "sha256:a0dcaf5648cecddc328e81a0421821a1f65a1d517b20746c94a1f0f5c36fb51a",
+                "sha256:a0ea10faa3bab0714d3a19c7e0921279a68d57552414d6eceaea99f97d7735db",
+                "sha256:a5b62d1805cc83d755972033c05cea78a1e177a159fc84da5c9c4ab6303ccbd9",
+                "sha256:a6cef5b31e27c31253c0f852b629a38d550ae66ec6850129c49d872f9ee428cb",
+                "sha256:a7bf8b05c214d32fb7ca7c001fde70b9b426378e897b0adbf77b85ea3569d56a",
+                "sha256:ac17a7e7b06ee426a4989f0b7f24ab1a592e39cdf56353a90f4e998bc0bf44d6",
+                "sha256:b3b687e905da32e5f2e5f16efa713f5d1fcd9fb8b8c697895de35c91fedeb086",
+                "sha256:b5e439d9e55d645f2a4dca63e2f66d68fe974c405053b132d61c7e98c25dfeb2",
+                "sha256:ba107add08e12600b072cf3c47aaa1ab85dd4d3c48107a5d3377d1bf80f8b235",
+                "sha256:d092b7ba63182d2dd427904e3eb58dd5c46ec67c5968de14a4b5007010a3a4cc",
+                "sha256:dc8c5c23e7056e126275dbf29efba817b3d94196690930d0968873ac3a94ab82",
+                "sha256:df0042cab69f4d246f4cb8fc297770ac4ae6ec2983f61836b04a117722037dcd",
+                "sha256:ee3d9ff16d749a9aa521bd7d86f0dbf256b2d2ac8ce31b19e4d2c86d2f2ff0b6",
+                "sha256:f23fbf70d2e80f4e03a83fc1206a8306d9bc50482fee4239f10676ce7e470c83",
+                "sha256:ff5d9fe518ad2de14ce82ab906b6ab5c2b0c7f4f984400ff8a7a905daa580a0a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.4.3"
+            "version": "==3.5.0"
         },
         "memory-profiler": {
             "hashes": [
@@ -215,11 +238,11 @@
         },
         "natsort": {
             "hashes": [
-                "sha256:00c603a42365830c4722a2eb7663a25919551217ec09a243d3399fa8dd4ac403",
-                "sha256:d0f4fc06ca163fa4a5ef638d9bf111c67f65eedcc7920f98dec08e489045b67e"
+                "sha256:5f5f4ea471d655b1b1611eef1cf0c6d3397095d2d3a1aab7098d6a50e4c3901a",
+                "sha256:a0a4fd71aee20a6d648da61e01180a63f7268e69983d0440bd3ad80ef1ba6981"
             ],
             "index": "pypi",
-            "version": "==7.1.1"
+            "version": "==8.0.0"
         },
         "numba": {
             "hashes": [
@@ -248,39 +271,47 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf",
-                "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2",
-                "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6",
-                "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad",
-                "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc",
-                "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254",
-                "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0",
-                "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218",
-                "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13",
-                "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef",
-                "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737",
-                "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723",
-                "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3",
-                "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1",
-                "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d",
-                "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52",
-                "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3",
-                "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5",
-                "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f",
-                "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496",
-                "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f",
-                "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724",
-                "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931",
-                "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f",
-                "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7",
-                "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38",
-                "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557",
-                "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57",
-                "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310",
-                "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419"
+                "sha256:0b78ecfa070460104934e2caf51694ccd00f37d5e5dbe76f021b1b0b0d221823",
+                "sha256:1247ef28387b7bb7f21caf2dbe4767f4f4175df44d30604d42ad9bd701ebb31f",
+                "sha256:1403b4e2181fc72664737d848b60e65150f272fe5a1c1cbc16145ed43884065a",
+                "sha256:170b2a0805c6891ca78c1d96ee72e4c3ed1ae0a992c75444b6ab20ff038ba2cd",
+                "sha256:2e4ed57f45f0aa38beca2a03b6532e70e548faf2debbeb3291cfc9b315d9be8f",
+                "sha256:32fe5b12061f6446adcbb32cf4060a14741f9c21e15aaee59a207b6ce6423469",
+                "sha256:34f3456f530ae8b44231c63082c8899fe9c983fd9b108c997c4b1c8c2d435333",
+                "sha256:4c9c23158b87ed0e70d9a50c67e5c0b3f75bcf2581a8e34668d4e9d7474d76c6",
+                "sha256:5d95668e727c75b3f5088ec7700e260f90ec83f488e4c0aaccb941148b2cd377",
+                "sha256:615d4e328af7204c13ae3d4df7615a13ff60a49cb0d9106fde07f541207883ca",
+                "sha256:69077388c5a4b997442b843dbdc3a85b420fb693ec8e33020bb24d647c164fa5",
+                "sha256:74b85a17528ca60cf98381a5e779fc0264b4a88b46025e6bcbe9621f46bb3e63",
+                "sha256:81225e58ef5fce7f1d80399575576fc5febec79a8a2742e8ef86d7b03beef49f",
+                "sha256:8890b3360f345e8360133bc078d2dacc2843b6ee6059b568781b15b97acbe39f",
+                "sha256:92aafa03da8658609f59f18722b88f0a73a249101169e28415b4fa148caf7e41",
+                "sha256:9864424631775b0c052f3bd98bc2712d131b3e2cd95d1c0c68b91709170890b0",
+                "sha256:9e6f5f50d1eff2f2f752b3089a118aee1ea0da63d56c44f3865681009b0af162",
+                "sha256:a3deb31bc84f2b42584b8c4001c85d1934dbfb4030827110bc36bfd11509b7bf",
+                "sha256:ad010846cdffe7ec27e3f933397f8a8d6c801a48634f419e3d075db27acf5880",
+                "sha256:b1e2312f5b8843a3e4e8224b2b48fe16119617b8fc0a54df8f50098721b5bed2",
+                "sha256:bc988afcea53e6156546e5b2885b7efab089570783d9d82caf1cfd323b0bb3dd",
+                "sha256:c449eb870616a7b62e097982c622d2577b3dbc800aaf8689254ec6e0197cbf1e",
+                "sha256:c74c699b122918a6c4611285cc2cad4a3aafdb135c22a16ec483340ef97d573c",
+                "sha256:c885bfc07f77e8fee3dc879152ba993732601f1f11de248d4f357f0ffea6a6d4",
+                "sha256:e3c3e990274444031482a31280bf48674441e0a5b55ddb168f3a6db3e0c38ec8",
+                "sha256:e4799be6a2d7d3c33699a6f77201836ac975b2e1b98c2a07f66a38f499cb50ce",
+                "sha256:e6c76a87633aa3fa16614b61ccedfae45b91df2767cf097aa9c933932a7ed1e0",
+                "sha256:e89717274b41ebd568cd7943fc9418eeb49b1785b66031bc8a7f6300463c5898",
+                "sha256:f5162ec777ba7138906c9c274353ece5603646c6965570d82905546579573f73",
+                "sha256:fde96af889262e85aa033f8ee1d3241e32bf36228318a61f1ace579df4e8170d"
             ],
             "markers": "python_version < '3.11' and python_version >= '3.7'",
-            "version": "==1.21.2"
+            "version": "==1.21.4"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
+                "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.3"
         },
         "pandas": {
             "hashes": [
@@ -306,62 +337,50 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30",
-                "sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9",
-                "sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71",
-                "sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9",
-                "sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b",
-                "sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630",
-                "sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875",
-                "sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2",
-                "sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1",
-                "sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7",
-                "sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3",
-                "sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b",
-                "sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6",
-                "sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba",
-                "sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4",
-                "sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864",
-                "sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056",
-                "sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228",
-                "sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8",
-                "sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb",
-                "sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d",
-                "sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da",
-                "sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073",
-                "sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3",
-                "sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616",
-                "sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa",
-                "sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979",
-                "sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a",
-                "sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b",
-                "sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6",
-                "sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441",
-                "sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624",
-                "sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd",
-                "sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550",
-                "sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09",
-                "sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196",
-                "sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b",
-                "sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1",
-                "sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6",
-                "sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83",
-                "sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f",
-                "sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4",
-                "sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19",
-                "sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341",
-                "sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96",
-                "sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355",
-                "sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c",
-                "sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c",
-                "sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629",
-                "sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2",
-                "sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87",
-                "sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5",
-                "sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"
+                "sha256:066f3999cb3b070a95c3652712cffa1a748cd02d60ad7b4e485c3748a04d9d76",
+                "sha256:0a0956fdc5defc34462bb1c765ee88d933239f9a94bc37d132004775241a7585",
+                "sha256:0b052a619a8bfcf26bd8b3f48f45283f9e977890263e4571f2393ed8898d331b",
+                "sha256:1394a6ad5abc838c5cd8a92c5a07535648cdf6d09e8e2d6df916dfa9ea86ead8",
+                "sha256:1bc723b434fbc4ab50bb68e11e93ce5fb69866ad621e3c2c9bdb0cd70e345f55",
+                "sha256:244cf3b97802c34c41905d22810846802a3329ddcb93ccc432870243211c79fc",
+                "sha256:25a49dc2e2f74e65efaa32b153527fc5ac98508d502fa46e74fa4fd678ed6645",
+                "sha256:2e4440b8f00f504ee4b53fe30f4e381aae30b0568193be305256b1462216feff",
+                "sha256:3862b7256046fcd950618ed22d1d60b842e3a40a48236a5498746f21189afbbc",
+                "sha256:3eb1ce5f65908556c2d8685a8f0a6e989d887ec4057326f6c22b24e8a172c66b",
+                "sha256:3f97cfb1e5a392d75dd8b9fd274d205404729923840ca94ca45a0af57e13dbe6",
+                "sha256:493cb4e415f44cd601fcec11c99836f707bb714ab03f5ed46ac25713baf0ff20",
+                "sha256:4acc0985ddf39d1bc969a9220b51d94ed51695d455c228d8ac29fcdb25810e6e",
+                "sha256:5503c86916d27c2e101b7f71c2ae2cddba01a2cf55b8395b0255fd33fa4d1f1a",
+                "sha256:5b7bb9de00197fb4261825c15551adf7605cf14a80badf1761d61e59da347779",
+                "sha256:5e9ac5f66616b87d4da618a20ab0a38324dbe88d8a39b55be8964eb520021e02",
+                "sha256:620582db2a85b2df5f8a82ddeb52116560d7e5e6b055095f04ad828d1b0baa39",
+                "sha256:62cc1afda735a8d109007164714e73771b499768b9bb5afcbbee9d0ff374b43f",
+                "sha256:70ad9e5c6cb9b8487280a02c0ad8a51581dcbbe8484ce058477692a27c151c0a",
+                "sha256:72b9e656e340447f827885b8d7a15fc8c4e68d410dc2297ef6787eec0f0ea409",
+                "sha256:72cbcfd54df6caf85cc35264c77ede902452d6df41166010262374155947460c",
+                "sha256:792e5c12376594bfcb986ebf3855aa4b7c225754e9a9521298e460e92fb4a488",
+                "sha256:7b7017b61bbcdd7f6363aeceb881e23c46583739cb69a3ab39cb384f6ec82e5b",
+                "sha256:81f8d5c81e483a9442d72d182e1fb6dcb9723f289a57e8030811bac9ea3fef8d",
+                "sha256:82aafa8d5eb68c8463b6e9baeb4f19043bb31fefc03eb7b216b51e6a9981ae09",
+                "sha256:84c471a734240653a0ec91dec0996696eea227eafe72a33bd06c92697728046b",
+                "sha256:8c803ac3c28bbc53763e6825746f05cc407b20e4a69d0122e526a582e3b5e153",
+                "sha256:93ce9e955cc95959df98505e4608ad98281fff037350d8c2671c9aa86bcf10a9",
+                "sha256:9a3e5ddc44c14042f0844b8cf7d2cd455f6cc80fd7f5eefbe657292cf601d9ad",
+                "sha256:a4901622493f88b1a29bd30ec1a2f683782e57c3c16a2dbc7f2595ba01f639df",
+                "sha256:a5a4532a12314149d8b4e4ad8ff09dde7427731fcfa5917ff16d0291f13609df",
+                "sha256:b8831cb7332eda5dc89b21a7bce7ef6ad305548820595033a4b03cf3091235ed",
+                "sha256:b8e2f83c56e141920c39464b852de3719dfbfb6e3c99a2d8da0edf4fb33176ed",
+                "sha256:c70e94281588ef053ae8998039610dbd71bc509e4acbc77ab59d7d2937b10698",
+                "sha256:c8a17b5d948f4ceeceb66384727dde11b240736fddeda54ca740b9b8b1556b29",
+                "sha256:d82cdb63100ef5eedb8391732375e6d05993b765f72cb34311fab92103314649",
+                "sha256:d89363f02658e253dbd171f7c3716a5d340a24ee82d38aab9183f7fdf0cdca49",
+                "sha256:d99ec152570e4196772e7a8e4ba5320d2d27bf22fdf11743dd882936ed64305b",
+                "sha256:ddc4d832a0f0b4c52fff973a0d44b6c99839a9d016fe4e6a1cb8f3eea96479c2",
+                "sha256:e3dacecfbeec9a33e932f00c6cd7996e62f53ad46fbe677577394aaa90ee419a",
+                "sha256:eb9fc393f3c61f9054e1ed26e6fe912c7321af2f41ff49d3f83d05bacf22cc78"
             ],
-            "index": "pypi",
-            "version": "==8.3.2"
+            "markers": "python_version >= '3.6'",
+            "version": "==8.4.0"
         },
         "psutil": {
             "hashes": [
@@ -421,40 +440,36 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
-                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
+                "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4",
+                "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.4.7"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.6"
         },
         "pyproj": {
             "hashes": [
-                "sha256:1563a854eff7b6ed430fb968f8f711d13fc02d587afb1dc93a88e5cda6d36462",
-                "sha256:1a0a55bbc2d903a6dada943ade11360a46d057ca42d305128c113273bae7cb58",
-                "sha256:2cc06ea53f8c6c97ff37a47a614b32adc4c4b667def08b528ea7f6a351fa8628",
-                "sha256:2e8e51e55bc6305b538c0c6576dbc37a45993b7f3e68699237411fbde9220aab",
-                "sha256:413cb8dfd358f3b89cb9756a3380295d34b3ea5059d47f327262e700f1c8baab",
-                "sha256:48df0d5ab085bd2dc6db3bca79e20bf15b08ffca4f4e42df6d87b566633b800c",
-                "sha256:582efc0cf0bae6ac5a4730280995fad6c20d844b56b70f41f4f04fdb1fdbd5d7",
-                "sha256:5974c1acbeb1d4f34bba111be547191f967ef182f77fc361c34984f8adbe6f66",
-                "sha256:5fb4887893d221a82082e95ac6dbf75cbbc739991fcaa80ba94fc16070321354",
-                "sha256:72e0c4409a0c2f83ba448ecdf6accc9615d3e4069b8cad2a2a37a464225d0582",
-                "sha256:8206748e1ef61151f19dd54378312e272812cb248226fb7909fda5052498f9b4",
-                "sha256:85b7f67a3606b8a846691effd80c187597ac4e2733ae818f3e3e8be33edcb582",
-                "sha256:87a2b36d7847149d1309c850d022da71de9398140a62e1496cfe64cf3e74040a",
-                "sha256:89c023b1f7c956343e87bc7165fe67986b5db0b840fd5adf0ca65c0b5dcb5007",
-                "sha256:8c0cd49eed818c9a02ea05cb9dc45d7209d1688fc4c9e08a37bc48a9ec72e852",
-                "sha256:911d711601d53208eb4dcca95df85661bee13592da6035c7f12ec0e8c7bbfea8",
-                "sha256:9a7007dea9d739a03afcc214c851dd43230db319876dd4bf3787031b67a57b77",
-                "sha256:b00e05f374d419be9689c53887149855e277b8fa6196f005b748c2c869ce260f",
-                "sha256:ca0838c2d979db4d71ae131214d336ad206b02b648a51e023e2ce2f5f8c2db11",
-                "sha256:cf52413347479035d361e5f760d1c29876b8941a763bb35e5f1505b3421707ae",
-                "sha256:e3415554fd1905c14632254f087f484e704b39865313aea8fe3e4d13017979c7",
-                "sha256:e95ad0255610b719e257150c4f989da51e025dea9a4d686b072c68f99ded538d",
-                "sha256:ff41af4b3bb33a1cdd0bfc1efe9eb0f0392ae946aeca5323309f66568d073efa"
+                "sha256:04c185102e659439c5bd428ac5473d36ef795fca8e225bbbe78e20643d804ec0",
+                "sha256:10dad599b9f7ce2194996dc25f1000e0aa15754ecef9db46b624713959c67957",
+                "sha256:1e88ebc4e08e661e9011b5c1ebfb32f0d311963a9824a6effb4168c7e07918b1",
+                "sha256:4f3ad09cf3352bf5664794042b28d98781362ec8d9774ad73f28a1a0101a27f1",
+                "sha256:5f8a8d982bde211e65dc2de1f8f36cf162f9cc7fcd8a7625046ea265284e5e65",
+                "sha256:67b94f4e694ae33fc90dfb7da0e6b5ed5f671dd0acc2f6cf46e9c39d56e16e1a",
+                "sha256:808f5992320e9631b2e45444028a65cd6ba3ee40229292934178ef07020a5ffd",
+                "sha256:8eda240225971b5cd0bac2d399ed6222068f0598ee92d5f6e847bd2019d2c8b0",
+                "sha256:911d773da9fa4d4f3f7580173858c391e3ee0b61acaf0be303baab323d2eae78",
+                "sha256:9cc464a1c51baad28ffb7a233116e8d4ce4c560b32039fa986d0f992ac3c431f",
+                "sha256:a162ed199cd2ec392cffe20b2fa3381b68e7a166d55f3f060eceb8d517e4f46d",
+                "sha256:aa87df0982aa0f4477478899d9c930cc0f97cd6d8a4ce84c43ac88ccf86d1da7",
+                "sha256:ae237492767e0225f99b53a0fd7110fde2b7e7cabc105bbc243c151a7497de88",
+                "sha256:ae5534fa7a3b74f20534694d297fce6f7483890ff6ca404394ecf372f3c589d4",
+                "sha256:b635e7e21fea5af74e90fc9e54d1a4c27078efdce6f214101c98dd93afae599a",
+                "sha256:b6c74bbec679199746a3e02c0e0fad093c3652df96dd63e086a2fbf2afe9dc0e",
+                "sha256:c4193e1069d165476b2d0f7d882b7712b3eab6e2e6fe2a0a78ef40de825a1f28",
+                "sha256:da88abc5e2f6a8fb07533855a57ca2a31845f58901a87f821b68b0db6b023978",
+                "sha256:ebbba7707fe83a01e54bce8e3e7342feb0b3e0d74ff8c28df12f8bc59b76827c"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.2.0"
+            "index": "pypi",
+            "version": "==3.1.0"
         },
         "python-dateutil": {
             "hashes": [
@@ -466,18 +481,18 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:dd8fe852847f4fbfadabf6183ddd4c824a9651f02d51714fa075c95561959c7d",
-                "sha256:effaac3c1e58d89b3ccb4d04a40dc7ad6e0275fda25fd75ae9d323e2465e202d"
+                "sha256:32b2bdc1873fd3a3c346da1c6db83d0053c3c62f28f1f38516070c4c8971b1d3",
+                "sha256:a5de49a31e953b45ff2d2fd434bbc2670e8db5273606c1e737cc6b93eff3655f"
             ],
             "index": "pypi",
-            "version": "==0.18.0"
+            "version": "==0.19.2"
         },
         "pytz": {
             "hashes": [
-                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
-                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
+                "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c",
+                "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"
             ],
-            "version": "==2021.1"
+            "version": "==2021.3"
         },
         "rasterio": {
             "hashes": [
@@ -524,28 +539,33 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:2a0eeaab01258e0870c4022a6cd329aef3b7c6c2b606bd7cf7bb2ba9820ae561",
-                "sha256:3304bd5bc32e00954ac4b3f4cc382ca8824719bf348aacbec6347337d6b125fe",
-                "sha256:3f52470e0548cdb74fb8ddf06773ffdcca7c97550f903b1c51312ec19243a7f7",
-                "sha256:4729b41a4cdaf4cd011aeac816b532f990bdf97710cef59149d3e293115cf467",
-                "sha256:4ee952f39a4a4c7ba775a32b664b1f4b74818548b65f765987adc14bb78f5802",
-                "sha256:611f9cb459d0707dd8e4de0c96f86e93f61aac7475fcb225e9ec71fecdc5cebf",
-                "sha256:6b47d5fa7ea651054362561a28b1ccc8da9368a39514c1bbf6c0977a1c376764",
-                "sha256:71cfc96297617eab911e22216e8a8597703202e95636d9406df9af5c2ac99a2b",
-                "sha256:787749110a23502031fb1643c55a2236c99c6b989cca703ea2114d65e21728ef",
-                "sha256:90c07ba5f34f33299a428b0d4fa24c30d2ceba44d63f8385b2b05be460819fcb",
-                "sha256:a496b42dbcd04ea9924f5e92be63af3d8e0f43a274b769bfaca0a297327d54ee",
-                "sha256:bc61e3e5ff92d2f32bb263621d54a9cff5e3f7c420af3d1fa122ce2529de2bd9",
-                "sha256:c9951e3746b68974125e5e3445008a4163dd6d20ae0bbdae22b38cb8951dc11b",
-                "sha256:d1388fbac9dd591ea630da75c455f4cc637a7ca5ecb31a6b6cef430914749cde",
-                "sha256:d13f31457f2216e5705304d9f28e2826edf75487410a57aa99263fa4ffd792c2",
-                "sha256:d648aa85dd5074b1ed83008ae987c3fbb53d68af619fce1dee231f4d8bd40e2f",
-                "sha256:da9c6b336e540def0b7fd65603da8abeb306c5fc9a5f4238665cbbb5ff95cf58",
-                "sha256:e101bceeb9e65a90dadbc5ca31283403a2d4667b9c178db29109750568e8d112",
-                "sha256:efdd3825d54c58df2cc394366ca4b9166cf940a0ebddeb87b6c10053deb625ea"
-            ],
-            "markers": "python_version < '3.10' and python_version >= '3.7'",
-            "version": "==1.7.1"
+                "sha256:1437073f1d4664990879aa8f9547524764372e0fef84a077be4b19e82bba7a8d",
+                "sha256:17fd991a275e4283453f89d404209aa92059ac68d76d804b4bc1716a3742e1b5",
+                "sha256:1ea6233f5a365cb7945b4304bd06323ece3ece85d6a3fa8598d2f53e513467c9",
+                "sha256:2d25272c03ee3c0fe5e0dff1bb7889280bb6c9e1766fa9c7bde81ad8a5f78694",
+                "sha256:30bdda199667e74b50208a793eb1ba47a04e5e3fa16f5ff06c6f7969ae78e4da",
+                "sha256:359b60a0cccd17723b9d5e329a5212a710e771a3ddde800e472fb93732756c46",
+                "sha256:39f838ea5ce8da868785193d88d05cf5a6d5c390804ec99de29a28e1dcdd53e6",
+                "sha256:4d175ba93e00d8eef8f7cd70d4d88a9106a86800c82ea03cf2268c36d6545483",
+                "sha256:5273d832fb9cd5724ee0d335c16a903b923441107dd973d27fc4293075a9f4e3",
+                "sha256:54951f51d731c832b1b8885e0a92e89f33d087de7e40d02078bf0d49c7cbdbb5",
+                "sha256:74f518ce542533054695f743e4271cb8986b63f95bb51d70fcee4f3929cbff7d",
+                "sha256:7b1d0f5f524518f1a86f288443528e4ff4a739c0966db663af4129b7ac7849f8",
+                "sha256:82c5befebf54d799d77e5f0205c03030f57f69ba2541baa44d2e6ad138c28cd3",
+                "sha256:8482c8e45857ab0a5446eb7460d2307a27cbbe659d6d2257820c6d6eb950fd0f",
+                "sha256:87cf3964db0f1cce17aeed5bfc1b89a6b4b07dbfc48e50d21fa3549e00456803",
+                "sha256:8b5726a0fedeaa6beb1095e4466998bdd1d1e960b28db9b5a16c89cbd7b2ebf1",
+                "sha256:97eb573e361a73a553b915dc195c6f72a08249964b1a33f157f9659f3b6210d1",
+                "sha256:a80eb01c43fd98257ec7a49ff5cec0edba32031b5f86503f55399a48cb2c5379",
+                "sha256:cac71d5476a6f56b50459da21f6221707e0051ebd428b2137db32ef4a43bb15e",
+                "sha256:d86abd1ddf421dea5e9cebfeb4de0d205b3dc04e78249afedba9c6c3b2227ff2",
+                "sha256:dc2d1bf41294e63c7302bf499973ac0c7f73c93c01763db43055f6525234bf11",
+                "sha256:e08b81fcd9bf98740b58dc6fdd7879e33a64dcb682201c1135f7d4a75216bb05",
+                "sha256:e3efe7ef75dfe627b354ab0af0dbc918eadee97cc80ff1aabea6d3e01114ebdd",
+                "sha256:fa2dbabaaecdb502641b0b3c00dec05fb475ae48655c66da16c9ed24eda1e711"
+            ],
+            "markers": "python_version < '3.11' and python_version >= '3.7'",
+            "version": "==1.7.2"
         },
         "seaborn": {
             "hashes": [
@@ -555,6 +575,14 @@
             "index": "pypi",
             "version": "==0.11.0"
         },
+        "setuptools-scm": {
+            "hashes": [
+                "sha256:4c64444b1d49c4063ae60bfe1680f611c8b13833d556fd1d6050c0023162a119",
+                "sha256:a49aa8081eeb3514eb9728fa5040f2eaa962d6c6f4ec9c32f6c1fba88f88a0f2"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==6.3.2"
+        },
         "shapely": {
             "hashes": [
                 "sha256:11090bd5b5f11d54e1924a11198226971dab6f392c2e5a3c74514857f764b971",
@@ -582,55 +610,70 @@
         },
         "simplejson": {
             "hashes": [
-                "sha256:065230b9659ac38c8021fa512802562d122afb0cf8d4b89e257014dcddb5730a",
-                "sha256:07707ba69324eaf58f0c6f59d289acc3e0ed9ec528dae5b0d4219c0d6da27dc5",
-                "sha256:10defa88dd10a0a4763f16c1b5504e96ae6dc68953cfe5fc572b4a8fcaf9409b",
-                "sha256:140eb58809f24d843736edb8080b220417e22c82ac07a3dfa473f57e78216b5f",
-                "sha256:188f2c78a8ac1eb7a70a4b2b7b9ad11f52181044957bf981fb3e399c719e30ee",
-                "sha256:1c2688365743b0f190392e674af5e313ebe9d621813d15f9332e874b7c1f2d04",
-                "sha256:24e413bd845bd17d4d72063d64e053898543fb7abc81afeae13e5c43cef9c171",
-                "sha256:2b59acd09b02da97728d0bae8ff48876d7efcbbb08e569c55e2d0c2e018324f5",
-                "sha256:2df15814529a4625ea6f7b354a083609b3944c269b954ece0d0e7455872e1b2a",
-                "sha256:352c11582aa1e49a2f0f7f7d8fd5ec5311da890d1354287e83c63ab6af857cf5",
-                "sha256:36b08b886027eac67e7a0e822e3a5bf419429efad7612e69501669d6252a21f2",
-                "sha256:376023f51edaf7290332dacfb055bc00ce864cb013c0338d0dea48731f37e42f",
-                "sha256:3ba82f8b421886f4a2311c43fb98faaf36c581976192349fef2a89ed0fcdbdef",
-                "sha256:3d72aa9e73134dacd049a2d6f9bd219f7be9c004d03d52395831611d66cedb71",
-                "sha256:40ece8fa730d1a947bff792bcc7824bd02d3ce6105432798e9a04a360c8c07b0",
-                "sha256:417b7e119d66085dc45bdd563dcb2c575ee10a3b1c492dd3502a029448d4be1c",
-                "sha256:42b7c7264229860fe879be961877f7466d9f7173bd6427b3ba98144a031d49fb",
-                "sha256:457d9cfe7ece1571770381edccdad7fc255b12cd7b5b813219441146d4f47595",
-                "sha256:4a6943816e10028eeed512ea03be52b54ea83108b408d1049b999f58a760089b",
-                "sha256:5b94df70bd34a3b946c0eb272022fb0f8a9eb27cad76e7f313fedbee2ebe4317",
-                "sha256:5f5051a13e7d53430a990604b532c9124253c5f348857e2d5106d45fc8533860",
-                "sha256:5f7f53b1edd4b23fb112b89208377480c0bcee45d43a03ffacf30f3290e0ed85",
-                "sha256:5fe8c6dcb9e6f7066bdc07d3c410a2fca78c0d0b4e0e72510ffd20a60a20eb8e",
-                "sha256:71a54815ec0212b0cba23adc1b2a731bdd2df7b9e4432718b2ed20e8aaf7f01a",
-                "sha256:7332f7b06d42153255f7bfeb10266141c08d48cc1a022a35473c95238ff2aebc",
-                "sha256:78c6f0ed72b440ebe1892d273c1e5f91e55e6861bea611d3b904e673152a7a4c",
-                "sha256:7c9b30a2524ae6983b708f12741a31fbc2fb8d6fecd0b6c8584a62fd59f59e09",
-                "sha256:86fcffc06f1125cb443e2bed812805739d64ceb78597ac3c1b2d439471a09717",
-                "sha256:87572213965fd8a4fb7a97f837221e01d8fddcfb558363c671b8aa93477fb6a2",
-                "sha256:8e595de17178dd3bbeb2c5b8ea97536341c63b7278639cb8ee2681a84c0ef037",
-                "sha256:917f01db71d5e720b731effa3ff4a2c702a1b6dacad9bcdc580d86a018dfc3ca",
-                "sha256:91cfb43fb91ff6d1e4258be04eee84b51a4ef40a28d899679b9ea2556322fb50",
-                "sha256:aa86cfdeb118795875855589934013e32895715ec2d9e8eb7a59be3e7e07a7e1",
-                "sha256:ade09aa3c284d11f39640aebdcbb748e1996f0c60504f8c4a0c5a9fec821e67a",
-                "sha256:b2a5688606dffbe95e1347a05b77eb90489fe337edde888e23bbb7fd81b0d93b",
-                "sha256:b92fbc2bc549c5045c8233d954f3260ccf99e0f3ec9edfd2372b74b350917752",
-                "sha256:c2d5334d935af711f6d6dfeec2d34e071cdf73ec0df8e8bd35ac435b26d8da97",
-                "sha256:cb0afc3bad49eb89a579103616574a54b523856d20fc539a4f7a513a0a8ba4b2",
-                "sha256:ce66f730031b9b3683b2fc6ad4160a18db86557c004c3d490a29bf8d450d7ab9",
-                "sha256:e29b9cea4216ec130df85d8c36efb9985fda1c9039e4706fb30e0fb6a67602ff",
-                "sha256:e2cc4b68e59319e3de778325e34fbff487bfdb2225530e89995402989898d681",
-                "sha256:e90d2e219c3dce1500dda95f5b893c293c4d53c4e330c968afbd4e7a90ff4a5b",
-                "sha256:f13c48cc4363829bdfecc0c181b6ddf28008931de54908a492dc8ccd0066cd60",
-                "sha256:f550730d18edec4ff9d4252784b62adfe885d4542946b6d5a54c8a6521b56afd",
-                "sha256:fa843ee0d34c7193f5a816e79df8142faff851549cab31e84b526f04878ac778",
-                "sha256:fe1c33f78d2060719d52ea9459d97d7ae3a5b707ec02548575c4fbed1d1d345b"
+                "sha256:04e31fa6ac8e326480703fb6ded1488bfa6f1d3f760d32e29dbf66d0838982ce",
+                "sha256:068670af975247acbb9fc3d5393293368cda17026db467bf7a51548ee8f17ee1",
+                "sha256:07ecaafc1b1501f275bf5acdee34a4ad33c7c24ede287183ea77a02dc071e0c0",
+                "sha256:0b4126cac7d69ac06ff22efd3e0b3328a4a70624fcd6bca4fc1b4e6d9e2e12bf",
+                "sha256:0de783e9c2b87bdd75b57efa2b6260c24b94605b5c9843517577d40ee0c3cc8a",
+                "sha256:12133863178a8080a3dccbf5cb2edfab0001bc41e5d6d2446af2a1131105adfe",
+                "sha256:1c9b1ed7ed282b36571638297525f8ef80f34b3e2d600a56f962c6044f24200d",
+                "sha256:23fe704da910ff45e72543cbba152821685a889cf00fc58d5c8ee96a9bad5f94",
+                "sha256:28221620f4dcabdeac310846629b976e599a13f59abb21616356a85231ebd6ad",
+                "sha256:35a49ebef25f1ebdef54262e54ae80904d8692367a9f208cdfbc38dbf649e00a",
+                "sha256:37bc0cf0e5599f36072077e56e248f3336917ded1d33d2688624d8ed3cefd7d2",
+                "sha256:3fe87570168b2ae018391e2b43fbf66e8593a86feccb4b0500d134c998983ccc",
+                "sha256:3ff5b3464e1ce86a8de8c88e61d4836927d5595c2162cab22e96ff551b916e81",
+                "sha256:401d40969cee3df7bda211e57b903a534561b77a7ade0dd622a8d1a31eaa8ba7",
+                "sha256:4b6bd8144f15a491c662f06814bd8eaa54b17f26095bb775411f39bacaf66837",
+                "sha256:4c09868ddb86bf79b1feb4e3e7e4a35cd6e61ddb3452b54e20cf296313622566",
+                "sha256:4d1c135af0c72cb28dd259cf7ba218338f4dc027061262e46fe058b4e6a4c6a3",
+                "sha256:4ff4ac6ff3aa8f814ac0f50bf218a2e1a434a17aafad4f0400a57a8cc62ef17f",
+                "sha256:521877c7bd060470806eb6335926e27453d740ac1958eaf0d8c00911bc5e1802",
+                "sha256:522fad7be85de57430d6d287c4b635813932946ebf41b913fe7e880d154ade2e",
+                "sha256:5540fba2d437edaf4aa4fbb80f43f42a8334206ad1ad3b27aef577fd989f20d9",
+                "sha256:5d6b4af7ad7e4ac515bc6e602e7b79e2204e25dbd10ab3aa2beef3c5a9cad2c7",
+                "sha256:5decdc78849617917c206b01e9fc1d694fd58caa961be816cb37d3150d613d9a",
+                "sha256:632ecbbd2228575e6860c9e49ea3cc5423764d5aa70b92acc4e74096fb434044",
+                "sha256:65b998193bd7b0c7ecdfffbc825d808eac66279313cb67d8892bb259c9d91494",
+                "sha256:67093a526e42981fdd954868062e56c9b67fdd7e712616cc3265ad0c210ecb51",
+                "sha256:681eb4d37c9a9a6eb9b3245a5e89d7f7b2b9895590bb08a20aa598c1eb0a1d9d",
+                "sha256:69bd56b1d257a91e763256d63606937ae4eb890b18a789b66951c00062afec33",
+                "sha256:724c1fe135aa437d5126138d977004d165a3b5e2ee98fc4eb3e7c0ef645e7e27",
+                "sha256:7255a37ff50593c9b2f1afa8fafd6ef5763213c1ed5a9e2c6f5b9cc925ab979f",
+                "sha256:743cd768affaa508a21499f4858c5b824ffa2e1394ed94eb85caf47ac0732198",
+                "sha256:80d3bc9944be1d73e5b1726c3bbfd2628d3d7fe2880711b1eb90b617b9b8ac70",
+                "sha256:82ff356ff91be0ab2293fc6d8d262451eb6ac4fd999244c4b5f863e049ba219c",
+                "sha256:8e8607d8f6b4f9d46fee11447e334d6ab50e993dd4dbfb22f674616ce20907ab",
+                "sha256:97202f939c3ff341fc3fa84d15db86156b1edc669424ba20b0a1fcd4a796a045",
+                "sha256:9b01e7b00654115965a206e3015f0166674ec1e575198a62a977355597c0bef5",
+                "sha256:9fa621b3c0c05d965882c920347b6593751b7ab20d8fa81e426f1735ca1a9fc7",
+                "sha256:a1aa6e4cae8e3b8d5321be4f51c5ce77188faf7baa9fe1e78611f93a8eed2882",
+                "sha256:a2d30d6c1652140181dc6861f564449ad71a45e4f165a6868c27d36745b65d40",
+                "sha256:a649d0f66029c7eb67042b15374bd93a26aae202591d9afd71e111dd0006b198",
+                "sha256:a7854326920d41c3b5d468154318fe6ba4390cb2410480976787c640707e0180",
+                "sha256:a89acae02b2975b1f8e4974cb8cdf9bf9f6c91162fb8dec50c259ce700f2770a",
+                "sha256:a8bbdb166e2fb816e43ab034c865147edafe28e1b19c72433147789ac83e2dda",
+                "sha256:ac786f6cb7aa10d44e9641c7a7d16d7f6e095b138795cd43503769d4154e0dc2",
+                "sha256:b09bc62e5193e31d7f9876220fb429ec13a6a181a24d897b9edfbbdbcd678851",
+                "sha256:b10556817f09d46d420edd982dd0653940b90151d0576f09143a8e773459f6fe",
+                "sha256:b81076552d34c27e5149a40187a8f7e2abb2d3185576a317aaf14aeeedad862a",
+                "sha256:bdfc54b4468ed4cd7415928cbe782f4d782722a81aeb0f81e2ddca9932632211",
+                "sha256:cf6e7d5fe2aeb54898df18db1baf479863eae581cce05410f61f6b4188c8ada1",
+                "sha256:cf98038d2abf63a1ada5730e91e84c642ba6c225b0198c3684151b1f80c5f8a6",
+                "sha256:d24a9e61df7a7787b338a58abfba975414937b609eb6b18973e25f573bc0eeeb",
+                "sha256:d74ee72b5071818a1a5dab47338e87f08a738cb938a3b0653b9e4d959ddd1fd9",
+                "sha256:dd16302d39c4d6f4afde80edd0c97d4db643327d355a312762ccd9bd2ca515ed",
+                "sha256:dd2fb11922f58df8528adfca123f6a84748ad17d066007e7ac977720063556bd",
+                "sha256:deac4bdafa19bbb89edfb73b19f7f69a52d0b5bd3bb0c4ad404c1bbfd7b4b7fd",
+                "sha256:e03c3b8cc7883a54c3f34a6a135c4a17bc9088a33f36796acdb47162791b02f6",
+                "sha256:e1ec8a9ee0987d4524ffd6299e778c16cc35fef6d1a2764e609f90962f0b293a",
+                "sha256:e8603e691580487f11306ecb066c76f1f4a8b54fb3bdb23fa40643a059509366",
+                "sha256:f444762fed1bc1fd75187ef14a20ed900c1fbb245d45be9e834b822a0223bc81",
+                "sha256:f63600ec06982cdf480899026f4fda622776f5fabed9a869fdb32d72bc17e99a",
+                "sha256:fb62d517a516128bacf08cb6a86ecd39fb06d08e7c4980251f5d5601d29989ba"
             ],
             "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.17.5"
+            "version": "==3.17.6"
         },
         "six": {
             "hashes": [
@@ -647,6 +690,14 @@
             ],
             "version": "==1.4.7"
         },
+        "tomli": {
+            "hashes": [
+                "sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee",
+                "sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.2.2"
+        },
         "tqdm": {
             "hashes": [
                 "sha256:6baa75a88582b1db6d34ce4690da5501d2a1cb65c34664840a456b2c9f794d29",
@@ -666,27 +717,27 @@
         },
         "decorator": {
             "hashes": [
-                "sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323",
-                "sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5"
+                "sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374",
+                "sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==5.0.9"
+            "version": "==5.1.0"
         },
         "ipython": {
             "hashes": [
-                "sha256:54bbd1fe3882457aaf28ae060a5ccdef97f212a741754e420028d4ec5c2291dc",
-                "sha256:aa21412f2b04ad1a652e30564fff6b4de04726ce875eab222c8430edc6db383a"
+                "sha256:4f69d7423a5a1972f6347ff233e38bbf4df6a150ef20fbb00c635442ac3060aa",
+                "sha256:a658beaf856ce46bc453366d5dc6b2ddc6c481efd3540cb28aa3943819caac9f"
             ],
             "index": "pypi",
-            "version": "==7.25.0"
+            "version": "==7.29.0"
         },
         "jedi": {
             "hashes": [
-                "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93",
-                "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"
+                "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d",
+                "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.0"
+            "version": "==0.18.1"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -721,11 +772,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:6076e46efae19b1e0ca1ec003ed37a933dc94b4d20f486235d436e64771dcd5c",
-                "sha256:eb71d5a6b72ce6db177af4a7d4d7085b99756bf656d98ffcc4fecd36850eea6c"
+                "sha256:449f333dd120bd01f5d296a8ce1452114ba3a71fae7288d2f0ae2c918764fa72",
+                "sha256:48d85cdca8b6c4f16480c7ce03fd193666b62b0a21667ca56b4bb5ad679d1170"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.20"
+            "version": "==3.0.22"
         },
         "ptyprocess": {
             "hashes": [
@@ -744,11 +795,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:03f172516916220b58c9f19d7f854734136dd9528103d04e9bf139a92c9f54c4",
-                "sha256:bd382d7ea181fbbcce157c133db9a829ce06edffe097bcf3ab945b435452b46d"
+                "sha256:059f456c5a7c1c82b98c2e8c799f39c9b8128f6d0d46941ee118daace9eb70c7",
+                "sha256:2d313cc50a42cd6c277e7d7dc8d4d7fedd06a2c215f78766ae7b1a66277e0033"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.1.0"
+            "version": "==5.1.1"
         },
         "wcwidth": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2ee0fa8b4..3ce00ed1d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v3.0.24.3 - 2021-11-29 - [PR #488](https://github.com/NOAA-OWP/cahaba/pull/488)
+
+Fixed projection issue in `synthesize_test_cases.py`.
+
+## Changes
+
+- `Pipfile`: Added `Pyproj` to `Pipfile` to specify a version that did not have the current projection issues.
+
+<br/><br/>
+
 ## v3.0.24.2 - 2021-11-18 - [PR #486](https://github.com/NOAA-OWP/cahaba/pull/486)
 
 Adding a new check to keep `usgs_elev_table.csv`, `src_base.csv`, `small_segments.csv` for runs not using the `-viz` flag. We unintentionally deleted some .csv files in `vary_mannings_n_composite.py` but need to maintain some of these for non `-viz` runs (e.g. `usgs_elev_table.csv` is used for sierra test input).

From 605b95a6bf389a165c263c1e02a841bcaf31f9b5 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Tue, 1 Feb 2022 14:08:26 -0600
Subject: [PATCH 125/359] FIM version 4

See Changelog v4.0.0.0 for full details.

This resolves #523.
---
 .gitignore                                    |   4 +-
 CHANGELOG.md                                  | 635 +++++++++++
 CITATION.cff                                  |  11 +
 Dockerfile                                    |   7 +-
 Dockerfile.prod                               | 103 ++
 Pipfile                                       |   4 +
 Pipfile.lock                                  | 411 +++++---
 README.md                                     |   7 +-
 config/deny_gms_branches_default.lst          |  58 +
 config/deny_gms_branches_min.lst              |  58 +
 config/deny_gms_unit_default.lst              |  18 +
 ....json => mannings_default_calibrated.json} |   0
 config/params_template.env                    |   5 +
 config/params_template_calibrated.env         |  46 +
 docs/CHANGELOG.md                             |  15 +
 gms_pipeline.sh                               |  54 +
 gms_run_branch.sh                             | 137 +++
 gms_run_unit.sh                               | 161 +++
 lib/aggregate_fim_outputs.py                  | 174 +++
 lib/fr_to_ms_raster_mask.py                   |  83 ++
 src/acquire_and_preprocess_inputs.py          |  63 +-
 src/add_crosswalk.py                          |  24 +-
 src/bathy_rc_adjust.py                        | 155 +++
 src/check_huc_inputs.py                       |   2 +-
 src/clip_vectors_to_wbd.py                    |  10 +-
 src/filter_catchments_and_add_attributes.py   |  18 +-
 src/gms/__init__.py                           |   0
 src/gms/aggregate_branch_lists.py             |  51 +
 src/gms/buffer_stream_branches.py             |  34 +
 src/gms/clip_rasters_to_branches.py           |  37 +
 src/gms/crosswalk_nwm_demDerived.py           | 158 +++
 src/gms/derive_level_paths.py                 | 168 +++
 src/gms/edit_points.py                        |  54 +
 src/gms/filter_gms_inputs_by_huc.py           |  33 +
 src/gms/finalize_srcs.py                      |  67 ++
 src/gms/generate_branch_list.py               |  29 +
 src/gms/outputs_cleanup.py                    |  47 +
 src/gms/query_vectors_by_branch_polygons.py   |  41 +
 src/gms/rem.py                                | 124 +++
 src/gms/reset_mannings.py                     | 209 ++++
 src/gms/run_by_branch.sh                      | 294 ++++++
 src/gms/run_by_unit.sh                        |  94 ++
 src/gms/stream_branches.py                    | 992 ++++++++++++++++++
 src/gms/subset_catch_list_by_branch_id.py     |  70 ++
 src/gms/test_new_crosswalk.sh                 |   4 +
 src/gms/time_and_tee_run_by_branch.sh         |  11 +
 src/gms/time_and_tee_run_by_unit.sh           |  12 +
 src/gms/toDo.md                               |  33 +
 src/make_stages_and_catchlist.py              |  10 +
 src/preprocess_rasters.py                     |  44 +-
 src/utils/reproject_dem.py                    |   8 +-
 src/utils/shared_functions.py                 |  59 +-
 src/utils/shared_variables.py                 |   9 +-
 tests/aggregate_metrics.py                    | 271 +++++
 tests/cache_metrics.py                        | 118 +++
 tests/run_test_case.py                        | 248 +++++
 tests/synthesize_test_cases.py                | 311 ++++++
 tests/utils/shared_functions.py               | 676 ++++++++++++
 tools/compare_gms_srcs_to_fr.py               | 156 +++
 tools/consolidate_metrics.py                  | 322 ++++++
 tools/cygnss_preprocessing.py                 | 228 ++++
 tools/gms_tools/__init__.py                   |   0
 .../compare_ms_and_non_ms_metrics.py          |  31 +
 tools/gms_tools/compile_comp_stats.py         |  75 ++
 .../gms_tools/compile_computational_stats.py  |  32 +
 tools/gms_tools/evaluate_continuity.py        | 136 +++
 tools/gms_tools/find_max_catchment_breadth.py |  73 ++
 tools/gms_tools/inundate_gms.py               |   3 +-
 tools/gms_tools/mosaic_inundation.py          |   6 +-
 tools/gms_tools/overlapping_inundation.py     |   4 +-
 tools/gms_tools/plots.py                      | 254 +++++
 tools/inundate_events.py                      |  98 ++
 tools/inundation.py                           | 158 ++-
 tools/make_boxes_from_bounds.py               |  74 ++
 tools/nesdis_preprocessing.py                 | 142 +++
 tools/plots/eval_plots.py                     | 343 ++++++
 tools/plots/utils/__init__.py                 |   0
 tools/plots/utils/shared_functions.py         | 321 ++++++
 tools/run_test_case.py                        | 218 +++-
 tools/shared_variables.py                     |  20 +
 tools/synthesize_test_cases.py                | 182 ++--
 tools/tools_shared_functions.py               |  66 +-
 82 files changed, 9160 insertions(+), 361 deletions(-)
 create mode 100644 CHANGELOG.md
 create mode 100644 CITATION.cff
 create mode 100644 Dockerfile.prod
 create mode 100644 config/deny_gms_branches_default.lst
 create mode 100644 config/deny_gms_branches_min.lst
 create mode 100644 config/deny_gms_unit_default.lst
 rename config/{mannings_calibrated.json => mannings_default_calibrated.json} (100%)
 create mode 100644 config/params_template_calibrated.env
 create mode 100755 gms_pipeline.sh
 create mode 100755 gms_run_branch.sh
 create mode 100755 gms_run_unit.sh
 create mode 100644 lib/aggregate_fim_outputs.py
 create mode 100755 lib/fr_to_ms_raster_mask.py
 create mode 100755 src/bathy_rc_adjust.py
 create mode 100644 src/gms/__init__.py
 create mode 100755 src/gms/aggregate_branch_lists.py
 create mode 100755 src/gms/buffer_stream_branches.py
 create mode 100755 src/gms/clip_rasters_to_branches.py
 create mode 100755 src/gms/crosswalk_nwm_demDerived.py
 create mode 100755 src/gms/derive_level_paths.py
 create mode 100755 src/gms/edit_points.py
 create mode 100755 src/gms/filter_gms_inputs_by_huc.py
 create mode 100755 src/gms/finalize_srcs.py
 create mode 100755 src/gms/generate_branch_list.py
 create mode 100755 src/gms/outputs_cleanup.py
 create mode 100755 src/gms/query_vectors_by_branch_polygons.py
 create mode 100755 src/gms/rem.py
 create mode 100755 src/gms/reset_mannings.py
 create mode 100755 src/gms/run_by_branch.sh
 create mode 100755 src/gms/run_by_unit.sh
 create mode 100755 src/gms/stream_branches.py
 create mode 100755 src/gms/subset_catch_list_by_branch_id.py
 create mode 100755 src/gms/test_new_crosswalk.sh
 create mode 100755 src/gms/time_and_tee_run_by_branch.sh
 create mode 100755 src/gms/time_and_tee_run_by_unit.sh
 create mode 100644 src/gms/toDo.md
 create mode 100644 tests/aggregate_metrics.py
 create mode 100644 tests/cache_metrics.py
 create mode 100755 tests/run_test_case.py
 create mode 100644 tests/synthesize_test_cases.py
 create mode 100644 tests/utils/shared_functions.py
 create mode 100755 tools/compare_gms_srcs_to_fr.py
 create mode 100755 tools/consolidate_metrics.py
 create mode 100755 tools/cygnss_preprocessing.py
 create mode 100644 tools/gms_tools/__init__.py
 create mode 100644 tools/gms_tools/compare_ms_and_non_ms_metrics.py
 create mode 100755 tools/gms_tools/compile_comp_stats.py
 create mode 100755 tools/gms_tools/compile_computational_stats.py
 create mode 100644 tools/gms_tools/evaluate_continuity.py
 create mode 100755 tools/gms_tools/find_max_catchment_breadth.py
 mode change 100644 => 100755 tools/gms_tools/inundate_gms.py
 mode change 100644 => 100755 tools/gms_tools/mosaic_inundation.py
 create mode 100755 tools/gms_tools/plots.py
 create mode 100755 tools/inundate_events.py
 create mode 100755 tools/make_boxes_from_bounds.py
 create mode 100755 tools/nesdis_preprocessing.py
 create mode 100755 tools/plots/eval_plots.py
 create mode 100644 tools/plots/utils/__init__.py
 create mode 100644 tools/plots/utils/shared_functions.py
 create mode 100644 tools/shared_variables.py

diff --git a/.gitignore b/.gitignore
index e18bb6432..47137b619 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,12 +1,14 @@
 *.pyc
 *.swp
+*.swo
+*.swn
 .nfs*
 __pycache__/
 config/**
 !config/*template*
 !config/*default*
-!config/*calibrated*
 !config/symbology/
+config/*.swp
 .vscode/
 **/.DS_Store
 **/*_pytest.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 000000000..af6f11b61
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,635 @@
+All notable changes to this project will be documented in this file.
+We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+<br/><br/>
+
+## v3.0.19.3 - 2021-07-09
+
+Hot fix to `synthesize_test_cases`.
+
+## Changes
+- Fixed if/elif/else statement in `synthesize_test_cases.py` that resulted in only IFC data being evaluated.
+
+<br/><br/>
+
+## v3.0.19.2 - 2021-07-01 - [PR #429](https://github.com/NOAA-OWP/cahaba/pull/429)
+
+Updates to evaluation scripts to allow for Alpha testing at Iowa Flood Center (IFC) sites. Also, `BAD_SITES` variable updates to omit sites not suitable for evaluation from metric calculations.
+
+## Changes
+- The `BAD_SITES` list in `tools_shared_variables.py` was updated and reasons for site omission are documented.
+- Refactored `run_test_case.py`, `synthesize_test_cases.py`, `tools_shared_variables.py`, and `eval_plots.py` to allow for IFC comparisons.
+
+<br/><br/>
+
+## v3.0.19.1 - 2021-06-17 - [PR #417](https://github.com/NOAA-OWP/cahaba/pull/417)
+
+Adding a thalweg profile tool to identify significant drops in thalweg elevation. Also setting lateral thalweg adjustment threshold in hydroconditioning.
+
+## Additions
+- `thalweg_drop_check.py` checks the elevation along the thalweg for each stream path downstream of MS headwaters within a HUC.
+
+## Removals
+- Removing `dissolveLinks` arg from `clip_vectors_to_wbd.py`.
+
+
+## Changes
+- Cleaned up code in `split_flows.py` to make it more readable.
+- Refactored `reduce_nhd_stream_density.py` and `adjust_headwater_streams.py` to limit MS headwater points in `agg_nhd_headwaters_adj.gpkg`.
+- Fixed a bug in `adjust_thalweg_lateral.py` lateral elevation replacement threshold; changed threshold to 3 meters.
+- Updated `aggregate_vector_inputs.py` to log intermediate processes.
+
+<br/><br/>
+
+## v3.0.19.0 - 2021-06-10 - [PR #415](https://github.com/NOAA-OWP/cahaba/pull/415)
+
+Feature to evaluate performance of alternative CatFIM techniques.
+
+## Additions
+- Added `eval_catfim_alt.py` to evaluate performance of alternative CatFIM techniques.
+
+<br/><br/>
+## v3.0.18.0 - 2021-06-09 - [PR #404](https://github.com/NOAA-OWP/cahaba/pull/404)
+
+To help analyze the memory consumption of the Fim Run process, the python module `memory-profiler` has been added to give insights into where peak memory usage is with in the codebase.
+
+In addition, the Dockerfile was previously broken due to the Taudem dependency removing the version that was previously being used by FIM. To fix this, and allow new docker images to be built, the Taudem version has been updated to the newest version on the Github repo and thus needs to be thoroughly tested to determine if this new version has affected the overall FIM outputs.
+
+## Additions
+- Added `memory-profiler` to `Pipfile` and `Pipfile.lock`.
+- Added `mprof` (memory-profiler cli utility) call to the `time_and_tee_run_by_unit.sh` to create overall memory usage graph location in the `/logs/{HUC}_memory.png` of the outputs directory.
+- Added `@profile` decorator to all functions within scripts used in the `run_by_unit.sh` script to allow for memory usage tracking, which is then recorded in the `/logs/{HUC}.log` file of the outputs directory.
+
+## Changes
+- Changed the Taudem version in `Dockerfile.dev` to `98137bb6541a0d0077a9c95becfed4e56d0aa0ac`.
+- Changed all calls of python scripts in `run_by_unit.s` to be called with the `-m memory-profiler` argument to allow scripts to also track memory usage.
+
+<br/><br/>
+## v3.0.17.1 - 2021-06-04 - [PR #395](https://github.com/NOAA-OWP/cahaba/pull/395)
+
+Bug fix to the `generate_nws_lid.py` script
+
+## Changes
+- Fixes incorrectly assigned attribute field "is_headwater" for some sites in the `nws_lid.gpkg` layer.
+- Updated `agg_nhd_headwaters_adj.gpkg`, `agg_nhd_streams_adj.gpkg`, `nwm_flows.gpkg`, and `nwm_catchments.gpkg` input layers using latest NWS LIDs.
+
+<br/><br/>
+## v3.0.17.0 - 2021-06-04 - [PR #393](https://github.com/NOAA-OWP/cahaba/pull/393)
+BARC updates to cap the bathy calculated xsec area in `bathy_rc_adjust.py` and allow user to choose input bankfull geometry.
+
+## Changes
+
+- Added new env variable to control which input file is used for the bankfull geometry input to bathy estimation workflow.
+- Modified the bathymetry cross section area calculation to cap the additional area value so that it cannot exceed the bankfull cross section area value for each stream segment (bankfull value obtained from regression equation dataset).
+- Modified the `rating_curve_comparison.py` plot output to always put the FIM rating curve on top of the USGS rating curve (avoids USGS points covering FIM).
+- Created a new aggregate csv file (aggregates for all hucs) for all of the `usgs_elev_table.csv` files (one per huc).
+- Evaluate the FIM Bathymetry Adjusted Rating Curve (BARC) tool performance using the estimated bankfull geometry dataset derived for the NWM route link dataset.
+
+<br/><br/>
+## v3.0.16.3 - 2021-05-21 - [PR #388](https://github.com/NOAA-OWP/cahaba/pull/388)
+
+Enhancement and bug fixes to `synthesize_test_cases.py`.
+
+## Changes
+- Addresses a bug where AHPS sites without benchmark data were receiving a CSI of 0 in the master metrics CSV produced by `synthesize_test_cases.py`.
+- Includes a feature enhancement to `synthesize_test_cases.py` that allows for the inclusion of user-specified testing versions in the master metrics CSV.
+- Removes some of the print statements used by `synthesize_test_cases.py`.
+
+<br/><br/>
+## v3.0.16.2 - 2021-05-18 - [PR #384](https://github.com/NOAA-OWP/cahaba/pull/384)
+
+Modifications and fixes to `run_test_case.py`, `eval_plots.py`, and AHPS preprocessing scripts.
+
+## Changes
+- Comment out return statement causing `run_test_case.py` to skip over sites/hucs when calculating contingency rasters.
+- Move bad sites list and query statement used to filter out bad sites to the `tools_shared_variables.py`.
+- Add print statements in `eval_plots.py` detailing the bad sites used and the query used to filter out bad sites.
+- Update AHPS preprocessing scripts to produce a domain shapefile.
+- Change output filenames produced in ahps preprocessing scripts.
+- Update workarounds for some sites in ahps preprocessing scripts.
+
+<br/><br/>
+## v3.0.16.1 - 2021-05-11 - [PR #380](https://github.com/NOAA-OWP/cahaba/pull/380)
+
+The current version of Eventlet used in the Connector module of the FIM API is outdated and vulnerable. This update bumps the version to the patched version.
+
+## Changes
+- Updated `api/node/connector/requirements.txt` to have the Eventlet version as 0.31.0
+
+<br/><br/>
+## v3.0.16.0 - 2021-05-07 - [PR #378](https://github.com/NOAA-OWP/cahaba/pull/378)
+
+New "Release" feature added to the FIM API. This feature will allow for automated FIM, CatFIM, and relevant metrics to be generated when a new FIM Version is released. See [#373](https://github.com/NOAA-OWP/cahaba/issues/373) for more detailed steps that take place in this feature.
+
+## Additions
+- Added new window to the UI in `api/frontend/gui/templates/index.html`.
+- Added new job type to `api/node/connector/connector.py` to allow these release jobs to run.
+- Added additional logic in `api/node/updater/updater.py` to run the new eval and CatFIM scripts used in the release feature.
+
+## Changes
+- Updated `api/frontend/output_handler/output_handler.py` to allow for copying more broad ranges of file paths instead of only the `/data/outputs` directory.
+
+<br/><br/>
+## v3.0.15.10 - 2021-05-06 - [PR #375](https://github.com/NOAA-OWP/cahaba/pull/375)
+
+Remove Great Lakes coastlines from WBD buffer.
+
+## Changes
+- `gl_water_polygons.gpkg` layer is used to mask out Great Lakes boundaries and remove NHDPlus HR coastline segments.
+
+<br/><br/>
+## v3.0.15.9 - 2021-05-03 - [PR #372](https://github.com/NOAA-OWP/cahaba/pull/372)
+
+Generate `nws_lid.gpkg`.
+
+## Additions
+- Generate `nws_lid.gpkg` with attributes indicating if site is a headwater `nws_lid` as well as if it is co-located with another `nws_lid` which is referenced to the same `nwm_feature_id` segment.
+
+<br/><br/>
+## v3.0.15.8 - 2021-04-29 - [PR #371](https://github.com/NOAA-OWP/cahaba/pull/371)
+
+Refactor NHDPlus HR preprocessing workflow. Resolves issue #238
+
+## Changes
+- Consolidate NHD streams, NWM catchments, and headwaters MS and FR layers with `mainstem` column.
+- HUC8 intersections are included in the input headwaters layer.
+- `clip_vectors_to_wbd.py` removes incoming stream segment from the selected layers.
+
+<br/><br/>
+## v3.0.15.7 - 2021-04-28 - [PR #367](https://github.com/NOAA-OWP/cahaba/pull/367)
+
+Refactor synthesize_test_case.py to handle exceptions during multiprocessing. Resolves issue #351
+
+## Changes
+- refactored `inundation.py` and `run_test_case.py` to handle exceptions without using `sys.exit()`.
+
+<br/><br/>
+## v3.0.15.6 - 2021-04-23 - [PR #365](https://github.com/NOAA-OWP/cahaba/pull/365)
+
+Implement CatFIM threshold flows to Sierra test and add AHPS benchmark preprocessing scripts.
+
+## Additions
+- Produce CatFIM flows file when running `rating_curve_get_usgs_gages.py`.
+- Several scripts to preprocess AHPS benchmark data. Requires numerous file dependencies not available through Cahaba.
+
+## Changes
+- Modify `rating_curve_comparison.py` to ingest CatFIM threshold flows in calculations.
+- Modify `eval_plots.py` to save all site specific bar plots in same parent directory instead of in subdirectories.
+- Add variables to `env.template` for AHPS benchmark preprocessing.
+
+<br/><br/>
+## v3.0.15.5 - 2021-04-20 - [PR #363](https://github.com/NOAA-OWP/cahaba/pull/363)
+
+Prevent eval_plots.py from erroring out when spatial argument enabled if certain datasets not analyzed.
+
+## Changes
+- Add check to make sure analyzed dataset is available prior to creating spatial dataset.
+
+<br/><br/>
+## v3.0.15.4 - 2021-04-20 - [PR #356](https://github.com/NOAA-OWP/cahaba/pull/356)
+
+Closing all multiprocessing Pool objects in repo.
+
+<br/><br/>
+## v3.0.15.3 - 2021-04-19 - [PR #358](https://github.com/NOAA-OWP/cahaba/pull/358)
+
+Preprocess NHDPlus HR rasters for consistent projections, nodata values, and convert from cm to meters.
+
+## Additions
+- `preprocess_rasters.py` reprojects raster, converts to meters, and updates nodata value to -9999.
+- Cleaned up log messages from `bathy_rc_adjust.py` and `usgs_gage_crosswalk.py`.
+- Outputs paths updated in `generate_categorical_fim_mapping.py` and `generate_categorical_fim.py`.
+- `update_raster_profile` cleans up raster crs, blocksize, nodata values, and converts elevation grids from cm to meters.
+- `reproject_dem.py` imports gdal to reproject elevation rasters because an error was occurring when using rasterio.
+
+## Changes
+- `burn_in_levees.py` replaces the `gdal_calc.py` command to resolve inconsistent outputs with burned in levee values.
+
+<br/><br/>
+## v3.0.15.2 - 2021-04-16 - [PR #359](https://github.com/NOAA-OWP/cahaba/pull/359)
+
+Hotfix to preserve desired files when production flag used in `fim_run.sh`.
+
+## Changes
+
+- Fixed production whitelisted files.
+
+<br/><br/>
+## v3.0.15.1 - 2021-04-13 - [PR #355](https://github.com/NOAA-OWP/cahaba/pull/355)
+
+Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field.
+
+## Changes
+
+- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "MS" run, only consider gages that contain rating curve information (via `curve` attribute) and are also mainstems gages (via `mainstems` attribute).
+- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "FR" run, only consider gages that contain rating curve information (via `curve` attribute) and are not mainstems gages (via `mainstems` attribute).
+- Modifies how mainstems segments are determined by using the `nwm_flows_ms.gpkg` as a lookup to determine if the NWM segment specified by WRDS for a gage site is a mainstems gage.
+
+## Additions
+
+- Adds a `mainstem` attribute field to `usgs_gages.gpkg` that indicates whether a gage is located on a mainstems river.
+- Adds `NWM_FLOWS_MS` variable to the `.env` and `.env.template` files.
+- Adds the `extent` argument specified by user when running `fim_run.sh` to `usgs_gage_crosswalk.py`.
+
+<br/><br/>
+## v3.0.15.0 - 2021-04-08 - [PR #340](https://github.com/NOAA-OWP/cahaba/pull/340)
+
+Implementing a prototype technique to estimate the missing bathymetric component in the HAND-derived synthetic rating curves. The new Bathymetric Adjusted Rating Curve (BARC) function is built within the `fim_run.sh` workflow and will ingest bankfull geometry estimates provided by the user to modify the cross section area used in the synthetic rating curve generation.
+
+### Changes
+ - `add_crosswalk.py` outputs the stream order variables to `src_full_crosswalked.csv` and calls the new `bathy_rc_adjust.py` if bathy env variable set to True and `extent=MS`.
+ - `run_by_unit.sh` includes a new csv outputs for reviewing BARC calculations.
+ - `params_template.env` & `params_calibrated.env` contain new BARC function input variables and on/off toggle variable.
+ - `eval_plots.py` now includes additional AHPS eval sites in the list of "bad_sites" (flagged issues with MS flowlines).
+
+### Additions
+ - `bathy_rc_adjust.py`:
+    - Imports the existing synthetic rating curve table and the bankfull geometry input data (topwidth and cross section area per COMID).
+    - Performs new synthetic rating curve calculations with bathymetry estimation modifications.
+    - Flags issues with the thalweg-notch artifact.
+
+<br/><br/>
+## v3.0.14.0 - 2021-04-05 - [PR #338](https://github.com/NOAA-OWP/cahaba/pull/338)
+
+Create tool to retrieve rating curves from USGS sites and convert to elevation (NAVD88). Intended to be used as part of the Sierra Test.
+
+### Changes
+ - Modify `usgs_gage_crosswalk.py` to:
+    1) Look for `location_id` instead of `site_no` attribute field in `usgs_gages.gpkg` file.
+    2) Filter out gages that do not have rating curves included in the `usgs_rating_curves.csv`.
+ - Modify `rating_curve_comparison.py` to perform a check on the age of the user specified `usgs_rating_curves.csv` and alert user to the age of the file and recommend updating if file is older the 30 days.
+
+### Additions
+ - Add `rating_curve_get_usgs_curves.py`. This script will generate the following files:
+     1) `usgs_rating_curves.csv`: A csv file that contains rating curves (including converted to NAVD88 elevation) for USGS gages in a format that is compatible with  `rating_curve_comparisons.py`. As it is is currently configured, only gages within CONUS will have rating curve data.
+     2) `log.csv`: A log file that records status for each gage and includes error messages.
+     3) `usgs_gages.gpkg`: A geospatial layer (in FIM projection) of all active USGS gages that meet a predefined criteria. Additionally, the `curve` attribute indicates whether a rating curve is found in the `usgs_rating_curves.csv`. This spatial file is only generated if the `all` option is passed with the `-l` argument.
+
+<br/><br/>
+## v3.0.13.0 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
+
+Created tool to compare synthetic rating curve with benchmark rating curve (Sierra Test).
+
+### Changes
+ - Update `aggregate_fim_outputs.py` call argument in `fim_run.sh` from 4 jobs to 6 jobs, to optimize API performance.
+ - Reroutes median elevation data from `add_crosswalk.py` and `rem.py` to new file (depreciating `hand_ref_elev_table.csv`).
+ - Adds new files to `viz_whitelist` in `output_cleanup.py`.
+
+### Additions
+ - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in `run_by_unit.py` with elevation and additional attributes at USGS gages.
+ - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data.
+
+<br/><br/>
+## v3.0.12.1 - 2021-03-31 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
+
+Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
+
+### Changes
+ - Removes file dependencies from spatial option. Does require the WBD layer which should be specified in `.env` file.
+ - Produces outputs in a format consistent with requirements needed for publishing.
+ - Preserves leading zeros in huc information for all outputs from `eval_plots.py`.
+
+### Additions
+ - Creates `fim_performance_points.shp`: this layer consists of all evaluated ahps points (with metrics). Spatial data retrieved from WRDS on the fly.
+ - Creates `fim_performance_polys.shp`: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.
+
+<br/><br/>
+## v3.0.12.0 - 2021-03-26 - [PR #327](https://github.com/NOAA-OWP/cahaba/pull/237)
+
+Add more detail/information to plotting capabilities.
+
+### Changes
+ - Merge `plot_functions.py` into `eval_plots.py` and move `eval_plots.py` into the tools directory.
+ - Remove `plots` subdirectory.
+
+### Additions
+ - Optional argument to create barplots of CSI for each individual site.
+ - Create a csv containing the data used to create the scatterplots.
+
+<br/><br/>
+## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
+
+Improvements to CatFIM service source data generation.
+
+### Changes
+ - Renamed `generate_categorical_fim.py` to `generate_categorical_fim_mapping.py`.
+ - Updated the status outputs of the `nws_lid_sites layer` and saved it in the same directory as the `merged catfim_library layer`.
+ - Additional stability fixes (such as improved compatability with WRDS updates).
+
+### Additions
+ - Added `generate_categorical_fim.py` to wrap `generate_categorical_fim_flows.py` and `generate_categorical_fim_mapping.py`.
+ - Create new `nws_lid_sites` shapefile located in same directory as the `catfim_library` shapefile.
+
+<br/><br/>
+## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
+
+Patch to synthesize_test_cases.py.
+
+### Changes
+ - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
+
+<br/><br/>
+## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
+
+Preprocessing of flow files for Categorical FIM.
+
+### Additions
+ - Generate Categorical FIM flow files for each category (action, minor, moderate, major).
+ - Generate point shapefile of Categorical FIM sites.
+ - Generate csv of attribute data in shapefile.
+ - Aggregate all shapefiles and csv files into one file in parent directory.
+ - Add flood of record category.
+
+ ### Changes
+ - Stability fixes to `generate_categorical_fim.py`.
+
+<br/><br/>
+## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)
+
+Enhancements to FIM API.
+
+### Changes
+ - `fim_run.sh` can now be run with jobs in parallel.
+ - Viz post-processing can now be selected in API interface.
+ - Jobs table shows jobs that end with errors.
+ - HUC preset lists can now be selected in interface.
+ - Better `output_handler` file writing.
+ - Overall better restart and retry handlers for networking problems.
+ - Jobs can now be canceled in API interface.
+ - Both FR and MS configs can be selected for a single job.
+
+<br/><br/>
+## v3.0.8.2 - 2021-03-11 - [PR #296](https://github.com/NOAA-OWP/cahaba/pull/296)
+
+Enhancements to post-processing for Viz-related use-cases.
+
+### Changes
+ - Aggregate grids are projected to Web Mercator during `-v` runs in `fim_run.sh`.
+ - HUC6 aggregation is parallelized.
+ - Aggregate grid blocksize is changed from 256 to 1024 for faster postprocessing.
+
+<br/><br/>
+## v3.0.8.1 - 2021-03-10 - [PR #302](https://github.com/NOAA-OWP/cahaba/pull/302)
+
+Patched import issue in `tools_shared_functions.py`.
+
+### Changes
+ - Changed `utils.` to `tools_` in `tools_shared_functions.py` after recent structural change to `tools` directory.
+
+<br/><br/>
+## v3.0.8.0 - 2021-03-09 - [PR #279](https://github.com/NOAA-OWP/cahaba/pull/279)
+
+Refactored NWS Flood Categorical HAND FIM (CatFIM) pipeline to open source.
+
+### Changes
+ - Added `VIZ_PROJECTION` to `shared_variables.py`.
+ - Added missing library referenced in `inundation.py`.
+ - Cleaned up and converted evaluation scripts in `generate_categorical_fim.py` to open source.
+ - Removed `util` folders under `tools` directory.
+
+<br/><br/>
+## v3.0.7.1 - 2021-03-02 - [PR #290](https://github.com/NOAA-OWP/cahaba/pull/290)
+
+Renamed benchmark layers in `test_cases` and updated variable names in evaluation scripts.
+
+### Changes
+ - Updated `run_test_case.py` with new benchmark layer names.
+ - Updated `run_test_case_calibration.py` with new benchmark layer names.
+
+<br/><br/>
+## v3.0.7.0 - 2021-03-01 - [PR #288](https://github.com/NOAA-OWP/cahaba/pull/288)
+
+Restructured the repository. This has no impact on hydrological work done in the codebase and is simply moving files and renaming directories.
+
+### Changes
+ - Moved the contents of the `lib` folder to a new folder called `src`.
+ - Moved the contents of the `tests` folder to the `tools` folder.
+ - Changed any instance of `lib` or `libDir` to `src` or `srcDir`.
+
+<br/><br/>
+## v3.0.6.0 - 2021-02-25 - [PR #276](https://github.com/NOAA-OWP/cahaba/pull/276)
+
+Enhancement that creates metric plots and summary statistics using metrics compiled by `synthesize_test_cases.py`.
+
+### Additions
+ - Added `eval_plots.py`, which produces:
+    - Boxplots of CSI, FAR, and POD/TPR
+    - Barplot of aggregated CSI scores
+    - Scatterplot of CSI comparing two FIM versions
+    - CSV of aggregated statistics (CSI, FAR, POD/TPR)
+    - CSV of analyzed data and analyzed sites
+
+<br/><br/>
+## v3.0.5.3 - 2021-02-23 - [PR #275](https://github.com/NOAA-OWP/cahaba/pull/275)
+
+Bug fixes to new evaluation code.
+
+### Changes
+
+ - Fixed a bug in `synthesize_test_cases.py` where the extent (MS/FR) was not being written to merged metrics file properly.
+ - Fixed a bug in `synthesize_test_cases.py` where only BLE test cases were being written to merged metrics file.
+ - Removed unused imports from `inundation.py`.
+ - Updated README.md
+
+<br/><br/>
+## v3.0.5.2 - 2021-02-23 - [PR #272](https://github.com/NOAA-OWP/cahaba/pull/272)
+
+Adds HAND synthetic rating curve (SRC) datum elevation values to `hydroTable.csv` output.
+
+### Changes
+
+ - Updated `add_crosswalk.py` to included "Median_Thal_Elev_m" variable outputs in `hydroTable.csv`.
+ - Renamed hydroid attribute in `rem.py` to "Median" in case we want to include other statistics in the future (e.g. min, max, range etc.).
+
+<br/><br/>
+## v3.0.5.1 - 2021-02-22
+
+Fixed `TEST_CASES_DIR` path in `tests/utils/shared_variables.py`.
+
+### Changes
+
+ - Removed `"_new"` from `TEST_CASES_DIR` variable.
+
+<br/><br/>
+## v3.0.5.0 - 2021-02-22 - [PR #267](https://github.com/NOAA-OWP/cahaba/pull/267)
+
+Enhancements to allow for evaluation at AHPS sites, the generation of a query-optimized metrics CSV, and the generation of categorical FIM. This merge requires that the `/test_cases` directory be updated for all machines performing evaluation.
+
+### Additions
+
+ - `generate_categorical_fim.py` was added to allow production of NWS Flood Categorical HAND FIM (CatFIM) source data. More changes on this script are to follow in subsequent branches.
+
+### Removals
+
+ - `ble_autoeval.sh` and `all_ble_stats_comparison.py` were deleted because `synthesize_test_cases.py` now handles the merging of metrics.
+ - The code block in `run_test_case.py` that was responsible for printing the colored metrics to screen has been commented out because of the new scale of evaluations (formerly in `run_test_case.py`, now in `shared_functions.py`)
+ - Remove unused imports from inundation wrappers in `/tools`.
+
+### Changes
+
+ - Updated `synthesize_test_cases.py` to allow for AHPS site evaluations.
+ - Reorganized `run_test_case.py` by moving more functions into `shared_functions.py`.
+ - Created more shared variables in `shared_variables.py` and updated import statements in relevant scripts.
+
+<br/><br/>
+
+## v3.0.4.4 - 2021-02-19 - [PR #266](https://github.com/NOAA-OWP/cahaba/pull/266)
+
+Rating curves for short stream segments are replaced with rating curves from upstream/downstream segments.
+
+### Changes
+
+ - Short stream segments are identified and are reassigned the channel geometry from upstream/downstream segment.
+ - `fossid` renamed to `fimid` and the attribute's starting value is now 1000 to avoid HydroIDs with leading zeroes.
+ - Addresses issue where HydroIDs were not included in final hydrotable.
+ - Added `import sys` to `inundation.py` (missing from previous feature branch).
+ - Variable names and general workflow are cleaned up.
+
+<br/><br/>
+## v3.0.4.3 - 2021-02-12 - [PR #254](https://github.com/NOAA-OWP/cahaba/pull/254)
+
+Modified `rem.py` with a new function to output HAND reference elev.
+
+### Changes
+
+ - Function `make_catchment_hydroid_dict` creates a df of pixel catchment ids and overlapping hydroids.
+ - Merge hydroid df and thalweg minimum elevation df.
+ - Produces new output containing all catchment ids and min thalweg elevation value named `hand_ref_elev_table.csv`.
+ - Overwrites the `demDerived_reaches_split.gpk` layer by adding additional attribute `Min_Thal_Elev_meters` to view the elevation value for each hydroid.
+
+<br/><br/>
+## v3.0.4.2 - 2021-02-12 - [PR #255](https://github.com/NOAA-OWP/cahaba/pull/255)
+
+Addresses issue when running on HUC6 scale.
+
+### Changes
+
+ - `src.json` should be fixed and slightly smaller by removing whitespace.
+ - Rasters are about the same size as running fim as huc6 (compressed and tiled; aggregated are slightly larger).
+ - Naming convention and feature id attribute are only added to the aggregated hucs.
+ - HydroIDs are different for huc6 vs aggregated huc8s mostly due to forced split at huc boundaries (so long we use consistent workflow it shouldn't matter).
+ - Fixed known issue where sometimes an incoming stream is not included in the final selection will affect aggregate outputs.
+
+<br/><br/>
+## v3.0.4.1 - 2021-02-12 - [PR #261](https://github.com/NOAA-OWP/cahaba/pull/261)
+
+Updated MS Crosswalk method to address gaps in FIM.
+
+### Changes
+
+ - Fixed typo in stream midpoint calculation in `split_flows.py` and `add_crosswalk.py`.
+ - `add_crosswalk.py` now restricts the MS crosswalk to NWM MS catchments.
+ - `add_crosswalk.py` now performs a secondary MS crosswalk selection by nearest NWM MS catchment.
+
+<br/><br/>
+## v3.0.4.0 - 2021-02-10 - [PR #256](https://github.com/NOAA-OWP/cahaba/pull/256)
+
+New python script "wrappers" for using `inundation.py`.
+
+### Additions
+
+ - Created `inundation_wrapper_nwm_flows.py` to produce inundation outputs using NWM recurrence flows: 1.5 year, 5 year, 10 year.
+ - Created `inundation_wrapper_custom_flow.py` to produce inundation outputs with user-created flow file.
+ - Created new `tools` parent directory to store `inundation_wrapper_nwm_flows.py` and  `inundation_wrapper_custom_flow.py`.
+
+<br/><br/>
+## v3.0.3.1 - 2021-02-04 - [PR #253](https://github.com/NOAA-OWP/cahaba/pull/253)
+
+Bug fixes to correct mismatched variable name and file path.
+
+### Changes
+
+ - Corrected variable name in `fim_run.sh`.
+ - `acquire_and_preprocess_inputs.py` now creates `huc_lists` folder and updates file path.
+
+<br/><br/>
+## v3.0.3.0 - 2021-02-04 - [PR #227](https://github.com/NOAA-OWP/cahaba/pull/227)
+
+Post-process to aggregate FIM outputs to HUC6 scale.
+
+### Additions
+
+ - Viz outputs aggregated to HUC6 scale; saves outputs to `aggregate_fim_outputs` folder.
+
+### Changes
+
+ - `split_flows.py` now splits streams at HUC8 boundaries to ensure consistent catchment boundaries along edges.
+ - `aggregate_fim_outputs.sh` has been depreciated but remains in the repo for potential FIM 4 development.
+ - Replaced geopandas driver arg with getDriver throughout repo.
+ - Organized parameters in environment files by group.
+ - Cleaned up variable names in `split_flows.py` and `build_stream_traversal.py`.
+ - `build_stream_traversal.py` is now assigning HydroID by midpoint instead centroid.
+ - Cleanup of `clip_vectors_to_wbd.py`.
+
+<br/><br/>
+## v3.0.2.0 - 2021-01-25 - [PR #218](https://github.com/NOAA-OWP/cahaba/pull/218)
+
+Addition of an API service to schedule, run and manage `fim_run` jobs through a user-friendly web interface.
+
+### Additions
+
+ - `api` folder that contains all the codebase for the new service.
+
+<br/><br/>
+## v3.0.1.0 - 2021-01-21 - [PR #206](https://github.com/NOAA-OWP/cahaba/pull/206)
+
+Preprocess MS and FR stream networks
+
+### Changes
+
+ - Headwater stream segments geometries are adjusted to align with with NWM streams.
+ - Incoming streams are selected using intersection points between NWM streams and HUC4 boundaries.
+ - `clip_vectors_to_wbd.py` handles local headwaters.
+ - Removes NHDPlus features categorized as coastline and underground conduit.  
+ - Added streams layer to production whitelist.
+ - Fixed progress bar in `lib/acquire_and_preprocess_inputs.py`.
+ - Added `getDriver` to shared `functions.py`.
+ - Cleaned up variable names and types.
+
+<br/><br/>
+## v3.0.0.4 - 2021-01-20 - [PR #230](https://github.com/NOAA-OWP/cahaba/pull/230)
+
+Changed the directory where the `included_huc*.lst` files are being read from.
+
+### Changes
+
+ - Changed the directory where the `included_huc*.lst` files are being read from.
+
+<br/><br/>
+## v3.0.0.3 - 2021-01-14 - [PR #210](https://github.com/NOAA-OWP/cahaba/pull/210)
+
+Hotfix for handling nodata value in rasterized levee lines.
+
+### Changes
+
+ - Resolves bug for HUCs where `$ndv > 0` (Great Lakes region).
+ - Initialize the `nld_rasterized_elev.tif` using a value of `-9999` instead of `$ndv`.
+
+ <br/><br/>
+## v3.0.0.2 - 2021-01-06 - [PR #200](https://github.com/NOAA-OWP/cahaba/pull/200)
+
+Patch to address AHPSs mapping errors.
+
+### Changes
+
+ - Checks `dtype` of `hydroTable.csv` columns to resolve errors caused in `inundation.py` when joining to flow forecast.
+ - Exits `inundation.py` when all hydrotable HydroIDs are lake features.
+ - Updates path to latest AHPs site layer.
+ - Updated [readme](https://github.com/NOAA-OWP/cahaba/commit/9bffb885f32dfcd95978c7ccd2639f9df56ff829)
+
+<br/><br/>
+## v3.0.0.1 - 2020-12-31 - [PR #184](https://github.com/NOAA-OWP/cahaba/pull/184)
+
+Modifications to build and run Docker image more reliably. Cleanup on some pre-processing scripts.
+
+### Changes
+
+ - Changed to noninteractive install of GRASS.
+ - Changed some paths from relative to absolute and cleaned up some python shebang lines.
+
+### Notes
+ - `aggregate_vector_inputs.py` doesn't work yet. Need to externally download required data to run fim_run.sh
+
+ <br/><br/>
+## v3.0.0.0 - 2020-12-22 - [PR #181](https://github.com/NOAA-OWP/cahaba/pull/181)
+
+The software released here builds on the flood inundation mapping capabilities demonstrated as part of the National Flood Interoperability Experiment, the Office of Water Prediction's Innovators Program and the National Water Center Summer Institute. The flood inundation mapping software implements the Height Above Nearest Drainage (HAND) algorithm and incorporates community feedback and lessons learned over several years. The software has been designed to meet the requirements set by stakeholders interested in flood prediction and has been developed in partnership with several entities across the water enterprise.
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 000000000..6d018da0c
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,11 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: "Aristizabal"
+  given-names: "Fernando"
+  orcid: "https://orcid.org/0000-0003-2525-4712"
+title: "Generalized Mainstems (GMS): An improvement of the Height Above Nearest Drainage Method (HAND) for flood inundation mapping (FIM) that uses processing units of unit stream order"
+version: alpha
+doi: None
+date-released: pending
+url: "https://github.com/NOAA-OWP/fim_4"
diff --git a/Dockerfile b/Dockerfile
index ef56576b8..5848ecaa5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -58,8 +58,6 @@ ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
 ## ADDING FIM GROUP ##
-# ARG GroupID=1370800120
-# ARG GroupName=apd_dev
 ARG GroupID=1370800235
 ARG GroupName=fim
 RUN addgroup --gid $GroupID $GroupName
@@ -69,8 +67,8 @@ ENV GN=$GroupName
 RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
-
-RUN apt update --fix-missing && apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
+RUN apt update --fix-missing 
+RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
 
 RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
 
@@ -83,6 +81,7 @@ ENV PYTHONUNBUFFERED=TRUE
 
 ## ADD TO PATHS ##
 ENV PATH="$projectDir:${PATH}"
+ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 
 ## install python 3 modules ##
 COPY Pipfile .
diff --git a/Dockerfile.prod b/Dockerfile.prod
new file mode 100644
index 000000000..5c5c7964b
--- /dev/null
+++ b/Dockerfile.prod
@@ -0,0 +1,103 @@
+## Temporary image to build the libraries and only save the needed artifacts
+FROM osgeo/gdal:ubuntu-full-3.1.2 AS builder
+WORKDIR /opt/builder
+ARG dataDir=/data
+ARG projectDir=/foss_fim
+ARG depDir=/dependencies
+ARG taudemVersion=98137bb6541a0d0077a9c95becfed4e56d0aa0ac
+ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
+ENV DEBIAN_FRONTEND noninteractive
+ENV taudemDir=$depDir/taudem/bin
+ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
+
+RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/* 
+
+RUN git clone https://github.com/dtarb/taudem.git
+RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
+
+RUN apt-get update --fix-missing && apt-get install -y cmake mpich \
+    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/* 
+
+## Compile Main taudem repo ##
+RUN mkdir -p taudem/bin
+RUN cd taudem \
+    && git checkout $taudemVersion \
+    && cd src \
+    && make
+
+## Compile taudem repo with accelerated flow directions ##
+RUN cd taudem_accelerated_flowDirections/taudem \
+    && git checkout $taudemVersion2 \
+    && mkdir build \
+    && cd build \
+    && cmake .. \
+    && make
+
+RUN mkdir -p $taudemDir
+RUN mkdir -p $taudemDir2
+
+## Move needed binaries to the next stage of the image
+RUN cd taudem/bin && mv -t $taudemDir flowdircond aread8 threshold streamnet gagewatershed catchhydrogeo dinfdistdown
+RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d8flowdir dinfflowdir
+
+
+
+
+###############################################################################################
+
+
+
+# Base Image that has GDAL, PROJ, etc
+FROM osgeo/gdal:ubuntu-full-3.1.2
+ARG dataDir=/data
+ARG projectDir=/foss_fim
+ARG depDir=/dependencies
+ENV inputDataDir=$dataDir/inputs
+ENV outputDataDir=$dataDir/outputs 
+ENV srcDir=$projectDir/src
+ENV taudemDir=$depDir/taudem/bin
+ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
+
+## ADDING FIM GROUP ##
+ARG GroupID=1370800235
+ARG GroupName=fim
+RUN addgroup --gid $GroupID $GroupName
+ENV GID=$GroupID
+ENV GN=$GroupName
+
+RUN mkdir -p $depDir
+COPY --from=builder $depDir $depDir
+
+RUN apt update --fix-missing
+RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
+
+RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
+#COPY install_grass.exp .
+#RUN ./install_grass.exp
+
+RUN apt auto-remove
+
+## adding environment variables for numba and python ##
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+ENV PYTHONUNBUFFERED=TRUE
+
+## ADD TO PATHS ##
+ENV PATH="$projectDir:${PATH}"
+ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
+
+## install python 3 modules ##
+COPY Pipfile .
+COPY Pipfile.lock .
+RUN pip3 install pipenv && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
+
+## Copy the source code to the image
+COPY . $projectDir/
+
+## Set user:group for running docker in detached mode
+USER root:$GroupName
+
+# RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
+ADD ./src/entrypoint.sh /
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
+
diff --git a/Pipfile b/Pipfile
index b54b7daf7..2376754e3 100755
--- a/Pipfile
+++ b/Pipfile
@@ -22,6 +22,10 @@ seaborn = "==0.11.0"
 python-dotenv = "*"
 natsort = "*"
 memory-profiler = "*"
+xarray = "==0.19.0"
+netcdf4 = "==1.5.7"
+tables = "==3.6.1"
+h5py = "==3.4.0"
 pyproj = "==3.1.0"
 
 [requires]
diff --git a/Pipfile.lock b/Pipfile.lock
index 4307ca71f..c6a3d3a0e 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f1296d064e5178d77fdf23130a571d8169821f2eeed04b37c1b2cf60fe6928ba"
+            "sha256": "3d3887794dbd15cc911a4223776cf9002eed9d36a62876bf3b96022626c530bf"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -38,6 +38,36 @@
             ],
             "version": "==2021.10.8"
         },
+        "cftime": {
+            "hashes": [
+                "sha256:01882a6f540f61b87db7e6c1a389d278e97b63b8c40457df7d5b98a25413c1a7",
+                "sha256:06033c132beb5e560f1957c048fdfbb3c876b3cc2f3e5e0f664927d98b5fae44",
+                "sha256:2ebdc5b20fb113ad0aa41c61fa7f9d52cb3a22b4b847ed2bf653d7e1f81ae8e2",
+                "sha256:35cb5cbcbd9f8083265a3ce4283ebc62114e28d589d6bf62d98532ada6cfc890",
+                "sha256:40517a5c2352c847f919b7f29963015c37c071ebc2e89ecc4f62aa4b80d9a72e",
+                "sha256:4260ca466a445af7b243d84a7c96833b9261bdfd15ba86028d9710b7d700f490",
+                "sha256:5850ed2ae6c218d57249cf75fa27accd4e3436617a8ea9cd2bea6cb031f449fd",
+                "sha256:6db9129b4f26202c1ba4610b667693e686922011bd7166e7573dea668bb70c88",
+                "sha256:6dc4d76ec7fe5a2d3c00dbe6604c757f1319613b75ef157554ef3648bf102a50",
+                "sha256:72eff06667df45cc87c7b50c87ee67471a32b73cdfaac903bb65945823a4bdf3",
+                "sha256:75dc17b031e85feece5a5770f00342dedeb4349dc530bcd1cd25deb7064cab32",
+                "sha256:940fcaec3159e8ba6ad93bd939108c665c35650fcf6cd026687b40b48bca8110",
+                "sha256:a220f9652f34038c0d9ca275bbfb50efc52dbf867733746cf8a7dcd3b5b84cf1",
+                "sha256:ae7151d4c0759f7263d3baf4ca7283f715f99a02cf3bd4cf2403ae0e5312b64e",
+                "sha256:bcca217713f6468ea0fe18637ff2a113bcb5cf9c7328ba535a0323033110e419",
+                "sha256:bcdf2a3d52a36e81e30d07ae93afdeb057cca8eb0a28c21cfef1f523ade1fe74",
+                "sha256:c5d0985d99a4c749d90016e993dedd6c37c5b6d459b5747eb3a14e9ad63dc669",
+                "sha256:cd33cd2e220c1c283f6fe3273352d7754795f86835ad04520f26a441a5b45193",
+                "sha256:db62b4383815596a92acb6cfa3779b208292aa17f90c10669ee50806b2795cee",
+                "sha256:e3196615252312762ce2196664b2614d254d5e4d2b0cf5c7866c5916e45cec66",
+                "sha256:f549ad17102403984a9c865313f758c346cbb04dd4550946cab81a5fea132ff3",
+                "sha256:fae15273aafd0c9382637b03ef6fa51274c28c84dc3b459a969be2121f41c648",
+                "sha256:fd2311da209bbcff4221d344cb6ee217ce6b465832b013106e1ae21e947818f8",
+                "sha256:fdfc3f3da4efc22ae22096c2b20ac9d5da01b3038c4ed4407fcd501efab26099",
+                "sha256:ff292fcd4d2322fe28bcc0c9e70a37e048858aa68a9aaa43710a73100fa1e77d"
+            ],
+            "version": "==1.5.1.1"
+        },
         "click": {
             "hashes": [
                 "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
@@ -58,7 +88,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cycler": {
@@ -87,11 +117,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:68071406009e7ef6a5fdcd85d95975cd6963867bb226f2b786bfffe15d1959ef",
-                "sha256:8c8f84131bf04f3b1dcf99b9763cec35c347164ab6ad006e18d2f99fcab05529"
+                "sha256:545c05d0f7903a863c2020e07b8f0a57517f2c40d940bded77076397872d14ca",
+                "sha256:edf251d5d2cc0580d5f72de4621c338d8c66c5f61abb50cf486640f73c8194d5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.28.1"
+            "version": "==4.28.5"
         },
         "geopandas": {
             "hashes": [
@@ -109,6 +139,22 @@
             "index": "pypi",
             "version": "==0.5"
         },
+        "h5py": {
+            "hashes": [
+                "sha256:0b0f002f5f341afe7d3d7e15198e80d9021da24a4d182d88068d79bfc91fba86",
+                "sha256:1edf33e722d47c6eb3878d51173b23dd848939f006f41b498bafceff87fb4cbd",
+                "sha256:46917f20021dde02865572a5fd2bb620945f7b7cd268bdc8e3f5720c32b38140",
+                "sha256:708ddff49af12c01d77e0f9782bb1a0364d96459ec0d1f85d90baea6d203764b",
+                "sha256:8745e5159830d7975a9cf38690455f22601509cda04de29b7e88b3fbdc747611",
+                "sha256:8e809149f95d9a3a33b1279bfbf894c78635a5497e8d5ac37420fa5ec0cf4f29",
+                "sha256:aa511bd05a9174c3008becdc93bd5785e254d34a6ab5f0425e6b2fbbc88afa6d",
+                "sha256:bb4ce46095e3b16c872aaf62adad33f40039fecae04674eb62c035386affcb91",
+                "sha256:be2a545f09074546f73305e0db6d36aaf1fb6ea2fcf1add2ce306b9c7f78e55a",
+                "sha256:ee1c683d91ab010d5e85cb61e8f9e7ee0d8eab545bf3dd50a9618f1d0e8f615e"
+            ],
+            "index": "pypi",
+            "version": "==3.4.0"
+        },
         "kiwisolver": {
             "hashes": [
                 "sha256:0007840186bacfaa0aba4466d5890334ea5938e0bb7e28078a0eb0e63b5b59d5",
@@ -183,51 +229,51 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:0abf8b51cc6d3ba34d1b15b26e329f23879848a0cf1216954c1f432ffc7e1af7",
-                "sha256:0e020a42f3338823a393dd2f80e39a2c07b9f941dfe2c778eb104eeb33d60bb5",
-                "sha256:13930a0c9bec0fd25f43c448b047a21af1353328b946f044a8fc3be077c6b1a8",
-                "sha256:153a0cf6a6ff4f406a0600d2034710c49988bacc6313d193b32716f98a697580",
-                "sha256:18f6e52386300db5cc4d1e9019ad9da2e80658bab018834d963ebb0aa5355095",
-                "sha256:2089b9014792dcc87bb1d620cde847913338abf7d957ef05587382b0cb76d44e",
-                "sha256:2eea16883aa7724c95eea0eb473ab585c6cf66f0e28f7f13e63deb38f4fd6d0f",
-                "sha256:38892a254420d95594285077276162a5e9e9c30b6da08bdc2a4d53331ad9a6fa",
-                "sha256:4b018ea6f26424a0852eb60eb406420d9f0d34f65736ea7bbfbb104946a66d86",
-                "sha256:65f877882b7ddede7090c7d87be27a0f4720fe7fc6fddd4409c06e1aa0f1ae8d",
-                "sha256:666d717a4798eb9c5d3ae83fe80c7bc6ed696b93e879cb01cb24a74155c73612",
-                "sha256:66b172610db0ececebebb09d146f54205f87c7b841454e408fba854764f91bdd",
-                "sha256:6db02c5605f063b67780f4d5753476b6a4944343284aa4e93c5e8ff6e9ec7f76",
-                "sha256:6e0e6b2111165522ad336705499b1f968c34a9e84d05d498ee5af0b5697d1efe",
-                "sha256:71a1851111f23f82fc43d2b6b2bfdd3f760579a664ebc939576fe21cc6133d01",
-                "sha256:7a7cb59ebd63a8ac4542ec1c61dd08724f82ec3aa7bb6b4b9e212d43c611ce3d",
-                "sha256:7baf23adb698d8c6ca7339c9dde00931bc47b2dd82fa912827fef9f93db77f5e",
-                "sha256:970aa97297537540369d05fe0fd1bb952593f9ab696c9b427c06990a83e2418b",
-                "sha256:9bac8eb1eccef540d7f4e844b6313d9f7722efd48c07e1b4bfec1056132127fd",
-                "sha256:a07ff2565da72a7b384a9e000b15b6b8270d81370af8a3531a16f6fbcee023cc",
-                "sha256:a0dcaf5648cecddc328e81a0421821a1f65a1d517b20746c94a1f0f5c36fb51a",
-                "sha256:a0ea10faa3bab0714d3a19c7e0921279a68d57552414d6eceaea99f97d7735db",
-                "sha256:a5b62d1805cc83d755972033c05cea78a1e177a159fc84da5c9c4ab6303ccbd9",
-                "sha256:a6cef5b31e27c31253c0f852b629a38d550ae66ec6850129c49d872f9ee428cb",
-                "sha256:a7bf8b05c214d32fb7ca7c001fde70b9b426378e897b0adbf77b85ea3569d56a",
-                "sha256:ac17a7e7b06ee426a4989f0b7f24ab1a592e39cdf56353a90f4e998bc0bf44d6",
-                "sha256:b3b687e905da32e5f2e5f16efa713f5d1fcd9fb8b8c697895de35c91fedeb086",
-                "sha256:b5e439d9e55d645f2a4dca63e2f66d68fe974c405053b132d61c7e98c25dfeb2",
-                "sha256:ba107add08e12600b072cf3c47aaa1ab85dd4d3c48107a5d3377d1bf80f8b235",
-                "sha256:d092b7ba63182d2dd427904e3eb58dd5c46ec67c5968de14a4b5007010a3a4cc",
-                "sha256:dc8c5c23e7056e126275dbf29efba817b3d94196690930d0968873ac3a94ab82",
-                "sha256:df0042cab69f4d246f4cb8fc297770ac4ae6ec2983f61836b04a117722037dcd",
-                "sha256:ee3d9ff16d749a9aa521bd7d86f0dbf256b2d2ac8ce31b19e4d2c86d2f2ff0b6",
-                "sha256:f23fbf70d2e80f4e03a83fc1206a8306d9bc50482fee4239f10676ce7e470c83",
-                "sha256:ff5d9fe518ad2de14ce82ab906b6ab5c2b0c7f4f984400ff8a7a905daa580a0a"
+                "sha256:14334b9902ec776461c4b8c6516e26b450f7ebe0b3ef8703bf5cdfbbaecf774a",
+                "sha256:2252bfac85cec7af4a67e494bfccf9080bcba8a0299701eab075f48847cca907",
+                "sha256:2e3484d8455af3fdb0424eae1789af61f6a79da0c80079125112fd5c1b604218",
+                "sha256:34a1fc29f8f96e78ec57a5eff5e8d8b53d3298c3be6df61e7aa9efba26929522",
+                "sha256:3e66497cd990b1a130e21919b004da2f1dc112132c01ac78011a90a0f9229778",
+                "sha256:40e0d7df05e8efe60397c69b467fc8f87a2affeb4d562fe92b72ff8937a2b511",
+                "sha256:456cc8334f6d1124e8ff856b42d2cc1c84335375a16448189999496549f7182b",
+                "sha256:506b210cc6e66a0d1c2bb765d055f4f6bc2745070fb1129203b67e85bbfa5c18",
+                "sha256:53273c5487d1c19c3bc03b9eb82adaf8456f243b97ed79d09dded747abaf1235",
+                "sha256:577ed20ec9a18d6bdedb4616f5e9e957b4c08563a9f985563a31fd5b10564d2a",
+                "sha256:6803299cbf4665eca14428d9e886de62e24f4223ac31ab9c5d6d5339a39782c7",
+                "sha256:68fa30cec89b6139dc559ed6ef226c53fd80396da1919a1b5ef672c911aaa767",
+                "sha256:6c094e4bfecd2fa7f9adffd03d8abceed7157c928c2976899de282f3600f0a3d",
+                "sha256:778d398c4866d8e36ee3bf833779c940b5f57192fa0a549b3ad67bc4c822771b",
+                "sha256:7a350ca685d9f594123f652ba796ee37219bf72c8e0fc4b471473d87121d6d34",
+                "sha256:87900c67c0f1728e6db17c6809ec05c025c6624dcf96a8020326ea15378fe8e7",
+                "sha256:8a77906dc2ef9b67407cec0bdbf08e3971141e535db888974a915be5e1e3efc6",
+                "sha256:8e70ae6475cfd0fad3816dcbf6cac536dc6f100f7474be58d59fa306e6e768a4",
+                "sha256:abf67e05a1b7f86583f6ebd01f69b693b9c535276f4e943292e444855870a1b8",
+                "sha256:b04fc29bcef04d4e2d626af28d9d892be6aba94856cb46ed52bcb219ceac8943",
+                "sha256:b19a761b948e939a9e20173aaae76070025f0024fc8f7ba08bef22a5c8573afc",
+                "sha256:b2e9810e09c3a47b73ce9cab5a72243a1258f61e7900969097a817232246ce1c",
+                "sha256:b71f3a7ca935fc759f2aed7cec06cfe10bc3100fadb5dbd9c435b04e557971e1",
+                "sha256:b8a4fb2a0c5afbe9604f8a91d7d0f27b1832c3e0b5e365f95a13015822b4cd65",
+                "sha256:bb1c613908f11bac270bc7494d68b1ef6e7c224b7a4204d5dacf3522a41e2bc3",
+                "sha256:d24e5bb8028541ce25e59390122f5e48c8506b7e35587e5135efcb6471b4ac6c",
+                "sha256:d70a32ee1f8b55eed3fd4e892f0286df8cccc7e0475c11d33b5d0a148f5c7599",
+                "sha256:e293b16cf303fe82995e41700d172a58a15efc5331125d08246b520843ef21ee",
+                "sha256:e2f28a07b4f82abb40267864ad7b3a4ed76f1b1663e81c7efc84a9b9248f672f",
+                "sha256:e3520a274a0e054e919f5b3279ee5dbccf5311833819ccf3399dab7c83e90a25",
+                "sha256:e3b6f3fd0d8ca37861c31e9a7cab71a0ef14c639b4c95654ea1dd153158bf0df",
+                "sha256:e486f60db0cd1c8d68464d9484fd2a94011c1ac8593d765d0211f9daba2bd535",
+                "sha256:e8c87cdaf06fd7b2477f68909838ff4176f105064a72ca9d24d3f2a29f73d393",
+                "sha256:edf5e4e1d5fb22c18820e8586fb867455de3b109c309cb4fce3aaed85d9468d1",
+                "sha256:fe8d40c434a8e2c68d64c6d6a04e77f21791a93ff6afe0dce169597c110d3079"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.5.0"
+            "version": "==3.5.1"
         },
         "memory-profiler": {
             "hashes": [
-                "sha256:01385ac0fec944fcf7969814ec4406c6d8a9c66c079d09276723c5a7680f44e5"
+                "sha256:6a12869511d6cebcb29b71ba26985675a58e16e06b3c523b49f67c5497a33d1c"
             ],
             "index": "pypi",
-            "version": "==0.58.0"
+            "version": "==0.60.0"
         },
         "munch": {
             "hashes": [
@@ -238,11 +284,52 @@
         },
         "natsort": {
             "hashes": [
-                "sha256:5f5f4ea471d655b1b1611eef1cf0c6d3397095d2d3a1aab7098d6a50e4c3901a",
-                "sha256:a0a4fd71aee20a6d648da61e01180a63f7268e69983d0440bd3ad80ef1ba6981"
+                "sha256:09cff3cabb617f9794f643758ab5b337c1c0af1a6df11edf5c04de496fd7fb26",
+                "sha256:feb87e0ce1dc1f8f3f21e18a85216c790e746d76a5ff6889563394605f504a2b"
             ],
             "index": "pypi",
-            "version": "==8.0.0"
+            "version": "==8.0.2"
+        },
+        "netcdf4": {
+            "hashes": [
+                "sha256:02c2317f377f23005e447e4b187ebce3cdeae76b9d0a443d9d6f5156a3980f04",
+                "sha256:087d09594c0739b03a18ed058d6eca8dc80cc672961b231473bdf2c728112ec8",
+                "sha256:0bf6b4375d99ef6aa9b5d82bdd2a92113c130821461e10bf9831609783f43a2a",
+                "sha256:1dd21bff8e29476decfe42602ab40694f10be84b1a585bc5d28c8b08c19f7283",
+                "sha256:1ee78d5c129fcfeafd6d4a6339984d6108dbabfb8a2cec651f9dc2132c792c78",
+                "sha256:206af8b5d110b89132ddea9b7b162efe82c26db37bc7a7633110750685243195",
+                "sha256:2106c5eaacb8e4dd1ad9837cb020fdf1b9c9e5630cd6648a6ac760690f098eed",
+                "sha256:34355a0129f05e44b26be22e352af183dee7dcd4c6504b398c10ef89ca9f9303",
+                "sha256:35f527f9d22b23c93cbb6b20e07fe1e3e27ed5ac80a699c2b1f2d150d5db70ee",
+                "sha256:393f9c7e92f03e08f8093100396dddb8619a43cb37ad2ac6beb70aaa71226dd5",
+                "sha256:41c0ff220a16c894bf147ce2b1c0266610a5b94bf7aed66c5ee2e5aeaa6bfb20",
+                "sha256:41ff33837732c94a9446ebc1562bf5d954c43e1009f1429423da9151bee5bac1",
+                "sha256:43c95d05458b730a496bbfa52080f4133ec60f07b3994a5b7f45034c65c6d31b",
+                "sha256:4933f948dba82f4f95dcb4d0c4227ab4e085c35e33048009c77ec446a86b2d63",
+                "sha256:50821324f804cd99043b7273d15e65e204cadbf7ca8af27059fade765fb2c430",
+                "sha256:56808afaf4f8afd6d7f6dcdba5ee39d37160028066abba140e0210bdb1d50712",
+                "sha256:5eb58e4b4ad9b79b72fceb8a806b5236b6ce4d5053f7020cff367fcc0bebc765",
+                "sha256:5f520457cf48541edb83655567b5b17cdb2df19a305703f015a85e7f2aceb560",
+                "sha256:7d31d1b736d402b2779f703479d0fbdee59fc6e1e880d1af6eb36fded934a982",
+                "sha256:8164915dc40baa26c65d0e5661d96cd8cbc87157563cf364167d9ec82a6ef7f6",
+                "sha256:8bb9e8b073e25c44cdcc42947e945a5b950ea93a519ba28fec642a92a1c983a9",
+                "sha256:8f4352717660ea48df8c21920f29c708ba69b7002f609dc89036507913b82608",
+                "sha256:a0a47694a123647ff7ef4c16a863510ad640fd44e75f8e2ee5042578ad2a0d67",
+                "sha256:ad705a74449bcefeebb6ae41a5e5987ab29942f508e092151dac7508803e12e8",
+                "sha256:aff200ee07495c73f4257f55d19d13f3678d7f3171d48a85481706e06a6db0e9",
+                "sha256:c06f59f54f73aab86b123838fabe68c728503be981cad9f28283dcfb627f7023",
+                "sha256:c270e4e5161955a35a152348a659270540c85fb2f238c13601c8649cd992032f",
+                "sha256:c3056cc36669abd35b152b978a0bc54b622995d9d7165f8d380c134fa60e27d6",
+                "sha256:c6b0c6ae9f9a050e9878f700e87405044fb2ee9a730559d6feb87f2361b361c6",
+                "sha256:ca8f6f519b290b68892186cb966ee8d2287faec8463128fe6aaec188a0c5ae96",
+                "sha256:cb374f259d0a6d0e436294607d4bc5999f20f61e1a7d365b50e07898ea43a7de",
+                "sha256:d145f9c12da29da3922d8b8aafea2a2a89501bcb28a219a46b7b828b57191594",
+                "sha256:e8d64208e69cd5e67112eb4378066946124dcd1fbb7c2aa3d12d488f4d024759",
+                "sha256:f7b8b19971ff5e1dc5267f1f917667214c6614816f879ff6620bc152132c2629",
+                "sha256:fc489ecca6fb2e5ab738eedf962f9213dcde005b6f9e6f9b2d3d9d6513811759"
+            ],
+            "index": "pypi",
+            "version": "==1.5.7"
         },
         "numba": {
             "hashes": [
@@ -269,41 +356,72 @@
             "index": "pypi",
             "version": "==0.50.1"
         },
+        "numexpr": {
+            "hashes": [
+                "sha256:1639561d056d2d790a56ddab7e7df40b6181ad50338b50fba94aa42874a00958",
+                "sha256:2247d92da60b85de619e797e59a80e9c7302fba82dcd0525de8f7dd729a0d60f",
+                "sha256:24fb5b2c17273a76e7de9cea7817c54262198657998a093fceb4030f273524c7",
+                "sha256:3bab5add6628fa8bb66fba7b1f0eed5d8d0ce05fdd2dcc326dde8a297a961c46",
+                "sha256:48258db3ba89ad028744e07b09dde963f82da7f081849d3a003bb0b96b112d4f",
+                "sha256:4eb79d9026f013cf8d16de8be74911c74c0c09362627bf4b39e2b7f1f3188c28",
+                "sha256:517f299c4bc8491b5117aa276e8f3cf7ee2e89223922e92e2ea78a32985d5087",
+                "sha256:5d0c98c4d8bcc25962e5859176e5728f69209cffb9b7f64bf6d1c801fe350946",
+                "sha256:6bce8a183afe157c25385d27be314be22f06ba644c89b611d20e2570a06bd5dd",
+                "sha256:79ec94295aa57f5a9d212116bb7359744cd2f9e05d477df0dee383b7f44b9588",
+                "sha256:80db25e2934fd1a1b787440d5fa7946adb79a1289d7dc64e2c8bcd6ceae660ad",
+                "sha256:920c6a3088790573765e103e20592864977aa4b4d1b819c298fa9d88771cde1b",
+                "sha256:a591f99ecbb413749725e8da4e52e663f0939dd5fbf1ae5a7c6c50ba734f57de",
+                "sha256:a97a087a5f5e56cd81c69215918fdaca60eb478a099daa757455e4ff887f7600",
+                "sha256:a9f046cb5752f08a9291dc1fd37a9cfd15770262188bb984e4418490fef9c9ec",
+                "sha256:ab6b2cb64bc9391f77f08203fda5af3647ed2abcefb928cc6282727854f97735",
+                "sha256:b57d3ab7054409d9b4d2148241ae70d738c0b0daeb1a0efd5ea89b9279752e22",
+                "sha256:bd402e43b8baf6436b7c2c14541f69eb4f97f023469585a7ad258c49622ff619",
+                "sha256:cd779aa44dd986c4ef10163519239602b027be06a527946656207acf1f58113b",
+                "sha256:cfd89f63028f8df3c9b11bf2c98085184f967a09f543a77c3335f4a0ec54f124",
+                "sha256:d148e99483e15de22d0acd5100136d39a336e91c8f8d37bf2e84e9f0ab4c0610",
+                "sha256:d2b4b6379763ec5d794d4aaa1834ae00f1bba82a36d0b99c6e2d559302a21e85",
+                "sha256:d7b64b125161e722c9dc8a27df282e755bd9a5adf826b2e3e1f038e3dfdc3307",
+                "sha256:da180aaec7f6c387540b251f6ec2b8d280220c0e45731778853c8b0d86c4ae22",
+                "sha256:ebf31aeed426840aefe1e94c89bb0b7530a72be36444ed4c73e4411865b79be5",
+                "sha256:fd6905bc80a11908e363c9821cbf8aeeca4dca5b6a2eea90a97b055bc73443e6"
+            ],
+            "version": "==2.8.1"
+        },
         "numpy": {
             "hashes": [
-                "sha256:0b78ecfa070460104934e2caf51694ccd00f37d5e5dbe76f021b1b0b0d221823",
-                "sha256:1247ef28387b7bb7f21caf2dbe4767f4f4175df44d30604d42ad9bd701ebb31f",
-                "sha256:1403b4e2181fc72664737d848b60e65150f272fe5a1c1cbc16145ed43884065a",
-                "sha256:170b2a0805c6891ca78c1d96ee72e4c3ed1ae0a992c75444b6ab20ff038ba2cd",
-                "sha256:2e4ed57f45f0aa38beca2a03b6532e70e548faf2debbeb3291cfc9b315d9be8f",
-                "sha256:32fe5b12061f6446adcbb32cf4060a14741f9c21e15aaee59a207b6ce6423469",
-                "sha256:34f3456f530ae8b44231c63082c8899fe9c983fd9b108c997c4b1c8c2d435333",
-                "sha256:4c9c23158b87ed0e70d9a50c67e5c0b3f75bcf2581a8e34668d4e9d7474d76c6",
-                "sha256:5d95668e727c75b3f5088ec7700e260f90ec83f488e4c0aaccb941148b2cd377",
-                "sha256:615d4e328af7204c13ae3d4df7615a13ff60a49cb0d9106fde07f541207883ca",
-                "sha256:69077388c5a4b997442b843dbdc3a85b420fb693ec8e33020bb24d647c164fa5",
-                "sha256:74b85a17528ca60cf98381a5e779fc0264b4a88b46025e6bcbe9621f46bb3e63",
-                "sha256:81225e58ef5fce7f1d80399575576fc5febec79a8a2742e8ef86d7b03beef49f",
-                "sha256:8890b3360f345e8360133bc078d2dacc2843b6ee6059b568781b15b97acbe39f",
-                "sha256:92aafa03da8658609f59f18722b88f0a73a249101169e28415b4fa148caf7e41",
-                "sha256:9864424631775b0c052f3bd98bc2712d131b3e2cd95d1c0c68b91709170890b0",
-                "sha256:9e6f5f50d1eff2f2f752b3089a118aee1ea0da63d56c44f3865681009b0af162",
-                "sha256:a3deb31bc84f2b42584b8c4001c85d1934dbfb4030827110bc36bfd11509b7bf",
-                "sha256:ad010846cdffe7ec27e3f933397f8a8d6c801a48634f419e3d075db27acf5880",
-                "sha256:b1e2312f5b8843a3e4e8224b2b48fe16119617b8fc0a54df8f50098721b5bed2",
-                "sha256:bc988afcea53e6156546e5b2885b7efab089570783d9d82caf1cfd323b0bb3dd",
-                "sha256:c449eb870616a7b62e097982c622d2577b3dbc800aaf8689254ec6e0197cbf1e",
-                "sha256:c74c699b122918a6c4611285cc2cad4a3aafdb135c22a16ec483340ef97d573c",
-                "sha256:c885bfc07f77e8fee3dc879152ba993732601f1f11de248d4f357f0ffea6a6d4",
-                "sha256:e3c3e990274444031482a31280bf48674441e0a5b55ddb168f3a6db3e0c38ec8",
-                "sha256:e4799be6a2d7d3c33699a6f77201836ac975b2e1b98c2a07f66a38f499cb50ce",
-                "sha256:e6c76a87633aa3fa16614b61ccedfae45b91df2767cf097aa9c933932a7ed1e0",
-                "sha256:e89717274b41ebd568cd7943fc9418eeb49b1785b66031bc8a7f6300463c5898",
-                "sha256:f5162ec777ba7138906c9c274353ece5603646c6965570d82905546579573f73",
-                "sha256:fde96af889262e85aa033f8ee1d3241e32bf36228318a61f1ace579df4e8170d"
+                "sha256:00c9fa73a6989895b8815d98300a20ac993c49ac36c8277e8ffeaa3631c0dbbb",
+                "sha256:025b497014bc33fc23897859350f284323f32a2fff7654697f5a5fc2a19e9939",
+                "sha256:08de8472d9f7571f9d51b27b75e827f5296295fa78817032e84464be8bb905bc",
+                "sha256:1964db2d4a00348b7a60ee9d013c8cb0c566644a589eaa80995126eac3b99ced",
+                "sha256:2a9add27d7fc0fdb572abc3b2486eb3b1395da71e0254c5552b2aad2a18b5441",
+                "sha256:2d8adfca843bc46ac199a4645233f13abf2011a0b2f4affc5c37cd552626f27b",
+                "sha256:301e408a052fdcda5cdcf03021ebafc3c6ea093021bf9d1aa47c54d48bdad166",
+                "sha256:311283acf880cfcc20369201bd75da907909afc4666966c7895cbed6f9d2c640",
+                "sha256:341dddcfe3b7b6427a28a27baa59af5ad51baa59bfec3264f1ab287aa3b30b13",
+                "sha256:3a5098df115340fb17fc93867317a947e1dcd978c3888c5ddb118366095851f8",
+                "sha256:3c978544be9e04ed12016dd295a74283773149b48f507d69b36f91aa90a643e5",
+                "sha256:3d893b0871322eaa2f8c7072cdb552d8e2b27645b7875a70833c31e9274d4611",
+                "sha256:4fe6a006557b87b352c04596a6e3f12a57d6e5f401d804947bd3188e6b0e0e76",
+                "sha256:507c05c7a37b3683eb08a3ff993bd1ee1e6c752f77c2f275260533b265ecdb6c",
+                "sha256:58ca1d7c8aef6e996112d0ce873ac9dfa1eaf4a1196b4ff7ff73880a09923ba7",
+                "sha256:61bada43d494515d5b122f4532af226fdb5ee08fe5b5918b111279843dc6836a",
+                "sha256:69a5a8d71c308d7ef33ef72371c2388a90e3495dbb7993430e674006f94797d5",
+                "sha256:6a5928bc6241264dce5ed509e66f33676fc97f464e7a919edc672fb5532221ee",
+                "sha256:7b9d6b14fc9a4864b08d1ba57d732b248f0e482c7b2ff55c313137e3ed4d8449",
+                "sha256:a7c4b701ca418cd39e28ec3b496e6388fe06de83f5f0cb74794fa31cfa384c02",
+                "sha256:a7e8f6216f180f3fd4efb73de5d1eaefb5f5a1ee5b645c67333033e39440e63a",
+                "sha256:b545ebadaa2b878c8630e5bcdb97fc4096e779f335fc0f943547c1c91540c815",
+                "sha256:c293d3c0321996cd8ffe84215ffe5d269fd9d1d12c6f4ffe2b597a7c30d3e593",
+                "sha256:c5562bcc1a9b61960fc8950ade44d00e3de28f891af0acc96307c73613d18f6e",
+                "sha256:ca9c23848292c6fe0a19d212790e62f398fd9609aaa838859be8459bfbe558aa",
+                "sha256:cc1b30205d138d1005adb52087ff45708febbef0e420386f58664f984ef56954",
+                "sha256:dbce7adeb66b895c6aaa1fad796aaefc299ced597f6fbd9ceddb0dd735245354",
+                "sha256:dc4b2fb01f1b4ddbe2453468ea0719f4dbb1f5caa712c8b21bb3dd1480cd30d9",
+                "sha256:eed2afaa97ec33b4411995be12f8bdb95c87984eaa28d76cf628970c8a2d689a",
+                "sha256:fc7a7d7b0ed72589fd8b8486b9b42a564f10b8762be8bd4d9df94b807af4a089"
             ],
             "markers": "python_version < '3.11' and python_version >= '3.7'",
-            "version": "==1.21.4"
+            "version": "==1.21.5"
         },
         "packaging": {
             "hashes": [
@@ -476,7 +594,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==2.8.2"
         },
         "python-dotenv": {
@@ -539,33 +657,38 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:1437073f1d4664990879aa8f9547524764372e0fef84a077be4b19e82bba7a8d",
-                "sha256:17fd991a275e4283453f89d404209aa92059ac68d76d804b4bc1716a3742e1b5",
-                "sha256:1ea6233f5a365cb7945b4304bd06323ece3ece85d6a3fa8598d2f53e513467c9",
-                "sha256:2d25272c03ee3c0fe5e0dff1bb7889280bb6c9e1766fa9c7bde81ad8a5f78694",
-                "sha256:30bdda199667e74b50208a793eb1ba47a04e5e3fa16f5ff06c6f7969ae78e4da",
-                "sha256:359b60a0cccd17723b9d5e329a5212a710e771a3ddde800e472fb93732756c46",
-                "sha256:39f838ea5ce8da868785193d88d05cf5a6d5c390804ec99de29a28e1dcdd53e6",
-                "sha256:4d175ba93e00d8eef8f7cd70d4d88a9106a86800c82ea03cf2268c36d6545483",
-                "sha256:5273d832fb9cd5724ee0d335c16a903b923441107dd973d27fc4293075a9f4e3",
-                "sha256:54951f51d731c832b1b8885e0a92e89f33d087de7e40d02078bf0d49c7cbdbb5",
-                "sha256:74f518ce542533054695f743e4271cb8986b63f95bb51d70fcee4f3929cbff7d",
-                "sha256:7b1d0f5f524518f1a86f288443528e4ff4a739c0966db663af4129b7ac7849f8",
-                "sha256:82c5befebf54d799d77e5f0205c03030f57f69ba2541baa44d2e6ad138c28cd3",
-                "sha256:8482c8e45857ab0a5446eb7460d2307a27cbbe659d6d2257820c6d6eb950fd0f",
-                "sha256:87cf3964db0f1cce17aeed5bfc1b89a6b4b07dbfc48e50d21fa3549e00456803",
-                "sha256:8b5726a0fedeaa6beb1095e4466998bdd1d1e960b28db9b5a16c89cbd7b2ebf1",
-                "sha256:97eb573e361a73a553b915dc195c6f72a08249964b1a33f157f9659f3b6210d1",
-                "sha256:a80eb01c43fd98257ec7a49ff5cec0edba32031b5f86503f55399a48cb2c5379",
-                "sha256:cac71d5476a6f56b50459da21f6221707e0051ebd428b2137db32ef4a43bb15e",
-                "sha256:d86abd1ddf421dea5e9cebfeb4de0d205b3dc04e78249afedba9c6c3b2227ff2",
-                "sha256:dc2d1bf41294e63c7302bf499973ac0c7f73c93c01763db43055f6525234bf11",
-                "sha256:e08b81fcd9bf98740b58dc6fdd7879e33a64dcb682201c1135f7d4a75216bb05",
-                "sha256:e3efe7ef75dfe627b354ab0af0dbc918eadee97cc80ff1aabea6d3e01114ebdd",
-                "sha256:fa2dbabaaecdb502641b0b3c00dec05fb475ae48655c66da16c9ed24eda1e711"
+                "sha256:033ce76ed4e9f62923e1f8124f7e2b0800db533828c853b402c7eec6e9465d80",
+                "sha256:173308efba2270dcd61cd45a30dfded6ec0085b4b6eb33b5eb11ab443005e088",
+                "sha256:21b66200cf44b1c3e86495e3a436fc7a26608f92b8d43d344457c54f1c024cbc",
+                "sha256:2c56b820d304dffcadbbb6cbfbc2e2c79ee46ea291db17e288e73cd3c64fefa9",
+                "sha256:304dfaa7146cffdb75fbf6bb7c190fd7688795389ad060b970269c8576d038e9",
+                "sha256:3f78181a153fa21c018d346f595edd648344751d7f03ab94b398be2ad083ed3e",
+                "sha256:4d242d13206ca4302d83d8a6388c9dfce49fc48fdd3c20efad89ba12f785bf9e",
+                "sha256:5d1cc2c19afe3b5a546ede7e6a44ce1ff52e443d12b231823268019f608b9b12",
+                "sha256:5f2cfc359379c56b3a41b17ebd024109b2049f878badc1e454f31418c3a18436",
+                "sha256:65bd52bf55f9a1071398557394203d881384d27b9c2cad7df9a027170aeaef93",
+                "sha256:7edd9a311299a61e9919ea4192dd477395b50c014cdc1a1ac572d7c27e2207fa",
+                "sha256:8499d9dd1459dc0d0fe68db0832c3d5fc1361ae8e13d05e6849b358dc3f2c279",
+                "sha256:866ada14a95b083dd727a845a764cf95dd13ba3dc69a16b99038001b05439709",
+                "sha256:87069cf875f0262a6e3187ab0f419f5b4280d3dcf4811ef9613c605f6e4dca95",
+                "sha256:93378f3d14fff07572392ce6a6a2ceb3a1f237733bd6dcb9eb6a2b29b0d19085",
+                "sha256:95c2d250074cfa76715d58830579c64dff7354484b284c2b8b87e5a38321672c",
+                "sha256:ab5875facfdef77e0a47d5fd39ea178b58e60e454a4c85aa1e52fcb80db7babf",
+                "sha256:b0e0aeb061a1d7dcd2ed59ea57ee56c9b23dd60100825f98238c06ee5cc4467e",
+                "sha256:b78a35c5c74d336f42f44106174b9851c783184a85a3fe3e68857259b37b9ffb",
+                "sha256:c9e04d7e9b03a8a6ac2045f7c5ef741be86727d8f49c45db45f244bdd2bcff17",
+                "sha256:ca36e7d9430f7481fc7d11e015ae16fbd5575615a8e9060538104778be84addf",
+                "sha256:ceebc3c4f6a109777c0053dfa0282fddb8893eddfb0d598574acfb734a926168",
+                "sha256:e2c036492e673aad1b7b0d0ccdc0cb30a968353d2c4bf92ac8e73509e1bf212c",
+                "sha256:eb326658f9b73c07081300daba90a8746543b5ea177184daed26528273157294",
+                "sha256:eb7ae2c4dbdb3c9247e07acc532f91077ae6dbc40ad5bd5dca0bb5a176ee9bda",
+                "sha256:edad1cf5b2ce1912c4d8ddad20e11d333165552aba262c882e28c78bbc09dbf6",
+                "sha256:eef93a446114ac0193a7b714ce67659db80caf940f3232bad63f4c7a81bc18df",
+                "sha256:f7eaea089345a35130bc9a39b89ec1ff69c208efa97b3f8b25ea5d4c41d88094",
+                "sha256:f99d206db1f1ae735a8192ab93bd6028f3a42f6fa08467d37a14eb96c9dd34a3"
             ],
             "markers": "python_version < '3.11' and python_version >= '3.7'",
-            "version": "==1.7.2"
+            "version": "==1.7.3"
         },
         "seaborn": {
             "hashes": [
@@ -575,13 +698,13 @@
             "index": "pypi",
             "version": "==0.11.0"
         },
-        "setuptools-scm": {
+        "setuptools": {
             "hashes": [
-                "sha256:4c64444b1d49c4063ae60bfe1680f611c8b13833d556fd1d6050c0023162a119",
-                "sha256:a49aa8081eeb3514eb9728fa5040f2eaa962d6c6f4ec9c32f6c1fba88f88a0f2"
+                "sha256:032e5949c02878c405552fb800743aad96940097460bc6f115340402852dcdde",
+                "sha256:989ab2d3e632ba23358b8d43950bd46babde5bbb1516760dc5f5ccfe7accdd45"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==6.3.2"
+            "markers": "python_version >= '3.7'",
+            "version": "==60.0.3"
         },
         "shapely": {
             "hashes": [
@@ -672,7 +795,7 @@
                 "sha256:f63600ec06982cdf480899026f4fda622776f5fabed9a869fdb32d72bc17e99a",
                 "sha256:fb62d517a516128bacf08cb6a86ecd39fb06d08e7c4980251f5d5601d29989ba"
             ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==3.17.6"
         },
         "six": {
@@ -680,7 +803,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.16.0"
         },
         "snuggs": {
@@ -690,13 +813,35 @@
             ],
             "version": "==1.4.7"
         },
-        "tomli": {
-            "hashes": [
-                "sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee",
-                "sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade"
+        "tables": {
+            "hashes": [
+                "sha256:169450bd11959c0e1c43137e768cf8b60b2a4f3b2ebf9a620e21865dc0c2d059",
+                "sha256:361da30289ecdcb39b7648c786d8185f9ab08879c0a58a3fc56dab026e122d8e",
+                "sha256:4628e762a8aacfa038cdae118d2d1df9a9ddd9b4a82d6993f4bcbfa7744a9f8a",
+                "sha256:49a972b8a7c27a8a173aeb05f67acb45fe608b64cd8e9fa667c0962a60b71b49",
+                "sha256:6055dd1d3ec03fd25c60bb93a4be396464f0640fd5845884230dae1deb7e6cc6",
+                "sha256:6e13a3eaf86f9eba582a04b44929ee1585a05dd539d207a10a22374b7e4552ca",
+                "sha256:7acbf0e2fb7132a40f441ebb53b53c97cee05fb88ce743afdd97c681d1d377d7",
+                "sha256:8c96c5d4a9ebe34b72b918b3189954e2d5b6f87cb211d4244b7c001661d8a861",
+                "sha256:8ea87231788cfd5773ffbe33f149f778f9ef4ab681149dec00cb88e1681bd299",
+                "sha256:94d7ccac04277089e3bb466bf5c8f7038dd53bb8f19ea9679b7fea62c5c3ae8f",
+                "sha256:950167d56b45ece117f79d839d5d55f0cb45bfca20290fa9dcd70255282f969e",
+                "sha256:9d06c5fda6657698bae4fbe841204625b501ddf2e2a77131c23f3d3ac072db82",
+                "sha256:acb3f905c63e437023071833744b3e5a83376dc457f413f0840d8d50dd5d402b",
+                "sha256:bfdbcacffec122ce8d1b0dd6ffc3c6051bedd6081e20264fa96165d43fc78f52",
+                "sha256:c0b97a7363941d9518573c217cb5bfe4b2b456748aac1e9420d3979f7d5e82d2",
+                "sha256:d95faa1174653a738ac8183a95f050a29a3f69efac6e71f70cde8d717e31af17",
+                "sha256:da9e1ee83c01ed4d1382c7b186d77b4c0ef80b340a48d11a66346e30342c5929",
+                "sha256:db163df08ded7804d596dee14d88397f6c55cdf4671b3992cb885c0b3890a54d",
+                "sha256:dedb959c00ac9e84562a69e80fa858d7aa06d91f96c6cb8cccbbbaf7a879436b",
+                "sha256:ea4b41ed95953ad588bcd6e557577414e50754011430c27934daf5dbd2d52251",
+                "sha256:eed1e030bb077476d585697e37f2b8e37db4157ff93b485b43f374254cff8698",
+                "sha256:f1327aeef8b6c0fec5aae9f5f5a57b2d8ec98c08495fd09471b749ea46de9eb0",
+                "sha256:f9c88511483c8fd39e7841fc60bc7038c96eeb87fe776092439172e1e6330f49",
+                "sha256:fd63c94960f8208cb13d41033a3114c0242e7737cb578f2454c6a087c5d246ec"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.2.2"
+            "index": "pypi",
+            "version": "==3.6.1"
         },
         "tqdm": {
             "hashes": [
@@ -705,6 +850,14 @@
             ],
             "index": "pypi",
             "version": "==4.48.0"
+        },
+        "xarray": {
+            "hashes": [
+                "sha256:3a365ce09127fc841ba88baa63f37ca61376ffe389a6c5e66d52f2c88c23a62b",
+                "sha256:da0f0d7719b5ee95143a34804e3afb82cb8abbe5c10f9750a9dbed156ad30f00"
+            ],
+            "index": "pypi",
+            "version": "==0.19.0"
         }
     },
     "develop": {
@@ -725,11 +878,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:4f69d7423a5a1972f6347ff233e38bbf4df6a150ef20fbb00c635442ac3060aa",
-                "sha256:a658beaf856ce46bc453366d5dc6b2ddc6c481efd3540cb28aa3943819caac9f"
+                "sha256:cb6aef731bf708a7727ab6cde8df87f0281b1427d41e65d62d4b68934fa54e97",
+                "sha256:fc60ef843e0863dd4e24ab2bb5698f071031332801ecf8d1aeb4fb622056545c"
             ],
             "index": "pypi",
-            "version": "==7.29.0"
+            "version": "==7.30.1"
         },
         "jedi": {
             "hashes": [
@@ -749,11 +902,11 @@
         },
         "parso": {
             "hashes": [
-                "sha256:12b83492c6239ce32ff5eed6d3639d6a536170723c6f3f1506869f1ace413398",
-                "sha256:a8c4922db71e4fdb90e0d0bc6e50f9b273d3397925e5e60a717e719201778d22"
+                "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0",
+                "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.8.2"
+            "version": "==0.8.3"
         },
         "pexpect": {
             "hashes": [
@@ -772,11 +925,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:449f333dd120bd01f5d296a8ce1452114ba3a71fae7288d2f0ae2c918764fa72",
-                "sha256:48d85cdca8b6c4f16480c7ce03fd193666b62b0a21667ca56b4bb5ad679d1170"
+                "sha256:1bb05628c7d87b645974a1bad3f17612be0c29fa39af9f7688030163f680bad6",
+                "sha256:e56f2ff799bacecd3e88165b1e2f5ebf9bcd59e80e06d395fa0cc4b8bd7bb506"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.22"
+            "version": "==3.0.24"
         },
         "ptyprocess": {
             "hashes": [
@@ -793,6 +946,14 @@
             "markers": "python_version >= '3.5'",
             "version": "==2.10.0"
         },
+        "setuptools": {
+            "hashes": [
+                "sha256:032e5949c02878c405552fb800743aad96940097460bc6f115340402852dcdde",
+                "sha256:989ab2d3e632ba23358b8d43950bd46babde5bbb1516760dc5f5ccfe7accdd45"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==60.0.3"
+        },
         "traitlets": {
             "hashes": [
                 "sha256:059f456c5a7c1c82b98c2e8c799f39c9b8128f6d0d46941ee118daace9eb70c7",
diff --git a/README.md b/README.md
index 24cee3ce7..6bdfbb6c8 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 ## Cahaba: Flood Inundation Mapping for U.S. National Water Model
 
-Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
+Flood inundation mapping softwaconfigured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
 
 This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids. This repository also includes functionality to generate flood inundation maps (FIMs) and evaluate FIM accuracy.
 
@@ -115,6 +115,11 @@ and include both `Pipfile` and `Pipfile.lock` in your commits. The docker image
 
 If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
 
+----
+## Citing This Work
+
+Please cite this work in your research and projects according to the CITATION.cff file found in the root of this repository.
+
 ----
 ### Known Issues & Getting Help
 
diff --git a/config/deny_gms_branches_default.lst b/config/deny_gms_branches_default.lst
new file mode 100644
index 000000000..718bea8a1
--- /dev/null
+++ b/config/deny_gms_branches_default.lst
@@ -0,0 +1,58 @@
+# List of files for gms branches to delete
+# Use comment to allow list the file
+# Use curly braces to denote branch_id
+agree_bufgrid.tif
+agree_bufgrid_allo.tif
+agree_bufgrid_dist.tif
+agree_smogrid.tif
+agree_smogrid_allo.tif
+agree_smogrid_dist.tif
+catch_list_{}.txt
+coordFile_{}.txt
+crosswalk_table_{}.csv
+demDerived_reaches_{}.dbf
+demDerived_reaches_{}.prj
+demDerived_reaches_{}.shp
+demDerived_reaches_{}.shx
+demDerived_reaches_split_{}.gpkg
+demDerived_reaches_split_filtered_{}.gpkg
+#demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+demDerived_reaches_split_points_{}.gpkg
+demDerived_streamPixels_{}.tif
+demDerived_streamPixels_ids_{}.tif
+demDerived_streamPixels_ids_{}_allo.tif
+demDerived_streamPixels_ids_{}_dist.tif
+dem_burned_{}.tif
+dem_burned_filled_{}.tif
+dem_lateral_thalweg_adj_{}.tif
+dem_meters_{}.tif
+#dem_thalwegCond_{}.tif
+flowaccum_d8_burned_filled_{}.tif
+flowdir_d8_burned_filled_{}.tif
+flowdir_d8_burned_filled_flows_{}.tif
+flows_grid_boolean_{}.tif
+flows_points_pixels_{}.gpkg
+gw_catchments_pixels_{}.tif
+gw_catchments_reaches_{}.gpkg
+gw_catchments_reaches_{}.tif
+#gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_{}.tif
+#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+headwaters_{}.tif
+#hydroTable_{}.csv
+idFile_{}.txt
+nld_rasterized_elev_{}.tif
+#nwm_catchments_proj_subset_levelPaths_{}.gpkg
+#nwm_subset_streams_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
+rem_{}.tif
+#rem_zeroed_masked_{}.tif
+slopes_d8_dem_meters_{}.tif
+slopes_d8_dem_meters_masked_{}.tif
+sn_catchments_reaches_{}.tif
+src_{}.json
+src_base_{}.csv
+#src_full_crosswalked_{}.csv
+stage_{}.txt
+streamOrder_{}.tif
+treeFile_{}.txt
diff --git a/config/deny_gms_branches_min.lst b/config/deny_gms_branches_min.lst
new file mode 100644
index 000000000..0ff709c82
--- /dev/null
+++ b/config/deny_gms_branches_min.lst
@@ -0,0 +1,58 @@
+# List of files for gms branches to delete
+# Use comment to allow list the file
+# Use curly braces to denote branch_id
+agree_bufgrid.tif
+agree_bufgrid_allo.tif
+agree_bufgrid_dist.tif
+agree_smogrid.tif
+agree_smogrid_allo.tif
+agree_smogrid_dist.tif
+catch_list_{}.txt
+coordFile_{}.txt
+crosswalk_table_{}.csv
+demDerived_reaches_{}.dbf
+demDerived_reaches_{}.prj
+demDerived_reaches_{}.shp
+demDerived_reaches_{}.shx
+demDerived_reaches_split_{}.gpkg
+demDerived_reaches_split_filtered_{}.gpkg
+demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+demDerived_reaches_split_points_{}.gpkg
+demDerived_streamPixels_{}.tif
+demDerived_streamPixels_ids_{}.tif
+demDerived_streamPixels_ids_{}_allo.tif
+demDerived_streamPixels_ids_{}_dist.tif
+dem_burned_{}.tif
+dem_burned_filled_{}.tif
+dem_lateral_thalweg_adj_{}.tif
+dem_meters_{}.tif
+dem_thalwegCond_{}.tif
+flowaccum_d8_burned_filled_{}.tif
+flowdir_d8_burned_filled_{}.tif
+flowdir_d8_burned_filled_flows_{}.tif
+flows_grid_boolean_{}.tif
+flows_points_pixels_{}.gpkg
+gw_catchments_pixels_{}.tif
+gw_catchments_reaches_{}.gpkg
+gw_catchments_reaches_{}.tif
+#gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_{}.tif
+gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+headwaters_{}.tif
+#hydroTable_{}.csv
+idFile_{}.txt
+nld_rasterized_elev_{}.tif
+nwm_catchments_proj_subset_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
+rem_{}.tif
+#rem_zeroed_masked_{}.tif
+slopes_d8_dem_meters_{}.tif
+slopes_d8_dem_meters_masked_{}.tif
+sn_catchments_reaches_{}.tif
+src_{}.json
+src_base_{}.csv
+#src_full_crosswalked_{}.csv
+stage_{}.txt
+streamOrder_{}.tif
+treeFile_{}.txt
diff --git a/config/deny_gms_unit_default.lst b/config/deny_gms_unit_default.lst
new file mode 100644
index 000000000..f19e57773
--- /dev/null
+++ b/config/deny_gms_unit_default.lst
@@ -0,0 +1,18 @@
+# List of files for gms branches to delete
+# Use comment to allow list the file
+NHDPlusBurnLineEvent_subset.gpkg
+#branch_id.lst
+#branch_polygons.gpkg
+nhd_headwater_points_subset.gpkg
+#nld_subset_levees.gpkg
+nwm_catchments_proj_subset.gpkg
+#nwm_catchments_proj_subset_levelPaths.gpkg
+nwm_headwaters.gpkg
+#nwm_lakes_proj_subset.gpkg
+nwm_subset_streams.gpkg
+#nwm_subset_streams_levelPaths.gpkg
+#nwm_subset_streams_levelPaths_dissolved.gpkg
+#nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+#wbd.gpkg
+#wbd8_clp.gpkg
+wbd_buffered.gpkg
diff --git a/config/mannings_calibrated.json b/config/mannings_default_calibrated.json
similarity index 100%
rename from config/mannings_calibrated.json
rename to config/mannings_default_calibrated.json
diff --git a/config/params_template.env b/config/params_template.env
index 41df712c6..72f58e75c 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -20,6 +20,11 @@ export slope_min=0.001
 export min_catchment_area=0.25
 export min_stream_length=0.5
 
+#### gms parameters ####
+export branch_id_attribute=levpa_id
+export branch_buffer_distance_meters=7000
+export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.
+
 #### bathy SRC estimation parameters ####
 export bathy_src_toggle=True # Toggle to run BARC routine (True=on; False=off)
 export bankfull_input_table="data/inputs/bathymetry/nwm_flow_bieger_qreg.csv" # input file location with feature_id and channel geometry attributes
diff --git a/config/params_template_calibrated.env b/config/params_template_calibrated.env
new file mode 100644
index 000000000..b6a0694f4
--- /dev/null
+++ b/config/params_template_calibrated.env
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+#### hydroconditioning parameters ####
+export negative_burn_value=1000
+export agree_DEM_buffer=70
+export wbd_buffer=5000
+export thalweg_lateral_elev_threshold=3
+
+#### geospatial parameters ####
+export max_split_distance_meters=1500
+export ms_buffer_dist=7000
+export lakes_buffer_dist_meters=20
+
+#### rating curve parameters ####
+export manning_n="/foss_fim/config/mannings_default_calibrated.json"
+export stage_min_meters=0
+export stage_interval_meters=0.3048
+export stage_max_meters=25
+export slope_min=0.001
+export min_catchment_area=0.25
+export min_stream_length=0.5
+
+#### gms parameters ####
+export branch_id_attribute=levpa_id
+export branch_buffer_distance_meters=7000
+export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.
+
+#### bathy SRC estimation parameters ####
+export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
+#export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
+export bathy_src_modification=True
+export surf_area_thalweg_ratio_flag=10
+export thalweg_stg_search_max_limit=3
+export bathy_xs_area_chg_flag=1
+export bankful_xs_area_ratio_flag=10
+export thalweg_hyd_radius_flag=10
+
+#### computational parameters ####
+export ncores_gw=1 # mpi number of cores for gagewatershed
+export ncores_fd=1 # mpi number of cores for flow directions
+export default_max_jobs=1 # default number of max concurrent jobs to run
+export memfree=0G # min free memory required to start a new job or keep youngest job alive
+
+#### logging parameters ####
+export startDiv="\n##########################################################################\n"
+export stopDiv="\n##########################################################################"
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3ce00ed1d..de3a8ac9a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.0.0 - 2022-02-01 - [PR #524](https://github.com/NOAA-OWP/cahaba/pull/524)
+
+FIM4 builds upon FIM3 and allows for better representation of inundation through the reduction of artificial restriction of inundation at catchment boundaries.
+
+More details will be made available through a publication by Aristizabal et. al. and will be included in the "Credits and References" section of the README.md, titled "Reducing Horton-Strahler Stream Order Can Enhance Flood Inundation Mapping Skill with Applications for the U.S. National Water Model."
+
+## Additions
+
+- `/src/gms`: A new directory containing scripts necessary to produce the FIM4 Height Above Nearest Drainage grids and synthetic rating curves needed for inundation mapping.
+- `/tools/gms_tools`: A new directory containing scripts necessary to generate and evaluate inundation maps produced from FIM4 Height Above Nearest Drainage grids and synthetic rating curves.
+
+
+<br/><br/>
+
+
 ## v3.0.24.3 - 2021-11-29 - [PR #488](https://github.com/NOAA-OWP/cahaba/pull/488)
 
 Fixed projection issue in `synthesize_test_cases.py`.
diff --git a/gms_pipeline.sh b/gms_pipeline.sh
new file mode 100755
index 000000000..b79efe891
--- /dev/null
+++ b/gms_pipeline.sh
@@ -0,0 +1,54 @@
+#!/bin/bash -e
+
+#################################
+#### Sample commands for GMS ####
+#################################
+
+# produce FIM 3 commands for preprocessing as normal
+# as a comment, we should look into consolidating these three steps below into one
+
+## acquire nhd and nwm inputs then prepare them
+# acquire_and_preprocess_inputs.py
+
+## reproject rasters and convert to meters
+# preprocess_rasters.py
+
+## aggregate nhd vectors
+# aggregate_vectors.py
+
+
+## GMS ##
+
+## produce gms hydrofabric at unit level first 
+gms_run_unit.sh -u 11140105 -n test_output -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst
+
+## produce level path or branch level datasets
+gms_run_branch.sh -n test_output -u 11140105 -c /foss_fim/config/params_template.env -j 7 -d /foss_fim/config/deny_gms_branches_default.lst
+
+
+## Do either FIM or Test case information below ## 
+
+## FIM ##
+
+# make sure you have an outputs directory for FIMs setup first
+# using /data/temp here for demo
+
+# produce level path scale FIM
+/foss_fim/tools/gms_tools/inundate_gms.py -y /data/outputs/test_output -u 11140105 -f /data/test_cases/ble_test_cases/validation_data_ble/11140105/100yr/ble_huc_11140105_flows_100yr.csv -w 7 -i /data/temp/11140105_inundation_test.tif -o /data/temp/11140105_inundation_test.csv -v
+
+# mosaic level path FIM
+/foss_fim/tools/gms_tools/mosaic_inundation.py -i /data/temp/11140105_inundation_test.csv -a /data/outputs/test_output/11140105/wbd.gpkg -v -t inundation_rasters -m /data/temp/11140105_inundation_test.tif
+
+## Test Cases ##
+
+# singular test case
+/foss_fim/tools/run_test_case.py -r test_output/11140105 -b test_output -t 11140105_ble -e GMS -m HUC -w 7 -v -vg
+
+# batch of test cases
+# synthesize_test_cases.py
+/foss_fim/tools/synthesize_test_cases.py -e GMS -jh 2 -jb 3 -b ble -vr -v test_output -s your_string
+
+# aggregate results from synthesize test cases
+# will fill in more here later
+/foss_fim/tools/consolidate_metrics.py -v test_output <a FR outputs dir> <a MS outputs dir> -o /data/temp/metrics.csv
+# if only one of each is provided above (FR, MS, GMS) then you can pass -i to impute FR metrics in HUCs with missing MSoutputs
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
new file mode 100755
index 000000000..31ffeef4e
--- /dev/null
+++ b/gms_run_branch.sh
@@ -0,0 +1,137 @@
+#!/bin/bash -e
+:
+usage ()
+{
+    echo 'Produce GMS hydrofabric at levelpath/branch scale. Execute gms_run_unit.sh prior to.'
+    echo 'Usage : gms_run_branch.sh [REQ: -c <config file> -n <run name> ] [OPT: -h -j <job limit>]'
+    echo ''
+    echo 'REQUIRED:'
+    echo '  -c/--config     : configuration file with bash environment variables to export'
+    echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
+    echo ''
+    echo 'OPTIONS:'
+    echo '  -h/--help       : help file'
+    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
+    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
+    echo '  -r/--retry      : retries failed jobs'
+    echo '  -o/--overwrite  : overwrite outputs if already exist'
+    echo '  -d/--denylist  : file with line delimited list of files in branches directories to remove upon completion'
+    echo '                   (see config/deny_gms_branches_default.lst for a starting point)'
+    echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
+    echo '                     also accepted. HUCs must present in inputs directory.'
+    exit
+}
+
+if [ "$#" -lt 4 ]
+then
+  usage
+fi
+
+while [ "$1" != "" ]; do
+case $1
+in
+    -c|--configFile )
+        shift
+        envFile=$1
+        ;;
+    -n|--runName)
+        shift
+        runName=$1
+        ;;
+    -j|--jobLimit)
+        shift
+        jobLimit=$1
+        ;;
+    -u|--hucList)
+        shift
+        hucList=$1
+        ;;
+    -h|--help)
+        shift
+        usage
+        ;;
+    -o|--overwrite)
+        overwrite=1
+        ;;
+    -r|--retry)
+        retry="--retry-failed"
+        overwrite=1
+        ;;
+    -d|--denylist)
+        shift
+        deny_gms_branches_list=$1
+        ;;
+    *) ;;
+    esac
+    shift
+done
+
+# print usage if arguments empty
+if [ "$envFile" = "" ]
+then
+    usage
+fi
+if [ "$runName" = "" ]
+then
+    usage
+fi
+if [ "$overwrite" = "" ]
+then
+    overwrite=0
+fi
+if [ -z "$retry" ]
+then
+    retry=""
+fi
+
+## SOURCE ENV FILE AND FUNCTIONS ##
+source $envFile
+source $srcDir/bash_functions.env
+
+# default values
+if [ "$jobLimit" = "" ] ; then
+    jobLimit=$default_max_jobs
+fi
+
+## Define Outputs Data Dir & Log File##
+export outputRunDataDir=$outputDataDir/$runName
+export deny_gms_branches_list=$deny_gms_branches_list
+logFile=$outputRunDataDir/logs/branch/summary_gms_branch.log
+export extent=GMS
+export overwrite=$overwrite
+
+## Check for run data directory ##
+if [ ! -d "$outputRunDataDir" ]; then 
+    echo "Depends on output from gms_run_unit.sh. Please produce data with gms_run_unit.sh first."
+    exit 1
+fi
+
+## Filter out hucs ##
+if [ "$hucList" = "" ]; then
+    gms_inputs=$outputRunDataDir/gms_inputs.csv
+else
+    $srcDir/gms/filter_gms_inputs_by_huc.py -g $outputRunDataDir/gms_inputs.csv -u $hucList -o $outputRunDataDir/gms_inputs_filtered.csv
+    gms_inputs=$outputRunDataDir/gms_inputs_filtered.csv
+fi
+
+# Echo intent to retry
+if [ "$retry" = "--retry-failed" ]; then
+    echo "Retrying failed unit level jobs for $runName"
+fi 
+
+# make log dir
+if [ ! -d "$outputRunDataDir/logs/branch" ]; then
+    mkdir -p $outputRunDataDir/logs/branch
+fi
+
+## RUN GMS BY BRANCH ##
+if [ "$jobLimit" -eq 1 ]; then
+    parallel $retry --verbose --timeout $branch_timeout --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
+else
+    parallel $retry --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
+fi
+
+
+# -------------------
+## GET NON ZERO EXIT CODES ##
+grep -ER 'Exit status: [1-9]' $outputRunDataDir/logs/branch/*_branch_*.log > $outputRunDataDir/logs/branch/non_zero_exit_codes.log
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
new file mode 100755
index 000000000..b75274167
--- /dev/null
+++ b/gms_run_unit.sh
@@ -0,0 +1,161 @@
+#!/bin/bash -e
+:
+usage ()
+{
+    echo 'Produce GMS hydrofabric datasets for unit scale. Run after fim_run.sh but before gms_run_branch.sh'
+    echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -c <config file> -n <run name> ] [OPT: -h -j <job limit>]'
+    echo ''
+    echo 'REQUIRED:'
+    echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
+    echo '                     also accepted. HUCs must present in inputs directory.'
+    echo '  -c/--config     : configuration file with bash environment variables to export'
+    echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
+    echo ''
+    echo 'OPTIONS:'
+    echo '  -h/--help       : help file'
+    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
+    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
+    echo '  -o/--overwrite  : overwrite outputs if already exist'
+    echo '  -r/--retry      : retries failed jobs'
+    echo '  -d/--denylist   : file with line delimited list of files in huc directories to remove upon completion'
+    echo '                   (see config/deny_gms_unit_default.lst for a starting point)'
+    exit
+}
+
+if [ "$#" -lt 6 ]
+then
+  usage
+fi
+
+
+while [ "$1" != "" ]; do
+case $1
+in
+    -u|--hucList)
+        shift
+        hucList="$1"
+        ;;
+    -c|--configFile )
+        shift
+        envFile=$1
+        ;;
+    -n|--runName)
+        shift
+        runName=$1
+        ;;
+    -j|--jobLimit)
+        shift
+        jobLimit=$1
+        ;;
+    -h|--help)
+        shift
+        usage
+        ;;
+    -o|--overwrite)
+        overwrite=1
+        ;;
+    -r|--retry)
+        retry="--retry-failed"
+        overwrite=1
+        ;;
+    -d|--denylist)
+        shift
+        deny_gms_unit_list=$1
+        ;;
+    *) ;;
+    esac
+    shift
+done
+
+# print usage if arguments empty
+if [ "$hucList" = "" ]
+then
+    usage
+fi
+if [ "$envFile" = "" ]
+then
+    usage
+fi
+if [ "$runName" = "" ]
+then
+    usage
+fi
+if [ "$deny_gms_unit_list" = "" ]
+then
+    usage
+fi
+if [ -z "$overwrite" ]
+then
+    overwrite=0
+fi
+if [ -z "$retry" ]
+then
+    retry=""
+fi
+
+## SOURCE ENV FILE AND FUNCTIONS ##
+source $envFile
+source $srcDir/bash_functions.env
+
+# default values
+if [ "$jobLimit" = "" ] ; then
+    jobLimit=$default_max_jobs
+fi
+
+## Define Outputs Data Dir & Log File##
+export outputRunDataDir=$outputDataDir/$runName
+logFile=$outputRunDataDir/logs/unit/summary_gms_unit.log
+export overwrite=$overwrite
+
+## Define inputs
+export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
+export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
+export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
+export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
+export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
+export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
+export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
+export deny_gms_unit_list=$deny_gms_unit_list
+
+## Input handling ##
+$srcDir/check_huc_inputs.py -u "$hucList"
+
+## Make output and data directories ##
+if [ "$retry" = "--retry-failed" ]; then
+    echo "Retrying failed unit level jobs for $runName"
+fi
+
+# make dirs
+if [ ! -d $outputRunDataDir ]; then
+    mkdir -p $outputRunDataDir
+fi
+if [ ! -d "$outputRunDataDir/logs/unit" ]; then
+    mkdir -p $outputRunDataDir/logs/unit
+fi
+
+# copy over config file
+cp -a $envFile $outputRunDataDir
+
+## GMS BY UNIT##
+if [ -f "$hucList" ]; then
+    if [ "$jobLimit" -eq 1 ]; then
+        parallel $retry --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
+    else
+        parallel $retry --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
+    fi
+else 
+    if [ "$jobLimit" -eq 1 ]; then
+        parallel $retry --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
+    else
+        parallel $retry --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
+    fi
+ fi
+
+
+## AGGREGATE BRANCH LISTS INTO ONE ##
+python3 $srcDir/gms/aggregate_branch_lists.py -l $hucList
+
+
+## GET NON ZERO EXIT CODES ##
+# get positive non-zero exit codes 
+grep -ER 'Exit status: [1-9]' $outputRunDataDir/logs/unit/*_unit.log > $outputRunDataDir/logs/unit/non_zero_exit_codes.log
diff --git a/lib/aggregate_fim_outputs.py b/lib/aggregate_fim_outputs.py
new file mode 100644
index 000000000..edafd93a3
--- /dev/null
+++ b/lib/aggregate_fim_outputs.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import pandas as pd
+import json
+import rasterio
+from rasterio.merge import merge
+import shutil
+import csv
+from utils.shared_variables import PREP_PROJECTION
+
+
+def aggregate_fim_outputs(fim_out_dir):
+
+    print ("aggregating outputs to HUC6 scale")
+
+    drop_folders = ['logs']
+    huc_list = [huc for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
+    huc6_list = [str(huc[0:6]) for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
+    huc6_list = list(set(huc6_list))
+
+    for huc in huc_list:
+
+        os.makedirs(os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6])), exist_ok=True)
+
+        # original file paths
+        hydrotable_filename = os.path.join(fim_out_dir,huc,'hydroTable.csv')
+        src_filename = os.path.join(fim_out_dir,huc,'src.json')
+
+        # aggregate file name paths
+        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),'hydroTable.csv')
+        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),f'rating_curves_{huc[0:6]}.json')
+
+        if len(huc)> 6:
+
+            # open hydrotable
+            hydrotable = pd.read_csv(hydrotable_filename)
+
+            # write/append aggregate hydrotable
+            if os.path.isfile(aggregate_hydrotable):
+                hydrotable.to_csv(aggregate_hydrotable,index=False, mode='a',header=False)
+            else:
+                hydrotable.to_csv(aggregate_hydrotable,index=False)
+
+            del hydrotable
+
+            # open src
+            src = open(src_filename)
+            src = json.load(src)
+
+            # write/append aggregate src
+            if os.path.isfile(aggregate_src):
+
+                with open(aggregate_src, "r+") as file:
+                    data = json.load(file)
+                    data.update(src)
+
+                with open(aggregate_src, 'w') as outfile:
+                    json.dump(data, outfile)
+            else:
+                with open(aggregate_src, 'w') as outfile:
+                        json.dump(src, outfile)
+
+            del src
+
+        else:
+            shutil.copy(hydrotable_filename, aggregate_hydrotable)
+            shutil.copy(src_filename, aggregate_src)
+
+    for huc6 in huc6_list:
+
+        ## add feature_id to aggregate src
+        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),'hydroTable.csv')
+        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),f'rating_curves_{huc6}.json')
+
+        # Open aggregate src for writing feature_ids to
+        src_data = {}
+        with open(aggregate_src) as jsonf:
+            src_data = json.load(jsonf)
+
+        with open(aggregate_hydrotable) as csvf:
+            csvReader = csv.DictReader(csvf)
+
+            for row in csvReader:
+                if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
+                    src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
+
+        # Write src_data to JSON file
+        with open(aggregate_src, 'w') as jsonf:
+            json.dump(src_data, jsonf)
+
+        ## aggregate rasters
+        huc6_dir = os.path.join(fim_out_dir,'aggregate_fim_outputs',huc6)
+
+        # aggregate file paths
+        rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}.tif')
+        catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}.tif')
+
+        if huc6 not in huc_list:
+
+            huc6_filter = [path.startswith(huc6) for path in huc_list]
+            subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
+
+            # aggregate and mosaic rem
+            rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
+
+            if len(rem_list) > 1:
+
+                rem_files_to_mosaic = []
+
+                for rem in rem_list:
+
+                    rem_src = rasterio.open(rem)
+                    rem_files_to_mosaic.append(rem_src)
+
+                mosaic, out_trans = merge(rem_files_to_mosaic)
+                out_meta = rem_src.meta.copy()
+                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
+
+                with rasterio.open(rem_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
+                    dest.write(mosaic)
+
+                del rem_files_to_mosaic,rem_src,out_meta,mosaic
+
+            elif len(rem_list)==1:
+
+                shutil.copy(rem_list[0], rem_mosaic)
+
+            # aggregate and mosaic catchments
+            catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
+
+            if len(catchment_list) > 1:
+
+                cat_files_to_mosaic = []
+
+                for cat in catchment_list:
+                    cat_src = rasterio.open(cat)
+                    cat_files_to_mosaic.append(cat_src)
+
+                mosaic, out_trans = merge(cat_files_to_mosaic)
+                out_meta = cat_src.meta.copy()
+
+                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
+
+                with rasterio.open(catchment_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
+                    dest.write(mosaic)
+
+                del cat_files_to_mosaic,cat_src,out_meta,mosaic
+
+            elif len(catchment_list)==1:
+
+                shutil.copy(catchment_list[0], catchment_mosaic)
+
+        else:
+            # original file paths
+            rem_filename = os.path.join(fim_out_dir,huc6,'rem_zeroed_masked.tif')
+            catchment_filename = os.path.join(fim_out_dir,huc6,'gw_catchments_reaches_filtered_addedAttributes.tif')
+
+            shutil.copy(rem_filename, rem_mosaic)
+            shutil.copy(catchment_filename, catchment_mosaic)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Aggregate layers buy HUC6')
+    parser.add_argument('-d','--fim-outputs-directory', help='FIM outputs directory', required=True)
+
+
+    args = vars(parser.parse_args())
+
+    fim_outputs_directory = args['fim_outputs_directory']
+
+    aggregate_fim_outputs(fim_outputs_directory)
diff --git a/lib/fr_to_ms_raster_mask.py b/lib/fr_to_ms_raster_mask.py
new file mode 100755
index 000000000..567b5cf19
--- /dev/null
+++ b/lib/fr_to_ms_raster_mask.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+'''
+ Description: Mask raster layers using 'mainstems' stream buffer
+'''
+
+import sys
+import os
+import geopandas as gpd
+import rasterio.mask
+
+split_flows_fileName    = sys.argv[1]
+fdr_fr                  = sys.argv[2]
+dem_fr                  = sys.argv[3]
+slope_fr                = sys.argv[4]
+fdr_ms_filename         = sys.argv[5]
+dem_ms_filename         = sys.argv[6]
+slope_ms_filename       = sys.argv[7]
+str_pixel_fr            = sys.argv[8]
+str_pixel_ms_filename   = sys.argv[9]
+ms_buffer_dist          = int(os.environ['ms_buffer_dist'])
+
+# create output layer names
+split_flows = gpd.read_file(split_flows_fileName)
+
+# Limit the rasters to the buffer distance around the draft streams.
+print ("Limiting rasters to buffer area ({} meters) around model streams".format(str(ms_buffer_dist)))
+
+split_flows_ms_buffer = split_flows.unary_union.buffer(ms_buffer_dist)
+
+print('Writing raster outputs ...')
+
+# Mask nhddem
+with rasterio.open(dem_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+    out_meta = src.meta
+
+out_meta.update({"driver": "GTiff",
+     "height": out_image.shape[1],
+     "width": out_image.shape[2],
+     "transform": out_transform})
+
+with rasterio.open(os.path.join(os.path.dirname(dem_fr), dem_ms_filename), "w", **out_meta) as dest:
+    dest.write(out_image)
+
+# Mask fdr
+with rasterio.open(fdr_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+    out_meta = src.meta
+
+out_meta.update({"driver": "GTiff",
+     "height": out_image.shape[1],
+     "width": out_image.shape[2],
+     "transform": out_transform})
+
+with rasterio.open(os.path.join(os.path.dirname(fdr_fr), fdr_ms_filename), "w", **out_meta) as dest:
+    dest.write(out_image)
+
+# Mask slope
+with rasterio.open(slope_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+    out_meta = src.meta
+
+out_meta.update({"driver": "GTiff",
+     "height": out_image.shape[1],
+     "width": out_image.shape[2],
+     "transform": out_transform})
+
+with rasterio.open(os.path.join(os.path.dirname(slope_fr), slope_ms_filename), "w", **out_meta) as dest:
+    dest.write(out_image)
+
+# Mask stream pixels
+with rasterio.open(str_pixel_fr) as src:
+    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
+    out_meta = src.meta
+
+out_meta.update({"driver": "GTiff",
+     "height": out_image.shape[1],
+     "width": out_image.shape[2],
+     "transform": out_transform})
+
+with rasterio.open(os.path.join(os.path.dirname(str_pixel_fr), str_pixel_ms_filename), "w", **out_meta) as dest:
+    dest.write(out_image)
diff --git a/src/acquire_and_preprocess_inputs.py b/src/acquire_and_preprocess_inputs.py
index 417cc4d93..37cb33e3d 100755
--- a/src/acquire_and_preprocess_inputs.py
+++ b/src/acquire_and_preprocess_inputs.py
@@ -19,12 +19,16 @@
                                     NHD_VECTOR_EXTRACTION_SUFFIX,
                                     PREP_PROJECTION,
                                     WBD_NATIONAL_URL,
-                                    FIM_ID
+                                    FIM_ID,
+                                    OVERWRITE_WBD,
+                                    OVERWRITE_NHD,
+                                    OVERWRITE_ALL,
+                                    nhd_raster_url_template,
+                                    nhd_vector_url_template
                                     )
 
-from utils.shared_functions import (pull_file, 
-                                    run_system_command,
-                                    delete_file,
+from utils.shared_functions import (pull_file, run_system_command,
+                                    subset_wbd_gpkg, delete_file,
                                     getDriver)
 
 NHDPLUS_VECTORS_DIRNAME = 'nhdplus_vectors'
@@ -32,14 +36,12 @@
 NWM_HYDROFABRIC_DIRNAME = 'nwm_hydrofabric'
 NWM_FILE_TO_SUBSET_WITH = 'nwm_flows.gpkg'
 
-
 def subset_wbd_to_nwm_domain(wbd,nwm_file_to_use):
 
     intersecting_indices = [not (gpd.read_file(nwm_file_to_use,mask=b).empty) for b in wbd.geometry]
 
     return(wbd[intersecting_indices])
 
-
 def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_use,overwrite_wbd,num_workers):
     """
     This helper function pulls and unzips Watershed Boundary Dataset (WBD) data. It uses the WBD URL defined by WBD_NATIONAL_URL.
@@ -126,7 +128,6 @@ def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_
 
     return(wbd_directory)
 
-
 def pull_and_prepare_nwm_hydrofabric(path_to_saved_data_parent_dir, path_to_preinputs_dir,num_workers):
     """
     This helper function pulls and unzips NWM hydrofabric data. It uses the NWM hydrofabric URL defined by NWM_HYDROFABRIC_URL.
@@ -156,20 +157,20 @@ def pull_and_prepare_nwm_hydrofabric(path_to_saved_data_parent_dir, path_to_prei
         pool.map(run_system_command, procs_list)
 
 
-def pull_and_prepare_nhd_data(args):
+def pull_and_prepare_nhd_data(nhd_raster_download_url,
+                              nhd_raster_extraction_path,
+                              nhd_vector_download_url,
+                              nhd_vector_extraction_path,
+                              overwrite_nhd_dem,
+                              overwrite_nhd_gdb
+                             ):
     """
     This helper function is designed to be multiprocessed. It pulls and unzips NHD raster and vector data.
     Args:
         args (list): A list of arguments in this format: [nhd_raster_download_url, nhd_raster_extraction_path, nhd_vector_download_url, nhd_vector_extraction_path]
     """
-    # Parse urls and extraction paths from procs_list.
-    nhd_raster_download_url = args[0]
-    nhd_raster_extraction_path = args[1]
-    nhd_vector_download_url = args[2]
-    nhd_vector_extraction_path = args[3]
-    overwrite_nhd = args[4]
-
-    nhd_gdb = nhd_vector_extraction_path.replace('.zip', '.gdb')  # Update extraction path from .zip to .gdb.
+    # Update extraction path from .zip to .gdb
+    nhd_gdb = nhd_vector_extraction_path.replace('.zip', '.gdb')
 
     # Download raster and vector, if not already in user's directory (exist check performed by pull_file()).
     nhd_raster_extraction_parent = os.path.dirname(nhd_raster_extraction_path)
@@ -181,7 +182,8 @@ def pull_and_prepare_nhd_data(args):
         os.mkdir(nhd_raster_parent_dir)
 
     elev_cm_tif = os.path.join(nhd_raster_parent_dir, 'elev_cm.tif')
-    if not os.path.exists(elev_cm_tif) or overwrite_nhd:
+    elev_m_tif = os.path.join(nhd_raster_parent_dir, 'elev_m.tif')
+    if not os.path.exists(elev_cm_tif) or overwrite_nhd_dem:
         pull_file(nhd_raster_download_url, nhd_raster_extraction_path)
         os.system("7za e {nhd_raster_extraction_path} -o{nhd_raster_parent_dir} elev_cm.tif -r ".format(nhd_raster_extraction_path=nhd_raster_extraction_path, nhd_raster_parent_dir=nhd_raster_parent_dir))
 
@@ -200,7 +202,7 @@ def pull_and_prepare_nhd_data(args):
     if not os.path.exists(nhd_vector_extraction_parent):
         os.mkdir(nhd_vector_extraction_parent)
 
-    if not os.path.exists(nhd_gdb) or overwrite_nhd:  # Only pull if not already pulled and processed.
+    if not os.path.exists(nhd_gdb) or overwrite_nhd_gdb:  # Only pull if not already pulled and processed.
         # Download and fully unzip downloaded GDB.
         pull_file(nhd_vector_download_url, nhd_vector_extraction_path)
         huc = os.path.split(nhd_vector_extraction_parent)[1]  # Parse HUC.
@@ -285,7 +287,12 @@ def build_huc_list_files(path_to_saved_data_parent_dir, wbd_directory):
             f.write("%s\n" % item)
 
 
-def manage_preprocessing(hucs_of_interest, num_workers=1,overwrite_nhd=False, overwrite_wbd=False):
+def manage_preprocessing( hucs_of_interest,
+                          num_workers=1,
+                          overwrite_nhd_dem=False,
+                          overwrite_nhd_gdb=False,
+                          overwrite_wbd=False
+                        ):
     """
     This functions manages the downloading and preprocessing of gridded and vector data for FIM production.
 
@@ -330,24 +337,29 @@ def manage_preprocessing(hucs_of_interest, num_workers=1,overwrite_nhd=False, ov
         except FileNotFoundError:
             huc_list = list(hucs_of_interest)
 
+    # get unique huc4s
+    huc_list = [h[0:4] for h in huc_list]
+    huc_list = list( set(huc_list) )
 
     # Construct paths to data to download and append to procs_list for multiprocessed pull, project, and converstion to geopackage.
     for huc in huc_list:
         huc = str(huc)  # Ensure huc is string.
 
         # Construct URL and extraction path for NHDPlus raster.
-        nhd_raster_download_url = os.path.join(NHD_URL_PARENT, NHD_URL_PREFIX + huc + NHD_RASTER_URL_SUFFIX)
+        #nhd_raster_download_url = os.path.join(NHD_URL_PARENT, NHD_URL_PREFIX + huc + NHD_RASTER_URL_SUFFIX)
+        nhd_raster_download_url = nhd_raster_url_template.format(huc) 
         nhd_raster_extraction_path = os.path.join(nhd_raster_dir, NHD_URL_PREFIX + huc + NHD_RASTER_URL_SUFFIX)
-
+        
         # Construct URL and extraction path for NHDPlus vector. Organize into huc-level subdirectories.
-        nhd_vector_download_url = os.path.join(NHD_URL_PARENT, NHD_URL_PREFIX + huc + NHD_VECTOR_URL_SUFFIX)
+        #nhd_vector_download_url = os.path.join(NHD_URL_PARENT, NHD_URL_PREFIX + huc + NHD_VECTOR_URL_SUFFIX)
+        nhd_vector_download_url = nhd_vector_url_template.format(huc) 
         nhd_vector_download_parent = os.path.join(vector_data_dir, huc)
         if not os.path.exists(nhd_vector_download_parent):
             os.mkdir(nhd_vector_download_parent)
         nhd_vector_extraction_path = os.path.join(nhd_vector_download_parent, NHD_VECTOR_EXTRACTION_PREFIX + huc + NHD_VECTOR_EXTRACTION_SUFFIX)
 
         # Append extraction instructions to nhd_procs_list.
-        nhd_procs_list.append([nhd_raster_download_url, nhd_raster_extraction_path, nhd_vector_download_url, nhd_vector_extraction_path, overwrite_nhd])
+        nhd_procs_list.append([nhd_raster_download_url, nhd_raster_extraction_path, nhd_vector_download_url, nhd_vector_extraction_path, overwrite_nhd_dem, overwrite_nhd_gdb])
 
     # Pull and prepare NHD data.
     # with Pool(processes=num_workers) as pool:
@@ -355,7 +367,7 @@ def manage_preprocessing(hucs_of_interest, num_workers=1,overwrite_nhd=False, ov
 
     for huc in nhd_procs_list:
         try:
-            pull_and_prepare_nhd_data(huc)
+            pull_and_prepare_nhd_data(*huc)
         except HTTPError:
             print("404 error for HUC4 {}".format(huc))
 
@@ -375,7 +387,8 @@ def manage_preprocessing(hucs_of_interest, num_workers=1,overwrite_nhd=False, ov
     parser = argparse.ArgumentParser(description='Acquires and preprocesses WBD and NHD data for use in fim_run.sh.')
     parser.add_argument('-u','--hucs-of-interest',help='HUC4, series of HUC4s, or path to a line-delimited file of HUC4s to acquire.',required=True,nargs='+')
     #parser.add_argument('-j','--num-workers',help='Number of workers to process with',required=False,default=1,type=int)
-    parser.add_argument('-n', '--overwrite-nhd', help='Optional flag to overwrite NHDPlus Data',required=False,action='store_true')
+    parser.add_argument('-nd', '--overwrite-nhd-dem', help='Optional flag to overwrite NHDPlus DEM Data',required=False,action='store_true',default=False)
+    parser.add_argument('-ng', '--overwrite-nhd-gdb', help='Optional flag to overwrite NHDPlus GDB Data',required=False,action='store_true',default=False)
     parser.add_argument('-w', '--overwrite-wbd', help='Optional flag to overwrite WBD Data',required=False,action='store_true')
 
     # Extract to dictionary and assign to variables.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 9b38418b2..37ee0a25d 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -8,10 +8,13 @@
 import json
 import argparse
 import sys
+from bathy_rc_adjust import bathy_rc_lookup
+from utils.shared_functions import getDriver
 # sys.path.append('/foss_fim/src')
 # sys.path.append('/foss_fim/config')
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
+from memory_profiler import profile
 
 
 @mem_profile
@@ -51,10 +54,14 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
         output_flows = output_flows.merge(relevant_input_nwmflows[['order_','feature_id']],on='feature_id')
         output_flows = output_flows.merge(output_catchments.filter(items=['HydroID','areasqkm']),on='HydroID')
 
-    elif extent == 'MS':
+    elif (extent == 'MS') | (extent == 'GMS'):
         ## crosswalk using stream segment midpoint method
         input_nwmcat = gpd.read_file(input_nwmcat_fileName, mask=input_huc)
-        input_nwmcat = input_nwmcat.loc[input_nwmcat.mainstem==1]
+
+        # only reduce nwm catchments to mainstems if running mainstems
+        if extent == 'MS':
+            input_nwmcat = input_nwmcat.loc[input_nwmcat.mainstem==1]
+
         input_nwmcat = input_nwmcat.rename(columns={'ID':'feature_id'})
         if input_nwmcat.feature_id.dtype != 'int': input_nwmcat.feature_id = input_nwmcat.feature_id.astype(int)
         input_nwmcat=input_nwmcat.set_index('feature_id')
@@ -103,6 +110,11 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
         if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
         output_flows = input_flows.merge(crosswalk,on='HydroID')
+
+        # added for GMS. Consider adding filter_catchments_and_add_attributes.py to run_by_branch.sh
+        if 'areasqkm' not in output_catchments.columns:
+            output_catchments['areasqkm'] = output_catchments.geometry.area/(1000**2)
+
         output_flows = output_flows.merge(output_catchments.filter(items=['HydroID','areasqkm']),on='HydroID')
 
     # read in manning's n values
@@ -213,12 +225,18 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
     if extent == 'FR':
         output_src = output_src.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
-    elif extent == 'MS':
+    elif (extent == 'MS') | (extent == 'GMS'):
         output_src = output_src.merge(crosswalk[['HydroID','feature_id']],on='HydroID')
 
     output_crosswalk = output_src[['HydroID','feature_id']]
     output_crosswalk = output_crosswalk.drop_duplicates(ignore_index=True)
 
+    ## bathy estimation integration in synthetic rating curve calculations
+    #if (bathy_src_calc == True and extent == 'MS'):
+    #    output_src = bathy_rc_lookup(output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName)
+    #else:
+    #    print('Note: NOT using bathy estimation approach to modify the SRC...')
+
     # make hydroTable
     output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)','BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)','Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
new file mode 100755
index 000000000..ebda61044
--- /dev/null
+++ b/src/bathy_rc_adjust.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+
+from os import environ
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+
+sa_ratio_flag             = float(environ['surf_area_thalweg_ratio_flag']) #10x
+thal_stg_limit            = float(environ['thalweg_stg_search_max_limit']) #3m
+bankful_xs_ratio_flag     = float(environ['bankful_xs_area_ratio_flag']) #10x
+bathy_xsarea_flag         = float(environ['bathy_xs_area_chg_flag']) #1x
+thal_hyd_radius_flag      = float(environ['thalweg_hyd_radius_flag']) #10x
+
+def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,):
+    ## Convert input_src_base featureid to integer
+    if input_src_base.feature_id.dtype != 'int': input_src_base.feature_id = input_src_base.feature_id.astype(int)
+
+    ## Read in the bankfull channel geometry text file
+    input_bathy = pd.read_csv(input_bathy_fileName, dtype= {'COMID': int})
+
+    ## Merge input_bathy and modified_src_base df using feature_id/COMID attributes
+    input_bathy = input_bathy.rename(columns={'COMID':'feature_id','BANKFULL_WIDTH':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA':'BANKFULL_XSEC_AREA (m2)'})
+    modified_src_base = input_src_base.merge(input_bathy.loc[:,['feature_id','BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA (m2)']],how='left',on='feature_id')
+
+    ## Check that the merge process returned matching feature_id entries
+    if modified_src_base['BANKFULL_WIDTH (m)'].count() == 0:
+        print('No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC!')
+        return(input_src_base)
+    else:
+        ## Use SurfaceArea variable to identify thalweg-restricted stage values for each hydroid
+        ## Calculate the interrow SurfaceArea ratio n/(n-1)
+        modified_src_base['SA_div'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
+        ## Mask SA_div when Stage = 0 or when the SA_div value (n / n-1) is > threshold value (i.e. 10x)
+        modified_src_base['SA_div'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
+        ## Create new df to filter and groupby HydroID
+        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div']]
+        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['Stage']<thal_stg_limit] # assuming thalweg burn-in is less than 3 meters
+        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div'].notnull()]
+        find_thalweg_notch = find_thalweg_notch.loc[find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()].reset_index(drop=True)
+        ## Assign thalweg_burn_elev variable to the stage value found in previous step
+        find_thalweg_notch['Thalweg_burn_elev'] = find_thalweg_notch['Stage']
+        ## Merge the Thalweg_burn_elev value back into the modified SRC --> this is used to mask the discharge after Manning's equation
+        modified_src_base = modified_src_base.merge(find_thalweg_notch.loc[:,['HydroID','Thalweg_burn_elev']],how='left',on='HydroID')
+
+        ## Calculate bankfull vs top width difference for each feature_id
+        modified_src_base['Top Width Diff (m)'] = (modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']).abs()
+        ## Calculate XS Area field (Channel Volume / Stream Length)
+        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (modified_src_base['LENGTHKM'] * 1000)
+
+        ## Groupby HydroID and find min of Top Width Diff (m)
+        output_bathy = modified_src_base[['feature_id','HydroID','order_','Stage','SurfaceArea (m2)','Thalweg_burn_elev','BANKFULL_WIDTH (m)','TopWidth (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','Top Width Diff (m)']]
+        ## filter out stage = 0 rows in SRC (assuming geom at stage 0 is not a valid channel geom)
+        output_bathy = output_bathy[output_bathy['Stage'] > 0]
+        ## filter SRC rows identified as Thalweg burned
+        output_bathy['Top Width Diff (m)'].mask(output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'],inplace=True)
+        ## ignore hydroid/featureid that did not have a valid Bankfull lookup (areas outside CONUS - i.e. Canada)
+        output_bathy = output_bathy[output_bathy['BANKFULL_XSEC_AREA (m2)'].notnull()]
+        ## ignore SRC entries with 0 surface area --> handles input SRC artifacts/errors in Great Lakes region
+        output_bathy = output_bathy[output_bathy['SurfaceArea (m2)'] > 0]
+        ## find index of minimum top width difference --> this will be used as the SRC "bankfull" row for future calcs
+        output_bathy = output_bathy.loc[output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()].reset_index(drop=True)
+        print('Average: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].mean()))
+        print('Minimum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].min()))
+        print('Maximum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].max()))
+        print('STD: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].std()) +'\n' + '#################')
+
+        ## Calculate XS Area difference between SRC and Bankfull database
+        output_bathy['XS Area Diff (m2)'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)'])
+        output_bathy['XS Bankfull Area Ratio'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']).round(2)
+        ## masking negative XS Area Diff and XS Area = 0
+        output_bathy['XS Bankfull Area Ratio'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        ## masking negative XS Area Diff and XS Area = 0
+        output_bathy['XS Area Diff (m2)'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
+        output_bathy['XS Area Diff (m2)'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
+        output_bathy['XS Bankfull Area Ratio'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        ## Print XS Area Diff statistics
+        print('Average: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].mean()))
+        print('Minimum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].min()))
+        print('Maximum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].max()))
+        print('STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std()))
+
+        ## Bin XS Bankfull Area Ratio by stream order
+        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']].copy()
+        ## mask stage values when XS Bankfull Area Ratio is null (need to filter to calculate the median for valid values below)
+        stream_order_bathy_ratio['Stage'].mask(stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(),inplace=True)
+        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(count=('XS Bankfull Area Ratio','count'),mean_xs_area_ratio=('XS Bankfull Area Ratio','mean'),median_stage_bankfull=('Stage','median'))
+        ## fill XS Bankfull Area Ratio and Stage values if no values were found in the grouby calcs
+        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill()+stream_order_bathy_ratio.bfill())/2
+        ## fill first and last stream order values if needed
+        stream_order_bathy_ratio = stream_order_bathy_ratio.bfill().ffill()
+        ## Get count_total tally of the total number of stream order hydroids in the HUC (not filtering anything out)
+        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage','count'))
+        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(stream_order_bathy_ratio_count,how='left',on='order_')
+        ## Fill any remaining null values: mean_xs_area_ratio --> 1 median_stage_bankfull --> 0
+        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(),1,inplace=True)
+        stream_order_bathy_ratio['median_stage_bankfull'].mask(stream_order_bathy_ratio['median_stage_bankfull'].isnull(),0,inplace=True)
+
+        ## Combine SRC df and df of XS Area for each hydroid and matching stage and order from bins above
+        output_bathy = output_bathy.merge(stream_order_bathy_ratio,how='left',on='order_')
+        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio,how='left',on='order_')
+
+        ## Calculate stage vs median_stage_bankfull difference for bankfull lookup
+        modified_src_base['lookup_stage_diff'] = (modified_src_base[['median_stage_bankfull','Thalweg_burn_elev']].max(axis=1) - modified_src_base['Stage']).abs()
+
+        ## If median_stage_bankfull is null then set lookup_stage_diff to 999 at stage 0 (handles errors for channels outside CONUS)
+        modified_src_base['lookup_stage_diff'].mask((modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),999,inplace=True)
+
+        ## Groupby HydroID again and find min of lookup_stage_diff
+        xs_area_hydroid_lookup = modified_src_base[['HydroID','BANKFULL_XSEC_AREA (m2)','XS Area (m2)','Stage','Thalweg_burn_elev','median_stage_bankfull','lookup_stage_diff','mean_xs_area_ratio']]
+        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()].reset_index(drop=True)
+
+        ## Calculate bathy adjusted XS Area ('XS Area (m2)' mutliplied by mean_xs_area_ratio)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']) - xs_area_hydroid_lookup['XS Area (m2)']
+
+        ## Calculate the ratio btw the lookup SRC XS_Area and the Bankfull_XSEC_AREA --> use this as a flag for potentially bad XS data
+        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'])
+        ## Set bath_cal_xs_area to 0 if the bankfull_XS_ratio_flag is > threshold --> 5x (assuming too large of difference to be a reliable bankfull calculation)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag,xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],inplace=True)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(),0,inplace=True)
+
+        ## Merge bathy_calc_xs_area to the modified_src_base
+        modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area']],how='left',on='HydroID')
+
+        ## Calculate new bathy adjusted channel geometry variables
+        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'Discharge (m3s-1)_nobathy'})
+        modified_src_base['XS Area (m2)_bathy_adj'] = modified_src_base['XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
+        modified_src_base['Volume (m3)_bathy_adj'] = modified_src_base['XS Area (m2)_bathy_adj'] * modified_src_base['LENGTHKM'] * 1000
+        modified_src_base['WetArea (m2)_bathy_adj'] = modified_src_base['Volume (m3)_bathy_adj']/modified_src_base['LENGTHKM']/1000
+        modified_src_base['HydraulicRadius (m)_bathy_adj'] = modified_src_base['WetArea (m2)_bathy_adj']/modified_src_base['WettedPerimeter (m)']
+        modified_src_base['HydraulicRadius (m)_bathy_adj'].fillna(0, inplace=True)
+        ## mask out negative top width differences (avoid thalweg burn notch)
+        modified_src_base['HydraulicRadius (m)_bathy_adj'].mask((modified_src_base['HydraulicRadius (m)_bathy_adj']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
+
+        ## Calculate Q using Manning's equation
+        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)_bathy_adj']* \
+        pow(modified_src_base['HydraulicRadius (m)_bathy_adj'],2.0/3)* \
+        pow(modified_src_base['SLOPE'],0.5)/modified_src_base['ManningN']
+        ## mask discharge values for stage = 0 rows in SRC (replace with 0) --> do we need SRC to start at 0??
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0,0,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'],0,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'],-999,inplace=True)
+
+        ## Organize bathy calc output variables for csv
+        output_bathy = output_bathy[['HydroID','order_','Stage','SurfaceArea (m2)','TopWidth (m)','BANKFULL_WIDTH (m)','Top Width Diff (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','XS Area Diff (m2)','XS Bankfull Area Ratio','count','median_stage_bankfull','mean_xs_area_ratio']]
+
+        ## Export bathy/bankful calculation tables for easy viewing
+        output_bathy.to_csv(output_bathy_fileName,index=False)
+        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName,index=True)
+        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName,index=True)
+        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName,index=True)
+
+        print('Completed Bathy Calculations...')
+        return(modified_src_base)
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index cd471cd88..9313b6ed8 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -8,7 +8,7 @@
 def __read_included_files(parent_dir_path):
 
     filename_patterns = glob(os.path.join(parent_dir_path,'included_huc*.lst'))
-
+    
     accepted_hucs_set = set()
     for filename in filename_patterns:
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index e5053627a..7a4c94492 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -127,7 +127,15 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
 
     # Subset nwm streams
     print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd_buffer)
+    if extent == 'GMS':
+        nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd)
+        nwm_streams = gpd.clip(nwm_streams,wbd)
+    else:
+        nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd_buffer)
+
+     # NWM can have duplicate records, but appear to always be identical duplicates
+    nwm_streams.drop_duplicates(subset="ID", keep="first", inplace=True)
+
     if extent == 'MS':
         nwm_streams = nwm_streams.loc[nwm_streams.mainstem==1]
     if len(nwm_streams) > 0:
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index 17aa5c611..2f4192662 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -40,18 +40,18 @@ def filter_catchments_and_add_attributes(input_catchments_filename, input_flows_
         # add geometry column
         output_catchments['areasqkm'] = output_catchments.geometry.area/(1000**2)
 
-        output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
-        output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
+        try:
+            output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
+            output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
+        except ValueError:
 
+            class NoFlowLinesInHUC(Exception):
+                pass
+
+            raise NoFlowLinesInHUC("There are no flowlines in the HUC. These should be better filtered in derive_level_paths.py")
 
-if __name__ == '__main__':
 
-    input_catchments_filename = sys.argv[1]
-    input_flows_filename = sys.argv[2]
-    output_catchments_filename = sys.argv[3]
-    output_flows_filename = sys.argv[4]
-    wbd_filename = sys.argv[5]
-    huc_code = str(sys.argv[6])
+if __name__ == '__main__':
 
     # Parse arguments.
     parser = argparse.ArgumentParser(description='filter_catchments_and_add_attributes.py')
diff --git a/src/gms/__init__.py b/src/gms/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/gms/aggregate_branch_lists.py b/src/gms/aggregate_branch_lists.py
new file mode 100755
index 000000000..eb03a5172
--- /dev/null
+++ b/src/gms/aggregate_branch_lists.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import argparse 
+import pandas as pd
+from os import environ
+from os.path import join
+from glob import glob
+
+
+def aggregate_inputs_for_gms(hucList):
+
+    try:
+        hucList = pd.read_csv(hucList[0],header=None,dtype=str).loc[:,0].tolist()
+    except FileNotFoundError:
+        pass
+
+    hucList = set(hucList)
+
+    # get branch lists
+    outputRunDataDir = environ['outputRunDataDir']
+    branch_id_files = glob(join(outputRunDataDir,'*','branch_id.lst'))
+
+    all_huc_numbers,all_bids = [],[]
+    for bid_file in branch_id_files:
+        huc_number = bid_file.split('/')[-2]
+        
+        if huc_number in hucList:
+            bids = pd.read_csv(bid_file,header=None).loc[:,0].tolist()
+            huc_number_list = [huc_number] * len(bids)
+
+            all_bids += bids
+            all_huc_numbers += huc_number_list
+        
+    output = pd.DataFrame({ 
+                            'huc': all_huc_numbers,
+                            'branch' : all_bids
+                          })
+    
+    output.to_csv(join(outputRunDataDir,'gms_inputs.csv'),index=False,header=False)
+
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Aggregate GMS Inputs')
+    parser.add_argument('-l','--hucList', help='huc list', required=True,nargs='+')
+    #parser.add_argument('-a','--input-flows-fileName', help='DEM derived streams', required=True)
+
+    args = vars(parser.parse_args())
+
+    aggregate_inputs_for_gms(**args)
diff --git a/src/gms/buffer_stream_branches.py b/src/gms/buffer_stream_branches.py
new file mode 100755
index 000000000..af7ef661d
--- /dev/null
+++ b/src/gms/buffer_stream_branches.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+
+from stream_branches import StreamNetwork
+from stream_branches import StreamBranchPolygons
+import argparse
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Generates branch polygons')
+    parser.add_argument('-s','--streams', help='Streams file to branch', required=True)
+    parser.add_argument('-i','--branch-id', help='Attribute with branch ids', required=True)
+    parser.add_argument('-d','--buffer-distance', help='Distance to buffer branches to create branch polygons', required=True,type=int)
+    parser.add_argument('-b','--branches', help='Branch polygons out file name', required=False,default=None)
+    parser.add_argument('-v','--verbose', help='Verbose printing', required=False,default=None,action='store_true')
+
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    streams_file , branch_id_attribute, buffer_distance, stream_polygons_file, verbose = args["streams"], args["branch_id"] , args["buffer_distance"], args["branches"] , args["verbose"]
+    
+    # load file
+    stream_network = StreamNetwork.from_file( filename=streams_file,branch_id_attribute=branch_id_attribute,
+                                              values_excluded=None,attribute_excluded=None, verbose = verbose)
+    
+    # make stream polygons 
+    stream_polys = StreamBranchPolygons.buffer_stream_branches( stream_network,
+                                                                buffer_distance=buffer_distance,
+                                                                verbose=verbose                  )
+    
+    stream_polys.write(stream_polygons_file,verbose=verbose)
+
diff --git a/src/gms/clip_rasters_to_branches.py b/src/gms/clip_rasters_to_branches.py
new file mode 100755
index 000000000..7057c830f
--- /dev/null
+++ b/src/gms/clip_rasters_to_branches.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+
+from stream_branches import StreamNetwork
+from stream_branches import StreamBranchPolygons
+import argparse
+from tqdm import tqdm
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Clips rasters to branch polygons')
+    parser.add_argument('-b','--branches', help='Branch polygons file name', required=True,default=None)
+    parser.add_argument('-d','--branch-id', help='Branch ID attribute', required=True,default=None)
+    parser.add_argument('-i','--branch-id-attribute', help='Branch ID attribute', required=True,default=None)
+    parser.add_argument('-r','--rasters', help='Raster file name to clip', required=True,default=None,nargs="+")
+    parser.add_argument('-c','--clipped-rasters', help='Branch polygons out file name', required=False,default=None,nargs="+")
+    parser.add_argument('-v','--verbose', help='Verbose printing', required=False,default=None,action='store_true')
+
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    stream_polygons_file, branch_id, branch_id_attribute, rasters, clipped_rasters, verbose = args["branches"], args["branch_id"],args["branch_id_attribute"],args["rasters"] , args["clipped_rasters"], args["verbose"]
+    
+    # load file
+    stream_polys = StreamBranchPolygons.from_file( filename=stream_polygons_file, 
+                                                   branch_id_attribute=branch_id_attribute,
+                                                   values_excluded=None,attribute_excluded=None, verbose = verbose)
+    
+    for raster, clipped_raster in tqdm(zip(rasters,clipped_rasters),disable=(not verbose),total=len(rasters)):
+        if verbose:
+            print("Clipping \'{}\' to branch polygons ...".format(raster.split('/')[-1].split('.')[0]))
+   
+        stream_polys.clip(raster,clipped_raster,branch_id,branch_id_attribute)
+    
+
diff --git a/src/gms/crosswalk_nwm_demDerived.py b/src/gms/crosswalk_nwm_demDerived.py
new file mode 100755
index 000000000..131b91aff
--- /dev/null
+++ b/src/gms/crosswalk_nwm_demDerived.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+import argparse
+from utils.shared_functions import getDriver
+from utils.shared_variables import FIM_ID
+from gms import stream_branches as sb
+from shapely.geometry import MultiLineString
+
+
+def Crosswalk_nwm_demDerived(nwm_streams, demDerived, wbd=None, node_prefix=None, sampling_size=None, 
+                             crosswalk_outfile=None, 
+                             demDerived_outfile=None, nwm_outfile=None, verbose=False):
+    
+    # load nwm streams
+    if isinstance(nwm_streams,sb.StreamNetwork):
+        pass
+    elif isinstance(nwm_streams,str):
+        nwm_streams = sb.StreamNetwork.from_file(nwm_streams)
+    else:
+        raise TypeError("For nwm_streams pass file path string or GeoDataFrame object")
+
+    # load demDerived
+    if isinstance(demDerived,sb.StreamNetwork):
+        pass
+    elif isinstance(demDerived,str):
+        demDerived = sb.StreamNetwork.from_file(demDerived)
+    else:
+        raise TypeError("demDerived pass file path string or GeoDataFrame object")
+
+    # clip nwm_streams
+    if wbd is not None:
+        nwm_streams = nwm_streams.clip(wbd,keep_geom_type=True,verbose=verbose)
+
+    # build traversal to nwm 
+    nwm_streams = Add_traversal_to_NWM(nwm_streams,node_prefix=node_prefix,
+                                       outfile=nwm_outfile,
+                                       verbose=verbose)
+
+    # create points for nwm and demDerived networks
+    nwm_points = nwm_streams.explode_to_points(reach_id_attribute='ID', sampling_size=sampling_size, 
+                                               verbose=verbose)
+    demDerived_points = demDerived.explode_to_points(reach_id_attribute='HydroID', sampling_size=sampling_size, 
+                                                     verbose=verbose)
+    
+    # conflate points
+    crosswalk_table = sb.StreamNetwork.conflate_points(demDerived_points, nwm_points,
+                                                         source_reach_id_attribute='HydroID',
+                                                         target_reach_id_attribute='ID',
+                                                         verbose=verbose)
+    
+    # merge crosswalk table
+    crosswalk_table.rename(columns={'ID':'feature_id'},inplace=True)
+    demDerived.drop(columns='feature_id',inplace=True,errors='raise')
+    demDerived['HydroID'] = demDerived['HydroID'].astype(int)
+    demDerived = demDerived.merge(crosswalk_table, how='left', left_on='HydroID', right_index=True)
+
+    """
+    demDerived = demDerived.conflate_branches(target_stream_network=nwm_streams,branch_id_attribute_left='levpa_id',
+                                                branch_id_attribute_right='levpa_id',
+                                                left_order_attribute='order_', right_order_attribute='order_',
+                                                crosswalk_attribute='nwm_levpa_id',
+                                                verbose=verbose)
+    """
+
+    if demDerived_outfile is not None:
+        demDerived.write(demDerived_outfile,index=False,verbose=verbose)
+
+    if crosswalk_outfile is not None:
+        crosswalk_table.to_csv(crosswalk_outfile,index=True)
+
+    #print(demDerived, crosswalk_table)
+    return(demDerived, crosswalk_table)
+
+
+def Add_traversal_to_NWM(nwm_streams,node_prefix=None,outfile=None,verbose=False):
+    
+    if isinstance(nwm_streams,sb.StreamNetwork):
+        pass
+    elif isinstance(nwm_streams,str):
+        nwm_streams = sb.StreamNetwork.from_file(nwm_streams)
+    else:
+        raise TypeError("nwm_streams_file pass file path string or GeoDataFrame object")
+
+    # remove multilinestrings if any
+    anyMultiLineStrings = np.any(np.array([isinstance(g, MultiLineString) for g in nwm_streams.geometry]))
+    if anyMultiLineStrings:
+        nwm_streams = nwm_streams.dissolve_by_branch(branch_id_attribute='ID', attribute_excluded=None,
+                                                     values_excluded=None, verbose=verbose)
+
+
+    # create stream node ids
+    nwm_streams = nwm_streams.derive_nodes(toNode_attribute='To_Node',fromNode_attribute='From_Node',
+                                           reach_id_attribute='ID',
+                                           outlet_linestring_index=-1,node_prefix=node_prefix,
+                                           max_node_digits=8,verbose=verbose)
+    # inlets and outlets
+    nwm_streams = nwm_streams.derive_outlets(toNode_attribute='To_Node',fromNode_attribute='From_Node',
+                                             outlets_attribute='outlet_id',verbose=verbose)
+    nwm_streams = nwm_streams.derive_inlets(toNode_attribute='To_Node',fromNode_attribute='From_Node',
+                                            inlets_attribute='inlet_id',verbose=verbose)
+
+    # upstream and downstream dictionaries
+    upstreams,downstreams = nwm_streams.make_up_and_downstream_dictionaries(reach_id_attribute='ID',
+                                                                            toNode_attribute='To_Node',
+                                                                            fromNode_attribute='From_Node',
+                                                                            verbose=verbose)
+
+    # derive arbolate sum
+    nwm_streams = nwm_streams.get_arbolate_sum(arbolate_sum_attribute='arbolate_sum',inlets_attribute='inlet_id',
+                         reach_id_attribute='ID',length_conversion_factor_to_km = 0.001,
+                         upstreams=upstreams, downstreams=downstreams,
+                         toNode_attribute='To_Node',
+                         fromNode_attribute='From_Node',
+                         verbose=verbose
+                        )
+
+    # derive levelpaths
+    nwm_streams = nwm_streams.derive_stream_branches(toNode_attribute='To_Node',
+                               fromNode_attribute='From_Node',
+                               upstreams=upstreams,
+                               outlet_attribute='outlet_id',
+                               branch_id_attribute='levpa_id',
+                               reach_id_attribute='ID',
+                               comparison_attributes='order_',
+                               comparison_function=max,
+                               max_branch_id_digits=6,
+                               verbose=verbose)
+
+    #nwm_streams = nwm_streams.dissolve_by_branch(branch_id_attribute='levpa_id', attribute_excluded=None,
+    #                                             values_excluded=None, verbose=verbose)
+    
+    nwm_streams.reset_index(drop=True,inplace=True)
+
+    if outfile is not None:
+        nwm_streams.write(outfile,index=False,verbose=verbose)
+
+    return(nwm_streams)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Crosswalking')
+    parser.add_argument('-n','--nwm-streams', help='NWM Streams', required=True)
+    parser.add_argument('-d','--demDerived', help='demDerived Streams', required=True)
+    parser.add_argument('-w','--wbd', help='WBD File', required=False,default=None)
+    parser.add_argument('-p','--node-prefix', help='Node Prefix', required=False,default=None)
+    parser.add_argument('-a','--sampling-size', help='Sample size for Points', required=False,default=None,type=int)
+    parser.add_argument('-c','--crosswalk-outfile', help='Crosswalk Out File', required=False,default=None)
+    parser.add_argument('-e','--demDerived-outfile', help='demDerived Out File', required=False,default=None)
+    parser.add_argument('-m','--nwm-outfile', help='NWM Streams Out File', required=False,default=None)
+    parser.add_argument('-v','--verbose', help='Verbose', required=False,default=False,action='store_true')
+
+    kwargs = vars(parser.parse_args())
+
+    Crosswalk_nwm_demDerived(**kwargs)
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
new file mode 100755
index 000000000..5a90d9ace
--- /dev/null
+++ b/src/gms/derive_level_paths.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+from stream_branches import StreamNetwork
+import argparse
+from utils.shared_functions import get_fossid_from_huc8
+import geopandas as gpd
+
+
+def Derive_level_paths(in_stream_network, out_stream_network,branch_id_attribute,
+                       out_stream_network_dissolved=None,huc_id=None,
+                       headwaters_outfile=None,catchments=None,
+                       catchments_outfile=None,
+                       branch_inlets_outfile=None,
+                       toNode_attribute='To_Node',fromNode_attribute='From_Node',
+                       reach_id_attribute='HydroID',verbose=False
+                       ):
+
+    # getting foss_id of huc8
+    #foss_id = get_fossid_from_huc8(huc8_id=huc_id,foss_id_attribute='fossid',
+                                   #hucs_layerName='WBDHU8')
+    
+    if verbose:
+        print("Deriving level paths ...")
+
+    # load file
+    if verbose:
+        print("Loading stream network ...")
+    stream_network = StreamNetwork.from_file(in_stream_network)
+
+    inlets_attribute = 'inlet_id'
+    outlets_attribute = 'outlet_id'
+    outlet_linestring_index = -1
+
+    # converts multi-linestrings to linestrings
+    stream_network = stream_network.multilinestrings_to_linestrings()
+
+    # derive nodes
+    stream_network = stream_network.derive_nodes(toNode_attribute=toNode_attribute,
+                                                  fromNode_attribute=fromNode_attribute,
+                                                  reach_id_attribute=reach_id_attribute,
+                                                  outlet_linestring_index=outlet_linestring_index,
+                                                  node_prefix=None,
+                                                  verbose=verbose)
+    
+    # derive outlets and inlets
+    stream_network = stream_network.derive_outlets(toNode_attribute,
+                                                   fromNode_attribute,
+                                                   outlets_attribute=outlets_attribute,
+                                                   verbose=verbose
+                                                  )
+    stream_network = stream_network.derive_inlets(toNode_attribute,
+                                                  fromNode_attribute,
+                                                  inlets_attribute=inlets_attribute,
+                                                  verbose=verbose
+                                                 )
+
+    # derive up and downstream networks
+    upstreams, downstreams = stream_network.make_up_and_downstream_dictionaries(
+                                                                         reach_id_attribute=reach_id_attribute,
+                                                                         toNode_attribute=toNode_attribute,
+                                                                         fromNode_attribute=fromNode_attribute,
+                                                                         verbose=True
+                                                                         )
+
+    # derive arbolate sum
+    stream_network = stream_network.get_arbolate_sum(arbolate_sum_attribute='arbolate_sum',
+                                                     inlets_attribute=inlets_attribute,
+                                                     reach_id_attribute=reach_id_attribute,
+                                                     upstreams=upstreams,
+                                                     downstreams=downstreams,
+                                                     length_conversion_factor_to_km = 0.001,
+                                                     verbose=verbose
+                                                    )
+
+    # derive stream branches
+    stream_network = stream_network.derive_stream_branches( toNode_attribute=toNode_attribute,
+                                                            fromNode_attribute=fromNode_attribute,
+                                                            upstreams=upstreams,
+                                                            branch_id_attribute=branch_id_attribute,
+                                                            reach_id_attribute=reach_id_attribute,
+                                                            comparison_attributes='arbolate_sum',
+                                                            comparison_function=max,
+                                                            verbose=verbose
+                                                           )
+    
+    # filter out streams with out catchments
+    if (catchments is not None) & (catchments_outfile is not None):
+
+        catchments = gpd.read_file(catchments)
+
+        stream_network = stream_network.remove_branches_without_catchments( 
+                                                                catchments,
+                                                                reach_id_attribute=reach_id_attribute,
+                                                                branch_id_attribute=branch_id_attribute,
+                                                                reach_id_attribute_in_catchments=reach_id_attribute,
+                                                                verbose=verbose
+                                                                              )
+
+        # subset which columns to merge
+        stream_network_to_merge = stream_network.filter(
+                                                        items = [reach_id_attribute,inlets_attribute,
+                                                                 outlets_attribute,branch_id_attribute]
+                                                       )
+
+        catchments = catchments.merge(stream_network_to_merge,how='inner',
+                                      left_on=reach_id_attribute,
+                                      right_on=reach_id_attribute
+                                     )
+
+        catchments.reset_index(drop=True,inplace=True)
+
+        catchments.to_file(catchments_outfile,index=False,driver='GPKG')
+
+    # derive headwaters
+    if headwaters_outfile is not None:
+        headwaters = stream_network.derive_headwater_points_with_inlets(
+                                                        fromNode_attribute=fromNode_attribute,
+                                                        inlets_attribute=inlets_attribute,
+                                                        outlet_linestring_index=outlet_linestring_index
+                                                       )
+        # headwaters write
+        headwaters.to_file(headwaters_outfile,index=False,driver='GPKG')
+
+    
+    if out_stream_network is not None:
+        if verbose:
+            print("Writing stream branches ...")
+        stream_network.write(out_stream_network,index=True)
+    
+    if out_stream_network_dissolved is not None:
+    
+        # dissolve by levelpath
+        stream_network = stream_network.dissolve_by_branch(branch_id_attribute=branch_id_attribute,
+                                                           attribute_excluded=None, #'order_',
+                                                           values_excluded=None, #[1,2],
+                                                           out_vector_files=out_stream_network_dissolved,
+                                                           verbose=verbose)
+
+        branch_inlets = stream_network.derive_inlet_points_by_feature( feature_attribute=branch_id_attribute,
+                                                                       outlet_linestring_index=outlet_linestring_index
+                                                                     )
+   
+        if branch_inlets_outfile is not None:
+            branch_inlets.to_file(branch_inlets_outfile,index=False,driver='GPKG')
+
+    return(stream_network)
+    
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Create stream network level paths')
+    parser.add_argument('-i','--in-stream-network', help='Input stream network', required=True)
+    parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
+    parser.add_argument('-u','--huc-id', help='Current HUC ID', required=False,default=None)
+    parser.add_argument('-r','--reach-id-attribute', help='Reach ID attribute to use in source file', required=False,default='HydroID')
+    parser.add_argument('-c','--catchments', help='NWM catchments to append level path data to', required=False, default=None)
+    parser.add_argument('-t','--catchments-outfile', help='NWM catchments outfile with appended level path data', required=False, default=None)
+    parser.add_argument('-n','--branch_inlets_outfile', help='Output level paths inlets', required=False,default=None)
+    parser.add_argument('-o','--out-stream-network', help='Output stream network', required=False,default=None)
+    parser.add_argument('-e','--headwaters-outfile', help='Output stream network headwater points', required=False,default=None)
+    parser.add_argument('-d','--out-stream-network-dissolved', help='Dissolved output stream network', required=False,default=None)
+    parser.add_argument('-v','--verbose', help='Verbose output', required=False,default=False,action='store_true')
+    
+    args = vars(parser.parse_args())
+
+    Derive_level_paths(**args)
+
diff --git a/src/gms/edit_points.py b/src/gms/edit_points.py
new file mode 100755
index 000000000..400386cc9
--- /dev/null
+++ b/src/gms/edit_points.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import argparse
+
+
+def Edit_points(stream_reaches,branch_id_attribute,reach_points,
+                out_reach_points=None,out_pixel_points=None,verbose=False):
+
+    if verbose:
+        print("Editing points files ...")
+
+    if verbose:
+        print("Loading files ...")
+    stream_reaches = gpd.read_file(stream_reaches)
+    stream_reaches = stream_reaches.astype({'HydroID':int})
+
+    reach_points=gpd.read_file(reach_points)
+    reach_points['HydroID'] = reach_points['id'].copy()
+    
+    # merge
+    if verbose:
+        print("Merging points ...")
+    reach_points = reach_points.merge(stream_reaches.loc[:,["HydroID",branch_id_attribute]],how='inner',on='HydroID')
+    
+    # join on HydroID to add branch_id
+    if out_reach_points is not None:
+        reach_points.to_file(out_reach_points,driver='GPKG',index=False)
+
+    # make pixel points
+    if verbose:
+        print("Generating pixel points ...")
+
+    pixel_points = reach_points.copy()
+    pixel_points['id'] = list(range(1,len(pixel_points)+1))
+
+    if out_pixel_points is not None:
+        pixel_points.to_file(out_pixel_points,driver='GPKG',index=False)
+
+    return(reach_points,pixel_points)
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Edit points to include branch ids')
+    parser.add_argument('-i','--stream-reaches', help='Input stream network', required=True)
+    parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
+    parser.add_argument('-r','--reach-points', help='Name of the branch attribute desired', required=True)
+    parser.add_argument('-o','--out-reach-points', help='Output stream network', required=False,default=None)
+    parser.add_argument('-p','--out-pixel-points', help='Dissolved output stream network', required=False,default=None)
+    parser.add_argument('-v','--verbose', help='Verbose output', required=False,default=False,action='store_true')
+    
+    args = vars(parser.parse_args())
+
+    Edit_points(**args)
diff --git a/src/gms/filter_gms_inputs_by_huc.py b/src/gms/filter_gms_inputs_by_huc.py
new file mode 100755
index 000000000..66ed631eb
--- /dev/null
+++ b/src/gms/filter_gms_inputs_by_huc.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import argparse
+
+def filter_gms_inputs_by_huc(gms_inputs,hucs,gms_outputs):
+
+    try:
+        with open(hucs[0]) as hf:
+            hucsList = set([str(h).rstrip() for h in hf])
+    except FileNotFoundError:
+        hucsList = set(hucs)
+    
+    gms_inputs = pd.read_csv(gms_inputs,header=None,dtype=str)
+    gms_inputs_mask = gms_inputs.loc[:,0].isin(hucsList)
+    gms_inputs = gms_inputs.loc[gms_inputs_mask,:]
+
+    assert len(gms_inputs) > 0, "Filtered GMS list is empty"
+
+    gms_inputs.to_csv(gms_outputs,index=False,header=False)
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Adjusts the elevation of the thalweg to the lateral zonal minimum.')
+    parser.add_argument('-g','--gms-inputs',help='Raster of elevation.',required=True)
+    parser.add_argument('-u','--hucs',help='Raster of elevation.',required=True,nargs='+')
+    parser.add_argument('-o','--gms-outputs',help='Raster of elevation.',required=True)
+
+    args = vars(parser.parse_args())
+
+    filter_gms_inputs_by_huc(**args)
diff --git a/src/gms/finalize_srcs.py b/src/gms/finalize_srcs.py
new file mode 100755
index 000000000..46b61ca8b
--- /dev/null
+++ b/src/gms/finalize_srcs.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import pandas as pd
+from numpy import unique
+import json
+import argparse
+from utils.shared_functions import getDriver
+
+def finalize_srcs(srcbase,srcfull,hydrotable,output_srcfull=None,output_hydrotable=None):
+
+
+    # calculate src_full
+    srcbase = pd.read_csv(srcbase, dtype= {'CatchId': int})
+    srcbase.rename(columns={'CatchId':'HydroID'},inplace=True)
+    srcbase = srcbase.rename(columns=lambda x: x.strip(" "))
+    
+    # read and merge in attributes from base hydrofabric src full
+    srcfull = pd.read_csv(srcfull, dtype={'CatchId': int})
+    srcfull.rename(columns={'CatchId':'HydroID'},inplace=True)
+    srcfull = srcfull.loc[:,["ManningN","HydroID","feature_id"]].drop_duplicates(subset='HydroID')
+    
+    srcbase = srcbase.merge(srcfull,how='inner',left_on='HydroID',right_on='HydroID')
+
+    srcbase = srcbase.apply(pd.to_numeric,**{'errors' : 'coerce'})
+    srcbase['TopWidth (m)'] = srcbase['SurfaceArea (m2)']/srcbase['LENGTHKM']/1000
+    srcbase['WettedPerimeter (m)'] = srcbase['BedArea (m2)']/srcbase['LENGTHKM']/1000
+    srcbase['WetArea (m2)'] = srcbase['Volume (m3)']/srcbase['LENGTHKM']/1000
+    srcbase['HydraulicRadius (m)'] = srcbase['WetArea (m2)']/srcbase['WettedPerimeter (m)']
+    srcbase['HydraulicRadius (m)'].fillna(0, inplace=True)
+    srcbase['Discharge (m3s-1)'] = srcbase['WetArea (m2)']* \
+                                          pow(srcbase['HydraulicRadius (m)'],2.0/3)* \
+                                          pow(srcbase['SLOPE'],0.5)/srcbase['ManningN']
+
+    # set nans to 0
+    srcbase.loc[srcbase['Stage']==0,['Discharge (m3s-1)']] = 0
+
+    if output_srcfull is not None:
+        srcbase.to_csv(output_srcfull,index=False)
+
+    hydrotable = pd.read_csv(hydrotable)
+    hydrotable.drop(columns=['stage','discharge_cms'],inplace=True)
+    
+    hydrotable.drop_duplicates(subset='HydroID',inplace=True)
+    #srcfull = srcfull.loc[:,["ManningN","HydroID","feature_id"]].drop_duplicates(subset='HydroID')
+    hydrotable = hydrotable.merge(srcbase.loc[:,['HydroID','Stage','Discharge (m3s-1)']],how='right',left_on='HydroID',right_on='HydroID')
+    hydrotable.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
+    #hydrotable.drop_duplicates(subset='stage',inplace=True)
+    
+    if output_hydrotable is not None:
+        hydrotable.to_csv(output_hydrotable,index=False)
+
+    return(srcbase,hydrotable)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='')
+    parser.add_argument('-b','--srcbase', help='Base synthetic rating curve table', required=True)
+    parser.add_argument('-f','--srcfull', help='Base synthetic rating curve table', required=True)
+    parser.add_argument('-w','--hydrotable',help='Input Hydro-Table',required=False)
+    parser.add_argument('-r','--output-srcfull', help='Output crosswalked synthetic rating curve table', required=False,default=None)
+    parser.add_argument('-t','--output-hydrotable',help='Hydrotable',required=False,default=None)
+
+    args = vars(parser.parse_args())
+
+    finalize_srcs(**args)
diff --git a/src/gms/generate_branch_list.py b/src/gms/generate_branch_list.py
new file mode 100755
index 000000000..7eb6d2d30
--- /dev/null
+++ b/src/gms/generate_branch_list.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+from stream_branches import StreamNetwork
+import argparse
+
+def Generate_branch_list(stream_network_dissolved,branch_id_attribute,output_branch_list):
+
+    # load stream network
+    stream_network_dissolved = StreamNetwork.from_file( stream_network_dissolved,
+                                                        branch_id_attribute=branch_id_attribute )
+
+    # reduce to branch id attribute and convert to pandas df
+    stream_network_dissolved = stream_network_dissolved.loc[:,branch_id_attribute]
+
+    # write
+    stream_network_dissolved.to_csv(output_branch_list,sep= " ",index=False,header=False)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Create branch list')
+    parser.add_argument('-d','--stream-network-dissolved', help='Dissolved stream network', required=True)
+    parser.add_argument('-b','--branch-id-attribute', help='Branch ID attribute to use in dissolved stream network', required=True)
+    parser.add_argument('-o','--output-branch-list', help='Output branch list', required=True)
+    
+    args = vars(parser.parse_args())
+
+    Generate_branch_list(**args)
diff --git a/src/gms/outputs_cleanup.py b/src/gms/outputs_cleanup.py
new file mode 100755
index 000000000..617a915dd
--- /dev/null
+++ b/src/gms/outputs_cleanup.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+
+def remove_deny_list_files(directory,deny_list,branches=False,verbose=False):
+
+    deny_list_file_object = open(deny_list)
+
+    file_removed_count = 0
+
+    for line in deny_list_file_object:
+        line = line.partition('#')[0]
+        filename = line.rstrip()
+
+        if len(filename) == 0:
+            continue
+        
+        filename = os.path.join(directory,filename)
+
+        # for use with branches. append branch id
+        if branches:
+            branch_id = directory.split('/')[-1]
+            filename = filename.format(branch_id)
+        
+        if os.path.exists(filename):
+            os.remove(filename)
+            file_removed_count +=1
+
+    if verbose:
+        print(f'Removed {file_removed_count} files')
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Clean up outputs given file with line delimineted files')
+    parser.add_argument('-d','--directory', help='Directory to find files', required=True)
+    parser.add_argument('-l','--deny-list', help='Path to deny list file. Must be line delimited', required=True)
+    parser.add_argument('-b','--branches', help='For use with branch files', required=False,default=False, action='store_true')
+    parser.add_argument('-v','--verbose', help='Verbose', required=False,default=False, action='store_true')
+
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    remove_deny_list_files(**args)
+
diff --git a/src/gms/query_vectors_by_branch_polygons.py b/src/gms/query_vectors_by_branch_polygons.py
new file mode 100755
index 000000000..27ca1a00e
--- /dev/null
+++ b/src/gms/query_vectors_by_branch_polygons.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+
+from stream_branches import StreamBranchPolygons
+import argparse
+from tqdm import tqdm
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Query vectors by unique attribute values')
+    parser.add_argument('-a','--vector-attributes', help='Vector with unique attributs', required=True,default=None)
+    parser.add_argument('-d','--branch-ids', help='Branch ID value', required=True,default=None, nargs='+')
+    parser.add_argument('-i','--attribute-id', help='Attribute Name', required=True,default=None)
+    parser.add_argument('-s','--subset-vectors', help='Vector file names to query by attribute', required=False,default=None,nargs="+")
+    parser.add_argument('-o','--out-files', help='Vector filenames to write to after query', required=False,default=None,nargs="+")
+    parser.add_argument('-v','--verbose', help='Verbose printing', required=False,default=None,action='store_true')
+
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    attributes_vector_file, branch_ids ,attribute, subset_vectors, out_files, verbose = args["vector_attributes"], args["branch_ids"],args["attribute_id"], args["subset_vectors"], args["out_files"], args["verbose"]
+    
+    # load file
+    #stream_polys = StreamBranchPolygons.from_file( filename=attributes_vector_file, 
+    #                                               branch_id_attribute=attribute,
+    #                                               values_excluded=None,attribute_excluded=None, verbose = verbose)
+    
+    for subset_vector,out_file in tqdm(zip(subset_vectors,out_files),disable=(not verbose),
+                                       total=len(subset_vectors),
+                                       desc="Query vectors"):
+
+        #if verbose:
+            #print("Query \'{}\' by attribute in \'{}\' ...".format(out_file.split('/')[-1].split('.')[0],
+            #                                                   attributes_vector_file.split('/')[-1].split('.')[0]))
+        StreamBranchPolygons.query_vectors_by_branch(subset_vector,
+                                                     branch_ids=branch_ids,
+                                                     branch_id_attribute=attribute,
+                                                     out_filename_template=out_file)
+
diff --git a/src/gms/rem.py b/src/gms/rem.py
new file mode 100755
index 000000000..aa7faeafa
--- /dev/null
+++ b/src/gms/rem.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+from numba import njit, typeof, typed, types
+import rasterio
+import numpy as np
+import argparse
+import os
+from osgeo import ogr, gdal
+
+
+def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster):
+    """
+        Calculates REM/HAND/Detrended DEM
+        
+        Parameters
+        ----------
+        dem_fileName : str
+            File name of pit filled DEM raster.
+        pixel_watersheds_fileName : str
+            File name of stream pixel watersheds raster.
+        rem_fileName : str
+            File name of output relative elevation raster.
+
+    """
+
+    
+    # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
+    # The following creates a dictionary of the catchment ids (key) and their elevation along the thalweg (value).
+    @njit
+    def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalweg_window):
+  
+        for i,cm in enumerate(flat_catchments):
+            if thalweg_window[i] == 1:  # Only allow reference elevation to be within thalweg.
+                # If the catchment really exists in the dictionary, compare elevation values.
+                if (cm in catchment_min_dict):
+                    if (flat_dem[i] < catchment_min_dict[cm]):
+                        # If the flat_dem's elevation value is less than the catchment_min_dict min, update the catchment_min_dict min.
+                        catchment_min_dict[cm] = flat_dem[i]                                                
+                else:
+                    catchment_min_dict[cm] = flat_dem[i]                
+        return(catchment_min_dict)
+
+    # Open the masked gw_catchments_pixels_masked and dem_thalwegCond_masked.
+    gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
+    dem_thalwegCond_masked_object = rasterio.open(dem_fileName)
+    thalweg_raster_object = rasterio.open(thalweg_raster)
+    
+    # Specify raster object metadata.
+    meta = dem_thalwegCond_masked_object.meta.copy()
+    meta['tiled'], meta['compress'] = True, 'lzw'
+    
+    # -- Create catchment_min_dict -- #
+    catchment_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums.
+    # Update catchment_min_dict with pixel sheds minimum.
+    for ji, window in dem_thalwegCond_masked_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
+        dem_window = dem_thalwegCond_masked_object.read(1,window=window).ravel()  # Define dem_window.
+        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window.
+        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window.
+        
+        # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
+        catchment_min_dict = make_catchment_min_dict(dem_window, catchment_min_dict, catchments_window, thalweg_window)
+
+    dem_thalwegCond_masked_object.close()
+    gw_catchments_pixels_masked_object.close()
+    thalweg_raster_object.close()
+    # ------------------------------------------------------------------------------------------------------------------------ #
+    
+    
+    # ------------------------------------------- Produce relative elevation model ------------------------------------------- #
+    @njit
+    def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
+
+        rem_window = np.zeros(len(flat_dem),dtype=np.float32)
+        for i,cm in enumerate(flat_catchments):
+            if cm in catchmentMinDict:
+                if catchmentMinDict[cm] == ndv:
+                    rem_window[i] = ndv
+                else:
+                    rem_window[i] = flat_dem[i] - catchmentMinDict[cm]
+
+        return(rem_window)
+
+    rem_rasterio_object = rasterio.open(rem_fileName,'w',**meta)  # Open rem_rasterio_object for writing to rem_fileName.
+    pixel_catchments_rasterio_object = rasterio.open(pixel_watersheds_fileName)  # Open pixel_catchments_rasterio_object
+    dem_rasterio_object = rasterio.open(dem_fileName)
+        
+    for ji, window in dem_rasterio_object.block_windows(1):
+        dem_window = dem_rasterio_object.read(1,window=window)
+        window_shape = dem_window.shape
+
+        dem_window = dem_window.ravel()
+        catchments_window = pixel_catchments_rasterio_object.read(1,window=window).ravel()
+        
+        rem_window = calculate_rem(dem_window, catchment_min_dict, catchments_window, meta['nodata'])
+        rem_window = rem_window.reshape(window_shape).astype(np.float32)
+        
+        rem_rasterio_object.write(rem_window, window=window, indexes=1)
+        
+    dem_rasterio_object.close()
+    pixel_catchments_rasterio_object.close()
+    rem_rasterio_object.close()
+    # ------------------------------------------------------------------------------------------------------------------------ #
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Relative elevation from pixel based watersheds')
+    parser.add_argument('-d','--dem', help='DEM to use within project path', required=True)
+    parser.add_argument('-w','--watersheds',help='Pixel based watersheds raster to use within project path',required=True)
+    parser.add_argument('-t','--thalweg-raster',help='A binary raster representing the thalweg. 1 for thalweg, 0 for non-thalweg.',required=True)
+    parser.add_argument('-o','--rem',help='Output REM raster',required=True)
+    
+    
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    # rename variable inputs
+    dem_fileName = args['dem']
+    pixel_watersheds_fileName = args['watersheds']
+    rem_fileName = args['rem']
+    thalweg_raster = args['thalweg_raster']
+
+    rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster)
diff --git a/src/gms/reset_mannings.py b/src/gms/reset_mannings.py
new file mode 100755
index 000000000..df911bbb9
--- /dev/null
+++ b/src/gms/reset_mannings.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import numpy as np
+import argparse
+import os
+from glob import iglob,glob
+from gms.stream_branches import StreamNetwork
+
+def Reset_mannings(hydrofabric_dir,mannings_value,overwrite_files=False):
+
+
+    src_table_filePaths, hydro_table_filePaths, stream_network_filePaths = make_file_paths_for_inputs(hydrofabric_dir)
+
+    single_stream_network = len(stream_network_filePaths) == 1
+
+    if single_stream_network:
+        stream_network = StreamNetwork.from_file(stream_network_filePaths[0])
+    
+    
+    for i,(srcFP,hydFP) in enumerate(zip(src_table_filePaths,hydro_table_filePaths)):
+
+        src_table = load_src_table(srcFP)
+        hydro_table = load_hydro_table(hydFP)
+        
+        src_table, hydro_table = reset_mannings_for_a_processing_unit(src_table,hydro_table,mannings_value)
+        
+        if not single_stream_network:
+            stream_network = StreamNetwork.from_file( stream_network_filePaths[i] )
+
+        small_segments = identify_small_reaches(stream_network,min_catchment_area=None,min_stream_length=None)
+        src_table, hydro_table = replace_discharges_of_small_segments(small_segments, src_table, hydro_table)
+
+        if overwrite_files:
+            src_table.to_csv(srcFP,index=False)
+            hydro_table.to_csv(hydFP,index=False)
+
+        #yield(src_table, hydro_table)
+
+
+def load_hydro_table(hydro_table_filePath):
+
+    hydro_table = pd.read_csv( hydro_table_filePath, 
+                               dtype= { 'HydroID' : str,
+                                        'feature_id' : str,
+                                        'stage' : float,
+                                        'discharge_cms': float,
+                                        'HUC' : str,
+                                        'LakeID' : str 
+                                       }
+                             )
+
+    return(hydro_table)
+
+
+def load_src_table(src_table_filePath):
+
+    src_table = pd.read_csv( src_table_filePath, 
+                             dtype= { 'HydroID' : str,
+                                      'feature_id' : str,
+                                      'stage' : float,
+                                      'discharge_cms': float,
+                                      'HUC' : str,
+                                      'LakeID' : str 
+                                     }
+                           )
+
+    return(src_table)
+
+
+def make_file_paths_for_inputs(hydrofabric_dir):
+
+    src_table_filePath_to_glob = os.path.join(hydrofabric_dir,'**','src_full_crosswalked*.csv')
+    hydro_table_filePath_to_glob = os.path.join(hydrofabric_dir,'**','hydroTable*.csv')
+    stream_network_filePath_to_glob = os.path.join(hydrofabric_dir,'**','demDerived_reaches_split_filtered_addedAttributes_crosswalked*.gpkg')
+
+    src_table_filePaths = iglob(src_table_filePath_to_glob,recursive=True)
+    hydro_table_filePaths = iglob(hydro_table_filePath_to_glob,recursive=True)
+    stream_network_filePaths = glob(stream_network_filePath_to_glob,recursive=True)
+
+
+    return(src_table_filePaths,hydro_table_filePaths,stream_network_filePaths)
+
+
+def reset_mannings_for_a_processing_unit(src_table,hydro_table,mannings_value):
+
+    src_table = override_mannings(src_table,mannings_value)
+
+    src_table = calculate_discharge(src_table)
+
+    hydro_table["discharge_cms"] = src_table["Discharge (m3s-1)"]
+
+    return(src_table,hydro_table)
+
+
+def override_mannings(table,mannings_value,mannings_attribute="ManningN"):
+
+    table[mannings_attribute] = mannings_value
+
+    return(table)
+
+
+def calculate_discharge(src_table):
+
+    src_table['Discharge (m3s-1)'] = src_table['WetArea (m2)']* \
+                                     pow(src_table['HydraulicRadius (m)'],2.0/3)* \
+                                     pow(src_table['SLOPE'],0.5)/src_table['ManningN']
+    
+    # set zero stage values to zero discharge
+    src_table.loc[src_table['Stage']==0,['Discharge (m3s-1)']] = 0
+
+    
+    return(src_table)
+
+
+def identify_small_reaches(stream_network,min_catchment_area=None,min_stream_length=None):
+
+    # Adjust short model reach rating curves
+    sml_segs = pd.DataFrame()
+
+    if min_catchment_area is None:
+        min_catchment_area = float(os.environ['min_catchment_area']) #0.25#
+
+    if min_stream_length is None:
+        min_stream_length = float(os.environ['min_stream_length']) #0.5#
+
+    # replace small segment geometry with neighboring stream
+    for stream_index in stream_network.index:
+
+        if stream_network["areasqkm"][stream_index] < min_catchment_area and stream_network["LengthKm"][stream_index] < min_stream_length and stream_network["LakeID"][stream_index] < 0:
+
+            short_id = stream_network['HydroID'][stream_index]
+            to_node = stream_network['To_Node'][stream_index]
+            from_node = stream_network['From_Node'][stream_index]
+
+            # multiple upstream segments
+            if len(stream_network.loc[stream_network['NextDownID'] == short_id]['HydroID']) > 1:
+                max_order = max(stream_network.loc[stream_network['NextDownID'] == short_id]['order_']) # drainage area would be better than stream order but we would need to calculate
+
+                if len(stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID']) == 1:
+                    update_id = stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID'].item()
+
+                else:
+                    update_id = stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+
+            # single upstream segments
+            elif len(stream_network.loc[stream_network['NextDownID'] == short_id]['HydroID']) == 1:
+                update_id = stream_network.loc[stream_network.To_Node==from_node]['HydroID'].item()
+
+            # no upstream segments; multiple downstream segments
+            elif len(stream_network.loc[stream_network.From_Node==to_node]['HydroID']) > 1:
+                max_order = max(stream_network.loc[stream_network.From_Node==to_node]['HydroID']['order_']) # drainage area would be better than stream order but we would need to calculate
+
+                if len(stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID']) == 1:
+                    update_id = stream_network.loc[(stream_network.From_Node==to_node) & (stream_network['order_'] == max_order)]['HydroID'].item()
+
+                else:
+                    update_id = stream_network.loc[(stream_network.From_Node==to_node) & (stream_network['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+
+            # no upstream segments; single downstream segment
+            elif len(stream_network.loc[stream_network.From_Node==to_node]['HydroID']) == 1:
+                    update_id = stream_network.loc[stream_network.From_Node==to_node]['HydroID'].item()
+
+            else:
+                update_id = stream_network.loc[stream_network.HydroID==short_id]['HydroID'].item()
+
+            str_order = stream_network.loc[stream_network.HydroID==short_id]['order_'].item()
+            sml_segs = sml_segs.append({'short_id':short_id, 'update_id':update_id, 'str_order':str_order}, ignore_index=True)
+
+    #print("Number of short reaches [{} < {} and {} < {}] = {}".format("areasqkm", min_catchment_area, "LengthKm", min_stream_length, len(sml_segs)))
+    
+    return(sml_segs)
+
+
+def replace_discharges_of_small_segments(sml_segs, src_table, hydro_table):
+    
+    # update rating curves
+    if len(sml_segs) == 0:
+        return(src_table,hydro_table)
+
+    #sml_segs.to_csv(small_segments_filename,index=False)
+    #print("Update rating curves for short reaches.")
+
+    for index, segment in sml_segs.iterrows():
+
+        short_id = segment[0]
+        update_id= segment[1]
+        new_values = src_table.loc[src_table['HydroID'] == update_id][['Stage', 'Discharge (m3s-1)']]
+
+        for src_index, src_stage in new_values.iterrows():
+            src_table.loc[(src_table['HydroID']== short_id) & (src_table['Stage']== src_stage[0]),['Discharge (m3s-1)']] = src_stage[1]
+
+    hydro_table["discharge_cms"] = src_table["Discharge (m3s-1)"]
+
+    return(src_table,hydro_table)
+
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Overwrites mannings n values and recomputes discharge values for SRCs and Hydro-Tables')
+    parser.add_argument('-y','--hydrofabric-dir', help='Hydrofabric directory', required=True)
+    parser.add_argument('-n','--mannings-value', help='Mannings N value to use', required=True, type=float)
+    parser.add_argument('-o','--overwrite-files', help='Overwrites original files if used', required=False, default=False,action='store_true')
+    
+    
+    args = vars(parser.parse_args())
+
+    Reset_mannings(**args)
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
new file mode 100755
index 000000000..cb6a599a2
--- /dev/null
+++ b/src/gms/run_by_branch.sh
@@ -0,0 +1,294 @@
+#!/bin/bash -e
+
+## INITIALIZE TOTAL TIME TIMER ##
+T_total_start
+
+## SOURCE BASH FUNCTIONS
+source $srcDir/bash_functions.env
+
+## SET VARIABLES AND FILE INPUTS ##
+hucNumber="$1"
+current_branch_id="$2"
+hucUnitLength=${#hucNumber}
+huc4Identifier=${hucNumber:0:4}
+huc2Identifier=${hucNumber:0:2}
+
+outputHucDataDir=$outputRunDataDir/$hucNumber
+outputBranchDataDir=$outputHucDataDir/branches
+outputCurrentBranchDataDir=$outputBranchDataDir/$current_branch_id
+
+# set input files
+input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m.tif
+input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
+input_bathy_bankfull=$inputDataDir/$bankfull_input_table
+input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
+
+## OVERWRITE
+if [ -d "$outputCurrentBranchDataDir" ];then
+    if [ $overwrite -eq 1 ]; then
+        rm -rf $outputCurrentBranchDataDir
+    else
+        echo "GMS branch data directories for $hucNumber - $current_branch_id already exist. Use -o/--overwrite to continue"
+        exit 1
+    fi
+fi
+
+## MAKE OUTPUT BRANCH DIRECTORY
+mkdir -p $outputCurrentBranchDataDir
+
+## START MESSAGE ##
+echo -e $startDiv$startDiv"Processing branch_id: $current_branch_id in HUC: $hucNumber ..."$stopDiv$stopDiv
+
+## CLIP RASTERS
+echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/gms/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $input_DEM -c $outputCurrentBranchDataDir/dem_meters.tif -v 
+Tcount
+
+## GET RASTER METADATA
+echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
+
+## SUBSET VECTORS
+echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+echo -e "Querying NWM streams ..."
+ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg
+echo -e "Querying NWM catchments ..."
+ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
+echo -e "Querying NWM Dissolved Levelpaths headwaters ..."
+ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+#echo -e "Querying NWM headwaters ..."
+#ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_headwaters.gpkg
+Tcount
+
+## RASTERIZE NLD MULTILINES ##
+echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+# REMAINS UNTESTED FOR AREAS WITH LEVEES
+[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
+gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputCurrentBranchDataDir/nld_rasterized_elev_$current_branch_id.tif
+Tcount
+
+## RASTERIZE REACH BOOLEAN (1 & 0) ##
+echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $outputCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
+Tcount
+
+## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
+echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif
+Tcount
+
+## BURN LEVEES INTO DEM ##
+echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwrite dem_meters.tif output) $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+# REMAINS UNTESTED FOR AREAS WITH LEVEES
+[ -f $outputCurrentBranchDataDir/nld_rasterized_elev_$current_branch_id.tif ] && \
+python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputCurrentBranchDataDir/nld_rasterized_elev_$current_branch_id.tif -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif
+Tcount
+
+## DEM Reconditioning ##
+# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
+# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
+echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif -d $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -w $outputCurrentBranchDataDir -g $outputCurrentBranchDataDir/temp_work -o $outputCurrentBranchDataDir/dem_burned_$current_branch_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+Tcount
+
+## PIT REMOVE BURNED DEM ##
+echo -e $startDiv"Pit remove Burned DEM $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+rd_depression_filling $outputCurrentBranchDataDir/dem_burned_$current_branch_id.tif $outputCurrentBranchDataDir/dem_burned_filled_$current_branch_id.tif
+Tcount
+
+## D8 FLOW DIR ##
+echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_burned_filled_$current_branch_id.tif -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif
+Tcount
+
+## D8 FLOW ACCUMULATIONS ##
+echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/aread8 -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -ad8  $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -wg  $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif -nc
+Tcount
+
+# THRESHOLD ACCUMULATIONS ##
+echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/threshold -ssa $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src  $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -thresh 1
+Tcount
+
+## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
+echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif -g $outputCurrentBranchDataDir/temp_grass
+Tcount
+
+## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
+echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -a $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif -d $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif -t 50 -o $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -th $thalweg_lateral_elev_threshold
+Tcount
+
+## MASK BURNED DEM FOR STREAMS ONLY ###
+echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -B $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif --calc="A/B" --outfile="$outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" --NoDataValue=0
+Tcount
+
+## FLOW CONDITION STREAMS ##
+echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/flowdircond -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif -z $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -zfdc $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
+Tcount
+
+## D8 SLOPES ##
+echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
+Tcount
+
+# STREAMNET FOR REACHES ##
+echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/streamnet -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -fel $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -ord $outputCurrentBranchDataDir/streamOrder_$current_branch_id.tif -tree $outputCurrentBranchDataDir/treeFile_$current_branch_id.txt -coord $outputCurrentBranchDataDir/coordFile_$current_branch_id.txt -w $outputCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif -net $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
+Tcount
+
+## SPLIT DERIVED REACHES ##
+echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+Tcount
+
+## GAGE WATERSHED FOR REACHES ##
+echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -gw $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
+Tcount
+
+## VECTORIZE FEATURE ID CENTROIDS ##
+echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/reachID_grid_to_vector_points.py -r $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -i featureID -p $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
+Tcount
+
+## GAGE WATERSHED FOR PIXELS ##
+echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif -gw $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
+Tcount
+
+# D8 REM ##
+echo -e $startDiv"D8 REM $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/gms/rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
+Tcount
+
+## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
+echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"$stopDiv
+date -u
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+Tcount
+
+## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
+echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+[ -f $outputHucDataDir/LandSea_subset.gpkg ] && \
+gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
+Tcount
+
+## POLYGONIZE REACH WATERSHEDS ##
+echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_polygonize.py -8 -f GPKG $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
+Tcount
+
+## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
+echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
+Tcount
+
+## RASTERIZE NEW CATCHMENTS AGAIN ##
+echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
+Tcount
+
+## MASK SLOPE TO CATCHMENTS ##
+echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"$stopDiv
+date -u
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif --calc="A*(B>0)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
+Tcount
+
+## MAKE CATCHMENT AND STAGE FILES ##
+echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/make_stages_and_catchlist.py -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/stage_$current_branch_id.txt -a $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
+Tcount
+
+## MASK REM RASTER TO REMOVE OCEAN AREAS ##
+echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+[ -f $outputCurrentBranchDataDir/LandSea_subset.tif ] && \
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LandSea_subset_$current_node_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+Tcount
+
+## HYDRAULIC PROPERTIES ##
+echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/catchhydrogeo -hand $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -catch $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif -catchlist $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -slp $outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif -h $outputCurrentBranchDataDir/stage_$current_branch_id.txt -table $outputCurrentBranchDataDir/src_base_$current_branch_id.csv
+Tcount
+
+## FINALIZE CATCHMENTS AND MODEL STREAMS ##
+echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/src_base_$current_branch_id.csv -l $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $outputCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $outputCurrentBranchDataDir/src_$current_branch_id.json -x $outputCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $outputCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg -y $outputCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg -p $extent -k $outputCurrentBranchDataDir/small_segments_$current_branch_id.csv
+Tcount
+
+## REMOVE FILES FROM DENY LIST ##
+if [ -f $deny_gms_branches_list ]; then
+    echo -e $startDiv"Remove files $hucNumber $current_branch_id"$stopDiv
+    date -u
+    Tstart
+    $srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_gms_branches_list -v -b
+    Tcount
+fi
+
+
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
new file mode 100755
index 000000000..b89f695cd
--- /dev/null
+++ b/src/gms/run_by_unit.sh
@@ -0,0 +1,94 @@
+#!/bin/bash -e
+
+## INITIALIZE TOTAL TIME TIMER ##
+T_total_start
+
+## SET OUTPUT DIRECTORY FOR UNIT ##
+hucNumber="$1"
+outputHucDataDir=$outputRunDataDir/$hucNumber
+outputBranchDataDir=$outputHucDataDir/branches
+
+## huc data
+if [ -d "$outputHucDataDir" ]; then
+    if [ $overwrite -eq 1 ]; then
+        rm -rf $outputHucDataDir
+    else
+        echo "Output dir $outputHucDataDir exists. Use overwrite -o to run."
+    fi
+fi
+
+# make outputs directory
+mkdir -p $outputHucDataDir
+
+# make branches outputs directory
+if [ ! -d "$outputBranchDataDir" ]; then
+    mkdir -p $outputBranchDataDir
+fi
+
+## SET VARIABLES AND FILE INPUTS ##
+hucUnitLength=${#hucNumber}
+huc4Identifier=${hucNumber:0:4}
+huc2Identifier=${hucNumber:0:2}
+input_NHD_WBHD_layer=WBDHU$hucUnitLength
+input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m.tif
+input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
+input_bathy_bankfull=$inputDataDir/$bankfull_input_table
+
+# Define the landsea water body mask using either Great Lakes or Ocean polygon input #
+if [[ $huc2Identifier == "04" ]] ; then
+  input_LANDSEA=$input_GL_boundaries
+  echo -e "Using $input_LANDSEA for water body mask (Great Lakes)"
+else
+  input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
+fi
+
+## GET WBD ##
+echo -e $startDiv"Get WBD $hucNumber"$stopDiv
+date -u
+Tstart
+ogr2ogr -f GPKG $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
+Tcount
+
+## Subset Vector Layers ##
+echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent GMS -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
+Tcount
+
+## Clip WBD8 ##
+echo -e $startDiv"Clip WBD8"$stopDiv
+date -u
+Tstart
+ogr2ogr -f GPKG -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+Tcount
+
+## DERIVE LEVELPATH  ##
+echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv
+date -u
+Tstart
+$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v
+Tcount
+
+## STREAM BRANCH POLYGONS
+echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"$stopDiv
+date -u
+Tstart
+$srcDir/gms/buffer_stream_branches.py -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg -v 
+Tcount
+
+## CREATE BRANCHID LIST FILE
+echo -e $startDiv"Create file of branch ids for $hucNumber"$stopDiv
+date -u
+Tstart
+$srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute
+Tcount
+
+## REMOVE FILES FROM DENY LIST ##
+if [ -f $deny_gms_unit_list ]; then
+    echo -e $startDiv"Remove files $hucNumber"$stopDiv
+    date -u
+    Tstart
+    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_gms_unit_list -v
+    Tcount
+fi
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
new file mode 100755
index 000000000..7a28741df
--- /dev/null
+++ b/src/gms/stream_branches.py
@@ -0,0 +1,992 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import pandas as pd
+import rasterio
+from rasterio.mask import mask
+from rasterio.io import DatasetReader
+from os.path import splitext
+import fiona
+from fiona.errors import DriverError
+from collections import deque
+import numpy as np
+from tqdm import tqdm
+from shapely.ops import linemerge, unary_union
+from shapely.geometry import MultiLineString, LineString, MultiPoint, Point
+from shapely.strtree import STRtree
+from random import sample
+from scipy.stats import mode
+
+class StreamNetwork(gpd.GeoDataFrame):
+
+    geom_name = 'geometry' # geometry attribute name
+    branch_id_attribute = None # branch id attribute name
+    values_excluded = None
+    attribute_excluded = None
+
+    def __init__(self,*args,**kwargs):
+        
+        if kwargs:
+            branch_id_attribute = kwargs.pop("branch_id_attribute",None)
+            values_excluded = kwargs.pop("values_excluded",None)
+            attribute_excluded = kwargs.pop("attribute_excluded",None)
+        
+        super().__init__(*args,**kwargs)
+        
+        self.branch_id_attribute = branch_id_attribute
+        self.values_excluded = values_excluded
+        self.attribute_excluded = attribute_excluded
+
+
+    @classmethod
+    def from_file(cls,filename,branch_id_attribute=None,values_excluded=None,attribute_excluded=None, verbose=False,*args,**kwargs):
+
+        """ loads stream network from file to streamnetwork geopandas """
+
+        if kwargs:
+            inputs = {  'branch_id_attribute' : kwargs.pop("branch_id_attribute",None) , 
+                        'values_excluded' :kwargs.pop("values_excluded",None) ,
+                        'attribute_excluded' : kwargs.pop("attribute_excluded",None)      }
+            
+            verbose = kwargs.pop('verbose',None)
+        else:
+            inputs = {  'branch_id_attribute' : branch_id_attribute , 
+                        'values_excluded' : values_excluded ,
+                        'attribute_excluded' : attribute_excluded      }
+            
+        if verbose: 
+            print('Loading file')
+        
+        return(cls(gpd.read_file(filename,*args,**kwargs),**inputs))
+
+
+    def write(self,fileName,layer=None,index=True,verbose=False):
+
+        """ Gets driver Name from file extension for Geopandas writing """
+
+        if verbose:
+            print("Writing to {}".format(fileName))
+
+        # sets driver
+        driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
+        driver = driverDictionary[splitext(fileName)[1]]
+    
+        self.to_file(fileName, driver=driver, layer=layer, index=index)
+
+    def apply(self,*args,**kwargs):
+        
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+        crs = self.crs
+        
+        self = super().apply(*args,**kwargs)
+        self = self.set_crs(crs)
+        
+        self = StreamNetwork( self,
+                              branch_id_attribute=branch_id_attribute,
+                              attribute_excluded=attribute_excluded,
+                              values_excluded=values_excluded
+                            )
+
+        return(self)
+
+
+    def multilinestrings_to_linestrings(self):
+        
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+        
+        def convert_to_linestring(row):
+
+            geometry = row['geometry']
+
+            if isinstance(geometry,MultiLineString):
+                linestring = LineString(sum([list(item.coords) for item in list(geometry.geoms)],[]))
+                row['geometry'] = linestring
+            
+            return(row)
+
+        
+        self = StreamNetwork( self.apply(convert_to_linestring,axis=1),
+                              branch_id_attribute=branch_id_attribute,
+                              attribute_excluded=attribute_excluded,
+                              values_excluded=values_excluded
+                            )
+        
+        return(self)
+
+
+    def explode(self,**kwargs):
+        
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+
+        self = StreamNetwork( super().explode(**kwargs),
+                             branch_id_attribute=branch_id_attribute,
+                             attribute_excluded=attribute_excluded,
+                             values_excluded=values_excluded
+                            )
+
+        return(self)
+
+
+    def to_df(self,*args,**kwargs):
+
+        """ Converts back to dataframe """
+        
+        self = pd.DataFrame(self,*args,**kwargs)
+
+        return(self)
+
+
+    def merge(self,*args,**kwargs):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+
+        self = super().merge(*args,**kwargs)
+
+        self = StreamNetwork(self,branch_id_attribute=branch_id_attribute,
+                             attribute_excluded=attribute_excluded,
+                             values_excluded=values_excluded)
+
+        return(self)
+
+
+    def merge_stream_branches(self,stream_branch_dataset,on='NHDPlusID',branch_id_attribute='LevelPathI',attributes='StreamOrde',stream_branch_layer_name=None):
+
+        """ Merges stream branch id attribute from another vector file """
+
+        # load vaas
+        if isinstance(stream_branch_dataset,str):
+            stream_branch_dataset = gpd.read_file(stream_branch_dataset,layer=stream_branch_layer_name)
+        elif isinstance(stream_branch_dataset,gpd.GeoDataFrame):
+            pass
+        else:
+            raise TypeError('Pass stream_branch_dataset argument as filepath or GeoDataframe')
+    
+        # merge and drop duplicate columns
+        if isinstance(attributes,list):
+            what = [on] + [branch_id_attribute] + attributes 
+        elif isinstance(attributes,str):
+            what = [on] + [branch_id_attribute] +[attributes]
+
+        self = self.merge(stream_branch_dataset[what],on=on, how='inner')
+        
+        # make sure it's the correct object type
+        self = StreamNetwork(self,branch_id_attribute=branch_id_attribute)
+
+        return(self)
+
+
+    @staticmethod
+    def flip_inlet_outlet_linestring_index(linestring_index):
+
+        # returns -1 for 0 and 0 for -1
+        inlet_outlet_linestring_index_dict = { 0 : -1, -1 : 0 }
+        
+        try:
+            return( inlet_outlet_linestring_index_dict[linestring_index] )
+        except KeyError:
+            raise ValueError('Linestring index should be 0 or -1')
+
+
+    def derive_nodes(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',reach_id_attribute='NHDPlusID',
+                     outlet_linestring_index=0,node_prefix=None,max_node_digits=8,verbose=False):
+        
+        if verbose:
+            print("Deriving nodes ...")
+
+        inlet_linestring_index = StreamNetwork.flip_inlet_outlet_linestring_index(outlet_linestring_index)
+        
+        # set node prefix to string
+        if node_prefix is None:
+            node_prefix=''
+        
+        # handle digits and values for node ids
+        max_post_node_digits = max_node_digits - len(node_prefix)
+        max_node_value = int('9' * max_post_node_digits)
+
+        # sets index of stream branches as reach id attribute
+        #if self.index.name != reach_id_attribute:
+            #self.set_index(reach_id_attribute,drop=True,inplace=True)
+        
+        inlet_coordinates, outlet_coordinates = dict(), dict()
+        node_coordinates = dict()
+        toNodes, fromNodes = [None] * len(self),[None] * len(self)
+        current_node_id = '1'.zfill(max_post_node_digits)
+
+        for i,(_,row) in enumerate(self.iterrows()):
+            
+            reach_id = row[reach_id_attribute]
+
+            # get foss id for node_prefix
+            if len(node_prefix) > 0:
+                current_node_prefix = node_prefix
+            else:
+                current_node_prefix = str(reach_id)[0:4]
+            
+            # makes list of coordinates. Merges multi-part geoms
+            reach_coordinates = list(row['geometry'].coords)
+
+            inlet_coordinate = reach_coordinates[inlet_linestring_index]
+            outlet_coordinate = reach_coordinates[outlet_linestring_index]
+            
+            if inlet_coordinate not in node_coordinates:
+                current_node_id_with_prefix = current_node_prefix + current_node_id
+                node_coordinates[inlet_coordinate] = current_node_id_with_prefix
+                fromNodes[i] = current_node_id_with_prefix
+                
+                current_node_id = int(current_node_id.lstrip('0')) + 1
+                if current_node_id > max_node_value:
+                    raise ValueError('Current Node ID exceeding max. Look at source code to change.')
+                current_node_id = str(current_node_id).zfill(max_post_node_digits)
+
+            else:
+                fromNodes[i] = node_coordinates[inlet_coordinate]
+
+            if outlet_coordinate not in node_coordinates:
+                current_node_id_with_prefix = current_node_prefix + current_node_id
+                node_coordinates[outlet_coordinate] = current_node_id_with_prefix
+                toNodes[i] = current_node_id_with_prefix
+
+                current_node_id = int(current_node_id.lstrip('0')) + 1
+                if current_node_id > max_node_value:
+                    raise ValueError('Current Node ID exceeding max. Look at source code to change.')
+                current_node_id = str(current_node_id).zfill(max_post_node_digits)
+
+            else:
+                toNodes[i] = node_coordinates[outlet_coordinate]
+
+        self.loc[:,fromNode_attribute] = fromNodes
+        self.loc[:,toNode_attribute] = toNodes
+        
+        
+        return(self)
+
+
+    def derive_outlets(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',outlets_attribute='outlet_id',
+                       verbose=False):
+
+        if verbose:
+            print("Deriving outlets ...")
+
+        fromNodes = set(i for i in self[fromNode_attribute])
+
+        outlets = [-1] * len(self)
+        
+        for i,tn in enumerate(self[toNode_attribute]):
+            if tn not in fromNodes:
+                outlets[i] = i + 1
+
+        self[outlets_attribute] = outlets
+
+        return(self)
+
+
+    def derive_inlets(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',
+                      inlets_attribute='inlet_id',
+                      verbose=False
+                     ):
+
+        if verbose:
+            print("Deriving inlets ...")
+
+        toNodes = set(i for i in self[toNode_attribute])
+
+        inlets = [-1] * len(self)
+
+        for i,fn in enumerate(self[fromNode_attribute]):
+            if fn not in toNodes:
+                inlets[i] = i + 1
+
+        self[inlets_attribute] = inlets
+
+        return(self)
+
+
+    def derive_inlet_points_by_feature( self,feature_attribute, outlet_linestring_index):
+        
+        """ Finds the upstream point of every feature in the stream network """
+
+        inlet_linestring_index = StreamNetwork.flip_inlet_outlet_linestring_index(outlet_linestring_index)
+
+        feature_inlet_points_gdf = gpd.GeoDataFrame(self.copy())
+
+        for idx, row in self.iterrows():
+            feature_inlet_point = Point(row.geometry.coords[inlet_linestring_index])  
+
+            feature_inlet_points_gdf.loc[idx,'geometry'] = feature_inlet_point
+
+        return(feature_inlet_points_gdf)
+
+
+    def derive_headwater_points_with_inlets( self,inlets_attribute='inlet_id',
+                                             fromNode_attribute='FromNode',
+                                             outlet_linestring_index=0
+                                           ):
+    
+        """ Derives headwater points file given inlets """
+        
+        # get inlet linestring index
+        inlet_linestring_index = StreamNetwork.flip_inlet_outlet_linestring_index(outlet_linestring_index)
+
+        inlet_indices = self.loc[:,inlets_attribute] != -1
+
+        inlets = self.loc[inlet_indices,:].reset_index(drop=True)
+
+        headwater_points_gdf = gpd.GeoDataFrame(inlets.copy())
+        
+        for idx,row in inlets.iterrows():
+            headwater_point = row.geometry.coords[inlet_linestring_index]
+
+            headwater_point = Point(headwater_point)
+
+            headwater_points_gdf.loc[idx,'geometry'] = headwater_point
+
+        return(headwater_points_gdf)
+
+    def remove_stream_segments_without_catchments( self,
+                                                   catchments,
+                                                   reach_id_attribute='NHDPlusID',
+                                                   reach_id_attribute_in_catchments='NHDPlusID',
+                                                   verbose=False
+                                                 ):
+
+        if verbose:
+            print("Removing stream segments without catchments ...")
+
+        # load cathments
+        if isinstance(catchments,gpd.GeoDataFrame):
+            pass
+        elif isinstance(catchments,str):
+            catchments = gpd.read_file(catchments)
+        else:
+            raise TypeError("Catchments needs to be GeoDataFame or path to vector file")
+        
+        self = self.merge(catchments.loc[:,reach_id_attribute_in_catchments],
+                          left_on=reach_id_attribute,
+                          right_on=reach_id_attribute_in_catchments,
+                          how='inner')
+
+        return(self)
+
+
+    def remove_branches_without_catchments(self,
+                                           catchments,
+                                           reach_id_attribute='NHDPlusID',
+                                           branch_id_attribute='branchID',
+                                           reach_id_attribute_in_catchments='NHDPlusID',
+                                           verbose=False
+                                           ):
+
+        if verbose:
+            print("Removing stream branches without catchments ...")
+
+        # load cathments
+        if isinstance(catchments,gpd.GeoDataFrame):
+            pass
+        elif isinstance(catchments,str):
+            catchments = gpd.read_file(catchments)
+        else:
+            raise TypeError("Catchments needs to be GeoDataFame or path to vector file")
+        
+        unique_stream_branches = self.loc[:,branch_id_attribute].unique()
+        unique_catchments = set(catchments.loc[:,reach_id_attribute_in_catchments].unique())
+
+        current_index_name = self.index.name
+        self.set_index(branch_id_attribute,drop=False,inplace=True)
+
+        for usb in unique_stream_branches:
+
+            try:
+                reach_ids_in_branch = set(self.loc[usb,reach_id_attribute].unique())
+            except AttributeError:
+                reach_ids_in_branch = set( [ self.loc[usb,reach_id_attribute] ] )
+
+            if len( reach_ids_in_branch & unique_catchments) == 0:
+                #print(f'Dropping {usb}')
+                self.drop(usb,inplace=True)
+
+        if current_index_name is None:
+            self.reset_index(drop=True,inplace=True)
+        else:
+            self.set_index(current_index_name,drop=True,inplace=True)
+
+        return(self)
+
+
+    def derive_stream_branches(self,toNode_attribute='ToNode',
+                               fromNode_attribute='FromNode',
+                               upstreams=None,
+                               outlet_attribute='outlet_id',
+                               branch_id_attribute='branchID',
+                               reach_id_attribute='NHDPlusID',
+                               comparison_attributes='StreamOrde',
+                               comparison_function=max,
+                               max_branch_id_digits=6,
+                               verbose=False):
+
+        """ Derives stream branches """
+
+        # checks inputs
+        allowed_comparison_function = {max,min}
+        if comparison_function not in allowed_comparison_function:
+            raise ValueError(f"Only {allowed_comparison_function} comparison functions allowed")
+
+        # sets index of stream branches as reach id attribute
+        reset_index = False
+        if self.index.name != reach_id_attribute:
+            self.set_index(reach_id_attribute,drop=True,inplace=True)
+            reset_index = True
+
+        # make upstream and downstream dictionaries if none are passed
+        if upstreams is None:
+            upstreams,_ = self.make_up_and_downstream_dictionaries(reach_id_attribute=reach_id_attribute,
+                                                                   toNode_attribute=toNode_attribute,
+                                                                   fromNode_attribute=fromNode_attribute,
+                                                                   verbose=verbose)
+        
+        # initialize empty queue, visited set, branch attribute column, and all toNodes set
+        Q = deque()
+        visited = set()
+        self[branch_id_attribute] = [-1] * len(self)
+
+        # progress bar
+        progress = tqdm(total=len(self),disable=(not verbose),desc='Stream branches')
+
+        outlet_boolean_mask = self[outlet_attribute] >= 0
+        outlet_reach_ids = self.index[outlet_boolean_mask].tolist()
+
+        branch_ids = [ str(h)[0:4] + str(b+1).zfill(max_branch_id_digits) for b,h in enumerate(outlet_reach_ids) ]
+
+        self.loc[outlet_reach_ids,branch_id_attribute] = branch_ids
+        Q = deque(outlet_reach_ids)
+        visited = set(outlet_reach_ids)
+        bid = int(branch_ids[-1][-max_branch_id_digits:].lstrip('0')) + 1
+        progress.update(bid-1)
+
+        # breath-first traversal
+        # while queue contains reaches
+        while Q:
+
+            # pop current reach id from queue
+            current_reach_id = Q.popleft()
+            
+            # update progress
+            progress.update(1)
+
+            # get current reach stream order and branch id
+            current_reach_comparison_value = self.at[current_reach_id,comparison_attributes]
+            current_reach_branch_id = self.at[current_reach_id,branch_id_attribute]
+
+            # get upstream ids
+            upstream_ids = upstreams[current_reach_id]
+
+            # identify upstreams by finding if fromNode exists in set of all toNodes
+            if upstream_ids:
+                
+                # determine if each upstream has been visited or not
+                not_visited_upstream_ids = [] # list to save not visited upstreams
+                for us in upstream_ids:
+
+                    # if upstream id has not been visited
+                    if us not in visited:
+
+                        # add to visited set and to queue
+                        visited.add(us)
+                        Q.append(us)
+                        not_visited_upstream_ids += [us]
+
+                # if upstreams that are not visited exist
+                if not_visited_upstream_ids:
+
+                    # find 
+                    upstream_reaches_compare_values = self.loc[not_visited_upstream_ids,comparison_attributes]
+                    matching_value = comparison_function(upstream_reaches_compare_values)
+                    
+                    matches = 0 # if upstream matches are more than 1, limits to only one match
+                    for usrcv,nvus in zip(upstream_reaches_compare_values,not_visited_upstream_ids):
+                        if (usrcv == matching_value) & (matches == 0):
+                            self.at[nvus,branch_id_attribute] = current_reach_branch_id
+                            matches += 1
+                        else:
+                            branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
+                            self.at[nvus,branch_id_attribute] = branch_id
+                            bid += 1
+        
+        progress.close()
+
+        if reset_index:
+            self.reset_index(drop=False,inplace=True)
+
+        return(self)
+
+
+    def make_up_and_downstream_dictionaries(self,reach_id_attribute='NHDPlusID',
+                                            toNode_attribute='ToNode',
+                                            fromNode_attribute='FromNode',
+                                            verbose=False):
+        
+        # sets index of stream branches as reach id attribute
+        #if self.index.name != reach_id_attribute:
+        #    self.set_index(reach_id_attribute,drop=True,inplace=True)
+
+        # find upstream and downstream dictionaries
+        upstreams,downstreams = dict(),dict()
+        
+        for _, row in tqdm(self.iterrows(),disable=(not verbose),
+                           total=len(self), desc='Upstream and downstream dictionaries'):
+            
+            reach_id = row[reach_id_attribute]
+            downstreams[reach_id] = self.loc[ self[fromNode_attribute] == row[toNode_attribute] , reach_id_attribute].tolist() 
+            upstreams[reach_id] = self.loc[ self[toNode_attribute] == row[fromNode_attribute] , reach_id_attribute].tolist()
+        
+        return(upstreams,downstreams)
+
+
+    def get_arbolate_sum(self,arbolate_sum_attribute='arbolate_sum',inlets_attribute='inlet_id',
+                         reach_id_attribute='NHDPlusID',length_conversion_factor_to_km = 0.001,
+                         upstreams=None, downstreams=None,
+                         toNode_attribute='ToNode',
+                         fromNode_attribute='FromNode',
+                         verbose=False
+                        ):
+        
+        # sets index of stream branches as reach id attribute
+        reset_index = False
+        if self.index.name != reach_id_attribute:
+            self.set_index(reach_id_attribute,drop=True,inplace=True)
+            reset_index = True
+
+        # make upstream and downstream dictionaries if none are passed
+        if (upstreams is None) | (downstreams is None):
+            upstreams, downstreams = self.make_up_and_downstream_dictionaries(reach_id_attribute=reach_id_attribute,
+                                                                              toNode_attribute=toNode_attribute,
+                                                                              fromNode_attribute=fromNode_attribute,
+                                                                              verbose=verbose)
+
+        # initialize queue, visited set, with inlet reach ids
+        inlet_reach_ids = self.index[self[inlets_attribute] >= 0].tolist()
+        S = deque(inlet_reach_ids)
+        visited = set()
+
+        # initialize arbolate sum, make length km column, make all from nodes set
+        self[arbolate_sum_attribute] = self.geometry.length * length_conversion_factor_to_km 
+        
+        # progress bar
+        progress = tqdm(total=len(self),disable=(not verbose), desc= "Arbolate sums")
+
+        # depth-first traversal
+        # while stack contains reaches
+        while S:
+            
+            # pop current reach id from queue
+            current_reach_id = S.pop()
+            
+            # current arbolate sum
+            current_reach_arbolate_sum = self.at[current_reach_id,arbolate_sum_attribute]
+
+            # if current reach id is not visited mark as visited
+            if current_reach_id not in visited:
+                visited.add(current_reach_id)
+                progress.update(n=1)
+
+            # get downstream ids
+            downstream_ids = downstreams[current_reach_id]
+            
+            if downstream_ids:
+                
+                for ds in downstream_ids:
+                    
+                    # figure out of all upstream reaches of ds have been visited
+                    upstream_of_ds_ids = set(upstreams[ds])
+                    all_upstream_ids_of_ds_are_visited = upstream_of_ds_ids.issubset(visited)
+                    
+                    # append downstream to stack
+                    if all_upstream_ids_of_ds_are_visited:
+                        S.append(ds)
+                        
+                    self.loc[ds,arbolate_sum_attribute] += current_reach_arbolate_sum
+        
+        progress.close()
+    
+        if reset_index:
+            self.reset_index(drop=False,inplace=True)
+
+        return(self)
+
+
+    def dissolve_by_branch(self,branch_id_attribute='LevelPathI',attribute_excluded='StreamOrde',
+                           values_excluded=[1,2],out_vector_files=None, verbose=False):
+        
+        if verbose:
+            print("Dissolving by branch ...")
+
+        # exclude attributes and their values
+        if (attribute_excluded is not None) & (values_excluded is not None):
+            values_excluded = set(values_excluded)
+            exclude_indices = [False if i in values_excluded else True for i in self[attribute_excluded]]
+            self = self.loc[exclude_indices,:]
+
+        # dissolve lines
+        self['bids_temp'] = self.loc[:,branch_id_attribute].copy()
+        self = self.dissolve(by=branch_id_attribute)
+        self.rename(columns={'bids_temp' : branch_id_attribute},inplace=True)
+        
+        # merges each multi-line string to a sigular linestring
+        for lpid,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc="Merging mult-part geoms"):
+            if isinstance(row.geometry,MultiLineString):
+                merged_line = linemerge(row.geometry)
+                #self.loc[lpid,'geometry'] = merged_line
+                try:
+                    self.loc[lpid,'geometry'] = merged_line
+                except ValueError:
+                    merged_line = list(merged_line.geoms)[0]
+                    self.loc[lpid,'geometry'] = merged_line
+        
+        #self[branch_id_attribute] = bids
+        self = StreamNetwork(self,branch_id_attribute=branch_id_attribute,
+                             attribute_excluded=attribute_excluded,
+                             values_excluded=values_excluded)
+
+        if out_vector_files is not None:
+            
+            base_file_path,extension = splitext(out_vector_files)
+            
+            if verbose:
+                print("Writing dissolved branches ...")
+            
+            #for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
+                #out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
+                
+                #bid_indices = self.loc[:,branch_id_attribute] == bid
+                #current_stream_network = StreamNetwork(self.loc[bid_indices,:])
+
+                #current_stream_network.write(out_vector_file,index=False)
+            self.write(out_vector_files,index=False)
+
+        return(self)
+
+
+    def derive_segments(self,inlets_attribute='inlet_id', reach_id_attribute='NHDPlusID'):
+        pass
+
+
+    def conflate_branches(self,target_stream_network,branch_id_attribute_left='branch_id',
+                          branch_id_attribute_right='branch_id', left_order_attribute='order_',
+                          right_order_attribute='order_',
+                          crosswalk_attribute='crosswalk_id', verbose=False):
+        
+        # get unique stream orders
+        orders = self.loc[:,right_order_attribute].unique()
+
+        # make a dictionary of STR trees for every stream order
+        trees = { o:STRtree(target_stream_network.geometry.tolist()) for o in orders }
+
+        # make the crosswalk id attribute and set index
+        self.loc[:,crosswalk_attribute] = [None] * len(self)
+        self.set_index(branch_id_attribute_left,inplace=True)
+
+        # loop through rows of self
+        for idx,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc="Conflating branches"):
+
+            g = row['geometry']
+            o = row[left_order_attribute]
+            
+            tree = trees[o]
+
+            # find nearest geom in target and its index
+            matching_geom = tree.nearest(g)
+            match_idx = target_stream_network.geometry == matching_geom
+            
+            # get the branch ids
+            right_branch_id = int(target_stream_network.loc[match_idx,branch_id_attribute_left])
+            left_branch_id = idx
+
+            # save the target matching branch id 
+            self.loc[left_branch_id,crosswalk_attribute] = right_branch_id
+
+        # reset indices
+        self.reset_index(inplace=True,drop=False)
+
+        return(self)
+        
+
+    def explode_to_points(self,reach_id_attribute='NHDPlusID', sampling_size=None,
+                          verbose=False):
+        
+        points_gdf = self.copy()
+        points_gdf.reset_index(inplace=True,drop=True)
+
+        all_exploded_points = [None] * len(points_gdf)
+        for idx,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc='Exploding Points'):
+            
+            geom = row['geometry']
+            
+            exploded_points = [p for p in iter(geom.coords)]
+
+            if sampling_size is None:
+                exploded_points = MultiPoint(exploded_points)
+            else:
+                try:
+                    exploded_points = MultiPoint( sample(exploded_points,sampling_size) )
+                except ValueError:
+                    exploded_points = MultiPoint( exploded_points )
+            
+            all_exploded_points[idx] = exploded_points
+
+        points_gdf['geometry'] = all_exploded_points
+        
+        points_gdf = points_gdf.explode()
+        points_gdf.reset_index(inplace=True,drop=True)
+
+        return(points_gdf)
+
+
+    @staticmethod
+    def conflate_points(source_points,target_points,source_reach_id_attribute,target_reach_id_attribute,verbose=False):
+
+        tree = STRtree(target_points.geometry.tolist())
+
+        # find matching geometry
+        matches_dict = dict.fromkeys(source_points.loc[:,source_reach_id_attribute].astype(int).tolist(),[])
+        for idx,row in tqdm(source_points.iterrows(),total=len(source_points),disable=(not verbose),desc="Conflating points"):
+
+            geom = row['geometry']
+            nearest_target_point = tree.nearest(geom)
+            match_idx = target_points.index[target_points.geometry == nearest_target_point].tolist()
+            
+            if len(match_idx) > 1:
+                match_idx = match_idx[0]
+            else:
+                match_idx = match_idx[0]
+
+            matched_id = int(target_points.loc[match_idx,target_reach_id_attribute])
+            source_id = int(row[source_reach_id_attribute])
+            matches_dict[source_id] = matches_dict[source_id] + [matched_id]
+
+            #if len(matches_dict[source_id])>1:
+            #    print(matches_dict[source_id])
+
+        # get mode of matches
+        if verbose:
+            print("Finding mode of matches ...")
+
+        for source_id,matches in matches_dict.items():
+            majority = mode(matches).mode
+            matches_dict[source_id] = majority[0]
+        
+        
+        # make dataframe
+        if verbose:
+            print("Generating crosswalk table ...")
+
+        crosswalk_table = pd.DataFrame.from_dict(matches_dict,orient='index',
+                                                 columns=[target_reach_id_attribute])
+        crosswalk_table.index.name = source_reach_id_attribute
+
+
+        return(crosswalk_table)
+
+
+    def clip(self,mask,keep_geom_type=False,verbose=False):
+
+        if verbose:
+            print("Clipping streams to mask ...")
+        
+        # load mask
+        if isinstance(mask,gpd.GeoDataFrame):
+            pass
+        elif isinstance(mask,str):
+            mask = gpd.read_file(mask)
+        else:
+            raise TypeError("mask needs to be GeoDataFame or path to vector file")
+
+
+        branch_id_attribute = self.branch_id_attribute 
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+    
+        self = StreamNetwork(
+                             gpd.clip(self,mask,keep_geom_type).reset_index(drop=True),
+                             branch_id_attribute=branch_id_attribute,
+                             attribute_excluded=attribute_excluded,
+                             values_excluded=values_excluded)
+
+        return(self)
+
+
+class StreamBranchPolygons(StreamNetwork):
+
+    #branch_id_attribute = None
+    #values_excluded = None
+    #attribute_excluded = None
+
+    def __init__(self,*args,**kwargs):
+        super().__init__(*args,**kwargs)
+     
+   
+    @classmethod
+    def buffer_stream_branches(cls,stream_network,buffer_distance,verbose=True):
+
+        """ Buffers stream branches by distance """
+
+        if verbose:
+            print("Buffering stream branches to polygons")
+
+        buffer_distance = int(buffer_distance)
+
+        # buffer lines
+        new_bids = [None] *len(stream_network) ; new_geoms = new_bids.copy()
+        i=0
+   
+        for _,row in tqdm(stream_network.iterrows(),disable=(not verbose),total=len(stream_network)):
+            new_geoms[i] = row[stream_network.geometry.name].buffer(buffer_distance)
+            new_bids[i] = i + 1
+            i += 1
+        
+        # create polys gpd
+        polys = stream_network.copy()
+
+        # assign to StreamBranchPolys
+        #polys[stream_network.branch_id_attribute] = new_bids
+        polys[stream_network.geom_name] = new_geoms
+        polys.set_geometry(stream_network.geom_name)
+        
+        # assign class and attributes
+        polys = cls(polys,branch_id_attribute=stream_network.branch_id_attribute,
+                    attribute_excluded=stream_network.attribute_excluded,
+                    values_excluded=stream_network.values_excluded)
+
+        return(polys)
+
+    @staticmethod
+    def query_vectors_by_branch(vector,branch_ids,branch_id_attribute,out_filename_template=None,vector_layer=None):
+        
+        # load vaas
+        if isinstance(vector,str):
+            vector_filename = vector
+            #vector = gpd.read_file(vector_filename,layer=vector_layer)
+            vector = fiona.open(vector_filename,'r',layer=vector_layer)
+        elif isinstance(vector,fiona.Collection):
+            pass
+        else:
+            raise TypeError('Pass vector argument as filepath or fiona collection')
+
+
+        def __find_matching_record(vector,attribute,value,matching='first'):
+
+            if matching not in ('first','all'):
+                raise ValueError("matching needs to be \'first\' or \'all\'")
+            
+            matches = []
+            for rec in vector:
+               if rec['properties'][attribute] == value:
+                   if matching == 'first':
+                       matches = [ rec ]
+                       break
+                   elif matching == 'all':
+                       matches += [ rec ] 
+
+            return(matches)
+
+
+        # get source information
+        source_meta = vector.meta
+
+        # out records
+        out_records = []
+        
+        for bid in branch_ids:
+            out_records += __find_matching_record(vector,branch_id_attribute,bid,matching='all')
+
+        if (out_filename_template is not None) & ( len(out_records) != 0):
+                base,ext = out_filename_template.split('.')
+                out_filename = base + "_{}.".format(bid) + ext
+                
+                with fiona.open(out_filename,'w',**source_meta) as out_file:
+                    out_file.writerecords(out_records)
+                
+        # close
+        vector.close()
+
+        return(out_records)
+
+
+    def clip(self,to_clip,out_filename_template=None,branch_id=None,branch_id_attribute=None):
+
+        """ Clips a raster or vector to the stream branch polygons """
+
+        fileType = "raster" #default
+        
+        # load raster or vector file
+        if isinstance(to_clip,DatasetReader): #if already rasterio dataset
+            pass
+        elif isinstance(to_clip,str): # if a string
+
+            try: # tries to open as rasterio first then geopanda
+                to_clip = rasterio.open(to_clip,'r')
+            except rasterio.errors.RasterioIOError:
+                try:
+                    to_clip = gpd.read_file(to_clip)
+                    fileType = "vector"
+                except DriverError: 
+                    raise IOError("{} file not found".format(to_clip))
+
+        elif isinstance(to_clip,gpd.GeoDataFrame): # if a geopanda dataframe
+            fileType = "vector"
+        else:
+            raise TypeError("Pass rasterio dataset,geopandas GeoDataFrame, or filepath to raster or vector file")
+
+        # generator to iterate
+        if branch_id is not None:
+            #print(iter(tuple([0,self.loc[self.loc[:,branch_id_attribute]==branch_id,:].squeeze()])))
+            generator_to_iterate = enumerate( [self.loc[self.loc[:,branch_id_attribute]==branch_id,:].squeeze()] ) 
+        else:
+            generator_to_iterate = self.iterrows()
+
+        return_list = [] # list to return rasterio objects or gdf's
+        
+        if fileType == "raster":
+            buffered_meta = to_clip.meta.copy()
+            buffered_meta.update(blockxsize=256, blockysize=256, tiled=True)
+
+            for i,row in generator_to_iterate:
+                buffered_array,buffered_transform = mask(to_clip,[row[self.geom_name]],crop=True)
+
+                buffered_meta.update(height = buffered_array.shape[1],
+                                     width = buffered_array.shape[2],
+                                     transform = buffered_transform
+                                    )
+
+                # write out files
+                if out_filename_template is not None:
+                    branch_id = row[self.branch_id_attribute]
+
+                    base,ext = out_filename_template.split('.')
+                    out_filename = base + "_{}.".format(branch_id) + ext
+                    
+                    with rasterio.open(out_filename,'w',**buffered_meta) as out:
+                        out.write(buffered_array)
+            
+                # return files in list
+                return_list += [out]
+
+            out.close()
+
+        if fileType == "vector":
+            for i,row in generator_to_iterate:
+                branch_id = row[self.branch_id_attribute]
+                out = gpd.clip(to_clip,row[self.geom_name],keep_geom_type=True)
+                return_list += [out]
+                
+                if (out_filename_template is not None) & (not out.empty):
+                    base,ext = out_filename_template.split('.')
+                    out_filename = base + "_{}.".format(branch_id) + ext
+                    StreamNetwork.write(out,out_filename)
+        
+        return(return_list)
diff --git a/src/gms/subset_catch_list_by_branch_id.py b/src/gms/subset_catch_list_by_branch_id.py
new file mode 100755
index 000000000..606d4e99e
--- /dev/null
+++ b/src/gms/subset_catch_list_by_branch_id.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import geopandas as gpd
+from stream_branches import StreamNetwork
+from os.path import splitext
+import argparse
+
+def Subset_catch_list(catch_list,stream_network,branch_id_attribute,branch_id_list=None,out_catch_list=None,verbose=False):
+
+    if verbose:
+        print("Loading files ....")
+    
+    # loading files
+    catch_list = pd.read_csv(catch_list,sep=" ",header=None,skiprows=1)
+    catch_list.rename(columns={0:"HydroID",1:"slopes",2:"lengthKM",3:"areasqkm"},inplace=True)
+    stream_network = StreamNetwork.from_file(stream_network,branch_id_attribute=branch_id_attribute)
+    stream_network = StreamNetwork(stream_network.astype({'HydroID':int}),branch_id_attribute=branch_id_attribute)
+
+    if verbose:
+        print("Merging HydroIDs ... ")
+    catch_list = catch_list.merge(stream_network.loc[:,["HydroID",branch_id_attribute]],on='HydroID',how='inner')
+
+    unique_branch_ids = catch_list.loc[:,branch_id_attribute].sort_values().unique()
+    base_file_path,extension = splitext(out_catch_list)
+
+    if branch_id_list:
+        # write unique branch ids to file
+        if verbose: 
+            print("Writing branch id list ...")
+    
+        unique_branch_ids.tofile(branch_id_list,sep="\n")
+
+    if verbose:
+        print("Writing catch list subsets ...")
+    for bid in unique_branch_ids:
+
+        # subsetting to branch id and getting number of hydroids
+        branch_catch_list = catch_list.loc[ catch_list.loc[:,branch_id_attribute] == bid, : ]
+        num_of_hydroIDs = len(branch_catch_list)
+
+        # dropping branch id attribute
+        branch_catch_list = branch_catch_list.drop(columns=branch_id_attribute)
+
+        # out file name management
+        out_branch_catch_list = "{}_{}{}".format(base_file_path,bid,extension)
+
+        # write number of hydroids
+        with open(out_branch_catch_list,'w') as f:
+            f.write("{}\n".format(num_of_hydroIDs))
+
+        # write out catch list in append mode
+        branch_catch_list.to_csv(out_branch_catch_list, mode = 'a',header=False,sep=" ",index=False)
+
+    return(catch_list)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Subsets catch list to branch scale')
+    parser.add_argument('-c','--catch-list', help='Input catchment list', required=True)
+    parser.add_argument('-s','--stream-network', help='Stream Network with HydroIDs and Branch IDs', required=True)
+    parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
+    parser.add_argument('-l','--branch-id-list', help='Output the branch id list file desired', required=False,default=None)
+    parser.add_argument('-o','--out-catch-list', help='Output catchment list', required=False,default=None)
+    parser.add_argument('-v','--verbose', help='Verbose output', required=False,default=False,action='store_true')
+    
+    args = vars(parser.parse_args())
+
+    Subset_catch_list(**args)
diff --git a/src/gms/test_new_crosswalk.sh b/src/gms/test_new_crosswalk.sh
new file mode 100755
index 000000000..620c94223
--- /dev/null
+++ b/src/gms/test_new_crosswalk.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -e
+
+
+python3 /foss_fim/src/gms/crosswalk_nwm_demDerived.py -n /data/outputs/first_batch_test_FR_c/12090301/nwm_subset_streams.gpkg -d /data/outputs/first_batch_test_FR_c/12090301/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -v -c /data/temp/continuity/cross_walk_$1.csv -e /data/temp/continuity/demDerived_crosswalked_$1.gpkg -m /data/temp/continuity/nwm_traversal_$1.gpkg -w /data/temp/continuity/wbd.gpkg -a $1
diff --git a/src/gms/time_and_tee_run_by_branch.sh b/src/gms/time_and_tee_run_by_branch.sh
new file mode 100755
index 000000000..2377fdb72
--- /dev/null
+++ b/src/gms/time_and_tee_run_by_branch.sh
@@ -0,0 +1,11 @@
+#!/bin/bash -e
+
+echo 
+echo "================================================================================"
+echo "Start Processing HUC $1 - Branch $2"
+
+/usr/bin/time -v $srcDir/gms/run_by_branch.sh $1 $2 |& tee $outputRunDataDir/logs/branch/$1_gms_branch_$2.log
+#exit ${PIPESTATUS[0]}
+
+echo "================================================================================"
+exit
diff --git a/src/gms/time_and_tee_run_by_unit.sh b/src/gms/time_and_tee_run_by_unit.sh
new file mode 100755
index 000000000..9ef8270c0
--- /dev/null
+++ b/src/gms/time_and_tee_run_by_unit.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+echo 
+echo "================================================================================"
+echo "Start Processing HUC $1"
+
+# TODO: Jan 10, 2020: This rarely receives anything but an exit code of 1 (even if a huc fails). 
+/usr/bin/time -v $srcDir/gms/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/unit/$1_gms_unit.log 
+
+echo "================================================================================"
+#exit ${PIPESTATUS[0]}
+exit
diff --git a/src/gms/toDo.md b/src/gms/toDo.md
new file mode 100644
index 000000000..fe7e131bf
--- /dev/null
+++ b/src/gms/toDo.md
@@ -0,0 +1,33 @@
+# To Do List
+## FIM 3 functionality
+- *NDV for elevation rasters:* setting NDV for rasters is exposed to the user in preprocess_raster.py. This should be set to value in src/shared_variables.py of elev_raster_ndv. This prevents issues. Make sure all references to NDV in entire pipeline are consistent.
+- *Pipeline Management:* the user pipeline should be strengthened. Preprocess, hydrofab production, inundation, evaluation steps should be consolidated. For example: there are many preprocessing steps now. Also every major pipeline step should have meta-data associated with it in some sort of pandas friendly json format. Example: every fim_run/gms_run should have a meta-data json detailing the model used, parameters, date ran, user, and the inherited data from the preprocessing steps of the pipeline.
+- *NLD Lines:* nld lines are first being rasterized then burned.
+    - the python script burn_levees could be avoided using gdal_calc and converting nld elevations to meters in preprocessing
+    - if conversion to meters is done in preprocessing. It maybe possible to burn the nld elevations directly into the dem with gdal_rasterize all in one step
+    - this could help both FIM 3 and GMS
+
+## Eval
+- *Testing Architecture & Modularity:* The test case functionality requires more modularity. There should be a clear set of tools that abstract away the model or test cases used. 
+    - User tools should allow any predicted raster and any benchmark raster to be compared given proper encoding.
+        - This function should be command line oriented and allow for a list or file to be evaluated using multiple CPUs
+    - Eval tools for FIM 3, Lisflood, GMS, etc can then wrap around these core eval tools
+    - Aggregation of metrics can be improved. The function create_metrics_metrics.py is spaghetti (15 indentations).See consolidate_metrics for modular tooling. Needs support for other test case formats.
+    - Try statements should only include very specific lines. Having very large try blocks (see run_test_case.py creates alot of issues in debugging nested modules).
+    - Any code put in if main == name part of a module is not exposed to the user if the main function is imported elsewhere in python. These code segments should modularized and grouped into functionality in main function.
+- *NoForecastFound Error:* inundation.py throws a NoForecastFound exception when no matches are found between forecast and hydrotable. In levelpaths with no forecast this avoids creating an inundation raster for that area. This changes the spatial extent of GMS to exclude some areas that FR/MS would write as FNs (see for an example 13020102).
+- *Evaluation extents:* The evaluation extents between FR, MS, GMS are not consistent. While this does have a small effect on CSI, MCC should be immune to it and generally shows a similar trend
+
+
+## GMS 
+- *NWM Divergences:* Levelpath derivation doesn't handle divergences (eg 12020002).
+    - it shortens the effective length of levelpaths thus reducing the rating curve height of the most upstream catchment
+    - it also creates more levelpaths and likely increases computational time
+- *Update rem.py*: rem.py underwent some change. Update to latest src/rem.py from src/gms/rem.py
+- *Unique Level Path and Hydro IDs:* unique identifiers for HydroIDs with GMS. Maybe FIMID, then branch ID, then HydroID.
+- *convenience wrapper for gms_run...sh:* Make a convenience wrapper for gms_run_unit.sh and gms_run_branch.sh. Be mindful of the two different processing loads and expose two different job numbers to the user.
+    - *Deny Listing for Units:* The files in the deny list for units is executed at the end of gms/run_by_unit.sh. This requires files used in run_by_branch.sh to be left while not necessary left behind. This should be moved later in the process possibly once the convenience wrapper is made.
+- *Update clip_vectors_to_wbd.py:* Clipping way too many vectors for GMS purposes. This creates extra processing and storage requirements.
+- *Small level paths just in the WBD area:* Small levelpaths just outside the wbd are being filtered out filter_catchments_and_add_attributes.py and lead to an exception. These cause known non-zero exit codes. To reduce non-zero exit codes to not known issues these levelpaths need to be removed in derive_level_paths.py. See 11140104 - 7057000003, 12020007 - 1475000002, 11110104 - 7756000488, 11110104 - 7756000505, 11090204 - 5128000164
+- *Stream order filtering:* You would likely gain computational time if you filtered out lower stream orders in the NWM for GMS input (creating a nwm_streams_gms.gpkg). By doing this you would need to mosaic with FR streams at the lower removed stream orders. GMS likely does very little at stream orders 1s and 2s and maybe even 3s and 4s.*
+- *Levees to run_by_unit*: burning in levees at the branch scale is likely less efficient than at the unit scale. Consider moving those two modules to the run_by_unit script for gms
diff --git a/src/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
index 524360915..b9210945d 100755
--- a/src/make_stages_and_catchlist.py
+++ b/src/make_stages_and_catchlist.py
@@ -23,6 +23,16 @@ def make_stages_and_catchlist(flows_filename, catchments_filename, stages_filena
     stages_max = stages_max + stages_interval
     stages = np.round(np.arange(stages_min,stages_max,stages_interval),4)
 
+    hydroIDs = flows['HydroID'].tolist()
+    len_of_hydroIDs = len(hydroIDs)
+    slopes = flows['S0'].tolist()
+    lengthkm = flows['LengthKm'].tolist()
+
+    try:
+        areasqkm = catchments['areasqkm'].tolist()
+    except KeyError:
+        areasqkm = catchments['geometry'].area/ 10**6
+        
     with open(stages_filename,'w') as f:
         f.write("Stage\n")
         for stage in stages:
diff --git a/src/preprocess_rasters.py b/src/preprocess_rasters.py
index 635e1f227..bf82d91fa 100755
--- a/src/preprocess_rasters.py
+++ b/src/preprocess_rasters.py
@@ -4,6 +4,8 @@
 import sys
 sys.path.append('/foss_fim/src')
 from multiprocessing import Pool
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
 import argparse
 from utils.reproject_dem import reproject_dem
 from utils.shared_functions import update_raster_profile
@@ -17,7 +19,9 @@
     parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
     parser.add_argument('-nodata','--nodata-val', help='DEM nodata value', required=False,type=float,default=-9999.0)
     parser.add_argument('-block','--blocksize', help='DEM blocksize', required=False,type=int,default=512)
-    parser.add_argument('-keep','--keep-intermediate', help='keep intermediate files', required=False,type=bool,default=True)
+    parser.add_argument('-keep','--keep-intermediate', help='keep intermediate files', required=False,default=True,action='store_true')
+    parser.add_argument('-op','--overwrite-projected', help='Overwrite projected files', required=False,default=False,action='store_true')
+    parser.add_argument('-om','--overwrite-meters', help='Overwrite meter files', required=False,default=False,action='store_true')
 
     args = vars(parser.parse_args())
 
@@ -26,6 +30,8 @@
     nodata_val         = args['nodata_val']
     blocksize          = args['blocksize']
     keep_intermediate  = args['keep_intermediate']
+    overwrite_projected  = args['overwrite_projected']
+    overwrite_meters  = args['overwrite_meters']
 
     reproject_procs_list = []
 
@@ -33,23 +39,41 @@
         raster_dir = os.path.join(dem_dir,huc)
         elev_cm = os.path.join(raster_dir, 'elev_cm.tif')
         elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
-        reproject_procs_list.append([raster_dir, elev_cm, elev_cm_proj, PREP_PROJECTION_CM])
+        reproject_procs_list.append([raster_dir, elev_cm, elev_cm_proj, PREP_PROJECTION_CM,overwrite_projected])
 
     # Multiprocess reprojection
     with Pool(processes=number_of_jobs) as pool:
         pool.map(reproject_dem, reproject_procs_list)
 
     profile_procs_list = []
-
+    
     for huc in os.listdir(dem_dir):
         elev_m_tif = os.path.join(dem_dir,huc, 'elev_m.tif')
-        if not os.path.exists(elev_m_tif):
-            raster_dir = os.path.join(dem_dir,huc)
-            elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
-            elev_m = os.path.join(raster_dir, 'elev_m.tif')
-            profile_procs_list.append([elev_cm_proj, elev_m,PREP_PROJECTION,nodata_val,blocksize,keep_intermediate])
+        raster_dir = os.path.join(dem_dir,huc)
+        elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
+        elev_m = os.path.join(raster_dir, 'elev_m.tif')
+        profile_procs_list.append((huc,[elev_cm_proj, elev_m,PREP_PROJECTION,nodata_val,blocksize,keep_intermediate,overwrite_meters]))
 
+    executor = ThreadPoolExecutor(max_workers=number_of_jobs)
+    
+    executor_generator = { 
+                          executor.submit(update_raster_profile,p):h for h,p in profile_procs_list
+                         }
+    
+    for future in tqdm(as_completed(executor_generator),
+                       total=len(profile_procs_list),
+                       desc="Processing rasters with {} workers ".format(number_of_jobs)
+                      ):
+        hucCode = executor_generator[future]
+        #try:
+        future.result()
+        #except Exception as exc:
+        #    print('{},{},{}'.format(hucCode,exc.__class__.__name__,exc))
+    
+    # power down pool
+    executor.shutdown(wait=True)
+    
     # Multiprocess update profile
-    with Pool(processes=2) as pool:
+ #   with Pool(processes=2) as pool:
         # TODO read in windows becasue gdal rasters are massive
-        pool.map(update_raster_profile, profile_procs_list)
+#        pool.map(update_raster_profile, profile_procs_list)
diff --git a/src/utils/reproject_dem.py b/src/utils/reproject_dem.py
index dba8f65de..056602b53 100755
--- a/src/utils/reproject_dem.py
+++ b/src/utils/reproject_dem.py
@@ -16,9 +16,15 @@ def reproject_dem(args):
     elev_cm               = args[1]
     elev_cm_proj          = args[2]
     reprojection          = args[3]
+    overwrite             = args[4]
 
-    if os.path.exists(elev_cm_proj):
+    if os.path.exists(elev_cm_proj) & overwrite:
         os.remove(elev_cm_proj)
+    elif not os.path.exists(elev_cm_proj):
+        pass
+    else:
+        print(f"Skipping {elev_cm_proj}. Raster already exists. Use overwrite option.")
+        return
 
     shutil.copy(elev_cm, elev_cm_proj)
 
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index b01533c7d..01904a222 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -2,6 +2,7 @@
 
 import os
 from os.path import splitext
+import fiona
 import rasterio
 import numpy as np
 from rasterio.warp import calculate_default_transform, reproject, Resampling
@@ -89,6 +90,17 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
     gdf.to_file(multilayer_wbd_geopackage, layer=layer_name,driver='GPKG',index=False)
 
 
+def get_fossid_from_huc8(huc8_id,foss_id_attribute='fossid',
+                         hucs=os.path.join(os.environ['inputDataDir'],'wbd','WBD_National.gpkg'),
+                         hucs_layerName=None):
+
+    hucs = fiona.open(hucs,'r',layer=hucs_layerName)
+
+    for huc in hucs:
+        if huc['properties']['HUC8'] == huc8_id:
+            return(huc['properties'][foss_id_attribute])
+
+
 def update_raster_profile(args):
 
     elev_cm_filename   = args[0]
@@ -97,6 +109,15 @@ def update_raster_profile(args):
     nodata_val         = args[3]
     blocksize          = args[4]
     keep_intermediate  = args[5]
+    overwrite  = args[6]
+
+    if os.path.exists(elev_m_filename) & overwrite:
+        os.remove(elev_m_filename)
+    elif not os.path.exists(elev_m_filename):
+        pass
+    else:
+        print(f"Skipping {elev_m_filename}. Use overwrite option.")
+        return
 
     if isinstance(blocksize, int):
         pass
@@ -111,29 +132,38 @@ def update_raster_profile(args):
 
     # Update nodata value and convert from cm to meters
     dem_cm = rasterio.open(elev_cm_filename)
-
+    
     no_data = dem_cm.nodata
-    data = dem_cm.read(1)
-
-    dem_m = np.where(data == int(no_data), nodata_val, (data/100).astype(rasterio.float32))
-
-    del data
-
+    
     dem_m_profile = dem_cm.profile.copy()
-
     dem_m_profile.update(driver='GTiff',tiled=True,nodata=nodata_val,
                          blockxsize=blocksize, blockysize=blocksize,
                          dtype='float32',crs=projection,compress='lzw',interleave='band')
 
-    with rasterio.open(elev_m_filename, "w", **dem_m_profile, BIGTIFF='YES') as dest:
-        dest.write(dem_m, indexes = 1)
+    dest = rasterio.open(elev_m_filename, "w", **dem_m_profile, BIGTIFF='YES')
 
-    if keep_intermediate == False:
-        os.remove(elev_cm_filename)
+    for idx,window in dem_cm.block_windows(1):
+        data = dem_cm.read(1,window=window)
+
+        # wrote out output of this line as the same variable.
+        data = np.where(data == int(no_data), nodata_val, (data/100).astype(rasterio.float32))
+
+    # removed this line to avoid having two array copies of data. Kills memory usage
+    #del data
+
+        dest.write(data, indexes = 1, window=window)
 
-    del dem_m
-    dem_cm.close()
 
+    # not necessary
+    #del dem_m
+
+    dem_cm.close()
+    dest.close()
+    
+    if keep_intermediate == False:
+        os.remove(elev_cm_filename)
+    
+    return(elev_m_filename)
 
 '''
 This function isn't currently used but is the preferred method for
@@ -209,7 +239,6 @@ def reproject_raster(input_raster_name,reprojection,blocksize=None,reprojected_r
                 del dst
         del src
 
-
 def mem_profile(func):
     def wrapper(*args, **kwargs):
         if (os.environ.get('mem') == "1"):
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 44bc27f8b..85130e7cf 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -23,6 +23,13 @@
 NHD_VECTOR_EXTRACTION_PREFIX = 'NHDPLUS_H_'
 NHD_VECTOR_EXTRACTION_SUFFIX = '_HU4_GDB.zip'
 
+## added here bc prefixs and suffixes just aren't necessary in python3
+nhd_raster_url_template = "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_{}_HU4_RASTER.7z"
+nhd_vector_url_template = "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_{}_HU4_GDB.zip"
+
+# --- Values ---- #
+elev_raster_ndv = -9999.0
+
 # -- Field Names -- #
 FIM_ID = 'fimid'
 
@@ -46,7 +53,7 @@
 os.environ['nhdplus_vectors_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors')
 os.environ['nwm_dir'] = os.path.join(os.environ.get('input_dir'),'nwm_hydrofabric')
 os.environ['wbd_dir'] = os.path.join(os.environ.get('input_dir'),'wbd')
-os.environ['ahps_dir'] = os.path.join(os.environ.get('input_dir'),'ahps_sites')
+os.environ['ahps_dir'] = os.path.join(os.environ.get('input_dir'),'ahp_sites')
 os.environ['nhdplus_aggregate_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors_aggregate')
 
 # File Paths
diff --git a/tests/aggregate_metrics.py b/tests/aggregate_metrics.py
new file mode 100644
index 000000000..d8a462d5b
--- /dev/null
+++ b/tests/aggregate_metrics.py
@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import csv
+    
+import argparse
+
+TEST_CASES_DIR = r'/data/test_cases_new/'
+# TEMP = r'/data/temp'
+
+# Search through all previous_versions in test_cases
+from utils.shared_functions import compute_stats_from_contingency_table
+
+def create_master_metrics_csv():
+            
+    # Construct header
+    metrics_to_write = ['true_negatives_count',
+                        'false_negatives_count',
+                        'true_positives_count',
+                        'false_positives_count',
+                        'contingency_tot_count',
+                        'cell_area_m2',
+                        'TP_area_km2',
+                        'FP_area_km2',
+                        'TN_area_km2',
+                        'FN_area_km2',
+                        'contingency_tot_area_km2',
+                        'predPositive_area_km2',
+                        'predNegative_area_km2',
+                        'obsPositive_area_km2',
+                        'obsNegative_area_km2',
+                        'positiveDiff_area_km2',
+                        'CSI',
+                        'FAR',
+                        'TPR',
+                        'TNR',
+                        'PPV',
+                        'NPV',
+                        'ACC',
+                        'Bal_ACC',
+                        'MCC',
+                        'EQUITABLE_THREAT_SCORE',
+                        'PREVALENCE',
+                        'BIAS',
+                        'F1_SCORE',
+                        'TP_perc',
+                        'FP_perc',
+                        'TN_perc',
+                        'FN_perc',
+                        'predPositive_perc',
+                        'predNegative_perc',
+                        'obsPositive_perc',
+                        'obsNegative_perc',
+                        'positiveDiff_perc',
+                        'masked_count',
+                        'masked_perc',
+                        'masked_area_km2'
+                        ]
+    
+    additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
+    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source']]
+    
+    for benchmark_type in ['ble', 'ahps']:
+        
+        if benchmark_type == 'ble':
+        
+            test_cases = r'/data/test_cases'
+            test_cases_list = os.listdir(test_cases)
+            # AHPS test_ids
+            versions_to_aggregate = ['fim_1_0_0', 'fim_2_3_3', 'fim_3_0_0_3_fr_c']
+                            
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
+                    
+                    for magnitude in ['100yr', '500yr']:
+                        for version in versions_to_aggregate:
+                            version_dir = os.path.join(previous_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+
+                            if os.path.exists(magnitude_dir):
+                                
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f:
+                                        flow = 'NA'
+                                        nws_lid = "NA"
+                                        benchmark_source = 'ble'
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        if os.path.exists(full_json_path):
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            
+                                            list_to_write.append(sub_list_to_append)
+                                                                                    
+                except ValueError:
+                    pass
+                
+        if benchmark_type == 'ahps':
+    
+            test_cases = r'/data/test_cases_ahps_testing'
+            test_cases_list = os.listdir(test_cases)
+            # AHPS test_ids
+            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021', 
+                                     'fim_2_x_ms_nws_1_21_2021', 'fim_2_x_ms_usgs_1_21_2021',
+                                     'fim_3_0_0_3_ms_c_nws_1_21_2021', 'fim_3_0_0_3_ms_c_usgs_1_21_2021',
+                                     'ms_xwalk_fill_missing_cal_nws', 'ms_xwalk_fill_missing_cal_usgs']
+            
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
+                    
+                    for magnitude in ['action', 'minor', 'moderate', 'major']:
+                        for version in versions_to_aggregate:
+                            
+                            if 'nws' in version:
+                                benchmark_source = 'ahps_nws'
+                            if 'usgs' in version:
+                                benchmark_source = 'ahps_usgs'
+                            
+                            version_dir = os.path.join(previous_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+                            
+                            if os.path.exists(magnitude_dir):
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f and 'total_area' not in f:
+                                        nws_lid = f[:5]
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        flow = ''
+                                        if os.path.exists(full_json_path):
+                                            # Get flow used to map.
+                                            if 'usgs' in version:
+                                                parent_dir = 'usgs_1_21_2021'
+                                            if 'nws' in version:
+                                                parent_dir = 'nws_1_21_2021'
+                                                
+                                            flow_file = os.path.join(test_cases, parent_dir, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                                            if os.path.exists(flow_file):
+                                                with open(flow_file, newline='') as csv_file:
+                                                    reader = csv.reader(csv_file)
+                                                    next(reader)
+                                                    for row in reader:
+                                                        flow = row[1]
+                                                    if nws_lid == 'mcc01':
+                                                        print(flow)
+                                            
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            list_to_write.append(sub_list_to_append)
+                                        
+                except ValueError:
+                    pass
+        
+    with open(output_csv, 'w', newline='') as csvfile:
+        csv_writer = csv.writer(csvfile)
+        csv_writer.writerows(list_to_write)
+
+
+
+def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfolder=""):
+
+    # Read hucs into list.
+    if hucs != "":
+        huc_list = [line.rstrip('\n') for line in open(hucs)]
+
+    else:
+        huc_list = None
+
+    if config == "DEV":
+        config_version = "development_versions"
+    elif config == "PREV":
+        config_version = "previous_versions"
+
+    # Make directory to store output aggregates.
+    if special_string != "":
+        special_string = "_" + special_string
+    aggregate_output_dir = os.path.join(outfolder, 'aggregate_metrics', branch + '_aggregate_metrics' + special_string)
+    if not os.path.exists(aggregate_output_dir):
+        os.makedirs(aggregate_output_dir)
+
+    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
+
+    for magnitude in ['100yr', '500yr', 'action', 'minor', 'moderate', 'major']:
+        huc_path_list = [['huc', 'path']]
+        true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = 0, 0, 0, 0, 0, 0
+        
+        for test_case in test_cases_dir_list:
+
+            if test_case not in ['other', 'validation_data_ble', 'validation_data_legacy', 'validation_data_ahps']:
+                branch_results_dir = os.path.join(TEST_CASES_DIR, test_case, 'performance_archive', config_version, branch)
+
+                huc = test_case.split('_')[0]
+                # Check that the huc is in the list of hucs to aggregate.
+                if huc_list != None and huc not in huc_list:
+                    continue
+
+                stats_json_path = os.path.join(branch_results_dir, magnitude, 'total_area_stats.json')
+
+                # If there is a stats json for the test case and branch name, use it when aggregating stats.
+                if os.path.exists(stats_json_path):
+                    json_dict = json.load(open(stats_json_path))
+
+                    true_positives += json_dict['true_positives_count']
+                    true_negatives += json_dict['true_negatives_count']
+                    false_positives += json_dict['false_positives_count']
+                    false_negatives += json_dict['false_negatives_count']
+                    masked_count += json_dict['masked_count']
+
+                    cell_area = json_dict['cell_area_m2']
+
+                    huc_path_list.append([huc, stats_json_path])
+                
+                    
+            if cell_area == 0:
+                continue
+            
+            # Pass all sums to shared function to calculate metrics.
+            stats_dict = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=cell_area, masked_count=masked_count)
+
+            list_to_write = [['metric', 'value']]  # Initialize header.
+
+            for stat in stats_dict:
+                list_to_write.append([stat, stats_dict[stat]])
+                
+            # Map path to output directory for aggregate metrics.
+            output_file = os.path.join(aggregate_output_dir, branch + '_aggregate_metrics_' + magnitude + special_string + '.csv')
+
+        if cell_area != 0:
+            with open(output_file, 'w', newline='') as csvfile:
+                csv_writer = csv.writer(csvfile)
+                csv_writer.writerows(list_to_write)
+                csv_writer.writerow([])
+                csv_writer.writerows(huc_path_list)
+    
+            print()
+            print("Finished aggregating for the '" + magnitude + "' magnitude. Aggregated metrics over " + str(len(huc_path_list)-1) + " test cases.")
+            print()
+            print("Results are at: " + output_file)
+            print()
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Aggregates a metric or metrics for multiple HUC8s.')
+    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=False)
+    parser.add_argument('-b','--branch',help='Name of branch to check all test_cases for and to aggregate.',required=True)
+    parser.add_argument('-u','--hucs',help='HUC8s to restrict the aggregation.',required=False, default="")
+    parser.add_argument('-s','--special_string',help='Special string to add to outputs.',required=False, default="")
+    parser.add_argument('-f','--outfolder',help='output folder',required=True,type=str)
+
+    args = vars(parser.parse_args())
+
+    aggregate_metrics(**args)
diff --git a/tests/cache_metrics.py b/tests/cache_metrics.py
new file mode 100644
index 000000000..3f601cc5c
--- /dev/null
+++ b/tests/cache_metrics.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import traceback
+
+from run_test_case import run_alpha_test
+from multiprocessing import Pool
+
+TEST_CASES_DIR = r'/data/test_cases_new/'  #TODO remove "_new"
+PREVIOUS_FIM_DIR = r'/data/previous_fim'
+OUTPUTS_DIR = r'/data/outputs'
+
+
+def process_alpha_test(args):
+    
+    fim_run_dir = args[0]
+    version = args[1]
+    test_id = args[2]
+    magnitude = args[3]
+    archive_results = args[4]
+    
+    mask_type = 'huc'    
+    
+    if archive_results == False:
+        compare_to_previous = True
+    else:
+        compare_to_previous = False
+
+    try:
+        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type)
+    except Exception:
+        traceback.print_exc()
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
+    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
+    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
+    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
+    parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
+    parser.add_argument('-b','--benchmark-category',help='Options include ble or ahps. Defaults to process both.',required=False, default=None)
+        
+    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
+    
+    args = vars(parser.parse_args())
+
+    config = args['config']
+    fim_version = args['fim_version']
+    job_number = int(args['job_number'])
+    special_string = args['special_string']
+    benchmark_category = args['benchmark_category']
+    
+    if fim_version != "all":
+        previous_fim_list = [fim_version]
+    else:
+        previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)    
+    
+    if config == 'PREV':
+        archive_results = True
+    elif config == 'DEV':
+        archive_results = False
+    else:
+        print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
+    
+    benchmark_category_list = []
+    
+    if benchmark_category == None:
+        for d in test_cases_dir_list:
+            if 'test_cases' in d:
+                benchmark_category_list.append(d.replace('_test_cases', ''))
+    else:
+        benchmark_category_list = [benchmark_category]
+
+    procs_list = []
+    for bench_cat in benchmark_category_list:
+        bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
+        
+        bench_cat_test_case_list = os.listdir(bench_cat_test_case_dir)
+    
+        for test_id in bench_cat_test_case_list:
+            if 'validation' and 'other' not in test_id:
+                            
+                current_huc = test_id.split('_')[0]
+                if test_id.split('_')[1] in bench_cat:
+                
+                    for version in previous_fim_list:
+                        
+                        if config == 'DEV':
+                            fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
+                        elif config == 'PREV':
+                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
+                                                
+                        if not os.path.exists(fim_run_dir):
+                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  # For previous versions of HAND computed at HUC6 scale
+                        
+                        if os.path.exists(fim_run_dir):
+                            if special_string != "":
+                                version = version + '_' + special_string
+                            
+                            if 'ble' in test_id:
+                                magnitude = ['100yr', '500yr']
+                            elif 'usgs' or 'nws' in test_id:
+                                magnitude = ['action', 'minor', 'moderate', 'major']
+                            else:
+                                continue
+                        
+                            print("Adding " + test_id + " to list of test_ids to process...")
+                            if job_number > 1:
+                                procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results])
+                            else:                            
+                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results])
+
+    if job_number > 1:
+        pool = Pool(job_number)
+        pool.map(process_alpha_test, procs_list)
\ No newline at end of file
diff --git a/tests/run_test_case.py b/tests/run_test_case.py
new file mode 100755
index 000000000..5beac4f16
--- /dev/null
+++ b/tests/run_test_case.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import shutil
+import argparse
+
+from utils.shared_functions import compute_contingency_stats_from_rasters
+from utils.shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
+from inundation import inundate
+
+
+def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True):
+    
+    benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
+    current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
+    
+    # Construct paths to development test results if not existent.
+    if archive_results:
+        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'official_versions', version)
+    else:
+        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'testing_versions', version)
+
+    # Delete the entire directory if it already exists.
+    if os.path.exists(version_test_case_dir_parent):
+        if overwrite == True:
+            shutil.rmtree(version_test_case_dir_parent)
+        else:
+            print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
+            return
+        
+    os.mkdir(version_test_case_dir_parent)
+
+    print("Running the alpha test for test_id: " + test_id + ", " + version + "...")
+    stats_modes_list = ['total_area']
+
+    fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
+    assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
+
+    # Create paths to fim_run outputs for use in inundate().
+    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
+    if not os.path.exists(rem):
+        rem = os.path.join(fim_run_parent, 'rem_clipped_zeroed_masked.tif')
+    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+    if not os.path.exists(catchments):
+        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_clipped_addedAttributes.tif')
+    if mask_type == 'huc':
+        catchment_poly = ''
+    else:
+        catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
+        
+    # Map necessary inputs for inundation().
+    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+
+    # Create list of shapefile paths to use as exclusion areas.
+    zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
+    mask_dict = {'levees': 
+                    {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
+                     'buffer': None,
+                     'operation': 'exclude'
+                     },
+                'waterbodies':
+                    {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
+                     'buffer': None,
+                     'operation': 'exclude',
+                     },
+                }
+            
+    if inclusion_area != '':
+        inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
+        mask_dict.update({inclusion_area_name: {'path': inclusion_area,
+                                                'buffer': int(inclusion_area_buffer),
+                                                'operation': 'include'}})
+        # Append the concatenated inclusion_area_name and buffer.
+        if inclusion_area_buffer == None:
+            inclusion_area_buffer = 0
+        stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm') 
+
+    # Check if magnitude is list of magnitudes or single value.
+    magnitude_list = magnitude
+    if type(magnitude_list) != list:
+        magnitude_list = [magnitude_list]
+
+    # Get path to validation_data_{benchmark} directory and huc_dir.
+    validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
+    
+    for magnitude in magnitude_list:
+        version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
+        if not os.path.exists(version_test_case_dir):
+            os.mkdir(version_test_case_dir)
+    
+        # Construct path to validation raster and forecast file.
+        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+            benchmark_raster_path_list, forecast_list = [], []
+            lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
+            lid_list, inundation_raster_list, extent_file_list = [], [], []
+            
+            for lid in lid_dir_list:
+                lid_dir = os.path.join(validation_data_path, current_huc, lid)
+                benchmark_raster_path_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_depth_' + magnitude + '.tif'))  # TEMP
+                forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
+                lid_list.append(lid)
+                inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
+                extent_file_list.append(os.path.join(lid_dir, lid + '_extent.shp'))
+
+        else:
+            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_depth_' + magnitude + '.tif')
+            benchmark_raster_path_list = [benchmark_raster_file]
+            forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
+            forecast_list = [forecast_path]
+            inundation_raster_list = [os.path.join(version_test_case_dir, 'inundation_extent.tif')]
+            
+        for index in range(0, len(benchmark_raster_path_list)):
+            benchmark_raster_path = benchmark_raster_path_list[index]
+            forecast = forecast_list[index]
+            inundation_raster = inundation_raster_list[index]
+            
+            # Only need to define ahps_lid and ahps_extent_file for AHPS_BENCHMARK_CATEGORIES.
+            if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                ahps_lid = lid_list[index]
+                ahps_extent_file = extent_file_list[index]
+                mask_dict.update({ahps_lid:
+                    {'path': ahps_extent_file,
+                     'buffer': None,
+                     'operation': 'include'}
+                        })
+        
+                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_extent_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
+                    continue
+            else:  # If not in AHPS_BENCHMARK_CATEGORIES.
+                if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
+                    continue
+    
+            # Run inundate.
+            print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
+            try:
+                inundate(
+                         rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                         subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
+                         depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
+                        )
+            
+                print("-----> Inundation mapping complete.")
+                predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
+        
+                # Define outputs for agreement_raster, stats_json, and stats_csv.
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+                else:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+         
+                compute_contingency_stats_from_rasters(predicted_raster_path,
+                                                       benchmark_raster_path,
+                                                       agreement_raster,
+                                                       stats_csv=stats_csv,
+                                                       stats_json=stats_json,
+                                                       mask_values=[],
+                                                       stats_modes_list=stats_modes_list,
+                                                       test_id=test_id,
+                                                       mask_dict=mask_dict,
+                                                       )
+        
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    del mask_dict[ahps_lid]
+                
+                print(" ")
+                print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
+                print(" ")
+            except Exception as e:
+                print(e)      
+        
+        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+            # -- Delete temp files -- #
+            # List all files in the output directory.
+            output_file_list = os.listdir(version_test_case_dir)
+            for output_file in output_file_list:
+                if "total_area" in output_file:
+                    full_output_file_path = os.path.join(version_test_case_dir, output_file)
+                    os.remove(full_output_file_path)
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
+    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
+    parser.add_argument('-b', '--version',help='The name of the working version in which features are being tested',required=True,default="")
+    parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
+    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
+    parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
+    parser.add_argument('-c', '--compare-to-previous', help='Compare to previous versions of HAND.', required=False,action='store_true')
+    parser.add_argument('-a', '--archive-results', help='Automatically copy results to the "previous_version" archive for test_id. For admin use only.', required=False,action='store_true')
+    parser.add_argument('-i', '--inclusion-area', help='Path to shapefile. Contingency metrics will be produced from pixels inside of shapefile extent.', required=False, default="")
+    parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
+    parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
+    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False)
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    valid_test_id_list = os.listdir(TEST_CASES_DIR)
+
+    exit_flag = False  # Default to False.
+    print()
+
+    # Ensure test_id is valid.
+#    if args['test_id'] not in valid_test_id_list:
+#        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
+#        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
+#        for test_id in valid_test_id_list:
+#          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
+#              print(CYAN_BOLD + test_id + ENDC)
+#        print()
+#        exit_flag = True
+
+    # Ensure fim_run_dir exists.
+    if not os.path.exists(os.path.join(os.environ['outputDataDir'], args['fim_run_dir'])):
+        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
+        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
+        print()
+        exit_flag = True
+    
+    # Ensure inclusion_area path exists.
+    if args['inclusion_area'] != "" and not os.path.exists(args['inclusion_area']):
+        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area (-i) " + CYAN_BOLD + args['inclusion_area'] + WHITE_BOLD + " could not be located." + ENDC)
+        exit_flag = True
+        
+    try:
+        inclusion_buffer = int(args['inclusion_area_buffer'])
+    except ValueError:
+        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area_buffer (-ib) " + CYAN_BOLD + args['inclusion_area_buffer'] + WHITE_BOLD + " is not a round number." + ENDC)
+
+    if args['magnitude'] == '':
+        if 'ble' in args['test_id'].split('_'):
+            args['magnitude'] = ['100yr', '500yr']
+        elif 'ahps' in args['test_id'].split('_'):
+            args['magnitude'] = ['action', 'minor', 'moderate', 'major']
+        else:
+            print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
+            exit_flag = True     
+            
+    if exit_flag:
+        print()
+        sys.exit()
+
+    else:
+        run_alpha_test(**args)
diff --git a/tests/synthesize_test_cases.py b/tests/synthesize_test_cases.py
new file mode 100644
index 000000000..1fdb0a4dc
--- /dev/null
+++ b/tests/synthesize_test_cases.py
@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+from multiprocessing import Pool
+import json
+import csv
+
+from run_test_case import run_alpha_test
+from utils.shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
+
+
+def create_master_metrics_csv(master_metrics_csv_output):
+            
+    # Construct header
+    metrics_to_write = ['true_negatives_count',
+                        'false_negatives_count',
+                        'true_positives_count',
+                        'false_positives_count',
+                        'contingency_tot_count',
+                        'cell_area_m2',
+                        'TP_area_km2',
+                        'FP_area_km2',
+                        'TN_area_km2',
+                        'FN_area_km2',
+                        'contingency_tot_area_km2',
+                        'predPositive_area_km2',
+                        'predNegative_area_km2',
+                        'obsPositive_area_km2',
+                        'obsNegative_area_km2',
+                        'positiveDiff_area_km2',
+                        'CSI',
+                        'FAR',
+                        'TPR',
+                        'TNR',
+                        'PPV',
+                        'NPV',
+                        'ACC',
+                        'Bal_ACC',
+                        'MCC',
+                        'EQUITABLE_THREAT_SCORE',
+                        'PREVALENCE',
+                        'BIAS',
+                        'F1_SCORE',
+                        'TP_perc',
+                        'FP_perc',
+                        'TN_perc',
+                        'FN_perc',
+                        'predPositive_perc',
+                        'predNegative_perc',
+                        'obsPositive_perc',
+                        'obsNegative_perc',
+                        'positiveDiff_perc',
+                        'masked_count',
+                        'masked_perc',
+                        'masked_area_km2'
+                        ]
+    
+    additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
+    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
+    
+    versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
+    
+    for benchmark_source in ['ble', 'nws', 'usgs']:
+        
+        benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
+        
+        if benchmark_source == 'ble':
+            test_cases_list = os.listdir(benchmark_test_case_dir)
+                            
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
+                    
+                    for magnitude in ['100yr', '500yr']:
+                        for version in versions_to_aggregate:
+                            if '_fr' in version:
+                                extent_config = 'FR'
+                            elif '_ms' in version:
+                                extent_config = 'MS'
+                            else:
+                                extent_config = 'FR'
+                            if "_c" in version and version.split('_c')[1] == "":
+                                calibrated = "yes"
+                            else:
+                                calibrated = "no"
+                            version_dir = os.path.join(official_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+
+                            if os.path.exists(magnitude_dir):
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f:
+                                        flow = 'NA'
+                                        nws_lid = "NA"
+                                        benchmark_source = 'ble'
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        if os.path.exists(full_json_path):
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            sub_list_to_append.append(extent_config)
+                                            sub_list_to_append.append(calibrated)
+                                            
+                                            list_to_write.append(sub_list_to_append)
+                except ValueError:
+                    pass
+                
+        if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
+            test_cases_list = os.listdir(benchmark_test_case_dir)
+
+            for test_case in test_cases_list:
+                try:
+                    int(test_case.split('_')[0])
+                    
+                    huc = test_case.split('_')[0]
+                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
+                    
+                    for magnitude in ['action', 'minor', 'moderate', 'major']:
+                        for version in versions_to_aggregate:
+                            if '_fr' in version:
+                                extent_config = 'FR'
+                            elif '_ms' in version:
+                                extent_config = 'MS'
+                            else:
+                                extent_config = 'FR'
+                            if "_c" in version and version.split('_c')[1] == "":
+                                calibrated = "yes"
+                            else:
+                                calibrated = "no"
+                                
+                            version_dir = os.path.join(official_versions, version)
+                            magnitude_dir = os.path.join(version_dir, magnitude)
+                            if os.path.exists(magnitude_dir):
+                                magnitude_dir_list = os.listdir(magnitude_dir)
+                                for f in magnitude_dir_list:
+                                    if '.json' in f and 'total_area' not in f:
+                                        nws_lid = f[:5]
+                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
+                                        full_json_path = os.path.join(magnitude_dir, f)
+                                        flow = ''
+                                        if os.path.exists(full_json_path):
+                                            
+                                            # Get flow used to map.                                                
+                                            flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                                            if os.path.exists(flow_file):
+                                                with open(flow_file, newline='') as csv_file:
+                                                    reader = csv.reader(csv_file)
+                                                    next(reader)
+                                                    for row in reader:
+                                                        flow = row[1]
+                                                    if nws_lid == 'mcc01':
+                                                        print(flow)
+                                            
+                                            stats_dict = json.load(open(full_json_path))
+                                            for metric in metrics_to_write:
+                                                sub_list_to_append.append(stats_dict[metric])
+                                            sub_list_to_append.append(full_json_path)
+                                            sub_list_to_append.append(flow)
+                                            sub_list_to_append.append(benchmark_source)
+                                            sub_list_to_append.append(extent_config)
+                                            sub_list_to_append.append(calibrated)
+                                            
+                                            list_to_write.append(sub_list_to_append)
+                except ValueError:
+                    pass
+        
+    with open(master_metrics_csv_output, 'w', newline='') as csvfile:
+        csv_writer = csv.writer(csvfile)
+        csv_writer.writerows(list_to_write)
+
+
+def process_alpha_test(args):
+    
+    fim_run_dir = args[0]
+    version = args[1]
+    test_id = args[2]
+    magnitude = args[3]
+    archive_results = args[4]
+    overwrite = args[5]
+    
+    mask_type = 'huc'    
+    
+    if archive_results == False:
+        compare_to_previous = True
+    else:
+        compare_to_previous = False
+
+    try:
+        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type, overwrite=overwrite)
+    except Exception as e:
+        print(e)
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
+    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
+    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
+    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
+    parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
+    parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
+    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
+    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=True)
+    
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+    config = args['config']
+    fim_version = args['fim_version']
+    job_number = int(args['job_number'])
+    special_string = args['special_string']
+    benchmark_category = args['benchmark_category']
+    overwrite = args['overwrite']
+    master_metrics_csv = args['master_metrics_csv']
+    
+    if overwrite:
+        if input("Are you sure you want to overwrite metrics? y/n: ") == "n":
+            quit
+        
+    # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
+    if fim_version != "all":
+        previous_fim_list = [fim_version]
+    else:
+        if config == 'PREV':
+            previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)
+        elif config == 'DEV':
+            previous_fim_list = os.listdir(OUTPUTS_DIR)
+    
+    # Define whether or not to archive metrics in "official_versions" or "testing_versions" for each test_id.
+    if config == 'PREV':
+        archive_results = True
+    elif config == 'DEV':
+        archive_results = False
+    else:
+        print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
+     
+    # List all available benchmark categories and test_cases.
+    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
+    benchmark_category_list = []
+    if benchmark_category == "all":
+        for d in test_cases_dir_list:
+            if 'test_cases' in d:
+                benchmark_category_list.append(d.replace('_test_cases', ''))
+    else:
+        benchmark_category_list = [benchmark_category]
+        
+    # Loop through benchmark categories.
+    procs_list = []
+    for bench_cat in benchmark_category_list:
+        
+        # Map path to appropriate test_cases folder and list test_ids into bench_cat_id_list.
+        bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
+        bench_cat_id_list = os.listdir(bench_cat_test_case_dir)
+    
+        # Loop through test_ids in bench_cat_id_list.
+        for test_id in bench_cat_id_list:
+            if 'validation' and 'other' not in test_id:
+                current_huc = test_id.split('_')[0]
+                if test_id.split('_')[1] in bench_cat:
+                
+                    # Loop through versions.
+                    for version in previous_fim_list:
+                        if config == 'DEV':
+                            fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
+                        elif config == 'PREV':
+                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
+                                                
+                        # For previous versions of HAND computed at HUC6 scale
+                        if not os.path.exists(fim_run_dir):
+                            if config == 'DEV':
+                                fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
+                            elif config == 'PREV':
+                                fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  
+                        
+                        if os.path.exists(fim_run_dir):
+                            
+                            # If a user supplies a specia_string (-s), then add it to the end of the created dirs.
+                            if special_string != "":
+                                version = version + '_' + special_string
+                            
+                            # Define the magnitude lists to use, depending on test_id.
+                            if 'ble' in test_id:
+                                magnitude = ['100yr', '500yr']
+                            elif 'usgs' or 'nws' in test_id:
+                                magnitude = ['action', 'minor', 'moderate', 'major']
+                            else:
+                                continue
+                        
+                            # Either add to list to multiprocess or process serially, depending on user specification.
+                            if job_number > 1:
+                                procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
+                            else:                            
+                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
+
+    # Multiprocess alpha test runs.
+    if job_number > 1:
+        pool = Pool(job_number)
+        pool.map(process_alpha_test, procs_list)
+    
+    # Do aggregate_metrics.
+    print("Creating master metrics CSV...")
+    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv)
+    
\ No newline at end of file
diff --git a/tests/utils/shared_functions.py b/tests/utils/shared_functions.py
new file mode 100644
index 000000000..d36c22814
--- /dev/null
+++ b/tests/utils/shared_functions.py
@@ -0,0 +1,676 @@
+#!/usr/bin/env python3
+
+import os
+import json
+import csv
+import rasterio
+import pandas as pd
+from utils.shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
+                                    ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
+
+def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
+
+    difference_dict = {}
+
+    # Compare stats_csv to previous_version_stats_file
+    stats_dict_to_test = json.load(open(stats_json_to_test))
+    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
+
+    for stat, value in stats_dict_to_test.items():
+        previous_version_value = previous_version_stats_dict[stat]
+        stat_value_diff = value - previous_version_value
+        difference_dict.update({stat + '_diff': stat_value_diff})
+
+    return difference_dict
+
+
+def compute_contingency_stats_from_rasters(predicted_raster_path, benchmark_raster_path, agreement_raster=None, stats_csv=None, stats_json=None, mask_values=None, stats_modes_list=['total_area'], test_id='', mask_dict={}):
+    """
+    This function contains FIM-specific logic to prepare raster datasets for use in the generic get_contingency_table_from_binary_rasters() function.
+    This function also calls the generic compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
+
+    Args:
+        predicted_raster_path (str): The path to the predicted, or modeled, FIM extent raster.
+        benchmark_raster_path (str): The path to the benchmark, or truth, FIM extent raster.
+        agreement_raster (str): Optional. An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive, 3: True Positive.
+        stats_csv (str): Optional. Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
+        stats_json (str): Optional. Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary in other processes.
+
+    Returns:
+        stats_dictionary (dict): A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and statistic values are the values.
+    """
+
+    # Get cell size of benchmark raster.
+    raster = rasterio.open(predicted_raster_path)
+    t = raster.transform
+    cell_x = t[0]
+    cell_y = t[4]
+    cell_area = abs(cell_x*cell_y)
+
+    # Get contingency table from two rasters.
+    contingency_table_dictionary = get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster, mask_values=mask_values, mask_dict=mask_dict)
+
+    stats_dictionary = {}
+
+    for stats_mode in contingency_table_dictionary:
+        true_negatives = contingency_table_dictionary[stats_mode]['true_negatives']
+        false_negatives = contingency_table_dictionary[stats_mode]['false_negatives']
+        false_positives = contingency_table_dictionary[stats_mode]['false_positives']
+        true_positives = contingency_table_dictionary[stats_mode]['true_positives']
+        masked_count = contingency_table_dictionary[stats_mode]['masked_count']
+        file_handle = contingency_table_dictionary[stats_mode]['file_handle']
+
+        # Produce statistics from continency table and assign to dictionary. cell_area argument optional (defaults to None).
+        mode_stats_dictionary = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area, masked_count)
+
+        # Write the mode_stats_dictionary to the stats_csv.
+        if stats_csv != None:
+            stats_csv = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.csv')
+            df = pd.DataFrame.from_dict(mode_stats_dictionary, orient="index", columns=['value'])
+            df.to_csv(stats_csv)
+
+        # Write the mode_stats_dictionary to the stats_json.
+        if stats_json != None:
+            stats_json = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.json')
+            with open(stats_json, "w") as outfile:
+                json.dump(mode_stats_dictionary, outfile)
+
+        stats_dictionary.update({stats_mode: mode_stats_dictionary})
+
+    return stats_dictionary
+
+
+def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
+    """
+    This function searches multiple directories and locates previously produced performance statistics.
+
+    Args:
+        archive_to_check (str): The directory path to search.
+        magnitude (str): Because a benchmark dataset may have multiple magnitudes, this argument defines
+                               which magnitude is to be used when searching for previous statistics.
+    Returns:
+        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain and magnitude.
+                                  {version: {agreement_raster: agreement_raster_path, stats_csv: stats_csv_path, stats_json: stats_json_path}}
+                                  *Will only add the paths to files that exist.
+
+    """
+
+    archive_dictionary = {}
+
+    # List through previous version and check for available stats and maps. If available, add to dictionary.
+    available_versions_list = os.listdir(archive_to_check)
+
+    if len(available_versions_list) == 0:
+        print("Cannot compare with -c flag because there are no data in the previous_versions directory.")
+        return
+
+    for version in available_versions_list:
+        version_magnitude_dir = os.path.join(archive_to_check, version, magnitude)
+        stats_json = os.path.join(version_magnitude_dir, stats_mode + '_stats.json')
+
+        if os.path.exists(stats_json):
+            archive_dictionary.update({version: {'stats_json': stats_json}})
+
+    return archive_dictionary
+
+
+#def compare_to_previous(benchmark_category, test_id, stats_modes_list, magnitude, version, test_version_dictionary, version_test_case_dir):
+#    text_block = []
+#    # Compare to previous stats files that are available.
+#    archive_to_check = os.path.join(TEST_CASES_DIR, benchmark_category + 'test_cases', test_id, 'official_versions')
+#    for stats_mode in stats_modes_list:
+#        archive_dictionary = profile_test_case_archive(archive_to_check, magnitude, stats_mode)
+#
+#        if archive_dictionary == {}:
+#            break
+#
+#        # Create header for section.
+#        header = [stats_mode]
+#        for previous_version, paths in archive_dictionary.items():
+#            header.append(previous_version)
+#        header.append(version)
+#        text_block.append(header)
+#
+#        # Loop through stats in PRINTWORTHY_STATS for left.
+#        for stat in PRINTWORTHY_STATS:
+#            stat_line = [stat]
+#            for previous_version, paths in archive_dictionary.items():
+#                # Load stats for previous version.
+#                previous_version_stats_json_path = paths['stats_json']
+#                if os.path.exists(previous_version_stats_json_path):
+#                    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
+#
+#                    # Append stat for the version to state_line.
+#                    stat_line.append(previous_version_stats_dict[stat])
+#
+#
+#            # Append stat for the current version to stat_line.
+#            stat_line.append(test_version_dictionary[stats_mode][stat])
+#
+#            text_block.append(stat_line)
+#
+#        text_block.append([" "])
+#
+#    regression_report_csv = os.path.join(version_test_case_dir, 'stats_summary.csv')
+#    with open(regression_report_csv, 'w', newline='') as csvfile:
+#        csv_writer = csv.writer(csvfile)
+#        csv_writer.writerows(text_block)
+#
+#    print()
+#    print("--------------------------------------------------------------------------------------------------")
+#    
+#    stats_mode = stats_modes_list[0]
+#    try:
+#        last_version_index = text_block[0].index('dev_latest')
+#    except ValueError:
+#        try:
+#            last_version_index = text_block[0].index('fim_2_3_3')
+#        except ValueError:
+#            try:
+#                last_version_index = text_block[0].index('fim_1_0_0')
+#            except ValueError:
+#                print(TRED_BOLD + "Warning: " + ENDC + "Cannot compare " + version + " to a previous version because no authoritative versions were found in previous_versions directory. Future version of run_test_case may allow for comparisons between dev versions.")
+#                print()
+#                continue
+#
+#    
+#
+#    for line in text_block:
+#        first_item = line[0]
+#        if first_item in stats_modes_list:
+#            current_version_index = line.index(version)
+#            if first_item != stats_mode:  # Update the stats_mode and print a separator.
+#                print()
+#                print()
+#                print("--------------------------------------------------------------------------------------------------")
+#            print()
+#            stats_mode = first_item
+#            print(CYAN_BOLD + current_huc + ": " + magnitude.upper(), ENDC)
+#            print(CYAN_BOLD + stats_mode.upper().replace('_', ' ') + " METRICS" + ENDC)
+#            print()
+#
+#            color = WHITE_BOLD
+#            metric_name = '      '.center(len(max(PRINTWORTHY_STATS, key=len)))
+#            percent_change_header = '% CHG'
+#            difference_header = 'DIFF'
+#            current_version_header = line[current_version_index].upper()
+#            last_version_header = line[last_version_index].upper()
+#            # Print Header.
+#            print(color + metric_name + "      " + percent_change_header.center((7)) + "       " + difference_header.center((15))  + "    " + current_version_header.center(18) + " " + last_version_header.center(18), ENDC)
+#        # Format and print stat row.
+#        elif first_item in PRINTWORTHY_STATS:
+#            stat_name = first_item.upper().center(len(max(PRINTWORTHY_STATS, key=len))).replace('_', ' ')
+#            current_version = round((line[current_version_index]), 3)
+#            last_version = round((line[last_version_index]) + 0.000, 3)
+#            difference = round(current_version - last_version, 3)
+#            if difference > 0:
+#                symbol = '+'
+#                if first_item in GO_UP_STATS:
+#                    color = TGREEN_BOLD
+#                elif first_item in GO_DOWN_STATS:
+#                    color = TRED_BOLD
+#                else:
+#                    color = TWHITE
+#            if difference < 0:
+#                symbol = '-'
+#                if first_item in GO_UP_STATS:
+#                    color = TRED_BOLD
+#                elif first_item in GO_DOWN_STATS:
+#                    color = TGREEN_BOLD
+#                else:
+#                    color = TWHITE
+#
+#            if difference == 0 :
+#                symbol, color = '+', TGREEN
+#            percent_change = round((difference / last_version)*100,2)
+#
+#            print(WHITE_BOLD + stat_name + ENDC + "     " + color + (symbol + " {:5.2f}".format(abs(percent_change)) + " %").rjust(len(percent_change_header)), ENDC + "    " + color + ("{:12.3f}".format((difference))).rjust(len(difference_header)), ENDC + "    " + "{:15.3f}".format(current_version).rjust(len(current_version_header)) + "   " + "{:15.3f}".format(last_version).rjust(len(last_version_header)) + "  ")
+#
+#    print()
+#    print()
+#    print()
+#    print("--------------------------------------------------------------------------------------------------")
+#    print()
+#
+
+
+def compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=None, masked_count=None):
+    """
+    This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.
+    Much of the calculations below were taken from older Python files. This is evident in the inconsistent use of case.
+    
+    Args:
+        true_negatives (int): The true negatives from a contingency table.
+        false_negatives (int): The false negatives from a contingency table.
+        false_positives (int): The false positives from a contingency table.
+        true_positives (int): The true positives from a contingency table.
+        cell_area (float or None): This optional argument allows for area-based statistics to be calculated, in the case that
+                                   contingency table metrics were derived from areal analysis.
+    
+    Returns:
+        stats_dictionary (dict): A dictionary of statistics. Statistic names are keys and statistic values are the values.
+                                 Refer to dictionary definition in bottom of function for statistic names.
+    
+    """
+    
+    import numpy as np
+    
+    total_population = true_negatives + false_negatives + false_positives + true_positives
+        
+    # Basic stats.
+#    Percent_correct = ((true_positives + true_negatives) / total_population) * 100
+#    pod             = true_positives / (true_positives + false_negatives)
+    
+    try:
+        FAR = false_positives / (true_positives + false_positives)
+    except ZeroDivisionError:
+        FAR = "NA"
+        
+    try:
+        CSI = true_positives / (true_positives + false_positives + false_negatives)
+    except ZeroDivisionError:
+        CSI = "NA"
+    
+    try:
+        BIAS = (true_positives + false_positives) / (true_positives + false_negatives)
+    except ZeroDivisionError:
+        BIAS = "NA"
+    
+    # Compute equitable threat score (ETS) / Gilbert Score. 
+    try:
+        a_ref = ((true_positives + false_positives)*(true_positives + false_negatives)) / total_population
+        EQUITABLE_THREAT_SCORE = (true_positives - a_ref) / (true_positives - a_ref + false_positives + false_negatives)
+    except ZeroDivisionError:
+        EQUITABLE_THREAT_SCORE = "NA"
+
+    if total_population == 0:
+        TP_perc, FP_perc, TN_perc, FN_perc = "NA", "NA", "NA", "NA"
+    else:
+        TP_perc = (true_positives / total_population) * 100
+        FP_perc = (false_positives / total_population) * 100
+        TN_perc = (true_negatives / total_population) * 100
+        FN_perc = (false_negatives / total_population) * 100
+    
+    predPositive = true_positives + false_positives
+    predNegative = true_negatives + false_negatives
+    obsPositive = true_positives + false_negatives
+    obsNegative = true_negatives + false_positives
+    
+    TP = float(true_positives)
+    TN = float(true_negatives)
+    FN = float(false_negatives)
+    FP = float(false_positives)
+    try:
+        MCC = (TP*TN - FP*FN)/ np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))
+    except ZeroDivisionError:
+        MCC = "NA"
+    
+    if masked_count != None:
+        total_pop_and_mask_pop = total_population + masked_count
+        if total_pop_and_mask_pop == 0:
+            masked_perc = "NA"
+        else:
+            masked_perc = (masked_count / total_pop_and_mask_pop) * 100
+    else:
+        masked_perc = None
+    
+    # This checks if a cell_area has been provided, thus making areal calculations possible.
+    sq_km_converter = 1000000
+        
+    if cell_area != None:
+        TP_area = (true_positives * cell_area) / sq_km_converter
+        FP_area = (false_positives * cell_area) / sq_km_converter
+        TN_area = (true_negatives * cell_area) / sq_km_converter
+        FN_area = (false_negatives * cell_area) / sq_km_converter
+        area = (total_population * cell_area) / sq_km_converter
+        
+        predPositive_area = (predPositive * cell_area) / sq_km_converter
+        predNegative_area = (predNegative * cell_area) / sq_km_converter
+        obsPositive_area =  (obsPositive * cell_area) / sq_km_converter
+        obsNegative_area =  (obsNegative * cell_area) / sq_km_converter
+        positiveDiff_area = predPositive_area - obsPositive_area
+        
+        if masked_count != None:
+            masked_area = (masked_count * cell_area) / sq_km_converter
+        else:
+            masked_area = None
+
+    # If no cell_area is provided, then the contingeny tables are likely not derived from areal analysis.
+    else:
+        TP_area = None
+        FP_area = None
+        TN_area = None
+        FN_area = None
+        area = None
+        
+        predPositive_area = None
+        predNegative_area = None
+        obsPositive_area =  None
+        obsNegative_area =  None
+        positiveDiff_area = None
+        MCC = None
+        
+    if total_population == 0:
+        predPositive_perc, predNegative_perc, obsPositive_perc, obsNegative_perc , positiveDiff_perc = "NA", "NA", "NA", "NA", "NA"
+    else:
+        predPositive_perc = (predPositive / total_population) * 100
+        predNegative_perc = (predNegative / total_population) * 100
+        obsPositive_perc = (obsPositive / total_population) * 100
+        obsNegative_perc = (obsNegative / total_population) * 100
+        
+        positiveDiff_perc = predPositive_perc - obsPositive_perc
+        
+    if total_population == 0:
+        prevalence = "NA"
+    else:
+        prevalence = (true_positives + false_negatives) / total_population
+    
+    try:
+        PPV = true_positives / predPositive
+    except ZeroDivisionError:
+        PPV = "NA"
+    
+    try:
+        NPV = true_negatives / predNegative
+    except ZeroDivisionError:
+        NPV = "NA"
+    
+    try:
+        TNR = true_negatives / obsNegative
+    except ZeroDivisionError:
+        TNR = "NA"
+        
+    try:
+        TPR = true_positives / obsPositive
+        
+    except ZeroDivisionError:
+        TPR = "NA"
+
+    try:
+        Bal_ACC = np.mean([TPR,TNR])
+    except TypeError:
+        Bal_ACC = "NA"
+    
+    if total_population == 0:
+        ACC = "NA"
+    else:
+        ACC = (true_positives + true_negatives) / total_population
+    
+    try:
+        F1_score = (2*true_positives) / (2*true_positives + false_positives + false_negatives)
+    except ZeroDivisionError:
+        F1_score = "NA"
+
+    stats_dictionary = {'true_negatives_count': int(true_negatives),
+                        'false_negatives_count': int(false_negatives),
+                        'true_positives_count': int(true_positives),
+                        'false_positives_count': int(false_positives),
+                        'contingency_tot_count': int(total_population),
+                        'cell_area_m2': cell_area,
+                        
+                        'TP_area_km2': TP_area,
+                        'FP_area_km2': FP_area,
+                        'TN_area_km2': TN_area,
+                        'FN_area_km2': FN_area,
+
+                        'contingency_tot_area_km2': area,
+                        'predPositive_area_km2': predPositive_area,
+                        'predNegative_area_km2': predNegative_area,
+                        'obsPositive_area_km2': obsPositive_area,
+                        'obsNegative_area_km2': obsNegative_area,
+                        'positiveDiff_area_km2': positiveDiff_area,
+
+                        'CSI': CSI,
+                        'FAR': FAR,
+                        'TPR': TPR,                        
+                        'TNR': TNR,                        
+                        
+                        'PPV': PPV,
+                        'NPV': NPV,
+                        'ACC': ACC,
+                        'Bal_ACC': Bal_ACC,
+                        'MCC': MCC,
+                        'EQUITABLE_THREAT_SCORE': EQUITABLE_THREAT_SCORE, 
+                        'PREVALENCE': prevalence,
+                        'BIAS': BIAS,
+                        'F1_SCORE': F1_score,
+
+                        'TP_perc': TP_perc,
+                        'FP_perc': FP_perc,
+                        'TN_perc': TN_perc,
+                        'FN_perc': FN_perc,
+                        'predPositive_perc': predPositive_perc,
+                        'predNegative_perc': predNegative_perc,
+                        'obsPositive_perc': obsPositive_perc,
+                        'obsNegative_perc': obsNegative_perc,
+                        'positiveDiff_perc': positiveDiff_perc,
+  
+                        'masked_count': int(masked_count),
+                        'masked_perc': masked_perc,
+                        'masked_area_km2': masked_area,
+                        
+                        }
+
+    return stats_dictionary
+
+
+def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster=None, mask_values=None, mask_dict={}):
+    """
+    Produces contingency table from 2 rasters and returns it. Also exports an agreement raster classified as:
+        0: True Negatives
+        1: False Negative
+        2: False Positive
+        3: True Positive
+        
+    Args:
+        benchmark_raster_path (str): Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
+        predicted_raster_path (str): Path to the predicted raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
+    
+    Returns:
+        contingency_table_dictionary (dict): A Python dictionary of a contingency table. Key/value pair formatted as:
+                                            {true_negatives: int, false_negatives: int, false_positives: int, true_positives: int}
+    
+    """
+    from rasterio.warp import reproject, Resampling
+    import rasterio
+    import numpy as np
+    import os
+    import rasterio.mask
+    import geopandas as gpd
+    from shapely.geometry import box
+            
+    print("-----> Evaluating performance across the total area...")
+    # Load rasters.
+    benchmark_src = rasterio.open(benchmark_raster_path)
+    predicted_src = rasterio.open(predicted_raster_path)
+    predicted_array = predicted_src.read(1)
+    
+    benchmark_array_original = benchmark_src.read(1)
+    
+    if benchmark_array_original.shape != predicted_array.shape:
+        benchmark_array = np.empty(predicted_array.shape, dtype=np.int8)
+        
+        reproject(benchmark_array_original, 
+              destination = benchmark_array,
+              src_transform = benchmark_src.transform, 
+              src_crs = benchmark_src.crs,
+              src_nodata = benchmark_src.nodata,
+              dst_transform = predicted_src.transform, 
+              dst_crs = predicted_src.crs,
+              dst_nodata = benchmark_src.nodata,
+              dst_resolution = predicted_src.res,
+              resampling = Resampling.nearest)
+    
+    predicted_array_raw = predicted_src.read(1)
+    
+    # Align the benchmark domain to the modeled domain.
+    benchmark_array = np.where(predicted_array==predicted_src.nodata, 10, benchmark_array)
+            
+    # Ensure zeros and ones for binary comparison. Assume that positive values mean flooding and 0 or negative values mean dry. 
+    predicted_array = np.where(predicted_array==predicted_src.nodata, 10, predicted_array)  # Reclassify NoData to 10
+    predicted_array = np.where(predicted_array<0, 0, predicted_array)
+    predicted_array = np.where(predicted_array>0, 1, predicted_array)
+    
+    benchmark_array = np.where(benchmark_array==benchmark_src.nodata, 10, benchmark_array)  # Reclassify NoData to 10        
+
+    agreement_array = np.add(benchmark_array, 2*predicted_array)
+    agreement_array = np.where(agreement_array>4, 10, agreement_array)
+    
+    del benchmark_src, benchmark_array, predicted_array, predicted_array_raw
+
+    # Loop through exclusion masks and mask the agreement_array.
+    if mask_dict != {}:
+        for poly_layer in mask_dict:
+            
+            operation = mask_dict[poly_layer]['operation']
+            
+            if operation == 'exclude':
+                
+                poly_path = mask_dict[poly_layer]['path']
+                buffer_val = mask_dict[poly_layer]['buffer']
+                
+                reference = predicted_src
+                            
+                bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
+                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).  
+                poly_all = gpd.read_file(poly_path, bbox = bounding_box)
+                
+                # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
+                if poly_all.empty:
+                    continue
+                
+                print("-----> Masking at " + poly_layer + "...")
+                #Project layer to reference crs.
+                poly_all_proj = poly_all.to_crs(reference.crs)
+                # check if there are any lakes within our reference raster extent.
+                if poly_all_proj.empty: 
+                    #If no features within reference raster extent, create a zero array of same shape as reference raster.
+                    poly_mask = np.zeros(reference.shape)
+                else:
+                    #Check if a buffer value is passed to function.
+                    if buffer_val is None:
+                        #If features are present and no buffer is passed, assign geometry to variable.
+                        geometry = poly_all_proj.geometry
+                    else:
+                        #If  features are present and a buffer is passed, assign buffered geometry to variable.
+                        geometry = poly_all_proj.buffer(buffer_val)
+                        
+                    #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
+                    in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
+                    #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
+                    poly_mask = np.where(in_poly == True, 1,0)
+
+                    # Perform mask.
+                    masked_agreement_array = np.where(poly_mask == 1, 4, agreement_array)
+                    
+                    # Get rid of masked values outside of the modeled domain.
+                    agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
+                    
+    contingency_table_dictionary = {}  # Initialize empty dictionary.
+    
+    # Only write the agreement raster if user-specified.
+    if agreement_raster != None:
+        with rasterio.Env():
+            profile = predicted_src.profile
+            profile.update(nodata=10)
+            with rasterio.open(agreement_raster, 'w', **profile) as dst:
+                dst.write(agreement_array, 1)
+         
+        # Write legend text file
+        legend_txt = os.path.join(os.path.split(agreement_raster)[0], 'read_me.txt')
+        
+        from datetime import datetime
+        
+        now = datetime.now()
+        current_time = now.strftime("%m/%d/%Y %H:%M:%S")
+                
+        with open(legend_txt, 'w') as f:
+            f.write("%s\n" % '0: True Negative')
+            f.write("%s\n" % '1: False Negative')
+            f.write("%s\n" % '2: False Positive')
+            f.write("%s\n" % '3: True Positive')
+            f.write("%s\n" % '4: Masked area (excluded from contingency table analysis). Mask layers: {mask_dict}'.format(mask_dict=mask_dict))
+            f.write("%s\n" % 'Results produced at: {current_time}'.format(current_time=current_time))
+                          
+    # Store summed pixel counts in dictionary.
+    contingency_table_dictionary.update({'total_area':{'true_negatives': int((agreement_array == 0).sum()),
+                                                      'false_negatives': int((agreement_array == 1).sum()),
+                                                      'false_positives': int((agreement_array == 2).sum()),
+                                                      'true_positives': int((agreement_array == 3).sum()),
+                                                      'masked_count': int((agreement_array == 4).sum()),
+                                                      'file_handle': 'total_area'
+                                                     
+                                                      }})                               
+        
+    # After agreement_array is masked with default mask layers, check for inclusion masks in mask_dict.
+    if mask_dict != {}:
+        for poly_layer in mask_dict:
+            
+            operation = mask_dict[poly_layer]['operation']
+            
+            if operation == 'include':
+                poly_path = mask_dict[poly_layer]['path']
+                buffer_val = mask_dict[poly_layer]['buffer']
+
+                reference = predicted_src
+                            
+                bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
+                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).  
+                poly_all = gpd.read_file(poly_path, bbox = bounding_box)
+                
+                # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
+                if poly_all.empty:
+                    continue
+                
+                print("-----> Evaluating performance at " + poly_layer + "...")
+                #Project layer to reference crs.
+                poly_all_proj = poly_all.to_crs(reference.crs)
+                # check if there are any lakes within our reference raster extent.
+                if poly_all_proj.empty: 
+                    #If no features within reference raster extent, create a zero array of same shape as reference raster.
+                    poly_mask = np.zeros(reference.shape)
+                else:
+                    #Check if a buffer value is passed to function.
+                    if buffer_val is None:
+                        #If features are present and no buffer is passed, assign geometry to variable.
+                        geometry = poly_all_proj.geometry
+                    else:
+                        #If  features are present and a buffer is passed, assign buffered geometry to variable.
+                        geometry = poly_all_proj.buffer(buffer_val)
+                        
+                    #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
+                    in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
+                    #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
+                    poly_mask = np.where(in_poly == True, 1, 0)
+
+                    # Perform mask.
+                    masked_agreement_array = np.where(poly_mask == 0, 4, agreement_array)  # Changed to poly_mask == 0
+                    
+                    # Get rid of masked values outside of the modeled domain.
+                    temp_agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
+                    
+                    if buffer_val == None:  # The buffer used is added to filename, and 0 is easier to read than None.
+                        buffer_val = 0
+                    
+                    poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
+                    
+                    # Write the layer_agreement_raster.
+                    layer_agreement_raster = os.path.join(os.path.split(agreement_raster)[0], poly_handle + '_agreement.tif')
+                    with rasterio.Env():
+                        profile = predicted_src.profile
+                        profile.update(nodata=10)
+                        with rasterio.open(layer_agreement_raster, 'w', **profile) as dst:
+                            dst.write(temp_agreement_array, 1)
+                    
+
+                    # Store summed pixel counts in dictionary.
+                    contingency_table_dictionary.update({poly_handle:{'true_negatives': int((temp_agreement_array == 0).sum()),
+                                                                     'false_negatives': int((temp_agreement_array == 1).sum()),
+                                                                     'false_positives': int((temp_agreement_array == 2).sum()),
+                                                                     'true_positives': int((temp_agreement_array == 3).sum()),
+                                                                     'masked_count': int((temp_agreement_array == 4).sum()),
+                                                                     'file_handle': poly_handle
+                                                                      }})
+
+    return contingency_table_dictionary
+    
diff --git a/tools/compare_gms_srcs_to_fr.py b/tools/compare_gms_srcs_to_fr.py
new file mode 100755
index 000000000..539c1ba65
--- /dev/null
+++ b/tools/compare_gms_srcs_to_fr.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import numpy as np
+from glob import iglob, glob
+from os.path import join, exists
+import argparse
+import matplotlib.pyplot as plt
+from matplotlib.gridspec import GridSpec
+
+
+def join_srcs(fr_data_dir,ms_data_dir,gms_data_dir,outputs_dir=None):
+
+    gms_srcs_file_list_generator = iglob( join(gms_data_dir,"*","branches","*","src_full_*.csv"), recursive=True)
+    
+    """
+    gms_srcs = None
+    for i in gms_srcs_file_list_generator:
+        current_gms_src = pd.read_csv(i)
+
+        if gms_srcs is None:
+            gms_srcs = current_gms_src
+        else:
+            gms_srcs = pd.concat((gms_srcs,current_gms_src),ignore_index=True)
+    """
+
+    gms_processed_data_file_name = join(outputs_dir,'gms_mean_srcs_ble.h5')
+    ms_processed_data_file_name = join(outputs_dir,'ms_mean_srcs_ble.h5')
+    fr_processed_data_file_name = join(outputs_dir,'fr_mean_srcs_ble.h5')
+
+    if exists(gms_processed_data_file_name):
+        gms_srcs = pd.read_hdf(gms_processed_data_file_name)
+    else:
+        # GMS srcs
+        print("Loading GMS SRCs")
+        gms_srcs = pd.concat( [ pd.read_csv(i) for i in gms_srcs_file_list_generator] , ignore_index=True)
+        gms_srcs.loc[:,'Number of Cells'] = gms_srcs.loc[:,'Number of Cells'].astype(int)
+        gms_srcs = gms_srcs.loc[:,['Stage','Discharge (m3s-1)','Volume (m3)','BedArea (m2)','Number of Cells']].groupby('Stage').agg('mean').reset_index(drop=False)
+
+   
+    if exists(fr_processed_data_file_name):
+        fr_srcs = pd.read_hdf(fr_processed_data_file_name)
+    else:
+        # fr srcsi
+        fr_srcs_file_name = glob( join(fr_data_dir,'*',"src_full_crosswalked.csv"))[0]
+        fr_srcs = pd.read_csv( fr_srcs_file_name )
+
+        fr_srcs_file_names = glob( join(fr_data_dir,'*',"src_full_crosswalked.csv"))
+        print("Loading FR SRCs")
+        fr_srcs = pd.concat( [ pd.read_csv(i) for i in fr_srcs_file_names ] , ignore_index = True )
+        # average out values by stage
+        fr_srcs = fr_srcs.loc[:,['Stage','Discharge (m3s-1)','Volume (m3)','BedArea (m2)','Number of Cells']].groupby('Stage').agg('mean').reset_index(drop=False)
+
+    # gms
+    if exists(ms_processed_data_file_name):
+        ms_srcs = pd.read_hdf(ms_processed_data_file_name)
+    else:
+        ms_src_file_names = glob( join(ms_data_dir,'*',"src_full_crosswalked.csv"))
+        print("Loading MS SRCs")
+        ms_srcs = pd.concat( [ pd.read_csv(i) for i in ms_src_file_names ] , ignore_index = True )
+
+        ms_srcs = pd.concat( (ms_srcs,fr_srcs), ignore_index=True )
+    
+        ms_srcs = ms_srcs.loc[:,['Stage','Discharge (m3s-1)','Volume (m3)','BedArea (m2)','Number of Cells']].groupby('Stage').agg('mean').reset_index(drop=False)
+    
+    # save files
+    if not exists(gms_processed_data_file_name):
+        gms_srcs.to_hdf(gms_processed_data_file_name,key='data')
+
+    if not exists(ms_processed_data_file_name):
+        ms_srcs.to_hdf(ms_processed_data_file_name,key='data')
+
+    if not exists(fr_processed_data_file_name):
+        fr_srcs.to_hdf(fr_processed_data_file_name,key='data')
+
+    # figure
+    fig = plt.figure(constrained_layout=True,figsize=(6,7))
+    gs = GridSpec(3,2,figure=fig)
+
+    # SRC plots
+    ax1 = fig.add_subplot(gs[0,:])
+    ax1.plot(gms_srcs.loc[:,'Discharge (m3s-1)'],gms_srcs.loc[:,'Stage'],label='GMS Mean')
+    ax1.plot(ms_srcs.loc[:,'Discharge (m3s-1)'],ms_srcs.loc[:,'Stage'],label='MS Mean')
+    ax1.plot(fr_srcs.loc[:,'Discharge (m3s-1)'],fr_srcs.loc[:,'Stage'],label="FR Mean")
+    ax1.set_xlabel(r'Discharge ($m^{3}/s$)'+'\n'+'(a)')
+    ax1.set_ylabel('Stage, m')
+    ax1.ticklabel_format(axis='x',style='sci',useMathText=True,scilimits=(0,4))
+    ax1.legend(loc='center',bbox_to_anchor=(0.5,1.25,0,0),ncol=3)
+
+    #ax1.legend()
+    #if outputs_dir is not None:
+    #    ax1.savefig( join(outputs_dir,"rating_curve.jpg") )
+    
+    # Volume plots
+    ax2 = fig.add_subplot(gs[1,0])
+    ax2.plot(gms_srcs.loc[:,'Stage'],gms_srcs.loc[:,'Volume (m3)'],label='GMS Mean')
+    ax2.plot(ms_srcs.loc[:,'Stage'],ms_srcs.loc[:,'Volume (m3)'],label='MS Mean')
+    ax2.plot(fr_srcs.loc[:,'Stage'],fr_srcs.loc[:,'Volume (m3)'],label="FR Mean")
+    ax2.set_xlabel('Stage, m'+'\n'+'(b)')
+    ax2.set_ylabel(r'Volume, $m^3$')
+    #ax2.legend()
+    #if outputs_dir is not None:
+    #    plt.savefig( join(outputs_dir,"volume.jpg") )
+    
+    # Bed Area plots
+    ax3 = fig.add_subplot(gs[1,1])
+    ax3.plot(gms_srcs.loc[:,'Stage'],gms_srcs.loc[:,'BedArea (m2)'],label='GMS Mean')
+    ax3.plot(ms_srcs.loc[:,'Stage'],ms_srcs.loc[:,'BedArea (m2)'],label='MS Mean')
+    ax3.plot(fr_srcs.loc[:,'Stage'],fr_srcs.loc[:,'BedArea (m2)'],label="FR Mean")
+    ax3.set_xlabel('Stage, m'+'\n'+'(c)')
+    ax3.set_ylabel(r'Bed Area, $m^2$')
+    #ax3.legend()
+    #if outputs_dir is not None:
+    #    plt.savefig( join(outputs_dir,"bedarea.jpg") )
+    
+    # Volume and Bed Area plots
+    ax4 = fig.add_subplot(gs[2,0])
+    ax4.plot(gms_srcs.loc[:,'Stage'],
+             np.power(gms_srcs.loc[:,'Volume (m3)'],5/3) * np.power(gms_srcs.loc[:,'BedArea (m2)'],-2/3),
+             label='GMS Mean')
+    ax4.plot(ms_srcs.loc[:,'Stage'],
+             np.power(ms_srcs.loc[:,'Volume (m3)'],5/3) * np.power(ms_srcs.loc[:,'BedArea (m2)'],-2/3),
+             label='MS Mean')
+    ax4.plot(fr_srcs.loc[:,'Stage'],
+             np.power(fr_srcs.loc[:,'Volume (m3)'],5/3) * np.power(fr_srcs.loc[:,'BedArea (m2)'],-2/3),
+             label="FR Mean")
+    ax4.set_xlabel('Stage, m'+'\n'+'(d)')
+    ax4.set_ylabel(r'$\dfrac{(Volume,\ m^3)^{5/3} }{(Bed\ Area,\ m^2) ^{2/3} }$')
+    #ax4.legend()
+    #if outputs_dir is not None:
+    #    plt.savefig( join(outputs_dir,"volume_and_bedarea.jpg") )
+
+    # Number of Cells plots
+    ax5 = fig.add_subplot(gs[2,1])
+    ax5.plot(gms_srcs.loc[:,'Stage'],gms_srcs.loc[:,'Number of Cells'],label='GMS Mean')
+    ax5.plot(ms_srcs.loc[:,'Stage'],ms_srcs.loc[:,'Number of Cells'],label='MS Mean')
+    ax5.plot(fr_srcs.loc[:,'Stage'],fr_srcs.loc[:,'Number of Cells'],label="FR Mean")
+    ax5.set_xlabel('Stage, m'+'\n'+'(e)')
+    ax5.set_ylabel('Pixels')
+    ax5.ticklabel_format(axis='y',style='sci',useMathText=True,scilimits=(0,3))
+    
+    if outputs_dir is not None:
+        fig.savefig( join(outputs_dir,"rating_curve_comparison.jpg"), dpi=300)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Comparing SRCs')
+    parser.add_argument('-f','--fr-data-dir', help='Directory with FR data', required=True)
+    parser.add_argument('-g','--gms-data-dir', help='Directory with GMS data', required=True)
+    parser.add_argument('-m','--ms-data-dir', help='Directory with MS data', required=True)
+    parser.add_argument('-o','--outputs-dir', help='Directory for outputs data', required=False,default=None)
+
+    args = vars(parser.parse_args())
+
+    join_srcs(**args)
diff --git a/tools/consolidate_metrics.py b/tools/consolidate_metrics.py
new file mode 100755
index 000000000..20f084ef8
--- /dev/null
+++ b/tools/consolidate_metrics.py
@@ -0,0 +1,322 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import os
+import re
+import json
+import numpy as np
+from glob import glob
+from tqdm import tqdm
+import argparse
+from collections import defaultdict
+from shared_variables import TEST_CASES_DIR,\
+                             PREVIOUS_FIM_DIR,\
+                             OUTPUTS_DIR,\
+                             INPUTS_DIR,\
+                             AHPS_BENCHMARK_CATEGORIES
+from tools_shared_functions import csi,far,tpr,mcc
+
+# set display options
+pd.set_option('display.max_rows', None)
+pd.set_option('display.max_columns', None)
+
+def Consolidate_metrics( benchmarks=['all'],versions=['all'],
+                         zones=['total_area'],matching_hucs_only=True,
+                         metrics_output_csv=None,
+                         impute_missing_ms=False
+                       ):
+
+    """ Consolidates metrics into single dataframe """
+
+    if isinstance(benchmarks,str):
+        benchmarks = [benchmarks]
+    elif isinstance(benchmarks,list):
+        pass
+    else:
+        benchmarks = list(benchmarks)
+    
+    if isinstance(versions,str):
+        versions = [versions]
+    elif isinstance(benchmarks,list):
+        pass
+    else:
+        versions = list(versions)
+
+    if isinstance(zones,str):
+        zones = [zones]
+    elif isinstance(zones,list):
+        pass
+    else:
+        zones = list(zones)
+    
+
+    # loop through benchmarks
+    consolidated_metrics_df = [ f for f in return_dataframe_for_benchmark_source(benchmarks,zones) ]
+    
+    # concat
+    consolidated_metrics_df = pd.concat(consolidated_metrics_df,ignore_index=True)
+
+    # find matching rows
+    consolidated_metrics_df = find_matching_rows_by_attribute_value(consolidated_metrics_df,'version',versions)
+
+    if impute_missing_ms:
+        consolidated_metrics_df = impute_missing_MS_with_FR(consolidated_metrics_df)
+
+    if metrics_output_csv is not None:
+        consolidated_metrics_df.to_csv(metrics_output_csv, index=False)
+
+    """
+    #print(consolidated_metrics_df.columns);exit()
+    consolidated_metrics_pivot = pd.pivot_table(
+                                                consolidated_metrics_df,
+                                                values=['FP_area_km2','FN_area_km2','TP_area_km2','contingency_tot_area_km2','obsPositive_area_km2'],
+                                                columns=['extent_config'],
+                                                index=['magnitude'], 
+                                                aggfunc=np.sum
+                                               )
+    print(consolidated_metrics_pivot);exit()"""
+
+    consolidated_secondary_metrics = pivot_metrics(consolidated_metrics_df)
+    print(consolidated_secondary_metrics)
+
+    return(consolidated_metrics_df,consolidated_secondary_metrics)
+
+
+def impute_missing_MS_with_FR(consolidated_metrics_df):
+
+
+    # make sure you have one version per extent_config
+    fr_extent_config = 'FR'
+    ms_extent_config = 'MS'
+    extent_config = [fr_extent_config, ms_extent_config]
+    
+    indices_of_ec = {ec : consolidated_metrics_df.index[consolidated_metrics_df.loc[:,'extent_config'] == ec] for ec in extent_config }
+
+    unique_hucs_dict = dict()
+    for ec in extent_config:
+
+        unique_version = consolidated_metrics_df.loc[ indices_of_ec[ec],'version'].unique()
+
+        if len(unique_version) > 1:
+            raise ValueError(f"{ec} version has more than one extent. Only pass one version per extent configuration.")
+        unique_hucs = set(consolidated_metrics_df.loc[indices_of_ec[ec],'huc'].unique())
+
+        unique_hucs_dict[ec] = unique_hucs
+
+
+    unique_hucs = unique_hucs_dict[fr_extent_config] - unique_hucs_dict[ms_extent_config]
+    
+    # copy those rows over with extent_config set to MS
+    unique_fr_mask = consolidated_metrics_df.loc[ indices_of_ec['FR'],'huc'].isin(unique_hucs)
+    unique_fr_indices = unique_fr_mask.index[unique_fr_mask]
+
+    fr_entries = consolidated_metrics_df.loc[unique_fr_indices,:].reset_index(drop=True)
+    fr_entries.loc[:,'extent_config'] = "MS"
+    fr_entries.loc[:,'version'] = consolidated_metrics_df.loc[ indices_of_ec['MS'], 'version'].unique()
+
+    consolidated_metrics_df = pd.concat((consolidated_metrics_df,fr_entries)).reset_index(drop=True)
+
+    return(consolidated_metrics_df)
+
+
+
+def pivot_metrics(consolidated_metrics_df):
+
+    ''' Pivots metrics to provide summary of results '''
+
+    # pivot out
+    consolidated_metrics_pivot = pd.pivot_table(
+                                                consolidated_metrics_df,
+                                                values=['true_positives_count','false_positives_count',
+                                                        'false_negatives_count','true_negatives_count'],
+                                                index=['magnitude','extent_config','version'], 
+                                                aggfunc=sum
+                                               )
+
+    
+    return( apply_metrics_to_pivot_table(consolidated_metrics_pivot) )
+
+
+
+def apply_metrics_to_pivot_table(consolidated_metrics_pivot):
+
+    # metrics to run
+    metrics_functions = { 'CSI': csi , 'TPR' : tpr, 'FAR' : far, 'MCC' : mcc }
+    #metrics_functions = { 'CSI': csi }
+    
+    def row_wise_function(df,mn,mf):
+        
+        ''' applies function along rows of dataframes '''
+
+        return( 
+                pd.Series(
+                          mf( 
+                             df['true_positives_count'],
+                             df['false_positives_count'],
+                             df['false_negatives_count'],
+                             df['true_negatives_count']
+                            ),
+                          name=mn
+                         )
+              )
+
+    column_generator = (  row_wise_function(consolidated_metrics_pivot,met_name,met_func) 
+                          for met_name,met_func in metrics_functions.items() 
+                       )
+
+    consolidated_secondary_metrics = pd.concat(column_generator,axis=1)
+
+    return(consolidated_secondary_metrics)
+
+
+def find_matching_rows_by_attribute_value(df,attributes,matches):
+    
+    if (len(matches) == 1) & ('all' in matches):
+        return(df)
+    else:
+        df = df.loc[df.loc[:,attributes].isin(matches)]
+
+    return(df)
+
+
+def return_dataframe_for_benchmark_source(benchmarks,zones=['total_area']):
+
+    """ returns a dataframe of results given a name for a benchmark source """
+
+    benchmark_function_dict = { 
+                                'ble' : consolidate_metrics('ble',zones),
+                                'ifc' : consolidate_metrics('ifc',zones)
+                              }
+
+    # if all cycle through all functions, else go through selection
+    if (len(benchmarks)== 1) & ('all' in benchmarks):
+        
+        for f in benchmark_function_dict.values():
+            yield(f)
+
+    else:
+
+        for b in benchmarks:
+            try:
+                yield( benchmark_function_dict[b] )
+            except KeyError: 
+                raise ValueError(f"Benchmark '{b}' not supported. "\
+                                 f"Pass 'all' or one of these"\
+                                 f"{list(benchmark_function_dict.keys())}")
+
+
+def consolidate_metrics(benchmark,zones=['total_area']):
+
+    """ consolidates ble metrics """
+
+    # get filenames for metrics for ble
+    files_to_consolidate = list()
+    for zone in zones:
+        file_pattern_to_glob = os.path.join(TEST_CASES_DIR,f'{benchmark}_test_cases','**',f'{zone}_stats.csv')
+        files_to_consolidate.extend(glob(file_pattern_to_glob, recursive=True))
+
+    # make a metrics dataframe generator
+    metrics_df_generator = metrics_data_frame_generator_from_files(files_to_consolidate)
+
+    # concat said generator of files
+    metrics_df = pd.concat(metrics_df_generator, ignore_index=True)
+
+    # make a meta-data
+    metadata_df = metadata_dataframe_from_file_names_for_ble(files_to_consolidate)
+    
+    # concat metrics and metadata dataframes
+    return(   
+            pd.concat( (metadata_df,metrics_df), axis=1) 
+          )
+
+
+def metrics_data_frame_generator_from_files(file_names):
+
+    """ Reads in metrics dataframes from filenames """
+
+    for f in file_names:
+        yield pd.read_csv(f,index_col=0).T.reset_index(drop=True)
+
+
+def metadata_dataframe_from_file_names_for_ble(file_names):
+    
+    """ Makes a dataframe for meta-data """
+
+    file_name_index_dict = { 
+                            'benchmark_source' : parse_benchmark_source(file_names),
+                            'version' : parse_version_name(file_names),
+                            'huc' : parse_huc(file_names),
+                            'magnitude' : parse_magnitude(file_names),
+                            'extent_config' : parse_eval_metadata(file_names,'model'),
+                            'calibrated' : parse_eval_metadata(file_names,'calibrated')
+                           }
+
+    return( 
+            pd.DataFrame(file_name_index_dict)
+          )
+
+
+def parse_benchmark_source(file_names):
+
+    """ Parses benchmark source """
+
+    for f in file_names:
+        yield f.split('/')[3].split('_')[0]
+
+
+def parse_version_name(file_names):
+
+    """ Parses version name """
+
+    for f in file_names:
+        yield f.split('/')[6]
+
+
+def parse_magnitude(file_names):
+
+    """ Parses magnitude """
+
+    for f in file_names:
+        yield f.split('/')[7]
+
+
+def parse_huc(file_names):
+
+    """ parses huc """
+
+    for f in file_names:
+        yield f.split('/')[4].split('_')[0]
+
+
+def parse_eval_metadata(file_names,metadata_field):
+   
+    """ parsing eval metadata json files """
+
+    for f in file_names:
+        root_dir = os.path.abspath(os.sep)
+        dir_of_metadata = f.split('/')[0:7]
+        eval_metadata_filepath = os.path.join(root_dir,*dir_of_metadata,'eval_metadata.json')
+        
+        # read eval data, if no file write None 
+        try:
+            with open(eval_metadata_filepath) as fObj:
+                eval_metadata = json.load(fObj)
+        except FileNotFoundError:
+            yield None
+        else:
+            yield eval_metadata[metadata_field]
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
+    parser.add_argument('-b','--benchmarks',help='Allowed benchmarks', required=False, default='all', nargs="+")
+    parser.add_argument('-v','--versions',help='Allowed versions', required=False, default='all', nargs="+")
+    parser.add_argument('-z','--zones',help='Allowed zones', required=False, default='total_area', nargs="+")
+    parser.add_argument('-o','--metrics-output-csv',help='File path to outputs csv', required=False, default=None)
+    parser.add_argument('-i','--impute-missing_ms',help='Imputes FR metrics in HUCS with no MS. Only supports one version per extent config', required=False, action='store_true',default=False)
+
+    args = vars(parser.parse_args())
+
+    Consolidate_metrics(**args)
diff --git a/tools/cygnss_preprocessing.py b/tools/cygnss_preprocessing.py
new file mode 100755
index 000000000..bbd08c7e0
--- /dev/null
+++ b/tools/cygnss_preprocessing.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+
+from shapely.geometry import box
+import datetime
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+import argparse
+from datetime import datetime
+from make_boxes_from_bounds import find_hucs_of_bounding_boxes
+import requests
+from concurrent.futures import ThreadPoolExecutor,as_completed
+import os
+from tqdm import tqdm
+from foss_fim.tools.inundation import read_nwm_forecast_file
+
+
+def cygnss_preprocessing(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',forecast_output_file=None,retrieve=True,workers=6,download_directory=None, daily_mean_forecast_files=None):
+
+    _, bounding_boxes = find_hucs_of_bounding_boxes(bounding_boxes_file,wbd=wbd,projection_of_boxes=projection_of_boxes,wbd_layer=wbd_layer)
+    
+    # load bounding box file
+    bounding_boxes['event_date'] = pd.to_datetime(bounding_boxes['event_date'],utc=True)
+    bounding_boxes.reset_index(drop=True,inplace=True)
+    
+    wbdcol_name = 'HUC'+wbd_layer[-1]
+
+    # expand dates
+    datetime_indices = bounding_boxes.apply(lambda df:pd.date_range(df['event_date'],periods=24,closed=None,freq='H',tz='UTC'),axis=1)
+
+    datetime_indices.name = 'date_time'
+    datetime_indices=pd.DataFrame(datetime_indices)
+    datetime_indices = datetime_indices.join(bounding_boxes[['Name',wbdcol_name]])
+
+    # append columns to expanded dates
+    forecast_df = pd.DataFrame()
+    for idx,row in datetime_indices.iterrows():
+        dt_df = row['date_time'].to_frame(index=False,name='date_time')
+        
+        row = row.drop('date_time')
+
+        dt_df = dt_df.join(pd.concat([pd.DataFrame(row).T]*len(dt_df),ignore_index=True))
+
+        forecast_df = pd.concat((forecast_df,dt_df),ignore_index=True)
+
+    # add extra dry date 
+    #additional_date_df = forecast_df[['Name',wbdcol_name]].merge(bounding_boxes[['additional_date',wbdcol_name]],left_on=wbdcol_name,right_on=wbdcol_name).drop_duplicates(ignore_index=True)
+
+    #forecast_df = pd.concat((forecast_df,additional_date_df.rename(columns={'additional_date':'date_time'})),ignore_index=True)
+
+    forecast_df = forecast_df.sort_values(['Name',wbdcol_name],ignore_index=True)
+
+
+    #forecast_df['date_time'] = forecast_df.apply(lambda df : df['date_time'].replace(hour=18,minute=0),axis=1)
+
+    forecast_df = forecast_df.rename(columns={wbdcol_name:'huc'})
+
+    forecast_df = construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=retrieve,workers=workers)
+   
+
+    # take daily means
+    def get_forecast(forecast_row):
+        
+        try:
+            forecast_table = read_nwm_forecast_file(forecast_row['forecast_file'])
+        except FileNotFoundError:
+            print(f"Skipping file {forecast_row['forecast_file']}")
+            return None
+
+        return(forecast_table)
+
+    # filter out 2021
+    years = forecast_df['date_time'].dt.year < 2021
+    forecast_df = forecast_df.loc[years,:]
+
+    # unique dates and hourly samples
+    unique_forecast_df = forecast_df.drop(columns='huc').drop_duplicates()
+    unique_forecast_df.reset_index(inplace=True,drop=True)
+
+    dates = unique_forecast_df["date_time"].map(lambda t: t.date())
+    unique_dates, hourly_samples_per_date = np.unique(dates,return_counts=True)
+
+    unique_dict = dict(zip(unique_dates,hourly_samples_per_date))
+    
+    final_forecast_df = forecast_df.groupby(pd.Grouper(key='date_time',freq='d')).first().dropna()
+    final_forecast_df.set_index(pd.to_datetime(final_forecast_df.index.date,utc=True),drop=True,inplace=True)
+    forecast_df_dates = forecast_df.copy()
+    forecast_df_dates.date_time = forecast_df.date_time.dt.date
+    final_forecast_df = final_forecast_df.merge(forecast_df,left_index=True, right_on='date_time')
+
+    final_forecast_df.reset_index(drop=True,inplace=True)
+    final_forecast_df['date_time'] = final_forecast_df.date_time.dt.date
+
+    final_forecast_df.drop(columns={'huc_x','Name_x','forecast_file_x','forecast_url_x','forecast_url_y'},inplace=True)
+
+    final_forecast_df.rename(columns={'Name_y':'Name','huc_y':'huc','forecast_file_y':'forecast_file'},inplace=True)
+    
+    def update_daily(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df):
+        daily_mean = daily_nwm_forecast_df.mean(axis=1).rename('discharge')
+        
+        current_date_string = current_date.strftime(format='%Y%m%d')
+        filename,ext = os.path.basename(daily_mean_forecast_files).split('.')
+        outfile = os.path.join(os.path.dirname(daily_mean_forecast_files),filename+'_'+ current_date_string + '.'+ext)
+        daily_mean.to_csv(outfile,index=True)
+        
+        final_forecast_df.loc[final_forecast_df['date_time'] == current_date,'forecast_file'] = outfile
+        
+        daily_nwm_forecast_df = None
+
+        return(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df)
+
+
+    daily_nwm_forecast_df = None
+    ii = 0
+    current_date_time = unique_forecast_df.loc[0,'date_time']
+    current_date = current_date_time.date()
+    for i,row in tqdm(unique_forecast_df.iterrows(),total=len(unique_forecast_df),desc='Daily Means'):
+        
+        current_nwm_forecast_df = get_forecast(row)
+        
+        if ii == unique_dict[current_date]:
+
+            daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df = update_daily(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df)
+
+        if current_nwm_forecast_df is not None:
+            
+            if daily_nwm_forecast_df is None:
+                ii = 0
+                daily_nwm_forecast_df = pd.DataFrame(np.empty((len(current_nwm_forecast_df),unique_dict[current_date])))
+                daily_nwm_forecast_df.set_index(current_nwm_forecast_df.index,inplace=True,drop=True)
+            
+            daily_nwm_forecast_df.loc[:,ii] = current_nwm_forecast_df.discharge
+            #daily_nwm_forecast_df.rename(columns={ii:current_date_time})
+            
+            ii += 1
+        
+        current_date_time = row['date_time']
+        current_date = current_date_time.date()
+
+    daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df = update_daily(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df)
+       
+
+    if forecast_output_file is not None:
+        #final_forecast_df.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d %H:%M:%S%Z')
+        final_forecast_df.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d')
+
+
+
+def construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=True,workers=1):
+    # make forecast file names for NWM and retrieve
+
+    #construct url
+    #url = f'{year}/{year}{month}{day}{time}.CHRTOUT_DOMAIN1.comp'
+
+    make_url = lambda df:  "https://noaa-nwm-retrospective-2-1-pds.s3.amazonaws.com/model_output/" + \
+                                          str(df['date_time'].year) + '/' + str(df['date_time'].year) + \
+                                           str(df['date_time'].month).zfill(2) + \
+                                           str(df['date_time'].day).zfill(2) + \
+                                           str(df['date_time'].hour).zfill(2) + \
+                                           str(df['date_time'].minute).zfill(2) +\
+                                           ".CHRTOUT_DOMAIN1.comp"
+        
+    def make_file_names(df,download_directory):
+        # assumes that the last segment after the / represents the file name
+        url = df['forecast_url']
+        file_name_start_pos = url.rfind("/") + 1
+        file_name = url[file_name_start_pos:]
+
+        file_name = os.path.join(download_directory,file_name)
+
+        return(file_name)
+
+
+    urls = forecast_df.apply(make_url,axis=1)
+    forecast_df['forecast_url'] = urls
+
+    file_names = forecast_df.apply(lambda df: make_file_names(df,download_directory),axis=1)
+    forecast_df['forecast_file'] = file_names
+
+    if not retrieve:
+        return(forecast_df)
+
+    download_df = forecast_df[['forecast_url','forecast_file']].drop_duplicates()
+
+
+    def download_url(url,file_name):
+        r = requests.get(url, stream=True)
+        if r.status_code == requests.codes.ok:
+             with open(file_name, 'wb') as f:
+                for data in r:
+                    f.write(data)
+        return url
+
+    pool = ThreadPoolExecutor(max_workers=workers)
+    
+    results = { pool.submit(download_url,*(url,file_name)) : (url,file_name) for idx,(url,file_name) in download_df.iterrows() }
+
+    for future in tqdm(as_completed(results),total=len(download_df),desc='Acquiring NWM forecasts'):
+
+        url,file_name = results[future]
+
+        try:
+            future.result()
+        except Exception as exc:
+            print('error',exc,url)
+
+    pool.shutdown(wait=True)
+    
+    return(forecast_df)
+
+
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
+    parser.add_argument('-b','--bounding-boxes-file', help='Bounding box file', required=True)
+    parser.add_argument('-w','--wbd', help='WBD file', required=True)
+    parser.add_argument('-f','--forecast-output-file', help='Forecast file', required=False,default=None)
+    parser.add_argument('-r','--retrieve', help='Forecast file', required=False,default=False,action='store_true')
+    parser.add_argument('-j','--workers', help='Forecast file', required=False,default=1,type=int)
+    parser.add_argument('-d','--download_directory', help='Forecast file', required=False,default=1)
+    parser.add_argument('-m','--daily-mean-forecast-files', help='Daily Mean Forecast file', required=False,default=None)
+
+    args=vars(parser.parse_args())
+
+    cygnss_preprocessing(**args)
diff --git a/tools/gms_tools/__init__.py b/tools/gms_tools/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tools/gms_tools/compare_ms_and_non_ms_metrics.py b/tools/gms_tools/compare_ms_and_non_ms_metrics.py
new file mode 100644
index 000000000..8edabc0e2
--- /dev/null
+++ b/tools/gms_tools/compare_ms_and_non_ms_metrics.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import numpy as np
+import os
+from tqdm import tqdm
+import argparse
+from concurrent.futures import ProcessPoolExecutor, as_completed
+                                   csi, mcc, tpr, far
+from tools.shared_variables import TEST_CASES_DIR, OUTPUTS_DIR
+from glob import glob
+from itertools import product
+
+
+def Compare_ms_and_non_ms_areas():
+    
+    return(None)
+
+
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
+    #parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
+    
+
+    #args = vars(parser.parse_args())
+    
+    Compare_ms_and_non_ms_areas()
diff --git a/tools/gms_tools/compile_comp_stats.py b/tools/gms_tools/compile_comp_stats.py
new file mode 100755
index 000000000..c5f8a6458
--- /dev/null
+++ b/tools/gms_tools/compile_comp_stats.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import numpy as np
+from glob import glob, iglob
+import argparse
+import os
+
+
+def Compile_comp_stats(hydrofabric_dirs):
+    
+    all_minutes = []
+    all_models = []
+    all_hucs = []
+
+    file_iterator = get_log_files(hydrofabric_dirs)
+
+    for entry in file_iterator:
+        model, log = entry
+        log_file = open(log,'r')
+        
+        huc8code = log.split('/')[-1][0:8]
+
+        for line in log_file:
+            if 'wall clock' in line:
+                time_string = line.strip().split(' ')[-1]
+
+                time_string_split = time_string.split(':')
+
+                if len(time_string_split) == 2:
+                    minutes, seconds = time_string_split
+                    
+                    total_minutes = float(minutes)+float(seconds)/60
+
+                if len(time_string_split) == 3:
+                    hours, minutes, seconds = time_string_split 
+                
+                    total_minutes = float(hours)*60+float(minutes)+float(seconds)/60
+
+                all_minutes.append(total_minutes)
+                all_models.append(model)
+                all_hucs.append(huc8code)
+
+    df = pd.DataFrame({'Minutes':all_minutes,'Model':all_models, 'HUC': all_hucs})
+
+    total_per_huc = df.pivot_table(values='Minutes',index=['Model','HUC'],aggfunc=sum)
+
+    print(total_per_huc.pivot_table(values='Minutes',index='Model',aggfunc=[np.mean,np.median,np.sum]))
+
+
+
+def get_log_files(hydrofabric_dirs):
+
+    for hydrofabric_dir in hydrofabric_dirs:
+        log_dir = os.path.join(hydrofabric_dir,'logs')
+
+        if os.path.join(log_dir,'branch'):
+            model = 'GMS'
+        if '_MS' in log_dir:
+            model = 'MS'
+        if '_FR' in log_dir:
+            model = 'FR'
+
+        for fn in iglob(os.path.join(log_dir,'**','[0-9]*.log'),recursive=True):
+            yield(model,fn)
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Get Comp Stats')
+    parser.add_argument('-y','--hydrofabric_dirs', help='Directory path to FIM hydrofabric by processing unit', required=True,nargs='+')
+    
+
+    Compile_comp_stats( **vars(parser.parse_args()) )
diff --git a/tools/gms_tools/compile_computational_stats.py b/tools/gms_tools/compile_computational_stats.py
new file mode 100755
index 000000000..f7ab6daed
--- /dev/null
+++ b/tools/gms_tools/compile_computational_stats.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import argparse
+import pandas as pd
+import numpy as np
+from glob import iglob
+from os.path import join
+
+# desired output for branches
+# dataframe columns: HUC, branch_id, exit status, ,time, ram, 
+
+def compile_summary(gms_output_dir,ouput=None):
+
+    unit_summary = join(gms_output_dir,logs, 'summary_gms_unit.log')
+    branch_summary = join(gms_output_dir,logs, 'summary_gms_branch.log')
+
+    unit_summary = pd.read_csv(unit_summary,sep='\t')
+    branch_summary = pd.read_csv(branch_summary,sep='\t')
+
+
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Create stream network level paths')
+    parser.add_argument('-d','--gms-output-dir', help='Input stream network', required=True)
+    parser.add_argument('-o','--output', help='Input stream network', required=True)
+    
+    args = vars(parser.parse_args())
+
+    compile_summary(**args)
+
diff --git a/tools/gms_tools/evaluate_continuity.py b/tools/gms_tools/evaluate_continuity.py
new file mode 100644
index 000000000..b16c6b140
--- /dev/null
+++ b/tools/gms_tools/evaluate_continuity.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import pandas as pd
+import numpy as np
+import argparse
+import matplotlib.pyplot as plt
+
+def evaluate_continuity(stream_network_file,forecast_file,stream_network_outfile=None,
+                        confluences_only=False,plot_file=None):
+
+    stream_network = gpd.read_file(stream_network_file)
+    forecast = pd.read_csv(forecast_file)
+    
+    stream_network = stream_network.merge(forecast,how='left',left_on='feature_id',right_on='feature_id')
+    
+    stream_network['discharge'] = stream_network['discharge'].fillna(0)
+
+ 
+    toNodes = set(stream_network['To_Node'].tolist())
+
+    # upstream dictionary and confluences
+    upstream_dictionary,confluences = {},set()
+    for idx,reach in stream_network.iterrows():
+    
+        fromNode = reach['From_Node']
+        hydroId= reach['HydroID']
+        
+        if fromNode in toNodes:
+            upstream_indices = stream_network['To_Node'] == fromNode
+            upstream_discharges = np.array( stream_network.loc[upstream_indices,'discharge'].tolist() )
+            upstream_dictionary[hydroId] = upstream_discharges
+
+            isconfluence = len(upstream_discharges) > 1
+            if isconfluence:
+                confluences.add(hydroId)
+
+    # filter out non-confluences
+    if confluences_only:
+        hydroIDs = stream_network['HydroID'].tolist()
+        confluence_bool = np.array(  [True if h in confluences else False for h in hydroIDs]  )
+        stream_network = stream_network.loc[confluence_bool,:]
+
+    actual_discharges, expected_discharges = [], []
+    expected_dischages_dict = dict()
+    for idx,reach in stream_network.iterrows():
+    
+        hydroId = reach['HydroID']
+        
+        try:
+            upstream_discharges = upstream_dictionary[hydroId]
+        except KeyError:
+            expected_dischages_dict[hydroId] = 0
+            continue
+
+        actual_discharges += [reach['discharge']]
+        expected_discharges += [np.sum(upstream_discharges)]
+        expected_dischages_dict[hydroId] = np.sum(upstream_discharges)
+
+
+    actual_discharges, expected_discharges = np.array(actual_discharges), np.array(expected_discharges)
+
+    # add to stream_network
+    expected_discharges_df = pd.DataFrame.from_dict(expected_dischages_dict,orient='index',
+                                                    columns=['expected_discharges'])
+    stream_network = stream_network.merge(expected_discharges_df, left_on='HydroID', right_index=True, how='left')
+    
+    number_of_reaches = len(stream_network)
+    SMAPE = smape(actual_discharges,expected_discharges)
+    diff = actual_discharges - expected_discharges
+
+    print("Number of No Flow Reaches: {} out of {}".format((stream_network['discharge']==0).sum(),number_of_reaches))
+    print("SMAPE = {}%".format(SMAPE[0]))
+    print("Diff (<0) = {}".format(np.sum(diff<0)))
+    print("Diff (>0) = {}".format(np.sum(diff>0)))
+    print("Diff (=0) = {}".format(np.sum(diff==0)))
+    print("Diff (>-10,<10) = {}".format(np.sum(np.logical_or(diff>-10,diff<10))))
+    print("Median diff: {}".format(np.nanmedian(diff)))
+    print("Mean diff: {}".format(np.nanmean(diff)))
+
+    if confluences_only:
+        nbins = 50
+        xlim = (-2000,2000)
+        ylim = (0,50)
+        title = 'Discharge Errors (CMS) At Confluence Reaches'
+    else:
+        nbins = 500
+        xlim = (-450,450)
+        ylim = (0,60)
+        title = 'Discharge Errors (CMS)'
+    
+    fig = plt.figure(1)
+    ax = plt.subplot(111)
+    n, bins, patches = ax.hist(diff, nbins, facecolor='blue', alpha=0.4)
+    try:
+        plt.xlim(xlim)
+        plt.ylim(ylim)
+    except UnboundLocalError:
+        pass
+    plt.title(title)
+    plt.xlabel('Discharge Errors (CMS) = actual - expected')
+    plt.ylabel('Count (Peak not show)')
+    
+    if plot_file is not None:
+        fig.savefig(plot_file)
+
+    if stream_network_outfile is not None:
+        stream_network.to_file(stream_network_outfile,index=False,driver='GPKG')
+
+    return(stream_network)
+
+
+def smape(predicted,actual):
+
+    assert len(predicted) == len(actual),"Predicted and actual need to have same length"
+
+    sape = 100 * (np.abs(predicted-actual) / (np.abs(predicted) + np.abs(actual)))
+
+    return (np.nanmean(sape),sape)
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Evaluating continuity')
+    parser.add_argument('-s','--stream-network-file',help='Stream Network',required=True)
+    parser.add_argument('-f','--forecast-file',help='Forecast File',required=True)
+    parser.add_argument('-o','--stream-network-outfile',help='Stream Network Outfile',required=False,default=None)
+    parser.add_argument('-c','--confluences-only',help='Only at confluence reaches',required=False,default=False,action='store_true')
+    parser.add_argument('-p','--plot-file',help='Plot File',required=False,default=None)
+
+
+
+    args = vars(parser.parse_args())
+
+    evaluate_continuity(**args)
diff --git a/tools/gms_tools/find_max_catchment_breadth.py b/tools/gms_tools/find_max_catchment_breadth.py
new file mode 100755
index 000000000..e8da34aa8
--- /dev/null
+++ b/tools/gms_tools/find_max_catchment_breadth.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+import os
+import pandas as pd
+import geopandas as gpd
+from glob import glob,iglob
+import argparse
+from shapely.geometry import Point
+
+
+def Find_max_catchment_breadth(hydrofabric_dir):
+
+    catchments_fileNames = glob_file_paths_for_catchments(hydrofabric_dir)
+
+    list_with_max_lengths_within_files = []
+
+    for idx,cfn in enumerate(catchments_fileNames):
+
+        catchments = gpd.read_file(cfn)
+
+        lengths = catchments.apply(get_length_and_width_of_poly,axis=1)
+
+        max_length_within_file = lengths.median()
+
+        list_with_max_lengths_within_files.append(max_length_within_file)
+
+        print(idx,max_length_within_file)
+
+
+    print(max(list_with_max_lengths_within_files))
+
+
+def glob_file_paths_for_catchments(hydrofabric_dir):
+
+    file_pattern_to_glob = os.path.join(hydrofabric_dir,'**','gw_catchments_reaches_filtered_addedAttributes_crosswalked*.gpkg')
+
+    catchments_fileNames = iglob(file_pattern_to_glob)
+
+    return(catchments_fileNames)
+
+
+def get_length_and_width_of_poly(geodataframe_row):
+
+    poly = geodataframe_row['geometry']
+    
+    # get minimum bounding box around polygon
+    box = poly.minimum_rotated_rectangle
+
+    # get coordinates of polygon vertices
+    x, y = box.exterior.coords.xy
+
+    # get length of bounding box edges
+    edge_length = (Point(x[0], y[0]).distance(Point(x[1], y[1])), Point(x[1], y[1]).distance(Point(x[2], y[2])))
+
+    # get length of polygon as the longest edge of the bounding box
+    length = max(edge_length)
+
+    # get width of polygon as the shortest edge of the bounding box
+    #width = min(edge_length)
+
+    return(length)
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Find the maximum breadth of catchments in given hydrofabric directory')
+    parser.add_argument('-y','--hydrofabric-dir',help='Path to hydrofabric directory',required=True)
+
+    args = vars(parser.parse_args())
+
+    Find_max_catchment_breadth(**args)
+
diff --git a/tools/gms_tools/inundate_gms.py b/tools/gms_tools/inundate_gms.py
old mode 100644
new mode 100755
index 0148da336..0b1c2f59b
--- a/tools/gms_tools/inundate_gms.py
+++ b/tools/gms_tools/inundate_gms.py
@@ -209,7 +209,7 @@ def __inundate_gms_generator(
                             'out_vector_profile' : None,
                             'quiet' : not verbose
                           }
-
+        
         yield (inundate_input,identifiers)
 
 
@@ -258,4 +258,3 @@ def __vprint(message,verbose):
     # extract to dictionary and run
     Inundate_gms( **vars(parser.parse_args()) )
     
-    
\ No newline at end of file
diff --git a/tools/gms_tools/mosaic_inundation.py b/tools/gms_tools/mosaic_inundation.py
old mode 100644
new mode 100755
index ea24cb75f..291c7287b
--- a/tools/gms_tools/mosaic_inundation.py
+++ b/tools/gms_tools/mosaic_inundation.py
@@ -7,7 +7,7 @@
 import os
 import pandas as pd
 from tqdm import tqdm
-from tools_shared_variables import elev_raster_ndv
+from utils.shared_variables import elev_raster_ndv
 
 def Mosaic_inundation(
                       map_file,mosaic_attribute='inundation_rasters',mosaic_output=None,
@@ -53,7 +53,7 @@ def Mosaic_inundation(
     else:
         tqdm_disable = True
 
-    for ag in tqdm(aggregation_units,disable=tqdm_disable,desc='Compositing MS and FR maps'):
+    for ag in tqdm(aggregation_units,disable=tqdm_disable,desc='Mosaicing FIMs'):
 
         try:
             inundation_maps_list = inundation_maps_df.loc[ag,mosaic_attribute].tolist()
@@ -142,4 +142,4 @@ def __append_id_to_file_name(file_name,identifier):
 
     args = vars(parser.parse_args())
     
-    Mosaic_inundation(**args)
\ No newline at end of file
+    Mosaic_inundation(**args)
diff --git a/tools/gms_tools/overlapping_inundation.py b/tools/gms_tools/overlapping_inundation.py
index 5dab3da96..a99abdeec 100644
--- a/tools/gms_tools/overlapping_inundation.py
+++ b/tools/gms_tools/overlapping_inundation.py
@@ -442,10 +442,12 @@ def merge_data(rst_data,
                  data]
 
         del data
+        
         with warnings.catch_warnings():
             # This `with` block supresses the RuntimeWarning thrown by numpy when aggregating nan values
             warnings.simplefilter("ignore", category=RuntimeWarning)
             window_data[row_slice, col_slice] = agg_function(merge)
+        
         window_data[np.isnan(window_data)] = nodata
         del merge
 
@@ -502,4 +504,4 @@ def merge_data(rst_data,
                                  (1, 1))
     overlap.merge_rasters(project_path + '/merged_final5.tif', threaded=False, workers=4)
 
-    print('end', time.localtime())
\ No newline at end of file
+    print('end', time.localtime())
diff --git a/tools/gms_tools/plots.py b/tools/gms_tools/plots.py
new file mode 100755
index 000000000..80a1b4fcb
--- /dev/null
+++ b/tools/gms_tools/plots.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+from matplotlib.ticker import FixedLocator, FixedFormatter, NullFormatter, FormatStrFormatter
+import argparse
+import pandas as pd
+import numpy as np
+from statsmodels.robust.robust_linear_model import RLM
+from itertools import product
+
+
+
+def gms_box_plots(metrics_table,output_fig=None):
+
+    if isinstance(metrics_table,pd.DataFrame):
+        pass
+    elif isinstance(metrics_table,str):
+        metrics_table = pd.read_csv(metrics_table)
+    elif isinstance(metrics_table,list):
+        metrics_table = pd.concat([pd.read_csv(mt) for mt in metrics_table],ignore_index=True)
+    else: 
+        ValueError("Pass metrics_table as DataFrame or path to CSV")
+
+    metrics_table = preparing_data_for_plotting(metrics_table)
+    
+    facetgrid = sns.catplot( 
+                    data=metrics_table,x='Model',y='Metric Value',
+                    hue='Magnitude',inner='quartile',split=True,
+                    order=['FR','MS','GMS'],
+                    cut=2,
+                    kind='violin',
+                    col='Mannings N',
+                    row='Metric',
+                    margin_titles=False,
+                    sharex=False,
+                    sharey=False,
+                    despine=True,
+                    scale='width',
+                    height=2.75,
+                    aspect=1,
+                    linewidth=1.5,
+                    legend=False,
+                    saturation=0.55
+                  )
+    
+    facetgrid.map_dataframe( sns.regplot,
+                             data=metrics_table,
+                             x='model_integer_encodings',
+                             y='Metric Value',
+                             scatter=False,
+                             ci=None,
+                             robust=True,
+                             color= 'darkgreen',
+                             truncate=False,
+                             line_kws= {'linewidth' : 2},
+                             label='Trend Line'
+                           )
+    
+    #facetgrid.fig.set_size_inches(10,15)
+
+    # legend
+    handles, labels = facetgrid.axes[0,0].get_legend_handles_labels()
+    facetgrid.fig.legend(loc='lower left',ncol=3,bbox_to_anchor=(0.22,-0.06),handles=handles,labels=labels)
+
+    # xlabel
+    plt.annotate('Model',xy=(0.52,0.054),xycoords='figure fraction')
+
+    # override axes params
+    facetgrid = set_axes(facetgrid)
+
+    facetgrid.fig.subplots_adjust( wspace=0,
+                                   hspace=0.1,
+                                 )
+    
+    # set margins to tight
+    #plt.tight_layout()
+    
+    # set rlm metrics
+    metrics_table = robust_linear_model(metrics_table)
+    facetgrid = annotate_rlm(facetgrid,metrics_table)
+
+    # set text sizes
+    facetgrid = set_text_sizes(facetgrid)
+
+    if output_fig is not None:
+        plt.savefig(output_fig,dpi=300,format='jpg',bbox_inches='tight')
+
+    plt.show()
+    
+
+def set_axes(facetgrid):
+
+    range_by_row = ( (0.45,0.75,0.05),
+                     (0.50,0.90,0.05),
+                     (0.00,0.40,0.05)
+                   )
+    
+    nrows,ncols = facetgrid.axes.shape
+
+    for rowIdx in range(nrows):
+        
+        all_ticks = np.arange(*range_by_row[rowIdx])
+        major_ticks = all_ticks[1:]
+        #minor_ticks = (all_ticks[0] , all_ticks[-1]+range_by_row[rowIdx][-1])
+        minor_ticks = np.arange( range_by_row[rowIdx][0],
+                                 range_by_row[rowIdx][1]+range_by_row[rowIdx][2],
+                                 range_by_row[rowIdx][2]*2
+                               )
+        major_ticks = np.arange( range_by_row[rowIdx][0]+range_by_row[rowIdx][2],
+                                 range_by_row[rowIdx][1],
+                                 range_by_row[rowIdx][2]*2
+                               )
+        
+        
+        for colIdx in range(ncols):
+            
+            facetgrid.axes[rowIdx,colIdx].set_ylim(range_by_row[rowIdx][:-1])
+            facetgrid.axes[rowIdx,colIdx].yaxis.set_major_locator(FixedLocator(major_ticks))
+            facetgrid.axes[rowIdx,colIdx].yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
+            facetgrid.axes[rowIdx,colIdx].yaxis.set_minor_locator(FixedLocator(minor_ticks))
+            facetgrid.axes[rowIdx,colIdx].yaxis.set_minor_formatter(NullFormatter())
+            facetgrid.axes[rowIdx,colIdx].tick_params(labelrotation=45,axis='y')
+            
+            # set axis titles
+            current_title = facetgrid.axes[rowIdx,colIdx].get_title()
+            facetgrid.axes[rowIdx,colIdx].set_title('')
+            values = [val.strip() for val in current_title.split('|')]
+            metric = values[0][-3:]
+            mannings = values[1][-4:]
+
+            # axis border
+            facetgrid.axes[rowIdx,colIdx].axhline(range_by_row[rowIdx][1],color='black')
+            
+            if colIdx == 1:
+                facetgrid.axes[rowIdx,colIdx].axvline(x=facetgrid.axes[rowIdx,colIdx].get_xlim()[1],color='black')
+
+            if rowIdx == 0:
+                facetgrid.axes[rowIdx,colIdx].set_title(f'Manning\'s N = {mannings}',pad=25)
+
+            if colIdx == 0:
+                facetgrid.axes[rowIdx,colIdx].set_ylabel(metric)
+
+            # removes x ticks from everything but bottom row
+            if rowIdx < (nrows - 1):
+                facetgrid.axes[rowIdx,colIdx].set_xticks([])
+        
+        # remove ticks from 2nd column
+        facetgrid.axes[rowIdx,1].set(yticklabels=[]) 
+
+    return(facetgrid)
+
+
+def preparing_data_for_plotting(metrics_table):
+
+    metrics_table.rename(columns= {'extent_config':'Model',
+                                   'magnitude': 'Magnitude',
+                                   'TPR' : 'POD'
+                                  },
+                         inplace=True)
+
+    set_mannings = lambda df : 0.06 if 'n_6' in df['version'] else 0.12
+
+    metrics_table['Mannings N'] = metrics_table.apply(set_mannings,axis=1)
+
+    metrics_table = pd.melt(metrics_table, id_vars=('Model','Magnitude','Mannings N'),
+                            value_vars=('CSI','POD','FAR'),
+                            var_name='Metric',
+                            value_name='Metric Value'
+                            )
+
+    model_to_value_dict = {'FR' : 0, 'MS': 1, 'GMS': 2}
+
+    set_model_integer_encodings = lambda df : model_to_value_dict[ df['Model'] ]
+
+    metrics_table['model_integer_encodings'] = metrics_table.apply(set_model_integer_encodings, axis=1)
+    
+    return(metrics_table)
+
+
+def robust_linear_model(metrics_table):
+
+    metrics = metrics_table.loc[:,'Metric'].unique()
+    mannings = metrics_table.loc[:,'Mannings N'].unique()
+
+    metric_indices = { m : metrics_table.loc[:,'Metric'] == m for m in metrics }
+    mannings_indices = { m : metrics_table.loc[:,'Mannings N'] == m for m in mannings }
+
+    metrics_table.set_index(['Metric','Mannings N'],inplace=True,drop=False)
+    metrics_table.sort_index(inplace=True)
+    for met,man in product(metrics,mannings):
+        
+        y = metrics_table.loc[(met,man),'Metric Value'].to_numpy()
+
+        X = metrics_table.loc[(met,man),'model_integer_encodings'].to_numpy()
+
+        X = np.stack((np.ones(X.shape),X),axis=1)
+        
+        model = RLM(y,X)
+        results = model.fit()
+
+        beta1 = results.params[1]
+        pval_beta1=results.pvalues[1]/2 # two tailed to one tail
+
+        metrics_table.loc[(met,man),'beta1'] = beta1
+        metrics_table.loc[(met,man),'beta1_pvalue'] = pval_beta1
+    
+    metrics_table.reset_index(drop=True,inplace=True)
+
+    return(metrics_table)
+
+
+def annotate_rlm(facetgrid,metrics_table):
+
+    metrics_table.set_index(['Metric','Mannings N'],inplace=True,drop=False)
+    metrics_table.sort_index(inplace=True)
+
+    metric_index_dict = {'CSI':0,'POD':1,'FAR':2}
+    mannings_index_dict = {0.06:0 , 0.12:1}
+    
+    for met,man in metrics_table.index.unique():
+
+        rowIdx = metric_index_dict[met]
+        colIdx = mannings_index_dict[man]
+
+        beta1 = metrics_table.loc[(met,man),'beta1'].unique()[0]
+        beta1_pvalue = metrics_table.loc[(met,man),'beta1_pvalue'].unique()[0]
+
+        facetgrid.axes[rowIdx,colIdx].annotate( r'$\beta_1$  =  {:.4f}  |  p-value  =  {:.3f}'.format(beta1,beta1_pvalue),
+                                                xy=(0.01,1.025),
+                                                xycoords='axes fraction',
+                                                color='darkgreen')
+    
+    return(facetgrid)
+
+
+def set_text_sizes(facetgrid):
+
+    plt.rc('font',size=12)
+
+    return(facetgrid)
+
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
+    parser.add_argument('-m','--metrics-table',help='Metrics table',required=True,nargs='+')
+    parser.add_argument('-o','--output-fig',help='Output figure',required=False,default=None)
+
+    args = vars(parser.parse_args())
+
+    gms_box_plots(**args)
diff --git a/tools/inundate_events.py b/tools/inundate_events.py
new file mode 100755
index 000000000..c3a42efc3
--- /dev/null
+++ b/tools/inundate_events.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+from shapely.geometry import box
+from concurrent.futures import ThreadPoolExecutor,as_completed
+from tqdm import tqdm
+import pandas as pd
+import geopandas as gpd
+import argparse
+from inundation import inundate
+import os
+
+
+def inundate_events(hydrofabric_dir,forecast_file,inundation_file,inundation_polygon=None,jobs=1):
+
+    forecast_df = pd.read_csv(forecast_file,infer_datetime_format=True,dtype={'huc':str},parse_dates=['date_time'])
+
+    #list_of_hucs_to_run = { '09020318','09020317' }
+    #forecast_df = forecast_df.loc[ forecast_df.huc.isin(list_of_hucs_to_run),:]
+
+    inputs = build_fim3_inputs(hydrofabric_dir,forecast_df,inundation_file,inundation_polygon)
+    
+    executor = ThreadPoolExecutor(max_workers=jobs)
+
+    results = {executor.submit(inundate,**kwargs) : (kwargs['hydro_table'],kwargs['forecast']) for kwargs in inputs}
+                         #rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=None,hucs_layerName=None,
+                 #subset_hucs=None,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
+                 #depths=None,out_raster_profile=None,out_vector_profile=None,quiet=False
+
+    for future in tqdm(as_completed(results),total=len(forecast_df)):
+
+        try:
+            future.result()
+        except Exception as exc:
+            print(exc,results[future])
+
+    executor.shutdown(wait=True)
+
+
+
+def build_fim3_inputs(hydrofabric_dir,forecast_df,inundation_file=None,inundation_polygons=None):
+
+
+    for idx,row in forecast_df.iterrows():
+
+        huc = row['huc']
+        rem = os.path.join(hydrofabric_dir,huc,'rem_zeroed_masked.tif')
+        catchments_raster = os.path.join(hydrofabric_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif')
+        catchment_poly = os.path.join(hydrofabric_dir,huc,'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+        hydrotable = os.path.join(hydrofabric_dir,huc,'hydroTable.csv')
+
+        # inundation raster, inundation poly add HUC,date,time to filename
+        inundation_file_for_row = append_meta_data_to_file_names(inundation_file,row)
+        inundation_poly_for_row = append_meta_data_to_file_names(inundation_polygons,row)
+
+        kwargs = { 'rem' : rem, 
+                   'catchments' : catchments_raster, 
+                   'catchment_poly' : catchment_poly , 
+                   'hydro_table': hydrotable,
+                   'mask_type' : 'huc',
+                   'forecast' : row['forecast_file'],
+                   'inundation_raster' : inundation_file_for_row,
+                   'inundation_polygon' : inundation_poly_for_row,
+                   'quiet' : True
+                }
+        
+        yield(kwargs)
+
+
+def append_meta_data_to_file_names(file_name,row):
+
+    if file_name is None:
+        return(file_name)
+
+    base_file_path, extension = os.path.splitext(file_name)
+    
+    hucCode = row['huc']
+    site_name = row['Name']
+    date_time = str(row['date_time'].year) + str(row['date_time'].month).zfill(2) + str(row['date_time'].day).zfill(2) + '_' +str(row['date_time'].hour).zfill(2) + str(row['date_time'].minute).zfill(2) + 'Z'
+
+
+    appended_file_path = f"{base_file_path}_{site_name}_{hucCode}_{date_time}{extension}"
+
+    return(appended_file_path)
+
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
+    parser.add_argument('-y','--hydrofabric-dir', help='Bounding box file', required=True)
+    parser.add_argument('-f','--forecast-file', help='WBD file', required=True)
+    parser.add_argument('-i','--inundation-file', help='WBD file', required=False,default=None)
+    parser.add_argument('-j','--jobs', help='WBD file', required=False,default=None,type=int)
+
+    args=vars(parser.parse_args())
+
+    inundate_events(**args)
diff --git a/tools/inundation.py b/tools/inundation.py
index 2b5a4599c..98cad20d9 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -11,12 +11,22 @@
 from shapely.geometry import shape
 from rasterio.mask import mask
 from rasterio.io import DatasetReader,DatasetWriter
-from rasterio.features import shapes
 from collections import OrderedDict
 import argparse
 from warnings import warn
 from gdal import BuildVRT
 import geopandas as gpd
+import sys
+import xarray as xr
+
+class hydroTableHasOnlyLakes(Exception): 
+    """ Raised when a Hydro-Table only has lakes """
+    pass
+
+
+class NoForecastFound(Exception):
+    """ Raised when no forecast is available for a given Hydro-Table """
+    pass
 
 class hydroTableHasOnlyLakes(Exception): 
     """ Raised when a Hydro-Table only has lakes """
@@ -163,6 +173,7 @@ def inundate(
     else:
         raise TypeError("Pass hydro table csv")
 
+        
     if catchmentStagesDict is not None:
         if src_table is not None:
             create_src_subset_csv(hydro_table,catchmentStagesDict,src_table)
@@ -184,7 +195,6 @@ def inundate(
             except Exception as exc:
                 __vprint("Exception {} for {}".format(exc,results[future]),not quiet)
             else:
-
                 if results[future] is not None:
                     __vprint("... {} complete".format(results[future]),not quiet)
                 else:
@@ -197,31 +207,14 @@ def inundate(
         # power down pool
         executor.shutdown(wait=True)
 
-        # optional aggregation
-        if (aggregate) & (hucs is not None):
-            # inun grid vrt
-            if inundation_raster is not None:
-                inun_vrt = BuildVRT(splitext(inundation_raster)[0]+'.vrt',inundation_rasters)
-                inun_vrt = None
-                #_ = run('gdalbuildvrt -q -overwrite {} {}'.format(splitext(inundation_raster)[0]+'.vrt'," ".join(inundation_rasters)),shell=True)
-            # depths vrt
-            if depths is not None:
-                depths_vrt = BuildVRT(splitext(depths)[0]+'.vrt',depth_rasters,resampleAlg='bilinear')
-                depths_vrt = None
-                #_ = run('gdalbuildvrt -q -overwrite -r bilinear {} {}'.format(splitext(depths)[0]+'.vrt'," ".join(depth_rasters)),shell=True)
-
-            # concat inun poly
-            if inundation_polygon is not None:
-                _ = run('ogrmerge.py -o {} {} -f GPKG -single -overwrite_ds'.format(inundation_polygon," ".join(inundation_polys)),shell=True)
-
-        # close datasets
-        rem.close()
-        catchments.close()
-
-        return(0)
+    # close datasets
+    rem.close()
+    catchments.close()
+
+
+
+    return(inundation_rasters,depth_rasters,inundation_polys)
 
-    else:
-        return(1)
 
 def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profile,catchments_profile,hucCode,
                       catchmentStagesDict,depths,inundation_raster,inundation_polygon,
@@ -326,7 +319,7 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
 
         # make generator for inundation polygons
         inundation_polygon_generator = shapes(inundation_array,mask=inundation_array>0,connectivity=8,transform=window_transform)
-
+        
         # generate records
         records = []
         for i,(g,h) in enumerate(inundation_polygon_generator):
@@ -334,7 +327,7 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
             record['geometry'] = g
             record['properties'] = {'HydroID' : int(h)}
             records += [record]
-
+        
         # write out
         inundation_polygon.writerecords(records)
 
@@ -358,6 +351,13 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
         ip_name = inundation_polygon.path
     except AttributeError:
         ip_name = None
+    
+    #print(ir_name)
+    #yield(ir_name,d_name,ip_name)
+    
+    if isinstance(depths,DatasetWriter): depths.close()
+    if isinstance(inundation_raster,DatasetWriter): inundation_raster.close()
+    if isinstance(inundation_polygon,fiona.Collection): inundation_polygon.close()
 
     return(ir_name,d_name,ip_name)
 
@@ -382,6 +382,7 @@ def __go_fast_mapping(rem,catchments,catchmentStagesDict,inundation,depths):
 def __make_windows_generator(rem,catchments,catchment_poly,mask_type,catchmentStagesDict,inundation_raster,inundation_polygon,
                              depths,out_raster_profile,out_vector_profile,quiet,hucs=None,hucSet=None):
 
+    
     if hucs is not None:
 
         # get attribute name for HUC column
@@ -429,6 +430,8 @@ def __return_huc_in_hucSet(hucCode,hucSet):
                     rem_array,window_transform = mask(rem,catchment_poly['geometry'],crop=True,indexes=1)
                     catchments_array,_ = mask(catchments,catchment_poly['geometry'],crop=True,indexes=1)
                     del catchment_poly
+                elif mask_type is None:
+                    pass
                 else:
                     print ("invalid mask type. Options are 'huc' or 'filter'")
             except ValueError: # shape doesn't overlap raster
@@ -480,6 +483,41 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
 
     hydroTable = hydroTable[hydroTable["LakeID"] == -999]  # Subset hydroTable to include only non-lake catchments.
 
+    # raises error if hydroTable is empty due to all segments being lakes
+    if hydroTable.empty:
+        raise hydroTableHasOnlyLakes("All stream segments in HUC are within lake boundaries.")
+
+
+    if isinstance(forecast,str):
+
+        try:
+            forecast = pd.read_csv(
+                                   forecast,
+                                   dtype={'feature_id' : str , 'discharge' : float}
+                                  )
+            forecast.set_index('feature_id',inplace=True)
+        except UnicodeDecodeError:
+            forecast = read_nwm_forecast_file(forecast)
+    
+    elif isinstance(forecast,pd.DataFrame):
+        pass # consider checking for dtypes, indices, and columns
+    else:
+        raise TypeError("Pass path to forecast file csv or Pandas DataFrame")
+
+    # susbset hucs if passed
+    if subset_hucs is not None:
+        if isinstance(subset_hucs,list):
+            if len(subset_hucs) == 1:
+                try:
+                    subset_hucs = open(subset_hucs[0]).read().split('\n')
+                except FileNotFoundError:
+                    pass
+        elif isinstance(subset_hucs,str):
+                try:
+                    subset_hucs = open(subset_hucs).read().split('\n')
+                except FileNotFoundError:
+                    subset_hucs = [subset_hucs]
+
     if not hydroTable.empty:
 
         if isinstance(forecast,str):
@@ -516,36 +554,58 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
 
             hydroTable = hydroTable[np.in1d(hydroTable.index.get_level_values('HUC'), subset_hucs)]
 
-        # join tables
-        try:
-            hydroTable = hydroTable.join(forecast,on=['feature_id'],how='inner')
+    # join tables
+    try:
+        hydroTable = hydroTable.join(forecast,on=['feature_id'],how='inner')
+    except AttributeError:
+        #print("FORECAST ERROR")
+        raise NoForecastFound("No forecast value found for the passed feature_ids in the Hydro-Table")
 
+    else:
 
-            # initialize dictionary
-            catchmentStagesDict = typed.Dict.empty(types.int32,types.float64)
+        # initialize dictionary
+        catchmentStagesDict = typed.Dict.empty(types.int32,types.float64)
 
-            # interpolate stages
-            for hid,sub_table in hydroTable.groupby(level='HydroID'):
+        # interpolate stages
+        for hid,sub_table in hydroTable.groupby(level='HydroID'):
 
-                interpolated_stage = np.interp(sub_table.loc[:,'discharge'].unique(),sub_table.loc[:,'discharge_cms'],sub_table.loc[:,'stage'])
+            interpolated_stage = np.interp(sub_table.loc[:,'discharge'].unique(),sub_table.loc[:,'discharge_cms'],sub_table.loc[:,'stage'])
 
-                # add this interpolated stage to catchment stages dict
-                h = round(interpolated_stage[0],4)
+            # add this interpolated stage to catchment stages dict
+            h = round(interpolated_stage[0],4)
 
-                hid = types.int32(hid) ; h = types.float32(h)
-                catchmentStagesDict[hid] = h
+            hid = types.int32(hid) ; h = types.float32(h)
+            catchmentStagesDict[hid] = h
 
-            # huc set
-            hucSet = [str(i) for i in hydroTable.index.get_level_values('HUC').unique().to_list()]
+        # huc set
+        hucSet = [str(i) for i in hydroTable.index.get_level_values('HUC').unique().to_list()]
 
-            return(catchmentStagesDict,hucSet)
+        return(catchmentStagesDict,hucSet)
+
+
+def read_nwm_forecast_file(forecast_file,rename_headers=True):
+        
+    """ Reads NWM netcdf comp files and converts to forecast data frame """
+
+    flows_nc = xr.open_dataset(forecast_file,decode_cf='feature_id',engine='netcdf4')
+    
+    flows_df = flows_nc.to_dataframe()
+    flows_df.reset_index(inplace=True)
+    
+    flows_df = flows_df[['streamflow','feature_id']]
+    
+    if rename_headers:
+        flows_df = flows_df.rename(columns={"streamflow": "discharge"})
+
+    convert_dict = {'feature_id': str,'discharge': float}
+    flows_df = flows_df.astype(convert_dict)
+    
+    flows_df.set_index('feature_id',inplace=True,drop=True)
+
+    flows_df.dropna(inplace=True)
+
+    return(flows_df)
 
-        except AttributeError:
-            print (f"No matching feature IDs between forecast and hydrotable for HUC(s): {subset_hucs}")
-            return(None,None)
-    else:
-        print(f"All stream segments in HUC(s): {huc_error} are within lake boundaries.")
-        return(None,None)
 
 def __vprint(message,verbose):
     if verbose:
diff --git a/tools/make_boxes_from_bounds.py b/tools/make_boxes_from_bounds.py
new file mode 100755
index 000000000..b9ea31d45
--- /dev/null
+++ b/tools/make_boxes_from_bounds.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+
+from shapely.geometry import box
+import pandas as pd
+import geopandas as gpd
+import argparse
+from datetime import datetime
+from foss_fim.src.utils.shared_functions import getDriver
+
+
+def find_hucs_of_bounding_boxes(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',huc_output_file=None,forecast_output_file=None,bounding_boxes_outfile=None):
+
+
+    # load bounding box file
+    bounding_boxes = pd.read_csv(bounding_boxes_file,
+                                 dtype={'minx':float,'miny':float,'maxx':float,'maxy':float},
+                                 comment='#')
+
+
+    make_box_geom = lambda df : box(df['minx'],df['miny'],df['maxx'],df['maxy'])
+
+    bounding_boxes['geometry'] = bounding_boxes.apply(make_box_geom,axis=1)
+
+    bounding_boxes = gpd.GeoDataFrame(bounding_boxes,crs=projection_of_boxes)
+
+    wbd_proj = gpd.read_file(wbd,layer=wbd_layer,rows=1).crs
+
+    bounding_boxes = bounding_boxes.to_crs(wbd_proj)
+
+    if bounding_boxes_outfile is not None:
+        bounding_boxes.to_file(bounding_boxes_outfile,driver=getDriver(bounding_boxes_outfile),index=False)
+
+    wbdcol_name = 'HUC'+wbd_layer[-1]
+
+    get_intersections = lambda bbdf : gpd.read_file(wbd,layer=wbd_layer,mask=bbdf.geometry)[wbdcol_name]
+
+    hucs = bounding_boxes.apply(get_intersections,axis=1)
+        
+    bounding_boxes.drop(columns=['geometry','minx','miny','maxx','maxy'],inplace=True)
+
+    hucs_columns = hucs.columns
+    bb_columns = bounding_boxes.columns
+    bounding_boxes = hucs.join(bounding_boxes)
+    bounding_boxes = pd.melt(bounding_boxes,id_vars=bb_columns,value_vars=hucs_columns,value_name='HUC8')
+    bounding_boxes.drop(columns=['variable'],inplace=True)
+    bounding_boxes.dropna(inplace=True)
+    bounding_boxes.reset_index(drop=True,inplace=True)
+
+    hucs_series = pd.Series(hucs.stack().reset_index(drop=True).unique())
+    
+    if huc_output_file is not None:
+        hucs_series.to_csv(huc_output_file,sep='\n',index=False,header=False)
+    
+    if forecast_output_file is not None:
+        bounding_boxes.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d %H:%M:%S%Z')
+    
+    return(hucs_series,bounding_boxes)
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
+    parser.add_argument('-b','--bounding-boxes-file', help='Bounding box file', required=True)
+    parser.add_argument('-w','--wbd', help='WBD file', required=True)
+    parser.add_argument('-p','--projection-of-boxes', help='Projection', required=False,default='EPSG:4329')
+    parser.add_argument('-o','--huc-output-file', help='Output file of HUCS', required=False,default=None)
+    parser.add_argument('-f','--forecast-output-file', help='Forecast file', required=False,default=None)
+    parser.add_argument('-u','--bounding-boxes-outfile', help='Bounding boxes outfile', required=False,default=None)
+
+    args=vars(parser.parse_args())
+
+    find_hucs_of_bounding_boxes(**args)
diff --git a/tools/nesdis_preprocessing.py b/tools/nesdis_preprocessing.py
new file mode 100755
index 000000000..ccf0eeb15
--- /dev/null
+++ b/tools/nesdis_preprocessing.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+from shapely.geometry import box
+import pandas as pd
+import geopandas as gpd
+import argparse
+from datetime import datetime
+from make_boxes_from_bounds import find_hucs_of_bounding_boxes
+import requests
+from concurrent.futures import ThreadPoolExecutor,as_completed
+import os
+from tqdm import tqdm
+
+
+def nesdis_preprocessing(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',forecast_output_file=None,retrieve=True,workers=6,download_directory=None):
+
+    _, bounding_boxes = find_hucs_of_bounding_boxes(bounding_boxes_file,wbd=wbd,projection_of_boxes=projection_of_boxes,wbd_layer=wbd_layer)
+
+    # load bounding box file
+    bounding_boxes['event_start'] = pd.to_datetime(bounding_boxes['event_start'],utc=True)
+    bounding_boxes['event_end'] = pd.to_datetime(bounding_boxes['event_end'],utc=True)
+    bounding_boxes['additional_date'] = pd.to_datetime(bounding_boxes['additional_date'],utc=True)
+
+    bounding_boxes.reset_index(drop=True,inplace=True)
+    
+    wbdcol_name = 'HUC'+wbd_layer[-1]
+
+    # expand dates
+    datetime_indices = bounding_boxes.apply(lambda df:pd.date_range(df['event_start'],df['event_end'],closed=None,freq='D',tz='UTC'),axis=1)
+    
+    datetime_indices.name = 'date_time'
+    datetime_indices=pd.DataFrame(datetime_indices)
+    datetime_indices = datetime_indices.join(bounding_boxes[['Name',wbdcol_name]])
+
+    # append columns to expanded dates
+    forecast_df = pd.DataFrame()
+    for idx,row in datetime_indices.iterrows():
+        dt_df = row['date_time'].to_frame(index=False,name='date_time')
+        
+        row = row.drop('date_time')
+
+        dt_df = dt_df.join(pd.concat([pd.DataFrame(row).T]*len(dt_df),ignore_index=True))
+
+        forecast_df = pd.concat((forecast_df,dt_df),ignore_index=True)
+
+    # add extra dry date 
+    additional_date_df = forecast_df[['Name',wbdcol_name]].merge(bounding_boxes[['additional_date',wbdcol_name]],left_on=wbdcol_name,right_on=wbdcol_name).drop_duplicates(ignore_index=True)
+
+    forecast_df = pd.concat((forecast_df,additional_date_df.rename(columns={'additional_date':'date_time'})),ignore_index=True)
+
+    forecast_df = forecast_df.sort_values(['Name',wbdcol_name],ignore_index=True)
+
+
+    forecast_df['date_time'] = forecast_df.apply(lambda df : df['date_time'].replace(hour=18,minute=0),axis=1)
+
+    forecast_df = forecast_df.rename(columns={wbdcol_name:'huc'})
+
+    forecast_df = construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=retrieve,workers=workers)
+   
+    if forecast_output_file is not None:
+        forecast_df.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d %H:%M:%S%Z')
+
+
+
+
+def construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=True,workers=1):
+    # make forecast file names for NWM and retrieve
+
+    #construct url
+    #url = f'{year}/{year}{month}{day}{time}.CHRTOUT_DOMAIN1.comp'
+
+    make_url = lambda df:  "https://noaa-nwm-retrospective-2-1-pds.s3.amazonaws.com/model_output/" + \
+                                          str(df['date_time'].year) + '/' + str(df['date_time'].year) + \
+                                           str(df['date_time'].month).zfill(2) + \
+                                           str(df['date_time'].day).zfill(2) + \
+                                           str(df['date_time'].hour).zfill(2) + \
+                                           str(df['date_time'].minute).zfill(2) +\
+                                           ".CHRTOUT_DOMAIN1.comp"
+        
+    def make_file_names(df,download_directory):
+        # assumes that the last segment after the / represents the file name
+        url = df['forecast_url']
+        file_name_start_pos = url.rfind("/") + 1
+        file_name = url[file_name_start_pos:]
+
+        file_name = os.path.join(download_directory,file_name)
+
+        return(file_name)
+
+
+    urls = forecast_df.apply(make_url,axis=1)
+    forecast_df['forecast_url'] = urls
+
+    file_names = forecast_df.apply(lambda df: make_file_names(df,download_directory),axis=1)
+    forecast_df['forecast_file'] = file_names
+
+    if not retrieve:
+        return(forecast_df)
+
+    download_df = forecast_df[['forecast_url','forecast_file']].drop_duplicates()
+
+
+    def download_url(url,file_name):
+        r = requests.get(url, stream=True)
+        if r.status_code == requests.codes.ok:
+             with open(file_name, 'wb') as f:
+                for data in r:
+                    f.write(data)
+        return url
+
+    pool = ThreadPoolExecutor(max_workers=workers)
+    
+    results = { pool.submit(download_url,*(url,file_name)) : (url,file_name) for idx,(url,file_name) in download_df.iterrows() }
+
+    for future in tqdm(as_completed(results),total=len(download_df)):
+
+        url,file_name = results[future]
+
+        try:
+            future.result()
+        except Exception as exc:
+            print('error',exc,url)
+
+    pool.shutdown(wait=True)
+    
+    return(forecast_df)
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
+    parser.add_argument('-b','--bounding-boxes-file', help='Bounding box file', required=True)
+    parser.add_argument('-w','--wbd', help='WBD file', required=True)
+    parser.add_argument('-f','--forecast-output-file', help='Forecast file', required=False,default=None)
+    parser.add_argument('-r','--retrieve', help='Forecast file', required=False,default=False,action='store_true')
+    parser.add_argument('-j','--workers', help='Forecast file', required=False,default=1,type=int)
+    parser.add_argument('-d','--download_directory', help='Forecast file', required=False,default=1)
+
+    args=vars(parser.parse_args())
+
+    nesdis_preprocessing(**args)
diff --git a/tools/plots/eval_plots.py b/tools/plots/eval_plots.py
new file mode 100755
index 000000000..9c29087e1
--- /dev/null
+++ b/tools/plots/eval_plots.py
@@ -0,0 +1,343 @@
+#!/usr/bin/env python3
+import pandas as pd
+from pathlib import Path
+import argparse
+from natsort import natsorted
+import geopandas as gpd
+from utils.shared_functions import filter_dataframe, boxplot, scatterplot, barplot
+def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial_ahps = False, fim_1_ms = False):
+
+    '''
+    Creates plots and summary statistics using metrics compiled from 
+    synthesize_test_cases. Required inputs are metrics_csv and workspace. 
+    Outputs include:
+        aggregate_<benchmark source>_<configuration>.csv: this csv 
+            contains the aggregated total statistics (i.e. CSI, FAR, POD)
+            using the summed area_sq_km fields
+        <benchmark source>_<configuration>_common_sites.csv: this csv 
+            contains the unique sites (e.g usgs/nws: nws_lid; ble: huc08) 
+            considered for aggregation/plots for each magnitude. The selected
+            sites occur in all versions analyzed. For example, if FIM 1,
+            FIM 2, FIM 3.0.0.3 were versions analyzed, the common sites 
+            would be those that had data for ALL versions. This 
+            analysis is then redone for each magnitude. As such, the number
+            of sites may vary with magnitude. The number of sites for each
+            magnitude is annotated on generated plots.
+        <benchmark source>_<configuration>_analyzed_data.csv: this is the 
+            dataset used to create plots and aggregate statistics. It is 
+            a subset of the input metrics file and consists of the common
+            sites.
+        csi_aggr_<benchmark source>_<configuration>.png: bar plot of the 
+            aggregated CSI scores. Number of common sites is annotated
+            (see list of sites listed in *_*_common_sites.csv).
+        csi_<benchmark source>_<configuration>.png: box plot of CSI scores 
+            (sites weighted equally). Number of common sites is annotated 
+            (see list of sites listed in *_*_common_sites.csv).
+        far_<benchmark source>_<configuration>*.png: box plot of FAR scores
+            (sites weighted equally). Number of common sites is annotated 
+            (see list of sites listed in *_*_common_sites.csv).
+        tpr_<benchmark source>_<configuration>*.png: box plot of TPR/POD 
+            scores (sites weighted equally). Number of common sites is 
+            annotated (see list of sites listed in *_*_common_sites.csv).
+        csi_scatter_<magnitude>_<configuration>*.png: scatter plot comparing 
+            two versions for a given magnitude. This is only generated if
+            there are exactly two versions analyzed.
+
+    Parameters
+    ----------
+    metrics_csv : STRING
+        Path to csv produced as part of synthesize_test_cases containing
+        all metrics across all versions.
+    workspace : STRING
+        Path to the output workspace. Subdirectories will be created 
+        reflecting the evaluation datasets.
+    versions: LIST
+        A list of versions to be aggregated/plotted. Uses the "startswith" 
+        approach. Versions should be supplied in the order they are to 
+        be plotted. For example: ['fim_', 'fb']; This will evaluate all 
+        versions that start with fim_ (e.g. fim_1, fim_2, fim_3) and any
+        feature branch that starts with "fb". To esbalish version order,
+        the fim versions are naturally sorted and then fb versions 
+        (naturally sorted) are appended. These versions are also used to 
+        filter the input metric csv as only these versions are retained 
+        for analysis. 
+    stats: LIST
+        A list of statistics to be plotted. Must be identical to column 
+        field in metrics_csv. CSI, POD, TPR are currently calculated, if 
+        additional statistics are desired formulas would need to be coded.
+    alternate_ahps_query : STRING, optional
+        The default is false. Currently the default ahps query is same 
+        as done for apg goals. If a different query is desired it can be 
+        supplied and it will supercede the default query. 
+    spatial_ahps : DICTIONARY, optional
+        The default is false. A dictionary with keys as follows: 
+            'static': Path to AHPS point file created during creation of
+                FIM 3 static libraries.
+            'evaluated': Path to extent file created during the creation
+                of the NWS/USGS AHPS preprocessing.
+            'metadata': Path to previously created file that contains 
+                metadata about each site (feature_id, wfo, rfc and etc).
+        No spatial layers will be created if set to False, if a dictionary
+        is supplied then a spatial layer is produced.
+    fim_1_ms: BOOL
+        Default is false. If True then fim_1 rows are duplicated with 
+        extent_config set to MS. This allows for FIM 1 to be included 
+        in MS plots/stats (helpful for nws/usgs ahps comparisons).
+
+    Returns
+    -------
+    all_datasets : DICT
+        Dictionary containing all datasets generated. 
+        Keys: (benchmark_source, extent_config), 
+        Values: (filtered dataframe, common sites)
+
+    '''
+    
+    #Import metrics csv as DataFrame and initialize all_datasets dictionary
+    csv_df = pd.read_csv(metrics_csv)
+
+    #fim_1_ms flag enables FIM 1 to be shown on MS plots/stats
+    if fim_1_ms:
+        #Query FIM 1 rows based on version beginning with "fim_1"
+        fim_1_rows = csv_df.query('version.str.startswith("fim_1")').copy()
+        #Set extent configuration to MS (instead of FR)
+        fim_1_rows['extent_config'] = 'MS'
+        #Append duplicate FIM 1 rows to original dataframe
+        csv_df = csv_df.append(fim_1_rows, ignore_index = True)
+        
+    #If versions are supplied then filter out    
+    if versions:
+        #Filter out versions based on supplied version list
+        metrics = csv_df.query('version.str.startswith(tuple(@versions))')
+    else:
+        metrics = csv_df
+       
+    #Group by benchmark source
+    benchmark_by_source = metrics.groupby(['benchmark_source', 'extent_config'])
+
+    #Iterate through benchmark_by_source. Pre-filter metrics dataframe 
+    #as needed (e.g. usgs/nws filter query). Then further filtering to 
+    #discard all hucs/nws_lid that are not present across all analyzed 
+    #versions for a given magnitude. The final filtered dataset is written 
+    #to a dictionary with the key (benchmark source, extent config) 
+    #and values (filtered dataframe, common sites).
+    all_datasets = {}
+    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:        
+                
+        #If source is usgs/nws define the base resolution and query 
+        #(use alternate query if passed). Append filtered datasets to 
+        #all_datasets dictionary.
+        if benchmark_source in ['usgs','nws']:
+            
+            #Set the base processing unit for the ahps runs.
+            base_resolution = 'nws_lid'
+            
+            #Default query (used for APG) it could be that bad_sites should be modified. If so pass an alternate query using the "alternate_ahps_query"
+            bad_sites = ['grfi2','ksdm7','hohn4','rwdn4']
+            query = "not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites"
+
+            #If alternate ahps evaluation query argument is passed, use that.
+            if alternate_ahps_query:
+                query = alternate_ahps_query
+
+            #Filter the dataset based on query
+            ahps_metrics = benchmark_metrics.query(query)
+            
+            #Filter out all instances where the base_resolution doesn't 
+            #exist across all desired fim versions for a given magnitude.
+            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
+                     
+        #If source is 'ble', set base_resolution and append ble dataset 
+        #to all_datasets dictionary
+        elif benchmark_source == 'ble':
+            
+            #Set the base processing unit for ble runs
+            base_resolution = 'huc'
+            
+            #Filter out all instances where base_resolution doesn't exist 
+            #across all desired fim versions for a given magnitude.
+            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(benchmark_metrics, base_resolution)
+            
+    #For each dataset in all_datasets, generate plots and aggregate statistics.
+    for (dataset_name,configuration), (dataset, sites) in all_datasets.items():
+        
+        #Define and create the output workspace as a subfolder within 
+        #the supplied workspace
+        output_workspace = Path(workspace) / dataset_name / configuration.lower()
+        output_workspace.mkdir(parents = True, exist_ok = True)         
+                
+        #Write out the filtered dataset and common sites to file
+        dataset.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_analyzed_data.csv'), index = False)
+        sites_pd = pd.DataFrame.from_dict(sites, orient = 'index').transpose()
+        sites_pd.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_common_sites.csv'), index = False)
+        
+        #set the order of the magnitudes and define base resolution.
+        if dataset_name == 'ble':
+            magnitude_order = ['100yr', '500yr']
+            base_resolution = 'huc'
+        elif dataset_name in ['usgs','nws']:
+            magnitude_order = ['action','minor','moderate','major']
+            base_resolution = 'nws_lid'
+
+        #Calculate aggregated metrics based on total_sq_km fields.
+        dataset_sums = dataset.groupby(['version', 'magnitude'])[['TP_area_km2','FP_area_km2','FN_area_km2']].sum()
+        dataset_sums['csi'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'] + dataset_sums['FN_area_km2'])
+        dataset_sums['far'] = dataset_sums['FP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'])
+        dataset_sums['pod'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FN_area_km2'])
+        dataset_sums = dataset_sums.reset_index()
+        
+        #Write aggregated metrics to file.
+        dataset_sums.to_csv(output_workspace / f'aggregate_{dataset_name}_{configuration.lower()}.csv', index = False )
+
+        #This section naturally orders analyzed versions which defines 
+        #the hue order for the generated plots.
+        #Get all versions in dataset
+        all_versions = list(dataset.version.unique())        
+        version_order = []        
+        #If versions are not specified then use all available versions 
+        #and assign to versions_list
+        if not versions:
+            versions_list = all_versions
+        #if versions are supplied assign to versions_list
+        else:
+            versions_list = versions        
+        #For each version supplied by the user
+        for version in versions_list:
+            #Select all the versions that start with the supplied version.
+            selected_versions = [sel_version for sel_version in all_versions if sel_version.startswith(version)]
+            #Naturally sort selected_versions
+            selected_versions = natsorted(selected_versions)
+            #Populate version order based on the sorted subsets.
+            version_order.extend(selected_versions)
+            
+        #Define textbox which will contain the counts of each magnitude.
+        textbox = []
+        for magnitude in sites:
+            count = len(sites[magnitude])
+            line_text = f'{magnitude.title()} Sites = {count}'
+            textbox.append(line_text)
+        textbox = '\n'.join(textbox)
+
+        #Create aggregate barplot
+        aggregate_file = output_workspace / (f'csi_aggr_{dataset_name}_{configuration.lower()}.png')        
+        barplot(dataframe = dataset_sums, x_field = 'magnitude', x_order = magnitude_order, y_field = 'csi', hue_field = 'version', ordered_hue = version_order, title_text = f'Aggregate {dataset_name.upper()} FIM Scores', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = aggregate_file)
+        
+        #Create box plots for each metric in supplied stats.
+        for stat in stats:
+            output_file = output_workspace / (f'{stat.lower()}_{dataset_name}_{configuration.lower()}.png')    
+            boxplot(dataframe = dataset, x_field = 'magnitude', x_order = magnitude_order, y_field = stat, hue_field = 'version', ordered_hue = version_order, title_text = f'{dataset_name.upper()} FIM Sites', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = output_file)
+        
+        #Get the last 2 versions from the version order for scatter plot.
+        if len(version_order) == 2:            
+            x_version, y_version = version_order
+            for magnitude in magnitude_order:
+                #Scatterplot comparison between last 2 versions.
+                x_csi = dataset.query(f'version == "{x_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
+                y_csi = dataset.query(f'version == "{y_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
+                plotdf = pd.merge(x_csi, y_csi, on = base_resolution, suffixes = (f"_{x_version}",f"_{y_version}"))
+                #Define arguments for scatterplot function.
+                title_text = f'CSI {magnitude}'
+                dest_file = output_workspace / f'csi_scatter_{magnitude}_{configuration.lower()}.png'
+                scatterplot(dataframe = plotdf, x_field = f'CSI_{x_version}', y_field = f'CSI_{y_version}', title_text = title_text, annotate = False, dest_file = dest_file)
+    
+
+    #######################################################################
+    #Create spatial layers with threshold and mapping information
+    ########################################################################
+    if spatial_ahps:
+
+        #Read in supplied shapefile layers
+        #Layer containing metadata for each site (feature_id, wfo, etc). 
+        #Convert nws_lid to lower case.
+        ahps_metadata = gpd.read_file(spatial_ahps['metadata'])
+        ahps_metadata['nws_lid'] = ahps_metadata['nws_lid'].str.lower()
+        metadata_crs = ahps_metadata.crs
+        
+        #Extent layer generated from preprocessing NWS/USGS datasets
+        evaluated_ahps_extent = gpd.read_file(spatial_ahps['evaluated'])
+        
+        #Extent layer generated from static ahps library preprocessing
+        static_library = gpd.read_file(spatial_ahps['static'])
+        
+        #Fields to keep
+        #Get list of fields to keep in merge
+        preserved_static_library_fields = ['nws_lid'] + [i for i in static_library.columns if i.startswith(('Q','S'))]                
+        #Get list of fields to keep in merge.
+        preserved_evaluated_ahps_fields = ['nws_lid', 'source', 'geometry'] + [i for i in evaluated_ahps_extent.columns if i.startswith(('action','minor','moderate','major'))]
+
+        #Join tables to evaluated_ahps_extent
+        evaluated_ahps_extent = evaluated_ahps_extent[preserved_evaluated_ahps_fields]
+        evaluated_ahps_extent = evaluated_ahps_extent.merge(ahps_metadata, on = 'nws_lid')
+        evaluated_ahps_extent['geometry'] = evaluated_ahps_extent['geometry_y']
+        evaluated_ahps_extent.drop(columns = ['geometry_y','geometry_x'], inplace = True)
+        evaluated_ahps_extent = evaluated_ahps_extent.merge(static_library[preserved_static_library_fields], on = 'nws_lid')
+        
+        #Join dataset metrics to evaluated_ahps_extent data. 
+        final_join = pd.DataFrame()
+        for (dataset_name, configuration), (dataset, sites) in all_datasets.items():
+            #Only select ahps from dataset if config is MS
+            if dataset_name in ['usgs','nws'] and configuration == 'MS':
+                #Select records from evaluated_ahps_extent that match the dataset name
+                subset = evaluated_ahps_extent.query(f'source == "{dataset_name}"')                        
+                #Join to dataset
+                dataset_with_subset = dataset.merge(subset, on = 'nws_lid')
+                #Append rows to final_join dataframe
+                final_join = final_join.append(dataset_with_subset)
+        
+        #Modify version field
+        final_join['version'] = final_join.version.str.split('_nws|_usgs').str[0]
+        
+        #Write geodataframe to file
+        gdf = gpd.GeoDataFrame(final_join, geometry = final_join['geometry'], crs = metadata_crs)
+        output_shapefile = Path(workspace) / 'nws_usgs_site_info.shp'
+        gdf.to_file(output_shapefile) 
+                    
+
+
+#######################################################################
+if __name__ == '__main__':
+    #Parse arguments
+    parser = argparse.ArgumentParser(description = 'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
+    parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
+    parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
+    parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
+    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR'], required = False)
+    parser.add_argument('-q', '--alternate_ahps_query',help = 'Alternate filter query for AHPS. Default is: "not nws_lid.isnull() & not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites" where bad_sites are (grfi2,ksdm7,hohn4,rwdn4)', default = False, required = False)
+    parser.add_argument('-sp', '--spatial_ahps', help = 'If spatial point layer is desired, supply a csv with 3 lines of the following format: metadata, path/to/metadata/shapefile\nevaluated, path/to/evaluated/shapefile\nstatic, path/to/static/shapefile.', default = False, required = False)
+    parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
+    
+    #Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+    
+    #If errors occur reassign error to True
+    error = False
+    #Create dictionary if file specified for spatial_ahps
+    if args['spatial_ahps']:
+        #Create dictionary
+        spatial_dict = {}
+        with open(args['spatial_ahps']) as file:
+            for line in file:
+                key, value = line.strip('\n').split(',')
+                spatial_dict[key] = Path(value)
+        args['spatial_ahps'] = spatial_dict
+        #Check that all required keys are present and overwrite args with spatial_dict
+        required_keys = set(['metadata', 'evaluated', 'static'])
+        if required_keys - spatial_dict.keys():
+          print('\n Required keys are: metadata, evaluated, static')
+          error = True
+        else:
+            args['spatial_ahps'] = spatial_dict
+
+
+    #Finalize Variables
+    m = args['metrics_csv']
+    w = args['workspace']
+    v = args['versions']
+    s = args['stats']
+    q = args['alternate_ahps_query']
+    sp= args['spatial_ahps']
+    f = args['fim_1_ms']
+
+    #Run eval_plots function
+    if not error:        
+        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f)
\ No newline at end of file
diff --git a/tools/plots/utils/__init__.py b/tools/plots/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tools/plots/utils/shared_functions.py b/tools/plots/utils/shared_functions.py
new file mode 100644
index 000000000..60342059e
--- /dev/null
+++ b/tools/plots/utils/shared_functions.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import re
+#########################################################################
+#Create boxplot
+#########################################################################
+def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):      
+    '''
+    Create boxplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis
+    x_order : List
+        Order to arrange the x-axis.
+    y_field : STR
+        Field to use for the y-axis
+    hue_field : STR
+        Field to use for hue (typically FIM version)
+    title_text : STR
+        Text for plot title.
+    fim_configuration: STR
+        Configuration of FIM (FR or MS or Composite).
+    simplify_legend : BOOL, optional
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        The default is False.
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. The default is False.
+
+    Returns
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    #Use seaborn to plot the boxplot
+    axes=sns.boxplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
+    #set title of plot
+    axes.set_title(f'{title_text} ({y_field})',fontsize=20, weight = 'bold')
+    #Set yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    for index,ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel('',fontsize=0,weight = 'bold')
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+    axes.legend(markerscale = 2, fontsize =20, loc = 'lower left')
+    
+    #If simple legend desired
+    if simplify_legend:
+        #trim labels to FIM 1, FIM 2, and the FIM 3 version    
+        handles, org_labels = axes.get_legend_handles_labels()
+        label_dict = {}
+        for label in org_labels:
+            if 'fim_1' in label:
+                label_dict[label] = 'FIM 1'
+            elif 'fim_2' in label:
+                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()                  
+            elif 'fim_3' in label:
+                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                if label.endswith('_c'):
+                    label_dict[label] = label_dict[label] + ' c'
+            else:
+                label_dict[label] = label + ' ' + fim_configuration.lower()
+        #Define simplified labels as a list.
+        new_labels = [label_dict[label] for label in org_labels]
+        #Define legend location. FAR needs to be in different location than CSI/POD.
+        if y_field == 'FAR':
+            legend_location = 'upper right'
+        else:
+            legend_location = 'lower left' 
+        #rename legend labels to the simplified labels.
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
+    #Print textbox if supplied
+    if textbox_str:
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=14, verticalalignment='top', bbox=box_props)
+
+    #If figure to be saved to disk, then do so, otherwise return figure
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+
+#########################################################################
+#Create scatter plot
+#########################################################################
+def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):      
+    '''
+    Create boxplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis (Assumes FIM 2)
+    y_field : STR
+        Field to use for the y-axis (Assumes FIM 3)
+    title_text : STR
+        Text for plot title.    
+    stats_text : STR or BOOL
+        Text for stats to place on chart. Default is false (no stats printed)
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. The default is False.
+
+    Returnsy
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    
+    #Use seaborn to plot the boxplot
+    axes=sns.scatterplot(data=dataframe, x=x_field, y=y_field, color = 'black', s = 150)
+    
+    #Set xticks and yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
+    axes.grid(b=True, which='major', axis='both')    
+    
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel(f'{x_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
+
+    #Plot diagonal line
+    diag_range = [0,1]
+    axes.plot(diag_range, diag_range, color='gray', transform=axes.transAxes)
+
+
+    #set title of plot
+    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
+   
+    if annotate:
+        #Set text for labels
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        textbox_str = 'Target Better'
+        axes.text(0.3, 0.6, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+        textbox_str = 'Baseline Better'
+        axes.text(0.5, 0.2, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+
+    if stats_text:
+        #Add statistics textbox
+        axes.text(0.01, 0.80, stats_text, transform=axes.transAxes, fontsize=24, verticalalignment='top', bbox=box_props)
+
+    #If figure to be saved to disk, then do so, otherwise return fig
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+#########################################################################
+#Create barplot
+#########################################################################
+def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):      
+    '''
+    Create barplots. 
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        Pandas dataframe data to be plotted.
+    x_field : STR
+        Field to use for x-axis
+    x_order : List
+        Order to arrange the x-axis.
+    y_field : STR
+        Field to use for the y-axis
+    hue_field : STR
+        Field to use for hue (typically FIM version)
+    title_text : STR
+        Text for plot title.
+    fim_configuration: STR
+        Configuration of FIM (FR or MS or Composite).
+    simplify_legend : BOOL, optional
+        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
+        Default is False.
+    display_values : BOOL, optional
+        If True, Y values will be displayed above bars. 
+        Default is False.
+    dest_file : STR or BOOL, optional
+        If STR provide the full path to the figure to be saved. If False 
+        no plot is saved to disk. Default is False.
+
+    Returns
+    -------
+    fig : MATPLOTLIB
+        Plot.
+
+    '''
+
+    #initialize plot
+    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
+    #Use seaborn to plot the boxplot
+    axes=sns.barplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
+    #set title of plot
+    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
+    #Set yticks and background horizontal line.
+    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
+    for index,ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
+    #Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.upper()}',fontsize='xx-large',weight = 'bold')
+    axes.set_xlabel('',fontsize=0,weight = 'bold')
+    #Set sizes of ticks and legend.
+    axes.tick_params(labelsize = 'xx-large')
+    axes.legend(markerscale = 2, fontsize =20, loc = 'upper right')
+    #If simple legend desired
+    if simplify_legend:
+        #trim labels to FIM 1, FIM 2, FIM 3    
+        handles, org_labels = axes.get_legend_handles_labels()
+        label_dict = {}
+        for label in org_labels:
+            if 'fim_1' in label:
+                label_dict[label] = 'FIM 1'
+            elif 'fim_2' in label:
+                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
+            elif 'fim_3' in label:
+                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                if label.endswith('_c'):
+                    label_dict[label] = label_dict[label] + ' c'
+            else:
+                label_dict[label] = label + ' ' + fim_configuration.lower()
+        #Define simplified labels as a list.
+        new_labels = [label_dict[label] for label in org_labels]
+        #rename legend labels to the simplified labels.
+        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
+    #Add Textbox
+    if textbox_str:
+        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
+        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=18, verticalalignment='top', bbox=box_props)
+
+    #Display Y values above bars
+    if display_values:
+        #Add values of bars directly above bar.
+        for patch in axes.patches:
+            value = round(patch.get_height(),3)
+            axes.text(patch.get_x()+patch.get_width()/2.,
+                    patch.get_height(),
+                    '{:1.3f}'.format(value),
+                    ha="center", fontsize=18)  
+    
+    #If figure to be saved to disk, then do so, otherwise return fig
+    if dest_file:
+        fig.savefig(dest_file)
+        plt.close(fig)
+    else:
+        return fig
+#######################################################################
+#Filter dataframe generated from csv file from run_test_case aggregation
+########################################################################
+def filter_dataframe(dataframe, unique_field):
+    '''
+
+    This script will filter out the sites (or hucs) which are not consistently 
+    found for all versions for a given magnitude. For example, an AHPS 
+    lid site must have output for all 3 versions (fim1, fim2, fim3) for 
+    a given magnitude (eg action) otherwise that lid is filtered out. 
+    Likewise for a BLE a huc must have output for all 3 versions 
+    (fim1, fim2, fim3) for a given magnitude (eg 100yr) otherwise it is 
+    filtered out.
+
+    Parameters
+    ----------
+    dataframe : Pandas DataFrame
+        Containing the input metrics originating from synthesize_test_cases
+    unique_field : STR
+        base resolution for each benchmark source: 'nws'/'usgs' (nws_lid)
+        ble (huc).
+
+    Returns
+    -------
+    final_filtered_dataframe : Pandas Dataframe
+        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude. 
+    unique_sites: DICT
+        The sites that were included in the dataframe for each magnitude.
+
+    '''
+    
+    #Get lists of sites for each magnitude/version
+    unique_sites = dataframe.groupby(['magnitude','version'])[unique_field].agg('unique')
+    #Get unique magnitudes
+    magnitudes = dataframe.magnitude.unique()
+    #Create new dataframe to hold metrics for the common sites as well as the actual lists of common sites.
+    final_filtered_dataframe = pd.DataFrame()
+    all_unique_sites = {}
+    #Cycle through each magnitude
+    for magnitude in magnitudes:
+        #Compile a list of sets containing unique lids pertaining to each threshold. List contains 3 unique sets [{fim1:unique lids},{fim2: unique lids},{fim3: unique lids}]
+        sites_per_magnitude=[set(a) for a in unique_sites[magnitude]]
+        #Intersect the sets to get the common lids per threshold then convert to list.
+        common_sites_per_magnitude = list(set.intersection(*sites_per_magnitude))
+        #Write common sites to dataframe
+        all_unique_sites[magnitude] = common_sites_per_magnitude
+        #Query filtered dataframe and only include data associated with the common sites for that magnitude
+        filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
+        #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
+        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)            
+    
+    return final_filtered_dataframe, all_unique_sites
+
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 9d4870565..756386ef7 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -4,12 +4,40 @@
 import sys
 import shutil
 import argparse
+import traceback
+from pathlib import Path
+import json
+import ast
+import pandas as pd
 
 from tools_shared_functions import compute_contingency_stats_from_rasters
-from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
+from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES, IFC_MAGNITUDE_LIST, BLE_MAGNITUDE_LIST )
 from inundation import inundate
-
-def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True):
+from gms_tools.inundate_gms import Inundate_gms
+from gms_tools.mosaic_inundation import Mosaic_inundation
+from gms_tools.overlapping_inundation import OverlapWindowMerge
+from glob import glob
+from utils.shared_variables import elev_raster_ndv
+
+def run_alpha_test( fim_run_dir, version, test_id, magnitude, 
+                calibrated, model,
+                compare_to_previous=False, archive_results=False, 
+                mask_type='filter', inclusion_area='', 
+                inclusion_area_buffer=0, light_run=False, 
+                overwrite=True, fr_run_dir=None, 
+                gms_workers=1,verbose=False,
+                gms_verbose=False
+              ):
+
+    # check eval_meta input
+    if model not in {None,'FR','MS','GMS'}:
+        raise ValueError("Model argument needs to be \'FR\', \'MS\', or \'GMS.\'")
+
+    # make bool
+    calibrated = bool( calibrated )
+
+    if (model == "MS") & (fr_run_dir is None):
+        raise ValueError("fr_run_dir argument needs to be specified with MS model")
 
     benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
     current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
@@ -24,18 +52,23 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     if os.path.exists(version_test_case_dir_parent):
         if overwrite == True:
             shutil.rmtree(version_test_case_dir_parent)
+        elif model == 'MS':
+            pass
         else:
             print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
             return
 
-    os.mkdir(version_test_case_dir_parent)
+    os.makedirs(version_test_case_dir_parent,exist_ok=True)
 
-    print("Running the alpha test for test_id: " + test_id + ", " + version + "...")
+    __vprint("Running the alpha test for test_id: " + test_id + ", " + version + "...",verbose)
     stats_modes_list = ['total_area']
 
     fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
     assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
 
+    # get hydrofabric directory
+    hydrofabric_dir = Path(fim_run_parent).parent.absolute()
+
     # Create paths to fim_run outputs for use in inundate().
     rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
     if not os.path.exists(rem):
@@ -80,6 +113,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     magnitude_list = magnitude
     if type(magnitude_list) != list:
         magnitude_list = [magnitude_list]
+        
 
 
     # Get path to validation_data_{benchmark} directory and huc_dir.
@@ -134,46 +168,115 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                 if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
                     continue
             # Run inundate.
-#            print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
+            __vprint("-----> Running inundate() to produce inundation extent for the " + magnitude + " magnitude...",verbose)
+            # The inundate adds the huc to the name so I account for that here.
+            predicted_raster_path = os.path.join(
+                                        os.path.split(inundation_raster)[0], 
+                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
+                                                )  
             try:
-                inundate_test = inundate(
-                         rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                         subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-                         depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
+                if model == 'GMS':
+                    
+                    map_file = Inundate_gms(
+                                             hydrofabric_dir=hydrofabric_dir, 
+                                             forecast=forecast, 
+                                             num_workers=gms_workers,
+                                             hucs=current_huc,
+                                             inundation_raster=inundation_raster,
+                                             inundation_polygon=None, depths_raster=None,
+                                             verbose=gms_verbose,
+                                             log_file=None,
+                                             output_fileNames=None
+                                            )
+                    
+                    mask_path_gms = os.path.join(fim_run_parent, 'wbd.gpkg')
+
+                    Mosaic_inundation(
+                                        map_file,mosaic_attribute='inundation_rasters',
+                                        mosaic_output=inundation_raster,
+                                        mask=mask_path_gms,unit_attribute_name='huc8',
+                                        nodata=elev_raster_ndv,workers=1,
+                                        remove_inputs=True,
+                                        subset=None,verbose=verbose
+                                      )
+                
+                else:
+                    inundate(
+                             rem,catchments,catchment_poly,hydro_table,forecast,
+                             mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
+                             subset_hucs=current_huc,num_workers=1,aggregate=False,
+                             inundation_raster=inundation_raster,inundation_polygon=None,
+                             depths=None,out_raster_profile=None,out_vector_profile=None,
+                             quiet=True
                         )
-                if inundate_test == 0:
-#                    print("-----> Inundation mapping complete.")
-                    predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
-
-                    # Define outputs for agreement_raster, stats_json, and stats_csv.
-                    if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                        agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-                    else:
-                        agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-
-                    compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                           benchmark_raster_path,
-                                                           agreement_raster,
-                                                           stats_csv=stats_csv,
-                                                           stats_json=stats_json,
-                                                           mask_values=[],
-                                                           stats_modes_list=stats_modes_list,
-                                                           test_id=test_id,
-                                                           mask_dict=mask_dict,
-                                                           )
-
-                    if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                        del mask_dict[ahps_lid]
-
-                    print(" ")
-                    print("Evaluation metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
-                    print(" ")
-                elif inundate_test == 1:
-                    pass
-                    print (f"No matching feature IDs between forecast and hydrotable for magnitude: {magnitude}")
-                    #return
+
+                if model =='MS':
+                    
+                    # Mainstems inundation
+                    #fr_run_parent = os.path.join(os.environ['outputDataDir'], fr_run_dir,current_huc)
+                    #assert os.path.exists(fr_run_parent), "Cannot locate " + fr_run_parent
+                    
+                    inundation_raster_ms = os.path.join(
+                                        os.path.split(inundation_raster)[0], 
+                                        os.path.split(inundation_raster)[1].replace('.tif', '_{}_MS.tif'.format(current_huc))
+                                           )  
+                    inundation_raster_fr = os.path.join(
+                                        os.path.split(version_test_case_dir_parent)[0],
+                                        fr_run_dir,
+                                        magnitude,
+                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
+                                           )  
+                    
+                    os.rename(predicted_raster_path,inundation_raster_ms)
+
+                    ms_inundation_map_file = { 
+                                               'huc8' : [current_huc] * 2,
+                                               'branchID' : [None] * 2,
+                                               'inundation_rasters' : [inundation_raster_fr,inundation_raster_ms],
+                                               'depths_rasters' : [None] * 2,
+                                               'inundation_polygons' : [None] * 2
+                                             }
+                    ms_inundation_map_file = pd.DataFrame(ms_inundation_map_file)
+                    
+                    Mosaic_inundation(
+                                        ms_inundation_map_file,mosaic_attribute='inundation_rasters',
+                                        mosaic_output=inundation_raster,
+                                        mask=catchment_poly,unit_attribute_name='huc8',
+                                        nodata=elev_raster_ndv,workers=1,
+                                        remove_inputs=False,
+                                        subset=None,verbose=verbose
+                                      )
+
+                __vprint("-----> Inundation mapping complete.",verbose)
+
+                # Define outputs for agreement_raster, stats_json, and stats_csv.
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+                else:
+                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+                
+                compute_contingency_stats_from_rasters(predicted_raster_path,
+                                                       benchmark_raster_path,
+                                                       agreement_raster,
+                                                       stats_csv=stats_csv,
+                                                       stats_json=stats_json,
+                                                       mask_values=[],
+                                                       stats_modes_list=stats_modes_list,
+                                                       test_id=test_id,
+                                                       mask_dict=mask_dict,
+                                                       )
+
+                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
+                    del mask_dict[ahps_lid]
+
+                __vprint(" ",verbose)
+                # print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC) # GMS
+                __vprint("Evaluation metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC,verbose) # cahaba/dev
+                __vprint(" ",verbose)
+
             except Exception as e:
-                print(e)
+                print(traceback.print_exc())
+                #print(e)
 
         if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
             # -- Delete temp files -- #
@@ -184,6 +287,17 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
                     full_output_file_path = os.path.join(version_test_case_dir, output_file)
                     os.remove(full_output_file_path)
 
+    # write out evaluation meta-data
+    with open(os.path.join(version_test_case_dir_parent,'eval_metadata.json'),'w') as meta:
+        eval_meta = { 'calibrated' : calibrated , 'model' : model }
+        meta.write( 
+                    json.dumps(eval_meta,indent=2) 
+                   )
+
+def __vprint(message,verbose):
+    if verbose:
+        print(message)
+
 
 if __name__ == '__main__':
 
@@ -192,7 +306,9 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
     parser.add_argument('-b', '--version',help='The name of the working version in which features are being tested',required=True,default="")
     parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
-    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
+    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method. MS and GMS are currently on supporting huc', required=False,default="filter")
+    parser.add_argument('-n','--calibrated',help='Denotes use of calibrated n values',required=False, default=False,action='store_true')
+    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed',required=True)
     parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
     parser.add_argument('-c', '--compare-to-previous', help='Compare to previous versions of HAND.', required=False,action='store_true')
     parser.add_argument('-a', '--archive-results', help='Automatically copy results to the "previous_version" archive for test_id. For admin use only.', required=False,action='store_true')
@@ -200,6 +316,10 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
     parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
     parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False, action='store_true')
+    parser.add_argument('-w','--gms-workers', help='Number of workers to use for GMS Branch Inundation', required=False, default=1)
+    parser.add_argument('-d','--fr-run-dir',help='Name of test case directory containing inundation for FR configuration',required=False,default=None)
+    parser.add_argument('-v', '--verbose', help='Verbose operation', required=False, action='store_true', default=False)
+    parser.add_argument('-vg', '--gms-verbose', help='Prints progress bar for GMS', required=False, action='store_true', default=False)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
@@ -207,7 +327,7 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     valid_test_id_list = os.listdir(TEST_CASES_DIR)
 
     exit_flag = False  # Default to False.
-    print()
+    __vprint("",args['verbose'])
 
     # Ensure test_id is valid.
 #    if args['test_id'] not in valid_test_id_list:
@@ -236,11 +356,15 @@ def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous
     except ValueError:
         print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area_buffer (-ib) " + CYAN_BOLD + args['inclusion_area_buffer'] + WHITE_BOLD + " is not a round number." + ENDC)
 
+    benchmark_category = args['test_id'].split('_')[1]
+
     if args['magnitude'] == '':
-        if 'ble' in args['test_id'].split('_'):
-            args['magnitude'] = ['100yr', '500yr']
-        elif 'nws' or 'usgs' in args['test_id'].split('_'):
+        if 'ble' == benchmark_category:
+            args['magnitude'] = BLE_MAGNITUDE_LIST
+        elif ('nws' == benchmark_category) | ('usgs' == benchmark_category):
             args['magnitude'] = ['action', 'minor', 'moderate', 'major']
+        elif 'ifc' == current_benchmark_category:
+            args['magnitude'] = IFC_MAGNITUDE_LIST
         else:
             print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
             exit_flag = True
diff --git a/tools/shared_variables.py b/tools/shared_variables.py
new file mode 100644
index 000000000..292f55de1
--- /dev/null
+++ b/tools/shared_variables.py
@@ -0,0 +1,20 @@
+import os
+
+# Environmental variables and constants.
+TEST_CASES_DIR = r'/data/test_cases/'
+PREVIOUS_FIM_DIR = r'/data/previous_fim'
+OUTPUTS_DIR = os.environ['outputDataDir']
+INPUTS_DIR = r'/data/inputs'
+AHPS_BENCHMARK_CATEGORIES = ['usgs', 'nws']
+PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
+GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
+GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
+
+# Colors.
+ENDC = '\033[m'
+TGREEN_BOLD = '\033[32;1m'
+TGREEN = '\033[32m'
+TRED_BOLD = '\033[31;1m'
+TWHITE = '\033[37m'
+WHITE_BOLD = '\033[37;1m'
+CYAN_BOLD = '\033[36;1m'
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index e49b1519d..e42024c4e 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -3,10 +3,14 @@
 import os
 import argparse
 from multiprocessing import Pool
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 import json
 import csv
+import ast
+from tqdm import tqdm
 
 from run_test_case import run_alpha_test
+
 from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT
 
 
@@ -86,8 +90,11 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                 magnitude_list = MAGNITUDE_DICT['ble']
             if benchmark_source == 'ifc':
                 magnitude_list = MAGNITUDE_DICT['ifc']
-            test_cases_list = os.listdir(benchmark_test_case_dir)
-
+            try:
+                test_cases_list = os.listdir(benchmark_test_case_dir)
+            except FileNotFoundError:
+                continue
+            
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
@@ -139,7 +146,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                 list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
-
+        
         if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
             test_cases_list = os.listdir(benchmark_test_case_dir)
 
@@ -156,7 +163,6 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
                         if iteration == "comparison":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
-                            versions_to_aggregate = dev_versions_to_include_list
 
                         for magnitude in ['action', 'minor', 'moderate', 'major']:
                             for version in versions_to_aggregate:
@@ -204,69 +210,57 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                 list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
-
+    
     with open(master_metrics_csv_output, 'w', newline='') as csvfile:
         csv_writer = csv.writer(csvfile)
         csv_writer.writerows(list_to_write)
 
 
-def process_alpha_test(args):
-    """
-    This function is designed to be used in multiprocessing. It handles the calling of the run_alpha_test function.
-    
-    Args:
-        args (list): Formatted [fim_run_dir (str), version (str), test_id (str), magnitude (str), archive_results (bool), overwrite (bool)]
-    
-    """
-    
-    
-    fim_run_dir = args[0]
-    version = args[1]
-    test_id = args[2]
-    magnitude = args[3]
-    archive_results = args[4]
-    overwrite = args[5]
-
-    mask_type = 'huc'
-
-    if archive_results == False:
-        compare_to_previous = True
-    else:
-        compare_to_previous = False
-
-    try:
-        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type, overwrite=overwrite)
-    except Exception as e:
-        print(e)
-
-
 if __name__ == '__main__':
 
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
+    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=False,default='DEV')
+    parser.add_argument('-l','--calibrated',help='Denotes use of calibrated n values. This should be taken from meta-data from hydrofabric dir',required=False, default=False,action='store_true')
+    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed. This should be taken from meta-data in hydrofabric dir.',required=True)
     parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
-    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
+    parser.add_argument('-jh','--job-number-huc',help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',required=False, default=1,type=int)
+    parser.add_argument('-jb','--job-number-branch',help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',required=False, default=1,type=int)
     parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
     parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
     parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
     parser.add_argument('-dc', '--dev-version-to-compare', nargs='+', help='Specify the name(s) of a dev (testing) version to include in master metrics CSV. Pass a space-delimited list.',required=False)
-    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=True)
+    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=False,default=None)
+    parser.add_argument('-d','--fr-run-dir',help='Name of test case directory containing FIM for FR model',required=False,default=None)
+    parser.add_argument('-vr','--verbose',help='Verbose',required=False,default=None,action='store_true')
+    parser.add_argument('-vg','--gms-verbose',help='GMS Verbose Progress Bar',required=False,default=None,action='store_true')
 
     # Assign variables from arguments.
     args = vars(parser.parse_args())
     config = args['config']
     fim_version = args['fim_version']
-    job_number = int(args['job_number'])
+    job_number_huc = args['job_number_huc']
+    job_number_branch = args['job_number_branch']
     special_string = args['special_string']
     benchmark_category = args['benchmark_category']
     overwrite = args['overwrite']
     dev_versions_to_compare = args['dev_version_to_compare']
     master_metrics_csv = args['master_metrics_csv']
-
-    if overwrite:
-        if input("Are you sure you want to overwrite metrics? y/n: ") == "n":
-            quit
+    fr_run_dir = args['fr_run_dir']
+    calibrated = args['calibrated']
+    model = args['model']
+    verbose = args['verbose']
+    gms_verbose = args['gms_verbose']
+
+    # check job numbers
+    total_cpus_requested = job_number_huc * job_number_branch
+    total_cpus_available = os.cpu_count() - 1
+    if total_cpus_requested > total_cpus_available:
+        raise ValueError('The HUC job number, {}, multiplied by the branch job number, {}, '\
+                          'exceeds your machine\'s available CPU count minus one. '\
+                          'Please lower the job_number_huc or job_number_branch'\
+                          'values accordingly.'.format(job_number_huc,job_number_branch)
+                        )
 
     # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
     if fim_version != "all":
@@ -296,18 +290,25 @@ def process_alpha_test(args):
         benchmark_category_list = [benchmark_category]
 
     # Loop through benchmark categories.
-    procs_list = []
+    procs_list = [] ; procs_dict = {}
     for bench_cat in benchmark_category_list:
         
         # Map path to appropriate test_cases folder and list test_ids into bench_cat_id_list.
         bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
         bench_cat_id_list = os.listdir(bench_cat_test_case_dir)
+        
+        # temp
+        #bench_cat_id_list = ['07060003_ifc']
 
+        #if job_number_huc == 1:
+            # something wrong about this lengtu
+            #pb = tqdm(total=len(bench_cat_id_list))
         # Loop through test_ids in bench_cat_id_list.
         for test_id in bench_cat_id_list:
             if 'validation' and 'other' not in test_id:
                 current_huc = test_id.split('_')[0]
-                if test_id.split('_')[1] in bench_cat:
+                current_benchmark_category = test_id.split('_')[1]
+                if current_benchmark_category in bench_cat:
                     # Loop through versions.
                     for version in previous_fim_list:
                         if config == 'DEV':
@@ -317,33 +318,89 @@ def process_alpha_test(args):
 
                         # For previous versions of HAND computed at HUC6 scale
                         if not os.path.exists(fim_run_dir):
-                            print(fim_run_dir)
                             if config == 'DEV':
                                 fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
                             elif config == 'PREV':
                                 fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])
-                                    
+                        
                         if os.path.exists(fim_run_dir):
+
                             # If a user supplies a special_string (-s), then add it to the end of the created dirs.
                             if special_string != "":
                                 version = version + '_' + special_string
 
             
                             # Define the magnitude lists to use, depending on test_id.
-                            benchmark_type = test_id.split('_')[1]
-                            magnitude = MAGNITUDE_DICT[benchmark_type]
-                            
-                            # Either add to list to multiprocess or process serially, depending on user specification.
-                            if job_number > 1:
-                                procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
+                            if 'ble' == current_benchmark_category:
+                                magnitude = MAGNITUDE_DICT['ble']
+                            elif ('usgs' == current_benchmark_category) | ('nws' == current_benchmark_category):
+                                magnitude = ['action', 'minor', 'moderate', 'major']
+                            elif 'ifc' == current_benchmark_category:
+                                magnitude_list = MAGNITUDE_DICT['ifc']
                             else:
-                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
+                                continue
+
+                            alpha_test_args = { 
+                                                'fim_run_dir': fim_run_dir, 
+                                                'version': version, 
+                                                'test_id': test_id, 
+                                                'magnitude': magnitude, 
+                                                'calibrated': calibrated,
+                                                'model': model,
+                                                'compare_to_previous': not archive_results, 
+                                                'archive_results': archive_results, 
+                                                'mask_type': 'huc',
+                                                'overwrite': overwrite, 
+                                                'fr_run_dir': fr_run_dir, 
+                                                'gms_workers': job_number_branch,
+                                                'verbose': False,
+                                                'gms_verbose': False
+                                              }
+                            
+                            procs_dict[current_huc] = alpha_test_args
+
+    if job_number_huc == 1:
+        
+        number_of_hucs = len(procs_dict)
+        verbose_by_huc = not number_of_hucs == 1
+
+        for current_huc, alpha_test_args in tqdm(procs_dict.items(),total=number_of_hucs,disable=(not verbose_by_huc)):
+            alpha_test_args.update({'gms_verbose': not verbose_by_huc})
+
+            try:
+                run_alpha_test(**alpha_test_args)
+            except Exception as exc:
+                print('{}, {}, {}'.format(test_id,exc.__class__.__name__,exc))
 
     # Multiprocess alpha test runs.
-    if job_number > 1:
-        with Pool(processes=job_number) as pool:
-            pool.map(process_alpha_test, procs_list)
-            
+    if job_number_huc > 1:
+        
+        #print(procs_dict)
+        executor = ProcessPoolExecutor(max_workers=job_number_huc)
+
+        executor_generator = { 
+                              executor.submit(run_alpha_test,**inp) : ids for ids,inp in procs_dict.items()
+                             }
+
+        for future in tqdm(as_completed(executor_generator),
+                           total=len(executor_generator),
+                           disable=(not verbose),
+                           desc="Running test cases with {} HUC workers "\
+                                "and {} Branch workers".format(job_number_huc,
+                                                               job_number_branch),
+                          ):
+        
+
+            hucCode = executor_generator[future]
+
+            try:
+                future.result()
+            except Exception as exc:
+                print('{}, {}, {}'.format(hucCode,exc.__class__.__name__,exc))
+    
+        # power down pool
+        executor.shutdown(wait=True)
+
     if config == 'DEV':
         if dev_versions_to_compare != None:
             dev_versions_to_include_list = dev_versions_to_compare + [version]
@@ -352,6 +409,9 @@ def process_alpha_test(args):
     if config == 'PREV':
         dev_versions_to_include_list = []
 
-    # Do aggregate_metrics.
-    print("Creating master metrics CSV...")
-    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, dev_versions_to_include_list=dev_versions_to_include_list)
+    if master_metrics_csv is not None:
+        # Do aggregate_metrics.
+        print("Creating master metrics CSV...")
+
+        # this function is not compatible with GMS
+        create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, versions_to_include_list=dev_versions_to_include_list)
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 6cf0e7685..2f303d642 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -499,6 +499,7 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
             operation = mask_dict[poly_layer]['operation']
 
             if operation == 'include':
+
                 poly_path = mask_dict[poly_layer]['path']
                 buffer_val = mask_dict[poly_layer]['buffer']
 
@@ -1447,4 +1448,67 @@ def process_grid(benchmark, benchmark_profile, domain, domain_profile, reference
     profile.update(nodata = new_nodata_value) 
     profile.update (width = new_benchmark_width)
     profile.update(height = new_benchmark_height)   
-    return classified_benchmark_projected, profile
\ No newline at end of file
+    
+    return classified_benchmark_projected, profile
+
+
+def calculate_metrics_from_agreement_raster(agreement_raster):
+
+    ''' Calculates metrics from an agreement raster '''
+
+    agreement_encoding_digits_to_names = { 0: "TN",
+                                           1: "FN",
+                                           2: "FP",
+                                           3: "TP"
+                                          }
+
+
+    if isinstance(agreement_raster,rasterio.DatasetReader):
+        pass
+    elif isinstance(agreement_raster,str):
+        agreement_raster = rasterio.open(agreement_raster)
+    else:
+        raise TypeError(f"{agreement_raster} is not a Rasterio Dataset Reader or a filepath to a raster")
+
+    # cycle through blocks 
+    totals = dict.from_keys(list(range(4)),0)
+    for idx,wind in agreement_raster.block_windows(1):
+        window_data = agreement_raster.read(1,window=wind)
+        values, counts = np.unique(window_data,return_counts=True)
+        for val,cts in values_counts:
+            totals[val] += cts
+
+    results = dict()
+    for digit,count in totals.items():
+        results[agreement_encoding_digits_to_names[digit]] = count
+   
+    return(results)
+
+
+# evaluation metric fucntions
+
+def csi(TP,FP,FN,TN=None):
+
+    ''' Critical Success Index '''
+
+    return TP / (FP + FN + TP)
+
+    
+def tpr(TP,FP,FN,TN=None):
+
+    ''' True Positive Rate '''
+    
+    return TP / (TP + FN)
+
+
+def far(TP,FP,FN,TN=None):
+
+    ''' False Alarm Rate '''
+
+    return FP / (TP + FP)
+
+
+def mcc(TP,FP,FN,TN=None):
+
+    ''' Matthew's Correlation Coefficient '''
+    return (TP*TN - FP*FN) / np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))

From fc8d0f80cb7b089dd2c8f5d995b0d166fb0ded06 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 2 Feb 2022 12:24:05 -0600
Subject: [PATCH 126/359] Adds basic unit test capabilities

See Changelog v4.0.1.0 for full details.
---
 docs/CHANGELOG.md                             |  24 ++++
 unit_tests/README.md                          |  79 ++++++++++++
 unit_tests/clip_vectors_to_wbd_params.json    |  23 ++++
 unit_tests/clip_vectors_to_wbd_unittests.py   |  94 ++++++++++++++
 unit_tests/gms/derive_level_paths_params.json |  14 ++
 .../gms/derive_level_paths_unittests.py       | 121 ++++++++++++++++++
 unit_tests/unit_tests_utils.py                |  11 ++
 7 files changed, 366 insertions(+)
 create mode 100644 unit_tests/README.md
 create mode 100644 unit_tests/clip_vectors_to_wbd_params.json
 create mode 100644 unit_tests/clip_vectors_to_wbd_unittests.py
 create mode 100644 unit_tests/gms/derive_level_paths_params.json
 create mode 100644 unit_tests/gms/derive_level_paths_unittests.py
 create mode 100644 unit_tests/unit_tests_utils.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index de3a8ac9a..9924ef72d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,30 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.1.0 - 2022-02-02 - [PR #525](https://github.com/NOAA-OWP/cahaba/pull/525)
+
+The addition of a very simple and evolving unit test system which has two unit tests against two py files.  This will set a precendence and will grow over time and may be automated, possibly during git check-in triggered. The embedded README.md has more details of what we currently have, how to use it, how to add new unit tests, and expected future enhancements.
+
+## Additions
+
+- `/unit_tests/` folder which has the following:
+
+   - `clip_vectors_to_wbd_params.json`: A set of default "happy path" values that are expected to pass validation for the clip_vectors_to_wbd.py -> clip_vectors_to_wbd (function).
+
+   - `clip_vectors_to_wbd_unittests.py`: A unit test file for src/clip_vectors_to_wbd.py. Incomplete but evolving.
+
+   - `README.md`: Some information about how to create unit tests and how to use them.
+
+   - `unit_tests_utils.py`: A python file where methods that are common to all unit tests can be placed.
+
+   - `gms/derive_level_paths_params.json`: A set of default "happy path" values that are expected to pass validation for the derive_level_paths_params.py -> Derive_level_paths (function). 
+
+   - `gms/derive_level_paths_unittests.py`: A unit test file for `src/derive_level_paths.py`. Incomplete but evolving.
+
+<br/><br/>
+
+
 ## v4.0.0.0 - 2022-02-01 - [PR #524](https://github.com/NOAA-OWP/cahaba/pull/524)
 
 FIM4 builds upon FIM3 and allows for better representation of inundation through the reduction of artificial restriction of inundation at catchment boundaries.
diff --git a/unit_tests/README.md b/unit_tests/README.md
new file mode 100644
index 000000000..b599ace65
--- /dev/null
+++ b/unit_tests/README.md
@@ -0,0 +1,79 @@
+## Cahaba: Flood Inundation Mapping for U.S. National Water Model
+
+Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
+
+#### For more information, see the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki).
+
+### This folder and files are for unit testing python files
+
+## Creating unit tests
+
+For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and it's json parms file. 
+
+The files should be named following FIM convention:
+
+{source py file name}_unittests.py     ie) derive_level_paths_unittests.py
+{source py file name}_params.json      ie) derive_level_paths_params.json
+
+
+## Tips to creating a new json file for a new unit test file.
+
+There are multiple way to figure out a set of default json parameters for the new unit test file. 
+
+One way is to use the incoming arg parser. Most py files, they include the code block of " __name__ == '__main__':", followed by external arg parsing, ie) the args = vars(parser.parse_args()). 
+	- Add a print(args) or similar, and get all the values including keys as output.
+	- Copy that into an editor being used to create the json file.
+	- Add a line break after every comma.
+	- Find/replace all single quotes to double quotes then cleanup the left tab formatting.
+
+
+## Running unit tests
+
+Start a docker container as you normally would for any development. ie) docker run --rm -it --name rob_levelpaths -v /home/{your name}/projects/gms_ms_level_paths/fim_4/:/foss_fim {your docker image name}
+
+At the root terminal window, run:  python ./foss_fim/unit_tests/gms/derive_level_paths_unittests.py  or python ./foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
+(replace with your own script and path name)
+
+
+## Key Notes for creating new unit tests
+1) All test functions must start with the phrase "test_". That is how the unit test engine picks it up. The rest of the function name does not have to match the pattern of {function name being tested} but should. Further, the rest of the function name should say what the test is about, ie) _failed_input_path.  ie) test_{some_function_name_from_the_source_code_file}_failed_input_path. It is fine that the function names get very long (common in the industry).
+
+2) The output for a selected "unittest" import engine can be ugly and hard to read. It sometimes mixed outputs from multiple unit test functions simulataneously instead of keeping all output together for a given unit test. We will try to make this better later.
+
+3) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
+
+4) You can not control the order that unit tests are run within a unit test file. (UnitTest engine limitation)
+
+5) There must be at one "{original py file name}_params.json" file.
+
+6) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to full pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass.
+
+7) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output, but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
+
+7) One py file = one "{original py file name}_unittests.py" file.
+
+
+## Future Enhancements
+1) We can automate triggers on these files for things like checking triggers or an single global "run_all_unittest" script, but for now.. its one offs.
+
+2) Better output from the unit tests.
+
+3) We will upgrade it so you can pass in a params.json file to the call to give you even more flexibility by passing in your own params.json (think params_template.json idea)
+
+4) At this time, the root json file has only one "node". We may consider have more than one node in the json file, which has a different set of data for different test conditions. 
+
+5) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
+
+
+## testing for failing conditions
+- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for `ONLY one variable value change`. Aka.. a "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
+
+- It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
+
+- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass. 
+
+An example is in unit_tests/gms/Derive_level_paths_unittests.py -> test_Derive_level_paths_invalid_input_stream_network (function). It is incomplete but give you the pattern.
+
+We have almost no "assert"s yet, but most unit test usually have one or more "assert" test. See https://docs.python.org/3/library/unittest.html for more details.
+
+
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
new file mode 100644
index 000000000..3f1b0f787
--- /dev/null
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -0,0 +1,23 @@
+{
+	"hucCode": "05030104",
+	"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
+	"nhd_streams": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg", 
+	"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
+	"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
+	"wbd": "/data/outputs/level_paths_3/05030104/wbd.gpkg",
+	"wbd_buffer": "/data/outputs/level_paths_3/05030104/wbd_buffered.gpkg",
+	"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", 
+	"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
+	"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
+	"subset_nhd_streams": "/data/outputs/level_paths_3/05030104/NHDPlusBurnLineEvent_subset.gpkg",
+	"subset_nld_lines": "/data/outputs/level_paths_3/05030104/nld_subset_levees.gpkg", 
+	"subset_lakes": "/data/outputs/level_paths_3/05030104/nwm_lakes_proj_subset.gpkg",
+	"subset_catchments": "/data/outputs/level_paths_3/05030104/nwm_catchments_proj_subset.gpkg",
+	"subset_nhd_headwaters": "/data/outputs/level_paths_3/05030104/nhd_headwater_points_subset.gpkg", 
+	"subset_nwm_streams": "/data/outputs/level_paths_3/05030104/nwm_subset_streams.gpkg", 
+	"subset_landsea": "/data/outputs/level_paths_3/05030104/LandSea_subset.gpkg", 
+	"extent": "GMS",
+	"great_lakes_filename": "/data/inputs/landsea/gl_water_polygons.gpkg", 
+	"wbd_buffer_distance": 5000,
+	"lake_buffer_distance": 20
+}
\ No newline at end of file
diff --git a/unit_tests/clip_vectors_to_wbd_unittests.py b/unit_tests/clip_vectors_to_wbd_unittests.py
new file mode 100644
index 000000000..322b4ac4b
--- /dev/null
+++ b/unit_tests/clip_vectors_to_wbd_unittests.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import unittest
+
+import unit_tests_utils as helpers
+
+# importing python folders in other direcories
+sys.path.append('/foss_fim/src/')
+import clip_vectors_to_wbd
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_clip_vectors_to_wbd(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = '/foss_fim/unit_tests/clip_vectors_to_wbd_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    def test_subset_vector_layers_success(self):
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params
+        
+        '''
+        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
+        Most of the output test and internal tests with this function will test a wide variety of conditions.
+        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
+        Only the basic return output value shoudl be tested to ensure it is as expected.
+        For now, we are adding the very basic "happy path" test.
+        '''
+       
+        # There is no default return value.
+        # Later we need to check in this function that files were being created on the file system
+        
+        # for now we are happy if no exceptions are thrown.
+        clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
+                                                nwm_streams_filename = params["nwm_streams"],
+                                                nhd_streams_filename = params["nhd_streams"],
+                                                nwm_lakes_filename = params["nwm_lakes"],
+                                                nld_lines_filename = params["nld_lines"],
+                                                nwm_catchments_filename = params["nwm_catchments"],
+                                                nhd_headwaters_filename = params["nhd_headwaters"],
+                                                landsea_filename = params["landsea"],
+                                                wbd_filename = params["wbd"],
+                                                wbd_buffer_filename = params["wbd_buffer"],
+                                                subset_nhd_streams_filename = params["subset_nhd_streams"],
+                                                subset_nld_lines_filename = params["subset_nld_lines"],
+                                                subset_nwm_lakes_filename = params["subset_lakes"],
+                                                subset_nwm_catchments_filename = params["subset_catchments"],
+                                                subset_nhd_headwaters_filename = params["subset_nhd_headwaters"],
+                                                subset_nwm_streams_filename = params["subset_nwm_streams"],
+                                                subset_landsea_filename = params["subset_landsea"],
+                                                extent = params["extent"],
+                                                great_lakes_filename = params["great_lakes_filename"],
+                                                wbd_buffer_distance = params["wbd_buffer_distance"],
+                                                lake_buffer_distance = params["lake_buffer_distance"])
+       
+       
+
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/gms/derive_level_paths_params.json b/unit_tests/gms/derive_level_paths_params.json
new file mode 100644
index 000000000..68a30f802
--- /dev/null
+++ b/unit_tests/gms/derive_level_paths_params.json
@@ -0,0 +1,14 @@
+{
+	"in_stream_network": "/data/outputs/level_paths_1/03110201/nwm_subset_streams.gpkg",
+	"out_stream_network": "/data/outputs/level_paths_1/03110201/nwm_subset_streams_levelPaths.gpkg",
+	"branch_id_attribute": "levpa_id",
+	"out_stream_network_dissolved": "/data/outputs/level_paths_1/03110201/nwm_subset_streams_levelPaths.gpkg",
+	"headwaters_outfile": "/data/outputs/level_paths_1/03110201/nwm_headwaters.gpkg",
+	"catchments": "/data/outputs/level_paths_1/03110201/nwm_catchments_proj_subset.gpkg",
+	"catchments_outfile": "/data/outputs/level_paths_1/03110201/nwm_catchments_proj_subset_levelPaths.gpkg",
+	"branch_inlets_outfile": "/data/outputs/level_paths_1/03110201/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+	"reach_id_attribute": "ID",
+	"verbose": false
+}
+
+
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
new file mode 100644
index 000000000..c7d9f0085
--- /dev/null
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+import unittest
+
+import json
+
+# importing python folders in other direcories
+sys.path.append('/foss_fim/unit_tests/')
+import unit_tests_utils as helpers
+
+# importing python folders in other direcories
+sys.path.append('/foss_fim/src/gms/')
+import derive_level_paths
+
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_Derive_level_paths(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+    
+        params_file_path = '/foss_fim/unit_tests/gms/derive_level_paths_params.json'    
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    def test_Derive_level_paths_success(self):
+       
+        helpers.print_unit_test_function_header()
+        
+        params = self.params
+
+        # Function Notes:
+        # huc_ids no longer used, so it is not submitted
+        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
+        # skip them here.
+        actual = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                                       out_stream_network = params["out_stream_network"],
+                                                       branch_id_attribute = params["branch_id_attribute"],
+                                                       out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                                       headwaters_outfile = params["headwaters_outfile"],
+                                                       catchments = params["catchments"],
+                                                       catchments_outfile = params["catchments_outfile"],
+                                                       branch_inlets_outfile = params["branch_inlets_outfile"],
+                                                       reach_id_attribute = params["reach_id_attribute"],
+                                                       verbose = params["verbose"] )
+        
+        
+        '''
+        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
+        Most of the output test and internal tests with this function will test a wide variety of conditions.
+        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
+        Only the basic return output value shoudl be tested to ensure it is as expected.
+        For now, we are adding the very basic "happy path" test.
+        '''
+        
+        print(actual)
+        
+        # Later, we should check here that the files outputed to the file system, exist and are valid and those do not have to be new "test_" methods.
+        # We can start using unit test "asserts" as we go as well.
+        
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")        
+
+
+    # Invalid Input stream
+    # MUST start with the name of "test_"    
+    def test_Derive_level_paths_invalid_input_stream_network(self):
+
+        helpers.print_unit_test_function_header()        
+        
+        try:
+        
+            params = self.params
+            #with open("/foss_fim/unit_tests/gms/derive_level_paths_params.json") as params_file:
+            #    params = json.load(params_file)
+
+            bad_input_stream_network = "some bad path"
+           
+            actual = derive_level_paths.Derive_level_paths(in_stream_network = bad_input_stream_network,
+                                                           out_stream_network = params["out_stream_network"],
+                                                           branch_id_attribute = params["branch_id_attribute"],
+                                                           out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                                           huc_id = params["huc_id"],
+                                                           headwaters_outfile = params["headwaters_outfile"],
+                                                           catchments = params["catchments"],
+                                                           catchments_outfile = params["catchments_outfile"],
+                                                           branch_inlets_outfile = params["branch_inlets_outfile"],
+                                                           reach_id_attribute = params["reach_id_attribute"],
+                                                           verbose = params["verbose"] )
+                                                           
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+            
+        except Exception as ex:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            print(ex)
+            
+        finally:
+            print("*************************************************************")             
+        
+if __name__ == '__main__':
+
+    print("*****************************")
+    print(f"Start of {os.path.basename(__file__)} tests")
+    print()
+            
+    unittest.main()
+    
+    print()    
+    print(f"End of {os.path.basename(__file__)} tests")
+    
diff --git a/unit_tests/unit_tests_utils.py b/unit_tests/unit_tests_utils.py
new file mode 100644
index 000000000..5db453323
--- /dev/null
+++ b/unit_tests/unit_tests_utils.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+import sys
+
+
+def print_unit_test_function_header():
+
+    print()    
+    print("*************************************************************") 
+    print(f"Start function: {sys._getframe(1).f_code.co_name}")
+    print()

From 8eba0eed5f924377e79fa5873d335b1483b8f807 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 2 Feb 2022 15:25:32 -0600
Subject: [PATCH 127/359] Preliminary readme fixes and deletion of old
 Changelog

---
 CHANGELOG.md | 635 ---------------------------------------------------
 README.md    |  20 +-
 2 files changed, 15 insertions(+), 640 deletions(-)
 delete mode 100644 CHANGELOG.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
deleted file mode 100644
index af6f11b61..000000000
--- a/CHANGELOG.md
+++ /dev/null
@@ -1,635 +0,0 @@
-All notable changes to this project will be documented in this file.
-We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
-<br/><br/>
-
-## v3.0.19.3 - 2021-07-09
-
-Hot fix to `synthesize_test_cases`.
-
-## Changes
-- Fixed if/elif/else statement in `synthesize_test_cases.py` that resulted in only IFC data being evaluated.
-
-<br/><br/>
-
-## v3.0.19.2 - 2021-07-01 - [PR #429](https://github.com/NOAA-OWP/cahaba/pull/429)
-
-Updates to evaluation scripts to allow for Alpha testing at Iowa Flood Center (IFC) sites. Also, `BAD_SITES` variable updates to omit sites not suitable for evaluation from metric calculations.
-
-## Changes
-- The `BAD_SITES` list in `tools_shared_variables.py` was updated and reasons for site omission are documented.
-- Refactored `run_test_case.py`, `synthesize_test_cases.py`, `tools_shared_variables.py`, and `eval_plots.py` to allow for IFC comparisons.
-
-<br/><br/>
-
-## v3.0.19.1 - 2021-06-17 - [PR #417](https://github.com/NOAA-OWP/cahaba/pull/417)
-
-Adding a thalweg profile tool to identify significant drops in thalweg elevation. Also setting lateral thalweg adjustment threshold in hydroconditioning.
-
-## Additions
-- `thalweg_drop_check.py` checks the elevation along the thalweg for each stream path downstream of MS headwaters within a HUC.
-
-## Removals
-- Removing `dissolveLinks` arg from `clip_vectors_to_wbd.py`.
-
-
-## Changes
-- Cleaned up code in `split_flows.py` to make it more readable.
-- Refactored `reduce_nhd_stream_density.py` and `adjust_headwater_streams.py` to limit MS headwater points in `agg_nhd_headwaters_adj.gpkg`.
-- Fixed a bug in `adjust_thalweg_lateral.py` lateral elevation replacement threshold; changed threshold to 3 meters.
-- Updated `aggregate_vector_inputs.py` to log intermediate processes.
-
-<br/><br/>
-
-## v3.0.19.0 - 2021-06-10 - [PR #415](https://github.com/NOAA-OWP/cahaba/pull/415)
-
-Feature to evaluate performance of alternative CatFIM techniques.
-
-## Additions
-- Added `eval_catfim_alt.py` to evaluate performance of alternative CatFIM techniques.
-
-<br/><br/>
-## v3.0.18.0 - 2021-06-09 - [PR #404](https://github.com/NOAA-OWP/cahaba/pull/404)
-
-To help analyze the memory consumption of the Fim Run process, the python module `memory-profiler` has been added to give insights into where peak memory usage is with in the codebase.
-
-In addition, the Dockerfile was previously broken due to the Taudem dependency removing the version that was previously being used by FIM. To fix this, and allow new docker images to be built, the Taudem version has been updated to the newest version on the Github repo and thus needs to be thoroughly tested to determine if this new version has affected the overall FIM outputs.
-
-## Additions
-- Added `memory-profiler` to `Pipfile` and `Pipfile.lock`.
-- Added `mprof` (memory-profiler cli utility) call to the `time_and_tee_run_by_unit.sh` to create overall memory usage graph location in the `/logs/{HUC}_memory.png` of the outputs directory.
-- Added `@profile` decorator to all functions within scripts used in the `run_by_unit.sh` script to allow for memory usage tracking, which is then recorded in the `/logs/{HUC}.log` file of the outputs directory.
-
-## Changes
-- Changed the Taudem version in `Dockerfile.dev` to `98137bb6541a0d0077a9c95becfed4e56d0aa0ac`.
-- Changed all calls of python scripts in `run_by_unit.s` to be called with the `-m memory-profiler` argument to allow scripts to also track memory usage.
-
-<br/><br/>
-## v3.0.17.1 - 2021-06-04 - [PR #395](https://github.com/NOAA-OWP/cahaba/pull/395)
-
-Bug fix to the `generate_nws_lid.py` script
-
-## Changes
-- Fixes incorrectly assigned attribute field "is_headwater" for some sites in the `nws_lid.gpkg` layer.
-- Updated `agg_nhd_headwaters_adj.gpkg`, `agg_nhd_streams_adj.gpkg`, `nwm_flows.gpkg`, and `nwm_catchments.gpkg` input layers using latest NWS LIDs.
-
-<br/><br/>
-## v3.0.17.0 - 2021-06-04 - [PR #393](https://github.com/NOAA-OWP/cahaba/pull/393)
-BARC updates to cap the bathy calculated xsec area in `bathy_rc_adjust.py` and allow user to choose input bankfull geometry.
-
-## Changes
-
-- Added new env variable to control which input file is used for the bankfull geometry input to bathy estimation workflow.
-- Modified the bathymetry cross section area calculation to cap the additional area value so that it cannot exceed the bankfull cross section area value for each stream segment (bankfull value obtained from regression equation dataset).
-- Modified the `rating_curve_comparison.py` plot output to always put the FIM rating curve on top of the USGS rating curve (avoids USGS points covering FIM).
-- Created a new aggregate csv file (aggregates for all hucs) for all of the `usgs_elev_table.csv` files (one per huc).
-- Evaluate the FIM Bathymetry Adjusted Rating Curve (BARC) tool performance using the estimated bankfull geometry dataset derived for the NWM route link dataset.
-
-<br/><br/>
-## v3.0.16.3 - 2021-05-21 - [PR #388](https://github.com/NOAA-OWP/cahaba/pull/388)
-
-Enhancement and bug fixes to `synthesize_test_cases.py`.
-
-## Changes
-- Addresses a bug where AHPS sites without benchmark data were receiving a CSI of 0 in the master metrics CSV produced by `synthesize_test_cases.py`.
-- Includes a feature enhancement to `synthesize_test_cases.py` that allows for the inclusion of user-specified testing versions in the master metrics CSV.
-- Removes some of the print statements used by `synthesize_test_cases.py`.
-
-<br/><br/>
-## v3.0.16.2 - 2021-05-18 - [PR #384](https://github.com/NOAA-OWP/cahaba/pull/384)
-
-Modifications and fixes to `run_test_case.py`, `eval_plots.py`, and AHPS preprocessing scripts.
-
-## Changes
-- Comment out return statement causing `run_test_case.py` to skip over sites/hucs when calculating contingency rasters.
-- Move bad sites list and query statement used to filter out bad sites to the `tools_shared_variables.py`.
-- Add print statements in `eval_plots.py` detailing the bad sites used and the query used to filter out bad sites.
-- Update AHPS preprocessing scripts to produce a domain shapefile.
-- Change output filenames produced in ahps preprocessing scripts.
-- Update workarounds for some sites in ahps preprocessing scripts.
-
-<br/><br/>
-## v3.0.16.1 - 2021-05-11 - [PR #380](https://github.com/NOAA-OWP/cahaba/pull/380)
-
-The current version of Eventlet used in the Connector module of the FIM API is outdated and vulnerable. This update bumps the version to the patched version.
-
-## Changes
-- Updated `api/node/connector/requirements.txt` to have the Eventlet version as 0.31.0
-
-<br/><br/>
-## v3.0.16.0 - 2021-05-07 - [PR #378](https://github.com/NOAA-OWP/cahaba/pull/378)
-
-New "Release" feature added to the FIM API. This feature will allow for automated FIM, CatFIM, and relevant metrics to be generated when a new FIM Version is released. See [#373](https://github.com/NOAA-OWP/cahaba/issues/373) for more detailed steps that take place in this feature.
-
-## Additions
-- Added new window to the UI in `api/frontend/gui/templates/index.html`.
-- Added new job type to `api/node/connector/connector.py` to allow these release jobs to run.
-- Added additional logic in `api/node/updater/updater.py` to run the new eval and CatFIM scripts used in the release feature.
-
-## Changes
-- Updated `api/frontend/output_handler/output_handler.py` to allow for copying more broad ranges of file paths instead of only the `/data/outputs` directory.
-
-<br/><br/>
-## v3.0.15.10 - 2021-05-06 - [PR #375](https://github.com/NOAA-OWP/cahaba/pull/375)
-
-Remove Great Lakes coastlines from WBD buffer.
-
-## Changes
-- `gl_water_polygons.gpkg` layer is used to mask out Great Lakes boundaries and remove NHDPlus HR coastline segments.
-
-<br/><br/>
-## v3.0.15.9 - 2021-05-03 - [PR #372](https://github.com/NOAA-OWP/cahaba/pull/372)
-
-Generate `nws_lid.gpkg`.
-
-## Additions
-- Generate `nws_lid.gpkg` with attributes indicating if site is a headwater `nws_lid` as well as if it is co-located with another `nws_lid` which is referenced to the same `nwm_feature_id` segment.
-
-<br/><br/>
-## v3.0.15.8 - 2021-04-29 - [PR #371](https://github.com/NOAA-OWP/cahaba/pull/371)
-
-Refactor NHDPlus HR preprocessing workflow. Resolves issue #238
-
-## Changes
-- Consolidate NHD streams, NWM catchments, and headwaters MS and FR layers with `mainstem` column.
-- HUC8 intersections are included in the input headwaters layer.
-- `clip_vectors_to_wbd.py` removes incoming stream segment from the selected layers.
-
-<br/><br/>
-## v3.0.15.7 - 2021-04-28 - [PR #367](https://github.com/NOAA-OWP/cahaba/pull/367)
-
-Refactor synthesize_test_case.py to handle exceptions during multiprocessing. Resolves issue #351
-
-## Changes
-- refactored `inundation.py` and `run_test_case.py` to handle exceptions without using `sys.exit()`.
-
-<br/><br/>
-## v3.0.15.6 - 2021-04-23 - [PR #365](https://github.com/NOAA-OWP/cahaba/pull/365)
-
-Implement CatFIM threshold flows to Sierra test and add AHPS benchmark preprocessing scripts.
-
-## Additions
-- Produce CatFIM flows file when running `rating_curve_get_usgs_gages.py`.
-- Several scripts to preprocess AHPS benchmark data. Requires numerous file dependencies not available through Cahaba.
-
-## Changes
-- Modify `rating_curve_comparison.py` to ingest CatFIM threshold flows in calculations.
-- Modify `eval_plots.py` to save all site specific bar plots in same parent directory instead of in subdirectories.
-- Add variables to `env.template` for AHPS benchmark preprocessing.
-
-<br/><br/>
-## v3.0.15.5 - 2021-04-20 - [PR #363](https://github.com/NOAA-OWP/cahaba/pull/363)
-
-Prevent eval_plots.py from erroring out when spatial argument enabled if certain datasets not analyzed.
-
-## Changes
-- Add check to make sure analyzed dataset is available prior to creating spatial dataset.
-
-<br/><br/>
-## v3.0.15.4 - 2021-04-20 - [PR #356](https://github.com/NOAA-OWP/cahaba/pull/356)
-
-Closing all multiprocessing Pool objects in repo.
-
-<br/><br/>
-## v3.0.15.3 - 2021-04-19 - [PR #358](https://github.com/NOAA-OWP/cahaba/pull/358)
-
-Preprocess NHDPlus HR rasters for consistent projections, nodata values, and convert from cm to meters.
-
-## Additions
-- `preprocess_rasters.py` reprojects raster, converts to meters, and updates nodata value to -9999.
-- Cleaned up log messages from `bathy_rc_adjust.py` and `usgs_gage_crosswalk.py`.
-- Outputs paths updated in `generate_categorical_fim_mapping.py` and `generate_categorical_fim.py`.
-- `update_raster_profile` cleans up raster crs, blocksize, nodata values, and converts elevation grids from cm to meters.
-- `reproject_dem.py` imports gdal to reproject elevation rasters because an error was occurring when using rasterio.
-
-## Changes
-- `burn_in_levees.py` replaces the `gdal_calc.py` command to resolve inconsistent outputs with burned in levee values.
-
-<br/><br/>
-## v3.0.15.2 - 2021-04-16 - [PR #359](https://github.com/NOAA-OWP/cahaba/pull/359)
-
-Hotfix to preserve desired files when production flag used in `fim_run.sh`.
-
-## Changes
-
-- Fixed production whitelisted files.
-
-<br/><br/>
-## v3.0.15.1 - 2021-04-13 - [PR #355](https://github.com/NOAA-OWP/cahaba/pull/355)
-
-Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field.
-
-## Changes
-
-- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "MS" run, only consider gages that contain rating curve information (via `curve` attribute) and are also mainstems gages (via `mainstems` attribute).
-- Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "FR" run, only consider gages that contain rating curve information (via `curve` attribute) and are not mainstems gages (via `mainstems` attribute).
-- Modifies how mainstems segments are determined by using the `nwm_flows_ms.gpkg` as a lookup to determine if the NWM segment specified by WRDS for a gage site is a mainstems gage.
-
-## Additions
-
-- Adds a `mainstem` attribute field to `usgs_gages.gpkg` that indicates whether a gage is located on a mainstems river.
-- Adds `NWM_FLOWS_MS` variable to the `.env` and `.env.template` files.
-- Adds the `extent` argument specified by user when running `fim_run.sh` to `usgs_gage_crosswalk.py`.
-
-<br/><br/>
-## v3.0.15.0 - 2021-04-08 - [PR #340](https://github.com/NOAA-OWP/cahaba/pull/340)
-
-Implementing a prototype technique to estimate the missing bathymetric component in the HAND-derived synthetic rating curves. The new Bathymetric Adjusted Rating Curve (BARC) function is built within the `fim_run.sh` workflow and will ingest bankfull geometry estimates provided by the user to modify the cross section area used in the synthetic rating curve generation.
-
-### Changes
- - `add_crosswalk.py` outputs the stream order variables to `src_full_crosswalked.csv` and calls the new `bathy_rc_adjust.py` if bathy env variable set to True and `extent=MS`.
- - `run_by_unit.sh` includes a new csv outputs for reviewing BARC calculations.
- - `params_template.env` & `params_calibrated.env` contain new BARC function input variables and on/off toggle variable.
- - `eval_plots.py` now includes additional AHPS eval sites in the list of "bad_sites" (flagged issues with MS flowlines).
-
-### Additions
- - `bathy_rc_adjust.py`:
-    - Imports the existing synthetic rating curve table and the bankfull geometry input data (topwidth and cross section area per COMID).
-    - Performs new synthetic rating curve calculations with bathymetry estimation modifications.
-    - Flags issues with the thalweg-notch artifact.
-
-<br/><br/>
-## v3.0.14.0 - 2021-04-05 - [PR #338](https://github.com/NOAA-OWP/cahaba/pull/338)
-
-Create tool to retrieve rating curves from USGS sites and convert to elevation (NAVD88). Intended to be used as part of the Sierra Test.
-
-### Changes
- - Modify `usgs_gage_crosswalk.py` to:
-    1) Look for `location_id` instead of `site_no` attribute field in `usgs_gages.gpkg` file.
-    2) Filter out gages that do not have rating curves included in the `usgs_rating_curves.csv`.
- - Modify `rating_curve_comparison.py` to perform a check on the age of the user specified `usgs_rating_curves.csv` and alert user to the age of the file and recommend updating if file is older the 30 days.
-
-### Additions
- - Add `rating_curve_get_usgs_curves.py`. This script will generate the following files:
-     1) `usgs_rating_curves.csv`: A csv file that contains rating curves (including converted to NAVD88 elevation) for USGS gages in a format that is compatible with  `rating_curve_comparisons.py`. As it is is currently configured, only gages within CONUS will have rating curve data.
-     2) `log.csv`: A log file that records status for each gage and includes error messages.
-     3) `usgs_gages.gpkg`: A geospatial layer (in FIM projection) of all active USGS gages that meet a predefined criteria. Additionally, the `curve` attribute indicates whether a rating curve is found in the `usgs_rating_curves.csv`. This spatial file is only generated if the `all` option is passed with the `-l` argument.
-
-<br/><br/>
-## v3.0.13.0 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
-
-Created tool to compare synthetic rating curve with benchmark rating curve (Sierra Test).
-
-### Changes
- - Update `aggregate_fim_outputs.py` call argument in `fim_run.sh` from 4 jobs to 6 jobs, to optimize API performance.
- - Reroutes median elevation data from `add_crosswalk.py` and `rem.py` to new file (depreciating `hand_ref_elev_table.csv`).
- - Adds new files to `viz_whitelist` in `output_cleanup.py`.
-
-### Additions
- - `usgs_gage_crosswalk.py`: generates `usgs_elev_table.csv` in `run_by_unit.py` with elevation and additional attributes at USGS gages.
- - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data.
-
-<br/><br/>
-## v3.0.12.1 - 2021-03-31 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
-
-Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
-
-### Changes
- - Removes file dependencies from spatial option. Does require the WBD layer which should be specified in `.env` file.
- - Produces outputs in a format consistent with requirements needed for publishing.
- - Preserves leading zeros in huc information for all outputs from `eval_plots.py`.
-
-### Additions
- - Creates `fim_performance_points.shp`: this layer consists of all evaluated ahps points (with metrics). Spatial data retrieved from WRDS on the fly.
- - Creates `fim_performance_polys.shp`: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.
-
-<br/><br/>
-## v3.0.12.0 - 2021-03-26 - [PR #327](https://github.com/NOAA-OWP/cahaba/pull/237)
-
-Add more detail/information to plotting capabilities.
-
-### Changes
- - Merge `plot_functions.py` into `eval_plots.py` and move `eval_plots.py` into the tools directory.
- - Remove `plots` subdirectory.
-
-### Additions
- - Optional argument to create barplots of CSI for each individual site.
- - Create a csv containing the data used to create the scatterplots.
-
-<br/><br/>
-## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
-
-Improvements to CatFIM service source data generation.
-
-### Changes
- - Renamed `generate_categorical_fim.py` to `generate_categorical_fim_mapping.py`.
- - Updated the status outputs of the `nws_lid_sites layer` and saved it in the same directory as the `merged catfim_library layer`.
- - Additional stability fixes (such as improved compatability with WRDS updates).
-
-### Additions
- - Added `generate_categorical_fim.py` to wrap `generate_categorical_fim_flows.py` and `generate_categorical_fim_mapping.py`.
- - Create new `nws_lid_sites` shapefile located in same directory as the `catfim_library` shapefile.
-
-<br/><br/>
-## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
-
-Patch to synthesize_test_cases.py.
-
-### Changes
- - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
-
-<br/><br/>
-## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
-
-Preprocessing of flow files for Categorical FIM.
-
-### Additions
- - Generate Categorical FIM flow files for each category (action, minor, moderate, major).
- - Generate point shapefile of Categorical FIM sites.
- - Generate csv of attribute data in shapefile.
- - Aggregate all shapefiles and csv files into one file in parent directory.
- - Add flood of record category.
-
- ### Changes
- - Stability fixes to `generate_categorical_fim.py`.
-
-<br/><br/>
-## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)
-
-Enhancements to FIM API.
-
-### Changes
- - `fim_run.sh` can now be run with jobs in parallel.
- - Viz post-processing can now be selected in API interface.
- - Jobs table shows jobs that end with errors.
- - HUC preset lists can now be selected in interface.
- - Better `output_handler` file writing.
- - Overall better restart and retry handlers for networking problems.
- - Jobs can now be canceled in API interface.
- - Both FR and MS configs can be selected for a single job.
-
-<br/><br/>
-## v3.0.8.2 - 2021-03-11 - [PR #296](https://github.com/NOAA-OWP/cahaba/pull/296)
-
-Enhancements to post-processing for Viz-related use-cases.
-
-### Changes
- - Aggregate grids are projected to Web Mercator during `-v` runs in `fim_run.sh`.
- - HUC6 aggregation is parallelized.
- - Aggregate grid blocksize is changed from 256 to 1024 for faster postprocessing.
-
-<br/><br/>
-## v3.0.8.1 - 2021-03-10 - [PR #302](https://github.com/NOAA-OWP/cahaba/pull/302)
-
-Patched import issue in `tools_shared_functions.py`.
-
-### Changes
- - Changed `utils.` to `tools_` in `tools_shared_functions.py` after recent structural change to `tools` directory.
-
-<br/><br/>
-## v3.0.8.0 - 2021-03-09 - [PR #279](https://github.com/NOAA-OWP/cahaba/pull/279)
-
-Refactored NWS Flood Categorical HAND FIM (CatFIM) pipeline to open source.
-
-### Changes
- - Added `VIZ_PROJECTION` to `shared_variables.py`.
- - Added missing library referenced in `inundation.py`.
- - Cleaned up and converted evaluation scripts in `generate_categorical_fim.py` to open source.
- - Removed `util` folders under `tools` directory.
-
-<br/><br/>
-## v3.0.7.1 - 2021-03-02 - [PR #290](https://github.com/NOAA-OWP/cahaba/pull/290)
-
-Renamed benchmark layers in `test_cases` and updated variable names in evaluation scripts.
-
-### Changes
- - Updated `run_test_case.py` with new benchmark layer names.
- - Updated `run_test_case_calibration.py` with new benchmark layer names.
-
-<br/><br/>
-## v3.0.7.0 - 2021-03-01 - [PR #288](https://github.com/NOAA-OWP/cahaba/pull/288)
-
-Restructured the repository. This has no impact on hydrological work done in the codebase and is simply moving files and renaming directories.
-
-### Changes
- - Moved the contents of the `lib` folder to a new folder called `src`.
- - Moved the contents of the `tests` folder to the `tools` folder.
- - Changed any instance of `lib` or `libDir` to `src` or `srcDir`.
-
-<br/><br/>
-## v3.0.6.0 - 2021-02-25 - [PR #276](https://github.com/NOAA-OWP/cahaba/pull/276)
-
-Enhancement that creates metric plots and summary statistics using metrics compiled by `synthesize_test_cases.py`.
-
-### Additions
- - Added `eval_plots.py`, which produces:
-    - Boxplots of CSI, FAR, and POD/TPR
-    - Barplot of aggregated CSI scores
-    - Scatterplot of CSI comparing two FIM versions
-    - CSV of aggregated statistics (CSI, FAR, POD/TPR)
-    - CSV of analyzed data and analyzed sites
-
-<br/><br/>
-## v3.0.5.3 - 2021-02-23 - [PR #275](https://github.com/NOAA-OWP/cahaba/pull/275)
-
-Bug fixes to new evaluation code.
-
-### Changes
-
- - Fixed a bug in `synthesize_test_cases.py` where the extent (MS/FR) was not being written to merged metrics file properly.
- - Fixed a bug in `synthesize_test_cases.py` where only BLE test cases were being written to merged metrics file.
- - Removed unused imports from `inundation.py`.
- - Updated README.md
-
-<br/><br/>
-## v3.0.5.2 - 2021-02-23 - [PR #272](https://github.com/NOAA-OWP/cahaba/pull/272)
-
-Adds HAND synthetic rating curve (SRC) datum elevation values to `hydroTable.csv` output.
-
-### Changes
-
- - Updated `add_crosswalk.py` to included "Median_Thal_Elev_m" variable outputs in `hydroTable.csv`.
- - Renamed hydroid attribute in `rem.py` to "Median" in case we want to include other statistics in the future (e.g. min, max, range etc.).
-
-<br/><br/>
-## v3.0.5.1 - 2021-02-22
-
-Fixed `TEST_CASES_DIR` path in `tests/utils/shared_variables.py`.
-
-### Changes
-
- - Removed `"_new"` from `TEST_CASES_DIR` variable.
-
-<br/><br/>
-## v3.0.5.0 - 2021-02-22 - [PR #267](https://github.com/NOAA-OWP/cahaba/pull/267)
-
-Enhancements to allow for evaluation at AHPS sites, the generation of a query-optimized metrics CSV, and the generation of categorical FIM. This merge requires that the `/test_cases` directory be updated for all machines performing evaluation.
-
-### Additions
-
- - `generate_categorical_fim.py` was added to allow production of NWS Flood Categorical HAND FIM (CatFIM) source data. More changes on this script are to follow in subsequent branches.
-
-### Removals
-
- - `ble_autoeval.sh` and `all_ble_stats_comparison.py` were deleted because `synthesize_test_cases.py` now handles the merging of metrics.
- - The code block in `run_test_case.py` that was responsible for printing the colored metrics to screen has been commented out because of the new scale of evaluations (formerly in `run_test_case.py`, now in `shared_functions.py`)
- - Remove unused imports from inundation wrappers in `/tools`.
-
-### Changes
-
- - Updated `synthesize_test_cases.py` to allow for AHPS site evaluations.
- - Reorganized `run_test_case.py` by moving more functions into `shared_functions.py`.
- - Created more shared variables in `shared_variables.py` and updated import statements in relevant scripts.
-
-<br/><br/>
-
-## v3.0.4.4 - 2021-02-19 - [PR #266](https://github.com/NOAA-OWP/cahaba/pull/266)
-
-Rating curves for short stream segments are replaced with rating curves from upstream/downstream segments.
-
-### Changes
-
- - Short stream segments are identified and are reassigned the channel geometry from upstream/downstream segment.
- - `fossid` renamed to `fimid` and the attribute's starting value is now 1000 to avoid HydroIDs with leading zeroes.
- - Addresses issue where HydroIDs were not included in final hydrotable.
- - Added `import sys` to `inundation.py` (missing from previous feature branch).
- - Variable names and general workflow are cleaned up.
-
-<br/><br/>
-## v3.0.4.3 - 2021-02-12 - [PR #254](https://github.com/NOAA-OWP/cahaba/pull/254)
-
-Modified `rem.py` with a new function to output HAND reference elev.
-
-### Changes
-
- - Function `make_catchment_hydroid_dict` creates a df of pixel catchment ids and overlapping hydroids.
- - Merge hydroid df and thalweg minimum elevation df.
- - Produces new output containing all catchment ids and min thalweg elevation value named `hand_ref_elev_table.csv`.
- - Overwrites the `demDerived_reaches_split.gpk` layer by adding additional attribute `Min_Thal_Elev_meters` to view the elevation value for each hydroid.
-
-<br/><br/>
-## v3.0.4.2 - 2021-02-12 - [PR #255](https://github.com/NOAA-OWP/cahaba/pull/255)
-
-Addresses issue when running on HUC6 scale.
-
-### Changes
-
- - `src.json` should be fixed and slightly smaller by removing whitespace.
- - Rasters are about the same size as running fim as huc6 (compressed and tiled; aggregated are slightly larger).
- - Naming convention and feature id attribute are only added to the aggregated hucs.
- - HydroIDs are different for huc6 vs aggregated huc8s mostly due to forced split at huc boundaries (so long we use consistent workflow it shouldn't matter).
- - Fixed known issue where sometimes an incoming stream is not included in the final selection will affect aggregate outputs.
-
-<br/><br/>
-## v3.0.4.1 - 2021-02-12 - [PR #261](https://github.com/NOAA-OWP/cahaba/pull/261)
-
-Updated MS Crosswalk method to address gaps in FIM.
-
-### Changes
-
- - Fixed typo in stream midpoint calculation in `split_flows.py` and `add_crosswalk.py`.
- - `add_crosswalk.py` now restricts the MS crosswalk to NWM MS catchments.
- - `add_crosswalk.py` now performs a secondary MS crosswalk selection by nearest NWM MS catchment.
-
-<br/><br/>
-## v3.0.4.0 - 2021-02-10 - [PR #256](https://github.com/NOAA-OWP/cahaba/pull/256)
-
-New python script "wrappers" for using `inundation.py`.
-
-### Additions
-
- - Created `inundation_wrapper_nwm_flows.py` to produce inundation outputs using NWM recurrence flows: 1.5 year, 5 year, 10 year.
- - Created `inundation_wrapper_custom_flow.py` to produce inundation outputs with user-created flow file.
- - Created new `tools` parent directory to store `inundation_wrapper_nwm_flows.py` and  `inundation_wrapper_custom_flow.py`.
-
-<br/><br/>
-## v3.0.3.1 - 2021-02-04 - [PR #253](https://github.com/NOAA-OWP/cahaba/pull/253)
-
-Bug fixes to correct mismatched variable name and file path.
-
-### Changes
-
- - Corrected variable name in `fim_run.sh`.
- - `acquire_and_preprocess_inputs.py` now creates `huc_lists` folder and updates file path.
-
-<br/><br/>
-## v3.0.3.0 - 2021-02-04 - [PR #227](https://github.com/NOAA-OWP/cahaba/pull/227)
-
-Post-process to aggregate FIM outputs to HUC6 scale.
-
-### Additions
-
- - Viz outputs aggregated to HUC6 scale; saves outputs to `aggregate_fim_outputs` folder.
-
-### Changes
-
- - `split_flows.py` now splits streams at HUC8 boundaries to ensure consistent catchment boundaries along edges.
- - `aggregate_fim_outputs.sh` has been depreciated but remains in the repo for potential FIM 4 development.
- - Replaced geopandas driver arg with getDriver throughout repo.
- - Organized parameters in environment files by group.
- - Cleaned up variable names in `split_flows.py` and `build_stream_traversal.py`.
- - `build_stream_traversal.py` is now assigning HydroID by midpoint instead centroid.
- - Cleanup of `clip_vectors_to_wbd.py`.
-
-<br/><br/>
-## v3.0.2.0 - 2021-01-25 - [PR #218](https://github.com/NOAA-OWP/cahaba/pull/218)
-
-Addition of an API service to schedule, run and manage `fim_run` jobs through a user-friendly web interface.
-
-### Additions
-
- - `api` folder that contains all the codebase for the new service.
-
-<br/><br/>
-## v3.0.1.0 - 2021-01-21 - [PR #206](https://github.com/NOAA-OWP/cahaba/pull/206)
-
-Preprocess MS and FR stream networks
-
-### Changes
-
- - Headwater stream segments geometries are adjusted to align with with NWM streams.
- - Incoming streams are selected using intersection points between NWM streams and HUC4 boundaries.
- - `clip_vectors_to_wbd.py` handles local headwaters.
- - Removes NHDPlus features categorized as coastline and underground conduit.  
- - Added streams layer to production whitelist.
- - Fixed progress bar in `lib/acquire_and_preprocess_inputs.py`.
- - Added `getDriver` to shared `functions.py`.
- - Cleaned up variable names and types.
-
-<br/><br/>
-## v3.0.0.4 - 2021-01-20 - [PR #230](https://github.com/NOAA-OWP/cahaba/pull/230)
-
-Changed the directory where the `included_huc*.lst` files are being read from.
-
-### Changes
-
- - Changed the directory where the `included_huc*.lst` files are being read from.
-
-<br/><br/>
-## v3.0.0.3 - 2021-01-14 - [PR #210](https://github.com/NOAA-OWP/cahaba/pull/210)
-
-Hotfix for handling nodata value in rasterized levee lines.
-
-### Changes
-
- - Resolves bug for HUCs where `$ndv > 0` (Great Lakes region).
- - Initialize the `nld_rasterized_elev.tif` using a value of `-9999` instead of `$ndv`.
-
- <br/><br/>
-## v3.0.0.2 - 2021-01-06 - [PR #200](https://github.com/NOAA-OWP/cahaba/pull/200)
-
-Patch to address AHPSs mapping errors.
-
-### Changes
-
- - Checks `dtype` of `hydroTable.csv` columns to resolve errors caused in `inundation.py` when joining to flow forecast.
- - Exits `inundation.py` when all hydrotable HydroIDs are lake features.
- - Updates path to latest AHPs site layer.
- - Updated [readme](https://github.com/NOAA-OWP/cahaba/commit/9bffb885f32dfcd95978c7ccd2639f9df56ff829)
-
-<br/><br/>
-## v3.0.0.1 - 2020-12-31 - [PR #184](https://github.com/NOAA-OWP/cahaba/pull/184)
-
-Modifications to build and run Docker image more reliably. Cleanup on some pre-processing scripts.
-
-### Changes
-
- - Changed to noninteractive install of GRASS.
- - Changed some paths from relative to absolute and cleaned up some python shebang lines.
-
-### Notes
- - `aggregate_vector_inputs.py` doesn't work yet. Need to externally download required data to run fim_run.sh
-
- <br/><br/>
-## v3.0.0.0 - 2020-12-22 - [PR #181](https://github.com/NOAA-OWP/cahaba/pull/181)
-
-The software released here builds on the flood inundation mapping capabilities demonstrated as part of the National Flood Interoperability Experiment, the Office of Water Prediction's Innovators Program and the National Water Center Summer Institute. The flood inundation mapping software implements the Height Above Nearest Drainage (HAND) algorithm and incorporates community feedback and lessons learned over several years. The software has been designed to meet the requirements set by stakeholders interested in flood prediction and has been developed in partnership with several entities across the water enterprise.
diff --git a/README.md b/README.md
index 6bdfbb6c8..69b280b63 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,14 @@
-## Cahaba: Flood Inundation Mapping for U.S. National Water Model
+## Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
 
-Flood inundation mapping softwaconfigured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
+This repository includes flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
 
 This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids. This repository also includes functionality to generate flood inundation maps (FIMs) and evaluate FIM accuracy.
 
-#### For more information, see the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki).
+#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/cahaba/wiki).
+
+---
+
+# FIM Version 3 
 
 ## Accessing Data through ESIP S3 Bucket
 The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.
@@ -115,6 +119,12 @@ and include both `Pipfile` and `Pipfile.lock` in your commits. The docker image
 
 If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
 
+
+---
+# FIM Version 4
+
+Details to come.
+
 ----
 ## Citing This Work
 
@@ -123,11 +133,11 @@ Please cite this work in your research and projects according to the CITATION.cf
 ----
 ### Known Issues & Getting Help
 
-Please see the issue tracker on GitHub and the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki/Cahaba-Wiki-Home) for known issues and getting help.
+Please see the issue tracker on GitHub and the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/cahaba/wiki/Cahaba-Wiki-Home) for known issues and getting help.
 
 ### Getting Involved
 
-NOAA's National Water Center welcomes anyone to contribute to the Cahaba repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
+NOAA's National Water Center welcomes anyone to contribute to the Inundation Mapping repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
 ### Open Source Licensing Info
 1. [TERMS](docs/TERMS.md)

From e81ec28072599248420b48764b6e43ee4e0fa716 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 9 Feb 2022 08:50:53 -0600
Subject: [PATCH 128/359] Unit tests improvements

---
 unit_tests/README.md                          | 20 +++++----
 unit_tests/clip_vectors_to_wbd_params.json    | 45 ++++++++++---------
 unit_tests/clip_vectors_to_wbd_unittests.py   | 13 +++---
 unit_tests/gms/derive_level_paths_params.json | 23 +++++-----
 .../gms/derive_level_paths_unittests.py       | 45 +++++++++----------
 5 files changed, 77 insertions(+), 69 deletions(-)

diff --git a/unit_tests/README.md b/unit_tests/README.md
index b599ace65..0238b4529 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -29,7 +29,8 @@ One way is to use the incoming arg parser. Most py files, they include the code
 
 ## Running unit tests
 
-Start a docker container as you normally would for any development. ie) docker run --rm -it --name rob_levelpaths -v /home/{your name}/projects/gms_ms_level_paths/fim_4/:/foss_fim {your docker image name}
+Start a docker container as you normally would for any development. ie) docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
+	- ie) docker run --rm -it --name Rob_GMS -v /home/rob_abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
 
 At the root terminal window, run:  python ./foss_fim/unit_tests/gms/derive_level_paths_unittests.py  or python ./foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
 (replace with your own script and path name)
@@ -46,23 +47,26 @@ At the root terminal window, run:  python ./foss_fim/unit_tests/gms/derive_level
 
 5) There must be at one "{original py file name}_params.json" file.
 
-6) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to full pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass.
+6) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to full pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass. Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data"
 
-7) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output, but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
+7) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data", if one does not already exist. Generally, the individual unit tests, will call the "valid_data" node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie.. major different test conditions).
 
-7) One py file = one "{original py file name}_unittests.py" file.
+8) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
+
+9) One py file = one "{original py file name}_unittests.py" file.
+
+10) Sometimes you may want to run a full successful "happy path" version through gms_run_by_unit.sh (or similar), to get all of the files you need in place to do your testing. However.. you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the "-d" parameter (denylist). ie) Normally:  gms_run_unit.sh -n gms_example_unit_tests -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o, but ours would be 
+gms_run_unit.sh -n gms_example_unit_tests -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o. 
 
 
 ## Future Enhancements
 1) We can automate triggers on these files for things like checking triggers or an single global "run_all_unittest" script, but for now.. its one offs.
 
-2) Better output from the unit tests.
+2) Better output from the unit tests including verbosity output control
 
 3) We will upgrade it so you can pass in a params.json file to the call to give you even more flexibility by passing in your own params.json (think params_template.json idea)
 
-4) At this time, the root json file has only one "node". We may consider have more than one node in the json file, which has a different set of data for different test conditions. 
-
-5) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
+4) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
 
 
 ## testing for failing conditions
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 3f1b0f787..004aef392 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -1,23 +1,26 @@
 {
-	"hucCode": "05030104",
-	"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
-	"nhd_streams": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg", 
-	"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
-	"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
-	"wbd": "/data/outputs/level_paths_3/05030104/wbd.gpkg",
-	"wbd_buffer": "/data/outputs/level_paths_3/05030104/wbd_buffered.gpkg",
-	"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", 
-	"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
-	"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
-	"subset_nhd_streams": "/data/outputs/level_paths_3/05030104/NHDPlusBurnLineEvent_subset.gpkg",
-	"subset_nld_lines": "/data/outputs/level_paths_3/05030104/nld_subset_levees.gpkg", 
-	"subset_lakes": "/data/outputs/level_paths_3/05030104/nwm_lakes_proj_subset.gpkg",
-	"subset_catchments": "/data/outputs/level_paths_3/05030104/nwm_catchments_proj_subset.gpkg",
-	"subset_nhd_headwaters": "/data/outputs/level_paths_3/05030104/nhd_headwater_points_subset.gpkg", 
-	"subset_nwm_streams": "/data/outputs/level_paths_3/05030104/nwm_subset_streams.gpkg", 
-	"subset_landsea": "/data/outputs/level_paths_3/05030104/LandSea_subset.gpkg", 
-	"extent": "GMS",
-	"great_lakes_filename": "/data/inputs/landsea/gl_water_polygons.gpkg", 
-	"wbd_buffer_distance": 5000,
-	"lake_buffer_distance": 20
+	"valid_data":
+	{
+		"hucCode": "05030104",
+		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
+		"nhd_streams": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg", 
+		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
+		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
+		"wbd": "/data/outputs/gms_example_unit_tests/05030104/wbd.gpkg",
+		"wbd_buffer": "/data/outputs/gms_example_unit_tests/05030104/wbd_buffered.gpkg",
+		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", 
+		"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
+		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
+		"subset_nhd_streams": "/data/outputs/gms_example_unit_tests/05030104/NHDPlusBurnLineEvent_subset.gpkg",
+		"subset_nld_lines": "/data/outputs/gms_example_unit_tests/05030104/nld_subset_levees.gpkg", 
+		"subset_lakes": "/data/outputs/gms_example_unit_tests/05030104/nwm_lakes_proj_subset.gpkg",
+		"subset_catchments": "/data/outputs/gms_example_unit_tests/05030104/nwm_catchments_proj_subset.gpkg",
+		"subset_nhd_headwaters": "/data/outputs/gms_example_unit_tests/05030104/nhd_headwater_points_subset.gpkg", 
+		"subset_nwm_streams": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams.gpkg", 
+		"subset_landsea": "/data/outputs/gms_example_unit_tests/05030104/LandSea_subset.gpkg", 
+		"extent": "GMS",
+		"great_lakes_filename": "/data/inputs/landsea/gl_water_polygons.gpkg", 
+		"wbd_buffer_distance": 5000,
+		"lake_buffer_distance": 20
+	}
 }
\ No newline at end of file
diff --git a/unit_tests/clip_vectors_to_wbd_unittests.py b/unit_tests/clip_vectors_to_wbd_unittests.py
index 322b4ac4b..47f4c1b36 100644
--- a/unit_tests/clip_vectors_to_wbd_unittests.py
+++ b/unit_tests/clip_vectors_to_wbd_unittests.py
@@ -33,11 +33,6 @@ def setUpClass(self):
     # MUST start with the name of "test_"
     def test_subset_vector_layers_success(self):
 
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
-        params = self.params
-        
         '''
         This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
         Most of the output test and internal tests with this function will test a wide variety of conditions.
@@ -45,6 +40,12 @@ def test_subset_vector_layers_success(self):
         Only the basic return output value shoudl be tested to ensure it is as expected.
         For now, we are adding the very basic "happy path" test.
         '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()
+        
        
         # There is no default return value.
         # Later we need to check in this function that files were being created on the file system
@@ -73,8 +74,6 @@ def test_subset_vector_layers_success(self):
                                                 lake_buffer_distance = params["lake_buffer_distance"])
        
        
-
-       
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")
 
diff --git a/unit_tests/gms/derive_level_paths_params.json b/unit_tests/gms/derive_level_paths_params.json
index 68a30f802..6e63343b7 100644
--- a/unit_tests/gms/derive_level_paths_params.json
+++ b/unit_tests/gms/derive_level_paths_params.json
@@ -1,14 +1,17 @@
 {
-	"in_stream_network": "/data/outputs/level_paths_1/03110201/nwm_subset_streams.gpkg",
-	"out_stream_network": "/data/outputs/level_paths_1/03110201/nwm_subset_streams_levelPaths.gpkg",
-	"branch_id_attribute": "levpa_id",
-	"out_stream_network_dissolved": "/data/outputs/level_paths_1/03110201/nwm_subset_streams_levelPaths.gpkg",
-	"headwaters_outfile": "/data/outputs/level_paths_1/03110201/nwm_headwaters.gpkg",
-	"catchments": "/data/outputs/level_paths_1/03110201/nwm_catchments_proj_subset.gpkg",
-	"catchments_outfile": "/data/outputs/level_paths_1/03110201/nwm_catchments_proj_subset_levelPaths.gpkg",
-	"branch_inlets_outfile": "/data/outputs/level_paths_1/03110201/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
-	"reach_id_attribute": "ID",
-	"verbose": false
+	"valid_data":
+	{
+		"in_stream_network": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams.gpkg",
+		"out_stream_network": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"branch_id_attribute": "levpa_id",
+		"out_stream_network_dissolved": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/gms_example_unit_tests/05030104/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"reach_id_attribute": "ID",
+		"verbose": false
+	}
 }
 
 
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
index c7d9f0085..bdd58c217 100644
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -34,10 +34,19 @@ def setUpClass(self):
 
     # MUST start with the name of "test_"
     def test_Derive_level_paths_success(self):
+
+        '''
+        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
+        Most of the output test and internal tests with this function will test a wide variety of conditions.
+        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
+        Only the basic return output value shoudl be tested to ensure it is as expected.
+        For now, we are adding the very basic "happy path" test.
+        '''
        
+        # makes output readability easier and consistant with other unit tests       
         helpers.print_unit_test_function_header()
         
-        params = self.params
+        params = self.params["valid_data"].copy()
 
         # Function Notes:
         # huc_ids no longer used, so it is not submitted
@@ -54,16 +63,7 @@ def test_Derive_level_paths_success(self):
                                                        reach_id_attribute = params["reach_id_attribute"],
                                                        verbose = params["verbose"] )
         
-        
-        '''
-        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
-        Most of the output test and internal tests with this function will test a wide variety of conditions.
-        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
-        Only the basic return output value shoudl be tested to ensure it is as expected.
-        For now, we are adding the very basic "happy path" test.
-        '''
-        
-        print(actual)
+        #print(actual)
         
         # Later, we should check here that the files outputed to the file system, exist and are valid and those do not have to be new "test_" methods.
         # We can start using unit test "asserts" as we go as well.
@@ -71,22 +71,20 @@ def test_Derive_level_paths_success(self):
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")        
 
-
+    
     # Invalid Input stream
     # MUST start with the name of "test_"    
     def test_Derive_level_paths_invalid_input_stream_network(self):
 
-        helpers.print_unit_test_function_header()        
+        # makes output readability easier and consistant with other unit tests 
+        helpers.print_unit_test_function_header()
         
         try:
         
-            params = self.params
-            #with open("/foss_fim/unit_tests/gms/derive_level_paths_params.json") as params_file:
-            #    params = json.load(params_file)
-
-            bad_input_stream_network = "some bad path"
-           
-            actual = derive_level_paths.Derive_level_paths(in_stream_network = bad_input_stream_network,
+            params = self.params["valid_data"].copy()
+            params["in_stream_network"] = "some bad path"
+          
+            actual = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
                                                            out_stream_network = params["out_stream_network"],
                                                            branch_id_attribute = params["branch_id_attribute"],
                                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
@@ -100,14 +98,15 @@ def test_Derive_level_paths_invalid_input_stream_network(self):
                                                            
             raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
             
-        except Exception as ex:
+        except Exception:
             print()
             print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            print(ex)
             
         finally:
             print("*************************************************************")             
-        
+       
+       
+       
 if __name__ == '__main__':
 
     print("*****************************")

From 24107570fcf7570404121f0c20474f502ffe3c01 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 9 Feb 2022 15:11:26 -0600
Subject: [PATCH 129/359] Adds unit test templates to make it easier to create
 new unit tests

---
 unit_tests/__example_nws_data_README.md    |  76 +++++++++++++
 unit_tests/__example_nws_data_params.json  |   6 +
 unit_tests/__example_nws_data_unittests.py | 124 +++++++++++++++++++++
 unit_tests/__template_params.json          |  11 ++
 unit_tests/__template_unittests.py         | 111 ++++++++++++++++++
 5 files changed, 328 insertions(+)
 create mode 100644 unit_tests/__example_nws_data_README.md
 create mode 100644 unit_tests/__example_nws_data_params.json
 create mode 100644 unit_tests/__example_nws_data_unittests.py
 create mode 100644 unit_tests/__template_params.json
 create mode 100644 unit_tests/__template_unittests.py

diff --git a/unit_tests/__example_nws_data_README.md b/unit_tests/__example_nws_data_README.md
new file mode 100644
index 000000000..151532420
--- /dev/null
+++ b/unit_tests/__example_nws_data_README.md
@@ -0,0 +1,76 @@
+
+## Sample unit tests - for demo purposes only#
+
+    Used to demonstrate __example_nws_data_unittests.py only.
+	 The __example_nws_data_unittest.py files shows how success and fails method look like as well as things like using actual results, expected results and asserts.
+
+# Here is a unit test being used in LISFLOOD
+
+`Here are some example code changes that would be in place if the __example files were valid (doesn't actually exist).`
+
+- There is a new variable added to src/utils/shared_variables -> DATA_SOURCE_PATH_NWS_LID_GPKG = "/data/inputs/ahp_sites/01_27_22/nws_lid.gpkg"
+
+- A python file exists in src called nws_data.py (again.. doesn't actually)
+
+Here are the contents of that pretend file (for this repo)
+
++++++++++++++++++++++++++++++++++++++++++++
+
+import geopandas as gpd
+import utils.shared_variables
+
+def get_huc8_by_nws_lid(nws_lid):
+
+    '''
+    OVERVIEW:
+        By passing in a nws_lid (also called a ahps id), it will open the ahps_sites/nws_lid.gpkg, find the record and return the huc8 value.
+
+    INPUT PARAMETERS:
+        - nws_lid: a string value (case sensitive) which should match the nws_lids.gpkg -> nws_lid column.
+
+    PROCESSING / VALIDATION:
+        - IF the nws record is not found, an exception will be raised.
+        - The nws_lid.gpkg path is defined by the new src/utils/data_sources.env parameter.
+        - If more than one nws record was found by nws_lid, it is assumed as bad data and an exception will be raised.
+        - nws_lid will be changed to upper case to match the database
+
+    RETURNS:
+       The discovered HUC8 value.
+    '''
+
+    # -----------------------------
+    # VALIDATION OF INCOMING VALUES
+
+    # Ensure the nws_lid value is not "None" or empty
+    if (not nws_lid):
+        raise Exception("nw_lid parameter appears to be empty or invalid")
+
+    # Ensure there are no spaces inside or on either end of the nws_lid value
+    if (" " in nws_lid):
+        raise Exception("nws_lid parameter appears to have invalid spaces either in or around the value. Please check the incoming param value.")
+
+
+    # -----------------------------
+    # PROCESSING:
+
+    nws_lid = nws_lid.upper()
+
+    # get the huc from the nws_lid.gpkg
+    nws_lid_data = gpd.read_file(utils.shared_variables.DATA_SOURCE_PATH_NWS_LID_GPKG)
+
+    nws_record = nws_lid_data.loc[nws_lid_data['nws_lid'] == nws_lid]
+
+    if nws_record.empty:
+        raise Exception("No nws_lid record found for nws_id of " + nws_lid)
+
+    if len(nws_record) > 1:
+        raise Exception("More than one nws_lid record was found for nws_id of " + nws_lid + ". Check the nws_lid.gpkg as there may be some bad data.")
+
+    huc8 = nws_record['HUC8'].values[0]
+    if (not huc8):
+        raise Exception("nws_lid record found but no huc value found. Check nws_lid.gpkg")
+
+    return(huc8)
+
+
++++++++++++++++++++++++++++++++++++++++++++
diff --git a/unit_tests/__example_nws_data_params.json b/unit_tests/__example_nws_data_params.json
new file mode 100644
index 000000000..b44e437ed
--- /dev/null
+++ b/unit_tests/__example_nws_data_params.json
@@ -0,0 +1,6 @@
+{
+	"valid_data":
+		{
+			"nws_lid": "smit2"
+		}
+}
diff --git a/unit_tests/__example_nws_data_unittests.py b/unit_tests/__example_nws_data_unittests.py
new file mode 100644
index 000000000..7c679ab2e
--- /dev/null
+++ b/unit_tests/__example_nws_data_unittests.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import unittest
+
+import unit_tests_utils as helpers
+
+# importing python folders in other direcories
+sys.path.append('/foss_fim/src/')
+import nws_data
+
+class test_get_huc8_by_nws_lid(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = '/foss_fim/unit_tests/nws_data_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_get_huc8_by_nws_lid_success(self):
+
+        '''
+            Tests that the correct huc8 is returned with a valid nws-lid
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()
+
+        print(params["nws_lid"])
+    
+        # for now we are happy if no exceptions are thrown.
+        actual_response = nws_data.get_huc8_by_nws_lid(nws_lid = params["nws_lid"])
+        expected_response = '12090301'
+
+        self.assertEqual(actual_response, expected_response, "HUC8 does not match expected result")
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_get_huc8_by_nws_lid_fail_invalid_nws_lid(self):
+
+        '''
+            Tests that the nws_lid does not exist in the nws_lid.gpkg
+            Expecting an exception to be thrown
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        #params = self.params["valid_data"].copy()
+
+        try:
+
+            nws_invalid_value = "abc"
+    
+            # for now we are happy if no exceptions are thrown.
+            actual_response = nws_data.get_huc8_by_nws_lid(nws_lid = nws_invalid_value)
+
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            
+        finally:
+            print("*************************************************************")
+
+
+    def test_get_huc8_by_nws_lid_fail_empty_nws_lid(self):
+
+        '''
+            Tests that the nws_lid is empty
+            Expecting an exception to be thrown
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        #params = self.params["valid_data"].copy()
+
+        try:
+
+            nws_invalid_value = " " # add a blank space to further test
+    
+            # for now we are happy if no exceptions are thrown.
+            actual_response = nws_data.get_huc8_by_nws_lid(nws_lid = nws_invalid_value)
+
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            
+        finally:
+            print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
\ No newline at end of file
diff --git a/unit_tests/__template_params.json b/unit_tests/__template_params.json
new file mode 100644
index 000000000..a6ce3fce8
--- /dev/null
+++ b/unit_tests/__template_params.json
@@ -0,0 +1,11 @@
+{
+	"valid_data":
+	{
+		"some param name (suggested to match the method param name": "some value",
+		"another one and as many lines as you need": "some value",
+		"for 'None', you can just leave this field out, but sometimes it seems like jsut passing the value of None seems to work (not validated) ": "some value",
+		"for numerics, just put the number": 20,
+	   "for lists/collections: ": ["value 1", "value 2"],
+		"inline json comments don't work (let us know if you figure it out)": "sorry"
+	}
+}
\ No newline at end of file
diff --git a/unit_tests/__template_unittests.py b/unit_tests/__template_unittests.py
new file mode 100644
index 000000000..6d0e7ee36
--- /dev/null
+++ b/unit_tests/__template_unittests.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import unittest
+
+import unit_tests_utils as helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import <Your original source python file name>   
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_<Your original source python file name>(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = '/foss_fim/unit_tests/<Your original source python file name>_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_<method name you are about to test>_success(self):
+
+        '''
+        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
+          If there is no return value from the method, please say so.>
+        
+        Dev Notes: (which can be removed after you make this file)
+            Remember... You need to validate the method output if there is any. However, if you have time, it
+            is also recommended that you validate other outputs such as writing or updating file on the file 
+            system, aka: Does the expected file exist. Don't worry about its contents.
+
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        
+       
+        # for now we are happy if no exceptions are thrown.
+        
+        # < See the readme.md, clip_vectors_to_wbd_unittests.py or gms/derive_level_paths_unittests.py 
+        # for examples.>
+        # Replace this stub example with your own.
+        # Try to use the same order to make it easier.
+        # Remember, if the method accepts **params, then you can sent that in here as well.
+        #   ie: my_py_class.my_method(** params)
+        
+        clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
+                                                nwm_streams_filename = params["nwm_streams"],
+                                                nhd_streams_filename = params["nhd_streams"],
+                                                etc, etc for each param)
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    # ***** REMOVE THIS BLOCK IF YOU ARE NOT USING IT ***
+    #       EXAMPLE SUCCESSFUL FAIL    
+    # 
+    #def test_subset_vector_layers_fail_invalid_stream_path(self):
+    #    '''
+    #    Notes about what the test is and the expected results (or expected exception if applicable)
+    #    '''
+    #   #global params_file
+    #   helpers.print_unit_test_function_header()
+        
+    #   params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+
+    #   params["nwm_streams"] = "some bad path"
+
+    #   clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
+    #                                            nwm_streams_filename = params["nwm_streams"],
+    #                                            nhd_streams_filename = params["nhd_streams"],
+    #                                            etc, etc for each param)
+    #   
+       
+    #    print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+    #    print("*************************************************************")
+        
+    # ***********************
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    

From 9d57655ec1601b2c21df05c6bedaa0419e89e4b2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 9 Feb 2022 15:27:07 -0600
Subject: [PATCH 130/359] Bump ipython from 7.29.0 to 7.31.1 (#510)

Bumps [ipython](https://github.com/ipython/ipython) from 7.29.0 to 7.31.1.
- [Release notes](https://github.com/ipython/ipython/releases)
- [Commits](https://github.com/ipython/ipython/compare/7.29.0...7.31.1)

---
updated-dependencies:
- dependency-name: ipython
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Pipfile.lock | 319 +++++++++++++++++++++++++--------------------------
 1 file changed, 156 insertions(+), 163 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index c6a3d3a0e..6474d104d 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -25,11 +25,11 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
-                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
+                "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",
+                "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==21.2.0"
+            "version": "==21.4.0"
         },
         "certifi": {
             "hashes": [
@@ -40,33 +40,39 @@
         },
         "cftime": {
             "hashes": [
-                "sha256:01882a6f540f61b87db7e6c1a389d278e97b63b8c40457df7d5b98a25413c1a7",
-                "sha256:06033c132beb5e560f1957c048fdfbb3c876b3cc2f3e5e0f664927d98b5fae44",
-                "sha256:2ebdc5b20fb113ad0aa41c61fa7f9d52cb3a22b4b847ed2bf653d7e1f81ae8e2",
-                "sha256:35cb5cbcbd9f8083265a3ce4283ebc62114e28d589d6bf62d98532ada6cfc890",
-                "sha256:40517a5c2352c847f919b7f29963015c37c071ebc2e89ecc4f62aa4b80d9a72e",
-                "sha256:4260ca466a445af7b243d84a7c96833b9261bdfd15ba86028d9710b7d700f490",
-                "sha256:5850ed2ae6c218d57249cf75fa27accd4e3436617a8ea9cd2bea6cb031f449fd",
-                "sha256:6db9129b4f26202c1ba4610b667693e686922011bd7166e7573dea668bb70c88",
-                "sha256:6dc4d76ec7fe5a2d3c00dbe6604c757f1319613b75ef157554ef3648bf102a50",
-                "sha256:72eff06667df45cc87c7b50c87ee67471a32b73cdfaac903bb65945823a4bdf3",
-                "sha256:75dc17b031e85feece5a5770f00342dedeb4349dc530bcd1cd25deb7064cab32",
-                "sha256:940fcaec3159e8ba6ad93bd939108c665c35650fcf6cd026687b40b48bca8110",
-                "sha256:a220f9652f34038c0d9ca275bbfb50efc52dbf867733746cf8a7dcd3b5b84cf1",
-                "sha256:ae7151d4c0759f7263d3baf4ca7283f715f99a02cf3bd4cf2403ae0e5312b64e",
-                "sha256:bcca217713f6468ea0fe18637ff2a113bcb5cf9c7328ba535a0323033110e419",
-                "sha256:bcdf2a3d52a36e81e30d07ae93afdeb057cca8eb0a28c21cfef1f523ade1fe74",
-                "sha256:c5d0985d99a4c749d90016e993dedd6c37c5b6d459b5747eb3a14e9ad63dc669",
-                "sha256:cd33cd2e220c1c283f6fe3273352d7754795f86835ad04520f26a441a5b45193",
-                "sha256:db62b4383815596a92acb6cfa3779b208292aa17f90c10669ee50806b2795cee",
-                "sha256:e3196615252312762ce2196664b2614d254d5e4d2b0cf5c7866c5916e45cec66",
-                "sha256:f549ad17102403984a9c865313f758c346cbb04dd4550946cab81a5fea132ff3",
-                "sha256:fae15273aafd0c9382637b03ef6fa51274c28c84dc3b459a969be2121f41c648",
-                "sha256:fd2311da209bbcff4221d344cb6ee217ce6b465832b013106e1ae21e947818f8",
-                "sha256:fdfc3f3da4efc22ae22096c2b20ac9d5da01b3038c4ed4407fcd501efab26099",
-                "sha256:ff292fcd4d2322fe28bcc0c9e70a37e048858aa68a9aaa43710a73100fa1e77d"
-            ],
-            "version": "==1.5.1.1"
+                "sha256:07e8f183ed9143308d098b149539ae2a29b367ca825126efaa835d9fda7c7c3f",
+                "sha256:1ae8313bce817bf4a8de0569bc97332101fc580e457bc9ff7afe42ac090c7c7b",
+                "sha256:1eac1695ffcd7cc02e0fc608c5ba7373f1bc39b56a8e76ba62136f1d9ca7069b",
+                "sha256:1f336afb2841ae769b33a720237ab2a4ccd8c14160f1332d2813a39ac9d1c045",
+                "sha256:1f3d8623b1f6c35a4b8c702bf1f4ec5340713b8e2944fd549f8f8a25abd2f0ff",
+                "sha256:291d7d5f19525d2b667fc4ba7d18bcd95ece8384224f8e8ee22c8e72ebddf4de",
+                "sha256:2b774870f21045c7b0728ac3cdea9c57a2121c01eb95bf17e0ed4aaf58a939ef",
+                "sha256:375d37d9ab8bf501c048e44efce2276296e3d67bb276e891e0e93b0a8bbb988a",
+                "sha256:3fb31f26828c909c0234834a8882c137428d75638734ba8d2ef9e66a57e81fec",
+                "sha256:4bd4a94376989a6cf69d4a59e9e5871d19ded6de9956e4fc6b5d2775d509c49e",
+                "sha256:5c8716983468ebfa9b6fba0040881f431334ea56ad9d1df075ae15cd37ae7985",
+                "sha256:5ff10b4ac0be73ee319c1dded6814e509d52ccb993da8a6496f153f5bb0e6efb",
+                "sha256:66af87cdde654d0cde0a0b936d11e0675b57697b369a142de8d5e5efae1e53d8",
+                "sha256:670bc389b7d7a7e61cf42e7842a820c059efb2adce50eacef51e25389daab01f",
+                "sha256:695f45898a00acceed4795f76b2ac4f1455945fe9e3a174b0ea9fff9dc2c9a17",
+                "sha256:6bfb44c72e277c12f60c4c4b6186963191743bf80a438246d37c7320a95fa54a",
+                "sha256:740d11d0b567390e2892d1dd81387cc2e547b1856ed3969dd51c4a089d9501b2",
+                "sha256:7410c1435d52758c8cf2d46b8021c5bfcabf2ad85e360a466bc2b6d224316a82",
+                "sha256:832c7674f6fec8bdacc09928bd976600d886035ad39d2e781897a0e180552fce",
+                "sha256:a818f59878ed029066748980ecfd435600be76b62f9269ddc06da91668ed506a",
+                "sha256:a9db712b1092771d7e79c3e44838f2ddd751fad874d90d05b37c209c9dd160ea",
+                "sha256:be258adaba9298d00cd420e27f5ba4a3e379aaf4d7009d8826f607ca58c9b813",
+                "sha256:bf7021e635126323ca84622838fef634f9dd5334c1c5658b7a9181bb4d819d27",
+                "sha256:bf9871732414fa2841452fd0f9233e52f4e1bf49ba47005ea81bff0f9c4f40a1",
+                "sha256:c49599320537db1c067f1d77e00264a3d3c53ebde13a0e62269c1c2cf9828d72",
+                "sha256:c8ad56b7185a68fa4b0b27fee949cb7af2de1ca37207e177efc9231a0785562b",
+                "sha256:cfebcf6e9e5afe015d848908cb7d56009aa98b3a63a28bcd6faee18b8d7b75de",
+                "sha256:dfc6a15acce854571d82565aac3d47df40317f2290e8e304a89dcb9351de19eb",
+                "sha256:e7fe234c7bbc04af7d1d8a2ee5649f9a2f2de2aa424996453ed2173302bd9638",
+                "sha256:f24330721024500b5dbe87f164aa30b82a0e785f62d3d09fb9efeac5e5623385",
+                "sha256:f904e3efa04adc727e787b1306ef50c181e9c448a71656938d7483cc104c90c8"
+            ],
+            "version": "==1.5.2"
         },
         "click": {
             "hashes": [
@@ -88,7 +94,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cycler": {
@@ -117,11 +123,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:545c05d0f7903a863c2020e07b8f0a57517f2c40d940bded77076397872d14ca",
-                "sha256:edf251d5d2cc0580d5f72de4621c338d8c66c5f61abb50cf486640f73c8194d5"
+                "sha256:1933415e0fbdf068815cb1baaa1f159e17830215f7e8624e5731122761627557",
+                "sha256:2b18a172120e32128a80efee04cff487d5d140fe7d817deb648b2eee023a40e4"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.28.5"
+            "version": "==4.29.1"
         },
         "geopandas": {
             "hashes": [
@@ -389,39 +395,31 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:00c9fa73a6989895b8815d98300a20ac993c49ac36c8277e8ffeaa3631c0dbbb",
-                "sha256:025b497014bc33fc23897859350f284323f32a2fff7654697f5a5fc2a19e9939",
-                "sha256:08de8472d9f7571f9d51b27b75e827f5296295fa78817032e84464be8bb905bc",
-                "sha256:1964db2d4a00348b7a60ee9d013c8cb0c566644a589eaa80995126eac3b99ced",
-                "sha256:2a9add27d7fc0fdb572abc3b2486eb3b1395da71e0254c5552b2aad2a18b5441",
-                "sha256:2d8adfca843bc46ac199a4645233f13abf2011a0b2f4affc5c37cd552626f27b",
-                "sha256:301e408a052fdcda5cdcf03021ebafc3c6ea093021bf9d1aa47c54d48bdad166",
-                "sha256:311283acf880cfcc20369201bd75da907909afc4666966c7895cbed6f9d2c640",
-                "sha256:341dddcfe3b7b6427a28a27baa59af5ad51baa59bfec3264f1ab287aa3b30b13",
-                "sha256:3a5098df115340fb17fc93867317a947e1dcd978c3888c5ddb118366095851f8",
-                "sha256:3c978544be9e04ed12016dd295a74283773149b48f507d69b36f91aa90a643e5",
-                "sha256:3d893b0871322eaa2f8c7072cdb552d8e2b27645b7875a70833c31e9274d4611",
-                "sha256:4fe6a006557b87b352c04596a6e3f12a57d6e5f401d804947bd3188e6b0e0e76",
-                "sha256:507c05c7a37b3683eb08a3ff993bd1ee1e6c752f77c2f275260533b265ecdb6c",
-                "sha256:58ca1d7c8aef6e996112d0ce873ac9dfa1eaf4a1196b4ff7ff73880a09923ba7",
-                "sha256:61bada43d494515d5b122f4532af226fdb5ee08fe5b5918b111279843dc6836a",
-                "sha256:69a5a8d71c308d7ef33ef72371c2388a90e3495dbb7993430e674006f94797d5",
-                "sha256:6a5928bc6241264dce5ed509e66f33676fc97f464e7a919edc672fb5532221ee",
-                "sha256:7b9d6b14fc9a4864b08d1ba57d732b248f0e482c7b2ff55c313137e3ed4d8449",
-                "sha256:a7c4b701ca418cd39e28ec3b496e6388fe06de83f5f0cb74794fa31cfa384c02",
-                "sha256:a7e8f6216f180f3fd4efb73de5d1eaefb5f5a1ee5b645c67333033e39440e63a",
-                "sha256:b545ebadaa2b878c8630e5bcdb97fc4096e779f335fc0f943547c1c91540c815",
-                "sha256:c293d3c0321996cd8ffe84215ffe5d269fd9d1d12c6f4ffe2b597a7c30d3e593",
-                "sha256:c5562bcc1a9b61960fc8950ade44d00e3de28f891af0acc96307c73613d18f6e",
-                "sha256:ca9c23848292c6fe0a19d212790e62f398fd9609aaa838859be8459bfbe558aa",
-                "sha256:cc1b30205d138d1005adb52087ff45708febbef0e420386f58664f984ef56954",
-                "sha256:dbce7adeb66b895c6aaa1fad796aaefc299ced597f6fbd9ceddb0dd735245354",
-                "sha256:dc4b2fb01f1b4ddbe2453468ea0719f4dbb1f5caa712c8b21bb3dd1480cd30d9",
-                "sha256:eed2afaa97ec33b4411995be12f8bdb95c87984eaa28d76cf628970c8a2d689a",
-                "sha256:fc7a7d7b0ed72589fd8b8486b9b42a564f10b8762be8bd4d9df94b807af4a089"
-            ],
-            "markers": "python_version < '3.11' and python_version >= '3.7'",
-            "version": "==1.21.5"
+                "sha256:0d245a2bf79188d3f361137608c3cd12ed79076badd743dc660750a9f3074f7c",
+                "sha256:26b4018a19d2ad9606ce9089f3d52206a41b23de5dfe8dc947d2ec49ce45d015",
+                "sha256:2db01d9838a497ba2aa9a87515aeaf458f42351d72d4e7f3b8ddbd1eba9479f2",
+                "sha256:3d62d6b0870b53799204515145935608cdeb4cebb95a26800b6750e48884cc5b",
+                "sha256:45a7dfbf9ed8d68fd39763940591db7637cf8817c5bce1a44f7b56c97cbe211e",
+                "sha256:4ac4d7c9f8ea2a79d721ebfcce81705fc3cd61a10b731354f1049eb8c99521e8",
+                "sha256:60f19c61b589d44fbbab8ff126640ae712e163299c2dd422bfe4edc7ec51aa9b",
+                "sha256:632e062569b0fe05654b15ef0e91a53c0a95d08ffe698b66f6ba0f927ad267c2",
+                "sha256:65f5e257987601fdfc63f1d02fca4d1c44a2b85b802f03bd6abc2b0b14648dd2",
+                "sha256:69958735d5e01f7b38226a6c6e7187d72b7e4d42b6b496aca5860b611ca0c193",
+                "sha256:78bfbdf809fc236490e7e65715bbd98377b122f329457fffde206299e163e7f3",
+                "sha256:7e957ca8112c689b728037cea9c9567c27cf912741fabda9efc2c7d33d29dfa1",
+                "sha256:800dfeaffb2219d49377da1371d710d7952c9533b57f3d51b15e61c4269a1b5b",
+                "sha256:831f2df87bd3afdfc77829bc94bd997a7c212663889d56518359c827d7113b1f",
+                "sha256:88d54b7b516f0ca38a69590557814de2dd638d7d4ed04864826acaac5ebb8f01",
+                "sha256:8d1563060e77096367952fb44fca595f2b2f477156de389ce7c0ade3aef29e21",
+                "sha256:b5ec9a5eaf391761c61fd873363ef3560a3614e9b4ead17347e4deda4358bca4",
+                "sha256:bcd19dab43b852b03868796f533b5f5561e6c0e3048415e675bec8d2e9d286c1",
+                "sha256:c51124df17f012c3b757380782ae46eee85213a3215e51477e559739f57d9bf6",
+                "sha256:e348ccf5bc5235fc405ab19d53bec215bb373300e5523c7b476cc0da8a5e9973",
+                "sha256:e60ef82c358ded965fdd3132b5738eade055f48067ac8a5a8ac75acc00cad31f",
+                "sha256:f8ad59e6e341f38266f1549c7c2ec70ea0e3d1effb62a44e5c3dba41c55f0187"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.22.1"
         },
         "packaging": {
             "hashes": [
@@ -455,84 +453,79 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:066f3999cb3b070a95c3652712cffa1a748cd02d60ad7b4e485c3748a04d9d76",
-                "sha256:0a0956fdc5defc34462bb1c765ee88d933239f9a94bc37d132004775241a7585",
-                "sha256:0b052a619a8bfcf26bd8b3f48f45283f9e977890263e4571f2393ed8898d331b",
-                "sha256:1394a6ad5abc838c5cd8a92c5a07535648cdf6d09e8e2d6df916dfa9ea86ead8",
-                "sha256:1bc723b434fbc4ab50bb68e11e93ce5fb69866ad621e3c2c9bdb0cd70e345f55",
-                "sha256:244cf3b97802c34c41905d22810846802a3329ddcb93ccc432870243211c79fc",
-                "sha256:25a49dc2e2f74e65efaa32b153527fc5ac98508d502fa46e74fa4fd678ed6645",
-                "sha256:2e4440b8f00f504ee4b53fe30f4e381aae30b0568193be305256b1462216feff",
-                "sha256:3862b7256046fcd950618ed22d1d60b842e3a40a48236a5498746f21189afbbc",
-                "sha256:3eb1ce5f65908556c2d8685a8f0a6e989d887ec4057326f6c22b24e8a172c66b",
-                "sha256:3f97cfb1e5a392d75dd8b9fd274d205404729923840ca94ca45a0af57e13dbe6",
-                "sha256:493cb4e415f44cd601fcec11c99836f707bb714ab03f5ed46ac25713baf0ff20",
-                "sha256:4acc0985ddf39d1bc969a9220b51d94ed51695d455c228d8ac29fcdb25810e6e",
-                "sha256:5503c86916d27c2e101b7f71c2ae2cddba01a2cf55b8395b0255fd33fa4d1f1a",
-                "sha256:5b7bb9de00197fb4261825c15551adf7605cf14a80badf1761d61e59da347779",
-                "sha256:5e9ac5f66616b87d4da618a20ab0a38324dbe88d8a39b55be8964eb520021e02",
-                "sha256:620582db2a85b2df5f8a82ddeb52116560d7e5e6b055095f04ad828d1b0baa39",
-                "sha256:62cc1afda735a8d109007164714e73771b499768b9bb5afcbbee9d0ff374b43f",
-                "sha256:70ad9e5c6cb9b8487280a02c0ad8a51581dcbbe8484ce058477692a27c151c0a",
-                "sha256:72b9e656e340447f827885b8d7a15fc8c4e68d410dc2297ef6787eec0f0ea409",
-                "sha256:72cbcfd54df6caf85cc35264c77ede902452d6df41166010262374155947460c",
-                "sha256:792e5c12376594bfcb986ebf3855aa4b7c225754e9a9521298e460e92fb4a488",
-                "sha256:7b7017b61bbcdd7f6363aeceb881e23c46583739cb69a3ab39cb384f6ec82e5b",
-                "sha256:81f8d5c81e483a9442d72d182e1fb6dcb9723f289a57e8030811bac9ea3fef8d",
-                "sha256:82aafa8d5eb68c8463b6e9baeb4f19043bb31fefc03eb7b216b51e6a9981ae09",
-                "sha256:84c471a734240653a0ec91dec0996696eea227eafe72a33bd06c92697728046b",
-                "sha256:8c803ac3c28bbc53763e6825746f05cc407b20e4a69d0122e526a582e3b5e153",
-                "sha256:93ce9e955cc95959df98505e4608ad98281fff037350d8c2671c9aa86bcf10a9",
-                "sha256:9a3e5ddc44c14042f0844b8cf7d2cd455f6cc80fd7f5eefbe657292cf601d9ad",
-                "sha256:a4901622493f88b1a29bd30ec1a2f683782e57c3c16a2dbc7f2595ba01f639df",
-                "sha256:a5a4532a12314149d8b4e4ad8ff09dde7427731fcfa5917ff16d0291f13609df",
-                "sha256:b8831cb7332eda5dc89b21a7bce7ef6ad305548820595033a4b03cf3091235ed",
-                "sha256:b8e2f83c56e141920c39464b852de3719dfbfb6e3c99a2d8da0edf4fb33176ed",
-                "sha256:c70e94281588ef053ae8998039610dbd71bc509e4acbc77ab59d7d2937b10698",
-                "sha256:c8a17b5d948f4ceeceb66384727dde11b240736fddeda54ca740b9b8b1556b29",
-                "sha256:d82cdb63100ef5eedb8391732375e6d05993b765f72cb34311fab92103314649",
-                "sha256:d89363f02658e253dbd171f7c3716a5d340a24ee82d38aab9183f7fdf0cdca49",
-                "sha256:d99ec152570e4196772e7a8e4ba5320d2d27bf22fdf11743dd882936ed64305b",
-                "sha256:ddc4d832a0f0b4c52fff973a0d44b6c99839a9d016fe4e6a1cb8f3eea96479c2",
-                "sha256:e3dacecfbeec9a33e932f00c6cd7996e62f53ad46fbe677577394aaa90ee419a",
-                "sha256:eb9fc393f3c61f9054e1ed26e6fe912c7321af2f41ff49d3f83d05bacf22cc78"
+                "sha256:03b27b197deb4ee400ed57d8d4e572d2d8d80f825b6634daf6e2c18c3c6ccfa6",
+                "sha256:0b281fcadbb688607ea6ece7649c5d59d4bbd574e90db6cd030e9e85bde9fecc",
+                "sha256:0ebd8b9137630a7bbbff8c4b31e774ff05bbb90f7911d93ea2c9371e41039b52",
+                "sha256:113723312215b25c22df1fdf0e2da7a3b9c357a7d24a93ebbe80bfda4f37a8d4",
+                "sha256:2d16b6196fb7a54aff6b5e3ecd00f7c0bab1b56eee39214b2b223a9d938c50af",
+                "sha256:2fd8053e1f8ff1844419842fd474fc359676b2e2a2b66b11cc59f4fa0a301315",
+                "sha256:31b265496e603985fad54d52d11970383e317d11e18e856971bdbb86af7242a4",
+                "sha256:3586e12d874ce2f1bc875a3ffba98732ebb12e18fb6d97be482bd62b56803281",
+                "sha256:47f5cf60bcb9fbc46011f75c9b45a8b5ad077ca352a78185bd3e7f1d294b98bb",
+                "sha256:490e52e99224858f154975db61c060686df8a6b3f0212a678e5d2e2ce24675c9",
+                "sha256:500d397ddf4bbf2ca42e198399ac13e7841956c72645513e8ddf243b31ad2128",
+                "sha256:52abae4c96b5da630a8b4247de5428f593465291e5b239f3f843a911a3cf0105",
+                "sha256:6579f9ba84a3d4f1807c4aab4be06f373017fc65fff43498885ac50a9b47a553",
+                "sha256:68e06f8b2248f6dc8b899c3e7ecf02c9f413aab622f4d6190df53a78b93d97a5",
+                "sha256:6c5439bfb35a89cac50e81c751317faea647b9a3ec11c039900cd6915831064d",
+                "sha256:72c3110228944019e5f27232296c5923398496b28be42535e3b2dc7297b6e8b6",
+                "sha256:72f649d93d4cc4d8cf79c91ebc25137c358718ad75f99e99e043325ea7d56100",
+                "sha256:7aaf07085c756f6cb1c692ee0d5a86c531703b6e8c9cae581b31b562c16b98ce",
+                "sha256:80fe92813d208ce8aa7d76da878bdc84b90809f79ccbad2a288e9bcbeac1d9bd",
+                "sha256:95545137fc56ce8c10de646074d242001a112a92de169986abd8c88c27566a05",
+                "sha256:97b6d21771da41497b81652d44191489296555b761684f82b7b544c49989110f",
+                "sha256:98cb63ca63cb61f594511c06218ab4394bf80388b3d66cd61d0b1f63ee0ea69f",
+                "sha256:9f3b4522148586d35e78313db4db0df4b759ddd7649ef70002b6c3767d0fdeb7",
+                "sha256:a09a9d4ec2b7887f7a088bbaacfd5c07160e746e3d47ec5e8050ae3b2a229e9f",
+                "sha256:b5050d681bcf5c9f2570b93bee5d3ec8ae4cf23158812f91ed57f7126df91762",
+                "sha256:bb47a548cea95b86494a26c89d153fd31122ed65255db5dcbc421a2d28eb3379",
+                "sha256:bc462d24500ba707e9cbdef436c16e5c8cbf29908278af053008d9f689f56dee",
+                "sha256:c2067b3bb0781f14059b112c9da5a91c80a600a97915b4f48b37f197895dd925",
+                "sha256:d154ed971a4cc04b93a6d5b47f37948d1f621f25de3e8fa0c26b2d44f24e3e8f",
+                "sha256:d5dcea1387331c905405b09cdbfb34611050cc52c865d71f2362f354faee1e9f",
+                "sha256:ee6e2963e92762923956fe5d3479b1fdc3b76c83f290aad131a2f98c3df0593e",
+                "sha256:fd0e5062f11cb3e730450a7d9f323f4051b532781026395c4323b8ad055523c4"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==8.4.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==9.0.0"
         },
         "psutil": {
             "hashes": [
-                "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64",
-                "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131",
-                "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c",
-                "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6",
-                "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023",
-                "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df",
-                "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394",
-                "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4",
-                "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b",
-                "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2",
-                "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d",
-                "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65",
-                "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d",
-                "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef",
-                "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7",
-                "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60",
-                "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6",
-                "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8",
-                "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b",
-                "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d",
-                "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac",
-                "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935",
-                "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d",
-                "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28",
-                "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876",
-                "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0",
-                "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3",
-                "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"
+                "sha256:072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5",
+                "sha256:1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a",
+                "sha256:1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4",
+                "sha256:3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841",
+                "sha256:32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d",
+                "sha256:3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d",
+                "sha256:3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0",
+                "sha256:4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845",
+                "sha256:539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf",
+                "sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b",
+                "sha256:58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07",
+                "sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618",
+                "sha256:742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2",
+                "sha256:7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd",
+                "sha256:76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666",
+                "sha256:7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce",
+                "sha256:7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3",
+                "sha256:8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d",
+                "sha256:869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25",
+                "sha256:90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492",
+                "sha256:9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b",
+                "sha256:b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d",
+                "sha256:c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2",
+                "sha256:c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203",
+                "sha256:cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2",
+                "sha256:d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94",
+                "sha256:df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9",
+                "sha256:e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64",
+                "sha256:e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56",
+                "sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3",
+                "sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c",
+                "sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3"
             ],
             "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.8.0"
+            "version": "==5.9.0"
         },
         "pygeos": {
             "hashes": [
@@ -558,11 +551,11 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4",
-                "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"
+                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
+                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==3.0.6"
+            "version": "==3.0.7"
         },
         "pyproj": {
             "hashes": [
@@ -594,7 +587,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.2"
         },
         "python-dotenv": {
@@ -700,11 +693,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:032e5949c02878c405552fb800743aad96940097460bc6f115340402852dcdde",
-                "sha256:989ab2d3e632ba23358b8d43950bd46babde5bbb1516760dc5f5ccfe7accdd45"
+                "sha256:c99207037c38984eae838c2fd986f39a9ddf4fabfe0fddd957e622d1d1dcdd05",
+                "sha256:eb83b1012ae6bf436901c2a2cee35d45b7260f31fd4b65fd1e50a9f99c11d7f8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==60.0.3"
+            "version": "==60.6.0"
         },
         "shapely": {
             "hashes": [
@@ -795,7 +788,7 @@
                 "sha256:f63600ec06982cdf480899026f4fda622776f5fabed9a869fdb32d72bc17e99a",
                 "sha256:fb62d517a516128bacf08cb6a86ecd39fb06d08e7c4980251f5d5601d29989ba"
             ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==3.17.6"
         },
         "six": {
@@ -803,7 +796,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "snuggs": {
@@ -870,19 +863,19 @@
         },
         "decorator": {
             "hashes": [
-                "sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374",
-                "sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7"
+                "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330",
+                "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==5.1.0"
+            "version": "==5.1.1"
         },
         "ipython": {
             "hashes": [
-                "sha256:cb6aef731bf708a7727ab6cde8df87f0281b1427d41e65d62d4b68934fa54e97",
-                "sha256:fc60ef843e0863dd4e24ab2bb5698f071031332801ecf8d1aeb4fb622056545c"
+                "sha256:55df3e0bd0f94e715abd968bedd89d4e8a7bce4bf498fb123fed4f5398fea874",
+                "sha256:b5548ec5329a4bcf054a5deed5099b0f9622eb9ea51aaa7104d215fece201d8c"
             ],
             "index": "pypi",
-            "version": "==7.30.1"
+            "version": "==7.31.1"
         },
         "jedi": {
             "hashes": [
@@ -925,11 +918,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:1bb05628c7d87b645974a1bad3f17612be0c29fa39af9f7688030163f680bad6",
-                "sha256:e56f2ff799bacecd3e88165b1e2f5ebf9bcd59e80e06d395fa0cc4b8bd7bb506"
+                "sha256:4bcf119be2200c17ed0d518872ef922f1de336eb6d1ddbd1e089ceb6447d97c6",
+                "sha256:a51d41a6a45fd9def54365bca8f0402c8f182f2b6f7e29c74d55faeb9fb38ac4"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.24"
+            "version": "==3.0.26"
         },
         "ptyprocess": {
             "hashes": [
@@ -940,19 +933,19 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380",
-                "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"
+                "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65",
+                "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==2.10.0"
+            "version": "==2.11.2"
         },
         "setuptools": {
             "hashes": [
-                "sha256:032e5949c02878c405552fb800743aad96940097460bc6f115340402852dcdde",
-                "sha256:989ab2d3e632ba23358b8d43950bd46babde5bbb1516760dc5f5ccfe7accdd45"
+                "sha256:c99207037c38984eae838c2fd986f39a9ddf4fabfe0fddd957e622d1d1dcdd05",
+                "sha256:eb83b1012ae6bf436901c2a2cee35d45b7260f31fd4b65fd1e50a9f99c11d7f8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==60.0.3"
+            "version": "==60.6.0"
         },
         "traitlets": {
             "hashes": [

From d133addd4d730b5c468dcf1a8f7dfab35c55cbd7 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 25 Feb 2022 16:08:22 -0600
Subject: [PATCH 131/359] Suppress unit tests warnings which made outputs
 harder to read

---
 unit_tests/__example_nws_data_unittests.py     | 3 +++
 unit_tests/__template_unittests.py             | 3 +++
 unit_tests/clip_vectors_to_wbd_unittests.py    | 3 +++
 unit_tests/gms/derive_level_paths_unittests.py | 6 +++++-
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/unit_tests/__example_nws_data_unittests.py b/unit_tests/__example_nws_data_unittests.py
index 7c679ab2e..8834b5b84 100644
--- a/unit_tests/__example_nws_data_unittests.py
+++ b/unit_tests/__example_nws_data_unittests.py
@@ -6,6 +6,7 @@
 
 import argparse
 import json
+import warnings
 import unittest
 
 import unit_tests_utils as helpers
@@ -22,6 +23,8 @@ class test_get_huc8_by_nws_lid(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
+        warnings.simplefilter('ignore')
+        
         params_file_path = '/foss_fim/unit_tests/nws_data_params.json'
     
         with open(params_file_path) as params_file:
diff --git a/unit_tests/__template_unittests.py b/unit_tests/__template_unittests.py
index 6d0e7ee36..01195723b 100644
--- a/unit_tests/__template_unittests.py
+++ b/unit_tests/__template_unittests.py
@@ -6,6 +6,7 @@
 
 import argparse
 import json
+import warnings
 import unittest
 
 import unit_tests_utils as helpers
@@ -24,6 +25,8 @@ class test_<Your original source python file name>(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
+        warnings.simplefilter('ignore')
+
         params_file_path = '/foss_fim/unit_tests/<Your original source python file name>_params.json'
     
         with open(params_file_path) as params_file:
diff --git a/unit_tests/clip_vectors_to_wbd_unittests.py b/unit_tests/clip_vectors_to_wbd_unittests.py
index 47f4c1b36..619a0dc6c 100644
--- a/unit_tests/clip_vectors_to_wbd_unittests.py
+++ b/unit_tests/clip_vectors_to_wbd_unittests.py
@@ -6,6 +6,7 @@
 
 import argparse
 import json
+import warnings
 import unittest
 
 import unit_tests_utils as helpers
@@ -24,6 +25,8 @@ class test_clip_vectors_to_wbd(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
+        warnings.simplefilter('ignore')
+
         params_file_path = '/foss_fim/unit_tests/clip_vectors_to_wbd_params.json'
     
         with open(params_file_path) as params_file:
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
index bdd58c217..ac2f9ba43 100644
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -3,9 +3,11 @@
 import inspect
 import os
 import sys
-import unittest
 
+import argparse
 import json
+import warnings
+import unittest
 
 # importing python folders in other direcories
 sys.path.append('/foss_fim/unit_tests/')
@@ -26,6 +28,8 @@ class test_Derive_level_paths(unittest.TestCase):
     @classmethod
     def setUpClass(self):
     
+        warnings.simplefilter('ignore')
+        
         params_file_path = '/foss_fim/unit_tests/gms/derive_level_paths_params.json'    
     
         with open(params_file_path) as params_file:

From 45c7d6cb3e9fd4c8c88cb828eb34b89b2219739d Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 3 Mar 2022 15:52:23 -0600
Subject: [PATCH 132/359] New Feature to remove stream orders 1 and 2 if
 desired via command arg

See Changelog v4.0.2.0 for full details.

This addresses #552.
---
 docs/CHANGELOG.md                             |  20 +++
 gms_run_unit.sh                               |  17 +-
 src/gms/derive_level_paths.py                 |  42 +++--
 src/gms/run_by_unit.sh                        |   2 +-
 src/gms/stream_branches.py                    |  31 +++-
 unit_tests/gms/derive_level_paths_params.json |   3 +-
 .../gms/derive_level_paths_unittests.py       | 149 ++++++++++++++++--
 7 files changed, 227 insertions(+), 37 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9924ef72d..d165bd915 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,26 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.2.0 - 2022-03-02 - [PR #548](https://github.com/NOAA-OWP/inundation-mapping/pull/548)
+
+Added a new optional system which allows an argument to be added to the `gms_run_unit.sh` command line to filter out stream orders 1 and 2 when calculating branches. 
+
+## Changes
+
+- `gms_run_unit.sh`: Add the new optional `-s` command line argument. Inclusion of this argument means "drop stream orders 1 and 2".
+
+- `src/gms`
+   - `run_by_unit.sh`: Capture and forward the drop stream orders flag to `derive_level_paths.py`
+	
+	- `derive_level_paths.py`: Capture the drop stream order flag and working with `stream_branches.py` to include/not include loading nwm stream with stream orders 1 and 2.
+	
+	- `stream_branchs.py`: A correction was put in place to allow for the filter of branch attributes and values to be excluded. The `from_file` method has the functionality but was incomplete. This was corrected and how could accept the values from `derive_level_paths.py` to use the branch attribute of "order_" (gkpg field) and values excluded of [1,2] when optionally desired.
+
+- `unit_tests/gms`
+	- `derive_level_paths_unittests.py` and `derive_level_paths_params.py`: Updated for testing for the new "drop stream orders 1 and 2" feature. Upgrades were also made to earlier existing incomplete test methods to test more output conditions.
+	
+<br/><br/>
+
 ## v4.0.1.0 - 2022-02-02 - [PR #525](https://github.com/NOAA-OWP/cahaba/pull/525)
 
 The addition of a very simple and evolving unit test system which has two unit tests against two py files.  This will set a precendence and will grow over time and may be automated, possibly during git check-in triggered. The embedded README.md has more details of what we currently have, how to use it, how to add new unit tests, and expected future enhancements.
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index b75274167..857c41712 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -3,7 +3,8 @@
 usage ()
 {
     echo 'Produce GMS hydrofabric datasets for unit scale. Run after fim_run.sh but before gms_run_branch.sh'
-    echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -c <config file> -n <run name> ] [OPT: -h -j <job limit>]'
+    echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -c <config file> -n <run name> ]'
+    echo '  	 						 [OPT: -h -j <job limit>] -o -r -d <deny list file> -s ]'
     echo ''
     echo 'REQUIRED:'
     echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
@@ -13,12 +14,14 @@ usage ()
     echo ''
     echo 'OPTIONS:'
     echo '  -h/--help       : help file'
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
+    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
     echo '  -r/--retry      : retries failed jobs'
     echo '  -d/--denylist   : file with line delimited list of files in huc directories to remove upon completion'
     echo '                   (see config/deny_gms_unit_default.lst for a starting point)'
+	 echo '  -s/--dropStreamOrder_1_2 : If this flag is included, the system will leave out stream orders 1 and 2 at the very'
+	 echo	'                     top initial load of the nwm_subset_streams'
     exit
 }
 
@@ -62,6 +65,9 @@ in
         shift
         deny_gms_unit_list=$1
         ;;
+	 -s|--dropLowStreamOrders)
+		  dropLowStreamOrders=1
+		  ;;
     *) ;;
     esac
     shift
@@ -92,6 +98,10 @@ if [ -z "$retry" ]
 then
     retry=""
 fi
+if [ -z "$dropLowStreamOrders" ]
+then
+    dropLowStreamOrders=0
+fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
@@ -105,7 +115,10 @@ fi
 ## Define Outputs Data Dir & Log File##
 export outputRunDataDir=$outputDataDir/$runName
 logFile=$outputRunDataDir/logs/unit/summary_gms_unit.log
+
+## Set misc global variables
 export overwrite=$overwrite
+export dropLowStreamOrders=$dropLowStreamOrders
 
 ## Define inputs
 export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index 5a90d9ace..185c55fba 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -6,18 +6,18 @@
 import geopandas as gpd
 
 
-def Derive_level_paths(in_stream_network, out_stream_network,branch_id_attribute,
-                       out_stream_network_dissolved=None,huc_id=None,
-                       headwaters_outfile=None,catchments=None,
+def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribute,
+                       out_stream_network_dissolved=None, huc_id=None,
+                       headwaters_outfile=None, catchments=None,
                        catchments_outfile=None,
                        branch_inlets_outfile=None,
-                       toNode_attribute='To_Node',fromNode_attribute='From_Node',
-                       reach_id_attribute='HydroID',verbose=False
-                       ):
-
+                       toNode_attribute='To_Node', fromNode_attribute='From_Node',
+                       reach_id_attribute='HydroID', verbose=False,
+                       drop_low_stream_orders=False ):
+    
     # getting foss_id of huc8
     #foss_id = get_fossid_from_huc8(huc8_id=huc_id,foss_id_attribute='fossid',
-                                   #hucs_layerName='WBDHU8')
+                                   #hucs_layerName='WBDHU8')    
     
     if verbose:
         print("Deriving level paths ...")
@@ -25,8 +25,15 @@ def Derive_level_paths(in_stream_network, out_stream_network,branch_id_attribute
     # load file
     if verbose:
         print("Loading stream network ...")
-    stream_network = StreamNetwork.from_file(in_stream_network)
-
+        
+    if (drop_low_stream_orders):
+        stream_network = StreamNetwork.from_file(filename=in_stream_network,
+                                                 branch_id_attribute="order_",
+                                                 values_excluded=[1,2]
+                                                 )
+    else:
+        stream_network = StreamNetwork.from_file(filename=in_stream_network)
+                                                 
     inlets_attribute = 'inlet_id'
     outlets_attribute = 'outlet_id'
     outlet_linestring_index = -1
@@ -152,15 +159,16 @@ def Derive_level_paths(in_stream_network, out_stream_network,branch_id_attribute
     parser = argparse.ArgumentParser(description='Create stream network level paths')
     parser.add_argument('-i','--in-stream-network', help='Input stream network', required=True)
     parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
-    parser.add_argument('-u','--huc-id', help='Current HUC ID', required=False,default=None)
-    parser.add_argument('-r','--reach-id-attribute', help='Reach ID attribute to use in source file', required=False,default='HydroID')
+    parser.add_argument('-u','--huc-id', help='Current HUC ID', required=False, default=None)
+    parser.add_argument('-r','--reach-id-attribute', help='Reach ID attribute to use in source file', required=False, default='HydroID')
     parser.add_argument('-c','--catchments', help='NWM catchments to append level path data to', required=False, default=None)
     parser.add_argument('-t','--catchments-outfile', help='NWM catchments outfile with appended level path data', required=False, default=None)
-    parser.add_argument('-n','--branch_inlets_outfile', help='Output level paths inlets', required=False,default=None)
-    parser.add_argument('-o','--out-stream-network', help='Output stream network', required=False,default=None)
-    parser.add_argument('-e','--headwaters-outfile', help='Output stream network headwater points', required=False,default=None)
-    parser.add_argument('-d','--out-stream-network-dissolved', help='Dissolved output stream network', required=False,default=None)
-    parser.add_argument('-v','--verbose', help='Verbose output', required=False,default=False,action='store_true')
+    parser.add_argument('-n','--branch_inlets_outfile', help='Output level paths inlets', required=False, default=None)
+    parser.add_argument('-o','--out-stream-network', help='Output stream network', required=False, default=None)
+    parser.add_argument('-e','--headwaters-outfile', help='Output stream network headwater points', required=False, default=None)
+    parser.add_argument('-d','--out-stream-network-dissolved', help='Dissolved output stream network', required=False, default=None)
+    parser.add_argument('-v','--verbose', help='Verbose output', required=False, default=False, action='store_true')
+    parser.add_argument('-s','--drop-low-stream-orders', help='Drop stream orders 1 and 2', type=int, required=False, default=False)
     
     args = vars(parser.parse_args())
 
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index b89f695cd..40a9b04cd 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -67,7 +67,7 @@ Tcount
 echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v
+$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -s $dropLowStreamOrders
 Tcount
 
 ## STREAM BRANCH POLYGONS
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index 7a28741df..fda5ed195 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -19,6 +19,20 @@
 
 class StreamNetwork(gpd.GeoDataFrame):
 
+    '''
+    Notes:
+        - Many of the methods support two attributes called branch_id_attribute and values_excluded.
+          This can be used to filter out records. 
+            ie) When calling the nwm_subset_streams.gpkg, you can filter some records like this:
+                StreamNetwork.from_file(filename=outputs/<some huc>/nwm_subset_streams.gpkg,
+                                                 branch_id_attribute="order_",
+                                                 values_excluded=[1,2]
+                (which means drop all records that have an order_ of 1 or 2.
+                
+        - Note: from_file is using its branch_id_attribute and values_excluded as intended but other
+             methods may not be incomplete and not filtering as expected.
+    '''
+    
     geom_name = 'geometry' # geometry attribute name
     branch_id_attribute = None # branch id attribute name
     values_excluded = None
@@ -39,7 +53,8 @@ def __init__(self,*args,**kwargs):
 
 
     @classmethod
-    def from_file(cls,filename,branch_id_attribute=None,values_excluded=None,attribute_excluded=None, verbose=False,*args,**kwargs):
+    def from_file(cls, filename, branch_id_attribute=None, values_excluded=None, 
+                  attribute_excluded=None, verbose=False, *args, **kwargs):
 
         """ loads stream network from file to streamnetwork geopandas """
 
@@ -56,8 +71,20 @@ def from_file(cls,filename,branch_id_attribute=None,values_excluded=None,attribu
             
         if verbose: 
             print('Loading file')
+            
+        raw_df = gpd.read_file(filename,*args,**kwargs)
+        filtered_df = gpd.GeoDataFrame()
+                      
+        if (branch_id_attribute is not None) and (values_excluded is not None):
+            filtered_df = raw_df[~raw_df[branch_id_attribute].isin(values_excluded)]
+        else:
+            filtered_df = raw_df
+       
+        if verbose:         
+             print("======" + filename)
+             print("Number of df rows = " + str(filtered_df.shape[0]))
         
-        return(cls(gpd.read_file(filename,*args,**kwargs),**inputs))
+        return(cls(filtered_df,**inputs))
 
 
     def write(self,fileName,layer=None,index=True,verbose=False):
diff --git a/unit_tests/gms/derive_level_paths_params.json b/unit_tests/gms/derive_level_paths_params.json
index 6e63343b7..e26c7aadf 100644
--- a/unit_tests/gms/derive_level_paths_params.json
+++ b/unit_tests/gms/derive_level_paths_params.json
@@ -10,7 +10,8 @@
 		"catchments_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
 		"branch_inlets_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
-		"verbose": false
+		"verbose": false,
+		"drop_low_stream_orders": false
 	}
 }
 
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
index ac2f9ba43..bca18320a 100644
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -16,6 +16,7 @@
 # importing python folders in other direcories
 sys.path.append('/foss_fim/src/gms/')
 import derive_level_paths
+import stream_branches
 
 
 # NOTE: This goes directly to the function.
@@ -36,15 +37,11 @@ def setUpClass(self):
             self.params = json.load(params_file)
 
 
-    # MUST start with the name of "test_"
-    def test_Derive_level_paths_success(self):
 
+    def test_Derive_level_paths_success_all_params(self):
+    
         '''
-        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
-        Most of the output test and internal tests with this function will test a wide variety of conditions.
-        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
-        Only the basic return output value shoudl be tested to ensure it is as expected.
-        For now, we are adding the very basic "happy path" test.
+        This test includes all params with many optional parms being set to the default value of the function
         '''
        
         # makes output readability easier and consistant with other unit tests       
@@ -56,7 +53,75 @@ def test_Derive_level_paths_success(self):
         # huc_ids no longer used, so it is not submitted
         # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
         # skip them here.
-        actual = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
+        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
+        actual_df = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                                       out_stream_network = params["out_stream_network"],
+                                                       branch_id_attribute = params["branch_id_attribute"],
+                                                       out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                                       headwaters_outfile = params["headwaters_outfile"],
+                                                       catchments = params["catchments"],
+                                                       catchments_outfile = params["catchments_outfile"],
+                                                       branch_inlets_outfile = params["branch_inlets_outfile"],
+                                                       reach_id_attribute = params["reach_id_attribute"],
+                                                       verbose = params["verbose"],
+                                                       drop_low_stream_orders=params["drop_low_stream_orders"])
+
+        # -----------
+        # test data type being return is as expected. Downstream code might to know that type
+        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
+        
+        # -----------
+        #**** NOTE: Based on 05030104         
+        # Test row count for dissolved level path GeoDataframe which is returned.
+        actual_row_count = len(actual_df) 
+        expected_row_count = 58
+        self.assertEqual(actual_row_count, expected_row_count)
+        
+        # -----------
+        # Test that output files exist as expected
+        if os.path.exists(params["out_stream_network"]) == False:
+            raise Exception(params["out_stream_network"] + " does not exist")
+            
+        if os.path.exists(params["out_stream_network_dissolved"]) == False:
+            raise Exception(params["out_stream_network_dissolved"] + " does not exist")
+
+        if os.path.exists(params["headwaters_outfile"]) == False:
+            raise Exception(params["headwaters_outfile"] + " does not exist")
+
+        #if os.path.exists(params["catchments_outfile"]) == False:
+        #    raise Exception(params["catchments_outfile"] + " does not exist")
+
+        if os.path.exists(params["catchments_outfile"]) == False:
+            raise Exception(params["catchments_outfile"] + " does not exist")
+
+        if os.path.exists(params["branch_inlets_outfile"]) == False:
+            raise Exception(params["branch_inlets_outfile"] + " does not exist")
+       
+        
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")        
+
+
+    def test_Derive_level_paths_success_drop_low_stream_orders_not_submitted(self):
+    
+        '''
+        This test includes most params but does not submit a drop_low_stream_orders param
+        and it should default to "false", meaning no filtering out of stream orders 1 and 2.
+        Note: Most path tests done in test_Derive_level_paths_success_all_params 
+        and are not repeated here.
+        '''
+        
+        # makes output readability easier and consistant with other unit tests       
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()
+
+        # Function Notes:
+        # huc_ids no longer used, so it is not submitted
+        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
+        # skip them here.
+        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
+        actual_df = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
                                                        out_stream_network = params["out_stream_network"],
                                                        branch_id_attribute = params["branch_id_attribute"],
                                                        out_stream_network_dissolved = params["out_stream_network_dissolved"],
@@ -65,24 +130,80 @@ def test_Derive_level_paths_success(self):
                                                        catchments_outfile = params["catchments_outfile"],
                                                        branch_inlets_outfile = params["branch_inlets_outfile"],
                                                        reach_id_attribute = params["reach_id_attribute"],
-                                                       verbose = params["verbose"] )
+                                                       verbose = params["verbose"])
+
+        # -----------
+        # test data type being return is as expected. Downstream code might to know that type
+        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
         
-        #print(actual)
+        # -----------
+        #**** NOTE: Based on 05030104         
+        # Test row count for dissolved level path GeoDataframe which is returned.
+        actual_row_count = len(actual_df) 
+        expected_row_count = 58    # should still be 58 with no filtering
+        self.assertEqual(actual_row_count, expected_row_count)
+        
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")        
         
-        # Later, we should check here that the files outputed to the file system, exist and are valid and those do not have to be new "test_" methods.
-        # We can start using unit test "asserts" as we go as well.
         
+
+    def test_Derive_level_paths_success_drop_low_stream_orders_is_true(self):
+    
+        '''
+        This test includes most params but does not submit a drop_low_stream_orders param
+        and it should default to "false", meaning no filtering out of stream orders 1 and 2.
+        Note: Most path tests done in test_Derive_level_paths_success_all_params 
+        and are not repeated here.
+        '''
+        
+        # makes output readability easier and consistant with other unit tests       
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()
+
+        params["drop_low_stream_orders"] = True
+
+        # Function Notes:
+        # huc_ids no longer used, so it is not submitted
+        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
+        # skip them here.
+        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
+        actual_df = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                                       out_stream_network = params["out_stream_network"],
+                                                       branch_id_attribute = params["branch_id_attribute"],
+                                                       out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                                       headwaters_outfile = params["headwaters_outfile"],
+                                                       catchments = params["catchments"],
+                                                       catchments_outfile = params["catchments_outfile"],
+                                                       branch_inlets_outfile = params["branch_inlets_outfile"],
+                                                       reach_id_attribute = params["reach_id_attribute"],
+                                                       verbose = params["verbose"],
+                                                       drop_low_stream_orders=params["drop_low_stream_orders"])
+
+        # -----------
+        # test data type being return is as expected. Downstream code might to know that type
+        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
+        
+        # -----------
+        #**** NOTE: Based on 05030104         
+        # Test row count for dissolved level path GeoDataframe which is returned.
+        actual_row_count = len(actual_df) 
+        expected_row_count = 4
+        self.assertEqual(actual_row_count, expected_row_count)
+
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")        
 
     
-    # Invalid Input stream
-    # MUST start with the name of "test_"    
+    # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
     def test_Derive_level_paths_invalid_input_stream_network(self):
 
         # makes output readability easier and consistant with other unit tests 
         helpers.print_unit_test_function_header()
         
+        # NOTE: As we are expecting an exception, this MUST have a try catch.
+        #   for "success" tests (as above functions), they CAN NOT have a try/catch
         try:
         
             params = self.params["valid_data"].copy()

From b11f98150a81d17887a60a4f11bd9d41f9d75cb5 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 4 Mar 2022 13:31:37 -0600
Subject: [PATCH 133/359] FIM 4 USGS gage crosswalk and Sierra Test

See Changelog v4.0.3.0 for full details.

This resolves #539.
---
 config/deny_gms_branches_default.lst          |   1 +
 config/deny_gms_branches_min.lst              |   1 +
 config/deny_gms_unit_default.lst              |   1 +
 docs/CHANGELOG.md                             |  27 ++
 gms_run_branch.sh                             |   2 +
 src/gms/run_by_branch.sh                      |   9 +
 src/gms/run_by_unit.sh                        |   7 +
 src/usgs_gage_aggregate.py                    |  98 ++++
 src/usgs_gage_crosswalk.py                    | 203 ++++----
 src/usgs_gage_unit_setup.py                   | 132 +++++
 tools/rating_curve_comparison.py              | 454 +++++++++++++++---
 .../rating_curve_comparison_params.json       |  44 ++
 .../rating_curve_comparison_unittests.py      | 105 ++++
 unit_tests/usgs_gage_crosswalk_params.json    |  13 +
 unit_tests/usgs_gage_crosswalk_unittests.py   |  89 ++++
 15 files changed, 1036 insertions(+), 150 deletions(-)
 create mode 100644 src/usgs_gage_aggregate.py
 create mode 100755 src/usgs_gage_unit_setup.py
 create mode 100644 unit_tests/rating_curve_comparison_params.json
 create mode 100644 unit_tests/rating_curve_comparison_unittests.py
 create mode 100644 unit_tests/usgs_gage_crosswalk_params.json
 create mode 100644 unit_tests/usgs_gage_crosswalk_unittests.py

diff --git a/config/deny_gms_branches_default.lst b/config/deny_gms_branches_default.lst
index 718bea8a1..ec7043e11 100644
--- a/config/deny_gms_branches_default.lst
+++ b/config/deny_gms_branches_default.lst
@@ -56,3 +56,4 @@ src_base_{}.csv
 stage_{}.txt
 streamOrder_{}.tif
 treeFile_{}.txt
+#usgs_elev_table.csv
\ No newline at end of file
diff --git a/config/deny_gms_branches_min.lst b/config/deny_gms_branches_min.lst
index 0ff709c82..b11c17d06 100644
--- a/config/deny_gms_branches_min.lst
+++ b/config/deny_gms_branches_min.lst
@@ -56,3 +56,4 @@ src_base_{}.csv
 stage_{}.txt
 streamOrder_{}.tif
 treeFile_{}.txt
+#usgs_elev_table.csv
diff --git a/config/deny_gms_unit_default.lst b/config/deny_gms_unit_default.lst
index f19e57773..dc5bb7277 100644
--- a/config/deny_gms_unit_default.lst
+++ b/config/deny_gms_unit_default.lst
@@ -13,6 +13,7 @@ nwm_subset_streams.gpkg
 #nwm_subset_streams_levelPaths.gpkg
 #nwm_subset_streams_levelPaths_dissolved.gpkg
 #nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+#usgs_subset_gages.gpkg
 #wbd.gpkg
 #wbd8_clp.gpkg
 wbd_buffered.gpkg
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d165bd915..4da8370ca 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,33 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.3.0 - 2022-03-03 - [PR #550](https://github.com/NOAA-OWP/inundation-mapping/pull/550)
+
+This PR ports the functionality of `usgs_gage_crosswalk.py` and `rating_curve_comparison.py` to FIM 4.
+
+## Additions
+
+- `src/`:
+    - `usgs_gage_aggregate.py`: Aggregates all instances of `usgs_elev_table.csv` to the HUC level. This makes it easier to view the gages in each HUC without having to hunt through branch folders and easier for the Sierra Test to run at the HUC level.
+    - `usgs_gage_unit_setup.py`: Assigns a branch to each USGS gage within a unit. The output of this module is `usgs_subset_gages.gpkg` at the HUC level containing the `levpa_id` attribute.
+
+## Changes
+
+- `gms_run_branch.sh`: Added a line to aggregate all `usgs_elev_table.csv` into the HUC directory level using `src/usgs_gage_aggregate.py`.
+- `src/`:
+    -  `gms/`
+        - `run_by_branch.sh`: Added a block to run `src/usgs_gage_crosswalk.py`. 
+        - `run_by_unit.sh`: Added a block to run `src/usgs_gage_unit_setup.py`.
+    - `usgs_gage_crosswalk.py`: Similar to it's functionality in FIM 3, this module snaps USGS gages to the stream network, samples the underlying DEMs, and writes the attributes to `usgs_elev_table.csv`. This CSV is later aggregated to the HUC level and eventually used in `tools/rating_curve_comparison.py`. Addresses #539 
+- `tools/rating_curve_comparison.py`: Updated Sierra Test to work with FIM 4 data structure.
+- `unit_tests/`:
+    - `rating_curve_comparison_unittests.py` & `rating_curve_comparison_params.json`: Unit test code and parameters for the Sierra Test.
+    - `usgs_gage_crosswalk_unittests.py` & `usgs_gage_crosswalk_params.json`: Unit test code and parameters for `usgs_gage_crosswalk.py`
+- `config/`:
+    - `deny_gms_branches_default.lst` & `config/deny_gms_branches_min.lst`: Add `usgs_elev_table.csv` to the lists as a comment so it doesn't get deleted during cleanup.
+    - `deny_gms_unit_default.lst`: Add `usgs_subset_gages.gpkg` to the lists as a comment so it doesn't get deleted during cleanup.
+
+<br/><br/>
 
 ## v4.0.2.0 - 2022-03-02 - [PR #548](https://github.com/NOAA-OWP/inundation-mapping/pull/548)
 
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 31ffeef4e..825e994f4 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -131,6 +131,8 @@ else
     parallel $retry --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
 fi
 
+## RUN AGGREGATE BRANCH ELEV TABLES ##
+python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
 
 # -------------------
 ## GET NON ZERO EXIT CODES ##
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index cb6a599a2..af2b5f344 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -282,6 +282,15 @@ Tstart
 python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/src_base_$current_branch_id.csv -l $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $outputCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $outputCurrentBranchDataDir/src_$current_branch_id.json -x $outputCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $outputCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg -y $outputCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg -p $extent -k $outputCurrentBranchDataDir/small_segments_$current_branch_id.csv
 Tcount
 
+## USGS CROSSWALK ##
+if [ -f $outputHucDataDir/usgs_subset_gages.gpkg ]; then
+    echo -e $startDiv"USGS Crosswalk $hucNumber $current_branch_id"$stopDiv
+    date -u
+    Tstart
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $current_branch_id
+    Tcount
+fi
+
 ## REMOVE FILES FROM DENY LIST ##
 if [ -f $deny_gms_branches_list ]; then
     echo -e $startDiv"Remove files $hucNumber $current_branch_id"$stopDiv
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 40a9b04cd..53f8b7daa 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -84,6 +84,13 @@ Tstart
 $srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute
 Tcount
 
+## CREATE USGS GAGES FILE
+echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg
+Tcount
+
 ## REMOVE FILES FROM DENY LIST ##
 if [ -f $deny_gms_unit_list ]; then
     echo -e $startDiv"Remove files $hucNumber"$stopDiv
diff --git a/src/usgs_gage_aggregate.py b/src/usgs_gage_aggregate.py
new file mode 100644
index 000000000..f6967c407
--- /dev/null
+++ b/src/usgs_gage_aggregate.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+import os
+from os.path import join
+import pandas as pd
+import re
+import argparse
+
+class HucDirectory(object):
+
+    def __init__(self, path, limit_branches=[]):
+
+        self.dir = path
+        self.name = os.path.basename(path)
+        self.limit_branches = limit_branches
+
+        self.usgs_dtypes = {'location_id':str,
+                            'nws_lid':str,
+                            'feature_id':int,
+                            'HydroID':int,
+                            'levpa_id':str,
+                            'dem_elevation':float,
+                            'dem_adj_elevation':float,
+                            'order_':int,
+                            'LakeID':object,
+                            'HUC8':str,
+                            'snap_distance':float}
+        self.agg_usgs_elev_table = pd.DataFrame(columns=list(self.usgs_dtypes.keys()))
+
+    def iter_branches(self):
+
+        if self.limit_branches:
+            for branch in self.limit_branches:
+                yield (branch, join(self.dir, 'branches', branch))
+
+        else:
+            for branch in os.listdir(join(self.dir, 'branches')):
+                yield (branch, join(self.dir, 'branches', branch))
+
+    def usgs_elev_table(self, branch_path):
+
+        usgs_elev_filename = join(branch_path, 'usgs_elev_table.csv')
+        if not os.path.isfile(usgs_elev_filename):
+            return
+
+        usgs_elev_table = pd.read_csv(usgs_elev_filename, dtype=self.usgs_dtypes)
+        self.agg_usgs_elev_table = self.agg_usgs_elev_table.append(usgs_elev_table)
+
+
+    def agg_function(self):
+
+        for branch_id, branch_path in self.iter_branches():
+
+            self.usgs_elev_table(branch_path)
+
+            ## Other aggregate funtions can go here
+        
+        ## After all of the branches are visited, the code below will write the aggregates
+        if os.path.isfile(join(self.dir, 'usgs_elev_table.csv')):
+            os.remove(join(self.dir, 'usgs_elev_table.csv'))
+
+        if not self.agg_usgs_elev_table.empty:
+            self.agg_usgs_elev_table.to_csv(join(self.dir, 'usgs_elev_table.csv'), index=False)
+        
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Aggregates usgs_elev_table.csv at the HUC level')
+    parser.add_argument('-fim','--fim_directory', help='Input FIM Directory', required=True)
+    parser.add_argument('-gms','--gms_inputs', help='Input gms_inputs CSV file', required=False)
+
+    args = vars(parser.parse_args())
+
+    fim_directory = args['fim_directory']
+    gms_inputs = args['gms_inputs']
+    assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
+
+    if gms_inputs:
+        gms_inputs = pd.read_csv(gms_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
+
+        for huc in gms_inputs.huc.unique():
+
+            branches = gms_inputs.loc[gms_inputs.huc == huc, 'levpa_id'].tolist()
+            huc = HucDirectory(join(fim_directory, huc), limit_branches=branches)
+            huc.agg_function()
+
+    else:
+        for huc_dir in [d for d in os.listdir(fim_directory) if re.match('\d{8}', d)]:
+
+            huc = HucDirectory(join(fim_directory, huc_dir))
+            huc.agg_function()
+
+
+        
+
+
+
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 3b7b54256..57b7c1cb8 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
 
+import os
 import geopandas as gpd
-import pandas as pd
 import rasterio
 import argparse
-import pygeos
-from shapely.wkb import dumps, loads
 import warnings
 from utils.shared_functions import mem_profile
 warnings.simplefilter("ignore")
@@ -16,118 +14,135 @@
     Parameters
     ----------
     usgs_gages_filename : str
-        File name of USGS stations layer.
-    dem_filename : str
-        File name of original DEM.
+        File path of USGS stations subset layer. i.e. '/data/path/usgs_subset_gages.gpkg'
     input_flows_filename : str
-        File name of FIM streams layer.
+        File path of FIM streams layer. i.e. '/data/path/demDerived_reaches_split_filtered_3246000257.gpkg'
     input_catchment_filename : str
-        File name of FIM catchment layer.
-    wbd_buffer_filename : str
-        File name of buffered wbd.
+        File path of FIM catchment layer. i.e. '/data/path/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg'
+    dem_filename : str
+        File path of original DEM. i.e. '/data/path/dem_meters_3246000257.tif'
     dem_adj_filename : str
-        File name of thalweg adjusted DEM.
+        File path of thalweg adjusted DEM. i.e. '/data/path/dem_thalwegCond_3246000257.tif'
     output_table_filename : str
-        File name of output table.
+        File path of output table. i.e. '/data/path/usgs_elev_table.csv'
+    branch_id: str
+        ID of the current branch i.e. '3246000257'
 '''
 
-
-@mem_profile
-def crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename,dem_adj_filename,output_table_filename,extent):
-
-    wbd_buffer = gpd.read_file(wbd_buffer_filename)
-    usgs_gages = gpd.read_file(usgs_gages_filename, mask=wbd_buffer)
-    dem_m = rasterio.open(dem_filename,'r')
-    input_flows = gpd.read_file(input_flows_filename)
-    input_catchment = gpd.read_file(input_catchment_filename)
-    dem_adj = rasterio.open(dem_adj_filename,'r')
-
-    #MS extent use gages that are mainstem
-    if extent == "MS":
-        usgs_gages = usgs_gages.query('curve == "yes" & mainstem == "yes"')
-    #FR extent use gages that are not mainstem
-    if extent == "FR":
-        usgs_gages = usgs_gages.query('curve == "yes" & mainstem == "no"')
-
-    if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
-
-    # Identify closest HydroID
-    closest_catchment = gpd.sjoin(usgs_gages, input_catchment, how='left', op='within').reset_index(drop=True)
-    closest_hydro_id = closest_catchment.filter(items=['location_id','HydroID','min_thal_elev','med_thal_elev','max_thal_elev', 'order_'])
-    closest_hydro_id = closest_hydro_id.dropna()
-
-    # Get USGS gages that are within catchment boundaries
-    usgs_gages = usgs_gages.loc[usgs_gages.location_id.isin(list(closest_hydro_id.location_id))]
-
-    columns = ['location_id','HydroID','dem_elevation','dem_adj_elevation','min_thal_elev', 'med_thal_elev','max_thal_elev','str_order']
-    gage_data = []
-
-    # Move USGS gage to stream
-    for index, gage in usgs_gages.iterrows():
-
-        # Get stream attributes
-        hydro_id = closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].HydroID.item()
-        str_order = str(int(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].order_.item()))
-        min_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].min_thal_elev.item(),2)
-        med_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].med_thal_elev.item(),2)
-        max_thal_elev = round(closest_hydro_id.loc[closest_hydro_id.location_id==gage.location_id].max_thal_elev.item(),2)
-
-        # Convert headwater point geometries to WKB representation
-        wkb_gages = dumps(gage.geometry)
-
-        # Create pygeos headwater point geometries from WKB representation
-        gage_bin_geom = pygeos.io.from_wkb(wkb_gages)
-
-        # Closest segment to headwater
-        closest_stream = input_flows.loc[input_flows.HydroID==hydro_id]
-        wkb_closest_stream = dumps(closest_stream.geometry.item())
-        stream_bin_geom = pygeos.io.from_wkb(wkb_closest_stream)
-
-        # Linear reference headwater to closest stream segment
-        gage_distance_to_line = pygeos.linear.line_locate_point(stream_bin_geom, gage_bin_geom)
-        referenced_gage = pygeos.linear.line_interpolate_point(stream_bin_geom, gage_distance_to_line)
-
-        # Convert geometries to wkb representation
-        bin_referenced_gage = pygeos.io.to_wkb(referenced_gage)
-
-        # Convert to shapely geometries
-        shply_referenced_gage = loads(bin_referenced_gage)
-
-        # Sample rasters at adjusted gage
-        dem_m_elev = round(list(rasterio.sample.sample_gen(dem_m,shply_referenced_gage.coords))[0].item(),2)
-        dem_adj_elev = round(list(rasterio.sample.sample_gen(dem_adj,shply_referenced_gage.coords))[0].item(),2)
-
-        # Append dem_m_elev, dem_adj_elev, hydro_id, and gage number to table
-        site_elevations = [str(gage.location_id), str(hydro_id), dem_m_elev, dem_adj_elev, min_thal_elev, med_thal_elev, max_thal_elev,str(str_order)]
-        gage_data.append(site_elevations)
-
-    elev_table = pd.DataFrame(gage_data, columns=columns)
-
-    if not elev_table.empty:
-        elev_table.to_csv(output_table_filename,index=False)
+class GageCrosswalk(object):
+
+    def __init__(self, usgs_subset_gages_filename, branch_id):
+        
+        self.branch_id = branch_id
+        self.gages = self._load_gages(usgs_subset_gages_filename)
+
+    def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_table_filename):
+        '''Run the gage crosswalk steps: 1) spatial join to branch catchments layer 2) snap sites to 
+        the dem-derived flows 3) sample both dems at the snapped points 4) write the crosswalked points
+        to usgs_elev_table.csv
+        '''
+
+        if self.gages.empty:
+            print(f'There are no gages for branch {branch_id}')
+            os._exit(0)
+        # Spatial join to fim catchments
+        self.catchment_sjoin(input_catchment_filename)
+        if self.gages.empty:
+            print(f'There are no gages for branch {branch_id}')
+            os._exit(0)
+        # Snap to dem derived flow lines
+        self.snap_to_dem_derived_flows(input_flows_filename)
+        # Sample DEM and thalweg adjusted DEM
+        self.sample_dem(dem_filename, 'dem_elevation')
+        self.sample_dem(dem_adj_filename, 'dem_adj_elevation')
+        # Write to csv
+        num_gages = len(self.gages)
+        print(f"{num_gages} gage{'' if num_gages == 1 else 's'} in branch {self.branch_id}")
+        self.write(output_table_filename)
+
+
+    def _load_gages(self, gages_filename):
+        '''Reads gage geopackage from huc level and filters based on current branch id'''
+
+        usgs_gages = gpd.read_file(gages_filename)
+        return  usgs_gages[usgs_gages.levpa_id == self.branch_id]
+
+    def catchment_sjoin(self, input_catchment_filename):
+        '''Spatial joins gages to FIM catchments'''
+
+        input_catchments = gpd.read_file(input_catchment_filename, dtype={'HydroID':int})
+        self.gages = gpd.sjoin(self.gages, input_catchments[['HydroID', 'LakeID', 'geometry']], how='inner')
+
+    def snap_to_dem_derived_flows(self, input_flows_filename):
+        '''Joins to dem derived flow line and produces snap_distance and geometry_snapped for sampling DEMs on the thalweg'''
+
+        input_flows = gpd.read_file(input_flows_filename)
+        input_flows['geometry_ln'] = input_flows.geometry
+        self.gages = self.gages.merge(input_flows[['HydroID', 'geometry_ln']], on='HydroID')
+        
+        # Snap each point to its feature_id line
+        self.gages['geometry_snapped'], self.gages['snap_distance'] = self.gages.apply(self.snap_to_line, axis=1,result_type='expand').T.values
+        self.gages.geometry_snapped = self.gages.geometry_snapped.astype('geometry')
+
+    def sample_dem(self, dem_filename, column_name):
+        '''Sample an input DEM at snapped points. Make sure to run self.gages.set_geometry("geometry_snapped") before runnig
+        this method, otherwise the DEM will be sampled at the actual gage locations.'''
+
+        coord_list = [(x,y) for x,y in zip(self.gages['geometry_snapped'].x , self.gages['geometry_snapped'].y)]
+        
+        with rasterio.open(dem_filename) as dem:
+            self.gages[column_name] = [x[0] for x in dem.sample(coord_list)]
+
+    def write(self, output_table_filename):
+        '''Write to csv file'''
+
+        elev_table = self.gages.copy()
+        # Elev table cleanup
+        elev_table.loc[elev_table['location_id'] == elev_table['nws_lid'], 'location_id'] = None # set location_id to None where there isn't a gage
+        elev_table = elev_table[['location_id', 'nws_lid', 'feature_id', 'HydroID', 'levpa_id', 'dem_elevation', 'dem_adj_elevation', 'order_', 'LakeID', 'HUC8', 'snap_distance']]
+
+        if not elev_table.empty:
+            elev_table.to_csv(output_table_filename, index=False)
+
+    @staticmethod
+    def snap_to_line(row):
+        if not row.geometry_ln:
+            return (None, None)
+        snap_geom = row.geometry_ln.interpolate(row.geometry_ln.project(row.geometry))
+        return (snap_geom, snap_geom.distance(row.geometry))
 
 
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Crosswalk USGS sites to HydroID and get elevations')
-    parser.add_argument('-gages','--usgs-gages-filename', help='USGS gages', required=True)
-    parser.add_argument('-dem','--dem-filename',help='DEM',required=True)
+    parser.add_argument('-gages','--usgs-gages-filename', help='USGS gage subset at the huc level', required=True)
     parser.add_argument('-flows','--input-flows-filename', help='DEM derived streams', required=True)
     parser.add_argument('-cat','--input-catchment-filename', help='DEM derived catchments', required=True)
-    parser.add_argument('-wbd','--wbd-buffer-filename', help='WBD buffer', required=True)
+    parser.add_argument('-dem','--dem-filename',help='DEM',required=True)
     parser.add_argument('-dem_adj','--dem-adj-filename', help='Thalweg adjusted DEM', required=True)
     parser.add_argument('-outtable','--output-table-filename', help='Table to append data', required=True)
-    parser.add_argument('-e', '--extent', help="extent configuration entered by user when running fim_run.sh", required = True)
+    parser.add_argument('-b','--branch-id', help='Branch ID used to filter the gages', type=str, required=True)
 
     args = vars(parser.parse_args())
 
     usgs_gages_filename = args['usgs_gages_filename']
-    dem_filename = args['dem_filename']
     input_flows_filename = args['input_flows_filename']
     input_catchment_filename = args['input_catchment_filename']
-    wbd_buffer_filename = args['wbd_buffer_filename']
+    dem_filename = args['dem_filename']
     dem_adj_filename = args['dem_adj_filename']
     output_table_filename = args['output_table_filename']
-    extent = args['extent']
+    branch_id = args['branch_id']
+
+    assert os.path.isfile(usgs_gages_filename), f"The input file {usgs_gages_filename} does not exist."
+
+    # Instantiate class
+    gage_crosswalk = GageCrosswalk(usgs_gages_filename, branch_id)
+    gage_crosswalk.run_crosswalk(input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_table_filename)
+
+"""
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /data/outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg -flows /data/outputs/carson_gms_bogus/02020005/branches/3246000305/demDerived_reaches_split_filtered_3246000305.gpkg -cat /data/outputs/carson_gms_bogus/02020005/branches/3246000305/gw_catchments_reaches_filtered_addedAttributes_3246000305.gpkg -dem /data/outputs/carson_gms_bogus/02020005/branches/3246000305/dem_meters_3246000305.tif -dem_adj /data/outputs/carson_gms_bogus/02020005/branches/3246000305/dem_thalwegCond_3246000305.tif -outtable /data/outputs/carson_gms_bogus/02020005/branches/3246000305/usgs_elev_table.csv -b 3246000305
+
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /data/outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg -flows /data/outputs/carson_gms_bogus/02020005/branches/3246000257/demDerived_reaches_split_filtered_3246000257.gpkg -cat /data/outputs/carson_gms_bogus/02020005/branches/3246000257/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg -dem /data/outputs/carson_gms_bogus/02020005/branches/3246000257/dem_meters_3246000257.tif -dem_adj /data/outputs/carson_gms_bogus/02020005/branches/3246000257/dem_thalwegCond_3246000257.tif -outtable /data/outputs/carson_gms_bogus/02020005/branches/3246000257/usgs_elev_table.csv -b 3246000257
 
-    crosswalk_usgs_gage(usgs_gages_filename,dem_filename,input_flows_filename,input_catchment_filename,wbd_buffer_filename, dem_adj_filename,output_table_filename, extent)
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /data/outputs/carson_gage_test/04130001/usgs_subset_gages.gpkg -flows /data/outputs/carson_gage_test/04130001/branches/9041000030/demDerived_reaches_split_filtered_9041000030.gpkg -cat /data/outputs/carson_gage_test/04130001/branches/9041000030/gw_catchments_reaches_filtered_addedAttributes_9041000030.gpkg -dem /data/outputs/carson_gage_test/04130001/branches/9041000030/dem_meters_9041000030.tif -dem_adj /data/outputs/carson_gage_test/04130001/branches/904100030/dem_thalwegCond_0941000030.tif -outtable /data/outputs/carson_gage_test/04130001/branches/9041000030/usgs_elev_table.csv -b 9041000030
+"""
\ No newline at end of file
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
new file mode 100755
index 000000000..085800e77
--- /dev/null
+++ b/src/usgs_gage_unit_setup.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+
+import os
+from posixpath import dirname
+import re
+import geopandas as gpd
+import pandas as pd
+import argparse
+import warnings
+from utils.shared_functions import mem_profile
+warnings.simplefilter("ignore")
+
+class Gage2Branch(object):
+
+    def __init__(self, usgs_gage_filename, ahps_filename, huc8):
+
+        self.usgs_gage_filename = usgs_gage_filename
+        self.ahps_filename = ahps_filename
+        self.huc8 = huc8
+        self.load_gages()
+
+    def load_gages(self):
+
+        # Filter USGS gages to huc
+        usgs_gages = gpd.read_file(self.usgs_gage_filename)
+        self.gages = usgs_gages[(usgs_gages.HUC8 == self.huc8) & (usgs_gages.curve == 'yes')]
+        
+        # Get AHPS sites within the HUC and add them to the USGS dataset
+        if self.ahps_filename:
+            ahps_sites = gpd.read_file(self.ahps_filename)
+            ahps_sites = ahps_sites[ahps_sites.HUC8 == self.huc8] # filter to HUC8
+            ahps_sites.rename(columns={'nwm_feature_id':'feature_id',
+                                'usgs_site_code':'location_id'}, inplace=True)
+            ahps_sites = ahps_sites[ahps_sites.location_id.isna()] # Filter sites that are already in the USGS dataset
+            self.gages = self.gages.append(ahps_sites[['feature_id', 'nws_lid', 'location_id', 'HUC8', 'name', 'states','geometry']])
+
+        # Create gages attribute
+        self.gages.location_id.fillna(usgs_gages.nws_lid, inplace=True)
+        self.gages.loc[self.gages['nws_lid'] == 'Bogus_ID', 'nws_lid'] = None
+
+    def sort_into_branch(self, nwm_subset_streams_levelPaths):
+
+        nwm_reaches = gpd.read_file(nwm_subset_streams_levelPaths)
+        nwm_reaches.rename(columns={'ID':'feature_id'}, inplace=True)
+
+        if not self.gages[self.gages.feature_id.isnull()].empty:
+            missing_feature_id = self.gages.loc[self.gages.feature_id.isnull()].copy()
+            nwm_reaches_union = nwm_reaches.geometry.unary_union
+            missing_feature_id['feature_id'] = missing_feature_id.apply(lambda row: self.sjoin_nearest_to_nwm(row.geometry, nwm_reaches, nwm_reaches_union), axis=1)
+
+            self.gages.update(missing_feature_id)
+
+            del nwm_reaches_union
+
+        # Left join gages with NWM streams to get the level path
+        self.gages.feature_id = self.gages.feature_id.astype(int)
+        self.gages = self.gages.merge(nwm_reaches[['feature_id','levpa_id','order_']], on='feature_id', how='left')
+        return self.gages
+    
+    def write(self, out_name):
+
+        self.gages.to_file(out_name,driver='GPKG',index=False)
+
+    @staticmethod
+    def sjoin_nearest_to_nwm(pnt, lines, union):
+
+        snap_geom = union.interpolate(union.project(pnt))
+        queried_index = lines.geometry.sindex.query(snap_geom)
+        if len(queried_index):
+            return int(lines.iloc[queried_index[0]].feature_id.item())
+
+    @staticmethod
+    def filter_gage_branches(gms_inputs_filename):
+
+        fim_dir = os.path.dirname(gms_inputs_filename)
+        gms_inputs = pd.read_csv(gms_inputs_filename, header=None, names=['huc', 'levpa_id'],
+            dtype={'huc':str, 'levpa_id':str})
+
+        for huc_dir in [d for d in os.listdir(fim_dir) if re.search('^\d{8}$', d)]:
+
+            gage_file = os.path.join(fim_dir, huc_dir, 'usgs_subset_gages.gpkg')
+            if not os.path.isfile(gage_file):
+                gms_inputs.drop(gms_inputs.loc[gms_inputs.huc == huc_dir].index, inplace=True)
+                continue
+
+            gages = gpd.read_file(gage_file)
+            level_paths = gages.levpa_id
+            gms_inputs.drop(gms_inputs.loc[(gms_inputs.huc == huc_dir) & (~gms_inputs.levpa_id.isin(level_paths))].index, inplace=True)
+        
+        gms_inputs.to_csv(gms_inputs_filename, index=False, header=False)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Assign HUC gages to branch and stage for usgs_gage_crosswalk.py')
+    parser.add_argument('-gages','--usgs-gages-filename', help='USGS gages', required=True)
+    parser.add_argument('-ahps','--nws-lid-filename', help='AHPS gages', required=False)
+    parser.add_argument('-nwm','--input-nwm-filename', help='NWM stream subset', required=True)
+    parser.add_argument('-o','--output-filename', help='Table to append data', required=True)
+    parser.add_argument('-huc','--huc8-id', help='HUC8 ID (to verify gage location huc)', type=str, required=True)
+    parser.add_argument('-ff','--filter-gms-inputs', help='WARNING: only run this parameter if you know exactly what you are doing', required=False)
+
+    args = vars(parser.parse_args())
+
+    usgs_gages_filename = args['usgs_gages_filename']
+    nws_lid_filename = args['nws_lid_filename']
+    input_nwm_filename = args['input_nwm_filename']
+    output_filename = args['output_filename']
+    huc8 = args['huc8_id']
+    filter_gms_inputs = args['filter_gms_inputs']
+
+    if not filter_gms_inputs:
+
+        usgs_gage_subset = Gage2Branch(usgs_gages_filename, nws_lid_filename, huc8)
+        if usgs_gage_subset.gages.empty:
+            print(f'There are no gages identified for {huc8}')
+            os._exit(0)
+        usgs_gage_subset.sort_into_branch(input_nwm_filename)
+        usgs_gage_subset.write(output_filename)
+
+    else:
+        ''' 
+        This is an easy way to filter gms_inputs so that only branches with gages will run during gms_run_branch.sh.
+        You can run this option after gms_run_unit.sh has completed.
+
+        example:
+        python3 src/usgs_gage_unit_setup.py -gages x -ahps x -nwm x -o x -huc x -ff /data/outputs/test_output/gms_inputs.csv
+        '''
+        assert os.path.isfile(filter_gms_inputs)
+        Gage2Branch.filter_gage_branches(filter_gms_inputs)
+
+    
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index d2adeba08..5e5985040 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -2,7 +2,8 @@
 
 import os
 import sys
-import pandas as pd
+import re
+import pandas as pd, geopandas as gpd
 import numpy as np
 import argparse
 import matplotlib.pyplot as plt
@@ -15,6 +16,10 @@
 from pathlib import Path
 import time
 warnings.simplefilter(action='ignore', category=FutureWarning)
+import rasterio
+from rasterio import features as riofeatures
+from rasterio import plot as rioplot
+from shapely.geometry import Polygon
 
 """
     Plot Rating Curves and Compare to USGS Gages
@@ -62,7 +67,7 @@ def check_file_age(file):
 def generate_rating_curve_metrics(args):
 
     elev_table_filename         = args[0]
-    hydrotable_filename         = args[1]
+    branches_folder             = args[1]
     usgs_gages_filename         = args[2]
     usgs_recurr_stats_filename  = args[3]
     nwm_recurr_data_filename    = args[4]
@@ -70,13 +75,32 @@ def generate_rating_curve_metrics(args):
     nwm_flow_dir                = args[6]
     catfim_flows_filename       = args[7]
     huc                         = args[8]
-
-    elev_table = pd.read_csv(elev_table_filename,dtype={'location_id': str})
-    hydrotable = pd.read_csv(hydrotable_filename,dtype={'HUC': str,'feature_id': str})
-    usgs_gages = pd.read_csv(usgs_gages_filename,dtype={'location_id': str})
+    alt_plot                    = args[9]
+
+    elev_table = pd.read_csv(elev_table_filename,dtype={'location_id': object, 'feature_id':object,'HydroID':object, 'levpa_id':object})
+    elev_table.dropna(subset=['location_id'], inplace=True)
+    usgs_gages = pd.read_csv(usgs_gages_filename,dtype={'location_id': object, 'feature_id':object})
+
+    # Aggregate FIM4 hydroTables
+    hydrotable = pd.DataFrame()
+    for branch in elev_table.levpa_id.unique():
+        branch_elev_table = elev_table.loc[elev_table.levpa_id == branch].copy()
+        branch_hydrotable = pd.read_csv(join(branches_folder, str(branch), f'hydroTable_{branch}.csv'),dtype={'HydroID':object,'feature_id':object})
+        # Only pull SRC for hydroids that are in this branch
+        branch_hydrotable = branch_hydrotable.loc[branch_hydrotable.HydroID.isin(branch_elev_table.HydroID)]
+        branch_hydrotable.drop(columns=['order_'], inplace=True)
+        # Join SRC with elevation data
+        branch_elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
+        branch_hydrotable = branch_hydrotable.merge(branch_elev_table, on="HydroID")
+        # Append to full rating curve dataframe
+        if hydrotable.empty:
+            hydrotable = branch_hydrotable
+        else:
+            hydrotable = hydrotable.append(branch_hydrotable)
 
     # Join rating curves with elevation data
-    hydrotable = hydrotable.merge(elev_table, on="HydroID")
+    #elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
+    #hydrotable = hydrotable.merge(elev_table, on="HydroID")
     relevant_gages = list(hydrotable.location_id.unique())
     usgs_gages = usgs_gages[usgs_gages['location_id'].isin(relevant_gages)]
     usgs_gages = usgs_gages.reset_index(drop=True)
@@ -91,35 +115,37 @@ def generate_rating_curve_metrics(args):
 
         hydrotable['source'] = "FIM"
         usgs_gages['source'] = "USGS"
-        limited_hydrotable = hydrotable.filter(items=['location_id','elevation_ft','discharge_cfs','source'])
+        limited_hydrotable = hydrotable.filter(items=['location_id','elevation_ft','discharge_cfs','source', 'HydroID', 'levpa_id', 'dem_adj_elevation'])
         select_usgs_gages = usgs_gages.filter(items=['location_id', 'elevation_ft', 'discharge_cfs','source'])
-
-        rating_curves = limited_hydrotable.append(select_usgs_gages)
+        if 'default_discharge_cms' in hydrotable.columns: # check if both "FIM" and "FIM_default" SRCs are available
+            hydrotable['default_discharge_cfs'] = hydrotable.default_discharge_cms * 35.3147
+            limited_hydrotable_default = hydrotable.filter(items=['location_id','elevation_ft', 'default_discharge_cfs'])
+            limited_hydrotable_default['discharge_cfs'] = limited_hydrotable_default.default_discharge_cfs
+            limited_hydrotable_default['source'] = "FIM_default"
+            rating_curves = limited_hydrotable.append(select_usgs_gages)
+            rating_curves = rating_curves.append(limited_hydrotable_default)
+        else:
+            rating_curves = limited_hydrotable.append(select_usgs_gages)
 
         # Add stream order
-        stream_orders = hydrotable.filter(items=['location_id','str_order']).drop_duplicates()
+        stream_orders = hydrotable.filter(items=['location_id','order_']).drop_duplicates()
         rating_curves = rating_curves.merge(stream_orders, on='location_id')
-        rating_curves['str_order'] = rating_curves['str_order'].astype('int')
+        rating_curves['order_'] = rating_curves['order_'].astype('int')
 
-        # plot rating curves
-        generate_facet_plot(rating_curves, rc_comparison_plot_filename)
 
         # NWM recurr intervals
-        recurr_1_5_yr_filename = join(nwm_flow_dir,'recurr_1_5_cms.csv')
-        recurr_5_yr_filename = join(nwm_flow_dir,'recurr_5_0_cms.csv')
-        recurr_10_yr_filename = join(nwm_flow_dir,'recurr_10_0_cms.csv')
-
-        # Update column names
-        recurr_1_5_yr = pd.read_csv(recurr_1_5_yr_filename,dtype={'feature_id': str})
-        recurr_1_5_yr = recurr_1_5_yr.rename(columns={"discharge": "1.5"})
-        recurr_5_yr = pd.read_csv(recurr_5_yr_filename,dtype={'feature_id': str})
-        recurr_5_yr = recurr_5_yr.rename(columns={"discharge": "5.0"})
-        recurr_10_yr = pd.read_csv(recurr_10_yr_filename,dtype={'feature_id': str})
-        recurr_10_yr = recurr_10_yr.rename(columns={"discharge": "10.0"})
+        recurr_intervals = ("2","5","10","25","50","100")
+        recurr_dfs = []
+        for interval in recurr_intervals:
+            recurr_file = join(nwm_flow_dir, 'nwm21_17C_recurr_{}_0_cms.csv'.format(interval))
+            df = pd.read_csv(recurr_file, dtype={'feature_id': str})
+            # Update column names
+            df = df.rename(columns={"discharge": interval})
+            recurr_dfs.append(df)
 
         # Merge NWM recurr intervals into a single layer
-        nwm_recurr_intervals_all = reduce(lambda x,y: pd.merge(x,y, on='feature_id', how='outer'), [recurr_1_5_yr, recurr_5_yr, recurr_10_yr])
-        nwm_recurr_intervals_all = pd.melt(nwm_recurr_intervals_all, id_vars=['feature_id'], value_vars=['1.5','5.0','10.0'], var_name='recurr_interval', value_name='discharge_cms')
+        nwm_recurr_intervals_all = reduce(lambda x,y: pd.merge(x,y, on='feature_id', how='outer'), recurr_dfs)
+        nwm_recurr_intervals_all = pd.melt(nwm_recurr_intervals_all, id_vars=['feature_id'], value_vars=recurr_intervals, var_name='recurr_interval', value_name='discharge_cms')
 
         # Append catfim data (already set up in format similar to nwm_recurr_intervals_all)
         cat_fim = pd.read_csv(catfim_flows_filename, dtype={'feature_id':str})
@@ -131,6 +157,7 @@ def generate_rating_curve_metrics(args):
 
         # Identify unique gages
         usgs_crosswalk = hydrotable.filter(items=['location_id', 'feature_id']).drop_duplicates()
+        usgs_crosswalk.dropna(subset=['location_id'], inplace=True)
 
         nwm_recurr_data_table = pd.DataFrame()
         usgs_recurr_data = pd.DataFrame()
@@ -145,7 +172,7 @@ def generate_rating_curve_metrics(args):
                 print(f"missing USGS rating curve data for usgs station {gage.location_id} in huc {huc}")
                 continue
 
-            str_order = np.unique(usgs_rc.str_order).item()
+            str_order = np.unique(usgs_rc.order_).item()
             feature_id = str(gage.feature_id)
 
             usgs_pred_elev = get_reccur_intervals(usgs_rc, usgs_crosswalk,nwm_recurr_intervals_all)
@@ -221,11 +248,21 @@ def generate_rating_curve_metrics(args):
             nwm_recurr_data_table.elevation_ft = np.round(nwm_recurr_data_table.elevation_ft,2)
             nwm_recurr_data_table.to_csv(nwm_recurr_data_filename,index=False)
 
+        # plot rating curves
+        if alt_plot:
+            generate_rc_and_rem_plots(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table, branches_folder)
+        else:
+            generate_facet_plot(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table)
+
     else:
         print(f"no USGS data for gage(s): {relevant_gages} in huc {huc}")
 
 def aggregate_metrics(output_dir,procs_list,stat_groups):
 
+    # Default stat group to location_id
+    if stat_groups is None:
+        stat_groups = ['location_id']
+
     # agg_usgs_interp_elev_stats = join(output_dir,'agg_usgs_interp_elev_stats.csv')
     agg_nwm_recurr_flow_elev = join(output_dir,'agg_nwm_recurr_flow_elevations.csv')
     agg_nwm_recurr_flow_elev_stats = join(output_dir,f"agg_nwm_recurr_flow_elev_stats_{'_'.join(stat_groups)}.csv")
@@ -264,17 +301,26 @@ def aggregate_metrics(output_dir,procs_list,stat_groups):
 
     agg_recurr_stats_table.to_csv(agg_nwm_recurr_flow_elev_stats,index=False)
 
+    return agg_recurr_stats_table
 
-def generate_facet_plot(rc, plot_filename):
+
+def generate_facet_plot(rc, plot_filename, recurr_data_table):
 
     # Filter FIM elevation based on USGS data
     for gage in rc.location_id.unique():
 
         min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
         max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
+        min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
+        max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
+        ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
+
+        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
 
-        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft > (max_elev + 2))].index)
-        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2)].index)
+        if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
 
     rc = rc.rename(columns={"location_id": "USGS Gage"})
 
@@ -286,10 +332,29 @@ def generate_facet_plot(rc, plot_filename):
         columns = 1
 
     sns.set(style="ticks")
-    g = sns.FacetGrid(rc, col="USGS Gage", hue="source", hue_order=['USGS','FIM'], sharex=False, sharey=False,col_wrap=columns)
-    g.map(sns.scatterplot, "discharge_cfs", "elevation_ft", palette="tab20c", marker="o")
+
+    # Plot both "FIM" and "FIM_default" rating curves
+    hue_order = ['USGS','FIM','FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM']
+    # Facet Grid
+    g = sns.FacetGrid(rc, col="USGS Gage", hue="source", hue_order=hue_order,
+                    sharex=False, sharey=False,col_wrap=columns,
+                    height=3.5, aspect=1.65)
+    g.map(plt.plot, "discharge_cfs", "elevation_ft", linewidth=2, alpha=0.8)
     g.set_axis_labels(x_var="Discharge (cfs)", y_var="Elevation (ft)")
 
+    ## Plot recurrence intervals
+    axes = g.axes_dict
+    for gage in axes:
+        ax = axes[gage]
+        plt.sca(ax)
+        recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
+            .filter(items=['recurr_interval', 'discharge_cfs'])
+        for i, r in recurr_data.iterrows():
+            if not r.recurr_interval.isnumeric(): continue # skip catfim flows
+            l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
+            plt.axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
+            plt.text(r.discharge_cfs, ax.get_ylim()[1] - (ax.get_ylim()[1]-ax.get_ylim()[0])*0.03, r.recurr_interval, size='small', c='purple')
+
     # Adjust the arrangement of the plots
     g.fig.tight_layout(w_pad=1)
     g.add_legend()
@@ -297,6 +362,112 @@ def generate_facet_plot(rc, plot_filename):
     plt.savefig(plot_filename)
     plt.close()
 
+def generate_rc_and_rem_plots(rc, plot_filename, recurr_data_table, branches_folder):
+
+    ## Set up figure
+    num_plots = len(rc["location_id"].unique())
+    fig = plt.figure(figsize=(6, 2.4*num_plots))
+    gs = fig.add_gridspec(num_plots, 2, width_ratios=[2, 3])
+    ax = gs.subplots()
+    if ax.ndim == 1:                     # hucs with only one plot will only have one-dimensional axes;
+        ax = np.expand_dims(ax, axis=0)  # the axes manipulations below require 2 dimensions
+    plt.tight_layout(w_pad=1)
+
+    # Create a dictionary with location_id as keys and branch id as values
+    gage_branch_dict = rc.groupby('location_id')['levpa_id'].first().to_dict()
+
+    for i, gage in enumerate(gage_branch_dict):
+    ###################################################################################################################
+    # Filter FIM elevation based on USGS data
+
+        min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
+        max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
+        min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
+        max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
+        ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
+
+        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
+
+        if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
+
+    ###################################################################################################################
+        ## Read in reaches, catchment raster, and rem raster
+        branch = gage_branch_dict[gage]
+        if os.path.isfile(os.path.join(branches_folder, branch, f'demDerived_reaches_split_filtered_addedAttributes_crosswalked_{branch}.gpkg')):
+            reaches = gpd.read_file(os.path.join(branches_folder, branch, f'demDerived_reaches_split_filtered_addedAttributes_crosswalked_{branch}.gpkg'))
+            reach = reaches[reaches.HydroID == hydroid]
+        with rasterio.open(os.path.join(branches_folder, branch, f'gw_catchments_reaches_filtered_addedAttributes_{branch}.tif')) as catch_rast:
+            catchments = catch_rast.read()
+        with rasterio.open(os.path.join(branches_folder, branch, f'rem_zeroed_masked_{branch}.tif')) as rem:
+            rem_transform = rem.transform
+            rem_extent = rioplot.plotting_extent(rem)
+            rem_sub25 = rem.read()
+            # Set all pixels above the SRC calculation height to nan
+            rem_sub25[np.where(rem_sub25 > 25.3)] = -9999.
+            rem_sub25[np.where(rem_sub25 == -9999.)] = np.nan
+        
+        # Plot the rating curve
+        ax[i,1].plot("discharge_cfs", "elevation_ft", data=rc[(rc.source == 'USGS') & (rc.location_id == gage)], linewidth=2, alpha=0.8, label='USGS')
+        ax[i,1].plot("discharge_cfs", "elevation_ft", data=rc[(rc.source == 'FIM') & (rc.location_id == gage)], linewidth=2, alpha=0.8, label='FIM')
+        if 'default_discharge_cfs' in rc.columns:
+            ax[i,1].plot("default_discharge_cfs", "elevation_ft", data=rc[(rc.source == 'FIM_default') & (rc.location_id == gage)], linewidth=2, alpha=0.8, label='FIM_default')
+        
+        # Plot the recurrence intervals
+        recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
+            .filter(items=['recurr_interval', 'discharge_cfs'])
+        for _, r in recurr_data.iterrows():
+            if not r.recurr_interval.isnumeric(): continue # skip catfim flows
+            l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
+            ax[i,1].axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
+            ax[i,1].text(r.discharge_cfs, ax[i,1].get_ylim()[1] - (ax[i,1].get_ylim()[1]-ax[i,1].get_ylim()[0])*0.06, r.recurr_interval, size='small', c='purple')
+
+        # Get the hydroid    
+        hydroid = rc[rc.location_id == gage].HydroID.unique()[0]
+        if not hydroid:
+            print(f'Gage {gage} in HUC {branch} has no HydroID')
+            continue
+
+        # Filter the reaches and REM by the hydroid
+        catchment_rem = rem_sub25.copy()
+        catchment_rem[np.where(catchments != int(hydroid))] = np.nan
+
+        # Convert raster to WSE feet and limit to upper bound of rating curve
+        dem_adj_elevation = rc[rc.location_id == gage].dem_adj_elevation.unique()[0]
+        catchment_rem = (catchment_rem + dem_adj_elevation) * 3.28084
+        max_elev = rc[(rc.source == 'FIM') & (rc.location_id == gage)].elevation_ft.max()
+        catchment_rem[np.where(catchment_rem > max_elev)] = np.nan         # <-- Comment out this line to get the full raster that is
+                                                                           # used in rating curve creation
+        # Create polygon for perimeter/area stats                          
+        catchment_rem_1s = catchment_rem.copy()
+        catchment_rem_1s[np.where(~np.isnan(catchment_rem_1s))] = 1
+        features = riofeatures.shapes(catchment_rem_1s, mask=~np.isnan(catchment_rem), transform=rem_transform, connectivity=8)
+        del catchment_rem_1s
+        features = [f for f in features]
+        geom = [Polygon(f[0]['coordinates'][0]) for f in features]
+        poly = gpd.GeoDataFrame({'geometry':geom})
+        #poly['perimeter'] = poly.length                                        # These lines are calculating perimeter/area stats
+        #poly['area'] = poly.area                                               # and can be removed if there is a separate process
+        #poly['perimeter_area_ratio'] = poly.length/poly.area                   # set up later that calculates these for all hydroids
+        #poly['perimeter_area_ratio_sqrt'] = poly.length/(poly.area**.5)        # within a catchment.
+        bounds = poly.total_bounds
+        bounds = ((bounds[0]-20, bounds[2]+20), (bounds[1]-20, bounds[3]+20))
+        
+        # REM plot
+        if 'reach' in locals():
+            reach.plot(ax=ax[i,0], color='#999999', linewidth=0.9)
+        im = ax[i,0].imshow(rasterio.plot.reshape_as_image(catchment_rem), cmap='gnuplot', extent=rem_extent, interpolation='none')
+        plt.colorbar(im, ax=ax[i,0], location='left')
+        ax[i,0].set_xbound(bounds[0]); ax[i,0].set_ybound(bounds[1])
+        ax[i,0].set_xticks([]); ax[i,0].set_yticks([])
+        ax[i,0].set_title(gage)
+
+    del catchments, rem_sub25, catchment_rem
+    ax[0,1].legend()
+    plt.savefig(plot_filename, dpi=200)
+    plt.close()
 
 def get_reccur_intervals(site_rc, usgs_crosswalk,nwm_recurr_intervals):
 
@@ -331,9 +502,6 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
         .rename_axis(None, axis=1)
      )
 
-    if stat_groups is None:
-        stat_groups = ['location_id']
-
     # Calculate variables for NRMSE
     rc_unmelt["yhat_minus_y"] = rc_unmelt[src_elev] - rc_unmelt[usgs_elev]
     rc_unmelt["yhat_minus_y_squared"] = rc_unmelt["yhat_minus_y"] ** 2
@@ -360,21 +528,157 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     nrmse_table_group = nrmse_table.groupby(stat_groups)
 
     # Calculate nrmse
-    nrmse = nrmse_table_group.apply(lambda x: ((x['sum_y_diff'] / x['n']) ** 0.5) / (x['y_max'] - x['y_min']))\
+    def NRMSE(x):
+        if x['n'][0] == 1:      # when n==1, NRME equation will return an `inf` 
+            return x['sum_y_diff'] ** 0.5
+        else:
+            return ((x['sum_y_diff'] / x['n']) ** 0.5) / (x['y_max'] - x['y_min'])
+
+    nrmse = nrmse_table_group.apply(NRMSE)\
         .reset_index(stat_groups, drop = False).rename({0: "nrmse"}, axis=1)
 
     # Calculate Mean Absolute Depth Difference
     mean_abs_y_diff = station_rc.apply(lambda x: (abs(x["yhat_minus_y"]).mean()))\
         .reset_index(stat_groups, drop = False).rename({0: "mean_abs_y_diff_ft"}, axis=1)
 
+    # Calculate Mean Depth Difference (non-absolute value)
+    mean_y_diff     = station_rc.apply(lambda x: (x["yhat_minus_y"].mean()))\
+        .reset_index(stat_groups, drop = False).rename({0: "mean_y_diff_ft"}, axis=1)
+
     # Calculate Percent Bias
     percent_bias = station_rc.apply(lambda x: 100 * (x["yhat_minus_y"].sum() / x[usgs_elev].sum()))\
         .reset_index(stat_groups, drop = False).rename({0: "percent_bias"}, axis=1)
 
-    rc_stat_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [nrmse, mean_abs_y_diff, percent_bias])
+    rc_stat_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [nrmse, mean_abs_y_diff, mean_y_diff, percent_bias])
 
     return rc_stat_table
 
+def create_static_gpkg(output_dir, output_gpkg, agg_recurr_stats_table, gages_gpkg_filepath):
+    '''
+    Merges the output dataframe from aggregate_metrics() with the usgs gages GIS data
+    '''
+    # Load in the usgs_gages geopackage
+    usgs_gages = gpd.read_file(gages_gpkg_filepath)
+    # Merge the stats for all of the recurrance intervals/thresholds
+    usgs_gages = usgs_gages.merge(agg_recurr_stats_table, on='location_id')
+    # Load in the rating curves file
+    agg_nwm_recurr_flow_elev = join(output_dir,'agg_nwm_recurr_flow_elevations.csv')
+    agg_stats = pd.read_csv(agg_nwm_recurr_flow_elev,dtype={'location_id': str,
+                                                        'feature_id': str})
+    diff_table = calculate_rc_diff(agg_stats)
+    # Merge recurrence interval difference table with points layer
+    usgs_gages = usgs_gages.merge(diff_table, on='location_id')
+    usgs_gages = usgs_gages.round(decimals=2)
+
+    # Write to file
+    usgs_gages.to_file(join(output_dir, output_gpkg), driver='GPKG', index=False)
+
+    # Create figure
+    usgs_gages.replace(np.inf, np.nan, inplace=True) # replace inf with nan for plotting
+    fig, ax = plt.subplots(2, 2, figsize=(18, 10))
+
+    # Bin data
+    max_bin = usgs_gages['mean_abs_y_diff_ft'].max()
+    bins = (0, 1, 3, 6, 9, max_bin if max_bin > 12 else 12)
+    usgs_gages['mean_abs_y_diff_ft'] = pd.cut(usgs_gages['mean_abs_y_diff_ft'], bins=bins)
+    
+    max_bin = usgs_gages['mean_y_diff_ft'].max()
+    min_bin = usgs_gages['mean_y_diff_ft'].min()
+    bins = (min_bin if min_bin < -12 else -12, -9, -6, -3, -1, 0, 1, 3, 6, 9, max_bin if max_bin > 12 else 12)
+    usgs_gages['mean_y_diff_ft'] = pd.cut(usgs_gages['mean_y_diff_ft'], bins=bins)
+
+    # Create subplots
+    sns.histplot(ax=ax[0,0], y='nrmse', data=usgs_gages, binwidth=0.2, binrange=(0, 10))
+    sns.countplot(ax=ax[1,0], y='mean_abs_y_diff_ft', data=usgs_gages)
+    sns.countplot(ax=ax[1,1], y='mean_y_diff_ft', data=usgs_gages)
+    sns.boxplot(ax=ax[0,1], data=usgs_gages[['2', '5', '10', '25', '50', '100', 'action', 'minor', 'moderate','major']])
+    ax[0,1].set(ylim=(-12, 12))
+
+    fig.tight_layout()
+    fig.savefig(join(output_dir, f'{output_gpkg}_summary_plots.png'.replace('.gpkg', '')))
+
+    return
+
+def calculate_rc_diff(rc):
+
+    usgs_elev = "USGS"
+    src_elev = "FIM"
+
+    # Collect any extra columns not associated with melt
+    col_index = list(rc.columns)
+    pivot_vars = ['source','elevation_ft']
+    col_index = [col for col in col_index if col not in pivot_vars]
+
+    # Unmelt elevation/source
+    rc_unmelt = (rc.set_index(col_index)
+        .pivot(columns="source")['elevation_ft']
+        .reset_index()
+        .rename_axis(None, axis=1)
+     )
+
+    # Calculate water surface elevation difference at recurrence intervals
+    rc_unmelt["yhat_minus_y"] = rc_unmelt[src_elev] - rc_unmelt[usgs_elev]
+    # Remove duplicate location_id-recurr_interval pairs and pivot 
+    rc_unmelt.set_index(['location_id', 'recurr_interval'], inplace=True, verify_integrity=False)
+    rc_unmelt = (rc_unmelt[~rc_unmelt.index.duplicated(keep='first')]
+        .reset_index()
+        .pivot(index='location_id', columns='recurr_interval', values='yhat_minus_y'))
+    # Reorder columns
+    rc_unmelt = rc_unmelt[['2', '5', '10', '25', '50', '100', 'action', 'minor', 'moderate','major']]
+
+    return rc_unmelt
+    
+def evaluate_results(sierra_results=[], labels=[], save_location=''):
+    '''
+    Compares multiple Sierra Test results using a boxplot.
+
+    Parameters
+    ------------
+    sierra_results : list
+        List of GeoDataFrames with sierra test results.
+    labels : list
+        List of strings that will be used as labels for sierra_results. Length must be equal to sierra_results.
+    save_location : str
+        Path to save output boxplot figure.
+
+    Example
+    ------------
+    from rating_curve_comparison import evaluate_results
+    import geopandas as gpd
+
+    sierra_1 = gpd.read_file("/data/path/to/fim_3_X_ms.gpkg")
+    sierra_new = gpd.read_file("/data/path/to/fim_experiment.gpkg")
+
+    evaluate_results([sierra_1, sierra_new], ["fim_3_X_ms", "fim_calibrate_SRC"], "/path/to/output.png")
+    '''
+
+    assert len(sierra_results) == len(labels), "Each Sierra Test results must also have a label"
+
+    # Define recurrence intervals to plot
+    recurr_intervals = ("2","5","10","25","50","100","action","minor","moderate","major")
+
+    # Assign labels to the input sierra test result dataframes
+    for df, label in zip(sierra_results, labels):
+        df['_version'] = label
+
+    # Combine all dataframes into one
+    all_results = sierra_results[0]
+    all_results = all_results.append(sierra_results[1:])
+
+    # Melt results for boxplotting
+    all_results_melted = all_results.melt(id_vars=["location_id", '_version'], 
+                                        value_vars=recurr_intervals, 
+                                        var_name='recurr_interval', 
+                                        value_name='error_ft')
+
+    # Plot all results in a comparison boxplot
+    fig, ax = plt.subplots(figsize=(10,5))
+    sns.boxplot(x='recurr_interval', y='error_ft', hue='_version', data=all_results_melted, ax=ax, fliersize=3)
+    ax.set(ylim=(-30, 30))
+    ax.grid()
+    ax.legend(bbox_to_anchor=(1, 1), loc='upper right', title='FIM Version')
+    ax.set_title('Sierra Test Results Comparison')
+    plt.savefig(save_location)
 
 if __name__ == '__main__':
 
@@ -385,7 +689,11 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     parser.add_argument('-flows','--nwm-flow-dir',help='NWM recurrence flows dir',required=True,type=str)
     parser.add_argument('-catfim', '--catfim-flows-filename', help='Categorical FIM flows file',required = True,type=str)
     parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-    parser.add_argument('-group','--stat-groups',help='column(s) to group stats',required=False,type=str)
+    parser.add_argument('-group','--stat-groups',help='column(s) to group stats',required=False,type=str,nargs='+')
+    parser.add_argument('-pnts','--stat-gages',help='takes 2 arguments: 1) file path of input usgs_gages.gpkg and 2) output GPKG name to write USGS gages with joined stats',required=False,type=str,nargs=2)
+    parser.add_argument('-alt','--alt-plot',help='Generate rating curve plots with REM maps',required=False,default=False,action='store_true')
+    parser.add_argument('-eval','--evaluate-results',help='Create a boxplot comparison of multiple input Sierra Test results. \
+        Expects 2 arguments: 1) path to the Sierra Test results for comparison and 2) the corresponding label for the boxplot.',required=False,nargs=2,action='append')
 
     args = vars(parser.parse_args())
 
@@ -396,8 +704,20 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     catfim_flows_filename = args['catfim_flows_filename']
     number_of_jobs = args['number_of_jobs']
     stat_groups = args['stat_groups']
+    alt_plot = args['alt_plot']
+    eval = args['evaluate_results']
+    if args['stat_gages']:
+        gages_gpkg_filepath = args['stat_gages'][0]
+        stat_gages = args['stat_gages'][1]
+        assert (os.path.exists(gages_gpkg_filepath)), f"{gages_gpkg_filepath} does not exist. Please specify a full path to a USGS geopackage (.gpkg)"
+    else:
+        stat_gages = None
 
-    stat_groups = stat_groups.split()
+    # Make sure that location_id is the only -group when using -pnts
+    assert (not stat_gages or (stat_gages and (not stat_groups or stat_groups == ['location_id']))), \
+        "location_id is the only acceptable stat_groups argument when producting an output GPKG"
+    # Make sure that the -pnts flag is used with the -eval flag
+    assert not eval or (eval and stat_gages), "You must use the -pnts flag with the -eval flag"
     procs_list = []
 
     plots_dir = join(output_dir,'plots')
@@ -414,22 +734,23 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     sys.stdout = log_file
 
     merged_elev_table = []
-    huc_list  = os.listdir(fim_dir)
+    huc_list  = [huc for huc in os.listdir(fim_dir) if re.search("^\d{6,8}$", huc)]
     for huc in huc_list:
 
-        if huc != 'logs':
-            elev_table_filename = join(fim_dir,huc,'usgs_elev_table.csv')
-            hydrotable_filename = join(fim_dir,huc,'hydroTable.csv')
-            usgs_recurr_stats_filename = join(tables_dir,f"usgs_interpolated_elevation_stats_{huc}.csv")
-            nwm_recurr_data_filename = join(tables_dir,f"nwm_recurrence_flow_elevations_{huc}.csv")
-            rc_comparison_plot_filename = join(plots_dir,f"FIM-USGS_rating_curve_comparison_{huc}.png")
-
-            if isfile(elev_table_filename):
-                procs_list.append([elev_table_filename, hydrotable_filename, usgs_gages_filename, usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,nwm_flow_dir, catfim_flows_filename, huc])
-                # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
-                read_elev_table = pd.read_csv(elev_table_filename)
-                read_elev_table['huc'] = huc
-                merged_elev_table.append(read_elev_table)
+        elev_table_filename = join(fim_dir,huc,'usgs_elev_table.csv')
+        branches_folder = join(fim_dir,huc,'branches')
+        usgs_recurr_stats_filename = join(tables_dir,f"usgs_interpolated_elevation_stats_{huc}.csv")
+        nwm_recurr_data_filename = join(tables_dir,f"nwm_recurrence_flow_elevations_{huc}.csv")
+        rc_comparison_plot_filename = join(plots_dir,f"FIM-USGS_rating_curve_comparison_{huc}.png")
+
+        if isfile(elev_table_filename):
+            procs_list.append([elev_table_filename, branches_folder, usgs_gages_filename, 
+                            usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,
+                            nwm_flow_dir, catfim_flows_filename, huc, alt_plot])
+            # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
+            read_elev_table = pd.read_csv(elev_table_filename, dtype={'location_id':str, 'HydroID':str, 'huc':str, 'feature_id':int})
+            read_elev_table['huc'] = huc
+            merged_elev_table.append(read_elev_table)
 
     # Output a concatenated elev_table to_csv
     if merged_elev_table:
@@ -443,12 +764,33 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     with Pool(processes=number_of_jobs) as pool:
         pool.map(generate_rating_curve_metrics, procs_list)
 
-    print(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
-    aggregate_metrics(output_dir,procs_list,stat_groups)
+    # Create point layer of usgs gages with joined stats attributes
+    if stat_gages:
+        print("Creating usgs gages GPKG with joined rating curve summary stats")
+        agg_recurr_stats_table = aggregate_metrics(output_dir,procs_list,['location_id'])
+        create_static_gpkg(output_dir, stat_gages, agg_recurr_stats_table, gages_gpkg_filepath)
+        del agg_recurr_stats_table # memory cleanup
+    else: # if not producing GIS layer, just aggregate metrics
+        print(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
+        aggregate_metrics(output_dir,procs_list,stat_groups)
 
     print('Delete intermediate tables')
     shutil.rmtree(tables_dir, ignore_errors=True)
 
+    # Compare current sierra test results to previous tests
+    if eval:
+        # Transpose comparison sierra results
+        sierra_test_paths, sierra_test_labels = np.array(eval).T.tolist()
+        # Add current sierra test to lists
+        sierra_test_paths = sierra_test_paths + [join(output_dir, stat_gages)]
+        sierra_test_labels = sierra_test_labels + [stat_gages.replace('.gpkg', '')]
+        # Read in all sierra test results
+        sierra_test_dfs = [gpd.read_file(i) for i in sierra_test_paths]
+        # Feed results into evaluation function
+        evaluate_results(sierra_test_dfs,
+                        sierra_test_labels,
+                        join(output_dir, 'Sierra_Test_Eval_boxplot.png'))
+
     # Close log file
     sys.stdout = sys.__stdout__
     log_file.close()
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
new file mode 100644
index 000000000..bc780e4cb
--- /dev/null
+++ b/unit_tests/rating_curve_comparison_params.json
@@ -0,0 +1,44 @@
+{
+	"valid_data":
+	{
+		"01010004":
+		[
+			"/data/outputs/carson_gage_test/01010004/usgs_elev_table.csv",
+			"/data/outputs/carson_gage_test/01010004/branches",
+			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_01010004.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_01010004.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_01010004.png",
+			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
+			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
+			"01010004",
+			false
+		],
+		"02020005":
+		[
+			"/data/outputs/carson_gage_test/02020005/usgs_elev_table.csv",
+			"/data/outputs/carson_gage_test/02020005/branches",
+			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
+			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
+			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
+			"02020005",
+			false
+		],
+		"02030103":
+		[
+			"/data/outputs/carson_gage_test/02030103/usgs_elev_table.csv",
+			"/data/outputs/carson_gage_test/02030103/branches",
+			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02030103.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02030103.csv",
+			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02030103.png",
+			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
+			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
+			"02030103",
+			false
+		]
+	}
+}
\ No newline at end of file
diff --git a/unit_tests/rating_curve_comparison_unittests.py b/unit_tests/rating_curve_comparison_unittests.py
new file mode 100644
index 000000000..c68450d40
--- /dev/null
+++ b/unit_tests/rating_curve_comparison_unittests.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+import warnings
+
+import argparse
+import json
+import unittest
+
+import unit_tests_utils as helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/tools/')   # *** update your folder path here if required ***
+from rating_curve_comparison import generate_rating_curve_metrics
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_rating_curve_comparison(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = '/foss_fim/unit_tests/rating_curve_comparison_params.json'
+        warnings.simplefilter('ignore')
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_generate_rating_curve_metrics_01010004_success(self):
+
+        '''
+        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
+          If there is no return value from the method, please say so.>
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+                
+        generate_rating_curve_metrics(params["01010004"])
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+    def test_generate_rating_curve_metrics_02020005_success(self):
+
+        '''
+        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
+          If there is no return value from the method, please say so.>
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+                
+        generate_rating_curve_metrics(params["02020005"])
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+    def test_generate_rating_curve_metrics_02030103_success(self):
+
+        '''
+        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
+          If there is no return value from the method, please say so.>
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+                
+        generate_rating_curve_metrics(params["02030103"])
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json
new file mode 100644
index 000000000..89e80f614
--- /dev/null
+++ b/unit_tests/usgs_gage_crosswalk_params.json
@@ -0,0 +1,13 @@
+{
+	"valid_data":
+	{
+		"usgs_gages_filename": "/data/outputs/gms_example_unit_tests/02020005/usgs_subset_gages.gpkg",
+		"input_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/demDerived_reaches_split_filtered_3246000257.gpkg",
+		"input_catchment_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg",
+		"dem_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/dem_meters_3246000257.tif",
+		"dem_adj_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/dem_thalwegCond_3246000257.tif",
+		"output_table_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/usgs_elev_table.csv",
+		"branch_id": "3246000257"
+
+	}
+}
\ No newline at end of file
diff --git a/unit_tests/usgs_gage_crosswalk_unittests.py b/unit_tests/usgs_gage_crosswalk_unittests.py
new file mode 100644
index 000000000..6fb287eca
--- /dev/null
+++ b/unit_tests/usgs_gage_crosswalk_unittests.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import unittest
+import warnings
+
+import unit_tests_utils as helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+from usgs_gage_crosswalk import GageCrosswalk
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_usgs_gage_crosswalk(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = '/foss_fim/unit_tests/usgs_gage_crosswalk_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_GageCrosswalk_success(self):
+
+        '''
+        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
+          If there is no return value from the method, please say so.>
+        
+        Dev Notes: (which can be removed after you make this file)
+            Remember... You need to validate the method output if there is any. However, if you have time, it
+            is also recommended that you validate other outputs such as writing or updating file on the file 
+            system, aka: Does the expected file exist. Don't worry about its contents.
+
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        
+       
+        # Delete the usgs_elev_table.csv if it exists
+        if os.path.isfile(params["output_table_filename"]):
+            os.remove(params["output_table_filename"])
+
+        # Instantiate and run GageCrosswalk
+        gage_crosswalk = GageCrosswalk(params["usgs_gages_filename"], params["branch_id"])
+
+        # Run crosswalk
+        gage_crosswalk.run_crosswalk(params["input_catchment_filename"], params["input_flows_filename"], 
+                            params["dem_filename"], params["dem_adj_filename"], params["output_table_filename"])
+        print(gage_crosswalk.gages)
+
+        # Make sure that the usgs_elev_table.csv was written
+        assert os.path.isfile(params["output_table_filename"]), f'{params["output_table_filename"]} does not exist'
+
+
+
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    

From f8eb1e0742af0527e3e99afbbf84b273a0bfd76a Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Thu, 10 Mar 2022 10:57:42 -0600
Subject: [PATCH 134/359] Bug fixes for the Alpha Test to run in FIM 4

See Changelog 4.0.3.1 for full details. This addressed #551.
---
 docs/CHANGELOG.md              | 12 ++++++++++++
 tools/eval_plots.py            |  4 ++--
 tools/run_test_case.py         |  2 +-
 tools/synthesize_test_cases.py |  5 +++--
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4da8370ca..830116661 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.3.1 - 2022-03-10 - [PR #561](https://github.com/NOAA-OWP/inundation-mapping/pull/561)
+
+Bug fixes to get the Alpha Test working in FIM 4.
+
+## Changes
+
+- `tools/sythesize_test_cases.py`: Fixed bugs that prevented multiple benchmark types in the same huc from running `run_test_case.py`.
+- `tools/run_test_case.py`: Fixed mall bug for IFC benchmark.
+- `tools/eval_plots.py`: Fixed Pandas query bugs.
+
+<br/><br/>
+
 ## v4.0.3.0 - 2022-03-03 - [PR #550](https://github.com/NOAA-OWP/inundation-mapping/pull/550)
 
 This PR ports the functionality of `usgs_gage_crosswalk.py` and `rating_curve_comparison.py` to FIM 4.
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 6c2dcf00f..be7fdcc71 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -454,7 +454,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     # If versions are supplied then filter out
     if versions:
         #Filter out versions based on supplied version list
-        metrics = csv_df.query('version.str.startswith(tuple(@versions))')
+        metrics = csv_df.query('version.str.startswith(tuple(@versions))', engine='python')
     else:
         metrics = csv_df
 
@@ -481,7 +481,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             base_resolution = 'nws_lid'
 
             # Filter the dataset based on query (IMPORTED FROM TOOLS_SHARED_VARIABLES.py)
-            ahps_metrics = benchmark_metrics.query(DISCARD_AHPS_QUERY)
+            ahps_metrics = benchmark_metrics.query(DISCARD_AHPS_QUERY, engine='python')
 
             # Filter out all instances where the base_resolution doesn't exist across all desired fim versions for a given magnitude
             all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 756386ef7..49891828d 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -363,7 +363,7 @@ def __vprint(message,verbose):
             args['magnitude'] = BLE_MAGNITUDE_LIST
         elif ('nws' == benchmark_category) | ('usgs' == benchmark_category):
             args['magnitude'] = ['action', 'minor', 'moderate', 'major']
-        elif 'ifc' == current_benchmark_category:
+        elif 'ifc' == benchmark_category:
             args['magnitude'] = IFC_MAGNITUDE_LIST
         else:
             print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index e42024c4e..09f08d777 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -163,6 +163,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
                         if iteration == "comparison":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
+                            versions_to_aggregate = dev_versions_to_include_list
 
                         for magnitude in ['action', 'minor', 'moderate', 'major']:
                             for version in versions_to_aggregate:
@@ -357,7 +358,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                 'gms_verbose': False
                                               }
                             
-                            procs_dict[current_huc] = alpha_test_args
+                            procs_dict[test_id] = alpha_test_args
 
     if job_number_huc == 1:
         
@@ -414,4 +415,4 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         print("Creating master metrics CSV...")
 
         # this function is not compatible with GMS
-        create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, versions_to_include_list=dev_versions_to_include_list)
+        create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, dev_versions_to_include_list=dev_versions_to_include_list)

From 153e638ae681d2bddb817a9e4018f0c6eae7b8cc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 14 Mar 2022 09:57:30 -0500
Subject: [PATCH 135/359] Bump pillow from 9.0.0 to 9.0.1 (#563)

Bumps [pillow](https://github.com/python-pillow/Pillow) from 9.0.0 to 9.0.1.
- [Release notes](https://github.com/python-pillow/Pillow/releases)
- [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst)
- [Commits](https://github.com/python-pillow/Pillow/compare/9.0.0...9.0.1)

---
updated-dependencies:
- dependency-name: pillow
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Pipfile.lock | 263 +++++++++++++++++++++++++--------------------------
 1 file changed, 129 insertions(+), 134 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 6474d104d..4085a268b 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -40,39 +40,39 @@
         },
         "cftime": {
             "hashes": [
-                "sha256:07e8f183ed9143308d098b149539ae2a29b367ca825126efaa835d9fda7c7c3f",
-                "sha256:1ae8313bce817bf4a8de0569bc97332101fc580e457bc9ff7afe42ac090c7c7b",
-                "sha256:1eac1695ffcd7cc02e0fc608c5ba7373f1bc39b56a8e76ba62136f1d9ca7069b",
-                "sha256:1f336afb2841ae769b33a720237ab2a4ccd8c14160f1332d2813a39ac9d1c045",
-                "sha256:1f3d8623b1f6c35a4b8c702bf1f4ec5340713b8e2944fd549f8f8a25abd2f0ff",
-                "sha256:291d7d5f19525d2b667fc4ba7d18bcd95ece8384224f8e8ee22c8e72ebddf4de",
-                "sha256:2b774870f21045c7b0728ac3cdea9c57a2121c01eb95bf17e0ed4aaf58a939ef",
-                "sha256:375d37d9ab8bf501c048e44efce2276296e3d67bb276e891e0e93b0a8bbb988a",
-                "sha256:3fb31f26828c909c0234834a8882c137428d75638734ba8d2ef9e66a57e81fec",
-                "sha256:4bd4a94376989a6cf69d4a59e9e5871d19ded6de9956e4fc6b5d2775d509c49e",
-                "sha256:5c8716983468ebfa9b6fba0040881f431334ea56ad9d1df075ae15cd37ae7985",
-                "sha256:5ff10b4ac0be73ee319c1dded6814e509d52ccb993da8a6496f153f5bb0e6efb",
-                "sha256:66af87cdde654d0cde0a0b936d11e0675b57697b369a142de8d5e5efae1e53d8",
-                "sha256:670bc389b7d7a7e61cf42e7842a820c059efb2adce50eacef51e25389daab01f",
-                "sha256:695f45898a00acceed4795f76b2ac4f1455945fe9e3a174b0ea9fff9dc2c9a17",
-                "sha256:6bfb44c72e277c12f60c4c4b6186963191743bf80a438246d37c7320a95fa54a",
-                "sha256:740d11d0b567390e2892d1dd81387cc2e547b1856ed3969dd51c4a089d9501b2",
-                "sha256:7410c1435d52758c8cf2d46b8021c5bfcabf2ad85e360a466bc2b6d224316a82",
-                "sha256:832c7674f6fec8bdacc09928bd976600d886035ad39d2e781897a0e180552fce",
-                "sha256:a818f59878ed029066748980ecfd435600be76b62f9269ddc06da91668ed506a",
-                "sha256:a9db712b1092771d7e79c3e44838f2ddd751fad874d90d05b37c209c9dd160ea",
-                "sha256:be258adaba9298d00cd420e27f5ba4a3e379aaf4d7009d8826f607ca58c9b813",
-                "sha256:bf7021e635126323ca84622838fef634f9dd5334c1c5658b7a9181bb4d819d27",
-                "sha256:bf9871732414fa2841452fd0f9233e52f4e1bf49ba47005ea81bff0f9c4f40a1",
-                "sha256:c49599320537db1c067f1d77e00264a3d3c53ebde13a0e62269c1c2cf9828d72",
-                "sha256:c8ad56b7185a68fa4b0b27fee949cb7af2de1ca37207e177efc9231a0785562b",
-                "sha256:cfebcf6e9e5afe015d848908cb7d56009aa98b3a63a28bcd6faee18b8d7b75de",
-                "sha256:dfc6a15acce854571d82565aac3d47df40317f2290e8e304a89dcb9351de19eb",
-                "sha256:e7fe234c7bbc04af7d1d8a2ee5649f9a2f2de2aa424996453ed2173302bd9638",
-                "sha256:f24330721024500b5dbe87f164aa30b82a0e785f62d3d09fb9efeac5e5623385",
-                "sha256:f904e3efa04adc727e787b1306ef50c181e9c448a71656938d7483cc104c90c8"
-            ],
-            "version": "==1.5.2"
+                "sha256:003abc800933a39d6f4ecec113f401aef9961642a4769aa1099ed34e0c1c3d46",
+                "sha256:09970124c0a8c84a66e34e8655c42f0254e1c29d73a766e71c9a871ce57f125b",
+                "sha256:1187fa1107974fe374f90936e39d2e09ac7cb4ffcd3b31314898684a2cebba87",
+                "sha256:13103e6650beea6552316bd5825d6aa3b7e98f5b8115026df4826798dff9f741",
+                "sha256:13e92118fa3606b2a1d95bbe1a18a0a260ba85704d9ff061e9599b0e745bb4a9",
+                "sha256:1888f0c0ab37d6ee23d07e74e51dcaae136da48784ceaa5d078115aa2e2b8b92",
+                "sha256:1c56556900b9ae3a5b4c9705699d78f865994e5c3c92cd230d54c89d669b88f0",
+                "sha256:1f43f1c839f5579bf045f13b0d2deb15048d314d209039664003ae1bf2544898",
+                "sha256:2da21d8626a7f549eaaae7d314110133914fa24726e4e7caab2775627cb7361a",
+                "sha256:4c608937082980e0905657ec2475b3ff2a2fc89255e185f44076da482102f617",
+                "sha256:4d39b666284c78726eb7ea8b4ad29aacd387e912177721b3eb9bdc508e2b86fa",
+                "sha256:517c50e29b3c141919d83bf3782892e4a4144034b63722e7a266db25488b3bb5",
+                "sha256:519d1c391ed7be3bb4d84990a51a76cbb1cc894bd776637c0227dfe1f6274f39",
+                "sha256:64bcb6ca9ef922b2332e9ed91cd6f6f2fe28de392fb10e752f2ac83ffdc50854",
+                "sha256:7900daf0efa495f263c3c5b0c96502cc909fb07ef56844791e3c8cdbb08a741c",
+                "sha256:80390ddb68a7e27b52bf83efc040c8e3ec2e3b0bb0e66084a75498fb39f63a63",
+                "sha256:9b31455449d2e347d033a6440f94305ce7eb48d32d863dfe0f53671c78e2561d",
+                "sha256:a49b023ca5cab9c43d33b3a43edc5a112f5d452f75fc1a85f8ca136f4fa9d440",
+                "sha256:a6c1e59299250299f23ab526d1ad640dcc3b9eb311468a8f9112f9d53d6106c6",
+                "sha256:a8a56edad64912b11d8f80245b5ce7e61057eee22b6af0718e9d086b381a17d9",
+                "sha256:b420e3811da55db538008229b3ca34055734d6ade12316889d1979d770d7dfaf",
+                "sha256:bb0d680149f128d181ef1c6561eaebbf99b572329dee090c83b6735d7284a703",
+                "sha256:bfb03c766250d86d6abf546d4c1fbe0db7401eaedeab5bbbd0e599fde7c7a7d3",
+                "sha256:c0b3aff3c955d18342c2097ad1102c78d0999dba5de178d0ce68a317caf6f87e",
+                "sha256:cab3fee686c58e2ce2f61e1bf138711eefde6c0235b0cfc736c8ea96381d8346",
+                "sha256:e020db7391b2112e4c1ec9db75ff82031f979666264903697ab5ad0bfb20d490",
+                "sha256:e195ed19638f324c40bb0094d13385d58e798607670b91cc8f42717a0a832263",
+                "sha256:eca80007816b9dd1cc154a393b8da2ada22baad1ac7b2b2a9cc01451796e3e93",
+                "sha256:ee740b8dd77cb1773ae1560b890cee3aa5b82a4a7ca056223f08f4b37ebb6466",
+                "sha256:f63c14e98022c3a67ff7c5c12edf17912a7353ddd95bbfde6db1e4bc05ea5bdc",
+                "sha256:f7007e7cd837cb37716149d7c735b90b947b3370b7a5652d58675ea940541542"
+            ],
+            "version": "==1.6.0"
         },
         "click": {
             "hashes": [
@@ -123,11 +123,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:1933415e0fbdf068815cb1baaa1f159e17830215f7e8624e5731122761627557",
-                "sha256:2b18a172120e32128a80efee04cff487d5d140fe7d817deb648b2eee023a40e4"
+                "sha256:084dd1762f083a1bf49e41da1bfeafb475c9dce46265690a6bdd33290b9a63f4",
+                "sha256:6985cc5380c06db07fdc73ade15e6adbd4ce6ff850d7561ca00f97090b4b263d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.29.1"
+            "version": "==4.30.0"
         },
         "geopandas": {
             "hashes": [
@@ -395,31 +395,29 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:0d245a2bf79188d3f361137608c3cd12ed79076badd743dc660750a9f3074f7c",
-                "sha256:26b4018a19d2ad9606ce9089f3d52206a41b23de5dfe8dc947d2ec49ce45d015",
-                "sha256:2db01d9838a497ba2aa9a87515aeaf458f42351d72d4e7f3b8ddbd1eba9479f2",
-                "sha256:3d62d6b0870b53799204515145935608cdeb4cebb95a26800b6750e48884cc5b",
-                "sha256:45a7dfbf9ed8d68fd39763940591db7637cf8817c5bce1a44f7b56c97cbe211e",
-                "sha256:4ac4d7c9f8ea2a79d721ebfcce81705fc3cd61a10b731354f1049eb8c99521e8",
-                "sha256:60f19c61b589d44fbbab8ff126640ae712e163299c2dd422bfe4edc7ec51aa9b",
-                "sha256:632e062569b0fe05654b15ef0e91a53c0a95d08ffe698b66f6ba0f927ad267c2",
-                "sha256:65f5e257987601fdfc63f1d02fca4d1c44a2b85b802f03bd6abc2b0b14648dd2",
-                "sha256:69958735d5e01f7b38226a6c6e7187d72b7e4d42b6b496aca5860b611ca0c193",
-                "sha256:78bfbdf809fc236490e7e65715bbd98377b122f329457fffde206299e163e7f3",
-                "sha256:7e957ca8112c689b728037cea9c9567c27cf912741fabda9efc2c7d33d29dfa1",
-                "sha256:800dfeaffb2219d49377da1371d710d7952c9533b57f3d51b15e61c4269a1b5b",
-                "sha256:831f2df87bd3afdfc77829bc94bd997a7c212663889d56518359c827d7113b1f",
-                "sha256:88d54b7b516f0ca38a69590557814de2dd638d7d4ed04864826acaac5ebb8f01",
-                "sha256:8d1563060e77096367952fb44fca595f2b2f477156de389ce7c0ade3aef29e21",
-                "sha256:b5ec9a5eaf391761c61fd873363ef3560a3614e9b4ead17347e4deda4358bca4",
-                "sha256:bcd19dab43b852b03868796f533b5f5561e6c0e3048415e675bec8d2e9d286c1",
-                "sha256:c51124df17f012c3b757380782ae46eee85213a3215e51477e559739f57d9bf6",
-                "sha256:e348ccf5bc5235fc405ab19d53bec215bb373300e5523c7b476cc0da8a5e9973",
-                "sha256:e60ef82c358ded965fdd3132b5738eade055f48067ac8a5a8ac75acc00cad31f",
-                "sha256:f8ad59e6e341f38266f1549c7c2ec70ea0e3d1effb62a44e5c3dba41c55f0187"
+                "sha256:07a8c89a04997625236c5ecb7afe35a02af3896c8aa01890a849913a2309c676",
+                "sha256:08d9b008d0156c70dc392bb3ab3abb6e7a711383c3247b410b39962263576cd4",
+                "sha256:201b4d0552831f7250a08d3b38de0d989d6f6e4658b709a02a73c524ccc6ffce",
+                "sha256:2c10a93606e0b4b95c9b04b77dc349b398fdfbda382d2a39ba5a822f669a0123",
+                "sha256:3ca688e1b9b95d80250bca34b11a05e389b1420d00e87a0d12dc45f131f704a1",
+                "sha256:48a3aecd3b997bf452a2dedb11f4e79bc5bfd21a1d4cc760e703c31d57c84b3e",
+                "sha256:568dfd16224abddafb1cbcce2ff14f522abe037268514dd7e42c6776a1c3f8e5",
+                "sha256:5bfb1bb598e8229c2d5d48db1860bcf4311337864ea3efdbe1171fb0c5da515d",
+                "sha256:639b54cdf6aa4f82fe37ebf70401bbb74b8508fddcf4797f9fe59615b8c5813a",
+                "sha256:8251ed96f38b47b4295b1ae51631de7ffa8260b5b087808ef09a39a9d66c97ab",
+                "sha256:92bfa69cfbdf7dfc3040978ad09a48091143cffb778ec3b03fa170c494118d75",
+                "sha256:97098b95aa4e418529099c26558eeb8486e66bd1e53a6b606d684d0c3616b168",
+                "sha256:a3bae1a2ed00e90b3ba5f7bd0a7c7999b55d609e0c54ceb2b076a25e345fa9f4",
+                "sha256:c34ea7e9d13a70bf2ab64a2532fe149a9aced424cd05a2c4ba662fd989e3e45f",
+                "sha256:dbc7601a3b7472d559dc7b933b18b4b66f9aa7452c120e87dfb33d02008c8a18",
+                "sha256:e7927a589df200c5e23c57970bafbd0cd322459aa7b1ff73b7c2e84d6e3eae62",
+                "sha256:f8c1f39caad2c896bc0018f699882b345b2a63708008be29b1f355ebf6f933fe",
+                "sha256:f950f8845b480cffe522913d35567e29dd381b0dc7e4ce6a4a9f9156417d2430",
+                "sha256:fade0d4f4d292b6f39951b6836d7a3c7ef5b2347f3c420cd9820a1d90d794802",
+                "sha256:fdf3c08bce27132395d3c3ba1503cac12e17282358cb4bddc25cc46b0aca07aa"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.22.1"
+            "version": "==1.22.3"
         },
         "packaging": {
             "hashes": [
@@ -453,41 +451,44 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:03b27b197deb4ee400ed57d8d4e572d2d8d80f825b6634daf6e2c18c3c6ccfa6",
-                "sha256:0b281fcadbb688607ea6ece7649c5d59d4bbd574e90db6cd030e9e85bde9fecc",
-                "sha256:0ebd8b9137630a7bbbff8c4b31e774ff05bbb90f7911d93ea2c9371e41039b52",
-                "sha256:113723312215b25c22df1fdf0e2da7a3b9c357a7d24a93ebbe80bfda4f37a8d4",
-                "sha256:2d16b6196fb7a54aff6b5e3ecd00f7c0bab1b56eee39214b2b223a9d938c50af",
-                "sha256:2fd8053e1f8ff1844419842fd474fc359676b2e2a2b66b11cc59f4fa0a301315",
-                "sha256:31b265496e603985fad54d52d11970383e317d11e18e856971bdbb86af7242a4",
-                "sha256:3586e12d874ce2f1bc875a3ffba98732ebb12e18fb6d97be482bd62b56803281",
-                "sha256:47f5cf60bcb9fbc46011f75c9b45a8b5ad077ca352a78185bd3e7f1d294b98bb",
-                "sha256:490e52e99224858f154975db61c060686df8a6b3f0212a678e5d2e2ce24675c9",
-                "sha256:500d397ddf4bbf2ca42e198399ac13e7841956c72645513e8ddf243b31ad2128",
-                "sha256:52abae4c96b5da630a8b4247de5428f593465291e5b239f3f843a911a3cf0105",
-                "sha256:6579f9ba84a3d4f1807c4aab4be06f373017fc65fff43498885ac50a9b47a553",
-                "sha256:68e06f8b2248f6dc8b899c3e7ecf02c9f413aab622f4d6190df53a78b93d97a5",
-                "sha256:6c5439bfb35a89cac50e81c751317faea647b9a3ec11c039900cd6915831064d",
-                "sha256:72c3110228944019e5f27232296c5923398496b28be42535e3b2dc7297b6e8b6",
-                "sha256:72f649d93d4cc4d8cf79c91ebc25137c358718ad75f99e99e043325ea7d56100",
-                "sha256:7aaf07085c756f6cb1c692ee0d5a86c531703b6e8c9cae581b31b562c16b98ce",
-                "sha256:80fe92813d208ce8aa7d76da878bdc84b90809f79ccbad2a288e9bcbeac1d9bd",
-                "sha256:95545137fc56ce8c10de646074d242001a112a92de169986abd8c88c27566a05",
-                "sha256:97b6d21771da41497b81652d44191489296555b761684f82b7b544c49989110f",
-                "sha256:98cb63ca63cb61f594511c06218ab4394bf80388b3d66cd61d0b1f63ee0ea69f",
-                "sha256:9f3b4522148586d35e78313db4db0df4b759ddd7649ef70002b6c3767d0fdeb7",
-                "sha256:a09a9d4ec2b7887f7a088bbaacfd5c07160e746e3d47ec5e8050ae3b2a229e9f",
-                "sha256:b5050d681bcf5c9f2570b93bee5d3ec8ae4cf23158812f91ed57f7126df91762",
-                "sha256:bb47a548cea95b86494a26c89d153fd31122ed65255db5dcbc421a2d28eb3379",
-                "sha256:bc462d24500ba707e9cbdef436c16e5c8cbf29908278af053008d9f689f56dee",
-                "sha256:c2067b3bb0781f14059b112c9da5a91c80a600a97915b4f48b37f197895dd925",
-                "sha256:d154ed971a4cc04b93a6d5b47f37948d1f621f25de3e8fa0c26b2d44f24e3e8f",
-                "sha256:d5dcea1387331c905405b09cdbfb34611050cc52c865d71f2362f354faee1e9f",
-                "sha256:ee6e2963e92762923956fe5d3479b1fdc3b76c83f290aad131a2f98c3df0593e",
-                "sha256:fd0e5062f11cb3e730450a7d9f323f4051b532781026395c4323b8ad055523c4"
+                "sha256:011233e0c42a4a7836498e98c1acf5e744c96a67dd5032a6f666cc1fb97eab97",
+                "sha256:0f29d831e2151e0b7b39981756d201f7108d3d215896212ffe2e992d06bfe049",
+                "sha256:12875d118f21cf35604176872447cdb57b07126750a33748bac15e77f90f1f9c",
+                "sha256:14d4b1341ac07ae07eb2cc682f459bec932a380c3b122f5540432d8977e64eae",
+                "sha256:1c3c33ac69cf059bbb9d1a71eeaba76781b450bc307e2291f8a4764d779a6b28",
+                "sha256:1d19397351f73a88904ad1aee421e800fe4bbcd1aeee6435fb62d0a05ccd1030",
+                "sha256:253e8a302a96df6927310a9d44e6103055e8fb96a6822f8b7f514bb7ef77de56",
+                "sha256:2632d0f846b7c7600edf53c48f8f9f1e13e62f66a6dbc15191029d950bfed976",
+                "sha256:335ace1a22325395c4ea88e00ba3dc89ca029bd66bd5a3c382d53e44f0ccd77e",
+                "sha256:413ce0bbf9fc6278b2d63309dfeefe452835e1c78398efb431bab0672fe9274e",
+                "sha256:5100b45a4638e3c00e4d2320d3193bdabb2d75e79793af7c3eb139e4f569f16f",
+                "sha256:514ceac913076feefbeaf89771fd6febde78b0c4c1b23aaeab082c41c694e81b",
+                "sha256:528a2a692c65dd5cafc130de286030af251d2ee0483a5bf50c9348aefe834e8a",
+                "sha256:6295f6763749b89c994fcb6d8a7f7ce03c3992e695f89f00b741b4580b199b7e",
+                "sha256:6c8bc8238a7dfdaf7a75f5ec5a663f4173f8c367e5a39f87e720495e1eed75fa",
+                "sha256:718856856ba31f14f13ba885ff13874be7fefc53984d2832458f12c38205f7f7",
+                "sha256:7f7609a718b177bf171ac93cea9fd2ddc0e03e84d8fa4e887bdfc39671d46b00",
+                "sha256:80ca33961ced9c63358056bd08403ff866512038883e74f3a4bf88ad3eb66838",
+                "sha256:80fe64a6deb6fcfdf7b8386f2cf216d329be6f2781f7d90304351811fb591360",
+                "sha256:81c4b81611e3a3cb30e59b0cf05b888c675f97e3adb2c8672c3154047980726b",
+                "sha256:855c583f268edde09474b081e3ddcd5cf3b20c12f26e0d434e1386cc5d318e7a",
+                "sha256:9bfdb82cdfeccec50aad441afc332faf8606dfa5e8efd18a6692b5d6e79f00fd",
+                "sha256:a5d24e1d674dd9d72c66ad3ea9131322819ff86250b30dc5821cbafcfa0b96b4",
+                "sha256:a9f44cd7e162ac6191491d7249cceb02b8116b0f7e847ee33f739d7cb1ea1f70",
+                "sha256:b5b3f092fe345c03bca1e0b687dfbb39364b21ebb8ba90e3fa707374b7915204",
+                "sha256:b9618823bd237c0d2575283f2939655f54d51b4527ec3972907a927acbcc5bfc",
+                "sha256:cef9c85ccbe9bee00909758936ea841ef12035296c748aaceee535969e27d31b",
+                "sha256:d21237d0cd37acded35154e29aec853e945950321dd2ffd1a7d86fe686814669",
+                "sha256:d3c5c79ab7dfce6d88f1ba639b77e77a17ea33a01b07b99840d6ed08031cb2a7",
+                "sha256:d9d7942b624b04b895cb95af03a23407f17646815495ce4547f0e60e0b06f58e",
+                "sha256:db6d9fac65bd08cea7f3540b899977c6dee9edad959fa4eaf305940d9cbd861c",
+                "sha256:ede5af4a2702444a832a800b8eb7f0a7a1c0eed55b644642e049c98d589e5092",
+                "sha256:effb7749713d5317478bb3acb3f81d9d7c7f86726d41c1facca068a04cf5bb4c",
+                "sha256:f154d173286a5d1863637a7dcd8c3437bb557520b01bddb0be0258dcb72696b5",
+                "sha256:f25ed6e28ddf50de7e7ea99d7a976d6a9c415f03adcaac9c41ff6ff41b6d86ac"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==9.0.0"
+            "index": "pypi",
+            "version": "==9.0.1"
         },
         "psutil": {
             "hashes": [
@@ -650,38 +651,32 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:033ce76ed4e9f62923e1f8124f7e2b0800db533828c853b402c7eec6e9465d80",
-                "sha256:173308efba2270dcd61cd45a30dfded6ec0085b4b6eb33b5eb11ab443005e088",
-                "sha256:21b66200cf44b1c3e86495e3a436fc7a26608f92b8d43d344457c54f1c024cbc",
-                "sha256:2c56b820d304dffcadbbb6cbfbc2e2c79ee46ea291db17e288e73cd3c64fefa9",
-                "sha256:304dfaa7146cffdb75fbf6bb7c190fd7688795389ad060b970269c8576d038e9",
-                "sha256:3f78181a153fa21c018d346f595edd648344751d7f03ab94b398be2ad083ed3e",
-                "sha256:4d242d13206ca4302d83d8a6388c9dfce49fc48fdd3c20efad89ba12f785bf9e",
-                "sha256:5d1cc2c19afe3b5a546ede7e6a44ce1ff52e443d12b231823268019f608b9b12",
-                "sha256:5f2cfc359379c56b3a41b17ebd024109b2049f878badc1e454f31418c3a18436",
-                "sha256:65bd52bf55f9a1071398557394203d881384d27b9c2cad7df9a027170aeaef93",
-                "sha256:7edd9a311299a61e9919ea4192dd477395b50c014cdc1a1ac572d7c27e2207fa",
-                "sha256:8499d9dd1459dc0d0fe68db0832c3d5fc1361ae8e13d05e6849b358dc3f2c279",
-                "sha256:866ada14a95b083dd727a845a764cf95dd13ba3dc69a16b99038001b05439709",
-                "sha256:87069cf875f0262a6e3187ab0f419f5b4280d3dcf4811ef9613c605f6e4dca95",
-                "sha256:93378f3d14fff07572392ce6a6a2ceb3a1f237733bd6dcb9eb6a2b29b0d19085",
-                "sha256:95c2d250074cfa76715d58830579c64dff7354484b284c2b8b87e5a38321672c",
-                "sha256:ab5875facfdef77e0a47d5fd39ea178b58e60e454a4c85aa1e52fcb80db7babf",
-                "sha256:b0e0aeb061a1d7dcd2ed59ea57ee56c9b23dd60100825f98238c06ee5cc4467e",
-                "sha256:b78a35c5c74d336f42f44106174b9851c783184a85a3fe3e68857259b37b9ffb",
-                "sha256:c9e04d7e9b03a8a6ac2045f7c5ef741be86727d8f49c45db45f244bdd2bcff17",
-                "sha256:ca36e7d9430f7481fc7d11e015ae16fbd5575615a8e9060538104778be84addf",
-                "sha256:ceebc3c4f6a109777c0053dfa0282fddb8893eddfb0d598574acfb734a926168",
-                "sha256:e2c036492e673aad1b7b0d0ccdc0cb30a968353d2c4bf92ac8e73509e1bf212c",
-                "sha256:eb326658f9b73c07081300daba90a8746543b5ea177184daed26528273157294",
-                "sha256:eb7ae2c4dbdb3c9247e07acc532f91077ae6dbc40ad5bd5dca0bb5a176ee9bda",
-                "sha256:edad1cf5b2ce1912c4d8ddad20e11d333165552aba262c882e28c78bbc09dbf6",
-                "sha256:eef93a446114ac0193a7b714ce67659db80caf940f3232bad63f4c7a81bc18df",
-                "sha256:f7eaea089345a35130bc9a39b89ec1ff69c208efa97b3f8b25ea5d4c41d88094",
-                "sha256:f99d206db1f1ae735a8192ab93bd6028f3a42f6fa08467d37a14eb96c9dd34a3"
-            ],
-            "markers": "python_version < '3.11' and python_version >= '3.7'",
-            "version": "==1.7.3"
+                "sha256:011d4386b53b933142f58a652aa0f149c9b9242abd4f900b9f4ea5fbafc86b89",
+                "sha256:16e09ef68b352d73befa8bcaf3ebe25d3941fe1a58c82909d5589856e6bc8174",
+                "sha256:31d4f2d6b724bc9a98e527b5849b8a7e589bf1ea630c33aa563eda912c9ff0bd",
+                "sha256:38aa39b6724cb65271e469013aeb6f2ce66fd44f093e241c28a9c6bc64fd79ed",
+                "sha256:3d573228c10a3a8c32b9037be982e6440e411b443a6267b067cac72f690b8d56",
+                "sha256:3d9dd6c8b93a22bf9a3a52d1327aca7e092b1299fb3afc4f89e8eba381be7b59",
+                "sha256:559a8a4c03a5ba9fe3232f39ed24f86457e4f3f6c0abbeae1fb945029f092720",
+                "sha256:5e73343c5e0d413c1f937302b2e04fb07872f5843041bcfd50699aef6e95e399",
+                "sha256:723b9f878095ed994756fa4ee3060c450e2db0139c5ba248ee3f9628bd64e735",
+                "sha256:87b01c7d5761e8a266a0fbdb9d88dcba0910d63c1c671bdb4d99d29f469e9e03",
+                "sha256:8f4d059a97b29c91afad46b1737274cb282357a305a80bdd9e8adf3b0ca6a3f0",
+                "sha256:92b2c2af4183ed09afb595709a8ef5783b2baf7f41e26ece24e1329c109691a7",
+                "sha256:937d28722f13302febde29847bbe554b89073fbb924a30475e5ed7b028898b5f",
+                "sha256:a279e27c7f4566ef18bab1b1e2c37d168e365080974758d107e7d237d3f0f484",
+                "sha256:ad5be4039147c808e64f99c0e8a9641eb5d2fa079ff5894dcd8240e94e347af4",
+                "sha256:ae3e327da323d82e918e593460e23babdce40d7ab21490ddf9fc06dec6b91a18",
+                "sha256:bb7088e89cd751acf66195d2f00cf009a1ea113f3019664032d9075b1e727b6c",
+                "sha256:c17a1878d00a5dd2797ccd73623ceca9d02375328f6218ee6d921e1325e61aff",
+                "sha256:c2bae431d127bf0b1da81fc24e4bba0a84d058e3a96b9dd6475dfcb3c5e8761e",
+                "sha256:de2e80ee1d925984c2504812a310841c241791c5279352be4707cdcd7c255039",
+                "sha256:e6f0cd9c0bd374ef834ee1e0f0999678d49dcc400ea6209113d81528958f97c7",
+                "sha256:f3720d0124aced49f6f2198a6900304411dbbeed12f56951d7c66ebef05e3df6",
+                "sha256:f4a6d3b9f9797eb2d43938ac2c5d96d02aed17ef170c8b38f11798717523ddba"
+            ],
+            "markers": "python_version < '3.11' and python_version >= '3.8'",
+            "version": "==1.8.0"
         },
         "seaborn": {
             "hashes": [
@@ -693,11 +688,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:c99207037c38984eae838c2fd986f39a9ddf4fabfe0fddd957e622d1d1dcdd05",
-                "sha256:eb83b1012ae6bf436901c2a2cee35d45b7260f31fd4b65fd1e50a9f99c11d7f8"
+                "sha256:2347b2b432c891a863acadca2da9ac101eae6169b1d3dfee2ec605ecd50dbfe5",
+                "sha256:e4f30b9f84e5ab3decf945113119649fec09c1fc3507c6ebffec75646c56e62b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==60.6.0"
+            "version": "==60.9.3"
         },
         "shapely": {
             "hashes": [
@@ -918,11 +913,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:4bcf119be2200c17ed0d518872ef922f1de336eb6d1ddbd1e089ceb6447d97c6",
-                "sha256:a51d41a6a45fd9def54365bca8f0402c8f182f2b6f7e29c74d55faeb9fb38ac4"
+                "sha256:30129d870dcb0b3b6a53efdc9d0a83ea96162ffd28ffe077e94215b233dc670c",
+                "sha256:9f1cd16b1e86c2968f2519d7fb31dd9d669916f515612c269d14e9ed52b51650"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.26"
+            "version": "==3.0.28"
         },
         "ptyprocess": {
             "hashes": [
@@ -941,11 +936,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:c99207037c38984eae838c2fd986f39a9ddf4fabfe0fddd957e622d1d1dcdd05",
-                "sha256:eb83b1012ae6bf436901c2a2cee35d45b7260f31fd4b65fd1e50a9f99c11d7f8"
+                "sha256:2347b2b432c891a863acadca2da9ac101eae6169b1d3dfee2ec605ecd50dbfe5",
+                "sha256:e4f30b9f84e5ab3decf945113119649fec09c1fc3507c6ebffec75646c56e62b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==60.6.0"
+            "version": "==60.9.3"
         },
         "traitlets": {
             "hashes": [

From 43bcf4901802a39eff07b05910f02e3e9975be2d Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Tue, 12 Apr 2022 16:07:25 -0500
Subject: [PATCH 136/359] Stream order selection bug fixes, new logging
 features

See Changelog 4.0.4.0 for full details.

This resolves #579
---
 docs/CHANGELOG.md                             | 101 +++++++++++++++++-
 fim_run.sh                                    |  17 +++
 gms_run_branch.sh                             |  35 +++++-
 gms_run_unit.sh                               |  28 +++--
 src/filter_catchments_and_add_attributes.py   |  45 ++++++--
 src/gms/derive_level_paths.py                 |  26 ++++-
 src/gms/generate_branch_list.py               |   3 +-
 src/gms/run_by_branch.sh                      |  25 +++--
 src/gms/run_by_unit.sh                        |   9 ++
 src/gms/stream_branches.py                    |   6 ++
 src/gms/time_and_tee_run_by_branch.sh         |  22 +++-
 src/gms/time_and_tee_run_by_unit.sh           |  23 +++-
 src/gms/toDo.md                               |   1 -
 src/run_by_unit.sh                            |  14 ++-
 src/split_flows.py                            |  40 +++++--
 src/time_and_tee_run_by_unit.sh               |  26 ++++-
 src/utils/fim_enums.py                        |  28 +++++
 ..._catchments_and_add_attributes_params.json |  12 +++
 ...catchments_and_add_attributes_unittests.py |  86 +++++++++++++++
 unit_tests/gms/derive_level_paths_params.json |  33 ++++--
 .../gms/derive_level_paths_unittests.py       |  38 ++++++-
 unit_tests/split_flows_params.json            |  15 +++
 unit_tests/split_flows_unittests.py           |  89 +++++++++++++++
 23 files changed, 653 insertions(+), 69 deletions(-)
 create mode 100644 src/utils/fim_enums.py
 create mode 100644 unit_tests/filter_catchments_and_add_attributes_params.json
 create mode 100644 unit_tests/filter_catchments_and_add_attributes_unittests.py
 create mode 100644 unit_tests/split_flows_params.json
 create mode 100644 unit_tests/split_flows_unittests.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 830116661..dac104ad7 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,101 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.4.0 - 2022-04-12 - [PR #557](https://github.com/NOAA-OWP/inundation-mapping/pull/557)
+
+During large scale testing of the new **filtering out stream orders 1 and 2** feature [PR #548](https://github.com/NOAA-OWP/inundation-mapping/pull/548), a bug was discovered with 14 HUCS that had no remaining streams after removing stream orders 1 and 2. This resulted in a number of unmanaged and unclear exceptions. An exception may be still raised will still be raised in this fix for logging purposes, but it is now very clear what happened. Other types of events are logged with clear codes to identify what happened.
+
+Fixes were put in place for a couple of new logging behaviors.
+
+1. Recognize that for system exit codes, there are times when an event is neither a success (code 0) nor a failure (code 1). During processing where stream orders are dropped, some HUCs had no remaining reaches, others had mismatched reaches and others as had missing flowlines (reaches) relating to dissolved level paths (merging individual reaches as part of GMS). When these occur, we want to abort the HUC (unit) or branch processing, identify that they were aborted for specific reasons and continue. A new custom system exit code system was adding using python enums. Logging was enhanced to recognize that some exit codes were not a 0 or a 1 and process them differently.
+
+2. Pathing and log management became an issue. It us not uncommon for tens or hundreds of thousands of branches to be processed. A new feature was to recognize what is happening with each branch or unit and have them easily found and recognizable. Futher, processing for failure (sys exit code of 1) are now copied into a unique folder as the occur to help with visualization of run time errors. Previously errors were not extracted until the end of the entire run which may be multiple days.
+
+3. A minor correction was made when dissolved level paths were created with the new merged level path not always having a valid stream order value.
+
+## File Additions
+
+- `src/`
+   - `utils/`
+      - `fim_enums.py`:
+         - A new class called `FIM_system_exit_codes` was added. This allows tracking and blocking of duplicate system exit codes when a custom system code is required.
+        
+
+## Changes
+
+- `fim_run.sh`: Added the gms `non-zero-exit-code` system to `fim_run` to help uncover and isolate errors during processing. Errors recorded in log files within in the logs/unit folder are now copied into a new folder called `unit_errors`.  
+    
+- `gms_run_branch.sh`:
+    -  Minor adjustments to how the `non-zero-exit code` logs were created. Testing uncovered that previous versions were not always reliable. This is now stablized and enhanced.
+    - In previous versions, only the `gms_unit.sh` was aware that **stream order filtering** was being done. Now all branch processing is also aware that filtering is in place. Processing in child files and classes can now make adjustments as/if required for stream order filtering.
+    - Small output adjustments were made to help with overall screen and log readability.  
+
+- `gms_run_unit.sh`:
+    - Minor adjustments to how the `non-zero-exit-code` logs were created similar to `gms_run_branch.sh.`
+    - Small text corrections, formatting and output corrections were added.
+    - A feature removing all log files at the start of the entire process run were added if the `overwrite` command line argument was added.
+
+- `src/`
+   - `filter_catchments_and_add_attributes.py`:
+      - Some minor formatting and readability adjustments were added.
+      - Additions were made to help this code be aware and responding accordingly if that stream order filtering has occurred. Previously recorded as bugs coming from this class, are now may recorded with the new custom exit code if applicable.
+
+   - `run_by_unit.sh` (supporting fim_run.sh):
+         - As a change was made to sub-process call to `filter_catchments_and_add_attributes.py` file, which is shared by gms, related to reach errors / events.
+
+   - `split_flows.py`:
+      - Some minor formatting and readability adjustments were added.
+      - Additions were made to recognize the same type of errors as being described in other files related to stream order filtering issues.
+      - A correction was made to be more precise and more explicit when a gms branch error existed. This was done to ensure that we were not letting other exceptions be trapped that were NOT related to stream flow filtering.
+      
+   - `time_and_tee_run_by_unit.sh`:
+      - The new custom system exit codes was added. Note that the values of 61 (responding system code) are hardcoded instead of using the python based `Fim_system_exit_code` system. This is related to limited communication between python and bash.
+
+   - `gms/`
+      - `derive_level_paths.py`:  
+          - Was upgraded to use the new fim_enums.Fim_system_exit_codes system. This occurs when no streams / flows remain after filtering.  Without this upgrade, standard exceptions were being issued with minimal details for the error.
+          - Minor adjustments to formatting for readability were made.
+
+      - `generate_branch_list.py` :  Minor adjustments to formatting for readability were made.
+
+      - `run_by_branch.sh`:
+         - Some minor formatting and readability adjustments were added.
+         - Additions to the subprocess call to `split_flows.py` were added so it was aware that branch filtering was being used. `split_flows.py` was one of the files that was throwing errors related to stream order filtering. A subprocess call to `filter_catchments_and_add_attributes.py` adjustment was also required for the same reason.
+
+      - `run_by_unit.sh`:
+         - Some minor formatting and readability adjustments were added.
+         - An addition was made to help trap errors that might be triggered by `derive_level_paths.py` for `stream order filtering`.
+
+      - `time_and_tee_run_by_branch.sh`:
+         - A system was added recognize if an non successful system exit code was sent back from `run_by_branch`. This includes true errors of code 1 and other new custom system exit codes. Upon detection of non-zero-exit codes, log files are immediately copied into special folders for quicker and easier visibility. Previously errors were not brought forth until the entire process was completed which ranged fro hours up to 18 days. Note: System exit codes of 60 and 61 were hardcoded instead of using the values from the new  `FIM_system_exit_codes` due to limitation of communication between python and bash.
+
+      - `time_and_tee_run_by_unit.sh`:
+         - The same upgrade as described above in `time_and_tee_run_by_branch.sh` was applied here.
+         - Minor readability and output formatting changes were made.
+
+      - `todo.md`
+         - An entry was removed from this list which talked about errors due to small level paths exactly as was fixed in this pull request set.
+
+- `unit_tests/`
+   - `gms/`
+      - `derive_level_paths_unittests.py` :  Added a new unit test specifically testing this type of condition with a known HUC that triggered the branch errors previously described..
+      - `derive_level_paths_params.json`:
+           - Added a new node with a HUC number known to fail.
+           - Changed pathing for unit test data pathing from `/data/outputs/gms_example_unit_tests` to `/data/outputs/fim_unit_test_data_do_not_remove`. The new folder is intended to be a more permanent folder for unit test data.
+           - Some additional tests were added validating the argument for dropping stream orders.
+
+
+## Unit Test File Additions:
+
+- `unit_tests/`
+   - `filter_catchments_and_add_attributes_unittests.py` and `filter_catchments_and_add_attributes_params.json`:
+
+   - `split_flows_unittests.py' and `split_flows_params.json`
+
+
+<br/><br/>
+
 ## v4.0.3.1 - 2022-03-10 - [PR #561](https://github.com/NOAA-OWP/inundation-mapping/pull/561)
 
 Bug fixes to get the Alpha Test working in FIM 4.
@@ -52,12 +147,12 @@ Added a new optional system which allows an argument to be added to the `gms_run
 - `src/gms`
    - `run_by_unit.sh`: Capture and forward the drop stream orders flag to `derive_level_paths.py`
 	
-	- `derive_level_paths.py`: Capture the drop stream order flag and working with `stream_branches.py` to include/not include loading nwm stream with stream orders 1 and 2.
+   - `derive_level_paths.py`: Capture the drop stream order flag and working with `stream_branches.py` to include/not include loading nwm stream with stream orders 1 and 2.
 	
-	- `stream_branchs.py`: A correction was put in place to allow for the filter of branch attributes and values to be excluded. The `from_file` method has the functionality but was incomplete. This was corrected and how could accept the values from `derive_level_paths.py` to use the branch attribute of "order_" (gkpg field) and values excluded of [1,2] when optionally desired.
+   - `stream_branchs.py`: A correction was put in place to allow for the filter of branch attributes and values to be excluded. The `from_file` method has the functionality but was incomplete. This was corrected and how could accept the values from `derive_level_paths.py` to use the branch attribute of "order_" (gkpg field) and values excluded of [1,2] when optionally desired.
 
 - `unit_tests/gms`
-	- `derive_level_paths_unittests.py` and `derive_level_paths_params.py`: Updated for testing for the new "drop stream orders 1 and 2" feature. Upgrades were also made to earlier existing incomplete test methods to test more output conditions.
+    - `derive_level_paths_unittests.py` and `derive_level_paths_params.py`: Updated for testing for the new "drop stream orders 1 and 2" feature. Upgrades were also made to earlier existing incomplete test methods to test more output conditions.
 	
 <br/><br/>
 
diff --git a/fim_run.sh b/fim_run.sh
index 6d4d58a7a..0b0b54cf1 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -135,7 +135,16 @@ elif [ -d "$outputRunDataDir" ] && [ -z "$overwrite" ] ; then
     echo "$runName data directories already exist. Use -o/--overwrite to continue"
     exit 1
 fi
+
 mkdir -p $outputRunDataDir/logs
+if [ ! -d "$outputRunDataDir/unit_errors" ]; then
+    mkdir -p $outputRunDataDir/unit_errors
+else
+    if [ $overwrite -eq 1 ]; then
+        rm -rf $outputRunDataDir/unit_errors/*.*
+        rm -rf $outputRunDataDir/logs/unit/*.*
+    fi
+fi
 
 ## RUN ##
 if [ -f "$hucList" ]; then
@@ -179,3 +188,11 @@ if [[ "$viz" -eq 1 ]]; then
     # aggregate outputs
     time python3 /foss_fim/src/aggregate_fim_outputs.py -d $outputRunDataDir -j 6
 fi
+
+## GET NON ZERO EXIT CODES ##
+# Needed in case aggregation fails, we will need the logs
+find $outputRunDataDir/logs/ -name "*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/unit_errors/non_zero_exit_codes.log"
+
+echo "================================================================================"
+echo "Unit processing is complete"
+echo
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 825e994f4..8acf58ddc 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -19,6 +19,9 @@ usage ()
     echo '                   (see config/deny_gms_branches_default.lst for a starting point)'
     echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
     echo '                     also accepted. HUCs must present in inputs directory.'
+	echo '  -s/--dropStreamOrders : If this flag is included, the system will leave out stream orders 1 and 2'
+	echo '                    at the initial load of the nwm_subset_streams'
+    echo
     exit
 }
 
@@ -61,6 +64,9 @@ in
         shift
         deny_gms_branches_list=$1
         ;;
+	 -s|--dropLowStreamOrders)
+		  dropLowStreamOrders=1
+		  ;;
     *) ;;
     esac
     shift
@@ -83,6 +89,10 @@ if [ -z "$retry" ]
 then
     retry=""
 fi
+if [ -z "$dropLowStreamOrders" ]
+then
+    dropLowStreamOrders=0
+fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
@@ -99,6 +109,7 @@ export deny_gms_branches_list=$deny_gms_branches_list
 logFile=$outputRunDataDir/logs/branch/summary_gms_branch.log
 export extent=GMS
 export overwrite=$overwrite
+export dropLowStreamOrders=$dropLowStreamOrders
 
 ## Check for run data directory ##
 if [ ! -d "$outputRunDataDir" ]; then 
@@ -123,17 +134,35 @@ fi
 if [ ! -d "$outputRunDataDir/logs/branch" ]; then
     mkdir -p $outputRunDataDir/logs/branch
 fi
+if [ ! -d "$outputRunDataDir/branch_errors" ]; then
+    mkdir -p "$outputRunDataDir/branch_errors"
+else
+    if [ $overwrite -eq 1 ]; then
+        rm -rf $outputRunDataDir/branch_errors/
+    fi
+fi
 
 ## RUN GMS BY BRANCH ##
+echo "================================================================================"
+echo "Start of branch processing"
 if [ "$jobLimit" -eq 1 ]; then
     parallel $retry --verbose --timeout $branch_timeout --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
 else
     parallel $retry --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
 fi
 
+# -------------------
+## GET NON ZERO EXIT CODES ##
+# Needed in case aggregation fails, we will need the logs
+find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log"
+
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
+# TODO: How do we skip aggregation if there is a branch error
+# maybe against the non_zero logs above
+echo 
+echo "Processing usgs gage aggregation"
 python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
 
-# -------------------
-## GET NON ZERO EXIT CODES ##
-grep -ER 'Exit status: [1-9]' $outputRunDataDir/logs/branch/*_branch_*.log > $outputRunDataDir/logs/branch/non_zero_exit_codes.log
+echo "================================================================================"
+echo "Branch processing is complete"
+echo
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index 857c41712..2ac411b18 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -4,7 +4,7 @@ usage ()
 {
     echo 'Produce GMS hydrofabric datasets for unit scale. Run after fim_run.sh but before gms_run_branch.sh'
     echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -c <config file> -n <run name> ]'
-    echo '  	 						 [OPT: -h -j <job limit>] -o -r -d <deny list file> -s ]'
+    echo '  	 				   [OPT: -h -j <job limit>] -o -r -d <deny list file> -s <drop stream orders 1 and 2>]'
     echo ''
     echo 'REQUIRED:'
     echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
@@ -20,8 +20,8 @@ usage ()
     echo '  -r/--retry      : retries failed jobs'
     echo '  -d/--denylist   : file with line delimited list of files in huc directories to remove upon completion'
     echo '                   (see config/deny_gms_unit_default.lst for a starting point)'
-	 echo '  -s/--dropStreamOrder_1_2 : If this flag is included, the system will leave out stream orders 1 and 2 at the very'
-	 echo	'                     top initial load of the nwm_subset_streams'
+	echo '  -s/--dropStreamOrder_1_2 : If this flag is included, the system will leave out stream orders 1 and 2'
+	echo '                    at the initial load of the nwm_subset_streams'
     exit
 }
 
@@ -145,6 +145,18 @@ fi
 if [ ! -d "$outputRunDataDir/logs/unit" ]; then
     mkdir -p $outputRunDataDir/logs/unit
 fi
+if [ ! -d "$outputRunDataDir/unit_errors" ]; then
+    mkdir -p $outputRunDataDir/unit_errors
+else
+    if [ $overwrite -eq 1 ]; then
+        rm -rf $outputRunDataDir/unit_errors/
+        rm -rf $outputRunDataDir/logs/unit/
+
+        # remove branch logs as well
+        rm -rf $outputRunDataDir/branch_errors/
+        rm -rf $outputRunDataDir/logs/branch/
+    fi
+fi
 
 # copy over config file
 cp -a $envFile $outputRunDataDir
@@ -164,11 +176,13 @@ else
     fi
  fi
 
-
 ## AGGREGATE BRANCH LISTS INTO ONE ##
 python3 $srcDir/gms/aggregate_branch_lists.py -l $hucList
 
-
 ## GET NON ZERO EXIT CODES ##
-# get positive non-zero exit codes 
-grep -ER 'Exit status: [1-9]' $outputRunDataDir/logs/unit/*_unit.log > $outputRunDataDir/logs/unit/non_zero_exit_codes.log
+# Needed in case aggregation fails, we will need the logs
+find $outputRunDataDir/logs/ -name "*_unit.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/unit_errors/non_zero_exit_codes.log"
+
+echo "================================================================================"
+echo "Unit processing is complete"
+echo
\ No newline at end of file
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index 2f4192662..6c9423ce3 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -4,27 +4,39 @@
 import geopandas as gpd
 import numpy as np
 import sys
+
 from utils.shared_variables import FIM_ID
 from utils.shared_functions import mem_profile
-
+from utils.fim_enums import FIM_system_exit_codes
 
 @mem_profile
-def filter_catchments_and_add_attributes(input_catchments_filename, input_flows_filename, output_catchments_filename, output_flows_filename, wbd_filename, huc_code):
+def filter_catchments_and_add_attributes(input_catchments_filename,
+                                         input_flows_filename,
+                                         output_catchments_filename,
+                                         output_flows_filename,
+                                         wbd_filename,
+                                         huc_code,
+                                         drop_stream_orders=False):
+
     input_catchments = gpd.read_file(input_catchments_filename)
     wbd = gpd.read_file(wbd_filename)
     input_flows = gpd.read_file(input_flows_filename)
 
     # filter segments within huc boundary
     select_flows = tuple(map(str,map(int,wbd[wbd.HUC8.str.contains(huc_code)][FIM_ID])))
-
-    if input_flows.HydroID.dtype != 'str': input_flows.HydroID = input_flows.HydroID.astype(str)
+    
+    if input_flows.HydroID.dtype != 'str':
+        input_flows.HydroID = input_flows.HydroID.astype(str)
     output_flows = input_flows[input_flows.HydroID.str.startswith(select_flows)].copy()
-    if output_flows.HydroID.dtype != 'int': output_flows.HydroID = output_flows.HydroID.astype(int)
+    
+    if output_flows.HydroID.dtype != 'int':
+        output_flows.HydroID = output_flows.HydroID.astype(int)
 
     if len(output_flows) > 0:
 
         # merges input flows attributes and filters hydroids
-        if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
+        if input_catchments.HydroID.dtype != 'int':
+            input_catchments.HydroID = input_catchments.HydroID.astype(int)
         output_catchments = input_catchments.merge(output_flows.drop(['geometry'],axis=1),on='HydroID')
 
         # filter out smaller duplicate features
@@ -44,11 +56,21 @@ def filter_catchments_and_add_attributes(input_catchments_filename, input_flows_
             output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
             output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
         except ValueError:
-
-            class NoFlowLinesInHUC(Exception):
-                pass
-
-            raise NoFlowLinesInHUC("There are no flowlines in the HUC. These should be better filtered in derive_level_paths.py")
+            if (drop_stream_orders):
+                # this is not an exception, but a custom exit code that can be trapped
+                print("There are no flowlines in the HUC after stream order filtering.")
+                sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+            else:
+                # if we are not dropping stream orders, then something is wrong
+                raise Exception("There are no flowlines in the HUC.")
+    else:
+        if (drop_stream_orders):
+            # this is not an exception, but a custom exit code that can be trapped
+            print("There are no flowlines in the HUC after stream order filtering.")
+            sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+        else:
+            # if we are not dropping stream orders, then something is wrong
+            raise Exception("There are no flowlines in the HUC.")
 
 
 if __name__ == '__main__':
@@ -61,6 +83,7 @@ class NoFlowLinesInHUC(Exception):
     parser.add_argument('-o', '--output-flows-filename', help='output-flows-filename', required=True)
     parser.add_argument('-w', '--wbd-filename', help='wbd-filename', required=True)
     parser.add_argument('-u', '--huc-code', help='huc-code', required=True)
+    parser.add_argument('-s', '--drop-stream-orders', help='Drop stream orders 1 and 2', type=int, required=False, default=False)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index 185c55fba..91cd063f4 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 
-from stream_branches import StreamNetwork
 import argparse
-from utils.shared_functions import get_fossid_from_huc8
 import geopandas as gpd
+import sys
 
+from stream_branches import StreamNetwork
+from utils.shared_functions import get_fossid_from_huc8
+from utils.fim_enums import FIM_system_exit_codes
 
 def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribute,
                        out_stream_network_dissolved=None, huc_id=None,
@@ -31,9 +33,25 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                  branch_id_attribute="order_",
                                                  values_excluded=[1,2]
                                                  )
+                                                 
+                                                 
     else:
         stream_network = StreamNetwork.from_file(filename=in_stream_network)
-                                                 
+
+    # if there are no reaches at this point (due to filtering if applicable)
+    if (len(stream_network) == 0) and (drop_low_stream_orders):
+        # This is technically not an error but we need to have it logged so the user know what
+        # happened to it and we need the huc to not be included in future processing. 
+        # We need it to be not included in the gms_input.csv at the end of the unit processing.
+        # Throw an exception with valid text. This will show up in the non-zero exit codes and explain why an error.
+        # Later, we can look at creating custom sys exit codes 
+        # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file or stream order filtering.")
+        print("Sorry, no branches exist and processing can not continue. This could be an empty file or stream order filtering.")
+        sys.exit(FIM_system_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
+
+    print("inside Derive. Should we be here?")
+
+
     inlets_attribute = 'inlet_id'
     outlets_attribute = 'outlet_id'
     outlet_linestring_index = -1
@@ -119,7 +137,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         catchments.to_file(catchments_outfile,index=False,driver='GPKG')
 
     # derive headwaters
-    if headwaters_outfile is not None:
+    if (headwaters_outfile is not None):
         headwaters = stream_network.derive_headwater_points_with_inlets(
                                                         fromNode_attribute=fromNode_attribute,
                                                         inlets_attribute=inlets_attribute,
diff --git a/src/gms/generate_branch_list.py b/src/gms/generate_branch_list.py
index 7eb6d2d30..100ef6bba 100755
--- a/src/gms/generate_branch_list.py
+++ b/src/gms/generate_branch_list.py
@@ -4,9 +4,10 @@
 from stream_branches import StreamNetwork
 import argparse
 
-def Generate_branch_list(stream_network_dissolved,branch_id_attribute,output_branch_list):
+def Generate_branch_list(stream_network_dissolved, branch_id_attribute, output_branch_list):
 
     # load stream network
+
     stream_network_dissolved = StreamNetwork.from_file( stream_network_dissolved,
                                                         branch_id_attribute=branch_id_attribute )
 
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index af2b5f344..cff4a97b3 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -37,7 +37,7 @@ fi
 mkdir -p $outputCurrentBranchDataDir
 
 ## START MESSAGE ##
-echo -e $startDiv$startDiv"Processing branch_id: $current_branch_id in HUC: $hucNumber ..."$stopDiv$stopDiv
+echo -e $startDiv"Processing branch_id: $current_branch_id in HUC: $hucNumber ..."$stopDiv
 
 ## CLIP RASTERS
 echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"$stopDiv
@@ -181,7 +181,7 @@ Tcount
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
-$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -ds $dropLowStreamOrders
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
@@ -219,12 +219,14 @@ gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIF
 Tcount
 
 ## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
-echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-[ -f $outputHucDataDir/LandSea_subset.gpkg ] && \
-gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
-Tcount
+if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
+    echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"$stopDiv
+    date -u
+    Tstart
+
+    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
+    Tcount
+fi
 
 ## POLYGONIZE REACH WATERSHEDS ##
 echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"$stopDiv
@@ -237,8 +239,8 @@ Tcount
 echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
-Tcount
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber -s $dropLowStreamOrders
+
 
 ## RASTERIZE NEW CATCHMENTS AGAIN ##
 echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"$stopDiv
@@ -300,4 +302,5 @@ if [ -f $deny_gms_branches_list ]; then
     Tcount
 fi
 
-
+echo -e $startDiv"End Processing $hucNumber $current_branch_id ..."
+echo
\ No newline at end of file
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 53f8b7daa..7f5223aba 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -8,6 +8,7 @@ hucNumber="$1"
 outputHucDataDir=$outputRunDataDir/$hucNumber
 outputBranchDataDir=$outputHucDataDir/branches
 
+
 ## huc data
 if [ -d "$outputHucDataDir" ]; then
     if [ $overwrite -eq 1 ]; then
@@ -34,6 +35,9 @@ input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
 input_bathy_bankfull=$inputDataDir/$bankfull_input_table
 
+## START MESSAGE ##
+echo -e $startDiv"Processing HUC: $hucNumber ..."$stopDiv
+
 # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
 if [[ $huc2Identifier == "04" ]] ; then
   input_LANDSEA=$input_GL_boundaries
@@ -68,6 +72,11 @@ echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv
 date -u
 Tstart
 $srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -s $dropLowStreamOrders
+
+# test if we received a non-zero code back from derive_level_paths.py
+subscript_exit_code=$?
+# we have to retrow it if it is not a zero (but it will stop further execution in this script)
+if [ $subscript_exit_code -ne 0 ]; then exit $subscript_exit_code; fi
 Tcount
 
 ## STREAM BRANCH POLYGONS
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index fda5ed195..cec199ba2 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -661,9 +661,15 @@ def dissolve_by_branch(self,branch_id_attribute='LevelPathI',attribute_excluded=
 
         # dissolve lines
         self['bids_temp'] = self.loc[:,branch_id_attribute].copy()
+
+        # ensure the new stream order has the order from it's highest child
+        max_stream_order = self.groupby(branch_id_attribute).max()['order_'].copy()        
+
         self = self.dissolve(by=branch_id_attribute)
         self.rename(columns={'bids_temp' : branch_id_attribute},inplace=True)
         
+        self["order_"] = max_stream_order.values
+
         # merges each multi-line string to a sigular linestring
         for lpid,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc="Merging mult-part geoms"):
             if isinstance(row.geometry,MultiLineString):
diff --git a/src/gms/time_and_tee_run_by_branch.sh b/src/gms/time_and_tee_run_by_branch.sh
index 2377fdb72..989c9da71 100755
--- a/src/gms/time_and_tee_run_by_branch.sh
+++ b/src/gms/time_and_tee_run_by_branch.sh
@@ -2,10 +2,28 @@
 
 echo 
 echo "================================================================================"
-echo "Start Processing HUC $1 - Branch $2"
 
-/usr/bin/time -v $srcDir/gms/run_by_branch.sh $1 $2 |& tee $outputRunDataDir/logs/branch/$1_gms_branch_$2.log
+/usr/bin/time -v $srcDir/gms/run_by_branch.sh $1 $2 |& tee $outputRunDataDir/logs/branch/$1_gms_branch_$2.log 
+
 #exit ${PIPESTATUS[0]}
+return_codes=( "${PIPESTATUS[@]}" )
+
+# we do this way instead of working directly with stderr and stdout
+# as they were messing with output logs which we always want.
+for code in "${return_codes[@]}"
+do
+    # Make an extra copy of the branch log in a new folder
+    # Note: It was tricky to load in the fim_enum into bash, so we will just 
+    # go with the code for now
+    if [ $code -eq 61 ]; then
+        echo
+        echo "***** Branch has no valid flowlines *****"
+    elif [ $code -ne 0 ]; then
+        echo
+        echo "***** An error has occured  *****"
+        cp $outputRunDataDir/logs/branch/$1_gms_branch_$2.log $outputRunDataDir/branch_errors
+    fi
+done
 
 echo "================================================================================"
 exit
diff --git a/src/gms/time_and_tee_run_by_unit.sh b/src/gms/time_and_tee_run_by_unit.sh
index 9ef8270c0..bae50e1ae 100755
--- a/src/gms/time_and_tee_run_by_unit.sh
+++ b/src/gms/time_and_tee_run_by_unit.sh
@@ -2,11 +2,28 @@
 
 echo 
 echo "================================================================================"
-echo "Start Processing HUC $1"
 
-# TODO: Jan 10, 2020: This rarely receives anything but an exit code of 1 (even if a huc fails). 
 /usr/bin/time -v $srcDir/gms/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/unit/$1_gms_unit.log 
 
-echo "================================================================================"
 #exit ${PIPESTATUS[0]}
+return_codes=( "${PIPESTATUS[@]}" )
+
+# we do this way instead of working directly with stderr and stdout
+# as they were messing with output logs which we always want.
+for code in "${return_codes[@]}"
+do
+    # Make an extra copy of the branch log in a new folder  if an error
+    # Note: It was tricky to load in the fim_enum into bash, so we will just 
+    # go with the code for now
+    if [ $code -eq 60 ]; then
+        echo
+        echo "***** Unit has no valid branches *****"
+    elif [ $code -ne 0 ]; then
+        echo
+        echo "***** An error has occured  *****"
+        cp $outputRunDataDir/logs/unit/$1_gms_unit.log $outputRunDataDir/unit_errors
+    fi
+done
+
+echo "================================================================================"
 exit
diff --git a/src/gms/toDo.md b/src/gms/toDo.md
index fe7e131bf..ebe514a1b 100644
--- a/src/gms/toDo.md
+++ b/src/gms/toDo.md
@@ -28,6 +28,5 @@
 - *convenience wrapper for gms_run...sh:* Make a convenience wrapper for gms_run_unit.sh and gms_run_branch.sh. Be mindful of the two different processing loads and expose two different job numbers to the user.
     - *Deny Listing for Units:* The files in the deny list for units is executed at the end of gms/run_by_unit.sh. This requires files used in run_by_branch.sh to be left while not necessary left behind. This should be moved later in the process possibly once the convenience wrapper is made.
 - *Update clip_vectors_to_wbd.py:* Clipping way too many vectors for GMS purposes. This creates extra processing and storage requirements.
-- *Small level paths just in the WBD area:* Small levelpaths just outside the wbd are being filtered out filter_catchments_and_add_attributes.py and lead to an exception. These cause known non-zero exit codes. To reduce non-zero exit codes to not known issues these levelpaths need to be removed in derive_level_paths.py. See 11140104 - 7057000003, 12020007 - 1475000002, 11110104 - 7756000488, 11110104 - 7756000505, 11090204 - 5128000164
 - *Stream order filtering:* You would likely gain computational time if you filtered out lower stream orders in the NWM for GMS input (creating a nwm_streams_gms.gpkg). By doing this you would need to mosaic with FR streams at the lower removed stream orders. GMS likely does very little at stream orders 1s and 2s and maybe even 3s and 4s.*
 - *Levees to run_by_unit*: burning in levees at the branch scale is likely less efficient than at the unit scale. Consider moving those two modules to the run_by_unit script for gms
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index b46f7ce9a..4bca99028 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -24,6 +24,7 @@ echo -e "memfree=$memfree"$stopDiv
 
 ## SET OUTPUT DIRECTORY FOR UNIT ##
 hucNumber="$1"
+# Test for standard errors
 outputHucDataDir=$outputRunDataDir/$hucNumber
 mkdir $outputHucDataDir
 
@@ -326,11 +327,14 @@ Tstart
 [ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ] && \
 python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputHucDataDir/gw_catchments_reaches.gpkg -f $outputHucDataDir/demDerived_reaches_split.gpkg -c $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -o $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
 
-if [[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ]] ; then
-  echo "EXIT FLAG!! (exit 65): No relevant streams within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
-  rm -rf $outputHucDataDir
-  exit 0
-fi
+# Tight now, while there is a system code enum in filter_catchments_and_add_attributes.py, 
+# it shoudl not happen at this time for standard fim_run.sh versions. 
+# This has been left in and commented for example purposes if other system codes are added.
+
+# test if we received a non-zero code back from filter_catchments_and_add_attributes.py
+#subscript_exit_code=$?
+# we have to retrow it if it is not a zero (but it will stop further execution in this script)
+#if [ $subscript_exit_code -ne 0 ]; then exit $subscript_exit_code; fi
 Tcount
 
 ## GET RASTER METADATA ## *****
diff --git a/src/split_flows.py b/src/split_flows.py
index 4dff74fe0..3c730c5b7 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -24,9 +24,20 @@
 import build_stream_traversal
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
+from utils.fim_enums import FIM_system_exit_codes
 
 @mem_profile
-def split_flows(max_length, slope_min, lakes_buffer_input, flows_filename, dem_filename, split_flows_filename, split_points_filename, wbd8_clp_filename, lakes_filename):
+def split_flows(max_length, 
+                slope_min, 
+                lakes_buffer_input, 
+                flows_filename,
+                dem_filename, 
+                split_flows_filename, 
+                split_points_filename, 
+                wbd8_clp_filename, 
+                lakes_filename,
+                drop_stream_orders=False):
+
     wbd = gpd.read_file(wbd8_clp_filename)
 
     toMetersConversion = 1e-3
@@ -34,9 +45,14 @@ def split_flows(max_length, slope_min, lakes_buffer_input, flows_filename, dem_f
     print('Loading data ...')
     flows = gpd.read_file(flows_filename)
 
-    if not len(flows) > 0:
-        print ("No relevant streams within HUC boundaries.")
-        sys.exit(0)
+    if (len(flows) == 0):
+        if (drop_stream_orders):
+            # this is not an exception, but a custom exit code that can be trapped
+            print("No relevant streams within HUC boundaries.")
+            sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+        else:
+            # if we are not dropping stream orders, then something is wrong
+            raise Exception("No flowlines exist.")
 
     wbd8 = gpd.read_file(wbd8_clp_filename)
     dem = rasterio.open(dem_filename,'r')
@@ -199,10 +215,21 @@ def split_flows(max_length, slope_min, lakes_buffer_input, flows_filename, dem_f
 
     if isfile(split_flows_filename):
         remove(split_flows_filename)
-    split_flows_gdf.to_file(split_flows_filename,driver=getDriver(split_flows_filename),index=False)
-
     if isfile(split_points_filename):
         remove(split_points_filename)
+
+    if (len(split_flows_gdf) == 0):
+        if (drop_stream_orders):
+            # this is not an exception, but a custom exit code that can be trapped
+            print("There are no flowlines after stream order filtering.")
+            sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+        else:
+            # if we are not dropping stream orders, then something is wrong
+            raise Exception("No flowlines exist.")
+    split_flows_gdf.to_file(split_flows_filename,driver=getDriver(split_flows_filename),index=False)
+
+    if len(split_points_gdf) == 0:
+        raise Exception("No points exist.")
     split_points_gdf.to_file(split_points_filename,driver=getDriver(split_points_filename),index=False)
 
 
@@ -219,6 +246,7 @@ def split_flows(max_length, slope_min, lakes_buffer_input, flows_filename, dem_f
     parser.add_argument('-p', '--split-points-filename', help='split-points-filename',required=True)
     parser.add_argument('-w', '--wbd8-clp-filename', help='wbd8-clp-filename',required=True)
     parser.add_argument('-l', '--lakes-filename', help='lakes-filename',required=True)
+    parser.add_argument('-ds', '--drop-stream-orders', help='Drop stream orders 1 and 2', type=int, required=False, default=False)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
diff --git a/src/time_and_tee_run_by_unit.sh b/src/time_and_tee_run_by_unit.sh
index 431beef97..1566e2d46 100755
--- a/src/time_and_tee_run_by_unit.sh
+++ b/src/time_and_tee_run_by_unit.sh
@@ -7,4 +7,28 @@ else
   /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log 
 fi
 
-exit ${PIPESTATUS[0]}
+#$exit ${PIPESTATUS[0]}
+
+return_codes=( "${PIPESTATUS[@]}" )
+
+# we do this way instead of working directly with stderr and stdout
+# as they were messing with output logs which we always want.
+for code in "${return_codes[@]}"
+do
+    # Make an extra copy of the  log in a new folder  if an error
+    # Note: It was tricky to load in the fim_enum into bash, so we will just 
+    # go with the code for now
+    if [ $code -eq 61 ]; then
+        echo
+        echo "***** HUC has no valid flowlines *****"  #(for fim_run this is invalid)
+    elif [ $code -ne 0 ]; then
+        echo
+        echo "***** An error has occured  *****"
+        cp $outputRunDataDir/logs/$1.log $outputRunDataDir/unit_errors
+        exit ${PIPESTATUS[0]}
+    fi
+done
+
+echo "================================================================================"
+exit
+
diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py
new file mode 100644
index 000000000..d545fdd23
--- /dev/null
+++ b/src/utils/fim_enums.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+from enum import Enum
+
+class FIM_system_exit_codes(Enum):
+    '''
+    This particular enum class allows for special system exit codes to be issued to tell different
+    code layers about what has happened. Currently, most of our code uses just sys.exit(0) and 
+    sys.exit(1)<which is an error>.
+    The list of enums lower (which can grow as needed) allows for more status to be return.
+    - Notes about system exit codes:
+        - Numerics of 0 to 255 are available.
+        - Some are already reserved can we can not use them for custom exit codes. Reserved codes are:
+            - 0: Success
+            - 1: Fail
+            - 2, 126, 127, 128, 130 and 255.
+            - see: https://tldp.org/LDP/abs/html/exitcodes.html
+        - More advanced combinations of codes can be used and we will keep it simple for now.
+    - Sample usage:
+        import utils/fim_enums
+        print(FIM_system_exit_codes.GMS_UNIT_NO_BRANCHES.value) -> 60   [this is used in gms/derive_level_paths.py]
+
+    - For more information : https://docs.python.org/3.11/howto/enum.html and https://docs.python.org/3/library/enum.html
+    '''
+
+    GMS_UNIT_NO_BRANCHES = 60
+    NO_FLOWLINES_EXIST = 61
+            
\ No newline at end of file
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
new file mode 100644
index 000000000..454a930b8
--- /dev/null
+++ b/unit_tests/filter_catchments_and_add_attributes_params.json
@@ -0,0 +1,12 @@
+{
+	"valid_data":
+	{
+		"input_catchments_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/gw_catchments_reaches_3246000305.gpkg",
+		"input_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_3246000305.gpkg",
+		"output_catchments_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/gw_catchments_reaches_filtered_addedAttributes_3246000305.gpkg",
+		"output_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_filtered_3246000305.gpkg",
+	    "wbd_filename": "/data/outputs/gms_example_unit_tests/02020005/wbd8_clp.gpkg",
+		"huc_code": "02020005",
+		"drop_stream_orders": false
+	}
+}
\ No newline at end of file
diff --git a/unit_tests/filter_catchments_and_add_attributes_unittests.py b/unit_tests/filter_catchments_and_add_attributes_unittests.py
new file mode 100644
index 000000000..07126629f
--- /dev/null
+++ b/unit_tests/filter_catchments_and_add_attributes_unittests.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import warnings
+import unittest
+
+import unit_tests_utils as helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import filter_catchments_and_add_attributes as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_filter_catchments_and_add_attributes(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+
+        params_file_path = '/foss_fim/unit_tests/filter_catchments_and_add_attributes_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_filter_catchments_and_add_attributes_success(self):
+
+        '''
+        The gw_catchments_reaches_filtered_addedAttributes_<branchID>.gpkg and
+        demDerived_reaches_split_filtered_<branchid>.gpkg should not exit prior to this test.
+        If the test is successful, these file will be created.
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        params = self.params["valid_data"].copy() 
+
+        # to setup the test, lets start by deleted the two expected output files to ensure
+        # that they are regenerated
+        if os.path.exists(params["output_catchments_filename"]):
+            os.remove(params["output_catchments_filename"])
+        if os.path.exists(params["output_flows_filename"]):
+            os.remove(params["output_flows_filename"])
+       
+        src.filter_catchments_and_add_attributes(input_catchments_filename = params["input_catchments_filename"],
+                                                input_flows_filename = params["input_flows_filename"],
+                                                output_catchments_filename = params["output_catchments_filename"],
+                                                output_flows_filename = params["output_flows_filename"],
+                                                wbd_filename = params["wbd_filename"],
+                                                huc_code = params["huc_code"],
+                                                drop_stream_orders = params["drop_stream_orders"])
+       
+        if os.path.exists(params["output_catchments_filename"]) == False:
+            raise Exception(params["output_catchments_filename"] + " does not exist")
+        if os.path.exists(params["output_flows_filename"]) == False:
+            raise Exception(params["output_flows_filename"] + " does not exist")
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/gms/derive_level_paths_params.json b/unit_tests/gms/derive_level_paths_params.json
index e26c7aadf..36b5a6cf9 100644
--- a/unit_tests/gms/derive_level_paths_params.json
+++ b/unit_tests/gms/derive_level_paths_params.json
@@ -1,18 +1,33 @@
 {
 	"valid_data":
 	{
-		"in_stream_network": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams.gpkg",
-		"out_stream_network": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
 		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/gms_example_unit_tests/05030104/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
-		"verbose": false,
+		"verbose": true,
 		"drop_low_stream_orders": false
-	}
+	},
+	
+	"dropped_stream_orders_no_branches_remaining":
+	{
+		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams.gpkg",
+		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
+		"branch_id_attribute": "levpa_id",
+		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"reach_id_attribute": "ID",
+		"verbose": true,
+		"drop_low_stream_orders": true
+	}	
 }
 
 
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
index bca18320a..c5e92e96e 100644
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -17,6 +17,7 @@
 sys.path.append('/foss_fim/src/gms/')
 import derive_level_paths
 import stream_branches
+from utils.fim_enums import FIM_system_exit_codes as fsec
 
 
 # NOTE: This goes directly to the function.
@@ -145,8 +146,7 @@ def test_Derive_level_paths_success_drop_low_stream_orders_not_submitted(self):
         
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")        
-        
-        
+                
 
     def test_Derive_level_paths_success_drop_low_stream_orders_is_true(self):
     
@@ -195,6 +195,40 @@ def test_Derive_level_paths_success_drop_low_stream_orders_is_true(self):
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")        
 
+
+    def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self):
+    
+        '''
+        This test includes a huc that after stream orders 1 and 2 are dropped, no 
+        branches are remaining (the original huc only had 1 and 2 reaches.
+
+        # expecting an exception with a specific note
+        # NOTE: this only works for a few hucs, one is 02030201
+
+        '''
+        
+        # makes output readability easier and consistant with other unit tests       
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["dropped_stream_orders_no_branches_remaining"].copy()
+
+        with self.assertRaises(SystemExit) as se:
+            derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                                out_stream_network = params["out_stream_network"],
+                                                branch_id_attribute = params["branch_id_attribute"],
+                                                out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                                headwaters_outfile = params["headwaters_outfile"],
+                                                catchments = params["catchments"],
+                                                catchments_outfile = params["catchments_outfile"],
+                                                branch_inlets_outfile = params["branch_inlets_outfile"],
+                                                reach_id_attribute = params["reach_id_attribute"],
+                                                verbose = params["verbose"],
+                                                drop_low_stream_orders=params["drop_low_stream_orders"])
+        self.assertEqual(se.exception.code, fsec.GMS_UNIT_NO_BRANCHES.value)
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")        
+
     
     # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
     def test_Derive_level_paths_invalid_input_stream_network(self):
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
new file mode 100644
index 000000000..1813f4222
--- /dev/null
+++ b/unit_tests/split_flows_params.json
@@ -0,0 +1,15 @@
+{
+	"valid_data":
+	{
+		"max_length": 1500,
+		"slope_min": 0.001,
+		"lakes_buffer_input": 20,
+		"flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_3246000305.shp",
+		"dem_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/dem_thalwegCond_3246000305.tif",
+	    "split_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_3246000305.gpkg",
+		"split_points_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_points_3246000305.gpkg",
+		"wbd8_clp_filename": "/data/outputs/gms_example_unit_tests/02020005/wbd8_clp.gpkg",
+		"lakes_filename": "/data/outputs/gms_example_unit_tests/02020005/nwm_lakes_proj_subset.gpkg",
+		"drop_stream_orders": false
+	}
+}
\ No newline at end of file
diff --git a/unit_tests/split_flows_unittests.py b/unit_tests/split_flows_unittests.py
new file mode 100644
index 000000000..0a32ab7ab
--- /dev/null
+++ b/unit_tests/split_flows_unittests.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import warnings
+import unittest
+
+import unit_tests_utils as helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import split_flows as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_split_flows(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+
+        params_file_path = '/foss_fim/unit_tests/split_flows_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_split_flows_success(self):
+
+        '''
+        The /data/outputs/gms_example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
+        /data/outputs/gms_example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
+        If the test is successful, these file will be created.
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        params = self.params["valid_data"].copy() 
+
+        # to setup the test, lets start by deleted the two expected output files to ensure
+        # that they are regenerated
+        if os.path.exists(params["split_flows_filename"]):
+            os.remove(params["split_flows_filename"])
+        if os.path.exists(params["split_points_filename"]):
+            os.remove(params["split_points_filename"])
+       
+        src.split_flows(max_length = params["max_length"],
+                        slope_min = params["slope_min"],
+                        lakes_buffer_input = params["lakes_buffer_input"],
+                        flows_filename = params["flows_filename"],
+                        dem_filename = params["dem_filename"],
+                        split_flows_filename = params["split_flows_filename"],
+                        split_points_filename = params["split_points_filename"],
+                        wbd8_clp_filename = params["wbd8_clp_filename"],
+                        lakes_filename = params["lakes_filename"],
+                        drop_stream_orders = params["drop_stream_orders"])
+       
+        if os.path.exists(params["split_flows_filename"]) == False:
+            raise Exception(params["split_flows_filename"] + " does not exist")
+        if os.path.exists(params["split_points_filename"]) == False:
+            raise Exception(params["split_points_filename"] + " does not exist")
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    

From 309b9ba9e11b159c36c3171f8b043362c94b8c6b Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 2 May 2022 16:15:32 -0500
Subject: [PATCH 137/359] [1pt] PR: New tools and adjustments for bulk gms
 processing for alpha tests (#587)

* A couple of new tools to create a list of hucs that have test case data and copy fim 3/4 folders tht match that huc list to another folder.

* Upgrade output prints for better traceability, plus changed aggregate_branch_lists.py to be re-usable as a stand alone command line tool that the copy_test_case_folders.py can use when isolating specific hucs for alpha tests.

* Update CHANGELOG.md
---
 docs/CHANGELOG.md                 |  25 +++++++
 gms_run_branch.sh                 |  28 ++++++--
 gms_run_unit.sh                   |  32 +++++++--
 src/gms/aggregate_branch_lists.py |  26 ++++---
 tools/copy_test_case_folders.py   | 115 ++++++++++++++++++++++++++++++
 tools/find_test_case_folders.py   |  77 ++++++++++++++++++++
 tools/run_test_case.py            |   2 +
 tools/synthesize_test_cases.py    |  52 ++++++++++++--
 8 files changed, 333 insertions(+), 24 deletions(-)
 create mode 100644 tools/copy_test_case_folders.py
 create mode 100644 tools/find_test_case_folders.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index dac104ad7..70e3fca68 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,31 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.4.1 - 2022-05-02 - [PR #587](https://github.com/NOAA-OWP/inundation-mapping/pull/587)
+
+While testing GMS against evaluation and inundation data, we discovered some challenges for running alpha testing at full scale. Part of it was related to the very large output volume for GMS which resulted in outputs being created on multiple servers and folders. Considering the GMS volume and processing, a tool was required to extract out the ~215 HUC's that we have evaluation data for. Next, we needed isolate valid HUC output folders from original 2,188 HUC's and its 100's of thousands of branches. The first new tool allows us to point to the `test_case` data folder and create a list of all HUC's that we have validation for.
+
+Now that we have a list of relavent HUC's, we need to consolidate output folders from the previously processed full CONUS+ output data. The new `copy_test_case_folders.py` tool extracts relavent HUC (gms unit) folders, based on the list created above, into a consolidated folder. The two tools combine result in significantly reduced overall processing time for running alpha tests at scale.
+
+`gms_run_unit.sh` and `aggregated_branch_lists.py` were adjusted to make a previously hardcoded file path and file name to be run-time parameters. By adding the two new arguments, the file could be used against the new `copy_test_case_folders.py`. `copy_test_case_folders.py` and `gms_run_unit.sh` can now call `aggregated_branch_lists.py` to create a key input file called `gms_inputs.csv` which is a key file required for alpha testing.
+
+A few other small adjustments were made for readability and traceability as well as a few small fixes discovered when running at scale.
+
+## Additions
+
+- `tools/find_test_case_folders.py`: A new tool for creating a list of HUC's that we have test/evaluation data for.
+- `tools/copy_test_case_folders.py`: A new tool for using the list created above, to scan through other fully processed output folders and extract only the HUC's (gms units) and it's branches into a consolidated folder, ready for alpha test processing (or other needs).
+
+## Changes
+
+- `src/gms/aggregate_branch_lists.py`: Adjusted to allow two previously hardcoded values to now be incoming arguments. Now this file can be used by both `gms_run_unit.sh` and `copy_test_case_folders.py`.
+- `tools/synthesize_test_cases.py`: Adjustments for readability and progress status. The embedded progress bars are not working and will be addressed later.
+- `tools/run_test_case.py`: A print statement was added to help with processing progess was added.
+- `gms_run_unit.sh`: This was adjusted to match the new input parameters for `aggregate_branch_lists.py` as well as additions for progress status. It now will show the entire progress period start datetime, end datetime and duration. 
+- `gms_run_branch.sh`: Also was upgraded to show the entire progress period start datetime, end datetime and duration.
+
+<br/><br/>
+
 ## v4.0.4.0 - 2022-04-12 - [PR #557](https://github.com/NOAA-OWP/inundation-mapping/pull/557)
 
 During large scale testing of the new **filtering out stream orders 1 and 2** feature [PR #548](https://github.com/NOAA-OWP/inundation-mapping/pull/548), a bug was discovered with 14 HUCS that had no remaining streams after removing stream orders 1 and 2. This resulted in a number of unmanaged and unclear exceptions. An exception may be still raised will still be raised in this fix for logging purposes, but it is now very clear what happened. Other types of events are logged with clear codes to identify what happened.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 8acf58ddc..fb043bb3a 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -133,7 +133,12 @@ fi
 # make log dir
 if [ ! -d "$outputRunDataDir/logs/branch" ]; then
     mkdir -p $outputRunDataDir/logs/branch
+elif [ $overwrite -eq 1 ]; then
+    # need to clean it out if we are overwriting
+    rm -rf $outputRunDataDir/logs/branch
+    mkdir -p $outputRunDataDir/logs/branch
 fi
+
 if [ ! -d "$outputRunDataDir/branch_errors" ]; then
     mkdir -p "$outputRunDataDir/branch_errors"
 else
@@ -145,16 +150,21 @@ fi
 ## RUN GMS BY BRANCH ##
 echo "================================================================================"
 echo "Start of branch processing"
+echo "Started: `date -u`" 
+
+## Track total time of the overall run
+T_total_start
+Tstart
+
 if [ "$jobLimit" -eq 1 ]; then
     parallel $retry --verbose --timeout $branch_timeout --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
 else
     parallel $retry --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
 fi
 
-# -------------------
-## GET NON ZERO EXIT CODES ##
-# Needed in case aggregation fails, we will need the logs
-find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log"
+echo "Branch processing is complete"
+Tcount
+date -u
 
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
 # TODO: How do we skip aggregation if there is a branch error
@@ -163,6 +173,14 @@ echo
 echo "Processing usgs gage aggregation"
 python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
 
+# -------------------
+## GET NON ZERO EXIT CODES ##
+# Needed in case aggregation fails, we will need the logs
+echo
+echo "Start non-zero exit code checking"
+find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
+
 echo "================================================================================"
-echo "Branch processing is complete"
+echo "GMS_run_branch complete"
+echo "Ended: `date -u`" 
 echo
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index 2ac411b18..9ea3da8fe 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -142,16 +142,22 @@ fi
 if [ ! -d $outputRunDataDir ]; then
     mkdir -p $outputRunDataDir
 fi
+
 if [ ! -d "$outputRunDataDir/logs/unit" ]; then
     mkdir -p $outputRunDataDir/logs/unit
+elif [ $overwrite -eq 1 ]; then
+    # clean it out if we are overwritting
+    rm -rf $outputRunDataDir/logs/unit/
+    mkdir -p $outputRunDataDir/logs/unit
 fi
+
 if [ ! -d "$outputRunDataDir/unit_errors" ]; then
     mkdir -p $outputRunDataDir/unit_errors
 else
     if [ $overwrite -eq 1 ]; then
         rm -rf $outputRunDataDir/unit_errors/
-        rm -rf $outputRunDataDir/logs/unit/
-
+        mkdir -p $outputRunDataDir/unit_errors
+        
         # remove branch logs as well
         rm -rf $outputRunDataDir/branch_errors/
         rm -rf $outputRunDataDir/logs/branch/
@@ -161,6 +167,15 @@ fi
 # copy over config file
 cp -a $envFile $outputRunDataDir
 
+## RUN GMS BY BRANCH ##
+echo "================================================================================"
+echo "Start of unit processing"
+echo "Started: `date -u`" 
+
+## Track total time of the overall run
+T_total_start
+Tstart
+
 ## GMS BY UNIT##
 if [ -f "$hucList" ]; then
     if [ "$jobLimit" -eq 1 ]; then
@@ -176,13 +191,20 @@ else
     fi
  fi
 
-## AGGREGATE BRANCH LISTS INTO ONE ##
-python3 $srcDir/gms/aggregate_branch_lists.py -l $hucList
+echo "Unit (HUC) processing is complete"
+Tcount
+date -u
 
 ## GET NON ZERO EXIT CODES ##
 # Needed in case aggregation fails, we will need the logs
-find $outputRunDataDir/logs/ -name "*_unit.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/unit_errors/non_zero_exit_codes.log"
+echo "Start of non zero exit codes check"
+find $outputRunDataDir/logs/ -name "*_unit.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/unit_errors/non_zero_exit_codes.log" &
+
+## AGGREGATE BRANCH LISTS INTO ONE ##
+echo "Start branch aggregation"
+python3 $srcDir/gms/aggregate_branch_lists.py -d $outputRunDataDir -f "gms_inputs.csv" -l $hucList
 
 echo "================================================================================"
 echo "Unit processing is complete"
+echo "Ended: `date -u`" 
 echo
\ No newline at end of file
diff --git a/src/gms/aggregate_branch_lists.py b/src/gms/aggregate_branch_lists.py
index eb03a5172..f83eb8c72 100755
--- a/src/gms/aggregate_branch_lists.py
+++ b/src/gms/aggregate_branch_lists.py
@@ -7,24 +7,29 @@
 from glob import glob
 
 
-def aggregate_inputs_for_gms(hucList):
-
+def aggregate_inputs_for_gms(huc_list, output_dir, output_file_name):
+
+    # bash will send huclist in as a colletion and not a string
+    if isinstance(huc_list, list):
+        huc_list_file = huc_list[0]
+    else:
+        huc_list_file = huc_list
+    print(huc_list_file)
     try:
-        hucList = pd.read_csv(hucList[0],header=None,dtype=str).loc[:,0].tolist()
+        huc_list = pd.read_csv(huc_list_file,header=None,dtype=str).loc[:,0].tolist()
     except FileNotFoundError:
         pass
 
-    hucList = set(hucList)
+    hucs = set(huc_list)
 
     # get branch lists
-    outputRunDataDir = environ['outputRunDataDir']
-    branch_id_files = glob(join(outputRunDataDir,'*','branch_id.lst'))
+    branch_id_files = glob(join(output_dir,'*','branch_id.lst'))
 
     all_huc_numbers,all_bids = [],[]
     for bid_file in branch_id_files:
         huc_number = bid_file.split('/')[-2]
         
-        if huc_number in hucList:
+        if huc_number in hucs:
             bids = pd.read_csv(bid_file,header=None).loc[:,0].tolist()
             huc_number_list = [huc_number] * len(bids)
 
@@ -36,15 +41,16 @@ def aggregate_inputs_for_gms(hucList):
                             'branch' : all_bids
                           })
     
-    output.to_csv(join(outputRunDataDir,'gms_inputs.csv'),index=False,header=False)
+    output.to_csv(join(output_dir, output_file_name),index=False,header=False)
 
 
 
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Aggregate GMS Inputs')
-    parser.add_argument('-l','--hucList', help='huc list', required=True,nargs='+')
-    #parser.add_argument('-a','--input-flows-fileName', help='DEM derived streams', required=True)
+    parser.add_argument('-d','--output_dir', help='output run data directory', required=True)
+    parser.add_argument('-f','--output_file_name', help='output file name', required=True)
+    parser.add_argument('-l','--huc_list', help='huc list', required=True,nargs='+')
 
     args = vars(parser.parse_args())
 
diff --git a/tools/copy_test_case_folders.py b/tools/copy_test_case_folders.py
new file mode 100644
index 000000000..fd65830fd
--- /dev/null
+++ b/tools/copy_test_case_folders.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import shutil
+import sys
+
+# importing python folders in other direcories
+sys.path.append('/foss_fim/src/gms/')
+import aggregate_branch_lists as agg
+
+def copy_folders(folder_name_list,
+                source_dir,
+                target_dir,
+                create_gms_input_list=False,
+                overwrite=False):
+
+    '''
+    Summary: Scans the source_directory looking for huc values from the huc list. Once found,
+        the huc directory is copied to the target directory. All recusive files/folders wil be copied.
+        
+        Line items each be on a new line  (ie "\n")
+        
+        ***** Note: The tool will still work, if the list is not a list of hucs. 
+              It can be just a list of folder names.
+    Input:
+        - folder_name_list: A file and path to a .txt or .lst file with a list of line delimited huc numbers.
+        - source_dir: The root folder where the huc (or named) folders reside.
+        - target_dir: The root folder where the huc folders will be copied to. Note. All contents of 
+             each huc folder, including branch folders if applicable, will be copied, in the extact
+             structure as the source directory. Note: The target folder need not pre-exist. 
+        - create_gms_input_list: If this flag is set to True, after coping the folders, the 
+            "aggregate_branch_lists.py" file will be called in order to make the gms_input.csv file.
+            The gms_input.csv is required for futher processing such as reprocessing branchs or set up
+            for test cases.
+        - overwrite:  if this value is set to true, the entire target_directory will be emptied of its
+             contents as this process starts if the folder exists. 
+    Output:
+        - A copy of huc directories (or named folders) as listed in the folder_name_list.
+    '''
+
+    if (not os.path.exists(folder_name_list)):
+        raise FileNotFoundError(f"Sorry. The file {folder_name_list} does not exist")
+
+    if (not os.path.exists(source_dir)):
+        raise NotADirectoryError(f"Sorry. Source folder of {source_dir} does not exist")
+
+    with open(folder_name_list, "r") as fp:
+        raw_folder_names = fp.read().split("\n")
+
+    # split on new line can add an extra row of a blank value if a newline char exists on the end.
+    # Some lines may have extra spaces, or dups. It is ok if the value is not necessarily a huc
+    folder_names = set() # this will ensure unique names
+    for folder_name in raw_folder_names:
+        folder_name = folder_name.strip()
+        if (folder_name) != '':
+            folder_names.add(folder_name)
+
+    sorted_folder_names = sorted(folder_names)
+
+    print(f"{str(len(sorted_folder_names))} folders to be copied")
+    ctr = 0
+    
+    for folder_name in sorted_folder_names:
+        src_folder = os.path.join(source_dir, folder_name)
+        if not os.path.exists(src_folder):
+            print(f"source folder not found: {src_folder}")
+        else:
+            target_folder = os.path.join(target_dir, folder_name)
+            print(f"coping folder : {target_folder}")
+            shutil.copytree(src_folder, target_folder, dirs_exist_ok=True)
+            ctr+=1
+
+    print(f"{str(ctr)} folders have been copied to {target_dir}")
+
+    if create_gms_input_list == True:
+        # call this code, which scans each huc (unit) directory looking for the branch_id.lst
+        # and adds them together to create the gms_inputs.csv file
+        # Note: folder_name_list needs to be a huc list to work)
+        agg.aggregate_inputs_for_gms(folder_name_list, target_dir, "gms_inputs.csv")
+        print("gms_inputs.csv created")
+    
+
+if __name__ == '__main__':
+
+# Remember: This is for pulling out only folders that are related to a huc list (such as an alpha test list)
+#   and it has to be run on each root folder, one at a time (for now. aka.. no wildcards)
+
+# Sample Usage: 
+#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_22020420.lst -s /outputs/rob_gms_test_synth/ -t /data/outputs/rob_gms_test_synth_combined -a
+
+#  NOTE the 'a' at the end meaning go ahead create the gms_input.csv. This is normally
+# left for the last folder to be copied over.
+
+    parser = argparse.ArgumentParser(description='Based on the huc list parameter, ' \
+                        'find and copy the full huc (or other) directories.')
+
+    parser.add_argument('-f','--folder_name_list',
+                            help='file and path to the huc list. Note: The list does not ' \
+                                'necessarily be a list of huc, just a list of unique values',
+                            required=True)                            
+    parser.add_argument('-s','--source_dir', 
+                            help='Source folder to be scanned for unique folders',
+                            required=True)
+    parser.add_argument('-t','--target_dir',
+                            help='Target folder where the folders will be copied to',
+                            required=True)
+
+    parser.add_argument('-a','--create_gms_input_list',
+                            help='Create the gms_input.csv list after coping',
+                            required=False, default=False, action='store_true')
+
+    args = vars(parser.parse_args())
+
+    copy_folders(**args)
diff --git a/tools/find_test_case_folders.py b/tools/find_test_case_folders.py
new file mode 100644
index 000000000..c71d397e4
--- /dev/null
+++ b/tools/find_test_case_folders.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import re
+
+def create_huc_list(input_root_search_folder,
+                    output_path,
+                    overwrite=False):
+
+    '''
+    Summary: This scans an input directory, such as test_cases, and looks for each
+         unique huc value through those folders. This gives us a single list of all hucs
+         that can be used for alpha or sierra tests. 
+         This looks for first level folders using the following convention:
+            - *_test_cases  (ie usgs_test_cases)
+                Then subfolders using the following convention:
+                - {8 numbers}_{3 to 7 characters}. ie) 01080005_usgs
+    Input:
+        - input_root_search_folder: A fully qualified root path (relative to Docker pathing) to the folder that all hucs
+            are in.
+        - output_path: a path and file name (preferred as .lst or .txt) where the list of line delimited hucs
+            will be copied. The file and path do not need to pre-exist.
+        - overwrite: If the file exists and the overwrite flag is false, an error will be issued. Else, it will
+            be fully overwritten.
+    Output:
+        - a line delimited list of all huc numbers that have test cases available.
+    '''
+
+    if (not os.path.exists(input_root_search_folder)):
+        raise NotADirectoryError(f"Sorry. Search_folder of {input_root_search_folder} does not exist")
+
+    if (os.path.exists(output_path) and not overwrite):
+        raise Exception(f"Sorry. The file {output_path} already exists. Use 'overwrite' argument if desired.")
+
+    hucs = set()
+
+    for test_case in [test_case for test_case in os.listdir(input_root_search_folder) 
+        if re.match('.*_test_cases', test_case)]:
+            for test_id in [test_id for test_id in os.listdir(os.path.join(input_root_search_folder, test_case))
+                if re.match('\d{8}_\w{3,7}', test_id)]:
+                    hucs.add(test_id[:8])
+
+    sorted_hucs = sorted(hucs)
+    #print(sorted_hucs)
+    print(f"{str(len(sorted_hucs))} hucs found")
+    
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+    # Save to disk
+    with (open(output_path, 'w')) as output_file:
+        #output_file.writelines(sorted_hucs)
+        for huc in sorted_hucs:
+            output_file.write(huc)
+            output_file.write('\n')
+
+    print(f"huc list saved at {output_path}")
+
+
+if __name__ == '__main__':
+
+# Sample Usage: python /foss_fim/tools/find_test_case_hucs.py -d /data/test_cases/ -f /some_directory/huc_list_for_tests_22020420.lst -o
+
+    parser = argparse.ArgumentParser(description='Finds all unique hucs that have test case data.')
+    parser.add_argument('-d','--input_root_search_folder', 
+                            help='Root folder to be scanned for unique hucs that have test case',
+                            required=True)
+    parser.add_argument('-f','--output_path',
+                            help='Folder path and file name to be saved (.txt or .lst suggested).',
+                            required=True)
+    parser.add_argument('-o','--overwrite', 
+                            help='Overwrite the file if already existing? (default false)',
+                            action='store_true' )
+
+    args = vars(parser.parse_args())
+
+    create_huc_list(**args)
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 49891828d..6df9710ca 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -36,6 +36,8 @@ def run_alpha_test( fim_run_dir, version, test_id, magnitude,
     # make bool
     calibrated = bool( calibrated )
 
+    print(f"run_alpha_test: test_id = {test_id}")
+
     if (model == "MS") & (fr_run_dir is None):
         raise ValueError("fr_run_dir argument needs to be specified with MS model")
 
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 09f08d777..07c3aa3e8 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -2,6 +2,7 @@
 
 import os
 import argparse
+from datetime import datetime
 from multiprocessing import Pool
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 import json
@@ -72,6 +73,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                         'masked_area_km2'
                         ]
 
+
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
     list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
 
@@ -217,8 +219,30 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         csv_writer.writerows(list_to_write)
 
 
+
 if __name__ == '__main__':
 
+    # Sample usage:
+    
+    #python /foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v gms_test_synth_combined -jh 2 -jb 40 -m /outputs/gms_test_synth_combined/gms_synth_metrics.csv -vg -o
+
+    # Notes:
+    #   - gms_input.csv MUST be in the folder suggested.
+    #   - the -v param is the name in the folder in the "outputs/" directory where the test hucs are at.
+    #       It also becomes the folder names inside the test_case folders when done.
+    #   - the -vg param may not be working (will be assessed better on later releases).
+    #   - Find a balance between -jh (number of jobs for hucs) versus -jb (number of jobs for branches)
+    #     on quick tests on a 96 core machine, we tried [1 @ 80], [2 @ 40], and [3 @ 25] (and others).
+    #   -jb 3 -jh 25 was noticably better. You can likely go more jb cores with better success, just
+    #     experiment.  Start times, End Times and duration are now included.
+    #   - The -m can be any path and any name.
+    
+    # To see your outputs in the test_case folder (hard coded path), you can check for outputs using
+    #     (cd .... to your test_case folder), then command becomes  find . -name gms_test_* -type d (Notice the
+    #     the -name can be a wildcard for your -v param (or the whole -v value))
+    # If you want to delete the test outputs, test the outputs as suggest immediately above, but this time your
+    #     command becomes:  find . -name gms_test_* -type d  -exec rm -rdf {} +
+
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
     parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=False,default='DEV')
@@ -253,6 +277,14 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
     verbose = args['verbose']
     gms_verbose = args['gms_verbose']
 
+
+    print("================================")
+    print("Start synthesize test cases")
+    start_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"started: {dt_string}")
+    print()
+    
     # check job numbers
     total_cpus_requested = job_number_huc * job_number_branch
     total_cpus_available = os.cpu_count() - 1
@@ -312,6 +344,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                 if current_benchmark_category in bench_cat:
                     # Loop through versions.
                     for version in previous_fim_list:
+                        
                         if config == 'DEV':
                             fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
                         elif config == 'PREV':
@@ -329,7 +362,6 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                             # If a user supplies a special_string (-s), then add it to the end of the created dirs.
                             if special_string != "":
                                 version = version + '_' + special_string
-
             
                             # Define the magnitude lists to use, depending on test_id.
                             if 'ble' == current_benchmark_category:
@@ -357,7 +389,6 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                 'verbose': False,
                                                 'gms_verbose': False
                                               }
-                            
                             procs_dict[test_id] = alpha_test_args
 
     if job_number_huc == 1:
@@ -367,7 +398,6 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
         for current_huc, alpha_test_args in tqdm(procs_dict.items(),total=number_of_hucs,disable=(not verbose_by_huc)):
             alpha_test_args.update({'gms_verbose': not verbose_by_huc})
-
             try:
                 run_alpha_test(**alpha_test_args)
             except Exception as exc:
@@ -415,4 +445,18 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         print("Creating master metrics CSV...")
 
         # this function is not compatible with GMS
-        create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, dev_versions_to_include_list=dev_versions_to_include_list)
+        create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, 
+                                  dev_versions_to_include_list=dev_versions_to_include_list)
+    
+
+    print("================================")
+    print("End synthesize test cases")
+
+    end_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"ended: {dt_string}")
+
+    # calculate duration
+    time_duration = end_time - start_time
+    print(f"Duration: {str(time_duration).split('.')[0]}")
+    print()

From cf2038e6dd1ff388e2b8416c0bf39208f6a2c6fb Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 5 May 2022 09:14:25 -0500
Subject: [PATCH 138/359] Update README.md (#589)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 69b280b63..bdd336e05 100644
--- a/README.md
+++ b/README.md
@@ -137,7 +137,7 @@ Please see the issue tracker on GitHub and the [Inundation Mapping Wiki](https:/
 
 ### Getting Involved
 
-NOAA's National Water Center welcomes anyone to contribute to the Inundation Mapping repository to improve flood inundation mapping capabilities. Please contact Brad Bates (bradford.bates@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
+NOAA's National Water Center welcomes anyone to contribute to the Inundation Mapping repository to improve flood inundation mapping capabilities. Please contact Carson Pruitt (carson.pruitt@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) to get started.
 
 ### Open Source Licensing Info
 1. [TERMS](docs/TERMS.md)

From b56f79fa66eec97509f56c5140d7018688eaccac Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Tue, 10 May 2022 16:28:35 -0500
Subject: [PATCH 139/359] [1pt] PR: Hotfixes and updates to FIM4 alpha eval and
 eval plots (#594)

* Hotfixes to resolve issues or missing eval metrics
- Updated IFC magnitudes list variable (previously not used)
- Copied FIM3 version of the workflow to assign FR, MS, COMP resolution variable
- Added PND metric to calculations and output csv file
- Added 'ras2fim' benchmark variable to list of data to evaluate

* Updated magnitude dictionary lookup code
- Added 'ras2fim' magnitude list to `tools_shared_variables.py`

* Added re import

* Copied Carson's eval_plots.py changes

* Replaced deprecated parameter name for matplotlib grid()

* Added Carson's latest eval_plots.py revisions to hotfix

* Added PND plots to eval_plots.py

* Updated changelog
---
 docs/CHANGELOG.md               | 13 +++++++++
 tools/eval_plots.py             | 52 +++++++++++++++++++++------------
 tools/synthesize_test_cases.py  | 41 ++++++++++----------------
 tools/tools_shared_functions.py |  6 ++++
 tools/tools_shared_variables.py |  5 ++--
 5 files changed, 71 insertions(+), 46 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 70e3fca68..73b953b68 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.4.2 - 2022-05-03 - [PR #594](https://github.com/NOAA-OWP/inundation-mapping/pull/594)
+
+This hotfix includes several revisions needed to fix/update the FIM4 area inundation evaluation scripts. These changes largely migrate revisions from the FIM3 evaluation code to the FIM4 evaluation code.
+
+## Changes
+
+- `tools/eval_plots.py`: Copied FIM3 code revisions to enable RAS2FIM evals and PND plots. Replaced deprecated parameter name for matplotlib grid()
+- `tools/synthesize_test_cases.py`: Copied FIM3 code revisions to assign FR, MS, COMP resolution variable and addressed magnitude list variable for IFC eval
+- `tools/tools_shared_functions.py`: Copied FIM3 code revisions to enable probability not detected (PND) metric calculation
+- `tools/tools_shared_variables.py`: Updated magnitude dictionary variables for RAS2FIM evals and PND plots
+
+<br/><br/>
+
 ## v4.0.4.1 - 2022-05-02 - [PR #587](https://github.com/NOAA-OWP/inundation-mapping/pull/587)
 
 While testing GMS against evaluation and inundation data, we discovered some challenges for running alpha testing at full scale. Part of it was related to the very large output volume for GMS which resulted in outputs being created on multiple servers and folders. Considering the GMS volume and processing, a tool was required to extract out the ~215 HUC's that we have evaluation data for. Next, we needed isolate valid HUC output folders from original 2,188 HUC's and its 100's of thousands of branches. The first new tool allows us to point to the `test_case` data folder and create a list of all HUC's that we have validation for.
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index be7fdcc71..a9a35293d 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -86,7 +86,7 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
             elif 'fim_2' in label:
                 label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
             elif 'fim_3' in label and len(label) < 20:
-                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                label_dict[label] = re.split('_fr|_ms|_comp', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
             else:
@@ -94,7 +94,7 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
         #Define simplified labels as a list.
         new_labels = [label_dict[label] for label in org_labels]
         #Define legend location. FAR needs to be in different location than CSI/POD.
-        if y_field == 'FAR':
+        if y_field in ['FAR', 'PND']:
             legend_location = 'upper right'
         else:
             legend_location = 'lower left'
@@ -151,7 +151,7 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
     #Set xticks and yticks and background horizontal line.
     axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
     axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
-    axes.grid(b=True, which='major', axis='both')
+    axes.grid(visible=True, which='major', axis='both')
 
     #Set sizes of ticks and legend.
     axes.tick_params(labelsize = 'xx-large')
@@ -253,7 +253,7 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
             elif 'fim_2' in label:
                 label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
             elif 'fim_3' in label and len(label) < 20:
-                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                label_dict[label] = re.split('_fr|_ms|_comp', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
             else:
@@ -339,7 +339,7 @@ def filter_dataframe(dataframe, unique_field):
 ##############################################################################
 #Main function to analyze metric csv.
 ##############################################################################
-def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , spatial = False, fim_1_ms = False, site_barplots = False):
+def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR','PND'] , spatial = False, fim_1_ms = False, site_barplots = False):
 
     '''
     Creates plots and summary statistics using metrics compiled from
@@ -487,7 +487,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
 
         # If source is 'ble', set base_resolution and append ble dataset to all_datasets dictionary
-        elif benchmark_source in ['ble', 'ifc']:
+        elif benchmark_source in ['ble', 'ifc', 'ras2fim']:
 
             # Set the base processing unit for ble runs
             base_resolution = 'huc'
@@ -514,6 +514,9 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         elif dataset_name == 'ifc':
             magnitude_order = ['2yr','5yr','10yr','25yr','50yr','100yr','200yr','500yr']
             base_resolution = 'huc'
+        elif dataset_name == 'ras2fim':
+            magnitude_order = ['2yr','5yr','10yr','25yr','50yr','100yr']
+            base_resolution = 'huc'
         elif dataset_name in ['usgs','nws']:
             magnitude_order = ['action','minor','moderate','major']
             base_resolution = 'nws_lid'
@@ -597,14 +600,19 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         ###############################################################
         #This section will join ahps metrics to a spatial point layer
         ###############################################################
-        if all_datasets.get(('nws','MS')) and all_datasets.get(('usgs','MS')):
+        if (all_datasets.get(('nws','MS')) and all_datasets.get(('usgs','MS'))) or \
+            (all_datasets.get(('nws','COMP')) and all_datasets.get(('usgs','COMP'))): # export composite metrics to shp
             #Get point data for ahps sites
             #Get metrics for usgs and nws benchmark sources
-            usgs_dataset,sites = all_datasets.get(('usgs','MS'))
-            nws_dataset, sites = all_datasets.get(('nws','MS'))
+            try:
+                usgs_dataset,sites = all_datasets.get(('usgs','MS'))
+                nws_dataset, sites = all_datasets.get(('nws','MS'))
+            except TypeError: # Composite metrics
+                usgs_dataset,sites = all_datasets.get(('usgs','COMP'))
+                nws_dataset, sites = all_datasets.get(('nws','COMP'))
             #Append usgs/nws dataframes and filter unnecessary columns and rename remaining.
-            all_ahps_datasets = usgs_dataset.append(nws_dataset)
-            all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source'])
+            all_ahps_datasets = pd.concat([usgs_dataset, nws_dataset])
+            all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','benchmark_source'])
             all_ahps_datasets.rename(columns = {'benchmark_source':'source'}, inplace = True)
 
             #Get spatial data from WRDS
@@ -631,24 +639,32 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         ################################################################
         #This section joins ble (FR) metrics to a spatial layer of HUCs.
         ################################################################
-        if all_datasets.get(('ble','FR')) and all_datasets.get(('ifc','FR')):
+        if (all_datasets.get(('ble','FR')) and all_datasets.get(('ifc','FR')) and all_datasets.get(('ras2fim','FR'))) or \
+            (all_datasets.get(('ble','COMP')) and all_datasets.get(('ifc','COMP')) and all_datasets.get(('ras2fim','COMP'))):
             #Select BLE, FR dataset.
-            ble_dataset, sites = all_datasets.get(('ble','FR'))
-            ifc_dataset, sites = all_datasets.get(('ifc','FR'))
-            huc_datasets = ble_dataset.append(ifc_dataset)
+            try:
+                ble_dataset, sites = all_datasets.get(('ble','FR'))
+                ifc_dataset, sites = all_datasets.get(('ifc','FR'))
+                ras2fim_dataset, sites = all_datasets.get(('ras2fim','FR'))
+            except TypeError:
+                ble_dataset, sites = all_datasets.get(('ble','COMP'))
+                ifc_dataset, sites = all_datasets.get(('ifc','COMP'))
+                ras2fim_dataset, sites = all_datasets.get(('ras2fim','COMP'))
+
+            huc_datasets = pd.concat([ble_dataset, ifc_dataset, ras2fim_dataset])
             #Read in HUC spatial layer
             wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')
             #Join metrics to HUC spatial layer
             wbd_with_metrics = wbd_gdf.merge(huc_datasets, how = 'inner', left_on = 'HUC8', right_on = 'huc')
             #Filter out unnecessary columns
-            wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','benchmark_source','geometry'])
+            wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','benchmark_source','geometry'])
             wbd_with_metrics.rename(columns = {'benchmark_source':'source'}, inplace = True )
             #Project to VIZ projection
             wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION)
             #Write out to file
             wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
         else:
-            print('BLE/IFC FR datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a FR version')
+            print('BLE/IFC/RAS2FIM FR datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a FR version')
 #######################################################################
 if __name__ == '__main__':
     # Parse arguments
@@ -656,7 +672,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
     parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
     parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
-    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR'], required = False)
+    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR','PND'], required = False)
     parser.add_argument('-sp', '--spatial', help = 'If enabled, creates spatial layers with metrics populated in attribute table.', action = 'store_true', required = False)
     parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
     parser.add_argument('-i', '--site_plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 07c3aa3e8..33d3fe0d0 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -9,6 +9,7 @@
 import csv
 import ast
 from tqdm import tqdm
+import re
 
 from run_test_case import run_alpha_test
 
@@ -50,6 +51,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                         'FAR',
                         'TPR',
                         'TNR',
+                        'PND',
                         'PPV',
                         'NPV',
                         'ACC',
@@ -84,18 +86,12 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
     else:
         iteration_list = ['official']
 
-    for benchmark_source in ['ble', 'nws', 'usgs', 'ifc']:
+    for benchmark_source in ['ble', 'nws', 'usgs', 'ifc','ras2fim']:
         benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
-        if benchmark_source in ['ble', 'ifc']:
+        test_cases_list = [d for d in os.listdir(benchmark_test_case_dir) if re.match('\d{8}_\w{3,7}', d)]
+        if benchmark_source in ['ble', 'ifc','ras2fim']:
             
-            if benchmark_source == 'ble':
-                magnitude_list = MAGNITUDE_DICT['ble']
-            if benchmark_source == 'ifc':
-                magnitude_list = MAGNITUDE_DICT['ifc']
-            try:
-                test_cases_list = os.listdir(benchmark_test_case_dir)
-            except FileNotFoundError:
-                continue
+            magnitude_list = MAGNITUDE_DICT[benchmark_source]
             
             for test_case in test_cases_list:
                 try:
@@ -114,12 +110,12 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
                         for magnitude in magnitude_list:
                             for version in versions_to_aggregate:
-                                if '_fr' in version:
-                                    extent_config = 'FR'
-                                elif '_ms' in version:
+                                if '_ms' in version:
                                     extent_config = 'MS'
-                                else:
+                                elif ('_fr' in version) or (version == 'fim_2_3_3'):
                                     extent_config = 'FR'
+                                else:
+                                    extent_config = 'COMP'
                                 if "_c" in version and version.split('_c')[1] == "":
                                     calibrated = "yes"
                                 else:
@@ -169,12 +165,12 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
                         for magnitude in ['action', 'minor', 'moderate', 'major']:
                             for version in versions_to_aggregate:
-                                if '_fr' in version:
-                                    extent_config = 'FR'
-                                elif '_ms' in version:
+                                if '_ms' in version:
                                     extent_config = 'MS'
-                                else:
+                                elif ('_fr' in version) or (version == 'fim_2_3_3'):
                                     extent_config = 'FR'
+                                else:
+                                    extent_config = 'COMP'
                                 if "_c" in version and version.split('_c')[1] == "":
                                     calibrated = "yes"
                                 else:
@@ -364,14 +360,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                 version = version + '_' + special_string
             
                             # Define the magnitude lists to use, depending on test_id.
-                            if 'ble' == current_benchmark_category:
-                                magnitude = MAGNITUDE_DICT['ble']
-                            elif ('usgs' == current_benchmark_category) | ('nws' == current_benchmark_category):
-                                magnitude = ['action', 'minor', 'moderate', 'major']
-                            elif 'ifc' == current_benchmark_category:
-                                magnitude_list = MAGNITUDE_DICT['ifc']
-                            else:
-                                continue
+                            magnitude = MAGNITUDE_DICT[current_benchmark_category]
 
                             alpha_test_args = { 
                                                 'fim_run_dir': fim_run_dir, 
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 2f303d642..119aee6c5 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -291,6 +291,11 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
     except ZeroDivisionError:
         F1_score = "NA"
 
+    if TPR == 'NA':
+        PND = 'NA'
+    else:
+        PND = 1.0 - TPR  # Probability Not Detected (PND)
+
     stats_dictionary = {'true_negatives_count': int(true_negatives),
                         'false_negatives_count': int(false_negatives),
                         'true_positives_count': int(true_positives),
@@ -314,6 +319,7 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
                         'FAR': FAR,
                         'TPR': TPR,
                         'TNR': TNR,
+                        'PND': PND,
 
                         'PPV': PPV,
                         'NPV': NPV,
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index 0c31a6fb8..82d63d0b0 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -10,11 +10,12 @@
 BLE_MAGNITUDE_LIST = ['100yr', '500yr']
 IFC_MAGNITUDE_LIST = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr', '200yr', '500yr']
 AHPS_MAGNITUDE_LIST = ['action', 'minor', 'moderate', 'major']
+RAS2FIM_MAGNITUDE_LIST = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr']
 
-MAGNITUDE_DICT = {'ble': BLE_MAGNITUDE_LIST, 'ifc': IFC_MAGNITUDE_LIST, 'usgs': AHPS_MAGNITUDE_LIST, 'nws': AHPS_MAGNITUDE_LIST}
+MAGNITUDE_DICT = {'ble': BLE_MAGNITUDE_LIST, 'ifc': IFC_MAGNITUDE_LIST, 'usgs': AHPS_MAGNITUDE_LIST, 'nws': AHPS_MAGNITUDE_LIST, 'ras2fim': RAS2FIM_MAGNITUDE_LIST}
 PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
 GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
-GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
+GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc', 'PND']
 
 # Variables for eval_plots.py
 BAD_SITES = [

From 57f4a76e1fa0f6d4352332950a742f87b52bb69a Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Tue, 17 May 2022 10:34:53 -0500
Subject: [PATCH 140/359] Update to fix error related to old version of flask
 (#599)

---
 api/frontend/gui/requirements.txt   | 2 +-
 api/node/connector/connector.py     | 1 +
 api/node/connector/requirements.txt | 3 ++-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/api/frontend/gui/requirements.txt b/api/frontend/gui/requirements.txt
index a78a884a3..00b4e3b79 100644
--- a/api/frontend/gui/requirements.txt
+++ b/api/frontend/gui/requirements.txt
@@ -1,4 +1,4 @@
-flask==1.1.2
+flask==2.1.2
 flask-socketio==5.0.0
 gevent==20.9.0
 gunicorn==20.0.4
\ No newline at end of file
diff --git a/api/node/connector/connector.py b/api/node/connector/connector.py
index 117448d4b..677518676 100644
--- a/api/node/connector/connector.py
+++ b/api/node/connector/connector.py
@@ -224,4 +224,5 @@ def ws_cancel_job(job_params):
    
 
 if __name__ == '__main__':
+
     socketio.run(app, host="0.0.0.0", port="6000")
diff --git a/api/node/connector/requirements.txt b/api/node/connector/requirements.txt
index e0b0b19de..6eb15401b 100644
--- a/api/node/connector/requirements.txt
+++ b/api/node/connector/requirements.txt
@@ -1,3 +1,4 @@
-flask==1.1.2
+flask==2.1.2
+werkzeug==2.0.0
 flask-socketio==5.0.0
 eventlet==0.31.0

From 1820af99d91cede1de1b618fb5bca77bdd3e4b42 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 26 May 2022 15:17:06 -0500
Subject: [PATCH 141/359] [1pt] Mosaic inundation maps for FIM3 FR and FIM4 GMS
 SO 3+ (#605)

* WIP: basic inundation work, next is compositing

* WIP: (t5) inundation and mosaic of a single huc fully operational. Now on to multi hucs

* WIP: Fix masking error

* WIP: Most is working, just adding multi processing to hucs

* Completed: composite_inundation operational, with various adjustment to secondary files like inundate_gms.py and others

* updated pathing for the unit test file

* Update CHANGELOG.md
---
 docs/CHANGELOG.md                        |  23 ++
 src/utils/shared_functions.py            |  39 ++
 tools/composite_inundation.py            | 478 +++++++++++++++++++++++
 tools/composite_ms_fr_inundation.py      | 213 ----------
 tools/gms_tools/inundate_gms.py          | 157 ++++----
 tools/gms_tools/mosaic_inundation.py     | 144 ++++---
 tools/inundation.py                      | 110 ++++--
 tools/synthesize_test_cases.py           |  52 ++-
 unit_tests/gms/inundate_gms_params.json  |  16 +
 unit_tests/gms/inundate_gms_unittests.py |  91 +++++
 unit_tests/shared_functions_params.json  |  13 +
 unit_tests/shared_functions_unittests.py | 116 ++++++
 12 files changed, 1046 insertions(+), 406 deletions(-)
 create mode 100644 tools/composite_inundation.py
 delete mode 100644 tools/composite_ms_fr_inundation.py
 create mode 100644 unit_tests/gms/inundate_gms_params.json
 create mode 100644 unit_tests/gms/inundate_gms_unittests.py
 create mode 100644 unit_tests/shared_functions_params.json
 create mode 100644 unit_tests/shared_functions_unittests.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 73b953b68..74105580a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,29 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.4.3 - 2022-05-26 - [PR #605](https://github.com/NOAA-OWP/inundation-mapping/pull/605)
+
+We needed a tool that could composite / mosaic inundation maps for FIM3 FR and FIM4 / GMS with stream orders 3 and higher. A tool previously existed named composite_fr_ms_inundation.py and it was renamed to composite_inundation.py and upgraded to handle any combination of 2 of 3 items (FIM3 FR, FIM3 MS and/or FIM4 GMS).
+
+## Additions
+
+- `tools/composite_inundation.py`: Technically it is a renamed from composite_ms_fr_inundation.py, and is based on that functionality, but has been heavily modified. It has a number of options, but primarily is designed to take two sets of output directories, inundate the files, then composite them into a single mosiac'd raster per huc. The primary usage is expected to be compositing FIM3 FR with FIM4 / GMS with stream orders 3 and higher. 
+
+- `unit_tests/gms/inundate_gms_unittests.py and inundate_gms_params.json`: for running unit tests against `tools/gms_tools/inunundate_gms.py`.
+- `unit_tests/shared_functions_unittests.py and shared_functions_params.json`: A new function named `append_id_to_file_name_single_identifier` was added to `src/utils/shared_functions.py` and some unit tests for that function was created.
+
+## Removed
+
+- `tools/composite_ms_fr_inundation.py`: replaced with upgraded version named `composite_inundation.py`.
+
+## Changes
+
+- `tools/gms_tools/inundate_gms.py`: some style, readabilty cleanup plus move a function up to `shared_functions.py`.
+- `tools/gms_tools/mosaic_inundation.py`: some style, readabilty cleanup plus move a function up to `shared_functions.py`.
+- `tools/inundation.py`: some style, readabilty cleanup.
+- `tools/synthesize_test_cases.py`: was updated primarily for sample usage notes. 
+<br/><br/>
+
 ## v4.0.4.2 - 2022-05-03 - [PR #594](https://github.com/NOAA-OWP/inundation-mapping/pull/594)
 
 This hotfix includes several revisions needed to fix/update the FIM4 area inundation evaluation scripts. These changes largely migrate revisions from the FIM3 evaluation code to the FIM4 evaluation code.
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 01904a222..653ad0245 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -246,3 +246,42 @@ def wrapper(*args, **kwargs):
         else:
             func(*args, **kwargs)
     return wrapper
+
+def append_id_to_file_name(file_name, identifier):
+    '''
+    Processing:
+        Takes an incoming file name and inserts an identifier into the name
+        just ahead of the extension, with an underscore added.
+        ie) filename = "/output/myfolder/a_raster.tif"
+            indentifer = "13090001"
+            Becomes: "/output/myfolder/a_raster_13090001.tif"
+        Note: 
+            - Can handle a single identifier or a list of identifier
+              ie) identifier = ["13090001", "123000001"]
+              Becomes: "/output/myfolder/a_raster_13090001_123000001.tif"
+            - This allows for file name to not be submitted and will return None
+    Inputs:
+        file_name: a single file name
+        identifier: a value or list of values to be inserted with an underscore 
+            added ahead of the extention
+    Output:
+        out_file_name: A single name with each identifer added at the end before
+            the extension, each with an underscore in front of the identifier.
+    '''
+
+    if file_name is not None:
+
+        root,extension = os.path.splitext(file_name)
+
+        if isinstance(identifier, list):
+
+            out_file_name = root
+            for i in identifier:
+                out_file_name += "_{}".format(i)
+            out_file_name += extension
+        else:
+            out_file_name = root + "_{}".format(identifier) + extension
+    else:
+        out_file_name = None
+
+    return(out_file_name)
diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py
new file mode 100644
index 000000000..5fdfabcd7
--- /dev/null
+++ b/tools/composite_inundation.py
@@ -0,0 +1,478 @@
+#!/usr/bin/env python3
+import os, argparse, copy, sys
+import json
+import rasterio
+import numpy as np
+import pandas as pd
+
+from datetime import datetime
+from multiprocessing import Pool
+#from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
+import concurrent.futures as cf
+from tqdm import tqdm
+
+from inundation import inundate
+from gms_tools.mosaic_inundation import Mosaic_inundation
+from gms_tools.inundate_gms import Inundate_gms
+from utils.shared_functions import append_id_to_file_name
+from utils.shared_variables import elev_raster_ndv
+
+class InundateModel_HUC(object):
+
+    def __init__(self, model, source_directory, huc):
+
+        self.model = model
+        self.source_directory = source_directory
+        self.huc = huc        
+
+    def inundate_huc(self, flows_file, composite_output_dir, output_name, log_file_path,
+                     num_workers_branches, no_cleanup, verbose):
+        '''
+        Processing:
+            Will inundate a single huc directory and if gms, will create an aggregate mosiac per huc
+
+        Returns:
+            The map file of the inundated raster.
+        '''
+
+        source_huc_dir = os.path.join(self.source_directory, self.huc)
+
+        # hucs do not need to exist as the list might have one huc from one
+        # source directory that is not in another. TODO: how to log this.
+        if (not os.path.exists(source_huc_dir)): 
+            print(f"HUC {self.huc} does not exist in {self.source_directory}")
+            return None 
+
+        output_huc_dir = os.path.join(composite_output_dir, self.huc)
+        # Create output directory if it does not exist
+        if not os.path.isdir(output_huc_dir):
+            os.mkdir(output_huc_dir)
+
+        if self.model == "ms":
+            extent_friendly = "mainstem (MS)"
+        elif self.model == "fr":
+            extent_friendly = "full-resolution (FR)"
+        else: # gms
+            extent_friendly = "FIM4 GMS"
+
+        inundation_map_file = None
+
+        output_raster_name = os.path.join(output_huc_dir, output_name)
+        output_raster_name = append_id_to_file_name(output_raster_name, [self.huc,self.model])
+
+        # adjust to add model and huc number
+        log_file = None
+        inundation_list_file = None        
+        if (log_file_path != None):
+            log_file = os.path.join(log_file_path, f"{self.huc}_{self.model}_error_logs.txt") 
+            inundation_list_file = os.path.join(log_file_path, 
+                                               f"{self.huc}_{self.model}_inundation_file_list.csv") 
+
+        if (verbose):
+            print(f'... Creating an inundation map for the {extent_friendly}'\
+                    f' configuration for HUC {self.huc}...')
+
+        if self.model in ["fr", "ms"]:
+
+            rem = os.path.join(source_huc_dir, 'rem_zeroed_masked.tif')
+            catchments = os.path.join(source_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+            catchment_poly = os.path.join(source_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+            hydro_table = os.path.join(source_huc_dir, 'hydroTable.csv')
+
+            # Ensure that all of the required files exist in the huc directory
+            for file in (rem, catchments, catchment_poly, hydro_table):
+                if not os.path.exists(file):
+                    raise Exception(f"The following file does not exist within the supplied FIM directory:\n{file}")
+
+            # Run inundation()
+            # Must set workers to one as we are only processing one huc.
+            map_file = inundate(rem, 
+                                catchments, 
+                                catchment_poly, 
+                                hydro_table,
+                                flows_file, 
+                                mask_type = None, 
+                                num_workers = 1,
+                                inundation_raster = output_raster_name,
+                                quiet = not verbose)
+
+            #if verbose:
+            #    print("Inundation Response:")
+            #    print(map_file)
+
+            if len(map_file) == 0:
+                raise Exception(f"Failed to inundate {extent_friendly} using the provided flows.")
+
+            mosaic_file_path = map_file[0][0]
+            inundation_map_file = [self.model, self.huc, mosaic_file_path]            
+
+        else:  # gms
+            # we are doing each huc one at a time
+            map_file = Inundate_gms( hydrofabric_dir = self.source_directory,
+                                    forecast = flows_file, 
+                                    num_workers = num_workers_branches,
+                                    hucs = self.huc,
+                                    inundation_raster = output_raster_name,
+                                    verbose = verbose,
+                                    log_file = log_file,
+                                    output_fileNames = inundation_list_file )
+            
+            mask_path_gms = os.path.join(source_huc_dir, 'wbd.gpkg')
+
+            # we are going to mosaic the gms files per huc
+            # NOTE: Leave workers as 1, it fails to composite correctly if more than one.
+            mosaic_file_path = Mosaic_inundation( map_file, 
+                                                  mosaic_attribute = 'inundation_rasters',
+                                                  mosaic_output = output_raster_name,
+                                                  mask = mask_path_gms, 
+                                                  unit_attribute_name = 'huc8',
+                                                  nodata = elev_raster_ndv,
+                                                  workers = 1,
+                                                  remove_inputs = not no_cleanup,
+                                                  subset = None,
+                                                  verbose = verbose )
+
+            inundation_map_file = [self.model, self.huc, mosaic_file_path]
+            
+            if verbose: print(f"Inundation for HUC {self.huc} is complete")
+
+        return inundation_map_file
+
+class Composite_HUC(object):
+
+    # Composites two source directories for a single huc
+    # Note: The huc does not need to exist in both source directories
+    @classmethod
+    def composite_huc(self, args):
+
+        huc = args["current_huc"]
+        print(f"Processing huc {huc}")        
+        composite_model_map_files = []
+        for model in args["models"]:
+
+            # setup original fim/gms processed directory
+            if model == "ms" : source_dir = args["fim_dir_ms"]
+            elif model == "fr" : source_dir = args["fim_dir_fr"]
+            else: source_dir = args["gms_dir"]
+            
+            ci = InundateModel_HUC(model, source_dir, huc)
+            map_file = ci.inundate_huc(args["flows_file"],
+                                        args["composite_output_dir"], 
+                                        args["output_name"],
+                                        args["log_file_path"],
+                                        args["num_workers_branches"],
+                                        args["no_cleanup"],
+                                        args["verbose"])
+            if map_file is not None:
+                composite_model_map_files.append(map_file)
+
+        # Composite the two final model outputs
+        inundation_map_file_df = pd.DataFrame(composite_model_map_files,
+                                              columns = ['model', 'huc8', 'inundation_rasters'])
+
+        if args["verbose"]:
+            print("inundation_map_file_df")
+            print(inundation_map_file_df)
+
+        composite_file_output = os.path.join(args["composite_output_dir"], huc, args["output_name"])
+        composite_file_output = append_id_to_file_name(composite_file_output, huc)
+
+        # NOTE: Leave workers as 1, it fails to composite correctly if more than one.
+        #    - Also. by adding the is_mosaic_for_gms_branches = False, Mosaic_inudation
+        #      will not auto add the HUC into the output name (its default behaviour)
+        mosaic_file_path = Mosaic_inundation( inundation_map_file_df,
+                                            mosaic_attribute='inundation_rasters',
+                                            mosaic_output = composite_file_output,
+                                            mask = None,
+                                            unit_attribute_name = 'huc8',
+                                            nodata = elev_raster_ndv,
+                                            workers = 1,
+                                            remove_inputs = not args["no_cleanup"],
+                                            subset = None,
+                                            verbose = args["verbose"] )
+
+        if args["is_bin_raster"]:
+            if args["verbose"]: print("Converting to binary")
+            CompositeInundation.hydroid_to_binary(composite_file_output)
+
+
+class CompositeInundation(object):
+
+    @classmethod
+    def run_composite(self, args):
+
+        def __validate_args(args):
+
+            """
+            Runs `inundate()` on any two of the following:
+                1) FIM 3.X mainstem (MS)
+                2) FIM 3.X full-resolution (FR)
+                3) FIM 4.x (gms)
+                All three need to have outputs and composites results. Assumes that all products 
+            necessary for `inundate()` are in each huc8 folder.
+
+            Parameters
+            ----------
+            fim_dir_ms : str
+                Path to MS FIM directory. This should be an output directory from `fim_run.sh`.
+            fim_dir_fr : str
+                Path to FR FIM directory. This should be an output directory from `fim_run.sh`.
+            gms_dir : str
+                Path to FIM4 GMS directory. This should be an output directory from `gms_run_unit, then gms_run_branch`.
+            huc: str, optional
+                If this value comes in, it shoudl be a single huc value. If it does not exist,
+                we use all hucs in the give source directories.
+            flows_file : str : 
+                Can be a single file path to a forecast csv or a comma-separated list of files.
+            composite_output_dir : str
+                Folder path to write outputs. It will be created if it does not exist.
+            output_name : str, optional
+                Name for output raster. If not specified, by default the raster will be named 'inundation_composite_{flows_root}.tif'.
+            is_bin_raster : bool, optional
+                Flag to create binary raster as output.
+            num_workers_huc : int, optional
+                defaults to 1 and means the number of processes to be used for processing hucs
+            num_workers_branches : int, optional
+                defaults to 1 and means the number of processes to be used for processing gms branches
+            no_cleanup : bool, optional
+                If False, intermediate files are deleted.
+            verbose : bool, optional
+                show extra output.
+            """
+            
+            if (args["fim_dir_ms"]) and (args["fim_dir_ms"].lower() == "none"):
+                args["fim_dir_ms"] = None
+            if (args["fim_dir_fr"]) and (args["fim_dir_fr"].lower() == "none"):
+                args["fim_dir_fr"] = None        
+            if (args["gms_dir"]) and (args["gms_dir"].lower() == "none"):
+                args["gms_dir"] = None
+            
+            # count number of input dir types and ensure their are no duplciates.
+            dir_list_lowercase = [] # we use a forced lowercase to help ensure dups (might be mixed case)
+            dir_list_raw = []
+            missing_dir_msg = "{} directory of {} does not exist"    
+            args["models"] = []
+            if (args["fim_dir_ms"] != None):
+                args["models"].append("ms")
+                assert os.path.isdir(args["fim_dir_ms"]), missing_dir_msg.format("ms", args["fim_dir_ms"])
+                dir_list_raw.append(args["fim_dir_ms"])
+                dir_list_lowercase.append(args["fim_dir_ms"].lower())
+
+            if (args["fim_dir_fr"] != None):
+                args["models"].append("fr")
+                assert os.path.isdir(args["fim_dir_fr"]), missing_dir_msg.format("fr", args["fim_dir_fr"])
+                dir_list_raw.append(args["fim_dir_fr"])
+                dir_list_lowercase.append(args["fim_dir_fr"].lower())        
+
+            if (args["gms_dir"] != None):
+                args["models"].append("gms")
+                assert os.path.isdir(args["gms_dir"]), missing_dir_msg.format("gms", args["gms_dir"])
+                dir_list_raw.append(args["gms_dir"])
+                dir_list_lowercase.append(args["gms_dir"].lower())                
+
+            if (len(args["models"]) != 2):
+                raise ValueError("Must submit exactly two directories (ms, fr and/or gms")
+
+            # check for duplicate dir names (submitting same dir for two args)
+            if len(dir_list_lowercase) != len(set(dir_list_lowercase)):
+                raise ValueError("The two sources directories are the same path.")
+
+            if not os.path.exists(args["flows_file"]):
+                print(f'{args["flows_file"]} does not exist. Please specify a flow file.')
+
+            # check job numbers
+            assert args["num_workers_huc"] >= 1, "Number of huc workers should be 1 or greater"
+            assert args["num_workers_branches"] >= 1, "Number of branch workers should be 1 or greater"
+
+            total_cpus_requested = args["num_workers_huc"] * args["num_workers_branches"]
+            total_cpus_available = os.cpu_count()
+            if total_cpus_requested > (total_cpus_available - 1):
+                raise ValueError('The HUC job num of workers, {}, multiplied by the branch workers number, {}, '\
+                                'exceeds your machine\'s available CPU count of {} minus one. '\
+                                'Please lower the num_workers_huc or num_workers_branches'\
+                                'values accordingly.'.format(args["num_workers_huc"], 
+                                total_cpus_available, args["num_workers_branches"]) )
+            
+            # Create output directory if it does not exist
+            if not os.path.isdir(args["composite_output_dir"]):
+                os.mkdir(args["composite_output_dir"])
+
+            # If no output name supplied, create one using the flows file name 
+            # the output_name is the final composite output file.
+            # we also extract the basic file name without extension for use as the log file name
+            if not args["output_name"]:
+                flows_root = os.path.splitext(os.path.basename(args["flows_file"]))
+                root_output_file_name = f'inundation_composite_{flows_root[0]}'
+                args["output_name"] = f"{root_output_file_name}.tif"
+            else:
+                # see if the file name has a path or not, fail if it does
+                output_file_name_split = os.path.split(args["output_name"])
+                if args["output_name"] == output_file_name_split[0]:
+                    raise ValueError("""If submitting the -n (output file name), please ensure 
+                                        it has no pathing. You can also leave it blank if you like.""")
+
+                root_output_file_name = os.path.splitext(args["output_name"])[0]
+
+            # setup log file and its directory
+            # Note: Log files are only created at this time if verbose\
+            args["log_file_path"] = None
+            if (args["verbose"]):
+                args["log_file_path"] = os.path.join(args["composite_output_dir"], "mosaic_logs")
+                if not os.path.isdir(args["log_file_path"]):
+                    os.mkdir(args["log_file_path"])
+
+            # Save run parameters up to this point
+            args_file = os.path.join(args["composite_output_dir"], root_output_file_name + '_args.json')
+            with open(args_file, 'w') as json_file:
+                json.dump(args, json_file)
+                print(f"Args printed to file at {args_file}")
+
+            # make combined huc list, NOTE: not all source dirs will have the same huc folders
+            huc_list = set()
+            if (args["huc"] != None):
+                if (len(args["huc"]) != 8) or (not args["huc"].isnumeric()):
+                    raise ValueError("Single huc value (-u arg) was submitted but appears invalid")
+                else:
+                    huc_list.add(args["huc"])
+            else:
+                for dir in dir_list_raw:
+                    sub_dirs = [ item for item in os.listdir(dir) if os.path.isdir(os.path.join(dir, item)) ]
+                    # Some directories may not be hucs (log folders, etc)            
+                    huc_folders = [ item for item in sub_dirs if item.isnumeric() ]
+                    huc_set = set(huc_folders)
+                    huc_list.update(huc_set) # will ensure no dups
+
+            args["huc_list"] = huc_list
+
+            return args
+
+        args =  __validate_args(args)
+
+        huc_list = args["huc_list"] 
+        number_huc_workers = args["num_workers_huc"]
+        #if len(huc_list == 1): # skip iterator
+        if (number_huc_workers == 1):
+            for huc in sorted(huc_list):
+                Composite_HUC.composite_huc(huc, args)
+        else:
+
+            print(f"Processing {len(huc_list)} hucs")
+            args_list = []
+            #sorted_hucs = sorted(huc_list)
+            #params_items = [(args, huc) for huc in huc_list]
+            for huc in sorted(huc_list):
+                huc_args = copy.deepcopy(args)
+                huc_args["current_huc"] = huc
+                args_list.append(huc_args)
+
+            with cf.ProcessPoolExecutor(max_workers = number_huc_workers) as executor:
+                executor_gen = { executor.submit(Composite_HUC.composite_huc, params): params for params in args_list }
+
+                for future in tqdm(cf.as_completed(executor_gen),
+                                total = len(executor_gen),
+                                desc = f"Running composite inundation with {number_huc_workers} workers" ):
+
+                    executor_gen[future]
+
+                try:
+                    future.result()
+                except Exception as exc:
+                    print('{}, {}, {}'.format(hucCode,exc.__class__.__name__,exc))
+
+
+            print("All hucs have been processed")            
+
+    @staticmethod
+    def hydroid_to_binary(hydroid_raster_filename):
+        #Converts hydroid positive/negative grid to 1/0
+        #to_bin = lambda x: np.where(x > 0, 1, np.where(x == 0, -9999, 0))
+        to_bin = lambda x: np.where(x > 0, 1, np.where(x != -9999, 0, -9999))
+        hydroid_raster = rasterio.open(hydroid_raster_filename)
+        profile = hydroid_raster.profile # get profile for new raster creation later on
+        profile['nodata'] = -9999
+        bin_raster = to_bin(hydroid_raster.read(1)) # converts neg/pos to 0/1
+        # Overwrite inundation raster
+        with rasterio.open(hydroid_raster_filename, "w", **profile) as out_raster:
+            out_raster.write(bin_raster.astype(hydroid_raster.profile['dtype']), 1)
+        del hydroid_raster,profile,bin_raster
+    
+
+
+if __name__ == '__main__':
+    
+    '''
+        Runs inundate and compositing on any and exactly two of the following:
+            1) FIM 3.X mainstem (MS)
+            2) FIM 3.X full-resolution (FR)
+            3) FIM 4.x (gms)
+
+        Examples of usage
+        Notice: arg keys and values for some of the variants
+        --------
+        a) ms and fr (single huc)
+        python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -fr /outputs/inundation_test_1_FIM3_fr -u /data/inputs/huc_lists/include_huc8.lst -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
+
+        a) ms and gms (all hucs in each folder)
+        python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -gms /outputs/inundation_test_1_gms -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -c -jh 3 -jb 20
+
+        b) fr and gms (single huc)
+        python3 /foss_fim/tools/composite_inundation.py -fr /outputs/inundation_test_1_FIM3_fr -gms /outputs/inundation_test_1_gms -u 13090001 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
+    '''
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description="""Inundate FIM 3 full resolution
+                and mainstem outputs using a flow file and composite the results.""")
+    parser.add_argument('-ms','--fim-dir-ms', help='Source directory that contains MS FIM outputs.',
+                        required=False, default=None)
+    parser.add_argument('-fr','--fim-dir-fr', help='Source directory that contains FR FIM outputs.', 
+                        required=False, default=None)
+    parser.add_argument('-gms','--gms-dir', help='Source directory that contains FIM4 GMS outputs.', 
+                        required=False, default=None)
+    parser.add_argument('-u','--huc', 
+                        help="""(Optional) If a single HUC is provided, only that HUC will be processed.
+                        If not submitted, all HUCs in the source directories will be used.""",
+                        required=False, default=None)
+    parser.add_argument('-f','--flows-file', 
+                        help='File path of flows csv.',
+                        required=True)
+    parser.add_argument('-o','--composite-output-dir', help='Folder to write Composite Raster output.',
+                        required=True)
+    parser.add_argument('-n','--output-name', help='File name for output(s).', 
+                        default=None, required=False)
+    parser.add_argument('-b','--is-bin-raster', 
+                        help="If flag is included, the output raster will be changed to wet/dry.", 
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-jh', '--num-workers-huc',
+                        help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',
+                        required=False, default=1, type=int)
+    parser.add_argument('-jb', '--num-workers-branches',
+                        help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',
+                        required=False, default=1, type=int)
+    parser.add_argument('-c','--no_cleanup', 
+                        help='If flag used, intermediate rasters are NOT cleaned up.',
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-v','--verbose', help='Show additional outputs.',
+                        required=False, default=False, action='store_true')
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    start_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print("================================")
+    print(f"Start composite inundation - {dt_string}")    
+    print()
+
+    ci = CompositeInundation()
+    ci.run_composite(args)
+
+    end_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print("================================")
+    print(f"End composite inundation - {dt_string}")
+
+    # calculate duration
+    time_duration = end_time - start_time
+    print(f"Duration: {str(time_duration).split('.')[0]}")
+    print()
+
diff --git a/tools/composite_ms_fr_inundation.py b/tools/composite_ms_fr_inundation.py
deleted file mode 100644
index e0153e143..000000000
--- a/tools/composite_ms_fr_inundation.py
+++ /dev/null
@@ -1,213 +0,0 @@
-#!/usr/bin/env python3
-import os, argparse, rasterio
-import numpy as np
-import pandas as pd
-
-from inundation import inundate
-from gms_tools.mosaic_inundation import Mosaic_inundation, __append_id_to_file_name
-
-
-def composite_inundation(fim_dir_ms, fim_dir_fr, huc, flows, composite_output_dir, ouput_name='',
-                         bin_rast_flag=False, depth_rast_flag=False, clean=True, quiet=True):
-    """
-    Runs `inundate()` on FIM 3.X mainstem (MS) and full-resolution (FR) outputs and composites results. Assumes that all `fim_run` products 
-    necessary for `inundate()` are in each huc8 folder.
-
-    Parameters
-    ----------
-    fim_dir_ms : str
-        Path to MS FIM directory. This should be an output directory from `fim_run.sh`.
-    fim_dir_fr : str
-        Path to FR FIM directory. This should be an output directory from `fim_run.sh`.
-    huc : str
-        HUC8 to run `inundate()`. This should be a folder within both `fim_dir_ms` and `fim_dir_fr`.
-    flows : str or pandas.DataFrame, can be a single file or a comma-separated list of files
-        File path to forecast csv or Pandas DataFrame with correct column names.
-    composite_output_dir : str
-        Folder path to write outputs. It will be created if it does not exist.
-    ouput_name : str, optional
-        Name for output raster. If not specified, by default the raster will be named 'inundation_composite_{flows_root}.tif'.
-    bin_rast_flag : bool, optional
-        Flag to create binary raster as output. If no raster flags are passed, this is the default behavior.
-    depth_rast_flag : bool, optional
-        Flag to create depth raster as output.
-    clean : bool, optional
-        If True, intermediate files are deleted.
-    quiet : bool, optional
-        Quiet output.
-
-    Returns
-    -------
-    None
-
-    Raises
-    ------
-    TypeError
-        Wrong input data types
-    AssertionError
-        Wrong input data types
-
-    Notes
-    -----
-    - Specifying a subset of the domain in rem or catchments to inundate on is achieved by the HUCs file or the forecast file.
-
-    Examples
-    --------
-    >>> import composite_ms_fr_inundation
-    >>> composite_ms_fr_inundation.composite_inundation(
-            '/home/user/fim_ouput_mainstem',
-            '/home/user/fim_ouput_fullres',
-            '12090301',
-            '/home/user/forecast_file.csv',
-            '/home/user/fim_inundation_composite',
-            'inundation_composite.tif',
-            True,
-            False)
-    """
-    # Set inundation raster to True if no output type flags are passed
-    if not (bin_rast_flag or depth_rast_flag):
-        bin_rast_flag = True
-    assert not (bin_rast_flag and depth_rast_flag), 'Output can only be binary or depth grid, not both'
-    assert os.path.isdir(fim_dir_ms), f'{fim_dir_ms} is not a directory. Please specify an existing MS FIM directory.'
-    assert os.path.isdir(fim_dir_fr), f'{fim_dir_fr} is not a directory. Please specify an existing FR FIM directory.'
-    assert os.path.exists(flows), f'{flows} does not exist. Please specify a flow file.'
-
-    # Instantiate output variables
-    var_keeper = {
-        'ms': {
-            'dir': fim_dir_ms,
-            'outputs': {
-                'inundation_rast': os.path.join(composite_output_dir, f'{huc}_inundation_ms.tif') if bin_rast_flag else None,
-                'depth_rast':      os.path.join(composite_output_dir, f'{huc}_depth_ms.tif') if depth_rast_flag else None
-            }
-        },
-        'fr': {
-            'dir': fim_dir_fr,
-            'outputs': {
-                'inundation_rast': os.path.join(composite_output_dir, f'{huc}_inundation_fr.tif') if bin_rast_flag else None,
-                'depth_rast':      os.path.join(composite_output_dir, f'{huc}_depth_fr.tif') if depth_rast_flag else None
-            }
-        }
-    }
-    # Build inputs to inundate() based on the input folders and huc
-    if not quiet: print(f"HUC {huc}")
-    for extent in var_keeper:
-        rem = os.path.join(var_keeper[extent]['dir'], huc, 'rem_zeroed_masked.tif')
-        catchments = os.path.join(var_keeper[extent]['dir'], huc, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-        catchment_poly = os.path.join(var_keeper[extent]['dir'], huc, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-        hydro_table = os.path.join(var_keeper[extent]['dir'], huc, 'hydroTable.csv')
-
-        # Ensure that all of the required files exist in the huc directory
-        for file in (rem, catchments, catchment_poly, hydro_table):
-            if not os.path.exists(file):
-                raise Exception(f"The following file does not exist within the supplied FIM directory:\n{file}")
-
-        # Run inundation()
-        extent_friendly = "mainstem (MS)" if extent=="ms" else "full-resolution (FR)"
-        grid_type = "an inundation" if bin_rast_flag else "a depth"
-        if not quiet: print(f"  Creating {grid_type} map for the {extent_friendly} configuration...")
-        result = inundate(rem,catchments,catchment_poly,hydro_table,flows,mask_type=None,
-                inundation_raster=  var_keeper[extent]['outputs']['inundation_rast'],
-                depths=             var_keeper[extent]['outputs']['depth_rast'],
-                quiet=              quiet)
-        if result != 0:
-            raise Exception(f"Failed to inundate {rem} using the provided flows.")
-
-    # If no output name supplied, create one using the flows file name 
-    if not ouput_name:
-        flows_root = os.path.splitext(os.path.basename(flows))[0]
-        ouput_name = os.path.join(composite_output_dir, f'inundation_composite_{flows_root}.tif')
-    else:
-        ouput_name = os.path.join(composite_output_dir, ouput_name)
-    
-    # Composite MS and FR
-    inundation_map_file = { 
-                    'huc8' : [huc] * 2,
-                    'branchID' : [None] * 2,
-                    'inundation_rasters':  [var_keeper['fr']['outputs']['inundation_rast'], 
-                                            var_keeper['ms']['outputs']['inundation_rast']],
-                    'depths_rasters':      [var_keeper['fr']['outputs']['depth_rast'], 
-                                            var_keeper['ms']['outputs']['depth_rast']]
-                    }
-    inundation_map_file = pd.DataFrame(inundation_map_file)
-    Mosaic_inundation(
-                    inundation_map_file,
-                    mosaic_attribute='depths_rasters' if depth_rast_flag else 'inundation_rasters',
-                    mosaic_output=ouput_name,
-                    mask=catchment_poly,
-                    unit_attribute_name='huc8',
-                    nodata=-9999,
-                    workers=1,
-                    remove_inputs=clean,
-                    subset=None,verbose=not quiet
-                    )
-    if bin_rast_flag:
-        hydroid_to_binary(__append_id_to_file_name(ouput_name, huc))
-
-def hydroid_to_binary(hydroid_raster_filename):
-    '''Converts hydroid positive/negative grid to 1/0'''
-
-    #to_bin = lambda x: np.where(x > 0, 1, np.where(x == 0, -9999, 0))
-    to_bin = lambda x: np.where(x > 0, 1, np.where(x != -9999, 0, -9999))
-    hydroid_raster = rasterio.open(hydroid_raster_filename)
-    profile = hydroid_raster.profile # get profile for new raster creation later on
-    profile['nodata'] = -9999
-    bin_raster = to_bin(hydroid_raster.read(1)) # converts neg/pos to 0/1
-    # Overwrite inundation raster
-    with rasterio.open(hydroid_raster_filename, "w", **profile) as out_raster:
-        out_raster.write(bin_raster.astype(hydroid_raster.profile['dtype']), 1)
-    del hydroid_raster,profile,bin_raster
-
-
-if __name__ == '__main__':
-    
-    # parse arguments
-    parser = argparse.ArgumentParser(description='Inundate FIM 3 full resolution and mainstem outputs using a flow file and composite the results.')
-    parser.add_argument('-ms','--fim-dir-ms',help='Directory that contains MS FIM outputs.',required=True)
-    parser.add_argument('-fr','--fim-dir-fr',help='Directory that contains FR FIM outputs.',required=True)
-    parser.add_argument('-u','--huc',help='HUC within FIM directories to inunundate. Can be a comma-separated list.',required=True)
-    parser.add_argument('-f','--flows-file',help='File path of flows csv or comma-separated list of paths if running multiple HUCs',required=True)
-    parser.add_argument('-o','--ouput-dir',help='Folder to write Composite Raster output.',required=True)
-    parser.add_argument('-n','--ouput-name',help='File name for output(s).',default=None,required=False)
-    parser.add_argument('-b','--bin-raster',help='Output raster is a binary wet/dry grid. This is the default if no raster flags are passed.',required=False,default=False,action='store_true')
-    parser.add_argument('-d','--depth-raster',help='Output raster is a depth grid.',required=False,default=False,action='store_true')
-    parser.add_argument('-j','--num-workers',help='Number of concurrent processesto run.',required=False,default=1,type=int)
-    parser.add_argument('-c','--clean',help='If flag used, intermediate rasters are NOT cleaned up.',required=False,default=True,action='store_false')
-    parser.add_argument('-q','--quiet',help='Quiet terminal output.',required=False,default=False,action='store_true')
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-    fim_dir_ms    = args['fim_dir_ms']
-    fim_dir_fr    = args['fim_dir_fr']
-    hucs          = args['huc'].replace(' ', '').split(',')
-    flows_files   = args['flows_file'].replace(' ', '').split(',')
-    num_workers   = int(args['num_workers'])
-    output_dir    = args['ouput_dir']
-    ouput_name    = args['ouput_name']
-    bin_raster    = bool(args['bin_raster'])
-    depth_raster  = bool(args['depth_raster'])
-    clean         = bool(args['clean'])
-    quiet         = bool(args['quiet'])
-
-    assert num_workers >= 1, "Number of workers should be 1 or greater"
-    assert len(flows_files) == len(hucs), "Number of hucs must be equal to the number of forecasts provided"
-    assert not (bin_raster and depth_raster), "Cannot use both -b and -d flags"
-
-    # Create output directory if it does not exist
-    if not os.path.isdir(output_dir):
-        os.mkdir(output_dir)
-
-    # Create nested list for input into multi-threading
-    arg_list = []
-    for huc, flows_file in zip(hucs, flows_files):
-        arg_list.append((fim_dir_ms, fim_dir_fr, huc, flows_file, output_dir, ouput_name, bin_raster, depth_raster, clean, quiet))
-
-    # Multi-thread for each huc in input hucs
-    if num_workers > 1:
-        from multiprocessing import Pool
-        with Pool(processes=num_workers) as pool:
-            # Run composite_inundation()
-            pool.starmap(composite_inundation, arg_list)
-    else: # run linear if jobs == 1
-        for arg in arg_list:
-            composite_inundation(*arg)
diff --git a/tools/gms_tools/inundate_gms.py b/tools/gms_tools/inundate_gms.py
index 0b1c2f59b..6bcca72a8 100755
--- a/tools/gms_tools/inundate_gms.py
+++ b/tools/gms_tools/inundate_gms.py
@@ -1,26 +1,27 @@
 #!/usr/bin/env python3
 
+import os
+import argparse
+import logging
+import sys
+import traceback
+
 import numpy as np
 import pandas as pd
-from inundation import inundate
-import os
+
 from tqdm import tqdm
-import argparse
+from inundation import inundate
 from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor,as_completed
 from inundation import hydroTableHasOnlyLakes, NoForecastFound
-import traceback
-import logging
+from utils.shared_functions import append_id_to_file_name
 
-
-def Inundate_gms(
-                  hydrofabric_dir, forecast, num_workers=1,
-                  hucs=None,
-                  inundation_raster=None,
-                  inundation_polygon=None, depths_raster=None,
-                  verbose=False,
-                  log_file=None,
-                  output_fileNames=None
-                 ):
+def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
+                  hucs = None,
+                  inundation_raster = None,
+                  inundation_polygon = None, depths_raster = None,
+                  verbose = False,
+                  log_file = None,
+                  output_fileNames = None ):
 
     # input handling
     if hucs is not None:
@@ -39,7 +40,8 @@ def Inundate_gms(
         if os.path.exists(log_file):
             os.remove(log_file)
         
-        print('HUC8,BranchID,Exception',file=open(log_file,'w'))
+        if verbose :
+            print('HUC8,BranchID,Exception',file=open(log_file,'w'))
     #if log_file:
         #logging.basicConfig(filename=log_file, level=logging.INFO)
         #logging.info('HUC8,BranchID,Exception')
@@ -47,8 +49,7 @@ def Inundate_gms(
     # load gms inputs
     hucs_branches = pd.read_csv( os.path.join(hydrofabric_dir,'gms_inputs.csv'),
                                  header=None,
-                                 dtype= {0:str,1:str}
-                               )
+                                 dtype= {0:str,1:str} )
     
     if hucs is not None:
         hucs = set(hucs)
@@ -59,19 +60,18 @@ def Inundate_gms(
     number_of_branches = len(hucs_branches)
     
     # make inundate generator
-    inundate_input_generator = __inundate_gms_generator(
-                                                          hucs_branches,number_of_branches,
-                                                          hydrofabric_dir,
-                                                          inundation_raster,
-                                                          inundation_polygon,
-                                                          depths_raster,
-                                                          forecast,
-                                                          verbose=False
-                                                        )
+    inundate_input_generator = __inundate_gms_generator( hucs_branches,
+                                                         number_of_branches,
+                                                         hydrofabric_dir,
+                                                         inundation_raster,
+                                                         inundation_polygon,
+                                                         depths_raster,
+                                                         forecast,
+                                                         verbose=False )
 
     # start up process pool
     # better results with Process pool
-    executor = ProcessPoolExecutor(max_workers=num_workers)
+    executor = ProcessPoolExecutor(max_workers = num_workers)
 
     # collect output filenames
     inundation_raster_fileNames = [None] * number_of_branches
@@ -88,14 +88,16 @@ def Inundate_gms(
     idx = 0
     for future in tqdm(as_completed(executor_generator),
                        total=len(executor_generator),
-                       disable=(not verbose),
-                       desc="Inundating branches with {} workers".format(num_workers)
-                      ):
+                       desc="Inundating branches with {} workers".format(num_workers),
+                       disable=(not verbose) ):
         
         hucCode, branch_id = executor_generator[future]
 
         try:
             future.result()
+
+            #print("future")
+            #print(future.result())
         
         except NoForecastFound as exc:
             if log_file is not None:
@@ -123,7 +125,7 @@ def Inundate_gms(
             branch_ids[idx] = branch_id
 
             try:
-                #print(hucCode,branch_id,future.result()[0][0])
+                #print(hucCode,branch_id,future.result()[0][0])                
                 inundation_raster_fileNames[idx] = future.result()[0][0]
             except TypeError:
                 pass
@@ -144,13 +146,14 @@ def Inundate_gms(
     executor.shutdown(wait=True)
 
     # make filename dataframe
-    output_fileNames_df = pd.DataFrame( { 
-                                          'huc8' : hucCodes,
+    output_fileNames_df = pd.DataFrame( { 'huc8' : hucCodes,
                                           'branchID' : branch_ids,
                                           'inundation_rasters' : inundation_raster_fileNames,
                                           'depths_rasters' : depths_raster_fileNames,
-                                          'inundation_polygons' : inundation_polygon_fileNames } 
-                                       )
+                                          'inundation_polygons' : inundation_polygon_fileNames } )
+
+    #print("output_fileNames_df")
+    #print(output_fileNames_df)
 
     if output_fileNames is not None:
         output_fileNames_df.to_csv(output_fileNames,index=False)
@@ -158,16 +161,13 @@ def Inundate_gms(
     return(output_fileNames_df)
 
 
-
-
-def __inundate_gms_generator( 
-                              hucs_branches,number_of_branches,
+def __inundate_gms_generator( hucs_branches,
+                              number_of_branches,
                               hydrofabric_dir,
                               inundation_raster,
                               inundation_polygon,
                               depths_raster,
-                              forecast,verbose=False
-                            ):
+                              forecast, verbose = False ):
 
     # iterate over branches
     for idx,row in hucs_branches.iterrows():
@@ -175,63 +175,60 @@ def __inundate_gms_generator(
         huc = str(row[0])
         branch_id = str(row[1])
 
-        gms_dir = os.path.join(hydrofabric_dir,huc,'branches')
+        gms_dir = os.path.join(hydrofabric_dir, huc, 'branches')
+
+        rem_file_name = 'rem_zeroed_masked_{}.tif'.format(branch_id)
+        rem_branch = os.path.join( gms_dir, branch_id, rem_file_name )
 
-        rem_branch = os.path.join( gms_dir,branch_id,'rem_zeroed_masked_{}.tif'.format(branch_id) )
-        catchments_branch = os.path.join( gms_dir,branch_id,
-                                          f'gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif' )
-        hydroTable_branch = os.path.join( gms_dir,branch_id,'hydroTable_{}.csv'.format(branch_id) )
-        catchment_poly = os.path.join( gms_dir, branch_id,
-                                       f'gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg' )
+        catchments_file_name = f'gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif'
+        catchments_branch = os.path.join( gms_dir, branch_id, catchments_file_name )
+
+        hydroTable_branch = os.path.join( gms_dir,branch_id, 'hydroTable_{}.csv'.format(branch_id) )
+
+        xwalked_file_name = f'gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg'
+        catchment_poly = os.path.join( gms_dir, branch_id, xwalked_file_name )
         
     
         # branch output
-        inundation_branch_raster = __append_id_to_file_name(inundation_raster,[huc,branch_id])
-        inundation_branch_polygon = __append_id_to_file_name(inundation_polygon,[huc,branch_id])
-        depths_branch_raster = __append_id_to_file_name(depths_raster,[huc,branch_id])
+        # Some other functions that call in here already added a huc, so only add it if not yet there
+        if (inundation_raster is not None) and (huc not in inundation_raster):
+            inundation_branch_raster = append_id_to_file_name(inundation_raster,[huc,branch_id])
+        else:
+            inundation_branch_raster = append_id_to_file_name(inundation_raster,branch_id)
+
+        if (inundation_polygon is not None) and (huc not in inundation_polygon):
+            inundation_branch_polygon = append_id_to_file_name(inundation_polygon,[huc,branch_id])
+        else:
+            inundation_branch_polygon = append_id_to_file_name(inundation_polygon,branch_id)
+
+        if (depths_raster is not None) and (huc not in depths_raster):
+            depths_branch_raster = append_id_to_file_name(depths_raster,[huc,branch_id])
+        else:
+            depths_branch_raster = append_id_to_file_name(depths_raster,branch_id)
 
         # identifiers
         identifiers = (huc,branch_id)
 
         # inundate input
-        inundate_input = { 
-                            'rem' : rem_branch, 'catchments' : catchments_branch, 'catchment_poly' : catchment_poly,
-                            'hydro_table' : hydroTable_branch,'forecast' : forecast,
+        inundate_input = {  'rem' : rem_branch, 
+                            'catchments' : catchments_branch, 
+                            'catchment_poly' : catchment_poly,
+                            'hydro_table' : hydroTable_branch,
+                            'forecast' : forecast,
                             'mask_type' : None,
                             'hucs' : None,
                             'hucs_layerName' : None,
-                            'subset_hucs' : None, 'num_workers' : 1,
+                            'subset_hucs' : None, 
+                            'num_workers' : 1,
                             'aggregate' : False,
                             'inundation_raster' : inundation_branch_raster,
                             'inundation_polygon' : inundation_branch_polygon,
                             'depths' : depths_branch_raster,
                             'out_raster_profile' : None,
                             'out_vector_profile' : None,
-                            'quiet' : not verbose
-                          }
+                            'quiet' : not verbose }
         
-        yield (inundate_input,identifiers)
-
-
-
-def __append_id_to_file_name(file_name,identifier):
-
-
-    if file_name is not None:
-
-        root,extension = os.path.splitext(file_name)
-
-        if isinstance(identifier,list):
-            for i in identifier:
-                out_file_name = root + "_{}".format(i)
-            out_file_name += extension
-        else:
-            out_file_name = root + "_{}".format(identifier) + extension
-
-    else:
-        out_file_name = None
-
-    return(out_file_name)
+        yield (inundate_input, identifiers)
 
 
 def __vprint(message,verbose):
diff --git a/tools/gms_tools/mosaic_inundation.py b/tools/gms_tools/mosaic_inundation.py
index 291c7287b..29a682e8c 100755
--- a/tools/gms_tools/mosaic_inundation.py
+++ b/tools/gms_tools/mosaic_inundation.py
@@ -1,22 +1,33 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from glob import glob
-from gms_tools.overlapping_inundation import OverlapWindowMerge
 import argparse
 import os
 import pandas as pd
+import sys
+
+from glob import glob
+from gms_tools.overlapping_inundation import OverlapWindowMerge
 from tqdm import tqdm
 from utils.shared_variables import elev_raster_ndv
-
-def Mosaic_inundation(
-                      map_file,mosaic_attribute='inundation_rasters',mosaic_output=None,
-                      mask=None,unit_attribute_name='huc8',
-                      nodata=elev_raster_ndv,workers=4,
-                      remove_inputs=False,
-                      subset=None,verbose=True
-                      ):
+from utils.shared_functions import append_id_to_file_name
+
+def Mosaic_inundation( map_file,
+                       mosaic_attribute = 'inundation_rasters',
+                       mosaic_output = None,
+                       mask = None,
+                       unit_attribute_name = 'huc8',
+                       nodata = elev_raster_ndv,
+                       workers = 1,
+                       remove_inputs = False,
+                       subset = None,
+                       verbose = True,
+                       is_mosaic_for_gms_branches = False ):
     
+    # Notes:
+    #    - If is_mosaic_for_gms_branches is true, the mosaic output name
+    #      will add the HUC into the output name for overwrite resons.
+
     # check input
     if mosaic_attribute not in ('inundation_rasters','depths_rasters'):
         raise ValueError('Pass inundation or depths for mosaic_attribute argument') 
@@ -27,13 +38,12 @@ def Mosaic_inundation(
         del map_file
     elif isinstance(map_file,str):
         inundation_maps_df = pd.read_csv(map_file,
-                                         dtype={unit_attribute_name:str,'branchID':str}
-                                        )
+                                         dtype={unit_attribute_name:str,'branchID':str})
     else:
         raise TypeError('Pass Pandas Dataframe or file path string to csv for map_file argument')
 
     # remove NaNs
-    inundation_maps_df.dropna(axis=0,how='all',inplace=True)
+    inundation_maps_df.dropna(axis=0, how='all', inplace=True)
 
     # subset
     if subset is not None:
@@ -43,7 +53,7 @@ def Mosaic_inundation(
     # unique aggregation units
     aggregation_units = inundation_maps_df.loc[:,unit_attribute_name].unique()
 
-    inundation_maps_df.set_index(unit_attribute_name,drop=True,inplace=True)
+    inundation_maps_df.set_index(unit_attribute_name, drop=True, inplace=True)
 
     # decide upon wheter to display 
     if verbose & len(aggregation_units) == 1:
@@ -53,17 +63,31 @@ def Mosaic_inundation(
     else:
         tqdm_disable = True
 
-    for ag in tqdm(aggregation_units,disable=tqdm_disable,desc='Mosaicing FIMs'):
+    for ag in tqdm(aggregation_units, disable = tqdm_disable, desc = 'Mosaicing FIMs'):
 
         try:
             inundation_maps_list = inundation_maps_df.loc[ag,mosaic_attribute].tolist()
         except AttributeError:
             inundation_maps_list = [ inundation_maps_df.loc[ag,mosaic_attribute] ]
 
-        ag_mosaic_output = __append_id_to_file_name(mosaic_output,ag)
+        # Some processes may have already added the ag value (if it is a huc) to 
+        # the file name, so don't re-add it.
+        # Only add the huc into the name if branches are being processed, as 
+        # sometimes the mosiac is not for gms branches but maybe mosaic of an
+        # fr set with a gms composite map.
+        ag_mosaic_output = mosaic_output
+        if (is_mosaic_for_gms_branches) and (ag not in mosaic_output):
+            ag_mosaic_output = append_id_to_file_name(mosaic_output, ag) # change it
+
+        #print(f"ag_mosaic_output is {ag_mosaic_output}")
         #try:
-        mosaic_by_unit(inundation_maps_list,ag_mosaic_output,nodata,
-                       workers=1,remove_inputs=remove_inputs,mask=mask,verbose=verbose)
+        mosaic_by_unit(inundation_maps_list, 
+                      ag_mosaic_output,
+                      nodata,
+                      workers = workers, 
+                      remove_inputs = remove_inputs,
+                      mask = mask,
+                      verbose = verbose)
         #except Exception as exc:
         #    print(ag,exc)
     
@@ -71,74 +95,72 @@ def Mosaic_inundation(
     # inundation maps
     inundation_maps_df.reset_index(drop=True)
 
+    # return file name and path of the final mosaic output file.
+    return ag_mosaic_output
 
 
-def mosaic_by_unit(inundation_maps_list,mosaic_output,nodata=elev_raster_ndv,
-                   workers=1,remove_inputs=False,mask=None,verbose=False):
-
+# Note: This uses threading and not processes. If the number of workers is more than 
+# the number of possible threads, no results will be returned. But it is usually
+# pretty fast anyways. This needs to be fixed.
+def mosaic_by_unit(inundation_maps_list,
+                   mosaic_output,
+                   nodata = elev_raster_ndv,
+                   workers = 1,
+                   remove_inputs = False,
+                   mask = None,
+                   verbose = False):
 
     # overlap object instance
     overlap = OverlapWindowMerge( inundation_maps_list, (30, 30) )
 
-    # mosaic
-    #if verbose:
-    #    print("Mosaicing ...")
-
     if mosaic_output is not None:
         if workers > 1:
             threaded = True
         else:
             threaded= False
         
-        overlap.merge_rasters(mosaic_output, threaded=threaded, workers=workers,nodata=nodata)
+        overlap.merge_rasters(mosaic_output, threaded=threaded, workers=workers, nodata=nodata)
 
         if mask:
-            #if verbose:
-            #    print("Masking ...")
-            overlap.mask_mosaic(mosaic_output,mask,outfile=mosaic_output)
+            if verbose:
+                print("Masking ...")
+            overlap.mask_mosaic(mosaic_output, mask, outfile=mosaic_output)
     
     if remove_inputs:
-        #if verbose:
-        #    print("Removing inputs ...")
+        if verbose:
+            print("Removing inputs ...")
 
         for inun_map in inundation_maps_list:
             if inun_map is not None:
                 if os.path.isfile(inun_map):
                     os.remove(inun_map)
 
-
-def __append_id_to_file_name(file_name,identifier):
-
-
-    if file_name is not None:
-
-        root,extension = os.path.splitext(file_name)
-
-        if isinstance(identifier,list):
-            for i in identifier:
-                out_file_name = root + "_{}".format(i)
-            out_file_name += extension
-        else:
-            out_file_name = root + "_{}".format(identifier) + extension
-
-    else:
-        out_file_name = None
-
-    return(out_file_name)
-
-
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Mosaic GMS Inundation Rasters')
-    parser.add_argument('-i','--map-file', help='List of file paths to inundation/depth maps to mosaic', required=True)
-    parser.add_argument('-a','--mask', help='File path to vector polygon mask to clip mosaic too', required=False,default=None)
-    parser.add_argument('-s','--subset', help='Subset units', required=False,default=None,type=str,nargs='+')
-    parser.add_argument('-n','--nodata', help='Inundation Maps', required=False,default=elev_raster_ndv)
-    parser.add_argument('-w','--workers', help='Number of Workers', required=False,default=4,type=int)
-    parser.add_argument('-t','--mosaic-attribute', help='Mosaiced inundation Maps', required=False,default=None)
-    parser.add_argument('-m','--mosaic-output', help='Mosaiced inundation Maps', required=False,default=None)
-    parser.add_argument('-r','--remove-inputs', help='Remove original input inundation Maps', required=False,default=False,action='store_true')
-    parser.add_argument('-v','--verbose', help='Remove original input inundation Maps', required=False,default=False,action='store_true')
+    parser.add_argument('-i','--map-file', 
+                        help='List of file paths to inundation/depth maps to mosaic',
+                        required=True)
+    parser.add_argument('-a','--mask', 
+                        help='File path to vector polygon mask to clip mosaic too',
+                        required=False, default=None)
+    parser.add_argument('-s','--subset', help='Subset units', 
+                        required=False, default=None, type=str, nargs='+')
+    parser.add_argument('-n','--nodata', help='Inundation Maps',
+                        required=False, default=elev_raster_ndv)
+    parser.add_argument('-w','--workers', help='Number of Workers', 
+                        required=False, default=4, type=int)
+    parser.add_argument('-t','--mosaic-attribute', help='Mosaiced inundation Maps', 
+                        required=False, default=None)
+    parser.add_argument('-m','--mosaic-output', help='Mosaiced inundation Maps file name',
+                        required=False, default=None)
+    parser.add_argument('-r','--remove-inputs', help='Remove original input inundation Maps',
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-v','--verbose', help='Remove original input inundation Maps', 
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-g','--is-mosaic-for-gms-branches', 
+                        help='If the mosaic is for gms branchs, include this arg',
+                        required=False, default=False, action='store_true')
 
     args = vars(parser.parse_args())
     
diff --git a/tools/inundation.py b/tools/inundation.py
index 98cad20d9..93031adaf 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -37,11 +37,12 @@ class NoForecastFound(Exception):
     """ Raised when no forecast is available for a given Hydro-Table """
     pass
 
-def inundate(
-             rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=None,hucs_layerName=None,
-             subset_hucs=None,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
-             depths=None,out_raster_profile=None,out_vector_profile=None,src_table=None,quiet=False
-            ):
+def inundate(rem, catchments, catchment_poly, hydro_table, forecast,
+             mask_type, hucs = None, hucs_layerName = None,
+             subset_hucs = None, num_workers = 1, aggregate = False, 
+             inundation_raster = None, inundation_polygon = None,
+             depths = None, out_raster_profile = None, out_vector_profile = None,
+             src_table = None, quiet = False):
     """
 
     Run inundation on FIM >=3.0 outputs at job-level scale or aggregated scale
@@ -102,10 +103,6 @@ def inundate(
     -----
     - Specifying a subset of the domain in rem or catchments to inundate on is achieved by the HUCs file or the forecast file.
 
-    Examples
-    --------
-    >>> import inundation
-    >>> inundation.inundate(rem,catchments,hydro_table,forecast,inundation_raster)
     """
 
     # check for num_workers
@@ -179,8 +176,11 @@ def inundate(
             create_src_subset_csv(hydro_table,catchmentStagesDict,src_table)
 
         # make windows generator
-        window_gen = __make_windows_generator(rem,catchments,catchment_poly,mask_type,catchmentStagesDict,inundation_raster,inundation_polygon,
-                                              depths,out_raster_profile,out_vector_profile,quiet,hucs=hucs,hucSet=hucSet)
+        window_gen = __make_windows_generator(rem, catchments, catchment_poly,
+                                              mask_type, catchmentStagesDict, inundation_raster,
+                                              inundation_polygon, depths, out_raster_profile,
+                                              out_vector_profile, quiet, 
+                                              hucs = hucs, hucSet = hucSet)
 
         # start up thread pool
         executor = ThreadPoolExecutor(max_workers=num_workers)
@@ -211,8 +211,6 @@ def inundate(
     rem.close()
     catchments.close()
 
-
-
     return(inundation_rasters,depth_rasters,inundation_polys)
 
 
@@ -379,8 +377,19 @@ def __go_fast_mapping(rem,catchments,catchmentStagesDict,inundation,depths):
     return(inundation,depths)
 
 
-def __make_windows_generator(rem,catchments,catchment_poly,mask_type,catchmentStagesDict,inundation_raster,inundation_polygon,
-                             depths,out_raster_profile,out_vector_profile,quiet,hucs=None,hucSet=None):
+def __make_windows_generator(rem, 
+                             catchments,
+                             catchment_poly,
+                             mask_type,
+                             catchmentStagesDict,
+                             inundation_raster,
+                             inundation_polygon,
+                             depths,
+                             out_raster_profile,
+                             out_vector_profile,
+                             quiet,
+                             hucs = None,
+                             hucSet = None):
 
     
     if hucs is not None:
@@ -439,10 +448,10 @@ def __return_huc_in_hucSet(hucCode,hucSet):
 
             hucCode = huc['properties'][hucColName]
 
-            yield (rem_array,catchments_array,rem.crs.wkt,
-                   window_transform,rem.profile,catchments.profile,hucCode,
-                   catchmentStagesDict,depths,inundation_raster,
-                   inundation_polygon,out_raster_profile,out_vector_profile,quiet)
+            yield (rem_array, catchments_array, rem.crs.wkt,
+                   window_transform, rem.profile, catchments.profile, hucCode,
+                   catchmentStagesDict, depths, inundation_raster,
+                   inundation_polygon, out_raster_profile, out_vector_profile, quiet)
 
     else:
         hucCode = None
@@ -626,22 +635,53 @@ def create_src_subset_csv(hydro_table,catchmentStagesDict,src_table):
 
     # parse arguments
     parser = argparse.ArgumentParser(description='Rapid inundation mapping for FOSS FIM. Operates in single-HUC and batch modes.')
-    parser.add_argument('-r','--rem', help='REM raster at job level or mosaic vrt. Must match catchments CRS.', required=True)
-    parser.add_argument('-c','--catchments',help='Catchments raster at job level or mosaic VRT. Must match rem CRS.',required=True)
-    parser.add_argument('-b','--catchment-poly',help='catchment_vector',required=True)
-    parser.add_argument('-t','--hydro-table',help='Hydro-table in csv file format',required=True)
-    parser.add_argument('-f','--forecast',help='Forecast discharges in CMS as CSV file',required=True)
-    parser.add_argument('-u','--hucs',help='Batch mode only: HUCs file to process at. Must match CRS of input rasters',required=False,default=None)
-    parser.add_argument('-l','--hucs-layerName',help='Batch mode only. Layer name in HUCs file to use',required=False,default=None)
-    parser.add_argument('-j','--num-workers',help='Batch mode only. Number of concurrent processes',required=False,default=1,type=int)
-    parser.add_argument('-s','--subset-hucs',help='Batch mode only. HUC code, series of HUC codes (no quotes required), or line delimited of HUCs to run within the hucs file that is passed',required=False,default=None,nargs='+')
-    parser.add_argument('-m', '--mask-type', help='Specify huc (FIM < 3) or filter (FIM >= 3) masking method', required=False,default="huc")
-    parser.add_argument('-a','--aggregate',help='Batch mode only. Aggregate outputs to VRT files. Currently, raises warning and sets to false if used.',required=False,action='store_true')
-    parser.add_argument('-i','--inundation-raster',help='Inundation Raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
-    parser.add_argument('-p','--inundation-polygon',help='Inundation polygon output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
-    parser.add_argument('-d','--depths',help='Depths raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
-    parser.add_argument('-n','--src-table',help='Output table with the SRC lookup/interpolation. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
-    parser.add_argument('-q','--quiet',help='Quiet terminal output',required=False,default=False,action='store_true')
+    parser.add_argument('-r', '--rem', 
+                        help='REM raster at job level or mosaic vrt. Must match catchments CRS.', 
+                        required=True)
+    parser.add_argument('-c', '--catchments',
+                        help='Catchments raster at job level or mosaic VRT. Must match rem CRS.',
+                        required=True)
+    parser.add_argument('-b', '--catchment-poly', help='catchment_vector',
+                        required=True)
+    parser.add_argument('-t', '--hydro-table', help='Hydro-table in csv file format', 
+                        required=True)
+    parser.add_argument('-f', '--forecast', help='Forecast discharges in CMS as CSV file',
+                        required=True)
+    parser.add_argument('-u', '--hucs',
+                        help='Batch mode only: HUCs file to process at. Must match CRS of input rasters',required=False, default=None)
+    parser.add_argument('-l', '--hucs-layerName', help='Batch mode only. Layer name in HUCs file to use',
+                        required=False, default=None)
+    parser.add_argument('-j', '--num-workers',help='Batch mode only. Number of concurrent processes',
+                        required=False, default=1, type=int)
+    parser.add_argument('-s', '--subset-hucs', help = """Batch mode only. HUC code, 
+            series of HUC codes (no quotes required), or line delimited of HUCs to run within 
+            the hucs file that is passed""",
+                        required=False, default=None, nargs='+')
+    parser.add_argument('-m', '--mask-type',
+                        help='Specify huc (FIM < 3) or filter (FIM >= 3) masking method',
+                        required=False, default="huc")
+    parser.add_argument('-a', '--aggregate',
+                        help="""Batch mode only. Aggregate outputs to VRT files.
+                        Currently, raises warning and sets to false if used.""",
+                        required=False, action='store_true')
+    parser.add_argument('-i', '--inundation-raster',
+                        help="""Inundation Raster output. Only writes if designated.
+                        Appends HUC code in batch mode.""",
+                        required=False, default=None)
+    parser.add_argument('-p', '--inundation-polygon',
+                        help="""Inundation polygon output. Only writes if designated. 
+                        Appends HUC code in batch mode.""",
+                        required=False, default=None)
+    parser.add_argument('-d', '--depths',
+                        help="""Depths raster output. Only writes if designated.
+                        Appends HUC code in batch mode.""",
+                        required=False, default=None)
+    parser.add_argument('-n', '--src-table',
+                        help="""Output table with the SRC lookup/interpolation.
+                        Only writes if designated. Appends HUC code in batch mode.""",
+                        required=False, default=None)
+    parser.add_argument('-q','--quiet', help='Quiet terminal output',
+                        required=False, default=False, action='store_true')
 
     # extract to dictionary
     args = vars(parser.parse_args())
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 33d3fe0d0..d1f0f7958 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -219,25 +219,43 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 if __name__ == '__main__':
 
     # Sample usage:
+    '''
+     === FOR GMS
+    python /foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v gms_test_synth_combined -jh 2 -jb 40 -m /outputs/gms_test_synth_combined/gms_synth_metrics.csv -vg -o
+
+     Notes:
+       - gms_input.csv MUST be in the folder suggested.
+       - the -v param is the name in the folder in the "outputs/" directory where the test hucs are at.
+           It also becomes the folder names inside the test_case folders when done.
+       - the -vg param may not be working (will be assessed better on later releases).
+       - Find a balance between -jh (number of jobs for hucs) versus -jb (number of jobs for branches)
+         on quick tests on a 96 core machine, we tried [1 @ 80], [2 @ 40], and [3 @ 25] (and others).
+       -jb 3 -jh 25 was noticably better. You can likely go more jb cores with better success, just
+         experiment.  Start times, End Times and duration are now included.
+       - The -m can be any path and any name.
     
-    #python /foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v gms_test_synth_combined -jh 2 -jb 40 -m /outputs/gms_test_synth_combined/gms_synth_metrics.csv -vg -o
-
-    # Notes:
-    #   - gms_input.csv MUST be in the folder suggested.
-    #   - the -v param is the name in the folder in the "outputs/" directory where the test hucs are at.
-    #       It also becomes the folder names inside the test_case folders when done.
-    #   - the -vg param may not be working (will be assessed better on later releases).
-    #   - Find a balance between -jh (number of jobs for hucs) versus -jb (number of jobs for branches)
-    #     on quick tests on a 96 core machine, we tried [1 @ 80], [2 @ 40], and [3 @ 25] (and others).
-    #   -jb 3 -jh 25 was noticably better. You can likely go more jb cores with better success, just
-    #     experiment.  Start times, End Times and duration are now included.
-    #   - The -m can be any path and any name.
+     To see your outputs in the test_case folder (hard coded path), you can check for outputs using
+         (cd .... to your test_case folder), then command becomes  find . -name gms_test_* -type d (Notice the
+         the -name can be a wildcard for your -v param (or the whole -v value))
+     If you want to delete the test outputs, test the outputs as suggest immediately above, but this time your
+         command becomes:  find . -name gms_test_* -type d  -exec rm -rdf {} +
+    '''
+    '''
+     === FOR FIM
+    python /foss_fim/tools/synthesize_test_cases.py -c DEV -e MS -v dev_fim_3_0_29_1_ms -jh 4 -m /outputs/dev_fim_3_0_29_1_ms/alpha/alpha_master_metrics_fim_3_0_29_1_ms_src_adjust.csv -vg -o
+
+     Notes:
+       - the -v param is the name in the folder in the "outputs/" directory where the test hucs are at.
+           It also becomes the folder names inside the test_case folders when done.
+       - the -vg param may not be working (will be assessed better on later releases).
+       - The -m can be any path and any name.
     
-    # To see your outputs in the test_case folder (hard coded path), you can check for outputs using
-    #     (cd .... to your test_case folder), then command becomes  find . -name gms_test_* -type d (Notice the
-    #     the -name can be a wildcard for your -v param (or the whole -v value))
-    # If you want to delete the test outputs, test the outputs as suggest immediately above, but this time your
-    #     command becomes:  find . -name gms_test_* -type d  -exec rm -rdf {} +
+     To see your outputs in the test_case folder (hard coded path), you can check for outputs using
+         (cd .... to your test_case folder), then command becomes  find . -name dev_fim_3_0_29_1_* -type d (Notice the
+         the -name can be a wildcard for your -v param (or the whole -v value))
+     If you want to delete the test outputs, test the outputs as suggest immediately above, but this time your
+         command becomes:  find . -name dev_fim_3_0_29_1_* -type d  -exec rm -rdf {} +
+    '''
 
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
diff --git a/unit_tests/gms/inundate_gms_params.json b/unit_tests/gms/inundate_gms_params.json
new file mode 100644
index 000000000..b6c2d9884
--- /dev/null
+++ b/unit_tests/gms/inundate_gms_params.json
@@ -0,0 +1,16 @@
+{
+	"valid_data_inudation_raster_single_huc":
+	{
+		"hydrofabric_dir": "/outputs/fim_unit_test_data_do_not_remove/",
+		"forecast": "data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv",
+		"num_workers": 1,
+		"hucs": "05030104",
+	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_1_gms.tif",
+		"inundation_polygon": null,
+		"depths_raster": null,
+		"verbose": true,
+		"log_file": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_logfile.txt",
+		"output_fileNames": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_file_list.csv"
+	}
+
+}
\ No newline at end of file
diff --git a/unit_tests/gms/inundate_gms_unittests.py b/unit_tests/gms/inundate_gms_unittests.py
new file mode 100644
index 000000000..b09034edb
--- /dev/null
+++ b/unit_tests/gms/inundate_gms_unittests.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import pandas as pd
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+import unit_tests_utils as helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/tools/gms_tools/') 
+import inundate_gms as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_inundate_gms(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+
+        params_file_path = '/foss_fim/unit_tests/gms/inundate_gms_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self):
+
+        '''
+        Test for creating a gms inundation raster, not a depth raster and no
+        inundation_polygons.
+        This test is based on creating a raster based on a single huc and its branches
+        within a gms output folder.
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["valid_data_inudation_raster_single_huc"].copy()         
+        
+        output_fileNames_df = src.Inundate_gms(hydrofabric_dir = params["hydrofabric_dir"],
+                                               forecast = params["forecast"],
+                                               num_workers = params["num_workers"],
+                                               hucs = params["hucs"],
+                                               inundation_raster = params["inundation_raster"],
+                                               inundation_polygon = params["inundation_polygon"],
+                                               depths_raster = params["depths_raster"],
+                                               verbose = params["verbose"], 
+                                               log_file = params["log_file"],
+                                               output_fileNames = params["output_fileNames"])
+
+        # check if output files df has records.
+        assert len(output_fileNames_df) > 0, "Expected as least one dataframe record"
+
+        # also check output log file and output raster. Can't... there will be multiple outputs
+        #assert os.path.exists(params["inundation_raster"]), "Inundation Raster does not exist"
+
+        assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+        
+    # ***********************
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json
new file mode 100644
index 000000000..16edb822c
--- /dev/null
+++ b/unit_tests/shared_functions_params.json
@@ -0,0 +1,13 @@
+{
+	"append_append_id_to_file_name_single_identifier_valid":
+	{
+		"file_name": "/output/myfolder/a_raster.tif",
+		"identifier": "13090001"
+	},
+
+	"append_append_id_to_file_name_identifier_list_valid":
+	{
+		"file_name": "/output/myfolder/a_raster.tif",
+		"identifier": ["13090001", "05010204"]
+	}	
+}
\ No newline at end of file
diff --git a/unit_tests/shared_functions_unittests.py b/unit_tests/shared_functions_unittests.py
new file mode 100644
index 000000000..1c3e11905
--- /dev/null
+++ b/unit_tests/shared_functions_unittests.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import argparse
+import json
+import warnings
+import unittest
+
+import unit_tests_utils as helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/utils') 
+import shared_functions as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_shared_functions(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+
+        params_file_path = '/foss_fim/unit_tests/shared_functions_params.json'
+    
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_append_id_to_file_name_single_identifier_success(self):
+
+        '''
+        Pass in a file name with the single identifier and get the single adjusted file name back
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
+        
+        expected_output = "/output/myfolder/a_raster_13090001.tif"
+
+        actual_output = src.append_id_to_file_name(file_name = params["file_name"],
+                                                   identifier = params["identifier"])
+
+        err_msg = "actual output does not match expected output"
+        self.assertEqual(expected_output, actual_output, err_msg)
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_append_id_to_file_name_indentifer_list_success(self):
+
+        '''
+        Pass in a file name with the list of identifiers and get a file name back with multiple identifers added
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()        
+        
+        expected_output = "/output/myfolder/a_raster_13090001_05010204.tif"
+
+        actual_output = src.append_id_to_file_name(file_name = params["file_name"],
+                                                   identifier = params["identifier"])
+
+        #print(actual_output)
+        err_msg = "actual output does not match expected output"
+        self.assertEqual(expected_output, actual_output, err_msg)
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+    def test_append_id_to_file_name_no_file_name_success(self):
+
+        '''
+        Pass in an non existant file name and get None back
+        '''
+
+        #global params_file
+        helpers.print_unit_test_function_header()
+        
+        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
+        
+        actual_output = src.append_id_to_file_name(None,
+                                                   identifier = params["identifier"])
+
+        if (actual_output is not None):
+            raise Exception("actual output should not have a value")
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    

From 235268fc6c3af1765a280df63004b3f0a626c794 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Thu, 16 Jun 2022 09:58:54 -0500
Subject: [PATCH 142/359] [1pt] PR: 'Branch zero' that utilizes the full stream
 network (#611)

* move DEM processing to unit level

* created  to run common processing between  and

* finalize branch zero

* branch zero code standardization and cleanup

* fixed gitignore

* Docker pathing issue resulted in not being able to see the new deny_gms_branch_zero.lst file. This is a temp work around.

* small bug fixes for branch zero

* bug fix for branch zero feature_id crosswalk errors

* remove redundant files

* remove tests directory

* branch errors folder bug

* update CHANGELOG.md

Co-authored-by: Rob Hanna <Robert.Hanna@NOAA.gov>
---
 .gitignore                                   |   1 +
 config/deny_gms_branch_zero.lst              |  59 ++
 config/deny_gms_unit_default.lst             |   4 +-
 config/params_template.env                   |   1 +
 docs/CHANGELOG.md                            |  33 +
 gms_run_branch.sh                            |   7 +-
 gms_run_unit.sh                              |   7 +-
 src/gms/delineate_hydros_and_produce_HAND.sh | 178 +++++
 src/gms/generate_branch_list.py              |   7 +-
 src/gms/run_by_branch.sh                     | 239 +------
 src/gms/run_by_unit.sh                       | 122 +++-
 src/output_cleanup.py                        |   6 +-
 tests/aggregate_metrics.py                   | 271 --------
 tests/cache_metrics.py                       | 118 ----
 tests/run_test_case.py                       | 248 -------
 tests/synthesize_test_cases.py               | 311 ---------
 tests/utils/shared_functions.py              | 676 -------------------
 tools/consolidate_metrics.py                 |   2 +-
 tools/shared_variables.py                    |  20 -
 19 files changed, 437 insertions(+), 1873 deletions(-)
 create mode 100644 config/deny_gms_branch_zero.lst
 create mode 100755 src/gms/delineate_hydros_and_produce_HAND.sh
 delete mode 100644 tests/aggregate_metrics.py
 delete mode 100644 tests/cache_metrics.py
 delete mode 100755 tests/run_test_case.py
 delete mode 100644 tests/synthesize_test_cases.py
 delete mode 100644 tests/utils/shared_functions.py
 delete mode 100644 tools/shared_variables.py

diff --git a/.gitignore b/.gitignore
index 47137b619..e5acc4ee8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
 .nfs*
 __pycache__/
 config/**
+!config/deny*.lst
 !config/*template*
 !config/*default*
 !config/symbology/
diff --git a/config/deny_gms_branch_zero.lst b/config/deny_gms_branch_zero.lst
new file mode 100644
index 000000000..3b80477a4
--- /dev/null
+++ b/config/deny_gms_branch_zero.lst
@@ -0,0 +1,59 @@
+# List of files for gms branches to delete
+# Use comment to allow list the file
+# Use curly braces to denote branch_id
+agree_bufgrid.tif
+agree_bufgrid_allo.tif
+agree_bufgrid_dist.tif
+agree_smogrid.tif
+agree_smogrid_allo.tif
+agree_smogrid_dist.tif
+catch_list_{}.txt
+coordFile_{}.txt
+crosswalk_table_{}.csv
+demDerived_reaches_{}.dbf
+demDerived_reaches_{}.prj
+demDerived_reaches_{}.shp
+demDerived_reaches_{}.shx
+demDerived_reaches_split_{}.gpkg
+demDerived_reaches_split_filtered_{}.gpkg
+demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+demDerived_reaches_split_points_{}.gpkg
+demDerived_streamPixels_{}.tif
+demDerived_streamPixels_ids_{}.tif
+demDerived_streamPixels_ids_{}_allo.tif
+demDerived_streamPixels_ids_{}_dist.tif
+dem_burned_{}.tif
+dem_burned_filled_{}.tif
+dem_lateral_thalweg_adj_{}.tif
+#dem_meters_{}.tif
+dem_thalwegCond_{}.tif
+flowaccum_d8_burned_filled_{}.tif
+#flowdir_d8_burned_filled_{}.tif
+flowdir_d8_burned_filled_flows_{}.tif
+flows_grid_boolean_{}.tif
+flows_points_pixels_{}.gpkg
+gw_catchments_pixels_{}.tif
+gw_catchments_reaches_{}.gpkg
+gw_catchments_reaches_{}.tif
+#gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_{}.tif
+gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+headwaters_{}.tif
+#hydroTable_{}.csv
+idFile_{}.txt
+nld_rasterized_elev_{}.tif
+nwm_catchments_proj_subset_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
+rem_{}.tif
+#rem_zeroed_masked_{}.tif
+slopes_d8_dem_meters_{}.tif
+slopes_d8_dem_meters_masked_{}.tif
+sn_catchments_reaches_{}.tif
+src_{}.json
+src_base_{}.csv
+#src_full_crosswalked_{}.csv
+stage_{}.txt
+streamOrder_{}.tif
+treeFile_{}.txt
+#usgs_elev_table.csv
diff --git a/config/deny_gms_unit_default.lst b/config/deny_gms_unit_default.lst
index dc5bb7277..f4f91920a 100644
--- a/config/deny_gms_unit_default.lst
+++ b/config/deny_gms_unit_default.lst
@@ -5,11 +5,11 @@ NHDPlusBurnLineEvent_subset.gpkg
 #branch_polygons.gpkg
 nhd_headwater_points_subset.gpkg
 #nld_subset_levees.gpkg
-nwm_catchments_proj_subset.gpkg
+#nwm_catchments_proj_subset.gpkg
 #nwm_catchments_proj_subset_levelPaths.gpkg
 nwm_headwaters.gpkg
 #nwm_lakes_proj_subset.gpkg
-nwm_subset_streams.gpkg
+#nwm_subset_streams.gpkg
 #nwm_subset_streams_levelPaths.gpkg
 #nwm_subset_streams_levelPaths_dissolved.gpkg
 #nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index 72f58e75c..4d967cbbf 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -24,6 +24,7 @@ export min_stream_length=0.5
 export branch_id_attribute=levpa_id
 export branch_buffer_distance_meters=7000
 export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.
+export branch_zero_id="0"
 
 #### bathy SRC estimation parameters ####
 export bathy_src_toggle=True # Toggle to run BARC routine (True=on; False=off)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 74105580a..e63a7f639 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,39 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.5.0 - 2022-06-16 - [PR #611](https://github.com/NOAA-OWP/inundation-mapping/pull/611)
+
+'Branch zero' is a new branch that runs the HUCs full stream network to make up for stream orders 1 & 2 being skipped by the GMS solution and is similar to the FR extent in FIM v3. This new branch is created during `run_by_unit.sh` and the processed DEM is used by the other GMS branches during `run_by_branch.sh` to improve efficiency.
+
+## Additions
+
+- `src/gms/delineate_hydros_and_produce_HAND.sh`: Runs all of the modules associated with delineating stream lines and catchments and building the HAND relative elevation model. This file is called once during `gms_run_unit` to produce the branch zero files and is also run for every GMS branch in `gms_run_branch`.
+- `config/deny_gms_branch_zero.lst`: A list specifically for branch zero that helps with cleanup (removing unneeded files after processing).
+
+## Changes
+
+- `src/`
+    - `output_cleanup.py`: Fixed bug for viz flag.
+    - `gms/`
+        - `run_by_unit.sh`: Added creation of "branch zero", DEM pre-processing, and now calls.
+        -  `delineate_hydros_and_produce_HAND.sh` to produce HAND outputs for the entire stream network.
+        - `run_by_branch.sh`: Removed DEM processing steps (now done in `run_by_unit.sh`), moved stream network delineation and HAND generation to `delineate_hydros_and_produce_HAND.sh`.
+        - `generate_branch_list.py`: Added argument and parameter to sure that the branch zero entry was added to the branch list.
+- `config/`
+     - `params_template.env`: Added `zero_branch_id` variable.
+- `tools`
+     - `run_test_case.py`: Some styling / readability upgrades plus some enhanced outputs.  Also changed the _verbose_ flag to _gms_verbose_ being passed into Mosaic_inundation function.
+     - `synthesize_test_cases.py`: arguments being passed into the _alpha_test_args_ from being hardcoded from flags to verbose (effectively turning on verbose outputs when applicable. Note: Progress bar was not affected.
+     - `tools_shared_functions.py`: Some styling / readability upgrades.
+- `gms_run_unit.sh`: Added export of extent variable.
+- `gms_run_branch.sh`: Fixed bug when using overwrite flag saying branch errors folder already exists.
+
+## Removals
+
+- `tests/`: Redundant
+- `tools/shared_variables`: Redundant
+
+<br/><br/>
 
 ## v4.0.4.3 - 2022-05-26 - [PR #605](https://github.com/NOAA-OWP/inundation-mapping/pull/605)
 
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index fb043bb3a..a8cda2e50 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -141,10 +141,9 @@ fi
 
 if [ ! -d "$outputRunDataDir/branch_errors" ]; then
     mkdir -p "$outputRunDataDir/branch_errors"
-else
-    if [ $overwrite -eq 1 ]; then
-        rm -rf $outputRunDataDir/branch_errors/
-    fi
+elif [ $overwrite -eq 1 ]; then
+    rm -rf $outputRunDataDir/branch_errors
+    mkdir -p $outputRunDataDir/branch_errors
 fi
 
 ## RUN GMS BY BRANCH ##
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index 9ea3da8fe..fe61c3d65 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -65,9 +65,9 @@ in
         shift
         deny_gms_unit_list=$1
         ;;
-	 -s|--dropLowStreamOrders)
-		  dropLowStreamOrders=1
-		  ;;
+    -s|--dropLowStreamOrders)
+        dropLowStreamOrders=1
+        ;;
     *) ;;
     esac
     shift
@@ -129,6 +129,7 @@ export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_strea
 export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
 export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
 export deny_gms_unit_list=$deny_gms_unit_list
+export extent=GMS
 
 ## Input handling ##
 $srcDir/check_huc_inputs.py -u "$hucList"
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
new file mode 100755
index 000000000..353903d4a
--- /dev/null
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -0,0 +1,178 @@
+#!/bin/bash -e
+
+## Level is equal to the parent script: 'unit' or 'branch'
+level=$1
+
+## INITIALIZE TOTAL TIME TIMER ##
+T_total_start
+
+## D8 FLOW ACCUMULATIONS ##
+echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/aread8 -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -wg $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif -nc
+Tcount
+
+# THRESHOLD ACCUMULATIONS ##
+echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/threshold -ssa $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -thresh 1
+Tcount
+
+## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
+echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif -g $outputCurrentBranchDataDir/temp_grass
+Tcount
+
+## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
+echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -a $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif -d $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif -t 50 -o $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -th $thalweg_lateral_elev_threshold
+Tcount
+
+## MASK BURNED DEM FOR STREAMS ONLY ###
+echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -B $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif --calc="A/B" --outfile="$outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" --NoDataValue=0
+Tcount
+
+## FLOW CONDITION STREAMS ##
+echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/flowdircond -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif -z $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -zfdc $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
+Tcount
+
+## D8 SLOPES ##
+echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
+Tcount
+
+# STREAMNET FOR REACHES ##
+echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/streamnet -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -fel $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -ord $outputCurrentBranchDataDir/streamOrder_$current_branch_id.tif -tree $outputCurrentBranchDataDir/treeFile_$current_branch_id.txt -coord $outputCurrentBranchDataDir/coordFile_$current_branch_id.txt -w $outputCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif -net $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
+Tcount
+
+## SPLIT DERIVED REACHES ##
+echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -ds $dropLowStreamOrders
+Tcount
+
+## GAGE WATERSHED FOR REACHES ##
+echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -gw $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
+Tcount
+
+## VECTORIZE FEATURE ID CENTROIDS ##
+echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/reachID_grid_to_vector_points.py -r $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -i featureID -p $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
+Tcount
+
+## GAGE WATERSHED FOR PIXELS ##
+echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif -gw $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
+Tcount
+
+# D8 REM ##
+echo -e $startDiv"D8 REM $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/gms/rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
+Tcount
+
+## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
+echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"$stopDiv
+date -u
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+Tcount
+
+## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
+if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
+    echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"$stopDiv
+    date -u
+    Tstart
+
+    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
+    Tcount
+fi
+
+## POLYGONIZE REACH WATERSHEDS ##
+echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_polygonize.py -8 -f GPKG $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
+Tcount
+
+## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
+echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber -s $dropLowStreamOrders
+Tcount
+
+## RASTERIZE NEW CATCHMENTS AGAIN ##
+echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
+Tcount
+
+## MASK SLOPE TO CATCHMENTS ##
+echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"$stopDiv
+date -u
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif --calc="A*(B>0)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
+Tcount
+
+## MAKE CATCHMENT AND STAGE FILES ##
+echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$srcDir/make_stages_and_catchlist.py -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/stage_$current_branch_id.txt -a $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
+Tcount
+
+## MASK REM RASTER TO REMOVE OCEAN AREAS ##
+echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+[ -f $outputCurrentBranchDataDir/LandSea_subset.tif ] && \
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LandSea_subset_$current_node_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+Tcount
+
+## HYDRAULIC PROPERTIES ##
+echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+$taudemDir/catchhydrogeo -hand $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -catch $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif -catchlist $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -slp $outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif -h $outputCurrentBranchDataDir/stage_$current_branch_id.txt -table $outputCurrentBranchDataDir/src_base_$current_branch_id.csv
+Tcount
+
+## FINALIZE CATCHMENTS AND MODEL STREAMS ##
+echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+if [ "$level" = "branch" ]; then
+    b_arg=$outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
+    z_arg=$outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg
+elif [ "$level" = "unit" ]; then
+    # Branch zero has a different source for -b and -z arguments
+    b_arg=$outputHucDataDir/nwm_subset_streams.gpkg
+    z_arg=$outputHucDataDir/nwm_catchments_proj_subset.gpkg
+fi
+python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/src_base_$current_branch_id.csv -l $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $outputCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $outputCurrentBranchDataDir/src_$current_branch_id.json -x $outputCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $outputCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $b_arg -y $outputCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -p $extent -k $outputCurrentBranchDataDir/small_segments_$current_branch_id.csv
+Tcount
diff --git a/src/gms/generate_branch_list.py b/src/gms/generate_branch_list.py
index 100ef6bba..96a57c1cf 100755
--- a/src/gms/generate_branch_list.py
+++ b/src/gms/generate_branch_list.py
@@ -4,7 +4,7 @@
 from stream_branches import StreamNetwork
 import argparse
 
-def Generate_branch_list(stream_network_dissolved, branch_id_attribute, output_branch_list):
+def Generate_branch_list(stream_network_dissolved, branch_id_attribute, output_branch_list, branch_zero):
 
     # load stream network
 
@@ -17,6 +17,10 @@ def Generate_branch_list(stream_network_dissolved, branch_id_attribute, output_b
     # write
     stream_network_dissolved.to_csv(output_branch_list,sep= " ",index=False,header=False)
 
+    # Add branch zero ID to branch list
+    if branch_zero:
+        with open(output_branch_list,'a') as branch_lst:
+            branch_lst.write(f'{branch_zero}')
 
 if __name__ == '__main__':
 
@@ -24,6 +28,7 @@ def Generate_branch_list(stream_network_dissolved, branch_id_attribute, output_b
     parser.add_argument('-d','--stream-network-dissolved', help='Dissolved stream network', required=True)
     parser.add_argument('-b','--branch-id-attribute', help='Branch ID attribute to use in dissolved stream network', required=True)
     parser.add_argument('-o','--output-branch-list', help='Output branch list', required=True)
+    parser.add_argument('-z','--branch-zero', help='Optional Branch Zero ID (str) to be added to the branch list. Usually this will be "0".', required=False)
     
     args = vars(parser.parse_args())
 
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index cff4a97b3..8b8c26562 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -13,6 +13,11 @@ hucUnitLength=${#hucNumber}
 huc4Identifier=${hucNumber:0:4}
 huc2Identifier=${hucNumber:0:2}
 
+# Skip branch zero
+if [ $current_branch_id = $branch_zero_id ]; then
+    exit 0
+fi
+
 outputHucDataDir=$outputRunDataDir/$hucNumber
 outputBranchDataDir=$outputHucDataDir/branches
 outputCurrentBranchDataDir=$outputBranchDataDir/$current_branch_id
@@ -39,19 +44,6 @@ mkdir -p $outputCurrentBranchDataDir
 ## START MESSAGE ##
 echo -e $startDiv"Processing branch_id: $current_branch_id in HUC: $hucNumber ..."$stopDiv
 
-## CLIP RASTERS
-echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$srcDir/gms/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $input_DEM -c $outputCurrentBranchDataDir/dem_meters.tif -v 
-Tcount
-
-## GET RASTER METADATA
-echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
-
 ## SUBSET VECTORS
 echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"$stopDiv
 date -u
@@ -66,15 +58,19 @@ ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentB
 #ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_headwaters.gpkg
 Tcount
 
-## RASTERIZE NLD MULTILINES ##
-echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $current_branch_id"$stopDiv
+## GET RASTERS FROM BRANCH ZERO AND CLIP TO CURRENT BRANCH BUFFER ##
+echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
-# REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
-gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputCurrentBranchDataDir/nld_rasterized_elev_$current_branch_id.tif
+$srcDir/gms/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $outputBranchDataDir/$branch_zero_id/dem_meters_$branch_zero_id.tif $outputBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $outputCurrentBranchDataDir/dem_meters.tif $outputCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
 Tcount
 
+## GET RASTER METADATA
+echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"$stopDiv
+date -u
+Tstart
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
+
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"$stopDiv
 date -u
@@ -89,200 +85,19 @@ Tstart
 gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif
 Tcount
 
-## BURN LEVEES INTO DEM ##
-echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwrite dem_meters.tif output) $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-# REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $outputCurrentBranchDataDir/nld_rasterized_elev_$current_branch_id.tif ] && \
-python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputCurrentBranchDataDir/nld_rasterized_elev_$current_branch_id.tif -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif
-Tcount
-
-## DEM Reconditioning ##
-# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
-# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
-echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif -d $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -w $outputCurrentBranchDataDir -g $outputCurrentBranchDataDir/temp_work -o $outputCurrentBranchDataDir/dem_burned_$current_branch_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
-Tcount
-
-## PIT REMOVE BURNED DEM ##
-echo -e $startDiv"Pit remove Burned DEM $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-rd_depression_filling $outputCurrentBranchDataDir/dem_burned_$current_branch_id.tif $outputCurrentBranchDataDir/dem_burned_filled_$current_branch_id.tif
-Tcount
-
-## D8 FLOW DIR ##
-echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_burned_filled_$current_branch_id.tif -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif
-Tcount
-
-## D8 FLOW ACCUMULATIONS ##
-echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$taudemDir/aread8 -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -ad8  $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -wg  $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif -nc
-Tcount
-
-# THRESHOLD ACCUMULATIONS ##
-echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$taudemDir/threshold -ssa $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src  $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -thresh 1
-Tcount
-
-## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
-echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif -g $outputCurrentBranchDataDir/temp_grass
-Tcount
-
-## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
-echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -a $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif -d $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif -t 50 -o $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -th $thalweg_lateral_elev_threshold
-Tcount
-
-## MASK BURNED DEM FOR STREAMS ONLY ###
-echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -B $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif --calc="A/B" --outfile="$outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" --NoDataValue=0
-Tcount
-
-## FLOW CONDITION STREAMS ##
-echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$taudemDir/flowdircond -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif -z $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -zfdc $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
-Tcount
-
-## D8 SLOPES ##
-echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
-Tcount
-
-# STREAMNET FOR REACHES ##
-echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$taudemDir/streamnet -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -fel $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -ord $outputCurrentBranchDataDir/streamOrder_$current_branch_id.tif -tree $outputCurrentBranchDataDir/treeFile_$current_branch_id.txt -coord $outputCurrentBranchDataDir/coordFile_$current_branch_id.txt -w $outputCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif -net $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
-Tcount
-
-## SPLIT DERIVED REACHES ##
-echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -ds $dropLowStreamOrders
-Tcount
-
-## GAGE WATERSHED FOR REACHES ##
-echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -gw $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
-Tcount
-
-## VECTORIZE FEATURE ID CENTROIDS ##
-echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$srcDir/reachID_grid_to_vector_points.py -r $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -i featureID -p $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
-Tcount
-
-## GAGE WATERSHED FOR PIXELS ##
-echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif -gw $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
-Tcount
-
-# D8 REM ##
-echo -e $startDiv"D8 REM $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$srcDir/gms/rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
-Tcount
-
-## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
-echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"$stopDiv
-date -u
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
-Tcount
-
-## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
-if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
-    echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"$stopDiv
-    date -u
-    Tstart
-
-    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
-    Tcount
-fi
-
-## POLYGONIZE REACH WATERSHEDS ##
-echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-gdal_polygonize.py -8 -f GPKG $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
-Tcount
-
-## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
-echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber -s $dropLowStreamOrders
-
-
-## RASTERIZE NEW CATCHMENTS AGAIN ##
-echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
-Tcount
-
-## MASK SLOPE TO CATCHMENTS ##
-echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"$stopDiv
-date -u
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif --calc="A*(B>0)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
-Tcount
-
-## MAKE CATCHMENT AND STAGE FILES ##
-echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$srcDir/make_stages_and_catchlist.py -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/stage_$current_branch_id.txt -a $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
-Tcount
-
-## MASK REM RASTER TO REMOVE OCEAN AREAS ##
-echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-[ -f $outputCurrentBranchDataDir/LandSea_subset.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LandSea_subset_$current_node_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
-Tcount
-
-## HYDRAULIC PROPERTIES ##
-echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-$taudemDir/catchhydrogeo -hand $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -catch $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif -catchlist $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -slp $outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif -h $outputCurrentBranchDataDir/stage_$current_branch_id.txt -table $outputCurrentBranchDataDir/src_base_$current_branch_id.csv
-Tcount
-
-## FINALIZE CATCHMENTS AND MODEL STREAMS ##
-echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/src_base_$current_branch_id.csv -l $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $outputCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $outputCurrentBranchDataDir/src_$current_branch_id.json -x $outputCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $outputCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg -y $outputCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg -p $extent -k $outputCurrentBranchDataDir/small_segments_$current_branch_id.csv
-Tcount
+## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
+export hucNumber=$hucNumber
+export current_branch_id=$current_branch_id
+export outputCurrentBranchDataDir=$outputCurrentBranchDataDir
+export outputHucDataDir=$outputHucDataDir
+export ndv=$ndv
+export xmin=$xmin
+export ymin=$ymin
+export xmax=$xmax
+export ymax=$ymax
+export ncols=$ncols
+export nrows=$nrows
+$srcDir/gms/delineate_hydros_and_produce_HAND.sh "branch"
 
 ## USGS CROSSWALK ##
 if [ -f $outputHucDataDir/usgs_subset_gages.gpkg ]; then
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 7f5223aba..3a7457bb5 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -7,7 +7,7 @@ T_total_start
 hucNumber="$1"
 outputHucDataDir=$outputRunDataDir/$hucNumber
 outputBranchDataDir=$outputHucDataDir/branches
-
+current_branch_id=$branch_zero_id
 
 ## huc data
 if [ -d "$outputHucDataDir" ]; then
@@ -57,7 +57,7 @@ Tcount
 echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent GMS -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
+python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
 Tcount
 
 ## Clip WBD8 ##
@@ -90,7 +90,123 @@ Tcount
 echo -e $startDiv"Create file of branch ids for $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute
+if [ $dropLowStreamOrders != 0 ]; then # only add branch zero to branch list if low stream orders are dropped
+    $srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -z $branch_zero_id
+else
+    $srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute
+fi
+
+Tcount
+
+## CREATE BRANCH ZERO ##
+echo -e $startDiv"Creating branch zero for $hucNumber"$stopDiv
+outputCurrentBranchDataDir=$outputBranchDataDir/$branch_zero_id
+
+## OVERWRITE
+if [ -d "$outputCurrentBranchDataDir" ];then
+    if [ $overwrite -eq 1 ]; then
+        rm -rf $outputCurrentBranchDataDir
+    else
+        echo "GMS branch data directories for $hucNumber - $branch_zero_id already exist. Use -o/--overwrite to continue"
+        exit 1
+    fi
+fi
+
+## MAKE OUTPUT BRANCH DIRECTORY
+mkdir -p $outputCurrentBranchDataDir
+
+## CLIP RASTERS
+echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+[ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \
+gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+Tcount
+
+## GET RASTER METADATA
+echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif)
+
+## RASTERIZE NLD MULTILINES ##
+echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+# REMAINS UNTESTED FOR AREAS WITH LEVEES
+[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
+gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif
+Tcount
+
+## BURN LEVEES INTO DEM ##
+echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+# REMAINS UNTESTED FOR AREAS WITH LEVEES
+[ -f $outputCurrentBranchDataDir/nld_subset_levees.tif ] && \
+python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -nld $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif -out $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+Tcount
+
+## RASTERIZE REACH BOOLEAN (1 & 0) ##
+echo -e $startDiv"Rasterize Reach Boolean $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_subset_streams.gpkg $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
+Tcount
+
+## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
+echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nhd_headwater_points_subset.gpkg $outputCurrentBranchDataDir/headwaters_$branch_zero_id.tif
+Tcount
+
+## DEM Reconditioning ##
+# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
+# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
+echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -w $outputCurrentBranchDataDir -g $outputCurrentBranchDataDir/temp_work -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+Tcount
+
+## PIT REMOVE BURNED DEM ##
+echo -e $startDiv"Pit remove Burned DEM $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+rd_depression_filling $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
+Tcount
+
+## D8 FLOW DIR ##
+echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $branch_zero_id"$stopDiv
+date -u
+Tstart
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
+Tcount
+
+## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
+export hucNumber=$hucNumber
+export current_branch_id=$current_branch_id
+export outputCurrentBranchDataDir=$outputCurrentBranchDataDir
+export outputHucDataDir=$outputHucDataDir
+export ndv=$ndv
+export xmin=$xmin
+export ymin=$ymin
+export xmax=$xmax
+export ymax=$ymax
+export ncols=$ncols
+export nrows=$nrows
+if [ $dropLowStreamOrders != 0 ]; then # only produce branch zero HAND if low stream orders are dropped
+    $srcDir/gms/delineate_hydros_and_produce_HAND.sh "unit"
+else
+    echo -e $startDiv"Skipping branch zero processing because there are no stream orders being dropped $hucNumber"$stopDiv
+fi
+
+## CLEANUP BRANCH ZERO OUTPUTS ##
+echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
+date -u
+Tstart
+$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -v -b
 Tcount
 
 ## CREATE USGS GAGES FILE
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index 2491f05a5..146b59308 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -5,7 +5,7 @@
 
 
 @mem_profile
-def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, viz_post_processing):
+def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, is_viz_post_processing):
     '''
     Processes all the final output files to cleanup and add post-processing
 
@@ -70,7 +70,7 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
 def whitelist_directory(directory_path, whitelist, additional_whitelist):
     # Add any additional files to the whitelist that the user wanted to keep
     if additional_whitelist:
-        whitelist = whitelist + [filename for filename in additional_whitelist.split(',')]
+        whitelist = whitelist + additional_whitelist
 
     # Delete any non-whitelisted files
     directory = os.fsencode(directory_path)
@@ -85,7 +85,7 @@ def whitelist_directory(directory_path, whitelist, additional_whitelist):
     parser = argparse.ArgumentParser(description = 'Cleanup output files')
     parser.add_argument('huc_number', type=str, help='The HUC')
     parser.add_argument('output_folder_path', type=str, help='Path to the outputs for the specific huc')
-    parser.add_argument('-w', '--additional_whitelist', type=str, help='List of additional files to keep in a production run')
+    parser.add_argument('-w', '--additional_whitelist', type=str, help='List of additional files to keep in a production run',default=None,nargs="+")
     parser.add_argument('-p', '--is_production', help='Keep only white-listed files for production runs', action='store_true')
     parser.add_argument('-v', '--is_viz_post_processing', help='Formats output files to be useful for Viz', action='store_true')
 
diff --git a/tests/aggregate_metrics.py b/tests/aggregate_metrics.py
deleted file mode 100644
index d8a462d5b..000000000
--- a/tests/aggregate_metrics.py
+++ /dev/null
@@ -1,271 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import csv
-    
-import argparse
-
-TEST_CASES_DIR = r'/data/test_cases_new/'
-# TEMP = r'/data/temp'
-
-# Search through all previous_versions in test_cases
-from utils.shared_functions import compute_stats_from_contingency_table
-
-def create_master_metrics_csv():
-            
-    # Construct header
-    metrics_to_write = ['true_negatives_count',
-                        'false_negatives_count',
-                        'true_positives_count',
-                        'false_positives_count',
-                        'contingency_tot_count',
-                        'cell_area_m2',
-                        'TP_area_km2',
-                        'FP_area_km2',
-                        'TN_area_km2',
-                        'FN_area_km2',
-                        'contingency_tot_area_km2',
-                        'predPositive_area_km2',
-                        'predNegative_area_km2',
-                        'obsPositive_area_km2',
-                        'obsNegative_area_km2',
-                        'positiveDiff_area_km2',
-                        'CSI',
-                        'FAR',
-                        'TPR',
-                        'TNR',
-                        'PPV',
-                        'NPV',
-                        'ACC',
-                        'Bal_ACC',
-                        'MCC',
-                        'EQUITABLE_THREAT_SCORE',
-                        'PREVALENCE',
-                        'BIAS',
-                        'F1_SCORE',
-                        'TP_perc',
-                        'FP_perc',
-                        'TN_perc',
-                        'FN_perc',
-                        'predPositive_perc',
-                        'predNegative_perc',
-                        'obsPositive_perc',
-                        'obsNegative_perc',
-                        'positiveDiff_perc',
-                        'masked_count',
-                        'masked_perc',
-                        'masked_area_km2'
-                        ]
-    
-    additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
-    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source']]
-    
-    for benchmark_type in ['ble', 'ahps']:
-        
-        if benchmark_type == 'ble':
-        
-            test_cases = r'/data/test_cases'
-            test_cases_list = os.listdir(test_cases)
-            # AHPS test_ids
-            versions_to_aggregate = ['fim_1_0_0', 'fim_2_3_3', 'fim_3_0_0_3_fr_c']
-                            
-            for test_case in test_cases_list:
-                try:
-                    int(test_case.split('_')[0])
-                    
-                    huc = test_case.split('_')[0]
-                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
-                    
-                    for magnitude in ['100yr', '500yr']:
-                        for version in versions_to_aggregate:
-                            version_dir = os.path.join(previous_versions, version)
-                            magnitude_dir = os.path.join(version_dir, magnitude)
-
-                            if os.path.exists(magnitude_dir):
-                                
-                                magnitude_dir_list = os.listdir(magnitude_dir)
-                                for f in magnitude_dir_list:
-                                    if '.json' in f:
-                                        flow = 'NA'
-                                        nws_lid = "NA"
-                                        benchmark_source = 'ble'
-                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
-                                        full_json_path = os.path.join(magnitude_dir, f)
-                                        if os.path.exists(full_json_path):
-                                            stats_dict = json.load(open(full_json_path))
-                                            for metric in metrics_to_write:
-                                                sub_list_to_append.append(stats_dict[metric])
-                                            sub_list_to_append.append(full_json_path)
-                                            sub_list_to_append.append(flow)
-                                            sub_list_to_append.append(benchmark_source)
-                                            
-                                            list_to_write.append(sub_list_to_append)
-                                                                                    
-                except ValueError:
-                    pass
-                
-        if benchmark_type == 'ahps':
-    
-            test_cases = r'/data/test_cases_ahps_testing'
-            test_cases_list = os.listdir(test_cases)
-            # AHPS test_ids
-            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021', 
-                                     'fim_2_x_ms_nws_1_21_2021', 'fim_2_x_ms_usgs_1_21_2021',
-                                     'fim_3_0_0_3_ms_c_nws_1_21_2021', 'fim_3_0_0_3_ms_c_usgs_1_21_2021',
-                                     'ms_xwalk_fill_missing_cal_nws', 'ms_xwalk_fill_missing_cal_usgs']
-            
-            for test_case in test_cases_list:
-                try:
-                    int(test_case.split('_')[0])
-                    
-                    huc = test_case.split('_')[0]
-                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
-                    
-                    for magnitude in ['action', 'minor', 'moderate', 'major']:
-                        for version in versions_to_aggregate:
-                            
-                            if 'nws' in version:
-                                benchmark_source = 'ahps_nws'
-                            if 'usgs' in version:
-                                benchmark_source = 'ahps_usgs'
-                            
-                            version_dir = os.path.join(previous_versions, version)
-                            magnitude_dir = os.path.join(version_dir, magnitude)
-                            
-                            if os.path.exists(magnitude_dir):
-                                magnitude_dir_list = os.listdir(magnitude_dir)
-                                for f in magnitude_dir_list:
-                                    if '.json' in f and 'total_area' not in f:
-                                        nws_lid = f[:5]
-                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
-                                        full_json_path = os.path.join(magnitude_dir, f)
-                                        flow = ''
-                                        if os.path.exists(full_json_path):
-                                            # Get flow used to map.
-                                            if 'usgs' in version:
-                                                parent_dir = 'usgs_1_21_2021'
-                                            if 'nws' in version:
-                                                parent_dir = 'nws_1_21_2021'
-                                                
-                                            flow_file = os.path.join(test_cases, parent_dir, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
-                                            if os.path.exists(flow_file):
-                                                with open(flow_file, newline='') as csv_file:
-                                                    reader = csv.reader(csv_file)
-                                                    next(reader)
-                                                    for row in reader:
-                                                        flow = row[1]
-                                                    if nws_lid == 'mcc01':
-                                                        print(flow)
-                                            
-                                            stats_dict = json.load(open(full_json_path))
-                                            for metric in metrics_to_write:
-                                                sub_list_to_append.append(stats_dict[metric])
-                                            sub_list_to_append.append(full_json_path)
-                                            sub_list_to_append.append(flow)
-                                            sub_list_to_append.append(benchmark_source)
-                                            list_to_write.append(sub_list_to_append)
-                                        
-                except ValueError:
-                    pass
-        
-    with open(output_csv, 'w', newline='') as csvfile:
-        csv_writer = csv.writer(csvfile)
-        csv_writer.writerows(list_to_write)
-
-
-
-def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfolder=""):
-
-    # Read hucs into list.
-    if hucs != "":
-        huc_list = [line.rstrip('\n') for line in open(hucs)]
-
-    else:
-        huc_list = None
-
-    if config == "DEV":
-        config_version = "development_versions"
-    elif config == "PREV":
-        config_version = "previous_versions"
-
-    # Make directory to store output aggregates.
-    if special_string != "":
-        special_string = "_" + special_string
-    aggregate_output_dir = os.path.join(outfolder, 'aggregate_metrics', branch + '_aggregate_metrics' + special_string)
-    if not os.path.exists(aggregate_output_dir):
-        os.makedirs(aggregate_output_dir)
-
-    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
-
-    for magnitude in ['100yr', '500yr', 'action', 'minor', 'moderate', 'major']:
-        huc_path_list = [['huc', 'path']]
-        true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = 0, 0, 0, 0, 0, 0
-        
-        for test_case in test_cases_dir_list:
-
-            if test_case not in ['other', 'validation_data_ble', 'validation_data_legacy', 'validation_data_ahps']:
-                branch_results_dir = os.path.join(TEST_CASES_DIR, test_case, 'performance_archive', config_version, branch)
-
-                huc = test_case.split('_')[0]
-                # Check that the huc is in the list of hucs to aggregate.
-                if huc_list != None and huc not in huc_list:
-                    continue
-
-                stats_json_path = os.path.join(branch_results_dir, magnitude, 'total_area_stats.json')
-
-                # If there is a stats json for the test case and branch name, use it when aggregating stats.
-                if os.path.exists(stats_json_path):
-                    json_dict = json.load(open(stats_json_path))
-
-                    true_positives += json_dict['true_positives_count']
-                    true_negatives += json_dict['true_negatives_count']
-                    false_positives += json_dict['false_positives_count']
-                    false_negatives += json_dict['false_negatives_count']
-                    masked_count += json_dict['masked_count']
-
-                    cell_area = json_dict['cell_area_m2']
-
-                    huc_path_list.append([huc, stats_json_path])
-                
-                    
-            if cell_area == 0:
-                continue
-            
-            # Pass all sums to shared function to calculate metrics.
-            stats_dict = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=cell_area, masked_count=masked_count)
-
-            list_to_write = [['metric', 'value']]  # Initialize header.
-
-            for stat in stats_dict:
-                list_to_write.append([stat, stats_dict[stat]])
-                
-            # Map path to output directory for aggregate metrics.
-            output_file = os.path.join(aggregate_output_dir, branch + '_aggregate_metrics_' + magnitude + special_string + '.csv')
-
-        if cell_area != 0:
-            with open(output_file, 'w', newline='') as csvfile:
-                csv_writer = csv.writer(csvfile)
-                csv_writer.writerows(list_to_write)
-                csv_writer.writerow([])
-                csv_writer.writerows(huc_path_list)
-    
-            print()
-            print("Finished aggregating for the '" + magnitude + "' magnitude. Aggregated metrics over " + str(len(huc_path_list)-1) + " test cases.")
-            print()
-            print("Results are at: " + output_file)
-            print()
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Aggregates a metric or metrics for multiple HUC8s.')
-    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=False)
-    parser.add_argument('-b','--branch',help='Name of branch to check all test_cases for and to aggregate.',required=True)
-    parser.add_argument('-u','--hucs',help='HUC8s to restrict the aggregation.',required=False, default="")
-    parser.add_argument('-s','--special_string',help='Special string to add to outputs.',required=False, default="")
-    parser.add_argument('-f','--outfolder',help='output folder',required=True,type=str)
-
-    args = vars(parser.parse_args())
-
-    aggregate_metrics(**args)
diff --git a/tests/cache_metrics.py b/tests/cache_metrics.py
deleted file mode 100644
index 3f601cc5c..000000000
--- a/tests/cache_metrics.py
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import argparse
-import traceback
-
-from run_test_case import run_alpha_test
-from multiprocessing import Pool
-
-TEST_CASES_DIR = r'/data/test_cases_new/'  #TODO remove "_new"
-PREVIOUS_FIM_DIR = r'/data/previous_fim'
-OUTPUTS_DIR = r'/data/outputs'
-
-
-def process_alpha_test(args):
-    
-    fim_run_dir = args[0]
-    version = args[1]
-    test_id = args[2]
-    magnitude = args[3]
-    archive_results = args[4]
-    
-    mask_type = 'huc'    
-    
-    if archive_results == False:
-        compare_to_previous = True
-    else:
-        compare_to_previous = False
-
-    try:
-        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type)
-    except Exception:
-        traceback.print_exc()
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
-    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
-    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
-    parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
-    parser.add_argument('-b','--benchmark-category',help='Options include ble or ahps. Defaults to process both.',required=False, default=None)
-        
-    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
-    
-    args = vars(parser.parse_args())
-
-    config = args['config']
-    fim_version = args['fim_version']
-    job_number = int(args['job_number'])
-    special_string = args['special_string']
-    benchmark_category = args['benchmark_category']
-    
-    if fim_version != "all":
-        previous_fim_list = [fim_version]
-    else:
-        previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)    
-    
-    if config == 'PREV':
-        archive_results = True
-    elif config == 'DEV':
-        archive_results = False
-    else:
-        print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
-    
-    benchmark_category_list = []
-    
-    if benchmark_category == None:
-        for d in test_cases_dir_list:
-            if 'test_cases' in d:
-                benchmark_category_list.append(d.replace('_test_cases', ''))
-    else:
-        benchmark_category_list = [benchmark_category]
-
-    procs_list = []
-    for bench_cat in benchmark_category_list:
-        bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
-        
-        bench_cat_test_case_list = os.listdir(bench_cat_test_case_dir)
-    
-        for test_id in bench_cat_test_case_list:
-            if 'validation' and 'other' not in test_id:
-                            
-                current_huc = test_id.split('_')[0]
-                if test_id.split('_')[1] in bench_cat:
-                
-                    for version in previous_fim_list:
-                        
-                        if config == 'DEV':
-                            fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
-                        elif config == 'PREV':
-                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
-                                                
-                        if not os.path.exists(fim_run_dir):
-                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  # For previous versions of HAND computed at HUC6 scale
-                        
-                        if os.path.exists(fim_run_dir):
-                            if special_string != "":
-                                version = version + '_' + special_string
-                            
-                            if 'ble' in test_id:
-                                magnitude = ['100yr', '500yr']
-                            elif 'usgs' or 'nws' in test_id:
-                                magnitude = ['action', 'minor', 'moderate', 'major']
-                            else:
-                                continue
-                        
-                            print("Adding " + test_id + " to list of test_ids to process...")
-                            if job_number > 1:
-                                procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results])
-                            else:                            
-                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results])
-
-    if job_number > 1:
-        pool = Pool(job_number)
-        pool.map(process_alpha_test, procs_list)
\ No newline at end of file
diff --git a/tests/run_test_case.py b/tests/run_test_case.py
deleted file mode 100755
index 5beac4f16..000000000
--- a/tests/run_test_case.py
+++ /dev/null
@@ -1,248 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import shutil
-import argparse
-
-from utils.shared_functions import compute_contingency_stats_from_rasters
-from utils.shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES)
-from inundation import inundate
-
-
-def run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=False, archive_results=False, mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True):
-    
-    benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
-    current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
-    
-    # Construct paths to development test results if not existent.
-    if archive_results:
-        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'official_versions', version)
-    else:
-        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'testing_versions', version)
-
-    # Delete the entire directory if it already exists.
-    if os.path.exists(version_test_case_dir_parent):
-        if overwrite == True:
-            shutil.rmtree(version_test_case_dir_parent)
-        else:
-            print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
-            return
-        
-    os.mkdir(version_test_case_dir_parent)
-
-    print("Running the alpha test for test_id: " + test_id + ", " + version + "...")
-    stats_modes_list = ['total_area']
-
-    fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
-    assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
-
-    # Create paths to fim_run outputs for use in inundate().
-    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
-    if not os.path.exists(rem):
-        rem = os.path.join(fim_run_parent, 'rem_clipped_zeroed_masked.tif')
-    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-    if not os.path.exists(catchments):
-        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_clipped_addedAttributes.tif')
-    if mask_type == 'huc':
-        catchment_poly = ''
-    else:
-        catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
-        
-    # Map necessary inputs for inundation().
-    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-
-    # Create list of shapefile paths to use as exclusion areas.
-    zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
-    mask_dict = {'levees': 
-                    {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
-                     'buffer': None,
-                     'operation': 'exclude'
-                     },
-                'waterbodies':
-                    {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
-                     'buffer': None,
-                     'operation': 'exclude',
-                     },
-                }
-            
-    if inclusion_area != '':
-        inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
-        mask_dict.update({inclusion_area_name: {'path': inclusion_area,
-                                                'buffer': int(inclusion_area_buffer),
-                                                'operation': 'include'}})
-        # Append the concatenated inclusion_area_name and buffer.
-        if inclusion_area_buffer == None:
-            inclusion_area_buffer = 0
-        stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm') 
-
-    # Check if magnitude is list of magnitudes or single value.
-    magnitude_list = magnitude
-    if type(magnitude_list) != list:
-        magnitude_list = [magnitude_list]
-
-    # Get path to validation_data_{benchmark} directory and huc_dir.
-    validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
-    
-    for magnitude in magnitude_list:
-        version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
-        if not os.path.exists(version_test_case_dir):
-            os.mkdir(version_test_case_dir)
-    
-        # Construct path to validation raster and forecast file.
-        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-            benchmark_raster_path_list, forecast_list = [], []
-            lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
-            lid_list, inundation_raster_list, extent_file_list = [], [], []
-            
-            for lid in lid_dir_list:
-                lid_dir = os.path.join(validation_data_path, current_huc, lid)
-                benchmark_raster_path_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_depth_' + magnitude + '.tif'))  # TEMP
-                forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
-                lid_list.append(lid)
-                inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
-                extent_file_list.append(os.path.join(lid_dir, lid + '_extent.shp'))
-
-        else:
-            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_depth_' + magnitude + '.tif')
-            benchmark_raster_path_list = [benchmark_raster_file]
-            forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
-            forecast_list = [forecast_path]
-            inundation_raster_list = [os.path.join(version_test_case_dir, 'inundation_extent.tif')]
-            
-        for index in range(0, len(benchmark_raster_path_list)):
-            benchmark_raster_path = benchmark_raster_path_list[index]
-            forecast = forecast_list[index]
-            inundation_raster = inundation_raster_list[index]
-            
-            # Only need to define ahps_lid and ahps_extent_file for AHPS_BENCHMARK_CATEGORIES.
-            if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                ahps_lid = lid_list[index]
-                ahps_extent_file = extent_file_list[index]
-                mask_dict.update({ahps_lid:
-                    {'path': ahps_extent_file,
-                     'buffer': None,
-                     'operation': 'include'}
-                        })
-        
-                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_extent_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
-                    continue
-            else:  # If not in AHPS_BENCHMARK_CATEGORIES.
-                if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
-                    continue
-    
-            # Run inundate.
-            print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
-            try:
-                inundate(
-                         rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                         subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-                         depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
-                        )
-            
-                print("-----> Inundation mapping complete.")
-                predicted_raster_path = os.path.join(os.path.split(inundation_raster)[0], os.path.split(inundation_raster)[1].replace('.tif', '_' + current_huc + '.tif'))  # The inundate adds the huc to the name so I account for that here.
-        
-                # Define outputs for agreement_raster, stats_json, and stats_csv.
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-                else:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-         
-                compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                       benchmark_raster_path,
-                                                       agreement_raster,
-                                                       stats_csv=stats_csv,
-                                                       stats_json=stats_json,
-                                                       mask_values=[],
-                                                       stats_modes_list=stats_modes_list,
-                                                       test_id=test_id,
-                                                       mask_dict=mask_dict,
-                                                       )
-        
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    del mask_dict[ahps_lid]
-                
-                print(" ")
-                print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC)
-                print(" ")
-            except Exception as e:
-                print(e)      
-        
-        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-            # -- Delete temp files -- #
-            # List all files in the output directory.
-            output_file_list = os.listdir(version_test_case_dir)
-            for output_file in output_file_list:
-                if "total_area" in output_file:
-                    full_output_file_path = os.path.join(version_test_case_dir, output_file)
-                    os.remove(full_output_file_path)
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-b', '--version',help='The name of the working version in which features are being tested',required=True,default="")
-    parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
-    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
-    parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
-    parser.add_argument('-c', '--compare-to-previous', help='Compare to previous versions of HAND.', required=False,action='store_true')
-    parser.add_argument('-a', '--archive-results', help='Automatically copy results to the "previous_version" archive for test_id. For admin use only.', required=False,action='store_true')
-    parser.add_argument('-i', '--inclusion-area', help='Path to shapefile. Contingency metrics will be produced from pixels inside of shapefile extent.', required=False, default="")
-    parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
-    parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
-    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False)
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    valid_test_id_list = os.listdir(TEST_CASES_DIR)
-
-    exit_flag = False  # Default to False.
-    print()
-
-    # Ensure test_id is valid.
-#    if args['test_id'] not in valid_test_id_list:
-#        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
-#        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
-#        for test_id in valid_test_id_list:
-#          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
-#              print(CYAN_BOLD + test_id + ENDC)
-#        print()
-#        exit_flag = True
-
-    # Ensure fim_run_dir exists.
-    if not os.path.exists(os.path.join(os.environ['outputDataDir'], args['fim_run_dir'])):
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
-        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
-        print()
-        exit_flag = True
-    
-    # Ensure inclusion_area path exists.
-    if args['inclusion_area'] != "" and not os.path.exists(args['inclusion_area']):
-        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area (-i) " + CYAN_BOLD + args['inclusion_area'] + WHITE_BOLD + " could not be located." + ENDC)
-        exit_flag = True
-        
-    try:
-        inclusion_buffer = int(args['inclusion_area_buffer'])
-    except ValueError:
-        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area_buffer (-ib) " + CYAN_BOLD + args['inclusion_area_buffer'] + WHITE_BOLD + " is not a round number." + ENDC)
-
-    if args['magnitude'] == '':
-        if 'ble' in args['test_id'].split('_'):
-            args['magnitude'] = ['100yr', '500yr']
-        elif 'ahps' in args['test_id'].split('_'):
-            args['magnitude'] = ['action', 'minor', 'moderate', 'major']
-        else:
-            print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
-            exit_flag = True     
-            
-    if exit_flag:
-        print()
-        sys.exit()
-
-    else:
-        run_alpha_test(**args)
diff --git a/tests/synthesize_test_cases.py b/tests/synthesize_test_cases.py
deleted file mode 100644
index 1fdb0a4dc..000000000
--- a/tests/synthesize_test_cases.py
+++ /dev/null
@@ -1,311 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import argparse
-from multiprocessing import Pool
-import json
-import csv
-
-from run_test_case import run_alpha_test
-from utils.shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES
-
-
-def create_master_metrics_csv(master_metrics_csv_output):
-            
-    # Construct header
-    metrics_to_write = ['true_negatives_count',
-                        'false_negatives_count',
-                        'true_positives_count',
-                        'false_positives_count',
-                        'contingency_tot_count',
-                        'cell_area_m2',
-                        'TP_area_km2',
-                        'FP_area_km2',
-                        'TN_area_km2',
-                        'FN_area_km2',
-                        'contingency_tot_area_km2',
-                        'predPositive_area_km2',
-                        'predNegative_area_km2',
-                        'obsPositive_area_km2',
-                        'obsNegative_area_km2',
-                        'positiveDiff_area_km2',
-                        'CSI',
-                        'FAR',
-                        'TPR',
-                        'TNR',
-                        'PPV',
-                        'NPV',
-                        'ACC',
-                        'Bal_ACC',
-                        'MCC',
-                        'EQUITABLE_THREAT_SCORE',
-                        'PREVALENCE',
-                        'BIAS',
-                        'F1_SCORE',
-                        'TP_perc',
-                        'FP_perc',
-                        'TN_perc',
-                        'FN_perc',
-                        'predPositive_perc',
-                        'predNegative_perc',
-                        'obsPositive_perc',
-                        'obsNegative_perc',
-                        'positiveDiff_perc',
-                        'masked_count',
-                        'masked_perc',
-                        'masked_area_km2'
-                        ]
-    
-    additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
-    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
-    
-    versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
-    
-    for benchmark_source in ['ble', 'nws', 'usgs']:
-        
-        benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
-        
-        if benchmark_source == 'ble':
-            test_cases_list = os.listdir(benchmark_test_case_dir)
-                            
-            for test_case in test_cases_list:
-                try:
-                    int(test_case.split('_')[0])
-                    
-                    huc = test_case.split('_')[0]
-                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-                    
-                    for magnitude in ['100yr', '500yr']:
-                        for version in versions_to_aggregate:
-                            if '_fr' in version:
-                                extent_config = 'FR'
-                            elif '_ms' in version:
-                                extent_config = 'MS'
-                            else:
-                                extent_config = 'FR'
-                            if "_c" in version and version.split('_c')[1] == "":
-                                calibrated = "yes"
-                            else:
-                                calibrated = "no"
-                            version_dir = os.path.join(official_versions, version)
-                            magnitude_dir = os.path.join(version_dir, magnitude)
-
-                            if os.path.exists(magnitude_dir):
-                                magnitude_dir_list = os.listdir(magnitude_dir)
-                                for f in magnitude_dir_list:
-                                    if '.json' in f:
-                                        flow = 'NA'
-                                        nws_lid = "NA"
-                                        benchmark_source = 'ble'
-                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
-                                        full_json_path = os.path.join(magnitude_dir, f)
-                                        if os.path.exists(full_json_path):
-                                            stats_dict = json.load(open(full_json_path))
-                                            for metric in metrics_to_write:
-                                                sub_list_to_append.append(stats_dict[metric])
-                                            sub_list_to_append.append(full_json_path)
-                                            sub_list_to_append.append(flow)
-                                            sub_list_to_append.append(benchmark_source)
-                                            sub_list_to_append.append(extent_config)
-                                            sub_list_to_append.append(calibrated)
-                                            
-                                            list_to_write.append(sub_list_to_append)
-                except ValueError:
-                    pass
-                
-        if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
-            test_cases_list = os.listdir(benchmark_test_case_dir)
-
-            for test_case in test_cases_list:
-                try:
-                    int(test_case.split('_')[0])
-                    
-                    huc = test_case.split('_')[0]
-                    official_versions = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-                    
-                    for magnitude in ['action', 'minor', 'moderate', 'major']:
-                        for version in versions_to_aggregate:
-                            if '_fr' in version:
-                                extent_config = 'FR'
-                            elif '_ms' in version:
-                                extent_config = 'MS'
-                            else:
-                                extent_config = 'FR'
-                            if "_c" in version and version.split('_c')[1] == "":
-                                calibrated = "yes"
-                            else:
-                                calibrated = "no"
-                                
-                            version_dir = os.path.join(official_versions, version)
-                            magnitude_dir = os.path.join(version_dir, magnitude)
-                            if os.path.exists(magnitude_dir):
-                                magnitude_dir_list = os.listdir(magnitude_dir)
-                                for f in magnitude_dir_list:
-                                    if '.json' in f and 'total_area' not in f:
-                                        nws_lid = f[:5]
-                                        sub_list_to_append = [version, nws_lid, magnitude, huc]
-                                        full_json_path = os.path.join(magnitude_dir, f)
-                                        flow = ''
-                                        if os.path.exists(full_json_path):
-                                            
-                                            # Get flow used to map.                                                
-                                            flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
-                                            if os.path.exists(flow_file):
-                                                with open(flow_file, newline='') as csv_file:
-                                                    reader = csv.reader(csv_file)
-                                                    next(reader)
-                                                    for row in reader:
-                                                        flow = row[1]
-                                                    if nws_lid == 'mcc01':
-                                                        print(flow)
-                                            
-                                            stats_dict = json.load(open(full_json_path))
-                                            for metric in metrics_to_write:
-                                                sub_list_to_append.append(stats_dict[metric])
-                                            sub_list_to_append.append(full_json_path)
-                                            sub_list_to_append.append(flow)
-                                            sub_list_to_append.append(benchmark_source)
-                                            sub_list_to_append.append(extent_config)
-                                            sub_list_to_append.append(calibrated)
-                                            
-                                            list_to_write.append(sub_list_to_append)
-                except ValueError:
-                    pass
-        
-    with open(master_metrics_csv_output, 'w', newline='') as csvfile:
-        csv_writer = csv.writer(csvfile)
-        csv_writer.writerows(list_to_write)
-
-
-def process_alpha_test(args):
-    
-    fim_run_dir = args[0]
-    version = args[1]
-    test_id = args[2]
-    magnitude = args[3]
-    archive_results = args[4]
-    overwrite = args[5]
-    
-    mask_type = 'huc'    
-    
-    if archive_results == False:
-        compare_to_previous = True
-    else:
-        compare_to_previous = False
-
-    try:
-        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type, overwrite=overwrite)
-    except Exception as e:
-        print(e)
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
-    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
-    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
-    parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
-    parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
-    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
-    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=True)
-    
-    # Assign variables from arguments.
-    args = vars(parser.parse_args())
-    config = args['config']
-    fim_version = args['fim_version']
-    job_number = int(args['job_number'])
-    special_string = args['special_string']
-    benchmark_category = args['benchmark_category']
-    overwrite = args['overwrite']
-    master_metrics_csv = args['master_metrics_csv']
-    
-    if overwrite:
-        if input("Are you sure you want to overwrite metrics? y/n: ") == "n":
-            quit
-        
-    # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
-    if fim_version != "all":
-        previous_fim_list = [fim_version]
-    else:
-        if config == 'PREV':
-            previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)
-        elif config == 'DEV':
-            previous_fim_list = os.listdir(OUTPUTS_DIR)
-    
-    # Define whether or not to archive metrics in "official_versions" or "testing_versions" for each test_id.
-    if config == 'PREV':
-        archive_results = True
-    elif config == 'DEV':
-        archive_results = False
-    else:
-        print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
-     
-    # List all available benchmark categories and test_cases.
-    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
-    benchmark_category_list = []
-    if benchmark_category == "all":
-        for d in test_cases_dir_list:
-            if 'test_cases' in d:
-                benchmark_category_list.append(d.replace('_test_cases', ''))
-    else:
-        benchmark_category_list = [benchmark_category]
-        
-    # Loop through benchmark categories.
-    procs_list = []
-    for bench_cat in benchmark_category_list:
-        
-        # Map path to appropriate test_cases folder and list test_ids into bench_cat_id_list.
-        bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
-        bench_cat_id_list = os.listdir(bench_cat_test_case_dir)
-    
-        # Loop through test_ids in bench_cat_id_list.
-        for test_id in bench_cat_id_list:
-            if 'validation' and 'other' not in test_id:
-                current_huc = test_id.split('_')[0]
-                if test_id.split('_')[1] in bench_cat:
-                
-                    # Loop through versions.
-                    for version in previous_fim_list:
-                        if config == 'DEV':
-                            fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
-                        elif config == 'PREV':
-                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
-                                                
-                        # For previous versions of HAND computed at HUC6 scale
-                        if not os.path.exists(fim_run_dir):
-                            if config == 'DEV':
-                                fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
-                            elif config == 'PREV':
-                                fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  
-                        
-                        if os.path.exists(fim_run_dir):
-                            
-                            # If a user supplies a specia_string (-s), then add it to the end of the created dirs.
-                            if special_string != "":
-                                version = version + '_' + special_string
-                            
-                            # Define the magnitude lists to use, depending on test_id.
-                            if 'ble' in test_id:
-                                magnitude = ['100yr', '500yr']
-                            elif 'usgs' or 'nws' in test_id:
-                                magnitude = ['action', 'minor', 'moderate', 'major']
-                            else:
-                                continue
-                        
-                            # Either add to list to multiprocess or process serially, depending on user specification.
-                            if job_number > 1:
-                                procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
-                            else:                            
-                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results, overwrite])
-
-    # Multiprocess alpha test runs.
-    if job_number > 1:
-        pool = Pool(job_number)
-        pool.map(process_alpha_test, procs_list)
-    
-    # Do aggregate_metrics.
-    print("Creating master metrics CSV...")
-    create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv)
-    
\ No newline at end of file
diff --git a/tests/utils/shared_functions.py b/tests/utils/shared_functions.py
deleted file mode 100644
index d36c22814..000000000
--- a/tests/utils/shared_functions.py
+++ /dev/null
@@ -1,676 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import json
-import csv
-import rasterio
-import pandas as pd
-from utils.shared_variables import (TEST_CASES_DIR, PRINTWORTHY_STATS, GO_UP_STATS, GO_DOWN_STATS,
-                                    ENDC, TGREEN_BOLD, TGREEN, TRED_BOLD, TWHITE, WHITE_BOLD, CYAN_BOLD)
-
-def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
-
-    difference_dict = {}
-
-    # Compare stats_csv to previous_version_stats_file
-    stats_dict_to_test = json.load(open(stats_json_to_test))
-    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
-
-    for stat, value in stats_dict_to_test.items():
-        previous_version_value = previous_version_stats_dict[stat]
-        stat_value_diff = value - previous_version_value
-        difference_dict.update({stat + '_diff': stat_value_diff})
-
-    return difference_dict
-
-
-def compute_contingency_stats_from_rasters(predicted_raster_path, benchmark_raster_path, agreement_raster=None, stats_csv=None, stats_json=None, mask_values=None, stats_modes_list=['total_area'], test_id='', mask_dict={}):
-    """
-    This function contains FIM-specific logic to prepare raster datasets for use in the generic get_contingency_table_from_binary_rasters() function.
-    This function also calls the generic compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
-
-    Args:
-        predicted_raster_path (str): The path to the predicted, or modeled, FIM extent raster.
-        benchmark_raster_path (str): The path to the benchmark, or truth, FIM extent raster.
-        agreement_raster (str): Optional. An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive, 3: True Positive.
-        stats_csv (str): Optional. Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
-        stats_json (str): Optional. Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary in other processes.
-
-    Returns:
-        stats_dictionary (dict): A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and statistic values are the values.
-    """
-
-    # Get cell size of benchmark raster.
-    raster = rasterio.open(predicted_raster_path)
-    t = raster.transform
-    cell_x = t[0]
-    cell_y = t[4]
-    cell_area = abs(cell_x*cell_y)
-
-    # Get contingency table from two rasters.
-    contingency_table_dictionary = get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster, mask_values=mask_values, mask_dict=mask_dict)
-
-    stats_dictionary = {}
-
-    for stats_mode in contingency_table_dictionary:
-        true_negatives = contingency_table_dictionary[stats_mode]['true_negatives']
-        false_negatives = contingency_table_dictionary[stats_mode]['false_negatives']
-        false_positives = contingency_table_dictionary[stats_mode]['false_positives']
-        true_positives = contingency_table_dictionary[stats_mode]['true_positives']
-        masked_count = contingency_table_dictionary[stats_mode]['masked_count']
-        file_handle = contingency_table_dictionary[stats_mode]['file_handle']
-
-        # Produce statistics from continency table and assign to dictionary. cell_area argument optional (defaults to None).
-        mode_stats_dictionary = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area, masked_count)
-
-        # Write the mode_stats_dictionary to the stats_csv.
-        if stats_csv != None:
-            stats_csv = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.csv')
-            df = pd.DataFrame.from_dict(mode_stats_dictionary, orient="index", columns=['value'])
-            df.to_csv(stats_csv)
-
-        # Write the mode_stats_dictionary to the stats_json.
-        if stats_json != None:
-            stats_json = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.json')
-            with open(stats_json, "w") as outfile:
-                json.dump(mode_stats_dictionary, outfile)
-
-        stats_dictionary.update({stats_mode: mode_stats_dictionary})
-
-    return stats_dictionary
-
-
-def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
-    """
-    This function searches multiple directories and locates previously produced performance statistics.
-
-    Args:
-        archive_to_check (str): The directory path to search.
-        magnitude (str): Because a benchmark dataset may have multiple magnitudes, this argument defines
-                               which magnitude is to be used when searching for previous statistics.
-    Returns:
-        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain and magnitude.
-                                  {version: {agreement_raster: agreement_raster_path, stats_csv: stats_csv_path, stats_json: stats_json_path}}
-                                  *Will only add the paths to files that exist.
-
-    """
-
-    archive_dictionary = {}
-
-    # List through previous version and check for available stats and maps. If available, add to dictionary.
-    available_versions_list = os.listdir(archive_to_check)
-
-    if len(available_versions_list) == 0:
-        print("Cannot compare with -c flag because there are no data in the previous_versions directory.")
-        return
-
-    for version in available_versions_list:
-        version_magnitude_dir = os.path.join(archive_to_check, version, magnitude)
-        stats_json = os.path.join(version_magnitude_dir, stats_mode + '_stats.json')
-
-        if os.path.exists(stats_json):
-            archive_dictionary.update({version: {'stats_json': stats_json}})
-
-    return archive_dictionary
-
-
-#def compare_to_previous(benchmark_category, test_id, stats_modes_list, magnitude, version, test_version_dictionary, version_test_case_dir):
-#    text_block = []
-#    # Compare to previous stats files that are available.
-#    archive_to_check = os.path.join(TEST_CASES_DIR, benchmark_category + 'test_cases', test_id, 'official_versions')
-#    for stats_mode in stats_modes_list:
-#        archive_dictionary = profile_test_case_archive(archive_to_check, magnitude, stats_mode)
-#
-#        if archive_dictionary == {}:
-#            break
-#
-#        # Create header for section.
-#        header = [stats_mode]
-#        for previous_version, paths in archive_dictionary.items():
-#            header.append(previous_version)
-#        header.append(version)
-#        text_block.append(header)
-#
-#        # Loop through stats in PRINTWORTHY_STATS for left.
-#        for stat in PRINTWORTHY_STATS:
-#            stat_line = [stat]
-#            for previous_version, paths in archive_dictionary.items():
-#                # Load stats for previous version.
-#                previous_version_stats_json_path = paths['stats_json']
-#                if os.path.exists(previous_version_stats_json_path):
-#                    previous_version_stats_dict = json.load(open(previous_version_stats_json_path))
-#
-#                    # Append stat for the version to state_line.
-#                    stat_line.append(previous_version_stats_dict[stat])
-#
-#
-#            # Append stat for the current version to stat_line.
-#            stat_line.append(test_version_dictionary[stats_mode][stat])
-#
-#            text_block.append(stat_line)
-#
-#        text_block.append([" "])
-#
-#    regression_report_csv = os.path.join(version_test_case_dir, 'stats_summary.csv')
-#    with open(regression_report_csv, 'w', newline='') as csvfile:
-#        csv_writer = csv.writer(csvfile)
-#        csv_writer.writerows(text_block)
-#
-#    print()
-#    print("--------------------------------------------------------------------------------------------------")
-#    
-#    stats_mode = stats_modes_list[0]
-#    try:
-#        last_version_index = text_block[0].index('dev_latest')
-#    except ValueError:
-#        try:
-#            last_version_index = text_block[0].index('fim_2_3_3')
-#        except ValueError:
-#            try:
-#                last_version_index = text_block[0].index('fim_1_0_0')
-#            except ValueError:
-#                print(TRED_BOLD + "Warning: " + ENDC + "Cannot compare " + version + " to a previous version because no authoritative versions were found in previous_versions directory. Future version of run_test_case may allow for comparisons between dev versions.")
-#                print()
-#                continue
-#
-#    
-#
-#    for line in text_block:
-#        first_item = line[0]
-#        if first_item in stats_modes_list:
-#            current_version_index = line.index(version)
-#            if first_item != stats_mode:  # Update the stats_mode and print a separator.
-#                print()
-#                print()
-#                print("--------------------------------------------------------------------------------------------------")
-#            print()
-#            stats_mode = first_item
-#            print(CYAN_BOLD + current_huc + ": " + magnitude.upper(), ENDC)
-#            print(CYAN_BOLD + stats_mode.upper().replace('_', ' ') + " METRICS" + ENDC)
-#            print()
-#
-#            color = WHITE_BOLD
-#            metric_name = '      '.center(len(max(PRINTWORTHY_STATS, key=len)))
-#            percent_change_header = '% CHG'
-#            difference_header = 'DIFF'
-#            current_version_header = line[current_version_index].upper()
-#            last_version_header = line[last_version_index].upper()
-#            # Print Header.
-#            print(color + metric_name + "      " + percent_change_header.center((7)) + "       " + difference_header.center((15))  + "    " + current_version_header.center(18) + " " + last_version_header.center(18), ENDC)
-#        # Format and print stat row.
-#        elif first_item in PRINTWORTHY_STATS:
-#            stat_name = first_item.upper().center(len(max(PRINTWORTHY_STATS, key=len))).replace('_', ' ')
-#            current_version = round((line[current_version_index]), 3)
-#            last_version = round((line[last_version_index]) + 0.000, 3)
-#            difference = round(current_version - last_version, 3)
-#            if difference > 0:
-#                symbol = '+'
-#                if first_item in GO_UP_STATS:
-#                    color = TGREEN_BOLD
-#                elif first_item in GO_DOWN_STATS:
-#                    color = TRED_BOLD
-#                else:
-#                    color = TWHITE
-#            if difference < 0:
-#                symbol = '-'
-#                if first_item in GO_UP_STATS:
-#                    color = TRED_BOLD
-#                elif first_item in GO_DOWN_STATS:
-#                    color = TGREEN_BOLD
-#                else:
-#                    color = TWHITE
-#
-#            if difference == 0 :
-#                symbol, color = '+', TGREEN
-#            percent_change = round((difference / last_version)*100,2)
-#
-#            print(WHITE_BOLD + stat_name + ENDC + "     " + color + (symbol + " {:5.2f}".format(abs(percent_change)) + " %").rjust(len(percent_change_header)), ENDC + "    " + color + ("{:12.3f}".format((difference))).rjust(len(difference_header)), ENDC + "    " + "{:15.3f}".format(current_version).rjust(len(current_version_header)) + "   " + "{:15.3f}".format(last_version).rjust(len(last_version_header)) + "  ")
-#
-#    print()
-#    print()
-#    print()
-#    print("--------------------------------------------------------------------------------------------------")
-#    print()
-#
-
-
-def compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=None, masked_count=None):
-    """
-    This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.
-    Much of the calculations below were taken from older Python files. This is evident in the inconsistent use of case.
-    
-    Args:
-        true_negatives (int): The true negatives from a contingency table.
-        false_negatives (int): The false negatives from a contingency table.
-        false_positives (int): The false positives from a contingency table.
-        true_positives (int): The true positives from a contingency table.
-        cell_area (float or None): This optional argument allows for area-based statistics to be calculated, in the case that
-                                   contingency table metrics were derived from areal analysis.
-    
-    Returns:
-        stats_dictionary (dict): A dictionary of statistics. Statistic names are keys and statistic values are the values.
-                                 Refer to dictionary definition in bottom of function for statistic names.
-    
-    """
-    
-    import numpy as np
-    
-    total_population = true_negatives + false_negatives + false_positives + true_positives
-        
-    # Basic stats.
-#    Percent_correct = ((true_positives + true_negatives) / total_population) * 100
-#    pod             = true_positives / (true_positives + false_negatives)
-    
-    try:
-        FAR = false_positives / (true_positives + false_positives)
-    except ZeroDivisionError:
-        FAR = "NA"
-        
-    try:
-        CSI = true_positives / (true_positives + false_positives + false_negatives)
-    except ZeroDivisionError:
-        CSI = "NA"
-    
-    try:
-        BIAS = (true_positives + false_positives) / (true_positives + false_negatives)
-    except ZeroDivisionError:
-        BIAS = "NA"
-    
-    # Compute equitable threat score (ETS) / Gilbert Score. 
-    try:
-        a_ref = ((true_positives + false_positives)*(true_positives + false_negatives)) / total_population
-        EQUITABLE_THREAT_SCORE = (true_positives - a_ref) / (true_positives - a_ref + false_positives + false_negatives)
-    except ZeroDivisionError:
-        EQUITABLE_THREAT_SCORE = "NA"
-
-    if total_population == 0:
-        TP_perc, FP_perc, TN_perc, FN_perc = "NA", "NA", "NA", "NA"
-    else:
-        TP_perc = (true_positives / total_population) * 100
-        FP_perc = (false_positives / total_population) * 100
-        TN_perc = (true_negatives / total_population) * 100
-        FN_perc = (false_negatives / total_population) * 100
-    
-    predPositive = true_positives + false_positives
-    predNegative = true_negatives + false_negatives
-    obsPositive = true_positives + false_negatives
-    obsNegative = true_negatives + false_positives
-    
-    TP = float(true_positives)
-    TN = float(true_negatives)
-    FN = float(false_negatives)
-    FP = float(false_positives)
-    try:
-        MCC = (TP*TN - FP*FN)/ np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))
-    except ZeroDivisionError:
-        MCC = "NA"
-    
-    if masked_count != None:
-        total_pop_and_mask_pop = total_population + masked_count
-        if total_pop_and_mask_pop == 0:
-            masked_perc = "NA"
-        else:
-            masked_perc = (masked_count / total_pop_and_mask_pop) * 100
-    else:
-        masked_perc = None
-    
-    # This checks if a cell_area has been provided, thus making areal calculations possible.
-    sq_km_converter = 1000000
-        
-    if cell_area != None:
-        TP_area = (true_positives * cell_area) / sq_km_converter
-        FP_area = (false_positives * cell_area) / sq_km_converter
-        TN_area = (true_negatives * cell_area) / sq_km_converter
-        FN_area = (false_negatives * cell_area) / sq_km_converter
-        area = (total_population * cell_area) / sq_km_converter
-        
-        predPositive_area = (predPositive * cell_area) / sq_km_converter
-        predNegative_area = (predNegative * cell_area) / sq_km_converter
-        obsPositive_area =  (obsPositive * cell_area) / sq_km_converter
-        obsNegative_area =  (obsNegative * cell_area) / sq_km_converter
-        positiveDiff_area = predPositive_area - obsPositive_area
-        
-        if masked_count != None:
-            masked_area = (masked_count * cell_area) / sq_km_converter
-        else:
-            masked_area = None
-
-    # If no cell_area is provided, then the contingeny tables are likely not derived from areal analysis.
-    else:
-        TP_area = None
-        FP_area = None
-        TN_area = None
-        FN_area = None
-        area = None
-        
-        predPositive_area = None
-        predNegative_area = None
-        obsPositive_area =  None
-        obsNegative_area =  None
-        positiveDiff_area = None
-        MCC = None
-        
-    if total_population == 0:
-        predPositive_perc, predNegative_perc, obsPositive_perc, obsNegative_perc , positiveDiff_perc = "NA", "NA", "NA", "NA", "NA"
-    else:
-        predPositive_perc = (predPositive / total_population) * 100
-        predNegative_perc = (predNegative / total_population) * 100
-        obsPositive_perc = (obsPositive / total_population) * 100
-        obsNegative_perc = (obsNegative / total_population) * 100
-        
-        positiveDiff_perc = predPositive_perc - obsPositive_perc
-        
-    if total_population == 0:
-        prevalence = "NA"
-    else:
-        prevalence = (true_positives + false_negatives) / total_population
-    
-    try:
-        PPV = true_positives / predPositive
-    except ZeroDivisionError:
-        PPV = "NA"
-    
-    try:
-        NPV = true_negatives / predNegative
-    except ZeroDivisionError:
-        NPV = "NA"
-    
-    try:
-        TNR = true_negatives / obsNegative
-    except ZeroDivisionError:
-        TNR = "NA"
-        
-    try:
-        TPR = true_positives / obsPositive
-        
-    except ZeroDivisionError:
-        TPR = "NA"
-
-    try:
-        Bal_ACC = np.mean([TPR,TNR])
-    except TypeError:
-        Bal_ACC = "NA"
-    
-    if total_population == 0:
-        ACC = "NA"
-    else:
-        ACC = (true_positives + true_negatives) / total_population
-    
-    try:
-        F1_score = (2*true_positives) / (2*true_positives + false_positives + false_negatives)
-    except ZeroDivisionError:
-        F1_score = "NA"
-
-    stats_dictionary = {'true_negatives_count': int(true_negatives),
-                        'false_negatives_count': int(false_negatives),
-                        'true_positives_count': int(true_positives),
-                        'false_positives_count': int(false_positives),
-                        'contingency_tot_count': int(total_population),
-                        'cell_area_m2': cell_area,
-                        
-                        'TP_area_km2': TP_area,
-                        'FP_area_km2': FP_area,
-                        'TN_area_km2': TN_area,
-                        'FN_area_km2': FN_area,
-
-                        'contingency_tot_area_km2': area,
-                        'predPositive_area_km2': predPositive_area,
-                        'predNegative_area_km2': predNegative_area,
-                        'obsPositive_area_km2': obsPositive_area,
-                        'obsNegative_area_km2': obsNegative_area,
-                        'positiveDiff_area_km2': positiveDiff_area,
-
-                        'CSI': CSI,
-                        'FAR': FAR,
-                        'TPR': TPR,                        
-                        'TNR': TNR,                        
-                        
-                        'PPV': PPV,
-                        'NPV': NPV,
-                        'ACC': ACC,
-                        'Bal_ACC': Bal_ACC,
-                        'MCC': MCC,
-                        'EQUITABLE_THREAT_SCORE': EQUITABLE_THREAT_SCORE, 
-                        'PREVALENCE': prevalence,
-                        'BIAS': BIAS,
-                        'F1_SCORE': F1_score,
-
-                        'TP_perc': TP_perc,
-                        'FP_perc': FP_perc,
-                        'TN_perc': TN_perc,
-                        'FN_perc': FN_perc,
-                        'predPositive_perc': predPositive_perc,
-                        'predNegative_perc': predNegative_perc,
-                        'obsPositive_perc': obsPositive_perc,
-                        'obsNegative_perc': obsNegative_perc,
-                        'positiveDiff_perc': positiveDiff_perc,
-  
-                        'masked_count': int(masked_count),
-                        'masked_perc': masked_perc,
-                        'masked_area_km2': masked_area,
-                        
-                        }
-
-    return stats_dictionary
-
-
-def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster=None, mask_values=None, mask_dict={}):
-    """
-    Produces contingency table from 2 rasters and returns it. Also exports an agreement raster classified as:
-        0: True Negatives
-        1: False Negative
-        2: False Positive
-        3: True Positive
-        
-    Args:
-        benchmark_raster_path (str): Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
-        predicted_raster_path (str): Path to the predicted raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
-    
-    Returns:
-        contingency_table_dictionary (dict): A Python dictionary of a contingency table. Key/value pair formatted as:
-                                            {true_negatives: int, false_negatives: int, false_positives: int, true_positives: int}
-    
-    """
-    from rasterio.warp import reproject, Resampling
-    import rasterio
-    import numpy as np
-    import os
-    import rasterio.mask
-    import geopandas as gpd
-    from shapely.geometry import box
-            
-    print("-----> Evaluating performance across the total area...")
-    # Load rasters.
-    benchmark_src = rasterio.open(benchmark_raster_path)
-    predicted_src = rasterio.open(predicted_raster_path)
-    predicted_array = predicted_src.read(1)
-    
-    benchmark_array_original = benchmark_src.read(1)
-    
-    if benchmark_array_original.shape != predicted_array.shape:
-        benchmark_array = np.empty(predicted_array.shape, dtype=np.int8)
-        
-        reproject(benchmark_array_original, 
-              destination = benchmark_array,
-              src_transform = benchmark_src.transform, 
-              src_crs = benchmark_src.crs,
-              src_nodata = benchmark_src.nodata,
-              dst_transform = predicted_src.transform, 
-              dst_crs = predicted_src.crs,
-              dst_nodata = benchmark_src.nodata,
-              dst_resolution = predicted_src.res,
-              resampling = Resampling.nearest)
-    
-    predicted_array_raw = predicted_src.read(1)
-    
-    # Align the benchmark domain to the modeled domain.
-    benchmark_array = np.where(predicted_array==predicted_src.nodata, 10, benchmark_array)
-            
-    # Ensure zeros and ones for binary comparison. Assume that positive values mean flooding and 0 or negative values mean dry. 
-    predicted_array = np.where(predicted_array==predicted_src.nodata, 10, predicted_array)  # Reclassify NoData to 10
-    predicted_array = np.where(predicted_array<0, 0, predicted_array)
-    predicted_array = np.where(predicted_array>0, 1, predicted_array)
-    
-    benchmark_array = np.where(benchmark_array==benchmark_src.nodata, 10, benchmark_array)  # Reclassify NoData to 10        
-
-    agreement_array = np.add(benchmark_array, 2*predicted_array)
-    agreement_array = np.where(agreement_array>4, 10, agreement_array)
-    
-    del benchmark_src, benchmark_array, predicted_array, predicted_array_raw
-
-    # Loop through exclusion masks and mask the agreement_array.
-    if mask_dict != {}:
-        for poly_layer in mask_dict:
-            
-            operation = mask_dict[poly_layer]['operation']
-            
-            if operation == 'exclude':
-                
-                poly_path = mask_dict[poly_layer]['path']
-                buffer_val = mask_dict[poly_layer]['buffer']
-                
-                reference = predicted_src
-                            
-                bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
-                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).  
-                poly_all = gpd.read_file(poly_path, bbox = bounding_box)
-                
-                # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
-                if poly_all.empty:
-                    continue
-                
-                print("-----> Masking at " + poly_layer + "...")
-                #Project layer to reference crs.
-                poly_all_proj = poly_all.to_crs(reference.crs)
-                # check if there are any lakes within our reference raster extent.
-                if poly_all_proj.empty: 
-                    #If no features within reference raster extent, create a zero array of same shape as reference raster.
-                    poly_mask = np.zeros(reference.shape)
-                else:
-                    #Check if a buffer value is passed to function.
-                    if buffer_val is None:
-                        #If features are present and no buffer is passed, assign geometry to variable.
-                        geometry = poly_all_proj.geometry
-                    else:
-                        #If  features are present and a buffer is passed, assign buffered geometry to variable.
-                        geometry = poly_all_proj.buffer(buffer_val)
-                        
-                    #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
-                    in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
-                    #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
-                    poly_mask = np.where(in_poly == True, 1,0)
-
-                    # Perform mask.
-                    masked_agreement_array = np.where(poly_mask == 1, 4, agreement_array)
-                    
-                    # Get rid of masked values outside of the modeled domain.
-                    agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
-                    
-    contingency_table_dictionary = {}  # Initialize empty dictionary.
-    
-    # Only write the agreement raster if user-specified.
-    if agreement_raster != None:
-        with rasterio.Env():
-            profile = predicted_src.profile
-            profile.update(nodata=10)
-            with rasterio.open(agreement_raster, 'w', **profile) as dst:
-                dst.write(agreement_array, 1)
-         
-        # Write legend text file
-        legend_txt = os.path.join(os.path.split(agreement_raster)[0], 'read_me.txt')
-        
-        from datetime import datetime
-        
-        now = datetime.now()
-        current_time = now.strftime("%m/%d/%Y %H:%M:%S")
-                
-        with open(legend_txt, 'w') as f:
-            f.write("%s\n" % '0: True Negative')
-            f.write("%s\n" % '1: False Negative')
-            f.write("%s\n" % '2: False Positive')
-            f.write("%s\n" % '3: True Positive')
-            f.write("%s\n" % '4: Masked area (excluded from contingency table analysis). Mask layers: {mask_dict}'.format(mask_dict=mask_dict))
-            f.write("%s\n" % 'Results produced at: {current_time}'.format(current_time=current_time))
-                          
-    # Store summed pixel counts in dictionary.
-    contingency_table_dictionary.update({'total_area':{'true_negatives': int((agreement_array == 0).sum()),
-                                                      'false_negatives': int((agreement_array == 1).sum()),
-                                                      'false_positives': int((agreement_array == 2).sum()),
-                                                      'true_positives': int((agreement_array == 3).sum()),
-                                                      'masked_count': int((agreement_array == 4).sum()),
-                                                      'file_handle': 'total_area'
-                                                     
-                                                      }})                               
-        
-    # After agreement_array is masked with default mask layers, check for inclusion masks in mask_dict.
-    if mask_dict != {}:
-        for poly_layer in mask_dict:
-            
-            operation = mask_dict[poly_layer]['operation']
-            
-            if operation == 'include':
-                poly_path = mask_dict[poly_layer]['path']
-                buffer_val = mask_dict[poly_layer]['buffer']
-
-                reference = predicted_src
-                            
-                bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
-                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).  
-                poly_all = gpd.read_file(poly_path, bbox = bounding_box)
-                
-                # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
-                if poly_all.empty:
-                    continue
-                
-                print("-----> Evaluating performance at " + poly_layer + "...")
-                #Project layer to reference crs.
-                poly_all_proj = poly_all.to_crs(reference.crs)
-                # check if there are any lakes within our reference raster extent.
-                if poly_all_proj.empty: 
-                    #If no features within reference raster extent, create a zero array of same shape as reference raster.
-                    poly_mask = np.zeros(reference.shape)
-                else:
-                    #Check if a buffer value is passed to function.
-                    if buffer_val is None:
-                        #If features are present and no buffer is passed, assign geometry to variable.
-                        geometry = poly_all_proj.geometry
-                    else:
-                        #If  features are present and a buffer is passed, assign buffered geometry to variable.
-                        geometry = poly_all_proj.buffer(buffer_val)
-                        
-                    #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
-                    in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
-                    #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
-                    poly_mask = np.where(in_poly == True, 1, 0)
-
-                    # Perform mask.
-                    masked_agreement_array = np.where(poly_mask == 0, 4, agreement_array)  # Changed to poly_mask == 0
-                    
-                    # Get rid of masked values outside of the modeled domain.
-                    temp_agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
-                    
-                    if buffer_val == None:  # The buffer used is added to filename, and 0 is easier to read than None.
-                        buffer_val = 0
-                    
-                    poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
-                    
-                    # Write the layer_agreement_raster.
-                    layer_agreement_raster = os.path.join(os.path.split(agreement_raster)[0], poly_handle + '_agreement.tif')
-                    with rasterio.Env():
-                        profile = predicted_src.profile
-                        profile.update(nodata=10)
-                        with rasterio.open(layer_agreement_raster, 'w', **profile) as dst:
-                            dst.write(temp_agreement_array, 1)
-                    
-
-                    # Store summed pixel counts in dictionary.
-                    contingency_table_dictionary.update({poly_handle:{'true_negatives': int((temp_agreement_array == 0).sum()),
-                                                                     'false_negatives': int((temp_agreement_array == 1).sum()),
-                                                                     'false_positives': int((temp_agreement_array == 2).sum()),
-                                                                     'true_positives': int((temp_agreement_array == 3).sum()),
-                                                                     'masked_count': int((temp_agreement_array == 4).sum()),
-                                                                     'file_handle': poly_handle
-                                                                      }})
-
-    return contingency_table_dictionary
-    
diff --git a/tools/consolidate_metrics.py b/tools/consolidate_metrics.py
index 20f084ef8..d9a80b9f2 100755
--- a/tools/consolidate_metrics.py
+++ b/tools/consolidate_metrics.py
@@ -9,7 +9,7 @@
 from tqdm import tqdm
 import argparse
 from collections import defaultdict
-from shared_variables import TEST_CASES_DIR,\
+from tools_shared_variables import TEST_CASES_DIR,\
                              PREVIOUS_FIM_DIR,\
                              OUTPUTS_DIR,\
                              INPUTS_DIR,\
diff --git a/tools/shared_variables.py b/tools/shared_variables.py
deleted file mode 100644
index 292f55de1..000000000
--- a/tools/shared_variables.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import os
-
-# Environmental variables and constants.
-TEST_CASES_DIR = r'/data/test_cases/'
-PREVIOUS_FIM_DIR = r'/data/previous_fim'
-OUTPUTS_DIR = os.environ['outputDataDir']
-INPUTS_DIR = r'/data/inputs'
-AHPS_BENCHMARK_CATEGORIES = ['usgs', 'nws']
-PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
-GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
-GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc']
-
-# Colors.
-ENDC = '\033[m'
-TGREEN_BOLD = '\033[32;1m'
-TGREEN = '\033[32m'
-TRED_BOLD = '\033[31;1m'
-TWHITE = '\033[37m'
-WHITE_BOLD = '\033[37;1m'
-CYAN_BOLD = '\033[36;1m'

From ad291bce4bfcd65a33dcaeeb9adeba649c31ab5e Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Tue, 28 Jun 2022 08:39:52 -0500
Subject: [PATCH 143/359] [1pt] PR: Alpha Test Refactor (#612)

* move DEM processing to unit level

* created  to run common processing between  and

* finalize branch zero

* updated alpha test to use test_case class from dev-fim3

* small bug fixes for the alpha test

* updated verbose printing, fix exception handing issue inside multi processing, created a generic re-usable verbose printing method and misc styling upgrades

* fix issue with multi-proc not working

* Update CHANGELOG.md

Co-authored-by: Rob Hanna <Robert.Hanna@NOAA.gov>
Co-authored-by: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
---
 docs/CHANGELOG.md                        |  29 +
 gms_run_branch.sh                        |   8 +-
 gms_run_unit.sh                          |   4 +-
 src/utils/shared_functions.py            | 110 ++--
 tools/composite_inundation.py            |   6 +-
 tools/gms_tools/inundate_gms.py          |  30 +-
 tools/gms_tools/mosaic_inundation.py     |  22 +-
 tools/run_test_case.py                   | 791 ++++++++++++-----------
 tools/synthesize_test_cases.py           | 229 +++----
 unit_tests/shared_functions_unittests.py |  11 +-
 10 files changed, 668 insertions(+), 572 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e63a7f639..4d91c8ad5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,35 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.5.1 - 2022-06-27 - [PR #612](https://github.com/NOAA-OWP/inundation-mapping/pull/612)
+
+`Alpha Test Refactor` An upgrade was made a few weeks back to the dev-fim3 branch that improved performance, usability and readability of running alpha tests. Some cleanup in other files for readability, debugging verbosity and styling were done as well. A newer, cleaner system for printing lines when the verbose flag is enabled was added.
+
+## Changes
+
+- `gms_run_branch.sh`:  Updated help instructions to about using multiple HUCs as command arguments.
+- `gms_run_unit.sh`:  Updated help instructions to about using multiple HUCs as command arguments.
+- `src/utils`
+    - `shared_functions.py`: 
+       - Added a new function called `vprint` which creates a simpler way (and better readability) for other python files when wanting to include a print line when the verbose flag is on.
+       - Added a new class named `FIM_Helpers` as a wrapper for the new `vprint` method. 
+       - With the new `FIM_Helpers` class, a previously existing method named `append_id_to_file_name` was moved into this class making it easier and quicker for usage in other classes.
+       
+- `tools`
+    - `composite_inundation.py`: Updated its usage of the `append_id_to_file_name` function to now call the`FIM_Helpers` method version of it.
+    - `gms_tools`
+       - `inundate_gms.py`: Updated for its adjusted usage of the `append_id_to_file_name` method, also removed its own `def __vprint` function in favour of the `FIM_Helpers.vprint` method. 
+       - `mosaic_inundation.py`: 
+          - Added adjustments for use of `append_id_to_file_name` and adjustments for `fh.vprint`.
+          - Fixed a bug for the variable `ag_mosaic_output` which was not pre-declared and would fail as using an undefined variable in certain conditions.
+    - `run_test_case.py`: Ported `test_case` class from FIM 3 and tweaked slightly to allow for GMS FIM. Also added more prints against the new fh.vprint method. Also added a default print line for progress / traceability for all alpha test regardless if the verbose flag is set.
+    - `synthesize_test_cases.py`: Ported `test_case` class from FIM 3.
+- `unit_tests`
+   - `shared_functions_unittests.py`: Update to match moving the `append_id_to_file_name` into the `FIM_Helpers` class. Also removed all "header print lines" for each unit test method (for output readability).
+
+<br/><br/>
+
 ## v4.0.5.0 - 2022-06-16 - [PR #611](https://github.com/NOAA-OWP/inundation-mapping/pull/611)
 
 'Branch zero' is a new branch that runs the HUCs full stream network to make up for stream orders 1 & 2 being skipped by the GMS solution and is similar to the FR extent in FIM v3. This new branch is created during `run_by_unit.sh` and the processed DEM is used by the other GMS branches during `run_by_branch.sh` to improve efficiency.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index a8cda2e50..43eb89e03 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -15,10 +15,10 @@ usage ()
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -r/--retry      : retries failed jobs'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -d/--denylist  : file with line delimited list of files in branches directories to remove upon completion'
-    echo '                   (see config/deny_gms_branches_default.lst for a starting point)'
-    echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
-    echo '                     also accepted. HUCs must present in inputs directory.'
+    echo '  -d/--denylist   : file with line delimited list of files in branches directories to remove '
+    echo '                    upon completion. ie) config/deny_gms_branches_default.lst'
+    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited).'
+    echo '                    A line delimited file also acceptable. HUCs must present in inputs directory.'
 	echo '  -s/--dropStreamOrders : If this flag is included, the system will leave out stream orders 1 and 2'
 	echo '                    at the initial load of the nwm_subset_streams'
     echo
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index fe61c3d65..31dc88357 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -7,8 +7,8 @@ usage ()
     echo '  	 				   [OPT: -h -j <job limit>] -o -r -d <deny list file> -s <drop stream orders 1 and 2>]'
     echo ''
     echo 'REQUIRED:'
-    echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
-    echo '                     also accepted. HUCs must present in inputs directory.'
+    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
+    echo '                    A line delimited file is also acceptable. HUCs must present in inputs directory.'
     echo '  -c/--config     : configuration file with bash environment variables to export'
     echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
     echo ''
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 653ad0245..ea464667a 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import os
+import os, inspect
 from os.path import splitext
 import fiona
 import rasterio
@@ -247,41 +247,77 @@ def wrapper(*args, **kwargs):
             func(*args, **kwargs)
     return wrapper
 
-def append_id_to_file_name(file_name, identifier):
-    '''
-    Processing:
-        Takes an incoming file name and inserts an identifier into the name
-        just ahead of the extension, with an underscore added.
-        ie) filename = "/output/myfolder/a_raster.tif"
-            indentifer = "13090001"
-            Becomes: "/output/myfolder/a_raster_13090001.tif"
-        Note: 
-            - Can handle a single identifier or a list of identifier
-              ie) identifier = ["13090001", "123000001"]
-              Becomes: "/output/myfolder/a_raster_13090001_123000001.tif"
-            - This allows for file name to not be submitted and will return None
-    Inputs:
-        file_name: a single file name
-        identifier: a value or list of values to be inserted with an underscore 
-            added ahead of the extention
-    Output:
-        out_file_name: A single name with each identifer added at the end before
-            the extension, each with an underscore in front of the identifier.
-    '''
-
-    if file_name is not None:
-
-        root,extension = os.path.splitext(file_name)
-
-        if isinstance(identifier, list):
-
-            out_file_name = root
-            for i in identifier:
-                out_file_name += "_{}".format(i)
-            out_file_name += extension
+
+class FIM_Helpers:
+
+    @staticmethod    
+    def append_id_to_file_name(file_name, identifier):
+        '''
+        Processing:
+            Takes an incoming file name and inserts an identifier into the name
+            just ahead of the extension, with an underscore added.
+            ie) filename = "/output/myfolder/a_raster.tif"
+                indentifer = "13090001"
+                Becomes: "/output/myfolder/a_raster_13090001.tif"
+            Note: 
+                - Can handle a single identifier or a list of identifier
+                ie) identifier = ["13090001", "123000001"]
+                Becomes: "/output/myfolder/a_raster_13090001_123000001.tif"
+                - This allows for file name to not be submitted and will return None
+        Inputs:
+            file_name: a single file name
+            identifier: a value or list of values to be inserted with an underscore 
+                added ahead of the extention
+        Output:
+            out_file_name: A single name with each identifer added at the end before
+                the extension, each with an underscore in front of the identifier.
+        '''
+
+        if file_name is not None:
+
+            root,extension = os.path.splitext(file_name)
+
+            if isinstance(identifier, list):
+
+                out_file_name = root
+                for i in identifier:
+                    out_file_name += "_{}".format(i)
+                out_file_name += extension
+            else:
+                out_file_name = root + "_{}".format(identifier) + extension
         else:
-            out_file_name = root + "_{}".format(identifier) + extension
-    else:
-        out_file_name = None
+            out_file_name = None
+
+        return(out_file_name)
+
+    @staticmethod
+    def vprint (message, is_verbose, show_caller = False):
+        '''
+        Processing: Will print a standard output message only when the 
+            verbose flag is set to True
+        Parameters:
+            message : str
+                The message for output
+                Note: this method puts a '...' in front of the message
+            is_verbose : bool
+                This exists so the call to vprint always exists and does not 
+                need a "if verbose: test for inline code
+                If this value is False, this method will simply return
+            show_caller : bool
+                Sometimes, it is desired to see the calling function, method or class
+        Returns:
+            str : the message starting with "... " and optionallly ending with
+                the calling function, method or class name
+        '''
+        if not is_verbose:
+            return
+
+        msg = f"... {message}"
+        if (show_caller):
+            caller_name = inspect.stack()[1][3]
+            if (caller_name == "<module"):
+                caller_name = inspect.stack()[1][1]
+            msg += f"  [from : {caller_name}]"
+        print (msg)
+
 
-    return(out_file_name)
diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py
index 5fdfabcd7..3eddb202e 100644
--- a/tools/composite_inundation.py
+++ b/tools/composite_inundation.py
@@ -14,7 +14,7 @@
 from inundation import inundate
 from gms_tools.mosaic_inundation import Mosaic_inundation
 from gms_tools.inundate_gms import Inundate_gms
-from utils.shared_functions import append_id_to_file_name
+from utils.shared_functions import FIM_Helpers as fh
 from utils.shared_variables import elev_raster_ndv
 
 class InundateModel_HUC(object):
@@ -58,7 +58,7 @@ def inundate_huc(self, flows_file, composite_output_dir, output_name, log_file_p
         inundation_map_file = None
 
         output_raster_name = os.path.join(output_huc_dir, output_name)
-        output_raster_name = append_id_to_file_name(output_raster_name, [self.huc,self.model])
+        output_raster_name = fh.append_id_to_file_name(output_raster_name, [self.huc,self.model])
 
         # adjust to add model and huc number
         log_file = None
@@ -175,7 +175,7 @@ def composite_huc(self, args):
             print(inundation_map_file_df)
 
         composite_file_output = os.path.join(args["composite_output_dir"], huc, args["output_name"])
-        composite_file_output = append_id_to_file_name(composite_file_output, huc)
+        composite_file_output = fh.append_id_to_file_name(composite_file_output, huc)
 
         # NOTE: Leave workers as 1, it fails to composite correctly if more than one.
         #    - Also. by adding the is_mosaic_for_gms_branches = False, Mosaic_inudation
diff --git a/tools/gms_tools/inundate_gms.py b/tools/gms_tools/inundate_gms.py
index 6bcca72a8..7ecd5ce4d 100755
--- a/tools/gms_tools/inundate_gms.py
+++ b/tools/gms_tools/inundate_gms.py
@@ -13,7 +13,7 @@
 from inundation import inundate
 from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor,as_completed
 from inundation import hydroTableHasOnlyLakes, NoForecastFound
-from utils.shared_functions import append_id_to_file_name
+from utils.shared_functions import FIM_Helpers as fh
 
 def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
                   hucs = None,
@@ -39,7 +39,7 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
     if log_file is not None:
         if os.path.exists(log_file):
             os.remove(log_file)
-        
+
         if verbose :
             print('HUC8,BranchID,Exception',file=open(log_file,'w'))
     #if log_file:
@@ -95,9 +95,6 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
 
         try:
             future.result()
-
-            #print("future")
-            #print(future.result())
         
         except NoForecastFound as exc:
             if log_file is not None:
@@ -152,9 +149,6 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
                                           'depths_rasters' : depths_raster_fileNames,
                                           'inundation_polygons' : inundation_polygon_fileNames } )
 
-    #print("output_fileNames_df")
-    #print(output_fileNames_df)
-
     if output_fileNames is not None:
         output_fileNames_df.to_csv(output_fileNames,index=False)
 
@@ -192,23 +186,25 @@ def __inundate_gms_generator( hucs_branches,
         # branch output
         # Some other functions that call in here already added a huc, so only add it if not yet there
         if (inundation_raster is not None) and (huc not in inundation_raster):
-            inundation_branch_raster = append_id_to_file_name(inundation_raster,[huc,branch_id])
+            inundation_branch_raster = fh.append_id_to_file_name(inundation_raster,[huc,branch_id])
         else:
-            inundation_branch_raster = append_id_to_file_name(inundation_raster,branch_id)
+            inundation_branch_raster = fh.append_id_to_file_name(inundation_raster,branch_id)
 
         if (inundation_polygon is not None) and (huc not in inundation_polygon):
-            inundation_branch_polygon = append_id_to_file_name(inundation_polygon,[huc,branch_id])
+            inundation_branch_polygon = fh.append_id_to_file_name(inundation_polygon,[huc,branch_id])
         else:
-            inundation_branch_polygon = append_id_to_file_name(inundation_polygon,branch_id)
+            inundation_branch_polygon = fh.append_id_to_file_name(inundation_polygon,branch_id)
 
         if (depths_raster is not None) and (huc not in depths_raster):
-            depths_branch_raster = append_id_to_file_name(depths_raster,[huc,branch_id])
+            depths_branch_raster = fh.append_id_to_file_name(depths_raster,[huc,branch_id])
         else:
-            depths_branch_raster = append_id_to_file_name(depths_raster,branch_id)
+            depths_branch_raster = fh.append_id_to_file_name(depths_raster,branch_id)
 
         # identifiers
         identifiers = (huc,branch_id)
 
+        #print(f"inundation_branch_raster is {inundation_branch_raster}")
+
         # inundate input
         inundate_input = {  'rem' : rem_branch, 
                             'catchments' : catchments_branch, 
@@ -230,12 +226,6 @@ def __inundate_gms_generator( hucs_branches,
         
         yield (inundate_input, identifiers)
 
-
-def __vprint(message,verbose):
-    if verbose:
-        print(message)
-
-
 if __name__ == '__main__':
 
     # parse arguments
diff --git a/tools/gms_tools/mosaic_inundation.py b/tools/gms_tools/mosaic_inundation.py
index 29a682e8c..ff6165a6c 100755
--- a/tools/gms_tools/mosaic_inundation.py
+++ b/tools/gms_tools/mosaic_inundation.py
@@ -10,7 +10,7 @@
 from gms_tools.overlapping_inundation import OverlapWindowMerge
 from tqdm import tqdm
 from utils.shared_variables import elev_raster_ndv
-from utils.shared_functions import append_id_to_file_name
+from utils.shared_functions import FIM_Helpers as fh
 
 def Mosaic_inundation( map_file,
                        mosaic_attribute = 'inundation_rasters',
@@ -63,6 +63,8 @@ def Mosaic_inundation( map_file,
     else:
         tqdm_disable = True
 
+    ag_mosaic_output = ""
+
     for ag in tqdm(aggregation_units, disable = tqdm_disable, desc = 'Mosaicing FIMs'):
 
         try:
@@ -75,12 +77,11 @@ def Mosaic_inundation( map_file,
         # Only add the huc into the name if branches are being processed, as 
         # sometimes the mosiac is not for gms branches but maybe mosaic of an
         # fr set with a gms composite map.
+
         ag_mosaic_output = mosaic_output
         if (is_mosaic_for_gms_branches) and (ag not in mosaic_output):
-            ag_mosaic_output = append_id_to_file_name(mosaic_output, ag) # change it
+            ag_mosaic_output = fh.append_id_to_file_name(mosaic_output, ag) # change it
 
-        #print(f"ag_mosaic_output is {ag_mosaic_output}")
-        #try:
         mosaic_by_unit(inundation_maps_list, 
                       ag_mosaic_output,
                       nodata,
@@ -88,14 +89,13 @@ def Mosaic_inundation( map_file,
                       remove_inputs = remove_inputs,
                       mask = mask,
                       verbose = verbose)
-        #except Exception as exc:
-        #    print(ag,exc)
-    
+
 
     # inundation maps
     inundation_maps_df.reset_index(drop=True)
 
-    # return file name and path of the final mosaic output file.
+    # Return file name and path of the final mosaic output file.
+    # Might be empty.
     return ag_mosaic_output
 
 
@@ -122,13 +122,11 @@ def mosaic_by_unit(inundation_maps_list,
         overlap.merge_rasters(mosaic_output, threaded=threaded, workers=workers, nodata=nodata)
 
         if mask:
-            if verbose:
-                print("Masking ...")
+            fh.vprint("Masking ...", verbose)                
             overlap.mask_mosaic(mosaic_output, mask, outfile=mosaic_output)
     
     if remove_inputs:
-        if verbose:
-            print("Removing inputs ...")
+        fh.vprint("Removing inputs ...", verbose)
 
         for inun_map in inundation_maps_list:
             if inun_map is not None:
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 6df9710ca..6d7efc5d6 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -1,379 +1,440 @@
 #!/usr/bin/env python3
 
-import os
-import sys
-import shutil
-import argparse
-import traceback
-from pathlib import Path
-import json
-import ast
+import os, re, shutil, json, sys
 import pandas as pd
 
-from tools_shared_functions import compute_contingency_stats_from_rasters
-from tools_shared_variables import (TEST_CASES_DIR, INPUTS_DIR, ENDC, TRED_BOLD, WHITE_BOLD, CYAN_BOLD, AHPS_BENCHMARK_CATEGORIES, IFC_MAGNITUDE_LIST, BLE_MAGNITUDE_LIST )
+from tools_shared_variables import TEST_CASES_DIR, INPUTS_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT, elev_raster_ndv
 from inundation import inundate
-from gms_tools.inundate_gms import Inundate_gms
 from gms_tools.mosaic_inundation import Mosaic_inundation
-from gms_tools.overlapping_inundation import OverlapWindowMerge
-from glob import glob
-from utils.shared_variables import elev_raster_ndv
-
-def run_alpha_test( fim_run_dir, version, test_id, magnitude, 
-                calibrated, model,
-                compare_to_previous=False, archive_results=False, 
-                mask_type='filter', inclusion_area='', 
-                inclusion_area_buffer=0, light_run=False, 
-                overwrite=True, fr_run_dir=None, 
-                gms_workers=1,verbose=False,
-                gms_verbose=False
-              ):
-
-    # check eval_meta input
-    if model not in {None,'FR','MS','GMS'}:
-        raise ValueError("Model argument needs to be \'FR\', \'MS\', or \'GMS.\'")
-
-    # make bool
-    calibrated = bool( calibrated )
-
-    print(f"run_alpha_test: test_id = {test_id}")
-
-    if (model == "MS") & (fr_run_dir is None):
-        raise ValueError("fr_run_dir argument needs to be specified with MS model")
-
-    benchmark_category = test_id.split('_')[1] # Parse benchmark_category from test_id.
-    current_huc = test_id.split('_')[0]  # Break off HUC ID and assign to variable.
-
-    # Construct paths to development test results if not existent.
-    if archive_results:
-        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'official_versions', version)
-    else:
-        version_test_case_dir_parent = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', test_id, 'testing_versions', version)
-
-    # Delete the entire directory if it already exists.
-    if os.path.exists(version_test_case_dir_parent):
-        if overwrite == True:
-            shutil.rmtree(version_test_case_dir_parent)
-        elif model == 'MS':
-            pass
-        else:
-            print("Metrics for ({version}: {test_id}) already exist. Use overwrite flag (-o) to overwrite metrics.".format(version=version, test_id=test_id))
-            return
-
-    os.makedirs(version_test_case_dir_parent,exist_ok=True)
-
-    __vprint("Running the alpha test for test_id: " + test_id + ", " + version + "...",verbose)
-    stats_modes_list = ['total_area']
-
-    fim_run_parent = os.path.join(os.environ['outputDataDir'], fim_run_dir)
-    assert os.path.exists(fim_run_parent), "Cannot locate " + fim_run_parent
-
-    # get hydrofabric directory
-    hydrofabric_dir = Path(fim_run_parent).parent.absolute()
-
-    # Create paths to fim_run outputs for use in inundate().
-    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
-    if not os.path.exists(rem):
-        rem = os.path.join(fim_run_parent, 'rem_clipped_zeroed_masked.tif')
-    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-    if not os.path.exists(catchments):
-        catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_clipped_addedAttributes.tif')
-    if mask_type == 'huc':
-        catchment_poly = ''
-    else:
-        catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
-
-    # Map necessary inputs for inundation().
-    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-
-    # Create list of shapefile paths to use as exclusion areas.
-    zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
-    mask_dict = {'levees':
-                    {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
-                     'buffer': None,
-                     'operation': 'exclude'
-                     },
-                'waterbodies':
-                    {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
-                     'buffer': None,
-                     'operation': 'exclude',
-                     },
-                }
-
-    if inclusion_area != '':
-        inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
-        mask_dict.update({inclusion_area_name: {'path': inclusion_area,
-                                                'buffer': int(inclusion_area_buffer),
-                                                'operation': 'include'}})
-        # Append the concatenated inclusion_area_name and buffer.
-        if inclusion_area_buffer == None:
-            inclusion_area_buffer = 0
-        stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm')
-
-    # Check if magnitude is list of magnitudes or single value.
-    magnitude_list = magnitude
-    if type(magnitude_list) != list:
-        magnitude_list = [magnitude_list]
-        
+from gms_tools.inundate_gms import Inundate_gms
+from tools_shared_functions import compute_contingency_stats_from_rasters
+from utils.shared_functions import FIM_Helpers as fh
 
+class benchmark(object):
 
-    # Get path to validation_data_{benchmark} directory and huc_dir.
-    validation_data_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category)
-    for magnitude in magnitude_list:
-        version_test_case_dir = os.path.join(version_test_case_dir_parent, magnitude)
-        if not os.path.exists(version_test_case_dir):
-            os.mkdir(version_test_case_dir)
-        # Construct path to validation raster and forecast file.
-        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-            benchmark_raster_path_list, forecast_list = [], []
-            lid_dir_list = os.listdir(os.path.join(validation_data_path, current_huc))
-            lid_list, inundation_raster_list, domain_file_list = [], [], []
-
-            for lid in lid_dir_list:
-                lid_dir = os.path.join(validation_data_path, current_huc, lid)
-                benchmark_lid_raster_path = os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
-
-                # Only compare if the benchmark data exist.
-                if os.path.exists(benchmark_lid_raster_path):
-                    benchmark_raster_path_list.append(benchmark_lid_raster_path)  # TEMP
-                    forecast_list.append(os.path.join(lid_dir, magnitude, 'ahps_' + lid + '_huc_' + current_huc + '_flows_' + magnitude + '.csv'))  # TEMP
-                    lid_list.append(lid)
-                    inundation_raster_list.append(os.path.join(version_test_case_dir, lid + '_inundation_extent.tif'))
-                    domain_file_list.append(os.path.join(lid_dir, lid + '_domain.shp'))
+    AHPS_BENCHMARK_CATEGORIES = AHPS_BENCHMARK_CATEGORIES
+    MAGNITUDE_DICT = MAGNITUDE_DICT
+    
+    def __init__(self, category):
+        """Class that handles benchmark data.
 
+            Parameters
+            ----------
+            category : str
+                Category of the benchmark site. Should be one of ['ble', 'ifc', 'nws', 'usgs', 'ras2fim'].
+        """
+        
+        self.category = category.lower()
+        assert category in list(self.MAGNITUDE_DICT.keys()), f"Category must be one of {list(self.MAGNITUDE_DICT.keys())}"
+        self.validation_data = os.path.join(TEST_CASES_DIR, f'{self.category}_test_cases', f'validation_data_{self.category}')
+        self.is_ahps = True if self.category in self.AHPS_BENCHMARK_CATEGORIES else False
+    
+    def magnitudes(self):
+        '''Returns the magnitudes associated with the benchmark category.'''
+        return self.MAGNITUDE_DICT[self.category]
+    
+    def huc_data(self):
+        '''Returns a dict of HUC8, magnitudes, and sites.'''
+        huc_mags = {}
+        for huc in os.listdir(self.validation_data):
+            if not re.match('\d{8}', huc): continue
+            huc_mags[huc] = self.data(huc)
+        return huc_mags
+    
+    def data(self, huc):
+        '''Returns a dict of magnitudes and sites for a given huc. Sites will be AHPS lids for 
+           AHPS sites and empty strings for non-AHPS sites.
+        '''
+        huc_dir = os.path.join(self.validation_data, huc)
+        if not os.path.isdir(huc_dir):
+            return {}
+        if self.is_ahps:
+            lids = os.listdir(huc_dir)
+
+            mag_dict = {}
+            for lid in lids:
+                lid_dir = os.path.join(huc_dir, lid)
+                for mag in [file for file in os.listdir(lid_dir) if file in self.magnitudes()]:
+                    if mag in mag_dict:
+                        mag_dict[mag].append(lid)
+                    else:
+                        mag_dict[mag] = [lid]
+            return mag_dict
         else:
-            benchmark_raster_file = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_extent_' + magnitude + '.tif')
-            benchmark_raster_path_list = [benchmark_raster_file]
-            forecast_path = os.path.join(TEST_CASES_DIR, benchmark_category + '_test_cases', 'validation_data_' + benchmark_category, current_huc, magnitude, benchmark_category + '_huc_' + current_huc + '_flows_' + magnitude + '.csv')
-            forecast_list = [forecast_path]
-            inundation_raster_list = [os.path.join(version_test_case_dir, 'inundation_extent.tif')]
-
-        for index in range(0, len(benchmark_raster_path_list)):
-            benchmark_raster_path = benchmark_raster_path_list[index]
-            forecast = forecast_list[index]
-            inundation_raster = inundation_raster_list[index]
-            # Only need to define ahps_lid and ahps_extent_file for AHPS_BENCHMARK_CATEGORIES.
-            if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                ahps_lid = lid_list[index]
-                ahps_domain_file = domain_file_list[index]
-                mask_dict.update({ahps_lid:
-                    {'path': ahps_domain_file,
-                     'buffer': None,
-                     'operation': 'include'}
-                        })
-
-
-                if not os.path.exists(benchmark_raster_path) or not os.path.exists(ahps_domain_file) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
-                    continue
-            else:  # If not in AHPS_BENCHMARK_CATEGORIES.
-                if not os.path.exists(benchmark_raster_path) or not os.path.exists(forecast):  # Skip loop instance if the benchmark raster doesn't exist.
-                    continue
-            # Run inundate.
-            __vprint("-----> Running inundate() to produce inundation extent for the " + magnitude + " magnitude...",verbose)
-            # The inundate adds the huc to the name so I account for that here.
-            predicted_raster_path = os.path.join(
-                                        os.path.split(inundation_raster)[0], 
-                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
-                                                )  
-            try:
-                if model == 'GMS':
-                    
-                    map_file = Inundate_gms(
-                                             hydrofabric_dir=hydrofabric_dir, 
-                                             forecast=forecast, 
-                                             num_workers=gms_workers,
-                                             hucs=current_huc,
-                                             inundation_raster=inundation_raster,
-                                             inundation_polygon=None, depths_raster=None,
-                                             verbose=gms_verbose,
-                                             log_file=None,
-                                             output_fileNames=None
-                                            )
-                    
-                    mask_path_gms = os.path.join(fim_run_parent, 'wbd.gpkg')
-
-                    Mosaic_inundation(
-                                        map_file,mosaic_attribute='inundation_rasters',
-                                        mosaic_output=inundation_raster,
-                                        mask=mask_path_gms,unit_attribute_name='huc8',
-                                        nodata=elev_raster_ndv,workers=1,
-                                        remove_inputs=True,
-                                        subset=None,verbose=verbose
-                                      )
-                
-                else:
-                    inundate(
-                             rem,catchments,catchment_poly,hydro_table,forecast,
-                             mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                             subset_hucs=current_huc,num_workers=1,aggregate=False,
-                             inundation_raster=inundation_raster,inundation_polygon=None,
-                             depths=None,out_raster_profile=None,out_vector_profile=None,
-                             quiet=True
-                        )
-
-                if model =='MS':
-                    
-                    # Mainstems inundation
-                    #fr_run_parent = os.path.join(os.environ['outputDataDir'], fr_run_dir,current_huc)
-                    #assert os.path.exists(fr_run_parent), "Cannot locate " + fr_run_parent
-                    
-                    inundation_raster_ms = os.path.join(
-                                        os.path.split(inundation_raster)[0], 
-                                        os.path.split(inundation_raster)[1].replace('.tif', '_{}_MS.tif'.format(current_huc))
-                                           )  
-                    inundation_raster_fr = os.path.join(
-                                        os.path.split(version_test_case_dir_parent)[0],
-                                        fr_run_dir,
-                                        magnitude,
-                                        os.path.split(inundation_raster)[1].replace('.tif', '_'+current_huc+'.tif')
-                                           )  
-                    
-                    os.rename(predicted_raster_path,inundation_raster_ms)
-
-                    ms_inundation_map_file = { 
-                                               'huc8' : [current_huc] * 2,
-                                               'branchID' : [None] * 2,
-                                               'inundation_rasters' : [inundation_raster_fr,inundation_raster_ms],
-                                               'depths_rasters' : [None] * 2,
-                                               'inundation_polygons' : [None] * 2
-                                             }
-                    ms_inundation_map_file = pd.DataFrame(ms_inundation_map_file)
-                    
-                    Mosaic_inundation(
-                                        ms_inundation_map_file,mosaic_attribute='inundation_rasters',
-                                        mosaic_output=inundation_raster,
-                                        mask=catchment_poly,unit_attribute_name='huc8',
-                                        nodata=elev_raster_ndv,workers=1,
-                                        remove_inputs=False,
-                                        subset=None,verbose=verbose
-                                      )
-
-                __vprint("-----> Inundation mapping complete.",verbose)
-
-                # Define outputs for agreement_raster, stats_json, and stats_csv.
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, lid + 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
+            mags = list(os.listdir(huc_dir))
+            return {mag:[''] for mag in mags}
+
+class test_case(benchmark):
+
+    def __init__(self, test_id, version, archive=True):
+        """Class that handles test cases, specifically running the alpha test.
+
+            Parameters
+            ----------
+            test_id : str
+                ID of the test case in huc8_category format, e.g. `12090201_ble`.
+            version : str
+                Version of FIM to which this test_case belongs. This should correspond to the fim directory
+                name in either `/data/previous_fim/` or `/data/outputs/`.
+            archive : bool
+                If true, this test case outputs will be placed into the `official_versions` folder
+                and the FIM model will be read from the `/data/previous_fim` folder.
+                If false, it will be saved to the `testing_versions/` folder and the FIM model 
+                will be read from the `/data/outputs/` folder.
+        
+        """
+        self.test_id = test_id
+        self.huc, self.benchmark_cat = test_id.split('_')
+        super().__init__(self.benchmark_cat)
+        self.version = version
+        self.archive = archive
+        # FIM run directory path - uses HUC 6 for FIM 1 & 2
+        self.fim_dir = os.path.join(PREVIOUS_FIM_DIR if archive else OUTPUTS_DIR, 
+                                    self.version, 
+                                    self.huc if not re.search('^fim_[1,2]', version, re.IGNORECASE) else self.huc[:6])
+        # Test case directory path
+        self.dir = os.path.join(TEST_CASES_DIR, f'{self.benchmark_cat}_test_cases', 
+                                test_id,
+                                'official_versions' if archive else 'testing_versions',
+                                version)
+        # Benchmark data path
+        self.benchmark_dir = os.path.join(self.validation_data, self.huc)
+
+        # Create list of shapefile paths to use as exclusion areas.
+        zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
+        self.mask_dict = {'levees':
+                        {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
+                        'buffer': None,
+                        'operation': 'exclude'
+                        },
+                    'waterbodies':
+                        {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
+                        'buffer': None,
+                        'operation': 'exclude',
+                        },
+                    }
+
+    @classmethod
+    def list_all_test_cases(cls, version, archive, benchmark_categories=[]):
+        """Returns a complete list of all benchmark category test cases as classes.
+
+            Parameters
+            ----------
+            version : str
+                Version of FIM to which this test_case belongs. This should correspond to the fim directory
+                name in either `/data/previous_fim/` or `/data/outputs/`.
+            archive : bool
+                If true, this test case outputs will be placed into the `official_versions` folder
+                and the FIM model will be read from the `/data/previous_fim` folder.
+                If false, it will be saved to the `testing_versions/` folder and the FIM model 
+                will be read from the `/data/outputs/` folder.
+        """
+        if not benchmark_categories:
+            benchmark_categories = list(cls.MAGNITUDE_DICT.keys())
+            
+        test_case_list = []
+        for bench_cat in benchmark_categories:
+
+            benchmark_class = benchmark(bench_cat)
+            benchmark_data = benchmark_class.huc_data()
+
+            for huc in benchmark_data.keys():
+                test_case_list.append(cls(f'{huc}_{bench_cat}', version, archive))
+
+        return test_case_list
+
+    def alpha_test(self, calibrated=False, model='', mask_type='huc', inclusion_area='',
+                inclusion_area_buffer=0, overwrite=True, verbose=False, gms_workers=1):
+        '''Compares a FIM directory with benchmark data from a variety of sources.
+        
+            Parameters
+            ----------
+            calibrated : bool
+                Whether or not this FIM version is calibrated.
+            model : str
+                MS or FR extent of the model. This value will be written to the eval_metadata.json.
+            mask_type : str
+                Mask type to feed into inundation.py.
+            inclusion_area : int
+                Area to include in agreement analysis.
+            inclusion_area_buffer : int
+                Buffer distance in meters to include outside of the model's domain.
+            overwrite : bool
+                If True, overwites pre-existing test cases within the test_cases directory.
+            verbose : bool
+                If True, prints out all pertinent data.
+            gms_workers : int
+                Number of worker processes assigned to GMS processing.
+        '''
+        
+        try:
+            if not overwrite and os.path.isdir(self.dir):
+                print(f"Metrics for {self.dir} already exist. Use overwrite flag (-o) to overwrite metrics.")
+                return
+
+            fh.vprint(f"Starting alpha test for {self.dir}", verbose)
+            
+            self.stats_modes_list = ['total_area']
+
+            # Create paths to fim_run outputs for use in inundate()
+            if model != 'GMS':
+                self.rem = os.path.join(self.fim_dir, 'rem_zeroed_masked.tif')
+                if not os.path.exists(self.rem):
+                    self.rem = os.path.join(self.fim_dir, 'rem_clipped_zeroed_masked.tif')
+                self.catchments = os.path.join(self.fim_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+                if not os.path.exists(self.catchments):
+                    self.catchments = os.path.join(self.fim_dir, 'gw_catchments_reaches_clipped_addedAttributes.tif')
+                self.mask_type = mask_type
+                if mask_type == 'huc':
+                    self.catchment_poly = ''
                 else:
-                    agreement_raster, stats_json, stats_csv = os.path.join(version_test_case_dir, 'total_area_agreement.tif'), os.path.join(version_test_case_dir, 'stats.json'), os.path.join(version_test_case_dir, 'stats.csv')
-                
-                compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                       benchmark_raster_path,
-                                                       agreement_raster,
-                                                       stats_csv=stats_csv,
-                                                       stats_json=stats_json,
-                                                       mask_values=[],
-                                                       stats_modes_list=stats_modes_list,
-                                                       test_id=test_id,
-                                                       mask_dict=mask_dict,
-                                                       )
-
-                if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-                    del mask_dict[ahps_lid]
-
-                __vprint(" ",verbose)
-                # print("Evaluation complete. All metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC) # GMS
-                __vprint("Evaluation metrics for " + test_id + ", " + version + ", " + magnitude + " are available at " + CYAN_BOLD + version_test_case_dir + ENDC,verbose) # cahaba/dev
-                __vprint(" ",verbose)
-
-            except Exception as e:
-                print(traceback.print_exc())
-                #print(e)
-
-        if benchmark_category in AHPS_BENCHMARK_CATEGORIES:
-            # -- Delete temp files -- #
-            # List all files in the output directory.
-            output_file_list = os.listdir(version_test_case_dir)
-            for output_file in output_file_list:
-                if "total_area" in output_file:
-                    full_output_file_path = os.path.join(version_test_case_dir, output_file)
-                    os.remove(full_output_file_path)
-
-    # write out evaluation meta-data
-    with open(os.path.join(version_test_case_dir_parent,'eval_metadata.json'),'w') as meta:
-        eval_meta = { 'calibrated' : calibrated , 'model' : model }
-        meta.write( 
-                    json.dumps(eval_meta,indent=2) 
-                   )
-
-def __vprint(message,verbose):
-    if verbose:
-        print(message)
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping and regression analysis for FOSS FIM. Regression analysis results are stored in the test directory.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-b', '--version',help='The name of the working version in which features are being tested',required=True,default="")
-    parser.add_argument('-t', '--test-id',help='The test_id to use. Format as: HUC_BENCHMARKTYPE, e.g. 12345678_ble.',required=True,default="")
-    parser.add_argument('-m', '--mask-type', help='Specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method. MS and GMS are currently on supporting huc', required=False,default="filter")
-    parser.add_argument('-n','--calibrated',help='Denotes use of calibrated n values',required=False, default=False,action='store_true')
-    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed',required=True)
-    parser.add_argument('-y', '--magnitude',help='The magnitude to run.',required=False, default="")
-    parser.add_argument('-c', '--compare-to-previous', help='Compare to previous versions of HAND.', required=False,action='store_true')
-    parser.add_argument('-a', '--archive-results', help='Automatically copy results to the "previous_version" archive for test_id. For admin use only.', required=False,action='store_true')
-    parser.add_argument('-i', '--inclusion-area', help='Path to shapefile. Contingency metrics will be produced from pixels inside of shapefile extent.', required=False, default="")
-    parser.add_argument('-ib','--inclusion-area-buffer', help='Buffer to use when masking contingency metrics with inclusion area.', required=False, default="0")
-    parser.add_argument('-l', '--light-run', help='Using the light_run option will result in only stat files being written, and NOT grid files.', required=False, action='store_true')
-    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, default=False, action='store_true')
-    parser.add_argument('-w','--gms-workers', help='Number of workers to use for GMS Branch Inundation', required=False, default=1)
-    parser.add_argument('-d','--fr-run-dir',help='Name of test case directory containing inundation for FR configuration',required=False,default=None)
-    parser.add_argument('-v', '--verbose', help='Verbose operation', required=False, action='store_true', default=False)
-    parser.add_argument('-vg', '--gms-verbose', help='Prints progress bar for GMS', required=False, action='store_true', default=False)
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    valid_test_id_list = os.listdir(TEST_CASES_DIR)
-
-    exit_flag = False  # Default to False.
-    __vprint("",args['verbose'])
-
-    # Ensure test_id is valid.
-#    if args['test_id'] not in valid_test_id_list:
-#        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided test_id (-t) " + CYAN_BOLD + args['test_id'] + WHITE_BOLD + " is not available." + ENDC)
-#        print(WHITE_BOLD + "Available test_ids include: " + ENDC)
-#        for test_id in valid_test_id_list:
-#          if 'validation' not in test_id.split('_') and 'ble' in test_id.split('_'):
-#              print(CYAN_BOLD + test_id + ENDC)
-#        print()
-#        exit_flag = True
-
-    # Ensure fim_run_dir exists.
-    if not os.path.exists(os.path.join(os.environ['outputDataDir'], args['fim_run_dir'])):
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
-        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. outputs/123456/123456." + ENDC)
-        print()
-        exit_flag = True
-
-    # Ensure inclusion_area path exists.
-    if args['inclusion_area'] != "" and not os.path.exists(args['inclusion_area']):
-        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area (-i) " + CYAN_BOLD + args['inclusion_area'] + WHITE_BOLD + " could not be located." + ENDC)
-        exit_flag = True
-
-    try:
-        inclusion_buffer = int(args['inclusion_area_buffer'])
-    except ValueError:
-        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided inclusion_area_buffer (-ib) " + CYAN_BOLD + args['inclusion_area_buffer'] + WHITE_BOLD + " is not a round number." + ENDC)
-
-    benchmark_category = args['test_id'].split('_')[1]
-
-    if args['magnitude'] == '':
-        if 'ble' == benchmark_category:
-            args['magnitude'] = BLE_MAGNITUDE_LIST
-        elif ('nws' == benchmark_category) | ('usgs' == benchmark_category):
-            args['magnitude'] = ['action', 'minor', 'moderate', 'major']
-        elif 'ifc' == benchmark_category:
-            args['magnitude'] = IFC_MAGNITUDE_LIST
+                    self.catchment_poly = os.path.join(self.fim_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+                self.hydro_table = os.path.join(self.fim_dir, 'hydroTable.csv')
+
+            # Map necessary inputs for inundate().
+            self.hucs, self.hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
+
+            if inclusion_area != '':
+                inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
+                self.mask_dict.update({inclusion_area_name: {'path': inclusion_area,
+                                                        'buffer': int(inclusion_area_buffer),
+                                                        'operation': 'include'}})
+                # Append the concatenated inclusion_area_name and buffer.
+                if inclusion_area_buffer == None:
+                    inclusion_area_buffer = 0
+                self.stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm')
+
+            # Delete the directory if it exists
+            if os.path.exists(self.dir):
+                shutil.rmtree(self.dir)
+            os.mkdir(self.dir)
+
+            # Get the magnitudes and lids for the current huc and loop through them
+            validation_data = self.data(self.huc)
+            for magnitude in validation_data:
+                for instance in validation_data[magnitude]:      # instance will be the lid for AHPS sites and '' for other sites
+                    # For each site, inundate the REM and compute aggreement raster with stats
+                    self._inundate_and_compute(magnitude, instance, model=model, verbose=verbose, gms_workers=gms_workers)
+
+                # Clean up 'total_area' outputs from AHPS sites
+                if self.is_ahps:
+                    self.clean_ahps_outputs(os.path.join(self.dir, magnitude))
+
+            # Write out evaluation meta-data
+            self.write_metadata(calibrated, model)
+
+        except KeyboardInterrupt:
+            print("Program aborted via keyboard interrupt")
+            sys.exit(1)
+        except Exception as ex:
+            print(ex)
+            sys.exit(1)
+        
+    def _inundate_and_compute(self, 
+                              magnitude, 
+                              lid, 
+                              compute_only = False, 
+                              model = '',
+                              verbose = False,
+                              gms_workers = 1):
+        '''Method for inundating and computing contingency rasters as part of the alpha_test.
+           Used by both the alpha_test() and composite() methods.
+
+            Parameters
+            ----------
+            magnitude : str
+                Magnitude of the current benchmark site.
+            lid : str
+                lid of the current benchmark site. For non-AHPS sites, this should be an empty string ('').
+            compute_only : bool
+                If true, skips inundation and only computes contingency stats.
+        '''
+        # Output files
+        fh.vprint("Creating output files", verbose)
+
+        test_case_out_dir     = os.path.join(self.dir, magnitude)
+        inundation_prefix     = lid + '_' if lid else ''
+        inundation_path       = os.path.join(test_case_out_dir, f'{inundation_prefix}inundation_extent.tif')
+        predicted_raster_path = inundation_path.replace('.tif', f'_{self.huc}.tif')
+        agreement_raster      = os.path.join(test_case_out_dir, (f'ahps_{lid}' if lid else '') +'total_area_agreement.tif')
+        stats_json            = os.path.join(test_case_out_dir, 'stats.json')
+        stats_csv             = os.path.join(test_case_out_dir, 'stats.csv')
+
+        # Create directory
+        if not os.path.isdir(test_case_out_dir):
+            os.mkdir(test_case_out_dir)
+
+        # Benchmark raster and flow files
+        benchmark_rast = (f'ahps_{lid}' if lid else self.benchmark_cat) + f'_huc_{self.huc}_extent_{magnitude}.tif'
+        benchmark_rast = os.path.join(self.benchmark_dir, lid, magnitude, benchmark_rast)
+        benchmark_flows = benchmark_rast.replace(f'_extent_{magnitude}.tif', f'_flows_{magnitude}.csv')
+        mask_dict_indiv = self.mask_dict.copy()
+        if self.is_ahps: # add domain shapefile to mask for AHPS sites
+            domain = os.path.join(self.benchmark_dir, lid, f'{lid}_domain.shp')
+            mask_dict_indiv.update({lid:
+                            {'path': domain,
+                            'buffer': None,
+                            'operation': 'include'}
+                                })
+        # Check to make sure all relevant files exist
+        if not os.path.isfile(benchmark_rast) or not os.path.isfile(benchmark_flows) or (self.is_ahps and not os.path.isfile(domain)):
+            return -1
+
+        # Inundate REM
+        if not compute_only:             # composite alpha tests don't need to be inundated
+            if model == 'GMS':
+                fh.vprint("Begin GMS Inundation", verbose)
+                map_file = Inundate_gms( hydrofabric_dir = os.path.dirname(self.fim_dir), 
+                                         forecast = benchmark_flows, 
+                                         num_workers = gms_workers,
+                                         hucs = self.huc,
+                                         inundation_raster = predicted_raster_path,
+                                         inundation_polygon = None,
+                                         depths_raster = None,
+                                         verbose = verbose,
+                                         log_file = None,
+                                         output_fileNames = None )
+                #if (len(map_file) > 0):
+                fh.vprint("Begin GMS Mosaic", verbose)
+                Mosaic_inundation( map_file,
+                                    mosaic_attribute = 'inundation_rasters',
+                                    mosaic_output = predicted_raster_path,
+                                    mask = os.path.join(self.fim_dir,'wbd.gpkg'),
+                                    unit_attribute_name = 'huc8',
+                                    nodata = elev_raster_ndv,
+                                    workers = 1,
+                                    remove_inputs = False,
+                                    subset = None,
+                                    verbose = verbose )
+            # FIM v3 and before
+            else:
+                fh.vprint("Begin FIM v3 (or earlier) Inundation", verbose)
+                inundate_result = inundate(self.rem, self.catchments, self.catchment_poly, self.hydro_table, benchmark_flows,
+                    self.mask_type,hucs=self.hucs,hucs_layerName=self.hucs_layerName,
+                    subset_hucs=self.huc,num_workers=1,aggregate=False,
+                    inundation_raster=inundation_path,inundation_polygon=None,
+                    depths=None,out_raster_profile=None,out_vector_profile=None,
+                    quiet=True)
+                if inundate_result != 0:
+                    return inundate_result
+
+        # Create contingency rasters and stats
+        fh.vprint("Begin creating contingency rasters and stats", verbose)
+        if os.path.isfile(predicted_raster_path):
+            compute_contingency_stats_from_rasters( predicted_raster_path,
+                                                    benchmark_rast,
+                                                    agreement_raster,
+                                                    stats_csv=stats_csv,
+                                                    stats_json=stats_json,
+                                                    mask_values=[],
+                                                    stats_modes_list=self.stats_modes_list,
+                                                    test_id=self.test_id,
+                                                    mask_dict=mask_dict_indiv )
+        return
+
+
+    @classmethod
+    def run_alpha_test(cls, version, test_id, magnitude, calibrated, model, archive_results=False, 
+                       mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True, verbose=False, gms_workers=1):
+        '''Class method for instantiating the test_case class and running alpha_test directly'''
+
+        alpha_class = cls(test_id, version, archive_results)
+        alpha_class.alpha_test(calibrated, model, mask_type, inclusion_area,
+                inclusion_area_buffer, overwrite, verbose, gms_workers)
+
+    def composite(self, version_2, calibrated = False, overwrite = True, verbose = False):
+        '''Class method for compositing MS and FR inundation and creating an agreement raster with stats
+
+            Parameters
+            ----------
+            version_2 : str
+                Version with which to composite.
+            calibrated : bool
+                Whether or not this FIM version is calibrated.
+            overwrite : bool
+                If True, overwites pre-existing test cases within the test_cases directory.
+        '''
+
+        if re.match(r'(.*)(_ms|_fr)', self.version):
+            composite_version_name = re.sub(r'(.*)(_ms|_fr)', r'\1_comp', self.version, count=1)
         else:
-            print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. ble options include: 100yr, 500yr. ahps options include action, minor, moderate, major." + ENDC)
-            exit_flag = True
+            composite_version_name = re.sub(r'(.*)(_ms|_fr)', r'\1_comp', version_2, count=1)
 
-    if exit_flag:
-        print()
-        sys.exit()
+        fh.vprint(f"Begin composite for version : {composite_version_name}", verbose)
+
+        composite_test_case = test_case(self.test_id, composite_version_name, self.archive)
+        input_test_case_2 = test_case(self.test_id, version_2, self.archive)
+        composite_test_case.stats_modes_list = ['total_area']
+
+        if not overwrite and os.path.isdir(composite_test_case.dir):
+            return
+
+        # Delete the directory if it exists
+        if os.path.exists(composite_test_case.dir):
+            shutil.rmtree(composite_test_case.dir)
+
+        validation_data = composite_test_case.data(composite_test_case.huc)
+        for magnitude in validation_data:
+            for instance in validation_data[magnitude]:      # instance will be the lid for AHPS sites and '' for other sites (ble/ifc/ras2fim)
+                inundation_prefix = instance + '_' if instance else ''
+
+                input_inundation = os.path.join(self.dir, magnitude, f'{inundation_prefix}inundation_extent_{self.huc}.tif')
+                input_inundation_2 = os.path.join(input_test_case_2.dir, magnitude, f'{inundation_prefix}inundation_extent_{input_test_case_2.huc}.tif')
+                output_inundation = os.path.join(composite_test_case.dir, magnitude, f'{inundation_prefix}inundation_extent.tif')
+
+                if os.path.isfile(input_inundation) and os.path.isfile(input_inundation_2):
+                    inundation_map_file = pd.DataFrame({ 
+                                            'huc8' : [composite_test_case.huc] * 2,
+                                            'branchID' : [None] * 2,
+                                            'inundation_rasters' : [input_inundation,input_inundation_2],
+                                            'depths_rasters' : [None] * 2,
+                                            'inundation_polygons' : [None] * 2
+                                        })
+                    os.makedirs(os.path.dirname(output_inundation), exist_ok=True)
+
+                    fh.vprint(f"Begin mosaic inundation for version : {composite_version_name}", verbose)
+                    Mosaic_inundation(inundation_map_file,mosaic_attribute='inundation_rasters',
+                                            mosaic_output=output_inundation, mask=None, unit_attribute_name='huc8',
+                                            nodata=elev_raster_ndv, workers=1, remove_inputs=False, subset=None, verbose=False
+                                            )
+                    composite_test_case._inundate_and_compute(magnitude, instance, compute_only=True)
+
+                elif os.path.isfile(input_inundation) or os.path.isfile(input_inundation_2): 
+                    # If only one model (MS or FR) has inundation, simply copy over all files as the composite
+                    single_test_case = self if os.path.isfile(input_inundation) else input_test_case_2
+                    shutil.copytree(single_test_case.dir, re.sub(r'(.*)(_ms|_fr)', r'\1_comp', single_test_case.dir, count=1))
+                    composite_test_case.write_metadata(calibrated, 'COMP')
+                    return
+
+            # Clean up 'total_area' outputs from AHPS sites
+            if composite_test_case.is_ahps:
+                composite_test_case.clean_ahps_outputs(os.path.join(composite_test_case.dir, magnitude))
+
+        composite_test_case.write_metadata(calibrated, 'COMP')
+
+    def write_metadata(self, calibrated, model):
+        '''Writes metadata files for a test_case directory.'''
+        with open(os.path.join(self.dir,'eval_metadata.json'),'w') as meta:
+            eval_meta = { 'calibrated' : calibrated , 'model' : model }
+            meta.write( 
+                        json.dumps(eval_meta,indent=2) 
+                    )
+    def clean_ahps_outputs(self, magnitude_directory):
+        '''Cleans up `total_area` files from an input AHPS magnitude directory.'''
+        output_file_list = [os.path.join(magnitude_directory, of) for of in os.listdir(magnitude_directory)]
+        for output_file in output_file_list:
+            if "total_area" in output_file:
+                os.remove(output_file)
+
+    def get_current_agreements(self):
+        '''Returns a list of all agreement rasters currently existing for the test_case.'''
+        agreement_list = []
+        for mag in os.listdir(self.dir):
+            mag_dir = os.path.join(self.dir, mag)
+            if not os.path.isdir(mag_dir): continue
+
+            for f in os.listdir(mag_dir):
+                if 'agreement.tif' in f:
+                    agreement_list.append(os.path.join(mag_dir, f))
+        return agreement_list
 
-    else:
-        run_alpha_test(**args)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index d1f0f7958..24ba9c0b0 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -1,18 +1,13 @@
 #!/usr/bin/env python3
 
-import os
-import argparse
+import os, argparse, json, csv, ast, re, sys, traceback, signal
 from datetime import datetime
 from multiprocessing import Pool
-from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
-import json
-import csv
-import ast
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 from tqdm import tqdm
-import re
-
-from run_test_case import run_alpha_test
 
+from utils.shared_functions import FIM_Helpers as fh
+from run_test_case import test_case
 from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT
 
 
@@ -214,6 +209,17 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         csv_writer = csv.writer(csvfile)
         csv_writer.writerows(list_to_write)
 
+def progress_bar_handler(executor_dict, verbose, desc):
+
+    for future in tqdm(as_completed(executor_dict),
+                    total=len(executor_dict),
+                    disable=(not verbose),
+                    desc=desc,
+                    ):
+        try:
+            future.result()
+        except Exception as exc:
+            print('{}, {}, {}'.format(executor_dict[future],exc.__class__.__name__,exc))
 
 
 if __name__ == '__main__':
@@ -288,9 +294,8 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
     fr_run_dir = args['fr_run_dir']
     calibrated = args['calibrated']
     model = args['model']
-    verbose = args['verbose']
-    gms_verbose = args['gms_verbose']
-
+    verbose = bool(args['verbose'])
+    gms_verbose = bool(args['gms_verbose'])
 
     print("================================")
     print("Start synthesize test cases")
@@ -298,7 +303,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
     print (f"started: {dt_string}")
     print()
-    
+
     # check job numbers
     total_cpus_requested = job_number_huc * job_number_branch
     total_cpus_available = os.cpu_count() - 1
@@ -326,124 +331,101 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
     else:
         print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
 
-    # List all available benchmark categories and test_cases.
-    test_cases_dir_list = os.listdir(TEST_CASES_DIR)
-    benchmark_category_list = []
-    if benchmark_category == "all":
-        for d in test_cases_dir_list:
-            if 'test_cases' in d:
-                benchmark_category_list.append(d.replace('_test_cases', ''))
-    else:
-        benchmark_category_list = [benchmark_category]
+    # Create a list of all test_cases for which we have validation data
+    all_test_cases = test_case.list_all_test_cases(version = fim_version, archive = archive_results,
+            benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
+    
+    # Set up multiprocessor
+    with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
 
-    # Loop through benchmark categories.
-    procs_list = [] ; procs_dict = {}
-    for bench_cat in benchmark_category_list:
-        
-        # Map path to appropriate test_cases folder and list test_ids into bench_cat_id_list.
-        bench_cat_test_case_dir = os.path.join(TEST_CASES_DIR, bench_cat + '_test_cases')
-        bench_cat_id_list = os.listdir(bench_cat_test_case_dir)
-        
-        # temp
-        #bench_cat_id_list = ['07060003_ifc']
-
-        #if job_number_huc == 1:
-            # something wrong about this lengtu
-            #pb = tqdm(total=len(bench_cat_id_list))
-        # Loop through test_ids in bench_cat_id_list.
-        for test_id in bench_cat_id_list:
-            if 'validation' and 'other' not in test_id:
-                current_huc = test_id.split('_')[0]
-                current_benchmark_category = test_id.split('_')[1]
-                if current_benchmark_category in bench_cat:
-                    # Loop through versions.
-                    for version in previous_fim_list:
-                        
-                        if config == 'DEV':
-                            fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
-                        elif config == 'PREV':
-                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
-
-                        # For previous versions of HAND computed at HUC6 scale
-                        if not os.path.exists(fim_run_dir):
-                            if config == 'DEV':
-                                fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc[:6])
-                            elif config == 'PREV':
-                                fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])
-                        
-                        if os.path.exists(fim_run_dir):
-
-                            # If a user supplies a special_string (-s), then add it to the end of the created dirs.
-                            if special_string != "":
-                                version = version + '_' + special_string
+        ## Loop through all test cases, build the alpha test arguments, and submit them to the process pool
+        executor_dict = {}
+        for test_case_class in all_test_cases:
             
-                            # Define the magnitude lists to use, depending on test_id.
-                            magnitude = MAGNITUDE_DICT[current_benchmark_category]
-
-                            alpha_test_args = { 
-                                                'fim_run_dir': fim_run_dir, 
-                                                'version': version, 
-                                                'test_id': test_id, 
-                                                'magnitude': magnitude, 
-                                                'calibrated': calibrated,
-                                                'model': model,
-                                                'compare_to_previous': not archive_results, 
-                                                'archive_results': archive_results, 
-                                                'mask_type': 'huc',
-                                                'overwrite': overwrite, 
-                                                'fr_run_dir': fr_run_dir, 
-                                                'gms_workers': job_number_branch,
-                                                'verbose': False,
-                                                'gms_verbose': False
-                                              }
-                            procs_dict[test_id] = alpha_test_args
-
-    if job_number_huc == 1:
-        
-        number_of_hucs = len(procs_dict)
-        verbose_by_huc = not number_of_hucs == 1
+            if not os.path.exists(test_case_class.fim_dir):
+                continue
 
-        for current_huc, alpha_test_args in tqdm(procs_dict.items(),total=number_of_hucs,disable=(not verbose_by_huc)):
-            alpha_test_args.update({'gms_verbose': not verbose_by_huc})
-            try:
-                run_alpha_test(**alpha_test_args)
-            except Exception as exc:
-                print('{}, {}, {}'.format(test_id,exc.__class__.__name__,exc))
+            fh.vprint(f"test_case_class.test_id is {test_case_class.test_id}", verbose)
 
-    # Multiprocess alpha test runs.
-    if job_number_huc > 1:
-        
-        #print(procs_dict)
-        executor = ProcessPoolExecutor(max_workers=job_number_huc)
-
-        executor_generator = { 
-                              executor.submit(run_alpha_test,**inp) : ids for ids,inp in procs_dict.items()
-                             }
-
-        for future in tqdm(as_completed(executor_generator),
-                           total=len(executor_generator),
-                           disable=(not verbose),
-                           desc="Running test cases with {} HUC workers "\
-                                "and {} Branch workers".format(job_number_huc,
-                                                               job_number_branch),
-                          ):
-        
-
-            hucCode = executor_generator[future]
+            alpha_test_args = { 
+                                'calibrated': calibrated,
+                                'model': model,
+                                'mask_type': 'huc',
+                                'overwrite': overwrite,
+                                'verbose':gms_verbose if model == 'GMS' else verbose,
+                                'gms_workers': job_number_branch
+                                }
 
             try:
-                future.result()
-            except Exception as exc:
-                print('{}, {}, {}'.format(hucCode,exc.__class__.__name__,exc))
-    
-        # power down pool
-        executor.shutdown(wait=True)
+                future = executor.submit(test_case_class.alpha_test, **alpha_test_args)
+                executor_dict[future] = test_case_class.test_id
+            except Exception as ex:
+                print(f"*** {ex}")
+                traceback.print_exc()
+                sys.exit(1)
+
+        # Send the executor to the progress bar and wait for all MS tasks to finish
+        progress_bar_handler(executor_dict, True, f"Running {model} alpha test cases with {job_number_huc} workers")
+        #wait(executor_dict.keys())
+
+    ## Composite alpha test run is initiated by a MS `model` and providing a `fr_run_dir`
+    if model == 'MS' and fr_run_dir:
+
+        ## Rebuild all test cases list with the FR version, loop through them and apply the alpha test
+        all_test_cases = test_case.list_all_test_cases(version = fr_run_dir, archive = archive_results,
+                benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
+
+        with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
+            executor_dict = {}
+            for test_case_class in all_test_cases:
+                if not os.path.exists(test_case_class.fim_dir):
+                    continue
+                alpha_test_args = { 
+                                    'calibrated': calibrated,
+                                    'model': model,
+                                    'mask_type': 'huc',
+                                    'verbose':verbose,
+                                    'overwrite': overwrite
+                                    }
+                try:
+                    future = executor.submit(test_case_class.alpha_test, **alpha_test_args)
+                    executor_dict[future] = test_case_class.test_id
+                except Exception as ex:
+                    print(f"*** {ex}")
+                    traceback.print_exc()
+                    sys.exit(1)
+
+            # Send the executor to the progress bar and wait for all FR tasks to finish
+            progress_bar_handler(executor_dict, True, f"Running FR test cases with {job_number_huc} workers")
+            #wait(executor_dict.keys())
+
+        # Loop through FR test cases, build composite arguments, and submit the composite method to the process pool
+        with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
+            executor_dict = {}
+            for test_case_class in all_test_cases:
+                composite_args = { 
+                                    'version_2': fim_version, # this is the MS version name since `all_test_cases` are FR
+                                    'calibrated': calibrated,
+                                    'overwrite': overwrite,
+                                    'verbose': verbose
+                                    }
+
+                try:
+                    future = executor.submit(test_case_class.alpha_test, **alpha_test_args)
+                    executor_dict[future] = test_case_class.test_id
+                except Exception as ex:
+                    print(f"*** {ex}")
+                    traceback.print_exc()
+                    sys.exit(1)
+
+            # Send the executor to the progress bar
+            progress_bar_handler(executor_dict, verbose, f"Compositing test cases with {job_number_huc} workers")
 
     if config == 'DEV':
         if dev_versions_to_compare != None:
-            dev_versions_to_include_list = dev_versions_to_compare + [version]
+            dev_versions_to_include_list = dev_versions_to_compare + previous_fim_list
         else:
-            dev_versions_to_include_list = [version]
+            dev_versions_to_include_list = previous_fim_list
     if config == 'PREV':
         dev_versions_to_include_list = []
 
@@ -452,10 +434,9 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         print("Creating master metrics CSV...")
 
         # this function is not compatible with GMS
-        create_master_metrics_csv(master_metrics_csv_output=master_metrics_csv, 
-                                  dev_versions_to_include_list=dev_versions_to_include_list)
+        create_master_metrics_csv(master_metrics_csv_output = master_metrics_csv, 
+                                  dev_versions_to_include_list = dev_versions_to_include_list)
     
-
     print("================================")
     print("End synthesize test cases")
 
diff --git a/unit_tests/shared_functions_unittests.py b/unit_tests/shared_functions_unittests.py
index 1c3e11905..fb6df76c1 100644
--- a/unit_tests/shared_functions_unittests.py
+++ b/unit_tests/shared_functions_unittests.py
@@ -12,8 +12,9 @@
 import unit_tests_utils as helpers
 
 # importing python folders in other directories
-sys.path.append('/foss_fim/src/utils') 
-import shared_functions as src
+#sys.path.append('/foss_fim/src/utils') 
+#import utils.shared_functions as src
+from utils.shared_functions import FIM_Helpers as src
 
 # NOTE: This goes directly to the function.
 # Ultimately, it should emulate going through command line (not import -> direct function call)
@@ -40,7 +41,7 @@ def test_append_id_to_file_name_single_identifier_success(self):
         '''
 
         #global params_file
-        helpers.print_unit_test_function_header()
+        #helpers.print_unit_test_function_header()
         
         params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
         
@@ -63,7 +64,7 @@ def test_append_id_to_file_name_indentifer_list_success(self):
         '''
 
         #global params_file
-        helpers.print_unit_test_function_header()
+        #helpers.print_unit_test_function_header()
         
         params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()        
         
@@ -86,7 +87,7 @@ def test_append_id_to_file_name_no_file_name_success(self):
         '''
 
         #global params_file
-        helpers.print_unit_test_function_header()
+        #helpers.print_unit_test_function_header()
         
         params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
         

From 1ca6b65f18ae4a4ac3f02237a3b3d4205a473050 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 21 Jul 2022 15:21:33 -0500
Subject: [PATCH 144/359] added version to pipenv (#634)

---
 Dockerfile      | 3 ++-
 Dockerfile.prod | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 5848ecaa5..87ee08259 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -86,7 +86,8 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 ## install python 3 modules ##
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
+RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
+
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
 ADD ./src/entrypoint.sh /
diff --git a/Dockerfile.prod b/Dockerfile.prod
index 5c5c7964b..a71933c6c 100644
--- a/Dockerfile.prod
+++ b/Dockerfile.prod
@@ -89,7 +89,7 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 ## install python 3 modules ##
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
+RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
 
 ## Copy the source code to the image
 COPY . $projectDir/

From e12c2ad59c8e97d4450e11c95e2ba2c7e25b07d8 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 29 Jul 2022 13:29:13 -0500
Subject: [PATCH 145/359] Updated unit tests based on latest code and adjusted
 some pathing (#622)

---
 config/deny_gms_branch_unittests.lst          |  59 ++++++++
 docs/CHANGELOG.md                             |   4 +
 unit_tests/README.md                          |  37 ++++-
 unit_tests/__example_nws_data_README.md       |  76 ----------
 unit_tests/__example_nws_data_params.json     |   6 -
 unit_tests/__example_nws_data_unittests.py    | 127 ----------------
 unit_tests/__template_unittests.py            |  19 +--
 unit_tests/clip_vectors_to_wbd_params.json    |  18 +--
 unit_tests/clip_vectors_to_wbd_unittests.py   |  58 ++++---
 ..._catchments_and_add_attributes_params.json |  14 +-
 ...catchments_and_add_attributes_unittests.py |  17 ++-
 .../gms/derive_level_paths_unittests.py       | 143 ++++++++----------
 .../rating_curve_comparison_params.json       |  30 ++--
 .../rating_curve_comparison_unittests.py      |  16 +-
 unit_tests/shared_functions_params.json       |  10 +-
 unit_tests/shared_functions_unittests.py      |  29 +---
 unit_tests/split_flows_params.json            |  14 +-
 unit_tests/split_flows_unittests.py           |  10 +-
 .../gms_tools}/inundate_gms_params.json       |   4 +-
 .../gms_tools}/inundate_gms_unittests.py      |  14 +-
 unit_tests/tools/inundate_params.json         |  24 +++
 unit_tests/tools/inundate_unittests.py        |  91 +++++++++++
 unit_tests/unit_tests_utils.py                |  23 ++-
 unit_tests/usgs_gage_crosswalk_params.json    |  15 +-
 unit_tests/usgs_gage_crosswalk_unittests.py   |  15 +-
 25 files changed, 401 insertions(+), 472 deletions(-)
 create mode 100644 config/deny_gms_branch_unittests.lst
 delete mode 100644 unit_tests/__example_nws_data_README.md
 delete mode 100644 unit_tests/__example_nws_data_params.json
 delete mode 100644 unit_tests/__example_nws_data_unittests.py
 rename unit_tests/{gms => tools/gms_tools}/inundate_gms_params.json (89%)
 rename unit_tests/{gms => tools/gms_tools}/inundate_gms_unittests.py (89%)
 create mode 100644 unit_tests/tools/inundate_params.json
 create mode 100644 unit_tests/tools/inundate_unittests.py

diff --git a/config/deny_gms_branch_unittests.lst b/config/deny_gms_branch_unittests.lst
new file mode 100644
index 000000000..530403984
--- /dev/null
+++ b/config/deny_gms_branch_unittests.lst
@@ -0,0 +1,59 @@
+# List of files for gms branches to delete
+# Use comment to allow list the file
+# Use curly braces to denote branch_id
+agree_bufgrid.tif
+agree_bufgrid_allo.tif
+agree_bufgrid_dist.tif
+agree_smogrid.tif
+agree_smogrid_allo.tif
+agree_smogrid_dist.tif
+catch_list_{}.txt
+coordFile_{}.txt
+crosswalk_table_{}.csv
+#demDerived_reaches_{}.dbf
+#demDerived_reaches_{}.prj
+#demDerived_reaches_{}.shp
+#demDerived_reaches_{}.shx
+#demDerived_reaches_split_{}.gpkg
+#demDerived_reaches_split_filtered_{}.gpkg
+demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+demDerived_reaches_split_points_{}.gpkg
+demDerived_streamPixels_{}.tif
+demDerived_streamPixels_ids_{}.tif
+demDerived_streamPixels_ids_{}_allo.tif
+demDerived_streamPixels_ids_{}_dist.tif
+dem_burned_{}.tif
+dem_burned_filled_{}.tif
+dem_lateral_thalweg_adj_{}.tif
+#dem_meters_{}.tif
+#dem_thalwegCond_{}.tif
+flowaccum_d8_burned_filled_{}.tif
+#flowdir_d8_burned_filled_{}.tif
+flowdir_d8_burned_filled_flows_{}.tif
+flows_grid_boolean_{}.tif
+flows_points_pixels_{}.gpkg
+gw_catchments_pixels_{}.tif
+#gw_catchments_reaches_{}.gpkg
+gw_catchments_reaches_{}.tif
+#gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_{}.tif
+gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+headwaters_{}.tif
+#hydroTable_{}.csv
+idFile_{}.txt
+nld_rasterized_elev_{}.tif
+nwm_catchments_proj_subset_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
+rem_{}.tif
+#rem_zeroed_masked_{}.tif
+slopes_d8_dem_meters_{}.tif
+slopes_d8_dem_meters_masked_{}.tif
+sn_catchments_reaches_{}.tif
+src_{}.json
+src_base_{}.csv
+#src_full_crosswalked_{}.csv
+stage_{}.txt
+streamOrder_{}.tif
+treeFile_{}.txt
+#usgs_elev_table.csv
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4d91c8ad5..0a9314424 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,10 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.5.2 - 2022-07-25 - [PR #622](https://github.com/NOAA-OWP/inundation-mapping/pull/622)
+
+Updates to unit tests including a minor update for outputs and loading in .json parameter files.
+<br><br>
 
 ## v4.0.5.1 - 2022-06-27 - [PR #612](https://github.com/NOAA-OWP/inundation-mapping/pull/612)
 
diff --git a/unit_tests/README.md b/unit_tests/README.md
index 0238b4529..39dd33174 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -32,10 +32,25 @@ One way is to use the incoming arg parser. Most py files, they include the code
 Start a docker container as you normally would for any development. ie) docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
 	- ie) docker run --rm -it --name Rob_GMS -v /home/rob_abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
 
+For unit tests to work, you need to run the following (if not already in place).
+Notice a modified branch "deny_gms_branch_unittests.lst"  (special for unittests)
+
+Here are the params and args you need if you need to re-run unit and branch
+
+gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o -s
+	
+gms_run_branch.sh -n fim_unit_test_data_do_not_remove -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_branch_unittests.lst -o -s
+
+**NOTICE: the deny file used for gms_run_branch... its a special one for unittests `deny_gms_branch_unittests.lst`.
+
+If you need to run inundation tests, fun the following:
+
+python3 foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v fim_unit_test_data_do_not_remove -jh 1 -jb 1 -m /outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
+
+If you want to test just one unit test, here is an example:
 At the root terminal window, run:  python ./foss_fim/unit_tests/gms/derive_level_paths_unittests.py  or python ./foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
 (replace with your own script and path name)
 
-
 ## Key Notes for creating new unit tests
 1) All test functions must start with the phrase "test_". That is how the unit test engine picks it up. The rest of the function name does not have to match the pattern of {function name being tested} but should. Further, the rest of the function name should say what the test is about, ie) _failed_input_path.  ie) test_{some_function_name_from_the_source_code_file}_failed_input_path. It is fine that the function names get very long (common in the industry).
 
@@ -55,18 +70,15 @@ At the root terminal window, run:  python ./foss_fim/unit_tests/gms/derive_level
 
 9) One py file = one "{original py file name}_unittests.py" file.
 
-10) Sometimes you may want to run a full successful "happy path" version through gms_run_by_unit.sh (or similar), to get all of the files you need in place to do your testing. However.. you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the "-d" parameter (denylist). ie) Normally:  gms_run_unit.sh -n gms_example_unit_tests -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o, but ours would be 
-gms_run_unit.sh -n gms_example_unit_tests -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o. 
-
+10) Sometimes you may want to run a full successful "happy path" version through gms_run_by_unit.sh (or similar), to get all of the files you need in place to do your testing. However.. you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the "-d" parameter (denylist). ie) Normally:  gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o, but ours would be 
+gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o. 
 
 ## Future Enhancements
 1) We can automate triggers on these files for things like checking triggers or an single global "run_all_unittest" script, but for now.. its one offs.
 
 2) Better output from the unit tests including verbosity output control
 
-3) We will upgrade it so you can pass in a params.json file to the call to give you even more flexibility by passing in your own params.json (think params_template.json idea)
-
-4) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
+3) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
 
 
 ## testing for failing conditions
@@ -80,4 +92,15 @@ An example is in unit_tests/gms/Derive_level_paths_unittests.py -> test_Derive_l
 
 We have almost no "assert"s yet, but most unit test usually have one or more "assert" test. See https://docs.python.org/3/library/unittest.html for more details.
 
+## Unit tests currently available
+python3 /foss_fim/unit_tests/gms/derive_level_paths_unittests.py
+python3 /foss_fim/unit_tests/tools/inundate_unittests.py
+python3 /foss_fim/unit_tests/tools/gms_tools/inundate_gms_unittests.py
+python3 /foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
+python3 /foss_fim/unit_tests/filter_catchments_and_add_attributes_unittests.py
+python3 /foss_fim/unit_tests/rating_curve_comparison_unittests.py
+python3 /foss_fim/unit_tests/shared_functions_unittests.py
+python3 /foss_fim/unit_tests/split_flows_unittests.py
+python3 /foss_fim/unit_tests/usgs_gage_crosswalk_unittests.py
+
 
diff --git a/unit_tests/__example_nws_data_README.md b/unit_tests/__example_nws_data_README.md
deleted file mode 100644
index 151532420..000000000
--- a/unit_tests/__example_nws_data_README.md
+++ /dev/null
@@ -1,76 +0,0 @@
-
-## Sample unit tests - for demo purposes only#
-
-    Used to demonstrate __example_nws_data_unittests.py only.
-	 The __example_nws_data_unittest.py files shows how success and fails method look like as well as things like using actual results, expected results and asserts.
-
-# Here is a unit test being used in LISFLOOD
-
-`Here are some example code changes that would be in place if the __example files were valid (doesn't actually exist).`
-
-- There is a new variable added to src/utils/shared_variables -> DATA_SOURCE_PATH_NWS_LID_GPKG = "/data/inputs/ahp_sites/01_27_22/nws_lid.gpkg"
-
-- A python file exists in src called nws_data.py (again.. doesn't actually)
-
-Here are the contents of that pretend file (for this repo)
-
-+++++++++++++++++++++++++++++++++++++++++++
-
-import geopandas as gpd
-import utils.shared_variables
-
-def get_huc8_by_nws_lid(nws_lid):
-
-    '''
-    OVERVIEW:
-        By passing in a nws_lid (also called a ahps id), it will open the ahps_sites/nws_lid.gpkg, find the record and return the huc8 value.
-
-    INPUT PARAMETERS:
-        - nws_lid: a string value (case sensitive) which should match the nws_lids.gpkg -> nws_lid column.
-
-    PROCESSING / VALIDATION:
-        - IF the nws record is not found, an exception will be raised.
-        - The nws_lid.gpkg path is defined by the new src/utils/data_sources.env parameter.
-        - If more than one nws record was found by nws_lid, it is assumed as bad data and an exception will be raised.
-        - nws_lid will be changed to upper case to match the database
-
-    RETURNS:
-       The discovered HUC8 value.
-    '''
-
-    # -----------------------------
-    # VALIDATION OF INCOMING VALUES
-
-    # Ensure the nws_lid value is not "None" or empty
-    if (not nws_lid):
-        raise Exception("nw_lid parameter appears to be empty or invalid")
-
-    # Ensure there are no spaces inside or on either end of the nws_lid value
-    if (" " in nws_lid):
-        raise Exception("nws_lid parameter appears to have invalid spaces either in or around the value. Please check the incoming param value.")
-
-
-    # -----------------------------
-    # PROCESSING:
-
-    nws_lid = nws_lid.upper()
-
-    # get the huc from the nws_lid.gpkg
-    nws_lid_data = gpd.read_file(utils.shared_variables.DATA_SOURCE_PATH_NWS_LID_GPKG)
-
-    nws_record = nws_lid_data.loc[nws_lid_data['nws_lid'] == nws_lid]
-
-    if nws_record.empty:
-        raise Exception("No nws_lid record found for nws_id of " + nws_lid)
-
-    if len(nws_record) > 1:
-        raise Exception("More than one nws_lid record was found for nws_id of " + nws_lid + ". Check the nws_lid.gpkg as there may be some bad data.")
-
-    huc8 = nws_record['HUC8'].values[0]
-    if (not huc8):
-        raise Exception("nws_lid record found but no huc value found. Check nws_lid.gpkg")
-
-    return(huc8)
-
-
-+++++++++++++++++++++++++++++++++++++++++++
diff --git a/unit_tests/__example_nws_data_params.json b/unit_tests/__example_nws_data_params.json
deleted file mode 100644
index b44e437ed..000000000
--- a/unit_tests/__example_nws_data_params.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-	"valid_data":
-		{
-			"nws_lid": "smit2"
-		}
-}
diff --git a/unit_tests/__example_nws_data_unittests.py b/unit_tests/__example_nws_data_unittests.py
deleted file mode 100644
index 8834b5b84..000000000
--- a/unit_tests/__example_nws_data_unittests.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import argparse
-import json
-import warnings
-import unittest
-
-import unit_tests_utils as helpers
-
-# importing python folders in other direcories
-sys.path.append('/foss_fim/src/')
-import nws_data
-
-class test_get_huc8_by_nws_lid(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        
-        params_file_path = '/foss_fim/unit_tests/nws_data_params.json'
-    
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    def test_get_huc8_by_nws_lid_success(self):
-
-        '''
-            Tests that the correct huc8 is returned with a valid nws-lid
-        '''
-
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
-        params = self.params["valid_data"].copy()
-
-        print(params["nws_lid"])
-    
-        # for now we are happy if no exceptions are thrown.
-        actual_response = nws_data.get_huc8_by_nws_lid(nws_lid = params["nws_lid"])
-        expected_response = '12090301'
-
-        self.assertEqual(actual_response, expected_response, "HUC8 does not match expected result")
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_get_huc8_by_nws_lid_fail_invalid_nws_lid(self):
-
-        '''
-            Tests that the nws_lid does not exist in the nws_lid.gpkg
-            Expecting an exception to be thrown
-        '''
-
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
-        #params = self.params["valid_data"].copy()
-
-        try:
-
-            nws_invalid_value = "abc"
-    
-            # for now we are happy if no exceptions are thrown.
-            actual_response = nws_data.get_huc8_by_nws_lid(nws_lid = nws_invalid_value)
-
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        finally:
-            print("*************************************************************")
-
-
-    def test_get_huc8_by_nws_lid_fail_empty_nws_lid(self):
-
-        '''
-            Tests that the nws_lid is empty
-            Expecting an exception to be thrown
-        '''
-
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
-        #params = self.params["valid_data"].copy()
-
-        try:
-
-            nws_invalid_value = " " # add a blank space to further test
-    
-            # for now we are happy if no exceptions are thrown.
-            actual_response = nws_data.get_huc8_by_nws_lid(nws_lid = nws_invalid_value)
-
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        finally:
-            print("*************************************************************")
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
\ No newline at end of file
diff --git a/unit_tests/__template_unittests.py b/unit_tests/__template_unittests.py
index 01195723b..4f1e465b1 100644
--- a/unit_tests/__template_unittests.py
+++ b/unit_tests/__template_unittests.py
@@ -4,16 +4,16 @@
 import os
 import sys
 
-import argparse
 import json
 import warnings
 import unittest
 
-import unit_tests_utils as helpers
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other directories
 sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import <Your original source python file name>   
+import <Your original source python file name> as src
 
 # NOTE: This goes directly to the function.
 # Ultimately, it should emulate going through command line (not import -> direct function call)
@@ -26,9 +26,7 @@ class test_<Your original source python file name>(unittest.TestCase):
     def setUpClass(self):
 
         warnings.simplefilter('ignore')
-
-        params_file_path = '/foss_fim/unit_tests/<Your original source python file name>_params.json'
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
@@ -49,7 +47,6 @@ def test_<method name you are about to test>_success(self):
         '''
 
         #global params_file
-        helpers.print_unit_test_function_header()
         
         params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
         
@@ -63,10 +60,10 @@ def test_<method name you are about to test>_success(self):
         # Remember, if the method accepts **params, then you can sent that in here as well.
         #   ie: my_py_class.my_method(** params)
         
-        clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
-                                                nwm_streams_filename = params["nwm_streams"],
-                                                nhd_streams_filename = params["nhd_streams"],
-                                                etc, etc for each param)
+        src.subset_vector_layers(hucCode = params["hucCode"],
+                                    nwm_streams_filename = params["nwm_streams"],
+                                    nhd_streams_filename = params["nhd_streams"],
+                                    etc, etc for each param)
        
        
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 004aef392..388ce1094 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -6,18 +6,18 @@
 		"nhd_streams": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg", 
 		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
 		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
-		"wbd": "/data/outputs/gms_example_unit_tests/05030104/wbd.gpkg",
-		"wbd_buffer": "/data/outputs/gms_example_unit_tests/05030104/wbd_buffered.gpkg",
+		"wbd": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
+		"wbd_buffer": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
 		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", 
 		"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
-		"subset_nhd_streams": "/data/outputs/gms_example_unit_tests/05030104/NHDPlusBurnLineEvent_subset.gpkg",
-		"subset_nld_lines": "/data/outputs/gms_example_unit_tests/05030104/nld_subset_levees.gpkg", 
-		"subset_lakes": "/data/outputs/gms_example_unit_tests/05030104/nwm_lakes_proj_subset.gpkg",
-		"subset_catchments": "/data/outputs/gms_example_unit_tests/05030104/nwm_catchments_proj_subset.gpkg",
-		"subset_nhd_headwaters": "/data/outputs/gms_example_unit_tests/05030104/nhd_headwater_points_subset.gpkg", 
-		"subset_nwm_streams": "/data/outputs/gms_example_unit_tests/05030104/nwm_subset_streams.gpkg", 
-		"subset_landsea": "/data/outputs/gms_example_unit_tests/05030104/LandSea_subset.gpkg", 
+		"subset_nhd_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/NHDPlusBurnLineEvent_subset.gpkg",
+		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg", 
+		"subset_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
+		"subset_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"subset_nhd_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg", 
+		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg", 
+		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg", 
 		"extent": "GMS",
 		"great_lakes_filename": "/data/inputs/landsea/gl_water_polygons.gpkg", 
 		"wbd_buffer_distance": 5000,
diff --git a/unit_tests/clip_vectors_to_wbd_unittests.py b/unit_tests/clip_vectors_to_wbd_unittests.py
index 619a0dc6c..b6cd35f9c 100644
--- a/unit_tests/clip_vectors_to_wbd_unittests.py
+++ b/unit_tests/clip_vectors_to_wbd_unittests.py
@@ -4,16 +4,16 @@
 import os
 import sys
 
-import argparse
 import json
 import warnings
 import unittest
 
-import unit_tests_utils as helpers
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other direcories
 sys.path.append('/foss_fim/src/')
-import clip_vectors_to_wbd
+import clip_vectors_to_wbd as src
 
 # NOTE: This goes directly to the function.
 # Ultimately, it should emulate going through command line (not import -> direct function call)
@@ -26,9 +26,7 @@ class test_clip_vectors_to_wbd(unittest.TestCase):
     def setUpClass(self):
 
         warnings.simplefilter('ignore')
-
-        params_file_path = '/foss_fim/unit_tests/clip_vectors_to_wbd_params.json'
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
@@ -44,39 +42,35 @@ def test_subset_vector_layers_success(self):
         For now, we are adding the very basic "happy path" test.
         '''
 
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
         params = self.params["valid_data"].copy()
-        
        
         # There is no default return value.
         # Later we need to check in this function that files were being created on the file system
         
         # for now we are happy if no exceptions are thrown.
-        clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
-                                                nwm_streams_filename = params["nwm_streams"],
-                                                nhd_streams_filename = params["nhd_streams"],
-                                                nwm_lakes_filename = params["nwm_lakes"],
-                                                nld_lines_filename = params["nld_lines"],
-                                                nwm_catchments_filename = params["nwm_catchments"],
-                                                nhd_headwaters_filename = params["nhd_headwaters"],
-                                                landsea_filename = params["landsea"],
-                                                wbd_filename = params["wbd"],
-                                                wbd_buffer_filename = params["wbd_buffer"],
-                                                subset_nhd_streams_filename = params["subset_nhd_streams"],
-                                                subset_nld_lines_filename = params["subset_nld_lines"],
-                                                subset_nwm_lakes_filename = params["subset_lakes"],
-                                                subset_nwm_catchments_filename = params["subset_catchments"],
-                                                subset_nhd_headwaters_filename = params["subset_nhd_headwaters"],
-                                                subset_nwm_streams_filename = params["subset_nwm_streams"],
-                                                subset_landsea_filename = params["subset_landsea"],
-                                                extent = params["extent"],
-                                                great_lakes_filename = params["great_lakes_filename"],
-                                                wbd_buffer_distance = params["wbd_buffer_distance"],
-                                                lake_buffer_distance = params["lake_buffer_distance"])
-       
+        src.subset_vector_layers(hucCode = params["hucCode"],
+                                nwm_streams_filename = params["nwm_streams"],
+                                nhd_streams_filename = params["nhd_streams"],
+                                nwm_lakes_filename = params["nwm_lakes"],
+                                nld_lines_filename = params["nld_lines"],
+                                nwm_catchments_filename = params["nwm_catchments"],
+                                nhd_headwaters_filename = params["nhd_headwaters"],
+                                landsea_filename = params["landsea"],
+                                wbd_filename = params["wbd"],
+                                wbd_buffer_filename = params["wbd_buffer"],
+                                subset_nhd_streams_filename = params["subset_nhd_streams"],
+                                subset_nld_lines_filename = params["subset_nld_lines"],
+                                subset_nwm_lakes_filename = params["subset_lakes"],
+                                subset_nwm_catchments_filename = params["subset_catchments"],
+                                subset_nhd_headwaters_filename = params["subset_nhd_headwaters"],
+                                subset_nwm_streams_filename = params["subset_nwm_streams"],
+                                subset_landsea_filename = params["subset_landsea"],
+                                extent = params["extent"],
+                                great_lakes_filename = params["great_lakes_filename"],
+                                wbd_buffer_distance = params["wbd_buffer_distance"],
+                                lake_buffer_distance = params["lake_buffer_distance"])
        
+
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")
 
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
index 454a930b8..b8c2378cb 100644
--- a/unit_tests/filter_catchments_and_add_attributes_params.json
+++ b/unit_tests/filter_catchments_and_add_attributes_params.json
@@ -1,12 +1,12 @@
 {
 	"valid_data":
 	{
-		"input_catchments_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/gw_catchments_reaches_3246000305.gpkg",
-		"input_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_3246000305.gpkg",
-		"output_catchments_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/gw_catchments_reaches_filtered_addedAttributes_3246000305.gpkg",
-		"output_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_filtered_3246000305.gpkg",
-	    "wbd_filename": "/data/outputs/gms_example_unit_tests/02020005/wbd8_clp.gpkg",
+		"input_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_3246000005.gpkg",
+		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
+		"output_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
+		"output_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
+	    "wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
 		"huc_code": "02020005",
-		"drop_stream_orders": false
+		"drop_stream_orders": true
 	}
-}
\ No newline at end of file
+}
diff --git a/unit_tests/filter_catchments_and_add_attributes_unittests.py b/unit_tests/filter_catchments_and_add_attributes_unittests.py
index 07126629f..c7e403081 100644
--- a/unit_tests/filter_catchments_and_add_attributes_unittests.py
+++ b/unit_tests/filter_catchments_and_add_attributes_unittests.py
@@ -4,12 +4,12 @@
 import os
 import sys
 
-import argparse
 import json
 import warnings
 import unittest
 
-import unit_tests_utils as helpers
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other directories
 sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
@@ -17,6 +17,12 @@
 
 # NOTE: This goes directly to the function.
 # Ultimately, it should emulate going through command line (not import -> direct function call)
+
+
+# *************
+# Important: For this to work, when you run gms_run_branch.sh, you have to 
+# use deny_gms_branches_default.lst or one that does not exist. Key files need to exist
+# for this unit test to work.
 class test_filter_catchments_and_add_attributes(unittest.TestCase):
 
     '''
@@ -26,13 +32,10 @@ class test_filter_catchments_and_add_attributes(unittest.TestCase):
     def setUpClass(self):
 
         warnings.simplefilter('ignore')
-
-        params_file_path = '/foss_fim/unit_tests/filter_catchments_and_add_attributes_params.json'
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-
     # MUST start with the name of "test_"
     # This is the (or one of the) valid test expected to pass
     def test_filter_catchments_and_add_attributes_success(self):
@@ -43,8 +46,6 @@ def test_filter_catchments_and_add_attributes_success(self):
         If the test is successful, these file will be created.
         '''
 
-        #global params_file
-        helpers.print_unit_test_function_header()
         params = self.params["valid_data"].copy() 
 
         # to setup the test, lets start by deleted the two expected output files to ensure
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
index c5e92e96e..fe7c1fa9e 100644
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -4,20 +4,19 @@
 import os
 import sys
 
-import argparse
 import json
 import warnings
 import unittest
 
 # importing python folders in other direcories
 sys.path.append('/foss_fim/unit_tests/')
-import unit_tests_utils as helpers
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other direcories
 sys.path.append('/foss_fim/src/gms/')
-import derive_level_paths
+import derive_level_paths as src
 import stream_branches
-from utils.fim_enums import FIM_system_exit_codes as fsec
+from utils.fim_enums import FIM_system_exit_codes as fec
 
 
 # NOTE: This goes directly to the function.
@@ -31,23 +30,16 @@ class test_Derive_level_paths(unittest.TestCase):
     def setUpClass(self):
     
         warnings.simplefilter('ignore')
-        
-        params_file_path = '/foss_fim/unit_tests/gms/derive_level_paths_params.json'    
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-
-
     def test_Derive_level_paths_success_all_params(self):
     
         '''
         This test includes all params with many optional parms being set to the default value of the function
         '''
-       
-        # makes output readability easier and consistant with other unit tests       
-        helpers.print_unit_test_function_header()
-        
+      
         params = self.params["valid_data"].copy()
 
         # Function Notes:
@@ -55,17 +47,17 @@ def test_Derive_level_paths_success_all_params(self):
         # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
         # skip them here.
         # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                                       out_stream_network = params["out_stream_network"],
-                                                       branch_id_attribute = params["branch_id_attribute"],
-                                                       out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                                       headwaters_outfile = params["headwaters_outfile"],
-                                                       catchments = params["catchments"],
-                                                       catchments_outfile = params["catchments_outfile"],
-                                                       branch_inlets_outfile = params["branch_inlets_outfile"],
-                                                       reach_id_attribute = params["reach_id_attribute"],
-                                                       verbose = params["verbose"],
-                                                       drop_low_stream_orders=params["drop_low_stream_orders"])
+        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                            out_stream_network = params["out_stream_network"],
+                                            branch_id_attribute = params["branch_id_attribute"],
+                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                            headwaters_outfile = params["headwaters_outfile"],
+                                            catchments = params["catchments"],
+                                            catchments_outfile = params["catchments_outfile"],
+                                            branch_inlets_outfile = params["branch_inlets_outfile"],
+                                            reach_id_attribute = params["reach_id_attribute"],
+                                            verbose = params["verbose"],
+                                            drop_low_stream_orders=params["drop_low_stream_orders"])
 
         # -----------
         # test data type being return is as expected. Downstream code might to know that type
@@ -111,10 +103,7 @@ def test_Derive_level_paths_success_drop_low_stream_orders_not_submitted(self):
         Note: Most path tests done in test_Derive_level_paths_success_all_params 
         and are not repeated here.
         '''
-        
-        # makes output readability easier and consistant with other unit tests       
-        helpers.print_unit_test_function_header()
-        
+       
         params = self.params["valid_data"].copy()
 
         # Function Notes:
@@ -122,16 +111,16 @@ def test_Derive_level_paths_success_drop_low_stream_orders_not_submitted(self):
         # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
         # skip them here.
         # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                                       out_stream_network = params["out_stream_network"],
-                                                       branch_id_attribute = params["branch_id_attribute"],
-                                                       out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                                       headwaters_outfile = params["headwaters_outfile"],
-                                                       catchments = params["catchments"],
-                                                       catchments_outfile = params["catchments_outfile"],
-                                                       branch_inlets_outfile = params["branch_inlets_outfile"],
-                                                       reach_id_attribute = params["reach_id_attribute"],
-                                                       verbose = params["verbose"])
+        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                            out_stream_network = params["out_stream_network"],
+                                            branch_id_attribute = params["branch_id_attribute"],
+                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                            headwaters_outfile = params["headwaters_outfile"],
+                                            catchments = params["catchments"],
+                                            catchments_outfile = params["catchments_outfile"],
+                                            branch_inlets_outfile = params["branch_inlets_outfile"],
+                                            reach_id_attribute = params["reach_id_attribute"],
+                                            verbose = params["verbose"])
 
         # -----------
         # test data type being return is as expected. Downstream code might to know that type
@@ -157,9 +146,6 @@ def test_Derive_level_paths_success_drop_low_stream_orders_is_true(self):
         and are not repeated here.
         '''
         
-        # makes output readability easier and consistant with other unit tests       
-        helpers.print_unit_test_function_header()
-        
         params = self.params["valid_data"].copy()
 
         params["drop_low_stream_orders"] = True
@@ -169,17 +155,17 @@ def test_Derive_level_paths_success_drop_low_stream_orders_is_true(self):
         # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
         # skip them here.
         # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                                       out_stream_network = params["out_stream_network"],
-                                                       branch_id_attribute = params["branch_id_attribute"],
-                                                       out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                                       headwaters_outfile = params["headwaters_outfile"],
-                                                       catchments = params["catchments"],
-                                                       catchments_outfile = params["catchments_outfile"],
-                                                       branch_inlets_outfile = params["branch_inlets_outfile"],
-                                                       reach_id_attribute = params["reach_id_attribute"],
-                                                       verbose = params["verbose"],
-                                                       drop_low_stream_orders=params["drop_low_stream_orders"])
+        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                            out_stream_network = params["out_stream_network"],
+                                            branch_id_attribute = params["branch_id_attribute"],
+                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                            headwaters_outfile = params["headwaters_outfile"],
+                                            catchments = params["catchments"],
+                                            catchments_outfile = params["catchments_outfile"],
+                                            branch_inlets_outfile = params["branch_inlets_outfile"],
+                                            reach_id_attribute = params["reach_id_attribute"],
+                                            verbose = params["verbose"],
+                                            drop_low_stream_orders=params["drop_low_stream_orders"])
 
         # -----------
         # test data type being return is as expected. Downstream code might to know that type
@@ -207,24 +193,22 @@ def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self
 
         '''
         
-        # makes output readability easier and consistant with other unit tests       
-        helpers.print_unit_test_function_header()
-        
         params = self.params["dropped_stream_orders_no_branches_remaining"].copy()
 
         with self.assertRaises(SystemExit) as se:
-            derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                                out_stream_network = params["out_stream_network"],
-                                                branch_id_attribute = params["branch_id_attribute"],
-                                                out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                                headwaters_outfile = params["headwaters_outfile"],
-                                                catchments = params["catchments"],
-                                                catchments_outfile = params["catchments_outfile"],
-                                                branch_inlets_outfile = params["branch_inlets_outfile"],
-                                                reach_id_attribute = params["reach_id_attribute"],
-                                                verbose = params["verbose"],
-                                                drop_low_stream_orders=params["drop_low_stream_orders"])
-        self.assertEqual(se.exception.code, fsec.GMS_UNIT_NO_BRANCHES.value)
+            src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                    out_stream_network = params["out_stream_network"],
+                                    branch_id_attribute = params["branch_id_attribute"],
+                                    out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                    headwaters_outfile = params["headwaters_outfile"],
+                                    catchments = params["catchments"],
+                                    catchments_outfile = params["catchments_outfile"],
+                                    branch_inlets_outfile = params["branch_inlets_outfile"],
+                                    reach_id_attribute = params["reach_id_attribute"],
+                                    verbose = params["verbose"],
+                                    drop_low_stream_orders=params["drop_low_stream_orders"])
+
+        self.assertEqual(se.exception.code, fec.GMS_UNIT_NO_BRANCHES.value)
 
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")        
@@ -233,9 +217,6 @@ def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self
     # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
     def test_Derive_level_paths_invalid_input_stream_network(self):
 
-        # makes output readability easier and consistant with other unit tests 
-        helpers.print_unit_test_function_header()
-        
         # NOTE: As we are expecting an exception, this MUST have a try catch.
         #   for "success" tests (as above functions), they CAN NOT have a try/catch
         try:
@@ -243,17 +224,17 @@ def test_Derive_level_paths_invalid_input_stream_network(self):
             params = self.params["valid_data"].copy()
             params["in_stream_network"] = "some bad path"
           
-            actual = derive_level_paths.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                                           out_stream_network = params["out_stream_network"],
-                                                           branch_id_attribute = params["branch_id_attribute"],
-                                                           out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                                           huc_id = params["huc_id"],
-                                                           headwaters_outfile = params["headwaters_outfile"],
-                                                           catchments = params["catchments"],
-                                                           catchments_outfile = params["catchments_outfile"],
-                                                           branch_inlets_outfile = params["branch_inlets_outfile"],
-                                                           reach_id_attribute = params["reach_id_attribute"],
-                                                           verbose = params["verbose"] )
+            actual = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                            out_stream_network = params["out_stream_network"],
+                                            branch_id_attribute = params["branch_id_attribute"],
+                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                            huc_id = params["huc_id"],
+                                            headwaters_outfile = params["headwaters_outfile"],
+                                            catchments = params["catchments"],
+                                            catchments_outfile = params["catchments_outfile"],
+                                            branch_inlets_outfile = params["branch_inlets_outfile"],
+                                            reach_id_attribute = params["reach_id_attribute"],
+                                            verbose = params["verbose"] )
                                                            
             raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
             
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
index bc780e4cb..44e2232ac 100644
--- a/unit_tests/rating_curve_comparison_params.json
+++ b/unit_tests/rating_curve_comparison_params.json
@@ -3,12 +3,12 @@
 	{
 		"01010004":
 		[
-			"/data/outputs/carson_gage_test/01010004/usgs_elev_table.csv",
-			"/data/outputs/carson_gage_test/01010004/branches",
+			"/data/outputs/fim_unit_test_data_do_not_remove/01010004/usgs_elev_table.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/01010004/branches",
 			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_01010004.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_01010004.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_01010004.png",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_01010004.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_01010004.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_01010004.png",
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"01010004",
@@ -16,12 +16,12 @@
 		],
 		"02020005":
 		[
-			"/data/outputs/carson_gage_test/02020005/usgs_elev_table.csv",
-			"/data/outputs/carson_gage_test/02020005/branches",
+			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_elev_table.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches",
 			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"02020005",
@@ -29,12 +29,12 @@
 		],
 		"02030103":
 		[
-			"/data/outputs/carson_gage_test/02030103/usgs_elev_table.csv",
-			"/data/outputs/carson_gage_test/02030103/branches",
+			"/data/outputs/fim_unit_test_data_do_not_remove/02030103/usgs_elev_table.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/02030103/branches",
 			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02030103.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02030103.csv",
-			"/data/outputs/gms_example_unit_tests/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02030103.png",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02030103.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02030103.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02030103.png",
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"02030103",
diff --git a/unit_tests/rating_curve_comparison_unittests.py b/unit_tests/rating_curve_comparison_unittests.py
index c68450d40..07ded5931 100644
--- a/unit_tests/rating_curve_comparison_unittests.py
+++ b/unit_tests/rating_curve_comparison_unittests.py
@@ -5,11 +5,11 @@
 import sys
 import warnings
 
-import argparse
 import json
 import unittest
 
-import unit_tests_utils as helpers
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other directories
 sys.path.append('/foss_fim/tools/')   # *** update your folder path here if required ***
@@ -25,9 +25,8 @@ class test_rating_curve_comparison(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        params_file_path = '/foss_fim/unit_tests/rating_curve_comparison_params.json'
         warnings.simplefilter('ignore')
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
@@ -41,9 +40,6 @@ def test_generate_rating_curve_metrics_01010004_success(self):
           If there is no return value from the method, please say so.>
         '''
 
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
         params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
                 
         generate_rating_curve_metrics(params["01010004"])
@@ -58,9 +54,6 @@ def test_generate_rating_curve_metrics_02020005_success(self):
         < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
           If there is no return value from the method, please say so.>
         '''
-
-        #global params_file
-        helpers.print_unit_test_function_header()
         
         params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
                 
@@ -76,9 +69,6 @@ def test_generate_rating_curve_metrics_02030103_success(self):
         < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
           If there is no return value from the method, please say so.>
         '''
-
-        #global params_file
-        helpers.print_unit_test_function_header()
         
         params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
                 
diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json
index 16edb822c..04b33df3e 100644
--- a/unit_tests/shared_functions_params.json
+++ b/unit_tests/shared_functions_params.json
@@ -1,13 +1,15 @@
 {
 	"append_append_id_to_file_name_single_identifier_valid":
 	{
-		"file_name": "/output/myfolder/a_raster.tif",
-		"identifier": "13090001"
+		"file_name": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
+		"identifier": "8888",
+		"expected_output": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003_8888.tif"
 	},
 
 	"append_append_id_to_file_name_identifier_list_valid":
 	{
-		"file_name": "/output/myfolder/a_raster.tif",
-		"identifier": ["13090001", "05010204"]
+		"file_name": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
+		"identifier": ["7777", "8888"],
+		"expected_output": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003_7777_8888.tif"
 	}	
 }
\ No newline at end of file
diff --git a/unit_tests/shared_functions_unittests.py b/unit_tests/shared_functions_unittests.py
index fb6df76c1..446aed235 100644
--- a/unit_tests/shared_functions_unittests.py
+++ b/unit_tests/shared_functions_unittests.py
@@ -4,12 +4,12 @@
 import os
 import sys
 
-import argparse
 import json
 import warnings
 import unittest
 
-import unit_tests_utils as helpers
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other directories
 #sys.path.append('/foss_fim/src/utils') 
@@ -27,9 +27,7 @@ class test_shared_functions(unittest.TestCase):
     def setUpClass(self):
 
         warnings.simplefilter('ignore')
-
-        params_file_path = '/foss_fim/unit_tests/shared_functions_params.json'
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
@@ -39,19 +37,13 @@ def test_append_id_to_file_name_single_identifier_success(self):
         '''
         Pass in a file name with the single identifier and get the single adjusted file name back
         '''
-
-        #global params_file
-        #helpers.print_unit_test_function_header()
-        
+       
         params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
-        
-        expected_output = "/output/myfolder/a_raster_13090001.tif"
-
         actual_output = src.append_id_to_file_name(file_name = params["file_name"],
                                                    identifier = params["identifier"])
 
         err_msg = "actual output does not match expected output"
-        self.assertEqual(expected_output, actual_output, err_msg)
+        self.assertEqual(params["expected_output"], actual_output, err_msg)
 
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")
@@ -62,9 +54,6 @@ def test_append_id_to_file_name_indentifer_list_success(self):
         '''
         Pass in a file name with the list of identifiers and get a file name back with multiple identifers added
         '''
-
-        #global params_file
-        #helpers.print_unit_test_function_header()
         
         params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()        
         
@@ -75,7 +64,7 @@ def test_append_id_to_file_name_indentifer_list_success(self):
 
         #print(actual_output)
         err_msg = "actual output does not match expected output"
-        self.assertEqual(expected_output, actual_output, err_msg)
+        self.assertEqual(params["expected_output"], actual_output, err_msg)
 
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")
@@ -85,14 +74,10 @@ def test_append_id_to_file_name_no_file_name_success(self):
         '''
         Pass in an non existant file name and get None back
         '''
-
-        #global params_file
-        #helpers.print_unit_test_function_header()
         
         params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
         
-        actual_output = src.append_id_to_file_name(None,
-                                                   identifier = params["identifier"])
+        actual_output = src.append_id_to_file_name(None, identifier = params["identifier"])
 
         if (actual_output is not None):
             raise Exception("actual output should not have a value")
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
index 1813f4222..15ab0bce9 100644
--- a/unit_tests/split_flows_params.json
+++ b/unit_tests/split_flows_params.json
@@ -4,12 +4,12 @@
 		"max_length": 1500,
 		"slope_min": 0.001,
 		"lakes_buffer_input": 20,
-		"flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_3246000305.shp",
-		"dem_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/dem_thalwegCond_3246000305.tif",
-	    "split_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_3246000305.gpkg",
-		"split_points_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000305/demDerived_reaches_split_points_3246000305.gpkg",
-		"wbd8_clp_filename": "/data/outputs/gms_example_unit_tests/02020005/wbd8_clp.gpkg",
-		"lakes_filename": "/data/outputs/gms_example_unit_tests/02020005/nwm_lakes_proj_subset.gpkg",
-		"drop_stream_orders": false
+		"flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
+		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+	    "split_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
+		"split_points_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
+		"wbd8_clp_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
+		"lakes_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_lakes_proj_subset.gpkg",
+		"drop_stream_orders": true
 	}
 }
\ No newline at end of file
diff --git a/unit_tests/split_flows_unittests.py b/unit_tests/split_flows_unittests.py
index 0a32ab7ab..e517f078a 100644
--- a/unit_tests/split_flows_unittests.py
+++ b/unit_tests/split_flows_unittests.py
@@ -4,12 +4,12 @@
 import os
 import sys
 
-import argparse
 import json
 import warnings
 import unittest
 
-import unit_tests_utils as helpers
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other directories
 sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
@@ -26,9 +26,7 @@ class test_split_flows(unittest.TestCase):
     def setUpClass(self):
 
         warnings.simplefilter('ignore')
-
-        params_file_path = '/foss_fim/unit_tests/split_flows_params.json'
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
@@ -43,8 +41,6 @@ def test_split_flows_success(self):
         If the test is successful, these file will be created.
         '''
 
-        #global params_file
-        helpers.print_unit_test_function_header()
         params = self.params["valid_data"].copy() 
 
         # to setup the test, lets start by deleted the two expected output files to ensure
diff --git a/unit_tests/gms/inundate_gms_params.json b/unit_tests/tools/gms_tools/inundate_gms_params.json
similarity index 89%
rename from unit_tests/gms/inundate_gms_params.json
rename to unit_tests/tools/gms_tools/inundate_gms_params.json
index b6c2d9884..a75e663c4 100644
--- a/unit_tests/gms/inundate_gms_params.json
+++ b/unit_tests/tools/gms_tools/inundate_gms_params.json
@@ -4,8 +4,8 @@
 		"hydrofabric_dir": "/outputs/fim_unit_test_data_do_not_remove/",
 		"forecast": "data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv",
 		"num_workers": 1,
-		"hucs": "05030104",
-	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_1_gms.tif",
+		"hucs": "02020005",
+	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
 		"inundation_polygon": null,
 		"depths_raster": null,
 		"verbose": true,
diff --git a/unit_tests/gms/inundate_gms_unittests.py b/unit_tests/tools/gms_tools/inundate_gms_unittests.py
similarity index 89%
rename from unit_tests/gms/inundate_gms_unittests.py
rename to unit_tests/tools/gms_tools/inundate_gms_unittests.py
index b09034edb..54c742f93 100644
--- a/unit_tests/gms/inundate_gms_unittests.py
+++ b/unit_tests/tools/gms_tools/inundate_gms_unittests.py
@@ -4,17 +4,14 @@
 import os
 import sys
 
-import argparse
 import json
-import pandas as pd
 import warnings
 import unittest
 
 sys.path.append('/foss_fim/unit_tests/')
-import unit_tests_utils as helpers
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-# importing python folders in other directories
-sys.path.append('/foss_fim/tools/gms_tools/') 
+sys.path.append('/foss_fim/tools/gms_tools')
 import inundate_gms as src
 
 # NOTE: This goes directly to the function.
@@ -28,9 +25,7 @@ class test_inundate_gms(unittest.TestCase):
     def setUpClass(self):
 
         warnings.simplefilter('ignore')
-
-        params_file_path = '/foss_fim/unit_tests/gms/inundate_gms_params.json'
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
@@ -44,9 +39,6 @@ def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self
         within a gms output folder.
         '''
 
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
         params = self.params["valid_data_inudation_raster_single_huc"].copy()         
         
         output_fileNames_df = src.Inundate_gms(hydrofabric_dir = params["hydrofabric_dir"],
diff --git a/unit_tests/tools/inundate_params.json b/unit_tests/tools/inundate_params.json
new file mode 100644
index 000000000..6e2b1d800
--- /dev/null
+++ b/unit_tests/tools/inundate_params.json
@@ -0,0 +1,24 @@
+{
+	"valid_data_inundate_branch":
+	{
+		"rem": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
+		"catchment_poly": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
+		"hydro_table": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/hydroTable_3246000001.csv",
+	    "forecast": "/data/test_cases/usgs_test_cases/validation_data_usgs/02020005/ptvn6/action/ahps_ptvn6_huc_02020005_flows_action.csv",
+		"mask_type": null,
+		"hucs": null,
+		"hucs_layerName": null,
+		"subset_hucs": null,
+		"num_workers": 1,
+		"aggregate": false,
+		"inundation_raster": "/data/test_cases/usgs_test_cases/02020005_usgs/testing_versions/fim_unit_test_data_do_not_remove/action/ptvn6_inundation_extent_02020005_3246000001.tif",
+		"inundation_polygon": null,
+		"depths": null,
+		"out_raster_profile": null,
+		"out_vector_profile": null,
+		"src_table": null,
+		"quiet": true,
+		"expected_inundation_raster": "/data/" 
+	}
+}
\ No newline at end of file
diff --git a/unit_tests/tools/inundate_unittests.py b/unit_tests/tools/inundate_unittests.py
new file mode 100644
index 000000000..66fb1ad16
--- /dev/null
+++ b/unit_tests/tools/inundate_unittests.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+sys.path.append('/foss_fim/tools/')
+import inundation as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_inundate(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_inundate_create_inundation_raster_single_branch_success(self):
+
+        '''
+        Test for creating a inundation branch raster, no depth raster and no
+        inundation_polygons, no subsets, no mask
+        '''
+
+        params = self.params["valid_data_inundate_branch"].copy()         
+        
+        # returns list of rasters and polys
+        inundation_rasters, depth_rasters, inundation_polys = src.inundate(
+                                            rem = params["rem"],
+                                            catchments = params["catchments"],
+                                            catchment_poly = params["catchment_poly"],
+                                            hydro_table = params["hydro_table"],
+                                            forecast = params["forecast"],
+                                            mask_type = params["mask_type"],
+                                            hucs = params["hucs"],
+                                            hucs_layerName = params["hucs_layerName"],
+                                            subset_hucs = params["subset_hucs"],
+                                            num_workers = params["num_workers"],
+                                            aggregate = params["aggregate"], 
+                                            inundation_raster = params["inundation_raster"],
+                                            inundation_polygon = params["inundation_polygon"],
+                                            depths = params["depths"],
+                                            out_raster_profile = params["out_raster_profile"],
+                                            out_vector_profile = params["out_vector_profile"],
+                                            src_table = params["src_table"],
+                                            quiet = params["quiet"]                                            
+                                            )
+
+        print(inundation_rasters)
+        assert len(inundation_rasters) == 1, "Expected exactly one inundation raster path records"
+        assert depth_rasters[0] == None, "Expected no depth raster path records"
+        assert inundation_polys[0] == None, "Expected no inundation_polys records"
+        
+        assert os.path.exists(params["expected_inundation_raster"]), \
+            f"Expected file {params['expected_inundation_raster']} but it does not exist."
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+        
+    # ***********************
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/unit_tests_utils.py b/unit_tests/unit_tests_utils.py
index 5db453323..018ae815a 100644
--- a/unit_tests/unit_tests_utils.py
+++ b/unit_tests/unit_tests_utils.py
@@ -1,11 +1,20 @@
 #!/usr/bin/env python3
 
+import errno
+import os
 import sys
 
-
-def print_unit_test_function_header():
-
-    print()    
-    print("*************************************************************") 
-    print(f"Start function: {sys._getframe(1).f_code.co_name}")
-    print()
+class FIM_unit_test_helpers(object):
+    
+    @staticmethod
+    def get_params_filename(unit_test_file_name):
+       
+        unittest_file_name = os.path.basename(unit_test_file_name)
+        params_file_name = unittest_file_name.replace("_unittests.py", "_params.json")
+        params_file_path = os.path.join(os.path.dirname(unit_test_file_name), params_file_name)
+        
+        if (not os.path.exists(params_file_path)):
+            raise FileNotFoundError(f"{params_file_path} does not exist")
+        
+        return params_file_path
+    
diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json
index 89e80f614..2d91c01aa 100644
--- a/unit_tests/usgs_gage_crosswalk_params.json
+++ b/unit_tests/usgs_gage_crosswalk_params.json
@@ -1,13 +1,12 @@
 {
 	"valid_data":
 	{
-		"usgs_gages_filename": "/data/outputs/gms_example_unit_tests/02020005/usgs_subset_gages.gpkg",
-		"input_flows_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/demDerived_reaches_split_filtered_3246000257.gpkg",
-		"input_catchment_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg",
-		"dem_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/dem_meters_3246000257.tif",
-		"dem_adj_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/dem_thalwegCond_3246000257.tif",
-		"output_table_filename": "/data/outputs/gms_example_unit_tests/02020005/branches/3246000257/usgs_elev_table.csv",
-		"branch_id": "3246000257"
-
+		"usgs_gages_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_subset_gages.gpkg",
+		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
+		"input_catchment_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
+		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_meters_3246000005.tif",
+		"dem_adj_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+		"output_table_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/usgs_elev_table.csv",
+		"branch_id": "3246000005"
 	}
 }
\ No newline at end of file
diff --git a/unit_tests/usgs_gage_crosswalk_unittests.py b/unit_tests/usgs_gage_crosswalk_unittests.py
index 6fb287eca..bc66d6faf 100644
--- a/unit_tests/usgs_gage_crosswalk_unittests.py
+++ b/unit_tests/usgs_gage_crosswalk_unittests.py
@@ -4,12 +4,12 @@
 import os
 import sys
 
-import argparse
 import json
 import unittest
 import warnings
 
-import unit_tests_utils as helpers
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # importing python folders in other directories
 sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
@@ -26,8 +26,7 @@ class test_usgs_gage_crosswalk(unittest.TestCase):
     def setUpClass(self):
 
         warnings.simplefilter('ignore')
-        params_file_path = '/foss_fim/unit_tests/usgs_gage_crosswalk_params.json'
-    
+        params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
@@ -46,12 +45,7 @@ def test_GageCrosswalk_success(self):
             system, aka: Does the expected file exist. Don't worry about its contents.
 
         '''
-
-        #global params_file
-        helpers.print_unit_test_function_header()
-        
         params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-        
        
         # Delete the usgs_elev_table.csv if it exists
         if os.path.isfile(params["output_table_filename"]):
@@ -68,9 +62,6 @@ def test_GageCrosswalk_success(self):
         # Make sure that the usgs_elev_table.csv was written
         assert os.path.isfile(params["output_table_filename"]), f'{params["output_table_filename"]} does not exist'
 
-
-
-
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")
 

From b5fd3db1413b6acd211ad9d00e56d47ef6cb5968 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 1 Aug 2022 15:12:21 -0500
Subject: [PATCH 146/359] Simplify usage of fim4 via gms_pipeline.sh (#641)

---
 README.md                                   |  60 +++--
 config/params_template_calibrated.env       |  46 ----
 docs/CHANGELOG.md                           |  42 ++-
 gms_pipeline.sh                             | 247 +++++++++++++----
 gms_run_branch.sh                           |  64 +++--
 gms_run_unit.sh                             | 100 +++----
 src/bash_functions.env                      |   9 +-
 src/check_huc_inputs.py                     |  64 +++--
 src/check_unit_errors.py                    |  81 ++++++
 src/filter_catchments_and_add_attributes.py |   6 +-
 src/gms/derive_level_paths.py               |  13 +-
 src/gms/run_by_unit.sh                      |  12 +
 src/split_flows.py                          |   6 +-
 src/utils/fim_enums.py                      |   5 +-
 src/utils/shared_variables.py               |   3 +
 tools/copy_test_case_folders.py             |   2 +-
 unit_tests/README.md                        |   6 +-
 unit_tests/check_unit_errors_params.json    |  11 +
 unit_tests/check_unit_errors_unittests.py   | 278 ++++++++++++++++++++
 19 files changed, 817 insertions(+), 238 deletions(-)
 delete mode 100644 config/params_template_calibrated.env
 create mode 100644 src/check_unit_errors.py
 create mode 100644 unit_tests/check_unit_errors_params.json
 create mode 100644 unit_tests/check_unit_errors_unittests.py

diff --git a/README.md b/README.md
index bdd336e05..a23e65155 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 
 ---
 
-# FIM Version 3 
+# FIM Version 4 
 
 ## Accessing Data through ESIP S3 Bucket
 The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.
@@ -36,8 +36,10 @@ aws s3 ls s3://noaa-nws-owp-fim/ --request-payer requester
 
 Download a directory of outputs for a HUC8:
 ```
-aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/fim_3_0_21_0/outputs/fr/12090301 12090301 --request-payer requester
+aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/fim_3_0_34_1/outputs/fr/12090301 12090301 --request-payer requester
 ```
+**Note**: There may be newer editions than fim_3_0_34_1, and it is recommended to adjust the command above for the latest version.
+
 
 ## Running the Code
 ### Input Data
@@ -48,7 +50,7 @@ Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP
 
 ### Installation
 1. Install Docker : [Docker](https://docs.docker.com/get-docker/)
-2. Build Docker Image : `docker build -f Dockerfile.dev -t <image_name>:<tag> <path/to/repository>`
+2. Build Docker Image : `docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>`
 3. Create FIM group on host machine:
     - Linux: `groupadd -g 1370800178 fim`
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
@@ -60,17 +62,22 @@ Make sure to set the config folder group to 'fim' recursively using the chown co
 - `params_template.env`
 - `mannings_default.json`
     - must change filepath in `params_template.env` in `manning_n` variable name
-- `params_calibrated.env`
-    - runs calibrated mannings parameters from `mannings_calibrated.json`
 
 ### Produce HAND Hydrofabric
 ```
-fim_run.sh -u <huc8> -c /foss_fim/config/<your_params_file.env> -n <name_your_run>
+gms_pipeline.sh -u <huc8> -n <name_your_run>
 ```
-- `-u` can be a single huc, a series passed in quotes, or a line-delimited file
-    i. To run entire domain of available data use one of the ```/data/inputs/included_huc[4,6,8].lst``` files
+- There are a wide number of options and defaulted values, for details run ```gms_pipeline.sh -h```
+- Manditory arguments:
+    - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited file
+    i. To run entire domain of available data use one of the ```/data/inputs/included_huc[4,6,8].lst``` files or a huc list file of your choice.
+    - `-n` is a name of your run (only alphanumeric)
 - Outputs can be found under ```/data/outputs/<name_your_run>```
 
+Processing of HUC's in FIM4 (GMS) comes in two pieces: gms_run_unit and gms_run_branch. `gms_pipeline.sh` above takes care of both steps however, you can run each part seperately for faster development if you like.
+
+If you choose to do the two step hydrofabric creation, then run `gms_run_unit.sh`, then `gms_run_branch.sh`. See each of those files for details on arguments.
+
 ### Testing in Other HUCs
 To test in HUCs other than the provided HUCs, the following processes can be followed to acquire and preprocess additional NHDPlus rasters and vectors. After these steps are run, the "Produce HAND Hydrofabric" step can be run for the new HUCs.
 
@@ -88,11 +95,11 @@ To test in HUCs other than the provided HUCs, the following processes can be fol
 
 ----
 ### Evaluating Inundation Map Performance
-After `fim_run.sh` completes, you can evaluate the model's skill. The evaluation benchmark datasets are available through ESIP in the `test_cases` directory.
+After `gms_pipeline.sh` completes, you can evaluate the model's skill. The evaluation benchmark datasets are available through ESIP in the `test_cases` directory.
 
 To evaluate model skill, run the following:
 ```
-python /foss_fim/tools/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs]
+python /foss_fim/tools/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs (cores and/or procs)]
 ```
 
 More information can be found by running:
@@ -103,27 +110,30 @@ python /foss_fim/tools/synthesize_test_cases.py --help
 ----
 ### Managing Dependencies
 
-Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/). To add new dependencies, from the projects's top-level directory:
+Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/). 
 
-```bash
-pipenv install ipython --dev
-```
+When you execute `docker build` from the `Installation` section above, all of the dependencies you need are included. This includes dependencies for you to work in JupyterLab for testing purposes. 
 
-The `--dev` flag adds development dependencies, omit it if you want to add a production dependency. If the environment looks goods after adding dependencies, lock it with:
+While very rare, you may want to add more dependencies. You can follow the following steps:
 
-```bash
-pipenv lock
-```
+- From inside your docker container, run the following command:
+    ```bash
+    pipenv install <your package name> --dev
+    ```
+    The `--dev` flag adds development dependencies, omit it if you want to add a production dependency.
+    
+    This will automatically update the Pipfile in the root of your docker container directory. If the environment looks goods after adding dependencies, lock it with:
 
-and include both `Pipfile` and `Pipfile.lock` in your commits. The docker image installs the environment from the lock file.
-
-If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
+    ```bash
+    pipenv lock
+    ```
 
+    This will update the `Pipfile.lock`. Copy the new updated `Pipfile` and `Pipfile.lock` in the source directory and include both in your git commits. The docker image installs the environment from the lock file. 
+    
+**Make sure you test it heavily including create new docker images and that it continues to work with the code.**
 
----
-# FIM Version 4
+If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
 
-Details to come.
 
 ----
 ## Citing This Work
@@ -133,7 +143,7 @@ Please cite this work in your research and projects according to the CITATION.cf
 ----
 ### Known Issues & Getting Help
 
-Please see the issue tracker on GitHub and the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/cahaba/wiki/Cahaba-Wiki-Home) for known issues and getting help.
+Please see the issue tracker on GitHub and the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki/Known-Shortcomings-and-Opportunities-for-Improvement) for known issues and getting help.
 
 ### Getting Involved
 
diff --git a/config/params_template_calibrated.env b/config/params_template_calibrated.env
deleted file mode 100644
index b6a0694f4..000000000
--- a/config/params_template_calibrated.env
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-#### hydroconditioning parameters ####
-export negative_burn_value=1000
-export agree_DEM_buffer=70
-export wbd_buffer=5000
-export thalweg_lateral_elev_threshold=3
-
-#### geospatial parameters ####
-export max_split_distance_meters=1500
-export ms_buffer_dist=7000
-export lakes_buffer_dist_meters=20
-
-#### rating curve parameters ####
-export manning_n="/foss_fim/config/mannings_default_calibrated.json"
-export stage_min_meters=0
-export stage_interval_meters=0.3048
-export stage_max_meters=25
-export slope_min=0.001
-export min_catchment_area=0.25
-export min_stream_length=0.5
-
-#### gms parameters ####
-export branch_id_attribute=levpa_id
-export branch_buffer_distance_meters=7000
-export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.
-
-#### bathy SRC estimation parameters ####
-export bankfull_input_table="/bathymetry/BANKFULL_CONUS.txt" # Option 1: Wieczorek (2018) bankfull geometry (Bieger 2015 regression)
-#export bankfull_input_table="/bathymetry/nwm_route_link_geom_BED.csv" # Option 2: NWM Route Link bankfull geometry (Blackburn-Lynch regression)
-export bathy_src_modification=True
-export surf_area_thalweg_ratio_flag=10
-export thalweg_stg_search_max_limit=3
-export bathy_xs_area_chg_flag=1
-export bankful_xs_area_ratio_flag=10
-export thalweg_hyd_radius_flag=10
-
-#### computational parameters ####
-export ncores_gw=1 # mpi number of cores for gagewatershed
-export ncores_fd=1 # mpi number of cores for flow directions
-export default_max_jobs=1 # default number of max concurrent jobs to run
-export memfree=0G # min free memory required to start a new job or keep youngest job alive
-
-#### logging parameters ####
-export startDiv="\n##########################################################################\n"
-export stopDiv="\n##########################################################################"
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0a9314424..32be5fc10 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,44 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.5.3 - 2022-07-27 - [PR #630](https://github.com/NOAA-OWP/inundation-mapping/issues/630)
+
+A file called gms_pipeline.sh already existed but was unusable. This has been updated and now can be used as a "one-command" execution of the fim4/gms run. While you still can run gms_run_unit.sh and gms_run_branch.sh as you did before, you no longer need to. Input arguments were simplified to allow for more default and this simplification was added to `gms_run_unit.sh` and `gms_run_branch.sh` as well. 
+
+A new feature was added that is being used for `gms_pipeline.sh` which tests the percent and number of errors after hucs are processed before continuing onto branch processing.
+
+New FIM4/gms usability is now just (at a minumum): `gms_pipeline.sh -n <output name> -u <HUC(s) or HUC list path>`
+	
+`gms_run_branch.sh` and `gms_run_branch.sh` have also been changed to add the new -a flag and default to dropping stream orders 1 and 2.
+
+## Additions
+
+- `src`
+    - `check_unit_errors.py`: as described above.
+- `unit_tests`
+    - `check_unit_errors_unittests.py` and `check_unit_errors_params.json`: to match new file.    
+
+## Changes
+
+- `README.md`:  Updated text for FIM4, gms_pipeline, S3 input updates, information about updating dependencies, misc link updates and misc text verbage.
+- `gms_pipeline.sh`: as described above.
+- `gms_run_unit.sh`: as described above. Also small updates to clean up folders and files in case of an overwrite.
+- `gms_run_branch.sh`: as described above.
+- `src`
+     - `utils`
+         - `fim_enums.py`:  FIM_system_exit_codes renamed to FIM_exit_codes.
+         - `shared_variables.py`: added configurable values for minimum number and percentage of unit errors.
+    - `bash_functions.env`:   Update to make the cumulative time screen outputs in mins/secs instead of just seconds.
+    - `check_huc_inputs.py`:  Now returns the number of HUCs being processed, needed by `gms_pipeline.sh` (Note: to get the value back to a bash file, it has to send it back via a "print" line and not a "return" value.  Improved input validation, 
+- `unit_tests`
+   - `README.md`: Misc text and link updates.
+
+## Removals
+
+- `config\params_template_calibrated.env`: No longer needed. Has been removed already from dev-fim3 and confirmed that it is not needed.
+<br><br>
+
 ## v4.0.5.2 - 2022-07-25 - [PR #622](https://github.com/NOAA-OWP/inundation-mapping/pull/622)
 
 Updates to unit tests including a minor update for outputs and loading in .json parameter files.
@@ -58,8 +96,8 @@ Updates to unit tests including a minor update for outputs and loading in .json
      - `run_test_case.py`: Some styling / readability upgrades plus some enhanced outputs.  Also changed the _verbose_ flag to _gms_verbose_ being passed into Mosaic_inundation function.
      - `synthesize_test_cases.py`: arguments being passed into the _alpha_test_args_ from being hardcoded from flags to verbose (effectively turning on verbose outputs when applicable. Note: Progress bar was not affected.
      - `tools_shared_functions.py`: Some styling / readability upgrades.
-- `gms_run_unit.sh`: Added export of extent variable.
-- `gms_run_branch.sh`: Fixed bug when using overwrite flag saying branch errors folder already exists.
+- `gms_run_unit.sh`: Added export of extent variable, dropped the -s flag and added the -a flag so it now defaults to dropping stream orders 1 and 2.
+- `gms_run_branch.sh`: Fixed bug when using overwrite flag saying branch errors folder already exists, dropped the -s flag and added the -a flag so it now defaults to dropping stream orders 1 and 2.
 
 ## Removals
 
diff --git a/gms_pipeline.sh b/gms_pipeline.sh
index b79efe891..f9d9c7034 100755
--- a/gms_pipeline.sh
+++ b/gms_pipeline.sh
@@ -1,54 +1,197 @@
 #!/bin/bash -e
+:
+usage ()
+{
+    echo
+    echo 'Produce GMS hydrofabric datasets for unit and branch scale.'
+    echo 'Usage : gms_pipeline.sh [REQ: -u <hucs> - -n <run name> ]'
+    echo '                        [OPT: -h -c <config file> -j <job limit>] -o -r'
+    echo '                         -ud <unit deny list file> -bd <branch deny list file>'
+    echo '                         -a <use all stream orders> ]'
+    echo ''
+    echo 'REQUIRED:'
+    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
+    echo '                    A line delimited file is also acceptable. HUCs must present in inputs directory.'
+    echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
+    echo 
+    echo 'OPTIONS:'
+    echo '  -h/--help       : help file'
+    echo '  -c/--config     : configuration file with bash environment variables to export'
+    echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
+    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
+    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
+    echo '  -o/--overwrite  : overwrite outputs if already exist'
+    echo '  -r/--retry      : retries failed jobs'
+    echo '  -ud/--unitDenylist : A file with line delimited list of files in UNIT (HUC) directories to remove'
+    echo '                    upon completion (see config/deny_gms_unit_default.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_default.lst'
+    echo '  -bd/--branchDenylist : A file with line delimited list of files in branches directories to remove' 
+    echo '                    upon completion (see config/deny_gms_branches_min.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches_min.lst'    
+	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
+	echo '                    at the initial load of the nwm_subset_streams.'
+	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
+    echo
+    exit
+}
+
+set -e
+
+while [ "$1" != "" ]; do
+case $1
+in
+    -u|--hucList)
+        shift
+        hucList=$1
+        ;;
+    -c|--configFile )
+        shift
+        envFile=$1
+        ;;
+    -n|--runName)
+        shift
+        runName=$1
+        ;;
+    -j|--jobLimit)
+        shift
+        jobLimit=$1
+        ;;
+    -h|--help)
+        shift
+        usage
+        ;;
+    -o|--overwrite)
+        overwrite=1
+        ;;
+    -r|--retry)
+        retry="--retry-failed"
+        overwrite=1
+        ;;
+    -ud|--unitDenylist)
+        shift
+        deny_gms_unit_list=$1
+        ;;
+    -bd|--branchDenylist)
+        shift
+        deny_gms_branches_list=$1
+        ;;
+    -a|--useAllStreamOrders)
+        useAllStreamOrders=1
+        ;;
+    *) ;;
+    esac
+    shift
+done
+
+# print usage if arguments empty
+if [ "$hucList" = "" ]
+then
+    echo "ERROR: Missing -u Huclist argument"
+    usage
+fi
+if [ "$runName" = "" ]
+then
+    echo "ERROR: Missing -n run time name argument"
+    usage
+fi
+
+if [ "$envFile" = "" ]
+then
+    envFile=/foss_fim/config/params_template.env
+fi
+if [ "$deny_gms_unit_list" = "" ]
+then
+   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_default.lst
+fi
+if [ "$deny_gms_branches_list" = "" ]
+then
+   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_min.lst
+fi
+if [ -z "$overwrite" ]
+then
+    # default is false (0)
+    overwrite=0
+fi
+if [ -z "$retry" ]
+then
+    retry=""
+fi
+
+# invert useAllStreamOrders boolean (to make it historically compatiable
+# with other files like gms/run_unit.sh and gms/run_branch.sh).
+# Yet help user understand that the inclusion of the -a flag means
+# to include the stream order (and not get mixed up with older versions
+# where -s mean drop stream orders)
+# This will encourage leaving stream orders 1 and 2 out.
+if [ "$useAllStreamOrders" == "1" ]; then
+    export dropLowStreamOrders=0
+else
+    export dropLowStreamOrders=1
+fi
+
+## SOURCE ENV FILE AND FUNCTIONS ##
+source $envFile
+source $srcDir/bash_functions.env
+
+# default values
+if [ "$jobLimit" = "" ] ; then
+    jobLimit=$default_max_jobs
+fi
+
+export outputRunDataDir=$outputDataDir/$runName
+
+if [ -d $outputRunDataDir ] && [ $overwrite -eq 0 ]; then
+    echo
+    echo "ERROR: Output dir $outputRunDataDir exists. Use overwrite -o to run."
+    echo        
+    usage
+fi
+
+pipeline_start_time=`date +%s`
+
+num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
+
+echo
+echo "======================= Start of gms_pipeline.sh ========================="
+echo "Number of HUCs to process is $num_hucs"
+
+## Produce gms hydrofabric at unit level first (gms_run_unit)
+
+# We have to build this as a string as some args are optional.
+# but the huclist doesn't always pass well, so just worry about
+# the rest of the params.
+run_cmd=" -n $runName"
+run_cmd+=" -c $envFile"
+run_cmd+=" -j $jobLimit"
+
+if [ $overwrite -eq 1 ]; then run_cmd+=" -o" ; fi
+if [ "$retry" == "--retry-failed" ]; then run_cmd+=" -r" ; fi
+if [ $dropLowStreamOrders -eq 1 ]; then run_cmd+=" -s" ; fi
+#echo "$run_cmd"
+. /foss_fim/gms_run_unit.sh -u "$hucList" $run_cmd -d $deny_gms_unit_list
+
+## CHECK IF OK TO CONTINUE ON TO BRANCH STEPS
+# Count the number of files in the $outputRunDataDir/unit_errors
+# If no errors, there will be only one file, non_zero_exit_codes.log.
+# Calculate the number of error files as a percent of the number of hucs 
+# originally submitted. If the percent error is over "x" threshold stop processing
+# Note: This applys only if there are a min number of hucs. Aka.. if min threshold
+# is set to 10, then only return a sys.exit of > 1, if there is at least 10 errors
+
+# if this has too many errors, it will return a sys.exit code (like 62 as per fim_enums)
+# and we will stop the rest of the process. We have to catch stnerr as well.
+python3 $srcDir/check_unit_errors.py -f $outputRunDataDir -n $num_hucs
+
+
+## Produce level path or branch level datasets
+. /foss_fim/gms_run_branch.sh $run_cmd -d $deny_gms_branches_list
+
+
+echo "======================== End of gms_pipeline.sh =========================="
+pipeline_end_time=`date +%s`
+total_sec=$(expr $pipeline_end_time - $pipeline_start_time)
+dur_min=$((total_sec / 60))
+dur_remainder_sec=$((total_sec % 60))
+echo "Total Run Time = $dur_min min(s) and $dur_remainder_sec sec"
+echo
 
-#################################
-#### Sample commands for GMS ####
-#################################
-
-# produce FIM 3 commands for preprocessing as normal
-# as a comment, we should look into consolidating these three steps below into one
-
-## acquire nhd and nwm inputs then prepare them
-# acquire_and_preprocess_inputs.py
-
-## reproject rasters and convert to meters
-# preprocess_rasters.py
-
-## aggregate nhd vectors
-# aggregate_vectors.py
-
-
-## GMS ##
-
-## produce gms hydrofabric at unit level first 
-gms_run_unit.sh -u 11140105 -n test_output -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst
-
-## produce level path or branch level datasets
-gms_run_branch.sh -n test_output -u 11140105 -c /foss_fim/config/params_template.env -j 7 -d /foss_fim/config/deny_gms_branches_default.lst
-
-
-## Do either FIM or Test case information below ## 
-
-## FIM ##
-
-# make sure you have an outputs directory for FIMs setup first
-# using /data/temp here for demo
-
-# produce level path scale FIM
-/foss_fim/tools/gms_tools/inundate_gms.py -y /data/outputs/test_output -u 11140105 -f /data/test_cases/ble_test_cases/validation_data_ble/11140105/100yr/ble_huc_11140105_flows_100yr.csv -w 7 -i /data/temp/11140105_inundation_test.tif -o /data/temp/11140105_inundation_test.csv -v
-
-# mosaic level path FIM
-/foss_fim/tools/gms_tools/mosaic_inundation.py -i /data/temp/11140105_inundation_test.csv -a /data/outputs/test_output/11140105/wbd.gpkg -v -t inundation_rasters -m /data/temp/11140105_inundation_test.tif
-
-## Test Cases ##
-
-# singular test case
-/foss_fim/tools/run_test_case.py -r test_output/11140105 -b test_output -t 11140105_ble -e GMS -m HUC -w 7 -v -vg
-
-# batch of test cases
-# synthesize_test_cases.py
-/foss_fim/tools/synthesize_test_cases.py -e GMS -jh 2 -jb 3 -b ble -vr -v test_output -s your_string
-
-# aggregate results from synthesize test cases
-# will fill in more here later
-/foss_fim/tools/consolidate_metrics.py -v test_output <a FR outputs dir> <a MS outputs dir> -o /data/temp/metrics.csv
-# if only one of each is provided above (FR, MS, GMS) then you can pass -i to impute FR metrics in HUCs with missing MSoutputs
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 43eb89e03..5efe91a90 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -3,33 +3,33 @@
 usage ()
 {
     echo 'Produce GMS hydrofabric at levelpath/branch scale. Execute gms_run_unit.sh prior to.'
-    echo 'Usage : gms_run_branch.sh [REQ: -c <config file> -n <run name> ] [OPT: -h -j <job limit>]'
+    echo 'Usage : gms_run_branch.sh [REQ: -n <run name> ]'
+    echo '  	 				    [OPT: -h -j <job limit> -o -r -d <deny list file>'
+    echo '                                -u <hucs> -a <use all stream orders>]'    
     echo ''
     echo 'REQUIRED:'
-    echo '  -c/--config     : configuration file with bash environment variables to export'
-    echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
+    echo '  -n/--runName    : A name to tag the output directories and log files as. could be a version tag.'
     echo ''
     echo 'OPTIONS:'
     echo '  -h/--help       : help file'
+    echo '  -c/--config     : configuration file with bash environment variables to export'    
+    echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -r/--retry      : retries failed jobs'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -d/--denylist   : file with line delimited list of files in branches directories to remove '
-    echo '                    upon completion. ie) config/deny_gms_branches_default.lst'
+    echo '  -d/--denylist    : A file with line delimited list of files in branches directories to remove' 
+    echo '                    upon completion (see config/deny_gms_branches_min.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches_min.lst'    
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited).'
     echo '                    A line delimited file also acceptable. HUCs must present in inputs directory.'
-	echo '  -s/--dropStreamOrders : If this flag is included, the system will leave out stream orders 1 and 2'
-	echo '                    at the initial load of the nwm_subset_streams'
+	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
+	echo '                    at the initial load of the nwm_subset_streams.'
+	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
     echo
     exit
 }
 
-if [ "$#" -lt 4 ]
-then
-  usage
-fi
-
 while [ "$1" != "" ]; do
 case $1
 in
@@ -64,23 +64,30 @@ in
         shift
         deny_gms_branches_list=$1
         ;;
-	 -s|--dropLowStreamOrders)
-		  dropLowStreamOrders=1
-		  ;;
+    -a|--useAllStreamOrders)
+        useAllStreamOrders=1
+        ;;
     *) ;;
     esac
     shift
 done
 
 # print usage if arguments empty
-if [ "$envFile" = "" ]
+if [ "$runName" = "" ]
 then
+    echo "ERROR: Missing -n run time name argument"
     usage
 fi
-if [ "$runName" = "" ]
+
+if [ "$envFile" = "" ]
 then
-    usage
+    envFile=/foss_fim/config/params_template.env
+fi
+if [ "$deny_gms_branches_list" = "" ]
+then
+   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_min.lst
 fi
+
 if [ "$overwrite" = "" ]
 then
     overwrite=0
@@ -89,9 +96,17 @@ if [ -z "$retry" ]
 then
     retry=""
 fi
-if [ -z "$dropLowStreamOrders" ]
-then
-    dropLowStreamOrders=0
+
+# invert useAllStreamOrders boolean (to make it historically compatiable
+# with other files like gms/run_unit.sh and gms/run_branch.sh).
+# Yet help user understand that the inclusion of the -a flag means
+# to include the stream order (and not get mixed up with older versions
+# where -s mean drop stream orders)
+# This will encourage leaving stream orders 1 and 2 out.
+if [ "$useAllStreamOrders" == "1" ]; then
+    export dropLowStreamOrders=0
+else
+    export dropLowStreamOrders=1
 fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##
@@ -109,7 +124,7 @@ export deny_gms_branches_list=$deny_gms_branches_list
 logFile=$outputRunDataDir/logs/branch/summary_gms_branch.log
 export extent=GMS
 export overwrite=$overwrite
-export dropLowStreamOrders=$dropLowStreamOrders
+
 
 ## Check for run data directory ##
 if [ ! -d "$outputRunDataDir" ]; then 
@@ -147,7 +162,7 @@ elif [ $overwrite -eq 1 ]; then
 fi
 
 ## RUN GMS BY BRANCH ##
-echo "================================================================================"
+echo "=========================================================================="
 echo "Start of branch processing"
 echo "Started: `date -u`" 
 
@@ -179,7 +194,8 @@ echo
 echo "Start non-zero exit code checking"
 find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
 
-echo "================================================================================"
+echo "=========================================================================="
 echo "GMS_run_branch complete"
+Tcount
 echo "Ended: `date -u`" 
 echo
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index 31dc88357..dd3edb091 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -2,35 +2,33 @@
 :
 usage ()
 {
-    echo 'Produce GMS hydrofabric datasets for unit scale. Run after fim_run.sh but before gms_run_branch.sh'
-    echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -c <config file> -n <run name> ]'
-    echo '  	 				   [OPT: -h -j <job limit>] -o -r -d <deny list file> -s <drop stream orders 1 and 2>]'
+    echo 'Produce GMS hydrofabric datasets for unit scale.'
+    echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -n <run name> ]'
+    echo '                        [OPT: -h -j <job limit>]  -c <config file>'
+    echo '                         -o -r -d <deny list file> -a <use all stream orders>]'
     echo ''
     echo 'REQUIRED:'
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
     echo '                    A line delimited file is also acceptable. HUCs must present in inputs directory.'
-    echo '  -c/--config     : configuration file with bash environment variables to export'
     echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
     echo ''
     echo 'OPTIONS:'
     echo '  -h/--help       : help file'
+    echo '  -c/--config     : configuration file with bash environment variables to export'
+    echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'        
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
     echo '  -r/--retry      : retries failed jobs'
     echo '  -d/--denylist   : file with line delimited list of files in huc directories to remove upon completion'
     echo '                   (see config/deny_gms_unit_default.lst for a starting point)'
-	echo '  -s/--dropStreamOrder_1_2 : If this flag is included, the system will leave out stream orders 1 and 2'
-	echo '                    at the initial load of the nwm_subset_streams'
+	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
+	echo '                    at the initial load of the nwm_subset_streams.'
+	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
+    echo
     exit
 }
 
-if [ "$#" -lt 6 ]
-then
-  usage
-fi
-
-
 while [ "$1" != "" ]; do
 case $1
 in
@@ -65,8 +63,8 @@ in
         shift
         deny_gms_unit_list=$1
         ;;
-    -s|--dropLowStreamOrders)
-        dropLowStreamOrders=1
+    -a|--useAllStreamOrders)
+        useAllStreamOrders=1
         ;;
     *) ;;
     esac
@@ -76,20 +74,24 @@ done
 # print usage if arguments empty
 if [ "$hucList" = "" ]
 then
+    echo "ERROR: Missing -u Huclist argument"
     usage
 fi
-if [ "$envFile" = "" ]
+if [ "$runName" = "" ]
 then
+    echo "ERROR: Missing -n run time name argument"
     usage
 fi
-if [ "$runName" = "" ]
+
+if [ "$envFile" = "" ]
 then
-    usage
+    envFile=/foss_fim/config/params_template.env
 fi
 if [ "$deny_gms_unit_list" = "" ]
 then
-    usage
+   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_default.lst
 fi
+
 if [ -z "$overwrite" ]
 then
     overwrite=0
@@ -98,9 +100,17 @@ if [ -z "$retry" ]
 then
     retry=""
 fi
-if [ -z "$dropLowStreamOrders" ]
-then
-    dropLowStreamOrders=0
+
+# invert useAllStreamOrders boolean (to make it historically compatiable
+# with other files like gms/run_unit.sh and gms/run_branch.sh).
+# Yet help user understand that the inclusion of the -a flag means
+# to include the stream order (and not get mixed up with older versions
+# where -s mean drop stream orders)
+# This will encourage leaving stream orders 1 and 2 out.
+if [ "$useAllStreamOrders" == "1" ]; then
+    export dropLowStreamOrders=0
+else
+    export dropLowStreamOrders=1
 fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##
@@ -116,6 +126,13 @@ fi
 export outputRunDataDir=$outputDataDir/$runName
 logFile=$outputRunDataDir/logs/unit/summary_gms_unit.log
 
+if [ -d $outputRunDataDir ] && [ $overwrite -eq 0 ]; then
+    echo
+    echo "ERROR: Output dir $outputRunDataDir exists. Use overwrite -o to run."
+    echo        
+    usage
+fi
+
 ## Set misc global variables
 export overwrite=$overwrite
 export dropLowStreamOrders=$dropLowStreamOrders
@@ -131,8 +148,8 @@ export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
 export deny_gms_unit_list=$deny_gms_unit_list
 export extent=GMS
 
-## Input handling ##
-$srcDir/check_huc_inputs.py -u "$hucList"
+# we are not using the variable output at this time, but keep it anways
+num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
 
 ## Make output and data directories ##
 if [ "$retry" = "--retry-failed" ]; then
@@ -144,32 +161,22 @@ if [ ! -d $outputRunDataDir ]; then
     mkdir -p $outputRunDataDir
 fi
 
-if [ ! -d "$outputRunDataDir/logs/unit" ]; then
-    mkdir -p $outputRunDataDir/logs/unit
-elif [ $overwrite -eq 1 ]; then
-    # clean it out if we are overwritting
-    rm -rf $outputRunDataDir/logs/unit/
-    mkdir -p $outputRunDataDir/logs/unit
-fi
+# we need to clean out the all log files overwrite or not
+rm -rf $outputRunDataDir/logs/unit/
+mkdir -p $outputRunDataDir/logs/unit
 
-if [ ! -d "$outputRunDataDir/unit_errors" ]; then
-    mkdir -p $outputRunDataDir/unit_errors
-else
-    if [ $overwrite -eq 1 ]; then
-        rm -rf $outputRunDataDir/unit_errors/
-        mkdir -p $outputRunDataDir/unit_errors
-        
-        # remove branch logs as well
-        rm -rf $outputRunDataDir/branch_errors/
-        rm -rf $outputRunDataDir/logs/branch/
-    fi
-fi
+rm -rf $outputRunDataDir/unit_errors/
+mkdir -p $outputRunDataDir/unit_errors
+
+# if it exists, but don't make a new one yet, let gms_run_branch do that.        
+rm -rf $outputRunDataDir/logs/branch
+rm -rf $outputRunDataDir/branch_errors
 
 # copy over config file
 cp -a $envFile $outputRunDataDir
 
 ## RUN GMS BY BRANCH ##
-echo "================================================================================"
+echo "=========================================================================="
 echo "Start of unit processing"
 echo "Started: `date -u`" 
 
@@ -205,7 +212,8 @@ find $outputRunDataDir/logs/ -name "*_unit.log" -type f | xargs grep -E "Exit st
 echo "Start branch aggregation"
 python3 $srcDir/gms/aggregate_branch_lists.py -d $outputRunDataDir -f "gms_inputs.csv" -l $hucList
 
-echo "================================================================================"
-echo "Unit processing is complete"
-echo "Ended: `date -u`" 
+echo "=========================================================================="
+echo "gms_run_unit processing is complete"
+Tcount
+date -u
 echo
\ No newline at end of file
diff --git a/src/bash_functions.env b/src/bash_functions.env
index 54eb9fca8..17eb08274 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -12,8 +12,13 @@ Tstart () {
 
 Tcount () {
     t2=`date +%s`
-    echo "Time  = `expr $t2 \- $t1`sec"
-    echo "Cumulative_Time = `expr $t2 \- $t0`sec"
+    #echo "Time  = `expr $t2 \- $t1`sec"
+    #echo "Cumulative_Time = `expr $t2 \- $t0`sec"
+
+    local total_sec=$(expr $t2 - $t1)
+    local dur_min=$((total_sec / 60))
+    local dur_remainder_sec=$((total_sec % 60))
+    echo "Cumulative Time = $dur_min min(s) and $dur_remainder_sec sec"
 }
 
 export -f T_total_start
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index 9313b6ed8..c5839e6e3 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
+from logging import exception
 import os
 import argparse
 from glob import glob
-
+import string
 
 def __read_included_files(parent_dir_path):
 
@@ -12,47 +13,72 @@ def __read_included_files(parent_dir_path):
     accepted_hucs_set = set()
     for filename in filename_patterns:
 
-        f = open(filename,'r')
-        fList = f.readlines()
-        f.close()
-
-        fList = [fl.rstrip() for fl in fList]
-
-        accepted_hucs_set.update(fList)
+        with open(filename,'r') as huc_list_file:
+            file_lines = huc_list_file.readlines()
+            f_list = [fl.rstrip() for fl in file_lines]
+            accepted_hucs_set.update(f_list)
 
     return(accepted_hucs_set)
 
 
 def __read_input_hucs(hucs):
 
-    hucs = [h.split() for h in hucs][0]
+    huc_list = set()
     if os.path.isfile(hucs[0]):
         with open(hucs[0],'r') as hucs_file:
-            hucs = hucs_file.readlines()
-            hucs = [h.split() for h in hucs][0]
-
-    return(hucs)
-
+            file_lines = hucs_file.readlines()
+            f_list = [__clean_huc_value(fl) for fl in file_lines]
+            huc_list.update(f_list)
+    else:
+        if (len(hucs) > 0):
+            for huc in hucs:
+                huc_list.add(__clean_huc_value(huc))
+        else:
+            huc_list.add(__clean_huc_value(hucs[0]))
+
+    return(huc_list)
+
+def __clean_huc_value(huc):
+    
+    # Strips the newline character plus
+    # single or double quotes (which sometimes happens)
+    huc = huc.strip().replace("\"", "")
+    huc = huc.replace("\'", "")
+    return huc
 
 def __check_for_membership(hucs,accepted_hucs_set):
 
     for huc in hucs:
+        if (type(huc) == string) and (not huc.isnumeric()):
+            msg = f"Huc value of {huc} does not appear to be a number."
+            msg += "It could be an incorrect value but also could be that the huc list "
+            msg += "(if you used one), is not unix encoded."
+            raise KeyError(msg)
+        
         if huc not in accepted_hucs_set:
             raise KeyError("HUC {} not found in available inputs. Edit HUC inputs or acquire datasets and try again".format(huc))
 
 
 def check_hucs(hucs):
-
-    accepted_hucs = __read_included_files(os.path.join(os.environ['inputDataDir'],'huc_lists'))
-    hucs = __read_input_hucs(hucs)
-    __check_for_membership(hucs,accepted_hucs)
+    
+    huc_list_path = os.path.join(os.environ['inputDataDir'],'huc_lists')
+    accepted_hucs = __read_included_files(huc_list_path)
+    list_hucs = __read_input_hucs(hucs)
+    __check_for_membership(list_hucs, accepted_hucs)
+    
+    # we need to return the number of hucs being used.
+    # it is not easy to return a value to bash, except with standard out.
+    # so we will just to a print line back (Note: This means there can be no other
+    # print commands in this file, even for debugging, as bash will pick up the 
+    # very first "print"
+    print(len(list_hucs))
 
 
 if __name__ == '__main__':
 
     # parse arguments
     parser = argparse.ArgumentParser(description='Checks input hucs for availability within inputs')
-    parser.add_argument('-u','--hucs',help='Line-delimited file or list of HUCs to check availibility for',required=True,nargs='+')
+    parser.add_argument('-u','--hucs',help='Line-delimited file or list of HUCs to check availibility for',required=True, nargs='+')
 
     # extract to dictionary
     args = vars(parser.parse_args())
diff --git a/src/check_unit_errors.py b/src/check_unit_errors.py
new file mode 100644
index 000000000..5eb467320
--- /dev/null
+++ b/src/check_unit_errors.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import argparse
+import math
+import os
+import sys
+
+from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
+                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
+from utils.fim_enums import FIM_exit_codes
+
+"""
+    Calculates the number of units/hucs that errored during processing.
+    Based on a percentage of original number of hucs to be processed,
+    this could respond with an abort processing code. It will also
+    only throw that code if a minimum number of error exist.
+    There should always be at least one (non_zero_exit_codes.log)
+    
+    Note: The percentage number as a whole number and the min number of 
+    errors are stored in the utils/shared_variables.py (kinda like constants)
+    
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders. (ie output_run_data_dir)
+    number_of_input_hucs : int
+        Number of hucs originally submitted for processing.
+    
+    Returns
+    ----------
+    return_code of 0 (success) or 62 (from fim_enums)
+"""
+def check_unit_errors(fim_dir, number_of_input_hucs):
+
+    return_code = 0 # default success return code.
+    
+    if (not os.path.isdir(fim_dir)):
+        raise Exception(f"The fim output directory of {fim_dir} does not exist")
+
+    unit_errors_dir = os.path.join(fim_dir, "unit_errors")
+
+    if (not os.path.isdir(unit_errors_dir)):
+        raise Exception("The unit errors directory inside the fim output"\
+                          f" directory of {fim_dir} does not exist")
+
+    error_file_count = 0    
+    for path in os.listdir(unit_errors_dir):
+        if ( os.path.isfile(os.path.join(unit_errors_dir, path)) and
+            ("non_zero_exit_codes.log" not in path)):
+            error_file_count += 1
+
+    # We will only error out if it is more than the min number of error files.
+    # This is done because sometimes during dev, you are expecting a bunch of errors
+    # and sometimes, the number of errors is too small to worry about.
+   
+    if (error_file_count > UNIT_ERRORS_MIN_NUMBER_THRESHOLD):
+
+        percentage_of_errors =  error_file_count / number_of_input_hucs * 100
+        
+        if (percentage_of_errors >= UNIT_ERRORS_MIN_PERCENT_THRESHOLD):
+            
+            errMsg = "Too many unit errors exist to continue,"\
+                f" code:{FIM_exit_codes.EXCESS_UNIT_ERRORS.value}"
+            raise Exception(errMsg)
+  
+    return return_code
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Check number of unit errors to determine if continue')
+    parser.add_argument('-f', '--fim_dir',help='root output folder for the process (output + name)',
+                            required=True)
+    parser.add_argument('-n', '--number_of_input_hucs', help = 'Original number of hucs to process',
+                            type=int, required=True)
+
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    # call function
+    check_unit_errors(**args)
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index 6c9423ce3..00c11c9ce 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -7,7 +7,7 @@
 
 from utils.shared_variables import FIM_ID
 from utils.shared_functions import mem_profile
-from utils.fim_enums import FIM_system_exit_codes
+from utils.fim_enums import FIM_exit_codes
 
 @mem_profile
 def filter_catchments_and_add_attributes(input_catchments_filename,
@@ -59,7 +59,7 @@ def filter_catchments_and_add_attributes(input_catchments_filename,
             if (drop_stream_orders):
                 # this is not an exception, but a custom exit code that can be trapped
                 print("There are no flowlines in the HUC after stream order filtering.")
-                sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+                sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
             else:
                 # if we are not dropping stream orders, then something is wrong
                 raise Exception("There are no flowlines in the HUC.")
@@ -67,7 +67,7 @@ def filter_catchments_and_add_attributes(input_catchments_filename,
         if (drop_stream_orders):
             # this is not an exception, but a custom exit code that can be trapped
             print("There are no flowlines in the HUC after stream order filtering.")
-            sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+            sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
         else:
             # if we are not dropping stream orders, then something is wrong
             raise Exception("There are no flowlines in the HUC.")
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index 91cd063f4..03d8f61fe 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -5,8 +5,7 @@
 import sys
 
 from stream_branches import StreamNetwork
-from utils.shared_functions import get_fossid_from_huc8
-from utils.fim_enums import FIM_system_exit_codes
+from utils.fim_enums import FIM_exit_codes
 
 def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribute,
                        out_stream_network_dissolved=None, huc_id=None,
@@ -16,10 +15,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                        toNode_attribute='To_Node', fromNode_attribute='From_Node',
                        reach_id_attribute='HydroID', verbose=False,
                        drop_low_stream_orders=False ):
-    
-    # getting foss_id of huc8
-    #foss_id = get_fossid_from_huc8(huc8_id=huc_id,foss_id_attribute='fossid',
-                                   #hucs_layerName='WBDHU8')    
+   
     
     if verbose:
         print("Deriving level paths ...")
@@ -47,10 +43,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         # Later, we can look at creating custom sys exit codes 
         # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file or stream order filtering.")
         print("Sorry, no branches exist and processing can not continue. This could be an empty file or stream order filtering.")
-        sys.exit(FIM_system_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
-
-    print("inside Derive. Should we be here?")
-
+        sys.exit(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     inlets_attribute = 'inlet_id'
     outlets_attribute = 'outlet_id'
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 3a7457bb5..67213acab 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -5,6 +5,18 @@ T_total_start
 
 ## SET OUTPUT DIRECTORY FOR UNIT ##
 hucNumber="$1"
+
+# Even though check_input_hucs at gms_run_unit validate that all values
+# are numbers, sometimes the huc list can come in as windows incoded and not
+# unix encoded. It can get missed but tee and time can parse it wrong.
+# so, we will strip a slash of the end if it exists, the re-validat that the
+# value is a number.  (Note: doesn't seem to work all of the time for encoding
+# issues (??))
+re='^[0-9]+$'
+if ! [[ $hucNumber =~ $re ]] ; then
+   echo "Error: hucNumber is not a number" >&2; exit 1
+fi
+
 outputHucDataDir=$outputRunDataDir/$hucNumber
 outputBranchDataDir=$outputHucDataDir/branches
 current_branch_id=$branch_zero_id
diff --git a/src/split_flows.py b/src/split_flows.py
index 3c730c5b7..80d77b0ca 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -24,7 +24,7 @@
 import build_stream_traversal
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
-from utils.fim_enums import FIM_system_exit_codes
+from utils.fim_enums import FIM_exit_codes
 
 @mem_profile
 def split_flows(max_length, 
@@ -49,7 +49,7 @@ def split_flows(max_length,
         if (drop_stream_orders):
             # this is not an exception, but a custom exit code that can be trapped
             print("No relevant streams within HUC boundaries.")
-            sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+            sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
         else:
             # if we are not dropping stream orders, then something is wrong
             raise Exception("No flowlines exist.")
@@ -222,7 +222,7 @@ def split_flows(max_length,
         if (drop_stream_orders):
             # this is not an exception, but a custom exit code that can be trapped
             print("There are no flowlines after stream order filtering.")
-            sys.exit(FIM_system_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+            sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
         else:
             # if we are not dropping stream orders, then something is wrong
             raise Exception("No flowlines exist.")
diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py
index d545fdd23..13c4fc0eb 100644
--- a/src/utils/fim_enums.py
+++ b/src/utils/fim_enums.py
@@ -2,7 +2,7 @@
 
 from enum import Enum
 
-class FIM_system_exit_codes(Enum):
+class FIM_exit_codes(Enum):
     '''
     This particular enum class allows for special system exit codes to be issued to tell different
     code layers about what has happened. Currently, most of our code uses just sys.exit(0) and 
@@ -18,11 +18,12 @@ class FIM_system_exit_codes(Enum):
         - More advanced combinations of codes can be used and we will keep it simple for now.
     - Sample usage:
         import utils/fim_enums
-        print(FIM_system_exit_codes.GMS_UNIT_NO_BRANCHES.value) -> 60   [this is used in gms/derive_level_paths.py]
+        print(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value) -> 60   [this is used in gms/derive_level_paths.py]
 
     - For more information : https://docs.python.org/3.11/howto/enum.html and https://docs.python.org/3/library/enum.html
     '''
 
     GMS_UNIT_NO_BRANCHES = 60
     NO_FLOWLINES_EXIST = 61
+    EXCESS_UNIT_ERRORS = 62    
             
\ No newline at end of file
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 85130e7cf..d8219b2e4 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -30,6 +30,9 @@
 # --- Values ---- #
 elev_raster_ndv = -9999.0
 
+UNIT_ERRORS_MIN_NUMBER_THRESHOLD = 10
+UNIT_ERRORS_MIN_PERCENT_THRESHOLD = 10  # as in 10% (should be a whole number)
+
 # -- Field Names -- #
 FIM_ID = 'fimid'
 
diff --git a/tools/copy_test_case_folders.py b/tools/copy_test_case_folders.py
index fd65830fd..1c05b4428 100644
--- a/tools/copy_test_case_folders.py
+++ b/tools/copy_test_case_folders.py
@@ -87,7 +87,7 @@ def copy_folders(folder_name_list,
 #   and it has to be run on each root folder, one at a time (for now. aka.. no wildcards)
 
 # Sample Usage: 
-#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_22020420.lst -s /outputs/rob_gms_test_synth/ -t /data/outputs/rob_gms_test_synth_combined -a
+#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_22020420.lst -s /outputs/gms_test_synth/ -t /data/outputs/gms_test_synth_combined -a
 
 #  NOTE the 'a' at the end meaning go ahead create the gms_input.csv. This is normally
 # left for the last folder to be copied over.
diff --git a/unit_tests/README.md b/unit_tests/README.md
index 39dd33174..666600d1c 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -1,8 +1,8 @@
-## Cahaba: Flood Inundation Mapping for U.S. National Water Model
+## Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
 
 Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
 
-#### For more information, see the [Cahaba Wiki](https://github.com/NOAA-OWP/cahaba/wiki).
+#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
 
 ### This folder and files are for unit testing python files
 
@@ -30,7 +30,7 @@ One way is to use the incoming arg parser. Most py files, they include the code
 ## Running unit tests
 
 Start a docker container as you normally would for any development. ie) docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
-	- ie) docker run --rm -it --name Rob_GMS -v /home/rob_abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
+	- ie) docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
 
 For unit tests to work, you need to run the following (if not already in place).
 Notice a modified branch "deny_gms_branch_unittests.lst"  (special for unittests)
diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json
new file mode 100644
index 000000000..d97d43084
--- /dev/null
+++ b/unit_tests/check_unit_errors_params.json
@@ -0,0 +1,11 @@
+{
+	"valid_data":
+	{
+		"fim_dir": "/outputs/gms_example_unit_tests"
+	},
+
+	"invalid_path":
+	{
+		"fim_dir": "/outputs/gms_example_unit_tests_not_not_valid"
+	}	
+}
\ No newline at end of file
diff --git a/unit_tests/check_unit_errors_unittests.py b/unit_tests/check_unit_errors_unittests.py
new file mode 100644
index 000000000..92fc10640
--- /dev/null
+++ b/unit_tests/check_unit_errors_unittests.py
@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+
+import inspect
+import math
+import os
+import shutil
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import check_unit_errors as src
+from utils.fim_enums import FIM_exit_codes
+
+from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
+                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
+
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_check_unit_errors(unittest.TestCase):
+
+
+    # CURRENT 
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        try:        
+            params_file_path = ut_helpers.get_params_filename(__file__)
+        except FileNotFoundError as ex:
+            print(f"params file not found. ({ex}). Check pathing and file name convention.")
+            sys.exit(1)
+
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+    '''
+    To make most of these unit test work, we have to do the following:
+    - rename the current unit_errors directory (if it exists)
+    - empty the unit_errors directory
+    - create a bunch of dummy files in it
+    - perform the unit test
+    - delete the unit_errors directory
+    - rename the original unit_test folder back to unit_tests
+    '''
+
+    def test_check_unit_errors_success_below_min_errors(self):
+
+        # Expecting no errors
+        # test to ensure the number of dummy files is less than the
+        # overall min number of error files.
+
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
+        
+            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
+                                                                            num_dummy_files_reqd)
+            expected_output = 0
+            actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
+            
+            err_msg = "actual output does not match expected output"
+            self.assertEqual(expected_output, actual_output, err_msg)
+            
+            if (ue_folder_existed):
+                self.__remove_temp_unit_errors_folder(params["fim_dir"])
+            
+            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+            print("*************************************************************")
+            
+        except BaseException as ex:
+            
+            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
+            
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+            # re throw for the unit test engine            
+            raise ex
+
+
+    def test_check_unit_errors_fail_above_min_errors(self):
+
+        # test to ensure the number of dummy files is more than the
+        # overall min number of error files.
+
+        # Expecting sys.exit of 62
+        # we do expect this to fail and if it fails, it is successful
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD + 1
+        
+            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
+            src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
+            
+            raise Exception("Test Failed: expected sys.exit error but did not receive it")
+            
+        except BaseException as ex:
+           
+            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
+                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            else:
+                print("ex value is")
+                print(ex)
+                raise Exception("Expected a system exit code but did not receive it")
+          
+        finally:
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+
+    def test_check_unit_errors_success_above_percent_errors(self):
+
+        # Expecting no errors
+        # test to ensure the number of dummy files is more than the
+        # overall min number of error files.
+        
+        # we do expect this not to to fail as it is greater than 10 errors
+        # but below the percent threshhold
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
+            
+            print(f"num_dummy_files_reqd is {num_dummy_files_reqd}")
+            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
+                                                                            num_dummy_files_reqd)
+            
+            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
+            expected_output = 0
+            actual_output = src.check_unit_errors(params["fim_dir"], num_total_units)
+            
+            err_msg = "actual output does not match expected output"
+            self.assertEqual(expected_output, actual_output, err_msg)
+            
+            if (ue_folder_existed):
+                self.__remove_temp_unit_errors_folder(params["fim_dir"])
+            
+            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+            print("*************************************************************")
+            
+        except BaseException as ex:
+            
+            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
+            
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+            # re throw for the unit test engine            
+            raise ex
+
+
+    def test_check_unit_errors_fail_below_percent_errors(self):
+
+        # Expecting sys.exit of 62
+        # test to ensure the number of dummy files is more than the
+        # overall min number of error files.
+        
+        # we do expect this to fail as it is greater than 10 errors
+        # AND below the percent threshhold (more percent errors than the threshold)
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
+            
+            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
+            
+            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) - 10
+            print(f" is {num_total_units}")
+
+            src.check_unit_errors(params["fim_dir"], num_total_units)
+            raise Exception("Test Failed: expected sys.exit error but did not receive it")
+                    
+        except BaseException as ex:
+           
+            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
+                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+                print("*************************************************************")                             
+            else:
+                print("ex value is")
+                print(ex)
+                raise Exception("Expected a system exit code but did not receive it")
+          
+        finally:
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+            
+
+    def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files):
+        '''
+        Process:
+            We want to preserve the original unit_errors folder if it exists,
+            so we wil rename it.
+            Then we will make a new unit_errors folder and fill it with a bunch of 
+            dummy files.
+            A dummy file for non_zero_exit_codes.log will also be created.
+        Input:
+            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
+            number_of_files: how many dummy files to create
+        Returns:
+            True if the 'unit_errors' folder did original exist and needs to be renamed back.
+            False if the 'unit_errors' folder never existed in the first place.
+        '''
+        
+        ue_folder_preexists = False
+        
+        if (not os.path.isdir(output_folder)):
+            raise Exception(f"unit test root folder of {output_folder} does not exist")
+        
+        ue_folder = os.path.join(output_folder, "unit_errors")
+        temp_ue_folder = ue_folder + "_temp"
+        if (os.path.isdir(ue_folder)):
+            ue_folder_preexists = True
+            os.rename(ue_folder, temp_ue_folder)
+        
+        os.mkdir(ue_folder)
+        
+        for i in range(0, number_of_files):
+            file_name = "sample_" + str(i) + ".txt"
+            file_path = os.path.join(ue_folder, file_name)
+            with open(file_path, 'w') as fp:
+                pass
+        
+        return ue_folder_preexists
+        
+        
+    def __remove_temp_unit_errors_folder(self, output_folder):
+        '''
+        Process:
+            We want to preserve the original unit_errors folder if it exists,
+            so we will delete our temp unit test version of 'unit_errors', and rename
+            the original back to 'unit_errors'
+            Note.. it is possible the temp folder does not exist,
+            but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
+        Input:
+            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
+        Returns:
+            nothing
+        '''
+
+        ue_folder = os.path.join(output_folder, "unit_errors")        
+        if (os.path.isdir(ue_folder)):
+            shutil.rmtree(ue_folder)       
+        
+        temp_ue_folder = ue_folder + "_temp"        
+        if (os.path.isdir(temp_ue_folder)):
+            os.rename(temp_ue_folder, ue_folder)
+      
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    

From 2ecf528194017e5729074e2113f74f5a95cd586f Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 1 Aug 2022 16:25:26 -0400
Subject: [PATCH 147/359] Bug fix for missing `testing_versions` folder (#642)

---
 docs/CHANGELOG.md      | 11 +++++++++++
 tools/run_test_case.py |  2 ++
 2 files changed, 13 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 32be5fc10..e30d271f0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.5.4 - 2022-08-01 - [PR #642](https://github.com/NOAA-OWP/inundation-mapping/pull/642)
+
+Fixes bug that causes [Errno2] No such file or directory error when running synthesize_test_cases.py if testing_versions folder doesn't exist (for example, after downloading test_cases from ESIP S3).
+
+## Additions
+
+- `run_test_case.py`: Checks for testing_versions folder in test_cases and adds it if it doesn't exist.
+
+<br/><br/>
+
+=======
 ## v4.0.5.3 - 2022-07-27 - [PR #630](https://github.com/NOAA-OWP/inundation-mapping/issues/630)
 
 A file called gms_pipeline.sh already existed but was unusable. This has been updated and now can be used as a "one-command" execution of the fim4/gms run. While you still can run gms_run_unit.sh and gms_run_branch.sh as you did before, you no longer need to. Input arguments were simplified to allow for more default and this simplification was added to `gms_run_unit.sh` and `gms_run_branch.sh` as well. 
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 6d7efc5d6..04ff96fb8 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -97,6 +97,8 @@ def __init__(self, test_id, version, archive=True):
                                 test_id,
                                 'official_versions' if archive else 'testing_versions',
                                 version)
+        if not os.path.exists(self.dir):
+            os.makedirs(self.dir)
         # Benchmark data path
         self.benchmark_dir = os.path.join(self.validation_data, self.huc)
 

From 6cad26acc67402bd844aeaa5150dee467d389a27 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 3 Aug 2022 13:01:46 -0500
Subject: [PATCH 148/359] Removing special characters from CHANGELOG (#649)

---
 docs/CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e30d271f0..e7c3fcdb0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -12,7 +12,6 @@ Fixes bug that causes [Errno2] No such file or directory error when running synt
 
 <br/><br/>
 
-=======
 ## v4.0.5.3 - 2022-07-27 - [PR #630](https://github.com/NOAA-OWP/inundation-mapping/issues/630)
 
 A file called gms_pipeline.sh already existed but was unusable. This has been updated and now can be used as a "one-command" execution of the fim4/gms run. While you still can run gms_run_unit.sh and gms_run_branch.sh as you did before, you no longer need to. Input arguments were simplified to allow for more default and this simplification was added to `gms_run_unit.sh` and `gms_run_branch.sh` as well. 

From 3701056fdce64bfa789295a87e8d8035f7e014b8 Mon Sep 17 00:00:00 2001
From: JimJam <89595253+JamesColl-NOAA@users.noreply.github.com>
Date: Wed, 10 Aug 2022 11:46:29 -0500
Subject: [PATCH 149/359] Adding step to trim DEM derived streamlines if they
 extend past branch streamlines (#614)

---
 docs/CHANGELOG.md                            | 10 ++++
 src/gms/delineate_hydros_and_produce_HAND.sh |  4 +-
 src/split_flows.py                           | 56 +++++++++++++++++---
 3 files changed, 60 insertions(+), 10 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e7c3fcdb0..24df1578c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.6.0 - 2022-08-10 - [PR #614](https://github.com/NOAA-OWP/inundation-mapping/pull/614)
+
+Addressing #560, this fix in run_by_branch trims the DEM derived streamline if it extends past the end of the branch streamline. It does this by finding the terminal point of the branch stream, snapping to the nearest point on the DEM derived stream, and cutting off the remaining downstream portion of the DEM derived stream.
+
+## Changes
+
+- `/src/split_flows.py`: Trims the DEM derived streamline if it flows past the terminus of the branch (or level path) streamline.
+- `/src/gms/delineate_hydros_and_produce_HAND.sh`: Added branch streamlines as an input to `split_flows.py`.
+
+<br/><br/>
 
 ## v4.0.5.4 - 2022-08-01 - [PR #642](https://github.com/NOAA-OWP/inundation-mapping/pull/642)
 
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
index 353903d4a..96e86b1a3 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -55,7 +55,7 @@ Tstart
 mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
 Tcount
 
-# STREAMNET FOR REACHES ##
+## STREAMNET FOR REACHES ##
 echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
@@ -66,7 +66,7 @@ Tcount
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
-$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -ds $dropLowStreamOrders
+$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg -ds $dropLowStreamOrders
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
diff --git a/src/split_flows.py b/src/split_flows.py
index 80d77b0ca..7d4555d85 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -9,19 +9,23 @@
     5) create points layer with segment verticies encoded with HydroID's (used for catchment delineation in next step)
 '''
 
-import sys
+import argparse
+import build_stream_traversal
 import geopandas as gpd
+import numpy as np
+import os
 import pandas as pd
-from shapely.geometry import Point, LineString, MultiPoint
 import rasterio
-import numpy as np
-import argparse
-from tqdm import tqdm
+import sys
 import time
-from os.path import isfile
-from os import remove,environ
+
 from collections import OrderedDict
-import build_stream_traversal
+from os import remove,environ,path
+from os.path import isfile,split,dirname
+from shapely import ops, wkt
+from shapely.geometry import Point, LineString, MultiPoint
+from shapely.ops import split as shapely_ops_split
+from tqdm import tqdm
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
 from utils.fim_enums import FIM_exit_codes
@@ -36,6 +40,7 @@ def split_flows(max_length,
                 split_points_filename, 
                 wbd8_clp_filename, 
                 lakes_filename,
+                nwm_streams_filename,
                 drop_stream_orders=False):
 
     wbd = gpd.read_file(wbd8_clp_filename)
@@ -73,6 +78,40 @@ def split_flows(max_length,
     slopes = []
     hydro_id = 'HydroID'
 
+    # If loop addressing: https://github.com/NOAA-OWP/inundation-mapping/issues/560
+    # if we are processing branch 0, skip this step
+    if (os.path.split(os.path.dirname(nwm_streams_filename))[1] != '0'):
+        print ('trimming DEM stream to NWM branch terminus')
+        # read in nwm lines, explode to ensure linestrings are the only geometry
+        levelpath_lines = gpd.read_file(nwm_streams_filename).explode()
+
+        # Dissolve the linestring (how much faith should I hold that these are digitized with flow?)
+        linestring_geo = levelpath_lines.iloc[0]['geometry']
+        if (len(levelpath_lines) > 1):
+            linestring_geo = ops.linemerge(levelpath_lines.dissolve(by='levpa_id').iloc[0]['geometry'])
+
+        # Identify the end vertex (most downstream, should be last), transform into geodataframe
+        terminal_nwm_point = []
+        first, last = linestring_geo.boundary
+        terminal_nwm_point.append({'ID':'teminal','geometry':last})
+        snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(levelpath_lines.crs)
+
+        # Snap to DEM flows
+        snapped_point['geometry'] = snapped_point.apply(lambda row: flows.interpolate(flows.project( row.geometry)), axis=1)
+
+        # Trim flows to snapped point
+        # buffer here because python precision issues, print(demDerived_reaches.distance(snapped_point) < 1e-8)
+        trimmed_line = shapely_ops_split(flows.iloc[0]['geometry'], snapped_point.iloc[0]['geometry'].buffer(1))
+        # Edge cases: line string not split?, nothing is returned, split does not preserve linestring order?
+        # Note to dear reader: last here is really the most upstream segmennt (see crevats above).  When we split we should get 3 segments, the most downstream one
+        # the tiny 1 meter segment that falls within the snapped point buffer, and the most upstream one.  We want that last one which is why we trimmed_line[len(trimmed_line)-1]
+        last_line_segment = pd.DataFrame({'id':['first'],'geometry':[trimmed_line[len(trimmed_line)-1].wkt]})
+        last_line_segment['geometry'] = last_line_segment['geometry'].apply(wkt.loads) # can be last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment) when we update geopandas verisons
+        last_line_segment_geodataframe = gpd.GeoDataFrame(last_line_segment).set_crs(flows.crs)
+
+        # replace geometry in merged flowine
+        flows['geometry'] = last_line_segment_geodataframe.iloc[0]['geometry']
+
     # split at HUC8 boundaries
     print ('splitting stream segments at HUC8 boundaries')
     flows = gpd.overlay(flows, wbd8, how='union').explode().reset_index(drop=True)
@@ -246,6 +285,7 @@ def split_flows(max_length,
     parser.add_argument('-p', '--split-points-filename', help='split-points-filename',required=True)
     parser.add_argument('-w', '--wbd8-clp-filename', help='wbd8-clp-filename',required=True)
     parser.add_argument('-l', '--lakes-filename', help='lakes-filename',required=True)
+    parser.add_argument('-n', '--nwm-streams-filename', help='nwm-streams-filename',required=True)
     parser.add_argument('-ds', '--drop-stream-orders', help='Drop stream orders 1 and 2', type=int, required=False, default=False)
 
     # Extract to dictionary and assign to variables.

From 0d11759c176f0fb8c3f0903f079cac0687d32f4a Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Tue, 16 Aug 2022 11:40:45 -0400
Subject: [PATCH 150/359] Prune branches that fail with Exit status: 61 in
 split_flows.py (#655)

---
 docs/CHANGELOG.md                | 14 +++++
 gms_run_branch.sh                |  7 +++
 src/gms/remove_error_branches.py | 93 ++++++++++++++++++++++++++++++++
 tools/gms_tools/inundate_gms.py  |  5 +-
 4 files changed, 115 insertions(+), 4 deletions(-)
 create mode 100644 src/gms/remove_error_branches.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 24df1578c..6299f48d2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.6.1 - 2022-08-12 - [PR #655](https://github.com/NOAA-OWP/inundation-mapping/pull/655)
+
+Prunes branches that fail with NO_FLOWLINES_EXIST (Exit code: 61) in `gms_run_branch.sh` after running `split_flows.py`
+
+## Additions
+- Adds `remove_error_branches.py` (called from `gms_run_branch.sh`)
+- Adds `gms_inputs_removed.csv` to log branches that have been removed across all HUCs
+
+## Removals
+- Deletes branch folders that fail
+- Deletes branch from `gms_inputs.csv`
+
+<br/><br/>
+
 ## v4.0.6.0 - 2022-08-10 - [PR #614](https://github.com/NOAA-OWP/inundation-mapping/pull/614)
 
 Addressing #560, this fix in run_by_branch trims the DEM derived streamline if it extends past the end of the branch streamline. It does this by finding the terminal point of the branch stream, snapping to the nearest point on the DEM derived stream, and cutting off the remaining downstream portion of the DEM derived stream.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 5efe91a90..057e3affc 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -194,6 +194,13 @@ echo
 echo "Start non-zero exit code checking"
 find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
 
+# -------------------
+## REMOVE FAILED BRANCHES ##
+# Needed in case aggregation fails, we will need the logs
+echo
+echo "Removing branches that failed with Exit status: 61"
+python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g $outputRunDataDir/gms_inputs.csv
+
 echo "=========================================================================="
 echo "GMS_run_branch complete"
 Tcount
diff --git a/src/gms/remove_error_branches.py b/src/gms/remove_error_branches.py
new file mode 100644
index 000000000..1299930b4
--- /dev/null
+++ b/src/gms/remove_error_branches.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+# Removes branches with Exit status: 61 following split_flows.py.
+# Removes both the branch folder and the reference in gms_inputs.csv
+
+import os
+import argparse
+import pandas as pd
+import shutil
+
+def remove_error_branches(logfile, gms_inputs):
+    if os.path.isfile(logfile):
+        try:
+            errors_df = pd.read_csv(logfile, sep=':', header=None)
+        except pd.errors.EmptyDataError:
+            print('\nLog file is empty. Skipping this HUC.\n')
+            return
+
+        gms_inputs_df = pd.read_csv(gms_inputs, header=None)
+
+        # Make copy of gms_inputs.csv
+        gms_inputs_copy = os.path.splitext(gms_inputs)[0] + '_original.csv'
+        if not os.path.isfile(gms_inputs_copy):
+            gms_inputs_df.to_csv(gms_inputs_copy, header=False, index=False)
+            
+        gms_inputs_removed = os.path.splitext(gms_inputs)[0] + '_removed.csv'
+        if not os.path.isfile(gms_inputs_removed):
+            error_branches = None
+        else:
+            error_branches = pd.read_csv(gms_inputs_removed, header=None, dtype=str)
+
+        first_occurrence = []
+        for i, row in errors_df.iterrows():
+            error_code = row[2]
+
+            if error_code == 61:
+                dirname, basename = os.path.split(row[0])
+
+                filename = os.path.splitext(basename)[0]
+
+                print(f"Removing {filename}")
+
+                split = str.split(filename, '_')
+
+                huc = split[0]
+                branch = split[3]
+                huc_level = len(huc)
+
+                if huc not in first_occurrence:
+                    # Ignore previous removals for this HUC
+                    if error_branches is not None:
+                        error_branches = error_branches[error_branches[0] != huc]
+
+                    first_occurrence.append(huc)
+
+                output_dir = os.path.split(os.path.split(dirname)[0])[0]
+                branch_dir = os.path.join(output_dir, huc, 'branches', branch)
+                if os.path.exists(branch_dir):
+                    shutil.rmtree(branch_dir)
+
+                # Remove bad branch from DataFrame
+                if int(branch) in gms_inputs_df.loc[:,1].values:
+                    gms_inputs_df = gms_inputs_df.drop(index=gms_inputs_df[gms_inputs_df.loc[:,1]==int(branch)].index[0])
+
+                tmp_df = pd.DataFrame([huc.zfill(huc_level), branch]).T
+                if error_branches is None:
+                    error_branches = tmp_df
+                else:
+                    error_branches = pd.concat([error_branches, tmp_df])
+
+        # Save list of removed branches
+        pd.DataFrame(error_branches).to_csv(gms_inputs_removed, header=False, index=False)
+
+        # Overwrite gms_inputs.csv with error branches removed
+        gms_inputs_df.to_csv(gms_inputs, header=False, index=False)
+
+        print('\nDone removing error branches\n')
+
+    else:
+        print('\nNo log file found\n')
+
+
+if __name__ == '__main__':
+
+    # parse arguments
+    parser = argparse.ArgumentParser(description='Remove branches with Exit status: 61')
+    parser.add_argument('-f','--logfile', help='Location of non_zero_exit_codes.log', required=True)
+    parser.add_argument('-g','--gms-inputs', help='Location of gms_inputs.csv', required=True)
+
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    remove_error_branches(args['logfile'], args['gms_inputs'])
\ No newline at end of file
diff --git a/tools/gms_tools/inundate_gms.py b/tools/gms_tools/inundate_gms.py
index 7ecd5ce4d..8259fad51 100755
--- a/tools/gms_tools/inundate_gms.py
+++ b/tools/gms_tools/inundate_gms.py
@@ -2,11 +2,8 @@
 
 import os
 import argparse
-import logging
-import sys
-import traceback
+# import logging
 
-import numpy as np
 import pandas as pd
 
 from tqdm import tqdm

From 194ca51f2f142d2d4f22f95bc7a0e17fd6a4a433 Mon Sep 17 00:00:00 2001
From: EricMyskowski-NOAA
 <108021418+EricMyskowski-NOAA@users.noreply.github.com>
Date: Tue, 16 Aug 2022 10:51:11 -0500
Subject: [PATCH 151/359] Adds USFIMR Data to Benchmark Capability (#639)

---
 docs/CHANGELOG.md          |  11 +++
 tools/fimr_to_benchmark.py | 160 +++++++++++++++++++++++++++++++++++++
 2 files changed, 171 insertions(+)
 create mode 100644 tools/fimr_to_benchmark.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 6299f48d2..adab50e16 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.6.2 - 2022-08-16 - [PR #639](https://github.com/NOAA-OWP/inundation-mapping/pull/639)
+
+This file converts USFIMR remote sensed inundation shapefiles into a raster that can be used to compare to the FIM data. It has to be run separately for each shapefile. This addresses [#629].
+
+## Additions
+
+- `/tools/fimr_to_benchmark.py`: This file converts USFIMR remote sensed inundation shapefiles into a raster that can be used to compare to the FIM data. It has to be run separately for each shapefile.
+
+<br/><br/>
+
 ## v4.0.6.1 - 2022-08-12 - [PR #655](https://github.com/NOAA-OWP/inundation-mapping/pull/655)
 
 Prunes branches that fail with NO_FLOWLINES_EXIST (Exit code: 61) in `gms_run_branch.sh` after running `split_flows.py`
diff --git a/tools/fimr_to_benchmark.py b/tools/fimr_to_benchmark.py
new file mode 100644
index 000000000..0d4184815
--- /dev/null
+++ b/tools/fimr_to_benchmark.py
@@ -0,0 +1,160 @@
+
+import geopandas as gpd
+import pandas as pd
+import os
+from tools_shared_functions import get_metadata
+from dotenv import load_dotenv
+load_dotenv()
+import argparse
+import rasterio
+#from rasterio import features
+import affine
+import rasterio.mask
+
+def fimr_to_benchmark(fimr_path, output_path):
+    '''
+    This function converts USFIMR shapefiles from this website (https://waterserv.ua.edu/datasets/usfimr/) to rasters that c an 
+    be used for FIM Alpha evaluations and calibration. Note only USFIMR data with flow values (q) can be used here. 
+
+    Inputs: 1. FIMR Path - Path where th FIMR Shapefile was downloaded from the aformentioned website.
+            2. Output Path_flow file - Path the flow file will be sent to. 
+            3. Output path raster - path the raster will be sent too. There will be a file for each huc with the huc number in it.
+
+    Outputs: 1. flow_file.csv - This is a flow file for the river. The flow values are from the most upstream usgs gage.
+             2. rasterized{huc number}.tif - This is the fimr data that has been converted to a raster. It is on the huc8 scale so it provides
+                each huc as a separate file. The inundated areas are assigned 1 and the dry areas are assigned 0.
+
+    Other Files Used: 1. wbd - The huc 8 boundaries throughout the country
+                      2. flowlines - the flowlines along the all the rivers in the country
+    '''
+    # input validation
+    if (not os.path.exists(fimr_path)):
+        raise Exception(f"Sorry. {fimr_path} does not exist")
+
+    if (not os.path.exists(output_path)):
+        raise Exception(f"Sorry. {output_path} does not exist")
+
+    API_BASE_URL = os.getenv("API_BASE_URL")
+
+    # Path for the wbd data and nwm flowlines
+    wbd_path = '/data/inputs/wbd/WBD_National.gpkg'
+    flowlines_path = '/data/inputs/nwm_hydrofabric/nwm_flows_ms_wrds.gpkg'
+
+    # Saving wbd, flowlines, and fimr data as variables
+    wbd = gpd.read_file(wbd_path, layer = 'WBDHU8')
+    flowlines = gpd.read_file(flowlines_path)
+    fimr = gpd.read_file(fimr_path)
+
+    # Joining the flowlines and fimr to output only the flowlines in the inundated area
+    joined_flowlines_fimr = gpd.sjoin(flowlines, fimr)
+    
+    # Renaming index_right to prevent it from causing an error in the next spatial join
+    joined_flowlines_fimr = joined_flowlines_fimr.rename(columns = {'index_right':'right_index'})
+    # Getting a list of all the unique feature Ids from the joined flowlines
+    unique_feature_ids = joined_flowlines_fimr.drop_duplicates('ID')
+    # Changing cfs to cms
+    unique_feature_ids['Q_at_Image_cms'] = unique_feature_ids['Q_at_Image']*0.028317
+    # Make a discharge dataframe with only the feture ID and the Q values
+    feature_discharge = unique_feature_ids[['ID','Q_at_Image_cms']]
+
+    #flow_file_path = os.path.join(output_path, 'flow_file.csv')
+    #feature_discharge.to_csv(flow_file_path, header = ['feature_id', 'discharge'],index = False)
+    #print('flow file created at ',flow_file_path)
+
+    # Takes the huc 8s that overlap the flowlines for the river 
+    huc_joined = gpd.sjoin(wbd, joined_flowlines_fimr)
+    huc_list = huc_joined.HUC8.unique()
+    # Overlay the hucs
+    fimr_hucs = wbd.loc[wbd.HUC8.isin(huc_list)]
+    fimr_huc_union = gpd.overlay(joined_flowlines_fimr, fimr_hucs, how = 'union')
+    fimr_huc_union = fimr_huc_union.rename(columns = {'Shape_Length':'Shapelength'})
+    # Renames Shape_Length as Shapelength to eliminate a duplicate column and let it be exported
+    huc_joined = huc_joined.rename(columns = {'Shape_Length':'Shapelength'})
+
+    # Takes the usgs location ids 
+    gages = fimr['USGS_Gage']
+    split_gages = gages.iloc[0].split( ", ")
+    # Takes the first usgs gauge idea that has the streamflow data for the whole fimr area
+    first_gage = (split_gages[0])
+ 
+    # Get metadata for all usgs_site_codes that are active in the U.S.
+    metadata_url = f'{API_BASE_URL}/metadata' 
+    # Define arguments to retrieve metadata and then get metadata from WRDS
+    select_by = 'usgs_site_code'
+    selector = [first_gage] #change to usgs location id from fimr dataframe
+    must_include = 'usgs_data.active'
+    metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = None )
+   
+    # Takes the stream order at the gauge
+    stream_order = metadata_df['nwm_feature_data_stream_order']
+    stream_order_float = stream_order[0]
+
+    # Gets flow lines where the streasm order is equal to the magnitude of the mainstem
+    correct_flow = joined_flowlines_fimr.loc[joined_flowlines_fimr['order_'] == stream_order_float]
+    
+    # Exploding the fimr inundaton polygon
+    exploded_fimr = fimr.explode()
+    # Taking only the fimr polygons that overlap the flowlines to minimize noise
+    clean_fimr = gpd.sjoin(exploded_fimr, correct_flow)
+    # Renaming columns to prevent duplicate column names in overlay
+    clean_fimr = clean_fimr.rename(columns = {'index_right':'index_from_right'})
+
+    # Overlaying the cleaned fimr data and the hucs it is in
+    fimr_huc_union = gpd.overlay(clean_fimr, fimr_hucs, how = 'union')
+    fimr_huc_union = fimr_huc_union.rename(columns = {'Shape_Length':'Shapelength'})
+    # Renames Shape_Length as Shapelength to eliminate a duplicate column and let it be exported
+    huc_joined = huc_joined.rename(columns = {'Shape_Length':'Shapelength'})
+    
+    # Looping through the three huc8s, getting the fimr data in each 
+    for huc in huc_list:
+      
+        fimr_huc = fimr_huc_union.loc[fimr_huc_union.HUC8 == huc]
+        shapes = []
+
+        for index,row in fimr_huc.iterrows():
+
+            if pd.isna(row.right_index):
+                value = 0
+            else:
+                value = 1
+            
+            shapes.append((row.geometry, value))
+
+        #Outputting Flow File
+    
+        flow_file_path = os.path.join(output_path, 'fimr_huc_' + huc + '_flows.csv')
+        feature_discharge.to_csv(flow_file_path, header = ['feature_id', 'discharge'],index = False)
+        print('flow file created at',flow_file_path)
+        # output path for the rasters, float causes it to keep all the digits 
+        out_path_raster = os.path.join(output_path,'fimr_huc_' + huc + '_extent.tif')
+
+        #Rasterizing
+
+        minx, miny, maxx, maxy = fimr_huc.geometry.total_bounds
+        transform = affine.Affine(10, 0, minx, 0, -10, maxy)
+
+        # Makeing the empty raster
+        raster_shape = (int((maxy - miny)//10), int((maxx - minx)//10))
+        raster = rasterio.features.rasterize(shapes,
+                                     out_shape = raster_shape,
+                                     transform = transform,
+                                     fill = -999)
+    
+        with rasterio.open(out_path_raster, 'w', driver = 'GTiff', height = raster_shape[0], width = raster_shape[1], count = 1, dtype = raster.dtype, nodata = -999,
+            crs = '+proj=aea +lat_0=23 +lon_0=-96 +lat_1=29.5 +lat_2=45.5 +x_0=0 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs',transform = transform) as dst_dataset:
+            # writing array into the empty raster
+            dst_dataset.write_band(1, raster)
+            print('Rasterized fimr file created at ', out_path_raster)
+
+     # Parse Arguments   
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description = 'Get FIMR data for the different rivers from the different files')
+    
+    parser.add_argument('-f','--fimr_path', type = str, help = 'the path for the shapefile inputed from the FIMR dastabase', required = True) 
+    parser.add_argument('-o','--output_path', type = str, help = 'output path for the flow file and FIMR rasters.', required = True)
+    args = vars(parser.parse_args())
+
+    fimr_to_benchmark(**args)
+
+

From dacbed0212083713434b3661d8d0dbfb1407b4f2 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 17 Aug 2022 20:33:20 -0500
Subject: [PATCH 152/359] Update docker python packages for calibration db
 (#652)

---
 Dockerfile        |   2 +-
 Pipfile           |   2 +
 Pipfile.lock      | 716 ++++++++++++++++++++++++++--------------------
 docs/CHANGELOG.md |   6 +
 4 files changed, 416 insertions(+), 310 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 87ee08259..da10b8e0c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -87,7 +87,7 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 COPY Pipfile .
 COPY Pipfile.lock .
 RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
-
+RUN pip3 install shapely
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
 ADD ./src/entrypoint.sh /
diff --git a/Pipfile b/Pipfile
index 2376754e3..313f02dcc 100755
--- a/Pipfile
+++ b/Pipfile
@@ -10,6 +10,7 @@ ipython = "*"
 fiona = "==1.8.17"
 geopandas = "==0.8.1"
 numba = "==0.50.1"
+numpy = "==1.22.4"
 pandas = "==1.0.5"
 pygeos = "==0.7.1"
 rasterio = "==1.1.5"
@@ -27,6 +28,7 @@ netcdf4 = "==1.5.7"
 tables = "==3.6.1"
 h5py = "==3.4.0"
 pyproj = "==3.1.0"
+psycopg2-binary = "==2.8.6"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 4085a268b..9f5e923e4 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "3d3887794dbd15cc911a4223776cf9002eed9d36a62876bf3b96022626c530bf"
+            "sha256": "076bbd38f0fa0777776db2a9f842b60cbc6a8d5f5bf8366cc79e6abf86efde4b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -18,61 +18,59 @@
     "default": {
         "affine": {
             "hashes": [
-                "sha256:2e045def1aa29e613c42e801a7e10e0b9bacfed1a7c6af0cadf8843530a15102",
-                "sha256:34b05b070d954c382e56f02c207a372d8a32621a87653cc30cdd31cd7f65799f"
+                "sha256:d676de66157ad6af99ffd94e0f54e89dfc35b0fb7252ead2ed0ad2dca431bdd0",
+                "sha256:de17839ff05e965580870c3b15e14cefd7992fa05dba9202a0879bbed0c171e4"
             ],
-            "version": "==2.3.0"
+            "version": "==2.3.1"
         },
         "attrs": {
             "hashes": [
-                "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",
-                "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"
+                "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6",
+                "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==21.4.0"
+            "markers": "python_version >= '3.5'",
+            "version": "==22.1.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
-                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
+                "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
+                "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"
             ],
-            "version": "==2021.10.8"
+            "markers": "python_version >= '3.6'",
+            "version": "==2022.6.15"
         },
         "cftime": {
             "hashes": [
-                "sha256:003abc800933a39d6f4ecec113f401aef9961642a4769aa1099ed34e0c1c3d46",
-                "sha256:09970124c0a8c84a66e34e8655c42f0254e1c29d73a766e71c9a871ce57f125b",
-                "sha256:1187fa1107974fe374f90936e39d2e09ac7cb4ffcd3b31314898684a2cebba87",
-                "sha256:13103e6650beea6552316bd5825d6aa3b7e98f5b8115026df4826798dff9f741",
-                "sha256:13e92118fa3606b2a1d95bbe1a18a0a260ba85704d9ff061e9599b0e745bb4a9",
-                "sha256:1888f0c0ab37d6ee23d07e74e51dcaae136da48784ceaa5d078115aa2e2b8b92",
-                "sha256:1c56556900b9ae3a5b4c9705699d78f865994e5c3c92cd230d54c89d669b88f0",
-                "sha256:1f43f1c839f5579bf045f13b0d2deb15048d314d209039664003ae1bf2544898",
-                "sha256:2da21d8626a7f549eaaae7d314110133914fa24726e4e7caab2775627cb7361a",
-                "sha256:4c608937082980e0905657ec2475b3ff2a2fc89255e185f44076da482102f617",
-                "sha256:4d39b666284c78726eb7ea8b4ad29aacd387e912177721b3eb9bdc508e2b86fa",
-                "sha256:517c50e29b3c141919d83bf3782892e4a4144034b63722e7a266db25488b3bb5",
-                "sha256:519d1c391ed7be3bb4d84990a51a76cbb1cc894bd776637c0227dfe1f6274f39",
-                "sha256:64bcb6ca9ef922b2332e9ed91cd6f6f2fe28de392fb10e752f2ac83ffdc50854",
-                "sha256:7900daf0efa495f263c3c5b0c96502cc909fb07ef56844791e3c8cdbb08a741c",
-                "sha256:80390ddb68a7e27b52bf83efc040c8e3ec2e3b0bb0e66084a75498fb39f63a63",
-                "sha256:9b31455449d2e347d033a6440f94305ce7eb48d32d863dfe0f53671c78e2561d",
-                "sha256:a49b023ca5cab9c43d33b3a43edc5a112f5d452f75fc1a85f8ca136f4fa9d440",
-                "sha256:a6c1e59299250299f23ab526d1ad640dcc3b9eb311468a8f9112f9d53d6106c6",
-                "sha256:a8a56edad64912b11d8f80245b5ce7e61057eee22b6af0718e9d086b381a17d9",
-                "sha256:b420e3811da55db538008229b3ca34055734d6ade12316889d1979d770d7dfaf",
-                "sha256:bb0d680149f128d181ef1c6561eaebbf99b572329dee090c83b6735d7284a703",
-                "sha256:bfb03c766250d86d6abf546d4c1fbe0db7401eaedeab5bbbd0e599fde7c7a7d3",
-                "sha256:c0b3aff3c955d18342c2097ad1102c78d0999dba5de178d0ce68a317caf6f87e",
-                "sha256:cab3fee686c58e2ce2f61e1bf138711eefde6c0235b0cfc736c8ea96381d8346",
-                "sha256:e020db7391b2112e4c1ec9db75ff82031f979666264903697ab5ad0bfb20d490",
-                "sha256:e195ed19638f324c40bb0094d13385d58e798607670b91cc8f42717a0a832263",
-                "sha256:eca80007816b9dd1cc154a393b8da2ada22baad1ac7b2b2a9cc01451796e3e93",
-                "sha256:ee740b8dd77cb1773ae1560b890cee3aa5b82a4a7ca056223f08f4b37ebb6466",
-                "sha256:f63c14e98022c3a67ff7c5c12edf17912a7353ddd95bbfde6db1e4bc05ea5bdc",
-                "sha256:f7007e7cd837cb37716149d7c735b90b947b3370b7a5652d58675ea940541542"
-            ],
-            "version": "==1.6.0"
+                "sha256:00aef598f14faa77e48518728bd0cead585e45da89be5f7eae7be8f41c10b401",
+                "sha256:01d40598c8e551202911f5126cdff52b816fdc8316f8a88f798b81fc1a8b55bf",
+                "sha256:041c7f9f620ac5465fbaf4eb6570e4f286535e55e56ac4a34cfcfb3e853d4093",
+                "sha256:0ee1d1f9ff4e9f54700dd02d8d707a383bd9dad5c5de9c566372caba071982d5",
+                "sha256:39bc35a0c05db5f24aeef384af2119a9f0820791617e198d06bd9dba50218fa9",
+                "sha256:3c8389df99216bf184141f11b84ba00b414177c92618bc4ee1297376163a20d5",
+                "sha256:3d6af9e713b911baed904fbc4ebeec7ca44f5970971f1c5e86a34cba82fe98e9",
+                "sha256:511215f45ed7cc79ead84020e88e1fc476b8aba71b47d2fcdef8e65242406927",
+                "sha256:54c2533f6069cda87bd641fb6df28542238679896a5231006d0de00afbdf3edf",
+                "sha256:646991eb76a9906a023ec982714a04d8b4e06346ea5836ea8c30350dccca079c",
+                "sha256:67a2a9a6a7bb799b6d171239ea809e70b572c37eeda6053b04401088e6e83bed",
+                "sha256:6842d4d37377e1aca392cee4feb0e9f05d36264db08ca2cccd2c3d168487d981",
+                "sha256:688b2c54e5cb17e5cb55f799df093c5d3d4974feee8e1e7224c9ac40de2580e2",
+                "sha256:7c0724249912efa3708c265181e880ae80d90c68849aede5ce26ae7649c9e87d",
+                "sha256:8b1d2becda38027398894836c979467d6649e090a050db8c353461d9537273bb",
+                "sha256:8dfe2cf7b75a6541a54e0f25665bed5240f4c0cf5d233f81d53b4b5c55dd1c6d",
+                "sha256:93e32bee6cf67488661a7a2a73ede4c089902ef40f6d20312474351c52aa10c7",
+                "sha256:b0c192eca13faf932bbc30ad6b19878c816fe228fb84803e30873f2a5e7bd500",
+                "sha256:b46f9139aa23d6ebff01dff6970dc5476293287bc6ba473ce64542ff20607bfe",
+                "sha256:c130f78e5d7eeb8c625192848075a8a8bb92595d07ad5b4cb95b007011d78e5a",
+                "sha256:cdf27919b88a089c72266992ce31de80ed52b334a892e1ce0853ddf82d2a636b",
+                "sha256:db836dd53a56fc177d4e1e792fb94306592b8e21dc0c2d671c7a10058081e341",
+                "sha256:e57381585d421b2fdbb9369d7cc53f8442152d3c7b415bda210cb575366d7480",
+                "sha256:f3eb51592e03c9b5d4328eae7a40c7c063725b071e037ddf813a5d68f6a5cb6e",
+                "sha256:f5eb773cd21e2a8a492834cf0afd586a41b757807f53b845257a0a7419c17538",
+                "sha256:fb8c2e40b74708e20e3ab13e6273f33143037b7d16349c604e1091d2e8175275",
+                "sha256:fdb23b1a75e7efb86858c8ff6ab6744558009d8127e5543df9df543036504d21"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.6.1"
         },
         "click": {
             "hashes": [
@@ -123,11 +121,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:084dd1762f083a1bf49e41da1bfeafb475c9dce46265690a6bdd33290b9a63f4",
-                "sha256:6985cc5380c06db07fdc73ade15e6adbd4ce6ff850d7561ca00f97090b4b263d"
+                "sha256:9a1c52488045cd6c6491fd07711a380f932466e317cb8e016fc4e99dc7eac2f0",
+                "sha256:d73f25b283cd8033367451122aa868a23de0734757a01984e4b30b18b9050c72"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.30.0"
+            "version": "==4.34.4"
         },
         "geopandas": {
             "hashes": [
@@ -163,53 +161,52 @@
         },
         "kiwisolver": {
             "hashes": [
-                "sha256:0007840186bacfaa0aba4466d5890334ea5938e0bb7e28078a0eb0e63b5b59d5",
-                "sha256:19554bd8d54cf41139f376753af1a644b63c9ca93f8f72009d50a2080f870f77",
-                "sha256:1d45d1c74f88b9f41062716c727f78f2a59a5476ecbe74956fafb423c5c87a76",
-                "sha256:1d819553730d3c2724582124aee8a03c846ec4362ded1034c16fb3ef309264e6",
-                "sha256:2210f28778c7d2ee13f3c2a20a3a22db889e75f4ec13a21072eabb5693801e84",
-                "sha256:22521219ca739654a296eea6d4367703558fba16f98688bd8ce65abff36eaa84",
-                "sha256:25405f88a37c5f5bcba01c6e350086d65e7465fd1caaf986333d2a045045a223",
-                "sha256:2b65bd35f3e06a47b5c30ea99e0c2b88f72c6476eedaf8cfbc8e66adb5479dcf",
-                "sha256:2ddb500a2808c100e72c075cbb00bf32e62763c82b6a882d403f01a119e3f402",
-                "sha256:2f8f6c8f4f1cff93ca5058d6ec5f0efda922ecb3f4c5fb76181f327decff98b8",
-                "sha256:30fa008c172355c7768159983a7270cb23838c4d7db73d6c0f6b60dde0d432c6",
-                "sha256:3dbb3cea20b4af4f49f84cffaf45dd5f88e8594d18568e0225e6ad9dec0e7967",
-                "sha256:4116ba9a58109ed5e4cb315bdcbff9838f3159d099ba5259c7c7fb77f8537492",
-                "sha256:44e6adf67577dbdfa2d9f06db9fbc5639afefdb5bf2b4dfec25c3a7fbc619536",
-                "sha256:5326ddfacbe51abf9469fe668944bc2e399181a2158cb5d45e1d40856b2a0589",
-                "sha256:70adc3658138bc77a36ce769f5f183169bc0a2906a4f61f09673f7181255ac9b",
-                "sha256:72be6ebb4e92520b9726d7146bc9c9b277513a57a38efcf66db0620aec0097e0",
-                "sha256:7843b1624d6ccca403a610d1277f7c28ad184c5aa88a1750c1a999754e65b439",
-                "sha256:7ba5a1041480c6e0a8b11a9544d53562abc2d19220bfa14133e0cdd9967e97af",
-                "sha256:80efd202108c3a4150e042b269f7c78643420cc232a0a771743bb96b742f838f",
-                "sha256:82f49c5a79d3839bc8f38cb5f4bfc87e15f04cbafa5fbd12fb32c941cb529cfb",
-                "sha256:83d2c9db5dfc537d0171e32de160461230eb14663299b7e6d18ca6dca21e4977",
-                "sha256:8d93a1095f83e908fc253f2fb569c2711414c0bfd451cab580466465b235b470",
-                "sha256:8dc3d842fa41a33fe83d9f5c66c0cc1f28756530cd89944b63b072281e852031",
-                "sha256:9661a04ca3c950a8ac8c47f53cbc0b530bce1b52f516a1e87b7736fec24bfff0",
-                "sha256:a498bcd005e8a3fedd0022bb30ee0ad92728154a8798b703f394484452550507",
-                "sha256:a7a4cf5bbdc861987a7745aed7a536c6405256853c94abc9f3287c3fa401b174",
-                "sha256:b5074fb09429f2b7bc82b6fb4be8645dcbac14e592128beeff5461dcde0af09f",
-                "sha256:b6a5431940f28b6de123de42f0eb47b84a073ee3c3345dc109ad550a3307dd28",
-                "sha256:ba677bcaff9429fd1bf01648ad0901cea56c0d068df383d5f5856d88221fe75b",
-                "sha256:bcadb05c3d4794eb9eee1dddf1c24215c92fb7b55a80beae7a60530a91060560",
-                "sha256:bf7eb45d14fc036514c09554bf983f2a72323254912ed0c3c8e697b62c4c158f",
-                "sha256:c358721aebd40c243894298f685a19eb0491a5c3e0b923b9f887ef1193ddf829",
-                "sha256:c4550a359c5157aaf8507e6820d98682872b9100ce7607f8aa070b4b8af6c298",
-                "sha256:c6572c2dab23c86a14e82c245473d45b4c515314f1f859e92608dcafbd2f19b8",
-                "sha256:cba430db673c29376135e695c6e2501c44c256a81495da849e85d1793ee975ad",
-                "sha256:dedc71c8eb9c5096037766390172c34fb86ef048b8e8958b4e484b9e505d66bc",
-                "sha256:e6f5eb2f53fac7d408a45fbcdeda7224b1cfff64919d0f95473420a931347ae9",
-                "sha256:ec2eba188c1906b05b9b49ae55aae4efd8150c61ba450e6721f64620c50b59eb",
-                "sha256:ee040a7de8d295dbd261ef2d6d3192f13e2b08ec4a954de34a6fb8ff6422e24c",
-                "sha256:eedd3b59190885d1ebdf6c5e0ca56828beb1949b4dfe6e5d0256a461429ac386",
-                "sha256:f441422bb313ab25de7b3dbfd388e790eceb76ce01a18199ec4944b369017009",
-                "sha256:f8eb7b6716f5b50e9c06207a14172cf2de201e41912ebe732846c02c830455b9",
-                "sha256:fc4453705b81d03568d5b808ad8f09c77c47534f6ac2e72e733f9ca4714aa75c"
+                "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b",
+                "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166",
+                "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c",
+                "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c",
+                "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0",
+                "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c",
+                "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6",
+                "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004",
+                "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf",
+                "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac",
+                "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766",
+                "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6",
+                "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d",
+                "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191",
+                "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d",
+                "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f",
+                "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454",
+                "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8",
+                "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de",
+                "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a",
+                "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9",
+                "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3",
+                "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32",
+                "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938",
+                "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1",
+                "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d",
+                "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824",
+                "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b",
+                "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd",
+                "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3",
+                "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae",
+                "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597",
+                "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955",
+                "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a",
+                "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea",
+                "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede",
+                "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408",
+                "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871",
+                "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29",
+                "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897",
+                "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0",
+                "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09",
+                "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.3.2"
+            "version": "==1.4.4"
         },
         "llvmlite": {
             "hashes": [
@@ -235,44 +232,44 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:14334b9902ec776461c4b8c6516e26b450f7ebe0b3ef8703bf5cdfbbaecf774a",
-                "sha256:2252bfac85cec7af4a67e494bfccf9080bcba8a0299701eab075f48847cca907",
-                "sha256:2e3484d8455af3fdb0424eae1789af61f6a79da0c80079125112fd5c1b604218",
-                "sha256:34a1fc29f8f96e78ec57a5eff5e8d8b53d3298c3be6df61e7aa9efba26929522",
-                "sha256:3e66497cd990b1a130e21919b004da2f1dc112132c01ac78011a90a0f9229778",
-                "sha256:40e0d7df05e8efe60397c69b467fc8f87a2affeb4d562fe92b72ff8937a2b511",
-                "sha256:456cc8334f6d1124e8ff856b42d2cc1c84335375a16448189999496549f7182b",
-                "sha256:506b210cc6e66a0d1c2bb765d055f4f6bc2745070fb1129203b67e85bbfa5c18",
-                "sha256:53273c5487d1c19c3bc03b9eb82adaf8456f243b97ed79d09dded747abaf1235",
-                "sha256:577ed20ec9a18d6bdedb4616f5e9e957b4c08563a9f985563a31fd5b10564d2a",
-                "sha256:6803299cbf4665eca14428d9e886de62e24f4223ac31ab9c5d6d5339a39782c7",
-                "sha256:68fa30cec89b6139dc559ed6ef226c53fd80396da1919a1b5ef672c911aaa767",
-                "sha256:6c094e4bfecd2fa7f9adffd03d8abceed7157c928c2976899de282f3600f0a3d",
-                "sha256:778d398c4866d8e36ee3bf833779c940b5f57192fa0a549b3ad67bc4c822771b",
-                "sha256:7a350ca685d9f594123f652ba796ee37219bf72c8e0fc4b471473d87121d6d34",
-                "sha256:87900c67c0f1728e6db17c6809ec05c025c6624dcf96a8020326ea15378fe8e7",
-                "sha256:8a77906dc2ef9b67407cec0bdbf08e3971141e535db888974a915be5e1e3efc6",
-                "sha256:8e70ae6475cfd0fad3816dcbf6cac536dc6f100f7474be58d59fa306e6e768a4",
-                "sha256:abf67e05a1b7f86583f6ebd01f69b693b9c535276f4e943292e444855870a1b8",
-                "sha256:b04fc29bcef04d4e2d626af28d9d892be6aba94856cb46ed52bcb219ceac8943",
-                "sha256:b19a761b948e939a9e20173aaae76070025f0024fc8f7ba08bef22a5c8573afc",
-                "sha256:b2e9810e09c3a47b73ce9cab5a72243a1258f61e7900969097a817232246ce1c",
-                "sha256:b71f3a7ca935fc759f2aed7cec06cfe10bc3100fadb5dbd9c435b04e557971e1",
-                "sha256:b8a4fb2a0c5afbe9604f8a91d7d0f27b1832c3e0b5e365f95a13015822b4cd65",
-                "sha256:bb1c613908f11bac270bc7494d68b1ef6e7c224b7a4204d5dacf3522a41e2bc3",
-                "sha256:d24e5bb8028541ce25e59390122f5e48c8506b7e35587e5135efcb6471b4ac6c",
-                "sha256:d70a32ee1f8b55eed3fd4e892f0286df8cccc7e0475c11d33b5d0a148f5c7599",
-                "sha256:e293b16cf303fe82995e41700d172a58a15efc5331125d08246b520843ef21ee",
-                "sha256:e2f28a07b4f82abb40267864ad7b3a4ed76f1b1663e81c7efc84a9b9248f672f",
-                "sha256:e3520a274a0e054e919f5b3279ee5dbccf5311833819ccf3399dab7c83e90a25",
-                "sha256:e3b6f3fd0d8ca37861c31e9a7cab71a0ef14c639b4c95654ea1dd153158bf0df",
-                "sha256:e486f60db0cd1c8d68464d9484fd2a94011c1ac8593d765d0211f9daba2bd535",
-                "sha256:e8c87cdaf06fd7b2477f68909838ff4176f105064a72ca9d24d3f2a29f73d393",
-                "sha256:edf5e4e1d5fb22c18820e8586fb867455de3b109c309cb4fce3aaed85d9468d1",
-                "sha256:fe8d40c434a8e2c68d64c6d6a04e77f21791a93ff6afe0dce169597c110d3079"
+                "sha256:03bbb3f5f78836855e127b5dab228d99551ad0642918ccbf3067fcd52ac7ac5e",
+                "sha256:24173c23d1bcbaed5bf47b8785d27933a1ac26a5d772200a0f3e0e38f471b001",
+                "sha256:2a0967d4156adbd0d46db06bc1a877f0370bce28d10206a5071f9ecd6dc60b79",
+                "sha256:2e8bda1088b941ead50caabd682601bece983cadb2283cafff56e8fcddbf7d7f",
+                "sha256:31fbc2af27ebb820763f077ec7adc79b5a031c2f3f7af446bd7909674cd59460",
+                "sha256:364e6bca34edc10a96aa3b1d7cd76eb2eea19a4097198c1b19e89bee47ed5781",
+                "sha256:3d8e129af95b156b41cb3be0d9a7512cc6d73e2b2109f82108f566dbabdbf377",
+                "sha256:44c6436868186564450df8fd2fc20ed9daaef5caad699aa04069e87099f9b5a8",
+                "sha256:48cf850ce14fa18067f2d9e0d646763681948487a8080ec0af2686468b4607a2",
+                "sha256:49a5938ed6ef9dda560f26ea930a2baae11ea99e1c2080c8714341ecfda72a89",
+                "sha256:4a05f2b37222319753a5d43c0a4fd97ed4ff15ab502113e3f2625c26728040cf",
+                "sha256:4a44cdfdb9d1b2f18b1e7d315eb3843abb097869cd1ef89cfce6a488cd1b5182",
+                "sha256:4fa28ca76ac5c2b2d54bc058b3dad8e22ee85d26d1ee1b116a6fd4d2277b6a04",
+                "sha256:5844cea45d804174bf0fac219b4ab50774e504bef477fc10f8f730ce2d623441",
+                "sha256:5a32ea6e12e80dedaca2d4795d9ed40f97bfa56e6011e14f31502fdd528b9c89",
+                "sha256:6c623b355d605a81c661546af7f24414165a8a2022cddbe7380a31a4170fa2e9",
+                "sha256:751d3815b555dcd6187ad35b21736dc12ce6925fc3fa363bbc6dc0f86f16484f",
+                "sha256:75c406c527a3aa07638689586343f4b344fcc7ab1f79c396699eb550cd2b91f7",
+                "sha256:77157be0fc4469cbfb901270c205e7d8adb3607af23cef8bd11419600647ceed",
+                "sha256:7d7705022df2c42bb02937a2a824f4ec3cca915700dd80dc23916af47ff05f1a",
+                "sha256:7f409716119fa39b03da3d9602bd9b41142fab7a0568758cd136cd80b1bf36c8",
+                "sha256:9480842d5aadb6e754f0b8f4ebeb73065ac8be1855baa93cd082e46e770591e9",
+                "sha256:9776e1a10636ee5f06ca8efe0122c6de57ffe7e8c843e0fb6e001e9d9256ec95",
+                "sha256:a91426ae910819383d337ba0dc7971c7cefdaa38599868476d94389a329e599b",
+                "sha256:b4fedaa5a9aa9ce14001541812849ed1713112651295fdddd640ea6620e6cf98",
+                "sha256:b6c63cd01cad0ea8704f1fd586e9dc5777ccedcd42f63cbbaa3eae8dd41172a1",
+                "sha256:b8d3f4e71e26307e8c120b72c16671d70c5cd08ae412355c11254aa8254fb87f",
+                "sha256:c4b82c2ae6d305fcbeb0eb9c93df2602ebd2f174f6e8c8a5d92f9445baa0c1d3",
+                "sha256:c772264631e5ae61f0bd41313bbe48e1b9bcc95b974033e1118c9caa1a84d5c6",
+                "sha256:c87973ddec10812bddc6c286b88fdd654a666080fbe846a1f7a3b4ba7b11ab78",
+                "sha256:e2b696699386766ef171a259d72b203a3c75d99d03ec383b97fc2054f52e15cf",
+                "sha256:ea75df8e567743207e2b479ba3d8843537be1c146d4b1e3e395319a4e1a77fe9",
+                "sha256:ebc27ad11df3c1661f4677a7762e57a8a91dd41b466c3605e90717c9a5f90c82",
+                "sha256:ee0b8e586ac07f83bb2950717e66cb305e2859baf6f00a9c39cc576e0ce9629c",
+                "sha256:ee175a571e692fc8ae8e41ac353c0e07259113f4cb063b0ec769eff9717e84bb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.5.1"
+            "version": "==3.5.2"
         },
         "memory-profiler": {
             "hashes": [
@@ -290,11 +287,11 @@
         },
         "natsort": {
             "hashes": [
-                "sha256:09cff3cabb617f9794f643758ab5b337c1c0af1a6df11edf5c04de496fd7fb26",
-                "sha256:feb87e0ce1dc1f8f3f21e18a85216c790e746d76a5ff6889563394605f504a2b"
+                "sha256:c7c1f3f27c375719a4dfcab353909fe39f26c2032a062a8c80cc844eaaca0445",
+                "sha256:f59988d2f24e77b6b56f8a8f882d5df6b3b637e09e075abc67b486d59fba1a4b"
             ],
             "index": "pypi",
-            "version": "==8.0.2"
+            "version": "==8.1.0"
         },
         "netcdf4": {
             "hashes": [
@@ -364,60 +361,61 @@
         },
         "numexpr": {
             "hashes": [
-                "sha256:1639561d056d2d790a56ddab7e7df40b6181ad50338b50fba94aa42874a00958",
-                "sha256:2247d92da60b85de619e797e59a80e9c7302fba82dcd0525de8f7dd729a0d60f",
-                "sha256:24fb5b2c17273a76e7de9cea7817c54262198657998a093fceb4030f273524c7",
-                "sha256:3bab5add6628fa8bb66fba7b1f0eed5d8d0ce05fdd2dcc326dde8a297a961c46",
-                "sha256:48258db3ba89ad028744e07b09dde963f82da7f081849d3a003bb0b96b112d4f",
-                "sha256:4eb79d9026f013cf8d16de8be74911c74c0c09362627bf4b39e2b7f1f3188c28",
-                "sha256:517f299c4bc8491b5117aa276e8f3cf7ee2e89223922e92e2ea78a32985d5087",
-                "sha256:5d0c98c4d8bcc25962e5859176e5728f69209cffb9b7f64bf6d1c801fe350946",
-                "sha256:6bce8a183afe157c25385d27be314be22f06ba644c89b611d20e2570a06bd5dd",
-                "sha256:79ec94295aa57f5a9d212116bb7359744cd2f9e05d477df0dee383b7f44b9588",
-                "sha256:80db25e2934fd1a1b787440d5fa7946adb79a1289d7dc64e2c8bcd6ceae660ad",
-                "sha256:920c6a3088790573765e103e20592864977aa4b4d1b819c298fa9d88771cde1b",
-                "sha256:a591f99ecbb413749725e8da4e52e663f0939dd5fbf1ae5a7c6c50ba734f57de",
-                "sha256:a97a087a5f5e56cd81c69215918fdaca60eb478a099daa757455e4ff887f7600",
-                "sha256:a9f046cb5752f08a9291dc1fd37a9cfd15770262188bb984e4418490fef9c9ec",
-                "sha256:ab6b2cb64bc9391f77f08203fda5af3647ed2abcefb928cc6282727854f97735",
-                "sha256:b57d3ab7054409d9b4d2148241ae70d738c0b0daeb1a0efd5ea89b9279752e22",
-                "sha256:bd402e43b8baf6436b7c2c14541f69eb4f97f023469585a7ad258c49622ff619",
-                "sha256:cd779aa44dd986c4ef10163519239602b027be06a527946656207acf1f58113b",
-                "sha256:cfd89f63028f8df3c9b11bf2c98085184f967a09f543a77c3335f4a0ec54f124",
-                "sha256:d148e99483e15de22d0acd5100136d39a336e91c8f8d37bf2e84e9f0ab4c0610",
-                "sha256:d2b4b6379763ec5d794d4aaa1834ae00f1bba82a36d0b99c6e2d559302a21e85",
-                "sha256:d7b64b125161e722c9dc8a27df282e755bd9a5adf826b2e3e1f038e3dfdc3307",
-                "sha256:da180aaec7f6c387540b251f6ec2b8d280220c0e45731778853c8b0d86c4ae22",
-                "sha256:ebf31aeed426840aefe1e94c89bb0b7530a72be36444ed4c73e4411865b79be5",
-                "sha256:fd6905bc80a11908e363c9821cbf8aeeca4dca5b6a2eea90a97b055bc73443e6"
-            ],
-            "version": "==2.8.1"
+                "sha256:052ec3a55cc1ccc447580ee5b828b2bd0bc14fea0756ddb81d9617b5472c77b5",
+                "sha256:08d8f8e31647815d979185eb455cb5b4d845e20ff808bd6f7f4edf5e0a35e2f6",
+                "sha256:1575a35190d650bf64d2efd8590a8ef3ca564ef20b9f8727428b57759712becb",
+                "sha256:19cd7563421862de85404bd5de06bee8a3ebff4fc9f718de09cc704bc3348f08",
+                "sha256:33be3bbbad71d97d14a39d84957c2bcc368fec775369664d0c24be030c50c359",
+                "sha256:3a1ce79b7d32c55cce334566e3c6716f7b646f6eceb2ace38adaa795848f3583",
+                "sha256:4ddc46c1e5d726b57d008169b75074ab66869e1827098614ebafa45d152f81b7",
+                "sha256:4f291f0df7b25d9530991f880cc232a644a7a722d130c61b43e593b98fb6523f",
+                "sha256:5532bd7164eb8a05410771faf94a661fc69e5ca72deb8612f1bedc26311ed3c8",
+                "sha256:5b014d1c426c444102fb9eea6438052ee86c82684d27dd20b531caf2c60bc4c9",
+                "sha256:5c660dea90935b963db9529d963493c40fabb2343684b52083fb86b2547d60c8",
+                "sha256:828926f5d4dc9ace2bebd2eec56bee852518afa31e6df175d1706e6631dfd1a2",
+                "sha256:854541cf4214d747ab2f87229e9dde052fddc52c379f59047d64f9b7e2f4d578",
+                "sha256:99b9a91811de8cd24bd7d7fbc1883653dad6485e8c683d85b1007a13868713f6",
+                "sha256:a6954d65d7140864d9bb2302b7580c60c88c4d12e00c59a0a53f1660573e922b",
+                "sha256:b127b0d0e1665b94adcc658c5f9d688ac4903ef81da5d8f4e956c995cf69d5c7",
+                "sha256:ba9aa03f4213f4e0c0d964afd6a920c9000e73d22b88c72c46b151d292ee5581",
+                "sha256:be9b8309a4a2f785197b1a29f7767a5ff217ea505e5a751b237336f3b50b7e48",
+                "sha256:c35669268f602ac6412c8c6244b256ebb4f31ffc926b936ca0d9cffda251db8a",
+                "sha256:cb647c9d9c785dae0759bf6c875cde2bec472b5c3f7a6015734b161ae766d141",
+                "sha256:cbd75ac287923bd0c5b95143915648c62d97f994b06dacd770bd205da014f6bd",
+                "sha256:eba7fad925e3063a0434844a667fbdea30b53fe1344efef73475b32d33aa0fec",
+                "sha256:f29b882a21b5381c0e472bc66e8d1c519b8920edc2522b8b4ede79e314d31d20",
+                "sha256:fe6b49631c3bf54e92b0fb334c8e59694685924492d80c325e1b44ecbbc0f22d"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.8.3"
         },
         "numpy": {
             "hashes": [
-                "sha256:07a8c89a04997625236c5ecb7afe35a02af3896c8aa01890a849913a2309c676",
-                "sha256:08d9b008d0156c70dc392bb3ab3abb6e7a711383c3247b410b39962263576cd4",
-                "sha256:201b4d0552831f7250a08d3b38de0d989d6f6e4658b709a02a73c524ccc6ffce",
-                "sha256:2c10a93606e0b4b95c9b04b77dc349b398fdfbda382d2a39ba5a822f669a0123",
-                "sha256:3ca688e1b9b95d80250bca34b11a05e389b1420d00e87a0d12dc45f131f704a1",
-                "sha256:48a3aecd3b997bf452a2dedb11f4e79bc5bfd21a1d4cc760e703c31d57c84b3e",
-                "sha256:568dfd16224abddafb1cbcce2ff14f522abe037268514dd7e42c6776a1c3f8e5",
-                "sha256:5bfb1bb598e8229c2d5d48db1860bcf4311337864ea3efdbe1171fb0c5da515d",
-                "sha256:639b54cdf6aa4f82fe37ebf70401bbb74b8508fddcf4797f9fe59615b8c5813a",
-                "sha256:8251ed96f38b47b4295b1ae51631de7ffa8260b5b087808ef09a39a9d66c97ab",
-                "sha256:92bfa69cfbdf7dfc3040978ad09a48091143cffb778ec3b03fa170c494118d75",
-                "sha256:97098b95aa4e418529099c26558eeb8486e66bd1e53a6b606d684d0c3616b168",
-                "sha256:a3bae1a2ed00e90b3ba5f7bd0a7c7999b55d609e0c54ceb2b076a25e345fa9f4",
-                "sha256:c34ea7e9d13a70bf2ab64a2532fe149a9aced424cd05a2c4ba662fd989e3e45f",
-                "sha256:dbc7601a3b7472d559dc7b933b18b4b66f9aa7452c120e87dfb33d02008c8a18",
-                "sha256:e7927a589df200c5e23c57970bafbd0cd322459aa7b1ff73b7c2e84d6e3eae62",
-                "sha256:f8c1f39caad2c896bc0018f699882b345b2a63708008be29b1f355ebf6f933fe",
-                "sha256:f950f8845b480cffe522913d35567e29dd381b0dc7e4ce6a4a9f9156417d2430",
-                "sha256:fade0d4f4d292b6f39951b6836d7a3c7ef5b2347f3c420cd9820a1d90d794802",
-                "sha256:fdf3c08bce27132395d3c3ba1503cac12e17282358cb4bddc25cc46b0aca07aa"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.22.3"
+                "sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207",
+                "sha256:1ce7ab2053e36c0a71e7a13a7475bd3b1f54750b4b433adc96313e127b870887",
+                "sha256:2d487e06ecbf1dc2f18e7efce82ded4f705f4bd0cd02677ffccfb39e5c284c7e",
+                "sha256:37431a77ceb9307c28382c9773da9f306435135fae6b80b62a11c53cfedd8802",
+                "sha256:3e1ffa4748168e1cc8d3cde93f006fe92b5421396221a02f2274aab6ac83b077",
+                "sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af",
+                "sha256:43a8ca7391b626b4c4fe20aefe79fec683279e31e7c79716863b4b25021e0e74",
+                "sha256:4c6036521f11a731ce0648f10c18ae66d7143865f19f7299943c985cdc95afb5",
+                "sha256:59d55e634968b8f77d3fd674a3cf0b96e85147cd6556ec64ade018f27e9479e1",
+                "sha256:64f56fc53a2d18b1924abd15745e30d82a5782b2cab3429aceecc6875bd5add0",
+                "sha256:7228ad13744f63575b3a972d7ee4fd61815b2879998e70930d4ccf9ec721dce0",
+                "sha256:9ce7df0abeabe7fbd8ccbf343dc0db72f68549856b863ae3dd580255d009648e",
+                "sha256:a911e317e8c826ea632205e63ed8507e0dc877dcdc49744584dfc363df9ca08c",
+                "sha256:b89bf9b94b3d624e7bb480344e91f68c1c6c75f026ed6755955117de00917a7c",
+                "sha256:ba9ead61dfb5d971d77b6c131a9dbee62294a932bf6a356e48c75ae684e635b3",
+                "sha256:c1d937820db6e43bec43e8d016b9b3165dcb42892ea9f106c70fb13d430ffe72",
+                "sha256:cc7f00008eb7d3f2489fca6f334ec19ca63e31371be28fd5dad955b16ec285bd",
+                "sha256:d4c5d5eb2ec8da0b4f50c9a843393971f31f1d60be87e0fb0917a49133d257d6",
+                "sha256:e96d7f3096a36c8754207ab89d4b3282ba7b49ea140e4973591852c77d09eb76",
+                "sha256:f0725df166cf4785c0bc4cbfb320203182b1ecd30fee6e541c8752a92df6aa32",
+                "sha256:f3eb268dbd5cfaffd9448113539e44e2dd1c5ca9ce25576f7c04a5453edc26fa",
+                "sha256:fb7a980c81dd932381f8228a426df8aeb70d59bbcda2af075b627bbc50207cba"
+            ],
+            "index": "pypi",
+            "version": "==1.22.4"
         },
         "packaging": {
             "hashes": [
@@ -451,82 +449,146 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:011233e0c42a4a7836498e98c1acf5e744c96a67dd5032a6f666cc1fb97eab97",
-                "sha256:0f29d831e2151e0b7b39981756d201f7108d3d215896212ffe2e992d06bfe049",
-                "sha256:12875d118f21cf35604176872447cdb57b07126750a33748bac15e77f90f1f9c",
-                "sha256:14d4b1341ac07ae07eb2cc682f459bec932a380c3b122f5540432d8977e64eae",
-                "sha256:1c3c33ac69cf059bbb9d1a71eeaba76781b450bc307e2291f8a4764d779a6b28",
-                "sha256:1d19397351f73a88904ad1aee421e800fe4bbcd1aeee6435fb62d0a05ccd1030",
-                "sha256:253e8a302a96df6927310a9d44e6103055e8fb96a6822f8b7f514bb7ef77de56",
-                "sha256:2632d0f846b7c7600edf53c48f8f9f1e13e62f66a6dbc15191029d950bfed976",
-                "sha256:335ace1a22325395c4ea88e00ba3dc89ca029bd66bd5a3c382d53e44f0ccd77e",
-                "sha256:413ce0bbf9fc6278b2d63309dfeefe452835e1c78398efb431bab0672fe9274e",
-                "sha256:5100b45a4638e3c00e4d2320d3193bdabb2d75e79793af7c3eb139e4f569f16f",
-                "sha256:514ceac913076feefbeaf89771fd6febde78b0c4c1b23aaeab082c41c694e81b",
-                "sha256:528a2a692c65dd5cafc130de286030af251d2ee0483a5bf50c9348aefe834e8a",
-                "sha256:6295f6763749b89c994fcb6d8a7f7ce03c3992e695f89f00b741b4580b199b7e",
-                "sha256:6c8bc8238a7dfdaf7a75f5ec5a663f4173f8c367e5a39f87e720495e1eed75fa",
-                "sha256:718856856ba31f14f13ba885ff13874be7fefc53984d2832458f12c38205f7f7",
-                "sha256:7f7609a718b177bf171ac93cea9fd2ddc0e03e84d8fa4e887bdfc39671d46b00",
-                "sha256:80ca33961ced9c63358056bd08403ff866512038883e74f3a4bf88ad3eb66838",
-                "sha256:80fe64a6deb6fcfdf7b8386f2cf216d329be6f2781f7d90304351811fb591360",
-                "sha256:81c4b81611e3a3cb30e59b0cf05b888c675f97e3adb2c8672c3154047980726b",
-                "sha256:855c583f268edde09474b081e3ddcd5cf3b20c12f26e0d434e1386cc5d318e7a",
-                "sha256:9bfdb82cdfeccec50aad441afc332faf8606dfa5e8efd18a6692b5d6e79f00fd",
-                "sha256:a5d24e1d674dd9d72c66ad3ea9131322819ff86250b30dc5821cbafcfa0b96b4",
-                "sha256:a9f44cd7e162ac6191491d7249cceb02b8116b0f7e847ee33f739d7cb1ea1f70",
-                "sha256:b5b3f092fe345c03bca1e0b687dfbb39364b21ebb8ba90e3fa707374b7915204",
-                "sha256:b9618823bd237c0d2575283f2939655f54d51b4527ec3972907a927acbcc5bfc",
-                "sha256:cef9c85ccbe9bee00909758936ea841ef12035296c748aaceee535969e27d31b",
-                "sha256:d21237d0cd37acded35154e29aec853e945950321dd2ffd1a7d86fe686814669",
-                "sha256:d3c5c79ab7dfce6d88f1ba639b77e77a17ea33a01b07b99840d6ed08031cb2a7",
-                "sha256:d9d7942b624b04b895cb95af03a23407f17646815495ce4547f0e60e0b06f58e",
-                "sha256:db6d9fac65bd08cea7f3540b899977c6dee9edad959fa4eaf305940d9cbd861c",
-                "sha256:ede5af4a2702444a832a800b8eb7f0a7a1c0eed55b644642e049c98d589e5092",
-                "sha256:effb7749713d5317478bb3acb3f81d9d7c7f86726d41c1facca068a04cf5bb4c",
-                "sha256:f154d173286a5d1863637a7dcd8c3437bb557520b01bddb0be0258dcb72696b5",
-                "sha256:f25ed6e28ddf50de7e7ea99d7a976d6a9c415f03adcaac9c41ff6ff41b6d86ac"
+                "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927",
+                "sha256:030e3460861488e249731c3e7ab59b07c7853838ff3b8e16aac9561bb345da14",
+                "sha256:0ed2c4ef2451de908c90436d6e8092e13a43992f1860275b4d8082667fbb2ffc",
+                "sha256:136659638f61a251e8ed3b331fc6ccd124590eeff539de57c5f80ef3a9594e58",
+                "sha256:13b725463f32df1bfeacbf3dd197fb358ae8ebcd8c5548faa75126ea425ccb60",
+                "sha256:1536ad017a9f789430fb6b8be8bf99d2f214c76502becc196c6f2d9a75b01b76",
+                "sha256:15928f824870535c85dbf949c09d6ae7d3d6ac2d6efec80f3227f73eefba741c",
+                "sha256:17d4cafe22f050b46d983b71c707162d63d796a1235cdf8b9d7a112e97b15bac",
+                "sha256:1802f34298f5ba11d55e5bb09c31997dc0c6aed919658dfdf0198a2fe75d5490",
+                "sha256:1cc1d2451e8a3b4bfdb9caf745b58e6c7a77d2e469159b0d527a4554d73694d1",
+                "sha256:1fd6f5e3c0e4697fa7eb45b6e93996299f3feee73a3175fa451f49a74d092b9f",
+                "sha256:254164c57bab4b459f14c64e93df11eff5ded575192c294a0c49270f22c5d93d",
+                "sha256:2ad0d4df0f5ef2247e27fc790d5c9b5a0af8ade9ba340db4a73bb1a4a3e5fb4f",
+                "sha256:2c58b24e3a63efd22554c676d81b0e57f80e0a7d3a5874a7e14ce90ec40d3069",
+                "sha256:2d33a11f601213dcd5718109c09a52c2a1c893e7461f0be2d6febc2879ec2402",
+                "sha256:337a74fd2f291c607d220c793a8135273c4c2ab001b03e601c36766005f36885",
+                "sha256:37ff6b522a26d0538b753f0b4e8e164fdada12db6c6f00f62145d732d8a3152e",
+                "sha256:3d1f14f5f691f55e1b47f824ca4fdcb4b19b4323fe43cc7bb105988cad7496be",
+                "sha256:408673ed75594933714482501fe97e055a42996087eeca7e5d06e33218d05aa8",
+                "sha256:4134d3f1ba5f15027ff5c04296f13328fecd46921424084516bdb1b2548e66ff",
+                "sha256:4ad2f835e0ad81d1689f1b7e3fbac7b01bb8777d5a985c8962bedee0cc6d43da",
+                "sha256:50dff9cc21826d2977ef2d2a205504034e3a4563ca6f5db739b0d1026658e004",
+                "sha256:510cef4a3f401c246cfd8227b300828715dd055463cdca6176c2e4036df8bd4f",
+                "sha256:5aed7dde98403cd91d86a1115c78d8145c83078e864c1de1064f52e6feb61b20",
+                "sha256:69bd1a15d7ba3694631e00df8de65a8cb031911ca11f44929c97fe05eb9b6c1d",
+                "sha256:6bf088c1ce160f50ea40764f825ec9b72ed9da25346216b91361eef8ad1b8f8c",
+                "sha256:6e8c66f70fb539301e064f6478d7453e820d8a2c631da948a23384865cd95544",
+                "sha256:727dd1389bc5cb9827cbd1f9d40d2c2a1a0c9b32dd2261db522d22a604a6eec9",
+                "sha256:74a04183e6e64930b667d321524e3c5361094bb4af9083db5c301db64cd341f3",
+                "sha256:75e636fd3e0fb872693f23ccb8a5ff2cd578801251f3a4f6854c6a5d437d3c04",
+                "sha256:7761afe0126d046974a01e030ae7529ed0ca6a196de3ec6937c11df0df1bc91c",
+                "sha256:7888310f6214f19ab2b6df90f3f06afa3df7ef7355fc025e78a3044737fab1f5",
+                "sha256:7b0554af24df2bf96618dac71ddada02420f946be943b181108cac55a7a2dcd4",
+                "sha256:7c7b502bc34f6e32ba022b4a209638f9e097d7a9098104ae420eb8186217ebbb",
+                "sha256:808add66ea764ed97d44dda1ac4f2cfec4c1867d9efb16a33d158be79f32b8a4",
+                "sha256:831e648102c82f152e14c1a0938689dbb22480c548c8d4b8b248b3e50967b88c",
+                "sha256:93689632949aff41199090eff5474f3990b6823404e45d66a5d44304e9cdc467",
+                "sha256:96b5e6874431df16aee0c1ba237574cb6dff1dcb173798faa6a9d8b399a05d0e",
+                "sha256:9a54614049a18a2d6fe156e68e188da02a046a4a93cf24f373bffd977e943421",
+                "sha256:a138441e95562b3c078746a22f8fca8ff1c22c014f856278bdbdd89ca36cff1b",
+                "sha256:a647c0d4478b995c5e54615a2e5360ccedd2f85e70ab57fbe817ca613d5e63b8",
+                "sha256:a9c9bc489f8ab30906d7a85afac4b4944a572a7432e00698a7239f44a44e6efb",
+                "sha256:ad2277b185ebce47a63f4dc6302e30f05762b688f8dc3de55dbae4651872cdf3",
+                "sha256:b6d5e92df2b77665e07ddb2e4dbd6d644b78e4c0d2e9272a852627cdba0d75cf",
+                "sha256:bc431b065722a5ad1dfb4df354fb9333b7a582a5ee39a90e6ffff688d72f27a1",
+                "sha256:bdd0de2d64688ecae88dd8935012c4a72681e5df632af903a1dca8c5e7aa871a",
+                "sha256:c79698d4cd9318d9481d89a77e2d3fcaeff5486be641e60a4b49f3d2ecca4e28",
+                "sha256:cb6259196a589123d755380b65127ddc60f4c64b21fc3bb46ce3a6ea663659b0",
+                "sha256:d5b87da55a08acb586bad5c3aa3b86505f559b84f39035b233d5bf844b0834b1",
+                "sha256:dcd7b9c7139dc8258d164b55696ecd16c04607f1cc33ba7af86613881ffe4ac8",
+                "sha256:dfe4c1fedfde4e2fbc009d5ad420647f7730d719786388b7de0999bf32c0d9fd",
+                "sha256:ea98f633d45f7e815db648fd7ff0f19e328302ac36427343e4432c84432e7ff4",
+                "sha256:ec52c351b35ca269cb1f8069d610fc45c5bd38c3e91f9ab4cbbf0aebc136d9c8",
+                "sha256:eef7592281f7c174d3d6cbfbb7ee5984a671fcd77e3fc78e973d492e9bf0eb3f",
+                "sha256:f07f1f00e22b231dd3d9b9208692042e29792d6bd4f6639415d2f23158a80013",
+                "sha256:f3fac744f9b540148fa7715a435d2283b71f68bfb6d4aae24482a890aed18b59",
+                "sha256:fa768eff5f9f958270b081bb33581b4b569faabf8774726b283edb06617101dc",
+                "sha256:fac2d65901fb0fdf20363fbd345c01958a742f2dc62a8dd4495af66e3ff502a4"
             ],
-            "index": "pypi",
-            "version": "==9.0.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==9.2.0"
         },
         "psutil": {
             "hashes": [
-                "sha256:072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5",
-                "sha256:1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a",
-                "sha256:1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4",
-                "sha256:3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841",
-                "sha256:32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d",
-                "sha256:3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d",
-                "sha256:3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0",
-                "sha256:4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845",
-                "sha256:539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf",
-                "sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b",
-                "sha256:58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07",
-                "sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618",
-                "sha256:742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2",
-                "sha256:7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd",
-                "sha256:76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666",
-                "sha256:7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce",
-                "sha256:7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3",
-                "sha256:8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d",
-                "sha256:869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25",
-                "sha256:90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492",
-                "sha256:9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b",
-                "sha256:b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d",
-                "sha256:c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2",
-                "sha256:c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203",
-                "sha256:cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2",
-                "sha256:d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94",
-                "sha256:df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9",
-                "sha256:e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64",
-                "sha256:e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56",
-                "sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3",
-                "sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c",
-                "sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.0"
+                "sha256:068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685",
+                "sha256:0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc",
+                "sha256:0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36",
+                "sha256:19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1",
+                "sha256:20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329",
+                "sha256:28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81",
+                "sha256:29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de",
+                "sha256:3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4",
+                "sha256:32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574",
+                "sha256:3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237",
+                "sha256:44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22",
+                "sha256:4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b",
+                "sha256:56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0",
+                "sha256:57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954",
+                "sha256:58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021",
+                "sha256:645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537",
+                "sha256:799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87",
+                "sha256:79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0",
+                "sha256:91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc",
+                "sha256:9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af",
+                "sha256:944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4",
+                "sha256:a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453",
+                "sha256:a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689",
+                "sha256:abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8",
+                "sha256:b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680",
+                "sha256:b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e",
+                "sha256:c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9",
+                "sha256:d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b",
+                "sha256:db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d",
+                "sha256:e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2",
+                "sha256:f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5",
+                "sha256:fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==5.9.1"
+        },
+        "psycopg2-binary": {
+            "hashes": [
+                "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c",
+                "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67",
+                "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0",
+                "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6",
+                "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db",
+                "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94",
+                "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52",
+                "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056",
+                "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b",
+                "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd",
+                "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550",
+                "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679",
+                "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83",
+                "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77",
+                "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2",
+                "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77",
+                "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2",
+                "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd",
+                "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859",
+                "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1",
+                "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25",
+                "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152",
+                "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf",
+                "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f",
+                "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729",
+                "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71",
+                "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66",
+                "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4",
+                "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449",
+                "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da",
+                "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a",
+                "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c",
+                "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb",
+                "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4",
+                "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5"
+            ],
+            "index": "pypi",
+            "version": "==2.8.6"
         },
         "pygeos": {
             "hashes": [
@@ -552,11 +614,11 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
-                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
+                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
+                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.0.7"
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.0.9"
         },
         "pyproj": {
             "hashes": [
@@ -593,18 +655,18 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:32b2bdc1873fd3a3c346da1c6db83d0053c3c62f28f1f38516070c4c8971b1d3",
-                "sha256:a5de49a31e953b45ff2d2fd434bbc2670e8db5273606c1e737cc6b93eff3655f"
+                "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f",
+                "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"
             ],
             "index": "pypi",
-            "version": "==0.19.2"
+            "version": "==0.20.0"
         },
         "pytz": {
             "hashes": [
-                "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c",
-                "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"
+                "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7",
+                "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"
             ],
-            "version": "==2021.3"
+            "version": "==2022.1"
         },
         "rasterio": {
             "hashes": [
@@ -651,32 +713,32 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:011d4386b53b933142f58a652aa0f149c9b9242abd4f900b9f4ea5fbafc86b89",
-                "sha256:16e09ef68b352d73befa8bcaf3ebe25d3941fe1a58c82909d5589856e6bc8174",
-                "sha256:31d4f2d6b724bc9a98e527b5849b8a7e589bf1ea630c33aa563eda912c9ff0bd",
-                "sha256:38aa39b6724cb65271e469013aeb6f2ce66fd44f093e241c28a9c6bc64fd79ed",
-                "sha256:3d573228c10a3a8c32b9037be982e6440e411b443a6267b067cac72f690b8d56",
-                "sha256:3d9dd6c8b93a22bf9a3a52d1327aca7e092b1299fb3afc4f89e8eba381be7b59",
-                "sha256:559a8a4c03a5ba9fe3232f39ed24f86457e4f3f6c0abbeae1fb945029f092720",
-                "sha256:5e73343c5e0d413c1f937302b2e04fb07872f5843041bcfd50699aef6e95e399",
-                "sha256:723b9f878095ed994756fa4ee3060c450e2db0139c5ba248ee3f9628bd64e735",
-                "sha256:87b01c7d5761e8a266a0fbdb9d88dcba0910d63c1c671bdb4d99d29f469e9e03",
-                "sha256:8f4d059a97b29c91afad46b1737274cb282357a305a80bdd9e8adf3b0ca6a3f0",
-                "sha256:92b2c2af4183ed09afb595709a8ef5783b2baf7f41e26ece24e1329c109691a7",
-                "sha256:937d28722f13302febde29847bbe554b89073fbb924a30475e5ed7b028898b5f",
-                "sha256:a279e27c7f4566ef18bab1b1e2c37d168e365080974758d107e7d237d3f0f484",
-                "sha256:ad5be4039147c808e64f99c0e8a9641eb5d2fa079ff5894dcd8240e94e347af4",
-                "sha256:ae3e327da323d82e918e593460e23babdce40d7ab21490ddf9fc06dec6b91a18",
-                "sha256:bb7088e89cd751acf66195d2f00cf009a1ea113f3019664032d9075b1e727b6c",
-                "sha256:c17a1878d00a5dd2797ccd73623ceca9d02375328f6218ee6d921e1325e61aff",
-                "sha256:c2bae431d127bf0b1da81fc24e4bba0a84d058e3a96b9dd6475dfcb3c5e8761e",
-                "sha256:de2e80ee1d925984c2504812a310841c241791c5279352be4707cdcd7c255039",
-                "sha256:e6f0cd9c0bd374ef834ee1e0f0999678d49dcc400ea6209113d81528958f97c7",
-                "sha256:f3720d0124aced49f6f2198a6900304411dbbeed12f56951d7c66ebef05e3df6",
-                "sha256:f4a6d3b9f9797eb2d43938ac2c5d96d02aed17ef170c8b38f11798717523ddba"
-            ],
-            "markers": "python_version < '3.11' and python_version >= '3.8'",
-            "version": "==1.8.0"
+                "sha256:01c2015e132774feefe059d5354055fec6b751d7a7d70ad2cf5ce314e7426e2a",
+                "sha256:0424d1bbbfa51d5ddaa16d067fd593863c9f2fb7c6840c32f8a08a8832f8e7a4",
+                "sha256:10417935486b320d98536d732a58362e3d37e84add98c251e070c59a6bfe0863",
+                "sha256:12005d30894e4fe7b247f7233ba0801a341f887b62e2eb99034dd6f2a8a33ad6",
+                "sha256:16207622570af10f9e6a2cdc7da7a9660678852477adbcd056b6d1057a036fef",
+                "sha256:45f0d6c0d6e55582d3b8f5c58ad4ca4259a02affb190f89f06c8cc02e21bba81",
+                "sha256:5d1b9cf3771fd921f7213b4b886ab2606010343bb36259b544a816044576d69e",
+                "sha256:693b3fe2e7736ce0dbc72b4d933798eb6ca8ce51b8b934e3f547cc06f48b2afb",
+                "sha256:73b704c5eea9be811919cae4caacf3180dd9212d9aed08477c1d2ba14900a9de",
+                "sha256:79dd7876614fc2869bf5d311ef33962d2066ea888bc66c80fd4fa80f8772e5a9",
+                "sha256:7bad16b91918bf3288089a78a4157e04892ea6475fb7a1d9bcdf32c30c8a3dba",
+                "sha256:8d541db2d441ef87afb60c4a2addb00c3af281633602a4967e733ef4b7050504",
+                "sha256:8f2232c9d9119ec356240255a715a289b3a33be828c3e4abac11fd052ce15b1e",
+                "sha256:97a1f1e51ea30782d7baa8d0c52f72c3f9f05cb609cf1b990664231c5102bccd",
+                "sha256:adb6c438c6ef550e2bb83968e772b9690cb421f2c6073f9c2cb6af15ee538bc9",
+                "sha256:bb687d245b6963673c639f318eea7e875d1ba147a67925586abed3d6f39bb7d8",
+                "sha256:bd490f77f35800d5620f4d9af669e372d9a88db1f76ef219e1609cc4ecdd1a24",
+                "sha256:c0dfd7d2429452e7e94904c6a3af63cbaa3cf51b348bd9d35b42db7e9ad42791",
+                "sha256:d3a326673ac5afa9ef5613a61626b9ec15c8f7222b4ecd1ce0fd8fcba7b83c59",
+                "sha256:e2004d2a3c397b26ca78e67c9d320153a1a9b71ae713ad33f4a3a3ab3d79cc65",
+                "sha256:e2ac088ea4aa61115b96b47f5f3d94b3fa29554340b6629cd2bfe6b0521ee33b",
+                "sha256:f7c3c578ff556333f3890c2df6c056955d53537bb176698359088108af73a58f",
+                "sha256:fc58c3fcb8a724b703ffbc126afdca5a8353d4d5945d5c92db85617e165299e7"
+            ],
+            "markers": "python_version < '3.12' and python_version >= '3.8'",
+            "version": "==1.9.0"
         },
         "seaborn": {
             "hashes": [
@@ -688,11 +750,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:2347b2b432c891a863acadca2da9ac101eae6169b1d3dfee2ec605ecd50dbfe5",
-                "sha256:e4f30b9f84e5ab3decf945113119649fec09c1fc3507c6ebffec75646c56e62b"
+                "sha256:7c7854ee1429a240090297628dc9f75b35318d193537968e2dc14010ee2f5bca",
+                "sha256:dc2662692f47d99cb8ae15a784529adeed535bcd7c277fee0beccf961522baf6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==60.9.3"
+            "version": "==63.4.1"
         },
         "shapely": {
             "hashes": [
@@ -849,6 +911,13 @@
         }
     },
     "develop": {
+        "asttokens": {
+            "hashes": [
+                "sha256:0844691e88552595a6f4a4281a9f7f79b8dd45ca4ccea82e5e05b4bbdb76705c",
+                "sha256:9a54c114f02c7a9480d56550932546a3f1fe71d8a02f1bc7ccd0ee3ee35cf4d5"
+            ],
+            "version": "==2.0.5"
+        },
         "backcall": {
             "hashes": [
                 "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e",
@@ -864,13 +933,20 @@
             "markers": "python_version >= '3.5'",
             "version": "==5.1.1"
         },
+        "executing": {
+            "hashes": [
+                "sha256:4ce4d6082d99361c0231fc31ac1a0f56979363cc6819de0b1410784f99e49105",
+                "sha256:ea278e2cf90cbbacd24f1080dd1f0ac25b71b2e21f50ab439b7ba45dd3195587"
+            ],
+            "version": "==0.9.1"
+        },
         "ipython": {
             "hashes": [
-                "sha256:55df3e0bd0f94e715abd968bedd89d4e8a7bce4bf498fb123fed4f5398fea874",
-                "sha256:b5548ec5329a4bcf054a5deed5099b0f9622eb9ea51aaa7104d215fece201d8c"
+                "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1",
+                "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd"
             ],
             "index": "pypi",
-            "version": "==7.31.1"
+            "version": "==8.4.0"
         },
         "jedi": {
             "hashes": [
@@ -913,11 +989,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:30129d870dcb0b3b6a53efdc9d0a83ea96162ffd28ffe077e94215b233dc670c",
-                "sha256:9f1cd16b1e86c2968f2519d7fb31dd9d669916f515612c269d14e9ed52b51650"
+                "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0",
+                "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.28"
+            "version": "==3.0.30"
         },
         "ptyprocess": {
             "hashes": [
@@ -926,29 +1002,51 @@
             ],
             "version": "==0.7.0"
         },
+        "pure-eval": {
+            "hashes": [
+                "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350",
+                "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"
+            ],
+            "version": "==0.2.2"
+        },
         "pygments": {
             "hashes": [
-                "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65",
-                "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a"
+                "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb",
+                "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.11.2"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.12.0"
         },
         "setuptools": {
             "hashes": [
-                "sha256:2347b2b432c891a863acadca2da9ac101eae6169b1d3dfee2ec605ecd50dbfe5",
-                "sha256:e4f30b9f84e5ab3decf945113119649fec09c1fc3507c6ebffec75646c56e62b"
+                "sha256:7c7854ee1429a240090297628dc9f75b35318d193537968e2dc14010ee2f5bca",
+                "sha256:dc2662692f47d99cb8ae15a784529adeed535bcd7c277fee0beccf961522baf6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==60.9.3"
+            "version": "==63.4.1"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.16.0"
+        },
+        "stack-data": {
+            "hashes": [
+                "sha256:77bec1402dcd0987e9022326473fdbcc767304892a533ed8c29888dacb7dddbc",
+                "sha256:aa1d52d14d09c7a9a12bb740e6bdfffe0f5e8f4f9218d85e7c73a8c37f7ae38d"
+            ],
+            "version": "==0.3.0"
         },
         "traitlets": {
             "hashes": [
-                "sha256:059f456c5a7c1c82b98c2e8c799f39c9b8128f6d0d46941ee118daace9eb70c7",
-                "sha256:2d313cc50a42cd6c277e7d7dc8d4d7fedd06a2c215f78766ae7b1a66277e0033"
+                "sha256:0bb9f1f9f017aa8ec187d8b1b2a7a6626a2a1d877116baba52a129bfa124f8e2",
+                "sha256:65fa18961659635933100db8ca120ef6220555286949774b9cfc106f941d1c7a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.1.1"
+            "version": "==5.3.0"
         },
         "wcwidth": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index adab50e16..e87b67b41 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,12 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.6.3 - 2022-08-04 - [PR #652](https://github.com/NOAA-OWP/inundation-mapping/pull/652)
+
+Updated `Dockerfile`, `Pipfile` and `Pipfile.lock` to add the new psycopg2 python package required for a WIP code fix for the new FIM4 calibration db.
+
+<br/><br/>
+
 
 ## v4.0.6.2 - 2022-08-16 - [PR #639](https://github.com/NOAA-OWP/inundation-mapping/pull/639)
 

From 89a978ffeda6c6e42c6e312b9130aac832b01246 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Wed, 17 Aug 2022 20:37:07 -0500
Subject: [PATCH 153/359] FIM4 synthetic rating curve calibration functionality
 (#657)

---
 config/deny_gms_branch_zero.lst   |   2 +-
 config/deny_gms_branches_min.lst  |   2 +-
 config/deny_gms_unit_default.lst  |   1 +
 config/params_template.env        |  12 +-
 docs/CHANGELOG.md                 |  37 +++
 gms_run_branch.sh                 |  28 +++
 src/add_crosswalk.py              |  15 ++
 src/gms/run_by_unit.sh            |  21 +-
 src/make_stages_and_catchlist.py  |  10 +-
 src/src_adjust_spatial_obs.py     | 382 ++++++++++++++++++++++++++++++
 src/src_adjust_usgs_rating.py     | 262 ++++++++++++++++++++
 src/src_roughness_optimization.py | 376 +++++++++++++++++++++++++++++
 src/usgs_gage_aggregate.py        |   2 +-
 src/usgs_gage_unit_setup.py       |  16 ++
 src/utils/shared_functions.py     |  49 +++-
 src/utils/shared_variables.py     |   5 +
 16 files changed, 1202 insertions(+), 18 deletions(-)
 create mode 100644 src/src_adjust_spatial_obs.py
 create mode 100644 src/src_adjust_usgs_rating.py
 create mode 100644 src/src_roughness_optimization.py

diff --git a/config/deny_gms_branch_zero.lst b/config/deny_gms_branch_zero.lst
index 3b80477a4..93bbf79b9 100644
--- a/config/deny_gms_branch_zero.lst
+++ b/config/deny_gms_branch_zero.lst
@@ -37,7 +37,7 @@ gw_catchments_reaches_{}.gpkg
 gw_catchments_reaches_{}.tif
 #gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 #gw_catchments_reaches_filtered_addedAttributes_{}.tif
-gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
diff --git a/config/deny_gms_branches_min.lst b/config/deny_gms_branches_min.lst
index b11c17d06..4492a170f 100644
--- a/config/deny_gms_branches_min.lst
+++ b/config/deny_gms_branches_min.lst
@@ -37,7 +37,7 @@ gw_catchments_reaches_{}.gpkg
 gw_catchments_reaches_{}.tif
 #gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 #gw_catchments_reaches_filtered_addedAttributes_{}.tif
-gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
diff --git a/config/deny_gms_unit_default.lst b/config/deny_gms_unit_default.lst
index f4f91920a..53f32b920 100644
--- a/config/deny_gms_unit_default.lst
+++ b/config/deny_gms_unit_default.lst
@@ -13,6 +13,7 @@ nwm_headwaters.gpkg
 #nwm_subset_streams_levelPaths.gpkg
 #nwm_subset_streams_levelPaths_dissolved.gpkg
 #nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+#usgs_elev_table.csv
 #usgs_subset_gages.gpkg
 #wbd.gpkg
 #wbd8_clp.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index 4d967cbbf..49db90523 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -52,6 +52,16 @@ export vrough_suffix="_vmann" # text to append to output log and src_full_crossw
 export vmann_input_file="data/inputs/rating_curve/variable_roughness/mannings_global_06_011.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
 export bankfull_attribute="chann_volume_ratio" # src_full_crosswalked_bankfull.csv attribute (column id) containing the channel vs overbank ratio values (generated in the identify_src_bankfull.py)
 
+#### apply SRC adjustments using USGS rating curve database ####
+export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; False=off)
+export nwm_recur_file="data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
+
+#### apply SRC adjustments using observed FIM/flow point database ####
+export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
+export fim_obs_pnt_data="data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
+#### path to env file with sensitive paths for accessing postgres database ####
+export CALB_DB_KEYS_FILE="/data/config/calb_db_keys.env"
+
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
@@ -60,4 +70,4 @@ export memfree=0G # min free memory required to start a new job or keep youngest
 
 #### logging parameters ####
 export startDiv="\n##########################################################################\n"
-export stopDiv="\n##########################################################################"
+export stopDiv="\n##########################################################################"
\ No newline at end of file
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e87b67b41..7afaa4cd8 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -8,6 +8,43 @@ Updated `Dockerfile`, `Pipfile` and `Pipfile.lock` to add the new psycopg2 pytho
 <br/><br/>
 
 
+## v4.0.7.0 - 2022-08-17 - [PR #657](https://github.com/NOAA-OWP/inundation-mapping/pull/657)
+
+Introduces synthetic rating curve calibration workflow. The calibration computes new Manning's coefficients for the HAND SRCs using input data: USGS gage locations, USGS rating curve csv, and a benchmark FIM extent point database stored in PostgreSQL database. This addresses [#535].
+
+## Additions
+
+- `src/src_adjust_spatial_obs.py`: new synthetic rating curve calibration routine that prepares all of the spatial (point data) benchmark data for ingest to the Manning's coefficient calculations performed in `src_roughness_optimization.py`
+- `src/src_adjust_usgs_rating.py`: new synthetic rating curve calibration routine that prepares all of the USGS gage location and observed rating curve data for ingest to the Manning's coefficient calculations performed in `src_roughness_optimization.py`
+- `src/src_roughness_optimization.py`: new SRC post-processing script that ingests observed data and HUC/branch FIM output data to compute optimized Manning's coefficient values and update the discharge values in the SRCs. Outputs a new hydroTable.csv.
+
+## Changes
+
+- `config/deny_gms_branch_zero.lst`: added `gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg` to list of files to keep (used in calibration workflow)
+- `config/deny_gms_branches_min.lst`: added `gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg` to list of files to keep (used in calibration workflow)
+- `config/deny_gms_unit_default.lst`: added `usgs_elev_table.csv` to list of files to keep (used in calibration workflow)
+- `config/params_template.env`: added new variables for user to control calibration
+  - `src_adjust_usgs`: Toggle to run src adjustment routine (True=on; False=off)
+  - `nwm_recur_file`: input file location with nwm feature_id and recurrence flow values
+  - `src_adjust_spatial`: Toggle to run src adjustment routine (True=on; False=off)
+  - `fim_obs_pnt_data`: input file location with benchmark point data used to populate the postgresql database
+  - `CALB_DB_KEYS_FILE`: path to env file with sensitive paths for accessing postgres database
+- `gms_run_branch.sh`: includes new steps in the workflow to connect to the calibration PostgreSQL database, run SRC calibration w/ USGS gage rating curves, run SRC calibration w/ benchmark point database
+- `src/add_crosswalk.py`: added step to create placeholder variables to be replaced in post-processing (as needed). Created here to ensure consistent column variables in the final hydrotable.csv
+- `src/gms/run_by_unit.sh`: added new steps to workflow to create the `usgs_subset_gages.gpkg` file for branch zero and then perform crosswalk and create `usgs_elev_table.csv` for branch zero
+- `src/make_stages_and_catchlist.py`: Reconcile flows and catchments hydroids
+- `src/usgs_gage_aggregate.py`: changed streamorder data type from integer to string to better handle missing values in `usgs_gage_unit_setup.py`
+- `src/usgs_gage_unit_setup.py`: added new inputs and function to populate `usgs_elev_table.csv` for branch zero using all available gages within the huc (not filtering to a specific branch)
+- `src/utils/shared_functions.py`: added two new functions for calibration workflow
+  - `check_file_age`: check the age of a file (use for flagging potentially outdated input)
+  - `concat_huc_csv`: concatenate huc csv files to a single dataframe/csv
+- `src/utils/shared_variables.py`: defined new SRC calibration threshold variables
+  - `DOWNSTREAM_THRESHOLD`: distance in km to propogate new roughness values downstream
+  - `ROUGHNESS_MAX_THRESH`: max allowable adjusted roughness value (void values larger than this)
+  - `ROUGHNESS_MIN_THRESH`: min allowable adjusted roughness value (void values smaller than this)
+
+<br/><br/>
+
 ## v4.0.6.2 - 2022-08-16 - [PR #639](https://github.com/NOAA-OWP/inundation-mapping/pull/639)
 
 This file converts USFIMR remote sensed inundation shapefiles into a raster that can be used to compare to the FIM data. It has to be run separately for each shapefile. This addresses [#629].
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 057e3affc..ecdcb026c 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -113,6 +113,21 @@ fi
 source $envFile
 source $srcDir/bash_functions.env
 
+## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
+if [ "$src_adjust_spatial" = "True" ]; then
+    if [ ! -f $CALB_DB_KEYS_FILE ]; then
+        echo "ERROR! - src_adjust_spatial parameter is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        exit 1
+    else
+        source $CALB_DB_KEYS_FILE
+        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes"
+        echo "Loading HUC Data"
+        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $input_WBD_gdb WBDHU8
+        echo "Loading Point Data"
+        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+    fi
+fi
+
 # default values
 if [ "$jobLimit" = "" ] ; then
     jobLimit=$default_max_jobs
@@ -187,6 +202,19 @@ echo
 echo "Processing usgs gage aggregation"
 python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
 
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
+if [ "$src_adjust_usgs" = "True" ]; then
+    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"$stopDiv
+    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
+    time python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+fi
+
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
+if [ "$src_adjust_spatial" = "True" ]; then
+    echo -e $startDiv"Performing SRC adjustments using benchmark point database"$stopDiv
+    time python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
+fi
+
 # -------------------
 ## GET NON ZERO EXIT CODES ##
 # Needed in case aggregation fails, we will need the logs
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 37ee0a25d..fe64d0f50 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -240,8 +240,23 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     # make hydroTable
     output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)','BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)','Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
+    ## Set placeholder variables to be replaced in post-processing (as needed). Create here to ensure consistent column vars
     output_hydro_table['barc_on'] = False # set barc_on attribute to Fasle (default) --> will be overwritten if BARC module runs
+    output_hydro_table['raw_discharge_cms'] = output_src['Discharge (m3s-1)']
+    output_hydro_table['raw_Volume (m3)'] = output_src['Volume (m3)']
+    output_hydro_table['raw_WetArea (m2)'] = output_src['WetArea (m2)']
+    output_hydro_table['raw_HydraulicRadius (m)'] = output_src['HydraulicRadius (m)']
     output_hydro_table['vmann_on'] = False # set vmann_on attribute to Fasle (default) --> will be overwritten if variable roughness module runs
+    output_hydro_table['raw_ManningN'] = output_src['ManningN']
+    output_hydro_table['vmann_discharge_cms'] = pd.NA
+    output_hydro_table['vmann_ManningN'] = pd.NA
+    output_hydro_table['adjust_src_on'] = False
+    output_hydro_table['last_updated'] = pd.NA
+    output_hydro_table['submitter'] = pd.NA
+    output_hydro_table['adjust_ManningN'] = pd.NA
+    output_hydro_table['obs_source'] = pd.NA
+    output_hydro_table['default_discharge_cms'] = pd.NA
+    output_hydro_table['default_ManningN'] = pd.NA
 
 
     if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 67213acab..c513e8b3d 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -214,18 +214,27 @@ else
     echo -e $startDiv"Skipping branch zero processing because there are no stream orders being dropped $hucNumber"$stopDiv
 fi
 
-## CLEANUP BRANCH ZERO OUTPUTS ##
-echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
+## CREATE USGS GAGES FILE
+echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -v -b
+python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id -bzero $dropLowStreamOrders
 Tcount
 
-## CREATE USGS GAGES FILE
-echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
+## USGS CROSSWALK ##
+if [ -f $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
+    echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"$stopDiv
+    date -u
+    Tstart
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $branch_zero_id
+    Tcount
+fi
+
+## CLEANUP BRANCH ZERO OUTPUTS ##
+echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg
+$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -v -b
 Tcount
 
 ## REMOVE FILES FROM DENY LIST ##
diff --git a/src/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
index b9210945d..677de1899 100755
--- a/src/make_stages_and_catchlist.py
+++ b/src/make_stages_and_catchlist.py
@@ -12,13 +12,9 @@ def make_stages_and_catchlist(flows_filename, catchments_filename, stages_filena
     flows = gpd.read_file(flows_filename)
     catchments = gpd.read_file(catchments_filename)
 
-
-    hydroIDs = flows['HydroID'].tolist()
-    len_of_hydroIDs = len(hydroIDs)
-    slopes = flows['S0'].tolist()
-    lengthkm = flows['LengthKm'].tolist()
-    areasqkm = catchments['areasqkm'].tolist()
-
+    # Reconcile flows and catchments hydroids
+    flows      = flows.merge(catchments[['HydroID']], on='HydroID', how='inner')
+    catchments = catchments.merge(flows[['HydroID']], on='HydroID', how='inner')
 
     stages_max = stages_max + stages_interval
     stages = np.round(np.arange(stages_min,stages_max,stages_interval),4)
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
new file mode 100644
index 000000000..abd77ace0
--- /dev/null
+++ b/src/src_adjust_spatial_obs.py
@@ -0,0 +1,382 @@
+import argparse
+import geopandas as gpd
+from geopandas.tools import sjoin
+import os
+import rasterio
+import pandas as pd
+import numpy as np
+import sys
+import json
+import datetime as dt
+from collections import deque
+import multiprocessing
+from multiprocessing import Pool
+from src_roughness_optimization import update_rating_curve
+import psycopg2 # python package for connecting to postgres
+from dotenv import load_dotenv
+import time
+
+#import variables from .env file
+load_dotenv()
+CALIBRATION_DB_HOST = os.getenv("CALIBRATION_DB_HOST")
+CALIBRATION_DB_USER_NAME = os.getenv("CALIBRATION_DB_USER_NAME")
+CALIBRATION_DB_PASS = os.getenv("CALIBRATION_DB_PASS")
+
+from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH
+'''
+The script imports a PostgreSQL database containing observed FIM extent points and associated flow data. This script attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow.
+
+Processing
+- Define CRS to use for initial geoprocessing and read wbd_path and points_layer.
+- Define paths to hydroTable.csv, HAND raster, catchments raster, and synthetic rating curve JSON.
+- Clip the points water_edge_df to the huc cathments polygons (for faster processing?)
+- Define coords variable to be used in point raster value attribution and use point geometry to determine catchment raster pixel values
+- Check that there are valid obs in the water_edge_df (not empty) and convert pandas series to dataframe to pass to update_rating_curve
+- Call update_rating_curve() to perform the rating curve calibration.
+
+Inputs
+- points_layer:         .gpkg layer containing observed/truth FIM extent points and associated flow value 
+- fim_directory:        fim directory containing individual HUC output dirs
+- wbd_path:             path the watershed boundary dataset layer (HUC polygon boundaries)
+- job_number:           number of multi-processing jobs to use
+- debug_outputs_option: optional flag to output intermediate files for reviewing/debugging
+
+Outputs
+- water_edge_median_df: dataframe containing "hydroid", "flow", "submitter", "coll_time", "flow_unit", "layer", and median "HAND" value
+'''
+
+def process_points(args):
+
+    '''
+    The funciton ingests geodataframe and attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow
+
+    Processing
+    - Extract x,y coordinates from geometry
+    - Projects the point data to matching CRS for HAND and hydroid rasters
+    - Samples the hydroid and HAND raster values for each point and stores the values in dataframe
+    - Calculates the median HAND value for all points by hydroid
+    '''
+
+    branch_dir = args[0]
+    huc = args[1]
+    branch_id = args[2]
+    hand_path = args[3]
+    catchments_path = args[4]
+    catchments_poly_path = args[5]
+    water_edge_df = args[6]
+    htable_path = args[7]
+    optional_outputs = args[8]
+
+    ## Define coords variable to be used in point raster value attribution.
+    coords = [(x,y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
+
+    ## Use point geometry to determine HAND raster pixel values.
+    hand_src = rasterio.open(hand_path)
+    hand_crs = hand_src.crs
+    water_edge_df.to_crs(hand_crs)  # Reproject geodataframe to match hand_src. Should be the same, but this is a double check.
+    water_edge_df['hand'] = [h[0] for h in hand_src.sample(coords)]
+    hand_src.close()
+    del hand_src, hand_crs,
+
+    ## Use point geometry to determine catchment raster pixel values.
+    catchments_src = rasterio.open(catchments_path)
+    catchments_crs = catchments_src.crs
+    water_edge_df.to_crs(catchments_crs)
+    water_edge_df['hydroid'] = [c[0] for c in catchments_src.sample(coords)]
+    catchments_src.close()
+    del catchments_src, catchments_crs
+
+    water_edge_df = water_edge_df[(water_edge_df['hydroid'].notnull()) & (water_edge_df['hand'] > 0) & (water_edge_df['hydroid'] > 0)]
+
+    ## Check that there are valid obs in the water_edge_df (not empty)
+    if water_edge_df.empty:
+        log_text = 'NOTE --> skipping HUC: ' + str(huc) + '  Branch: ' + str(branch_id) + ': no valid observation points found within the branch catchments'
+    else:
+        ## Intermediate output for debugging
+        if optional_outputs:
+            branch_debug_pts_out_gpkg = os.path.join(branch_dir, 'export_water_edge_df_' + branch_id + '.gpkg')
+            water_edge_df.to_file(branch_debug_pts_out_gpkg, driver='GPKG', index=False)
+            
+        #print('Processing points for HUC: ' + str(huc) + '  Branch: ' + str(branch_id))
+        ## Get median HAND value for appropriate groups.
+        water_edge_median_ds = water_edge_df.groupby(["hydroid", "flow", "submitter", "coll_time", "flow_unit","layer"])['hand'].median()
+
+        ## Write user_supplied_n_vals to CSV for next step.
+        pt_n_values_csv = os.path.join(branch_dir, 'user_supplied_n_vals_' + branch_id + '.csv')
+        water_edge_median_ds.to_csv(pt_n_values_csv)
+        ## Convert pandas series to dataframe to pass to update_rating_curve
+        water_edge_median_df = water_edge_median_ds.reset_index()
+        water_edge_median_df['coll_time'] = water_edge_median_df.coll_time.astype(str)
+        del water_edge_median_ds
+
+        ## Additional arguments for src_roughness_optimization
+        source_tag = 'point_obs' # tag to use in source attribute field
+        merge_prev_adj = True # merge in previous SRC adjustment calculations
+
+        ## Call update_rating_curve() to perform the rating curve calibration.
+        log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc, branch_id, catchments_poly_path, optional_outputs, source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
+        ## Still testing: use code below to print out any exceptions.
+        '''
+        try:
+            log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc, catchments_poly_path, optional_outputs, source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
+        except Exception as e:
+            print(str(huc) + ' --> ' + str(e))
+            log_text = 'ERROR!!!: HUC ' + str(huc) + ' --> ' + str(e)
+        '''
+    return(log_text)
+
+
+def find_points_in_huc(huc_id, conn):
+    # Point data in the database is already attributed with HUC8 id
+    '''
+    The function queries the PostgreSQL database and returns all points attributed with the input huc id.
+
+    Processing
+    - Query the PostgreSQL database for points attributed with huc id.
+    - Reads the filtered database result into a pandas geodataframe
+
+    Inputs
+    - conn:         connection to PostgreSQL db
+    - huc_id:       HUC id to query the db
+
+    Outputs
+    - water_edge_df: geodataframe with point data
+    '''
+
+    huc_pt_query = """SELECT ST_X(P.geom), ST_Y(P.geom), P.submitter, P.flow, P.coll_time, P.flow_unit, P.layer, P.geom 
+    FROM points P 
+    JOIN hucs H ON ST_Contains(H.geom, P.geom)
+    WHERE H.huc8 = %s """
+    
+    # Need to hard code the CRS to use EPSG:5070 instead of the default ESRI:102039 (gdal pyproj throws an error with crs 102039)
+    # Appears that EPSG:5070 is functionally equivalent to ESRI:102039: https://gis.stackexchange.com/questions/329123/crs-interpretation-in-qgis
+    water_edge_df = gpd.GeoDataFrame.from_postgis(huc_pt_query, con=conn, params=[huc_id], crs="EPSG:5070", parse_dates=['coll_time'])
+    water_edge_df = water_edge_df.drop(columns=['st_x','st_y'])
+    
+    return water_edge_df
+
+def find_hucs_with_points(conn,fim_out_huc_list):
+    '''
+    The function queries the PostgreSQL database and returns a list of all the HUCs that contain calb point data.
+
+    Processing
+    - Query the PostgreSQL database for all unique huc ids
+
+    Inputs
+    - conn:         connection to PostgreSQL db
+
+    Outputs
+    - hucs_wpoints: list with all unique huc ids
+    '''
+
+    cursor = conn.cursor()
+    '''
+    cursor.execute("""
+        SELECT DISTINCT H.huc8
+        FROM points P JOIN hucs H ON ST_Contains(H.geom, P.geom);
+    """)
+    '''
+    cursor.execute("SELECT DISTINCT H.huc8 FROM points P JOIN hucs H ON ST_Contains(H.geom, P.geom) WHERE H.huc8 = ANY(%s);", (fim_out_huc_list,))
+    hucs_fetch = cursor.fetchall() # list with tuple with the attributes defined above (need to convert to df?)
+    hucs_wpoints = []
+    for huc in hucs_fetch:
+        hucs_wpoints.append(huc[0])
+    cursor.close()
+    return hucs_wpoints
+
+def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_file):
+    '''
+    The function obtains all points within a given huc, locates the corresponding FIM output files for each huc (confirms all necessary files exist), and then passes a proc list of huc organized data to process_points function.
+
+    Processing
+    - Query the PostgreSQL database for all unique huc ids that have calb points
+    - Loop through all HUCs with calb data and locate necessary fim output files to pass to calb workflow
+
+    Inputs
+    - fim_directory:        parent directory of fim ouputs (contains HUC directories)
+    - job_number:           number of multiprocessing jobs to use for processing hucs
+    - debug_outputs_option: optional flag to output intermediate files
+
+    Outputs
+    - procs_list:           passes multiprocessing list of input args for process_points function input
+    '''
+    conn = connect() # Connect to the PostgreSQL db once
+    log_file.write('Connected to database via host: ' + str(CALIBRATION_DB_HOST) + '\n')
+    print("Finding all fim_output hucs that contain calibration points...")
+    fim_out_huc_list  = [ item for item in os.listdir(fim_directory) if os.path.isdir(os.path.join(fim_directory, item)) ]
+    fim_out_huc_list.remove('logs')
+    ## Record run time and close log file
+    run_time_start = dt.datetime.now()
+    log_file.write('Finding all hucs that contain calibration points...' + '\n')
+    huc_list_db = find_hucs_with_points(conn,fim_out_huc_list)
+    run_time_end = dt.datetime.now()
+    task_run_time = run_time_end - run_time_start
+    log_file.write('HUC SEARCH TASK RUN TIME: ' + str(task_run_time) + '\n')
+    print(f"{len(huc_list_db)} hucs found in point database" + '\n')
+    log_file.write(f"{len(huc_list_db)} hucs found in point database" + '\n')
+    log_file.write('#########################################################\n')
+
+    ## Ensure HUC id is either HUC8
+    huc_list = []
+    for huc in huc_list_db:
+        ## zfill to the appropriate scale to ensure leading zeros are present, if necessary.
+        if len(huc) == 7:
+            huc = huc.zfill(8)
+        if huc not in huc_list:
+            huc_list.append(huc)
+            log_file.write(str(huc) + '\n')
+
+    procs_list = []  # Initialize proc list for mulitprocessing.
+
+    #huc_list = ['12040103']
+    ## Define paths to relevant HUC HAND data.
+    for huc in huc_list:
+        huc_branches_dir = os.path.join(fim_directory, huc,'branches')
+        water_edge_df = find_points_in_huc(huc, conn).reset_index()
+        print(f"{len(water_edge_df)} points found in " + str(huc))
+        log_file.write(f"{len(water_edge_df)} points found in " + str(huc) + '\n')
+
+        ## Create X and Y location columns by extracting from geometry.
+        water_edge_df['X'] = water_edge_df['geom'].x
+        water_edge_df['Y'] = water_edge_df['geom'].y
+
+        ## Check to make sure the HUC directory exists in the current fim_directory
+        if not os.path.exists(os.path.join(fim_directory, huc)):
+            log_file.write("FIM Directory for huc: " + str(huc) + " does not exist --> skipping SRC adjustments for this HUC (obs points found)\n")
+
+        ## Intermediate output for debugging
+        if debug_outputs_option:
+            huc_debug_pts_out = os.path.join(fim_directory, huc, 'debug_water_edge_df_' + huc + '.csv')
+            water_edge_df.to_csv(huc_debug_pts_out)
+            huc_debug_pts_out_gpkg = os.path.join(fim_directory, huc, 'export_water_edge_df_' + huc + '.gpkg')
+            water_edge_df.to_file(huc_debug_pts_out_gpkg, driver='GPKG', index=False)
+        
+        for branch_id in os.listdir(huc_branches_dir):
+            branch_dir = os.path.join(huc_branches_dir,branch_id)
+            ## Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
+            hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
+            catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
+            htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
+            catchments_poly_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
+
+            # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
+            if not os.path.exists(hand_path):
+                print("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+            elif not os.path.exists(catchments_path):
+                print("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')        
+            elif not os.path.exists(htable_path):
+                print("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+            else:
+                procs_list.append([branch_dir, huc, branch_id, hand_path, catchments_path, catchments_poly_path, water_edge_df, htable_path, debug_outputs_option])
+
+    with Pool(processes=job_number) as pool:
+                log_output = pool.map(process_points, procs_list)
+                log_file.writelines(["%s\n" % item  for item in log_output])
+    log_file.write('#########################################################\n')
+    disconnect(conn) # move this to happen at the end of the huc looping
+
+def connect():
+    """ Connect to the PostgreSQL database server """
+
+    print('Connecting to the PostgreSQL database...')
+    conn = None
+    not_connected = True
+    while not_connected:
+        try:
+
+            # connect to the PostgreSQL server
+            conn = psycopg2.connect(
+                host=CALIBRATION_DB_HOST,
+                database="calibration",
+                user=CALIBRATION_DB_USER_NAME,
+                password=CALIBRATION_DB_PASS)
+
+            # create a cursor
+            cur = conn.cursor()
+
+            # execute a statement
+            print('Host name: ' + CALIBRATION_DB_HOST)
+            print('PostgreSQL database version:')
+            cur.execute('SELECT version()')
+
+            # display the PostgreSQL database server version
+            db_version = cur.fetchone()
+            print(db_version)
+
+               # close the communication with the PostgreSQL
+            cur.close()
+            not_connected = False
+            print("Connected to database\n\n")
+        except (Exception, psycopg2.DatabaseError) as error:
+            print("Waiting for database to come online")
+            time.sleep(5)
+
+    return conn
+
+def disconnect(conn):
+    """ Disconnect from the PostgreSQL database server """
+
+    if conn is not None:
+        conn.close()
+        print('Database connection closed.')
+
+def run_prep(fim_directory,debug_outputs_option,ds_thresh_override,DOWNSTREAM_THRESHOLD,job_number):
+    assert os.path.isdir(fim_directory), 'ERROR: could not find the input fim_dir location: ' + str(fim_directory)
+
+    available_cores = multiprocessing.cpu_count()
+    if job_number > available_cores:
+        job_number = available_cores - 1
+        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+
+    if ds_thresh_override != DOWNSTREAM_THRESHOLD:
+        print('ALERT!! - Using a downstream distance threshold value (' + str(float(ds_thresh_override)) + 'km) different than the default (' + str(DOWNSTREAM_THRESHOLD) + 'km) - interpret results accordingly')
+        DOWNSTREAM_THRESHOLD = float(ds_thresh_override)
+
+    ## Create output dir for log file
+    output_dir = os.path.join(fim_directory,"logs","src_optimization")
+    if not os.path.isdir(output_dir):
+        os.makedirs(output_dir)
+        
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+
+    ## Create log file for processing records
+    print('This may take a few minutes...')
+    sys.__stdout__ = sys.stdout
+    log_file = open(os.path.join(output_dir,'log_spatial_src_adjust.log'),"w")
+    log_file.write('#########################################################\n')
+    log_file.write('Parameter Values:\n' + 'DOWNSTREAM_THRESHOLD= ' + str(DOWNSTREAM_THRESHOLD) + '\n' + 'ROUGHNESS_MIN_THRESH= ' + str( ROUGHNESS_MIN_THRESH) + '\n' + 'ROUGHNESS_MAX_THRESH=' + str(ROUGHNESS_MAX_THRESH) + '\n')
+    log_file.write('#########################################################\n\n')
+    log_file.write('START TIME: ' + str(begin_time) + '\n')
+
+    ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_file)
+
+    ## Record run time and close log file
+    end_time = dt.datetime.now()
+    log_file.write('END TIME: ' + str(end_time) + '\n')
+    tot_run_time = end_time - begin_time
+    log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+    sys.stdout = sys.__stdout__
+    log_file.close()
+
+if __name__ == '__main__':
+    ## Parse arguments.
+    parser = argparse.ArgumentParser(description='Adjusts rating curve given a shapefile containing points of known water boundary.')
+    #parser.add_argument('-db','--points-layer',help='Path to points layer containing known water boundary locations',required=True)
+    parser.add_argument('-fim_dir','--fim-directory',help='Parent directory of FIM-required datasets.',required=True)
+    parser.add_argument('-debug','--extra-outputs',help='OPTIONAL flag: Use this to keep intermediate output files for debugging/testing',default=False,required=False, action='store_true')
+    parser.add_argument('-dthresh','--downstream-thresh',help='OPTIONAL Override: distance in km to propogate modified roughness values downstream', default=DOWNSTREAM_THRESHOLD,required=False)
+    parser.add_argument('-j','--job-number',help='OPTIONAL: Number of jobs to use',required=False,default=2)
+
+    ## Assign variables from arguments.
+    args = vars(parser.parse_args())
+    #points_layer = args['points_layer']
+    fim_directory = args['fim_directory']
+    debug_outputs_option = args['extra_outputs']
+    ds_thresh_override = args['downstream_thresh']
+    job_number = int(args['job_number'])
+
+    run_prep(fim_directory,debug_outputs_option,ds_thresh_override,DOWNSTREAM_THRESHOLD,job_number)
\ No newline at end of file
diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py
new file mode 100644
index 000000000..d0df1f5e3
--- /dev/null
+++ b/src/src_adjust_usgs_rating.py
@@ -0,0 +1,262 @@
+import argparse
+import os
+import pandas as pd
+import sys
+import json
+import datetime as dt
+from pathlib import Path
+from collections import deque
+import multiprocessing
+from multiprocessing import Pool
+from utils.shared_functions import check_file_age, concat_huc_csv
+from src_roughness_optimization import update_rating_curve
+'''
+The script ingests a USGS rating curve csv and a NWM flow recurrence interval database. The gage location will be associated to the corresponding hydroID and attributed with the HAND elevation value
+
+Processing
+- Read in USGS rating curve from csv and convert WSE navd88 values to meters
+- Read in the aggregate USGS elev table csv from the HUC fim directory (output from usgs_gage_crosswalk.py)
+- Filter null entries and convert usgs flow from cfs to cms
+- Calculate HAND elevation value for each gage location (NAVD88 elevation - NHD DEM thalweg elevation)
+- Read in the NWM recurr csv file and convert flow to cfs
+- Calculate the closest SRC discharge value to the NWM flow value
+- Create dataframe with crosswalked USGS flow and NWM recurr flow and assign metadata attributes
+- Calculate flow difference (variance) to check for large discrepancies btw NWM flow and USGS closest flow
+- Log any signifant differences (or negative HAND values) btw the NWM flow value and closest USGS rating flow
+- Produce log file
+- Call update_rating_curve() to perform the rating curve calibration.
+
+Inputs
+- branch_dir:        fim directory containing individual HUC output dirs
+- usgs_rc_filepath:     USGS rating curve database (produced by rating_curve_get_usgs_curves.py)
+- nwm_recurr_filepath:  NWM flow recurrence interval dataset
+- debug_outputs_option: optional flag to output intermediate files for reviewing/debugging
+- job_number:           number of multi-processing jobs to use
+
+Outputs
+- water_edge_median_ds: dataframe containing 'location_id','hydroid','feature_id','huc','hand','discharge_cms','nwm_recur_flow_cms','nwm_recur','layer'
+'''
+
+def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir):
+    start_time = dt.datetime.now()
+    print('Reading USGS rating curve from csv...')
+    log_text = 'Processing database for USGS flow/WSE at NWM flow recur intervals...\n'
+    col_usgs = ["location_id", "flow", "stage", "elevation_navd88"]
+    usgs_rc_df = pd.read_csv(usgs_rc_filepath, dtype={'location_id': object}, usecols=col_usgs)#, nrows=30000)
+    print('Duration (read usgs_rc_csv): {}'.format(dt.datetime.now() - start_time))
+    
+    # convert WSE navd88 values to meters
+    usgs_rc_df['elevation_navd88_m'] = usgs_rc_df['elevation_navd88'] / 3.28084
+    
+    # read in the aggregate USGS elev table csv
+    start_time = dt.datetime.now()
+    cross_df = usgs_elev_df[["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_adj_elevation"]].copy()
+    cross_df.rename(columns={'dem_adj_elevation':'hand_datum', 'HydroID':'hydroid', 'HUC8':'huc'}, inplace=True)
+    
+    # filter null location_id rows from cross_df (removes ahps lide entries that aren't associated with USGS gage)
+    cross_df = cross_df[cross_df.location_id.notnull()]
+    
+    # convert usgs flow from cfs to cms
+    usgs_rc_df['discharge_cms'] = usgs_rc_df.flow / 35.3147
+    usgs_rc_df = usgs_rc_df.drop(columns=["flow"])
+    
+    # merge usgs ratings with crosswalk attributes
+    usgs_rc_df = usgs_rc_df.merge(cross_df, how='left', on='location_id')
+    usgs_rc_df = usgs_rc_df[usgs_rc_df['hydroid'].notna()]
+    
+    # calculate hand elevation
+    usgs_rc_df['hand'] = usgs_rc_df['elevation_navd88_m'] - usgs_rc_df['hand_datum']
+    usgs_rc_df = usgs_rc_df[['location_id','feature_id','hydroid','levpa_id','huc','hand','discharge_cms']]
+    usgs_rc_df['feature_id'] = usgs_rc_df['feature_id'].astype(int)
+    
+    # read in the NWM recurr csv file
+    nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
+    nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
+    nwm_recur_df.rename(columns={'2_0_year_recurrence_flow_17C':'2_0_year','5_0_year_recurrence_flow_17C':'5_0_year','10_0_year_recurrence_flow_17C':'10_0_year','25_0_year_recurrence_flow_17C':'25_0_year','50_0_year_recurrence_flow_17C':'50_0_year','100_0_year_recurrence_flow_17C':'100_0_year'}, inplace=True)
+    
+    #convert cfs to cms (x 0.028317)
+    nwm_recur_df.loc[:, ['2_0_year','5_0_year','10_0_year','25_0_year','50_0_year','100_0_year']] *= 0.028317
+    
+    # merge nwm recurr with usgs_rc
+    merge_df = usgs_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
+    
+    # NWM recurr intervals
+    recurr_intervals = ("2","5","10","25","50","100")
+    final_df = pd.DataFrame() # create empty dataframe to append flow interval dataframes
+    for interval in recurr_intervals:
+        log_text += ('\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n')
+        print('Processing: ' + str(interval) + '-year NWM recurr intervals')
+        ## Calculate the closest SRC discharge value to the NWM flow value
+        merge_df['Q_find'] = (merge_df['discharge_cms'] - merge_df[interval+"_0_year"]).abs()
+        
+        ## Check for any missing/null entries in the input SRC
+        if merge_df['Q_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
+            log_text += 'HUC: ' + str(merge_df['huc']) + ' : feature_id' + str(merge_df['feature_id']) + ' --> Null values found in "Q_find" calc. These will be filled with 999999 () \n'
+            ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
+            merge_df['Q_find'] = merge_df['Q_find'].fillna(999999)
+        if merge_df['hydroid'].isnull().values.any():
+            log_text += 'HUC: ' + str(merge_df['huc']) + ' --> Null values found in "hydroid"... \n'
+        
+        # Create dataframe with crosswalked USGS flow and NWM recurr flow
+        calc_df = merge_df.loc[merge_df.groupby(['location_id','levpa_id'])['Q_find'].idxmin()].reset_index(drop=True) # find the index of the Q_1_5_find (closest matching flow)
+        # Calculate flow difference (variance) to check for large discrepancies btw NWM flow and USGS closest flow
+        calc_df['check_variance'] = ((calc_df['discharge_cms'] - calc_df[interval+"_0_year"])/calc_df['discharge_cms']).abs()
+        # Assign new metadata attributes
+        calc_df['nwm_recur'] = interval+"_0_year"
+        calc_df['layer'] = '_usgs-gage____' + interval+"-year"
+        calc_df.rename(columns={interval+"_0_year":'nwm_recur_flow_cms'}, inplace=True)
+        # Subset calc_df for final output
+        calc_df = calc_df[['location_id','hydroid','feature_id','levpa_id','huc','hand','discharge_cms','check_variance','nwm_recur_flow_cms','nwm_recur','layer']]
+        final_df = final_df.append(calc_df, ignore_index=True)
+        # Log any negative HAND elev values and remove from database
+        log_text += ('Warning: Negative HAND stage values -->\n')
+        log_text += (calc_df[calc_df['hand']<0].to_string() +'\n')
+        final_df = final_df[final_df['hand']>0]
+        # Log any signifant differences btw the NWM flow value and closest USGS rating flow (this ensures that we consistently sample the USGS rating curves at known intervals - NWM recur flow)
+        log_text += ('Warning: Large variance (>10%) between NWM flow and closest USGS flow -->\n')
+        log_text += (calc_df[calc_df['check_variance']>0.1].to_string() +'\n')
+        final_df = final_df[final_df['check_variance']<0.1]
+        final_df['submitter'] = 'usgs_rating_wrds_api_' + final_df['location_id']
+        # Get datestamp from usgs rating curve file to use as coll_time attribute in hydroTable.csv
+        datestamp = check_file_age(usgs_rc_filepath)
+        final_df['coll_time'] = str(datestamp)[:15]
+
+    # Rename attributes (for ingest to update_rating_curve) and output csv with the USGS RC database
+    final_df.rename(columns={'discharge_cms':'flow'}, inplace=True)  
+    final_df.to_csv(os.path.join(log_dir,"usgs_rc_nwm_recurr.csv"),index=False)
+
+    # Output log text to log file
+    log_text += ('#########\nTotal entries per USGS gage location -->\n')
+    loc_id_df = final_df.groupby(['location_id']).size().reset_index(name='count') 
+    log_text += (loc_id_df.to_string() +'\n')
+    log_text += ('#########\nTotal entries per NWM recur value -->\n')
+    recur_count_df = final_df.groupby(['nwm_recur']).size().reset_index(name='count') 
+    log_text += (recur_count_df.to_string() +'\n')
+    log_usgs_db = open(os.path.join(log_dir,'log_usgs_rc_database.log'),"w")
+    log_usgs_db.write(log_text)
+    log_usgs_db.close()
+    return(final_df)
+
+def branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file):
+    huc_list = usgs_df['huc'].tolist()
+    huc_list = list(set(huc_list))
+    procs_list = []  # Initialize list for mulitprocessing.
+
+    # loop through all unique level paths that have a USGS gage
+    #branch_huc_dict = pd.Series(usgs_df.levpa_id.values,index=usgs_df.huc).to_dict('list')
+    #branch_huc_dict = usgs_df.set_index('huc').T.to_dict('list')
+    huc_branch_dict = usgs_df.groupby('huc')['levpa_id'].apply(set).to_dict()
+
+    for huc in huc_branch_dict:
+        branch_set = huc_branch_dict[huc]
+        for branch_id in branch_set: 
+            # Define paths to branch HAND data.
+            # Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
+            # Assumes outputs are for HUC8 (not HUC6)
+            branch_dir = os.path.join(run_dir,huc,'branches',branch_id)
+            hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
+            catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
+            catchments_poly_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
+            htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
+            water_edge_median_ds = usgs_df[(usgs_df['huc']==huc) & (usgs_df['levpa_id']==branch_id)]
+
+            # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
+            if not os.path.exists(hand_path):
+                print("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+            elif not os.path.exists(catchments_path):
+                print("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')        
+            elif not os.path.exists(htable_path):
+                print("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+            else:
+                ## Additional arguments for src_roughness_optimization
+                source_tag = 'usgs_rating' # tag to use in source attribute field
+                merge_prev_adj = False # merge in previous SRC adjustment calculations
+
+                print('Will perform SRC adjustments for huc: ' + str(huc) + ' - branch-id: ' + str(branch_id))
+                procs_list.append([branch_dir, water_edge_median_ds, htable_path, huc, branch_id, catchments_poly_path, debug_outputs_option, source_tag, merge_prev_adj])
+
+    # multiprocess all available branches
+    print(f"Calculating new SRCs for {len(procs_list)} branches using {job_number} jobs...")
+    with Pool(processes=job_number) as pool:
+            log_output = pool.starmap(update_rating_curve, procs_list)
+            log_file.writelines(["%s\n" % item  for item in log_output])
+    # try statement for debugging 
+    # try:
+    #     with Pool(processes=job_number) as pool:
+    #         log_output = pool.starmap(update_rating_curve, procs_list)
+    #         log_file.writelines(["%s\n" % item  for item in log_output])
+    # except Exception as e:
+    #     print(str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e))
+    #     log_file.write('ERROR!!!: HUC ' + str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e) + '\n')
+
+def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number):
+    ## Check input args are valid
+    assert os.path.isdir(run_dir), 'ERROR: could not find the input fim_dir location: ' + str(run_dir)
+
+    ## Create an aggregate dataframe with all usgs_elev_table.csv entries for hucs in fim_dir
+    print('Reading USGS gage HAND elevation from usgs_elev_table.csv files...')
+    #usgs_elev_file = os.path.join(branch_dir,'usgs_elev_table.csv')
+    #usgs_elev_df = pd.read_csv(usgs_elev_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int})
+    csv_name = 'usgs_elev_table.csv'
+    usgs_elev_df = concat_huc_csv(run_dir,csv_name)
+    if usgs_elev_df is None:
+        print('WARNING: usgs_elev_df not created - check that usgs_elev_table.csv files exist in fim_dir!')
+    elif usgs_elev_df.empty:
+        print('WARNING: usgs_elev_df is empty - check that usgs_elev_table.csv files exist in fim_dir!')
+    
+    available_cores = multiprocessing.cpu_count()
+    if job_number > available_cores:
+        job_number = available_cores - 1
+        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+
+    ## Create output dir for log and usgs rc database
+    log_dir = os.path.join(run_dir,"logs","src_optimization")
+    print("Log file output here: " + str(log_dir))
+    if not os.path.isdir(log_dir):
+        os.makedirs(log_dir)
+
+    print('This may take a few minutes...')
+    usgs_df = create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir)
+
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+    # Create log file for processing records
+    log_file = open(os.path.join(log_dir,'log_usgs_rc_src_adjust.log'),"w")
+    log_file.write('START TIME: ' + str(begin_time) + '\n')
+    log_file.write('#########################################################\n\n')
+
+    ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
+    branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file)
+
+    ## Record run time and close log file
+    log_file.write('#########################################################\n\n')
+    end_time = dt.datetime.now()
+    log_file.write('END TIME: ' + str(end_time) + '\n')
+    tot_run_time = end_time - begin_time
+    log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+    sys.stdout = sys.__stdout__
+    log_file.close()
+
+if __name__ == '__main__':
+    ## Parse arguments.
+    parser = argparse.ArgumentParser(description='Adjusts rating curve with database of USGS rating curve (calculated WSE/flow).')
+    parser.add_argument('-run_dir','--run-dir',help='Parent directory of FIM run.',required=True)
+    parser.add_argument('-usgs_rc','--usgs-ratings',help='Path to USGS rating curve csv file',required=True)
+    parser.add_argument('-nwm_recur','--nwm_recur',help='Path to NWM recur file (multiple NWM flow intervals). NOTE: assumes flow units are cfs!!',required=True)
+    parser.add_argument('-debug','--extra-outputs',help='Optional flag: Use this to keep intermediate output files for debugging/testing',default=False,required=False, action='store_true')
+    parser.add_argument('-j','--job-number',help='Number of jobs to use',required=False,default=1)
+
+    ## Assign variables from arguments.
+    args = vars(parser.parse_args())
+    run_dir = args['run_dir']
+    usgs_rc_filepath = args['usgs_ratings']
+    nwm_recurr_filepath = args['nwm_recur']
+    debug_outputs_option = args['extra_outputs']
+    job_number = int(args['job_number'])
+
+    ## Prepare/check inputs, create log file, and spin up the proc list
+    run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number)
+    
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
new file mode 100644
index 000000000..d0607de08
--- /dev/null
+++ b/src/src_roughness_optimization.py
@@ -0,0 +1,376 @@
+import argparse
+import geopandas as gpd
+from geopandas.tools import sjoin
+import os
+import rasterio
+import pandas as pd
+import numpy as np
+import sys
+import json
+import datetime as dt
+from collections import deque
+import multiprocessing
+from multiprocessing import Pool
+
+from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH
+
+
+def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, branch_id, catchments_poly_path, debug_outputs_option, source_tag, merge_prev_adj=False, down_dist_thresh=DOWNSTREAM_THRESHOLD):
+    '''
+    This script ingests a dataframe containing observed data (HAND elevation and flow) and then calculates new SRC roughness values via Manning's equation. The new roughness values are averaged for each HydroID and then progated downstream and a new discharge value is calculated where applicable.
+
+    Processing Steps:
+    - Read in the hydroTable.csv and check whether it has previously been updated (rename default columns if needed)
+    - Loop through the user provided point data --> stage/flow dataframe row by row and copy the corresponding htable values for the matching stage->HAND lookup
+    - Calculate new HydroID roughness values for input obs data using Manning's equation
+    - Create dataframe to check for erroneous Manning's n values (values set in tools_shared_variables.py: >0.6 or <0.001 --> see input args)
+    - Create magnitude and ahps column by subsetting the "layer" attribute
+    - Create df grouped by hydroid with ahps_lid and huc number and then pivot the magnitude column to display n value for each magnitude at each hydroid
+    - Create df with the most recent collection time entry and submitter attribs
+    - Cacluate median ManningN to handle cases with multiple hydroid entries and create a df with the median hydroid_ManningN value per feature_id
+    - Rename the original hydrotable variables to allow new calculations to use the primary var name 
+    - Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a singel hydroid)
+    - Create attributes to traverse the flow network between HydroIDs
+    - Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg)
+    - Create the adjust_ManningN column by combining the hydroid_ManningN with the featid_ManningN (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
+    - Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new adjusted roughness value
+    - Update the catchments polygon .gpkg with joined attribute - "src_calibrated"
+    - Merge the final ManningN dataframe to the original hydroTable
+    - Create the ManningN column by combining the hydroid_ManningN with the default_ManningN (use modified where available)
+    - Calculate new discharge_cms with new adjusted ManningN
+    - Export a new hydroTable.csv and overwrite the previous version and output new src json (overwrite previous)
+
+    Inputs:
+    - fim_directory:        fim directory containing individual HUC output dirs
+    - water_edge_median_df: dataframe containing observation data (attributes: "hydroid", "flow", "submitter", "coll_time", "flow_unit", "layer", "HAND")
+    - htable_path:          path to the current HUC hydroTable.csv
+    - huc:                  string variable for the HUC id # (huc8 or huc6)
+    - branch_id:            string variable for the branch id
+    - catchments_poly_path: path to the current HUC catchments polygon layer .gpkg 
+    - debug_outputs_option: optional input argument to output additional intermediate data files (csv files with SRC calculations)
+    - source_tag:           input text tag used to specify the type/source of the input obs data used for the SRC adjustments (e.g. usgs_rating or point_obs)
+    - merge_prev_adj:       boolean argument to specify when to merge previous SRC adjustments vs. overwrite (default=False)
+    - down_dist_thresh:     optional input argument to override the env variable that controls the downstream distance new roughness values are applied downstream of locations with valid obs data
+
+    Ouputs:
+    - output_catchments:    same input "catchments_poly_path" .gpkg with appened attributes for SRC adjustments fields
+    - df_htable:            same input "htable_path" --> updated hydroTable.csv with new/modified attributes
+    - output_src_json:      src.json file with new SRC discharge values
+
+    '''
+    #print("Processing huc --> " + str(huc))
+    log_text = "\nProcessing huc --> " + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+    log_text += "DOWNSTREAM_THRESHOLD: " + str(down_dist_thresh) + 'km\n'
+    log_text += "Merge Previous Adj Values: " + str(merge_prev_adj) + '\n'
+    df_nvalues = water_edge_median_df.copy()
+    df_nvalues = df_nvalues[ (df_nvalues.hydroid.notnull()) & (df_nvalues.hydroid > 0) ] # remove null entries that do not have a valid hydroid
+
+    ## Read in the hydroTable.csv and check wether it has previously been updated (rename default columns if needed)
+    df_htable = pd.read_csv(htable_path, dtype={'HUC': object, 'last_updated':object, 'submitter':object, 'obs_source':object})
+    df_prev_adj = pd.DataFrame() # initialize empty df for populating/checking later
+    if 'default_discharge_cms' not in df_htable.columns: # need this column to exist before continuing
+        df_htable['adjust_src_on'] = False
+        df_htable['last_updated'] = pd.NA
+        df_htable['submitter'] = pd.NA
+        df_htable['adjust_ManningN'] = pd.NA
+        df_htable['obs_source'] = pd.NA
+        df_htable['default_discharge_cms'] = pd.NA
+        df_htable['default_ManningN'] = pd.NA
+    if df_htable['default_discharge_cms'].isnull().values.any(): # check if there are not valid values in the column (True = no previous calibration outputs)
+        df_htable['default_discharge_cms'] = df_htable['discharge_cms'].values
+        df_htable['default_ManningN'] = df_htable['ManningN'].values
+
+    if merge_prev_adj and not df_htable['adjust_ManningN'].isnull().all(): # check if the merge_prev_adj setting is True and there are valid 'adjust_ManningN' values from previous calibration outputs
+        # Create a subset of hydrotable with previous adjusted SRC attributes
+        df_prev_adj_htable = df_htable.copy()[['HydroID','submitter','last_updated','adjust_ManningN','obs_source']] 
+        df_prev_adj_htable.rename(columns={'submitter':'submitter_prev','last_updated':'last_updated_prev','adjust_ManningN':'adjust_ManningN_prev','obs_source':'obs_source_prev'}, inplace=True)
+        df_prev_adj_htable = df_prev_adj_htable.groupby(["HydroID"]).first()
+        # Only keep previous USGS rating curve adjustments (previous spatial obs adjustments are not retained)
+        df_prev_adj = df_prev_adj_htable[df_prev_adj_htable['obs_source_prev'].str.contains("usgs_rating", na=False)] 
+        log_text += 'HUC: ' + str(huc) + '  Branch: ' + str(branch_id) + ': found previous hydroTable calibration attributes --> retaining previous calb attributes for blending...\n'
+    # Delete previous adj columns to prevent duplicate variable issues (if src_roughness_optimization.py was previously applied)
+    df_htable.drop(['ManningN','discharge_cms','submitter','last_updated','adjust_ManningN','adjust_src_on','obs_source'], axis=1, inplace=True) 
+    df_htable.rename(columns={'default_discharge_cms':'discharge_cms','default_ManningN':'ManningN'}, inplace=True)
+
+    ## loop through the user provided point data --> stage/flow dataframe row by row
+    for index, row in df_nvalues.iterrows():
+        if row.hydroid not in df_htable['HydroID'].values:
+            print('ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid))
+            log_text += 'ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid) + '\n'
+        else:
+            df_htable_hydroid = df_htable[df_htable.HydroID == row.hydroid] # filter htable for entries with matching hydroid
+            if df_htable_hydroid.empty:
+                print('ERROR: df_htable_hydroid is empty but expected data: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid))
+                log_text += 'ERROR: df_htable_hydroid is empty but expected data: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid) + '\n'
+                
+            find_src_stage = df_htable_hydroid.loc[df_htable_hydroid['stage'].sub(row.hand).abs().idxmin()] # find closest matching stage to the user provided HAND value
+            ## copy the corresponding htable values for the matching stage->HAND lookup
+            df_nvalues.loc[index,'feature_id'] = find_src_stage.feature_id
+            df_nvalues.loc[index,'LakeID'] = find_src_stage.LakeID
+            df_nvalues.loc[index,'NextDownID'] = find_src_stage.NextDownID
+            df_nvalues.loc[index,'LENGTHKM'] = find_src_stage.LENGTHKM
+            df_nvalues.loc[index,'src_stage'] = find_src_stage.stage
+            df_nvalues.loc[index,'ManningN'] = find_src_stage.ManningN
+            df_nvalues.loc[index,'SLOPE'] = find_src_stage.SLOPE
+            df_nvalues.loc[index,'HydraulicRadius_m'] = find_src_stage['HydraulicRadius (m)']
+            df_nvalues.loc[index,'WetArea_m2'] = find_src_stage['WetArea (m2)']
+            df_nvalues.loc[index,'discharge_cms'] = find_src_stage.discharge_cms
+
+    ## mask src values that crosswalk to the SRC zero point (src_stage ~ 0 or discharge <= 0)
+    print(str(huc) + '  branch id: ' + str(branch_id))
+    df_nvalues[['HydraulicRadius_m','WetArea_m2']] = df_nvalues[['HydraulicRadius_m','WetArea_m2']].mask((df_nvalues['src_stage'] <= 0.1) | (df_nvalues['discharge_cms'] <= 0.0), np.nan)
+
+    ## Calculate roughness using Manning's equation
+    df_nvalues.rename(columns={'ManningN':'default_ManningN','hydroid':'HydroID'}, inplace=True) # rename the previous ManningN column
+    df_nvalues['hydroid_ManningN'] = df_nvalues['WetArea_m2']* \
+    pow(df_nvalues['HydraulicRadius_m'],2.0/3)* \
+    pow(df_nvalues['SLOPE'],0.5)/df_nvalues['flow']
+
+    ## Create dataframe to check for erroneous Manning's n values (values set in tools_shared_variables.py --> >0.6 or <0.001)
+    df_nvalues['Mann_flag'] = np.where((df_nvalues['hydroid_ManningN'] >= ROUGHNESS_MAX_THRESH) | (df_nvalues['hydroid_ManningN'] <= ROUGHNESS_MIN_THRESH) | (df_nvalues['hydroid_ManningN'].isnull()),'Fail','Pass')
+    df_mann_flag = df_nvalues[(df_nvalues['Mann_flag'] == 'Fail')][['HydroID','hydroid_ManningN']]
+    if not df_mann_flag.empty:
+        log_text += '!!! Flaged Mannings Roughness values below !!!' +'\n'
+        log_text += df_mann_flag.to_string() + '\n'
+
+    ## Create magnitude and ahps column by subsetting the "layer" attribute
+    df_nvalues['magnitude'] = df_nvalues['layer'].str.split("_").str[5]
+    df_nvalues['ahps_lid'] = df_nvalues['layer'].str.split("_").str[1]
+    df_nvalues['huc'] = str(huc)
+    df_nvalues.drop(['layer'], axis=1, inplace=True)
+
+    ## Create df grouped by hydroid with ahps_lid and huc number
+    df_huc_lid = df_nvalues.groupby(["HydroID"]).first()[['ahps_lid','huc']]
+    df_huc_lid.columns = pd.MultiIndex.from_product([['info'], df_huc_lid.columns])
+
+    ## pivot the magnitude column to display n value for each magnitude at each hydroid
+    df_nvalues_mag = df_nvalues.pivot_table(index='HydroID', columns='magnitude', values=['hydroid_ManningN'], aggfunc='mean') # if there are multiple entries per hydroid and magnitude - aggregate using mean
+    
+    ## Optional: Export csv with the newly calculated Manning's N values
+    if debug_outputs_option:
+        output_calc_n_csv = os.path.join(fim_directory, 'calc_src_n_vals_' + branch_id + '.csv')
+        df_nvalues.to_csv(output_calc_n_csv,index=False)
+
+    ## filter the modified Manning's n dataframe for values out side allowable range
+    df_nvalues = df_nvalues[df_nvalues['Mann_flag'] == 'Pass']
+
+    ## Check that there are valid entries in the calculate roughness df after filtering
+    if not df_nvalues.empty:
+        ## Create df with the most recent collection time entry and submitter attribs
+        df_updated = df_nvalues[['HydroID','coll_time','submitter','ahps_lid']] # subset the dataframe
+        df_updated = df_updated.sort_values('coll_time').drop_duplicates(['HydroID'],keep='last') # sort by collection time and then drop duplicate HydroIDs (keep most recent coll_time per HydroID)
+        df_updated.rename(columns={'coll_time':'last_updated'}, inplace=True)
+
+        ## cacluate median ManningN to handle cases with multiple hydroid entries
+        df_mann_hydroid = df_nvalues.groupby(["HydroID"])[['hydroid_ManningN']].median()
+
+        ## Create a df with the median hydroid_ManningN value per feature_id
+        #df_mann_featid = df_nvalues.groupby(["feature_id"])[['hydroid_ManningN']].mean()
+        #df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
+
+        ## Rename the original hydrotable variables to allow new calculations to use the primary var name
+        df_htable.rename(columns={'ManningN':'default_ManningN','discharge_cms':'default_discharge_cms'}, inplace=True)
+
+        ## Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a singel hydroid)
+        df_nrange = df_nvalues.groupby('HydroID').agg({'hydroid_ManningN': ['median', 'min', 'max', 'std', 'count']})
+        df_nrange['hydroid_ManningN','range'] = df_nrange['hydroid_ManningN','max'] - df_nrange['hydroid_ManningN','min']
+        df_nrange = df_nrange.join(df_nvalues_mag, how='outer') # join the df_nvalues_mag containing hydroid_manningn values per flood magnitude category
+        df_nrange = df_nrange.merge(df_huc_lid, how='outer', on='HydroID') # join the df_huc_lid df to add attributes for lid and huc#
+        log_text += 'Statistics for Modified Roughness Calcs -->' +'\n'
+        log_text += df_nrange.to_string() + '\n'
+        log_text += '----------------------------------------\n'
+
+        ## Optional: Output csv with SRC calc stats
+        if debug_outputs_option:
+            output_stats_n_csv = os.path.join(fim_directory, 'stats_src_n_vals_' + branch_id + '.csv')
+            df_nrange.to_csv(output_stats_n_csv,index=True)
+
+        ## subset the original hydrotable dataframe and subset to one row per HydroID
+        df_nmerge = df_htable[['HydroID','feature_id','NextDownID','LENGTHKM','LakeID','order_']].drop_duplicates(['HydroID'],keep='first') 
+
+        ## Need to check that there are non-lake hydroids in the branch hydrotable (prevents downstream error)
+        df_htable_check_lakes = df_nmerge.loc[df_nmerge['LakeID'] == -999]
+        if not df_htable_check_lakes.empty:
+
+            ## Create attributes to traverse the flow network between HydroIDs
+            df_nmerge = branch_network(df_nmerge)
+
+            ## Merge the newly caluclated ManningN dataframes
+            df_nmerge = df_nmerge.merge(df_mann_hydroid, how='left', on='HydroID')
+            df_nmerge = df_nmerge.merge(df_updated, how='left', on='HydroID')
+
+            ## Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg)
+            df_nmerge = group_manningn_calc(df_nmerge, down_dist_thresh)
+
+            ## Create a df with the median hydroid_ManningN value per feature_id
+            df_mann_featid = df_nmerge.groupby(["feature_id"])[['hydroid_ManningN']].mean()
+            df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
+            df_mann_featid_attrib = df_nmerge.groupby('feature_id').first() # create a seperate df with attributes to apply to other hydroids that share a featureid
+            df_mann_featid_attrib = df_mann_featid_attrib[df_mann_featid_attrib['submitter'].notna()][['last_updated','submitter']]
+            df_nmerge = df_nmerge.merge(df_mann_featid, how='left', on='feature_id').set_index('feature_id')
+            df_nmerge = df_nmerge.combine_first(df_mann_featid_attrib).reset_index()
+            
+            if not df_nmerge['hydroid_ManningN'].isnull().all():
+                ## Temp testing filter to only use the hydroid manning n values (drop the featureid and group ManningN variables)
+                #df_nmerge = df_nmerge.assign(featid_ManningN=np.nan)
+                #df_nmerge = df_nmerge.assign(group_ManningN=np.nan)
+
+                ## Create the adjust_ManningN column by combining the hydroid_ManningN with the featid_ManningN (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
+                conditions  = [ (df_nmerge['hydroid_ManningN'].isnull()) & (df_nmerge['featid_ManningN'].notnull()), (df_nmerge['hydroid_ManningN'].isnull()) & (df_nmerge['featid_ManningN'].isnull()) & (df_nmerge['group_ManningN'].notnull()) ]
+                choices     = [ df_nmerge['featid_ManningN'], df_nmerge['group_ManningN'] ]
+                df_nmerge['adjust_ManningN'] = np.select(conditions, choices, default=df_nmerge['hydroid_ManningN'])
+                df_nmerge['obs_source'] = np.where(df_nmerge['adjust_ManningN'].notnull(), source_tag, pd.NA)
+                df_nmerge.drop(['feature_id','NextDownID','LENGTHKM','LakeID','order_'], axis=1, inplace=True) # drop these columns to avoid duplicates where merging with the full hydroTable df
+
+                ## Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new adjusted roughness value
+                if not df_prev_adj.empty:
+                    df_nmerge = pd.merge(df_nmerge,df_prev_adj, on='HydroID', how='outer')
+                    df_nmerge['submitter'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['submitter_prev'],df_nmerge['submitter'])
+                    df_nmerge['last_updated'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['last_updated_prev'],df_nmerge['last_updated'])
+                    df_nmerge['obs_source'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['obs_source_prev'],df_nmerge['obs_source'])
+                    df_nmerge['adjust_ManningN'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['adjust_ManningN_prev'],df_nmerge['adjust_ManningN'])
+                    df_nmerge.drop(['submitter_prev','last_updated_prev','adjust_ManningN_prev','obs_source_prev'], axis=1, inplace=True)
+                
+                ## Update the catchments polygon .gpkg with joined attribute - "src_calibrated"
+                if os.path.isfile(catchments_poly_path):
+                    input_catchments = gpd.read_file(catchments_poly_path)
+                    ## Create new "src_calibrated" column for viz query
+                    if 'src_calibrated' in input_catchments.columns: # check if this attribute already exists and drop if needed
+                        input_catchments.drop(['src_calibrated'], axis=1, inplace=True)
+                    df_nmerge['src_calibrated'] = np.where(df_nmerge['adjust_ManningN'].notnull(), 'True', 'False')
+                    output_catchments = input_catchments.merge(df_nmerge[['HydroID','src_calibrated']], how='left', on='HydroID')
+                    output_catchments['src_calibrated'].fillna('False', inplace=True)
+                    output_catchments.to_file(catchments_poly_path,driver="GPKG",index=False) # overwrite the previous layer
+                    df_nmerge.drop(['src_calibrated'], axis=1, inplace=True)
+                ## Optional ouputs: 1) merge_n_csv csv with all of the calculated n values and 2) a catchments .gpkg with new joined attributes
+                if debug_outputs_option:
+                    output_merge_n_csv = os.path.join(fim_directory, 'merge_src_n_vals_' + branch_id + '.csv')
+                    df_nmerge.to_csv(output_merge_n_csv,index=False)
+                    ## output new catchments polygon layer with several new attributes appended
+                    if os.path.isfile(catchments_poly_path):
+                        input_catchments = gpd.read_file(catchments_poly_path)
+                        output_catchments_fileName = os.path.join(os.path.split(catchments_poly_path)[0],"gw_catchments_src_adjust_" + str(branch_id) + ".gpkg")
+                        output_catchments = input_catchments.merge(df_nmerge, how='left', on='HydroID')
+                        output_catchments.to_file(output_catchments_fileName,driver="GPKG",index=False)
+
+                ## Merge the final ManningN dataframe to the original hydroTable
+                df_nmerge.drop(['ahps_lid','start_catch','route_count','branch_id','hydroid_ManningN','featid_ManningN','group_ManningN',], axis=1, inplace=True) # drop these columns to avoid duplicates where merging with the full hydroTable df
+                df_htable = df_htable.merge(df_nmerge, how='left', on='HydroID')
+                df_htable['adjust_src_on'] = np.where(df_htable['adjust_ManningN'].notnull(), 'True', 'False') # create true/false column to clearly identify where new roughness values are applied
+
+                ## Create the ManningN column by combining the hydroid_ManningN with the default_ManningN (use modified where available)
+                df_htable['ManningN'] = np.where(df_htable['adjust_ManningN'].isnull(),df_htable['default_ManningN'],df_htable['adjust_ManningN'])
+
+                ## Calculate new discharge_cms with new adjusted ManningN
+                df_htable['discharge_cms'] = df_htable['WetArea (m2)']* \
+                pow(df_htable['HydraulicRadius (m)'],2.0/3)* \
+                pow(df_htable['SLOPE'],0.5)/df_htable['ManningN']
+
+                ## Replace discharge_cms with 0 or -999 if present in the original discharge (carried over from thalweg notch workaround in SRC post-processing)
+                df_htable['discharge_cms'].mask(df_htable['default_discharge_cms']==0.0,0.0,inplace=True)
+                df_htable['discharge_cms'].mask(df_htable['default_discharge_cms']==-999,-999,inplace=True)
+
+                ## Export a new hydroTable.csv and overwrite the previous version
+                out_htable = os.path.join(fim_directory, 'hydroTable_' + branch_id + '.csv')
+                df_htable.to_csv(out_htable,index=False)
+
+            else:
+                print('ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid hydroid roughness calculations after removing lakeid catchments from consideration')
+                log_text += 'ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid hydroid roughness calculations after removing lakeid catchments from consideration\n'
+        else:
+                print('WARNING!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> hydrotable is empty after removing lake catchments (will ignore branch)')
+                log_text += 'ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> hydrotable is empty after removing lake catchments (will ignore branch)\n'
+    else:
+        print('ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid roughness calculations - please check point data and src calculations to evaluate')
+        log_text += 'ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid roughness calculations - please check point data and src calculations to evaluate\n'
+
+    log_text += 'Completed: ' + str(huc) + ' --> branch: ' + str(branch_id) + '\n'
+    log_text += '#########################################################\n'
+    print("Completed huc: " + str(huc) + ' --> branch: ' + str(branch_id))
+    return(log_text)
+
+def branch_network(df_input_htable):
+    df_input_htable = df_input_htable.astype({'NextDownID': 'int64'}) # ensure attribute has consistent format as int
+    df_input_htable = df_input_htable.loc[df_input_htable['LakeID'] == -999] # remove all hydroids associated with lake/water body (these often have disjoined artifacts in the network)
+    df_input_htable["start_catch"] = ~df_input_htable['HydroID'].isin(df_input_htable['NextDownID']) # define start catchments as hydroids that are not found in the "NextDownID" attribute for all other hydroids
+            
+    df_input_htable.set_index('HydroID',inplace=True,drop=False) # set index to the hydroid
+    branch_heads = deque(df_input_htable[df_input_htable['start_catch'] == True]['HydroID'].tolist()) # create deque of hydroids to define start points in the while loop
+    visited = set() # create set to keep track of all hydroids that have been accounted for
+    branch_count = 0 # start branch id 
+    while branch_heads:
+        hid = branch_heads.popleft() # pull off left most hydroid from deque of start hydroids
+        Q = deque(df_input_htable[df_input_htable['HydroID'] == hid]['HydroID'].tolist()) # create a new deque that will be used to populate all relevant downstream hydroids
+        vert_count = 0; branch_count += 1
+        while Q:
+            q = Q.popleft()
+            if q not in visited:
+                df_input_htable.loc[df_input_htable.HydroID==q,'route_count'] = vert_count # assign var with flow order ranking
+                df_input_htable.loc[df_input_htable.HydroID==q,'branch_id'] = branch_count # assign var with current branch id
+                vert_count += 1
+                visited.add(q)
+                nextid = df_input_htable.loc[q,'NextDownID'] # find the id for the next downstream hydroid
+                order = df_input_htable.loc[q,'order_'] # find the streamorder for the current hydroid
+            
+                if nextid not in visited and nextid in df_input_htable.HydroID:
+                    check_confluence = (df_input_htable.NextDownID == nextid).sum() > 1 # check if the NextDownID is referenced by more than one hydroid (>1 means this is a confluence)
+                    nextorder = df_input_htable.loc[nextid,'order_'] # find the streamorder for the next downstream hydroid
+                    if nextorder > order and check_confluence == True: # check if the nextdownid streamorder is greater than the current hydroid order and the nextdownid is a confluence (more than 1 upstream hydroid draining to it)
+                        branch_heads.append(nextid) # found a terminal point in the network (append to branch_heads for second pass)
+                        continue # if above conditions are True than stop traversing downstream and move on to next starting hydroid
+                    Q.append(nextid)
+    df_input_htable.reset_index(drop=True, inplace=True) # reset index (previously using hydroid as index)
+    df_input_htable.sort_values(['branch_id','route_count'], inplace=True) # sort the dataframe by branch_id and then by route_count (need this ordered to ensure upstream to downstream ranking for each branch)
+    return(df_input_htable)
+
+def group_manningn_calc(df_nmerge, down_dist_thresh):
+    ## Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg
+    #df_nmerge.sort_values(by=['NextDownID'], inplace=True)
+    dist_accum = 0; hyid_count = 0; hyid_accum_count = 0; 
+    run_accum_mann = 0; group_ManningN = 0; branch_start = 1                                        # initialize counter and accumulation variables
+    lid_count = 0; prev_lid = 'x'
+    for index, row in df_nmerge.iterrows():                                                         # loop through the df (parse by hydroid)
+        if int(df_nmerge.loc[index,'branch_id']) != branch_start:                                   # check if start of new branch
+            dist_accum = 0; hyid_count = 0; hyid_accum_count = 0;                                   # initialize counter vars
+            run_accum_mann = 0; group_ManningN = 0                                                  # initialize counter vars
+            branch_start = int(df_nmerge.loc[index,'branch_id'])                                    # reassign the branch_start var to evaluate on next iteration
+            # use the code below to withold downstream hydroid_ManningN values (use this for downstream evaluation tests)
+            '''
+            lid_count = 0                                                                         
+        if not pd.isna(df_nmerge.loc[index,'ahps_lid']):
+            if df_nmerge.loc[index,'ahps_lid'] == prev_lid:
+                lid_count += 1
+                if lid_count > 3: # only keep the first 3 HydroID n values (everything else set to null for downstream application)
+                    df_nmerge.loc[index,'hydroid_ManningN'] = np.nan
+                    df_nmerge.loc[index,'featid_ManningN'] = np.nan
+            else:
+                lid_count = 1
+            prev_lid = df_nmerge.loc[index,'ahps_lid']
+            '''
+        if np.isnan(df_nmerge.loc[index,'hydroid_ManningN']):                                       # check if the hydroid_ManningN value is nan (indicates a non-calibrated hydroid)
+            df_nmerge.loc[index,'accum_dist'] = row['LENGTHKM'] + dist_accum                        # calculate accumulated river distance
+            dist_accum += row['LENGTHKM']                                                           # add hydroid length to the dist_accum var
+            hyid_count = 0                                                                          # reset the hydroid counter to 0
+            df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hydroid accum counter
+            if dist_accum < down_dist_thresh:                                                   # check if the accum distance is less than Xkm downstream from valid hydroid_ManningN group value
+                if hyid_accum_count > 1:                                                            # only apply the group_ManningN if there are 2 or more valid hydorids that contributed to the upstream group_ManningN
+                    df_nmerge.loc[index,'group_ManningN'] = group_ManningN                          # output the group_ManningN var
+            else:
+                run_avg_mann = 0                                                                    # reset the running average manningn variable (greater than 10km downstream)
+        else:                                                                                       # performs the following for hydroids that have a valid hydroid_ManningN value
+            dist_accum = 0; hyid_count += 1                                                         # initialize vars
+            df_nmerge.loc[index,'accum_dist'] = 0                                                   # output the accum_dist value (set to 0)
+            if hyid_count == 1:                                                                     # checks if this the first in a series of valid hydroid_ManningN values
+                run_accum_mann = 0; hyid_accum_count = 0                                            # initialize counter and running accumulated manningN value
+            group_ManningN = (row['hydroid_ManningN'] + run_accum_mann)/float(hyid_count)           # calculate the group_ManningN (NOTE: this will continue to change as more hydroid values are accumulated in the "group" moving downstream)
+            df_nmerge.loc[index,'group_ManningN'] = group_ManningN                                  # output the group_ManningN var 
+            df_nmerge.loc[index,'hyid_count'] = hyid_count                                          # output the hyid_count var 
+            run_accum_mann += row['hydroid_ManningN']                                               # add current hydroid manningn value to the running accum mann var
+            hyid_accum_count += 1                                                                   # increase the # of hydroid accum counter
+            df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hyid_accum_count var
+
+    ## Delete unnecessary intermediate outputs
+    if 'hyid_count' in df_nmerge.columns:
+        df_nmerge.drop(['hyid_count','accum_dist','hyid_accum_count'], axis=1, inplace=True) # drop hydroid counter if it exists
+    #df_nmerge.drop(['accum_dist','hyid_accum_count'], axis=1, inplace=True) # drop accum vars from group calc
+    return(df_nmerge)
\ No newline at end of file
diff --git a/src/usgs_gage_aggregate.py b/src/usgs_gage_aggregate.py
index f6967c407..24d6d26c4 100644
--- a/src/usgs_gage_aggregate.py
+++ b/src/usgs_gage_aggregate.py
@@ -21,7 +21,7 @@ def __init__(self, path, limit_branches=[]):
                             'levpa_id':str,
                             'dem_elevation':float,
                             'dem_adj_elevation':float,
-                            'order_':int,
+                            'order_':str,
                             'LakeID':object,
                             'HUC8':str,
                             'snap_distance':float}
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 085800e77..93c4095f6 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -56,6 +56,12 @@ def sort_into_branch(self, nwm_subset_streams_levelPaths):
         self.gages.feature_id = self.gages.feature_id.astype(int)
         self.gages = self.gages.merge(nwm_reaches[['feature_id','levpa_id','order_']], on='feature_id', how='left')
         return self.gages
+
+    def branch_zero(self, bzero_id):
+
+        # note that some gages will not have a valid "order_" attribute (not attributed to a level path in the step before - likely a gage on dropped stream order)
+        self.gages.levpa_id = bzero_id
+        return self.gages
     
     def write(self, out_name):
 
@@ -98,6 +104,8 @@ def filter_gage_branches(gms_inputs_filename):
     parser.add_argument('-nwm','--input-nwm-filename', help='NWM stream subset', required=True)
     parser.add_argument('-o','--output-filename', help='Table to append data', required=True)
     parser.add_argument('-huc','--huc8-id', help='HUC8 ID (to verify gage location huc)', type=str, required=True)
+    parser.add_argument('-bzero','--branch-zero-check', help='Check for determining if branch zero is created', type=int, required=True)
+    parser.add_argument('-bzero_id','--branch-zero-id', help='Branch zero ID value', type=str, required=True)
     parser.add_argument('-ff','--filter-gms-inputs', help='WARNING: only run this parameter if you know exactly what you are doing', required=False)
 
     args = vars(parser.parse_args())
@@ -107,6 +115,8 @@ def filter_gage_branches(gms_inputs_filename):
     input_nwm_filename = args['input_nwm_filename']
     output_filename = args['output_filename']
     huc8 = args['huc8_id']
+    bzero_check = args['branch_zero_check']
+    bzero_id = args['branch_zero_id']
     filter_gms_inputs = args['filter_gms_inputs']
 
     if not filter_gms_inputs:
@@ -118,6 +128,12 @@ def filter_gage_branches(gms_inputs_filename):
         usgs_gage_subset.sort_into_branch(input_nwm_filename)
         usgs_gage_subset.write(output_filename)
 
+        # Create seperate output for branch zero
+        if bzero_check != 0:
+            output_filename_zero = os.path.splitext(output_filename)[0] + '_' + bzero_id + os.path.splitext(output_filename)[-1]
+            usgs_gage_subset.branch_zero(bzero_id)
+            usgs_gage_subset.write(output_filename_zero)
+
     else:
         ''' 
         This is an easy way to filter gms_inputs so that only branches with gages will run during gms_run_branch.sh.
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index ea464667a..dbf97333b 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -2,11 +2,14 @@
 
 import os, inspect
 from os.path import splitext
+from datetime import datetime, timezone
 import fiona
 import rasterio
 import numpy as np
+from pathlib import Path
 from rasterio.warp import calculate_default_transform, reproject, Resampling
 from pyproj.crs import CRS
+import pandas as pd
 
 def getDriver(fileName):
 
@@ -320,4 +323,48 @@ def vprint (message, is_verbose, show_caller = False):
             msg += f"  [from : {caller_name}]"
         print (msg)
 
-
+########################################################################
+#Function to check the age of a file (use for flagging potentially outdated input)
+########################################################################
+def check_file_age(file):
+    '''
+    Checks if file exists, determines the file age
+    Returns
+    -------
+    None.
+    '''
+    file = Path(file)
+    if file.is_file():
+        modified_date = datetime.fromtimestamp(file.stat().st_mtime, tz=timezone.utc)
+
+        return modified_date
+
+
+########################################################################
+#Function to concatenate huc csv files to a single dataframe/csv
+########################################################################
+def concat_huc_csv(fim_dir,csv_name):
+    '''
+    Checks if huc csv file exist, concatenates contents of csv
+    Returns
+    -------
+    None.
+    '''
+
+    merged_csv = []
+    huc_list  = os.listdir(fim_dir)
+    for huc in huc_list:
+        if huc != 'logs':
+            csv_file = os.path.join(fim_dir,huc,str(csv_name))
+            if Path(csv_file).is_file():
+                # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
+                read_csv = pd.read_csv(csv_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int, 'levpa_id': object})
+                # Add huc field to dataframe
+                read_csv['HUC8'] = huc 
+                merged_csv.append(read_csv)
+
+    # Create and return a concatenated pd dataframe
+    if merged_csv:
+        print(f"Creating aggregate csv")
+        concat_df = pd.concat(merged_csv)
+        return concat_df
\ No newline at end of file
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index d8219b2e4..4e704aa46 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -47,6 +47,11 @@
 OVERWRITE_NHD = 'OVERWRITE_NHD'
 OVERWRITE_ALL = 'OVERWRITE_ALL'
 
+# Rating Curve Adjustment (local calibration) variables
+DOWNSTREAM_THRESHOLD = 10 # distance in km to propogate new roughness values downstream
+ROUGHNESS_MAX_THRESH = 0.6 # max allowable adjusted roughness value (void values larger than this)
+ROUGHNESS_MIN_THRESH = 0.001 # min allowable adjusted roughness value (void values smaller than this)
+
 ## Input Paths and Directories
 # Directories
 os.environ['src_dir'] = '/foss_fim/src'

From 79bcbb2305188d4af2300b37a6a48d0ffefc896c Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 19 Aug 2022 12:01:20 -0500
Subject: [PATCH 154/359] Patches for FIM4 alpha test (#658)

---
 src/gms/remove_error_branches.py | 9 ++++-----
 tools/eval_plots.py              | 4 ++++
 tools/run_test_case.py           | 2 +-
 tools/synthesize_test_cases.py   | 8 +++++++-
 4 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/gms/remove_error_branches.py b/src/gms/remove_error_branches.py
index 1299930b4..00a9c8ee4 100644
--- a/src/gms/remove_error_branches.py
+++ b/src/gms/remove_error_branches.py
@@ -16,7 +16,7 @@ def remove_error_branches(logfile, gms_inputs):
             print('\nLog file is empty. Skipping this HUC.\n')
             return
 
-        gms_inputs_df = pd.read_csv(gms_inputs, header=None)
+        gms_inputs_df = pd.read_csv(gms_inputs, header=None, dtype={0:str,1:str})
 
         # Make copy of gms_inputs.csv
         gms_inputs_copy = os.path.splitext(gms_inputs)[0] + '_original.csv'
@@ -44,7 +44,6 @@ def remove_error_branches(logfile, gms_inputs):
 
                 huc = split[0]
                 branch = split[3]
-                huc_level = len(huc)
 
                 if huc not in first_occurrence:
                     # Ignore previous removals for this HUC
@@ -59,10 +58,10 @@ def remove_error_branches(logfile, gms_inputs):
                     shutil.rmtree(branch_dir)
 
                 # Remove bad branch from DataFrame
-                if int(branch) in gms_inputs_df.loc[:,1].values:
-                    gms_inputs_df = gms_inputs_df.drop(index=gms_inputs_df[gms_inputs_df.loc[:,1]==int(branch)].index[0])
+                if branch in gms_inputs_df.loc[:,1].values:
+                    gms_inputs_df = gms_inputs_df.drop(index=gms_inputs_df[gms_inputs_df.loc[:,1]==branch].index[0])
 
-                tmp_df = pd.DataFrame([huc.zfill(huc_level), branch]).T
+                tmp_df = pd.DataFrame([huc, branch]).T
                 if error_branches is None:
                     error_branches = tmp_df
                 else:
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index a9a35293d..eb1bfc63f 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -89,6 +89,8 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
                 label_dict[label] = re.split('_fr|_ms|_comp', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
+            elif 'fim_4' in label and len(label) < 20:
+                label_dict[label] = label.replace('_','.').replace('fim.','FIM ')
             else:
                 label_dict[label] = label
         #Define simplified labels as a list.
@@ -256,6 +258,8 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
                 label_dict[label] = re.split('_fr|_ms|_comp', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
+            elif 'fim_4' in label and len(label) < 20:
+                label_dict[label] = label.replace('_','.').replace('fim.','FIM ')
             else:
                 label_dict[label] = label
         #Define simplified labels as a list.
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 04ff96fb8..abba132b9 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -305,7 +305,7 @@ def _inundate_and_compute(self,
                                     unit_attribute_name = 'huc8',
                                     nodata = elev_raster_ndv,
                                     workers = 1,
-                                    remove_inputs = False,
+                                    remove_inputs = True,
                                     subset = None,
                                     verbose = verbose )
             # FIM v3 and before
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 24ba9c0b0..a117464a3 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -99,6 +99,9 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                         if iteration == "official":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
+                            # add in composite of versions
+                            composite_versions = [v.replace('_ms', '_comp') for v in versions_to_aggregate if '_ms' in v]
+                            versions_to_aggregate += composite_versions
                         if iteration == "comparison":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = dev_versions_to_include_list
@@ -154,6 +157,9 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                         if iteration == "official":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
+                            # add in composite of versions
+                            composite_versions = [v.replace('_ms', '_comp') for v in versions_to_aggregate if '_ms' in v]
+                            versions_to_aggregate += composite_versions
                         if iteration == "comparison":
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = dev_versions_to_include_list
@@ -427,7 +433,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
         else:
             dev_versions_to_include_list = previous_fim_list
     if config == 'PREV':
-        dev_versions_to_include_list = []
+        dev_versions_to_include_list = dev_versions_to_compare
 
     if master_metrics_csv is not None:
         # Do aggregate_metrics.

From 010f3710fdc31cf32a75eaa8f16c7904e55d42f1 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Mon, 29 Aug 2022 07:41:12 -0500
Subject: [PATCH 155/359] Hotfix for addressing missing input variable in
 gms_run_branch.sh (#665)

---
 docs/CHANGELOG.md                 | 16 +++++++++++++---
 gms_run_branch.sh                 |  2 +-
 src/src_adjust_spatial_obs.py     |  2 +-
 src/src_roughness_optimization.py |  1 -
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7afaa4cd8..ad1d53906 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,12 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-## v4.0.6.3 - 2022-08-04 - [PR #652](https://github.com/NOAA-OWP/inundation-mapping/pull/652)
+## v4.0.7.1 - 2022-08-22 - [PR #665](https://github.com/NOAA-OWP/inundation-mapping/pull/665)
 
-Updated `Dockerfile`, `Pipfile` and `Pipfile.lock` to add the new psycopg2 python package required for a WIP code fix for the new FIM4 calibration db.
+Hotfix for addressing missing input variable when running `gms_run_branch.sh` outside of `gms_pipeline.sh`. 
 
-<br/><br/>
+## Changes
+- `gms_run_branch.sh`: defining path to WBD HUC input file directly in ogr2ogr call rather than using the $input_WBD_gdb defined in `gms_run_unit.sh`
+- `src/src_adjust_spatial_obs.py`: removed an extra print statement
+- `src/src_roughness_optimization.py`: removed a log file write that contained sensitive host name
 
+<br/><br/>
 
 ## v4.0.7.0 - 2022-08-17 - [PR #657](https://github.com/NOAA-OWP/inundation-mapping/pull/657)
 
@@ -45,6 +49,12 @@ Introduces synthetic rating curve calibration workflow. The calibration computes
 
 <br/><br/>
 
+## v4.0.6.3 - 2022-08-04 - [PR #652](https://github.com/NOAA-OWP/inundation-mapping/pull/652)
+
+Updated `Dockerfile`, `Pipfile` and `Pipfile.lock` to add the new psycopg2 python package required for a WIP code fix for the new FIM4 calibration db.
+
+<br/><br/>
+
 ## v4.0.6.2 - 2022-08-16 - [PR #639](https://github.com/NOAA-OWP/inundation-mapping/pull/639)
 
 This file converts USFIMR remote sensed inundation shapefiles into a raster that can be used to compare to the FIM data. It has to be run separately for each shapefile. This addresses [#629].
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index ecdcb026c..41ca20376 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -122,7 +122,7 @@ if [ "$src_adjust_spatial" = "True" ]; then
         source $CALB_DB_KEYS_FILE
         echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes"
         echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $input_WBD_gdb WBDHU8
+        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
         echo "Loading Point Data"
         time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
     fi
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index abd77ace0..9407f3e1d 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -201,7 +201,7 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
     - procs_list:           passes multiprocessing list of input args for process_points function input
     '''
     conn = connect() # Connect to the PostgreSQL db once
-    log_file.write('Connected to database via host: ' + str(CALIBRATION_DB_HOST) + '\n')
+    log_file.write('Connected to database via host\n')
     print("Finding all fim_output hucs that contain calibration points...")
     fim_out_huc_list  = [ item for item in os.listdir(fim_directory) if os.path.isdir(os.path.join(fim_directory, item)) ]
     fim_out_huc_list.remove('logs')
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index d0607de08..56663d16a 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -117,7 +117,6 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
             df_nvalues.loc[index,'discharge_cms'] = find_src_stage.discharge_cms
 
     ## mask src values that crosswalk to the SRC zero point (src_stage ~ 0 or discharge <= 0)
-    print(str(huc) + '  branch id: ' + str(branch_id))
     df_nvalues[['HydraulicRadius_m','WetArea_m2']] = df_nvalues[['HydraulicRadius_m','WetArea_m2']].mask((df_nvalues['src_stage'] <= 0.1) | (df_nvalues['discharge_cms'] <= 0.0), np.nan)
 
     ## Calculate roughness using Manning's equation

From 40d2258dc7d5bb460dea091ce21dccd831fa97c0 Mon Sep 17 00:00:00 2001
From: CalebOliven-NOAA <97476664+CalebOliven-NOAA@users.noreply.github.com>
Date: Mon, 29 Aug 2022 15:08:42 -0600
Subject: [PATCH 156/359] Update Inundate_nation.py for FIM4 (#654)

---
 docs/CHANGELOG.md               |  14 ++
 tools/gms_tools/inundate_gms.py |   3 +-
 tools/inundate_nation.py        | 240 +++++++++++++++++++++++---------
 3 files changed, 193 insertions(+), 64 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ad1d53906..a0a6ebf9b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.7.2 - 2022-08-11 - [PR #654](https://github.com/NOAA-OWP/inundation-mapping/pull/654)
+
+`inundate_nation.py` A change to switch the inundate nation function away from refrences to `inundate.py`, and rather use `inundate_gms.py` and `mosaic_inundation.py`
+
+## Changes
+
+- `inundate_gms`:  Changed `mask_type = 'filter'`
+
+<br/><br/>
+
+
 ## v4.0.7.1 - 2022-08-22 - [PR #665](https://github.com/NOAA-OWP/inundation-mapping/pull/665)
 
 Hotfix for addressing missing input variable when running `gms_run_branch.sh` outside of `gms_pipeline.sh`. 
@@ -79,6 +91,7 @@ Prunes branches that fail with NO_FLOWLINES_EXIST (Exit code: 61) in `gms_run_br
 
 <br/><br/>
 
+
 ## v4.0.6.0 - 2022-08-10 - [PR #614](https://github.com/NOAA-OWP/inundation-mapping/pull/614)
 
 Addressing #560, this fix in run_by_branch trims the DEM derived streamline if it extends past the end of the branch streamline. It does this by finding the terminal point of the branch stream, snapping to the nearest point on the DEM derived stream, and cutting off the remaining downstream portion of the DEM derived stream.
@@ -142,6 +155,7 @@ New FIM4/gms usability is now just (at a minumum): `gms_pipeline.sh -n <output n
 Updates to unit tests including a minor update for outputs and loading in .json parameter files.
 <br><br>
 
+
 ## v4.0.5.1 - 2022-06-27 - [PR #612](https://github.com/NOAA-OWP/inundation-mapping/pull/612)
 
 `Alpha Test Refactor` An upgrade was made a few weeks back to the dev-fim3 branch that improved performance, usability and readability of running alpha tests. Some cleanup in other files for readability, debugging verbosity and styling were done as well. A newer, cleaner system for printing lines when the verbose flag is enabled was added.
diff --git a/tools/gms_tools/inundate_gms.py b/tools/gms_tools/inundate_gms.py
index 8259fad51..4437fadab 100755
--- a/tools/gms_tools/inundate_gms.py
+++ b/tools/gms_tools/inundate_gms.py
@@ -208,7 +208,7 @@ def __inundate_gms_generator( hucs_branches,
                             'catchment_poly' : catchment_poly,
                             'hydro_table' : hydroTable_branch,
                             'forecast' : forecast,
-                            'mask_type' : None,
+                            'mask_type' : 'filter',
                             'hucs' : None,
                             'hucs_layerName' : None,
                             'subset_hucs' : None, 
@@ -241,4 +241,3 @@ def __inundate_gms_generator( hucs_branches,
     
     # extract to dictionary and run
     Inundate_gms( **vars(parser.parse_args()) )
-    
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index 6fbcdf146..d5b79af7e 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -1,102 +1,218 @@
 import argparse
 import os
+import rasterio
+from rasterio.merge import merge
+from osgeo import gdal, ogr
 
 from inundation import inundate
+import multiprocessing
 from multiprocessing import Pool
+from gms_tools.mosaic_inundation import Mosaic_inundation
+from gms_tools.inundate_gms import Inundate_gms
+from tools_shared_variables import elev_raster_ndv
+import re
+
 
 INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
+INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
 INPUTS_DIR = r'/data/inputs'
+OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/'
+DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
+PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'
 
 
 def run_inundation(args):
     """
-    This script is basically a wrapper for the inundate function and is designed for multiprocessing.
+    This script is a wrapper for the inundate function and is designed for multiprocessing.
     
     Args:
-        args (list): [fim_run_dir (str), huc (str), magnitude (str), magnitude_output_dir (str), config (str)]
+        args (list): [fim_run_dir (str), huc_list (list), magnitude (str), magnitude_output_dir (str), config (str), forecast (str), job_number (int)]
     
     """
     
-    fim_run_dir = args[0]
-    huc = args[1]
+    fim_run_dir = args[0]  
+    huc_list = args[1]
     magnitude = args[2]
     magnitude_output_dir = args[3]
     config = args[4]
+    forecast = args[5]
+    job_number = args[6]
+
     
     # Define file paths for use in inundate().
-    fim_run_parent = os.path.join(fim_run_dir, huc)
-    rem = os.path.join(fim_run_parent, 'rem_zeroed_masked.tif')
-    catchments = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-    mask_type = 'huc'
-    catchment_poly = ''
-    hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
-    catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+ 
+
     inundation_raster = os.path.join(magnitude_output_dir, magnitude + '_' + config + '_inund_extent.tif')
-    depth_raster = os.path.join(magnitude_output_dir, magnitude + '_' + config + '_inund_depth.tif')
-    forecast = os.path.join(INUN_REVIEW_DIR, 'nwm_recurr_flow_data', 'recurr_' + magnitude + '_cms.csv')
-    hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-
-    # Run inundate() once for depth and once for extent.
-    if not os.path.exists(depth_raster):
-        print("Running the NWM recurrence intervals for HUC: " + huc + ", " + magnitude + "...")
-        inundate(
-                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
-                 depths=depth_raster,out_raster_profile=None,out_vector_profile=None,quiet=True
-                )
-        
-    if not os.path.exists(inundation_raster):
-        inundate(
-                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-                 depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
-                )
-        
+    
+    print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude))
 
-if __name__ == '__main__':
+    map_file = Inundate_gms( hydrofabric_dir = fim_run_dir, 
+                                    forecast = forecast, 
+                                    num_workers = job_number,
+                                    hucs = huc_list,
+                                    inundation_raster = inundation_raster,
+                                    inundation_polygon = None,
+                                    depths_raster = None,
+                                    verbose = True,
+                                    log_file = None,
+                                    output_fileNames = None )
+    
+    Mosaic_inundation( map_file,
+                                    mosaic_attribute = 'inundation_rasters',
+                                    mosaic_output = inundation_raster,
+                                    #mask = os.path.join(fim_run_dir,huc8,'wbd.gpkg'),
+                                    mask = None,
+                                    unit_attribute_name = 'huc8',
+                                    nodata = elev_raster_ndv,
+                                    workers = 1,
+                                    remove_inputs = True,
+                                    subset = None,
+                                    verbose = True,
+                                    is_mosaic_for_gms_branches = True )
+
+def create_bool_rasters(args):
+    in_raster_dir = args[0]
+    rasfile = args[1]
+    output_bool_dir = args[2]
+
+    print("Calculating boolean inundate raster: " + rasfile)
+    p = in_raster_dir + os.sep + rasfile
+    raster = rasterio.open(p)
+    profile = raster.profile
+    array = raster.read()
+    del raster
+    array[array>0] = 1
+    array[array<=0] = 0
+    # And then change the band count to 1, set the
+    # dtype to uint8, and specify LZW compression.
+    profile.update(driver="GTiff",
+                height=array.shape[1],
+                width=array.shape[2],
+                tiled=True,
+                nodata=0,
+                blockxsize=512, 
+                blockysize=512,
+                dtype='int8',
+                crs=PREP_PROJECTION,
+                compress='lzw')
+    with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, 'w', **profile) as dst:
+        dst.write(array.astype(rasterio.int8))
+
+
+def vrt_raster_mosaic(output_bool_dir, output_mos_dir, fim_version_tag):
     
+    #raster_to_mosaic = ['data/temp/ryan/inundate_nation/25_0_ms/25_0_ms_inund_extent_12090301.tif','data/temp/ryan/inundate_nation/25_0_ms/25_0_ms_inund_extent_12090302.tif']
+    raster_to_mosaic = []
+    for rasfile in os.listdir(output_bool_dir):
+        if rasfile.endswith('.tif') and "extent" in rasfile:
+            p = output_bool_dir + os.sep + rasfile
+            print("Processing: " + p)
+            raster_to_mosaic.append(p)
+
+    print("Creating virtual raster...")
+    vrt = gdal.BuildVRT(output_mos_dir + "merged.vrt", raster_to_mosaic)
+
+    print("Building raster mosaic: " + str(output_mos_dir + fim_version_tag + "_mosaic.tif"))
+    gdal.Translate(output_mos_dir + fim_version_tag + "_mosaic.tif", vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2'])
+    vrt = None        
+
+
+
+
+
+if __name__ == '__main__':
+    available_cores = multiprocessing.cpu_count()
+
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow recurrence interflow data. Inundation outputs are stored in the /inundation_review/inundation_nwm_recurr/ directory.')
     parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)',required=True)
-    parser.add_argument('-o', '--output-dir',help='The path to a directory to write the outputs. If not used, the inundation_review directory is used by default -> type=str',required=False, default="")
+    parser.add_argument('-o', '--output-dir',help='Optional: The path to a directory to write the outputs. If not used, the inundation_nation directory is used by default -> type=str',required=False, default="")
+    parser.add_argument('-m', '--magnitude-list', help = 'List of NWM recurr flow intervals to process (Default: 100_0) (Other options: 2_0 5_0 10_0 25_0 50_0 100_0)', nargs = '+', default = ['100_0'], required = False)
+    parser.add_argument('-d', '--depth',help='not functional. Optional flag to produce inundation depth rasters (extent raster created by default)', action='store_true')
+    parser.add_argument('-s', '--mosaic',help='Optional flag to produce mosaic of FIM extent rasters', action='store_true')
     parser.add_argument('-j', '--job-number',help='The number of jobs',required=False,default=1)
         
     args = vars(parser.parse_args())
 
     fim_run_dir = args['fim_run_dir']
     output_dir = args['output_dir']
-    magnitude_list = ['1_5']
-    
+    depth_option = args['depth']
+    magnitude_list = args['magnitude_list']
+    mosaic_option = args['mosaic']
     job_number = int(args['job_number'])
 
-    huc_list = os.listdir(fim_run_dir)
+    assert os.path.isdir(fim_run_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_run_dir)
+    print("Input FIM Directory: " + str(fim_run_dir))
+    fim_version = os.path.basename(os.path.normpath(fim_run_dir))
+    print("Using fim version: " + str(fim_version))
+    for magnitude in magnitude_list:
+        print("Preparing to generate inundation outputs for magnitude: " + str(magnitude))
+        nwm_recurr_file = os.path.join(INUN_REVIEW_DIR, 'nwm_recurr_flow_data', 'nwm21_17C_recurr_' + magnitude + '_cms.csv')
+        assert os.path.isfile(nwm_recurr_file), 'ERROR: could not find the input NWM recurr flow file: ' + str(nwm_recurr_file)
+        print("Input flow file: " + str(nwm_recurr_file))
         
-    fim_version = os.path.split(fim_run_dir)[1]
-    
-    if output_dir == "":
-        output_dir = os.path.join(INUN_REVIEW_DIR, fim_version)
+        config = 'gms'
+
+        if output_dir == "":
+            output_dir = INUN_OUTPUT_DIR
+        if not os.path.exists(output_dir):
+            print('Creating new output directory: ' + str(output_dir))
+            os.mkdir(output_dir)
             
-    if not os.path.exists(output_dir):
-        os.mkdir(output_dir)
-    
-    if 'ms' in fim_version:
-        config = 'ms'
-    if 'fr' in fim_version:
-        config = 'fr'
-        
-    procs_list = []
-    
-    for huc in huc_list:
-        if huc != 'logs':
-            for magnitude in magnitude_list:
-                magnitude_output_dir = os.path.join(output_dir, magnitude + '_' + config)
-                if not os.path.exists(magnitude_output_dir):
-                    os.mkdir(magnitude_output_dir)
-                    print(magnitude_output_dir)
-                procs_list.append([fim_run_dir, huc, magnitude, magnitude_output_dir, config])
+        huc_list = []
+        for huc in os.listdir(fim_run_dir):
+            
+            #if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
+            if re.match('\d{8}', huc):    
+                huc_list.append(huc)
+            
+        magnitude_output_dir = os.path.join(output_dir, magnitude + '_' + config  + '_' + fim_version)
+        if not os.path.exists(magnitude_output_dir):
+            os.mkdir(magnitude_output_dir)
+
+        print('Inundation outputs here: ' + str(magnitude_output_dir))
+        run_inundation([fim_run_dir, huc_list, magnitude, magnitude_output_dir, config, nwm_recurr_file, job_number])
+                   
+
+
+        # Perform mosaic operation
+        if mosaic_option:
+            print("Performing mosaic process...")
+            fim_version_tag = os.path.basename(os.path.normpath(magnitude_output_dir))
+            print("fim_version: " + fim_version_tag)
+            output_bool_dir = os.path.join(OUTPUT_BOOL_PARENT_DIR, fim_version_tag)
+            if not os.path.exists(output_bool_dir):
+                print('Creating new output directory for boolean temporary outputs: ' + str(output_bool_dir))
+                os.mkdir(output_bool_dir)
+
+
+            output_mos_dir = output_dir
+            if output_dir == "":
+                output_mos_dir = DEFAULT_OUTPUT_DIR
+            
+            
+            if not os.path.exists(output_mos_dir):
+                print('Creating new output directory: ' + str(output_mos_dir))
+                os.mkdir(output_mos_dir)
+
+            if job_number > available_cores:
+                job_number = available_cores - 1
+                print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+
+            procs_list = []
+            for rasfile in os.listdir(magnitude_output_dir):
+                if rasfile.endswith('.tif') and "extent" in rasfile:
+                    #p = magnitude_output_dir + rasfile
+                    procs_list.append([magnitude_output_dir,rasfile,output_bool_dir])
+
+            # Multiprocess --> create boolean inundation rasters for all hucs
+            if len(procs_list) > 0:
+                with Pool(processes=job_number) as pool:
+                    pool.map(create_bool_rasters, procs_list)
+            else:
+                print('Did not find any valid FIM extent rasters: ' + magnitude_output_dir)
+
+            # Perform VRT creation and final mosaic using boolean rasters
             
-    # Multiprocess.
-    if job_number > 1:
-        with Pool(processes=job_number) as pool:
-            pool.map(run_inundation, procs_list)
+            vrt_raster_mosaic(output_bool_dir,output_mos_dir,fim_version_tag)
 

From 68e76431311523575417b796d758a9b6a948d282 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 31 Aug 2022 16:26:03 -0400
Subject: [PATCH 157/359] Prune branches in waterbodies (#671)

---
 docs/CHANGELOG.md             |  10 ++-
 src/gms/derive_level_paths.py |  25 ++++---
 src/gms/run_by_unit.sh        |   2 +-
 src/gms/stream_branches.py    | 122 +++++++++++++++++++++++++++++++---
 4 files changed, 138 insertions(+), 21 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a0a6ebf9b..b65f4f586 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,11 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## 4.0.8.0 - 2022-08-26 - [PR #671](https://github.com/NOAA-OWP/inundation-mapping/pull/671)
+
+Trims ends of branches that are in waterbodies; also removes branches if they are entirely in a waterbody.
+
+<br/><br/>
 
 ## v4.0.7.2 - 2022-08-11 - [PR #654](https://github.com/NOAA-OWP/inundation-mapping/pull/654)
 
@@ -10,8 +15,11 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 - `inundate_gms`:  Changed `mask_type = 'filter'`
 
-<br/><br/>
+## Changes
+
+- `src/gms/stream_branches.py`: adds functionality to trim and prune branches in waterbodies.
 
+<br/><br/>
 
 ## v4.0.7.1 - 2022-08-22 - [PR #665](https://github.com/NOAA-OWP/inundation-mapping/pull/665)
 
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index 03d8f61fe..96e4cf5fa 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -9,7 +9,7 @@
 
 def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribute,
                        out_stream_network_dissolved=None, huc_id=None,
-                       headwaters_outfile=None, catchments=None,
+                       headwaters_outfile=None, catchments=None, waterbodies=None,
                        catchments_outfile=None,
                        branch_inlets_outfile=None,
                        toNode_attribute='To_Node', fromNode_attribute='From_Node',
@@ -101,9 +101,8 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                             verbose=verbose
                                                            )
     
-    # filter out streams with out catchments
+    # filter out streams without catchments
     if (catchments is not None) & (catchments_outfile is not None):
-
         catchments = gpd.read_file(catchments)
 
         stream_network = stream_network.remove_branches_without_catchments( 
@@ -137,16 +136,18 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                         outlet_linestring_index=outlet_linestring_index
                                                        )
         # headwaters write
-        headwaters.to_file(headwaters_outfile,index=False,driver='GPKG')
+        headwaters.to_file(headwaters_outfile, index=False, driver='GPKG')
 
-    
     if out_stream_network is not None:
         if verbose:
             print("Writing stream branches ...")
-        stream_network.write(out_stream_network,index=True)
+        stream_network.write(out_stream_network, index=True)
     
     if out_stream_network_dissolved is not None:
-    
+        stream_network = stream_network.trim_branches_in_waterbodies(branch_id_attribute=branch_id_attribute,
+                                                    verbose=verbose
+                                                                    )
+
         # dissolve by levelpath
         stream_network = stream_network.dissolve_by_branch(branch_id_attribute=branch_id_attribute,
                                                            attribute_excluded=None, #'order_',
@@ -154,8 +155,13 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                            out_vector_files=out_stream_network_dissolved,
                                                            verbose=verbose)
 
-        branch_inlets = stream_network.derive_inlet_points_by_feature( feature_attribute=branch_id_attribute,
-                                                                       outlet_linestring_index=outlet_linestring_index
+        stream_network = stream_network.remove_branches_in_waterbodies(waterbodies=waterbodies,
+                                                                       out_vector_files=out_stream_network_dissolved,
+                                                                       verbose=False
+                                                                      )
+                                       
+        branch_inlets = stream_network.derive_inlet_points_by_feature(feature_attribute=branch_id_attribute,
+                                                                      outlet_linestring_index=outlet_linestring_index
                                                                      )
    
         if branch_inlets_outfile is not None:
@@ -174,6 +180,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
     parser.add_argument('-r','--reach-id-attribute', help='Reach ID attribute to use in source file', required=False, default='HydroID')
     parser.add_argument('-c','--catchments', help='NWM catchments to append level path data to', required=False, default=None)
     parser.add_argument('-t','--catchments-outfile', help='NWM catchments outfile with appended level path data', required=False, default=None)
+    parser.add_argument('-w','--waterbodies', help='NWM waterbodies to eliminate branches from', required=False, default=None)
     parser.add_argument('-n','--branch_inlets_outfile', help='Output level paths inlets', required=False, default=None)
     parser.add_argument('-o','--out-stream-network', help='Output stream network', required=False, default=None)
     parser.add_argument('-e','--headwaters-outfile', help='Output stream network headwater points', required=False, default=None)
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index c513e8b3d..fa25e9cc3 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -83,7 +83,7 @@ Tcount
 echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -s $dropLowStreamOrders
+$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -s $dropLowStreamOrders -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 
 # test if we received a non-zero code back from derive_level_paths.py
 subscript_exit_code=$?
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index cec199ba2..cb16c7112 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -291,7 +291,6 @@ def derive_nodes(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',re
         self.loc[:,fromNode_attribute] = fromNodes
         self.loc[:,toNode_attribute] = toNodes
         
-        
         return(self)
 
 
@@ -386,7 +385,7 @@ def remove_stream_segments_without_catchments( self,
         if verbose:
             print("Removing stream segments without catchments ...")
 
-        # load cathments
+        # load catchments
         if isinstance(catchments,gpd.GeoDataFrame):
             pass
         elif isinstance(catchments,str):
@@ -413,8 +412,8 @@ def remove_branches_without_catchments(self,
         if verbose:
             print("Removing stream branches without catchments ...")
 
-        # load cathments
-        if isinstance(catchments,gpd.GeoDataFrame):
+        # load catchments
+        if isinstance(catchments, gpd.GeoDataFrame):
             pass
         elif isinstance(catchments,str):
             catchments = gpd.read_file(catchments)
@@ -425,7 +424,7 @@ def remove_branches_without_catchments(self,
         unique_catchments = set(catchments.loc[:,reach_id_attribute_in_catchments].unique())
 
         current_index_name = self.index.name
-        self.set_index(branch_id_attribute,drop=False,inplace=True)
+        self.set_index(branch_id_attribute, drop=False, inplace=True)
 
         for usb in unique_stream_branches:
 
@@ -439,9 +438,112 @@ def remove_branches_without_catchments(self,
                 self.drop(usb,inplace=True)
 
         if current_index_name is None:
-            self.reset_index(drop=True,inplace=True)
+            self.reset_index(drop=True, inplace=True)
         else:
-            self.set_index(current_index_name,drop=True,inplace=True)
+            self.set_index(current_index_name, drop=True, inplace=True)
+
+        return(self)
+
+
+    def trim_branches_in_waterbodies(self,
+                                       branch_id_attribute,
+                                       verbose=False
+                                       ):
+        """
+        Recursively trims the reaches from the ends of the branches if they are in a 
+        waterbody (determined by the Lake attribute).
+        """
+
+        def find_downstream_reaches_in_waterbodies(tmp_self, downstream_ID, tmp_IDs=[]):
+            downstream_ID = int(tmp_self.ID[tmp_self.to==downstream_ID])
+
+            # Stop if lowest reach is not in a lake
+            if int(tmp_self.Lake[tmp_self.ID==downstream_ID]) == -9999:
+                return tmp_IDs
+            else:
+                tmp_IDs.append(downstream_ID)
+
+                # Find next lowest downstream reach
+                if downstream_ID in tmp_self.to.values:
+                    return find_downstream_reaches_in_waterbodies(tmp_self, downstream_ID, tmp_IDs)
+                else:
+                    return tmp_IDs
+
+        def find_upstream_reaches_in_waterbodies(tmp_self, upstream_ID, tmp_IDs=[]):
+            upstream_ID = int(tmp_self.to[tmp_self.ID==upstream_ID])
+
+            # Stop if uppermost reach is not in a lake
+            if int(tmp_self.Lake[tmp_self.ID==upstream_ID]) == -9999:
+                return tmp_IDs
+            else:
+                tmp_IDs.append(upstream_ID)
+
+                # Find next highest upstream reach
+                if upstream_ID in tmp_self.ID.values:
+                    return find_upstream_reaches_in_waterbodies(tmp_self, upstream_ID, tmp_IDs)
+                else:
+                    return tmp_IDs
+
+        if verbose:
+            print("Trimming stream branches in waterbodies ...")
+
+        for branch in self[branch_id_attribute].unique():
+            tmp_self = self[self[branch_id_attribute]==branch]
+
+            # If entire branch is in waterbody
+            if all(tmp_self.Lake.values != -9999):
+                tmp_IDs = tmp_self.ID
+                
+            else:
+                # Find bottom up
+                downstream_ID = int(tmp_self.ID[~tmp_self.to.isin(tmp_self.ID)]) # ID of most downstream reach
+                if not int(tmp_self.Lake[tmp_self.ID==downstream_ID]) == -9999:
+                    tmp_IDs = find_downstream_reaches_in_waterbodies(tmp_self, downstream_ID, [downstream_ID])
+                else:
+                    tmp_IDs = []
+
+                # Find top down
+                upstream_ID = int(tmp_self.ID[~tmp_self.ID.isin(tmp_self.to)])
+                if (len(tmp_IDs) != len(tmp_self)) and not (int(tmp_self.Lake[tmp_self.ID==upstream_ID]) == -9999):
+                    tmp_IDs.append(upstream_ID)
+                    tmp_IDs.append(find_upstream_reaches_in_waterbodies(tmp_self, upstream_ID, tmp_IDs))
+
+            if len(tmp_IDs) > 0:
+                self.drop(tmp_self[tmp_self.ID.isin(tmp_IDs)].index, inplace=True)
+
+        return(self)
+
+
+    def remove_branches_in_waterbodies(self,
+                                       waterbodies,
+                                       out_vector_files=None,
+                                       verbose=False
+                                       ):
+        """
+        Removes branches completely in waterbodies
+        """
+
+        if verbose:
+            print('Removing branches in waterbodies')
+
+        # load waterbodies
+        if isinstance(waterbodies,gpd.GeoDataFrame):
+            pass
+        elif isinstance(waterbodies,str):
+            waterbodies = gpd.read_file(waterbodies)
+        else:
+            raise TypeError("Waterbodies needs to be GeoDataFame or path to vector file")
+
+        # Find branches in waterbodies
+        sjoined = gpd.sjoin(self, waterbodies, op='within')
+        self.drop(sjoined.index, inplace=True)
+
+        if out_vector_files is not None:
+            
+            if verbose:
+                print("Writing pruned branches ...")
+            
+            self.write(out_vector_files, index=False)
 
         return(self)
 
@@ -647,8 +749,8 @@ def get_arbolate_sum(self,arbolate_sum_attribute='arbolate_sum',inlets_attribute
         return(self)
 
 
-    def dissolve_by_branch(self,branch_id_attribute='LevelPathI',attribute_excluded='StreamOrde',
-                           values_excluded=[1,2],out_vector_files=None, verbose=False):
+    def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_excluded='StreamOrde',
+                           values_excluded=[1,2], out_vector_files=None, verbose=False):
         
         if verbose:
             print("Dissolving by branch ...")
@@ -700,7 +802,7 @@ def dissolve_by_branch(self,branch_id_attribute='LevelPathI',attribute_excluded=
                 #current_stream_network = StreamNetwork(self.loc[bid_indices,:])
 
                 #current_stream_network.write(out_vector_file,index=False)
-            self.write(out_vector_files,index=False)
+            self.write(out_vector_files, index=False)
 
         return(self)
 

From 29c859d13c5e765b7c358eae372a8060bea578bd Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 9 Sep 2022 15:50:04 -0500
Subject: [PATCH 158/359] Update derive level paths to use stream order (#672)

---
 docs/CHANGELOG.md                             |  6 +++
 src/gms/derive_level_paths.py                 |  2 +-
 src/gms/stream_branches.py                    | 40 ++++++++++++++-----
 .../gms/derive_level_paths_unittests.py       |  2 +-
 4 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b65f4f586..0eb989ff8 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,12 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## 4.0.9.0 - 2022-09-09 - [PR #672](https://github.com/NOAA-OWP/inundation-mapping/pull/672)
+
+When deriving level paths, this improvement allows stream order to override arbolate sum when selecting the proper upstream segment to continue the current branch.
+
+<br/><br/>
+
 ## 4.0.8.0 - 2022-08-26 - [PR #671](https://github.com/NOAA-OWP/inundation-mapping/pull/671)
 
 Trims ends of branches that are in waterbodies; also removes branches if they are entirely in a waterbody.
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index 96e4cf5fa..7102a6732 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -96,7 +96,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                             upstreams=upstreams,
                                                             branch_id_attribute=branch_id_attribute,
                                                             reach_id_attribute=reach_id_attribute,
-                                                            comparison_attributes='arbolate_sum',
+                                                            comparison_attributes=['arbolate_sum', 'order_'],
                                                             comparison_function=max,
                                                             verbose=verbose
                                                            )
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index cb16c7112..3d4173119 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -609,7 +609,7 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
             progress.update(1)
 
             # get current reach stream order and branch id
-            current_reach_comparison_value = self.at[current_reach_id,comparison_attributes]
+            #current_reach_comparison_value = self.at[current_reach_id,comparison_attributes]
             current_reach_branch_id = self.at[current_reach_id,branch_id_attribute]
 
             # get upstream ids
@@ -636,16 +636,34 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
                     # find 
                     upstream_reaches_compare_values = self.loc[not_visited_upstream_ids,comparison_attributes]
                     matching_value = comparison_function(upstream_reaches_compare_values)
-                    
-                    matches = 0 # if upstream matches are more than 1, limits to only one match
-                    for usrcv,nvus in zip(upstream_reaches_compare_values,not_visited_upstream_ids):
-                        if (usrcv == matching_value) & (matches == 0):
-                            self.at[nvus,branch_id_attribute] = current_reach_branch_id
-                            matches += 1
-                        else:
-                            branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
-                            self.at[nvus,branch_id_attribute] = branch_id
-                            bid += 1
+
+                    #==================================================================================
+                    # If the two stream orders aren't the same, then follow the highest order, otherwise use arbolate sum
+                    if upstream_reaches_compare_values.idxmax()['order_'] == upstream_reaches_compare_values.idxmin()['order_']:
+                        decision_attribute = 'arbolate_sum'
+                    else:
+                        decision_attribute = 'order_'
+                    # Continue the current branch up the larger stream
+                    continue_id = upstream_reaches_compare_values.idxmax()[decision_attribute]
+                    self.loc[continue_id,branch_id_attribute] = current_reach_branch_id
+                    # Create a new level path for the smaller tributary
+                    if len(not_visited_upstream_ids) == 1: continue # only create a new branch if there are 2 upstreams
+                    new_id = upstream_reaches_compare_values.idxmin()[decision_attribute]
+                    branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
+                    self.loc[new_id,branch_id_attribute] = branch_id
+                    bid += 1
+                    #==================================================================================
+                    ''' NOTE: The above logic uses stream order to override arbolate sum. Use the commented section
+                     below if this turns out to be a bad idea!'''
+                    #matches = 0 # if upstream matches are more than 1, limits to only one match
+                    #for usrcv,nvus in zip(upstream_reaches_compare_values,not_visited_upstream_ids):
+                    #    if (usrcv == matching_value) & (matches == 0):
+                    #        self.at[nvus,branch_id_attribute] = current_reach_branch_id
+                    #        matches += 1
+                    #    else:
+                    #        branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
+                    #        self.at[nvus,branch_id_attribute] = branch_id
+                    #        bid += 1
         
         progress.close()
 
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
index fe7c1fa9e..be613204a 100644
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -16,7 +16,7 @@
 sys.path.append('/foss_fim/src/gms/')
 import derive_level_paths as src
 import stream_branches
-from utils.fim_enums import FIM_system_exit_codes as fec
+from utils.fim_enums import FIM_exit_codes as fec
 
 
 # NOTE: This goes directly to the function.

From 23ae13ced9da616385a4bc38b32d2c68bd21237d Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 9 Sep 2022 16:13:26 -0500
Subject: [PATCH 159/359] Add tool to push data to S3 for Hydroviz (#664)

---
 .gitignore                      |    1 +
 Dockerfile                      |    6 +-
 Pipfile                         |    4 +
 Pipfile.lock                    | 1100 +++++++++++++++++++++++++++++--
 config/aws_s3_put_whitelist.lst |    3 +
 data/README.md                  |   16 +
 data/__init__.py                |    0
 data/aws/.gitignore             |    2 +
 data/aws/aws_base.py            |  124 ++++
 data/aws/aws_creds_template.env |    3 +
 data/aws/s3.py                  |  221 +++++++
 docs/CHANGELOG.md               |   34 +
 gms_run_branch.sh               |   30 +-
 gms_run_unit.sh                 |   12 +-
 src/utils/shared_functions.py   |  219 ++++--
 15 files changed, 1642 insertions(+), 133 deletions(-)
 create mode 100644 config/aws_s3_put_whitelist.lst
 create mode 100644 data/README.md
 create mode 100644 data/__init__.py
 create mode 100644 data/aws/.gitignore
 create mode 100644 data/aws/aws_base.py
 create mode 100644 data/aws/aws_creds_template.env
 create mode 100644 data/aws/s3.py

diff --git a/.gitignore b/.gitignore
index e5acc4ee8..b34cb48fd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@
 __pycache__/
 config/**
 !config/deny*.lst
+!config/aws*.lst
 !config/*template*
 !config/*default*
 !config/symbology/
diff --git a/Dockerfile b/Dockerfile
index da10b8e0c..a9cf355a1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -74,6 +74,11 @@ RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=
 
 RUN apt auto-remove
 
+## adding AWS CLI (for bash) ##
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+	./aws/install
+
 ## adding environment variables for numba and python ##
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
@@ -87,7 +92,6 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 COPY Pipfile .
 COPY Pipfile.lock .
 RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
-RUN pip3 install shapely
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
 ADD ./src/entrypoint.sh /
diff --git a/Pipfile b/Pipfile
index 313f02dcc..ce0fc83ad 100755
--- a/Pipfile
+++ b/Pipfile
@@ -29,6 +29,10 @@ tables = "==3.6.1"
 h5py = "==3.4.0"
 pyproj = "==3.1.0"
 psycopg2-binary = "==2.8.6"
+boto3 = "*"
+jupyter = "*"
+jupyterlab = "*"
+ipympl = "*"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 9f5e923e4..2e5578067 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "076bbd38f0fa0777776db2a9f842b60cbc6a8d5f5bf8366cc79e6abf86efde4b"
+            "sha256": "f48eed8e1c745d0ea7de9caa2c2fb3264541a2ef93f9408e0156c4544359bc55"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -23,6 +23,56 @@
             ],
             "version": "==2.3.1"
         },
+        "anyio": {
+            "hashes": [
+                "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b",
+                "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"
+            ],
+            "markers": "python_full_version >= '3.6.2'",
+            "version": "==3.6.1"
+        },
+        "argon2-cffi": {
+            "hashes": [
+                "sha256:8c976986f2c5c0e5000919e6de187906cfd81fb1c72bf9d88c01177e77da7f80",
+                "sha256:d384164d944190a7dd7ef22c6aa3ff197da12962bd04b17f64d4e93d934dba5b"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.3.0"
+        },
+        "argon2-cffi-bindings": {
+            "hashes": [
+                "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670",
+                "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f",
+                "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583",
+                "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194",
+                "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c",
+                "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a",
+                "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082",
+                "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5",
+                "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f",
+                "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7",
+                "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d",
+                "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f",
+                "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae",
+                "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3",
+                "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86",
+                "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367",
+                "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d",
+                "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93",
+                "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb",
+                "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e",
+                "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.2.0"
+        },
+        "asttokens": {
+            "hashes": [
+                "sha256:c61e16246ecfb2cde2958406b4c8ebc043c9e6d73aaa83c941673b35e5d3a76b",
+                "sha256:e3305297c744ae53ffa032c45dc347286165e4ffce6875dc662b205db0623d86"
+            ],
+            "version": "==2.0.8"
+        },
         "attrs": {
             "hashes": [
                 "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6",
@@ -31,6 +81,53 @@
             "markers": "python_version >= '3.5'",
             "version": "==22.1.0"
         },
+        "babel": {
+            "hashes": [
+                "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51",
+                "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.10.3"
+        },
+        "backcall": {
+            "hashes": [
+                "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e",
+                "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"
+            ],
+            "version": "==0.2.0"
+        },
+        "beautifulsoup4": {
+            "hashes": [
+                "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
+                "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==4.11.1"
+        },
+        "bleach": {
+            "hashes": [
+                "sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a",
+                "sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==5.0.1"
+        },
+        "boto3": {
+            "hashes": [
+                "sha256:09fb94008bd1f2566ef79ec7d8dd7a233b5da67f3b2536e30432eb8e88022d4d",
+                "sha256:9fab8acec76008797be4671d514f39f1163ba044c0a0bce8d5afe8b52cbd5550"
+            ],
+            "index": "pypi",
+            "version": "==1.24.56"
+        },
+        "botocore": {
+            "hashes": [
+                "sha256:81101abab2013992a84019739d85a6fa8ec6f1f42fb09ad07e0bf4c8f0efce60",
+                "sha256:81df81b1a1c5608b3a7aca6a837acb27552c3ecc92584a58b171c60754b1d4eb"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.27.56"
+        },
         "certifi": {
             "hashes": [
                 "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
@@ -39,6 +136,75 @@
             "markers": "python_version >= '3.6'",
             "version": "==2022.6.15"
         },
+        "cffi": {
+            "hashes": [
+                "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5",
+                "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef",
+                "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104",
+                "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426",
+                "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405",
+                "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375",
+                "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a",
+                "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e",
+                "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc",
+                "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf",
+                "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185",
+                "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497",
+                "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3",
+                "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35",
+                "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c",
+                "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83",
+                "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21",
+                "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca",
+                "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984",
+                "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac",
+                "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd",
+                "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee",
+                "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a",
+                "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2",
+                "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192",
+                "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7",
+                "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585",
+                "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f",
+                "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e",
+                "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27",
+                "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b",
+                "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e",
+                "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e",
+                "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d",
+                "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c",
+                "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415",
+                "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82",
+                "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02",
+                "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314",
+                "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325",
+                "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c",
+                "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3",
+                "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914",
+                "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045",
+                "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d",
+                "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9",
+                "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5",
+                "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2",
+                "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c",
+                "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3",
+                "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2",
+                "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8",
+                "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d",
+                "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d",
+                "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9",
+                "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162",
+                "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76",
+                "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4",
+                "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e",
+                "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9",
+                "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6",
+                "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b",
+                "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01",
+                "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"
+            ],
+            "version": "==1.15.1"
+        },
         "cftime": {
             "hashes": [
                 "sha256:00aef598f14faa77e48518728bd0cead585e45da89be5f7eae7be8f41c10b401",
@@ -72,6 +238,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.6.1"
         },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
+                "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.1.1"
+        },
         "click": {
             "hashes": [
                 "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
@@ -103,6 +277,68 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.11.0"
         },
+        "debugpy": {
+            "hashes": [
+                "sha256:34d2cdd3a7c87302ba5322b86e79c32c2115be396f3f09ca13306d8a04fe0f16",
+                "sha256:3c9f985944a30cfc9ae4306ac6a27b9c31dba72ca943214dad4a0ab3840f6161",
+                "sha256:4e255982552b0edfe3a6264438dbd62d404baa6556a81a88f9420d3ed79b06ae",
+                "sha256:5ad571a36cec137ae6ed951d0ff75b5e092e9af6683da084753231150cbc5b25",
+                "sha256:6efc30325b68e451118b795eff6fe8488253ca3958251d5158106d9c87581bc6",
+                "sha256:7c302095a81be0d5c19f6529b600bac971440db3e226dce85347cc27e6a61908",
+                "sha256:84c39940a0cac410bf6aa4db00ba174f973eef521fbe9dd058e26bcabad89c4f",
+                "sha256:86d784b72c5411c833af1cd45b83d80c252b77c3bfdb43db17c441d772f4c734",
+                "sha256:adcfea5ea06d55d505375995e150c06445e2b20cd12885bcae566148c076636b",
+                "sha256:b8deaeb779699350deeed835322730a3efec170b88927debc9ba07a1a38e2585",
+                "sha256:c4b2bd5c245eeb49824bf7e539f95fb17f9a756186e51c3e513e32999d8846f3",
+                "sha256:c4cd6f37e3c168080d61d698390dfe2cd9e74ebf80b448069822a15dadcda57d",
+                "sha256:cca23cb6161ac89698d629d892520327dd1be9321c0960e610bbcb807232b45d",
+                "sha256:d5c814596a170a0a58fa6fad74947e30bfd7e192a5d2d7bd6a12156c2899e13a",
+                "sha256:daadab4403427abd090eccb38d8901afd8b393e01fd243048fab3f1d7132abb4",
+                "sha256:dda8652520eae3945833e061cbe2993ad94a0b545aebd62e4e6b80ee616c76b2",
+                "sha256:e8922090514a890eec99cfb991bab872dd2e353ebb793164d5f01c362b9a40bf",
+                "sha256:fc233a0160f3b117b20216f1169e7211b83235e3cd6749bcdd8dbb72177030c7"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.6.3"
+        },
+        "decorator": {
+            "hashes": [
+                "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330",
+                "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==5.1.1"
+        },
+        "defusedxml": {
+            "hashes": [
+                "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
+                "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.7.1"
+        },
+        "entrypoints": {
+            "hashes": [
+                "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4",
+                "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.4"
+        },
+        "executing": {
+            "hashes": [
+                "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e",
+                "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"
+            ],
+            "version": "==0.10.0"
+        },
+        "fastjsonschema": {
+            "hashes": [
+                "sha256:2f7158c4de792555753d6c2277d6a2af2d406dfd97aeca21d17173561ede4fe6",
+                "sha256:d6fa3ffbe719768d70e298b9fb847484e2bdfdb7241ed052b8d57a9294a8c334"
+            ],
+            "version": "==2.16.1"
+        },
         "fiona": {
             "hashes": [
                 "sha256:2563d9a9f21390a7376556c82d9cef90a20a362949a9c7cb66f4ce63d90b66cd",
@@ -121,11 +357,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:9a1c52488045cd6c6491fd07711a380f932466e317cb8e016fc4e99dc7eac2f0",
-                "sha256:d73f25b283cd8033367451122aa868a23de0734757a01984e4b30b18b9050c72"
+                "sha256:cb91ef8d5a435d90aeb3ab814b2548c6b515df5bc13b4c5adaa23778f2f79823",
+                "sha256:e637d2fe06bddabbfc488e02ef32d04d561e3c71e9ba11abc7782ea753ceb218"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.34.4"
+            "version": "==4.36.0"
         },
         "geopandas": {
             "hashes": [
@@ -159,6 +395,181 @@
             "index": "pypi",
             "version": "==3.4.0"
         },
+        "idna": {
+            "hashes": [
+                "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
+                "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==3.3"
+        },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670",
+                "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"
+            ],
+            "markers": "python_version < '3.10'",
+            "version": "==4.12.0"
+        },
+        "importlib-resources": {
+            "hashes": [
+                "sha256:5481e97fb45af8dcf2f798952625591c58fe599d0735d86b10f54de086a61681",
+                "sha256:f78a8df21a79bcc30cfd400bdc38f314333de7c0fb619763f6b9dabab8268bb7"
+            ],
+            "markers": "python_version < '3.9'",
+            "version": "==5.9.0"
+        },
+        "ipykernel": {
+            "hashes": [
+                "sha256:37acc3254caa8a0dafcddddc8dc863a60ad1b46487b68aee361d9a15bda98112",
+                "sha256:d8969c5b23b0e453a23166da5a669c954db399789293fcb03fec5cb25367e43c"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.15.1"
+        },
+        "ipympl": {
+            "hashes": [
+                "sha256:3505b472d412178fd115e255764d1e7e7632d6c82e9de6d62b25438a35374680",
+                "sha256:64096900c6392d53761a68ef6dd07541cc75672f9c3f84a3bb44bf8f9bbc3787"
+            ],
+            "index": "pypi",
+            "version": "==0.9.1"
+        },
+        "ipython": {
+            "hashes": [
+                "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1",
+                "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==8.4.0"
+        },
+        "ipython-genutils": {
+            "hashes": [
+                "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8",
+                "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"
+            ],
+            "version": "==0.2.0"
+        },
+        "ipywidgets": {
+            "hashes": [
+                "sha256:3d47a7826cc6e2644d7cb90db26699451f8b42379cf63b761431b63d19984ca2",
+                "sha256:449ab8e7872d0f388ee5c5b3666b9d6af5e5618a5749fd62652680be37dff2af"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==7.7.2"
+        },
+        "jedi": {
+            "hashes": [
+                "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d",
+                "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.18.1"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
+                "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.1.2"
+        },
+        "jmespath": {
+            "hashes": [
+                "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
+                "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.1"
+        },
+        "json5": {
+            "hashes": [
+                "sha256:993189671e7412e9cdd8be8dc61cf402e8e579b35f1d1bb20ae6b09baa78bbce",
+                "sha256:ad9f048c5b5a4c3802524474ce40a622fae789860a86f10cc4f7e5f9cf9b46ab"
+            ],
+            "version": "==0.9.10"
+        },
+        "jsonschema": {
+            "hashes": [
+                "sha256:3776512df4f53f74e6e28fe35717b5b223c1756875486984a31bc9165e7fc920",
+                "sha256:870a61bb45050b81103faf6a4be00a0a906e06636ffcf0b84f5a2e51faf901ff"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==4.13.0"
+        },
+        "jupyter": {
+            "hashes": [
+                "sha256:3e1f86076bbb7c8c207829390305a2b1fe836d471ed54be66a3b8c41e7f46cc7",
+                "sha256:5b290f93b98ffbc21c0c7e749f054b3267782166d72fa5e3ed1ed4eaf34a2b78",
+                "sha256:d9dc4b3318f310e34c82951ea5d6683f67bed7def4b259fafbfe4f1beb1d8e5f"
+            ],
+            "index": "pypi",
+            "version": "==1.0.0"
+        },
+        "jupyter-client": {
+            "hashes": [
+                "sha256:17d74b0d0a7b24f1c8c527b24fcf4607c56bee542ffe8e3418e50b21e514b621",
+                "sha256:aa9a6c32054b290374f95f73bb0cae91455c58dfb84f65c8591912b8f65e6d56"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==7.3.4"
+        },
+        "jupyter-console": {
+            "hashes": [
+                "sha256:172f5335e31d600df61613a97b7f0352f2c8250bbd1092ef2d658f77249f89fb",
+                "sha256:756df7f4f60c986e7bc0172e4493d3830a7e6e75c08750bbe59c0a5403ad6dee"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.4.4"
+        },
+        "jupyter-core": {
+            "hashes": [
+                "sha256:2e5f244d44894c4154d06aeae3419dd7f1b0ef4494dc5584929b398c61cfd314",
+                "sha256:715e22bb6cc7db3718fddfac1f69f1c7e899ca00e42bdfd4bf3705452b9fd84a"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==4.11.1"
+        },
+        "jupyter-server": {
+            "hashes": [
+                "sha256:022759b09c96a4e2feb95de59ce4283e04e17782efe197b91d23a47521609b77",
+                "sha256:2b72fc595bccae292260aad8157a0ead8da2c703ec6ae1bb7b36dbad0e267ea7"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.18.1"
+        },
+        "jupyterlab": {
+            "hashes": [
+                "sha256:3181d60518e6101b9e43f6da231aa4be812a59d81ddcc8fe680dd33066fddae6",
+                "sha256:472f6b7996c75f6991592483c26d9fe205a59a71ccbce15842400155dc64f59b"
+            ],
+            "index": "pypi",
+            "version": "==3.4.5"
+        },
+        "jupyterlab-pygments": {
+            "hashes": [
+                "sha256:2405800db07c9f770863bcf8049a529c3dd4d3e28536638bd7c1c01d2748309f",
+                "sha256:7405d7fde60819d905a9fa8ce89e4cd830e318cdad22a0030f7a901da705585d"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.2.2"
+        },
+        "jupyterlab-server": {
+            "hashes": [
+                "sha256:0e327d7a346874fd8e94c1bcbd69906d18a8558df8f13115c5afd183c3107756",
+                "sha256:a91c515e0e7971a8f7c3c9834b748857f7dac502f93604bf283987991fd987ef"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.15.0"
+        },
+        "jupyterlab-widgets": {
+            "hashes": [
+                "sha256:67d0ef1e407e0c42c8ab60b9d901cd7a4c68923650763f75bf17fb06c1943b79",
+                "sha256:90ab47d99da03a3697074acb23b2975ead1d6171aa41cb2812041a7f2a08177a"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.1.1"
+        },
         "kiwisolver": {
             "hashes": [
                 "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b",
@@ -166,42 +577,67 @@
                 "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c",
                 "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c",
                 "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0",
+                "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4",
+                "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9",
+                "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286",
+                "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767",
                 "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c",
                 "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6",
+                "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b",
                 "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004",
                 "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf",
+                "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494",
                 "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac",
+                "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626",
                 "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766",
+                "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514",
                 "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6",
+                "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f",
                 "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d",
                 "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191",
                 "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d",
+                "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51",
                 "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f",
+                "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8",
                 "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454",
+                "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb",
+                "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da",
                 "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8",
                 "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de",
                 "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a",
                 "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9",
+                "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008",
                 "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3",
                 "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32",
                 "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938",
                 "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1",
+                "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9",
                 "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d",
                 "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824",
                 "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b",
                 "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd",
+                "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2",
+                "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5",
+                "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69",
                 "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3",
                 "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae",
                 "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597",
+                "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e",
                 "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955",
+                "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca",
                 "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a",
                 "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea",
                 "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede",
+                "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4",
+                "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6",
+                "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686",
                 "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408",
                 "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871",
                 "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29",
+                "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750",
                 "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897",
                 "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0",
+                "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2",
                 "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09",
                 "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"
             ],
@@ -230,46 +666,176 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.33.0"
         },
+        "lxml": {
+            "hashes": [
+                "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318",
+                "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c",
+                "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b",
+                "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000",
+                "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73",
+                "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d",
+                "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb",
+                "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8",
+                "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2",
+                "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345",
+                "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94",
+                "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e",
+                "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b",
+                "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc",
+                "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a",
+                "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9",
+                "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc",
+                "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387",
+                "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb",
+                "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7",
+                "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4",
+                "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97",
+                "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67",
+                "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627",
+                "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7",
+                "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd",
+                "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3",
+                "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7",
+                "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130",
+                "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b",
+                "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036",
+                "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785",
+                "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca",
+                "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91",
+                "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc",
+                "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536",
+                "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391",
+                "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3",
+                "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d",
+                "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21",
+                "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3",
+                "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d",
+                "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29",
+                "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715",
+                "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed",
+                "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25",
+                "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c",
+                "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785",
+                "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837",
+                "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4",
+                "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b",
+                "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2",
+                "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067",
+                "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448",
+                "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d",
+                "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2",
+                "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc",
+                "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c",
+                "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5",
+                "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84",
+                "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8",
+                "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf",
+                "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7",
+                "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e",
+                "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb",
+                "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b",
+                "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3",
+                "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad",
+                "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8",
+                "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==4.9.1"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003",
+                "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88",
+                "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5",
+                "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7",
+                "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a",
+                "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603",
+                "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1",
+                "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135",
+                "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247",
+                "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6",
+                "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601",
+                "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77",
+                "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02",
+                "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e",
+                "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63",
+                "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f",
+                "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980",
+                "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b",
+                "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812",
+                "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff",
+                "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96",
+                "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1",
+                "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925",
+                "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a",
+                "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6",
+                "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e",
+                "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f",
+                "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4",
+                "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f",
+                "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3",
+                "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c",
+                "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a",
+                "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417",
+                "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a",
+                "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a",
+                "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37",
+                "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452",
+                "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933",
+                "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a",
+                "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.1.1"
+        },
         "matplotlib": {
             "hashes": [
-                "sha256:03bbb3f5f78836855e127b5dab228d99551ad0642918ccbf3067fcd52ac7ac5e",
-                "sha256:24173c23d1bcbaed5bf47b8785d27933a1ac26a5d772200a0f3e0e38f471b001",
-                "sha256:2a0967d4156adbd0d46db06bc1a877f0370bce28d10206a5071f9ecd6dc60b79",
-                "sha256:2e8bda1088b941ead50caabd682601bece983cadb2283cafff56e8fcddbf7d7f",
-                "sha256:31fbc2af27ebb820763f077ec7adc79b5a031c2f3f7af446bd7909674cd59460",
-                "sha256:364e6bca34edc10a96aa3b1d7cd76eb2eea19a4097198c1b19e89bee47ed5781",
-                "sha256:3d8e129af95b156b41cb3be0d9a7512cc6d73e2b2109f82108f566dbabdbf377",
-                "sha256:44c6436868186564450df8fd2fc20ed9daaef5caad699aa04069e87099f9b5a8",
-                "sha256:48cf850ce14fa18067f2d9e0d646763681948487a8080ec0af2686468b4607a2",
-                "sha256:49a5938ed6ef9dda560f26ea930a2baae11ea99e1c2080c8714341ecfda72a89",
-                "sha256:4a05f2b37222319753a5d43c0a4fd97ed4ff15ab502113e3f2625c26728040cf",
-                "sha256:4a44cdfdb9d1b2f18b1e7d315eb3843abb097869cd1ef89cfce6a488cd1b5182",
-                "sha256:4fa28ca76ac5c2b2d54bc058b3dad8e22ee85d26d1ee1b116a6fd4d2277b6a04",
-                "sha256:5844cea45d804174bf0fac219b4ab50774e504bef477fc10f8f730ce2d623441",
-                "sha256:5a32ea6e12e80dedaca2d4795d9ed40f97bfa56e6011e14f31502fdd528b9c89",
-                "sha256:6c623b355d605a81c661546af7f24414165a8a2022cddbe7380a31a4170fa2e9",
-                "sha256:751d3815b555dcd6187ad35b21736dc12ce6925fc3fa363bbc6dc0f86f16484f",
-                "sha256:75c406c527a3aa07638689586343f4b344fcc7ab1f79c396699eb550cd2b91f7",
-                "sha256:77157be0fc4469cbfb901270c205e7d8adb3607af23cef8bd11419600647ceed",
-                "sha256:7d7705022df2c42bb02937a2a824f4ec3cca915700dd80dc23916af47ff05f1a",
-                "sha256:7f409716119fa39b03da3d9602bd9b41142fab7a0568758cd136cd80b1bf36c8",
-                "sha256:9480842d5aadb6e754f0b8f4ebeb73065ac8be1855baa93cd082e46e770591e9",
-                "sha256:9776e1a10636ee5f06ca8efe0122c6de57ffe7e8c843e0fb6e001e9d9256ec95",
-                "sha256:a91426ae910819383d337ba0dc7971c7cefdaa38599868476d94389a329e599b",
-                "sha256:b4fedaa5a9aa9ce14001541812849ed1713112651295fdddd640ea6620e6cf98",
-                "sha256:b6c63cd01cad0ea8704f1fd586e9dc5777ccedcd42f63cbbaa3eae8dd41172a1",
-                "sha256:b8d3f4e71e26307e8c120b72c16671d70c5cd08ae412355c11254aa8254fb87f",
-                "sha256:c4b82c2ae6d305fcbeb0eb9c93df2602ebd2f174f6e8c8a5d92f9445baa0c1d3",
-                "sha256:c772264631e5ae61f0bd41313bbe48e1b9bcc95b974033e1118c9caa1a84d5c6",
-                "sha256:c87973ddec10812bddc6c286b88fdd654a666080fbe846a1f7a3b4ba7b11ab78",
-                "sha256:e2b696699386766ef171a259d72b203a3c75d99d03ec383b97fc2054f52e15cf",
-                "sha256:ea75df8e567743207e2b479ba3d8843537be1c146d4b1e3e395319a4e1a77fe9",
-                "sha256:ebc27ad11df3c1661f4677a7762e57a8a91dd41b466c3605e90717c9a5f90c82",
-                "sha256:ee0b8e586ac07f83bb2950717e66cb305e2859baf6f00a9c39cc576e0ce9629c",
-                "sha256:ee175a571e692fc8ae8e41ac353c0e07259113f4cb063b0ec769eff9717e84bb"
+                "sha256:0bcdfcb0f976e1bac6721d7d457c17be23cf7501f977b6a38f9d38a3762841f7",
+                "sha256:1e64ac9be9da6bfff0a732e62116484b93b02a0b4d4b19934fb4f8e7ad26ad6a",
+                "sha256:22227c976ad4dc8c5a5057540421f0d8708c6560744ad2ad638d48e2984e1dbc",
+                "sha256:2886cc009f40e2984c083687251821f305d811d38e3df8ded414265e4583f0c5",
+                "sha256:2e6d184ebe291b9e8f7e78bbab7987d269c38ea3e062eace1fe7d898042ef804",
+                "sha256:3211ba82b9f1518d346f6309df137b50c3dc4421b4ed4815d1d7eadc617f45a1",
+                "sha256:339cac48b80ddbc8bfd05daae0a3a73414651a8596904c2a881cfd1edb65f26c",
+                "sha256:35a8ad4dddebd51f94c5d24bec689ec0ec66173bf614374a1244c6241c1595e0",
+                "sha256:3b4fa56159dc3c7f9250df88f653f085068bcd32dcd38e479bba58909254af7f",
+                "sha256:43e9d3fa077bf0cc95ded13d331d2156f9973dce17c6f0c8b49ccd57af94dbd9",
+                "sha256:57f1b4e69f438a99bb64d7f2c340db1b096b41ebaa515cf61ea72624279220ce",
+                "sha256:5c096363b206a3caf43773abebdbb5a23ea13faef71d701b21a9c27fdcef72f4",
+                "sha256:6bb93a0492d68461bd458eba878f52fdc8ac7bdb6c4acdfe43dba684787838c2",
+                "sha256:6ea6aef5c4338e58d8d376068e28f80a24f54e69f09479d1c90b7172bad9f25b",
+                "sha256:6fe807e8a22620b4cd95cfbc795ba310dc80151d43b037257250faf0bfcd82bc",
+                "sha256:73dd93dc35c85dece610cca8358003bf0760d7986f70b223e2306b4ea6d1406b",
+                "sha256:839d47b8ead7ad9669aaacdbc03f29656dc21f0d41a6fea2d473d856c39c8b1c",
+                "sha256:874df7505ba820e0400e7091199decf3ff1fde0583652120c50cd60d5820ca9a",
+                "sha256:879c7e5fce4939c6aa04581dfe08d57eb6102a71f2e202e3314d5fbc072fd5a0",
+                "sha256:94ff86af56a3869a4ae26a9637a849effd7643858a1a04dd5ee50e9ab75069a7",
+                "sha256:99482b83ebf4eb6d5fc6813d7aacdefdd480f0d9c0b52dcf9f1cc3b2c4b3361a",
+                "sha256:9ab29589cef03bc88acfa3a1490359000c18186fc30374d8aa77d33cc4a51a4a",
+                "sha256:9befa5954cdbc085e37d974ff6053da269474177921dd61facdad8023c4aeb51",
+                "sha256:a206a1b762b39398efea838f528b3a6d60cdb26fe9d58b48265787e29cd1d693",
+                "sha256:ab8d26f07fe64f6f6736d635cce7bfd7f625320490ed5bfc347f2cdb4fae0e56",
+                "sha256:b28de401d928890187c589036857a270a032961411934bdac4cf12dde3d43094",
+                "sha256:b428076a55fb1c084c76cb93e68006f27d247169f056412607c5c88828d08f88",
+                "sha256:bf618a825deb6205f015df6dfe6167a5d9b351203b03fab82043ae1d30f16511",
+                "sha256:c995f7d9568f18b5db131ab124c64e51b6820a92d10246d4f2b3f3a66698a15b",
+                "sha256:cd45a6f3e93a780185f70f05cf2a383daed13c3489233faad83e81720f7ede24",
+                "sha256:d2484b350bf3d32cae43f85dcfc89b3ed7bd2bcd781ef351f93eb6fb2cc483f9",
+                "sha256:d62880e1f60e5a30a2a8484432bcb3a5056969dc97258d7326ad465feb7ae069",
+                "sha256:dacddf5bfcec60e3f26ec5c0ae3d0274853a258b6c3fc5ef2f06a8eb23e042be",
+                "sha256:f3840c280ebc87a48488a46f760ea1c0c0c83fcf7abbe2e6baf99d033fd35fd8",
+                "sha256:f814504e459c68118bf2246a530ed953ebd18213dc20e3da524174d84ed010b2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.5.2"
+            "version": "==3.5.3"
+        },
+        "matplotlib-inline": {
+            "hashes": [
+                "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311",
+                "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==0.1.6"
         },
         "memory-profiler": {
             "hashes": [
@@ -278,6 +844,13 @@
             "index": "pypi",
             "version": "==0.60.0"
         },
+        "mistune": {
+            "hashes": [
+                "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e",
+                "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4"
+            ],
+            "version": "==0.8.4"
+        },
         "munch": {
             "hashes": [
                 "sha256:2d735f6f24d4dba3417fa448cae40c6e896ec1fdab6cdb5e6510999758a4dbd2",
@@ -293,6 +866,46 @@
             "index": "pypi",
             "version": "==8.1.0"
         },
+        "nbclassic": {
+            "hashes": [
+                "sha256:4b01076effdac53e775cd1b6a4e891663568b32621468e205b502a23b2921899",
+                "sha256:f03111b2cebaa69b88370a7b23b19b2b37c9bb71767f1828cdfd7a047eae8edd"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.4.3"
+        },
+        "nbclient": {
+            "hashes": [
+                "sha256:09bae4ea2df79fa6bc50aeb8278d8b79d2036792824337fa6eee834afae17312",
+                "sha256:0df76a7961d99a681b4796c74a1f2553b9f998851acc01896dce064ad19a9027"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.6.6"
+        },
+        "nbconvert": {
+            "hashes": [
+                "sha256:10ed693c4cfd3c63583c87ca5c3a2f6ed874145103595f3824efcc8dfcb7522c",
+                "sha256:2564bb5125d862949f72475de0c0348392add7ea62cc950985347bfe7bbc2034"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.5.3"
+        },
+        "nbformat": {
+            "hashes": [
+                "sha256:0d6072aaec95dddc39735c144ee8bbc6589c383fb462e4058abc855348152dad",
+                "sha256:44ba5ca6acb80c5d5a500f1e5b83ede8cbe364d5a495c4c8cf60aaf1ba656501"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==5.4.0"
+        },
+        "nest-asyncio": {
+            "hashes": [
+                "sha256:b98e3ec1b246135e4642eceffa5a6c23a3ab12c82ff816a92c612d68205813b2",
+                "sha256:e442291cd942698be619823a17a86a5759eabe1f8613084790de189fe9e16d65"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.5.5"
+        },
         "netcdf4": {
             "hashes": [
                 "sha256:02c2317f377f23005e447e4b187ebce3cdeae76b9d0a443d9d6f5156a3980f04",
@@ -334,6 +947,22 @@
             "index": "pypi",
             "version": "==1.5.7"
         },
+        "notebook": {
+            "hashes": [
+                "sha256:6268c9ec9048cff7a45405c990c29ac9ca40b0bc3ec29263d218c5e01f2b4e86",
+                "sha256:8c07a3bb7640e371f8a609bdbb2366a1976c6a2589da8ef917f761a61e3ad8b1"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.4.12"
+        },
+        "notebook-shim": {
+            "hashes": [
+                "sha256:02432d55a01139ac16e2100888aa2b56c614720cec73a27e71f40a5387e45324",
+                "sha256:7897e47a36d92248925a2143e3596f19c60597708f7bef50d81fcd31d7263e85"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.1.0"
+        },
         "numba": {
             "hashes": [
                 "sha256:24852c21fbf7edf9e000eeec9fbd1b24d1ca17c86ae449b06a3707bcdec95479",
@@ -447,6 +1076,37 @@
             "index": "pypi",
             "version": "==1.0.5"
         },
+        "pandocfilters": {
+            "hashes": [
+                "sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38",
+                "sha256:33aae3f25fd1a026079f5d27bdd52496f0e0803b3469282162bafdcbdf6ef14f"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.5.0"
+        },
+        "parso": {
+            "hashes": [
+                "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0",
+                "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.8.3"
+        },
+        "pexpect": {
+            "hashes": [
+                "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
+                "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"
+            ],
+            "markers": "sys_platform != 'win32'",
+            "version": "==4.8.0"
+        },
+        "pickleshare": {
+            "hashes": [
+                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
+                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
+            ],
+            "version": "==0.7.5"
+        },
         "pillow": {
             "hashes": [
                 "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927",
@@ -511,6 +1171,30 @@
             "markers": "python_version >= '3.7'",
             "version": "==9.2.0"
         },
+        "pkgutil-resolve-name": {
+            "hashes": [
+                "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174",
+                "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"
+            ],
+            "markers": "python_version < '3.9'",
+            "version": "==1.3.10"
+        },
+        "prometheus-client": {
+            "hashes": [
+                "sha256:522fded625282822a89e2773452f42df14b5a8e84a86433e3f8a189c1d54dc01",
+                "sha256:5459c427624961076277fdc6dc50540e2bacb98eebde99886e59ec55ed92093a"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.14.1"
+        },
+        "prompt-toolkit": {
+            "hashes": [
+                "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0",
+                "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"
+            ],
+            "markers": "python_full_version >= '3.6.2'",
+            "version": "==3.0.30"
+        },
         "psutil": {
             "hashes": [
                 "sha256:068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685",
@@ -590,6 +1274,28 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "ptyprocess": {
+            "hashes": [
+                "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35",
+                "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"
+            ],
+            "markers": "os_name != 'nt'",
+            "version": "==0.7.0"
+        },
+        "pure-eval": {
+            "hashes": [
+                "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350",
+                "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"
+            ],
+            "version": "==0.2.2"
+        },
+        "pycparser": {
+            "hashes": [
+                "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
+                "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
+            ],
+            "version": "==2.21"
+        },
         "pygeos": {
             "hashes": [
                 "sha256:0dbb816ebede8bb87f4e4c87c2303b9ed9e4281f8c4cac2263e7e2ac2271a8dc",
@@ -612,6 +1318,14 @@
             "index": "pypi",
             "version": "==0.7.1"
         },
+        "pygments": {
+            "hashes": [
+                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
+                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.13.0"
+        },
         "pyparsing": {
             "hashes": [
                 "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
@@ -645,6 +1359,33 @@
             "index": "pypi",
             "version": "==3.1.0"
         },
+        "pyrsistent": {
+            "hashes": [
+                "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c",
+                "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc",
+                "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e",
+                "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26",
+                "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec",
+                "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286",
+                "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045",
+                "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec",
+                "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8",
+                "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c",
+                "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca",
+                "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22",
+                "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a",
+                "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96",
+                "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc",
+                "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1",
+                "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07",
+                "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6",
+                "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b",
+                "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5",
+                "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.18.1"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
@@ -663,10 +1404,105 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7",
-                "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"
+                "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197",
+                "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5"
+            ],
+            "version": "==2022.2.1"
+        },
+        "pyzmq": {
+            "hashes": [
+                "sha256:022cf5ea7bcaa8a06a03c2706e0ae66904b6138b2155577cd34c64bc7cc637ab",
+                "sha256:044447ae4b2016a6b8697571fd633f799f860b19b76c4a2fd9b1140d52ee6745",
+                "sha256:07ed8aaf7ffe150af873269690cc654ffeca7491f62aae0f3821baa181f8d5fe",
+                "sha256:10d1910ec381b851aeb024a042a13db178cb1edf125e76a4e9d2548ad103aadb",
+                "sha256:12e62ff0d5223ec09b597ab6d73858b9f64a51221399f3cb08aa495e1dff7935",
+                "sha256:1f368a82b29f80071781b20663c0fc0c8f6b13273f9f5abe1526af939534f90f",
+                "sha256:20bafc4095eab00f41a510579363a3f5e1f5c69d7ee10f1d88895c4df0259183",
+                "sha256:2141e6798d5981be04c08996d27962086a1aa3ea536fe9cf7e89817fd4523f86",
+                "sha256:23e708fbfdf4ee3107422b69ca65da1b9f056b431fc0888096a8c1d6cd908e8f",
+                "sha256:28dbdb90b2f6b131f8f10e6081012e4e25234213433420e67e0c1162de537113",
+                "sha256:29b74774a0bfd3c4d98ac853f0bdca55bd9ec89d5b0def5486407cca54472ef8",
+                "sha256:2b381aa867ece7d0a82f30a0c7f3d4387b7cf2e0697e33efaa5bed6c5784abcd",
+                "sha256:2f67b63f53c6994d601404fd1a329e6d940ac3dd1d92946a93b2b9c70df67b9f",
+                "sha256:342ca3077f47ec2ee41b9825142b614e03e026347167cbc72a59b618c4f6106c",
+                "sha256:35e635343ff367f697d00fa1484262bb68e36bc74c9b80737eac5a1e04c4e1b1",
+                "sha256:385609812eafd9970c3752c51f2f6c4f224807e3e441bcfd8c8273877d00c8a8",
+                "sha256:38e106b64bad744fe469dc3dd864f2764d66399178c1bf39d45294cc7980f14f",
+                "sha256:39dd252b683816935702825e5bf775df16090619ced9bb4ba68c2d0b6f0c9b18",
+                "sha256:407f909c4e8fde62fbdad9ebd448319792258cc0550c2815567a4d9d8d9e6d18",
+                "sha256:415ff62ac525d9add1e3550430a09b9928d2d24a20cc4ce809e67caac41219ab",
+                "sha256:4805af9614b0b41b7e57d17673459facf85604dac502a5a9244f6e8c9a4de658",
+                "sha256:48400b96788cdaca647021bf19a9cd668384f46e4d9c55cf045bdd17f65299c8",
+                "sha256:49d30ba7074f469e8167917abf9eb854c6503ae10153034a6d4df33618f1db5f",
+                "sha256:4bb798bef181648827019001f6be43e1c48b34b477763b37a8d27d8c06d197b8",
+                "sha256:4d6f110c56f7d5b4d64dde3a382ae61b6d48174e30742859d8e971b18b6c9e5c",
+                "sha256:55568a020ad2cae9ae36da6058e7ca332a56df968f601cbdb7cf6efb2a77579a",
+                "sha256:565bd5ab81f6964fc4067ccf2e00877ad0fa917308975694bbb54378389215f8",
+                "sha256:5c558b50402fca1acc94329c5d8f12aa429738904a5cfb32b9ed3c61235221bb",
+                "sha256:5e05492be125dce279721d6b54fd1b956546ecc4bcdfcf8e7b4c413bc0874c10",
+                "sha256:624fd38071a817644acdae075b92a23ea0bdd126a58148288e8284d23ec361ce",
+                "sha256:650389bbfca73955b262b2230423d89992f38ec48033307ae80e700eaa2fbb63",
+                "sha256:67975a9e1237b9ccc78f457bef17691bbdd2055a9d26e81ee914ba376846d0ce",
+                "sha256:6b1e79bba24f6df1712e3188d5c32c480d8eda03e8ecff44dc8ecb0805fa62f3",
+                "sha256:6fd5d0d50cbcf4bc376861529a907bed026a4cbe8c22a500ff8243231ef02433",
+                "sha256:71b32a1e827bdcbf73750e60370d3b07685816ff3d8695f450f0f8c3226503f8",
+                "sha256:794871988c34727c7f79bdfe2546e6854ae1fa2e1feb382784f23a9c6c63ecb3",
+                "sha256:79a87831b47a9f6161ad23fa5e89d5469dc585abc49f90b9b07fea8905ae1234",
+                "sha256:7e0113d70b095339e99bb522fe7294f5ae6a7f3b2b8f52f659469a74b5cc7661",
+                "sha256:84678153432241bcdca2210cf4ff83560b200556867aea913ffbb960f5d5f340",
+                "sha256:8a68f57b7a3f7b6b52ada79876be1efb97c8c0952423436e84d70cc139f16f0d",
+                "sha256:8c02a0cd39dc01659b3d6cb70bb3a41aebd9885fd78239acdd8d9c91351c4568",
+                "sha256:8c842109d31a9281d678f668629241c405928afbebd913c48a5a8e7aee61f63d",
+                "sha256:8dc66f109a245653b19df0f44a5af7a3f14cb8ad6c780ead506158a057bd36ce",
+                "sha256:90d88f9d9a2ae6cfb1dc4ea2d1710cdf6456bc1b9a06dd1bb485c5d298f2517e",
+                "sha256:9269fbfe3a4eb2009199120861c4571ef1655fdf6951c3e7f233567c94e8c602",
+                "sha256:929d548b74c0f82f7f95b54e4a43f9e4ce2523cfb8a54d3f7141e45652304b2a",
+                "sha256:99a5a77a10863493a1ee8dece02578c6b32025fb3afff91b40476bc489e81648",
+                "sha256:9a39ddb0431a68954bd318b923230fa5b649c9c62b0e8340388820c5f1b15bd2",
+                "sha256:9d0ab2936085c85a1fc6f9fd8f89d5235ae99b051e90ec5baa5e73ad44346e1f",
+                "sha256:9e5bf6e7239fc9687239de7a283aa8b801ab85371116045b33ae20132a1325d6",
+                "sha256:a0f09d85c45f58aa8e715b42f8b26beba68b3b63a8f7049113478aca26efbc30",
+                "sha256:a114992a193577cb62233abf8cb2832970f9975805a64740e325d2f895e7f85a",
+                "sha256:a3fd44b5046d247e7f0f1660bcafe7b5fb0db55d0934c05dd57dda9e1f823ce7",
+                "sha256:ad28ddb40db8e450d7d4bf8a1d765d3f87b63b10e7e9a825a3c130c6371a8c03",
+                "sha256:aecd6ceaccc4b594e0092d6513ef3f1c0fa678dd89f86bb8ff1a47014b8fca35",
+                "sha256:b815991c7d024bf461f358ad871f2be1135576274caed5749c4828859e40354e",
+                "sha256:b861db65f6b8906c8d6db51dde2448f266f0c66bf28db2c37aea50f58a849859",
+                "sha256:c3ebf1668664d20c8f7d468955f18379b7d1f7bc8946b13243d050fa3888c7ff",
+                "sha256:c56b1a62a1fb87565343c57b6743fd5da6e138b8c6562361d7d9b5ce4acf399a",
+                "sha256:c780acddd2934c6831ff832ecbf78a45a7b62d4eb216480f863854a8b7d54fa7",
+                "sha256:c890309296f53f9aa32ffcfc51d805705e1982bffd27c9692a8f1e1b8de279f4",
+                "sha256:c9cfaf530e6a7ff65f0afe275e99f983f68b54dfb23ea401f0bc297a632766b6",
+                "sha256:d904f6595acfaaf99a1a61881fea068500c40374d263e5e073aa4005e5f9c28a",
+                "sha256:e06747014a5ad1b28cebf5bc1ddcdaccfb44e9b441d35e6feb1286c8a72e54be",
+                "sha256:e1fe30bcd5aea5948c42685fad910cd285eacb2518ea4dc6c170d6b535bee95d",
+                "sha256:e753eee6d3b93c5354e8ba0a1d62956ee49355f0a36e00570823ef64e66183f5",
+                "sha256:ec9803aca9491fd6f0d853d2a6147f19f8deaaa23b1b713d05c5d09e56ea7142",
+                "sha256:efb9e38b2a590282704269585de7eb33bf43dc294cad092e1b172e23d4c217e5",
+                "sha256:f07016e3cf088dbfc6e7c5a7b3f540db5c23b0190d539e4fd3e2b5e6beffa4b5",
+                "sha256:f392cbea531b7142d1958c0d4a0c9c8d760dc451e5848d8dd3387804d3e3e62c",
+                "sha256:f619fd38fc2641abfb53cca719c165182500600b82c695cc548a0f05f764be05",
+                "sha256:fefdf9b685fda4141b95ebec975946076a5e0723ff70b037032b2085c5317684",
+                "sha256:ffc6b1623d0f9affb351db4ca61f432dca3628a5ee015f9bf2bfbe9c6836881c"
             ],
-            "version": "==2022.1"
+            "markers": "python_version >= '3.6'",
+            "version": "==23.2.1"
+        },
+        "qtconsole": {
+            "hashes": [
+                "sha256:b73723fac43938b684dcb237a88510dc7721c43a726cea8ade179a2927c0a2f3",
+                "sha256:d364592d7ede3257f1e17fcdbfd339c26e2cc638ca4fa4ee56a724e26ea13c81"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==5.3.1"
+        },
+        "qtpy": {
+            "hashes": [
+                "sha256:d283cfba378b0dbe36a55b68aea8ee2f86cd6ccf06c023af25bbe705ffbb29e5",
+                "sha256:d85f1b121f24a41ad26c55c446e66abdb7c528839f8c4f11f156ec4541903914"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.2.0"
         },
         "rasterio": {
             "hashes": [
@@ -692,6 +1528,14 @@
             "index": "pypi",
             "version": "==0.15.0"
         },
+        "requests": {
+            "hashes": [
+                "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
+                "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
+            ],
+            "markers": "python_version >= '3.7' and python_version < '4'",
+            "version": "==2.28.1"
+        },
         "richdem": {
             "hashes": [
                 "sha256:0c5988b58bea942a31ea9f5c5502a6a34aacad5cd1ab6b5e32ac627630a36b9b",
@@ -711,6 +1555,14 @@
             "index": "pypi",
             "version": "==0.3.4"
         },
+        "s3transfer": {
+            "hashes": [
+                "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd",
+                "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.6.0"
+        },
         "scipy": {
             "hashes": [
                 "sha256:01c2015e132774feefe059d5354055fec6b751d7a7d70ad2cf5ce314e7426e2a",
@@ -748,13 +1600,20 @@
             "index": "pypi",
             "version": "==0.11.0"
         },
+        "send2trash": {
+            "hashes": [
+                "sha256:d2c24762fd3759860a0aff155e45871447ea58d2be6bdd39b5c8f966a0c99c2d",
+                "sha256:f20eaadfdb517eaca5ce077640cb261c7d2698385a6a0f072a4a5447fd49fa08"
+            ],
+            "version": "==1.8.0"
+        },
         "setuptools": {
             "hashes": [
-                "sha256:7c7854ee1429a240090297628dc9f75b35318d193537968e2dc14010ee2f5bca",
-                "sha256:dc2662692f47d99cb8ae15a784529adeed535bcd7c277fee0beccf961522baf6"
+                "sha256:10602cd0a6f5feab6656e9587f9075292ab777c5200f3bf00293ecd23d9f2788",
+                "sha256:d2e010624c781b26ad6629a8de9832327cf853dea93894487979e55f9ad06857"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==63.4.1"
+            "version": "==65.1.0"
         },
         "shapely": {
             "hashes": [
@@ -856,6 +1715,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
+        "sniffio": {
+            "hashes": [
+                "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663",
+                "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.2.0"
+        },
         "snuggs": {
             "hashes": [
                 "sha256:501cf113fe3892e14e2fee76da5cd0606b7e149c411c271898e6259ebde2617b",
@@ -863,6 +1730,21 @@
             ],
             "version": "==1.4.7"
         },
+        "soupsieve": {
+            "hashes": [
+                "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759",
+                "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.3.2.post1"
+        },
+        "stack-data": {
+            "hashes": [
+                "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7",
+                "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa"
+            ],
+            "version": "==0.4.0"
+        },
         "tables": {
             "hashes": [
                 "sha256:169450bd11959c0e1c43137e768cf8b60b2a4f3b2ebf9a620e21865dc0c2d059",
@@ -893,6 +1775,39 @@
             "index": "pypi",
             "version": "==3.6.1"
         },
+        "terminado": {
+            "hashes": [
+                "sha256:0d5f126fbfdb5887b25ae7d9d07b0d716b1cc0ccaacc71c1f3c14d228e065197",
+                "sha256:ab4eeedccfcc1e6134bfee86106af90852c69d602884ea3a1e8ca6d4486e9bfe"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.15.0"
+        },
+        "tinycss2": {
+            "hashes": [
+                "sha256:b2e44dd8883c360c35dd0d1b5aad0b610e5156c2cb3b33434634e539ead9d8bf",
+                "sha256:fe794ceaadfe3cf3e686b22155d0da5780dd0e273471a51846d0a02bc204fec8"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.1.1"
+        },
+        "tornado": {
+            "hashes": [
+                "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca",
+                "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72",
+                "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23",
+                "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8",
+                "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b",
+                "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9",
+                "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13",
+                "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75",
+                "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac",
+                "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e",
+                "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.2"
+        },
         "tqdm": {
             "hashes": [
                 "sha256:6baa75a88582b1db6d34ce4690da5501d2a1cb65c34664840a456b2c9f794d29",
@@ -901,6 +1816,52 @@
             "index": "pypi",
             "version": "==4.48.0"
         },
+        "traitlets": {
+            "hashes": [
+                "sha256:0bb9f1f9f017aa8ec187d8b1b2a7a6626a2a1d877116baba52a129bfa124f8e2",
+                "sha256:65fa18961659635933100db8ca120ef6220555286949774b9cfc106f941d1c7a"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==5.3.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc",
+                "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
+            "version": "==1.26.11"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
+                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
+            ],
+            "version": "==0.2.5"
+        },
+        "webencodings": {
+            "hashes": [
+                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
+                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+            ],
+            "version": "==0.5.1"
+        },
+        "websocket-client": {
+            "hashes": [
+                "sha256:5d55652dc1d0b3c734f044337d929aaf83f4f9138816ec680c1aefefb4dc4877",
+                "sha256:d58c5f284d6a9bf8379dab423259fe8f85b70d5fa5d2916d5791a84594b122b1"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.3"
+        },
+        "widgetsnbextension": {
+            "hashes": [
+                "sha256:954e0faefdd414e4e013f17dbc7fd86f24cf1d243a3ac85d5f0fc2c2d2b50c66",
+                "sha256:9c84ae64c2893c7cbe2eaafc7505221a795c27d68938454034ac487319a75b10"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.6.1"
+        },
         "xarray": {
             "hashes": [
                 "sha256:3a365ce09127fc841ba88baa63f37ca61376ffe389a6c5e66d52f2c88c23a62b",
@@ -908,15 +1869,23 @@
             ],
             "index": "pypi",
             "version": "==0.19.0"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2",
+                "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.8.1"
         }
     },
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:0844691e88552595a6f4a4281a9f7f79b8dd45ca4ccea82e5e05b4bbdb76705c",
-                "sha256:9a54c114f02c7a9480d56550932546a3f1fe71d8a02f1bc7ccd0ee3ee35cf4d5"
+                "sha256:c61e16246ecfb2cde2958406b4c8ebc043c9e6d73aaa83c941673b35e5d3a76b",
+                "sha256:e3305297c744ae53ffa032c45dc347286165e4ffce6875dc662b205db0623d86"
             ],
-            "version": "==2.0.5"
+            "version": "==2.0.8"
         },
         "backcall": {
             "hashes": [
@@ -935,17 +1904,17 @@
         },
         "executing": {
             "hashes": [
-                "sha256:4ce4d6082d99361c0231fc31ac1a0f56979363cc6819de0b1410784f99e49105",
-                "sha256:ea278e2cf90cbbacd24f1080dd1f0ac25b71b2e21f50ab439b7ba45dd3195587"
+                "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e",
+                "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"
             ],
-            "version": "==0.9.1"
+            "version": "==0.10.0"
         },
         "ipython": {
             "hashes": [
                 "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1",
                 "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd"
             ],
-            "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==8.4.0"
         },
         "jedi": {
@@ -958,11 +1927,11 @@
         },
         "matplotlib-inline": {
             "hashes": [
-                "sha256:a04bfba22e0d1395479f866853ec1ee28eea1485c1d69a6faf00dc3e24ff34ee",
-                "sha256:aed605ba3b72462d64d475a21a9296f400a19c4f74a31b59103d2a99ffd5aa5c"
+                "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311",
+                "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.1.3"
+            "version": "==0.1.6"
         },
         "parso": {
             "hashes": [
@@ -1000,6 +1969,7 @@
                 "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35",
                 "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"
             ],
+            "markers": "os_name != 'nt'",
             "version": "==0.7.0"
         },
         "pure-eval": {
@@ -1011,19 +1981,19 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb",
-                "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"
+                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
+                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.12.0"
+            "version": "==2.13.0"
         },
         "setuptools": {
             "hashes": [
-                "sha256:7c7854ee1429a240090297628dc9f75b35318d193537968e2dc14010ee2f5bca",
-                "sha256:dc2662692f47d99cb8ae15a784529adeed535bcd7c277fee0beccf961522baf6"
+                "sha256:10602cd0a6f5feab6656e9587f9075292ab777c5200f3bf00293ecd23d9f2788",
+                "sha256:d2e010624c781b26ad6629a8de9832327cf853dea93894487979e55f9ad06857"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==63.4.1"
+            "version": "==65.1.0"
         },
         "six": {
             "hashes": [
@@ -1035,10 +2005,10 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:77bec1402dcd0987e9022326473fdbcc767304892a533ed8c29888dacb7dddbc",
-                "sha256:aa1d52d14d09c7a9a12bb740e6bdfffe0f5e8f4f9218d85e7c73a8c37f7ae38d"
+                "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7",
+                "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa"
             ],
-            "version": "==0.3.0"
+            "version": "==0.4.0"
         },
         "traitlets": {
             "hashes": [
diff --git a/config/aws_s3_put_whitelist.lst b/config/aws_s3_put_whitelist.lst
new file mode 100644
index 000000000..431404077
--- /dev/null
+++ b/config/aws_s3_put_whitelist.lst
@@ -0,0 +1,3 @@
+hydroTable{}.csv
+gw_catchments_reaches_filtered_addedAttributes{}.tif
+rem_zeroed_masked{}.tif
diff --git a/data/README.md b/data/README.md
new file mode 100644
index 000000000..d6c573a69
--- /dev/null
+++ b/data/README.md
@@ -0,0 +1,16 @@
+## Source Data Folder
+
+This folder is for python and other files to communicate with external data sources.
+
+The first implementation is for communication to AWS (Amazon Web Service) and has a sub-folder to match.
+
+The first AWS api implementation in the form of talking to AWS S3 buckets has also been adeed and a sub-folder to match.
+
+Generally the pattern will be expected as:
+
+- `src/data`
+   -  `{a root data source name}`: Such as AWS, or possibily NWM, NHD, 3Dep or whatever.
+       - `A service class`: A python file for any basic API or service you are communication with. In this case there is one for S3. Any communication for S3 buckets will be in this file, including get, pull, get bucket lists, etc. If other AWS api is added, you would generally have an other python file. For example, if we start communicating interactively with ECR or Lambda, there is an argument for it being a new python file.
+       - `A parent class`: In the case of AWS, one thing all aws interactions have some things in commons, such as the need to authenticate. By having a base class, all AWS child (inherited) classes automatically have that code available. Using inheritance helps keep standardization of how we communicate with AWS. Helper or common utility classes can also be added if required. One example that has been added is that many AWS communication need an boto3 client object. A one line call by child classes mean the child class has a boto3 client object if required.  Later, if we need a boto3 resource object, that can be added as well. Note: Broad utilities methods that are greater than just AWS, will likely be added at the higher folders (such src/utils or src/data/).
+
+The new `s3.py` is constucted so it can be used as a CLI tool (input param args), but any function can be called directly. You will notice that a starting system of passing in a parameter for `action_type` has been added. At this time, only the option of `put` is available. If a need arises later to pull data from S3, the existing command line parameters will not need to change. Only minor adjustements to the __ main __ and add new methods for "get".  If another python file calls directly over to this python file, it will not go through the __ main __ method but straight to the "get" or "put" or whatever.
diff --git a/data/__init__.py b/data/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/data/aws/.gitignore b/data/aws/.gitignore
new file mode 100644
index 000000000..a3ad1a24c
--- /dev/null
+++ b/data/aws/.gitignore
@@ -0,0 +1,2 @@
+*.env
+!aws_creds_template.env
\ No newline at end of file
diff --git a/data/aws/aws_base.py b/data/aws/aws_base.py
new file mode 100644
index 000000000..0dadce969
--- /dev/null
+++ b/data/aws/aws_base.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+# standard library imports (https://docs.python.org/3/library/
+# (imports first, then "froms", in alpha order)
+import os
+import subprocess
+import sys
+
+sys.path.append('/foss_fim/src')
+from utils.shared_functions import FIM_Helpers as fh
+
+# third party imports
+# (imports first, then "froms", in alpha order)
+from dotenv import load_dotenv
+
+'''
+This implements all common variables related when communicating to AWS
+'''
+class AWS_Base(object):
+    
+    def __init__(self, path_to_cred_env_file, *args, **kwargs):
+
+        '''
+        Overview
+        ----------
+        This will load the aws credentials enviroment file.
+        For now, we will feed it via an env file. Eventuallly, it should be
+        changed to ~/.aws/credentials (and maybe profiles)
+        
+        The aws_credentials_file will be loaded and an aws client
+        will be created ready for use.
+
+        Parameters
+        ----------
+        path_to_cred_env_file : str
+            File path of the aws_credentials_file as an .env
+
+        is_verbose : bool
+            If True, then debugging output will be included
+
+        '''
+        
+        if (not os.path.exists(path_to_cred_env_file)):
+            raise FileNotFoundError("AWS credentials file not found")
+            
+        load_dotenv(path_to_cred_env_file)
+        
+        if kwargs:
+            is_verbose = kwargs.pop("is_verbose",None)
+
+        self.is_verbose = is_verbose
+        
+        # TODO: validate service name with AWS (which will help
+        # validate the connection)
+    
+    
+    def get_aws_cli_credentials(self):
+        
+        '''
+        Overview
+        ----------
+        To run aws cli commands (subprocess), aws creds need to be set up
+        in the command environment. This method will take care of that
+        via bash "exports"
+        
+        Returns
+        -----------
+        A string that can be concatenated to the front of a subprocess cmd
+        and includes the export creds.
+        
+        '''
+        
+        fh.vprint("getting aws credential string", self.is_verbose, True)
+        
+        cmd = "export AWS_ACCESS_KEY_ID=" + os.getenv('AWS_ACCESS_KEY')
+        cmd += " && export AWS_SECRET_ACCESS_KEY=" + os.getenv('AWS_SECRET_ACCESS_KEY')
+        cmd += " && export AWS_DEFAULT_REGION=" + os.getenv('AWS_REGION')
+        
+        return cmd
+   
+   
+    def create_aws_cli_include_argument(self, whitelist_file_names):
+        '''
+        Overview
+        ----------
+        Creates a string valid for aws_cli include commands.
+        
+        When using an "include", this string will automatically add
+        --exclude "*" , without it, the includes will not work.
+        
+        If the whitelist_file_names is empty, then an empty string will be returned.
+
+        Parameters
+        ----------
+        whitelist_file_names : list
+            a list of file names to be whitelisted. The file names should already be adjusted
+             to the "*" pattern already and stripped as applicable.
+
+        Returns
+        ----------
+        A string that can be added straight into a aws cli command. 
+        
+        example: export AWS_ACCESS_KEY_ID=A{somekey}Q && export 
+        AWS_SECRET_ACCESS_KEY=KW(examplekey)80 && 
+        export AWS_DEFAULT_REGION=us-west-1
+        '''
+        
+        if (whitelist_file_names is None) or (len(whitelist_file_names) == 0):
+            return ""  # empty string
+        
+        # For there to be "includes", for aws cli, you must have exclude "all"
+        cli_whitelist = '--exclude "*"' 
+        
+        for whitelist_file_name in whitelist_file_names:
+            if not whitelist_file_name.startswith("*"):
+                whitelist_file_name = "*" + whitelist_file_name
+            
+            whitelist_file_name = whitelist_file_name.replace("{}", "*")
+
+            cli_whitelist += f' --include "{whitelist_file_name}"'
+        
+        fh.vprint(f"cli include string is {cli_whitelist}", self.is_verbose, True)
+        
+        return cli_whitelist
diff --git a/data/aws/aws_creds_template.env b/data/aws/aws_creds_template.env
new file mode 100644
index 000000000..b236ef3f1
--- /dev/null
+++ b/data/aws/aws_creds_template.env
@@ -0,0 +1,3 @@
+AWS_REGION=
+AWS_ACCESS_KEY=
+AWS_SECRET_ACCESS_KEY=
\ No newline at end of file
diff --git a/data/aws/s3.py b/data/aws/s3.py
new file mode 100644
index 000000000..7da3b2273
--- /dev/null
+++ b/data/aws/s3.py
@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+
+# standard library imports (https://docs.python.org/3/library/
+# (imports first, then "froms", in alpha order)
+from operator import truediv
+import os
+from pickle import FALSE
+import sys
+from datetime import datetime
+
+# third party imports
+# (imports first, then "froms", in alpha order)
+import argparse
+import subprocess
+
+sys.path.append('/foss_fim/src')
+from aws_base import *
+from utils.shared_functions import FIM_Helpers as fh
+
+'''
+This file is for communicating to any AWS S3 buckets.
+For now, it can only push to a bucket (such as hydroviz)
+
+Note: For now, we will add a parameter for the aws credentials .env file.
+This means the .env file does not need to automatically be with the source code.
+Later, this will be changed to aws credentials profiles (or ~/aws/credentials (folders))
+'''
+
+class S3(AWS_Base):
+    
+    def put_to_bucket(self, src_folder_path,
+                      aws_target_path, whitelist_file_path = None):
+        
+        '''
+        Overview
+        ----------
+        Push a data folder to hydroviz visualization (primarily for now)
+        
+        If the aws_target_folder_path folder does not exist, it will be created. If it does exist, files will
+        be just added or overwritten (no pre-clean)
+       
+        The source folder will copy all contents (files and subfolders)
+        ie) aws_target_path = s3://some-address-us-east-1/test_uploads/220818
+            src_folder_path = /outputs/fim_0_34_1_fr/(some files and folders)
+            
+            Becomes: some-aws-address/test_uploads/220818/(some files and folders)
+            
+        Note:
+            - This is build on aws cli as boto3 can push up single files but is very slow.
+              As of Aug 2022, they do not have a bulk boto3 recursive folder load
+
+        Parameters
+        ----------
+       
+        - src_folder_path : str
+            folder path of files to be copied up
+             
+        - aws_target_path : str
+            A s3 bucket and folder locations in AWS (can not be blank) 
+             
+        - whitelist_file_path : str
+            A file with a set of line delimited file names that can be copied up to S3.
+            Note: make sure the list file is unix encoded and not windows encoded.
+            If None, then all files / folders will be pushed up
+            Note: wildcard variables of * is available.
+            
+        '''
+        whitelist_file_names = [] # if blank, it is assumed to load all
+
+        # --- Validate incoming values
+        # test the whitelist path and load it into a collection
+        if (whitelist_file_path is not None) and (len(whitelist_file_path.strip()) > 1):
+            
+            # if this list has items, then we load those items only 
+            whitelist_file_names = fh.load_list_file(whitelist_file_path.strip())
+            
+        if (self.is_verbose) and (len(whitelist_file_names) > 0):
+            print("whitelist entries")
+            print(whitelist_file_names)
+    
+        # test src_folder_path
+        if (not os.path.exists(src_folder_path)):
+            raise ValueError("Invalid local_folder_path (src directory)")
+        src_folder_path = src_folder_path.rstrip('/') # trailing slashes
+        
+        # test aws target folder
+        if (aws_target_path is None) or (len(aws_target_path.strip()) == 0):
+            raise ValueError("aws target folder path not supplied")
+        
+        aws_target_path = aws_target_path.strip() # spaces
+        aws_target_path = aws_target_path.strip('/') # leading/trailing slashes
+        
+
+        # --- Upload the files
+        print("************************************")
+        print("--- Start uploading files ")
+        fh.print_current_date_time()
+        start_dt = datetime.now()
+
+        self.bulk_upload(src_folder_path, aws_target_path, whitelist_file_names)        
+
+        print("--- End uploading files")        
+        fh.print_current_date_time()
+        end_dt = datetime.now()
+        fh.print_date_time_duration(start_dt, end_dt)
+
+        print("************************************")        
+
+
+    def bulk_upload(self, src_folder_path, aws_target_path, whitelist_file_names = []):
+        
+        '''
+        Overview
+        ----------
+        Files will be loaded keeping their folder structure and will be loaded
+        via Bash AWS CLI. It is preferred to load via boto3 but as of Aug 2022, it does
+        not have a bulk loader, only a single file loader and it is very slow.
+        
+        With CLI, we have a bit of trouble with outputs, but will track it as best we can
+        
+        Parameters
+        ----------
+        - src_folder_path : str
+            fully pathed location of where the files are being copied from
+            
+        - aws_target_path : str
+            The full url including subfolder path
+            For example: s3://example-us-east-1/test_upload/test_2
+        
+        - whitelist_file_names : list (but can be empty)
+            A list of file names to be included in the transfer. If the file name includes
+            a {}, it will be adjusted for aws cli format automatically.
+
+        Returns
+        -----------------
+        True (successful copy ) or False (unsuccessful)
+
+        '''
+
+        is_copy_successful = False
+
+        cmd = self.get_aws_cli_credentials()
+        
+        # used cp (copy and replace. could have used 'sync' which looks for updates only)
+        cmd += f' && aws s3 cp --recursive {src_folder_path}'
+        
+        if not aws_target_path.startswith("s3://"):
+            aws_target_path = "s3://" + aws_target_path
+        
+        cmd += f' {aws_target_path}'
+        
+        if (whitelist_file_names is not None) and (len(whitelist_file_names) > 1):
+            cmd += f' {self.create_aws_cli_include_argument(whitelist_file_names)} '
+
+        fh.vprint(f"cmd is {cmd}", self.is_verbose, True)
+        print("")
+        
+        process = subprocess.Popen(cmd, shell = True, bufsize = 1,
+                                stdout = subprocess.PIPE, 
+                                stderr = subprocess.STDOUT,
+                                errors = 'replace'
+                                ) 
+        
+        while True:
+            
+            realtime_output = process.stdout.readline()
+            if realtime_output == '' and process.poll() is not None:
+                break
+            if realtime_output:
+                # AWS spits out a tons of "completes"
+                if (not realtime_output.startswith("Completed")):
+                    print(realtime_output.strip(), flush=False)
+                sys.stdout.flush()
+                    
+        is_copy_successful = True  # catching of correct error not quite working
+            
+        return is_copy_successful
+
+
+if __name__ == '__main__':
+
+    # Sample Usage
+    #python3 /foss_fim/data/aws/s3.py -a put -c /data/config/aws_creds.env -s /data/previous_fim/fim_4_0_7_0 -t "s3://example-us-east-1/fim_4_0_7_0" -w /foss_fim/config/aws_s3_put_whitelist.lst
+    
+    # You can leave the -w flag off to load all files/folders from a directory
+    
+    # this works for hydroviz but can work with other s3 sites as well (ie esip)
+            
+    parser = argparse.ArgumentParser(description='Communication with aws s3 data services')
+    parser.add_argument('-a','--action_type', 
+                        help='value of get or put (defaults to put)',
+                        default="put", required=False)
+    parser.add_argument('-c','--aws_cred_env_file', 
+                        help='path to aws credentials env file', required=True)
+    parser.add_argument('-s','--local_folder_path',
+                        help='folder path of all files to be saved to or from', required=True)
+    parser.add_argument('-t','--aws_target_path',
+                        help='s3 bucket address and folder', required=True)
+    parser.add_argument('-w','--whitelist_file_path', 
+                        help='A file with the last of file names to be copied up (line delimited)',
+                        required=False)
+    parser.add_argument('-v','--is_verbose', 
+                        help='Adding this flag will give additional tracing output',
+                        required=False, default=False, action='store_true')
+
+    args = vars(parser.parse_args())
+    
+    if (args['action_type'] == "put"):
+        
+        s3 = S3(path_to_cred_env_file = args['aws_cred_env_file'],
+                is_verbose = args['is_verbose'])
+        s3.put_to_bucket(src_folder_path = args['local_folder_path'],
+                         aws_target_path = args['aws_target_path'],
+                         whitelist_file_path = args['whitelist_file_path'])
+        
+    elif (args["action_type"] == "get"):
+        raise Exception("Error: get method not yet implemented or available")
+    else:
+        raise Exception("Error: action type value invalid. Current options are: 'put' (more coming soon) ")
+
+
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0eb989ff8..624e9bde0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,40 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.9.1 - 2022-09-01 - [PR #664](https://github.com/NOAA-OWP/inundation-mapping/pull/664)
+
+A couple of changes:
+1) Addition of a new tool for pushing files / folders up to an AWS (Amazon Web Service) S3 bucket.
+2) Updates to the Docker image creation files to include new packages for boto3 (for AWS) and also added `jupyter`, `jupterlab` and `ipympl` to make it easier to use those tools during development.
+3) Correct an oversight of `logs\src_optimization` not being cleared upon `overwrite` run.
+
+## Additions
+
+- `src`
+   - `data`
+       - `README.md`: Details on how the new system for `data` folders (for communication for external data sources/services).
+       - `aws`
+           - `aws_base.py`:  A file using a class and inheritance system (parent / child). This file has properties and a method that all child class will be expected to use and share. This makes it quicker and easier to added new AWS tools and helps keep consistant patterns and standards.
+           - `aws_creds_template.env`: There are a number of ways to validate credentials to send data up to S3. We have chosen to use an `.env` file that can be passed into the tool from any location. This is the template for that `.env` file. Later versions may be changed to use AWS profile security system.
+           - `s3.py`: This file pushes file and folders up to a defined S3 bucket and root folder. Note: while it is designed only for `puts` (pushing to S3), hooks were added in case functional is added later for `gets` (pull from S3).
+
+
+## Changes
+
+- `utils`
+   - `shared_functions.py`:  A couple of new features
+       -  Added a method which accepts a path to a .lst or .txt file with a collection of data and load it into a  python list object. It can be used for a list of HUCS, file paths, or almost anything. 
+       - A new method for quick addition of current date/time in output.
+       - A new method for quick calculation and formatting of time duration in hours, min and seconds.
+       - A new method for search for a string in a given python list. It was designed with the following in mind, we already have a python list loaded with whitelist of files to be included in an S3 push. As we iterate through files from the file system, we can use this tool to see if the file should be pushed to S3. This tool can easily be used contexts and there is similar functionality in other FIM4 code that might be able to this method.
+
+- `Dockerfile` : Removed a line for reloading Shapely in recent PRs, which for some reason is no longer needed after adding the new BOTO3 python package. Must be related to python packages dependencies. This removed Shapely warning seen as a result of another recent PR. Also added AWS CLI for bash commands.
+
+- `Pipfile` and `Pipfile.lock`:  Updates for the four new python packages, `boto3` (for AWS), `jupyter`, `jupyterlab` and `ipympl`. We have some staff that use Jupyter in their dev actitivies. Adding this package into the base Docker image will make it easier for them.
+
+<br/><br/>
+
 ## 4.0.9.0 - 2022-09-09 - [PR #672](https://github.com/NOAA-OWP/inundation-mapping/pull/672)
 
 When deriving level paths, this improvement allows stream order to override arbolate sum when selecting the proper upstream segment to continue the current branch.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 41ca20376..6dc72644a 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -113,21 +113,6 @@ fi
 source $envFile
 source $srcDir/bash_functions.env
 
-## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
-if [ "$src_adjust_spatial" = "True" ]; then
-    if [ ! -f $CALB_DB_KEYS_FILE ]; then
-        echo "ERROR! - src_adjust_spatial parameter is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
-        exit 1
-    else
-        source $CALB_DB_KEYS_FILE
-        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes"
-        echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
-        echo "Loading Point Data"
-        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
-    fi
-fi
-
 # default values
 if [ "$jobLimit" = "" ] ; then
     jobLimit=$default_max_jobs
@@ -176,6 +161,21 @@ elif [ $overwrite -eq 1 ]; then
     mkdir -p $outputRunDataDir/branch_errors
 fi
 
+## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
+if [ "$src_adjust_spatial" = "True" ]; then
+    if [ ! -f $CALB_DB_KEYS_FILE ]; then
+        echo "ERROR! - src_adjust_spatial parameter is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        exit 1
+    else
+        source $CALB_DB_KEYS_FILE
+        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes"
+        echo "Loading HUC Data"
+        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+        echo "Loading Point Data"
+        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+    fi
+fi
+
 ## RUN GMS BY BRANCH ##
 echo "=========================================================================="
 echo "Start of branch processing"
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index dd3edb091..bd6b69180 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -161,17 +161,15 @@ if [ ! -d $outputRunDataDir ]; then
     mkdir -p $outputRunDataDir
 fi
 
+# remove these directories on a new or overwrite run
+rm -rf $outputRunDataDir/logs
+rm -rf $outputRunDataDir/branch_errors
+rm -rf $outputRunDataDir/unit_errors
+
 # we need to clean out the all log files overwrite or not
-rm -rf $outputRunDataDir/logs/unit/
 mkdir -p $outputRunDataDir/logs/unit
-
-rm -rf $outputRunDataDir/unit_errors/
 mkdir -p $outputRunDataDir/unit_errors
 
-# if it exists, but don't make a new one yet, let gms_run_branch do that.        
-rm -rf $outputRunDataDir/logs/branch
-rm -rf $outputRunDataDir/branch_errors
-
 # copy over config file
 cp -a $envFile $outputRunDataDir
 
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index dbf97333b..6caa2873a 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -1,15 +1,21 @@
 #!/usr/bin/env python3
 
-import os, inspect
+import os
+import inspect
+import re
+
 from os.path import splitext
 from datetime import datetime, timezone
+from pathlib import Path
+
 import fiona
-import rasterio
+import pandas as pd
 import numpy as np
-from pathlib import Path
+import rasterio
+
 from rasterio.warp import calculate_default_transform, reproject, Resampling
 from pyproj.crs import CRS
-import pandas as pd
+
 
 def getDriver(fileName):
 
@@ -251,8 +257,62 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
+
+########################################################################
+#Function to check the age of a file (use for flagging potentially outdated input)
+########################################################################
+def check_file_age(file):
+    '''
+    Checks if file exists, determines the file age
+    Returns
+    -------
+    None.
+    '''
+    file = Path(file)
+    if file.is_file():
+        modified_date = datetime.fromtimestamp(file.stat().st_mtime, tz=timezone.utc)
+
+        return modified_date
+
+
+########################################################################
+#Function to concatenate huc csv files to a single dataframe/csv
+########################################################################
+def concat_huc_csv(fim_dir,csv_name):
+    '''
+    Checks if huc csv file exist, concatenates contents of csv
+    Returns
+    -------
+    None.
+    '''
+
+    merged_csv = []
+    huc_list  = os.listdir(fim_dir)
+    for huc in huc_list:
+        if huc != 'logs':
+            csv_file = os.path.join(fim_dir,huc,str(csv_name))
+            if Path(csv_file).is_file():
+                # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
+                read_csv = pd.read_csv(csv_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int, 'levpa_id': object})
+                # Add huc field to dataframe
+                read_csv['HUC8'] = huc 
+                merged_csv.append(read_csv)
+
+    # Create and return a concatenated pd dataframe
+    if merged_csv:
+        print(f"Creating aggregate csv")
+        concat_df = pd.concat(merged_csv)
+        return concat_df
+      
+
+
+
+
+
+# #####################################
 class FIM_Helpers:
 
+    # -----------------------------------------------------------
     @staticmethod    
     def append_id_to_file_name(file_name, identifier):
         '''
@@ -267,13 +327,22 @@ def append_id_to_file_name(file_name, identifier):
                 ie) identifier = ["13090001", "123000001"]
                 Becomes: "/output/myfolder/a_raster_13090001_123000001.tif"
                 - This allows for file name to not be submitted and will return None
+        -------
+        
         Inputs:
             file_name: a single file name
             identifier: a value or list of values to be inserted with an underscore 
                 added ahead of the extention
+        
+        -------
         Output:
             out_file_name: A single name with each identifer added at the end before
                 the extension, each with an underscore in front of the identifier.
+
+        -------                
+        Usage:
+            from utils.shared_functions import FIM_Helpers as fh
+            composite_file_output = fh.append_id_to_file_name(composite_file_output, huc)
         '''
 
         if file_name is not None:
@@ -293,11 +362,14 @@ def append_id_to_file_name(file_name, identifier):
 
         return(out_file_name)
 
+    # -----------------------------------------------------------
     @staticmethod
     def vprint (message, is_verbose, show_caller = False):
         '''
         Processing: Will print a standard output message only when the 
             verbose flag is set to True
+        -------
+        
         Parameters:
             message : str
                 The message for output
@@ -308,9 +380,16 @@ def vprint (message, is_verbose, show_caller = False):
                 If this value is False, this method will simply return
             show_caller : bool
                 Sometimes, it is desired to see the calling function, method or class
+        
+        -------
         Returns:
             str : the message starting with "... " and optionallly ending with
                 the calling function, method or class name
+        
+        ------- 
+        Usage:
+            from utils.shared_functions import FIM_Helpers as fh
+            fh.vprint(f"Starting alpha test for {self.dir}", verbose)
         '''
         if not is_verbose:
             return
@@ -323,48 +402,98 @@ def vprint (message, is_verbose, show_caller = False):
             msg += f"  [from : {caller_name}]"
         print (msg)
 
-########################################################################
-#Function to check the age of a file (use for flagging potentially outdated input)
-########################################################################
-def check_file_age(file):
-    '''
-    Checks if file exists, determines the file age
-    Returns
-    -------
-    None.
-    '''
-    file = Path(file)
-    if file.is_file():
-        modified_date = datetime.fromtimestamp(file.stat().st_mtime, tz=timezone.utc)
-
-        return modified_date
 
+    # -----------------------------------------------------------
+    @staticmethod
+    def load_list_file(file_name_and_path):
+        '''
+        Process:
+        -------
+        Attempts to load a .txt or .lst file of line delimited values into a python list
+        
+        Parameters:
+        -------
+        file_name_and_path : str
+            path and file name of data to be loaded.
+        
+        Returns:
+        -------
+        a Python list
+        
+        -------
+        Usage:
+            from utils.shared_functions import FIM_Helpers as fh
+            fh.vprint(f"Starting alpha test for {self.dir}", verbose)
+        
+        '''
+        
+        if (not os.path.isfile(file_name_and_path)):
+            raise ValueError(f"Sorry, file {file_name_and_path} does not exist. Check name and path.")
+        
+        line_values = []
+        
+        with open(file_name_and_path, "r") as data_file:
+            data = data_file.read()
+            # replacing end splitting the text 
+            # when newline ('\n') is seen.
+            line_values_raw = data.split("\n")
+            line_values_stripped = [i.strip() for i in line_values_raw] # removes extra spaces
+            
+            # dending on comments in the file or an extra line break at the end, we might
+            # get empty entries in the line_values collection. We remove them here
+            line_values = [ele for ele in line_values_stripped if ele != ""]
+            
+        if (len(line_values) == 0):
+            raise Exception("Sorry, there are no value were in the list")
+            
+        return line_values       
+
+    # -----------------------------------------------------------
+    @staticmethod
+    def print_current_date_time():
+        '''
+        Process:
+        -------
+        prints the following:
+        
+            Current date and time: 2022-08-19 15:22:49
+        
+        -------
+        Usage:
+            from utils.shared_functions import FIM_Helpers as fh
+            fh.print_current_date_time()
+        
+        '''
+        d1 = datetime.now()
+        dt_stamp = "Current date and time : "
+        dt_stamp += d1.strftime("%Y-%m-%d %H:%M:%S")
+        print (dt_stamp)
 
-########################################################################
-#Function to concatenate huc csv files to a single dataframe/csv
-########################################################################
-def concat_huc_csv(fim_dir,csv_name):
-    '''
-    Checks if huc csv file exist, concatenates contents of csv
-    Returns
-    -------
-    None.
-    '''
+    # -----------------------------------------------------------
+    @staticmethod
+    def print_date_time_duration(start_dt, end_dt):
+        '''
+        Process:
+        -------
+        Calcuates the diffenence in time between the start and end time
+        and prints is as:
+        
+            Duration: 4 hours 23 mins 15 secs
+        
+        -------
+        Usage:
+            from utils.shared_functions import FIM_Helpers as fh
+            fh.print_current_date_time()
+        
+        '''
+        time_delta = (end_dt - start_dt)
+        total_seconds = int(time_delta.total_seconds())
 
-    merged_csv = []
-    huc_list  = os.listdir(fim_dir)
-    for huc in huc_list:
-        if huc != 'logs':
-            csv_file = os.path.join(fim_dir,huc,str(csv_name))
-            if Path(csv_file).is_file():
-                # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
-                read_csv = pd.read_csv(csv_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int, 'levpa_id': object})
-                # Add huc field to dataframe
-                read_csv['HUC8'] = huc 
-                merged_csv.append(read_csv)
+        total_days, rem_seconds = divmod(total_seconds, 60 * 60 * 24)        
+        total_hours, rem_seconds = divmod(rem_seconds, 60 * 60)
+        total_mins, seconds = divmod(rem_seconds, 60)
 
-    # Create and return a concatenated pd dataframe
-    if merged_csv:
-        print(f"Creating aggregate csv")
-        concat_df = pd.concat(merged_csv)
-        return concat_df
\ No newline at end of file
+        time_fmt = f"{total_hours:02d} hours {total_mins:02d} mins {seconds:02d} secs"
+        
+        print("Duration: " + time_fmt)
+        

From f83d8191b35324d8d7bc36a655c2a1a56a3fd8d9 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Tue, 13 Sep 2022 15:59:33 -0400
Subject: [PATCH 160/359] v4.0.9.2 Bug fixes for pruning lake branches and
 stream order prioritizing (#678)

---
 docs/CHANGELOG.md          |  21 +++++--
 src/gms/stream_branches.py | 114 ++++++++++++++++++-------------------
 2 files changed, 73 insertions(+), 62 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 624e9bde0..85764a5bc 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.9.2 - 2022-09-12 - [PR #678](https://github.com/NOAA-OWP/inundation-mapping/pull/678)
+
+This fixes several bugs related to branch definition and trimming due to waterbodies.
+
+## Changes
+
+- `src/gms/stream_branches.py`
+   - Bypasses erroneous stream network data in the to ID field by using the Node attribute instead.
+   - Adds check if no nwm_lakes_proj_subset.gpkg file is found due to no waterbodies in the HUC.
+   - Allows for multiple upstream branches when stream order overrides arbolate sum.
+
+<br/><br/>
+
 ## v4.0.9.1 - 2022-09-01 - [PR #664](https://github.com/NOAA-OWP/inundation-mapping/pull/664)
 
 A couple of changes:
@@ -45,6 +58,10 @@ When deriving level paths, this improvement allows stream order to override arbo
 
 Trims ends of branches that are in waterbodies; also removes branches if they are entirely in a waterbody.
 
+## Changes
+
+- `src/gms/stream_branches.py`: adds `trim_branches_in_waterbodies()` and `remove_branches_in_waterbodies()` to trim and prune branches in waterbodies.
+
 <br/><br/>
 
 ## v4.0.7.2 - 2022-08-11 - [PR #654](https://github.com/NOAA-OWP/inundation-mapping/pull/654)
@@ -55,10 +72,6 @@ Trims ends of branches that are in waterbodies; also removes branches if they ar
 
 - `inundate_gms`:  Changed `mask_type = 'filter'`
 
-## Changes
-
-- `src/gms/stream_branches.py`: adds functionality to trim and prune branches in waterbodies.
-
 <br/><br/>
 
 ## v4.0.7.1 - 2022-08-22 - [PR #665](https://github.com/NOAA-OWP/inundation-mapping/pull/665)
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index 3d4173119..fdfaf1ee6 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -5,7 +5,7 @@
 import rasterio
 from rasterio.mask import mask
 from rasterio.io import DatasetReader
-from os.path import splitext
+from os.path import splitext, isfile
 import fiona
 from fiona.errors import DriverError
 from collections import deque
@@ -454,62 +454,62 @@ def trim_branches_in_waterbodies(self,
         waterbody (determined by the Lake attribute).
         """
 
-        def find_downstream_reaches_in_waterbodies(tmp_self, downstream_ID, tmp_IDs=[]):
-            downstream_ID = int(tmp_self.ID[tmp_self.to==downstream_ID])
+        def find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
+            # Find lowest reach(es)
+            downstream_IDs = [int(x) for x in tmp_self.From_Node[~tmp_self.To_Node.isin(tmp_self.From_Node)]] # IDs of most downstream reach(es)
 
-            # Stop if lowest reach is not in a lake
-            if int(tmp_self.Lake[tmp_self.ID==downstream_ID]) == -9999:
-                return tmp_IDs
-            else:
-                tmp_IDs.append(downstream_ID)
-
-                # Find next lowest downstream reach
-                if downstream_ID in tmp_self.to.values:
-                    return find_downstream_reaches_in_waterbodies(tmp_self, downstream_ID, tmp_IDs)
+            for downstream_ID in downstream_IDs:
+                # Stop if lowest reach is not in a lake
+                if int(tmp_self.Lake[tmp_self.From_Node.astype(int)==downstream_ID]) == -9999:
+                    continue
                 else:
-                    return tmp_IDs
+                    # Remove reach from tmp_self
+                    tmp_IDs.append(downstream_ID)
+                    tmp_self.drop(tmp_self[tmp_self.From_Node.astype(int).isin([downstream_ID,])].index, inplace=True)
 
-        def find_upstream_reaches_in_waterbodies(tmp_self, upstream_ID, tmp_IDs=[]):
-            upstream_ID = int(tmp_self.to[tmp_self.ID==upstream_ID])
+                    # Repeat for next lowest downstream reach
+                    if downstream_ID in tmp_self.To_Node.astype(int).values:
+                        return find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
 
-            # Stop if uppermost reach is not in a lake
-            if int(tmp_self.Lake[tmp_self.ID==upstream_ID]) == -9999:
-                return tmp_IDs
-            else:
-                tmp_IDs.append(upstream_ID)
+            return tmp_IDs
 
-                # Find next highest upstream reach
-                if upstream_ID in tmp_self.ID.values:
-                    return find_upstream_reaches_in_waterbodies(tmp_self, upstream_ID, tmp_IDs)
+        def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
+            # Find highest reach(es)
+            upstream_IDs = [int(x) for x in tmp_self.From_Node[~tmp_self.From_Node.isin(tmp_self.To_Node)]] # IDs of most upstream reach(es)
+
+            for upstream_ID in upstream_IDs:
+                # Stop if uppermost reach is not in a lake
+                if int(tmp_self.Lake[tmp_self.From_Node.astype(int)==upstream_ID]) == -9999:
+                    continue
                 else:
-                    return tmp_IDs
+                    # Remove reach from tmp_self
+                    tmp_IDs.append(upstream_ID)
+                    tmp_self.drop(tmp_self[tmp_self.From_Node.astype(int).isin([upstream_ID,])].index, inplace=True)
+
+                    # Repeat for next highest upstream reach
+                    return find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
+
+            return tmp_IDs
 
         if verbose:
             print("Trimming stream branches in waterbodies ...")
 
-        for branch in self[branch_id_attribute].unique():
-            tmp_self = self[self[branch_id_attribute]==branch]
+        for branch in self[branch_id_attribute].astype(int).unique():
+            tmp_self = self[self[branch_id_attribute].astype(int)==branch]
 
             # If entire branch is in waterbody
             if all(tmp_self.Lake.values != -9999):
-                tmp_IDs = tmp_self.ID
+                tmp_IDs = tmp_self.From_Node.astype(int)
                 
             else:
                 # Find bottom up
-                downstream_ID = int(tmp_self.ID[~tmp_self.to.isin(tmp_self.ID)]) # ID of most downstream reach
-                if not int(tmp_self.Lake[tmp_self.ID==downstream_ID]) == -9999:
-                    tmp_IDs = find_downstream_reaches_in_waterbodies(tmp_self, downstream_ID, [downstream_ID])
-                else:
-                    tmp_IDs = []
+                tmp_IDs = find_downstream_reaches_in_waterbodies(tmp_self)
 
                 # Find top down
-                upstream_ID = int(tmp_self.ID[~tmp_self.ID.isin(tmp_self.to)])
-                if (len(tmp_IDs) != len(tmp_self)) and not (int(tmp_self.Lake[tmp_self.ID==upstream_ID]) == -9999):
-                    tmp_IDs.append(upstream_ID)
-                    tmp_IDs.append(find_upstream_reaches_in_waterbodies(tmp_self, upstream_ID, tmp_IDs))
+                tmp_IDs = find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
 
             if len(tmp_IDs) > 0:
-                self.drop(tmp_self[tmp_self.ID.isin(tmp_IDs)].index, inplace=True)
+                self.drop(self[self.From_Node.astype(int).isin(tmp_IDs)].index, inplace=True)
 
         return(self)
 
@@ -527,23 +527,20 @@ def remove_branches_in_waterbodies(self,
             print('Removing branches in waterbodies')
 
         # load waterbodies
-        if isinstance(waterbodies,gpd.GeoDataFrame):
-            pass
-        elif isinstance(waterbodies,str):
+        if isinstance(waterbodies,str) and isfile(waterbodies):
             waterbodies = gpd.read_file(waterbodies)
-        else:
-            raise TypeError("Waterbodies needs to be GeoDataFame or path to vector file")
 
-        # Find branches in waterbodies
-        sjoined = gpd.sjoin(self, waterbodies, op='within')
-        self.drop(sjoined.index, inplace=True)
+        if isinstance(waterbodies,gpd.GeoDataFrame):
+            # Find branches in waterbodies
+            sjoined = gpd.sjoin(self, waterbodies, op='within')
+            self.drop(sjoined.index, inplace=True)
 
-        if out_vector_files is not None:
-            
-            if verbose:
-                print("Writing pruned branches ...")
-            
-            self.write(out_vector_files, index=False)
+            if out_vector_files is not None:
+                
+                if verbose:
+                    print("Writing pruned branches ...")
+                
+                self.write(out_vector_files, index=False)
 
         return(self)
 
@@ -646,12 +643,13 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
                     # Continue the current branch up the larger stream
                     continue_id = upstream_reaches_compare_values.idxmax()[decision_attribute]
                     self.loc[continue_id,branch_id_attribute] = current_reach_branch_id
-                    # Create a new level path for the smaller tributary
+                    # Create a new level path for the smaller tributary(ies)
                     if len(not_visited_upstream_ids) == 1: continue # only create a new branch if there are 2 upstreams
-                    new_id = upstream_reaches_compare_values.idxmin()[decision_attribute]
-                    branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
-                    self.loc[new_id,branch_id_attribute] = branch_id
-                    bid += 1
+                    new_upstream_branches = upstream_reaches_compare_values.loc[~upstream_reaches_compare_values.index.isin([continue_id,])]
+                    for new_up_id in new_upstream_branches.index:
+                        branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
+                        self.loc[new_up_id,branch_id_attribute] = branch_id
+                        bid += 1
                     #==================================================================================
                     ''' NOTE: The above logic uses stream order to override arbolate sum. Use the commented section
                      below if this turns out to be a bad idea!'''
@@ -790,7 +788,7 @@ def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_exclude
         
         self["order_"] = max_stream_order.values
 
-        # merges each multi-line string to a sigular linestring
+        # merges each multi-line string to a singular linestring
         for lpid,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc="Merging mult-part geoms"):
             if isinstance(row.geometry,MultiLineString):
                 merged_line = linemerge(row.geometry)
@@ -808,7 +806,7 @@ def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_exclude
 
         if out_vector_files is not None:
             
-            base_file_path,extension = splitext(out_vector_files)
+            # base_file_path,extension = splitext(out_vector_files)
             
             if verbose:
                 print("Writing dissolved branches ...")

From aba8c28578cc1c2bee8f0e6ada70ea6d999eea75 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 16 Sep 2022 15:20:59 -0400
Subject: [PATCH 161/359] Cleanup CHANGELOG.md (#688)

---
 docs/CHANGELOG.md | 234 +++++++++++++++++++++++++++-------------------
 1 file changed, 139 insertions(+), 95 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 85764a5bc..cdd28546c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -6,7 +6,7 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 This fixes several bugs related to branch definition and trimming due to waterbodies.
 
-## Changes
+### Changes
 
 - `src/gms/stream_branches.py`
    - Bypasses erroneous stream network data in the to ID field by using the Node attribute instead.
@@ -22,7 +22,7 @@ A couple of changes:
 2) Updates to the Docker image creation files to include new packages for boto3 (for AWS) and also added `jupyter`, `jupterlab` and `ipympl` to make it easier to use those tools during development.
 3) Correct an oversight of `logs\src_optimization` not being cleared upon `overwrite` run.
 
-## Additions
+### Additions
 
 - `src`
    - `data`
@@ -33,7 +33,7 @@ A couple of changes:
            - `s3.py`: This file pushes file and folders up to a defined S3 bucket and root folder. Note: while it is designed only for `puts` (pushing to S3), hooks were added in case functional is added later for `gets` (pull from S3).
 
 
-## Changes
+### Changes
 
 - `utils`
    - `shared_functions.py`:  A couple of new features
@@ -68,7 +68,7 @@ Trims ends of branches that are in waterbodies; also removes branches if they ar
 
 `inundate_nation.py` A change to switch the inundate nation function away from refrences to `inundate.py`, and rather use `inundate_gms.py` and `mosaic_inundation.py`
 
-## Changes
+### Changes
 
 - `inundate_gms`:  Changed `mask_type = 'filter'`
 
@@ -78,7 +78,7 @@ Trims ends of branches that are in waterbodies; also removes branches if they ar
 
 Hotfix for addressing missing input variable when running `gms_run_branch.sh` outside of `gms_pipeline.sh`. 
 
-## Changes
+### Changes
 - `gms_run_branch.sh`: defining path to WBD HUC input file directly in ogr2ogr call rather than using the $input_WBD_gdb defined in `gms_run_unit.sh`
 - `src/src_adjust_spatial_obs.py`: removed an extra print statement
 - `src/src_roughness_optimization.py`: removed a log file write that contained sensitive host name
@@ -89,13 +89,13 @@ Hotfix for addressing missing input variable when running `gms_run_branch.sh` ou
 
 Introduces synthetic rating curve calibration workflow. The calibration computes new Manning's coefficients for the HAND SRCs using input data: USGS gage locations, USGS rating curve csv, and a benchmark FIM extent point database stored in PostgreSQL database. This addresses [#535].
 
-## Additions
+### Additions
 
 - `src/src_adjust_spatial_obs.py`: new synthetic rating curve calibration routine that prepares all of the spatial (point data) benchmark data for ingest to the Manning's coefficient calculations performed in `src_roughness_optimization.py`
 - `src/src_adjust_usgs_rating.py`: new synthetic rating curve calibration routine that prepares all of the USGS gage location and observed rating curve data for ingest to the Manning's coefficient calculations performed in `src_roughness_optimization.py`
 - `src/src_roughness_optimization.py`: new SRC post-processing script that ingests observed data and HUC/branch FIM output data to compute optimized Manning's coefficient values and update the discharge values in the SRCs. Outputs a new hydroTable.csv.
 
-## Changes
+### Changes
 
 - `config/deny_gms_branch_zero.lst`: added `gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg` to list of files to keep (used in calibration workflow)
 - `config/deny_gms_branches_min.lst`: added `gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg` to list of files to keep (used in calibration workflow)
@@ -132,7 +132,7 @@ Updated `Dockerfile`, `Pipfile` and `Pipfile.lock` to add the new psycopg2 pytho
 
 This file converts USFIMR remote sensed inundation shapefiles into a raster that can be used to compare to the FIM data. It has to be run separately for each shapefile. This addresses [#629].
 
-## Additions
+### Additions
 
 - `/tools/fimr_to_benchmark.py`: This file converts USFIMR remote sensed inundation shapefiles into a raster that can be used to compare to the FIM data. It has to be run separately for each shapefile.
 
@@ -142,11 +142,11 @@ This file converts USFIMR remote sensed inundation shapefiles into a raster that
 
 Prunes branches that fail with NO_FLOWLINES_EXIST (Exit code: 61) in `gms_run_branch.sh` after running `split_flows.py`
 
-## Additions
+### Additions
 - Adds `remove_error_branches.py` (called from `gms_run_branch.sh`)
 - Adds `gms_inputs_removed.csv` to log branches that have been removed across all HUCs
 
-## Removals
+### Removals
 - Deletes branch folders that fail
 - Deletes branch from `gms_inputs.csv`
 
@@ -157,7 +157,7 @@ Prunes branches that fail with NO_FLOWLINES_EXIST (Exit code: 61) in `gms_run_br
 
 Addressing #560, this fix in run_by_branch trims the DEM derived streamline if it extends past the end of the branch streamline. It does this by finding the terminal point of the branch stream, snapping to the nearest point on the DEM derived stream, and cutting off the remaining downstream portion of the DEM derived stream.
 
-## Changes
+### Changes
 
 - `/src/split_flows.py`: Trims the DEM derived streamline if it flows past the terminus of the branch (or level path) streamline.
 - `/src/gms/delineate_hydros_and_produce_HAND.sh`: Added branch streamlines as an input to `split_flows.py`.
@@ -168,7 +168,7 @@ Addressing #560, this fix in run_by_branch trims the DEM derived streamline if i
 
 Fixes bug that causes [Errno2] No such file or directory error when running synthesize_test_cases.py if testing_versions folder doesn't exist (for example, after downloading test_cases from ESIP S3).
 
-## Additions
+### Additions
 
 - `run_test_case.py`: Checks for testing_versions folder in test_cases and adds it if it doesn't exist.
 
@@ -184,14 +184,14 @@ New FIM4/gms usability is now just (at a minumum): `gms_pipeline.sh -n <output n
 	
 `gms_run_branch.sh` and `gms_run_branch.sh` have also been changed to add the new -a flag and default to dropping stream orders 1 and 2.
 
-## Additions
+### Additions
 
 - `src`
     - `check_unit_errors.py`: as described above.
 - `unit_tests`
     - `check_unit_errors_unittests.py` and `check_unit_errors_params.json`: to match new file.    
 
-## Changes
+### Changes
 
 - `README.md`:  Updated text for FIM4, gms_pipeline, S3 input updates, information about updating dependencies, misc link updates and misc text verbage.
 - `gms_pipeline.sh`: as described above.
@@ -206,7 +206,7 @@ New FIM4/gms usability is now just (at a minumum): `gms_pipeline.sh -n <output n
 - `unit_tests`
    - `README.md`: Misc text and link updates.
 
-## Removals
+### Removals
 
 - `config\params_template_calibrated.env`: No longer needed. Has been removed already from dev-fim3 and confirmed that it is not needed.
 <br><br>
@@ -221,7 +221,7 @@ Updates to unit tests including a minor update for outputs and loading in .json
 
 `Alpha Test Refactor` An upgrade was made a few weeks back to the dev-fim3 branch that improved performance, usability and readability of running alpha tests. Some cleanup in other files for readability, debugging verbosity and styling were done as well. A newer, cleaner system for printing lines when the verbose flag is enabled was added.
 
-## Changes
+### Changes
 
 - `gms_run_branch.sh`:  Updated help instructions to about using multiple HUCs as command arguments.
 - `gms_run_unit.sh`:  Updated help instructions to about using multiple HUCs as command arguments.
@@ -249,12 +249,12 @@ Updates to unit tests including a minor update for outputs and loading in .json
 
 'Branch zero' is a new branch that runs the HUCs full stream network to make up for stream orders 1 & 2 being skipped by the GMS solution and is similar to the FR extent in FIM v3. This new branch is created during `run_by_unit.sh` and the processed DEM is used by the other GMS branches during `run_by_branch.sh` to improve efficiency.
 
-## Additions
+### Additions
 
 - `src/gms/delineate_hydros_and_produce_HAND.sh`: Runs all of the modules associated with delineating stream lines and catchments and building the HAND relative elevation model. This file is called once during `gms_run_unit` to produce the branch zero files and is also run for every GMS branch in `gms_run_branch`.
 - `config/deny_gms_branch_zero.lst`: A list specifically for branch zero that helps with cleanup (removing unneeded files after processing).
 
-## Changes
+### Changes
 
 - `src/`
     - `output_cleanup.py`: Fixed bug for viz flag.
@@ -272,7 +272,7 @@ Updates to unit tests including a minor update for outputs and loading in .json
 - `gms_run_unit.sh`: Added export of extent variable, dropped the -s flag and added the -a flag so it now defaults to dropping stream orders 1 and 2.
 - `gms_run_branch.sh`: Fixed bug when using overwrite flag saying branch errors folder already exists, dropped the -s flag and added the -a flag so it now defaults to dropping stream orders 1 and 2.
 
-## Removals
+### Removals
 
 - `tests/`: Redundant
 - `tools/shared_variables`: Redundant
@@ -283,30 +283,31 @@ Updates to unit tests including a minor update for outputs and loading in .json
 
 We needed a tool that could composite / mosaic inundation maps for FIM3 FR and FIM4 / GMS with stream orders 3 and higher. A tool previously existed named composite_fr_ms_inundation.py and it was renamed to composite_inundation.py and upgraded to handle any combination of 2 of 3 items (FIM3 FR, FIM3 MS and/or FIM4 GMS).
 
-## Additions
+### Additions
 
 - `tools/composite_inundation.py`: Technically it is a renamed from composite_ms_fr_inundation.py, and is based on that functionality, but has been heavily modified. It has a number of options, but primarily is designed to take two sets of output directories, inundate the files, then composite them into a single mosiac'd raster per huc. The primary usage is expected to be compositing FIM3 FR with FIM4 / GMS with stream orders 3 and higher. 
 
 - `unit_tests/gms/inundate_gms_unittests.py and inundate_gms_params.json`: for running unit tests against `tools/gms_tools/inunundate_gms.py`.
 - `unit_tests/shared_functions_unittests.py and shared_functions_params.json`: A new function named `append_id_to_file_name_single_identifier` was added to `src/utils/shared_functions.py` and some unit tests for that function was created.
 
-## Removed
+### Removed
 
 - `tools/composite_ms_fr_inundation.py`: replaced with upgraded version named `composite_inundation.py`.
 
-## Changes
+### Changes
 
 - `tools/gms_tools/inundate_gms.py`: some style, readabilty cleanup plus move a function up to `shared_functions.py`.
 - `tools/gms_tools/mosaic_inundation.py`: some style, readabilty cleanup plus move a function up to `shared_functions.py`.
 - `tools/inundation.py`: some style, readabilty cleanup.
-- `tools/synthesize_test_cases.py`: was updated primarily for sample usage notes. 
+- `tools/synthesize_test_cases.py`: was updated primarily for sample usage notes.
+
 <br/><br/>
 
 ## v4.0.4.2 - 2022-05-03 - [PR #594](https://github.com/NOAA-OWP/inundation-mapping/pull/594)
 
 This hotfix includes several revisions needed to fix/update the FIM4 area inundation evaluation scripts. These changes largely migrate revisions from the FIM3 evaluation code to the FIM4 evaluation code.
 
-## Changes
+### Changes
 
 - `tools/eval_plots.py`: Copied FIM3 code revisions to enable RAS2FIM evals and PND plots. Replaced deprecated parameter name for matplotlib grid()
 - `tools/synthesize_test_cases.py`: Copied FIM3 code revisions to assign FR, MS, COMP resolution variable and addressed magnitude list variable for IFC eval
@@ -325,12 +326,12 @@ Now that we have a list of relavent HUC's, we need to consolidate output folders
 
 A few other small adjustments were made for readability and traceability as well as a few small fixes discovered when running at scale.
 
-## Additions
+### Additions
 
 - `tools/find_test_case_folders.py`: A new tool for creating a list of HUC's that we have test/evaluation data for.
 - `tools/copy_test_case_folders.py`: A new tool for using the list created above, to scan through other fully processed output folders and extract only the HUC's (gms units) and it's branches into a consolidated folder, ready for alpha test processing (or other needs).
 
-## Changes
+### Changes
 
 - `src/gms/aggregate_branch_lists.py`: Adjusted to allow two previously hardcoded values to now be incoming arguments. Now this file can be used by both `gms_run_unit.sh` and `copy_test_case_folders.py`.
 - `tools/synthesize_test_cases.py`: Adjustments for readability and progress status. The embedded progress bars are not working and will be addressed later.
@@ -352,7 +353,7 @@ Fixes were put in place for a couple of new logging behaviors.
 
 3. A minor correction was made when dissolved level paths were created with the new merged level path not always having a valid stream order value.
 
-## File Additions
+### File Additions
 
 - `src/`
    - `utils/`
@@ -360,7 +361,7 @@ Fixes were put in place for a couple of new logging behaviors.
          - A new class called `FIM_system_exit_codes` was added. This allows tracking and blocking of duplicate system exit codes when a custom system code is required.
         
 
-## Changes
+### Changes
 
 - `fim_run.sh`: Added the gms `non-zero-exit-code` system to `fim_run` to help uncover and isolate errors during processing. Errors recorded in log files within in the logs/unit folder are now copied into a new folder called `unit_errors`.  
     
@@ -423,22 +424,20 @@ Fixes were put in place for a couple of new logging behaviors.
            - Changed pathing for unit test data pathing from `/data/outputs/gms_example_unit_tests` to `/data/outputs/fim_unit_test_data_do_not_remove`. The new folder is intended to be a more permanent folder for unit test data.
            - Some additional tests were added validating the argument for dropping stream orders.
 
-
-## Unit Test File Additions:
+### Unit Test File Additions:
 
 - `unit_tests/`
    - `filter_catchments_and_add_attributes_unittests.py` and `filter_catchments_and_add_attributes_params.json`:
 
    - `split_flows_unittests.py' and `split_flows_params.json`
 
-
 <br/><br/>
 
 ## v4.0.3.1 - 2022-03-10 - [PR #561](https://github.com/NOAA-OWP/inundation-mapping/pull/561)
 
 Bug fixes to get the Alpha Test working in FIM 4.
 
-## Changes
+### Changes
 
 - `tools/sythesize_test_cases.py`: Fixed bugs that prevented multiple benchmark types in the same huc from running `run_test_case.py`.
 - `tools/run_test_case.py`: Fixed mall bug for IFC benchmark.
@@ -450,13 +449,13 @@ Bug fixes to get the Alpha Test working in FIM 4.
 
 This PR ports the functionality of `usgs_gage_crosswalk.py` and `rating_curve_comparison.py` to FIM 4.
 
-## Additions
+### Additions
 
 - `src/`:
     - `usgs_gage_aggregate.py`: Aggregates all instances of `usgs_elev_table.csv` to the HUC level. This makes it easier to view the gages in each HUC without having to hunt through branch folders and easier for the Sierra Test to run at the HUC level.
     - `usgs_gage_unit_setup.py`: Assigns a branch to each USGS gage within a unit. The output of this module is `usgs_subset_gages.gpkg` at the HUC level containing the `levpa_id` attribute.
 
-## Changes
+### Changes
 
 - `gms_run_branch.sh`: Added a line to aggregate all `usgs_elev_table.csv` into the HUC directory level using `src/usgs_gage_aggregate.py`.
 - `src/`:
@@ -478,7 +477,7 @@ This PR ports the functionality of `usgs_gage_crosswalk.py` and `rating_curve_co
 
 Added a new optional system which allows an argument to be added to the `gms_run_unit.sh` command line to filter out stream orders 1 and 2 when calculating branches. 
 
-## Changes
+### Changes
 
 - `gms_run_unit.sh`: Add the new optional `-s` command line argument. Inclusion of this argument means "drop stream orders 1 and 2".
 
@@ -498,7 +497,7 @@ Added a new optional system which allows an argument to be added to the `gms_run
 
 The addition of a very simple and evolving unit test system which has two unit tests against two py files.  This will set a precendence and will grow over time and may be automated, possibly during git check-in triggered. The embedded README.md has more details of what we currently have, how to use it, how to add new unit tests, and expected future enhancements.
 
-## Additions
+### Additions
 
 - `/unit_tests/` folder which has the following:
 
@@ -516,27 +515,24 @@ The addition of a very simple and evolving unit test system which has two unit t
 
 <br/><br/>
 
-
 ## v4.0.0.0 - 2022-02-01 - [PR #524](https://github.com/NOAA-OWP/cahaba/pull/524)
 
 FIM4 builds upon FIM3 and allows for better representation of inundation through the reduction of artificial restriction of inundation at catchment boundaries.
 
 More details will be made available through a publication by Aristizabal et. al. and will be included in the "Credits and References" section of the README.md, titled "Reducing Horton-Strahler Stream Order Can Enhance Flood Inundation Mapping Skill with Applications for the U.S. National Water Model."
 
-## Additions
+### Additions
 
 - `/src/gms`: A new directory containing scripts necessary to produce the FIM4 Height Above Nearest Drainage grids and synthetic rating curves needed for inundation mapping.
 - `/tools/gms_tools`: A new directory containing scripts necessary to generate and evaluate inundation maps produced from FIM4 Height Above Nearest Drainage grids and synthetic rating curves.
 
-
 <br/><br/>
 
-
 ## v3.0.24.3 - 2021-11-29 - [PR #488](https://github.com/NOAA-OWP/cahaba/pull/488)
 
 Fixed projection issue in `synthesize_test_cases.py`.
 
-## Changes
+### Changes
 
 - `Pipfile`: Added `Pyproj` to `Pipfile` to specify a version that did not have the current projection issues.
 
@@ -546,7 +542,7 @@ Fixed projection issue in `synthesize_test_cases.py`.
 
 Adding a new check to keep `usgs_elev_table.csv`, `src_base.csv`, `small_segments.csv` for runs not using the `-viz` flag. We unintentionally deleted some .csv files in `vary_mannings_n_composite.py` but need to maintain some of these for non `-viz` runs (e.g. `usgs_elev_table.csv` is used for sierra test input).
 
-## Changes
+### Changes
 
 - `fim_run.sh`: passing `-v` flag to `vary_mannings_n_composite.py` to determine which csv files to delete. Setting `$viz` = 0 for non `-v` runs.
 - `src/vary_mannings_n_composite.py`: added `-v` input arg and if statement to check which .csv files to delete.
@@ -559,11 +555,11 @@ Adding a new check to keep `usgs_elev_table.csv`, `src_base.csv`, `small_segment
 
 Patch to clean up unnecessary files and create better names for intermediate raster files.
 
-## Removals
+### Removals
 
 - `tools/run_test_case_gms.py`: Unnecessary file.
 
-## Changes
+### Changes
 
 - `tools/composite_ms_fr_inundation.py`: Clean up documentation and intermediate file names.
 - `tools/run_test_case.py`: Remove unnecessary imports.
@@ -574,11 +570,11 @@ Patch to clean up unnecessary files and create better names for intermediate ras
 
 Adds `composite_ms_fr_inundation.py` to allow for the generation of an inundation map given a "flow file" CSV and full-resolution (FR) and mainstem (MS) relative elevation models, synthetic rating curves, and catchments rasters created by the `fim_run.sh` script.
 
-## Additions
+### Additions
 - `composite_ms_fr_inundation.py`: New module that is used to inundate both MS and FR FIM and composite the two inundation rasters.
 - `/tools/gms_tools/`: Three modules (`inundate_gms.py`, `mosaic_inundation.py`, `overlapping_inundation.py`) ported from the GMS branch used to composite inundation rasters.
 
-## Changes
+### Changes
 - `inundation.py`: Added 2 exception classes ported from the GMS branch.
 
 <br/><br/>
@@ -586,7 +582,7 @@ Adds `composite_ms_fr_inundation.py` to allow for the generation of an inundatio
 ## v3.0.23.3 - 2021-11-04 - [PR #481](https://github.com/NOAA-OWP/cahaba/pull/481)
 Includes additional hydraulic properties to the `hydroTable.csv`: `Number of Cells`, `SurfaceArea (m2)`, `BedArea (m2)`, `Volume (m3)`, `SLOPE`, `LENGTHKM`, `AREASQKM`, `Roughness`, `TopWidth (m)`, `WettedPerimeter (m)`. Also adds `demDerived_reaches_split_points.gpkg`, `flowdir_d8_burned_filled.tif`, and `dem_thalwegCond.tif` to `-v` whitelist.
 
-## Changes
+### Changes
 - `run_by_unit.sh`: Added `EXIT FLAG` tag and previous non-zero exit code tag to the print statement to allow log lookup.
 - `add_crosswalk.py`: Added extra attributes to the hydroTable.csv. Includes a default `barc_on` and `vmann_on` (=False) attribute that is overwritten (=True) if SRC post-processing modules are run.
 - `bathy_src_adjust_topwidth.py`: Overwrites the `barc_on` attribute where applicable and includes the BARC-modified Volume property.
@@ -598,7 +594,7 @@ Includes additional hydraulic properties to the `hydroTable.csv`: `Number of Cel
 ## v3.0.23.2 - 2021-11-04 - [PR #480](https://github.com/NOAA-OWP/cahaba/pull/480)
 Hotfix for `vary_manning_n_composite.py` to address null discharge values for non-CONUS hucs.
 
-## Changes
+### Changes
 - `vary_manning_n_composite.py`: Add numpy where clause to set final discharge value to the original value if `vmann=False`
 
 <br/><br/>
@@ -606,7 +602,7 @@ Hotfix for `vary_manning_n_composite.py` to address null discharge values for no
 ## v3.0.23.1 - 2021-11-03 - [PR #479](https://github.com/NOAA-OWP/cahaba/pull/479)
 Patches the API updater. The `params_calibrated.env` is replaced with `params_template.env` because the BARC and Multi-N modules supplant the calibrated values.
 
-## Changes
+### Changes
 - `api/node/updater/updater.py`: Changed `params_calibrated.env` to `params_template.env`
 
 <br/><br/>
@@ -615,14 +611,14 @@ Patches the API updater. The `params_calibrated.env` is replaced with `params_te
 
 Moved the synthetic rating curve (SRC) processes from the `\tools` directory to `\src` directory to support post-processing in `fim_run.sh`. These SRC post-processing modules will now run as part of the default `fim_run.sh` workflow. Reconfigured bathymetry adjusted rating curve (BARC) module to use the 1.5yr flow from NWM v2 recurrence flow data in combination with the Bieger et al. (2015) regression equations with bankfull discharge predictor variable input.
 
-## Additions
+### Additions
 - `src/bathy_src_adjust_topwidth.py` --> New version of bathymetry adjusted rating curve (BARC) module that is configured to use the Bieger et al. (2015) regression equation with input bankfull discharge as the predictor variable (previous version used the drainage area version of the regression equations). Also added log output capability, added reconfigured output content in `src_full_crosswalked_BARC.csv` and `hydroTable.csv`, and included modifications to allow BARC to run as a post-processing step in `fim_run.sh`. Reminder: BARC is only configured for MS extent.
 
-## Removals
+### Removals
 - `config/params_calibrated.env` --> deprecated the calibrated roughness values by stream order with the new introduction of variable/composite roughness module
 - `src/bathy_rc_adjust.py` --> deprecated the previous BARC version
 
-## Changes
+### Changes
 - `src/identify_src_bankfull.py` --> Moved this script from /tools to /src, added more doc strings, cleaned up output log, and reconfigured to allow execution from fim_run.sh post-processing.
 - `src/vary_mannings_n_composite.py` --> Moved this script from /tools to /src, added more doc strings, cleaned up output log, added/reconfigured output content in src_full_crosswalked_vmann.csv and hydroTable.csv, and reconfigured to allow execution from fim_run.sh post-processing.
 - `config/params_template.env` --> Added additional parameter/variables for input to `identify_src_bankfull.py`, `vary_mannings_n_composite.py`, and `bathy_src_adjust_topwidth.py`.
@@ -641,7 +637,7 @@ Moved the synthetic rating curve (SRC) processes from the `\tools` directory to
 
 Manually filtering segments from stream input layer to fix flow reversal of the MS River (HUC 08030100).
 
-## Changes
+### Changes
 - `clip_vectors_to_wbd.py`: Fixes bug where flow direction is reversed for HUC 08030100. The issue is resolved by filtering incoming stream segments that intersect with the elevation grid boundary.
 
 <br/><br/>
@@ -650,11 +646,11 @@ Manually filtering segments from stream input layer to fix flow reversal of the
 
 These "tool" enhancements 1) delineate in-channel vs. out-of-channel geometry to allow more targeted development of key physical drivers influencing the SRC calculations (e.g. bathymetry & Manning’s n) #418 and 2) applies a variable/composite Manning’s roughness (n) using user provided csv with in-channel vs. overbank roughness values #419 & #410.
 
-## Additions
+### Additions
 - `identify_src_bankfull.p`: new post-processing tool that ingests a flow csv (e.g. NWM 1.5yr recurr flow) to approximate the bankfull STG and then calculate the channel vs. overbank proportions using the volume and hydraulic radius variables
 - `vary_mannings_n_composite.py`: new post-processing tool that ingests a csv containing feature_id, channel roughness, and overbank roughness and then generates composite n values via the channel ratio variable
 
-## Changes
+### Changes
 - `eval_plots.py`: modified the plot legend text to display full label for development tests
 - `inundation.py`: added new optional argument (-n) and corresponding function to produce a csv containing the stage value (and SRC variables) calculated from the flow to stage interpolation.
 
@@ -664,7 +660,7 @@ These "tool" enhancements 1) delineate in-channel vs. out-of-channel geometry to
 
 This new workflow ingests FIM point observations from users and “corrects” the synthetic rating curves to produce the desired FIM extent at locations where feedback is available (locally calibrate FIM).
 
-## Changes
+### Changes
 - `add_crosswalk.py`: added `NextDownID` and `order_` attributes to the exported `hydroTable.csv`. This will potentially be used in future enhancements to extend SRC changes to upstream/downstream catchments.
 - `adjust_rc_with_feedback.py`: added a new workflow to perform the SRC modifications (revised discharge) using the existing HAND geometry variables combined with the user provided point location flow and stage data.
 - `inundation_wrapper_custom_flow.py`: updated code to allow for huc6 processing to generate custom inundation outputs.
@@ -675,7 +671,7 @@ This new workflow ingests FIM point observations from users and “corrects” t
 
 Patches an issue where only certain benchmark categories were being used in evaluation.
 
-## Changes
+### Changes
 - In `tools/tools_shared_variables.py`, created a variable `MAGNITUDE_DICT` to store benchmark category magnitudes.
 - `synthesize_test_cases.py` imports `MAGNITUDE_DICT` and uses it to assign magnitudes.
 
@@ -685,7 +681,7 @@ Patches an issue where only certain benchmark categories were being used in eval
 
 Renames the BARC modified variables that are exported to `src_full_crosswalked.csv` to replace the original variables. The default/original variables are renamed with `orig_` prefix. This change is needed to ensure downstream uses of the `src_full_crosswalked.csv` are able to reference the authoritative version of the channel geometry variables (i.e. BARC-adjust where available).
 
-## Changes
+### Changes
 - In `src_full_crosswalked.csv`, default/original variables are renamed with `orig_` prefix and `SA_div` is renamed to `SA_div_flag`.
 
 <br/><br/>
@@ -694,7 +690,7 @@ Renames the BARC modified variables that are exported to `src_full_crosswalked.c
 
 This fixes a bug in the `get_metadata()` function in `/tools/tools_shared_functions.py` that arose because of a WRDS update. Previously the `metadata_source` response was returned as independent variables, but now it is returned a list of strings. Another issue was observed where the `EVALUATED_SITES_CSV` variable was being misdefined (at least on the development VM) through the OS environmental variable setting.
 
-## Changes
+### Changes
 - In `tools_shared_functions.py`, changed parsing of WRDS `metadata_sources` to account for new list type.
 - In `generate_categorical_fim_flows.py`, changed the way the `EVALUATED_SITES_CSV` path is defined from OS environmental setting to a relative path that will work within Docker container.
 
@@ -704,7 +700,7 @@ This fixes a bug in the `get_metadata()` function in `/tools/tools_shared_functi
 
 This merge addresses an issues with the bathymetry adjusted rating curve (BARC) calculations exacerbating single-pixel inundation issues for the lower Mississippi River. This fix allows the user to specify a stream order value that will be ignored in BARC calculations (reverts to using the original/default rating curve). If/when the "thalweg notch" issue is addressed, this change may be unmade.
 
-## Changes
+### Changes
 - Added new env variable `ignore_streamorders` set to 10.
 - Added new BARC code to set the bathymetry adjusted cross-section area to 0 (reverts to using the default SRC values) based on the streamorder env variable.
 
@@ -714,7 +710,7 @@ This merge addresses an issues with the bathymetry adjusted rating curve (BARC)
 
 Patches the minimum stream length in the template parameters file.
 
-## Changes
+### Changes
 - Changes `max_split_distance_meters` in `params_template.env` to 1500.
 
 <br/><br/>
@@ -723,7 +719,7 @@ Patches the minimum stream length in the template parameters file.
 
 This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in future issues. The primary function that performs the HAND value and hydroid extraction is ingest_points_layer() but this may change as the overall synthetic rating curve automatic update machanism evolves.
 
-## Additions
+### Additions
 - Added `adjust_rc_with_feedback.py` with `ingest_points_layer()`, a function to extract HAND and hydroid values for use in an automatic synthetic rating curve updating mechanism.
 
 <br/><br/>
@@ -732,7 +728,7 @@ This adds a script, `adjust_rc_with_feedback.py`, that will be expanded  in futu
 
 General repository cleanup, made memory-profiling an optional flag, API's release feature now saves outputs.
 
-## Changes
+### Changes
 - Remove `Dockerfile.prod`, rename `Dockerfile.dev` to just `Dockerfile`, and remove `.dockerignore`.
 - Clean up `Dockerfile` and remove any unused* packages or variables.
 - Remove any unused* Python packages from the `Pipfile`.
@@ -749,7 +745,7 @@ General repository cleanup, made memory-profiling an optional flag, API's releas
 
 This merge modifies `clip_vectors_to_wbd.py` to check for relevant input data.
 
-## Changes
+### Changes
 - `clip_vectors_to_wbd.py` now checks that there are NWM stream segments within the buffered HUC boundary.
 - `included_huc8_ms.lst` has several additional HUC8s.
 
@@ -759,7 +755,7 @@ This merge modifies `clip_vectors_to_wbd.py` to check for relevant input data.
 
 This merge improves documentation in various scripts.
 
-## Changes
+### Changes
 This PR better documents the following:
 
 - `inundate_nation.py`
@@ -773,7 +769,7 @@ This PR better documents the following:
 
 This merge adds two new scripts into `/tools/` for use in QAQC.
 
-## Additions
+### Additions
 - `inundate_nation.py` to produce inundation maps for the entire country for use in QAQC.
 - `check_deep_flooding.py` to check for depths of inundation greater than a user-supplied threshold at specific areas defined by a user-supplied shapefile.
 
@@ -789,11 +785,11 @@ Updating `README.md`.
 
 Updating logging and fixing bug in vector preprocessing.
 
-## Additions
+### Additions
 - `fim_completion_check.py` adds message to docker log to log any HUCs that were requested but did not finish `run_by_unit.sh`.
 - Adds `input_data_edits_changelog.txt` to the inputs folder to track any manual or version/location specific changes that were made to data used in FIM 3.
 
-## Changes
+### Changes
 - Provides unique exit codes to relevant domain checkpoints within `run_by_unit.sh`.
 - Bug fixes in `reduce_nhd_stream_density.py`, `mprof plot` call.
 - Improved error handling in `add_crosswalk.py`.
@@ -804,7 +800,7 @@ Updating logging and fixing bug in vector preprocessing.
 
 Hot fix to `synthesize_test_cases`.
 
-## Changes
+### Changes
 - Fixed if/elif/else statement in `synthesize_test_cases.py` that resulted in only IFC data being evaluated.
 
 <br/><br/>
@@ -813,7 +809,7 @@ Hot fix to `synthesize_test_cases`.
 
 Updates to evaluation scripts to allow for Alpha testing at Iowa Flood Center (IFC) sites. Also, `BAD_SITES` variable updates to omit sites not suitable for evaluation from metric calculations.
 
-## Changes
+### Changes
 - The `BAD_SITES` list in `tools_shared_variables.py` was updated and reasons for site omission are documented.
 - Refactored `run_test_case.py`, `synthesize_test_cases.py`, `tools_shared_variables.py`, and `eval_plots.py` to allow for IFC comparisons.
 
@@ -823,13 +819,13 @@ Updates to evaluation scripts to allow for Alpha testing at Iowa Flood Center (I
 
 Adding a thalweg profile tool to identify significant drops in thalweg elevation. Also setting lateral thalweg adjustment threshold in hydroconditioning.
 
-## Additions
+### Additions
 - `thalweg_drop_check.py` checks the elevation along the thalweg for each stream path downstream of MS headwaters within a HUC.
 
-## Removals
+### Removals
 - Removing `dissolveLinks` arg from `clip_vectors_to_wbd.py`.
 
-## Changes
+### Changes
 - Cleaned up code in `split_flows.py` to make it more readable.
 - Refactored `reduce_nhd_stream_density.py` and `adjust_headwater_streams.py` to limit MS headwater points in `agg_nhd_headwaters_adj.gpkg`.
 - Fixed a bug in `adjust_thalweg_lateral.py` lateral elevation replacement threshold; changed threshold to 3 meters.
@@ -841,39 +837,42 @@ Adding a thalweg profile tool to identify significant drops in thalweg elevation
 
 Feature to evaluate performance of alternative CatFIM techniques.
 
-## Additions
+### Additions
 - Added `eval_catfim_alt.py` to evaluate performance of alternative CatFIM techniques.
 
 <br/><br/>
+
 ## v3.0.18.0 - 2021-06-09 - [PR #404](https://github.com/NOAA-OWP/cahaba/pull/404)
 
 To help analyze the memory consumption of the Fim Run process, the python module `memory-profiler` has been added to give insights into where peak memory usage is with in the codebase.
 
 In addition, the Dockerfile was previously broken due to the Taudem dependency removing the version that was previously being used by FIM. To fix this, and allow new docker images to be built, the Taudem version has been updated to the newest version on the Github repo and thus needs to be thoroughly tested to determine if this new version has affected the overall FIM outputs.
 
-## Additions
+### Additions
 - Added `memory-profiler` to `Pipfile` and `Pipfile.lock`.
 - Added `mprof` (memory-profiler cli utility) call to the `time_and_tee_run_by_unit.sh` to create overall memory usage graph location in the `/logs/{HUC}_memory.png` of the outputs directory.
 - Added `@profile` decorator to all functions within scripts used in the `run_by_unit.sh` script to allow for memory usage tracking, which is then recorded in the `/logs/{HUC}.log` file of the outputs directory.
 
-## Changes
+### Changes
 - Changed the Taudem version in `Dockerfile.dev` to `98137bb6541a0d0077a9c95becfed4e56d0aa0ac`.
 - Changed all calls of python scripts in `run_by_unit.s` to be called with the `-m memory-profiler` argument to allow scripts to also track memory usage.
 
 <br/><br/>
+
 ## v3.0.17.1 - 2021-06-04 - [PR #395](https://github.com/NOAA-OWP/cahaba/pull/395)
 
 Bug fix to the `generate_nws_lid.py` script
 
-## Changes
+### Changes
 - Fixes incorrectly assigned attribute field "is_headwater" for some sites in the `nws_lid.gpkg` layer.
 - Updated `agg_nhd_headwaters_adj.gpkg`, `agg_nhd_streams_adj.gpkg`, `nwm_flows.gpkg`, and `nwm_catchments.gpkg` input layers using latest NWS LIDs.
 
 <br/><br/>
+
 ## v3.0.17.0 - 2021-06-04 - [PR #393](https://github.com/NOAA-OWP/cahaba/pull/393)
 BARC updates to cap the bathy calculated xsec area in `bathy_rc_adjust.py` and allow user to choose input bankfull geometry.
 
-## Changes
+### Changes
 
 - Added new env variable to control which input file is used for the bankfull geometry input to bathy estimation workflow.
 - Modified the bathymetry cross section area calculation to cap the additional area value so that it cannot exceed the bankfull cross section area value for each stream segment (bankfull value obtained from regression equation dataset).
@@ -882,21 +881,23 @@ BARC updates to cap the bathy calculated xsec area in `bathy_rc_adjust.py` and a
 - Evaluate the FIM Bathymetry Adjusted Rating Curve (BARC) tool performance using the estimated bankfull geometry dataset derived for the NWM route link dataset.
 
 <br/><br/>
+
 ## v3.0.16.3 - 2021-05-21 - [PR #388](https://github.com/NOAA-OWP/cahaba/pull/388)
 
 Enhancement and bug fixes to `synthesize_test_cases.py`.
 
-## Changes
+### Changes
 - Addresses a bug where AHPS sites without benchmark data were receiving a CSI of 0 in the master metrics CSV produced by `synthesize_test_cases.py`.
 - Includes a feature enhancement to `synthesize_test_cases.py` that allows for the inclusion of user-specified testing versions in the master metrics CSV.
 - Removes some of the print statements used by `synthesize_test_cases.py`.
 
 <br/><br/>
+
 ## v3.0.16.2 - 2021-05-18 - [PR #384](https://github.com/NOAA-OWP/cahaba/pull/384)
 
 Modifications and fixes to `run_test_case.py`, `eval_plots.py`, and AHPS preprocessing scripts.
 
-## Changes
+### Changes
 - Comment out return statement causing `run_test_case.py` to skip over sites/hucs when calculating contingency rasters.
 - Move bad sites list and query statement used to filter out bad sites to the `tools_shared_variables.py`.
 - Add print statements in `eval_plots.py` detailing the bad sites used and the query used to filter out bad sites.
@@ -905,129 +906,142 @@ Modifications and fixes to `run_test_case.py`, `eval_plots.py`, and AHPS preproc
 - Update workarounds for some sites in ahps preprocessing scripts.
 
 <br/><br/>
+
 ## v3.0.16.1 - 2021-05-11 - [PR #380](https://github.com/NOAA-OWP/cahaba/pull/380)
 
 The current version of Eventlet used in the Connector module of the FIM API is outdated and vulnerable. This update bumps the version to the patched version.
 
-## Changes
+### Changes
 - Updated `api/node/connector/requirements.txt` to have the Eventlet version as 0.31.0
 
 <br/><br/>
+
 ## v3.0.16.0 - 2021-05-07 - [PR #378](https://github.com/NOAA-OWP/cahaba/pull/378)
 
 New "Release" feature added to the FIM API. This feature will allow for automated FIM, CatFIM, and relevant metrics to be generated when a new FIM Version is released. See [#373](https://github.com/NOAA-OWP/cahaba/issues/373) for more detailed steps that take place in this feature.
 
-## Additions
+### Additions
 - Added new window to the UI in `api/frontend/gui/templates/index.html`.
 - Added new job type to `api/node/connector/connector.py` to allow these release jobs to run.
 - Added additional logic in `api/node/updater/updater.py` to run the new eval and CatFIM scripts used in the release feature.
 
-## Changes
+### Changes
 - Updated `api/frontend/output_handler/output_handler.py` to allow for copying more broad ranges of file paths instead of only the `/data/outputs` directory.
 
 <br/><br/>
+
 ## v3.0.15.10 - 2021-05-06 - [PR #375](https://github.com/NOAA-OWP/cahaba/pull/375)
 
 Remove Great Lakes coastlines from WBD buffer.
 
-## Changes
+### Changes
 - `gl_water_polygons.gpkg` layer is used to mask out Great Lakes boundaries and remove NHDPlus HR coastline segments.
 
 <br/><br/>
+
 ## v3.0.15.9 - 2021-05-03 - [PR #372](https://github.com/NOAA-OWP/cahaba/pull/372)
 
 Generate `nws_lid.gpkg`.
 
-## Additions
+### Additions
 - Generate `nws_lid.gpkg` with attributes indicating if site is a headwater `nws_lid` as well as if it is co-located with another `nws_lid` which is referenced to the same `nwm_feature_id` segment.
 
 <br/><br/>
+
 ## v3.0.15.8 - 2021-04-29 - [PR #371](https://github.com/NOAA-OWP/cahaba/pull/371)
 
 Refactor NHDPlus HR preprocessing workflow. Resolves issue #238
 
-## Changes
+### Changes
 - Consolidate NHD streams, NWM catchments, and headwaters MS and FR layers with `mainstem` column.
 - HUC8 intersections are included in the input headwaters layer.
 - `clip_vectors_to_wbd.py` removes incoming stream segment from the selected layers.
 
 <br/><br/>
+
 ## v3.0.15.7 - 2021-04-28 - [PR #367](https://github.com/NOAA-OWP/cahaba/pull/367)
 
 Refactor synthesize_test_case.py to handle exceptions during multiprocessing. Resolves issue #351
 
-## Changes
+### Changes
 - refactored `inundation.py` and `run_test_case.py` to handle exceptions without using `sys.exit()`.
 
 <br/><br/>
+
 ## v3.0.15.6 - 2021-04-23 - [PR #365](https://github.com/NOAA-OWP/cahaba/pull/365)
 
 Implement CatFIM threshold flows to Sierra test and add AHPS benchmark preprocessing scripts.
 
-## Additions
+### Additions
 - Produce CatFIM flows file when running `rating_curve_get_usgs_gages.py`.
 - Several scripts to preprocess AHPS benchmark data. Requires numerous file dependencies not available through Cahaba.
 
-## Changes
+### Changes
 - Modify `rating_curve_comparison.py` to ingest CatFIM threshold flows in calculations.
 - Modify `eval_plots.py` to save all site specific bar plots in same parent directory instead of in subdirectories.
 - Add variables to `env.template` for AHPS benchmark preprocessing.
 
 <br/><br/>
+
 ## v3.0.15.5 - 2021-04-20 - [PR #363](https://github.com/NOAA-OWP/cahaba/pull/363)
 
 Prevent eval_plots.py from erroring out when spatial argument enabled if certain datasets not analyzed.
 
-## Changes
+### Changes
 - Add check to make sure analyzed dataset is available prior to creating spatial dataset.
 
 <br/><br/>
+
 ## v3.0.15.4 - 2021-04-20 - [PR #356](https://github.com/NOAA-OWP/cahaba/pull/356)
 
 Closing all multiprocessing Pool objects in repo.
 
 <br/><br/>
+
 ## v3.0.15.3 - 2021-04-19 - [PR #358](https://github.com/NOAA-OWP/cahaba/pull/358)
 
 Preprocess NHDPlus HR rasters for consistent projections, nodata values, and convert from cm to meters.
 
-## Additions
+### Additions
 - `preprocess_rasters.py` reprojects raster, converts to meters, and updates nodata value to -9999.
 - Cleaned up log messages from `bathy_rc_adjust.py` and `usgs_gage_crosswalk.py`.
 - Outputs paths updated in `generate_categorical_fim_mapping.py` and `generate_categorical_fim.py`.
 - `update_raster_profile` cleans up raster crs, blocksize, nodata values, and converts elevation grids from cm to meters.
 - `reproject_dem.py` imports gdal to reproject elevation rasters because an error was occurring when using rasterio.
 
-## Changes
+### Changes
 - `burn_in_levees.py` replaces the `gdal_calc.py` command to resolve inconsistent outputs with burned in levee values.
 
 <br/><br/>
+
 ## v3.0.15.2 - 2021-04-16 - [PR #359](https://github.com/NOAA-OWP/cahaba/pull/359)
 
 Hotfix to preserve desired files when production flag used in `fim_run.sh`.
 
-## Changes
+### Changes
 
 - Fixed production whitelisted files.
 
 <br/><br/>
+
 ## v3.0.15.1 - 2021-04-13 - [PR #355](https://github.com/NOAA-OWP/cahaba/pull/355)
 
 Sierra test considered all USGS gage locations to be mainstems even though many actually occurred with tributaries. This resulted in unrealistic comparisons as incorrect gages were assigned to mainstems segments. This feature branch identifies gages that are on mainstems via attribute field.
 
-## Changes
+### Changes
 
 - Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "MS" run, only consider gages that contain rating curve information (via `curve` attribute) and are also mainstems gages (via `mainstems` attribute).
 - Modifies `usgs_gage_crosswalk.py` to filter out gages from the `usgs_gages.gpkg` layer such that for a "FR" run, only consider gages that contain rating curve information (via `curve` attribute) and are not mainstems gages (via `mainstems` attribute).
 - Modifies how mainstems segments are determined by using the `nwm_flows_ms.gpkg` as a lookup to determine if the NWM segment specified by WRDS for a gage site is a mainstems gage.
 
-## Additions
+### Additions
 
 - Adds a `mainstem` attribute field to `usgs_gages.gpkg` that indicates whether a gage is located on a mainstems river.
 - Adds `NWM_FLOWS_MS` variable to the `.env` and `.env.template` files.
 - Adds the `extent` argument specified by user when running `fim_run.sh` to `usgs_gage_crosswalk.py`.
 
 <br/><br/>
+
 ## v3.0.15.0 - 2021-04-08 - [PR #340](https://github.com/NOAA-OWP/cahaba/pull/340)
 
 Implementing a prototype technique to estimate the missing bathymetric component in the HAND-derived synthetic rating curves. The new Bathymetric Adjusted Rating Curve (BARC) function is built within the `fim_run.sh` workflow and will ingest bankfull geometry estimates provided by the user to modify the cross section area used in the synthetic rating curve generation.
@@ -1045,6 +1059,7 @@ Implementing a prototype technique to estimate the missing bathymetric component
     - Flags issues with the thalweg-notch artifact.
 
 <br/><br/>
+
 ## v3.0.14.0 - 2021-04-05 - [PR #338](https://github.com/NOAA-OWP/cahaba/pull/338)
 
 Create tool to retrieve rating curves from USGS sites and convert to elevation (NAVD88). Intended to be used as part of the Sierra Test.
@@ -1062,6 +1077,7 @@ Create tool to retrieve rating curves from USGS sites and convert to elevation (
      3) `usgs_gages.gpkg`: A geospatial layer (in FIM projection) of all active USGS gages that meet a predefined criteria. Additionally, the `curve` attribute indicates whether a rating curve is found in the `usgs_rating_curves.csv`. This spatial file is only generated if the `all` option is passed with the `-l` argument.
 
 <br/><br/>
+
 ## v3.0.13.0 - 2021-04-01 - [PR #332](https://github.com/NOAA-OWP/cahaba/pull/332)
 
 Created tool to compare synthetic rating curve with benchmark rating curve (Sierra Test).
@@ -1076,6 +1092,7 @@ Created tool to compare synthetic rating curve with benchmark rating curve (Sier
  - `rating_curve_comparison.py`: post-processing script to plot and calculate metrics between synthetic rating curves and USGS rating curve data.
 
 <br/><br/>
+
 ## v3.0.12.1 - 2021-03-31 - [PR #336](https://github.com/NOAA-OWP/cahaba/pull/336)
 
 Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
@@ -1090,6 +1107,7 @@ Fix spatial option in `eval_plots.py` when creating plots and spatial outputs.
  - Creates `fim_performance_polys.shp`: this layer consists of all evaluated huc8s (with metrics). Spatial data retrieved from WBD layer.
 
 <br/><br/>
+
 ## v3.0.12.0 - 2021-03-26 - [PR #327](https://github.com/NOAA-OWP/cahaba/pull/237)
 
 Add more detail/information to plotting capabilities.
@@ -1103,6 +1121,7 @@ Add more detail/information to plotting capabilities.
  - Create a csv containing the data used to create the scatterplots.
 
 <br/><br/>
+
 ## v3.0.11.0 - 2021-03-22 - [PR #319](https://github.com/NOAA-OWP/cahaba/pull/298)
 
 Improvements to CatFIM service source data generation.
@@ -1117,6 +1136,7 @@ Improvements to CatFIM service source data generation.
  - Create new `nws_lid_sites` shapefile located in same directory as the `catfim_library` shapefile.
 
 <br/><br/>
+
 ## v3.0.10.1 - 2021-03-24 - [PR #320](https://github.com/NOAA-OWP/cahaba/pull/320)
 
 Patch to synthesize_test_cases.py.
@@ -1125,6 +1145,7 @@ Patch to synthesize_test_cases.py.
  - Bug fix to `synthesize_test_cases.py` to allow comparison between `testing` version and `official` versions.
 
 <br/><br/>
+
 ## v3.0.10.0 - 2021-03-12 - [PR #298](https://github.com/NOAA-OWP/cahaba/pull/298)
 
 Preprocessing of flow files for Categorical FIM.
@@ -1140,6 +1161,7 @@ Preprocessing of flow files for Categorical FIM.
  - Stability fixes to `generate_categorical_fim.py`.
 
 <br/><br/>
+
 ## v3.0.9.0 - 2021-03-12 - [PR #297](https://github.com/NOAA-OWP/cahaba/pull/297)
 
 Enhancements to FIM API.
@@ -1155,6 +1177,7 @@ Enhancements to FIM API.
  - Both FR and MS configs can be selected for a single job.
 
 <br/><br/>
+
 ## v3.0.8.2 - 2021-03-11 - [PR #296](https://github.com/NOAA-OWP/cahaba/pull/296)
 
 Enhancements to post-processing for Viz-related use-cases.
@@ -1165,6 +1188,7 @@ Enhancements to post-processing for Viz-related use-cases.
  - Aggregate grid blocksize is changed from 256 to 1024 for faster postprocessing.
 
 <br/><br/>
+
 ## v3.0.8.1 - 2021-03-10 - [PR #302](https://github.com/NOAA-OWP/cahaba/pull/302)
 
 Patched import issue in `tools_shared_functions.py`.
@@ -1173,6 +1197,7 @@ Patched import issue in `tools_shared_functions.py`.
  - Changed `utils.` to `tools_` in `tools_shared_functions.py` after recent structural change to `tools` directory.
 
 <br/><br/>
+
 ## v3.0.8.0 - 2021-03-09 - [PR #279](https://github.com/NOAA-OWP/cahaba/pull/279)
 
 Refactored NWS Flood Categorical HAND FIM (CatFIM) pipeline to open source.
@@ -1184,6 +1209,7 @@ Refactored NWS Flood Categorical HAND FIM (CatFIM) pipeline to open source.
  - Removed `util` folders under `tools` directory.
 
 <br/><br/>
+
 ## v3.0.7.1 - 2021-03-02 - [PR #290](https://github.com/NOAA-OWP/cahaba/pull/290)
 
 Renamed benchmark layers in `test_cases` and updated variable names in evaluation scripts.
@@ -1193,6 +1219,7 @@ Renamed benchmark layers in `test_cases` and updated variable names in evaluatio
  - Updated `run_test_case_calibration.py` with new benchmark layer names.
 
 <br/><br/>
+
 ## v3.0.7.0 - 2021-03-01 - [PR #288](https://github.com/NOAA-OWP/cahaba/pull/288)
 
 Restructured the repository. This has no impact on hydrological work done in the codebase and is simply moving files and renaming directories.
@@ -1203,6 +1230,7 @@ Restructured the repository. This has no impact on hydrological work done in the
  - Changed any instance of `lib` or `libDir` to `src` or `srcDir`.
 
 <br/><br/>
+
 ## v3.0.6.0 - 2021-02-25 - [PR #276](https://github.com/NOAA-OWP/cahaba/pull/276)
 
 Enhancement that creates metric plots and summary statistics using metrics compiled by `synthesize_test_cases.py`.
@@ -1216,6 +1244,7 @@ Enhancement that creates metric plots and summary statistics using metrics compi
     - CSV of analyzed data and analyzed sites
 
 <br/><br/>
+
 ## v3.0.5.3 - 2021-02-23 - [PR #275](https://github.com/NOAA-OWP/cahaba/pull/275)
 
 Bug fixes to new evaluation code.
@@ -1228,6 +1257,7 @@ Bug fixes to new evaluation code.
  - Updated README.md
 
 <br/><br/>
+
 ## v3.0.5.2 - 2021-02-23 - [PR #272](https://github.com/NOAA-OWP/cahaba/pull/272)
 
 Adds HAND synthetic rating curve (SRC) datum elevation values to `hydroTable.csv` output.
@@ -1247,6 +1277,7 @@ Fixed `TEST_CASES_DIR` path in `tests/utils/shared_variables.py`.
  - Removed `"_new"` from `TEST_CASES_DIR` variable.
 
 <br/><br/>
+
 ## v3.0.5.0 - 2021-02-22 - [PR #267](https://github.com/NOAA-OWP/cahaba/pull/267)
 
 Enhancements to allow for evaluation at AHPS sites, the generation of a query-optimized metrics CSV, and the generation of categorical FIM. This merge requires that the `/test_cases` directory be updated for all machines performing evaluation.
@@ -1282,6 +1313,7 @@ Rating curves for short stream segments are replaced with rating curves from ups
  - Variable names and general workflow are cleaned up.
 
 <br/><br/>
+
 ## v3.0.4.3 - 2021-02-12 - [PR #254](https://github.com/NOAA-OWP/cahaba/pull/254)
 
 Modified `rem.py` with a new function to output HAND reference elev.
@@ -1294,6 +1326,7 @@ Modified `rem.py` with a new function to output HAND reference elev.
  - Overwrites the `demDerived_reaches_split.gpk` layer by adding additional attribute `Min_Thal_Elev_meters` to view the elevation value for each hydroid.
 
 <br/><br/>
+
 ## v3.0.4.2 - 2021-02-12 - [PR #255](https://github.com/NOAA-OWP/cahaba/pull/255)
 
 Addresses issue when running on HUC6 scale.
@@ -1307,6 +1340,7 @@ Addresses issue when running on HUC6 scale.
  - Fixed known issue where sometimes an incoming stream is not included in the final selection will affect aggregate outputs.
 
 <br/><br/>
+
 ## v3.0.4.1 - 2021-02-12 - [PR #261](https://github.com/NOAA-OWP/cahaba/pull/261)
 
 Updated MS Crosswalk method to address gaps in FIM.
@@ -1318,6 +1352,7 @@ Updated MS Crosswalk method to address gaps in FIM.
  - `add_crosswalk.py` now performs a secondary MS crosswalk selection by nearest NWM MS catchment.
 
 <br/><br/>
+
 ## v3.0.4.0 - 2021-02-10 - [PR #256](https://github.com/NOAA-OWP/cahaba/pull/256)
 
 New python script "wrappers" for using `inundation.py`.
@@ -1329,6 +1364,7 @@ New python script "wrappers" for using `inundation.py`.
  - Created new `tools` parent directory to store `inundation_wrapper_nwm_flows.py` and  `inundation_wrapper_custom_flow.py`.
 
 <br/><br/>
+
 ## v3.0.3.1 - 2021-02-04 - [PR #253](https://github.com/NOAA-OWP/cahaba/pull/253)
 
 Bug fixes to correct mismatched variable name and file path.
@@ -1339,6 +1375,7 @@ Bug fixes to correct mismatched variable name and file path.
  - `acquire_and_preprocess_inputs.py` now creates `huc_lists` folder and updates file path.
 
 <br/><br/>
+
 ## v3.0.3.0 - 2021-02-04 - [PR #227](https://github.com/NOAA-OWP/cahaba/pull/227)
 
 Post-process to aggregate FIM outputs to HUC6 scale.
@@ -1358,6 +1395,7 @@ Post-process to aggregate FIM outputs to HUC6 scale.
  - Cleanup of `clip_vectors_to_wbd.py`.
 
 <br/><br/>
+
 ## v3.0.2.0 - 2021-01-25 - [PR #218](https://github.com/NOAA-OWP/cahaba/pull/218)
 
 Addition of an API service to schedule, run and manage `fim_run` jobs through a user-friendly web interface.
@@ -1367,6 +1405,7 @@ Addition of an API service to schedule, run and manage `fim_run` jobs through a
  - `api` folder that contains all the codebase for the new service.
 
 <br/><br/>
+
 ## v3.0.1.0 - 2021-01-21 - [PR #206](https://github.com/NOAA-OWP/cahaba/pull/206)
 
 Preprocess MS and FR stream networks
@@ -1383,6 +1422,7 @@ Preprocess MS and FR stream networks
  - Cleaned up variable names and types.
 
 <br/><br/>
+
 ## v3.0.0.4 - 2021-01-20 - [PR #230](https://github.com/NOAA-OWP/cahaba/pull/230)
 
 Changed the directory where the `included_huc*.lst` files are being read from.
@@ -1392,6 +1432,7 @@ Changed the directory where the `included_huc*.lst` files are being read from.
  - Changed the directory where the `included_huc*.lst` files are being read from.
 
 <br/><br/>
+
 ## v3.0.0.3 - 2021-01-14 - [PR #210](https://github.com/NOAA-OWP/cahaba/pull/210)
 
 Hotfix for handling nodata value in rasterized levee lines.
@@ -1402,6 +1443,7 @@ Hotfix for handling nodata value in rasterized levee lines.
  - Initialize the `nld_rasterized_elev.tif` using a value of `-9999` instead of `$ndv`.
 
  <br/><br/>
+
 ## v3.0.0.2 - 2021-01-06 - [PR #200](https://github.com/NOAA-OWP/cahaba/pull/200)
 
 Patch to address AHPSs mapping errors.
@@ -1414,6 +1456,7 @@ Patch to address AHPSs mapping errors.
  - Updated [readme](https://github.com/NOAA-OWP/cahaba/commit/9bffb885f32dfcd95978c7ccd2639f9df56ff829)
 
 <br/><br/>
+
 ## v3.0.0.1 - 2020-12-31 - [PR #184](https://github.com/NOAA-OWP/cahaba/pull/184)
 
 Modifications to build and run Docker image more reliably. Cleanup on some pre-processing scripts.
@@ -1427,6 +1470,7 @@ Modifications to build and run Docker image more reliably. Cleanup on some pre-p
  - `aggregate_vector_inputs.py` doesn't work yet. Need to externally download required data to run fim_run.sh
 
  <br/><br/>
+
 ## v3.0.0.0 - 2020-12-22 - [PR #181](https://github.com/NOAA-OWP/cahaba/pull/181)
 
 The software released here builds on the flood inundation mapping capabilities demonstrated as part of the National Flood Interoperability Experiment, the Office of Water Prediction's Innovators Program and the National Water Center Summer Institute. The flood inundation mapping software implements the Height Above Nearest Drainage (HAND) algorithm and incorporates community feedback and lessons learned over several years. The software has been designed to meet the requirements set by stakeholders interested in flood prediction and has been developed in partnership with several entities across the water enterprise.

From ba37f9c67db590ad7c5f4cedaa8cad0d18200331 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Tue, 27 Sep 2022 10:56:25 -0500
Subject: [PATCH 162/359] v4.0.9.3 Download USGS 3DEP Dems (#681)

---
 Dockerfile                                    |   2 +-
 Dockerfile.prod                               | 103 ------
 .../acquire_and_preprocess_inputs.py          |   0
 data/create_vrt_file.py                       | 114 ++++++
 {tools => data/nws}/preprocess_ahps_nws.py    |  17 +-
 {src => data}/preprocess_rasters.py           |   2 +
 data/usgs/acquire_and_preprocess_3dep_dems.py | 337 ++++++++++++++++++
 {tools => data/usgs}/preprocess_ahps_usgs.py  |  13 +-
 .../usgs}/preprocess_download_usgs_grids.py   |  13 +-
 docs/CHANGELOG.md                             |  45 +++
 src/utils/shared_functions.py                 | 108 +++++-
 src/utils/shared_variables.py                 |  14 +-
 12 files changed, 634 insertions(+), 134 deletions(-)
 delete mode 100644 Dockerfile.prod
 rename {src => data}/acquire_and_preprocess_inputs.py (100%)
 create mode 100644 data/create_vrt_file.py
 rename {tools => data/nws}/preprocess_ahps_nws.py (99%)
 rename {src => data}/preprocess_rasters.py (99%)
 create mode 100644 data/usgs/acquire_and_preprocess_3dep_dems.py
 rename {tools => data/usgs}/preprocess_ahps_usgs.py (99%)
 rename {tools => data/usgs}/preprocess_download_usgs_grids.py (99%)

diff --git a/Dockerfile b/Dockerfile
index a9cf355a1..f62059ae6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -68,7 +68,7 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 RUN apt update --fix-missing 
-RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
+RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux
 
 RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
 
diff --git a/Dockerfile.prod b/Dockerfile.prod
deleted file mode 100644
index a71933c6c..000000000
--- a/Dockerfile.prod
+++ /dev/null
@@ -1,103 +0,0 @@
-## Temporary image to build the libraries and only save the needed artifacts
-FROM osgeo/gdal:ubuntu-full-3.1.2 AS builder
-WORKDIR /opt/builder
-ARG dataDir=/data
-ARG projectDir=/foss_fim
-ARG depDir=/dependencies
-ARG taudemVersion=98137bb6541a0d0077a9c95becfed4e56d0aa0ac
-ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
-ENV DEBIAN_FRONTEND noninteractive
-ENV taudemDir=$depDir/taudem/bin
-ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
-
-RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/* 
-
-RUN git clone https://github.com/dtarb/taudem.git
-RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
-
-RUN apt-get update --fix-missing && apt-get install -y cmake mpich \
-    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/* 
-
-## Compile Main taudem repo ##
-RUN mkdir -p taudem/bin
-RUN cd taudem \
-    && git checkout $taudemVersion \
-    && cd src \
-    && make
-
-## Compile taudem repo with accelerated flow directions ##
-RUN cd taudem_accelerated_flowDirections/taudem \
-    && git checkout $taudemVersion2 \
-    && mkdir build \
-    && cd build \
-    && cmake .. \
-    && make
-
-RUN mkdir -p $taudemDir
-RUN mkdir -p $taudemDir2
-
-## Move needed binaries to the next stage of the image
-RUN cd taudem/bin && mv -t $taudemDir flowdircond aread8 threshold streamnet gagewatershed catchhydrogeo dinfdistdown
-RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d8flowdir dinfflowdir
-
-
-
-
-###############################################################################################
-
-
-
-# Base Image that has GDAL, PROJ, etc
-FROM osgeo/gdal:ubuntu-full-3.1.2
-ARG dataDir=/data
-ARG projectDir=/foss_fim
-ARG depDir=/dependencies
-ENV inputDataDir=$dataDir/inputs
-ENV outputDataDir=$dataDir/outputs 
-ENV srcDir=$projectDir/src
-ENV taudemDir=$depDir/taudem/bin
-ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
-
-## ADDING FIM GROUP ##
-ARG GroupID=1370800235
-ARG GroupName=fim
-RUN addgroup --gid $GroupID $GroupName
-ENV GID=$GroupID
-ENV GN=$GroupName
-
-RUN mkdir -p $depDir
-COPY --from=builder $depDir $depDir
-
-RUN apt update --fix-missing
-RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1
-
-RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
-#COPY install_grass.exp .
-#RUN ./install_grass.exp
-
-RUN apt auto-remove
-
-## adding environment variables for numba and python ##
-ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
-ENV PYTHONUNBUFFERED=TRUE
-
-## ADD TO PATHS ##
-ENV PATH="$projectDir:${PATH}"
-ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
-
-## install python 3 modules ##
-COPY Pipfile .
-COPY Pipfile.lock .
-RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
-
-## Copy the source code to the image
-COPY . $projectDir/
-
-## Set user:group for running docker in detached mode
-USER root:$GroupName
-
-# RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
-ADD ./src/entrypoint.sh /
-ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
-
diff --git a/src/acquire_and_preprocess_inputs.py b/data/acquire_and_preprocess_inputs.py
similarity index 100%
rename from src/acquire_and_preprocess_inputs.py
rename to data/acquire_and_preprocess_inputs.py
diff --git a/data/create_vrt_file.py b/data/create_vrt_file.py
new file mode 100644
index 000000000..ca15dc147
--- /dev/null
+++ b/data/create_vrt_file.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import sys
+
+from datetime import datetime
+from osgeo import gdal
+
+sys.path.append('/foss_fim/src')
+from utils.shared_functions import FIM_Helpers as fh
+
+def create_vrt_file(src_directory, vrt_file_name):
+    
+    '''
+    Overview
+    ----------
+    Takes all .tif files in a given directory and creates a vrt from them.
+    Note: This assumes all .tifs share a common directory.
+    
+    Parameters
+    ----------
+        - src_directory (str):
+            Location where the .tifs are at
+        - vrt_file_name (str):
+            The name of the vrt file to be created. Note: it will be in the same
+            directory as the tifs.
+    '''
+  
+    # -------------------
+    # Validation
+    if (not os.path.exists(src_directory)):
+        raise ValueError(f'src_directory value of {src_directory}'\
+                          ' not set to a valid path')
+
+    if (vrt_file_name is None) or (vrt_file_name == ""):
+        raise ValueError(f'vrt_file_name not defined.')
+        
+    if (not vrt_file_name.endswith(".vrt")):
+        vrt_file_name += ".vrt"
+
+    # -------------------
+    
+    target_vrt_file_path = os.path.join(src_directory, vrt_file_name)
+
+    # -------------------
+    # setup logs
+    start_time = datetime.now()
+    fh.print_start_header('Creating vrt file', start_time)
+
+    __setup_logger(src_directory)
+    logging.info(f"saving vrt to {target_vrt_file_path}")
+
+    # -------------------
+    # processing
+    
+    tif_file_names = fh.get_file_names(src_directory, '.tif')
+        
+    __create_vrt(tif_file_names, target_vrt_file_path)
+
+    end_time = datetime.now()
+    fh.print_end_header('Loading 3dep dems', start_time, end_time)
+    logging.info(fh.print_date_time_duration(start_time, end_time))
+
+
+def __create_vrt(tif_file_names, target_vrt_file_path):
+    
+    logging.info("Files included:")
+    for file_name in tif_file_names:
+        logging.info(f" - {file_name}")
+    
+    result = gdal.BuildVRT(target_vrt_file_path, tif_file_names)
+    logging.info(result)
+
+
+def __setup_logger(output_folder_path):
+
+    start_time = datetime.now()
+    file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
+    log_file_name = f"vrt_build-{file_dt_string}.log"
+
+    log_file_path = os.path.join(output_folder_path, log_file_name)
+
+    file_handler = logging.FileHandler(log_file_path)
+    file_handler.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.DEBUG)
+
+    logger = logging.getLogger()
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+    logger.setLevel(logging.DEBUG)    
+    
+    logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
+    logging.info("----------------")
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Create a vrt using all tifs in a given directory')
+
+    parser.add_argument('-s','--src_directory', help='A directory of where the .tif files '\
+                        'files exist. If the -f (tif-file) param is empty then all .tif files '\
+                        'in this directory will be used.', 
+                        required=True)
+    
+    parser.add_argument('-n','--vrt_file_name', help='Name of the vrt file (name only) to be created. ' \
+                        'Note: it will be created in the source directory.', 
+                        required=True)
+    
+    args = vars(parser.parse_args())
+
+    create_vrt_file(**args)
diff --git a/tools/preprocess_ahps_nws.py b/data/nws/preprocess_ahps_nws.py
similarity index 99%
rename from tools/preprocess_ahps_nws.py
rename to data/nws/preprocess_ahps_nws.py
index 8c5a5f5ec..4659e3dc8 100644
--- a/tools/preprocess_ahps_nws.py
+++ b/data/nws/preprocess_ahps_nws.py
@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
+import argparse
+import geopandas as gpd
 import numpy as np
-from pathlib import Path
+import os
 import pandas as pd
-import geopandas as gpd
 import rasterio
-from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
-import argparse
-from dotenv import load_dotenv
-import os
-import traceback
 import sys
-sys.path.append('/foss_fim/src')
+import traceback
+
+from dotenv import load_dotenv
+from pathlib import Path
+sys.path.append('/foss_fim/tools')
+from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 
 
 def get_env_paths():
diff --git a/src/preprocess_rasters.py b/data/preprocess_rasters.py
similarity index 99%
rename from src/preprocess_rasters.py
rename to data/preprocess_rasters.py
index bf82d91fa..9bcad5fa2 100755
--- a/src/preprocess_rasters.py
+++ b/data/preprocess_rasters.py
@@ -7,6 +7,8 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 import argparse
+
+sys.path.append('/foss_fim/src')
 from utils.reproject_dem import reproject_dem
 from utils.shared_functions import update_raster_profile
 from utils.shared_variables import PREP_PROJECTION, PREP_PROJECTION_CM
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
new file mode 100644
index 000000000..6ad83f8e1
--- /dev/null
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+
+import argparse
+import glob
+import logging
+import os
+import subprocess
+import sys
+import traceback
+
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+from datetime import datetime
+#from tqdm import tqdm
+
+sys.path.append('/foss_fim/src')
+import utils.shared_variables as sv
+import utils.shared_functions as sf
+
+from utils.shared_functions import FIM_Helpers as fh
+
+# local constants (until changed to input param)
+__USGS_3DEP_10M_VRT_URL = r'/vsicurl/https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/TIFF/USGS_Seamless_DEM_13.vrt'  # 10m = 13 (1/3 arc second)
+
+
+def acquire_and_preprocess_3dep_dems(extent_file_path,
+                                     target_output_folder_path = '', 
+                                     number_of_jobs = 1, 
+                                     retry = False):
+    
+    '''
+    Overview
+    ----------
+    This will download 3dep rasters from USGS using USGS vrts.
+    By default USGS 3Dep stores all their rasters in lat/long (northing and easting).
+    By us downloading the rasters using WBD HUC4 clips and gdal, we an accomplish a few extra
+    steps.
+        1) Ensure the projection types that are downloaded are consistant and controlled.
+           We are going to download them as NAD83 basic (espg: 4269) which is consistant
+           with other data sources, even though FIM defaults to ESRI:102039. We will
+           change that as we add the clipped version per HUC8.
+        2) ensure we are adjusting blocksizes, compression and other raster params
+        3) Create the 3dep rasters in the size we want (default at HUC4 for now)
+        
+    Notes:
+        - As this is a very low use tool, all values such as the USGS vrt path, output
+          folder paths, huc unit level (huc4), etc are all hardcoded
+        
+    Parameters
+    ----------
+        - extent_file_path (str):
+            Location of where the extent files that are to be used as clip extent against
+            the USGS 3Dep vrt url.
+            ie) \data\inputs\wbd\HUC4
+            
+        - target_output_folder_path (str):
+            The output location of the new 3dep dem files. When the param is not submitted,
+            it will be sent to /data/input/usgs/3dep_dems/10m/.
+    
+        - number_of_jobs (int):
+            This program supports multiple procs if multiple procs/cores are available.
+            
+        - retry (True / False):
+            If retry is True and the file exists (either the raw downloaded DEM and/or)
+            the projected one, then skip it
+    '''
+    # -------------------
+    # Validation
+    total_cpus_available = os.cpu_count() - 1
+    if number_of_jobs > total_cpus_available:
+        raise ValueError('The number of jobs {number_of_jobs}'\
+                          'exceeds your machine\'s available CPU count minus one. '\
+                          'Please lower the number of jobs '\
+                          'values accordingly.'.format(number_of_jobs)
+                        )
+
+    if (not os.path.exists(extent_file_path)):
+        raise ValueError(f'extent_file_path value of {extent_file_path}'\
+                          ' not set to a valid path')
+    
+    if (target_output_folder_path is None) or (target_output_folder_path == ""):
+        target_output_folder_path = sv.INPUT_DEMS_3DEP_10M_DIR
+    
+    if (not os.path.exists(target_output_folder_path)):
+        raise ValueError(f"Output folder path {target_output_folder_path} does not exist" )
+   
+    # -------------------
+    # setup logs
+    start_time = datetime.now()
+    fh.print_start_header('Loading 3dep dems', start_time)
+   
+    print(f"Downloading to {target_output_folder_path}")
+    __setup_logger(target_output_folder_path)
+    logging.info(f"Downloading to {target_output_folder_path}")
+    
+    
+    # -------------------
+    # processing
+    
+    # Get the WBD .gpkg files (or clip extent)
+    extent_file_names = fh.get_file_names(extent_file_path, 'gpkg')
+    msg = f"Extent files coming from {extent_file_path}"
+    print(msg)
+    logging.info(msg)
+   
+    # download dems, setting projection, block size, etc
+    __download_usgs_dems(extent_file_names, target_output_folder_path, number_of_jobs, retry)
+    
+    end_time = datetime.now()
+    fh.print_end_header('Loading 3dep dems', start_time, end_time)
+    logging.info(fh.print_date_time_duration(start_time, end_time))
+
+
+def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry):
+    
+    '''
+    Process:
+    ----------
+    download the actual raw (non reprojected files) from the USGS
+    based on stated embedded arguments
+    
+    Parameters
+    ----------
+        - fl (object of fim_logger (must have been created))
+        - remaining params are defined in acquire_and_preprocess_3dep_dems
+    '''
+
+    print(f"==========================================================")
+    print(f"-- Downloading USGS DEMs Starting")
+    
+    base_cmd =  'gdalwarp {0} {1}'
+    base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
+    base_cmd += ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
+    base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES"'
+    base_cmd += ' -t_srs EPSG:5070'
+   
+    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
+
+        executor_dict = {}
+        
+        for idx, extent_file in enumerate(extent_files):
+            
+            download_dem_args = { 
+                                'extent_file': extent_file,
+                                'output_folder_path': output_folder_path,
+                                'download_url': __USGS_3DEP_10M_VRT_URL,
+                                'base_cmd':base_cmd,
+                                'retry': retry
+                                }
+        
+            try:
+                future = executor.submit(download_usgs_dem_file, **download_dem_args)
+                executor_dict[future] = extent_file
+            except Exception as ex:
+                
+                summary = traceback.StackSummary.extract(
+                        traceback.walk_stack(None))
+                print(f"*** {ex}")                
+                print(''.join(summary.format()))    
+                
+                logging.critical(f"*** {ex}")
+                logging.critical(''.join(summary.format()))
+
+                sys.exit(1)
+            
+        # Send the executor to the progress bar and wait for all FR tasks to finish
+        sf.progress_bar_handler(executor_dict, f"Downloading USGG 3Dep Dems")
+
+    print(f"-- Downloading USGS DEMs Completed")
+    logging.info(f"-- Downloading USGS DEMs Completed")
+    print(f"==========================================================")    
+    
+        
+def download_usgs_dem_file(extent_file, 
+                           output_folder_path, 
+                           download_url,
+                           base_cmd,
+                           retry):
+
+    '''
+    Process:
+    ----------    
+        Downloads just one dem file from USGS. This is setup as a method
+        to allow for multi-processing.
+
+        
+    Parameters:
+    ----------
+        - extent_file (str)
+             When the dem is downloaded, it is clipped against this extent (.gkpg) file.
+        - output_folder_path (str)
+             Location of where the output file will be stored
+        - download_url (str)
+             URL for the USGS download site (note: Should include '/vsicurl/' at the 
+             front of the URL)
+        - base_cmd (str)
+             The basic GDAL command with string formatting wholes for key values.
+             See the cmd variable below.
+             ie)
+                base_cmd =  'gdalwarp {0} {1}'
+                base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
+                base_cmd +=  ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
+                base_cmd +=  ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES"'
+                base_cmd +=  ' -t_srs EPSG:5070'
+        - retry (bool)
+             If True, and the file exists (and is over 0k), downloading will be skipped.
+        
+    '''
+    
+    basic_file_name = os.path.basename(extent_file).split('.')[0]
+    target_file_name_raw = f"{basic_file_name}_dem.tif" # as downloaded
+    target_path_raw = os.path.join(output_folder_path,
+                                    target_file_name_raw)
+    
+    # File might exist from a previous failed run. If it was aborted or failed
+    # on a previous attempt, it's size less than 1mg, so delete it.
+    # 
+    # IMPORTANT:
+    #
+    # it might be compromised on a previous run but GREATER 1mg (part written).
+    # That scenerio is not handled as we can not tell if it completed.
+    
+    if (retry) and (os.path.exists(target_path_raw)):
+        if (os.stat(target_path_raw).st_size < 1000000):
+            os.remove(target_path_raw)
+        else:
+            msg = f" - Downloading -- {target_file_name_raw} - Skipped (already exists (see retry flag))"
+            print(msg)  
+            logging.info(msg)
+            return
+    
+    msg = f" - Downloading -- {target_file_name_raw} - Started"
+    print(msg)
+    logging.info(msg)
+            
+    cmd = base_cmd.format(download_url,
+                            target_path_raw,
+                            extent_file)
+    #PREP_PROJECTION_EPSG
+    #fh.vprint(f"cmd is {cmd}", self.is_verbose, True)
+    #print(f"cmd is {cmd}")
+    
+    # didn't use Popen becuase of how it interacts with multi proc
+    # was creating some issues. Run worked much better.
+    process = subprocess.run(cmd, shell = True,
+                                stdout = subprocess.PIPE, 
+                                stderr = subprocess.PIPE,
+                                check = True,
+                                universal_newlines=True) 
+
+    msg = process.stdout
+    print(msg)
+    logging.info(msg)
+    
+    if (process.stderr != ""):
+        if ("ERROR" in process.stderr.upper()):
+            msg = f" - Downloading -- {target_file_name_raw}"\
+                    f"  ERROR -- details: ({process.stderr})"
+            print(msg)
+            logging.error(msg)
+            os.remove(target_path_raw)
+    else:
+        msg = f" - Downloading -- {target_file_name_raw} - Complete"
+        print(msg)
+        logging.info(msg)
+
+
+def __setup_logger(output_folder_path):
+
+    start_time = datetime.now()
+    file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
+    log_file_name = f"3Dep_downloaded-{file_dt_string}.log"
+
+    log_file_path = os.path.join(output_folder_path, log_file_name)
+
+    file_handler = logging.FileHandler(log_file_path)
+    file_handler.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.DEBUG)
+
+    logger = logging.getLogger()
+    logger.addHandler(file_handler)
+    # logger.addHandler(console_handler)
+    logger.setLevel(logging.DEBUG)    
+    
+    logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
+    logging.info("----------------")
+
+
+if __name__ == '__main__':
+
+    # Parse arguments.
+    
+    # sample usage (min params):
+    # - python3 /foss_fim/data/usgs/acquire_and_preprocess_3dep_dems.py -e /data/inputs/wbd/HUC4
+    
+    # Notes:
+    #   - This is a very low use tool. So for now, this only can load 10m (1/3 arc second) and is using
+    #     hardcoded paths for the wbd gpkg to be used for clipping (no buffer for now).
+    #     Also hardcoded usgs 3dep urls, etc.  Minor
+    #     upgrades can easily be made for different urls, output folder paths, huc units, etc
+    #     as/if needed (command line params)
+    #   - The output path can be adjusted in case of a test reload of newer data for 3dep.
+    #     The default is /data/input/usgs/3dep_dems/10m/
+    #   - While you can (and should use more than one job number (if manageable by your server)),
+    #     this tool is memory intensive and needs more RAM then it needs cores / cpus. Go ahead and 
+    #     anyways and increase the job number so you are getting the most out of your RAM. Or
+    #     depending on your machine performance, maybe half of your cpus / cores. This tool will
+    #     not fail or freeze depending on the number of jobs / cores you select.
+        
+        
+    # IMPORTANT: 
+    # (Sept 2022): we do not process HUC2 of 19 (alaska) or 22 (misc US pacific islands).
+    # They need to be removed from the input src clip directory in the first place.
+    # They can not be reliably removed in code.
+       
+    parser = argparse.ArgumentParser(description='Acquires and preprocesses USGS 3Dep dems')
+
+    parser.add_argument('-e','--extent_file_path', help='location the gpkg files that will'\
+                        ' are being used as clip regions (aka.. huc4_*.gpkg or whatever).'\
+                        ' All gpkgs in this folder will be used.', required=True)
+
+    parser.add_argument('-j','--number_of_jobs', help='Number of (jobs) cores/processes to used.', 
+                        required=False, default=1, type=int)
+
+    parser.add_argument('-r','--retry', help='If included, it will skip files that already exist.'\
+                        ' Default is all will be loaded/reloaded.', 
+                        required=False, action='store_true', default=False)
+
+    parser.add_argument('-t','--target_output_folder_path', help='location of where the 3dep files'\
+                        ' will be saved', required=False, default='')
+
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    acquire_and_preprocess_3dep_dems(**args)
+
diff --git a/tools/preprocess_ahps_usgs.py b/data/usgs/preprocess_ahps_usgs.py
similarity index 99%
rename from tools/preprocess_ahps_usgs.py
rename to data/usgs/preprocess_ahps_usgs.py
index e34725b4d..1c8b78be3 100644
--- a/tools/preprocess_ahps_usgs.py
+++ b/data/usgs/preprocess_ahps_usgs.py
@@ -1,17 +1,18 @@
 #!/usr/bin/env python3
+import argparse
 import numpy as np
-from pathlib import Path
+import os
 import pandas as pd
 import rasterio
 import requests
-from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
-import argparse
-from dotenv import load_dotenv
-import os
 import sys
-sys.path.append('/foss_fim/src')
 import traceback
 
+from dotenv import load_dotenv
+from pathlib import Path
+
+sys.path.append('/foss_fim/tools')
+from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 
 def get_env_paths():
     load_dotenv()
diff --git a/tools/preprocess_download_usgs_grids.py b/data/usgs/preprocess_download_usgs_grids.py
similarity index 99%
rename from tools/preprocess_download_usgs_grids.py
rename to data/usgs/preprocess_download_usgs_grids.py
index 332a30ed1..8d85a6e1e 100644
--- a/tools/preprocess_download_usgs_grids.py
+++ b/data/usgs/preprocess_download_usgs_grids.py
@@ -1,13 +1,14 @@
 #!/usr/bin/env python3
-import urllib.request 
-from pathlib import Path
-from dotenv import load_dotenv
-import os
 import argparse
+import os
+import pandas as pd
 import requests
-from collections import defaultdict
 import urllib
-import pandas as pd
+import urllib.request 
+
+from collections import defaultdict
+from dotenv import load_dotenv
+from pathlib import Path
 
 load_dotenv()
 USGS_DOWNLOAD_URL = os.getenv("USGS_DOWNLOAD_URL")
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index cdd28546c..1e5b26802 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,51 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.9.3 - 2022-09-13 - [PR #681](https://github.com/NOAA-OWP/inundation-mapping/pull/681)
+
+Created a new tool to downloaded USGS 3Dep DEM's via their S3 bucket.
+
+Other changes:
+ - Some code file re-organization in favour of the new `data` folder which is designed for getting, setting, and processing data from external sources such as AWS, WBD, NHD, NWM, etc.
+ - Added tmux as a new tool embedded inside the docker images.
+
+### Additions
+
+- `data`
+   - `usgs`
+      - `acquire_and_preprocess_3dep_dems.py`:  The new tool as described above. For now it is hardcoded to a set path for USGS AWS S3 vrt file but may change later for it to become parameter driven.
+ - `create_vrt_file.py`: This is also a new tool that can take a directory of geotiff files and create a gdal virtual file, .vrt extention, also called a `virtual raster`. Instead of clipping against HUC4, 6, 8's raster files, and run risks of boundary issues, vrt's actual like all of the tif's are one giant mosaiced raster and can be clipped as one.
+
+### Removals
+
+- 'Dockerfile.prod`:  No longer being used (never was used)
+
+### Changes
+
+- `Dockerfile`:  Added apt install for tmux. This tool will now be available in docker images and assists developers.
+
+- `data`
+   - `acquire_and_preprocess_inputs.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+   - `nws`
+      - `preprocess_ahps_nws.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+      - `preprocess_rasters.py`: moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+    - `usgs`
+         - `preprocess_ahps_usgs.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+         - `preprocess_download_usgs_grids.py`: moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+
+ - `src`
+     - `utils`
+         - `shared_functions.py`:  changes made were
+              - Cleanup the "imports" section of the file (including a change to how the utils.shared_variables file is loaded.
+              - Added `progress_bar_handler` function which can be re-used by other code files.
+              - Added `get_file_names` which can create a list of files from a given directory matching a given extension. 
+              - Modified `print_current_date_time` and `print_date_time_duration` and  methods to return the date time strings. These helper methods exist to help with standardization of logging and output console messages.
+              - Added `print_start_header` and `print_end_header` to help with standardization of console and logging output messages.
+          - `shared_variables.py`: Additions in support of near future functionality of having fim load DEM's from USGS 3DEP instead of NHD rasters.
+
+<br/><br/>
+
+
 ## v4.0.9.2 - 2022-09-12 - [PR #678](https://github.com/NOAA-OWP/inundation-mapping/pull/678)
 
 This fixes several bugs related to branch definition and trimming due to waterbodies.
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 6caa2873a..f109ac4ea 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -1,20 +1,25 @@
 #!/usr/bin/env python3
 
 import os
+import glob
 import inspect
 import re
+import sys
 
-from os.path import splitext
+from concurrent.futures import as_completed
 from datetime import datetime, timezone
 from pathlib import Path
+from os.path import splitext
 
 import fiona
-import pandas as pd
 import numpy as np
+import pandas as pd
 import rasterio
+import utils.shared_variables as sv
 
-from rasterio.warp import calculate_default_transform, reproject, Resampling
 from pyproj.crs import CRS
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+from tqdm import tqdm
 
 
 def getDriver(fileName):
@@ -71,7 +76,7 @@ def run_system_command(args):
 def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
 
     import geopandas as gp
-    from utils.shared_variables import CONUS_STATE_LIST, PREP_PROJECTION
+
 
     print("Subsetting " + wbd_gpkg + "...")
     # Read geopackage into dataframe.
@@ -82,12 +87,12 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
         state = row["STATES"]
         if state != None:  # Some polygons are empty in the STATES field.
             keep_flag = False  # Default to Fault, i.e. to delete the polygon.
-            if state in CONUS_STATE_LIST:
+            if state in sv.CONUS_STATE_LIST:
                 keep_flag = True
             # Only split if multiple states present. More efficient this way.
             elif len(state) > 2:
                 for wbd_state in state.split(","):  # Some polygons have multiple states, separated by a comma.
-                    if wbd_state in CONUS_STATE_LIST:  # Check each polygon to make sure it's state abbrev name is allowed.
+                    if wbd_state in sv.CONUS_STATE_LIST:  # Check each polygon to make sure it's state abbrev name is allowed.
                         keep_flag = True
                         break
             if not keep_flag:
@@ -95,7 +100,7 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
 
     # Overwrite geopackage.
     layer_name = os.path.split(wbd_gpkg)[1].strip('.gpkg')
-    gdf.crs = PREP_PROJECTION
+    gdf.crs = sv.PREP_PROJECTION
     gdf.to_file(multilayer_wbd_geopackage, layer=layer_name,driver='GPKG',index=False)
 
 
@@ -303,10 +308,18 @@ def concat_huc_csv(fim_dir,csv_name):
         print(f"Creating aggregate csv")
         concat_df = pd.concat(merged_csv)
         return concat_df
-      
 
 
+def progress_bar_handler(executor_dict, desc):
 
+    for future in tqdm(as_completed(executor_dict),
+                    total=len(executor_dict),
+                    desc=desc
+                    ):
+        try:
+            future.result()
+        except Exception as exc:
+            print('{}, {}, {}'.format(executor_dict[future],exc.__class__.__name__,exc))
 
 
 # #####################################
@@ -448,6 +461,53 @@ def load_list_file(file_name_and_path):
             
         return line_values       
 
+    # -----------------------------------------------------------
+    @staticmethod
+    def get_file_names(src_folder, file_extension):
+        '''
+        Process
+        ----------
+        Get a list of file names and paths matching the file extension
+        
+        Parameters
+        ----------
+            - src_folder (str)
+                Location of the files.
+
+            - file_extension (str)
+                All files matching this file_extension will be added to the list.
+
+        
+        Returns
+        ----------
+        A list of file names and paths
+        '''
+
+        if (not file_extension) and (len(file_extension.strip()) == 0):
+            raise ValueError(f"file_extension value not set")
+        
+        # remove the starting . if it exists
+        if (file_extension.startswith(".")):
+            file_extension = file_extension[1:]
+        
+        # test that folder exists
+        if (not os.path.exists(src_folder)):
+            raise ValueError(f"{file_extension} src folder of {src_folder} not found")
+        
+        if (not src_folder.endswith("/")):
+            src_folder += "/"
+        
+        glob_pattern = f"{src_folder}*.{file_extension}"
+        file_list = glob.glob(glob_pattern)
+    
+        if (len(file_list) == 0):
+            raise Exception(f"files with the extension of {file_extension} "\
+                            f" in the {src_folder} did not load or do not exist")
+        
+        file_list.sort()
+        
+        return file_list
+
     # -----------------------------------------------------------
     @staticmethod
     def print_current_date_time():
@@ -462,12 +522,18 @@ def print_current_date_time():
         Usage:
             from utils.shared_functions import FIM_Helpers as fh
             fh.print_current_date_time()
+
+        -------
+        Returns:
+            Current date / time as a formatted string
         
         '''
         d1 = datetime.now()
         dt_stamp = "Current date and time : "
         dt_stamp += d1.strftime("%Y-%m-%d %H:%M:%S")
         print (dt_stamp)
+        
+        return dt_stamp
 
     # -----------------------------------------------------------
     @staticmethod
@@ -485,6 +551,10 @@ def print_date_time_duration(start_dt, end_dt):
             from utils.shared_functions import FIM_Helpers as fh
             fh.print_current_date_time()
         
+        -------
+        Returns:
+            Duration as a formatted string
+            
         '''
         time_delta = (end_dt - start_dt)
         total_seconds = int(time_delta.total_seconds())
@@ -495,5 +565,25 @@ def print_date_time_duration(start_dt, end_dt):
 
         time_fmt = f"{total_hours:02d} hours {total_mins:02d} mins {seconds:02d} secs"
         
-        print("Duration: " + time_fmt)
+        duration_msg = "Duration: " + time_fmt
+        print(duration_msg)
+        
+        return duration_msg
+
+    # -----------------------------------------------------------
+    @staticmethod
+    def print_start_header(friendly_program_name, start_time):
+        
+        print("================================")
+        dt_string = start_time.strftime("%m/%d/%Y %H:%M:%S")
+        print(f"Start {friendly_program_name} : {dt_string}")
+        print()
+
+    # -----------------------------------------------------------
+    @staticmethod
+    def print_end_header(friendly_program_name, start_time, end_time):
         
+        print("================================")
+        dt_string = end_time.strftime("%m/%d/%Y %H:%M:%S")
+        print (f"End {friendly_program_name} : {dt_string}")
+        print()
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 4e704aa46..3cac204be 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -4,15 +4,27 @@
 
 # Projections.
 #PREP_PROJECTION = "+proj=aea +datum=NAD83 +x_0=0.0 +y_0=0.0 +lon_0=96dW +lat_0=23dN +lat_1=29d30'N +lat_2=45d30'N +towgs84=-0.9956000824677655,1.901299877314078,0.5215002840524426,0.02591500053005733,0.009425998542707753,0.01159900118427752,-0.00062000005129903 +no_defs +units=m"
+
+# These two are ESRI:102039 (USA_Contiguous_Albers_Equal_Area_Conic_USGS_version)
 PREP_PROJECTION_CM = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Albers"],PARAMETER["false_easting",0.0],PARAMETER["false_northing",0.0],PARAMETER["central_meridian",-96.0],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_origin",23.0],UNIT["Meter",1.0],VERTCS["NAVD_1988",VDATUM["North_American_Vertical_Datum_1988"],PARAMETER["Vertical_Shift",0.0],PARAMETER["Direction",1.0],UNIT["Centimeter",0.01]]]'
-PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'
+PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'  
+DEFAULT_FIM_PROJECTION_CRS = 'ESRI:102039'  
 VIZ_PROJECTION ='PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
+
 # -- Data URLs-- #
 NHD_URL_PARENT = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/'
 NWM_HYDROFABRIC_URL = r'http://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_channel_hydrofabric.tar.gz'  # Temporary
 WBD_NATIONAL_URL = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip'
 WBD_HU2_URL_PARENT = r'http://prd-tnm.s3-website-us-west-2.amazonaws.com/?prefix=StagedProducts/Hydrography/WBD/HU2/GDB'
 
+
+# -- Local Data paths -- #
+# For 10m (1/3 arc second)
+INPUT_DEMS_3DEP_10M_DIR = '/data/inputs/usgs/3dep_dems/10m'
+INPUT_DEMS_3DEP_10M_FIM_VRT = '/data/inputs/usgs/3dep_dems/10m/FIM_Seamless_DEM_10m.vrt'
+WBD_HUC4_DIR = '/data/inputs/wbd/HUC4'
+
+
 # -- Prefixes and Suffixes -- #
 NHD_URL_PREFIX = 'NHDPLUS_H_'
 NHD_RASTER_URL_SUFFIX = '_HU4_RASTER.7z'

From 5cf2c95422290fe3a6f955a531a96a90738e7c7e Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 30 Sep 2022 14:36:11 -0500
Subject: [PATCH 163/359] v4.0.9.4 Cleanup Branch Zero output at the end of a
 processing run (#691)

---
 config/deny_gms_branch_unittests.lst          |   5 +-
 config/deny_gms_branch_zero.lst               |   1 +
 ..._default.lst => deny_gms_branches_dev.lst} |   2 +
 ...hes_min.lst => deny_gms_branches_prod.lst} |   4 +-
 ...nit_default.lst => deny_gms_unit_prod.lst} |   1 +
 config/params_template.env                    |   2 +-
 docs/CHANGELOG.md                             |  42 ++++-
 gms_pipeline.sh                               |  20 ++-
 gms_run_branch.sh                             |  26 ++-
 gms_run_unit.sh                               |  10 +-
 src/gms/outputs_cleanup.py                    | 112 +++++++++---
 src/gms/run_by_branch.sh                      |   2 +-
 src/gms/run_by_unit.sh                        |   4 +-
 unit_tests/README.md                          |   4 +-
 unit_tests/__template_unittests.py            |   1 -
 ...catchments_and_add_attributes_unittests.py |   4 +-
 unit_tests/gms/outputs_cleanup_params.json    |  25 +++
 unit_tests/gms/outputs_cleanup_unittests.py   | 163 ++++++++++++++++++
 18 files changed, 371 insertions(+), 57 deletions(-)
 rename config/{deny_gms_branches_default.lst => deny_gms_branches_dev.lst} (97%)
 rename config/{deny_gms_branches_min.lst => deny_gms_branches_prod.lst} (93%)
 rename config/{deny_gms_unit_default.lst => deny_gms_unit_prod.lst} (96%)
 create mode 100644 unit_tests/gms/outputs_cleanup_params.json
 create mode 100644 unit_tests/gms/outputs_cleanup_unittests.py

diff --git a/config/deny_gms_branch_unittests.lst b/config/deny_gms_branch_unittests.lst
index 530403984..cb947ee1c 100644
--- a/config/deny_gms_branch_unittests.lst
+++ b/config/deny_gms_branch_unittests.lst
@@ -16,7 +16,7 @@ crosswalk_table_{}.csv
 #demDerived_reaches_{}.shx
 #demDerived_reaches_split_{}.gpkg
 #demDerived_reaches_split_filtered_{}.gpkg
-demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+#demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
 demDerived_reaches_split_points_{}.gpkg
 demDerived_streamPixels_{}.tif
 demDerived_streamPixels_ids_{}.tif
@@ -37,10 +37,11 @@ gw_catchments_pixels_{}.tif
 gw_catchments_reaches_{}.tif
 #gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 #gw_catchments_reaches_filtered_addedAttributes_{}.tif
-gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
+LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
diff --git a/config/deny_gms_branch_zero.lst b/config/deny_gms_branch_zero.lst
index 93bbf79b9..a75f8c5ef 100644
--- a/config/deny_gms_branch_zero.lst
+++ b/config/deny_gms_branch_zero.lst
@@ -41,6 +41,7 @@ gw_catchments_reaches_{}.tif
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
+#LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
diff --git a/config/deny_gms_branches_default.lst b/config/deny_gms_branches_dev.lst
similarity index 97%
rename from config/deny_gms_branches_default.lst
rename to config/deny_gms_branches_dev.lst
index ec7043e11..c9009488a 100644
--- a/config/deny_gms_branches_default.lst
+++ b/config/deny_gms_branches_dev.lst
@@ -41,7 +41,9 @@ gw_catchments_reaches_{}.tif
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
+LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
+nld_subset_levees_{}.tif
 #nwm_catchments_proj_subset_levelPaths_{}.gpkg
 #nwm_subset_streams_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
diff --git a/config/deny_gms_branches_min.lst b/config/deny_gms_branches_prod.lst
similarity index 93%
rename from config/deny_gms_branches_min.lst
rename to config/deny_gms_branches_prod.lst
index 4492a170f..a5d1ecbba 100644
--- a/config/deny_gms_branches_min.lst
+++ b/config/deny_gms_branches_prod.lst
@@ -16,7 +16,7 @@ demDerived_reaches_{}.shp
 demDerived_reaches_{}.shx
 demDerived_reaches_split_{}.gpkg
 demDerived_reaches_split_filtered_{}.gpkg
-demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+#demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
 demDerived_reaches_split_points_{}.gpkg
 demDerived_streamPixels_{}.tif
 demDerived_streamPixels_ids_{}.tif
@@ -41,7 +41,9 @@ gw_catchments_reaches_{}.tif
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
+LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
+nld_subset_levees_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
diff --git a/config/deny_gms_unit_default.lst b/config/deny_gms_unit_prod.lst
similarity index 96%
rename from config/deny_gms_unit_default.lst
rename to config/deny_gms_unit_prod.lst
index 53f32b920..419187fbb 100644
--- a/config/deny_gms_unit_default.lst
+++ b/config/deny_gms_unit_prod.lst
@@ -3,6 +3,7 @@
 NHDPlusBurnLineEvent_subset.gpkg
 #branch_id.lst
 #branch_polygons.gpkg
+LandSea_subset.gpkg
 nhd_headwater_points_subset.gpkg
 #nld_subset_levees.gpkg
 #nwm_catchments_proj_subset.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index 49db90523..9e14210d6 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -57,7 +57,7 @@ export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; F
 export nwm_recur_file="data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
 
 #### apply SRC adjustments using observed FIM/flow point database ####
-export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
+export src_adjust_spatial="False" # Toggle to run src adjustment routine (True=on; False=off)
 export fim_obs_pnt_data="data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 #### path to env file with sensitive paths for accessing postgres database ####
 export CALB_DB_KEYS_FILE="/data/config/calb_db_keys.env"
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1e5b26802..016591a31 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,47 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.9.4 - 2022-09-30 - [PR #691](https://github.com/NOAA-OWP/inundation-mapping/pull/691)
+
+Cleanup Branch Zero output at the end of a processing run. Without this fix, some very large files were being left on the file system. Adjustments and cleanup changed the full BED output run from appx 2 TB output to appx 1 TB output.
+
+### Additions
+
+- `unit_tests`
+    - `gms`
+        - `outputs_cleanup_params.json` and `outputs_cleanup_unittests.py`: The usual unit test files.
+
+### Changes
+
+- `gms_pipeline.sh`: changed variables and text to reflect the renamed default `deny_gms_branchs_prod.lst` and `deny_gms_unit_prod.lst` files. Also tells how a user can use the word 'none' for the deny list parameter (both or either unit or branch deny list) to skip output cleanup(s).
+
+- `gms_run_unit.sh`: changed variables and text to reflect the renamed default `deny_gms_unit_prod.lst` files. Also added a bit of minor output text (styling). Also tells how a user can use the word 'none' for the deny list parameter to skip output cleanup.
+
+- `gms_run_branch.sh`:
+       ... changed variables and text to reflect the renamed default `deny_gms_branches.lst` files. 
+       ... added a bit of minor output text (styling).
+       ... also tells how a user can use the word 'none' for the deny list parameter to skip output cleanup.
+       ... added a new section that calls the `outputs_cleanup.py` file and will do post cleanup on branch zero output files.
+
+- `src`
+    - `gms`
+        - `outputs_cleanup.py`: pretty much rewrote it in its entirety. Now accepts a manditory branch id (can be zero) and can recursively search subdirectories. ie) We can submit a whole output directory with all hucs and ask to cleanup branch 0 folder OR cleanup files in any particular directory as we did before (per branch id).
+          
+          - `run_by_unit.sh`:  updated to pass in a branch id (or the value of "0" meaning branch zero) to outputs_cleanup.py.
+          - `run_by_branch.sh`:  updated to pass in a branch id to outputs_cleanup.py.
+      
+- `unit_tests`
+    - `README.md`: updated to talk about the specific deny list for unit_testing.
+    - `__template_unittests.py`: updated for the latest code standards for unit tests. 
+
+- `config`
+    - `deny_gms_branch_unittest.lst`: Added some new files to be deleted, updated others.
+    - `deny_gms_branch_zero.lst`: Added some new files to be deleted.
+    - `deny_gms_branches_dev.lst`:  Renamed from `deny_gms_branches_default.lst` and some new files to be deleted, updated others. Now used primarily for development and testing use.
+    - `deny_gms_branches_prod.lst`:  Renamed from `deny_gms_branches_min` and some new files to be deleted, updated others. Now used primarily for when releasing a version to production.
+    -  `deny_gms_unit_prod.lst`: Renamed from `deny_gms_unit_default.lst`, yes... there currently is no "dev" version.  Added some new files to be deleted.
+
+<br/><br/>
 
 ## v4.0.9.3 - 2022-09-13 - [PR #681](https://github.com/NOAA-OWP/inundation-mapping/pull/681)
 
@@ -46,7 +87,6 @@ Other changes:
 
 <br/><br/>
 
-
 ## v4.0.9.2 - 2022-09-12 - [PR #678](https://github.com/NOAA-OWP/inundation-mapping/pull/678)
 
 This fixes several bugs related to branch definition and trimming due to waterbodies.
diff --git a/gms_pipeline.sh b/gms_pipeline.sh
index f9d9c7034..103c76a5c 100755
--- a/gms_pipeline.sh
+++ b/gms_pipeline.sh
@@ -23,14 +23,18 @@ usage ()
     echo '  -o/--overwrite  : overwrite outputs if already exist'
     echo '  -r/--retry      : retries failed jobs'
     echo '  -ud/--unitDenylist : A file with line delimited list of files in UNIT (HUC) directories to remove'
-    echo '                    upon completion (see config/deny_gms_unit_default.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_default.lst'
+    echo '                    upon completion (see config/deny_gms_unit_prod.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_prod.lst'
+    echo '                    -- Note: if you want all output files (aka.. no files removed),'
+    echo '                    use the word none as this value for this parameter.'
     echo '  -bd/--branchDenylist : A file with line delimited list of files in branches directories to remove' 
-    echo '                    upon completion (see config/deny_gms_branches_min.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches_min.lst'    
+    echo '                    upon completion (see config/deny_gms_branches_prod.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'   
+    echo '                    -- Note: if you want all output files (aka.. no files removed),'
+    echo '                    use the word none as this value for this parameter.'
 	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
-	echo '                    at the initial load of the nwm_subset_streams.'
-	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
+    echo '                    at the initial load of the nwm_subset_streams.'
+    echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
     echo
     exit
 }
@@ -101,11 +105,11 @@ then
 fi
 if [ "$deny_gms_unit_list" = "" ]
 then
-   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_default.lst
+   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_prod.lst
 fi
 if [ "$deny_gms_branches_list" = "" ]
 then
-   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_min.lst
+   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_prod.lst
 fi
 if [ -z "$overwrite" ]
 then
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 6dc72644a..018c934a3 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -19,13 +19,15 @@ usage ()
     echo '  -r/--retry      : retries failed jobs'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
     echo '  -d/--denylist    : A file with line delimited list of files in branches directories to remove' 
-    echo '                    upon completion (see config/deny_gms_branches_min.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches_min.lst'    
+    echo '                    upon completion (see config/deny_gms_branches_prod.lst for a starting point)'
+    echo '                    -- Note: if you want all output files (aka.. no files removed),'
+    echo '                    use the word none as this value for this parameter.'
+    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'    
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited).'
     echo '                    A line delimited file also acceptable. HUCs must present in inputs directory.'
-	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
-	echo '                    at the initial load of the nwm_subset_streams.'
-	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
+    echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
+    echo '                    at the initial load of the nwm_subset_streams.'
+    echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
     echo
     exit
 }
@@ -85,7 +87,7 @@ then
 fi
 if [ "$deny_gms_branches_list" = "" ]
 then
-   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_min.lst
+   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_prod.lst
 fi
 
 if [ "$overwrite" = "" ]
@@ -215,18 +217,26 @@ if [ "$src_adjust_spatial" = "True" ]; then
     time python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
 fi
 
+# -------------------
+## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO##
+if [ -f $deny_gms_branches_list ]; then
+    echo -e $startDiv"Cleaning up (Removing) files all branch zero for all HUCs"$stopDiv
+    date -u
+    $srcDir/gms/outputs_cleanup.py -d $outputRunDataDir -l $deny_gms_branches_list -b 0
+fi
+
 # -------------------
 ## GET NON ZERO EXIT CODES ##
 # Needed in case aggregation fails, we will need the logs
 echo
-echo "Start non-zero exit code checking"
+echo -e $startDiv"Start non-zero exit code checking"$stopDiv
 find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
 
 # -------------------
 ## REMOVE FAILED BRANCHES ##
 # Needed in case aggregation fails, we will need the logs
 echo
-echo "Removing branches that failed with Exit status: 61"
+echo -e $startDiv"Removing branches that failed with Exit status: 61"$stopDiv
 python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g $outputRunDataDir/gms_inputs.csv
 
 echo "=========================================================================="
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index bd6b69180..be302f5a0 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -21,7 +21,9 @@ usage ()
     echo '  -o/--overwrite  : overwrite outputs if already exist'
     echo '  -r/--retry      : retries failed jobs'
     echo '  -d/--denylist   : file with line delimited list of files in huc directories to remove upon completion'
-    echo '                   (see config/deny_gms_unit_default.lst for a starting point)'
+    echo '                   (see config/deny_gms_unit_prod.lst for a starting point)'
+    echo '                    -- Note: if you want all output files (aka.. no files removed),'
+    echo '                    use the word none as this value for this parameter.'
 	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
 	echo '                    at the initial load of the nwm_subset_streams.'
 	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
@@ -89,7 +91,7 @@ then
 fi
 if [ "$deny_gms_unit_list" = "" ]
 then
-   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_default.lst
+   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_prod.lst
 fi
 
 if [ -z "$overwrite" ]
@@ -203,11 +205,11 @@ date -u
 
 ## GET NON ZERO EXIT CODES ##
 # Needed in case aggregation fails, we will need the logs
-echo "Start of non zero exit codes check"
+echo -e $startDiv"Start of non zero exit codes check"$stopDiv
 find $outputRunDataDir/logs/ -name "*_unit.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/unit_errors/non_zero_exit_codes.log" &
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
-echo "Start branch aggregation"
+echo -e $startDiv"Start branch aggregation"$stopDiv
 python3 $srcDir/gms/aggregate_branch_lists.py -d $outputRunDataDir -f "gms_inputs.csv" -l $hucList
 
 echo "=========================================================================="
diff --git a/src/gms/outputs_cleanup.py b/src/gms/outputs_cleanup.py
index 617a915dd..48d7ba8ea 100755
--- a/src/gms/outputs_cleanup.py
+++ b/src/gms/outputs_cleanup.py
@@ -1,44 +1,110 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import os
 
-def remove_deny_list_files(directory,deny_list,branches=False,verbose=False):
-
-    deny_list_file_object = open(deny_list)
+from pathlib import Path
+from utils.shared_functions import FIM_Helpers as fh
 
-    file_removed_count = 0
+def remove_deny_list_files(src_dir, deny_list, branch_id, verbose=False):
 
-    for line in deny_list_file_object:
-        line = line.partition('#')[0]
-        filename = line.rstrip()
+    '''
+    Overview
+    ----------
+    Delete a set of files in a given directory (and/or subdirectories) based
+    on values in the deny list.
+        
+    Notes:
+        - Strange, but.. if you want to skip deleting file, have the value for
+          the 'deny_list' param to be the value of "none" (this is a by product
+          of using bash as part of our system)
+        - In the deny list, any line starting with a # will be skipped. Any line
+          value which contains the value of {}, will be replaced with submitted
+          branch id. If the line does not have a {} in it, it will be searched
+          and removed for an exact file match.
+        - Technically, we don't validate that the branch id is a number, and will
+          work with any value.
 
-        if len(filename) == 0:
-            continue
+    Parameters
+    ----------
+    
+    - src_dir : str
+        Folder path where the files are to be deleted (recursive).
+        Will error if does not exist.
+            
+    - deny_list : str
+        If not the value of "none" (any case), the file must exist and it contains
+        the list of files to be deleted. Will error if does not exist.
+            
+    - branch_id : str
+        Needs to have a value and will be subsituted into any {} value. 
+        Will error if does not exist.
         
-        filename = os.path.join(directory,filename)
+    Returns
+    ----------
+        None
+    '''
+    
+    # Yes.. this is a little strange.
+    # if the user submitts the deny list name of "none" (any case)
+    # we skip this
+    if (deny_list.upper() == 'NONE'): 
+        print("file clean via the deny list skipped")
+        return
+
+    if (not os.path.isdir(src_dir)):
+        raise ValueError(f"Sorry, the directory {src_dir} does not exist")
+
+    if (branch_id.strip() == ""):
+        raise ValueError(f"Sorry, branch id value must exist")
 
-        # for use with branches. append branch id
-        if branches:
-            branch_id = directory.split('/')[-1]
-            filename = filename.format(branch_id)
+    # Note: some of the deny_file_names might be a comment line
+    # this will validate file exists
+    deny_file_names = fh.load_list_file(deny_list.strip())
+
+    fh.vprint(f"source folder is {src_dir}", verbose)
+    fh.vprint(f"deny_list is {deny_list}", verbose)
+
+    file_removed_count = 0
+
+    for deny_file_name in deny_file_names:
+    
+        # Only keep lines that do no start with a #
+        # aka.. we are only deleting files that do not start a line with #
+        deny_file_name = deny_file_name.strip()
+        if (deny_file_name.startswith("#")): 
+            continue 
         
-        if os.path.exists(filename):
-            os.remove(filename)
+        # the file name may / may not have a {} in it . If it does
+        # has a {} it in, we will replace it with the branch ID.
+        # if the file name does not have a {} in it, that file
+        # will be deleted.
+        # We will search all directories recursively.
+        deny_file_name = deny_file_name.replace("{}", branch_id)
+        
+        found_files = Path(src_dir).rglob(f"{deny_file_name}")
+        
+        for found_file in found_files:
+            fh.vprint(f"found file: {found_file}", verbose, False)
+
+            if os.path.exists(found_file):
+                os.remove(found_file)
+            
             file_removed_count +=1
 
-    if verbose:
-        print(f'Removed {file_removed_count} files')
+    fh.vprint(f"Removed {file_removed_count} files", verbose, True)
 
 
 if __name__ == '__main__':
 
     # parse arguments
     parser = argparse.ArgumentParser(description='Clean up outputs given file with line delimineted files')
-    parser.add_argument('-d','--directory', help='Directory to find files', required=True)
-    parser.add_argument('-l','--deny-list', help='Path to deny list file. Must be line delimited', required=True)
-    parser.add_argument('-b','--branches', help='For use with branch files', required=False,default=False, action='store_true')
-    parser.add_argument('-v','--verbose', help='Verbose', required=False,default=False, action='store_true')
+    parser.add_argument('-d','--src_dir', help='Directory to find files', required=True)
+    parser.add_argument('-l','--deny_list', help='Path to deny list file. Must be line delimited',
+                        required=True)
+    parser.add_argument('-b','--branch_id', help='Branch id value', required=True)
+    parser.add_argument('-v','--verbose', help='Verbose', required=False, 
+                        default=False, action='store_true')
 
     # extract to dictionary
     args = vars(parser.parse_args())
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index 8b8c26562..8c0d185c3 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -113,7 +113,7 @@ if [ -f $deny_gms_branches_list ]; then
     echo -e $startDiv"Remove files $hucNumber $current_branch_id"$stopDiv
     date -u
     Tstart
-    $srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_gms_branches_list -v -b
+    $srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_gms_branches_list -b $current_branch_id
     Tcount
 fi
 
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index fa25e9cc3..883a8980d 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -234,7 +234,7 @@ fi
 echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -v -b
+$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -b 0
 Tcount
 
 ## REMOVE FILES FROM DENY LIST ##
@@ -242,6 +242,6 @@ if [ -f $deny_gms_unit_list ]; then
     echo -e $startDiv"Remove files $hucNumber"$stopDiv
     date -u
     Tstart
-    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_gms_unit_list -v
+    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_gms_unit_list -b $hucNumber
     Tcount
 fi
diff --git a/unit_tests/README.md b/unit_tests/README.md
index 666600d1c..4f7a1955a 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -37,9 +37,7 @@ Notice a modified branch "deny_gms_branch_unittests.lst"  (special for unittests
 
 Here are the params and args you need if you need to re-run unit and branch
 
-gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o -s
-	
-gms_run_branch.sh -n fim_unit_test_data_do_not_remove -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_branch_unittests.lst -o -s
+gms_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -c /foss_fim/config/params_template.env -j 1 -bd /foss_fim/config/deny_gms_branch_unittests.lst -ud None -o -s
 
 **NOTICE: the deny file used for gms_run_branch... its a special one for unittests `deny_gms_branch_unittests.lst`.
 
diff --git a/unit_tests/__template_unittests.py b/unit_tests/__template_unittests.py
index 4f1e465b1..678605bcc 100644
--- a/unit_tests/__template_unittests.py
+++ b/unit_tests/__template_unittests.py
@@ -78,7 +78,6 @@ def test_<method name you are about to test>_success(self):
     #    Notes about what the test is and the expected results (or expected exception if applicable)
     #    '''
     #   #global params_file
-    #   helpers.print_unit_test_function_header()
         
     #   params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
 
diff --git a/unit_tests/filter_catchments_and_add_attributes_unittests.py b/unit_tests/filter_catchments_and_add_attributes_unittests.py
index c7e403081..2243f7edc 100644
--- a/unit_tests/filter_catchments_and_add_attributes_unittests.py
+++ b/unit_tests/filter_catchments_and_add_attributes_unittests.py
@@ -21,8 +21,8 @@
 
 # *************
 # Important: For this to work, when you run gms_run_branch.sh, you have to 
-# use deny_gms_branches_default.lst or one that does not exist. Key files need to exist
-# for this unit test to work.
+# use deny_gms_branches_dev.lst or the word "none" for the deny list arguments
+# (unit and branch deny list parameters). Key files need to exist for this unit test to work.
 class test_filter_catchments_and_add_attributes(unittest.TestCase):
 
     '''
diff --git a/unit_tests/gms/outputs_cleanup_params.json b/unit_tests/gms/outputs_cleanup_params.json
new file mode 100644
index 000000000..963e676fe
--- /dev/null
+++ b/unit_tests/gms/outputs_cleanup_params.json
@@ -0,0 +1,25 @@
+{
+	"valid_specific_branch_data":
+	{
+		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
+		"deny_list": "/foss_fim/config/deny_gms_branches_prod.lst",
+		"branch_id": "3246000009",
+		"verbose": true
+	},
+
+	"valid_directory_data":
+	{
+		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
+		"deny_list": "/foss_fim/config/deny_gms_branches_prod.lst",
+		"branch_id": "0",
+		"verbose": true
+	},
+
+	"skip_clean":
+	{
+		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
+		"deny_list": "None",
+		"branch_id": "",
+		"verbose": true
+	}	
+}
\ No newline at end of file
diff --git a/unit_tests/gms/outputs_cleanup_unittests.py b/unit_tests/gms/outputs_cleanup_unittests.py
new file mode 100644
index 000000000..006c95ab0
--- /dev/null
+++ b/unit_tests/gms/outputs_cleanup_unittests.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/') 
+import gms.outputs_cleanup as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_outputs_cleanup(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_remove_deny_list_files_specific_branch_success(self):
+
+        '''
+        This validates removal of files for a directory already pointing to a 
+        specific branch in a HUC
+        '''
+
+        params = self.params["valid_specific_branch_data"].copy()
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+                                   
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_files_huc_level_success(self):
+
+        '''
+        This validates removal of files for all files and subdirectory files.
+        Normally used for covering all hucs and their branch zeros but 
+        can be anything
+        '''
+
+        params = self.params["valid_directory_data"].copy()
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+                                   
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_skip_cleaning_success(self):
+
+        '''
+        This validates removal of files for all files and subdirectory files.
+        Normally used for covering all hucs and their branch zeros but 
+        can be anything
+        '''
+
+        params = self.params["skip_clean"].copy()
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+                                   
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_files_invalid_src_directory(self):
+
+        '''
+        Double check the src directory exists
+        '''
+
+        # NOTE: As we are expecting an exception, this MUST have a try catch.
+        #   for "success" tests (as above functions), they CAN NOT have a try/catch
+        
+        try:
+
+            params = self.params["valid_specific_branch_data"].copy()    
+            params["src_dir"] = "/data/does_no_exist"
+            
+            src.remove_deny_list_files(src_dir = params["src_dir"],
+                                    deny_list = params["deny_list"],
+                                    branch_id = params["branch_id"],
+                                    verbose = params["verbose"])
+            
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+            
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_files_invalid_deny_list_does_not_exist(self):
+
+        '''
+        Double check the deny list exists
+        '''
+
+        # NOTE: As we are expecting an exception, this MUST have a try catch.
+        #   for "success" tests (as above functions), they CAN NOT have a try/catch
+        try:
+
+            params = self.params["valid_specific_branch_data"].copy()    
+            params["deny_list"] = "invalid_file_name.txt"
+            
+            src.remove_deny_list_files(src_dir = params["src_dir"],
+                                    deny_list = params["deny_list"],
+                                    branch_id = params["branch_id"],
+                                    verbose = params["verbose"])
+            
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+            
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+   
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    

From ebd25b5bb1dc9adb6919cad5374eccc1a28d55f0 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Tue, 4 Oct 2022 12:03:46 -0500
Subject: [PATCH 164/359] v4.0.9.5 HOTFIX: fixes for calib system, landSea deny
 list and params (#696)

---
 config/deny_gms_unit_prod.lst |  2 +-
 config/params_template.env    |  2 +-
 docs/CHANGELOG.md             | 21 +++++++++++++++++++++
 gms_run_branch.sh             | 21 ++++++++++++++++-----
 src/src_adjust_spatial_obs.py | 16 ++++++++++++++--
 src/src_adjust_usgs_rating.py | 29 +++++++++++++++++++----------
 6 files changed, 72 insertions(+), 19 deletions(-)

diff --git a/config/deny_gms_unit_prod.lst b/config/deny_gms_unit_prod.lst
index 419187fbb..6c5b5d9a1 100644
--- a/config/deny_gms_unit_prod.lst
+++ b/config/deny_gms_unit_prod.lst
@@ -3,7 +3,7 @@
 NHDPlusBurnLineEvent_subset.gpkg
 #branch_id.lst
 #branch_polygons.gpkg
-LandSea_subset.gpkg
+#LandSea_subset.gpkg
 nhd_headwater_points_subset.gpkg
 #nld_subset_levees.gpkg
 #nwm_catchments_proj_subset.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index 9e14210d6..49db90523 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -57,7 +57,7 @@ export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; F
 export nwm_recur_file="data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
 
 #### apply SRC adjustments using observed FIM/flow point database ####
-export src_adjust_spatial="False" # Toggle to run src adjustment routine (True=on; False=off)
+export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
 export fim_obs_pnt_data="data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 #### path to env file with sensitive paths for accessing postgres database ####
 export CALB_DB_KEYS_FILE="/data/config/calb_db_keys.env"
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 016591a31..3193ec260 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,27 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.9.5 - 2022-10-3 - [PR #696](https://github.com/NOAA-OWP/inundation-mapping/pull/696)
+
+- Fixed deny_gms_unit_prod.lst to comment LandSea_subset.gpkg, so it does not get removed. It is needed for processing in some branches
+- Change default for params_template.env -> src_adjust_spatial="False", back to default of "True"
+- Fixed an infinite loop when src_adjust_usgs_rating.py was unable to talk to the calib db.
+- Fixed src_adjsust_usgs_rating.py for when the usgs_elev_table.csv may not exist.
+
+### Changes
+
+- `gms_run_branch.sh`:  removed some "time" command in favour of using fim commands from bash_functions.sh which give better time and output messages.
+
+- `config`
+    - `deny_gms_unit_prod.lst`: Commented out LandSea_subset.gpkg as some HUCs need that file in place.
+    - `params_template.env`: Changed default src_adjust_spatial back to True
+
+- `src`
+    - `src_adjust_spatial_obs.py`:  Added code to a while loop (line 298) so it is not an indefinite loop that never stops running. It will now attempts to contact the calibration db after 6 attempts. Small adjustments to output and logging were also made and validation that a connection to the calib db was actually successful.
+    - `src_adjust_usgs_rating.py`: Discovered that a usgs_elev_df might not exist (particularly when processing was being done for hucs that have no usgs guage data). If the usgs_elev_df does not exist, it no longer errors out.
+
+<br/><br/>
+
 ## v4.0.9.4 - 2022-09-30 - [PR #691](https://github.com/NOAA-OWP/inundation-mapping/pull/691)
 
 Cleanup Branch Zero output at the end of a processing run. Without this fix, some very large files were being left on the file system. Adjustments and cleanup changed the full BED output run from appx 2 TB output to appx 1 TB output.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 018c934a3..c268b1448 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -163,6 +163,10 @@ elif [ $overwrite -eq 1 ]; then
     mkdir -p $outputRunDataDir/branch_errors
 fi
 
+## Track total time of the overall run
+T_total_start
+Tstart
+
 ## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
 if [ "$src_adjust_spatial" = "True" ]; then
     if [ ! -f $CALB_DB_KEYS_FILE ]; then
@@ -178,13 +182,14 @@ if [ "$src_adjust_spatial" = "True" ]; then
     fi
 fi
 
+Tcount
+Tstart
+date -u
+
 ## RUN GMS BY BRANCH ##
 echo "=========================================================================="
 echo "Start of branch processing"
 echo "Started: `date -u`" 
-
-## Track total time of the overall run
-T_total_start
 Tstart
 
 if [ "$jobLimit" -eq 1 ]; then
@@ -206,15 +211,21 @@ python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
 if [ "$src_adjust_usgs" = "True" ]; then
+    Tstart
     echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"$stopDiv
     # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
-    time python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+    Tcount
+    date -u
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
 if [ "$src_adjust_spatial" = "True" ]; then
+    Tstart
     echo -e $startDiv"Performing SRC adjustments using benchmark point database"$stopDiv
-    time python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
+    python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
+    Tcount
+    date -u
 fi
 
 # -------------------
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index 9407f3e1d..2b0faebfe 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -200,15 +200,25 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
     Outputs
     - procs_list:           passes multiprocessing list of input args for process_points function input
     '''
+    
+    log_file.write('Connecting to database via host\n')    
     conn = connect() # Connect to the PostgreSQL db once
+    
+    if (conn is None):
+        msg = "unable to connect to calibration db\n"
+        print(msg)
+        log_file.write(msg)
+        return
+    
     log_file.write('Connected to database via host\n')
     print("Finding all fim_output hucs that contain calibration points...")
     fim_out_huc_list  = [ item for item in os.listdir(fim_directory) if os.path.isdir(os.path.join(fim_directory, item)) ]
+
     fim_out_huc_list.remove('logs')
     ## Record run time and close log file
     run_time_start = dt.datetime.now()
     log_file.write('Finding all hucs that contain calibration points...' + '\n')
-    huc_list_db = find_hucs_with_points(conn,fim_out_huc_list)
+    huc_list_db = find_hucs_with_points(conn, fim_out_huc_list)
     run_time_end = dt.datetime.now()
     task_run_time = run_time_end - run_time_start
     log_file.write('HUC SEARCH TASK RUN TIME: ' + str(task_run_time) + '\n')
@@ -284,7 +294,8 @@ def connect():
     print('Connecting to the PostgreSQL database...')
     conn = None
     not_connected = True
-    while not_connected:
+    fail_ctr = 0
+    while not_connected and fail_ctr < 6:
         try:
 
             # connect to the PostgreSQL server
@@ -312,6 +323,7 @@ def connect():
             print("Connected to database\n\n")
         except (Exception, psycopg2.DatabaseError) as error:
             print("Waiting for database to come online")
+            fail_ctr += 1
             time.sleep(5)
 
     return conn
diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py
index d0df1f5e3..a9a086a48 100644
--- a/src/src_adjust_usgs_rating.py
+++ b/src/src_adjust_usgs_rating.py
@@ -201,11 +201,6 @@ def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,j
     #usgs_elev_file = os.path.join(branch_dir,'usgs_elev_table.csv')
     #usgs_elev_df = pd.read_csv(usgs_elev_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int})
     csv_name = 'usgs_elev_table.csv'
-    usgs_elev_df = concat_huc_csv(run_dir,csv_name)
-    if usgs_elev_df is None:
-        print('WARNING: usgs_elev_df not created - check that usgs_elev_table.csv files exist in fim_dir!')
-    elif usgs_elev_df.empty:
-        print('WARNING: usgs_elev_df is empty - check that usgs_elev_table.csv files exist in fim_dir!')
     
     available_cores = multiprocessing.cpu_count()
     if job_number > available_cores:
@@ -218,18 +213,32 @@ def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,j
     if not os.path.isdir(log_dir):
         os.makedirs(log_dir)
 
-    print('This may take a few minutes...')
-    usgs_df = create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir)
-
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
     # Create log file for processing records
     log_file = open(os.path.join(log_dir,'log_usgs_rc_src_adjust.log'),"w")
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
+   
+    usgs_elev_df = concat_huc_csv(run_dir,csv_name)
+
+    if usgs_elev_df is None:
+        warn_err = 'WARNING: usgs_elev_df not created - check that usgs_elev_table.csv files exist in fim_dir!'
+        print(warn_err)
+        log_file.write(warn_err)
+        
+    elif usgs_elev_df.empty:
+        warn_err = 'WARNING: usgs_elev_df is empty - check that usgs_elev_table.csv files exist in fim_dir!'
+        print(warn_err)
+        log_file.write(warn_err)
+
+    else:
+        print('This may take a few minutes...')
+        log_file.write("starting create usgs rating db")
+        usgs_df = create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir)
 
-    ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
-    branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file)
+        ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
+        branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file)
 
     ## Record run time and close log file
     log_file.write('#########################################################\n\n')

From 8b0e7d6b11c33435b4e7703e86dbcd9319cb0ae7 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 17 Oct 2022 13:13:51 -0500
Subject: [PATCH 165/359] v4.0.9.6 Inundate-nation bug fixes and some minor
 upgrades (#711)

---
 docs/CHANGELOG.md        |  16 +++
 tools/inundate_nation.py | 290 ++++++++++++++++++++++++---------------
 2 files changed, 196 insertions(+), 110 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3193ec260..8474a3fd1 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.9.6 - 2022-10-17 - [PR #711](https://github.com/NOAA-OWP/inundation-mapping/pull/711)
+
+Bug fix and formatting upgrades. It was also upgraded to allow for misc other inundation data such as high water data.
+
+### Changes
+
+- `tools`
+    - `inundate_nation.py`:  As stated above.
+
+### Testing
+
+- it was run in a production model against fim 4.0.9.2 at 100 yr and 2 yr as well as a new High Water dataset.
+
+<br/><br/>
+
+
 ## v4.0.9.5 - 2022-10-3 - [PR #696](https://github.com/NOAA-OWP/inundation-mapping/pull/696)
 
 - Fixed deny_gms_unit_prod.lst to comment LandSea_subset.gpkg, so it does not get removed. It is needed for processing in some branches
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index d5b79af7e..693017e5b 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -1,26 +1,130 @@
 import argparse
+import logging
+import multiprocessing
 import os
 import rasterio
+import re
+import shutil
+import sys
+
+from multiprocessing import Pool
 from rasterio.merge import merge
 from osgeo import gdal, ogr
 
-from inundation import inundate
-import multiprocessing
-from multiprocessing import Pool
+sys.path.append('/foss_fim/src')
+from datetime import datetime
 from gms_tools.mosaic_inundation import Mosaic_inundation
 from gms_tools.inundate_gms import Inundate_gms
+from inundation import inundate
 from tools_shared_variables import elev_raster_ndv
-import re
+from utils.shared_functions import FIM_Helpers as fh
 
 
-INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
-INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
-INPUTS_DIR = r'/data/inputs'
-OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/'
-DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
+#INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
+#INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
+#INPUTS_DIR = r'/data/inputs'
+#OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/'
+#DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
 PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'
 
 
+def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, inc_mosaic, job_number):
+
+    
+    assert os.path.isdir(fim_run_dir), f'ERROR: could not find the input fim_dir location: {fim_run_dir}'
+    
+    assert os.path.exists(flow_file), f'ERROR: could not find the flow file: {flow_file}'
+
+    if job_number > available_cores:
+        job_number = available_cores - 1
+        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+
+    fim_version = os.path.basename(os.path.normpath(fim_run_dir))
+    logging.info(f'Using fim version: {fim_version}')
+    output_base_file_name = magnitude_key + "_" + fim_version
+    #print(output_base_file_name)
+    
+    __setup_logger(output_dir, output_base_file_name )
+
+    start_dt = datetime.now()
+
+    logging.info(f'Input FIM Directory: {fim_run_dir}')
+    logging.info(f'output_dir: {output_dir}')
+    logging.info(f'magnitude_key: {magnitude_key}')
+    logging.info(f'flow_file: {flow_file}')
+    logging.info(f'inc_mosaic: {str(inc_mosaic)}')
+
+    print("Preparing to generate inundation outputs for magnitude: " + magnitude_key)
+    print("Input flow file: " + flow_file)
+    
+    magnitude_output_dir = os.path.join(output_dir, output_base_file_name)
+
+    if not os.path.exists(magnitude_output_dir):
+        print('Creating new output directory for raw mosaic files: ' + magnitude_output_dir)
+        os.mkdir(magnitude_output_dir)
+    else:
+        # we need to empty it. we will kill it and remake it (using rmtree to force it)
+        shutil.rmtree(magnitude_output_dir, ignore_errors=True)
+        os.mkdir(magnitude_output_dir)
+        
+    huc_list = []
+    for huc in os.listdir(fim_run_dir):
+        
+        #if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
+        if re.match('\d{8}', huc):    
+            huc_list.append(huc)
+
+    print('Inundation raw mosaic outputs here: ' + magnitude_output_dir)
+    
+    config = "GMS"
+    
+    run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, config, flow_file, job_number])
+                
+    # Perform mosaic operation
+    if inc_mosaic:
+        fh.print_current_date_time()
+        logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))
+        print("Performing bool mosaic process...")
+        logging.info("Performing bool mosaic process...")
+        
+        output_bool_dir = os.path.join(output_dir, "bool_temp")
+        if not os.path.exists(output_bool_dir):
+            os.mkdir(output_bool_dir)
+        else:
+            # we need to empty it. we will kill it and remake it (using rmtree to force it)
+            shutil.rmtree(output_bool_dir, ignore_errors=True)
+            os.mkdir(output_bool_dir)
+
+        procs_list = []
+        for rasfile in os.listdir(magnitude_output_dir):
+            if rasfile.endswith('.tif') and "extent" in rasfile:
+                #p = magnitude_output_dir + rasfile
+                procs_list.append([magnitude_output_dir, rasfile, output_bool_dir])
+
+        # Multiprocess --> create boolean inundation rasters for all hucs
+        if len(procs_list) > 0:
+            with Pool(processes=job_number) as pool:
+                pool.map(create_bool_rasters, procs_list)
+        else:
+            msg = f'Did not find any valid FIM extent rasters: {magnitude_output_dir}'
+            print(msg)
+            logging.info(msg)
+
+        # Perform VRT creation and final mosaic using boolean rasters
+        vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name)
+        
+        # now cleanup the raw mosiac directories
+        shutil.rmtree(output_bool_dir, ignore_errors=True)
+
+    # now cleanup the raw mosiac directories
+    shutil.rmtree(magnitude_output_dir, ignore_errors=True)
+        
+    fh.print_current_date_time()
+    logging.info(logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S")))
+    end_time = datetime.now()
+    logging.info(fh.print_date_time_duration(start_dt, end_time))
+
+
 def run_inundation(args):
     """
     This script is a wrapper for the inundate function and is designed for multiprocessing.
@@ -37,10 +141,8 @@ def run_inundation(args):
     config = args[4]
     forecast = args[5]
     job_number = args[6]
-
-    
+   
     # Define file paths for use in inundate().
- 
 
     inundation_raster = os.path.join(magnitude_output_dir, magnitude + '_' + config + '_inund_extent.tif')
     
@@ -99,120 +201,88 @@ def create_bool_rasters(args):
         dst.write(array.astype(rasterio.int8))
 
 
-def vrt_raster_mosaic(output_bool_dir, output_mos_dir, fim_version_tag):
+def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag):
     
-    #raster_to_mosaic = ['data/temp/ryan/inundate_nation/25_0_ms/25_0_ms_inund_extent_12090301.tif','data/temp/ryan/inundate_nation/25_0_ms/25_0_ms_inund_extent_12090302.tif']
-    raster_to_mosaic = []
+  
+    # NOTE: Oct 2022.. we no longer need the VRT, only the large mosaic'd raster. 
+    # this code is about to be deprecated, so we will leave it as is.
+    
+    rasters_to_mosaic = []
     for rasfile in os.listdir(output_bool_dir):
         if rasfile.endswith('.tif') and "extent" in rasfile:
             p = output_bool_dir + os.sep + rasfile
             print("Processing: " + p)
-            raster_to_mosaic.append(p)
-
-    print("Creating virtual raster...")
-    vrt = gdal.BuildVRT(output_mos_dir + "merged.vrt", raster_to_mosaic)
+            rasters_to_mosaic.append(p)
 
-    print("Building raster mosaic: " + str(output_mos_dir + fim_version_tag + "_mosaic.tif"))
-    gdal.Translate(output_mos_dir + fim_version_tag + "_mosaic.tif", vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2'])
+    logging.info(fh.print_current_date_time())
+    output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt")
+    print("Creating virtual raster: " + output_mosiac_vrt)
+    logging.info("Creating virtual raster: " + output_mosiac_vrt)
+    vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic)
+    
+    output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif")    
+    print("Building raster mosaic: " + output_mosiac_raster)
+    logging.info("Building raster mosaic: " + output_mosiac_raster)
+    print("This can take a number of hours, watch 'docker stats' cpu value to ensure the process"\
+        "to ensure the process is still working")
+    gdal.Translate(output_mosiac_raster, vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2'])
     vrt = None        
 
 
+def __setup_logger(output_folder_path, log_file_name_key):
+
+    start_time = datetime.now()
+    file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
+    log_file_name = f"{log_file_name_key}-{file_dt_string}.log"
+
+    log_file_path = os.path.join(output_folder_path, log_file_name)
+    
+    logging.basicConfig(filename=log_file_path, 
+                        level=logging.DEBUG,
+                        format='%(message)s')
 
+    # yes.. this can do console logs as well, but it can be a bit unstable and ugly
+    
+    logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
+    logging.info("----------------")
 
 
 if __name__ == '__main__':
+
+    # Sample usage: python3 /foss_fim/tools/inundate_nation.py -r /outputs/fim_4_0_9_2 -m 100_0 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_100_0_cms.csv -s -j 10
+    # outputs become /data/inundation_review/inundate_nation/100_0_fim_4_0_9_2_mosiac.tif (.log, etc)
+
+    # Sample usage: python3 /foss_fim/tools/inundate_nation.py -r /outputs/fim_4_0_9_2 -m hw -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv -s -j 10
+    # outputs become /data/inundation_review/inundate_nation/hw_fim_4_0_9_2_mosiac.tif (.log, etc)
+
+    # if run on UCS2, you can map docker as -v /dev_fim_share... etc/:/data -v /local....outputs:/outputs -v foss_fim as normal.
+    
     available_cores = multiprocessing.cpu_count()
 
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow recurrence interflow data. Inundation outputs are stored in the /inundation_review/inundation_nwm_recurr/ directory.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)',required=True)
-    parser.add_argument('-o', '--output-dir',help='Optional: The path to a directory to write the outputs. If not used, the inundation_nation directory is used by default -> type=str',required=False, default="")
-    parser.add_argument('-m', '--magnitude-list', help = 'List of NWM recurr flow intervals to process (Default: 100_0) (Other options: 2_0 5_0 10_0 25_0 50_0 100_0)', nargs = '+', default = ['100_0'], required = False)
-    parser.add_argument('-d', '--depth',help='not functional. Optional flag to produce inundation depth rasters (extent raster created by default)', action='store_true')
-    parser.add_argument('-s', '--mosaic',help='Optional flag to produce mosaic of FIM extent rasters', action='store_true')
-    parser.add_argument('-j', '--job-number',help='The number of jobs',required=False,default=1)
+    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow '\
+        'recurrence interflow data. Inundation outputs are stored in the '\
+        '/inundation_review/inundation_nwm_recurr/ directory.')
         
-    args = vars(parser.parse_args())
-
-    fim_run_dir = args['fim_run_dir']
-    output_dir = args['output_dir']
-    depth_option = args['depth']
-    magnitude_list = args['magnitude_list']
-    mosaic_option = args['mosaic']
-    job_number = int(args['job_number'])
-
-    assert os.path.isdir(fim_run_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_run_dir)
-    print("Input FIM Directory: " + str(fim_run_dir))
-    fim_version = os.path.basename(os.path.normpath(fim_run_dir))
-    print("Using fim version: " + str(fim_version))
-    for magnitude in magnitude_list:
-        print("Preparing to generate inundation outputs for magnitude: " + str(magnitude))
-        nwm_recurr_file = os.path.join(INUN_REVIEW_DIR, 'nwm_recurr_flow_data', 'nwm21_17C_recurr_' + magnitude + '_cms.csv')
-        assert os.path.isfile(nwm_recurr_file), 'ERROR: could not find the input NWM recurr flow file: ' + str(nwm_recurr_file)
-        print("Input flow file: " + str(nwm_recurr_file))
+    parser.add_argument('-r', '--fim-run-dir', help='Name of directory containing outputs '\
+        'of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)', required=True)
+        
+    parser.add_argument('-o', '--output-dir', help='Optional: The path to a directory to write the '\
+        'outputs. If not used, the inundation_nation directory is used by default '\
+        'ie) /data/inundation_review/inundate_nation/',
+        default='/data/inundation_review/inundate_nation/', required=False)
+       
+    parser.add_argument('-m', '--magnitude_key', help = 'used in output folders names and temp files, '\
+        'added to output_file_name_key ie 100_0, 2_0, hw, etc)', required = True)
         
-        config = 'gms'
-
-        if output_dir == "":
-            output_dir = INUN_OUTPUT_DIR
-        if not os.path.exists(output_dir):
-            print('Creating new output directory: ' + str(output_dir))
-            os.mkdir(output_dir)
-            
-        huc_list = []
-        for huc in os.listdir(fim_run_dir):
-            
-            #if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
-            if re.match('\d{8}', huc):    
-                huc_list.append(huc)
-            
-        magnitude_output_dir = os.path.join(output_dir, magnitude + '_' + config  + '_' + fim_version)
-        if not os.path.exists(magnitude_output_dir):
-            os.mkdir(magnitude_output_dir)
-
-        print('Inundation outputs here: ' + str(magnitude_output_dir))
-        run_inundation([fim_run_dir, huc_list, magnitude, magnitude_output_dir, config, nwm_recurr_file, job_number])
-                   
-
-
-        # Perform mosaic operation
-        if mosaic_option:
-            print("Performing mosaic process...")
-            fim_version_tag = os.path.basename(os.path.normpath(magnitude_output_dir))
-            print("fim_version: " + fim_version_tag)
-            output_bool_dir = os.path.join(OUTPUT_BOOL_PARENT_DIR, fim_version_tag)
-            if not os.path.exists(output_bool_dir):
-                print('Creating new output directory for boolean temporary outputs: ' + str(output_bool_dir))
-                os.mkdir(output_bool_dir)
-
-
-            output_mos_dir = output_dir
-            if output_dir == "":
-                output_mos_dir = DEFAULT_OUTPUT_DIR
-            
-            
-            if not os.path.exists(output_mos_dir):
-                print('Creating new output directory: ' + str(output_mos_dir))
-                os.mkdir(output_mos_dir)
-
-            if job_number > available_cores:
-                job_number = available_cores - 1
-                print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
-
-            procs_list = []
-            for rasfile in os.listdir(magnitude_output_dir):
-                if rasfile.endswith('.tif') and "extent" in rasfile:
-                    #p = magnitude_output_dir + rasfile
-                    procs_list.append([magnitude_output_dir,rasfile,output_bool_dir])
-
-            # Multiprocess --> create boolean inundation rasters for all hucs
-            if len(procs_list) > 0:
-                with Pool(processes=job_number) as pool:
-                    pool.map(create_bool_rasters, procs_list)
-            else:
-                print('Did not find any valid FIM extent rasters: ' + magnitude_output_dir)
-
-            # Perform VRT creation and final mosaic using boolean rasters
-            
-            vrt_raster_mosaic(output_bool_dir,output_mos_dir,fim_version_tag)
+    parser.add_argument('-f', '--flow_file', help = 'the path and flow file to be used. '\
+        'ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv', required = True)
+        
+    parser.add_argument('-s', '--inc_mosaic',help='Optional flag to produce mosaic of FIM extent rasters',
+                        action='store_true')
+                        
+    parser.add_argument('-j', '--job-number', help='The number of jobs', required=False, default=1, type=int)
+        
+    args = vars(parser.parse_args())
 
+    inundate_nation(**args)
\ No newline at end of file

From 1fe93fe3150853d83771c47fe1b6fc1879ac832d Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 17 Oct 2022 13:18:35 -0500
Subject: [PATCH 166/359] v4.0.9.7 Add fim3 s3 whitelist and rename the fim s3
 whitelist (#703)

---
 config/aws_s3_put_fim3_whitelist.lst          |  4 ++++
 ...list.lst => aws_s3_put_fim4_whitelist.lst} |  0
 docs/CHANGELOG.md                             | 19 ++++++++++++++++++-
 3 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 config/aws_s3_put_fim3_whitelist.lst
 rename config/{aws_s3_put_whitelist.lst => aws_s3_put_fim4_whitelist.lst} (100%)

diff --git a/config/aws_s3_put_fim3_whitelist.lst b/config/aws_s3_put_fim3_whitelist.lst
new file mode 100644
index 000000000..b81404832
--- /dev/null
+++ b/config/aws_s3_put_fim3_whitelist.lst
@@ -0,0 +1,4 @@
+hydroTable{}.csv
+gw_catchments_reaches_filtered_addedAttributes{}.tif
+rem_zeroed_masked{}.tif
+usgs_elev_table.csv
\ No newline at end of file
diff --git a/config/aws_s3_put_whitelist.lst b/config/aws_s3_put_fim4_whitelist.lst
similarity index 100%
rename from config/aws_s3_put_whitelist.lst
rename to config/aws_s3_put_fim4_whitelist.lst
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8474a3fd1..b0bd105b3 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.9.7 - 2022-10-7 - [PR #703](https://github.com/NOAA-OWP/inundation-mapping/pull/703)
+
+During a recent release of a FIM 3 version, it was discovered that FIM3 has slightly different AWS S3 upload requirements. A new s3 whitelist file has been created for FIM3 and the other s3 file was renamed to include the phrase "fim4" in it.
+
+This is being added to source control as it might be used again and we don't want to loose it.
+
+### Additions
+
+- `config`
+   - `aws_s3_put_fim3_whitelist.lst`
+   
+### Renamed
+
+- `config`
+   - `aws_s3_put_fim4_whitelist.lst`: renamed from aws_s3_put_whitelist.lst
+
+<br/><br/>
+
 ## v4.0.9.6 - 2022-10-17 - [PR #711](https://github.com/NOAA-OWP/inundation-mapping/pull/711)
 
 Bug fix and formatting upgrades. It was also upgraded to allow for misc other inundation data such as high water data.
@@ -16,7 +34,6 @@ Bug fix and formatting upgrades. It was also upgraded to allow for misc other in
 
 <br/><br/>
 
-
 ## v4.0.9.5 - 2022-10-3 - [PR #696](https://github.com/NOAA-OWP/inundation-mapping/pull/696)
 
 - Fixed deny_gms_unit_prod.lst to comment LandSea_subset.gpkg, so it does not get removed. It is needed for processing in some branches

From 00bf50080bc7eaa84f546d70aec69bdf0a7f0c9e Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 17 Oct 2022 13:26:32 -0500
Subject: [PATCH 167/359] v4.0.9.8 Added calibration database engine to FIM4
 (#701)

---
 README.md                                     |  4 +-
 config/calb_db_keys_template.env              | 12 +++
 docs/CHANGELOG.md                             | 29 +++++++
 gms_run_branch.sh                             | 22 ++++-
 src/src_adjust_spatial_obs.py                 | 84 ++++++++++++-------
 tools/calibration-db/README.md                | 81 ++++++++++++++++++
 tools/calibration-db/docker-compose.yml       | 18 ++++
 .../docker-entrypoint-initdb.d/init-db.sh     |  6 ++
 8 files changed, 223 insertions(+), 33 deletions(-)
 create mode 100755 config/calb_db_keys_template.env
 create mode 100755 tools/calibration-db/README.md
 create mode 100755 tools/calibration-db/docker-compose.yml
 create mode 100755 tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh

diff --git a/README.md b/README.md
index a23e65155..7f9033268 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 # FIM Version 4 
 
 ## Accessing Data through ESIP S3 Bucket
-The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.
+The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.  You will need permission from ESIP to access this data. Please contact Carson Pruitt (carson.pruitt@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) for assistance.
 
 AWS Region: `US East (N. Virginia) us-east-1`
 
@@ -63,6 +63,8 @@ Make sure to set the config folder group to 'fim' recursively using the chown co
 - `mannings_default.json`
     - must change filepath in `params_template.env` in `manning_n` variable name
 
+This system has an optional tool called the `calibration database tool`. In order to use this system, you will need to install the calibration database service or disable it in the `params_template.env` file. See [calibration tool README](https://github.com/NOAA-OWP/inundation-mapping/blob/dev/tools/calibration-db/README.md) for more details.
+
 ### Produce HAND Hydrofabric
 ```
 gms_pipeline.sh -u <huc8> -n <name_your_run>
diff --git a/config/calb_db_keys_template.env b/config/calb_db_keys_template.env
new file mode 100755
index 000000000..a6287bb5b
--- /dev/null
+++ b/config/calb_db_keys_template.env
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# SENSITIVE INFO!!! NOT FOR PUBLIC DISSEMENATION
+
+# Notes:
+#  - The DB_Host name is the name of the server running the calibration db.
+#  - More than one computer can call across the network to one calibraion db server.
+#  - You can change the name of the database if you like, but not recommended.
+CALIBRATION_DB_HOST=
+CALIBRATION_DB_NAME=calibration
+CALIBRATION_DB_USER_NAME=
+CALIBRATION_DB_PASS=
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b0bd105b3..12822df1f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,35 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.9.8 - 2022-10-06 - [PR #701](https://github.com/NOAA-OWP/inundation-mapping/pull/701)
+
+Moved the calibration tool from dev-fim3 branch into "dev" (fim4) branch. Git history not available.
+
+Also updated making it easier to deploy, along with better information for external contributors.
+
+Changed the system so the calibration database name is configurable. This allows test databases to be setup in the same postgres db / server system. You can have more than one calb_db_keys.env running in different computers (or even more than one on one server) pointing to the same actual postgres server and service. ie) multiple dev machine can call a single production server which hosts the database.
+
+For more details see /tools/calibration-db/README.md
+
+### Changes
+
+- `tools`
+    - `calibration-db`
+        - `docker-compose.yml`: changed to allow for configurable database name. (allows for more then one database in a postgres database system (one for prod, another for test if needed))
+
+### Additions
+
+- `config`
+    - `calb_db_keys_template.env`: a new template verison of the required config values.
+
+### Removals
+
+- `tools`
+    - `calibration-db`
+        - `start_db.sh`: Removed as the command should be run on demand and not specifically scripted because of its configurable location of the env file.
+
+<br/><br/>
+
 ## v4.0.9.7 - 2022-10-7 - [PR #703](https://github.com/NOAA-OWP/inundation-mapping/pull/703)
 
 During a recent release of a FIM 3 version, it was discovered that FIM3 has slightly different AWS S3 upload requirements. A new s3 whitelist file has been created for FIM3 and the other s3 file was renamed to include the phrase "fim4" in it.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index c268b1448..3e8cff749 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -167,18 +167,32 @@ fi
 T_total_start
 Tstart
 
+: '
+This makes the local variables from the calb_db_keys files
+into global variables that can be used in other files, including python.
+
+Why not just leave the word export in front of each of the keys in the
+calb_db_keys.env? Becuase that file is used against docker-compose
+when we start up that part of the sytem and it does not like the word
+export.
+'
+export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
+export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
+export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
+export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
+
 ## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
 if [ "$src_adjust_spatial" = "True" ]; then
     if [ ! -f $CALB_DB_KEYS_FILE ]; then
-        echo "ERROR! - src_adjust_spatial parameter is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
         exit 1
     else
         source $CALB_DB_KEYS_FILE
-        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes"
+        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
         echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
         echo "Loading Point Data"
-        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=calibration user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
     fi
 fi
 
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index 2b0faebfe..237e7365a 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -1,28 +1,32 @@
+#!/usr/bin/env python3
+
 import argparse
+import datetime as dt
 import geopandas as gpd
-from geopandas.tools import sjoin
+#import json
+import multiprocessing
+import numpy as np
 import os
-import rasterio
 import pandas as pd
-import numpy as np
+import psycopg2 # python package for connecting to postgres
+import rasterio
 import sys
-import json
-import datetime as dt
+import time
+
 from collections import deque
-import multiprocessing
+from dotenv import load_dotenv
+from geopandas.tools import sjoin
 from multiprocessing import Pool
 from src_roughness_optimization import update_rating_curve
-import psycopg2 # python package for connecting to postgres
-from dotenv import load_dotenv
-import time
+from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH
 
 #import variables from .env file
 load_dotenv()
 CALIBRATION_DB_HOST = os.getenv("CALIBRATION_DB_HOST")
+CALIBRATION_DB_NAME = os.getenv("CALIBRATION_DB_NAME")
 CALIBRATION_DB_USER_NAME = os.getenv("CALIBRATION_DB_USER_NAME")
 CALIBRATION_DB_PASS = os.getenv("CALIBRATION_DB_PASS")
 
-from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH
 '''
 The script imports a PostgreSQL database containing observed FIM extent points and associated flow data. This script attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow.
 
@@ -90,11 +94,13 @@ def process_points(args):
 
     ## Check that there are valid obs in the water_edge_df (not empty)
     if water_edge_df.empty:
-        log_text = 'NOTE --> skipping HUC: ' + str(huc) + '  Branch: ' + str(branch_id) + ': no valid observation points found within the branch catchments'
+        log_text = 'NOTE --> skipping HUC: ' + str(huc) + '  Branch: ' + str(branch_id)\
+            + ': no valid observation points found within the branch catchments'
     else:
         ## Intermediate output for debugging
         if optional_outputs:
-            branch_debug_pts_out_gpkg = os.path.join(branch_dir, 'export_water_edge_df_' + branch_id + '.gpkg')
+            branch_debug_pts_out_gpkg = os.path.join(branch_dir, 'export_water_edge_df_'
+                                                     + branch_id + '.gpkg')
             water_edge_df.to_file(branch_debug_pts_out_gpkg, driver='GPKG', index=False)
             
         #print('Processing points for HUC: ' + str(huc) + '  Branch: ' + str(branch_id))
@@ -114,7 +120,9 @@ def process_points(args):
         merge_prev_adj = True # merge in previous SRC adjustment calculations
 
         ## Call update_rating_curve() to perform the rating curve calibration.
-        log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc, branch_id, catchments_poly_path, optional_outputs, source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
+        log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc, 
+                                       branch_id, catchments_poly_path, optional_outputs, 
+                                       source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
         ## Still testing: use code below to print out any exceptions.
         '''
         try:
@@ -150,12 +158,14 @@ def find_points_in_huc(huc_id, conn):
     
     # Need to hard code the CRS to use EPSG:5070 instead of the default ESRI:102039 (gdal pyproj throws an error with crs 102039)
     # Appears that EPSG:5070 is functionally equivalent to ESRI:102039: https://gis.stackexchange.com/questions/329123/crs-interpretation-in-qgis
-    water_edge_df = gpd.GeoDataFrame.from_postgis(huc_pt_query, con=conn, params=[huc_id], crs="EPSG:5070", parse_dates=['coll_time'])
+    water_edge_df = gpd.GeoDataFrame.from_postgis(huc_pt_query, con=conn, 
+                                                  params=[huc_id], crs="EPSG:5070",
+                                                  parse_dates=['coll_time'])
     water_edge_df = water_edge_df.drop(columns=['st_x','st_y'])
     
     return water_edge_df
 
-def find_hucs_with_points(conn,fim_out_huc_list):
+def find_hucs_with_points(conn, fim_out_huc_list):
     '''
     The function queries the PostgreSQL database and returns a list of all the HUCs that contain calb point data.
 
@@ -267,7 +277,8 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
             hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
             catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
             htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
-            catchments_poly_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
+            catchments_poly_path = os.path.join(branch_dir, 
+                'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
 
             # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
             if not os.path.exists(hand_path):
@@ -285,6 +296,7 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
     with Pool(processes=job_number) as pool:
                 log_output = pool.map(process_points, procs_list)
                 log_file.writelines(["%s\n" % item  for item in log_output])
+                
     log_file.write('#########################################################\n')
     disconnect(conn) # move this to happen at the end of the huc looping
 
@@ -301,7 +313,7 @@ def connect():
             # connect to the PostgreSQL server
             conn = psycopg2.connect(
                 host=CALIBRATION_DB_HOST,
-                database="calibration",
+                database=CALIBRATION_DB_NAME,
                 user=CALIBRATION_DB_USER_NAME,
                 password=CALIBRATION_DB_PASS)
 
@@ -321,6 +333,7 @@ def connect():
             cur.close()
             not_connected = False
             print("Connected to database\n\n")
+            
         except (Exception, psycopg2.DatabaseError) as error:
             print("Waiting for database to come online")
             fail_ctr += 1
@@ -335,20 +348,25 @@ def disconnect(conn):
         conn.close()
         print('Database connection closed.')
 
-def run_prep(fim_directory,debug_outputs_option,ds_thresh_override,DOWNSTREAM_THRESHOLD,job_number):
-    assert os.path.isdir(fim_directory), 'ERROR: could not find the input fim_dir location: ' + str(fim_directory)
+def run_prep(fim_directory, debug_outputs_option, ds_thresh_override, DOWNSTREAM_THRESHOLD, job_number):
+    
+    assert os.path.isdir(fim_directory), 'ERROR: could not find the input fim_dir location: '\
+        + str(fim_directory)
 
     available_cores = multiprocessing.cpu_count()
     if job_number > available_cores:
         job_number = available_cores - 1
-        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+        print("Provided job number exceeds the number of available cores. " \
+            + str(job_number) + " max jobs will be used instead.")
 
     if ds_thresh_override != DOWNSTREAM_THRESHOLD:
-        print('ALERT!! - Using a downstream distance threshold value (' + str(float(ds_thresh_override)) + 'km) different than the default (' + str(DOWNSTREAM_THRESHOLD) + 'km) - interpret results accordingly')
+        print('ALERT!! - Using a downstream distance threshold value ('
+            + str(float(ds_thresh_override)) + 'km) different than the default ('\
+            + str(DOWNSTREAM_THRESHOLD) + 'km) - interpret results accordingly')
         DOWNSTREAM_THRESHOLD = float(ds_thresh_override)
 
     ## Create output dir for log file
-    output_dir = os.path.join(fim_directory,"logs","src_optimization")
+    output_dir = os.path.join(fim_directory, "logs", "src_optimization")
     if not os.path.isdir(output_dir):
         os.makedirs(output_dir)
         
@@ -360,7 +378,9 @@ def run_prep(fim_directory,debug_outputs_option,ds_thresh_override,DOWNSTREAM_TH
     sys.__stdout__ = sys.stdout
     log_file = open(os.path.join(output_dir,'log_spatial_src_adjust.log'),"w")
     log_file.write('#########################################################\n')
-    log_file.write('Parameter Values:\n' + 'DOWNSTREAM_THRESHOLD= ' + str(DOWNSTREAM_THRESHOLD) + '\n' + 'ROUGHNESS_MIN_THRESH= ' + str( ROUGHNESS_MIN_THRESH) + '\n' + 'ROUGHNESS_MAX_THRESH=' + str(ROUGHNESS_MAX_THRESH) + '\n')
+    log_file.write('Parameter Values:\n' + 'DOWNSTREAM_THRESHOLD = ' + str(DOWNSTREAM_THRESHOLD)\
+        + '\n' + 'ROUGHNESS_MIN_THRESH = ' + str( ROUGHNESS_MIN_THRESH) + '\n'\
+        + 'ROUGHNESS_MAX_THRESH = ' + str(ROUGHNESS_MAX_THRESH) + '\n')
     log_file.write('#########################################################\n\n')
     log_file.write('START TIME: ' + str(begin_time) + '\n')
 
@@ -378,10 +398,17 @@ def run_prep(fim_directory,debug_outputs_option,ds_thresh_override,DOWNSTREAM_TH
     ## Parse arguments.
     parser = argparse.ArgumentParser(description='Adjusts rating curve given a shapefile containing points of known water boundary.')
     #parser.add_argument('-db','--points-layer',help='Path to points layer containing known water boundary locations',required=True)
-    parser.add_argument('-fim_dir','--fim-directory',help='Parent directory of FIM-required datasets.',required=True)
-    parser.add_argument('-debug','--extra-outputs',help='OPTIONAL flag: Use this to keep intermediate output files for debugging/testing',default=False,required=False, action='store_true')
-    parser.add_argument('-dthresh','--downstream-thresh',help='OPTIONAL Override: distance in km to propogate modified roughness values downstream', default=DOWNSTREAM_THRESHOLD,required=False)
-    parser.add_argument('-j','--job-number',help='OPTIONAL: Number of jobs to use',required=False,default=2)
+    parser.add_argument('-fim_dir','--fim-directory',
+                        help='Parent directory of FIM-required datasets.', required=True)
+    parser.add_argument('-debug','--extra-outputs',
+                        help='OPTIONAL flag: Use this to keep intermediate output files for debugging/testing',
+                        default=False, required=False, action='store_true')
+    parser.add_argument('-dthresh','--downstream-thresh',
+                        help='OPTIONAL Override: distance in km to propogate modified roughness values downstream', 
+                        default=DOWNSTREAM_THRESHOLD, required=False)
+    parser.add_argument('-j','--job-number',
+                        help='OPTIONAL: Number of jobs to use', required=False,
+                        default=2)
 
     ## Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -391,4 +418,5 @@ def run_prep(fim_directory,debug_outputs_option,ds_thresh_override,DOWNSTREAM_TH
     ds_thresh_override = args['downstream_thresh']
     job_number = int(args['job_number'])
 
-    run_prep(fim_directory,debug_outputs_option,ds_thresh_override,DOWNSTREAM_THRESHOLD,job_number)
\ No newline at end of file
+    run_prep(fim_directory, debug_outputs_option, ds_thresh_override, 
+             DOWNSTREAM_THRESHOLD, job_number)
\ No newline at end of file
diff --git a/tools/calibration-db/README.md b/tools/calibration-db/README.md
new file mode 100755
index 000000000..ef85eeb21
--- /dev/null
+++ b/tools/calibration-db/README.md
@@ -0,0 +1,81 @@
+## The calibration database service
+
+The **calibration-db tool** is an optional use tool. When started, it creates a service, via a docker container, that creates a postgres database. The calibration-db tool will make minor updates to hydrotables.csv's, but only to applicable HUCs. The applicable HUCs generally have AHPS library data or good USGS guage data.
+
+-------------------------------
+To setup the system, it needs to be done in two parts.
+
+**A)** Update the `/config/params_template.env` file and set `src_adjust_spatial` to "True".  If you are not using the tool, set the flag to "False".
+
+**B)** Setup the calibration database service. See steps below.
+
+-------------------------------
+## Creating the calibration database service
+
+When you start the service, it will start up a new docker container named "fim_calibration_db", if enabled in the params_template.env.
+
+To run commands, use a bash terminal window.
+
+Steps:
+1. Copy `/config/calb_db_keys_template.env` and rename it to a name of your choice (ie: `calb_db_keys.env` or whatever). We recommend saving the file outside the fim source code folder for security purposes. 
+
+2. Update the values in the new .env file. 
+	Note: The `CALIBRATION_DB_HOST` is the computer/server where the docker container is physically running. This may be your server name, but can also be a different server if you like. One calibration database service can be shared by multiple other servers, each with their own FIM source code folders (a "client" if you will). Each "client" server can have it's own calb_db_keys.env file and/or can share the .env with other servers.  ie) dev-1 and prod-1 can share a .env file if they have a shared drive.
+	
+3. Start the service from the /tools/calibration-db directory of the server that will run the calibration database service. 
+   Using a bash terminal window, run
+	  `docker-compose --env-file {path to .env file}/{your key file name} up --build -d` 
+	  ie) docker-compose --env-file /my_server/config/calb_db_keys.env up --build -d
+
+	   - If you get an error of permissions denied, you might need to upgrade the permissions to your calibration-db folder. From your /tools directory, run `chmod -R 777 calibration-db` (yes.. 777).  You may need to add the word "sudo" at the front of the command, depending on your system configuration.
+	   - If you get an error saying `command not found: docker compose`, you may need to install it via `sudo apt install docker-compose` or check your system configuration.
+
+4. You should be able to see a container named `fim_calibration_db` via a bash terminal command of `docker stats --no-stream`.
+
+-------------------------------
+## Destroying the database (from the /tools/calibration-db directory)
+
+Using a bash terminal window, run
+    `docker-compose down`
+    `rm -rf pgdata`
+
+-------------------------------
+## Debugging Postgres DB
+
+Use the following command to connect to the postgres DB (using values from your .env file):
+
+`docker run -it --rm postgis/postgis psql -h CALIBRATION_DB_HOST -U CALIBRATION_DB_USER_NAME` (and follow prompt to enter password (CALIBRATION_DB_PASS))
+ie) `docker run -it --rm postgis/postgis psql -h my_server_name -U fim_postgres` (and follow prompt to enter password (CALIBRATION_DB_PASS))
+
+### View names of databases (you can have more than one database in a postgres database server. ie) calibation and calibration-test.
+
+`\l`
+
+    #### Note: FIM will create a database as it is being used. If you have not yet processed any HUCs, there will not be a database yet.
+
+### Connect to calibration DB  
+
+`\c calibration`  (or the name of the database you gave in the calib_db_keys.env (CALIBRATION_DB_NAME))
+
+### View tables in DB (now connected)
+
+`\dt`
+
+### View columns in tables
+
+`\d+ hucs`
+`\d+ points`
+
+### View number of rows in a table
+
+'select count(*) from table_name;`  (you need the ";" on the end)
+
+Postgres uses psql. See [https://www.postgresql.org/docs/current/sql-commands.html]9https://www.postgresql.org/docs/current/sql-commands.html) for details on commands.
+
+
+## System Notes
+
+The `docker-entrypoint-initdb.d` scripts only run once, when the DB is initially created. To rerun these scripts, you will need to destroy the database.
+The `docker-entrypoint-initdb.d` scripts do not create the tables for the database, but rather simply enables the PostGIS extension.
+If you are trying to connect to a database that is on the same server that you are working on, you have to use the full servername and not `localhost`.
+
diff --git a/tools/calibration-db/docker-compose.yml b/tools/calibration-db/docker-compose.yml
new file mode 100755
index 000000000..36c962140
--- /dev/null
+++ b/tools/calibration-db/docker-compose.yml
@@ -0,0 +1,18 @@
+version: '3.5'
+services:
+  calibration-db:
+    image: postgis/postgis
+    container_name: fim_calibration_db
+    restart: always
+    environment:
+      - POSTGRES_USER=${CALIBRATION_DB_USER_NAME}
+      - POSTGRES_PASSWORD=${CALIBRATION_DB_PASS}
+      - POSTGRES_DB=${CALIBRATION_DB_NAME}
+      - PGDATA=/var/lib/postgresql/data/fim/pgdata
+    ports:
+      - "5432:5432"
+    volumes:
+      - calibration_db_data:/var/lib/postgresql/data/fim
+      - ./docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+volumes:
+    calibration_db_data:
diff --git a/tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh b/tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh
new file mode 100755
index 000000000..c60ee4d18
--- /dev/null
+++ b/tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+  CREATE EXTENSION postgis;
+EOSQL

From 290573da8047cffb7a6989f2b738b7725e523d32 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 17 Oct 2022 13:44:51 -0500
Subject: [PATCH 168/359] v4.0.10.0 Implementing USGS 3DEP DEMs into the FIM 4
 workflow (#697)

---
 config/params_template.env                    |   2 +-
 data/usgs/acquire_and_preprocess_3dep_dems.py |  17 +-
 docs/CHANGELOG.md                             |  73 +++++++-
 fim_run.sh                                    |   8 +
 lib/aggregate_fim_outputs.py                  | 174 ------------------
 lib/fr_to_ms_raster_mask.py                   |  83 ---------
 src/gms/run_by_branch.sh                      |   6 -
 src/gms/run_by_unit.sh                        |   6 +-
 src/run_by_unit.sh                            |   9 +
 src/utils/shared_variables.py                 |  28 ++-
 10 files changed, 116 insertions(+), 290 deletions(-)
 delete mode 100644 lib/aggregate_fim_outputs.py
 delete mode 100755 lib/fr_to_ms_raster_mask.py

diff --git a/config/params_template.env b/config/params_template.env
index 49db90523..4a8a5daaf 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -52,7 +52,7 @@ export vrough_suffix="_vmann" # text to append to output log and src_full_crossw
 export vmann_input_file="data/inputs/rating_curve/variable_roughness/mannings_global_06_011.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
 export bankfull_attribute="chann_volume_ratio" # src_full_crosswalked_bankfull.csv attribute (column id) containing the channel vs overbank ratio values (generated in the identify_src_bankfull.py)
 
-#### apply SRC adjustments using USGS rating curve database ####
+#### apply SRC adjustments using USGS rating curve database (calibration db) ####
 export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; False=off)
 export nwm_recur_file="data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
 
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index 6ad83f8e1..a74d5accc 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -78,7 +78,7 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
                           ' not set to a valid path')
     
     if (target_output_folder_path is None) or (target_output_folder_path == ""):
-        target_output_folder_path = sv.INPUT_DEMS_3DEP_10M_DIR
+        target_output_folder_path = os.environ['usgs_3dep_dems_10m']
     
     if (not os.path.exists(target_output_folder_path)):
         raise ValueError(f"Output folder path {target_output_folder_path} does not exist" )
@@ -88,7 +88,7 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
     start_time = datetime.now()
     fh.print_start_header('Loading 3dep dems', start_time)
    
-    print(f"Downloading to {target_output_folder_path}")
+    #print(f"Downloading to {target_output_folder_path}")
     __setup_logger(target_output_folder_path)
     logging.info(f"Downloading to {target_output_folder_path}")
     
@@ -122,6 +122,11 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     ----------
         - fl (object of fim_logger (must have been created))
         - remaining params are defined in acquire_and_preprocess_3dep_dems
+        
+    Notes
+    ----------
+        - pixel size set to 10 x 10 (m)
+        - block size (256) (sometimes we use 512)
     '''
 
     print(f"==========================================================")
@@ -130,7 +135,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     base_cmd =  'gdalwarp {0} {1}'
     base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
     base_cmd += ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
-    base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES"'
+    base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
     base_cmd += ' -t_srs EPSG:5070'
    
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
@@ -199,7 +204,7 @@ def download_usgs_dem_file(extent_file,
                 base_cmd =  'gdalwarp {0} {1}'
                 base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
                 base_cmd +=  ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
-                base_cmd +=  ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES"'
+                base_cmd +=  ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
                 base_cmd +=  ' -t_srs EPSG:5070'
         - retry (bool)
              If True, and the file exists (and is over 0k), downloading will be skipped.
@@ -279,7 +284,6 @@ def __setup_logger(output_folder_path):
 
     logger = logging.getLogger()
     logger.addHandler(file_handler)
-    # logger.addHandler(console_handler)
     logger.setLevel(logging.DEBUG)    
     
     logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
@@ -309,7 +313,8 @@ def __setup_logger(output_folder_path):
         
         
     # IMPORTANT: 
-    # (Sept 2022): we do not process HUC2 of 19 (alaska) or 22 (misc US pacific islands).
+    # (Sept 2022): we do not process HUC2 of 22 (misc US pacific islands).
+    # We left in HUC2 of 19 (alaska) as we hope to get there in the semi near future
     # They need to be removed from the input src clip directory in the first place.
     # They can not be reliably removed in code.
        
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 12822df1f..3621dca71 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,32 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.10.0 - 2022-10-4 - [PR #697](https://github.com/NOAA-OWP/inundation-mapping/pull/697)
+
+Change FIM to load DEM's from the new USGS 3Dep files instead of the original NHD Rasters.
+
+### Changes
+
+- `config`
+    - `params_template.env`: Change default of the calib db back to true:  src_adjust_spatial back to "True". Plus a few text updates.
+- `src`
+    - `gms`
+        - `run_by_unit.sh`: Change input_DEM value to the new vrt `$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt` to load the new 3Dep DEM's. Note: The 3Dep DEM's are projected as CRS 5070, but for now, our code is using ESRI:102039. Later all code and input will be changed to CRS:5070. We now are defining the FIM desired projection (102039), so we need to reproject on the fly from 5070 to 102039 during the gdalwarp cut.
+        - `run_by_branch.sh`: Removed unused lines.
+    - `utils`
+        - `shared_variables.py`: Changes to use the new 3Dep DEM rasters instead of the NHD rasters. Moved some values (grouped some variables). Added some new variables for 3Dep. Note: At this time, some of these new enviro variables for 3Dep are not used but are expected to be used shortly.
+- `data`
+    - `usgs`
+        - `acquire_and_preprocess_3dep_dems.py`: Minor updates for adjustments of environmental variables. Adjustments to ensure the cell sizes are fully defined as 10 x 10 as source has a different resolution. The data we downloaded to the new `inputs/3dep_dems/10m_5070` was loaded as 10x10, CRS:5070 rasters.
+
+### Removals
+
+- `lib`
+    - `aggregate_fim_outputs.py` : obsolete. Had been deprecated for a while and replaced by other files.
+    - `fr_to_mr_raster_mask.py` : obsolete. Had been deprecated for a while and replaced by other files.
+
+<br/><br/>
+
 ## v4.0.9.8 - 2022-10-06 - [PR #701](https://github.com/NOAA-OWP/inundation-mapping/pull/701)
 
 Moved the calibration tool from dev-fim3 branch into "dev" (fim4) branch. Git history not available.
@@ -122,7 +148,7 @@ Cleanup Branch Zero output at the end of a processing run. Without this fix, som
     - `deny_gms_branch_zero.lst`: Added some new files to be deleted.
     - `deny_gms_branches_dev.lst`:  Renamed from `deny_gms_branches_default.lst` and some new files to be deleted, updated others. Now used primarily for development and testing use.
     - `deny_gms_branches_prod.lst`:  Renamed from `deny_gms_branches_min` and some new files to be deleted, updated others. Now used primarily for when releasing a version to production.
-    -  `deny_gms_unit_prod.lst`: Renamed from `deny_gms_unit_default.lst`, yes... there currently is no "dev" version.  Added some new files to be deleted.
+    - `deny_gms_unit_prod.lst`: Renamed from `deny_gms_unit_default.lst`, yes... there currently is no "dev" version.  Added some new files to be deleted.
 
 <br/><br/>
 
@@ -170,6 +196,51 @@ Other changes:
 
 <br/><br/>
 
+## v4.0.(pending) - 2022-09-13 - [PR #681](https://github.com/NOAA-OWP/inundation-mapping/pull/681)
+
+Created a new tool to downloaded USGS 3Dep DEM's via their S3 bucket.
+
+Other changes:
+ - Some code file re-organization in favour of the new `data` folder which is designed for getting, setting, and processing data from external sources such as AWS, WBD, NHD, NWM, etc.
+ - Added tmux as a new tool embedded inside the docker images.
+
+## Additions
+
+- `data`
+   - `usgs`
+      - `acquire_and_preprocess_3dep_dems.py`:  The new tool as described above. For now it is hardcoded to a set path for USGS AWS S3 vrt file but may change later for it to become parameter driven.
+ - `create_vrt_file.py`: This is also a new tool that can take a directory of geotiff files and create a gdal virtual file, .vrt extention, also called a `virtual raster`. Instead of clipping against HUC4, 6, 8's raster files, and run risks of boundary issues, vrt's actual like all of the tif's are one giant mosaiced raster and can be clipped as one.
+
+## Removals
+
+- 'Dockerfile.prod`:  No longer being used (never was used)
+
+## Changes
+
+- `Dockerfile`:  Added apt install for tmux. This tool will now be available in docker images and assists developers.
+
+- `data`
+   - `acquire_and_preprocess_inputs.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+   - `nws`
+      - `preprocess_ahps_nws.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+      - `preprocess_rasters.py`: moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+    - `usgs`
+         - `preprocess_ahps_usgs.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+         - `preprocess_download_usgs_grids.py`: moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
+
+ - `src`
+     - `utils`
+         - `shared_functions.py`:  changes made were
+              - Cleanup the "imports" section of the file (including a change to how the utils.shared_variables file is loaded.
+              - Added `progress_bar_handler` function which can be re-used by other code files.
+              - Added `get_file_names` which can create a list of files from a given directory matching a given extension. 
+              - Modified `print_current_date_time` and `print_date_time_duration` and  methods to return the date time strings. These helper methods exist to help with standardization of logging and output console messages.
+              - Added `print_start_header` and `print_end_header` to help with standardization of console and logging output messages.
+          - `shared_variables.py`: Additions in support of near future functionality of having fim load DEM's from USGS 3DEP instead of NHD rasters.
+
+<br/><br/>
+
+
 ## v4.0.9.2 - 2022-09-12 - [PR #678](https://github.com/NOAA-OWP/inundation-mapping/pull/678)
 
 This fixes several bugs related to branch definition and trimming due to waterbodies.
diff --git a/fim_run.sh b/fim_run.sh
index 0b0b54cf1..676fc0bda 100755
--- a/fim_run.sh
+++ b/fim_run.sh
@@ -25,6 +25,14 @@ usage ()
     exit
 }
 
+##################################
+##
+#### Deprecated:  Oct 1, 2022 ####
+echo "Deprecated: Oct 1, 2022"
+##
+##################################
+
+
 if [ "$#" -lt 7 ]
 then
   usage
diff --git a/lib/aggregate_fim_outputs.py b/lib/aggregate_fim_outputs.py
deleted file mode 100644
index edafd93a3..000000000
--- a/lib/aggregate_fim_outputs.py
+++ /dev/null
@@ -1,174 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import argparse
-import pandas as pd
-import json
-import rasterio
-from rasterio.merge import merge
-import shutil
-import csv
-from utils.shared_variables import PREP_PROJECTION
-
-
-def aggregate_fim_outputs(fim_out_dir):
-
-    print ("aggregating outputs to HUC6 scale")
-
-    drop_folders = ['logs']
-    huc_list = [huc for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
-    huc6_list = [str(huc[0:6]) for huc in os.listdir(fim_out_dir) if huc not in drop_folders]
-    huc6_list = list(set(huc6_list))
-
-    for huc in huc_list:
-
-        os.makedirs(os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6])), exist_ok=True)
-
-        # original file paths
-        hydrotable_filename = os.path.join(fim_out_dir,huc,'hydroTable.csv')
-        src_filename = os.path.join(fim_out_dir,huc,'src.json')
-
-        # aggregate file name paths
-        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),'hydroTable.csv')
-        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc[0:6]),f'rating_curves_{huc[0:6]}.json')
-
-        if len(huc)> 6:
-
-            # open hydrotable
-            hydrotable = pd.read_csv(hydrotable_filename)
-
-            # write/append aggregate hydrotable
-            if os.path.isfile(aggregate_hydrotable):
-                hydrotable.to_csv(aggregate_hydrotable,index=False, mode='a',header=False)
-            else:
-                hydrotable.to_csv(aggregate_hydrotable,index=False)
-
-            del hydrotable
-
-            # open src
-            src = open(src_filename)
-            src = json.load(src)
-
-            # write/append aggregate src
-            if os.path.isfile(aggregate_src):
-
-                with open(aggregate_src, "r+") as file:
-                    data = json.load(file)
-                    data.update(src)
-
-                with open(aggregate_src, 'w') as outfile:
-                    json.dump(data, outfile)
-            else:
-                with open(aggregate_src, 'w') as outfile:
-                        json.dump(src, outfile)
-
-            del src
-
-        else:
-            shutil.copy(hydrotable_filename, aggregate_hydrotable)
-            shutil.copy(src_filename, aggregate_src)
-
-    for huc6 in huc6_list:
-
-        ## add feature_id to aggregate src
-        aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),'hydroTable.csv')
-        aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),f'rating_curves_{huc6}.json')
-
-        # Open aggregate src for writing feature_ids to
-        src_data = {}
-        with open(aggregate_src) as jsonf:
-            src_data = json.load(jsonf)
-
-        with open(aggregate_hydrotable) as csvf:
-            csvReader = csv.DictReader(csvf)
-
-            for row in csvReader:
-                if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
-                    src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
-
-        # Write src_data to JSON file
-        with open(aggregate_src, 'w') as jsonf:
-            json.dump(src_data, jsonf)
-
-        ## aggregate rasters
-        huc6_dir = os.path.join(fim_out_dir,'aggregate_fim_outputs',huc6)
-
-        # aggregate file paths
-        rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}.tif')
-        catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}.tif')
-
-        if huc6 not in huc_list:
-
-            huc6_filter = [path.startswith(huc6) for path in huc_list]
-            subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
-
-            # aggregate and mosaic rem
-            rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
-
-            if len(rem_list) > 1:
-
-                rem_files_to_mosaic = []
-
-                for rem in rem_list:
-
-                    rem_src = rasterio.open(rem)
-                    rem_files_to_mosaic.append(rem_src)
-
-                mosaic, out_trans = merge(rem_files_to_mosaic)
-                out_meta = rem_src.meta.copy()
-                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
-
-                with rasterio.open(rem_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
-                    dest.write(mosaic)
-
-                del rem_files_to_mosaic,rem_src,out_meta,mosaic
-
-            elif len(rem_list)==1:
-
-                shutil.copy(rem_list[0], rem_mosaic)
-
-            # aggregate and mosaic catchments
-            catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
-
-            if len(catchment_list) > 1:
-
-                cat_files_to_mosaic = []
-
-                for cat in catchment_list:
-                    cat_src = rasterio.open(cat)
-                    cat_files_to_mosaic.append(cat_src)
-
-                mosaic, out_trans = merge(cat_files_to_mosaic)
-                out_meta = cat_src.meta.copy()
-
-                out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
-
-                with rasterio.open(catchment_mosaic, "w", **out_meta, tiled=True, blockxsize=256, blockysize=256, BIGTIFF='YES') as dest:
-                    dest.write(mosaic)
-
-                del cat_files_to_mosaic,cat_src,out_meta,mosaic
-
-            elif len(catchment_list)==1:
-
-                shutil.copy(catchment_list[0], catchment_mosaic)
-
-        else:
-            # original file paths
-            rem_filename = os.path.join(fim_out_dir,huc6,'rem_zeroed_masked.tif')
-            catchment_filename = os.path.join(fim_out_dir,huc6,'gw_catchments_reaches_filtered_addedAttributes.tif')
-
-            shutil.copy(rem_filename, rem_mosaic)
-            shutil.copy(catchment_filename, catchment_mosaic)
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Aggregate layers buy HUC6')
-    parser.add_argument('-d','--fim-outputs-directory', help='FIM outputs directory', required=True)
-
-
-    args = vars(parser.parse_args())
-
-    fim_outputs_directory = args['fim_outputs_directory']
-
-    aggregate_fim_outputs(fim_outputs_directory)
diff --git a/lib/fr_to_ms_raster_mask.py b/lib/fr_to_ms_raster_mask.py
deleted file mode 100755
index 567b5cf19..000000000
--- a/lib/fr_to_ms_raster_mask.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-
-'''
- Description: Mask raster layers using 'mainstems' stream buffer
-'''
-
-import sys
-import os
-import geopandas as gpd
-import rasterio.mask
-
-split_flows_fileName    = sys.argv[1]
-fdr_fr                  = sys.argv[2]
-dem_fr                  = sys.argv[3]
-slope_fr                = sys.argv[4]
-fdr_ms_filename         = sys.argv[5]
-dem_ms_filename         = sys.argv[6]
-slope_ms_filename       = sys.argv[7]
-str_pixel_fr            = sys.argv[8]
-str_pixel_ms_filename   = sys.argv[9]
-ms_buffer_dist          = int(os.environ['ms_buffer_dist'])
-
-# create output layer names
-split_flows = gpd.read_file(split_flows_fileName)
-
-# Limit the rasters to the buffer distance around the draft streams.
-print ("Limiting rasters to buffer area ({} meters) around model streams".format(str(ms_buffer_dist)))
-
-split_flows_ms_buffer = split_flows.unary_union.buffer(ms_buffer_dist)
-
-print('Writing raster outputs ...')
-
-# Mask nhddem
-with rasterio.open(dem_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(dem_fr), dem_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
-
-# Mask fdr
-with rasterio.open(fdr_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(fdr_fr), fdr_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
-
-# Mask slope
-with rasterio.open(slope_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(slope_fr), slope_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
-
-# Mask stream pixels
-with rasterio.open(str_pixel_fr) as src:
-    out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-    out_meta = src.meta
-
-out_meta.update({"driver": "GTiff",
-     "height": out_image.shape[1],
-     "width": out_image.shape[2],
-     "transform": out_transform})
-
-with rasterio.open(os.path.join(os.path.dirname(str_pixel_fr), str_pixel_ms_filename), "w", **out_meta) as dest:
-    dest.write(out_image)
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index 8c0d185c3..76e133098 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -22,12 +22,6 @@ outputHucDataDir=$outputRunDataDir/$hucNumber
 outputBranchDataDir=$outputHucDataDir/branches
 outputCurrentBranchDataDir=$outputBranchDataDir/$current_branch_id
 
-# set input files
-input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m.tif
-input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
-input_bathy_bankfull=$inputDataDir/$bankfull_input_table
-input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
-
 ## OVERWRITE
 if [ -d "$outputCurrentBranchDataDir" ];then
     if [ $overwrite -eq 1 ]; then
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 883a8980d..f0ef94023 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -43,7 +43,9 @@ hucUnitLength=${#hucNumber}
 huc4Identifier=${hucNumber:0:4}
 huc2Identifier=${hucNumber:0:2}
 input_NHD_WBHD_layer=WBDHU$hucUnitLength
-input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m.tif
+
+default_projection_crs="ESRI:102039"
+input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
 input_bathy_bankfull=$inputDataDir/$bankfull_input_table
 
@@ -132,7 +134,7 @@ echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"$stopD
 date -u
 Tstart
 [ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \
-gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $default_projection_crs $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
 Tcount
 
 ## GET RASTER METADATA
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
index 4bca99028..679a1351b 100755
--- a/src/run_by_unit.sh
+++ b/src/run_by_unit.sh
@@ -3,6 +3,15 @@
 ## INITIALIZE TOTAL TIME TIMER ##
 T_total_start
 
+##################################
+##
+#### Deprecated:  Oct 1, 2022 ####
+echo "Deprecated: Oct 1, 2022"
+##
+##################################
+
+
+
 echo -e $startDiv"Parameter Values"
 echo -e "extent=$extent"
 echo -e "agree_DEM_buffer=$agree_DEM_buffer"
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 3cac204be..d92d10a0b 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -18,19 +18,13 @@
 WBD_HU2_URL_PARENT = r'http://prd-tnm.s3-website-us-west-2.amazonaws.com/?prefix=StagedProducts/Hydrography/WBD/HU2/GDB'
 
 
-# -- Local Data paths -- #
-# For 10m (1/3 arc second)
-INPUT_DEMS_3DEP_10M_DIR = '/data/inputs/usgs/3dep_dems/10m'
-INPUT_DEMS_3DEP_10M_FIM_VRT = '/data/inputs/usgs/3dep_dems/10m/FIM_Seamless_DEM_10m.vrt'
-WBD_HUC4_DIR = '/data/inputs/wbd/HUC4'
-
-
 # -- Prefixes and Suffixes -- #
 NHD_URL_PREFIX = 'NHDPLUS_H_'
 NHD_RASTER_URL_SUFFIX = '_HU4_RASTER.7z'
 NHD_VECTOR_URL_SUFFIX = '_HU4_GDB.zip'
-NHD_RASTER_EXTRACTION_PREFIX = 'HRNHDPlusRasters'
-NHD_RASTER_EXTRACTION_SUFFIX = 'elev_cm.tif'
+
+    #NHD_RASTER_EXTRACTION_PREFIX = 'HRNHDPlusRasters'
+    #NHD_RASTER_EXTRACTION_SUFFIX = 'elev_cm.tif'
 
 NHD_VECTOR_EXTRACTION_PREFIX = 'NHDPLUS_H_'
 NHD_VECTOR_EXTRACTION_SUFFIX = '_HU4_GDB.zip'
@@ -69,22 +63,22 @@
 os.environ['src_dir'] = '/foss_fim/src'
 os.environ['input_dir'] = 'data/inputs'
 
-os.environ['nhdplus_rasters_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_rasters')
+#os.environ['nhdplus_rasters_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_rasters')
 os.environ['nhdplus_vectors_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors')
 os.environ['nwm_dir'] = os.path.join(os.environ.get('input_dir'),'nwm_hydrofabric')
 os.environ['wbd_dir'] = os.path.join(os.environ.get('input_dir'),'wbd')
 os.environ['ahps_dir'] = os.path.join(os.environ.get('input_dir'),'ahp_sites')
-os.environ['nhdplus_aggregate_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors_aggregate')
+os.environ['ahps_filename'] = os.path.join(os.environ.get('ahps_dir'),'nws_lid.gpkg')
+
+os.environ['3dep_dems_10m_5070'] = os.path.join(os.environ.get('input_dir'),'3dep_dems/10m_5070')
+os.environ['3dep_dems_10m_5070_vrt'] = os.path.join(os.environ.get('3dep_dems_10m_5070'),
+                                                    'fim_seamless_3dep_dem_10m_5070.vrt')
 
-# File Paths
-os.environ['wbd_filename'] = os.path.join(os.environ.get('wbd_dir'),'WBD_National.gpkg')
+os.environ['nwm_dir'] = os.path.join(os.environ.get('input_dir'),'nwm_hydrofabric')
 os.environ['nwm_streams_orig_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_flows_original.gpkg')
 os.environ['nwm_streams_all_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_flows.gpkg')
 os.environ['nwm_headwaters_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_headwaters.gpkg')
 os.environ['nwm_huc4_intersections_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_huc4_intersections.gpkg')
-os.environ['nhd_huc8_intersections_filename'] = os.path.join(os.environ.get('nwm_dir'),'nhd_huc8_intersections.gpkg')
-os.environ['ahps_filename'] = os.path.join(os.environ.get('ahps_dir'),'nws_lid.gpkg')
-os.environ['agg_nhd_headwaters_adj_fileName'] = os.path.join(os.environ.get('nhdplus_aggregate_dir'),'agg_nhd_headwaters_adj.gpkg')
-os.environ['agg_nhd_streams_adj_fileName'] = os.path.join(os.environ.get('nhdplus_aggregate_dir'),'agg_nhd_streams_adj.gpkg')
 os.environ['nwm_catchments_orig_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_catchments_original.gpkg')
 os.environ['nwm_catchments_all_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_catchments.gpkg')
+os.environ['nhd_huc8_intersections_filename'] = os.path.join(os.environ.get('nwm_dir'),'nhd_huc8_intersections.gpkg')

From 1658f2b05124dfe0f5f7e8518e1c8a6eed7a55c2 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Mon, 17 Oct 2022 14:09:40 -0500
Subject: [PATCH 169/359] v4.0.10.1 Hotfix for FIM4 sierra test plots with
 branch zero (#695)

---
 docs/CHANGELOG.md                |  14 +
 tools/rating_curve_comparison.py | 468 +++++++++++++++++--------------
 2 files changed, 269 insertions(+), 213 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3621dca71..f7390c91b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.10.1 - 2022-10-5 - [PR #695](https://github.com/NOAA-OWP/inundation-mapping/pull/695)
+
+This hotfix address a bug with how the rating curve comparison (sierra test) handles the branch zero synthetic rating curve in the comparison plots. Address #676 
+
+### Changes
+
+- `tools/rating_curve_comparison.py`
+  - Added logging function to print and write to log file
+  - Added new filters to ignore AHPS only sites (these are sites that we need for CatFIM but do not have a USGS gage or USGS rating curve available for sierra test analysis)
+  - Added functionality to identify branch zero SRCs
+  - Added new plot formatting to distinguish branch zero from other branches
+
+<br/><br/>
+
 ## v4.0.10.0 - 2022-10-4 - [PR #697](https://github.com/NOAA-OWP/inundation-mapping/pull/697)
 
 Change FIM to load DEM's from the new USGS 3Dep files instead of the original NHD Rasters.
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 5e5985040..4cd002ee8 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -12,6 +12,8 @@
 from multiprocessing import Pool
 from os.path import isfile, join
 import shutil
+import traceback
+import logging
 import warnings
 from pathlib import Path
 import time
@@ -77,185 +79,201 @@ def generate_rating_curve_metrics(args):
     huc                         = args[8]
     alt_plot                    = args[9]
 
+    logging.info("Generating rating curve metrics for huc: " + str(huc))
     elev_table = pd.read_csv(elev_table_filename,dtype={'location_id': object, 'feature_id':object,'HydroID':object, 'levpa_id':object})
+    
+    # Filter out null and non-integer location_id entries (the crosswalk steps tries to fill AHPS only sites with the nws_lid)
     elev_table.dropna(subset=['location_id'], inplace=True)
+    elev_table = elev_table[elev_table['location_id'].apply(lambda x: str(x).isdigit())]
+
+    # Read in the USGS gages rating curve database csv
     usgs_gages = pd.read_csv(usgs_gages_filename,dtype={'location_id': object, 'feature_id':object})
 
     # Aggregate FIM4 hydroTables
-    hydrotable = pd.DataFrame()
-    for branch in elev_table.levpa_id.unique():
-        branch_elev_table = elev_table.loc[elev_table.levpa_id == branch].copy()
-        branch_hydrotable = pd.read_csv(join(branches_folder, str(branch), f'hydroTable_{branch}.csv'),dtype={'HydroID':object,'feature_id':object})
-        # Only pull SRC for hydroids that are in this branch
-        branch_hydrotable = branch_hydrotable.loc[branch_hydrotable.HydroID.isin(branch_elev_table.HydroID)]
-        branch_hydrotable.drop(columns=['order_'], inplace=True)
-        # Join SRC with elevation data
-        branch_elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
-        branch_hydrotable = branch_hydrotable.merge(branch_elev_table, on="HydroID")
-        # Append to full rating curve dataframe
-        if hydrotable.empty:
-            hydrotable = branch_hydrotable
-        else:
-            hydrotable = hydrotable.append(branch_hydrotable)
-
-    # Join rating curves with elevation data
-    #elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
-    #hydrotable = hydrotable.merge(elev_table, on="HydroID")
-    relevant_gages = list(hydrotable.location_id.unique())
-    usgs_gages = usgs_gages[usgs_gages['location_id'].isin(relevant_gages)]
-    usgs_gages = usgs_gages.reset_index(drop=True)
-
-    if len(usgs_gages) > 0:
-
-        # Adjust rating curve to elevation
-        hydrotable['elevation_ft'] = (hydrotable.stage + hydrotable.dem_adj_elevation) * 3.28084 # convert from m to ft
-        # hydrotable['raw_elevation_ft'] = (hydrotable.stage + hydrotable.dem_elevation) * 3.28084 # convert from m to ft
-        hydrotable['discharge_cfs'] = hydrotable.discharge_cms * 35.3147
-        usgs_gages = usgs_gages.rename(columns={"flow": "discharge_cfs", "elevation_navd88": "elevation_ft"})
-
-        hydrotable['source'] = "FIM"
-        usgs_gages['source'] = "USGS"
-        limited_hydrotable = hydrotable.filter(items=['location_id','elevation_ft','discharge_cfs','source', 'HydroID', 'levpa_id', 'dem_adj_elevation'])
-        select_usgs_gages = usgs_gages.filter(items=['location_id', 'elevation_ft', 'discharge_cfs','source'])
-        if 'default_discharge_cms' in hydrotable.columns: # check if both "FIM" and "FIM_default" SRCs are available
-            hydrotable['default_discharge_cfs'] = hydrotable.default_discharge_cms * 35.3147
-            limited_hydrotable_default = hydrotable.filter(items=['location_id','elevation_ft', 'default_discharge_cfs'])
-            limited_hydrotable_default['discharge_cfs'] = limited_hydrotable_default.default_discharge_cfs
-            limited_hydrotable_default['source'] = "FIM_default"
-            rating_curves = limited_hydrotable.append(select_usgs_gages)
-            rating_curves = rating_curves.append(limited_hydrotable_default)
-        else:
-            rating_curves = limited_hydrotable.append(select_usgs_gages)
-
-        # Add stream order
-        stream_orders = hydrotable.filter(items=['location_id','order_']).drop_duplicates()
-        rating_curves = rating_curves.merge(stream_orders, on='location_id')
-        rating_curves['order_'] = rating_curves['order_'].astype('int')
-
-
-        # NWM recurr intervals
-        recurr_intervals = ("2","5","10","25","50","100")
-        recurr_dfs = []
-        for interval in recurr_intervals:
-            recurr_file = join(nwm_flow_dir, 'nwm21_17C_recurr_{}_0_cms.csv'.format(interval))
-            df = pd.read_csv(recurr_file, dtype={'feature_id': str})
-            # Update column names
-            df = df.rename(columns={"discharge": interval})
-            recurr_dfs.append(df)
-
-        # Merge NWM recurr intervals into a single layer
-        nwm_recurr_intervals_all = reduce(lambda x,y: pd.merge(x,y, on='feature_id', how='outer'), recurr_dfs)
-        nwm_recurr_intervals_all = pd.melt(nwm_recurr_intervals_all, id_vars=['feature_id'], value_vars=recurr_intervals, var_name='recurr_interval', value_name='discharge_cms')
-
-        # Append catfim data (already set up in format similar to nwm_recurr_intervals_all)
-        cat_fim = pd.read_csv(catfim_flows_filename, dtype={'feature_id':str})
-        nwm_recurr_intervals_all = nwm_recurr_intervals_all.append(cat_fim)
-
-        # Convert discharge to cfs and filter
-        nwm_recurr_intervals_all['discharge_cfs'] = nwm_recurr_intervals_all.discharge_cms * 35.3147
-        nwm_recurr_intervals_all = nwm_recurr_intervals_all.filter(items=['discharge_cfs', 'recurr_interval','feature_id']).drop_duplicates()
-
-        # Identify unique gages
-        usgs_crosswalk = hydrotable.filter(items=['location_id', 'feature_id']).drop_duplicates()
-        usgs_crosswalk.dropna(subset=['location_id'], inplace=True)
-
-        nwm_recurr_data_table = pd.DataFrame()
-        usgs_recurr_data = pd.DataFrame()
-
-        # Interpolate USGS/FIM elevation at each gage
-        for index, gage in usgs_crosswalk.iterrows():
-
-            # Interpolate USGS elevation at NWM recurrence intervals
-            usgs_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="USGS")]
-
-            if len(usgs_rc) <1:
-                print(f"missing USGS rating curve data for usgs station {gage.location_id} in huc {huc}")
-                continue
-
-            str_order = np.unique(usgs_rc.order_).item()
-            feature_id = str(gage.feature_id)
-
-            usgs_pred_elev = get_reccur_intervals(usgs_rc, usgs_crosswalk,nwm_recurr_intervals_all)
-
-            # Handle sites missing data
-            if len(usgs_pred_elev) <1:
-                print(f"missing USGS elevation data for usgs station {gage.location_id} in huc {huc}")
-                continue
-
-            # Clean up data
-            usgs_pred_elev['location_id'] = gage.location_id
-            usgs_pred_elev = usgs_pred_elev.filter(items=['location_id','recurr_interval', 'discharge_cfs','pred_elev'])
-            usgs_pred_elev = usgs_pred_elev.rename(columns={"pred_elev": "USGS"})
-
-            # Interpolate FIM elevation at NWM recurrence intervals
-            fim_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="FIM")]
-
-            if len(fim_rc) <1:
-                print(f"missing FIM rating curve data for usgs station {gage.location_id} in huc {huc}")
-                continue
-
-            fim_pred_elev = get_reccur_intervals(fim_rc, usgs_crosswalk,nwm_recurr_intervals_all)
-
-            # Handle sites missing data
-            if len(fim_pred_elev) <1:
-                print(f"missing FIM elevation data for usgs station {gage.location_id} in huc {huc}")
-                continue
-
-            # Clean up data
-            fim_pred_elev = fim_pred_elev.rename(columns={"pred_elev": "FIM"})
-            fim_pred_elev = fim_pred_elev.filter(items=['recurr_interval', 'discharge_cfs','FIM'])
-            usgs_pred_elev = usgs_pred_elev.merge(fim_pred_elev, on=['recurr_interval','discharge_cfs'])
-
-            # Add attributes
-            usgs_pred_elev['HUC'] = huc
-            usgs_pred_elev['HUC4'] = huc[0:4]
-            usgs_pred_elev['str_order'] = str_order
-            usgs_pred_elev['feature_id'] = feature_id
-
-            # Melt dataframe
-            usgs_pred_elev = pd.melt(usgs_pred_elev, id_vars=['location_id','feature_id','recurr_interval','discharge_cfs','HUC','HUC4','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
-            nwm_recurr_data_table = nwm_recurr_data_table.append(usgs_pred_elev)
-
-            # Interpolate FIM elevation at USGS observations
-            # fim_rc = fim_rc.merge(usgs_crosswalk, on="location_id")
-            # usgs_rc = usgs_rc.rename(columns={"elevation_ft": "USGS"})
-            #
-            # # Sort stage in ascending order
-            # usgs_rc = usgs_rc.sort_values('USGS',ascending=True)
-            #
-            # # Interpolate FIM elevation at USGS observations
-            # usgs_rc['FIM'] = np.interp(usgs_rc.discharge_cfs.values, fim_rc['discharge_cfs'], fim_rc['elevation_ft'], left = np.nan, right = np.nan)
-            # usgs_rc = usgs_rc[usgs_rc['FIM'].notna()]
-            # usgs_rc = usgs_rc.drop(columns=["source"])
-            #
-            # # Melt dataframe
-            # usgs_rc = pd.melt(usgs_rc, id_vars=['location_id','discharge_cfs','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
-            #
-            # if not usgs_rc.empty:
-            #     usgs_recurr_data = usgs_recurr_data.append(usgs_rc)
-
-        # Generate stats for all sites in huc
-        # if not usgs_recurr_data.empty:
-        #     usgs_recurr_stats_table = calculate_rc_stats_elev(usgs_recurr_data)
-        #     usgs_recurr_stats_table.to_csv(usgs_recurr_stats_filename,index=False)
-
-        # # Generate plots (not currently being used)
-        # fim_elev_at_USGS_rc_plot_filename = join(dirname(rc_comparison_plot_filename),'FIM_elevations_at_USGS_rc_' + str(huc) +'.png')
-        # generate_facet_plot(usgs_recurr_data, fim_elev_at_USGS_rc_plot_filename)
-
-        if not nwm_recurr_data_table.empty:
-            nwm_recurr_data_table.discharge_cfs = np.round(nwm_recurr_data_table.discharge_cfs,2)
-            nwm_recurr_data_table.elevation_ft = np.round(nwm_recurr_data_table.elevation_ft,2)
-            nwm_recurr_data_table.to_csv(nwm_recurr_data_filename,index=False)
-
-        # plot rating curves
-        if alt_plot:
-            generate_rc_and_rem_plots(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table, branches_folder)
+    if not elev_table.empty:
+        hydrotable = pd.DataFrame()
+        for branch in elev_table.levpa_id.unique():
+            branch_elev_table = elev_table.loc[elev_table.levpa_id == branch].copy()
+            #branch_elev_table = elev_table.loc[(elev_table.levpa_id == branch) & (elev_table.location_id.notnull())].copy()
+            branch_hydrotable = pd.read_csv(join(branches_folder, str(branch), f'hydroTable_{branch}.csv'),dtype={'HydroID':object,'feature_id':object,'obs_source':object,'last_updated':object,'submitter':object})
+            # Only pull SRC for hydroids that are in this branch
+            branch_hydrotable = branch_hydrotable.loc[branch_hydrotable.HydroID.isin(branch_elev_table.HydroID)]
+            branch_hydrotable.drop(columns=['order_'], inplace=True)
+            # Join SRC with elevation data
+            branch_elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
+            branch_hydrotable = branch_hydrotable.merge(branch_elev_table, on="HydroID")
+            # Append to full rating curve dataframe
+            if hydrotable.empty:
+                hydrotable = branch_hydrotable
+            else:
+                hydrotable = hydrotable.append(branch_hydrotable)
+
+        # Join rating curves with elevation data
+        #elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
+        #hydrotable = hydrotable.merge(elev_table, on="HydroID")
+        if 'location_id' in hydrotable.columns:
+            relevant_gages = list(hydrotable.location_id.unique())
         else:
-            generate_facet_plot(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table)
+            relevant_gages = []
+        usgs_gages = usgs_gages[usgs_gages['location_id'].isin(relevant_gages)]
+        usgs_gages = usgs_gages.reset_index(drop=True)
+
+        if len(usgs_gages) > 0:
+
+            # Adjust rating curve to elevation
+            hydrotable['elevation_ft'] = (hydrotable.stage + hydrotable.dem_adj_elevation) * 3.28084 # convert from m to ft
+            # hydrotable['raw_elevation_ft'] = (hydrotable.stage + hydrotable.dem_elevation) * 3.28084 # convert from m to ft
+            hydrotable['discharge_cfs'] = hydrotable.discharge_cms * 35.3147
+            usgs_gages = usgs_gages.rename(columns={"flow": "discharge_cfs", "elevation_navd88": "elevation_ft"})
+
+            hydrotable['source'] = "FIM"
+            usgs_gages['source'] = "USGS"
+            limited_hydrotable = hydrotable.filter(items=['location_id','elevation_ft','discharge_cfs','source', 'HydroID', 'levpa_id', 'dem_adj_elevation'])
+            select_usgs_gages = usgs_gages.filter(items=['location_id', 'elevation_ft', 'discharge_cfs','source'])
+            if 'default_discharge_cms' in hydrotable.columns: # check if both "FIM" and "FIM_default" SRCs are available
+                hydrotable['default_discharge_cfs'] = hydrotable.default_discharge_cms * 35.3147
+                limited_hydrotable_default = hydrotable.filter(items=['location_id','elevation_ft', 'default_discharge_cfs','HydroID', 'levpa_id', 'dem_adj_elevation'])
+                limited_hydrotable_default['discharge_cfs'] = limited_hydrotable_default.default_discharge_cfs
+                limited_hydrotable_default['source'] = "FIM_default"
+                rating_curves = limited_hydrotable.append(select_usgs_gages)
+                rating_curves = rating_curves.append(limited_hydrotable_default)
+            else:
+                rating_curves = limited_hydrotable.append(select_usgs_gages)
+
+            # Add stream order
+            stream_orders = hydrotable.filter(items=['location_id','order_']).drop_duplicates()
+            rating_curves = rating_curves.merge(stream_orders, on='location_id')
+            rating_curves['order_'].fillna(0,inplace=True)
+            rating_curves['order_'] = rating_curves['order_'].astype('int')
+
+
+            # NWM recurr intervals
+            recurr_intervals = ("2","5","10","25","50","100")
+            recurr_dfs = []
+            for interval in recurr_intervals:
+                recurr_file = join(nwm_flow_dir, 'nwm21_17C_recurr_{}_0_cms.csv'.format(interval))
+                df = pd.read_csv(recurr_file, dtype={'feature_id': str})
+                # Update column names
+                df = df.rename(columns={"discharge": interval})
+                recurr_dfs.append(df)
+
+            # Merge NWM recurr intervals into a single layer
+            nwm_recurr_intervals_all = reduce(lambda x,y: pd.merge(x,y, on='feature_id', how='outer'), recurr_dfs)
+            nwm_recurr_intervals_all = pd.melt(nwm_recurr_intervals_all, id_vars=['feature_id'], value_vars=recurr_intervals, var_name='recurr_interval', value_name='discharge_cms')
+
+            # Append catfim data (already set up in format similar to nwm_recurr_intervals_all)
+            cat_fim = pd.read_csv(catfim_flows_filename, dtype={'feature_id':str})
+            nwm_recurr_intervals_all = nwm_recurr_intervals_all.append(cat_fim)
+
+            # Convert discharge to cfs and filter
+            nwm_recurr_intervals_all['discharge_cfs'] = nwm_recurr_intervals_all.discharge_cms * 35.3147
+            nwm_recurr_intervals_all = nwm_recurr_intervals_all.filter(items=['discharge_cfs', 'recurr_interval','feature_id']).drop_duplicates()
+
+            # Identify unique gages
+            usgs_crosswalk = hydrotable.filter(items=['location_id', 'feature_id']).drop_duplicates()
+            usgs_crosswalk.dropna(subset=['location_id'], inplace=True)
+
+            nwm_recurr_data_table = pd.DataFrame()
+            usgs_recurr_data = pd.DataFrame()
+
+            # Interpolate USGS/FIM elevation at each gage
+            for index, gage in usgs_crosswalk.iterrows():
+
+                # Interpolate USGS elevation at NWM recurrence intervals
+                usgs_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="USGS")]
+
+                if len(usgs_rc) <1:
+                    logging.info(f"missing USGS rating curve data for usgs station {gage.location_id} in huc {huc}")
+                    continue
+
+                str_order = np.unique(usgs_rc.order_).item()
+                feature_id = str(gage.feature_id)
+
+                usgs_pred_elev = get_reccur_intervals(usgs_rc, usgs_crosswalk,nwm_recurr_intervals_all)
+
+                # Handle sites missing data
+                if len(usgs_pred_elev) <1:
+                    logging.info(f"WARNING: missing USGS elevation data for usgs station {gage.location_id} in huc {huc}")
+                    continue
+
+                # Clean up data
+                usgs_pred_elev['location_id'] = gage.location_id
+                usgs_pred_elev = usgs_pred_elev.filter(items=['location_id','recurr_interval', 'discharge_cfs','pred_elev'])
+                usgs_pred_elev = usgs_pred_elev.rename(columns={"pred_elev": "USGS"})
+
+                # Interpolate FIM elevation at NWM recurrence intervals
+                fim_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="FIM")]
+
+                if len(fim_rc) <1:
+                    logging.info(f"missing FIM rating curve data for usgs station {gage.location_id} in huc {huc}")
+                    continue
+
+                fim_pred_elev = get_reccur_intervals(fim_rc, usgs_crosswalk,nwm_recurr_intervals_all)
+
+                # Handle sites missing data
+                if len(fim_pred_elev) <1:
+                    logging.info(f"WARNING: missing FIM elevation data for usgs station {gage.location_id} in huc {huc}")
+                    continue
+
+                # Clean up data
+                fim_pred_elev = fim_pred_elev.rename(columns={"pred_elev": "FIM"})
+                fim_pred_elev = fim_pred_elev.filter(items=['recurr_interval', 'discharge_cfs','FIM'])
+                usgs_pred_elev = usgs_pred_elev.merge(fim_pred_elev, on=['recurr_interval','discharge_cfs'])
+
+                # Add attributes
+                usgs_pred_elev['HUC'] = huc
+                usgs_pred_elev['HUC4'] = huc[0:4]
+                usgs_pred_elev['str_order'] = str_order
+                usgs_pred_elev['feature_id'] = feature_id
+
+                # Melt dataframe
+                usgs_pred_elev = pd.melt(usgs_pred_elev, id_vars=['location_id','feature_id','recurr_interval','discharge_cfs','HUC','HUC4','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
+                nwm_recurr_data_table = nwm_recurr_data_table.append(usgs_pred_elev)
+
+                # Interpolate FIM elevation at USGS observations
+                # fim_rc = fim_rc.merge(usgs_crosswalk, on="location_id")
+                # usgs_rc = usgs_rc.rename(columns={"elevation_ft": "USGS"})
+                #
+                # # Sort stage in ascending order
+                # usgs_rc = usgs_rc.sort_values('USGS',ascending=True)
+                #
+                # # Interpolate FIM elevation at USGS observations
+                # usgs_rc['FIM'] = np.interp(usgs_rc.discharge_cfs.values, fim_rc['discharge_cfs'], fim_rc['elevation_ft'], left = np.nan, right = np.nan)
+                # usgs_rc = usgs_rc[usgs_rc['FIM'].notna()]
+                # usgs_rc = usgs_rc.drop(columns=["source"])
+                #
+                # # Melt dataframe
+                # usgs_rc = pd.melt(usgs_rc, id_vars=['location_id','discharge_cfs','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
+                #
+                # if not usgs_rc.empty:
+                #     usgs_recurr_data = usgs_recurr_data.append(usgs_rc)
+
+            # Generate stats for all sites in huc
+            # if not usgs_recurr_data.empty:
+            #     usgs_recurr_stats_table = calculate_rc_stats_elev(usgs_recurr_data)
+            #     usgs_recurr_stats_table.to_csv(usgs_recurr_stats_filename,index=False)
+
+            # # Generate plots (not currently being used)
+            # fim_elev_at_USGS_rc_plot_filename = join(dirname(rc_comparison_plot_filename),'FIM_elevations_at_USGS_rc_' + str(huc) +'.png')
+            # generate_facet_plot(usgs_recurr_data, fim_elev_at_USGS_rc_plot_filename)
+
+            if not nwm_recurr_data_table.empty:
+                nwm_recurr_data_table.discharge_cfs = np.round(nwm_recurr_data_table.discharge_cfs,2)
+                nwm_recurr_data_table.elevation_ft = np.round(nwm_recurr_data_table.elevation_ft,2)
+                nwm_recurr_data_table.to_csv(nwm_recurr_data_filename,index=False)
+            if 'location_id' not in nwm_recurr_data_table.columns:
+                logging.info(f"WARNING: nwm_recurr_data_table is missing location_id column for gage {relevant_gages} in huc {huc}")
+
+            # plot rating curves
+            if alt_plot:
+                generate_rc_and_rem_plots(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table, branches_folder)
+            else:
+                generate_facet_plot(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table)
 
+        else:
+            logging.info(f"no USGS data for gage(s): {relevant_gages} in huc {huc}")
     else:
-        print(f"no USGS data for gage(s): {relevant_gages} in huc {huc}")
+        logging.info(f"no valid USGS gages found in huc {huc} (note: may be ahps sites without UGSG gages)")
 
 def aggregate_metrics(output_dir,procs_list,stat_groups):
 
@@ -305,25 +323,34 @@ def aggregate_metrics(output_dir,procs_list,stat_groups):
 
 
 def generate_facet_plot(rc, plot_filename, recurr_data_table):
-
+    
     # Filter FIM elevation based on USGS data
     for gage in rc.location_id.unique():
-
-        min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
-        max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
-        min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
-        max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
-        ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
-
-        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
-
-        if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
+        #print(recurr_data_table.head)
+        try:
+            min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
+            max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
+            min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
+            max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
+            ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
+
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
+
+            if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
+                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
+        except Exception as ex:
+            summary = traceback.StackSummary.extract(
+                    traceback.walk_stack(None))
+            logging.info("WARNING: rating curve dataframe not processed correctly...")
 
     rc = rc.rename(columns={"location_id": "USGS Gage"})
 
+    # split out branch 0 FIM data
+    rc['source_branch'] = np.where((rc.source == 'FIM') & (rc.levpa_id == '0'), 'FIM_b0', np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source))
+    #rc['source_branch'] = np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source)
+
     ## Generate rating curve plots
     num_plots = len(rc["USGS Gage"].unique())
     if num_plots > 3:
@@ -334,11 +361,16 @@ def generate_facet_plot(rc, plot_filename, recurr_data_table):
     sns.set(style="ticks")
 
     # Plot both "FIM" and "FIM_default" rating curves
-    hue_order = ['USGS','FIM','FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM']
+    if '0' in rc.levpa_id.values: # checks to see if branch zero data exists in the rating curve df
+        hue_order = ['USGS','FIM','FIM_default','FIM_b0','FIM_default_b0'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM','FIM_b0']
+        kw = {'color': ['blue','green','orange','green','orange'], 'linestyle' : ["-","-","-","--","--"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green','green'], 'linestyle' : ["-","-","--"]}
+    else:
+        hue_order = ['USGS','FIM','FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM']
+        kw = {'color': ['blue','green','orange'], 'linestyle' : ["-","-","-"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green'], 'linestyle' : ["-","_"]}
     # Facet Grid
-    g = sns.FacetGrid(rc, col="USGS Gage", hue="source", hue_order=hue_order,
+    g = sns.FacetGrid(rc, col="USGS Gage", hue="source_branch", hue_order=hue_order,
                     sharex=False, sharey=False,col_wrap=columns,
-                    height=3.5, aspect=1.65)
+                    height=3.5, aspect=1.65, hue_kws=kw)
     g.map(plt.plot, "discharge_cfs", "elevation_ft", linewidth=2, alpha=0.8)
     g.set_axis_labels(x_var="Discharge (cfs)", y_var="Elevation (ft)")
 
@@ -347,13 +379,18 @@ def generate_facet_plot(rc, plot_filename, recurr_data_table):
     for gage in axes:
         ax = axes[gage]
         plt.sca(ax)
-        recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
-            .filter(items=['recurr_interval', 'discharge_cfs'])
-        for i, r in recurr_data.iterrows():
-            if not r.recurr_interval.isnumeric(): continue # skip catfim flows
-            l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
-            plt.axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
-            plt.text(r.discharge_cfs, ax.get_ylim()[1] - (ax.get_ylim()[1]-ax.get_ylim()[0])*0.03, r.recurr_interval, size='small', c='purple')
+        try:
+            recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
+                .filter(items=['recurr_interval', 'discharge_cfs'])
+            for i, r in recurr_data.iterrows():
+                if not r.recurr_interval.isnumeric(): continue # skip catfim flows
+                l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
+                plt.axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
+                plt.text(r.discharge_cfs, ax.get_ylim()[1] - (ax.get_ylim()[1]-ax.get_ylim()[0])*0.03, r.recurr_interval, size='small', c='purple')
+        except Exception as ex:
+            summary = traceback.StackSummary.extract(
+                    traceback.walk_stack(None))
+            logging.info("WARNING: Could not plot recurrence intervals...")
 
     # Adjust the arrangement of the plots
     g.fig.tight_layout(w_pad=1)
@@ -427,7 +464,7 @@ def generate_rc_and_rem_plots(rc, plot_filename, recurr_data_table, branches_fol
         # Get the hydroid    
         hydroid = rc[rc.location_id == gage].HydroID.unique()[0]
         if not hydroid:
-            print(f'Gage {gage} in HUC {branch} has no HydroID')
+            logging.info(f'Gage {gage} in HUC {branch} has no HydroID')
             continue
 
         # Filter the reaches and REM by the hydroid
@@ -475,11 +512,18 @@ def get_reccur_intervals(site_rc, usgs_crosswalk,nwm_recurr_intervals):
     nwm_ids = len(usgs_site.feature_id.drop_duplicates())
 
     if nwm_ids > 0:
-
-        nwm_recurr_intervals = nwm_recurr_intervals.copy().loc[nwm_recurr_intervals.feature_id==usgs_site.feature_id.drop_duplicates().item()]
-        nwm_recurr_intervals['pred_elev'] = np.interp(nwm_recurr_intervals.discharge_cfs.values, usgs_site['discharge_cfs'], usgs_site['elevation_ft'], left = np.nan, right = np.nan)
-
-        return nwm_recurr_intervals
+        try:
+            nwm_recurr_intervals = nwm_recurr_intervals.copy().loc[nwm_recurr_intervals.feature_id==usgs_site.feature_id.drop_duplicates().item()]
+            nwm_recurr_intervals['pred_elev'] = np.interp(nwm_recurr_intervals.discharge_cfs.values, usgs_site['discharge_cfs'], usgs_site['elevation_ft'], left = np.nan, right = np.nan)
+
+            return nwm_recurr_intervals
+        except Exception as ex:
+            summary = traceback.StackSummary.extract(
+                    traceback.walk_stack(None))
+            #logging.info("WARNING: get_recurr_intervals failed for some reason....")
+            #logging.info(f"*** {ex}")                
+            #logging.info(''.join(summary.format()))
+            return []
 
     else:
         return []
@@ -729,9 +773,11 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     print(check_file_age(usgs_gages_filename))
 
     # Open log file
-    sys.__stdout__ = sys.stdout
-    log_file = open(join(output_dir,'rating_curve_comparison.log'),"w")
-    sys.stdout = log_file
+    # Create log output
+    level    = logging.INFO # using WARNING level to avoid benign? info messages ("Failed to auto identify EPSG: 7")
+    format   = '  %(message)s'
+    handlers = [logging.FileHandler(os.path.join(output_dir,'rating_curve_comparison.log')), logging.StreamHandler()]
+    logging.basicConfig(level = level, format = format, handlers = handlers)
 
     merged_elev_table = []
     huc_list  = [huc for huc in os.listdir(fim_dir) if re.search("^\d{6,8}$", huc)]
@@ -754,27 +800,27 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
 
     # Output a concatenated elev_table to_csv
     if merged_elev_table:
-        print(f"Creating aggregate elev table csv")
+        logging.info(f"Creating aggregate elev table csv")
         concat_elev_table = pd.concat(merged_elev_table)
         concat_elev_table['thal_burn_depth_meters'] = concat_elev_table['dem_elevation'] - concat_elev_table['dem_adj_elevation']
         concat_elev_table.to_csv(join(output_dir,'agg_usgs_elev_table.csv'),index=False)
 
     # Initiate multiprocessing
-    print(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    logging.info(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")
     with Pool(processes=number_of_jobs) as pool:
         pool.map(generate_rating_curve_metrics, procs_list)
 
     # Create point layer of usgs gages with joined stats attributes
     if stat_gages:
-        print("Creating usgs gages GPKG with joined rating curve summary stats")
+        logging.info("Creating usgs gages GPKG with joined rating curve summary stats")
         agg_recurr_stats_table = aggregate_metrics(output_dir,procs_list,['location_id'])
         create_static_gpkg(output_dir, stat_gages, agg_recurr_stats_table, gages_gpkg_filepath)
         del agg_recurr_stats_table # memory cleanup
     else: # if not producing GIS layer, just aggregate metrics
-        print(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
+        logging.info(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
         aggregate_metrics(output_dir,procs_list,stat_groups)
 
-    print('Delete intermediate tables')
+    logging.info('Delete intermediate tables')
     shutil.rmtree(tables_dir, ignore_errors=True)
 
     # Compare current sierra test results to previous tests
@@ -790,7 +836,3 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
         evaluate_results(sierra_test_dfs,
                         sierra_test_labels,
                         join(output_dir, 'Sierra_Test_Eval_boxplot.png'))
-
-    # Close log file
-    sys.stdout = sys.__stdout__
-    log_file.close()

From 7405cffcfd6ca5ff05dcfff08c10306a49464810 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 28 Oct 2022 15:14:00 -0400
Subject: [PATCH 170/359] v4.0.10.2 Runs branch 0 on HUCs without other
 branches after stream order filtering (#723)

---
 docs/CHANGELOG.md                 | 18 ++++++++++++++++++
 src/gms/buffer_stream_branches.py | 25 +++++++++++++------------
 src/gms/derive_level_paths.py     | 30 ++++++++++++++++--------------
 src/gms/generate_branch_list.py   | 19 ++++++++++---------
 src/gms/stream_branches.py        | 12 ++++++++++++
 5 files changed, 69 insertions(+), 35 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f7390c91b..8904088a0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.10.2 - 2022-10-24 - [PR #723](https://github.com/NOAA-OWP/inundation-mapping/pull/723)
+
+Runs branch 0 on HUCs with no other branches remaining after filtering stream orders if `drop_low_stream_orders` is used.
+
+### Additions
+
+- `src/gms`
+    - `stream_branches.py`: adds `exclude_attribute_values()` to filter out stream orders 1&2 outside of `load_file()`
+
+### Changes
+
+- `src/gms`
+    - `buffer_stream_branches.py`: adds check for `streams_file`
+    - `derive_level_paths.py`: checks length of `stream_network` before filtering out stream orders 1&2, then filters using `stream_network.exclude_attribute_values()`
+    - `generate_branch_list.py`: adds check for `stream_network_dissolved`
+
+<br/><br/>
+    
 ## v4.0.10.1 - 2022-10-5 - [PR #695](https://github.com/NOAA-OWP/inundation-mapping/pull/695)
 
 This hotfix address a bug with how the rating curve comparison (sierra test) handles the branch zero synthetic rating curve in the comparison plots. Address #676 
diff --git a/src/gms/buffer_stream_branches.py b/src/gms/buffer_stream_branches.py
index af7ef661d..cb1426d2c 100755
--- a/src/gms/buffer_stream_branches.py
+++ b/src/gms/buffer_stream_branches.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-
+import os
 from stream_branches import StreamNetwork
 from stream_branches import StreamBranchPolygons
 import argparse
@@ -19,16 +19,17 @@
     # extract to dictionary
     args = vars(parser.parse_args())
 
-    streams_file , branch_id_attribute, buffer_distance, stream_polygons_file, verbose = args["streams"], args["branch_id"] , args["buffer_distance"], args["branches"] , args["verbose"]
-    
-    # load file
-    stream_network = StreamNetwork.from_file( filename=streams_file,branch_id_attribute=branch_id_attribute,
-                                              values_excluded=None,attribute_excluded=None, verbose = verbose)
-    
-    # make stream polygons 
-    stream_polys = StreamBranchPolygons.buffer_stream_branches( stream_network,
-                                                                buffer_distance=buffer_distance,
-                                                                verbose=verbose                  )
+    streams_file, branch_id_attribute, buffer_distance, stream_polygons_file, verbose = args["streams"], args["branch_id"] , args["buffer_distance"], args["branches"] , args["verbose"]
     
-    stream_polys.write(stream_polygons_file,verbose=verbose)
+    if os.path.exists(streams_file):
+        # load file
+        stream_network = StreamNetwork.from_file( filename=streams_file,branch_id_attribute=branch_id_attribute,
+                                                values_excluded=None,attribute_excluded=None, verbose = verbose)
+        
+        # make stream polygons 
+        stream_polys = StreamBranchPolygons.buffer_stream_branches( stream_network,
+                                                                    buffer_distance=buffer_distance,
+                                                                    verbose=verbose                  )
+        
+        stream_polys.write(stream_polygons_file,verbose=verbose)
 
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index 7102a6732..fec0a4472 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -24,27 +24,29 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
     if verbose:
         print("Loading stream network ...")
         
-    if (drop_low_stream_orders):
-        stream_network = StreamNetwork.from_file(filename=in_stream_network,
-                                                 branch_id_attribute="order_",
-                                                 values_excluded=[1,2]
-                                                 )
-                                                 
-                                                 
-    else:
-        stream_network = StreamNetwork.from_file(filename=in_stream_network)
+    stream_network = StreamNetwork.from_file(filename=in_stream_network)
 
-    # if there are no reaches at this point (due to filtering if applicable)
-    if (len(stream_network) == 0) and (drop_low_stream_orders):
+    # if there are no reaches at this point
+    if (len(stream_network) == 0):
         # This is technically not an error but we need to have it logged so the user know what
         # happened to it and we need the huc to not be included in future processing. 
         # We need it to be not included in the gms_input.csv at the end of the unit processing.
         # Throw an exception with valid text. This will show up in the non-zero exit codes and explain why an error.
         # Later, we can look at creating custom sys exit codes 
-        # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file or stream order filtering.")
-        print("Sorry, no branches exist and processing can not continue. This could be an empty file or stream order filtering.")
+        # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file.")
+        print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
         sys.exit(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
-
+                                                 
+    elif (drop_low_stream_orders):
+        stream_network = stream_network.exclude_attribute_values(branch_id_attribute="order_",
+                                               values_excluded=[1,2]
+                                              )
+
+        # if there are no reaches at this point (due to filtering)
+        if (len(stream_network) == 0):
+            print("No branches exist but branch zero processing will continue. This could be due to stream order filtering.")
+            return
+                                                 
     inlets_attribute = 'inlet_id'
     outlets_attribute = 'outlet_id'
     outlet_linestring_index = -1
diff --git a/src/gms/generate_branch_list.py b/src/gms/generate_branch_list.py
index 96a57c1cf..269263f63 100755
--- a/src/gms/generate_branch_list.py
+++ b/src/gms/generate_branch_list.py
@@ -1,25 +1,26 @@
 #!/usr/bin/env python3
 
+import os
 import pandas as pd
 from stream_branches import StreamNetwork
 import argparse
 
 def Generate_branch_list(stream_network_dissolved, branch_id_attribute, output_branch_list, branch_zero):
 
-    # load stream network
+    if os.path.exists(stream_network_dissolved):
+        # load stream network
+        stream_network_dissolved = StreamNetwork.from_file( stream_network_dissolved,
+                                                            branch_id_attribute=branch_id_attribute )
 
-    stream_network_dissolved = StreamNetwork.from_file( stream_network_dissolved,
-                                                        branch_id_attribute=branch_id_attribute )
+        # reduce to branch id attribute and convert to pandas df
+        stream_network_dissolved = stream_network_dissolved.loc[:,branch_id_attribute]
 
-    # reduce to branch id attribute and convert to pandas df
-    stream_network_dissolved = stream_network_dissolved.loc[:,branch_id_attribute]
-
-    # write
-    stream_network_dissolved.to_csv(output_branch_list,sep= " ",index=False,header=False)
+        # write
+        stream_network_dissolved.to_csv(output_branch_list,sep= " ",index=False,header=False)
 
     # Add branch zero ID to branch list
     if branch_zero:
-        with open(output_branch_list,'a') as branch_lst:
+        with open(output_branch_list, 'a') as branch_lst:
             branch_lst.write(f'{branch_zero}')
 
 if __name__ == '__main__':
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index fdfaf1ee6..90d928631 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -375,6 +375,18 @@ def derive_headwater_points_with_inlets( self,inlets_attribute='inlet_id',
 
         return(headwater_points_gdf)
 
+
+    def exclude_attribute_values(self, branch_id_attribute=None, values_excluded=None, verbose=False):
+                      
+        if (branch_id_attribute is not None) and (values_excluded is not None):
+            self = StreamNetwork(self[~self[branch_id_attribute].isin(values_excluded)], branch_id_attribute=branch_id_attribute)
+       
+        if verbose:         
+             print("Number of df rows = " + str(self.shape[0]))
+        
+        return(self)
+
+
     def remove_stream_segments_without_catchments( self,
                                                    catchments,
                                                    reach_id_attribute='NHDPlusID',

From 31436ac5f006994fdb0314691cac77918c211459 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 28 Oct 2022 15:18:54 -0400
Subject: [PATCH 171/359] v4.0.10.3 Port thalweg notch fix from FIM3 (#718)

---
 docs/CHANGELOG.md          |  10 +++
 src/clip_vectors_to_wbd.py | 148 +++++++++++++++++--------------------
 src/gms/run_by_unit.sh     |   2 +-
 3 files changed, 80 insertions(+), 80 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8904088a0..d781b67ba 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.10.3 - 2022-10-19 - [PR #718](https://github.com/NOAA-OWP/inundation-mapping/pull/718)
+
+Fixes thalweg notch by clipping upstream ends of the stream segments to prevent the stream network from reaching the edge of the DEM and being treated as outlets when pit filling the burned DEM.
+
+### Changes
+
+- `src/clip_vectors_to_wbd.py`: Uses a slightly smaller buffer than wbd_buffer (wbd_buffer_distance-2*(DEM cell size)) to clip stream network inside of DEM extent.
+
+<br/><br/>
+
 ## v4.0.10.2 - 2022-10-24 - [PR #723](https://github.com/NOAA-OWP/inundation-mapping/pull/723)
 
 Runs branch 0 on HUCs with no other branches remaining after filtering stream orders if `drop_low_stream_orders` is used.
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 7a4c94492..9684d377d 100755
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -1,54 +1,68 @@
 #!/usr/bin/env python3
 
+import os
 import sys
+import pandas as pd
 import geopandas as gpd
 import argparse
 from shapely.geometry import MultiPolygon,Polygon
 from utils.shared_functions import getDriver, mem_profile
-
+import rasterio as rio
 
 @mem_profile
-def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance):
+def subset_vector_layers(hucCode, nwm_streams_filename, nhd_streams_filename, nwm_lakes_filename, nld_lines_filename, nwm_catchments_filename, nhd_headwaters_filename, landsea_filename, wbd_filename, wbd_buffer_filename, subset_nhd_streams_filename, subset_nld_lines_filename, subset_nwm_lakes_filename, subset_nwm_catchments_filename, subset_nhd_headwaters_filename, subset_nwm_streams_filename, subset_landsea_filename, great_lakes_filename, wbd_buffer_distance, lake_buffer_distance, dem_filename):
 
     hucUnitLength = len(str(hucCode))
 
+    with rio.open(dem_filename) as dem_raster:
+        dem_cellsize = max(dem_raster.res)
+
     # Get wbd buffer
     wbd = gpd.read_file(wbd_filename)
     wbd_buffer = wbd.copy()
-    wbd_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance,resolution=32)
-    projection = wbd_buffer.crs
+    wbd_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance, resolution=32)
+
+    # Make the streams buffer smaller than the wbd_buffer so streams don't reach the edge of the DEM
+    wbd_buffer_filename_split = os.path.splitext(wbd_buffer_filename)
+    wbd_streams_buffer_filename = wbd_buffer_filename_split[0] + '_streams' + wbd_buffer_filename_split[1]
+    wbd_streams_buffer = wbd.copy()
+    wbd_streams_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance-3*dem_cellsize, resolution=32)
 
-    great_lakes = gpd.read_file(great_lakes_filename, mask = wbd_buffer).reset_index(drop=True)
+    # projection = wbd_buffer.crs
+
+    great_lakes = gpd.read_file(great_lakes_filename, mask=wbd_buffer).reset_index(drop=True)
 
     if not great_lakes.empty:
-        print("Masking Great Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
+        print("Masking Great Lakes for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
 
         # Clip excess lake area
         great_lakes = gpd.clip(great_lakes, wbd_buffer)
+        great_lakes_streams = gpd.clip(great_lakes, wbd_streams_buffer)
 
         # Buffer remaining lake area
         great_lakes.geometry = great_lakes.buffer(lake_buffer_distance)
+        great_lakes_streams.geometry = great_lakes_streams.buffer(lake_buffer_distance)
 
         # Removed buffered GL from WBD buffer
         wbd_buffer = gpd.overlay(wbd_buffer, great_lakes, how='difference')
-        wbd_buffer = wbd_buffer[['geometry']]
-        wbd_buffer.to_file(wbd_buffer_filename,driver=getDriver(wbd_buffer_filename),index=False)
+        wbd_streams_buffer = gpd.overlay(wbd_streams_buffer, great_lakes, how='difference')
 
-    else:
-        wbd_buffer = wbd_buffer[['geometry']]
-        wbd_buffer.to_file(wbd_buffer_filename,driver=getDriver(wbd_buffer_filename),index=False)
+    wbd_buffer = wbd_buffer[['geometry']]
+    wbd_streams_buffer = wbd_streams_buffer[['geometry']]
+    wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False)
+    wbd_streams_buffer.to_file(wbd_streams_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False)
 
     del great_lakes
 
     # Clip ocean water polygon for future masking ocean areas (where applicable)
-    landsea = gpd.read_file(landsea_filename, mask = wbd_buffer)
+    landsea = gpd.read_file(landsea_filename, mask=wbd_buffer)
     if not landsea.empty:
-        landsea.to_file(subset_landsea_filename,driver=getDriver(subset_landsea_filename),index=False)
+        landsea.to_file(subset_landsea_filename, driver=getDriver(subset_landsea_filename), index=False)
     del landsea
 
     # Find intersecting lakes and writeout
     print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_lakes = gpd.read_file(nwm_lakes_filename, mask = wbd_buffer)
+    nwm_lakes = gpd.read_file(nwm_lakes_filename, mask=wbd_buffer)
     nwm_lakes = nwm_lakes.loc[nwm_lakes.Shape_Area < 18990454000.0]
 
     if not nwm_lakes.empty:
@@ -58,90 +72,66 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
         # Loop through the filled polygons and insert the new geometry
         for i in range(len(nwm_lakes_fill_holes)):
             nwm_lakes.loc[i,'geometry'] = nwm_lakes_fill_holes[i]
-        nwm_lakes.to_file(subset_nwm_lakes_filename,driver=getDriver(subset_nwm_lakes_filename),index=False)
+        nwm_lakes.to_file(subset_nwm_lakes_filename, driver=getDriver(subset_nwm_lakes_filename), index=False)
     del nwm_lakes
 
     # Find intersecting levee lines
-    print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nld_lines = gpd.read_file(nld_lines_filename, mask = wbd_buffer)
+    print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    nld_lines = gpd.read_file(nld_lines_filename, mask=wbd_buffer)
     if not nld_lines.empty:
-        nld_lines.to_file(subset_nld_lines_filename,driver=getDriver(subset_nld_lines_filename),index=False)
+        nld_lines.to_file(subset_nld_lines_filename, driver=getDriver(subset_nld_lines_filename), index=False)
     del nld_lines
 
     # Subset nhd headwaters
-    print("Subsetting NHD Headwater Points for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nhd_headwaters = gpd.read_file(nhd_headwaters_filename, mask = wbd_buffer)
-    if extent == 'MS':
-        nhd_headwaters = nhd_headwaters.loc[nhd_headwaters.mainstem==1]
+    print("Subsetting NHD Headwater Points for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    nhd_headwaters = gpd.read_file(nhd_headwaters_filename, mask=wbd_streams_buffer)
 
     if len(nhd_headwaters) > 0:
-        nhd_headwaters.to_file(subset_nhd_headwaters_filename,driver=getDriver(subset_nhd_headwaters_filename),index=False)
+        nhd_headwaters.to_file(subset_nhd_headwaters_filename, driver=getDriver(subset_nhd_headwaters_filename), index=False)
     else:
-        print ("No headwater point(s) within HUC " + str(hucCode) +  " boundaries.")
+        print ("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nhd_headwaters
 
     # Subset nhd streams
-    print("Querying NHD Streams for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nhd_streams = gpd.read_file(nhd_streams_filename, mask = wbd_buffer)
-
-    if extent == 'MS':
-        nhd_streams = nhd_streams.loc[nhd_streams.mainstem==1]
+    print("Querying NHD Streams for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    nhd_streams = gpd.read_file(nhd_streams_filename, mask=wbd_streams_buffer)
 
     if len(nhd_streams) > 0:
-
-        # Find incoming stream segments (to WBD buffer) and identify which are upstream
-        threshold_segments = gpd.overlay(nhd_streams, wbd_buffer, how='symmetric_difference')
-        from_list = threshold_segments.FromNode.to_list()
-        to_list = nhd_streams.ToNode.to_list()
-        missing_segments = list(set(from_list) - set(to_list))
-
-        # special case: stream meanders in and out of WBD buffer boundary
-        if str(hucCode) == '10030203':
-            missing_segments = missing_segments + [23001300001840.0, 23001300016571.0]
-            
-        if str(hucCode) == '08030100':
-            missing_segments = missing_segments + [20000600011559.0, 20000600045761.0, 20000600002821.0]
-
-        # Remove incoming stream segment so it won't be routed as outflow during hydroconditioning
-        nhd_streams = nhd_streams.loc[~nhd_streams.FromNode.isin(missing_segments)]
+        nhd_streams = gpd.clip(nhd_streams, wbd_streams_buffer)
 
         nhd_streams.to_file(subset_nhd_streams_filename,driver=getDriver(subset_nhd_streams_filename),index=False)
     else:
-        print ("No NHD streams within HUC " + str(hucCode) +  " boundaries.")
+        print ("No NHD streams within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nhd_streams
 
     # Find intersecting nwm_catchments
-    print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_catchments = gpd.read_file(nwm_catchments_filename, mask = wbd_buffer)
-    if extent == 'MS':
-        nwm_catchments = nwm_catchments.loc[nwm_catchments.mainstem==1]
+    print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    nwm_catchments = gpd.read_file(nwm_catchments_filename, mask=wbd_buffer)
 
     if len(nwm_catchments) > 0:
-        nwm_catchments.to_file(subset_nwm_catchments_filename,driver=getDriver(subset_nwm_catchments_filename),index=False)
+        nwm_catchments.to_file(subset_nwm_catchments_filename, driver=getDriver(subset_nwm_catchments_filename), index=False)
     else:
-        print ("No NHD catchments within HUC " + str(hucCode) +  " boundaries.")
+        print ("No NHD catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_catchments
 
     # Subset nwm streams
-    print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    if extent == 'GMS':
-        nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd)
-        nwm_streams = gpd.clip(nwm_streams,wbd)
-    else:
-        nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd_buffer)
+    print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+
+    nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd)
+    nwm_streams = gpd.clip(nwm_streams, wbd)
 
      # NWM can have duplicate records, but appear to always be identical duplicates
     nwm_streams.drop_duplicates(subset="ID", keep="first", inplace=True)
 
-    if extent == 'MS':
-        nwm_streams = nwm_streams.loc[nwm_streams.mainstem==1]
     if len(nwm_streams) > 0:
-        nwm_streams.to_file(subset_nwm_streams_filename,driver=getDriver(subset_nwm_streams_filename),index=False)
+        nwm_streams = gpd.clip(nwm_streams, wbd_streams_buffer)
+
+        nwm_streams.to_file(subset_nwm_streams_filename, driver=getDriver(subset_nwm_streams_filename), index=False)
     else:
-        print ("No NWM stream segments within HUC " + str(hucCode) +  " boundaries.")
+        print ("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_streams
 
@@ -154,22 +144,22 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     parser.add_argument('-s','--nhd-streams',help='NHDPlus HR burnline',required=True)
     parser.add_argument('-l','--nwm-lakes', help='NWM Lakes', required=True)
     parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path', required=True)
-    parser.add_argument('-g','--wbd',help='HUC boundary',required=True)
-    parser.add_argument('-f','--wbd-buffer',help='Buffered HUC boundary',required=True)
+    parser.add_argument('-g','--wbd', help='HUC boundary', required=True)
+    parser.add_argument('-i','--dem-filename', help='DEM filename', required=True)
+    parser.add_argument('-f','--wbd-buffer', help='Buffered HUC boundary', required=True)
     parser.add_argument('-m','--nwm-catchments', help='NWM catchments', required=True)
-    parser.add_argument('-y','--nhd-headwaters',help='NHD headwaters',required=True)
-    parser.add_argument('-v','--landsea',help='LandSea - land boundary',required=True)
-    parser.add_argument('-c','--subset-nhd-streams',help='NHD streams subset',required=True)
-    parser.add_argument('-z','--subset-nld-lines',help='Subset of NLD levee vectors for HUC',required=True)
-    parser.add_argument('-a','--subset-lakes',help='NWM lake subset',required=True)
-    parser.add_argument('-n','--subset-catchments',help='NWM catchments subset',required=True)
-    parser.add_argument('-e','--subset-nhd-headwaters',help='NHD headwaters subset',required=True,default=None)
-    parser.add_argument('-b','--subset-nwm-streams',help='NWM streams subset',required=True)
-    parser.add_argument('-x','--subset-landsea',help='LandSea subset',required=True)
-    parser.add_argument('-extent','--extent',help='FIM extent',required=True)
-    parser.add_argument('-gl','--great-lakes-filename',help='Great Lakes layer',required=True)
-    parser.add_argument('-wb','--wbd-buffer-distance',help='WBD Mask buffer distance',required=True,type=int)
-    parser.add_argument('-lb','--lake-buffer-distance',help='Great Lakes Mask buffer distance',required=True,type=int)
+    parser.add_argument('-y','--nhd-headwaters', help='NHD headwaters', required=True)
+    parser.add_argument('-v','--landsea', help='LandSea - land boundary', required=True)
+    parser.add_argument('-c','--subset-nhd-streams', help='NHD streams subset', required=True)
+    parser.add_argument('-z','--subset-nld-lines', help='Subset of NLD levee vectors for HUC', required=True)
+    parser.add_argument('-a','--subset-lakes', help='NWM lake subset', required=True)
+    parser.add_argument('-n','--subset-catchments', help='NWM catchments subset', required=True)
+    parser.add_argument('-e','--subset-nhd-headwaters', help='NHD headwaters subset', required=True, default=None)
+    parser.add_argument('-b','--subset-nwm-streams', help='NWM streams subset', required=True)
+    parser.add_argument('-x','--subset-landsea', help='LandSea subset', required=True)
+    parser.add_argument('-gl','--great-lakes-filename', help='Great Lakes layer', required=True)
+    parser.add_argument('-wb','--wbd-buffer-distance', help='WBD Mask buffer distance', required=True, type=int)
+    parser.add_argument('-lb','--lake-buffer-distance', help='Great Lakes Mask buffer distance', required=True, type=int)
 
     args = vars(parser.parse_args())
 
@@ -190,9 +180,9 @@ def subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_l
     subset_nhd_headwaters_filename = args['subset_nhd_headwaters']
     subset_nwm_streams_filename = args['subset_nwm_streams']
     subset_landsea_filename = args['subset_landsea']
-    extent = args['extent']
     great_lakes_filename = args['great_lakes_filename']
     wbd_buffer_distance = args['wbd_buffer_distance']
     lake_buffer_distance  = args['lake_buffer_distance']
+    dem_filename = args['dem_filename']
 
-    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename,subset_nhd_headwaters_filename,subset_nwm_streams_filename,subset_landsea_filename,extent,great_lakes_filename,wbd_buffer_distance,lake_buffer_distance)
+    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename, subset_nhd_headwaters_filename, subset_nwm_streams_filename, subset_landsea_filename, great_lakes_filename,wbd_buffer_distance,lake_buffer_distance, dem_filename)
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index f0ef94023..b7fba718a 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -71,7 +71,7 @@ Tcount
 echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
+python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM
 Tcount
 
 ## Clip WBD8 ##

From e3c34c8c812f4c7008b5f65020531c777c3116be Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 28 Oct 2022 15:21:19 -0400
Subject: [PATCH 172/359] v4.0.10.4 Create HydroID - feature_id crosswalk table
 (#727)

---
 docs/CHANGELOG.md                           | 11 ++++++
 gms_run_branch.sh                           | 13 ++++++-
 tools/gms_tools/combine_crosswalk_tables.py | 39 +++++++++++++++++++++
 3 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 tools/gms_tools/combine_crosswalk_tables.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d781b67ba..ac7567645 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.10.4 - 2022-10-27 - [PR #727](https://github.com/NOAA-OWP/inundation-mapping/pull/727)
+
+Creates a single crosswalk table containing HUC (huc8), BranchID, HydroID, feature_id (and optionally LakeID) from branch-level hydroTables.csv files.
+
+### Additions
+
+- `tools/gms_tools/combine_crosswalk_tables.py`: reads and concatenates hydroTable.csv files, writes crosswalk table
+- `gms_run_branch.sh`: Adds `tools/gms_tools/make_complete_hydrotable.py` to post-processing
+
+<br/><br/>
+
 ## v4.0.10.3 - 2022-10-19 - [PR #718](https://github.com/NOAA-OWP/inundation-mapping/pull/718)
 
 Fixes thalweg notch by clipping upstream ends of the stream segments to prevent the stream network from reaching the edge of the DEM and being treated as outlets when pit filling the burned DEM.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 3e8cff749..2c04583ea 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -262,10 +262,21 @@ find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -
 # Needed in case aggregation fails, we will need the logs
 echo
 echo -e $startDiv"Removing branches that failed with Exit status: 61"$stopDiv
+Tstart
 python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g $outputRunDataDir/gms_inputs.csv
+Tcount
+date -u
+
+echo
+echo -e $startDiv"Combining crosswalk tables"$stopDiv
+# aggregate outputs
+Tstart
+python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
+Tcount
+date -u
 
 echo "=========================================================================="
 echo "GMS_run_branch complete"
 Tcount
-echo "Ended: `date -u`" 
+echo "Ended: `date -u`"
 echo
diff --git a/tools/gms_tools/combine_crosswalk_tables.py b/tools/gms_tools/combine_crosswalk_tables.py
new file mode 100644
index 000000000..fdb6afcb7
--- /dev/null
+++ b/tools/gms_tools/combine_crosswalk_tables.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import argparse
+import pandas as pd
+
+def combine_crosswalk_tables(data_directory, output_filename):
+    """
+    Combines all hydrotables from a run into a single crosswalk table with HUC, BranchID, HydroID, feature_id, and LakeID
+    """
+
+    file_list = sorted(glob.glob(os.path.join(data_directory, '*', 'branches', '*', 'hydroTable_*.csv')))
+
+    dfs = list()
+    for filename in file_list:
+        if os.path.exists(filename):
+            file_df = pd.read_csv(filename, usecols=['HUC', 'HydroID', 'feature_id', 'LakeID'], dtype={'HUC':str})
+            file_df.drop_duplicates(inplace=True)
+            file_df.rename(columns={'HUC':'huc8'}, inplace=True)
+            file_df['BranchID'] = os.path.split(os.path.dirname(filename))[1]
+
+            dfs.append(file_df)
+        else:
+            print(f'{filename} is missing.')
+
+    df = pd.concat(dfs)
+
+    df.to_csv(output_filename, index=False)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Combines hydrotables from HUC/branch into a single crosswalk table')
+    parser.add_argument('-d', '--data-directory', help='Data directory (name of run)', type=str, required=True)
+    parser.add_argument('-o', '--output-filename', help='Filename for output', type=str, required=True)
+
+    args = vars(parser.parse_args())
+
+    combine_crosswalk_tables(**args)
\ No newline at end of file

From 9a8e3d8fbcb46e93f47e6aa1356e2cc2ac29252a Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 28 Oct 2022 14:24:48 -0500
Subject: [PATCH 173/359] v4.0.10.5 Add cblend (buffer) to acquire 3dep
 downloads (#720)

---
 data/create_vrt_file.py                       |  2 ++
 data/usgs/acquire_and_preprocess_3dep_dems.py | 10 +++++++---
 docs/CHANGELOG.md                             | 18 ++++++++++++++++++
 src/gms/run_by_unit.sh                        |  1 +
 4 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/data/create_vrt_file.py b/data/create_vrt_file.py
index ca15dc147..0e3c1a96b 100644
--- a/data/create_vrt_file.py
+++ b/data/create_vrt_file.py
@@ -98,6 +98,8 @@ def __setup_logger(output_folder_path):
 
 if __name__ == '__main__':
 
+    # Sample Usage:  python3 /foss_fim/data/create_vrt_file.py -s /data/inputs/3dep_dems/10m_5070/ -n "fim_seamless_3dep_dem_10m_5070.vrt"
+
     parser = argparse.ArgumentParser(description='Create a vrt using all tifs in a given directory')
 
     parser.add_argument('-s','--src_directory', help='A directory of where the .tif files '\
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index a74d5accc..dbff22924 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -107,6 +107,7 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
     
     end_time = datetime.now()
     fh.print_end_header('Loading 3dep dems', start_time, end_time)
+    print(f'---- NOTE: Remember to scan the log file for any failures')
     logging.info(fh.print_date_time_duration(start_time, end_time))
 
 
@@ -127,6 +128,9 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     ----------
         - pixel size set to 10 x 10 (m)
         - block size (256) (sometimes we use 512)
+        - cblend 6 add's a small buffer when pulling down the tif (ensuring seamless
+          overlap at the borders.)    
+    
     '''
 
     print(f"==========================================================")
@@ -136,7 +140,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
     base_cmd += ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
     base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
-    base_cmd += ' -t_srs EPSG:5070'
+    base_cmd += ' -t_srs EPSG:5070 -cblend 6'
    
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
 
@@ -205,7 +209,7 @@ def download_usgs_dem_file(extent_file,
                 base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
                 base_cmd +=  ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
                 base_cmd +=  ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
-                base_cmd +=  ' -t_srs EPSG:5070'
+                base_cmd +=  ' -t_srs EPSG:5070 -cblend 6'
         - retry (bool)
              If True, and the file exists (and is over 0k), downloading will be skipped.
         
@@ -295,7 +299,7 @@ def __setup_logger(output_folder_path):
     # Parse arguments.
     
     # sample usage (min params):
-    # - python3 /foss_fim/data/usgs/acquire_and_preprocess_3dep_dems.py -e /data/inputs/wbd/HUC4
+    # - python3 /foss_fim/data/usgs/acquire_and_preprocess_3dep_dems.py -e /data/inputs/wbd/HUC6_ESPG_5070/ -t /data/inputs/3dep_dems/10m_5070/ -r -j 20
     
     # Notes:
     #   - This is a very low use tool. So for now, this only can load 10m (1/3 arc second) and is using
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ac7567645..836d999b4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.10.5 - 2022-10-21 - [PR #720](https://github.com/NOAA-OWP/inundation-mapping/pull/720)
+
+Earlier versions of the acquire_and_preprocess_3dep_dems.py did not have any buffer added when downloading HUC6 DEMs. This resulted in 1 pixel nodata gaps in the final REM outputs in some cases at HUC8 sharing a HUC6 border. Adding the param of cblend 6 to the gdalwarp command meant put a 6 extra pixels all around perimeter. Testing showed that 6 pixels was plenty sufficient as the gaps were never more than 1 pixel on borders of no-data.
+
+### Changes
+
+- `data`
+    - `usgs`
+        - `acquire_and_preprocess_3dep_dems.py`: Added the `cblend 6` param to the gdalwarp call for when the dem is downloaded from USGS.
+    - `create_vrt_file.py`:  Added sample usage comment.
+ - `src`
+     - `gms`
+         `run_by_unit.sh`: Added a comment about gdal as it relates to run_by_unit.
+
+Note: the new replacement inputs/3dep_dems/10m_5070/ files can / will be copied before PR approval as the true fix was replacment DEM's. There is zero risk of overwriting prior to code merge.
+
+<br/><br/>
+
 ## v4.0.10.4 - 2022-10-27 - [PR #727](https://github.com/NOAA-OWP/inundation-mapping/pull/727)
 
 Creates a single crosswalk table containing HUC (huc8), BranchID, HydroID, feature_id (and optionally LakeID) from branch-level hydroTables.csv files.
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index b7fba718a..f78cb0d3c 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -131,6 +131,7 @@ mkdir -p $outputCurrentBranchDataDir
 
 ## CLIP RASTERS
 echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"$stopDiv
+# Note: don't need to use gdalwarp -cblend as we are using a buffered wbd
 date -u
 Tstart
 [ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \

From 628474d54855d6a7a6a6c3efa314b5e62dc6ccfc Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 31 Oct 2022 17:08:22 -0400
Subject: [PATCH 174/359] v4.0.11.0 Mask levee-protected areas from Relative
 Elevation Model (#690)

---
 config/deny_gms_branch_zero.lst              |   1 +
 config/deny_gms_branches_dev.lst             |   1 +
 config/deny_gms_branches_prod.lst            |   1 +
 config/deny_gms_unit_prod.lst                |   1 +
 config/params_template.env                   |   3 +
 docs/CHANGELOG.md                            |  23 +++
 gms_run_branch.sh                            |   4 +-
 gms_run_unit.sh                              |   7 +-
 src/clip_vectors_to_wbd.py                   | 167 +++++++++++--------
 src/gms/delineate_hydros_and_produce_HAND.sh |  36 +++-
 src/gms/rasterize_by_order.py                |  53 ++++++
 src/gms/remove_error_branches.py             |   0
 src/gms/run_by_branch.sh                     |   1 +
 src/gms/run_by_unit.sh                       |  46 ++++-
 src/gms/stream_branches.py                   |   4 +-
 src/split_flows.py                           |   2 -
 tools/eval_alt_catfim.py                     |   2 +-
 tools/run_test_case.py                       |   2 +-
 unit_tests/README.md                         |   2 +-
 unit_tests/clip_vectors_to_wbd_params.json   |  32 ++--
 unit_tests/clip_vectors_to_wbd_unittests.py  |  42 ++---
 21 files changed, 304 insertions(+), 126 deletions(-)
 mode change 100755 => 100644 src/clip_vectors_to_wbd.py
 create mode 100755 src/gms/rasterize_by_order.py
 mode change 100644 => 100755 src/gms/remove_error_branches.py

diff --git a/config/deny_gms_branch_zero.lst b/config/deny_gms_branch_zero.lst
index a75f8c5ef..5b6543be1 100644
--- a/config/deny_gms_branch_zero.lst
+++ b/config/deny_gms_branch_zero.lst
@@ -42,6 +42,7 @@ headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
 #LandSea_subset_{}.tif
+LeveeProtectedAreas_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
diff --git a/config/deny_gms_branches_dev.lst b/config/deny_gms_branches_dev.lst
index c9009488a..01f9084a4 100644
--- a/config/deny_gms_branches_dev.lst
+++ b/config/deny_gms_branches_dev.lst
@@ -41,6 +41,7 @@ gw_catchments_reaches_{}.tif
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
+LeveeProtectedAreas_subset_{}.tif
 LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nld_subset_levees_{}.tif
diff --git a/config/deny_gms_branches_prod.lst b/config/deny_gms_branches_prod.lst
index a5d1ecbba..ca8b460e5 100644
--- a/config/deny_gms_branches_prod.lst
+++ b/config/deny_gms_branches_prod.lst
@@ -41,6 +41,7 @@ gw_catchments_reaches_{}.tif
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
+LeveeProtectedAreas_subset_{}.tif
 LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nld_subset_levees_{}.tif
diff --git a/config/deny_gms_unit_prod.lst b/config/deny_gms_unit_prod.lst
index 6c5b5d9a1..faeb3b092 100644
--- a/config/deny_gms_unit_prod.lst
+++ b/config/deny_gms_unit_prod.lst
@@ -4,6 +4,7 @@ NHDPlusBurnLineEvent_subset.gpkg
 #branch_id.lst
 #branch_polygons.gpkg
 #LandSea_subset.gpkg
+#LeveeProtectedAreas_subset.gpkg
 nhd_headwater_points_subset.gpkg
 #nld_subset_levees.gpkg
 #nwm_catchments_proj_subset.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index 4a8a5daaf..d9ff518cc 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -26,6 +26,9 @@ export branch_buffer_distance_meters=7000
 export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.
 export branch_zero_id="0"
 
+#### mask levee-protected areas from relative elevation model
+export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from relative elevation model
+
 #### bathy SRC estimation parameters ####
 export bathy_src_toggle=True # Toggle to run BARC routine (True=on; False=off)
 export bankfull_input_table="data/inputs/bathymetry/nwm_flow_bieger_qreg.csv" # input file location with feature_id and channel geometry attributes
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 836d999b4..1d0fee582 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,29 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.11.0 - 2022-09-21 - [PR #690](https://github.com/NOAA-OWP/inundation-mapping/pull/690)
+
+Masks levee-protected areas from Relative Elevation Model if branch 0 or if branch stream order exceeds a threshold.
+
+### Additions
+
+- `src/gms/`
+   - `delineate_hydros_and_produce_HAND.sh`
+      - Reprojects and creates HUC-level raster of levee-protected areas from polygon layer
+      - Uses that raster to mask/remove those areas from the Relative Elevation Model
+   - `rasterize_by_order.py`: Subsets levee-protected area branch-level raster if branch 0 or if order exceeds a threshold (default threshold: max order - 1)
+- `config/`
+   - `deny_gms_branches_default.lst`, and `deny_gms_branches_min.lst`: Added LeveeProtectedAreas_subset_{}.tif
+   - `params_template.env`: Adds mask_leveed_area_toggle
+
+### Changes
+
+- `src/gms/delineate_hydros_and_produce_HAND.sh`: Fixes a bug in ocean/Great Lakes masking
+- `tools/`
+    - `eval_alt_catfim.py` and `run_test_case.py`: Changes the levee mask to the updated inputs/nld_vectors/Levee_protected_areas.gpkg
+
+<br/><br/>
+
 ## v4.0.10.5 - 2022-10-21 - [PR #720](https://github.com/NOAA-OWP/inundation-mapping/pull/720)
 
 Earlier versions of the acquire_and_preprocess_3dep_dems.py did not have any buffer added when downloading HUC6 DEMs. This resulted in 1 pixel nodata gaps in the final REM outputs in some cases at HUC8 sharing a HUC6 border. Adding the param of cblend 6 to the gdalwarp command meant put a 6 extra pixels all around perimeter. Testing showed that 6 pixels was plenty sufficient as the gaps were never more than 1 pixel on borders of no-data.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 2c04583ea..b72705bad 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -152,14 +152,14 @@ if [ ! -d "$outputRunDataDir/logs/branch" ]; then
     mkdir -p $outputRunDataDir/logs/branch
 elif [ $overwrite -eq 1 ]; then
     # need to clean it out if we are overwriting
-    rm -rf $outputRunDataDir/logs/branch
+    rm -rdf $outputRunDataDir/logs/branch
     mkdir -p $outputRunDataDir/logs/branch
 fi
 
 if [ ! -d "$outputRunDataDir/branch_errors" ]; then
     mkdir -p "$outputRunDataDir/branch_errors"
 elif [ $overwrite -eq 1 ]; then
-    rm -rf $outputRunDataDir/branch_errors
+    rm -rdf $outputRunDataDir/branch_errors
     mkdir -p $outputRunDataDir/branch_errors
 fi
 
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index be302f5a0..285a4b05d 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -147,6 +147,7 @@ export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
 export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
 export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
 export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
+export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Levee_protected_areas.gpkg
 export deny_gms_unit_list=$deny_gms_unit_list
 export extent=GMS
 
@@ -164,9 +165,9 @@ if [ ! -d $outputRunDataDir ]; then
 fi
 
 # remove these directories on a new or overwrite run
-rm -rf $outputRunDataDir/logs
-rm -rf $outputRunDataDir/branch_errors
-rm -rf $outputRunDataDir/unit_errors
+rm -rdf $outputRunDataDir/logs
+rm -rdf $outputRunDataDir/branch_errors
+rm -rdf $outputRunDataDir/unit_errors
 
 # we need to clean out the all log files overwrite or not
 mkdir -p $outputRunDataDir/logs/unit
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
old mode 100755
new mode 100644
index 9684d377d..189136781
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -5,13 +5,36 @@
 import pandas as pd
 import geopandas as gpd
 import argparse
+import rasterio as rio
+
 from shapely.geometry import MultiPolygon,Polygon
 from utils.shared_functions import getDriver, mem_profile
-import rasterio as rio
 
 @mem_profile
-def subset_vector_layers(hucCode, nwm_streams_filename, nhd_streams_filename, nwm_lakes_filename, nld_lines_filename, nwm_catchments_filename, nhd_headwaters_filename, landsea_filename, wbd_filename, wbd_buffer_filename, subset_nhd_streams_filename, subset_nld_lines_filename, subset_nwm_lakes_filename, subset_nwm_catchments_filename, subset_nhd_headwaters_filename, subset_nwm_streams_filename, subset_landsea_filename, great_lakes_filename, wbd_buffer_distance, lake_buffer_distance, dem_filename):
-
+def subset_vector_layers(subset_nwm_lakes, 
+                         subset_nwm_streams,
+                         subset_nhd_streams,
+                         hucCode,
+                         subset_nhd_headwaters,
+                         wbd_buffer_filename,
+                         wbd_filename,
+                         dem_filename,
+                         nwm_lakes,
+                         nwm_catchments,
+                         subset_nwm_catchments,
+                         nld_lines,
+                         nhd_streams,
+                         landsea,
+                         nwm_streams,
+                         subset_landsea,
+                         nhd_headwaters,
+                         subset_nld_lines,
+                         great_lakes,
+                         lake_buffer_distance,
+                         wbd_buffer_distance,
+                         levee_protected_areas,
+                         subset_levee_protected_areas):
+        
     hucUnitLength = len(str(hucCode))
 
     with rio.open(dem_filename) as dem_raster:
@@ -30,7 +53,7 @@ def subset_vector_layers(hucCode, nwm_streams_filename, nhd_streams_filename, nw
 
     # projection = wbd_buffer.crs
 
-    great_lakes = gpd.read_file(great_lakes_filename, mask=wbd_buffer).reset_index(drop=True)
+    great_lakes = gpd.read_file(great_lakes, mask=wbd_buffer).reset_index(drop=True)
 
     if not great_lakes.empty:
         print("Masking Great Lakes for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
@@ -55,39 +78,48 @@ def subset_vector_layers(hucCode, nwm_streams_filename, nhd_streams_filename, nw
     del great_lakes
 
     # Clip ocean water polygon for future masking ocean areas (where applicable)
-    landsea = gpd.read_file(landsea_filename, mask=wbd_buffer)
+    landsea = gpd.read_file(landsea, mask=wbd_buffer)
     if not landsea.empty:
-        landsea.to_file(subset_landsea_filename, driver=getDriver(subset_landsea_filename), index=False)
+        landsea.to_file(subset_landsea, driver = getDriver(subset_landsea), index=False)
     del landsea
 
+    # Clip levee-protected areas polygons for future masking ocean areas (where applicable)
+    print("Subsetting Levee Protected Areas", flush=True)
+    levee_protected_areas = gpd.read_file(levee_protected_areas, mask=wbd_buffer)
+    if not levee_protected_areas.empty:
+        levee_protected_areas = levee_protected_areas.to_crs('ESRI:102039')
+        levee_protected_areas.to_file(subset_levee_protected_areas, driver = getDriver
+                                      (subset_levee_protected_areas), index=False)
+    del levee_protected_areas
+
     # Find intersecting lakes and writeout
-    print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength,hucCode),flush=True)
-    nwm_lakes = gpd.read_file(nwm_lakes_filename, mask=wbd_buffer)
+    print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    nwm_lakes = gpd.read_file(nwm_lakes, mask = wbd_buffer)
     nwm_lakes = nwm_lakes.loc[nwm_lakes.Shape_Area < 18990454000.0]
 
     if not nwm_lakes.empty:
         # Perform fill process to remove holes/islands in the NWM lake polygons
         nwm_lakes = nwm_lakes.explode()
-        nwm_lakes_fill_holes=MultiPolygon(Polygon(p.exterior) for p in nwm_lakes['geometry']) # remove donut hole geometries
+        nwm_lakes_fill_holes = MultiPolygon(Polygon(p.exterior) for p in nwm_lakes['geometry']) # remove donut hole geometries
         # Loop through the filled polygons and insert the new geometry
         for i in range(len(nwm_lakes_fill_holes)):
-            nwm_lakes.loc[i,'geometry'] = nwm_lakes_fill_holes[i]
-        nwm_lakes.to_file(subset_nwm_lakes_filename, driver=getDriver(subset_nwm_lakes_filename), index=False)
+            nwm_lakes.loc[i, 'geometry'] = nwm_lakes_fill_holes[i]
+        nwm_lakes.to_file(subset_nwm_lakes, driver = getDriver(subset_nwm_lakes), index=False)
     del nwm_lakes
 
     # Find intersecting levee lines
     print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
-    nld_lines = gpd.read_file(nld_lines_filename, mask=wbd_buffer)
+    nld_lines = gpd.read_file(nld_lines, mask = wbd_buffer)
     if not nld_lines.empty:
-        nld_lines.to_file(subset_nld_lines_filename, driver=getDriver(subset_nld_lines_filename), index=False)
+        nld_lines.to_file(subset_nld_lines, driver = getDriver(subset_nld_lines), index=False)
     del nld_lines
 
     # Subset nhd headwaters
     print("Subsetting NHD Headwater Points for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
-    nhd_headwaters = gpd.read_file(nhd_headwaters_filename, mask=wbd_streams_buffer)
+    nhd_headwaters = gpd.read_file(nhd_headwaters, mask=wbd_streams_buffer)
 
     if len(nhd_headwaters) > 0:
-        nhd_headwaters.to_file(subset_nhd_headwaters_filename, driver=getDriver(subset_nhd_headwaters_filename), index=False)
+        nhd_headwaters.to_file(subset_nhd_headwaters, driver=getDriver(subset_nhd_headwaters), index=False)
     else:
         print ("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
@@ -95,12 +127,13 @@ def subset_vector_layers(hucCode, nwm_streams_filename, nhd_streams_filename, nw
 
     # Subset nhd streams
     print("Querying NHD Streams for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
-    nhd_streams = gpd.read_file(nhd_streams_filename, mask=wbd_streams_buffer)
+    nhd_streams = gpd.read_file(nhd_streams, mask=wbd_streams_buffer)
 
     if len(nhd_streams) > 0:
         nhd_streams = gpd.clip(nhd_streams, wbd_streams_buffer)
 
-        nhd_streams.to_file(subset_nhd_streams_filename,driver=getDriver(subset_nhd_streams_filename),index=False)
+        nhd_streams.to_file(subset_nhd_streams, 
+                            driver = getDriver(subset_nhd_streams), index=False)
     else:
         print ("No NHD streams within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
@@ -108,28 +141,30 @@ def subset_vector_layers(hucCode, nwm_streams_filename, nhd_streams_filename, nw
 
     # Find intersecting nwm_catchments
     print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
-    nwm_catchments = gpd.read_file(nwm_catchments_filename, mask=wbd_buffer)
+    nwm_catchments = gpd.read_file(nwm_catchments, mask=wbd_buffer)
 
     if len(nwm_catchments) > 0:
-        nwm_catchments.to_file(subset_nwm_catchments_filename, driver=getDriver(subset_nwm_catchments_filename), index=False)
+        nwm_catchments.to_file(subset_nwm_catchments, driver=getDriver(subset_nwm_catchments), index=False)
     else:
         print ("No NHD catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_catchments
 
     # Subset nwm streams
+
     print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
 
-    nwm_streams = gpd.read_file(nwm_streams_filename, mask = wbd)
+    nwm_streams = gpd.read_file(nwm_streams, mask = wbd)
     nwm_streams = gpd.clip(nwm_streams, wbd)
 
      # NWM can have duplicate records, but appear to always be identical duplicates
     nwm_streams.drop_duplicates(subset="ID", keep="first", inplace=True)
 
+
     if len(nwm_streams) > 0:
         nwm_streams = gpd.clip(nwm_streams, wbd_streams_buffer)
 
-        nwm_streams.to_file(subset_nwm_streams_filename, driver=getDriver(subset_nwm_streams_filename), index=False)
+        nwm_streams.to_file(subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False)
     else:
         print ("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
@@ -138,51 +173,53 @@ def subset_vector_layers(hucCode, nwm_streams_filename, nhd_streams_filename, nw
 
 if __name__ == '__main__':
 
+    #print(sys.argv)
+
     parser = argparse.ArgumentParser(description='Subset vector layers')
-    parser.add_argument('-d','--hucCode', help='HUC boundary ID', required=True,type=str)
-    parser.add_argument('-w','--nwm-streams', help='NWM flowlines', required=True)
-    parser.add_argument('-s','--nhd-streams',help='NHDPlus HR burnline',required=True)
-    parser.add_argument('-l','--nwm-lakes', help='NWM Lakes', required=True)
-    parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path', required=True)
-    parser.add_argument('-g','--wbd', help='HUC boundary', required=True)
+    parser.add_argument('-a','--subset-nwm-lakes', help='NWM lake subset', 
+                        required=True)
+    parser.add_argument('-b','--subset-nwm-streams', help='NWM streams subset', 
+                        required=True)
+    parser.add_argument('-c','--subset-nhd-streams', help='NHD streams subset', 
+                        required=True)
+    parser.add_argument('-d','--hucCode', help='HUC boundary ID', required=True,
+                        type=str)
+    parser.add_argument('-e','--subset-nhd-headwaters', help='NHD headwaters subset', 
+                        required=True, default=None)
+    parser.add_argument('-f','--wbd_buffer_filename', help='Buffered HUC boundary', 
+                        required=True)
+    parser.add_argument('-g','--wbd-filename', help='HUC boundary', required=True)
     parser.add_argument('-i','--dem-filename', help='DEM filename', required=True)
-    parser.add_argument('-f','--wbd-buffer', help='Buffered HUC boundary', required=True)
-    parser.add_argument('-m','--nwm-catchments', help='NWM catchments', required=True)
-    parser.add_argument('-y','--nhd-headwaters', help='NHD headwaters', required=True)
-    parser.add_argument('-v','--landsea', help='LandSea - land boundary', required=True)
-    parser.add_argument('-c','--subset-nhd-streams', help='NHD streams subset', required=True)
-    parser.add_argument('-z','--subset-nld-lines', help='Subset of NLD levee vectors for HUC', required=True)
-    parser.add_argument('-a','--subset-lakes', help='NWM lake subset', required=True)
-    parser.add_argument('-n','--subset-catchments', help='NWM catchments subset', required=True)
-    parser.add_argument('-e','--subset-nhd-headwaters', help='NHD headwaters subset', required=True, default=None)
-    parser.add_argument('-b','--subset-nwm-streams', help='NWM streams subset', required=True)
-    parser.add_argument('-x','--subset-landsea', help='LandSea subset', required=True)
-    parser.add_argument('-gl','--great-lakes-filename', help='Great Lakes layer', required=True)
-    parser.add_argument('-wb','--wbd-buffer-distance', help='WBD Mask buffer distance', required=True, type=int)
-    parser.add_argument('-lb','--lake-buffer-distance', help='Great Lakes Mask buffer distance', required=True, type=int)
-
+    parser.add_argument('-l','--nwm-lakes', help='NWM Lakes', required=True)    
+    parser.add_argument('-m','--nwm-catchments', help='NWM catchments',
+                        required=True)	 
+    parser.add_argument('-n','--subset-nwm-catchments', help='NWM catchments subset', 
+                        required=True)
+    parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path',
+                        required=True)
+    parser.add_argument('-s','--nhd-streams', help='NHDPlus HR burnline',
+                        required=True)
+    parser.add_argument('-v','--landsea', help='LandSea - land boundary',
+                        required=True)	 
+    parser.add_argument('-w','--nwm-streams', help='NWM flowlines',
+                        required=True)
+    parser.add_argument('-x','--subset-landsea', help='LandSea subset', 
+                        required=True)
+    parser.add_argument('-y','--nhd-headwaters', help='NHD headwaters',
+                        required=True)	 
+    parser.add_argument('-z','--subset-nld-lines', help='Subset of NLD levee vectors for HUC',
+                        required=True)
+    parser.add_argument('-gl','--great-lakes', help='Great Lakes layer', 
+                        required=True)
+    parser.add_argument('-lb','--lake-buffer-distance', help='Great Lakes Mask buffer distance',
+                        required=True, type=int)
+    parser.add_argument('-wb','--wbd-buffer-distance', help='WBD Mask buffer distance', 
+                        required=True, type=int)
+    parser.add_argument('-lpf','--levee-protected-areas', 
+                        help='Levee-protected areas filename', required=True)    
+    parser.add_argument('-lps','--subset-levee-protected-areas', 
+                        help='Levee-protected areas subset', required=True)
+    
     args = vars(parser.parse_args())
 
-    hucCode = args['hucCode']
-    nwm_streams_filename = args['nwm_streams']
-    nhd_streams_filename = args['nhd_streams']
-    nwm_lakes_filename = args['nwm_lakes']
-    nld_lines_filename = args['nld_lines']
-    wbd_filename = args['wbd']
-    wbd_buffer_filename = args['wbd_buffer']
-    nwm_catchments_filename = args['nwm_catchments']
-    nhd_headwaters_filename = args['nhd_headwaters']
-    landsea_filename = args['landsea']
-    subset_nhd_streams_filename = args['subset_nhd_streams']
-    subset_nld_lines_filename = args['subset_nld_lines']
-    subset_nwm_lakes_filename = args['subset_lakes']
-    subset_nwm_catchments_filename = args['subset_catchments']
-    subset_nhd_headwaters_filename = args['subset_nhd_headwaters']
-    subset_nwm_streams_filename = args['subset_nwm_streams']
-    subset_landsea_filename = args['subset_landsea']
-    great_lakes_filename = args['great_lakes_filename']
-    wbd_buffer_distance = args['wbd_buffer_distance']
-    lake_buffer_distance  = args['lake_buffer_distance']
-    dem_filename = args['dem_filename']
-
-    subset_vector_layers(hucCode,nwm_streams_filename,nhd_streams_filename,nwm_lakes_filename,nld_lines_filename,nwm_catchments_filename,nhd_headwaters_filename,landsea_filename,wbd_filename,wbd_buffer_filename,subset_nhd_streams_filename,subset_nld_lines_filename,subset_nwm_lakes_filename,subset_nwm_catchments_filename, subset_nhd_headwaters_filename, subset_nwm_streams_filename, subset_landsea_filename, great_lakes_filename,wbd_buffer_distance,lake_buffer_distance, dem_filename)
+    subset_vector_layers(**args)
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
index 96e86b1a3..bb2939e03 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -108,11 +108,23 @@ if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
     echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"$stopDiv
     date -u
     Tstart
-
     gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
     Tcount
 fi
 
+## RASTERIZE LEVEE-PROTECTED AREAS POLYGON (IF APPLICABLE) ##
+if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
+    echo -e $startDiv"Rasterize levee-protected areas polygons $hucNumber $current_branch_id"$stopDiv
+    date -u
+    Tstart
+    if [ $current_branch_id = $branch_zero_id ]; then 
+        gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LeveeProtectedAreas_subset.gpkg $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif
+    else
+        $srcDir/gms/rasterize_by_order.py -c $ncols -r $nrows -n $ndv --x-min $xmin --y-min $ymin --x-max $xmax --y-max $ymax -v $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -f $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif -s $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -i $current_branch_id
+    fi
+    Tcount
+fi
+
 ## POLYGONIZE REACH WATERSHEDS ##
 echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"$stopDiv
 date -u
@@ -148,12 +160,22 @@ $srcDir/make_stages_and_catchlist.py -f $outputCurrentBranchDataDir/demDerived_r
 Tcount
 
 ## MASK REM RASTER TO REMOVE OCEAN AREAS ##
-echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"$stopDiv
-date -u
-Tstart
-[ -f $outputCurrentBranchDataDir/LandSea_subset.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LandSea_subset_$current_node_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
-Tcount
+if  [ -f $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
+    echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"$stopDiv
+    date -u
+    Tstart
+    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+    Tcount
+fi
+
+## MASK REM RASTER TO REMOVE LEVEE-PROTECTED AREAS ##
+if [ -f $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif ]; then
+    echo -e $startDiv"Additional masking to REM raster to remove levee-protected areas $hucNumber $current_branch_id"$stopDiv
+    date -u
+    Tstart
+    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+    Tcount
+fi
 
 ## HYDRAULIC PROPERTIES ##
 echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"$stopDiv
diff --git a/src/gms/rasterize_by_order.py b/src/gms/rasterize_by_order.py
new file mode 100755
index 000000000..8e5072d76
--- /dev/null
+++ b/src/gms/rasterize_by_order.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import argparse
+from osgeo import gdal
+import geopandas as gpd
+
+def rasterize_by_order(vector_filename, raster_filename, x_min, y_min, x_max, y_max, columns, rows, nodata_value, stream_layer, branch_id_attribute, branch_id, order_attribute):
+    """
+    Rasterizes polygon layers exceeding a stream order threshold
+    """
+
+    streams_df = gpd.read_file(stream_layer, ignore_geometry=True)
+
+    # Rasterize if branch is at least the minimum order
+    if streams_df.loc[streams_df[branch_id_attribute].astype(int)==branch_id, order_attribute].max() >= streams_df[order_attribute].max() - 1:
+
+        # Open the data source and read in the extent
+        source_ds = gdal.OpenEx(vector_filename)
+
+        gdal.Rasterize(raster_filename,
+                        source_ds,
+                        format='GTIFF',
+                        outputType=gdal.GDT_Int32,
+                        creationOptions=["COMPRESS=LZW", "BIGTIFF=YES", "TILES=YES"],
+                        noData=nodata_value,
+                        initValues=1,
+                        width=columns,
+                        height=rows,
+                        allTouched=True,
+                        burnValues=nodata_value,
+                        outputBounds=[x_min, y_min, x_max, y_max])
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Rasterize a polygon layer')
+    parser.add_argument('-v', '--vector-filename', help='Input vector filename', required=True)
+    parser.add_argument('-f', '--raster-filename', help='Output raster filename', required=True)
+    parser.add_argument('--x-min', help='Minimum x coordinate', required=True)
+    parser.add_argument('--y-min', help='Minimun y coordinate', required=True)
+    parser.add_argument('--x-max', help='Maximum x coordinate', required=True)
+    parser.add_argument('--y-max', help='Maximum y coordinate', required=True)
+    parser.add_argument('-c', '--columns', help='Output raster columns', required=True)
+    parser.add_argument('-r', '--rows', help='Output raster rows', required=True)
+    parser.add_argument('-n', '--nodata-value', help='NoData value', required=True)
+    parser.add_argument('-s', '--stream-layer', help='Stream layer filename', required=True)
+    parser.add_argument('-b', '--branch-id-attribute', help='Branch ID attribute name', required=False, default='levpa_id')
+    parser.add_argument('-i', '--branch-id', help='Branch ID', type=int, required='True')
+    parser.add_argument('-a', '--order-attribute', help='Stream order attribute name', required=False, default='order_')
+
+    args = vars(parser.parse_args())
+
+    rasterize_by_order(**args)
diff --git a/src/gms/remove_error_branches.py b/src/gms/remove_error_branches.py
old mode 100644
new mode 100755
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index 76e133098..5c547cb4c 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -64,6 +64,7 @@ echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
 read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
+Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"$stopDiv
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index f78cb0d3c..9b6124946 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -6,12 +6,14 @@ T_total_start
 ## SET OUTPUT DIRECTORY FOR UNIT ##
 hucNumber="$1"
 
-# Even though check_input_hucs at gms_run_unit validate that all values
-# are numbers, sometimes the huc list can come in as windows incoded and not
-# unix encoded. It can get missed but tee and time can parse it wrong.
-# so, we will strip a slash of the end if it exists, the re-validat that the
-# value is a number.  (Note: doesn't seem to work all of the time for encoding
-# issues (??))
+: '
+  Even though check_input_hucs at gms_run_unit validate that all values
+  are numbers, sometimes the huc list can come in as windows incoded and not
+  unix encoded. It can get missed but tee and time can parse it wrong.
+  so, we will strip a slash of the end if it exists, the re-validat that the
+  value is a number.  (Note: doesn''t seem to work all of the time for encoding
+  issues (??))
+'
 re='^[0-9]+$'
 if ! [[ $hucNumber =~ $re ]] ; then
    echo "Error: hucNumber is not a number" >&2; exit 1
@@ -71,8 +73,38 @@ Tcount
 echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM
+
+cmd_args=" -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg"
+cmd_args+=" -b $outputHucDataDir/nwm_subset_streams.gpkg"
+cmd_args+=" -c $outputHucDataDir/NHPlusBurnLineEvent_subset.gpkg"
+cmd_args+=" -d $hucNumber"
+cmd_args+=" -e $outputHucDataDir/nhd_headwater_points_subset.gpkg"
+cmd_args+=" -f $outputHucDataDir/wbd_buffered.gpkg"
+cmd_args+=" -g $outputHucDataDir/wbd.gpkg"
+cmd_args+=" -i $input_DEM"
+cmd_args+=" -l $input_nwm_lakes"
+cmd_args+=" -m $input_nwm_catchments"
+cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"
+cmd_args+=" -r $input_NLD"
+cmd_args+=" -s $input_nhd_flowlines"
+cmd_args+=" -v $input_LANDSEA"
+cmd_args+=" -w $input_nwm_flows"
+cmd_args+=" -x $outputHucDataDir/LandSea_subset.gpkg"
+cmd_args+=" -y $input_nhd_headwaters"
+cmd_args+=" -z $outputHucDataDir/nld_subset_levees.gpkg"
+cmd_args+=" -gl $input_GL_boundaries"
+cmd_args+=" -lb $lakes_buffer_dist_meters"
+cmd_args+=" -wb $wbd_buffer"
+cmd_args+=" -lpf $input_nld_levee_protected_areas"
+cmd_args+=" -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg"
+
 Tcount
+#echo "$cmd_args"
+python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
+
+: '
+python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM
+'
 
 ## Clip WBD8 ##
 echo -e $startDiv"Clip WBD8"$stopDiv
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index 90d928631..9fae162a7 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -618,7 +618,7 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
             progress.update(1)
 
             # get current reach stream order and branch id
-            #current_reach_comparison_value = self.at[current_reach_id,comparison_attributes]
+            # current_reach_comparison_value = self.at[current_reach_id,comparison_attributes]
             current_reach_branch_id = self.at[current_reach_id,branch_id_attribute]
 
             # get upstream ids
@@ -644,7 +644,7 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
 
                     # find 
                     upstream_reaches_compare_values = self.loc[not_visited_upstream_ids,comparison_attributes]
-                    matching_value = comparison_function(upstream_reaches_compare_values)
+                    # matching_value = comparison_function(upstream_reaches_compare_values)
 
                     #==================================================================================
                     # If the two stream orders aren't the same, then follow the highest order, otherwise use arbolate sum
diff --git a/src/split_flows.py b/src/split_flows.py
index 7d4555d85..daf4e757b 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -43,8 +43,6 @@ def split_flows(max_length,
                 nwm_streams_filename,
                 drop_stream_orders=False):
 
-    wbd = gpd.read_file(wbd8_clp_filename)
-
     toMetersConversion = 1e-3
 
     print('Loading data ...')
diff --git a/tools/eval_alt_catfim.py b/tools/eval_alt_catfim.py
index ffe86cf0e..6ddf9165d 100644
--- a/tools/eval_alt_catfim.py
+++ b/tools/eval_alt_catfim.py
@@ -195,7 +195,7 @@ def process_alt_comparison(args):
                                 mask_values=None
                                 stats_modes_list=['total_area']
                                 test_id=''
-                                mask_dict={'levees': {'path': '/data/test_cases/other/zones/leveed_areas_conus.shp', 'buffer': None, 'operation': 'exclude'}, 
+                                mask_dict={'levees': {'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg', 'buffer': None, 'operation': 'exclude'}, 
                                             'waterbodies': {'path': '/data/test_cases/other/zones/nwm_v2_reservoirs.shp', 'buffer': None, 'operation': 'exclude'},
                                             site: {'path': '/data/test_cases/{benchmark_type}_test_cases/validation_data_{benchmark_type}/{huc}/{site}/{site}_domain.shp'.format(benchmark_type=benchmark_type, site=site, category=category, huc=huc), 'buffer': None, 'operation': 'include'}}
                                 
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index abba132b9..a850aaa3b 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -105,7 +105,7 @@ def __init__(self, test_id, version, archive=True):
         # Create list of shapefile paths to use as exclusion areas.
         zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
         self.mask_dict = {'levees':
-                        {'path': os.path.join(zones_dir, 'leveed_areas_conus.shp'),
+                        {'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg',
                         'buffer': None,
                         'operation': 'exclude'
                         },
diff --git a/unit_tests/README.md b/unit_tests/README.md
index 4f7a1955a..037a22f7a 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -37,7 +37,7 @@ Notice a modified branch "deny_gms_branch_unittests.lst"  (special for unittests
 
 Here are the params and args you need if you need to re-run unit and branch
 
-gms_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -c /foss_fim/config/params_template.env -j 1 -bd /foss_fim/config/deny_gms_branch_unittests.lst -ud None -o -s
+gms_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -bd /foss_fim/config/deny_gms_branch_unittests.lst -ud None -j 1 -o
 
 **NOTICE: the deny file used for gms_run_branch... its a special one for unittests `deny_gms_branch_unittests.lst`.
 
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 388ce1094..be56c91ee 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -1,26 +1,28 @@
 {
 	"valid_data":
 	{
+		"subset_nwm_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
+		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg", 		
+		"subset_nhd_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/NHDPlusBurnLineEvent_subset.gpkg",
 		"hucCode": "05030104",
-		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
-		"nhd_streams": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg", 
+		"subset_nhd_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg", 
+		"wbd_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
+		"wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
+		"dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
 		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
-		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
-		"wbd": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
-		"wbd_buffer": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
 		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", 
-		"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
+		"subset_nwm_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
+		"nhd_streams": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg", 
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
-		"subset_nhd_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/NHDPlusBurnLineEvent_subset.gpkg",
-		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg", 
-		"subset_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
-		"subset_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
-		"subset_nhd_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg", 
-		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg", 
+		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
 		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg", 
-		"extent": "GMS",
-		"great_lakes_filename": "/data/inputs/landsea/gl_water_polygons.gpkg", 
+		"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
+		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg", 
+		"great_lakes": "/data/inputs/landsea/gl_water_polygons.gpkg", 
+		"lake_buffer_distance": 20,		
 		"wbd_buffer_distance": 5000,
-		"lake_buffer_distance": 20
+		"levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
+		"subset_levee_protected_areas": "/data/outputs/rob_test_2/05030104/LeveeProtectedAreas_subset.gpkg"
 	}
 }
\ No newline at end of file
diff --git a/unit_tests/clip_vectors_to_wbd_unittests.py b/unit_tests/clip_vectors_to_wbd_unittests.py
index b6cd35f9c..adb3cbe46 100644
--- a/unit_tests/clip_vectors_to_wbd_unittests.py
+++ b/unit_tests/clip_vectors_to_wbd_unittests.py
@@ -48,27 +48,29 @@ def test_subset_vector_layers_success(self):
         # Later we need to check in this function that files were being created on the file system
         
         # for now we are happy if no exceptions are thrown.
-        src.subset_vector_layers(hucCode = params["hucCode"],
-                                nwm_streams_filename = params["nwm_streams"],
-                                nhd_streams_filename = params["nhd_streams"],
-                                nwm_lakes_filename = params["nwm_lakes"],
-                                nld_lines_filename = params["nld_lines"],
-                                nwm_catchments_filename = params["nwm_catchments"],
-                                nhd_headwaters_filename = params["nhd_headwaters"],
-                                landsea_filename = params["landsea"],
-                                wbd_filename = params["wbd"],
-                                wbd_buffer_filename = params["wbd_buffer"],
-                                subset_nhd_streams_filename = params["subset_nhd_streams"],
-                                subset_nld_lines_filename = params["subset_nld_lines"],
-                                subset_nwm_lakes_filename = params["subset_lakes"],
-                                subset_nwm_catchments_filename = params["subset_catchments"],
-                                subset_nhd_headwaters_filename = params["subset_nhd_headwaters"],
-                                subset_nwm_streams_filename = params["subset_nwm_streams"],
-                                subset_landsea_filename = params["subset_landsea"],
-                                extent = params["extent"],
-                                great_lakes_filename = params["great_lakes_filename"],
+        src.subset_vector_layers(subset_nwm_lakes = params["subset_nwm_lakes"],
+                                subset_nwm_streams = params["subset_nwm_streams"],
+                                subset_nhd_streams = params["subset_nhd_streams"],
+                                hucCode = params["hucCode"],
+                                subset_nhd_headwaters = params["subset_nhd_headwaters"],
+                                wbd_buffer_filename = params["wbd_buffer_filename"],
+                                wbd_filename = params["wbd_filename"],
+                                dem_filename = params["dem_filename"],
+                                nwm_lakes = params["nwm_lakes"],
+                                nwm_catchments = params["nwm_catchments"],
+                                subset_nwm_catchments = params["subset_nwm_catchments"],
+                                nld_lines = params["nld_lines"],
+                                nhd_streams = params["nhd_streams"],
+                                landsea = params["landsea"],
+                                nwm_streams = params["nwm_streams"],
+                                subset_landsea = params["subset_landsea"],
+                                nhd_headwaters = params["nhd_headwaters"],
+                                subset_nld_lines = params["subset_nld_lines"],
+                                great_lakes = params["great_lakes"],                                
+                                lake_buffer_distance = params["lake_buffer_distance"],
                                 wbd_buffer_distance = params["wbd_buffer_distance"],
-                                lake_buffer_distance = params["lake_buffer_distance"])
+                                levee_protected_areas = params["levee_protected_areas"],
+                                subset_levee_protected_areas = params["subset_levee_protected_areas"])
        
 
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")

From ecb592a83c77aacd1932d6ae219130d93cf59269 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 3 Nov 2022 15:47:53 -0500
Subject: [PATCH 175/359] v4.0.11.1 Upgrade jupyter core (#732)

---
 Pipfile           |    1 +
 Pipfile.lock      | 1116 +++++++++++++++++++++++----------------------
 docs/CHANGELOG.md |   11 +
 3 files changed, 575 insertions(+), 553 deletions(-)

diff --git a/Pipfile b/Pipfile
index ce0fc83ad..08a1c3150 100755
--- a/Pipfile
+++ b/Pipfile
@@ -33,6 +33,7 @@ boto3 = "*"
 jupyter = "*"
 jupyterlab = "*"
 ipympl = "*"
+jupyter-core = "~=4.11.2"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 2e5578067..6d2621fbe 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f48eed8e1c745d0ea7de9caa2c2fb3264541a2ef93f9408e0156c4544359bc55"
+            "sha256": "efc2e94e622e9f6160ed7ddc395420f566ca21527936677b5ad0184c1d26bb1a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -25,11 +25,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b",
-                "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"
+                "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421",
+                "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.6.1"
+            "version": "==3.6.2"
         },
         "argon2-cffi": {
             "hashes": [
@@ -68,10 +68,10 @@
         },
         "asttokens": {
             "hashes": [
-                "sha256:c61e16246ecfb2cde2958406b4c8ebc043c9e6d73aaa83c941673b35e5d3a76b",
-                "sha256:e3305297c744ae53ffa032c45dc347286165e4ffce6875dc662b205db0623d86"
+                "sha256:1b28ed85e254b724439afc783d4bee767f780b936c3fe8b3275332f42cf5f561",
+                "sha256:4aa76401a151c8cc572d906aad7aea2a841780834a19d780f4321c0fe1b54635"
             ],
-            "version": "==2.0.8"
+            "version": "==2.1.0"
         },
         "attrs": {
             "hashes": [
@@ -114,27 +114,27 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:09fb94008bd1f2566ef79ec7d8dd7a233b5da67f3b2536e30432eb8e88022d4d",
-                "sha256:9fab8acec76008797be4671d514f39f1163ba044c0a0bce8d5afe8b52cbd5550"
+                "sha256:17ce7b6b702f9e844a33ce3ce9612f09d9d73eb7d34aaeffc77353ec036a9615",
+                "sha256:aec7db139429fe0f3fbe723170461192b0483b0070114a4b56351e374e0f294d"
             ],
             "index": "pypi",
-            "version": "==1.24.56"
+            "version": "==1.25.5"
         },
         "botocore": {
             "hashes": [
-                "sha256:81101abab2013992a84019739d85a6fa8ec6f1f42fb09ad07e0bf4c8f0efce60",
-                "sha256:81df81b1a1c5608b3a7aca6a837acb27552c3ecc92584a58b171c60754b1d4eb"
+                "sha256:5eb98c85e80fab6640342377ce947c86a003bbaf8427c2e7a03c8da0bdbb796c",
+                "sha256:f322d7b62163219ffeb787a116d318273dfb7243c3b49d95f5bfff8daa1df4e0"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.27.56"
+            "version": "==1.28.5"
         },
         "certifi": {
             "hashes": [
-                "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
-                "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"
+                "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14",
+                "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2022.6.15"
+            "version": "==2022.9.24"
         },
         "cffi": {
             "hashes": [
@@ -207,36 +207,30 @@
         },
         "cftime": {
             "hashes": [
-                "sha256:00aef598f14faa77e48518728bd0cead585e45da89be5f7eae7be8f41c10b401",
-                "sha256:01d40598c8e551202911f5126cdff52b816fdc8316f8a88f798b81fc1a8b55bf",
-                "sha256:041c7f9f620ac5465fbaf4eb6570e4f286535e55e56ac4a34cfcfb3e853d4093",
-                "sha256:0ee1d1f9ff4e9f54700dd02d8d707a383bd9dad5c5de9c566372caba071982d5",
-                "sha256:39bc35a0c05db5f24aeef384af2119a9f0820791617e198d06bd9dba50218fa9",
-                "sha256:3c8389df99216bf184141f11b84ba00b414177c92618bc4ee1297376163a20d5",
-                "sha256:3d6af9e713b911baed904fbc4ebeec7ca44f5970971f1c5e86a34cba82fe98e9",
-                "sha256:511215f45ed7cc79ead84020e88e1fc476b8aba71b47d2fcdef8e65242406927",
-                "sha256:54c2533f6069cda87bd641fb6df28542238679896a5231006d0de00afbdf3edf",
-                "sha256:646991eb76a9906a023ec982714a04d8b4e06346ea5836ea8c30350dccca079c",
-                "sha256:67a2a9a6a7bb799b6d171239ea809e70b572c37eeda6053b04401088e6e83bed",
-                "sha256:6842d4d37377e1aca392cee4feb0e9f05d36264db08ca2cccd2c3d168487d981",
-                "sha256:688b2c54e5cb17e5cb55f799df093c5d3d4974feee8e1e7224c9ac40de2580e2",
-                "sha256:7c0724249912efa3708c265181e880ae80d90c68849aede5ce26ae7649c9e87d",
-                "sha256:8b1d2becda38027398894836c979467d6649e090a050db8c353461d9537273bb",
-                "sha256:8dfe2cf7b75a6541a54e0f25665bed5240f4c0cf5d233f81d53b4b5c55dd1c6d",
-                "sha256:93e32bee6cf67488661a7a2a73ede4c089902ef40f6d20312474351c52aa10c7",
-                "sha256:b0c192eca13faf932bbc30ad6b19878c816fe228fb84803e30873f2a5e7bd500",
-                "sha256:b46f9139aa23d6ebff01dff6970dc5476293287bc6ba473ce64542ff20607bfe",
-                "sha256:c130f78e5d7eeb8c625192848075a8a8bb92595d07ad5b4cb95b007011d78e5a",
-                "sha256:cdf27919b88a089c72266992ce31de80ed52b334a892e1ce0853ddf82d2a636b",
-                "sha256:db836dd53a56fc177d4e1e792fb94306592b8e21dc0c2d671c7a10058081e341",
-                "sha256:e57381585d421b2fdbb9369d7cc53f8442152d3c7b415bda210cb575366d7480",
-                "sha256:f3eb51592e03c9b5d4328eae7a40c7c063725b071e037ddf813a5d68f6a5cb6e",
-                "sha256:f5eb773cd21e2a8a492834cf0afd586a41b757807f53b845257a0a7419c17538",
-                "sha256:fb8c2e40b74708e20e3ab13e6273f33143037b7d16349c604e1091d2e8175275",
-                "sha256:fdb23b1a75e7efb86858c8ff6ab6744558009d8127e5543df9df543036504d21"
+                "sha256:055d5d60a756c6c1857cf84d77655bb707057bb6c4a4fbb104a550e76c40aad9",
+                "sha256:07fdef2f75a0f0952b0376fa4cd08ef8a1dad3b963976ac07517811d434936b7",
+                "sha256:0955e1f3e1c09a9e0296b50f135ff9719cb2466f81c8ad4a10ef06fa394de984",
+                "sha256:29c18601abea0fd160fbe423e05c7a56fe1d38dd250a6b010de499a132d3fe18",
+                "sha256:2abdac6ca5b8b6102f319122546739dfc42406b816c16f2a98a8f0cd406d3bf0",
+                "sha256:2ba7909a0cd4adcb16797d8d6ab2767e7ddb980b2bf9dbabfc71b3bdd94f072b",
+                "sha256:3042048324b4d6a1066c978ec78101effdd84320e8862bfdbf8122d7ad7588ec",
+                "sha256:455cec3627e6ca8694b0d9201da6581eb4381b58389f1fbcb51a14fa0e2b3d94",
+                "sha256:56d0242fc4990584b265622622b25bb262a178097711d2d95e53ef52a9d23e7e",
+                "sha256:8614c00fb8a5046de304fdd86dbd224f99408185d7b245ac6628d0276596e6d2",
+                "sha256:86fe550b94525c327578a90b2e13418ca5ba6c636d5efe3edec310e631757eea",
+                "sha256:892d5dc38f8b998c83a2a01f131e63896d020586de473e1878f9e85acc70ad44",
+                "sha256:8d49d69c64cee2c175478eed84c3a57fce083da4ceebce16440f72be561a8489",
+                "sha256:93f00f454329c1f2588ebca2650e8edf7607d6189dbdcc81b5f3be2080155cc4",
+                "sha256:acb294fdb80e33545ae54b4421df35c4e578708a5ffce1c00408b2294e70ecef",
+                "sha256:aedfb7a783d19d7a30cb41951310f3bfe98f9f21fffc723c8af08a11962b0b17",
+                "sha256:afb5b38b51b8bc02f1656a9f15c52b0b20a3999adbe1ab9ac57f926e0065b48a",
+                "sha256:b4d2a1920f0aad663f25700b30621ff64af373499e52b544da1148dd8c09409a",
+                "sha256:e83db2fdda900eb154a9f79dfb665ac6190781c61d2e18151996de5ee7ffd8a2",
+                "sha256:eb7f8cd0996640b83020133b5ef6b97fc9216c3129eaeeaca361abdff5d82166",
+                "sha256:ee70fa069802652cf534de1dd3fc590b7d22d4127447bf96ac9849abcdadadf1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.6.1"
+            "version": "==1.6.2"
         },
         "charset-normalizer": {
             "hashes": [
@@ -269,6 +263,81 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
+        "contourpy": {
+            "hashes": [
+                "sha256:0236875c5a0784215b49d00ebbe80c5b6b5d5244b3655a36dda88105334dea17",
+                "sha256:03d1b9c6b44a9e30d554654c72be89af94fab7510b4b9f62356c64c81cec8b7d",
+                "sha256:0537cc1195245bbe24f2913d1f9211b8f04eb203de9044630abd3664c6cc339c",
+                "sha256:06ca79e1efbbe2df795822df2fa173d1a2b38b6e0f047a0ec7903fbca1d1847e",
+                "sha256:08e8d09d96219ace6cb596506fb9b64ea5f270b2fb9121158b976d88871fcfd1",
+                "sha256:0b1e66346acfb17694d46175a0cea7d9036f12ed0c31dfe86f0f405eedde2bdd",
+                "sha256:0b97454ed5b1368b66ed414c754cba15b9750ce69938fc6153679787402e4cdf",
+                "sha256:0e4854cc02006ad6684ce092bdadab6f0912d131f91c2450ce6dbdea78ee3c0b",
+                "sha256:12a7dc8439544ed05c6553bf026d5e8fa7fad48d63958a95d61698df0e00092b",
+                "sha256:1b1ee48a130da4dd0eb8055bbab34abf3f6262957832fd575e0cab4979a15a41",
+                "sha256:1c0e1308307a75e07d1f1b5f0f56b5af84538a5e9027109a7bcf6cb47c434e72",
+                "sha256:1dedf4c64185a216c35eb488e6f433297c660321275734401760dafaeb0ad5c2",
+                "sha256:208bc904889c910d95aafcf7be9e677726df9ef71e216780170dbb7e37d118fa",
+                "sha256:211dfe2bd43bf5791d23afbe23a7952e8ac8b67591d24be3638cabb648b3a6eb",
+                "sha256:341330ed19074f956cb20877ad8d2ae50e458884bfa6a6df3ae28487cc76c768",
+                "sha256:344cb3badf6fc7316ad51835f56ac387bdf86c8e1b670904f18f437d70da4183",
+                "sha256:358f6364e4873f4d73360b35da30066f40387dd3c427a3e5432c6b28dd24a8fa",
+                "sha256:371f6570a81dfdddbb837ba432293a63b4babb942a9eb7aaa699997adfb53278",
+                "sha256:375d81366afd547b8558c4720337218345148bc2fcffa3a9870cab82b29667f2",
+                "sha256:3a1917d3941dd58732c449c810fa7ce46cc305ce9325a11261d740118b85e6f3",
+                "sha256:4081918147fc4c29fad328d5066cfc751da100a1098398742f9f364be63803fc",
+                "sha256:444fb776f58f4906d8d354eb6f6ce59d0a60f7b6a720da6c1ccb839db7c80eb9",
+                "sha256:46deb310a276cc5c1fd27958e358cce68b1e8a515fa5a574c670a504c3a3fe30",
+                "sha256:494efed2c761f0f37262815f9e3c4bb9917c5c69806abdee1d1cb6611a7174a0",
+                "sha256:50627bf76abb6ba291ad08db583161939c2c5fab38c38181b7833423ab9c7de3",
+                "sha256:5641927cc5ae66155d0c80195dc35726eae060e7defc18b7ab27600f39dd1fe7",
+                "sha256:5b117d29433fc8393b18a696d794961464e37afb34a6eeb8b2c37b5f4128a83e",
+                "sha256:613c665529899b5d9fade7e5d1760111a0b011231277a0d36c49f0d3d6914bd6",
+                "sha256:6e459ebb8bb5ee4c22c19cc000174f8059981971a33ce11e17dddf6aca97a142",
+                "sha256:6f56515e7c6fae4529b731f6c117752247bef9cdad2b12fc5ddf8ca6a50965a5",
+                "sha256:730c27978a0003b47b359935478b7d63fd8386dbb2dcd36c1e8de88cbfc1e9de",
+                "sha256:75a2e638042118118ab39d337da4c7908c1af74a8464cad59f19fbc5bbafec9b",
+                "sha256:78ced51807ccb2f45d4ea73aca339756d75d021069604c2fccd05390dc3c28eb",
+                "sha256:7ee394502026d68652c2824348a40bf50f31351a668977b51437131a90d777ea",
+                "sha256:8468b40528fa1e15181cccec4198623b55dcd58306f8815a793803f51f6c474a",
+                "sha256:84c593aeff7a0171f639da92cb86d24954bbb61f8a1b530f74eb750a14685832",
+                "sha256:913bac9d064cff033cf3719e855d4f1db9f1c179e0ecf3ba9fdef21c21c6a16a",
+                "sha256:9447c45df407d3ecb717d837af3b70cfef432138530712263730783b3d016512",
+                "sha256:9b0e7fe7f949fb719b206548e5cde2518ffb29936afa4303d8a1c4db43dcb675",
+                "sha256:9bc407a6af672da20da74823443707e38ece8b93a04009dca25856c2d9adadb1",
+                "sha256:9e8e686a6db92a46111a1ee0ee6f7fbfae4048f0019de207149f43ac1812cf95",
+                "sha256:9fc4e7973ed0e1fe689435842a6e6b330eb7ccc696080dda9a97b1a1b78e41db",
+                "sha256:a457ee72d9032e86730f62c5eeddf402e732fdf5ca8b13b41772aa8ae13a4563",
+                "sha256:a628bba09ba72e472bf7b31018b6281fd4cc903f0888049a3724afba13b6e0b8",
+                "sha256:a79d239fc22c3b8d9d3de492aa0c245533f4f4c7608e5749af866949c0f1b1b9",
+                "sha256:aa4674cf3fa2bd9c322982644967f01eed0c91bb890f624e0e0daf7a5c3383e9",
+                "sha256:acd2bd02f1a7adff3a1f33e431eb96ab6d7987b039d2946a9b39fe6fb16a1036",
+                "sha256:b3b1bd7577c530eaf9d2bc52d1a93fef50ac516a8b1062c3d1b9bcec9ebe329b",
+                "sha256:b48d94386f1994db7c70c76b5808c12e23ed7a4ee13693c2fc5ab109d60243c0",
+                "sha256:b64f747e92af7da3b85631a55d68c45a2d728b4036b03cdaba4bd94bcc85bd6f",
+                "sha256:b98c820608e2dca6442e786817f646d11057c09a23b68d2b3737e6dcb6e4a49b",
+                "sha256:c1baa49ab9fedbf19d40d93163b7d3e735d9cd8d5efe4cce9907902a6dad391f",
+                "sha256:c38c6536c2d71ca2f7e418acaf5bca30a3af7f2a2fa106083c7d738337848dbe",
+                "sha256:c78bfbc1a7bff053baf7e508449d2765964d67735c909b583204e3240a2aca45",
+                "sha256:cd2bc0c8f2e8de7dd89a7f1c10b8844e291bca17d359373203ef2e6100819edd",
+                "sha256:d2eff2af97ea0b61381828b1ad6cd249bbd41d280e53aea5cccd7b2b31b8225c",
+                "sha256:d8834c14b8c3dd849005e06703469db9bf96ba2d66a3f88ecc539c9a8982e0ee",
+                "sha256:d912f0154a20a80ea449daada904a7eb6941c83281a9fab95de50529bfc3a1da",
+                "sha256:da1ef35fd79be2926ba80fbb36327463e3656c02526e9b5b4c2b366588b74d9a",
+                "sha256:dbe6fe7a1166b1ddd7b6d887ea6fa8389d3f28b5ed3f73a8f40ece1fc5a3d340",
+                "sha256:dcd556c8fc37a342dd636d7eef150b1399f823a4462f8c968e11e1ebeabee769",
+                "sha256:e13b31d1b4b68db60b3b29f8e337908f328c7f05b9add4b1b5c74e0691180109",
+                "sha256:e1739496c2f0108013629aa095cc32a8c6363444361960c07493818d0dea2da4",
+                "sha256:e43255a83835a129ef98f75d13d643844d8c646b258bebd11e4a0975203e018f",
+                "sha256:e626cefff8491bce356221c22af5a3ea528b0b41fbabc719c00ae233819ea0bf",
+                "sha256:eadad75bf91897f922e0fb3dca1b322a58b1726a953f98c2e5f0606bd8408621",
+                "sha256:f33da6b5d19ad1bb5e7ad38bb8ba5c426d2178928bc2b2c44e8823ea0ecb6ff3",
+                "sha256:f4052a8a4926d4468416fc7d4b2a7b2a3e35f25b39f4061a7e2a3a2748c4fc48",
+                "sha256:f6ca38dd8d988eca8f07305125dec6f54ac1c518f1aaddcc14d08c01aebb6efc"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.6"
+        },
         "cycler": {
             "hashes": [
                 "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3",
@@ -327,17 +396,17 @@
         },
         "executing": {
             "hashes": [
-                "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e",
-                "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"
+                "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc",
+                "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"
             ],
-            "version": "==0.10.0"
+            "version": "==1.2.0"
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:2f7158c4de792555753d6c2277d6a2af2d406dfd97aeca21d17173561ede4fe6",
-                "sha256:d6fa3ffbe719768d70e298b9fb847484e2bdfdb7241ed052b8d57a9294a8c334"
+                "sha256:01e366f25d9047816fe3d288cbfc3e10541daf0af2044763f3d0ade42476da18",
+                "sha256:21f918e8d9a1a4ba9c22e09574ba72267a6762d47822db9add95f6454e51cc1c"
             ],
-            "version": "==2.16.1"
+            "version": "==2.16.2"
         },
         "fiona": {
             "hashes": [
@@ -357,11 +426,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:cb91ef8d5a435d90aeb3ab814b2548c6b515df5bc13b4c5adaa23778f2f79823",
-                "sha256:e637d2fe06bddabbfc488e02ef32d04d561e3c71e9ba11abc7782ea753ceb218"
+                "sha256:2bb244009f9bf3fa100fc3ead6aeb99febe5985fa20afbfbaa2f8946c2fbdaf1",
+                "sha256:820466f43c8be8c3009aef8b87e785014133508f0de64ec469e4efb643ae54fb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.36.0"
+            "version": "==4.38.0"
         },
         "geopandas": {
             "hashes": [
@@ -397,51 +466,51 @@
         },
         "idna": {
             "hashes": [
-                "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
-                "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+                "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
+                "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.3"
+            "version": "==3.4"
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670",
-                "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"
+                "sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab",
+                "sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.12.0"
+            "version": "==5.0.0"
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:5481e97fb45af8dcf2f798952625591c58fe599d0735d86b10f54de086a61681",
-                "sha256:f78a8df21a79bcc30cfd400bdc38f314333de7c0fb619763f6b9dabab8268bb7"
+                "sha256:c01b1b94210d9849f286b86bb51bcea7cd56dde0600d8db721d7b81330711668",
+                "sha256:ee17ec648f85480d523596ce49eae8ead87d5631ae1551f913c0100b5edd3437"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==5.9.0"
+            "version": "==5.10.0"
         },
         "ipykernel": {
             "hashes": [
-                "sha256:37acc3254caa8a0dafcddddc8dc863a60ad1b46487b68aee361d9a15bda98112",
-                "sha256:d8969c5b23b0e453a23166da5a669c954db399789293fcb03fec5cb25367e43c"
+                "sha256:301fdb487587c9bf277025001da97b53697aab73ae1268d9d1ba972a2c5fc801",
+                "sha256:e195cf6d8c3dd5d41f3cf8ad831d9891f95d7d18fa6d5fb4d30a713df99b26a4"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==6.15.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==6.17.0"
         },
         "ipympl": {
             "hashes": [
-                "sha256:3505b472d412178fd115e255764d1e7e7632d6c82e9de6d62b25438a35374680",
-                "sha256:64096900c6392d53761a68ef6dd07541cc75672f9c3f84a3bb44bf8f9bbc3787"
+                "sha256:655604f0bf6d264cf599766950a5b26e292d107cc23e197503647e75417981cc",
+                "sha256:c865c1992248f9966fbe4b6006239ae2959b00fc7e887ae32b0bd389808f0f8b"
             ],
             "index": "pypi",
-            "version": "==0.9.1"
+            "version": "==0.9.2"
         },
         "ipython": {
             "hashes": [
-                "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1",
-                "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd"
+                "sha256:7c959e3dedbf7ed81f9b9d8833df252c430610e2a4a6464ec13cd20975ce20a5",
+                "sha256:91ef03016bcf72dd17190f863476e7c799c6126ec7e8be97719d1bc9a78a59a4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.4.0"
+            "version": "==8.6.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -452,11 +521,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:3d47a7826cc6e2644d7cb90db26699451f8b42379cf63b761431b63d19984ca2",
-                "sha256:449ab8e7872d0f388ee5c5b3666b9d6af5e5618a5749fd62652680be37dff2af"
+                "sha256:08cb75c6e0a96836147cbfdc55580ae04d13e05d26ffbc377b4e1c68baa28b1f",
+                "sha256:1dc3dd4ee19ded045ea7c86eb273033d238d8e43f9e7872c52d092683f263891"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.7.2"
+            "version": "==8.0.2"
         },
         "jedi": {
             "hashes": [
@@ -491,11 +560,11 @@
         },
         "jsonschema": {
             "hashes": [
-                "sha256:3776512df4f53f74e6e28fe35717b5b223c1756875486984a31bc9165e7fc920",
-                "sha256:870a61bb45050b81103faf6a4be00a0a906e06636ffcf0b84f5a2e51faf901ff"
+                "sha256:165059f076eff6971bae5b742fc029a7b4ef3f9bcf04c14e4776a7605de14b23",
+                "sha256:9e74b8f9738d6a946d70705dc692b74b5429cd0960d58e79ffecfc43b2221eb9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.13.0"
+            "version": "==4.16.0"
         },
         "jupyter": {
             "hashes": [
@@ -508,11 +577,11 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:17d74b0d0a7b24f1c8c527b24fcf4607c56bee542ffe8e3418e50b21e514b621",
-                "sha256:aa9a6c32054b290374f95f73bb0cae91455c58dfb84f65c8591912b8f65e6d56"
+                "sha256:1c1d418ef32a45a1fae0b243e6f01cc9bf65fa8ddbd491a034b9ba6ac6502951",
+                "sha256:5616db609ac720422e6a4b893d6572b8d655ff41e058367f4459a0d2c0726832"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.3.4"
+            "version": "==7.4.4"
         },
         "jupyter-console": {
             "hashes": [
@@ -524,27 +593,27 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:2e5f244d44894c4154d06aeae3419dd7f1b0ef4494dc5584929b398c61cfd314",
-                "sha256:715e22bb6cc7db3718fddfac1f69f1c7e899ca00e42bdfd4bf3705452b9fd84a"
+                "sha256:3815e80ec5272c0c19aad087a0d2775df2852cfca8f5a17069e99c9350cecff8",
+                "sha256:c2909b9bc7dca75560a6c5ae78c34fd305ede31cd864da3c0d0bb2ed89aa9337"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==4.11.1"
+            "index": "pypi",
+            "version": "==4.11.2"
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:022759b09c96a4e2feb95de59ce4283e04e17782efe197b91d23a47521609b77",
-                "sha256:2b72fc595bccae292260aad8157a0ead8da2c703ec6ae1bb7b36dbad0e267ea7"
+                "sha256:992531008544d77e05a16251cdfbd0bdff1b1efa14760c79b9cc776ac9214cf1",
+                "sha256:d0adca19913a3763359be7f0b8c2ea8bfde356f4b8edd8e3149d7d0fbfaa248b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.18.1"
+            "version": "==1.21.0"
         },
         "jupyterlab": {
             "hashes": [
-                "sha256:3181d60518e6101b9e43f6da231aa4be812a59d81ddcc8fe680dd33066fddae6",
-                "sha256:472f6b7996c75f6991592483c26d9fe205a59a71ccbce15842400155dc64f59b"
+                "sha256:e02556c8ea1b386963c4b464e4618aee153c5416b07ab481425c817a033323a2",
+                "sha256:f433059fe0e12d75ea90a81a0b6721113bb132857e3ec2197780b6fe84cbcbde"
             ],
             "index": "pypi",
-            "version": "==3.4.5"
+            "version": "==3.5.0"
         },
         "jupyterlab-pygments": {
             "hashes": [
@@ -556,19 +625,19 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:0e327d7a346874fd8e94c1bcbd69906d18a8558df8f13115c5afd183c3107756",
-                "sha256:a91c515e0e7971a8f7c3c9834b748857f7dac502f93604bf283987991fd987ef"
+                "sha256:07007a3a0a30bfc6424b28b76df8d67386cc2d5f9f42886773b1b3c473cb9a3f",
+                "sha256:7ad1a37a716f6d10e90185c636c122d55a58ef3141ae50f9d0601d3ccf54d43e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.15.0"
+            "version": "==2.16.2"
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:67d0ef1e407e0c42c8ab60b9d901cd7a4c68923650763f75bf17fb06c1943b79",
-                "sha256:90ab47d99da03a3697074acb23b2975ead1d6171aa41cb2812041a7f2a08177a"
+                "sha256:6aa1bc0045470d54d76b9c0b7609a8f8f0087573bae25700a370c11f82cb38c8",
+                "sha256:c767181399b4ca8b647befe2d913b1260f51bf9d8ef9b7a14632d4c1a7b536bd"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.1.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.0.3"
         },
         "kiwisolver": {
             "hashes": [
@@ -666,82 +735,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.33.0"
         },
-        "lxml": {
-            "hashes": [
-                "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318",
-                "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c",
-                "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b",
-                "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000",
-                "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73",
-                "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d",
-                "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb",
-                "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8",
-                "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2",
-                "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345",
-                "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94",
-                "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e",
-                "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b",
-                "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc",
-                "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a",
-                "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9",
-                "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc",
-                "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387",
-                "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb",
-                "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7",
-                "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4",
-                "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97",
-                "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67",
-                "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627",
-                "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7",
-                "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd",
-                "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3",
-                "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7",
-                "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130",
-                "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b",
-                "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036",
-                "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785",
-                "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca",
-                "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91",
-                "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc",
-                "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536",
-                "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391",
-                "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3",
-                "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d",
-                "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21",
-                "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3",
-                "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d",
-                "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29",
-                "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715",
-                "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed",
-                "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25",
-                "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c",
-                "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785",
-                "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837",
-                "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4",
-                "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b",
-                "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2",
-                "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067",
-                "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448",
-                "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d",
-                "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2",
-                "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc",
-                "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c",
-                "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5",
-                "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84",
-                "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8",
-                "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf",
-                "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7",
-                "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e",
-                "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb",
-                "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b",
-                "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3",
-                "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad",
-                "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8",
-                "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==4.9.1"
-        },
         "markupsafe": {
             "hashes": [
                 "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003",
@@ -790,44 +783,50 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:0bcdfcb0f976e1bac6721d7d457c17be23cf7501f977b6a38f9d38a3762841f7",
-                "sha256:1e64ac9be9da6bfff0a732e62116484b93b02a0b4d4b19934fb4f8e7ad26ad6a",
-                "sha256:22227c976ad4dc8c5a5057540421f0d8708c6560744ad2ad638d48e2984e1dbc",
-                "sha256:2886cc009f40e2984c083687251821f305d811d38e3df8ded414265e4583f0c5",
-                "sha256:2e6d184ebe291b9e8f7e78bbab7987d269c38ea3e062eace1fe7d898042ef804",
-                "sha256:3211ba82b9f1518d346f6309df137b50c3dc4421b4ed4815d1d7eadc617f45a1",
-                "sha256:339cac48b80ddbc8bfd05daae0a3a73414651a8596904c2a881cfd1edb65f26c",
-                "sha256:35a8ad4dddebd51f94c5d24bec689ec0ec66173bf614374a1244c6241c1595e0",
-                "sha256:3b4fa56159dc3c7f9250df88f653f085068bcd32dcd38e479bba58909254af7f",
-                "sha256:43e9d3fa077bf0cc95ded13d331d2156f9973dce17c6f0c8b49ccd57af94dbd9",
-                "sha256:57f1b4e69f438a99bb64d7f2c340db1b096b41ebaa515cf61ea72624279220ce",
-                "sha256:5c096363b206a3caf43773abebdbb5a23ea13faef71d701b21a9c27fdcef72f4",
-                "sha256:6bb93a0492d68461bd458eba878f52fdc8ac7bdb6c4acdfe43dba684787838c2",
-                "sha256:6ea6aef5c4338e58d8d376068e28f80a24f54e69f09479d1c90b7172bad9f25b",
-                "sha256:6fe807e8a22620b4cd95cfbc795ba310dc80151d43b037257250faf0bfcd82bc",
-                "sha256:73dd93dc35c85dece610cca8358003bf0760d7986f70b223e2306b4ea6d1406b",
-                "sha256:839d47b8ead7ad9669aaacdbc03f29656dc21f0d41a6fea2d473d856c39c8b1c",
-                "sha256:874df7505ba820e0400e7091199decf3ff1fde0583652120c50cd60d5820ca9a",
-                "sha256:879c7e5fce4939c6aa04581dfe08d57eb6102a71f2e202e3314d5fbc072fd5a0",
-                "sha256:94ff86af56a3869a4ae26a9637a849effd7643858a1a04dd5ee50e9ab75069a7",
-                "sha256:99482b83ebf4eb6d5fc6813d7aacdefdd480f0d9c0b52dcf9f1cc3b2c4b3361a",
-                "sha256:9ab29589cef03bc88acfa3a1490359000c18186fc30374d8aa77d33cc4a51a4a",
-                "sha256:9befa5954cdbc085e37d974ff6053da269474177921dd61facdad8023c4aeb51",
-                "sha256:a206a1b762b39398efea838f528b3a6d60cdb26fe9d58b48265787e29cd1d693",
-                "sha256:ab8d26f07fe64f6f6736d635cce7bfd7f625320490ed5bfc347f2cdb4fae0e56",
-                "sha256:b28de401d928890187c589036857a270a032961411934bdac4cf12dde3d43094",
-                "sha256:b428076a55fb1c084c76cb93e68006f27d247169f056412607c5c88828d08f88",
-                "sha256:bf618a825deb6205f015df6dfe6167a5d9b351203b03fab82043ae1d30f16511",
-                "sha256:c995f7d9568f18b5db131ab124c64e51b6820a92d10246d4f2b3f3a66698a15b",
-                "sha256:cd45a6f3e93a780185f70f05cf2a383daed13c3489233faad83e81720f7ede24",
-                "sha256:d2484b350bf3d32cae43f85dcfc89b3ed7bd2bcd781ef351f93eb6fb2cc483f9",
-                "sha256:d62880e1f60e5a30a2a8484432bcb3a5056969dc97258d7326ad465feb7ae069",
-                "sha256:dacddf5bfcec60e3f26ec5c0ae3d0274853a258b6c3fc5ef2f06a8eb23e042be",
-                "sha256:f3840c280ebc87a48488a46f760ea1c0c0c83fcf7abbe2e6baf99d033fd35fd8",
-                "sha256:f814504e459c68118bf2246a530ed953ebd18213dc20e3da524174d84ed010b2"
+                "sha256:02561141c434154f7bae8e5449909d152367cb40aa57bfb2a27f2748b9c5f95f",
+                "sha256:05e86446562063d6186ff6d700118c0dbd5dccc403a6187351ee526c48878f10",
+                "sha256:0bab7564aafd5902128d54b68dca04f5755413fb6b502100bb0235a545882c48",
+                "sha256:12ab21d0cad122f5b23688d453a0280676e7c42f634f0dbd093d15d42d142b1f",
+                "sha256:183bf3ac6a6023ee590aa4b677f391ceed65ec0d6b930901a8483c267bd12995",
+                "sha256:1e2c75d5d1ff6b7ef9870360bfa23bea076b8dc0945a60d19453d7619ed9ea8f",
+                "sha256:220314c2d6b9ca11570d7cd4b841c9f3137546f188336003b9fb8def4dcb804d",
+                "sha256:2469f57e4c5cc0e85eddc7b30995ea9c404a78c0b1856da75d1a5887156ca350",
+                "sha256:27337bcb38d5db7430c14f350924542d75416ec1546d5d9d9f39b362b71db3fb",
+                "sha256:2cc5d726d4d42865f909c5208a7841109d76584950dd0587b01a77cc279d4ab7",
+                "sha256:3c53486278a0629fd892783271dc994b962fba8dfe207445d039e14f1928ea46",
+                "sha256:4648f0d79a87bf50ee740058305c91091ee5e1fbb71a7d2f5fe6707bfe328d1c",
+                "sha256:47cb088bbce82ae9fc2edf3c25e56a5c6142ce2553fea2b781679f960a70c207",
+                "sha256:4a3d903588b519b38ed085d0ae762a1dcd4b70164617292175cfd91b90d6c415",
+                "sha256:4d3b0e0a4611bd22065bbf47e9b2f689ac9e575bcb850a9f0ae2bbed75cab956",
+                "sha256:52935b7d4ccbf0dbc9cf454dbb10ca99c11cbe8da9467596b96e5e21fd4dfc5c",
+                "sha256:563896ba269324872ace436a57775dcc8322678a9496b28a8c25cdafa5ec2b92",
+                "sha256:565f514dec81a41cbed10eb6011501879695087fc2787fb89423a466508abbbd",
+                "sha256:5f97141e05baf160c3ec125f06ceb2a44c9bb62f42fcb8ee1c05313c73e99432",
+                "sha256:6f5788168da2661b42f7468063b725cc73fdbeeb80f2704cb2d8c415e9a57c50",
+                "sha256:71eced071825005011cdc64efbae2e2c76b8209c18aa487dedf69796fe4b1e40",
+                "sha256:7730e60e751cfcfe7fcb223cf03c0b979e9a064c239783ad37929d340a364cef",
+                "sha256:8245e85fd793f58edf29b8a9e3be47e8ecf76ea1a1e8240545f2746181ca5787",
+                "sha256:85948b303534b69fd771126764cf883fde2af9b003eb5778cb60f3b46f93d3f6",
+                "sha256:87027ff7b2edeb14476900261ef04d4beae949e1dfa0a3eb3ad6a6efbf9d0e1d",
+                "sha256:87bdbd37d0a41e025879863fe9b17bab15c0421313bc33e77e5e1aa54215c9c5",
+                "sha256:8dc25473319afabe49150267e54648ac559c33b0fc2a80c8caecfbbc2948a820",
+                "sha256:8ddd58324dc9a77e2e56d7b7aea7dbd0575b6f7cd1333c3ca9d388ac70978344",
+                "sha256:9403764017d20ff570f7ce973a8b9637f08a6109118f4e0ce6c7493d8849a0d3",
+                "sha256:9dd40505ccc526acaf9a5db1b3029e237c64b58f1249983b28a291c2d6a1d0fa",
+                "sha256:a4de03085afb3b80fab341afaf8e60dfe06ce439b6dfed55d657cf34a7bc3c40",
+                "sha256:a68b91ac7e6bb26100a540a033f54c95fe06d9c0aa51312c2a52d07d1bde78f4",
+                "sha256:b53387d4e59432ff221540a4ffb5ee9669c69417805e4faf0148a00d701c61f9",
+                "sha256:c1effccef0cea2d4da9feeed22079adf6786f92c800a7d0d2ef2104318a1c66c",
+                "sha256:c9756a8e69f6e1f76d47eb42132175b6814da1fbeae0545304c6d0fc2aae252a",
+                "sha256:d0161ebf87518ecfe0980c942d5f0d5df0e080c1746ebaab2027a969967014b7",
+                "sha256:e2d1b7225666f7e1bcc94c0bc9c587a82e3e8691da4757e357e5c2515222ee37",
+                "sha256:e3c116e779fbbf421a9e4d3060db259a9bb486d98f4e3c5a0877c599bd173582",
+                "sha256:e4c8b5a243dd29d50289d694e931bd6cb6ae0b5bd654d12c647543d63862540c",
+                "sha256:e632f66218811d4cf8b7a2a649e25ec15406c3c498f72d19e2bcf8377f38445d",
+                "sha256:fad858519bd6d52dbfeebdbe04d00dd8e932ed436f1c535e61bcc970a96c11e4"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.5.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==3.6.1"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -846,10 +845,10 @@
         },
         "mistune": {
             "hashes": [
-                "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e",
-                "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4"
+                "sha256:182cc5ee6f8ed1b807de6b7bb50155df7b66495412836b9a74c8fbdfc75fe36d",
+                "sha256:9ee0a66053e2267aba772c71e06891fa8f1af6d4b01d5e84e267b4570d4d9808"
             ],
-            "version": "==0.8.4"
+            "version": "==2.0.4"
         },
         "munch": {
             "hashes": [
@@ -860,51 +859,51 @@
         },
         "natsort": {
             "hashes": [
-                "sha256:c7c1f3f27c375719a4dfcab353909fe39f26c2032a062a8c80cc844eaaca0445",
-                "sha256:f59988d2f24e77b6b56f8a8f882d5df6b3b637e09e075abc67b486d59fba1a4b"
+                "sha256:04fe18fdd2b9e5957f19f687eb117f102ef8dde6b574764e536e91194bed4f5f",
+                "sha256:57f85b72c688b09e053cdac302dd5b5b53df5f73ae20b4874fcbffd8bf783d11"
             ],
             "index": "pypi",
-            "version": "==8.1.0"
+            "version": "==8.2.0"
         },
         "nbclassic": {
             "hashes": [
-                "sha256:4b01076effdac53e775cd1b6a4e891663568b32621468e205b502a23b2921899",
-                "sha256:f03111b2cebaa69b88370a7b23b19b2b37c9bb71767f1828cdfd7a047eae8edd"
+                "sha256:1e0470583b55089c427940ed31b8a866ffef7ccab101494e409efe5ac7ba9897",
+                "sha256:d71d18aa6605eaf59e9b99b34c96360af45847f2a30ee2fefbe2f7bed4bc3df2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.4.3"
+            "version": "==0.4.7"
         },
         "nbclient": {
             "hashes": [
-                "sha256:09bae4ea2df79fa6bc50aeb8278d8b79d2036792824337fa6eee834afae17312",
-                "sha256:0df76a7961d99a681b4796c74a1f2553b9f998851acc01896dce064ad19a9027"
+                "sha256:434c91385cf3e53084185334d675a0d33c615108b391e260915d1aa8e86661b8",
+                "sha256:a1d844efd6da9bc39d2209bf996dbd8e07bf0f36b796edfabaa8f8a9ab77c3aa"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.6.6"
+            "version": "==0.7.0"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:10ed693c4cfd3c63583c87ca5c3a2f6ed874145103595f3824efcc8dfcb7522c",
-                "sha256:2564bb5125d862949f72475de0c0348392add7ea62cc950985347bfe7bbc2034"
+                "sha256:66326174c190dc4f0a6cbbff96f30c632774b441fa3c7565662bb3d41992fb0f",
+                "sha256:7ae7ccc68495b565dab153459ee7e65039970913eb115070da6e2c673cf0e9f8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.5.3"
+            "version": "==7.2.3"
         },
         "nbformat": {
             "hashes": [
-                "sha256:0d6072aaec95dddc39735c144ee8bbc6589c383fb462e4058abc855348152dad",
-                "sha256:44ba5ca6acb80c5d5a500f1e5b83ede8cbe364d5a495c4c8cf60aaf1ba656501"
+                "sha256:1b05ec2c552c2f1adc745f4eddce1eac8ca9ffd59bb9fd859e827eaa031319f9",
+                "sha256:1d4760c15c1a04269ef5caf375be8b98dd2f696e5eb9e603ec2bf091f9b0d3f3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.0"
+            "version": "==5.7.0"
         },
         "nest-asyncio": {
             "hashes": [
-                "sha256:b98e3ec1b246135e4642eceffa5a6c23a3ab12c82ff816a92c612d68205813b2",
-                "sha256:e442291cd942698be619823a17a86a5759eabe1f8613084790de189fe9e16d65"
+                "sha256:b9a953fb40dceaa587d109609098db21900182b16440652454a146cffb06e8b8",
+                "sha256:d267cc1ff794403f7df692964d1d2a3fa9418ffea2a3f6859a439ff482fef290"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==1.5.5"
+            "version": "==1.5.6"
         },
         "netcdf4": {
             "hashes": [
@@ -949,19 +948,19 @@
         },
         "notebook": {
             "hashes": [
-                "sha256:6268c9ec9048cff7a45405c990c29ac9ca40b0bc3ec29263d218c5e01f2b4e86",
-                "sha256:8c07a3bb7640e371f8a609bdbb2366a1976c6a2589da8ef917f761a61e3ad8b1"
+                "sha256:c1897e5317e225fc78b45549a6ab4b668e4c996fd03a04e938fe5e7af2bfffd0",
+                "sha256:e04f9018ceb86e4fa841e92ea8fb214f8d23c1cedfde530cc96f92446924f0e4"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.4.12"
+            "version": "==6.5.2"
         },
         "notebook-shim": {
             "hashes": [
-                "sha256:02432d55a01139ac16e2100888aa2b56c614720cec73a27e71f40a5387e45324",
-                "sha256:7897e47a36d92248925a2143e3596f19c60597708f7bef50d81fcd31d7263e85"
+                "sha256:481711abddfb2e5305b83cf0efe18475824eb47d1ba9f87f66a8c574b8b5c9e4",
+                "sha256:fdb81febb05932c6d19e44e10382ce05469cac5e1b6e99b49be6159ddb5e4804"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.1.0"
+            "version": "==0.2.0"
         },
         "numba": {
             "hashes": [
@@ -990,33 +989,39 @@
         },
         "numexpr": {
             "hashes": [
-                "sha256:052ec3a55cc1ccc447580ee5b828b2bd0bc14fea0756ddb81d9617b5472c77b5",
-                "sha256:08d8f8e31647815d979185eb455cb5b4d845e20ff808bd6f7f4edf5e0a35e2f6",
-                "sha256:1575a35190d650bf64d2efd8590a8ef3ca564ef20b9f8727428b57759712becb",
-                "sha256:19cd7563421862de85404bd5de06bee8a3ebff4fc9f718de09cc704bc3348f08",
-                "sha256:33be3bbbad71d97d14a39d84957c2bcc368fec775369664d0c24be030c50c359",
-                "sha256:3a1ce79b7d32c55cce334566e3c6716f7b646f6eceb2ace38adaa795848f3583",
-                "sha256:4ddc46c1e5d726b57d008169b75074ab66869e1827098614ebafa45d152f81b7",
-                "sha256:4f291f0df7b25d9530991f880cc232a644a7a722d130c61b43e593b98fb6523f",
-                "sha256:5532bd7164eb8a05410771faf94a661fc69e5ca72deb8612f1bedc26311ed3c8",
-                "sha256:5b014d1c426c444102fb9eea6438052ee86c82684d27dd20b531caf2c60bc4c9",
-                "sha256:5c660dea90935b963db9529d963493c40fabb2343684b52083fb86b2547d60c8",
-                "sha256:828926f5d4dc9ace2bebd2eec56bee852518afa31e6df175d1706e6631dfd1a2",
-                "sha256:854541cf4214d747ab2f87229e9dde052fddc52c379f59047d64f9b7e2f4d578",
-                "sha256:99b9a91811de8cd24bd7d7fbc1883653dad6485e8c683d85b1007a13868713f6",
-                "sha256:a6954d65d7140864d9bb2302b7580c60c88c4d12e00c59a0a53f1660573e922b",
-                "sha256:b127b0d0e1665b94adcc658c5f9d688ac4903ef81da5d8f4e956c995cf69d5c7",
-                "sha256:ba9aa03f4213f4e0c0d964afd6a920c9000e73d22b88c72c46b151d292ee5581",
-                "sha256:be9b8309a4a2f785197b1a29f7767a5ff217ea505e5a751b237336f3b50b7e48",
-                "sha256:c35669268f602ac6412c8c6244b256ebb4f31ffc926b936ca0d9cffda251db8a",
-                "sha256:cb647c9d9c785dae0759bf6c875cde2bec472b5c3f7a6015734b161ae766d141",
-                "sha256:cbd75ac287923bd0c5b95143915648c62d97f994b06dacd770bd205da014f6bd",
-                "sha256:eba7fad925e3063a0434844a667fbdea30b53fe1344efef73475b32d33aa0fec",
-                "sha256:f29b882a21b5381c0e472bc66e8d1c519b8920edc2522b8b4ede79e314d31d20",
-                "sha256:fe6b49631c3bf54e92b0fb334c8e59694685924492d80c325e1b44ecbbc0f22d"
+                "sha256:059546e8f6283ccdb47c683101a890844f667fa6d56258d48ae2ecf1b3875957",
+                "sha256:17ac9cfe6d0078c5fc06ba1c1bbd20b8783f28c6f475bbabd3cad53683075cab",
+                "sha256:3f039321d1c17962c33079987b675fb251b273dbec0f51aac0934e932446ccc3",
+                "sha256:5538b30199bfc68886d2be18fcef3abd11d9271767a7a69ff3688defe782800a",
+                "sha256:655d84eb09adfee3c09ecf4a89a512225da153fdb7de13c447404b7d0523a9a7",
+                "sha256:6931b1e9d4f629f43c14b21d44f3f77997298bea43790cfcdb4dd98804f90783",
+                "sha256:6c368aa35ae9b18840e78b05f929d3a7b3abccdba9630a878c7db74ca2368339",
+                "sha256:6ee9db7598dd4001138b482342b96d78110dd77cefc051ec75af3295604dde6a",
+                "sha256:77898fdf3da6bb96aa8a4759a8231d763a75d848b2f2e5c5279dad0b243c8dfe",
+                "sha256:7bca95f4473b444428061d4cda8e59ac564dc7dc6a1dea3015af9805c6bc2946",
+                "sha256:7d71add384adc9119568d7e9ffa8a35b195decae81e0abf54a2b7779852f0637",
+                "sha256:845a6aa0ed3e2a53239b89c1ebfa8cf052d3cc6e053c72805e8153300078c0b1",
+                "sha256:90f12cc851240f7911a47c91aaf223dba753e98e46dff3017282e633602e76a7",
+                "sha256:9400781553541f414f82eac056f2b4c965373650df9694286b9bd7e8d413f8d8",
+                "sha256:9e34931089a6bafc77aaae21f37ad6594b98aa1085bb8b45d5b3cd038c3c17d9",
+                "sha256:9f096d707290a6a00b6ffdaf581ee37331109fb7b6c8744e9ded7c779a48e517",
+                "sha256:a38664e699526cb1687aefd9069e2b5b9387da7feac4545de446141f1ef86f46",
+                "sha256:a6d2d7740ae83ba5f3531e83afc4b626daa71df1ef903970947903345c37bd03",
+                "sha256:a75967d46b6bd56455dd32da6285e5ffabe155d0ee61eef685bbfb8dafb2e484",
+                "sha256:b076db98ca65eeaf9bd224576e3ac84c05e451c0bd85b13664b7e5f7b62e2c70",
+                "sha256:b318541bf3d8326682ebada087ba0050549a16d8b3fa260dd2585d73a83d20a7",
+                "sha256:b96334fc1748e9ec4f93d5fadb1044089d73fb08208fdb8382ed77c893f0be01",
+                "sha256:c867cc36cf815a3ec9122029874e00d8fbcef65035c4a5901e9b120dd5d626a2",
+                "sha256:d5432537418d18691b9115d615d6daa17ee8275baef3edf1afbbf8bc69806147",
+                "sha256:db93cf1842f068247de631bfc8af20118bf1f9447cd929b531595a5e0efc9346",
+                "sha256:df35324666b693f13a016bc7957de7cc4d8801b746b81060b671bf78a52b9037",
+                "sha256:df3a1f6b24214a1ab826e9c1c99edf1686c8e307547a9aef33910d586f626d01",
+                "sha256:eaec59e9bf70ff05615c34a8b8d6c7bd042bd9f55465d7b495ea5436f45319d0",
+                "sha256:f3a920bfac2645017110b87ddbe364c9c7a742870a4d2f6120b8786c25dc6db3",
+                "sha256:ff5835e8af9a212e8480003d731aad1727aaea909926fd009e8ae6a1cba7f141"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.8.3"
+            "version": "==2.8.4"
         },
         "numpy": {
             "hashes": [
@@ -1109,67 +1114,68 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927",
-                "sha256:030e3460861488e249731c3e7ab59b07c7853838ff3b8e16aac9561bb345da14",
-                "sha256:0ed2c4ef2451de908c90436d6e8092e13a43992f1860275b4d8082667fbb2ffc",
-                "sha256:136659638f61a251e8ed3b331fc6ccd124590eeff539de57c5f80ef3a9594e58",
-                "sha256:13b725463f32df1bfeacbf3dd197fb358ae8ebcd8c5548faa75126ea425ccb60",
-                "sha256:1536ad017a9f789430fb6b8be8bf99d2f214c76502becc196c6f2d9a75b01b76",
-                "sha256:15928f824870535c85dbf949c09d6ae7d3d6ac2d6efec80f3227f73eefba741c",
-                "sha256:17d4cafe22f050b46d983b71c707162d63d796a1235cdf8b9d7a112e97b15bac",
-                "sha256:1802f34298f5ba11d55e5bb09c31997dc0c6aed919658dfdf0198a2fe75d5490",
-                "sha256:1cc1d2451e8a3b4bfdb9caf745b58e6c7a77d2e469159b0d527a4554d73694d1",
-                "sha256:1fd6f5e3c0e4697fa7eb45b6e93996299f3feee73a3175fa451f49a74d092b9f",
-                "sha256:254164c57bab4b459f14c64e93df11eff5ded575192c294a0c49270f22c5d93d",
-                "sha256:2ad0d4df0f5ef2247e27fc790d5c9b5a0af8ade9ba340db4a73bb1a4a3e5fb4f",
-                "sha256:2c58b24e3a63efd22554c676d81b0e57f80e0a7d3a5874a7e14ce90ec40d3069",
-                "sha256:2d33a11f601213dcd5718109c09a52c2a1c893e7461f0be2d6febc2879ec2402",
-                "sha256:337a74fd2f291c607d220c793a8135273c4c2ab001b03e601c36766005f36885",
-                "sha256:37ff6b522a26d0538b753f0b4e8e164fdada12db6c6f00f62145d732d8a3152e",
-                "sha256:3d1f14f5f691f55e1b47f824ca4fdcb4b19b4323fe43cc7bb105988cad7496be",
-                "sha256:408673ed75594933714482501fe97e055a42996087eeca7e5d06e33218d05aa8",
-                "sha256:4134d3f1ba5f15027ff5c04296f13328fecd46921424084516bdb1b2548e66ff",
-                "sha256:4ad2f835e0ad81d1689f1b7e3fbac7b01bb8777d5a985c8962bedee0cc6d43da",
-                "sha256:50dff9cc21826d2977ef2d2a205504034e3a4563ca6f5db739b0d1026658e004",
-                "sha256:510cef4a3f401c246cfd8227b300828715dd055463cdca6176c2e4036df8bd4f",
-                "sha256:5aed7dde98403cd91d86a1115c78d8145c83078e864c1de1064f52e6feb61b20",
-                "sha256:69bd1a15d7ba3694631e00df8de65a8cb031911ca11f44929c97fe05eb9b6c1d",
-                "sha256:6bf088c1ce160f50ea40764f825ec9b72ed9da25346216b91361eef8ad1b8f8c",
-                "sha256:6e8c66f70fb539301e064f6478d7453e820d8a2c631da948a23384865cd95544",
-                "sha256:727dd1389bc5cb9827cbd1f9d40d2c2a1a0c9b32dd2261db522d22a604a6eec9",
-                "sha256:74a04183e6e64930b667d321524e3c5361094bb4af9083db5c301db64cd341f3",
-                "sha256:75e636fd3e0fb872693f23ccb8a5ff2cd578801251f3a4f6854c6a5d437d3c04",
-                "sha256:7761afe0126d046974a01e030ae7529ed0ca6a196de3ec6937c11df0df1bc91c",
-                "sha256:7888310f6214f19ab2b6df90f3f06afa3df7ef7355fc025e78a3044737fab1f5",
-                "sha256:7b0554af24df2bf96618dac71ddada02420f946be943b181108cac55a7a2dcd4",
-                "sha256:7c7b502bc34f6e32ba022b4a209638f9e097d7a9098104ae420eb8186217ebbb",
-                "sha256:808add66ea764ed97d44dda1ac4f2cfec4c1867d9efb16a33d158be79f32b8a4",
-                "sha256:831e648102c82f152e14c1a0938689dbb22480c548c8d4b8b248b3e50967b88c",
-                "sha256:93689632949aff41199090eff5474f3990b6823404e45d66a5d44304e9cdc467",
-                "sha256:96b5e6874431df16aee0c1ba237574cb6dff1dcb173798faa6a9d8b399a05d0e",
-                "sha256:9a54614049a18a2d6fe156e68e188da02a046a4a93cf24f373bffd977e943421",
-                "sha256:a138441e95562b3c078746a22f8fca8ff1c22c014f856278bdbdd89ca36cff1b",
-                "sha256:a647c0d4478b995c5e54615a2e5360ccedd2f85e70ab57fbe817ca613d5e63b8",
-                "sha256:a9c9bc489f8ab30906d7a85afac4b4944a572a7432e00698a7239f44a44e6efb",
-                "sha256:ad2277b185ebce47a63f4dc6302e30f05762b688f8dc3de55dbae4651872cdf3",
-                "sha256:b6d5e92df2b77665e07ddb2e4dbd6d644b78e4c0d2e9272a852627cdba0d75cf",
-                "sha256:bc431b065722a5ad1dfb4df354fb9333b7a582a5ee39a90e6ffff688d72f27a1",
-                "sha256:bdd0de2d64688ecae88dd8935012c4a72681e5df632af903a1dca8c5e7aa871a",
-                "sha256:c79698d4cd9318d9481d89a77e2d3fcaeff5486be641e60a4b49f3d2ecca4e28",
-                "sha256:cb6259196a589123d755380b65127ddc60f4c64b21fc3bb46ce3a6ea663659b0",
-                "sha256:d5b87da55a08acb586bad5c3aa3b86505f559b84f39035b233d5bf844b0834b1",
-                "sha256:dcd7b9c7139dc8258d164b55696ecd16c04607f1cc33ba7af86613881ffe4ac8",
-                "sha256:dfe4c1fedfde4e2fbc009d5ad420647f7730d719786388b7de0999bf32c0d9fd",
-                "sha256:ea98f633d45f7e815db648fd7ff0f19e328302ac36427343e4432c84432e7ff4",
-                "sha256:ec52c351b35ca269cb1f8069d610fc45c5bd38c3e91f9ab4cbbf0aebc136d9c8",
-                "sha256:eef7592281f7c174d3d6cbfbb7ee5984a671fcd77e3fc78e973d492e9bf0eb3f",
-                "sha256:f07f1f00e22b231dd3d9b9208692042e29792d6bd4f6639415d2f23158a80013",
-                "sha256:f3fac744f9b540148fa7715a435d2283b71f68bfb6d4aae24482a890aed18b59",
-                "sha256:fa768eff5f9f958270b081bb33581b4b569faabf8774726b283edb06617101dc",
-                "sha256:fac2d65901fb0fdf20363fbd345c01958a742f2dc62a8dd4495af66e3ff502a4"
+                "sha256:03150abd92771742d4a8cd6f2fa6246d847dcd2e332a18d0c15cc75bf6703040",
+                "sha256:073adb2ae23431d3b9bcbcff3fe698b62ed47211d0716b067385538a1b0f28b8",
+                "sha256:0b07fffc13f474264c336298d1b4ce01d9c5a011415b79d4ee5527bb69ae6f65",
+                "sha256:0b7257127d646ff8676ec8a15520013a698d1fdc48bc2a79ba4e53df792526f2",
+                "sha256:12ce4932caf2ddf3e41d17fc9c02d67126935a44b86df6a206cf0d7161548627",
+                "sha256:15c42fb9dea42465dfd902fb0ecf584b8848ceb28b41ee2b58f866411be33f07",
+                "sha256:18498994b29e1cf86d505edcb7edbe814d133d2232d256db8c7a8ceb34d18cef",
+                "sha256:1c7c8ae3864846fc95f4611c78129301e203aaa2af813b703c55d10cc1628535",
+                "sha256:22b012ea2d065fd163ca096f4e37e47cd8b59cf4b0fd47bfca6abb93df70b34c",
+                "sha256:276a5ca930c913f714e372b2591a22c4bd3b81a418c0f6635ba832daec1cbcfc",
+                "sha256:2e0918e03aa0c72ea56edbb00d4d664294815aa11291a11504a377ea018330d3",
+                "sha256:3033fbe1feb1b59394615a1cafaee85e49d01b51d54de0cbf6aa8e64182518a1",
+                "sha256:3168434d303babf495d4ba58fc22d6604f6e2afb97adc6a423e917dab828939c",
+                "sha256:3dd6caf940756101205dffc5367babf288a30043d35f80936f9bfb37f8355b32",
+                "sha256:40e1ce476a7804b0fb74bcfa80b0a2206ea6a882938eaba917f7a0f004b42502",
+                "sha256:41e0051336807468be450d52b8edd12ac60bebaa97fe10c8b660f116e50b30e4",
+                "sha256:4390e9ce199fc1951fcfa65795f239a8a4944117b5935a9317fb320e7767b40f",
+                "sha256:502526a2cbfa431d9fc2a079bdd9061a2397b842bb6bc4239bb176da00993812",
+                "sha256:51e0e543a33ed92db9f5ef69a0356e0b1a7a6b6a71b80df99f1d181ae5875636",
+                "sha256:57751894f6618fd4308ed8e0c36c333e2f5469744c34729a27532b3db106ee20",
+                "sha256:5d77adcd56a42d00cc1be30843d3426aa4e660cab4a61021dc84467123f7a00c",
+                "sha256:655a83b0058ba47c7c52e4e2df5ecf484c1b0b0349805896dd350cbc416bdd91",
+                "sha256:68943d632f1f9e3dce98908e873b3a090f6cba1cbb1b892a9e8d97c938871fbe",
+                "sha256:6c738585d7a9961d8c2821a1eb3dcb978d14e238be3d70f0a706f7fa9316946b",
+                "sha256:73bd195e43f3fadecfc50c682f5055ec32ee2c933243cafbfdec69ab1aa87cad",
+                "sha256:772a91fc0e03eaf922c63badeca75e91baa80fe2f5f87bdaed4280662aad25c9",
+                "sha256:77ec3e7be99629898c9a6d24a09de089fa5356ee408cdffffe62d67bb75fdd72",
+                "sha256:7db8b751ad307d7cf238f02101e8e36a128a6cb199326e867d1398067381bff4",
+                "sha256:801ec82e4188e935c7f5e22e006d01611d6b41661bba9fe45b60e7ac1a8f84de",
+                "sha256:82409ffe29d70fd733ff3c1025a602abb3e67405d41b9403b00b01debc4c9a29",
+                "sha256:828989c45c245518065a110434246c44a56a8b2b2f6347d1409c787e6e4651ee",
+                "sha256:829f97c8e258593b9daa80638aee3789b7df9da5cf1336035016d76f03b8860c",
+                "sha256:871b72c3643e516db4ecf20efe735deb27fe30ca17800e661d769faab45a18d7",
+                "sha256:89dca0ce00a2b49024df6325925555d406b14aa3efc2f752dbb5940c52c56b11",
+                "sha256:90fb88843d3902fe7c9586d439d1e8c05258f41da473952aa8b328d8b907498c",
+                "sha256:97aabc5c50312afa5e0a2b07c17d4ac5e865b250986f8afe2b02d772567a380c",
+                "sha256:9aaa107275d8527e9d6e7670b64aabaaa36e5b6bd71a1015ddd21da0d4e06448",
+                "sha256:9f47eabcd2ded7698106b05c2c338672d16a6f2a485e74481f524e2a23c2794b",
+                "sha256:a0a06a052c5f37b4ed81c613a455a81f9a3a69429b4fd7bb913c3fa98abefc20",
+                "sha256:ab388aaa3f6ce52ac1cb8e122c4bd46657c15905904b3120a6248b5b8b0bc228",
+                "sha256:ad58d27a5b0262c0c19b47d54c5802db9b34d38bbf886665b626aff83c74bacd",
+                "sha256:ae5331c23ce118c53b172fa64a4c037eb83c9165aba3a7ba9ddd3ec9fa64a699",
+                "sha256:af0372acb5d3598f36ec0914deed2a63f6bcdb7b606da04dc19a88d31bf0c05b",
+                "sha256:afa4107d1b306cdf8953edde0534562607fe8811b6c4d9a486298ad31de733b2",
+                "sha256:b03ae6f1a1878233ac620c98f3459f79fd77c7e3c2b20d460284e1fb370557d4",
+                "sha256:b0915e734b33a474d76c28e07292f196cdf2a590a0d25bcc06e64e545f2d146c",
+                "sha256:b4012d06c846dc2b80651b120e2cdd787b013deb39c09f407727ba90015c684f",
+                "sha256:b472b5ea442148d1c3e2209f20f1e0bb0eb556538690fa70b5e1f79fa0ba8dc2",
+                "sha256:b59430236b8e58840a0dfb4099a0e8717ffb779c952426a69ae435ca1f57210c",
+                "sha256:b90f7616ea170e92820775ed47e136208e04c967271c9ef615b6fbd08d9af0e3",
+                "sha256:b9a65733d103311331875c1dca05cb4606997fd33d6acfed695b1232ba1df193",
+                "sha256:bac18ab8d2d1e6b4ce25e3424f709aceef668347db8637c2296bcf41acb7cf48",
+                "sha256:bca31dd6014cb8b0b2db1e46081b0ca7d936f856da3b39744aef499db5d84d02",
+                "sha256:be55f8457cd1eac957af0c3f5ece7bc3f033f89b114ef30f710882717670b2a8",
+                "sha256:c7025dce65566eb6e89f56c9509d4f628fddcedb131d9465cacd3d8bac337e7e",
+                "sha256:c935a22a557a560108d780f9a0fc426dd7459940dc54faa49d83249c8d3e760f",
+                "sha256:dbb8e7f2abee51cef77673be97760abff1674ed32847ce04b4af90f610144c7b",
+                "sha256:ebf2029c1f464c59b8bdbe5143c79fa2045a581ac53679733d3a91d400ff9efb",
+                "sha256:f1ff2ee69f10f13a9596480335f406dd1f70c3650349e2be67ca3139280cade0"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==9.2.0"
+            "version": "==9.3.0"
         },
         "pkgutil-resolve-name": {
             "hashes": [
@@ -1181,57 +1187,61 @@
         },
         "prometheus-client": {
             "hashes": [
-                "sha256:522fded625282822a89e2773452f42df14b5a8e84a86433e3f8a189c1d54dc01",
-                "sha256:5459c427624961076277fdc6dc50540e2bacb98eebde99886e59ec55ed92093a"
+                "sha256:be26aa452490cfcf6da953f9436e95a9f2b4d578ca80094b4458930e5f584ab1",
+                "sha256:db7c05cbd13a0f79975592d112320f2605a325969b270a94b71dcabc47b931d2"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.14.1"
+            "version": "==0.15.0"
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0",
-                "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"
+                "sha256:9696f386133df0fc8ca5af4895afe5d78f5fcfe5258111c2a79a1c3e41ffa96d",
+                "sha256:9ada952c9d1787f52ff6d5f3484d0b4df8952787c087edf6a1f7c2cb1ea88148"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.30"
+            "version": "==3.0.31"
         },
         "psutil": {
             "hashes": [
-                "sha256:068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685",
-                "sha256:0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc",
-                "sha256:0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36",
-                "sha256:19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1",
-                "sha256:20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329",
-                "sha256:28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81",
-                "sha256:29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de",
-                "sha256:3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4",
-                "sha256:32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574",
-                "sha256:3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237",
-                "sha256:44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22",
-                "sha256:4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b",
-                "sha256:56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0",
-                "sha256:57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954",
-                "sha256:58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021",
-                "sha256:645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537",
-                "sha256:799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87",
-                "sha256:79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0",
-                "sha256:91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc",
-                "sha256:9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af",
-                "sha256:944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4",
-                "sha256:a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453",
-                "sha256:a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689",
-                "sha256:abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8",
-                "sha256:b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680",
-                "sha256:b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e",
-                "sha256:c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9",
-                "sha256:d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b",
-                "sha256:db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d",
-                "sha256:e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2",
-                "sha256:f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5",
-                "sha256:fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676"
+                "sha256:07d880053c6461c9b89cd5d4808f3b8336665fa3acdefd6777662c5ed73a851a",
+                "sha256:12500d761ac091f2426567f19f95fd3f15a197d96befb44a5c1e3cbe6db5752c",
+                "sha256:1b540599481c73408f6b392cdffef5b01e8ff7a2ac8caae0a91b8222e88e8f1e",
+                "sha256:35feafe232d1aaf35d51bd42790cbccb882456f9f18cdc411532902370d660df",
+                "sha256:3a7826e68b0cf4ce2c1ee385d64eab7d70e3133171376cac53d7c1790357ec8f",
+                "sha256:46907fa62acaac364fff0b8a9da7b360265d217e4fdeaca0a2397a6883dffba2",
+                "sha256:4bd4854f0c83aa84a5a40d3b5d0eb1f3c128f4146371e03baed4589fe4f3c931",
+                "sha256:538fcf6ae856b5e12d13d7da25ad67f02113c96f5989e6ad44422cb5994ca7fc",
+                "sha256:547ebb02031fdada635452250ff39942db8310b5c4a8102dfe9384ee5791e650",
+                "sha256:5e8b50241dd3c2ed498507f87a6602825073c07f3b7e9560c58411c14fe1e1c9",
+                "sha256:5fa88e3d5d0b480602553d362c4b33a63e0c40bfea7312a7bf78799e01e0810b",
+                "sha256:68fa227c32240c52982cb931801c5707a7f96dd8927f9102d6c7771ea1ff5698",
+                "sha256:6ced1ad823ecfa7d3ce26fe8aa4996e2e53fb49b7fed8ad81c80958501ec0619",
+                "sha256:71b1206e7909792d16933a0d2c1c7f04ae196186c51ba8567abae1d041f06dcb",
+                "sha256:767ef4fa33acda16703725c0473a91e1832d296c37c63896c7153ba81698f1ab",
+                "sha256:7ccfcdfea4fc4b0a02ca2c31de7fcd186beb9cff8207800e14ab66f79c773af6",
+                "sha256:7e4939ff75149b67aef77980409f156f0082fa36accc475d45c705bb00c6c16a",
+                "sha256:828c9dc9478b34ab96be75c81942d8df0c2bb49edbb481f597314d92b6441d89",
+                "sha256:8a4e07611997acf178ad13b842377e3d8e9d0a5bac43ece9bfc22a96735d9a4f",
+                "sha256:941a6c2c591da455d760121b44097781bc970be40e0e43081b9139da485ad5b7",
+                "sha256:9a4af6ed1094f867834f5f07acd1250605a0874169a5fcadbcec864aec2496a6",
+                "sha256:9ec296f565191f89c48f33d9544d8d82b0d2af7dd7d2d4e6319f27a818f8d1cc",
+                "sha256:9ec95df684583b5596c82bb380c53a603bb051cf019d5c849c47e117c5064395",
+                "sha256:a04a1836894c8279e5e0a0127c0db8e198ca133d28be8a2a72b4db16f6cf99c1",
+                "sha256:a3d81165b8474087bb90ec4f333a638ccfd1d69d34a9b4a1a7eaac06648f9fbe",
+                "sha256:b4a247cd3feaae39bb6085fcebf35b3b8ecd9b022db796d89c8f05067ca28e71",
+                "sha256:ba38cf9984d5462b506e239cf4bc24e84ead4b1d71a3be35e66dad0d13ded7c1",
+                "sha256:beb57d8a1ca0ae0eb3d08ccaceb77e1a6d93606f0e1754f0d60a6ebd5c288837",
+                "sha256:d266cd05bd4a95ca1c2b9b5aac50d249cf7c94a542f47e0b22928ddf8b80d1ef",
+                "sha256:d8c3cc6bb76492133474e130a12351a325336c01c96a24aae731abf5a47fe088",
+                "sha256:db8e62016add2235cc87fb7ea000ede9e4ca0aa1f221b40cef049d02d5d2593d",
+                "sha256:e7507f6c7b0262d3e7b0eeda15045bf5881f4ada70473b87bc7b7c93b992a7d7",
+                "sha256:ed15edb14f52925869250b1375f0ff58ca5c4fa8adefe4883cfb0737d32f5c02",
+                "sha256:f57d63a2b5beaf797b87024d018772439f9d3103a395627b77d17a8d72009543",
+                "sha256:fa5e32c7d9b60b2528108ade2929b115167fe98d59f89555574715054f50fa31",
+                "sha256:fe79b4ad4836e3da6c4650cb85a663b3a51aef22e1a829c384e18fae87e5e727"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.1"
+            "version": "==5.9.3"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -1361,30 +1371,31 @@
         },
         "pyrsistent": {
             "hashes": [
-                "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c",
-                "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc",
-                "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e",
-                "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26",
-                "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec",
-                "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286",
-                "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045",
-                "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec",
-                "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8",
-                "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c",
-                "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca",
-                "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22",
-                "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a",
-                "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96",
-                "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc",
-                "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1",
-                "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07",
-                "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6",
-                "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b",
-                "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5",
-                "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6"
+                "sha256:06579d46d8ad69529b28f88711191a7fe7103c92d04a9f338dc754f71b92efa0",
+                "sha256:1d0620474d509172e1c50b79d5626bfe1899f174bf650186a50c6ce31289ff52",
+                "sha256:2032d971711643049b4f2c3ca5155a855d507d73bad26dac8d4349e5c5dd6758",
+                "sha256:2c641111c3f110379bb9001dbb26b34eb8cafab3d0fa855dc161c391461a4aab",
+                "sha256:327f99800d04a9abcf580daecfd6dd4bfdb4a7e61c71bf2cd1189ef1ca44bade",
+                "sha256:39f15ad754384e744ac8b00805913bfa66c41131faaa3e4c45c4af0731f3e8f6",
+                "sha256:4c58bd93c4d502f52938fccdbe6c9d70df3a585c6b39d900fab5f76b604282aa",
+                "sha256:62a41037387ae849a493cd945e22b34d167a843d57f75b07dbfad6d96cef485c",
+                "sha256:62b704f18526a8fc243152de8f3f40ae39c5172baff10f50c0c5d5331d6f2342",
+                "sha256:6df99c3578dc4eb33f3eb26bc28277ab40a720b71649d940bff9c1f704377772",
+                "sha256:6ef7430e45c5fa0bb6c361cada4a08ed9c184b5ed086815a85c3bc8c5054566b",
+                "sha256:73b2db09fe15b6e444c0bd566a125a385ca6493456224ce8b367d734f079f576",
+                "sha256:73d4ec2997716af3c8f28f7e3d3a565d273a598982d2fe95639e07ce4db5da45",
+                "sha256:73e3e2fd9da009d558050697cc22ad689f89a14a2ef2e67304628a913e59c947",
+                "sha256:890f577aec554f142e01daf890221d10e4f93a9b1107998d631d3f075b55e8f8",
+                "sha256:8a34a2a8b220247658f7ced871197c390b3a6371d796a5869ab1c62abe0be527",
+                "sha256:8bc23e9ddcb523c3ffb4d712aa0bd5bc67b34ff4e2b23fb557012171bdb4013a",
+                "sha256:945297fc344fef4d540135180ce7babeb2291d124698cc6282f3eac624aa5e82",
+                "sha256:aaa869d9199d7d4c70a57678aff21654cc179c0c32bcfde87f1d65d0ff47e520",
+                "sha256:bc33fc20ddfd89b86b7710142963490d8c4ee8307ed6cc5e189a58fa72390eb9",
+                "sha256:cfe6d8b293d123255fd3b475b5f4e851eb5cbaee2064c8933aa27344381744ae",
+                "sha256:d16ac5ab3d9db78fed40c884d67079524e4cf8276639211ad9e6fa73e727727e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.18.1"
+            "version": "==0.19.1"
         },
         "python-dateutil": {
             "hashes": [
@@ -1396,113 +1407,114 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f",
-                "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"
+                "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5",
+                "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"
             ],
             "index": "pypi",
-            "version": "==0.20.0"
+            "version": "==0.21.0"
         },
         "pytz": {
             "hashes": [
-                "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197",
-                "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5"
+                "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22",
+                "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914"
             ],
-            "version": "==2022.2.1"
+            "version": "==2022.5"
         },
         "pyzmq": {
             "hashes": [
-                "sha256:022cf5ea7bcaa8a06a03c2706e0ae66904b6138b2155577cd34c64bc7cc637ab",
-                "sha256:044447ae4b2016a6b8697571fd633f799f860b19b76c4a2fd9b1140d52ee6745",
-                "sha256:07ed8aaf7ffe150af873269690cc654ffeca7491f62aae0f3821baa181f8d5fe",
-                "sha256:10d1910ec381b851aeb024a042a13db178cb1edf125e76a4e9d2548ad103aadb",
-                "sha256:12e62ff0d5223ec09b597ab6d73858b9f64a51221399f3cb08aa495e1dff7935",
-                "sha256:1f368a82b29f80071781b20663c0fc0c8f6b13273f9f5abe1526af939534f90f",
-                "sha256:20bafc4095eab00f41a510579363a3f5e1f5c69d7ee10f1d88895c4df0259183",
-                "sha256:2141e6798d5981be04c08996d27962086a1aa3ea536fe9cf7e89817fd4523f86",
-                "sha256:23e708fbfdf4ee3107422b69ca65da1b9f056b431fc0888096a8c1d6cd908e8f",
-                "sha256:28dbdb90b2f6b131f8f10e6081012e4e25234213433420e67e0c1162de537113",
-                "sha256:29b74774a0bfd3c4d98ac853f0bdca55bd9ec89d5b0def5486407cca54472ef8",
-                "sha256:2b381aa867ece7d0a82f30a0c7f3d4387b7cf2e0697e33efaa5bed6c5784abcd",
-                "sha256:2f67b63f53c6994d601404fd1a329e6d940ac3dd1d92946a93b2b9c70df67b9f",
-                "sha256:342ca3077f47ec2ee41b9825142b614e03e026347167cbc72a59b618c4f6106c",
-                "sha256:35e635343ff367f697d00fa1484262bb68e36bc74c9b80737eac5a1e04c4e1b1",
-                "sha256:385609812eafd9970c3752c51f2f6c4f224807e3e441bcfd8c8273877d00c8a8",
-                "sha256:38e106b64bad744fe469dc3dd864f2764d66399178c1bf39d45294cc7980f14f",
-                "sha256:39dd252b683816935702825e5bf775df16090619ced9bb4ba68c2d0b6f0c9b18",
-                "sha256:407f909c4e8fde62fbdad9ebd448319792258cc0550c2815567a4d9d8d9e6d18",
-                "sha256:415ff62ac525d9add1e3550430a09b9928d2d24a20cc4ce809e67caac41219ab",
-                "sha256:4805af9614b0b41b7e57d17673459facf85604dac502a5a9244f6e8c9a4de658",
-                "sha256:48400b96788cdaca647021bf19a9cd668384f46e4d9c55cf045bdd17f65299c8",
-                "sha256:49d30ba7074f469e8167917abf9eb854c6503ae10153034a6d4df33618f1db5f",
-                "sha256:4bb798bef181648827019001f6be43e1c48b34b477763b37a8d27d8c06d197b8",
-                "sha256:4d6f110c56f7d5b4d64dde3a382ae61b6d48174e30742859d8e971b18b6c9e5c",
-                "sha256:55568a020ad2cae9ae36da6058e7ca332a56df968f601cbdb7cf6efb2a77579a",
-                "sha256:565bd5ab81f6964fc4067ccf2e00877ad0fa917308975694bbb54378389215f8",
-                "sha256:5c558b50402fca1acc94329c5d8f12aa429738904a5cfb32b9ed3c61235221bb",
-                "sha256:5e05492be125dce279721d6b54fd1b956546ecc4bcdfcf8e7b4c413bc0874c10",
-                "sha256:624fd38071a817644acdae075b92a23ea0bdd126a58148288e8284d23ec361ce",
-                "sha256:650389bbfca73955b262b2230423d89992f38ec48033307ae80e700eaa2fbb63",
-                "sha256:67975a9e1237b9ccc78f457bef17691bbdd2055a9d26e81ee914ba376846d0ce",
-                "sha256:6b1e79bba24f6df1712e3188d5c32c480d8eda03e8ecff44dc8ecb0805fa62f3",
-                "sha256:6fd5d0d50cbcf4bc376861529a907bed026a4cbe8c22a500ff8243231ef02433",
-                "sha256:71b32a1e827bdcbf73750e60370d3b07685816ff3d8695f450f0f8c3226503f8",
-                "sha256:794871988c34727c7f79bdfe2546e6854ae1fa2e1feb382784f23a9c6c63ecb3",
-                "sha256:79a87831b47a9f6161ad23fa5e89d5469dc585abc49f90b9b07fea8905ae1234",
-                "sha256:7e0113d70b095339e99bb522fe7294f5ae6a7f3b2b8f52f659469a74b5cc7661",
-                "sha256:84678153432241bcdca2210cf4ff83560b200556867aea913ffbb960f5d5f340",
-                "sha256:8a68f57b7a3f7b6b52ada79876be1efb97c8c0952423436e84d70cc139f16f0d",
-                "sha256:8c02a0cd39dc01659b3d6cb70bb3a41aebd9885fd78239acdd8d9c91351c4568",
-                "sha256:8c842109d31a9281d678f668629241c405928afbebd913c48a5a8e7aee61f63d",
-                "sha256:8dc66f109a245653b19df0f44a5af7a3f14cb8ad6c780ead506158a057bd36ce",
-                "sha256:90d88f9d9a2ae6cfb1dc4ea2d1710cdf6456bc1b9a06dd1bb485c5d298f2517e",
-                "sha256:9269fbfe3a4eb2009199120861c4571ef1655fdf6951c3e7f233567c94e8c602",
-                "sha256:929d548b74c0f82f7f95b54e4a43f9e4ce2523cfb8a54d3f7141e45652304b2a",
-                "sha256:99a5a77a10863493a1ee8dece02578c6b32025fb3afff91b40476bc489e81648",
-                "sha256:9a39ddb0431a68954bd318b923230fa5b649c9c62b0e8340388820c5f1b15bd2",
-                "sha256:9d0ab2936085c85a1fc6f9fd8f89d5235ae99b051e90ec5baa5e73ad44346e1f",
-                "sha256:9e5bf6e7239fc9687239de7a283aa8b801ab85371116045b33ae20132a1325d6",
-                "sha256:a0f09d85c45f58aa8e715b42f8b26beba68b3b63a8f7049113478aca26efbc30",
-                "sha256:a114992a193577cb62233abf8cb2832970f9975805a64740e325d2f895e7f85a",
-                "sha256:a3fd44b5046d247e7f0f1660bcafe7b5fb0db55d0934c05dd57dda9e1f823ce7",
-                "sha256:ad28ddb40db8e450d7d4bf8a1d765d3f87b63b10e7e9a825a3c130c6371a8c03",
-                "sha256:aecd6ceaccc4b594e0092d6513ef3f1c0fa678dd89f86bb8ff1a47014b8fca35",
-                "sha256:b815991c7d024bf461f358ad871f2be1135576274caed5749c4828859e40354e",
-                "sha256:b861db65f6b8906c8d6db51dde2448f266f0c66bf28db2c37aea50f58a849859",
-                "sha256:c3ebf1668664d20c8f7d468955f18379b7d1f7bc8946b13243d050fa3888c7ff",
-                "sha256:c56b1a62a1fb87565343c57b6743fd5da6e138b8c6562361d7d9b5ce4acf399a",
-                "sha256:c780acddd2934c6831ff832ecbf78a45a7b62d4eb216480f863854a8b7d54fa7",
-                "sha256:c890309296f53f9aa32ffcfc51d805705e1982bffd27c9692a8f1e1b8de279f4",
-                "sha256:c9cfaf530e6a7ff65f0afe275e99f983f68b54dfb23ea401f0bc297a632766b6",
-                "sha256:d904f6595acfaaf99a1a61881fea068500c40374d263e5e073aa4005e5f9c28a",
-                "sha256:e06747014a5ad1b28cebf5bc1ddcdaccfb44e9b441d35e6feb1286c8a72e54be",
-                "sha256:e1fe30bcd5aea5948c42685fad910cd285eacb2518ea4dc6c170d6b535bee95d",
-                "sha256:e753eee6d3b93c5354e8ba0a1d62956ee49355f0a36e00570823ef64e66183f5",
-                "sha256:ec9803aca9491fd6f0d853d2a6147f19f8deaaa23b1b713d05c5d09e56ea7142",
-                "sha256:efb9e38b2a590282704269585de7eb33bf43dc294cad092e1b172e23d4c217e5",
-                "sha256:f07016e3cf088dbfc6e7c5a7b3f540db5c23b0190d539e4fd3e2b5e6beffa4b5",
-                "sha256:f392cbea531b7142d1958c0d4a0c9c8d760dc451e5848d8dd3387804d3e3e62c",
-                "sha256:f619fd38fc2641abfb53cca719c165182500600b82c695cc548a0f05f764be05",
-                "sha256:fefdf9b685fda4141b95ebec975946076a5e0723ff70b037032b2085c5317684",
-                "sha256:ffc6b1623d0f9affb351db4ca61f432dca3628a5ee015f9bf2bfbe9c6836881c"
+                "sha256:0108358dab8c6b27ff6b985c2af4b12665c1bc659648284153ee501000f5c107",
+                "sha256:07bec1a1b22dacf718f2c0e71b49600bb6a31a88f06527dfd0b5aababe3fa3f7",
+                "sha256:0e8f482c44ccb5884bf3f638f29bea0f8dc68c97e38b2061769c4cb697f6140d",
+                "sha256:0ec91f1bad66f3ee8c6deb65fa1fe418e8ad803efedd69c35f3b5502f43bd1dc",
+                "sha256:0f14cffd32e9c4c73da66db97853a6aeceaac34acdc0fae9e5bbc9370281864c",
+                "sha256:15975747462ec49fdc863af906bab87c43b2491403ab37a6d88410635786b0f4",
+                "sha256:1724117bae69e091309ffb8255412c4651d3f6355560d9af312d547f6c5bc8b8",
+                "sha256:1a7c280185c4da99e0cc06c63bdf91f5b0b71deb70d8717f0ab870a43e376db8",
+                "sha256:1b7928bb7580736ffac5baf814097be342ba08d3cfdfb48e52773ec959572287",
+                "sha256:2032d9cb994ce3b4cba2b8dfae08c7e25bc14ba484c770d4d3be33c27de8c45b",
+                "sha256:20e7eeb1166087db636c06cae04a1ef59298627f56fb17da10528ab52a14c87f",
+                "sha256:216f5d7dbb67166759e59b0479bca82b8acf9bed6015b526b8eb10143fb08e77",
+                "sha256:28b119ba97129d3001673a697b7cce47fe6de1f7255d104c2f01108a5179a066",
+                "sha256:3104f4b084ad5d9c0cb87445cc8cfd96bba710bef4a66c2674910127044df209",
+                "sha256:3e6192dbcefaaa52ed81be88525a54a445f4b4fe2fffcae7fe40ebb58bd06bfd",
+                "sha256:42d4f97b9795a7aafa152a36fe2ad44549b83a743fd3e77011136def512e6c2a",
+                "sha256:44e706bac34e9f50779cb8c39f10b53a4d15aebb97235643d3112ac20bd577b4",
+                "sha256:47b11a729d61a47df56346283a4a800fa379ae6a85870d5a2e1e4956c828eedc",
+                "sha256:4854f9edc5208f63f0841c0c667260ae8d6846cfa233c479e29fdc85d42ebd58",
+                "sha256:48f721f070726cd2a6e44f3c33f8ee4b24188e4b816e6dd8ba542c8c3bb5b246",
+                "sha256:52afb0ac962963fff30cf1be775bc51ae083ef4c1e354266ab20e5382057dd62",
+                "sha256:54d8b9c5e288362ec8595c1d98666d36f2070fd0c2f76e2b3c60fbad9bd76227",
+                "sha256:5bd3d7dfd9cd058eb68d9a905dec854f86649f64d4ddf21f3ec289341386c44b",
+                "sha256:613010b5d17906c4367609e6f52e9a2595e35d5cc27d36ff3f1b6fa6e954d944",
+                "sha256:624321120f7e60336be8ec74a172ae7fba5c3ed5bf787cc85f7e9986c9e0ebc2",
+                "sha256:65c94410b5a8355cfcf12fd600a313efee46ce96a09e911ea92cf2acf6708804",
+                "sha256:6640f83df0ae4ae1104d4c62b77e9ef39be85ebe53f636388707d532bee2b7b8",
+                "sha256:687700f8371643916a1d2c61f3fdaa630407dd205c38afff936545d7b7466066",
+                "sha256:77c2713faf25a953c69cf0f723d1b7dd83827b0834e6c41e3fb3bbc6765914a1",
+                "sha256:78068e8678ca023594e4a0ab558905c1033b2d3e806a0ad9e3094e231e115a33",
+                "sha256:7a23ccc1083c260fa9685c93e3b170baba45aeed4b524deb3f426b0c40c11639",
+                "sha256:7abddb2bd5489d30ffeb4b93a428130886c171b4d355ccd226e83254fcb6b9ef",
+                "sha256:80093b595921eed1a2cead546a683b9e2ae7f4a4592bb2ab22f70d30174f003a",
+                "sha256:8242543c522d84d033fe79be04cb559b80d7eb98ad81b137ff7e0a9020f00ace",
+                "sha256:838812c65ed5f7c2bd11f7b098d2e5d01685a3f6d1f82849423b570bae698c00",
+                "sha256:83ea1a398f192957cb986d9206ce229efe0ee75e3c6635baff53ddf39bd718d5",
+                "sha256:8421aa8c9b45ea608c205db9e1c0c855c7e54d0e9c2c2f337ce024f6843cab3b",
+                "sha256:858375573c9225cc8e5b49bfac846a77b696b8d5e815711b8d4ba3141e6e8879",
+                "sha256:86de64468cad9c6d269f32a6390e210ca5ada568c7a55de8e681ca3b897bb340",
+                "sha256:87f7ac99b15270db8d53f28c3c7b968612993a90a5cf359da354efe96f5372b4",
+                "sha256:8bad8210ad4df68c44ff3685cca3cda448ee46e20d13edcff8909eba6ec01ca4",
+                "sha256:8bb4af15f305056e95ca1bd086239b9ebc6ad55e9f49076d27d80027f72752f6",
+                "sha256:8c78bfe20d4c890cb5580a3b9290f700c570e167d4cdcc55feec07030297a5e3",
+                "sha256:8f3f3154fde2b1ff3aa7b4f9326347ebc89c8ef425ca1db8f665175e6d3bd42f",
+                "sha256:94010bd61bc168c103a5b3b0f56ed3b616688192db7cd5b1d626e49f28ff51b3",
+                "sha256:941fab0073f0a54dc33d1a0460cb04e0d85893cb0c5e1476c785000f8b359409",
+                "sha256:9dca7c3956b03b7663fac4d150f5e6d4f6f38b2462c1e9afd83bcf7019f17913",
+                "sha256:a180dbd5ea5d47c2d3b716d5c19cc3fb162d1c8db93b21a1295d69585bfddac1",
+                "sha256:a2712aee7b3834ace51738c15d9ee152cc5a98dc7d57dd93300461b792ab7b43",
+                "sha256:a435ef8a3bd95c8a2d316d6e0ff70d0db524f6037411652803e118871d703333",
+                "sha256:abb756147314430bee5d10919b8493c0ccb109ddb7f5dfd2fcd7441266a25b75",
+                "sha256:abe6eb10122f0d746a0d510c2039ae8edb27bc9af29f6d1b05a66cc2401353ff",
+                "sha256:acbd0a6d61cc954b9f535daaa9ec26b0a60a0d4353c5f7c1438ebc88a359a47e",
+                "sha256:ae08ac90aa8fa14caafc7a6251bd218bf6dac518b7bff09caaa5e781119ba3f2",
+                "sha256:ae61446166983c663cee42c852ed63899e43e484abf080089f771df4b9d272ef",
+                "sha256:afe1f3bc486d0ce40abb0a0c9adb39aed3bbac36ebdc596487b0cceba55c21c1",
+                "sha256:b946da90dc2799bcafa682692c1d2139b2a96ec3c24fa9fc6f5b0da782675330",
+                "sha256:b947e264f0e77d30dcbccbb00f49f900b204b922eb0c3a9f0afd61aaa1cedc3d",
+                "sha256:bb5635c851eef3a7a54becde6da99485eecf7d068bd885ac8e6d173c4ecd68b0",
+                "sha256:bcbebd369493d68162cddb74a9c1fcebd139dfbb7ddb23d8f8e43e6c87bac3a6",
+                "sha256:c31805d2c8ade9b11feca4674eee2b9cce1fec3e8ddb7bbdd961a09dc76a80ea",
+                "sha256:c8840f064b1fb377cffd3efeaad2b190c14d4c8da02316dae07571252d20b31f",
+                "sha256:ccb94342d13e3bf3ffa6e62f95b5e3f0bc6bfa94558cb37f4b3d09d6feb536ff",
+                "sha256:d66689e840e75221b0b290b0befa86f059fb35e1ee6443bce51516d4d61b6b99",
+                "sha256:dabf1a05318d95b1537fd61d9330ef4313ea1216eea128a17615038859da3b3b",
+                "sha256:db03704b3506455d86ec72c3358a779e9b1d07b61220dfb43702b7b668edcd0d",
+                "sha256:de4217b9eb8b541cf2b7fde4401ce9d9a411cc0af85d410f9d6f4333f43640be",
+                "sha256:df0841f94928f8af9c7a1f0aaaffba1fb74607af023a152f59379c01c53aee58",
+                "sha256:dfb992dbcd88d8254471760879d48fb20836d91baa90f181c957122f9592b3dc",
+                "sha256:e7e66b4e403c2836ac74f26c4b65d8ac0ca1eef41dfcac2d013b7482befaad83",
+                "sha256:e8012bce6836d3f20a6c9599f81dfa945f433dab4dbd0c4917a6fb1f998ab33d",
+                "sha256:f01de4ec083daebf210531e2cca3bdb1608dbbbe00a9723e261d92087a1f6ebc",
+                "sha256:f0d945a85b70da97ae86113faf9f1b9294efe66bd4a5d6f82f2676d567338b66",
+                "sha256:fa0ae3275ef706c0309556061185dd0e4c4cd3b7d6f67ae617e4e677c7a41e2e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==23.2.1"
+            "version": "==24.0.1"
         },
         "qtconsole": {
             "hashes": [
-                "sha256:b73723fac43938b684dcb237a88510dc7721c43a726cea8ade179a2927c0a2f3",
-                "sha256:d364592d7ede3257f1e17fcdbfd339c26e2cc638ca4fa4ee56a724e26ea13c81"
+                "sha256:8eadf012e83ab018295803c247c6ab7eacd3d5ab1e1d88a0f37fdcfdab9295a3",
+                "sha256:c29d24464f57cdbaa17d6f6060be6e6d5e29126e7feb57eebc1747433382b3d1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.1"
+            "version": "==5.3.2"
         },
         "qtpy": {
             "hashes": [
-                "sha256:d283cfba378b0dbe36a55b68aea8ee2f86cd6ccf06c023af25bbe705ffbb29e5",
-                "sha256:d85f1b121f24a41ad26c55c446e66abdb7c528839f8c4f11f156ec4541903914"
+                "sha256:268cf5328f41353be1b127e04a81bc74ec9a9b54c9ac75dd8fe0ff48d8ad6ead",
+                "sha256:7d5231133b772e40b4ee514b6673aca558331e4b88ca038b26c9e16c5c95524f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.2.0"
+            "version": "==2.2.1"
         },
         "rasterio": {
             "hashes": [
@@ -1565,32 +1577,30 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:01c2015e132774feefe059d5354055fec6b751d7a7d70ad2cf5ce314e7426e2a",
-                "sha256:0424d1bbbfa51d5ddaa16d067fd593863c9f2fb7c6840c32f8a08a8832f8e7a4",
-                "sha256:10417935486b320d98536d732a58362e3d37e84add98c251e070c59a6bfe0863",
-                "sha256:12005d30894e4fe7b247f7233ba0801a341f887b62e2eb99034dd6f2a8a33ad6",
-                "sha256:16207622570af10f9e6a2cdc7da7a9660678852477adbcd056b6d1057a036fef",
-                "sha256:45f0d6c0d6e55582d3b8f5c58ad4ca4259a02affb190f89f06c8cc02e21bba81",
-                "sha256:5d1b9cf3771fd921f7213b4b886ab2606010343bb36259b544a816044576d69e",
-                "sha256:693b3fe2e7736ce0dbc72b4d933798eb6ca8ce51b8b934e3f547cc06f48b2afb",
-                "sha256:73b704c5eea9be811919cae4caacf3180dd9212d9aed08477c1d2ba14900a9de",
-                "sha256:79dd7876614fc2869bf5d311ef33962d2066ea888bc66c80fd4fa80f8772e5a9",
-                "sha256:7bad16b91918bf3288089a78a4157e04892ea6475fb7a1d9bcdf32c30c8a3dba",
-                "sha256:8d541db2d441ef87afb60c4a2addb00c3af281633602a4967e733ef4b7050504",
-                "sha256:8f2232c9d9119ec356240255a715a289b3a33be828c3e4abac11fd052ce15b1e",
-                "sha256:97a1f1e51ea30782d7baa8d0c52f72c3f9f05cb609cf1b990664231c5102bccd",
-                "sha256:adb6c438c6ef550e2bb83968e772b9690cb421f2c6073f9c2cb6af15ee538bc9",
-                "sha256:bb687d245b6963673c639f318eea7e875d1ba147a67925586abed3d6f39bb7d8",
-                "sha256:bd490f77f35800d5620f4d9af669e372d9a88db1f76ef219e1609cc4ecdd1a24",
-                "sha256:c0dfd7d2429452e7e94904c6a3af63cbaa3cf51b348bd9d35b42db7e9ad42791",
-                "sha256:d3a326673ac5afa9ef5613a61626b9ec15c8f7222b4ecd1ce0fd8fcba7b83c59",
-                "sha256:e2004d2a3c397b26ca78e67c9d320153a1a9b71ae713ad33f4a3a3ab3d79cc65",
-                "sha256:e2ac088ea4aa61115b96b47f5f3d94b3fa29554340b6629cd2bfe6b0521ee33b",
-                "sha256:f7c3c578ff556333f3890c2df6c056955d53537bb176698359088108af73a58f",
-                "sha256:fc58c3fcb8a724b703ffbc126afdca5a8353d4d5945d5c92db85617e165299e7"
-            ],
-            "markers": "python_version < '3.12' and python_version >= '3.8'",
-            "version": "==1.9.0"
+                "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31",
+                "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108",
+                "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0",
+                "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b",
+                "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e",
+                "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e",
+                "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5",
+                "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840",
+                "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58",
+                "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523",
+                "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd",
+                "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab",
+                "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c",
+                "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb",
+                "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096",
+                "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0",
+                "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc",
+                "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9",
+                "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c",
+                "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95",
+                "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.9.3"
         },
         "seaborn": {
             "hashes": [
@@ -1609,11 +1619,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:10602cd0a6f5feab6656e9587f9075292ab777c5200f3bf00293ecd23d9f2788",
-                "sha256:d2e010624c781b26ad6629a8de9832327cf853dea93894487979e55f9ad06857"
+                "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17",
+                "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==65.1.0"
+            "version": "==65.5.0"
         },
         "shapely": {
             "hashes": [
@@ -1717,11 +1727,11 @@
         },
         "sniffio": {
             "hashes": [
-                "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663",
-                "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"
+                "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101",
+                "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==1.2.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.0"
         },
         "snuggs": {
             "hashes": [
@@ -1740,10 +1750,10 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7",
-                "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa"
+                "sha256:8e515439f818efaa251036af72d89e4026e2b03993f3453c000b200fb4f2d6aa",
+                "sha256:b92d206ef355a367d14316b786ab41cb99eb453a21f2cb216a4204625ff7bc07"
             ],
-            "version": "==0.4.0"
+            "version": "==0.6.0"
         },
         "tables": {
             "hashes": [
@@ -1777,19 +1787,27 @@
         },
         "terminado": {
             "hashes": [
-                "sha256:0d5f126fbfdb5887b25ae7d9d07b0d716b1cc0ccaacc71c1f3c14d228e065197",
-                "sha256:ab4eeedccfcc1e6134bfee86106af90852c69d602884ea3a1e8ca6d4486e9bfe"
+                "sha256:520feaa3aeab8ad64a69ca779be54be9234edb2d0d6567e76c93c2c9a4e6e43f",
+                "sha256:bf6fe52accd06d0661d7611cc73202121ec6ee51e46d8185d489ac074ca457c2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.15.0"
+            "version": "==0.17.0"
         },
         "tinycss2": {
             "hashes": [
-                "sha256:b2e44dd8883c360c35dd0d1b5aad0b610e5156c2cb3b33434634e539ead9d8bf",
-                "sha256:fe794ceaadfe3cf3e686b22155d0da5780dd0e273471a51846d0a02bc204fec8"
+                "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847",
+                "sha256:8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.1.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.2.1"
+        },
+        "tomli": {
+            "hashes": [
+                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.1"
         },
         "tornado": {
             "hashes": [
@@ -1818,19 +1836,19 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:0bb9f1f9f017aa8ec187d8b1b2a7a6626a2a1d877116baba52a129bfa124f8e2",
-                "sha256:65fa18961659635933100db8ca120ef6220555286949774b9cfc106f941d1c7a"
+                "sha256:1201b2c9f76097195989cdf7f65db9897593b0dfd69e4ac96016661bb6f0d30f",
+                "sha256:b122f9ff2f2f6c1709dab289a05555be011c87828e911c0cf4074b85cb780a79"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.0"
+            "version": "==5.5.0"
         },
         "urllib3": {
             "hashes": [
-                "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc",
-                "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"
+                "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
+                "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
-            "version": "==1.26.11"
+            "version": "==1.26.12"
         },
         "wcwidth": {
             "hashes": [
@@ -1848,19 +1866,19 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:5d55652dc1d0b3c734f044337d929aaf83f4f9138816ec680c1aefefb4dc4877",
-                "sha256:d58c5f284d6a9bf8379dab423259fe8f85b70d5fa5d2916d5791a84594b122b1"
+                "sha256:398909eb7e261f44b8f4bd474785b6ec5f5b499d4953342fe9755e01ef624090",
+                "sha256:f9611eb65c8241a67fb373bef040b3cf8ad377a9f6546a12b620b6511e8ea9ef"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.3.3"
+            "version": "==1.4.1"
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:954e0faefdd414e4e013f17dbc7fd86f24cf1d243a3ac85d5f0fc2c2d2b50c66",
-                "sha256:9c84ae64c2893c7cbe2eaafc7505221a795c27d68938454034ac487319a75b10"
+                "sha256:34824864c062b0b3030ad78210db5ae6a3960dfb61d5b27562d6631774de0286",
+                "sha256:7f3b0de8fda692d31ef03743b598620e31c2668b835edbd3962d080ccecf31eb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.6.1"
+            "version": "==4.0.3"
         },
         "xarray": {
             "hashes": [
@@ -1872,20 +1890,20 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2",
-                "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"
+                "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1",
+                "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.8.1"
+            "version": "==3.10.0"
         }
     },
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:c61e16246ecfb2cde2958406b4c8ebc043c9e6d73aaa83c941673b35e5d3a76b",
-                "sha256:e3305297c744ae53ffa032c45dc347286165e4ffce6875dc662b205db0623d86"
+                "sha256:1b28ed85e254b724439afc783d4bee767f780b936c3fe8b3275332f42cf5f561",
+                "sha256:4aa76401a151c8cc572d906aad7aea2a841780834a19d780f4321c0fe1b54635"
             ],
-            "version": "==2.0.8"
+            "version": "==2.1.0"
         },
         "backcall": {
             "hashes": [
@@ -1904,18 +1922,18 @@
         },
         "executing": {
             "hashes": [
-                "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e",
-                "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"
+                "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc",
+                "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"
             ],
-            "version": "==0.10.0"
+            "version": "==1.2.0"
         },
         "ipython": {
             "hashes": [
-                "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1",
-                "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd"
+                "sha256:7c959e3dedbf7ed81f9b9d8833df252c430610e2a4a6464ec13cd20975ce20a5",
+                "sha256:91ef03016bcf72dd17190f863476e7c799c6126ec7e8be97719d1bc9a78a59a4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.4.0"
+            "version": "==8.6.0"
         },
         "jedi": {
             "hashes": [
@@ -1958,11 +1976,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0",
-                "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"
+                "sha256:9696f386133df0fc8ca5af4895afe5d78f5fcfe5258111c2a79a1c3e41ffa96d",
+                "sha256:9ada952c9d1787f52ff6d5f3484d0b4df8952787c087edf6a1f7c2cb1ea88148"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.30"
+            "version": "==3.0.31"
         },
         "ptyprocess": {
             "hashes": [
@@ -1987,14 +2005,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==2.13.0"
         },
-        "setuptools": {
-            "hashes": [
-                "sha256:10602cd0a6f5feab6656e9587f9075292ab777c5200f3bf00293ecd23d9f2788",
-                "sha256:d2e010624c781b26ad6629a8de9832327cf853dea93894487979e55f9ad06857"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==65.1.0"
-        },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@@ -2005,18 +2015,18 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7",
-                "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa"
+                "sha256:8e515439f818efaa251036af72d89e4026e2b03993f3453c000b200fb4f2d6aa",
+                "sha256:b92d206ef355a367d14316b786ab41cb99eb453a21f2cb216a4204625ff7bc07"
             ],
-            "version": "==0.4.0"
+            "version": "==0.6.0"
         },
         "traitlets": {
             "hashes": [
-                "sha256:0bb9f1f9f017aa8ec187d8b1b2a7a6626a2a1d877116baba52a129bfa124f8e2",
-                "sha256:65fa18961659635933100db8ca120ef6220555286949774b9cfc106f941d1c7a"
+                "sha256:1201b2c9f76097195989cdf7f65db9897593b0dfd69e4ac96016661bb6f0d30f",
+                "sha256:b122f9ff2f2f6c1709dab289a05555be011c87828e911c0cf4074b85cb780a79"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.0"
+            "version": "==5.5.0"
         },
         "wcwidth": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1d0fee582..635d862a0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.11.1 - 2022-11-01 - [PR #732](https://github.com/NOAA-OWP/inundation-mapping/pull/732)
+
+Due to a recent IT security scan, it was determined that Jupyter-core needed to be upgraded.
+
+### Changes
+
+- `Pipfile` and `Pipfile.lock`:  Added a specific version of Jupyter Core that is compliant with IT.
+
+<br/><br/>
+
 ## v4.0.11.0 - 2022-09-21 - [PR #690](https://github.com/NOAA-OWP/inundation-mapping/pull/690)
 
 Masks levee-protected areas from Relative Elevation Model if branch 0 or if branch stream order exceeds a threshold.

From 8bedee7bc76ed7edd2de9108bd2fffd04be7b6af Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 23 Nov 2022 10:50:42 -0600
Subject: [PATCH 176/359] v4.0.11.2 Fim4 branch zero override (#737)

---
 config/deny_gms_branch_zero.lst |  2 +-
 docs/CHANGELOG.md               | 22 ++++++++
 gms_pipeline.sh                 | 79 ++++++++++++++++++++--------
 gms_run_branch.sh               | 93 ++++++++++++++++++++++++++-------
 gms_run_unit.sh                 | 52 ++++++++++++++----
 src/gms/run_by_branch.sh        |  4 +-
 src/gms/run_by_unit.sh          | 10 ++--
 src/output_cleanup.py           | 14 +++++
 8 files changed, 216 insertions(+), 60 deletions(-)

diff --git a/config/deny_gms_branch_zero.lst b/config/deny_gms_branch_zero.lst
index 5b6543be1..2167ee332 100644
--- a/config/deny_gms_branch_zero.lst
+++ b/config/deny_gms_branch_zero.lst
@@ -16,7 +16,7 @@ demDerived_reaches_{}.shp
 demDerived_reaches_{}.shx
 demDerived_reaches_split_{}.gpkg
 demDerived_reaches_split_filtered_{}.gpkg
-demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+#demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
 demDerived_reaches_split_points_{}.gpkg
 demDerived_streamPixels_{}.tif
 demDerived_streamPixels_ids_{}.tif
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 635d862a0..9dd405adf 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,28 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.11.2 - 2022-11-07 - [PR #737](https://github.com/NOAA-OWP/inundation-mapping/pull/737)
+
+Add an extra input args to the gms_**.sh files to allow for an override of the branch zero deny list, same as we can do with the unit and branch deny list overrides. This is needed for debugging purposes.
+
+Also, if there is no override for the deny branch zero list and is not using the word "none", then use the default or overridden standard branch deny list.  This will keep the branch zero's and branch output folders similar but not identical for outputs.
+
+### Changes
+
+- `gms_pipeline.sh`:  Add new param to allow for branch zero deny list override. Plus added better logic for catching bad deny lists earlier.
+- `gms_run_branch.sh`:  Add new param to allow for branch zero deny list override.  Add logic to cleanup all branch zero output folders with the default branch deny list (not the branch zero list), UNLESS an override exists for the branch zero deny list.
+- `gms_run_unit.sh`: Add new param to allow for branch zero deny list override.
+- `config`
+    - `deny_gms_branch_zero.lst`: update to keep an additional file in the outputs.
+- `src`
+    - `output_cleanup.py`: added note saying it is deprecated.
+    - `gms`
+        - `run_by_branch.sh`: variable name change (matching new names in related files for deny lists)
+        - `run_by_unit.sh`: Add new param to allow for branch zero deny list override.
+
+<br/><br/>
+
+
 ## v4.0.11.1 - 2022-11-01 - [PR #732](https://github.com/NOAA-OWP/inundation-mapping/pull/732)
 
 Due to a recent IT security scan, it was determined that Jupyter-core needed to be upgraded.
diff --git a/gms_pipeline.sh b/gms_pipeline.sh
index 103c76a5c..4aa9c3216 100755
--- a/gms_pipeline.sh
+++ b/gms_pipeline.sh
@@ -7,7 +7,7 @@ usage ()
     echo 'Usage : gms_pipeline.sh [REQ: -u <hucs> - -n <run name> ]'
     echo '                        [OPT: -h -c <config file> -j <job limit>] -o -r'
     echo '                         -ud <unit deny list file> -bd <branch deny list file>'
-    echo '                         -a <use all stream orders> ]'
+    echo '                         -zd <branch zero deny list file> -a <use all stream orders> ]'
     echo ''
     echo 'REQUIRED:'
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
@@ -18,20 +18,27 @@ usage ()
     echo '  -h/--help       : help file'
     echo '  -c/--config     : configuration file with bash environment variables to export'
     echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
+    echo '  -ud/--unitDenylist : A file with a line delimited list of files in UNIT (HUC) directories to be removed'
+    echo '                    upon completion (see config/deny_gms_unit_prod.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_prod.lst'
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'
+    echo '  -bd/--branchDenylist : A file with a line delimited list of files in BRANCHES directories to be removed' 
+    echo '                    upon completion of branch processing.'
+    echo '                    (see config/deny_gms_branches_prod.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'   
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'
+    echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
+    echo '                    be removed upon completion of branch zero processing.'
+    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'    
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
     echo '  -r/--retry      : retries failed jobs'
-    echo '  -ud/--unitDenylist : A file with line delimited list of files in UNIT (HUC) directories to remove'
-    echo '                    upon completion (see config/deny_gms_unit_prod.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_prod.lst'
-    echo '                    -- Note: if you want all output files (aka.. no files removed),'
-    echo '                    use the word none as this value for this parameter.'
-    echo '  -bd/--branchDenylist : A file with line delimited list of files in branches directories to remove' 
-    echo '                    upon completion (see config/deny_gms_branches_prod.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'   
-    echo '                    -- Note: if you want all output files (aka.. no files removed),'
-    echo '                    use the word none as this value for this parameter.'
 	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
     echo '                    at the initial load of the nwm_subset_streams.'
     echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
@@ -73,12 +80,16 @@ in
         ;;
     -ud|--unitDenylist)
         shift
-        deny_gms_unit_list=$1
+        deny_unit_list=$1
         ;;
     -bd|--branchDenylist)
         shift
-        deny_gms_branches_list=$1
+        deny_branches_list=$1
         ;;
+    -zd|--branchZeroDenylist)
+        shift
+        deny_branch_zero_list=$1
+        ;;        
     -a|--useAllStreamOrders)
         useAllStreamOrders=1
         ;;
@@ -103,14 +114,7 @@ if [ "$envFile" = "" ]
 then
     envFile=/foss_fim/config/params_template.env
 fi
-if [ "$deny_gms_unit_list" = "" ]
-then
-   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_prod.lst
-fi
-if [ "$deny_gms_branches_list" = "" ]
-then
-   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_prod.lst
-fi
+
 if [ -z "$overwrite" ]
 then
     # default is false (0)
@@ -121,6 +125,35 @@ then
     retry=""
 fi
 
+# The tests for the deny lists are duplicated here on to help catch
+# them earlier (ie.. don't have to wait to process units to find an
+# pathing error with the branch deny list)
+if [ "$deny_unit_list" != "" ] && \
+   [ "${deny_unit_list^^}" != "NONE" ] && \
+   [ ! -f "$deny_unit_list" ]
+then
+    # NONE is not case sensitive
+    echo "Error: The -ud <unit deny file> does not exist and is not the word NONE"
+    usage
+fi
+
+if [ "$deny_branches_list" != "" ] && \
+   [ "${deny_branches_list^^}" != "NONE" ] && \
+   [ ! -f "$deny_branches_list" ]
+then
+    # NONE is not case sensitive
+    echo "Error: The -bd <branch deny file> does not exist and is not the word NONE"
+    usage
+fi
+
+if [ "$deny_branch_zero_list" != "" ] && \
+   [ "${deny_branch_zero_list^^}" != "NONE" ] && \
+   [ ! -f "$deny_branch_zero_list" ]
+then
+    echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
+    usage
+fi
+
 # invert useAllStreamOrders boolean (to make it historically compatiable
 # with other files like gms/run_unit.sh and gms/run_branch.sh).
 # Yet help user understand that the inclusion of the -a flag means
@@ -172,7 +205,7 @@ if [ $overwrite -eq 1 ]; then run_cmd+=" -o" ; fi
 if [ "$retry" == "--retry-failed" ]; then run_cmd+=" -r" ; fi
 if [ $dropLowStreamOrders -eq 1 ]; then run_cmd+=" -s" ; fi
 #echo "$run_cmd"
-. /foss_fim/gms_run_unit.sh -u "$hucList" $run_cmd -d $deny_gms_unit_list
+. /foss_fim/gms_run_unit.sh -u "$hucList" $run_cmd -ud "$deny_unit_list" -zd "$deny_branch_zero_list"
 
 ## CHECK IF OK TO CONTINUE ON TO BRANCH STEPS
 # Count the number of files in the $outputRunDataDir/unit_errors
@@ -188,7 +221,7 @@ python3 $srcDir/check_unit_errors.py -f $outputRunDataDir -n $num_hucs
 
 
 ## Produce level path or branch level datasets
-. /foss_fim/gms_run_branch.sh $run_cmd -d $deny_gms_branches_list
+. /foss_fim/gms_run_branch.sh $run_cmd -bd "$deny_branches_list" -zd "$deny_branch_zero_list"
 
 
 echo "======================== End of gms_pipeline.sh =========================="
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index b72705bad..f78e29e0d 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -4,8 +4,9 @@ usage ()
 {
     echo 'Produce GMS hydrofabric at levelpath/branch scale. Execute gms_run_unit.sh prior to.'
     echo 'Usage : gms_run_branch.sh [REQ: -n <run name> ]'
-    echo '  	 				    [OPT: -h -j <job limit> -o -r -d <deny list file>'
-    echo '                                -u <hucs> -a <use all stream orders>]'    
+    echo '  	 				    [OPT: -h -j <job limit> -o -r '
+    echo '  	 				    -bd <branch deny list file> -zd <branch zero deny list file>'
+    echo '                          -u <hucs> -a <use all stream orders>]'
     echo ''
     echo 'REQUIRED:'
     echo '  -n/--runName    : A name to tag the output directories and log files as. could be a version tag.'
@@ -14,17 +15,24 @@ usage ()
     echo '  -h/--help       : help file'
     echo '  -c/--config     : configuration file with bash environment variables to export'    
     echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
+    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited).'
+    echo '                    A line delimited file also acceptable. HUCs must present in inputs directory.'
+    echo '  -bd/--branchDenylist : A file with a line delimited list of files in BRANCHES directories to be removed' 
+    echo '                    upon completion of branch processing.'
+    echo '                    (see config/deny_gms_branches_prod.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'   
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'
+    echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
+    echo '                    be removed upon completion of branch zero processing.'
+    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'    
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -r/--retry      : retries failed jobs'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -d/--denylist    : A file with line delimited list of files in branches directories to remove' 
-    echo '                    upon completion (see config/deny_gms_branches_prod.lst for a starting point)'
-    echo '                    -- Note: if you want all output files (aka.. no files removed),'
-    echo '                    use the word none as this value for this parameter.'
-    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'    
-    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited).'
-    echo '                    A line delimited file also acceptable. HUCs must present in inputs directory.'
     echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
     echo '                    at the initial load of the nwm_subset_streams.'
     echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
@@ -62,10 +70,14 @@ in
         retry="--retry-failed"
         overwrite=1
         ;;
-    -d|--denylist)
+    -bd|--branchDenylist)
         shift
-        deny_gms_branches_list=$1
+        deny_branches_list=$1
         ;;
+    -zd|--branchZeroDenylist)
+        shift
+        deny_branch_zero_list_for_branches=$1
+        ;;        
     -a|--useAllStreamOrders)
         useAllStreamOrders=1
         ;;
@@ -85,9 +97,36 @@ if [ "$envFile" = "" ]
 then
     envFile=/foss_fim/config/params_template.env
 fi
-if [ "$deny_gms_branches_list" = "" ]
+
+if [ "$deny_branches_list" = "" ]
+then
+   deny_branches_list=/foss_fim/config/deny_gms_branches_prod.lst
+elif [ "${deny_branches_list^^}" != "NONE" ] && [ ! -f "$deny_branches_list" ]
+then
+    # NONE is not case sensitive
+    echo "Error: The -bd <branch deny file> does not exist and is not the word NONE"
+    usage
+fi
+
+# Yes.. we have to have a different variable names for the deny_branch_zero_list_for_branches
+# and deny_branch_zero_list_for_units. While they both use the same input arg, they use
+# the value slightly different and when using gms_pipeline, the values can impact
+# other bash files.
+if [ "$deny_branch_zero_list_for_branches" = "" ]
 then
-   deny_gms_branches_list=/foss_fim/config/deny_gms_branches_prod.lst
+    deny_branch_zero_list_for_branches=/foss_fim/config/deny_gms_branch_zero.lst
+elif [ "${deny_branch_zero_list_for_branches^^}" != "NONE" ]   # NONE is not case sensitive
+then
+    if [ ! -f "$deny_branch_zero_list_for_branches" ]
+    then
+        echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
+        usage
+    else
+        # only if the deny branch zero has been overwritten and file exists
+        has_deny_branch_zero_override=1 
+    fi
+else
+    has_deny_branch_zero_override=1 # it is the value of NONE and is overridden
 fi
 
 if [ "$overwrite" = "" ]
@@ -122,7 +161,7 @@ fi
 
 ## Define Outputs Data Dir & Log File##
 export outputRunDataDir=$outputDataDir/$runName
-export deny_gms_branches_list=$deny_gms_branches_list
+export deny_branches_list=$deny_branches_list
 logFile=$outputRunDataDir/logs/branch/summary_gms_branch.log
 export extent=GMS
 export overwrite=$overwrite
@@ -243,11 +282,27 @@ if [ "$src_adjust_spatial" = "True" ]; then
 fi
 
 # -------------------
-## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO##
-if [ -f $deny_gms_branches_list ]; then
-    echo -e $startDiv"Cleaning up (Removing) files all branch zero for all HUCs"$stopDiv
-    date -u
-    $srcDir/gms/outputs_cleanup.py -d $outputRunDataDir -l $deny_gms_branches_list -b 0
+## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO (but using normal branch deny) ##
+## but also do not remove if branch zero deny is NONE (any case)
+
+# If the deny branch zero has been overridden, then use it (file path proven above).
+# Override might be the value of None (case not sensitive)
+# Else then use the default deny branch (not the zero) which might also be None and that is ok
+
+# If deny branch zero is NONE.. then skip
+# if deny branch zero has an override deny list, use it.
+# if deny branch zero is not NONE and deny list is not overridden
+#    then see if reg deny branch is NONE.  If so.. skip
+#    else, use the deny branch list instead to do final cleanup on branch zero
+
+if [ "$has_deny_branch_zero_override" == "1" ]
+then
+    echo -e $startDiv"Cleaning up (Removing) files for branch zero for all HUCs"
+    $srcDir/gms/outputs_cleanup.py -d $outputRunDataDir -l $deny_branch_zero_list_for_branches -b 0
+
+else 
+    echo -e $startDiv"Cleaning up (Removing) files all branch zero for all HUCs using the default branch deny list"
+    $srcDir/gms/outputs_cleanup.py -d $outputRunDataDir -l $deny_branches_list -b 0
 fi
 
 # -------------------
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index 285a4b05d..f676f94cd 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -5,7 +5,8 @@ usage ()
     echo 'Produce GMS hydrofabric datasets for unit scale.'
     echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -n <run name> ]'
     echo '                        [OPT: -h -j <job limit>]  -c <config file>'
-    echo '                         -o -r -d <deny list file> -a <use all stream orders>]'
+    echo '                         -o -r -ud <unit deny list file>'
+    echo '                         -zd <branch zero deny list file> -a <use all stream orders>]'
     echo ''
     echo 'REQUIRED:'
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
@@ -16,14 +17,21 @@ usage ()
     echo '  -h/--help       : help file'
     echo '  -c/--config     : configuration file with bash environment variables to export'
     echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'        
+    echo '  -ud/--unitDenylist : A file with a line delimited list of files in UNIT (HUC) directories to be removed'
+    echo '                    upon completion (see config/deny_gms_unit_prod.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_prod.lst'
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'
+    echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
+    echo '                    be removed upon completion of branch zero processing.'
+    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'    
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
     echo '  -r/--retry      : retries failed jobs'
-    echo '  -d/--denylist   : file with line delimited list of files in huc directories to remove upon completion'
-    echo '                   (see config/deny_gms_unit_prod.lst for a starting point)'
-    echo '                    -- Note: if you want all output files (aka.. no files removed),'
-    echo '                    use the word none as this value for this parameter.'
 	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
 	echo '                    at the initial load of the nwm_subset_streams.'
 	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
@@ -61,10 +69,14 @@ in
         retry="--retry-failed"
         overwrite=1
         ;;
-    -d|--denylist)
+    -ud|--unitDenylist)
         shift
-        deny_gms_unit_list=$1
+        deny_unit_list=$1
         ;;
+    -zd|--branchZeroDenylist)
+        shift
+        deny_branch_zero_list_for_units=$1
+        ;;        
     -a|--useAllStreamOrders)
         useAllStreamOrders=1
         ;;
@@ -89,9 +101,27 @@ if [ "$envFile" = "" ]
 then
     envFile=/foss_fim/config/params_template.env
 fi
-if [ "$deny_gms_unit_list" = "" ]
+
+if [ "$deny_unit_list" = "" ]
+then
+    deny_unit_list=/foss_fim/config/deny_gms_unit_prod.lst
+elif [ "${deny_unit_list^^}" != "NONE" ] && [ ! -f "$deny_unit_list" ]
+then
+    # NONE is not case sensitive
+    echo "Error: The -ud <unit deny file> does not exist and is not the word NONE"
+    usage
+fi
+
+if [ "$deny_branch_zero_list_for_units" = "" ]
+then
+    deny_branch_zero_list_for_units=/foss_fim/config/deny_gms_branch_zero.lst
+elif [ "${deny_branch_zero_list_for_units^^}" != "NONE" ]   # NONE is not case sensitive
 then
-   deny_gms_unit_list=/foss_fim/config/deny_gms_unit_prod.lst
+    if [ ! -f "$deny_branch_zero_list_for_units" ]
+    then
+        echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
+        usage
+    fi
 fi
 
 if [ -z "$overwrite" ]
@@ -151,6 +181,10 @@ export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Levee_protected
 export deny_gms_unit_list=$deny_gms_unit_list
 export extent=GMS
 
+export deny_unit_list=$deny_unit_list
+export deny_branch_zero_list_for_units=$deny_branch_zero_list_for_units
+
+
 # we are not using the variable output at this time, but keep it anways
 num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
 
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index 5c547cb4c..a7dd1c3f8 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -104,11 +104,11 @@ if [ -f $outputHucDataDir/usgs_subset_gages.gpkg ]; then
 fi
 
 ## REMOVE FILES FROM DENY LIST ##
-if [ -f $deny_gms_branches_list ]; then
+if [ -f $deny_branches_list ]; then
     echo -e $startDiv"Remove files $hucNumber $current_branch_id"$stopDiv
     date -u
     Tstart
-    $srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_gms_branches_list -b $current_branch_id
+    $srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branches_list -b $current_branch_id
     Tcount
 fi
 
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 9b6124946..88fd96960 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -267,16 +267,14 @@ fi
 
 ## CLEANUP BRANCH ZERO OUTPUTS ##
 echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
-date -u
-Tstart
-$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $srcDir/../config/deny_gms_branch_zero.lst -b 0
-Tcount
+$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branch_zero_list_for_units -b 0
+
 
 ## REMOVE FILES FROM DENY LIST ##
-if [ -f $deny_gms_unit_list ]; then
+if [ -f $deny_unit_list ]; then
     echo -e $startDiv"Remove files $hucNumber"$stopDiv
     date -u
     Tstart
-    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_gms_unit_list -b $hucNumber
+    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
     Tcount
 fi
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
index 146b59308..34b83672a 100755
--- a/src/output_cleanup.py
+++ b/src/output_cleanup.py
@@ -3,6 +3,12 @@
 import argparse
 from utils.shared_functions import mem_profile
 
+##################################
+##
+#### Deprecated (as part of fim_run):  Oct 1, 2022 ####
+##
+##################################
+
 
 @mem_profile
 def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, is_viz_post_processing):
@@ -23,6 +29,14 @@ def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_prod
         Determine whether or not to process outputs for Viz
     '''
 
+    ##################################
+    ##
+    #### Deprecated (as part of fim_run):  Oct 1, 2022 ####
+    print('#### Deprecated (as part of fim_run):  Oct 1, 2022 ####')
+    ##
+    ##################################
+
+
     # List of files that will be saved during a production run
     production_whitelist = [
         'rem_zeroed_masked.tif',

From 557e8511be36b1e24ed8697d41c74a5328be8f99 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 23 Nov 2022 11:04:31 -0600
Subject: [PATCH 177/359] v4.0.11.3 Add tool to preprocess downloaded levee
 prot areas (#739)

---
 data/nld/preprocess_levee_protected_areas.py | 180 +++++++++++++++++++
 data/preprocess_rasters.py                   |   5 +
 docs/CHANGELOG.md                            |  21 ++-
 src/utils/shared_functions.py                |  12 +-
 4 files changed, 214 insertions(+), 4 deletions(-)
 create mode 100644 data/nld/preprocess_levee_protected_areas.py

diff --git a/data/nld/preprocess_levee_protected_areas.py b/data/nld/preprocess_levee_protected_areas.py
new file mode 100644
index 000000000..57654d263
--- /dev/null
+++ b/data/nld/preprocess_levee_protected_areas.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+import argparse
+import geopandas as gpd
+import logging
+import os
+import pathlib
+import sys
+
+from datetime import datetime
+
+sys.path.append('/foss_fim/src')
+import utils.shared_functions as sf
+from utils.shared_functions import FIM_Helpers as fh
+
+def preprocess_levee_protected_areas(source_file_name_and_path,
+                                     target_output_folder_path = '',
+                                     target_output_filename = 'levee_protected_areas.gpkg',
+                                     projection_crs_id = 'EPSG:5070',
+                                     overwrite = False):
+                                     
+    '''
+    Overview
+    ----------
+    This takes a geojson file in (assuming it has geometry), converts it to a gkpg,
+    and converts it's projection (id) (defaulting to EPSG:5070)
+    
+    Parameters:
+    ----------
+        - source_file_name_and_path (str):
+            Name and location of the geojson file. ie) /data/input_temp/LeveedArea.geojson
+        - target_output_folder_path (str):
+            Location where the output gkpg will be saved (defaults to input folder)
+        - target_output_filename (str):
+            Output file name (defaults to levee_protected_areas.gpkg)
+        - projection_crs_id (str):
+            CRS ID that it will be reprojected to (defaults to EPSG:5070). 
+            Remember.. projections can and often do get changed during clipping.
+            
+    Returns:
+    ----------
+        None
+    ''' 
+
+    # -------------------
+    # Validation and setup variables
+    
+    # source file
+    if (source_file_name_and_path is None) or (source_file_name_and_path == ''):
+        raise ValueError("source file name and path can not be empty")
+    source_file_name_and_path = source_file_name_and_path.strip()
+    
+    if (not os.path.exists(source_file_name_and_path)):
+        raise FileNotFoundError("levee geojson source file not found.")
+    
+    source_file_extension = pathlib.Path(source_file_name_and_path).suffix
+    if (source_file_extension.lower() != ".geojson"):
+        raise ValueError("source file does not appear to be a geojson file")
+
+    # target file and path
+    if (target_output_folder_path is not None) and (target_output_folder_path != ''):
+        if (not os.path.exists(target_output_folder_path)):
+            os.mkdir(target_output_folder_path)
+    else:
+        target_output_folder_path = os.path.dirname(source_file_name_and_path)
+        
+    if (target_output_filename is not None) and (target_output_filename != ''):
+        target_file_extension = pathlib.Path(target_output_filename).suffix
+        if (target_file_extension.lower() != ".gpkg"):
+            raise ValueError("target file name does not appear to be a gpkg file")
+    else:
+        target_output_filename = 'levee_protected_areas.gpkg'
+
+    target_output_file = os.path.join(target_output_folder_path, target_output_filename)
+    if (not overwrite) and os.path.exists(target_output_file):
+        raise Exception("Target output file of {target_output_file} already exists. "\
+                        "Add overwrite flag or change target path or target file name")
+       
+    # projection crs
+    if (projection_crs_id is None) or (projection_crs_id == ''):
+        projection_crs_id = 'EPSG:5070'
+
+    # -------------------
+    # setup logs
+    #start_time = datetime.now()
+    print("=======================")
+    print("Starting preprocessing levee protected area geojson file")
+   
+    __setup_logger(target_output_folder_path)
+    #sf.setup_logger('preprocess_level_protected_areas', target_output_folder_path)
+    logging.info(f"Downloading to {target_output_folder_path}")
+    
+    # -------------------
+    # processing
+    
+    # load geojson into a dataframe
+    # Note:: the logs record a weird line from geopandas
+    #    Skipping field stewardOrgIds: invalid type 1 but seems to be fine.
+    gdf = gpd.read_file(source_file_name_and_path)
+    if (len(gdf) == 0) or (gdf.empty):
+        raise Exception("Source geojson file appears to be empty or invalid")
+   
+    # check current CRS
+    gdf_crs = str(gdf.crs)
+    #print(gdf_crs)
+    
+    # don't change it if the projection's already match    
+    if (gdf_crs.lower() != projection_crs_id.lower()):
+        gdf.to_crs(projection_crs_id, inplace=True)           
+    
+    gdf.to_file(target_output_file, driver="GPKG", index=False)
+    
+    msg = f"Preprocessing Complete: file created at {target_output_file}"
+    print(msg)
+    logging.info(msg)
+    
+    #end_time = datetime.now()
+    #logging.info(fh.print_date_time_duration(start_time, end_time))
+        
+def __setup_logger(output_folder_path):
+    
+    root_file_name = "preprocess_level_protected_areas"
+    start_time = datetime.now()
+    file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
+    log_file_name = f"{root_file_name}-{file_dt_string}.log"
+
+    log_file_path = os.path.join(output_folder_path, log_file_name)
+
+    file_handler = logging.FileHandler(log_file_path)
+    file_handler.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.DEBUG)
+
+    logger = logging.getLogger()
+    logger.addHandler(file_handler)
+    logger.setLevel(logging.DEBUG)    
+    
+    logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
+    logging.info("----------------")
+
+        
+
+if __name__ == '__main__':
+    
+    # Acquire Notes as of Nov 8, 2022
+    '''
+    Working with gkpg''s in code is 10 to 20 times faster than working directly with geojson files
+    so we will convert it. We have to download the geojson file by hand for now as USACE doesn't have an API
+    that allows you to download the entire CONUS+. They do have api's that download levee geojson files at a huc
+    level but they don't have geometry. You have to call once for the geojson per huc, then go back for the geometries.
+    For now, we will stay with downloading it by hand.
+    '''
+    
+    # Instructions for Downloading (as of Nov 8, 2022)
+    #  1) go to :https://levees.sec.usace.army.mil/#/levees/search/&viewType=map&resultsType=systems&advanced=true&hideList=false&eventSystem=false
+    #  2) Click on the "Download Data" button in the top right corner and wait for a bit. It takes a bit as it is 200mg but it isn't bad.
+    #     It downloads one big LeevedArea.geojson with geometry.
+
+    # Sample Usage: python3 /foss_fim/data/nld/preprocess_levee_protected_areas.py -s /data/inputs/LeveedArea.geojson
+    
+    parser = argparse.ArgumentParser(description='Preprocess the NLD levee protected areas geojson into a gkpg.')
+
+    parser.add_argument('-s','--source-file-name-and-path', help='location of the geojson that will converted.', 
+                        required=True)
+    
+    parser.add_argument('-t','--target-output-folder-path', help='location of where the gkpg file'\
+                        ' will be saved (defaults to input folder path).', required=False)
+
+    parser.add_argument('-f', '--target-output-filename', help='output gpkg file name'\
+                        ' (defaults to levee_protected_area.gkpg).', required=False)
+
+    parser.add_argument('-c','--projection-crs-id', help='Reproject to CRS ID (default = EPSG:5070).', 
+                        required=False)
+
+    parser.add_argument('-o','--overwrite', help='Overwrite if the file already exists.',
+                        required=False,  action='store_true')
+
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+
+    preprocess_levee_protected_areas(**args)
diff --git a/data/preprocess_rasters.py b/data/preprocess_rasters.py
index 9bcad5fa2..189a30d8a 100755
--- a/data/preprocess_rasters.py
+++ b/data/preprocess_rasters.py
@@ -13,6 +13,11 @@
 from utils.shared_functions import update_raster_profile
 from utils.shared_variables import PREP_PROJECTION, PREP_PROJECTION_CM
 
+##################################
+##
+#### Deprecated (will be replaced with new files and new logic):  Nov 8, 2022 ####
+##
+##################################
 
 if __name__ == '__main__':
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9dd405adf..21ce5dc34 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,26 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.11.3 - 2022-11-10 - [PR #739](https://github.com/NOAA-OWP/inundation-mapping/pull/739)
+
+New tool with instructions of downloading levee protected areas and a tool to pre-process it, ready for FIM.
+
+### Additions
+
+- `data`
+    - `nld`
+         - `preprocess_levee_protected_areas.py`:  as described above
+
+### Changes
+
+- `data`
+     - `preprocess_rasters.py`: added deprecation note. It will eventually be replaced in it's entirety.
+- `src`
+    - `utils`
+        - `shared_functions.py`: a few styling adjustments.
+
+<br/><br/>
+
 ## v4.0.11.2 - 2022-11-07 - [PR #737](https://github.com/NOAA-OWP/inundation-mapping/pull/737)
 
 Add an extra input args to the gms_**.sh files to allow for an override of the branch zero deny list, same as we can do with the unit and branch deny list overrides. This is needed for debugging purposes.
@@ -23,7 +43,6 @@ Also, if there is no override for the deny branch zero list and is not using the
 
 <br/><br/>
 
-
 ## v4.0.11.1 - 2022-11-01 - [PR #732](https://github.com/NOAA-OWP/inundation-mapping/pull/732)
 
 Due to a recent IT security scan, it was determined that Jupyter-core needed to be upgraded.
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index f109ac4ea..df4ce8224 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -12,6 +12,7 @@
 from os.path import splitext
 
 import fiona
+import geopandas as gp
 import numpy as np
 import pandas as pd
 import rasterio
@@ -75,8 +76,6 @@ def run_system_command(args):
 
 def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
 
-    import geopandas as gp
-
 
     print("Subsetting " + wbd_gpkg + "...")
     # Read geopackage into dataframe.
@@ -193,7 +192,12 @@ def update_raster_profile(args):
 
 '''
 
-def reproject_raster(input_raster_name,reprojection,blocksize=None,reprojected_raster_name=None):
+def reproject_raster(input_raster_name,
+                     reprojection, 
+                     blocksize=None,
+                     reprojected_raster_name=None):
+
+    # TODO: Might be removed (Nov 8, 2022), not in use
 
     if blocksize is not None:
         if isinstance(blocksize, int):
@@ -253,6 +257,7 @@ def reproject_raster(input_raster_name,reprojection,blocksize=None,reprojected_r
                 del dst
         del src
 
+
 def mem_profile(func):
     def wrapper(*args, **kwargs):
         if (os.environ.get('mem') == "1"):
@@ -310,6 +315,7 @@ def concat_huc_csv(fim_dir,csv_name):
         return concat_df
 
 
+# -----------------------------------------------------------
 def progress_bar_handler(executor_dict, desc):
 
     for future in tqdm(as_completed(executor_dict),

From 11c85273cff88802914a09b2ecf250433667f498 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 23 Nov 2022 12:08:42 -0500
Subject: [PATCH 178/359] v4.0.11.4 Adds capability to produce single FIM4
 rating curve comparison plots for each gage (#709)

---
 docs/CHANGELOG.md                | 13 ++++-
 tools/rating_curve_comparison.py | 95 +++++++++++++++++++++++++++++++-
 2 files changed, 104 insertions(+), 4 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 21ce5dc34..5f1e1acd9 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.11.4 - 2022-10-12 - [PR #709](https://github.com/NOAA-OWP/inundation-mapping/pull/709)
+
+Adds capability to produce single rating curve comparison plots for each gage.
+
+### Changes
+
+- `tools/rating_curve_comparison.py`
+    - Adds generate_single_plot() to make a single rating curve comparison plot for each gage in a given HUC
+    - Adds command line switch to generate single plots
+    
+<br/><br/>
+
 ## v4.0.11.3 - 2022-11-10 - [PR #739](https://github.com/NOAA-OWP/inundation-mapping/pull/739)
 
 New tool with instructions of downloading levee protected areas and a tool to pre-process it, ready for FIM.
@@ -537,7 +549,6 @@ Prunes branches that fail with NO_FLOWLINES_EXIST (Exit code: 61) in `gms_run_br
 
 <br/><br/>
 
-
 ## v4.0.6.0 - 2022-08-10 - [PR #614](https://github.com/NOAA-OWP/inundation-mapping/pull/614)
 
 Addressing #560, this fix in run_by_branch trims the DEM derived streamline if it extends past the end of the branch streamline. It does this by finding the terminal point of the branch stream, snapping to the nearest point on the DEM derived stream, and cutting off the remaining downstream portion of the DEM derived stream.
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 4cd002ee8..53ed5c97c 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -78,6 +78,7 @@ def generate_rating_curve_metrics(args):
     catfim_flows_filename       = args[7]
     huc                         = args[8]
     alt_plot                    = args[9]
+    single_plot                 = args[10]
 
     logging.info("Generating rating curve metrics for huc: " + str(huc))
     elev_table = pd.read_csv(elev_table_filename,dtype={'location_id': object, 'feature_id':object,'HydroID':object, 'levpa_id':object})
@@ -267,9 +268,10 @@ def generate_rating_curve_metrics(args):
             # plot rating curves
             if alt_plot:
                 generate_rc_and_rem_plots(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table, branches_folder)
+            elif single_plot:
+                generate_single_plot(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table)
             else:
                 generate_facet_plot(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table)
-
         else:
             logging.info(f"no USGS data for gage(s): {relevant_gages} in huc {huc}")
     else:
@@ -322,6 +324,92 @@ def aggregate_metrics(output_dir,procs_list,stat_groups):
     return agg_recurr_stats_table
 
 
+def generate_single_plot(rc, plot_filename, recurr_data_table):
+
+    tmp_rc = rc.copy()
+    
+    # Filter FIM elevation based on USGS data
+    for gage in rc.location_id.unique():
+        rc = rc[rc.location_id==gage]
+
+        plot_filename_splitext = os.path.splitext(plot_filename)
+        gage_plot_filename = plot_filename_splitext[0] + '_' + gage +  plot_filename_splitext[1]
+
+        #print(recurr_data_table.head)
+        try:
+            min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
+            max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
+            min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
+            max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
+            ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
+
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
+
+            if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
+                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
+                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
+        except Exception as ex:
+            summary = traceback.StackSummary.extract(
+                    traceback.walk_stack(None))
+            logging.info("WARNING: rating curve dataframe not processed correctly...")
+
+        rc = rc.rename(columns={"location_id": "USGS Gage"})
+
+        # split out branch 0 FIM data
+        rc['source_branch'] = np.where((rc.source == 'FIM') & (rc.levpa_id == '0'), 'FIM_b0', np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source))
+        #rc['source_branch'] = np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source)
+
+        ## Generate rating curve plots
+        num_plots = len(rc["USGS Gage"].unique())
+        if num_plots > 3:
+            columns = num_plots // 3
+        else:
+            columns = 1
+
+        sns.set(style="ticks")
+
+        # Plot both "FIM" and "FIM_default" rating curves
+        if '0' in rc.levpa_id.values: # checks to see if branch zero data exists in the rating curve df
+            hue_order = ['USGS','FIM','FIM_default','FIM_b0','FIM_default_b0'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM','FIM_b0']
+            kw = {'color': ['blue','green','orange','green','orange'], 'linestyle' : ["-","-","-","--","--"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green','green'], 'linestyle' : ["-","-","--"]}
+        else:
+            hue_order = ['USGS','FIM','FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM']
+            kw = {'color': ['blue','green','orange'], 'linestyle' : ["-","-","-"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green'], 'linestyle' : ["-","_"]}
+        # Facet Grid
+        g = sns.FacetGrid(rc, col="USGS Gage", hue="source_branch", hue_order=hue_order,
+                        sharex=False, sharey=False,col_wrap=columns,
+                        height=3.5, aspect=1.65, hue_kws=kw)
+        g.map(plt.plot, "discharge_cfs", "elevation_ft", linewidth=2, alpha=0.8)
+        g.set_axis_labels(x_var="Discharge (cfs)", y_var="Elevation (ft)")
+
+        ## Plot recurrence intervals
+        axes = g.axes_dict
+        for gage in axes:
+            ax = axes[gage]
+            plt.sca(ax)
+            try:
+                recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
+                    .filter(items=['recurr_interval', 'discharge_cfs'])
+                for i, r in recurr_data.iterrows():
+                    if not r.recurr_interval.isnumeric(): continue # skip catfim flows
+                    l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
+                    plt.axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
+                    plt.text(r.discharge_cfs, ax.get_ylim()[1] - (ax.get_ylim()[1]-ax.get_ylim()[0])*0.03, r.recurr_interval, size='small', c='purple')
+            except Exception as ex:
+                summary = traceback.StackSummary.extract(
+                        traceback.walk_stack(None))
+                logging.info("WARNING: Could not plot recurrence intervals...")
+
+        # Adjust the arrangement of the plots
+        g.fig.tight_layout(w_pad=1)
+        g.add_legend()
+
+        plt.savefig(gage_plot_filename)
+        plt.close()
+
+        rc = tmp_rc
+
 def generate_facet_plot(rc, plot_filename, recurr_data_table):
     
     # Filter FIM elevation based on USGS data
@@ -738,7 +826,7 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     parser.add_argument('-alt','--alt-plot',help='Generate rating curve plots with REM maps',required=False,default=False,action='store_true')
     parser.add_argument('-eval','--evaluate-results',help='Create a boxplot comparison of multiple input Sierra Test results. \
         Expects 2 arguments: 1) path to the Sierra Test results for comparison and 2) the corresponding label for the boxplot.',required=False,nargs=2,action='append')
-
+    parser.add_argument('-s', '--single-plot', help='Create single plots', action='store_true')
     args = vars(parser.parse_args())
 
     fim_dir = args['fim_dir']
@@ -750,6 +838,7 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     stat_groups = args['stat_groups']
     alt_plot = args['alt_plot']
     eval = args['evaluate_results']
+    single_plot = args['single_plot']
     if args['stat_gages']:
         gages_gpkg_filepath = args['stat_gages'][0]
         stat_gages = args['stat_gages'][1]
@@ -792,7 +881,7 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
         if isfile(elev_table_filename):
             procs_list.append([elev_table_filename, branches_folder, usgs_gages_filename, 
                             usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,
-                            nwm_flow_dir, catfim_flows_filename, huc, alt_plot])
+                            nwm_flow_dir, catfim_flows_filename, huc, alt_plot, single_plot])
             # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
             read_elev_table = pd.read_csv(elev_table_filename, dtype={'location_id':str, 'HydroID':str, 'huc':str, 'feature_id':int})
             read_elev_table['huc'] = huc

From d1d74f985fa65404af0d0f915377f576ef912179 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 23 Nov 2022 12:13:17 -0500
Subject: [PATCH 179/359] v4.0.11.5 Skip `usgs_gage_unit_setup.py` if no level
 paths (#746)

---
 docs/CHANGELOG.md      | 10 ++++++++++
 src/gms/run_by_unit.sh | 12 +++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5f1e1acd9..5c11ac681 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.11.5 - 2022-11-18 - [PR #746](https://github.com/NOAA-OWP/inundation-mapping/pull/746)
+
+Skips `src/usgs_gage_unit_setup.py` if no level paths exist. This may happen if a HUC has no stream orders > 2. This is a bug fix for #723 for the case that the HUC also has USGS gages.
+
+### Changes
+
+- `src/gms/run_by_unit.sh`: Adds check for `nwm_subset_streams_levelPaths.gpkg` before running `usgs_gage_unit_setup.py`
+
+<br/><br/>
+
 ## v4.0.11.4 - 2022-10-12 - [PR #709](https://github.com/NOAA-OWP/inundation-mapping/pull/709)
 
 Adds capability to produce single rating curve comparison plots for each gage.
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 88fd96960..618235f35 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -250,11 +250,13 @@ else
 fi
 
 ## CREATE USGS GAGES FILE
-echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id -bzero $dropLowStreamOrders
-Tcount
+if [ -f $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
+    echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
+    date -u
+    Tstart
+    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id -bzero $dropLowStreamOrders
+    Tcount
+fi
 
 ## USGS CROSSWALK ##
 if [ -f $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then

From 5b4dff0b6f10db63fe34ab1b0ea8f0981fa3a743 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 2 Dec 2022 11:09:58 -0600
Subject: [PATCH 180/359] v4.0.12.0 Channel/overbank subdivision framework
 (#736)

---
 config/params_template.env         |  27 +--
 docs/CHANGELOG.md                  |  25 +++
 gms_run_branch.sh                  |  84 ++++---
 src/add_crosswalk.py               |  32 +--
 src/identify_src_bankfull.py       | 337 ++++++++++++++++-------------
 src/src_adjust_spatial_obs.py      |   3 +-
 src/src_adjust_usgs_rating.py      |   4 +-
 src/src_roughness_optimization.py  | 176 +++++++--------
 src/subdiv_chan_obank_src.py       | 277 ++++++++++++++++++++++++
 src/utils/shared_variables.py      |   2 +-
 src/vary_mannings_n_composite.py   | 256 ----------------------
 tools/aggregate_csv_files.py       |  63 ++++++
 tools/eval_plots.py                |  10 +-
 tools/synthesize_test_cases.py     |  11 +-
 tools/vary_mannings_n_composite.py | 273 +++++++++++++++++++++++
 15 files changed, 998 insertions(+), 582 deletions(-)
 create mode 100755 src/subdiv_chan_obank_src.py
 delete mode 100755 src/vary_mannings_n_composite.py
 create mode 100644 tools/aggregate_csv_files.py
 create mode 100755 tools/vary_mannings_n_composite.py

diff --git a/config/params_template.env b/config/params_template.env
index d9ff518cc..12ced021c 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -29,31 +29,14 @@ export branch_zero_id="0"
 #### mask levee-protected areas from relative elevation model
 export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from relative elevation model
 
-#### bathy SRC estimation parameters ####
-export bathy_src_toggle=True # Toggle to run BARC routine (True=on; False=off)
-export bankfull_input_table="data/inputs/bathymetry/nwm_flow_bieger_qreg.csv" # input file location with feature_id and channel geometry attributes
-# Option 1: Bieger et al. 2015 discharge regression --> nwm_flow_bieger_qreg.csv
-# Option 2: Bieger et al. 2015 drainage area regression (obtained from Wieczorek (2018) database) --> BANKFULL_CONUS.txt
-# Option 3: NWM Route Link bankfull geometry (Blackburn-Lynch regression) --> nwm_route_link_geom_BED.csv
-export src_plot_option="False" # optional toggle to create SRC comparison plots for each hydroid (Warning: longer run times)
-export surf_area_thalweg_ratio_flag=10 # Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
-export thalweg_stg_search_max_limit=3 # Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
-export bathy_xs_area_chg_flag=1 # Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
-export bankful_xs_area_ratio_flag=10 # Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
-export thalweg_hyd_radius_flag=10 # Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
-export ignore_streamorders=10 # Ignore BARC calculations for streamorders >= this value (10 is Mississippi R)
-
 #### estimating bankfull stage in SRCs ####
 export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
-export src_bankfull_plot_option="False" # optional toggle to create SRC comparison plots for each hydroid (Warning: longer run times)
-export bankfull_flows_file="data/inputs/rating_curve/bankfull_flows/nwm_v2_0_recurr_1_5_cms.csv" # input file location with nwm feature_id and recurrence flow values
+export bankfull_flows_file="data/inputs/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv" # input file location with nwm feature_id and recurrence flow values
 
-#### applying variable/composite roughness curve to SRCs ####
-export src_vrough_toggle="True" # Toggle to run composite roughness src routine (True=on; False=off)
-export src_vrough_plot_option="False" # optional toggle to create SRC comparison plots for each hydroid (Warning: longer run times)
-export vrough_suffix="_vmann" # text to append to output log and src_full_crosswalked file names
-export vmann_input_file="data/inputs/rating_curve/variable_roughness/mannings_global_06_011.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export bankfull_attribute="chann_volume_ratio" # src_full_crosswalked_bankfull.csv attribute (column id) containing the channel vs overbank ratio values (generated in the identify_src_bankfull.py)
+#### applying channel/overbank subdivision routine to SRCs ####
+export src_subdiv_toggle="True" # Toggle to run composite roughness src routine (True=on; False=off)
+export vrough_suffix="" # text to append to output log and hydrotable file names (use for testing/debugging)
+export vmann_input_file="data/inputs/rating_curve/variable_roughness/mannings_global_06_12.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
 
 #### apply SRC adjustments using USGS rating curve database (calibration db) ####
 export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; False=off)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5c11ac681..ef39833de 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,31 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.0.12.0 - 2022-11-28 - [PR #736](https://github.com/NOAA-OWP/inundation-mapping/pull/736)
+
+This feature branch introduces a new methodology for computing Manning's equation for the synthetic rating curves. The new subdivision approach 1) estimates bankfull stage by crosswalking "bankfull" proxy discharge data to the raw SRC discharge values 2) identifies in-channel vs. overbank geometry values 3) applies unique in-channel and overbank Manning's n value (user provided values) to compute Manning's equation separately for channel and overbank discharge and adds the two components together for total discharge 4) computes a calibration coefficient (where benchmark data exists) that applies to the  calibrated total discharge calculation.
+
+### Additions
+
+- `src/subdiv_chan_obank_src.py`: new script that performs all subdiv calculations and then produce a new (modified) `hydroTable.csv`. Inputs include `src_full_crosswalked.csv` for each huc/branch and a Manning's roughness csv file (containing: featureid, channel n, overbank n; file located in the `/inputs/rating_curve/variable_roughness/`). Note that the `identify_src_bankfull.py` script must be run prior to running the subdiv workflow.
+
+### Changes
+
+- `config/params_template.env`: removed BARC and composite roughness parameters; added new subdivision parameters; default Manning's n file set to `mannings_global_06_12.csv`
+- `gms_run_branch.sh`: moved the PostgreSQL database steps to occur immediately before the SRC calibration steps; added new subdivision step; added condition to SRC calibration to ensure subdivision routine is run
+- `src/add_crosswalk.py`: removed BARC function call; update placeholder value list (removed BARC and composite roughness variables) - these placeholder variables ensure that all hydrotables have the same dimensions
+- `src/identify_src_bankfull.py`: revised FIM3 starting code to work with FIM4 framework; stripped out unnecessary calculations; restricted bankfull identification to stage values > 0
+- `src/src_adjust_spatial_obs.py`: added huc sort function to help user track progress from console outputs
+- `src/src_adjust_usgs_rating.py`: added huc sort function to help user track progress from console outputs
+- `src/src_roughness_optimization.py`: reconfigured code to compute a calibration coefficient and apply adjustments using the subdivision variables; renamed numerous variables; simplified code where possible
+- `src/utils/shared_variables.py`: increased `ROUGHNESS_MAX_THRESH` from 0.6 to 0.8
+- `tools/vary_mannings_n_composite.py`: *moved this script from /src to /tools*; updated this code from FIM3 to work with FIM4 structure; however, it is not currently implemented (the subdivision routine replaces this)
+- `tools/aggregate_csv_files.py`: helper tool to search for csv files by name/wildcard and concatenate all found files into one csv (used for aggregating previous calibrated roughness values)
+- `tools/eval_plots.py`: updated list of metrics to plot to also include equitable threat score and mathews correlation coefficient (MCC)
+- `tools/synthesize_test_cases.py`: updated the list of FIM version metrics that the `PREV` flag will use to create the final aggregated metrics csv; this change will combine the dev versions provided with the `-dc` flag along with the existing `previous_fim_list` 
+
+<br/><br/>
+
 ## v4.0.11.5 - 2022-11-18 - [PR #746](https://github.com/NOAA-OWP/inundation-mapping/pull/746)
 
 Skips `src/usgs_gage_unit_setup.py` if no level paths exist. This may happen if a HUC has no stream orders > 2. This is a bug fix for #723 for the case that the HUC also has USGS gages.
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index f78e29e0d..c313bcc4f 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -204,40 +204,6 @@ fi
 
 ## Track total time of the overall run
 T_total_start
-Tstart
-
-: '
-This makes the local variables from the calb_db_keys files
-into global variables that can be used in other files, including python.
-
-Why not just leave the word export in front of each of the keys in the
-calb_db_keys.env? Becuase that file is used against docker-compose
-when we start up that part of the sytem and it does not like the word
-export.
-'
-export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
-export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
-export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
-export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
-
-## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
-if [ "$src_adjust_spatial" = "True" ]; then
-    if [ ! -f $CALB_DB_KEYS_FILE ]; then
-        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
-        exit 1
-    else
-        source $CALB_DB_KEYS_FILE
-        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
-        echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
-        echo "Loading Point Data"
-        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
-    fi
-fi
-
-Tcount
-Tstart
-date -u
 
 ## RUN GMS BY BRANCH ##
 echo "=========================================================================="
@@ -262,8 +228,54 @@ echo
 echo "Processing usgs gage aggregation"
 python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
 
+## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
+if [ "$src_bankfull_toggle" = "True" ]; then
+    echo -e $startDiv"Estimating bankfull stage in SRCs"$stopDiv
+    # Run SRC bankfull estimation routine routine
+    Tstart
+    time python3 /foss_fim/src/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit
+    Tcount
+fi
+
+## RUN SYNTHETIC RATING SUBDIVISION ROUTINE ##
+if [ "$src_subdiv_toggle" = "True" ]; then
+    echo -e $startDiv"Performing SRC channel/overbank subdivision routine"$stopDiv
+    # Run SRC Subdivision & Variable Roughness routine
+    Tstart
+    time python3 /foss_fim/src/subdiv_chan_obank_src.py -fim_dir $outputRunDataDir -mann $vmann_input_file -j $jobLimit
+    Tcount
+fi
+
+## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
+if [ "$src_adjust_spatial" = "True" ]; then
+    if [ ! -f $CALB_DB_KEYS_FILE ]; then
+        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        exit 1
+    else
+        source $CALB_DB_KEYS_FILE
+        : '
+        This makes the local variables from the calb_db_keys files
+        into global variables that can be used in other files, including python.
+
+        Why not just leave the word export in front of each of the keys in the
+        calb_db_keys.env? Becuase that file is used against docker-compose
+        when we start up that part of the sytem and it does not like the word
+        export.
+        '
+        export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
+        export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
+        export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
+        export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
+        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
+        echo "Loading HUC Data"
+        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+        echo "Loading Point Data"
+        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+    fi
+fi
+
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
-if [ "$src_adjust_usgs" = "True" ]; then
+if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
     Tstart
     echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"$stopDiv
     # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
@@ -273,7 +285,7 @@ if [ "$src_adjust_usgs" = "True" ]; then
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
-if [ "$src_adjust_spatial" = "True" ]; then
+if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
     Tstart
     echo -e $startDiv"Performing SRC adjustments using benchmark point database"$stopDiv
     python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index fe64d0f50..4bc49d3cf 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -8,7 +8,6 @@
 import json
 import argparse
 import sys
-from bathy_rc_adjust import bathy_rc_lookup
 from utils.shared_functions import getDriver
 # sys.path.append('/foss_fim/src')
 # sys.path.append('/foss_fim/config')
@@ -241,23 +240,26 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)','BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)','Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
     ## Set placeholder variables to be replaced in post-processing (as needed). Create here to ensure consistent column vars
-    output_hydro_table['barc_on'] = False # set barc_on attribute to Fasle (default) --> will be overwritten if BARC module runs
-    output_hydro_table['raw_discharge_cms'] = output_src['Discharge (m3s-1)']
-    output_hydro_table['raw_Volume (m3)'] = output_src['Volume (m3)']
-    output_hydro_table['raw_WetArea (m2)'] = output_src['WetArea (m2)']
-    output_hydro_table['raw_HydraulicRadius (m)'] = output_src['HydraulicRadius (m)']
-    output_hydro_table['vmann_on'] = False # set vmann_on attribute to Fasle (default) --> will be overwritten if variable roughness module runs
-    output_hydro_table['raw_ManningN'] = output_src['ManningN']
-    output_hydro_table['vmann_discharge_cms'] = pd.NA
-    output_hydro_table['vmann_ManningN'] = pd.NA
-    output_hydro_table['adjust_src_on'] = False
+    ## These variables represent the original unmodified values
+    output_hydro_table['default_discharge_cms'] = output_src['Discharge (m3s-1)']
+    output_hydro_table['default_Volume (m3)'] = output_src['Volume (m3)']
+    output_hydro_table['default_WetArea (m2)'] = output_src['WetArea (m2)']
+    output_hydro_table['default_HydraulicRadius (m)'] = output_src['HydraulicRadius (m)']
+    output_hydro_table['default_ManningN'] = output_src['ManningN']
+    ## Placeholder vars for subdivision routine
+    output_hydro_table['subdiv_applied'] = False
+    output_hydro_table['overbank_n'] = pd.NA
+    output_hydro_table['channel_n'] = pd.NA
+    output_hydro_table['subdiv_discharge_cms'] = pd.NA
+    ## Placeholder vars for the calibration routine
+    output_hydro_table['calb_applied'] = pd.NA
     output_hydro_table['last_updated'] = pd.NA
     output_hydro_table['submitter'] = pd.NA
-    output_hydro_table['adjust_ManningN'] = pd.NA
     output_hydro_table['obs_source'] = pd.NA
-    output_hydro_table['default_discharge_cms'] = pd.NA
-    output_hydro_table['default_ManningN'] = pd.NA
-
+    output_hydro_table['precalb_discharge_cms'] = pd.NA
+    output_hydro_table['calb_coef_usgs'] = pd.NA
+    output_hydro_table['calb_coef_spatial'] = pd.NA
+    output_hydro_table['calb_coef_final'] = pd.NA
 
     if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
     output_hydro_table[FIM_ID] = output_hydro_table.loc[:,'HydroID'].apply(lambda x : str(x)[0:4])
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index 3be19fae1..4bd5d7c14 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -1,123 +1,146 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
 import sys
 import pandas as pd
-import argparse
 import matplotlib.pyplot as plt
+import multiprocessing
 import seaborn as sns
-from functools import reduce
-from multiprocessing import Pool
-from os.path import isfile, join, dirname, isdir
 import shutil
 import warnings
-from pathlib import Path
 import datetime as dt
+import re
+import traceback
+from pathlib import Path
+from functools import reduce
+from multiprocessing import Pool
+from os.path import isfile, join, dirname, isdir
+from tqdm import tqdm
 sns.set_theme(style="whitegrid")
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
 """
-    Identify the SRC bankfull stage values using the NWM 1.5yr flows
+    Identify the SRC bankfull stage values using the NWM bankfull estimated flows
 
     Parameters
     ----------
     fim_dir : str
         Directory containing FIM output folders.
     bankfull_flow_dir : str
-        Directory containing "bankfull" flows files (e.g. NWM 1.5yr recurr).
+        Directory containing "bankfull" flows files (e.g. NWM bankfull estimated recurr).
     number_of_jobs : str
         Number of jobs.
     plots : str
-        Flag to create SRC plots for all hydroids (True/False)
+        Optional: Flag to create SRC plots for all hydroids (True/False)
 """
 
 def src_bankfull_lookup(args):
 
     src_full_filename           = args[0]
-    src_modify_filename         = args[1]
+    src_usecols                 = args[1]
     df_bflows                   = args[2]
     huc                         = args[3]
-    src_plot_option             = args[4]
-    huc_output_dir              = args[5]
+    branch_id                   = args[4]
+    src_plot_option             = args[5]
+    huc_output_dir              = args[6]
 
     ## Read the src_full_crosswalked.csv
-    #print('Processing: ' + str(huc))
-    log_text = 'Calculating: ' + str(huc) + '\n'
-    df_src = pd.read_csv(src_full_filename,dtype={'HydroID': int,'feature_id': int})
-
-    ## NWM recurr rename discharge var
-    df_bflows = df_bflows.rename(columns={'discharge':'discharge_1_5'})
-
-    ## Combine the nwm 1.5yr flows into the SRC via feature_id
-    df_src = df_src.merge(df_bflows,how='left',on='feature_id')
-
-    ## Check if there are any missing data, negative or zero flow values in the discharge_1_5
-    check_null = df_src['discharge_1_5'].isnull().sum()
-    if check_null > 0:
-        log_text += 'Missing feature_id in crosswalk for huc: ' + str(huc) + ' --> these featureids will be ignored in bankfull calcs (~' + str(check_null/84) +  ' features) \n'
-        ## Fill missing/nan nwm discharge_1_5 values with -999 to handle later
-        df_src['discharge_1_5'] = df_src['discharge_1_5'].fillna(-999)
-    negative_flows = len(df_src.loc[(df_src.discharge_1_5 <= 0) & (df_src.discharge_1_5 != -999)])
-    if negative_flows > 0:
-        log_text += 'HUC: ' + str(huc) + ' --> Negative or zero flow values found (likely lakeid loc)\n'
-
-    ## Define the channel geometry variable names to use from the src
-    hradius_var = 'HydraulicRadius (m)'
-    volume_var = 'Volume (m3)'
-
-    ## Locate the closest SRC discharge value to the NWM 1.5yr flow
-    df_src['Q_1_5_find'] = (df_src['discharge_1_5'] - df_src['Discharge (m3s-1)']).abs()
-
-    ## Check for any missing/null entries in the input SRC
-    if df_src['Q_1_5_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
-        log_text += 'HUC: ' + str(huc) + ' --> Null values found in "Q_1_5_find" calc. These will be filled with 999999 () \n'
-        ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
-        df_src['Q_1_5_find'] = df_src['Q_1_5_find'].fillna(999999)
-    if df_src['HydroID'].isnull().values.any():
-        log_text += 'HUC: ' + str(huc) + ' --> Null values found in "HydroID"... \n'
-
-    df_1_5 = df_src[['Stage','HydroID',volume_var,hradius_var,'Q_1_5_find']] # create new subset df to perform the Q_1_5 lookup
-    df_1_5 = df_1_5[df_1_5['Stage'] > 0.0] # Ensure bankfull stage is greater than stage=0
-    df_1_5.reset_index(drop=True, inplace=True)
-    df_1_5 = df_1_5.loc[df_1_5.groupby('HydroID')['Q_1_5_find'].idxmin()].reset_index(drop=True) # find the index of the Q_1_5_find (closest matching flow)
-    df_1_5 = df_1_5.rename(columns={'Stage':'Stage_1_5',volume_var:'Volume_bankfull',hradius_var:'HRadius_bankfull'}) # rename volume to use later for channel portion calc
-    df_src = df_src.merge(df_1_5[['Stage_1_5','HydroID','Volume_bankfull','HRadius_bankfull']],how='left',on='HydroID')
-    df_src.drop(['Q_1_5_find'], axis=1, inplace=True)
-
-    ## Calculate the channel portion of bankfull Volume
-    df_src['chann_volume_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
-    df_src['chann_volume_ratio'].where(df_src['Stage'] == 0, df_src['Volume_bankfull'] / (df_src[volume_var]),inplace=True)
-    #df_src['chann_volume_ratio'] = df_src['chann_volume_ratio'].clip_upper(1.0)
-    df_src['chann_volume_ratio'].where(df_src['chann_volume_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
-    df_src['chann_volume_ratio'].where(df_src['discharge_1_5'] > 0.0, 0.0, inplace=True) # if the discharge_1_5 value <= 0 then set channel ratio to 0 (will use global overbank manning n)
-    #df_src.drop(['Volume_bankfull'], axis=1, inplace=True)
-
-    ## Calculate the channel portion of bankfull Hydraulic Radius
-    df_src['chann_hradius_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
-    df_src['chann_hradius_ratio'].where(df_src['Stage'] == 0, df_src['HRadius_bankfull'] / (df_src[hradius_var]),inplace=True)
-    #df_src['chann_hradius_ratio'] = df_src['HRadius_bankfull'] / (df_src[hradius_var]+.0001) # old adding 0.01 to avoid dividing by 0 at stage=0
-    df_src['chann_hradius_ratio'].where(df_src['chann_hradius_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
-    df_src['chann_hradius_ratio'].where(df_src['discharge_1_5'] > 0.0, 0.0, inplace=True) # if the discharge_1_5 value <= 0 then set channel ratio to 0 (will use global overbank manning n)
-    #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
-
-    ## mask bankfull variables when the 1.5yr flow value is <= 0
-    df_src['Stage_1_5'].mask(df_src['discharge_1_5'] <= 0.0,inplace=True)
-
-    ## Create a new column to identify channel/floodplain via the bankfull stage value
-    df_src.loc[df_src['Stage'] <= df_src['Stage_1_5'], 'channel_fplain_1_5'] = 'channel'
-    df_src.loc[df_src['Stage'] > df_src['Stage_1_5'], 'channel_fplain_1_5'] = 'floodplain'
-    df_src['channel_fplain_1_5'] = df_src['channel_fplain_1_5'].fillna('channel')
-
-    ## Output new SRC with bankfull column
-    df_src.to_csv(src_modify_filename,index=False)
-    log_text += 'Completed: ' + str(huc)
-
-    ## plot rating curves (optional arg)
-    if src_plot_option == 'True':
-        if isdir(huc_output_dir) == False:
-            os.mkdir(huc_output_dir)
-        generate_src_plot(df_src, huc_output_dir)
-
+    print('Calculating bankfull: ' + str(huc) + '  branch id: ' + str(branch_id))
+    log_text = 'Calculating: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+    try:
+        df_src = pd.read_csv(src_full_filename,usecols=src_usecols,dtype={'HydroID': int,'feature_id': int})
+
+        ## NWM recurr rename discharge var
+        df_bflows = df_bflows.rename(columns={'discharge':'bankfull_flow'})
+
+        ## Combine the nwm bankfull estimated flows into the SRC via feature_id
+        df_src = df_src.merge(df_bflows,how='left',on='feature_id')
+
+        ## Check if there are any missing data, negative or zero flow values in the bankfull_flow
+        check_null = df_src['bankfull_flow'].isnull().sum()
+        if check_null > 0:
+            log_text += 'WARNING: Missing feature_id in crosswalk for huc: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> these featureids will be ignored in bankfull calcs (~' + str(check_null/84) +  ' features) \n'
+            ## Fill missing/nan nwm bankfull_flow values with -999 to handle later
+            df_src['bankfull_flow'] = df_src['bankfull_flow'].fillna(-999)
+        negative_flows = len(df_src.loc[(df_src.bankfull_flow <= 0) & (df_src.bankfull_flow != -999)])
+        if negative_flows > 0:
+            log_text += 'WARNING: HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> Negative or zero flow values found in the input bankfull flows csv (posible lakeid loc)\n'
+
+        ## Define the channel geometry variable names to use from the src
+        hradius_var = 'HydraulicRadius (m)'
+        volume_var = 'Volume (m3)'
+        surface_area_var = 'SurfaceArea (m2)'
+        bedarea_var = 'BedArea (m2)'
+
+        ## Locate the closest SRC discharge value to the NWM bankfull estimated flow
+        df_src['Q_bfull_find'] = (df_src['bankfull_flow'] - df_src['Discharge (m3s-1)']).abs()
+
+        ## Check for any missing/null entries in the input SRC
+        if df_src['Q_bfull_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
+            log_text += 'WARNING: HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> Null values found in "Q_bfull_find" calc. These will be filled with 999999 () \n'
+            ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
+            df_src['Q_bfull_find'] = df_src['Q_bfull_find'].fillna(999999)
+        if df_src['HydroID'].isnull().values.any():
+            log_text += 'WARNING: HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> Null values found in "HydroID"... \n'
+
+        df_bankfull_calc = df_src[['Stage','HydroID',bedarea_var,volume_var,hradius_var,surface_area_var,'Q_bfull_find']] # create new subset df to perform the Q_1_5 lookup
+        df_bankfull_calc = df_bankfull_calc[df_bankfull_calc['Stage'] > 0.0] # Ensure bankfull stage is greater than stage=0
+        df_bankfull_calc.reset_index(drop=True, inplace=True)
+        df_bankfull_calc = df_bankfull_calc.loc[df_bankfull_calc.groupby('HydroID')['Q_bfull_find'].idxmin()].reset_index(drop=True) # find the index of the Q_bfull_find (closest matching flow)
+        df_bankfull_calc = df_bankfull_calc.rename(columns={'Stage':'Stage_bankfull',bedarea_var:'BedArea_bankfull',volume_var:'Volume_bankfull',hradius_var:'HRadius_bankfull',surface_area_var:'SurfArea_bankfull'}) # rename volume to use later for channel portion calc
+        df_src = df_src.merge(df_bankfull_calc[['Stage_bankfull','HydroID','BedArea_bankfull','Volume_bankfull','HRadius_bankfull','SurfArea_bankfull']],how='left',on='HydroID')
+        df_src.drop(['Q_bfull_find'], axis=1, inplace=True)
+
+        ## The bankfull ratio variables below were previously used for the composite variable roughness routine (not currently implimented)
+        # ## Calculate the channel portion of bankfull Volume
+        # df_src['chann_volume_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
+        # df_src['chann_volume_ratio'].where(df_src['Stage'] == 0, df_src['Volume_bankfull'] / (df_src[volume_var]),inplace=True)
+        # #df_src['chann_volume_ratio'] = df_src['chann_volume_ratio'].clip_upper(1.0)
+        # df_src['chann_volume_ratio'].where(df_src['chann_volume_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
+        # df_src['chann_volume_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True) # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # #df_src.drop(['Volume_bankfull'], axis=1, inplace=True)
+
+        # ## Calculate the channel portion of bankfull Hydraulic Radius
+        # df_src['chann_hradius_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
+        # df_src['chann_hradius_ratio'].where(df_src['Stage'] == 0, df_src['HRadius_bankfull'] / (df_src[hradius_var]),inplace=True)
+        # #df_src['chann_hradius_ratio'] = df_src['HRadius_bankfull'] / (df_src[hradius_var]+.0001) # old adding 0.01 to avoid dividing by 0 at stage=0
+        # df_src['chann_hradius_ratio'].where(df_src['chann_hradius_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
+        # df_src['chann_hradius_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True) # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
+
+        # ## Calculate the channel portion of bankfull Surface Area
+        # df_src['chann_surfarea_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
+        # df_src['chann_surfarea_ratio'].where(df_src['Stage'] == 0, df_src['SurfArea_bankfull'] / (df_src[surface_area_var]),inplace=True)
+        # df_src['chann_surfarea_ratio'].where(df_src['chann_surfarea_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
+        # df_src['chann_surfarea_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True) # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
+
+        ## mask bankfull variables when the bankfull estimated flow value is <= 0
+        df_src['Stage_bankfull'].mask(df_src['bankfull_flow'] <= 0.0,inplace=True)
+
+        ## Create a new column to identify channel/floodplain via the bankfull stage value
+        df_src.loc[df_src['Stage'] <= df_src['Stage_bankfull'], 'bankfull_proxy'] = 'channel'
+        df_src.loc[df_src['Stage'] > df_src['Stage_bankfull'], 'bankfull_proxy'] = 'floodplain'
+        df_src['bankfull_proxy'] = df_src['bankfull_proxy'].fillna('channel')
+
+        ## Output new SRC with bankfull column
+        df_src.to_csv(src_full_filename,index=False)
+        log_text += 'Completed: ' + str(huc)
+
+        ## plot rating curves (optional arg)
+        if src_plot_option:
+            if isdir(huc_output_dir) == False:
+                os.mkdir(huc_output_dir)
+            generate_src_plot(df_src, huc_output_dir)
+
+    except Exception as ex:
+        summary = traceback.StackSummary.extract(
+                traceback.walk_stack(None))
+        print(str(huc) + '  branch id: ' + str(branch_id) + " failed for some reason")
+        print(f"*** {ex}")                
+        print(''.join(summary.format()))    
+        log_text += 'ERROR --> ' + str(huc) + '  branch id: ' + str(branch_id) + " failed (details: " + (f"*** {ex}") + (''.join(summary.format())) + '\n'
     return(log_text)
 
 def generate_src_plot(df_src, plt_out_dir):
@@ -136,86 +159,104 @@ def generate_src_plot(df_src, plt_out_dir):
         axes[1].set_title('Channel Volume vs. HRadius Ratio')
         sns.despine(fig, left=True, bottom=True)
         sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, ax=axes[0])
-        sns.lineplot(x='Discharge (m3s-1)', y='Stage_1_5', data=plot_df, color='green', ax=axes[0])
-        axes[0].fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_1_5'],alpha=0.5)
-        axes[0].text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+        sns.lineplot(x='Discharge (m3s-1)', y='Stage_bankfull', data=plot_df, color='green', ax=axes[0])
+        axes[0].fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'],alpha=0.5)
+        axes[0].text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_bankfull'].median(), "Bankfull Proxy Stage: " + str(plot_df['Stage_bankfull'].median()))
         sns.scatterplot(x='chann_volume_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_volume_ratio", s=38)
         sns.scatterplot(x='chann_hradius_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_hradius_ratio", s=12)
+        sns.scatterplot(x='chann_surfarea_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_surfarea_ratio", s=12)
         axes[1].legend()
         plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_bankfull.png',dpi=100, bbox_inches='tight')
         plt.close()
 
-def multi_process(src_bankfull_lookup, procs_list):
+def multi_process(src_bankfull_lookup, procs_list, log_file, number_of_jobs, verbose):
     ## Initiate multiprocessing
+    available_cores = multiprocessing.cpu_count()
+    if number_of_jobs > available_cores:
+        number_of_jobs = available_cores - 2
+        print("Provided job number exceeds the number of available cores. " + str(number_of_jobs) + " max jobs will be used instead.")
+
     print(f"Identifying bankfull stage for {len(procs_list)} hucs using {number_of_jobs} jobs")
     with Pool(processes=number_of_jobs) as pool:
-        map_output = pool.map(src_bankfull_lookup, procs_list)
+        #progress_bar = tqdm(total=len(procs_list[0]))
+        if verbose:
+            map_output = tqdm(pool.imap(src_bankfull_lookup, procs_list), total=len(procs_list))
+            tuple(map_output)  # fetch the lazy results
+        else:
+            map_output = pool.map(src_bankfull_lookup, procs_list)
     log_file.writelines(["%s\n" % item  for item in map_output])
 
+def run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_option):
+    procs_list = []
+
+    ## Print message to user and initiate run clock
+    print('Writing progress to log file here: ' + str(join(fim_dir,'bankfull_detect.log')))
+    print('This may take a few minutes...')
+
+    ## Check that the input fim_dir exists
+    assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
+    ## Check that the bankfull flow filepath exists and read to dataframe
+    assert os.path.isfile(bankfull_flow_filepath), 'ERROR: Can not find the input bankfull flow file: ' + str(bankfull_flow_filepath)
+    
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+    ## initiate log file
+    log_file = open(join(fim_dir,'logs','log_bankfull_indentify.log'),"w")
+    log_file.write('START TIME: ' + str(begin_time) + '\n')
+    log_file.write('#########################################################\n\n')
+
+    ## List of columns in SRC_full_crosswalk to read in (ignores other columns that may have been added by previous post-proccessing runs)
+    src_usecols=['Stage','Number of Cells','SurfaceArea (m2)','BedArea (m2)','Volume (m3)','SLOPE','LENGTHKM','AREASQKM','ManningN','HydroID','NextDownID','order_','TopWidth (m)','WettedPerimeter (m)','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)','feature_id']
+
+    df_bflows = pd.read_csv(bankfull_flow_filepath,dtype={'feature_id': int})
+    huc_list  = os.listdir(fim_dir)
+    huc_list.sort() # sort huc_list for helping track progress in future print statments
+    huc_pass_list = []
+    for huc in huc_list:
+        #if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+        if re.match('\d{8}', huc):    
+            huc_branches_dir = os.path.join(fim_dir, huc,'branches')
+            for branch_id in os.listdir(huc_branches_dir):
+                branch_dir = os.path.join(huc_branches_dir,branch_id)
+                src_orig_full_filename = join(branch_dir,'src_full_crosswalked_' + branch_id + '.csv')
+                huc_output_dir = join(branch_dir,'src_plots')
+                ## check if BARC modified src_full_crosswalked_BARC.csv exists otherwise use the orginial src_full_crosswalked.csv
+                if isfile(src_orig_full_filename):
+                    huc_pass_list.append(str(huc) + " --> src_full_crosswalked.csv")
+                    procs_list.append([src_orig_full_filename, src_usecols, df_bflows, huc, branch_id, src_plot_option, huc_output_dir])
+                else:
+                    print('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + 'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+                    log_file.write('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + 'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+
+    log_file.writelines(["%s\n" % item  for item in huc_pass_list])
+    log_file.write('#########################################################\n\n')
+
+    ## Pass huc procs_list to multiprocessing function
+    multi_process(src_bankfull_lookup, procs_list, log_file, number_of_jobs, verbose)
+
+    ## Record run time and close log file
+    end_time = dt.datetime.now()
+    log_file.write('END TIME: ' + str(end_time) + '\n')
+    tot_run_time = end_time - begin_time
+    log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+    log_file.close()
 
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Identify bankfull stage for each hydroid synthetic rating curve')
     parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-flows','--bankfull-flow-input',help='NWM recurrence flows dir',required=True,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-    parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
+    parser.add_argument('-flows','--bankfull-flow-input',help='NWM recurrence flows dir (flow units in CMS!!!)',required=True,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='OPTIONAL: number of workers (default=8)',required=False,default=8,type=int)
+    parser.add_argument('-vb','--verbose',help='OPTIONAL: verbose progress bar',required=False,default=None,action='store_true')
+    parser.add_argument('-plots','--src-plot-option',help='OPTIONAL flag: use this flag to create src plots for all hydroids (helpful for evaluating). WARNING - long runtime',default=False,required=False, action='store_true')
 
     args = vars(parser.parse_args())
 
     fim_dir = args['fim_dir']
     bankfull_flow_filepath = args['bankfull_flow_input']
     number_of_jobs = args['number_of_jobs']
+    verbose = bool(args['verbose'])
     src_plot_option = args['src_plot_option']
-    procs_list = []
-
-    ## Print message to user and initiate run clock
-    print('Writing progress to log file here: ' + str(join(fim_dir,'bankfull_detect.log')))
-    print('This may take a few minutes...')
-    ## Create a time var to log run time
-    begin_time = dt.datetime.now()
-
-    ## Check that the bankfull flow filepath exists and read to dataframe
-    if not isfile(bankfull_flow_filepath):
-        print('!!!ERROR: Can not find the input bankfull flow file: ' + str(bankfull_flow_filepath))
-    else:
-        df_bflows = pd.read_csv(bankfull_flow_filepath,dtype={'feature_id': int})
-        huc_list  = os.listdir(fim_dir)
-        huc_pass_list = []
-        for huc in huc_list:
-            if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
-                src_barc_full_filename = join(fim_dir,huc,'src_full_crosswalked_BARC.csv')
-                src_orig_full_filename = join(fim_dir,huc,'src_full_crosswalked.csv')
-                src_modify_filename = join(fim_dir,huc,'src_full_crosswalked_bankfull.csv')
-                huc_output_dir = join(fim_dir,huc,'src_plots')
-                ## check if BARC modified src_full_crosswalked_BARC.csv exists otherwise use the orginial src_full_crosswalked.csv
-                if isfile(src_barc_full_filename):
-                    print(str(huc))
-                    huc_pass_list.append(str(huc) + " --> src_full_crosswalked_BARC.csv")
-                    procs_list.append([src_barc_full_filename, src_modify_filename, df_bflows, huc, src_plot_option, huc_output_dir])
-                elif isfile(src_orig_full_filename):
-                    print(str(huc))
-                    huc_pass_list.append(str(huc) + " --> src_full_crosswalked.csv")
-                    procs_list.append([src_orig_full_filename, src_modify_filename, df_bflows, huc, src_plot_option, huc_output_dir])
-                else:
-                    print(str(huc) + 'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: ' + str(join(fim_dir,huc)) + ' - skipping this HUC!!!\n')
-
-        ## initiate log file
-        print(f"Identifying bankfull stage for {len(procs_list)} hucs using {number_of_jobs} jobs")
-        sys.__stdout__ = sys.stdout
-        log_file = open(join(fim_dir,'logs','log_bankfull_indentify.log'),"w")
-        sys.stdout = log_file
-        log_file.write('START TIME: ' + str(begin_time) + '\n')
-        log_file.writelines(["%s\n" % item  for item in huc_pass_list])
-        log_file.write('#########################################################\n\n')
-
-        ## Pass huc procs_list to multiprocessing function
-        multi_process(src_bankfull_lookup, procs_list)
-
-        ## Record run time and close log file
-        end_time = dt.datetime.now()
-        log_file.write('END TIME: ' + str(end_time) + '\n')
-        tot_run_time = end_time - begin_time
-        log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
-        sys.stdout = sys.__stdout__
-        log_file.close()
+    
+    ## Prepare/check inputs, create log file, and spin up the proc list
+    run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_option)
\ No newline at end of file
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index 237e7365a..8122c9ece 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -250,6 +250,7 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
 
     #huc_list = ['12040103']
     ## Define paths to relevant HUC HAND data.
+    huc_list.sort() # sort huc_list for helping track progress in future print statments
     for huc in huc_list:
         huc_branches_dir = os.path.join(fim_directory, huc,'branches')
         water_edge_df = find_points_in_huc(huc, conn).reset_index()
@@ -419,4 +420,4 @@ def run_prep(fim_directory, debug_outputs_option, ds_thresh_override, DOWNSTREAM
     job_number = int(args['job_number'])
 
     run_prep(fim_directory, debug_outputs_option, ds_thresh_override, 
-             DOWNSTREAM_THRESHOLD, job_number)
\ No newline at end of file
+             DOWNSTREAM_THRESHOLD, job_number)
diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py
index a9a086a48..1db6f3eec 100644
--- a/src/src_adjust_usgs_rating.py
+++ b/src/src_adjust_usgs_rating.py
@@ -138,8 +138,6 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
     return(final_df)
 
 def branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file):
-    huc_list = usgs_df['huc'].tolist()
-    huc_list = list(set(huc_list))
     procs_list = []  # Initialize list for mulitprocessing.
 
     # loop through all unique level paths that have a USGS gage
@@ -147,7 +145,7 @@ def branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file):
     #branch_huc_dict = usgs_df.set_index('huc').T.to_dict('list')
     huc_branch_dict = usgs_df.groupby('huc')['levpa_id'].apply(set).to_dict()
 
-    for huc in huc_branch_dict:
+    for huc in sorted(huc_branch_dict.keys()): # sort huc_list for helping track progress in future print statments
         branch_set = huc_branch_dict[huc]
         for branch_id in branch_set: 
             # Define paths to branch HAND data.
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index 56663d16a..9be4b397e 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -29,9 +29,9 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     - Create df with the most recent collection time entry and submitter attribs
     - Cacluate median ManningN to handle cases with multiple hydroid entries and create a df with the median hydroid_ManningN value per feature_id
     - Rename the original hydrotable variables to allow new calculations to use the primary var name 
-    - Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a singel hydroid)
+    - Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a single hydroid)
     - Create attributes to traverse the flow network between HydroIDs
-    - Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg)
+    - Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg)
     - Create the adjust_ManningN column by combining the hydroid_ManningN with the featid_ManningN (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
     - Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new adjusted roughness value
     - Update the catchments polygon .gpkg with joined attribute - "src_calibrated"
@@ -65,32 +65,40 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     df_nvalues = water_edge_median_df.copy()
     df_nvalues = df_nvalues[ (df_nvalues.hydroid.notnull()) & (df_nvalues.hydroid > 0) ] # remove null entries that do not have a valid hydroid
 
+    ## Determine calibration data type for naming calb dataframe column
+    if source_tag == 'point_obs':
+        calb_type = 'calb_coef_spatial'
+    if source_tag == 'usgs_rating':
+        calb_type = 'calb_coef_usgs'
+
     ## Read in the hydroTable.csv and check wether it has previously been updated (rename default columns if needed)
     df_htable = pd.read_csv(htable_path, dtype={'HUC': object, 'last_updated':object, 'submitter':object, 'obs_source':object})
     df_prev_adj = pd.DataFrame() # initialize empty df for populating/checking later
-    if 'default_discharge_cms' not in df_htable.columns: # need this column to exist before continuing
-        df_htable['adjust_src_on'] = False
+    if 'precalb_discharge_cms' not in df_htable.columns: # need this column to exist before continuing
+        df_htable['calb_applied'] = False
         df_htable['last_updated'] = pd.NA
         df_htable['submitter'] = pd.NA
-        df_htable['adjust_ManningN'] = pd.NA
         df_htable['obs_source'] = pd.NA
-        df_htable['default_discharge_cms'] = pd.NA
-        df_htable['default_ManningN'] = pd.NA
-    if df_htable['default_discharge_cms'].isnull().values.any(): # check if there are not valid values in the column (True = no previous calibration outputs)
-        df_htable['default_discharge_cms'] = df_htable['discharge_cms'].values
-        df_htable['default_ManningN'] = df_htable['ManningN'].values
-
-    if merge_prev_adj and not df_htable['adjust_ManningN'].isnull().all(): # check if the merge_prev_adj setting is True and there are valid 'adjust_ManningN' values from previous calibration outputs
+        df_htable['precalb_discharge_cms'] = pd.NA
+        df_htable['calb_coef_usgs'] = pd.NA
+        df_htable['calb_coef_spatial'] = pd.NA
+        df_htable['calb_coef_final'] = pd.NA
+    if df_htable['precalb_discharge_cms'].isnull().values.any(): # check if there are not valid values in the column (True = no previous calibration outputs)
+        df_htable['precalb_discharge_cms'] = df_htable['discharge_cms'].values
+
+    ## The section below allows for previous calibration modifications (i.e. usgs rating calbs) to be available in the final calibration outputs
+    if merge_prev_adj and not df_htable['calb_coef_final'].isnull().all(): # check if the merge_prev_adj setting is True and there are valid 'calb_coef_final' values from previous calibration outputs
         # Create a subset of hydrotable with previous adjusted SRC attributes
-        df_prev_adj_htable = df_htable.copy()[['HydroID','submitter','last_updated','adjust_ManningN','obs_source']] 
-        df_prev_adj_htable.rename(columns={'submitter':'submitter_prev','last_updated':'last_updated_prev','adjust_ManningN':'adjust_ManningN_prev','obs_source':'obs_source_prev'}, inplace=True)
+        df_prev_adj_htable = df_htable.copy()[['HydroID','submitter','last_updated','obs_source','calb_coef_final']] 
+        df_prev_adj_htable.rename(columns={'submitter':'submitter_prev','last_updated':'last_updated_prev','calb_coef_final':'calb_coef_final_prev','obs_source':'obs_source_prev'}, inplace=True)
         df_prev_adj_htable = df_prev_adj_htable.groupby(["HydroID"]).first()
         # Only keep previous USGS rating curve adjustments (previous spatial obs adjustments are not retained)
         df_prev_adj = df_prev_adj_htable[df_prev_adj_htable['obs_source_prev'].str.contains("usgs_rating", na=False)] 
         log_text += 'HUC: ' + str(huc) + '  Branch: ' + str(branch_id) + ': found previous hydroTable calibration attributes --> retaining previous calb attributes for blending...\n'
+    
     # Delete previous adj columns to prevent duplicate variable issues (if src_roughness_optimization.py was previously applied)
-    df_htable.drop(['ManningN','discharge_cms','submitter','last_updated','adjust_ManningN','adjust_src_on','obs_source'], axis=1, inplace=True) 
-    df_htable.rename(columns={'default_discharge_cms':'discharge_cms','default_ManningN':'ManningN'}, inplace=True)
+    df_htable.drop(['discharge_cms','submitter','last_updated',calb_type,'calb_coef_final','calb_applied','obs_source'], axis=1, inplace=True, errors='ignore') 
+    df_htable.rename(columns={'precalb_discharge_cms':'discharge_cms'}, inplace=True)
 
     ## loop through the user provided point data --> stage/flow dataframe row by row
     for index, row in df_nvalues.iterrows():
@@ -98,7 +106,7 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
             print('ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid))
             log_text += 'ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid) + '\n'
         else:
-            df_htable_hydroid = df_htable[df_htable.HydroID == row.hydroid] # filter htable for entries with matching hydroid
+            df_htable_hydroid = df_htable[(df_htable.HydroID == row.hydroid) & (df_htable.stage > 0)] # filter htable for entries with matching hydroid and ignore stage 0 (first possible stage match at 1ft)
             if df_htable_hydroid.empty:
                 print('ERROR: df_htable_hydroid is empty but expected data: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid))
                 log_text += 'ERROR: df_htable_hydroid is empty but expected data: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid) + '\n'
@@ -110,24 +118,21 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
             df_nvalues.loc[index,'NextDownID'] = find_src_stage.NextDownID
             df_nvalues.loc[index,'LENGTHKM'] = find_src_stage.LENGTHKM
             df_nvalues.loc[index,'src_stage'] = find_src_stage.stage
-            df_nvalues.loc[index,'ManningN'] = find_src_stage.ManningN
-            df_nvalues.loc[index,'SLOPE'] = find_src_stage.SLOPE
-            df_nvalues.loc[index,'HydraulicRadius_m'] = find_src_stage['HydraulicRadius (m)']
-            df_nvalues.loc[index,'WetArea_m2'] = find_src_stage['WetArea (m2)']
+            df_nvalues.loc[index,'channel_n'] = find_src_stage.channel_n
+            df_nvalues.loc[index,'overbank_n'] = find_src_stage.overbank_n
             df_nvalues.loc[index,'discharge_cms'] = find_src_stage.discharge_cms
 
-    ## mask src values that crosswalk to the SRC zero point (src_stage ~ 0 or discharge <= 0)
-    df_nvalues[['HydraulicRadius_m','WetArea_m2']] = df_nvalues[['HydraulicRadius_m','WetArea_m2']].mask((df_nvalues['src_stage'] <= 0.1) | (df_nvalues['discharge_cms'] <= 0.0), np.nan)
+    ## Calculate calibration coefficient
+    df_nvalues.rename(columns={'hydroid':'HydroID'}, inplace=True) # rename the previous ManningN column
+    df_nvalues['hydroid_calb_coef'] = df_nvalues['discharge_cms']/df_nvalues['flow'] # Qobs / Qsrc
 
-    ## Calculate roughness using Manning's equation
-    df_nvalues.rename(columns={'ManningN':'default_ManningN','hydroid':'HydroID'}, inplace=True) # rename the previous ManningN column
-    df_nvalues['hydroid_ManningN'] = df_nvalues['WetArea_m2']* \
-    pow(df_nvalues['HydraulicRadius_m'],2.0/3)* \
-    pow(df_nvalues['SLOPE'],0.5)/df_nvalues['flow']
+    ## Calcuate a "calibration adjusted" n value using channel and overbank n-values multiplied by calb_coef
+    df_nvalues['channel_n_calb'] = df_nvalues['hydroid_calb_coef']*df_nvalues['channel_n']
+    df_nvalues['overbank_n_calb'] = df_nvalues['hydroid_calb_coef']*df_nvalues['overbank_n']
 
-    ## Create dataframe to check for erroneous Manning's n values (values set in tools_shared_variables.py --> >0.6 or <0.001)
-    df_nvalues['Mann_flag'] = np.where((df_nvalues['hydroid_ManningN'] >= ROUGHNESS_MAX_THRESH) | (df_nvalues['hydroid_ManningN'] <= ROUGHNESS_MIN_THRESH) | (df_nvalues['hydroid_ManningN'].isnull()),'Fail','Pass')
-    df_mann_flag = df_nvalues[(df_nvalues['Mann_flag'] == 'Fail')][['HydroID','hydroid_ManningN']]
+    ## Create dataframe to check for unrealistic/egregious calibration adjustments by applying the calibration coefficient to the Manning's n values and setting an acceptable range (values set in tools_shared_variables.py --> >0.8 or <0.001)
+    df_nvalues['Mann_flag'] = np.where((df_nvalues['channel_n_calb'] >= ROUGHNESS_MAX_THRESH) | (df_nvalues['overbank_n_calb'] >= ROUGHNESS_MAX_THRESH) | (df_nvalues['channel_n_calb'] <= ROUGHNESS_MIN_THRESH) | (df_nvalues['overbank_n_calb'] <= ROUGHNESS_MIN_THRESH) | (df_nvalues['hydroid_calb_coef'].isnull()),'Fail','Pass')
+    df_mann_flag = df_nvalues[(df_nvalues['Mann_flag'] == 'Fail')][['HydroID','hydroid_calb_coef','channel_n_calb','overbank_n_calb']]
     if not df_mann_flag.empty:
         log_text += '!!! Flaged Mannings Roughness values below !!!' +'\n'
         log_text += df_mann_flag.to_string() + '\n'
@@ -143,11 +148,11 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     df_huc_lid.columns = pd.MultiIndex.from_product([['info'], df_huc_lid.columns])
 
     ## pivot the magnitude column to display n value for each magnitude at each hydroid
-    df_nvalues_mag = df_nvalues.pivot_table(index='HydroID', columns='magnitude', values=['hydroid_ManningN'], aggfunc='mean') # if there are multiple entries per hydroid and magnitude - aggregate using mean
+    df_nvalues_mag = df_nvalues.pivot_table(index='HydroID', columns='magnitude', values=['hydroid_calb_coef'], aggfunc='mean') # if there are multiple entries per hydroid and magnitude - aggregate using mean
     
     ## Optional: Export csv with the newly calculated Manning's N values
     if debug_outputs_option:
-        output_calc_n_csv = os.path.join(fim_directory, 'calc_src_n_vals_' + branch_id + '.csv')
+        output_calc_n_csv = os.path.join(fim_directory, calb_type + '_src_calcs_' + branch_id + '.csv')
         df_nvalues.to_csv(output_calc_n_csv,index=False)
 
     ## filter the modified Manning's n dataframe for values out side allowable range
@@ -161,18 +166,18 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
         df_updated.rename(columns={'coll_time':'last_updated'}, inplace=True)
 
         ## cacluate median ManningN to handle cases with multiple hydroid entries
-        df_mann_hydroid = df_nvalues.groupby(["HydroID"])[['hydroid_ManningN']].median()
+        df_mann_hydroid = df_nvalues.groupby(["HydroID"])[['hydroid_calb_coef']].median()
 
         ## Create a df with the median hydroid_ManningN value per feature_id
         #df_mann_featid = df_nvalues.groupby(["feature_id"])[['hydroid_ManningN']].mean()
         #df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
 
         ## Rename the original hydrotable variables to allow new calculations to use the primary var name
-        df_htable.rename(columns={'ManningN':'default_ManningN','discharge_cms':'default_discharge_cms'}, inplace=True)
+        df_htable.rename(columns={'discharge_cms':'precalb_discharge_cms'}, inplace=True)
 
         ## Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a singel hydroid)
-        df_nrange = df_nvalues.groupby('HydroID').agg({'hydroid_ManningN': ['median', 'min', 'max', 'std', 'count']})
-        df_nrange['hydroid_ManningN','range'] = df_nrange['hydroid_ManningN','max'] - df_nrange['hydroid_ManningN','min']
+        df_nrange = df_nvalues.groupby('HydroID').agg({'hydroid_calb_coef': ['median', 'min', 'max', 'std', 'count']})
+        df_nrange['hydroid_calb_coef','range'] = df_nrange['hydroid_calb_coef','max'] - df_nrange['hydroid_calb_coef','min']
         df_nrange = df_nrange.join(df_nvalues_mag, how='outer') # join the df_nvalues_mag containing hydroid_manningn values per flood magnitude category
         df_nrange = df_nrange.merge(df_huc_lid, how='outer', on='HydroID') # join the df_huc_lid df to add attributes for lid and huc#
         log_text += 'Statistics for Modified Roughness Calcs -->' +'\n'
@@ -181,7 +186,7 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
 
         ## Optional: Output csv with SRC calc stats
         if debug_outputs_option:
-            output_stats_n_csv = os.path.join(fim_directory, 'stats_src_n_vals_' + branch_id + '.csv')
+            output_stats_n_csv = os.path.join(fim_directory, calb_type + '_src_coef_vals_stats_' + branch_id + '.csv')
             df_nrange.to_csv(output_stats_n_csv,index=True)
 
         ## subset the original hydrotable dataframe and subset to one row per HydroID
@@ -192,7 +197,7 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
         if not df_htable_check_lakes.empty:
 
             ## Create attributes to traverse the flow network between HydroIDs
-            df_nmerge = branch_network(df_nmerge)
+            df_nmerge = branch_network_tracer(df_nmerge)
 
             ## Merge the newly caluclated ManningN dataframes
             df_nmerge = df_nmerge.merge(df_mann_hydroid, how='left', on='HydroID')
@@ -201,49 +206,49 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
             ## Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg)
             df_nmerge = group_manningn_calc(df_nmerge, down_dist_thresh)
 
-            ## Create a df with the median hydroid_ManningN value per feature_id
-            df_mann_featid = df_nmerge.groupby(["feature_id"])[['hydroid_ManningN']].mean()
-            df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
+            ## Create a df with the median hydroid_calb_coef value per feature_id
+            df_mann_featid = df_nmerge.groupby(["feature_id"])[['hydroid_calb_coef']].mean()
+            df_mann_featid.rename(columns={'hydroid_calb_coef':'featid_calb_coef'}, inplace=True)
             df_mann_featid_attrib = df_nmerge.groupby('feature_id').first() # create a seperate df with attributes to apply to other hydroids that share a featureid
             df_mann_featid_attrib = df_mann_featid_attrib[df_mann_featid_attrib['submitter'].notna()][['last_updated','submitter']]
             df_nmerge = df_nmerge.merge(df_mann_featid, how='left', on='feature_id').set_index('feature_id')
             df_nmerge = df_nmerge.combine_first(df_mann_featid_attrib).reset_index()
             
-            if not df_nmerge['hydroid_ManningN'].isnull().all():
-                ## Temp testing filter to only use the hydroid manning n values (drop the featureid and group ManningN variables)
-                #df_nmerge = df_nmerge.assign(featid_ManningN=np.nan)
-                #df_nmerge = df_nmerge.assign(group_ManningN=np.nan)
-
-                ## Create the adjust_ManningN column by combining the hydroid_ManningN with the featid_ManningN (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
-                conditions  = [ (df_nmerge['hydroid_ManningN'].isnull()) & (df_nmerge['featid_ManningN'].notnull()), (df_nmerge['hydroid_ManningN'].isnull()) & (df_nmerge['featid_ManningN'].isnull()) & (df_nmerge['group_ManningN'].notnull()) ]
-                choices     = [ df_nmerge['featid_ManningN'], df_nmerge['group_ManningN'] ]
-                df_nmerge['adjust_ManningN'] = np.select(conditions, choices, default=df_nmerge['hydroid_ManningN'])
-                df_nmerge['obs_source'] = np.where(df_nmerge['adjust_ManningN'].notnull(), source_tag, pd.NA)
-                df_nmerge.drop(['feature_id','NextDownID','LENGTHKM','LakeID','order_'], axis=1, inplace=True) # drop these columns to avoid duplicates where merging with the full hydroTable df
-
+            if not df_nmerge['hydroid_calb_coef'].isnull().all():
+
+                ## Create the calibration coefficient column by combining the hydroid_calb_coef with the featid_calb_coef (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_calb_coef value(s))
+                conditions  = [ (df_nmerge['hydroid_calb_coef'].isnull()) & (df_nmerge['featid_calb_coef'].notnull()), (df_nmerge['hydroid_calb_coef'].isnull()) & (df_nmerge['featid_calb_coef'].isnull()) & (df_nmerge['group_calb_coef'].notnull()) ]
+                choices     = [ df_nmerge['featid_calb_coef'], df_nmerge['group_calb_coef'] ]
+                df_nmerge[calb_type] = np.select(conditions, choices, default=df_nmerge['hydroid_calb_coef'])
+                df_nmerge['obs_source'] = np.where(df_nmerge[calb_type].notnull(), source_tag, pd.NA)
+                df_nmerge.drop(['feature_id','NextDownID','LENGTHKM','LakeID','order_'], axis=1, inplace=True, errors='ignore') # drop these columns to avoid duplicates where merging with the full hydroTable df
+                
                 ## Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new adjusted roughness value
                 if not df_prev_adj.empty:
                     df_nmerge = pd.merge(df_nmerge,df_prev_adj, on='HydroID', how='outer')
-                    df_nmerge['submitter'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['submitter_prev'],df_nmerge['submitter'])
-                    df_nmerge['last_updated'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['last_updated_prev'],df_nmerge['last_updated'])
-                    df_nmerge['obs_source'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['obs_source_prev'],df_nmerge['obs_source'])
-                    df_nmerge['adjust_ManningN'] = np.where((df_nmerge['adjust_ManningN'].isnull() & df_nmerge['adjust_ManningN_prev'].notnull()),df_nmerge['adjust_ManningN_prev'],df_nmerge['adjust_ManningN'])
-                    df_nmerge.drop(['submitter_prev','last_updated_prev','adjust_ManningN_prev','obs_source_prev'], axis=1, inplace=True)
+                    df_nmerge['submitter'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['submitter_prev'],df_nmerge['submitter'])
+                    df_nmerge['last_updated'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['last_updated_prev'],df_nmerge['last_updated'])
+                    df_nmerge['obs_source'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['obs_source_prev'],df_nmerge['obs_source'])
+                    df_nmerge['calb_coef_final'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['calb_coef_final_prev'],df_nmerge[calb_type])
+                    df_nmerge.drop(['submitter_prev','last_updated_prev','calb_coef_final_prev','obs_source_prev'], axis=1, inplace=True, errors='ignore')
+                else:
+                    df_nmerge['calb_coef_final'] = df_nmerge[calb_type]
+                
                 
                 ## Update the catchments polygon .gpkg with joined attribute - "src_calibrated"
                 if os.path.isfile(catchments_poly_path):
                     input_catchments = gpd.read_file(catchments_poly_path)
                     ## Create new "src_calibrated" column for viz query
                     if 'src_calibrated' in input_catchments.columns: # check if this attribute already exists and drop if needed
-                        input_catchments.drop(['src_calibrated'], axis=1, inplace=True)
-                    df_nmerge['src_calibrated'] = np.where(df_nmerge['adjust_ManningN'].notnull(), 'True', 'False')
+                        input_catchments.drop(['src_calibrated'], axis=1, inplace=True, errors='ignore')
+                    df_nmerge['src_calibrated'] = np.where(df_nmerge['calb_coef_final'].notnull(), 'True', 'False')
                     output_catchments = input_catchments.merge(df_nmerge[['HydroID','src_calibrated']], how='left', on='HydroID')
                     output_catchments['src_calibrated'].fillna('False', inplace=True)
                     output_catchments.to_file(catchments_poly_path,driver="GPKG",index=False) # overwrite the previous layer
-                    df_nmerge.drop(['src_calibrated'], axis=1, inplace=True)
+                    df_nmerge.drop(['src_calibrated'], axis=1, inplace=True, errors='ignore')
                 ## Optional ouputs: 1) merge_n_csv csv with all of the calculated n values and 2) a catchments .gpkg with new joined attributes
                 if debug_outputs_option:
-                    output_merge_n_csv = os.path.join(fim_directory, 'merge_src_n_vals_' + branch_id + '.csv')
+                    output_merge_n_csv = os.path.join(fim_directory, calb_type + '_merge_vals_' + branch_id + '.csv')
                     df_nmerge.to_csv(output_merge_n_csv,index=False)
                     ## output new catchments polygon layer with several new attributes appended
                     if os.path.isfile(catchments_poly_path):
@@ -253,21 +258,16 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
                         output_catchments.to_file(output_catchments_fileName,driver="GPKG",index=False)
 
                 ## Merge the final ManningN dataframe to the original hydroTable
-                df_nmerge.drop(['ahps_lid','start_catch','route_count','branch_id','hydroid_ManningN','featid_ManningN','group_ManningN',], axis=1, inplace=True) # drop these columns to avoid duplicates where merging with the full hydroTable df
+                df_nmerge.drop(['ahps_lid','start_catch','route_count','branch_id','hydroid_calb_coef','featid_calb_coef','group_calb_coef',], axis=1, inplace=True, errors='ignore') # drop these columns to avoid duplicates where merging with the full hydroTable df
                 df_htable = df_htable.merge(df_nmerge, how='left', on='HydroID')
-                df_htable['adjust_src_on'] = np.where(df_htable['adjust_ManningN'].notnull(), 'True', 'False') # create true/false column to clearly identify where new roughness values are applied
-
-                ## Create the ManningN column by combining the hydroid_ManningN with the default_ManningN (use modified where available)
-                df_htable['ManningN'] = np.where(df_htable['adjust_ManningN'].isnull(),df_htable['default_ManningN'],df_htable['adjust_ManningN'])
+                df_htable['calb_applied'] = np.where(df_htable['calb_coef_final'].notnull(), 'True', 'False') # create true/false column to clearly identify where new roughness values are applied
 
                 ## Calculate new discharge_cms with new adjusted ManningN
-                df_htable['discharge_cms'] = df_htable['WetArea (m2)']* \
-                pow(df_htable['HydraulicRadius (m)'],2.0/3)* \
-                pow(df_htable['SLOPE'],0.5)/df_htable['ManningN']
+                df_htable['discharge_cms'] = np.where(df_htable['calb_coef_final'].isnull(), df_htable['precalb_discharge_cms'], df_htable['precalb_discharge_cms']/df_htable['calb_coef_final'])
 
                 ## Replace discharge_cms with 0 or -999 if present in the original discharge (carried over from thalweg notch workaround in SRC post-processing)
-                df_htable['discharge_cms'].mask(df_htable['default_discharge_cms']==0.0,0.0,inplace=True)
-                df_htable['discharge_cms'].mask(df_htable['default_discharge_cms']==-999,-999,inplace=True)
+                df_htable['discharge_cms'].mask(df_htable['precalb_discharge_cms']==0.0,0.0,inplace=True)
+                df_htable['discharge_cms'].mask(df_htable['precalb_discharge_cms']==-999,-999,inplace=True)
 
                 ## Export a new hydroTable.csv and overwrite the previous version
                 out_htable = os.path.join(fim_directory, 'hydroTable_' + branch_id + '.csv')
@@ -288,7 +288,7 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     print("Completed huc: " + str(huc) + ' --> branch: ' + str(branch_id))
     return(log_text)
 
-def branch_network(df_input_htable):
+def branch_network_tracer(df_input_htable):
     df_input_htable = df_input_htable.astype({'NextDownID': 'int64'}) # ensure attribute has consistent format as int
     df_input_htable = df_input_htable.loc[df_input_htable['LakeID'] == -999] # remove all hydroids associated with lake/water body (these often have disjoined artifacts in the network)
     df_input_htable["start_catch"] = ~df_input_htable['HydroID'].isin(df_input_htable['NextDownID']) # define start catchments as hydroids that are not found in the "NextDownID" attribute for all other hydroids
@@ -323,17 +323,17 @@ def branch_network(df_input_htable):
     return(df_input_htable)
 
 def group_manningn_calc(df_nmerge, down_dist_thresh):
-    ## Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg
+    ## Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg
     #df_nmerge.sort_values(by=['NextDownID'], inplace=True)
     dist_accum = 0; hyid_count = 0; hyid_accum_count = 0; 
-    run_accum_mann = 0; group_ManningN = 0; branch_start = 1                                        # initialize counter and accumulation variables
+    run_accum_mann = 0; group_calb_coef = 0; branch_start = 1                                        # initialize counter and accumulation variables
     lid_count = 0; prev_lid = 'x'
     for index, row in df_nmerge.iterrows():                                                         # loop through the df (parse by hydroid)
         if int(df_nmerge.loc[index,'branch_id']) != branch_start:                                   # check if start of new branch
             dist_accum = 0; hyid_count = 0; hyid_accum_count = 0;                                   # initialize counter vars
-            run_accum_mann = 0; group_ManningN = 0                                                  # initialize counter vars
+            run_accum_mann = 0; group_calb_coef = 0                                                  # initialize counter vars
             branch_start = int(df_nmerge.loc[index,'branch_id'])                                    # reassign the branch_start var to evaluate on next iteration
-            # use the code below to withold downstream hydroid_ManningN values (use this for downstream evaluation tests)
+            # use the code below to withold downstream hydroid_calb_coef values (use this for downstream evaluation tests)
             '''
             lid_count = 0                                                                         
         if not pd.isna(df_nmerge.loc[index,'ahps_lid']):
@@ -346,30 +346,30 @@ def group_manningn_calc(df_nmerge, down_dist_thresh):
                 lid_count = 1
             prev_lid = df_nmerge.loc[index,'ahps_lid']
             '''
-        if np.isnan(df_nmerge.loc[index,'hydroid_ManningN']):                                       # check if the hydroid_ManningN value is nan (indicates a non-calibrated hydroid)
+        if np.isnan(df_nmerge.loc[index,'hydroid_calb_coef']):                                       # check if the hydroid_calb_coef value is nan (indicates a non-calibrated hydroid)
             df_nmerge.loc[index,'accum_dist'] = row['LENGTHKM'] + dist_accum                        # calculate accumulated river distance
             dist_accum += row['LENGTHKM']                                                           # add hydroid length to the dist_accum var
             hyid_count = 0                                                                          # reset the hydroid counter to 0
             df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hydroid accum counter
-            if dist_accum < down_dist_thresh:                                                   # check if the accum distance is less than Xkm downstream from valid hydroid_ManningN group value
-                if hyid_accum_count > 1:                                                            # only apply the group_ManningN if there are 2 or more valid hydorids that contributed to the upstream group_ManningN
-                    df_nmerge.loc[index,'group_ManningN'] = group_ManningN                          # output the group_ManningN var
+            if dist_accum < down_dist_thresh:                                                   # check if the accum distance is less than Xkm downstream from valid hydroid_calb_coef group value
+                if hyid_accum_count > 1:                                                            # only apply the group_calb_coef if there are 2 or more valid hydorids that contributed to the upstream group_calb_coef
+                    df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                          # output the group_calb_coef var
             else:
                 run_avg_mann = 0                                                                    # reset the running average manningn variable (greater than 10km downstream)
-        else:                                                                                       # performs the following for hydroids that have a valid hydroid_ManningN value
+        else:                                                                                       # performs the following for hydroids that have a valid hydroid_calb_coef value
             dist_accum = 0; hyid_count += 1                                                         # initialize vars
             df_nmerge.loc[index,'accum_dist'] = 0                                                   # output the accum_dist value (set to 0)
-            if hyid_count == 1:                                                                     # checks if this the first in a series of valid hydroid_ManningN values
+            if hyid_count == 1:                                                                     # checks if this the first in a series of valid hydroid_calb_coef values
                 run_accum_mann = 0; hyid_accum_count = 0                                            # initialize counter and running accumulated manningN value
-            group_ManningN = (row['hydroid_ManningN'] + run_accum_mann)/float(hyid_count)           # calculate the group_ManningN (NOTE: this will continue to change as more hydroid values are accumulated in the "group" moving downstream)
-            df_nmerge.loc[index,'group_ManningN'] = group_ManningN                                  # output the group_ManningN var 
+            group_calb_coef = (row['hydroid_calb_coef'] + run_accum_mann)/float(hyid_count)           # calculate the group_calb_coef (NOTE: this will continue to change as more hydroid values are accumulated in the "group" moving downstream)
+            df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                                  # output the group_calb_coef var 
             df_nmerge.loc[index,'hyid_count'] = hyid_count                                          # output the hyid_count var 
-            run_accum_mann += row['hydroid_ManningN']                                               # add current hydroid manningn value to the running accum mann var
+            run_accum_mann += row['hydroid_calb_coef']                                               # add current hydroid manningn value to the running accum mann var
             hyid_accum_count += 1                                                                   # increase the # of hydroid accum counter
             df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hyid_accum_count var
 
     ## Delete unnecessary intermediate outputs
     if 'hyid_count' in df_nmerge.columns:
-        df_nmerge.drop(['hyid_count','accum_dist','hyid_accum_count'], axis=1, inplace=True) # drop hydroid counter if it exists
+        df_nmerge.drop(['hyid_count','accum_dist','hyid_accum_count'], axis=1, inplace=True, errors='ignore') # drop hydroid counter if it exists
     #df_nmerge.drop(['accum_dist','hyid_accum_count'], axis=1, inplace=True) # drop accum vars from group calc
     return(df_nmerge)
\ No newline at end of file
diff --git a/src/subdiv_chan_obank_src.py b/src/subdiv_chan_obank_src.py
new file mode 100755
index 000000000..ae09508df
--- /dev/null
+++ b/src/subdiv_chan_obank_src.py
@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+import os
+import sys
+import argparse
+import datetime as dt
+import matplotlib.pyplot as plt
+import multiprocessing
+import numpy as np
+import pandas as pd
+import re
+import seaborn as sns
+import shutil
+import traceback
+import warnings
+from functools import reduce
+from multiprocessing import Pool
+from os.path import isfile, join, dirname, isdir
+from pathlib import Path
+from tqdm import tqdm
+
+sns.set_theme(style="whitegrid")
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+"""
+    Compute channel geomety and Manning's equation using subdivision method (separate in-channel vs. overbank).
+    Also apply unique Manning's n-values for channel and overbank using a user provided feature_id csv
+
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders.
+    mann_n_table : str
+        Path to a csv file containing Manning's n values by feature_id (must contain variables "feature_id", "channel_n", "overbank_n")
+    file_suffix : str
+        Optional: Suffix to append to the output log file
+    number_of_jobs : str
+        Number of jobs.
+    src_plot_option : str
+        Optional (True or False): use this flag to crate src plots for all hydroids (long run time)
+"""
+
+def variable_mannings_calc(args):
+
+    in_src_bankfull_filename    = args[0]
+    df_mann                     = args[1]
+    huc                         = args[2]
+    branch_id                   = args[3]
+    htable_filename             = args[4]
+    output_suffix               = args[5]
+    src_plot_option             = args[6]
+    huc_output_dir              = args[7]
+
+    ## Read the src_full_crosswalked.csv
+    log_text = 'Calculating modified SRC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+    try:
+        df_src_orig = pd.read_csv(in_src_bankfull_filename,dtype={'feature_id': 'int64'})
+
+        ## Check that the channel ratio column the user specified exists in the def
+        if 'Stage_bankfull' not in df_src_orig.columns:
+            print('WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified bankfull column: ' + 'Stage_bankfull')
+            print('Skipping --> ' + str(huc) + '  branch id: ' + str(branch_id))
+            log_text += 'WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified bankfull column: ' + 'Stage_bankfull'  + '\n'
+        else:
+            df_src_orig.drop(['channel_n','overbank_n','subdiv_applied','Discharge (m3s-1)_subdiv','Volume_chan (m3)','Volume_obank (m3)','BedArea_chan (m2)','BedArea_obank (m2)','WettedPerimeter_chan (m)','WettedPerimeter_obank (m)'], axis=1, inplace=True, errors='ignore') # drop these cols (in case vmann was previously performed)
+            
+            ## Calculate subdiv geometry variables
+            print('Calculating subdiv variables for SRC: ' + str(huc) + '  branch id: ' + str(branch_id))
+            log_text = 'Calculating subdiv variables for SRC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+            df_src = subdiv_geometry(df_src_orig)
+            
+            ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the 'Stage_bankfull')
+            df_src = df_src.merge(df_mann,  how='left', on='feature_id')
+            check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
+            if check_null > 0:
+                log_text += str(huc) + '  branch id: ' + str(branch_id) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
+
+            ## Check if there are any missing data in the 'Stage_bankfull' column (these are locations where subdiv will not be applied)
+            df_src['subdiv_applied'] = np.where(df_src['Stage_bankfull'].isnull(), False, True) # create field to identify where vmann is applied (True=yes; False=no)        
+
+            ## Calculate Manning's equation discharge for channel, overbank, and total
+            df_src = subdiv_mannings_eq(df_src)
+
+            ## Use the default discharge column when vmann is not being applied
+            df_src['Discharge (m3s-1)_subdiv'] = np.where(df_src['subdiv_applied']==False, df_src['Discharge (m3s-1)'], df_src['Discharge (m3s-1)_subdiv']) # reset the discharge value back to the original if vmann=false
+
+            ## Output new SRC with bankfull column
+            df_src.to_csv(in_src_bankfull_filename,index=False)
+
+            ## Output new hydroTable with updated discharge and ManningN column
+            df_src_trim = df_src[['HydroID','Stage','subdiv_applied','channel_n','overbank_n','Discharge (m3s-1)_subdiv']]
+            df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_subdiv': 'subdiv_discharge_cms'})
+            df_src_trim['discharge_cms'] = df_src_trim['subdiv_discharge_cms'] # create a copy of vmann modified discharge (used to track future changes)
+            df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
+
+            ## drop the previously modified discharge column to be replaced with updated version
+            df_htable.drop(['subdiv_applied','discharge_cms','overbank_n','channel_n','subdiv_discharge_cms'], axis=1, errors='ignore', inplace=True) 
+            df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
+
+            ## Output new hydroTable csv
+            if output_suffix != "":
+                htable_filename = os.path.splitext(htable_filename)[0] + output_suffix + '.csv' 
+            df_htable.to_csv(htable_filename,index=False)
+
+            log_text += 'Completed: ' + str(huc)
+
+            ## plot rating curves
+            if src_plot_option:
+                if isdir(huc_output_dir) == False:
+                    os.mkdir(huc_output_dir)
+                generate_src_plot(df_src, huc_output_dir)
+    except Exception as ex:
+        summary = traceback.StackSummary.extract(
+                traceback.walk_stack(None))
+        print('WARNING: ' + str(huc) + '  branch id: ' + str(branch_id) + " subdivision failed for some reason")
+        #print(f"*** {ex}")                
+        #print(''.join(summary.format()))    
+        log_text += 'ERROR --> ' + str(huc) + '  branch id: ' + str(branch_id) + " subdivision failed (details: " + (f"*** {ex}") + (''.join(summary.format())) + '\n'
+
+    return(log_text)
+
+def subdiv_geometry(df_src):
+
+    ## Calculate in-channel volume & bed area
+    df_src['Volume_chan (m3)'] = np.where(df_src['Stage']<=df_src['Stage_bankfull'], df_src['Volume (m3)'], (df_src['Volume_bankfull'] + ((df_src['Stage'] - df_src['Stage_bankfull']) * df_src['SurfArea_bankfull'])))
+    df_src['BedArea_chan (m2)'] = np.where(df_src['Stage']<=df_src['Stage_bankfull'], df_src['BedArea (m2)'], df_src['BedArea_bankfull'])
+    df_src['WettedPerimeter_chan (m)'] = np.where(df_src['Stage']<=df_src['Stage_bankfull'], (df_src['BedArea_chan (m2)']/df_src['LENGTHKM']/1000), (df_src['BedArea_chan (m2)']/df_src['LENGTHKM']/1000) + ((df_src['Stage'] - df_src['Stage_bankfull'])*2))
+
+    ## Calculate overbank volume & bed area
+    df_src['Volume_obank (m3)'] = np.where(df_src['Stage']>df_src['Stage_bankfull'], (df_src['Volume (m3)'] - df_src['Volume_chan (m3)']), 0.0)
+    df_src['BedArea_obank (m2)'] = np.where(df_src['Stage']>df_src['Stage_bankfull'], (df_src['BedArea (m2)'] - df_src['BedArea_chan (m2)']), 0.0)
+    df_src['WettedPerimeter_obank (m)'] = df_src['BedArea_obank (m2)']/df_src['LENGTHKM']/1000
+    return(df_src)
+
+def subdiv_mannings_eq(df_src):
+    ## Calculate discharge (channel) using Manning's equation
+    df_src.drop(['WetArea_chan (m2)','HydraulicRadius_chan (m)','Discharge_chan (m3s-1)','Velocity_chan (m/s)'], axis=1, inplace=True, errors='ignore') # drop these cols (in case subdiv was previously performed)
+    df_src['WetArea_chan (m2)'] = df_src['Volume_chan (m3)']/df_src['LENGTHKM']/1000
+    df_src['HydraulicRadius_chan (m)'] = df_src['WetArea_chan (m2)']/df_src['WettedPerimeter_chan (m)']
+    df_src['HydraulicRadius_chan (m)'].fillna(0, inplace=True)
+    df_src['Discharge_chan (m3s-1)'] = df_src['WetArea_chan (m2)']* \
+    pow(df_src['HydraulicRadius_chan (m)'],2.0/3)* \
+    pow(df_src['SLOPE'],0.5)/df_src['channel_n']
+    df_src['Velocity_chan (m/s)'] = df_src['Discharge_chan (m3s-1)']/df_src['WetArea_chan (m2)']
+    df_src['Velocity_chan (m/s)'].fillna(0, inplace=True)
+
+    ## Calculate discharge (overbank) using Manning's equation
+    df_src.drop(['WetArea_obank (m2)','HydraulicRadius_obank (m)','Discharge_obank (m3s-1)','Velocity_obank (m/s)'], axis=1, inplace=True, errors='ignore') # drop these cols (in case subdiv was previously performed)
+    df_src['WetArea_obank (m2)'] = df_src['Volume_obank (m3)']/df_src['LENGTHKM']/1000
+    df_src['HydraulicRadius_obank (m)'] = df_src['WetArea_obank (m2)']/df_src['WettedPerimeter_obank (m)']
+    df_src.replace([np.inf, -np.inf], np.nan, inplace=True) # need to replace inf instances (divide by 0)
+    df_src['HydraulicRadius_obank (m)'].fillna(0, inplace=True)
+    df_src['Discharge_obank (m3s-1)'] = df_src['WetArea_obank (m2)']* \
+    pow(df_src['HydraulicRadius_obank (m)'],2.0/3)* \
+    pow(df_src['SLOPE'],0.5)/df_src['overbank_n']
+    df_src['Velocity_obank (m/s)'] = df_src['Discharge_obank (m3s-1)']/df_src['WetArea_obank (m2)']
+    df_src['Velocity_obank (m/s)'].fillna(0, inplace=True)
+
+    ## Calcuate the total of the subdivided discharge (channel + overbank)
+    df_src.drop(['Discharge (m3s-1)_subdiv'], axis=1, inplace=True, errors='ignore') # drop these cols (in case subdiv was previously performed)
+    df_src['Discharge (m3s-1)_subdiv'] = df_src['Discharge_chan (m3s-1)'] + df_src['Discharge_obank (m3s-1)']
+    return(df_src)
+
+def generate_src_plot(df_src, plt_out_dir):
+
+    ## create list of unique hydroids
+    hydroids = df_src.HydroID.unique().tolist()
+
+    ## plot each hydroid SRC in the huc
+    for hydroid in hydroids:
+        print("Creating SRC plot: " + str(hydroid))
+        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+
+        f, ax = plt.subplots(figsize=(6.5, 6.5))
+        ax.set_title(str(hydroid))
+        sns.despine(f, left=True, bottom=True)
+        sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
+        sns.scatterplot(x='Discharge (m3s-1)_subdiv', y='Stage', data=plot_df, label="SRC w/ Subdiv", ax=ax, color='orange')
+        sns.scatterplot(x='Discharge_chan (m3s-1)', y='Stage', data=plot_df, label="SRC Channel", ax=ax, color='green', s=8)
+        sns.scatterplot(x='Discharge_obank (m3s-1)', y='Stage', data=plot_df, label="SRC Overbank", ax=ax, color='purple', s=8)
+        sns.lineplot(x='Discharge (m3s-1)', y='Stage_bankfull', data=plot_df, color='green', ax=ax)
+        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'],alpha=0.5)
+        plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_bankfull'].median(), "NWM Bankfull Approx: " + str(plot_df['Stage_bankfull'].median()))
+        ax.legend()
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png',dpi=175, bbox_inches='tight')
+        plt.close()
+
+def multi_process(variable_mannings_calc, procs_list, log_file, number_of_jobs, verbose):
+    ## Initiate multiprocessing
+    available_cores = multiprocessing.cpu_count()
+    if number_of_jobs > available_cores:
+        number_of_jobs = available_cores - 2
+        print("Provided job number exceeds the number of available cores. " + str(number_of_jobs) + " max jobs will be used instead.")
+
+    print(f"Computing subdivided SRC and applying variable Manning's n to channel/overbank for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    with Pool(processes=number_of_jobs) as pool:
+        if verbose:
+            map_output = tqdm(pool.imap(variable_mannings_calc, procs_list), total=len(procs_list))
+            tuple(map_output)  # fetch the lazy results
+        else:
+            map_output = pool.map(variable_mannings_calc, procs_list)
+    log_file.writelines(["%s\n" % item  for item in map_output])
+
+def run_prep(fim_dir,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option):
+    procs_list = []
+
+    print('Writing progress to log file here: ' + str(join(fim_dir,'logs','subdiv_src_' + output_suffix + '.log')))
+    print('This may take a few minutes...')
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+
+    ## initiate log file
+    log_file = open(join(fim_dir,'logs','subdiv_src_' + output_suffix + '.log'),"w")
+    log_file.write('START TIME: ' + str(begin_time) + '\n')
+    log_file.write('#########################################################\n\n')
+
+    ## Check that the input fim_dir exists
+    assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
+    ## Check that the manning's roughness input filepath exists and then read to dataframe
+    assert os.path.isfile(mann_n_table), 'Can not find the input roughness/feature_id file: ' + str(mann_n_table)
+    
+    ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
+    print('Importing the Manning roughness data file: ' + mann_n_table)
+    df_mann = pd.read_csv(mann_n_table,dtype={'feature_id': 'int64'})
+    if 'channel_n' not in df_mann.columns or 'overbank_n' not in df_mann.columns or 'feature_id' not in df_mann.columns:
+        print('Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann)
+    else:
+        print('Running the variable_mannings_calc function...')
+
+        ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
+        huc_list  = os.listdir(fim_dir)
+        huc_list.sort() # sort huc_list for helping track progress in future print statments
+        for huc in huc_list:
+            #if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+            if re.match('\d{8}', huc):    
+                huc_branches_dir = os.path.join(fim_dir, huc,'branches')
+                for branch_id in os.listdir(huc_branches_dir):
+                    branch_dir = os.path.join(huc_branches_dir,branch_id)
+                    in_src_bankfull_filename = join(branch_dir,'src_full_crosswalked_' + branch_id + '.csv')
+                    htable_filename = join(branch_dir,'hydroTable_' + branch_id + '.csv')
+                    huc_plot_output_dir = join(branch_dir,'src_plots')
+
+                    if isfile(in_src_bankfull_filename) and isfile(htable_filename):
+                        procs_list.append([in_src_bankfull_filename, df_mann, huc, branch_id, htable_filename, output_suffix, src_plot_option, huc_plot_output_dir])
+                    else:
+                        print('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+                        log_file.write('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+
+        ## Pass huc procs_list to multiprocessing function
+        multi_process(variable_mannings_calc, procs_list, log_file, number_of_jobs, verbose)
+
+        ## Record run time and close log file
+        end_time = dt.datetime.now()
+        log_file.write('END TIME: ' + str(end_time) + '\n')
+        tot_run_time = end_time - begin_time
+        log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+        log_file.close()
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description="Subdivide the default SRC to compute a seperate channel component and overbank component. Impliment user provided Manning's n values for in-channel vs. overbank flow. Recalculate Manning's eq for discharge")
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-mann','--mann-n-table',help="Path to a csv file containing Manning's n values by featureid",required=True,type=str)
+    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",default="",required=False,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='OPTIONAL: number of workers (default=8)',required=False,default=8,type=int)
+    parser.add_argument('-vb','--verbose',help='OPTIONAL: verbose progress bar',required=False,default=None,action='store_true')
+    parser.add_argument('-plots','--src-plot-option',help='OPTIONAL flag: use this flag to create src plots for all hydroids. WARNING - long runtime',default=False,required=False, action='store_true')
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    mann_n_table = args['mann_n_table']
+    output_suffix = args['output_suffix']
+    number_of_jobs = args['number_of_jobs']
+    verbose = bool(args['verbose'])
+    src_plot_option = args['src_plot_option']
+
+    run_prep(fim_dir,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option)
\ No newline at end of file
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index d92d10a0b..3a5801aaf 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -55,7 +55,7 @@
 
 # Rating Curve Adjustment (local calibration) variables
 DOWNSTREAM_THRESHOLD = 10 # distance in km to propogate new roughness values downstream
-ROUGHNESS_MAX_THRESH = 0.6 # max allowable adjusted roughness value (void values larger than this)
+ROUGHNESS_MAX_THRESH = 0.8 # max allowable adjusted roughness value (void values larger than this)
 ROUGHNESS_MIN_THRESH = 0.001 # min allowable adjusted roughness value (void values smaller than this)
 
 ## Input Paths and Directories
diff --git a/src/vary_mannings_n_composite.py b/src/vary_mannings_n_composite.py
deleted file mode 100755
index 0490cc368..000000000
--- a/src/vary_mannings_n_composite.py
+++ /dev/null
@@ -1,256 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import pandas as pd
-import numpy as np
-import argparse
-import matplotlib.pyplot as plt
-import seaborn as sns
-from functools import reduce
-from multiprocessing import Pool
-from os.path import isfile, join, dirname, isdir
-import shutil
-import warnings
-from pathlib import Path
-import datetime as dt
-sns.set_theme(style="whitegrid")
-warnings.simplefilter(action='ignore', category=FutureWarning)
-
-"""
-    Vary the Manning's n values for in-channel vs. floodplain
-
-    Parameters
-    ----------
-    fim_dir : str
-        Directory containing FIM output folders.
-    channel_ratio_src_column : str
-        SRC attribute containing the channel vs. floodplain attribute
-    mann_n_table : str
-        Path to a csv file containing Manning's n values by feature_id
-    file_suffix : str
-        Suffix to append to the output log file
-    number_of_jobs : str
-        Number of jobs.
-    src_plot_option : str
-        Optional (True or False): use this flag to crate src plots for all hydroids
-"""
-
-def variable_mannings_calc(args):
-
-    in_src_bankfull_filename    = args[0]
-    channel_ratio_src_column    = args[1]
-    df_mann                     = args[2]
-    huc                         = args[3]
-    out_src_vmann_filename   = args[4]
-    htable_filename             = args[5]
-    src_plot_option             = args[6]
-    huc_output_dir              = args[7]
-    viz_clean_flag              = args[8]
-
-    ## Read the src_full_crosswalked.csv
-    log_text = 'Calculating: ' + str(huc) + '\n'
-    df_src = pd.read_csv(in_src_bankfull_filename,dtype={'feature_id': 'int64'})
-
-    ## Check that the channel ratio column the user specified exists in the def
-    if channel_ratio_src_column not in df_src.columns:
-        log_text += 'WARNING --> ' + str(huc) + in_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
-    else:
-        ## Raname the current discharge & ManningN columns
-        df_src = df_src.rename(columns={'Discharge (m3s-1)':'default_Discharge (m3s-1)','ManningN':'default_ManningN'})
-        ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the channel_ratio_src_column)
-        df_src = df_src.merge(df_mann,  how='left', on='feature_id')
-        check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
-        if check_null > 0:
-            log_text += str(huc) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
-
-        ## Calculate composite Manning's n using the channel geometry ratio attribute given by user (e.g. chann_hradius_ratio or chann_vol_ratio)
-        df_src['comp_ManningN'] = (df_src[channel_ratio_src_column]*df_src['channel_n']) + ((1.0 - df_src[channel_ratio_src_column])*df_src['overbank_n'])
-        #print('Done calculating composite Manning n (' + channel_ratio_src_column + '): ' + str(huc))
-
-        ## Check if there are any missing data in the composite ManningN column
-        check_null_comp = df_src['comp_ManningN'].isnull().sum()
-        if check_null_comp > 0:
-            log_text += str(huc) + ' --> ' + 'Missing values in the comp_ManningN calculation' + ' --> missing entries= ' + str(check_null_comp/84)  + '\n'
-        df_src['vmann_on'] = np.where(df_src['comp_ManningN'].isnull(), False, True) # create field to identify where vmann is applied (True=yes; False=no)
-
-        ## Define the channel geometry variable names to use from the src
-        hydr_radius = 'HydraulicRadius (m)'
-        wet_area = 'WetArea (m2)'
-
-        ## Calculate Q using Manning's equation
-        #df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True) # rename the previous Discharge column
-        df_src['Discharge (m3s-1)_varMann'] = df_src[wet_area]* \
-        pow(df_src[hydr_radius],2.0/3)* \
-        pow(df_src['SLOPE'],0.5)/df_src['comp_ManningN']
-
-        ## Set Q values to 0 and -999 for specified criteria
-        df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == 0,0,inplace=True)
-        if 'Thalweg_burn_elev' in df_src:
-            df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == df_src['Thalweg_burn_elev'],0,inplace=True)
-            df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] < df_src['Thalweg_burn_elev'],-999,inplace=True)
-
-        ## Use the default discharge column when vmann is not being applied
-        df_src['Discharge (m3s-1)_varMann'] = np.where(df_src['vmann_on']==False, df_src['default_Discharge (m3s-1)'], df_src['Discharge (m3s-1)_varMann']) # reset the discharge value back to the original if vmann=false
-        df_src['comp_ManningN'] = np.where(df_src['vmann_on']==False, df_src['default_ManningN'], df_src['comp_ManningN']) # reset the ManningN value back to the original if vmann=false
-
-        ## Output new SRC with bankfull column
-        df_src.to_csv(out_src_vmann_filename,index=False)
-
-        ## Output new hydroTable with updated discharge and ManningN column
-        df_src_trim = df_src[['HydroID','Stage','vmann_on',channel_ratio_src_column,'Discharge (m3s-1)_varMann','comp_ManningN']]
-        df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_varMann': 'discharge_cms','comp_ManningN':'ManningN'})
-        df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
-        df_htable.rename(columns={'ManningN':'orig_ManningN'},inplace=True)
-        df_htable.drop(['vmann_on'], axis=1, inplace=True) # drop the default "vmann_on" variable from add_crosswalk.py
-        if not set(['orig_discharge_cms']).issubset(df_htable.columns):
-            df_htable.rename(columns={'discharge_cms':'orig_discharge_cms'},inplace=True)
-        else:
-            df_htable.drop(['discharge_cms'], axis=1, inplace=True) # drop the previously modified discharge column to be replaced with updated version
-        df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
-
-        # Delete intermediate CSVs outputs. Todo delete this block later.
-        htable_parent_dir = os.path.split(htable_filename)[0]
-        # List all CSVs.
-        file_list = os.listdir(htable_parent_dir)
-        for f in file_list:
-            if viz_clean_flag == 1: # if using the viz flag then delete all intermediate csv files
-                if '.csv' in f:
-                    if f != 'hydroTable.csv':
-                        os.remove(os.path.join(htable_parent_dir, f))
-            else:
-                keep_files = ['usgs_elev_table.csv', 'src_base.csv', 'small_segments.csv']
-                if '.csv' in f:
-                    if f not in keep_files:
-                        os.remove(os.path.join(htable_parent_dir, f))
-
-        df_htable.to_csv(htable_filename,index=False)
-
-        log_text += 'Completed: ' + str(huc)
-
-        ## plot rating curves
-        if src_plot_option == 'True':
-            if isdir(huc_output_dir) == False:
-                os.mkdir(huc_output_dir)
-            generate_src_plot(df_src, huc_output_dir)
-
-    return(log_text)
-
-def generate_src_plot(df_src, plt_out_dir):
-
-    ## create list of unique hydroids
-    hydroids = df_src.HydroID.unique().tolist()
-
-    ## plot each hydroid SRC in the huc
-    for hydroid in hydroids:
-        print("Creating SRC plot: " + str(hydroid))
-        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
-
-        f, ax = plt.subplots(figsize=(6.5, 6.5))
-        ax.set_title(str(hydroid))
-        sns.despine(f, left=True, bottom=True)
-        sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
-        sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
-        sns.lineplot(x='Discharge (m3s-1)', y='Stage_1_5', data=plot_df, color='green', ax=ax)
-        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_1_5'],alpha=0.5)
-        plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
-        ax.legend()
-        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png',dpi=175, bbox_inches='tight')
-        plt.close()
-
-#    for hydroid in hydroids:
-#        print("Creating SRC plot: " + str(hydroid))
-#        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
-#
-#        f, ax = plt.subplots(figsize=(6.5, 6.5))
-#        ax.set_title(str(hydroid))
-#        sns.despine(f, left=True, bottom=True)
-#        sns.scatterplot(x='comp_ManningN', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
-#        #sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
-#        sns.lineplot(x='comp_ManningN', y='Stage_1_5', data=plot_df, color='green', ax=ax)
-#        plt.fill_between(plot_df['comp_ManningN'], plot_df['Stage_1_5'],alpha=0.5)
-#        plt.text(plot_df['comp_ManningN'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
-#        ax.legend()
-#        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
-#        plt.close()
-
-def multi_process(variable_mannings_calc, procs_list):
-    ## Initiate multiprocessing
-    print(f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
-    with Pool(processes=number_of_jobs) as pool:
-        map_output = pool.map(variable_mannings_calc, procs_list)
-    log_file.writelines(["%s\n" % item  for item in map_output])
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description="Vary the Manning's n values for in-channel vs. floodplain (recalculate Manning's eq for Discharge)")
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-bc','--channel-ratio-src-column',help='SRC attribute containing the channel vs. overbank geometry ratio (for composite calc)',required=False,type=str,default='chann_hradius_ratio')
-    parser.add_argument('-mann','--mann-n-table',help="Path to a csv file containing Manning's n values by featureid",required=True,type=str)
-    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",required=True,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-    parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
-    parser.add_argument('-viz_clean','--viz-clean',help='Optional (Viz flag): pass the viz flag (0 or 1) to delete intermediate csv files',required=False,default=0,type=int)
-
-    args = vars(parser.parse_args())
-
-    fim_dir = args['fim_dir']
-    channel_ratio_src_column = args['channel_ratio_src_column']
-    mann_n_table = args['mann_n_table']
-    output_suffix = args['output_suffix']
-    number_of_jobs = args['number_of_jobs']
-    src_plot_option = args['src_plot_option']
-    viz_clean_flag = args['viz_clean']
-    procs_list = []
-
-    print('Writing progress to log file here: ' + str(join(fim_dir,'log_composite_n' + output_suffix + '.log')))
-    print('This may take a few minutes...')
-    ## Create a time var to log run time
-    begin_time = dt.datetime.now()
-
-    ## Check that the bankfull flow filepath exists and read to dataframe
-    if not isfile(mann_n_table):
-        print('!!! Can not find the input roughness/feature_id file: ' + str(mann_n_table))
-    else:
-        ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
-        print('Importing the Manning roughness data file: ' + mann_n_table)
-        df_mann = pd.read_csv(mann_n_table,dtype={'feature_id': 'int64'})
-        if 'channel_n' not in df_mann.columns or 'overbank_n' not in df_mann.columns or 'feature_id' not in df_mann.columns:
-            print('Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann)
-        else:
-            print('Running the variable_mannings_calc function...')
-
-            ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
-            huc_list  = os.listdir(fim_dir)
-            skip_hucs_log = ""
-            for huc in huc_list:
-                if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
-                    in_src_bankfull_filename = join(fim_dir,huc,'src_full_crosswalked_bankfull.csv')
-                    out_src_vmann_filename = join(fim_dir,huc,'src_full_crosswalked_vmann.csv')
-                    htable_filename = join(fim_dir,huc,'hydroTable.csv')
-                    huc_plot_output_dir = join(fim_dir,huc,'src_plots')
-
-                    if isfile(in_src_bankfull_filename):
-                        print(str(huc))
-                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, out_src_vmann_filename, htable_filename, src_plot_option, huc_plot_output_dir,viz_clean_flag])
-                    else:
-                        print(str(huc) + '\nWARNING --> can not find the src_full_crosswalked_bankfull.csv in the fim output dir: ' + str(join(fim_dir,huc)) + ' - skipping this HUC!!!\n')
-
-            ## initiate log file
-            print(f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
-            sys.__stdout__ = sys.stdout
-            log_file = open(join(fim_dir,'logs','log_composite_n' + output_suffix + '.log'),"w")
-            sys.stdout = log_file
-            log_file.write('START TIME: ' + str(begin_time) + '\n')
-            log_file.write('#########################################################\n\n')
-
-            ## Pass huc procs_list to multiprocessing function
-            multi_process(variable_mannings_calc, procs_list)
-
-            ## Record run time and close log file
-            end_time = dt.datetime.now()
-            log_file.write('END TIME: ' + str(end_time) + '\n')
-            tot_run_time = end_time - begin_time
-            log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
-            sys.stdout = sys.__stdout__
-            log_file.close()
diff --git a/tools/aggregate_csv_files.py b/tools/aggregate_csv_files.py
new file mode 100644
index 000000000..cfd531eba
--- /dev/null
+++ b/tools/aggregate_csv_files.py
@@ -0,0 +1,63 @@
+import os, fnmatch
+import pandas as pd
+import argparse
+
+def locate(pattern, root_path):
+    for path, dirs, files in os.walk(os.path.abspath(root_path)):
+        for filename in fnmatch.filter(files, pattern):
+            yield os.path.join(path, filename)
+
+def read_csvs_to_df(files_to_merge,head_row):
+    #df = pd.concat((pd.read_csv(f,usecols=["HUC", "feature_id", "HydroID", "last_updated", "submitter", "adjust_ManningN"],dtype={'feature_id': 'int64','HUC': str}) for f in files_to_merge), ignore_index=True)
+    li = []
+    for file_in in files_to_merge:
+        print(file_in)
+        ## Use below for merging hydroTables for calib n value data
+        #df = pd.read_csv(file_in,usecols=["HUC", "feature_id", "HydroID", "last_updated", "submitter", "adjust_ManningN"],dtype={'feature_id': 'int64','HUC': str}, index_col=None, header=0)
+        #df = df[df['adjust_ManningN'].notna()]
+        #df.drop_duplicates(subset=['HydroID'],inplace=True)
+        df = pd.read_csv(file_in, index_col=None, header=head_row) #dtype={'feature_id': 'int64'}
+        df = df[df['Unnamed: 0'] != 'HydroID']
+        li.append(df)
+
+    frame = pd.concat(li, axis=0, ignore_index=True)
+    return(frame)
+
+def write_aggregate(frame,output_file):
+    print('Writing new csv file: ' + output_file)
+    frame.to_csv(output_file,index=False)
+
+def concat_files(files_to_merge):
+    # joining files with concat and read_csv
+    print('Concatenating all matching csv files...')
+    df_concat = pd.concat(map(pd.read_csv, files_to_merge), ignore_index=True)
+    
+def run_prep(fim_dir,file_search_str,head_row,output_file):
+    assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
+
+    files_to_merge = [js for js in locate('*' + file_search_str + '*.csv', fim_dir)]
+    if len(files_to_merge) > 0:
+        print('Found files: ' + str(len(files_to_merge)))
+
+        aggreg_df = read_csvs_to_df(files_to_merge,head_row)
+        write_aggregate(aggreg_df,output_file)
+
+    else:
+        print('Did not find any files using tag: ' + '*' + file_search_str + '*.csv')
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description="Simple tool to search for csv files (using wildcard text) within a fim output directory and then aggregate all files into a single csv")
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir (e.g. data/outputs/xxxx/', required=True,type=str)
+    parser.add_argument('-search_str','--file-search-str', help='File search string', required=True,type=str)
+    parser.add_argument('-head_row','--header-row', help='Optional: header row to parse (default=0)',default=0, required=False,type=int)
+    parser.add_argument('-out_csv','--output-csv', help='full filepath to write new csv', required=True,type=str)
+
+    args = vars(parser.parse_args())
+    fim_dir = args['fim_dir']
+    file_search_str = args['file_search_str']
+    head_row = args['header_row']
+    output_file = args['output_csv']
+
+    run_prep(fim_dir,file_search_str,head_row,output_file)
+
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index eb1bfc63f..560c6c05d 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -96,7 +96,7 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_
         #Define simplified labels as a list.
         new_labels = [label_dict[label] for label in org_labels]
         #Define legend location. FAR needs to be in different location than CSI/POD.
-        if y_field in ['FAR', 'PND']:
+        if y_field in ['FAR', 'PND','EQUITABLE_THREAT_SCORE']:
             legend_location = 'upper right'
         else:
             legend_location = 'lower left'
@@ -343,7 +343,7 @@ def filter_dataframe(dataframe, unique_field):
 ##############################################################################
 #Main function to analyze metric csv.
 ##############################################################################
-def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR','PND'] , spatial = False, fim_1_ms = False, site_barplots = False):
+def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE'] , spatial = False, fim_1_ms = False, site_barplots = False):
 
     '''
     Creates plots and summary statistics using metrics compiled from
@@ -616,7 +616,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
                 nws_dataset, sites = all_datasets.get(('nws','COMP'))
             #Append usgs/nws dataframes and filter unnecessary columns and rename remaining.
             all_ahps_datasets = pd.concat([usgs_dataset, nws_dataset])
-            all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','benchmark_source'])
+            all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE','benchmark_source'])
             all_ahps_datasets.rename(columns = {'benchmark_source':'source'}, inplace = True)
 
             #Get spatial data from WRDS
@@ -661,7 +661,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             #Join metrics to HUC spatial layer
             wbd_with_metrics = wbd_gdf.merge(huc_datasets, how = 'inner', left_on = 'HUC8', right_on = 'huc')
             #Filter out unnecessary columns
-            wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','benchmark_source','geometry'])
+            wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE','benchmark_source','geometry'])
             wbd_with_metrics.rename(columns = {'benchmark_source':'source'}, inplace = True )
             #Project to VIZ projection
             wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION)
@@ -676,7 +676,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
     parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
     parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
-    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR','PND'], required = False)
+    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR','PND','MCC','EQUITABLE_THREAT_SCORE'], required = False)
     parser.add_argument('-sp', '--spatial', help = 'If enabled, creates spatial layers with metrics populated in attribute table.', action = 'store_true', required = False)
     parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
     parser.add_argument('-i', '--site_plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index a117464a3..6182cc105 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -427,13 +427,10 @@ def progress_bar_handler(executor_dict, verbose, desc):
             # Send the executor to the progress bar
             progress_bar_handler(executor_dict, verbose, f"Compositing test cases with {job_number_huc} workers")
 
-    if config == 'DEV':
-        if dev_versions_to_compare != None:
-            dev_versions_to_include_list = dev_versions_to_compare + previous_fim_list
-        else:
-            dev_versions_to_include_list = previous_fim_list
-    if config == 'PREV':
-        dev_versions_to_include_list = dev_versions_to_compare
+    if dev_versions_to_compare != None:
+        dev_versions_to_include_list = dev_versions_to_compare + previous_fim_list
+    else:
+        dev_versions_to_include_list = previous_fim_list
 
     if master_metrics_csv is not None:
         # Do aggregate_metrics.
diff --git a/tools/vary_mannings_n_composite.py b/tools/vary_mannings_n_composite.py
new file mode 100755
index 000000000..450734d18
--- /dev/null
+++ b/tools/vary_mannings_n_composite.py
@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+import datetime as dt
+import re
+import os
+import sys
+import pandas as pd
+import numpy as np
+import argparse
+import matplotlib.pyplot as plt
+import seaborn as sns
+import shutil
+import traceback
+import warnings
+from functools import reduce
+from multiprocessing import Pool
+from os.path import isfile, join, dirname, isdir
+from pathlib import Path
+from tqdm import tqdm
+
+sns.set_theme(style="whitegrid")
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+"""
+    Vary the Manning's n values for in-channel vs. floodplain
+
+    Parameters
+    ----------
+    fim_dir : str
+        Directory containing FIM output folders.
+    channel_ratio_src_column : str
+        SRC attribute containing the channel vs. floodplain attribute
+    mann_n_table : str
+        Path to a csv file containing Manning's n values by feature_id
+    file_suffix : str
+        Suffix to append to the output log file
+    number_of_jobs : str
+        Number of jobs.
+    src_plot_option : str
+        Optional (True or False): use this flag to crate src plots for all hydroids
+"""
+
+def variable_mannings_calc(args):
+
+    in_src_bankfull_filename    = args[0]
+    channel_ratio_src_column    = args[1]
+    df_mann                     = args[2]
+    huc                         = args[3]
+    branch_id                   = args[4]
+    htable_filename             = args[5]
+    output_suffix               = args[6]
+    src_plot_option             = args[7]
+    huc_output_dir              = args[8]
+
+    ## Read the src_full_crosswalked.csv
+    print('Calculating variable roughness: ' + str(huc) + '  branch id: ' + str(branch_id))
+    log_text = 'Calculating variable roughness: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+    df_src = pd.read_csv(in_src_bankfull_filename,dtype={'feature_id': 'int64'})
+
+    ## Check that the channel ratio column the user specified exists in the def
+    if channel_ratio_src_column not in df_src.columns:
+        print('WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column)
+        print('Skipping --> ' + str(huc) + '  branch id: ' + str(branch_id))
+        log_text += 'WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
+    else:
+        try:
+            if 'comp_ManningN' in df_src.columns:
+                df_src.drop(['channel_n','overbank_n','comp_ManningN','vmann_on','Discharge (m3s-1)_varMann'], axis=1, inplace=True) # drop these cols (in case vmann was previously performed)
+            
+            ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the channel_ratio_src_column)
+            df_src = df_src.merge(df_mann,  how='left', on='feature_id')
+            check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
+            if check_null > 0:
+                log_text += str(huc) + '  branch id: ' + str(branch_id) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
+
+            ## Calculate composite Manning's n using the channel geometry ratio attribute given by user (e.g. chann_hradius_ratio or chann_vol_ratio)
+            df_src['comp_ManningN'] = (df_src[channel_ratio_src_column]*df_src['channel_n']) + ((1.0 - df_src[channel_ratio_src_column])*df_src['overbank_n'])
+            #print('Done calculating composite Manning n (' + channel_ratio_src_column + '): ' + str(huc))
+
+            ## Check if there are any missing data in the composite ManningN column
+            check_null_comp = df_src['comp_ManningN'].isnull().sum()
+            if check_null_comp > 0:
+                log_text += str(huc) + '  branch id: ' + str(branch_id) + ' --> ' + 'Missing values in the comp_ManningN calculation' + ' --> missing entries= ' + str(check_null_comp/84)  + '\n'
+            df_src['vmann_on'] = np.where(df_src['comp_ManningN'].isnull(), False, True) # create field to identify where vmann is applied (True=yes; False=no)
+
+            ## Define the channel geometry variable names to use from the src
+            hydr_radius = 'HydraulicRadius (m)'
+            wet_area = 'WetArea (m2)'
+
+            ## Calculate Q using Manning's equation
+            #df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True) # rename the previous Discharge column
+            df_src['Discharge (m3s-1)_varMann'] = df_src[wet_area]* \
+            pow(df_src[hydr_radius],2.0/3)* \
+            pow(df_src['SLOPE'],0.5)/df_src['comp_ManningN']
+
+            ## Set Q values to 0 and -999 for specified criteria (thalweg notch check happens in BARC)
+            # df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == 0,0,inplace=True)
+            # if 'Thalweg_burn_elev' in df_src:
+            #     df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == df_src['Thalweg_burn_elev'],0,inplace=True)
+            #     df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] < df_src['Thalweg_burn_elev'],-999,inplace=True)
+
+            ## Use the default discharge column when vmann is not being applied
+            df_src['Discharge (m3s-1)_varMann'] = np.where(df_src['vmann_on']==False, df_src['Discharge (m3s-1)'], df_src['Discharge (m3s-1)_varMann']) # reset the discharge value back to the original if vmann=false
+            df_src['comp_ManningN'] = np.where(df_src['vmann_on']==False, df_src['ManningN'], df_src['comp_ManningN']) # reset the ManningN value back to the original if vmann=false
+
+            ## Output new SRC with bankfull column
+            df_src.to_csv(in_src_bankfull_filename,index=False)
+
+            ## Output new hydroTable with updated discharge and ManningN column
+            df_src_trim = df_src[['HydroID','Stage','vmann_on','comp_ManningN','Discharge (m3s-1)_varMann']]
+            df_src_trim = df_src_trim.rename(columns={'Stage':'stage','comp_ManningN':'vmann_ManningN','Discharge (m3s-1)_varMann': 'vmann_discharge_cms'})
+            df_src_trim['ManningN'] = df_src_trim['vmann_ManningN'] # create a copy of vmann modified ManningN (used to track future changes)
+            df_src_trim['discharge_cms'] = df_src_trim['vmann_discharge_cms'] # create a copy of vmann modified discharge (used to track future changes)
+            df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
+
+            ## Check if BARC ran
+            # if not set(['orig_discharge_cms']).issubset(df_htable.columns):
+            #     df_htable.rename(columns={'discharge_cms':'orig_discharge_cms'},inplace=True)
+            #     df_htable.rename(columns={'ManningN':'orig_ManningN'},inplace=True)
+            # else:
+
+            ## drop the previously modified discharge column to be replaced with updated version
+            df_htable.drop(['vmann_on','discharge_cms','ManningN','vmann_discharge_cms','vmann_ManningN'], axis=1, inplace=True) 
+            df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
+
+            df_htable['vmann_on'] = np.where(df_htable['LakeID']>0, False, df_htable['vmann_on']) # reset the ManningN value back to the original if vmann=false
+
+            ## Output new hydroTable csv
+            if output_suffix != "":
+                htable_filename = os.path.splitext(htable_filename)[0] + output_suffix + '.csv' 
+            df_htable.to_csv(htable_filename,index=False)
+
+            log_text += 'Completed: ' + str(huc)
+
+            ## plot rating curves
+            if src_plot_option:
+                if isdir(huc_output_dir) == False:
+                    os.mkdir(huc_output_dir)
+                generate_src_plot(df_src, huc_output_dir)
+        except Exception as ex:
+            summary = traceback.StackSummary.extract(
+                    traceback.walk_stack(None))
+            print(str(huc) + '  branch id: ' + str(branch_id) + " failed for some reason")
+            print(f"*** {ex}")                
+            print(''.join(summary.format()))    
+            log_text += 'ERROR --> ' + str(huc) + '  branch id: ' + str(branch_id) + " failed (details: " + (f"*** {ex}") + (''.join(summary.format())) + '\n'
+
+    return(log_text)
+
+def generate_src_plot(df_src, plt_out_dir):
+
+    ## create list of unique hydroids
+    hydroids = df_src.HydroID.unique().tolist()
+
+    ## plot each hydroid SRC in the huc
+    for hydroid in hydroids:
+        print("Creating SRC plot: " + str(hydroid))
+        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+
+        f, ax = plt.subplots(figsize=(6.5, 6.5))
+        ax.set_title(str(hydroid))
+        sns.despine(f, left=True, bottom=True)
+        sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
+        sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
+        sns.lineplot(x='Discharge (m3s-1)', y='Stage_bankfull', data=plot_df, color='green', ax=ax)
+        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'],alpha=0.5)
+        plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_bankfull'].median(), "NWM Bankfull Approx: " + str(plot_df['Stage_bankfull'].median()))
+        ax.legend()
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png',dpi=175, bbox_inches='tight')
+        plt.close()
+
+#    for hydroid in hydroids:
+#        print("Creating SRC plot: " + str(hydroid))
+#        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+#
+#        f, ax = plt.subplots(figsize=(6.5, 6.5))
+#        ax.set_title(str(hydroid))
+#        sns.despine(f, left=True, bottom=True)
+#        sns.scatterplot(x='comp_ManningN', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
+#        #sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
+#        sns.lineplot(x='comp_ManningN', y='Stage_1_5', data=plot_df, color='green', ax=ax)
+#        plt.fill_between(plot_df['comp_ManningN'], plot_df['Stage_1_5'],alpha=0.5)
+#        plt.text(plot_df['comp_ManningN'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+#        ax.legend()
+#        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
+#        plt.close()
+
+def multi_process(variable_mannings_calc, procs_list, log_file, verbose):
+    ## Initiate multiprocessing
+    print(f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    with Pool(processes=number_of_jobs) as pool:
+        if verbose:
+            map_output = tqdm(pool.imap(variable_mannings_calc, procs_list), total=len(procs_list))
+            tuple(map_output)  # fetch the lazy results
+        else:
+            map_output = pool.map(variable_mannings_calc, procs_list)
+    log_file.writelines(["%s\n" % item  for item in map_output])
+
+def run_prep(fim_dir,channel_ratio_src_column,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option):
+    procs_list = []
+
+    print('Writing progress to log file here: ' + str(join(fim_dir,'log_composite_n' + output_suffix + '.log')))
+    print('This may take a few minutes...')
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+
+    ## initiate log file
+    log_file = open(join(fim_dir,'logs','log_composite_n' + output_suffix + '.log'),"w")
+    log_file.write('START TIME: ' + str(begin_time) + '\n')
+    log_file.write('#########################################################\n\n')
+
+    ## Check that the input fim_dir exists
+    assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
+    ## Check that the manning's roughness input filepath exists and then read to dataframe
+    assert os.path.isfile(mann_n_table), 'Can not find the input roughness/feature_id file: ' + str(mann_n_table)
+    
+    ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
+    print('Importing the Manning roughness data file: ' + mann_n_table)
+    df_mann = pd.read_csv(mann_n_table,dtype={'feature_id': 'int64'})
+    if 'channel_n' not in df_mann.columns or 'overbank_n' not in df_mann.columns or 'feature_id' not in df_mann.columns:
+        print('Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann)
+    else:
+        print('Running the variable_mannings_calc function...')
+
+        ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
+        huc_list  = os.listdir(fim_dir)
+        for huc in huc_list:
+            #if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+            if re.match('\d{8}', huc):    
+                huc_branches_dir = os.path.join(fim_dir, huc,'branches')
+                for branch_id in os.listdir(huc_branches_dir):
+                    branch_dir = os.path.join(huc_branches_dir,branch_id)
+                    in_src_bankfull_filename = join(branch_dir,'src_full_crosswalked_' + branch_id + '.csv')
+                    htable_filename = join(branch_dir,'hydroTable_' + branch_id + '.csv')
+                    huc_plot_output_dir = join(branch_dir,'src_plots')
+
+                    if isfile(in_src_bankfull_filename) and isfile(htable_filename):
+                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, branch_id, htable_filename, output_suffix, src_plot_option, huc_plot_output_dir])
+                    else:
+                        print('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+                        log_file.write('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+
+        ## Pass huc procs_list to multiprocessing function
+        multi_process(variable_mannings_calc, procs_list, log_file, verbose)
+
+        ## Record run time and close log file
+        end_time = dt.datetime.now()
+        log_file.write('END TIME: ' + str(end_time) + '\n')
+        tot_run_time = end_time - begin_time
+        log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+        log_file.close()
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description="Vary the Manning's n values for in-channel vs. floodplain (recalculate Manning's eq for Discharge)")
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-bc','--channel-ratio-src-column',help='SRC attribute containing the channel vs. overbank geometry ratio (for composite calc)',required=False,type=str,default='chann_hradius_ratio')
+    parser.add_argument('-mann','--mann-n-table',help="Path to a csv file containing Manning's n values by featureid",required=True,type=str)
+    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",default="",required=False,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
+    parser.add_argument('-vb','--verbose',help='Optional verbose progress bar',required=False,default=None,action='store_true')
+    parser.add_argument('-plots','--src-plot-option',help='OPTIONAL flag: use this flag to create src plots for all hydroids. WARNING - long runtime',default=False,required=False, action='store_true')
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    channel_ratio_src_column = args['channel_ratio_src_column']
+    mann_n_table = args['mann_n_table']
+    output_suffix = args['output_suffix']
+    number_of_jobs = args['number_of_jobs']
+    verbose = bool(args['verbose'])
+    src_plot_option = args['src_plot_option']
+
+    run_prep(fim_dir,channel_ratio_src_column,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option)
\ No newline at end of file

From 300db015e0fa258b054b832bfc5218addda905a7 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 2 Dec 2022 11:18:25 -0600
Subject: [PATCH 181/359] v4.0.12.1 Cleanup a few deny list files (#751)

---
 config/deny_gms_branches_dev.lst  |  2 +-
 config/deny_gms_branches_prod.lst |  2 +-
 config/deny_gms_unit_prod.lst     |  3 ++-
 docs/CHANGELOG.md                 | 11 +++++++++++
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/config/deny_gms_branches_dev.lst b/config/deny_gms_branches_dev.lst
index 01f9084a4..6fea451b0 100644
--- a/config/deny_gms_branches_dev.lst
+++ b/config/deny_gms_branches_dev.lst
@@ -35,7 +35,7 @@ flows_points_pixels_{}.gpkg
 gw_catchments_pixels_{}.tif
 gw_catchments_reaches_{}.gpkg
 gw_catchments_reaches_{}.tif
-#gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
+gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 #gw_catchments_reaches_filtered_addedAttributes_{}.tif
 #gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
diff --git a/config/deny_gms_branches_prod.lst b/config/deny_gms_branches_prod.lst
index ca8b460e5..4b9e17d61 100644
--- a/config/deny_gms_branches_prod.lst
+++ b/config/deny_gms_branches_prod.lst
@@ -35,7 +35,7 @@ flows_points_pixels_{}.gpkg
 gw_catchments_pixels_{}.tif
 gw_catchments_reaches_{}.gpkg
 gw_catchments_reaches_{}.tif
-#gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
+gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 #gw_catchments_reaches_filtered_addedAttributes_{}.tif
 #gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
diff --git a/config/deny_gms_unit_prod.lst b/config/deny_gms_unit_prod.lst
index faeb3b092..5fc4b75b3 100644
--- a/config/deny_gms_unit_prod.lst
+++ b/config/deny_gms_unit_prod.lst
@@ -1,6 +1,6 @@
 # List of files for gms branches to delete
 # Use comment to allow list the file
-NHDPlusBurnLineEvent_subset.gpkg
+NHPlusBurnLineEvent_subset.gpkg
 #branch_id.lst
 #branch_polygons.gpkg
 #LandSea_subset.gpkg
@@ -20,3 +20,4 @@ nwm_headwaters.gpkg
 #wbd.gpkg
 #wbd8_clp.gpkg
 wbd_buffered.gpkg
+#wbd_buffered_streams.gpkg
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ef39833de..30b8c8b2e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.12.1 - 2022-11-30 - [PR #751](https://github.com/NOAA-OWP/inundation-mapping/pull/751)
+
+Updating a few deny list files.
+
+### Changes
+
+- `config`:
+    - `deny_gms_branches_dev.lst`, `deny_gms_branches_prod.lst`, and `deny_gms_unit_prod.lst`
+
+<br/><br/>
+
 
 ## v4.0.12.0 - 2022-11-28 - [PR #736](https://github.com/NOAA-OWP/inundation-mapping/pull/736)
 

From c227b7f10374944b4c03fc530994f55f1b08b02b Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 9 Dec 2022 11:46:04 -0500
Subject: [PATCH 182/359] v4.0.12.2 Hotfix: Stop writing
 `gms_inputs_removed.csv` if empty (#754)

---
 docs/CHANGELOG.md                | 10 ++++++++++
 src/gms/remove_error_branches.py | 12 ++++++++----
 2 files changed, 18 insertions(+), 4 deletions(-)
 mode change 100755 => 100644 src/gms/remove_error_branches.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 30b8c8b2e..1156e7631 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## V4.0.12.2 - 2022-12-04 - [PR #754](https://github.com/NOAA-OWP/inundation-mapping/pull/754)
+
+Stop writing `gms_inputs_removed.csv` if no branches are removed with Error status 61.
+
+### Changes
+
+- `src/gms/remove_error_branches.py`: Checks if error branches is not empty before saving gms_inputs_removed.csv
+
+<br/><br/>
+
 ## v4.0.12.1 - 2022-11-30 - [PR #751](https://github.com/NOAA-OWP/inundation-mapping/pull/751)
 
 Updating a few deny list files.
diff --git a/src/gms/remove_error_branches.py b/src/gms/remove_error_branches.py
old mode 100755
new mode 100644
index 00a9c8ee4..7dfaf26f8
--- a/src/gms/remove_error_branches.py
+++ b/src/gms/remove_error_branches.py
@@ -68,12 +68,16 @@ def remove_error_branches(logfile, gms_inputs):
                     error_branches = pd.concat([error_branches, tmp_df])
 
         # Save list of removed branches
-        pd.DataFrame(error_branches).to_csv(gms_inputs_removed, header=False, index=False)
+        if len(error_branches) > 0:
+            pd.DataFrame(error_branches).to_csv(gms_inputs_removed, header=False, index=False)
 
-        # Overwrite gms_inputs.csv with error branches removed
-        gms_inputs_df.to_csv(gms_inputs, header=False, index=False)
+            # Overwrite gms_inputs.csv with error branches removed
+            gms_inputs_df.to_csv(gms_inputs, header=False, index=False)
 
-        print('\nDone removing error branches\n')
+            print('\nDone removing error branches\n')
+
+        else:
+            print('\nDone -- no branches to remove')
 
     else:
         print('\nNo log file found\n')

From b8a209ff2e5e802ff9fe115e6f8730b8de22ea82 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 9 Dec 2022 11:51:39 -0500
Subject: [PATCH 183/359] v4.0.13.0 Replace NHD headwaters with NWM-derived
 headwaters and remove NHD references (#744)

---
 config/deny_gms_unit_prod.lst               |  2 -
 docs/CHANGELOG.md                           | 59 +++++----------------
 gms_run_unit.sh                             |  3 +-
 src/clip_vectors_to_wbd.py                  | 54 +++++--------------
 src/gms/crosswalk_nwm_demDerived.py         | 12 +----
 src/gms/derive_level_paths.py               | 53 +++++++++---------
 src/gms/run_by_unit.sh                      | 13 +++--
 src/gms/stream_branches.py                  | 25 +++++----
 unit_tests/__template_unittests.py          |  2 -
 unit_tests/clip_vectors_to_wbd_params.json  | 19 +++----
 unit_tests/clip_vectors_to_wbd_unittests.py |  5 +-
 11 files changed, 81 insertions(+), 166 deletions(-)

diff --git a/config/deny_gms_unit_prod.lst b/config/deny_gms_unit_prod.lst
index 5fc4b75b3..0ad10cc82 100644
--- a/config/deny_gms_unit_prod.lst
+++ b/config/deny_gms_unit_prod.lst
@@ -1,11 +1,9 @@
 # List of files for gms branches to delete
 # Use comment to allow list the file
-NHPlusBurnLineEvent_subset.gpkg
 #branch_id.lst
 #branch_polygons.gpkg
 #LandSea_subset.gpkg
 #LeveeProtectedAreas_subset.gpkg
-nhd_headwater_points_subset.gpkg
 #nld_subset_levees.gpkg
 #nwm_catchments_proj_subset.gpkg
 #nwm_catchments_proj_subset_levelPaths.gpkg
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1156e7631..d2d654681 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.13.0 - 2022-11-16 - [PR #744](https://github.com/NOAA-OWP/inundation-mapping/pull/744)
+
+Changes branch 0 headwaters data source from NHD to NWS to be consistent with branches. Removes references to NHD flowlines and headwater data.
+
+### Changes
+
+- `src/gms/derive_level_paths.py`: Generates headwaters before stream branch filtering
+
+### Removals
+
+- Removes NHD flowlines and headwater references from `gms_run_unit.sh`, `config/deny_gms_unit_prod.lst`, `src/clip_vectors_to_wbd.py`, `src/gms/run_by_unit.sh`, `unit_tests/__template_unittests.py`, `unit_tests/clip_vectors_to_wbd_params.json`, and `unit_tests/clip_vectors_to_wbd_unittests.py`
+
+<br/><br/>
+
 ## V4.0.12.2 - 2022-12-04 - [PR #754](https://github.com/NOAA-OWP/inundation-mapping/pull/754)
 
 Stop writing `gms_inputs_removed.csv` if no branches are removed with Error status 61.
@@ -410,51 +424,6 @@ Other changes:
 
 <br/><br/>
 
-## v4.0.(pending) - 2022-09-13 - [PR #681](https://github.com/NOAA-OWP/inundation-mapping/pull/681)
-
-Created a new tool to downloaded USGS 3Dep DEM's via their S3 bucket.
-
-Other changes:
- - Some code file re-organization in favour of the new `data` folder which is designed for getting, setting, and processing data from external sources such as AWS, WBD, NHD, NWM, etc.
- - Added tmux as a new tool embedded inside the docker images.
-
-## Additions
-
-- `data`
-   - `usgs`
-      - `acquire_and_preprocess_3dep_dems.py`:  The new tool as described above. For now it is hardcoded to a set path for USGS AWS S3 vrt file but may change later for it to become parameter driven.
- - `create_vrt_file.py`: This is also a new tool that can take a directory of geotiff files and create a gdal virtual file, .vrt extention, also called a `virtual raster`. Instead of clipping against HUC4, 6, 8's raster files, and run risks of boundary issues, vrt's actual like all of the tif's are one giant mosaiced raster and can be clipped as one.
-
-## Removals
-
-- 'Dockerfile.prod`:  No longer being used (never was used)
-
-## Changes
-
-- `Dockerfile`:  Added apt install for tmux. This tool will now be available in docker images and assists developers.
-
-- `data`
-   - `acquire_and_preprocess_inputs.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
-   - `nws`
-      - `preprocess_ahps_nws.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
-      - `preprocess_rasters.py`: moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
-    - `usgs`
-         - `preprocess_ahps_usgs.py`:  moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
-         - `preprocess_download_usgs_grids.py`: moved from the `tools` directory but not other changes made. Note: will required review/adjustments before being used again.
-
- - `src`
-     - `utils`
-         - `shared_functions.py`:  changes made were
-              - Cleanup the "imports" section of the file (including a change to how the utils.shared_variables file is loaded.
-              - Added `progress_bar_handler` function which can be re-used by other code files.
-              - Added `get_file_names` which can create a list of files from a given directory matching a given extension. 
-              - Modified `print_current_date_time` and `print_date_time_duration` and  methods to return the date time strings. These helper methods exist to help with standardization of logging and output console messages.
-              - Added `print_start_header` and `print_end_header` to help with standardization of console and logging output messages.
-          - `shared_variables.py`: Additions in support of near future functionality of having fim load DEM's from USGS 3DEP instead of NHD rasters.
-
-<br/><br/>
-
-
 ## v4.0.9.2 - 2022-09-12 - [PR #678](https://github.com/NOAA-OWP/inundation-mapping/pull/678)
 
 This fixes several bugs related to branch definition and trimming due to waterbodies.
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index f676f94cd..c64030e5c 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -174,8 +174,7 @@ export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
 export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
 export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
 export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
-export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
+export input_nwm_headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
 export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
 export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Levee_protected_areas.gpkg
 export deny_gms_unit_list=$deny_gms_unit_list
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 189136781..5c07cf957 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 
-import os
 import sys
-import pandas as pd
 import geopandas as gpd
 import argparse
 import rasterio as rio
@@ -13,9 +11,8 @@
 @mem_profile
 def subset_vector_layers(subset_nwm_lakes, 
                          subset_nwm_streams,
-                         subset_nhd_streams,
                          hucCode,
-                         subset_nhd_headwaters,
+                         subset_nwm_headwaters,
                          wbd_buffer_filename,
                          wbd_filename,
                          dem_filename,
@@ -23,11 +20,10 @@ def subset_vector_layers(subset_nwm_lakes,
                          nwm_catchments,
                          subset_nwm_catchments,
                          nld_lines,
-                         nhd_streams,
                          landsea,
                          nwm_streams,
                          subset_landsea,
-                         nhd_headwaters,
+                         nwm_headwaters,
                          subset_nld_lines,
                          great_lakes,
                          lake_buffer_distance,
@@ -46,13 +42,9 @@ def subset_vector_layers(subset_nwm_lakes,
     wbd_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance, resolution=32)
 
     # Make the streams buffer smaller than the wbd_buffer so streams don't reach the edge of the DEM
-    wbd_buffer_filename_split = os.path.splitext(wbd_buffer_filename)
-    wbd_streams_buffer_filename = wbd_buffer_filename_split[0] + '_streams' + wbd_buffer_filename_split[1]
     wbd_streams_buffer = wbd.copy()
     wbd_streams_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance-3*dem_cellsize, resolution=32)
 
-    # projection = wbd_buffer.crs
-
     great_lakes = gpd.read_file(great_lakes, mask=wbd_buffer).reset_index(drop=True)
 
     if not great_lakes.empty:
@@ -73,7 +65,6 @@ def subset_vector_layers(subset_nwm_lakes,
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
     wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False)
-    wbd_streams_buffer.to_file(wbd_streams_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False)
 
     del great_lakes
 
@@ -114,30 +105,16 @@ def subset_vector_layers(subset_nwm_lakes,
         nld_lines.to_file(subset_nld_lines, driver = getDriver(subset_nld_lines), index=False)
     del nld_lines
 
-    # Subset nhd headwaters
-    print("Subsetting NHD Headwater Points for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
-    nhd_headwaters = gpd.read_file(nhd_headwaters, mask=wbd_streams_buffer)
+    # Subset NWM headwaters
+    print("Subsetting NWM Headwater Points for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer)
 
-    if len(nhd_headwaters) > 0:
-        nhd_headwaters.to_file(subset_nhd_headwaters, driver=getDriver(subset_nhd_headwaters), index=False)
+    if len(nwm_headwaters) > 0:
+        nwm_headwaters.to_file(subset_nwm_headwaters, driver=getDriver(subset_nwm_headwaters), index=False)
     else:
         print ("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
-    del nhd_headwaters
-
-    # Subset nhd streams
-    print("Querying NHD Streams for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
-    nhd_streams = gpd.read_file(nhd_streams, mask=wbd_streams_buffer)
-
-    if len(nhd_streams) > 0:
-        nhd_streams = gpd.clip(nhd_streams, wbd_streams_buffer)
-
-        nhd_streams.to_file(subset_nhd_streams, 
-                            driver = getDriver(subset_nhd_streams), index=False)
-    else:
-        print ("No NHD streams within HUC " + str(hucCode) + " boundaries.")
-        sys.exit(0)
-    del nhd_streams
+    del nwm_headwaters
 
     # Find intersecting nwm_catchments
     print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
@@ -146,21 +123,18 @@ def subset_vector_layers(subset_nwm_lakes,
     if len(nwm_catchments) > 0:
         nwm_catchments.to_file(subset_nwm_catchments, driver=getDriver(subset_nwm_catchments), index=False)
     else:
-        print ("No NHD catchments within HUC " + str(hucCode) + " boundaries.")
+        print ("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_catchments
 
     # Subset nwm streams
-
-    print("Subsetting NWM Streams and deriving headwaters for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    print("Subsetting NWM Streams for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
 
     nwm_streams = gpd.read_file(nwm_streams, mask = wbd)
-    nwm_streams = gpd.clip(nwm_streams, wbd)
 
      # NWM can have duplicate records, but appear to always be identical duplicates
     nwm_streams.drop_duplicates(subset="ID", keep="first", inplace=True)
 
-
     if len(nwm_streams) > 0:
         nwm_streams = gpd.clip(nwm_streams, wbd_streams_buffer)
 
@@ -180,11 +154,9 @@ def subset_vector_layers(subset_nwm_lakes,
                         required=True)
     parser.add_argument('-b','--subset-nwm-streams', help='NWM streams subset', 
                         required=True)
-    parser.add_argument('-c','--subset-nhd-streams', help='NHD streams subset', 
-                        required=True)
     parser.add_argument('-d','--hucCode', help='HUC boundary ID', required=True,
                         type=str)
-    parser.add_argument('-e','--subset-nhd-headwaters', help='NHD headwaters subset', 
+    parser.add_argument('-e','--subset-nwm-headwaters', help='NWM headwaters subset', 
                         required=True, default=None)
     parser.add_argument('-f','--wbd_buffer_filename', help='Buffered HUC boundary', 
                         required=True)
@@ -197,15 +169,13 @@ def subset_vector_layers(subset_nwm_lakes,
                         required=True)
     parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path',
                         required=True)
-    parser.add_argument('-s','--nhd-streams', help='NHDPlus HR burnline',
-                        required=True)
     parser.add_argument('-v','--landsea', help='LandSea - land boundary',
                         required=True)	 
     parser.add_argument('-w','--nwm-streams', help='NWM flowlines',
                         required=True)
     parser.add_argument('-x','--subset-landsea', help='LandSea subset', 
                         required=True)
-    parser.add_argument('-y','--nhd-headwaters', help='NHD headwaters',
+    parser.add_argument('-y','--nwm-headwaters', help='NWM headwaters',
                         required=True)	 
     parser.add_argument('-z','--subset-nld-lines', help='Subset of NLD levee vectors for HUC',
                         required=True)
diff --git a/src/gms/crosswalk_nwm_demDerived.py b/src/gms/crosswalk_nwm_demDerived.py
index 131b91aff..97f815ffd 100755
--- a/src/gms/crosswalk_nwm_demDerived.py
+++ b/src/gms/crosswalk_nwm_demDerived.py
@@ -40,9 +40,9 @@ def Crosswalk_nwm_demDerived(nwm_streams, demDerived, wbd=None, node_prefix=None
                                        verbose=verbose)
 
     # create points for nwm and demDerived networks
-    nwm_points = nwm_streams.explode_to_points(reach_id_attribute='ID', sampling_size=sampling_size, 
+    nwm_points = nwm_streams.explode_to_points(sampling_size=sampling_size, 
                                                verbose=verbose)
-    demDerived_points = demDerived.explode_to_points(reach_id_attribute='HydroID', sampling_size=sampling_size, 
+    demDerived_points = demDerived.explode_to_points(sampling_size=sampling_size, 
                                                      verbose=verbose)
     
     # conflate points
@@ -57,14 +57,6 @@ def Crosswalk_nwm_demDerived(nwm_streams, demDerived, wbd=None, node_prefix=None
     demDerived['HydroID'] = demDerived['HydroID'].astype(int)
     demDerived = demDerived.merge(crosswalk_table, how='left', left_on='HydroID', right_index=True)
 
-    """
-    demDerived = demDerived.conflate_branches(target_stream_network=nwm_streams,branch_id_attribute_left='levpa_id',
-                                                branch_id_attribute_right='levpa_id',
-                                                left_order_attribute='order_', right_order_attribute='order_',
-                                                crosswalk_attribute='nwm_levpa_id',
-                                                verbose=verbose)
-    """
-
     if demDerived_outfile is not None:
         demDerived.write(demDerived_outfile,index=False,verbose=verbose)
 
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index fec0a4472..90971d55c 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -36,8 +36,8 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file.")
         print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
         sys.exit(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
-                                                 
-    elif (drop_low_stream_orders):
+
+    if (drop_low_stream_orders):
         stream_network = stream_network.exclude_attribute_values(branch_id_attribute="order_",
                                                values_excluded=[1,2]
                                               )
@@ -56,11 +56,11 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
 
     # derive nodes
     stream_network = stream_network.derive_nodes(toNode_attribute=toNode_attribute,
-                                                  fromNode_attribute=fromNode_attribute,
-                                                  reach_id_attribute=reach_id_attribute,
-                                                  outlet_linestring_index=outlet_linestring_index,
-                                                  node_prefix=None,
-                                                  verbose=verbose)
+                                                 fromNode_attribute=fromNode_attribute,
+                                                 reach_id_attribute=reach_id_attribute,
+                                                 outlet_linestring_index=outlet_linestring_index,
+                                                 node_prefix=None,
+                                                 verbose=verbose)
     
     # derive outlets and inlets
     stream_network = stream_network.derive_outlets(toNode_attribute,
@@ -72,9 +72,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                   fromNode_attribute,
                                                   inlets_attribute=inlets_attribute,
                                                   verbose=verbose
-                                                 )
-
-    # derive up and downstream networks
+                                                 )    # derive up and downstream networks
     upstreams, downstreams = stream_network.make_up_and_downstream_dictionaries(
                                                                          reach_id_attribute=reach_id_attribute,
                                                                          toNode_attribute=toNode_attribute,
@@ -93,15 +91,15 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                     )
 
     # derive stream branches
-    stream_network = stream_network.derive_stream_branches( toNode_attribute=toNode_attribute,
-                                                            fromNode_attribute=fromNode_attribute,
-                                                            upstreams=upstreams,
-                                                            branch_id_attribute=branch_id_attribute,
-                                                            reach_id_attribute=reach_id_attribute,
-                                                            comparison_attributes=['arbolate_sum', 'order_'],
-                                                            comparison_function=max,
-                                                            verbose=verbose
-                                                           )
+    stream_network = stream_network.derive_stream_branches(toNode_attribute=toNode_attribute,
+                                                           fromNode_attribute=fromNode_attribute,
+                                                           upstreams=upstreams,
+                                                           branch_id_attribute=branch_id_attribute,
+                                                           reach_id_attribute=reach_id_attribute,
+                                                           comparison_attributes=['arbolate_sum', 'order_'],
+                                                           comparison_function=max,
+                                                           verbose=verbose
+                                                          )
     
     # filter out streams without catchments
     if (catchments is not None) & (catchments_outfile is not None):
@@ -116,8 +114,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                                               )
 
         # subset which columns to merge
-        stream_network_to_merge = stream_network.filter(
-                                                        items = [reach_id_attribute,inlets_attribute,
+        stream_network_to_merge = stream_network.filter(items = [reach_id_attribute,inlets_attribute,
                                                                  outlets_attribute,branch_id_attribute]
                                                        )
 
@@ -126,9 +123,9 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                       right_on=reach_id_attribute
                                      )
 
-        catchments.reset_index(drop=True,inplace=True)
+        catchments.reset_index(drop=True, inplace=True)
 
-        catchments.to_file(catchments_outfile,index=False,driver='GPKG')
+        catchments.to_file(catchments_outfile, index=False, driver='GPKG')
 
     # derive headwaters
     if (headwaters_outfile is not None):
@@ -162,12 +159,12 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                                        verbose=False
                                                                       )
                                        
+    if branch_inlets_outfile is not None:
         branch_inlets = stream_network.derive_inlet_points_by_feature(feature_attribute=branch_id_attribute,
-                                                                      outlet_linestring_index=outlet_linestring_index
-                                                                     )
-   
-        if branch_inlets_outfile is not None:
-            branch_inlets.to_file(branch_inlets_outfile,index=False,driver='GPKG')
+                                                                        outlet_linestring_index=outlet_linestring_index
+                                                                        )
+
+        branch_inlets.to_file(branch_inlets_outfile, index=False, driver='GPKG')
 
     return(stream_network)
     
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 618235f35..1fdc8be38 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -76,9 +76,8 @@ Tstart
 
 cmd_args=" -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg"
 cmd_args+=" -b $outputHucDataDir/nwm_subset_streams.gpkg"
-cmd_args+=" -c $outputHucDataDir/NHPlusBurnLineEvent_subset.gpkg"
 cmd_args+=" -d $hucNumber"
-cmd_args+=" -e $outputHucDataDir/nhd_headwater_points_subset.gpkg"
+cmd_args+=" -e $outputHucDataDir/nwm_headwater_points_subset.gpkg"
 cmd_args+=" -f $outputHucDataDir/wbd_buffered.gpkg"
 cmd_args+=" -g $outputHucDataDir/wbd.gpkg"
 cmd_args+=" -i $input_DEM"
@@ -86,11 +85,10 @@ cmd_args+=" -l $input_nwm_lakes"
 cmd_args+=" -m $input_nwm_catchments"
 cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"
 cmd_args+=" -r $input_NLD"
-cmd_args+=" -s $input_nhd_flowlines"
 cmd_args+=" -v $input_LANDSEA"
 cmd_args+=" -w $input_nwm_flows"
 cmd_args+=" -x $outputHucDataDir/LandSea_subset.gpkg"
-cmd_args+=" -y $input_nhd_headwaters"
+cmd_args+=" -y $input_nwm_headwaters"
 cmd_args+=" -z $outputHucDataDir/nld_subset_levees.gpkg"
 cmd_args+=" -gl $input_GL_boundaries"
 cmd_args+=" -lb $lakes_buffer_dist_meters"
@@ -103,7 +101,7 @@ Tcount
 python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
 
 : '
-python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM
+python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nwm_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nwm_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM
 '
 
 ## Clip WBD8 ##
@@ -119,6 +117,7 @@ date -u
 Tstart
 $srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -s $dropLowStreamOrders -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 
+
 # test if we received a non-zero code back from derive_level_paths.py
 subscript_exit_code=$?
 # we have to retrow it if it is not a zero (but it will stop further execution in this script)
@@ -202,10 +201,10 @@ gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -c
 Tcount
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
-echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $branch_zero_id"$stopDiv
+echo -e $startDiv"Rasterize NWM Headwaters $hucNumber $branch_zero_id"$stopDiv
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nhd_headwater_points_subset.gpkg $outputCurrentBranchDataDir/headwaters_$branch_zero_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_headwater_points_subset.gpkg $outputCurrentBranchDataDir/headwaters_$branch_zero_id.tif
 Tcount
 
 ## DEM Reconditioning ##
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index 9fae162a7..996231ed1 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -183,7 +183,7 @@ def merge(self,*args,**kwargs):
         return(self)
 
 
-    def merge_stream_branches(self,stream_branch_dataset,on='NHDPlusID',branch_id_attribute='LevelPathI',attributes='StreamOrde',stream_branch_layer_name=None):
+    def merge_stream_branches(self,stream_branch_dataset,on='ID',branch_id_attribute='LevelPathI',attributes='StreamOrde',stream_branch_layer_name=None):
 
         """ Merges stream branch id attribute from another vector file """
 
@@ -221,7 +221,7 @@ def flip_inlet_outlet_linestring_index(linestring_index):
             raise ValueError('Linestring index should be 0 or -1')
 
 
-    def derive_nodes(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',reach_id_attribute='NHDPlusID',
+    def derive_nodes(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',reach_id_attribute='ID',
                      outlet_linestring_index=0,node_prefix=None,max_node_digits=8,verbose=False):
         
         if verbose:
@@ -389,8 +389,8 @@ def exclude_attribute_values(self, branch_id_attribute=None, values_excluded=Non
 
     def remove_stream_segments_without_catchments( self,
                                                    catchments,
-                                                   reach_id_attribute='NHDPlusID',
-                                                   reach_id_attribute_in_catchments='NHDPlusID',
+                                                   reach_id_attribute='ID',
+                                                   reach_id_attribute_in_catchments='ID',
                                                    verbose=False
                                                  ):
 
@@ -415,9 +415,9 @@ def remove_stream_segments_without_catchments( self,
 
     def remove_branches_without_catchments(self,
                                            catchments,
-                                           reach_id_attribute='NHDPlusID',
+                                           reach_id_attribute='ID',
                                            branch_id_attribute='branchID',
-                                           reach_id_attribute_in_catchments='NHDPlusID',
+                                           reach_id_attribute_in_catchments='ID',
                                            verbose=False
                                            ):
 
@@ -562,7 +562,7 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
                                upstreams=None,
                                outlet_attribute='outlet_id',
                                branch_id_attribute='branchID',
-                               reach_id_attribute='NHDPlusID',
+                               reach_id_attribute='ID',
                                comparison_attributes='StreamOrde',
                                comparison_function=max,
                                max_branch_id_digits=6,
@@ -683,7 +683,7 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
         return(self)
 
 
-    def make_up_and_downstream_dictionaries(self,reach_id_attribute='NHDPlusID',
+    def make_up_and_downstream_dictionaries(self,reach_id_attribute='ID',
                                             toNode_attribute='ToNode',
                                             fromNode_attribute='FromNode',
                                             verbose=False):
@@ -706,7 +706,7 @@ def make_up_and_downstream_dictionaries(self,reach_id_attribute='NHDPlusID',
 
 
     def get_arbolate_sum(self,arbolate_sum_attribute='arbolate_sum',inlets_attribute='inlet_id',
-                         reach_id_attribute='NHDPlusID',length_conversion_factor_to_km = 0.001,
+                         reach_id_attribute='ID',length_conversion_factor_to_km = 0.001,
                          upstreams=None, downstreams=None,
                          toNode_attribute='ToNode',
                          fromNode_attribute='FromNode',
@@ -812,7 +812,7 @@ def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_exclude
                     self.loc[lpid,'geometry'] = merged_line
         
         #self[branch_id_attribute] = bids
-        self = StreamNetwork(self,branch_id_attribute=branch_id_attribute,
+        self = StreamNetwork(self, branch_id_attribute=branch_id_attribute,
                              attribute_excluded=attribute_excluded,
                              values_excluded=values_excluded)
 
@@ -835,7 +835,7 @@ def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_exclude
         return(self)
 
 
-    def derive_segments(self,inlets_attribute='inlet_id', reach_id_attribute='NHDPlusID'):
+    def derive_segments(self,inlets_attribute='inlet_id', reach_id_attribute='ID'):
         pass
 
 
@@ -879,7 +879,7 @@ def conflate_branches(self,target_stream_network,branch_id_attribute_left='branc
         return(self)
         
 
-    def explode_to_points(self,reach_id_attribute='NHDPlusID', sampling_size=None,
+    def explode_to_points(self,reach_id_attribute='ID', sampling_size=None,
                           verbose=False):
         
         points_gdf = self.copy()
@@ -909,7 +909,6 @@ def explode_to_points(self,reach_id_attribute='NHDPlusID', sampling_size=None,
 
         return(points_gdf)
 
-
     @staticmethod
     def conflate_points(source_points,target_points,source_reach_id_attribute,target_reach_id_attribute,verbose=False):
 
diff --git a/unit_tests/__template_unittests.py b/unit_tests/__template_unittests.py
index 678605bcc..44e960e59 100644
--- a/unit_tests/__template_unittests.py
+++ b/unit_tests/__template_unittests.py
@@ -62,7 +62,6 @@ def test_<method name you are about to test>_success(self):
         
         src.subset_vector_layers(hucCode = params["hucCode"],
                                     nwm_streams_filename = params["nwm_streams"],
-                                    nhd_streams_filename = params["nhd_streams"],
                                     etc, etc for each param)
        
        
@@ -85,7 +84,6 @@ def test_<method name you are about to test>_success(self):
 
     #   clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
     #                                            nwm_streams_filename = params["nwm_streams"],
-    #                                            nhd_streams_filename = params["nhd_streams"],
     #                                            etc, etc for each param)
     #   
        
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index be56c91ee..04dc82c74 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -1,26 +1,23 @@
 {
-	"valid_data":
-	{
+	"valid_data": {
 		"subset_nwm_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
-		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg", 		
-		"subset_nhd_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/NHDPlusBurnLineEvent_subset.gpkg",
+		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
 		"hucCode": "05030104",
-		"subset_nhd_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg", 
+		"subset_nhd_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
 		"wbd_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
 		"wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
 		"dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
 		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
-		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", 
+		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
 		"subset_nwm_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
 		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
-		"nhd_streams": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg", 
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
 		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
-		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg", 
+		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
 		"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
-		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg", 
-		"great_lakes": "/data/inputs/landsea/gl_water_polygons.gpkg", 
-		"lake_buffer_distance": 20,		
+		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
+		"great_lakes": "/data/inputs/landsea/gl_water_polygons.gpkg",
+		"lake_buffer_distance": 20,
 		"wbd_buffer_distance": 5000,
 		"levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
 		"subset_levee_protected_areas": "/data/outputs/rob_test_2/05030104/LeveeProtectedAreas_subset.gpkg"
diff --git a/unit_tests/clip_vectors_to_wbd_unittests.py b/unit_tests/clip_vectors_to_wbd_unittests.py
index adb3cbe46..1e4193def 100644
--- a/unit_tests/clip_vectors_to_wbd_unittests.py
+++ b/unit_tests/clip_vectors_to_wbd_unittests.py
@@ -50,17 +50,14 @@ def test_subset_vector_layers_success(self):
         # for now we are happy if no exceptions are thrown.
         src.subset_vector_layers(subset_nwm_lakes = params["subset_nwm_lakes"],
                                 subset_nwm_streams = params["subset_nwm_streams"],
-                                subset_nhd_streams = params["subset_nhd_streams"],
                                 hucCode = params["hucCode"],
-                                subset_nhd_headwaters = params["subset_nhd_headwaters"],
-                                wbd_buffer_filename = params["wbd_buffer_filename"],
+                                subset_nhd_headwaters = params["subset_nhd_headwaters"],                                wbd_buffer_filename = params["wbd_buffer_filename"],
                                 wbd_filename = params["wbd_filename"],
                                 dem_filename = params["dem_filename"],
                                 nwm_lakes = params["nwm_lakes"],
                                 nwm_catchments = params["nwm_catchments"],
                                 subset_nwm_catchments = params["subset_nwm_catchments"],
                                 nld_lines = params["nld_lines"],
-                                nhd_streams = params["nhd_streams"],
                                 landsea = params["landsea"],
                                 nwm_streams = params["nwm_streams"],
                                 subset_landsea = params["subset_landsea"],

From 8a043d6825b5c506da7d4e87128a422e275e3a68 Mon Sep 17 00:00:00 2001
From: CalebOliven-NOAA <97476664+CalebOliven-NOAA@users.noreply.github.com>
Date: Fri, 9 Dec 2022 09:56:56 -0700
Subject: [PATCH 184/359] V4.0.13.1 fim4 alpha by catchments (#743)

---
 docs/CHANGELOG.md                |  13 ++
 tools/pixel_counter.py           | 291 +++++++++++++++++++++++++++++++
 tools/pixel_counter_functions.py | 110 ++++++++++++
 tools/pixel_counter_wrapper.py   | 102 +++++++++++
 tools/test_case_by_hydro_id.py   | 219 +++++++++++++++++++++++
 5 files changed, 735 insertions(+)
 create mode 100644 tools/pixel_counter.py
 create mode 100644 tools/pixel_counter_functions.py
 create mode 100644 tools/pixel_counter_wrapper.py
 create mode 100644 tools/test_case_by_hydro_id.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d2d654681..265765d1b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.13.1 - 2022-12-09 - [PR #743](https://github.com/NOAA-OWP/inundation-mapping/pull/743)
+
+This merge adds the tools required to generate Alpha metrics by hydroid. It summarizes the Apha metrics by branch 0 catchment for use in the Hydrovis "FIM Performance" service.
+
+## Additions
+
+- `pixel_counter.py`:  A script to perform zonal statistics against raster data and geometries
+- `pixel_counter_functions.py`: Supporting functions
+- `pixel_counter_wrapper.py`: a script that wraps `pixel_counter.py` for batch processing
+- `test_case_by_hydroid.py`: the main script to orchestrate the generation of alpha metrics by catchment
+
+<br/><br/>
+
 ## v4.0.13.0 - 2022-11-16 - [PR #744](https://github.com/NOAA-OWP/inundation-mapping/pull/744)
 
 Changes branch 0 headwaters data source from NHD to NWS to be consistent with branches. Removes references to NHD flowlines and headwater data.
diff --git a/tools/pixel_counter.py b/tools/pixel_counter.py
new file mode 100644
index 000000000..b7ca06602
--- /dev/null
+++ b/tools/pixel_counter.py
@@ -0,0 +1,291 @@
+'''Created on 02/21/2022.
+Written by:
+Anuska Narayanan (The University of Alabama Department of Geography, anarayanan1@crimson.ua.edu;
+Sophie Williams (The University of Alabama Department of Geography, scwilliams8@crimson.ua.edu; and
+Brad Bates (NOAA, Lynker, and the National Water Center, bradford.bates@noaa.gov)
+
+Derived from a Python version of a zonal statistics function written by Matthew Perry (@perrygeo).
+
+Description: This script isolates the number of pixels per class of a raster within the outlines of
+one or more polygons and displays them in a table. It accomplishes this by rasterizing the vector file,
+masking out the desired areas of both rasters, and then summarizing them in a dataframe. It makes use
+of the gdal, numpy, and pandas function libraries.
+Inputs: one raster file with at least one set of attributes; one vector file containing one or more polygon
+boundaries
+Output: a dataframe table with rows displayed by each polygon within the vector file, and columns
+displaying the pixel count of each raster attribute class in the polygon
+'''
+
+# Import raster and vector function libraries
+#from types import NoneType
+from osgeo import gdal, ogr
+from osgeo.gdalconst import *
+# Import numerical data library
+import numpy as np
+# Import file management library
+import sys
+import os
+# Import data analysis library
+import pandas as pd
+import argparse
+from pandas import DataFrame
+import copy
+import pathlib
+import tempfile
+
+from pixel_counter_functions import (get_nlcd_counts, get_levee_counts, get_bridge_counts, get_nlcd_counts_inside_flood,get_mask_value_counts)
+
+
+# Set up error handler
+gdal.PushErrorHandler('CPLQuietErrorHandler')
+
+# Function to pologonize flood extent
+def make_flood_extent_polygon(flood_extent):
+    flood_extent_dataset = gdal.Open(flood_extent)
+    cols = flood_extent_dataset.RasterXSize
+    rows = flood_extent_dataset.RasterYSize
+
+    # Get some metadata to filter out NaN values
+    flood_extent_raster = flood_extent_dataset.GetRasterBand(1)
+    noDataVal = flood_extent_raster.GetNoDataValue()  # no data value
+    scaleFactor = flood_extent_raster.GetScale()  # scale factor
+
+    # Assign flood_extent No Data Values to NaN
+    flood_extent_array = flood_extent_dataset.GetRasterBand(1).ReadAsArray(0, 0, cols, rows).astype(np.float)
+    flood_extent_array[flood_extent_array == int(noDataVal)] = np.nan
+    flood_extent_array = flood_extent_array / scaleFactor
+
+    # Assign flood_extent Negative Values to NaN
+    flood_extent_nonzero_array = copy.copy(flood_extent_array)
+    flood_extent_nonzero_array[flood_extent_array < 0] = np.nan
+
+    # make temporary output file
+    mem_drv = gdal.GetDriverByName('MEM')
+    target = mem_drv.Create('temp_tif', cols, rows, 1, gdal.GDT_Float32)
+    target.GetRasterBand(1).WriteArray(flood_extent_nonzero_array)
+
+    # Add GeoTranform and Projection
+    geotrans = flood_extent_dataset.GetGeoTransform()
+    proj = flood_extent_dataset.GetProjection()
+    target.SetGeoTransform(geotrans)
+    target.SetProjection(proj)
+    target.FlushCache()
+
+    # set up inputs for converting flood extent raster to polygon
+    band = target.GetRasterBand(1)
+    band.ReadAsArray()
+
+    outshape_location = tempfile.gettempdir()
+    outshape_location_path = os.path.abspath(outshape_location)
+    outShapefile = outshape_location_path + "/" + "polygonized.shp"
+    driver = ogr.GetDriverByName("ESRI Shapefile")
+    outDatasource = driver.CreateDataSource(outShapefile)
+    outLayer = outDatasource.CreateLayer("buffalo", srs=None)
+
+    # Add the DN field
+    newField = ogr.FieldDefn('HydroID', ogr.OFTInteger)
+    outLayer.CreateField(newField)
+
+    # Polygonize
+    gdal.Polygonize(band, None, outLayer, 0, [], callback=None)
+    outDatasource.Destroy()
+    sourceRaster = None
+
+    fullpath = os.path.abspath(outShapefile)
+    print(fullpath)
+    print(type(fullpath))
+
+    return fullpath
+# Function that transforms vector dataset to raster
+def bbox_to_pixel_offsets(gt, bbox):
+    originX = gt[0]
+    originY = gt[3]
+    pixel_width = gt[1]
+    pixel_height = gt[5]
+    x1 = int((bbox[0] - originX) / pixel_width)
+    x2 = int((bbox[1] - originX) / pixel_width) + 1
+
+    y1 = int((bbox[3] - originY) / pixel_height)
+    y2 = int((bbox[2] - originY) / pixel_height) + 1
+
+    xsize = x2 - x1
+    ysize = y2 - y1
+    return (x1, y1, xsize, ysize)
+
+
+# Main function that determines zonal statistics of raster classes in a polygon area
+def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_extent=False):
+
+    # Loop through different raster paths in the raster_path_dict and
+    # perform zonal statistics on the files.
+    for layer in raster_path_dict:
+        raster_path = raster_path_dict[layer]
+        if raster_path == "":  # Only process if a raster path is provided
+            continue
+        if layer == 'flood_extent' and raster_path_dict["nlcd"]!= "":
+            vector_path = make_flood_extent_polygon(flood_extent)
+            raster_path = raster_path_dict["nlcd"]
+
+
+        # Opens raster file and sets path
+        rds = gdal.Open(raster_path, GA_ReadOnly)
+        
+        assert rds
+        rb = rds.GetRasterBand(1)
+        rgt = rds.GetGeoTransform()
+
+        if nodata_value:
+            nodata_value = float(nodata_value)
+            rb.SetNoDataValue(nodata_value)
+        if vector_path == "":
+            print('No vector path provided. Continuing to next layer.')
+            continue
+        # Opens vector file and sets path
+        
+        try:
+            vds = ogr.Open(vector_path)
+            vlyr = vds.GetLayer(0)
+        except:
+            continue
+        
+        # Creates an in-memory numpy array of the source raster data covering the whole extent of the vector layer
+        if global_src_extent:
+            # use global source extent
+            # useful only when disk IO or raster scanning inefficiencies are your limiting factor
+            # advantage: reads raster data in one pass
+            # disadvantage: large vector extents may have big memory requirements
+            src_offset = bbox_to_pixel_offsets(rgt, vlyr.GetExtent())
+            src_array = rb.ReadAsArray(*src_offset)
+
+            # calculate new geotransform of the layer subset
+            new_gt = (
+                (rgt[0] + (src_offset[0] * rgt[1])),
+                rgt[1],
+                0.0,
+                (rgt[3] + (src_offset[1] * rgt[5])),
+                0.0,
+                rgt[5]
+            )
+
+        mem_drv = ogr.GetDriverByName('Memory')
+        driver = gdal.GetDriverByName('MEM')
+
+        # Loop through vectors, as many as exist in file
+        # Creates new list to contain their stats
+        stats = []
+        feat = vlyr.GetNextFeature()
+        while feat is not None:
+
+            if not global_src_extent:
+                # use local source extent
+                # fastest option when you have fast disks and well indexed raster (ie tiled Geotiff)
+                # advantage: each feature uses the smallest raster chunk
+                # disadvantage: lots of reads on the source raster
+                src_offset = bbox_to_pixel_offsets(rgt, feat.geometry().GetEnvelope())
+                src_array = rb.ReadAsArray(*src_offset)
+
+                # calculate new geotransform of the feature subset
+                new_gt = (
+                    (rgt[0] + (src_offset[0] * rgt[1])),
+                    rgt[1],
+                    0.0,
+                    (rgt[3] + (src_offset[1] * rgt[5])),
+                    0.0,
+                    rgt[5]
+                )
+
+            # Create a temporary vector layer in memory
+            mem_ds = mem_drv.CreateDataSource('out')
+            mem_layer = mem_ds.CreateLayer('poly', None, ogr.wkbPolygon)
+            mem_layer.CreateFeature(feat.Clone())
+
+            # Rasterize temporary vector layer
+            rvds = driver.Create('', src_offset[2], src_offset[3], 1, gdal.GDT_Byte)
+            rvds.SetGeoTransform(new_gt)
+            gdal.RasterizeLayer(rvds, [1], mem_layer, burn_values=[1])
+            rv_array = rvds.ReadAsArray()
+
+            # Mask the source data array with our current feature and get statistics (pixel count) of masked areas
+            # we take the logical_not to flip 0<->1 to get the correct mask effect
+            # we also mask out nodata values explictly
+            if src_array is None:
+                feat = vlyr.GetNextFeature()
+                continue
+            masked = np.ma.MaskedArray(
+                src_array,
+                mask=np.logical_or(
+                    src_array == nodata_value,
+                    np.logical_not(rv_array)
+                )
+            )
+
+            # Call different counter functions depending on the raster's source.
+            if layer == "nlcd":
+                feature_stats = get_nlcd_counts(feat, masked)
+            if layer == "agreement_raster":
+                feature_stats = get_mask_value_counts(feat, masked)
+            if layer == "levees":
+                feature_stats = get_levee_counts(feat, masked)
+            if layer == "bridges":
+                feature_stats = get_bridge_counts(feat, masked)
+            if layer == "flood_extent":
+                feature_stats = get_nlcd_counts_inside_flood(feat, masked)
+
+            stats.append(feature_stats)
+
+            rvds = None
+            mem_ds = None
+            feat = vlyr.GetNextFeature()
+
+    vds = None
+    rds = None
+    return stats
+
+
+# Creates and prints dataframe containing desired statistics
+if __name__ == "__main__":
+    # opts = {'VECTOR': sys.argv[1:], 'RASTER': sys.argv[2:]}
+    # stats = zonal_stats(opts['VECTOR'], opts['RASTER'])
+
+    parser = argparse.ArgumentParser(description='Computes pixel counts for raster classes within a vector area.')
+    parser.add_argument('-v', '--vector',
+                        help='Path to vector file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-n', '--nlcd',
+                        help='Path to National Land Cover Database raster file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-l', '--levees',
+                        help='Path to levees raster file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-b', '--bridges',
+                        help='Path to bridges file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-f', '--flood_extent',
+                        help='Path to flood extent file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-c', '--csv',
+                        help='Path to export csv file.',
+                        required=True)
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+    vector = args['vector']
+    nlcd = args['nlcd']
+    levees = args['levees']
+    bridges = args['bridges']
+    flood_extent = args['flood_extent']
+
+    csv = args['csv']
+    
+    raster_path_dict = {'nlcd': nlcd, 'levees': levees, 'bridges': bridges, 'flood_extent': flood_extent}
+    stats = zonal_stats(vector, raster_path_dict)
+
+    # Export CSV
+    df = pd.DataFrame(stats)
+    result = df[(df >= 0).all(axis=1)]
+    df2 = pd.DataFrame(result)
+    df2.to_csv(csv, index=False)
diff --git a/tools/pixel_counter_functions.py b/tools/pixel_counter_functions.py
new file mode 100644
index 000000000..63add9861
--- /dev/null
+++ b/tools/pixel_counter_functions.py
@@ -0,0 +1,110 @@
+import numpy as np
+def get_nlcd_counts_inside_flood(feat,masked):
+    # Gets NLCD Counds inside flood extent
+    feature_stats = {
+        'f_HydroID': feat.GetField('HydroID'),
+        'f_TotalPixels': int(masked.count()),
+        'f_lulc_11': np.count_nonzero((masked == [11])),
+        'f_lulc_12': np.count_nonzero((masked == [12])),
+        'f_lulc_21': np.count_nonzero((masked == [21])),
+        'f_lulc_22': np.count_nonzero((masked == [22])),
+        'f_lulc_23': np.count_nonzero((masked == [23])),
+        'f_lulc_24': np.count_nonzero((masked == [24])),
+        'f_lulc_31': np.count_nonzero((masked == [31])),
+        'f_lulc_41': np.count_nonzero((masked == [41])),
+        'f_lulc_42': np.count_nonzero((masked == [42])),
+        'f_lulc_43': np.count_nonzero((masked == [43])),
+        'f_lulc_51': np.count_nonzero((masked == [51])),
+        'f_lulc_52': np.count_nonzero((masked == [52])),
+        'f_lulc_71': np.count_nonzero((masked == [71])),
+        'f_lulc_72': np.count_nonzero((masked == [72])),
+        'f_lulc_73': np.count_nonzero((masked == [73])),
+        'f_lulc_74': np.count_nonzero((masked == [74])),
+        'f_lulc_81': np.count_nonzero((masked == [81])),
+        'f_lulc_82': np.count_nonzero((masked == [82])),
+        'f_lulc_90': np.count_nonzero((masked == [90])),
+        'f_lulc_95': np.count_nonzero((masked == [95])),
+        'f_lulc_1': np.count_nonzero((masked == [11])) + np.count_nonzero((masked == [12])),
+        'f_lulc_2': np.count_nonzero((masked == [21])) + np.count_nonzero((masked == [22]))
+                  + np.count_nonzero((masked == [23])) + np.count_nonzero((masked == [24])),
+        'f_lulc_3': np.count_nonzero((masked == [31])),
+        'f_lulc_4': np.count_nonzero((masked == [41])) + np.count_nonzero((masked == [42]))
+                  + np.count_nonzero((masked == [43])),
+        'f_lulc_5': np.count_nonzero((masked == [51])) + np.count_nonzero((masked == [52])),
+        'f_lulc_7': np.count_nonzero((masked == [71])) + np.count_nonzero((masked == [72]))
+                  + np.count_nonzero((masked == [73])) + np.count_nonzero((masked == [74])),
+        'f_lulc_8': np.count_nonzero((masked == [81])) + np.count_nonzero((masked == [82])),
+        'f_lulc_9': np.count_nonzero((masked == [90])) + np.count_nonzero((masked == [95]))
+
+    }
+    return feature_stats
+
+def get_nlcd_counts(feat, masked):
+    # Acquires information for table on each raster attribute per poly feature
+    feature_stats = {
+        'FID': int(feat.GetFID()),
+        'HydroID': feat.GetField('HydroID'),
+        'TotalPixels': int(masked.count()),
+        'lulc_11': np.count_nonzero((masked == [11])),
+        'lulc_12': np.count_nonzero((masked == [12])),
+        'lulc_21': np.count_nonzero((masked == [21])),
+        'lulc_22': np.count_nonzero((masked == [22])),
+        'lulc_23': np.count_nonzero((masked == [23])),
+        'lulc_24': np.count_nonzero((masked == [24])),
+        'lulc_31': np.count_nonzero((masked == [31])),
+        'lulc_41': np.count_nonzero((masked == [41])),
+        'lulc_42': np.count_nonzero((masked == [42])),
+        'lulc_43': np.count_nonzero((masked == [43])),
+        'lulc_51': np.count_nonzero((masked == [51])),
+        'lulc_52': np.count_nonzero((masked == [52])),
+        'lulc_71': np.count_nonzero((masked == [71])),
+        'lulc_72': np.count_nonzero((masked == [72])),
+        'lulc_73': np.count_nonzero((masked == [73])),
+        'lulc_74': np.count_nonzero((masked == [74])),
+        'lulc_81': np.count_nonzero((masked == [81])),
+        'lulc_82': np.count_nonzero((masked == [82])),
+        'lulc_90': np.count_nonzero((masked == [90])),
+        'lulc_95': np.count_nonzero((masked == [95])),
+        'lulc_1': np.count_nonzero((masked == [11])) + np.count_nonzero((masked == [12])),
+        'lulc_2': np.count_nonzero((masked == [21])) + np.count_nonzero((masked == [22]))
+                  + np.count_nonzero((masked == [23])) + np.count_nonzero((masked == [24])),
+        'lulc_3': np.count_nonzero((masked == [31])),
+        'lulc_4': np.count_nonzero((masked == [41])) + np.count_nonzero((masked == [42]))
+                  + np.count_nonzero((masked == [43])),
+        'lulc_5': np.count_nonzero((masked == [51])) + np.count_nonzero((masked == [52])),
+        'lulc_7': np.count_nonzero((masked == [71])) + np.count_nonzero((masked == [72]))
+                  + np.count_nonzero((masked == [73])) + np.count_nonzero((masked == [74])),
+        'lulc_8': np.count_nonzero((masked == [81])) + np.count_nonzero((masked == [82])),
+        'lulc_9': np.count_nonzero((masked == [90])) + np.count_nonzero((masked == [95]))
+
+    }
+    return feature_stats
+
+
+def get_levee_counts(feat, masked):
+
+    # Acquires information for table on each levee attribute per catchment
+    feature_stats = {
+        'HydroID': feat.GetField('HydroID'),
+        'TotalLeveePixels': int(masked.count()),
+    }
+    return feature_stats
+
+def get_bridge_counts(feat_masked):
+    pass
+
+
+def get_mask_value_counts(feat, masked):
+    # Acquires information for table on each raster attribute per poly feature
+    feature_stats = {
+        'FID': int(feat.GetFID()),
+        'HydroID': feat.GetField('HydroID'),
+        'TotalPixels': int(masked.count()),
+        'tn': np.count_nonzero((masked == [0])),
+        'fn': np.count_nonzero((masked == [1])),
+        'fp': np.count_nonzero((masked == [2])),
+        'tp': np.count_nonzero((masked == [3])),
+        'mp': np.count_nonzero((masked == [4]))
+        }
+    
+    return feature_stats
diff --git a/tools/pixel_counter_wrapper.py b/tools/pixel_counter_wrapper.py
new file mode 100644
index 000000000..575486646
--- /dev/null
+++ b/tools/pixel_counter_wrapper.py
@@ -0,0 +1,102 @@
+import argparse
+import pandas as pd
+import os
+from multiprocessing import Pool
+import traceback
+
+from pixel_counter import zonal_stats
+
+
+def queue_zonal_stats(fim_run_dir, raster_path_dict, output_dir, job_number):
+    """
+    This function sets up multiprocessing of the process_zonal_stats() function.
+    
+    """
+    
+    # Make output directory if it doesn't exist
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+        
+    fim_version = os.path.split(fim_run_dir)[1]  # Parse FIM Version
+    
+    fim_run_dir_list = os.listdir(fim_run_dir)  # List all HUCs in FIM run dir
+    
+    # Define variables to pass into process_zonal_stats()
+    procs_list = []
+    for huc in fim_run_dir_list:
+        vector = os.path.join(fim_run_dir, huc, 'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg')
+        csv = os.path.join(output_dir, fim_version + '_' + huc + '_pixel_counts.csv')
+        print(csv)
+        procs_list.append([vector, csv, raster_path_dict])
+        
+    # Initiate multiprocessing
+    with Pool(processes=job_number) as pool:
+        pool.map(process_zonal_stats, procs_list)
+
+
+def process_zonal_stats(args):
+    """
+    This function calls zonal_stats() in multiprocessing mode.
+    
+    """
+    # Extract variables from args
+    vector = args[0]
+    csv = args[1]
+    raster = args[2]
+    
+    # Do the zonal stats
+    try:
+        stats = zonal_stats(vector,raster)  # Call function
+        
+        # Export CSV
+        df = pd.DataFrame(stats)
+        df.to_csv(csv)
+        print("Finished writing: " + csv)
+        
+    # Save traceback to error file if error is encountered.
+    except Exception:
+        error_file = csv.replace('.csv', '_error.txt')
+        with open(error_file, 'w+') as f:
+            traceback.print_exc(file=f)
+
+
+if __name__ == '__main__':
+    
+    parser = argparse.ArgumentParser(description='Computes pixel counts for raster classes within a vector area.')
+    parser.add_argument('-d', '--fim-run-dir',
+                        help='Path to vector file.',
+                        required=True)
+    parser.add_argument('-n', '--nlcd',
+                        help='Path to National Land Cover Database raster file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-l', '--levees',
+                        help='Path to levees raster file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-b', '--bridges',
+                        help='Path to bridges file.',
+                        required=False,
+                        default="")
+    parser.add_argument('-o', '--output-dir',
+                        help='Path to output directory where CSV files will be written.',
+                        required=False,
+                        default="")
+    parser.add_argument('-j', '--job-number',
+                        help='Number of jobs to use.',
+                        required=False,
+                        default="")
+
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+    nlcd = args['nlcd']
+    levees = args['levees']
+    bridges = args['bridges']
+
+    raster_path_dict = {'nlcd': nlcd, 'levees': levees, 'bridges': bridges}
+    
+    args = vars(parser.parse_args())
+    
+    raster_path_dict = {'nlcd': args['nlcd']}
+    
+    queue_zonal_stats(args['fim_run_dir'], raster_path_dict, args['output_dir'], int(args['job_number']))
diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py
new file mode 100644
index 000000000..3db6c7d6c
--- /dev/null
+++ b/tools/test_case_by_hydro_id.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+
+# Import file management library
+import os
+# Import data analysis library
+import pandas as pd
+import geopandas as gpd
+import argparse
+from pixel_counter import zonal_stats
+from tools_shared_functions import compute_stats_from_contingency_table
+from run_test_case import test_case
+from shapely.validation import make_valid
+
+#####################################################
+# Perform zonal stats is a funtion stored in pixel_counter.py.
+# The input huc_gpkg is a single huc8 geopackage, the second input argument must be input as a dict.
+# For the purposes of assembling the alpha metrics by hydroid, always use agreement_raster total area agreement tiff.
+# This function is called automatically. Returns stats dict of pixel counts.
+#####################################################
+def perform_zonal_stats(huc_gpkg,agree_rast):
+    stats = zonal_stats(huc_gpkg,{"agreement_raster":agree_rast}, nodata_value=10)
+    return stats
+
+#####################################################
+# Creates a pandas df containing Alpha stats by hydroid.
+# Stats input is the output of zonal_stats function.
+# Huc8 is the huc8 string and is passed via the directory loop during execution.
+# Mag is the magnitude (100y, action, minor etc.) is passed via the directory loop.
+# Bench is the benchmark source.
+#####################################################
+def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
+    in_mem_df = pd.DataFrame(columns=['HydroID', 'huc8','contingency_tot_area_km2',
+                                    'CSI', 'FAR', 'TPR', 'TNR',  'PPV', 'NPV', 'Bal_ACC',
+                                    'MCC', 'EQUITABLE_THREAT_SCORE', 'PREVALENCE', 'BIAS', 'F1_SCORE',
+                                    'masked_perc', 'MAG','BENCH'])
+
+    for dicts in stats:
+        tot_pop = dicts['tn'] +dicts['fn'] + dicts['fp'] + dicts['tp']
+        if tot_pop == 0:
+            continue
+        
+        stats_dictionary = compute_stats_from_contingency_table(dicts['tn'], dicts['fn'], dicts['fp'], dicts['tp'], cell_area=100, masked_count= dicts['mp'])
+        # Calls compute_stats_from_contingency_table from run_test_case.py
+                    
+        hydroid = dicts['HydroID']
+        stats_dictionary['HydroID'] = hydroid
+               
+        contingency_tot_area_km2 = float(stats_dictionary['contingency_tot_area_km2'])
+        if contingency_tot_area_km2 != 'NA':
+            contingency_tot_area_km2 = round(contingency_tot_area_km2,2)
+       
+        CSI = stats_dictionary['CSI']
+        if CSI != 'NA':
+            CSI = round(CSI,2)
+
+        FAR = stats_dictionary['FAR']
+        if FAR != 'NA':
+            FAR = round(FAR,2)
+        
+        TPR = stats_dictionary['TPR']
+        if TPR != 'NA':
+            TPR = round(TPR,2)
+
+        TNR = stats_dictionary['TNR']
+        if TNR != 'NA':
+            TNR = round(TNR,2)
+
+        PPV = stats_dictionary['PPV']
+        if PPV != 'NA':
+            PPV = round(PPV,2)
+
+        NPV = stats_dictionary['NPV']
+        if NPV != 'NA':
+            NPV = round(NPV,2)
+
+        Bal_ACC = stats_dictionary['Bal_ACC']
+        if Bal_ACC != 'NA':
+            Bal_ACC = round(Bal_ACC,2)
+
+        MCC = float(stats_dictionary['MCC'])
+        if MCC != 'NA':
+            MCC = round(MCC,2)
+
+        EQUITABLE_THREAT_SCORE = stats_dictionary['EQUITABLE_THREAT_SCORE']
+        if EQUITABLE_THREAT_SCORE != 'NA':
+            EQUITABLE_THREAT_SCORE = round(EQUITABLE_THREAT_SCORE,2)
+
+        PREVALENCE = stats_dictionary['PREVALENCE']
+        if PREVALENCE != 'NA':
+            PREVALENCE = round(PREVALENCE,2)
+
+        BIAS = stats_dictionary['BIAS']
+        if BIAS != 'NA':
+            BIAS = round(BIAS,2)   
+
+        F1_SCORE = stats_dictionary['F1_SCORE']
+        if F1_SCORE != 'NA':
+            F1_SCORE = round(F1_SCORE,2)    
+
+        masked_perc = stats_dictionary['masked_perc']
+        if masked_perc != 'NA':
+            masked_perc = round(masked_perc,2)
+
+        HydroID = stats_dictionary['HydroID']
+
+        dict_with_list_values = {'HydroID': [HydroID],'huc8':[huc8], 'contingency_tot_area_km2': [contingency_tot_area_km2],
+        'CSI': [CSI], 'FAR': [FAR], 'TPR': [TPR], 'TNR': [TNR], 'PPV': [PPV], 'NPV': [NPV],
+        'Bal_ACC': [Bal_ACC], 'MCC': [MCC], 'EQUITABLE_THREAT_SCORE': [EQUITABLE_THREAT_SCORE], 'PREVALENCE': [PREVALENCE],
+        'BIAS': [BIAS], 'F1_SCORE': [F1_SCORE], 'masked_perc': [masked_perc], 'MAG':[mag],'BENCH':[bench]}
+        
+        dict_to_df = pd.DataFrame(dict_with_list_values,columns=['HydroID','huc8', 'contingency_tot_area_km2',
+        'CSI', 'FAR', 'TPR', 'TNR', 'PPV', 'NPV', 'Bal_ACC',
+        'MCC', 'EQUITABLE_THREAT_SCORE', 'PREVALENCE', 'BIAS', 'F1_SCORE', 'masked_perc', 'MAG','BENCH'])
+        
+        concat_list = [in_mem_df, dict_to_df]
+        in_mem_df = pd.concat(concat_list, sort=False)
+        
+    return in_mem_df
+
+
+if __name__ == "__main__":
+    
+    parser = argparse.ArgumentParser(description='Produces alpha metrics by hyrdoid.')
+
+    parser.add_argument('-b', '--benchmark_category',
+                        help='Choice of truth data. Options are: all, ble, ifc, nws, usgs, ras2fim',
+                        required=True)
+    parser.add_argument('-v', '--version',
+                        help='The fim version to use. Should be similar to fim_3_0_24_14_ms',
+                        required=True)
+    parser.add_argument('-g', '--gpkg',
+                        help='filepath and filename to hold exported gpkg (and csv) file. Similar to /data/path/output.gpkg Need to use gpkg as output. ',
+                        required=True)
+
+
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+    benchmark_category = args['benchmark_category']
+    version = args['version']
+    csv = args['gpkg']
+        
+    # Execution code
+    csv_output = gpd.GeoDataFrame(columns=['HydroID', 'huc8','contingency_tot_area_km2',
+                                    'CSI', 'FAR', 'TPR', 'TNR',  'PPV', 'NPV', 'Bal_ACC',
+                                    'MCC', 'EQUITABLE_THREAT_SCORE', 'PREVALENCE', 'BIAS', 'F1_SCORE',
+                                    'masked_perc', 'MAG','BENCH','geometry'], geometry = 'geometry')
+
+    # This funtion, relies on the test_case class defined in run_test_case.py to list all available test cases
+    print('listing_test_cases_with_updates')
+    all_test_cases = test_case.list_all_test_cases(version=version, archive=True, benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
+    
+    for test_case_class in all_test_cases:
+
+        if not os.path.exists(test_case_class.fim_dir):
+            print(f'{test_case_class.fim_dir} does not exist')
+            continue
+        else:
+            
+            print(test_case_class.fim_dir, end='\r')
+                       
+            agreement_dict = test_case_class.get_current_agreements()
+           
+            for agree_rast in agreement_dict:
+                    
+                    print('performing_zonal_stats')
+
+                    branches_dir = os.path.join(test_case_class.fim_dir,'branches')
+                    for branches in os.listdir(branches_dir):
+                        if branches != "0":
+                            continue
+                        huc_gpkg = os.path.join(branches_dir,branches,)
+                                                
+                        string_manip = "gw_catchments_reaches_filtered_addedAttributes_crosswalked_" + branches + ".gpkg"
+                        
+                        huc_gpkg = os.path.join(huc_gpkg, string_manip)
+                        
+                        define_mag = agree_rast.split(version)
+
+                        define_mag_1 = define_mag[1].split('/')
+                        
+                        mag = define_mag_1[1]
+                   
+                        stats = perform_zonal_stats(huc_gpkg,agree_rast)
+                        
+                        print('assembling_hydroalpha_for_single_huc')
+                        get_geom = gpd.read_file(huc_gpkg)
+                        
+                        get_geom['geometry'] = get_geom.apply(lambda row: make_valid(row.geometry), axis=1)
+                                            
+                        in_mem_df = assemble_hydro_alpha_for_single_huc(stats, test_case_class.huc, mag, test_case_class.benchmark_cat)
+                        
+                        hydro_geom_df = get_geom[["HydroID", "geometry"]]
+                        
+                        geom_output = hydro_geom_df.merge(in_mem_df, on='HydroID', how ='inner')
+                        
+                        concat_df_list = [geom_output, csv_output]
+
+                        csv_output = pd.concat(concat_df_list, sort=False)
+                                           
+    print('projecting to 3857')
+    csv_output = csv_output.to_crs('EPSG:3857')
+
+    print('manipulating the input string to exclude gpkg and include csv')
+    csv_path_list = csv.split(".")
+    csv_path = csv_path_list[0]
+    csv_path_dot = csv_path + ".csv"
+    
+    print('writing_to_gpkg')
+    csv_output.to_file(csv, driver="GPKG")
+    
+    # Add version information to csv_output dataframe
+    csv_output['version'] = version
+
+    print('writing_to_csv')
+    csv_output.to_csv(csv_path_dot) # Save to CSV
+
+
+    
+

From b2328f58951f119134130eeccd3510155fe395ef Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Dec 2022 09:47:16 -0600
Subject: [PATCH 185/359] Bump certifi from 2022.9.24 to 2022.12.7 (#757)

---
 Pipfile.lock | 576 +++++++++++++++++++++++++--------------------------
 1 file changed, 282 insertions(+), 294 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 6d2621fbe..af80a8c82 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -68,10 +68,10 @@
         },
         "asttokens": {
             "hashes": [
-                "sha256:1b28ed85e254b724439afc783d4bee767f780b936c3fe8b3275332f42cf5f561",
-                "sha256:4aa76401a151c8cc572d906aad7aea2a841780834a19d780f4321c0fe1b54635"
+                "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
+                "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"
             ],
-            "version": "==2.1.0"
+            "version": "==2.2.1"
         },
         "attrs": {
             "hashes": [
@@ -83,11 +83,11 @@
         },
         "babel": {
             "hashes": [
-                "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51",
-                "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"
+                "sha256:1ad3eca1c885218f6dce2ab67291178944f810a10a9b5f3cb8382a5a232b64fe",
+                "sha256:5ef4b3226b0180dedded4229651c8b0e1a3a6a2837d45a073272f313e4cf97f6"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.10.3"
+            "version": "==2.11.0"
         },
         "backcall": {
             "hashes": [
@@ -130,11 +130,11 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14",
-                "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"
+                "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
+                "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2022.9.24"
+            "index": "pypi",
+            "version": "==2022.12.7"
         },
         "cffi": {
             "hashes": [
@@ -263,6 +263,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
+        "comm": {
+            "hashes": [
+                "sha256:3e2f5826578e683999b93716285b3b1f344f157bf75fa9ce0a797564e742f062",
+                "sha256:9f3abf3515112fa7c55a42a6a5ab358735c9dccc8b5910a9d8e3ef5998130666"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.1.2"
+        },
         "contourpy": {
             "hashes": [
                 "sha256:0236875c5a0784215b49d00ebbe80c5b6b5d5244b3655a36dda88105334dea17",
@@ -348,27 +356,27 @@
         },
         "debugpy": {
             "hashes": [
-                "sha256:34d2cdd3a7c87302ba5322b86e79c32c2115be396f3f09ca13306d8a04fe0f16",
-                "sha256:3c9f985944a30cfc9ae4306ac6a27b9c31dba72ca943214dad4a0ab3840f6161",
-                "sha256:4e255982552b0edfe3a6264438dbd62d404baa6556a81a88f9420d3ed79b06ae",
-                "sha256:5ad571a36cec137ae6ed951d0ff75b5e092e9af6683da084753231150cbc5b25",
-                "sha256:6efc30325b68e451118b795eff6fe8488253ca3958251d5158106d9c87581bc6",
-                "sha256:7c302095a81be0d5c19f6529b600bac971440db3e226dce85347cc27e6a61908",
-                "sha256:84c39940a0cac410bf6aa4db00ba174f973eef521fbe9dd058e26bcabad89c4f",
-                "sha256:86d784b72c5411c833af1cd45b83d80c252b77c3bfdb43db17c441d772f4c734",
-                "sha256:adcfea5ea06d55d505375995e150c06445e2b20cd12885bcae566148c076636b",
-                "sha256:b8deaeb779699350deeed835322730a3efec170b88927debc9ba07a1a38e2585",
-                "sha256:c4b2bd5c245eeb49824bf7e539f95fb17f9a756186e51c3e513e32999d8846f3",
-                "sha256:c4cd6f37e3c168080d61d698390dfe2cd9e74ebf80b448069822a15dadcda57d",
-                "sha256:cca23cb6161ac89698d629d892520327dd1be9321c0960e610bbcb807232b45d",
-                "sha256:d5c814596a170a0a58fa6fad74947e30bfd7e192a5d2d7bd6a12156c2899e13a",
-                "sha256:daadab4403427abd090eccb38d8901afd8b393e01fd243048fab3f1d7132abb4",
-                "sha256:dda8652520eae3945833e061cbe2993ad94a0b545aebd62e4e6b80ee616c76b2",
-                "sha256:e8922090514a890eec99cfb991bab872dd2e353ebb793164d5f01c362b9a40bf",
-                "sha256:fc233a0160f3b117b20216f1169e7211b83235e3cd6749bcdd8dbb72177030c7"
+                "sha256:143f79d0798a9acea21cd1d111badb789f19d414aec95fa6389cfea9485ddfb1",
+                "sha256:1caee68f7e254267df908576c0d0938f8f88af16383f172cb9f0602e24c30c01",
+                "sha256:2a39e7da178e1f22f4bc04b57f085e785ed1bcf424aaf318835a1a7129eefe35",
+                "sha256:3d9c31baf64bf959a593996c108e911c5a9aa1693a296840e5469473f064bcec",
+                "sha256:40e2a83d31a16b83666f19fa06d97b2cc311af88e6266590579737949971a17e",
+                "sha256:4ab5e938925e5d973f567d6ef32751b17d10f3be3a8c4d73c52f53e727f69bf1",
+                "sha256:563f148f94434365ec0ce94739c749aabf60bf67339e68a9446499f3582d62f3",
+                "sha256:62ba4179b372a62abf9c89b56997d70a4100c6dea6c2a4e0e4be5f45920b3253",
+                "sha256:67edf033f9e512958f7b472975ff9d9b7ff64bf4440f6f6ae44afdc66b89e6b6",
+                "sha256:6ae238943482c78867ac707c09122688efb700372b617ffd364261e5e41f7a2f",
+                "sha256:82229790442856962aec4767b98ba2559fe0998f897e9f21fb10b4fd24b6c436",
+                "sha256:86bd25f38f8b6c5d430a5e2931eebbd5f580c640f4819fcd236d0498790c7204",
+                "sha256:d2968e589bda4e485a9c61f113754a28e48d88c5152ed8e0b2564a1fadbe50a5",
+                "sha256:d5ab9bd3f4e7faf3765fd52c7c43c074104ab1e109621dc73219099ed1a5399d",
+                "sha256:d8df268e9f72fc06efc2e75e8dc8e2b881d6a397356faec26efb2ee70b6863b7",
+                "sha256:e62b8034ede98932b92268669318848a0d42133d857087a3b9cec03bb844c615",
+                "sha256:e886a1296cd20a10172e94788009ce74b759e54229ebd64a43fa5c2b4e62cd76",
+                "sha256:ea4bf208054e6d41749f17612066da861dff10102729d32c85b47f155223cf2b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.6.3"
+            "version": "==1.6.4"
         },
         "decorator": {
             "hashes": [
@@ -474,27 +482,27 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab",
-                "sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43"
+                "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b",
+                "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==5.0.0"
+            "version": "==5.1.0"
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:c01b1b94210d9849f286b86bb51bcea7cd56dde0600d8db721d7b81330711668",
-                "sha256:ee17ec648f85480d523596ce49eae8ead87d5631ae1551f913c0100b5edd3437"
+                "sha256:32bb095bda29741f6ef0e5278c42df98d135391bee5f932841efc0041f748dc3",
+                "sha256:c09b067d82e72c66f4f8eb12332f5efbebc9b007c0b6c40818108c9870adc363"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==5.10.0"
+            "version": "==5.10.1"
         },
         "ipykernel": {
             "hashes": [
-                "sha256:301fdb487587c9bf277025001da97b53697aab73ae1268d9d1ba972a2c5fc801",
-                "sha256:e195cf6d8c3dd5d41f3cf8ad831d9891f95d7d18fa6d5fb4d30a713df99b26a4"
+                "sha256:1374a55c57ca7a7286c3d8b15799cd76e1a2381b6b1fea99c494b955988926b6",
+                "sha256:1ab68d3d3654196266baa93990055413e167263ffbe4cfe834f871bcd3d3506d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.17.0"
+            "version": "==6.19.2"
         },
         "ipympl": {
             "hashes": [
@@ -506,11 +514,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:7c959e3dedbf7ed81f9b9d8833df252c430610e2a4a6464ec13cd20975ce20a5",
-                "sha256:91ef03016bcf72dd17190f863476e7c799c6126ec7e8be97719d1bc9a78a59a4"
+                "sha256:352042ddcb019f7c04e48171b4dd78e4c4bb67bf97030d170e154aac42b656d9",
+                "sha256:882899fe78d5417a0aa07f995db298fa28b58faeba2112d2e3a4c95fe14bb738"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.6.0"
+            "version": "==8.7.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -521,19 +529,19 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:08cb75c6e0a96836147cbfdc55580ae04d13e05d26ffbc377b4e1c68baa28b1f",
-                "sha256:1dc3dd4ee19ded045ea7c86eb273033d238d8e43f9e7872c52d092683f263891"
+                "sha256:2ec50df8538a1d4ddd5d454830d010922ad1015e81ac23efb27c0908bbc1eece",
+                "sha256:db7dd35fb1217636cbdbe0ba0bd2216d91a7695cb28b5c1dca17e62cd51378de"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.0.2"
+            "version": "==8.0.3"
         },
         "jedi": {
             "hashes": [
-                "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d",
-                "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab"
+                "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
+                "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.1"
+            "version": "==0.18.2"
         },
         "jinja2": {
             "hashes": [
@@ -560,11 +568,11 @@
         },
         "jsonschema": {
             "hashes": [
-                "sha256:165059f076eff6971bae5b742fc029a7b4ef3f9bcf04c14e4776a7605de14b23",
-                "sha256:9e74b8f9738d6a946d70705dc692b74b5429cd0960d58e79ffecfc43b2221eb9"
+                "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d",
+                "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.16.0"
+            "version": "==4.17.3"
         },
         "jupyter": {
             "hashes": [
@@ -577,11 +585,11 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:1c1d418ef32a45a1fae0b243e6f01cc9bf65fa8ddbd491a034b9ba6ac6502951",
-                "sha256:5616db609ac720422e6a4b893d6572b8d655ff41e058367f4459a0d2c0726832"
+                "sha256:109a3c33b62a9cf65aa8325850a0999a795fac155d9de4f7555aef5f310ee35a",
+                "sha256:d4a67ae86ee014bcb96bd8190714f6af921f2b0f52f4208b086aa5acfd9f8d65"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.4.4"
+            "version": "==7.4.8"
         },
         "jupyter-console": {
             "hashes": [
@@ -601,11 +609,11 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:992531008544d77e05a16251cdfbd0bdff1b1efa14760c79b9cc776ac9214cf1",
-                "sha256:d0adca19913a3763359be7f0b8c2ea8bfde356f4b8edd8e3149d7d0fbfaa248b"
+                "sha256:438496cac509709cc85e60172e5538ca45b4c8a0862bb97cd73e49f2ace419cb",
+                "sha256:f7f7a2f9d36f4150ad125afef0e20b1c76c8ff83eb5e39fb02d3b9df0f9b79ab"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.21.0"
+            "version": "==1.23.3"
         },
         "jupyterlab": {
             "hashes": [
@@ -625,19 +633,19 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:07007a3a0a30bfc6424b28b76df8d67386cc2d5f9f42886773b1b3c473cb9a3f",
-                "sha256:7ad1a37a716f6d10e90185c636c122d55a58ef3141ae50f9d0601d3ccf54d43e"
+                "sha256:02862dcba01981fc0a0cb82198436301bb5a07ea59c05efeca59c8e9e9066309",
+                "sha256:61381c48268f0a8a84a27858b49db1b02e5aec6549e85e22fcdbf3eb14a2193a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.16.2"
+            "version": "==2.16.5"
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:6aa1bc0045470d54d76b9c0b7609a8f8f0087573bae25700a370c11f82cb38c8",
-                "sha256:c767181399b4ca8b647befe2d913b1260f51bf9d8ef9b7a14632d4c1a7b536bd"
+                "sha256:4c9275daa6d20fc96c3aea45756ece7110850d035b0b93a6a40e918016b927da",
+                "sha256:9a568e022b8bb53ab23291f6ddb52f8002b789c2c5763378cbc882be1d619be8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.3"
+            "version": "==3.0.4"
         },
         "kiwisolver": {
             "hashes": [
@@ -783,50 +791,50 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:02561141c434154f7bae8e5449909d152367cb40aa57bfb2a27f2748b9c5f95f",
-                "sha256:05e86446562063d6186ff6d700118c0dbd5dccc403a6187351ee526c48878f10",
-                "sha256:0bab7564aafd5902128d54b68dca04f5755413fb6b502100bb0235a545882c48",
-                "sha256:12ab21d0cad122f5b23688d453a0280676e7c42f634f0dbd093d15d42d142b1f",
-                "sha256:183bf3ac6a6023ee590aa4b677f391ceed65ec0d6b930901a8483c267bd12995",
-                "sha256:1e2c75d5d1ff6b7ef9870360bfa23bea076b8dc0945a60d19453d7619ed9ea8f",
-                "sha256:220314c2d6b9ca11570d7cd4b841c9f3137546f188336003b9fb8def4dcb804d",
-                "sha256:2469f57e4c5cc0e85eddc7b30995ea9c404a78c0b1856da75d1a5887156ca350",
-                "sha256:27337bcb38d5db7430c14f350924542d75416ec1546d5d9d9f39b362b71db3fb",
-                "sha256:2cc5d726d4d42865f909c5208a7841109d76584950dd0587b01a77cc279d4ab7",
-                "sha256:3c53486278a0629fd892783271dc994b962fba8dfe207445d039e14f1928ea46",
-                "sha256:4648f0d79a87bf50ee740058305c91091ee5e1fbb71a7d2f5fe6707bfe328d1c",
-                "sha256:47cb088bbce82ae9fc2edf3c25e56a5c6142ce2553fea2b781679f960a70c207",
-                "sha256:4a3d903588b519b38ed085d0ae762a1dcd4b70164617292175cfd91b90d6c415",
-                "sha256:4d3b0e0a4611bd22065bbf47e9b2f689ac9e575bcb850a9f0ae2bbed75cab956",
-                "sha256:52935b7d4ccbf0dbc9cf454dbb10ca99c11cbe8da9467596b96e5e21fd4dfc5c",
-                "sha256:563896ba269324872ace436a57775dcc8322678a9496b28a8c25cdafa5ec2b92",
-                "sha256:565f514dec81a41cbed10eb6011501879695087fc2787fb89423a466508abbbd",
-                "sha256:5f97141e05baf160c3ec125f06ceb2a44c9bb62f42fcb8ee1c05313c73e99432",
-                "sha256:6f5788168da2661b42f7468063b725cc73fdbeeb80f2704cb2d8c415e9a57c50",
-                "sha256:71eced071825005011cdc64efbae2e2c76b8209c18aa487dedf69796fe4b1e40",
-                "sha256:7730e60e751cfcfe7fcb223cf03c0b979e9a064c239783ad37929d340a364cef",
-                "sha256:8245e85fd793f58edf29b8a9e3be47e8ecf76ea1a1e8240545f2746181ca5787",
-                "sha256:85948b303534b69fd771126764cf883fde2af9b003eb5778cb60f3b46f93d3f6",
-                "sha256:87027ff7b2edeb14476900261ef04d4beae949e1dfa0a3eb3ad6a6efbf9d0e1d",
-                "sha256:87bdbd37d0a41e025879863fe9b17bab15c0421313bc33e77e5e1aa54215c9c5",
-                "sha256:8dc25473319afabe49150267e54648ac559c33b0fc2a80c8caecfbbc2948a820",
-                "sha256:8ddd58324dc9a77e2e56d7b7aea7dbd0575b6f7cd1333c3ca9d388ac70978344",
-                "sha256:9403764017d20ff570f7ce973a8b9637f08a6109118f4e0ce6c7493d8849a0d3",
-                "sha256:9dd40505ccc526acaf9a5db1b3029e237c64b58f1249983b28a291c2d6a1d0fa",
-                "sha256:a4de03085afb3b80fab341afaf8e60dfe06ce439b6dfed55d657cf34a7bc3c40",
-                "sha256:a68b91ac7e6bb26100a540a033f54c95fe06d9c0aa51312c2a52d07d1bde78f4",
-                "sha256:b53387d4e59432ff221540a4ffb5ee9669c69417805e4faf0148a00d701c61f9",
-                "sha256:c1effccef0cea2d4da9feeed22079adf6786f92c800a7d0d2ef2104318a1c66c",
-                "sha256:c9756a8e69f6e1f76d47eb42132175b6814da1fbeae0545304c6d0fc2aae252a",
-                "sha256:d0161ebf87518ecfe0980c942d5f0d5df0e080c1746ebaab2027a969967014b7",
-                "sha256:e2d1b7225666f7e1bcc94c0bc9c587a82e3e8691da4757e357e5c2515222ee37",
-                "sha256:e3c116e779fbbf421a9e4d3060db259a9bb486d98f4e3c5a0877c599bd173582",
-                "sha256:e4c8b5a243dd29d50289d694e931bd6cb6ae0b5bd654d12c647543d63862540c",
-                "sha256:e632f66218811d4cf8b7a2a649e25ec15406c3c498f72d19e2bcf8377f38445d",
-                "sha256:fad858519bd6d52dbfeebdbe04d00dd8e932ed436f1c535e61bcc970a96c11e4"
+                "sha256:0844523dfaaff566e39dbfa74e6f6dc42e92f7a365ce80929c5030b84caa563a",
+                "sha256:0eda9d1b43f265da91fb9ae10d6922b5a986e2234470a524e6b18f14095b20d2",
+                "sha256:168093410b99f647ba61361b208f7b0d64dde1172b5b1796d765cd243cadb501",
+                "sha256:1836f366272b1557a613f8265db220eb8dd883202bbbabe01bad5a4eadfd0c95",
+                "sha256:19d61ee6414c44a04addbe33005ab1f87539d9f395e25afcbe9a3c50ce77c65c",
+                "sha256:252957e208c23db72ca9918cb33e160c7833faebf295aaedb43f5b083832a267",
+                "sha256:32d29c8c26362169c80c5718ce367e8c64f4dd068a424e7110df1dd2ed7bd428",
+                "sha256:380d48c15ec41102a2b70858ab1dedfa33eb77b2c0982cb65a200ae67a48e9cb",
+                "sha256:3964934731fd7a289a91d315919cf757f293969a4244941ab10513d2351b4e83",
+                "sha256:3cef89888a466228fc4e4b2954e740ce8e9afde7c4315fdd18caa1b8de58ca17",
+                "sha256:4426c74761790bff46e3d906c14c7aab727543293eed5a924300a952e1a3a3c1",
+                "sha256:5024b8ed83d7f8809982d095d8ab0b179bebc07616a9713f86d30cf4944acb73",
+                "sha256:52c2bdd7cd0bf9d5ccdf9c1816568fd4ccd51a4d82419cc5480f548981b47dd0",
+                "sha256:54fa9fe27f5466b86126ff38123261188bed568c1019e4716af01f97a12fe812",
+                "sha256:5ba73aa3aca35d2981e0b31230d58abb7b5d7ca104e543ae49709208d8ce706a",
+                "sha256:5e16dcaecffd55b955aa5e2b8a804379789c15987e8ebd2f32f01398a81e975b",
+                "sha256:5ecfc6559132116dedfc482d0ad9df8a89dc5909eebffd22f3deb684132d002f",
+                "sha256:74153008bd24366cf099d1f1e83808d179d618c4e32edb0d489d526523a94d9f",
+                "sha256:78ec3c3412cf277e6252764ee4acbdbec6920cc87ad65862272aaa0e24381eee",
+                "sha256:795ad83940732b45d39b82571f87af0081c120feff2b12e748d96bb191169e33",
+                "sha256:7f716b6af94dc1b6b97c46401774472f0867e44595990fe80a8ba390f7a0a028",
+                "sha256:83dc89c5fd728fdb03b76f122f43b4dcee8c61f1489e232d9ad0f58020523e1c",
+                "sha256:8a0ae37576ed444fe853709bdceb2be4c7df6f7acae17b8378765bd28e61b3ae",
+                "sha256:8a8dbe2cb7f33ff54b16bb5c500673502a35f18ac1ed48625e997d40c922f9cc",
+                "sha256:8a9d899953c722b9afd7e88dbefd8fb276c686c3116a43c577cfabf636180558",
+                "sha256:8d0068e40837c1d0df6e3abf1cdc9a34a6d2611d90e29610fa1d2455aeb4e2e5",
+                "sha256:9347cc6822f38db2b1d1ce992f375289670e595a2d1c15961aacbe0977407dfc",
+                "sha256:9f335e5625feb90e323d7e3868ec337f7b9ad88b5d633f876e3b778813021dab",
+                "sha256:b03fd10a1709d0101c054883b550f7c4c5e974f751e2680318759af005964990",
+                "sha256:b0ca2c60d3966dfd6608f5f8c49b8a0fcf76de6654f2eda55fc6ef038d5a6f27",
+                "sha256:b2604c6450f9dd2c42e223b1f5dca9643a23cfecc9fde4a94bb38e0d2693b136",
+                "sha256:ca0e7a658fbafcddcaefaa07ba8dae9384be2343468a8e011061791588d839fa",
+                "sha256:d0e9ac04065a814d4cf2c6791a2ad563f739ae3ae830d716d54245c2b96fead6",
+                "sha256:d50e8c1e571ee39b5dfbc295c11ad65988879f68009dd281a6e1edbc2ff6c18c",
+                "sha256:d840adcad7354be6f2ec28d0706528b0026e4c3934cc6566b84eac18633eab1b",
+                "sha256:e0bbee6c2a5bf2a0017a9b5e397babb88f230e6f07c3cdff4a4c4bc75ed7c617",
+                "sha256:e5afe0a7ea0e3a7a257907060bee6724a6002b7eec55d0db16fd32409795f3e1",
+                "sha256:e68be81cd8c22b029924b6d0ee814c337c0e706b8d88495a617319e5dd5441c3",
+                "sha256:ec9be0f4826cdb3a3a517509dcc5f87f370251b76362051ab59e42b6b765f8c4",
+                "sha256:f04f97797df35e442ed09f529ad1235d1f1c0f30878e2fe09a2676b71a8801e0",
+                "sha256:f41e57ad63d336fe50d3a67bb8eaa26c09f6dda6a59f76777a99b8ccd8e26aec"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.6.1"
+            "version": "==3.6.2"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -867,11 +875,11 @@
         },
         "nbclassic": {
             "hashes": [
-                "sha256:1e0470583b55089c427940ed31b8a866ffef7ccab101494e409efe5ac7ba9897",
-                "sha256:d71d18aa6605eaf59e9b99b34c96360af45847f2a30ee2fefbe2f7bed4bc3df2"
+                "sha256:c74d8a500f8e058d46b576a41e5bc640711e1032cf7541dde5f73ea49497e283",
+                "sha256:cbf05df5842b420d5cece0143462380ea9d308ff57c2dc0eb4d6e035b18fbfb3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.4.7"
+            "version": "==0.4.8"
         },
         "nbclient": {
             "hashes": [
@@ -883,11 +891,11 @@
         },
         "nbconvert": {
             "hashes": [
-                "sha256:66326174c190dc4f0a6cbbff96f30c632774b441fa3c7565662bb3d41992fb0f",
-                "sha256:7ae7ccc68495b565dab153459ee7e65039970913eb115070da6e2c673cf0e9f8"
+                "sha256:c9c0e4b26326f7658ebf4cda0acc591b9727c4e3ee3ede962f70c11833b71b40",
+                "sha256:f933e82fe48b9a421e4252249f6c0a9a9940dc555642b4729f3f1f526bb16779"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.2.3"
+            "version": "==7.2.6"
         },
         "nbformat": {
             "hashes": [
@@ -956,11 +964,11 @@
         },
         "notebook-shim": {
             "hashes": [
-                "sha256:481711abddfb2e5305b83cf0efe18475824eb47d1ba9f87f66a8c574b8b5c9e4",
-                "sha256:fdb81febb05932c6d19e44e10382ce05469cac5e1b6e99b49be6159ddb5e4804"
+                "sha256:090e0baf9a5582ff59b607af523ca2db68ff216da0c69956b62cab2ef4fc9c3f",
+                "sha256:9c6c30f74c4fbea6fce55c1be58e7fd0409b1c681b075dcedceb005db5026949"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.2.0"
+            "version": "==0.2.2"
         },
         "numba": {
             "hashes": [
@@ -1053,11 +1061,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
-                "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+                "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3",
+                "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==21.3"
+            "markers": "python_version >= '3.7'",
+            "version": "==22.0"
         },
         "pandas": {
             "hashes": [
@@ -1127,6 +1135,7 @@
                 "sha256:2e0918e03aa0c72ea56edbb00d4d664294815aa11291a11504a377ea018330d3",
                 "sha256:3033fbe1feb1b59394615a1cafaee85e49d01b51d54de0cbf6aa8e64182518a1",
                 "sha256:3168434d303babf495d4ba58fc22d6604f6e2afb97adc6a423e917dab828939c",
+                "sha256:32a44128c4bdca7f31de5be641187367fe2a450ad83b833ef78910397db491aa",
                 "sha256:3dd6caf940756101205dffc5367babf288a30043d35f80936f9bfb37f8355b32",
                 "sha256:40e1ce476a7804b0fb74bcfa80b0a2206ea6a882938eaba917f7a0f004b42502",
                 "sha256:41e0051336807468be450d52b8edd12ac60bebaa97fe10c8b660f116e50b30e4",
@@ -1171,6 +1180,7 @@
                 "sha256:c7025dce65566eb6e89f56c9509d4f628fddcedb131d9465cacd3d8bac337e7e",
                 "sha256:c935a22a557a560108d780f9a0fc426dd7459940dc54faa49d83249c8d3e760f",
                 "sha256:dbb8e7f2abee51cef77673be97760abff1674ed32847ce04b4af90f610144c7b",
+                "sha256:e6ea6b856a74d560d9326c0f5895ef8050126acfdc7ca08ad703eb0081e82b74",
                 "sha256:ebf2029c1f464c59b8bdbe5143c79fa2045a581ac53679733d3a91d400ff9efb",
                 "sha256:f1ff2ee69f10f13a9596480335f406dd1f70c3650349e2be67ca3139280cade0"
             ],
@@ -1195,53 +1205,31 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:9696f386133df0fc8ca5af4895afe5d78f5fcfe5258111c2a79a1c3e41ffa96d",
-                "sha256:9ada952c9d1787f52ff6d5f3484d0b4df8952787c087edf6a1f7c2cb1ea88148"
+                "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63",
+                "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.31"
+            "version": "==3.0.36"
         },
         "psutil": {
             "hashes": [
-                "sha256:07d880053c6461c9b89cd5d4808f3b8336665fa3acdefd6777662c5ed73a851a",
-                "sha256:12500d761ac091f2426567f19f95fd3f15a197d96befb44a5c1e3cbe6db5752c",
-                "sha256:1b540599481c73408f6b392cdffef5b01e8ff7a2ac8caae0a91b8222e88e8f1e",
-                "sha256:35feafe232d1aaf35d51bd42790cbccb882456f9f18cdc411532902370d660df",
-                "sha256:3a7826e68b0cf4ce2c1ee385d64eab7d70e3133171376cac53d7c1790357ec8f",
-                "sha256:46907fa62acaac364fff0b8a9da7b360265d217e4fdeaca0a2397a6883dffba2",
-                "sha256:4bd4854f0c83aa84a5a40d3b5d0eb1f3c128f4146371e03baed4589fe4f3c931",
-                "sha256:538fcf6ae856b5e12d13d7da25ad67f02113c96f5989e6ad44422cb5994ca7fc",
-                "sha256:547ebb02031fdada635452250ff39942db8310b5c4a8102dfe9384ee5791e650",
-                "sha256:5e8b50241dd3c2ed498507f87a6602825073c07f3b7e9560c58411c14fe1e1c9",
-                "sha256:5fa88e3d5d0b480602553d362c4b33a63e0c40bfea7312a7bf78799e01e0810b",
-                "sha256:68fa227c32240c52982cb931801c5707a7f96dd8927f9102d6c7771ea1ff5698",
-                "sha256:6ced1ad823ecfa7d3ce26fe8aa4996e2e53fb49b7fed8ad81c80958501ec0619",
-                "sha256:71b1206e7909792d16933a0d2c1c7f04ae196186c51ba8567abae1d041f06dcb",
-                "sha256:767ef4fa33acda16703725c0473a91e1832d296c37c63896c7153ba81698f1ab",
-                "sha256:7ccfcdfea4fc4b0a02ca2c31de7fcd186beb9cff8207800e14ab66f79c773af6",
-                "sha256:7e4939ff75149b67aef77980409f156f0082fa36accc475d45c705bb00c6c16a",
-                "sha256:828c9dc9478b34ab96be75c81942d8df0c2bb49edbb481f597314d92b6441d89",
-                "sha256:8a4e07611997acf178ad13b842377e3d8e9d0a5bac43ece9bfc22a96735d9a4f",
-                "sha256:941a6c2c591da455d760121b44097781bc970be40e0e43081b9139da485ad5b7",
-                "sha256:9a4af6ed1094f867834f5f07acd1250605a0874169a5fcadbcec864aec2496a6",
-                "sha256:9ec296f565191f89c48f33d9544d8d82b0d2af7dd7d2d4e6319f27a818f8d1cc",
-                "sha256:9ec95df684583b5596c82bb380c53a603bb051cf019d5c849c47e117c5064395",
-                "sha256:a04a1836894c8279e5e0a0127c0db8e198ca133d28be8a2a72b4db16f6cf99c1",
-                "sha256:a3d81165b8474087bb90ec4f333a638ccfd1d69d34a9b4a1a7eaac06648f9fbe",
-                "sha256:b4a247cd3feaae39bb6085fcebf35b3b8ecd9b022db796d89c8f05067ca28e71",
-                "sha256:ba38cf9984d5462b506e239cf4bc24e84ead4b1d71a3be35e66dad0d13ded7c1",
-                "sha256:beb57d8a1ca0ae0eb3d08ccaceb77e1a6d93606f0e1754f0d60a6ebd5c288837",
-                "sha256:d266cd05bd4a95ca1c2b9b5aac50d249cf7c94a542f47e0b22928ddf8b80d1ef",
-                "sha256:d8c3cc6bb76492133474e130a12351a325336c01c96a24aae731abf5a47fe088",
-                "sha256:db8e62016add2235cc87fb7ea000ede9e4ca0aa1f221b40cef049d02d5d2593d",
-                "sha256:e7507f6c7b0262d3e7b0eeda15045bf5881f4ada70473b87bc7b7c93b992a7d7",
-                "sha256:ed15edb14f52925869250b1375f0ff58ca5c4fa8adefe4883cfb0737d32f5c02",
-                "sha256:f57d63a2b5beaf797b87024d018772439f9d3103a395627b77d17a8d72009543",
-                "sha256:fa5e32c7d9b60b2528108ade2929b115167fe98d59f89555574715054f50fa31",
-                "sha256:fe79b4ad4836e3da6c4650cb85a663b3a51aef22e1a829c384e18fae87e5e727"
+                "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff",
+                "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1",
+                "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62",
+                "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549",
+                "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08",
+                "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7",
+                "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e",
+                "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe",
+                "sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24",
+                "sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad",
+                "sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94",
+                "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8",
+                "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7",
+                "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.3"
+            "version": "==5.9.4"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -1371,31 +1359,31 @@
         },
         "pyrsistent": {
             "hashes": [
-                "sha256:06579d46d8ad69529b28f88711191a7fe7103c92d04a9f338dc754f71b92efa0",
-                "sha256:1d0620474d509172e1c50b79d5626bfe1899f174bf650186a50c6ce31289ff52",
-                "sha256:2032d971711643049b4f2c3ca5155a855d507d73bad26dac8d4349e5c5dd6758",
-                "sha256:2c641111c3f110379bb9001dbb26b34eb8cafab3d0fa855dc161c391461a4aab",
-                "sha256:327f99800d04a9abcf580daecfd6dd4bfdb4a7e61c71bf2cd1189ef1ca44bade",
-                "sha256:39f15ad754384e744ac8b00805913bfa66c41131faaa3e4c45c4af0731f3e8f6",
-                "sha256:4c58bd93c4d502f52938fccdbe6c9d70df3a585c6b39d900fab5f76b604282aa",
-                "sha256:62a41037387ae849a493cd945e22b34d167a843d57f75b07dbfad6d96cef485c",
-                "sha256:62b704f18526a8fc243152de8f3f40ae39c5172baff10f50c0c5d5331d6f2342",
-                "sha256:6df99c3578dc4eb33f3eb26bc28277ab40a720b71649d940bff9c1f704377772",
-                "sha256:6ef7430e45c5fa0bb6c361cada4a08ed9c184b5ed086815a85c3bc8c5054566b",
-                "sha256:73b2db09fe15b6e444c0bd566a125a385ca6493456224ce8b367d734f079f576",
-                "sha256:73d4ec2997716af3c8f28f7e3d3a565d273a598982d2fe95639e07ce4db5da45",
-                "sha256:73e3e2fd9da009d558050697cc22ad689f89a14a2ef2e67304628a913e59c947",
-                "sha256:890f577aec554f142e01daf890221d10e4f93a9b1107998d631d3f075b55e8f8",
-                "sha256:8a34a2a8b220247658f7ced871197c390b3a6371d796a5869ab1c62abe0be527",
-                "sha256:8bc23e9ddcb523c3ffb4d712aa0bd5bc67b34ff4e2b23fb557012171bdb4013a",
-                "sha256:945297fc344fef4d540135180ce7babeb2291d124698cc6282f3eac624aa5e82",
-                "sha256:aaa869d9199d7d4c70a57678aff21654cc179c0c32bcfde87f1d65d0ff47e520",
-                "sha256:bc33fc20ddfd89b86b7710142963490d8c4ee8307ed6cc5e189a58fa72390eb9",
-                "sha256:cfe6d8b293d123255fd3b475b5f4e851eb5cbaee2064c8933aa27344381744ae",
-                "sha256:d16ac5ab3d9db78fed40c884d67079524e4cf8276639211ad9e6fa73e727727e"
+                "sha256:055ab45d5911d7cae397dc418808d8802fb95262751872c841c170b0dbf51eed",
+                "sha256:111156137b2e71f3a9936baf27cb322e8024dac3dc54ec7fb9f0bcf3249e68bb",
+                "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a",
+                "sha256:21455e2b16000440e896ab99e8304617151981ed40c29e9507ef1c2e4314ee95",
+                "sha256:2aede922a488861de0ad00c7630a6e2d57e8023e4be72d9d7147a9fcd2d30712",
+                "sha256:3ba4134a3ff0fc7ad225b6b457d1309f4698108fb6b35532d015dca8f5abed73",
+                "sha256:456cb30ca8bff00596519f2c53e42c245c09e1a4543945703acd4312949bfd41",
+                "sha256:71d332b0320642b3261e9fee47ab9e65872c2bd90260e5d225dabeed93cbd42b",
+                "sha256:879b4c2f4d41585c42df4d7654ddffff1239dc4065bc88b745f0341828b83e78",
+                "sha256:9cd3e9978d12b5d99cbdc727a3022da0430ad007dacf33d0bf554b96427f33ab",
+                "sha256:a178209e2df710e3f142cbd05313ba0c5ebed0a55d78d9945ac7a4e09d923308",
+                "sha256:b39725209e06759217d1ac5fcdb510e98670af9e37223985f330b611f62e7425",
+                "sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2",
+                "sha256:bfd880614c6237243ff53a0539f1cb26987a6dc8ac6e66e0c5a40617296a045e",
+                "sha256:c43bec251bbd10e3cb58ced80609c5c1eb238da9ca78b964aea410fb820d00d6",
+                "sha256:d690b18ac4b3e3cab73b0b7aa7dbe65978a172ff94970ff98d82f2031f8971c2",
+                "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a",
+                "sha256:dec3eac7549869365fe263831f576c8457f6c833937c68542d08fde73457d291",
+                "sha256:e371b844cec09d8dc424d940e54bba8f67a03ebea20ff7b7b0d56f526c71d584",
+                "sha256:e5d8f84d81e3729c3b506657dddfe46e8ba9c330bf1858ee33108f8bb2adb38a",
+                "sha256:ea6b79a02a28550c98b6ca9c35b9f492beaa54d7c5c9e9949555893c8a9234d0",
+                "sha256:f1258f4e6c42ad0b20f9cfcc3ada5bd6b83374516cd01c0960e3cb75fdca6770"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.19.1"
+            "version": "==0.19.2"
         },
         "python-dateutil": {
             "hashes": [
@@ -1415,10 +1403,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22",
-                "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914"
+                "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427",
+                "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"
             ],
-            "version": "==2022.5"
+            "version": "==2022.6"
         },
         "pyzmq": {
             "hashes": [
@@ -1502,19 +1490,19 @@
         },
         "qtconsole": {
             "hashes": [
-                "sha256:8eadf012e83ab018295803c247c6ab7eacd3d5ab1e1d88a0f37fdcfdab9295a3",
-                "sha256:c29d24464f57cdbaa17d6f6060be6e6d5e29126e7feb57eebc1747433382b3d1"
+                "sha256:57748ea2fd26320a0b77adba20131cfbb13818c7c96d83fafcb110ff55f58b35",
+                "sha256:be13560c19bdb3b54ed9741a915aa701a68d424519e8341ac479a91209e694b2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.2"
+            "version": "==5.4.0"
         },
         "qtpy": {
             "hashes": [
-                "sha256:268cf5328f41353be1b127e04a81bc74ec9a9b54c9ac75dd8fe0ff48d8ad6ead",
-                "sha256:7d5231133b772e40b4ee514b6673aca558331e4b88ca038b26c9e16c5c95524f"
+                "sha256:0603c9c83ccc035a4717a12908bf6bc6cb22509827ea2ec0e94c2da7c9ed57c5",
+                "sha256:8d6d544fc20facd27360ea189592e6135c614785f0dec0b4f083289de6beb408"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.2.1"
+            "version": "==2.3.0"
         },
         "rasterio": {
             "hashes": [
@@ -1619,11 +1607,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17",
-                "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"
+                "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54",
+                "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==65.5.0"
+            "version": "==65.6.3"
         },
         "shapely": {
             "hashes": [
@@ -1652,70 +1640,70 @@
         },
         "simplejson": {
             "hashes": [
-                "sha256:04e31fa6ac8e326480703fb6ded1488bfa6f1d3f760d32e29dbf66d0838982ce",
-                "sha256:068670af975247acbb9fc3d5393293368cda17026db467bf7a51548ee8f17ee1",
-                "sha256:07ecaafc1b1501f275bf5acdee34a4ad33c7c24ede287183ea77a02dc071e0c0",
-                "sha256:0b4126cac7d69ac06ff22efd3e0b3328a4a70624fcd6bca4fc1b4e6d9e2e12bf",
-                "sha256:0de783e9c2b87bdd75b57efa2b6260c24b94605b5c9843517577d40ee0c3cc8a",
-                "sha256:12133863178a8080a3dccbf5cb2edfab0001bc41e5d6d2446af2a1131105adfe",
-                "sha256:1c9b1ed7ed282b36571638297525f8ef80f34b3e2d600a56f962c6044f24200d",
-                "sha256:23fe704da910ff45e72543cbba152821685a889cf00fc58d5c8ee96a9bad5f94",
-                "sha256:28221620f4dcabdeac310846629b976e599a13f59abb21616356a85231ebd6ad",
-                "sha256:35a49ebef25f1ebdef54262e54ae80904d8692367a9f208cdfbc38dbf649e00a",
-                "sha256:37bc0cf0e5599f36072077e56e248f3336917ded1d33d2688624d8ed3cefd7d2",
-                "sha256:3fe87570168b2ae018391e2b43fbf66e8593a86feccb4b0500d134c998983ccc",
-                "sha256:3ff5b3464e1ce86a8de8c88e61d4836927d5595c2162cab22e96ff551b916e81",
-                "sha256:401d40969cee3df7bda211e57b903a534561b77a7ade0dd622a8d1a31eaa8ba7",
-                "sha256:4b6bd8144f15a491c662f06814bd8eaa54b17f26095bb775411f39bacaf66837",
-                "sha256:4c09868ddb86bf79b1feb4e3e7e4a35cd6e61ddb3452b54e20cf296313622566",
-                "sha256:4d1c135af0c72cb28dd259cf7ba218338f4dc027061262e46fe058b4e6a4c6a3",
-                "sha256:4ff4ac6ff3aa8f814ac0f50bf218a2e1a434a17aafad4f0400a57a8cc62ef17f",
-                "sha256:521877c7bd060470806eb6335926e27453d740ac1958eaf0d8c00911bc5e1802",
-                "sha256:522fad7be85de57430d6d287c4b635813932946ebf41b913fe7e880d154ade2e",
-                "sha256:5540fba2d437edaf4aa4fbb80f43f42a8334206ad1ad3b27aef577fd989f20d9",
-                "sha256:5d6b4af7ad7e4ac515bc6e602e7b79e2204e25dbd10ab3aa2beef3c5a9cad2c7",
-                "sha256:5decdc78849617917c206b01e9fc1d694fd58caa961be816cb37d3150d613d9a",
-                "sha256:632ecbbd2228575e6860c9e49ea3cc5423764d5aa70b92acc4e74096fb434044",
-                "sha256:65b998193bd7b0c7ecdfffbc825d808eac66279313cb67d8892bb259c9d91494",
-                "sha256:67093a526e42981fdd954868062e56c9b67fdd7e712616cc3265ad0c210ecb51",
-                "sha256:681eb4d37c9a9a6eb9b3245a5e89d7f7b2b9895590bb08a20aa598c1eb0a1d9d",
-                "sha256:69bd56b1d257a91e763256d63606937ae4eb890b18a789b66951c00062afec33",
-                "sha256:724c1fe135aa437d5126138d977004d165a3b5e2ee98fc4eb3e7c0ef645e7e27",
-                "sha256:7255a37ff50593c9b2f1afa8fafd6ef5763213c1ed5a9e2c6f5b9cc925ab979f",
-                "sha256:743cd768affaa508a21499f4858c5b824ffa2e1394ed94eb85caf47ac0732198",
-                "sha256:80d3bc9944be1d73e5b1726c3bbfd2628d3d7fe2880711b1eb90b617b9b8ac70",
-                "sha256:82ff356ff91be0ab2293fc6d8d262451eb6ac4fd999244c4b5f863e049ba219c",
-                "sha256:8e8607d8f6b4f9d46fee11447e334d6ab50e993dd4dbfb22f674616ce20907ab",
-                "sha256:97202f939c3ff341fc3fa84d15db86156b1edc669424ba20b0a1fcd4a796a045",
-                "sha256:9b01e7b00654115965a206e3015f0166674ec1e575198a62a977355597c0bef5",
-                "sha256:9fa621b3c0c05d965882c920347b6593751b7ab20d8fa81e426f1735ca1a9fc7",
-                "sha256:a1aa6e4cae8e3b8d5321be4f51c5ce77188faf7baa9fe1e78611f93a8eed2882",
-                "sha256:a2d30d6c1652140181dc6861f564449ad71a45e4f165a6868c27d36745b65d40",
-                "sha256:a649d0f66029c7eb67042b15374bd93a26aae202591d9afd71e111dd0006b198",
-                "sha256:a7854326920d41c3b5d468154318fe6ba4390cb2410480976787c640707e0180",
-                "sha256:a89acae02b2975b1f8e4974cb8cdf9bf9f6c91162fb8dec50c259ce700f2770a",
-                "sha256:a8bbdb166e2fb816e43ab034c865147edafe28e1b19c72433147789ac83e2dda",
-                "sha256:ac786f6cb7aa10d44e9641c7a7d16d7f6e095b138795cd43503769d4154e0dc2",
-                "sha256:b09bc62e5193e31d7f9876220fb429ec13a6a181a24d897b9edfbbdbcd678851",
-                "sha256:b10556817f09d46d420edd982dd0653940b90151d0576f09143a8e773459f6fe",
-                "sha256:b81076552d34c27e5149a40187a8f7e2abb2d3185576a317aaf14aeeedad862a",
-                "sha256:bdfc54b4468ed4cd7415928cbe782f4d782722a81aeb0f81e2ddca9932632211",
-                "sha256:cf6e7d5fe2aeb54898df18db1baf479863eae581cce05410f61f6b4188c8ada1",
-                "sha256:cf98038d2abf63a1ada5730e91e84c642ba6c225b0198c3684151b1f80c5f8a6",
-                "sha256:d24a9e61df7a7787b338a58abfba975414937b609eb6b18973e25f573bc0eeeb",
-                "sha256:d74ee72b5071818a1a5dab47338e87f08a738cb938a3b0653b9e4d959ddd1fd9",
-                "sha256:dd16302d39c4d6f4afde80edd0c97d4db643327d355a312762ccd9bd2ca515ed",
-                "sha256:dd2fb11922f58df8528adfca123f6a84748ad17d066007e7ac977720063556bd",
-                "sha256:deac4bdafa19bbb89edfb73b19f7f69a52d0b5bd3bb0c4ad404c1bbfd7b4b7fd",
-                "sha256:e03c3b8cc7883a54c3f34a6a135c4a17bc9088a33f36796acdb47162791b02f6",
-                "sha256:e1ec8a9ee0987d4524ffd6299e778c16cc35fef6d1a2764e609f90962f0b293a",
-                "sha256:e8603e691580487f11306ecb066c76f1f4a8b54fb3bdb23fa40643a059509366",
-                "sha256:f444762fed1bc1fd75187ef14a20ed900c1fbb245d45be9e834b822a0223bc81",
-                "sha256:f63600ec06982cdf480899026f4fda622776f5fabed9a869fdb32d72bc17e99a",
-                "sha256:fb62d517a516128bacf08cb6a86ecd39fb06d08e7c4980251f5d5601d29989ba"
+                "sha256:002f069c7bb9a86826616a78f1214fea5b993435720990eecb0bf10955b9cd0e",
+                "sha256:00b673f0b3caf37a3d993bccf30a97290da6313b6ecc7d66937e9cd906d8f840",
+                "sha256:07e408222931b1a2aab71e60e5f169fa7c0d74cacd4e0a6a0199716cb18dad76",
+                "sha256:0de746c8f76355c79fd15eccd7ecde0b137cd911bdcdc463fc5c36ec3d8b98ea",
+                "sha256:0f33d16fa7b5e2ed6ea85d7b31bc84cf8c73c40cc2c9f87071e0fffcd52f5342",
+                "sha256:0f49858b5fc802081b71269f4a3aa5c5500ec6553637c9a0630f30a2a6541ea7",
+                "sha256:17dbc7f71fa5b7e4a2acef38cf0be30461ae6659456a978ce7eeebeb5bdf9e1a",
+                "sha256:17ec5e408fb6615250c1f18fb4eac3b2b99a85e8613bfc2dfa54827d0bf7f3e1",
+                "sha256:1b4085151e00ab7ca66f269aff7153f0ec18589cb22e7ceb8b365709c723fdd0",
+                "sha256:1f169402069f8cf93e359f607725b1d920c4dbe5bda4c520025d5fad8d20c1b7",
+                "sha256:1fbacdbba3cf5a471c67a9ca6cd270bba9578d5bc22aef6028faebbdb98bbb15",
+                "sha256:252f7cc5524bb5507a08377a4a75aa7ff4645f3dfca814d38bdbcf0f3c34d1ce",
+                "sha256:2aeed35db00cdf5d49ff1e7d878afd38c86a5fead0f1d364d539ad4d7a869e0e",
+                "sha256:2cc76435569e6c19574a8e913cfccbed832249b2b3b360caee9a4caf8ff866bf",
+                "sha256:448ab14fa67b3ac235a8445d14ec6d56268c3dabbce78720f9efa6d698466710",
+                "sha256:4609feb2ae66c132c6dcbe01dbfd4f6431afb4ff17303e37ca128fb6297cebd2",
+                "sha256:46bafa7e794f0e91fde850d906b0dc29a624c726b27e75d23bc8c3e35a48f28b",
+                "sha256:4a6199d302ec7d889e1aa6b493aa8e40b4dfa4bd85708f8c8f0c64ce5b8e0986",
+                "sha256:4d8d016f70d241f82189bc9f6d1eb8558b3599861f2c501b3f32da7fdf4e92ac",
+                "sha256:503da91993cc671fe7ebbf120c3ce868278de8226f158336afde874f7b7aa871",
+                "sha256:54c63cc7857f16a20aa170ffda9ebce45a3b7ba764b67a5a95bfe7ae613a2710",
+                "sha256:58a429d2c2fa80834115b923ff689622de8f214cf0dc4afa9f59e824b444ab31",
+                "sha256:599e9c53d3203bc36ef68efec138ca76d201da7ac06a114fae78536a8c10e35b",
+                "sha256:5f3dd31309ae5cc9f2df51d2d5cac89722dac3c853042ebefcaf7ad06ca19387",
+                "sha256:6187cbea7fdede732fe0347ad08cd920ebd9faa30b6c48782cee494051ca97c6",
+                "sha256:622cf0e1f870f189a0757fdcad7998a0c1dd46b0e53aeac9960556c141319c83",
+                "sha256:638bdd2deaccd3b8e02b1783280bd82341df5e1faa59c4f0276f03f16eec13ea",
+                "sha256:6804ad50aaf581df5c982fc101b0d932638066fe191074ded783602eb1c8982a",
+                "sha256:7a4d9b266ae6db578719f1255c742e76ee4676593087f4f6b79a2bbae2b1dcc5",
+                "sha256:7a9476dcd72aeba7d55c4800b9cd2204201af3539894b8512d74597e35a3033a",
+                "sha256:7b95c5cf71c16e4fdaa724719aaf8ccbed533e2df57a20bcff825ceeead27688",
+                "sha256:8493d2c1a940471b07d7c9c356a3f4eee780df073da2917418d0fe8669b54f99",
+                "sha256:875cfb43b622672218045dc927a86fc7c4c8111264c1d303aca5de33d5df479e",
+                "sha256:8d762267c4af617e1798bd0151f626105d06a88f214e3874b77eb89106f899fe",
+                "sha256:94c17d01e4c65e63deec46c984bb810de5e3a1259eb6bacdca63f3efc9c4c673",
+                "sha256:96979ff7f0daf47422d5f95d2d006da3210e0490a166bce2529f59f55047fc67",
+                "sha256:97139bf5134d713710665a6edb9500d69b93642c4b6b44b20800232dbd0f5b39",
+                "sha256:989b31d586954e65170ad3ec597218a6790c401b82da6193e8a897a06aa7946e",
+                "sha256:98b4c824f15436f1b22fe6d73c42ffacb246f7efc4d9dbbee542dd72355ecc43",
+                "sha256:9aff3c24017a7819c76b2f177d4fe8334b3d4cb6f702a2d7c666b3d57c36ffb4",
+                "sha256:9db78e18624f94d7b5642bf487244f803dab844e771d92e83f85f22da21ffe2d",
+                "sha256:a0e6dd5a0b8c76fb7522470789f1af793d39d6edbd4e40853e7be550ad49c430",
+                "sha256:a2f70d8170c7e02166a4c91462581e6ae5f35e3351a6b6c5142adcb04c7153ac",
+                "sha256:a814227fa08cae435ac7a42dcd2a04a7ec4a3cee23b7f83f9544cd26f452dcc4",
+                "sha256:aa9ecdd1d7ecbc7d1066c37cfbe52f65adf64b11b22d481a98fe1d3675dfff4b",
+                "sha256:b2b19d7aa4e9a1e7bf8caaf5f478a790190c60136314f45bb7702cb5a9337266",
+                "sha256:b4997bd8332cef3923402a07351571788f552f55ea1394ffbfccd4d203a8a05f",
+                "sha256:b71fef8ee41d59509c7f4afac7f627ed143c9e6db9eb08cfbba85e4c4dc5e67b",
+                "sha256:bd67d6fad7f4cd7c9cb7fad32d78ce32862fdb574b898447987a5de22fd37d73",
+                "sha256:ca22993a1a00440392c6c76f39addab8d97c706d2a8bcc2c9b2b6cb2cd7f41df",
+                "sha256:ce1c0580372d3c9bfa151bd0721a9bd5647b9b2245d0588d813fdbd2eb5d6f22",
+                "sha256:d522f28f7b252454df86ac3db5a0e1fe5ae03c8fc0cd1592c912b07c9fad6c29",
+                "sha256:d5d25cc5dad31a10d7a8196125515cc3aa68187c8953459fcaf127c2c8410f51",
+                "sha256:d9f7a692c11de20cb8ec680584815315e03d1404a6e299d36489b0fb6447d98d",
+                "sha256:d9fa2ad4cabb5054faa8d4a44b84134b0ec9d1421f5e9264d057d6be4d13c7fa",
+                "sha256:db53a85f4db0dbd9e5f6277d9153bcaa2ccb87b0d672c6a35f19432b3f2301a3",
+                "sha256:db9d36c4c7997c2a2513a5d218fd90b53bfeaf7e727f94aaf3576973378b3bce",
+                "sha256:e80f02e68d25c222471fcc5d1933275b8eb396e5e40b7863e4e0a43b3c810059",
+                "sha256:e84bd1c29e83ec74a95de070473742eb52d08502f2428eff5751671081e0a0a6",
+                "sha256:f0e12bdafdf7e32c5ad4a073e325ea0d659d4277af8b3d8eccf3101c56879619",
+                "sha256:fd56a9e0c63a1f9c37621fe298c77795aefd2a26dca80dcae27688586c40b4bb"
             ],
             "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.17.6"
+            "version": "==3.18.0"
         },
         "six": {
             "hashes": [
@@ -1750,10 +1738,10 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:8e515439f818efaa251036af72d89e4026e2b03993f3453c000b200fb4f2d6aa",
-                "sha256:b92d206ef355a367d14316b786ab41cb99eb453a21f2cb216a4204625ff7bc07"
+                "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815",
+                "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"
             ],
-            "version": "==0.6.0"
+            "version": "==0.6.2"
         },
         "tables": {
             "hashes": [
@@ -1787,11 +1775,11 @@
         },
         "terminado": {
             "hashes": [
-                "sha256:520feaa3aeab8ad64a69ca779be54be9234edb2d0d6567e76c93c2c9a4e6e43f",
-                "sha256:bf6fe52accd06d0661d7611cc73202121ec6ee51e46d8185d489ac074ca457c2"
+                "sha256:6ccbbcd3a4f8a25a5ec04991f39a0b8db52dfcd487ea0e578d977e6752380333",
+                "sha256:8650d44334eba354dd591129ca3124a6ba42c3d5b70df5051b6921d506fdaeae"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.17.0"
+            "version": "==0.17.1"
         },
         "tinycss2": {
             "hashes": [
@@ -1836,19 +1824,19 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:1201b2c9f76097195989cdf7f65db9897593b0dfd69e4ac96016661bb6f0d30f",
-                "sha256:b122f9ff2f2f6c1709dab289a05555be011c87828e911c0cf4074b85cb780a79"
+                "sha256:61832ea7b7f910f5745e27e9bb269a181fd15af76027d99560299209d5b17c94",
+                "sha256:bd0fca5c890a09bf66b33cce67ca14156b080429bc39c7ef26b075a4bd4f9fc3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.5.0"
+            "version": "==5.7.0"
         },
         "urllib3": {
             "hashes": [
-                "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
-                "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
+                "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc",
+                "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
-            "version": "==1.26.12"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==1.26.13"
         },
         "wcwidth": {
             "hashes": [
@@ -1866,19 +1854,19 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:398909eb7e261f44b8f4bd474785b6ec5f5b499d4953342fe9755e01ef624090",
-                "sha256:f9611eb65c8241a67fb373bef040b3cf8ad377a9f6546a12b620b6511e8ea9ef"
+                "sha256:d6b06432f184438d99ac1f456eaf22fe1ade524c3dd16e661142dc54e9cba574",
+                "sha256:d6e8f90ca8e2dd4e8027c4561adeb9456b54044312dba655e7cae652ceb9ae59"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.4.1"
+            "version": "==1.4.2"
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:34824864c062b0b3030ad78210db5ae6a3960dfb61d5b27562d6631774de0286",
-                "sha256:7f3b0de8fda692d31ef03743b598620e31c2668b835edbd3962d080ccecf31eb"
+                "sha256:44c69f18237af0f610557d6c1c7ef76853f5856a0e604c0a517f2320566bb775",
+                "sha256:fa0e840719ec95dd2ec85c3a48913f1a0c29d323eacbcdb0b29bfed0cc6da678"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.0.3"
+            "version": "==4.0.4"
         },
         "xarray": {
             "hashes": [
@@ -1890,20 +1878,20 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1",
-                "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"
+                "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa",
+                "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.10.0"
+            "version": "==3.11.0"
         }
     },
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:1b28ed85e254b724439afc783d4bee767f780b936c3fe8b3275332f42cf5f561",
-                "sha256:4aa76401a151c8cc572d906aad7aea2a841780834a19d780f4321c0fe1b54635"
+                "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
+                "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"
             ],
-            "version": "==2.1.0"
+            "version": "==2.2.1"
         },
         "backcall": {
             "hashes": [
@@ -1929,19 +1917,19 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:7c959e3dedbf7ed81f9b9d8833df252c430610e2a4a6464ec13cd20975ce20a5",
-                "sha256:91ef03016bcf72dd17190f863476e7c799c6126ec7e8be97719d1bc9a78a59a4"
+                "sha256:352042ddcb019f7c04e48171b4dd78e4c4bb67bf97030d170e154aac42b656d9",
+                "sha256:882899fe78d5417a0aa07f995db298fa28b58faeba2112d2e3a4c95fe14bb738"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.6.0"
+            "version": "==8.7.0"
         },
         "jedi": {
             "hashes": [
-                "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d",
-                "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab"
+                "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
+                "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.1"
+            "version": "==0.18.2"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -1976,11 +1964,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:9696f386133df0fc8ca5af4895afe5d78f5fcfe5258111c2a79a1c3e41ffa96d",
-                "sha256:9ada952c9d1787f52ff6d5f3484d0b4df8952787c087edf6a1f7c2cb1ea88148"
+                "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63",
+                "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.31"
+            "version": "==3.0.36"
         },
         "ptyprocess": {
             "hashes": [
@@ -2015,18 +2003,18 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:8e515439f818efaa251036af72d89e4026e2b03993f3453c000b200fb4f2d6aa",
-                "sha256:b92d206ef355a367d14316b786ab41cb99eb453a21f2cb216a4204625ff7bc07"
+                "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815",
+                "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"
             ],
-            "version": "==0.6.0"
+            "version": "==0.6.2"
         },
         "traitlets": {
             "hashes": [
-                "sha256:1201b2c9f76097195989cdf7f65db9897593b0dfd69e4ac96016661bb6f0d30f",
-                "sha256:b122f9ff2f2f6c1709dab289a05555be011c87828e911c0cf4074b85cb780a79"
+                "sha256:61832ea7b7f910f5745e27e9bb269a181fd15af76027d99560299209d5b17c94",
+                "sha256:bd0fca5c890a09bf66b33cce67ca14156b080429bc39c7ef26b075a4bd4f9fc3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.5.0"
+            "version": "==5.7.0"
         },
         "wcwidth": {
             "hashes": [

From 8f199d3a3978cc4b973db4dfaf8dc7d0b7d2e5b0 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Thu, 22 Dec 2022 10:55:08 -0500
Subject: [PATCH 186/359] Bug fix to inundation.py ignoring nodata

---
 docs/CHANGELOG.md   | 10 ++++++++++
 tools/inundation.py |  8 +++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 265765d1b..82b390e3e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.13.2 - 2022-12-20 - [PR #767](https://github.com/NOAA-OWP/inundation-mapping/pull/767)
+
+Fixes inundation of nodata areas of REM.
+
+### Changes
+
+- `tools/inundation.py`: Assigns depth a value of `0` if REM is less than `0`
+
+<br/><br/>
+
 ## v4.0.13.1 - 2022-12-09 - [PR #743](https://github.com/NOAA-OWP/inundation-mapping/pull/743)
 
 This merge adds the tools required to generate Alpha metrics by hydroid. It summarizes the Apha metrics by branch 0 catchment for use in the Hydrovis "FIM Performance" service.
diff --git a/tools/inundation.py b/tools/inundation.py
index 93031adaf..e2e947d08 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -365,9 +365,11 @@ def __go_fast_mapping(rem,catchments,catchmentStagesDict,inundation,depths):
 
     for i,(r,cm) in enumerate(zip(rem,catchments)):
         if cm in catchmentStagesDict:
-
-            depth = catchmentStagesDict[cm] - r
-            depths[i] = max(depth,0) # set negative depths to 0
+            if r >= 0:
+                depth = catchmentStagesDict[cm] - r
+                depths[i] = max(depth,0) # set negative depths to 0
+            else:
+                depths[i] = 0
 
             if depths[i] > 0: # set positive depths to positive
                 inundation[i] *= -1

From b4528ba5c546e99b73ceb81a98c71a47b70c3e91 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 23 Dec 2022 14:20:53 -0500
Subject: [PATCH 187/359] Mask levee-protected areas from DEM

---
 docs/CHANGELOG.md                            | 12 +++++
 src/gms/delineate_hydros_and_produce_HAND.sh |  9 ++++
 src/gms/mask_dem.py                          | 46 ++++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 src/gms/mask_dem.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 82b390e3e..786a02260 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.14.0 - 2022-12-20 - [PR #769](https://github.com/NOAA-OWP/inundation-mapping/pull/769)
+
+Masks levee-protected areas from the DEM in branch 0 and in highest two stream order branches.
+
+### Additions
+
+- `src/gms/`
+    - `mask_dem.py`: Masks levee-protected areas from the DEM in branch 0 and in highest two stream order branches
+    - `delineate_hydros_and_produce_HAND.sh`: Adds `src/gms/mask_dem.py`
+
+<br/><br/>
+
 ## v4.0.13.2 - 2022-12-20 - [PR #767](https://github.com/NOAA-OWP/inundation-mapping/pull/767)
 
 Fixes inundation of nodata areas of REM.
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
index bb2939e03..96e9ae5bd 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -6,6 +6,15 @@ level=$1
 ## INITIALIZE TOTAL TIME TIMER ##
 T_total_start
 
+## MASK LEVEE-PROTECTED AREAS FROM DEM ##
+if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
+    echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"$stopDiv
+    date -u
+    Tstart
+    python3 -m memory_profiler $srcDir/gms/mask_dem.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif  -s $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -i $current_branch_id -b0 $branch_zero_id
+    Tcount
+fi
+
 ## D8 FLOW ACCUMULATIONS ##
 echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"$stopDiv
 date -u
diff --git a/src/gms/mask_dem.py b/src/gms/mask_dem.py
new file mode 100644
index 000000000..d8916c2fb
--- /dev/null
+++ b/src/gms/mask_dem.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+import geopandas as gpd
+import fiona
+import rasterio as rio
+from rasterio.mask import mask
+import argparse
+from utils.shared_functions import mem_profile
+
+@mem_profile
+def mask_dem(dem_filename, nld_filename, out_dem_filename, stream_layer, branch_id_attribute, branch_id, order_attribute, branch_zero_id):
+    """
+    Masks levee-protected areas from DEM in branch 0 or if stream order is at least the minimum order (max - 1)
+    """
+    streams_df = gpd.read_file(stream_layer, ignore_geometry=True)
+
+    # Rasterize if branch zero or if stream order is at least the minimum order (max - 1)
+    if (branch_id == branch_zero_id) or (streams_df.loc[streams_df[branch_id_attribute].astype(int)==branch_id, order_attribute].max() >= streams_df[order_attribute].max() - 1):
+
+        with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
+            dem_data = dem.read(1)
+            dem_profile = dem.profile.copy()
+
+            geoms = [feature["geometry"] for feature in leveed]
+
+            # Mask out levee-protected areas from DEM
+            out_dem_masked, _ = mask(dem, geoms, invert=True)
+
+            with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
+                dest.write(out_dem_masked[0,:,:], indexes=1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Mask levee-protected areas from DEM')
+    parser.add_argument('-dem','--dem-filename', help='DEM filename', required=True,type=str)
+    parser.add_argument('-nld','--nld-filename', help='NLD filename', required=True,type=str)
+    parser.add_argument('-out','--out-dem-filename', help='out DEM filename', required=True,type=str)
+    parser.add_argument('-s', '--stream-layer', help='Stream layer filename', required=True)
+    parser.add_argument('-b', '--branch-id-attribute', help='Branch ID attribute name', required=False, default='levpa_id')
+    parser.add_argument('-i', '--branch-id', help='Branch ID', type=int, required='True')
+    parser.add_argument('-a', '--order-attribute', help='Stream order attribute name', required=False, default='order_')
+    parser.add_argument('-b0', '--branch-zero-id', help='Branch zero ID', type=int, required=False, default=0)
+
+    args = vars(parser.parse_args())
+
+    mask_dem(**args)

From 0355e2c639f7f1111aaa1855cec38f78585142cd Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 23 Dec 2022 14:36:03 -0500
Subject: [PATCH 188/359] Clip WBD to 3DEP DEM domain

---
 data/usgs/acquire_and_preprocess_3dep_dems.py | 50 +++++++++++++++++++
 docs/CHANGELOG.md                             | 13 +++++
 src/clip_vectors_to_wbd.py                    |  6 +++
 src/gms/run_by_unit.sh                        |  1 +
 4 files changed, 70 insertions(+)

diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index dbff22924..a8be1252b 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -7,6 +7,7 @@
 import subprocess
 import sys
 import traceback
+import geopandas as gpd
 
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 from datetime import datetime
@@ -104,6 +105,8 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
    
     # download dems, setting projection, block size, etc
     __download_usgs_dems(extent_file_names, target_output_folder_path, number_of_jobs, retry)
+
+    polygonize(target_output_folder_path)
     
     end_time = datetime.now()
     fh.print_end_header('Loading 3dep dems', start_time, end_time)
@@ -273,6 +276,53 @@ def download_usgs_dem_file(extent_file,
         logging.info(msg)
 
 
+def polygonize(target_output_folder_path):
+    """
+    Create a polygon of 3DEP domain from individual HUC6 DEMS which are then dissolved into a single polygon
+    """
+    dem_domain_file = os.path.join(target_output_folder_path, 'HUC6_dem_domain.gpkg')
+
+    msg = f" - Polygonizing -- {dem_domain_file} - Started"
+    print(msg)
+    logging.info(msg)
+            
+    dem_files = glob.glob(os.path.join(target_output_folder_path, '*_dem.tif'))
+    dem_gpkgs = gpd.GeoDataFrame()
+
+    for n, dem_file in enumerate(dem_files):
+        edge_tif = f'{os.path.splitext(dem_file)[0]}_edge.tif'
+        edge_gpkg = f'{os.path.splitext(edge_tif)[0]}.gpkg'
+
+        # Calculate a constant valued raster from valid DEM cells
+        if not os.path.exists(edge_tif):
+            subprocess.run(['gdal_calc.py', '-A', dem_file, f'--outfile={edge_tif}', '--calc=where(A > -900, 1, 0)', '--co', 'BIGTIFF=YES', '--co', 'NUM_THREADS=ALL_CPUS', '--co', 'TILED=YES', '--co', 'COMPRESS=LZW', '--co', 'SPARSE_OK=TRUE', '--type=Byte', '--quiet'])
+
+        # Polygonize constant valued raster
+        subprocess.run(['gdal_polygonize.py', '-8', edge_tif, '-q', '-f', 'GPKG', edge_gpkg])
+
+        gdf = gpd.read_file(edge_gpkg)
+
+        if n == 0:
+            dem_gpkgs = gdf
+        else:
+            dem_gpkgs = dem_gpkgs.append(gdf)
+
+        os.remove(edge_tif)
+        
+    dem_gpkgs['DN'] = 1
+    dem_dissolved = dem_gpkgs.dissolve(by='DN')
+    dem_dissolved.to_file(dem_domain_file, driver='GPKG')
+
+    if not os.path.exists(dem_domain_file):
+        msg = f" - Polygonizing -- {dem_domain_file} - Failed"
+        print(msg)
+        logging.error(msg)
+    else:
+        msg = f" - Polygonizing -- {dem_domain_file} - Complete"
+        print(msg)
+        logging.info(msg)
+
+
 def __setup_logger(output_folder_path):
 
     start_time = datetime.now()
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 786a02260..0b95bae1d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.14.1 - 2022-12-03 - [PR #753](https://github.com/NOAA-OWP/inundation-mapping/pull/753)
+
+Creates a polygon of 3DEP DEM domain (to eliminate errors caused by stream networks with no DEM data in areas of HUCs that are outside of the U.S. border) and uses the polygon layer to clip the WBD and stream network (to a buffer inside the WBD).
+
+### Additions
+- `data/usgs/acquire_and_preprocess_3dep_dems.py`: Adds creation of 3DEP domain polygon by polygonizing all HUC6 3DEP DEMs and then dissolving them.
+- `src/gms/run_by_unit.sh`: Adds 3DEP domain polygon .gpkg as input to `src/clip_vectors_to_wbd.py`
+
+### Changes
+- `src/clip_vectors_to_wbd.py`: Clips WBD to 3DEP domain polygon and clips streams to a buffer inside the clipped WBD polygon.
+
+<br/><br/>
+
 ## v4.0.14.0 - 2022-12-20 - [PR #769](https://github.com/NOAA-OWP/inundation-mapping/pull/769)
 
 Masks levee-protected areas from the DEM in branch 0 and in highest two stream order branches.
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 5c07cf957..60f8639a9 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -16,6 +16,7 @@ def subset_vector_layers(subset_nwm_lakes,
                          wbd_buffer_filename,
                          wbd_filename,
                          dem_filename,
+                         dem_domain,
                          nwm_lakes,
                          nwm_catchments,
                          subset_nwm_catchments,
@@ -41,6 +42,10 @@ def subset_vector_layers(subset_nwm_lakes,
     wbd_buffer = wbd.copy()
     wbd_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance, resolution=32)
 
+    # Erase area outside 3DEP domain
+    dem_domain = gpd.read_file(dem_domain)
+    wbd_buffer = gpd.clip(wbd_buffer, dem_domain)
+
     # Make the streams buffer smaller than the wbd_buffer so streams don't reach the edge of the DEM
     wbd_streams_buffer = wbd.copy()
     wbd_streams_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance-3*dem_cellsize, resolution=32)
@@ -162,6 +167,7 @@ def subset_vector_layers(subset_nwm_lakes,
                         required=True)
     parser.add_argument('-g','--wbd-filename', help='HUC boundary', required=True)
     parser.add_argument('-i','--dem-filename', help='DEM filename', required=True)
+    parser.add_argument('-j','--dem-domain', help='DEM domain polygon', required=True)
     parser.add_argument('-l','--nwm-lakes', help='NWM Lakes', required=True)    
     parser.add_argument('-m','--nwm-catchments', help='NWM catchments',
                         required=True)	 
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 1fdc8be38..c20e81d13 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -81,6 +81,7 @@ cmd_args+=" -e $outputHucDataDir/nwm_headwater_points_subset.gpkg"
 cmd_args+=" -f $outputHucDataDir/wbd_buffered.gpkg"
 cmd_args+=" -g $outputHucDataDir/wbd.gpkg"
 cmd_args+=" -i $input_DEM"
+cmd_args+=" -j $inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg"
 cmd_args+=" -l $input_nwm_lakes"
 cmd_args+=" -m $input_nwm_catchments"
 cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"

From 9116498653f1b36cf6563812b6a63d5ab19cc3a5 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 23 Dec 2022 13:46:21 -0600
Subject: [PATCH 189/359] Update and renamed whitelists for hydrovis S3 put

---
 ...lst => aws_s3_put_fim3_hydrovis_whitelist.lst} |  0
 ...lst => aws_s3_put_fim4_hydrovis_whitelist.lst} |  1 +
 data/aws/s3.py                                    |  4 +++-
 docs/CHANGELOG.md                                 | 15 +++++++++++++++
 4 files changed, 19 insertions(+), 1 deletion(-)
 rename config/{aws_s3_put_fim3_whitelist.lst => aws_s3_put_fim3_hydrovis_whitelist.lst} (100%)
 rename config/{aws_s3_put_fim4_whitelist.lst => aws_s3_put_fim4_hydrovis_whitelist.lst} (82%)

diff --git a/config/aws_s3_put_fim3_whitelist.lst b/config/aws_s3_put_fim3_hydrovis_whitelist.lst
similarity index 100%
rename from config/aws_s3_put_fim3_whitelist.lst
rename to config/aws_s3_put_fim3_hydrovis_whitelist.lst
diff --git a/config/aws_s3_put_fim4_whitelist.lst b/config/aws_s3_put_fim4_hydrovis_whitelist.lst
similarity index 82%
rename from config/aws_s3_put_fim4_whitelist.lst
rename to config/aws_s3_put_fim4_hydrovis_whitelist.lst
index 431404077..5d8ed5ca7 100644
--- a/config/aws_s3_put_fim4_whitelist.lst
+++ b/config/aws_s3_put_fim4_hydrovis_whitelist.lst
@@ -1,3 +1,4 @@
 hydroTable{}.csv
 gw_catchments_reaches_filtered_addedAttributes{}.tif
 rem_zeroed_masked{}.tif
+usgs_elev_table.csv
diff --git a/data/aws/s3.py b/data/aws/s3.py
index 7da3b2273..adc2f23b0 100644
--- a/data/aws/s3.py
+++ b/data/aws/s3.py
@@ -180,9 +180,10 @@ def bulk_upload(self, src_folder_path, aws_target_path, whitelist_file_names = [
 if __name__ == '__main__':
 
     # Sample Usage
-    #python3 /foss_fim/data/aws/s3.py -a put -c /data/config/aws_creds.env -s /data/previous_fim/fim_4_0_7_0 -t "s3://example-us-east-1/fim_4_0_7_0" -w /foss_fim/config/aws_s3_put_whitelist.lst
+    #python3 /foss_fim/data/aws/s3.py -a put -c /data/config/aws_creds.env -s /data/previous_fim/fim_4_0_13_1 -t "s3://example-us-east-1/fim_4_0_13_1" 
     
     # You can leave the -w flag off to load all files/folders from a directory
+    # but default is to -w /foss_fim/config/aws_s3_put_fim4_hydrovis_whitelist.lst
     
     # this works for hydroviz but can work with other s3 sites as well (ie esip)
             
@@ -198,6 +199,7 @@ def bulk_upload(self, src_folder_path, aws_target_path, whitelist_file_names = [
                         help='s3 bucket address and folder', required=True)
     parser.add_argument('-w','--whitelist_file_path', 
                         help='A file with the last of file names to be copied up (line delimited)',
+                        default='/foss_fim/config/aws_s3_put_fim4_hydrovis_whitelist.lst',
                         required=False)
     parser.add_argument('-v','--is_verbose', 
                         help='Adding this flag will give additional tracing output',
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0b95bae1d..48df89672 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.14.2 - 2022-12-22 - [PR #772](https://github.com/NOAA-OWP/inundation-mapping/pull/772)
+
+Added `usgs_elev_table.csv` to hydrovis whitelist files.  Also updated the name to include the word "hydrovis" in them (anticipating more s3 whitelist files).
+
+### Changes
+
+- `config`
+    - `aws_s3_put_fim4_hydrovis_whitelist.lst`:  File name updated and added usgs_elev_table.csv so it gets push up as well.
+    - `aws_s3_put_fim3_hydrovis_whitelist.lst`: File name updated
+
+- `data/aws`
+   - `s3.py`: added `/foss_fim/config/aws_s3_put_fim4_hydrovis_whitelist.lst` as a default to the -w param.
+
+<br/><br/>
+
 ## v4.0.14.1 - 2022-12-03 - [PR #753](https://github.com/NOAA-OWP/inundation-mapping/pull/753)
 
 Creates a polygon of 3DEP DEM domain (to eliminate errors caused by stream networks with no DEM data in areas of HUCs that are outside of the U.S. border) and uses the polygon layer to clip the WBD and stream network (to a buffer inside the WBD).

From 5de3995634bebedadbc7adf6559042bb69b64c39 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 23 Dec 2022 13:54:28 -0600
Subject: [PATCH 190/359] Enhancements to CatFIM, rating curve generation
 script, and other files

---
 docs/CHANGELOG.md                         |  33 +-
 src/usgs_gage_crosswalk.py                |   6 +-
 src/usgs_gage_unit_setup.py               |   2 +-
 tools/eval_plots.py                       |  18 +
 tools/generate_categorical_fim.py         | 704 ++++++++++++++++++++--
 tools/generate_categorical_fim_flows.py   | 134 ++--
 tools/generate_categorical_fim_mapping.py | 352 +++++------
 tools/gms_tools/inundate_gms.py           |   4 +-
 tools/pixel_counter.py                    |   7 +-
 tools/rating_curve_get_usgs_curves.py     | 129 ++--
 tools/test_case_by_hydro_id.py            |  80 +--
 tools/tools_shared_functions.py           |  67 +-
 tools/tools_shared_variables.py           |  13 +
 13 files changed, 1131 insertions(+), 418 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 48df89672..401a91b75 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,37 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.15.0 - 2022-12-20 - [PR #758](https://github.com/NOAA-OWP/inundation-mapping/pull/758)
+
+This merge addresses feedback received from field users regarding CatFIM. Users wanted a Stage-Based version of CatFIM, they wanted maps created for multiple intervals between flood categories, and they wanted documentation as to why many sites are absent from the Stage-Based CatFIM service. This merge seeks to address this feedback. CatFIM will continue to evolve with more feedback over time.
+
+## Changes
+- `/src/gms/usgs_gage_crosswalk.py`: Removed filtering of extra attributes when writing table
+- `/src/gms/usgs_gage_unit_setup.py`: Removed filter of gages where `rating curve == yes`. The filtering happens later on now.
+- `/tools/eval_plots.py`: Added a post-processing step to produce CSVs of spatial data
+- `/tools/generate_categorical_fim.py`:
+  - New arguments to support more advanced multiprocessing, support production of Stage-Based CatFIM, specific output directory pathing, upstream and downstream distance, controls on how high past "major" magnitude to go when producing interval maps for Stage-Based, the ability to run a single AHPS site.
+- `/tools/generate_categorical_fim_flows.py`:
+  - Allows for flows to be retrieved for only one site (useful for testing)
+  - More logging
+  - Filtering stream segments according to stream order
+- `/tools/generate_categorical_fim_mapping.py`:
+  - Support for Stage-Based CatFIM production
+  - Enhanced multiprocessing
+  - Improved post-processing
+- `/tools/pixel_counter.py`: fixed a bug where Nonetypes were being returned
+- `/tools/rating_curve_get_usgs_rating_curves.py`:
+  - Removed filtering when producing `usgs_gages.gpkg`, but adding attribute as to whether or not it meets acceptance criteria, as defined in `gms_tools/tools_shared_variables.py`.
+  - Creating a lookup list to filter out unacceptable gages before they're written to `usgs_rating_curves.csv`
+  - The `usgs_gages.gpkg` now includes two fields indicating whether or not gages pass acceptance criteria (defined in `tools_shared_variables.py`. The fields are `acceptable_codes` and `acceptable_alt_error`
+- `/tools/tools_shared_functions.py`:
+  - Added `get_env_paths()` function to retrieve environmental variable information used by CatFIM and rating curves scripts
+  - `Added `filter_nwm_segments_by_stream_order()` function that uses WRDS to filter out NWM feature_ids from a list if their stream order is different than a desired stream order.
+- `/tools/tools_shared_variables.py`: Added the acceptance criteria and URLS for gages as non-constant variables. These can be modified and tracked through version changes. These variables are imported by the CatFIM and USGS rating curve and gage generation scripts.
+- `/tools/test_case_by_hydroid.py`: reformatting code, recommend adding more comments/docstrings in future commit
+
+<br/><br/>
+
 ## v4.0.14.2 - 2022-12-22 - [PR #772](https://github.com/NOAA-OWP/inundation-mapping/pull/772)
 
 Added `usgs_elev_table.csv` to hydrovis whitelist files.  Also updated the name to include the word "hydrovis" in them (anticipating more s3 whitelist files).
@@ -49,7 +80,7 @@ Fixes inundation of nodata areas of REM.
 
 - `tools/inundation.py`: Assigns depth a value of `0` if REM is less than `0`
 
-<br/><br/>
+
 
 ## v4.0.13.1 - 2022-12-09 - [PR #743](https://github.com/NOAA-OWP/inundation-mapping/pull/743)
 
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 57b7c1cb8..a12494754 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -41,7 +41,7 @@ def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_file
         the dem-derived flows 3) sample both dems at the snapped points 4) write the crosswalked points
         to usgs_elev_table.csv
         '''
-
+        
         if self.gages.empty:
             print(f'There are no gages for branch {branch_id}')
             os._exit(0)
@@ -96,10 +96,10 @@ def sample_dem(self, dem_filename, column_name):
     def write(self, output_table_filename):
         '''Write to csv file'''
 
+        # Prep and write out file
         elev_table = self.gages.copy()
-        # Elev table cleanup
         elev_table.loc[elev_table['location_id'] == elev_table['nws_lid'], 'location_id'] = None # set location_id to None where there isn't a gage
-        elev_table = elev_table[['location_id', 'nws_lid', 'feature_id', 'HydroID', 'levpa_id', 'dem_elevation', 'dem_adj_elevation', 'order_', 'LakeID', 'HUC8', 'snap_distance']]
+        elev_table = elev_table[elev_table['location_id'].notna()]
 
         if not elev_table.empty:
             elev_table.to_csv(output_table_filename, index=False)
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 93c4095f6..476777da0 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -23,7 +23,7 @@ def load_gages(self):
 
         # Filter USGS gages to huc
         usgs_gages = gpd.read_file(self.usgs_gage_filename)
-        self.gages = usgs_gages[(usgs_gages.HUC8 == self.huc8) & (usgs_gages.curve == 'yes')]
+        self.gages = usgs_gages[(usgs_gages.HUC8 == self.huc8)]
         
         # Get AHPS sites within the HUC and add them to the USGS dataset
         if self.ahps_filename:
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 560c6c05d..d0b041f99 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -8,6 +8,7 @@
 import matplotlib.pyplot as plt
 import seaborn as sns
 import re
+import glob
 import os
 import sys
 sys.path.append('/foss_fim/src')
@@ -669,6 +670,19 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
             wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
         else:
             print('BLE/IFC/RAS2FIM FR datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a FR version')
+        
+def convert_shapes_to_csv(workspace):
+    
+    # Convert any geopackage in the root level of output_mapping_dir to CSV and rename.
+    shape_list = glob.glob(os.path.join(workspace, '*.shp'))
+    for shape in shape_list:
+        gdf = gpd.read_file(shape)
+        parent_directory = os.path.split(shape)[0]
+        file_name = shape.replace('.shp', '.csv')
+        csv_output_path = os.path.join(parent_directory, file_name)
+        gdf.to_csv(csv_output_path)
+    
+        
 #######################################################################
 if __name__ == '__main__':
     # Parse arguments
@@ -697,3 +711,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
     print('The following AHPS sites are considered "BAD_SITES":  ' + ', '.join(BAD_SITES))
     print('The following query is used to filter AHPS:  ' + DISCARD_AHPS_QUERY)
     eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, spatial = sp, fim_1_ms = f, site_barplots = i)
+    
+    # Convert output shapefiles to CSV
+    print("Converting to CSVs...")
+    convert_shapes_to_csv(w)
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index bcd6fc14d..4974fcb09 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -1,15 +1,148 @@
 #!/usr/bin/env python3
 
 import os
-import subprocess
 import argparse
+import csv
+import traceback
+import sys
 import time
 from pathlib import Path
 import geopandas as gpd
 import pandas as pd
+import rasterio
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+import glob
+from generate_categorical_fim_flows import generate_catfim_flows
+from generate_categorical_fim_mapping import manage_catfim_mapping, post_process_cat_fim_for_viz
+from tools_shared_functions import get_thresholds, get_nwm_segs, get_datum, ngvd_to_navd_ft, filter_nwm_segments_by_stream_order
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+import numpy as np
+from utils.shared_variables import VIZ_PROJECTION
+from tools_shared_variables import (acceptable_coord_acc_code_list, 
+                                    acceptable_coord_method_code_list, 
+                                    acceptable_alt_acc_thresh, 
+                                    acceptable_alt_meth_code_list, 
+                                    acceptable_site_type_list)
 
 
-def update_mapping_status(output_mapping_dir, output_flows_dir):
+def process_generate_categorical_fim(fim_run_dir, job_number_huc, job_number_inundate, 
+                                     stage_based, output_folder, overwrite, search, 
+                                     lid_to_run, job_number_intervals, past_major_interval_cap):
+        
+    # Check job numbers and raise error if necessary
+    total_cpus_requested = job_number_huc * job_number_inundate * job_number_intervals
+    total_cpus_available = os.cpu_count() - 1
+    if total_cpus_requested > total_cpus_available:
+        raise ValueError('The HUC job number, {}, multiplied by the inundate job number, {}, '\
+                          'exceeds your machine\'s available CPU count minus one. '\
+                          'Please lower the job_number_huc or job_number_inundate '\
+                          'values accordingly.'.format(job_number_huc, job_number_inundate) )
+
+    # Define default arguments. Modify these if necessary
+    fim_version = os.path.split(fim_run_dir)[1]
+    
+    # Append option configuration (flow_based or stage_based) to output folder name.
+    if stage_based:
+        file_handle_appendage, catfim_method = "_stage_based", "STAGE-BASED"
+    else:
+        file_handle_appendage, catfim_method = "_flow_based", "FLOW-BASED"
+    
+    # Define output directories
+    output_catfim_dir_parent = output_folder + file_handle_appendage
+    output_flows_dir = os.path.join(output_catfim_dir_parent, 'flows')
+    output_mapping_dir = os.path.join(output_catfim_dir_parent, 'mapping')
+    attributes_dir = os.path.join(output_catfim_dir_parent, 'attributes')
+    
+    # Create output directories
+    if not os.path.exists(output_catfim_dir_parent): os.mkdir(output_catfim_dir_parent)
+    if not os.path.exists(output_flows_dir): os.mkdir(output_flows_dir)
+    if not os.path.exists(output_mapping_dir): os.mkdir(output_mapping_dir)
+    if not os.path.exists(attributes_dir): os.mkdir(attributes_dir)
+    
+    # Define upstream and downstream search in miles
+    nwm_us_search, nwm_ds_search = search, search
+    fim_dir = fim_version
+    
+    # Set up logging
+    log_dir = os.path.join(output_catfim_dir_parent, 'logs')
+    log_file = os.path.join(log_dir, 'errors.log')
+    
+    # STAGE-BASED
+    if stage_based:
+        # Generate Stage-Based CatFIM mapping
+        nws_sites_layer = generate_stage_based_categorical_fim(output_mapping_dir, fim_version, fim_run_dir,
+                                                               nwm_us_search, nwm_ds_search, job_number_inundate,
+                                                               lid_to_run, attributes_dir, job_number_intervals,
+                                                               past_major_interval_cap, job_number_huc)
+        
+        job_number_tif = job_number_inundate * job_number_intervals
+        print("Post-processing TIFs...")
+        post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_mapping_dir, attributes_dir, log_file=log_file, fim_version=fim_version)
+    
+        # Updating mapping status
+        print('Updating mapping status...')
+        update_mapping_status(str(output_mapping_dir), str(output_flows_dir), nws_sites_layer, stage_based)
+
+    # FLOW-BASED
+    else:
+        fim_dir = ""
+        print('Creating flow files using the ' + catfim_method + ' technique...')
+        start = time.time()
+        nws_sites_layer = generate_catfim_flows(output_flows_dir, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir)
+        end = time.time()
+        elapsed_time = (end-start)/60
+        print(f'Finished creating flow files in {elapsed_time} minutes')
+        # Generate CatFIM mapping
+        print('Begin mapping')
+        start = time.time()
+        manage_catfim_mapping(fim_run_dir, output_flows_dir, output_mapping_dir, attributes_dir, job_number_huc, job_number_inundate, overwrite, depthtif=False)
+        end = time.time()
+        elapsed_time = (end-start)/60
+        print(f'Finished mapping in {elapsed_time} minutes')
+
+        # Updating mapping status
+        print('Updating mapping status')
+        update_mapping_status(str(output_mapping_dir), str(output_flows_dir), nws_sites_layer, stage_based)
+    
+    # Create CSV versions of the final geopackages.
+    print('Creating CSVs. This may take several minutes.')
+    reformatted_catfim_method = catfim_method.lower().replace('-', '_')
+    create_csvs(output_mapping_dir, reformatted_catfim_method)
+
+
+def create_csvs(output_mapping_dir, reformatted_catfim_method):
+    '''
+    Produces CSV versions of desired geopackage in the output_mapping_dir.
+
+    Parameters
+    ----------
+    output_mapping_dir : STR
+        Path to the output directory of all inundation maps.
+    reformatted_catfim_method : STR
+        Text to append to CSV to communicate the type of CatFIM.
+
+    Returns
+    -------
+    None.
+
+    '''
+    
+    # Convert any geopackage in the root level of output_mapping_dir to CSV and rename.
+    gpkg_list = glob.glob(os.path.join(output_mapping_dir, '*.gpkg'))
+    for gpkg in gpkg_list:
+        print(f"creating CSV for {gpkg}")
+        gdf = gpd.read_file(gpkg)
+        parent_directory = os.path.split(gpkg)[0]
+        if 'catfim_library' in gpkg:
+            file_name = reformatted_catfim_method + '_catfim.csv'
+        if 'nws_lid_sites' in gpkg:
+            file_name = reformatted_catfim_method + '_catfim_sites.csv'
+        
+        csv_output_path = os.path.join(parent_directory, file_name)
+        gdf.to_csv(csv_output_path)
+
+
+def update_mapping_status(output_mapping_dir, output_flows_dir, nws_sites_layer, stage_based):
     '''
     Updates the status for nws_lids from the flows subdirectory. Status
     is updated for sites where the inundation.py routine was not able to
@@ -38,9 +171,8 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     mapping_df['did_it_map'] = 'no'
     mapping_df['map_status'] = ' and all categories failed to map'
 
-    # Import shapefile output from flows creation
-    shapefile = Path(output_flows_dir)/'nws_lid_flows_sites.shp'
-    flows_df = gpd.read_file(shapefile)
+    # Import geopackage output from flows creation
+    flows_df = gpd.read_file(nws_sites_layer)
 
     # Join failed sites to flows df
     flows_df = flows_df.merge(mapping_df, how = 'left', on = 'nws_lid')
@@ -48,63 +180,535 @@ def update_mapping_status(output_mapping_dir, output_flows_dir):
     # Switch mapped column to no for failed sites and update status
     flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
     flows_df.loc[flows_df['did_it_map']=='no','status'] = flows_df['status'] + flows_df['map_status']
-
-    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
-    flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
-    mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
-    missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
     
-    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
-    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
-    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
+#    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
+#    if stage_based:
+#        missing_mapping_hucs = 
+#    else:
+#        flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
+#        mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
+#        missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
+#    
+#    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
+#    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
+#    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
 
     # Clean up GeoDataFrame and rename columns for consistency
     flows_df = flows_df.drop(columns = ['did_it_map','map_status'])
     flows_df = flows_df.rename(columns = {'nws_lid':'ahps_lid'})
 
     # Write out to file
-    nws_lid_path = Path(output_mapping_dir) / 'nws_lid_sites.shp'
-    flows_df.to_file(nws_lid_path)
+    flows_df.to_file(nws_sites_layer)
+
+
+def produce_inundation_map_with_stage_and_feature_ids(rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch):
+    
+    # Open rem_path and catchment_path using rasterio.
+    rem_src = rasterio.open(rem_path)
+    catchments_src = rasterio.open(catchments_path)
+    rem_array = rem_src.read(1)
+    catchments_array = catchments_src.read(1)
+    
+    # Use numpy.where operation to reclassify rem_path on the condition that the pixel values are <= to hand_stage and the catchments
+    # value is in the hydroid_list.
+    reclass_rem_array = np.where((rem_array<=hand_stage) & (rem_array != rem_src.nodata), 1, 0).astype('uint8')
+    hydroid_mask = np.isin(catchments_array, hydroid_list)   
+    target_catchments_array = np.where((hydroid_mask == True) & (catchments_array != catchments_src.nodata), 1, 0).astype('uint8')
+    masked_reclass_rem_array = np.where((reclass_rem_array == 1) & (target_catchments_array == 1), 1, 0).astype('uint8')
+        
+    # Save resulting array to new tif with appropriate name. brdc1_record_extent_18060005.tif
+    is_all_zero = np.all((masked_reclass_rem_array == 0))
+    
+    if not is_all_zero:
+        output_tif = os.path.join(lid_directory, lid + '_' + category + '_extent_' + huc + '_' + branch + '.tif')
+        with rasterio.Env():
+            profile = rem_src.profile
+            profile.update(dtype=rasterio.uint8)
+            profile.update(nodata=10)
+            
+            with rasterio.open(output_tif, 'w', **profile) as dst:
+                dst.write(masked_reclass_rem_array, 1)
+    
+
+def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, threshold_url, flood_categories, all_lists, past_major_interval_cap, number_of_jobs, number_of_interval_jobs, attributes_dir):
+    missing_huc_files = []
+    all_messages = []
+    stage_based_att_dict = {}
+    
+    print(f'Iterating through {huc}...') 
+    # Make output directory for huc.
+    huc_directory = os.path.join(workspace, huc)
+    if not os.path.exists(huc_directory):
+        os.mkdir(huc_directory)
+    
+    # Define paths to necessary HAND and HAND-related files.
+    usgs_elev_table = os.path.join(fim_dir, huc, 'usgs_elev_table.csv')
+    branch_dir = os.path.join(fim_dir, huc, 'branches')
+    
+    # Loop through each lid in nws_lids list
+    nws_lids = huc_dictionary[huc]
+    for lid in nws_lids:
+        lid = lid.lower() # Convert lid to lower case
+        # -- If necessary files exist, continue -- #
+        if not os.path.exists(usgs_elev_table):
+            all_messages.append([f'{lid}:usgs_elev_table missing, likely unacceptable gage datum error--more details to come in future release'])
+            continue
+        if not os.path.exists(branch_dir):
+            all_messages.append([f'{lid}:branch directory missing'])
+            continue
+        usgs_elev_df = pd.read_csv(usgs_elev_table)
+        # Make lid_directory.
+        
+        lid_directory = os.path.join(huc_directory, lid)
+        if not os.path.exists(lid_directory):
+            os.mkdir(lid_directory)
+        # Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
+        stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
+                
+        if stages == None:
+            all_messages.append([f'{lid}:error getting thresholds from WRDS API'])
+            continue
+        # Check if stages are supplied, if not write message and exit. 
+        if all(stages.get(category, None)==None for category in flood_categories):
+            all_messages.append([f'{lid}:missing threshold stages'])
+            continue
+
+        # Drop columns that offend acceptance criteria
+        usgs_elev_df['acceptable_codes'] = (usgs_elev_df['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
+                                    & usgs_elev_df['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
+                                    & usgs_elev_df['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
+                                    & usgs_elev_df['usgs_data_site_type'].isin(acceptable_site_type_list))
+        
+        usgs_elev_df = usgs_elev_df.astype({'usgs_data_alt_accuracy_code': float})
+        usgs_elev_df['acceptable_alt_error'] = np.where(usgs_elev_df['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False)
+        
+        acceptable_usgs_elev_df = usgs_elev_df[(usgs_elev_df['acceptable_codes'] == True) & (usgs_elev_df['acceptable_alt_error'] == True)]
+        
+        # Get the dem_adj_elevation value from usgs_elev_table.csv. Prioritize the value that is not from branch 0.
+        try:
+            matching_rows = acceptable_usgs_elev_df.loc[acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation']
+            if len(matching_rows) == 2:  # It means there are two level paths, use the one that is not 0
+                lid_usgs_elev = acceptable_usgs_elev_df.loc[(acceptable_usgs_elev_df['nws_lid'] == lid.upper()) & ('levpa_id' != 0), 'dem_adj_elevation'].values[0]
+            else:
+                lid_usgs_elev = acceptable_usgs_elev_df.loc[acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'].values[0]
+        except IndexError:  # Occurs when LID is missing from table
+            all_messages.append([f'{lid}:likely unacceptable gage datum error or accuracy code(s); please see acceptance criteria'])
+            continue
+        # Initialize nested dict for lid attributes
+        stage_based_att_dict.update({lid:{}})
+            
+        # Find lid metadata from master list of metadata dictionaries.
+        metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
+        lid_altitude = metadata['usgs_data']['altitude']
+   
+        ### --- Do Datum Offset --- ###
+        #determine source of interpolated threshold flows, this will be the rating curve that will be used.
+        rating_curve_source = flows.get('source')
+        if rating_curve_source is None:
+            all_messages.append([f'{lid}:No source for rating curve'])
+            continue
+        # Get the datum and adjust to NAVD if necessary.
+        nws_datum_info, usgs_datum_info = get_datum(metadata)
+        if rating_curve_source == 'USGS Rating Depot':
+            datum_data = usgs_datum_info
+        elif rating_curve_source == 'NRLDB':
+            datum_data = nws_datum_info
+                    
+        # If datum not supplied, skip to new site
+        datum = datum_data.get('datum', None)
+        if datum is None:
+            all_messages.append([f'{lid}:datum info unavailable'])
+            continue
+        # _________________________________________________________________________________________________________#
+        # SPECIAL CASE: Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null). 
+        # Modifying rating curve source will influence the rating curve and datum retrieved for benchmark determinations.
+        if lid == 'bmbp1':
+            rating_curve_source = 'NRLDB'
+        # ___________________________________________________________________#
+        
+        # SPECIAL CASE: Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29 conversion to NAVD88
+        # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7, stkn4, wlln7 
+        # Assumed to be NAD83 (no info from USGS or NWS data): dlrt2, eagi1, eppt2, jffw3, ldot2, rgdt2
+        if lid in ['bgwn7', 'dlrt2','eagi1','eppt2','fatw3','jffw3','ldot2','mnvn4','nhpp1','pinn4','rgdt2','rgln4','rssk1','sign4','smfn7','stkn4','wlln7' ]:
+            datum_data.update(crs = 'NAD83')
+        # ___________________________________________________________________#
+        
+        # SPECIAL CASE: Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and is actually NAD27. 
+        # This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and it matches NRLDB coordinates.
+        if lid == 'bmbp1':
+            datum_data.update(crs = 'NAD27')
+        # ___________________________________________________________________#
+        
+        # SPECIAL CASE: Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure datum reported in NAVD88. 
+        # If NGVD29 it is converted to NAVD88.
+        # bgwn7, eagi1 vertical datum unknown, assume navd88
+        # fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
+        # wlln7 USGS data indicates vcs is NGVD29 (USGS and NWS info agree on datum value).
+        if lid in ['bgwn7','eagi1','fatw3']:
+            datum_data.update(vcs = 'NAVD88')
+        elif lid == 'wlln7':
+            datum_data.update(vcs = 'NGVD29')
+        # _________________________________________________________________________________________________________#
+        
+        # Adjust datum to NAVD88 if needed
+        # Default datum_adj_ft to 0.0
+        datum_adj_ft = 0.0
+        crs = datum_data.get('crs')
+        if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929', 'NGVD,1929', 'NGVD OF 1929', 'NGVD']:
+            # Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
+            try:
+                datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
+            except Exception as e:
+                e = str(e)
+                if crs == None:
+                    all_messages.append([f'{lid}:NOAA VDatum adjustment error, CRS is missing'])
+                if 'HTTPSConnectionPool' in e:
+                    time.sleep(10)  # Maybe the API needs a break, so wait 10 seconds
+                    try:
+                        datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
+                    except Exception:
+                        all_messages.append([f'{lid}:NOAA VDatum adjustment error, possible API issue'])
+                if 'Invalid projection' in e:
+                    all_messages.append([f'{lid}:NOAA VDatum adjustment error, invalid projection: crs={crs}'])
+                continue
+        
+        ### -- Concluded Datum Offset --- ###
+        # Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
+        unfiltered_segments = list(set(get_nwm_segs(metadata)))
+        
+        # Filter segments to be of like stream order.
+        desired_order = metadata['nwm_feature_data']['stream_order']
+        segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order)
+        action_stage = stages['action']
+        minor_stage = stages['minor']
+        moderate_stage = stages['moderate']
+        major_stage = stages['major']
+        stage_list = [i for i in [action_stage, minor_stage, moderate_stage, major_stage] if i is not None]
+        # Create a list of stages, incrementing by 1 ft.
+        if stage_list == []:
+            all_messages.append([f'{lid}:no stage values available'])
+            continue
+        interval_list = np.arange(min(stage_list), max(stage_list) + past_major_interval_cap, 1.0)  # Go an extra 10 ft beyond the max stage, arbitrary
+        # For each flood category
+        for category in flood_categories:
+            
+            # Pull stage value and confirm it's valid, then process
+            stage = stages[category]
+            
+            if stage != None and datum_adj_ft != None and lid_altitude != None:
+                
+                # Call function to execute mapping of the TIFs.
+                messages, hand_stage, datum_adj_wse, datum_adj_wse_m = produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs)
+                all_messages += messages
+                                                
+                # Extra metadata for alternative CatFIM technique. TODO Revisit because branches complicate things
+                stage_based_att_dict[lid].update({category: {'datum_adj_wse_ft': datum_adj_wse,
+                                                             'datum_adj_wse_m': datum_adj_wse_m,
+                                                             'hand_stage': hand_stage,
+                                                             'datum_adj_ft': datum_adj_ft,
+                                                             'lid_alt_ft': lid_altitude,
+                                                             'lid_alt_m': lid_altitude*0.3048}})
+            # If missing HUC file data, write message
+            if huc in missing_huc_files:
+                all_messages.append([f'{lid}:missing some HUC data'])
+                
+        # Now that the "official" category maps are made, produce the incremental maps.
+        for interval_stage in interval_list:
+            try:
+                with ProcessPoolExecutor(max_workers=number_of_interval_jobs) as executor:
+                    # Determine category the stage value belongs with.
+                    if action_stage <= interval_stage < minor_stage:
+                        category = 'action_' + str(interval_stage).replace('.', 'p') + 'ft'
+                    if minor_stage <= interval_stage < moderate_stage:
+                        category = 'minor_' + str(interval_stage).replace('.', 'p') + 'ft'
+                    if moderate_stage <= interval_stage < major_stage:
+                        category = 'moderate_' + str(interval_stage).replace('.', 'p') + 'ft'
+                    if interval_stage >= major_stage:
+                        category = 'major_' + str(interval_stage).replace('.', 'p') + 'ft'
+                    executor.submit(produce_stage_based_catfim_tifs, stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs)
+            except TypeError:  # sometimes the thresholds are Nonetypes
+                pass
+                    
+        # Create a csv with same information as geopackage but with each threshold as new record.
+        # Probably a less verbose way.
+        csv_df = pd.DataFrame()
+        for threshold in flood_categories:
+            try:
+                line_df = pd.DataFrame({'nws_lid': [lid], 
+                                        'name':metadata['nws_data']['name'], 
+                                        'WFO': metadata['nws_data']['wfo'], 
+                                        'rfc':metadata['nws_data']['rfc'], 
+                                        'huc':[huc], 
+                                        'state':metadata['nws_data']['state'], 
+                                        'county':metadata['nws_data']['county'], 
+                                        'magnitude': threshold, 
+                                        'q':flows[threshold], 
+                                        'q_uni':flows['units'], 
+                                        'q_src':flows['source'], 
+                                        'stage':stages[threshold], 
+                                        'stage_uni':stages['units'], 
+                                        's_src':stages['source'], 
+                                        'wrds_time':stages['wrds_timestamp'], 
+                                        'nrldb_time':metadata['nrldb_timestamp'],
+                                        'nwis_time':metadata['nwis_timestamp'], 
+                                        'lat':[float(metadata['nws_preferred']['latitude'])], 
+                                        'lon':[float(metadata['nws_preferred']['longitude'])],
+                                        'dtm_adj_ft': stage_based_att_dict[lid][threshold]['datum_adj_ft'],
+                                        'dadj_w_ft': stage_based_att_dict[lid][threshold]['datum_adj_wse_ft'],
+                                        'dadj_w_m': stage_based_att_dict[lid][threshold]['datum_adj_wse_m'],
+                                        'lid_alt_ft': stage_based_att_dict[lid][threshold]['lid_alt_ft'],
+                                        'lid_alt_m': stage_based_att_dict[lid][threshold]['lid_alt_m']})
+                csv_df = csv_df.append(line_df)
+                
+            except Exception as e:
+                print("Exception")
+                print(e)
+          
+        # Round flow and stage columns to 2 decimal places.
+        csv_df = csv_df.round({'q':2,'stage':2})
+        # If a site folder exists (ie a flow file was written) save files containing site attributes.
+        output_dir = os.path.join(workspace, huc, lid)
+        if os.path.exists(output_dir):
+            # Export DataFrame to csv containing attributes
+            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index = False)
+        else:
+            all_messages.append([f'{lid}:missing all calculated flows'])
+            
+        # If it made it to this point (i.e. no continues), there were no major preventers of mapping
+        all_messages.append([f'{lid}:OK'])
+    # Write all_messages by HUC to be scraped later.
+    messages_dir = os.path.join(workspace, 'messages')
+    if not os.path.exists(messages_dir):
+        os.mkdir(messages_dir)
+    huc_messages_csv = os.path.join(messages_dir, huc + '_messages.csv')
+    with open(huc_messages_csv, 'w') as output_csv:
+        writer = csv.writer(output_csv)
+        writer.writerows(all_messages)
+
+    
+def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us_search, nwm_ds_search, number_of_jobs, lid_to_run, attributes_dir, number_of_interval_jobs, past_major_interval_cap, job_number_huc):
+    
+    flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
+
+    huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists = generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based=True, fim_dir=fim_dir, lid_to_run=lid_to_run)
+    
+    with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
+        for huc in huc_dictionary:
+            executor.submit(iterate_through_huc_stage_based, workspace, huc, 
+                            fim_dir, huc_dictionary, threshold_url, flood_categories,
+                            all_lists, past_major_interval_cap, number_of_jobs, 
+                            number_of_interval_jobs, attributes_dir)
+                
+    print('Wrapping up Stage-Based CatFIM...')
+    csv_files = os.listdir(attributes_dir)
+    all_csv_df = pd.DataFrame()
+    refined_csv_files_list = []
+    for csv_file in csv_files:
+        full_csv_path = os.path.join(attributes_dir, csv_file)
+        # HUC has to be read in as string to preserve leading zeros.
+        try:
+            temp_df = pd.read_csv(full_csv_path, dtype={'huc':str})
+            all_csv_df = all_csv_df.append(temp_df, ignore_index = True)
+            refined_csv_files_list.append(csv_file)
+        except Exception:  # Happens if a file is empty (i.e. no mapping)
+            pass
+    # Write to file
+    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index = False)
+   
+    # This section populates a geopackage of all potential sites and details
+    # whether it was mapped or not (mapped field) and if not, why (status field).
+    
+    # Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
+    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)
+    viz_out_gdf.rename(columns = {'identifiers_nwm_feature_id': 'nwm_seg', 'identifiers_nws_lid':'nws_lid', 'identifiers_usgs_site_code':'usgs_gage'}, inplace = True)
+    viz_out_gdf['nws_lid'] = viz_out_gdf['nws_lid'].str.lower()
+    
+    # Using list of csv_files, populate DataFrame of all nws_lids that had
+    # a flow file produced and denote with "mapped" column.
+    nws_lids = []
+    for csv_file in csv_files:
+        nws_lids.append(csv_file.split('_attributes')[0])
+    lids_df = pd.DataFrame(nws_lids, columns = ['nws_lid'])
+    lids_df['mapped'] = 'yes'
+    
+    # Identify what lids were mapped by merging with lids_df. Populate 
+    # 'mapped' column with 'No' if sites did not map.
+    viz_out_gdf = viz_out_gdf.merge(lids_df, how = 'left', on = 'nws_lid')    
+    viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
+    
+    # Create list from all messages in messages dir.
+    messages_dir = os.path.join(workspace, 'messages')
+    all_messages = []
+    all_message_csvs = os.listdir(messages_dir)
+    for message_csv in all_message_csvs:
+        full_message_csv_path = os.path.join(messages_dir, message_csv)
+        with open(full_message_csv_path, newline='') as message_file:
+            reader = csv.reader(message_file)
+            for row in reader:
+                all_messages.append(row)
+    
+    # Write messages to DataFrame, split into columns, aggregate messages.
+    messages_df  = pd.DataFrame(all_messages, columns = ['message'])
+
+    messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
+    status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
+    
+    # Join messages to populate status field to candidate sites. Assign 
+    # status for null fields.
+    viz_out_gdf = viz_out_gdf.merge(status_df, how = 'left', on = 'nws_lid')
+    
+#    viz_out_gdf['status'] = viz_out_gdf['status'].fillna('OK')
+    
+    # Filter out columns and write out to file
+    nws_sites_layer = os.path.join(workspace, 'nws_lid_sites.gpkg')
+    
+    # Add acceptance criteria to viz_out_gdf before writing
+    viz_out_gdf['acceptable_coord_acc_code_list'] = str(acceptable_coord_acc_code_list)
+    viz_out_gdf['acceptable_coord_method_code_list'] = str(acceptable_coord_method_code_list)
+    viz_out_gdf['acceptable_alt_acc_thresh'] = float(acceptable_alt_acc_thresh)
+    viz_out_gdf['acceptable_alt_meth_code_list'] = str(acceptable_alt_meth_code_list)
+    viz_out_gdf['acceptable_site_type_list'] = str(acceptable_site_type_list)
+
+    viz_out_gdf.to_file(nws_sites_layer, driver='GPKG')
+    
+    return nws_sites_layer
+    
+
+def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs):
+    messages = []
+    
+    # Determine datum-offset water surface elevation (from above).
+    datum_adj_wse = stage + datum_adj_ft + lid_altitude
+    datum_adj_wse_m = datum_adj_wse*0.3048  # Convert ft to m
+    
+    # Subtract HAND gage elevation from HAND WSE to get HAND stage.
+    hand_stage = datum_adj_wse_m - lid_usgs_elev
+    
+    # Produce extent tif hand_stage. Multiprocess across branches.
+    branches = os.listdir(branch_dir)
+    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
+        for branch in branches:
+            # Define paths to necessary files to produce inundation grids.
+            full_branch_path = os.path.join(branch_dir, branch)
+            rem_path = os.path.join(fim_dir, huc, full_branch_path, 'rem_zeroed_masked_' + branch + '.tif')
+            catchments_path = os.path.join(fim_dir, huc, full_branch_path, 'gw_catchments_reaches_filtered_addedAttributes_' + branch + '.tif')
+            hydrotable_path = os.path.join(fim_dir, huc, full_branch_path, 'hydroTable_' + branch + '.csv')
+            
+            if not os.path.exists(rem_path):
+                messages.append([f"{lid}:rem doesn't exist"])
+                continue
+            if not os.path.exists(catchments_path):
+                messages.append([f"{lid}:catchments files don't exist"])
+                continue
+            if not os.path.exists(hydrotable_path):
+                messages.append([f"{lid}:hydrotable doesn't exist"])
+                continue
+            
+            # Use hydroTable to determine hydroid_list from site_ms_segments.
+            hydrotable_df = pd.read_csv(hydrotable_path)
+            hydroid_list = []
+            
+            # Determine hydroids at which to perform inundation
+            for feature_id in segments:
+                try:
+                    subset_hydrotable_df = hydrotable_df[hydrotable_df['feature_id'] == int(feature_id)]
+                    hydroid_list += list(subset_hydrotable_df.HydroID.unique())
+                except IndexError:
+                    pass
+                
+            if len(hydroid_list) == 0:
+#                messages.append(f"{lid}:no matching hydroids")  # Some branches don't have matching hydroids
+                continue
+
+            # If no segments, write message and exit out
+            if not segments:
+                messages.append([f'{lid}:missing nwm segments'])
+                continue
+            
+            # Create inundation maps with branch and stage data
+            try:
+                print("Running inundation for " + huc + " and branch " + branch)
+                executor.submit(produce_inundation_map_with_stage_and_feature_ids, rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch)
+            except Exception:
+                messages.append([f'{lid}:inundation failed at {category}'])
+                
+    # -- MOSAIC -- #
+    # Merge all rasters in lid_directory that have the same magnitude/category.
+    path_list = []
+    lid_dir_list = os.listdir(lid_directory)
+    print("Merging " + category)
+    for f in lid_dir_list:
+        if category in f:
+            path_list.append(os.path.join(lid_directory, f))
+    path_list.sort()  # To force branch 0 first in list, sort
+    
+    if len(path_list) > 0:
+        zero_branch_grid = path_list[0]
+        zero_branch_src = rasterio.open(zero_branch_grid)
+        zero_branch_array = zero_branch_src.read(1)
+        summed_array = zero_branch_array  # Initialize it as the branch zero array
+        
+        # Loop through remaining items in list and sum them with summed_array
+        for remaining_raster in path_list[1:]:
+            remaining_raster_src = rasterio.open(remaining_raster)
+            remaining_raster_array_original = remaining_raster_src.read(1)
+        
+            # Reproject non-branch-zero grids so I can sum them with the branch zero grid
+            remaining_raster_array = np.empty(zero_branch_array.shape, dtype=np.int8)
+            reproject(remaining_raster_array_original,
+                  destination = remaining_raster_array,
+                  src_transform = remaining_raster_src.transform,
+                  src_crs = remaining_raster_src.crs,
+                  src_nodata = remaining_raster_src.nodata,
+                  dst_transform = zero_branch_src.transform,
+                  dst_crs = zero_branch_src.crs,
+                  dst_nodata = -1,
+                  dst_resolution = zero_branch_src.res,
+                  resampling = Resampling.nearest)
+            # Sum rasters
+            summed_array = summed_array + remaining_raster_array
+            
+        del zero_branch_array  # Clean up
+        
+        # Define path to merged file, in same format as expected by post_process_cat_fim_for_viz function
+        output_tif = os.path.join(lid_directory, lid + '_' + category + '_extent.tif')  
+        profile = zero_branch_src.profile
+        summed_array = summed_array.astype('uint8')
+        with rasterio.open(output_tif, 'w', **profile) as dst:
+            dst.write(summed_array, 1)
+        del summed_array
+
+    return messages, hand_stage, datum_adj_wse, datum_adj_wse_m
 
 
 if __name__ == '__main__':
 
+    
     # Parse arguments
     parser = argparse.ArgumentParser(description = 'Run Categorical FIM')
-    parser.add_argument('-f','--fim_version',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-j','--number_of_jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
+    parser.add_argument('-f', '--fim_run_dir', help='Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_0_9_2',
+                        required=True)
+    parser.add_argument('-jh','--job_number_huc',help='Number of processes to use for HUC scale operations.'\
+        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count of the'\
+        ' machine. CatFIM sites generally only have 2-3 branches overlapping a site, so this number can be kept low (2-4)', required=False, default=1, type=int)
+    parser.add_argument('-jn','--job_number_inundate', help='Number of processes to use for inundating'\
+        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count'\
+        ' of the machine.', required=False, default=1, type=int)    
+    parser.add_argument('-a', '--stage_based', help = 'Run stage-based CatFIM instead of flow-based?'\
+        ' NOTE: flow-based CatFIM is the default.', required=False, default=False, action='store_true')
+    parser.add_argument('-t', '--output_folder', help = 'Target: Where the output folder will be',
+                        required = False, default = '/data/catfim/')
+    parser.add_argument('-o','--overwrite', help='Overwrite files', required=False, action="store_true")
+    parser.add_argument('-s','--search', help='Upstream and downstream search in miles. How far up and downstream do you want to go?',
+                        required=False, default='5')
+    parser.add_argument('-l','--lid_to_run', help='NWS LID, lowercase, to produce CatFIM for. Currently only accepts one. Default is all sites',
+                        required=False, default='all')
+    parser.add_argument('-ji','--job_number_intervals', help='Number of processes to use for inundating multiple intervals in stage-based'\
+        ' inundation and interval job numbers should multiply to no more than one less than the CPU count'\
+        ' of the machine.', required=False, default=1, type=int)
+    parser.add_argument('-mc','--past_major_interval_cap', help='Stage-Based Only. How many feet past major do you want to go for the interval FIMs?'\
+        ' of the machine.', required=False, default=5.0, type=float)    
+    
     args = vars(parser.parse_args())
+    process_generate_categorical_fim(**args)
 
-    # Get arguments
-    fim_version = args['fim_version']
-    number_of_jobs = args['number_of_jobs']
-
-    # Define default arguments. Modify these if necessary
-    fim_run_dir = Path(f'{fim_version}')
-    fim_version_folder = os.path.basename(fim_version)
-    output_flows_dir = Path(f'/data/catfim/{fim_version_folder}/flows')
-    output_mapping_dir = Path(f'/data/catfim/{fim_version_folder}/mapping')
-    nwm_us_search = '5'
-    nwm_ds_search = '5'
-    write_depth_tiff = False
-
-    ## Run CatFIM scripts in sequence
-    # Generate CatFIM flow files
-    print('Creating flow files')
-    start = time.time()
-    subprocess.call(['python3','/foss_fim/tools/generate_categorical_fim_flows.py', '-w' , str(output_flows_dir), '-u', nwm_us_search, '-d', nwm_ds_search])
-    end = time.time()
-    elapsed_time = (end-start)/60
-    print(f'Finished creating flow files in {elapsed_time} minutes')
-
-    # Generate CatFIM mapping
-    print('Begin mapping')
-    start = time.time()
-    subprocess.call(['python3','/foss_fim/tools/generate_categorical_fim_mapping.py', '-r' , str(fim_run_dir), '-s', str(output_flows_dir), '-o', str(output_mapping_dir), '-j', str(number_of_jobs)])
-    end = time.time()
-    elapsed_time = (end-start)/60
-    print(f'Finished mapping in {elapsed_time} minutes')
-
-    # Updating mapping status
-    print('Updating mapping status')
-    update_mapping_status(str(output_mapping_dir), str(output_flows_dir))
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index d2f5f0501..667b3ec9d 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 from pathlib import Path
 import pandas as pd
+import numpy as np
+import geopandas as gpd
 import time
-from tools_shared_functions import aggregate_wbd_hucs, mainstem_nwm_segs, get_thresholds, flow_data, get_metadata, get_nwm_segs
+from tools_shared_functions import aggregate_wbd_hucs, mainstem_nwm_segs, get_thresholds, flow_data, get_metadata, get_nwm_segs, get_datum, ngvd_to_navd_ft, filter_nwm_segments_by_stream_order
 import argparse
 from dotenv import load_dotenv
 import os
@@ -10,8 +12,6 @@
 sys.path.append('/foss_fim/src')
 from utils.shared_variables import VIZ_PROJECTION
 
-EVALUATED_SITES_CSV = r'/data/inputs/ahps_sites/evaluated_ahps_sites.csv'
-
 
 def get_env_paths():
     load_dotenv()
@@ -21,7 +21,7 @@ def get_env_paths():
     return API_BASE_URL, WBD_LAYER
 
 
-def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
+def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir=""):
     '''
     This will create static flow files for all nws_lids and save to the 
     workspace directory with the following format:
@@ -33,7 +33,6 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     This will use the WRDS API to get the nwm segments as well as the flow 
     values for each threshold at each nws_lid and then create the necessary 
     flow file to use for inundation mapping.
-
     Parameters
     ----------
     workspace : STR
@@ -48,10 +47,10 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     Returns
     -------
     None.
-
     '''
-    
+        
     all_start = time.time()
+    API_BASE_URL, WBD_LAYER = get_env_paths()
     #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
     workspace = Path(workspace)
     nwm_us_search = int(nwm_us_search)
@@ -63,44 +62,49 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     #Create workspace
     workspace.mkdir(parents=True,exist_ok = True)
 
-    print('Retrieving metadata...')
+    print(f'Retrieving metadata for site(s): {lid_to_run}...')
     #Get metadata for 'CONUS'
-    conus_list, conus_dataframe = get_metadata(metadata_url, select_by = 'nws_lid', selector = ['all'], must_include = 'nws_data.rfc_forecast_point', upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search )
-
-    #Get metadata for Islands
-    islands_list, islands_dataframe = get_metadata(metadata_url, select_by = 'state', selector = ['HI','PR'] , must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
-    
-    #Append the dataframes and lists
-    all_lists = conus_list + islands_list
-    
+    print(metadata_url)
+    if lid_to_run != 'all':
+        all_lists, conus_dataframe = get_metadata(metadata_url, select_by = 'nws_lid', selector = [lid_to_run], must_include = 'nws_data.rfc_forecast_point', upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+    else:
+        # Get CONUS metadata
+        conus_list, conus_dataframe = get_metadata(metadata_url, select_by = 'nws_lid', selector = ['all'], must_include = 'nws_data.rfc_forecast_point', upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        # Get metadata for Islands
+        islands_list, islands_dataframe = get_metadata(metadata_url, select_by = 'state', selector = ['HI','PR'] , must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        # Append the dataframes and lists
+        all_lists = conus_list + islands_list
+    print(len(all_lists))
     print('Determining HUC using WBD layer...')
-    #Assign HUCs to all sites using a spatial join of the FIM 3 HUC layer. 
-    #Get a dictionary of hucs (key) and sites (values) as well as a GeoDataFrame
-    #of all sites used later in script.
-    huc_dictionary, out_gdf = aggregate_wbd_hucs(metadata_list = all_lists, wbd_huc8_path = WBD_LAYER)
+    # Assign HUCs to all sites using a spatial join of the FIM 3 HUC layer. 
+    # Get a dictionary of hucs (key) and sites (values) as well as a GeoDataFrame
+    # of all sites used later in script.
+    huc_dictionary, out_gdf = aggregate_wbd_hucs(metadata_list = all_lists, wbd_huc8_path = WBD_LAYER, retain_attributes=True)
+    # Drop list fields if invalid
+    out_gdf = out_gdf.drop(['downstream_nwm_features'], axis=1, errors='ignore')
+    out_gdf = out_gdf.drop(['upstream_nwm_features'], axis=1, errors='ignore')
+    out_gdf = out_gdf.astype({'metadata_sources': str})
 
-    #Get all possible mainstem segments
-    print('Getting list of mainstem segments')
-    #Import list of evaluated sites
-    print(EVALUATED_SITES_CSV)
-    print(os.path.exists(EVALUATED_SITES_CSV))
-    list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
-    #The entire routine to get mainstems is hardcoded in this function.
-    ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
-    
-    #Loop through each huc unit, first define message variable and flood categories.
+    # Loop through each huc unit, first define message variable and flood categories.
     all_messages = []
     flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
+        
+    if stage_based:
+        return huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists
+    
     for huc in huc_dictionary:
         print(f'Iterating through {huc}')
         #Get list of nws_lids
         nws_lids = huc_dictionary[huc]
         #Loop through each lid in list to create flow file
-        for lid in nws_lids:
+        for lid in nws_lids:            
             #Convert lid to lower case
             lid = lid.lower()
             #Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
             stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
+            if stages == None or flows == None:
+                print("Likely WRDS error")
+                continue
             #Check if stages are supplied, if not write message and exit. 
             if all(stages.get(category, None)==None for category in flood_categories):
                 message = f'{lid}:missing threshold stages'
@@ -111,14 +115,15 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
                 message = f'{lid}:missing calculated flows'
                 all_messages.append(message)
                 continue
-
             #find lid metadata from master list of metadata dictionaries (line 66).
             metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
-        
+                                      
             #Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
-            segments = get_nwm_segs(metadata)        
-            site_ms_segs = set(segments).intersection(ms_segs)
-            segments = list(site_ms_segs)       
+            unfiltered_segments = list(set(get_nwm_segs(metadata)))
+            
+            desired_order = metadata['nwm_feature_data']['stream_order']
+            # Filter segments to be of like stream order.
+            segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order)
             #if no segments, write message and exit out
             if not segments:
                 print(f'{lid} no segments')
@@ -143,10 +148,11 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
                 else:
                     message = f'{lid}:{category} is missing calculated flow'
                     all_messages.append(message)
-
             #Get various attributes of the site.
-            lat = float(metadata['usgs_preferred']['latitude'])
-            lon = float(metadata['usgs_preferred']['longitude'])
+#            lat = float(metadata['usgs_preferred']['latitude'])
+#            lon = float(metadata['usgs_preferred']['longitude'])
+            lat = float(metadata['nws_preferred']['latitude'])
+            lon = float(metadata['nws_preferred']['longitude'])
             wfo = metadata['nws_data']['wfo']
             rfc = metadata['nws_data']['rfc']
             state = metadata['nws_data']['state']
@@ -159,7 +165,7 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
             wrds_timestamp = stages['wrds_timestamp']
             nrldb_timestamp = metadata['nrldb_timestamp']
             nwis_timestamp = metadata['nwis_timestamp']
-            
+                        
             #Create a csv with same information as shapefile but with each threshold as new record.
             csv_df = pd.DataFrame()
             for threshold in flood_categories:
@@ -172,41 +178,44 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
             output_dir = workspace / huc / lid
             if output_dir.exists():
                 #Export DataFrame to csv containing attributes
-                csv_df.to_csv(output_dir / f'{lid}_attributes.csv', index = False)
+                csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index = False)
+                message = f'{lid}:flows available'
+                all_messages.append(message)
             else:
                 message = f'{lid}:missing all calculated flows'
                 all_messages.append(message)
-        
-    print('wrapping up...')
+    print('Wrapping up...')
     #Recursively find all *_attributes csv files and append
-    csv_files = list(workspace.rglob('*_attributes.csv'))
+    csv_files = os.listdir(attributes_dir)
     all_csv_df = pd.DataFrame()
     for csv in csv_files:
-        #Huc has to be read in as string to preserve leading zeros.
-        temp_df = pd.read_csv(csv, dtype={'huc':str})
+        full_csv_path = os.path.join(attributes_dir, csv)
+        # Huc has to be read in as string to preserve leading zeros.
+        temp_df = pd.read_csv(full_csv_path, dtype={'huc':str})
         all_csv_df = all_csv_df.append(temp_df, ignore_index = True)
-    #Write to file
-    all_csv_df.to_csv(workspace / 'nws_lid_attributes.csv', index = False)
+    # Write to file
+    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index = False)
    
-    #This section populates a shapefile of all potential sites and details
-    #whether it was mapped or not (mapped field) and if not, why (status field).
+    # This section populates a shapefile of all potential sites and details
+    # whether it was mapped or not (mapped field) and if not, why (status field).
     
-    #Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
+    # Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
     viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)    
     viz_out_gdf.rename(columns = {'identifiers_nwm_feature_id': 'nwm_seg', 'identifiers_nws_lid':'nws_lid', 'identifiers_usgs_site_code':'usgs_gage'}, inplace = True)
     viz_out_gdf['nws_lid'] = viz_out_gdf['nws_lid'].str.lower()
     
-    #Using list of csv_files, populate DataFrame of all nws_lids that had
-    #a flow file produced and denote with "mapped" column.
-    nws_lids = [file.stem.split('_attributes')[0] for file in csv_files]
+    # Using list of csv_files, populate DataFrame of all nws_lids that had
+    # a flow file produced and denote with "mapped" column.
+    nws_lids = []
+    for csv_file in csv_files:
+        nws_lids.append(csv_file.split('_attributes')[0])
     lids_df = pd.DataFrame(nws_lids, columns = ['nws_lid'])
     lids_df['mapped'] = 'yes'
     
-    #Identify what lids were mapped by merging with lids_df. Populate 
-    #'mapped' column with 'No' if sites did not map.
+    # Identify what lids were mapped by merging with lids_df. Populate 
+    # 'mapped' column with 'No' if sites did not map.
     viz_out_gdf = viz_out_gdf.merge(lids_df, how = 'left', on = 'nws_lid')    
     viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
-    
     #Write messages to DataFrame, split into columns, aggregate messages.
     messages_df  = pd.DataFrame(all_messages, columns = ['message'])
     messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
@@ -218,13 +227,17 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     viz_out_gdf['status'] = viz_out_gdf['status'].fillna('all calculated flows available')
     
     #Filter out columns and write out to file
-    viz_out_gdf = viz_out_gdf.filter(['nws_lid','usgs_gage','nwm_seg','HUC8','mapped','status','geometry'])
-    viz_out_gdf.to_file(workspace /'nws_lid_flows_sites.shp')
+#    viz_out_gdf = viz_out_gdf.filter(['nws_lid','usgs_gage','nwm_seg','HUC8','mapped','status','geometry'])
+    nws_lid_layer = os.path.join(workspace, 'nws_lid_sites.gpkg').replace('flows', 'mapping')
+
+    viz_out_gdf.to_file(nws_lid_layer, driver='GPKG')
     
     #time operation
     all_end = time.time()
     print(f'total time is {round((all_end - all_start)/60),1} minutes')
     
+    return nws_lid_layer
+
     
 if __name__ == '__main__':
     #Parse arguments
@@ -232,8 +245,9 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search):
     parser.add_argument('-w', '--workspace', help = 'Workspace where all data will be stored.', required = True)
     parser.add_argument('-u', '--nwm_us_search',  help = 'Walk upstream on NWM network this many miles', required = True)
     parser.add_argument('-d', '--nwm_ds_search', help = 'Walk downstream on NWM network this many miles', required = True)
+    parser.add_argument('-a', '--stage_based', help = 'Run stage-based CatFIM instead of flow-based? NOTE: flow-based CatFIM is the default.', required=False, default=False, action='store_true')
+    parser.add_argument('-f', '--fim-dir', help='Path to FIM outputs directory. Only use this option if you are running in alt-catfim mode.',required=False,default="")
     args = vars(parser.parse_args())
 
     #Run get_env_paths and static_flow_lids
-    API_BASE_URL, WBD_LAYER = get_env_paths()
     generate_catfim_flows(**args)
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index 0827d3f08..743552664 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -2,33 +2,24 @@
 
 import sys
 import os
-from multiprocessing import Pool
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 import argparse
 import traceback
 import rasterio
 import geopandas as gpd
 import pandas as pd
-import shutil
 from rasterio.features import shapes
 from shapely.geometry.polygon import Polygon
 from shapely.geometry.multipolygon import MultiPolygon
-from inundation import inundate
 sys.path.append('/foss_fim/src')
 from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
 from utils.shared_functions import getDriver
+from gms_tools.mosaic_inundation import Mosaic_inundation
+from gms_tools.inundate_gms import Inundate_gms
 
-INPUTS_DIR = r'/data/inputs'
-magnitude_list = ['action', 'minor', 'moderate','major', 'record']
 
-# Define necessary variables for inundation()
-hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-mask_type, catchment_poly = 'huc', ''
-
-
-def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif, log_file):
-
-    no_data_list = []
-    procs_list = []
+def generate_categorical_fim(fim_run_dir, source_flow_dir, output_catfim_dir, 
+                             job_number_huc, job_number_inundate, depthtif, log_file):
 
     source_flow_dir_list = os.listdir(source_flow_dir)
     output_flow_dir_list = os.listdir(fim_run_dir)
@@ -43,36 +34,18 @@ def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, n
 
     # Loop through matching huc directories in the source_flow directory
     matching_hucs = list(set(output_flow_dir_list) & set(source_flow_dir_list))
-    for huc in matching_hucs:
-
-        if "." not in huc:
+    
+    with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
+        for huc in matching_hucs:
+            if "." in huc:
+                continue
 
             # Get list of AHPS site directories
             ahps_site_dir = os.path.join(source_flow_dir, huc)
             ahps_site_dir_list = os.listdir(ahps_site_dir)
 
-            # Map paths to HAND files needed for inundation()
-            fim_run_huc_dir = os.path.join(fim_run_dir, huc)
-            rem = os.path.join(fim_run_huc_dir, 'rem_zeroed_masked.tif')
-            catchments = os.path.join(fim_run_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-            hydroTable =  os.path.join(fim_run_huc_dir, 'hydroTable.csv')
-
-            exit_flag = False  # Default to False.
-
-            # Check if necessary data exist; set exit_flag to True if they don't exist
-            for f in [rem, catchments, hydroTable]:
-                if not os.path.exists(f):
-                    no_data_list.append(f)
-                    exit_flag = True
-
-            # Log missing data
-            if exit_flag == True:
-                f = open(log_file, 'a+')
-                f.write(f"Missing data for: {fim_run_huc_dir}\n")
-                f.close()
-
-            # Map path to huc directory inside out output_cat_fim_dir
-            cat_fim_huc_dir = os.path.join(output_cat_fim_dir, huc)
+            # Map path to huc directory inside out output_catfim_dir
+            cat_fim_huc_dir = os.path.join(output_catfim_dir, huc)
             if not os.path.exists(cat_fim_huc_dir):
                 os.mkdir(cat_fim_huc_dir)
 
@@ -89,49 +62,46 @@ def generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, n
 
                 # Loop through thresholds/magnitudes and define inundation output files paths
                 for magnitude in thresholds_dir_list:
-
-                    if "." not in magnitude:
-
-                        magnitude_flows_csv = os.path.join(ahps_site_parent, magnitude, 'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv')
-
-                        if os.path.exists(magnitude_flows_csv):
-
-                            output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
-
-                            if depthtif:
-                                output_depth_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_depth.tif')
-                            else:
-                                output_depth_grid = None
-
-                            # Append necessary variables to list for multiprocessing.
-                            procs_list.append([rem, catchments, magnitude_flows_csv, huc, hydroTable, output_extent_grid, output_depth_grid, ahps_site, magnitude, log_file])
-
-    # Initiate multiprocessing
-    print(f"Running inundation for {len(procs_list)} sites using {number_of_jobs} jobs")
-    with Pool(processes=number_of_jobs) as pool:
-        pool.map(run_inundation, procs_list)
-
-
-def run_inundation(args):
-
-    rem = args[0]
-    catchments = args[1]
-    magnitude_flows_csv = args[2]
-    huc = args[3]
-    hydroTable = args[4]
-    output_extent_grid = args[5]
-    output_depth_grid = args[6]
-    ahps_site = args[7]
-    magnitude = args[8]
-    log_file = args[9]
-
+                    if "." in magnitude:
+                        continue
+                    magnitude_flows_csv = os.path.join(ahps_site_parent, magnitude, 'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                    if os.path.exists(magnitude_flows_csv):
+                        output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
+                        try:
+                            executor.submit(run_inundation, magnitude_flows_csv, huc, output_extent_grid, ahps_site, magnitude, log_file, fim_run_dir, job_number_inundate)
+                        except Exception:
+                            traceback.print_exc()
+                            sys.exit(1)
+                                
+
+def run_inundation(magnitude_flows_csv, huc, output_extent_grid, ahps_site, magnitude, log_file, fim_run_dir, job_number_inundate):
+
+    huc_dir = os.path.join(fim_run_dir, huc)
     try:
-        inundate(rem,catchments,catchment_poly,hydroTable,magnitude_flows_csv,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=huc,num_workers=1,aggregate=False,inundation_raster=output_extent_grid,inundation_polygon=None,
-                 depths=output_depth_grid,out_raster_profile=None,out_vector_profile=None,quiet=True
-                )
-
-    except:
+        print("Running Inundate_gms for " + huc)
+        map_file = Inundate_gms(  hydrofabric_dir = fim_run_dir, 
+                                         forecast = magnitude_flows_csv, 
+                                         num_workers = job_number_inundate,
+                                         hucs = huc,
+                                         inundation_raster = output_extent_grid,
+                                         inundation_polygon = None,
+                                         depths_raster = None,
+                                         verbose = False,
+                                         log_file = None,
+                                         output_fileNames = None )
+        print("Mosaicking for " + huc)
+        Mosaic_inundation( map_file,
+                            mosaic_attribute = 'inundation_rasters',
+                            mosaic_output = output_extent_grid,
+                            mask = os.path.join(huc_dir,'wbd.gpkg'),
+                            unit_attribute_name = 'huc8',
+                            nodata = -9999,
+                            workers = 1,
+                            remove_inputs = False,
+                            subset = None,
+                            verbose = False )
+
+    except Exception:
         # Log errors and their tracebacks
         f = open(log_file, 'a+')
         f.write(f"{output_extent_grid} - inundation error: {traceback.format_exc()}\n")
@@ -147,143 +117,172 @@ def run_inundation(args):
             f.write('FAILURE_huc_{}:{}:{} map failed to create\n'.format(huc,ahps_site,magnitude))
 
 
-def post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir, nws_lid_attributes_filename, log_file):
-
+def post_process_huc_level(job_number_tif, ahps_dir_list, huc_dir, attributes_dir, gpkg_dir, fim_version, huc):
+    
+    tifs_to_reformat_list = []
+
+    # Loop through ahps sites
+    for ahps_lid in ahps_dir_list:
+        ahps_lid_dir = os.path.join(huc_dir, ahps_lid)
+
+        # Append desired filenames to list.
+        tif_list = os.listdir(ahps_lid_dir)
+        for tif in tif_list:
+            if 'extent.tif' in tif:
+                tifs_to_reformat_list.append(os.path.join(ahps_lid_dir, tif))
+
+        # Stage-Based CatFIM uses attributes from individual CSVs instead of the master CSV.
+        nws_lid_attributes_filename = os.path.join(attributes_dir, ahps_lid + '_attributes.csv')
+    
+        print(f"Reformatting TIFs {ahps_lid} for {huc_dir}")
+        with ProcessPoolExecutor(max_workers=job_number_tif) as executor:
+            for tif_to_process in tifs_to_reformat_list:
+                if not os.path.exists(tif_to_process):
+                    continue
+                try:
+                    magnitude = os.path.split(tif_to_process)[1].split('_')[1]
+                    try:
+                        interval_stage = float((os.path.split(tif_to_process)[1].split('_')[2]).replace('p', '.').replace("ft", ""))
+                        if interval_stage == 'extent':
+                            interval_stage = None
+                    except ValueError:
+                        interval_stage = None
+                    executor.submit(reformat_inundation_maps, ahps_lid, tif_to_process, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename, interval_stage)
+                except Exception as ex:
+                    print(f"*** {ex}")
+                    traceback.print_exc()
+                
+
+def post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file="", fim_version=""):
+    
+    print("In post processing...")
     # Create workspace
-    gpkg_dir = os.path.join(output_cat_fim_dir, 'gpkg')
+    gpkg_dir = os.path.join(output_catfim_dir, 'gpkg')
     if not os.path.exists(gpkg_dir):
         os.mkdir(gpkg_dir)
 
     # Find the FIM version
-    fim_version  = os.path.basename(output_cat_fim_dir)
-    merged_layer = os.path.join(output_cat_fim_dir, 'catfim_library.shp')
-
-    if not os.path.exists(merged_layer): # prevents appending to existing output
-
-        huc_ahps_dir_list = os.listdir(output_cat_fim_dir)
-        skip_list=['errors','logs','gpkg',merged_layer]
-
-        for magnitude in magnitude_list:
-
-            procs_list = []
-
-            # Loop through all categories
+    merged_layer = os.path.join(output_catfim_dir, 'catfim_library.gpkg')
+    if not os.path.exists(merged_layer): # prevents appending to existing output        
+        huc_ahps_dir_list = os.listdir(output_catfim_dir)
+        skip_list=['errors','logs','gpkg','missing_files.txt','messages',merged_layer]
+
+        # Loop through all categories
+        print("Building list of TIFs to reformat...")
+        with ProcessPoolExecutor(max_workers=job_number_huc) as huc_exector:
+            
             for huc in huc_ahps_dir_list:
 
-                if huc not in skip_list:
-
-                    huc_dir = os.path.join(output_cat_fim_dir, huc)
+                if huc in skip_list:
+                    continue
+                huc_dir = os.path.join(output_catfim_dir, huc)
+                try:
                     ahps_dir_list = os.listdir(huc_dir)
-
-                    # Loop through ahps sites
-                    for ahps_lid in ahps_dir_list:
-                        ahps_lid_dir = os.path.join(huc_dir, ahps_lid)
-
-                        extent_grid = os.path.join(ahps_lid_dir, ahps_lid + '_' + magnitude + '_extent_' + huc + '.tif')
-
-                        if os.path.exists(extent_grid):
-                            procs_list.append([ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename])
-
-                        else:
-                            try:
-                                f = open(log_file, 'a+')
-                                f.write(f"Missing layers: {extent_gpkg}\n")
-                                f.close()
-                            except:
-                                pass
-
-            # Multiprocess with instructions
-            with Pool(processes=number_of_jobs) as pool:
-                pool.map(reformat_inundation_maps, procs_list)
-
+                except NotADirectoryError:
+                    continue
+                # If there's no mapping for a HUC, delete the HUC directory.
+                if ahps_dir_list == []:
+                    os.rmdir(huc_dir)
+                    continue
+                
+                huc_exector.submit(post_process_huc_level, job_number_tif, ahps_dir_list, huc_dir, attributes_dir, gpkg_dir, fim_version, huc)
+                
         # Merge all layers
         print(f"Merging {len(os.listdir(gpkg_dir))} layers...")
-
         for layer in os.listdir(gpkg_dir):
-
+            # Open dissolved extent layers
             diss_extent_filename = os.path.join(gpkg_dir, layer)
-
-            # Open diss_extent
             diss_extent = gpd.read_file(diss_extent_filename)
             diss_extent['viz'] = 'yes'
-
+    
             # Write/append aggregate diss_extent
+            print(f"Merging layer: {layer}")
             if os.path.isfile(merged_layer):
                 diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False, mode='a')
             else:
                 diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False)
-
             del diss_extent
-
-        shutil.rmtree(gpkg_dir)
+    
+            #shutil.rmtree(gpkg_dir)  # TODO
 
     else:
         print(f"{merged_layer} already exists.")
 
 
-def reformat_inundation_maps(args):
+def reformat_inundation_maps(ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename, interval_stage=None):
 
     try:
-        lid = args[0]
-        grid_path = args[1]
-        gpkg_dir = args[2]
-        fim_version = args[3]
-        huc = args[4]
-        magnitude = args[5]
-        nws_lid_attributes_filename = args[6]
-
         # Convert raster to to shapes
-        with rasterio.open(grid_path) as src:
+        with rasterio.open(extent_grid) as src:
             image = src.read(1)
             mask = image > 0
-
+            
         # Aggregate shapes
         results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
 
         # Convert list of shapes to polygon
-        extent_poly  = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
-
+        extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
         # Dissolve polygons
         extent_poly_diss = extent_poly.dissolve(by='extent')
 
         # Update attributes
         extent_poly_diss = extent_poly_diss.reset_index(drop=True)
-        extent_poly_diss['ahps_lid'] = lid
+        extent_poly_diss['ahps_lid'] = ahps_lid
         extent_poly_diss['magnitude'] = magnitude
         extent_poly_diss['version'] = fim_version
         extent_poly_diss['huc'] = huc
-
+        extent_poly_diss['interval_stage'] = interval_stage
         # Project to Web Mercator
         extent_poly_diss = extent_poly_diss.to_crs(VIZ_PROJECTION)
 
         # Join attributes
         nws_lid_attributes_table = pd.read_csv(nws_lid_attributes_filename, dtype={'huc':str})
-        nws_lid_attributes_table = nws_lid_attributes_table.loc[(nws_lid_attributes_table.magnitude==magnitude) & (nws_lid_attributes_table.nws_lid==lid)]
-
-
+        nws_lid_attributes_table = nws_lid_attributes_table.loc[(nws_lid_attributes_table.magnitude==magnitude) & (nws_lid_attributes_table.nws_lid==ahps_lid)]
         extent_poly_diss = extent_poly_diss.merge(nws_lid_attributes_table, left_on=['ahps_lid','magnitude','huc'], right_on=['nws_lid','magnitude','huc'])
-
         extent_poly_diss = extent_poly_diss.drop(columns='nws_lid')
-
         # Save dissolved multipolygon
-        handle = os.path.split(grid_path)[1].replace('.tif', '')
-
-        diss_extent_filename = os.path.join(gpkg_dir, handle + "_dissolved.gpkg")
-
+        handle = os.path.split(extent_grid)[1].replace('.tif', '')
+        diss_extent_filename = os.path.join(gpkg_dir, f"{handle}_{huc}_dissolved.gpkg")
         extent_poly_diss["geometry"] = [MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in extent_poly_diss["geometry"]]
-
+        
         if not extent_poly_diss.empty:
-
             extent_poly_diss.to_file(diss_extent_filename,driver=getDriver(diss_extent_filename),index=False)
 
-    except Exception as e:
+    except Exception:
+        pass
         # Log and clean out the gdb so it's not merged in later
-        try:
-            f = open(log_file, 'a+')
-            f.write(str(diss_extent_filename) + " - dissolve error: " + str(e))
-            f.close()
-        except:
-            pass
+#        try:
+#            print(e)
+##            f = open(log_file, 'a+')
+##            f.write(str(diss_extent_filename) + " - dissolve error: " + str(e))
+##            f.close()
+#        except:
+#            pass
+
+
+def manage_catfim_mapping(fim_run_dir, source_flow_dir, output_catfim_dir, attributes_dir, job_number_huc, job_number_inundate, overwrite, depthtif):
+    
+    # Create output directory
+    if not os.path.exists(output_catfim_dir):
+        os.mkdir(output_catfim_dir)
+
+    # Create log directory
+    log_dir = os.path.join(output_catfim_dir, 'logs')
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+
+    # Create error log path
+    log_file = os.path.join(log_dir, 'errors.log')
+
+    job_number_tif = job_number_inundate
+
+    print("Generating Categorical FIM")
+    generate_categorical_fim(fim_run_dir, source_flow_dir, output_catfim_dir, job_number_huc, job_number_inundate, depthtif, log_file)
+
+    print("Aggregating Categorical FIM")
+    # Get fim_version.
+    fim_version = os.path.basename(os.path.normpath(fim_run_dir)).replace('fim_','').replace('_ms_c', '').replace('_', '.')
+    post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file, fim_version)
 
 
 if __name__ == '__main__':
@@ -292,7 +291,7 @@ def reformat_inundation_maps(args):
     parser = argparse.ArgumentParser(description='Categorical inundation mapping for FOSS FIM.')
     parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
     parser.add_argument('-s', '--source-flow-dir',help='Path to directory containing flow CSVs to use to generate categorical FIM.',required=True, default="")
-    parser.add_argument('-o', '--output-cat-fim-dir',help='Path to directory where categorical FIM outputs will be written.',required=True, default="")
+    parser.add_argument('-o', '--output-catfim-dir',help='Path to directory where categorical FIM outputs will be written.',required=True, default="")
     parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
     parser.add_argument('-depthtif','--write-depth-tiff',help='Using this option will write depth TIFFs.',required=False, action='store_true')
 
@@ -300,27 +299,8 @@ def reformat_inundation_maps(args):
 
     fim_run_dir = args['fim_run_dir']
     source_flow_dir = args['source_flow_dir']
-    output_cat_fim_dir = args['output_cat_fim_dir']
+    output_catfim_dir = args['output_catfim_dir']
     number_of_jobs = int(args['number_of_jobs'])
     depthtif = args['write_depth_tiff']
-
-    # Create output directory
-    if not os.path.exists(output_cat_fim_dir):
-        os.mkdir(output_cat_fim_dir)
-
-    # Create log directory
-    log_dir = os.path.join(output_cat_fim_dir, 'logs')
-    if not os.path.exists(log_dir):
-        os.mkdir(log_dir)
-
-    # Create error log path
-    log_file = os.path.join(log_dir, 'errors.log')
-
-    # Map path to points with attributes
-    nws_lid_attributes_filename = os.path.join(source_flow_dir, 'nws_lid_attributes.csv')
-
-    print("Generating Categorical FIM")
-    generate_categorical_fim(fim_run_dir, source_flow_dir, output_cat_fim_dir, number_of_jobs, depthtif,log_file)
-
-    print("Aggregating Categorical FIM")
-    post_process_cat_fim_for_viz(number_of_jobs, output_cat_fim_dir,nws_lid_attributes_filename,log_file)
+    
+    manage_catfim_mapping(fim_run_dir, source_flow_dir, output_catfim_dir, number_of_jobs, depthtif)
diff --git a/tools/gms_tools/inundate_gms.py b/tools/gms_tools/inundate_gms.py
index 4437fadab..7b481dd70 100755
--- a/tools/gms_tools/inundate_gms.py
+++ b/tools/gms_tools/inundate_gms.py
@@ -77,11 +77,9 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
     hucCodes = [None] * number_of_branches
     branch_ids = [None] * number_of_branches
        
-
     executor_generator = { 
                 executor.submit(inundate,**inp) : ids for inp,ids in inundate_input_generator 
-                }
-
+                }          
     idx = 0
     for future in tqdm(as_completed(executor_generator),
                        total=len(executor_generator),
diff --git a/tools/pixel_counter.py b/tools/pixel_counter.py
index b7ca06602..e4a96dce9 100644
--- a/tools/pixel_counter.py
+++ b/tools/pixel_counter.py
@@ -96,6 +96,8 @@ def make_flood_extent_polygon(flood_extent):
     print(type(fullpath))
 
     return fullpath
+
+
 # Function that transforms vector dataset to raster
 def bbox_to_pixel_offsets(gt, bbox):
     originX = gt[0]
@@ -239,7 +241,10 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
 
     vds = None
     rds = None
-    return stats
+    if stats != []:
+        return stats
+    else:
+        return []
 
 
 # Creates and prints dataframe containing desired statistics
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
index cb8a33f56..2c152850b 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -6,10 +6,16 @@
 from tools_shared_functions import get_metadata, get_datum, ngvd_to_navd_ft, get_rating_curve, aggregate_wbd_hucs, get_thresholds, flow_data
 from dotenv import load_dotenv
 import os
+import numpy as np
 import argparse
 import sys
 sys.path.append('/foss_fim/src')
 from utils.shared_variables import PREP_PROJECTION
+from tools_shared_variables import (acceptable_coord_acc_code_list, 
+                                    acceptable_coord_method_code_list, 
+                                    acceptable_alt_acc_thresh, 
+                                    acceptable_alt_meth_code_list, 
+                                    acceptable_site_type_list)
 
 '''
 This script calls the NOAA Tidal API for datum conversions. Experience shows that
@@ -26,9 +32,10 @@
 EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
 NWM_FLOWS_MS = os.getenv("NWM_FLOWS_MS")
 
+
 def get_all_active_usgs_sites():
     '''
-    Compile a list of all active usgs gage sites that meet certain criteria. 
+    Compile a list of all active usgs gage sites. 
     Return a GeoDataFrame of all sites.
 
     Returns
@@ -43,61 +50,15 @@ def get_all_active_usgs_sites():
     selector = ['all']
     must_include = 'usgs_data.active'
     metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = None )
-
-    #Filter out sites based quality of site. These acceptable codes were initially
-    #decided upon and may need fine tuning. A link where more information
-    #regarding the USGS attributes is provided. 
-    
-    #https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html
-    acceptable_coord_acc_code = ['H','1','5','S','R','B','C','D','E']
-    #https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html
-    acceptable_coord_method_code = ['C','D','W','X','Y','Z','N','M','L','G','R','F','S']
-    #https://help.waterdata.usgs.gov/codes-and-parameters/codes#SI
-    acceptable_alt_acc_thresh = 1
-    #https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html
-    acceptable_alt_meth_code = ['A','D','F','I','J','L','N','R','W','X','Y','Z']
-    #https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html
-    acceptable_site_type = ['ST']
-    
-    #Cycle through each site and filter out if site doesn't meet criteria.
-    acceptable_sites_metadata = []
-    for metadata in metadata_list:
-        #Get the usgs info from each site
-        usgs_data = metadata['usgs_data']
-                
-        #Get site quality attributes      
-        coord_accuracy_code = usgs_data.get('coord_accuracy_code')        
-        coord_method_code =   usgs_data.get('coord_method_code')
-        alt_accuracy_code =   usgs_data.get('alt_accuracy_code')
-        alt_method_code =     usgs_data.get('alt_method_code')
-        site_type =           usgs_data.get('site_type')
-        
-        #Check to make sure that none of the codes were null, if null values are found, skip to next.
-        if not all([coord_accuracy_code, coord_method_code, alt_accuracy_code, alt_method_code, site_type]):
-            continue
-        
-        #Test if site meets criteria.
-        if (coord_accuracy_code in acceptable_coord_acc_code and 
-            coord_method_code in acceptable_coord_method_code and
-            alt_accuracy_code <= acceptable_alt_acc_thresh and 
-            alt_method_code in acceptable_alt_meth_code and
-            site_type in acceptable_site_type):
-            
-            #If nws_lid is not populated then add a dummy ID so that 'aggregate_wbd_hucs' works correctly.
-            if not metadata.get('identifiers').get('nws_lid'):
-                metadata['identifiers']['nws_lid'] = 'Bogus_ID' 
-            
-            #Append metadata of acceptable site to acceptable_sites list.
-            acceptable_sites_metadata.append(metadata)  
-        
     #Get a geospatial layer (gdf) for all acceptable sites
-    dictionary, gdf = aggregate_wbd_hucs(acceptable_sites_metadata, Path(WBD_LAYER), retain_attributes = False)
+    print("Aggregating WBD HUCs...")
+    dictionary, gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes=True)
     #Get a list of all sites in gdf
     list_of_sites = gdf['identifiers_usgs_site_code'].to_list()
     #Rename gdf fields
     gdf.columns = gdf.columns.str.replace('identifiers_','')
 
-    return gdf, list_of_sites, acceptable_sites_metadata
+    return gdf, list_of_sites, metadata_list
             
 ##############################################################################
 #Generate categorical flows for each category across all sites.
@@ -217,14 +178,19 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
         all input sites. Additional metadata also contained in DataFrame
 
     '''
+    
     #Define URLs for metadata and rating curve
     metadata_url = f'{API_BASE_URL}/metadata'
     rating_curve_url = f'{API_BASE_URL}/rating_curve'
 
-    #If 'all' option passed to list of gages sites, it retrieves all acceptable sites within CONUS.
+    # Create workspace directory if it doesn't exist
+    if not os.path.exists(workspace):
+        os.mkdir(workspace)
+    
+    #If 'all' option passed to list of gages sites, it retrieves all sites within CONUS.
     print('getting metadata for all sites')
     if list_of_gage_sites == ['all']:
-        acceptable_sites_gdf, acceptable_sites_list, metadata_list = get_all_active_usgs_sites()
+        sites_gdf, sites_list, metadata_list = get_all_active_usgs_sites()
     #Otherwise, if a list of sites is passed, retrieve sites from WRDS.
     else:        
         #Define arguments to retrieve metadata and then get metadata from WRDS
@@ -254,22 +220,24 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
     #For each site in metadata_list
     for metadata in metadata_list:
         
+        print("get_datum")
         #Get datum information for site (only need usgs_data)
         nws, usgs = get_datum(metadata)        
         
         #Filter out sites that are not in contiguous US. If this section is removed be sure to test with datum adjustment section (region will need changed)
         if usgs['state'] in ['Alaska', 'Puerto Rico', 'Virgin Islands', 'Hawaii']:
             continue        
-        
         #Get rating curve for site
         location_ids = usgs['usgs_site_code']
+        if location_ids == None:  # Some sites don't have a value for usgs_site_code, skip them
+            continue
         curve = get_rating_curve(rating_curve_url, location_ids = [location_ids])
         #If no rating curve was returned, skip site.
         if curve.empty:
             message = f'{location_ids}: has no rating curve'
             regular_messages.append(message)
             continue
-
+        
         #Adjust datum to NAVD88 if needed. If datum unknown, skip site.
         if usgs['vcs'] == 'NGVD29':
             #To prevent time-out errors
@@ -283,7 +251,6 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
                 api_failure_messages.append(api_message)
                 print(api_message)
                 continue
-
             #If datum adjustment succeeded, calculate datum in NAVD88            
             navd88_datum = round(usgs['datum'] + datum_adj_ft, 2)
             message = f'{location_ids}:succesfully converted NGVD29 to NAVD88'
@@ -298,7 +265,8 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
             message = f"{location_ids}: datum unknown"
             regular_messages.append(message)
             continue
-
+        
+        print("Populating..")
         #Populate rating curve with metadata and use navd88 datum to convert stage to elevation.
         curve['active'] = usgs['active']
         curve['datum'] = usgs['datum']
@@ -306,23 +274,53 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
         curve['navd88_datum'] = navd88_datum
         curve['elevation_navd88'] = curve['stage'] + navd88_datum
         #Append all rating curves to a dataframe
-        all_rating_curves = all_rating_curves.append(curve)        
+        all_rating_curves = all_rating_curves.append(curve)
 
     #Rename columns and add attribute indicating if rating curve exists
-    acceptable_sites_gdf.rename(columns = {'nwm_feature_id':'feature_id','usgs_site_code':'location_id'}, inplace = True)
+    sites_gdf.rename(columns = {'nwm_feature_id':'feature_id','usgs_site_code':'location_id'}, inplace = True)
     sites_with_data = pd.DataFrame({'location_id':all_rating_curves['location_id'].unique(),'curve':'yes'})
-    acceptable_sites_gdf = acceptable_sites_gdf.merge(sites_with_data, on = 'location_id', how = 'left')
-    acceptable_sites_gdf.fillna({'curve':'no'},inplace = True)    
+    sites_gdf = sites_gdf.merge(sites_with_data, on = 'location_id', how = 'left')
+    sites_gdf.fillna({'curve':'no'},inplace = True)    
     #Add mainstems attribute to acceptable sites
     print('Attributing mainstems sites')
     #Import mainstems segments used in run_by_unit.sh
     ms_df = gpd.read_file(NWM_FLOWS_MS)
     ms_segs = ms_df.ID.astype(str).to_list()
     #Populate mainstems attribute field
-    acceptable_sites_gdf['mainstem'] = 'no'
-    acceptable_sites_gdf.loc[acceptable_sites_gdf.eval('feature_id in @ms_segs'),'mainstem'] = 'yes' 
+    sites_gdf['mainstem'] = 'no'
+    sites_gdf.loc[sites_gdf.eval('feature_id in @ms_segs'),'mainstem'] = 'yes' 
+    sites_gdf.to_csv(os.path.join(workspace, 'acceptable_sites_pre.csv'))
+    
+    sites_gdf = sites_gdf.drop(['upstream_nwm_features'], axis=1, errors='ignore')
+    sites_gdf = sites_gdf.drop(['downstream_nwm_features'], axis=1, errors='ignore')
     
+    print("Recasting...")
+    sites_gdf = sites_gdf.astype({'metadata_sources': str})
     
+    # -- Filter all_rating_curves according to acceptance criteria -- #
+    # -- We only want acceptable gages in the rating curve CSV -- #           
+    sites_gdf['acceptable_codes'] = (sites_gdf['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
+                                    & sites_gdf['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
+                                    & sites_gdf['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
+                                    & sites_gdf['usgs_data_site_type'].isin(acceptable_site_type_list))
+    
+    sites_gdf = sites_gdf.astype({'usgs_data_alt_accuracy_code': float})
+    sites_gdf['acceptable_alt_error'] = np.where(sites_gdf['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False)
+    
+    sites_gdf.to_file(os.path.join(workspace, 'sites_bool_flags.gpkg'), driver='GPKG')
+        
+    # Filter and save filtered file for viewing
+    acceptable_sites_gdf = sites_gdf[(sites_gdf['acceptable_codes'] == True) & (sites_gdf['acceptable_alt_error'] == True)]
+    acceptable_sites_gdf = acceptable_sites_gdf[acceptable_sites_gdf['curve'] == 'yes']
+    acceptable_sites_gdf.to_csv(os.path.join(workspace, 'acceptable_sites_for_rating_curves.csv'))
+    acceptable_sites_gdf.to_file(os.path.join(workspace, 'acceptable_sites_for_rating_curves.gpkg'),driver='GPKG')
+    
+    # Make list of acceptable sites
+    acceptable_sites_list = acceptable_sites_gdf['location_id'].tolist()
+        
+    # Filter out all_rating_curves by list
+    all_rating_curves = all_rating_curves[all_rating_curves['location_id'].isin(acceptable_sites_list)]
+
     #If workspace is specified, write data to file.
     if workspace:
         #Write rating curve dataframe to file
@@ -334,10 +332,10 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
         regular_messages = api_failure_messages + regular_messages                
         all_messages = pd.DataFrame({'Messages':regular_messages})
         all_messages.to_csv(Path(workspace) / 'log.csv', index = False)
-        #If 'all' option specified, reproject then write out shapefile of acceptable sites.
+        # If 'all' option specified, reproject then write out shapefile of acceptable sites.
         if list_of_gage_sites == ['all']:            
-            acceptable_sites_gdf = acceptable_sites_gdf.to_crs(PREP_PROJECTION)
-            acceptable_sites_gdf.to_file(Path(workspace) / 'usgs_gages.gpkg', layer = 'usgs_gages', driver = 'GPKG')
+            sites_gdf = sites_gdf.to_crs(PREP_PROJECTION)
+            sites_gdf.to_file(Path(workspace) / 'usgs_gages.gpkg', layer = 'usgs_gages', driver = 'GPKG')
         
         #Write out flow files for each threshold across all sites
         all_data = write_categorical_flow_files(metadata_list, workspace)
@@ -366,5 +364,6 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
     t = float(args['sleep_timer'])           
     
     #Generate USGS rating curves
+    print("Executing...")
     usgs_rating_to_elev(list_of_gage_sites = l, workspace=w, sleep_time = t)
     
\ No newline at end of file
diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py
index 3db6c7d6c..4d7657ff6 100644
--- a/tools/test_case_by_hydro_id.py
+++ b/tools/test_case_by_hydro_id.py
@@ -21,6 +21,7 @@ def perform_zonal_stats(huc_gpkg,agree_rast):
     stats = zonal_stats(huc_gpkg,{"agreement_raster":agree_rast}, nodata_value=10)
     return stats
 
+
 #####################################################
 # Creates a pandas df containing Alpha stats by hydroid.
 # Stats input is the output of zonal_stats function.
@@ -154,48 +155,49 @@ def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
         if not os.path.exists(test_case_class.fim_dir):
             print(f'{test_case_class.fim_dir} does not exist')
             continue
-        else:
             
-            print(test_case_class.fim_dir, end='\r')
-                       
-            agreement_dict = test_case_class.get_current_agreements()
-           
-            for agree_rast in agreement_dict:
-                    
-                    print('performing_zonal_stats')
-
-                    branches_dir = os.path.join(test_case_class.fim_dir,'branches')
-                    for branches in os.listdir(branches_dir):
-                        if branches != "0":
-                            continue
-                        huc_gpkg = os.path.join(branches_dir,branches,)
-                                                
-                        string_manip = "gw_catchments_reaches_filtered_addedAttributes_crosswalked_" + branches + ".gpkg"
-                        
-                        huc_gpkg = os.path.join(huc_gpkg, string_manip)
-                        
-                        define_mag = agree_rast.split(version)
-
-                        define_mag_1 = define_mag[1].split('/')
-                        
-                        mag = define_mag_1[1]
+        print(test_case_class.fim_dir, end='\r')
                    
-                        stats = perform_zonal_stats(huc_gpkg,agree_rast)
-                        
-                        print('assembling_hydroalpha_for_single_huc')
-                        get_geom = gpd.read_file(huc_gpkg)
-                        
-                        get_geom['geometry'] = get_geom.apply(lambda row: make_valid(row.geometry), axis=1)
+        agreement_dict = test_case_class.get_current_agreements()
+       
+        for agree_rast in agreement_dict:
+                
+                print('performing_zonal_stats')
+
+                branches_dir = os.path.join(test_case_class.fim_dir,'branches')
+                for branches in os.listdir(branches_dir):
+                    if branches != "0":
+                        continue
+                    huc_gpkg = os.path.join(branches_dir,branches,)
                                             
-                        in_mem_df = assemble_hydro_alpha_for_single_huc(stats, test_case_class.huc, mag, test_case_class.benchmark_cat)
-                        
-                        hydro_geom_df = get_geom[["HydroID", "geometry"]]
-                        
-                        geom_output = hydro_geom_df.merge(in_mem_df, on='HydroID', how ='inner')
-                        
-                        concat_df_list = [geom_output, csv_output]
-
-                        csv_output = pd.concat(concat_df_list, sort=False)
+                    string_manip = "gw_catchments_reaches_filtered_addedAttributes_crosswalked_" + branches + ".gpkg"
+                    
+                    huc_gpkg = os.path.join(huc_gpkg, string_manip)
+                    
+                    define_mag = agree_rast.split(version)
+
+                    define_mag_1 = define_mag[1].split('/')
+                    
+                    mag = define_mag_1[1]
+               
+                    stats = perform_zonal_stats(huc_gpkg,agree_rast)
+                    if stats == []:
+                        continue
+                    
+                    print('assembling_hydroalpha_for_single_huc')
+                    get_geom = gpd.read_file(huc_gpkg)
+                    
+                    get_geom['geometry'] = get_geom.apply(lambda row: make_valid(row.geometry), axis=1)
+                                        
+                    in_mem_df = assemble_hydro_alpha_for_single_huc(stats, test_case_class.huc, mag, test_case_class.benchmark_cat)
+                    
+                    hydro_geom_df = get_geom[["HydroID", "geometry"]]
+                    
+                    geom_output = hydro_geom_df.merge(in_mem_df, on='HydroID', how ='inner')
+                    
+                    concat_df_list = [geom_output, csv_output]
+
+                    csv_output = pd.concat(concat_df_list, sort=False)
                                            
     print('projecting to 3857')
     csv_output = csv_output.to_crs('EPSG:3857')
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 119aee6c5..cc317e867 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -16,6 +16,42 @@
 from shapely.geometry import shape
 from shapely.geometry import Polygon
 from shapely.geometry import MultiPolygon
+from dotenv import load_dotenv
+
+
+def get_env_paths():
+    load_dotenv()
+    #import variables from .env file
+    API_BASE_URL = os.getenv("API_BASE_URL")
+    WBD_LAYER = os.getenv("WBD_LAYER")
+    return API_BASE_URL, WBD_LAYER
+
+
+def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order):
+    """
+    This function uses the WRDS API to filter out NWM segments from a list if their stream order is different than
+    the target stream order.
+    
+    Args:
+        unfiltered_segments (list):  A list of NWM feature_id strings.
+        desired_order (str): The desired stream order.
+    Returns:
+        filtered_segments (list): A list of NWM feature_id strings, paired down to only those that share the target order.        
+    
+    """
+    API_BASE_URL, WBD_LAYER = get_env_paths()
+    #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
+    metadata_url = f'{API_BASE_URL}/metadata'
+    
+    filtered_segments = []
+    for feature_id in unfiltered_segments:
+        all_lists = get_metadata(metadata_url, select_by = 'nwm_feature_id', selector = [feature_id])
+        feature_id_metadata = next(item for item in all_lists)[0]
+        stream_order = feature_id_metadata['nwm_feature_data']['stream_order']
+        if stream_order == desired_order:
+            filtered_segments.append(feature_id)
+
+    return filtered_segments
 
 
 def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
@@ -613,6 +649,8 @@ def get_metadata(metadata_url, select_by, selector, must_include = None, upstrea
     params['downstream_trace_distance'] = downstream_trace_distance
     #Request data from url
     response = requests.get(url, params = params)
+#    print(response)
+#    print(url)
     if response.ok:
         #Convert data response to a json
         metadata_json = response.json()
@@ -685,13 +723,16 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
 
     '''
     #Import huc8 layer as geodataframe and retain necessary columns
+    print("Reading WBD...")
     huc8 = gpd.read_file(wbd_huc8_path, layer = 'WBDHU8')
+    print("WBD read.")
     huc8 = huc8[['HUC8','name','states', 'geometry']]
     #Define EPSG codes for possible latlon datum names (default of NAD83 if unassigned)
     crs_lookup ={'NAD27':'EPSG:4267', 'NAD83':'EPSG:4269', 'WGS84': 'EPSG:4326'}
     #Create empty geodataframe and define CRS for potential horizontal datums
     metadata_gdf = gpd.GeoDataFrame()
     #Iterate through each site
+    print("Iterating through metadata list...")
     for metadata in metadata_list:
         #Convert metadata to json
         df = pd.json_normalize(metadata)
@@ -701,6 +742,7 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
         df.dropna(subset = ['identifiers_nws_lid','usgs_preferred_latitude', 'usgs_preferred_longitude'], inplace = True)
         #If dataframe still has data
         if not df.empty:
+#            print(df[:5])
             #Get horizontal datum
             h_datum = df['usgs_preferred_latlon_datum_name'].item()
             #Look up EPSG code, if not returned Assume NAD83 as default. 
@@ -716,15 +758,16 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
             site_gdf = site_gdf.to_crs(huc8.crs)
             #Append site geodataframe to metadata geodataframe
             metadata_gdf = metadata_gdf.append(site_gdf, ignore_index = True)
-
+    
     #Trim metadata to only have certain fields.
     if not retain_attributes:
         metadata_gdf = metadata_gdf[['identifiers_nwm_feature_id', 'identifiers_nws_lid', 'identifiers_usgs_site_code', 'geometry']]
     #If a list of attributes is supplied then use that list.
-    elif isinstance(retain_attributes,list):
-        metadata_gdf = metadata_gdf[retain_attributes]
-
+#    elif isinstance(retain_attributes,list):
+#        metadata_gdf = metadata_gdf[retain_attributes]
+    print("Performing spatial and tabular operations on geodataframe...")
     #Perform a spatial join to get the WBD HUC 8 assigned to each AHPS
+    print(metadata_gdf)
     joined_gdf = gpd.sjoin(metadata_gdf, huc8, how = 'inner', op = 'intersects', lsuffix = 'ahps', rsuffix = 'wbd')
     joined_gdf = joined_gdf.drop(columns = 'index_wbd')
 
@@ -913,7 +956,10 @@ def get_thresholds(threshold_url, select_by, selector, threshold = 'all'):
                 flows['usgs_site_code'] = threshold_data.get('metadata').get('usgs_site_code')
                 stages['units'] = threshold_data.get('metadata').get('stage_units')
                 flows['units'] = threshold_data.get('metadata').get('calc_flow_units')
-    return stages, flows
+        return stages, flows
+    else:
+        print("WRDS response error: ")
+#        print(response)
 
 ########################################################################
 # Function to write flow file
@@ -1071,7 +1117,7 @@ def ngvd_to_navd_ft(datum_info, region = 'contiguous'):
         lon = datum_info['lon']
     
     #Define url for datum API
-    datum_url = 'https://vdatum.noaa.gov/vdatumweb/api/tidal'     
+    datum_url = 'https://vdatum.noaa.gov/vdatumweb/api/convert'     
     
     #Define parameters. Hard code most parameters to convert NGVD to NAVD.    
     params = {}
@@ -1087,16 +1133,18 @@ def ngvd_to_navd_ft(datum_info, region = 'contiguous'):
     
     #Call the API
     response = requests.get(datum_url, params = params)
-    #If succesful get the navd adjustment
+
+    #If successful get the navd adjustment
     if response:
         results = response.json()
         #Get adjustment in meters (NGVD29 to NAVD88)
-        adjustment = results['tar_height']
+        adjustment = results['t_z']
         #convert meters to feet
         adjustment_ft = round(float(adjustment) * 3.28084,2)                
     else:
         adjustment_ft = None
-    return adjustment_ft       
+    return adjustment_ft    
+   
 #######################################################################
 #Function to download rating curve from API
 #######################################################################
@@ -1121,6 +1169,7 @@ def get_rating_curve(rating_curve_url, location_ids):
     #Define DataFrame to contain all returned curves.
     all_curves = pd.DataFrame()
     
+    print(location_ids)
     #Define call to retrieve all rating curve information from WRDS.
     joined_location_ids = '%2C'.join(location_ids)
     url = f'{rating_curve_url}/{joined_location_ids}'
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index 82d63d0b0..e63d1cf80 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -61,3 +61,16 @@
 TWHITE = '\033[37m'
 WHITE_BOLD = '\033[37;1m'
 CYAN_BOLD = '\033[36;1m'
+
+# USGS gages acceptance criteria. Likely not constants, so not using all caps.
+# ANY CHANGE TO THESE VALUES SHOULD WARRANT A CODE VERSION CHANGE
+# https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html
+acceptable_coord_acc_code_list = ['H','1','5','S','R','B','C','D','E']
+# https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html
+acceptable_coord_method_code_list = ['C','D','W','X','Y','Z','N','M','L','G','R','F','S']
+# https://help.waterdata.usgs.gov/codes-and-parameters/codes#SI
+acceptable_alt_acc_thresh = 1.0
+# https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html
+acceptable_alt_meth_code_list = ['A','D','F','I','J','L','N','R','W','X','Y','Z']
+# https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html
+acceptable_site_type_list = ['ST']

From 99f16010cb36e245f005fda93d8b029ad3f21203 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 23 Dec 2022 14:03:40 -0600
Subject: [PATCH 191/359] Make GMS HAND post-processing independent

---
 docs/CHANGELOG.md                             |  54 +++++-
 gms_pipeline.sh                               |  52 ++---
 gms_run_branch.sh                             | 159 +++------------
 gms_run_post_processing.sh                    | 182 ++++++++++++++++++
 gms_run_unit.sh                               |  75 ++------
 src/bash_functions.env                        |  10 +
 src/filter_catchments_and_add_attributes.py   |  22 +--
 src/gms/delineate_hydros_and_produce_HAND.sh  |   4 +-
 src/gms/derive_level_paths.py                 |  56 +++---
 src/gms/run_by_unit.sh                        |  19 +-
 src/split_flows.py                            |  25 +--
 src/usgs_gage_unit_setup.py                   |   9 +-
 unit_tests/gms/derive_level_paths_params.json |   6 +-
 .../gms/derive_level_paths_unittests.py       |  94 +--------
 14 files changed, 354 insertions(+), 413 deletions(-)
 create mode 100755 gms_run_post_processing.sh

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 401a91b75..90abdebc9 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,56 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.16.0 - 2022-12-20 - [PR #768](https://github.com/NOAA-OWP/inundation-mapping/pull/768)
+
+`gms_run_branch.sh` was processing all of the branches iteratively, then continuing on to a large post processing portion of code. That has now be split to two files, one for branch iteration and the other file for just post processing.
+
+Other minor changes include:
+- Removing the system where a user could override `DropStreamOrders` where they could process streams with stream orders 1 and 2 independently like other GMS branches.  This option is now removed, so it will only allow stream orders 3 and higher as gms branches and SO 1 and 2 will always be in branch zero.
+
+- The `retry` flag on the three gms*.sh files has been removed. It did not work correctly and was not being used. Usage of it would have created unreliable results. 
+
+### Additions
+
+- `gms_run_post_processing.sh`
+   - handles all tasks from after `gms_run_branch.sh` to this file, except for output cleanup, which stayed in `gms_run_branch.sh`.
+   - Can be run completely independent from `gms_run_unit.sh` or gms_run_branch.sh` as long as all of the files are in place. And can be re-run if desired.
+
+### Changes
+
+- `gms_pipeline.sh`
+   - Remove "retry" system.
+   - Remove "dropLowStreamOrders" system.
+   - Updated for newer reusable output date/time/duration system.
+   - Add call to new `gms_run_post_processing.sh` file.
+
+- `gms_run_branch.sh`
+   - Remove "retry" system.
+   - Remove "dropLowStreamOrders" system.
+   - Updated for newer reusable output date/time/duration system.
+   - Removed most code from below the branch iterator to the new `gms_run_post_processing.sh` file. However, it did keep the branch files output cleanup and non-zero exit code checking.
+
+- `gms_run_unit.sh`
+   - Remove "retry" system.
+   - Remove "dropLowStreamOrders" system.
+   - Updated for newer reusable output date/time/duration system.
+
+- `src`
+    - `bash_functions.env`:  Added a new method to make it easier / simpler to calculation and display duration time. 
+    - `filter_catchments_and_add_attributes.py`:  Remove "dropLowStreamOrders" system.
+    - `split_flows.py`: Remove "dropLowStreamOrders" system.
+    - `usgs_gage_unit_setup.py`:  Remove "dropLowStreamOrders" system.
+
+- `gms`  
+    - `delineate_hydros_and_produced_HAND.sh` : Remove "dropLowStreamOrders" system.
+    - `derive_level_paths.py`: Remove "dropLowStreamOrders" system and some small style updates.
+    - `run_by_unit.sh`: Remove "dropLowStreamOrders" system.
+
+- `unit_tests/gms`
+    - `derive_level_paths_params.json` and `derive_level_paths_unittests.py`: Remove "dropLowStreamOrders" system.
+
+<br/><br/>
+
 ## v4.0.15.0 - 2022-12-20 - [PR #758](https://github.com/NOAA-OWP/inundation-mapping/pull/758)
 
 This merge addresses feedback received from field users regarding CatFIM. Users wanted a Stage-Based version of CatFIM, they wanted maps created for multiple intervals between flood categories, and they wanted documentation as to why many sites are absent from the Stage-Based CatFIM service. This merge seeks to address this feedback. CatFIM will continue to evolve with more feedback over time.
@@ -80,13 +130,11 @@ Fixes inundation of nodata areas of REM.
 
 - `tools/inundation.py`: Assigns depth a value of `0` if REM is less than `0`
 
-
-
 ## v4.0.13.1 - 2022-12-09 - [PR #743](https://github.com/NOAA-OWP/inundation-mapping/pull/743)
 
 This merge adds the tools required to generate Alpha metrics by hydroid. It summarizes the Apha metrics by branch 0 catchment for use in the Hydrovis "FIM Performance" service.
 
-## Additions
+### Additions
 
 - `pixel_counter.py`:  A script to perform zonal statistics against raster data and geometries
 - `pixel_counter_functions.py`: Supporting functions
diff --git a/gms_pipeline.sh b/gms_pipeline.sh
index 4aa9c3216..2907ce04e 100755
--- a/gms_pipeline.sh
+++ b/gms_pipeline.sh
@@ -4,10 +4,11 @@ usage ()
 {
     echo
     echo 'Produce GMS hydrofabric datasets for unit and branch scale.'
-    echo 'Usage : gms_pipeline.sh [REQ: -u <hucs> - -n <run name> ]'
-    echo '                        [OPT: -h -c <config file> -j <job limit>] -o -r'
-    echo '                         -ud <unit deny list file> -bd <branch deny list file>'
-    echo '                         -zd <branch zero deny list file> -a <use all stream orders> ]'
+    echo 'Usage : gms_pipeline.sh [REQ: -u <hucs> -n <run name> ]'
+    echo '                        [OPT: -h -c <config file> -j <job limit>] -o'
+    echo '                         -ud <unit deny list file>'
+    echo '                         -bd <branch deny list file>'
+    echo '                         -zd <branch zero deny list file>]'
     echo ''
     echo 'REQUIRED:'
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
@@ -38,10 +39,6 @@ usage ()
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -r/--retry      : retries failed jobs'
-	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
-    echo '                    at the initial load of the nwm_subset_streams.'
-    echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
     echo
     exit
 }
@@ -74,10 +71,6 @@ in
     -o|--overwrite)
         overwrite=1
         ;;
-    -r|--retry)
-        retry="--retry-failed"
-        overwrite=1
-        ;;
     -ud|--unitDenylist)
         shift
         deny_unit_list=$1
@@ -90,9 +83,6 @@ in
         shift
         deny_branch_zero_list=$1
         ;;        
-    -a|--useAllStreamOrders)
-        useAllStreamOrders=1
-        ;;
     *) ;;
     esac
     shift
@@ -120,10 +110,6 @@ then
     # default is false (0)
     overwrite=0
 fi
-if [ -z "$retry" ]
-then
-    retry=""
-fi
 
 # The tests for the deny lists are duplicated here on to help catch
 # them earlier (ie.. don't have to wait to process units to find an
@@ -154,18 +140,6 @@ then
     usage
 fi
 
-# invert useAllStreamOrders boolean (to make it historically compatiable
-# with other files like gms/run_unit.sh and gms/run_branch.sh).
-# Yet help user understand that the inclusion of the -a flag means
-# to include the stream order (and not get mixed up with older versions
-# where -s mean drop stream orders)
-# This will encourage leaving stream orders 1 and 2 out.
-if [ "$useAllStreamOrders" == "1" ]; then
-    export dropLowStreamOrders=0
-else
-    export dropLowStreamOrders=1
-fi
-
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
 source $srcDir/bash_functions.env
@@ -202,8 +176,7 @@ run_cmd+=" -c $envFile"
 run_cmd+=" -j $jobLimit"
 
 if [ $overwrite -eq 1 ]; then run_cmd+=" -o" ; fi
-if [ "$retry" == "--retry-failed" ]; then run_cmd+=" -r" ; fi
-if [ $dropLowStreamOrders -eq 1 ]; then run_cmd+=" -s" ; fi
+
 #echo "$run_cmd"
 . /foss_fim/gms_run_unit.sh -u "$hucList" $run_cmd -ud "$deny_unit_list" -zd "$deny_branch_zero_list"
 
@@ -217,18 +190,19 @@ if [ $dropLowStreamOrders -eq 1 ]; then run_cmd+=" -s" ; fi
 
 # if this has too many errors, it will return a sys.exit code (like 62 as per fim_enums)
 # and we will stop the rest of the process. We have to catch stnerr as well.
+# This stops the run from continuing to run, drastically filing drive and killing disk space.
 python3 $srcDir/check_unit_errors.py -f $outputRunDataDir -n $num_hucs
 
-
 ## Produce level path or branch level datasets
 . /foss_fim/gms_run_branch.sh $run_cmd -bd "$deny_branches_list" -zd "$deny_branch_zero_list"
 
 
+## continue on to post processing
+. /foss_fim/gms_run_post_processing.sh $run_cmd
+
+echo
 echo "======================== End of gms_pipeline.sh =========================="
-pipeline_end_time=`date +%s`
-total_sec=$(expr $pipeline_end_time - $pipeline_start_time)
-dur_min=$((total_sec / 60))
-dur_remainder_sec=$((total_sec % 60))
-echo "Total Run Time = $dur_min min(s) and $dur_remainder_sec sec"
+date -u
+Calc_Duration $pipeline_start_time
 echo
 
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index c313bcc4f..c8029af9b 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -4,19 +4,19 @@ usage ()
 {
     echo 'Produce GMS hydrofabric at levelpath/branch scale. Execute gms_run_unit.sh prior to.'
     echo 'Usage : gms_run_branch.sh [REQ: -n <run name> ]'
-    echo '  	 				    [OPT: -h -j <job limit> -o -r '
-    echo '  	 				    -bd <branch deny list file> -zd <branch zero deny list file>'
-    echo '                          -u <hucs> -a <use all stream orders>]'
+    echo '                          [OPT: -h -u <hucs> -c <config file> -j <job limit>] -o'
+    echo '  	 				    -bd <branch deny list file>'
+    echo '                          -zd <branch zero deny list file>]'
     echo ''
     echo 'REQUIRED:'
     echo '  -n/--runName    : A name to tag the output directories and log files as. could be a version tag.'
     echo ''
     echo 'OPTIONS:'
     echo '  -h/--help       : help file'
-    echo '  -c/--config     : configuration file with bash environment variables to export'    
-    echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited).'
     echo '                    A line delimited file also acceptable. HUCs must present in inputs directory.'
+    echo '  -c/--config     : configuration file with bash environment variables to export'    
+    echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
     echo '  -bd/--branchDenylist : A file with a line delimited list of files in BRANCHES directories to be removed' 
     echo '                    upon completion of branch processing.'
     echo '                    (see config/deny_gms_branches_prod.lst for a starting point)'
@@ -31,11 +31,7 @@ usage ()
     echo '                    use the word NONE as this value for this parameter.'    
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    echo '  -r/--retry      : retries failed jobs'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
-    echo '                    at the initial load of the nwm_subset_streams.'
-    echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
     echo
     exit
 }
@@ -66,10 +62,6 @@ in
     -o|--overwrite)
         overwrite=1
         ;;
-    -r|--retry)
-        retry="--retry-failed"
-        overwrite=1
-        ;;
     -bd|--branchDenylist)
         shift
         deny_branches_list=$1
@@ -78,9 +70,6 @@ in
         shift
         deny_branch_zero_list_for_branches=$1
         ;;        
-    -a|--useAllStreamOrders)
-        useAllStreamOrders=1
-        ;;
     *) ;;
     esac
     shift
@@ -133,22 +122,6 @@ if [ "$overwrite" = "" ]
 then
     overwrite=0
 fi
-if [ -z "$retry" ]
-then
-    retry=""
-fi
-
-# invert useAllStreamOrders boolean (to make it historically compatiable
-# with other files like gms/run_unit.sh and gms/run_branch.sh).
-# Yet help user understand that the inclusion of the -a flag means
-# to include the stream order (and not get mixed up with older versions
-# where -s mean drop stream orders)
-# This will encourage leaving stream orders 1 and 2 out.
-if [ "$useAllStreamOrders" == "1" ]; then
-    export dropLowStreamOrders=0
-else
-    export dropLowStreamOrders=1
-fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
@@ -163,12 +136,11 @@ fi
 export outputRunDataDir=$outputDataDir/$runName
 export deny_branches_list=$deny_branches_list
 logFile=$outputRunDataDir/logs/branch/summary_gms_branch.log
-export extent=GMS
 export overwrite=$overwrite
+export extent=GMS
 
-
-## Check for run data directory ##
-if [ ! -d "$outputRunDataDir" ]; then 
+## Check for run data directory and the file. If gms_run_unit failed, the file will not be there ##
+if [ ! -f "$outputRunDataDir/gms_inputs.csv" ]; then 
     echo "Depends on output from gms_run_unit.sh. Please produce data with gms_run_unit.sh first."
     exit 1
 fi
@@ -181,11 +153,6 @@ else
     gms_inputs=$outputRunDataDir/gms_inputs_filtered.csv
 fi
 
-# Echo intent to retry
-if [ "$retry" = "--retry-failed" ]; then
-    echo "Retrying failed unit level jobs for $runName"
-fi 
-
 # make log dir
 if [ ! -d "$outputRunDataDir/logs/branch" ]; then
     mkdir -p $outputRunDataDir/logs/branch
@@ -195,6 +162,9 @@ elif [ $overwrite -eq 1 ]; then
     mkdir -p $outputRunDataDir/logs/branch
 fi
 
+# Note: Other parts of the program will check for the existance of the file
+# /branch_errors/non_zero_exit_codes.log. It has to be removed no matter
+# what on each run of gms_run_branch
 if [ ! -d "$outputRunDataDir/branch_errors" ]; then
     mkdir -p "$outputRunDataDir/branch_errors"
 elif [ $overwrite -eq 1 ]; then
@@ -202,97 +172,25 @@ elif [ $overwrite -eq 1 ]; then
     mkdir -p $outputRunDataDir/branch_errors
 fi
 
-## Track total time of the overall run
-T_total_start
-
 ## RUN GMS BY BRANCH ##
-echo "=========================================================================="
-echo "Start of branch processing"
-echo "Started: `date -u`" 
+echo
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- Start of branch processing"
+echo "---- Started: `date -u`" 
+T_total_start
 Tstart
+all_branches_start_time=`date +%s`
 
 if [ "$jobLimit" -eq 1 ]; then
-    parallel $retry --verbose --timeout $branch_timeout --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
+    parallel --verbose --timeout $branch_timeout --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
 else
-    parallel $retry --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
+    parallel --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
 fi
 
 echo "Branch processing is complete"
 Tcount
 date -u
 
-## RUN AGGREGATE BRANCH ELEV TABLES ##
-# TODO: How do we skip aggregation if there is a branch error
-# maybe against the non_zero logs above
-echo 
-echo "Processing usgs gage aggregation"
-python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
-
-## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
-if [ "$src_bankfull_toggle" = "True" ]; then
-    echo -e $startDiv"Estimating bankfull stage in SRCs"$stopDiv
-    # Run SRC bankfull estimation routine routine
-    Tstart
-    time python3 /foss_fim/src/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit
-    Tcount
-fi
-
-## RUN SYNTHETIC RATING SUBDIVISION ROUTINE ##
-if [ "$src_subdiv_toggle" = "True" ]; then
-    echo -e $startDiv"Performing SRC channel/overbank subdivision routine"$stopDiv
-    # Run SRC Subdivision & Variable Roughness routine
-    Tstart
-    time python3 /foss_fim/src/subdiv_chan_obank_src.py -fim_dir $outputRunDataDir -mann $vmann_input_file -j $jobLimit
-    Tcount
-fi
-
-## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
-if [ "$src_adjust_spatial" = "True" ]; then
-    if [ ! -f $CALB_DB_KEYS_FILE ]; then
-        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
-        exit 1
-    else
-        source $CALB_DB_KEYS_FILE
-        : '
-        This makes the local variables from the calb_db_keys files
-        into global variables that can be used in other files, including python.
-
-        Why not just leave the word export in front of each of the keys in the
-        calb_db_keys.env? Becuase that file is used against docker-compose
-        when we start up that part of the sytem and it does not like the word
-        export.
-        '
-        export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
-        export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
-        export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
-        export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
-        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
-        echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
-        echo "Loading Point Data"
-        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
-    fi
-fi
-
-## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
-if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
-    Tstart
-    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"$stopDiv
-    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
-    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
-    Tcount
-    date -u
-fi
-
-## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
-if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
-    Tstart
-    echo -e $startDiv"Performing SRC adjustments using benchmark point database"$stopDiv
-    python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
-    Tcount
-    date -u
-fi
-
 # -------------------
 ## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO (but using normal branch deny) ##
 ## but also do not remove if branch zero deny is NONE (any case)
@@ -317,9 +215,13 @@ else
     $srcDir/gms/outputs_cleanup.py -d $outputRunDataDir -l $deny_branches_list -b 0
 fi
 
+
 # -------------------
 ## GET NON ZERO EXIT CODES ##
 # Needed in case aggregation fails, we will need the logs
+# Note: Other parts of the program (gms_run_post_processing.sh) check to see
+# if the branch_errors/non_zero_exit_codes.log exists. If it does not, it assumes
+# that gms_run_branch did not complete (or was not run)
 echo
 echo -e $startDiv"Start non-zero exit code checking"$stopDiv
 find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
@@ -335,15 +237,8 @@ Tcount
 date -u
 
 echo
-echo -e $startDiv"Combining crosswalk tables"$stopDiv
-# aggregate outputs
-Tstart
-python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
-Tcount
-date -u
-
-echo "=========================================================================="
-echo "GMS_run_branch complete"
-Tcount
-echo "Ended: `date -u`"
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- gms_run_branch complete"
+echo "---- Ended: `date -u`"
+Calc_Duration $all_branches_start_time
 echo
diff --git a/gms_run_post_processing.sh b/gms_run_post_processing.sh
new file mode 100755
index 000000000..8397d44b2
--- /dev/null
+++ b/gms_run_post_processing.sh
@@ -0,0 +1,182 @@
+#!/bin/bash -e
+:
+usage ()
+{
+    echo 'Produce GMS hydrofabric at levelpath/branch scale. Execute gms_run_unit.sh and gms_run_branch prior to.'
+    echo 'Usage : gms_run_post_processing.sh [REQ: -n <run name> ]'
+    echo '  	 				             [OPT: -h -c -j <job limit>]'
+    echo ''
+    echo 'REQUIRED:'
+    echo '  -n/--runName    : A name to tag the output directories and log files as. could be a version tag.'
+    echo ''
+    echo 'OPTIONS:'
+    echo '  -h/--help       : help file'
+    echo '  -c/--config     : configuration file with bash environment variables to export'    
+    echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
+    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
+    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
+    echo
+    exit
+}
+
+while [ "$1" != "" ]; do
+case $1
+in
+    -c|--configFile )
+        shift
+        envFile=$1
+        ;;
+    -n|--runName)
+        shift
+        runName=$1
+        ;;
+    -j|--jobLimit)
+        shift
+        jobLimit=$1
+        ;;
+    -h|--help)
+        shift
+        usage
+        ;;
+    *) ;;
+    esac
+    shift
+done
+
+# print usage if arguments empty
+if [ "$runName" = "" ]
+then
+    echo "ERROR: Missing -n run time name argument"
+    usage
+fi
+
+if [ "$envFile" = "" ]
+then
+    envFile=/foss_fim/config/params_template.env
+fi
+
+## SOURCE ENV FILE AND FUNCTIONS ##
+source $envFile
+source $srcDir/bash_functions.env
+
+# default values
+if [ "$jobLimit" = "" ] ; then
+    jobLimit=$default_max_jobs
+fi
+
+## Define Outputs Data Dir & Log File##
+export outputRunDataDir=$outputDataDir/$runName
+export extent=GMS
+
+## Check for run data directory ##
+if [ ! -d "$outputRunDataDir" ]; then 
+    echo "Depends on output from gms_run_unit.sh. Please produce data with gms_run_unit.sh first."
+    exit 1
+fi
+
+## Check to ensure gms_run_branch completed ##
+if [ ! -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" ]; then
+    echo "Depends on output from gms_run_branch.sh. Please run gms_run_branch.sh or check if it failed."
+    exit 1
+fi
+
+# Clean out the other post processing files before starting
+rm -rdf $outputRunDataDir/logs/src_optimization
+rm -f $outputRunDataDir/logs/log_bankfull_indentify.log
+rm -f $outputRunDataDir/logs/subdiv_src_.log
+
+gms_inputs=$outputRunDataDir/gms_inputs.csv
+
+echo
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- Start of gms_run_post_processing"
+echo "---- Started: `date -u`" 
+T_total_start
+post_proc_start_time=`date +%s`
+
+## RUN AGGREGATE BRANCH ELEV TABLES ##
+# TODO: How do we skip aggregation if there is a branch error
+# maybe against the non_zero logs above
+echo 
+echo "Processing usgs gage aggregation"
+python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
+
+## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
+if [ "$src_bankfull_toggle" = "True" ]; then
+    echo -e $startDiv"Estimating bankfull stage in SRCs"$stopDiv
+    # Run SRC bankfull estimation routine routine
+    Tstart
+    time python3 /foss_fim/src/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit
+    Tcount
+fi
+
+## RUN SYNTHETIC RATING SUBDIVISION ROUTINE ##
+if [ "$src_subdiv_toggle" = "True" ]; then
+    echo -e $startDiv"Performing SRC channel/overbank subdivision routine"$stopDiv
+    # Run SRC Subdivision & Variable Roughness routine
+    Tstart
+    time python3 /foss_fim/src/subdiv_chan_obank_src.py -fim_dir $outputRunDataDir -mann $vmann_input_file -j $jobLimit
+    Tcount
+fi
+
+## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
+if [ "$src_adjust_spatial" = "True" ]; then
+    if [ ! -f $CALB_DB_KEYS_FILE ]; then
+        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        exit 1
+    else
+        source $CALB_DB_KEYS_FILE
+        : '
+        This makes the local variables from the calb_db_keys files
+        into global variables that can be used in other files, including python.
+
+        Why not just leave the word export in front of each of the keys in the
+        calb_db_keys.env? Becuase that file is used against docker-compose
+        when we start up that part of the sytem and it does not like the word
+        export.
+        '
+        export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
+        export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
+        export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
+        export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
+        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
+        echo "Loading HUC Data"
+        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+        echo "Loading Point Data"
+        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+    fi
+fi
+
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
+if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
+    Tstart
+    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"$stopDiv
+    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
+    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+    Tcount
+    date -u
+fi
+
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
+if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
+    Tstart
+    echo -e $startDiv"Performing SRC adjustments using benchmark point database"$stopDiv
+    python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
+    Tcount
+    date -u
+fi
+
+echo
+echo -e $startDiv"Combining crosswalk tables"$stopDiv
+# aggregate outputs
+Tstart
+python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
+Tcount
+date -u
+
+echo
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- gms_run_post_processing complete"
+echo "---- Ended: `date -u`"
+Calc_Duration $post_proc_start_time
+echo
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index c64030e5c..bc2361e01 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -4,9 +4,9 @@ usage ()
 {
     echo 'Produce GMS hydrofabric datasets for unit scale.'
     echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -n <run name> ]'
-    echo '                        [OPT: -h -j <job limit>]  -c <config file>'
-    echo '                         -o -r -ud <unit deny list file>'
-    echo '                         -zd <branch zero deny list file> -a <use all stream orders>]'
+    echo '                        [OPT: -h -c <config file> -j <job limit> -o'
+    echo '                         -ud <unit deny list file>'
+    echo '                         -zd <branch zero deny list file>]'
     echo ''
     echo 'REQUIRED:'
     echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
@@ -31,10 +31,6 @@ usage ()
     echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
     echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
     echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -r/--retry      : retries failed jobs'
-	echo '  -a/--UseAllStreamOrders : If this flag is included, the system will INCLUDE stream orders 1 and 2'
-	echo '                    at the initial load of the nwm_subset_streams.'
-	echo '                    Default (if arg not added) is false and stream orders 1 and 2 will be dropped'    
     echo
     exit
 }
@@ -65,10 +61,6 @@ in
     -o|--overwrite)
         overwrite=1
         ;;
-    -r|--retry)
-        retry="--retry-failed"
-        overwrite=1
-        ;;
     -ud|--unitDenylist)
         shift
         deny_unit_list=$1
@@ -77,9 +69,6 @@ in
         shift
         deny_branch_zero_list_for_units=$1
         ;;        
-    -a|--useAllStreamOrders)
-        useAllStreamOrders=1
-        ;;
     *) ;;
     esac
     shift
@@ -128,22 +117,6 @@ if [ -z "$overwrite" ]
 then
     overwrite=0
 fi
-if [ -z "$retry" ]
-then
-    retry=""
-fi
-
-# invert useAllStreamOrders boolean (to make it historically compatiable
-# with other files like gms/run_unit.sh and gms/run_branch.sh).
-# Yet help user understand that the inclusion of the -a flag means
-# to include the stream order (and not get mixed up with older versions
-# where -s mean drop stream orders)
-# This will encourage leaving stream orders 1 and 2 out.
-if [ "$useAllStreamOrders" == "1" ]; then
-    export dropLowStreamOrders=0
-else
-    export dropLowStreamOrders=1
-fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
@@ -167,7 +140,6 @@ fi
 
 ## Set misc global variables
 export overwrite=$overwrite
-export dropLowStreamOrders=$dropLowStreamOrders
 
 ## Define inputs
 export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
@@ -183,15 +155,9 @@ export extent=GMS
 export deny_unit_list=$deny_unit_list
 export deny_branch_zero_list_for_units=$deny_branch_zero_list_for_units
 
-
 # we are not using the variable output at this time, but keep it anways
 num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
 
-## Make output and data directories ##
-if [ "$retry" = "--retry-failed" ]; then
-    echo "Retrying failed unit level jobs for $runName"
-fi
-
 # make dirs
 if [ ! -d $outputRunDataDir ]; then
     mkdir -p $outputRunDataDir
@@ -202,39 +168,37 @@ rm -rdf $outputRunDataDir/logs
 rm -rdf $outputRunDataDir/branch_errors
 rm -rdf $outputRunDataDir/unit_errors
 
-# we need to clean out the all log files overwrite or not
+# we need to clean out the all log files and some other files overwrite or not
 mkdir -p $outputRunDataDir/logs/unit
 mkdir -p $outputRunDataDir/unit_errors
+rm -f $outputRunDataDir/gms_inputs*
 
 # copy over config file
 cp -a $envFile $outputRunDataDir
 
-## RUN GMS BY BRANCH ##
-echo "=========================================================================="
-echo "Start of unit processing"
-echo "Started: `date -u`" 
-
-## Track total time of the overall run
-T_total_start
-Tstart
+## RUN GMS BY UNIT ##
+echo
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- Start of gms_run_unit"
+echo "---- Started: `date -u`" 
+all_units_start_time=`date +%s`
 
 ## GMS BY UNIT##
 if [ -f "$hucList" ]; then
     if [ "$jobLimit" -eq 1 ]; then
-        parallel $retry --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
+        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
     else
-        parallel $retry --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
+        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
     fi
 else 
     if [ "$jobLimit" -eq 1 ]; then
-        parallel $retry --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
+        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
     else
-        parallel $retry --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
+        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
     fi
  fi
 
 echo "Unit (HUC) processing is complete"
-Tcount
 date -u
 
 ## GET NON ZERO EXIT CODES ##
@@ -246,8 +210,9 @@ find $outputRunDataDir/logs/ -name "*_unit.log" -type f | xargs grep -E "Exit st
 echo -e $startDiv"Start branch aggregation"$stopDiv
 python3 $srcDir/gms/aggregate_branch_lists.py -d $outputRunDataDir -f "gms_inputs.csv" -l $hucList
 
-echo "=========================================================================="
-echo "gms_run_unit processing is complete"
-Tcount
-date -u
+echo
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- gms_run_unit is complete"
+echo "---- Ended: `date -u`"
+Calc_Duration $all_units_start_time
 echo
\ No newline at end of file
diff --git a/src/bash_functions.env b/src/bash_functions.env
index 17eb08274..f5975995a 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -21,6 +21,16 @@ Tcount () {
     echo "Cumulative Time = $dur_min min(s) and $dur_remainder_sec sec"
 }
 
+Calc_Duration() {
+    start_time=$1
+    end_time=`date +%s`
+
+    total_sec=$(expr $end_time - $start_time)
+    dur_min=$((total_sec / 60))
+    dur_remainder_sec=$((total_sec % 60))
+    echo "Duration = $dur_min min(s) and $dur_remainder_sec sec"
+}
+
 export -f T_total_start
 export -f Tstart
 export -f Tcount
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index 00c11c9ce..7801b613d 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -15,8 +15,7 @@ def filter_catchments_and_add_attributes(input_catchments_filename,
                                          output_catchments_filename,
                                          output_flows_filename,
                                          wbd_filename,
-                                         huc_code,
-                                         drop_stream_orders=False):
+                                         huc_code):
 
     input_catchments = gpd.read_file(input_catchments_filename)
     wbd = gpd.read_file(wbd_filename)
@@ -56,21 +55,13 @@ def filter_catchments_and_add_attributes(input_catchments_filename,
             output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
             output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
         except ValueError:
-            if (drop_stream_orders):
-                # this is not an exception, but a custom exit code that can be trapped
-                print("There are no flowlines in the HUC after stream order filtering.")
-                sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
-            else:
-                # if we are not dropping stream orders, then something is wrong
-                raise Exception("There are no flowlines in the HUC.")
-    else:
-        if (drop_stream_orders):
             # this is not an exception, but a custom exit code that can be trapped
             print("There are no flowlines in the HUC after stream order filtering.")
             sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
-        else:
-            # if we are not dropping stream orders, then something is wrong
-            raise Exception("There are no flowlines in the HUC.")
+    else:
+        # this is not an exception, but a custom exit code that can be trapped
+        print("There are no flowlines in the HUC after stream order filtering.")
+        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
 
 if __name__ == '__main__':
@@ -83,8 +74,7 @@ def filter_catchments_and_add_attributes(input_catchments_filename,
     parser.add_argument('-o', '--output-flows-filename', help='output-flows-filename', required=True)
     parser.add_argument('-w', '--wbd-filename', help='wbd-filename', required=True)
     parser.add_argument('-u', '--huc-code', help='huc-code', required=True)
-    parser.add_argument('-s', '--drop-stream-orders', help='Drop stream orders 1 and 2', type=int, required=False, default=False)
-
+    
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
 
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
index 96e9ae5bd..1d3759a67 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -75,7 +75,7 @@ Tcount
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
-$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg -ds $dropLowStreamOrders
+$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
@@ -145,7 +145,7 @@ Tcount
 echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber -s $dropLowStreamOrders
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
 Tcount
 
 ## RASTERIZE NEW CATCHMENTS AGAIN ##
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index 90971d55c..aaf2eeb19 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -13,8 +13,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                        catchments_outfile=None,
                        branch_inlets_outfile=None,
                        toNode_attribute='To_Node', fromNode_attribute='From_Node',
-                       reach_id_attribute='HydroID', verbose=False,
-                       drop_low_stream_orders=False ):
+                       reach_id_attribute='HydroID', verbose=False):
    
     
     if verbose:
@@ -37,15 +36,15 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
         sys.exit(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
 
-    if (drop_low_stream_orders):
-        stream_network = stream_network.exclude_attribute_values(branch_id_attribute="order_",
-                                               values_excluded=[1,2]
-                                              )
+    # values_exluded of 1 and 2 mean where are dropping stream orders 1 and 2. We are leaving those
+    # for branch zero.
+    stream_network = stream_network.exclude_attribute_values(branch_id_attribute="order_",
+                                            values_excluded=[1,2] )
 
-        # if there are no reaches at this point (due to filtering)
-        if (len(stream_network) == 0):
-            print("No branches exist but branch zero processing will continue. This could be due to stream order filtering.")
-            return
+    # if there are no reaches at this point (due to filtering)
+    if (len(stream_network) == 0):
+        print("No branches exist but branch zero processing will continue. This could be due to stream order filtering.")
+        return
                                                  
     inlets_attribute = 'inlet_id'
     outlets_attribute = 'outlet_id'
@@ -66,19 +65,18 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
     stream_network = stream_network.derive_outlets(toNode_attribute,
                                                    fromNode_attribute,
                                                    outlets_attribute=outlets_attribute,
-                                                   verbose=verbose
-                                                  )
+                                                   verbose=verbose)
+    
     stream_network = stream_network.derive_inlets(toNode_attribute,
                                                   fromNode_attribute,
                                                   inlets_attribute=inlets_attribute,
                                                   verbose=verbose
-                                                 )    # derive up and downstream networks
+                                                  )    # derive up and downstream networks
     upstreams, downstreams = stream_network.make_up_and_downstream_dictionaries(
                                                                          reach_id_attribute=reach_id_attribute,
                                                                          toNode_attribute=toNode_attribute,
                                                                          fromNode_attribute=fromNode_attribute,
-                                                                         verbose=True
-                                                                         )
+                                                                         verbose=True)
 
     # derive arbolate sum
     stream_network = stream_network.get_arbolate_sum(arbolate_sum_attribute='arbolate_sum',
@@ -87,8 +85,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                      upstreams=upstreams,
                                                      downstreams=downstreams,
                                                      length_conversion_factor_to_km = 0.001,
-                                                     verbose=verbose
-                                                    )
+                                                     verbose=verbose)
 
     # derive stream branches
     stream_network = stream_network.derive_stream_branches(toNode_attribute=toNode_attribute,
@@ -98,8 +95,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                            reach_id_attribute=reach_id_attribute,
                                                            comparison_attributes=['arbolate_sum', 'order_'],
                                                            comparison_function=max,
-                                                           verbose=verbose
-                                                          )
+                                                           verbose=verbose)
     
     # filter out streams without catchments
     if (catchments is not None) & (catchments_outfile is not None):
@@ -110,18 +106,15 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
                                                                 reach_id_attribute=reach_id_attribute,
                                                                 branch_id_attribute=branch_id_attribute,
                                                                 reach_id_attribute_in_catchments=reach_id_attribute,
-                                                                verbose=verbose
-                                                                              )
+                                                                verbose=verbose)
 
         # subset which columns to merge
         stream_network_to_merge = stream_network.filter(items = [reach_id_attribute,inlets_attribute,
-                                                                 outlets_attribute,branch_id_attribute]
-                                                       )
+                                                                 outlets_attribute,branch_id_attribute])
 
         catchments = catchments.merge(stream_network_to_merge,how='inner',
                                       left_on=reach_id_attribute,
-                                      right_on=reach_id_attribute
-                                     )
+                                      right_on=reach_id_attribute)
 
         catchments.reset_index(drop=True, inplace=True)
 
@@ -132,8 +125,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         headwaters = stream_network.derive_headwater_points_with_inlets(
                                                         fromNode_attribute=fromNode_attribute,
                                                         inlets_attribute=inlets_attribute,
-                                                        outlet_linestring_index=outlet_linestring_index
-                                                       )
+                                                        outlet_linestring_index=outlet_linestring_index)
         # headwaters write
         headwaters.to_file(headwaters_outfile, index=False, driver='GPKG')
 
@@ -144,8 +136,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
     
     if out_stream_network_dissolved is not None:
         stream_network = stream_network.trim_branches_in_waterbodies(branch_id_attribute=branch_id_attribute,
-                                                    verbose=verbose
-                                                                    )
+                                                    verbose=verbose)
 
         # dissolve by levelpath
         stream_network = stream_network.dissolve_by_branch(branch_id_attribute=branch_id_attribute,
@@ -156,13 +147,11 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
 
         stream_network = stream_network.remove_branches_in_waterbodies(waterbodies=waterbodies,
                                                                        out_vector_files=out_stream_network_dissolved,
-                                                                       verbose=False
-                                                                      )
+                                                                       verbose=False)
                                        
     if branch_inlets_outfile is not None:
         branch_inlets = stream_network.derive_inlet_points_by_feature(feature_attribute=branch_id_attribute,
-                                                                        outlet_linestring_index=outlet_linestring_index
-                                                                        )
+                                                                    outlet_linestring_index=outlet_linestring_index)
 
         branch_inlets.to_file(branch_inlets_outfile, index=False, driver='GPKG')
 
@@ -185,7 +174,6 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
     parser.add_argument('-e','--headwaters-outfile', help='Output stream network headwater points', required=False, default=None)
     parser.add_argument('-d','--out-stream-network-dissolved', help='Dissolved output stream network', required=False, default=None)
     parser.add_argument('-v','--verbose', help='Verbose output', required=False, default=False, action='store_true')
-    parser.add_argument('-s','--drop-low-stream-orders', help='Drop stream orders 1 and 2', type=int, required=False, default=False)
     
     args = vars(parser.parse_args())
 
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index c20e81d13..38a70d6ac 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -116,7 +116,7 @@ Tcount
 echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -s $dropLowStreamOrders -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 
 
 # test if we received a non-zero code back from derive_level_paths.py
@@ -136,12 +136,7 @@ Tcount
 echo -e $startDiv"Create file of branch ids for $hucNumber"$stopDiv
 date -u
 Tstart
-if [ $dropLowStreamOrders != 0 ]; then # only add branch zero to branch list if low stream orders are dropped
-    $srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -z $branch_zero_id
-else
-    $srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute
-fi
-
+$srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -z $branch_zero_id
 Tcount
 
 ## CREATE BRANCH ZERO ##
@@ -243,18 +238,16 @@ export xmax=$xmax
 export ymax=$ymax
 export ncols=$ncols
 export nrows=$nrows
-if [ $dropLowStreamOrders != 0 ]; then # only produce branch zero HAND if low stream orders are dropped
-    $srcDir/gms/delineate_hydros_and_produce_HAND.sh "unit"
-else
-    echo -e $startDiv"Skipping branch zero processing because there are no stream orders being dropped $hucNumber"$stopDiv
-fi
+
+## PRODUCE BRANCH ZERO HAND
+$srcDir/gms/delineate_hydros_and_produce_HAND.sh "unit"
 
 ## CREATE USGS GAGES FILE
 if [ -f $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id -bzero $dropLowStreamOrders
+    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id
     Tcount
 fi
 
diff --git a/src/split_flows.py b/src/split_flows.py
index daf4e757b..9d83122d0 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -40,8 +40,7 @@ def split_flows(max_length,
                 split_points_filename, 
                 wbd8_clp_filename, 
                 lakes_filename,
-                nwm_streams_filename,
-                drop_stream_orders=False):
+                nwm_streams_filename):
 
     toMetersConversion = 1e-3
 
@@ -49,13 +48,9 @@ def split_flows(max_length,
     flows = gpd.read_file(flows_filename)
 
     if (len(flows) == 0):
-        if (drop_stream_orders):
-            # this is not an exception, but a custom exit code that can be trapped
-            print("No relevant streams within HUC boundaries.")
-            sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
-        else:
-            # if we are not dropping stream orders, then something is wrong
-            raise Exception("No flowlines exist.")
+        # this is not an exception, but a custom exit code that can be trapped
+        print("No relevant streams within HUC boundaries.")
+        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
     wbd8 = gpd.read_file(wbd8_clp_filename)
     dem = rasterio.open(dem_filename,'r')
@@ -256,13 +251,10 @@ def split_flows(max_length,
         remove(split_points_filename)
 
     if (len(split_flows_gdf) == 0):
-        if (drop_stream_orders):
-            # this is not an exception, but a custom exit code that can be trapped
-            print("There are no flowlines after stream order filtering.")
-            sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
-        else:
-            # if we are not dropping stream orders, then something is wrong
-            raise Exception("No flowlines exist.")
+        # this is not an exception, but a custom exit code that can be trapped
+        print("There are no flowlines after stream order filtering.")
+        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+
     split_flows_gdf.to_file(split_flows_filename,driver=getDriver(split_flows_filename),index=False)
 
     if len(split_points_gdf) == 0:
@@ -284,7 +276,6 @@ def split_flows(max_length,
     parser.add_argument('-w', '--wbd8-clp-filename', help='wbd8-clp-filename',required=True)
     parser.add_argument('-l', '--lakes-filename', help='lakes-filename',required=True)
     parser.add_argument('-n', '--nwm-streams-filename', help='nwm-streams-filename',required=True)
-    parser.add_argument('-ds', '--drop-stream-orders', help='Drop stream orders 1 and 2', type=int, required=False, default=False)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 476777da0..d3efc61c2 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -104,7 +104,6 @@ def filter_gage_branches(gms_inputs_filename):
     parser.add_argument('-nwm','--input-nwm-filename', help='NWM stream subset', required=True)
     parser.add_argument('-o','--output-filename', help='Table to append data', required=True)
     parser.add_argument('-huc','--huc8-id', help='HUC8 ID (to verify gage location huc)', type=str, required=True)
-    parser.add_argument('-bzero','--branch-zero-check', help='Check for determining if branch zero is created', type=int, required=True)
     parser.add_argument('-bzero_id','--branch-zero-id', help='Branch zero ID value', type=str, required=True)
     parser.add_argument('-ff','--filter-gms-inputs', help='WARNING: only run this parameter if you know exactly what you are doing', required=False)
 
@@ -115,7 +114,6 @@ def filter_gage_branches(gms_inputs_filename):
     input_nwm_filename = args['input_nwm_filename']
     output_filename = args['output_filename']
     huc8 = args['huc8_id']
-    bzero_check = args['branch_zero_check']
     bzero_id = args['branch_zero_id']
     filter_gms_inputs = args['filter_gms_inputs']
 
@@ -129,10 +127,9 @@ def filter_gage_branches(gms_inputs_filename):
         usgs_gage_subset.write(output_filename)
 
         # Create seperate output for branch zero
-        if bzero_check != 0:
-            output_filename_zero = os.path.splitext(output_filename)[0] + '_' + bzero_id + os.path.splitext(output_filename)[-1]
-            usgs_gage_subset.branch_zero(bzero_id)
-            usgs_gage_subset.write(output_filename_zero)
+        output_filename_zero = os.path.splitext(output_filename)[0] + '_' + bzero_id + os.path.splitext(output_filename)[-1]
+        usgs_gage_subset.branch_zero(bzero_id)
+        usgs_gage_subset.write(output_filename_zero)
 
     else:
         ''' 
diff --git a/unit_tests/gms/derive_level_paths_params.json b/unit_tests/gms/derive_level_paths_params.json
index 36b5a6cf9..97e5e1a8b 100644
--- a/unit_tests/gms/derive_level_paths_params.json
+++ b/unit_tests/gms/derive_level_paths_params.json
@@ -10,8 +10,7 @@
 		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
 		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
-		"verbose": true,
-		"drop_low_stream_orders": false
+		"verbose": true
 	},
 	
 	"dropped_stream_orders_no_branches_remaining":
@@ -25,8 +24,7 @@
 		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset_levelPaths.gpkg",
 		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
-		"verbose": true,
-		"drop_low_stream_orders": true
+		"verbose": true
 	}	
 }
 
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
index be613204a..7027aa51c 100644
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ b/unit_tests/gms/derive_level_paths_unittests.py
@@ -56,8 +56,7 @@ def test_Derive_level_paths_success_all_params(self):
                                             catchments_outfile = params["catchments_outfile"],
                                             branch_inlets_outfile = params["branch_inlets_outfile"],
                                             reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"],
-                                            drop_low_stream_orders=params["drop_low_stream_orders"])
+                                            verbose = params["verbose"])
 
         # -----------
         # test data type being return is as expected. Downstream code might to know that type
@@ -94,94 +93,6 @@ def test_Derive_level_paths_success_all_params(self):
         print(f"Test Success: {inspect.currentframe().f_code.co_name}")
         print("*************************************************************")        
 
-
-    def test_Derive_level_paths_success_drop_low_stream_orders_not_submitted(self):
-    
-        '''
-        This test includes most params but does not submit a drop_low_stream_orders param
-        and it should default to "false", meaning no filtering out of stream orders 1 and 2.
-        Note: Most path tests done in test_Derive_level_paths_success_all_params 
-        and are not repeated here.
-        '''
-       
-        params = self.params["valid_data"].copy()
-
-        # Function Notes:
-        # huc_ids no longer used, so it is not submitted
-        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
-        # skip them here.
-        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                            out_stream_network = params["out_stream_network"],
-                                            branch_id_attribute = params["branch_id_attribute"],
-                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                            headwaters_outfile = params["headwaters_outfile"],
-                                            catchments = params["catchments"],
-                                            catchments_outfile = params["catchments_outfile"],
-                                            branch_inlets_outfile = params["branch_inlets_outfile"],
-                                            reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"])
-
-        # -----------
-        # test data type being return is as expected. Downstream code might to know that type
-        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
-        
-        # -----------
-        #**** NOTE: Based on 05030104         
-        # Test row count for dissolved level path GeoDataframe which is returned.
-        actual_row_count = len(actual_df) 
-        expected_row_count = 58    # should still be 58 with no filtering
-        self.assertEqual(actual_row_count, expected_row_count)
-        
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")        
-                
-
-    def test_Derive_level_paths_success_drop_low_stream_orders_is_true(self):
-    
-        '''
-        This test includes most params but does not submit a drop_low_stream_orders param
-        and it should default to "false", meaning no filtering out of stream orders 1 and 2.
-        Note: Most path tests done in test_Derive_level_paths_success_all_params 
-        and are not repeated here.
-        '''
-        
-        params = self.params["valid_data"].copy()
-
-        params["drop_low_stream_orders"] = True
-
-        # Function Notes:
-        # huc_ids no longer used, so it is not submitted
-        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
-        # skip them here.
-        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                            out_stream_network = params["out_stream_network"],
-                                            branch_id_attribute = params["branch_id_attribute"],
-                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                            headwaters_outfile = params["headwaters_outfile"],
-                                            catchments = params["catchments"],
-                                            catchments_outfile = params["catchments_outfile"],
-                                            branch_inlets_outfile = params["branch_inlets_outfile"],
-                                            reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"],
-                                            drop_low_stream_orders=params["drop_low_stream_orders"])
-
-        # -----------
-        # test data type being return is as expected. Downstream code might to know that type
-        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
-        
-        # -----------
-        #**** NOTE: Based on 05030104         
-        # Test row count for dissolved level path GeoDataframe which is returned.
-        actual_row_count = len(actual_df) 
-        expected_row_count = 4
-        self.assertEqual(actual_row_count, expected_row_count)
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")        
-
-
     def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self):
     
         '''
@@ -205,8 +116,7 @@ def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self
                                     catchments_outfile = params["catchments_outfile"],
                                     branch_inlets_outfile = params["branch_inlets_outfile"],
                                     reach_id_attribute = params["reach_id_attribute"],
-                                    verbose = params["verbose"],
-                                    drop_low_stream_orders=params["drop_low_stream_orders"])
+                                    verbose = params["verbose"])
 
         self.assertEqual(se.exception.code, fec.GMS_UNIT_NO_BRANCHES.value)
 

From cab36146460c684ccdd5f63d4dca11bd6bf243b6 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Tue, 27 Dec 2022 09:37:55 -0600
Subject: [PATCH 192/359] Add rsync to docker image

---
 Dockerfile        |  2 +-
 docs/CHANGELOG.md | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f62059ae6..7cd1f264e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -68,7 +68,7 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 RUN apt update --fix-missing 
-RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux
+RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync
 
 RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 90abdebc9..f278fea22 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.17.0 - 2022-12-21 - [PR #771](https://github.com/NOAA-OWP/inundation-mapping/pull/771)
+
+Added rysnc to docker images. rysnc can now be used inside the images to move data around via docker mounts.
+
+### Changes
+
+- `Dockerfile` : added rsync 
+
+<br/><br/>
+
 ## v4.0.16.0 - 2022-12-20 - [PR #768](https://github.com/NOAA-OWP/inundation-mapping/pull/768)
 
 `gms_run_branch.sh` was processing all of the branches iteratively, then continuing on to a large post processing portion of code. That has now be split to two files, one for branch iteration and the other file for just post processing.

From 1e24ec73366b3ae59e3cf0b016cc8e8ddcacf46c Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 29 Dec 2022 13:43:09 -0600
Subject: [PATCH 193/359] Bug fix to Stage-Based CatFIM

---
 docs/CHANGELOG.md                 | 8 ++++++++
 tools/generate_categorical_fim.py | 3 +--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f278fea22..d48cf0ba6 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.17.1 - 2022-12-29 - [PR #778](https://github.com/NOAA-OWP/inundation-mapping/pull/778)
+
+This merge fixes a bug where all of the Stage-Based intervals were the same.
+
+### Changes
+- `/tools/generate_categorical_fim.py`: Changed `stage` variable to `interval_stage` variable in `produce_stage_based_catfim_tifs` function call.
+
+<br/><br/>
 
 ## v4.0.17.0 - 2022-12-21 - [PR #771](https://github.com/NOAA-OWP/inundation-mapping/pull/771)
 
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index 4974fcb09..dfb25438b 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -425,7 +425,7 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
                         category = 'moderate_' + str(interval_stage).replace('.', 'p') + 'ft'
                     if interval_stage >= major_stage:
                         category = 'major_' + str(interval_stage).replace('.', 'p') + 'ft'
-                    executor.submit(produce_stage_based_catfim_tifs, stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs)
+                    executor.submit(produce_stage_based_catfim_tifs, interval_stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs)
             except TypeError:  # sometimes the thresholds are Nonetypes
                 pass
                     
@@ -682,7 +682,6 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
 
 
 if __name__ == '__main__':
-
     
     # Parse arguments
     parser = argparse.ArgumentParser(description = 'Run Categorical FIM')

From cc42ef9a2793d93ae59fbc34d8a94f0ae917e12d Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Thu, 29 Dec 2022 20:57:37 -0700
Subject: [PATCH 194/359] Remove `other` folder from `test_cases`

---
 config/symbology/esri/agreement_raster.lyr | Bin 7168 -> 9216 bytes
 config/symbology/qgis/agreement_raster.qml |   2 +-
 docs/CHANGELOG.md                          |  18 ++++++++++++++++++
 tools/aggregate_metrics.py                 |   2 +-
 tools/cache_metrics.py                     |   2 +-
 tools/eval_alt_catfim.py                   |   2 +-
 tools/run_test_case.py                     |   3 +--
 7 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/config/symbology/esri/agreement_raster.lyr b/config/symbology/esri/agreement_raster.lyr
index d6e27030d633b26c3a3b0a65b952a42c56601dff..7142cd6eb9ad7597508528045372789b35b91d83 100644
GIT binary patch
delta 1113
zcmZp$Xz<vO!^VE$_SL3+(l;j;v$ZlUlbC#w#cL7=$7CP&CxW~T3=Dig%n!r@K>TmB
zAk%)vNn8@520&geLk>eJ5F0YYGbAw>Fyv0w=ZeHBBhU1WVe$bkn`9@3AO=SU7X~*5
ze}+1sW<3TcAU}j51}Nsw;LH%p;LG3&mWctf@_|MtGDI+xFr+dR0j-Q-C<a=Y2qZIr
z>=K|z0gz4s$|W+SgXw&Rg2}nu>VnvuCp-BFw^Mx;(DBX;RzQ8BKyyIm76aLIa2L=P
z28KnGC-Yd-B^2y=LkHNejGGntR<ZJf(vUAW(z1cUoif>8NOm)$$UP<n0Zr${OA|#L
z1(wd7kmIiF2;x6&m2~X5<m0iW!N75{B8$Xiaq%Mi2aF610YUy=&cV@2p}~&v&iQ#I
zndznZrN!}%IZ3HS#qq9%rHMK5jzy`7Aj!<+_|RbY;P^72RAzqO<Tc`_*xeFyic{lV
zC)Y|ya6{OB`9&oelN%&NIh|AUN{WE`eJAgekd6#4Nz6-0EJ}$FNCXPzq~^pMD(M(m
z>Y2vs1ULpc`nra=21P4jsxng2F)>9`<&#*FSyGyk8t<PL??2f?a)vwsPfz|Y*%)$A
z-C)-#&kpzQD64FX+UGrLzz}2wVpJgDkf&`C_A8Cs;ppKr7Y!bKvPYL^0EQdq^K|P=
zlegP%Zj`ELoIF9QkfUVlr;zV^s^KD)9HL&C>ao*okwk(6`e$TpJdO}iDT`6K@$cy~
dh#h_l9R$E&f<4d*h63R!`kP(8fmBB^0sv^V9OM81

delta 1021
zcmZqhXt3Cj!^Z9~<JGM#1=l7Qv$Zl!5SV<C#cL7=$7CP&Cyf6#3o@N&oWvy|Zp6UA
zP{NQ8#EA?!4Dmp+h#{3Bak4H~Bpx|=CN{>&d$?@slYq(-8B!S1fJPJnX+4G{Aio4?
zL@`4QgA16>23t@7<mbTUN`O|S0A<U7v;~7XgC#>Ug9(EHkYxfCF=Q|TvMqtw7>Esl
zIt_ue0YfFwju-}O1_g#7ph?9HC6hC`C55p&Uv}~aZm0SVp!b{^tbn>gf$CF%`a$lZ
zo2&VdoXx<nVR9jl^#BDK1A_!GD1gB>S(ev*po4`27A%|Z@@`<21i8!?9HH63C`tiF
zuOl#`K&g;XNOtp1;d@L9KotxXCs+=G=vC=zjv#sg%hLlOdX>4R<77n^iOJ{1ij;3L
zGB5-L`FlABM=LqzB&8M=$2;feWhTeF7M3RF#5)$HCQfb;KgFJwm{Xh@pE@~ELITRp
zn_M6vS)ZJmS5gF2n44OZnUa~9r=+86X{Kios}tZD<ml@f;u;jKR9upnmy%eN5?_!A
zRGO2T6K|-bV`QmkY5)g@1T+{a>6n;e(I5h~A}6sVv!pa7H9kKrK7TTs<P3R`Pw|An
O<h_!Oo0mv6FaiK-$mX8_

diff --git a/config/symbology/qgis/agreement_raster.qml b/config/symbology/qgis/agreement_raster.qml
index 327ecdeff..7c74b4258 100644
--- a/config/symbology/qgis/agreement_raster.qml
+++ b/config/symbology/qgis/agreement_raster.qml
@@ -27,7 +27,7 @@
         <paletteEntry label="FN" value="1" alpha="255" color="#fdae61"/>
         <paletteEntry label="FP" value="2" alpha="255" color="#d7191c"/>
         <paletteEntry label="TP" value="3" alpha="255" color="#2c7bb6"/>
-        <paletteEntry label="Waterbody mask" value="4" alpha="255" color="#b2b2b2"/>
+        <paletteEntry label="Masked" value="4" alpha="255" color="#b2b2b2"/>
       </colorPalette>
       <colorramp type="randomcolors" name="[source]"/>
     </rasterrenderer>
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d48cf0ba6..6d58f1666 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.17.2 - 2022-12-29 - [PR #779](https://github.com/NOAA-OWP/inundation-mapping/pull/779)
+
+Remove dependency on `other` folder in `test_cases`. Also updates ESRI and QGIS agreement raster symbology label to include the addition of levee-protected areas as a mask.
+
+### Removals
+
+- `tools/`
+    - `aggregate_metrics.py` and `cache_metrics.py`: Removes reference to test_cases/other folder
+
+### Changes
+
+- `config/symbology/`
+    - `esri/agreement_raster.lyr` and `qgis/agreement_raster.qml`: Updates label from Waterbody mask to Masked since mask also now includes levee-protected areas
+- `tools/`
+    - `eval_alt_catfim.py` and `run_test_case.py`: Updates waterbody mask to dataset located in /inputs folder
+
+<br/><br/>
+
 ## v4.0.17.1 - 2022-12-29 - [PR #778](https://github.com/NOAA-OWP/inundation-mapping/pull/778)
 
 This merge fixes a bug where all of the Stage-Based intervals were the same.
diff --git a/tools/aggregate_metrics.py b/tools/aggregate_metrics.py
index 7cc5951b5..bd5a99fdd 100755
--- a/tools/aggregate_metrics.py
+++ b/tools/aggregate_metrics.py
@@ -204,7 +204,7 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
 
         for test_case in test_cases_dir_list:
 
-            if test_case not in ['other', 'validation_data_ble', 'validation_data_legacy', 'validation_data_ahps']:
+            if test_case not in ['validation_data_ble', 'validation_data_legacy', 'validation_data_ahps']:
                 branch_results_dir = os.path.join(TEST_CASES_DIR, test_case, 'performance_archive', config_version, branch)
 
                 huc = test_case.split('_')[0]
diff --git a/tools/cache_metrics.py b/tools/cache_metrics.py
index 0d02fb217..22ef4d9bf 100755
--- a/tools/cache_metrics.py
+++ b/tools/cache_metrics.py
@@ -81,7 +81,7 @@ def process_alpha_test(args):
         bench_cat_test_case_list = os.listdir(bench_cat_test_case_dir)
 
         for test_id in bench_cat_test_case_list:
-            if 'validation' and 'other' not in test_id:
+            if 'validation' not in test_id:
 
                 current_huc = test_id.split('_')[0]
                 if test_id.split('_')[1] in bench_cat:
diff --git a/tools/eval_alt_catfim.py b/tools/eval_alt_catfim.py
index 6ddf9165d..d5ce867d8 100644
--- a/tools/eval_alt_catfim.py
+++ b/tools/eval_alt_catfim.py
@@ -196,7 +196,7 @@ def process_alt_comparison(args):
                                 stats_modes_list=['total_area']
                                 test_id=''
                                 mask_dict={'levees': {'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg', 'buffer': None, 'operation': 'exclude'}, 
-                                            'waterbodies': {'path': '/data/test_cases/other/zones/nwm_v2_reservoirs.shp', 'buffer': None, 'operation': 'exclude'},
+                                            'waterbodies': {'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg', 'buffer': None, 'operation': 'exclude'},
                                             site: {'path': '/data/test_cases/{benchmark_type}_test_cases/validation_data_{benchmark_type}/{huc}/{site}/{site}_domain.shp'.format(benchmark_type=benchmark_type, site=site, category=category, huc=huc), 'buffer': None, 'operation': 'include'}}
                                 
                                 json_list.append(stats_json)
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index a850aaa3b..9d46ca582 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -103,14 +103,13 @@ def __init__(self, test_id, version, archive=True):
         self.benchmark_dir = os.path.join(self.validation_data, self.huc)
 
         # Create list of shapefile paths to use as exclusion areas.
-        zones_dir = os.path.join(TEST_CASES_DIR, 'other', 'zones')
         self.mask_dict = {'levees':
                         {'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg',
                         'buffer': None,
                         'operation': 'exclude'
                         },
                     'waterbodies':
-                        {'path': os.path.join(zones_dir, 'nwm_v2_reservoirs.shp'),
+                        {'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg',
                         'buffer': None,
                         'operation': 'exclude',
                         },

From e92eb8dbe6f366f98ca741b4f76c49b5516a5834 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 30 Dec 2022 18:56:58 -0700
Subject: [PATCH 195/359] Cleanup REM masking

---
 config/deny_gms_branch_zero.lst              |  1 -
 config/deny_gms_branches_dev.lst             |  1 -
 config/deny_gms_branches_prod.lst            |  1 -
 docs/CHANGELOG.md                            | 18 +++++++
 src/gms/delineate_hydros_and_produce_HAND.sh | 22 --------
 src/gms/rasterize_by_order.py                | 53 --------------------
 src/gms/rem.py                               |  2 +-
 7 files changed, 19 insertions(+), 79 deletions(-)
 delete mode 100755 src/gms/rasterize_by_order.py

diff --git a/config/deny_gms_branch_zero.lst b/config/deny_gms_branch_zero.lst
index 2167ee332..3bda34079 100644
--- a/config/deny_gms_branch_zero.lst
+++ b/config/deny_gms_branch_zero.lst
@@ -42,7 +42,6 @@ headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
 #LandSea_subset_{}.tif
-LeveeProtectedAreas_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
diff --git a/config/deny_gms_branches_dev.lst b/config/deny_gms_branches_dev.lst
index 6fea451b0..d52d4b855 100644
--- a/config/deny_gms_branches_dev.lst
+++ b/config/deny_gms_branches_dev.lst
@@ -41,7 +41,6 @@ gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
-LeveeProtectedAreas_subset_{}.tif
 LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nld_subset_levees_{}.tif
diff --git a/config/deny_gms_branches_prod.lst b/config/deny_gms_branches_prod.lst
index 4b9e17d61..b1edb0261 100644
--- a/config/deny_gms_branches_prod.lst
+++ b/config/deny_gms_branches_prod.lst
@@ -41,7 +41,6 @@ gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
-LeveeProtectedAreas_subset_{}.tif
 LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nld_subset_levees_{}.tif
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 6d58f1666..0bc066d57 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.17.3 - 2022-12-23 - [PR#773](https://github.com/NOAA-OWP/inundation-mapping/pull/773)
+
+Cleans up REM masking of levee-protected areas and fixes associated error.
+
+### Removals
+
+- `src/gms/`
+    - `delineate_hydros_and_produce_HAND.sh`: removes rasterization and masking of levee-protected areas from the REM
+    - `rasterize_by_order`: removes this file
+- `config/`
+    - `deny_gms_branch_zero.lst`, `deny_gms_branches_dev.lst`, and `deny_gms_branches_prod.lst`: removes `LeveeProtectedAreas_subset_{}.tif`
+
+### Changes
+
+- `src/gms/rem.py`: fixes an error where the nodata value of the DEM was overlooked
+
+<br/><br/>
+
 ## v4.0.17.2 - 2022-12-29 - [PR #779](https://github.com/NOAA-OWP/inundation-mapping/pull/779)
 
 Remove dependency on `other` folder in `test_cases`. Also updates ESRI and QGIS agreement raster symbology label to include the addition of levee-protected areas as a mask.
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
index 1d3759a67..274fe1727 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -121,19 +121,6 @@ if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
     Tcount
 fi
 
-## RASTERIZE LEVEE-PROTECTED AREAS POLYGON (IF APPLICABLE) ##
-if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
-    echo -e $startDiv"Rasterize levee-protected areas polygons $hucNumber $current_branch_id"$stopDiv
-    date -u
-    Tstart
-    if [ $current_branch_id = $branch_zero_id ]; then 
-        gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LeveeProtectedAreas_subset.gpkg $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif
-    else
-        $srcDir/gms/rasterize_by_order.py -c $ncols -r $nrows -n $ndv --x-min $xmin --y-min $ymin --x-max $xmax --y-max $ymax -v $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -f $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif -s $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -i $current_branch_id
-    fi
-    Tcount
-fi
-
 ## POLYGONIZE REACH WATERSHEDS ##
 echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"$stopDiv
 date -u
@@ -177,15 +164,6 @@ if  [ -f $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; th
     Tcount
 fi
 
-## MASK REM RASTER TO REMOVE LEVEE-PROTECTED AREAS ##
-if [ -f $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif ]; then
-    echo -e $startDiv"Additional masking to REM raster to remove levee-protected areas $hucNumber $current_branch_id"$stopDiv
-    date -u
-    Tstart
-    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LeveeProtectedAreas_subset_$current_branch_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
-    Tcount
-fi
-
 ## HYDRAULIC PROPERTIES ##
 echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"$stopDiv
 date -u
diff --git a/src/gms/rasterize_by_order.py b/src/gms/rasterize_by_order.py
deleted file mode 100755
index 8e5072d76..000000000
--- a/src/gms/rasterize_by_order.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-from osgeo import gdal
-import geopandas as gpd
-
-def rasterize_by_order(vector_filename, raster_filename, x_min, y_min, x_max, y_max, columns, rows, nodata_value, stream_layer, branch_id_attribute, branch_id, order_attribute):
-    """
-    Rasterizes polygon layers exceeding a stream order threshold
-    """
-
-    streams_df = gpd.read_file(stream_layer, ignore_geometry=True)
-
-    # Rasterize if branch is at least the minimum order
-    if streams_df.loc[streams_df[branch_id_attribute].astype(int)==branch_id, order_attribute].max() >= streams_df[order_attribute].max() - 1:
-
-        # Open the data source and read in the extent
-        source_ds = gdal.OpenEx(vector_filename)
-
-        gdal.Rasterize(raster_filename,
-                        source_ds,
-                        format='GTIFF',
-                        outputType=gdal.GDT_Int32,
-                        creationOptions=["COMPRESS=LZW", "BIGTIFF=YES", "TILES=YES"],
-                        noData=nodata_value,
-                        initValues=1,
-                        width=columns,
-                        height=rows,
-                        allTouched=True,
-                        burnValues=nodata_value,
-                        outputBounds=[x_min, y_min, x_max, y_max])
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Rasterize a polygon layer')
-    parser.add_argument('-v', '--vector-filename', help='Input vector filename', required=True)
-    parser.add_argument('-f', '--raster-filename', help='Output raster filename', required=True)
-    parser.add_argument('--x-min', help='Minimum x coordinate', required=True)
-    parser.add_argument('--y-min', help='Minimun y coordinate', required=True)
-    parser.add_argument('--x-max', help='Maximum x coordinate', required=True)
-    parser.add_argument('--y-max', help='Maximum y coordinate', required=True)
-    parser.add_argument('-c', '--columns', help='Output raster columns', required=True)
-    parser.add_argument('-r', '--rows', help='Output raster rows', required=True)
-    parser.add_argument('-n', '--nodata-value', help='NoData value', required=True)
-    parser.add_argument('-s', '--stream-layer', help='Stream layer filename', required=True)
-    parser.add_argument('-b', '--branch-id-attribute', help='Branch ID attribute name', required=False, default='levpa_id')
-    parser.add_argument('-i', '--branch-id', help='Branch ID', type=int, required='True')
-    parser.add_argument('-a', '--order-attribute', help='Stream order attribute name', required=False, default='order_')
-
-    args = vars(parser.parse_args())
-
-    rasterize_by_order(**args)
diff --git a/src/gms/rem.py b/src/gms/rem.py
index aa7faeafa..b90753fee 100755
--- a/src/gms/rem.py
+++ b/src/gms/rem.py
@@ -73,7 +73,7 @@ def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
         rem_window = np.zeros(len(flat_dem),dtype=np.float32)
         for i,cm in enumerate(flat_catchments):
             if cm in catchmentMinDict:
-                if catchmentMinDict[cm] == ndv:
+                if catchmentMinDict[cm] == ndv or flat_dem[i] == ndv:
                     rem_window[i] = ndv
                 else:
                     rem_window[i] = flat_dem[i] - catchmentMinDict[cm]

From f25179c34a05524e0df55e6f29092ec10ade51ca Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 6 Jan 2023 15:40:04 -0600
Subject: [PATCH 196/359] v4.0.17.4 Update hydrovis S3 push whitelist (#781)

---
 config/aws_s3_put_fim4_hydrovis_whitelist.lst |  1 +
 docs/CHANGELOG.md                             | 13 +++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/config/aws_s3_put_fim4_hydrovis_whitelist.lst b/config/aws_s3_put_fim4_hydrovis_whitelist.lst
index 5d8ed5ca7..0b1345ef3 100644
--- a/config/aws_s3_put_fim4_hydrovis_whitelist.lst
+++ b/config/aws_s3_put_fim4_hydrovis_whitelist.lst
@@ -2,3 +2,4 @@ hydroTable{}.csv
 gw_catchments_reaches_filtered_addedAttributes{}.tif
 rem_zeroed_masked{}.tif
 usgs_elev_table.csv
+crosswalk_table.csv
\ No newline at end of file
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0bc066d57..84e549954 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.17.4 - 2023-01-06 - [PR#781](https://github.com/NOAA-OWP/inundation-mapping/pull/781)
+
+Added crosswalk_table.csv from the root output folder as being a file push up to Hydrovis s3 bucket after FIM BED runs.
+
+### Changes
+
+- `config`
+    - `aws_s3_put_fim4_hydrovis_whitelist.lst`:  Added crosswalk_table.csv to whitelist.
+
+
+<br/><br/>
+
 ## v4.0.17.3 - 2022-12-23 - [PR#773](https://github.com/NOAA-OWP/inundation-mapping/pull/773)
 
 Cleans up REM masking of levee-protected areas and fixes associated error.

From 0466ec4a3cb9f2a24671a2ba3e9ac4087cc2c19f Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 6 Jan 2023 14:44:51 -0700
Subject: [PATCH 197/359] v4.0.18.0 Clip WBD and branch buffer polygons to DEM
 domain (#780)

---
 docs/CHANGELOG.md                 | 18 ++++++++++++++++++
 src/clip_vectors_to_wbd.py        | 18 +++++++++---------
 src/gms/buffer_stream_branches.py | 14 ++++++++++----
 src/gms/derive_level_paths.py     | 10 +++++++---
 src/gms/mask_dem.py               | 24 +++++++++++++-----------
 src/gms/remove_error_branches.py  |  2 +-
 src/gms/run_by_unit.sh            |  8 ++++----
 src/gms/toDo.md                   |  0
 src/split_flows.py                | 15 +++++++++++++--
 9 files changed, 75 insertions(+), 34 deletions(-)
 mode change 100644 => 100755 src/gms/mask_dem.py
 mode change 100644 => 100755 src/gms/remove_error_branches.py
 mode change 100644 => 100755 src/gms/toDo.md

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 84e549954..5bd79bc5f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.18.0 - 2023-01-03 - [PR#780](https://github.com/NOAA-OWP/inundation-mapping/pull/780)
+
+Clips WBD and stream branch buffer polygons to DEM domain.
+
+### Changes
+
+- `src/`
+    - `clip_vectors_to_wbd.py`: Clips WBD polygon to DEM domain
+
+    - `gms/`
+        - `buffer_stream_branches.py`: Clips branch buffer polygons to DEM domain
+        - `derive_level_paths.py`: Stop processing if no branches exist
+        - `mask_dem.py`: Checks if stream file exists before continuing
+        - `remove_error_branches.py`: Checks if error_branches has data before continuing
+        - `run_by_unit.sh`: Adds DEM domain as bash variable and adds it as an argument to calling `clip_vectors_to_wbd.py` and `buffer_stream_branches.py`
+
+<br/><br/>
+
 
 ## v4.0.17.4 - 2023-01-06 - [PR#781](https://github.com/NOAA-OWP/inundation-mapping/pull/781)
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 60f8639a9..b4cd7d27a 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -37,18 +37,20 @@ def subset_vector_layers(subset_nwm_lakes,
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
 
-    # Get wbd buffer
-    wbd = gpd.read_file(wbd_filename)
-    wbd_buffer = wbd.copy()
-    wbd_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance, resolution=32)
-
     # Erase area outside 3DEP domain
+    wbd = gpd.read_file(wbd_filename)
     dem_domain = gpd.read_file(dem_domain)
+    wbd = gpd.clip(wbd, dem_domain)
+    wbd.to_file(wbd_filename, layer='WBDHU8')
+
+    # Get wbd buffer
+    wbd_buffer = wbd.copy()
+    wbd_buffer.geometry = wbd_buffer.geometry.buffer(wbd_buffer_distance, resolution=32)
     wbd_buffer = gpd.clip(wbd_buffer, dem_domain)
 
     # Make the streams buffer smaller than the wbd_buffer so streams don't reach the edge of the DEM
-    wbd_streams_buffer = wbd.copy()
-    wbd_streams_buffer.geometry = wbd.geometry.buffer(wbd_buffer_distance-3*dem_cellsize, resolution=32)
+    wbd_streams_buffer = wbd_buffer.copy()
+    wbd_streams_buffer.geometry = wbd_streams_buffer.geometry.buffer(-3*dem_cellsize, resolution=32)
 
     great_lakes = gpd.read_file(great_lakes, mask=wbd_buffer).reset_index(drop=True)
 
@@ -57,11 +59,9 @@ def subset_vector_layers(subset_nwm_lakes,
 
         # Clip excess lake area
         great_lakes = gpd.clip(great_lakes, wbd_buffer)
-        great_lakes_streams = gpd.clip(great_lakes, wbd_streams_buffer)
 
         # Buffer remaining lake area
         great_lakes.geometry = great_lakes.buffer(lake_buffer_distance)
-        great_lakes_streams.geometry = great_lakes_streams.buffer(lake_buffer_distance)
 
         # Removed buffered GL from WBD buffer
         wbd_buffer = gpd.overlay(wbd_buffer, great_lakes, how='difference')
diff --git a/src/gms/buffer_stream_branches.py b/src/gms/buffer_stream_branches.py
index cb1426d2c..06313f5ad 100755
--- a/src/gms/buffer_stream_branches.py
+++ b/src/gms/buffer_stream_branches.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import geopandas as gpd
 from stream_branches import StreamNetwork
 from stream_branches import StreamBranchPolygons
 import argparse
@@ -10,6 +11,7 @@
 
     # parse arguments
     parser = argparse.ArgumentParser(description='Generates branch polygons')
+    parser.add_argument('-a', '--dem-domain', help='DEM domain file', required=False, type=str)
     parser.add_argument('-s','--streams', help='Streams file to branch', required=True)
     parser.add_argument('-i','--branch-id', help='Attribute with branch ids', required=True)
     parser.add_argument('-d','--buffer-distance', help='Distance to buffer branches to create branch polygons', required=True,type=int)
@@ -19,17 +21,21 @@
     # extract to dictionary
     args = vars(parser.parse_args())
 
-    streams_file, branch_id_attribute, buffer_distance, stream_polygons_file, verbose = args["streams"], args["branch_id"] , args["buffer_distance"], args["branches"] , args["verbose"]
+    streams_file, branch_id_attribute, buffer_distance, stream_polygons_file, dem_domain, verbose = args["streams"], args["branch_id"] , args["buffer_distance"], args["branches"], args['dem_domain'] , args["verbose"]
     
     if os.path.exists(streams_file):
         # load file
         stream_network = StreamNetwork.from_file( filename=streams_file,branch_id_attribute=branch_id_attribute,
-                                                values_excluded=None,attribute_excluded=None, verbose = verbose)
+                                                values_excluded=None, attribute_excluded=None, verbose = verbose)
         
         # make stream polygons 
         stream_polys = StreamBranchPolygons.buffer_stream_branches( stream_network,
                                                                     buffer_distance=buffer_distance,
                                                                     verbose=verbose                  )
-        
-        stream_polys.write(stream_polygons_file,verbose=verbose)
 
+        # Clip to DEM domain
+        if os.path.exists(dem_domain):
+            dem_domain = gpd.read_file(dem_domain)
+            stream_polys.geometry = gpd.clip(stream_polys, dem_domain).geometry
+        
+        stream_polys.write(stream_polygons_file,verbose=verbose)
\ No newline at end of file
diff --git a/src/gms/derive_level_paths.py b/src/gms/derive_level_paths.py
index aaf2eeb19..fd74ddbd3 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/gms/derive_level_paths.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import os
 import argparse
 import geopandas as gpd
 import sys
@@ -23,7 +24,11 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
     if verbose:
         print("Loading stream network ...")
         
-    stream_network = StreamNetwork.from_file(filename=in_stream_network)
+    if os.path.exists(in_stream_network):
+        stream_network = StreamNetwork.from_file(filename=in_stream_network)
+    else:
+        print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
+        sys.exit(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # if there are no reaches at this point
     if (len(stream_network) == 0):
@@ -38,8 +43,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
 
     # values_exluded of 1 and 2 mean where are dropping stream orders 1 and 2. We are leaving those
     # for branch zero.
-    stream_network = stream_network.exclude_attribute_values(branch_id_attribute="order_",
-                                            values_excluded=[1,2] )
+    stream_network = stream_network.exclude_attribute_values(branch_id_attribute="order_", values_excluded=[1,2] )
 
     # if there are no reaches at this point (due to filtering)
     if (len(stream_network) == 0):
diff --git a/src/gms/mask_dem.py b/src/gms/mask_dem.py
old mode 100644
new mode 100755
index d8916c2fb..42f0a8feb
--- a/src/gms/mask_dem.py
+++ b/src/gms/mask_dem.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import os
 import geopandas as gpd
 import fiona
 import rasterio as rio
@@ -12,22 +13,23 @@ def mask_dem(dem_filename, nld_filename, out_dem_filename, stream_layer, branch_
     """
     Masks levee-protected areas from DEM in branch 0 or if stream order is at least the minimum order (max - 1)
     """
-    streams_df = gpd.read_file(stream_layer, ignore_geometry=True)
+    if os.path.exists(stream_layer):
+        streams_df = gpd.read_file(stream_layer, ignore_geometry=True)
 
-    # Rasterize if branch zero or if stream order is at least the minimum order (max - 1)
-    if (branch_id == branch_zero_id) or (streams_df.loc[streams_df[branch_id_attribute].astype(int)==branch_id, order_attribute].max() >= streams_df[order_attribute].max() - 1):
+        # Rasterize if branch zero or if stream order is at least the minimum order (max - 1)
+        if (branch_id == branch_zero_id) or (streams_df.loc[streams_df[branch_id_attribute].astype(int)==branch_id, order_attribute].max() >= streams_df[order_attribute].max() - 1):
 
-        with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
-            dem_data = dem.read(1)
-            dem_profile = dem.profile.copy()
+            with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
+                dem_data = dem.read(1)
+                dem_profile = dem.profile.copy()
 
-            geoms = [feature["geometry"] for feature in leveed]
+                geoms = [feature["geometry"] for feature in leveed]
 
-            # Mask out levee-protected areas from DEM
-            out_dem_masked, _ = mask(dem, geoms, invert=True)
+                # Mask out levee-protected areas from DEM
+                out_dem_masked, _ = mask(dem, geoms, invert=True)
 
-            with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
-                dest.write(out_dem_masked[0,:,:], indexes=1)
+                with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
+                    dest.write(out_dem_masked[0,:,:], indexes=1)
 
 
 if __name__ == '__main__':
diff --git a/src/gms/remove_error_branches.py b/src/gms/remove_error_branches.py
old mode 100644
new mode 100755
index 7dfaf26f8..30afbc44b
--- a/src/gms/remove_error_branches.py
+++ b/src/gms/remove_error_branches.py
@@ -68,7 +68,7 @@ def remove_error_branches(logfile, gms_inputs):
                     error_branches = pd.concat([error_branches, tmp_df])
 
         # Save list of removed branches
-        if len(error_branches) > 0:
+        if error_branches is not None and len(error_branches) > 0:
             pd.DataFrame(error_branches).to_csv(gms_inputs_removed, header=False, index=False)
 
             # Overwrite gms_inputs.csv with error branches removed
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 38a70d6ac..67bc2e244 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -48,6 +48,7 @@ input_NHD_WBHD_layer=WBDHU$hucUnitLength
 
 default_projection_crs="ESRI:102039"
 input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
+input_DEM_domain=$inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
 input_bathy_bankfull=$inputDataDir/$bankfull_input_table
 
@@ -81,7 +82,7 @@ cmd_args+=" -e $outputHucDataDir/nwm_headwater_points_subset.gpkg"
 cmd_args+=" -f $outputHucDataDir/wbd_buffered.gpkg"
 cmd_args+=" -g $outputHucDataDir/wbd.gpkg"
 cmd_args+=" -i $input_DEM"
-cmd_args+=" -j $inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg"
+cmd_args+=" -j $input_DEM_domain"
 cmd_args+=" -l $input_nwm_lakes"
 cmd_args+=" -m $input_nwm_catchments"
 cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"
@@ -102,7 +103,7 @@ Tcount
 python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
 
 : '
-python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nwm_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nwm_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM
+python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nwm_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nwm_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM -j $input_DEM_domain
 '
 
 ## Clip WBD8 ##
@@ -118,7 +119,6 @@ date -u
 Tstart
 $srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 
-
 # test if we received a non-zero code back from derive_level_paths.py
 subscript_exit_code=$?
 # we have to retrow it if it is not a zero (but it will stop further execution in this script)
@@ -129,7 +129,7 @@ Tcount
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"$stopDiv
 date -u
 Tstart
-$srcDir/gms/buffer_stream_branches.py -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg -v 
+$srcDir/gms/buffer_stream_branches.py -a $input_DEM_domain -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg -v
 Tcount
 
 ## CREATE BRANCHID LIST FILE
diff --git a/src/gms/toDo.md b/src/gms/toDo.md
old mode 100644
new mode 100755
diff --git a/src/split_flows.py b/src/split_flows.py
index 9d83122d0..100be57b1 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -108,9 +108,15 @@ def split_flows(max_length,
     # split at HUC8 boundaries
     print ('splitting stream segments at HUC8 boundaries')
     flows = gpd.overlay(flows, wbd8, how='union').explode().reset_index(drop=True)
+    flows = flows[~flows.is_empty]
+
+    if (len(flows) == 0):
+        # this is not an exception, but a custom exit code that can be trapped
+        print("No relevant streams within HUC boundaries.")
+        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
     # check for lake features
-    if lakes is not None:
+    if lakes is not None and len(flows) > 0 :
         if len(lakes) > 0:
           print ('splitting stream segments at ' + str(len(lakes)) + ' waterbodies')
           #create splits at lake boundaries
@@ -125,6 +131,11 @@ def split_flows(max_length,
     # remove empty geometries
     flows = flows.loc[~flows.is_empty,:]
 
+    if (len(flows) == 0):
+        # this is not an exception, but a custom exit code that can be trapped
+        print("No relevant streams within HUC boundaries.")
+        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+
     for i,lineString in tqdm(enumerate(flows.geometry),total=len(flows.geometry)):
         # Reverse geometry order (necessary for BurnLines)
         lineString = LineString(lineString.coords[::-1])
@@ -266,7 +277,7 @@ def split_flows(max_length,
     max_length             = float(environ['max_split_distance_meters'])
     slope_min              = float(environ['slope_min'])
     lakes_buffer_input     = float(environ['lakes_buffer_dist_meters'])
-
+    
     # Parse arguments.
     parser = argparse.ArgumentParser(description='splitflows.py')
     parser.add_argument('-f', '--flows-filename', help='flows-filename',required=True)

From 501b48acf0fe7ad88369ad568dda4ebdeebf77a1 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 13 Jan 2023 11:10:29 -0600
Subject: [PATCH 198/359] v4.0.18.1 Stacked Bar Eval Plots (#760)

---
 docs/CHANGELOG.md              |  12 ++
 tools/eval_plots_stackedbar.py | 324 +++++++++++++++++++++++++++++++++
 2 files changed, 336 insertions(+)
 create mode 100644 tools/eval_plots_stackedbar.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5bd79bc5f..1727b3954 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.18.1 - 2022-12-13 - [PR #760](https://github.com/NOAA-OWP/inundation-mapping/pull/760)
+
+Adds stacked bar eval plots.
+
+### Additions
+
+- `/tools/eval_plots_stackedbar.py`: produces stacked bar eval plots in the same manner as `eval_plots.py`.
+
+<br/><br/>
+
 ## v4.0.18.0 - 2023-01-03 - [PR#780](https://github.com/NOAA-OWP/inundation-mapping/pull/780)
 
 Clips WBD and stream branch buffer polygons to DEM domain.
@@ -216,6 +226,8 @@ Fixes inundation of nodata areas of REM.
 
 - `tools/inundation.py`: Assigns depth a value of `0` if REM is less than `0`
 
+<br/><br/>
+
 ## v4.0.13.1 - 2022-12-09 - [PR #743](https://github.com/NOAA-OWP/inundation-mapping/pull/743)
 
 This merge adds the tools required to generate Alpha metrics by hydroid. It summarizes the Apha metrics by branch 0 catchment for use in the Hydrovis "FIM Performance" service.
diff --git a/tools/eval_plots_stackedbar.py b/tools/eval_plots_stackedbar.py
new file mode 100644
index 000000000..8cf8705fc
--- /dev/null
+++ b/tools/eval_plots_stackedbar.py
@@ -0,0 +1,324 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import sys
+import argparse
+from pathlib import Path
+sys.path.append('/foss_fim/tools/')
+from tools_shared_variables import BAD_SITES
+
+from matplotlib.patches import Patch
+from matplotlib.lines import Line2D
+import mpl_toolkits.axisartist as AA
+mag_dict = {'action':'1_act', 'minor':'2_min', 'moderate':'3_mod', 'major':'4_maj',
+           '100yr':100, '500yr':500, '2yr':2,'5yr':5, 
+            '10yr':10, '25yr':25, '50yr':50, '200yr':200}
+inverted_mag = dict((v, k) for k, v in mag_dict.items())
+
+def eval_plot_stack_data_prep(metric_csv, versions=[]):
+    
+    # Load in FIM 4 CSV and select the proper version metrics
+    metrics = pd.read_csv(metric_csv, dtype={'huc':str})
+    if versions:
+        versions = list(versions)
+        # Check to make sure requested versions are in the metrics file
+        for v in versions:
+            assert v in metrics.version.unique(), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
+        # Filter the metrics to the requested versions
+        metrics = metrics.loc[metrics.version.isin(versions)]
+    # Change the magnitudes to make them more sort-friendly for the plot below
+    metrics.magnitude = metrics.magnitude.apply(lambda m: mag_dict[m])
+    # Fill the non-AHPS sites with values for nws_lid
+    metrics.nws_lid.fillna(metrics.huc +'_'+ metrics.benchmark_source, inplace=True)
+    # Create multi-index for easy merging and plotting
+    metrics.set_index(['benchmark_source', 'nws_lid', 'magnitude'], inplace=True, drop=False)
+    
+    # Normalize data to the total of TP + FN
+    metrics['TP_FN_sum'] = metrics['false_negatives_count'] + metrics['true_positives_count']
+    metrics['TP_norm'] = metrics['true_positives_count'] / metrics['TP_FN_sum']
+    metrics['FN_norm'] = metrics['false_negatives_count'] / metrics['TP_FN_sum']
+    metrics['FP_norm'] = metrics['false_positives_count'] / metrics['TP_FN_sum']
+    metrics['FP_norm'].replace([np.inf, -np.inf], np.nan, inplace=True) # some bad sites will divide by 0
+    
+    return metrics
+
+def eval_plot_stack_indiv(metric_csv, versions, outfig, category):
+
+    # Check inputs
+    assert category in ('nws','usgs','ble','ifc','ras2fim'), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
+    
+    metrics_df = eval_plot_stack_data_prep(metric_csv, versions)
+
+    # Filter the plotting data to the selected category
+    data = metrics_df.loc[(category)]
+    num_subplots = len(data.nws_lid.unique())
+    xmax = data['FP_norm'].max() + 1
+    # Save version input order for plotting
+    version_dict = {v:i for i,v in enumerate(versions)}
+    data = data.assign(plot_order=data['version'])
+    data['plot_order'] = data.plot_order.map(version_dict)
+    data['magnitude_'] = data.magnitude
+    # Create the plot
+    fig, ax = plt.subplots(num_subplots, 1, figsize=(8, len(data)*0.25), dpi=100, facecolor='white', subplot_kw={'axes_class':AA.Axes})
+    # Create a subplot for every site (nws_lid)
+    for i, site in enumerate(sorted(data.nws_lid.unique())):
+        subplot_data = data.loc[(site)]
+        subplot_data = subplot_data.sort_values(['magnitude_','plot_order'], ascending=False)
+        new_y = [j*1.25 for j in range(len(subplot_data))]
+        # Stacked horizontal bar plots
+        ax[i].barh(y=new_y, width='TP_norm', left=0.0, color='#2c7bb6',
+                   linewidth=0.5, data=subplot_data)
+        ax[i].barh(y=new_y, width='FN_norm', left='TP_norm', color='#fdae61',
+                   linewidth=0.5, data=subplot_data)
+        ax[i].barh(y=new_y, width='FP_norm', left=1.0, color='#d7191c',
+                   linewidth=0.5, data=subplot_data)
+        # Plot the CSI and MCC scores
+        ax[i].scatter(y=new_y, x=subplot_data['CSI'].array, c='k', s=15, marker=r'x', zorder=3, linewidth=0.75)
+        ax[i].scatter(y=new_y, x=subplot_data['MCC'].array, s=15, marker=r'o', zorder=3, linewidths=.75, facecolor='None', edgecolors='k')
+        # Various axis customizations
+        ax[i].set_yticks(new_y, labels=subplot_data.version + ' ' + subplot_data.magnitude.map(inverted_mag).astype(str))
+        ax[i].axis["left"].label.set(visible=True, text=site, rotation=90, pad=10, ha='right')
+        ax[i].axis["left"].major_ticks.set(tick_out=True)
+        ax[i].axis["left"].major_ticklabels.set(ha='left')
+        ax[i].set_xlim(0, xmax)
+        ax[i].set_ylim(-1, new_y[-1]+1)
+        num_mags = len(subplot_data.magnitude.unique())
+        hlines = [j for j in np.linspace(new_y[-1]/num_mags, new_y[-1], num_mags-1, endpoint=False)]
+        ax[i].hlines(hlines, xmin=0, xmax=xmax, colors=['0.9' if i % 2 == 0 else 'w']*(num_mags-1), linestyles='dotted', zorder=0)
+        ax[i].grid(axis='x', color='0.8', zorder=0)
+        ax[i].axis['bottom'].set_visible(True if i == num_subplots-1 else False)
+        ax[i].axis['top'].set_visible(True if i == 0 else False)
+        ax[i].axis['top'].major_ticklabels.set_visible(True if i == 0 else False)
+        ax[i].set_facecolor('w' if i % 2 == 0 else '0.9')
+        # Label sites that have been identified as "Bad"
+        if site and site in BAD_SITES:
+            ax[i].text(xmax/2, new_y[-1]/2, '--BAD SITE--', horizontalalignment='center', verticalalignment='center')
+            ax[i].set_facecolor('0.67')
+    plt.subplots_adjust(wspace=0, hspace=0)
+    ax[0].set_title(f'{category.upper()} FIM Evaluation | Individual Sites', loc='center', pad=40)
+    TP_patch = Patch(color='#2c7bb6', linewidth=0.5, label=f'True Positive')
+    FN_patch = Patch(color='#fdae61', linewidth=0.5, label=f'False Negative')
+    FP_patch = Patch(color='#d7191c', linewidth=0.5, label=f'False Positive')
+    x_marker = Line2D([0], [0], marker='x', color='None', markeredgecolor='k', markerfacecolor='k', label='CSI Score', markersize=5, linewidth=0.5)
+    o_marker = Line2D([0], [0], marker='o', color='None', markeredgecolor='k', markerfacecolor='None', label='MCC Score', markersize=5, linewidth=10)
+    # Get the height of the figure in pixels so we can put the legend in a consistent position
+    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    handles = [TP_patch, FN_patch, FP_patch, x_marker, o_marker]
+    ax[0].legend(loc='center', ncol=len(handles), handles=handles, fontsize=8, columnspacing=1,
+                bbox_to_anchor=(0.5, (0.3+ax_pixel_height)/ax_pixel_height))
+    plt.savefig(outfig, bbox_inches='tight')
+    
+    return metrics_df
+
+def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
+
+    # Check inputs
+    assert category in ('nws','usgs','ble','ifc','ras2fim'), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
+    
+    metrics_df = eval_plot_stack_data_prep(metric_csv, versions)
+    metrics_df = metrics_df.loc[~metrics_df.nws_lid.isin(BAD_SITES)]
+    grouped = metrics_df.reset_index(drop=True).groupby(['benchmark_source','version','magnitude'], sort=False)
+    count_df = grouped.count()['CSI']
+    metrics_df = grouped.median()
+    metrics_df['TP_norm_q1'] = metrics_df['TP_norm'] - grouped['TP_norm'].quantile(0.25)
+    metrics_df['TP_norm_q3'] = grouped['TP_norm'].quantile(0.75) - metrics_df['TP_norm']
+    metrics_df['FP_norm_q1'] = metrics_df['FP_norm'] - grouped['FP_norm'].quantile(0.25)
+    metrics_df['FP_norm_q3'] = grouped['FP_norm'].quantile(0.75) - metrics_df['FP_norm']
+    
+    # Filter the plotting data to the selected category
+    data = metrics_df.loc[(category)].swaplevel()
+    num_subplots = len(data.reset_index().magnitude.unique())
+    num_ver = len(data.reset_index().version.unique())
+    xmax = 1+ data['FP_norm'].max() + data['FP_norm_q3'][data['FP_norm'].idxmax()]
+    # Save version input order for plotting
+    version_dict = {v:i for i,v in enumerate(versions)}
+    data = data.assign(plot_order=data.index.get_level_values('version'))
+    data['plot_order'] = data.plot_order.map(version_dict)
+    
+    # Create the plot
+    fig, ax = plt.subplots(num_subplots, 1, figsize=(8, len(data)*0.25), dpi=100, facecolor='white', subplot_kw={'axes_class':AA.Axes})
+    # Create a subplot for every flow (nws_lid)
+    for i, mag in enumerate(sorted(data.reset_index().magnitude.unique())):
+        subplot_data = data.loc[(mag)]
+        subplot_data = subplot_data.sort_values(['plot_order'], ascending=False)
+        new_y = [j*1.25 for j in range(len(subplot_data))]
+        # Stacked horizontal bar plots
+        ax[i].barh(y=new_y, width='TP_norm', left=0.0, color='#2c7bb6', 
+                   linewidth=0.5, data=subplot_data,
+                   xerr=[subplot_data['TP_norm_q1'], subplot_data['TP_norm_q3']] if show_iqr else None,
+                   error_kw=dict(elinewidth=1))
+        ax[i].barh(y=new_y, width='FN_norm', left='TP_norm', color='#fdae61', 
+                   linewidth=0.5, data=subplot_data)
+        ax[i].barh(y=new_y, width='FP_norm', left=1.0, color='#d7191c',
+                   linewidth=0.5, data=subplot_data, 
+                   xerr=[subplot_data['FP_norm_q1'], subplot_data['FP_norm_q3']] if show_iqr else None,
+                   error_kw=dict(elinewidth=1))
+        # Plot the CSI and MCC scores
+        ax[i].scatter(y=new_y, x=subplot_data['CSI'].array, c='k', s=15, marker=r'x', zorder=3, linewidth=0.75)
+        ax[i].scatter(y=new_y, x=subplot_data['MCC'].array, s=15, marker=r'o', zorder=3, linewidths=.75, facecolor='None', edgecolors='k')
+        # Various axis customizations
+        ax[i].set_yticks(new_y, labels=subplot_data.index)#, ha='left')
+        ax[i].axis["left"].label.set(visible=True, text=inverted_mag[mag], rotation=90, pad=10, ha='right')
+        ax[i].axis["left"].major_ticks.set(tick_out=True)
+        ax[i].axis["left"].major_ticklabels.set(ha='left')
+        n = count_df.loc[(category, versions[0], mag)]
+        ax[i].axis["right"].label.set(visible=True, text=f'n={n}', rotation=270, pad=5, ha='left')
+        ax[i].set_xlim(0, xmax)
+        ax[i].set_ylim(-1.25, num_ver+1.25)
+        ax[i].grid(axis='x', color='0.8', zorder=0)
+        ax[i].axis['bottom'].set_visible(True if i == num_subplots-1 else False)
+        ax[i].axis['top'].set_visible(True if i == 0 else False)
+        ax[i].axis['top'].major_ticklabels.set_visible(True if i == 0 else False)
+        ax[i].set_facecolor('w' if i % 2 == 0 else '0.9')
+    plt.subplots_adjust(wspace=0, hspace=0)
+    ax[0].set_title(f'{category.upper()} FIM Evaluation', loc='center', pad=35)
+    TP_patch = Patch(color='#2c7bb6', linewidth=0.5, label=f'True Positive')
+    FN_patch = Patch(color='#fdae61', linewidth=0.5, label=f'False Negative')
+    FP_patch = Patch(color='#d7191c', linewidth=0.5, label=f'False Positive')
+    x_marker = Line2D([0], [0], marker='x', color='None', markeredgecolor='k', markerfacecolor='k', label='CSI Score', markersize=5, linewidth=10)
+    o_marker = Line2D([0], [0], marker='o', color='None', markeredgecolor='k', markerfacecolor='None', label='MCC Score', markersize=5, linewidth=10)
+    # Get the height of the figure in pixels so we can put the legend in a consistent position
+    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    handles = [TP_patch, FN_patch, FP_patch, x_marker, o_marker]
+    ax[0].legend(loc='center', ncol=len(handles), handles=handles, fontsize=8, columnspacing=1,
+                bbox_to_anchor=(0.5, (0.3+ax_pixel_height)/ax_pixel_height))
+    plt.savefig(outfig, bbox_inches='tight')
+    
+    return data
+
+def diff_bar_plots(versions, metric_csv, category, outfig, stat='CSI'): 
+    
+    # Check inputs
+    assert category in ('nws','usgs','ble','ifc','ras2fim'), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
+    assert stat in ('CSI', 'MCC', 'TPR', 'PND', 'FAR'), f"stat must be one of ('CSI', 'MCC', 'TPR', 'PND', 'FAR'), not {stat}"
+    assert len(versions) == 2, f"You can only produce the comparison plots when comparing 2 fim versions. You have input {len(versions)}."
+    # CSI, MCC, and TPR scores are better when closer to 1, however PND and FAR scores are better when closer to 0
+    better_score = 'positive' if stat in ('CSI', 'MCC', 'TPR') else 'negative' 
+    
+    # Load in FIM 4 CSV and select the proper version metrics
+    metrics = pd.read_csv(metric_csv, dtype={'huc':str})
+    # Check to make sure requested versions are in the metrics file
+    for v in versions:
+        assert v in metrics.version.unique(), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
+    # Change the magnitudes to make them more sort-friendly for the plot below
+    metrics.magnitude = metrics.magnitude.apply(lambda m: mag_dict[m])
+    # Fill the non-AHPS sites with values for nws_lid
+    metrics.nws_lid.fillna(metrics.huc +'_'+ metrics.benchmark_source, inplace=True)
+    # Create multi-index for easy merging and plotting
+    metrics.set_index(['benchmark_source', 'nws_lid', 'magnitude'], inplace=True, drop=False)
+
+    # Separate versions into 2 dataframes
+    metrics_base = metrics.loc[metrics.version == versions[0]].copy()
+    metrics_new  = metrics.loc[metrics.version == versions[1]].copy()
+    
+    # Create a new column in the base dataframe and merge it into the comparison dataframe
+    metrics_base[f'BASE_{stat}'] = metrics_base[stat]
+    metrics_base.drop(columns=stat, inplace=True)
+    fim_compare = metrics_base.merge(metrics_new[stat], left_index=True, right_index=True, how='inner').sort_index(level=0,sort_remaining=True)
+    fim_compare[f'{stat}_diff'] = fim_compare[stat] - fim_compare[f'BASE_{stat}']
+    # Color the boxes according to improved vs regressed scores. See definition of 'better_score' above for details.
+    if better_score == 'positive':
+        fim_compare['color'] = fim_compare.apply(lambda row: 'None' if row[f'{stat}_diff'] < 0 else 'g', axis=1)
+        fim_compare['edge_color'] = fim_compare.apply(lambda row: 'r' if row[f'{stat}_diff'] < 0 else 'g', axis=1)
+    elif better_score == 'negative':
+        fim_compare['color'] = fim_compare.apply(lambda row: 'None' if row[f'{stat}_diff'] > 0 else 'g', axis=1)
+        fim_compare['edge_color'] = fim_compare.apply(lambda row: 'r' if row[f'{stat}_diff'] > 0 else 'g', axis=1)
+    
+    # Filter the plotting data to the selected category
+    data = fim_compare.loc[(category)]
+    num_subplots = len(data.nws_lid.unique())
+    num_mags = len(data.magnitude.unique())
+    # Create the plot
+    fig, ax = plt.subplots(num_subplots, 1, figsize=(8, len(data)*0.18), dpi=100, facecolor='white')
+    # Create a subplot for every site (nws_lid)
+    for i, site in enumerate(data.nws_lid.unique()):
+        subplot_data = data.loc[(site)]
+        ax[i].barh(y=np.arange(len(subplot_data)), width=f'{stat}_diff', left=f'BASE_{stat}', color='color',
+                   edgecolor='edge_color', linewidth=0.5, data=subplot_data, zorder=3)
+        ax[i].set_yticks(np.arange(len(subplot_data)), labels=subplot_data.index.map(inverted_mag).astype(str))
+        ax[i].set_xticks(np.linspace(0,1,11))
+        ax[i].set_ylabel(site, rotation='horizontal', labelpad=40)
+        ax[i].set_xlim(0, 1)
+        ax[i].set_ylim(-0.5, num_mags-0.25)
+        ax[i].grid(axis='x', color='0.8', zorder=0)
+        ax[i].spines['bottom'].set_color('0' if i == num_subplots-1 else 'None') 
+        ax[i].spines['top'].set_color('0' if i == 0 else '0.8') 
+        ax[i].set_facecolor('w' if i % 2 == 0 else '0.9')
+        ax[i].tick_params(axis="x", 
+                          bottom=True if i == num_subplots-1 else False, 
+                          top=True if i == 0 else False, 
+                          labelbottom=True if i == num_subplots-1 else False, 
+                          labeltop=True if i == 0 else False)
+        # Label sites that have been identified as "Bad"
+        if site and site in BAD_SITES:
+            ax[i].text(0.1, 1.5, '--BAD SITE--', horizontalalignment='center', verticalalignment='center')
+            ax[i].set_facecolor('0.67')
+    plt.subplots_adjust(wspace=0, hspace=0)
+    ax[0].set_title(f'{category.upper()} {stat} Comparison\n{versions[0]}  v  {versions[1]}', loc='center', pad=45)
+    g_patch = Patch(color='g', linewidth=0.5, label=f'{stat} Score Improvement')
+    r_patch = Patch(facecolor='None', edgecolor='r', linewidth=0.5, label=f'{stat} Score Regression')
+    # Get the height of the figure in pixels so we can put the legend in a consistent position
+    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    ax[0].legend(loc='center', ncol=2, handles=[g_patch, r_patch], fontsize=8,
+                bbox_to_anchor=(0.5, (0.4+ax_pixel_height)/ax_pixel_height))
+    plt.savefig(outfig, bbox_inches='tight')
+    
+    return fim_compare
+
+def iter_benchmarks(metric_csv, workspace, versions=[], individual_plots=False, show_iqr=False, diff_stat=False):
+    
+    # Import metrics csv as DataFrame and initialize all_datasets dictionary
+    csv_df = pd.read_csv(metric_csv, dtype = {'huc':str})
+
+    # If versions are supplied then filter out
+    if versions:
+        #Filter out versions based on supplied version list
+        metrics = csv_df.query('version.str.startswith(tuple(@versions))', engine='python')
+    else:
+        metrics = csv_df
+
+    # Group by benchmark source
+    benchmark_by_source = metrics.groupby(['benchmark_source', 'extent_config'])
+    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:
+
+        # Define and create the output workspace as a subfolder within the supplied workspace
+        output_workspace = Path(workspace) / benchmark_source / extent_configuration.lower()
+        output_workspace.mkdir(parents = True, exist_ok = True)
+        output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar{'_indiv'if individual_plots else ''}.png"
+        if diff_stat:
+            output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_diff_plot_{diff_stat}.png"
+            diff_bar_plots(versions, metric_csv, category=benchmark_source, outfig=output_png, stat=diff_stat)
+        elif individual_plots:
+            output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar_indiv.png"
+            eval_plot_stack_indiv(metric_csv=metric_csv, versions=versions, category=benchmark_source, outfig=output_png)
+        else:
+            output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar.png"
+            eval_plot_stack(metric_csv=metric_csv, versions=versions, category=benchmark_source, show_iqr=show_iqr, outfig=output_png)
+
+
+if __name__ == '__main__':
+    # Parse arguments
+    parser = argparse.ArgumentParser(description = f'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
+    parser.add_argument('-m','--metric_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
+    parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
+    parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
+    parser.add_argument('-i', '--site-plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
+    parser.add_argument('-iqr', '--show-iqr', help = 'If enabled, inter-quartile range error bars will be added.', action = 'store_true', required = False)
+    parser.add_argument('-d', '--diff-stat', help = 'This option creates diff plots instead of stacked bar plots. Only 2 versions can be used with this option. ' + \
+        "Input one of the following statistics to be used for comparison: ('CSI', 'MCC', 'TPR', 'PND', 'FAR').", default = False, required = False)
+
+    # Extract to dictionary and assign to variables
+    args = vars(parser.parse_args())
+
+    # Finalize Variables
+    m = args['metric_csv']
+    w = args['workspace']
+    v = args['versions']
+    i = args['site_plots']
+    iqr = args['show_iqr']
+    d = args['diff_stat']
+
+    # Run eval_plots function
+    print('The following AHPS sites are considered "BAD_SITES":  ' + ', '.join(BAD_SITES))
+    iter_benchmarks(m, w, v, i, iqr, d)
+

From 0868c6f925e7a3d43581b496d48d7711122901aa Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 13 Jan 2023 12:14:17 -0500
Subject: [PATCH 199/359] v4.0.18.2 Remove Great Lakes clipping (#790)

---
 docs/CHANGELOG.md          | 14 ++++++++++++++
 src/clip_vectors_to_wbd.py | 23 -----------------------
 src/gms/run_by_unit.sh     |  4 +---
 3 files changed, 15 insertions(+), 26 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1727b3954..0144ccb68 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.18.2 - 2023-01-11 - [PR#790](https://github.com/NOAA-OWP/inundation-mapping/pull/790)
+
+Remove Great Lakes clipping
+
+### Changes
+
+- `src/`
+    - `clip_vectors_to_wbd.py`: Removes Great Lakes clipping and references to Great Lakes polygons and lake buffer size
+
+    - `gms/run_by_unit.sh`: Removes Great Lakes polygon and lake buffer size arguments to `src/clip_vectors_to_wbd.py`
+
+<br/><br/>
+
 ## v4.0.18.1 - 2022-12-13 - [PR #760](https://github.com/NOAA-OWP/inundation-mapping/pull/760)
 
 Adds stacked bar eval plots.
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index b4cd7d27a..09e0ee898 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -26,8 +26,6 @@ def subset_vector_layers(subset_nwm_lakes,
                          subset_landsea,
                          nwm_headwaters,
                          subset_nld_lines,
-                         great_lakes,
-                         lake_buffer_distance,
                          wbd_buffer_distance,
                          levee_protected_areas,
                          subset_levee_protected_areas):
@@ -52,27 +50,10 @@ def subset_vector_layers(subset_nwm_lakes,
     wbd_streams_buffer = wbd_buffer.copy()
     wbd_streams_buffer.geometry = wbd_streams_buffer.geometry.buffer(-3*dem_cellsize, resolution=32)
 
-    great_lakes = gpd.read_file(great_lakes, mask=wbd_buffer).reset_index(drop=True)
-
-    if not great_lakes.empty:
-        print("Masking Great Lakes for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
-
-        # Clip excess lake area
-        great_lakes = gpd.clip(great_lakes, wbd_buffer)
-
-        # Buffer remaining lake area
-        great_lakes.geometry = great_lakes.buffer(lake_buffer_distance)
-
-        # Removed buffered GL from WBD buffer
-        wbd_buffer = gpd.overlay(wbd_buffer, great_lakes, how='difference')
-        wbd_streams_buffer = gpd.overlay(wbd_streams_buffer, great_lakes, how='difference')
-
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
     wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False)
 
-    del great_lakes
-
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea, mask=wbd_buffer)
     if not landsea.empty:
@@ -185,10 +166,6 @@ def subset_vector_layers(subset_nwm_lakes,
                         required=True)	 
     parser.add_argument('-z','--subset-nld-lines', help='Subset of NLD levee vectors for HUC',
                         required=True)
-    parser.add_argument('-gl','--great-lakes', help='Great Lakes layer', 
-                        required=True)
-    parser.add_argument('-lb','--lake-buffer-distance', help='Great Lakes Mask buffer distance',
-                        required=True, type=int)
     parser.add_argument('-wb','--wbd-buffer-distance', help='WBD Mask buffer distance', 
                         required=True, type=int)
     parser.add_argument('-lpf','--levee-protected-areas', 
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index 67bc2e244..b40a264f1 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -92,8 +92,6 @@ cmd_args+=" -w $input_nwm_flows"
 cmd_args+=" -x $outputHucDataDir/LandSea_subset.gpkg"
 cmd_args+=" -y $input_nwm_headwaters"
 cmd_args+=" -z $outputHucDataDir/nld_subset_levees.gpkg"
-cmd_args+=" -gl $input_GL_boundaries"
-cmd_args+=" -lb $lakes_buffer_dist_meters"
 cmd_args+=" -wb $wbd_buffer"
 cmd_args+=" -lpf $input_nld_levee_protected_areas"
 cmd_args+=" -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg"
@@ -103,7 +101,7 @@ Tcount
 python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
 
 : '
-python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nwm_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nwm_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer -i $input_DEM -j $input_DEM_domain
+python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nwm_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nwm_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -wb $wbd_buffer -i $input_DEM -j $input_DEM_domain
 '
 
 ## Clip WBD8 ##

From 48c1fbf9fc2bbfe0b9b0f7506001c45bc91c9e45 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 13 Jan 2023 11:21:03 -0600
Subject: [PATCH 200/359] v.4.0.19.0 Change the projection of HAND processing
 to EPSG:5070 (#782)

---
 data/nld/preprocess_levee_protected_areas.py  |  5 ++--
 data/usgs/acquire_and_preprocess_3dep_dems.py |  7 ++---
 docs/CHANGELOG.md                             | 18 +++++++++++++
 gms_run_post_processing.sh                    |  4 +--
 src/clip_vectors_to_wbd.py                    | 20 +++++++-------
 src/gms/run_by_unit.sh                        |  4 +--
 src/gms/stream_branches.py                    |  6 +++++
 src/src_adjust_spatial_obs.py                 | 27 +++++++------------
 src/utils/shared_variables.py                 | 12 ++++-----
 tools/inundate_nation.py                      |  5 ++--
 10 files changed, 61 insertions(+), 47 deletions(-)

diff --git a/data/nld/preprocess_levee_protected_areas.py b/data/nld/preprocess_levee_protected_areas.py
index 57654d263..d3f20a351 100644
--- a/data/nld/preprocess_levee_protected_areas.py
+++ b/data/nld/preprocess_levee_protected_areas.py
@@ -9,13 +9,14 @@
 from datetime import datetime
 
 sys.path.append('/foss_fim/src')
+from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 import utils.shared_functions as sf
 from utils.shared_functions import FIM_Helpers as fh
 
 def preprocess_levee_protected_areas(source_file_name_and_path,
                                      target_output_folder_path = '',
                                      target_output_filename = 'levee_protected_areas.gpkg',
-                                     projection_crs_id = 'EPSG:5070',
+                                     projection_crs_id = DEFAULT_FIM_PROJECTION_CRS,
                                      overwrite = False):
                                      
     '''
@@ -77,7 +78,7 @@ def preprocess_levee_protected_areas(source_file_name_and_path,
        
     # projection crs
     if (projection_crs_id is None) or (projection_crs_id == ''):
-        projection_crs_id = 'EPSG:5070'
+        projection_crs_id = DEFAULT_FIM_PROJECTION_CRS
 
     # -------------------
     # setup logs
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index a8be1252b..828c9cdef 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -143,7 +143,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
     base_cmd += ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
     base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
-    base_cmd += ' -t_srs EPSG:5070 -cblend 6'
+    base_cmd += ' -t_srs {3} -cblend 6'
    
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
 
@@ -212,7 +212,7 @@ def download_usgs_dem_file(extent_file,
                 base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
                 base_cmd +=  ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
                 base_cmd +=  ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
-                base_cmd +=  ' -t_srs EPSG:5070 -cblend 6'
+                base_cmd +=  ' -t_srs {3} -cblend 6'
         - retry (bool)
              If True, and the file exists (and is over 0k), downloading will be skipped.
         
@@ -246,7 +246,8 @@ def download_usgs_dem_file(extent_file,
             
     cmd = base_cmd.format(download_url,
                             target_path_raw,
-                            extent_file)
+                            extent_file,
+                            sv.DEFAULT_FIM_PROJECTION_CRS)
     #PREP_PROJECTION_EPSG
     #fh.vprint(f"cmd is {cmd}", self.is_verbose, True)
     #print(f"cmd is {cmd}")
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0144ccb68..d9d7ef81d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.19.0 - 2023-01-06 - [PR#782](https://github.com/NOAA-OWP/inundation-mapping/pull/782)
+
+Changes the projection of HAND processing to EPSG 5070.
+
+### Changes
+
+- `gms_run_post_processing.sh`: Adds target projection for `points`
+- `data/nld/preprocess_levee_protected_areas.py`: Changed to use `utils.shared_variables.DEFAULT_FIM_PROJECTION_CRS`
+- `src/`
+    - `clip_vectors_to_wbd.py`: Save intermediate outputs in EPSG:5070
+    - `src_adjust_spatial_obs.py`: Changed to use `utils.shared_variables.DEFAULT_FIM_PROJECTION_CRS`
+    - `utils/shared_variables.py`: Changes the designated projection variables
+    - `gms/`
+        - `stream_branches.py`: Checks the projection of the input streams and changes if necessary
+        - `run_by_unit.py`: Changes the default projection crs variable and added as HUC target projection
+- `tools/inundate_nation.py`: Changed to use `utils.shared_variables.PREP_PROJECTION`
+
+<br/><br/>
 
 ## v4.0.18.2 - 2023-01-11 - [PR#790](https://github.com/NOAA-OWP/inundation-mapping/pull/790)
 
diff --git a/gms_run_post_processing.sh b/gms_run_post_processing.sh
index 8397d44b2..b2659565d 100755
--- a/gms_run_post_processing.sh
+++ b/gms_run_post_processing.sh
@@ -141,9 +141,9 @@ if [ "$src_adjust_spatial" = "True" ]; then
         export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
         echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
         echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -a_srs ESRI:102039 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+        time ogr2ogr -overwrite -nln hucs -t_srs EPSG:5070 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
         echo "Loading Point Data"
-        time ogr2ogr -overwrite -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+        time ogr2ogr -overwrite -t_srs EPSG:5070 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
     fi
 fi
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 09e0ee898..ec660ea6c 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -6,6 +6,7 @@
 import rasterio as rio
 
 from shapely.geometry import MultiPolygon,Polygon
+from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 from utils.shared_functions import getDriver, mem_profile
 
 @mem_profile
@@ -39,7 +40,7 @@ def subset_vector_layers(subset_nwm_lakes,
     wbd = gpd.read_file(wbd_filename)
     dem_domain = gpd.read_file(dem_domain)
     wbd = gpd.clip(wbd, dem_domain)
-    wbd.to_file(wbd_filename, layer='WBDHU8')
+    wbd.to_file(wbd_filename, layer='WBDHU8', crs=DEFAULT_FIM_PROJECTION_CRS)
 
     # Get wbd buffer
     wbd_buffer = wbd.copy()
@@ -52,21 +53,20 @@ def subset_vector_layers(subset_nwm_lakes,
 
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
-    wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False)
+    wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
 
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea, mask=wbd_buffer)
     if not landsea.empty:
-        landsea.to_file(subset_landsea, driver = getDriver(subset_landsea), index=False)
+        landsea.to_file(subset_landsea, driver = getDriver(subset_landsea), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del landsea
 
     # Clip levee-protected areas polygons for future masking ocean areas (where applicable)
     print("Subsetting Levee Protected Areas", flush=True)
     levee_protected_areas = gpd.read_file(levee_protected_areas, mask=wbd_buffer)
     if not levee_protected_areas.empty:
-        levee_protected_areas = levee_protected_areas.to_crs('ESRI:102039')
         levee_protected_areas.to_file(subset_levee_protected_areas, driver = getDriver
-                                      (subset_levee_protected_areas), index=False)
+                                      (subset_levee_protected_areas), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del levee_protected_areas
 
     # Find intersecting lakes and writeout
@@ -81,14 +81,14 @@ def subset_vector_layers(subset_nwm_lakes,
         # Loop through the filled polygons and insert the new geometry
         for i in range(len(nwm_lakes_fill_holes)):
             nwm_lakes.loc[i, 'geometry'] = nwm_lakes_fill_holes[i]
-        nwm_lakes.to_file(subset_nwm_lakes, driver = getDriver(subset_nwm_lakes), index=False)
+        nwm_lakes.to_file(subset_nwm_lakes, driver = getDriver(subset_nwm_lakes), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del nwm_lakes
 
     # Find intersecting levee lines
     print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
     nld_lines = gpd.read_file(nld_lines, mask = wbd_buffer)
     if not nld_lines.empty:
-        nld_lines.to_file(subset_nld_lines, driver = getDriver(subset_nld_lines), index=False)
+        nld_lines.to_file(subset_nld_lines, driver = getDriver(subset_nld_lines), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del nld_lines
 
     # Subset NWM headwaters
@@ -96,7 +96,7 @@ def subset_vector_layers(subset_nwm_lakes,
     nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer)
 
     if len(nwm_headwaters) > 0:
-        nwm_headwaters.to_file(subset_nwm_headwaters, driver=getDriver(subset_nwm_headwaters), index=False)
+        nwm_headwaters.to_file(subset_nwm_headwaters, driver=getDriver(subset_nwm_headwaters), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     else:
         print ("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
@@ -107,7 +107,7 @@ def subset_vector_layers(subset_nwm_lakes,
     nwm_catchments = gpd.read_file(nwm_catchments, mask=wbd_buffer)
 
     if len(nwm_catchments) > 0:
-        nwm_catchments.to_file(subset_nwm_catchments, driver=getDriver(subset_nwm_catchments), index=False)
+        nwm_catchments.to_file(subset_nwm_catchments, driver=getDriver(subset_nwm_catchments), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     else:
         print ("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
@@ -124,7 +124,7 @@ def subset_vector_layers(subset_nwm_lakes,
     if len(nwm_streams) > 0:
         nwm_streams = gpd.clip(nwm_streams, wbd_streams_buffer)
 
-        nwm_streams.to_file(subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False)
+        nwm_streams.to_file(subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     else:
         print ("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index b40a264f1..acfee68db 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -46,7 +46,7 @@ huc4Identifier=${hucNumber:0:4}
 huc2Identifier=${hucNumber:0:2}
 input_NHD_WBHD_layer=WBDHU$hucUnitLength
 
-default_projection_crs="ESRI:102039"
+default_projection_crs="EPSG:5070"
 input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
 input_DEM_domain=$inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
@@ -108,7 +108,7 @@ python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $inp
 echo -e $startDiv"Clip WBD8"$stopDiv
 date -u
 Tstart
-ogr2ogr -f GPKG -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+ogr2ogr -f GPKG -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8 -t_srs $default_projection_crs
 Tcount
 
 ## DERIVE LEVELPATH  ##
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index 996231ed1..453cf716c 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -16,6 +16,7 @@
 from shapely.strtree import STRtree
 from random import sample
 from scipy.stats import mode
+from utils.shared_variables import PREP_CRS
 
 class StreamNetwork(gpd.GeoDataFrame):
 
@@ -73,6 +74,11 @@ def from_file(cls, filename, branch_id_attribute=None, values_excluded=None,
             print('Loading file')
             
         raw_df = gpd.read_file(filename,*args,**kwargs)
+
+        # Reproject
+        if raw_df.crs.to_authority() != PREP_CRS.to_authority():
+            raw_df.to_crs(PREP_CRS)
+
         filtered_df = gpd.GeoDataFrame()
                       
         if (branch_id_attribute is not None) and (values_excluded is not None):
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index 8122c9ece..1f2d08f1d 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -18,7 +18,7 @@
 from geopandas.tools import sjoin
 from multiprocessing import Pool
 from src_roughness_optimization import update_rating_curve
-from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH
+from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH, DEFAULT_FIM_PROJECTION_CRS
 
 #import variables from .env file
 load_dotenv()
@@ -52,7 +52,7 @@
 def process_points(args):
 
     '''
-    The funciton ingests geodataframe and attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow
+    The function ingests geodataframe and attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow
 
     Processing
     - Extract x,y coordinates from geometry
@@ -74,21 +74,12 @@ def process_points(args):
     ## Define coords variable to be used in point raster value attribution.
     coords = [(x,y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
 
+    water_edge_df.to_crs(DEFAULT_FIM_PROJECTION_CRS)
+    
     ## Use point geometry to determine HAND raster pixel values.
-    hand_src = rasterio.open(hand_path)
-    hand_crs = hand_src.crs
-    water_edge_df.to_crs(hand_crs)  # Reproject geodataframe to match hand_src. Should be the same, but this is a double check.
-    water_edge_df['hand'] = [h[0] for h in hand_src.sample(coords)]
-    hand_src.close()
-    del hand_src, hand_crs,
-
-    ## Use point geometry to determine catchment raster pixel values.
-    catchments_src = rasterio.open(catchments_path)
-    catchments_crs = catchments_src.crs
-    water_edge_df.to_crs(catchments_crs)
-    water_edge_df['hydroid'] = [c[0] for c in catchments_src.sample(coords)]
-    catchments_src.close()
-    del catchments_src, catchments_crs
+    with rasterio.open(hand_path) as hand_src, rasterio.open(catchments_path) as catchments_src:
+        water_edge_df['hand'] = [h[0] for h in hand_src.sample(coords)]
+        water_edge_df['hydroid'] = [c[0] for c in catchments_src.sample(coords)]
 
     water_edge_df = water_edge_df[(water_edge_df['hydroid'].notnull()) & (water_edge_df['hand'] > 0) & (water_edge_df['hydroid'] > 0)]
 
@@ -156,10 +147,10 @@ def find_points_in_huc(huc_id, conn):
     JOIN hucs H ON ST_Contains(H.geom, P.geom)
     WHERE H.huc8 = %s """
     
-    # Need to hard code the CRS to use EPSG:5070 instead of the default ESRI:102039 (gdal pyproj throws an error with crs 102039)
+    # Use EPSG:5070 instead of the default ESRI:102039 (gdal pyproj throws an error with crs 102039)
     # Appears that EPSG:5070 is functionally equivalent to ESRI:102039: https://gis.stackexchange.com/questions/329123/crs-interpretation-in-qgis
     water_edge_df = gpd.GeoDataFrame.from_postgis(huc_pt_query, con=conn, 
-                                                  params=[huc_id], crs="EPSG:5070",
+                                                  params=[huc_id], crs=DEFAULT_FIM_PROJECTION_CRS,
                                                   parse_dates=['coll_time'])
     water_edge_df = water_edge_df.drop(columns=['st_x','st_y'])
     
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 3a5801aaf..1c492c8bc 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
 
 import os
+from pyproj import CRS
 
-# Projections.
-#PREP_PROJECTION = "+proj=aea +datum=NAD83 +x_0=0.0 +y_0=0.0 +lon_0=96dW +lat_0=23dN +lat_1=29d30'N +lat_2=45d30'N +towgs84=-0.9956000824677655,1.901299877314078,0.5215002840524426,0.02591500053005733,0.009425998542707753,0.01159900118427752,-0.00062000005129903 +no_defs +units=m"
-
-# These two are ESRI:102039 (USA_Contiguous_Albers_Equal_Area_Conic_USGS_version)
-PREP_PROJECTION_CM = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Albers"],PARAMETER["false_easting",0.0],PARAMETER["false_northing",0.0],PARAMETER["central_meridian",-96.0],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_origin",23.0],UNIT["Meter",1.0],VERTCS["NAVD_1988",VDATUM["North_American_Vertical_Datum_1988"],PARAMETER["Vertical_Shift",0.0],PARAMETER["Direction",1.0],UNIT["Centimeter",0.01]]]'
-PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'  
-DEFAULT_FIM_PROJECTION_CRS = 'ESRI:102039'  
+# -- Projections: EPSG 5070-- #
+PREP_PROJECTION = 'PROJCRS["NAD83 / Conus Albers",BASEGEOGCRS["NAD83",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4269]],CONVERSION["Conus Albers",METHOD["Albers Equal Area",ID["EPSG",9822]],PARAMETER["Latitude of false origin",23,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8821]],PARAMETER["Longitude of false origin",-96,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8822]],PARAMETER["Latitude of 1st standard parallel",29.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8823]],PARAMETER["Latitude of 2nd standard parallel",45.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8824]],PARAMETER["Easting at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8826]],PARAMETER["Northing at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8827]]],CS[Cartesian,2],AXIS["easting (X)",east,ORDER[1],LENGTHUNIT["metre",1]],AXIS["northing (Y)",north,ORDER[2],LENGTHUNIT["metre",1]],USAGE[SCOPE["Data analysis and small scale data presentation for contiguous lower 48 states."],AREA["United States (USA) - CONUS onshore - Alabama; Arizona; Arkansas; California; Colorado; Connecticut; Delaware; Florida; Georgia; Idaho; Illinois; Indiana; Iowa; Kansas; Kentucky; Louisiana; Maine; Maryland; Massachusetts; Michigan; Minnesota; Mississippi; Missouri; Montana; Nebraska; Nevada; New Hampshire; New Jersey; New Mexico; New York; North Carolina; North Dakota; Ohio; Oklahoma; Oregon; Pennsylvania; Rhode Island; South Carolina; South Dakota; Tennessee; Texas; Utah; Vermont; Virginia; Washington; West Virginia; Wisconsin; Wyoming."],BBOX[24.41,-124.79,49.38,-66.91]],ID["EPSG",5070]]' 
+DEFAULT_FIM_PROJECTION_CRS = 'EPSG:5070'  
+PREP_CRS = CRS(PREP_PROJECTION)
 VIZ_PROJECTION ='PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
 
 # -- Data URLs-- #
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index 693017e5b..74a7c1bb2 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -16,16 +16,15 @@
 from gms_tools.mosaic_inundation import Mosaic_inundation
 from gms_tools.inundate_gms import Inundate_gms
 from inundation import inundate
-from tools_shared_variables import elev_raster_ndv
+from utils.shared_variables import elev_raster_ndv, PREP_PROJECTION
 from utils.shared_functions import FIM_Helpers as fh
 
 
 #INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
 #INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
 #INPUTS_DIR = r'/data/inputs'
-#OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/'
+#OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/
 #DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
-PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'
 
 
 def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, inc_mosaic, job_number):

From 3ade71833092a8f7d24920337458ded5ff0c19ce Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 20 Jan 2023 12:13:15 -0500
Subject: [PATCH 201/359] v4.0.19.1 Bug fix: combine_crosswalk_tables.py fails
 if only one HUC is run (#796)

---
 docs/CHANGELOG.md                           | 8 ++++++++
 tools/gms_tools/combine_crosswalk_tables.py | 5 +++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d9d7ef81d..1fe7b9421 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.19.1 - 2023-01-17 - [PR#796](https://github.com/NOAA-OWP/inundation-mapping/pull/796)
+
+### Changes
+
+- `tools/gms_tools/combine_crosswalk_tables.py`: Checks length of dataframe list before concatenating
+
+<br/><br/>
+
 ## v4.0.19.0 - 2023-01-06 - [PR#782](https://github.com/NOAA-OWP/inundation-mapping/pull/782)
 
 Changes the projection of HAND processing to EPSG 5070.
diff --git a/tools/gms_tools/combine_crosswalk_tables.py b/tools/gms_tools/combine_crosswalk_tables.py
index fdb6afcb7..df2aabcf4 100644
--- a/tools/gms_tools/combine_crosswalk_tables.py
+++ b/tools/gms_tools/combine_crosswalk_tables.py
@@ -24,9 +24,10 @@ def combine_crosswalk_tables(data_directory, output_filename):
         else:
             print(f'{filename} is missing.')
 
-    df = pd.concat(dfs)
+    if len(dfs) > 1:
+        df = pd.concat(dfs)
 
-    df.to_csv(output_filename, index=False)
+        df.to_csv(output_filename, index=False)
 
 
 if __name__ == '__main__':

From 6b933f7e4665600bc3d91d2f944c9b26c58de123 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 20 Jan 2023 12:20:41 -0500
Subject: [PATCH 202/359] v4.0.19.2 Consolidate bash environment variables into
 a single file (#797)

---
 config/mannings_default.json            | 16 ---------
 config/mannings_default_calibrated.json |  1 -
 config/params_template.env              |  2 +-
 docs/CHANGELOG.md                       | 28 +++++++++++++++
 gms_run_branch.sh                       |  2 +-
 gms_run_post_processing.sh              |  6 ++--
 gms_run_unit.sh                         | 14 ++------
 src/add_crosswalk.py                    | 12 +------
 src/aggregate_vector_inputs.py          | 48 +------------------------
 src/bash_variables.env                  | 13 +++++++
 src/gms/run_by_branch.sh                |  8 ++---
 src/gms/run_by_unit.sh                  | 13 ++++---
 src/utils/shared_variables.py           | 33 +++--------------
 13 files changed, 65 insertions(+), 131 deletions(-)
 delete mode 100644 config/mannings_default.json
 delete mode 100644 config/mannings_default_calibrated.json
 create mode 100644 src/bash_variables.env

diff --git a/config/mannings_default.json b/config/mannings_default.json
deleted file mode 100644
index d10b72607..000000000
--- a/config/mannings_default.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-     "1" : 0.06,
-     "2" : 0.06,
-     "3" : 0.06,
-     "4" : 0.06,
-     "5" : 0.06,
-     "6" : 0.06,
-     "7" : 0.06,
-     "8" : 0.06,
-     "9" : 0.06,
-    "10" : 0.06,
-    "11" : 0.06,
-    "12" : 0.06,
-    "13" : 0.06,
-    "14" : 0.06
-}
diff --git a/config/mannings_default_calibrated.json b/config/mannings_default_calibrated.json
deleted file mode 100644
index f75627ff4..000000000
--- a/config/mannings_default_calibrated.json
+++ /dev/null
@@ -1 +0,0 @@
-{"1": 0.125, "2": 0.125, "3": 0.125, "4": 0.125, "5": 0.125, "6": 0.125, "7": 0.125, "8": 0.105, "9": 0.1, "10": 0.05, "11": 0.06, "12": 0.06, "13": 0.06, "14": 0.06}
\ No newline at end of file
diff --git a/config/params_template.env b/config/params_template.env
index 12ced021c..384378de2 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -12,7 +12,7 @@ export ms_buffer_dist=7000
 export lakes_buffer_dist_meters=20
 
 #### rating curve parameters ####
-export manning_n="/foss_fim/config/mannings_default.json"
+export manning_n=0.06
 export stage_min_meters=0
 export stage_interval_meters=0.3048
 export stage_max_meters=25
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1fe7b9421..436a78443 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,34 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.19.2 - 2023-01-17 - [PR#797](https://github.com/NOAA-OWP/inundation-mapping/pull/797)
+
+Consolidates global bash environment variables into a new `src/bash_variables.env` file. Additionally, Python environment variables have been moved into this file and `src/utils/shared_variables.py` now references this file. Hardcoded projections have been replaced by an environment variable. This also replaces the Manning's N file in `config/params_template.env` with a constant and updates relevant code. Unused environment variables have been removed.
+
+### Additions
+
+- `src/bash_variables.env`: Adds file for global environment variables
+
+### Removals
+
+- `config/`
+    - `mannings_default.json`
+    - `mannings_default_calibrated.json`
+
+### Changes
+
+- `config/params_template.env`: Changes manning_n from filename to default value of 0.06
+- `gms_run_branch.sh`: Adds `bash_variables.env`
+- `gms_run_post_processing.sh`: Adds `bash_variables.env` and changes projection from hardcoded to environment variable
+- `gms_run_unit.sh`: Adds `bash_variables.env`
+- `src/`
+    - `add_crosswalk.py`: Assigns default manning_n value and removes assignments by stream orders
+    - `aggregate_vector_inputs.py`: Removes unused references to environment variables and function
+    - `gms/run_by_unit.sh`: Removes environment variable assignments and uses projection from environment variables
+    - `utils/shared_variables.py`: Removes environment variables and instead references src/bash_variables.env
+
+<br/><br/>
+
 ## v4.0.19.1 - 2023-01-17 - [PR#796](https://github.com/NOAA-OWP/inundation-mapping/pull/796)
 
 ### Changes
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index c8029af9b..928a381da 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -126,6 +126,7 @@ fi
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
 source $srcDir/bash_functions.env
+source $srcDir/bash_variables.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
@@ -137,7 +138,6 @@ export outputRunDataDir=$outputDataDir/$runName
 export deny_branches_list=$deny_branches_list
 logFile=$outputRunDataDir/logs/branch/summary_gms_branch.log
 export overwrite=$overwrite
-export extent=GMS
 
 ## Check for run data directory and the file. If gms_run_unit failed, the file will not be there ##
 if [ ! -f "$outputRunDataDir/gms_inputs.csv" ]; then 
diff --git a/gms_run_post_processing.sh b/gms_run_post_processing.sh
index b2659565d..7d2fa95bd 100755
--- a/gms_run_post_processing.sh
+++ b/gms_run_post_processing.sh
@@ -58,6 +58,7 @@ fi
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
 source $srcDir/bash_functions.env
+source $srcDir/bash_variables.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
@@ -139,11 +140,12 @@ if [ "$src_adjust_spatial" = "True" ]; then
         export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
         export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
         export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
+        export DEFAULT_FIM_PROJECTION_CRS=$DEFAULT_FIM_PROJECTION_CRS
         echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
         echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -t_srs EPSG:5070 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+        time ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
         echo "Loading Point Data"
-        time ogr2ogr -overwrite -t_srs EPSG:5070 -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+        time ogr2ogr -overwrite -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
     fi
 fi
 
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index bc2361e01..e8adf3e2e 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -121,6 +121,7 @@ fi
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $envFile
 source $srcDir/bash_functions.env
+source $srcDir/bash_variables.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
@@ -138,20 +139,9 @@ if [ -d $outputRunDataDir ] && [ $overwrite -eq 0 ]; then
     usage
 fi
 
-## Set misc global variables
-export overwrite=$overwrite
-
 ## Define inputs
-export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
-export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
-export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
-export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_nwm_headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
-export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
-export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Levee_protected_areas.gpkg
+export overwrite=$overwrite
 export deny_gms_unit_list=$deny_gms_unit_list
-export extent=GMS
-
 export deny_unit_list=$deny_unit_list
 export deny_branch_zero_list_for_units=$deny_branch_zero_list_for_units
 
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 4bc49d3cf..88398aa95 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -116,17 +116,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
         output_flows = output_flows.merge(output_catchments.filter(items=['HydroID','areasqkm']),on='HydroID')
 
-    # read in manning's n values
-    if calibration_mode == False:
-        with open(mannings_n, "r") as read_file:
-            mannings_dict = json.load(read_file)
-    else:
-        mannings_dict = {}
-        for cnt,value in enumerate(mannings_n.split(",")[2:]):
-            streamorder = cnt+1
-            mannings_dict[str(streamorder)] = value
-
-    output_flows['ManningN'] = output_flows['order_'].astype(str).map(mannings_dict)
+    output_flows['ManningN'] = mannings_n
 
     if output_flows.NextDownID.dtype != 'int': output_flows.NextDownID = output_flows.NextDownID.astype(int)
 
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
index aa47342b8..32b25739e 100755
--- a/src/aggregate_vector_inputs.py
+++ b/src/aggregate_vector_inputs.py
@@ -14,18 +14,7 @@
 from shapely.wkb import dumps, loads
 import pygeos
 
-nhdplus_vectors_dir = os.environ.get('nhdplus_vectors_dir')
-wbd_filename = os.environ.get('wbd_filename')
-nwm_streams_orig_filename = os.environ.get('nwm_streams_orig_filename')
-nwm_streams_all_filename = os.environ.get('nwm_streams_all_filename')
-nwm_headwaters_filename = os.environ.get('nwm_headwaters_filename')
-nwm_catchments_orig_filename = os.environ.get('nwm_catchments_orig_filename')
-nwm_catchments_all_filename = os.environ.get('nwm_catchments_all_filename')
-ahps_filename = os.environ.get('ahps_filename')
-nwm_huc4_intersections_filename = os.environ.get('nwm_huc4_intersections_filename')
-nhd_huc8_intersections_filename = os.environ.get('nhd_huc8_intersections_filename')
-agg_nhd_headwaters_adj_fileName = os.environ['agg_nhd_headwaters_adj_fileName']
-agg_nhd_streams_adj_fileName = os.environ['agg_nhd_streams_adj_fileName']
+
 
 
 def identify_nwm_ms_streams(nwm_streams_filename,ahps_filename,nwm_streams_all_filename):
@@ -314,37 +303,6 @@ def subset_stream_networks(args, huc):
     print(f"finished stream subset for HUC {huc}",flush=True)
 
 
-def aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,huc_list):
-
-    for huc in huc_list:
-
-        # aggregated final filenames
-        nhd_agg_adj_huc_subset = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
-        nhd_agg_adj_headwaters_subset = os.path.join(nhdplus_vectors_dir,huc,'nhd' + str(huc) + '_headwaters_adj.gpkg')
-
-        if os.path.isfile(nhd_agg_adj_huc_subset):
-            adj_nhd_streams_all = gpd.read_file(nhd_agg_adj_huc_subset)
-
-            # Write out FR adjusted
-            if os.path.isfile(agg_nhd_streams_adj_fileName):
-                adj_nhd_streams_all.to_file(agg_nhd_streams_adj_fileName,driver=getDriver(agg_nhd_streams_adj_fileName),index=False, mode='a')
-            else:
-                adj_nhd_streams_all.to_file(agg_nhd_streams_adj_fileName,driver=getDriver(agg_nhd_streams_adj_fileName),index=False)
-
-            del adj_nhd_streams_all
-
-        if os.path.isfile(nhd_agg_adj_headwaters_subset):
-            adj_nhd_headwater_points_all = gpd.read_file(nhd_agg_adj_headwaters_subset)
-
-            # Write out FR adjusted
-            if os.path.isfile(agg_nhd_headwaters_adj_fileName):
-                adj_nhd_headwater_points_all.to_file(agg_nhd_headwaters_adj_fileName,driver=getDriver(agg_nhd_headwaters_adj_fileName),index=False, mode='a')
-            else:
-                adj_nhd_headwater_points_all.to_file(agg_nhd_headwaters_adj_fileName,driver=getDriver(agg_nhd_headwaters_adj_fileName),index=False)
-
-            del adj_nhd_headwater_points_all
-
-
 def clean_up_intermediate_files(nhdplus_vectors_dir):
 
     for huc in os.listdir(nhdplus_vectors_dir):
@@ -393,9 +351,5 @@ def clean_up_intermediate_files(nhdplus_vectors_dir):
 
     del wbd4,wbd8
 
-    # Aggregate subset nhd networks for entire nwm domain
-    print ('Aggregating subset NHD networks for entire NWM domain')
-    aggregate_stream_networks(nhdplus_vectors_dir,agg_nhd_headwaters_adj_fileName,agg_nhd_streams_adj_fileName,missing_subsets)
-
     # Remove intermediate files
     # clean_up_intermediate_files(nhdplus_vectors_dir)
diff --git a/src/bash_variables.env b/src/bash_variables.env
new file mode 100644
index 000000000..6dfde858d
--- /dev/null
+++ b/src/bash_variables.env
@@ -0,0 +1,13 @@
+## Define inputs
+# NOTE: $inputDataDir is defined in Dockerfile
+export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
+export extent=GMS
+export input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
+export input_DEM_domain=$inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
+export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
+export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Levee_protected_areas.gpkg
+export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
+export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
+export input_nwm_headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
+export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
+export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index a7dd1c3f8..515bb60f9 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -43,13 +43,13 @@ echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"$
 date -u
 Tstart
 echo -e "Querying NWM streams ..."
-ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg
 echo -e "Querying NWM catchments ..."
-ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
 echo -e "Querying NWM Dissolved Levelpaths headwaters ..."
-ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
 #echo -e "Querying NWM headwaters ..."
-#ogr2ogr -f GPKG -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_headwaters.gpkg
+#ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_headwaters.gpkg
 Tcount
 
 ## GET RASTERS FROM BRANCH ZERO AND CLIP TO CURRENT BRANCH BUFFER ##
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
index acfee68db..25d5aeec8 100755
--- a/src/gms/run_by_unit.sh
+++ b/src/gms/run_by_unit.sh
@@ -3,6 +3,9 @@
 ## INITIALIZE TOTAL TIME TIMER ##
 T_total_start
 
+## SOURCE BASH FUNCTIONS
+source $srcDir/bash_variables.env
+
 ## SET OUTPUT DIRECTORY FOR UNIT ##
 hucNumber="$1"
 
@@ -46,11 +49,7 @@ huc4Identifier=${hucNumber:0:4}
 huc2Identifier=${hucNumber:0:2}
 input_NHD_WBHD_layer=WBDHU$hucUnitLength
 
-default_projection_crs="EPSG:5070"
-input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
-input_DEM_domain=$inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
 input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
-input_bathy_bankfull=$inputDataDir/$bankfull_input_table
 
 ## START MESSAGE ##
 echo -e $startDiv"Processing HUC: $hucNumber ..."$stopDiv
@@ -67,7 +66,7 @@ fi
 echo -e $startDiv"Get WBD $hucNumber"$stopDiv
 date -u
 Tstart
-ogr2ogr -f GPKG $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
 Tcount
 
 ## Subset Vector Layers ##
@@ -108,7 +107,7 @@ python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $inp
 echo -e $startDiv"Clip WBD8"$stopDiv
 date -u
 Tstart
-ogr2ogr -f GPKG -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8 -t_srs $default_projection_crs
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8
 Tcount
 
 ## DERIVE LEVELPATH  ##
@@ -160,7 +159,7 @@ echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"$stopD
 date -u
 Tstart
 [ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \
-gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $default_projection_crs $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
 Tcount
 
 ## GET RASTER METADATA
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 1c492c8bc..bdf19cec9 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -1,11 +1,12 @@
-#!/usr/bin/env python3
-
 import os
 from pyproj import CRS
 
+from dotenv import load_dotenv
+load_dotenv('/foss_fim/src/bash_variables.env')
+
 # -- Projections: EPSG 5070-- #
 PREP_PROJECTION = 'PROJCRS["NAD83 / Conus Albers",BASEGEOGCRS["NAD83",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4269]],CONVERSION["Conus Albers",METHOD["Albers Equal Area",ID["EPSG",9822]],PARAMETER["Latitude of false origin",23,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8821]],PARAMETER["Longitude of false origin",-96,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8822]],PARAMETER["Latitude of 1st standard parallel",29.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8823]],PARAMETER["Latitude of 2nd standard parallel",45.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8824]],PARAMETER["Easting at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8826]],PARAMETER["Northing at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8827]]],CS[Cartesian,2],AXIS["easting (X)",east,ORDER[1],LENGTHUNIT["metre",1]],AXIS["northing (Y)",north,ORDER[2],LENGTHUNIT["metre",1]],USAGE[SCOPE["Data analysis and small scale data presentation for contiguous lower 48 states."],AREA["United States (USA) - CONUS onshore - Alabama; Arizona; Arkansas; California; Colorado; Connecticut; Delaware; Florida; Georgia; Idaho; Illinois; Indiana; Iowa; Kansas; Kentucky; Louisiana; Maine; Maryland; Massachusetts; Michigan; Minnesota; Mississippi; Missouri; Montana; Nebraska; Nevada; New Hampshire; New Jersey; New Mexico; New York; North Carolina; North Dakota; Ohio; Oklahoma; Oregon; Pennsylvania; Rhode Island; South Carolina; South Dakota; Tennessee; Texas; Utah; Vermont; Virginia; Washington; West Virginia; Wisconsin; Wyoming."],BBOX[24.41,-124.79,49.38,-66.91]],ID["EPSG",5070]]' 
-DEFAULT_FIM_PROJECTION_CRS = 'EPSG:5070'  
+DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
 PREP_CRS = CRS(PREP_PROJECTION)
 VIZ_PROJECTION ='PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
 
@@ -13,7 +14,6 @@
 NHD_URL_PARENT = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/'
 NWM_HYDROFABRIC_URL = r'http://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_channel_hydrofabric.tar.gz'  # Temporary
 WBD_NATIONAL_URL = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip'
-WBD_HU2_URL_PARENT = r'http://prd-tnm.s3-website-us-west-2.amazonaws.com/?prefix=StagedProducts/Hydrography/WBD/HU2/GDB'
 
 
 # -- Prefixes and Suffixes -- #
@@ -55,28 +55,3 @@
 DOWNSTREAM_THRESHOLD = 10 # distance in km to propogate new roughness values downstream
 ROUGHNESS_MAX_THRESH = 0.8 # max allowable adjusted roughness value (void values larger than this)
 ROUGHNESS_MIN_THRESH = 0.001 # min allowable adjusted roughness value (void values smaller than this)
-
-## Input Paths and Directories
-# Directories
-os.environ['src_dir'] = '/foss_fim/src'
-os.environ['input_dir'] = 'data/inputs'
-
-#os.environ['nhdplus_rasters_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_rasters')
-os.environ['nhdplus_vectors_dir'] = os.path.join(os.environ.get('input_dir'),'nhdplus_vectors')
-os.environ['nwm_dir'] = os.path.join(os.environ.get('input_dir'),'nwm_hydrofabric')
-os.environ['wbd_dir'] = os.path.join(os.environ.get('input_dir'),'wbd')
-os.environ['ahps_dir'] = os.path.join(os.environ.get('input_dir'),'ahp_sites')
-os.environ['ahps_filename'] = os.path.join(os.environ.get('ahps_dir'),'nws_lid.gpkg')
-
-os.environ['3dep_dems_10m_5070'] = os.path.join(os.environ.get('input_dir'),'3dep_dems/10m_5070')
-os.environ['3dep_dems_10m_5070_vrt'] = os.path.join(os.environ.get('3dep_dems_10m_5070'),
-                                                    'fim_seamless_3dep_dem_10m_5070.vrt')
-
-os.environ['nwm_dir'] = os.path.join(os.environ.get('input_dir'),'nwm_hydrofabric')
-os.environ['nwm_streams_orig_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_flows_original.gpkg')
-os.environ['nwm_streams_all_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_flows.gpkg')
-os.environ['nwm_headwaters_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_headwaters.gpkg')
-os.environ['nwm_huc4_intersections_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_huc4_intersections.gpkg')
-os.environ['nwm_catchments_orig_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_catchments_original.gpkg')
-os.environ['nwm_catchments_all_filename'] = os.path.join(os.environ.get('nwm_dir'),'nwm_catchments.gpkg')
-os.environ['nhd_huc8_intersections_filename'] = os.path.join(os.environ.get('nwm_dir'),'nhd_huc8_intersections.gpkg')

From c1542872c60bd7a555825722c019ff2cee65b9ea Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 20 Jan 2023 11:27:46 -0600
Subject: [PATCH 203/359] v4.0.19.3 Remove fim3 files and refs (#794)

---
 api/.gitignore                                |   1 -
 api/README.md                                 |   0
 api/frontend/.env-template                    |   2 -
 api/frontend/docker-compose-dev.yml           |  52 --
 api/frontend/docker-compose-prod.yml          |  47 --
 api/frontend/gui/Dockerfile                   |  14 -
 api/frontend/gui/entrypoint.sh                |   5 -
 api/frontend/gui/gui.py                       |  16 -
 api/frontend/gui/requirements.txt             |   4 -
 api/frontend/gui/templates/index.html         | 560 ------------------
 api/frontend/gui/wsgi.py                      |   4 -
 api/frontend/nginx-dev.conf                   |  72 ---
 api/frontend/nginx-prod.conf                  |  55 --
 api/frontend/output_handler/Dockerfile        |  12 -
 api/frontend/output_handler/entrypoint.sh     |   6 -
 api/frontend/output_handler/output_handler.py |  46 --
 api/frontend/output_handler/requirements.txt  |   2 -
 api/node/.env-template                        |   6 -
 api/node/connector/Dockerfile                 |  16 -
 api/node/connector/connector.py               | 228 -------
 api/node/connector/entrypoint.sh              |   5 -
 api/node/connector/requirements.txt           |   4 -
 api/node/docker-compose-dev.yml               |  38 --
 api/node/docker-compose-prod.yml              |  53 --
 api/node/nginx.conf                           |  64 --
 api/node/updater/Dockerfile                   |  14 -
 api/node/updater/entrypoint.sh                |   5 -
 api/node/updater/requirements.txt             |   2 -
 api/node/updater/updater.py                   | 530 -----------------
 data/preprocess_rasters.py                    |  86 ---
 docs/CHANGELOG.md                             |  34 ++
 fim_run.sh                                    | 206 -------
 src/adjust_headwater_streams.py               |  15 +
 src/aggregate_fim_outputs.sh                  |  33 --
 src/fr_to_ms_raster_mask.py                   |  98 ---
 src/get_all_huc_in_inputs.py                  |  38 --
 src/gms/delineate_hydros_and_produce_HAND.sh  |   2 +-
 src/gms/{rem.py => make_rem.py}               |   0
 src/gms/toDo.md                               |   8 -
 src/reduce_nhd_stream_density.py              | 232 --------
 src/rem.py                                    | 189 ------
 src/run_by_unit.sh                            | 435 --------------
 src/time_and_tee_run_by_unit.sh               |  34 --
 src/utils/archive_cleanup.py                  |  63 --
 tools/compare_gms_srcs_to_fr.py               | 156 -----
 tools/cygnss_preprocessing.py                 |  16 +-
 tools/nesdis_preprocessing.py                 |  15 +
 tools/preprocess_fimx.py                      | 125 ----
 48 files changed, 80 insertions(+), 3568 deletions(-)
 delete mode 100644 api/.gitignore
 delete mode 100644 api/README.md
 delete mode 100644 api/frontend/.env-template
 delete mode 100644 api/frontend/docker-compose-dev.yml
 delete mode 100644 api/frontend/docker-compose-prod.yml
 delete mode 100644 api/frontend/gui/Dockerfile
 delete mode 100755 api/frontend/gui/entrypoint.sh
 delete mode 100644 api/frontend/gui/gui.py
 delete mode 100644 api/frontend/gui/requirements.txt
 delete mode 100644 api/frontend/gui/templates/index.html
 delete mode 100644 api/frontend/gui/wsgi.py
 delete mode 100644 api/frontend/nginx-dev.conf
 delete mode 100644 api/frontend/nginx-prod.conf
 delete mode 100644 api/frontend/output_handler/Dockerfile
 delete mode 100755 api/frontend/output_handler/entrypoint.sh
 delete mode 100644 api/frontend/output_handler/output_handler.py
 delete mode 100644 api/frontend/output_handler/requirements.txt
 delete mode 100644 api/node/.env-template
 delete mode 100644 api/node/connector/Dockerfile
 delete mode 100644 api/node/connector/connector.py
 delete mode 100755 api/node/connector/entrypoint.sh
 delete mode 100644 api/node/connector/requirements.txt
 delete mode 100644 api/node/docker-compose-dev.yml
 delete mode 100644 api/node/docker-compose-prod.yml
 delete mode 100644 api/node/nginx.conf
 delete mode 100644 api/node/updater/Dockerfile
 delete mode 100755 api/node/updater/entrypoint.sh
 delete mode 100644 api/node/updater/requirements.txt
 delete mode 100644 api/node/updater/updater.py
 delete mode 100755 data/preprocess_rasters.py
 delete mode 100755 fim_run.sh
 delete mode 100755 src/aggregate_fim_outputs.sh
 delete mode 100755 src/fr_to_ms_raster_mask.py
 delete mode 100755 src/get_all_huc_in_inputs.py
 rename src/gms/{rem.py => make_rem.py} (100%)
 delete mode 100644 src/reduce_nhd_stream_density.py
 delete mode 100755 src/rem.py
 delete mode 100755 src/run_by_unit.sh
 delete mode 100755 src/time_and_tee_run_by_unit.sh
 delete mode 100755 src/utils/archive_cleanup.py
 delete mode 100755 tools/compare_gms_srcs_to_fr.py
 delete mode 100755 tools/preprocess_fimx.py

diff --git a/api/.gitignore b/api/.gitignore
deleted file mode 100644
index 2eea525d8..000000000
--- a/api/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-.env
\ No newline at end of file
diff --git a/api/README.md b/api/README.md
deleted file mode 100644
index e69de29bb..000000000
diff --git a/api/frontend/.env-template b/api/frontend/.env-template
deleted file mode 100644
index afd955443..000000000
--- a/api/frontend/.env-template
+++ /dev/null
@@ -1,2 +0,0 @@
-DATA_PATH=
-SOCKET_URL=
\ No newline at end of file
diff --git a/api/frontend/docker-compose-dev.yml b/api/frontend/docker-compose-dev.yml
deleted file mode 100644
index 1f7b0ca2d..000000000
--- a/api/frontend/docker-compose-dev.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-version: '3.5'
-services:
-  fim_frontend_gui:
-    image: fim_frontend_gui
-    build:
-      context: ./gui
-    container_name: fim_frontend_gui
-    restart: always
-    env_file:
-      - .env
-    expose:
-      - "5000"
-    networks:
-      - fim
-    volumes:
-      - ./gui/templates/:/opt/gui/templates/
-      - ./gui/gui.py:/opt/gui/gui.py
-  fim_frontend_output_handler:
-    image: fim_frontend_output_handler
-    build:
-      context: ./output_handler
-    container_name: fim_frontend_output_handler
-    restart: always
-    external_links: 
-      - fim_node_connector
-    env_file:
-      - .env
-    networks:
-      - fim
-    volumes:
-      - ${DATA_PATH}:/data/
-      - ./output_handler/output_handler.py:/opt/output_handler/output_handler.py
-  fim_nginx:
-    image: nginx
-    container_name: fim_nginx
-    restart: always
-    depends_on:
-      - fim_frontend_gui
-    external_links: 
-      - fim_node_connector
-    ports:
-      - "80:80"
-    volumes:
-      - ./nginx-dev.conf:/etc/nginx/nginx.conf:ro
-    networks:
-      - fim
-    command: [nginx]
-
-networks:
-  fim:
-    name: fim
-          
\ No newline at end of file
diff --git a/api/frontend/docker-compose-prod.yml b/api/frontend/docker-compose-prod.yml
deleted file mode 100644
index b408fe054..000000000
--- a/api/frontend/docker-compose-prod.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-version: '3.5'
-services:
-  fim_frontend_gui:
-    image: fim_frontend_gui
-    build:
-      context: ./gui
-    container_name: fim_frontend_gui
-    restart: always
-    env_file:
-      - .env
-    expose:
-      - "5000"
-    networks:
-      - fim
-  fim_frontend_output_handler:
-    image: fim_frontend_output_handler
-    build:
-      context: ./output_handler
-    container_name: fim_frontend_output_handler
-    restart: always
-    external_links: 
-      - fim_node_connector
-    env_file:
-      - .env
-    networks:
-      - fim
-    volumes:
-      - ${DATA_PATH}:/data/
-  fim_nginx:
-    image: nginx
-    container_name: fim_nginx
-    restart: always
-    depends_on:
-      - fim_frontend_gui
-    external_links: 
-      - fim_node_connector
-    ports:
-      - "80:80"
-    volumes:
-      - ./nginx-prod.conf:/etc/nginx/nginx.conf:ro
-    networks:
-      - fim
-    command: [nginx]
-
-networks:
-  fim:
-    name: fim
\ No newline at end of file
diff --git a/api/frontend/gui/Dockerfile b/api/frontend/gui/Dockerfile
deleted file mode 100644
index fb5d0750e..000000000
--- a/api/frontend/gui/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM python:3.8.5-slim-buster
-
-ENV PYTHONUNBUFFERED 1
-RUN mkdir -p /opt/gui
-WORKDIR /opt/gui
-
-COPY requirements.txt .
-RUN pip install -r requirements.txt --no-cache-dir
-COPY . /opt/gui
-
-EXPOSE 5000
-
-RUN chmod +x /opt/gui/entrypoint.sh
-ENTRYPOINT ["/opt/gui/entrypoint.sh"]
diff --git a/api/frontend/gui/entrypoint.sh b/api/frontend/gui/entrypoint.sh
deleted file mode 100755
index fbf7d3145..000000000
--- a/api/frontend/gui/entrypoint.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-cd /opt/gui/
-echo "Starting Gunicorn"
-exec gunicorn --bind 0.0.0.0:5000 --reload wsgi:app
\ No newline at end of file
diff --git a/api/frontend/gui/gui.py b/api/frontend/gui/gui.py
deleted file mode 100644
index 456622d72..000000000
--- a/api/frontend/gui/gui.py
+++ /dev/null
@@ -1,16 +0,0 @@
-import os
-from gevent import monkey
-monkey.patch_all()
-
-from flask import Flask, render_template, request
-
-SOCKET_URL = os.environ.get('SOCKET_URL')
-
-app = Flask(__name__)
-
-@app.route('/')
-def main():
-    return render_template('index.html', socket_url=SOCKET_URL)
-
-if __name__ == '__main__':
-    app.run("0.0.0.0", port=5000)
\ No newline at end of file
diff --git a/api/frontend/gui/requirements.txt b/api/frontend/gui/requirements.txt
deleted file mode 100644
index 00b4e3b79..000000000
--- a/api/frontend/gui/requirements.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-flask==2.1.2
-flask-socketio==5.0.0
-gevent==20.9.0
-gunicorn==20.0.4
\ No newline at end of file
diff --git a/api/frontend/gui/templates/index.html b/api/frontend/gui/templates/index.html
deleted file mode 100644
index 421050dbd..000000000
--- a/api/frontend/gui/templates/index.html
+++ /dev/null
@@ -1,560 +0,0 @@
-<html>
-    <head>
-        <title>Cahaba API</title>
-
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.2/css/all.min.css" integrity="sha512-HK5fgLBL+xu6dm/Ii3z4xhlSUyZgTT9tuc/hSrtw6uzJOvgRr2a9jyxxT1ely+B+xFAmJKVSTbpM/CuL7qxO8w==" crossorigin="anonymous" />
-        <link rel="preconnect" href="https://fonts.gstatic.com">
-        <link href="https://fonts.googleapis.com/css2?family=Roboto&display=swap" rel="stylesheet">
-        <style>
-            html, body {
-                display: flex;
-                width: 100%;
-                height: 100%;
-
-                flex-direction: column;
-                align-items: center;
-                justify-content: center;
-            }
-
-            p, span, h1, h2, h3, h4, h5, h6, div, form, input, label, table {
-                font-family: 'Roboto', sans-serif;
-            }
-
-            #request-container {
-                display: flex;
-                width: 25rem;
-
-                flex-direction: column;
-
-                cursor: inherit;
-                opacity: inherit;
-            }
-
-            #request-container.loading {
-                cursor: progress;
-                opacity: 0.3;
-            }
-
-            #request-container.loading > div {
-                pointer-events: none;
-            }
-
-            #request-tabs-container {
-                display: flex;
-                width: 100%;
-                height: 2rem;
-
-                justify-content: space-evenly;
-
-                border: 2px solid black;
-            }
-
-            .request-tab {
-                display: flex;
-                flex: 1;
-                height: 100%;
-
-                align-items: center;
-                justify-content: center;
-
-                color: black;
-                background-color: white;
-                font-size: 0.9rem;
-
-                transition: all 0.1s linear;
-
-                cursor: pointer;
-            }
-
-            .request-tab:hover {
-                color: white;
-                background-color: rgb(99, 99, 99);
-            }
-
-            .request-tab.active {
-                color: white;
-                background-color: rgb(56, 56, 56);
-            }
-
-            .request-form {
-                display: flex;
-                width: 100%;
-                min-height: 20rem;
-                flex-direction: column;
-
-                justify-content: space-between;
-
-                border: 2px solid black;
-                border-top: none;
-                border-radius: 0 0 8px 8px;
-
-                margin: 0;
-                overflow: hidden;
-            }
-
-            .error-list {
-                display: flex;
-                width: 100%;
-                min-height: 4rem;
-                flex-direction: column;
-
-                align-items: center;
-                justify-content: center;
-
-                color: red;
-            }
-
-            .form-content {
-                display: flex;
-                flex-direction: column;
-                margin-bottom: 1rem;
-
-                padding: 1rem;
-            }
-
-            .start-button {
-                display: flex;
-                width: 100%;
-                height: 4rem;
-
-                align-items: center;
-                justify-content: center;
-
-                font-size: 2rem;
-                color: white;
-                background-color: #286ebe;
-                cursor: pointer;
-
-                transition: all 0.5s linear;
-                pointer-events: inherit;
-            }
-            
-            .start-button.not-connected {
-                background-color: #d11111;
-                pointer-events: none;
-            }
-
-            .start-button:hover {
-                background-color: #093568;
-            }
-
-            #release-button {
-                display: flex;
-                width: 100%;
-                height: 100%;
-
-                align-items: center;
-                justify-content: center;
-
-                font-size: 1.2rem;
-                color: white;
-                background-color: #28be99;
-                cursor: pointer;
-
-                transition: all 0.5s linear;
-                pointer-events: inherit;
-            }
-
-            #release-button:hover, #release-button.active {
-                background-color: #09685b;
-            }
-
-            .form-field {
-                display: flex;
-                width: 0;
-
-                flex: 1;
-            }
-
-            .container-horizontal {
-                display: flex;
-
-                margin-bottom: 1rem;
-            }
-
-            .input-with-label {
-                display: flex;
-
-                margin-right: 2rem;
-            }
-
-            .styled-table-wrapper {
-                display: none;
-                height: 12rem;
-
-                margin-top: 2rem;
-
-                overflow-y: auto;
-            }
-
-            .styled-table {
-                border-collapse: collapse;
-                min-width: 10rem;
-                box-shadow: 0 0 20px rgba(0, 0, 0, 0.15);
-            }
-
-            .styled-table th {
-                position: sticky;
-                top: 0;
-
-                background-color: #286ebe;
-                color: #ffffff;
-                font-weight: normal;
-                text-align: left;
-            }
-
-            .styled-table th,
-            .styled-table td {
-                padding: 1rem;
-            }
-
-            .styled-table tbody tr {
-                border-bottom: thin solid #dddddd;
-            }
-
-            .styled-table tbody tr:nth-of-type(even) {
-                background-color: #f3f3f3;
-            }
-
-            .styled-table tbody tr:last-of-type {
-                border-bottom: 2px solid #286ebe;
-            }
-
-            .styled-table tbody tr.job-completed {
-                font-weight: bold;
-                color: #2fd107;
-            }
-            .styled-table tbody tr.job-error {
-                font-weight: bold;
-                color: #d10707;
-            }
-
-            .cancel-button {
-                color: red;
-                cursor: pointer;
-                font-size: 1.5rem;
-            }
-
-            .info-button {
-                color: #286ebe;
-                cursor: pointer;
-                font-size: 1.5rem;
-            }
-        </style>
-    </head>
-    <body>
-        <!-- <h1>FIM Run API</h1> -->
-        <div id="request-container">
-            <div id="request-tabs-container">
-                <div class="request-tab" onclick="selectForm(event, 'release')">Release</div>
-                <div class="request-tab active" onclick="selectForm(event, 'fim-run')">FIM Run</div>
-                <div class="request-tab" onclick="selectForm(event, 'calibration')">Calibration</div>
-                <div class="request-tab" onclick="selectForm(event, 'preprocessing')">Pre-processing</div>
-            </div>
-            <div id="release" class="request-form" style="display: None;">
-                <div class="form-content">
-                    <h3 style="margin: 0;">FIM Version</h3>
-                    <div class="container-horizontal" style="width: 10rem;">
-                        <span>FIM_3_</span>
-                        <input type="text" id="release-job-version-major" class="form-field" placeholder="0"></input>
-                        <span>_</span>
-                        <input type="text" id="release-job-version-minor" class="form-field" placeholder="0"></input>
-                        <span>_</span>
-                        <input type="text" id="release-job-version-patch" class="form-field" placeholder="0"></input>
-                    </div>
-                    <h3 style="margin: 0;">Previous Major FIM Version</h3>
-                    <div class="container-horizontal" style="width: 10rem;">
-                        <span>FIM_3_</span>
-                        <input type="text" id="prev-release-job-version-major" class="form-field" placeholder="0"></input>
-                        <span>_</span>
-                        <input type="text" id="prev-release-job-version-minor" class="form-field" placeholder="0"></input>
-                        <span>_</span>
-                        <input type="text" id="prev-release-job-version-patch" class="form-field" placeholder="0"></input>
-                    </div>
-                    <div class="error-list" id="release-errors"></div>
-                </div>
-                <div class="start-button not-connected" onclick="submitRequest('release')">Not Connected</div>
-            </div>
-            <div id="fim-run" class="request-form">
-                <div class="form-content">
-                    <h3 style="margin: 0;">Basic</h3>
-                    <div class="container-horizontal">
-                        <input type="text" id="fim-run-job-name" class="form-field" placeholder="Job Name"></input>
-                        <select id="presets-list" class="form-field" onchange="presetListSelected(event)"></select>
-                        <input type="text" id="hucs" class="form-field" placeholder="HUC(s)" style="display: none;"></input>
-                        <input type="text" id="git-branch" class="form-field" placeholder="Git Branch"></input>
-                    </div>
-                    <h3 style="margin: 0;">Configuration</h3>
-                    <div class="container-horizontal">
-                        <div class="input-with-label">
-                            <input type="radio" name="configuration" value="default" checked>
-                            <label for="default">Default</label>
-                        </div>
-                        <div class="input-with-label">
-                            <input type="radio" name="configuration" value="calibrated">
-                            <label for="calibrated">Calibrated</label>
-                        </div>
-                    </div>
-                    <h3 style="margin: 0;">Extent</h3>
-                    <div class="container-horizontal">
-                        <div class="input-with-label">
-                            <input type="checkbox" name="extent" value="FR" checked>
-                            <label for="FR">Full Resolution</label>
-                        </div>
-                        <div class="input-with-label">
-                            <input type="checkbox" name="extent" value="MS">
-                            <label for="MS">Main Stem</label>
-                        </div>
-                    </div>
-                    <div class="input-with-label">
-                        <input type="checkbox" name="dev-run" value="dev-run">
-                        <label for="dev-run">Include all intermediate output files?</label>
-                    </div>
-                    <div class="input-with-label">
-                        <input type="checkbox" name="viz-run" value="viz-run" onclick="disableDevRun(event)">
-                        <label for="viz-run">Include Viz Post-Processing?</label>
-                    </div>
-                    <div class="error-list" id="fim-run-errors">
-    
-                    </div>
-                </div>
-                <div class="start-button not-connected" onclick="submitRequest('fim_run')">Not Connected</div>
-            </div>
-            <div id="calibration" class="request-form" style="display: None;">
-                Calibration stuff goes here
-            </div>
-            <div id="preprocessing" class="request-form" style="display: None;">
-                Pre-processing stuff goes here
-            </div>
-        </div>
-        <div id="job-list-wrapper" class="styled-table-wrapper">
-            <table class="styled-table">
-                <thead>
-                    <tr>
-                        <th>Name</th>
-                        <th>Time Elapsed</th>
-                        <th>Status</th>
-                        <th>Outputs Saved</th>
-                        <th></th>
-                        <th></th>
-                    </tr>
-                </thead>
-                <tbody id="job-list">
-                </tbody>
-            </table>
-        </div>
-
-        <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/socket.io/3.0.4/socket.io.min.js"></script>
-
-        <script type="text/javascript">
-            const socket = io.connect('{{socket_url}}')
-
-            document.addEventListener("DOMContentLoaded", () => {
-                // Socket.io related code
-                socket.on('connect', () => {
-                    const buttons = document.getElementsByClassName('start-button')
-                    for (let i = 0; i < buttons.length; i++) {
-                        buttons[i].classList.remove('not-connected')
-                        buttons[i].innerHTML = 'Start'
-                    }
-                })
-
-                socket.on('disconnect', () => {
-                    const buttons = document.getElementsByClassName('start-button')
-                    for (let i = 0; i < buttons.length; i++) {
-                        buttons[i].classList.add('not-connected')
-                        buttons[i].innerHTML = 'Not Connected'
-                    }
-                })
-
-                socket.on('client_update', ({jobUpdates, presetsList}) => {
-                    if(document.querySelector('#presets-list').innerHTML === '') {
-                        let presetsSelectBody = ''
-                        if (Object.keys(presetsList).length === 0)
-                            document.querySelector('#presets-list').style.display = 'none'
-                        else
-                            document.querySelector('#presets-list').style.display = 'block'
-                        Object.values(presetsList).forEach(p => {
-                            presetsSelectBody += `<option value="${p}">${p}</option>`
-                        })
-                        presetsSelectBody += `<option value="custom">Custom</option>`
-                        document.querySelector('#presets-list').innerHTML = presetsSelectBody
-                    }
-
-
-                    let jobsTableBody = ''
-                    if (jobUpdates.length === 0)
-                        document.querySelector('#job-list-wrapper').style.display = 'none'
-                    else
-                        document.querySelector('#job-list-wrapper').style.display = 'block'
-                    jobUpdates.forEach(j => {
-                        let outputs_saved = "N/A"
-                        if (j['total_output_files_length']) {
-                            outputs_saved = `${j['current_output_files_saved_length']}/${j['total_output_files_length']}`
-                        }
-
-                        statusClass = ''
-                        if (j['status'] === 'Completed') statusClass = 'class="job-completed"'
-                        if (j['status'] === 'Error' || j['exit_code'] !== 0) statusClass = 'class="job-error"'
-
-                        jobsTableBody += `<tr ${statusClass}>
-                                            <td>${j['nice_name'].replace(/_/g, ' ')}</td>
-                                            <td>${new Date(j['time_elapsed'] * 1000).toISOString().substr(11, 8)}</td>
-                                            <td>${j['status']}</td>
-                                            <td>${outputs_saved}</td>
-                                            <td><i class="fas fa-info info-button" onclick="getJobInfo('${j['job_name']}')"></i></td>
-                                            <td><i class="fas fa-times cancel-button" onclick="cancelJob('${j['job_name']}')"></i></td>
-                                        </tr>`
-                    })
-                    document.querySelector('#job-list').innerHTML = jobsTableBody
-                })
-
-                socket.on('validation_errors', validation_errors => {
-                    document.querySelector('#request-container').classList.remove('loading')
-                    document.querySelector('#fim-run-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
-                })
-
-                socket.on('job_added', job_type => {
-                    document.querySelector('#request-container').classList.remove('loading')
-                    if (job_type === 'fim_run') {
-                        document.querySelector('#fim-run-job-name').value = ''
-                        document.querySelector('#hucs').value = ''
-                        document.querySelector('#git-branch').value = ''
-                        document.querySelector("input[name='dev-run']").checked = false
-                        document.querySelector("input[name='dev-run']").disabled = false
-                        document.querySelector("input[name='viz-run']").checked = false
-                        document.querySelector("input[name='configuration'][value='default']").checked = true
-                        document.querySelector("input[name='extent'][value='FR']").checked = true
-                        document.querySelector("input[name='extent'][value='MS']").checked = false
-                        document.querySelector('#fim-run-errors').innerHTML = ''
-                    } else if (job_type === 'release') {
-                        console.log('release job has been added')
-                        document.querySelector('#release-job-version-major').value = ''
-                        document.querySelector('#release-job-version-minor').value = ''
-                        document.querySelector('#release-job-version-patch').value = ''
-                        document.querySelector('#prev-release-job-version-major').value = ''
-                        document.querySelector('#prev-release-job-version-minor').value = ''
-                        document.querySelector('#prev-release-job-version-patch').value = ''
-                        document.querySelector('#release-errors').innerHTML = ''
-                    }
-                })
-                socket.on('job_canceled', () => {
-                    console.log("Job has been cancelled")
-                })
-            })
-
-            const submitRequest = requestName => {
-                if (requestName === 'fim_run'){
-                    // Fim_run related code
-                    let validation_errors = []
-                    const job_name = document.querySelector('#fim-run-job-name').value
-                    const preset = document.querySelector('#presets-list').value
-                    const hucs = document.querySelector('#hucs').value
-                    const git_branch = document.querySelector('#git-branch').value
-                    const dev_run = document.querySelector("input[name='dev-run']").checked
-                    const viz_run = document.querySelector("input[name='viz-run']").checked
-                    const configuration =  Array.from(document.querySelectorAll("input[name='configuration']")).filter(c => c.checked)[0].value
-                    const extents =  Array.from(document.querySelectorAll("input[name='extent']")).filter(c => c.checked).map(c => c.value)
-
-                    if (job_name === '') validation_errors.push('Job Name Cannot Be Empty')
-                    if (preset === 'custom' && hucs === '') validation_errors.push('Huc(s) Cannot Be Empty')
-                    if (git_branch === '') validation_errors.push('Git Branch Cannot Be Empty')
-
-                    if (validation_errors.length > 0){
-                        document.querySelector('#fim-run-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
-                        return
-                    }
-
-                    document.querySelector('#request-container').classList.add('loading')
-                    
-                    socket.emit('new_job', {
-                        job_type: 'fim_run',
-                        job_name,
-                        preset,
-                        hucs,
-                        git_branch,
-                        dev_run,
-                        viz_run,
-                        configuration,
-                        extents
-                    })
-                } else if (requestName === 'release'){
-                    // Release related code
-                    let validation_errors = []
-                    const job_version_major = document.querySelector('#release-job-version-major').value
-                    const job_version_minor = document.querySelector('#release-job-version-minor').value
-                    const job_version_patch = document.querySelector('#release-job-version-patch').value
-                    const prev_job_version_major = document.querySelector('#prev-release-job-version-major').value
-                    const prev_job_version_minor = document.querySelector('#prev-release-job-version-minor').value
-                    const prev_job_version_patch = document.querySelector('#prev-release-job-version-patch').value
-
-                    if (job_version_major === '' || job_version_minor === '' || job_version_patch === '')
-                        validation_errors.push('Fim Version must be completely filled out')
-                    if (prev_job_version_major === '' || prev_job_version_minor === '' || prev_job_version_patch === '')
-                        validation_errors.push('Previous Major Fim Version must be completely filled out')
-
-                    if (validation_errors.length > 0){
-                        document.querySelector('#release-errors').innerHTML = validation_errors.map(e => `<span>${e}</span>`).join('')
-                        return
-                    }
-
-                    document.querySelector('#request-container').classList.add('loading')
-
-                    socket.emit('new_job', {
-                        job_type: 'release',
-                        job_version_major,
-                        job_version_minor,
-                        job_version_patch,
-                        prev_job_version_major,
-                        prev_job_version_minor,
-                        prev_job_version_patch
-                    })
-                }
-            }
-
-            const disableDevRun = e => {
-                if (e.target.checked){
-                    document.querySelector("input[name='dev-run']").checked = false
-                    document.querySelector("input[name='dev-run']").disabled = true
-                } else {
-                    document.querySelector("input[name='dev-run']").disabled = false
-                }
-            }
-
-            const presetListSelected = e => {
-                if (e.target.value === 'custom')
-                    document.querySelector('#hucs').style.display = 'flex'
-                else
-                    document.querySelector('#hucs').style.display = 'none'
-            }
-
-            const getJobInfo = jobName => {
-                console.log(jobName)
-            }
-
-            const cancelJob = jobName => {
-                console.log(jobName)
-                socket.emit('cancel_job', {job_name: jobName})
-            }
-
-            const selectForm = (e, formName) => {
-                // Close all forms and remove active from all tabs
-                const forms = document.getElementsByClassName('request-form')
-                for (let i = 0; i < forms.length; i++) {
-                    forms[i].style.display = 'none'
-                }
-
-                const tabs = document.getElementsByClassName('request-tab')
-                for (let i = 0; i < tabs.length; i++) {
-                    tabs[i].classList.remove('active')
-                }
-                
-                // Add active class to newly selected tab
-                e.target.classList.add('active')
-
-                // Make newly selected form visible
-                document.getElementById(formName).style.display = 'flex'
-            }
-        </script>
-    </body>
-</html>
\ No newline at end of file
diff --git a/api/frontend/gui/wsgi.py b/api/frontend/gui/wsgi.py
deleted file mode 100644
index b9303d13e..000000000
--- a/api/frontend/gui/wsgi.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from gui import app
-
-if __name__ == "__main__":
-    app.run()
\ No newline at end of file
diff --git a/api/frontend/nginx-dev.conf b/api/frontend/nginx-dev.conf
deleted file mode 100644
index 97e37c1f0..000000000
--- a/api/frontend/nginx-dev.conf
+++ /dev/null
@@ -1,72 +0,0 @@
-user nginx;
-worker_processes 1;
-pid /var/run/nginx.pid;
-daemon off;
-
-events {
-  worker_connections 512;
-  # multi_accept on;
-}
-
-http {
-  sendfile on;
-  tcp_nopush on;
-  tcp_nodelay on;
-  proxy_connect_timeout 300;
-  proxy_send_timeout 300;
-  proxy_read_timeout 90m;
-  send_timeout 300;
-  keepalive_timeout 65;
-  types_hash_max_size 2048;
-
-  include /etc/nginx/mime.types;
-  default_type application/octet-stream;
-
-  log_format upstream_time '$remote_addr - $remote_user [$time_local] '
-                             '"$request" $status $body_bytes_sent '
-                             '"$http_referer" "$http_user_agent"'
-                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
-
-  access_log /var/log/nginx/access.log upstream_time;
-  error_log /var/log/nginx/error.log warn;
-
-  server {
-    listen 80;
-    client_header_buffer_size 64k;
-    large_client_header_buffers 4 64k;
-    server_name _;
-    root /var/www/;
-
-    gzip on;
-    gzip_types application/json;
-    proxy_http_version 1.1;
-
-    location /stats/nginx {
-      stub_status on;
-    }
-
-    # Node side (these should only be used if the frontend is on the same machine as the connector)
-    location / {
-      proxy_pass http://fim_node_connector:6000/;
-      proxy_set_header Host            $host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-    }
-
-    location /socket.io {
-      proxy_http_version 1.1;
-      proxy_buffering off;
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection "Upgrade";
-      proxy_pass http://fim_node_connector:6000/socket.io;
-      proxy_set_header Host            $host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-    }
-
-    # Frontend Side
-    location /api {
-      proxy_pass http://fim_frontend_gui:5000/;
-      proxy_set_header Host            $host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-    }
-  }
-}
diff --git a/api/frontend/nginx-prod.conf b/api/frontend/nginx-prod.conf
deleted file mode 100644
index 878d423fc..000000000
--- a/api/frontend/nginx-prod.conf
+++ /dev/null
@@ -1,55 +0,0 @@
-user nginx;
-worker_processes 1;
-pid /var/run/nginx.pid;
-daemon off;
-
-events {
-  worker_connections 512;
-  # multi_accept on;
-}
-
-http {
-  sendfile on;
-  tcp_nopush on;
-  tcp_nodelay on;
-  proxy_connect_timeout 300;
-  proxy_send_timeout 300;
-  proxy_read_timeout 90m;
-  send_timeout 300;
-  keepalive_timeout 65;
-  types_hash_max_size 2048;
-
-  include /etc/nginx/mime.types;
-  default_type application/octet-stream;
-
-  log_format upstream_time '$remote_addr - $remote_user [$time_local] '
-                             '"$request" $status $body_bytes_sent '
-                             '"$http_referer" "$http_user_agent"'
-                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
-
-  access_log /var/log/nginx/access.log upstream_time;
-  error_log /var/log/nginx/error.log warn;
-
-  server {
-    listen 80;
-    client_header_buffer_size 64k;
-    large_client_header_buffers 4 64k;
-    server_name _;
-    root /var/www/;
-
-    gzip on;
-    gzip_types application/json;
-    proxy_http_version 1.1;
-
-    location /stats/nginx {
-      stub_status on;
-    }
-    
-    # Frontend Side
-    location /api {
-      proxy_pass http://fim_frontend_gui:5000/;
-      proxy_set_header Host            $host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-    }
-  }
-}
diff --git a/api/frontend/output_handler/Dockerfile b/api/frontend/output_handler/Dockerfile
deleted file mode 100644
index 68498a6d3..000000000
--- a/api/frontend/output_handler/Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM python:3.8.5-slim-buster
-
-ENV PYTHONUNBUFFERED 1
-RUN mkdir -p /opt/output_handler
-WORKDIR /opt/output_handler
-
-COPY requirements.txt .
-RUN pip install -r requirements.txt --no-cache-dir
-COPY . /opt/output_handler
-
-RUN chmod +x /opt/output_handler/entrypoint.sh
-ENTRYPOINT ["/opt/output_handler/entrypoint.sh"]
diff --git a/api/frontend/output_handler/entrypoint.sh b/api/frontend/output_handler/entrypoint.sh
deleted file mode 100755
index 248541b09..000000000
--- a/api/frontend/output_handler/entrypoint.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-umask 002
-cd /opt/output_handler/
-echo "Starting Output Handler"
-python ./output_handler.py
\ No newline at end of file
diff --git a/api/frontend/output_handler/output_handler.py b/api/frontend/output_handler/output_handler.py
deleted file mode 100644
index 1d803a696..000000000
--- a/api/frontend/output_handler/output_handler.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-import time
-
-import socketio
-
-SOCKET_URL = os.environ.get('SOCKET_URL')
-
-def handle_outputs(data):
-    job_name = data['job_name']
-    directory_path = data['directory_path']
-    file_name = data['file_name']
-    file_chunk = data['file_chunk']
-    chunk_index = data['chunk_index']
-
-    # Create folder if it doesn't yet exist and set writing mode
-    mode = 'ab'
-    if chunk_index == 0:
-        mode = 'wb'
-        try:
-            os.makedirs(directory_path)
-        except:
-            pass
-        
-    # Write binary data to file
-    with open(f"{directory_path}/{file_name}", mode) as binary_file:
-        print(f"Writing chunk {chunk_index} for file {directory_path}/{file_name}")
-        binary_file.write(file_chunk)
-
-    sio.emit('output_handler_finished_file_chunk', {'job_name': job_name, 'file_path': f"{directory_path}/{file_name}"})
-
-sio = socketio.Client()
-
-@sio.event
-def connect():
-    print("Output Handler Connected!")
-    sio.emit('output_handler_connected')
-
-@sio.event
-def disconnect():
-    print('disconnected from server')
-
-@sio.on('new_job_outputs')
-def ws_new_job_outputs(data):
-    handle_outputs(data)
-
-sio.connect(SOCKET_URL)
\ No newline at end of file
diff --git a/api/frontend/output_handler/requirements.txt b/api/frontend/output_handler/requirements.txt
deleted file mode 100644
index a2217b4e9..000000000
--- a/api/frontend/output_handler/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-python-engineio[client]==4.0.0
-python-socketio[client]==5.0.3
\ No newline at end of file
diff --git a/api/node/.env-template b/api/node/.env-template
deleted file mode 100644
index 352f2d2ca..000000000
--- a/api/node/.env-template
+++ /dev/null
@@ -1,6 +0,0 @@
-DATA_PATH=
-DOCKER_IMAGE_PATH=
-SOCKET_URL=
-FRONTEND_URL=
-GITHUB_REPO=https://github.com/NOAA-OWP/cahaba.git
-MAX_ALLOWED_CPU_CORES=
diff --git a/api/node/connector/Dockerfile b/api/node/connector/Dockerfile
deleted file mode 100644
index 091fdb364..000000000
--- a/api/node/connector/Dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-FROM docker:20.10.2-dind
-
-RUN apk add --no-cache python3 python3-dev py3-pip build-base openssl-dev libffi-dev git
-
-ENV PYTHONUNBUFFERED 1
-RUN mkdir -p /opt/connector
-WORKDIR /opt/connector
-
-COPY requirements.txt .
-RUN pip3 install -r requirements.txt --no-cache-dir
-COPY . /opt/connector
-
-EXPOSE 6000
-
-RUN chmod +x /opt/connector/entrypoint.sh
-ENTRYPOINT ["/opt/connector/entrypoint.sh"]
diff --git a/api/node/connector/connector.py b/api/node/connector/connector.py
deleted file mode 100644
index 677518676..000000000
--- a/api/node/connector/connector.py
+++ /dev/null
@@ -1,228 +0,0 @@
-import eventlet
-eventlet.monkey_patch()
-
-import os
-import re
-import time
-import random
-import logging
-import subprocess
-from datetime import date
-
-from flask import Flask, request
-from flask_socketio import SocketIO, emit
-
-DATA_PATH = os.environ.get('DATA_PATH')
-DOCKER_IMAGE_PATH = os.environ.get('DOCKER_IMAGE_PATH')
-SOCKET_URL = os.environ.get('SOCKET_URL')
-FRONTEND_URL = os.environ.get('FRONTEND_URL')
-GITHUB_REPO = os.environ.get('GITHUB_REPO')
-
-app = Flask(__name__)
-socketio = SocketIO(app, cors_allowed_origins=[SOCKET_URL, FRONTEND_URL, "http://fim_node_connector:6000"])
-
-shared_data = {
-    'handler_sid': None,
-    'updater_sid': None
-}
-
-@app.route('/')
-def main():
-    return '<h1>Nothing to see here....</h1>'
-
-@socketio.on('connect')
-def ws_conn():
-    print('user connected!')
-    emit('is_connected', True)
-
-@socketio.on('disconnect')
-def ws_disconn():
-    print('user disconnected!')
-    emit('is_connected', False)
-
-@socketio.on('update')
-def ws_update(data):
-    emit('client_update', data, broadcast=True)
-
-@socketio.on('output_handler_connected')
-def ws_output_handler_connected():
-    print('handler_sid: ', request.sid)
-    shared_data['handler_sid'] = request.sid
-    emit('retry_saving_files', room=shared_data['updater_sid'])
-
-@socketio.on('updater_connected')
-def ws_updater_connected():
-    print('updater_sid: ', request.sid)
-    shared_data['updater_sid'] = request.sid
-    emit('retry_saving_files', room=shared_data['updater_sid'])
-
-@socketio.on('ready_for_output_handler')
-def ws_ready_for_output_handler(data):
-    job_name = data['job_name']
-    path = data['path']
-    chunk_index = data['chunk_index']
-
-    if shared_data['handler_sid'] == None:
-        print("output handler not connected!")
-        emit('retry_saving_files')
-        return
-
-    # Split up path into parts for the output handler
-    path_parts = re.search(rf"(.+)/(.+)", path)
-    directory_path = path_parts.group(1)
-    file_name = path_parts.group(2)
-
-    file_read_start = time.time()
-    with open(path, "rb") as binary_file:
-        # Read and emit file chunk by chunk (50MB at a time)
-        binary_file.seek(chunk_index * 52428800)
-        file_chunk = binary_file.read(52428800)
-
-        if len(file_chunk) == 0:
-            print('End of File')
-            emit('file_saved', {
-                'job_name': job_name,
-                'file_path': path
-            }, room=shared_data['updater_sid'])
-            return
-
-        print("Sending to output handler", path, "Chunk:", chunk_index)
-        emit('new_job_outputs', {
-            'job_name': job_name,
-            'directory_path': directory_path,
-            'file_name': file_name,
-            'file_chunk': file_chunk,
-            'chunk_index': chunk_index
-        }, room=shared_data['handler_sid'])
-
-@socketio.on('output_handler_finished_file_chunk')
-def output_handler_finished_file_chunk(data):
-    job_name = data['job_name']
-    file_path = data['file_path']
-
-    print('done saving chunk', job_name, file_path)
-    emit('file_chunk_saved', {
-        'job_name': job_name,
-        'file_path': file_path,
-    }, room=shared_data['updater_sid'])
-
-@socketio.on('new_job')
-def ws_new_job(job_params):
-    job_type = job_params['job_type']
-
-    if job_type == 'fim_run':
-        validation_errors = []
-
-        # Get Preset Option
-        preset = job_params['preset']
-
-        # Validate Hucs Name Option
-        if preset == 'custom':
-            hucs_raw = job_params['hucs'].replace(',', ' ').split()
-            parallel_jobs = len(hucs_raw)
-            hucs_type = len(hucs_raw[0])
-            hucs = ' '.join(hucs_raw)
-            invalid_hucs = re.search('[^0-9 ]', hucs)
-            if invalid_hucs: validation_errors.append('Invalid Huc(s)')
-        else:
-            hucs = f"/data/inputs/huc_lists/{preset}"
-            parallel_jobs = 0
-            hucs_type = 0
-
-        # Validate Git Branch Option
-        branch = ''
-        branch_exists = subprocess.run(['git', 'ls-remote', '--heads', GITHUB_REPO, job_params['git_branch'].replace(' ', '_')], stdout=subprocess.PIPE).stdout.decode('utf-8')
-        if branch_exists: branch = job_params['git_branch'].replace(' ', '_')
-        else: validation_errors.append('Git Branch Does Not Exist')
-
-        # Validate Extent Option
-        valid_extents = ['FR', 'MS']
-        extents = []
-        for extent in job_params['extents']:
-            if extent in valid_extents:
-                extents.append(extent)
-            else:
-                validation_errors.append('Invalid Extent Option')
-
-        # Validate Configuration Option
-        config_path = ''
-        if job_params['configuration'] == 'default': config_path = './foss_fim/config/params_template.env'
-        elif job_params['configuration'] == 'calibrated': config_path = './foss_fim/config/params_calibrated.env'
-        else: validation_errors.append('Invalid Configuration Option')
-        
-        # Validate Dev Run Option
-        if job_params['dev_run'] : dev_run = True
-        else: dev_run = False
-
-        # Validate Viz Run Option
-        if job_params['viz_run'] : viz_run = True
-        else: viz_run = False
-
-        if len(validation_errors) == 0:
-            for extent in extents:
-                # Validate Job Name Option
-                job_name = f"apijob_{job_params['job_name'].replace(' ', '_')[0:50]}_fim_run_{extent.lower()}{'_c' if job_params['configuration'] == 'calibrated' else ''}{'_v' if viz_run == True else ''}_apijob_{branch}_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
-                print(f"adding job {job_name} {branch} {preset} {hucs} {parallel_jobs} {hucs_type} {extent.lower()} {config_path} {dev_run} {viz_run}")
-                emit('add_job_to_queue', {
-                    'job_type': 'fim_run',
-                    'job_name': job_name,
-                    'branch': branch,
-                    'hucs': hucs,
-                    'parallel_jobs': parallel_jobs,
-                    'hucs_type': hucs_type,
-                    'extent': extent,
-                    'config_path': config_path,
-                    'dev_run': dev_run,
-                    'viz_run': viz_run,
-                }, room=shared_data['updater_sid'])
-                print('fim_run job added')
-                emit('job_added', 'fim_run')
-        else:
-            emit('validation_errors', validation_errors)
-
-    elif job_type == 'release':
-        job_version_major = job_params['job_version_major']
-        job_version_minor = job_params['job_version_minor']
-        job_version_patch = job_params['job_version_patch']
-
-        # TODO: validate version number
-
-        job_name_base = f"fim_3_{job_version_major}_{job_version_minor}_{job_version_patch}"
-
-        prev_job_version_major = job_params['prev_job_version_major']
-        prev_job_version_minor = job_params['prev_job_version_minor']
-        prev_job_version_patch = job_params['prev_job_version_patch']
-
-        prev_version_base = f"fim_3_{prev_job_version_major}_{prev_job_version_minor}_{prev_job_version_patch}"
-
-        huc_lists = ['/data/inputs/huc_lists/included_huc8.lst', '/data/inputs/huc_lists/included_huc8_ms.lst']
-        extents = ['FR', 'MS']
-
-        for hucs, extent in zip(huc_lists, extents):
-            # Validate Job Name Option
-            prev_version = f"{prev_version_base}_{extent.lower()}_c"
-            job_name = f"apijob_{job_name_base}_{extent.lower()}_c_apijob_dev_{date.today().strftime('%d%m%Y')}_{random.randint(0, 99999)}"
-            print(f"adding job {job_name} {hucs} {extent.lower()}")
-            emit('add_job_to_queue', {
-                'job_type': 'release',
-                'job_name': job_name,
-                'hucs': hucs,
-                'extent': extent,
-                'previous_major_fim_version': prev_version
-            }, room=shared_data['updater_sid'])
-            print('release job added')
-            emit('job_added', 'release')
-    
-    @socketio.on('cancel_job')
-    def ws_cancel_job(job_params):
-        # Validate Job Name Option
-        job_name = job_params['job_name']
-
-        emit('remove_job_from_queue', {'job_name': job_name}, room=shared_data['updater_sid'])
-        print('job canceled')
-        emit('job_canceled', 'fim_run')
-   
-
-if __name__ == '__main__':
-
-    socketio.run(app, host="0.0.0.0", port="6000")
diff --git a/api/node/connector/entrypoint.sh b/api/node/connector/entrypoint.sh
deleted file mode 100755
index d6d853d6a..000000000
--- a/api/node/connector/entrypoint.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-cd /opt/connector/
-echo "Starting Connector"
-python3 ./connector.py
diff --git a/api/node/connector/requirements.txt b/api/node/connector/requirements.txt
deleted file mode 100644
index 6eb15401b..000000000
--- a/api/node/connector/requirements.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-flask==2.1.2
-werkzeug==2.0.0
-flask-socketio==5.0.0
-eventlet==0.31.0
diff --git a/api/node/docker-compose-dev.yml b/api/node/docker-compose-dev.yml
deleted file mode 100644
index 9823afad9..000000000
--- a/api/node/docker-compose-dev.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-version: '3.5'
-services:
-  fim_node_connector:
-    image: fim_node_connector
-    build:
-      context: ./connector
-    container_name: fim_node_connector
-    env_file:
-      - .env
-    restart: always
-    expose:
-      - "6000"
-    networks:
-      fim:
-        aliases: 
-          - fimnodeconnector
-    volumes:
-      - ${DATA_PATH}:/data/
-      - /var/run/docker.sock:/var/run/docker.sock
-      - ./connector/connector.py:/opt/connector/connector.py
-  fim_node_updater:
-    image: fim_node_updater
-    build:
-      context: ./updater
-    container_name: fim_node_updater
-    restart: always
-    depends_on: 
-      - fim_node_connector
-    networks:
-      - fim
-    volumes:
-      - ${DATA_PATH}:/data/
-      - /var/run/docker.sock:/var/run/docker.sock
-      - ./updater/updater.py:/opt/updater/updater.py
-networks:
-  fim:
-    name: fim
-          
\ No newline at end of file
diff --git a/api/node/docker-compose-prod.yml b/api/node/docker-compose-prod.yml
deleted file mode 100644
index f9787ab59..000000000
--- a/api/node/docker-compose-prod.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-version: '3.5'
-services:
-  fim_node_connector:
-    image: fim_node_connector
-    build:
-      context: ./connector
-    container_name: fim_node_connector
-    env_file:
-      - .env
-    restart: always
-    expose:
-      - "6000"
-    networks:
-      fim:
-        aliases: 
-          - fimnodeconnector
-    volumes:
-      - ${DATA_PATH}:/data/
-      - /var/run/docker.sock:/var/run/docker.sock
-  fim_node_updater:
-    image: fim_node_updater
-    build:
-      context: ./updater
-    container_name: fim_node_updater
-    env_file:
-      - .env
-    restart: always
-    depends_on: 
-      - fim_node_connector
-    networks:
-      - fim
-    volumes:
-      - ${DATA_PATH}:/data/
-      - /var/run/docker.sock:/var/run/docker.sock
-  fim_nginx:
-    image: nginx
-    container_name: fim_nginx
-    restart: always
-    depends_on:
-      - fim_node_connector
-    ports:
-      - "80:80"
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf:ro
-    networks:
-      - fim
-    command: [nginx]
-
-networks:
-  fim:
-    name: fim
-          
-      
\ No newline at end of file
diff --git a/api/node/nginx.conf b/api/node/nginx.conf
deleted file mode 100644
index c2fc935bd..000000000
--- a/api/node/nginx.conf
+++ /dev/null
@@ -1,64 +0,0 @@
-user nginx;
-worker_processes 1;
-pid /var/run/nginx.pid;
-daemon off;
-
-events {
-  worker_connections 512;
-  # multi_accept on;
-}
-
-http {
-  sendfile on;
-  tcp_nopush on;
-  tcp_nodelay on;
-  proxy_connect_timeout 300;
-  proxy_send_timeout 300;
-  proxy_read_timeout 90m;
-  send_timeout 300;
-  keepalive_timeout 65;
-  types_hash_max_size 2048;
-
-  include /etc/nginx/mime.types;
-  default_type application/octet-stream;
-
-  log_format upstream_time '$remote_addr - $remote_user [$time_local] '
-                             '"$request" $status $body_bytes_sent '
-                             '"$http_referer" "$http_user_agent"'
-                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
-
-  access_log /var/log/nginx/access.log upstream_time;
-  error_log /var/log/nginx/error.log warn;
-
-  server {
-    listen 80;
-    client_header_buffer_size 64k;
-    large_client_header_buffers 4 64k;
-    server_name _;
-    root /var/www/;
-
-    gzip on;
-    gzip_types application/json;
-    proxy_http_version 1.1;
-
-    location /stats/nginx {
-      stub_status on;
-    }
-
-    location / {
-      proxy_pass http://fim_node_connector:6000/;
-      proxy_set_header Host            $host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-    }
-
-    location /socket.io {
-      proxy_http_version 1.1;
-      proxy_buffering off;
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection "Upgrade";
-      proxy_pass http://fim_node_connector:6000/socket.io;
-      proxy_set_header Host            $host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-    }
-  }
-}
\ No newline at end of file
diff --git a/api/node/updater/Dockerfile b/api/node/updater/Dockerfile
deleted file mode 100644
index d62a77652..000000000
--- a/api/node/updater/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM docker:20.10.2-dind
-
-RUN apk add --no-cache python3 python3-dev py3-pip build-base openssl-dev libffi-dev git
-
-ENV PYTHONUNBUFFERED 1
-RUN mkdir -p /opt/updater
-WORKDIR /opt/updater
-
-COPY requirements.txt .
-RUN pip3 install -r requirements.txt --no-cache-dir
-COPY . /opt/updater
-
-RUN chmod +x /opt/updater/entrypoint.sh
-ENTRYPOINT ["/opt/updater/entrypoint.sh"]
diff --git a/api/node/updater/entrypoint.sh b/api/node/updater/entrypoint.sh
deleted file mode 100755
index 0f74f2541..000000000
--- a/api/node/updater/entrypoint.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-cd /opt/updater/
-echo "Starting Update Loop"
-python3 ./updater.py
diff --git a/api/node/updater/requirements.txt b/api/node/updater/requirements.txt
deleted file mode 100644
index a2217b4e9..000000000
--- a/api/node/updater/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-python-engineio[client]==4.0.0
-python-socketio[client]==5.0.3
\ No newline at end of file
diff --git a/api/node/updater/updater.py b/api/node/updater/updater.py
deleted file mode 100644
index 90fc0da71..000000000
--- a/api/node/updater/updater.py
+++ /dev/null
@@ -1,530 +0,0 @@
-import os
-import re
-import glob
-import math
-import time
-import json
-import shutil
-import logging
-import subprocess
-
-import socketio
-
-DATA_PATH = os.environ.get('DATA_PATH')
-DOCKER_IMAGE_PATH = os.environ.get('DOCKER_IMAGE_PATH')
-GITHUB_REPO = os.environ.get('GITHUB_REPO')
-MAX_ALLOWED_CPU_CORES = int(os.environ.get('MAX_ALLOWED_CPU_CORES'))
-
-shared_data = {
-    'connected': False,
-    'current_saving_job': ''
-}
-
-buffer_jobs = []
-buffer_remove_jobs = []
-current_jobs = {}
-if os.path.exists('/data/outputs/current_jobs.json'):
-    with open('/data/outputs/current_jobs.json') as f:
-        current_jobs = json.load(f)
-        for job_name in current_jobs.keys():
-            if 'is_actively_saving' in current_jobs[job_name] and current_jobs[job_name]['is_actively_saving'] == True:
-                shared_data['current_saving_job'] = current_jobs[job_name]
-
-
-# Get all the current running jobs from the list of docker containers, store that data in a dictionary
-# along with any other needed metadata (like if it's still running, doing post processing, copying outputs
-# to its destination, etc), and then update the websocket server of the status of the jobs.
-def update_loop():
-    while True:
-        # If there are no current jobs, just check every 10 seconds till there is
-        if len(current_jobs.keys()) == 0: sio.sleep(10)
-
-        while len(buffer_jobs) > 0:
-            new_job = buffer_jobs.pop()
-            current_jobs[new_job['job_name']] = new_job
-
-        while len(buffer_remove_jobs) > 0:
-            job_to_remove = buffer_remove_jobs.pop()
-            if job_to_remove['job_name'] in current_jobs:
-                current_jobs[job_to_remove['job_name']]['status'] = 'Cancelled'
-
-        # Get list of current docker containers that are fim run jobs
-                                #    docker ps --all --filter=name=apijob --format='{{.Names}} {{.State}}'
-        containers_raw = os.popen("docker ps --all --filter=name=apijob --format='{{.Names}} {{.State}}'").read().splitlines()
-        containers_split = [ line.split() for line in containers_raw ]
-        container_states = { name: state for (name, state) in containers_split }
-
-        jobs_to_delete = []
-        for job_name in current_jobs.keys():
-            sio.sleep(0)
-            if job_name in container_states:
-                current_jobs[job_name]['container_state'] = container_states[job_name]
-
-            # If the user chooses to cancel the job early
-            if current_jobs[job_name]['status'] == 'Cancelled':
-                # If the docker container is running, stop and remove it
-                if current_jobs[job_name]['time_elapsed'] > 0 and current_jobs[job_name]['container_state'] != 'exited':
-                    subprocess.call(f"docker container stop {job_name}", shell=True)
-                    subprocess.call(f"docker container rm {job_name}", shell=True)
-
-                print("output_handler finished, deleted temp source files and output files")
-                temp_path = f"/data/temp/{job_name}"
-                if os.path.isdir(temp_path):
-                    shutil.rmtree(temp_path)
-
-                outputs_path = f"/data/outputs/{current_jobs[job_name]['nice_name']}"
-                if os.path.isdir(outputs_path):
-                    shutil.rmtree(outputs_path)
-
-                jobs_to_delete.append(job_name)
-
-            active_statuses = [
-                'In Progress',
-                'Ready for Synthesize Test Cases',
-                'Running Synthesize Test Cases',
-                'Ready for Eval Plots',
-                'Running Eval Plots',
-                'Ready for Generate Categorical FIM',
-                'Running Generate Categorical FIM',
-                'Ready to Save File',
-                'Saving File'
-            ]
-            # TODO: separate list for queuing so that one job can save and another run
-
-            # Update the time elapsed for all jobs that are currently in progress or saving outputs
-            if current_jobs[job_name]['status'] in active_statuses:
-                current_jobs[job_name]['time_elapsed'] = math.ceil(time.time() - current_jobs[job_name]['time_started'])
-
-            # TODO: While job is in progress, keep track of how many hucs are done and overall progress %
-
-            # Once resources recome available, start a new job that is in queue
-
-            if current_jobs[job_name]['status'] == 'In Queue':
-                current_jobs[job_name]['time_started'] = time.time()
-
-                total_active_cores = 0
-                for j in current_jobs.keys():
-                    if current_jobs[j]['status'] in active_statuses:
-                        # This is to account for the fact that HUC6's take a lot more resources to run.
-                        # (not necessarily cpu cores but rather RAM, so this artificially reduces how many jobs can run when HUC6's
-                        #  are running)
-                        # HACK: this is more of a temporary solution until we no longer need to run HUC6's
-                        if current_jobs[j]['hucs_type'] == '6':
-                            total_active_cores += current_jobs[j]['parallel_jobs'] * 5
-                        else:
-                            total_active_cores += current_jobs[j]['parallel_jobs']
-
-                # Machine has enough resources to run a new job
-                potential_active_cores = 0
-                if current_jobs[job_name]['hucs_type'] == '6':
-                    potential_active_cores = current_jobs[job_name]['parallel_jobs'] * 5 + total_active_cores
-                else:
-                    potential_active_cores = current_jobs[job_name]['parallel_jobs'] + total_active_cores
-
-                # print(f"Checking whether a new job can start {potential_active_cores} <= {MAX_ALLOWED_CPU_CORES}")
-                # print(potential_active_cores <= MAX_ALLOWED_CPU_CORES)
-                if potential_active_cores <= MAX_ALLOWED_CPU_CORES:
-                    job_name = current_jobs[job_name]['job_name']
-                    nice_name = current_jobs[job_name]['nice_name']
-                    branch = current_jobs[job_name]['branch']
-                    hucs = current_jobs[job_name]['hucs']
-                    parallel_jobs = current_jobs[job_name]['parallel_jobs']
-                    extent = current_jobs[job_name]['extent']
-                    config_path = current_jobs[job_name]['config_path']
-                    dev_run = current_jobs[job_name]['dev_run']
-                    viz_run = current_jobs[job_name]['viz_run']
-
-                    # Clone github repo, with specific branch, to a temp folder
-                    print(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name} && chmod -R 777 {job_name} && cp .env {job_name}/tools/.env')
-                    subprocess.call(f'cd /data/temp && git clone -b {branch} {GITHUB_REPO} {job_name} && chmod -R 777 {job_name} && cp .env {job_name}/tools/.env', shell=True)
-
-                    # Kick off the new job as a docker container with the new cloned repo as the volume
-                    print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {nice_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}")
-                    subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim {DOCKER_IMAGE_PATH} fim_run.sh -u \"{hucs}\" -e {extent} -c {config_path} -n {nice_name} -o {'' if dev_run else '-p'} {'-v' if viz_run else ''} -j {parallel_jobs}", shell=True)
-                    current_jobs[job_name]['status'] = 'In Progress'
-
-            # Once the Docker container is done, either save outputs or run release
-            if current_jobs[job_name]['status'] == 'In Progress' and current_jobs[job_name]['container_state'] == 'exited':
-
-                # Get container exit code, get the docker log, and then remove container
-                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
-
-                print("Exit code")
-                print(exit_code_raw)
-                print(exit_code_raw[0])
-                try:
-                    print(int(exit_code_raw[0]))
-                except:
-                    pass
-
-                exit_code = int(exit_code_raw[0])
-                current_jobs[job_name]['exit_code'] = exit_code
-                subprocess.call(f"docker logs {job_name} >& /data/outputs/{current_jobs[job_name]['nice_name']}/logs/docker.log", shell=True)
-                subprocess.call(f"docker container rm {job_name}", shell=True)
-
-                if current_jobs[job_name]['job_type'] == 'fim_run':
-                    for path, folders, files in os.walk(f"/data/outputs/{current_jobs[job_name]['nice_name']}"):
-                        for file in files:
-                            current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = 0
-
-                    current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
-                    current_jobs[job_name]['status'] = 'Ready to Save File'
-                elif current_jobs[job_name]['job_type'] == 'release':
-                    # Move outputs to previous_fim and set them to be copied to the dev machine
-                    if os.path.isdir(f"/data/previous_fim/{current_jobs[job_name]['nice_name']}"):
-                        shutil.rmtree(f"/data/previous_fim/{current_jobs[job_name]['nice_name']}")
-                    if os.path.isdir(f"/data/outputs/{current_jobs[job_name]['nice_name']}"): shutil.move(f"/data/outputs/{current_jobs[job_name]['nice_name']}", '/data/previous_fim')
-                    for path, folders, files in os.walk(f"/data/previous_fim/{current_jobs[job_name]['nice_name']}"):
-                        for file in files:
-                            current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = 0
-                    current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
-                    current_jobs[job_name]['status'] = 'Ready for Synthesize Test Cases'
-
-            if current_jobs[job_name]['status'] == 'Ready for Synthesize Test Cases':
-                job_name = current_jobs[job_name]['job_name']
-                nice_name = current_jobs[job_name]['nice_name']
-                parallel_jobs = current_jobs[job_name]['parallel_jobs']
-
-                # Kick off the new job as a docker container to run eval metrics
-                print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/synthesize_test_cases.py -c PREV --fim-version {nice_name} --job-number {parallel_jobs} -m /data/test_cases/metrics_library/all_official_versions.csv")
-                subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/synthesize_test_cases.py -c PREV --fim-version {nice_name} --job-number {parallel_jobs} -m /data/test_cases/metrics_library/all_official_versions.csv", shell=True)
-                current_jobs[job_name]['container_state'] = 'running'
-                current_jobs[job_name]['status'] = 'Running Synthesize Test Cases'
-
-            # Once the Docker container is done, save outputs
-            if current_jobs[job_name]['status'] == 'Running Synthesize Test Cases' and current_jobs[job_name]['container_state'] == 'exited':
-                # Get container exit code, get the docker log, and then remove container
-                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
-
-                print("Exit code")
-                print(exit_code_raw)
-                print(exit_code_raw[0])
-                try:
-                    print(int(exit_code_raw[0]))
-                except:
-                    pass
-
-                exit_code = int(exit_code_raw[0])
-                current_jobs[job_name]['exit_code'] = exit_code
-                subprocess.call(f"docker logs {job_name} >& /data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/synthesize_test_cases_docker.log", shell=True)
-                subprocess.call(f"docker container rm {job_name}", shell=True)
-
-                current_jobs[job_name]['output_files_saved']['/data/test_cases/metrics_library/all_official_versions.csv'] = 0
-                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/synthesize_test_cases_docker.log"] = 0
-                current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
-                current_jobs[job_name]['status'] = 'Ready for Eval Plots'
-
-            if current_jobs[job_name]['status'] == 'Ready for Eval Plots':
-                job_name = current_jobs[job_name]['job_name']
-                nice_name = current_jobs[job_name]['nice_name']
-                previous_major_fim_version = current_jobs[job_name]['previous_major_fim_version']
-
-                # Kick off the new job as a docker container to run eval plots
-                print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/eval_plots.py -m /data/test_cases/metrics_library/all_official_versions.csv -w /data/test_cases/metrics_library/all_official_versions_viz -v {previous_major_fim_version} {nice_name} -sp")
-                subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/eval_plots.py -m /data/test_cases/metrics_library/all_official_versions.csv -w /data/test_cases/metrics_library/all_official_versions_viz -v {previous_major_fim_version} {nice_name} -sp", shell=True)
-                current_jobs[job_name]['container_state'] = 'running'
-                current_jobs[job_name]['status'] = 'Running Eval Plots'
-
-            # Once the Docker container is done, save outputs
-            if current_jobs[job_name]['status'] == 'Running Eval Plots' and current_jobs[job_name]['container_state'] == 'exited':
-                # Get container exit code, get the docker log, and then remove container
-                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
-
-                print("Exit code")
-                print(exit_code_raw)
-                print(exit_code_raw[0])
-                try:
-                    print(int(exit_code_raw[0]))
-                except:
-                    pass
-
-                exit_code = int(exit_code_raw[0])
-                current_jobs[job_name]['exit_code'] = exit_code
-                subprocess.call(f"docker logs {job_name} >& /data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/eval_plots_docker.log", shell=True)
-                subprocess.call(f"docker container rm {job_name}", shell=True)
-
-                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/eval_plots_docker.log"] = 0
-                for path, folders, files in os.walk('/data/test_cases/metrics_library/all_official_versions_viz'):
-                    for file in files:
-                        current_jobs[job_name]['output_files_saved'][os.path.join(path, file)] = 0
-                current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
-
-                current_jobs[job_name]['status'] = 'Ready for Generate Categorical FIM'
-
-            if current_jobs[job_name]['status'] == 'Ready for Generate Categorical FIM':
-                job_name = current_jobs[job_name]['job_name']
-                nice_name = current_jobs[job_name]['nice_name']
-                parallel_jobs = current_jobs[job_name]['parallel_jobs']
-
-                # Kick off the new job as a docker container to run CatFIM
-                print(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/generate_categorical_fim.py -f {nice_name} -j {parallel_jobs}")
-                subprocess.call(f"docker run -d --name {job_name} -v {DATA_PATH}:/data/ -v {DATA_PATH}temp/{job_name}/:/foss_fim -w /foss_fim/tools {DOCKER_IMAGE_PATH} /foss_fim/tools/generate_categorical_fim.py -f /data/previous_fim/{nice_name} -j {parallel_jobs}", shell=True)
-                current_jobs[job_name]['container_state'] = 'running'
-                current_jobs[job_name]['status'] = 'Running Generate Categorical FIM'
-
-            # Once the Docker container is done, save outputs
-            if current_jobs[job_name]['status'] == 'Running Generate Categorical FIM' and current_jobs[job_name]['container_state'] == 'exited':
-                # Get container exit code, get the docker log, and then remove container
-                exit_code_raw = os.popen(f"docker inspect {job_name}" + " --format='{{.State.ExitCode}}'").read().splitlines()
-
-                print("Exit code")
-                print(exit_code_raw)
-                print(exit_code_raw[0])
-                try:
-                    print(int(exit_code_raw[0]))
-                except:
-                    pass
-
-                exit_code = int(exit_code_raw[0])
-                current_jobs[job_name]['exit_code'] = exit_code
-                subprocess.call(f"docker logs {job_name} >& /data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/generate_categorical_fim.log", shell=True)
-                subprocess.call(f"docker container rm {job_name}", shell=True)
-
-                os.makedirs(f"/data/catfim/{current_jobs[job_name]['nice_name']}_temp")
-                if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"):
-                    for path in glob.glob(f"/data/catfim/{current_jobs[job_name]['nice_name']}/**/mapping/*", recursive=True):
-                        if not os.path.isdir(path):
-                            shutil.move(path, f"/data/catfim/{current_jobs[job_name]['nice_name']}_temp")
-                            filename = os.path.basename(path)
-                            current_jobs[job_name]['output_files_saved'][os.path.join(f"/data/catfim/{current_jobs[job_name]['nice_name']}", filename)] = 0
-                    shutil.rmtree(f"/data/catfim/{current_jobs[job_name]['nice_name']}")
-                shutil.move(f"/data/catfim/{current_jobs[job_name]['nice_name']}_temp", f"/data/catfim/{current_jobs[job_name]['nice_name']}")
-
-                current_jobs[job_name]['output_files_saved'][f"/data/previous_fim/{current_jobs[job_name]['nice_name']}/logs/generate_categorical_fim.log"] = 0
-                current_jobs[job_name]['total_output_files_length'] = len(current_jobs[job_name]['output_files_saved'].keys())
-                current_jobs[job_name]['status'] = 'Ready to Save File'
-
-            # Trigger connector to transmit the outputs to the output_handler
-            # If the output_handler is offline, it will keep retrying until the output_handler is online
-            if current_jobs[job_name]['status'] == 'Ready to Save File' and (shared_data['current_saving_job'] == '' or shared_data['current_saving_job'] == current_jobs[job_name]):
-                print(f"{job_name} ready for output handler")
-
-                shared_data['current_saving_job'] = current_jobs[job_name]
-                current_jobs[job_name]['is_actively_saving'] = True
-                output_to_save = {}
-                for path in current_jobs[job_name]['output_files_saved']:
-                    if current_jobs[job_name]['output_files_saved'][path] != -1:
-                        output_to_save = {'path': path, 'chunk_index': current_jobs[job_name]['output_files_saved'][path]}
-
-                if output_to_save != {}:
-                    if shared_data['connected']:
-                        sio.emit('ready_for_output_handler', {
-                            'nice_name': current_jobs[job_name]['nice_name'],
-                            'job_name': job_name,
-                            'path': output_to_save['path'],
-                            'chunk_index': output_to_save['chunk_index']
-                        })
-                current_jobs[job_name]['status'] = 'Saving File'
-
-            # Once the output_handler is done getting the outputs and the connector deletes the temp repo source,
-            # mark as completed
-            if current_jobs[job_name]['status'] == 'Saving File':
-                is_done = True
-                for path in current_jobs[job_name]['output_files_saved']:
-                    if current_jobs[job_name]['output_files_saved'][path] != -1:
-                        is_done = False
-                        break
-
-                if is_done:
-                    print("output_handler finished, deleted temp source files and output files")
-                    temp_path = f"/data/temp/{job_name}"
-                    if os.path.isdir(temp_path):
-                        shutil.rmtree(temp_path)
-
-                    outputs_path = f"/data/outputs/{current_jobs[job_name]['nice_name']}"
-                    if current_jobs[job_name]['job_type'] == 'release':
-                        outputs_path = f"/data/previous_fim/{current_jobs[job_name]['nice_name']}"
-                        destination = f"/data/viz/{current_jobs[job_name]['nice_name']}"
-
-                        if os.path.isdir(destination):
-                            shutil.rmtree(destination)
-                        if os.path.isdir(f"{outputs_path}/aggregate_fim_outputs"): shutil.move(f"{outputs_path}/aggregate_fim_outputs", destination)
-                        if os.path.isdir(f"{outputs_path}/logs"): shutil.move(f"{outputs_path}/logs", f"{destination}/logs")
-                        if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"): shutil.move(f"/data/catfim/{current_jobs[job_name]['nice_name']}", f"{destination}/catfim")
-
-                    if os.path.isdir(outputs_path):
-                        shutil.rmtree(outputs_path)
-                    if current_jobs[job_name]['job_type'] == 'release':
-                        if os.path.isdir(f"/data/catfim/{current_jobs[job_name]['nice_name']}"):
-                            shutil.rmtree(f"/data/catfim/{current_jobs[job_name]['nice_name']}")
-                        try:
-                            os.makedirs(outputs_path)
-                        except:
-                            pass
-
-                    current_jobs[job_name]['status'] = 'Completed' if current_jobs[job_name]['exit_code'] == 0 else 'Error'
-
-                    shared_data['current_saving_job'] = ''
-                    current_jobs[job_name]['is_actively_saving'] = False
-                    print(f"{job_name} completed")
-                    # TODO: Insert Slack notification here for finished job
-
-            # Remove job from list after it's been completed for more than 15 minutes
-            if (current_jobs[job_name]['status'] == 'Completed' or current_jobs[job_name]['status'] == 'Error') and \
-                time.time() >= current_jobs[job_name]['time_started'] + current_jobs[job_name]['time_elapsed'] + 900:
-                print(f"{job_name} removed from job list")
-                jobs_to_delete.append(job_name)
-
-        for job in jobs_to_delete:
-            del current_jobs[job]
-
-        presets_list = []
-        for path, folders, files in os.walk(f"/data/inputs/huc_lists"):
-            for file in files:
-                presets_list.append(file)
-
-        # Send updates to the connector and write job progress to file
-        job_updates = [ {
-            'job_name': job['job_name'],
-            'nice_name': job['nice_name'],
-            'status': job['status'],
-            'exit_code': job['exit_code'],
-            'time_elapsed': job['time_elapsed'],
-            'total_output_files_length': job['total_output_files_length'],
-            'current_output_files_saved_length': job['current_output_files_saved_length'],
-        } for job in current_jobs.values()]
-
-        if shared_data['connected']: sio.emit('update', {'jobUpdates': job_updates, 'presetsList': presets_list})
-        with open('/data/outputs/current_jobs.json.temp', 'w') as f:
-            json.dump(current_jobs, f)
-            shutil.move('/data/outputs/current_jobs.json.temp', '/data/outputs/current_jobs.json')
-
-sio = socketio.Client()
-
-@sio.event
-def connect():
-    print("Update Loop Connected!")
-    sio.emit('updater_connected')
-    shared_data['connected'] = True
-
-@sio.event
-def disconnect():
-    print('disconnected from server')
-    shared_data['connected'] = False
-
-@sio.on('add_job_to_queue')
-def ws_add_job_to_queue(data):
-    job_type = data['job_type']
-    if job_type == 'fim_run':
-        job_name = data['job_name']
-        branch = data['branch']
-        hucs = data['hucs']
-        parallel_jobs = data['parallel_jobs']
-        hucs_type = data['hucs_type']
-        extent = data['extent']
-        config_path = data['config_path']
-        dev_run = data['dev_run']
-        viz_run = data['viz_run']
-
-        # This is a preset list instead of a custom list of hucs
-        if hucs_type == 0:
-            if os.path.exists(hucs):
-                with open(hucs, "r") as preset_file:
-                    hucs_raw = preset_file.read().splitlines()
-                    parallel_jobs = len(hucs_raw)
-                    hucs_type = len(hucs_raw[0])
-
-        parallel_jobs = parallel_jobs if parallel_jobs <= MAX_ALLOWED_CPU_CORES else MAX_ALLOWED_CPU_CORES
-
-        buffer_jobs.append({
-            'job_type': job_type,
-            'job_name': job_name,
-            'branch': branch,
-            'hucs': hucs,
-            'parallel_jobs': parallel_jobs,
-            'hucs_type': hucs_type,
-            'extent': extent,
-            'config_path': config_path,
-            'dev_run': dev_run,
-            'viz_run': viz_run,
-            'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
-            'status': 'In Queue',
-            'time_started': 0,
-            'time_elapsed': 0,
-            'output_files_saved': {},
-            'total_output_files_length': 0,
-            'current_output_files_saved_length': 0,
-            'output_files_saved': {},
-            'container_state': 'running',
-            'exit_code': 0,
-            'is_actively_saving': False
-        })
-    elif job_type == 'release':
-        job_name = data['job_name']
-        hucs = data['hucs']
-        extent = data['extent']
-        branch = 'dev'
-        config_path = './foss_fim/config/params_template.env'
-        dev_run = False
-        viz_run = True
-        previous_major_fim_version = data['previous_major_fim_version']
-
-        if os.path.exists(hucs):
-            with open(hucs, "r") as preset_file:
-                hucs_raw = preset_file.read().splitlines()
-                parallel_jobs = len(hucs_raw)
-                hucs_type = len(hucs_raw[0])
-
-        parallel_jobs = parallel_jobs if parallel_jobs <= MAX_ALLOWED_CPU_CORES else MAX_ALLOWED_CPU_CORES
-
-        buffer_jobs.append({
-            'job_type': job_type,
-            'job_name': job_name,
-            'branch': branch,
-            'hucs': hucs,
-            'parallel_jobs': parallel_jobs,
-            'hucs_type': hucs_type,
-            'extent': extent,
-            'config_path': config_path,
-            'dev_run': dev_run,
-            'viz_run': viz_run,
-            'nice_name': re.search(r"apijob_(.+)_apijob.+", job_name).group(1),
-            'status': 'In Queue',
-            'time_started': 0,
-            'time_elapsed': 0,
-            'output_files_saved': {},
-            'total_output_files_length': 0,
-            'current_output_files_saved_length': 0,
-            'output_files_saved': {},
-            'container_state': 'running',
-            'exit_code': 0,
-            'is_actively_saving': False,
-            'previous_major_fim_version': previous_major_fim_version
-        })
-
-@sio.on('remove_job_from_queue')
-def ws_remove_job_from_queue(data):
-    job_name = data['job_name']
-    buffer_remove_jobs.append({'job_name': job_name})
-
-# If the output_handler is offline, try the saving process again
-@sio.on('retry_saving_files')
-def ws_retry_saving_files():
-    print('saving files failed, retrying')
-    for job_name in current_jobs:
-        if current_jobs[job_name]['status'] == "Saving File":
-            for path in current_jobs[job_name]['output_files_saved']:
-                if current_jobs[job_name]['output_files_saved'][path] != -1:
-                    current_jobs[job_name]['output_files_saved'][path] = 0
-
-            current_jobs[job_name]['status'] = 'Ready to Save File'
-
-@sio.on('file_chunk_saved')
-def ws_file_chunk_saved(data):
-    job_name = data['job_name']
-    file_path = data['file_path']
-
-    current_jobs[job_name]['output_files_saved'][file_path] += 1
-    current_jobs[job_name]['status'] = 'Ready to Save File'
-
-@sio.on('file_saved')
-def ws_file_saved(data):
-    job_name = data['job_name']
-    file_path = data['file_path']
-
-    current_jobs[job_name]['output_files_saved'][file_path] = -1
-    current_jobs[job_name]['current_output_files_saved_length'] += 1
-    current_jobs[job_name]['status'] = 'Ready to Save File'
-
-sio.connect('http://fim_node_connector:6000/')
-update_loop()
diff --git a/data/preprocess_rasters.py b/data/preprocess_rasters.py
deleted file mode 100755
index 189a30d8a..000000000
--- a/data/preprocess_rasters.py
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-sys.path.append('/foss_fim/src')
-from multiprocessing import Pool
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from tqdm import tqdm
-import argparse
-
-sys.path.append('/foss_fim/src')
-from utils.reproject_dem import reproject_dem
-from utils.shared_functions import update_raster_profile
-from utils.shared_variables import PREP_PROJECTION, PREP_PROJECTION_CM
-
-##################################
-##
-#### Deprecated (will be replaced with new files and new logic):  Nov 8, 2022 ####
-##
-##################################
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Reproject Elevation rasters and update profile')
-    parser.add_argument('-dem_dir','--dem-dir', help='DEM filename', required=True,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
-    parser.add_argument('-nodata','--nodata-val', help='DEM nodata value', required=False,type=float,default=-9999.0)
-    parser.add_argument('-block','--blocksize', help='DEM blocksize', required=False,type=int,default=512)
-    parser.add_argument('-keep','--keep-intermediate', help='keep intermediate files', required=False,default=True,action='store_true')
-    parser.add_argument('-op','--overwrite-projected', help='Overwrite projected files', required=False,default=False,action='store_true')
-    parser.add_argument('-om','--overwrite-meters', help='Overwrite meter files', required=False,default=False,action='store_true')
-
-    args = vars(parser.parse_args())
-
-    dem_dir            = args['dem_dir']
-    number_of_jobs     = args['number_of_jobs']
-    nodata_val         = args['nodata_val']
-    blocksize          = args['blocksize']
-    keep_intermediate  = args['keep_intermediate']
-    overwrite_projected  = args['overwrite_projected']
-    overwrite_meters  = args['overwrite_meters']
-
-    reproject_procs_list = []
-
-    for huc in os.listdir(dem_dir):
-        raster_dir = os.path.join(dem_dir,huc)
-        elev_cm = os.path.join(raster_dir, 'elev_cm.tif')
-        elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
-        reproject_procs_list.append([raster_dir, elev_cm, elev_cm_proj, PREP_PROJECTION_CM,overwrite_projected])
-
-    # Multiprocess reprojection
-    with Pool(processes=number_of_jobs) as pool:
-        pool.map(reproject_dem, reproject_procs_list)
-
-    profile_procs_list = []
-    
-    for huc in os.listdir(dem_dir):
-        elev_m_tif = os.path.join(dem_dir,huc, 'elev_m.tif')
-        raster_dir = os.path.join(dem_dir,huc)
-        elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
-        elev_m = os.path.join(raster_dir, 'elev_m.tif')
-        profile_procs_list.append((huc,[elev_cm_proj, elev_m,PREP_PROJECTION,nodata_val,blocksize,keep_intermediate,overwrite_meters]))
-
-    executor = ThreadPoolExecutor(max_workers=number_of_jobs)
-    
-    executor_generator = { 
-                          executor.submit(update_raster_profile,p):h for h,p in profile_procs_list
-                         }
-    
-    for future in tqdm(as_completed(executor_generator),
-                       total=len(profile_procs_list),
-                       desc="Processing rasters with {} workers ".format(number_of_jobs)
-                      ):
-        hucCode = executor_generator[future]
-        #try:
-        future.result()
-        #except Exception as exc:
-        #    print('{},{},{}'.format(hucCode,exc.__class__.__name__,exc))
-    
-    # power down pool
-    executor.shutdown(wait=True)
-    
-    # Multiprocess update profile
- #   with Pool(processes=2) as pool:
-        # TODO read in windows becasue gdal rasters are massive
-#        pool.map(update_raster_profile, profile_procs_list)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 436a78443..a537454ea 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,40 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.0.19.3 - 2023-01-17 - [PR#794](https://github.com/NOAA-OWP/inundation-mapping/pull/794)
+
+Removing FIM3 files and references.  Anything still required for FIM 3 are held in the dev-fim3 branch.
+
+### Removals
+
+- `data`
+    - `preprocess_rasters.py`: no longer valid as it is for NHD DEM rasters.
+- `fim_run.sh`
+- ` src`
+    - `aggregate_fim_outputs.sh`
+    - `fr_to_ms_raster.mask.py`
+    - `get_all_huc_in_inputs.py`
+    - `reduce_nhd_stream_density.py`
+    - `rem.py`:  There are two files named `rem.py`, one in the src directory and one in the gms directory. This version in the src directory is no longer valid. The `rem.py` in the gms directory is being renamed to avoid future enhancements of moving files.
+    - `run_by_unit.sh`:  There are two files named `run_by_unit.sh`, one in the src directory and one in the gms directory. This version in the src directory is for fim3. For the remaining `run_by_unit.sh`, it is NOT being renamed at this time as it will likely be renamed in the near future.
+    - `time_and_tee_run_by_unit.sh`:  Same not as above for `run_by_unit.sh`.
+    - `utils`
+        - `archive_cleanup.py`
+ - `tools`
+     - `compare_gms_srcs_to_fr.py`
+     - `preprocess_fimx.py`
+
+### Changes
+
+- `src`
+    - `adjust_headwater_streams.py`: Likely deprecated but kept for safety reason. Deprecation note added.
+- `tools`
+    - `cygnss_preprocess.py`: Likely deprecated but kept for safety reason. Deprecation note added.
+    - `nesdis_preprocess.py`: Likely deprecated but kept for safety reason. Deprecation note added.
+
+<br/><br/>
+
 ## v4.0.19.2 - 2023-01-17 - [PR#797](https://github.com/NOAA-OWP/inundation-mapping/pull/797)
 
 Consolidates global bash environment variables into a new `src/bash_variables.env` file. Additionally, Python environment variables have been moved into this file and `src/utils/shared_variables.py` now references this file. Hardcoded projections have been replaced by an environment variable. This also replaces the Manning's N file in `config/params_template.env` with a constant and updates relevant code. Unused environment variables have been removed.
diff --git a/fim_run.sh b/fim_run.sh
deleted file mode 100755
index 676fc0bda..000000000
--- a/fim_run.sh
+++ /dev/null
@@ -1,206 +0,0 @@
-#!/bin/bash -e
-:
-usage ()
-{
-    echo 'Produce FIM datasets'
-    echo 'Usage : fim_run.sh [REQ: -u <hucs> -c <config file> -n <run name> ] [OPT: -h -j <job limit>]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -u/--hucList    : HUC 4,6,or 8 to run or multiple passed in quotes. Line delimited file'
-    echo '                     also accepted. HUCs must present in inputs directory.'
-    echo '  -e/--extent     : full resolution or mainstem method; options are MS or FR'
-    echo '  -c/--config     : configuration file with bash environment variables to export'
-    echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
-    echo ''
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
-    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo '  -p/--production : only save final inundation outputs'
-    echo '  -w/--whitelist  : list of files to save in a production run in addition to final inundation outputs'
-    echo '                     ex: file1.tif,file2.json,file3.csv'
-    echo '  -v/--viz        : compute post-processing on outputs to be used in viz'
-    echo '  -m/--mem        : enable memory profiling'
-    exit
-}
-
-##################################
-##
-#### Deprecated:  Oct 1, 2022 ####
-echo "Deprecated: Oct 1, 2022"
-##
-##################################
-
-
-if [ "$#" -lt 7 ]
-then
-  usage
-fi
-
-while [ "$1" != "" ]; do
-case $1
-in
-    -u|--hucList)
-        shift
-        hucList="$1"
-        ;;
-    -c|--configFile )
-        shift
-        envFile=$1
-        ;;
-    -e|--extent)
-        shift
-        extent=$1
-        ;;
-    -n|--runName)
-        shift
-        runName=$1
-        ;;
-    -j|--jobLimit)
-        shift
-        jobLimit=$1
-        ;;
-    -h|--help)
-        shift
-        usage
-        ;;
-    -o|--overwrite)
-        overwrite=1
-        ;;
-    -p|--production)
-        production=1
-        ;;
-    -w|--whitelist)
-        shift
-        whitelist="$1"
-        ;;
-    -v|--viz)
-        viz=1
-        ;;
-    -m|--mem)
-        mem=1
-        ;;
-    *) ;;
-    esac
-    shift
-done
-
-# print usage if arguments empty
-if [ "$hucList" = "" ]
-then
-    usage
-fi
-if [ "$extent" = "" ]
-then
-    usage
-fi
-if [ "$envFile" = "" ]
-then
-    usage
-fi
-if [ "$runName" = "" ]
-then
-    usage
-fi
-
-## SOURCE ENV FILE AND FUNCTIONS ##
-source $envFile
-source $srcDir/bash_functions.env
-
-# default values
-if [ "$jobLimit" = "" ] ; then
-    jobLimit=$default_max_jobs
-fi
-if [ "$viz" = "" ] ; then
-    viz=0
-fi
-
-## Define Outputs Data Dir & Log File##
-export outputRunDataDir=$outputDataDir/$runName
-export extent=$extent
-export production=$production
-export whitelist=$whitelist
-export viz=$viz
-export mem=$mem
-logFile=$outputRunDataDir/logs/summary.log
-
-## Define inputs
-export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
-export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
-export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
-export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_nhd_flowlines=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_streams_adj.gpkg
-export input_nhd_headwaters=$inputDataDir/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg
-export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
-## Input handling ##
-$srcDir/check_huc_inputs.py -u "$hucList"
-
-## Make output and data directories ##
-if [ -d "$outputRunDataDir" ] && [  "$overwrite" -eq 1 ]; then
-    rm -rf "$outputRunDataDir"
-elif [ -d "$outputRunDataDir" ] && [ -z "$overwrite" ] ; then
-    echo "$runName data directories already exist. Use -o/--overwrite to continue"
-    exit 1
-fi
-
-mkdir -p $outputRunDataDir/logs
-if [ ! -d "$outputRunDataDir/unit_errors" ]; then
-    mkdir -p $outputRunDataDir/unit_errors
-else
-    if [ $overwrite -eq 1 ]; then
-        rm -rf $outputRunDataDir/unit_errors/*.*
-        rm -rf $outputRunDataDir/logs/unit/*.*
-    fi
-fi
-
-## RUN ##
-if [ -f "$hucList" ]; then
-    if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh :::: $hucList
-    else
-        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh :::: $hucList
-    fi
-else
-    if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh ::: $hucList
-    else
-        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/time_and_tee_run_by_unit.sh ::: $hucList
-    fi
-fi
-
-# identify missing HUCs
-# time python3 /foss_fim/tools/fim_completion_check.py -i $hucList -o $outputRunDataDir
-if [ "$extent" = "MS" ] && [ "$bathy_src_toggle" = "True" ]; then
-    # Run BARC routine
-    echo -e $startDiv"Performing Bathy Adjusted Rating Curve routine"$stopDiv
-    time python3 /foss_fim/src/bathy_src_adjust_topwidth.py -fim_dir $outputRunDataDir -bfull_geom $bankfull_input_table -j $jobLimit -plots $src_plot_option
-else
-    echo -e $startDiv"SKIPPING Bathy Adjusted Rating Curve routine"$stopDiv
-fi
-
-echo -e $startDiv"Estimating bankfull stage in SRCs"$stopDiv
-if [ "$src_bankfull_toggle" = "True" ]; then
-    # Run BARC routine
-    time python3 /foss_fim/src/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit -plots $src_bankfull_plot_option
-fi
-
-echo -e $startDiv"Applying variable roughness in SRCs"$stopDiv
-if [ "$src_vrough_toggle" = "True" ]; then
-    # Run BARC routine
-    time python3 /foss_fim/src/vary_mannings_n_composite.py -fim_dir $outputRunDataDir -mann $vmann_input_file -bc $bankfull_attribute -suff $vrough_suffix -j $jobLimit -plots $src_vrough_plot_option -viz_clean $viz
-fi
-
-echo "$viz"
-if [[ "$viz" -eq 1 ]]; then
-    # aggregate outputs
-    time python3 /foss_fim/src/aggregate_fim_outputs.py -d $outputRunDataDir -j 6
-fi
-
-## GET NON ZERO EXIT CODES ##
-# Needed in case aggregation fails, we will need the logs
-find $outputRunDataDir/logs/ -name "*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/unit_errors/non_zero_exit_codes.log"
-
-echo "================================================================================"
-echo "Unit processing is complete"
-echo
diff --git a/src/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
index 8350311c5..300e542b0 100644
--- a/src/adjust_headwater_streams.py
+++ b/src/adjust_headwater_streams.py
@@ -13,6 +13,12 @@
 import warnings
 warnings.simplefilter("ignore")
 
+##################################
+##
+## Likely Deprecated: File not in use. Noticed Jan 16, 2023
+## Might want to be kept for possible re-use at a later time?
+##
+##################################
 
 def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
@@ -162,6 +168,14 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
 if __name__ == '__main__':
 
+##################################
+##
+## Likely Deprecated: File not in use. Noticed Jan 16, 2023
+## Might want to be kept for possible re-use at a later time?
+##
+##################################
+
+
     parser = argparse.ArgumentParser(description='adjust headwater stream geometery based on headwater start points')
     parser.add_argument('-f','--huc',help='huc number',required=True)
     parser.add_argument('-l','--nhd-streams',help='NHDPlus HR geodataframe',required=True)
@@ -186,3 +200,4 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
     if adj_headwater_points_fileName is not None:
         adj_headwaters_gdf.to_file(args['adj_headwater_points_fileName'],driver=getDriver(args['adj_headwater_points_fileName']))
+
diff --git a/src/aggregate_fim_outputs.sh b/src/aggregate_fim_outputs.sh
deleted file mode 100755
index 7e632d586..000000000
--- a/src/aggregate_fim_outputs.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash -e
-
-#outputRunDataDir=$1
-
-echo "Aggregating FIM required outputs"
-
-# make aggregate fim outputs dir
-outputRunDataDir=$1
-fimAggregateOutputsDir=$outputRunDataDir/aggregate_fim_outputs
-mkdir $fimAggregateOutputsDir
-
-# cd to make vrt paths relative
-cd $fimAggregateOutputsDir
-
-# build rem vrt
-gdalbuildvrt -q rem.vrt ../*/rem_zeroed_masked.tif
-
-# build catchments vrt
-gdalbuildvrt -q catchments.vrt ../*/gw_catchments_reaches_filtered_addedAttributes.tif
-
-# aggregate hydro-table
-i=0 #inialize counter variable
-for f in $(find $outputRunDataDir -type f -name hydroTable.csv); do
-    if [ "$i" -gt 0 ];then # aggregate remaining files without header
-        tail -n+2 $f >> $fimAggregateOutputsDir/hydroTable.csv
-    else # copy first file over with header
-        cat $f > $fimAggregateOutputsDir/hydroTable.csv
-    fi
-    ((i=i+1)) #counter variable
-done
-
-# cd back
-cd $OLDPWD
diff --git a/src/fr_to_ms_raster_mask.py b/src/fr_to_ms_raster_mask.py
deleted file mode 100755
index 2ebc713f0..000000000
--- a/src/fr_to_ms_raster_mask.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-'''
- Description: Mask raster layers using 'mainstems' stream buffer
-'''
-
-import sys
-import os
-import argparse
-import geopandas as gpd
-import rasterio.mask
-from utils.shared_functions import mem_profile
-
-
-@mem_profile
-def fr_to_ms_raster_mask(ms_buffer_dist, split_flows_filename, fdr_fr, dem_fr, slope_fr, fdr_ms_filename, dem_ms_filename, slope_ms_filename, str_pixel_fr, str_pixel_ms_filename):
-    # create output layer names
-    split_flows = gpd.read_file(split_flows_filename)
-
-    # Limit the rasters to the buffer distance around the draft streams.
-    print ("Limiting rasters to buffer area ({} meters) around model streams".format(str(ms_buffer_dist)))
-
-    split_flows_ms_buffer = split_flows.unary_union.buffer(ms_buffer_dist)
-
-    print('Writing raster outputs ...')
-
-    # Mask nhddem
-    with rasterio.open(dem_fr) as src:
-        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-        out_meta = src.meta
-
-    out_meta.update({"driver": "GTiff",
-         "height": out_image.shape[1],
-         "width": out_image.shape[2],
-         "transform": out_transform})
-
-    with rasterio.open(os.path.join(os.path.dirname(dem_fr), dem_ms_filename), "w", **out_meta) as dest:
-        dest.write(out_image)
-
-    # Mask fdr
-    with rasterio.open(fdr_fr) as src:
-        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-        out_meta = src.meta
-
-    out_meta.update({"driver": "GTiff",
-         "height": out_image.shape[1],
-         "width": out_image.shape[2],
-         "transform": out_transform})
-
-    with rasterio.open(os.path.join(os.path.dirname(fdr_fr), fdr_ms_filename), "w", **out_meta) as dest:
-        dest.write(out_image)
-
-    # Mask slope
-    with rasterio.open(slope_fr) as src:
-        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-        out_meta = src.meta
-
-    out_meta.update({"driver": "GTiff",
-         "height": out_image.shape[1],
-         "width": out_image.shape[2],
-         "transform": out_transform})
-
-    with rasterio.open(os.path.join(os.path.dirname(slope_fr), slope_ms_filename), "w", **out_meta) as dest:
-        dest.write(out_image)
-
-    # Mask stream pixels
-    with rasterio.open(str_pixel_fr) as src:
-        out_image, out_transform = rasterio.mask.mask(src, [split_flows_ms_buffer], crop=True)
-        out_meta = src.meta
-
-    out_meta.update({"driver": "GTiff",
-         "height": out_image.shape[1],
-         "width": out_image.shape[2],
-         "transform": out_transform})
-
-    with rasterio.open(os.path.join(os.path.dirname(str_pixel_fr), str_pixel_ms_filename), "w", **out_meta) as dest:
-        dest.write(out_image)
-
-
-if __name__ == '__main__':
-    ms_buffer_dist = int(os.environ['ms_buffer_dist'])
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='fr_to_ms_raster_mask.py')
-    parser.add_argument('-s', '--split-flows-filename', help='split-flows-filename', required=True)
-    parser.add_argument('-f', '--fdr-fr', help='fdr-fr', required=True)
-    parser.add_argument('-d', '--dem-fr', help='dem-fr', required=True)
-    parser.add_argument('-r', '--slope-fr', help='slope-fr', required=True)
-    parser.add_argument('-m', '--fdr-ms-filename', help='fdr-ms-filename', required=True)
-    parser.add_argument('-n', '--dem-ms-filename', help='dem-ms-filename', required=True)
-    parser.add_argument('-o', '--slope-ms-filename', help='slope-ms-filename', required=True)
-    parser.add_argument('-p', '--str-pixel-fr', help='str-pixel-fr', required=True)
-    parser.add_argument('-q', '--str-pixel-ms-filename', help='str-pixel-ms-filename', required=True)
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    fr_to_ms_raster_mask(ms_buffer_dist, **args)
diff --git a/src/get_all_huc_in_inputs.py b/src/get_all_huc_in_inputs.py
deleted file mode 100755
index af507b10b..000000000
--- a/src/get_all_huc_in_inputs.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python3
-
-import geopandas as gpd
-import pandas as pd
-import numpy as np
-from glob import glob
-import argparse
-from os.path import join
-from tqdm import tqdm
-
-
-def find_unique_hucs(inputsDir,hucLength):
-
-    # get file list with glob
-    huc4_vaa_list = glob(join(inputsDir,'NHDPlusFlowlineVAA_*.gpkg'))
-
-    unique_hucs = np.array([])
-    for vaa_file in tqdm(huc4_vaa_list):
-        reachCodes = gpd.read_file(vaa_file)['ReachCode']
-        reachCodes = reachCodes.astype(str)
-        reachCodes = reachCodes.apply(lambda x : x[0:8])
-        unique_hucs = np.append(unique_hucs,reachCodes.apply(lambda x: x[0:hucLength]).unique())
-    
-    unique_hucs = pd.Series(unique_hucs)
-    unique_hucs.to_csv(join(inputsDir,'included_huc{}.lst'.format(hucLength)),header=False,index=False)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Get unique HUCs in results data dir')
-    parser.add_argument('-i','--inputs-directory',help='Basins polygons to use within project path',required=True)
-    parser.add_argument('-l','--huc-length',help='Basins polygons to use within proiject path',required=True,type=int)
-
-    args = vars(parser.parse_args())
-
-    inputsDir = args['inputs_directory']
-    hucLength = args['huc_length']
-
-    find_unique_hucs(inputsDir,hucLength)
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
index 274fe1727..8e6d63971 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -103,7 +103,7 @@ Tcount
 echo -e $startDiv"D8 REM $hucNumber $current_branch_id"$stopDiv
 date -u
 Tstart
-$srcDir/gms/rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
+$srcDir/gms/make_rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
 Tcount
 
 ## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
diff --git a/src/gms/rem.py b/src/gms/make_rem.py
similarity index 100%
rename from src/gms/rem.py
rename to src/gms/make_rem.py
diff --git a/src/gms/toDo.md b/src/gms/toDo.md
index ebe514a1b..53bd35845 100755
--- a/src/gms/toDo.md
+++ b/src/gms/toDo.md
@@ -1,11 +1,4 @@
 # To Do List
-## FIM 3 functionality
-- *NDV for elevation rasters:* setting NDV for rasters is exposed to the user in preprocess_raster.py. This should be set to value in src/shared_variables.py of elev_raster_ndv. This prevents issues. Make sure all references to NDV in entire pipeline are consistent.
-- *Pipeline Management:* the user pipeline should be strengthened. Preprocess, hydrofab production, inundation, evaluation steps should be consolidated. For example: there are many preprocessing steps now. Also every major pipeline step should have meta-data associated with it in some sort of pandas friendly json format. Example: every fim_run/gms_run should have a meta-data json detailing the model used, parameters, date ran, user, and the inherited data from the preprocessing steps of the pipeline.
-- *NLD Lines:* nld lines are first being rasterized then burned.
-    - the python script burn_levees could be avoided using gdal_calc and converting nld elevations to meters in preprocessing
-    - if conversion to meters is done in preprocessing. It maybe possible to burn the nld elevations directly into the dem with gdal_rasterize all in one step
-    - this could help both FIM 3 and GMS
 
 ## Eval
 - *Testing Architecture & Modularity:* The test case functionality requires more modularity. There should be a clear set of tools that abstract away the model or test cases used. 
@@ -23,7 +16,6 @@
 - *NWM Divergences:* Levelpath derivation doesn't handle divergences (eg 12020002).
     - it shortens the effective length of levelpaths thus reducing the rating curve height of the most upstream catchment
     - it also creates more levelpaths and likely increases computational time
-- *Update rem.py*: rem.py underwent some change. Update to latest src/rem.py from src/gms/rem.py
 - *Unique Level Path and Hydro IDs:* unique identifiers for HydroIDs with GMS. Maybe FIMID, then branch ID, then HydroID.
 - *convenience wrapper for gms_run...sh:* Make a convenience wrapper for gms_run_unit.sh and gms_run_branch.sh. Be mindful of the two different processing loads and expose two different job numbers to the user.
     - *Deny Listing for Units:* The files in the deny list for units is executed at the end of gms/run_by_unit.sh. This requires files used in run_by_branch.sh to be left while not necessary left behind. This should be moved later in the process possibly once the convenience wrapper is made.
diff --git a/src/reduce_nhd_stream_density.py b/src/reduce_nhd_stream_density.py
deleted file mode 100644
index e11472746..000000000
--- a/src/reduce_nhd_stream_density.py
+++ /dev/null
@@ -1,232 +0,0 @@
-#!/usr/bin/env python3
-
-import geopandas as gpd
-import pandas as pd
-import numpy as np
-from collections import deque
-import argparse
-import pygeos
-from shapely.wkb import dumps
-from shapely.geometry import Point
-from utils.shared_functions import getDriver
-
-
-def subset_nhd_network(huc4,huc4_mask,selected_wbd8,nhd_streams_,headwaters_filename,headwater_id,nwm_intersections_filename,mainstem_flag=False):
-
-    headwater_streams = pd.DataFrame()
-
-    if mainstem_flag == False:
-        nhd_streams = gpd.read_file(nhd_streams_)
-        headwater_col = 'is_headwater'
-        id_col = 'headwaters_id'
-        n = -1
-    else:
-        nhd_streams = nhd_streams_.copy()
-        headwater_col = 'mainstem'
-        id_col = 'nws_lid'
-        n = ''
-
-    # Locate the closest NHDPlus HR stream segment to NWM headwater points. Done by HUC8 to reduce processing time and to contain NWM headwater in the same HUC
-    for index, row in selected_wbd8.iterrows():
-        huc = row["HUC8"]
-
-        # Double check that this is a nested HUC
-        if huc.startswith(str(huc4)):
-
-            huc8_mask = selected_wbd8.loc[selected_wbd8.HUC8==huc]
-            huc8_mask = huc8_mask.reset_index(drop=True)
-
-            # Masking headwaters by HUC8
-            headwaters_mask = gpd.read_file(headwaters_filename, mask = huc8_mask)
-            headwaters_mask = headwaters_mask.reset_index(drop=True)
-
-            # Masking subset streams by HUC8
-            if mainstem_flag == False:
-                streams_subset = gpd.read_file(nhd_streams_, mask = huc8_mask)
-            else:
-                streams_subset = nhd_streams.loc[nhd_streams.HUC8==huc].copy()
-                if headwaters_mask.is_headwater.dtype != 'int': headwaters_mask.is_headwater = headwaters_mask.is_headwater.astype('int')
-                if headwaters_mask.is_colocated.dtype != 'int': headwaters_mask.is_colocated = headwaters_mask.is_colocated.astype('int')
-                headwaters_mask = headwaters_mask.loc[headwaters_mask.is_headwater==True]
-
-            if not streams_subset.empty:
-                streams_subset[headwater_col] = False
-                streams_subset = streams_subset.reset_index(drop=True)
-
-                # Create WKB geometry column
-                streams_subset['b_geom'] = None
-                for index, linestring in enumerate(streams_subset.geometry):
-                    streams_subset.at[index, 'b_geom'] = dumps(linestring)
-
-                # Create pygeos nhd stream geometries from WKB representation
-                streambin_geom = pygeos.io.from_wkb(streams_subset['b_geom'])
-
-                # Add HUC8 column
-                streams_subset['HUC8'] = str(huc)
-
-                # Add headwaters_id column
-                streams_subset[id_col] = n
-                distance_from_upstream = {}
-                for index, point in headwaters_mask.iterrows():
-
-                    # Convert headwater point geometries to WKB representation
-                    wkb_point = dumps(point.geometry)
-
-                    # Create pygeos headwater point geometries from WKB representation
-                    pointbin_geom = pygeos.io.from_wkb(wkb_point)
-
-                    # Distance to each stream segment
-                    distances = pygeos.measurement.distance(streambin_geom, pointbin_geom)
-
-                    # Find minimum distance
-                    min_index = np.argmin(distances)
-                    headwater_point_name = point[headwater_id]
-
-                    # Find stream segment closest to headwater point
-                    if mainstem_flag==True:
-
-                        if point.is_colocated==True:
-
-                            closest_stream = streams_subset.iloc[min_index]
-                            distance_to_line = point.geometry.distance(Point(closest_stream.geometry.coords[-1]))
-                            print(f"{point.nws_lid} distance on line {closest_stream.NHDPlusID}:  {np.round(distance_to_line,1)}")
-
-                            if not closest_stream.NHDPlusID in distance_from_upstream.keys():
-                                distance_from_upstream[closest_stream.NHDPlusID] = [point.nws_lid,distance_to_line]
-
-                            elif distance_from_upstream[closest_stream.NHDPlusID][1] > distance_to_line:
-                                distance_from_upstream[closest_stream.NHDPlusID] = [point.nws_lid,distance_to_line]
-
-                            headwater_point_name = distance_from_upstream[closest_stream.NHDPlusID][0]
-
-                    # Closest segment to headwater
-                    streams_subset.loc[min_index,headwater_col] = True
-                    streams_subset.loc[min_index,id_col] = headwater_point_name
-
-                headwater_streams = headwater_streams.append(streams_subset[['NHDPlusID',headwater_col,id_col,'HUC8']])
-
-    headwater_streams = headwater_streams.sort_values(headwater_col, ascending=False).drop_duplicates('NHDPlusID') # keeps headwater=True for conflicting duplicates
-
-    if mainstem_flag == False:
-        nhd_streams = nhd_streams.merge(headwater_streams,on='NHDPlusID',how='inner')
-    else:
-        headwater_streams = headwater_streams.drop(columns=['HUC8'])
-        nhd_streams = nhd_streams.merge(headwater_streams,on='NHDPlusID',how='outer')
-        nhd_streams[id_col] = nhd_streams[id_col].fillna(n)
-        nhd_streams[headwater_col] = nhd_streams[headwater_col].fillna(0)
-
-    del selected_wbd8, streams_subset, headwater_streams
-
-    huc4_mask_buffer = huc4_mask.buffer(10)
-
-    # Identify inflowing streams
-    nwm_intersections = gpd.read_file(nwm_intersections_filename, mask=huc4_mask_buffer)
-
-    if mainstem_flag == False:
-        nhd_streams['downstream_of_headwater'] = False
-        nhd_streams['is_relevant_stream'] = nhd_streams['is_headwater'].copy()
-    else:
-        nwm_intersections = nwm_intersections.loc[nwm_intersections.mainstem==1]
-
-    nhd_streams = nhd_streams.explode()
-    nhd_streams = nhd_streams.reset_index(drop=True)
-
-
-
-    # Find stream segment closest to nwm intersection point
-    for index, point in nwm_intersections.iterrows():
-
-        # Distance to each stream segment
-        distances = nhd_streams.distance(point.geometry)
-
-        # Find minimum distance
-        min_index = np.argmin(distances)
-
-        # Update attributes for incoming stream
-        nhd_streams.loc[min_index,headwater_col] = True
-
-        if mainstem_flag == False:
-            nhd_streams.loc[min_index,'downstream_of_headwater'] = True
-            nhd_streams['is_relevant_stream'] = nhd_streams[headwater_col].copy()
-
-    # Trace down from headwaters
-    nhd_streams.set_index('NHDPlusID',inplace=True,drop=False)
-
-    nhd_streams = get_downstream_segments(nhd_streams.copy(),headwater_col,mainstem_flag)
-
-    # nhd_streams.fillna(value = {"is_relevant_stream": False}, inplace=True)
-    nhd_streams = nhd_streams.loc[nhd_streams['is_relevant_stream'],:]
-    nhd_streams.reset_index(drop=True,inplace=True)
-
-    return nhd_streams
-
-
-def get_downstream_segments(streams, attribute,mainstem_flag):
-
-    Q = deque(streams.loc[streams[attribute],'NHDPlusID'].tolist())
-    visited = set()
-
-    while Q:
-        q = Q.popleft()
-        if q in visited:
-            continue
-
-        visited.add(q)
-        toNode,DnLevelPat = streams.loc[q,['ToNode','DnLevelPat']]
-
-        try:
-            downstream_ids = streams.loc[streams['FromNode'] == toNode,:].index.tolist()
-        except ValueError: # 18050002 has duplicate nhd stream feature
-            if len(toNode.unique()) == 1:
-                toNode = toNode.iloc[0]
-                downstream_ids = streams.loc[streams['FromNode'] == toNode,:].index.tolist()
-
-        # If multiple downstream_ids are returned select the ids that are along the main flow path (i.e. exclude segments that are diversions)
-        if len(set(downstream_ids))>1: # special case: remove duplicate NHDPlusIDs
-            relevant_ids = [segment for segment in downstream_ids if DnLevelPat == streams.loc[segment,'LevelPathI']]
-        else:
-            relevant_ids = downstream_ids
-
-        if mainstem_flag == False:
-
-            streams.loc[relevant_ids,'is_relevant_stream'] = True
-            streams.loc[relevant_ids,'downstream_of_headwater'] = True
-        else:
-            streams.loc[relevant_ids,'mainstem'] = True
-
-        for i in relevant_ids:
-            if i not in visited:
-                Q.append(i)
-
-    return streams
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Reduce NHDPlus HR network based on headwater points')
-    parser.add_argument('-n','--huc-number',help='HUC number',required=True,type=str)
-    parser.add_argument('-b','--huc4-mask',help='HUC4 mask',required=True)
-    parser.add_argument('-w','--selected-wbd8',help='WBD8 layer',required=True)
-    parser.add_argument('-t','--nhd-streams',help='NHDPlus HR geodataframe',required=True)
-    parser.add_argument('-a','--headwaters-filename',help='Headwaters points layer name',required=True,type=str)
-    parser.add_argument('-s','--subset-nhd-streams-fileName',help='Output streams layer name',required=False,type=str,default=None)
-    parser.add_argument('-i','--headwater-id',help='Headwater points ID column',required=True)
-    parser.add_argument('-c','--nwm-intersections-filename',help='NWM HUC4 intersection points',required=True)
-    parser.add_argument('-d','--mainstem-flag',help='flag for mainstems network',required=False,default=False)
-
-    args = vars(parser.parse_args())
-
-    huc_number = args['huc_number']
-    huc4_mask = args['huc4_mask']
-    selected_wbd8 = args['selected_wbd8']
-    nhd_streams = args['nhd_streams']
-    headwaters_filename = args['headwaters_filename']
-    subset_nhd_streams_fileName = args['subset_nhd_streams_fileName']
-    headwater_id = args['headwater_id']
-    nwm_intersections_filename = args['nwm_intersections_filename']
-    mainstem_flag = args['mainstem_flag']
-
-    subset_streams_gdf = subset_nhd_network(huc_number,huc4_mask,selected_wbd8,nhd_streams,headwaters_filename,headwater_id,nwm_intersections_filename,mainstem_flag=False)
-
-    if subset_nhd_streams_fileName is not None:
-        subset_streams_gdf.to_file(subset_nhd_streams_fileName,driver=getDriver(subset_nhd_streams_fileName),index=False)
diff --git a/src/rem.py b/src/rem.py
deleted file mode 100755
index c064710ed..000000000
--- a/src/rem.py
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env python3
-
-from numba import njit, typed, types
-import rasterio
-import numpy as np
-import argparse
-import pandas as pd
-import geopandas as gpd
-from utils.shared_functions import getDriver, mem_profile
-
-
-@mem_profile
-def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, dem_reaches_filename):
-    """
-        Calculates REM/HAND/Detrended DEM
-
-        Parameters
-        ----------
-        dem_fileName : str
-            File name of pit filled DEM raster.
-        pixel_watersheds_fileName : str
-            File name of stream pixel watersheds raster.
-        rem_fileName : str
-            File name of output relative elevation raster.
-        hydroid_fileName : str
-            File name of the hydroid raster (i.e. gw_catchments_reaches.tif)
-        dem_reaches_filename
-            File name of the reaches layer to populate HAND elevation attribute values and overwrite as output
-
-    """
-
-    # ------------------------------------------- Get catchment_hydroid_dict --------------------------------------------------- #
-    # The following creates a dictionary of the catchment ids (key) and their hydroid along the thalweg (value).
-    # This is needed to produce a HAND zero reference elevation by hydroid dataframe (helpful for evaluating rating curves & bathy properties)
-    @njit
-    def make_catchment_hydroid_dict(flat_value_raster, catchment_hydroid_dict, flat_catchments, thalweg_window):
-
-        for i,cm in enumerate(flat_catchments):
-            if thalweg_window[i] == 1:  # Only allow reference hydroid to be within thalweg.
-                catchment_hydroid_dict[cm] = flat_value_raster[i]
-        return(catchment_hydroid_dict)
-
-    # Open files.
-    gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
-    hydroid_pixels_object = rasterio.open(hydroid_fileName)
-    thalweg_raster_object = rasterio.open(thalweg_raster)
-
-    # Specify raster object metadata.
-    meta = hydroid_pixels_object.meta.copy()
-    meta['tiled'], meta['compress'] = True, 'lzw'
-
-    # -- Create catchment_hydroid_dict -- #
-    catchment_hydroid_dict = typed.Dict.empty(types.int64,types.int64)  # Initialize an empty dictionary to store the catchment hydroid.
-    # Update catchment_hydroid_dict with each pixel sheds hydroid.
-    # Creating dictionary containing catchment ids (key) and corresponding hydroid within the thalweg...
-    for ji, window in hydroid_pixels_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
-        hydroid_window = hydroid_pixels_object.read(1,window=window).ravel()  # Define hydroid_window
-        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window
-        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window
-
-        # Call numba-optimized function to update catchment_hydroid_dict with pixel sheds overlapping hydroid.
-        catchment_hydroid_dict = make_catchment_hydroid_dict(hydroid_window, catchment_hydroid_dict, catchments_window, thalweg_window)
-
-    hydroid_pixels_object.close()
-    gw_catchments_pixels_masked_object.close()
-    thalweg_raster_object.close()
-    # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
-    # The following creates a dictionary of the catchment ids (key) and their elevation along the thalweg (value).
-    @njit
-    def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalweg_window):
-
-        for i,cm in enumerate(flat_catchments):
-            if thalweg_window[i] == 1:  # Only allow reference elevation to be within thalweg
-                # If the catchment really exists in the dictionary, compare elevation values
-                if (cm in catchment_min_dict):
-                    if (flat_dem[i] < catchment_min_dict[cm]):
-                        # If the flat_dem's elevation value is less than the catchment_min_dict min, update the catchment_min_dict min
-                        catchment_min_dict[cm] = flat_dem[i]
-                else:
-                    catchment_min_dict[cm] = flat_dem[i]
-        return(catchment_min_dict)
-
-    # Open files.
-    gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
-    dem_thalwegCond_masked_object = rasterio.open(dem_fileName)
-    thalweg_raster_object = rasterio.open(thalweg_raster)
-
-    # Specify raster object metadata.
-    meta = dem_thalwegCond_masked_object.meta.copy()
-    meta['tiled'], meta['compress'] = True, 'lzw'
-
-    # -- Create catchment_min_dict -- #
-    catchment_min_dict = typed.Dict.empty(types.int64,types.float32)  # Initialize an empty dictionary to store the catchment minimums
-    # Update catchment_min_dict with pixel sheds minimum.
-    # Creating dictionary containing catchment ids (key) and corresponding elevation within the thalweg (value)...
-    for ji, window in dem_thalwegCond_masked_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
-        dem_window = dem_thalwegCond_masked_object.read(1,window=window).ravel()  # Define dem_window
-        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window
-        thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define thalweg_window
-
-        # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
-        catchment_min_dict = make_catchment_min_dict(dem_window, catchment_min_dict, catchments_window, thalweg_window)
-
-    dem_thalwegCond_masked_object.close()
-    gw_catchments_pixels_masked_object.close()
-    thalweg_raster_object.close()
-
-    # Merge and export dictionary to csv.
-    catchment_min_dict_df = pd.DataFrame.from_dict(catchment_min_dict, orient='index') # convert dict to dataframe
-    catchment_min_dict_df.columns = ['Median_Thal_Elev_m']
-    catchment_hydroid_dict_df = pd.DataFrame.from_dict(catchment_hydroid_dict, orient='index') # convert dict to dataframe
-    catchment_hydroid_dict_df.columns = ['HydroID']
-    merge_df = catchment_hydroid_dict_df.merge(catchment_min_dict_df, left_index=True, right_index=True)
-    merge_df.index.name = 'pixelcatch_id'
-
-    # Merge the HAND reference elevation by HydroID dataframe with the demDerived_reaches layer (add new layer attribute)
-    min_by_hydroid = merge_df.groupby(['HydroID']).min() # min value of all med_thal_elev for pixel catchments in each HydroID reach
-    min_by_hydroid.columns = ['min_thal_elev']
-    med_by_hydroid = merge_df.groupby(['HydroID']).median() # median value of all med_thal_elev for pixel catchments in each HydroID reach
-    med_by_hydroid.columns = ['med_thal_elev']
-    max_by_hydroid = merge_df.groupby(['HydroID']).max() # max value of all med_thal_elev for pixel catchments in each HydroID reach
-    max_by_hydroid.columns = ['max_thal_elev']
-    input_reaches = gpd.read_file(dem_reaches_filename)
-    input_reaches = input_reaches.merge(min_by_hydroid, on='HydroID') # merge dataframes by HydroID variable
-    input_reaches = input_reaches.merge(med_by_hydroid, on='HydroID') # merge dataframes by HydroID variable
-    input_reaches = input_reaches.merge(max_by_hydroid, on='HydroID') # merge dataframes by HydroID variable
-    input_reaches.to_file(dem_reaches_filename,driver=getDriver(dem_reaches_filename),index=False)
-    # ------------------------------------------------------------------------------------------------------------------------ #
-
-
-    # ------------------------------------------- Produce relative elevation model ------------------------------------------- #
-    @njit
-    def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
-        rem_window = np.zeros(len(flat_dem),dtype=np.float32)
-        for i,cm in enumerate(flat_catchments):
-            if cm in catchmentMinDict:
-                if catchmentMinDict[cm] == ndv:
-                    rem_window[i] = ndv
-                else:
-                    rem_window[i] = flat_dem[i] - catchmentMinDict[cm]
-
-        return(rem_window)
-
-    rem_rasterio_object = rasterio.open(rem_fileName,'w',**meta)  # Open rem_rasterio_object for writing to rem_fileName.
-    pixel_catchments_rasterio_object = rasterio.open(pixel_watersheds_fileName)  # Open pixel_catchments_rasterio_object
-    dem_rasterio_object = rasterio.open(dem_fileName)
-
-    # Producing relative elevation model raster
-    for ji, window in dem_rasterio_object.block_windows(1):
-        dem_window = dem_rasterio_object.read(1,window=window)
-        window_shape = dem_window.shape
-
-        dem_window = dem_window.ravel()
-        catchments_window = pixel_catchments_rasterio_object.read(1,window=window).ravel()
-
-        rem_window = calculate_rem(dem_window, catchment_min_dict, catchments_window, meta['nodata'])
-        rem_window = rem_window.reshape(window_shape).astype(np.float32)
-
-        rem_rasterio_object.write(rem_window, window=window, indexes=1)
-
-    dem_rasterio_object.close()
-    pixel_catchments_rasterio_object.close()
-    rem_rasterio_object.close()
-    # ------------------------------------------------------------------------------------------------------------------------ #
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Relative elevation from pixel based watersheds')
-    parser.add_argument('-d','--dem', help='DEM to use within project path', required=True)
-    parser.add_argument('-w','--watersheds',help='Pixel based watersheds raster to use within project path',required=True)
-    parser.add_argument('-t','--thalweg-raster',help='A binary raster representing the thalweg. 1 for thalweg, 0 for non-thalweg.',required=True)
-    parser.add_argument('-o','--rem',help='Output REM raster',required=True)
-    parser.add_argument('-i','--hydroid', help='HydroID raster to use within project path', required=True)
-    parser.add_argument('-s','--dem_reaches_in_out',help='DEM derived reach layer to join HAND reference elevation attribute',required=True)
-
-    # Extract arguments to dictionary.
-    args = vars(parser.parse_args())
-
-    # Define variable inputs.
-    dem_fileName = args['dem']
-    pixel_watersheds_fileName = args['watersheds']
-    rem_fileName = args['rem']
-    thalweg_raster = args['thalweg_raster']
-    hydroid_fileName = args['hydroid']
-    dem_reaches_filename = args['dem_reaches_in_out']
-
-    rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster, hydroid_fileName, dem_reaches_filename)
diff --git a/src/run_by_unit.sh b/src/run_by_unit.sh
deleted file mode 100755
index 679a1351b..000000000
--- a/src/run_by_unit.sh
+++ /dev/null
@@ -1,435 +0,0 @@
-#!/bin/bash -e
-
-## INITIALIZE TOTAL TIME TIMER ##
-T_total_start
-
-##################################
-##
-#### Deprecated:  Oct 1, 2022 ####
-echo "Deprecated: Oct 1, 2022"
-##
-##################################
-
-
-
-echo -e $startDiv"Parameter Values"
-echo -e "extent=$extent"
-echo -e "agree_DEM_buffer=$agree_DEM_buffer"
-echo -e "wbd_buffer=$wbd_buffer"
-echo -e "ms_buffer_dist=$ms_buffer_dist"
-echo -e "lakes_buffer_dist_meters=$lakes_buffer_dist_meters"
-echo -e "negative_burn_value=$negative_burn_value"
-echo -e "max_split_distance_meters=$max_split_distance_meters"
-echo -e "mannings_n=$manning_n"
-echo -e "stage_min_meters=$stage_min_meters"
-echo -e "stage_interval_meters=$stage_interval_meters"
-echo -e "stage_max_meters=$stage_max_meters"
-echo -e "slope_min=$slope_min"
-echo -e "ms_buffer_dist=$ms_buffer_dist"
-echo -e "ncores_gw=$ncores_gw"
-echo -e "ncores_fd=$ncores_fd"
-echo -e "default_max_jobs=$default_max_jobs"
-echo -e "memfree=$memfree"$stopDiv
-
-## SET OUTPUT DIRECTORY FOR UNIT ##
-hucNumber="$1"
-# Test for standard errors
-outputHucDataDir=$outputRunDataDir/$hucNumber
-mkdir $outputHucDataDir
-
-## SET VARIABLES AND FILE INPUTS ##
-hucUnitLength=${#hucNumber}
-huc4Identifier=${hucNumber:0:4}
-huc2Identifier=${hucNumber:0:2}
-input_NHD_WBHD_layer=WBDHU$hucUnitLength
-input_DEM=$inputDataDir/nhdplus_rasters/HRNHDPlusRasters"$huc4Identifier"/elev_m.tif
-input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
-input_bathy_bankfull=$inputDataDir/$bankfull_input_table
-
-# Define the landsea water body mask using either Great Lakes or Ocean polygon input #
-if [[ $huc2Identifier == "04" ]] ; then
-  input_LANDSEA=$input_GL_boundaries
-  echo -e "Using $input_LANDSEA for water body mask (Great Lakes)"
-else
-  input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
-fi
-
-## GET WBD ##
-echo -e $startDiv"Get WBD $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/wbd.gpkg ] && \
-ogr2ogr -f GPKG $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
-Tcount
-
-## Subset Vector Layers ##
-echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg ] && \
-python3 -m memory_profiler $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -s $input_nhd_flowlines -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nhd_headwaters -v $input_LANDSEA -c $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nhd_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -extent $extent -gl $input_GL_boundaries -lb $lakes_buffer_dist_meters -wb $wbd_buffer
-Tcount
-
-if [ "$extent" = "MS" ]; then
-  if [[ ! -f $outputHucDataDir/nhd_headwater_points_subset.gpkg ]] ; then
-    echo "EXIT FLAG!! (exit 55): No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
-    rm -rf $outputHucDataDir
-    exit 0
-  fi
-fi
-
-## Clip WBD8 ##
-echo -e $startDiv"Clip WBD8"$stopDiv
-date -u
-Tstart
-ogr2ogr -f GPKG -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8
-Tcount
-
-## CLIP DEM ##
-echo -e $startDiv"Clip DEM $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/dem_meters.tif ] && \
-gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" $input_DEM $outputHucDataDir/dem_meters.tif
-Tcount
-
-## GET RASTER METADATA
-echo -e $startDiv"Get DEM Metadata $hucNumber"$stopDiv
-date -u
-Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/dem_meters.tif)
-
-## RASTERIZE NLD MULTILINES ##
-echo -e $startDiv"Rasterize all NLD multilines using zelev vertices"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/nld_rasterized_elev.tif ] && [ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
-gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputHucDataDir/nld_rasterized_elev.tif
-Tcount
-
-## RASTERIZE REACH BOOLEAN (1 & 0) ##
-echo -e $startDiv"Rasterize Reach Boolean $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/flows_grid_boolean.tif ] && \
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/NHDPlusBurnLineEvent_subset.gpkg $outputHucDataDir/flows_grid_boolean.tif
-Tcount
-
-## RASTERIZE NHD HEADWATERS (1 & 0) ##
-echo -e $startDiv"Rasterize NHD Headwaters $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/headwaters.tif ] && \
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nhd_headwater_points_subset.gpkg $outputHucDataDir/headwaters.tif
-Tcount
-
-if [ "$extent" = "FR" ]; then
-  # RASTERIZE NWM CATCHMENTS ##
-  echo -e $startDiv"Raster NWM Catchments $hucNumber"$stopDiv
-  date -u
-  Tstart
-  [ ! -f $outputHucDataDir/nwm_catchments_proj_subset.tif ] && \
-  gdal_rasterize -ot Int32 -a ID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_catchments_proj_subset.gpkg $outputHucDataDir/nwm_catchments_proj_subset.tif
-  Tcount
-fi
-
-## BURN LEVEES INTO DEM ##
-echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwrite dem_meters.tif output) $hucNumber"$stopDiv
-date -u
-Tstart
-[ -f $outputHucDataDir/nld_rasterized_elev.tif ] && \
-python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputHucDataDir/dem_meters.tif -nld $outputHucDataDir/nld_rasterized_elev.tif -out $outputHucDataDir/dem_meters.tif
-Tcount
-
-## DEM Reconditioning ##
-# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
-# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
-echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/dem_burned.tif ] && \
-python3 -m memory_profiler $srcDir/agreedem.py -r $outputHucDataDir/flows_grid_boolean.tif -d $outputHucDataDir/dem_meters.tif -w $outputHucDataDir -g $outputHucDataDir/temp_work -o $outputHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
-Tcount
-
-## PIT REMOVE BURNED DEM ##
-echo -e $startDiv"Pit remove Burned DEM $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/dem_burned_filled.tif ] && \
-rd_depression_filling $outputHucDataDir/dem_burned.tif $outputHucDataDir/dem_burned_filled.tif
-Tcount
-
-## D8 FLOW DIR ##
-echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/flowdir_d8_burned_filled.tif ] && \
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputHucDataDir/dem_burned_filled.tif -p $outputHucDataDir/flowdir_d8_burned_filled.tif
-Tcount
-
-## DINF FLOW DIR ##
-# echo -e $startDiv"DINF on Filled Thalweg Conditioned DEM"$stopDiv
-# date -u
-# Tstart
-# [ ! -f $outputHucDataDir/flowdir_dinf_thalwegCond.tif] && \
-# mpiexec -n $ncores_fd $taudemDir2/dinfflowdir -fel $outputHucDataDir/dem_thalwegCond_filled.tif -ang $outputHucDataDir/flowdir_dinf_thalwegCond.tif -slp $outputHucDataDir/slopes_dinf.tif
-# Tcount
-
-## D8 FLOW ACCUMULATIONS ##
-echo -e $startDiv"D8 Flow Accumulations $hucNumber"$stopDiv
-date -u
-Tstart
-$taudemDir/aread8 -p $outputHucDataDir/flowdir_d8_burned_filled.tif -ad8  $outputHucDataDir/flowaccum_d8_burned_filled.tif -wg  $outputHucDataDir/headwaters.tif -nc
-Tcount
-
-# THRESHOLD ACCUMULATIONS ##
-echo -e $startDiv"Threshold Accumulations $hucNumber"$stopDiv
-date -u
-Tstart
-$taudemDir/threshold -ssa $outputHucDataDir/flowaccum_d8_burned_filled.tif -src  $outputHucDataDir/demDerived_streamPixels.tif -thresh 1
-Tcount
-
-## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
-echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputHucDataDir/demDerived_streamPixels.tif -o $outputHucDataDir/demDerived_streamPixels_ids.tif -g $outputHucDataDir/temp_grass
-Tcount
-
-## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
-echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputHucDataDir/dem_meters.tif -s $outputHucDataDir/demDerived_streamPixels.tif -a $outputHucDataDir/demDerived_streamPixels_ids_allo.tif -d $outputHucDataDir/demDerived_streamPixels_ids_dist.tif -t 50 -o $outputHucDataDir/dem_lateral_thalweg_adj.tif -th $thalweg_lateral_elev_threshold
-Tcount
-
-## MASK BURNED DEM FOR STREAMS ONLY ###
-echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/flowdir_d8_burned_filled_flows.tif ] && \
-gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputHucDataDir/flowdir_d8_burned_filled.tif -B $outputHucDataDir/demDerived_streamPixels.tif --calc="A/B" --outfile="$outputHucDataDir/flowdir_d8_burned_filled_flows.tif" --NoDataValue=0
-Tcount
-
-## FLOW CONDITION STREAMS ##
-echo -e $startDiv"Flow Condition Thalweg $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/dem_thalwegCond.tif ] && \
-$taudemDir/flowdircond -p $outputHucDataDir/flowdir_d8_burned_filled_flows.tif -z $outputHucDataDir/dem_lateral_thalweg_adj.tif -zfdc $outputHucDataDir/dem_thalwegCond.tif
-Tcount
-
-## D8 SLOPES ##
-echo -e $startDiv"D8 Slopes from DEM $hucNumber"$stopDiv
-date -u
-Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputHucDataDir/dem_lateral_thalweg_adj.tif -sd8 $outputHucDataDir/slopes_d8_dem_meters.tif
-Tcount
-
-# STREAMNET FOR REACHES ##
-echo -e $startDiv"Stream Net for Reaches $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/demDerived_reaches.shp ] && \
-$taudemDir/streamnet -p $outputHucDataDir/flowdir_d8_burned_filled.tif -fel $outputHucDataDir/dem_thalwegCond.tif -ad8 $outputHucDataDir/flowaccum_d8_burned_filled.tif -src $outputHucDataDir/demDerived_streamPixels.tif -ord $outputHucDataDir/streamOrder.tif -tree $outputHucDataDir/treeFile.txt -coord $outputHucDataDir/coordFile.txt -w $outputHucDataDir/sn_catchments_reaches.tif -net $outputHucDataDir/demDerived_reaches.shp
-Tcount
-
-## SPLIT DERIVED REACHES ##
-echo -e $startDiv"Split Derived Reaches $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ] && \
-python3 -m memory_profiler $srcDir/split_flows.py -f $outputHucDataDir/demDerived_reaches.shp -d $outputHucDataDir/dem_thalwegCond.tif -s $outputHucDataDir/demDerived_reaches_split.gpkg -p $outputHucDataDir/demDerived_reaches_split_points.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg
-Tcount
-
-if [[ ! -f $outputHucDataDir/demDerived_reaches_split.gpkg ]] ; then
-  echo "EXIT FLAG!! (exit 56): No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
-  rm -rf $outputHucDataDir
-  exit 0
-fi
-
-if [ "$extent" = "MS" ]; then
-  ## MASK RASTERS BY MS BUFFER ##
-  echo -e $startDiv"Mask Rasters with Stream Buffer $hucNumber"$stopDiv
-  date -u
-  Tstart
-  python3 -m memory_profiler $srcDir/fr_to_ms_raster_mask.py -s $outputHucDataDir/demDerived_reaches_split.gpkg -f $outputHucDataDir/flowdir_d8_burned_filled.tif -d $outputHucDataDir/dem_thalwegCond.tif -r $outputHucDataDir/slopes_d8_dem_meters.tif -m $outputHucDataDir/flowdir_d8_MS.tif -n $outputHucDataDir/dem_thalwegCond_MS.tif -o $outputHucDataDir/slopes_d8_dem_metersMS.tif -p $outputHucDataDir/demDerived_streamPixels.tif -q $outputHucDataDir/demDerived_streamPixelsMS.tif
-  Tcount
-
-  if [[ ! -f $outputHucDataDir/dem_thalwegCond_MS.tif ]] ; then
-    echo "EXIT FLAG!! (exit 57): No AHPs point(s) within HUC $hucNumber boundaries. Aborting run_by_unit.sh"
-    rm -rf $outputHucDataDir
-    exit 0
-  fi
-
-  dem_thalwegCond=$outputHucDataDir/dem_thalwegCond_MS.tif
-  slopes_d8_dem_meters=$outputHucDataDir/slopes_d8_dem_metersMS.tif
-  flowdir_d8_burned_filled=$outputHucDataDir/flowdir_d8_MS.tif
-  demDerived_streamPixels=$outputHucDataDir/demDerived_streamPixelsMS.tif
-else
-  dem_thalwegCond=$outputHucDataDir/dem_thalwegCond.tif
-  slopes_d8_dem_meters=$outputHucDataDir/slopes_d8_dem_meters.tif
-  flowdir_d8_burned_filled=$outputHucDataDir/flowdir_d8_burned_filled.tif
-  demDerived_streamPixels=$outputHucDataDir/demDerived_streamPixels.tif
-fi
-
-## GAGE WATERSHED FOR REACHES ##
-echo -e $startDiv"Gage Watershed for Reaches $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/gw_catchments_reaches.tif ] && \
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $flowdir_d8_burned_filled -gw $outputHucDataDir/gw_catchments_reaches.tif -o $outputHucDataDir/demDerived_reaches_split_points.gpkg -id $outputHucDataDir/idFile.txt
-Tcount
-
-## VECTORIZE FEATURE ID CENTROIDS ##
-echo -e $startDiv"Vectorize Pixel Centroids $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/flows_points_pixels.gpkg ] && \
-python3 -m memory_profiler $srcDir/reachID_grid_to_vector_points.py -r $demDerived_streamPixels -i featureID -p $outputHucDataDir/flows_points_pixels.gpkg
-Tcount
-
-## GAGE WATERSHED FOR PIXELS ##
-echo -e $startDiv"Gage Watershed for Pixels $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/gw_catchments_pixels.tif ] && \
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $flowdir_d8_burned_filled -gw $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/flows_points_pixels.gpkg -id $outputHucDataDir/idFile.txt
-Tcount
-
-# D8 REM ##
-echo -e $startDiv"D8 REM $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/rem.tif ] && \
-python3 -m memory_profiler $srcDir/rem.py -d $dem_thalwegCond -w $outputHucDataDir/gw_catchments_pixels.tif -o $outputHucDataDir/rem.tif -t $demDerived_streamPixels -i $outputHucDataDir/gw_catchments_reaches.tif -s $outputHucDataDir/demDerived_reaches_split.gpkg
-Tcount
-
-## DINF DISTANCE DOWN ##
-# echo -e $startDiv"DINF Distance Down on Filled Thalweg Conditioned DEM $hucNumber"$stopDiv
-# date -u
-# Tstart
-# [ ! -f $outputHucDataDir/flowdir_dinf_thalwegCond.tif] && \
-# mpiexec -n $ncores_fd $taudemDir/dinfdistdown -ang $outputHucDataDir/flowdir_dinf_thalwegCond.tif -fel $outputHucDataDir/dem_thalwegCond_filled.tif -src $demDerived_streamPixels -dd $outputHucDataDir/rem.tif -m ave h
-# Tcount
-
-## BRING DISTANCE DOWN TO ZERO ##
-echo -e $startDiv"Zero out negative values in distance down grid $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/rem_zeroed.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputHucDataDir/rem.tif --calc="(A*(A>=0))" --NoDataValue=$ndv --outfile=$outputHucDataDir/"rem_zeroed.tif"
-Tcount
-
-## POLYGONIZE REACH WATERSHEDS ##
-echo -e $startDiv"Polygonize Reach Watersheds $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/gw_catchments_reaches.gpkg ] && \
-gdal_polygonize.py -8 -f GPKG $outputHucDataDir/gw_catchments_reaches.tif $outputHucDataDir/gw_catchments_reaches.gpkg catchments HydroID
-Tcount
-
-## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
-echo -e $startDiv"Process catchments and model streams step 1 $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg ] && \
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputHucDataDir/gw_catchments_reaches.gpkg -f $outputHucDataDir/demDerived_reaches_split.gpkg -c $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -o $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
-
-# Tight now, while there is a system code enum in filter_catchments_and_add_attributes.py, 
-# it shoudl not happen at this time for standard fim_run.sh versions. 
-# This has been left in and commented for example purposes if other system codes are added.
-
-# test if we received a non-zero code back from filter_catchments_and_add_attributes.py
-#subscript_exit_code=$?
-# we have to retrow it if it is not a zero (but it will stop further execution in this script)
-#if [ $subscript_exit_code -ne 0 ]; then exit $subscript_exit_code; fi
-Tcount
-
-## GET RASTER METADATA ## *****
-echo -e $startDiv"Get Clipped Raster Metadata $hucNumber"$stopDiv
-date -u
-Tstart
-read fsize ncols nrows ndv_clipped xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/gw_catchments_reaches.tif)
-Tcount
-
-## RASTERIZE NEW CATCHMENTS AGAIN ##
-echo -e $startDiv"Rasterize filtered catchments $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.tif ] && \
-gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.tif
-Tcount
-
-## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
-echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber"$stopDiv
-date -u
-Tstart
-[ -f $outputHucDataDir/LandSea_subset.gpkg ] && [ ! -f $outputHucDataDir/LandSea_subset.tif ] && \
-gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputHucDataDir/LandSea_subset.tif
-Tcount
-
-## MASK SLOPE RASTER ##
-echo -e $startDiv"Masking Slope Raster to HUC $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/slopes_d8_dem_meters_masked.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $slopes_d8_dem_meters -B $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.tif --calc="(A*(B>0))+((B<=0)*-1)" --NoDataValue=-1 --outfile=$outputHucDataDir/"slopes_d8_dem_meters_masked.tif"
-Tcount
-
-## MASK REM RASTER ##
-echo -e $startDiv"Masking REM Raster to HUC $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/rem_zeroed_masked.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputHucDataDir/rem_zeroed.tif -B $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.tif --calc="(A*(B>0))" --NoDataValue=$ndv --outfile=$outputHucDataDir/"rem_zeroed_masked.tif"
-Tcount
-
-## MASK REM RASTER TO REMOVE OCEAN AREAS ##
-echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas in HUC $hucNumber"$stopDiv
-date -u
-Tstart
-[ -f $outputHucDataDir/LandSea_subset.tif ] && \
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputHucDataDir/rem_zeroed_masked.tif -B $outputHucDataDir/LandSea_subset.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputHucDataDir/"rem_zeroed_masked.tif"
-Tcount
-
-## MAKE CATCHMENT AND STAGE FILES ##
-echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/make_stages_and_catchlist.py -f $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -c $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -s $outputHucDataDir/stage.txt -a $outputHucDataDir/catchment_list.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
-Tcount
-
-## HYDRAULIC PROPERTIES ##
-echo -e $startDiv"Hydraulic Properties $hucNumber"$stopDiv
-date -u
-Tstart
-[ ! -f $outputHucDataDir/src_base.csv ] && \
-$taudemDir/catchhydrogeo -hand $outputHucDataDir/rem_zeroed_masked.tif -catch $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.tif -catchlist $outputHucDataDir/catchment_list.txt -slp $outputHucDataDir/slopes_d8_dem_meters_masked.tif -h $outputHucDataDir/stage.txt -table $outputHucDataDir/src_base.csv
-Tcount
-
-## FINALIZE CATCHMENTS AND MODEL STREAMS ##
-echo -e $startDiv"Finalize catchments and model streams $hucNumber"$stopDiv output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,
-date -u
-Tstart
-[ ! -f $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg ] && \
-python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes.gpkg -a $outputHucDataDir/demDerived_reaches_split_filtered.gpkg -s $outputHucDataDir/src_base.csv -l $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -f $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -r $outputHucDataDir/src_full_crosswalked.csv -j $outputHucDataDir/src.json -x $outputHucDataDir/crosswalk_table.csv -t $outputHucDataDir/hydroTable.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -y $outputHucDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $input_nwm_catchments -p $extent -k $outputHucDataDir/small_segments.csv
-Tcount
-
-## USGS CROSSWALK ##
-echo -e $startDiv"USGS Crosswalk $hucNumber"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/usgs_gage_crosswalk.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -dem $outputHucDataDir/dem_meters.tif -flows $outputHucDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -cat $outputHucDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg -wbd $outputHucDataDir/wbd_buffered.gpkg -dem_adj $dem_thalwegCond -outtable $outputHucDataDir/usgs_elev_table.csv -e $extent
-Tcount
-
-## CLEANUP OUTPUTS ##
-echo -e $startDiv"Cleaning up outputs $hucNumber"$stopDiv
-args=()
-[[ ! -z "$whitelist" ]] && args+=( "-w$whitelist" )
-(( production == 1 )) && args+=( '-p' )
-(( viz == 1 )) && args+=( '-v' )
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/output_cleanup.py $hucNumber $outputHucDataDir "${args[@]}"
-Tcount
diff --git a/src/time_and_tee_run_by_unit.sh b/src/time_and_tee_run_by_unit.sh
deleted file mode 100755
index 1566e2d46..000000000
--- a/src/time_and_tee_run_by_unit.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash -e
-
-if [[ "$mem" == "1" ]] ; then
-  mprof run -o $1.dat --include-children /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log
-  mprof plot -o $outputRunDataDir/logs/$1_memory $1.dat
-else
-  /usr/bin/time -v $srcDir/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/$1.log 
-fi
-
-#$exit ${PIPESTATUS[0]}
-
-return_codes=( "${PIPESTATUS[@]}" )
-
-# we do this way instead of working directly with stderr and stdout
-# as they were messing with output logs which we always want.
-for code in "${return_codes[@]}"
-do
-    # Make an extra copy of the  log in a new folder  if an error
-    # Note: It was tricky to load in the fim_enum into bash, so we will just 
-    # go with the code for now
-    if [ $code -eq 61 ]; then
-        echo
-        echo "***** HUC has no valid flowlines *****"  #(for fim_run this is invalid)
-    elif [ $code -ne 0 ]; then
-        echo
-        echo "***** An error has occured  *****"
-        cp $outputRunDataDir/logs/$1.log $outputRunDataDir/unit_errors
-        exit ${PIPESTATUS[0]}
-    fi
-done
-
-echo "================================================================================"
-exit
-
diff --git a/src/utils/archive_cleanup.py b/src/utils/archive_cleanup.py
deleted file mode 100755
index b0a08aafa..000000000
--- a/src/utils/archive_cleanup.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python3
-import os
-import argparse
-
-def archive_cleanup(archive_cleanup_path, additional_whitelist):
-    '''
-    Processes all archived job outputs from a given path to keep only necessary files
-
-    Parameters
-    ----------
-    archive_cleanup_path : STR
-        Path to the archived outputs
-    additional_whitelist : STR
-        Additional list of files to keep
-    '''
-
-    # List of files that will be saved by default
-    whitelist = [
-        'rem_zeroed_masked.tif',
-        'rem_clipped_zeroed_masked.tif',
-        'gw_catchments_reaches_filtered_addedAttributes.tif',
-        'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg',
-        'gw_catchments_reaches_clipped_addedAttributes.tif',
-        'gw_catchments_reaches_clipped_addedAttributes_crosswalked.gpkg',
-        'hydroTable.csv',
-        'gw_catchments_pixels.tif',
-        'dem_burned_filled.tif',
-        'demDerived_reaches.dbf',
-        'demDerived_reaches.prj',
-        'demDerived_reaches.shp',
-        'demDerived_reaches.shx'
-    ]
-
-    # Add any additional files to the whitelist that the user wanted to keep
-    if additional_whitelist:
-        whitelist = whitelist + [filename for filename in additional_whitelist.split(',')]
-
-    # Delete any non-whitelisted files
-    directory = os.fsencode(archive_cleanup_path)
-    for subdir in os.listdir(directory):
-        subdirname = os.fsdecode(subdir)
-        if subdirname != 'logs' and subdirname != 'aggregate_fim_outputs':
-            for file in os.listdir(os.path.join(archive_cleanup_path, subdirname)):
-                filename = os.fsdecode(file)
-                if filename not in whitelist: 
-                    os.remove(os.path.join(archive_cleanup_path, subdirname, filename))
-
-if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Cleanup archived output files')
-    parser.add_argument('archive_cleanup_path', type=str, help='Path to the archived job outputs')
-    parser.add_argument('-w', '--additional_whitelist', type=str, help='List of additional files to keep')
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    # Rename variable inputs
-    archive_cleanup_path = args['archive_cleanup_path']
-    additional_whitelist = args['additional_whitelist']
-    
-    # Run archive_cleanup
-    archive_cleanup(archive_cleanup_path, additional_whitelist)
-    
\ No newline at end of file
diff --git a/tools/compare_gms_srcs_to_fr.py b/tools/compare_gms_srcs_to_fr.py
deleted file mode 100755
index 539c1ba65..000000000
--- a/tools/compare_gms_srcs_to_fr.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#!/usr/bin/env python3
-
-import pandas as pd
-import numpy as np
-from glob import iglob, glob
-from os.path import join, exists
-import argparse
-import matplotlib.pyplot as plt
-from matplotlib.gridspec import GridSpec
-
-
-def join_srcs(fr_data_dir,ms_data_dir,gms_data_dir,outputs_dir=None):
-
-    gms_srcs_file_list_generator = iglob( join(gms_data_dir,"*","branches","*","src_full_*.csv"), recursive=True)
-    
-    """
-    gms_srcs = None
-    for i in gms_srcs_file_list_generator:
-        current_gms_src = pd.read_csv(i)
-
-        if gms_srcs is None:
-            gms_srcs = current_gms_src
-        else:
-            gms_srcs = pd.concat((gms_srcs,current_gms_src),ignore_index=True)
-    """
-
-    gms_processed_data_file_name = join(outputs_dir,'gms_mean_srcs_ble.h5')
-    ms_processed_data_file_name = join(outputs_dir,'ms_mean_srcs_ble.h5')
-    fr_processed_data_file_name = join(outputs_dir,'fr_mean_srcs_ble.h5')
-
-    if exists(gms_processed_data_file_name):
-        gms_srcs = pd.read_hdf(gms_processed_data_file_name)
-    else:
-        # GMS srcs
-        print("Loading GMS SRCs")
-        gms_srcs = pd.concat( [ pd.read_csv(i) for i in gms_srcs_file_list_generator] , ignore_index=True)
-        gms_srcs.loc[:,'Number of Cells'] = gms_srcs.loc[:,'Number of Cells'].astype(int)
-        gms_srcs = gms_srcs.loc[:,['Stage','Discharge (m3s-1)','Volume (m3)','BedArea (m2)','Number of Cells']].groupby('Stage').agg('mean').reset_index(drop=False)
-
-   
-    if exists(fr_processed_data_file_name):
-        fr_srcs = pd.read_hdf(fr_processed_data_file_name)
-    else:
-        # fr srcsi
-        fr_srcs_file_name = glob( join(fr_data_dir,'*',"src_full_crosswalked.csv"))[0]
-        fr_srcs = pd.read_csv( fr_srcs_file_name )
-
-        fr_srcs_file_names = glob( join(fr_data_dir,'*',"src_full_crosswalked.csv"))
-        print("Loading FR SRCs")
-        fr_srcs = pd.concat( [ pd.read_csv(i) for i in fr_srcs_file_names ] , ignore_index = True )
-        # average out values by stage
-        fr_srcs = fr_srcs.loc[:,['Stage','Discharge (m3s-1)','Volume (m3)','BedArea (m2)','Number of Cells']].groupby('Stage').agg('mean').reset_index(drop=False)
-
-    # gms
-    if exists(ms_processed_data_file_name):
-        ms_srcs = pd.read_hdf(ms_processed_data_file_name)
-    else:
-        ms_src_file_names = glob( join(ms_data_dir,'*',"src_full_crosswalked.csv"))
-        print("Loading MS SRCs")
-        ms_srcs = pd.concat( [ pd.read_csv(i) for i in ms_src_file_names ] , ignore_index = True )
-
-        ms_srcs = pd.concat( (ms_srcs,fr_srcs), ignore_index=True )
-    
-        ms_srcs = ms_srcs.loc[:,['Stage','Discharge (m3s-1)','Volume (m3)','BedArea (m2)','Number of Cells']].groupby('Stage').agg('mean').reset_index(drop=False)
-    
-    # save files
-    if not exists(gms_processed_data_file_name):
-        gms_srcs.to_hdf(gms_processed_data_file_name,key='data')
-
-    if not exists(ms_processed_data_file_name):
-        ms_srcs.to_hdf(ms_processed_data_file_name,key='data')
-
-    if not exists(fr_processed_data_file_name):
-        fr_srcs.to_hdf(fr_processed_data_file_name,key='data')
-
-    # figure
-    fig = plt.figure(constrained_layout=True,figsize=(6,7))
-    gs = GridSpec(3,2,figure=fig)
-
-    # SRC plots
-    ax1 = fig.add_subplot(gs[0,:])
-    ax1.plot(gms_srcs.loc[:,'Discharge (m3s-1)'],gms_srcs.loc[:,'Stage'],label='GMS Mean')
-    ax1.plot(ms_srcs.loc[:,'Discharge (m3s-1)'],ms_srcs.loc[:,'Stage'],label='MS Mean')
-    ax1.plot(fr_srcs.loc[:,'Discharge (m3s-1)'],fr_srcs.loc[:,'Stage'],label="FR Mean")
-    ax1.set_xlabel(r'Discharge ($m^{3}/s$)'+'\n'+'(a)')
-    ax1.set_ylabel('Stage, m')
-    ax1.ticklabel_format(axis='x',style='sci',useMathText=True,scilimits=(0,4))
-    ax1.legend(loc='center',bbox_to_anchor=(0.5,1.25,0,0),ncol=3)
-
-    #ax1.legend()
-    #if outputs_dir is not None:
-    #    ax1.savefig( join(outputs_dir,"rating_curve.jpg") )
-    
-    # Volume plots
-    ax2 = fig.add_subplot(gs[1,0])
-    ax2.plot(gms_srcs.loc[:,'Stage'],gms_srcs.loc[:,'Volume (m3)'],label='GMS Mean')
-    ax2.plot(ms_srcs.loc[:,'Stage'],ms_srcs.loc[:,'Volume (m3)'],label='MS Mean')
-    ax2.plot(fr_srcs.loc[:,'Stage'],fr_srcs.loc[:,'Volume (m3)'],label="FR Mean")
-    ax2.set_xlabel('Stage, m'+'\n'+'(b)')
-    ax2.set_ylabel(r'Volume, $m^3$')
-    #ax2.legend()
-    #if outputs_dir is not None:
-    #    plt.savefig( join(outputs_dir,"volume.jpg") )
-    
-    # Bed Area plots
-    ax3 = fig.add_subplot(gs[1,1])
-    ax3.plot(gms_srcs.loc[:,'Stage'],gms_srcs.loc[:,'BedArea (m2)'],label='GMS Mean')
-    ax3.plot(ms_srcs.loc[:,'Stage'],ms_srcs.loc[:,'BedArea (m2)'],label='MS Mean')
-    ax3.plot(fr_srcs.loc[:,'Stage'],fr_srcs.loc[:,'BedArea (m2)'],label="FR Mean")
-    ax3.set_xlabel('Stage, m'+'\n'+'(c)')
-    ax3.set_ylabel(r'Bed Area, $m^2$')
-    #ax3.legend()
-    #if outputs_dir is not None:
-    #    plt.savefig( join(outputs_dir,"bedarea.jpg") )
-    
-    # Volume and Bed Area plots
-    ax4 = fig.add_subplot(gs[2,0])
-    ax4.plot(gms_srcs.loc[:,'Stage'],
-             np.power(gms_srcs.loc[:,'Volume (m3)'],5/3) * np.power(gms_srcs.loc[:,'BedArea (m2)'],-2/3),
-             label='GMS Mean')
-    ax4.plot(ms_srcs.loc[:,'Stage'],
-             np.power(ms_srcs.loc[:,'Volume (m3)'],5/3) * np.power(ms_srcs.loc[:,'BedArea (m2)'],-2/3),
-             label='MS Mean')
-    ax4.plot(fr_srcs.loc[:,'Stage'],
-             np.power(fr_srcs.loc[:,'Volume (m3)'],5/3) * np.power(fr_srcs.loc[:,'BedArea (m2)'],-2/3),
-             label="FR Mean")
-    ax4.set_xlabel('Stage, m'+'\n'+'(d)')
-    ax4.set_ylabel(r'$\dfrac{(Volume,\ m^3)^{5/3} }{(Bed\ Area,\ m^2) ^{2/3} }$')
-    #ax4.legend()
-    #if outputs_dir is not None:
-    #    plt.savefig( join(outputs_dir,"volume_and_bedarea.jpg") )
-
-    # Number of Cells plots
-    ax5 = fig.add_subplot(gs[2,1])
-    ax5.plot(gms_srcs.loc[:,'Stage'],gms_srcs.loc[:,'Number of Cells'],label='GMS Mean')
-    ax5.plot(ms_srcs.loc[:,'Stage'],ms_srcs.loc[:,'Number of Cells'],label='MS Mean')
-    ax5.plot(fr_srcs.loc[:,'Stage'],fr_srcs.loc[:,'Number of Cells'],label="FR Mean")
-    ax5.set_xlabel('Stage, m'+'\n'+'(e)')
-    ax5.set_ylabel('Pixels')
-    ax5.ticklabel_format(axis='y',style='sci',useMathText=True,scilimits=(0,3))
-    
-    if outputs_dir is not None:
-        fig.savefig( join(outputs_dir,"rating_curve_comparison.jpg"), dpi=300)
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Comparing SRCs')
-    parser.add_argument('-f','--fr-data-dir', help='Directory with FR data', required=True)
-    parser.add_argument('-g','--gms-data-dir', help='Directory with GMS data', required=True)
-    parser.add_argument('-m','--ms-data-dir', help='Directory with MS data', required=True)
-    parser.add_argument('-o','--outputs-dir', help='Directory for outputs data', required=False,default=None)
-
-    args = vars(parser.parse_args())
-
-    join_srcs(**args)
diff --git a/tools/cygnss_preprocessing.py b/tools/cygnss_preprocessing.py
index bbd08c7e0..5d4450834 100755
--- a/tools/cygnss_preprocessing.py
+++ b/tools/cygnss_preprocessing.py
@@ -15,6 +15,14 @@
 from foss_fim.tools.inundation import read_nwm_forecast_file
 
 
+##################################
+##
+## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
+## Might want to be kept for possible re-use at a later time?
+##
+##################################
+
+
 def cygnss_preprocessing(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',forecast_output_file=None,retrieve=True,workers=6,download_directory=None, daily_mean_forecast_files=None):
 
     _, bounding_boxes = find_hucs_of_bounding_boxes(bounding_boxes_file,wbd=wbd,projection_of_boxes=projection_of_boxes,wbd_layer=wbd_layer)
@@ -209,9 +217,15 @@ def download_url(url,file_name):
     return(forecast_df)
 
 
+if __name__ == '__main__':
 
 
-if __name__ == '__main__':
+    ##################################
+    ##
+    ## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
+    ## Might want to be kept for possible re-use at a later time?
+    ##
+    ##################################
 
     # parse arguments
     parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
diff --git a/tools/nesdis_preprocessing.py b/tools/nesdis_preprocessing.py
index ccf0eeb15..45be46247 100755
--- a/tools/nesdis_preprocessing.py
+++ b/tools/nesdis_preprocessing.py
@@ -11,6 +11,13 @@
 import os
 from tqdm import tqdm
 
+##################################
+##
+## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
+## Might want to be kept for possible re-use at a later time?
+##
+##################################
+
 
 def nesdis_preprocessing(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',forecast_output_file=None,retrieve=True,workers=6,download_directory=None):
 
@@ -128,6 +135,14 @@ def download_url(url,file_name):
 
 if __name__ == '__main__':
 
+    ##################################
+    ##
+    ## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
+    ## Might want to be kept for possible re-use at a later time?
+    ##
+    ##################################
+
+
     # parse arguments
     parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
     parser.add_argument('-b','--bounding-boxes-file', help='Bounding box file', required=True)
diff --git a/tools/preprocess_fimx.py b/tools/preprocess_fimx.py
deleted file mode 100755
index cad6058d0..000000000
--- a/tools/preprocess_fimx.py
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/env python3
-
-import rasterio
-from rasterio.warp import calculate_default_transform, reproject, Resampling
-from rasterio import features
-import numpy as np
-import geopandas as gpd
-import os
-import argparse
-
-def fimx_to_fim3(catchments_path, raster_value_field, hand_raster_path, template_raster, out_hand_path = None, out_catchment_path = None):
-    '''
-    This function will produce a hand and catchment grid from fim1/fim2 for use in the fim3 inundation.py script. To accomplish this it:
-        1) reprojects the hand raster to the template raster crs.
-        2) It reprojects the catchment polygons to the template raster crs and then converts the polygons to a raster. The reprojected HAND raster properties are applied (extent, resolution)
-        3) Performs the intersection of the two rasters (HAND/catchments) and applies NODATA if either dataset has NODATA
-        4) Writes out the preprocessed HAND and Catchments raster if a path is specified.
-
-    Parameters
-    ----------
-    catchments_path : STRING
-        The path to the catchments vector data (assumes it is in a database).
-    raster_value_field : STRING
-        Attribute field in catchments layer whos values will be used for raster creation.
-    hand_raster_path : STRING
-        The path to the HAND raster dataset (ARC GRID is acceptable).
-    template_raster : STRING
-        Path to the template raster so that a CRS can be applied to output.
-    out_hand_path : STRING, optional
-        Path to the output HAND grid (Gtif format). The default is None.
-    out_catchment_path : STRING, optional
-        Path to the output catchment grid (Gtif format). The default is None.
-
-    Returns
-    -------
-    hand_masked : Numpy Array
-        Preprocessed HAND raster array.
-    hand_profile : LIST
-        Preprocessed HAND profile.
-    catchment_masked : Numpy Array
-        Preprocessed Catchment raster array.
-    catchment_profile : LIST
-        Preprocessed catchment raster profile.
-
-    '''
-
-
-    # Read in template raster as band object
-    reference = rasterio.open(template_raster)
-
-    ## Step 1: Convert HAND grid
-    # Read in the hand raster
-    hand = rasterio.open(hand_raster_path)
-    hand_arr = hand.read(1)
-    #Determine the new transform and dimensions of reprojected raster (CRS = reference raster)
-    new_transform, new_width, new_height = calculate_default_transform(hand.crs, reference.crs, hand.width, hand.height, *hand.bounds)
-    # Define an empty array that is same dimensions as output by the "calculate_default_transform" command
-    hand_proj = np.empty((new_height,new_width), dtype=np.float)
-    # Reproject to target dataset (resample method is bilinear due to elevation type data)
-    hand_nodata_value = -2147483648
-    reproject(hand_arr,
-              destination = hand_proj,
-              src_transform = hand.transform,
-              src_crs = hand.crs,
-              src_nodata = hand.nodata,
-              dst_transform = new_transform,
-              dst_crs = reference.crs,
-              dst_nodata = hand_nodata_value,
-              dst_resolution = hand.res,
-              resampling = Resampling.bilinear)
-
-    # Update profile data type and no data value
-    hand_profile = reference.profile
-    hand_profile.update(dtype = rasterio.float32)
-    hand_profile.update(nodata = hand_nodata_value)
-    hand_profile.update(width = new_width)
-    hand_profile.update(height = new_height)
-    hand_profile.update(transform = new_transform)
-
-    ## Step 2: Catchments to Polygons (same extent as the HAND raster)
-    # Read in the catchment layer to geopandas dataframe and convert to same CRS as reference raster
-    gdbpath, layername = os.path.split(catchments_path)
-    gdb_layer=gpd.read_file(gdbpath, driver='FileGDB', layer=layername)
-    proj_gdb_layer = gdb_layer.to_crs(reference.crs)
-    # Prepare vector data to be written to raster
-    shapes = list(zip(proj_gdb_layer['geometry'],proj_gdb_layer[raster_value_field].astype('int32')))
-    # Write vector data to raster image. Fill raster with zeros for areas that do not have data. We will set nodata to be zero later
-    catchment_proj = features.rasterize(((geometry, value) for geometry, value in shapes), fill = 0, out_shape=hand_proj.shape, transform=hand_profile['transform'], dtype = 'int32' )
-    # Save raster image to in-memory dataset. Reset dtype and nodata values.
-    catchment_profile = hand_profile.copy()
-    catchment_profile.update(dtype = 'int32')
-    catchment_profile.update(nodata=0)
-
-    ## Step 3: Union of NODATA locations applied to both HAND and Catchment grids
-    catchment_masked = np.where(np.logical_or(hand_proj == hand_profile['nodata'], catchment_proj == catchment_profile['nodata']), catchment_profile['nodata'],catchment_proj)
-        # Assign NODATA to hand where both catchment and hand have NODATA else assign hand values.
-    hand_masked = np.where(np.logical_or(hand_proj == hand_profile['nodata'], catchment_proj == catchment_profile['nodata']), hand_profile['nodata'],hand_proj)
-
-    ## Step 4: Write out hand and catchment rasters to file if path is specified
-    if out_hand_path is not None:
-        os.makedirs(os.path.split(out_hand_path)[0], exist_ok = True)
-        with rasterio.Env():
-            with rasterio.open(out_hand_path, 'w', **hand_profile) as hnd_dst:
-                hnd_dst.write(hand_masked.astype('float32'),1)
-    if out_catchment_path is not None:
-        os.makedirs(os.path.split(out_catchment_path)[0], exist_ok = True)
-        with rasterio.Env():
-            with rasterio.open(out_catchment_path, 'w', **catchment_profile) as cat_dst:
-                cat_dst.write(catchment_masked.astype('int32'),1)
-
-    return hand_masked, hand_profile, catchment_masked, catchment_profile
-
-if __name__ == '__main__':
-    # Parse arguments
-    parser = argparse.ArgumentParser(description = 'Preprocess FIM 1 and FIM 2 HAND and Catchment grids to be compatible with FIM 3.')
-    parser.add_argument('-c','--catchments-path', help = 'Path to catchments vector file', required = True)
-    parser.add_argument('-f', '--raster-value-field', help = 'Attribute ID field from which raster values will be assigned. Typically this will be "HydroID" for FIM2 and "feature_ID" for fim 1.', required = True)
-    parser.add_argument('-ha', '--hand-raster-path', help = 'Path to HAND raster (can be in ESRI GRID format)', required = True)
-    parser.add_argument('-t', '--template-raster', help = 'Path to a template raster. Properties (CRS, resolution) of the template raster will be used to preprocess HAND and Catchments grids', required = True)
-    parser.add_argument('-oh', '--out-hand-path', help = 'Path to the output HAND raster. Raster must be named "rem_clipped_zeroed_masked.tif', required = True)
-    parser.add_argument('-oc', '--out-catchment-path', help = 'Path to the output Catchment raster. Raster must be named "gw_catchments_reaches_clipped_addedAttributes.tif"', required = True)
-    # Extract to dictionary and assign to variables
-    args = vars(parser.parse_args())
-    # Run fimx to fim3 function
-    fimx_to_fim3(**args)

From 597b7f08da4d819245beceb52d2725fa6d792ffd Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 27 Jan 2023 11:09:35 -0600
Subject: [PATCH 204/359] v4.0.19.4 branch trimming for lake segments (#802)

---
 docs/CHANGELOG.md          | 9 +++++++++
 src/gms/stream_branches.py | 9 ++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a537454ea..accc0ef2b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.19.4 - 2023-01-25 - [PR#802](https://github.com/NOAA-OWP/inundation-mapping/pull/802)
+
+This revision includes a slight alteration to the filtering technique used to trim/remove lakeid nwm_reaches that exist at the upstream end of each branch network. By keeping a single lakeid reach at the branch level, we can avoid issues with the branch headwater point starting at a lake boundary. This ensures the headwater catchments for some branches are properly identified as a lake catchment (no inundation produced). 
+
+### Changes
+
+- `src/gms/stream_branches.py`: New changes to the `find_upstream_reaches_in_waterbodies` function: Added a step to create a list of nonlake segments (lakeid = -9999) . Use the list of nonlake reaches to allow the filter to keep a the first lakeid reach that connects to a nonlake segment.
+
+<br/><br/>
 
 ## v4.0.19.3 - 2023-01-17 - [PR#794](https://github.com/NOAA-OWP/inundation-mapping/pull/794)
 
diff --git a/src/gms/stream_branches.py b/src/gms/stream_branches.py
index 453cf716c..c5e68a512 100755
--- a/src/gms/stream_branches.py
+++ b/src/gms/stream_branches.py
@@ -484,29 +484,28 @@ def find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
                     # Remove reach from tmp_self
                     tmp_IDs.append(downstream_ID)
                     tmp_self.drop(tmp_self[tmp_self.From_Node.astype(int).isin([downstream_ID,])].index, inplace=True)
-
                     # Repeat for next lowest downstream reach
                     if downstream_ID in tmp_self.To_Node.astype(int).values:
                         return find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
-
             return tmp_IDs
 
         def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
             # Find highest reach(es)
             upstream_IDs = [int(x) for x in tmp_self.From_Node[~tmp_self.From_Node.isin(tmp_self.To_Node)]] # IDs of most upstream reach(es)
-
+            nonlake_reaches = [int(x) for x in tmp_self.From_Node[tmp_self.Lake  == -9999]] # IDs of most  reach(es) that are not designated as lake reaches
+            
             for upstream_ID in upstream_IDs:
                 # Stop if uppermost reach is not in a lake
                 if int(tmp_self.Lake[tmp_self.From_Node.astype(int)==upstream_ID]) == -9999:
                     continue
                 else:
+                    if int(tmp_self.To_Node[tmp_self.From_Node.astype(int)==upstream_ID]) in nonlake_reaches:
+                        continue
                     # Remove reach from tmp_self
                     tmp_IDs.append(upstream_ID)
                     tmp_self.drop(tmp_self[tmp_self.From_Node.astype(int).isin([upstream_ID,])].index, inplace=True)
-
                     # Repeat for next highest upstream reach
                     return find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
-
             return tmp_IDs
 
         if verbose:

From 5fe925ca362b88746917444e7060e12a6ab59c2e Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 27 Jan 2023 11:12:48 -0600
Subject: [PATCH 205/359] v4.0.19.5 Fix error msg of stats object not
 referenced (#801)

---
 docs/CHANGELOG.md              | 12 ++++++++++++
 tools/pixel_counter.py         |  6 ++++--
 tools/test_case_by_hydro_id.py | 26 +++++++++++++++++++++++---
 3 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index accc0ef2b..d952d5a9a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.0.19.5 - 2023-01-24 - [PR#801](https://github.com/NOAA-OWP/inundation-mapping/pull/801)
+
+When running tools/test_case_by_hydroid.py, it throws an error of local variable 'stats' referenced before assignment.
+
+### Changes
+
+- `tools`
+    - `pixel_counter.py`: declare stats object and remove the GA_Readonly flag
+    - `test_case_by_hydroid_id_py`: Added more logging.
+
+<br/><br/>
+
 ## v4.0.19.4 - 2023-01-25 - [PR#802](https://github.com/NOAA-OWP/inundation-mapping/pull/802)
 
 This revision includes a slight alteration to the filtering technique used to trim/remove lakeid nwm_reaches that exist at the upstream end of each branch network. By keeping a single lakeid reach at the branch level, we can avoid issues with the branch headwater point starting at a lake boundary. This ensures the headwater catchments for some branches are properly identified as a lake catchment (no inundation produced). 
diff --git a/tools/pixel_counter.py b/tools/pixel_counter.py
index e4a96dce9..b1d3afabb 100644
--- a/tools/pixel_counter.py
+++ b/tools/pixel_counter.py
@@ -118,6 +118,8 @@ def bbox_to_pixel_offsets(gt, bbox):
 # Main function that determines zonal statistics of raster classes in a polygon area
 def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_extent=False):
 
+    stats = []
+
     # Loop through different raster paths in the raster_path_dict and
     # perform zonal statistics on the files.
     for layer in raster_path_dict:
@@ -130,7 +132,7 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
 
 
         # Opens raster file and sets path
-        rds = gdal.Open(raster_path, GA_ReadOnly)
+        rds = gdal.Open(raster_path)
         
         assert rds
         rb = rds.GetRasterBand(1)
@@ -174,7 +176,7 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
 
         # Loop through vectors, as many as exist in file
         # Creates new list to contain their stats
-        stats = []
+
         feat = vlyr.GetNextFeature()
         while feat is not None:
 
diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py
index 4d7657ff6..7ff336cd7 100644
--- a/tools/test_case_by_hydro_id.py
+++ b/tools/test_case_by_hydro_id.py
@@ -6,6 +6,8 @@
 import pandas as pd
 import geopandas as gpd
 import argparse
+from datetime import datetime
+
 from pixel_counter import zonal_stats
 from tools_shared_functions import compute_stats_from_contingency_table
 from run_test_case import test_case
@@ -130,7 +132,7 @@ def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
                         help='The fim version to use. Should be similar to fim_3_0_24_14_ms',
                         required=True)
     parser.add_argument('-g', '--gpkg',
-                        help='filepath and filename to hold exported gpkg (and csv) file. Similar to /data/path/output.gpkg Need to use gpkg as output. ',
+                        help='filepath and filename to hold exported gpkg (and csv) file. Similar to /data/path/fim_performance_catchments.gpkg Need to use gpkg as output. ',
                         required=True)
 
 
@@ -139,6 +141,13 @@ def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
     benchmark_category = args['benchmark_category']
     version = args['version']
     csv = args['gpkg']
+
+    print("================================")
+    print("Start test_case_by_hydroid.py")
+    start_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"started: {dt_string}")
+    print()
         
     # Execution code
     csv_output = gpd.GeoDataFrame(columns=['HydroID', 'huc8','contingency_tot_area_km2',
@@ -162,7 +171,7 @@ def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
        
         for agree_rast in agreement_dict:
                 
-                print('performing_zonal_stats')
+                print(f'performing_zonal_stats for {agree_rast}')
 
                 branches_dir = os.path.join(test_case_class.fim_dir,'branches')
                 for branches in os.listdir(branches_dir):
@@ -216,6 +225,17 @@ def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
     print('writing_to_csv')
     csv_output.to_csv(csv_path_dot) # Save to CSV
 
+    print("================================")
+    print("End test_case_by_hydroid.py")
+
+    end_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"ended: {dt_string}")
+
+    # calculate duration
+    time_duration = end_time - start_time
+    print(f"Duration: {str(time_duration).split('.')[0]}")
+    print()
 
-    
 
+  
\ No newline at end of file

From 2f44e731047b0cc60b6401b2dde2216b7029e79c Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 13 Feb 2023 11:31:11 -0600
Subject: [PATCH 206/359] v4.1.0.0 One HUC processes its own branches (#806)

---
 README.md                                     |  28 +-
 ...ranches_prod.lst => deny_gms_branches.lst} |   0
 config/deny_gms_branches_dev.lst              |  61 ----
 ...ny_gms_unit_prod.lst => deny_gms_unit.lst} |   0
 config/params_template.env                    |   5 -
 docs/CHANGELOG.md                             |  71 +++++
 fim_pipeline.sh                               |  76 +++++
 fim_post_processing.sh                        | 203 +++++++++++++
 fim_pre_processing.sh                         | 247 ++++++++++++++++
 fim_process_unit_wb.sh                        | 129 +++++++++
 gms_pipeline.sh                               |  10 +-
 gms_run_branch.sh                             |   8 +-
 gms_run_post_processing.sh                    |   2 +-
 gms_run_unit.sh                               |   8 +-
 src/aggregate_branch_lists.py                 |  34 +++
 src/bash_functions.env                        |   4 +-
 src/bash_variables.env                        |   4 +
 src/check_huc_inputs.py                       |  12 +-
 src/clip_vectors_to_wbd.py                    |  14 +-
 src/generate_branch_list.py                   |  50 ++++
 src/generate_branch_list_csv.py               |  55 ++++
 src/gms/delineate_hydros_and_produce_HAND.sh  |  48 ++--
 src/gms/run_by_branch.sh                      |  28 +-
 src/identify_src_bankfull.py                  |   2 +-
 src/process_branch.sh                         |  57 ++++
 src/run_unit_wb.sh                            | 268 ++++++++++++++++++
 unit_tests/aggregate_branch_lists_params.json |   8 +
 .../aggregate_branch_lists_unittests.py       |  58 ++++
 .../generate_branch_list_csv_params.json      |  20 ++
 .../generate_branch_list_csv_unittests.py     | 104 +++++++
 unit_tests/generate_branch_list_params.json   |   8 +
 unit_tests/generate_branch_list_unittests.py  |  59 ++++
 unit_tests/gms/outputs_cleanup_params.json    |   4 +-
 33 files changed, 1541 insertions(+), 144 deletions(-)
 rename config/{deny_gms_branches_prod.lst => deny_gms_branches.lst} (100%)
 delete mode 100644 config/deny_gms_branches_dev.lst
 rename config/{deny_gms_unit_prod.lst => deny_gms_unit.lst} (100%)
 create mode 100755 fim_pipeline.sh
 create mode 100755 fim_post_processing.sh
 create mode 100755 fim_pre_processing.sh
 create mode 100755 fim_process_unit_wb.sh
 create mode 100755 src/aggregate_branch_lists.py
 create mode 100755 src/generate_branch_list.py
 create mode 100755 src/generate_branch_list_csv.py
 create mode 100755 src/process_branch.sh
 create mode 100755 src/run_unit_wb.sh
 create mode 100644 unit_tests/aggregate_branch_lists_params.json
 create mode 100644 unit_tests/aggregate_branch_lists_unittests.py
 create mode 100644 unit_tests/generate_branch_list_csv_params.json
 create mode 100644 unit_tests/generate_branch_list_csv_unittests.py
 create mode 100644 unit_tests/generate_branch_list_params.json
 create mode 100644 unit_tests/generate_branch_list_unittests.py

diff --git a/README.md b/README.md
index 7f9033268..dc5225b8e 100644
--- a/README.md
+++ b/README.md
@@ -36,9 +36,10 @@ aws s3 ls s3://noaa-nws-owp-fim/ --request-payer requester
 
 Download a directory of outputs for a HUC8:
 ```
-aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/fim_3_0_34_1/outputs/fr/12090301 12090301 --request-payer requester
+aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_0_18_02/12090301 /your_local_folder_name/12090301 --request-payer requester
 ```
-**Note**: There may be newer editions than fim_3_0_34_1, and it is recommended to adjust the command above for the latest version.
+By adjusting pathing, you can also download entire directories such as the fim_4_0_18_0 folder.
+**Note**: There may be newer editions than fim_4_0_18_0, and it is recommended to adjust the command above for the latest version.
 
 
 ## Running the Code
@@ -60,25 +61,25 @@ Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP
 This software is configurable via parameters found in the `config` directory. Copy files before editing and remove "template" pattern from the filename.
 Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
 - `params_template.env`
-- `mannings_default.json`
-    - must change filepath in `params_template.env` in `manning_n` variable name
 
 This system has an optional tool called the `calibration database tool`. In order to use this system, you will need to install the calibration database service or disable it in the `params_template.env` file. See [calibration tool README](https://github.com/NOAA-OWP/inundation-mapping/blob/dev/tools/calibration-db/README.md) for more details.
 
 ### Produce HAND Hydrofabric
 ```
-gms_pipeline.sh -u <huc8> -n <name_your_run>
+fim_pipeline.sh -u <huc8> -n <name_your_run>
 ```
-- There are a wide number of options and defaulted values, for details run ```gms_pipeline.sh -h```
+- There are a wide number of options and defaulted values, for details run ```fim_pipeline.sh -h```.
 - Manditory arguments:
-    - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited file
-    i. To run entire domain of available data use one of the ```/data/inputs/included_huc[4,6,8].lst``` files or a huc list file of your choice.
+    - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited (.lst) file. To run the entire domain of available data use one of the ```/data/inputs/included_huc8.lst``` files or a HUC list file of your choice.  Depending on the performance of your server, especially the number of CPU cores, running the full domain can take multiple days.
     - `-n` is a name of your run (only alphanumeric)
-- Outputs can be found under ```/data/outputs/<name_your_run>```
+- Outputs can be found under ```/data/outputs/<name_your_run>```.
 
-Processing of HUC's in FIM4 (GMS) comes in two pieces: gms_run_unit and gms_run_branch. `gms_pipeline.sh` above takes care of both steps however, you can run each part seperately for faster development if you like.
+Processing of HUC's in FIM4 comes in three pieces. You can run `fim_pipeline.sh` which automatically runs all of three major section, but you can run each of the sections independently if you like. The three sections are:
+- `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
+- `fim_process_unit_wb.sh` : This script processes one and exactly one HUC8 plus all of it's related branches. While it can only process one, you can run this script multiple times, each with different HUC (or overwriting a HUC). When you run `fim_pipeline.sh`, it automatically iterates when more than one HUC number has been supplied either by command line arguments or via a HUC list. For each HUC provided, `fim_pipeline.sh` will `fim_process_unit_wb.sh`. Using the `fim_process_unit_wb.sh`  script allows for a run / rerun of a HUC, or running other HUCs at different times / days or even different docker containers.
+- `fim_post_processing.sh` : This section takes all of the HUCs that have been processed, aggregates key information from each HUC directory and looks for errors across all HUC folders. It also processes the group in sub-steps such as usgs guages processesing, rating curve adjustments and more. Naturally, running or re-running this script can only be done after running `fim_pre_processing.sh` and at least one run of `fim_process_unit_wb.sh`.
 
-If you choose to do the two step hydrofabric creation, then run `gms_run_unit.sh`, then `gms_run_branch.sh`. See each of those files for details on arguments.
+Running the `fim_pipeline.sh` is a quicker process than running all three steps independently.
 
 ### Testing in Other HUCs
 To test in HUCs other than the provided HUCs, the following processes can be followed to acquire and preprocess additional NHDPlus rasters and vectors. After these steps are run, the "Produce HAND Hydrofabric" step can be run for the new HUCs.
@@ -86,6 +87,8 @@ To test in HUCs other than the provided HUCs, the following processes can be fol
 ```
 /foss_fim/src/acquire_and_preprocess_inputs.py -u <huc4s_to_process>
 ```
+    Note: This tool is deprecated, updates will be coming soon.
+
 - `-u` can be a single HUC4, series of HUC4s (e.g. 1209 1210), path to line-delimited file with HUC4s.
 - Please run `/foss_fim/src/acquire_and_preprocess_inputs.py --help` for more information.
 - See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
@@ -94,10 +97,11 @@ To test in HUCs other than the provided HUCs, the following processes can be fol
 ```
 /foss_fim/src/preprocess_rasters.py
 ```
+    Note: This tool is deprecated, updates will be coming soon.
 
 ----
 ### Evaluating Inundation Map Performance
-After `gms_pipeline.sh` completes, you can evaluate the model's skill. The evaluation benchmark datasets are available through ESIP in the `test_cases` directory.
+After `fim_pipeline.sh` completes, or combinations of the three major steps described above, you can evaluate the model's skill. The evaluation benchmark datasets are available through ESIP in the `test_cases` directory.
 
 To evaluate model skill, run the following:
 ```
diff --git a/config/deny_gms_branches_prod.lst b/config/deny_gms_branches.lst
similarity index 100%
rename from config/deny_gms_branches_prod.lst
rename to config/deny_gms_branches.lst
diff --git a/config/deny_gms_branches_dev.lst b/config/deny_gms_branches_dev.lst
deleted file mode 100644
index d52d4b855..000000000
--- a/config/deny_gms_branches_dev.lst
+++ /dev/null
@@ -1,61 +0,0 @@
-# List of files for gms branches to delete
-# Use comment to allow list the file
-# Use curly braces to denote branch_id
-agree_bufgrid.tif
-agree_bufgrid_allo.tif
-agree_bufgrid_dist.tif
-agree_smogrid.tif
-agree_smogrid_allo.tif
-agree_smogrid_dist.tif
-catch_list_{}.txt
-coordFile_{}.txt
-crosswalk_table_{}.csv
-demDerived_reaches_{}.dbf
-demDerived_reaches_{}.prj
-demDerived_reaches_{}.shp
-demDerived_reaches_{}.shx
-demDerived_reaches_split_{}.gpkg
-demDerived_reaches_split_filtered_{}.gpkg
-#demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
-demDerived_reaches_split_points_{}.gpkg
-demDerived_streamPixels_{}.tif
-demDerived_streamPixels_ids_{}.tif
-demDerived_streamPixels_ids_{}_allo.tif
-demDerived_streamPixels_ids_{}_dist.tif
-dem_burned_{}.tif
-dem_burned_filled_{}.tif
-dem_lateral_thalweg_adj_{}.tif
-dem_meters_{}.tif
-#dem_thalwegCond_{}.tif
-flowaccum_d8_burned_filled_{}.tif
-flowdir_d8_burned_filled_{}.tif
-flowdir_d8_burned_filled_flows_{}.tif
-flows_grid_boolean_{}.tif
-flows_points_pixels_{}.gpkg
-gw_catchments_pixels_{}.tif
-gw_catchments_reaches_{}.gpkg
-gw_catchments_reaches_{}.tif
-gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
-#gw_catchments_reaches_filtered_addedAttributes_{}.tif
-#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
-headwaters_{}.tif
-#hydroTable_{}.csv
-idFile_{}.txt
-LandSea_subset_{}.tif
-nld_rasterized_elev_{}.tif
-nld_subset_levees_{}.tif
-#nwm_catchments_proj_subset_levelPaths_{}.gpkg
-#nwm_subset_streams_levelPaths_{}.gpkg
-nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
-rem_{}.tif
-#rem_zeroed_masked_{}.tif
-slopes_d8_dem_meters_{}.tif
-slopes_d8_dem_meters_masked_{}.tif
-sn_catchments_reaches_{}.tif
-src_{}.json
-src_base_{}.csv
-#src_full_crosswalked_{}.csv
-stage_{}.txt
-streamOrder_{}.tif
-treeFile_{}.txt
-#usgs_elev_table.csv
\ No newline at end of file
diff --git a/config/deny_gms_unit_prod.lst b/config/deny_gms_unit.lst
similarity index 100%
rename from config/deny_gms_unit_prod.lst
rename to config/deny_gms_unit.lst
diff --git a/config/params_template.env b/config/params_template.env
index 384378de2..1895e6a25 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -51,9 +51,4 @@ export CALB_DB_KEYS_FILE="/data/config/calb_db_keys.env"
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
-export default_max_jobs=1 # default number of max concurrent jobs to run
 export memfree=0G # min free memory required to start a new job or keep youngest job alive
-
-#### logging parameters ####
-export startDiv="\n##########################################################################\n"
-export stopDiv="\n##########################################################################"
\ No newline at end of file
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d952d5a9a..1845a039b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,77 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.1.0.0 - 2023-01-30 - [PR#806](https://github.com/NOAA-OWP/inundation-mapping/pull/806)
+
+As we move to Amazon Web Service, AWS, we need to change our processing system. Currently, it is `gms_pipeline.sh` using bash "parallel" as an iterator which then first processes all HUCs, but not their branches. One of `gms_pipeline.sh`'s next steps is to do branch processing which is again iterated via "parallel". AKA. Units processed as one step, branches processed as second independent step.
+
+**Note:** While we are taking steps to move to AWS, we will continue to maintain the ability of doing all processing on a single server using a single docker container as we have for a long time. Moving to AWS is simply taking portions of code from FIM and adding it to AWS tools for performance of large scale production runs.
+
+Our new processing system, starting with this PR,  is to allow each HUC to process it's own branches.
+
+A further requirement was to split up the overall processing flow to independent steps, with each step being able to process itself without relying on "export" variables from other files. Note: There are still a few exceptions.  The basic flow now becomes
+- `fim_pre_processing.sh`, 
+- one or more calls to `fim_process_unit_wb.sh` (calling this file for each single HUC to be processed).
+- followed by a call to `fim_post_processing.sh`. 
+
+
+Note: This is a very large, complex PR with alot of critical details. Please read the details at [PR 806](https://github.com/NOAA-OWP/inundation-mapping/pull/806). 
+
+### CRITICAL NOTE
+The new `fim_pipeline.sh` and by proxy `fim_pre_processing.sh` has two new key input args, one named **-jh** (job HUCs) and one named **-jb** (job branches).  You can assign the number of cores/CPU's are used for processing a HUC versus the number of branches.  For the -jh number arg, it only is used against the `fim_pipeline.sh` file when it is processing more than one HUC or a list of HUCs as it is the iterator for HUCs.   The -jb flag says how many cores/CPU's can be used when processing branches (note.. the average HUC has 26 branches). 
+
+BUT.... you have to be careful not to overload your system.  **You need to multiply the -jh and the -jb values together, but only when using the `fim_pipeline.sh` script.**  Why? _If you have 16 CPU's available on your machine, and you assign -jh as 10 and -jb as 26, you are actually asking for 126 cores (10 x 26) but your machine only has 16 cores._   If you are not using `fim_pipeline.sh` but using the three processing steps independently, then the -jh value has not need to be anything but the number of 1 as each actual HUC can only be processed one at a time. (aka.. no iterator).
+</br>
+
+### Additions
+
+- `fim_pipeline.sh` :  The wrapper for the three new major "FIM" processing steps. This script allows processing in one command, same as the current tool of `gms_pipeline.sh`.
+- `fim_pre_processing.sh`: This file handles all argument input from the user, validates those inputs and sets up or cleans up folders. It also includes a new system of taking most input parameters and some key enviro variables and writing them out to a files called `runtime_args.env`.  Future processing steps need minimal input arguments as it can read most values it needs from this new `runtime_args.env`. This allows the three major steps to work independently from each other. Someone can now come in, run `fim_pre_processing.sh`, then run `fim_process_unit_wb.sh`, each with one HUC, as many time as they like, each adding just its own HUC folder to the output runtime folder. 
+- `fim_post_processing.sh`: Scans all HUC folders inside the runtime folders to handle a number of processing steps which include (to name a few): 
+    - aggregating errors
+    - aggregating to create a single list (gms_inputs.csv) for all valid HUCs and their branch ids
+    - usgs gage aggregation
+    - adjustments to SRV's
+    - and more    
+- `fim_process_unit_wb.sh`: Accepts only input args of runName and HUC number. It then sets up global variable, folders, etc to process just the one HUC. The logic for processing the HUC is in `run_unit_wb.sh` but managed by this `fim_process_unit_wb.sh` file including all error trapping.
+- `src`
+    - `aggregate_branch_lists.py`:  When each HUC is being processed, it creates it's own .csv file with its branch id's. In post processing we need one master csv list and this file aggregates them. Note: This is a similar file already in the `src/gms` folder but that version operates a bit different and will be deprecated soon.
+    - `generate_branch_list.py`: This creates the single .lst for a HUC defining each branch id. With this list, `run_unit_wb.sh` can do a parallelized iteration over each of its branches for processing. Note: This is also similar to the current `src/gms` file of the same name and the gms folder version will also be deprecated soon.
+    - `generate_branch_list_csv.py`. As each branch, including branch zero, has processed and if it was successful, it will add to a .csv list in the HUC directory. At the end, it becomes a list of all successful branches. This file will be aggregates with all similar .csv in post processing for future processing.
+    - `run_unit_wb.sh`:  The actual HUC processing logic. Note: This is fundamentally the same as the current HUC processing logic that exists currently in `src/gms/run_by_unit.sh`, which will be removed in the very near future. However, at the end of this file, it creates and manages a parallelized iterator for processing each of it's branches.
+    - `process_branch.sh`:  Same concept as `process_unit_wb.sh` but this one is for processing a single branch. This file manages the true branch processing file of `src/gms/run_by_branch.sh`.  It is a wrapper file to `src/gms/run_by_branch.sh` and catches all error and copies error files as applicable. This allows the parent processing files to continue despite branch errors. Both the new fim processing system and the older gms processing system currently share the branch processing file of `src/gms/run_by_branch.sh`. When the gms processing file is removed, this file will likely not change, only moved one directory up and be no longer in the `gms` sub-folder.
+- `unit_tests`
+    - `aggregate_branch_lists_unittests.py' and `aggregate_branch_lists_params.json`  (based on the newer `src` directory edition of `aggregate_branch_lists.py`).
+    - `generate_branch_list_unittest.py` and `generate_branch_list_params.json` (based on the newer `src` directory edition of `generate_branch_list.py`).
+    -  `generate_branch_list_csv_unittest.py` and `generate_branch_list_csv_params.json` 
+
+### Changes
+
+- `config`
+    - `params_template.env`: Removed the `default_max_jobs` value and moved the `startDiv` and `stopDiv` to the `bash_variables.env` file.
+    - `deny_gms_unit.lst` : Renamed from `deny_gms_unit_prod.lst`
+    - `deny_gms_branches.lst` : Renamed from `deny_gms_branches_prod.lst`
+
+- `gms_pipeline.sh`, `gms_run_branch.sh`, `gms_run_unit.sh`, and `gms_post_processing.sh` :  Changed to hardcode the `default_max_jobs` to the value of 1. (we don't want this to be changed at all). They were also changed for minor adjustments for the `deny` list files names.
+
+- `src`
+    - `bash_functions.env`: Fix error with calculating durations.
+    - `bash_variables.env`:  Adds the two export lines (stopDiv and startDiv) from `params_template.env`
+    - `clip_vectors_to_wbd.py`: Cleaned up some print statements for better output traceability.
+    - `check_huc_inputs.py`: Added logic to ensure the file was an .lst file. Other file formats were not be handled correctly.
+    - `gms`
+        - `delineate_hydros_and_produce_HAND.sh`: Removed all `stopDiv` variable to reduce log and screen output. 
+        - `run_by_branch.sh`: Removed an unnecessary test for overriding outputs.
+
+### Removed
+
+- `config`
+    - `deny_gms_branches_dev.lst`
+
+<br/><br/>
+
+
 ## v4.0.19.5 - 2023-01-24 - [PR#801](https://github.com/NOAA-OWP/inundation-mapping/pull/801)
 
 When running tools/test_case_by_hydroid.py, it throws an error of local variable 'stats' referenced before assignment.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
new file mode 100755
index 000000000..0e5a81a18
--- /dev/null
+++ b/fim_pipeline.sh
@@ -0,0 +1,76 @@
+#!/bin/bash -e
+
+: '
+fim_pipeline.sh -u <huc8> -n <name_your_run>
+
+For more details on 
+
+- There are a wide number of options and defaulted values, for details run ```fim_pipeline.sh -h```
+- Manditory arguments:
+    - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited file
+    i. To run entire domain of available data use the ```/data/inputs/included_huc8.lst``` file or a huc list file of your choice.
+    - `-n` is a name of your run (only alphanumeric)
+- Outputs can be found under ```/data/outputs/<name_your_run>```
+
+Processing of HUC''s in FIM4 comes in three pieces. You can run `fim_pipeline.sh` which automatically runs all of three major section, but you can run each of the sections independently if you like. The three sections are:
+- `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
+- `fim_process_unit_wb.sh` : This script processes one and exactly one HUC8 plus all of it''s related branches. While it can only process one, you can run this script multiple times, each with different HUC (or overwriting a HUC). When you run `fim_pipeline.sh`, it automatically iterates when more than one HUC number has been supplied either by command line arguments or via a HUC list. For each HUC provided, `fim_pipeline.sh` will `fim_process_unit_wb.sh`. Using the `fim_process_unit_wb.sh`  script allows for a run / rerun of a HUC, or running other HUCs at different times / days or even different docker containers.
+- `fim_post_processing.sh` : This section takes all of the HUCs that have been processed, aggregates key information from each HUC directory and looks for errors across all HUC folders. It also processes the group in sub-steps such as usgs guages processesing, rating curve adjustments and more. Naturally, running or re-running this script can only be done after running `fim_pre_processing.sh` and at least one run of `fim_process_unit_wb.sh`.
+
+Running the `fim_pipeline.sh` is a quicker process than running all three steps independently.
+'
+
+set -e
+
+# TODO
+# upgrade Dockerfile to add this as an env value
+projectDir=/foss_fim
+
+# See fim_pre_processing.sh for details of how to use this script. fim_pre_processing.sh
+# is a proxy for collecting and validating input.
+
+echo
+echo "======================= Start of fim_pipeline.sh ========================="
+echo "---- Started: `date -u`" 
+
+## LOAD AND VALIDATE INCOMING ARGUMENTS
+source $srcDir/bash_functions.env
+. $projectDir/fim_pre_processing.sh "$@"
+
+
+logFile=$outputRunDataDir/logs/unit/pipeline_summary_unit.log
+process_wb_file=$projectDir/fim_process_unit_wb.sh
+
+pipeline_start_time=`date +%s`
+
+# PROCESS THE UNITS (And branches)
+# Why an if and else? watch the number of colons
+if [ -f "$hucList" ]; then
+    if [ "$jobHucLimit" = "1" ]; then
+                parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList 
+    else
+        parallel --eta -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList
+    fi
+else 
+    if [ "$jobHucLimit" = "1" ]; then
+                parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName ::: $hucList
+    else
+        parallel --eta -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file  $runName  ::: $hucList
+    fi
+fi
+
+echo
+echo "---- Unit (HUC) processing is complete"
+date -u
+
+## POST PROCESSING
+
+# TODO: multiply the two job limits together for the limit here ??
+. $projectDir/fim_post_processing.sh -n $runName -j $jobHucLimit
+
+echo
+echo "======================== End of fim_pipeline.sh =========================="
+date -u
+Calc_Duration $pipeline_start_time
+echo
+
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
new file mode 100755
index 000000000..c105eacca
--- /dev/null
+++ b/fim_post_processing.sh
@@ -0,0 +1,203 @@
+#!/bin/bash -e
+
+:
+usage ()
+{
+    echo 'Post processing for creating FIM hydrofabric.'
+    echo 'Usage : fim_post_processing.sh [REQ: -n <run name> ]'
+    echo '  	 				         [OPT: -h -j <job limit>]'
+    echo ''
+    echo 'REQUIRED:'
+    echo '  -n/--runName    : A name to tag the output directories and log files.'
+    echo ''
+    echo 'OPTIONS:'
+    echo '  -h/--help       : help file'
+    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
+    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
+    echo '                    Note: Not the same variable name as fim_pipeline or fim_pre_processing'
+    echo '                    and can be the multiplication of jobHucLimit and jobBranchLimit'
+    echo
+    exit
+}
+
+while [ "$1" != "" ]; do
+case $1
+in
+    -n|--runName)
+        shift
+        runName=$1
+        ;;
+    -j|--jobLimit)
+        shift
+        jobLimit=$1
+        ;;
+    -h|--help)
+        shift
+        usage
+        ;;
+    *) ;;
+    esac
+    shift
+done
+
+# TODO
+# upgrade Dockerfile to add this as an env value
+projectDir=/foss_fim
+
+
+# print usage if arguments empty
+if [ "$runName" = "" ]
+then
+    echo "ERROR: Missing -n run time name argument"
+    usage
+fi
+
+outputRunDataDir=$outputDataDir/$runName
+
+## Check for run data directory ##
+if [ ! -d "$outputRunDataDir" ]; then 
+    echo "Depends on output from units and branches. Please provide an output folder name that has hucs/branches run."
+    exit 1
+fi
+
+if [ "$jobLimit" = "" ]; then jobLimit=1; fi
+
+# Clean out the other post processing files before starting
+rm -rdf $outputRunDataDir/logs/src_optimization
+rm -f $outputRunDataDir/logs/log_bankfull_indentify.log
+rm -f $outputRunDataDir/logs/subdiv_src_.log
+
+# load up enviromental information
+args_file=$outputRunDataDir/runtime_args.env
+gms_inputs=$outputRunDataDir/gms_inputs.csv
+
+source $args_file
+source $outputRunDataDir/params.env
+source $srcDir/bash_functions.env
+source $srcDir/bash_variables.env
+
+
+echo
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- Start of fim_post_processing"
+echo "---- Started: `date -u`" 
+T_total_start
+post_proc_start_time=`date +%s`
+
+
+## AGGREGATE BRANCH LISTS INTO ONE ##
+echo -e $startDiv"Start branch aggregation"
+python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $gms_inputs
+
+## GET NON ZERO EXIT CODES FOR UNITS ##
+## No longer applicable
+
+## GET NON ZERO EXIT CODES FOR BRANCHES ##
+echo -e $startDiv"Start non-zero exit code checking"
+find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
+
+
+## REMOVE FAILED BRANCHES ##
+# Needed in case aggregation fails, we will need the logs
+#echo
+#echo -e $startDiv"Removing branches that failed with Exit status: 61"
+#Tstart
+#python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g #$gms_inputs
+
+## RUN AGGREGATE BRANCH ELEV TABLES ##
+echo "Processing usgs gage aggregation"
+python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
+
+## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
+if [ "$src_bankfull_toggle" = "True" ]; then
+    echo -e $startDiv"Estimating bankfull stage in SRCs"
+    # Run SRC bankfull estimation routine routine
+    Tstart
+    python3 $srcDir/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit
+    Tcount
+fi
+
+## RUN SYNTHETIC RATING SUBDIVISION ROUTINE ##
+if [ "$src_subdiv_toggle" = "True" ]; then
+    echo -e $startDiv"Performing SRC channel/overbank subdivision routine"
+    # Run SRC Subdivision & Variable Roughness routine
+    Tstart
+    python3 $srcDir/subdiv_chan_obank_src.py -fim_dir $outputRunDataDir -mann $vmann_input_file -j $jobLimit
+    Tcount
+fi
+
+## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
+if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
+    if [ ! -f $CALB_DB_KEYS_FILE ]; then
+        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        exit 1
+    else
+        source $CALB_DB_KEYS_FILE
+
+        : '
+        This makes the local variables from the calb_db_keys files
+        into global variables that can be used in other files, including python.
+
+        Why not just leave the word export in front of each of the keys in the
+        calb_db_keys.env? Becuase that file is used against docker-compose
+        when we start up that part of the sytem and it does not like the word
+        export.
+        '
+        export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
+        export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
+        export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
+        export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
+        export DEFAULT_FIM_PROJECTION_CRS=$DEFAULT_FIM_PROJECTION_CRS
+
+        Tstart
+        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
+        echo "Loading HUC Data"
+        echo
+
+        ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+
+        echo "Loading Point Data"
+        echo
+        ogr2ogr -overwrite -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
+
+        Tcount
+    fi
+else
+    echo "Skipping Populate PostgrSQL database"
+fi
+
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
+if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
+    Tstart
+    echo    
+    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
+    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
+    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+    Tcount
+    date -u
+fi
+
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
+if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [ "$skipcal" = "0" ]; then
+    Tstart
+    echo
+    echo -e $startDiv"Performing SRC adjustments using benchmark point database"
+    python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
+    Tcount
+    date -u
+fi
+
+echo
+echo -e $startDiv"Combining crosswalk tables"
+# aggregate outputs
+Tstart
+python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
+Tcount
+date -u
+
+echo
+echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+echo "---- Start of fim_post_processing complete"
+echo "---- Ended: `date -u`"
+Calc_Duration $post_proc_start_time
+echo
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
new file mode 100755
index 000000000..260823699
--- /dev/null
+++ b/fim_pre_processing.sh
@@ -0,0 +1,247 @@
+#!/bin/bash -e
+:
+usage ()
+{
+    echo
+    echo 'Produce FIM hydrofabric datasets for unit and branch scale.'
+    echo 'Usage : [REQ: -u <hucs> -n <run name> ]'
+    echo '        [OPT: -h -c <config file> -o'
+    echo '          -ud <unit deny list file>'
+    echo '          -bd <branch deny list file>'
+    echo '          -zd <branch zero deny list file>'
+    echo '          -jh <number of jobs for HUCs>'
+    echo '          -jb <number of jobs for branches>'
+    echo '          -skipcal <skip the PostGres calibration process>'
+    echo '          -isaws ]'
+    echo ''
+    echo 'REQUIRED:'
+    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
+    echo '                    A line delimited file, with a .lst extension, is also acceptable.'
+    echo '                    HUCs must present in inputs directory.'
+    echo '  -n/--runName    : A name to tag the output directories and log files.'
+    echo 
+    echo 'OPTIONS:'
+    echo '  -h/--help       : Help file'
+    echo '  -c/--config     : Configuration file with bash environment variables to export'
+    echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
+    echo '  -ud/--unitDenylist'
+    echo '                    A file with a line delimited list of files in UNIT (HUC) directories to be removed'
+    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'
+    echo '  -bd/--branchDenylist'
+    echo '                    A file with a line delimited list of files in BRANCHES directories to be removed' 
+    echo '                    upon completion of branch processing.'
+    echo '                    (see config/deny_gms_branches.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'
+    echo '  -zd/--branchZeroDenylist'
+    echo '                    A file with a line delimited list of files in BRANCH ZERO directories to' 
+    echo '                    be removed upon completion of branch zero processing.'
+    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
+    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
+    echo '                    use the word NONE as this value for this parameter.'    
+    echo '  -jh/--jobLimit   : Max number of concurrent HUC jobs to run. Default 1 job at time.'
+    echo '  -jb/--jobBranchLimit : Max number of concurrent Branch jobs to run. Default 1 job at time.' 
+    echo '                         NOTE: Make sure that the multiplication of jh and jb subtract 2 (jh x jb -2) does not'
+    echo '                         exceed the total number of cores available.'
+    echo '  -o               : Overwrite outputs if already exist'
+    echo '  -skipcal         : If this param is included, post gres calibration system will be skipped'    
+    echo '  -isaws           : If this param is included, the code will use AWS objects where possible'
+    echo '                   : Note: This feature is not yet implemented'
+    echo
+    exit
+}
+
+set -e
+
+while [ "$1" != "" ]; do
+case $1
+in
+    -u|--hucList)
+        shift
+        hucList=$1
+        ;;
+    -c|--configFile )
+        shift
+        envFile=$1
+        ;;
+    -n|--runName)
+        shift
+        runName=$1
+        ;;
+    -jh|--jobHucLimit)
+        shift
+        jobHucLimit=$1
+        ;;
+    -jb|--jobBranchLimit)
+        shift
+        jobBranchLimit=$1
+        ;;        
+    -h|--help)
+        shift
+        usage
+        ;;
+    -ud|--unitDenylist)
+        shift
+        deny_unit_list=$1
+        ;;
+    -bd|--branchDenylist)
+        shift
+        deny_branches_list=$1
+        ;;
+    -zd|--branchZeroDenylist)
+        shift
+        deny_branch_zero_list=$1
+        ;;        
+    -o)
+        overwrite=1
+        ;;
+    -skipcal)
+        skipcal=1
+        ;;
+    -isaws)
+        isAWS=1
+        ;;
+    *) ;;
+    esac
+    shift
+done
+
+# print usage if arguments empty
+if [ "$hucList" = "" ]
+then
+    echo "ERROR: Missing -u Huclist argument"
+    usage
+fi
+if [ "$runName" = "" ]
+then
+    echo "ERROR: Missing -n run time name argument"
+    usage
+fi
+
+# TODO
+# upgrade Dockerfile to add this as an env value
+projectDir=/foss_fim
+
+# outputDataDir, srcDir and others come from the Dockerfile
+outputRunDataDir=$outputDataDir/$runName
+
+# default values
+if [ "$envFile" = "" ]; then envFile=/$projectDir/config/params_template.env; fi
+if [ "$jobHucLimit" = "" ]; then jobHucLimit=1; fi
+if [ "$jobBranchLimit" = "" ]; then jobBranchLimit=1; fi
+if [ -z "$overwrite" ]; then overwrite=0; fi
+if [ -z "$skipcal" ]; then skipcal=0; fi
+if [ -z "$isAWS" ]; then isAWS=0; fi
+
+# validate and set defaults for the deny lists
+if [ "$deny_unit_list" = "" ]
+then
+    deny_unit_list=$projectDir/config/deny_gms_unit.lst
+elif [ "${deny_unit_list^^}" != "NONE" ] && [ ! -f "$deny_unit_list" ]
+then
+    # NONE is not case sensitive
+    echo "Error: The -ud <unit deny file> does not exist and is not the word NONE"
+    usage
+fi
+
+# validate and set defaults for the deny lists
+if [ "$deny_branches_list" = "" ]
+then
+    deny_branches_list=$projectDir/config/deny_gms_branches.lst
+elif [ "${deny_branches_list^^}" != "NONE" ] && [ ! -f "$deny_branches_list" ]
+then
+    # NONE is not case sensitive
+    echo "Error: The -bd <branch deny file> does not exist and is not the word NONE"
+    usage
+fi
+
+# We do a 1st cleanup of branch zero using branchZeroDenylist (which might be none).
+# Later we do a 2nd cleanup of the branch zero that make the final output of branch zero
+# to match what all other branch folders have for remaining files. But.. if we override
+# branchZeroDenylist, we don't want it to be cleaned a second time.
+has_deny_branch_zero_override=0
+if [ "$deny_branch_zero_list" = "" ]
+then
+    deny_branch_zero_list=$projectDir/config/deny_gms_branch_zero.lst
+elif [ "${deny_branch_zero_list^^}" != "NONE" ]   # NONE is not case sensitive
+then
+    if [ ! -f "$deny_branch_zero_list" ]
+    then
+        echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
+        usage
+    else
+        # only if the deny branch zero has been overwritten and file exists
+        has_deny_branch_zero_override=1 
+    fi
+else
+    has_deny_branch_zero_override=1 # it is the value of NONE and is overridden
+fi
+
+# Safety feature to avoid accidentaly overwrites
+if [ -d $outputRunDataDir ] && [ $overwrite -eq 0 ]; then
+    echo
+    echo "ERROR: Output dir $outputRunDataDir exists. Use overwrite -o to run."
+    echo        
+    usage
+fi
+
+## SOURCE ENV FILE AND FUNCTIONS ##
+source $srcDir/bash_functions.env
+
+# these export are for fim_pipeline only.
+export runName=$runName
+export jobHucLimit=$jobHucLimit
+
+num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
+echo
+echo "--- Number of HUCs to process is $num_hucs"
+
+# make dirs
+if [ ! -d $outputRunDataDir ]; then
+    mkdir -p $outputRunDataDir
+else
+    # remove these directories and files on a new or overwrite run
+    rm -rdf $outputRunDataDir/logs
+    rm -rdf $outputRunDataDir/branch_errors
+    rm -rdf $outputRunDataDir/unit_errors
+    rm -rdf $outputRunDataDir/eval
+    rm -f $outputRunDataDir/crosswalk_table.csv
+    rm -f $outputRunDataDir/gms_inputs*
+    rm -f $outputRunDataDir/*.env
+fi
+
+#logFile=$outputRunDataDir/logs/unit/summary_unit.log
+mkdir -p $outputRunDataDir/logs/unit
+mkdir -p $outputRunDataDir/logs/branch
+mkdir -p $outputRunDataDir/unit_errors
+mkdir -p $outputRunDataDir/branch_errors
+
+# copy over config file and rename it (note.. yes, the envFile file can still be
+# loaded from command line and have its own values, it simply gets renamed and saved)
+cp $envFile $outputRunDataDir/params.env
+
+# create an new .env file on the fly that contains all runtime values
+# that any unit can load it independently (in seperate AWS objects, AWS fargates)
+# or via pipeline. There is likely a more elegent way to do this.
+
+args_file=$outputRunDataDir/runtime_args.env
+
+# the jobHucLimit is not from the args files, only jobBranchLimit
+echo "export runName=$runName" >> $args_file
+echo "export jobHucLimit=$jobHucLimit" >> $args_file
+echo "export jobBranchLimit=$jobBranchLimit" >> $args_file
+echo "export deny_unit_list=$deny_unit_list" >> $args_file
+echo "export deny_branches_list=$deny_branches_list" >> $args_file
+echo "export deny_branch_zero_list=$deny_branch_zero_list" >> $args_file
+echo "export has_deny_branch_zero_override=$has_deny_branch_zero_override" >> $args_file
+echo "export isAWS=$isAWS" >> $args_file
+echo "export skipcal=$skipcal" >> $args_file
+
+echo "--- Pre-processing is complete"
+
+echo
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
new file mode 100755
index 000000000..0f3bb8b69
--- /dev/null
+++ b/fim_process_unit_wb.sh
@@ -0,0 +1,129 @@
+#!/bin/bash -e
+
+# Why is this file here and it appears to be using duplicate export variables?
+# For AWS, we need to make a direct call to this files with two params, hucNumber first, 
+# then the runName same as the -n flag in fim_pipeline and fim_pre_processing.sh
+
+# This file will also catch any and all errors from src/run_unit_wb.sh file, even script aborts from that file
+
+# You really can not call directly to src/run_unit_wb.sh as that file relys on export values
+# from this file.
+# run_unit_wb will futher process branches with its own iterator (parallelization).
+
+# Sample Usage: /foss_fim/fim_process_unit_wb.sh rob_test_wb_1 05030104
+
+## START MESSAGE ##
+
+echo
+
+usage ()
+{
+    echo
+    echo 'Produce FIM hydrofabric datasets for a single unit and branch scale.'
+    echo 'NOTE: fim_pre_processing must have been already run and this tool'
+    echo '      will not include post processing. Only single independent single'
+    echo '      huc and its branches.'    
+    echo 'Usage : There are no arg keys (aka.. no dashes)'
+    echo '        you need the run name first, then the huc.'
+    echo '        Arguments:'
+    echo '           1) run name'
+    echo '           2) HUC number'
+    echo '        Example:'
+    echo '           /foss_fim/fim_process_unit_wb.sh rob_test_1 05030104'
+    echo
+    exit
+}
+
+export runName=$1
+export hucNumber=$2
+
+# print usage if arguments empty
+if [ "$runName" = "" ]
+then
+    echo "ERROR: Missing run time name argument (1st argument)"
+    usage
+fi
+
+if [ "$hucNumber" = "" ]
+then
+    echo "ERROR: Missing hucNumber argument (2nd argument)"
+    usage
+fi
+
+re='^[0-9]+$'
+if ! [[ $hucNumber =~ $re ]] ; then
+   echo "Error: hucNumber is not a number" >&2; exit 1
+   usage
+fi
+
+echo "=========================================================================="
+echo "---- Start of huc processing for $hucNumber"
+
+# outputDataDir, srcDir and others come from the Dockerfile
+
+export outputRunDataDir=/$outputDataDir/$runName
+export outputHucDataDir=$outputRunDataDir/$hucNumber
+export outputBranchDataDir=$outputHucDataDir/branches
+export current_branch_id=0
+
+## huc data
+if [ -d "$outputHucDataDir" ]; then
+    rm -rf $outputHucDataDir
+fi
+
+# make outputs directory
+mkdir -p $outputHucDataDir
+mkdir -p $outputBranchDataDir
+
+# Clean out previous unit logs and branch logs starting with this huc
+rm -f $outputRunDataDir/logs/unit/"$hucNumber"_unit.log
+rm -f $outputRunDataDir/logs/branch/"$hucNumber"_summary_branch.log
+rm -f $outputRunDataDir/logs/branch/"$hucNumber"*.log
+rm -f $outputRunDataDir/unit_errors/"$hucNumber"*.log
+rm -f $outputRunDataDir/branch_errors/"$hucNumber"*.log
+hucLogFileName=$outputRunDataDir/logs/unit/"$hucNumber"_unit.log
+
+# Process the actual huc
+/usr/bin/time -v $srcDir/run_unit_wb.sh 2>&1 | tee $hucLogFileName
+
+#exit ${PIPESTATUS[0]} (and yes.. there can be more than one)
+# and yes.. we can not use the $? as we are messing with exit codes
+return_codes=( "${PIPESTATUS[@]}" )
+
+#echo "huc return codes are:"
+#echo $return_codes
+
+# we do this way instead of working directly with stderr and stdout
+# as they were messing with output logs which we always want.
+err_exists=0
+for code in "${return_codes[@]}"
+do
+    # Make an extra copy of the unit log into a new folder.
+
+    # Note: It was tricky to load in the fim_enum into bash, so we will just 
+    # go with the code for now
+    if [ $code -eq 0 ]; then
+        echo
+        # do nothing
+    elif [ $code -eq 60 ]; then
+        echo
+        echo "***** Unit has no valid branches *****"
+        err_exists=1
+    elif [ $code -eq 61 ]; then
+        echo
+        echo "***** Unit has no remaining valid flowlines *****"   
+        err_exists=1        
+    else
+        echo
+        echo "***** An error has occured  *****"
+        err_exists=1        
+    fi
+done
+
+if [ "$err_exists" = "1" ]; then
+    # copy the error log over to the unit_errors folder to better isolate it
+    cp $hucLogFileName $outputRunDataDir/unit_errors
+fi
+echo "=========================================================================="
+# we always return a success at this point (so we don't stop the loops / iterator)
+exit 0
diff --git a/gms_pipeline.sh b/gms_pipeline.sh
index 2907ce04e..26c7694f1 100755
--- a/gms_pipeline.sh
+++ b/gms_pipeline.sh
@@ -20,14 +20,14 @@ usage ()
     echo '  -c/--config     : configuration file with bash environment variables to export'
     echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
     echo '  -ud/--unitDenylist : A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_gms_unit_prod.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_prod.lst'
+    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -bd/--branchDenylist : A file with a line delimited list of files in BRANCHES directories to be removed' 
     echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_gms_branches_prod.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'   
+    echo '                    (see config/deny_gms_branches.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
@@ -146,7 +146,7 @@ source $srcDir/bash_functions.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
-    jobLimit=$default_max_jobs
+    jobLimit=1
 fi
 
 export outputRunDataDir=$outputDataDir/$runName
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
index 928a381da..9c38437d8 100755
--- a/gms_run_branch.sh
+++ b/gms_run_branch.sh
@@ -19,8 +19,8 @@ usage ()
     echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
     echo '  -bd/--branchDenylist : A file with a line delimited list of files in BRANCHES directories to be removed' 
     echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_gms_branches_prod.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches_prod.lst'   
+    echo '                    (see config/deny_gms_branches.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
@@ -89,7 +89,7 @@ fi
 
 if [ "$deny_branches_list" = "" ]
 then
-   deny_branches_list=/foss_fim/config/deny_gms_branches_prod.lst
+   deny_branches_list=/foss_fim/config/deny_gms_branches.lst
 elif [ "${deny_branches_list^^}" != "NONE" ] && [ ! -f "$deny_branches_list" ]
 then
     # NONE is not case sensitive
@@ -130,7 +130,7 @@ source $srcDir/bash_variables.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
-    jobLimit=$default_max_jobs
+    jobLimit=1
 fi
 
 ## Define Outputs Data Dir & Log File##
diff --git a/gms_run_post_processing.sh b/gms_run_post_processing.sh
index 7d2fa95bd..e83404dca 100755
--- a/gms_run_post_processing.sh
+++ b/gms_run_post_processing.sh
@@ -62,7 +62,7 @@ source $srcDir/bash_variables.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
-    jobLimit=$default_max_jobs
+    jobLimit=1
 fi
 
 ## Define Outputs Data Dir & Log File##
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
index e8adf3e2e..dd4b46d08 100755
--- a/gms_run_unit.sh
+++ b/gms_run_unit.sh
@@ -18,8 +18,8 @@ usage ()
     echo '  -c/--config     : configuration file with bash environment variables to export'
     echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'        
     echo '  -ud/--unitDenylist : A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_gms_unit_prod.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit_prod.lst'
+    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
@@ -93,7 +93,7 @@ fi
 
 if [ "$deny_unit_list" = "" ]
 then
-    deny_unit_list=/foss_fim/config/deny_gms_unit_prod.lst
+    deny_unit_list=/foss_fim/config/deny_gms_unit.lst
 elif [ "${deny_unit_list^^}" != "NONE" ] && [ ! -f "$deny_unit_list" ]
 then
     # NONE is not case sensitive
@@ -125,7 +125,7 @@ source $srcDir/bash_variables.env
 
 # default values
 if [ "$jobLimit" = "" ] ; then
-    jobLimit=$default_max_jobs
+    jobLimit=1
 fi
 
 ## Define Outputs Data Dir & Log File##
diff --git a/src/aggregate_branch_lists.py b/src/aggregate_branch_lists.py
new file mode 100755
index 000000000..9cff6bb13
--- /dev/null
+++ b/src/aggregate_branch_lists.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import argparse 
+import pandas as pd
+import sys
+
+from os.path import join
+from glob import glob
+
+def aggregate_branch_lists(output_dir, file_name, output_file_name):
+
+    file_names = glob(join(output_dir, '*', file_name))
+    
+    if (len(file_names) == 0):
+        print ("Error: No Branches available to aggregate. Program terminated.", flush=True)
+        sys.exit(1)
+
+    df_combined = pd.concat([pd.read_csv(f, header=None, dtype ='str') for f in file_names ], ignore_index=True) 
+   
+    df_combined.to_csv(output_file_name, index=False, header=False)
+
+if __name__ == '__main__':
+
+    # This tool takes in a single file name and searchs all directories
+    # recusively for the same file name and merges them.
+    
+    parser = argparse.ArgumentParser(description='Aggregate')
+    parser.add_argument('-d','--output_dir', help='output run data directory', required=True)
+    parser.add_argument('-f','--file_name', help='file name to match', required=True)    
+    parser.add_argument('-o','--output_file_name', help='output file name', required=True)
+
+    args = vars(parser.parse_args())
+
+    aggregate_branch_lists(**args)
diff --git a/src/bash_functions.env b/src/bash_functions.env
index f5975995a..11ed8ea46 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -15,7 +15,7 @@ Tcount () {
     #echo "Time  = `expr $t2 \- $t1`sec"
     #echo "Cumulative_Time = `expr $t2 \- $t0`sec"
 
-    local total_sec=$(expr $t2 - $t1)
+    local total_sec=$(( $t2 - $t1))
     local dur_min=$((total_sec / 60))
     local dur_remainder_sec=$((total_sec % 60))
     echo "Cumulative Time = $dur_min min(s) and $dur_remainder_sec sec"
@@ -25,7 +25,7 @@ Calc_Duration() {
     start_time=$1
     end_time=`date +%s`
 
-    total_sec=$(expr $end_time - $start_time)
+    total_sec=$(( $end_time - $start_time ))
     dur_min=$((total_sec / 60))
     dur_remainder_sec=$((total_sec % 60))
     echo "Duration = $dur_min min(s) and $dur_remainder_sec sec"
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 6dfde858d..336ed61e3 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -11,3 +11,7 @@ export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
 export input_nwm_headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
 export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
 export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
+
+# Styling
+export startDiv="\n-----------------------------------------------------------------\n"
+export stopDiv="\n-----------------------------------------------------------------\n"
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index c5839e6e3..f44725180 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 
-from logging import exception
+
 import os
 import argparse
-from glob import glob
 import string
+import pathlib
+from glob import glob
+from logging import exception
 
 def __read_included_files(parent_dir_path):
 
@@ -25,6 +27,12 @@ def __read_input_hucs(hucs):
 
     huc_list = set()
     if os.path.isfile(hucs[0]):
+        
+        source_file_extension = pathlib.Path(hucs[0]).suffix   
+       
+        if (source_file_extension.lower() != ".lst" ):
+            raise Exception("Incoming file must be in .lst format if submitting a file name and path.")
+        
         with open(hucs[0],'r') as hucs_file:
             file_lines = hucs_file.readlines()
             f_list = [__clean_huc_value(fl) for fl in file_lines]
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index ec660ea6c..f311fc196 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -33,16 +33,19 @@ def subset_vector_layers(subset_nwm_lakes,
         
     hucUnitLength = len(str(hucCode))
 
+    print("Getting Cell Size", flush=True)
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
 
     # Erase area outside 3DEP domain
+    print("Erase area outside 3DEP domain", flush=True)
     wbd = gpd.read_file(wbd_filename)
     dem_domain = gpd.read_file(dem_domain)
     wbd = gpd.clip(wbd, dem_domain)
     wbd.to_file(wbd_filename, layer='WBDHU8', crs=DEFAULT_FIM_PROJECTION_CRS)
 
     # Get wbd buffer
+    print("Create wbd buffer", flush=True)
     wbd_buffer = wbd.copy()
     wbd_buffer.geometry = wbd_buffer.geometry.buffer(wbd_buffer_distance, resolution=32)
     wbd_buffer = gpd.clip(wbd_buffer, dem_domain)
@@ -58,6 +61,7 @@ def subset_vector_layers(subset_nwm_lakes,
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea, mask=wbd_buffer)
     if not landsea.empty:
+        print("Create landsea gpkg", flush=True)
         landsea.to_file(subset_landsea, driver = getDriver(subset_landsea), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del landsea
 
@@ -70,7 +74,7 @@ def subset_vector_layers(subset_nwm_lakes,
     del levee_protected_areas
 
     # Find intersecting lakes and writeout
-    print("Subsetting NWM Lakes for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    print("Subsetting NWM Lakes", flush=True)
     nwm_lakes = gpd.read_file(nwm_lakes, mask = wbd_buffer)
     nwm_lakes = nwm_lakes.loc[nwm_lakes.Shape_Area < 18990454000.0]
 
@@ -85,14 +89,14 @@ def subset_vector_layers(subset_nwm_lakes,
     del nwm_lakes
 
     # Find intersecting levee lines
-    print("Subsetting NLD levee lines for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    print("Subsetting NLD levee lines", flush=True)
     nld_lines = gpd.read_file(nld_lines, mask = wbd_buffer)
     if not nld_lines.empty:
         nld_lines.to_file(subset_nld_lines, driver = getDriver(subset_nld_lines), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del nld_lines
 
     # Subset NWM headwaters
-    print("Subsetting NWM Headwater Points for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    print("Subsetting NWM Headwater Points", flush=True)
     nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer)
 
     if len(nwm_headwaters) > 0:
@@ -103,7 +107,7 @@ def subset_vector_layers(subset_nwm_lakes,
     del nwm_headwaters
 
     # Find intersecting nwm_catchments
-    print("Subsetting NWM Catchments for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    print("Subsetting NWM Catchments", flush=True)
     nwm_catchments = gpd.read_file(nwm_catchments, mask=wbd_buffer)
 
     if len(nwm_catchments) > 0:
@@ -114,7 +118,7 @@ def subset_vector_layers(subset_nwm_lakes,
     del nwm_catchments
 
     # Subset nwm streams
-    print("Subsetting NWM Streams for HUC{} {}".format(hucUnitLength, hucCode), flush=True)
+    print("Subsetting NWM Streams", flush=True)
 
     nwm_streams = gpd.read_file(nwm_streams, mask = wbd)
 
diff --git a/src/generate_branch_list.py b/src/generate_branch_list.py
new file mode 100755
index 000000000..6e1240de4
--- /dev/null
+++ b/src/generate_branch_list.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import pandas as pd
+import sys
+
+sys.path.append('/foss_fim/src/gms/')
+from stream_branches import StreamNetwork
+
+def generate_branch_list(stream_network_dissolved, branch_id_attribute,
+                         output_branch_list_file):
+
+    '''
+    Processing:
+        This create a branch_ids.lst file which is required at the very start of processing
+        hucs.  This becomes the list that run_unit_wb.sh needs to iterate over branches
+        
+        Note: The .csv twin to this is appended to each time a branch completes, 
+        resulting in a list that only contains successfully processed branches.
+    Params:
+        - stream_network_dissolved (str): the gkpg that contains the list of disolved branch ids
+        - branch_id_attribute (str): the id of the field in the gkpg that has the branch ids.
+            (ie. like levpa_id (from params_template.env) )
+        - output_branch_list_file (str): file name and path of the list to be created.
+    Output:
+        - create a file (likely a .lst file) with branch ids (not including branch zero)
+    '''
+    
+    if os.path.exists(stream_network_dissolved):
+        # load stream network
+        stream_network_dissolved = StreamNetwork.from_file( stream_network_dissolved,
+                                                            branch_id_attribute=branch_id_attribute )
+        # reduce to branch id attribute and convert to pandas df
+        stream_network_dissolved = stream_network_dissolved.loc[:,branch_id_attribute]
+        
+        # write out the list version (just branch numbers)
+        stream_network_dissolved.to_csv(output_branch_list_file, sep= " ", index=False, header=False)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Create branch list')
+    parser.add_argument('-d','--stream-network-dissolved', help='Dissolved stream network', required=True)
+    parser.add_argument('-b','--branch-id-attribute', help='Branch ID attribute to use in dissolved stream network', required=True)
+    parser.add_argument('-o','--output-branch-list-file', help='Output branch list', required=True)    
+    
+    args = vars(parser.parse_args())
+
+    generate_branch_list(**args)
diff --git a/src/generate_branch_list_csv.py b/src/generate_branch_list_csv.py
new file mode 100755
index 000000000..e355e9f25
--- /dev/null
+++ b/src/generate_branch_list_csv.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import pandas as pd
+import pathlib
+
+def generate_branch_list_csv(huc_id, branch_id, output_branch_csv):
+
+    '''
+    Processing:
+        This create a branch_ids.csv file which is required for various post processing tasks.
+        If the csv already, then the new huc, branch id wil be appended.
+        If it does not yet exist, a new csv will be created
+        
+    Params:
+        - huc_id
+        - branch_id
+        - output_branch_csv (str): csv file name and path of the list to be created. (likely branch_list.csv)
+        
+    Output:
+        - create a csv file (assuming the format coming in is a csv
+    '''
+    # validations
+    file_extension = pathlib.Path(output_branch_csv).suffix
+   
+    if (file_extension != ".csv"):
+        raise ValueError("The output branch csv file does not have a .csv extension")
+    
+    if (len(huc_id) != 8) or (not huc_id.isnumeric()):
+        raise ValueError("The huc_id does not appear to be an eight digit number")
+    
+    if (not branch_id.isnumeric()):
+        raise ValueError("The branch_id does not appear to be a valid number")
+    
+    df_csv = None
+    new_data = [[huc_id, branch_id]]
+    col_names = ["huc_id","branch_id"]
+    df_csv = pd.DataFrame(new_data, columns=col_names)    
+
+    if (not os.path.exists(output_branch_csv)):
+        df_csv.to_csv(output_branch_csv, index=False, header=False)
+    else:
+        df_csv.to_csv(output_branch_csv, mode='a', index=False, header=False)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Create branch list')
+    parser.add_argument('-b','--branch-id', help='Branch ID', required=True)
+    parser.add_argument('-o','--output-branch-csv', help='Output branch csv list', required=True)
+    parser.add_argument('-u','--huc-id', help='HUC number being aggregated', required=True)
+    args = vars(parser.parse_args())
+
+    generate_branch_list_csv(**args)
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/gms/delineate_hydros_and_produce_HAND.sh
index 8e6d63971..c27de85cd 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/gms/delineate_hydros_and_produce_HAND.sh
@@ -8,7 +8,7 @@ T_total_start
 
 ## MASK LEVEE-PROTECTED AREAS FROM DEM ##
 if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
-    echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"$stopDiv
+    echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
     date -u
     Tstart
     python3 -m memory_profiler $srcDir/gms/mask_dem.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif  -s $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -i $current_branch_id -b0 $branch_zero_id
@@ -16,105 +16,105 @@ if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtec
 fi
 
 ## D8 FLOW ACCUMULATIONS ##
-echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"
 date -u
 Tstart
 $taudemDir/aread8 -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -wg $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif -nc
 Tcount
 
 # THRESHOLD ACCUMULATIONS ##
-echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"
 date -u
 Tstart
 $taudemDir/threshold -ssa $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -thresh 1
 Tcount
 
 ## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
-echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
 python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif -g $outputCurrentBranchDataDir/temp_grass
 Tcount
 
 ## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
-echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
 python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -a $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif -d $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif -t 50 -o $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -th $thalweg_lateral_elev_threshold
 Tcount
 
 ## MASK BURNED DEM FOR STREAMS ONLY ###
-echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"
 date -u
 Tstart
 gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -B $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif --calc="A/B" --outfile="$outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" --NoDataValue=0
 Tcount
 
 ## FLOW CONDITION STREAMS ##
-echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"
 date -u
 Tstart
 $taudemDir/flowdircond -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif -z $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -zfdc $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
 Tcount
 
 ## D8 SLOPES ##
-echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"
 date -u
 Tstart
 mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
 Tcount
 
 ## STREAMNET FOR REACHES ##
-echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
 $taudemDir/streamnet -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -fel $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -ord $outputCurrentBranchDataDir/streamOrder_$current_branch_id.tif -tree $outputCurrentBranchDataDir/treeFile_$current_branch_id.txt -coord $outputCurrentBranchDataDir/coordFile_$current_branch_id.txt -w $outputCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif -net $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
 Tcount
 
 ## SPLIT DERIVED REACHES ##
-echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
 $srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
-echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
 mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -gw $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
 Tcount
 
 ## VECTORIZE FEATURE ID CENTROIDS ##
-echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"
 date -u
 Tstart
 $srcDir/reachID_grid_to_vector_points.py -r $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -i featureID -p $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
 Tcount
 
 ## GAGE WATERSHED FOR PIXELS ##
-echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"
 date -u
 Tstart
 mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif -gw $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
 Tcount
 
 # D8 REM ##
-echo -e $startDiv"D8 REM $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"D8 REM $hucNumber $current_branch_id"
 date -u
 Tstart
 $srcDir/gms/make_rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
 Tcount
 
 ## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
-echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"
 date -u
 gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
 Tcount
 
 ## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
 if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
-    echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"$stopDiv
+    echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"
     date -u
     Tstart
     gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
@@ -122,34 +122,34 @@ if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
 fi
 
 ## POLYGONIZE REACH WATERSHEDS ##
-echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"
 date -u
 Tstart
 gdal_polygonize.py -8 -f GPKG $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
 Tcount
 
 ## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
-echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"
 date -u
 Tstart
 python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
 Tcount
 
 ## RASTERIZE NEW CATCHMENTS AGAIN ##
-echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"
 date -u
 Tstart
 gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
 Tcount
 
 ## MASK SLOPE TO CATCHMENTS ##
-echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"
 date -u
 gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif --calc="A*(B>0)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
 Tcount
 
 ## MAKE CATCHMENT AND STAGE FILES ##
-echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"
 date -u
 Tstart
 $srcDir/make_stages_and_catchlist.py -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/stage_$current_branch_id.txt -a $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
@@ -157,7 +157,7 @@ Tcount
 
 ## MASK REM RASTER TO REMOVE OCEAN AREAS ##
 if  [ -f $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
-    echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"$stopDiv
+    echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"
     date -u
     Tstart
     gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
@@ -165,14 +165,14 @@ if  [ -f $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; th
 fi
 
 ## HYDRAULIC PROPERTIES ##
-echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"
 date -u
 Tstart
 $taudemDir/catchhydrogeo -hand $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -catch $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif -catchlist $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -slp $outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif -h $outputCurrentBranchDataDir/stage_$current_branch_id.txt -table $outputCurrentBranchDataDir/src_base_$current_branch_id.csv
 Tcount
 
 ## FINALIZE CATCHMENTS AND MODEL STREAMS ##
-echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"
 date -u
 Tstart
 if [ "$level" = "branch" ]; then
diff --git a/src/gms/run_by_branch.sh b/src/gms/run_by_branch.sh
index 515bb60f9..2c141d282 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/gms/run_by_branch.sh
@@ -23,23 +23,19 @@ outputBranchDataDir=$outputHucDataDir/branches
 outputCurrentBranchDataDir=$outputBranchDataDir/$current_branch_id
 
 ## OVERWRITE
-if [ -d "$outputCurrentBranchDataDir" ];then
-    if [ $overwrite -eq 1 ]; then
-        rm -rf $outputCurrentBranchDataDir
-    else
-        echo "GMS branch data directories for $hucNumber - $current_branch_id already exist. Use -o/--overwrite to continue"
-        exit 1
-    fi
+if [ -d "$outputCurrentBranchDataDir" ]; then
+    rm -rf $outputCurrentBranchDataDir
 fi
 
 ## MAKE OUTPUT BRANCH DIRECTORY
 mkdir -p $outputCurrentBranchDataDir
 
 ## START MESSAGE ##
-echo -e $startDiv"Processing branch_id: $current_branch_id in HUC: $hucNumber ..."$stopDiv
+echo -e $startDiv"Processing HUC: $hucNumber - branch_id: $current_branch_id"
+echo
 
 ## SUBSET VECTORS
-echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"
 date -u
 Tstart
 echo -e "Querying NWM streams ..."
@@ -53,28 +49,28 @@ ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="
 Tcount
 
 ## GET RASTERS FROM BRANCH ZERO AND CLIP TO CURRENT BRANCH BUFFER ##
-echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"
 date -u
 Tstart
 $srcDir/gms/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $outputBranchDataDir/$branch_zero_id/dem_meters_$branch_zero_id.tif $outputBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $outputCurrentBranchDataDir/dem_meters.tif $outputCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
 Tcount
 
 ## GET RASTER METADATA
-echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"
 date -u
 Tstart
 read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
-echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"
 date -u
 Tstart
 gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $outputCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
 Tcount
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
-echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $current_branch_id"$stopDiv
+echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $current_branch_id"
 date -u
 Tstart
 gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif
@@ -96,7 +92,7 @@ $srcDir/gms/delineate_hydros_and_produce_HAND.sh "branch"
 
 ## USGS CROSSWALK ##
 if [ -f $outputHucDataDir/usgs_subset_gages.gpkg ]; then
-    echo -e $startDiv"USGS Crosswalk $hucNumber $current_branch_id"$stopDiv
+    echo -e $startDiv"USGS Crosswalk $hucNumber $current_branch_id"
     date -u
     Tstart
     python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $current_branch_id
@@ -105,12 +101,12 @@ fi
 
 ## REMOVE FILES FROM DENY LIST ##
 if [ -f $deny_branches_list ]; then
-    echo -e $startDiv"Remove files $hucNumber $current_branch_id"$stopDiv
+    echo -e $startDiv"Remove files $hucNumber $current_branch_id"
     date -u
     Tstart
     $srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branches_list -b $current_branch_id
     Tcount
 fi
 
-echo -e $startDiv"End Processing $hucNumber $current_branch_id ..."
+echo -e $startDiv"End Branch Processing $hucNumber $current_branch_id ..."
 echo
\ No newline at end of file
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index 4bd5d7c14..585e6ae74 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -190,7 +190,7 @@ def run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_opti
     procs_list = []
 
     ## Print message to user and initiate run clock
-    print('Writing progress to log file here: ' + str(join(fim_dir,'bankfull_detect.log')))
+    print('Writing progress to log file here: /logs/log_bankfull_indentify.log')
     print('This may take a few minutes...')
 
     ## Check that the input fim_dir exists
diff --git a/src/process_branch.sh b/src/process_branch.sh
new file mode 100755
index 000000000..627cfa182
--- /dev/null
+++ b/src/process_branch.sh
@@ -0,0 +1,57 @@
+#!/bin/bash -e
+
+# it is strongly recommended that you do not call directly to src/gms/run_by_branch.sh
+# but call this file and let is call run_by_branch.
+# This file will auto trap any exceptions from run_by_branch.
+
+# also.. remember.. that this file will rarely need to be called (but can be)
+# as it is usually called through a parallelizing iterator in run_unit_wb.sh
+
+# this also has no named command line arguments, onlly positional args.
+
+runName=$1
+hucNumber=$2
+branchId=$3
+
+# outputDataDir, srcDir and others come from the Dockerfile
+export outputRunDataDir=$outputDataDir/$runName
+branchLogFileName=$outputRunDataDir/logs/branch/"$hucNumber"_branch_"$branchId".log
+branch_list_csv_file=$outputRunDataDir/$hucNumber/branch_ids.csv
+
+/usr/bin/time -v $srcDir/gms/run_by_branch.sh $hucNumber $branchId 2>&1 | tee $branchLogFileName 
+
+#exit ${PIPESTATUS[0]}
+return_codes=( "${PIPESTATUS[@]}" )
+
+# we do this way instead of working directly with stderr and stdout
+# as they were messing with output logs which we always want.
+err_exists=0
+for code in "${return_codes[@]}"
+do
+    # Make an extra copy of the branch log in a new folder
+    # Note: It was tricky to load in the fim_enum into bash, so we will just 
+    # go with the code for now
+    if [ $code -eq 0 ]; then
+        echo
+        # do nothing
+    elif [ $code -eq 61 ]; then
+        echo
+        err_exists=1
+        echo "***** Branch has no valid flowlines *****"
+    elif [ $code -ne 0 ]; then
+        echo
+        err_exists=1
+        echo "***** An error has occured  *****"
+        cp $branchLogFileName $outputRunDataDir/branch_errors
+    fi
+done
+
+# Note: For branches, we do not copy over the log file for codes of 60 and 61.
+
+if [ "$err_exists" = "0" ]; then
+    # Only add the huc and branch number to the csv is the branch was successful at processing
+    # We also don't want to include 60's and 61's
+    $srcDir/generate_branch_list_csv.py -o $branch_list_csv_file -u $hucNumber -b $branchId
+fi
+
+exit 0  # we always return a success at this point (so we don't stop the loops / iterator)
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
new file mode 100755
index 000000000..e1516d9df
--- /dev/null
+++ b/src/run_unit_wb.sh
@@ -0,0 +1,268 @@
+#!/bin/bash -e
+
+# Do not call this file directly. Call fim_process_unit_wb.sh which calls
+# this file.
+
+## SOURCE FILE AND FUNCTIONS ##
+# load the various enviro files
+args_file=$outputRunDataDir/runtime_args.env
+
+source $args_file
+source $outputRunDataDir/params.env
+source $srcDir/bash_functions.env
+source $srcDir/bash_variables.env
+
+branch_list_csv_file=$outputHucDataDir/branch_ids.csv
+branch_list_lst_file=$outputHucDataDir/branch_ids.lst
+
+branchSummaryLogFile=$outputRunDataDir/logs/branch/"$hucNumber"_summary_branch.log
+
+## INITIALIZE TOTAL TIME TIMER ##
+T_total_start
+huc_start_time=`date +%s`
+
+## SET VARIABLES AND FILE INPUTS ##
+hucUnitLength=${#hucNumber}
+huc4Identifier=${hucNumber:0:4}
+huc2Identifier=${hucNumber:0:2}
+input_NHD_WBHD_layer=WBDHU$hucUnitLength
+
+input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
+
+# Define the landsea water body mask using either Great Lakes or Ocean polygon input #
+if [[ $huc2Identifier == "04" ]] ; then
+  input_LANDSEA=$input_GL_boundaries
+  #echo -e "Using $input_LANDSEA for water body mask (Great Lakes)"
+else
+  input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
+fi
+
+## GET WBD ##
+echo -e $startDiv"Get WBD $hucNumber"
+date -u
+Tstart
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
+Tcount
+
+## Subset Vector Layers ##
+echo -e $startDiv"Get Vector Layers and Subset $hucNumber"
+date -u
+Tstart
+
+cmd_args=" -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg"
+cmd_args+=" -b $outputHucDataDir/nwm_subset_streams.gpkg"
+cmd_args+=" -d $hucNumber"
+cmd_args+=" -e $outputHucDataDir/nwm_headwater_points_subset.gpkg"
+cmd_args+=" -f $outputHucDataDir/wbd_buffered.gpkg"
+cmd_args+=" -g $outputHucDataDir/wbd.gpkg"
+cmd_args+=" -i $input_DEM"
+cmd_args+=" -j $input_DEM_domain"
+cmd_args+=" -l $input_nwm_lakes"
+cmd_args+=" -m $input_nwm_catchments"
+cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"
+cmd_args+=" -r $input_NLD"
+cmd_args+=" -v $input_LANDSEA"
+cmd_args+=" -w $input_nwm_flows"
+cmd_args+=" -x $outputHucDataDir/LandSea_subset.gpkg"
+cmd_args+=" -y $input_nwm_headwaters"
+cmd_args+=" -z $outputHucDataDir/nld_subset_levees.gpkg"
+cmd_args+=" -wb $wbd_buffer"
+cmd_args+=" -lpf $input_nld_levee_protected_areas"
+cmd_args+=" -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg"
+
+#echo "$cmd_args"
+python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
+Tcount
+
+## Clip WBD8 ##
+echo -e $startDiv"Clip WBD8"
+date -u
+Tstart
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+Tcount
+
+## DERIVE LEVELPATH  ##
+echo -e $startDiv"Generating Level Paths for $hucNumber"
+date -u
+Tstart
+$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+
+# test if we received a non-zero code back from derive_level_paths.py
+subscript_exit_code=$?
+# we have to retrow it if it is not a zero (but it will stop further execution in this script)
+if [ $subscript_exit_code -ne 0 ]; then exit $subscript_exit_code; fi
+Tcount
+
+## STREAM BRANCH POLYGONS
+echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
+date -u
+Tstart
+$srcDir/gms/buffer_stream_branches.py -a $input_DEM_domain -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg
+Tcount
+
+## CREATE BRANCHID LIST FILE
+echo -e $startDiv"Create list file of branch ids for $hucNumber"
+date -u
+Tstart
+$srcDir/generate_branch_list.py -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -o $branch_list_lst_file
+Tcount
+
+## CREATE BRANCH ZERO ##
+echo -e $startDiv"Creating branch zero for $hucNumber"
+outputCurrentBranchDataDir=$outputBranchDataDir/$branch_zero_id
+
+## MAKE OUTPUT BRANCH DIRECTORY
+mkdir -p $outputCurrentBranchDataDir
+
+## CLIP RASTERS
+echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"
+# Note: don't need to use gdalwarp -cblend as we are using a buffered wbd
+date -u
+Tstart
+[ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \
+gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+Tcount
+
+## GET RASTER METADATA
+echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"
+date -u
+Tstart
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif)
+
+## RASTERIZE NLD MULTILINES ##
+echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $branch_zero_id"
+date -u
+Tstart
+# REMAINS UNTESTED FOR AREAS WITH LEVEES
+[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
+gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif
+Tcount
+
+## BURN LEVEES INTO DEM ##
+echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
+date -u
+Tstart
+# REMAINS UNTESTED FOR AREAS WITH LEVEES
+[ -f $outputCurrentBranchDataDir/nld_subset_levees.tif ] && \
+python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -nld $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif -out $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+Tcount
+
+## RASTERIZE REACH BOOLEAN (1 & 0) ##
+echo -e $startDiv"Rasterize Reach Boolean $hucNumber $branch_zero_id"
+date -u
+Tstart
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_subset_streams.gpkg $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
+Tcount
+
+## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
+echo -e $startDiv"Rasterize NWM Headwaters $hucNumber $branch_zero_id"
+date -u
+Tstart
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_headwater_points_subset.gpkg $outputCurrentBranchDataDir/headwaters_$branch_zero_id.tif
+Tcount
+
+## DEM Reconditioning ##
+# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
+# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
+echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
+date -u
+Tstart
+python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -w $outputCurrentBranchDataDir -g $outputCurrentBranchDataDir/temp_work -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+Tcount
+
+## PIT REMOVE BURNED DEM ##
+echo -e $startDiv"Pit remove Burned DEM $hucNumber $branch_zero_id"
+date -u
+Tstart
+rd_depression_filling $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
+Tcount
+
+## D8 FLOW DIR ##
+echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $branch_zero_id"
+date -u
+Tstart
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
+Tcount
+
+## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
+export hucNumber=$hucNumber
+export current_branch_id=$current_branch_id
+export outputCurrentBranchDataDir=$outputCurrentBranchDataDir
+export outputHucDataDir=$outputHucDataDir
+export ndv=$ndv
+export xmin=$xmin
+export ymin=$ymin
+export xmax=$xmax
+export ymax=$ymax
+export ncols=$ncols
+export nrows=$nrows
+
+## PRODUCE BRANCH ZERO HAND
+$srcDir/gms/delineate_hydros_and_produce_HAND.sh "unit"
+
+## CREATE USGS GAGES FILE
+if [ -f $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
+    echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"
+    date -u
+    Tstart
+    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id
+    Tcount
+fi
+
+## USGS CROSSWALK ##
+if [ -f $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
+    echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"
+    date -u
+    Tstart
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $branch_zero_id
+    Tcount
+fi
+
+## CLEANUP BRANCH ZERO OUTPUTS ##
+echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"
+$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branch_zero_list -b $branch_zero_id
+
+
+## REMOVE FILES FROM DENY LIST ##
+if [ -f $deny_unit_list ]; then
+    echo -e $startDiv"Remove files $hucNumber"
+    date -u
+    Tstart
+    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
+    Tcount
+fi
+
+# -------------------
+## Start the local csv branch list
+$srcDir/generate_branch_list_csv.py -o $branch_list_csv_file -u $hucNumber -b $branch_zero_id
+
+# -------------------
+## Processing Branches ##
+echo
+echo "---- Start of branch processing for $hucNumber"
+branch_processing_start_time=`date +%s`
+
+parallel --eta --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSummaryLogFile --colsep ',' -- $srcDir/process_branch.sh $runName $hucNumber :::: $branch_list_lst_file
+
+# -------------------
+## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO (but using normal branch deny) ##
+if [ "$has_deny_branch_zero_override" == "1" ]
+then
+    echo -e $startDiv"Second cleanup of files for branch zero (none default)"
+    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_branch_zero_list -b 0
+
+else 
+    echo -e $startDiv"Second cleanup of files for branch zero using the default branch deny list"
+    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_branches_list -b 0
+fi
+
+echo "---- All huc for $hucNumber branches have been now processed"
+Calc_Duration $branch_processing_start_time
+echo
+
+date -u
+echo "---- HUC processing for $hucNumber is complete"
+Calc_Duration $huc_start_time
+echo
+
+
diff --git a/unit_tests/aggregate_branch_lists_params.json b/unit_tests/aggregate_branch_lists_params.json
new file mode 100644
index 000000000..9b9e4bcdd
--- /dev/null
+++ b/unit_tests/aggregate_branch_lists_params.json
@@ -0,0 +1,8 @@
+{
+	"valid_data":
+	{
+		"output_dir": "/outputs/rob_wb_3/",
+		"file_name": "branch_ids.csv",
+		"output_file_name": "/outputs/rob_wb_3/test.csv"
+	}
+}
diff --git a/unit_tests/aggregate_branch_lists_unittests.py b/unit_tests/aggregate_branch_lists_unittests.py
new file mode 100644
index 000000000..72896e87e
--- /dev/null
+++ b/unit_tests/aggregate_branch_lists_unittests.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/') 
+import aggregate_branch_lists as src
+
+class test_aggregate_branch_lists(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+    def test_aggregate_branch_lists_success(self):
+
+        #global params_file
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+
+        src.aggregate_branch_lists(output_dir = params["output_dir"],
+                                   file_name = params["file_name"],
+                                   output_file_name = params["output_file_name"])
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/generate_branch_list_csv_params.json b/unit_tests/generate_branch_list_csv_params.json
new file mode 100644
index 000000000..d627a1356
--- /dev/null
+++ b/unit_tests/generate_branch_list_csv_params.json
@@ -0,0 +1,20 @@
+{
+	"valid_data_add_branch_zero":
+	{
+		"huc_id": "05030104",
+		"branch_id": "0",
+		"output_branch_csv": "/outputs/rob_wb_3/05030104/branch_ids.csv"
+	},
+	"valid_data_add_branch":
+	{
+		"huc_id": "05030104",
+		"branch_id": "1946000003",
+		"output_branch_csv": "/outputs/rob_wb_3/05030104/branch_ids.csv"
+	},	
+	"invalid_bad_file_extension":
+	{
+		"huc_id": "05030104",
+		"branch_id": "1946000003",
+		"output_branch_csv": "/outputs/rob_wb_3/05030104/branch_ids2"
+	}	
+}
diff --git a/unit_tests/generate_branch_list_csv_unittests.py b/unit_tests/generate_branch_list_csv_unittests.py
new file mode 100644
index 000000000..191d2fc01
--- /dev/null
+++ b/unit_tests/generate_branch_list_csv_unittests.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import generate_branch_list_csv as src
+
+class test_generate_branch_list_csv(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+        # for these tests to work, we have to check if the .csv exists and remove it 
+        # prior to exections of the tests.
+        
+        params = self.params["valid_data_add_branch_zero"].copy()
+        if (os.path.exists(params["output_branch_csv"])):
+            os.remove(params["output_branch_csv"])
+        
+
+    def test_generate_branch_list_csv_valid_data_add_branch_zero_success(self):
+
+        # yes.. we know that we can not control the order
+        
+        #global params_file
+        params = self.params["valid_data_add_branch_zero"].copy()
+
+        src.generate_branch_list_csv(huc_id = params["huc_id"],
+                                     branch_id = params["branch_id"],
+                                     output_branch_csv = params["output_branch_csv"])
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_generate_branch_list_csv_valid_data_add_branch_success(self):
+    
+        # yes.. we know that we can not control the order
+        
+        #global params_file
+        params = self.params["valid_data_add_branch"].copy()
+
+        src.generate_branch_list_csv(huc_id = params["huc_id"],
+                                     branch_id = params["branch_id"],
+                                     output_branch_csv = params["output_branch_csv"])
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+    def test_generate_branch_list_csv_invalid_bad_file_extension(self):
+        
+        #global params_file
+        params = self.params["invalid_bad_file_extension"].copy()
+
+        # we expect this to fail. If it does fail with an exception, then this test
+        # is sucessful.
+        try:
+            src.generate_branch_list_csv(huc_id = params["huc_id"],
+                                        branch_id = params["branch_id"],
+                                        output_branch_csv = params["output_branch_csv"])
+       
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+            
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+       
+        finally:
+            print("*************************************************************")             
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/generate_branch_list_params.json b/unit_tests/generate_branch_list_params.json
new file mode 100644
index 000000000..02a89bcea
--- /dev/null
+++ b/unit_tests/generate_branch_list_params.json
@@ -0,0 +1,8 @@
+{
+	"valid_data":
+	{
+		"stream_network_dissolved": "/outputs/rob_wb_3/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
+		"branch_id_attribute": "levpa_id",
+		"output_branch_list_file": "/outputs/rob_wb_3/05030104/branch_ids.lst"
+	}
+}
diff --git a/unit_tests/generate_branch_list_unittests.py b/unit_tests/generate_branch_list_unittests.py
new file mode 100644
index 000000000..fc7ac7776
--- /dev/null
+++ b/unit_tests/generate_branch_list_unittests.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import generate_branch_list as src
+
+class test_Generate_branch_list(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_Generate_branch_list_success(self):
+
+        #global params_file
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+
+        src.generate_branch_list(stream_network_dissolved = params["stream_network_dissolved"],
+                                 branch_id_attribute = params["branch_id_attribute"],
+                                 output_branch_list_file = params["output_branch_list_file"])
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/gms/outputs_cleanup_params.json b/unit_tests/gms/outputs_cleanup_params.json
index 963e676fe..456f03463 100644
--- a/unit_tests/gms/outputs_cleanup_params.json
+++ b/unit_tests/gms/outputs_cleanup_params.json
@@ -2,7 +2,7 @@
 	"valid_specific_branch_data":
 	{
 		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
-		"deny_list": "/foss_fim/config/deny_gms_branches_prod.lst",
+		"deny_list": "/foss_fim/config/deny_gms_branches.lst",
 		"branch_id": "3246000009",
 		"verbose": true
 	},
@@ -10,7 +10,7 @@
 	"valid_directory_data":
 	{
 		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
-		"deny_list": "/foss_fim/config/deny_gms_branches_prod.lst",
+		"deny_list": "/foss_fim/config/deny_gms_branches.lst",
 		"branch_id": "0",
 		"verbose": true
 	},

From 32eab723b918de94c80fb2a30b7bb6a087510752 Mon Sep 17 00:00:00 2001
From: LauraKeys-NOAA <70279180+LauraKeys-NOAA@users.noreply.github.com>
Date: Fri, 17 Feb 2023 10:52:02 -0600
Subject: [PATCH 207/359] v4.1.1.0 Updated 1-foot interval CatFIM with restart
 capabilities (#809)

---
 tools/generate_categorical_fim.py | 172 +++++++++++++++++++-----------
 1 file changed, 111 insertions(+), 61 deletions(-)

diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index dfb25438b..ee0c5ea03 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -130,7 +130,7 @@ def create_csvs(output_mapping_dir, reformatted_catfim_method):
     # Convert any geopackage in the root level of output_mapping_dir to CSV and rename.
     gpkg_list = glob.glob(os.path.join(output_mapping_dir, '*.gpkg'))
     for gpkg in gpkg_list:
-        print(f"creating CSV for {gpkg}")
+        print(f"Creating CSV for {gpkg}")
         gdf = gpd.read_file(gpkg)
         parent_directory = os.path.split(gpkg)[0]
         if 'catfim_library' in gpkg:
@@ -174,32 +174,34 @@ def update_mapping_status(output_mapping_dir, output_flows_dir, nws_sites_layer,
     # Import geopackage output from flows creation
     flows_df = gpd.read_file(nws_sites_layer)
 
-    # Join failed sites to flows df
-    flows_df = flows_df.merge(mapping_df, how = 'left', on = 'nws_lid')
-
-    # Switch mapped column to no for failed sites and update status
-    flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
-    flows_df.loc[flows_df['did_it_map']=='no','status'] = flows_df['status'] + flows_df['map_status']
-    
-#    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
-#    if stage_based:
-#        missing_mapping_hucs = 
-#    else:
-#        flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
-#        mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
-#        missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
-#    
-#    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
-#    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
-#    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
-
-    # Clean up GeoDataFrame and rename columns for consistency
-    flows_df = flows_df.drop(columns = ['did_it_map','map_status'])
-    flows_df = flows_df.rename(columns = {'nws_lid':'ahps_lid'})
-
-    # Write out to file
-    flows_df.to_file(nws_sites_layer)
+    try:
+        # Join failed sites to flows df
+        flows_df = flows_df.merge(mapping_df, how = 'left', on = 'nws_lid')
 
+        # Switch mapped column to no for failed sites and update status
+        flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
+        flows_df.loc[flows_df['did_it_map']=='no','status'] = flows_df['status'] + flows_df['map_status']
+        
+    #    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
+    #    if stage_based:
+    #        missing_mapping_hucs = 
+    #    else:
+    #        flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
+    #        mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
+    #        missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
+    #    
+    #    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
+    #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
+    #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
+
+        # Clean up GeoDataFrame and rename columns for consistency
+        flows_df = flows_df.drop(columns = ['did_it_map','map_status'])
+        flows_df = flows_df.rename(columns = {'nws_lid':'ahps_lid'})
+
+        # Write out to file
+        flows_df.to_file(nws_sites_layer)
+    except:
+        print("No LIDs")
 
 def produce_inundation_map_with_stage_and_feature_ids(rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch):
     
@@ -229,6 +231,10 @@ def produce_inundation_map_with_stage_and_feature_ids(rem_path, catchments_path,
             with rasterio.open(output_tif, 'w', **profile) as dst:
                 dst.write(masked_reclass_rem_array, 1)
     
+def mark_complete(site_directory):
+    marker_file = Path(site_directory) / 'complete.txt'
+    marker_file.touch()
+    return
 
 def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, threshold_url, flood_categories, all_lists, past_major_interval_cap, number_of_jobs, number_of_interval_jobs, attributes_dir):
     missing_huc_files = []
@@ -262,9 +268,14 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
         lid_directory = os.path.join(huc_directory, lid)
         if not os.path.exists(lid_directory):
             os.mkdir(lid_directory)
+        else:
+            complete_marker = os.path.join(lid_directory, 'complete.txt')
+            if os.path.exists(complete_marker):
+                all_messages.append([f"{lid}: already completed in previous run."])
+                continue
         # Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
         stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
-                
+
         if stages == None:
             all_messages.append([f'{lid}:error getting thresholds from WRDS API'])
             continue
@@ -273,20 +284,28 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
             all_messages.append([f'{lid}:missing threshold stages'])
             continue
 
-        # Drop columns that offend acceptance criteria
-        usgs_elev_df['acceptable_codes'] = (usgs_elev_df['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
-                                    & usgs_elev_df['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
-                                    & usgs_elev_df['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
-                                    & usgs_elev_df['usgs_data_site_type'].isin(acceptable_site_type_list))
-        
-        usgs_elev_df = usgs_elev_df.astype({'usgs_data_alt_accuracy_code': float})
-        usgs_elev_df['acceptable_alt_error'] = np.where(usgs_elev_df['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False)
-        
-        acceptable_usgs_elev_df = usgs_elev_df[(usgs_elev_df['acceptable_codes'] == True) & (usgs_elev_df['acceptable_alt_error'] == True)]
-        
+        try:
+            # Drop columns that offend acceptance criteria
+            usgs_elev_df['acceptable_codes'] = (usgs_elev_df['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
+                                        & usgs_elev_df['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
+                                        & usgs_elev_df['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
+                                        & usgs_elev_df['usgs_data_site_type'].isin(acceptable_site_type_list))
+           
+            usgs_elev_df = usgs_elev_df.astype({'usgs_data_alt_accuracy_code': float})
+            usgs_elev_df['acceptable_alt_error'] = np.where(usgs_elev_df['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False)
+            
+            acceptable_usgs_elev_df = usgs_elev_df[(usgs_elev_df['acceptable_codes'] == True) & (usgs_elev_df['acceptable_alt_error'] == True)]
+        except Exception as e:
+            # Not sure any of the sites actually have those USGS-related
+            # columns in this particular file, so just assume it's fine to use
+
+            #print("(Various columns related to USGS probably not in this csv)")
+            acceptable_usgs_elev_df = usgs_elev_df
+
         # Get the dem_adj_elevation value from usgs_elev_table.csv. Prioritize the value that is not from branch 0.
         try:
             matching_rows = acceptable_usgs_elev_df.loc[acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation']
+
             if len(matching_rows) == 2:  # It means there are two level paths, use the one that is not 0
                 lid_usgs_elev = acceptable_usgs_elev_df.loc[(acceptable_usgs_elev_df['nws_lid'] == lid.upper()) & ('levpa_id' != 0), 'dem_adj_elevation'].values[0]
             else:
@@ -301,6 +320,32 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
         metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
         lid_altitude = metadata['usgs_data']['altitude']
    
+        # Filter out sites that don't have "good" data
+        try:
+            if not metadata['usgs_data']['coord_accuracy_code'] in \
+                    acceptable_coord_acc_code_list:
+                print(f"\t{lid}: {metadata['usgs_data']['coord_accuracy_code']} Not in acceptable coord acc codes")
+                continue
+            if not metadata['usgs_data']['coord_method_code'] in \
+                    acceptable_coord_method_code_list:
+                print(f"\t{lid}: Not in acceptable coord method codes")
+                continue
+            if not metadata['usgs_data']['alt_method_code'] in \
+                    acceptable_alt_meth_code_list:
+                print(f"\t{lid}: Not in acceptable alt method codes")
+                continue
+            if not metadata['usgs_data']['site_type'] in \
+                    acceptable_site_type_list:
+                print(f"\t{lid}: Not in acceptable site type codes")
+                continue
+            if not float(metadata['usgs_data']['alt_accuracy_code']) <= \
+                    acceptable_alt_acc_thresh:
+                print(f"\t{lid}: Not in acceptable threshold range")
+                continue
+        except Exception as e:
+            print(e)
+            continue
+
         ### --- Do Datum Offset --- ###
         #determine source of interpolated threshold flows, this will be the rating curve that will be used.
         rating_curve_source = flows.get('source')
@@ -461,7 +506,6 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
                 csv_df = csv_df.append(line_df)
                 
             except Exception as e:
-                print("Exception")
                 print(e)
           
         # Round flow and stage columns to 2 decimal places.
@@ -476,6 +520,7 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
             
         # If it made it to this point (i.e. no continues), there were no major preventers of mapping
         all_messages.append([f'{lid}:OK'])
+        mark_complete(output_dir)
     # Write all_messages by HUC to be scraped later.
     messages_dir = os.path.join(workspace, 'messages')
     if not os.path.exists(messages_dir):
@@ -547,30 +592,35 @@ def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us
             for row in reader:
                 all_messages.append(row)
     
-    # Write messages to DataFrame, split into columns, aggregate messages.
-    messages_df  = pd.DataFrame(all_messages, columns = ['message'])
-
-    messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
-    status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
-    
-    # Join messages to populate status field to candidate sites. Assign 
-    # status for null fields.
-    viz_out_gdf = viz_out_gdf.merge(status_df, how = 'left', on = 'nws_lid')
-    
-#    viz_out_gdf['status'] = viz_out_gdf['status'].fillna('OK')
-    
     # Filter out columns and write out to file
     nws_sites_layer = os.path.join(workspace, 'nws_lid_sites.gpkg')
-    
-    # Add acceptance criteria to viz_out_gdf before writing
-    viz_out_gdf['acceptable_coord_acc_code_list'] = str(acceptable_coord_acc_code_list)
-    viz_out_gdf['acceptable_coord_method_code_list'] = str(acceptable_coord_method_code_list)
-    viz_out_gdf['acceptable_alt_acc_thresh'] = float(acceptable_alt_acc_thresh)
-    viz_out_gdf['acceptable_alt_meth_code_list'] = str(acceptable_alt_meth_code_list)
-    viz_out_gdf['acceptable_site_type_list'] = str(acceptable_site_type_list)
-
-    viz_out_gdf.to_file(nws_sites_layer, driver='GPKG')
-    
+
+    # Only write to sites geopackage if it didn't exist yet
+    # (and this line shouldn't have been reached if we had an interrupted
+    # run previously and are picking back up with a restart)
+    if not os.path.exists(nws_sites_layer):
+        
+        # Write messages to DataFrame, split into columns, aggregate messages.
+        messages_df  = pd.DataFrame(all_messages, columns = ['message'])
+
+        messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
+        status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
+        
+        # Join messages to populate status field to candidate sites. Assign 
+        # status for null fields.
+        viz_out_gdf = viz_out_gdf.merge(status_df, how = 'left', on = 'nws_lid')
+        
+        #    viz_out_gdf['status'] = viz_out_gdf['status'].fillna('OK')
+        
+        # Add acceptance criteria to viz_out_gdf before writing
+        viz_out_gdf['acceptable_coord_acc_code_list'] = str(acceptable_coord_acc_code_list)
+        viz_out_gdf['acceptable_coord_method_code_list'] = str(acceptable_coord_method_code_list)
+        viz_out_gdf['acceptable_alt_acc_thresh'] = float(acceptable_alt_acc_thresh)
+        viz_out_gdf['acceptable_alt_meth_code_list'] = str(acceptable_alt_meth_code_list)
+        viz_out_gdf['acceptable_site_type_list'] = str(acceptable_site_type_list)
+
+        viz_out_gdf.to_file(nws_sites_layer, driver='GPKG')
+        
     return nws_sites_layer
     
 

From 4ddaa21d46ca265cc41527a298cda52bc1f7482c Mon Sep 17 00:00:00 2001
From: LauraKeys-NOAA <70279180+LauraKeys-NOAA@users.noreply.github.com>
Date: Fri, 17 Feb 2023 10:53:34 -0600
Subject: [PATCH 208/359] v4.1.1.0 Update CHANGELOG.md (#817)

---
 docs/CHANGELOG.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1845a039b..661fc5e00 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.1.1.0 - 2023-02-16 - [PR#809](https://github.com/NOAA-OWP/inundation-mapping/pull/809)
+
+The CatFIM code was updated to allow 1-foot interval processing across all stage-based AHPS sites ranging from action stage to 5 feet above major stage, along with restart capability for interrupted processing runs.
+
+### Changes
+
+- `tools/generate_categorical_fim.py` (all changes made here)
+    - Added try-except blocks for code that didn't allow most sites to actually get processed because it was trying to check values of some USGS-related variables that most of the sites didn't have
+    - Overwrite abilities of the different outputs for the viz team were not consistent (i.e., one of the files had the ability to be overwritten but another didn't), so that has been made consistent to disallow any overwrites of the existing final outputs for a specified output folder.
+    - The code also has the ability to restart from an interrupted run and resume processing uncompleted HUCs by first checking for a simple "complete" file for each HUC. If a HUC has that file, then it is skipped (because it already completed processing during a run for a particular output folder / run name).
+    - When a HUC is successfully processed, an empty "complete" text file is created / touched.
+
+<br/><br/>
+
 
 ## v4.1.0.0 - 2023-01-30 - [PR#806](https://github.com/NOAA-OWP/inundation-mapping/pull/806)
 

From 5995bbf823554fafc9f807e719954009d8a0782f Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 17 Feb 2023 10:01:35 -0700
Subject: [PATCH 209/359] v4.1.2.0 Refactor `/unit_tests` to use Pytest. (#808)

---
 Pipfile                                       |    2 +-
 Pipfile.lock                                  | 1480 ++++++++++-------
 config/params_template.env                    |    8 +-
 docs/CHANGELOG.md                             |   59 +
 unit_tests/README.md                          |  134 +-
 ...__template_params.json => __template.json} |    6 +-
 unit_tests/__template.py                      |   77 +
 unit_tests/__template_unittests.py            |  108 --
 unit_tests/aggregate_branch_lists_params.json |    4 +-
 ...ests.py => aggregate_branch_lists_test.py} |   33 +-
 unit_tests/check_unit_errors_params.json      |    4 +-
 unit_tests/check_unit_errors_test.py          |  226 +++
 unit_tests/check_unit_errors_unittests.py     |  278 ----
 unit_tests/clip_vectors_to_wbd_params.json    |   13 +-
 unit_tests/clip_vectors_to_wbd_test.py        |   66 +
 unit_tests/clip_vectors_to_wbd_unittests.py   |   89 -
 ..._catchments_and_add_attributes_params.json |   12 +-
 ...ter_catchments_and_add_attributes_test.py} |   59 +-
 .../generate_branch_list_csv_params.json      |    6 +-
 ...ts.py => generate_branch_list_csv_test.py} |   46 +-
 unit_tests/generate_branch_list_params.json   |    4 +-
 unit_tests/generate_branch_list_test.py       |   34 +
 unit_tests/generate_branch_list_unittests.py  |   59 -
 unit_tests/gms/__init__.py                    |    0
 unit_tests/gms/derive_level_paths_params.json |   17 +-
 unit_tests/gms/derive_level_paths_test.py     |   94 ++
 .../gms/derive_level_paths_unittests.py       |  170 --
 unit_tests/gms/outputs_cleanup_test.py        |  188 +++
 unit_tests/gms/outputs_cleanup_unittests.py   |  163 --
 unit_tests/pyproject.toml                     |    7 +
 .../rating_curve_comparison_params.json       |    3 +
 unit_tests/rating_curve_comparison_test.py    |  103 ++
 .../rating_curve_comparison_unittests.py      |   95 --
 unit_tests/shared_functions_params.json       |   10 +-
 ..._unittests.py => shared_functions_test.py} |   63 +-
 unit_tests/split_flows_params.json            |    5 +-
 ...flows_unittests.py => split_flows_test.py} |   53 +-
 unit_tests/tools/__init__.py                  |    0
 .../{gms_tools => }/inundate_gms_params.json  |    9 +-
 ..._gms_unittests.py => inundate_gms_test.py} |   38 +-
 unit_tests/tools/inundate_unittests.py        |   91 -
 ...ate_params.json => inundation_params.json} |    0
 unit_tests/tools/inundation_test.py           |   62 +
 unit_tests/unit_tests_utils.py                |    8 +-
 unit_tests/usgs_gage_crosswalk_test.py        |   55 +
 unit_tests/usgs_gage_crosswalk_unittests.py   |   80 -
 46 files changed, 2109 insertions(+), 2012 deletions(-)
 rename unit_tests/{__template_params.json => __template.json} (72%)
 create mode 100644 unit_tests/__template.py
 delete mode 100644 unit_tests/__template_unittests.py
 rename unit_tests/{aggregate_branch_lists_unittests.py => aggregate_branch_lists_test.py} (51%)
 create mode 100644 unit_tests/check_unit_errors_test.py
 delete mode 100644 unit_tests/check_unit_errors_unittests.py
 create mode 100644 unit_tests/clip_vectors_to_wbd_test.py
 delete mode 100644 unit_tests/clip_vectors_to_wbd_unittests.py
 rename unit_tests/{filter_catchments_and_add_attributes_unittests.py => filter_catchments_and_add_attributes_test.py} (59%)
 rename unit_tests/{generate_branch_list_csv_unittests.py => generate_branch_list_csv_test.py} (62%)
 create mode 100644 unit_tests/generate_branch_list_test.py
 delete mode 100644 unit_tests/generate_branch_list_unittests.py
 create mode 100644 unit_tests/gms/__init__.py
 create mode 100644 unit_tests/gms/derive_level_paths_test.py
 delete mode 100644 unit_tests/gms/derive_level_paths_unittests.py
 create mode 100644 unit_tests/gms/outputs_cleanup_test.py
 delete mode 100644 unit_tests/gms/outputs_cleanup_unittests.py
 create mode 100644 unit_tests/pyproject.toml
 create mode 100644 unit_tests/rating_curve_comparison_test.py
 delete mode 100644 unit_tests/rating_curve_comparison_unittests.py
 rename unit_tests/{shared_functions_unittests.py => shared_functions_test.py} (51%)
 rename unit_tests/{split_flows_unittests.py => split_flows_test.py} (57%)
 create mode 100644 unit_tests/tools/__init__.py
 rename unit_tests/tools/{gms_tools => }/inundate_gms_params.json (66%)
 rename unit_tests/tools/{gms_tools/inundate_gms_unittests.py => inundate_gms_test.py} (67%)
 delete mode 100644 unit_tests/tools/inundate_unittests.py
 rename unit_tests/tools/{inundate_params.json => inundation_params.json} (100%)
 create mode 100644 unit_tests/tools/inundation_test.py
 create mode 100644 unit_tests/usgs_gage_crosswalk_test.py
 delete mode 100644 unit_tests/usgs_gage_crosswalk_unittests.py

diff --git a/Pipfile b/Pipfile
index 08a1c3150..e62db0d1e 100755
--- a/Pipfile
+++ b/Pipfile
@@ -33,7 +33,7 @@ boto3 = "*"
 jupyter = "*"
 jupyterlab = "*"
 ipympl = "*"
-jupyter-core = "~=4.11.2"
+pytest = "*"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index af80a8c82..12f51c8ee 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "efc2e94e622e9f6160ed7ddc395420f566ca21527936677b5ad0184c1d26bb1a"
+            "sha256": "f05b5609b9137d06ed14e862c850d362334b299b913f52899fb234449fbb231a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -18,10 +18,27 @@
     "default": {
         "affine": {
             "hashes": [
-                "sha256:d676de66157ad6af99ffd94e0f54e89dfc35b0fb7252ead2ed0ad2dca431bdd0",
-                "sha256:de17839ff05e965580870c3b15e14cefd7992fa05dba9202a0879bbed0c171e4"
+                "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92",
+                "sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea"
             ],
-            "version": "==2.3.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.4.0"
+        },
+        "aiofiles": {
+            "hashes": [
+                "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
+                "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
+            ],
+            "markers": "python_version >= '3.7' and python_version < '4'",
+            "version": "==22.1.0"
+        },
+        "aiosqlite": {
+            "hashes": [
+                "sha256:c3511b841e3a2c5614900ba1d179f366826857586f78abd75e7cbeb88e75a557",
+                "sha256:faa843ef5fb08bafe9a9b3859012d3d9d6f77ce3637899de20606b7fc39aa213"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.18.0"
         },
         "anyio": {
             "hashes": [
@@ -66,6 +83,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==21.2.0"
         },
+        "arrow": {
+            "hashes": [
+                "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1",
+                "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.2.3"
+        },
         "asttokens": {
             "hashes": [
                 "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
@@ -75,11 +100,11 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6",
-                "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"
+                "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836",
+                "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==22.1.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==22.2.0"
         },
         "babel": {
             "hashes": [
@@ -98,42 +123,41 @@
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
-                "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"
+                "sha256:0e79446b10b3ecb499c1556f7e228a53e64a2bfcebd455f370d8927cb5b59e39",
+                "sha256:bc4bdda6717de5a2987436fb8d72f45dc90dd856bdfd512a1314ce90349a0106"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==4.11.1"
+            "version": "==4.11.2"
         },
         "bleach": {
             "hashes": [
-                "sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a",
-                "sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c"
+                "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414",
+                "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.0.1"
+            "version": "==6.0.0"
         },
         "boto3": {
             "hashes": [
-                "sha256:17ce7b6b702f9e844a33ce3ce9612f09d9d73eb7d34aaeffc77353ec036a9615",
-                "sha256:aec7db139429fe0f3fbe723170461192b0483b0070114a4b56351e374e0f294d"
+                "sha256:159ab54ab301a213aea0dbebfe97c12909fd9180110a2b78aaa712c2d7d1a929"
             ],
             "index": "pypi",
-            "version": "==1.25.5"
+            "version": "==1.26.67"
         },
         "botocore": {
             "hashes": [
-                "sha256:5eb98c85e80fab6640342377ce947c86a003bbaf8427c2e7a03c8da0bdbb796c",
-                "sha256:f322d7b62163219ffeb787a116d318273dfb7243c3b49d95f5bfff8daa1df4e0"
+                "sha256:0ccec4a906b6b8c7bb6bc5226509059ee9ed94d3cf1014487ef5b8e56801e6a3",
+                "sha256:424c5fa21913d85ea990928acf35622a4f2b7f0750133b96adaa8c642bb6a878"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.28.5"
+            "version": "==1.29.67"
         },
         "certifi": {
             "hashes": [
                 "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
                 "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
             ],
-            "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==2022.12.7"
         },
         "cffi": {
@@ -234,11 +258,97 @@
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
-                "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
+                "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b",
+                "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42",
+                "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d",
+                "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b",
+                "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a",
+                "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59",
+                "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154",
+                "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1",
+                "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c",
+                "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a",
+                "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d",
+                "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6",
+                "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b",
+                "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b",
+                "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783",
+                "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5",
+                "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918",
+                "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555",
+                "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639",
+                "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786",
+                "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e",
+                "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed",
+                "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820",
+                "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8",
+                "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3",
+                "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541",
+                "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14",
+                "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be",
+                "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e",
+                "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76",
+                "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b",
+                "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c",
+                "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b",
+                "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3",
+                "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc",
+                "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6",
+                "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59",
+                "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4",
+                "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d",
+                "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d",
+                "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3",
+                "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a",
+                "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea",
+                "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6",
+                "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e",
+                "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603",
+                "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24",
+                "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a",
+                "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58",
+                "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678",
+                "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a",
+                "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c",
+                "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6",
+                "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18",
+                "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174",
+                "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317",
+                "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f",
+                "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc",
+                "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837",
+                "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41",
+                "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c",
+                "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579",
+                "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753",
+                "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8",
+                "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291",
+                "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087",
+                "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866",
+                "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3",
+                "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d",
+                "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1",
+                "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca",
+                "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e",
+                "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db",
+                "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72",
+                "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d",
+                "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc",
+                "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539",
+                "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d",
+                "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af",
+                "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b",
+                "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602",
+                "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f",
+                "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478",
+                "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c",
+                "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e",
+                "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479",
+                "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7",
+                "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.1.1"
+            "version": "==3.0.1"
         },
         "click": {
             "hashes": [
@@ -273,78 +383,64 @@
         },
         "contourpy": {
             "hashes": [
-                "sha256:0236875c5a0784215b49d00ebbe80c5b6b5d5244b3655a36dda88105334dea17",
-                "sha256:03d1b9c6b44a9e30d554654c72be89af94fab7510b4b9f62356c64c81cec8b7d",
-                "sha256:0537cc1195245bbe24f2913d1f9211b8f04eb203de9044630abd3664c6cc339c",
-                "sha256:06ca79e1efbbe2df795822df2fa173d1a2b38b6e0f047a0ec7903fbca1d1847e",
-                "sha256:08e8d09d96219ace6cb596506fb9b64ea5f270b2fb9121158b976d88871fcfd1",
-                "sha256:0b1e66346acfb17694d46175a0cea7d9036f12ed0c31dfe86f0f405eedde2bdd",
-                "sha256:0b97454ed5b1368b66ed414c754cba15b9750ce69938fc6153679787402e4cdf",
-                "sha256:0e4854cc02006ad6684ce092bdadab6f0912d131f91c2450ce6dbdea78ee3c0b",
-                "sha256:12a7dc8439544ed05c6553bf026d5e8fa7fad48d63958a95d61698df0e00092b",
-                "sha256:1b1ee48a130da4dd0eb8055bbab34abf3f6262957832fd575e0cab4979a15a41",
-                "sha256:1c0e1308307a75e07d1f1b5f0f56b5af84538a5e9027109a7bcf6cb47c434e72",
-                "sha256:1dedf4c64185a216c35eb488e6f433297c660321275734401760dafaeb0ad5c2",
-                "sha256:208bc904889c910d95aafcf7be9e677726df9ef71e216780170dbb7e37d118fa",
-                "sha256:211dfe2bd43bf5791d23afbe23a7952e8ac8b67591d24be3638cabb648b3a6eb",
-                "sha256:341330ed19074f956cb20877ad8d2ae50e458884bfa6a6df3ae28487cc76c768",
-                "sha256:344cb3badf6fc7316ad51835f56ac387bdf86c8e1b670904f18f437d70da4183",
-                "sha256:358f6364e4873f4d73360b35da30066f40387dd3c427a3e5432c6b28dd24a8fa",
-                "sha256:371f6570a81dfdddbb837ba432293a63b4babb942a9eb7aaa699997adfb53278",
-                "sha256:375d81366afd547b8558c4720337218345148bc2fcffa3a9870cab82b29667f2",
-                "sha256:3a1917d3941dd58732c449c810fa7ce46cc305ce9325a11261d740118b85e6f3",
-                "sha256:4081918147fc4c29fad328d5066cfc751da100a1098398742f9f364be63803fc",
-                "sha256:444fb776f58f4906d8d354eb6f6ce59d0a60f7b6a720da6c1ccb839db7c80eb9",
-                "sha256:46deb310a276cc5c1fd27958e358cce68b1e8a515fa5a574c670a504c3a3fe30",
-                "sha256:494efed2c761f0f37262815f9e3c4bb9917c5c69806abdee1d1cb6611a7174a0",
-                "sha256:50627bf76abb6ba291ad08db583161939c2c5fab38c38181b7833423ab9c7de3",
-                "sha256:5641927cc5ae66155d0c80195dc35726eae060e7defc18b7ab27600f39dd1fe7",
-                "sha256:5b117d29433fc8393b18a696d794961464e37afb34a6eeb8b2c37b5f4128a83e",
-                "sha256:613c665529899b5d9fade7e5d1760111a0b011231277a0d36c49f0d3d6914bd6",
-                "sha256:6e459ebb8bb5ee4c22c19cc000174f8059981971a33ce11e17dddf6aca97a142",
-                "sha256:6f56515e7c6fae4529b731f6c117752247bef9cdad2b12fc5ddf8ca6a50965a5",
-                "sha256:730c27978a0003b47b359935478b7d63fd8386dbb2dcd36c1e8de88cbfc1e9de",
-                "sha256:75a2e638042118118ab39d337da4c7908c1af74a8464cad59f19fbc5bbafec9b",
-                "sha256:78ced51807ccb2f45d4ea73aca339756d75d021069604c2fccd05390dc3c28eb",
-                "sha256:7ee394502026d68652c2824348a40bf50f31351a668977b51437131a90d777ea",
-                "sha256:8468b40528fa1e15181cccec4198623b55dcd58306f8815a793803f51f6c474a",
-                "sha256:84c593aeff7a0171f639da92cb86d24954bbb61f8a1b530f74eb750a14685832",
-                "sha256:913bac9d064cff033cf3719e855d4f1db9f1c179e0ecf3ba9fdef21c21c6a16a",
-                "sha256:9447c45df407d3ecb717d837af3b70cfef432138530712263730783b3d016512",
-                "sha256:9b0e7fe7f949fb719b206548e5cde2518ffb29936afa4303d8a1c4db43dcb675",
-                "sha256:9bc407a6af672da20da74823443707e38ece8b93a04009dca25856c2d9adadb1",
-                "sha256:9e8e686a6db92a46111a1ee0ee6f7fbfae4048f0019de207149f43ac1812cf95",
-                "sha256:9fc4e7973ed0e1fe689435842a6e6b330eb7ccc696080dda9a97b1a1b78e41db",
-                "sha256:a457ee72d9032e86730f62c5eeddf402e732fdf5ca8b13b41772aa8ae13a4563",
-                "sha256:a628bba09ba72e472bf7b31018b6281fd4cc903f0888049a3724afba13b6e0b8",
-                "sha256:a79d239fc22c3b8d9d3de492aa0c245533f4f4c7608e5749af866949c0f1b1b9",
-                "sha256:aa4674cf3fa2bd9c322982644967f01eed0c91bb890f624e0e0daf7a5c3383e9",
-                "sha256:acd2bd02f1a7adff3a1f33e431eb96ab6d7987b039d2946a9b39fe6fb16a1036",
-                "sha256:b3b1bd7577c530eaf9d2bc52d1a93fef50ac516a8b1062c3d1b9bcec9ebe329b",
-                "sha256:b48d94386f1994db7c70c76b5808c12e23ed7a4ee13693c2fc5ab109d60243c0",
-                "sha256:b64f747e92af7da3b85631a55d68c45a2d728b4036b03cdaba4bd94bcc85bd6f",
-                "sha256:b98c820608e2dca6442e786817f646d11057c09a23b68d2b3737e6dcb6e4a49b",
-                "sha256:c1baa49ab9fedbf19d40d93163b7d3e735d9cd8d5efe4cce9907902a6dad391f",
-                "sha256:c38c6536c2d71ca2f7e418acaf5bca30a3af7f2a2fa106083c7d738337848dbe",
-                "sha256:c78bfbc1a7bff053baf7e508449d2765964d67735c909b583204e3240a2aca45",
-                "sha256:cd2bc0c8f2e8de7dd89a7f1c10b8844e291bca17d359373203ef2e6100819edd",
-                "sha256:d2eff2af97ea0b61381828b1ad6cd249bbd41d280e53aea5cccd7b2b31b8225c",
-                "sha256:d8834c14b8c3dd849005e06703469db9bf96ba2d66a3f88ecc539c9a8982e0ee",
-                "sha256:d912f0154a20a80ea449daada904a7eb6941c83281a9fab95de50529bfc3a1da",
-                "sha256:da1ef35fd79be2926ba80fbb36327463e3656c02526e9b5b4c2b366588b74d9a",
-                "sha256:dbe6fe7a1166b1ddd7b6d887ea6fa8389d3f28b5ed3f73a8f40ece1fc5a3d340",
-                "sha256:dcd556c8fc37a342dd636d7eef150b1399f823a4462f8c968e11e1ebeabee769",
-                "sha256:e13b31d1b4b68db60b3b29f8e337908f328c7f05b9add4b1b5c74e0691180109",
-                "sha256:e1739496c2f0108013629aa095cc32a8c6363444361960c07493818d0dea2da4",
-                "sha256:e43255a83835a129ef98f75d13d643844d8c646b258bebd11e4a0975203e018f",
-                "sha256:e626cefff8491bce356221c22af5a3ea528b0b41fbabc719c00ae233819ea0bf",
-                "sha256:eadad75bf91897f922e0fb3dca1b322a58b1726a953f98c2e5f0606bd8408621",
-                "sha256:f33da6b5d19ad1bb5e7ad38bb8ba5c426d2178928bc2b2c44e8823ea0ecb6ff3",
-                "sha256:f4052a8a4926d4468416fc7d4b2a7b2a3e35f25b39f4061a7e2a3a2748c4fc48",
-                "sha256:f6ca38dd8d988eca8f07305125dec6f54ac1c518f1aaddcc14d08c01aebb6efc"
+                "sha256:031154ed61f7328ad7f97662e48660a150ef84ee1bc8876b6472af88bf5a9b98",
+                "sha256:0f9d350b639db6c2c233d92c7f213d94d2e444d8e8fc5ca44c9706cf72193772",
+                "sha256:130230b7e49825c98edf0b428b7aa1125503d91732735ef897786fe5452b1ec2",
+                "sha256:152fd8f730c31fd67fe0ffebe1df38ab6a669403da93df218801a893645c6ccc",
+                "sha256:1c71fdd8f1c0f84ffd58fca37d00ca4ebaa9e502fb49825484da075ac0b0b803",
+                "sha256:24847601071f740837aefb730e01bd169fbcaa610209779a78db7ebb6e6a7051",
+                "sha256:2e9ebb4425fc1b658e13bace354c48a933b842d53c458f02c86f371cecbedecc",
+                "sha256:30676ca45084ee61e9c3da589042c24a57592e375d4b138bd84d8709893a1ba4",
+                "sha256:31a55dccc8426e71817e3fe09b37d6d48ae40aae4ecbc8c7ad59d6893569c436",
+                "sha256:366a0cf0fc079af5204801786ad7a1c007714ee3909e364dbac1729f5b0849e5",
+                "sha256:38e2e577f0f092b8e6774459317c05a69935a1755ecfb621c0a98f0e3c09c9a5",
+                "sha256:3c184ad2433635f216645fdf0493011a4667e8d46b34082f5a3de702b6ec42e3",
+                "sha256:3caea6365b13119626ee996711ab63e0c9d7496f65641f4459c60a009a1f3e80",
+                "sha256:3e927b3868bd1e12acee7cc8f3747d815b4ab3e445a28d2e5373a7f4a6e76ba1",
+                "sha256:4ee3ee247f795a69e53cd91d927146fb16c4e803c7ac86c84104940c7d2cabf0",
+                "sha256:54d43960d809c4c12508a60b66cb936e7ed57d51fb5e30b513934a4a23874fae",
+                "sha256:57119b0116e3f408acbdccf9eb6ef19d7fe7baf0d1e9aaa5381489bc1aa56556",
+                "sha256:58569c491e7f7e874f11519ef46737cea1d6eda1b514e4eb5ac7dab6aa864d02",
+                "sha256:5a011cf354107b47c58ea932d13b04d93c6d1d69b8b6dce885e642531f847566",
+                "sha256:5caeacc68642e5f19d707471890f037a13007feba8427eb7f2a60811a1fc1350",
+                "sha256:5dd34c1ae752515318224cba7fc62b53130c45ac6a1040c8b7c1a223c46e8967",
+                "sha256:60835badb5ed5f4e194a6f21c09283dd6e007664a86101431bf870d9e86266c4",
+                "sha256:62398c80ef57589bdbe1eb8537127321c1abcfdf8c5f14f479dbbe27d0322e66",
+                "sha256:6381fa66866b0ea35e15d197fc06ac3840a9b2643a6475c8fff267db8b9f1e69",
+                "sha256:64757f6460fc55d7e16ed4f1de193f362104285c667c112b50a804d482777edd",
+                "sha256:69f8ff4db108815addd900a74df665e135dbbd6547a8a69333a68e1f6e368ac2",
+                "sha256:6c180d89a28787e4b73b07e9b0e2dac7741261dbdca95f2b489c4f8f887dd810",
+                "sha256:71b0bf0c30d432278793d2141362ac853859e87de0a7dee24a1cea35231f0d50",
+                "sha256:769eef00437edf115e24d87f8926955f00f7704bede656ce605097584f9966dc",
+                "sha256:7f6979d20ee5693a1057ab53e043adffa1e7418d734c1532e2d9e915b08d8ec2",
+                "sha256:87f4d8941a9564cda3f7fa6a6cd9b32ec575830780677932abdec7bcb61717b0",
+                "sha256:89ba9bb365446a22411f0673abf6ee1fea3b2cf47b37533b970904880ceb72f3",
+                "sha256:8acf74b5d383414401926c1598ed77825cd530ac7b463ebc2e4f46638f56cce6",
+                "sha256:9056c5310eb1daa33fc234ef39ebfb8c8e2533f088bbf0bc7350f70a29bde1ac",
+                "sha256:95c3acddf921944f241b6773b767f1cbce71d03307270e2d769fd584d5d1092d",
+                "sha256:9e20e5a1908e18aaa60d9077a6d8753090e3f85ca25da6e25d30dc0a9e84c2c6",
+                "sha256:a1e97b86f73715e8670ef45292d7cc033548266f07d54e2183ecb3c87598888f",
+                "sha256:a877ada905f7d69b2a31796c4b66e31a8068b37aa9b78832d41c82fc3e056ddd",
+                "sha256:a9d7587d2fdc820cc9177139b56795c39fb8560f540bba9ceea215f1f66e1566",
+                "sha256:abf298af1e7ad44eeb93501e40eb5a67abbf93b5d90e468d01fc0c4451971afa",
+                "sha256:ae90d5a8590e5310c32a7630b4b8618cef7563cebf649011da80874d0aa8f414",
+                "sha256:b6d0f9e1d39dbfb3977f9dd79f156c86eb03e57a7face96f199e02b18e58d32a",
+                "sha256:b8d587cc39057d0afd4166083d289bdeff221ac6d3ee5046aef2d480dc4b503c",
+                "sha256:c5210e5d5117e9aec8c47d9156d1d3835570dd909a899171b9535cb4a3f32693",
+                "sha256:cc331c13902d0f50845099434cd936d49d7a2ca76cb654b39691974cb1e4812d",
+                "sha256:ce41676b3d0dd16dbcfabcc1dc46090aaf4688fd6e819ef343dbda5a57ef0161",
+                "sha256:d8165a088d31798b59e91117d1f5fc3df8168d8b48c4acc10fc0df0d0bdbcc5e",
+                "sha256:e7281244c99fd7c6f27c1c6bfafba878517b0b62925a09b586d88ce750a016d2",
+                "sha256:e96a08b62bb8de960d3a6afbc5ed8421bf1a2d9c85cc4ea73f4bc81b4910500f",
+                "sha256:ed33433fc3820263a6368e532f19ddb4c5990855e4886088ad84fd7c4e561c71",
+                "sha256:efb8f6d08ca7998cf59eaf50c9d60717f29a1a0a09caa46460d33b2924839dbd",
+                "sha256:efe99298ba37e37787f6a2ea868265465410822f7bea163edcc1bd3903354ea9",
+                "sha256:f99e9486bf1bb979d95d5cffed40689cb595abb2b841f2991fc894b3452290e8",
+                "sha256:fc1464c97579da9f3ab16763c32e5c5d5bb5fa1ec7ce509a4ca6108b61b84fab",
+                "sha256:fd7dc0e6812b799a34f6d12fcb1000539098c249c8da54f3566c6a6461d0dbad"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.0.6"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.7"
         },
         "cycler": {
             "hashes": [
@@ -356,27 +452,26 @@
         },
         "debugpy": {
             "hashes": [
-                "sha256:143f79d0798a9acea21cd1d111badb789f19d414aec95fa6389cfea9485ddfb1",
-                "sha256:1caee68f7e254267df908576c0d0938f8f88af16383f172cb9f0602e24c30c01",
-                "sha256:2a39e7da178e1f22f4bc04b57f085e785ed1bcf424aaf318835a1a7129eefe35",
-                "sha256:3d9c31baf64bf959a593996c108e911c5a9aa1693a296840e5469473f064bcec",
-                "sha256:40e2a83d31a16b83666f19fa06d97b2cc311af88e6266590579737949971a17e",
-                "sha256:4ab5e938925e5d973f567d6ef32751b17d10f3be3a8c4d73c52f53e727f69bf1",
-                "sha256:563f148f94434365ec0ce94739c749aabf60bf67339e68a9446499f3582d62f3",
-                "sha256:62ba4179b372a62abf9c89b56997d70a4100c6dea6c2a4e0e4be5f45920b3253",
-                "sha256:67edf033f9e512958f7b472975ff9d9b7ff64bf4440f6f6ae44afdc66b89e6b6",
-                "sha256:6ae238943482c78867ac707c09122688efb700372b617ffd364261e5e41f7a2f",
-                "sha256:82229790442856962aec4767b98ba2559fe0998f897e9f21fb10b4fd24b6c436",
-                "sha256:86bd25f38f8b6c5d430a5e2931eebbd5f580c640f4819fcd236d0498790c7204",
-                "sha256:d2968e589bda4e485a9c61f113754a28e48d88c5152ed8e0b2564a1fadbe50a5",
-                "sha256:d5ab9bd3f4e7faf3765fd52c7c43c074104ab1e109621dc73219099ed1a5399d",
-                "sha256:d8df268e9f72fc06efc2e75e8dc8e2b881d6a397356faec26efb2ee70b6863b7",
-                "sha256:e62b8034ede98932b92268669318848a0d42133d857087a3b9cec03bb844c615",
-                "sha256:e886a1296cd20a10172e94788009ce74b759e54229ebd64a43fa5c2b4e62cd76",
-                "sha256:ea4bf208054e6d41749f17612066da861dff10102729d32c85b47f155223cf2b"
+                "sha256:0ea1011e94416e90fb3598cc3ef5e08b0a4dd6ce6b9b33ccd436c1dffc8cd664",
+                "sha256:23363e6d2a04d726bbc1400bd4e9898d54419b36b2cdf7020e3e215e1dcd0f8e",
+                "sha256:23c29e40e39ad7d869d408ded414f6d46d82f8a93b5857ac3ac1e915893139ca",
+                "sha256:549ae0cb2d34fc09d1675f9b01942499751d174381b6082279cf19cdb3c47cbe",
+                "sha256:70ab53918fd907a3ade01909b3ed783287ede362c80c75f41e79596d5ccacd32",
+                "sha256:72687b62a54d9d9e3fb85e7a37ea67f0e803aaa31be700e61d2f3742a5683917",
+                "sha256:78739f77c58048ec006e2b3eb2e0cd5a06d5f48c915e2fc7911a337354508110",
+                "sha256:7aa7e103610e5867d19a7d069e02e72eb2b3045b124d051cfd1538f1d8832d1b",
+                "sha256:87755e173fcf2ec45f584bb9d61aa7686bb665d861b81faa366d59808bbd3494",
+                "sha256:9b5d1b13d7c7bf5d7cf700e33c0b8ddb7baf030fcf502f76fc061ddd9405d16c",
+                "sha256:a771739902b1ae22a120dbbb6bd91b2cae6696c0e318b5007c5348519a4211c6",
+                "sha256:b9c2130e1c632540fbf9c2c88341493797ddf58016e7cba02e311de9b0a96b67",
+                "sha256:be596b44448aac14eb3614248c91586e2bc1728e020e82ef3197189aae556115",
+                "sha256:c05349890804d846eca32ce0623ab66c06f8800db881af7a876dc073ac1c2225",
+                "sha256:de4a045fbf388e120bb6ec66501458d3134f4729faed26ff95de52a754abddb1",
+                "sha256:dff595686178b0e75580c24d316aa45a8f4d56e2418063865c114eef651a982e",
+                "sha256:f6383c29e796203a0bba74a250615ad262c4279d398e89d895a69d3069498305"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.6.4"
+            "version": "==1.6.6"
         },
         "decorator": {
             "hashes": [
@@ -394,13 +489,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.7.1"
         },
-        "entrypoints": {
+        "exceptiongroup": {
             "hashes": [
-                "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4",
-                "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f"
+                "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e",
+                "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.4"
+            "markers": "python_version < '3.11'",
+            "version": "==1.1.0"
         },
         "executing": {
             "hashes": [
@@ -440,6 +535,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==4.38.0"
         },
+        "fqdn": {
+            "hashes": [
+                "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f",
+                "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014"
+            ],
+            "version": "==1.5.1"
+        },
         "geopandas": {
             "hashes": [
                 "sha256:e28a729e44ac53c1891b54b1aca60e3bc0bb9e88ad0f2be8e301a03b9510f6e2",
@@ -482,27 +584,35 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b",
-                "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"
+                "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad",
+                "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==5.1.0"
+            "version": "==6.0.0"
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:32bb095bda29741f6ef0e5278c42df98d135391bee5f932841efc0041f748dc3",
-                "sha256:c09b067d82e72c66f4f8eb12332f5efbebc9b007c0b6c40818108c9870adc363"
+                "sha256:7d543798b0beca10b6a01ac7cafda9f822c54db9e8376a6bf57e0cbd74d486b6",
+                "sha256:e4a96c8cc0339647ff9a5e0550d9f276fc5a01ffa276012b58ec108cfd7b8484"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==5.10.1"
+            "version": "==5.10.2"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
+                "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.0"
         },
         "ipykernel": {
             "hashes": [
-                "sha256:1374a55c57ca7a7286c3d8b15799cd76e1a2381b6b1fea99c494b955988926b6",
-                "sha256:1ab68d3d3654196266baa93990055413e167263ffbe4cfe834f871bcd3d3506d"
+                "sha256:1a04bb359212e23e46adc0116ec82ea128c1e5bd532fde4fbe679787ff36f0cf",
+                "sha256:a0f8eece39cab1ee352c9b59ec67bbe44d8299f8238e4c16ff7f4cf0052d3378"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.19.2"
+            "version": "==6.21.1"
         },
         "ipympl": {
             "hashes": [
@@ -514,11 +624,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:352042ddcb019f7c04e48171b4dd78e4c4bb67bf97030d170e154aac42b656d9",
-                "sha256:882899fe78d5417a0aa07f995db298fa28b58faeba2112d2e3a4c95fe14bb738"
+                "sha256:71618e82e6d59487bea059626e7c79fb4a5b760d1510d02fab1160db6fdfa1f7",
+                "sha256:9c207b0ef2d276d1bfcfeb9a62804336abbe4b170574ea061500952319b1d78c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.7.0"
+            "version": "==8.9.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -529,11 +639,18 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:2ec50df8538a1d4ddd5d454830d010922ad1015e81ac23efb27c0908bbc1eece",
-                "sha256:db7dd35fb1217636cbdbe0ba0bd2216d91a7695cb28b5c1dca17e62cd51378de"
+                "sha256:c0005a77a47d77889cafed892b58e33b4a2a96712154404c6548ec22272811ea",
+                "sha256:ebb195e743b16c3947fe8827190fb87b4d00979c0fbf685afe4d2c4927059fa1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.0.3"
+            "version": "==8.0.4"
+        },
+        "isoduration": {
+            "hashes": [
+                "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9",
+                "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042"
+            ],
+            "version": "==20.11.0"
         },
         "jedi": {
             "hashes": [
@@ -561,12 +678,22 @@
         },
         "json5": {
             "hashes": [
-                "sha256:993189671e7412e9cdd8be8dc61cf402e8e579b35f1d1bb20ae6b09baa78bbce",
-                "sha256:ad9f048c5b5a4c3802524474ce40a622fae789860a86f10cc4f7e5f9cf9b46ab"
+                "sha256:1aa54b80b5e507dfe31d12b7743a642e2ffa6f70bf73b8e3d7d1d5fba83d99bd",
+                "sha256:4f1e196acc55b83985a51318489f345963c7ba84aa37607e49073066c562e99b"
+            ],
+            "version": "==0.9.11"
+        },
+        "jsonpointer": {
+            "hashes": [
+                "sha256:51801e558539b4e9cd268638c078c6c5746c9ac96bc38152d443400e4f3793e9",
+                "sha256:97cba51526c829282218feb99dab1b1e6bdf8efd1c43dc9d57be093c0d69c99a"
             ],
-            "version": "==0.9.10"
+            "version": "==2.3"
         },
         "jsonschema": {
+            "extras": [
+                "format-nongpl"
+            ],
             "hashes": [
                 "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d",
                 "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"
@@ -585,11 +712,11 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:109a3c33b62a9cf65aa8325850a0999a795fac155d9de4f7555aef5f310ee35a",
-                "sha256:d4a67ae86ee014bcb96bd8190714f6af921f2b0f52f4208b086aa5acfd9f8d65"
+                "sha256:47ac9f586dbcff4d79387ec264faf0fdeb5f14845fa7345fd7d1e378f8096011",
+                "sha256:c53731eb590b68839b0ce04bf46ff8c4f03278f5d9fe5c3b0f268a57cc2bd97e"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==7.4.8"
+            "markers": "python_version >= '3.8'",
+            "version": "==8.0.2"
         },
         "jupyter-console": {
             "hashes": [
@@ -601,27 +728,67 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:3815e80ec5272c0c19aad087a0d2775df2852cfca8f5a17069e99c9350cecff8",
-                "sha256:c2909b9bc7dca75560a6c5ae78c34fd305ede31cd864da3c0d0bb2ed89aa9337"
+                "sha256:1407cdb4c79ee467696c04b76633fc1884015fa109323365a6372c8e890cc83f",
+                "sha256:4bdc2928c37f6917130c667d8b8708f20aee539d8283c6be72aabd2a4b4c83b0"
             ],
-            "index": "pypi",
-            "version": "==4.11.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.2.0"
+        },
+        "jupyter-events": {
+            "hashes": [
+                "sha256:6f7b67bf42b8a370c992187194ed02847dfa02307a7aebe9913e2d3979b9b6b8",
+                "sha256:e27ffdd6138699d47d42cb65ae6d79334ff7c0d923694381c991ce56a140f2cd"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.5.0"
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:438496cac509709cc85e60172e5538ca45b4c8a0862bb97cd73e49f2ace419cb",
-                "sha256:f7f7a2f9d36f4150ad125afef0e20b1c76c8ff83eb5e39fb02d3b9df0f9b79ab"
+                "sha256:5afb8a0cdfee37d02d69bdf470ae9cbb1dee5d4788f9bc6cc8e54bd8c83fb096",
+                "sha256:854fb7d49f6b7f545d4f8354172b004dcda887ba0699def7112daf785ba3c9ce"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.2.1"
+        },
+        "jupyter-server-fileid": {
+            "hashes": [
+                "sha256:a12209bdef4f2f9d57051b7556a089299fb9f26b501f643946854220c955be14",
+                "sha256:ac36436611b281cebbb5b9936a6f4850271bb411e13a287780a022dd0d2c3bf7"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.6.0"
+        },
+        "jupyter-server-terminals": {
+            "hashes": [
+                "sha256:57ab779797c25a7ba68e97bcfb5d7740f2b5e8a83b5e8102b10438041a7eac5d",
+                "sha256:75779164661cec02a8758a5311e18bb8eb70c4e86c6b699403100f1585a12a36"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.4.4"
+        },
+        "jupyter-server-ydoc": {
+            "hashes": [
+                "sha256:18275ff1ce7e93bbda2301ca066273b3951fc50b0d9c8fc33788374134ad7920",
+                "sha256:ab10864708c81fa41ab9f2ed3626b54ff6926eaf14545d1d439714978dad6e9f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.6.1"
+        },
+        "jupyter-ydoc": {
+            "hashes": [
+                "sha256:3163bd4745eedd46d4bba6df52ab26be3c5c44c3a8aaf247635062486ea8f84f",
+                "sha256:596a9ae5986b59f8776c42430b5ad516405963574078ab801781933c9690be93"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.23.3"
+            "version": "==0.2.2"
         },
         "jupyterlab": {
             "hashes": [
-                "sha256:e02556c8ea1b386963c4b464e4618aee153c5416b07ab481425c817a033323a2",
-                "sha256:f433059fe0e12d75ea90a81a0b6721113bb132857e3ec2197780b6fe84cbcbde"
+                "sha256:ad6707dd0149b629d0ed5b56916cfcdb816b376c6af3190337faba09e27ea29e",
+                "sha256:aee98c174180e98a30470297d10b959e8e64f2288970c0de65f0a6d2b4807034"
             ],
             "index": "pypi",
-            "version": "==3.5.0"
+            "version": "==3.6.1"
         },
         "jupyterlab-pygments": {
             "hashes": [
@@ -633,19 +800,19 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:02862dcba01981fc0a0cb82198436301bb5a07ea59c05efeca59c8e9e9066309",
-                "sha256:61381c48268f0a8a84a27858b49db1b02e5aec6549e85e22fcdbf3eb14a2193a"
+                "sha256:51f6922e34f9f3db875051f4f7b57539a04ddd030f42d9ce6062dedf67bf7f2f",
+                "sha256:9aec21a2183bbedd9f91a86628355449575f1862d88b28ad5f905019d31e6c21"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.16.5"
+            "version": "==2.19.0"
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:4c9275daa6d20fc96c3aea45756ece7110850d035b0b93a6a40e918016b927da",
-                "sha256:9a568e022b8bb53ab23291f6ddb52f8002b789c2c5763378cbc882be1d619be8"
+                "sha256:a04a42e50231b355b7087e16a818f541e53589f7647144ea0344c4bf16f300e5",
+                "sha256:eeaecdeaf6c03afc960ddae201ced88d5979b4ca9c3891bcb8f6631af705f5ef"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.4"
+            "version": "==3.0.5"
         },
         "kiwisolver": {
             "hashes": [
@@ -745,96 +912,106 @@
         },
         "markupsafe": {
             "hashes": [
-                "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003",
-                "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88",
-                "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5",
-                "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7",
-                "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a",
-                "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603",
-                "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1",
-                "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135",
-                "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247",
-                "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6",
-                "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601",
-                "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77",
-                "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02",
-                "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e",
-                "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63",
-                "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f",
-                "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980",
-                "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b",
-                "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812",
-                "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff",
-                "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96",
-                "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1",
-                "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925",
-                "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a",
-                "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6",
-                "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e",
-                "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f",
-                "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4",
-                "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f",
-                "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3",
-                "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c",
-                "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a",
-                "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417",
-                "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a",
-                "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a",
-                "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37",
-                "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452",
-                "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933",
-                "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a",
-                "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"
+                "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed",
+                "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc",
+                "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2",
+                "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460",
+                "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7",
+                "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0",
+                "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1",
+                "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa",
+                "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03",
+                "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323",
+                "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65",
+                "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013",
+                "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036",
+                "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f",
+                "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4",
+                "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419",
+                "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2",
+                "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619",
+                "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a",
+                "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a",
+                "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd",
+                "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7",
+                "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666",
+                "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65",
+                "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859",
+                "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625",
+                "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff",
+                "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156",
+                "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd",
+                "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba",
+                "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f",
+                "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1",
+                "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094",
+                "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a",
+                "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513",
+                "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed",
+                "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d",
+                "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3",
+                "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147",
+                "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c",
+                "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603",
+                "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601",
+                "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a",
+                "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1",
+                "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d",
+                "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3",
+                "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54",
+                "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2",
+                "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6",
+                "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.1.1"
+            "version": "==2.1.2"
         },
         "matplotlib": {
             "hashes": [
-                "sha256:0844523dfaaff566e39dbfa74e6f6dc42e92f7a365ce80929c5030b84caa563a",
-                "sha256:0eda9d1b43f265da91fb9ae10d6922b5a986e2234470a524e6b18f14095b20d2",
-                "sha256:168093410b99f647ba61361b208f7b0d64dde1172b5b1796d765cd243cadb501",
-                "sha256:1836f366272b1557a613f8265db220eb8dd883202bbbabe01bad5a4eadfd0c95",
-                "sha256:19d61ee6414c44a04addbe33005ab1f87539d9f395e25afcbe9a3c50ce77c65c",
-                "sha256:252957e208c23db72ca9918cb33e160c7833faebf295aaedb43f5b083832a267",
-                "sha256:32d29c8c26362169c80c5718ce367e8c64f4dd068a424e7110df1dd2ed7bd428",
-                "sha256:380d48c15ec41102a2b70858ab1dedfa33eb77b2c0982cb65a200ae67a48e9cb",
-                "sha256:3964934731fd7a289a91d315919cf757f293969a4244941ab10513d2351b4e83",
-                "sha256:3cef89888a466228fc4e4b2954e740ce8e9afde7c4315fdd18caa1b8de58ca17",
-                "sha256:4426c74761790bff46e3d906c14c7aab727543293eed5a924300a952e1a3a3c1",
-                "sha256:5024b8ed83d7f8809982d095d8ab0b179bebc07616a9713f86d30cf4944acb73",
-                "sha256:52c2bdd7cd0bf9d5ccdf9c1816568fd4ccd51a4d82419cc5480f548981b47dd0",
-                "sha256:54fa9fe27f5466b86126ff38123261188bed568c1019e4716af01f97a12fe812",
-                "sha256:5ba73aa3aca35d2981e0b31230d58abb7b5d7ca104e543ae49709208d8ce706a",
-                "sha256:5e16dcaecffd55b955aa5e2b8a804379789c15987e8ebd2f32f01398a81e975b",
-                "sha256:5ecfc6559132116dedfc482d0ad9df8a89dc5909eebffd22f3deb684132d002f",
-                "sha256:74153008bd24366cf099d1f1e83808d179d618c4e32edb0d489d526523a94d9f",
-                "sha256:78ec3c3412cf277e6252764ee4acbdbec6920cc87ad65862272aaa0e24381eee",
-                "sha256:795ad83940732b45d39b82571f87af0081c120feff2b12e748d96bb191169e33",
-                "sha256:7f716b6af94dc1b6b97c46401774472f0867e44595990fe80a8ba390f7a0a028",
-                "sha256:83dc89c5fd728fdb03b76f122f43b4dcee8c61f1489e232d9ad0f58020523e1c",
-                "sha256:8a0ae37576ed444fe853709bdceb2be4c7df6f7acae17b8378765bd28e61b3ae",
-                "sha256:8a8dbe2cb7f33ff54b16bb5c500673502a35f18ac1ed48625e997d40c922f9cc",
-                "sha256:8a9d899953c722b9afd7e88dbefd8fb276c686c3116a43c577cfabf636180558",
-                "sha256:8d0068e40837c1d0df6e3abf1cdc9a34a6d2611d90e29610fa1d2455aeb4e2e5",
-                "sha256:9347cc6822f38db2b1d1ce992f375289670e595a2d1c15961aacbe0977407dfc",
-                "sha256:9f335e5625feb90e323d7e3868ec337f7b9ad88b5d633f876e3b778813021dab",
-                "sha256:b03fd10a1709d0101c054883b550f7c4c5e974f751e2680318759af005964990",
-                "sha256:b0ca2c60d3966dfd6608f5f8c49b8a0fcf76de6654f2eda55fc6ef038d5a6f27",
-                "sha256:b2604c6450f9dd2c42e223b1f5dca9643a23cfecc9fde4a94bb38e0d2693b136",
-                "sha256:ca0e7a658fbafcddcaefaa07ba8dae9384be2343468a8e011061791588d839fa",
-                "sha256:d0e9ac04065a814d4cf2c6791a2ad563f739ae3ae830d716d54245c2b96fead6",
-                "sha256:d50e8c1e571ee39b5dfbc295c11ad65988879f68009dd281a6e1edbc2ff6c18c",
-                "sha256:d840adcad7354be6f2ec28d0706528b0026e4c3934cc6566b84eac18633eab1b",
-                "sha256:e0bbee6c2a5bf2a0017a9b5e397babb88f230e6f07c3cdff4a4c4bc75ed7c617",
-                "sha256:e5afe0a7ea0e3a7a257907060bee6724a6002b7eec55d0db16fd32409795f3e1",
-                "sha256:e68be81cd8c22b029924b6d0ee814c337c0e706b8d88495a617319e5dd5441c3",
-                "sha256:ec9be0f4826cdb3a3a517509dcc5f87f370251b76362051ab59e42b6b765f8c4",
-                "sha256:f04f97797df35e442ed09f529ad1235d1f1c0f30878e2fe09a2676b71a8801e0",
-                "sha256:f41e57ad63d336fe50d3a67bb8eaa26c09f6dda6a59f76777a99b8ccd8e26aec"
+                "sha256:01b7f521a9a73c383825813af255f8c4485d1706e4f3e2ed5ae771e4403a40ab",
+                "sha256:11011c97d62c1db7bc20509572557842dbb8c2a2ddd3dd7f20501aa1cde3e54e",
+                "sha256:1183877d008c752d7d535396096c910f4663e4b74a18313adee1213328388e1e",
+                "sha256:12f999661589981e74d793ee2f41b924b3b87d65fd929f6153bf0f30675c59b1",
+                "sha256:1c235bf9be052347373f589e018988cad177abb3f997ab1a2e2210c41562cc0c",
+                "sha256:1f4d69707b1677560cd952544ee4962f68ff07952fb9069ff8c12b56353cb8c9",
+                "sha256:1fcc4cad498533d3c393a160975acc9b36ffa224d15a6b90ae579eacee5d8579",
+                "sha256:2787a16df07370dcba385fe20cdd0cc3cfaabd3c873ddabca78c10514c799721",
+                "sha256:29f17b7f2e068dc346687cbdf80b430580bab42346625821c2d3abf3a1ec5417",
+                "sha256:38d38cb1ea1d80ee0f6351b65c6f76cad6060bbbead015720ba001348ae90f0c",
+                "sha256:3f56a7252eee8f3438447f75f5e1148a1896a2756a92285fe5d73bed6deebff4",
+                "sha256:5223affa21050fb6118353c1380c15e23aedfb436bf3e162c26dc950617a7519",
+                "sha256:57ad1aee29043163374bfa8990e1a2a10ff72c9a1bfaa92e9c46f6ea59269121",
+                "sha256:59400cc9451094b7f08cc3f321972e6e1db4cd37a978d4e8a12824bf7fd2f03b",
+                "sha256:68d94a436f62b8a861bf3ace82067a71bafb724b4e4f9133521e4d8012420dd7",
+                "sha256:6adc441b5b2098a4b904bbf9d9e92fb816fef50c55aa2ea6a823fc89b94bb838",
+                "sha256:6d81b11ede69e3a751424b98dc869c96c10256b2206bfdf41f9c720eee86844c",
+                "sha256:73b93af33634ed919e72811c9703e1105185cd3fb46d76f30b7f4cfbbd063f89",
+                "sha256:77b384cee7ab8cf75ffccbfea351a09b97564fc62d149827a5e864bec81526e5",
+                "sha256:79e501eb847f4a489eb7065bb8d3187117f65a4c02d12ea3a19d6c5bef173bcc",
+                "sha256:809119d1cba3ece3c9742eb01827fe7a0e781ea3c5d89534655a75e07979344f",
+                "sha256:80c166a0e28512e26755f69040e6bf2f946a02ffdb7c00bf6158cca3d2b146e6",
+                "sha256:81b409b2790cf8d7c1ef35920f01676d2ae7afa8241844e7aa5484fdf493a9a0",
+                "sha256:994637e2995b0342699b396a320698b07cd148bbcf2dd2fa2daba73f34dd19f2",
+                "sha256:9ceebaf73f1a3444fa11014f38b9da37ff7ea328d6efa1652241fe3777bfdab9",
+                "sha256:9fb8fb19d03abf3c5dab89a8677e62c4023632f919a62b6dd1d6d2dbf42cd9f5",
+                "sha256:acc3b1a4bddbf56fe461e36fb9ef94c2cb607fc90d24ccc650040bfcc7610de4",
+                "sha256:bbddfeb1495484351fb5b30cf5bdf06b3de0bc4626a707d29e43dfd61af2a780",
+                "sha256:bbf269e1d24bc25247095d71c7a969813f7080e2a7c6fa28931a603f747ab012",
+                "sha256:bebcff4c3ed02c6399d47329f3554193abd824d3d53b5ca02cf583bcd94470e2",
+                "sha256:c3f08df2ac4636249b8bc7a85b8b82c983bef1441595936f62c2918370ca7e1d",
+                "sha256:ca94f0362f6b6f424b555b956971dcb94b12d0368a6c3e07dc7a40d32d6d873d",
+                "sha256:d00c248ab6b92bea3f8148714837937053a083ff03b4c5e30ed37e28fc0e7e56",
+                "sha256:d2cfaa7fd62294d945b8843ea24228a27c8e7c5b48fa634f3c168153b825a21b",
+                "sha256:d5f18430f5cfa5571ab8f4c72c89af52aa0618e864c60028f11a857d62200cba",
+                "sha256:debeab8e2ab07e5e3dac33e12456da79c7e104270d2b2d1df92b9e40347cca75",
+                "sha256:dfba7057609ca9567b9704626756f0142e97ec8c5ba2c70c6e7bd1c25ef99f06",
+                "sha256:e0a64d7cc336b52e90f59e6d638ae847b966f68582a7af041e063d568e814740",
+                "sha256:eb9421c403ffd387fbe729de6d9a03005bf42faba5e8432f4e51e703215b49fc",
+                "sha256:faff486b36530a836a6b4395850322e74211cd81fc17f28b4904e1bd53668e3e",
+                "sha256:ff2aa84e74f80891e6bcf292ebb1dd57714ffbe13177642d65fee25384a30894"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.6.2"
+            "version": "==3.6.3"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -846,17 +1023,18 @@
         },
         "memory-profiler": {
             "hashes": [
-                "sha256:6a12869511d6cebcb29b71ba26985675a58e16e06b3c523b49f67c5497a33d1c"
+                "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84",
+                "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0"
             ],
             "index": "pypi",
-            "version": "==0.60.0"
+            "version": "==0.61.0"
         },
         "mistune": {
             "hashes": [
-                "sha256:182cc5ee6f8ed1b807de6b7bb50155df7b66495412836b9a74c8fbdfc75fe36d",
-                "sha256:9ee0a66053e2267aba772c71e06891fa8f1af6d4b01d5e84e267b4570d4d9808"
+                "sha256:0246113cb2492db875c6be56974a7c893333bf26cd92891c85f63151cee09d34",
+                "sha256:bad7f5d431886fcbaf5f758118ecff70d31f75231b34024a1341120340a65ce8"
             ],
-            "version": "==2.0.4"
+            "version": "==2.0.5"
         },
         "munch": {
             "hashes": [
@@ -875,35 +1053,35 @@
         },
         "nbclassic": {
             "hashes": [
-                "sha256:c74d8a500f8e058d46b576a41e5bc640711e1032cf7541dde5f73ea49497e283",
-                "sha256:cbf05df5842b420d5cece0143462380ea9d308ff57c2dc0eb4d6e035b18fbfb3"
+                "sha256:32c235e1f22f4048f3b877d354c198202898797cf9c2085856827598cead001b",
+                "sha256:8e8ffce7582bb7a4baf11fa86a3d88b184e8e7df78eed4ead69f15aa4fc0e323"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.4.8"
+            "version": "==0.5.1"
         },
         "nbclient": {
             "hashes": [
-                "sha256:434c91385cf3e53084185334d675a0d33c615108b391e260915d1aa8e86661b8",
-                "sha256:a1d844efd6da9bc39d2209bf996dbd8e07bf0f36b796edfabaa8f8a9ab77c3aa"
+                "sha256:884a3f4a8c4fc24bb9302f263e0af47d97f0d01fe11ba714171b320c8ac09547",
+                "sha256:d97ac6257de2794f5397609df754fcbca1a603e94e924eb9b99787c031ae2e7c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.7.0"
+            "version": "==0.7.2"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:c9c0e4b26326f7658ebf4cda0acc591b9727c4e3ee3ede962f70c11833b71b40",
-                "sha256:f933e82fe48b9a421e4252249f6c0a9a9940dc555642b4729f3f1f526bb16779"
+                "sha256:495638c5e06005f4a5ce828d8a81d28e34f95c20f4384d5d7a22254b443836e7",
+                "sha256:a42c3ac137c64f70cbe4d763111bf358641ea53b37a01a5c202ed86374af5234"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.2.6"
+            "version": "==7.2.9"
         },
         "nbformat": {
             "hashes": [
-                "sha256:1b05ec2c552c2f1adc745f4eddce1eac8ca9ffd59bb9fd859e827eaa031319f9",
-                "sha256:1d4760c15c1a04269ef5caf375be8b98dd2f696e5eb9e603ec2bf091f9b0d3f3"
+                "sha256:22a98a6516ca216002b0a34591af5bcb8072ca6c63910baffc901cfa07fefbf0",
+                "sha256:4b021fca24d3a747bf4e626694033d792d594705829e5e35b14ee3369f9f6477"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.7.0"
+            "version": "==5.7.3"
         },
         "nest-asyncio": {
             "hashes": [
@@ -1061,11 +1239,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3",
-                "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"
+                "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2",
+                "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==22.0"
+            "version": "==23.0"
         },
         "pandas": {
             "hashes": [
@@ -1122,70 +1300,86 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:03150abd92771742d4a8cd6f2fa6246d847dcd2e332a18d0c15cc75bf6703040",
-                "sha256:073adb2ae23431d3b9bcbcff3fe698b62ed47211d0716b067385538a1b0f28b8",
-                "sha256:0b07fffc13f474264c336298d1b4ce01d9c5a011415b79d4ee5527bb69ae6f65",
-                "sha256:0b7257127d646ff8676ec8a15520013a698d1fdc48bc2a79ba4e53df792526f2",
-                "sha256:12ce4932caf2ddf3e41d17fc9c02d67126935a44b86df6a206cf0d7161548627",
-                "sha256:15c42fb9dea42465dfd902fb0ecf584b8848ceb28b41ee2b58f866411be33f07",
-                "sha256:18498994b29e1cf86d505edcb7edbe814d133d2232d256db8c7a8ceb34d18cef",
-                "sha256:1c7c8ae3864846fc95f4611c78129301e203aaa2af813b703c55d10cc1628535",
-                "sha256:22b012ea2d065fd163ca096f4e37e47cd8b59cf4b0fd47bfca6abb93df70b34c",
-                "sha256:276a5ca930c913f714e372b2591a22c4bd3b81a418c0f6635ba832daec1cbcfc",
-                "sha256:2e0918e03aa0c72ea56edbb00d4d664294815aa11291a11504a377ea018330d3",
-                "sha256:3033fbe1feb1b59394615a1cafaee85e49d01b51d54de0cbf6aa8e64182518a1",
-                "sha256:3168434d303babf495d4ba58fc22d6604f6e2afb97adc6a423e917dab828939c",
-                "sha256:32a44128c4bdca7f31de5be641187367fe2a450ad83b833ef78910397db491aa",
-                "sha256:3dd6caf940756101205dffc5367babf288a30043d35f80936f9bfb37f8355b32",
-                "sha256:40e1ce476a7804b0fb74bcfa80b0a2206ea6a882938eaba917f7a0f004b42502",
-                "sha256:41e0051336807468be450d52b8edd12ac60bebaa97fe10c8b660f116e50b30e4",
-                "sha256:4390e9ce199fc1951fcfa65795f239a8a4944117b5935a9317fb320e7767b40f",
-                "sha256:502526a2cbfa431d9fc2a079bdd9061a2397b842bb6bc4239bb176da00993812",
-                "sha256:51e0e543a33ed92db9f5ef69a0356e0b1a7a6b6a71b80df99f1d181ae5875636",
-                "sha256:57751894f6618fd4308ed8e0c36c333e2f5469744c34729a27532b3db106ee20",
-                "sha256:5d77adcd56a42d00cc1be30843d3426aa4e660cab4a61021dc84467123f7a00c",
-                "sha256:655a83b0058ba47c7c52e4e2df5ecf484c1b0b0349805896dd350cbc416bdd91",
-                "sha256:68943d632f1f9e3dce98908e873b3a090f6cba1cbb1b892a9e8d97c938871fbe",
-                "sha256:6c738585d7a9961d8c2821a1eb3dcb978d14e238be3d70f0a706f7fa9316946b",
-                "sha256:73bd195e43f3fadecfc50c682f5055ec32ee2c933243cafbfdec69ab1aa87cad",
-                "sha256:772a91fc0e03eaf922c63badeca75e91baa80fe2f5f87bdaed4280662aad25c9",
-                "sha256:77ec3e7be99629898c9a6d24a09de089fa5356ee408cdffffe62d67bb75fdd72",
-                "sha256:7db8b751ad307d7cf238f02101e8e36a128a6cb199326e867d1398067381bff4",
-                "sha256:801ec82e4188e935c7f5e22e006d01611d6b41661bba9fe45b60e7ac1a8f84de",
-                "sha256:82409ffe29d70fd733ff3c1025a602abb3e67405d41b9403b00b01debc4c9a29",
-                "sha256:828989c45c245518065a110434246c44a56a8b2b2f6347d1409c787e6e4651ee",
-                "sha256:829f97c8e258593b9daa80638aee3789b7df9da5cf1336035016d76f03b8860c",
-                "sha256:871b72c3643e516db4ecf20efe735deb27fe30ca17800e661d769faab45a18d7",
-                "sha256:89dca0ce00a2b49024df6325925555d406b14aa3efc2f752dbb5940c52c56b11",
-                "sha256:90fb88843d3902fe7c9586d439d1e8c05258f41da473952aa8b328d8b907498c",
-                "sha256:97aabc5c50312afa5e0a2b07c17d4ac5e865b250986f8afe2b02d772567a380c",
-                "sha256:9aaa107275d8527e9d6e7670b64aabaaa36e5b6bd71a1015ddd21da0d4e06448",
-                "sha256:9f47eabcd2ded7698106b05c2c338672d16a6f2a485e74481f524e2a23c2794b",
-                "sha256:a0a06a052c5f37b4ed81c613a455a81f9a3a69429b4fd7bb913c3fa98abefc20",
-                "sha256:ab388aaa3f6ce52ac1cb8e122c4bd46657c15905904b3120a6248b5b8b0bc228",
-                "sha256:ad58d27a5b0262c0c19b47d54c5802db9b34d38bbf886665b626aff83c74bacd",
-                "sha256:ae5331c23ce118c53b172fa64a4c037eb83c9165aba3a7ba9ddd3ec9fa64a699",
-                "sha256:af0372acb5d3598f36ec0914deed2a63f6bcdb7b606da04dc19a88d31bf0c05b",
-                "sha256:afa4107d1b306cdf8953edde0534562607fe8811b6c4d9a486298ad31de733b2",
-                "sha256:b03ae6f1a1878233ac620c98f3459f79fd77c7e3c2b20d460284e1fb370557d4",
-                "sha256:b0915e734b33a474d76c28e07292f196cdf2a590a0d25bcc06e64e545f2d146c",
-                "sha256:b4012d06c846dc2b80651b120e2cdd787b013deb39c09f407727ba90015c684f",
-                "sha256:b472b5ea442148d1c3e2209f20f1e0bb0eb556538690fa70b5e1f79fa0ba8dc2",
-                "sha256:b59430236b8e58840a0dfb4099a0e8717ffb779c952426a69ae435ca1f57210c",
-                "sha256:b90f7616ea170e92820775ed47e136208e04c967271c9ef615b6fbd08d9af0e3",
-                "sha256:b9a65733d103311331875c1dca05cb4606997fd33d6acfed695b1232ba1df193",
-                "sha256:bac18ab8d2d1e6b4ce25e3424f709aceef668347db8637c2296bcf41acb7cf48",
-                "sha256:bca31dd6014cb8b0b2db1e46081b0ca7d936f856da3b39744aef499db5d84d02",
-                "sha256:be55f8457cd1eac957af0c3f5ece7bc3f033f89b114ef30f710882717670b2a8",
-                "sha256:c7025dce65566eb6e89f56c9509d4f628fddcedb131d9465cacd3d8bac337e7e",
-                "sha256:c935a22a557a560108d780f9a0fc426dd7459940dc54faa49d83249c8d3e760f",
-                "sha256:dbb8e7f2abee51cef77673be97760abff1674ed32847ce04b4af90f610144c7b",
-                "sha256:e6ea6b856a74d560d9326c0f5895ef8050126acfdc7ca08ad703eb0081e82b74",
-                "sha256:ebf2029c1f464c59b8bdbe5143c79fa2045a581ac53679733d3a91d400ff9efb",
-                "sha256:f1ff2ee69f10f13a9596480335f406dd1f70c3650349e2be67ca3139280cade0"
+                "sha256:013016af6b3a12a2f40b704677f8b51f72cb007dac785a9933d5c86a72a7fe33",
+                "sha256:0845adc64fe9886db00f5ab68c4a8cd933ab749a87747555cec1c95acea64b0b",
+                "sha256:0884ba7b515163a1a05440a138adeb722b8a6ae2c2b33aea93ea3118dd3a899e",
+                "sha256:09b89ddc95c248ee788328528e6a2996e09eaccddeeb82a5356e92645733be35",
+                "sha256:0dd4c681b82214b36273c18ca7ee87065a50e013112eea7d78c7a1b89a739153",
+                "sha256:0e51f608da093e5d9038c592b5b575cadc12fd748af1479b5e858045fff955a9",
+                "sha256:0f3269304c1a7ce82f1759c12ce731ef9b6e95b6df829dccd9fe42912cc48569",
+                "sha256:16a8df99701f9095bea8a6c4b3197da105df6f74e6176c5b410bc2df2fd29a57",
+                "sha256:19005a8e58b7c1796bc0167862b1f54a64d3b44ee5d48152b06bb861458bc0f8",
+                "sha256:1b4b4e9dda4f4e4c4e6896f93e84a8f0bcca3b059de9ddf67dac3c334b1195e1",
+                "sha256:28676836c7796805914b76b1837a40f76827ee0d5398f72f7dcc634bae7c6264",
+                "sha256:2968c58feca624bb6c8502f9564dd187d0e1389964898f5e9e1fbc8533169157",
+                "sha256:3f4cc516e0b264c8d4ccd6b6cbc69a07c6d582d8337df79be1e15a5056b258c9",
+                "sha256:3fa1284762aacca6dc97474ee9c16f83990b8eeb6697f2ba17140d54b453e133",
+                "sha256:43521ce2c4b865d385e78579a082b6ad1166ebed2b1a2293c3be1d68dd7ca3b9",
+                "sha256:451f10ef963918e65b8869e17d67db5e2f4ab40e716ee6ce7129b0cde2876eab",
+                "sha256:46c259e87199041583658457372a183636ae8cd56dbf3f0755e0f376a7f9d0e6",
+                "sha256:46f39cab8bbf4a384ba7cb0bc8bae7b7062b6a11cfac1ca4bc144dea90d4a9f5",
+                "sha256:519e14e2c49fcf7616d6d2cfc5c70adae95682ae20f0395e9280db85e8d6c4df",
+                "sha256:53dcb50fbdc3fb2c55431a9b30caeb2f7027fcd2aeb501459464f0214200a503",
+                "sha256:54614444887e0d3043557d9dbc697dbb16cfb5a35d672b7a0fcc1ed0cf1c600b",
+                "sha256:575d8912dca808edd9acd6f7795199332696d3469665ef26163cd090fa1f8bfa",
+                "sha256:5dd5a9c3091a0f414a963d427f920368e2b6a4c2f7527fdd82cde8ef0bc7a327",
+                "sha256:5f532a2ad4d174eb73494e7397988e22bf427f91acc8e6ebf5bb10597b49c493",
+                "sha256:60e7da3a3ad1812c128750fc1bc14a7ceeb8d29f77e0a2356a8fb2aa8925287d",
+                "sha256:653d7fb2df65efefbcbf81ef5fe5e5be931f1ee4332c2893ca638c9b11a409c4",
+                "sha256:6663977496d616b618b6cfa43ec86e479ee62b942e1da76a2c3daa1c75933ef4",
+                "sha256:6abfb51a82e919e3933eb137e17c4ae9c0475a25508ea88993bb59faf82f3b35",
+                "sha256:6c6b1389ed66cdd174d040105123a5a1bc91d0aa7059c7261d20e583b6d8cbd2",
+                "sha256:6d9dfb9959a3b0039ee06c1a1a90dc23bac3b430842dcb97908ddde05870601c",
+                "sha256:765cb54c0b8724a7c12c55146ae4647e0274a839fb6de7bcba841e04298e1011",
+                "sha256:7a21222644ab69ddd9967cfe6f2bb420b460dae4289c9d40ff9a4896e7c35c9a",
+                "sha256:7ac7594397698f77bce84382929747130765f66406dc2cd8b4ab4da68ade4c6e",
+                "sha256:7cfc287da09f9d2a7ec146ee4d72d6ea1342e770d975e49a8621bf54eaa8f30f",
+                "sha256:83125753a60cfc8c412de5896d10a0a405e0bd88d0470ad82e0869ddf0cb3848",
+                "sha256:847b114580c5cc9ebaf216dd8c8dbc6b00a3b7ab0131e173d7120e6deade1f57",
+                "sha256:87708d78a14d56a990fbf4f9cb350b7d89ee8988705e58e39bdf4d82c149210f",
+                "sha256:8a2b5874d17e72dfb80d917213abd55d7e1ed2479f38f001f264f7ce7bae757c",
+                "sha256:8f127e7b028900421cad64f51f75c051b628db17fb00e099eb148761eed598c9",
+                "sha256:94cdff45173b1919350601f82d61365e792895e3c3a3443cf99819e6fbf717a5",
+                "sha256:99d92d148dd03fd19d16175b6d355cc1b01faf80dae93c6c3eb4163709edc0a9",
+                "sha256:9a3049a10261d7f2b6514d35bbb7a4dfc3ece4c4de14ef5876c4b7a23a0e566d",
+                "sha256:9d9a62576b68cd90f7075876f4e8444487db5eeea0e4df3ba298ee38a8d067b0",
+                "sha256:9e5f94742033898bfe84c93c831a6f552bb629448d4072dd312306bab3bd96f1",
+                "sha256:a1c2d7780448eb93fbcc3789bf3916aa5720d942e37945f4056680317f1cd23e",
+                "sha256:a2e0f87144fcbbe54297cae708c5e7f9da21a4646523456b00cc956bd4c65815",
+                "sha256:a4dfdae195335abb4e89cc9762b2edc524f3c6e80d647a9a81bf81e17e3fb6f0",
+                "sha256:a96e6e23f2b79433390273eaf8cc94fec9c6370842e577ab10dabdcc7ea0a66b",
+                "sha256:aabdab8ec1e7ca7f1434d042bf8b1e92056245fb179790dc97ed040361f16bfd",
+                "sha256:b222090c455d6d1a64e6b7bb5f4035c4dff479e22455c9eaa1bdd4c75b52c80c",
+                "sha256:b52ff4f4e002f828ea6483faf4c4e8deea8d743cf801b74910243c58acc6eda3",
+                "sha256:b70756ec9417c34e097f987b4d8c510975216ad26ba6e57ccb53bc758f490dab",
+                "sha256:b8c2f6eb0df979ee99433d8b3f6d193d9590f735cf12274c108bd954e30ca858",
+                "sha256:b9b752ab91e78234941e44abdecc07f1f0d8f51fb62941d32995b8161f68cfe5",
+                "sha256:ba6612b6548220ff5e9df85261bddc811a057b0b465a1226b39bfb8550616aee",
+                "sha256:bd752c5ff1b4a870b7661234694f24b1d2b9076b8bf337321a814c612665f343",
+                "sha256:c3c4ed2ff6760e98d262e0cc9c9a7f7b8a9f61aa4d47c58835cdaf7b0b8811bb",
+                "sha256:c5c1362c14aee73f50143d74389b2c158707b4abce2cb055b7ad37ce60738d47",
+                "sha256:cb362e3b0976dc994857391b776ddaa8c13c28a16f80ac6522c23d5257156bed",
+                "sha256:d197df5489004db87d90b918033edbeee0bd6df3848a204bca3ff0a903bef837",
+                "sha256:d3b56206244dc8711f7e8b7d6cad4663917cd5b2d950799425076681e8766286",
+                "sha256:d5b2f8a31bd43e0f18172d8ac82347c8f37ef3e0b414431157718aa234991b28",
+                "sha256:d7081c084ceb58278dd3cf81f836bc818978c0ccc770cbbb202125ddabec6628",
+                "sha256:db74f5562c09953b2c5f8ec4b7dfd3f5421f31811e97d1dbc0a7c93d6e3a24df",
+                "sha256:df41112ccce5d47770a0c13651479fbcd8793f34232a2dd9faeccb75eb5d0d0d",
+                "sha256:e1339790c083c5a4de48f688b4841f18df839eb3c9584a770cbd818b33e26d5d",
+                "sha256:e621b0246192d3b9cb1dc62c78cfa4c6f6d2ddc0ec207d43c0dedecb914f152a",
+                "sha256:e8c5cf126889a4de385c02a2c3d3aba4b00f70234bfddae82a5eaa3ee6d5e3e6",
+                "sha256:e9d7747847c53a16a729b6ee5e737cf170f7a16611c143d95aa60a109a59c336",
+                "sha256:eaef5d2de3c7e9b21f1e762f289d17b726c2239a42b11e25446abf82b26ac132",
+                "sha256:ed3e4b4e1e6de75fdc16d3259098de7c6571b1a6cc863b1a49e7d3d53e036070",
+                "sha256:ef21af928e807f10bf4141cad4746eee692a0dd3ff56cfb25fce076ec3cc8abe",
+                "sha256:f09598b416ba39a8f489c124447b007fe865f786a89dbfa48bb5cf395693132a",
+                "sha256:f0caf4a5dcf610d96c3bd32932bfac8aee61c96e60481c2a0ea58da435e25acd",
+                "sha256:f6e78171be3fb7941f9910ea15b4b14ec27725865a73c15277bc39f5ca4f8391",
+                "sha256:f715c32e774a60a337b2bb8ad9839b4abf75b267a0f18806f6f4f5f1688c4b5a",
+                "sha256:fb5c1ad6bad98c57482236a21bf985ab0ef42bd51f7ad4e4538e89a997624e12"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==9.3.0"
+            "version": "==9.4.0"
         },
         "pkgutil-resolve-name": {
             "hashes": [
@@ -1195,13 +1389,29 @@
             "markers": "python_version < '3.9'",
             "version": "==1.3.10"
         },
+        "platformdirs": {
+            "hashes": [
+                "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9",
+                "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.0.0"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.0"
+        },
         "prometheus-client": {
             "hashes": [
-                "sha256:be26aa452490cfcf6da953f9436e95a9f2b4d578ca80094b4458930e5f584ab1",
-                "sha256:db7c05cbd13a0f79975592d112320f2605a325969b270a94b71dcabc47b931d2"
+                "sha256:0836af6eb2c8f4fed712b2f279f6c0a8bbab29f9f4aa15276b91c7cb0d1616ab",
+                "sha256:a03e35b359f14dd1630898543e2120addfdeacd1a6069c1367ae90fd93ad3f48"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.15.0"
+            "version": "==0.16.0"
         },
         "prompt-toolkit": {
             "hashes": [
@@ -1318,11 +1528,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
-                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
+                "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297",
+                "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.13.0"
+            "version": "==2.14.0"
         },
         "pyparsing": {
             "hashes": [
@@ -1359,31 +1569,44 @@
         },
         "pyrsistent": {
             "hashes": [
-                "sha256:055ab45d5911d7cae397dc418808d8802fb95262751872c841c170b0dbf51eed",
-                "sha256:111156137b2e71f3a9936baf27cb322e8024dac3dc54ec7fb9f0bcf3249e68bb",
-                "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a",
-                "sha256:21455e2b16000440e896ab99e8304617151981ed40c29e9507ef1c2e4314ee95",
-                "sha256:2aede922a488861de0ad00c7630a6e2d57e8023e4be72d9d7147a9fcd2d30712",
-                "sha256:3ba4134a3ff0fc7ad225b6b457d1309f4698108fb6b35532d015dca8f5abed73",
-                "sha256:456cb30ca8bff00596519f2c53e42c245c09e1a4543945703acd4312949bfd41",
-                "sha256:71d332b0320642b3261e9fee47ab9e65872c2bd90260e5d225dabeed93cbd42b",
-                "sha256:879b4c2f4d41585c42df4d7654ddffff1239dc4065bc88b745f0341828b83e78",
-                "sha256:9cd3e9978d12b5d99cbdc727a3022da0430ad007dacf33d0bf554b96427f33ab",
-                "sha256:a178209e2df710e3f142cbd05313ba0c5ebed0a55d78d9945ac7a4e09d923308",
-                "sha256:b39725209e06759217d1ac5fcdb510e98670af9e37223985f330b611f62e7425",
-                "sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2",
-                "sha256:bfd880614c6237243ff53a0539f1cb26987a6dc8ac6e66e0c5a40617296a045e",
-                "sha256:c43bec251bbd10e3cb58ced80609c5c1eb238da9ca78b964aea410fb820d00d6",
-                "sha256:d690b18ac4b3e3cab73b0b7aa7dbe65978a172ff94970ff98d82f2031f8971c2",
-                "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a",
-                "sha256:dec3eac7549869365fe263831f576c8457f6c833937c68542d08fde73457d291",
-                "sha256:e371b844cec09d8dc424d940e54bba8f67a03ebea20ff7b7b0d56f526c71d584",
-                "sha256:e5d8f84d81e3729c3b506657dddfe46e8ba9c330bf1858ee33108f8bb2adb38a",
-                "sha256:ea6b79a02a28550c98b6ca9c35b9f492beaa54d7c5c9e9949555893c8a9234d0",
-                "sha256:f1258f4e6c42ad0b20f9cfcc3ada5bd6b83374516cd01c0960e3cb75fdca6770"
+                "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8",
+                "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440",
+                "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a",
+                "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c",
+                "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3",
+                "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393",
+                "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9",
+                "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da",
+                "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf",
+                "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64",
+                "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a",
+                "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3",
+                "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98",
+                "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2",
+                "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8",
+                "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf",
+                "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc",
+                "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7",
+                "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28",
+                "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2",
+                "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b",
+                "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a",
+                "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64",
+                "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19",
+                "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1",
+                "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9",
+                "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.19.2"
+            "version": "==0.19.3"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5",
+                "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"
+            ],
+            "index": "pypi",
+            "version": "==7.2.1"
         },
         "python-dateutil": {
             "hashes": [
@@ -1395,98 +1618,155 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5",
-                "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"
+                "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49",
+                "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"
             ],
             "index": "pypi",
-            "version": "==0.21.0"
+            "version": "==0.21.1"
+        },
+        "python-json-logger": {
+            "hashes": [
+                "sha256:3b03487b14eb9e4f77e4fc2a023358b5394b82fd89cecf5586259baed57d8c6f",
+                "sha256:764d762175f99fcc4630bd4853b09632acb60a6224acb27ce08cd70f0b1b81bd"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.4"
         },
         "pytz": {
             "hashes": [
-                "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427",
-                "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"
+                "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0",
+                "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"
+            ],
+            "version": "==2022.7.1"
+        },
+        "pyyaml": {
+            "hashes": [
+                "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf",
+                "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293",
+                "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b",
+                "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57",
+                "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b",
+                "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4",
+                "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07",
+                "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba",
+                "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9",
+                "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287",
+                "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513",
+                "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0",
+                "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782",
+                "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0",
+                "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92",
+                "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f",
+                "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2",
+                "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc",
+                "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1",
+                "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c",
+                "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86",
+                "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4",
+                "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c",
+                "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34",
+                "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b",
+                "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d",
+                "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c",
+                "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb",
+                "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7",
+                "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737",
+                "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3",
+                "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d",
+                "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358",
+                "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53",
+                "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78",
+                "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803",
+                "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a",
+                "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f",
+                "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174",
+                "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"
             ],
-            "version": "==2022.6"
+            "markers": "python_version >= '3.6'",
+            "version": "==6.0"
         },
         "pyzmq": {
             "hashes": [
-                "sha256:0108358dab8c6b27ff6b985c2af4b12665c1bc659648284153ee501000f5c107",
-                "sha256:07bec1a1b22dacf718f2c0e71b49600bb6a31a88f06527dfd0b5aababe3fa3f7",
-                "sha256:0e8f482c44ccb5884bf3f638f29bea0f8dc68c97e38b2061769c4cb697f6140d",
-                "sha256:0ec91f1bad66f3ee8c6deb65fa1fe418e8ad803efedd69c35f3b5502f43bd1dc",
-                "sha256:0f14cffd32e9c4c73da66db97853a6aeceaac34acdc0fae9e5bbc9370281864c",
-                "sha256:15975747462ec49fdc863af906bab87c43b2491403ab37a6d88410635786b0f4",
-                "sha256:1724117bae69e091309ffb8255412c4651d3f6355560d9af312d547f6c5bc8b8",
-                "sha256:1a7c280185c4da99e0cc06c63bdf91f5b0b71deb70d8717f0ab870a43e376db8",
-                "sha256:1b7928bb7580736ffac5baf814097be342ba08d3cfdfb48e52773ec959572287",
-                "sha256:2032d9cb994ce3b4cba2b8dfae08c7e25bc14ba484c770d4d3be33c27de8c45b",
-                "sha256:20e7eeb1166087db636c06cae04a1ef59298627f56fb17da10528ab52a14c87f",
-                "sha256:216f5d7dbb67166759e59b0479bca82b8acf9bed6015b526b8eb10143fb08e77",
-                "sha256:28b119ba97129d3001673a697b7cce47fe6de1f7255d104c2f01108a5179a066",
-                "sha256:3104f4b084ad5d9c0cb87445cc8cfd96bba710bef4a66c2674910127044df209",
-                "sha256:3e6192dbcefaaa52ed81be88525a54a445f4b4fe2fffcae7fe40ebb58bd06bfd",
-                "sha256:42d4f97b9795a7aafa152a36fe2ad44549b83a743fd3e77011136def512e6c2a",
-                "sha256:44e706bac34e9f50779cb8c39f10b53a4d15aebb97235643d3112ac20bd577b4",
-                "sha256:47b11a729d61a47df56346283a4a800fa379ae6a85870d5a2e1e4956c828eedc",
-                "sha256:4854f9edc5208f63f0841c0c667260ae8d6846cfa233c479e29fdc85d42ebd58",
-                "sha256:48f721f070726cd2a6e44f3c33f8ee4b24188e4b816e6dd8ba542c8c3bb5b246",
-                "sha256:52afb0ac962963fff30cf1be775bc51ae083ef4c1e354266ab20e5382057dd62",
-                "sha256:54d8b9c5e288362ec8595c1d98666d36f2070fd0c2f76e2b3c60fbad9bd76227",
-                "sha256:5bd3d7dfd9cd058eb68d9a905dec854f86649f64d4ddf21f3ec289341386c44b",
-                "sha256:613010b5d17906c4367609e6f52e9a2595e35d5cc27d36ff3f1b6fa6e954d944",
-                "sha256:624321120f7e60336be8ec74a172ae7fba5c3ed5bf787cc85f7e9986c9e0ebc2",
-                "sha256:65c94410b5a8355cfcf12fd600a313efee46ce96a09e911ea92cf2acf6708804",
-                "sha256:6640f83df0ae4ae1104d4c62b77e9ef39be85ebe53f636388707d532bee2b7b8",
-                "sha256:687700f8371643916a1d2c61f3fdaa630407dd205c38afff936545d7b7466066",
-                "sha256:77c2713faf25a953c69cf0f723d1b7dd83827b0834e6c41e3fb3bbc6765914a1",
-                "sha256:78068e8678ca023594e4a0ab558905c1033b2d3e806a0ad9e3094e231e115a33",
-                "sha256:7a23ccc1083c260fa9685c93e3b170baba45aeed4b524deb3f426b0c40c11639",
-                "sha256:7abddb2bd5489d30ffeb4b93a428130886c171b4d355ccd226e83254fcb6b9ef",
-                "sha256:80093b595921eed1a2cead546a683b9e2ae7f4a4592bb2ab22f70d30174f003a",
-                "sha256:8242543c522d84d033fe79be04cb559b80d7eb98ad81b137ff7e0a9020f00ace",
-                "sha256:838812c65ed5f7c2bd11f7b098d2e5d01685a3f6d1f82849423b570bae698c00",
-                "sha256:83ea1a398f192957cb986d9206ce229efe0ee75e3c6635baff53ddf39bd718d5",
-                "sha256:8421aa8c9b45ea608c205db9e1c0c855c7e54d0e9c2c2f337ce024f6843cab3b",
-                "sha256:858375573c9225cc8e5b49bfac846a77b696b8d5e815711b8d4ba3141e6e8879",
-                "sha256:86de64468cad9c6d269f32a6390e210ca5ada568c7a55de8e681ca3b897bb340",
-                "sha256:87f7ac99b15270db8d53f28c3c7b968612993a90a5cf359da354efe96f5372b4",
-                "sha256:8bad8210ad4df68c44ff3685cca3cda448ee46e20d13edcff8909eba6ec01ca4",
-                "sha256:8bb4af15f305056e95ca1bd086239b9ebc6ad55e9f49076d27d80027f72752f6",
-                "sha256:8c78bfe20d4c890cb5580a3b9290f700c570e167d4cdcc55feec07030297a5e3",
-                "sha256:8f3f3154fde2b1ff3aa7b4f9326347ebc89c8ef425ca1db8f665175e6d3bd42f",
-                "sha256:94010bd61bc168c103a5b3b0f56ed3b616688192db7cd5b1d626e49f28ff51b3",
-                "sha256:941fab0073f0a54dc33d1a0460cb04e0d85893cb0c5e1476c785000f8b359409",
-                "sha256:9dca7c3956b03b7663fac4d150f5e6d4f6f38b2462c1e9afd83bcf7019f17913",
-                "sha256:a180dbd5ea5d47c2d3b716d5c19cc3fb162d1c8db93b21a1295d69585bfddac1",
-                "sha256:a2712aee7b3834ace51738c15d9ee152cc5a98dc7d57dd93300461b792ab7b43",
-                "sha256:a435ef8a3bd95c8a2d316d6e0ff70d0db524f6037411652803e118871d703333",
-                "sha256:abb756147314430bee5d10919b8493c0ccb109ddb7f5dfd2fcd7441266a25b75",
-                "sha256:abe6eb10122f0d746a0d510c2039ae8edb27bc9af29f6d1b05a66cc2401353ff",
-                "sha256:acbd0a6d61cc954b9f535daaa9ec26b0a60a0d4353c5f7c1438ebc88a359a47e",
-                "sha256:ae08ac90aa8fa14caafc7a6251bd218bf6dac518b7bff09caaa5e781119ba3f2",
-                "sha256:ae61446166983c663cee42c852ed63899e43e484abf080089f771df4b9d272ef",
-                "sha256:afe1f3bc486d0ce40abb0a0c9adb39aed3bbac36ebdc596487b0cceba55c21c1",
-                "sha256:b946da90dc2799bcafa682692c1d2139b2a96ec3c24fa9fc6f5b0da782675330",
-                "sha256:b947e264f0e77d30dcbccbb00f49f900b204b922eb0c3a9f0afd61aaa1cedc3d",
-                "sha256:bb5635c851eef3a7a54becde6da99485eecf7d068bd885ac8e6d173c4ecd68b0",
-                "sha256:bcbebd369493d68162cddb74a9c1fcebd139dfbb7ddb23d8f8e43e6c87bac3a6",
-                "sha256:c31805d2c8ade9b11feca4674eee2b9cce1fec3e8ddb7bbdd961a09dc76a80ea",
-                "sha256:c8840f064b1fb377cffd3efeaad2b190c14d4c8da02316dae07571252d20b31f",
-                "sha256:ccb94342d13e3bf3ffa6e62f95b5e3f0bc6bfa94558cb37f4b3d09d6feb536ff",
-                "sha256:d66689e840e75221b0b290b0befa86f059fb35e1ee6443bce51516d4d61b6b99",
-                "sha256:dabf1a05318d95b1537fd61d9330ef4313ea1216eea128a17615038859da3b3b",
-                "sha256:db03704b3506455d86ec72c3358a779e9b1d07b61220dfb43702b7b668edcd0d",
-                "sha256:de4217b9eb8b541cf2b7fde4401ce9d9a411cc0af85d410f9d6f4333f43640be",
-                "sha256:df0841f94928f8af9c7a1f0aaaffba1fb74607af023a152f59379c01c53aee58",
-                "sha256:dfb992dbcd88d8254471760879d48fb20836d91baa90f181c957122f9592b3dc",
-                "sha256:e7e66b4e403c2836ac74f26c4b65d8ac0ca1eef41dfcac2d013b7482befaad83",
-                "sha256:e8012bce6836d3f20a6c9599f81dfa945f433dab4dbd0c4917a6fb1f998ab33d",
-                "sha256:f01de4ec083daebf210531e2cca3bdb1608dbbbe00a9723e261d92087a1f6ebc",
-                "sha256:f0d945a85b70da97ae86113faf9f1b9294efe66bd4a5d6f82f2676d567338b66",
-                "sha256:fa0ae3275ef706c0309556061185dd0e4c4cd3b7d6f67ae617e4e677c7a41e2e"
+                "sha256:00c94fd4c9dd3c95aace0c629a7fa713627a5c80c1819326b642adf6c4b8e2a2",
+                "sha256:01d53958c787cfea34091fcb8ef36003dbb7913b8e9f8f62a0715234ebc98b70",
+                "sha256:0282bba9aee6e0346aa27d6c69b5f7df72b5a964c91958fc9e0c62dcae5fdcdc",
+                "sha256:02f5cb60a7da1edd5591a15efa654ffe2303297a41e1b40c3c8942f8f11fc17c",
+                "sha256:0645b5a2d2a06fd8eb738018490c514907f7488bf9359c6ee9d92f62e844b76f",
+                "sha256:0a154ef810d44f9d28868be04641f837374a64e7449df98d9208e76c260c7ef1",
+                "sha256:0a90b2480a26aef7c13cff18703ba8d68e181facb40f78873df79e6d42c1facc",
+                "sha256:0e8d00228db627ddd1b418c7afd81820b38575f237128c9650365f2dd6ac3443",
+                "sha256:17e1cb97d573ea84d7cd97188b42ca6f611ab3ee600f6a75041294ede58e3d20",
+                "sha256:183e18742be3621acf8908903f689ec520aee3f08449bfd29f583010ca33022b",
+                "sha256:1f6116991568aac48b94d6d8aaed6157d407942ea385335a6ed313692777fb9d",
+                "sha256:20638121b0bdc80777ce0ec8c1f14f1ffec0697a1f88f0b564fa4a23078791c4",
+                "sha256:2754fa68da08a854f4816e05160137fa938a2347276471103d31e04bcee5365c",
+                "sha256:28bcb2e66224a7ac2843eb632e4109d6b161479e7a2baf24e37210461485b4f1",
+                "sha256:293a7c2128690f496057f1f1eb6074f8746058d13588389981089ec45d8fdc77",
+                "sha256:2a73af6504e0d2805e926abf136ebf536735a13c22f709be7113c2ec65b4bec3",
+                "sha256:2d05d904f03ddf1e0d83d97341354dfe52244a619b5a1440a5f47a5b3451e84e",
+                "sha256:2e7b87638ee30ab13230e37ce5331b3e730b1e0dda30120b9eeec3540ed292c8",
+                "sha256:3100dddcada66ec5940ed6391ebf9d003cc3ede3d320748b2737553019f58230",
+                "sha256:31e523d067ce44a04e876bed3ff9ea1ff8d1b6636d16e5fcace9d22f8c564369",
+                "sha256:3594c0ff604e685d7e907860b61d0e10e46c74a9ffca168f6e9e50ea934ee440",
+                "sha256:3670e8c5644768f214a3b598fe46378a4a6f096d5fb82a67dfd3440028460565",
+                "sha256:4046d03100aca266e70d54a35694cb35d6654cfbef633e848b3c4a8d64b9d187",
+                "sha256:4725412e27612f0d7d7c2f794d89807ad0227c2fc01dd6146b39ada49c748ef9",
+                "sha256:484c2c4ee02c1edc07039f42130bd16e804b1fe81c4f428e0042e03967f40c20",
+                "sha256:487305c2a011fdcf3db1f24e8814bb76d23bc4d2f46e145bc80316a59a9aa07d",
+                "sha256:4a1bc30f0c18444d51e9b0d0dd39e3a4e7c53ee74190bebef238cd58de577ea9",
+                "sha256:4c25c95416133942280faaf068d0fddfd642b927fb28aaf4ab201a738e597c1e",
+                "sha256:4cbb885f347eba7ab7681c450dee5b14aed9f153eec224ec0c3f299273d9241f",
+                "sha256:4d3d604fe0a67afd1aff906e54da557a5203368a99dcc50a70eef374f1d2abef",
+                "sha256:4e295f7928a31ae0f657e848c5045ba6d693fe8921205f408ca3804b1b236968",
+                "sha256:5049e75cc99db65754a3da5f079230fb8889230cf09462ec972d884d1704a3ed",
+                "sha256:5050f5c50b58a6e38ccaf9263a356f74ef1040f5ca4030225d1cb1a858c5b7b6",
+                "sha256:526f884a27e8bba62fe1f4e07c62be2cfe492b6d432a8fdc4210397f8cf15331",
+                "sha256:531866c491aee5a1e967c286cfa470dffac1e2a203b1afda52d62b58782651e9",
+                "sha256:5605621f2181f20b71f13f698944deb26a0a71af4aaf435b34dd90146092d530",
+                "sha256:58fc3ad5e1cfd2e6d24741fbb1e216b388115d31b0ca6670f894187f280b6ba6",
+                "sha256:60ecbfe7669d3808ffa8a7dd1487d6eb8a4015b07235e3b723d4b2a2d4de7203",
+                "sha256:610d2d112acd4e5501fac31010064a6c6efd716ceb968e443cae0059eb7b86de",
+                "sha256:6136bfb0e5a9cf8c60c6ac763eb21f82940a77e6758ea53516c8c7074f4ff948",
+                "sha256:62b9e80890c0d2408eb42d5d7e1fc62a5ce71be3288684788f74cf3e59ffd6e2",
+                "sha256:656281d496aaf9ca4fd4cea84e6d893e3361057c4707bd38618f7e811759103c",
+                "sha256:66509c48f7446b640eeae24b60c9c1461799a27b1b0754e438582e36b5af3315",
+                "sha256:6bf3842af37af43fa953e96074ebbb5315f6a297198f805d019d788a1021dbc8",
+                "sha256:731b208bc9412deeb553c9519dca47136b5a01ca66667cafd8733211941b17e4",
+                "sha256:75243e422e85a62f0ab7953dc315452a56b2c6a7e7d1a3c3109ac3cc57ed6b47",
+                "sha256:7877264aa851c19404b1bb9dbe6eed21ea0c13698be1eda3784aab3036d1c861",
+                "sha256:81f99fb1224d36eb91557afec8cdc2264e856f3464500b55749020ce4c848ef2",
+                "sha256:8539216173135e9e89f6b1cc392e74e6b935b91e8c76106cf50e7a02ab02efe5",
+                "sha256:85456f0d8f3268eecd63dede3b99d5bd8d3b306310c37d4c15141111d22baeaf",
+                "sha256:866eabf7c1315ef2e93e34230db7cbf672e0d7c626b37c11f7e870c8612c3dcc",
+                "sha256:926236ca003aec70574754f39703528947211a406f5c6c8b3e50eca04a9e87fc",
+                "sha256:930e6ad4f2eaac31a3d0c2130619d25db754b267487ebc186c6ad18af2a74018",
+                "sha256:94f0a7289d0f5c80807c37ebb404205e7deb737e8763eb176f4770839ee2a287",
+                "sha256:9a2d5e419bd39a1edb6cdd326d831f0120ddb9b1ff397e7d73541bf393294973",
+                "sha256:9ca6db34b26c4d3e9b0728841ec9aa39484eee272caa97972ec8c8e231b20c7e",
+                "sha256:9f72ea279b2941a5203e935a4588b9ba8a48aeb9a926d9dfa1986278bd362cb8",
+                "sha256:a0e7ef9ac807db50b4eb6f534c5dcc22f998f5dae920cc28873d2c1d080a4fc9",
+                "sha256:a1cd4a95f176cdc0ee0a82d49d5830f13ae6015d89decbf834c273bc33eeb3d3",
+                "sha256:a9c464cc508177c09a5a6122b67f978f20e2954a21362bf095a0da4647e3e908",
+                "sha256:ac97e7d647d5519bcef48dd8d3d331f72975afa5c4496c95f6e854686f45e2d9",
+                "sha256:af1fbfb7ad6ac0009ccee33c90a1d303431c7fb594335eb97760988727a37577",
+                "sha256:b055a1cddf8035966ad13aa51edae5dc8f1bba0b5d5e06f7a843d8b83dc9b66b",
+                "sha256:b6f75b4b8574f3a8a0d6b4b52606fc75b82cb4391471be48ab0b8677c82f9ed4",
+                "sha256:b90bb8dfbbd138558f1f284fecfe328f7653616ff9a972433a00711d9475d1a9",
+                "sha256:be05504af0619d1cffa500af1e0ede69fb683f301003851f5993b5247cc2c576",
+                "sha256:c21a5f4e54a807df5afdef52b6d24ec1580153a6bcf0607f70a6e1d9fa74c5c3",
+                "sha256:c48f257da280b3be6c94e05bd575eddb1373419dbb1a72c3ce64e88f29d1cd6d",
+                "sha256:cac602e02341eaaf4edfd3e29bd3fdef672e61d4e6dfe5c1d065172aee00acee",
+                "sha256:ccb3e1a863222afdbda42b7ca8ac8569959593d7abd44f5a709177d6fa27d266",
+                "sha256:e1081d7030a1229c8ff90120346fb7599b54f552e98fcea5170544e7c6725aab",
+                "sha256:e14df47c1265356715d3d66e90282a645ebc077b70b3806cf47efcb7d1d630cb",
+                "sha256:e4bba04ea779a3d7ef25a821bb63fd0939142c88e7813e5bd9c6265a20c523a2",
+                "sha256:e99629a976809fe102ef73e856cf4b2660acd82a412a51e80ba2215e523dfd0a",
+                "sha256:f330a1a2c7f89fd4b0aa4dcb7bf50243bf1c8da9a2f1efc31daf57a2046b31f2",
+                "sha256:f3f96d452e9580cb961ece2e5a788e64abaecb1232a80e61deffb28e105ff84a",
+                "sha256:fc7c1421c5b1c916acf3128bf3cc7ea7f5018b58c69a6866d70c14190e600ce9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==24.0.1"
+            "version": "==25.0.0"
         },
         "qtconsole": {
             "hashes": [
@@ -1530,11 +1810,27 @@
         },
         "requests": {
             "hashes": [
-                "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
-                "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
+                "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa",
+                "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"
             ],
             "markers": "python_version >= '3.7' and python_version < '4'",
-            "version": "==2.28.1"
+            "version": "==2.28.2"
+        },
+        "rfc3339-validator": {
+            "hashes": [
+                "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b",
+                "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.1.4"
+        },
+        "rfc3986-validator": {
+            "hashes": [
+                "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9",
+                "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.1.1"
         },
         "richdem": {
             "hashes": [
@@ -1565,30 +1861,30 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31",
-                "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108",
-                "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0",
-                "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b",
-                "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e",
-                "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e",
-                "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5",
-                "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840",
-                "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58",
-                "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523",
-                "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd",
-                "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab",
-                "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c",
-                "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb",
-                "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096",
-                "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0",
-                "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc",
-                "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9",
-                "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c",
-                "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95",
-                "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.9.3"
+                "sha256:0490dc499fe23e4be35b8b6dd1e60a4a34f0c4adb30ac671e6332446b3cbbb5a",
+                "sha256:0ab2a58064836632e2cec31ca197d3695c86b066bc4818052b3f5381bfd2a728",
+                "sha256:151f066fe7d6653c3ffefd489497b8fa66d7316e3e0d0c0f7ff6acca1b802809",
+                "sha256:16ba05d3d1b9f2141004f3f36888e05894a525960b07f4c2bfc0456b955a00be",
+                "sha256:27e548276b5a88b51212b61f6dda49a24acf5d770dff940bd372b3f7ced8c6c2",
+                "sha256:2ad449db4e0820e4b42baccefc98ec772ad7818dcbc9e28b85aa05a536b0f1a2",
+                "sha256:2f9ea0a37aca111a407cb98aa4e8dfde6e5d9333bae06dfa5d938d14c80bb5c3",
+                "sha256:38bfbd18dcc69eeb589811e77fae552fa923067fdfbb2e171c9eac749885f210",
+                "sha256:3afcbddb4488ac950ce1147e7580178b333a29cd43524c689b2e3543a080a2c8",
+                "sha256:42ab8b9e7dc1ebe248e55f54eea5307b6ab15011a7883367af48dd781d1312e4",
+                "sha256:441cab2166607c82e6d7a8683779cb89ba0f475b983c7e4ab88f3668e268c143",
+                "sha256:4bd0e3278126bc882d10414436e58fa3f1eca0aa88b534fcbf80ed47e854f46c",
+                "sha256:4df25a28bd22c990b22129d3c637fd5c3be4b7c94f975dca909d8bab3309b694",
+                "sha256:5cd7a30970c29d9768a7164f564d1fbf2842bfc77b7d114a99bc32703ce0bf48",
+                "sha256:6e4497e5142f325a5423ff5fda2fff5b5d953da028637ff7c704378c8c284ea7",
+                "sha256:6faf86ef7717891195ae0537e48da7524d30bc3b828b30c9b115d04ea42f076f",
+                "sha256:954ff69d2d1bf666b794c1d7216e0a746c9d9289096a64ab3355a17c7c59db54",
+                "sha256:9b878c671655864af59c108c20e4da1e796154bd78c0ed6bb02bc41c84625686",
+                "sha256:b901b423c91281a974f6cd1c36f5c6c523e665b5a6d5e80fcb2334e14670eefd",
+                "sha256:c8b3cbc636a87a89b770c6afc999baa6bcbb01691b5ccbbc1b1791c7c0a07540",
+                "sha256:e096b062d2efdea57f972d232358cb068413dc54eec4f24158bcbb5cb8bddfd8"
+            ],
+            "markers": "python_version < '3.12' and python_version >= '3.8'",
+            "version": "==1.10.0"
         },
         "seaborn": {
             "hashes": [
@@ -1607,11 +1903,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54",
-                "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"
+                "sha256:16ccf598aab3b506593c17378473978908a2734d7336755a8769b480906bec1c",
+                "sha256:b440ee5f7e607bb8c9de15259dba2583dd41a38879a7abc1d43a71c59524da48"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==65.6.3"
+            "version": "==67.2.0"
         },
         "shapely": {
             "hashes": [
@@ -1640,70 +1936,82 @@
         },
         "simplejson": {
             "hashes": [
-                "sha256:002f069c7bb9a86826616a78f1214fea5b993435720990eecb0bf10955b9cd0e",
-                "sha256:00b673f0b3caf37a3d993bccf30a97290da6313b6ecc7d66937e9cd906d8f840",
-                "sha256:07e408222931b1a2aab71e60e5f169fa7c0d74cacd4e0a6a0199716cb18dad76",
-                "sha256:0de746c8f76355c79fd15eccd7ecde0b137cd911bdcdc463fc5c36ec3d8b98ea",
-                "sha256:0f33d16fa7b5e2ed6ea85d7b31bc84cf8c73c40cc2c9f87071e0fffcd52f5342",
-                "sha256:0f49858b5fc802081b71269f4a3aa5c5500ec6553637c9a0630f30a2a6541ea7",
-                "sha256:17dbc7f71fa5b7e4a2acef38cf0be30461ae6659456a978ce7eeebeb5bdf9e1a",
-                "sha256:17ec5e408fb6615250c1f18fb4eac3b2b99a85e8613bfc2dfa54827d0bf7f3e1",
-                "sha256:1b4085151e00ab7ca66f269aff7153f0ec18589cb22e7ceb8b365709c723fdd0",
-                "sha256:1f169402069f8cf93e359f607725b1d920c4dbe5bda4c520025d5fad8d20c1b7",
-                "sha256:1fbacdbba3cf5a471c67a9ca6cd270bba9578d5bc22aef6028faebbdb98bbb15",
-                "sha256:252f7cc5524bb5507a08377a4a75aa7ff4645f3dfca814d38bdbcf0f3c34d1ce",
-                "sha256:2aeed35db00cdf5d49ff1e7d878afd38c86a5fead0f1d364d539ad4d7a869e0e",
-                "sha256:2cc76435569e6c19574a8e913cfccbed832249b2b3b360caee9a4caf8ff866bf",
-                "sha256:448ab14fa67b3ac235a8445d14ec6d56268c3dabbce78720f9efa6d698466710",
-                "sha256:4609feb2ae66c132c6dcbe01dbfd4f6431afb4ff17303e37ca128fb6297cebd2",
-                "sha256:46bafa7e794f0e91fde850d906b0dc29a624c726b27e75d23bc8c3e35a48f28b",
-                "sha256:4a6199d302ec7d889e1aa6b493aa8e40b4dfa4bd85708f8c8f0c64ce5b8e0986",
-                "sha256:4d8d016f70d241f82189bc9f6d1eb8558b3599861f2c501b3f32da7fdf4e92ac",
-                "sha256:503da91993cc671fe7ebbf120c3ce868278de8226f158336afde874f7b7aa871",
-                "sha256:54c63cc7857f16a20aa170ffda9ebce45a3b7ba764b67a5a95bfe7ae613a2710",
-                "sha256:58a429d2c2fa80834115b923ff689622de8f214cf0dc4afa9f59e824b444ab31",
-                "sha256:599e9c53d3203bc36ef68efec138ca76d201da7ac06a114fae78536a8c10e35b",
-                "sha256:5f3dd31309ae5cc9f2df51d2d5cac89722dac3c853042ebefcaf7ad06ca19387",
-                "sha256:6187cbea7fdede732fe0347ad08cd920ebd9faa30b6c48782cee494051ca97c6",
-                "sha256:622cf0e1f870f189a0757fdcad7998a0c1dd46b0e53aeac9960556c141319c83",
-                "sha256:638bdd2deaccd3b8e02b1783280bd82341df5e1faa59c4f0276f03f16eec13ea",
-                "sha256:6804ad50aaf581df5c982fc101b0d932638066fe191074ded783602eb1c8982a",
-                "sha256:7a4d9b266ae6db578719f1255c742e76ee4676593087f4f6b79a2bbae2b1dcc5",
-                "sha256:7a9476dcd72aeba7d55c4800b9cd2204201af3539894b8512d74597e35a3033a",
-                "sha256:7b95c5cf71c16e4fdaa724719aaf8ccbed533e2df57a20bcff825ceeead27688",
-                "sha256:8493d2c1a940471b07d7c9c356a3f4eee780df073da2917418d0fe8669b54f99",
-                "sha256:875cfb43b622672218045dc927a86fc7c4c8111264c1d303aca5de33d5df479e",
-                "sha256:8d762267c4af617e1798bd0151f626105d06a88f214e3874b77eb89106f899fe",
-                "sha256:94c17d01e4c65e63deec46c984bb810de5e3a1259eb6bacdca63f3efc9c4c673",
-                "sha256:96979ff7f0daf47422d5f95d2d006da3210e0490a166bce2529f59f55047fc67",
-                "sha256:97139bf5134d713710665a6edb9500d69b93642c4b6b44b20800232dbd0f5b39",
-                "sha256:989b31d586954e65170ad3ec597218a6790c401b82da6193e8a897a06aa7946e",
-                "sha256:98b4c824f15436f1b22fe6d73c42ffacb246f7efc4d9dbbee542dd72355ecc43",
-                "sha256:9aff3c24017a7819c76b2f177d4fe8334b3d4cb6f702a2d7c666b3d57c36ffb4",
-                "sha256:9db78e18624f94d7b5642bf487244f803dab844e771d92e83f85f22da21ffe2d",
-                "sha256:a0e6dd5a0b8c76fb7522470789f1af793d39d6edbd4e40853e7be550ad49c430",
-                "sha256:a2f70d8170c7e02166a4c91462581e6ae5f35e3351a6b6c5142adcb04c7153ac",
-                "sha256:a814227fa08cae435ac7a42dcd2a04a7ec4a3cee23b7f83f9544cd26f452dcc4",
-                "sha256:aa9ecdd1d7ecbc7d1066c37cfbe52f65adf64b11b22d481a98fe1d3675dfff4b",
-                "sha256:b2b19d7aa4e9a1e7bf8caaf5f478a790190c60136314f45bb7702cb5a9337266",
-                "sha256:b4997bd8332cef3923402a07351571788f552f55ea1394ffbfccd4d203a8a05f",
-                "sha256:b71fef8ee41d59509c7f4afac7f627ed143c9e6db9eb08cfbba85e4c4dc5e67b",
-                "sha256:bd67d6fad7f4cd7c9cb7fad32d78ce32862fdb574b898447987a5de22fd37d73",
-                "sha256:ca22993a1a00440392c6c76f39addab8d97c706d2a8bcc2c9b2b6cb2cd7f41df",
-                "sha256:ce1c0580372d3c9bfa151bd0721a9bd5647b9b2245d0588d813fdbd2eb5d6f22",
-                "sha256:d522f28f7b252454df86ac3db5a0e1fe5ae03c8fc0cd1592c912b07c9fad6c29",
-                "sha256:d5d25cc5dad31a10d7a8196125515cc3aa68187c8953459fcaf127c2c8410f51",
-                "sha256:d9f7a692c11de20cb8ec680584815315e03d1404a6e299d36489b0fb6447d98d",
-                "sha256:d9fa2ad4cabb5054faa8d4a44b84134b0ec9d1421f5e9264d057d6be4d13c7fa",
-                "sha256:db53a85f4db0dbd9e5f6277d9153bcaa2ccb87b0d672c6a35f19432b3f2301a3",
-                "sha256:db9d36c4c7997c2a2513a5d218fd90b53bfeaf7e727f94aaf3576973378b3bce",
-                "sha256:e80f02e68d25c222471fcc5d1933275b8eb396e5e40b7863e4e0a43b3c810059",
-                "sha256:e84bd1c29e83ec74a95de070473742eb52d08502f2428eff5751671081e0a0a6",
-                "sha256:f0e12bdafdf7e32c5ad4a073e325ea0d659d4277af8b3d8eccf3101c56879619",
-                "sha256:fd56a9e0c63a1f9c37621fe298c77795aefd2a26dca80dcae27688586c40b4bb"
+                "sha256:04a4b9a297cccbc9e1d66fe652fbffd55b36d6579c43132e821d315957302194",
+                "sha256:063db62a9251e61ea0c17e49c3e7bed465bfcc5359655abcb8c0bc6130a4e0d4",
+                "sha256:070ab073ce72f1624107dfd6d095c87ac32aafe7ba54a5c5055a3dd83cb06e51",
+                "sha256:099bbd3b5b4ea83159a980348cd481a34984dee5fe1b9fac31a9137158f46960",
+                "sha256:0baf8c60efef74944ed4adb034d14bcf737731576f0e4c3c56fb875ea256af69",
+                "sha256:0e7c3fae6c9540064e06a653780b4f263675cd69ca6841345029fee3e27e9bb5",
+                "sha256:141782a0a25c1792627575b37b4951583358ccc7137623aa45947f8425ee8d96",
+                "sha256:14b35fb90083218e59df5dba733c7086655f2938f3fcabe36ad849623941d660",
+                "sha256:169c2c7446ef33439c304a6aa5b7b5a2dbc938c9c2dd882dd3f2553f9518ebf6",
+                "sha256:16cc750d19852fa5ebafd55da86fa357f87991e07b4e2afb37a5975dfdde0153",
+                "sha256:1907d49d70c75530976119c13785db91168d2599288debaca7d25da9cd2f3747",
+                "sha256:1b79e2607ac5ba98381c2e068727acc1e4dd385a6d216914c0613f8f568a06a5",
+                "sha256:1e49c84df6e71e3c23169d3df481565dd607cbee4aa1e0af15c493cccad7c745",
+                "sha256:23fce984045804194f513a2739dcd82be350198470d5ade5058da019a48cf3f8",
+                "sha256:24823364fee93bab141621b3a2e10612e31be7ca58788bf9b2cd2b1ce37ab07d",
+                "sha256:290bbcdcbb37af3f7e43378f592ab7a9168fca640da6af63d42cdb535f96bbf2",
+                "sha256:2a1b3222bc8f6ac91b5ebe3263111c7dc4dc4b01c52f0153f5bb1f3ef3bf0023",
+                "sha256:2b0f6de11f5ce4b80f51bc49d08b898602e190547f8efe4e44af8ae3cda7779d",
+                "sha256:2be75f4cb9951efeb2616e16f944ee4f9a09768475a3f5c40a6ac4dc5ee68dfd",
+                "sha256:2c7ee643ee93684bf76196e2d84a2090c6df8f01737a016e869b579593827b6e",
+                "sha256:37bdef13412c0bc338db2993a38f3911d5bd2a0ba8d00b3bc66d1063edd7c33e",
+                "sha256:3bab9ea49ff477c926c5787f79ec47cf51c7ffb15c9d8dd0f09e728807d44f4b",
+                "sha256:44d6c52d4f5c0c087a6e88a92bf9f94234321d21be32c6471ba39856e304bbe3",
+                "sha256:4b8d4d958c5ab3489d1174917a7fad82da642560c39ce559a624e63deaaa36b1",
+                "sha256:4de9fed1166aeedee44150fa83bc059aca6b612940281f8b5a39374781f16196",
+                "sha256:502d86fbfe914263642479b87ed61af3b27b9e039df77acd2416cfccfc892e68",
+                "sha256:508342d7227ed66beecfbba7a38b46e1a713faeb034216f43f03ec5c175e0622",
+                "sha256:50f4b6d52f3a2d1cffd11834a1fe7f9516f0e3f20cbe78027aa88ff990fad7d6",
+                "sha256:52465a5578cfc2c5e374a574df14dfb75e04c6cb6a100b7abc8bf6c89bea8f5e",
+                "sha256:55aa983575b0aef143845f5bfbb35075475eccaebf7d4b30f4037a2fe8414666",
+                "sha256:55df3dfd8777bf134e1078d2f195352432a77f23ccb90b92b08218123d56adc9",
+                "sha256:56f186d44a9f625b5e5d9ba4b9551e263604000a7df60cb373b3e789ca603b2a",
+                "sha256:5780e3929435a8d39671537174f8ce0ccafb4f6e0c748ffe139916ffbdca39d3",
+                "sha256:59a629240cfbc5b4f390a8578dca74ae77ab617de971862acb946822d2eb1b11",
+                "sha256:5b009342e712026ffabe8a471d5b4a4ff2a038687387e74eae601574c04dae33",
+                "sha256:62628ea5df8c830d00a7417d5ecd949a1b24a8d0a5063a2a77f7ec7522110a0f",
+                "sha256:694332fd6fd10fe8868c2508583220d1a1a7be9ff049dab5bd6b9aedfb9edc50",
+                "sha256:6a49665169c18f27a0fc10935466332ee7406ee14ced8dc0a1b4d465547299aa",
+                "sha256:6b997739fdbc9b7030ff490fc8e5f8c144b8ec80f3605eff643983672bb8cfde",
+                "sha256:6bd81d10cb3384f64242316da8a2b2f88618776bc1ef38bcc79f1afe8ad36616",
+                "sha256:6c4c56c5abb82e22877b913186e5c0fd7d9eef0c930719e28fa451d3f11defb4",
+                "sha256:6fe1173b4146641c872bafa6f9a21f3a2012f502d54fbb523a76e6320024fae9",
+                "sha256:75eb555dc349d0cbe2c95ea2be665b306c6ac6d5b64e3a3920af9b805ecdb5f7",
+                "sha256:7c26fe63755ecc59c502ddde8e58ce8b765bf4fdd3f5858d2b7c8ab28bc2a9c8",
+                "sha256:7e73d9d6af3c29b60a92e28b3144d951110f59a3d05fc402c3f6c5248b883400",
+                "sha256:7ff65b475091084e5bdb7f26e9c555956be7355b573ce494fa96f9f8e34541ac",
+                "sha256:8209c40279ed9b2cd5fbe2d617a29a074e90ea97fce7c07a0128a01cb3e8afc5",
+                "sha256:88f59a07873dc1f06fd9e6712dd71286f1b297a066ad2fd9110ad080d3cb011c",
+                "sha256:96ade36640734b54176c4765d00a60767bd7fae5b7a5b3574accc055ac18e34c",
+                "sha256:9cf299fbb7d476676dfea372a3262654af98694bd1df35b060ce0fe1b68087f1",
+                "sha256:a2960b95f3ba822d077d1afa7e1fea9799cfb2990028cf010e666f64195ecb5a",
+                "sha256:a80bd9a3db88a76a401155c64e3499376c702307c2206cb381cc2a8dd9cc4f1f",
+                "sha256:aad323e92cb1bd3b1db6f57c007dca964d13c52247ad844203ce381e94066601",
+                "sha256:ab5bdf0b8d07f7fd603b2d0c1982412cd9f8ade997088ddced251f7e656c7fd4",
+                "sha256:b0352428b35da859a98770949e7353866ae65463026f1c8e4c89a6395d4b5fd7",
+                "sha256:b2c4e8b65987f3c6529149495d28e23efe213e94dc3659176c4ab22d18a9ee4a",
+                "sha256:bcd9eac304a133ee4af58e68c5ded4c5ba663d3ee4602e8613359b776a1f8c8f",
+                "sha256:c3b696770b504f881f271f97b94a687487ec1ef20bfbd5f20d92bbab7a85952d",
+                "sha256:c4514675f6571da8190fea52a110bca686fa844972e8b2b3bc07ace9e632ee4f",
+                "sha256:c98fddc374468158778a8afb3fd7296412a2b2fc34cebba64212ac3e018e7382",
+                "sha256:cde5a3ff5e0bd5d6da676314dfae86c9e99bff77bca03d30223c9718a58f9e83",
+                "sha256:cf7168b2046db0eceb83d8ed2ee31c0847ce18b2d8baf3e93de9560f3921a8c3",
+                "sha256:d774782159347d66563cd7ac18b9dd37010438a825160cde4818caa18110a746",
+                "sha256:d990ea42ba908cb57a3df97d283aa26c1822f10a0a60e250b54ee21cd08c48d0",
+                "sha256:e762e9d8556fa9f3a99f8a278eeba50a35b5f554b82deeb282ddbdd85816e638",
+                "sha256:e8a4750e8db92109e6f1f7783a7faae4254d6d5dc28a41ff7eff7d2265f0586b",
+                "sha256:eb81cfef0c0039010f0212f4e5eb6909641b8a54c761584054ac97fd7bd0c21a",
+                "sha256:ebb53837c5ffcb6100646018565d3f1afed6f4b185b14b2c9cbccf874fe40157",
+                "sha256:efa70fd9b6c7b57b048ecadb909683acd535cddebc5b22f3c05ba3b369739caf",
+                "sha256:f73bae5e315adf7bc8cb7f0a13a1e9e33bead42e8ce174be83ac9ecc2513c86a",
+                "sha256:f89f078114cacedb9a3392615cc099cf02a51efa7507f90e2006bf7ec38c880d",
+                "sha256:f9f72d2b539512f382a48cc9ad6cea2d3a572e71e92c40e03d2140041eeaa233",
+                "sha256:fc8df5831b645e96a318ea51a66ce6e2bb869eebc3fa9a860bbf67aecd270055"
             ],
             "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.18.0"
+            "version": "==3.18.3"
         },
         "six": {
             "hashes": [
@@ -1794,7 +2102,7 @@
                 "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
                 "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version < '3.11'",
             "version": "==2.0.1"
         },
         "tornado": {
@@ -1824,26 +2132,40 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:61832ea7b7f910f5745e27e9bb269a181fd15af76027d99560299209d5b17c94",
-                "sha256:bd0fca5c890a09bf66b33cce67ca14156b080429bc39c7ef26b075a4bd4f9fc3"
+                "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8",
+                "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.7.0"
+            "version": "==5.9.0"
+        },
+        "uri-template": {
+            "hashes": [
+                "sha256:934e4d09d108b70eb8a24410af8615294d09d279ce0e7cbcdaef1bd21f932b06",
+                "sha256:f1699c77b73b925cf4937eae31ab282a86dc885c333f2e942513f08f691fc7db"
+            ],
+            "version": "==1.2.0"
         },
         "urllib3": {
             "hashes": [
-                "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc",
-                "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"
+                "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
+                "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.13"
+            "version": "==1.26.14"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
-                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
+                "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
+                "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"
+            ],
+            "version": "==0.2.6"
+        },
+        "webcolors": {
+            "hashes": [
+                "sha256:16d043d3a08fd6a1b1b7e3e9e62640d09790dce80d2bdd4792a175b35fe794a9",
+                "sha256:d98743d81d498a2d3eaf165196e65481f0d2ea85281463d856b1e51b09f62dce"
             ],
-            "version": "==0.2.5"
+            "version": "==1.12"
         },
         "webencodings": {
             "hashes": [
@@ -1854,19 +2176,19 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:d6b06432f184438d99ac1f456eaf22fe1ade524c3dd16e661142dc54e9cba574",
-                "sha256:d6e8f90ca8e2dd4e8027c4561adeb9456b54044312dba655e7cae652ceb9ae59"
+                "sha256:3f09e6d8230892547132177f575a4e3e73cfdf06526e20cc02aa1c3b47184d40",
+                "sha256:cdf5877568b7e83aa7cf2244ab56a3213de587bbe0ce9d8b9600fc77b455d89e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.4.2"
+            "version": "==1.5.1"
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:44c69f18237af0f610557d6c1c7ef76853f5856a0e604c0a517f2320566bb775",
-                "sha256:fa0e840719ec95dd2ec85c3a48913f1a0c29d323eacbcdb0b29bfed0cc6da678"
+                "sha256:003f716d930d385be3fd9de42dd9bf008e30053f73bddde235d14fbeaeff19af",
+                "sha256:eaaaf434fb9b08bd197b2a14ffe45ddb5ac3897593d43c69287091e5f3147bf7"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.0.4"
+            "version": "==4.0.5"
         },
         "xarray": {
             "hashes": [
@@ -1876,13 +2198,61 @@
             "index": "pypi",
             "version": "==0.19.0"
         },
+        "y-py": {
+            "hashes": [
+                "sha256:00c81e71abbbeab3b29ccc806176677a11397e637b2fd60c1fede24b0a7b4f32",
+                "sha256:0a14b9293a94a75d4659001fb6d7dba9c5386c8b752968f32a573830645c4e14",
+                "sha256:0e6ec7744ec1614fbcd5fdd9cfe2b1b7755c8fc9e2766782d0c3960b1f71fb4f",
+                "sha256:0edfa91505e3de0276daa44cf1f00a059c0cfc460e3fbb3f5188fefab30358ea",
+                "sha256:1258dd7ab1a03895b493bc6683d888adff8521b0d44d17b963be432395555f43",
+                "sha256:17dac9b53cc7c1c9845d6fb028e8f59b44da673f0dda9bb4a73eb86944181ce0",
+                "sha256:23285cb734a8036dbba07323a96f858e040b31fde888fc19daf2904ec62524f6",
+                "sha256:36558e6e117e70c7af47e5b07614abff481dd549d97394759ff513398add6363",
+                "sha256:3d318e922a8a193e2b7761594f0aa90a2e6d559dfdc96db4812995050887e9c5",
+                "sha256:3ff13618977bd24031387e32ba3c90e795992a5a8bac55707ff58e4c0839301d",
+                "sha256:400691bdf3be9ee950d73806a18fa81128b8ed3c74fde6b12f9d26ef311df877",
+                "sha256:410bfd166c8fcd6384655471e02449abc6051f0510d75954c3fa7f0aa43cf8aa",
+                "sha256:44a1e40582386309b873c6ab8a24e47f02fe2482359a85cf4b0a65353c00702c",
+                "sha256:52f03aa7bda1e862560fec043cc8580633dff811949d7df86ee3aa313d89ee78",
+                "sha256:6cd577e7a1b777c78127f6d89707b91b9a5935c632ebfa1791624d11c462a874",
+                "sha256:743b69a1920fa38696961d04e2921621f5794945b95e43797110830732c9f092",
+                "sha256:77e7a3e73e0fd71b016247a2d7dda21eeccb3ac53783c264fe13ea7801497084",
+                "sha256:822c04279e14a178c3726c9ccb16b447c5d03ebedb3410028f4aeb399f482d1f",
+                "sha256:8b0abfffed321b63850bbafb6223815db7336e86068f8ac841c2dbf5bf6326cd",
+                "sha256:90bcfd4de15063b41cb451911f67c0be2dafe701e66cf59c54c549ec309a3d48",
+                "sha256:95969ae8f9158cc8ba73ae631213e7030eac080330cb4aaa58a033932ee807a4",
+                "sha256:9be9a3981f4ed0983339b883d7eb1414a253ddcf248fb3b420e4d82da38b42e2",
+                "sha256:a15182ffc4d68659aea49374f7a7057e708dac8f49a9a5d8966482cef9c1cba2",
+                "sha256:a7668feef5734a769a5cff120b9bf79338b35dedbd5e6dfa99ac0c70640e11d4",
+                "sha256:a8ccd6baf2ac27e92a0c5bc05ee03528d897f9b86e2ae8fc27b04dcf0c462655",
+                "sha256:b36cd7b8d43782d932e7476b5381c13ecc02dcaabb988fdf920c2c30c8c93148",
+                "sha256:b4ccbd7e9e9ffab54bd5e1debe933dee495cf93783e1bfd0811ede9527a6c12d",
+                "sha256:df1fe3118d66127eacca14916d398099ff1016e2f003a5dab5f3fba4b449e7eb",
+                "sha256:df64934fe00bf10c1afb1a1b6ecf487b71fb233678faf075f4c0a9f384f99c70",
+                "sha256:e3b7d4150fb775a97d5454ca0e3eb2527ba83ec0e67a9244e4061abb7d91b16f",
+                "sha256:e66d9145d122339502ddb80d5aaed10592310c776b07f19f28e3391521cd19ec",
+                "sha256:f222bab71d8d3df9a40b2e5ab3a767d734c6ce11998e9a30a02fb83ab3e090b3",
+                "sha256:f4f1dd115c9493b4dc054521f85c69da15b9a036227199706757adf14013e3a7",
+                "sha256:f5b5ea683cb1cebb30c2a0a04a8b15cf01eccf4e1aee9406c97c8830d5913e64",
+                "sha256:f61d173bbe980b35f5720d94a9537bc09aec8621f8bba61d5f4db236679940d1"
+            ],
+            "version": "==0.5.5"
+        },
+        "ypy-websocket": {
+            "hashes": [
+                "sha256:491b2cc4271df4dde9be83017c15f4532b597dc43148472eb20c5aeb838a5b46",
+                "sha256:9049d5a7d61c26c2b5a39757c9ffcbe2274bf3553adeea8de7fe1c04671d4145"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.8.2"
+        },
         "zipp": {
             "hashes": [
-                "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa",
-                "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"
+                "sha256:6c4fe274b8f85ec73c37a8e4e3fa00df9fb9335da96fb789e3b96b318e5097b3",
+                "sha256:a3cac813d40993596b39ea9e93a18e8a2076d5c378b8bc88ec32ab264e04ad02"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.11.0"
+            "version": "==3.12.1"
         }
     },
     "develop": {
@@ -1917,11 +2287,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:352042ddcb019f7c04e48171b4dd78e4c4bb67bf97030d170e154aac42b656d9",
-                "sha256:882899fe78d5417a0aa07f995db298fa28b58faeba2112d2e3a4c95fe14bb738"
+                "sha256:71618e82e6d59487bea059626e7c79fb4a5b760d1510d02fab1160db6fdfa1f7",
+                "sha256:9c207b0ef2d276d1bfcfeb9a62804336abbe4b170574ea061500952319b1d78c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.7.0"
+            "version": "==8.9.0"
         },
         "jedi": {
             "hashes": [
@@ -1987,11 +2357,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
-                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
+                "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297",
+                "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.13.0"
+            "version": "==2.14.0"
         },
         "six": {
             "hashes": [
@@ -2010,18 +2380,18 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:61832ea7b7f910f5745e27e9bb269a181fd15af76027d99560299209d5b17c94",
-                "sha256:bd0fca5c890a09bf66b33cce67ca14156b080429bc39c7ef26b075a4bd4f9fc3"
+                "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8",
+                "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.7.0"
+            "version": "==5.9.0"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
-                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
+                "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
+                "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"
             ],
-            "version": "==0.2.5"
+            "version": "==0.2.6"
         }
     }
 }
diff --git a/config/params_template.env b/config/params_template.env
index 1895e6a25..f278139e5 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -31,20 +31,20 @@ export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from
 
 #### estimating bankfull stage in SRCs ####
 export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
-export bankfull_flows_file="data/inputs/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv" # input file location with nwm feature_id and recurrence flow values
+export bankfull_flows_file="/data/inputs/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv" # input file location with nwm feature_id and recurrence flow values
 
 #### applying channel/overbank subdivision routine to SRCs ####
 export src_subdiv_toggle="True" # Toggle to run composite roughness src routine (True=on; False=off)
 export vrough_suffix="" # text to append to output log and hydrotable file names (use for testing/debugging)
-export vmann_input_file="data/inputs/rating_curve/variable_roughness/mannings_global_06_12.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
+export vmann_input_file="/data/inputs/rating_curve/variable_roughness/mannings_global_06_12.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
 
 #### apply SRC adjustments using USGS rating curve database (calibration db) ####
 export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; False=off)
-export nwm_recur_file="data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
+export nwm_recur_file="/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
 
 #### apply SRC adjustments using observed FIM/flow point database ####
 export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
-export fim_obs_pnt_data="data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
+export fim_obs_pnt_data="/data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 #### path to env file with sensitive paths for accessing postgres database ####
 export CALB_DB_KEYS_FILE="/data/config/calb_db_keys.env"
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 661fc5e00..4deda8f5e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,65 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.1.2.0 - 2023-02-15 - [PR#808](https://github.com/NOAA-OWP/inundation-mapping/pull/808)
+
+Add `pytest` package and refactor existing unit tests. Update parameters to unit tests (`/unit_tests/*_params.json`) to valid paths. Add leading slash to paths in `/config/params_template.env`.
+
+### Additions
+
+- `/unit_tests`
+  - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
+  - `pyproject.toml`  - used to specify which warnings are excluded/filtered.
+  - `/gms`  
+    - `__init__.py` - needed for `pytest` command line executable to pick up tests.
+  - `/tools`
+    - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
+    - `inundate_gms_params.json` - file moved up into this directory
+    - `inundate_gms_test.py`     - file moved up into this directory
+    - `inundation_params.json`   - file moved up into this directory
+    - `inundation_test.py`       - file moved up into this directory
+
+### Removals
+
+- `/unit_tests/tools/gms_tools/` directory removed, and files moved up into `/unit_tests/tools`    
+ 
+### Changes
+
+- `Pipfile` - updated to include pytest as a dependency
+- `Pipfile.lock` - updated to include pytest as a dependency
+
+- `/config`
+  - `params_template.env` - leading slash added to paths
+  
+- `/unit_tests/` - All of the `*_test.py` files were refactored to follow the `pytest` paradigm.  
+  - `*_params.json` - valid paths on `fim-dev1` provided
+  - `README.md`  - updated to include documentation on pytest.  
+  - `unit_tests_utils.py`  
+  - `__template_unittests.py` -> `__template.py` - exclude the `_test` suffix to remove from test suite. Updated example on new format for pytest.
+  - `check_unit_errors_test.py`  
+  - `clip_vectors_to_wbd_test.py`  
+  - `filter_catchments_and_add_attributes_test.py`  
+  - `rating_curve_comparison_test.py`  
+  - `shared_functions_test.py`  
+  - `split_flow_test.py`  
+  - `usgs_gage_crosswalk_test.py`  
+  - `aggregate_branch_lists_test.py`  
+  - `generate_branch_list_test.py`  
+  - `generate_branch_list_csv_test.py`  
+  - `aggregate_branch_lists_test.py`  
+  - `generate_branch_list_csv_test.py`  
+  - `generate_branch_list_test.py`  
+    - `/gms`  
+      - `derive_level_paths_test.py`  
+      - `outputs_cleanup_test.py`
+    - `/tools`
+      - `inundate_unittests.py` -> `inundation_test.py`  
+      - `inundate_gms_test.py`
+
+
+<br/><br/>
+
+
 ## v4.1.1.0 - 2023-02-16 - [PR#809](https://github.com/NOAA-OWP/inundation-mapping/pull/809)
 
 The CatFIM code was updated to allow 1-foot interval processing across all stage-based AHPS sites ranging from action stage to 5 feet above major stage, along with restart capability for interrupted processing runs.
diff --git a/unit_tests/README.md b/unit_tests/README.md
index 037a22f7a..9725aa8cf 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -4,101 +4,141 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 
 #### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
 
-### This folder and files are for unit testing python files
+# This folder (`/unit_tests`) holds files for unit testing python files
 
 ## Creating unit tests
 
-For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and it's json parms file. 
+For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and an accompanying json paramerters file. 
 
 The files should be named following FIM convention:
 
-{source py file name}_unittests.py     ie) derive_level_paths_unittests.py
-{source py file name}_params.json      ie) derive_level_paths_params.json
+{source py file name}_test.py ->     `derive_level_paths_test.py`  
+{source py file name}_params.json -> `derive_level_paths_params.json`
 
 
-## Tips to creating a new json file for a new unit test file.
+## Tips to create a new json file for a new python unit test file.
 
 There are multiple way to figure out a set of default json parameters for the new unit test file. 
 
-One way is to use the incoming arg parser. Most py files, they include the code block of " __name__ == '__main__':", followed by external arg parsing, ie) the args = vars(parser.parse_args()). 
-	- Add a print(args) or similar, and get all the values including keys as output.
-	- Copy that into an editor being used to create the json file.
-	- Add a line break after every comma.
-	- Find/replace all single quotes to double quotes then cleanup the left tab formatting.
+One way is to use the incoming arg parser. Most python files include the code block of ` __name__ == '__main__':`, followed by external arg parsing (`args = vars(parser.parse_args()`). 
+* Add a `print(args)` or similar, and get all the values including keys as output.  
+* Copy that into an editor being used to create the json file.  
+* Add a line break after every comma.  
+* Find/replace all single quotes to double quotes then cleanup the left tab formatting.
 
 
 ## Running unit tests
 
-Start a docker container as you normally would for any development. ie) docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
-	- ie) docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
+Start a docker container as you normally would for any development. 
+```bash 
+docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
+```
+Example:
+```bash 
+docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
+```
 
 For unit tests to work, you need to run the following (if not already in place).
 Notice a modified branch "deny_gms_branch_unittests.lst"  (special for unittests)
 
 Here are the params and args you need if you need to re-run unit and branch
 
-gms_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -bd /foss_fim/config/deny_gms_branch_unittests.lst -ud None -j 1 -o
+```bash
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -bd /foss_fim/config/deny_gms_branch_unittests.lst -ud None -j 1 -o
+```
 
 **NOTICE: the deny file used for gms_run_branch... its a special one for unittests `deny_gms_branch_unittests.lst`.
 
 If you need to run inundation tests, fun the following:
 
+```bash
 python3 foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v fim_unit_test_data_do_not_remove -jh 1 -jb 1 -m /outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
+```
+### If you'd like to test the whole unit test suite:
+```
+pytest /foss_fim/unit_tests
+```
 
-If you want to test just one unit test, here is an example:
-At the root terminal window, run:  python ./foss_fim/unit_tests/gms/derive_level_paths_unittests.py  or python ./foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
-(replace with your own script and path name)
+This is not 100% stable, as accurate paths for the parameters `.json` files are not included in this repository, are not uniform accross machines, and are subject to change. 
+ 
+### If you want to test just one unit test (from the root terminal window):
 
-## Key Notes for creating new unit tests
-1) All test functions must start with the phrase "test_". That is how the unit test engine picks it up. The rest of the function name does not have to match the pattern of {function name being tested} but should. Further, the rest of the function name should say what the test is about, ie) _failed_input_path.  ie) test_{some_function_name_from_the_source_code_file}_failed_input_path. It is fine that the function names get very long (common in the industry).
+```bash
+pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py 
+						or  
+pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py
+```
 
-2) The output for a selected "unittest" import engine can be ugly and hard to read. It sometimes mixed outputs from multiple unit test functions simulataneously instead of keeping all output together for a given unit test. We will try to make this better later.
+### If you'd like to run a particular test, you can, for example:
+```
+pytest -v -s -k test_append_id_to_file_name_single_identifier_success
+```
 
-3) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
+If one test case is choosen, it will scan all of the test files, and scan for the method (test case) specified. 
 
-4) You can not control the order that unit tests are run within a unit test file. (UnitTest engine limitation)
+## Key Notes for creating new unit tests
+1) All test functions must start with the phrase `test_`. That is how pytest picks them up. The rest of the function name does not have to match the pattern of `function_name_being_tested` but should. Further, the rest of the function name should say what the test is about, ie) `_failed_input_path`.  ie) `test_{some_function_name_from_the_source_code_file}_failed_input_path`. It is fine that the function names get very long (common in the industry).
 
-5) There must be at one "{original py file name}_params.json" file.
+2) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
 
-6) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to full pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass. Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data"
+3) As of now, you can not control the order that unit tests are run within a unit test file. 
 
-7) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data", if one does not already exist. Generally, the individual unit tests, will call the "valid_data" node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie.. major different test conditions).
+4) There must be at least one associated `{original py file name}_params.json` file per unit test.
 
-8) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
+5) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to fully pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass.
 
-9) One py file = one "{original py file name}_unittests.py" file.
+6) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called `valid_data`, if one does not already exist. Generally, the individual unit tests, will call the `valid_data` node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie: major different test conditions).
 
-10) Sometimes you may want to run a full successful "happy path" version through gms_run_by_unit.sh (or similar), to get all of the files you need in place to do your testing. However.. you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the "-d" parameter (denylist). ie) Normally:  gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o, but ours would be 
-gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o. 
+7) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
 
-## Future Enhancements
-1) We can automate triggers on these files for things like checking triggers or an single global "run_all_unittest" script, but for now.. its one offs.
+8) One Python file = one `{original py file name}_test.py` file.
+
+9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist). 
+ie:
+```bash
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o
+```
+but ours would be:
+```bash 
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o
+```
+
+## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
 
-2) Better output from the unit tests including verbosity output control
+The `pyproject.toml` file has been added, which contains the build system requirements of Python projects.  This file used to specify which warnings are disabled to pass our unit tests. 
 
-3) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
+A `__init__.py` file has been added to both subdirectories (`/gms` & `/tools`) in order for the `pytest` command run in the `/unit_tests` to pick up the tests in those directories as well.
 
+Luckily, `pytest` works well with The Python Standard Library `unittest`. This made the migration of previous unit tests using `unittest` over to `pytest` quite simple. The caveat is that our current unit tests employ elements of both libraries. A full transition to `pytest` will ideally take place at a future date.
 
-## testing for failing conditions
-- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for `ONLY one variable value change`. Aka.. a "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
+## Testing for failing conditions
+- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for ONLY one variable value change. A "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
 
 - It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
 
 - When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass. 
 
-An example is in unit_tests/gms/Derive_level_paths_unittests.py -> test_Derive_level_paths_invalid_input_stream_network (function). It is incomplete but give you the pattern.
+An example is in `unit_tests/gms/Derive_level_paths_test.py` -> `test_Derive_level_paths_invalid_input_stream_network` (function). This example gives you the pattern implemented in Pytest.
 
-We have almost no "assert"s yet, but most unit test usually have one or more "assert" test. See https://docs.python.org/3/library/unittest.html for more details.
-
-## Unit tests currently available
-python3 /foss_fim/unit_tests/gms/derive_level_paths_unittests.py
-python3 /foss_fim/unit_tests/tools/inundate_unittests.py
-python3 /foss_fim/unit_tests/tools/gms_tools/inundate_gms_unittests.py
-python3 /foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
-python3 /foss_fim/unit_tests/filter_catchments_and_add_attributes_unittests.py
-python3 /foss_fim/unit_tests/rating_curve_comparison_unittests.py
-python3 /foss_fim/unit_tests/shared_functions_unittests.py
-python3 /foss_fim/unit_tests/split_flows_unittests.py
-python3 /foss_fim/unit_tests/usgs_gage_crosswalk_unittests.py
+## Future Enhancements
+1) Full transition to the `pytest` library, removing classes of `unittest.TestCase` and taking full advantage of available code re-use patterns offered through `pytest`.  
 
+2) Over time, it is expected that python files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each python file which makes unit testing harder and less specific. Generally function will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample .tif or small .gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: The files `/gms/derive_level_paths_test.py` and `clip_vectors_to_wbd_test.py` are not complete as they do not yet test all output from a method.
 
+## Unit tests currently available
+```
+pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py  
+pytest /foss_fim/unit_tests/gms/outputs_cleanup_test.py  
+pytest /foss_fim/unit_tests/tools/inundate_gms_test.py  
+pytest /foss_fim/unit_tests/tools/inundation_test.py  
+pytest /foss_fim/unit_tests/check_unit_errors_test.py  
+pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py  
+pytest /foss_fim/unit_tests/filter_catchments_and_add_attributes_test.py  
+pytest /foss_fim/unit_tests/rating_curve_comparison_test.py  
+pytest /foss_fim/unit_tests/shared_functions_test.py  
+pytest /foss_fim/unit_tests/split_flows_test.py  
+pytest /foss_fim/unit_tests/usgs_gage_crosswalk_test.py
+pytest /foss_fim/unit_tests/aggregate_branch_lists_test.py
+pytest /foss_fim/unit_tests/generate_branch_list_csv_test.py
+pytest /foss_fim/unit_tests/generate_branch_list_test.py
+```
diff --git a/unit_tests/__template_params.json b/unit_tests/__template.json
similarity index 72%
rename from unit_tests/__template_params.json
rename to unit_tests/__template.json
index a6ce3fce8..20d34f7c4 100644
--- a/unit_tests/__template_params.json
+++ b/unit_tests/__template.json
@@ -1,11 +1,11 @@
 {
 	"valid_data":
 	{
-		"some param name (suggested to match the method param name": "some value",
+		"some param name (suggested to match the method param name)": "some value",
 		"another one and as many lines as you need": "some value",
 		"for 'None', you can just leave this field out, but sometimes it seems like jsut passing the value of None seems to work (not validated) ": "some value",
 		"for numerics, just put the number": 20,
-	   "for lists/collections: ": ["value 1", "value 2"],
+	    "for lists/collections: ": ["value 1", "value 2"],
 		"inline json comments don't work (let us know if you figure it out)": "sorry"
 	}
-}
\ No newline at end of file
+}
diff --git a/unit_tests/__template.py b/unit_tests/__template.py
new file mode 100644
index 000000000..f3d6d483e
--- /dev/null
+++ b/unit_tests/__template.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import os
+import json
+import unittest
+import pytest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+import <Your original source python file name> as src
+
+class test_<Your original source python file name>(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+# Test Cases:
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_<method name you are about to test>_success(self):
+
+        '''
+        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
+          If there is no return value from the method, please say so.>
+        
+        Dev Notes: (which can be removed after you make this file)
+            Remember... You need to validate the method output if there is any. However, if you have time, it
+            is also recommended that you validate other outputs such as writing or updating file on the file 
+            system, aka: Does the expected file exist. Don't worry about its contents.
+        '''
+
+        #global params_file
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        
+        # for now we are happy if no exceptions are thrown.
+        
+        # See the readme.md, clip_vectors_to_wbd_test.py or gms/derive_level_paths_test.py for examples.
+        # Replace this stub example with your own.
+        # Try to use the same order to make it easier.
+        # Remember, if the method accepts **params, then you can sent that in here as well.
+        # ie: my_py_class.my_method(** params)
+        
+        src.subset_vector_layers(hucCode = params["hucCode"],
+                                 nwm_streams_filename = params["nwm_streams"],
+                                 etc, etc for each param)
+       
+        # This is what we are actually testing-
+        # An assert that evaluates as True passes, one that evaluates to False fails.
+        # A message (string) can be added after the assert statement to provide detail on the case being tested, and
+        # why it failed.
+        assert os.path.exists(params["nwm_streams"]) == True, "The nwm_streams file does not exist"
+    
+    
+    # EXAMPLE SUCCESSFUL TEST CASE WHICH CAPTURES AN EXCEPTION (FAILURE)   
+    
+    def test_subset_vector_layers_fail_invalid_stream_path(self):
+        '''
+        Notes about what the test is and the expected results (or expected exception if applicable)
+        '''
+
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+
+        params["nwm_streams"] = "/some/bad/path/"  
+
+        with pytest.raises(Exception) as e_info:
+            clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
+                                                     nwm_streams_filename = params["nwm_streams"],
+                                                     etc, etc for each param)
+
diff --git a/unit_tests/__template_unittests.py b/unit_tests/__template_unittests.py
deleted file mode 100644
index 44e960e59..000000000
--- a/unit_tests/__template_unittests.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import <Your original source python file name> as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_<Your original source python file name>(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
-    def test_<method name you are about to test>_success(self):
-
-        '''
-        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
-          If there is no return value from the method, please say so.>
-        
-        Dev Notes: (which can be removed after you make this file)
-            Remember... You need to validate the method output if there is any. However, if you have time, it
-            is also recommended that you validate other outputs such as writing or updating file on the file 
-            system, aka: Does the expected file exist. Don't worry about its contents.
-
-        '''
-
-        #global params_file
-        
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-        
-       
-        # for now we are happy if no exceptions are thrown.
-        
-        # < See the readme.md, clip_vectors_to_wbd_unittests.py or gms/derive_level_paths_unittests.py 
-        # for examples.>
-        # Replace this stub example with your own.
-        # Try to use the same order to make it easier.
-        # Remember, if the method accepts **params, then you can sent that in here as well.
-        #   ie: my_py_class.my_method(** params)
-        
-        src.subset_vector_layers(hucCode = params["hucCode"],
-                                    nwm_streams_filename = params["nwm_streams"],
-                                    etc, etc for each param)
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    # ***** REMOVE THIS BLOCK IF YOU ARE NOT USING IT ***
-    #       EXAMPLE SUCCESSFUL FAIL    
-    # 
-    #def test_subset_vector_layers_fail_invalid_stream_path(self):
-    #    '''
-    #    Notes about what the test is and the expected results (or expected exception if applicable)
-    #    '''
-    #   #global params_file
-        
-    #   params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-
-    #   params["nwm_streams"] = "some bad path"
-
-    #   clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
-    #                                            nwm_streams_filename = params["nwm_streams"],
-    #                                            etc, etc for each param)
-    #   
-       
-    #    print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-    #    print("*************************************************************")
-        
-    # ***********************
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/aggregate_branch_lists_params.json b/unit_tests/aggregate_branch_lists_params.json
index 9b9e4bcdd..fa5f36a29 100644
--- a/unit_tests/aggregate_branch_lists_params.json
+++ b/unit_tests/aggregate_branch_lists_params.json
@@ -1,8 +1,8 @@
 {
 	"valid_data":
 	{
-		"output_dir": "/outputs/rob_wb_3/",
+		"output_dir": "/outputs/fim_unit_test_data_do_not_remove/",
 		"file_name": "branch_ids.csv",
-		"output_file_name": "/outputs/rob_wb_3/test.csv"
+		"output_file_name": "/outputs/fim_unit_test_data_do_not_remove/test.csv"
 	}
 }
diff --git a/unit_tests/aggregate_branch_lists_unittests.py b/unit_tests/aggregate_branch_lists_test.py
similarity index 51%
rename from unit_tests/aggregate_branch_lists_unittests.py
rename to unit_tests/aggregate_branch_lists_test.py
index 72896e87e..e54cd5c85 100644
--- a/unit_tests/aggregate_branch_lists_unittests.py
+++ b/unit_tests/aggregate_branch_lists_test.py
@@ -1,20 +1,14 @@
 #!/usr/bin/env python3
 
-import inspect
-import os
-import sys
-
 import json
-import warnings
 import unittest
+import pytest
 
-sys.path.append('/foss_fim/unit_tests/')
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/') 
 import aggregate_branch_lists as src
 
+
 class test_aggregate_branch_lists(unittest.TestCase):
 
     '''
@@ -23,36 +17,19 @@ class test_aggregate_branch_lists(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        warnings.simplefilter('ignore')
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
+# Test Cases:
+
     def test_aggregate_branch_lists_success(self):
 
         #global params_file
         
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        params = self.params["valid_data"].copy()
 
         src.aggregate_branch_lists(output_dir = params["output_dir"],
                                    file_name = params["file_name"],
                                    output_file_name = params["output_file_name"])
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
 
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json
index d97d43084..d29b989af 100644
--- a/unit_tests/check_unit_errors_params.json
+++ b/unit_tests/check_unit_errors_params.json
@@ -1,11 +1,11 @@
 {
 	"valid_data":
 	{
-		"fim_dir": "/outputs/gms_example_unit_tests"
+		"fim_dir": "/data/outputs/fim_unit_test_data_do_not_remove"
 	},
 
 	"invalid_path":
 	{
-		"fim_dir": "/outputs/gms_example_unit_tests_not_not_valid"
+		"fim_dir": "/data/outputs/gms_example_unit_tests_not_not_valid"
 	}	
 }
\ No newline at end of file
diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py
new file mode 100644
index 000000000..b2229220a
--- /dev/null
+++ b/unit_tests/check_unit_errors_test.py
@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+
+import math
+import os
+import shutil
+import json
+import unittest
+import pytest
+
+from utils.fim_enums import FIM_exit_codes
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
+                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
+import check_unit_errors as src
+
+
+class test_check_unit_errors(unittest.TestCase):
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    
+    @classmethod
+    def setUpClass(self):
+        
+        # get_params_filename function in ./unit_test_utils handles errors
+        params_file_path = ut_helpers.get_params_filename(__file__)
+
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+    '''
+    To make most of this unit test, we have to do the following:
+    - rename the current unit_errors directory (if it exists)
+    - empty the unit_errors directory
+    - create a bunch of dummy files in it
+    - perform the unit test
+    - delete the unit_errors directory
+    - rename the original unit_test folder back to unit_tests
+    '''
+
+# Test Cases:
+
+    def test_check_unit_errors_success_below_min_errors(self):
+
+        # Expecting no errors.
+        # Test to ensure the number of dummy files is less than the overall min number of error files.
+
+        params = self.params["valid_data"].copy()
+        
+        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
+        ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"], 
+                                                                        num_dummy_files_reqd)
+        expected_output = 0
+        actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
+        
+        err_msg = "Number of dummy files IS NOT less than the overall min number of error files."
+        assert expected_output == actual_output, err_msg
+        
+        if (ue_folder_existed):
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+        
+            
+    def test_check_unit_errors_fail_above_min_errors(self):
+
+        # Test to ensure the number of dummy files is more than the overall min number of error files.
+        # Expecting sys.exit of 62
+        # We do expect this to fail and if it fails, it is successful.
+        # Here we expect an exception, and are capturing it using pytest.raises(Exception)
+        # To query the exception, or validate that it is the correct one, it is captured in the `e_info` object
+
+        params = self.params["valid_data"].copy()
+        
+        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD + 1
+        
+        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
+        
+        with pytest.raises(Exception) as e_info:
+            src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
+          
+        # We have to put the unit_errors folders back to the way it was
+        self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+
+    def test_check_unit_errors_success_above_percent_errors(self):
+
+        # Expecting no errors.
+        # Test to ensure the number of dummy files is more than the overall min number of error files.
+        # We do expect this not to to fail as it is greater than 10 errors but below the percent threshhold.
+
+        params = self.params["valid_data"].copy()
+        
+        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
+            
+        ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
+                                                                            num_dummy_files_reqd)
+            
+        num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
+        expected_output = 0
+        actual_output = src.check_unit_errors(params["fim_dir"], num_total_units)
+            
+        err_msg = "Number of dummy files IS NOT more than the overall min number of error files"
+        assert expected_output == actual_output, err_msg
+            
+        if (ue_folder_existed):
+            self.__remove_temp_unit_errors_folder(params["fim_dir"]) 
+            
+        # We have to put the unit_errors folders back to the way is was.
+        self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+
+    def test_check_unit_errors_fail_below_percent_errors(self):
+
+        # Expecting sys.exit of 62
+        # Test to ensure the number of dummy files is more than the overall min number of error files.
+        
+        # We do expect this to fail as it is greater than 10 errors
+        # AND below the percent threshhold (more percent errors than the threshold)
+        # Here we expect an exception, and are capturing it using pytest.raises(Exception)
+        # To query the exception, or validate that it is the correct one, it is captured in the `e_info` object
+
+        params = self.params["valid_data"].copy()
+        
+        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
+            
+        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
+            
+        num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) - 10
+
+        with pytest.raises(Exception) as e_info:
+            src.check_unit_errors(params["fim_dir"], num_total_units)
+          
+        # We have to put the unit_errors folders back to the way it was.
+        self.__remove_temp_unit_errors_folder(params["fim_dir"])
+    
+
+# Helper functions:
+
+    def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files):
+        '''
+        Process:
+            We want to preserve the original unit_errors folder if it exists,
+            so we wil rename it.
+            Then we will make a new unit_errors folder and fill it with a bunch of 
+            dummy files.
+            A dummy file for non_zero_exit_codes.log will also be created.
+        Input:
+            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
+            number_of_files: how many dummy files to create
+        Returns:
+            True if the 'unit_errors' folder did original exist and needs to be renamed back.
+            False if the 'unit_errors' folder never existed in the first place.
+        '''
+        
+        ue_folder_preexists = False
+        
+        if (not os.path.isdir(output_folder)):
+            raise Exception(f"unit test root folder of {output_folder} does not exist")
+        
+        ue_folder = os.path.join(output_folder, "unit_errors")
+        temp_ue_folder = ue_folder + "_temp"
+        if (os.path.isdir(ue_folder)):
+            ue_folder_preexists = True
+            os.rename(ue_folder, temp_ue_folder)
+        
+        os.mkdir(ue_folder)
+        
+        for i in range(0, number_of_files):
+            file_name = "sample_" + str(i) + ".txt"
+            file_path = os.path.join(ue_folder, file_name)
+            with open(file_path, 'w') as fp:
+                pass
+        
+        return ue_folder_preexists
+        
+        
+    def __remove_temp_unit_errors_folder(self, output_folder):
+        '''
+        Process:
+            We want to preserve the original unit_errors folder if it exists,
+            so we will delete our temp unit test version of 'unit_errors', and rename
+            the original back to 'unit_errors'
+            Note.. it is possible the temp folder does not exist,
+            but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
+        Input:
+            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
+        Returns:
+            nothing
+        '''
+
+        ue_folder = os.path.join(output_folder, "unit_errors")        
+        if (os.path.isdir(ue_folder)):
+            shutil.rmtree(ue_folder)       
+        
+        temp_ue_folder = ue_folder + "_temp"        
+        if (os.path.isdir(temp_ue_folder)):
+            os.rename(temp_ue_folder, ue_folder)
+    
+
+# Test Cases for Helper funcitons:
+
+    def test_create_temp_unit_errors_folder_files(self):
+
+        # Here we are testing our helper function to see if it raise exceptions appropriately with a bad path. 
+        # In this case, we want the exception to be raised if there is an invalid path. 
+
+        params = self.params["invalid_path"]
+        invalid_folder = params["fim_dir"]
+
+        with pytest.raises(Exception) as e_info:
+            self.__create_temp_unit_errors_folder_files(invalid_folder, 4)
+    
+    
+    def test_remove_temp_unit_errors_folder(self):
+        
+        # Test of out helper function to see if the temp folder was removed. 
+
+        params = self.params["valid_data"].copy()
+
+        self.__create_temp_unit_errors_folder_files(params["fim_dir"], 2)
+
+        self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+        temp_folder_created = os.path.join(params["fim_dir"], "unit_errors") + "_temp"
+
+        assert os.path.exists(temp_folder_created) == False
+
diff --git a/unit_tests/check_unit_errors_unittests.py b/unit_tests/check_unit_errors_unittests.py
deleted file mode 100644
index 92fc10640..000000000
--- a/unit_tests/check_unit_errors_unittests.py
+++ /dev/null
@@ -1,278 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import math
-import os
-import shutil
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import check_unit_errors as src
-from utils.fim_enums import FIM_exit_codes
-
-from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
-                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
-
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_check_unit_errors(unittest.TestCase):
-
-
-    # CURRENT 
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        try:        
-            params_file_path = ut_helpers.get_params_filename(__file__)
-        except FileNotFoundError as ex:
-            print(f"params file not found. ({ex}). Check pathing and file name convention.")
-            sys.exit(1)
-
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    '''
-    To make most of these unit test work, we have to do the following:
-    - rename the current unit_errors directory (if it exists)
-    - empty the unit_errors directory
-    - create a bunch of dummy files in it
-    - perform the unit test
-    - delete the unit_errors directory
-    - rename the original unit_test folder back to unit_tests
-    '''
-
-    def test_check_unit_errors_success_below_min_errors(self):
-
-        # Expecting no errors
-        # test to ensure the number of dummy files is less than the
-        # overall min number of error files.
-
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
-        
-            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
-                                                                            num_dummy_files_reqd)
-            expected_output = 0
-            actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-            
-            err_msg = "actual output does not match expected output"
-            self.assertEqual(expected_output, actual_output, err_msg)
-            
-            if (ue_folder_existed):
-                self.__remove_temp_unit_errors_folder(params["fim_dir"])
-            
-            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-            print("*************************************************************")
-            
-        except BaseException as ex:
-            
-            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
-            
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-            # re throw for the unit test engine            
-            raise ex
-
-
-    def test_check_unit_errors_fail_above_min_errors(self):
-
-        # test to ensure the number of dummy files is more than the
-        # overall min number of error files.
-
-        # Expecting sys.exit of 62
-        # we do expect this to fail and if it fails, it is successful
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD + 1
-        
-            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-            src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-            
-            raise Exception("Test Failed: expected sys.exit error but did not receive it")
-            
-        except BaseException as ex:
-           
-            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
-                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            else:
-                print("ex value is")
-                print(ex)
-                raise Exception("Expected a system exit code but did not receive it")
-          
-        finally:
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-
-    def test_check_unit_errors_success_above_percent_errors(self):
-
-        # Expecting no errors
-        # test to ensure the number of dummy files is more than the
-        # overall min number of error files.
-        
-        # we do expect this not to to fail as it is greater than 10 errors
-        # but below the percent threshhold
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-            
-            print(f"num_dummy_files_reqd is {num_dummy_files_reqd}")
-            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
-                                                                            num_dummy_files_reqd)
-            
-            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
-            expected_output = 0
-            actual_output = src.check_unit_errors(params["fim_dir"], num_total_units)
-            
-            err_msg = "actual output does not match expected output"
-            self.assertEqual(expected_output, actual_output, err_msg)
-            
-            if (ue_folder_existed):
-                self.__remove_temp_unit_errors_folder(params["fim_dir"])
-            
-            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-            print("*************************************************************")
-            
-        except BaseException as ex:
-            
-            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
-            
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-            # re throw for the unit test engine            
-            raise ex
-
-
-    def test_check_unit_errors_fail_below_percent_errors(self):
-
-        # Expecting sys.exit of 62
-        # test to ensure the number of dummy files is more than the
-        # overall min number of error files.
-        
-        # we do expect this to fail as it is greater than 10 errors
-        # AND below the percent threshhold (more percent errors than the threshold)
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-            
-            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-            
-            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) - 10
-            print(f" is {num_total_units}")
-
-            src.check_unit_errors(params["fim_dir"], num_total_units)
-            raise Exception("Test Failed: expected sys.exit error but did not receive it")
-                    
-        except BaseException as ex:
-           
-            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
-                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-                print("*************************************************************")                             
-            else:
-                print("ex value is")
-                print(ex)
-                raise Exception("Expected a system exit code but did not receive it")
-          
-        finally:
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-            
-
-    def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files):
-        '''
-        Process:
-            We want to preserve the original unit_errors folder if it exists,
-            so we wil rename it.
-            Then we will make a new unit_errors folder and fill it with a bunch of 
-            dummy files.
-            A dummy file for non_zero_exit_codes.log will also be created.
-        Input:
-            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
-            number_of_files: how many dummy files to create
-        Returns:
-            True if the 'unit_errors' folder did original exist and needs to be renamed back.
-            False if the 'unit_errors' folder never existed in the first place.
-        '''
-        
-        ue_folder_preexists = False
-        
-        if (not os.path.isdir(output_folder)):
-            raise Exception(f"unit test root folder of {output_folder} does not exist")
-        
-        ue_folder = os.path.join(output_folder, "unit_errors")
-        temp_ue_folder = ue_folder + "_temp"
-        if (os.path.isdir(ue_folder)):
-            ue_folder_preexists = True
-            os.rename(ue_folder, temp_ue_folder)
-        
-        os.mkdir(ue_folder)
-        
-        for i in range(0, number_of_files):
-            file_name = "sample_" + str(i) + ".txt"
-            file_path = os.path.join(ue_folder, file_name)
-            with open(file_path, 'w') as fp:
-                pass
-        
-        return ue_folder_preexists
-        
-        
-    def __remove_temp_unit_errors_folder(self, output_folder):
-        '''
-        Process:
-            We want to preserve the original unit_errors folder if it exists,
-            so we will delete our temp unit test version of 'unit_errors', and rename
-            the original back to 'unit_errors'
-            Note.. it is possible the temp folder does not exist,
-            but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
-        Input:
-            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
-        Returns:
-            nothing
-        '''
-
-        ue_folder = os.path.join(output_folder, "unit_errors")        
-        if (os.path.isdir(ue_folder)):
-            shutil.rmtree(ue_folder)       
-        
-        temp_ue_folder = ue_folder + "_temp"        
-        if (os.path.isdir(temp_ue_folder)):
-            os.rename(temp_ue_folder, ue_folder)
-      
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 04dc82c74..62f3dc137 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -1,12 +1,14 @@
 {
 	"valid_data": {
+		"outputRunDataDir":  "/data/outputs/fim_unit_test_data_do_not_remove",
 		"subset_nwm_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
 		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
 		"hucCode": "05030104",
-		"subset_nhd_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
+		"subset_nwm_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
 		"wbd_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
 		"wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
 		"dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
+		"dem_domain" : "/data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg",
 		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
 		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
 		"subset_nwm_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
@@ -14,12 +16,11 @@
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
 		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
 		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
-		"nhd_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
+		"nwm_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
 		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
-		"great_lakes": "/data/inputs/landsea/gl_water_polygons.gpkg",
-		"lake_buffer_distance": 20,
 		"wbd_buffer_distance": 5000,
 		"levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
-		"subset_levee_protected_areas": "/data/outputs/rob_test_2/05030104/LeveeProtectedAreas_subset.gpkg"
+		"subset_levee_protected_areas": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LeveeProtectedAreas_subset.gpkg"
 	}
-}
\ No newline at end of file
+}
+
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
new file mode 100644
index 000000000..66857c1a8
--- /dev/null
+++ b/unit_tests/clip_vectors_to_wbd_test.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+import os 
+import json
+import unittest
+import pytest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+import clip_vectors_to_wbd as src
+
+
+class test_clip_vectors_to_wbd(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+# Test Cases:
+
+    def test_subset_vector_layers_success(self):
+
+        '''
+        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
+        Most of the output test and internal tests with this function will test a wide variety of conditions.
+        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
+        Only the basic return output value should be tested to ensure it is as expected.
+        For now, we are adding the very basic "happy path" test.
+        '''
+
+        params = self.params["valid_data"].copy()
+       
+        # There is no default return value.
+        # Later we need to check in this function that files were being created on the file system
+        
+        # For now we are happy if no exceptions are thrown.
+        try:
+            src.subset_vector_layers(subset_nwm_lakes = params["subset_nwm_lakes"],
+                                     subset_nwm_streams = params["subset_nwm_streams"],
+                                     hucCode = params["hucCode"],
+                                     subset_nwm_headwaters = params["subset_nwm_headwaters"],                               
+                                     wbd_buffer_filename = params["wbd_buffer_filename"],
+                                     wbd_filename = params["wbd_filename"],
+                                     dem_filename = params["dem_filename"],
+                                     dem_domain = params["dem_domain"],
+                                     nwm_lakes = params["nwm_lakes"],
+                                     nwm_catchments = params["nwm_catchments"],
+                                     subset_nwm_catchments = params["subset_nwm_catchments"],
+                                     nld_lines = params["nld_lines"],
+                                     landsea = params["landsea"],
+                                     nwm_streams = params["nwm_streams"],
+                                     subset_landsea = params["subset_landsea"],
+                                     nwm_headwaters = params["nwm_headwaters"],
+                                     subset_nld_lines = params["subset_nld_lines"],
+                                     wbd_buffer_distance = params["wbd_buffer_distance"],
+                                     levee_protected_areas = params["levee_protected_areas"],
+                                     subset_levee_protected_areas = params["subset_levee_protected_areas"])
+
+        except (RuntimeError, TypeError, NameError) as e_info:
+            pytest.fail("Error in subset_vector_layers function", e_info)
+
diff --git a/unit_tests/clip_vectors_to_wbd_unittests.py b/unit_tests/clip_vectors_to_wbd_unittests.py
deleted file mode 100644
index 1e4193def..000000000
--- a/unit_tests/clip_vectors_to_wbd_unittests.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other direcories
-sys.path.append('/foss_fim/src/')
-import clip_vectors_to_wbd as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_clip_vectors_to_wbd(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    # MUST start with the name of "test_"
-    def test_subset_vector_layers_success(self):
-
-        '''
-        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
-        Most of the output test and internal tests with this function will test a wide variety of conditions.
-        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
-        Only the basic return output value shoudl be tested to ensure it is as expected.
-        For now, we are adding the very basic "happy path" test.
-        '''
-
-        params = self.params["valid_data"].copy()
-       
-        # There is no default return value.
-        # Later we need to check in this function that files were being created on the file system
-        
-        # for now we are happy if no exceptions are thrown.
-        src.subset_vector_layers(subset_nwm_lakes = params["subset_nwm_lakes"],
-                                subset_nwm_streams = params["subset_nwm_streams"],
-                                hucCode = params["hucCode"],
-                                subset_nhd_headwaters = params["subset_nhd_headwaters"],                                wbd_buffer_filename = params["wbd_buffer_filename"],
-                                wbd_filename = params["wbd_filename"],
-                                dem_filename = params["dem_filename"],
-                                nwm_lakes = params["nwm_lakes"],
-                                nwm_catchments = params["nwm_catchments"],
-                                subset_nwm_catchments = params["subset_nwm_catchments"],
-                                nld_lines = params["nld_lines"],
-                                landsea = params["landsea"],
-                                nwm_streams = params["nwm_streams"],
-                                subset_landsea = params["subset_landsea"],
-                                nhd_headwaters = params["nhd_headwaters"],
-                                subset_nld_lines = params["subset_nld_lines"],
-                                great_lakes = params["great_lakes"],                                
-                                lake_buffer_distance = params["lake_buffer_distance"],
-                                wbd_buffer_distance = params["wbd_buffer_distance"],
-                                levee_protected_areas = params["levee_protected_areas"],
-                                subset_levee_protected_areas = params["subset_levee_protected_areas"])
-       
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
index b8c2378cb..be247ac5a 100644
--- a/unit_tests/filter_catchments_and_add_attributes_params.json
+++ b/unit_tests/filter_catchments_and_add_attributes_params.json
@@ -1,12 +1,12 @@
 {
 	"valid_data":
 	{
-		"input_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_3246000005.gpkg",
-		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
-		"output_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
-		"output_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
+		"outputRunDataDir":  "/data/outputs/fim_unit_test_data_do_not_remove",
+		"input_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
+		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
+		"output_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
+		"output_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
 	    "wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
-		"huc_code": "02020005",
-		"drop_stream_orders": true
+		"huc_code": "02020005"
 	}
 }
diff --git a/unit_tests/filter_catchments_and_add_attributes_unittests.py b/unit_tests/filter_catchments_and_add_attributes_test.py
similarity index 59%
rename from unit_tests/filter_catchments_and_add_attributes_unittests.py
rename to unit_tests/filter_catchments_and_add_attributes_test.py
index 2243f7edc..258ff2349 100644
--- a/unit_tests/filter_catchments_and_add_attributes_unittests.py
+++ b/unit_tests/filter_catchments_and_add_attributes_test.py
@@ -5,20 +5,12 @@
 import sys
 
 import json
-import warnings
 import unittest
+import pytest
 
-sys.path.append('/foss_fim/unit_tests/')
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
 import filter_catchments_and_add_attributes as src
 
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-
-
 # *************
 # Important: For this to work, when you run gms_run_branch.sh, you have to 
 # use deny_gms_branches_dev.lst or the word "none" for the deny list arguments
@@ -31,13 +23,12 @@ class test_filter_catchments_and_add_attributes(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        warnings.simplefilter('ignore')
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
+# Test Cases:
+
     def test_filter_catchments_and_add_attributes_success(self):
 
         '''
@@ -48,40 +39,28 @@ def test_filter_catchments_and_add_attributes_success(self):
 
         params = self.params["valid_data"].copy() 
 
-        # to setup the test, lets start by deleted the two expected output files to ensure
-        # that they are regenerated
-        if os.path.exists(params["output_catchments_filename"]):
-            os.remove(params["output_catchments_filename"])
+        # To setup the test, lets start by deleted the two expected output files to ensure
+        # that they are regenerated.
         if os.path.exists(params["output_flows_filename"]):
             os.remove(params["output_flows_filename"])
-       
+        if os.path.exists(params["output_catchments_filename"]):
+            os.remove(params["output_catchments_filename"])
+        
+        # Test that the files were deleted
+        assert os.path.exists(params["output_flows_filename"]) == False
+
+        assert os.path.exists(params["output_catchments_filename"]) == False
+        
         src.filter_catchments_and_add_attributes(input_catchments_filename = params["input_catchments_filename"],
                                                 input_flows_filename = params["input_flows_filename"],
                                                 output_catchments_filename = params["output_catchments_filename"],
                                                 output_flows_filename = params["output_flows_filename"],
                                                 wbd_filename = params["wbd_filename"],
-                                                huc_code = params["huc_code"],
-                                                drop_stream_orders = params["drop_stream_orders"])
-       
-        if os.path.exists(params["output_catchments_filename"]) == False:
-            raise Exception(params["output_catchments_filename"] + " does not exist")
-        if os.path.exists(params["output_flows_filename"]) == False:
-            raise Exception(params["output_flows_filename"] + " does not exist")
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-if __name__ == '__main__':
+                                                huc_code = params["huc_code"])
+    
 
-    script_file_name = os.path.basename(__file__)
+        # Test that the files were created by filer_catchments_and_add_attributes       
+        assert os.path.exists(params["output_flows_filename"]) == True
+        
+        assert os.path.exists(params["output_catchments_filename"]) == True
 
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/generate_branch_list_csv_params.json b/unit_tests/generate_branch_list_csv_params.json
index d627a1356..004c5719b 100644
--- a/unit_tests/generate_branch_list_csv_params.json
+++ b/unit_tests/generate_branch_list_csv_params.json
@@ -3,18 +3,18 @@
 	{
 		"huc_id": "05030104",
 		"branch_id": "0",
-		"output_branch_csv": "/outputs/rob_wb_3/05030104/branch_ids.csv"
+		"output_branch_csv": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
 	},
 	"valid_data_add_branch":
 	{
 		"huc_id": "05030104",
 		"branch_id": "1946000003",
-		"output_branch_csv": "/outputs/rob_wb_3/05030104/branch_ids.csv"
+		"output_branch_csv": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
 	},	
 	"invalid_bad_file_extension":
 	{
 		"huc_id": "05030104",
 		"branch_id": "1946000003",
-		"output_branch_csv": "/outputs/rob_wb_3/05030104/branch_ids2"
+		"output_branch_csv": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids2"
 	}	
 }
diff --git a/unit_tests/generate_branch_list_csv_unittests.py b/unit_tests/generate_branch_list_csv_test.py
similarity index 62%
rename from unit_tests/generate_branch_list_csv_unittests.py
rename to unit_tests/generate_branch_list_csv_test.py
index 191d2fc01..5d2d75281 100644
--- a/unit_tests/generate_branch_list_csv_unittests.py
+++ b/unit_tests/generate_branch_list_csv_test.py
@@ -1,18 +1,12 @@
 #!/usr/bin/env python3
 
-import inspect
 import os
-import sys
-
 import json
-import warnings
 import unittest
+import pytest
 
-sys.path.append('/foss_fim/unit_tests/')
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
 import generate_branch_list_csv as src
 
 class test_generate_branch_list_csv(unittest.TestCase):
@@ -23,7 +17,6 @@ class test_generate_branch_list_csv(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        warnings.simplefilter('ignore')
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
@@ -34,7 +27,8 @@ def setUpClass(self):
         params = self.params["valid_data_add_branch_zero"].copy()
         if (os.path.exists(params["output_branch_csv"])):
             os.remove(params["output_branch_csv"])
-        
+
+# Test Cases:      
 
     def test_generate_branch_list_csv_valid_data_add_branch_zero_success(self):
 
@@ -47,9 +41,6 @@ def test_generate_branch_list_csv_valid_data_add_branch_zero_success(self):
                                      branch_id = params["branch_id"],
                                      output_branch_csv = params["output_branch_csv"])
        
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
 
 
     def test_generate_branch_list_csv_valid_data_add_branch_success(self):
@@ -63,42 +54,15 @@ def test_generate_branch_list_csv_valid_data_add_branch_success(self):
                                      branch_id = params["branch_id"],
                                      output_branch_csv = params["output_branch_csv"])
        
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
 
     def test_generate_branch_list_csv_invalid_bad_file_extension(self):
         
         #global params_file
         params = self.params["invalid_bad_file_extension"].copy()
 
-        # we expect this to fail. If it does fail with an exception, then this test
-        # is sucessful.
-        try:
+        # we expect this to fail. If it does fail with an exception, then this test is sucessful.
+        with pytest.raises(Exception) as e_info:
             src.generate_branch_list_csv(huc_id = params["huc_id"],
                                         branch_id = params["branch_id"],
                                         output_branch_csv = params["output_branch_csv"])
-       
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-            
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-       
-        finally:
-            print("*************************************************************")             
-
-
-if __name__ == '__main__':
 
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/generate_branch_list_params.json b/unit_tests/generate_branch_list_params.json
index 02a89bcea..b8c9bb4cc 100644
--- a/unit_tests/generate_branch_list_params.json
+++ b/unit_tests/generate_branch_list_params.json
@@ -1,8 +1,8 @@
 {
 	"valid_data":
 	{
-		"stream_network_dissolved": "/outputs/rob_wb_3/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
+		"stream_network_dissolved": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
 		"branch_id_attribute": "levpa_id",
-		"output_branch_list_file": "/outputs/rob_wb_3/05030104/branch_ids.lst"
+		"output_branch_list_file": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.lst"
 	}
 }
diff --git a/unit_tests/generate_branch_list_test.py b/unit_tests/generate_branch_list_test.py
new file mode 100644
index 000000000..d05d322c4
--- /dev/null
+++ b/unit_tests/generate_branch_list_test.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import os
+import json
+import unittest
+import pytest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+import generate_branch_list as src
+
+class test_Generate_branch_list(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+# Test Cases:
+
+    def test_Generate_branch_list_success(self):
+  
+        params = self.params["valid_data"].copy()
+
+        src.generate_branch_list(stream_network_dissolved = params["stream_network_dissolved"],
+                                 branch_id_attribute = params["branch_id_attribute"],
+                                 output_branch_list_file = params["output_branch_list_file"])
+
+
diff --git a/unit_tests/generate_branch_list_unittests.py b/unit_tests/generate_branch_list_unittests.py
deleted file mode 100644
index fc7ac7776..000000000
--- a/unit_tests/generate_branch_list_unittests.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import generate_branch_list as src
-
-class test_Generate_branch_list(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    def test_Generate_branch_list_success(self):
-
-        #global params_file
-        
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-
-        src.generate_branch_list(stream_network_dissolved = params["stream_network_dissolved"],
-                                 branch_id_attribute = params["branch_id_attribute"],
-                                 output_branch_list_file = params["output_branch_list_file"])
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/gms/__init__.py b/unit_tests/gms/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/unit_tests/gms/derive_level_paths_params.json b/unit_tests/gms/derive_level_paths_params.json
index 97e5e1a8b..d41e79fd2 100644
--- a/unit_tests/gms/derive_level_paths_params.json
+++ b/unit_tests/gms/derive_level_paths_params.json
@@ -1,6 +1,7 @@
 {
 	"valid_data":
 	{
+		"outputRunDataDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
 		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
 		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
 		"branch_id_attribute": "levpa_id",
@@ -11,21 +12,7 @@
 		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
 		"verbose": true
-	},
-	
-	"dropped_stream_orders_no_branches_remaining":
-	{
-		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams.gpkg",
-		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
-		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
-		"reach_id_attribute": "ID",
-		"verbose": true
-	}	
+	}
 }
 
 
diff --git a/unit_tests/gms/derive_level_paths_test.py b/unit_tests/gms/derive_level_paths_test.py
new file mode 100644
index 000000000..b9d4b6e5a
--- /dev/null
+++ b/unit_tests/gms/derive_level_paths_test.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import json
+import unittest
+import pytest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# # importing python folders in other direcories
+sys.path.append('/foss_fim/src/gms/')
+import derive_level_paths as src
+import stream_branches
+from utils.fim_enums import FIM_exit_codes as fec
+
+
+class test_Derive_level_paths(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+    
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+# Test Cases:
+
+    def test_Derive_level_paths_success_all_params(self):
+        '''
+        This test includes all params with many optional parms being set to the default value of the function
+        '''
+      
+        params = self.params["valid_data"].copy()
+
+        # Notes:
+        # Other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__ ,
+        # so we skip them here.
+        # Returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
+        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                           out_stream_network = params["out_stream_network"],
+                                           branch_id_attribute = params["branch_id_attribute"],
+                                           out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                           headwaters_outfile = params["headwaters_outfile"],
+                                           catchments = params["catchments"],
+                                           catchments_outfile = params["catchments_outfile"],
+                                           branch_inlets_outfile = params["branch_inlets_outfile"],
+                                           reach_id_attribute = params["reach_id_attribute"],
+                                           verbose = params["verbose"])
+
+        # test data type being return is as expected. Downstream code might to know that type
+        assert isinstance(actual_df, stream_branches.StreamNetwork)
+        
+
+        #**** NOTE: Based on 05030104         
+        # Test row count for dissolved level path GeoDataframe which is returned.
+        actual_row_count = len(actual_df) 
+        expected_row_count = 4
+        assert actual_row_count == expected_row_count
+        
+        # Test that output files exist as expected
+        assert os.path.exists(params["out_stream_network"]) == True, f"Expected file {params['out_stream_network']} but it does not exist."
+        assert os.path.exists(params["out_stream_network_dissolved"]) == True, f"Expected file {params['out_stream_network_dissolved']} but it does not exist."
+        assert os.path.exists(params["headwaters_outfile"]) == True, f"Expected file {params['headwaters_outfile']} but it does not exist."
+        assert os.path.exists(params["catchments_outfile"]) == True, f"Expected file {params['catchments_outfile']} but it does not exist."
+        assert os.path.exists(params["catchments_outfile"]) == True, f"Expected file {params['catchments_outfile']} but it does not exist."
+        assert os.path.exists(params["branch_inlets_outfile"]) == True, f"Expected file {params['branch_inlets_outfile']} but it does not exist."
+
+    
+    # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
+    def test_Derive_level_paths_invalid_input_stream_network(self):
+
+        # NOTE: As we are expecting an exception, we use pytest.raises(Exception).
+                
+        params = self.params["valid_data"].copy()
+        params["in_stream_network"] = "some bad path"
+        
+        with pytest.raises(Exception) as e_info:
+            actual = src.Derive_level_paths(in_stream_network = ut_helpers.json_concat(params, "outputRunDataDir", "in_stream_network"),
+                                            out_stream_network = params["out_stream_network"],
+                                            branch_id_attribute = params["branch_id_attribute"],
+                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                            huc_id = params["huc_id"],
+                                            headwaters_outfile = params["headwaters_outfile"],
+                                            catchments = params["catchments"],
+                                            catchments_outfile = params["catchments_outfile"],
+                                            branch_inlets_outfile = params["branch_inlets_outfile"],
+                                            reach_id_attribute = params["reach_id_attribute"],
+                                            verbose = params["verbose"] )       
+
diff --git a/unit_tests/gms/derive_level_paths_unittests.py b/unit_tests/gms/derive_level_paths_unittests.py
deleted file mode 100644
index 7027aa51c..000000000
--- a/unit_tests/gms/derive_level_paths_unittests.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-# importing python folders in other direcories
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other direcories
-sys.path.append('/foss_fim/src/gms/')
-import derive_level_paths as src
-import stream_branches
-from utils.fim_enums import FIM_exit_codes as fec
-
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_Derive_level_paths(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-    
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    def test_Derive_level_paths_success_all_params(self):
-    
-        '''
-        This test includes all params with many optional parms being set to the default value of the function
-        '''
-      
-        params = self.params["valid_data"].copy()
-
-        # Function Notes:
-        # huc_ids no longer used, so it is not submitted
-        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
-        # skip them here.
-        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                            out_stream_network = params["out_stream_network"],
-                                            branch_id_attribute = params["branch_id_attribute"],
-                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                            headwaters_outfile = params["headwaters_outfile"],
-                                            catchments = params["catchments"],
-                                            catchments_outfile = params["catchments_outfile"],
-                                            branch_inlets_outfile = params["branch_inlets_outfile"],
-                                            reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"])
-
-        # -----------
-        # test data type being return is as expected. Downstream code might to know that type
-        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
-        
-        # -----------
-        #**** NOTE: Based on 05030104         
-        # Test row count for dissolved level path GeoDataframe which is returned.
-        actual_row_count = len(actual_df) 
-        expected_row_count = 58
-        self.assertEqual(actual_row_count, expected_row_count)
-        
-        # -----------
-        # Test that output files exist as expected
-        if os.path.exists(params["out_stream_network"]) == False:
-            raise Exception(params["out_stream_network"] + " does not exist")
-            
-        if os.path.exists(params["out_stream_network_dissolved"]) == False:
-            raise Exception(params["out_stream_network_dissolved"] + " does not exist")
-
-        if os.path.exists(params["headwaters_outfile"]) == False:
-            raise Exception(params["headwaters_outfile"] + " does not exist")
-
-        #if os.path.exists(params["catchments_outfile"]) == False:
-        #    raise Exception(params["catchments_outfile"] + " does not exist")
-
-        if os.path.exists(params["catchments_outfile"]) == False:
-            raise Exception(params["catchments_outfile"] + " does not exist")
-
-        if os.path.exists(params["branch_inlets_outfile"]) == False:
-            raise Exception(params["branch_inlets_outfile"] + " does not exist")
-       
-        
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")        
-
-    def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self):
-    
-        '''
-        This test includes a huc that after stream orders 1 and 2 are dropped, no 
-        branches are remaining (the original huc only had 1 and 2 reaches.
-
-        # expecting an exception with a specific note
-        # NOTE: this only works for a few hucs, one is 02030201
-
-        '''
-        
-        params = self.params["dropped_stream_orders_no_branches_remaining"].copy()
-
-        with self.assertRaises(SystemExit) as se:
-            src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                    out_stream_network = params["out_stream_network"],
-                                    branch_id_attribute = params["branch_id_attribute"],
-                                    out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                    headwaters_outfile = params["headwaters_outfile"],
-                                    catchments = params["catchments"],
-                                    catchments_outfile = params["catchments_outfile"],
-                                    branch_inlets_outfile = params["branch_inlets_outfile"],
-                                    reach_id_attribute = params["reach_id_attribute"],
-                                    verbose = params["verbose"])
-
-        self.assertEqual(se.exception.code, fec.GMS_UNIT_NO_BRANCHES.value)
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")        
-
-    
-    # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
-    def test_Derive_level_paths_invalid_input_stream_network(self):
-
-        # NOTE: As we are expecting an exception, this MUST have a try catch.
-        #   for "success" tests (as above functions), they CAN NOT have a try/catch
-        try:
-        
-            params = self.params["valid_data"].copy()
-            params["in_stream_network"] = "some bad path"
-          
-            actual = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                            out_stream_network = params["out_stream_network"],
-                                            branch_id_attribute = params["branch_id_attribute"],
-                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                            huc_id = params["huc_id"],
-                                            headwaters_outfile = params["headwaters_outfile"],
-                                            catchments = params["catchments"],
-                                            catchments_outfile = params["catchments_outfile"],
-                                            branch_inlets_outfile = params["branch_inlets_outfile"],
-                                            reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"] )
-                                                           
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-            
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        finally:
-            print("*************************************************************")             
-       
-       
-       
-if __name__ == '__main__':
-
-    print("*****************************")
-    print(f"Start of {os.path.basename(__file__)} tests")
-    print()
-            
-    unittest.main()
-    
-    print()    
-    print(f"End of {os.path.basename(__file__)} tests")
-    
diff --git a/unit_tests/gms/outputs_cleanup_test.py b/unit_tests/gms/outputs_cleanup_test.py
new file mode 100644
index 000000000..ab1a46a92
--- /dev/null
+++ b/unit_tests/gms/outputs_cleanup_test.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import json
+from pathlib import Path
+import io
+from contextlib import redirect_stdout
+import unittest
+import pytest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+from utils.shared_functions import FIM_Helpers as fh
+
+
+sys.path.append('/foss_fim/src/gms') 
+import outputs_cleanup as src
+
+
+class test_outputs_cleanup(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+# Test Cases: 
+
+    def test_remove_deny_list_files_specific_branch_success(self):
+        '''
+        This validates removal of files for a directory already pointing to a 
+        specific branch in a HUC
+        '''
+
+        params = self.params["valid_specific_branch_data"].copy()
+
+        # Gather all of the file names into an array from the deny_list
+        deny_files = self.__get_deny_list_filenames(params["src_dir"], params["deny_list"], params["branch_id"])
+
+        # Test whether we have a list of files to check 
+        assert len(deny_files) > 0
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+
+        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) == True
+
+
+    def test_remove_deny_list_files_huc_level_success(self):
+        '''
+        This validates removal of files for all files and subdirectory files.
+        Normally used for covering all hucs and their branch zeros but 
+        can be anything
+        '''
+
+        params = self.params["valid_directory_data"].copy()
+
+        # Gather all of the file names into an array from the deny_list
+        deny_files = self.__get_deny_list_filenames(params["src_dir"], params["deny_list"], params["branch_id"])
+
+        # Test whether we have a list of files to check 
+        assert len(deny_files) > 0
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+                                   
+        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) == True
+
+
+    def test_remove_deny_list_skip_cleaning_success(self):
+        '''
+        This validates removal of files for all files and subdirectory files.
+        Normally used for covering all hucs and their branch zeros but 
+        can be anything
+        '''
+
+        params = self.params["skip_clean"].copy()
+
+        deny_files = self.__get_deny_list_filenames(params["src_dir"], params["deny_list"], params["branch_id"])
+
+        # Ensure we have a value of "None" for a deny_list value, 
+        # __get_deny_list_filenames returns an empty array if "None" is provided 
+        assert len(deny_files) == 0
+        
+        # This is a little tricky, as we're capturing the stdout (return statement) from remove_deny_list_files,
+        # to verify the function is returning at the correct place, and not removing files when we do not provide
+        # a deny list file. We set f to the io stream, and redirect it using redirect_stdout. 
+        f = io.StringIO()
+        with redirect_stdout(f):
+            src.remove_deny_list_files(src_dir = params["src_dir"],
+                                       deny_list = params["deny_list"],
+                                       branch_id = params["branch_id"],
+                                       verbose = params["verbose"])
+        
+        # Get the stdout value of remove_deny_list_files and set it to skip_clean_out
+        skip_clean_out = f.getvalue()                           
+        
+        # This string must match the print statement in /src/gms/outputs_cleanup.py, including the \n newline,
+        # which occurs "behind the scenes" with every call to print() in Python
+        assert skip_clean_out == "file clean via the deny list skipped\n"
+
+
+    def test_remove_deny_list_files_invalid_src_directory(self):
+        '''
+        Double check the src directory exists
+        '''
+
+        params = self.params["valid_specific_branch_data"].copy()    
+        params["src_dir"] = "/data/does_no_exist"
+
+        # We want an exception to be thrown here, if so, the test passes.
+        with pytest.raises(Exception) as e_info:
+
+            src.remove_deny_list_files(src_dir = params["src_dir"],
+                                       deny_list = params["deny_list"],
+                                       branch_id = params["branch_id"],
+                                       verbose = params["verbose"])
+
+
+    def test_remove_deny_list_files_invalid_deny_list_does_not_exist(self):
+        '''
+        Double check the deny list exists
+        '''
+
+        params = self.params["valid_specific_branch_data"].copy()    
+        params["deny_list"] = "invalid_file_name.txt"
+
+        # We want an exception to be thrown here, if so, the test passes.
+        with pytest.raises(Exception) as e_info:
+        
+            src.remove_deny_list_files(src_dir = params["src_dir"],
+                                       deny_list = params["deny_list"],
+                                       branch_id = params["branch_id"],
+                                       verbose = params["verbose"])
+
+
+# Helper Functions:
+
+    def __get_deny_list_filenames(self, src_dir, deny_list, branch_id):
+
+        deny_list_files = []
+
+        if (deny_list == "None"):
+            return deny_list_files
+
+        # Note: some of the deny_file_names might be a comment line
+        # this will validate file exists
+        deny_file_names = fh.load_list_file(deny_list.strip())
+
+        for deny_file_name in deny_file_names:
+        
+            # Only add files to the list that do not start with a #
+            deny_file_name = deny_file_name.strip()
+            if (deny_file_name.startswith("#")):
+                continue 
+            
+            deny_file_name = deny_file_name.replace("{}", branch_id)
+
+            deny_list_files.append(deny_file_name)
+                    
+        return deny_list_files
+
+
+    def __check_no_deny_list_files_exist(self, src_dir, deny_array):
+
+        found_files = []
+
+        for file_name in deny_array:
+
+            found_files.append(os.path.join(src_dir, file_name))
+
+        for found_file in found_files:
+
+            if os.path.exists(found_file):
+                return False
+        
+        return True
+
diff --git a/unit_tests/gms/outputs_cleanup_unittests.py b/unit_tests/gms/outputs_cleanup_unittests.py
deleted file mode 100644
index 006c95ab0..000000000
--- a/unit_tests/gms/outputs_cleanup_unittests.py
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/') 
-import gms.outputs_cleanup as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_outputs_cleanup(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    def test_remove_deny_list_files_specific_branch_success(self):
-
-        '''
-        This validates removal of files for a directory already pointing to a 
-        specific branch in a HUC
-        '''
-
-        params = self.params["valid_specific_branch_data"].copy()
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
-                                   
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_files_huc_level_success(self):
-
-        '''
-        This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but 
-        can be anything
-        '''
-
-        params = self.params["valid_directory_data"].copy()
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
-                                   
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_skip_cleaning_success(self):
-
-        '''
-        This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but 
-        can be anything
-        '''
-
-        params = self.params["skip_clean"].copy()
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
-                                   
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_files_invalid_src_directory(self):
-
-        '''
-        Double check the src directory exists
-        '''
-
-        # NOTE: As we are expecting an exception, this MUST have a try catch.
-        #   for "success" tests (as above functions), they CAN NOT have a try/catch
-        
-        try:
-
-            params = self.params["valid_specific_branch_data"].copy()    
-            params["src_dir"] = "/data/does_no_exist"
-            
-            src.remove_deny_list_files(src_dir = params["src_dir"],
-                                    deny_list = params["deny_list"],
-                                    branch_id = params["branch_id"],
-                                    verbose = params["verbose"])
-            
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-            
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_files_invalid_deny_list_does_not_exist(self):
-
-        '''
-        Double check the deny list exists
-        '''
-
-        # NOTE: As we are expecting an exception, this MUST have a try catch.
-        #   for "success" tests (as above functions), they CAN NOT have a try/catch
-        try:
-
-            params = self.params["valid_specific_branch_data"].copy()    
-            params["deny_list"] = "invalid_file_name.txt"
-            
-            src.remove_deny_list_files(src_dir = params["src_dir"],
-                                    deny_list = params["deny_list"],
-                                    branch_id = params["branch_id"],
-                                    verbose = params["verbose"])
-            
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-            
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-   
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/pyproject.toml b/unit_tests/pyproject.toml
new file mode 100644
index 000000000..142fbd053
--- /dev/null
+++ b/unit_tests/pyproject.toml
@@ -0,0 +1,7 @@
+[tool.pytest.ini_options]
+filterwarnings = [
+    "error",
+    "ignore::UserWarning",
+    "ignore::DeprecationWarning",
+    "ignore::PendingDeprecationWarning"
+]
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
index 44e2232ac..e233b3520 100644
--- a/unit_tests/rating_curve_comparison_params.json
+++ b/unit_tests/rating_curve_comparison_params.json
@@ -12,6 +12,7 @@
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"01010004",
+			false,
 			false
 		],
 		"02020005":
@@ -25,6 +26,7 @@
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"02020005",
+			false,
 			false
 		],
 		"02030103":
@@ -38,6 +40,7 @@
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"02030103",
+			false, 
 			false
 		]
 	}
diff --git a/unit_tests/rating_curve_comparison_test.py b/unit_tests/rating_curve_comparison_test.py
new file mode 100644
index 000000000..058662376
--- /dev/null
+++ b/unit_tests/rating_curve_comparison_test.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+import os
+import json
+import unittest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+from rating_curve_comparison import generate_rating_curve_metrics
+
+
+class test_rating_curve_comparison(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+# Test Cases
+
+    def test_generate_rating_curve_metrics_01010004_success(self):
+
+        '''
+        We are testing whether a .png file was created for a FIM-USGS rating curve comparison, 
+        for HUC 0101004 using the `generate_rating_curve_metrics` function.
+        The 5th index (parameter) for each HUC in `rating_curve_comparison_params.json` specifies
+        the FIM-USGS rating curve comparison .png filepath.
+        '''
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+
+        _indiv_huc_params = params["01010004"]
+
+        # To setup the test, lets start by deleted the expected output file to ensure
+        # that it is regenerated.
+        if os.path.exists(_indiv_huc_params[5]):
+            os.remove(_indiv_huc_params[5])
+        
+        # Test that the file was deleted
+        assert os.path.exists(_indiv_huc_params[5]) == False
+
+        generate_rating_curve_metrics(_indiv_huc_params)
+
+        # Test that the file was created by generate_rating_curve_metrics       
+        assert os.path.exists(_indiv_huc_params[5]) == True
+        
+
+    def test_generate_rating_curve_metrics_02020005_success(self):
+
+        '''
+        We are testing whether a .png file was created for a FIM-USGS rating curve comparison, 
+        for HUC 02020005 using the `generate_rating_curve_metrics` function.
+        The 5th index (parameter) for each HUC in `rating_curve_comparison_params.json` specifies
+        the FIM-USGS rating curve comparison .png filepath.
+        '''
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        
+        _indiv_huc_params = params["02020005"] 
+
+        # To setup the test, lets start by deleted the expected output file to ensure
+        # that it is regenerated.
+        if os.path.exists(_indiv_huc_params[5]):
+            os.remove(_indiv_huc_params[5])
+        
+        # Test that the file was deleted
+        assert os.path.exists(_indiv_huc_params[5]) == False
+
+        generate_rating_curve_metrics(_indiv_huc_params)
+
+        # Test that the file was created by generate_rating_curve_metrics       
+        assert os.path.exists(_indiv_huc_params[5]) == True
+
+
+    def test_generate_rating_curve_metrics_02030103_success(self):
+
+        '''
+        We are testing whether a .png file was created for a FIM-USGS rating curve comparison, 
+        for HUC 02030103 using the `generate_rating_curve_metrics` function.
+        The 5th index (parameter) for each HUC in `rating_curve_comparison_params.json` specifies
+        the FIM-USGS rating curve comparison .png filepath.
+        '''
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        
+        _indiv_huc_params = params["02030103"]
+
+        # To setup the test, lets start by deleted the expected output file to ensure
+        # that it is regenerated.
+        if os.path.exists(_indiv_huc_params[5]):
+            os.remove(_indiv_huc_params[5])
+        
+        # Test that the file was deleted
+        assert os.path.exists(_indiv_huc_params[5]) == False
+
+        generate_rating_curve_metrics(_indiv_huc_params)
+
+        # Test that the file was created by generate_rating_curve_metrics       
+        assert os.path.exists(_indiv_huc_params[5]) == True
+
diff --git a/unit_tests/rating_curve_comparison_unittests.py b/unit_tests/rating_curve_comparison_unittests.py
deleted file mode 100644
index 07ded5931..000000000
--- a/unit_tests/rating_curve_comparison_unittests.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-import warnings
-
-import json
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/tools/')   # *** update your folder path here if required ***
-from rating_curve_comparison import generate_rating_curve_metrics
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_rating_curve_comparison(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
-    def test_generate_rating_curve_metrics_01010004_success(self):
-
-        '''
-        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
-          If there is no return value from the method, please say so.>
-        '''
-
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-                
-        generate_rating_curve_metrics(params["01010004"])
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-    def test_generate_rating_curve_metrics_02020005_success(self):
-
-        '''
-        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
-          If there is no return value from the method, please say so.>
-        '''
-        
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-                
-        generate_rating_curve_metrics(params["02020005"])
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-    def test_generate_rating_curve_metrics_02030103_success(self):
-
-        '''
-        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
-          If there is no return value from the method, please say so.>
-        '''
-        
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-                
-        generate_rating_curve_metrics(params["02030103"])
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json
index 04b33df3e..9dce603d1 100644
--- a/unit_tests/shared_functions_params.json
+++ b/unit_tests/shared_functions_params.json
@@ -1,15 +1,17 @@
 {
 	"append_append_id_to_file_name_single_identifier_valid":
 	{
-		"file_name": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
+		"outputRunDataDir": "/output/fim_unit_test_data_do_not_remove/",
+		"file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
 		"identifier": "8888",
-		"expected_output": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003_8888.tif"
+		"expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_8888.tif"
 	},
 
 	"append_append_id_to_file_name_identifier_list_valid":
 	{
-		"file_name": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
+		"outputRunDataDir": "/output/fim_unit_test_data_do_not_remove/",
+		"file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
 		"identifier": ["7777", "8888"],
-		"expected_output": "/output/fim_unit_test_data_do_not_remove/02020005/branches/3246000003/rem_zeroed_masked_3246000003_7777_8888.tif"
+		"expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_7777_8888.tif"
 	}	
 }
\ No newline at end of file
diff --git a/unit_tests/shared_functions_unittests.py b/unit_tests/shared_functions_test.py
similarity index 51%
rename from unit_tests/shared_functions_unittests.py
rename to unit_tests/shared_functions_test.py
index 446aed235..3f9de9829 100644
--- a/unit_tests/shared_functions_unittests.py
+++ b/unit_tests/shared_functions_test.py
@@ -1,23 +1,12 @@
 #!/usr/bin/env python3
 
-import inspect
-import os
-import sys
-
 import json
-import warnings
 import unittest
+import pytest
 
-sys.path.append('/foss_fim/unit_tests/')
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-#sys.path.append('/foss_fim/src/utils') 
-#import utils.shared_functions as src
 from utils.shared_functions import FIM_Helpers as src
 
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
 class test_shared_functions(unittest.TestCase):
 
     '''
@@ -26,27 +15,28 @@ class test_shared_functions(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        warnings.simplefilter('ignore')
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
-
+    
+# Test Cases:
 
     def test_append_id_to_file_name_single_identifier_success(self):
 
         '''
         Pass in a file name with the single identifier and get the single adjusted file name back
         '''
-       
-        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
-        actual_output = src.append_id_to_file_name(file_name = params["file_name"],
+
+        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy() 
+
+        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputRunDataDir" , "file_name"),
                                                    identifier = params["identifier"])
 
         err_msg = "actual output does not match expected output"
-        self.assertEqual(params["expected_output"], actual_output, err_msg)
 
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
+        expected_output = ut_helpers.json_concat(params, "outputRunDataDir" , "expected_output")
+        
+        assert  expected_output == actual_output, err_msg
 
 
     def test_append_id_to_file_name_indentifer_list_success(self):
@@ -56,18 +46,16 @@ def test_append_id_to_file_name_indentifer_list_success(self):
         '''
         
         params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()        
-        
-        expected_output = "/output/myfolder/a_raster_13090001_05010204.tif"
 
-        actual_output = src.append_id_to_file_name(file_name = params["file_name"],
+        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputRunDataDir" , "file_name"),
                                                    identifier = params["identifier"])
 
-        #print(actual_output)
         err_msg = "actual output does not match expected output"
-        self.assertEqual(params["expected_output"], actual_output, err_msg)
+        
+        expected_output = ut_helpers.json_concat(params, "outputRunDataDir" , "expected_output")
+        
+        assert expected_output == actual_output, err_msg
 
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
 
     def test_append_id_to_file_name_no_file_name_success(self):
 
@@ -79,24 +67,7 @@ def test_append_id_to_file_name_no_file_name_success(self):
         
         actual_output = src.append_id_to_file_name(None, identifier = params["identifier"])
 
-        if (actual_output is not None):
-            raise Exception("actual output should not have a value")
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
+        error_msg = "actual output should not have a value"
 
+        assert actual_output == None, error_msg
 
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
index 15ab0bce9..6bd115d23 100644
--- a/unit_tests/split_flows_params.json
+++ b/unit_tests/split_flows_params.json
@@ -1,6 +1,7 @@
 {
 	"valid_data":
 	{
+		"outputRunDataDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
 		"max_length": 1500,
 		"slope_min": 0.001,
 		"lakes_buffer_input": 20,
@@ -10,6 +11,6 @@
 		"split_points_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
 		"wbd8_clp_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
 		"lakes_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_lakes_proj_subset.gpkg",
-		"drop_stream_orders": true
+		"nwm_streams_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_subset_streams_levelPaths.gpkg"
 	}
-}
\ No newline at end of file
+}
diff --git a/unit_tests/split_flows_unittests.py b/unit_tests/split_flows_test.py
similarity index 57%
rename from unit_tests/split_flows_unittests.py
rename to unit_tests/split_flows_test.py
index e517f078a..84d986a47 100644
--- a/unit_tests/split_flows_unittests.py
+++ b/unit_tests/split_flows_test.py
@@ -1,22 +1,14 @@
 #!/usr/bin/env python3
-
-import inspect
 import os
-import sys
-
 import json
-import warnings
 import unittest
+import pytest
 
-sys.path.append('/foss_fim/unit_tests/')
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
 import split_flows as src
 
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
+
 class test_split_flows(unittest.TestCase):
 
     '''
@@ -25,20 +17,19 @@ class test_split_flows(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        warnings.simplefilter('ignore')
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
+# Test Cases:
 
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
+    # Ensure split_flows_filename & split_points_filename are created by the split_flows function 
     def test_split_flows_success(self):
 
         '''
         The /data/outputs/gms_example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
         /data/outputs/gms_example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
-        If the test is successful, these file will be created.
+        If the test is successful, these files will be created.
         '''
 
         params = self.params["valid_data"].copy() 
@@ -49,6 +40,12 @@ def test_split_flows_success(self):
             os.remove(params["split_flows_filename"])
         if os.path.exists(params["split_points_filename"]):
             os.remove(params["split_points_filename"])
+
+        error_msg = params["split_flows_filename"] + " does exist, when it should not (post os.remove call)"
+        assert os.path.exists(params["split_flows_filename"]) == False, error_msg 
+        
+        error_msg = params["split_points_filename"] + " does exist, when it should not (post os.remove call)"
+        assert os.path.exists(params["split_points_filename"]) == False, error_msg
        
         src.split_flows(max_length = params["max_length"],
                         slope_min = params["slope_min"],
@@ -59,27 +56,11 @@ def test_split_flows_success(self):
                         split_points_filename = params["split_points_filename"],
                         wbd8_clp_filename = params["wbd8_clp_filename"],
                         lakes_filename = params["lakes_filename"],
-                        drop_stream_orders = params["drop_stream_orders"])
+                        nwm_streams_filename = params["nwm_streams_filename"])
        
-        if os.path.exists(params["split_flows_filename"]) == False:
-            raise Exception(params["split_flows_filename"] + " does not exist")
-        if os.path.exists(params["split_points_filename"]) == False:
-            raise Exception(params["split_points_filename"] + " does not exist")
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
+        error_msg = params["split_flows_filename"] + " does not exist"
+        assert os.path.exists(params["split_flows_filename"]) == True, error_msg 
+        
+        error_msg = params["split_points_filename"] + " does not exist"
+        assert os.path.exists(params["split_points_filename"]) == True, error_msg
 
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/tools/__init__.py b/unit_tests/tools/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/unit_tests/tools/gms_tools/inundate_gms_params.json b/unit_tests/tools/inundate_gms_params.json
similarity index 66%
rename from unit_tests/tools/gms_tools/inundate_gms_params.json
rename to unit_tests/tools/inundate_gms_params.json
index a75e663c4..a2fd24d72 100644
--- a/unit_tests/tools/gms_tools/inundate_gms_params.json
+++ b/unit_tests/tools/inundate_gms_params.json
@@ -1,10 +1,10 @@
 {
 	"valid_data_inudation_raster_single_huc":
 	{
-		"hydrofabric_dir": "/outputs/fim_unit_test_data_do_not_remove/",
-		"forecast": "data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv",
+		"hydrofabric_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
+		"forecast": "/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv",
 		"num_workers": 1,
-		"hucs": "02020005",
+		"hucs": "02020005, 05030104",
 	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
 		"inundation_polygon": null,
 		"depths_raster": null,
@@ -12,5 +12,4 @@
 		"log_file": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_logfile.txt",
 		"output_fileNames": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_file_list.csv"
 	}
-
-}
\ No newline at end of file
+}
diff --git a/unit_tests/tools/gms_tools/inundate_gms_unittests.py b/unit_tests/tools/inundate_gms_test.py
similarity index 67%
rename from unit_tests/tools/gms_tools/inundate_gms_unittests.py
rename to unit_tests/tools/inundate_gms_test.py
index 54c742f93..9e0f195d6 100644
--- a/unit_tests/tools/gms_tools/inundate_gms_unittests.py
+++ b/unit_tests/tools/inundate_gms_test.py
@@ -1,12 +1,11 @@
 #!/usr/bin/env python3
 
-import inspect
 import os
 import sys
 
 import json
-import warnings
 import unittest
+import pytest
 
 sys.path.append('/foss_fim/unit_tests/')
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
@@ -14,8 +13,7 @@
 sys.path.append('/foss_fim/tools/gms_tools')
 import inundate_gms as src
 
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
+
 class test_inundate_gms(unittest.TestCase):
 
     '''
@@ -24,12 +22,12 @@ class test_inundate_gms(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        warnings.simplefilter('ignore')
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-
+# Test Cases:
+    @pytest.mark.skip(reason="Inundate_gms will be rebuilt in the future, so this test will be left broken.")
     def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self):
 
         '''
@@ -49,35 +47,15 @@ def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self
                                                inundation_polygon = params["inundation_polygon"],
                                                depths_raster = params["depths_raster"],
                                                verbose = params["verbose"], 
-                                               log_file = params["log_file"],
-                                               output_fileNames = params["output_fileNames"])
+                                            #    log_file = None,
+                                            #    output_fileNames = params["output_fileNames"]
+                                               )
 
         # check if output files df has records.
         assert len(output_fileNames_df) > 0, "Expected as least one dataframe record"
 
         # also check output log file and output raster. Can't... there will be multiple outputs
-        #assert os.path.exists(params["inundation_raster"]), "Inundation Raster does not exist"
+        assert os.path.exists(params["inundation_raster"]), "Inundation Raster does not exist"
 
         assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-        
-    # ***********************
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
 
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/tools/inundate_unittests.py b/unit_tests/tools/inundate_unittests.py
deleted file mode 100644
index 66fb1ad16..000000000
--- a/unit_tests/tools/inundate_unittests.py
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-sys.path.append('/foss_fim/tools/')
-import inundation as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_inundate(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    def test_inundate_create_inundation_raster_single_branch_success(self):
-
-        '''
-        Test for creating a inundation branch raster, no depth raster and no
-        inundation_polygons, no subsets, no mask
-        '''
-
-        params = self.params["valid_data_inundate_branch"].copy()         
-        
-        # returns list of rasters and polys
-        inundation_rasters, depth_rasters, inundation_polys = src.inundate(
-                                            rem = params["rem"],
-                                            catchments = params["catchments"],
-                                            catchment_poly = params["catchment_poly"],
-                                            hydro_table = params["hydro_table"],
-                                            forecast = params["forecast"],
-                                            mask_type = params["mask_type"],
-                                            hucs = params["hucs"],
-                                            hucs_layerName = params["hucs_layerName"],
-                                            subset_hucs = params["subset_hucs"],
-                                            num_workers = params["num_workers"],
-                                            aggregate = params["aggregate"], 
-                                            inundation_raster = params["inundation_raster"],
-                                            inundation_polygon = params["inundation_polygon"],
-                                            depths = params["depths"],
-                                            out_raster_profile = params["out_raster_profile"],
-                                            out_vector_profile = params["out_vector_profile"],
-                                            src_table = params["src_table"],
-                                            quiet = params["quiet"]                                            
-                                            )
-
-        print(inundation_rasters)
-        assert len(inundation_rasters) == 1, "Expected exactly one inundation raster path records"
-        assert depth_rasters[0] == None, "Expected no depth raster path records"
-        assert inundation_polys[0] == None, "Expected no inundation_polys records"
-        
-        assert os.path.exists(params["expected_inundation_raster"]), \
-            f"Expected file {params['expected_inundation_raster']} but it does not exist."
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-        
-    # ***********************
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/tools/inundate_params.json b/unit_tests/tools/inundation_params.json
similarity index 100%
rename from unit_tests/tools/inundate_params.json
rename to unit_tests/tools/inundation_params.json
diff --git a/unit_tests/tools/inundation_test.py b/unit_tests/tools/inundation_test.py
new file mode 100644
index 000000000..dc44207dc
--- /dev/null
+++ b/unit_tests/tools/inundation_test.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import json
+import unittest
+import pytest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+import inundation as src
+
+
+class test_inundate(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+# Test Cases:
+
+    def test_inundate_create_inundation_raster_single_branch_success(self):
+        '''
+        Test for creating a inundation branch raster, no depth raster and no
+        inundation_polygons, no subsets, no mask
+        '''
+
+        params = self.params["valid_data_inundate_branch"].copy()         
+        
+        # returns list of rasters and polys
+        in_rasters, depth_rasters, in_polys = src.inundate(rem = params["rem"],
+                                                           catchments = params["catchments"],
+                                                           catchment_poly = params["catchment_poly"],
+                                                           hydro_table = params["hydro_table"],
+                                                           forecast = params["forecast"],
+                                                           mask_type = params["mask_type"],
+                                                           hucs = params["hucs"],
+                                                           hucs_layerName = params["hucs_layerName"],
+                                                           subset_hucs = params["subset_hucs"],
+                                                           num_workers = params["num_workers"],
+                                                           aggregate = params["aggregate"], 
+                                                           inundation_raster = params["inundation_raster"],
+                                                           inundation_polygon = params["inundation_polygon"],
+                                                           depths = params["depths"],
+                                                           out_raster_profile = params["out_raster_profile"],
+                                                           out_vector_profile = params["out_vector_profile"],
+                                                           src_table = params["src_table"],
+                                                           quiet = params["quiet"])
+        
+        assert len(in_rasters) == 1, "Expected exactly one inundation raster path records"
+        assert depth_rasters[0] == None, "Expected no depth raster path records"
+        assert in_polys[0] == None, "Expected no inundation_polys records"
+        
+        msg = f"Expected file {params['expected_inundation_raster']} but it does not exist."
+        assert os.path.exists(params["expected_inundation_raster"]) == True, msg
+
diff --git a/unit_tests/unit_tests_utils.py b/unit_tests/unit_tests_utils.py
index 018ae815a..d2c7223eb 100644
--- a/unit_tests/unit_tests_utils.py
+++ b/unit_tests/unit_tests_utils.py
@@ -10,7 +10,7 @@ class FIM_unit_test_helpers(object):
     def get_params_filename(unit_test_file_name):
        
         unittest_file_name = os.path.basename(unit_test_file_name)
-        params_file_name = unittest_file_name.replace("_unittests.py", "_params.json")
+        params_file_name = unittest_file_name.replace("_test.py", "_params.json")
         params_file_path = os.path.join(os.path.dirname(unit_test_file_name), params_file_name)
         
         if (not os.path.exists(params_file_path)):
@@ -18,3 +18,9 @@ def get_params_filename(unit_test_file_name):
         
         return params_file_path
     
+    @staticmethod
+    def json_concat(obj, keyone, keytwo):
+        
+        concat_string = obj[keyone] + obj[keytwo] 
+        
+        return concat_string
\ No newline at end of file
diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py
new file mode 100644
index 000000000..4b9bedc1a
--- /dev/null
+++ b/unit_tests/usgs_gage_crosswalk_test.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import json
+import unittest
+import pytest
+
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+from usgs_gage_crosswalk import GageCrosswalk
+
+
+class test_usgs_gage_crosswalk(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+# Test Cases:
+
+    def test_GageCrosswalk_success(self):
+
+        '''
+        Test whether the GageCrosswalk object can be instantiated, and test that the run_crosswalk method can successfully 
+        create the output table (usgs_elev_table.csv).
+        '''
+
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+       
+        # Delete the usgs_elev_table.csv if it exists
+        if os.path.exists(params["output_table_filename"]):
+            os.remove(params["output_table_filename"])
+
+        # Verify the usgs_elev_table.csv was deleted
+        msg = f'{params["output_table_filename"]} does exist, when it should have been deleted'
+        assert os.path.exists(params["output_table_filename"]) == False, msg 
+
+        # Instantiate and run GageCrosswalk
+        gage_crosswalk = GageCrosswalk(params["usgs_gages_filename"], params["branch_id"])
+
+        # Run crosswalk
+        gage_crosswalk.run_crosswalk(params["input_catchment_filename"], params["input_flows_filename"], 
+                            params["dem_filename"], params["dem_adj_filename"], params["output_table_filename"])
+
+        # Make sure that the usgs_elev_table.csv was written
+        msg = f'{params["output_table_filename"]} does not exist'
+        assert os.path.exists(params["output_table_filename"]) == True, msg 
+
diff --git a/unit_tests/usgs_gage_crosswalk_unittests.py b/unit_tests/usgs_gage_crosswalk_unittests.py
deleted file mode 100644
index bc66d6faf..000000000
--- a/unit_tests/usgs_gage_crosswalk_unittests.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import unittest
-import warnings
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-from usgs_gage_crosswalk import GageCrosswalk
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_usgs_gage_crosswalk(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
-    def test_GageCrosswalk_success(self):
-
-        '''
-        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
-          If there is no return value from the method, please say so.>
-        
-        Dev Notes: (which can be removed after you make this file)
-            Remember... You need to validate the method output if there is any. However, if you have time, it
-            is also recommended that you validate other outputs such as writing or updating file on the file 
-            system, aka: Does the expected file exist. Don't worry about its contents.
-
-        '''
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-       
-        # Delete the usgs_elev_table.csv if it exists
-        if os.path.isfile(params["output_table_filename"]):
-            os.remove(params["output_table_filename"])
-
-        # Instantiate and run GageCrosswalk
-        gage_crosswalk = GageCrosswalk(params["usgs_gages_filename"], params["branch_id"])
-
-        # Run crosswalk
-        gage_crosswalk.run_crosswalk(params["input_catchment_filename"], params["input_flows_filename"], 
-                            params["dem_filename"], params["dem_adj_filename"], params["output_table_filename"])
-        print(gage_crosswalk.gages)
-
-        # Make sure that the usgs_elev_table.csv was written
-        assert os.path.isfile(params["output_table_filename"]), f'{params["output_table_filename"]} does not exist'
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    

From 264f90655e6b36974ea0893e6a2eff05e0a2fd73 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 17 Feb 2023 11:12:35 -0600
Subject: [PATCH 210/359] v4.1.3.0 Update for calib db pathing to work in AWS
 (#812)

---
 docs/CHANGELOG.md              | 15 +++++++++++++--
 fim_post_processing.sh         |  6 ++++++
 tools/calibration-db/README.md |  4 +++-
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4deda8f5e..6135948ee 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.1.3.0 - 2023-02-13 - [PR#812](https://github.com/NOAA-OWP/inundation-mapping/pull/812)
+
+An update was required to adjust host name when in the AWS environment
+
+### Changes
+
+- `fim_post_processing.sh`: Added an "if isAWS" flag system based on the input command args from fim_pipeline.sh or 
+
+- `tools/calibration-db`
+    - `README.md`: Minor text correction.
+
+<br/><br/>
+
 ## v4.1.2.0 - 2023-02-15 - [PR#808](https://github.com/NOAA-OWP/inundation-mapping/pull/808)
 
 Add `pytest` package and refactor existing unit tests. Update parameters to unit tests (`/unit_tests/*_params.json`) to valid paths. Add leading slash to paths in `/config/params_template.env`.
@@ -59,7 +72,6 @@ Add `pytest` package and refactor existing unit tests. Update parameters to unit
 
 <br/><br/>
 
-
 ## v4.1.1.0 - 2023-02-16 - [PR#809](https://github.com/NOAA-OWP/inundation-mapping/pull/809)
 
 The CatFIM code was updated to allow 1-foot interval processing across all stage-based AHPS sites ranging from action stage to 5 feet above major stage, along with restart capability for interrupted processing runs.
@@ -74,7 +86,6 @@ The CatFIM code was updated to allow 1-foot interval processing across all stage
 
 <br/><br/>
 
-
 ## v4.1.0.0 - 2023-01-30 - [PR#806](https://github.com/NOAA-OWP/inundation-mapping/pull/806)
 
 As we move to Amazon Web Service, AWS, we need to change our processing system. Currently, it is `gms_pipeline.sh` using bash "parallel" as an iterator which then first processes all HUCs, but not their branches. One of `gms_pipeline.sh`'s next steps is to do branch processing which is again iterated via "parallel". AKA. Units processed as one step, branches processed as second independent step.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index c105eacca..0869b453f 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -143,6 +143,12 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
         when we start up that part of the sytem and it does not like the word
         export.
         '
+
+        # Pick up the docker parent host machine name and override the one coming from the config file (aws only)
+        if [ "$isAWS" = "1" ]; then
+            CALIBRATION_DB_HOST=$(curl http://169.254.169.254/latest/meta-data/local-ipv4 -s)
+        fi
+
         export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
         export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
         export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
diff --git a/tools/calibration-db/README.md b/tools/calibration-db/README.md
index ef85eeb21..6235d877c 100755
--- a/tools/calibration-db/README.md
+++ b/tools/calibration-db/README.md
@@ -38,7 +38,9 @@ Steps:
 Using a bash terminal window, run
     `docker-compose down`
     `rm -rf pgdata`
-
+  
+If that command errors out, use `docker-compose down --rmi local -v --remove-orphans`. 
+ 
 -------------------------------
 ## Debugging Postgres DB
 

From ff82fd363d9e0f6f053458931b701e0370170a06 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 17 Feb 2023 15:12:57 -0600
Subject: [PATCH 211/359] Delete check_unit_errors_params.json (#819)

---
 unit_tests/check_unit_errors_params.json | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 unit_tests/check_unit_errors_params.json

diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json
deleted file mode 100644
index d29b989af..000000000
--- a/unit_tests/check_unit_errors_params.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-	"valid_data":
-	{
-		"fim_dir": "/data/outputs/fim_unit_test_data_do_not_remove"
-	},
-
-	"invalid_path":
-	{
-		"fim_dir": "/data/outputs/gms_example_unit_tests_not_not_valid"
-	}	
-}
\ No newline at end of file

From 72c8cc66e41ace0bd84d71f43a9cf571f6c6d8b7 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 17 Feb 2023 15:13:41 -0600
Subject: [PATCH 212/359] Delete README.md (#818)

---
 unit_tests/README.md | 144 -------------------------------------------
 1 file changed, 144 deletions(-)
 delete mode 100644 unit_tests/README.md

diff --git a/unit_tests/README.md b/unit_tests/README.md
deleted file mode 100644
index 9725aa8cf..000000000
--- a/unit_tests/README.md
+++ /dev/null
@@ -1,144 +0,0 @@
-## Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
-
-Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
-
-#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
-
-# This folder (`/unit_tests`) holds files for unit testing python files
-
-## Creating unit tests
-
-For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and an accompanying json paramerters file. 
-
-The files should be named following FIM convention:
-
-{source py file name}_test.py ->     `derive_level_paths_test.py`  
-{source py file name}_params.json -> `derive_level_paths_params.json`
-
-
-## Tips to create a new json file for a new python unit test file.
-
-There are multiple way to figure out a set of default json parameters for the new unit test file. 
-
-One way is to use the incoming arg parser. Most python files include the code block of ` __name__ == '__main__':`, followed by external arg parsing (`args = vars(parser.parse_args()`). 
-* Add a `print(args)` or similar, and get all the values including keys as output.  
-* Copy that into an editor being used to create the json file.  
-* Add a line break after every comma.  
-* Find/replace all single quotes to double quotes then cleanup the left tab formatting.
-
-
-## Running unit tests
-
-Start a docker container as you normally would for any development. 
-```bash 
-docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
-```
-Example:
-```bash 
-docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
-```
-
-For unit tests to work, you need to run the following (if not already in place).
-Notice a modified branch "deny_gms_branch_unittests.lst"  (special for unittests)
-
-Here are the params and args you need if you need to re-run unit and branch
-
-```bash
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -bd /foss_fim/config/deny_gms_branch_unittests.lst -ud None -j 1 -o
-```
-
-**NOTICE: the deny file used for gms_run_branch... its a special one for unittests `deny_gms_branch_unittests.lst`.
-
-If you need to run inundation tests, fun the following:
-
-```bash
-python3 foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v fim_unit_test_data_do_not_remove -jh 1 -jb 1 -m /outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
-```
-### If you'd like to test the whole unit test suite:
-```
-pytest /foss_fim/unit_tests
-```
-
-This is not 100% stable, as accurate paths for the parameters `.json` files are not included in this repository, are not uniform accross machines, and are subject to change. 
- 
-### If you want to test just one unit test (from the root terminal window):
-
-```bash
-pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py 
-						or  
-pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py
-```
-
-### If you'd like to run a particular test, you can, for example:
-```
-pytest -v -s -k test_append_id_to_file_name_single_identifier_success
-```
-
-If one test case is choosen, it will scan all of the test files, and scan for the method (test case) specified. 
-
-## Key Notes for creating new unit tests
-1) All test functions must start with the phrase `test_`. That is how pytest picks them up. The rest of the function name does not have to match the pattern of `function_name_being_tested` but should. Further, the rest of the function name should say what the test is about, ie) `_failed_input_path`.  ie) `test_{some_function_name_from_the_source_code_file}_failed_input_path`. It is fine that the function names get very long (common in the industry).
-
-2) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
-
-3) As of now, you can not control the order that unit tests are run within a unit test file. 
-
-4) There must be at least one associated `{original py file name}_params.json` file per unit test.
-
-5) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to fully pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass.
-
-6) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called `valid_data`, if one does not already exist. Generally, the individual unit tests, will call the `valid_data` node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie: major different test conditions).
-
-7) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
-
-8) One Python file = one `{original py file name}_test.py` file.
-
-9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist). 
-ie:
-```bash
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_gms_unit_default.lst -o
-```
-but ours would be:
-```bash 
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o
-```
-
-## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
-
-The `pyproject.toml` file has been added, which contains the build system requirements of Python projects.  This file used to specify which warnings are disabled to pass our unit tests. 
-
-A `__init__.py` file has been added to both subdirectories (`/gms` & `/tools`) in order for the `pytest` command run in the `/unit_tests` to pick up the tests in those directories as well.
-
-Luckily, `pytest` works well with The Python Standard Library `unittest`. This made the migration of previous unit tests using `unittest` over to `pytest` quite simple. The caveat is that our current unit tests employ elements of both libraries. A full transition to `pytest` will ideally take place at a future date.
-
-## Testing for failing conditions
-- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for ONLY one variable value change. A "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
-
-- It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
-
-- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass. 
-
-An example is in `unit_tests/gms/Derive_level_paths_test.py` -> `test_Derive_level_paths_invalid_input_stream_network` (function). This example gives you the pattern implemented in Pytest.
-
-## Future Enhancements
-1) Full transition to the `pytest` library, removing classes of `unittest.TestCase` and taking full advantage of available code re-use patterns offered through `pytest`.  
-
-2) Over time, it is expected that python files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each python file which makes unit testing harder and less specific. Generally function will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample .tif or small .gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: The files `/gms/derive_level_paths_test.py` and `clip_vectors_to_wbd_test.py` are not complete as they do not yet test all output from a method.
-
-## Unit tests currently available
-```
-pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py  
-pytest /foss_fim/unit_tests/gms/outputs_cleanup_test.py  
-pytest /foss_fim/unit_tests/tools/inundate_gms_test.py  
-pytest /foss_fim/unit_tests/tools/inundation_test.py  
-pytest /foss_fim/unit_tests/check_unit_errors_test.py  
-pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py  
-pytest /foss_fim/unit_tests/filter_catchments_and_add_attributes_test.py  
-pytest /foss_fim/unit_tests/rating_curve_comparison_test.py  
-pytest /foss_fim/unit_tests/shared_functions_test.py  
-pytest /foss_fim/unit_tests/split_flows_test.py  
-pytest /foss_fim/unit_tests/usgs_gage_crosswalk_test.py
-pytest /foss_fim/unit_tests/aggregate_branch_lists_test.py
-pytest /foss_fim/unit_tests/generate_branch_list_csv_test.py
-pytest /foss_fim/unit_tests/generate_branch_list_test.py
-```

From 18b1c62eb751d8d9f5c9e169aa176fbe7292363f Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 17 Feb 2023 15:17:27 -0600
Subject: [PATCH 213/359] Delete derive_level_paths_params.json (#820)

---
 unit_tests/gms/derive_level_paths_params.json | 18 ------------------
 1 file changed, 18 deletions(-)
 delete mode 100644 unit_tests/gms/derive_level_paths_params.json

diff --git a/unit_tests/gms/derive_level_paths_params.json b/unit_tests/gms/derive_level_paths_params.json
deleted file mode 100644
index d41e79fd2..000000000
--- a/unit_tests/gms/derive_level_paths_params.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-	"valid_data":
-	{
-		"outputRunDataDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
-		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
-		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
-		"reach_id_attribute": "ID",
-		"verbose": true
-	}
-}
-
-

From c40b3cf868eaec069aed38639a1a6409d94b163b Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 17 Feb 2023 15:22:37 -0600
Subject: [PATCH 214/359] v4.2.0.0 Remove FIM3 and most references to GMS
 (#816)

---
 ...nittests.lst => deny_branch_unittests.lst} |   2 +-
 ...s_branch_zero.lst => deny_branch_zero.lst} |   2 +-
 ...eny_gms_branches.lst => deny_branches.lst} |   2 +-
 config/{deny_gms_unit.lst => deny_unit.lst}   |   2 +-
 config/params_template.env                    |   2 +-
 data/acquire_and_preprocess_inputs.py         | 397 ------------------
 data/nws/preprocess_ahps_nws.py               |  18 +
 data/usgs/acquire_and_preprocess_3dep_dems.py |   2 +-
 docs/CHANGELOG.md                             | 193 +++++++++
 fim_post_processing.sh                        |  16 +-
 fim_pre_processing.sh                         |  20 +-
 gms_pipeline.sh                               | 208 ---------
 gms_run_branch.sh                             | 244 -----------
 gms_run_post_processing.sh                    | 184 --------
 gms_run_unit.sh                               | 208 ---------
 src/add_crosswalk.py                          |  45 +-
 src/bash_variables.env                        |   1 -
 src/{gms => }/buffer_stream_branches.py       |   0
 src/{gms => }/clip_rasters_to_branches.py     |   0
 src/{gms => }/crosswalk_nwm_demDerived.py     |   4 +-
 .../delineate_hydros_and_produce_HAND.sh      |   6 +-
 src/{gms => }/derive_level_paths.py           |   6 +-
 src/{gms => }/edit_points.py                  |   0
 ...puts_by_huc.py => filter_inputs_by_huc.py} |  18 +-
 src/{gms => }/finalize_srcs.py                |   0
 src/generate_branch_list.py                   |   1 -
 src/gms/__init__.py                           |   0
 src/gms/aggregate_branch_lists.py             |  57 ---
 src/gms/generate_branch_list.py               |  36 --
 src/gms/remove_error_branches.py              |  96 -----
 src/gms/run_by_unit.sh                        | 272 ------------
 src/gms/test_new_crosswalk.sh                 |   4 -
 src/gms/time_and_tee_run_by_branch.sh         |  29 --
 src/gms/time_and_tee_run_by_unit.sh           |  29 --
 src/{gms => }/make_rem.py                     |   0
 src/{gms => }/mask_dem.py                     |   0
 src/output_cleanup.py                         | 117 ------
 src/{gms => }/outputs_cleanup.py              |   0
 src/process_branch.sh                         |   6 +-
 .../query_vectors_by_branch_polygons.py       |   0
 src/{gms => }/reset_mannings.py               |   2 +-
 src/{gms => }/run_by_branch.sh                |   6 +-
 src/run_unit_wb.sh                            |  14 +-
 src/{gms => }/stream_branches.py              |   0
 .../subset_catch_list_by_branch_id.py         |   0
 src/{gms => }/toDo.md                         |   0
 src/usgs_gage_aggregate.py                    |  12 +-
 src/usgs_gage_unit_setup.py                   |  27 +-
 src/utils/fim_enums.py                        |   4 +-
 .../combine_crosswalk_tables.py               |   0
 .../compare_ms_and_non_ms_metrics.py          |   0
 tools/{gms_tools => }/compile_comp_stats.py   |  17 +
 .../compile_computational_stats.py            |  18 +-
 tools/composite_inundation.py                 |  31 +-
 tools/consolidate_metrics.py                  |  17 +
 tools/copy_test_case_folders.py               |  32 +-
 tools/eval_plots.py                           |  12 +-
 tools/{gms_tools => }/evaluate_continuity.py  |   0
 .../find_max_catchment_breadth.py             |   0
 tools/generate_categorical_fim_mapping.py     |   4 +-
 tools/gms_tools/__init__.py                   |   0
 tools/{gms_tools => }/inundate_gms.py         |  16 +-
 tools/inundate_nation.py                      |  63 ++-
 tools/inundation.py                           |   6 +-
 tools/make_boxes_from_bounds.py               |   4 +-
 tools/{gms_tools => }/mosaic_inundation.py    |  12 +-
 .../{gms_tools => }/overlapping_inundation.py |   0
 tools/{gms_tools => }/plots.py                |   0
 tools/run_test_case.py                        |   8 +-
 tools/synthesize_test_cases.py                |   8 +-
 unit_tests/fim3_README.md                     | 104 +++++
 unit_tests/fim3__template_unittests.py        | 108 +++++
 unit_tests/fim3_check_unit_errors_params.json |  11 +
 .../fim3_check_unit_errors_unittests.py       | 278 ++++++++++++
 .../fim3_derive_level_paths_params.json       |  31 ++
 .../fim3_derive_level_paths_unittests.py      | 170 ++++++++
 ...catchments_and_add_attributes_unittests.py |  87 ++++
 unit_tests/fim3_split_flows_unittests.py      |  85 ++++
 .../{gms => }/outputs_cleanup_params.json     |   4 +-
 unit_tests/outputs_cleanup_unittests.py       | 163 +++++++
 .../tools/fim3_inundate_gms_params.json       |  16 +
 unit_tests/tools/inundate_gms_unittests.py    |  83 ++++
 82 files changed, 1597 insertions(+), 2083 deletions(-)
 rename config/{deny_gms_branch_unittests.lst => deny_branch_unittests.lst} (97%)
 rename config/{deny_gms_branch_zero.lst => deny_branch_zero.lst} (97%)
 rename config/{deny_gms_branches.lst => deny_branches.lst} (97%)
 rename config/{deny_gms_unit.lst => deny_unit.lst} (92%)
 delete mode 100755 data/acquire_and_preprocess_inputs.py
 delete mode 100755 gms_pipeline.sh
 delete mode 100755 gms_run_branch.sh
 delete mode 100755 gms_run_post_processing.sh
 delete mode 100755 gms_run_unit.sh
 rename src/{gms => }/buffer_stream_branches.py (100%)
 rename src/{gms => }/clip_rasters_to_branches.py (100%)
 rename src/{gms => }/crosswalk_nwm_demDerived.py (99%)
 rename src/{gms => }/delineate_hydros_and_produce_HAND.sh (94%)
 rename src/{gms => }/derive_level_paths.py (97%)
 rename src/{gms => }/edit_points.py (100%)
 rename src/{gms/filter_gms_inputs_by_huc.py => filter_inputs_by_huc.py} (52%)
 rename src/{gms => }/finalize_srcs.py (100%)
 delete mode 100644 src/gms/__init__.py
 delete mode 100755 src/gms/aggregate_branch_lists.py
 delete mode 100755 src/gms/generate_branch_list.py
 delete mode 100755 src/gms/remove_error_branches.py
 delete mode 100755 src/gms/run_by_unit.sh
 delete mode 100755 src/gms/test_new_crosswalk.sh
 delete mode 100755 src/gms/time_and_tee_run_by_branch.sh
 delete mode 100755 src/gms/time_and_tee_run_by_unit.sh
 rename src/{gms => }/make_rem.py (100%)
 rename src/{gms => }/mask_dem.py (100%)
 delete mode 100755 src/output_cleanup.py
 rename src/{gms => }/outputs_cleanup.py (100%)
 rename src/{gms => }/query_vectors_by_branch_polygons.py (100%)
 rename src/{gms => }/reset_mannings.py (99%)
 rename src/{gms => }/run_by_branch.sh (89%)
 rename src/{gms => }/stream_branches.py (100%)
 rename src/{gms => }/subset_catch_list_by_branch_id.py (100%)
 rename src/{gms => }/toDo.md (100%)
 rename tools/{gms_tools => }/combine_crosswalk_tables.py (100%)
 rename tools/{gms_tools => }/compare_ms_and_non_ms_metrics.py (100%)
 rename tools/{gms_tools => }/compile_comp_stats.py (81%)
 rename tools/{gms_tools => }/compile_computational_stats.py (64%)
 rename tools/{gms_tools => }/evaluate_continuity.py (100%)
 rename tools/{gms_tools => }/find_max_catchment_breadth.py (100%)
 delete mode 100644 tools/gms_tools/__init__.py
 rename tools/{gms_tools => }/inundate_gms.py (94%)
 rename tools/{gms_tools => }/mosaic_inundation.py (93%)
 rename tools/{gms_tools => }/overlapping_inundation.py (100%)
 rename tools/{gms_tools => }/plots.py (100%)
 create mode 100644 unit_tests/fim3_README.md
 create mode 100644 unit_tests/fim3__template_unittests.py
 create mode 100644 unit_tests/fim3_check_unit_errors_params.json
 create mode 100644 unit_tests/fim3_check_unit_errors_unittests.py
 create mode 100644 unit_tests/fim3_derive_level_paths_params.json
 create mode 100644 unit_tests/fim3_derive_level_paths_unittests.py
 create mode 100644 unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py
 create mode 100644 unit_tests/fim3_split_flows_unittests.py
 rename unit_tests/{gms => }/outputs_cleanup_params.json (80%)
 create mode 100644 unit_tests/outputs_cleanup_unittests.py
 create mode 100644 unit_tests/tools/fim3_inundate_gms_params.json
 create mode 100644 unit_tests/tools/inundate_gms_unittests.py

diff --git a/config/deny_gms_branch_unittests.lst b/config/deny_branch_unittests.lst
similarity index 97%
rename from config/deny_gms_branch_unittests.lst
rename to config/deny_branch_unittests.lst
index cb947ee1c..5e9a0968e 100644
--- a/config/deny_gms_branch_unittests.lst
+++ b/config/deny_branch_unittests.lst
@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif
diff --git a/config/deny_gms_branch_zero.lst b/config/deny_branch_zero.lst
similarity index 97%
rename from config/deny_gms_branch_zero.lst
rename to config/deny_branch_zero.lst
index 3bda34079..01bfaee85 100644
--- a/config/deny_gms_branch_zero.lst
+++ b/config/deny_branch_zero.lst
@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branch zero to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif
diff --git a/config/deny_gms_branches.lst b/config/deny_branches.lst
similarity index 97%
rename from config/deny_gms_branches.lst
rename to config/deny_branches.lst
index b1edb0261..9629d0317 100644
--- a/config/deny_gms_branches.lst
+++ b/config/deny_branches.lst
@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif
diff --git a/config/deny_gms_unit.lst b/config/deny_unit.lst
similarity index 92%
rename from config/deny_gms_unit.lst
rename to config/deny_unit.lst
index 0ad10cc82..b4680544e 100644
--- a/config/deny_gms_unit.lst
+++ b/config/deny_unit.lst
@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for units to delete
 # Use comment to allow list the file
 #branch_id.lst
 #branch_polygons.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index f278139e5..2c8eb2a1b 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -20,7 +20,7 @@ export slope_min=0.001
 export min_catchment_area=0.25
 export min_stream_length=0.5
 
-#### gms parameters ####
+#### branch parameters ####
 export branch_id_attribute=levpa_id
 export branch_buffer_distance_meters=7000
 export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.
diff --git a/data/acquire_and_preprocess_inputs.py b/data/acquire_and_preprocess_inputs.py
deleted file mode 100755
index 37cb33e3d..000000000
--- a/data/acquire_and_preprocess_inputs.py
+++ /dev/null
@@ -1,397 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import argparse
-import csv
-import sys
-sys.path.append('/foss_fim/src')
-import shutil
-from multiprocessing import Pool
-import geopandas as gpd
-from urllib.error import HTTPError
-from tqdm import tqdm
-
-from utils.shared_variables import (NHD_URL_PARENT,
-                                    NHD_URL_PREFIX,
-                                    NHD_RASTER_URL_SUFFIX,
-                                    NHD_VECTOR_URL_SUFFIX,
-                                    NHD_VECTOR_EXTRACTION_PREFIX,
-                                    NHD_VECTOR_EXTRACTION_SUFFIX,
-                                    PREP_PROJECTION,
-                                    WBD_NATIONAL_URL,
-                                    FIM_ID,
-                                    OVERWRITE_WBD,
-                                    OVERWRITE_NHD,
-                                    OVERWRITE_ALL,
-                                    nhd_raster_url_template,
-                                    nhd_vector_url_template
-                                    )
-
-from utils.shared_functions import (pull_file, run_system_command,
-                                    subset_wbd_gpkg, delete_file,
-                                    getDriver)
-
-NHDPLUS_VECTORS_DIRNAME = 'nhdplus_vectors'
-NHDPLUS_RASTERS_DIRNAME = 'nhdplus_rasters'
-NWM_HYDROFABRIC_DIRNAME = 'nwm_hydrofabric'
-NWM_FILE_TO_SUBSET_WITH = 'nwm_flows.gpkg'
-
-def subset_wbd_to_nwm_domain(wbd,nwm_file_to_use):
-
-    intersecting_indices = [not (gpd.read_file(nwm_file_to_use,mask=b).empty) for b in wbd.geometry]
-
-    return(wbd[intersecting_indices])
-
-def pull_and_prepare_wbd(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_use,overwrite_wbd,num_workers):
-    """
-    This helper function pulls and unzips Watershed Boundary Dataset (WBD) data. It uses the WBD URL defined by WBD_NATIONAL_URL.
-    This function also subsets the WBD layers (HU4, HU6, HU8) to CONUS and converts to geopkacage layers.
-
-    Args:
-        path_to_saved_data_parent_dir (str): The system path to where the WBD will be downloaded, unzipped, and preprocessed.
-
-    """
-
-    # Construct path to wbd_directory and create if not existent.
-    wbd_directory = os.path.join(path_to_saved_data_parent_dir, 'wbd')
-    if not os.path.exists(wbd_directory):
-        os.mkdir(wbd_directory)
-
-    wbd_gdb_path = os.path.join(wbd_directory, 'WBD_National_GDB.gdb')
-    pulled_wbd_zipped_path = os.path.join(wbd_directory, 'WBD_National_GDB.zip')
-
-    multilayer_wbd_geopackage = os.path.join(wbd_directory, 'WBD_National.gpkg')
-
-    nwm_huc_list_file_template = os.path.join(wbd_directory,'nwm_wbd{}.csv')
-
-    nwm_file_to_use = os.path.join(path_to_saved_data_parent_dir,nwm_dir_name,nwm_file_to_use)
-    if not os.path.isfile(nwm_file_to_use):
-        raise IOError("NWM File to Subset Too Not Available: {}".format(nwm_file_to_use))
-
-    if not os.path.exists(multilayer_wbd_geopackage) or overwrite_wbd:
-        # Download WBD and unzip if it's not already done.
-        if not os.path.exists(wbd_gdb_path):
-            if not os.path.exists(pulled_wbd_zipped_path):
-                pull_file(WBD_NATIONAL_URL, pulled_wbd_zipped_path)
-            os.system("7za x {pulled_wbd_zipped_path} -o{wbd_directory}".format(pulled_wbd_zipped_path=pulled_wbd_zipped_path, wbd_directory=wbd_directory))
-
-        procs_list, wbd_gpkg_list = [], []
-        multilayer_wbd_geopackage = os.path.join(wbd_directory, 'WBD_National.gpkg')
-        # Add fimid to HU8, project, and convert to geopackage.
-        if os.path.isfile(multilayer_wbd_geopackage):
-            os.remove(multilayer_wbd_geopackage)
-        print("Making National WBD GPKG...")
-        print("\tWBDHU8")
-        wbd_hu8 = gpd.read_file(wbd_gdb_path, layer='WBDHU8')
-        wbd_hu8 = wbd_hu8.rename(columns={'huc8':'HUC8'}) # rename column to caps
-        wbd_hu8 = wbd_hu8.sort_values('HUC8')
-        fimids = [str(item).zfill(4) for item in list(range(1000, 1000 + len(wbd_hu8)))]
-        wbd_hu8[FIM_ID] = fimids
-        wbd_hu8 = wbd_hu8.to_crs(PREP_PROJECTION)  # Project.
-        wbd_hu8 = subset_wbd_to_nwm_domain(wbd_hu8,nwm_file_to_use)
-        wbd_hu8.geometry = wbd_hu8.buffer(0)
-        wbd_hu8.to_file(multilayer_wbd_geopackage,layer='WBDHU8',driver=getDriver(multilayer_wbd_geopackage),index=False)  # Save.
-        wbd_hu8.HUC8.to_csv(nwm_huc_list_file_template.format('8'),index=False,header=False)
-        #wbd_gpkg_list.append(os.path.join(wbd_directory, 'WBDHU8.gpkg'))  # Append to wbd_gpkg_list for subsetting later.
-        del wbd_hu8
-
-        # Prepare procs_list for multiprocessed geopackaging.
-        for wbd_layer_num in ['4', '6']:
-            wbd_layer = 'WBDHU' + wbd_layer_num
-            print("\t{}".format(wbd_layer))
-            wbd = gpd.read_file(wbd_gdb_path,layer=wbd_layer)
-            wbd = wbd.to_crs(PREP_PROJECTION)
-            wbd = wbd.rename(columns={'huc'+wbd_layer_num : 'HUC' + wbd_layer_num})
-            wbd = subset_wbd_to_nwm_domain(wbd,nwm_file_to_use)
-            wbd.geometry = wbd.buffer(0)
-            wbd.to_file(multilayer_wbd_geopackage,layer=wbd_layer,driver=getDriver(multilayer_wbd_geopackage),index=False)
-            wbd['HUC{}'.format(wbd_layer_num)].to_csv(nwm_huc_list_file_template.format(wbd_layer_num),index=False,header=False)
-            #output_gpkg = os.path.join(wbd_directory, wbd_layer + '.gpkg')
-            #wbd_gpkg_list.append(output_gpkg)
-            #procs_list.append(['ogr2ogr -overwrite -progress -f GPKG -t_srs "{projection}" {output_gpkg} {wbd_gdb_path} {wbd_layer}'.format(output_gpkg=output_gpkg, wbd_gdb_path=wbd_gdb_path, wbd_layer=wbd_layer, projection=PREP_PROJECTION)])
-
-        # with Pool(processes=num_workers) as pool:
-            # pool.map(run_system_command, procs_list)
-
-        # Subset WBD layers to CONUS and add to single geopackage.
-        #print("Subsetting WBD layers to CONUS...")
-        #multilayer_wbd_geopackage = os.path.join(wbd_directory, 'WBD_National.gpkg')
-        #for gpkg in wbd_gpkg_list:
-        #    subset_wbd_gpkg(gpkg, multilayer_wbd_geopackage)
-
-    # Clean up temporary files.
-    #for temp_layer in ['WBDHU4', 'WBDHU6', 'WBDHU8']:
-    #    delete_file(os.path.join(wbd_directory, temp_layer + '.gpkg'))
-    #pulled_wbd_zipped_path = os.path.join(wbd_directory, 'WBD_National_GDB.zip')
-    #delete_file(pulled_wbd_zipped_path)
-    #delete_file(os.path.join(wbd_directory, 'WBD_National_GDB.jpg'))
-
-    return(wbd_directory)
-
-def pull_and_prepare_nwm_hydrofabric(path_to_saved_data_parent_dir, path_to_preinputs_dir,num_workers):
-    """
-    This helper function pulls and unzips NWM hydrofabric data. It uses the NWM hydrofabric URL defined by NWM_HYDROFABRIC_URL.
-
-    Args:
-        path_to_saved_data_parent_dir (str): The system path to where a 'nwm' subdirectory will be created and where NWM hydrofabric
-        will be downloaded, unzipped, and preprocessed.
-
-    """
-
-    # -- Acquire and preprocess NWM data -- #
-    nwm_hydrofabric_directory = os.path.join(path_to_saved_data_parent_dir, 'nwm_hydrofabric')
-    if not os.path.exists(nwm_hydrofabric_directory):
-        os.mkdir(nwm_hydrofabric_directory)
-
-    nwm_hydrofabric_gdb = os.path.join(path_to_preinputs_dir, 'nwm_v21.gdb')
-
-    # Project and convert to geopackage.
-    print("Projecting and converting NWM layers to geopackage...")
-    procs_list = []
-    for nwm_layer in ['nwm_flows', 'nwm_lakes', 'nwm_catchments']:  # I had to project the catchments and waterbodies because these 3 layers had varying CRSs.
-        print("Operating on " + nwm_layer)
-        output_gpkg = os.path.join(nwm_hydrofabric_directory, nwm_layer + '_proj.gpkg')
-        procs_list.append(['ogr2ogr -overwrite -progress -f GPKG -t_srs "{projection}" {output_gpkg} {nwm_hydrofabric_gdb} {nwm_layer}'.format(projection=PREP_PROJECTION, output_gpkg=output_gpkg, nwm_hydrofabric_gdb=nwm_hydrofabric_gdb, nwm_layer=nwm_layer)])
-
-    with Pool(processes=num_workers) as pool:
-        pool.map(run_system_command, procs_list)
-
-
-def pull_and_prepare_nhd_data(nhd_raster_download_url,
-                              nhd_raster_extraction_path,
-                              nhd_vector_download_url,
-                              nhd_vector_extraction_path,
-                              overwrite_nhd_dem,
-                              overwrite_nhd_gdb
-                             ):
-    """
-    This helper function is designed to be multiprocessed. It pulls and unzips NHD raster and vector data.
-    Args:
-        args (list): A list of arguments in this format: [nhd_raster_download_url, nhd_raster_extraction_path, nhd_vector_download_url, nhd_vector_extraction_path]
-    """
-    # Update extraction path from .zip to .gdb
-    nhd_gdb = nhd_vector_extraction_path.replace('.zip', '.gdb')
-
-    # Download raster and vector, if not already in user's directory (exist check performed by pull_file()).
-    nhd_raster_extraction_parent = os.path.dirname(nhd_raster_extraction_path)
-    huc = os.path.basename(nhd_raster_extraction_path).split('_')[2]
-
-    nhd_raster_parent_dir = os.path.join(nhd_raster_extraction_parent, 'HRNHDPlusRasters' + huc)
-
-    if not os.path.exists(nhd_raster_parent_dir):
-        os.mkdir(nhd_raster_parent_dir)
-
-    elev_cm_tif = os.path.join(nhd_raster_parent_dir, 'elev_cm.tif')
-    elev_m_tif = os.path.join(nhd_raster_parent_dir, 'elev_m.tif')
-    if not os.path.exists(elev_cm_tif) or overwrite_nhd_dem:
-        pull_file(nhd_raster_download_url, nhd_raster_extraction_path)
-        os.system("7za e {nhd_raster_extraction_path} -o{nhd_raster_parent_dir} elev_cm.tif -r ".format(nhd_raster_extraction_path=nhd_raster_extraction_path, nhd_raster_parent_dir=nhd_raster_parent_dir))
-
-        file_list = os.listdir(nhd_raster_parent_dir)
-        for f in file_list:
-            full_path = os.path.join(nhd_raster_parent_dir, f)
-            if 'elev_cm' not in f:
-                if os.path.isdir(full_path):
-                    shutil.rmtree(full_path)
-                elif os.path.isfile(full_path):
-                    os.remove(full_path)
-        os.remove(nhd_raster_extraction_path)
-
-    nhd_vector_extraction_parent = os.path.dirname(nhd_vector_extraction_path)
-
-    if not os.path.exists(nhd_vector_extraction_parent):
-        os.mkdir(nhd_vector_extraction_parent)
-
-    if not os.path.exists(nhd_gdb) or overwrite_nhd_gdb:  # Only pull if not already pulled and processed.
-        # Download and fully unzip downloaded GDB.
-        pull_file(nhd_vector_download_url, nhd_vector_extraction_path)
-        huc = os.path.split(nhd_vector_extraction_parent)[1]  # Parse HUC.
-        os.system("7za x {nhd_vector_extraction_path} -o{nhd_vector_extraction_parent}".format(nhd_vector_extraction_path=nhd_vector_extraction_path, nhd_vector_extraction_parent=nhd_vector_extraction_parent))
-        # extract input stream network
-        nhd = gpd.read_file(nhd_gdb,layer='NHDPlusBurnLineEvent')
-        nhd = nhd.to_crs(PREP_PROJECTION)
-        nhd.to_file(os.path.join(nhd_vector_extraction_parent, 'NHDPlusBurnLineEvent' + huc + '.gpkg'),driver='GPKG')
-        # extract flowlines for FType attributes
-        nhd = gpd.read_file(nhd_gdb,layer='NHDFlowline')
-        nhd = nhd.to_crs(PREP_PROJECTION)
-        nhd.to_file(os.path.join(nhd_vector_extraction_parent, 'NHDFlowline' + huc + '.gpkg'),driver='GPKG')
-        # extract attributes
-        nhd = gpd.read_file(nhd_gdb,layer='NHDPlusFlowLineVAA')
-        nhd.to_file(os.path.join(nhd_vector_extraction_parent, 'NHDPlusFlowLineVAA' + huc + '.gpkg'),driver='GPKG')
-        # -- Project and convert NHDPlusBurnLineEvent and NHDPlusFlowLineVAA vectors to geopackage -- #
-        #for nhd_layer in ['NHDPlusBurnLineEvent', 'NHDPlusFlowlineVAA']:
-        #    run_system_command(['ogr2ogr -overwrite -progress -f GPKG -t_srs "{projection}" {output_gpkg} {nhd_gdb} {nhd_layer}'.format(projection=PREP_PROJECTION, output_gpkg=output_gpkg, nhd_gdb=nhd_gdb, nhd_layer=nhd_layer)])  # Use list because function is configured for multiprocessing.
-    # Delete unnecessary files.
-    delete_file(nhd_vector_extraction_path.replace('.zip', '.jpg'))
-    delete_file(nhd_vector_extraction_path)  # Delete the zipped GDB.
-
-
-def build_huc_list_files(path_to_saved_data_parent_dir, wbd_directory):
-    """
-    This function builds a list of available HUC4s, HUC6s, and HUC8s and saves the lists to .lst files.
-
-    Args:
-        path_to_saved_data_parent_dir (str): The path to the parent directory where the .lst files will be saved.
-        wbd_directory (str): The path to the directory storing the WBD geopackages which are used to determine which HUCs are available for processing.
-
-    """
-
-    print("Building included HUC lists...")
-    # Identify all saved NHDPlus Vectors.
-    nhd_plus_raster_dir = os.path.join(path_to_saved_data_parent_dir, NHDPLUS_RASTERS_DIRNAME)
-    nhd_plus_vector_dir = os.path.join(path_to_saved_data_parent_dir, NHDPLUS_VECTORS_DIRNAME)
-
-    huc4_list = [i[-4:] for i in os.listdir(nhd_plus_raster_dir)]
-    huc6_list, huc8_list = [], []
-
-    # Read WBD into dataframe.
-    full_huc_gpkg = os.path.join(wbd_directory, 'WBD_National.gpkg')
-    huc_gpkg = 'WBDHU8' # The WBDHU4 are handled by the nhd_plus_raster_dir name.
-
-    # Open geopackage.
-    wbd = gpd.read_file(full_huc_gpkg, layer=huc_gpkg)
-
-    # Loop through entries and compare against the huc4_list to get available HUCs within the geopackage domain.
-    for index, row in tqdm(wbd.iterrows(),total=len(wbd)):
-        huc = row["HUC" + huc_gpkg[-1]]
-        huc_mask = wbd.loc[wbd[str("HUC" + huc_gpkg[-1])]==huc].geometry
-        burnline = os.path.join(nhd_plus_vector_dir, huc[0:4], 'NHDPlusBurnLineEvent' + huc[0:4] + '.gpkg')
-        if os.path.exists(burnline):
-            nhd_test = len(gpd.read_file(burnline, mask = huc_mask)) # this is slow, iterates through 2000+ HUC8s
-            # Append huc to huc8 list.
-            if (str(huc[:4]) in huc4_list) & (nhd_test>0):
-                huc8_list.append(huc)
-
-    huc6_list = [w[:6] for w in huc8_list]
-    huc6_list = set(huc6_list)
-
-    # Write huc lists to appropriate .lst files.
-    huc_lists_dir = os.path.join(path_to_saved_data_parent_dir, 'huc_lists')
-    if not os.path.exists(huc_lists_dir):
-        os.mkdir(huc_lists_dir)
-    included_huc4_file = os.path.join(huc_lists_dir, 'included_huc4.lst')
-    included_huc6_file = os.path.join(huc_lists_dir, 'included_huc6.lst')
-    included_huc8_file = os.path.join(huc_lists_dir, 'included_huc8.lst')
-
-    # Overly verbose file writing loops. Doing this in a pinch.
-    with open(included_huc4_file, 'w') as f:
-        for item in huc4_list:
-            f.write("%s\n" % item)
-
-    with open(included_huc6_file, 'w') as f:
-        for item in huc6_list:
-            f.write("%s\n" % item)
-
-    with open(included_huc8_file, 'w') as f:
-        for item in huc8_list:
-            f.write("%s\n" % item)
-
-
-def manage_preprocessing( hucs_of_interest,
-                          num_workers=1,
-                          overwrite_nhd_dem=False,
-                          overwrite_nhd_gdb=False,
-                          overwrite_wbd=False
-                        ):
-    """
-    This functions manages the downloading and preprocessing of gridded and vector data for FIM production.
-
-    Args:
-        hucs_of_interest (str): Path to a user-supplied config file of hydrologic unit codes to be pulled and post-processed.
-
-    """
-
-    #get input data dir
-    path_to_saved_data_parent_dir = os.environ['inputDataDir']
-
-    nhd_procs_list = []  # Initialize procs_list for multiprocessing.
-
-    # Create the parent directory if nonexistent.
-    if not os.path.exists(path_to_saved_data_parent_dir):
-        os.mkdir(path_to_saved_data_parent_dir)
-
-    # Create NHDPlus raster parent directory if nonexistent.
-    nhd_raster_dir = os.path.join(path_to_saved_data_parent_dir, NHDPLUS_RASTERS_DIRNAME)
-    if not os.path.exists(nhd_raster_dir):
-        os.mkdir(nhd_raster_dir)
-
-    # Create the vector data parent directory if nonexistent.
-    vector_data_dir = os.path.join(path_to_saved_data_parent_dir, NHDPLUS_VECTORS_DIRNAME)
-    if not os.path.exists(vector_data_dir):
-        os.mkdir(vector_data_dir)
-
-    # Parse HUCs from hucs_of_interest.
-    if isinstance(hucs_of_interest,list):
-        if len(hucs_of_interest) == 1:
-            try:
-                with open(hucs_of_interest[0]) as csv_file:  # Does not have to be CSV format.
-                    huc_list = [i[0] for i in csv.reader(csv_file)]
-            except FileNotFoundError:
-                huc_list = hucs_of_interest
-        else:
-                huc_list = hucs_of_interest
-    elif isinstance(hucs_of_interest,str):
-        try:
-            with open(hucs_of_interest) as csv_file:  # Does not have to be CSV format.
-                huc_list = [i[0] for i in csv.reader(csv_file)]
-        except FileNotFoundError:
-            huc_list = list(hucs_of_interest)
-
-    # get unique huc4s
-    huc_list = [h[0:4] for h in huc_list]
-    huc_list = list( set(huc_list) )
-
-    # Construct paths to data to download and append to procs_list for multiprocessed pull, project, and converstion to geopackage.
-    for huc in huc_list:
-        huc = str(huc)  # Ensure huc is string.
-
-        # Construct URL and extraction path for NHDPlus raster.
-        #nhd_raster_download_url = os.path.join(NHD_URL_PARENT, NHD_URL_PREFIX + huc + NHD_RASTER_URL_SUFFIX)
-        nhd_raster_download_url = nhd_raster_url_template.format(huc) 
-        nhd_raster_extraction_path = os.path.join(nhd_raster_dir, NHD_URL_PREFIX + huc + NHD_RASTER_URL_SUFFIX)
-        
-        # Construct URL and extraction path for NHDPlus vector. Organize into huc-level subdirectories.
-        #nhd_vector_download_url = os.path.join(NHD_URL_PARENT, NHD_URL_PREFIX + huc + NHD_VECTOR_URL_SUFFIX)
-        nhd_vector_download_url = nhd_vector_url_template.format(huc) 
-        nhd_vector_download_parent = os.path.join(vector_data_dir, huc)
-        if not os.path.exists(nhd_vector_download_parent):
-            os.mkdir(nhd_vector_download_parent)
-        nhd_vector_extraction_path = os.path.join(nhd_vector_download_parent, NHD_VECTOR_EXTRACTION_PREFIX + huc + NHD_VECTOR_EXTRACTION_SUFFIX)
-
-        # Append extraction instructions to nhd_procs_list.
-        nhd_procs_list.append([nhd_raster_download_url, nhd_raster_extraction_path, nhd_vector_download_url, nhd_vector_extraction_path, overwrite_nhd_dem, overwrite_nhd_gdb])
-
-    # Pull and prepare NHD data.
-    # with Pool(processes=num_workers) as pool:
-        # pool.map(pull_and_prepare_nhd_data, nhd_procs_list)
-
-    for huc in nhd_procs_list:
-        try:
-            pull_and_prepare_nhd_data(*huc)
-        except HTTPError:
-            print("404 error for HUC4 {}".format(huc))
-
-    # Pull and prepare NWM data.
-    #pull_and_prepare_nwm_hydrofabric(path_to_saved_data_parent_dir, path_to_preinputs_dir,num_workers)  # Commented out for now.
-
-    # Pull and prepare WBD data.
-    wbd_directory = pull_and_prepare_wbd(path_to_saved_data_parent_dir,NWM_HYDROFABRIC_DIRNAME,NWM_FILE_TO_SUBSET_WITH,overwrite_wbd,num_workers)
-
-    # Create HUC list files.
-    build_huc_list_files(path_to_saved_data_parent_dir, wbd_directory)
-
-
-if __name__ == '__main__':
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Acquires and preprocesses WBD and NHD data for use in fim_run.sh.')
-    parser.add_argument('-u','--hucs-of-interest',help='HUC4, series of HUC4s, or path to a line-delimited file of HUC4s to acquire.',required=True,nargs='+')
-    #parser.add_argument('-j','--num-workers',help='Number of workers to process with',required=False,default=1,type=int)
-    parser.add_argument('-nd', '--overwrite-nhd-dem', help='Optional flag to overwrite NHDPlus DEM Data',required=False,action='store_true',default=False)
-    parser.add_argument('-ng', '--overwrite-nhd-gdb', help='Optional flag to overwrite NHDPlus GDB Data',required=False,action='store_true',default=False)
-    parser.add_argument('-w', '--overwrite-wbd', help='Optional flag to overwrite WBD Data',required=False,action='store_true')
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    manage_preprocessing(**args)
diff --git a/data/nws/preprocess_ahps_nws.py b/data/nws/preprocess_ahps_nws.py
index 4659e3dc8..9bba37573 100644
--- a/data/nws/preprocess_ahps_nws.py
+++ b/data/nws/preprocess_ahps_nws.py
@@ -14,6 +14,15 @@
 from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 
 
+########################################################
+'''
+Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+   a significant review and/or upgrade.
+'''
+
+########################################################
+
+
 def get_env_paths():
     load_dotenv()
     #import variables from .env file
@@ -354,6 +363,15 @@ def preprocess_nws(source_dir, destination, reference_raster):
 
 
 if __name__ == '__main__':
+    
+    ########################################################
+    '''
+    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+    a significant review and/or upgrade.
+    '''
+
+    ########################################################
+    
     #Parse arguments
     parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
     parser.add_argument('-s', '--source_dir', help = 'Workspace where all source data is located.', required = True)
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index 828c9cdef..1323c8572 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -174,7 +174,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
 
                 sys.exit(1)
             
-        # Send the executor to the progress bar and wait for all FR tasks to finish
+        # Send the executor to the progress bar and wait for all tasks to finish
         sf.progress_bar_handler(executor_dict, f"Downloading USGG 3Dep Dems")
 
     print(f"-- Downloading USGS DEMs Completed")
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 6135948ee..d8b9770ae 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -70,6 +70,199 @@ Add `pytest` package and refactor existing unit tests. Update parameters to unit
       - `inundate_gms_test.py`
 
 
+<br/><br/>
+
+## v4.1.1.0 - 2023-02-16 - [PR#809](https://github.com/NOAA-OWP/inundation-mapping/pull/809)
+
+The CatFIM code was updated to allow 1-foot interval processing across all stage-based AHPS sites ranging from action stage to 5 feet above major stage, along with restart capability for interrupted processing runs.
+
+### Changes
+
+- `tools/generate_categorical_fim.py` (all changes made here)
+    - Added try-except blocks for code that didn't allow most sites to actually get processed because it was trying to check values of some USGS-related variables that most of the sites didn't have
+    - Overwrite abilities of the different outputs for the viz team were not consistent (i.e., one of the files had the ability to be overwritten but another didn't), so that has been made consistent to disallow any overwrites of the existing final outputs for a specified output folder.
+    - The code also has the ability to restart from an interrupted run and resume processing uncompleted HUCs by first checking for a simple "complete" file for each HUC. If a HUC has that file, then it is skipped (because it already completed processing during a run for a particular output folder / run name).
+    - When a HUC is successfully processed, an empty "complete" text file is created / touched.
+
+<br/><br/>
+
+## v4.2.0.0 - 2023-02-16 - [PR#816](https://github.com/NOAA-OWP/inundation-mapping/pull/816)
+
+This update removes the remaining elements of FIM3 code.  It further removes the phrases "GMS" as basically the entire FIM4 model. FIM4 is GMS. With removing FIM3, it also means remove concepts of "MS" and "FR" which were no longer relevant in FIM4.  There are only a few remaining places that will continue with the phrase "GMS" which is in some inundation files which are being re-evaluated.  Some deprecated files have been removed and some subfolders removed.
+
+There are a lot of duplicate explanations for some of the changes, so here is a shortcut system.
+
+- desc 1:  Remove or rename values based on phrase "GMS, MS and/or FR"
+- desc 2:  Moved file from the /src/gms folder to /src  or /tools/gms_tools to /tools
+- desc 3:  No longer needed as we now use the `fim_pipeline.sh` processing model.
+
+### Removals
+
+- `data`
+    - `acquire_and_preprocess_inputs.py`:  No longer needed
+- `gms_pipeline.sh` : see desc 3
+- `gms_run_branch.sh` : see desc 3
+- `gms_run_post_processing.sh` : see desc 3
+- `gms_run_unit.sh` : see desc 3
+- `src`
+    - `gms`
+        - `init.py` : folder removed, no longer needed.
+        - `aggregate_branch_lists.py`: no longer needed.  Newer version already exists in src directory.
+        - `remove_error_branches.py` :  see desc 3
+        - `run_by_unit.sh` : see desc 3
+        - `test_new_crosswalk.sh` : no longer needed
+        - `time_and_tee_run_by_branch.sh` : see desc 3
+        - `time_and_tee_run_by_unit.sh` : see desc 3
+    - `output_cleanup.py` : see desc 3
+ - `tools/gms_tools`
+     - `init.py` : folder removed, no longer needed.
+
+### Changes
+
+- `config`
+   - `deny_branch_unittests.lst` :  renamed from `deny_gms_branch_unittests.lst`
+   - `deny_branch_zero.lst` : renamed from `deny_gms_branch_zero.lst`
+   - `deny_branches.lst` :  renamed from `deny_gms_branches.lst`
+   - `deny_unit.lst`  : renamed from `deny_gms_unit.lst`
+   - `params_template.env` : see desc 1
+ 
+- `data`
+    - `nws`
+        - `preprocess_ahps_nws.py`:   Added deprecation note: If reused, it needs review and/or upgrades.
+    - `acquire_and_preprocess_3dep_dems.py` : see desc 1
+ - `fim_post_processing.sh` : see desc 1, plus a small pathing change.
+ - `fim_pre_processing.sh` : see desc 1
+ - ` src`
+     - `add_crosswalk.py` : see desc 1. Also cleaned up some formatting and commented out a code block in favor of a better way to pass args from "__main__"
+     - `bash_variables.env` : see desc 1
+     - `buffer_stream_branches.py` : see desc 2
+     - `clip_rasters_to_branches.py` : see desc 2
+     - `crosswalk_nwm_demDerived.py` :  see desc 1 and desc 2
+     - `delineate_hydros_and_produce_HAND.sh` : see desc 1 and desc 2
+     - `derive_level_paths.py`  :  see desc 1 and desc 2
+     - `edit_points.py` : see desc  2
+     - `filter_inputs_by_huc.py`: see desc 1 and desc 2
+     - `finalize_srcs.py`:  see desc 2
+     - `generate_branch_list.py` : see desc 1
+     - `make_rem.py` : see desc 2
+     - `make_dem.py` : see desc  2
+     - `outputs_cleanup.py`:  see desc 1
+     - `process_branch.sh`:  see desc 1
+     - `query_vectors_by_branch_polygons.py`: see desc 2
+     - `reset_mannings.py` : see desc 2
+     - `run_by_branch.sh`:  see desc 1
+     - `run_unit_wb.sh`: see desc 1
+     - `stream_branches.py`:  see desc 2
+     - `subset_catch_list_by_branch_id.py`: see desc 2
+     - `toDo.md`: see desc 2
+     - `usgs_gage_aggregate.py`:  see desc 1
+     - `usgs_gage_unit_setup.py` : see desc 1
+     - `utils`
+         - `fim_enums.py` : see desc 1
+         
+- `tools`
+    - `combine_crosswalk_tables.py` : see desc 2
+    - `compare_ms_and_non_ms_metrics.py` : see desc 2
+    - `compile_comp_stats.py`: see desc 2  and added note about possible deprecation.
+    - `compile_computation_stats.py` : see desc 2  and added note about possible deprecation.
+    - `composite_inundation.py` : see desc 1 : note.. references a file called inundate_gms which retains it's name for now.
+    - `consolidate_metrics.py`: added note about possible deprecation.
+    - `copy_test_case_folders.py`: see desc 1
+    - `eval_plots.py` : see desc 1
+    - `evaluate_continuity.py`: see desc 2
+    - `find_max_catchment_breadth.py` : see desc 2
+    - `generate_categorical_fim_mapping.py` : see desc 1
+    - `inundate_gms.py`: see desc 1 and desc 2. Note: This file has retained its name with the phrase "gms" in it as it might be upgraded later and there are some similar files with similar names.
+    - `inundate_nation.py` : see desc 1
+    - `inundation.py`:  text styling change
+    - `make_boxes_from_bounds.py`: text styling change
+    - `mosaic_inundation.py`:  see desc 1 and desc 2
+    - `overlapping_inundation.py`: see desc 2
+    - `plots.py` : see desc 2
+    - `run_test_case.py`:  see desc 1
+    - `synthesize_test_cases.py`: see desc 1
+    
+- `unit_tests`
+    - `README.md`: see desc 1
+    - `__template_unittests.py`: see desc 1
+    - `check_unit_errors_params.json`  and `check_unit_errors_unittests.py` : see desc 1
+    - `derive_level_paths_params.json` and `derive_level_paths_unittests.py` : see desc 1 and desc 2
+    - `filter_catchments_and_add_attributes_unittests.py`: see desc 1
+    - `outputs_cleanup_params.json` and `outputs_cleanup_unittests.py`: see desc 1 and desc 2
+    - `split_flows_unittests.py` : see desc 1
+    - `tools`
+        - `inundate_gms_params.json` and `inundate_gms_unittests.py`: see desc 1 and desc 2
+
+<br/><br/>
+## v4.1.3.0 - 2023-02-13 - [PR#812](https://github.com/NOAA-OWP/inundation-mapping/pull/812)
+
+An update was required to adjust host name when in the AWS environment
+
+### Changes
+
+- `fim_post_processing.sh`: Added an "if isAWS" flag system based on the input command args from fim_pipeline.sh or 
+
+- `tools/calibration-db`
+    - `README.md`: Minor text correction.
+
+<br/><br/>
+
+## v4.1.2.0 - 2023-02-15 - [PR#808](https://github.com/NOAA-OWP/inundation-mapping/pull/808)
+
+Add `pytest` package and refactor existing unit tests. Update parameters to unit tests (`/unit_tests/*_params.json`) to valid paths. Add leading slash to paths in `/config/params_template.env`.
+
+### Additions
+
+- `/unit_tests`
+  - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
+  - `pyproject.toml`  - used to specify which warnings are excluded/filtered.
+  - `/gms`  
+    - `__init__.py` - needed for `pytest` command line executable to pick up tests.
+  - `/tools`
+    - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
+    - `inundate_gms_params.json` - file moved up into this directory
+    - `inundate_gms_test.py`     - file moved up into this directory
+    - `inundation_params.json`   - file moved up into this directory
+    - `inundation_test.py`       - file moved up into this directory
+
+### Removals
+
+- `/unit_tests/tools/gms_tools/` directory removed, and files moved up into `/unit_tests/tools`    
+ 
+### Changes
+
+- `Pipfile` - updated to include pytest as a dependency
+- `Pipfile.lock` - updated to include pytest as a dependency
+
+- `/config`
+  - `params_template.env` - leading slash added to paths
+  
+- `/unit_tests/` - All of the `*_test.py` files were refactored to follow the `pytest` paradigm.  
+  - `*_params.json` - valid paths on `fim-dev1` provided
+  - `README.md`  - updated to include documentation on pytest.  
+  - `unit_tests_utils.py`  
+  - `__template_unittests.py` -> `__template.py` - exclude the `_test` suffix to remove from test suite. Updated example on new format for pytest.
+  - `check_unit_errors_test.py`  
+  - `clip_vectors_to_wbd_test.py`  
+  - `filter_catchments_and_add_attributes_test.py`  
+  - `rating_curve_comparison_test.py`  
+  - `shared_functions_test.py`  
+  - `split_flow_test.py`  
+  - `usgs_gage_crosswalk_test.py`  
+  - `aggregate_branch_lists_test.py`  
+  - `generate_branch_list_test.py`  
+  - `generate_branch_list_csv_test.py`  
+  - `aggregate_branch_lists_test.py`  
+  - `generate_branch_list_csv_test.py`  
+  - `generate_branch_list_test.py`  
+    - `/gms`  
+      - `derive_level_paths_test.py`  
+      - `outputs_cleanup_test.py`
+    - `/tools`
+      - `inundate_unittests.py` -> `inundation_test.py`  
+      - `inundate_gms_test.py`
+
+
 <br/><br/>
 
 ## v4.1.1.0 - 2023-02-16 - [PR#809](https://github.com/NOAA-OWP/inundation-mapping/pull/809)
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 0869b453f..0367f24cc 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -69,7 +69,7 @@ rm -f $outputRunDataDir/logs/subdiv_src_.log
 
 # load up enviromental information
 args_file=$outputRunDataDir/runtime_args.env
-gms_inputs=$outputRunDataDir/gms_inputs.csv
+fim_inputs=$outputRunDataDir/fim_inputs.csv
 
 source $args_file
 source $outputRunDataDir/params.env
@@ -87,7 +87,7 @@ post_proc_start_time=`date +%s`
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
 echo -e $startDiv"Start branch aggregation"
-python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $gms_inputs
+python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $fim_inputs
 
 ## GET NON ZERO EXIT CODES FOR UNITS ##
 ## No longer applicable
@@ -96,17 +96,9 @@ python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.cs
 echo -e $startDiv"Start non-zero exit code checking"
 find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
 
-
-## REMOVE FAILED BRANCHES ##
-# Needed in case aggregation fails, we will need the logs
-#echo
-#echo -e $startDiv"Removing branches that failed with Exit status: 61"
-#Tstart
-#python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g #$gms_inputs
-
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
 echo "Processing usgs gage aggregation"
-python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
+python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -i $fim_inputs
 
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
@@ -197,7 +189,7 @@ echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
 Tstart
-python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
+python3 /foss_fim/tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
 Tcount
 date -u
 
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 260823699..1a5b409c4 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -26,22 +26,22 @@ usage ()
     echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
     echo '  -ud/--unitDenylist'
     echo '                    A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
+    echo '                    upon completion (see config/deny_unit.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_unit.lst'
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -bd/--branchDenylist'
     echo '                    A file with a line delimited list of files in BRANCHES directories to be removed' 
     echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_gms_branches.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
+    echo '                    (see config/deny_branches.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_branches.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -zd/--branchZeroDenylist'
     echo '                    A file with a line delimited list of files in BRANCH ZERO directories to' 
     echo '                    be removed upon completion of branch zero processing.'
-    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
+    echo '                    (see config/deny_branch_zero.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_branch_zero.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'    
     echo '  -jh/--jobLimit   : Max number of concurrent HUC jobs to run. Default 1 job at time.'
@@ -141,7 +141,7 @@ if [ -z "$isAWS" ]; then isAWS=0; fi
 # validate and set defaults for the deny lists
 if [ "$deny_unit_list" = "" ]
 then
-    deny_unit_list=$projectDir/config/deny_gms_unit.lst
+    deny_unit_list=$projectDir/config/deny_unit.lst
 elif [ "${deny_unit_list^^}" != "NONE" ] && [ ! -f "$deny_unit_list" ]
 then
     # NONE is not case sensitive
@@ -152,7 +152,7 @@ fi
 # validate and set defaults for the deny lists
 if [ "$deny_branches_list" = "" ]
 then
-    deny_branches_list=$projectDir/config/deny_gms_branches.lst
+    deny_branches_list=$projectDir/config/deny_branches.lst
 elif [ "${deny_branches_list^^}" != "NONE" ] && [ ! -f "$deny_branches_list" ]
 then
     # NONE is not case sensitive
@@ -167,7 +167,7 @@ fi
 has_deny_branch_zero_override=0
 if [ "$deny_branch_zero_list" = "" ]
 then
-    deny_branch_zero_list=$projectDir/config/deny_gms_branch_zero.lst
+    deny_branch_zero_list=$projectDir/config/deny_branch_zero.lst
 elif [ "${deny_branch_zero_list^^}" != "NONE" ]   # NONE is not case sensitive
 then
     if [ ! -f "$deny_branch_zero_list" ]
@@ -211,7 +211,7 @@ else
     rm -rdf $outputRunDataDir/unit_errors
     rm -rdf $outputRunDataDir/eval
     rm -f $outputRunDataDir/crosswalk_table.csv
-    rm -f $outputRunDataDir/gms_inputs*
+    rm -f $outputRunDataDir/fim_inputs*
     rm -f $outputRunDataDir/*.env
 fi
 
diff --git a/gms_pipeline.sh b/gms_pipeline.sh
deleted file mode 100755
index 26c7694f1..000000000
--- a/gms_pipeline.sh
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/bin/bash -e
-:
-usage ()
-{
-    echo
-    echo 'Produce GMS hydrofabric datasets for unit and branch scale.'
-    echo 'Usage : gms_pipeline.sh [REQ: -u <hucs> -n <run name> ]'
-    echo '                        [OPT: -h -c <config file> -j <job limit>] -o'
-    echo '                         -ud <unit deny list file>'
-    echo '                         -bd <branch deny list file>'
-    echo '                         -zd <branch zero deny list file>]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
-    echo '                    A line delimited file is also acceptable. HUCs must present in inputs directory.'
-    echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
-    echo 
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -c/--config     : configuration file with bash environment variables to export'
-    echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
-    echo '  -ud/--unitDenylist : A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'
-    echo '  -bd/--branchDenylist : A file with a line delimited list of files in BRANCHES directories to be removed' 
-    echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_gms_branches.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'
-    echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
-    echo '                    be removed upon completion of branch zero processing.'
-    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'    
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
-    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo
-    exit
-}
-
-set -e
-
-while [ "$1" != "" ]; do
-case $1
-in
-    -u|--hucList)
-        shift
-        hucList=$1
-        ;;
-    -c|--configFile )
-        shift
-        envFile=$1
-        ;;
-    -n|--runName)
-        shift
-        runName=$1
-        ;;
-    -j|--jobLimit)
-        shift
-        jobLimit=$1
-        ;;
-    -h|--help)
-        shift
-        usage
-        ;;
-    -o|--overwrite)
-        overwrite=1
-        ;;
-    -ud|--unitDenylist)
-        shift
-        deny_unit_list=$1
-        ;;
-    -bd|--branchDenylist)
-        shift
-        deny_branches_list=$1
-        ;;
-    -zd|--branchZeroDenylist)
-        shift
-        deny_branch_zero_list=$1
-        ;;        
-    *) ;;
-    esac
-    shift
-done
-
-# print usage if arguments empty
-if [ "$hucList" = "" ]
-then
-    echo "ERROR: Missing -u Huclist argument"
-    usage
-fi
-if [ "$runName" = "" ]
-then
-    echo "ERROR: Missing -n run time name argument"
-    usage
-fi
-
-if [ "$envFile" = "" ]
-then
-    envFile=/foss_fim/config/params_template.env
-fi
-
-if [ -z "$overwrite" ]
-then
-    # default is false (0)
-    overwrite=0
-fi
-
-# The tests for the deny lists are duplicated here on to help catch
-# them earlier (ie.. don't have to wait to process units to find an
-# pathing error with the branch deny list)
-if [ "$deny_unit_list" != "" ] && \
-   [ "${deny_unit_list^^}" != "NONE" ] && \
-   [ ! -f "$deny_unit_list" ]
-then
-    # NONE is not case sensitive
-    echo "Error: The -ud <unit deny file> does not exist and is not the word NONE"
-    usage
-fi
-
-if [ "$deny_branches_list" != "" ] && \
-   [ "${deny_branches_list^^}" != "NONE" ] && \
-   [ ! -f "$deny_branches_list" ]
-then
-    # NONE is not case sensitive
-    echo "Error: The -bd <branch deny file> does not exist and is not the word NONE"
-    usage
-fi
-
-if [ "$deny_branch_zero_list" != "" ] && \
-   [ "${deny_branch_zero_list^^}" != "NONE" ] && \
-   [ ! -f "$deny_branch_zero_list" ]
-then
-    echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
-    usage
-fi
-
-## SOURCE ENV FILE AND FUNCTIONS ##
-source $envFile
-source $srcDir/bash_functions.env
-
-# default values
-if [ "$jobLimit" = "" ] ; then
-    jobLimit=1
-fi
-
-export outputRunDataDir=$outputDataDir/$runName
-
-if [ -d $outputRunDataDir ] && [ $overwrite -eq 0 ]; then
-    echo
-    echo "ERROR: Output dir $outputRunDataDir exists. Use overwrite -o to run."
-    echo        
-    usage
-fi
-
-pipeline_start_time=`date +%s`
-
-num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
-
-echo
-echo "======================= Start of gms_pipeline.sh ========================="
-echo "Number of HUCs to process is $num_hucs"
-
-## Produce gms hydrofabric at unit level first (gms_run_unit)
-
-# We have to build this as a string as some args are optional.
-# but the huclist doesn't always pass well, so just worry about
-# the rest of the params.
-run_cmd=" -n $runName"
-run_cmd+=" -c $envFile"
-run_cmd+=" -j $jobLimit"
-
-if [ $overwrite -eq 1 ]; then run_cmd+=" -o" ; fi
-
-#echo "$run_cmd"
-. /foss_fim/gms_run_unit.sh -u "$hucList" $run_cmd -ud "$deny_unit_list" -zd "$deny_branch_zero_list"
-
-## CHECK IF OK TO CONTINUE ON TO BRANCH STEPS
-# Count the number of files in the $outputRunDataDir/unit_errors
-# If no errors, there will be only one file, non_zero_exit_codes.log.
-# Calculate the number of error files as a percent of the number of hucs 
-# originally submitted. If the percent error is over "x" threshold stop processing
-# Note: This applys only if there are a min number of hucs. Aka.. if min threshold
-# is set to 10, then only return a sys.exit of > 1, if there is at least 10 errors
-
-# if this has too many errors, it will return a sys.exit code (like 62 as per fim_enums)
-# and we will stop the rest of the process. We have to catch stnerr as well.
-# This stops the run from continuing to run, drastically filing drive and killing disk space.
-python3 $srcDir/check_unit_errors.py -f $outputRunDataDir -n $num_hucs
-
-## Produce level path or branch level datasets
-. /foss_fim/gms_run_branch.sh $run_cmd -bd "$deny_branches_list" -zd "$deny_branch_zero_list"
-
-
-## continue on to post processing
-. /foss_fim/gms_run_post_processing.sh $run_cmd
-
-echo
-echo "======================== End of gms_pipeline.sh =========================="
-date -u
-Calc_Duration $pipeline_start_time
-echo
-
diff --git a/gms_run_branch.sh b/gms_run_branch.sh
deleted file mode 100755
index 9c38437d8..000000000
--- a/gms_run_branch.sh
+++ /dev/null
@@ -1,244 +0,0 @@
-#!/bin/bash -e
-:
-usage ()
-{
-    echo 'Produce GMS hydrofabric at levelpath/branch scale. Execute gms_run_unit.sh prior to.'
-    echo 'Usage : gms_run_branch.sh [REQ: -n <run name> ]'
-    echo '                          [OPT: -h -u <hucs> -c <config file> -j <job limit>] -o'
-    echo '  	 				    -bd <branch deny list file>'
-    echo '                          -zd <branch zero deny list file>]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -n/--runName    : A name to tag the output directories and log files as. could be a version tag.'
-    echo ''
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited).'
-    echo '                    A line delimited file also acceptable. HUCs must present in inputs directory.'
-    echo '  -c/--config     : configuration file with bash environment variables to export'    
-    echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
-    echo '  -bd/--branchDenylist : A file with a line delimited list of files in BRANCHES directories to be removed' 
-    echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_gms_branches.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'
-    echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
-    echo '                    be removed upon completion of branch zero processing.'
-    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'    
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
-    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo
-    exit
-}
-
-while [ "$1" != "" ]; do
-case $1
-in
-    -c|--configFile )
-        shift
-        envFile=$1
-        ;;
-    -n|--runName)
-        shift
-        runName=$1
-        ;;
-    -j|--jobLimit)
-        shift
-        jobLimit=$1
-        ;;
-    -u|--hucList)
-        shift
-        hucList=$1
-        ;;
-    -h|--help)
-        shift
-        usage
-        ;;
-    -o|--overwrite)
-        overwrite=1
-        ;;
-    -bd|--branchDenylist)
-        shift
-        deny_branches_list=$1
-        ;;
-    -zd|--branchZeroDenylist)
-        shift
-        deny_branch_zero_list_for_branches=$1
-        ;;        
-    *) ;;
-    esac
-    shift
-done
-
-# print usage if arguments empty
-if [ "$runName" = "" ]
-then
-    echo "ERROR: Missing -n run time name argument"
-    usage
-fi
-
-if [ "$envFile" = "" ]
-then
-    envFile=/foss_fim/config/params_template.env
-fi
-
-if [ "$deny_branches_list" = "" ]
-then
-   deny_branches_list=/foss_fim/config/deny_gms_branches.lst
-elif [ "${deny_branches_list^^}" != "NONE" ] && [ ! -f "$deny_branches_list" ]
-then
-    # NONE is not case sensitive
-    echo "Error: The -bd <branch deny file> does not exist and is not the word NONE"
-    usage
-fi
-
-# Yes.. we have to have a different variable names for the deny_branch_zero_list_for_branches
-# and deny_branch_zero_list_for_units. While they both use the same input arg, they use
-# the value slightly different and when using gms_pipeline, the values can impact
-# other bash files.
-if [ "$deny_branch_zero_list_for_branches" = "" ]
-then
-    deny_branch_zero_list_for_branches=/foss_fim/config/deny_gms_branch_zero.lst
-elif [ "${deny_branch_zero_list_for_branches^^}" != "NONE" ]   # NONE is not case sensitive
-then
-    if [ ! -f "$deny_branch_zero_list_for_branches" ]
-    then
-        echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
-        usage
-    else
-        # only if the deny branch zero has been overwritten and file exists
-        has_deny_branch_zero_override=1 
-    fi
-else
-    has_deny_branch_zero_override=1 # it is the value of NONE and is overridden
-fi
-
-if [ "$overwrite" = "" ]
-then
-    overwrite=0
-fi
-
-## SOURCE ENV FILE AND FUNCTIONS ##
-source $envFile
-source $srcDir/bash_functions.env
-source $srcDir/bash_variables.env
-
-# default values
-if [ "$jobLimit" = "" ] ; then
-    jobLimit=1
-fi
-
-## Define Outputs Data Dir & Log File##
-export outputRunDataDir=$outputDataDir/$runName
-export deny_branches_list=$deny_branches_list
-logFile=$outputRunDataDir/logs/branch/summary_gms_branch.log
-export overwrite=$overwrite
-
-## Check for run data directory and the file. If gms_run_unit failed, the file will not be there ##
-if [ ! -f "$outputRunDataDir/gms_inputs.csv" ]; then 
-    echo "Depends on output from gms_run_unit.sh. Please produce data with gms_run_unit.sh first."
-    exit 1
-fi
-
-## Filter out hucs ##
-if [ "$hucList" = "" ]; then
-    gms_inputs=$outputRunDataDir/gms_inputs.csv
-else
-    $srcDir/gms/filter_gms_inputs_by_huc.py -g $outputRunDataDir/gms_inputs.csv -u $hucList -o $outputRunDataDir/gms_inputs_filtered.csv
-    gms_inputs=$outputRunDataDir/gms_inputs_filtered.csv
-fi
-
-# make log dir
-if [ ! -d "$outputRunDataDir/logs/branch" ]; then
-    mkdir -p $outputRunDataDir/logs/branch
-elif [ $overwrite -eq 1 ]; then
-    # need to clean it out if we are overwriting
-    rm -rdf $outputRunDataDir/logs/branch
-    mkdir -p $outputRunDataDir/logs/branch
-fi
-
-# Note: Other parts of the program will check for the existance of the file
-# /branch_errors/non_zero_exit_codes.log. It has to be removed no matter
-# what on each run of gms_run_branch
-if [ ! -d "$outputRunDataDir/branch_errors" ]; then
-    mkdir -p "$outputRunDataDir/branch_errors"
-elif [ $overwrite -eq 1 ]; then
-    rm -rdf $outputRunDataDir/branch_errors
-    mkdir -p $outputRunDataDir/branch_errors
-fi
-
-## RUN GMS BY BRANCH ##
-echo
-echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- Start of branch processing"
-echo "---- Started: `date -u`" 
-T_total_start
-Tstart
-all_branches_start_time=`date +%s`
-
-if [ "$jobLimit" -eq 1 ]; then
-    parallel --verbose --timeout $branch_timeout --lb  -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
-else
-    parallel --eta --timeout $branch_timeout -j $jobLimit --joblog $logFile --colsep ',' -- $srcDir/gms/time_and_tee_run_by_branch.sh :::: $gms_inputs
-fi
-
-echo "Branch processing is complete"
-Tcount
-date -u
-
-# -------------------
-## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO (but using normal branch deny) ##
-## but also do not remove if branch zero deny is NONE (any case)
-
-# If the deny branch zero has been overridden, then use it (file path proven above).
-# Override might be the value of None (case not sensitive)
-# Else then use the default deny branch (not the zero) which might also be None and that is ok
-
-# If deny branch zero is NONE.. then skip
-# if deny branch zero has an override deny list, use it.
-# if deny branch zero is not NONE and deny list is not overridden
-#    then see if reg deny branch is NONE.  If so.. skip
-#    else, use the deny branch list instead to do final cleanup on branch zero
-
-if [ "$has_deny_branch_zero_override" == "1" ]
-then
-    echo -e $startDiv"Cleaning up (Removing) files for branch zero for all HUCs"
-    $srcDir/gms/outputs_cleanup.py -d $outputRunDataDir -l $deny_branch_zero_list_for_branches -b 0
-
-else 
-    echo -e $startDiv"Cleaning up (Removing) files all branch zero for all HUCs using the default branch deny list"
-    $srcDir/gms/outputs_cleanup.py -d $outputRunDataDir -l $deny_branches_list -b 0
-fi
-
-
-# -------------------
-## GET NON ZERO EXIT CODES ##
-# Needed in case aggregation fails, we will need the logs
-# Note: Other parts of the program (gms_run_post_processing.sh) check to see
-# if the branch_errors/non_zero_exit_codes.log exists. If it does not, it assumes
-# that gms_run_branch did not complete (or was not run)
-echo
-echo -e $startDiv"Start non-zero exit code checking"$stopDiv
-find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
-
-# -------------------
-## REMOVE FAILED BRANCHES ##
-# Needed in case aggregation fails, we will need the logs
-echo
-echo -e $startDiv"Removing branches that failed with Exit status: 61"$stopDiv
-Tstart
-python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g $outputRunDataDir/gms_inputs.csv
-Tcount
-date -u
-
-echo
-echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- gms_run_branch complete"
-echo "---- Ended: `date -u`"
-Calc_Duration $all_branches_start_time
-echo
diff --git a/gms_run_post_processing.sh b/gms_run_post_processing.sh
deleted file mode 100755
index e83404dca..000000000
--- a/gms_run_post_processing.sh
+++ /dev/null
@@ -1,184 +0,0 @@
-#!/bin/bash -e
-:
-usage ()
-{
-    echo 'Produce GMS hydrofabric at levelpath/branch scale. Execute gms_run_unit.sh and gms_run_branch prior to.'
-    echo 'Usage : gms_run_post_processing.sh [REQ: -n <run name> ]'
-    echo '  	 				             [OPT: -h -c -j <job limit>]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -n/--runName    : A name to tag the output directories and log files as. could be a version tag.'
-    echo ''
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -c/--config     : configuration file with bash environment variables to export'    
-    echo '                    default (if arg not added) : /foss_fim/config/params_template.env'            
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
-    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    echo
-    exit
-}
-
-while [ "$1" != "" ]; do
-case $1
-in
-    -c|--configFile )
-        shift
-        envFile=$1
-        ;;
-    -n|--runName)
-        shift
-        runName=$1
-        ;;
-    -j|--jobLimit)
-        shift
-        jobLimit=$1
-        ;;
-    -h|--help)
-        shift
-        usage
-        ;;
-    *) ;;
-    esac
-    shift
-done
-
-# print usage if arguments empty
-if [ "$runName" = "" ]
-then
-    echo "ERROR: Missing -n run time name argument"
-    usage
-fi
-
-if [ "$envFile" = "" ]
-then
-    envFile=/foss_fim/config/params_template.env
-fi
-
-## SOURCE ENV FILE AND FUNCTIONS ##
-source $envFile
-source $srcDir/bash_functions.env
-source $srcDir/bash_variables.env
-
-# default values
-if [ "$jobLimit" = "" ] ; then
-    jobLimit=1
-fi
-
-## Define Outputs Data Dir & Log File##
-export outputRunDataDir=$outputDataDir/$runName
-export extent=GMS
-
-## Check for run data directory ##
-if [ ! -d "$outputRunDataDir" ]; then 
-    echo "Depends on output from gms_run_unit.sh. Please produce data with gms_run_unit.sh first."
-    exit 1
-fi
-
-## Check to ensure gms_run_branch completed ##
-if [ ! -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" ]; then
-    echo "Depends on output from gms_run_branch.sh. Please run gms_run_branch.sh or check if it failed."
-    exit 1
-fi
-
-# Clean out the other post processing files before starting
-rm -rdf $outputRunDataDir/logs/src_optimization
-rm -f $outputRunDataDir/logs/log_bankfull_indentify.log
-rm -f $outputRunDataDir/logs/subdiv_src_.log
-
-gms_inputs=$outputRunDataDir/gms_inputs.csv
-
-echo
-echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- Start of gms_run_post_processing"
-echo "---- Started: `date -u`" 
-T_total_start
-post_proc_start_time=`date +%s`
-
-## RUN AGGREGATE BRANCH ELEV TABLES ##
-# TODO: How do we skip aggregation if there is a branch error
-# maybe against the non_zero logs above
-echo 
-echo "Processing usgs gage aggregation"
-python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
-
-## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
-if [ "$src_bankfull_toggle" = "True" ]; then
-    echo -e $startDiv"Estimating bankfull stage in SRCs"$stopDiv
-    # Run SRC bankfull estimation routine routine
-    Tstart
-    time python3 /foss_fim/src/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit
-    Tcount
-fi
-
-## RUN SYNTHETIC RATING SUBDIVISION ROUTINE ##
-if [ "$src_subdiv_toggle" = "True" ]; then
-    echo -e $startDiv"Performing SRC channel/overbank subdivision routine"$stopDiv
-    # Run SRC Subdivision & Variable Roughness routine
-    Tstart
-    time python3 /foss_fim/src/subdiv_chan_obank_src.py -fim_dir $outputRunDataDir -mann $vmann_input_file -j $jobLimit
-    Tcount
-fi
-
-## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
-if [ "$src_adjust_spatial" = "True" ]; then
-    if [ ! -f $CALB_DB_KEYS_FILE ]; then
-        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
-        exit 1
-    else
-        source $CALB_DB_KEYS_FILE
-        : '
-        This makes the local variables from the calb_db_keys files
-        into global variables that can be used in other files, including python.
-
-        Why not just leave the word export in front of each of the keys in the
-        calb_db_keys.env? Becuase that file is used against docker-compose
-        when we start up that part of the sytem and it does not like the word
-        export.
-        '
-        export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
-        export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
-        export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
-        export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
-        export DEFAULT_FIM_PROJECTION_CRS=$DEFAULT_FIM_PROJECTION_CRS
-        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
-        echo "Loading HUC Data"
-        time ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
-        echo "Loading Point Data"
-        time ogr2ogr -overwrite -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
-    fi
-fi
-
-## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
-if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
-    Tstart
-    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"$stopDiv
-    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
-    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
-    Tcount
-    date -u
-fi
-
-## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
-if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
-    Tstart
-    echo -e $startDiv"Performing SRC adjustments using benchmark point database"$stopDiv
-    python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
-    Tcount
-    date -u
-fi
-
-echo
-echo -e $startDiv"Combining crosswalk tables"$stopDiv
-# aggregate outputs
-Tstart
-python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
-Tcount
-date -u
-
-echo
-echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- gms_run_post_processing complete"
-echo "---- Ended: `date -u`"
-Calc_Duration $post_proc_start_time
-echo
diff --git a/gms_run_unit.sh b/gms_run_unit.sh
deleted file mode 100755
index dd4b46d08..000000000
--- a/gms_run_unit.sh
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/bin/bash -e
-:
-usage ()
-{
-    echo 'Produce GMS hydrofabric datasets for unit scale.'
-    echo 'Usage : gms_run_unit.sh [REQ: -u <hucs> -n <run name> ]'
-    echo '                        [OPT: -h -c <config file> -j <job limit> -o'
-    echo '                         -ud <unit deny list file>'
-    echo '                         -zd <branch zero deny list file>]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
-    echo '                    A line delimited file is also acceptable. HUCs must present in inputs directory.'
-    echo '  -n/--runName    : a name to tag the output directories and log files as. could be a version tag.'
-    echo ''
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -c/--config     : configuration file with bash environment variables to export'
-    echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'        
-    echo '  -ud/--unitDenylist : A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'
-    echo '  -zd/--branchZeroDenylist : A file with a line delimited list of files in BRANCH ZERO directories to' 
-    echo '                    be removed upon completion of branch zero processing.'
-    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'    
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time.'
-    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    echo '  -o/--overwrite  : overwrite outputs if already exist'
-    echo
-    exit
-}
-
-while [ "$1" != "" ]; do
-case $1
-in
-    -u|--hucList)
-        shift
-        hucList="$1"
-        ;;
-    -c|--configFile )
-        shift
-        envFile=$1
-        ;;
-    -n|--runName)
-        shift
-        runName=$1
-        ;;
-    -j|--jobLimit)
-        shift
-        jobLimit=$1
-        ;;
-    -h|--help)
-        shift
-        usage
-        ;;
-    -o|--overwrite)
-        overwrite=1
-        ;;
-    -ud|--unitDenylist)
-        shift
-        deny_unit_list=$1
-        ;;
-    -zd|--branchZeroDenylist)
-        shift
-        deny_branch_zero_list_for_units=$1
-        ;;        
-    *) ;;
-    esac
-    shift
-done
-
-# print usage if arguments empty
-if [ "$hucList" = "" ]
-then
-    echo "ERROR: Missing -u Huclist argument"
-    usage
-fi
-if [ "$runName" = "" ]
-then
-    echo "ERROR: Missing -n run time name argument"
-    usage
-fi
-
-if [ "$envFile" = "" ]
-then
-    envFile=/foss_fim/config/params_template.env
-fi
-
-if [ "$deny_unit_list" = "" ]
-then
-    deny_unit_list=/foss_fim/config/deny_gms_unit.lst
-elif [ "${deny_unit_list^^}" != "NONE" ] && [ ! -f "$deny_unit_list" ]
-then
-    # NONE is not case sensitive
-    echo "Error: The -ud <unit deny file> does not exist and is not the word NONE"
-    usage
-fi
-
-if [ "$deny_branch_zero_list_for_units" = "" ]
-then
-    deny_branch_zero_list_for_units=/foss_fim/config/deny_gms_branch_zero.lst
-elif [ "${deny_branch_zero_list_for_units^^}" != "NONE" ]   # NONE is not case sensitive
-then
-    if [ ! -f "$deny_branch_zero_list_for_units" ]
-    then
-        echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
-        usage
-    fi
-fi
-
-if [ -z "$overwrite" ]
-then
-    overwrite=0
-fi
-
-## SOURCE ENV FILE AND FUNCTIONS ##
-source $envFile
-source $srcDir/bash_functions.env
-source $srcDir/bash_variables.env
-
-# default values
-if [ "$jobLimit" = "" ] ; then
-    jobLimit=1
-fi
-
-## Define Outputs Data Dir & Log File##
-export outputRunDataDir=$outputDataDir/$runName
-logFile=$outputRunDataDir/logs/unit/summary_gms_unit.log
-
-if [ -d $outputRunDataDir ] && [ $overwrite -eq 0 ]; then
-    echo
-    echo "ERROR: Output dir $outputRunDataDir exists. Use overwrite -o to run."
-    echo        
-    usage
-fi
-
-## Define inputs
-export overwrite=$overwrite
-export deny_gms_unit_list=$deny_gms_unit_list
-export deny_unit_list=$deny_unit_list
-export deny_branch_zero_list_for_units=$deny_branch_zero_list_for_units
-
-# we are not using the variable output at this time, but keep it anways
-num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
-
-# make dirs
-if [ ! -d $outputRunDataDir ]; then
-    mkdir -p $outputRunDataDir
-fi
-
-# remove these directories on a new or overwrite run
-rm -rdf $outputRunDataDir/logs
-rm -rdf $outputRunDataDir/branch_errors
-rm -rdf $outputRunDataDir/unit_errors
-
-# we need to clean out the all log files and some other files overwrite or not
-mkdir -p $outputRunDataDir/logs/unit
-mkdir -p $outputRunDataDir/unit_errors
-rm -f $outputRunDataDir/gms_inputs*
-
-# copy over config file
-cp -a $envFile $outputRunDataDir
-
-## RUN GMS BY UNIT ##
-echo
-echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- Start of gms_run_unit"
-echo "---- Started: `date -u`" 
-all_units_start_time=`date +%s`
-
-## GMS BY UNIT##
-if [ -f "$hucList" ]; then
-    if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
-    else
-        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh :::: $hucList
-    fi
-else 
-    if [ "$jobLimit" -eq 1 ]; then
-        parallel --verbose --lb  -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
-    else
-        parallel --eta -j $jobLimit --joblog $logFile -- $srcDir/gms/time_and_tee_run_by_unit.sh ::: $hucList
-    fi
- fi
-
-echo "Unit (HUC) processing is complete"
-date -u
-
-## GET NON ZERO EXIT CODES ##
-# Needed in case aggregation fails, we will need the logs
-echo -e $startDiv"Start of non zero exit codes check"$stopDiv
-find $outputRunDataDir/logs/ -name "*_unit.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" >"$outputRunDataDir/unit_errors/non_zero_exit_codes.log" &
-
-## AGGREGATE BRANCH LISTS INTO ONE ##
-echo -e $startDiv"Start branch aggregation"$stopDiv
-python3 $srcDir/gms/aggregate_branch_lists.py -d $outputRunDataDir -f "gms_inputs.csv" -l $hucList
-
-echo
-echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- gms_run_unit is complete"
-echo "---- Ended: `date -u`"
-Calc_Duration $all_units_start_time
-echo
\ No newline at end of file
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 88398aa95..15dfe2efc 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -15,9 +15,27 @@
 from utils.shared_variables import FIM_ID
 from memory_profiler import profile
 
+# Feb 17, 2023
+# We want to explore using FR methodology as branch zero
 
 @mem_profile
-def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode=False):
+def add_crosswalk(input_catchments_fileName,
+                  input_flows_fileName,
+                  input_srcbase_fileName,
+                  output_catchments_fileName,
+                  output_flows_fileName,
+                  output_src_fileName,
+                  output_src_json_fileName,
+                  output_crosswalk_fileName,
+                  output_hydro_table_fileName,
+                  input_huc_fileName,
+                  input_nwmflows_fileName,
+                  input_nwmcatras_fileName,
+                  mannings_n,
+                  input_nwmcat_fileName,
+                  extent,
+                  small_segments_filename,
+                  calibration_mode=False):
 
     input_catchments = gpd.read_file(input_catchments_fileName)
     input_flows = gpd.read_file(input_flows_fileName)
@@ -298,7 +316,7 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
 
 if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description='Crosswalk for MS/FR networks; calculate synthetic rating curves; update short rating curves')
+    parser = argparse.ArgumentParser(description='Crosswalk for MS/FR/GMS networks; calculate synthetic rating curves; update short rating curves')
     parser.add_argument('-d','--input-catchments-fileName', help='DEM derived catchments', required=True)
     parser.add_argument('-a','--input-flows-fileName', help='DEM derived streams', required=True)
     parser.add_argument('-s','--input-srcbase-fileName', help='Base synthetic rating curve table', required=True)
@@ -313,28 +331,11 @@ def add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_f
     parser.add_argument('-y','--input-nwmcatras-fileName',help='NWM catchment raster',required=False)
     parser.add_argument('-m','--mannings-n',help='Mannings n. Accepts single parameter set or list of parameter set in calibration mode. Currently input as csv.',required=True)
     parser.add_argument('-z','--input-nwmcat-fileName',help='NWM catchment polygon',required=True)
-    parser.add_argument('-p','--extent',help='MS or FR extent',required=True)
+    parser.add_argument('-p','--extent',help='GMS only for now', default='GMS', required=False)
     parser.add_argument('-k','--small-segments-filename',help='output list of short segments',required=True)
     parser.add_argument('-c','--calibration-mode',help='Mannings calibration flag',required=False,action='store_true')
 
     args = vars(parser.parse_args())
 
-    input_catchments_fileName = args['input_catchments_fileName']
-    input_flows_fileName = args['input_flows_fileName']
-    input_srcbase_fileName = args['input_srcbase_fileName']
-    output_catchments_fileName = args['output_catchments_fileName']
-    output_flows_fileName = args['output_flows_fileName']
-    output_src_fileName = args['output_src_fileName']
-    output_src_json_fileName = args['output_src_json_fileName']
-    output_crosswalk_fileName = args['output_crosswalk_fileName']
-    output_hydro_table_fileName = args['output_hydro_table_fileName']
-    input_huc_fileName = args['input_huc_fileName']
-    input_nwmflows_fileName = args['input_nwmflows_fileName']
-    input_nwmcatras_fileName = args['input_nwmcatras_fileName']
-    mannings_n = args['mannings_n']
-    input_nwmcat_fileName = args['input_nwmcat_fileName']
-    extent = args['extent']
-    small_segments_filename = args['small_segments_filename']
-    calibration_mode = args['calibration_mode']
-
-    add_crosswalk(input_catchments_fileName,input_flows_fileName,input_srcbase_fileName,output_catchments_fileName,output_flows_fileName,output_src_fileName,output_src_json_fileName,output_crosswalk_fileName,output_hydro_table_fileName,input_huc_fileName,input_nwmflows_fileName,input_nwmcatras_fileName,mannings_n,input_nwmcat_fileName,extent,small_segments_filename,calibration_mode)
+    add_crosswalk(**args)
+   
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 336ed61e3..c2ba361ab 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -1,7 +1,6 @@
 ## Define inputs
 # NOTE: $inputDataDir is defined in Dockerfile
 export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
-export extent=GMS
 export input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
 export input_DEM_domain=$inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
 export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
diff --git a/src/gms/buffer_stream_branches.py b/src/buffer_stream_branches.py
similarity index 100%
rename from src/gms/buffer_stream_branches.py
rename to src/buffer_stream_branches.py
diff --git a/src/gms/clip_rasters_to_branches.py b/src/clip_rasters_to_branches.py
similarity index 100%
rename from src/gms/clip_rasters_to_branches.py
rename to src/clip_rasters_to_branches.py
diff --git a/src/gms/crosswalk_nwm_demDerived.py b/src/crosswalk_nwm_demDerived.py
similarity index 99%
rename from src/gms/crosswalk_nwm_demDerived.py
rename to src/crosswalk_nwm_demDerived.py
index 97f815ffd..55ef7e36e 100755
--- a/src/gms/crosswalk_nwm_demDerived.py
+++ b/src/crosswalk_nwm_demDerived.py
@@ -4,12 +4,12 @@
 import pandas as pd
 import numpy as np
 import argparse
+import stream_branches as sb
+
 from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
-from gms import stream_branches as sb
 from shapely.geometry import MultiLineString
 
-
 def Crosswalk_nwm_demDerived(nwm_streams, demDerived, wbd=None, node_prefix=None, sampling_size=None, 
                              crosswalk_outfile=None, 
                              demDerived_outfile=None, nwm_outfile=None, verbose=False):
diff --git a/src/gms/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
similarity index 94%
rename from src/gms/delineate_hydros_and_produce_HAND.sh
rename to src/delineate_hydros_and_produce_HAND.sh
index c27de85cd..022c36e0e 100755
--- a/src/gms/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -11,7 +11,7 @@ if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtec
     echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/gms/mask_dem.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif  -s $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -i $current_branch_id -b0 $branch_zero_id
+    python3 -m memory_profiler $srcDir/mask_dem.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif  -s $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -i $current_branch_id -b0 $branch_zero_id
     Tcount
 fi
 
@@ -103,7 +103,7 @@ Tcount
 echo -e $startDiv"D8 REM $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/gms/make_rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
+$srcDir/make_rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
 Tcount
 
 ## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
@@ -183,5 +183,5 @@ elif [ "$level" = "unit" ]; then
     b_arg=$outputHucDataDir/nwm_subset_streams.gpkg
     z_arg=$outputHucDataDir/nwm_catchments_proj_subset.gpkg
 fi
-python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/src_base_$current_branch_id.csv -l $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $outputCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $outputCurrentBranchDataDir/src_$current_branch_id.json -x $outputCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $outputCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $b_arg -y $outputCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -p $extent -k $outputCurrentBranchDataDir/small_segments_$current_branch_id.csv
+python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/src_base_$current_branch_id.csv -l $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $outputCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $outputCurrentBranchDataDir/src_$current_branch_id.json -x $outputCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $outputCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $b_arg -y $outputCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -k $outputCurrentBranchDataDir/small_segments_$current_branch_id.csv
 Tcount
diff --git a/src/gms/derive_level_paths.py b/src/derive_level_paths.py
similarity index 97%
rename from src/gms/derive_level_paths.py
rename to src/derive_level_paths.py
index fd74ddbd3..f34dc494c 100755
--- a/src/gms/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -28,18 +28,18 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         stream_network = StreamNetwork.from_file(filename=in_stream_network)
     else:
         print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
-        sys.exit(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
+        sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # if there are no reaches at this point
     if (len(stream_network) == 0):
         # This is technically not an error but we need to have it logged so the user know what
         # happened to it and we need the huc to not be included in future processing. 
-        # We need it to be not included in the gms_input.csv at the end of the unit processing.
+        # We need it to be not included in the fim_input.csv at the end of the unit processing.
         # Throw an exception with valid text. This will show up in the non-zero exit codes and explain why an error.
         # Later, we can look at creating custom sys exit codes 
         # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file.")
         print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
-        sys.exit(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value)  # will send a 60 back
+        sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # values_exluded of 1 and 2 mean where are dropping stream orders 1 and 2. We are leaving those
     # for branch zero.
diff --git a/src/gms/edit_points.py b/src/edit_points.py
similarity index 100%
rename from src/gms/edit_points.py
rename to src/edit_points.py
diff --git a/src/gms/filter_gms_inputs_by_huc.py b/src/filter_inputs_by_huc.py
similarity index 52%
rename from src/gms/filter_gms_inputs_by_huc.py
rename to src/filter_inputs_by_huc.py
index 66ed631eb..4543adacc 100755
--- a/src/gms/filter_gms_inputs_by_huc.py
+++ b/src/filter_inputs_by_huc.py
@@ -3,7 +3,7 @@
 import pandas as pd
 import argparse
 
-def filter_gms_inputs_by_huc(gms_inputs,hucs,gms_outputs):
+def filter_inputs_by_huc(fim_inputs, hucs, fim_outputs):
 
     try:
         with open(hucs[0]) as hf:
@@ -11,23 +11,23 @@ def filter_gms_inputs_by_huc(gms_inputs,hucs,gms_outputs):
     except FileNotFoundError:
         hucsList = set(hucs)
     
-    gms_inputs = pd.read_csv(gms_inputs,header=None,dtype=str)
-    gms_inputs_mask = gms_inputs.loc[:,0].isin(hucsList)
-    gms_inputs = gms_inputs.loc[gms_inputs_mask,:]
+    fim_inputs = pd.read_csv(fim_inputs,header=None,dtype=str)
+    fim_inputs_mask = fim_inputs.loc[:,0].isin(hucsList)
+    fim_inputs = fim_inputs.loc[fim_inputs_mask,:]
 
-    assert len(gms_inputs) > 0, "Filtered GMS list is empty"
+    assert len(fim_inputs) > 0, "Filtered FIM list is empty"
 
-    gms_inputs.to_csv(gms_outputs,index=False,header=False)
+    fim_inputs.to_csv(fim_outputs, index=False, header=False)
 
 
 if __name__ == '__main__':
 
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Adjusts the elevation of the thalweg to the lateral zonal minimum.')
-    parser.add_argument('-g','--gms-inputs',help='Raster of elevation.',required=True)
+    parser.add_argument('-g','--fim-inputs',help='Raster of elevation.',required=True)
     parser.add_argument('-u','--hucs',help='Raster of elevation.',required=True,nargs='+')
-    parser.add_argument('-o','--gms-outputs',help='Raster of elevation.',required=True)
+    parser.add_argument('-o','--fim-outputs',help='Raster of elevation.',required=True)
 
     args = vars(parser.parse_args())
 
-    filter_gms_inputs_by_huc(**args)
+    filter_inputs_by_huc(**args)
diff --git a/src/gms/finalize_srcs.py b/src/finalize_srcs.py
similarity index 100%
rename from src/gms/finalize_srcs.py
rename to src/finalize_srcs.py
diff --git a/src/generate_branch_list.py b/src/generate_branch_list.py
index 6e1240de4..2529a223e 100755
--- a/src/generate_branch_list.py
+++ b/src/generate_branch_list.py
@@ -5,7 +5,6 @@
 import pandas as pd
 import sys
 
-sys.path.append('/foss_fim/src/gms/')
 from stream_branches import StreamNetwork
 
 def generate_branch_list(stream_network_dissolved, branch_id_attribute,
diff --git a/src/gms/__init__.py b/src/gms/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/src/gms/aggregate_branch_lists.py b/src/gms/aggregate_branch_lists.py
deleted file mode 100755
index f83eb8c72..000000000
--- a/src/gms/aggregate_branch_lists.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse 
-import pandas as pd
-from os import environ
-from os.path import join
-from glob import glob
-
-
-def aggregate_inputs_for_gms(huc_list, output_dir, output_file_name):
-
-    # bash will send huclist in as a colletion and not a string
-    if isinstance(huc_list, list):
-        huc_list_file = huc_list[0]
-    else:
-        huc_list_file = huc_list
-    print(huc_list_file)
-    try:
-        huc_list = pd.read_csv(huc_list_file,header=None,dtype=str).loc[:,0].tolist()
-    except FileNotFoundError:
-        pass
-
-    hucs = set(huc_list)
-
-    # get branch lists
-    branch_id_files = glob(join(output_dir,'*','branch_id.lst'))
-
-    all_huc_numbers,all_bids = [],[]
-    for bid_file in branch_id_files:
-        huc_number = bid_file.split('/')[-2]
-        
-        if huc_number in hucs:
-            bids = pd.read_csv(bid_file,header=None).loc[:,0].tolist()
-            huc_number_list = [huc_number] * len(bids)
-
-            all_bids += bids
-            all_huc_numbers += huc_number_list
-        
-    output = pd.DataFrame({ 
-                            'huc': all_huc_numbers,
-                            'branch' : all_bids
-                          })
-    
-    output.to_csv(join(output_dir, output_file_name),index=False,header=False)
-
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Aggregate GMS Inputs')
-    parser.add_argument('-d','--output_dir', help='output run data directory', required=True)
-    parser.add_argument('-f','--output_file_name', help='output file name', required=True)
-    parser.add_argument('-l','--huc_list', help='huc list', required=True,nargs='+')
-
-    args = vars(parser.parse_args())
-
-    aggregate_inputs_for_gms(**args)
diff --git a/src/gms/generate_branch_list.py b/src/gms/generate_branch_list.py
deleted file mode 100755
index 269263f63..000000000
--- a/src/gms/generate_branch_list.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import pandas as pd
-from stream_branches import StreamNetwork
-import argparse
-
-def Generate_branch_list(stream_network_dissolved, branch_id_attribute, output_branch_list, branch_zero):
-
-    if os.path.exists(stream_network_dissolved):
-        # load stream network
-        stream_network_dissolved = StreamNetwork.from_file( stream_network_dissolved,
-                                                            branch_id_attribute=branch_id_attribute )
-
-        # reduce to branch id attribute and convert to pandas df
-        stream_network_dissolved = stream_network_dissolved.loc[:,branch_id_attribute]
-
-        # write
-        stream_network_dissolved.to_csv(output_branch_list,sep= " ",index=False,header=False)
-
-    # Add branch zero ID to branch list
-    if branch_zero:
-        with open(output_branch_list, 'a') as branch_lst:
-            branch_lst.write(f'{branch_zero}')
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Create branch list')
-    parser.add_argument('-d','--stream-network-dissolved', help='Dissolved stream network', required=True)
-    parser.add_argument('-b','--branch-id-attribute', help='Branch ID attribute to use in dissolved stream network', required=True)
-    parser.add_argument('-o','--output-branch-list', help='Output branch list', required=True)
-    parser.add_argument('-z','--branch-zero', help='Optional Branch Zero ID (str) to be added to the branch list. Usually this will be "0".', required=False)
-    
-    args = vars(parser.parse_args())
-
-    Generate_branch_list(**args)
diff --git a/src/gms/remove_error_branches.py b/src/gms/remove_error_branches.py
deleted file mode 100755
index 30afbc44b..000000000
--- a/src/gms/remove_error_branches.py
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/usr/bin/env python3
-
-# Removes branches with Exit status: 61 following split_flows.py.
-# Removes both the branch folder and the reference in gms_inputs.csv
-
-import os
-import argparse
-import pandas as pd
-import shutil
-
-def remove_error_branches(logfile, gms_inputs):
-    if os.path.isfile(logfile):
-        try:
-            errors_df = pd.read_csv(logfile, sep=':', header=None)
-        except pd.errors.EmptyDataError:
-            print('\nLog file is empty. Skipping this HUC.\n')
-            return
-
-        gms_inputs_df = pd.read_csv(gms_inputs, header=None, dtype={0:str,1:str})
-
-        # Make copy of gms_inputs.csv
-        gms_inputs_copy = os.path.splitext(gms_inputs)[0] + '_original.csv'
-        if not os.path.isfile(gms_inputs_copy):
-            gms_inputs_df.to_csv(gms_inputs_copy, header=False, index=False)
-            
-        gms_inputs_removed = os.path.splitext(gms_inputs)[0] + '_removed.csv'
-        if not os.path.isfile(gms_inputs_removed):
-            error_branches = None
-        else:
-            error_branches = pd.read_csv(gms_inputs_removed, header=None, dtype=str)
-
-        first_occurrence = []
-        for i, row in errors_df.iterrows():
-            error_code = row[2]
-
-            if error_code == 61:
-                dirname, basename = os.path.split(row[0])
-
-                filename = os.path.splitext(basename)[0]
-
-                print(f"Removing {filename}")
-
-                split = str.split(filename, '_')
-
-                huc = split[0]
-                branch = split[3]
-
-                if huc not in first_occurrence:
-                    # Ignore previous removals for this HUC
-                    if error_branches is not None:
-                        error_branches = error_branches[error_branches[0] != huc]
-
-                    first_occurrence.append(huc)
-
-                output_dir = os.path.split(os.path.split(dirname)[0])[0]
-                branch_dir = os.path.join(output_dir, huc, 'branches', branch)
-                if os.path.exists(branch_dir):
-                    shutil.rmtree(branch_dir)
-
-                # Remove bad branch from DataFrame
-                if branch in gms_inputs_df.loc[:,1].values:
-                    gms_inputs_df = gms_inputs_df.drop(index=gms_inputs_df[gms_inputs_df.loc[:,1]==branch].index[0])
-
-                tmp_df = pd.DataFrame([huc, branch]).T
-                if error_branches is None:
-                    error_branches = tmp_df
-                else:
-                    error_branches = pd.concat([error_branches, tmp_df])
-
-        # Save list of removed branches
-        if error_branches is not None and len(error_branches) > 0:
-            pd.DataFrame(error_branches).to_csv(gms_inputs_removed, header=False, index=False)
-
-            # Overwrite gms_inputs.csv with error branches removed
-            gms_inputs_df.to_csv(gms_inputs, header=False, index=False)
-
-            print('\nDone removing error branches\n')
-
-        else:
-            print('\nDone -- no branches to remove')
-
-    else:
-        print('\nNo log file found\n')
-
-
-if __name__ == '__main__':
-
-    # parse arguments
-    parser = argparse.ArgumentParser(description='Remove branches with Exit status: 61')
-    parser.add_argument('-f','--logfile', help='Location of non_zero_exit_codes.log', required=True)
-    parser.add_argument('-g','--gms-inputs', help='Location of gms_inputs.csv', required=True)
-
-    # extract to dictionary
-    args = vars(parser.parse_args())
-
-    remove_error_branches(args['logfile'], args['gms_inputs'])
\ No newline at end of file
diff --git a/src/gms/run_by_unit.sh b/src/gms/run_by_unit.sh
deleted file mode 100755
index 25d5aeec8..000000000
--- a/src/gms/run_by_unit.sh
+++ /dev/null
@@ -1,272 +0,0 @@
-#!/bin/bash -e
-
-## INITIALIZE TOTAL TIME TIMER ##
-T_total_start
-
-## SOURCE BASH FUNCTIONS
-source $srcDir/bash_variables.env
-
-## SET OUTPUT DIRECTORY FOR UNIT ##
-hucNumber="$1"
-
-: '
-  Even though check_input_hucs at gms_run_unit validate that all values
-  are numbers, sometimes the huc list can come in as windows incoded and not
-  unix encoded. It can get missed but tee and time can parse it wrong.
-  so, we will strip a slash of the end if it exists, the re-validat that the
-  value is a number.  (Note: doesn''t seem to work all of the time for encoding
-  issues (??))
-'
-re='^[0-9]+$'
-if ! [[ $hucNumber =~ $re ]] ; then
-   echo "Error: hucNumber is not a number" >&2; exit 1
-fi
-
-outputHucDataDir=$outputRunDataDir/$hucNumber
-outputBranchDataDir=$outputHucDataDir/branches
-current_branch_id=$branch_zero_id
-
-## huc data
-if [ -d "$outputHucDataDir" ]; then
-    if [ $overwrite -eq 1 ]; then
-        rm -rf $outputHucDataDir
-    else
-        echo "Output dir $outputHucDataDir exists. Use overwrite -o to run."
-    fi
-fi
-
-# make outputs directory
-mkdir -p $outputHucDataDir
-
-# make branches outputs directory
-if [ ! -d "$outputBranchDataDir" ]; then
-    mkdir -p $outputBranchDataDir
-fi
-
-## SET VARIABLES AND FILE INPUTS ##
-hucUnitLength=${#hucNumber}
-huc4Identifier=${hucNumber:0:4}
-huc2Identifier=${hucNumber:0:2}
-input_NHD_WBHD_layer=WBDHU$hucUnitLength
-
-input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
-
-## START MESSAGE ##
-echo -e $startDiv"Processing HUC: $hucNumber ..."$stopDiv
-
-# Define the landsea water body mask using either Great Lakes or Ocean polygon input #
-if [[ $huc2Identifier == "04" ]] ; then
-  input_LANDSEA=$input_GL_boundaries
-  echo -e "Using $input_LANDSEA for water body mask (Great Lakes)"
-else
-  input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
-fi
-
-## GET WBD ##
-echo -e $startDiv"Get WBD $hucNumber"$stopDiv
-date -u
-Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
-Tcount
-
-## Subset Vector Layers ##
-echo -e $startDiv"Get Vector Layers and Subset $hucNumber"$stopDiv
-date -u
-Tstart
-
-cmd_args=" -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg"
-cmd_args+=" -b $outputHucDataDir/nwm_subset_streams.gpkg"
-cmd_args+=" -d $hucNumber"
-cmd_args+=" -e $outputHucDataDir/nwm_headwater_points_subset.gpkg"
-cmd_args+=" -f $outputHucDataDir/wbd_buffered.gpkg"
-cmd_args+=" -g $outputHucDataDir/wbd.gpkg"
-cmd_args+=" -i $input_DEM"
-cmd_args+=" -j $input_DEM_domain"
-cmd_args+=" -l $input_nwm_lakes"
-cmd_args+=" -m $input_nwm_catchments"
-cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"
-cmd_args+=" -r $input_NLD"
-cmd_args+=" -v $input_LANDSEA"
-cmd_args+=" -w $input_nwm_flows"
-cmd_args+=" -x $outputHucDataDir/LandSea_subset.gpkg"
-cmd_args+=" -y $input_nwm_headwaters"
-cmd_args+=" -z $outputHucDataDir/nld_subset_levees.gpkg"
-cmd_args+=" -wb $wbd_buffer"
-cmd_args+=" -lpf $input_nld_levee_protected_areas"
-cmd_args+=" -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg"
-
-Tcount
-#echo "$cmd_args"
-python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
-
-: '
-python3 $srcDir/clip_vectors_to_wbd.py -d $hucNumber -w $input_nwm_flows -l $input_nwm_lakes -r $input_NLD -g $outputHucDataDir/wbd.gpkg -f $outputHucDataDir/wbd_buffered.gpkg -m $input_nwm_catchments -y $input_nwm_headwaters -v $input_LANDSEA -lpf $input_nld_levee_protected_areas -z $outputHucDataDir/nld_subset_levees.gpkg -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg -e $outputHucDataDir/nwm_headwater_points_subset.gpkg -b $outputHucDataDir/nwm_subset_streams.gpkg -x $outputHucDataDir/LandSea_subset.gpkg -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -wb $wbd_buffer -i $input_DEM -j $input_DEM_domain
-'
-
-## Clip WBD8 ##
-echo -e $startDiv"Clip WBD8"$stopDiv
-date -u
-Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8
-Tcount
-
-## DERIVE LEVELPATH  ##
-echo -e $startDiv"Generating Level Paths for $hucNumber"$stopDiv
-date -u
-Tstart
-$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -v -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
-
-# test if we received a non-zero code back from derive_level_paths.py
-subscript_exit_code=$?
-# we have to retrow it if it is not a zero (but it will stop further execution in this script)
-if [ $subscript_exit_code -ne 0 ]; then exit $subscript_exit_code; fi
-Tcount
-
-## STREAM BRANCH POLYGONS
-echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"$stopDiv
-date -u
-Tstart
-$srcDir/gms/buffer_stream_branches.py -a $input_DEM_domain -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg -v
-Tcount
-
-## CREATE BRANCHID LIST FILE
-echo -e $startDiv"Create file of branch ids for $hucNumber"$stopDiv
-date -u
-Tstart
-$srcDir/gms/generate_branch_list.py -o $outputHucDataDir/branch_id.lst -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -z $branch_zero_id
-Tcount
-
-## CREATE BRANCH ZERO ##
-echo -e $startDiv"Creating branch zero for $hucNumber"$stopDiv
-outputCurrentBranchDataDir=$outputBranchDataDir/$branch_zero_id
-
-## OVERWRITE
-if [ -d "$outputCurrentBranchDataDir" ];then
-    if [ $overwrite -eq 1 ]; then
-        rm -rf $outputCurrentBranchDataDir
-    else
-        echo "GMS branch data directories for $hucNumber - $branch_zero_id already exist. Use -o/--overwrite to continue"
-        exit 1
-    fi
-fi
-
-## MAKE OUTPUT BRANCH DIRECTORY
-mkdir -p $outputCurrentBranchDataDir
-
-## CLIP RASTERS
-echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"$stopDiv
-# Note: don't need to use gdalwarp -cblend as we are using a buffered wbd
-date -u
-Tstart
-[ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \
-gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
-Tcount
-
-## GET RASTER METADATA
-echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif)
-
-## RASTERIZE NLD MULTILINES ##
-echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-# REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
-gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif
-Tcount
-
-## BURN LEVEES INTO DEM ##
-echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-# REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $outputCurrentBranchDataDir/nld_subset_levees.tif ] && \
-python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -nld $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif -out $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
-Tcount
-
-## RASTERIZE REACH BOOLEAN (1 & 0) ##
-echo -e $startDiv"Rasterize Reach Boolean $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_subset_streams.gpkg $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
-Tcount
-
-## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
-echo -e $startDiv"Rasterize NWM Headwaters $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_headwater_points_subset.gpkg $outputCurrentBranchDataDir/headwaters_$branch_zero_id.tif
-Tcount
-
-## DEM Reconditioning ##
-# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
-# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
-echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -w $outputCurrentBranchDataDir -g $outputCurrentBranchDataDir/temp_work -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
-Tcount
-
-## PIT REMOVE BURNED DEM ##
-echo -e $startDiv"Pit remove Burned DEM $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-rd_depression_filling $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
-Tcount
-
-## D8 FLOW DIR ##
-echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $branch_zero_id"$stopDiv
-date -u
-Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
-Tcount
-
-## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
-export hucNumber=$hucNumber
-export current_branch_id=$current_branch_id
-export outputCurrentBranchDataDir=$outputCurrentBranchDataDir
-export outputHucDataDir=$outputHucDataDir
-export ndv=$ndv
-export xmin=$xmin
-export ymin=$ymin
-export xmax=$xmax
-export ymax=$ymax
-export ncols=$ncols
-export nrows=$nrows
-
-## PRODUCE BRANCH ZERO HAND
-$srcDir/gms/delineate_hydros_and_produce_HAND.sh "unit"
-
-## CREATE USGS GAGES FILE
-if [ -f $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
-    echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"$stopDiv
-    date -u
-    Tstart
-    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id
-    Tcount
-fi
-
-## USGS CROSSWALK ##
-if [ -f $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
-    echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"$stopDiv
-    date -u
-    Tstart
-    python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $branch_zero_id
-    Tcount
-fi
-
-## CLEANUP BRANCH ZERO OUTPUTS ##
-echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"$stopDiv
-$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branch_zero_list_for_units -b 0
-
-
-## REMOVE FILES FROM DENY LIST ##
-if [ -f $deny_unit_list ]; then
-    echo -e $startDiv"Remove files $hucNumber"$stopDiv
-    date -u
-    Tstart
-    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
-    Tcount
-fi
diff --git a/src/gms/test_new_crosswalk.sh b/src/gms/test_new_crosswalk.sh
deleted file mode 100755
index 620c94223..000000000
--- a/src/gms/test_new_crosswalk.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash -e
-
-
-python3 /foss_fim/src/gms/crosswalk_nwm_demDerived.py -n /data/outputs/first_batch_test_FR_c/12090301/nwm_subset_streams.gpkg -d /data/outputs/first_batch_test_FR_c/12090301/demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg -v -c /data/temp/continuity/cross_walk_$1.csv -e /data/temp/continuity/demDerived_crosswalked_$1.gpkg -m /data/temp/continuity/nwm_traversal_$1.gpkg -w /data/temp/continuity/wbd.gpkg -a $1
diff --git a/src/gms/time_and_tee_run_by_branch.sh b/src/gms/time_and_tee_run_by_branch.sh
deleted file mode 100755
index 989c9da71..000000000
--- a/src/gms/time_and_tee_run_by_branch.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash -e
-
-echo 
-echo "================================================================================"
-
-/usr/bin/time -v $srcDir/gms/run_by_branch.sh $1 $2 |& tee $outputRunDataDir/logs/branch/$1_gms_branch_$2.log 
-
-#exit ${PIPESTATUS[0]}
-return_codes=( "${PIPESTATUS[@]}" )
-
-# we do this way instead of working directly with stderr and stdout
-# as they were messing with output logs which we always want.
-for code in "${return_codes[@]}"
-do
-    # Make an extra copy of the branch log in a new folder
-    # Note: It was tricky to load in the fim_enum into bash, so we will just 
-    # go with the code for now
-    if [ $code -eq 61 ]; then
-        echo
-        echo "***** Branch has no valid flowlines *****"
-    elif [ $code -ne 0 ]; then
-        echo
-        echo "***** An error has occured  *****"
-        cp $outputRunDataDir/logs/branch/$1_gms_branch_$2.log $outputRunDataDir/branch_errors
-    fi
-done
-
-echo "================================================================================"
-exit
diff --git a/src/gms/time_and_tee_run_by_unit.sh b/src/gms/time_and_tee_run_by_unit.sh
deleted file mode 100755
index bae50e1ae..000000000
--- a/src/gms/time_and_tee_run_by_unit.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash -e
-
-echo 
-echo "================================================================================"
-
-/usr/bin/time -v $srcDir/gms/run_by_unit.sh $1 |& tee $outputRunDataDir/logs/unit/$1_gms_unit.log 
-
-#exit ${PIPESTATUS[0]}
-return_codes=( "${PIPESTATUS[@]}" )
-
-# we do this way instead of working directly with stderr and stdout
-# as they were messing with output logs which we always want.
-for code in "${return_codes[@]}"
-do
-    # Make an extra copy of the branch log in a new folder  if an error
-    # Note: It was tricky to load in the fim_enum into bash, so we will just 
-    # go with the code for now
-    if [ $code -eq 60 ]; then
-        echo
-        echo "***** Unit has no valid branches *****"
-    elif [ $code -ne 0 ]; then
-        echo
-        echo "***** An error has occured  *****"
-        cp $outputRunDataDir/logs/unit/$1_gms_unit.log $outputRunDataDir/unit_errors
-    fi
-done
-
-echo "================================================================================"
-exit
diff --git a/src/gms/make_rem.py b/src/make_rem.py
similarity index 100%
rename from src/gms/make_rem.py
rename to src/make_rem.py
diff --git a/src/gms/mask_dem.py b/src/mask_dem.py
similarity index 100%
rename from src/gms/mask_dem.py
rename to src/mask_dem.py
diff --git a/src/output_cleanup.py b/src/output_cleanup.py
deleted file mode 100755
index 34b83672a..000000000
--- a/src/output_cleanup.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env python3
-import os
-import argparse
-from utils.shared_functions import mem_profile
-
-##################################
-##
-#### Deprecated (as part of fim_run):  Oct 1, 2022 ####
-##
-##################################
-
-
-@mem_profile
-def output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, is_viz_post_processing):
-    '''
-    Processes all the final output files to cleanup and add post-processing
-
-    Parameters
-    ----------
-    huc_number : STR
-        The HUC
-    output_folder_path : STR
-        Path to the outputs for the specific huc
-    additional_whitelist : STR
-        Additional list of files to keep during a production run
-    is_production : BOOL
-        Determine whether or not to only keep whitelisted production files
-    is_viz_post_processing : BOOL
-        Determine whether or not to process outputs for Viz
-    '''
-
-    ##################################
-    ##
-    #### Deprecated (as part of fim_run):  Oct 1, 2022 ####
-    print('#### Deprecated (as part of fim_run):  Oct 1, 2022 ####')
-    ##
-    ##################################
-
-
-    # List of files that will be saved during a production run
-    production_whitelist = [
-        'rem_zeroed_masked.tif',
-        'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg',
-        'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg',
-        'gw_catchments_reaches_filtered_addedAttributes.tif',
-        'hydroTable.csv',
-        'src.json',
-        'small_segments.csv',
-        'bathy_crosswalk_calcs.csv',
-        'bathy_stream_order_calcs.csv',
-        'bathy_thalweg_flag.csv',
-        'bathy_xs_area_hydroid_lookup.csv',
-        'src_full_crosswalked.csv',
-        'usgs_elev_table.csv',
-        'hand_ref_elev_table.csv',
-    ]
-
-    # List of files that will be saved during a viz run
-    viz_whitelist = [
-        'rem_zeroed_masked.tif',
-        'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg',
-        'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg',
-        'gw_catchments_reaches_filtered_addedAttributes.tif',
-        'hydroTable.csv',
-        'src.json',
-        'small_segments.csv',
-        'src_full_crosswalked.csv',
-        'demDerived_reaches_split_points.gpkg',
-        'flowdir_d8_burned_filled.tif',
-        'dem_thalwegCond.tif'
-    ]
-
-    # If "production" run, only keep whitelisted files
-    if is_production and not is_viz_post_processing:
-        whitelist_directory(output_folder_path, production_whitelist, additional_whitelist)
-
-    # If Viz post-processing is enabled, form output files to Viz specifications
-    if is_viz_post_processing:
-        # Step 1, keep only files that Viz needs
-        whitelist_directory(output_folder_path, viz_whitelist, additional_whitelist)
-
-
-@mem_profile
-def whitelist_directory(directory_path, whitelist, additional_whitelist):
-    # Add any additional files to the whitelist that the user wanted to keep
-    if additional_whitelist:
-        whitelist = whitelist + additional_whitelist
-
-    # Delete any non-whitelisted files
-    directory = os.fsencode(directory_path)
-    for file in os.listdir(directory_path):
-        filename = os.fsdecode(file)
-        if filename not in whitelist:
-            os.remove(os.path.join(directory_path, filename))
-
-
-if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Cleanup output files')
-    parser.add_argument('huc_number', type=str, help='The HUC')
-    parser.add_argument('output_folder_path', type=str, help='Path to the outputs for the specific huc')
-    parser.add_argument('-w', '--additional_whitelist', type=str, help='List of additional files to keep in a production run',default=None,nargs="+")
-    parser.add_argument('-p', '--is_production', help='Keep only white-listed files for production runs', action='store_true')
-    parser.add_argument('-v', '--is_viz_post_processing', help='Formats output files to be useful for Viz', action='store_true')
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    # Rename variable inputs
-    huc_number = args['huc_number']
-    output_folder_path = args['output_folder_path']
-    additional_whitelist = args['additional_whitelist']
-    is_production = args['is_production']
-    is_viz_post_processing = args['is_viz_post_processing']
-
-    # Run output_cleanup
-    output_cleanup(huc_number, output_folder_path, additional_whitelist, is_production, is_viz_post_processing)
diff --git a/src/gms/outputs_cleanup.py b/src/outputs_cleanup.py
similarity index 100%
rename from src/gms/outputs_cleanup.py
rename to src/outputs_cleanup.py
diff --git a/src/process_branch.sh b/src/process_branch.sh
index 627cfa182..bd1d4d90f 100755
--- a/src/process_branch.sh
+++ b/src/process_branch.sh
@@ -1,13 +1,13 @@
 #!/bin/bash -e
 
-# it is strongly recommended that you do not call directly to src/gms/run_by_branch.sh
+# it is strongly recommended that you do not call directly to src/run_by_branch.sh
 # but call this file and let is call run_by_branch.
 # This file will auto trap any exceptions from run_by_branch.
 
 # also.. remember.. that this file will rarely need to be called (but can be)
 # as it is usually called through a parallelizing iterator in run_unit_wb.sh
 
-# this also has no named command line arguments, onlly positional args.
+# this also has no named command line arguments, only positional args.
 
 runName=$1
 hucNumber=$2
@@ -18,7 +18,7 @@ export outputRunDataDir=$outputDataDir/$runName
 branchLogFileName=$outputRunDataDir/logs/branch/"$hucNumber"_branch_"$branchId".log
 branch_list_csv_file=$outputRunDataDir/$hucNumber/branch_ids.csv
 
-/usr/bin/time -v $srcDir/gms/run_by_branch.sh $hucNumber $branchId 2>&1 | tee $branchLogFileName 
+/usr/bin/time -v $srcDir/run_by_branch.sh $hucNumber $branchId 2>&1 | tee $branchLogFileName 
 
 #exit ${PIPESTATUS[0]}
 return_codes=( "${PIPESTATUS[@]}" )
diff --git a/src/gms/query_vectors_by_branch_polygons.py b/src/query_vectors_by_branch_polygons.py
similarity index 100%
rename from src/gms/query_vectors_by_branch_polygons.py
rename to src/query_vectors_by_branch_polygons.py
diff --git a/src/gms/reset_mannings.py b/src/reset_mannings.py
similarity index 99%
rename from src/gms/reset_mannings.py
rename to src/reset_mannings.py
index df911bbb9..888e156e3 100755
--- a/src/gms/reset_mannings.py
+++ b/src/reset_mannings.py
@@ -5,7 +5,7 @@
 import argparse
 import os
 from glob import iglob,glob
-from gms.stream_branches import StreamNetwork
+from stream_branches import StreamNetwork
 
 def Reset_mannings(hydrofabric_dir,mannings_value,overwrite_files=False):
 
diff --git a/src/gms/run_by_branch.sh b/src/run_by_branch.sh
similarity index 89%
rename from src/gms/run_by_branch.sh
rename to src/run_by_branch.sh
index 2c141d282..3bd8f8658 100755
--- a/src/gms/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -52,7 +52,7 @@ Tcount
 echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/gms/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $outputBranchDataDir/$branch_zero_id/dem_meters_$branch_zero_id.tif $outputBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $outputCurrentBranchDataDir/dem_meters.tif $outputCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
+$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $outputBranchDataDir/$branch_zero_id/dem_meters_$branch_zero_id.tif $outputBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $outputCurrentBranchDataDir/dem_meters.tif $outputCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
 Tcount
 
 ## GET RASTER METADATA
@@ -88,7 +88,7 @@ export xmax=$xmax
 export ymax=$ymax
 export ncols=$ncols
 export nrows=$nrows
-$srcDir/gms/delineate_hydros_and_produce_HAND.sh "branch"
+$srcDir/delineate_hydros_and_produce_HAND.sh "branch"
 
 ## USGS CROSSWALK ##
 if [ -f $outputHucDataDir/usgs_subset_gages.gpkg ]; then
@@ -104,7 +104,7 @@ if [ -f $deny_branches_list ]; then
     echo -e $startDiv"Remove files $hucNumber $current_branch_id"
     date -u
     Tstart
-    $srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branches_list -b $current_branch_id
+    $srcDir/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branches_list -b $current_branch_id
     Tcount
 fi
 
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index e1516d9df..e7d5e2d16 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -85,7 +85,7 @@ Tcount
 echo -e $startDiv"Generating Level Paths for $hucNumber"
 date -u
 Tstart
-$srcDir/gms/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+$srcDir/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
 
 # test if we received a non-zero code back from derive_level_paths.py
 subscript_exit_code=$?
@@ -97,7 +97,7 @@ Tcount
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
 date -u
 Tstart
-$srcDir/gms/buffer_stream_branches.py -a $input_DEM_domain -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg
+$srcDir/buffer_stream_branches.py -a $input_DEM_domain -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg
 Tcount
 
 ## CREATE BRANCHID LIST FILE
@@ -198,7 +198,7 @@ export ncols=$ncols
 export nrows=$nrows
 
 ## PRODUCE BRANCH ZERO HAND
-$srcDir/gms/delineate_hydros_and_produce_HAND.sh "unit"
+$srcDir/delineate_hydros_and_produce_HAND.sh "unit"
 
 ## CREATE USGS GAGES FILE
 if [ -f $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
@@ -220,7 +220,7 @@ fi
 
 ## CLEANUP BRANCH ZERO OUTPUTS ##
 echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"
-$srcDir/gms/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branch_zero_list -b $branch_zero_id
+$srcDir/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branch_zero_list -b $branch_zero_id
 
 
 ## REMOVE FILES FROM DENY LIST ##
@@ -228,7 +228,7 @@ if [ -f $deny_unit_list ]; then
     echo -e $startDiv"Remove files $hucNumber"
     date -u
     Tstart
-    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
+    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
     Tcount
 fi
 
@@ -249,11 +249,11 @@ parallel --eta --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSumm
 if [ "$has_deny_branch_zero_override" == "1" ]
 then
     echo -e $startDiv"Second cleanup of files for branch zero (none default)"
-    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_branch_zero_list -b 0
+    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_branch_zero_list -b 0
 
 else 
     echo -e $startDiv"Second cleanup of files for branch zero using the default branch deny list"
-    $srcDir/gms/outputs_cleanup.py -d $outputHucDataDir -l $deny_branches_list -b 0
+    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_branches_list -b 0
 fi
 
 echo "---- All huc for $hucNumber branches have been now processed"
diff --git a/src/gms/stream_branches.py b/src/stream_branches.py
similarity index 100%
rename from src/gms/stream_branches.py
rename to src/stream_branches.py
diff --git a/src/gms/subset_catch_list_by_branch_id.py b/src/subset_catch_list_by_branch_id.py
similarity index 100%
rename from src/gms/subset_catch_list_by_branch_id.py
rename to src/subset_catch_list_by_branch_id.py
diff --git a/src/gms/toDo.md b/src/toDo.md
similarity index 100%
rename from src/gms/toDo.md
rename to src/toDo.md
diff --git a/src/usgs_gage_aggregate.py b/src/usgs_gage_aggregate.py
index 24d6d26c4..1a8993e83 100644
--- a/src/usgs_gage_aggregate.py
+++ b/src/usgs_gage_aggregate.py
@@ -68,20 +68,20 @@ def agg_function(self):
 
     parser = argparse.ArgumentParser(description='Aggregates usgs_elev_table.csv at the HUC level')
     parser.add_argument('-fim','--fim_directory', help='Input FIM Directory', required=True)
-    parser.add_argument('-gms','--gms_inputs', help='Input gms_inputs CSV file', required=False)
+    parser.add_argument('-i','--fim_inputs', help='Input fim_inputs CSV file', required=False)
 
     args = vars(parser.parse_args())
 
     fim_directory = args['fim_directory']
-    gms_inputs = args['gms_inputs']
+    fim_inputs = args['fim_inputs']
     assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
 
-    if gms_inputs:
-        gms_inputs = pd.read_csv(gms_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
+    if fim_inputs:
+        fim_inputs = pd.read_csv(fim_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
 
-        for huc in gms_inputs.huc.unique():
+        for huc in fim_inputs.huc.unique():
 
-            branches = gms_inputs.loc[gms_inputs.huc == huc, 'levpa_id'].tolist()
+            branches = fim_inputs.loc[fim_inputs.huc == huc, 'levpa_id'].tolist()
             huc = HucDirectory(join(fim_directory, huc), limit_branches=branches)
             huc.agg_function()
 
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index d3efc61c2..187bb3657 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -76,24 +76,24 @@ def sjoin_nearest_to_nwm(pnt, lines, union):
             return int(lines.iloc[queried_index[0]].feature_id.item())
 
     @staticmethod
-    def filter_gage_branches(gms_inputs_filename):
+    def filter_gage_branches(fim_inputs_filename):
 
-        fim_dir = os.path.dirname(gms_inputs_filename)
-        gms_inputs = pd.read_csv(gms_inputs_filename, header=None, names=['huc', 'levpa_id'],
+        fim_dir = os.path.dirname(fim_inputs_filename)
+        fim_inputs = pd.read_csv(fim_inputs_filename, header=None, names=['huc', 'levpa_id'],
             dtype={'huc':str, 'levpa_id':str})
 
         for huc_dir in [d for d in os.listdir(fim_dir) if re.search('^\d{8}$', d)]:
 
             gage_file = os.path.join(fim_dir, huc_dir, 'usgs_subset_gages.gpkg')
             if not os.path.isfile(gage_file):
-                gms_inputs.drop(gms_inputs.loc[gms_inputs.huc == huc_dir].index, inplace=True)
+                fim_inputs.drop(fim_inputs.loc[fim_inputs.huc == huc_dir].index, inplace=True)
                 continue
 
             gages = gpd.read_file(gage_file)
             level_paths = gages.levpa_id
-            gms_inputs.drop(gms_inputs.loc[(gms_inputs.huc == huc_dir) & (~gms_inputs.levpa_id.isin(level_paths))].index, inplace=True)
+            fim_inputs.drop(fim_inputs.loc[(fim_inputs.huc == huc_dir) & (~fim_inputs.levpa_id.isin(level_paths))].index, inplace=True)
         
-        gms_inputs.to_csv(gms_inputs_filename, index=False, header=False)
+        fim_inputs.to_csv(fim_inputs_filename, index=False, header=False)
 
 
 if __name__ == '__main__':
@@ -105,7 +105,7 @@ def filter_gage_branches(gms_inputs_filename):
     parser.add_argument('-o','--output-filename', help='Table to append data', required=True)
     parser.add_argument('-huc','--huc8-id', help='HUC8 ID (to verify gage location huc)', type=str, required=True)
     parser.add_argument('-bzero_id','--branch-zero-id', help='Branch zero ID value', type=str, required=True)
-    parser.add_argument('-ff','--filter-gms-inputs', help='WARNING: only run this parameter if you know exactly what you are doing', required=False)
+    parser.add_argument('-ff','--filter-fim-inputs', help='WARNING: only run this parameter if you know exactly what you are doing', required=False)
 
     args = vars(parser.parse_args())
 
@@ -115,9 +115,9 @@ def filter_gage_branches(gms_inputs_filename):
     output_filename = args['output_filename']
     huc8 = args['huc8_id']
     bzero_id = args['branch_zero_id']
-    filter_gms_inputs = args['filter_gms_inputs']
+    filter_fim_inputs = args['filter_fim_inputs']
 
-    if not filter_gms_inputs:
+    if not filter_fim_inputs:
 
         usgs_gage_subset = Gage2Branch(usgs_gages_filename, nws_lid_filename, huc8)
         if usgs_gage_subset.gages.empty:
@@ -133,13 +133,12 @@ def filter_gage_branches(gms_inputs_filename):
 
     else:
         ''' 
-        This is an easy way to filter gms_inputs so that only branches with gages will run during gms_run_branch.sh.
-        You can run this option after gms_run_unit.sh has completed.
+        This is an easy way to filter fim_inputs so that only branches with gages will run during fim_process_unit_wb.sh.
 
         example:
-        python3 src/usgs_gage_unit_setup.py -gages x -ahps x -nwm x -o x -huc x -ff /data/outputs/test_output/gms_inputs.csv
+        python3 src/usgs_gage_unit_setup.py -gages x -ahps x -nwm x -o x -huc x -ff /data/outputs/test_output/fim_inputs.csv
         '''
-        assert os.path.isfile(filter_gms_inputs)
-        Gage2Branch.filter_gage_branches(filter_gms_inputs)
+        assert os.path.isfile(filter_fim_inputs)
+        Gage2Branch.filter_gage_branches(filter_fim_inputs)
 
     
diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py
index 13c4fc0eb..5af4d8451 100644
--- a/src/utils/fim_enums.py
+++ b/src/utils/fim_enums.py
@@ -18,12 +18,12 @@ class FIM_exit_codes(Enum):
         - More advanced combinations of codes can be used and we will keep it simple for now.
     - Sample usage:
         import utils/fim_enums
-        print(FIM_exit_codes.GMS_UNIT_NO_BRANCHES.value) -> 60   [this is used in gms/derive_level_paths.py]
+        print(FIM_exit_codes.UNIT_NO_BRANCHES.value) -> 60   [this is used in derive_level_paths.py]
 
     - For more information : https://docs.python.org/3.11/howto/enum.html and https://docs.python.org/3/library/enum.html
     '''
 
-    GMS_UNIT_NO_BRANCHES = 60
+    UNIT_NO_BRANCHES = 60
     NO_FLOWLINES_EXIST = 61
     EXCESS_UNIT_ERRORS = 62    
             
\ No newline at end of file
diff --git a/tools/gms_tools/combine_crosswalk_tables.py b/tools/combine_crosswalk_tables.py
similarity index 100%
rename from tools/gms_tools/combine_crosswalk_tables.py
rename to tools/combine_crosswalk_tables.py
diff --git a/tools/gms_tools/compare_ms_and_non_ms_metrics.py b/tools/compare_ms_and_non_ms_metrics.py
similarity index 100%
rename from tools/gms_tools/compare_ms_and_non_ms_metrics.py
rename to tools/compare_ms_and_non_ms_metrics.py
diff --git a/tools/gms_tools/compile_comp_stats.py b/tools/compile_comp_stats.py
similarity index 81%
rename from tools/gms_tools/compile_comp_stats.py
rename to tools/compile_comp_stats.py
index c5f8a6458..b2bb0fe08 100755
--- a/tools/gms_tools/compile_comp_stats.py
+++ b/tools/compile_comp_stats.py
@@ -6,6 +6,14 @@
 import argparse
 import os
 
+########################################################
+'''
+Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+   a significant review and/or upgrade.
+'''
+
+########################################################
+
 
 def Compile_comp_stats(hydrofabric_dirs):
     
@@ -67,6 +75,15 @@ def get_log_files(hydrofabric_dirs):
 
 if __name__ == '__main__':
 
+    ########################################################
+    '''
+    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+    a significant review and/or upgrade.
+    '''
+
+    ########################################################
+
+
     # parse arguments
     parser = argparse.ArgumentParser(description='Get Comp Stats')
     parser.add_argument('-y','--hydrofabric_dirs', help='Directory path to FIM hydrofabric by processing unit', required=True,nargs='+')
diff --git a/tools/gms_tools/compile_computational_stats.py b/tools/compile_computational_stats.py
similarity index 64%
rename from tools/gms_tools/compile_computational_stats.py
rename to tools/compile_computational_stats.py
index f7ab6daed..e9f10cda4 100755
--- a/tools/gms_tools/compile_computational_stats.py
+++ b/tools/compile_computational_stats.py
@@ -9,6 +9,15 @@
 # desired output for branches
 # dataframe columns: HUC, branch_id, exit status, ,time, ram, 
 
+########################################################
+'''
+Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+   a significant review and/or upgrade.
+'''
+
+########################################################
+
+
 def compile_summary(gms_output_dir,ouput=None):
 
     unit_summary = join(gms_output_dir,logs, 'summary_gms_unit.log')
@@ -18,9 +27,16 @@ def compile_summary(gms_output_dir,ouput=None):
     branch_summary = pd.read_csv(branch_summary,sep='\t')
 
 
+if __name__ == '__main__':
 
+    ########################################################
+    '''
+    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+    a significant review and/or upgrade.
+    '''
+
+    ########################################################
 
-if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Create stream network level paths')
     parser.add_argument('-d','--gms-output-dir', help='Input stream network', required=True)
diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py
index 3eddb202e..d5784594b 100644
--- a/tools/composite_inundation.py
+++ b/tools/composite_inundation.py
@@ -12,16 +12,24 @@
 from tqdm import tqdm
 
 from inundation import inundate
-from gms_tools.mosaic_inundation import Mosaic_inundation
-from gms_tools.inundate_gms import Inundate_gms
+from mosaic_inundation import Mosaic_inundation
+from inundate_gms import Inundate_gms
 from utils.shared_functions import FIM_Helpers as fh
 from utils.shared_variables import elev_raster_ndv
 
+########################################################
+'''
+Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+   a significant review and/or upgrade.
+'''
+
+########################################################
+
+
 class InundateModel_HUC(object):
 
-    def __init__(self, model, source_directory, huc):
+    def __init__(self, source_directory, huc):
 
-        self.model = model
         self.source_directory = source_directory
         self.huc = huc        
 
@@ -48,13 +56,6 @@ def inundate_huc(self, flows_file, composite_output_dir, output_name, log_file_p
         if not os.path.isdir(output_huc_dir):
             os.mkdir(output_huc_dir)
 
-        if self.model == "ms":
-            extent_friendly = "mainstem (MS)"
-        elif self.model == "fr":
-            extent_friendly = "full-resolution (FR)"
-        else: # gms
-            extent_friendly = "FIM4 GMS"
-
         inundation_map_file = None
 
         output_raster_name = os.path.join(output_huc_dir, output_name)
@@ -64,12 +65,12 @@ def inundate_huc(self, flows_file, composite_output_dir, output_name, log_file_p
         log_file = None
         inundation_list_file = None        
         if (log_file_path != None):
-            log_file = os.path.join(log_file_path, f"{self.huc}_{self.model}_error_logs.txt") 
+            log_file = os.path.join(log_file_path, f"{self.huc}_error_logs.txt") 
             inundation_list_file = os.path.join(log_file_path, 
-                                               f"{self.huc}_{self.model}_inundation_file_list.csv") 
+                                               f"{self.huc}_inundation_file_list.csv") 
 
         if (verbose):
-            print(f'... Creating an inundation map for the {extent_friendly}'\
+            print(f'... Creating an inundation map for the FIM4'\
                     f' configuration for HUC {self.huc}...')
 
         if self.model in ["fr", "ms"]:
@@ -150,7 +151,7 @@ def composite_huc(self, args):
         composite_model_map_files = []
         for model in args["models"]:
 
-            # setup original fim/gms processed directory
+            # setup original fim processed directory
             if model == "ms" : source_dir = args["fim_dir_ms"]
             elif model == "fr" : source_dir = args["fim_dir_fr"]
             else: source_dir = args["gms_dir"]
diff --git a/tools/consolidate_metrics.py b/tools/consolidate_metrics.py
index d9a80b9f2..0b470631d 100755
--- a/tools/consolidate_metrics.py
+++ b/tools/consolidate_metrics.py
@@ -20,6 +20,15 @@
 pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None)
 
+
+########################################################
+'''
+Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+   a significant review and/or upgrade.
+'''
+
+########################################################
+
 def Consolidate_metrics( benchmarks=['all'],versions=['all'],
                          zones=['total_area'],matching_hucs_only=True,
                          metrics_output_csv=None,
@@ -309,6 +318,14 @@ def parse_eval_metadata(file_names,metadata_field):
 
 if __name__ == '__main__':
 
+    ########################################################
+    '''
+    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+    a significant review and/or upgrade.
+    '''
+
+    ########################################################
+
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
     parser.add_argument('-b','--benchmarks',help='Allowed benchmarks', required=False, default='all', nargs="+")
diff --git a/tools/copy_test_case_folders.py b/tools/copy_test_case_folders.py
index 1c05b4428..4e601e224 100644
--- a/tools/copy_test_case_folders.py
+++ b/tools/copy_test_case_folders.py
@@ -6,14 +6,18 @@
 import sys
 
 # importing python folders in other direcories
-sys.path.append('/foss_fim/src/gms/')
+sys.path.append('/foss_fim/src/')
 import aggregate_branch_lists as agg
 
+#######
+#  Feb 16, 2023: This file is likely now deprecated. If it is used again, the aggregate_branch_lists file will need to 
+#    be reconsiiled as a newer one exists in the src directory with a similar name.
+#######
+
 def copy_folders(folder_name_list,
                 source_dir,
                 target_dir,
-                create_gms_input_list=False,
-                overwrite=False):
+                create_fim_input_list=False):
 
     '''
     Summary: Scans the source_directory looking for huc values from the huc list. Once found,
@@ -29,12 +33,10 @@ def copy_folders(folder_name_list,
         - target_dir: The root folder where the huc folders will be copied to. Note. All contents of 
              each huc folder, including branch folders if applicable, will be copied, in the extact
              structure as the source directory. Note: The target folder need not pre-exist. 
-        - create_gms_input_list: If this flag is set to True, after coping the folders, the 
-            "aggregate_branch_lists.py" file will be called in order to make the gms_input.csv file.
-            The gms_input.csv is required for futher processing such as reprocessing branchs or set up
+        - create_fim_input_list: If this flag is set to True, after coping the folders, the 
+            "aggregate_branch_lists.py" file will be called in order to make the fim_input.csv file.
+            The fim_input.csv is required for futher processing such as reprocessing branchs or set up
             for test cases.
-        - overwrite:  if this value is set to true, the entire target_directory will be emptied of its
-             contents as this process starts if the folder exists. 
     Output:
         - A copy of huc directories (or named folders) as listed in the folder_name_list.
     '''
@@ -73,12 +75,12 @@ def copy_folders(folder_name_list,
 
     print(f"{str(ctr)} folders have been copied to {target_dir}")
 
-    if create_gms_input_list == True:
+    if create_fim_input_list == True:
         # call this code, which scans each huc (unit) directory looking for the branch_id.lst
-        # and adds them together to create the gms_inputs.csv file
+        # and adds them together to create the fim_inputs.csv file
         # Note: folder_name_list needs to be a huc list to work)
-        agg.aggregate_inputs_for_gms(folder_name_list, target_dir, "gms_inputs.csv")
-        print("gms_inputs.csv created")
+        agg.aggregate_branch_lists(folder_name_list, target_dir, "fim_inputs.csv")
+        print("fim_inputs.csv created")
     
 
 if __name__ == '__main__':
@@ -87,7 +89,7 @@ def copy_folders(folder_name_list,
 #   and it has to be run on each root folder, one at a time (for now. aka.. no wildcards)
 
 # Sample Usage: 
-#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_22020420.lst -s /outputs/gms_test_synth/ -t /data/outputs/gms_test_synth_combined -a
+#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_20220420.lst -s /outputs/copy_test_synth/ -t /data/outputs/copy_test_synth_combined -a
 
 #  NOTE the 'a' at the end meaning go ahead create the gms_input.csv. This is normally
 # left for the last folder to be copied over.
@@ -106,8 +108,8 @@ def copy_folders(folder_name_list,
                             help='Target folder where the folders will be copied to',
                             required=True)
 
-    parser.add_argument('-a','--create_gms_input_list',
-                            help='Create the gms_input.csv list after coping',
+    parser.add_argument('-a','--create_fim_input_list',
+                            help='Create the fim_input.csv list after copying',
                             required=False, default=False, action='store_true')
 
     args = vars(parser.parse_args())
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index d0b041f99..256342015 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -25,7 +25,9 @@
 #########################################################################
 #Create boxplot
 #########################################################################
-def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):
+def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue,
+            title_text, fim_configuration, textbox_str = False, 
+            simplify_legend = False, dest_file = False):
     '''
     Create boxplots.
 
@@ -192,7 +194,9 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
 #########################################################################
 #Create barplot
 #########################################################################
-def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):
+def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, 
+            title_text, fim_configuration, textbox_str = False, 
+            simplify_legend = False, display_values = False, dest_file = False):
     '''
     Create barplots.
 
@@ -344,7 +348,9 @@ def filter_dataframe(dataframe, unique_field):
 ##############################################################################
 #Main function to analyze metric csv.
 ##############################################################################
-def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE'] , spatial = False, fim_1_ms = False, site_barplots = False):
+def eval_plots(metrics_csv, workspace, versions = [], 
+               stats = ['CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE'] , spatial = False,
+               fim_1_ms = False, site_barplots = False):
 
     '''
     Creates plots and summary statistics using metrics compiled from
diff --git a/tools/gms_tools/evaluate_continuity.py b/tools/evaluate_continuity.py
similarity index 100%
rename from tools/gms_tools/evaluate_continuity.py
rename to tools/evaluate_continuity.py
diff --git a/tools/gms_tools/find_max_catchment_breadth.py b/tools/find_max_catchment_breadth.py
similarity index 100%
rename from tools/gms_tools/find_max_catchment_breadth.py
rename to tools/find_max_catchment_breadth.py
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index 743552664..db9328dbf 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -14,8 +14,8 @@
 sys.path.append('/foss_fim/src')
 from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
 from utils.shared_functions import getDriver
-from gms_tools.mosaic_inundation import Mosaic_inundation
-from gms_tools.inundate_gms import Inundate_gms
+from mosaic_inundation import Mosaic_inundation
+from inundate_gms import Inundate_gms
 
 
 def generate_categorical_fim(fim_run_dir, source_flow_dir, output_catfim_dir, 
diff --git a/tools/gms_tools/__init__.py b/tools/gms_tools/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tools/gms_tools/inundate_gms.py b/tools/inundate_gms.py
similarity index 94%
rename from tools/gms_tools/inundate_gms.py
rename to tools/inundate_gms.py
index 7b481dd70..e1a90fa67 100755
--- a/tools/gms_tools/inundate_gms.py
+++ b/tools/inundate_gms.py
@@ -43,8 +43,8 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
         #logging.basicConfig(filename=log_file, level=logging.INFO)
         #logging.info('HUC8,BranchID,Exception')
 
-    # load gms inputs
-    hucs_branches = pd.read_csv( os.path.join(hydrofabric_dir,'gms_inputs.csv'),
+    # load fim inputs
+    hucs_branches = pd.read_csv( os.path.join(hydrofabric_dir,'fim_inputs.csv'),
                                  header=None,
                                  dtype= {0:str,1:str} )
     
@@ -164,18 +164,18 @@ def __inundate_gms_generator( hucs_branches,
         huc = str(row[0])
         branch_id = str(row[1])
 
-        gms_dir = os.path.join(hydrofabric_dir, huc, 'branches')
+        huc_dir = os.path.join(hydrofabric_dir, huc, 'branches')
 
         rem_file_name = 'rem_zeroed_masked_{}.tif'.format(branch_id)
-        rem_branch = os.path.join( gms_dir, branch_id, rem_file_name )
+        rem_branch = os.path.join( huc_dir, branch_id, rem_file_name )
 
         catchments_file_name = f'gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif'
-        catchments_branch = os.path.join( gms_dir, branch_id, catchments_file_name )
+        catchments_branch = os.path.join( huc_dir, branch_id, catchments_file_name )
 
-        hydroTable_branch = os.path.join( gms_dir,branch_id, 'hydroTable_{}.csv'.format(branch_id) )
+        hydroTable_branch = os.path.join( huc_dir, branch_id, 'hydroTable_{}.csv'.format(branch_id) )
 
         xwalked_file_name = f'gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg'
-        catchment_poly = os.path.join( gms_dir, branch_id, xwalked_file_name )
+        catchment_poly = os.path.join( huc_dir, branch_id, xwalked_file_name )
         
     
         # branch output
@@ -224,7 +224,7 @@ def __inundate_gms_generator( hucs_branches,
 if __name__ == '__main__':
 
     # parse arguments
-    parser = argparse.ArgumentParser(description='Inundate GMS')
+    parser = argparse.ArgumentParser(description='Inundate FIM')
     parser.add_argument('-y','--hydrofabric_dir', help='Directory path to FIM hydrofabric by processing unit', required=True)
     parser.add_argument('-u','--hucs',help='List of HUCS to run',required=False,default=None,type=str,nargs='+')
     parser.add_argument('-f','--forecast',help='Forecast discharges in CMS as CSV file',required=True)
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index 74a7c1bb2..a07f9c9d9 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -13,8 +13,8 @@
 
 sys.path.append('/foss_fim/src')
 from datetime import datetime
-from gms_tools.mosaic_inundation import Mosaic_inundation
-from gms_tools.inundate_gms import Inundate_gms
+from mosaic_inundation import Mosaic_inundation
+from inundate_gms import Inundate_gms
 from inundation import inundate
 from utils.shared_variables import elev_raster_ndv, PREP_PROJECTION
 from utils.shared_functions import FIM_Helpers as fh
@@ -27,7 +27,8 @@
 #DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
 
 
-def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, inc_mosaic, job_number):
+def inundate_nation(fim_run_dir, output_dir, magnitude_key, 
+                    flow_file, inc_mosaic, job_number):
 
     
     assert os.path.isdir(fim_run_dir), f'ERROR: could not find the input fim_dir location: {fim_run_dir}'
@@ -74,10 +75,9 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, inc_mosai
             huc_list.append(huc)
 
     print('Inundation raw mosaic outputs here: ' + magnitude_output_dir)
-    
-    config = "GMS"
-    
-    run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, config, flow_file, job_number])
+   
+    run_inundation([fim_run_dir, huc_list, magnitude_key,
+                    magnitude_output_dir, flow_file, job_number])
                 
     # Perform mosaic operation
     if inc_mosaic:
@@ -129,7 +129,7 @@ def run_inundation(args):
     This script is a wrapper for the inundate function and is designed for multiprocessing.
     
     Args:
-        args (list): [fim_run_dir (str), huc_list (list), magnitude (str), magnitude_output_dir (str), config (str), forecast (str), job_number (int)]
+        args (list): [fim_run_dir (str), huc_list (list), magnitude (str), magnitude_output_dir (str), forecast (str), job_number (int)]
     
     """
     
@@ -137,39 +137,38 @@ def run_inundation(args):
     huc_list = args[1]
     magnitude = args[2]
     magnitude_output_dir = args[3]
-    config = args[4]
-    forecast = args[5]
-    job_number = args[6]
+    forecast = args[4]
+    job_number = args[5]
    
     # Define file paths for use in inundate().
 
-    inundation_raster = os.path.join(magnitude_output_dir, magnitude + '_' + config + '_inund_extent.tif')
+    inundation_raster = os.path.join(magnitude_output_dir, magnitude + '_inund_extent.tif')
     
     print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude))
 
     map_file = Inundate_gms( hydrofabric_dir = fim_run_dir, 
-                                    forecast = forecast, 
-                                    num_workers = job_number,
-                                    hucs = huc_list,
-                                    inundation_raster = inundation_raster,
-                                    inundation_polygon = None,
-                                    depths_raster = None,
-                                    verbose = True,
-                                    log_file = None,
-                                    output_fileNames = None )
+                             forecast = forecast, 
+                             num_workers = job_number,
+                             hucs = huc_list,
+                             inundation_raster = inundation_raster,
+                             inundation_polygon = None,
+                             depths_raster = None,
+                             verbose = True,
+                             log_file = None,
+                             output_fileNames = None )
     
     Mosaic_inundation( map_file,
-                                    mosaic_attribute = 'inundation_rasters',
-                                    mosaic_output = inundation_raster,
-                                    #mask = os.path.join(fim_run_dir,huc8,'wbd.gpkg'),
-                                    mask = None,
-                                    unit_attribute_name = 'huc8',
-                                    nodata = elev_raster_ndv,
-                                    workers = 1,
-                                    remove_inputs = True,
-                                    subset = None,
-                                    verbose = True,
-                                    is_mosaic_for_gms_branches = True )
+                       mosaic_attribute = 'inundation_rasters',
+                       mosaic_output = inundation_raster,
+                       #mask = os.path.join(fim_run_dir,huc8,'wbd.gpkg'),
+                       mask = None,
+                       unit_attribute_name = 'huc8',
+                       nodata = elev_raster_ndv,
+                       workers = 1,
+                       remove_inputs = True,
+                       subset = None,
+                       verbose = True,
+                       is_mosaic_for_branches = True )
 
 def create_bool_rasters(args):
     in_raster_dir = args[0]
diff --git a/tools/inundation.py b/tools/inundation.py
index e2e947d08..188f0bce0 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -270,10 +270,8 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
         if out_vector_profile is None:
             out_vector_profile = {'crs' : crs , 'driver' : 'GPKG'}
 
-        out_vector_profile['schema'] = {
-                                         'geometry' : 'Polygon',
-                                         'properties' : OrderedDict([('HydroID' , 'int')])
-                                       }
+        out_vector_profile['schema'] = { 'geometry' : 'Polygon',
+                                         'properties' : OrderedDict([('HydroID' , 'int')]) }
 
         # open output inundation polygons
         if isinstance(inundation_polygon,str):
diff --git a/tools/make_boxes_from_bounds.py b/tools/make_boxes_from_bounds.py
index b9ea31d45..69b9b5a8b 100755
--- a/tools/make_boxes_from_bounds.py
+++ b/tools/make_boxes_from_bounds.py
@@ -9,7 +9,9 @@
 from foss_fim.src.utils.shared_functions import getDriver
 
 
-def find_hucs_of_bounding_boxes(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',huc_output_file=None,forecast_output_file=None,bounding_boxes_outfile=None):
+def find_hucs_of_bounding_boxes(bounding_boxes_file, wbd=None, projection_of_boxes='EPSG:4329',
+                                wbd_layer='WBDHU8', huc_output_file=None,
+                                forecast_output_file=None, bounding_boxes_outfile=None):
 
 
     # load bounding box file
diff --git a/tools/gms_tools/mosaic_inundation.py b/tools/mosaic_inundation.py
similarity index 93%
rename from tools/gms_tools/mosaic_inundation.py
rename to tools/mosaic_inundation.py
index ff6165a6c..215e6511c 100755
--- a/tools/gms_tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -7,7 +7,7 @@
 import sys
 
 from glob import glob
-from gms_tools.overlapping_inundation import OverlapWindowMerge
+from overlapping_inundation import OverlapWindowMerge
 from tqdm import tqdm
 from utils.shared_variables import elev_raster_ndv
 from utils.shared_functions import FIM_Helpers as fh
@@ -22,10 +22,10 @@ def Mosaic_inundation( map_file,
                        remove_inputs = False,
                        subset = None,
                        verbose = True,
-                       is_mosaic_for_gms_branches = False ):
+                       is_mosaic_for_branches = False ):
     
     # Notes:
-    #    - If is_mosaic_for_gms_branches is true, the mosaic output name
+    #    - If is_mosaic_for_branches is true, the mosaic output name
     #      will add the HUC into the output name for overwrite resons.
 
     # check input
@@ -79,7 +79,7 @@ def Mosaic_inundation( map_file,
         # fr set with a gms composite map.
 
         ag_mosaic_output = mosaic_output
-        if (is_mosaic_for_gms_branches) and (ag not in mosaic_output):
+        if (is_mosaic_for_branches) and (ag not in mosaic_output):
             ag_mosaic_output = fh.append_id_to_file_name(mosaic_output, ag) # change it
 
         mosaic_by_unit(inundation_maps_list, 
@@ -156,8 +156,8 @@ def mosaic_by_unit(inundation_maps_list,
                         required=False, default=False, action='store_true')
     parser.add_argument('-v','--verbose', help='Remove original input inundation Maps', 
                         required=False, default=False, action='store_true')
-    parser.add_argument('-g','--is-mosaic-for-gms-branches', 
-                        help='If the mosaic is for gms branchs, include this arg',
+    parser.add_argument('-g','--is-mosaic-for-branches', 
+                        help='If the mosaic is for branchs, include this arg',
                         required=False, default=False, action='store_true')
 
     args = vars(parser.parse_args())
diff --git a/tools/gms_tools/overlapping_inundation.py b/tools/overlapping_inundation.py
similarity index 100%
rename from tools/gms_tools/overlapping_inundation.py
rename to tools/overlapping_inundation.py
diff --git a/tools/gms_tools/plots.py b/tools/plots.py
similarity index 100%
rename from tools/gms_tools/plots.py
rename to tools/plots.py
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 9d46ca582..3920f540d 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -5,8 +5,8 @@
 
 from tools_shared_variables import TEST_CASES_DIR, INPUTS_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT, elev_raster_ndv
 from inundation import inundate
-from gms_tools.mosaic_inundation import Mosaic_inundation
-from gms_tools.inundate_gms import Inundate_gms
+from mosaic_inundation import Mosaic_inundation
+from inundate_gms import Inundate_gms
 from tools_shared_functions import compute_contingency_stats_from_rasters
 from utils.shared_functions import FIM_Helpers as fh
 
@@ -284,7 +284,7 @@ def _inundate_and_compute(self,
         # Inundate REM
         if not compute_only:             # composite alpha tests don't need to be inundated
             if model == 'GMS':
-                fh.vprint("Begin GMS Inundation", verbose)
+                fh.vprint("Begin FIM4 Inundation", verbose)
                 map_file = Inundate_gms( hydrofabric_dir = os.path.dirname(self.fim_dir), 
                                          forecast = benchmark_flows, 
                                          num_workers = gms_workers,
@@ -296,7 +296,7 @@ def _inundate_and_compute(self,
                                          log_file = None,
                                          output_fileNames = None )
                 #if (len(map_file) > 0):
-                fh.vprint("Begin GMS Mosaic", verbose)
+                fh.vprint("Begin FIM4 Mosaic", verbose)
                 Mosaic_inundation( map_file,
                                     mosaic_attribute = 'inundation_rasters',
                                     mosaic_output = predicted_raster_path,
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 6182cc105..21c5d47d4 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -232,11 +232,11 @@ def progress_bar_handler(executor_dict, verbose, desc):
 
     # Sample usage:
     '''
-     === FOR GMS
+     === FOR (FIM 4)
     python /foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v gms_test_synth_combined -jh 2 -jb 40 -m /outputs/gms_test_synth_combined/gms_synth_metrics.csv -vg -o
 
      Notes:
-       - gms_input.csv MUST be in the folder suggested.
+       - fim_input.csv MUST be in the folder suggested.
        - the -v param is the name in the folder in the "outputs/" directory where the test hucs are at.
            It also becomes the folder names inside the test_case folders when done.
        - the -vg param may not be working (will be assessed better on later releases).
@@ -253,7 +253,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
          command becomes:  find . -name gms_test_* -type d  -exec rm -rdf {} +
     '''
     '''
-     === FOR FIM
+     === FOR FIM 3
     python /foss_fim/tools/synthesize_test_cases.py -c DEV -e MS -v dev_fim_3_0_29_1_ms -jh 4 -m /outputs/dev_fim_3_0_29_1_ms/alpha/alpha_master_metrics_fim_3_0_29_1_ms_src_adjust.csv -vg -o
 
      Notes:
@@ -273,7 +273,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
     parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=False,default='DEV')
     parser.add_argument('-l','--calibrated',help='Denotes use of calibrated n values. This should be taken from meta-data from hydrofabric dir',required=False, default=False,action='store_true')
-    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed. This should be taken from meta-data in hydrofabric dir.',required=True)
+    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed. This should be taken from meta-data in hydrofabric dir.', default='GMS', required=False)
     parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
     parser.add_argument('-jh','--job-number-huc',help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',required=False, default=1,type=int)
     parser.add_argument('-jb','--job-number-branch',help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',required=False, default=1,type=int)
diff --git a/unit_tests/fim3_README.md b/unit_tests/fim3_README.md
new file mode 100644
index 000000000..b7f37d1d3
--- /dev/null
+++ b/unit_tests/fim3_README.md
@@ -0,0 +1,104 @@
+## Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
+
+Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
+
+#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
+
+### This folder and files are for unit testing python files
+
+## Creating unit tests
+
+For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and it's json parms file. 
+
+The files should be named following FIM convention:
+
+{source py file name}_unittests.py     ie) derive_level_paths_unittests.py
+{source py file name}_params.json      ie) derive_level_paths_params.json
+
+
+## Tips to creating a new json file for a new unit test file.
+
+There are multiple way to figure out a set of default json parameters for the new unit test file. 
+
+One way is to use the incoming arg parser. Most py files, they include the code block of " __name__ == '__main__':", followed by external arg parsing, ie) the args = vars(parser.parse_args()). 
+	- Add a print(args) or similar, and get all the values including keys as output.
+	- Copy that into an editor being used to create the json file.
+	- Add a line break after every comma.
+	- Find/replace all single quotes to double quotes then cleanup the left tab formatting.
+
+
+## Running unit tests
+
+Start a docker container as you normally would for any development. ie) docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
+	- ie) docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
+
+For unit tests to work, you need to run the following (if not already in place).
+Notice a modified branch "deny_branch_unittests.lst"  (special for unittests)
+
+Here are the params and args you need if you need to re-run unit and branch
+
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -bd /foss_fim/config/deny_branch_unittests.lst -ud None -j 1 -o
+
+**NOTICE: the deny file used for fim_pipeline... its a special one for unittests `deny_branch_unittests.lst`.
+
+If you need to run inundation tests, fun the following:
+
+python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_not_remove -jh 1 -jb 1 -m /outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
+
+If you want to test just one unit test, here is an example:
+At the root terminal window, run:  python ./foss_fim/unit_tests/derive_level_paths_unittests.py  or python ./foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
+(replace with your own script and path name)
+
+## Key Notes for creating new unit tests
+1) All test functions must start with the phrase "test_". That is how the unit test engine picks it up. The rest of the function name does not have to match the pattern of {function name being tested} but should. Further, the rest of the function name should say what the test is about, ie) _failed_input_path.  ie) test_{some_function_name_from_the_source_code_file}_failed_input_path. It is fine that the function names get very long (common in the industry).
+
+2) The output for a selected "unittest" import engine can be ugly and hard to read. It sometimes mixed outputs from multiple unit test functions simulataneously instead of keeping all output together for a given unit test. We will try to make this better later.
+
+3) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
+
+4) You can not control the order that unit tests are run within a unit test file. (UnitTest engine limitation)
+
+5) There must be at one "{original py file name}_params.json" file.
+
+6) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to full pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass. Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data"
+
+7) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data", if one does not already exist. Generally, the individual unit tests, will call the "valid_data" node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie.. major different test conditions).
+
+8) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
+
+9) One py file = one "{original py file name}_unittests.py" file.
+
+10) Sometimes you may want to run a full successful "happy path" version through fim_pipeline.sh (or similar), to get all of the files you need in place to do your testing. However.. you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the "-d" parameter (denylist). ie) Normally:  fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o, but ours would be 
+gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o. 
+
+## Future Enhancements
+1) We can automate triggers on these files for things like checking triggers or an single global "run_all_unittest" script, but for now.. its one offs.
+
+2) Better output from the unit tests including verbosity output control
+
+3) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
+
+
+## testing for failing conditions
+- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for `ONLY one variable value change`. Aka.. a "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
+
+- It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
+
+- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass. 
+
+An example is in unit_tests/Derive_level_paths_unittests.py -> test_Derive_level_paths_invalid_input_stream_network (function). It is incomplete but give you the pattern.
+
+We have almost no "assert"s yet, but most unit test usually have one or more "assert" test. See https://docs.python.org/3/library/unittest.html for more details.
+
+## Unit tests currently available
+python3 /foss_fim/unit_tests/derive_level_paths_unittests.py
+python3 /foss_fim/unit_tests/tools/inundate_unittests.py
+python3 /foss_fim/unit_tests/tools/inundate_gms_unittests.py
+python3 /foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
+python3 /foss_fim/unit_tests/filter_catchments_and_add_attributes_unittests.py
+python3 /foss_fim/unit_tests/rating_curve_comparison_unittests.py
+python3 /foss_fim/unit_tests/shared_functions_unittests.py
+python3 /foss_fim/unit_tests/split_flows_unittests.py
+python3 /foss_fim/unit_tests/usgs_gage_crosswalk_unittests.py
+
+
diff --git a/unit_tests/fim3__template_unittests.py b/unit_tests/fim3__template_unittests.py
new file mode 100644
index 000000000..1ba2981ce
--- /dev/null
+++ b/unit_tests/fim3__template_unittests.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import <Your original source python file name> as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_<Your original source python file name>(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_<method name you are about to test>_success(self):
+
+        '''
+        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
+          If there is no return value from the method, please say so.>
+        
+        Dev Notes: (which can be removed after you make this file)
+            Remember... You need to validate the method output if there is any. However, if you have time, it
+            is also recommended that you validate other outputs such as writing or updating file on the file 
+            system, aka: Does the expected file exist. Don't worry about its contents.
+
+        '''
+
+        #global params_file
+        
+        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        
+       
+        # for now we are happy if no exceptions are thrown.
+        
+        # < See the readme.md, clip_vectors_to_wbd_unittests.py or derive_level_paths_unittests.py 
+        # for examples.>
+        # Replace this stub example with your own.
+        # Try to use the same order to make it easier.
+        # Remember, if the method accepts **params, then you can sent that in here as well.
+        #   ie: my_py_class.my_method(** params)
+        
+        src.subset_vector_layers(hucCode = params["hucCode"],
+                                    nwm_streams_filename = params["nwm_streams"],
+                                    etc, etc for each param)
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    # ***** REMOVE THIS BLOCK IF YOU ARE NOT USING IT ***
+    #       EXAMPLE SUCCESSFUL FAIL    
+    # 
+    #def test_subset_vector_layers_fail_invalid_stream_path(self):
+    #    '''
+    #    Notes about what the test is and the expected results (or expected exception if applicable)
+    #    '''
+    #   #global params_file
+        
+    #   params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+
+    #   params["nwm_streams"] = "some bad path"
+
+    #   clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
+    #                                            nwm_streams_filename = params["nwm_streams"],
+    #                                            etc, etc for each param)
+    #   
+       
+    #    print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+    #    print("*************************************************************")
+        
+    # ***********************
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/fim3_check_unit_errors_params.json b/unit_tests/fim3_check_unit_errors_params.json
new file mode 100644
index 000000000..bf89913a1
--- /dev/null
+++ b/unit_tests/fim3_check_unit_errors_params.json
@@ -0,0 +1,11 @@
+{
+	"valid_data":
+	{
+		"fim_dir": "/outputs/check_errors_example_unit_tests"
+	},
+
+	"invalid_path":
+	{
+		"fim_dir": "/outputs/check_errors_example_unit_tests_not_not_valid"
+	}	
+}
\ No newline at end of file
diff --git a/unit_tests/fim3_check_unit_errors_unittests.py b/unit_tests/fim3_check_unit_errors_unittests.py
new file mode 100644
index 000000000..04ebdfda7
--- /dev/null
+++ b/unit_tests/fim3_check_unit_errors_unittests.py
@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+
+import inspect
+import math
+import os
+import shutil
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import check_unit_errors as src
+from utils.fim_enums import FIM_exit_codes
+
+from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
+                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
+
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_check_unit_errors(unittest.TestCase):
+
+
+    # CURRENT 
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        try:        
+            params_file_path = ut_helpers.get_params_filename(__file__)
+        except FileNotFoundError as ex:
+            print(f"params file not found. ({ex}). Check pathing and file name convention.")
+            sys.exit(1)
+
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+    '''
+    To make most of these unit test work, we have to do the following:
+    - rename the current unit_errors directory (if it exists)
+    - empty the unit_errors directory
+    - create a bunch of dummy files in it
+    - perform the unit test
+    - delete the unit_errors directory
+    - rename the original unit_test folder back to unit_tests
+    '''
+
+    def test_check_unit_errors_success_below_min_errors(self):
+
+        # Expecting no errors
+        # test to ensure the number of dummy files is less than the
+        # overall min number of error files.
+
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
+        
+            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
+                                                                            num_dummy_files_reqd)
+            expected_output = 0
+            actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
+            
+            err_msg = "actual output does not match expected output"
+            self.assertEqual(expected_output, actual_output, err_msg)
+            
+            if (ue_folder_existed):
+                self.__remove_temp_unit_errors_folder(params["fim_dir"])
+            
+            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+            print("*************************************************************")
+            
+        except BaseException as ex:
+            
+            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
+            
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+            # re throw for the unit test engine            
+            raise ex
+
+
+    def test_check_unit_errors_fail_above_min_errors(self):
+
+        # test to ensure the number of dummy files is more than the
+        # overall min number of error files.
+
+        # Expecting sys.exit of 62
+        # we do expect this to fail and if it fails, it is successful
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD + 1
+        
+            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
+            src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
+            
+            raise Exception("Test Failed: expected sys.exit error but did not receive it")
+            
+        except BaseException as ex:
+           
+            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
+                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            else:
+                print("ex value is")
+                print(ex)
+                raise Exception("Expected a system exit code but did not receive it")
+          
+        finally:
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+
+    def test_check_unit_errors_success_above_percent_errors(self):
+
+        # Expecting no errors
+        # test to ensure the number of dummy files is more than the
+        # overall min number of error files.
+        
+        # we do expect this not to to fail as it is greater than 10 errors
+        # but below the percent threshhold
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
+            
+            print(f"num_dummy_files_reqd is {num_dummy_files_reqd}")
+            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
+                                                                            num_dummy_files_reqd)
+            
+            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
+            expected_output = 0
+            actual_output = src.check_unit_errors(params["fim_dir"], num_total_units)
+            
+            err_msg = "actual output does not match expected output"
+            self.assertEqual(expected_output, actual_output, err_msg)
+            
+            if (ue_folder_existed):
+                self.__remove_temp_unit_errors_folder(params["fim_dir"])
+            
+            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+            print("*************************************************************")
+            
+        except BaseException as ex:
+            
+            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
+            
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+
+            # re throw for the unit test engine            
+            raise ex
+
+
+    def test_check_unit_errors_fail_below_percent_errors(self):
+
+        # Expecting sys.exit of 62
+        # test to ensure the number of dummy files is more than the
+        # overall min number of error files.
+        
+        # we do expect this to fail as it is greater than 10 errors
+        # AND below the percent threshhold (more percent errors than the threshold)
+        params = self.params["valid_data"].copy()
+        
+        try:
+            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
+            
+            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
+            
+            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) - 10
+            print(f" is {num_total_units}")
+
+            src.check_unit_errors(params["fim_dir"], num_total_units)
+            raise Exception("Test Failed: expected sys.exit error but did not receive it")
+                    
+        except BaseException as ex:
+           
+            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
+                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+                print("*************************************************************")                             
+            else:
+                print("ex value is")
+                print(ex)
+                raise Exception("Expected a system exit code but did not receive it")
+          
+        finally:
+            # no matter what, we have to put the unit_errors folders back to the way
+            # is was, so try again if we have too
+            self.__remove_temp_unit_errors_folder(params["fim_dir"])
+            
+
+    def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files):
+        '''
+        Process:
+            We want to preserve the original unit_errors folder if it exists,
+            so we wil rename it.
+            Then we will make a new unit_errors folder and fill it with a bunch of 
+            dummy files.
+            A dummy file for non_zero_exit_codes.log will also be created.
+        Input:
+            output_folder: the root output folder (ie. /outputs/check_errors_example_unit_tests/)
+            number_of_files: how many dummy files to create
+        Returns:
+            True if the 'unit_errors' folder did original exist and needs to be renamed back.
+            False if the 'unit_errors' folder never existed in the first place.
+        '''
+        
+        ue_folder_preexists = False
+        
+        if (not os.path.isdir(output_folder)):
+            raise Exception(f"unit test root folder of {output_folder} does not exist")
+        
+        ue_folder = os.path.join(output_folder, "unit_errors")
+        temp_ue_folder = ue_folder + "_temp"
+        if (os.path.isdir(ue_folder)):
+            ue_folder_preexists = True
+            os.rename(ue_folder, temp_ue_folder)
+        
+        os.mkdir(ue_folder)
+        
+        for i in range(0, number_of_files):
+            file_name = "sample_" + str(i) + ".txt"
+            file_path = os.path.join(ue_folder, file_name)
+            with open(file_path, 'w') as fp:
+                pass
+        
+        return ue_folder_preexists
+        
+        
+    def __remove_temp_unit_errors_folder(self, output_folder):
+        '''
+        Process:
+            We want to preserve the original unit_errors folder if it exists,
+            so we will delete our temp unit test version of 'unit_errors', and rename
+            the original back to 'unit_errors'
+            Note.. it is possible the temp folder does not exist,
+            but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
+        Input:
+            output_folder: the root output folder (ie. /outputs/check_errors_example_unit_tests/)
+        Returns:
+            nothing
+        '''
+
+        ue_folder = os.path.join(output_folder, "unit_errors")        
+        if (os.path.isdir(ue_folder)):
+            shutil.rmtree(ue_folder)       
+        
+        temp_ue_folder = ue_folder + "_temp"        
+        if (os.path.isdir(temp_ue_folder)):
+            os.rename(temp_ue_folder, ue_folder)
+      
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/fim3_derive_level_paths_params.json b/unit_tests/fim3_derive_level_paths_params.json
new file mode 100644
index 000000000..97e5e1a8b
--- /dev/null
+++ b/unit_tests/fim3_derive_level_paths_params.json
@@ -0,0 +1,31 @@
+{
+	"valid_data":
+	{
+		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"branch_id_attribute": "levpa_id",
+		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"reach_id_attribute": "ID",
+		"verbose": true
+	},
+	
+	"dropped_stream_orders_no_branches_remaining":
+	{
+		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams.gpkg",
+		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
+		"branch_id_attribute": "levpa_id",
+		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"reach_id_attribute": "ID",
+		"verbose": true
+	}	
+}
+
+
diff --git a/unit_tests/fim3_derive_level_paths_unittests.py b/unit_tests/fim3_derive_level_paths_unittests.py
new file mode 100644
index 000000000..5f68a72d8
--- /dev/null
+++ b/unit_tests/fim3_derive_level_paths_unittests.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+# importing python folders in other direcories
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other direcories
+sys.path.append('/foss_fim/src/')
+import derive_level_paths as src
+import stream_branches
+from utils.fim_enums import FIM_exit_codes as fec
+
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_Derive_level_paths(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+    
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+    def test_Derive_level_paths_success_all_params(self):
+    
+        '''
+        This test includes all params with many optional parms being set to the default value of the function
+        '''
+      
+        params = self.params["valid_data"].copy()
+
+        # Function Notes:
+        # huc_ids no longer used, so it is not submitted
+        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
+        # skip them here.
+        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
+        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                            out_stream_network = params["out_stream_network"],
+                                            branch_id_attribute = params["branch_id_attribute"],
+                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                            headwaters_outfile = params["headwaters_outfile"],
+                                            catchments = params["catchments"],
+                                            catchments_outfile = params["catchments_outfile"],
+                                            branch_inlets_outfile = params["branch_inlets_outfile"],
+                                            reach_id_attribute = params["reach_id_attribute"],
+                                            verbose = params["verbose"])
+
+        # -----------
+        # test data type being return is as expected. Downstream code might to know that type
+        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
+        
+        # -----------
+        #**** NOTE: Based on 05030104         
+        # Test row count for dissolved level path GeoDataframe which is returned.
+        actual_row_count = len(actual_df) 
+        expected_row_count = 58
+        self.assertEqual(actual_row_count, expected_row_count)
+        
+        # -----------
+        # Test that output files exist as expected
+        if os.path.exists(params["out_stream_network"]) == False:
+            raise Exception(params["out_stream_network"] + " does not exist")
+            
+        if os.path.exists(params["out_stream_network_dissolved"]) == False:
+            raise Exception(params["out_stream_network_dissolved"] + " does not exist")
+
+        if os.path.exists(params["headwaters_outfile"]) == False:
+            raise Exception(params["headwaters_outfile"] + " does not exist")
+
+        #if os.path.exists(params["catchments_outfile"]) == False:
+        #    raise Exception(params["catchments_outfile"] + " does not exist")
+
+        if os.path.exists(params["catchments_outfile"]) == False:
+            raise Exception(params["catchments_outfile"] + " does not exist")
+
+        if os.path.exists(params["branch_inlets_outfile"]) == False:
+            raise Exception(params["branch_inlets_outfile"] + " does not exist")
+       
+        
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")        
+
+    def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self):
+    
+        '''
+        This test includes a huc that after stream orders 1 and 2 are dropped, no 
+        branches are remaining (the original huc only had 1 and 2 reaches.
+
+        # expecting an exception with a specific note
+        # NOTE: this only works for a few hucs, one is 02030201
+
+        '''
+        
+        params = self.params["dropped_stream_orders_no_branches_remaining"].copy()
+
+        with self.assertRaises(SystemExit) as se:
+            src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                    out_stream_network = params["out_stream_network"],
+                                    branch_id_attribute = params["branch_id_attribute"],
+                                    out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                    headwaters_outfile = params["headwaters_outfile"],
+                                    catchments = params["catchments"],
+                                    catchments_outfile = params["catchments_outfile"],
+                                    branch_inlets_outfile = params["branch_inlets_outfile"],
+                                    reach_id_attribute = params["reach_id_attribute"],
+                                    verbose = params["verbose"])
+
+        self.assertEqual(se.exception.code, fec.UNIT_NO_BRANCHES.value)
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")        
+
+    
+    # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
+    def test_Derive_level_paths_invalid_input_stream_network(self):
+
+        # NOTE: As we are expecting an exception, this MUST have a try catch.
+        #   for "success" tests (as above functions), they CAN NOT have a try/catch
+        try:
+        
+            params = self.params["valid_data"].copy()
+            params["in_stream_network"] = "some bad path"
+          
+            actual = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                            out_stream_network = params["out_stream_network"],
+                                            branch_id_attribute = params["branch_id_attribute"],
+                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
+                                            huc_id = params["huc_id"],
+                                            headwaters_outfile = params["headwaters_outfile"],
+                                            catchments = params["catchments"],
+                                            catchments_outfile = params["catchments_outfile"],
+                                            branch_inlets_outfile = params["branch_inlets_outfile"],
+                                            reach_id_attribute = params["reach_id_attribute"],
+                                            verbose = params["verbose"] )
+                                                           
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+            
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            
+        finally:
+            print("*************************************************************")             
+       
+       
+       
+if __name__ == '__main__':
+
+    print("*****************************")
+    print(f"Start of {os.path.basename(__file__)} tests")
+    print()
+            
+    unittest.main()
+    
+    print()    
+    print(f"End of {os.path.basename(__file__)} tests")
+    
diff --git a/unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py b/unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py
new file mode 100644
index 000000000..f838593ba
--- /dev/null
+++ b/unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import filter_catchments_and_add_attributes as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+
+
+# *************
+# Important: For this to work, when you run fim_pipeline.sh, you have to 
+# use deny_branches_dev.lst or the word "none" for the deny list arguments
+# (unit and branch deny list parameters). Key files need to exist for this unit test to work.
+class test_filter_catchments_and_add_attributes(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_filter_catchments_and_add_attributes_success(self):
+
+        '''
+        The gw_catchments_reaches_filtered_addedAttributes_<branchID>.gpkg and
+        demDerived_reaches_split_filtered_<branchid>.gpkg should not exit prior to this test.
+        If the test is successful, these file will be created.
+        '''
+
+        params = self.params["valid_data"].copy() 
+
+        # to setup the test, lets start by deleted the two expected output files to ensure
+        # that they are regenerated
+        if os.path.exists(params["output_catchments_filename"]):
+            os.remove(params["output_catchments_filename"])
+        if os.path.exists(params["output_flows_filename"]):
+            os.remove(params["output_flows_filename"])
+       
+        src.filter_catchments_and_add_attributes(input_catchments_filename = params["input_catchments_filename"],
+                                                input_flows_filename = params["input_flows_filename"],
+                                                output_catchments_filename = params["output_catchments_filename"],
+                                                output_flows_filename = params["output_flows_filename"],
+                                                wbd_filename = params["wbd_filename"],
+                                                huc_code = params["huc_code"],
+                                                drop_stream_orders = params["drop_stream_orders"])
+       
+        if os.path.exists(params["output_catchments_filename"]) == False:
+            raise Exception(params["output_catchments_filename"] + " does not exist")
+        if os.path.exists(params["output_flows_filename"]) == False:
+            raise Exception(params["output_flows_filename"] + " does not exist")
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/fim3_split_flows_unittests.py b/unit_tests/fim3_split_flows_unittests.py
new file mode 100644
index 000000000..451ec15cd
--- /dev/null
+++ b/unit_tests/fim3_split_flows_unittests.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
+import split_flows as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_split_flows(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    # MUST start with the name of "test_"
+    # This is the (or one of the) valid test expected to pass
+    def test_split_flows_success(self):
+
+        '''
+        The /data/outputs/example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
+        /data/outputs/example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
+        If the test is successful, these file will be created.
+        '''
+
+        params = self.params["valid_data"].copy() 
+
+        # to setup the test, lets start by deleted the two expected output files to ensure
+        # that they are regenerated
+        if os.path.exists(params["split_flows_filename"]):
+            os.remove(params["split_flows_filename"])
+        if os.path.exists(params["split_points_filename"]):
+            os.remove(params["split_points_filename"])
+       
+        src.split_flows(max_length = params["max_length"],
+                        slope_min = params["slope_min"],
+                        lakes_buffer_input = params["lakes_buffer_input"],
+                        flows_filename = params["flows_filename"],
+                        dem_filename = params["dem_filename"],
+                        split_flows_filename = params["split_flows_filename"],
+                        split_points_filename = params["split_points_filename"],
+                        wbd8_clp_filename = params["wbd8_clp_filename"],
+                        lakes_filename = params["lakes_filename"],
+                        drop_stream_orders = params["drop_stream_orders"])
+       
+        if os.path.exists(params["split_flows_filename"]) == False:
+            raise Exception(params["split_flows_filename"] + " does not exist")
+        if os.path.exists(params["split_points_filename"]) == False:
+            raise Exception(params["split_points_filename"] + " does not exist")
+
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/gms/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json
similarity index 80%
rename from unit_tests/gms/outputs_cleanup_params.json
rename to unit_tests/outputs_cleanup_params.json
index 456f03463..a545e6753 100644
--- a/unit_tests/gms/outputs_cleanup_params.json
+++ b/unit_tests/outputs_cleanup_params.json
@@ -2,7 +2,7 @@
 	"valid_specific_branch_data":
 	{
 		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
-		"deny_list": "/foss_fim/config/deny_gms_branches.lst",
+		"deny_list": "/foss_fim/config/deny_branches.lst",
 		"branch_id": "3246000009",
 		"verbose": true
 	},
@@ -10,7 +10,7 @@
 	"valid_directory_data":
 	{
 		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
-		"deny_list": "/foss_fim/config/deny_gms_branches.lst",
+		"deny_list": "/foss_fim/config/deny_branches.lst",
 		"branch_id": "0",
 		"verbose": true
 	},
diff --git a/unit_tests/outputs_cleanup_unittests.py b/unit_tests/outputs_cleanup_unittests.py
new file mode 100644
index 000000000..3d241f681
--- /dev/null
+++ b/unit_tests/outputs_cleanup_unittests.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+# importing python folders in other directories
+sys.path.append('/foss_fim/src/') 
+import outputs_cleanup as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_outputs_cleanup(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_remove_deny_list_files_specific_branch_success(self):
+
+        '''
+        This validates removal of files for a directory already pointing to a 
+        specific branch in a HUC
+        '''
+
+        params = self.params["valid_specific_branch_data"].copy()
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+                                   
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_files_huc_level_success(self):
+
+        '''
+        This validates removal of files for all files and subdirectory files.
+        Normally used for covering all hucs and their branch zeros but 
+        can be anything
+        '''
+
+        params = self.params["valid_directory_data"].copy()
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+                                   
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_skip_cleaning_success(self):
+
+        '''
+        This validates removal of files for all files and subdirectory files.
+        Normally used for covering all hucs and their branch zeros but 
+        can be anything
+        '''
+
+        params = self.params["skip_clean"].copy()
+        
+        src.remove_deny_list_files(src_dir = params["src_dir"],
+                                   deny_list = params["deny_list"],
+                                   branch_id = params["branch_id"],
+                                   verbose = params["verbose"])
+                                   
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_files_invalid_src_directory(self):
+
+        '''
+        Double check the src directory exists
+        '''
+
+        # NOTE: As we are expecting an exception, this MUST have a try catch.
+        #   for "success" tests (as above functions), they CAN NOT have a try/catch
+        
+        try:
+
+            params = self.params["valid_specific_branch_data"].copy()    
+            params["src_dir"] = "/data/does_no_exist"
+            
+            src.remove_deny_list_files(src_dir = params["src_dir"],
+                                    deny_list = params["deny_list"],
+                                    branch_id = params["branch_id"],
+                                    verbose = params["verbose"])
+            
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+            
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+
+    def test_remove_deny_list_files_invalid_deny_list_does_not_exist(self):
+
+        '''
+        Double check the deny list exists
+        '''
+
+        # NOTE: As we are expecting an exception, this MUST have a try catch.
+        #   for "success" tests (as above functions), they CAN NOT have a try/catch
+        try:
+
+            params = self.params["valid_specific_branch_data"].copy()    
+            params["deny_list"] = "invalid_file_name.txt"
+            
+            src.remove_deny_list_files(src_dir = params["src_dir"],
+                                    deny_list = params["deny_list"],
+                                    branch_id = params["branch_id"],
+                                    verbose = params["verbose"])
+            
+            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
+            
+        except Exception:
+            print()
+            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
+            
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+   
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    
diff --git a/unit_tests/tools/fim3_inundate_gms_params.json b/unit_tests/tools/fim3_inundate_gms_params.json
new file mode 100644
index 000000000..a75e663c4
--- /dev/null
+++ b/unit_tests/tools/fim3_inundate_gms_params.json
@@ -0,0 +1,16 @@
+{
+	"valid_data_inudation_raster_single_huc":
+	{
+		"hydrofabric_dir": "/outputs/fim_unit_test_data_do_not_remove/",
+		"forecast": "data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv",
+		"num_workers": 1,
+		"hucs": "02020005",
+	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
+		"inundation_polygon": null,
+		"depths_raster": null,
+		"verbose": true,
+		"log_file": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_logfile.txt",
+		"output_fileNames": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_file_list.csv"
+	}
+
+}
\ No newline at end of file
diff --git a/unit_tests/tools/inundate_gms_unittests.py b/unit_tests/tools/inundate_gms_unittests.py
new file mode 100644
index 000000000..ade2bfa30
--- /dev/null
+++ b/unit_tests/tools/inundate_gms_unittests.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+import json
+import warnings
+import unittest
+
+sys.path.append('/foss_fim/unit_tests/')
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
+sys.path.append('/foss_fim/tools')
+import inundate_gms as src
+
+# NOTE: This goes directly to the function.
+# Ultimately, it should emulate going through command line (not import -> direct function call)
+class test_inundate_gms(unittest.TestCase):
+
+    '''
+    Allows the params to be loaded one and used for all test methods
+    '''
+    @classmethod
+    def setUpClass(self):
+
+        warnings.simplefilter('ignore')
+        params_file_path = ut_helpers.get_params_filename(__file__)
+        with open(params_file_path) as params_file:
+            self.params = json.load(params_file)
+
+
+    def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self):
+
+        '''
+        Test for creating a gms inundation raster, not a depth raster and no
+        inundation_polygons.
+        This test is based on creating a raster based on a single huc and its branches
+        within a gms output folder.
+        '''
+
+        params = self.params["valid_data_inudation_raster_single_huc"].copy()         
+        
+        output_fileNames_df = src.Inundate_gms(hydrofabric_dir = params["hydrofabric_dir"],
+                                               forecast = params["forecast"],
+                                               num_workers = params["num_workers"],
+                                               hucs = params["hucs"],
+                                               inundation_raster = params["inundation_raster"],
+                                               inundation_polygon = params["inundation_polygon"],
+                                               depths_raster = params["depths_raster"],
+                                               verbose = params["verbose"], 
+                                               log_file = params["log_file"],
+                                               output_fileNames = params["output_fileNames"])
+
+        # check if output files df has records.
+        assert len(output_fileNames_df) > 0, "Expected as least one dataframe record"
+
+        # also check output log file and output raster. Can't... there will be multiple outputs
+        #assert os.path.exists(params["inundation_raster"]), "Inundation Raster does not exist"
+
+        assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
+       
+       
+        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
+        print("*************************************************************")
+
+        
+    # ***********************
+
+
+if __name__ == '__main__':
+
+    script_file_name = os.path.basename(__file__)
+
+    print("*****************************")
+    print(f"Start of {script_file_name} tests")
+    print()
+   
+    unittest.main()
+    
+    print()    
+    print(f"End of {script_file_name} tests")
+    

From b7cff3bee8f3fd7846eb77e958cdb7123bf51b07 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 24 Feb 2023 11:08:43 -0600
Subject: [PATCH 215/359] v4.2.0.1 Fix for missing branch_id.lst file (#827)

---
 docs/CHANGELOG.md  | 82 +++-------------------------------------------
 src/run_unit_wb.sh | 10 ++++--
 2 files changed, 12 insertions(+), 80 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d8b9770ae..c84ba1223 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,88 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-## v4.1.3.0 - 2023-02-13 - [PR#812](https://github.com/NOAA-OWP/inundation-mapping/pull/812)
-
-An update was required to adjust host name when in the AWS environment
-
-### Changes
-
-- `fim_post_processing.sh`: Added an "if isAWS" flag system based on the input command args from fim_pipeline.sh or 
-
-- `tools/calibration-db`
-    - `README.md`: Minor text correction.
-
-<br/><br/>
-
-## v4.1.2.0 - 2023-02-15 - [PR#808](https://github.com/NOAA-OWP/inundation-mapping/pull/808)
-
-Add `pytest` package and refactor existing unit tests. Update parameters to unit tests (`/unit_tests/*_params.json`) to valid paths. Add leading slash to paths in `/config/params_template.env`.
-
-### Additions
-
-- `/unit_tests`
-  - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
-  - `pyproject.toml`  - used to specify which warnings are excluded/filtered.
-  - `/gms`  
-    - `__init__.py` - needed for `pytest` command line executable to pick up tests.
-  - `/tools`
-    - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
-    - `inundate_gms_params.json` - file moved up into this directory
-    - `inundate_gms_test.py`     - file moved up into this directory
-    - `inundation_params.json`   - file moved up into this directory
-    - `inundation_test.py`       - file moved up into this directory
+## v4.2.0.1 - 2023-02-16 - [PR#827](https://github.com/NOAA-OWP/inundation-mapping/pull/827)
 
-### Removals
+FIM 4.2.0.0. was throwing errors for 14 HUCs that did not have any level paths. These are HUCs that have only stream orders 1 and 2 and are covered under branch zero, but no stream orders 3+ (no level paths).  This has now been changed to not throw an error but continue to process of the HUC.
 
-- `/unit_tests/tools/gms_tools/` directory removed, and files moved up into `/unit_tests/tools`    
- 
 ### Changes
 
-- `Pipfile` - updated to include pytest as a dependency
-- `Pipfile.lock` - updated to include pytest as a dependency
-
-- `/config`
-  - `params_template.env` - leading slash added to paths
-  
-- `/unit_tests/` - All of the `*_test.py` files were refactored to follow the `pytest` paradigm.  
-  - `*_params.json` - valid paths on `fim-dev1` provided
-  - `README.md`  - updated to include documentation on pytest.  
-  - `unit_tests_utils.py`  
-  - `__template_unittests.py` -> `__template.py` - exclude the `_test` suffix to remove from test suite. Updated example on new format for pytest.
-  - `check_unit_errors_test.py`  
-  - `clip_vectors_to_wbd_test.py`  
-  - `filter_catchments_and_add_attributes_test.py`  
-  - `rating_curve_comparison_test.py`  
-  - `shared_functions_test.py`  
-  - `split_flow_test.py`  
-  - `usgs_gage_crosswalk_test.py`  
-  - `aggregate_branch_lists_test.py`  
-  - `generate_branch_list_test.py`  
-  - `generate_branch_list_csv_test.py`  
-  - `aggregate_branch_lists_test.py`  
-  - `generate_branch_list_csv_test.py`  
-  - `generate_branch_list_test.py`  
-    - `/gms`  
-      - `derive_level_paths_test.py`  
-      - `outputs_cleanup_test.py`
-    - `/tools`
-      - `inundate_unittests.py` -> `inundation_test.py`  
-      - `inundate_gms_test.py`
-
-
-<br/><br/>
-
-## v4.1.1.0 - 2023-02-16 - [PR#809](https://github.com/NOAA-OWP/inundation-mapping/pull/809)
-
-The CatFIM code was updated to allow 1-foot interval processing across all stage-based AHPS sites ranging from action stage to 5 feet above major stage, along with restart capability for interrupted processing runs.
-
-### Changes
-
-- `tools/generate_categorical_fim.py` (all changes made here)
-    - Added try-except blocks for code that didn't allow most sites to actually get processed because it was trying to check values of some USGS-related variables that most of the sites didn't have
-    - Overwrite abilities of the different outputs for the viz team were not consistent (i.e., one of the files had the ability to be overwritten but another didn't), so that has been made consistent to disallow any overwrites of the existing final outputs for a specified output folder.
-    - The code also has the ability to restart from an interrupted run and resume processing uncompleted HUCs by first checking for a simple "complete" file for each HUC. If a HUC has that file, then it is skipped (because it already completed processing during a run for a particular output folder / run name).
-    - When a HUC is successfully processed, an empty "complete" text file is created / touched.
+- `src`
+    - `run_unit_wb.sh`: Test if branch_id.lst exists, which legitimately might not. Also a bit of text cleanup.
 
 <br/><br/>
 
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index e7d5e2d16..527b7be92 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -242,7 +242,13 @@ echo
 echo "---- Start of branch processing for $hucNumber"
 branch_processing_start_time=`date +%s`
 
-parallel --eta --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSummaryLogFile --colsep ',' -- $srcDir/process_branch.sh $runName $hucNumber :::: $branch_list_lst_file
+if [ -f $branch_list_lst_file ]; then
+    # There may not be a branch_ids.lst if there were no level paths (no stream orders 3+)
+    # but there will still be a branch zero
+    parallel --eta --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSummaryLogFile --colsep ',' -- $srcDir/process_branch.sh $runName $hucNumber :::: $branch_list_lst_file
+else
+    echo "No level paths exist with this HUC. Processing branch zero only."
+fi
 
 # -------------------
 ## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO (but using normal branch deny) ##
@@ -256,7 +262,7 @@ else
     $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_branches_list -b 0
 fi
 
-echo "---- All huc for $hucNumber branches have been now processed"
+echo "---- HUC $hucNumber - branches have now been processed"
 Calc_Duration $branch_processing_start_time
 echo
 

From 06ca69d342efb22ffdbc46a01750fa7fc6c168fe Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 24 Feb 2023 11:16:19 -0600
Subject: [PATCH 216/359] v4.2.1.0 Fix broken unit tests (#829)

---
 Pipfile.lock                                  | 358 ++++++++++--------
 data/usgs/acquire_and_preprocess_3dep_dems.py |   3 +
 docs/CHANGELOG.md                             |  32 +-
 unit_tests/README.md                          | 126 ++++++
 ...ams.json => check_unit_errors_params.json} |   2 +-
 unit_tests/check_unit_errors_test.py          |   2 +-
 unit_tests/derive_level_paths_params.json     |  18 +
 .../{gms => }/derive_level_paths_test.py      |   2 +-
 unit_tests/fim3_README.md                     | 104 -----
 unit_tests/fim3__template_unittests.py        | 108 ------
 .../fim3_check_unit_errors_unittests.py       | 278 --------------
 .../fim3_derive_level_paths_params.json       |  31 --
 .../fim3_derive_level_paths_unittests.py      | 170 ---------
 ...catchments_and_add_attributes_unittests.py |  87 -----
 unit_tests/fim3_split_flows_unittests.py      |  85 -----
 unit_tests/gms/__init__.py                    |   0
 unit_tests/{gms => }/outputs_cleanup_test.py  |   0
 unit_tests/outputs_cleanup_unittests.py       | 163 --------
 .../rating_curve_comparison_params.json       |  28 --
 unit_tests/rating_curve_comparison_test.py    |  58 +--
 unit_tests/split_flows_test.py                |   4 +-
 .../tools/fim3_inundate_gms_params.json       |  16 -
 unit_tests/tools/inundate_gms_params.json     |   2 +-
 unit_tests/tools/inundation_test.py           |   4 +
 24 files changed, 384 insertions(+), 1297 deletions(-)
 create mode 100644 unit_tests/README.md
 rename unit_tests/{fim3_check_unit_errors_params.json => check_unit_errors_params.json} (67%)
 create mode 100644 unit_tests/derive_level_paths_params.json
 rename unit_tests/{gms => }/derive_level_paths_test.py (99%)
 delete mode 100644 unit_tests/fim3_README.md
 delete mode 100644 unit_tests/fim3__template_unittests.py
 delete mode 100644 unit_tests/fim3_check_unit_errors_unittests.py
 delete mode 100644 unit_tests/fim3_derive_level_paths_params.json
 delete mode 100644 unit_tests/fim3_derive_level_paths_unittests.py
 delete mode 100644 unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py
 delete mode 100644 unit_tests/fim3_split_flows_unittests.py
 delete mode 100644 unit_tests/gms/__init__.py
 rename unit_tests/{gms => }/outputs_cleanup_test.py (100%)
 delete mode 100644 unit_tests/outputs_cleanup_unittests.py
 delete mode 100644 unit_tests/tools/fim3_inundate_gms_params.json

diff --git a/Pipfile.lock b/Pipfile.lock
index 12f51c8ee..7f1f8c827 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4'",
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -139,18 +139,19 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:159ab54ab301a213aea0dbebfe97c12909fd9180110a2b78aaa712c2d7d1a929"
+                "sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027",
+                "sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729"
             ],
             "index": "pypi",
-            "version": "==1.26.67"
+            "version": "==1.26.76"
         },
         "botocore": {
             "hashes": [
-                "sha256:0ccec4a906b6b8c7bb6bc5226509059ee9ed94d3cf1014487ef5b8e56801e6a3",
-                "sha256:424c5fa21913d85ea990928acf35622a4f2b7f0750133b96adaa8c642bb6a878"
+                "sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7",
+                "sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.29.67"
+            "version": "==1.29.76"
         },
         "certifi": {
             "hashes": [
@@ -370,7 +371,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
             "version": "==0.7.2"
         },
         "comm": {
@@ -592,11 +593,11 @@
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:7d543798b0beca10b6a01ac7cafda9f822c54db9e8376a6bf57e0cbd74d486b6",
-                "sha256:e4a96c8cc0339647ff9a5e0550d9f276fc5a01ffa276012b58ec108cfd7b8484"
+                "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6",
+                "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==5.10.2"
+            "version": "==5.12.0"
         },
         "iniconfig": {
             "hashes": [
@@ -608,27 +609,27 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:1a04bb359212e23e46adc0116ec82ea128c1e5bd532fde4fbe679787ff36f0cf",
-                "sha256:a0f8eece39cab1ee352c9b59ec67bbe44d8299f8238e4c16ff7f4cf0052d3378"
+                "sha256:430d00549b6aaf49bd0f5393150691edb1815afa62d457ee6b1a66b25cb17874",
+                "sha256:6e9213484e4ce1fb14267ee435e18f23cc3a0634e635b9fb4ed4677b84e0fdf8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.21.1"
+            "version": "==6.21.2"
         },
         "ipympl": {
             "hashes": [
-                "sha256:655604f0bf6d264cf599766950a5b26e292d107cc23e197503647e75417981cc",
-                "sha256:c865c1992248f9966fbe4b6006239ae2959b00fc7e887ae32b0bd389808f0f8b"
+                "sha256:49bab75c05673a6881d1aaec5d8ac81d4624f73d292d154c5fb7096f10236a2b",
+                "sha256:d113cd55891bafe9b27ef99b6dd111a87beb6bb2ae550c404292272103be8013"
             ],
             "index": "pypi",
-            "version": "==0.9.2"
+            "version": "==0.9.3"
         },
         "ipython": {
             "hashes": [
-                "sha256:71618e82e6d59487bea059626e7c79fb4a5b760d1510d02fab1160db6fdfa1f7",
-                "sha256:9c207b0ef2d276d1bfcfeb9a62804336abbe4b170574ea061500952319b1d78c"
+                "sha256:b13a1d6c1f5818bd388db53b7107d17454129a70de2b87481d555daede5eb49e",
+                "sha256:b38c31e8fc7eff642fc7c597061fff462537cf2314e3225a19c906b7b0d8a345"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.9.0"
+            "version": "==8.10.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -712,19 +713,19 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:47ac9f586dbcff4d79387ec264faf0fdeb5f14845fa7345fd7d1e378f8096011",
-                "sha256:c53731eb590b68839b0ce04bf46ff8c4f03278f5d9fe5c3b0f268a57cc2bd97e"
+                "sha256:be48ac6bd659cbbddb7a674cf06b3b8afbf53f228253cf58bde604c03bd487b0",
+                "sha256:ed65498bea6d876ef9d8da3e0db3dd33c5d129f5b2645f56ae03993782966bd0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.0.2"
+            "version": "==8.0.3"
         },
         "jupyter-console": {
             "hashes": [
-                "sha256:172f5335e31d600df61613a97b7f0352f2c8250bbd1092ef2d658f77249f89fb",
-                "sha256:756df7f4f60c986e7bc0172e4493d3830a7e6e75c08750bbe59c0a5403ad6dee"
+                "sha256:5931212d5cbc1f956f6fd615755b5e15f389a8eaa697288dbbe4377017615ecc",
+                "sha256:b32b5cb673a90909911988a2f1906ced005b07e8dadab6c8664d3f5de44a8191"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.4.4"
+            "version": "==6.6.1"
         },
         "jupyter-core": {
             "hashes": [
@@ -736,27 +737,27 @@
         },
         "jupyter-events": {
             "hashes": [
-                "sha256:6f7b67bf42b8a370c992187194ed02847dfa02307a7aebe9913e2d3979b9b6b8",
-                "sha256:e27ffdd6138699d47d42cb65ae6d79334ff7c0d923694381c991ce56a140f2cd"
+                "sha256:57a2749f87ba387cd1bfd9b22a0875b889237dbf2edc2121ebb22bde47036c17",
+                "sha256:9a6e9995f75d1b7146b436ea24d696ce3a35bfa8bfe45e0c33c334c79464d0b3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.5.0"
+            "version": "==0.6.3"
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:5afb8a0cdfee37d02d69bdf470ae9cbb1dee5d4788f9bc6cc8e54bd8c83fb096",
-                "sha256:854fb7d49f6b7f545d4f8354172b004dcda887ba0699def7112daf785ba3c9ce"
+                "sha256:29d6657bfb160b0e39b9030d67f33f918a188f2eba28065314a933b327fef872",
+                "sha256:b15078954120886d580e19d1746e2b62a3dc7bd082cb4716115c25fcd7061b00"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.2.1"
+            "version": "==2.3.0"
         },
         "jupyter-server-fileid": {
             "hashes": [
-                "sha256:a12209bdef4f2f9d57051b7556a089299fb9f26b501f643946854220c955be14",
-                "sha256:ac36436611b281cebbb5b9936a6f4850271bb411e13a287780a022dd0d2c3bf7"
+                "sha256:0b580a9d75cc7a7132132d6bb3faa4645e34527bff8760861e9e6de51bec7397",
+                "sha256:340e86b45875d51a60e0e93d8d3bcb609c2bc8d315ec07c003f36d561f637c0e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.6.0"
+            "version": "==0.7.0"
         },
         "jupyter-server-terminals": {
             "hashes": [
@@ -968,50 +969,50 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:01b7f521a9a73c383825813af255f8c4485d1706e4f3e2ed5ae771e4403a40ab",
-                "sha256:11011c97d62c1db7bc20509572557842dbb8c2a2ddd3dd7f20501aa1cde3e54e",
-                "sha256:1183877d008c752d7d535396096c910f4663e4b74a18313adee1213328388e1e",
-                "sha256:12f999661589981e74d793ee2f41b924b3b87d65fd929f6153bf0f30675c59b1",
-                "sha256:1c235bf9be052347373f589e018988cad177abb3f997ab1a2e2210c41562cc0c",
-                "sha256:1f4d69707b1677560cd952544ee4962f68ff07952fb9069ff8c12b56353cb8c9",
-                "sha256:1fcc4cad498533d3c393a160975acc9b36ffa224d15a6b90ae579eacee5d8579",
-                "sha256:2787a16df07370dcba385fe20cdd0cc3cfaabd3c873ddabca78c10514c799721",
-                "sha256:29f17b7f2e068dc346687cbdf80b430580bab42346625821c2d3abf3a1ec5417",
-                "sha256:38d38cb1ea1d80ee0f6351b65c6f76cad6060bbbead015720ba001348ae90f0c",
-                "sha256:3f56a7252eee8f3438447f75f5e1148a1896a2756a92285fe5d73bed6deebff4",
-                "sha256:5223affa21050fb6118353c1380c15e23aedfb436bf3e162c26dc950617a7519",
-                "sha256:57ad1aee29043163374bfa8990e1a2a10ff72c9a1bfaa92e9c46f6ea59269121",
-                "sha256:59400cc9451094b7f08cc3f321972e6e1db4cd37a978d4e8a12824bf7fd2f03b",
-                "sha256:68d94a436f62b8a861bf3ace82067a71bafb724b4e4f9133521e4d8012420dd7",
-                "sha256:6adc441b5b2098a4b904bbf9d9e92fb816fef50c55aa2ea6a823fc89b94bb838",
-                "sha256:6d81b11ede69e3a751424b98dc869c96c10256b2206bfdf41f9c720eee86844c",
-                "sha256:73b93af33634ed919e72811c9703e1105185cd3fb46d76f30b7f4cfbbd063f89",
-                "sha256:77b384cee7ab8cf75ffccbfea351a09b97564fc62d149827a5e864bec81526e5",
-                "sha256:79e501eb847f4a489eb7065bb8d3187117f65a4c02d12ea3a19d6c5bef173bcc",
-                "sha256:809119d1cba3ece3c9742eb01827fe7a0e781ea3c5d89534655a75e07979344f",
-                "sha256:80c166a0e28512e26755f69040e6bf2f946a02ffdb7c00bf6158cca3d2b146e6",
-                "sha256:81b409b2790cf8d7c1ef35920f01676d2ae7afa8241844e7aa5484fdf493a9a0",
-                "sha256:994637e2995b0342699b396a320698b07cd148bbcf2dd2fa2daba73f34dd19f2",
-                "sha256:9ceebaf73f1a3444fa11014f38b9da37ff7ea328d6efa1652241fe3777bfdab9",
-                "sha256:9fb8fb19d03abf3c5dab89a8677e62c4023632f919a62b6dd1d6d2dbf42cd9f5",
-                "sha256:acc3b1a4bddbf56fe461e36fb9ef94c2cb607fc90d24ccc650040bfcc7610de4",
-                "sha256:bbddfeb1495484351fb5b30cf5bdf06b3de0bc4626a707d29e43dfd61af2a780",
-                "sha256:bbf269e1d24bc25247095d71c7a969813f7080e2a7c6fa28931a603f747ab012",
-                "sha256:bebcff4c3ed02c6399d47329f3554193abd824d3d53b5ca02cf583bcd94470e2",
-                "sha256:c3f08df2ac4636249b8bc7a85b8b82c983bef1441595936f62c2918370ca7e1d",
-                "sha256:ca94f0362f6b6f424b555b956971dcb94b12d0368a6c3e07dc7a40d32d6d873d",
-                "sha256:d00c248ab6b92bea3f8148714837937053a083ff03b4c5e30ed37e28fc0e7e56",
-                "sha256:d2cfaa7fd62294d945b8843ea24228a27c8e7c5b48fa634f3c168153b825a21b",
-                "sha256:d5f18430f5cfa5571ab8f4c72c89af52aa0618e864c60028f11a857d62200cba",
-                "sha256:debeab8e2ab07e5e3dac33e12456da79c7e104270d2b2d1df92b9e40347cca75",
-                "sha256:dfba7057609ca9567b9704626756f0142e97ec8c5ba2c70c6e7bd1c25ef99f06",
-                "sha256:e0a64d7cc336b52e90f59e6d638ae847b966f68582a7af041e063d568e814740",
-                "sha256:eb9421c403ffd387fbe729de6d9a03005bf42faba5e8432f4e51e703215b49fc",
-                "sha256:faff486b36530a836a6b4395850322e74211cd81fc17f28b4904e1bd53668e3e",
-                "sha256:ff2aa84e74f80891e6bcf292ebb1dd57714ffbe13177642d65fee25384a30894"
+                "sha256:01681566e95b9423021b49dea6a2395c16fa054604eacb87f0f4c439750f9114",
+                "sha256:03eb2c8ff8d85da679b71e14c7c95d16d014c48e0c0bfa14db85f6cdc5c92aad",
+                "sha256:092e6abc80cdf8a95f7d1813e16c0e99ceda8d5b195a3ab859c680f3487b80a2",
+                "sha256:0a776462a4a63c0bfc9df106c15a0897aa2dbab6795c693aa366e8e283958854",
+                "sha256:0dfd4a0cbd151f6439e6d7f8dca5292839ca311e7e650596d073774847ca2e4f",
+                "sha256:111ef351f28fd823ed7177632070a6badd6f475607122bc9002a526f2502a0b5",
+                "sha256:21269450243d6928da81a9bed201f0909432a74e7d0d65db5545b9fa8a0d0223",
+                "sha256:21a8aeac39b4a795e697265d800ce52ab59bdeb6bb23082e2d971f3041074f02",
+                "sha256:21bd4033c40b95abd5b8453f036ed5aa70856e56ecbd887705c37dce007a4c21",
+                "sha256:3493b48e56468c39bd9c1532566dff3b8062952721b7521e1f394eb6791495f4",
+                "sha256:3a10428d4f8d1a478ceabd652e61a175b2fdeed4175ab48da4a7b8deb561e3fa",
+                "sha256:3d1e52365d8d5af699f04581ca191112e1d1220a9ce4386b57d807124d8b55e6",
+                "sha256:3da8b9618188346239e51f1ea6c0f8f05c6e218cfcc30b399dd7dd7f52e8bceb",
+                "sha256:4497d88c559b76da320b7759d64db442178beeea06a52dc0c629086982082dcd",
+                "sha256:46ca923e980f76d34c1c633343a72bb042d6ba690ecc649aababf5317997171d",
+                "sha256:4f640534ec2760e270801056bc0d8a10777c48b30966eef78a7c35d8590915ba",
+                "sha256:51fb664c37714cbaac69c16d6b3719f517a13c96c3f76f4caadd5a0aa7ed0329",
+                "sha256:56b7b79488209041a9bf7ddc34f1b069274489ce69e34dc63ae241d0d6b4b736",
+                "sha256:691ef1f15360e439886186d0db77b5345b24da12cbc4fc57b26c4826db4d6cab",
+                "sha256:71b751d06b2ed1fd017de512d7439c0259822864ea16731522b251a27c0b2ede",
+                "sha256:7d0dcd1a0bf8d56551e8617d6dc3881d8a1c7fb37d14e5ec12cbb293f3e6170a",
+                "sha256:827e78239292e561cfb70abf356a9d7eaf5bf6a85c97877f254009f20b892f89",
+                "sha256:8665855f3919c80551f377bc16df618ceabf3ef65270bc14b60302dce88ca9ab",
+                "sha256:8f6efd313430d7ef70a38a3276281cb2e8646b3a22b3b21eb227da20e15e6813",
+                "sha256:9d85355c48ef8b9994293eb7c00f44aa8a43cad7a297fbf0770a25cdb2244b91",
+                "sha256:a06a6c9822e80f323549c6bc9da96d4f233178212ad9a5f4ab87fd153077a507",
+                "sha256:b51ab8a5d5d3bbd4527af633a638325f492e09e45e78afdf816ef55217a09664",
+                "sha256:c0592ba57217c22987b7322df10f75ef95bc44dce781692b4b7524085de66019",
+                "sha256:c5465735eaaafd1cfaec3fed60aee776aeb3fd3992aa2e49f4635339c931d443",
+                "sha256:c849aa94ff2a70fb71f318f48a61076d1205c6013b9d3885ade7f992093ac434",
+                "sha256:c869b646489c6a94375714032e5cec08e3aa8d3f7d4e8ef2b0fb50a52b317ce6",
+                "sha256:cb52aa97b92acdee090edfb65d1cb84ea60ab38e871ba8321a10bbcebc2a3540",
+                "sha256:cf119eee4e57389fba5ac8b816934e95c256535e55f0b21628b4205737d1de85",
+                "sha256:cf6346644e8fe234dc847e6232145dac199a650d3d8025b3ef65107221584ba4",
+                "sha256:de20eb1247725a2f889173d391a6d9e7e0f2540feda24030748283108b0478ec",
+                "sha256:eb2e76cd429058d8954121c334dddfcd11a6186c6975bca61f3f248c99031b05",
+                "sha256:f336e7014889c38c59029ebacc35c59236a852e4b23836708cfd3f43d1eaeed5",
+                "sha256:f4ddac5f59e78d04b20469bc43853a8e619bb6505c7eac8ffb343ff2c516d72f",
+                "sha256:f910d924da8b9fb066b5beae0b85e34ed1b6293014892baadcf2a51da1c65807",
+                "sha256:f91d35b3ef51d29d9c661069b9e4ba431ce283ffc533b981506889e144b5b40e",
+                "sha256:fb0304c1cd802e9a25743414c887e8a7cd51d96c9ec96d388625d2cd1c137ae3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.6.3"
+            "version": "==3.7.0"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -1053,11 +1054,11 @@
         },
         "nbclassic": {
             "hashes": [
-                "sha256:32c235e1f22f4048f3b877d354c198202898797cf9c2085856827598cead001b",
-                "sha256:8e8ffce7582bb7a4baf11fa86a3d88b184e8e7df78eed4ead69f15aa4fc0e323"
+                "sha256:40f11bbcc59e8956c3d5ef132dec8e5a853e893ecf831e791d54da0d8a50d79d",
+                "sha256:6403a996562dadefa7fee9c49e17b663b5fd508241de5df655b90011cf3342d9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.5.1"
+            "version": "==0.5.2"
         },
         "nbclient": {
             "hashes": [
@@ -1415,11 +1416,10 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63",
-                "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305"
+                "sha256:6a2948ec427dfcc7c983027b1044b355db6aaa8be374f54ad2015471f7d81c5b"
             ],
-            "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.36"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.0.37"
         },
         "psutil": {
             "hashes": [
@@ -1626,11 +1626,11 @@
         },
         "python-json-logger": {
             "hashes": [
-                "sha256:3b03487b14eb9e4f77e4fc2a023358b5394b82fd89cecf5586259baed57d8c6f",
-                "sha256:764d762175f99fcc4630bd4853b09632acb60a6224acb27ce08cd70f0b1b81bd"
+                "sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c",
+                "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.0.4"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.0.7"
         },
         "pytz": {
             "hashes": [
@@ -1813,7 +1813,7 @@
                 "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa",
                 "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4'",
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==2.28.2"
         },
         "rfc3339-validator": {
@@ -1861,30 +1861,30 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:0490dc499fe23e4be35b8b6dd1e60a4a34f0c4adb30ac671e6332446b3cbbb5a",
-                "sha256:0ab2a58064836632e2cec31ca197d3695c86b066bc4818052b3f5381bfd2a728",
-                "sha256:151f066fe7d6653c3ffefd489497b8fa66d7316e3e0d0c0f7ff6acca1b802809",
-                "sha256:16ba05d3d1b9f2141004f3f36888e05894a525960b07f4c2bfc0456b955a00be",
-                "sha256:27e548276b5a88b51212b61f6dda49a24acf5d770dff940bd372b3f7ced8c6c2",
-                "sha256:2ad449db4e0820e4b42baccefc98ec772ad7818dcbc9e28b85aa05a536b0f1a2",
-                "sha256:2f9ea0a37aca111a407cb98aa4e8dfde6e5d9333bae06dfa5d938d14c80bb5c3",
-                "sha256:38bfbd18dcc69eeb589811e77fae552fa923067fdfbb2e171c9eac749885f210",
-                "sha256:3afcbddb4488ac950ce1147e7580178b333a29cd43524c689b2e3543a080a2c8",
-                "sha256:42ab8b9e7dc1ebe248e55f54eea5307b6ab15011a7883367af48dd781d1312e4",
-                "sha256:441cab2166607c82e6d7a8683779cb89ba0f475b983c7e4ab88f3668e268c143",
-                "sha256:4bd0e3278126bc882d10414436e58fa3f1eca0aa88b534fcbf80ed47e854f46c",
-                "sha256:4df25a28bd22c990b22129d3c637fd5c3be4b7c94f975dca909d8bab3309b694",
-                "sha256:5cd7a30970c29d9768a7164f564d1fbf2842bfc77b7d114a99bc32703ce0bf48",
-                "sha256:6e4497e5142f325a5423ff5fda2fff5b5d953da028637ff7c704378c8c284ea7",
-                "sha256:6faf86ef7717891195ae0537e48da7524d30bc3b828b30c9b115d04ea42f076f",
-                "sha256:954ff69d2d1bf666b794c1d7216e0a746c9d9289096a64ab3355a17c7c59db54",
-                "sha256:9b878c671655864af59c108c20e4da1e796154bd78c0ed6bb02bc41c84625686",
-                "sha256:b901b423c91281a974f6cd1c36f5c6c523e665b5a6d5e80fcb2334e14670eefd",
-                "sha256:c8b3cbc636a87a89b770c6afc999baa6bcbb01691b5ccbbc1b1791c7c0a07540",
-                "sha256:e096b062d2efdea57f972d232358cb068413dc54eec4f24158bcbb5cb8bddfd8"
+                "sha256:049a8bbf0ad95277ffba9b3b7d23e5369cc39e66406d60422c8cfef40ccc8415",
+                "sha256:07c3457ce0b3ad5124f98a86533106b643dd811dd61b548e78cf4c8786652f6f",
+                "sha256:0f1564ea217e82c1bbe75ddf7285ba0709ecd503f048cb1236ae9995f64217bd",
+                "sha256:1553b5dcddd64ba9a0d95355e63fe6c3fc303a8fd77c7bc91e77d61363f7433f",
+                "sha256:15a35c4242ec5f292c3dd364a7c71a61be87a3d4ddcc693372813c0b73c9af1d",
+                "sha256:1b4735d6c28aad3cdcf52117e0e91d6b39acd4272f3f5cd9907c24ee931ad601",
+                "sha256:2cf9dfb80a7b4589ba4c40ce7588986d6d5cebc5457cad2c2880f6bc2d42f3a5",
+                "sha256:39becb03541f9e58243f4197584286e339029e8908c46f7221abeea4b749fa88",
+                "sha256:43b8e0bcb877faf0abfb613d51026cd5cc78918e9530e375727bf0625c82788f",
+                "sha256:4b3f429188c66603a1a5c549fb414e4d3bdc2a24792e061ffbd607d3d75fd84e",
+                "sha256:4c0ff64b06b10e35215abce517252b375e580a6125fd5fdf6421b98efbefb2d2",
+                "sha256:51af417a000d2dbe1ec6c372dfe688e041a7084da4fdd350aeb139bd3fb55353",
+                "sha256:5678f88c68ea866ed9ebe3a989091088553ba12c6090244fdae3e467b1139c35",
+                "sha256:79c8e5a6c6ffaf3a2262ef1be1e108a035cf4f05c14df56057b64acc5bebffb6",
+                "sha256:7ff7f37b1bf4417baca958d254e8e2875d0cc23aaadbe65b3d5b3077b0eb23ea",
+                "sha256:aaea0a6be54462ec027de54fca511540980d1e9eea68b2d5c1dbfe084797be35",
+                "sha256:bce5869c8d68cf383ce240e44c1d9ae7c06078a9396df68ce88a1230f93a30c1",
+                "sha256:cd9f1027ff30d90618914a64ca9b1a77a431159df0e2a195d8a9e8a04c78abf9",
+                "sha256:d925fa1c81b772882aa55bcc10bf88324dadb66ff85d548c71515f6689c6dac5",
+                "sha256:e7354fd7527a4b0377ce55f286805b34e8c54b91be865bac273f527e1b839019",
+                "sha256:fae8a7b898c42dffe3f7361c40d5952b6bf32d10c4569098d276b4c547905ee1"
             ],
             "markers": "python_version < '3.12' and python_version >= '3.8'",
-            "version": "==1.10.0"
+            "version": "==1.10.1"
         },
         "seaborn": {
             "hashes": [
@@ -1903,11 +1903,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:16ccf598aab3b506593c17378473978908a2734d7336755a8769b480906bec1c",
-                "sha256:b440ee5f7e607bb8c9de15259dba2583dd41a38879a7abc1d43a71c59524da48"
+                "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330",
+                "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==67.2.0"
+            "version": "==67.4.0"
         },
         "shapely": {
             "hashes": [
@@ -2038,11 +2038,11 @@
         },
         "soupsieve": {
             "hashes": [
-                "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759",
-                "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"
+                "sha256:49e5368c2cda80ee7e84da9dbe3e110b70a4575f196efb74e51b94549d921955",
+                "sha256:e28dba9ca6c7c00173e34e4ba57448f0688bb681b7c5e8bf4971daafc093d69a"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.3.2.post1"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.4"
         },
         "stack-data": {
             "hashes": [
@@ -2200,43 +2200,74 @@
         },
         "y-py": {
             "hashes": [
-                "sha256:00c81e71abbbeab3b29ccc806176677a11397e637b2fd60c1fede24b0a7b4f32",
-                "sha256:0a14b9293a94a75d4659001fb6d7dba9c5386c8b752968f32a573830645c4e14",
-                "sha256:0e6ec7744ec1614fbcd5fdd9cfe2b1b7755c8fc9e2766782d0c3960b1f71fb4f",
-                "sha256:0edfa91505e3de0276daa44cf1f00a059c0cfc460e3fbb3f5188fefab30358ea",
-                "sha256:1258dd7ab1a03895b493bc6683d888adff8521b0d44d17b963be432395555f43",
-                "sha256:17dac9b53cc7c1c9845d6fb028e8f59b44da673f0dda9bb4a73eb86944181ce0",
-                "sha256:23285cb734a8036dbba07323a96f858e040b31fde888fc19daf2904ec62524f6",
-                "sha256:36558e6e117e70c7af47e5b07614abff481dd549d97394759ff513398add6363",
-                "sha256:3d318e922a8a193e2b7761594f0aa90a2e6d559dfdc96db4812995050887e9c5",
-                "sha256:3ff13618977bd24031387e32ba3c90e795992a5a8bac55707ff58e4c0839301d",
-                "sha256:400691bdf3be9ee950d73806a18fa81128b8ed3c74fde6b12f9d26ef311df877",
-                "sha256:410bfd166c8fcd6384655471e02449abc6051f0510d75954c3fa7f0aa43cf8aa",
-                "sha256:44a1e40582386309b873c6ab8a24e47f02fe2482359a85cf4b0a65353c00702c",
-                "sha256:52f03aa7bda1e862560fec043cc8580633dff811949d7df86ee3aa313d89ee78",
-                "sha256:6cd577e7a1b777c78127f6d89707b91b9a5935c632ebfa1791624d11c462a874",
-                "sha256:743b69a1920fa38696961d04e2921621f5794945b95e43797110830732c9f092",
-                "sha256:77e7a3e73e0fd71b016247a2d7dda21eeccb3ac53783c264fe13ea7801497084",
-                "sha256:822c04279e14a178c3726c9ccb16b447c5d03ebedb3410028f4aeb399f482d1f",
-                "sha256:8b0abfffed321b63850bbafb6223815db7336e86068f8ac841c2dbf5bf6326cd",
-                "sha256:90bcfd4de15063b41cb451911f67c0be2dafe701e66cf59c54c549ec309a3d48",
-                "sha256:95969ae8f9158cc8ba73ae631213e7030eac080330cb4aaa58a033932ee807a4",
-                "sha256:9be9a3981f4ed0983339b883d7eb1414a253ddcf248fb3b420e4d82da38b42e2",
-                "sha256:a15182ffc4d68659aea49374f7a7057e708dac8f49a9a5d8966482cef9c1cba2",
-                "sha256:a7668feef5734a769a5cff120b9bf79338b35dedbd5e6dfa99ac0c70640e11d4",
-                "sha256:a8ccd6baf2ac27e92a0c5bc05ee03528d897f9b86e2ae8fc27b04dcf0c462655",
-                "sha256:b36cd7b8d43782d932e7476b5381c13ecc02dcaabb988fdf920c2c30c8c93148",
-                "sha256:b4ccbd7e9e9ffab54bd5e1debe933dee495cf93783e1bfd0811ede9527a6c12d",
-                "sha256:df1fe3118d66127eacca14916d398099ff1016e2f003a5dab5f3fba4b449e7eb",
-                "sha256:df64934fe00bf10c1afb1a1b6ecf487b71fb233678faf075f4c0a9f384f99c70",
-                "sha256:e3b7d4150fb775a97d5454ca0e3eb2527ba83ec0e67a9244e4061abb7d91b16f",
-                "sha256:e66d9145d122339502ddb80d5aaed10592310c776b07f19f28e3391521cd19ec",
-                "sha256:f222bab71d8d3df9a40b2e5ab3a767d734c6ce11998e9a30a02fb83ab3e090b3",
-                "sha256:f4f1dd115c9493b4dc054521f85c69da15b9a036227199706757adf14013e3a7",
-                "sha256:f5b5ea683cb1cebb30c2a0a04a8b15cf01eccf4e1aee9406c97c8830d5913e64",
-                "sha256:f61d173bbe980b35f5720d94a9537bc09aec8621f8bba61d5f4db236679940d1"
-            ],
-            "version": "==0.5.5"
+                "sha256:05f805b58422d5d7c8e7e8e2141d1c3cac4daaa4557ae6a9b84b141fe8d6289e",
+                "sha256:065f90501cf008375d70be6ce72dd41745e09d088f0b545f5f914d2c3f04f7ae",
+                "sha256:0c0e333c20b0a6ce4a5851203d45898ab93f16426c342420b931e190c5b71d3d",
+                "sha256:13b9d2959d9a26536b6ad118fb026ff19bd79da52e4addf6f3a562e7c01d516e",
+                "sha256:1906f13e8d5ebfbd9c7948f57bc6f6f53b451b19c99350f42a0f648147a8acfe",
+                "sha256:1f54625b9ed4e787872c45d3044dcfd04c0da4258d9914f3d32308830b35246c",
+                "sha256:202b2a3e42e0a1eaedee26f8a3bc73cd9f994c4c2b15511ea56b9838178eb380",
+                "sha256:2532ea5aefb223fd688c93860199d348a7601d814aac9e8784d816314588ddeb",
+                "sha256:25637e3d011ca6f877a24f3083ff2549d1d619406d7e8a1455c445527205046c",
+                "sha256:2692c808bf28f797f8d693f45dc86563ac3b1626579f67ce9546dca69644d687",
+                "sha256:27c1e9a866146d250e9e16d99fe22a40c82f5b592ab85da97e5679fc3841c7ce",
+                "sha256:2ffebe5e62cbfee6e24593927dedba77dc13ac4cfb9c822074ab566b1fb63d59",
+                "sha256:50cfa0532bcee27edb8c64743b49570e28bb76a00cd384ead1d84b6f052d9368",
+                "sha256:55098440e32339c2dc3d652fb36bb77a4927dee5fd4ab0cb1fe12fdd163fd4f5",
+                "sha256:5dbd8d177ec7b9fef4a7b6d22eb2f8d5606fd5aac31cf2eab0dc18f0b3504c7c",
+                "sha256:63ef8e5b76cd54578a7fd5f72d8c698d9ccd7c555c7900ebfd38a24d397c3b15",
+                "sha256:73200c59bb253b880825466717941ac57267f2f685b053e183183cb6fe82874d",
+                "sha256:7353af0e9c1f42fbf0ab340e253eeb333d58c890fa91d3eadb1b9adaf9336732",
+                "sha256:742c486d5b792c4ad76e09426161302edddca85efe826fa01dcee50907326cd7",
+                "sha256:753aaae817d658a1e9d271663439d8e83d9d8effa45590ecdcadc600c7cf77e3",
+                "sha256:76b3480e7037ac9390c450e2aff9e46e2c9e61520c0d88afe228110ec728adc5",
+                "sha256:800e73d2110b97a74c52db2c8ce03a78e96f0d66a7e0c87d8254170a67c2db0e",
+                "sha256:85585e669d7679126e4a04e4bc0a063a641175a74eecfe47539e8da3e5b1da6e",
+                "sha256:8d4dfc276f988175baaa4ab321c3321a16ce33db3356c9bc5f4dea0db3de55aa",
+                "sha256:91be189fae8ba242528333e266e38d65cae3d9a09fe45867fab8578a3ddf2ea2",
+                "sha256:9484a3fc33f812234e58a5ee834b42bb0a628054d61b5c06c323aa56c12e557d",
+                "sha256:9513ae81fcc805671ae134c4c7421ca322acf92ce8b33817e1775ea8c0176973",
+                "sha256:95d13b38c9055d607565b77cbae12e2bf0c1671c5cb8f2ee2e1230d41d2d6d34",
+                "sha256:9983e99e3a61452b39ffce98206c7e4c6d260f4e917c8fe53fb54aaf25df89a3",
+                "sha256:9a59603cf42c20d02ee5add2e3d0ce48e89c480a2a02f642fb77f142c4f37958",
+                "sha256:a57d81260e048caacf43a2f851766687f53e8a8356df6947fb0eee7336a7e2de",
+                "sha256:a7977eeaceaeb0dfffcc5643c985c337ebc33a0b1d792ae0a9b1331cdd97366f",
+                "sha256:add793f5f5c7c7a3eb1b09ffc771bdaae10a0bd482a370bf696b83f8dee8d1b4",
+                "sha256:ae82a6d9cbaff8cb7505e81b5b7f9cd7756bb7e7110aef7914375fe56b012a90",
+                "sha256:af6df5ec1d66ee2d962026635d60e84ad35fc01b2a1e36b993360c0ce60ae349",
+                "sha256:afa9a11aa2880dd8689894f3269b653e6d3bd1956963d5329be9a5bf021dab62",
+                "sha256:b0ed760e6aa5316227a0ba2d5d29634a4ef2d72c8bc55169ac01664e17e4b536",
+                "sha256:b44473bb32217c78e18db66f497f6c8be33e339bab5f52398bb2468c904d5140",
+                "sha256:b67dad339f9b6701f74ff7a6e901c7909eca4eea02cf955b28d87a42650bd1be",
+                "sha256:bc9052a814e8b7ec756371a191f38de68b956437e0bb429c2dd503e658f298f9",
+                "sha256:c1f5f287cc7ae127ed6a2fb1546e631b316a41d087d7d2db9caa3e5f59906dcf",
+                "sha256:c3ae6d22b7cc599220a26b06da6ead9fd582eea5fdb6273b06fa3f060d0a26a7",
+                "sha256:c42f3a6cd20153925b00c49af855a3277989d411bb8ea849095be943ee160821",
+                "sha256:c7ca64a2a97f708569dcabd55865915943e30267bf6d26c4d212d005951efe62",
+                "sha256:caf9b1feb69379d424a1d3d7c899b8e0389a3fb3131d39c3c03dcc3d4a93dbdc",
+                "sha256:cb68445414940efe547291340e91604c7b8379b60822678ef29f4fc2a0e11c62",
+                "sha256:cc8e5f38842a4b043c9592bfa9a740147ddb8fac2d7a5b7bf6d52466c090ec23",
+                "sha256:cd6f373dbf592ad83aaf95c16abebc8678928e49bd509ebd593259e1908345ae",
+                "sha256:d2da2a9e28dceab4832945a745cad507579f52b4d0c9e2f54ae156eb56875861",
+                "sha256:d373c6bb8e21d5f7ec0833b76fa1ab480086ada602ef5bbf4724a25a21a00b6a",
+                "sha256:d722d6a27230c1f395535da5cee6a9a16497c6343afd262c846090075c083009",
+                "sha256:db1ac7f2d1862eb4c448cf76183399d555a63dbe2452bafecb1c2f691e36d687",
+                "sha256:df78a0409dca11554a4b6442d7a8e61f762c3cfc78d55d98352392869a6b9ae0",
+                "sha256:e30fe2491d095c6d695a2c96257967fd3e2497f0f777030c8492d03c18d46e2a",
+                "sha256:e370ce076781adea161b04d2f666e8b4f89bc7e8927ef842fbb0283d3bfa73e0",
+                "sha256:ecd3cb0d13ac92e7b9235d1024dba9af0788161246f12dcf1f635d634ccb206a",
+                "sha256:ed0fd5265905cc7e23709479bc152d69f4972dec32fa322d20cb77f749707e78",
+                "sha256:f6d87d0c2e87990bc00c049742d36a5dbbb1510949459af17198728890ee748a",
+                "sha256:f7434c77cd23592973ed63341b8d337e6aebaba5ed40d7f22e2d43dfd0c3a56e",
+                "sha256:f8b67ae37af8aac6160fda66c0f73bcdf65c06da9022eb76192c3fc45cfab994",
+                "sha256:f8f238144a302f17eb26b122cad9382fcff5ec6653b8a562130b9a5e44010098",
+                "sha256:fa685f7e43ce490dfb1e392ac48f584b75cd21f05dc526c160d15308236ce8a0",
+                "sha256:fce5feb57f6231376eb10d1fb68c60da106ffa0b520b3129471c466eff0304cc",
+                "sha256:fdafb93bfd5532b13a53c4090675bcd31724160017ecc73e492dc1211bc0377a",
+                "sha256:fe70d0134fe2115c08866f0cac0eb5c0788093872b5026eb438a74e1ebafd659",
+                "sha256:ff3ddedaa95284f4f22a92b362f658f3d92f272d8c0fa009051bd5490c4d5a04"
+            ],
+            "version": "==0.5.9"
         },
         "ypy-websocket": {
             "hashes": [
@@ -2248,11 +2279,11 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:6c4fe274b8f85ec73c37a8e4e3fa00df9fb9335da96fb789e3b96b318e5097b3",
-                "sha256:a3cac813d40993596b39ea9e93a18e8a2076d5c378b8bc88ec32ab264e04ad02"
+                "sha256:188834565033387710d046e3fe96acfc9b5e86cbca7f39ff69cf21a4128198b7",
+                "sha256:9e5421e176ef5ab4c0ad896624e87a7b2f07aca746c9b2aa305952800cb8eecb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.12.1"
+            "version": "==3.14.0"
         }
     },
     "develop": {
@@ -2287,11 +2318,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:71618e82e6d59487bea059626e7c79fb4a5b760d1510d02fab1160db6fdfa1f7",
-                "sha256:9c207b0ef2d276d1bfcfeb9a62804336abbe4b170574ea061500952319b1d78c"
+                "sha256:b13a1d6c1f5818bd388db53b7107d17454129a70de2b87481d555daede5eb49e",
+                "sha256:b38c31e8fc7eff642fc7c597061fff462537cf2314e3225a19c906b7b0d8a345"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.9.0"
+            "version": "==8.10.0"
         },
         "jedi": {
             "hashes": [
@@ -2334,11 +2365,10 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63",
-                "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305"
+                "sha256:6a2948ec427dfcc7c983027b1044b355db6aaa8be374f54ad2015471f7d81c5b"
             ],
-            "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.36"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.0.37"
         },
         "ptyprocess": {
             "hashes": [
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index 1323c8572..d69b0eea3 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -20,6 +20,9 @@
 from utils.shared_functions import FIM_Helpers as fh
 
 # local constants (until changed to input param)
+# This URL is part of a series of vrt data available from USGS via an S3 Bucket.
+# for more info see: "http://prd-tnm.s3.amazonaws.com/index.html?prefix=StagedProducts/Elevation/". The odd folder numbering is
+# a translation of arc seconds with 13m  being 1/3 arc second or 10 meters.
 __USGS_3DEP_10M_VRT_URL = r'/vsicurl/https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/TIFF/USGS_Seamless_DEM_13.vrt'  # 10m = 13 (1/3 arc second)
 
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c84ba1223..9801a3c1f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,31 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.2.1.0 - 2023-02-21 - [PR#829](https://github.com/NOAA-OWP/inundation-mapping/pull/829)
+
+During the merge from remove-fim3 PR into dev, merge conflicts were discovered in the unit_tests folders and files. Attempts to fix them at that time failed, so some files were removed, other renamed, other edited to get the merge to work.  Here are the fixes to put the unit tests system back to par.
+
+Note: some unit tests are now temporarily disabled due to dependencies on other files / folders which may not exist in other environments.
+
+Also.. the Changelog.md was broken and is being restored here.
+
+Also.. a minor text addition was added to the acquire_and_preprocess_3dep_dems.py files (not directly related to this PR)
+
+For file changes directly related to unit_test folder and it's file, please see [PR#829](https://github.com/NOAA-OWP/inundation-mapping/pull/829)
+
+Other file changes:
+
+### Changes
+- `Pipfile.lock` : rebuilt and updated as a safety pre-caution.
+- `docs`
+    - `CHANGELOG.md`: additions to this file for FIM 4.2.0.0 were not merged correctly.  (re-added just below in the 4.2.0.0 section)
+- `data`
+    - `usgs`
+        - `acquire_and_preprocess_3dep_dems.py`: Added text on data input URL source.
+        
+<br/><br/>
+
 ## v4.2.0.1 - 2023-02-16 - [PR#827](https://github.com/NOAA-OWP/inundation-mapping/pull/827)
 
 FIM 4.2.0.0. was throwing errors for 14 HUCs that did not have any level paths. These are HUCs that have only stream orders 1 and 2 and are covered under branch zero, but no stream orders 3+ (no level paths).  This has now been changed to not throw an error but continue to process of the HUC.
@@ -51,7 +76,7 @@ There are a lot of duplicate explanations for some of the changes, so here is a
    - `deny_branches.lst` :  renamed from `deny_gms_branches.lst`
    - `deny_unit.lst`  : renamed from `deny_gms_unit.lst`
    - `params_template.env` : see desc 1
- 
+
 - `data`
     - `nws`
         - `preprocess_ahps_nws.py`:   Added deprecation note: If reused, it needs review and/or upgrades.
@@ -85,7 +110,7 @@ There are a lot of duplicate explanations for some of the changes, so here is a
      - `usgs_gage_unit_setup.py` : see desc 1
      - `utils`
          - `fim_enums.py` : see desc 1
-         
+
 - `tools`
     - `combine_crosswalk_tables.py` : see desc 2
     - `compare_ms_and_non_ms_metrics.py` : see desc 2
@@ -107,7 +132,7 @@ There are a lot of duplicate explanations for some of the changes, so here is a
     - `plots.py` : see desc 2
     - `run_test_case.py`:  see desc 1
     - `synthesize_test_cases.py`: see desc 1
-    
+
 - `unit_tests`
     - `README.md`: see desc 1
     - `__template_unittests.py`: see desc 1
@@ -120,6 +145,7 @@ There are a lot of duplicate explanations for some of the changes, so here is a
         - `inundate_gms_params.json` and `inundate_gms_unittests.py`: see desc 1 and desc 2
 
 <br/><br/>
+
 ## v4.1.3.0 - 2023-02-13 - [PR#812](https://github.com/NOAA-OWP/inundation-mapping/pull/812)
 
 An update was required to adjust host name when in the AWS environment
diff --git a/unit_tests/README.md b/unit_tests/README.md
new file mode 100644
index 000000000..1bf2435c1
--- /dev/null
+++ b/unit_tests/README.md
@@ -0,0 +1,126 @@
+## Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
+
+Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
+
+#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
+
+# This folder (`/unit_tests`) holds files for unit testing python files
+
+## Creating unit tests
+
+For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and an accompanying json paramerters file. 
+
+The files should be named following FIM convention:
+
+{source py file name}_test.py ->     `derive_level_paths_test.py`  
+{source py file name}_params.json -> `derive_level_paths_params.json`
+
+
+## Tips to create a new json file for a new python unit test file.
+
+There are multiple way to figure out a set of default json parameters for the new unit test file. 
+
+One way is to use the incoming arg parser. Most python files include the code block of ` __name__ == '__main__':`, followed by external arg parsing (`args = vars(parser.parse_args()`). 
+* Add a `print(args)` or similar, and get all the values including keys as output.  
+* Copy that into an editor being used to create the json file.  
+* Add a line break after every comma.  
+* Find/replace all single quotes to double quotes then cleanup the left tab formatting.
+
+
+## Running unit tests
+
+Start a docker container as you normally would for any development. 
+```bash 
+docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
+```
+Example:
+```bash 
+docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
+```
+
+For unit tests to work, you need to run the following (if not already in place).
+Notice a modified branch "deny_branch_unittests.lst"  (special for unittests)
+
+Here are the params and args you need if you need to re-run unit and branch
+
+```bash
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 05030104" -bd /foss_fim/config/deny_branch_unittests.lst -ud None -j 1 -o
+```
+
+**NOTICE: the deny file used for fim_pipeline.sh, has a special one for unittests `deny_branch_unittests.lst`.
+
+If you need to run inundation tests, fun the following:
+
+```bash
+python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_not_remove -jh 1 -jb 1 -m /outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
+```
+### If you'd like to test the whole unit test suite:
+```
+pytest /foss_fim/unit_tests
+```
+
+This is not 100% stable, as accurate paths for the parameters `.json` files are not included in this repository, are not uniform accross machines, and are subject to change. 
+ 
+### If you want to test just one unit test (from the root terminal window):
+
+```bash
+pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py 
+						or  
+pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py
+```
+
+### If you'd like to run a particular test, you can, for example:
+```
+pytest -v -s -k test_append_id_to_file_name_single_identifier_success
+```
+
+If one test case is choosen, it will scan all of the test files, and scan for the method (test case) specified. 
+
+## Key Notes for creating new unit tests
+1) All test functions must start with the phrase `test_`. That is how pytest picks them up. The rest of the function name does not have to match the pattern of `function_name_being_tested` but should. Further, the rest of the function name should say what the test is about, ie) `_failed_input_path`.  ie) `test_{some_function_name_from_the_source_code_file}_failed_input_path`. It is fine that the function names get very long (common in the industry).
+
+2) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
+
+3) As of now, you can not control the order that unit tests are run within a unit test file. 
+
+4) There must be at least one associated `{original py file name}_params.json` file per unit test.
+
+5) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to fully pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass.
+
+6) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called `valid_data`, if one does not already exist. Generally, the individual unit tests, will call the `valid_data` node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie: major different test conditions).
+
+7) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
+
+8) One Python file = one `{original py file name}_test.py` file.
+
+9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist). 
+ie:
+```bash
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o
+```
+but ours would be:
+```bash 
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o
+```
+
+## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
+
+The `pyproject.toml` file has been added, which contains the build system requirements of Python projects.  This file used to specify which warnings are disabled to pass our unit tests. 
+
+A `__init__.py` file has been added to the subdirectory of `/tools` in order for the `pytest` command run in the `/unit_tests` to pick up the tests in those directories as well.
+
+Luckily, `pytest` works well with The Python Standard Library `unittest`. This made the migration of previous unit tests using `unittest` over to `pytest` quite simple. The caveat is that our current unit tests employ elements of both libraries. A full transition to `pytest` will ideally take place at a future date.
+
+## Testing for failing conditions
+- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for ONLY one variable value change. A "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
+
+- It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
+
+- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass. 
+
+An example is in `unit_tests/Derive_level_paths_test.py` -> `test_Derive_level_paths_invalid_input_stream_network` (function). This example gives you the pattern implemented in Pytest.
+
+## Future Enhancements
+1) Full transition to the `pytest` library, removing classes of `unittest.TestCase` and taking full advantage of available code re-use patterns offered through `pytest`.  
+
+2) Over time, it is expected that python files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each python file which makes unit testing harder and less specific. Generally function will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample .tif or small .gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: The files `/derive_level_paths_test.py` and `clip_vectors_to_wbd_test.py` are not complete as they do not yet test all output from a method.
diff --git a/unit_tests/fim3_check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json
similarity index 67%
rename from unit_tests/fim3_check_unit_errors_params.json
rename to unit_tests/check_unit_errors_params.json
index bf89913a1..d97ec466f 100644
--- a/unit_tests/fim3_check_unit_errors_params.json
+++ b/unit_tests/check_unit_errors_params.json
@@ -1,7 +1,7 @@
 {
 	"valid_data":
 	{
-		"fim_dir": "/outputs/check_errors_example_unit_tests"
+		"fim_dir": "/outputs/fim_unit_test_data_do_not_remove"
 	},
 
 	"invalid_path":
diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py
index b2229220a..d10080e96 100644
--- a/unit_tests/check_unit_errors_test.py
+++ b/unit_tests/check_unit_errors_test.py
@@ -182,7 +182,7 @@ def __remove_temp_unit_errors_folder(self, output_folder):
             Note.. it is possible the temp folder does not exist,
             but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
         Input:
-            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
+            output_folder: the root output folder (ie. /outputs/fim_unit_test_data_do_not_remove/)
         Returns:
             nothing
         '''
diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json
new file mode 100644
index 000000000..a465efc1a
--- /dev/null
+++ b/unit_tests/derive_level_paths_params.json
@@ -0,0 +1,18 @@
+{
+	"valid_data":
+	{
+		"outputRunDataDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
+		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"branch_id_attribute": "levpa_id",
+		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"reach_id_attribute": "ID",
+		"verbose": true
+	}	
+}
+
+
diff --git a/unit_tests/gms/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py
similarity index 99%
rename from unit_tests/gms/derive_level_paths_test.py
rename to unit_tests/derive_level_paths_test.py
index b9d4b6e5a..8f395e561 100644
--- a/unit_tests/gms/derive_level_paths_test.py
+++ b/unit_tests/derive_level_paths_test.py
@@ -9,7 +9,7 @@
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # # importing python folders in other direcories
-sys.path.append('/foss_fim/src/gms/')
+sys.path.append('/foss_fim/src/')
 import derive_level_paths as src
 import stream_branches
 from utils.fim_enums import FIM_exit_codes as fec
diff --git a/unit_tests/fim3_README.md b/unit_tests/fim3_README.md
deleted file mode 100644
index b7f37d1d3..000000000
--- a/unit_tests/fim3_README.md
+++ /dev/null
@@ -1,104 +0,0 @@
-## Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
-
-Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
-
-#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
-
-### This folder and files are for unit testing python files
-
-## Creating unit tests
-
-For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and it's json parms file. 
-
-The files should be named following FIM convention:
-
-{source py file name}_unittests.py     ie) derive_level_paths_unittests.py
-{source py file name}_params.json      ie) derive_level_paths_params.json
-
-
-## Tips to creating a new json file for a new unit test file.
-
-There are multiple way to figure out a set of default json parameters for the new unit test file. 
-
-One way is to use the incoming arg parser. Most py files, they include the code block of " __name__ == '__main__':", followed by external arg parsing, ie) the args = vars(parser.parse_args()). 
-	- Add a print(args) or similar, and get all the values including keys as output.
-	- Copy that into an editor being used to create the json file.
-	- Add a line break after every comma.
-	- Find/replace all single quotes to double quotes then cleanup the left tab formatting.
-
-
-## Running unit tests
-
-Start a docker container as you normally would for any development. ie) docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
-	- ie) docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
-
-For unit tests to work, you need to run the following (if not already in place).
-Notice a modified branch "deny_branch_unittests.lst"  (special for unittests)
-
-Here are the params and args you need if you need to re-run unit and branch
-
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 02030201 05030104" -bd /foss_fim/config/deny_branch_unittests.lst -ud None -j 1 -o
-
-**NOTICE: the deny file used for fim_pipeline... its a special one for unittests `deny_branch_unittests.lst`.
-
-If you need to run inundation tests, fun the following:
-
-python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_not_remove -jh 1 -jb 1 -m /outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
-
-If you want to test just one unit test, here is an example:
-At the root terminal window, run:  python ./foss_fim/unit_tests/derive_level_paths_unittests.py  or python ./foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
-(replace with your own script and path name)
-
-## Key Notes for creating new unit tests
-1) All test functions must start with the phrase "test_". That is how the unit test engine picks it up. The rest of the function name does not have to match the pattern of {function name being tested} but should. Further, the rest of the function name should say what the test is about, ie) _failed_input_path.  ie) test_{some_function_name_from_the_source_code_file}_failed_input_path. It is fine that the function names get very long (common in the industry).
-
-2) The output for a selected "unittest" import engine can be ugly and hard to read. It sometimes mixed outputs from multiple unit test functions simulataneously instead of keeping all output together for a given unit test. We will try to make this better later.
-
-3) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
-
-4) You can not control the order that unit tests are run within a unit test file. (UnitTest engine limitation)
-
-5) There must be at one "{original py file name}_params.json" file.
-
-6) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to full pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass. Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data"
-
-7) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called "valid_data", if one does not already exist. Generally, the individual unit tests, will call the "valid_data" node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie.. major different test conditions).
-
-8) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
-
-9) One py file = one "{original py file name}_unittests.py" file.
-
-10) Sometimes you may want to run a full successful "happy path" version through fim_pipeline.sh (or similar), to get all of the files you need in place to do your testing. However.. you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the "-d" parameter (denylist). ie) Normally:  fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o, but ours would be 
-gms_run_unit.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o. 
-
-## Future Enhancements
-1) We can automate triggers on these files for things like checking triggers or an single global "run_all_unittest" script, but for now.. its one offs.
-
-2) Better output from the unit tests including verbosity output control
-
-3) Over time, it is expected that python code files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each code file which makes unit testing harder and less specific. Generally for each function in a python code file will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample tif or small gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: Our first two files of derive_level_paths_unittests and clip_vectors_to_wbd_unittests are not complete as they do not yet test all output from a method.
-
-
-## testing for failing conditions
-- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for `ONLY one variable value change`. Aka.. a "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
-
-- It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
-
-- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass. 
-
-An example is in unit_tests/Derive_level_paths_unittests.py -> test_Derive_level_paths_invalid_input_stream_network (function). It is incomplete but give you the pattern.
-
-We have almost no "assert"s yet, but most unit test usually have one or more "assert" test. See https://docs.python.org/3/library/unittest.html for more details.
-
-## Unit tests currently available
-python3 /foss_fim/unit_tests/derive_level_paths_unittests.py
-python3 /foss_fim/unit_tests/tools/inundate_unittests.py
-python3 /foss_fim/unit_tests/tools/inundate_gms_unittests.py
-python3 /foss_fim/unit_tests/clip_vectors_to_wbd_unittests.py
-python3 /foss_fim/unit_tests/filter_catchments_and_add_attributes_unittests.py
-python3 /foss_fim/unit_tests/rating_curve_comparison_unittests.py
-python3 /foss_fim/unit_tests/shared_functions_unittests.py
-python3 /foss_fim/unit_tests/split_flows_unittests.py
-python3 /foss_fim/unit_tests/usgs_gage_crosswalk_unittests.py
-
-
diff --git a/unit_tests/fim3__template_unittests.py b/unit_tests/fim3__template_unittests.py
deleted file mode 100644
index 1ba2981ce..000000000
--- a/unit_tests/fim3__template_unittests.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import <Your original source python file name> as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_<Your original source python file name>(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
-    def test_<method name you are about to test>_success(self):
-
-        '''
-        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
-          If there is no return value from the method, please say so.>
-        
-        Dev Notes: (which can be removed after you make this file)
-            Remember... You need to validate the method output if there is any. However, if you have time, it
-            is also recommended that you validate other outputs such as writing or updating file on the file 
-            system, aka: Does the expected file exist. Don't worry about its contents.
-
-        '''
-
-        #global params_file
-        
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-        
-       
-        # for now we are happy if no exceptions are thrown.
-        
-        # < See the readme.md, clip_vectors_to_wbd_unittests.py or derive_level_paths_unittests.py 
-        # for examples.>
-        # Replace this stub example with your own.
-        # Try to use the same order to make it easier.
-        # Remember, if the method accepts **params, then you can sent that in here as well.
-        #   ie: my_py_class.my_method(** params)
-        
-        src.subset_vector_layers(hucCode = params["hucCode"],
-                                    nwm_streams_filename = params["nwm_streams"],
-                                    etc, etc for each param)
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    # ***** REMOVE THIS BLOCK IF YOU ARE NOT USING IT ***
-    #       EXAMPLE SUCCESSFUL FAIL    
-    # 
-    #def test_subset_vector_layers_fail_invalid_stream_path(self):
-    #    '''
-    #    Notes about what the test is and the expected results (or expected exception if applicable)
-    #    '''
-    #   #global params_file
-        
-    #   params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-
-    #   params["nwm_streams"] = "some bad path"
-
-    #   clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
-    #                                            nwm_streams_filename = params["nwm_streams"],
-    #                                            etc, etc for each param)
-    #   
-       
-    #    print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-    #    print("*************************************************************")
-        
-    # ***********************
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/fim3_check_unit_errors_unittests.py b/unit_tests/fim3_check_unit_errors_unittests.py
deleted file mode 100644
index 04ebdfda7..000000000
--- a/unit_tests/fim3_check_unit_errors_unittests.py
+++ /dev/null
@@ -1,278 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import math
-import os
-import shutil
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import check_unit_errors as src
-from utils.fim_enums import FIM_exit_codes
-
-from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
-                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
-
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_check_unit_errors(unittest.TestCase):
-
-
-    # CURRENT 
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        try:        
-            params_file_path = ut_helpers.get_params_filename(__file__)
-        except FileNotFoundError as ex:
-            print(f"params file not found. ({ex}). Check pathing and file name convention.")
-            sys.exit(1)
-
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    '''
-    To make most of these unit test work, we have to do the following:
-    - rename the current unit_errors directory (if it exists)
-    - empty the unit_errors directory
-    - create a bunch of dummy files in it
-    - perform the unit test
-    - delete the unit_errors directory
-    - rename the original unit_test folder back to unit_tests
-    '''
-
-    def test_check_unit_errors_success_below_min_errors(self):
-
-        # Expecting no errors
-        # test to ensure the number of dummy files is less than the
-        # overall min number of error files.
-
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
-        
-            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
-                                                                            num_dummy_files_reqd)
-            expected_output = 0
-            actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-            
-            err_msg = "actual output does not match expected output"
-            self.assertEqual(expected_output, actual_output, err_msg)
-            
-            if (ue_folder_existed):
-                self.__remove_temp_unit_errors_folder(params["fim_dir"])
-            
-            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-            print("*************************************************************")
-            
-        except BaseException as ex:
-            
-            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
-            
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-            # re throw for the unit test engine            
-            raise ex
-
-
-    def test_check_unit_errors_fail_above_min_errors(self):
-
-        # test to ensure the number of dummy files is more than the
-        # overall min number of error files.
-
-        # Expecting sys.exit of 62
-        # we do expect this to fail and if it fails, it is successful
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD + 1
-        
-            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-            src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-            
-            raise Exception("Test Failed: expected sys.exit error but did not receive it")
-            
-        except BaseException as ex:
-           
-            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
-                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            else:
-                print("ex value is")
-                print(ex)
-                raise Exception("Expected a system exit code but did not receive it")
-          
-        finally:
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-
-    def test_check_unit_errors_success_above_percent_errors(self):
-
-        # Expecting no errors
-        # test to ensure the number of dummy files is more than the
-        # overall min number of error files.
-        
-        # we do expect this not to to fail as it is greater than 10 errors
-        # but below the percent threshhold
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-            
-            print(f"num_dummy_files_reqd is {num_dummy_files_reqd}")
-            ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
-                                                                            num_dummy_files_reqd)
-            
-            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
-            expected_output = 0
-            actual_output = src.check_unit_errors(params["fim_dir"], num_total_units)
-            
-            err_msg = "actual output does not match expected output"
-            self.assertEqual(expected_output, actual_output, err_msg)
-            
-            if (ue_folder_existed):
-                self.__remove_temp_unit_errors_folder(params["fim_dir"])
-            
-            print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-            print("*************************************************************")
-            
-        except BaseException as ex:
-            
-            print(f"Test Failed: {inspect.currentframe().f_code.co_name}")
-            
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-            # re throw for the unit test engine            
-            raise ex
-
-
-    def test_check_unit_errors_fail_below_percent_errors(self):
-
-        # Expecting sys.exit of 62
-        # test to ensure the number of dummy files is more than the
-        # overall min number of error files.
-        
-        # we do expect this to fail as it is greater than 10 errors
-        # AND below the percent threshhold (more percent errors than the threshold)
-        params = self.params["valid_data"].copy()
-        
-        try:
-            num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-            
-            self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-            
-            num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) - 10
-            print(f" is {num_total_units}")
-
-            src.check_unit_errors(params["fim_dir"], num_total_units)
-            raise Exception("Test Failed: expected sys.exit error but did not receive it")
-                    
-        except BaseException as ex:
-           
-            if (str(FIM_exit_codes.EXCESS_UNIT_ERRORS.value) in str(ex)):
-                print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-                print("*************************************************************")                             
-            else:
-                print("ex value is")
-                print(ex)
-                raise Exception("Expected a system exit code but did not receive it")
-          
-        finally:
-            # no matter what, we have to put the unit_errors folders back to the way
-            # is was, so try again if we have too
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-            
-
-    def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files):
-        '''
-        Process:
-            We want to preserve the original unit_errors folder if it exists,
-            so we wil rename it.
-            Then we will make a new unit_errors folder and fill it with a bunch of 
-            dummy files.
-            A dummy file for non_zero_exit_codes.log will also be created.
-        Input:
-            output_folder: the root output folder (ie. /outputs/check_errors_example_unit_tests/)
-            number_of_files: how many dummy files to create
-        Returns:
-            True if the 'unit_errors' folder did original exist and needs to be renamed back.
-            False if the 'unit_errors' folder never existed in the first place.
-        '''
-        
-        ue_folder_preexists = False
-        
-        if (not os.path.isdir(output_folder)):
-            raise Exception(f"unit test root folder of {output_folder} does not exist")
-        
-        ue_folder = os.path.join(output_folder, "unit_errors")
-        temp_ue_folder = ue_folder + "_temp"
-        if (os.path.isdir(ue_folder)):
-            ue_folder_preexists = True
-            os.rename(ue_folder, temp_ue_folder)
-        
-        os.mkdir(ue_folder)
-        
-        for i in range(0, number_of_files):
-            file_name = "sample_" + str(i) + ".txt"
-            file_path = os.path.join(ue_folder, file_name)
-            with open(file_path, 'w') as fp:
-                pass
-        
-        return ue_folder_preexists
-        
-        
-    def __remove_temp_unit_errors_folder(self, output_folder):
-        '''
-        Process:
-            We want to preserve the original unit_errors folder if it exists,
-            so we will delete our temp unit test version of 'unit_errors', and rename
-            the original back to 'unit_errors'
-            Note.. it is possible the temp folder does not exist,
-            but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
-        Input:
-            output_folder: the root output folder (ie. /outputs/check_errors_example_unit_tests/)
-        Returns:
-            nothing
-        '''
-
-        ue_folder = os.path.join(output_folder, "unit_errors")        
-        if (os.path.isdir(ue_folder)):
-            shutil.rmtree(ue_folder)       
-        
-        temp_ue_folder = ue_folder + "_temp"        
-        if (os.path.isdir(temp_ue_folder)):
-            os.rename(temp_ue_folder, ue_folder)
-      
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/fim3_derive_level_paths_params.json b/unit_tests/fim3_derive_level_paths_params.json
deleted file mode 100644
index 97e5e1a8b..000000000
--- a/unit_tests/fim3_derive_level_paths_params.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-	"valid_data":
-	{
-		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
-		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
-		"reach_id_attribute": "ID",
-		"verbose": true
-	},
-	
-	"dropped_stream_orders_no_branches_remaining":
-	{
-		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams.gpkg",
-		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
-		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/02030201/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
-		"reach_id_attribute": "ID",
-		"verbose": true
-	}	
-}
-
-
diff --git a/unit_tests/fim3_derive_level_paths_unittests.py b/unit_tests/fim3_derive_level_paths_unittests.py
deleted file mode 100644
index 5f68a72d8..000000000
--- a/unit_tests/fim3_derive_level_paths_unittests.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-# importing python folders in other direcories
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other direcories
-sys.path.append('/foss_fim/src/')
-import derive_level_paths as src
-import stream_branches
-from utils.fim_enums import FIM_exit_codes as fec
-
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_Derive_level_paths(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-    
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    def test_Derive_level_paths_success_all_params(self):
-    
-        '''
-        This test includes all params with many optional parms being set to the default value of the function
-        '''
-      
-        params = self.params["valid_data"].copy()
-
-        # Function Notes:
-        # huc_ids no longer used, so it is not submitted
-        # other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__, so I will
-        # skip them here.
-        # returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                            out_stream_network = params["out_stream_network"],
-                                            branch_id_attribute = params["branch_id_attribute"],
-                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                            headwaters_outfile = params["headwaters_outfile"],
-                                            catchments = params["catchments"],
-                                            catchments_outfile = params["catchments_outfile"],
-                                            branch_inlets_outfile = params["branch_inlets_outfile"],
-                                            reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"])
-
-        # -----------
-        # test data type being return is as expected. Downstream code might to know that type
-        self.assertIsInstance(actual_df, stream_branches.StreamNetwork)
-        
-        # -----------
-        #**** NOTE: Based on 05030104         
-        # Test row count for dissolved level path GeoDataframe which is returned.
-        actual_row_count = len(actual_df) 
-        expected_row_count = 58
-        self.assertEqual(actual_row_count, expected_row_count)
-        
-        # -----------
-        # Test that output files exist as expected
-        if os.path.exists(params["out_stream_network"]) == False:
-            raise Exception(params["out_stream_network"] + " does not exist")
-            
-        if os.path.exists(params["out_stream_network_dissolved"]) == False:
-            raise Exception(params["out_stream_network_dissolved"] + " does not exist")
-
-        if os.path.exists(params["headwaters_outfile"]) == False:
-            raise Exception(params["headwaters_outfile"] + " does not exist")
-
-        #if os.path.exists(params["catchments_outfile"]) == False:
-        #    raise Exception(params["catchments_outfile"] + " does not exist")
-
-        if os.path.exists(params["catchments_outfile"]) == False:
-            raise Exception(params["catchments_outfile"] + " does not exist")
-
-        if os.path.exists(params["branch_inlets_outfile"]) == False:
-            raise Exception(params["branch_inlets_outfile"] + " does not exist")
-       
-        
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")        
-
-    def test_Derive_level_paths_success_drop_low_stream_orders_no_branches_left(self):
-    
-        '''
-        This test includes a huc that after stream orders 1 and 2 are dropped, no 
-        branches are remaining (the original huc only had 1 and 2 reaches.
-
-        # expecting an exception with a specific note
-        # NOTE: this only works for a few hucs, one is 02030201
-
-        '''
-        
-        params = self.params["dropped_stream_orders_no_branches_remaining"].copy()
-
-        with self.assertRaises(SystemExit) as se:
-            src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                    out_stream_network = params["out_stream_network"],
-                                    branch_id_attribute = params["branch_id_attribute"],
-                                    out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                    headwaters_outfile = params["headwaters_outfile"],
-                                    catchments = params["catchments"],
-                                    catchments_outfile = params["catchments_outfile"],
-                                    branch_inlets_outfile = params["branch_inlets_outfile"],
-                                    reach_id_attribute = params["reach_id_attribute"],
-                                    verbose = params["verbose"])
-
-        self.assertEqual(se.exception.code, fec.UNIT_NO_BRANCHES.value)
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")        
-
-    
-    # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
-    def test_Derive_level_paths_invalid_input_stream_network(self):
-
-        # NOTE: As we are expecting an exception, this MUST have a try catch.
-        #   for "success" tests (as above functions), they CAN NOT have a try/catch
-        try:
-        
-            params = self.params["valid_data"].copy()
-            params["in_stream_network"] = "some bad path"
-          
-            actual = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                            out_stream_network = params["out_stream_network"],
-                                            branch_id_attribute = params["branch_id_attribute"],
-                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                            huc_id = params["huc_id"],
-                                            headwaters_outfile = params["headwaters_outfile"],
-                                            catchments = params["catchments"],
-                                            catchments_outfile = params["catchments_outfile"],
-                                            branch_inlets_outfile = params["branch_inlets_outfile"],
-                                            reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"] )
-                                                           
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-            
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        finally:
-            print("*************************************************************")             
-       
-       
-       
-if __name__ == '__main__':
-
-    print("*****************************")
-    print(f"Start of {os.path.basename(__file__)} tests")
-    print()
-            
-    unittest.main()
-    
-    print()    
-    print(f"End of {os.path.basename(__file__)} tests")
-    
diff --git a/unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py b/unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py
deleted file mode 100644
index f838593ba..000000000
--- a/unit_tests/fim3_filter_catchments_and_add_attributes_unittests.py
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import filter_catchments_and_add_attributes as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-
-
-# *************
-# Important: For this to work, when you run fim_pipeline.sh, you have to 
-# use deny_branches_dev.lst or the word "none" for the deny list arguments
-# (unit and branch deny list parameters). Key files need to exist for this unit test to work.
-class test_filter_catchments_and_add_attributes(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
-    def test_filter_catchments_and_add_attributes_success(self):
-
-        '''
-        The gw_catchments_reaches_filtered_addedAttributes_<branchID>.gpkg and
-        demDerived_reaches_split_filtered_<branchid>.gpkg should not exit prior to this test.
-        If the test is successful, these file will be created.
-        '''
-
-        params = self.params["valid_data"].copy() 
-
-        # to setup the test, lets start by deleted the two expected output files to ensure
-        # that they are regenerated
-        if os.path.exists(params["output_catchments_filename"]):
-            os.remove(params["output_catchments_filename"])
-        if os.path.exists(params["output_flows_filename"]):
-            os.remove(params["output_flows_filename"])
-       
-        src.filter_catchments_and_add_attributes(input_catchments_filename = params["input_catchments_filename"],
-                                                input_flows_filename = params["input_flows_filename"],
-                                                output_catchments_filename = params["output_catchments_filename"],
-                                                output_flows_filename = params["output_flows_filename"],
-                                                wbd_filename = params["wbd_filename"],
-                                                huc_code = params["huc_code"],
-                                                drop_stream_orders = params["drop_stream_orders"])
-       
-        if os.path.exists(params["output_catchments_filename"]) == False:
-            raise Exception(params["output_catchments_filename"] + " does not exist")
-        if os.path.exists(params["output_flows_filename"]) == False:
-            raise Exception(params["output_flows_filename"] + " does not exist")
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/fim3_split_flows_unittests.py b/unit_tests/fim3_split_flows_unittests.py
deleted file mode 100644
index 451ec15cd..000000000
--- a/unit_tests/fim3_split_flows_unittests.py
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/')   # *** update your folder path here if required ***
-import split_flows as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_split_flows(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
-    def test_split_flows_success(self):
-
-        '''
-        The /data/outputs/example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
-        /data/outputs/example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
-        If the test is successful, these file will be created.
-        '''
-
-        params = self.params["valid_data"].copy() 
-
-        # to setup the test, lets start by deleted the two expected output files to ensure
-        # that they are regenerated
-        if os.path.exists(params["split_flows_filename"]):
-            os.remove(params["split_flows_filename"])
-        if os.path.exists(params["split_points_filename"]):
-            os.remove(params["split_points_filename"])
-       
-        src.split_flows(max_length = params["max_length"],
-                        slope_min = params["slope_min"],
-                        lakes_buffer_input = params["lakes_buffer_input"],
-                        flows_filename = params["flows_filename"],
-                        dem_filename = params["dem_filename"],
-                        split_flows_filename = params["split_flows_filename"],
-                        split_points_filename = params["split_points_filename"],
-                        wbd8_clp_filename = params["wbd8_clp_filename"],
-                        lakes_filename = params["lakes_filename"],
-                        drop_stream_orders = params["drop_stream_orders"])
-       
-        if os.path.exists(params["split_flows_filename"]) == False:
-            raise Exception(params["split_flows_filename"] + " does not exist")
-        if os.path.exists(params["split_points_filename"]) == False:
-            raise Exception(params["split_points_filename"] + " does not exist")
-
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/gms/__init__.py b/unit_tests/gms/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/unit_tests/gms/outputs_cleanup_test.py b/unit_tests/outputs_cleanup_test.py
similarity index 100%
rename from unit_tests/gms/outputs_cleanup_test.py
rename to unit_tests/outputs_cleanup_test.py
diff --git a/unit_tests/outputs_cleanup_unittests.py b/unit_tests/outputs_cleanup_unittests.py
deleted file mode 100644
index 3d241f681..000000000
--- a/unit_tests/outputs_cleanup_unittests.py
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-# importing python folders in other directories
-sys.path.append('/foss_fim/src/') 
-import outputs_cleanup as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_outputs_cleanup(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    def test_remove_deny_list_files_specific_branch_success(self):
-
-        '''
-        This validates removal of files for a directory already pointing to a 
-        specific branch in a HUC
-        '''
-
-        params = self.params["valid_specific_branch_data"].copy()
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
-                                   
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_files_huc_level_success(self):
-
-        '''
-        This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but 
-        can be anything
-        '''
-
-        params = self.params["valid_directory_data"].copy()
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
-                                   
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_skip_cleaning_success(self):
-
-        '''
-        This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but 
-        can be anything
-        '''
-
-        params = self.params["skip_clean"].copy()
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
-                                   
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_files_invalid_src_directory(self):
-
-        '''
-        Double check the src directory exists
-        '''
-
-        # NOTE: As we are expecting an exception, this MUST have a try catch.
-        #   for "success" tests (as above functions), they CAN NOT have a try/catch
-        
-        try:
-
-            params = self.params["valid_specific_branch_data"].copy()    
-            params["src_dir"] = "/data/does_no_exist"
-            
-            src.remove_deny_list_files(src_dir = params["src_dir"],
-                                    deny_list = params["deny_list"],
-                                    branch_id = params["branch_id"],
-                                    verbose = params["verbose"])
-            
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-            
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-
-    def test_remove_deny_list_files_invalid_deny_list_does_not_exist(self):
-
-        '''
-        Double check the deny list exists
-        '''
-
-        # NOTE: As we are expecting an exception, this MUST have a try catch.
-        #   for "success" tests (as above functions), they CAN NOT have a try/catch
-        try:
-
-            params = self.params["valid_specific_branch_data"].copy()    
-            params["deny_list"] = "invalid_file_name.txt"
-            
-            src.remove_deny_list_files(src_dir = params["src_dir"],
-                                    deny_list = params["deny_list"],
-                                    branch_id = params["branch_id"],
-                                    verbose = params["verbose"])
-            
-            raise AssertionError("Fail = excepted a thrown exception but did not get it but was received. Unit Test has 'failed'")
-            
-        except Exception:
-            print()
-            print(f"Test Success (failed as expected): {inspect.currentframe().f_code.co_name}")
-            
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-   
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
index e233b3520..24949dcb6 100644
--- a/unit_tests/rating_curve_comparison_params.json
+++ b/unit_tests/rating_curve_comparison_params.json
@@ -1,20 +1,6 @@
 {
 	"valid_data":
 	{
-		"01010004":
-		[
-			"/data/outputs/fim_unit_test_data_do_not_remove/01010004/usgs_elev_table.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/01010004/branches",
-			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_01010004.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_01010004.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_01010004.png",
-			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
-			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
-			"01010004",
-			false,
-			false
-		],
 		"02020005":
 		[
 			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_elev_table.csv",
@@ -28,20 +14,6 @@
 			"02020005",
 			false,
 			false
-		],
-		"02030103":
-		[
-			"/data/outputs/fim_unit_test_data_do_not_remove/02030103/usgs_elev_table.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/02030103/branches",
-			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02030103.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02030103.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02030103.png",
-			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
-			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
-			"02030103",
-			false, 
-			false
 		]
 	}
 }
\ No newline at end of file
diff --git a/unit_tests/rating_curve_comparison_test.py b/unit_tests/rating_curve_comparison_test.py
index 058662376..f1f4ec3d3 100644
--- a/unit_tests/rating_curve_comparison_test.py
+++ b/unit_tests/rating_curve_comparison_test.py
@@ -3,6 +3,7 @@
 import os
 import json
 import unittest
+import pytest
 
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 from rating_curve_comparison import generate_rating_curve_metrics
@@ -21,33 +22,9 @@ def setUpClass(self):
             self.params = json.load(params_file)
 
 # Test Cases
-
-    def test_generate_rating_curve_metrics_01010004_success(self):
-
-        '''
-        We are testing whether a .png file was created for a FIM-USGS rating curve comparison, 
-        for HUC 0101004 using the `generate_rating_curve_metrics` function.
-        The 5th index (parameter) for each HUC in `rating_curve_comparison_params.json` specifies
-        the FIM-USGS rating curve comparison .png filepath.
-        '''
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-
-        _indiv_huc_params = params["01010004"]
-
-        # To setup the test, lets start by deleted the expected output file to ensure
-        # that it is regenerated.
-        if os.path.exists(_indiv_huc_params[5]):
-            os.remove(_indiv_huc_params[5])
-        
-        # Test that the file was deleted
-        assert os.path.exists(_indiv_huc_params[5]) == False
-
-        generate_rating_curve_metrics(_indiv_huc_params)
-
-        # Test that the file was created by generate_rating_curve_metrics       
-        assert os.path.exists(_indiv_huc_params[5]) == True
-        
-
+     
+    @pytest.mark.skip(reason="rating_curve has invalid pathing based on files "\
+        "that are normally not in a runtime folder. This test will disabled for now.")
     def test_generate_rating_curve_metrics_02020005_success(self):
 
         '''
@@ -74,30 +51,3 @@ def test_generate_rating_curve_metrics_02020005_success(self):
         # Test that the file was created by generate_rating_curve_metrics       
         assert os.path.exists(_indiv_huc_params[5]) == True
 
-
-    def test_generate_rating_curve_metrics_02030103_success(self):
-
-        '''
-        We are testing whether a .png file was created for a FIM-USGS rating curve comparison, 
-        for HUC 02030103 using the `generate_rating_curve_metrics` function.
-        The 5th index (parameter) for each HUC in `rating_curve_comparison_params.json` specifies
-        the FIM-USGS rating curve comparison .png filepath.
-        '''
-        
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-        
-        _indiv_huc_params = params["02030103"]
-
-        # To setup the test, lets start by deleted the expected output file to ensure
-        # that it is regenerated.
-        if os.path.exists(_indiv_huc_params[5]):
-            os.remove(_indiv_huc_params[5])
-        
-        # Test that the file was deleted
-        assert os.path.exists(_indiv_huc_params[5]) == False
-
-        generate_rating_curve_metrics(_indiv_huc_params)
-
-        # Test that the file was created by generate_rating_curve_metrics       
-        assert os.path.exists(_indiv_huc_params[5]) == True
-
diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py
index 84d986a47..6b94fd2c7 100644
--- a/unit_tests/split_flows_test.py
+++ b/unit_tests/split_flows_test.py
@@ -27,8 +27,8 @@ def setUpClass(self):
     def test_split_flows_success(self):
 
         '''
-        The /data/outputs/gms_example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
-        /data/outputs/gms_example_unit_tests/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
+        The /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
+        /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
         If the test is successful, these files will be created.
         '''
 
diff --git a/unit_tests/tools/fim3_inundate_gms_params.json b/unit_tests/tools/fim3_inundate_gms_params.json
deleted file mode 100644
index a75e663c4..000000000
--- a/unit_tests/tools/fim3_inundate_gms_params.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-	"valid_data_inudation_raster_single_huc":
-	{
-		"hydrofabric_dir": "/outputs/fim_unit_test_data_do_not_remove/",
-		"forecast": "data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv",
-		"num_workers": 1,
-		"hucs": "02020005",
-	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
-		"inundation_polygon": null,
-		"depths_raster": null,
-		"verbose": true,
-		"log_file": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_logfile.txt",
-		"output_fileNames": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_file_list.csv"
-	}
-
-}
\ No newline at end of file
diff --git a/unit_tests/tools/inundate_gms_params.json b/unit_tests/tools/inundate_gms_params.json
index a2fd24d72..236614d6e 100644
--- a/unit_tests/tools/inundate_gms_params.json
+++ b/unit_tests/tools/inundate_gms_params.json
@@ -2,7 +2,7 @@
 	"valid_data_inudation_raster_single_huc":
 	{
 		"hydrofabric_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
-		"forecast": "/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv",
+		"forecast": "/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv",
 		"num_workers": 1,
 		"hucs": "02020005, 05030104",
 	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
diff --git a/unit_tests/tools/inundation_test.py b/unit_tests/tools/inundation_test.py
index dc44207dc..3cb4c6517 100644
--- a/unit_tests/tools/inundation_test.py
+++ b/unit_tests/tools/inundation_test.py
@@ -25,6 +25,7 @@ def setUpClass(self):
 
 # Test Cases:
 
+    @pytest.mark.skip(reason="Inundate_gms will be rebuilt in the future, so this test will be left broken.")
     def test_inundate_create_inundation_raster_single_branch_success(self):
         '''
         Test for creating a inundation branch raster, no depth raster and no
@@ -53,6 +54,9 @@ def test_inundate_create_inundation_raster_single_branch_success(self):
                                                            src_table = params["src_table"],
                                                            quiet = params["quiet"])
         
+        print("in_rasters")
+        print(in_rasters)
+        
         assert len(in_rasters) == 1, "Expected exactly one inundation raster path records"
         assert depth_rasters[0] == None, "Expected no depth raster path records"
         assert in_polys[0] == None, "Expected no inundation_polys records"

From ad87a7485876e2ef82967afe0671d8e16330849f Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 24 Feb 2023 12:33:51 -0500
Subject: [PATCH 217/359] v4.3.0.0 Replace GRASS with Whitebox (#814)

---
 Dockerfile                               |  14 +-
 Pipfile                                  |   4 +-
 Pipfile.lock                             |  18 +-
 docs/CHANGELOG.md                        |  23 +-
 src/adjust_thalweg_lateral.py            |  78 ++---
 src/agreedem.py                          | 381 ++++++++++++-----------
 src/delineate_hydros_and_produce_HAND.sh |   2 +-
 src/mask_dem.py                          |   1 -
 src/r_grow_distance.py                   | 101 ------
 src/run_unit_wb.sh                       |   2 +-
 src/unique_pixel_and_allocation.py       |  41 ++-
 11 files changed, 305 insertions(+), 360 deletions(-)
 delete mode 100755 src/r_grow_distance.py

diff --git a/Dockerfile b/Dockerfile
index 7cd1f264e..34c01455d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -52,7 +52,7 @@ ARG dataDir=/data
 ARG projectDir=/foss_fim
 ARG depDir=/dependencies
 ENV inputDataDir=$dataDir/inputs
-ENV outputDataDir=$dataDir/outputs
+ENV outputDataDir=/outputs
 ENV srcDir=$projectDir/src
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
@@ -70,14 +70,12 @@ COPY --from=builder $depDir $depDir
 RUN apt update --fix-missing 
 RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync
 
-RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
-
 RUN apt auto-remove
 
 ## adding AWS CLI (for bash) ##
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
     unzip awscliv2.zip && \
-	./aws/install
+    ./aws/install
 
 ## adding environment variables for numba and python ##
 ENV LC_ALL=C.UTF-8
@@ -92,6 +90,14 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 COPY Pipfile .
 COPY Pipfile.lock .
 RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
+#RUN pip3 install pipenv==2022.4.8 && pipenv install --system --deploy --ignore-pipfile (too slow to
+#     leave out shapely,pygeos at this time. Likely better after upgrading)
+
+# TEMP FIX as neither shapely or Shapely is staying in the pip list. If we manually add
+# it with pip (not pipenv), it works. Notice case for Shapely versus shapely. 
+# This temp fix works for now until we can reconsile the shapely package,
+# pygeos, geopanda's and possibly others (coming soon)
+RUN pip install shapely==1.7.0
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
 ADD ./src/entrypoint.sh /
diff --git a/Pipfile b/Pipfile
index e62db0d1e..a84391528 100755
--- a/Pipfile
+++ b/Pipfile
@@ -17,8 +17,6 @@ rasterio = "==1.1.5"
 rasterstats = "==0.15.0"
 richdem = "==0.3.4"
 tqdm = "==4.48.0"
-Shapely = "==1.7.0"
-grass-session = "==0.5"
 seaborn = "==0.11.0"
 python-dotenv = "*"
 natsort = "*"
@@ -34,6 +32,8 @@ jupyter = "*"
 jupyterlab = "*"
 ipympl = "*"
 pytest = "*"
+whitebox = "*"
+shapely = "==1.7.0"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 7f1f8c827..03826cafa 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f05b5609b9137d06ed14e862c850d362334b299b913f52899fb234449fbb231a"
+            "sha256": "4ef68f0d8f3ea64518ebd604722e5283603cf6ffe3401320f3db3be365224ac0"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -551,14 +551,6 @@
             "index": "pypi",
             "version": "==0.8.1"
         },
-        "grass-session": {
-            "hashes": [
-                "sha256:7155314535790145da8e2e31b0d20cd2be91477d54083a738b5c319164e7f03b",
-                "sha256:ce03a53e28cc14bc7fff91482e83ed4f174a1325732c4333ed183dd15de39f8d"
-            ],
-            "index": "pypi",
-            "version": "==0.5"
-        },
         "h5py": {
             "hashes": [
                 "sha256:0b0f002f5f341afe7d3d7e15198e80d9021da24a4d182d88068d79bfc91fba86",
@@ -2182,6 +2174,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.5.1"
         },
+        "whitebox": {
+            "hashes": [
+                "sha256:0455cf727233adb53abe1bbe221fe0ac597d3c2609a3f7e74c1243f8ffb4b190",
+                "sha256:8afc78a2f59e1b58e1cae2509f9489b7fc9f2105a5ee7d84a4005e5905dcea01"
+            ],
+            "index": "pypi",
+            "version": "==2.2.0"
+        },
         "widgetsnbextension": {
             "hashes": [
                 "sha256:003f716d930d385be3fd9de42dd9bf008e30053f73bddde235d14fbeaeff19af",
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9801a3c1f..e8cb443ef 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,28 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.0.0 - 2023-02-15 - [PR#814](https://github.com/NOAA-OWP/inundation-mapping/pull/814)
+
+Replaces GRASS with Whitebox. This addresses several issues, including Windows permissions and GRASS projection issues. Whitebox also has a slight performance benefit over GRASS.
+
+### Removals
+
+- `src/r_grow_distance.py`: Deletes file
+
+### Changes
+
+- `Dockerfile`: Removes GRASS, update `$outputDataDir` from `/data/outputs` to `/outputs`
+- `Pipfile` and `Pipfile.lock`: Adds Whitebox and removes GRASS
+- `src/`
+    - `agreedem.py`: Removes `r_grow_distance`; refactors to use with context and removes redundant raster reads.
+    - `adjust_lateral_thalweg.py` and `agreedem.py`: Refactors to use `with` context and removes redundant raster reads
+    - `unique_pixel_and_allocation.py`: Replaces GRASS with Whitebox and remove `r_grow_distance`
+    - `gms/`
+        - `delineate_hydros_and_produce_HAND.sh` and `run_by_unit.sh`: Removes GRASS parameter
+        - `mask_dem.py`: Removes unnecessary line
+
+<br/><br/>
+
 ## v4.2.1.0 - 2023-02-21 - [PR#829](https://github.com/NOAA-OWP/inundation-mapping/pull/829)
 
 During the merge from remove-fim3 PR into dev, merge conflicts were discovered in the unit_tests folders and files. Attempts to fix them at that time failed, so some files were removed, other renamed, other edited to get the merge to work.  Here are the fixes to put the unit tests system back to par.
@@ -300,7 +322,6 @@ BUT.... you have to be careful not to overload your system.  **You need to multi
 
 <br/><br/>
 
-
 ## v4.0.19.5 - 2023-01-24 - [PR#801](https://github.com/NOAA-OWP/inundation-mapping/pull/801)
 
 When running tools/test_case_by_hydroid.py, it throws an error of local variable 'stats' referenced before assignment.
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index c9930f480..f838a299b 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -32,32 +32,6 @@ def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window
 
         return(zone_min_dict)
 
-    # Open files.
-    elevation_raster_object = rasterio.open(elevation_raster)
-    allocation_zone_raster_object = rasterio.open(allocation_raster)
-    cost_distance_raster_object = rasterio.open(cost_distance_raster)
-
-    meta = elevation_raster_object.meta.copy()
-    meta['tiled'], meta['compress'] = True, 'lzw'
-
-    # -- Create zone_min_dict -- #
-    zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums
-    # Update catchment_min_dict with pixel sheds minimum.
-    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template
-        elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window
-        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window
-        cost_window = cost_distance_raster_object.read(1, window=window).ravel()  # Define cost_window
-
-        # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
-        zone_min_dict = make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, int(cost_distance_tolerance), meta['nodata'])
-
-    # ------------------------------------------------------------------------------------------------------------------------ #
-
-    elevation_raster_object.close()
-    allocation_zone_raster_object.close()
-    cost_distance_raster_object.close()
-
-
     # ------------------------------------------- Assign zonal min to thalweg ------------------------------------------------ #
     @njit
     def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window):
@@ -82,30 +56,46 @@ def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_w
 
         return(dem_window_to_return)
 
-    # Specify raster object metadata.
-    elevation_raster_object = rasterio.open(elevation_raster)
-    allocation_zone_raster_object = rasterio.open(allocation_raster)
-    thalweg_object = rasterio.open(stream_raster)
+    # Open files.
+    with rasterio.open(elevation_raster) as elevation_raster_object, rasterio.open(allocation_raster) as allocation_zone_raster_object:
+
+        with rasterio.open(cost_distance_raster) as cost_distance_raster_object:
+
+            meta = elevation_raster_object.meta.copy()
+            meta['tiled'], meta['compress'] = True, 'lzw'
+            ndv = meta['nodata']
+
+            # -- Create zone_min_dict -- #
+            zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums
+            # Update catchment_min_dict with pixel sheds minimum.
+            for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template
+                elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window
+                zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window
+                cost_window = cost_distance_raster_object.read(1, window=window).ravel()  # Define cost_window
+
+                # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
+                zone_min_dict = make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, int(cost_distance_tolerance), ndv)
+
+            # ------------------------------------------------------------------------------------------------------------------------ #
+
 
-    dem_lateral_thalweg_adj_object = rasterio.open(dem_lateral_thalweg_adj, 'w', **meta)
+        # Specify raster object metadata.
+        with rasterio.open(stream_raster) as thalweg_object, rasterio.open(dem_lateral_thalweg_adj, 'w', **meta) as dem_lateral_thalweg_adj_object:
 
-    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
-        dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window
-        window_shape = dem_window.shape
-        dem_window = dem_window.ravel()
+            for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
+                dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window
+                window_shape = dem_window.shape
+                dem_window = dem_window.ravel()
 
-        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window
-        thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window
+                zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window
+                thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window
 
-        # Call numba-optimized function to reassign thalweg cell values to catchment minimum value.
-        minimized_dem_window = minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window)
-        minimized_dem_window = minimized_dem_window.reshape(window_shape).astype(np.float32)
+                # Call numba-optimized function to reassign thalweg cell values to catchment minimum value.
+                minimized_dem_window = minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window)
+                minimized_dem_window = minimized_dem_window.reshape(window_shape).astype(np.float32)
 
-        dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)
+                dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)
 
-    elevation_raster_object.close()
-    allocation_zone_raster_object.close()
-    cost_distance_raster_object.close()
 
 
 if __name__ == '__main__':
diff --git a/src/agreedem.py b/src/agreedem.py
index 7d15b7f80..88e293fcd 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -3,14 +3,14 @@
 import numpy as np
 import os
 import argparse
-from r_grow_distance import r_grow_distance
+import whitebox
 from utils.shared_functions import mem_profile
 
 
 @mem_profile
-def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data):
+def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data):
     '''
-    Produces a hydroconditioned raster using the AGREE DEM methodology as described by Ferdi Hellweger (https://www.caee.utexas.edu/prof/maidment/gishydro/ferdi/research/agree/agree.html). The GRASS gis tool r.grow.distance is used to calculate intermediate allocation and proximity rasters.
+    Produces a hydroconditioned raster using the AGREE DEM methodology as described by Ferdi Hellweger (https://www.caee.utexas.edu/prof/maidment/gishydro/ferdi/research/agree/agree.html). Whiteboxtools is used to calculate intermediate allocation and proximity rasters.
 
     Parameters
     ----------
@@ -22,8 +22,6 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
         Path to output raster. For example, dem_burned.tif
     workspace : STR
         Path to workspace to save all intermediate files.
-    grass_workspace : STR
-        Path to the temporary workspace for grass inputs. This temporary workspace is deleted once grass datasets are produced and exported to tif files.
     buffer_dist : FLOAT
         AGREE stream buffer distance (in meters) on either side of stream.
     smooth_drop : FLOAT
@@ -38,6 +36,10 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
     None.
 
     '''
+    # Set wbt envs
+    wbt = whitebox.WhiteboxTools()
+    wbt.set_verbose_mode(False)
+    
     #------------------------------------------------------------------
     # 1. From Hellweger documentation: Compute the vector grid
     # (vectgrid). The cells in the vector grid corresponding to the
@@ -45,182 +47,192 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
     # data.
 
     # Import dem layer and river layer and get dem profile.
-    elev = rasterio.open(dem)
-    dem_profile = elev.profile
-
-    rivers = rasterio.open(rivers_raster)
-
-    # Define smogrid profile and output file
-    smo_profile = dem_profile.copy()
-    smo_profile.update(nodata = 0)
-    smo_profile.update(dtype = 'float32')
-    smo_output = os.path.join(workspace, 'agree_smogrid.tif')
-
-    # Windowed reading/calculating/writing
-    with rasterio.Env():
-        with rasterio.open(smo_output, 'w', **smo_profile) as raster:
-            for ji, window in elev.block_windows(1):
-                # read elevation data and mask information
-                elev_data_window = elev.read(1, window = window)
-                elev_mask_window = elev.read_masks(1, window = window).astype('bool')
-                # Import boolean river raster and apply same NODATA mask as dem
-                # layer. In case rivers extend beyond valid data regions of DEM.
-                river_raw_data_window = rivers.read(1, window = window)
-                river_data_window = np.where(elev_mask_window == True, river_raw_data_window, 0)
-
-                #---------------------------------------------------------------
-                # 2. From Hellweger documentation: Compute the smooth drop/raise
-                # grid (smogrid). The cells in the smooth drop/raise grid
-                # corresponding to the vector lines have an elevation equal to that
-                # of the original DEM (oelevgrid) plus a certain distance
-                # (smoothdist). All other cells have no data.
-
-                # Assign smooth distance and calculate the smogrid.
-                smooth_dist = -1 * smooth_drop # in meters.
-                smogrid_window = river_data_window*(elev_data_window + smooth_dist)
-
-                # Write out raster
-                raster.write(smogrid_window.astype('float32'), indexes = 1, window = window)
-
-    elev.close()
-    rivers.close()
-    raster.close()
-    #------------------------------------------------------------------
-    # 3. From Hellweger documentation: Compute the vector distance grids
-    # (vectdist and vectallo). The cells in the vector distance grid
-    # (vectdist) store the distance to the closest vector cell. The
-    # cells in vector allocation grid (vectallo) store the elevation of
-    # the closest vector cell.
-
-    # Compute allocation and proximity grid using GRASS gis
-    # r.grow.distance tool. Output distance grid in meters. Set datatype
-    # for output allocation and proximity grids to float32.
-    vectdist_grid, vectallo_grid = r_grow_distance(smo_output, grass_workspace, 'Float32', 'Float32')
-
-    #------------------------------------------------------------------
-    # 4. From Hellweger documentation: Compute the buffer grid
-    # (bufgrid2). The cells in the buffer grid outside the buffer
-    # distance (buffer) store the original elevation. The cells in the
-    # buffer grid inside the buffer distance have no data.
-
-    # Open distance, allocation, elevation grids.
-    vectdist = rasterio.open(vectdist_grid)
-    vectallo = rasterio.open(vectallo_grid)
-    elev = rasterio.open(dem)
-
-    # Define bufgrid profile and output file.
-    buf_output = os.path.join(workspace, 'agree_bufgrid.tif')
-    buf_profile = dem_profile.copy()
-    buf_profile.update(dtype = 'float32')
-
-    # Windowed reading/calculating/writing
-    with rasterio.Env():
-        with rasterio.open(buf_output, 'w', **buf_profile) as raster:
-            for ji, window in elev.block_windows(1):
-                # read distance, allocation, and elevation datasets
-                vectdist_data_window = vectdist.read(1, window = window)
-                vectallo_data_window = vectallo.read(1, window = window)
-                elev_data_window = elev.read(1, window = window)
-
-                # Define buffer distance and calculate adjustment to compute the
-                # bufgrid.
-                # half_res adjustment equal to half distance of one cell
-                half_res = elev.res[0]/2
-                final_buffer = buffer_dist - half_res # assume all units in meters.
-
-                # Calculate bufgrid. Assign NODATA to areas where vectdist_data <=
-                # buffered value.
-                bufgrid_window = np.where(vectdist_data_window > final_buffer, elev_data_window, dem_profile['nodata'])
-
-                # Write out raster.
-                raster.write(bufgrid_window.astype('float32'), indexes = 1, window = window)
-
-    vectdist.close()
-    vectallo.close()
-    elev.close()
-    #------------------------------------------------------------------
-    # 5. From Hellweger documentation: Compute the buffer distance grids
-    # (bufdist and bufallo). The cells in the buffer distance grid
-    # (bufdist) store the distance to the closest valued buffer grid
-    # cell (bufgrid2). The cells in buffer allocation grid (bufallo)
-    # store the elevation of the closest valued buffer cell.
-
-    # Compute allocation and proximity grid using GRASS gis
-    # r.grow.distance. Output distance grid in meters. Set datatype for
-    # output allocation and proximity grids to float32.
-    bufdist_grid, bufallo_grid = r_grow_distance(buf_output, grass_workspace, 'Float32', 'Float32')
-
-    # Open distance, allocation, elevation grids.
-    bufdist = rasterio.open(bufdist_grid)
-    bufallo = rasterio.open(bufallo_grid)
-    vectdist = rasterio.open(vectdist_grid)
-    vectallo = rasterio.open(vectallo_grid)
-    rivers = rasterio.open(rivers_raster)
-    elev = rasterio.open(dem)
-
-    # Define profile output file.
-    agree_output = output_raster
-    agree_profile = dem_profile.copy()
-    agree_profile.update(dtype = 'float32')
-
-    # Windowed reading/calculating/writing
-    with rasterio.Env():
-        with rasterio.open(agree_output, 'w', **agree_profile) as raster:
-            for ji, window in elev.block_windows(1):
-                # Read elevation data and mask, distance and allocation grids, and river data.
-                elev_data_window = elev.read(1, window = window)
-                elev_mask_window = elev.read_masks(1, window = window).astype('bool')
-                bufdist_data_window = bufdist.read(1, window = window)
-                bufallo_data_window = bufallo.read(1, window = window)
-                vectdist_data_window = vectdist.read(1, window = window)
-                vectallo_data_window = vectallo.read(1, window = window)
-                river_raw_data_window = rivers.read(1, window = window)
-
-
-                river_data_window = np.where(elev_mask_window == True, river_raw_data_window, -20.0)
-                #------------------------------------------------------------------
-                # 6. From Hellweger documentation: Compute the smooth modified
-                # elevation grid (smoelev). The cells in the smooth modified
-                # elevation grid store the results of the smooth surface
-                # reconditioning process. Note that for cells outside the buffer the
-                # equation below assigns the original elevation.
-
-                # Calculate smoelev.
-                smoelev_window = vectallo_data_window + ((bufallo_data_window - vectallo_data_window)/(bufdist_data_window + vectdist_data_window)) * vectdist_data_window
-
-                #------------------------------------------------------------------
-                # 7. From Hellweger documentation: Compute the sharp drop/raise grid
-                # (shagrid). The cells in the sharp drop/raise grid corresponding to
-                # the vector lines have an elevation equal to that of the smooth
-                # modified elevation grid (smoelev) plus a certain distance
-                # (sharpdist). All other cells have no data.
-
-                # Define sharp drop distance and calculate the sharp drop grid where
-                # only river cells are dropped by the sharp_dist amount.
-                sharp_dist = -1 * sharp_drop # in meters.
-                shagrid_window = (smoelev_window + sharp_dist) * river_data_window
-
-                #------------------------------------------------------------------
-                # 8. From Hellweger documentation: Compute the modified elevation
-                # grid (elevgrid). The cells in the modified elevation grid store
-                # the results of the surface reconditioning process. Note that for
-                # cells outside the buffer the the equation below assigns the
-                # original elevation.
-
-                # Merge sharp drop grid with smoelev grid. Then apply the same
-                # NODATA mask as original elevation grid.
-                elevgrid_window = np.where(river_data_window == 0, smoelev_window, shagrid_window)
-                agree_dem_window = np.where(elev_mask_window == True, elevgrid_window, dem_profile['nodata'])
-
-                # Write out to raster
-                raster.write(agree_dem_window.astype('float32'), indexes = 1, window = window)
-
-    bufdist.close()
-    bufallo.close()
-    vectdist.close()
-    vectallo.close()
-    rivers.close()
-    elev.close()
+    with rasterio.open(dem) as elev, rasterio.open(rivers_raster) as rivers:
+        dem_profile = elev.profile
+
+        # Define smogrid profile and output file
+        smo_profile = dem_profile.copy()
+        smo_profile.update(nodata = 0)
+        smo_profile.update(dtype = 'float32')
+        smo_output = os.path.join(workspace, 'agree_smogrid.tif')
+        vectdist_grid = os.path.join(workspace,'agree_smogrid_dist.tif')
+        vectallo_grid = os.path.join(workspace,'agree_smogrid_allo.tif')
+        
+        # Windowed reading/calculating/writing
+        with rasterio.Env():
+            with rasterio.open(smo_output, 'w', **smo_profile) as raster:
+                for ji, window in elev.block_windows(1):
+                    # read elevation data and mask information
+                    elev_data_window = elev.read(1, window = window)
+                    elev_mask_window = elev.read_masks(1, window = window).astype('bool')
+                    # Import boolean river raster and apply same NODATA mask as dem
+                    # layer. In case rivers extend beyond valid data regions of DEM.
+                    river_raw_data_window = rivers.read(1, window = window)
+                    river_data_window = np.where(elev_mask_window == True, river_raw_data_window, 0)
+
+                    #---------------------------------------------------------------
+                    # 2. From Hellweger documentation: Compute the smooth drop/raise
+                    # grid (smogrid). The cells in the smooth drop/raise grid
+                    # corresponding to the vector lines have an elevation equal to that
+                    # of the original DEM (oelevgrid) plus a certain distance
+                    # (smoothdist). All other cells have no data.
+
+                    # Assign smooth distance and calculate the smogrid.
+                    smooth_dist = -1 * smooth_drop # in meters.
+                    smogrid_window = river_data_window*(elev_data_window + smooth_dist)
+
+                    # Write out raster
+                    raster.write(smogrid_window.astype('float32'), indexes = 1, window = window)
+
+        #------------------------------------------------------------------
+        # 3. From Hellweger documentation: Compute the vector distance grids
+        # (vectdist and vectallo). The cells in the vector distance grid
+        # (vectdist) store the distance to the closest vector cell. The
+        # cells in vector allocation grid (vectallo) store the elevation of
+        # the closest vector cell.
+
+        # Compute allocation and proximity grid using WhiteboxTools
+        smo_output_zerod = os.path.join(workspace, 'agree_smogrid_zerod.tif')
+        wbt.euclidean_distance(rivers_raster,vectdist_grid)
+        wbt.convert_nodata_to_zero(smo_output,smo_output_zerod)
+        wbt.euclidean_allocation(smo_output_zerod,vectallo_grid)
+
+        #------------------------------------------------------------------
+        # 4. From Hellweger documentation: Compute the buffer grid
+        # (bufgrid2). The cells in the buffer grid outside the buffer
+        # distance (buffer) store the original elevation. The cells in the
+        # buffer grid inside the buffer distance have no data.
+
+        # Open distance, allocation, elevation grids.
+        with rasterio.open(vectdist_grid) as vectdist:
+
+            # Define bufgrid profile and output file.
+            buf_output = os.path.join(workspace, 'agree_bufgrid.tif')
+            bufdist_grid = os.path.join(workspace,'agree_bufgrid_dist.tif')
+            bufallo_grid = os.path.join(workspace,'agree_bufgrid_allo.tif')
+            buf_profile = dem_profile.copy()
+            buf_profile.update(dtype = 'float32')
+
+            # Windowed reading/calculating/writing
+            with rasterio.Env():
+                with rasterio.open(buf_output, 'w', **buf_profile) as raster:
+                    for ji, window in elev.block_windows(1):
+                        # read distance, allocation, and elevation datasets
+                        vectdist_data_window = vectdist.read(1, window = window)
+                        elev_data_window = elev.read(1, window = window)
+
+                        # Define buffer distance and calculate adjustment to compute the
+                        # bufgrid.
+                        # half_res adjustment equal to half distance of one cell
+                        half_res = elev.res[0]/2
+                        final_buffer = buffer_dist - half_res # assume all units in meters.
+
+                        # Calculate bufgrid. Assign NODATA to areas where vectdist_data <=
+                        # buffered value.
+                        bufgrid_window = np.where(vectdist_data_window > final_buffer, elev_data_window, dem_profile['nodata'])
+
+                        # Write out raster.
+                        raster.write(bufgrid_window.astype('float32'), indexes = 1, window = window)
+
+        
+            #------------------------------------------------------------------
+            # 5. From Hellweger documentation: Compute the buffer distance grids
+            # (bufdist and bufallo). The cells in the buffer distance grid
+            # (bufdist) store the distance to the closest valued buffer grid
+            # cell (bufgrid2). The cells in buffer allocation grid (bufallo)
+            # store the elevation of the closest valued buffer cell.
+
+            # # Transform the buffer grid (bufgrid2) to binary raster
+            bin_buf_output = os.path.join(workspace, 'agree_binary_bufgrid.tif')
+            with rasterio.open(buf_output) as agree_bufgrid:
+                agree_bufgrid_profile = agree_bufgrid.profile
+                bin_buf_output_profile = agree_bufgrid_profile.copy()
+                bin_buf_output_profile.update(dtype = 'float32')
+
+                with rasterio.Env():
+                    with rasterio.open(bin_buf_output, 'w', **bin_buf_output_profile) as raster:
+                        for ji, window in agree_bufgrid.block_windows(1):
+                            # read distance, allocation, and elevation datasets
+                            agree_bufgrid_data_window = agree_bufgrid.read(1, window = window)
+
+                            # Calculate bufgrid. Assign NODATA to areas where vectdist_data <=
+                            agree_bufgrid_data_window = np.where(agree_bufgrid_data_window>-10000, 1, 0)
+
+                            # Write out raster.
+                            raster.write(agree_bufgrid_data_window.astype('float32'), indexes = 1, window = window)
+
+            # Compute allocation and proximity grid using WhiteboxTools
+            buf_output_zerod = os.path.join(workspace, 'agree_bufgrid_zerod.tif')
+            wbt.euclidean_distance(bin_buf_output,bufdist_grid)
+            wbt.convert_nodata_to_zero(buf_output,buf_output_zerod)
+            wbt.euclidean_allocation(buf_output_zerod,bufallo_grid)
+
+            # Open distance, allocation, elevation grids.
+            with rasterio.open(bufdist_grid) as bufdist, rasterio.open(bufallo_grid) as bufallo, rasterio.open(vectallo_grid) as vectallo:
+
+                # Define profile output file.
+                agree_output = output_raster
+                agree_profile = dem_profile.copy()
+                agree_profile.update(dtype = 'float32')
+
+                # Windowed reading/calculating/writing
+                with rasterio.Env():
+                    with rasterio.open(agree_output, 'w', **agree_profile) as raster:
+                        for ji, window in elev.block_windows(1):
+                            # Read elevation data and mask, distance and allocation grids, and river data.
+                            elev_data_window = elev.read(1, window = window)
+                            elev_mask_window = elev.read_masks(1, window = window).astype('bool')
+                            bufdist_data_window = bufdist.read(1, window = window)
+                            bufallo_data_window = bufallo.read(1, window = window)
+                            vectdist_data_window = vectdist.read(1, window = window)
+                            vectallo_data_window = vectallo.read(1, window = window)
+                            river_raw_data_window = rivers.read(1, window = window)
+
+                            bufallo_data_window = np.where(bufallo_data_window == -32768., elev_data_window, bufallo_data_window)
+
+                            vectallo_data_window = np.where(vectallo_data_window == -32768., elev_data_window-10, vectallo_data_window)
+
+                            river_raw_data_window = river_raw_data_window.astype(np.float32)
+
+                            river_data_window = np.where(elev_mask_window == True, river_raw_data_window, -20.0)
+                            #------------------------------------------------------------------
+                            # 6. From Hellweger documentation: Compute the smooth modified
+                            # elevation grid (smoelev). The cells in the smooth modified
+                            # elevation grid store the results of the smooth surface
+                            # reconditioning process. Note that for cells outside the buffer the
+                            # equation below assigns the original elevation.
+
+                            # Calculate smoelev.
+                            smoelev_window = vectallo_data_window + ((bufallo_data_window - vectallo_data_window)/(bufdist_data_window + vectdist_data_window)) * vectdist_data_window
+
+                            #------------------------------------------------------------------
+                            # 7. From Hellweger documentation: Compute the sharp drop/raise grid
+                            # (shagrid). The cells in the sharp drop/raise grid corresponding to
+                            # the vector lines have an elevation equal to that of the smooth
+                            # modified elevation grid (smoelev) plus a certain distance
+                            # (sharpdist). All other cells have no data.
+
+                            # Define sharp drop distance and calculate the sharp drop grid where
+                            # only river cells are dropped by the sharp_dist amount.
+                            sharp_dist = -1 * sharp_drop # in meters.
+                            shagrid_window = (smoelev_window + sharp_dist) * river_data_window
+
+                            #------------------------------------------------------------------
+                            # 8. From Hellweger documentation: Compute the modified elevation
+                            # grid (elevgrid). The cells in the modified elevation grid store
+                            # the results of the surface reconditioning process. Note that for
+                            # cells outside the buffer the the equation below assigns the
+                            # original elevation.
+
+                            # Merge sharp drop grid with smoelev grid. Then apply the same
+                            # NODATA mask as original elevation grid.
+                            elevgrid_window = np.where(river_data_window == 0, smoelev_window, shagrid_window)
+                            agree_dem_window = np.where(elev_mask_window == True, elevgrid_window, dem_profile['nodata'])
+
+                            # Write out to raster
+                            raster.write(agree_dem_window.astype('float32'), indexes = 1, window = window)
+
+            
     # If the '-t' flag is called, intermediate data is removed.
     if delete_intermediate_data:
         os.remove(smo_output)
@@ -229,6 +241,9 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
         os.remove(vectallo_grid)
         os.remove(bufdist_grid)
         os.remove(bufallo_grid)
+        os.remove(bin_buf_output)
+        os.remove(buf_output_zerod)
+        os.remove(smo_output_zerod)
 
 
 if __name__ == '__main__':
@@ -238,7 +253,6 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
     parser.add_argument('-r', '--rivers', help = 'flows grid boolean layer', required = True)
     parser.add_argument('-d', '--dem_m',  help = 'DEM raster in meters', required = True)
     parser.add_argument('-w', '--workspace', help = 'Workspace', required = True)
-    parser.add_argument('-g', '--grass_workspace', help = 'Temporary GRASS workspace', required = True)
     parser.add_argument('-o',  '--output', help = 'Path to output raster', required = True)
     parser.add_argument('-b',  '--buffer', help = 'Buffer distance (m) on either side of channel', required = True)
     parser.add_argument('-sm', '--smooth', help = 'Smooth drop (m)', required = True)
@@ -252,7 +266,6 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
     rivers_raster = args['rivers']
     dem = args['dem_m']
     workspace = args['workspace']
-    grass_workspace = args['grass_workspace']
     output_raster = args['output']
     buffer_dist = float(args['buffer'])
     smooth_drop = float(args['smooth'])
@@ -260,4 +273,4 @@ def agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buff
     delete_intermediate_data = args['del']
 
     #Run agreedem
-    agreedem(rivers_raster, dem, output_raster, workspace, grass_workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data)
+    agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data)
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 022c36e0e..f92d39e61 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -33,7 +33,7 @@ Tcount
 echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif -g $outputCurrentBranchDataDir/temp_grass
+python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif
 Tcount
 
 ## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
diff --git a/src/mask_dem.py b/src/mask_dem.py
index 42f0a8feb..f7310c74c 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -20,7 +20,6 @@ def mask_dem(dem_filename, nld_filename, out_dem_filename, stream_layer, branch_
         if (branch_id == branch_zero_id) or (streams_df.loc[streams_df[branch_id_attribute].astype(int)==branch_id, order_attribute].max() >= streams_df[order_attribute].max() - 1):
 
             with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
-                dem_data = dem.read(1)
                 dem_profile = dem.profile.copy()
 
                 geoms = [feature["geometry"] for feature in leveed]
diff --git a/src/r_grow_distance.py b/src/r_grow_distance.py
deleted file mode 100755
index d42e88c0d..000000000
--- a/src/r_grow_distance.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/env python3
-from grass_session import Session
-import os
-import shutil 
-import grass.script as gscript
-import argparse
-
-
-def r_grow_distance(input_raster, grass_workspace, proximity_dtype, allocation_dtype):
-    '''
-    Runs the r.grow.distance GRASS gis tool which given an input raster will produce an output proximity (or distance) and euclidian allocation tool.
-
-    Parameters
-    ----------
-    input_raster : STR
-        Path to input raster. For example, see flows_grid_boolean.tif
-    grass_workspace : STR
-        Path to TEMPORARY directory to store intermediate GRASS data. This directory is deleted upon completion of this function.
-    proximity_dtype: STR
-        Data type for the proximity output. Typically 'Float32'.    
-    allocation_dtype: STR
-        Data type for the allocation output. Typically 'Float32' (AGREE processing) or 'Float64' (thalweg adjustment processing)
-
-    Returns
-    -------
-    output_proximity_path : STR
-        The path to the output proximity (or distance) raster (in tif format).
-    output_allocation_path : STR
-        The path to the output euclidian allocation raster (in tif format).
-
-    '''
-    
-    # Define parent directory of input raster and get input raster name
-    input_raster_directory = os.path.dirname(input_raster)
-    input_raster_name = os.path.splitext(os.path.basename(input_raster))[0]
-    
-    # Set up variables for use in GRASS
-    grass_gisdb = grass_workspace
-    grass_location = 'temporary_location'
-    grass_mapset = 'temporary_mapset'
-    projected_file = input_raster
-
-    # Start and close PERMANENT session.
-    PERMANENT = Session()
-    PERMANENT.open(gisdb = grass_gisdb, location = grass_location, create_opts = projected_file)
-    PERMANENT.close()
-
-    # Open a temporary session.
-    temporary_session = Session()
-    temporary_session.open(gisdb = grass_gisdb, location = grass_location, mapset = grass_mapset, create_opts = projected_file)
-    
-    #Import input raster into temporary session.
-    imported_grass_raster = input_raster_name + '@' + grass_mapset
-    gscript.run_command('r.in.gdal', input = input_raster, output = imported_grass_raster, quiet = True)
-
-    # Define names for proximity and allocation rasters. Run 
-    # r.grow.distance tool.
-    proximity_grass_name = 'proximity@' + grass_mapset
-    allocation_grass_name = 'allocation@'+ grass_mapset
-    gscript.run_command('r.grow.distance', flags = 'm', input = imported_grass_raster, distance = proximity_grass_name, value = allocation_grass_name, quiet = True)
-    
-    # Export proximity raster. Saved to same directory as input raster. 
-    # Dtype for proximity always float32.
-    proximity_filename = input_raster_name + '_dist.tif'
-    output_proximity_path=os.path.join(input_raster_directory,proximity_filename)
-    gscript.run_command('r.out.gdal', flags = 'cf', input = proximity_grass_name, output = output_proximity_path, format = 'GTiff', quiet = True, type = proximity_dtype, createopt = 'COMPRESS=LZW')
-
-    # Export allocation raster. Saved to same directory as input raster. 
-    # Dtype assigned via the allocation_dtype input.
-    allocation_filename = input_raster_name + '_allo.tif'
-    output_allocation_path = os.path.join(input_raster_directory, allocation_filename)
-    gscript.run_command('r.out.gdal', flags = 'cf', input = allocation_grass_name, output = output_allocation_path, format = 'GTiff', quiet = True, type = allocation_dtype, createopt = 'COMPRESS=LZW')
-    
-    # Close down temporary session and remove temporary workspace.
-    temporary_session.close()
-    shutil.rmtree(grass_gisdb)
-    
-    return output_proximity_path,output_allocation_path
-
-
-if __name__ == '__main__':
-
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Calculate AGREE DEM')
-    parser.add_argument('-i', '--in_raster', help = 'raster to perform r.grow.distance', required = True)
-    parser.add_argument('-g', '--grass_workspace', help = 'Temporary GRASS workspace', required = True)
-    parser.add_argument('-p', '--prox_dtype', help = 'Output proximity raster datatype', required = True)
-    parser.add_argument('-a', '--allo_dtype', help = 'Output allocation raster datatype', required = True)
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    # Rename variable inputs
-    input_raster = args['in_raster']
-    grass_workspace = args['grass_workspace']
-    proximity_dtype = args['prox_dtype']
-    allocation_dtype = args['allo_dtype']
-    
-    # Run r_grow_distance
-    r_grow_distance(input_raster, grass_workspace, proximity_dtype, allocation_dtype)
-    
\ No newline at end of file
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 527b7be92..13955ea7b 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -167,7 +167,7 @@ Tcount
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -w $outputCurrentBranchDataDir -g $outputCurrentBranchDataDir/temp_work -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -w $outputCurrentBranchDataDir -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
 Tcount
 
 ## PIT REMOVE BURNED DEM ##
diff --git a/src/unique_pixel_and_allocation.py b/src/unique_pixel_and_allocation.py
index 0df89560c..b100d4c86 100755
--- a/src/unique_pixel_and_allocation.py
+++ b/src/unique_pixel_and_allocation.py
@@ -4,15 +4,16 @@
 
 @author: trevor.grout
 """
+import os
 import rasterio
 import numpy as np
 import argparse
-from r_grow_distance import r_grow_distance
+import whitebox
 from utils.shared_functions import mem_profile
 
 
 @mem_profile
-def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
+def stream_pixel_zones(stream_pixels, unique_stream_pixels):
     '''
     This function will assign a unique ID for each stream pixel and writes to file. It then uses this raster to run GRASS r.grow.distance tool to create the allocation and proximity rasters required to complete the lateral thalweg conditioning.
 
@@ -22,8 +23,6 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
         Path to stream raster with value of 1. For example, demDerived_streamPixels.tif.
     unique_stream_pixels : STR
         Output path of raster containing unique ids for each stream pixel.
-    grass_workspace : STR
-        Path to temporary GRASS directory which is deleted.
 
     Returns
     -------
@@ -33,7 +32,15 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
         Path to output allocation raster.
 
     '''
-
+    # Set wbt envs
+    wbt = whitebox.WhiteboxTools()
+    wbt.set_verbose_mode(False)
+
+    workspace = os.path.dirname(unique_stream_pixels)
+    base = os.path.basename(unique_stream_pixels)
+    distance_grid = os.path.join(workspace,os.path.splitext(base)[0]+'_dist.tif')
+    allocation_grid = os.path.join(workspace,os.path.splitext(base)[0]+'_allo.tif')
+    
     # Import stream pixel raster
     with rasterio.open(stream_pixels) as temp:
         streams_profile = temp.profile
@@ -42,8 +49,8 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
     # Create array that matches shape of streams raster with unique values for each cell. Dataype is float64.
     unique_vals = np.arange(streams.size, dtype = 'float64').reshape(*streams.shape)
 
-    # At streams return the unique array value otherwise return NODATA value from input streams layer. NODATA value for demDerived_streamPixels.tif is -32768.
-    stream_pixel_values = np.where(streams == 1, unique_vals, streams_profile['nodata'])
+    # At streams return the unique array value otherwise return 0 values
+    stream_pixel_values = np.where(streams == 1, unique_vals, 0)
 
     # Reassign dtype to be float64 (needs to be float64)
     streams_profile.update(dtype = 'float64')
@@ -53,8 +60,20 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
         with rasterio.open(unique_stream_pixels, 'w', **streams_profile) as raster:
             raster.write(stream_pixel_values,1)
 
-    # Compute allocation and proximity grid using r.grow.distance. Output distance grid in meters. Set datatype for output allocation (needs to be float64) and proximity grids (float32).
-    distance_grid, allocation_grid = r_grow_distance(unique_stream_pixels, grass_workspace, 'Float32', 'Float64')
+    # Compute allocation and proximity grids.
+    wbt.euclidean_distance(stream_pixels,distance_grid)
+    wbt.euclidean_allocation(unique_stream_pixels,allocation_grid)
+
+    with rasterio.open(allocation_grid) as allocation_ds:
+        allocation = allocation_ds.read(1)
+        allocation_profile = allocation_ds.profile
+
+    # Add stream channel ids
+    allocation = np.where(allocation > 0, allocation, stream_pixel_values)
+
+    with rasterio.open(allocation_grid, 'w', **allocation_profile) as allocation_ds:
+        allocation_ds.write(allocation, 1)
+
 
     return distance_grid, allocation_grid
 
@@ -65,7 +84,6 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
     parser = argparse.ArgumentParser(description = 'Produce unique stream pixel values and allocation/proximity grids')
     parser.add_argument('-s', '--stream', help = 'raster to perform r.grow.distance', required = True)
     parser.add_argument('-o', '--out', help = 'output raster of unique ids for each stream pixel', required = True)
-    parser.add_argument('-g', '--grass_workspace', help = 'Temporary GRASS workspace', required = True)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
@@ -73,7 +91,6 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace):
     # Rename variable inputs
     stream_pixels = args['stream']
     unique_stream_pixels = args['out']
-    grass_workspace = args['grass_workspace']
 
     # Run stream_pixel_zones
-    stream_pixel_zones(stream_pixels, unique_stream_pixels, grass_workspace)
+    stream_pixel_zones(stream_pixels, unique_stream_pixels)

From 979705eb37840c4c5ef6143ff9786a98c036f0ec Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 10 Mar 2023 11:54:39 -0500
Subject: [PATCH 218/359] v4.3.0.1 Delete intermediate agreedem files (#841)

---
 config/deny_branch_unittests.lst |  3 +++
 config/deny_branch_zero.lst      |  3 +++
 config/deny_branches.lst         |  3 +++
 config/deny_unit.lst             |  9 +++++++++
 docs/CHANGELOG.md                | 10 ++++++++++
 5 files changed, 28 insertions(+)

diff --git a/config/deny_branch_unittests.lst b/config/deny_branch_unittests.lst
index 5e9a0968e..aec38c3b9 100644
--- a/config/deny_branch_unittests.lst
+++ b/config/deny_branch_unittests.lst
@@ -1,12 +1,15 @@
 # List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
+agree_binary_bufgrid.tif
 agree_bufgrid.tif
 agree_bufgrid_allo.tif
 agree_bufgrid_dist.tif
+agree_bufgrid_zerod.tif
 agree_smogrid.tif
 agree_smogrid_allo.tif
 agree_smogrid_dist.tif
+agree_smogrid_zerod.tif
 catch_list_{}.txt
 coordFile_{}.txt
 crosswalk_table_{}.csv
diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst
index 01bfaee85..34caa2694 100644
--- a/config/deny_branch_zero.lst
+++ b/config/deny_branch_zero.lst
@@ -1,12 +1,15 @@
 # List of files for branch zero to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
+agree_binary_bufgrid.tif
 agree_bufgrid.tif
 agree_bufgrid_allo.tif
 agree_bufgrid_dist.tif
+agree_bufgrid_zerod.tif
 agree_smogrid.tif
 agree_smogrid_allo.tif
 agree_smogrid_dist.tif
+agree_smogrid_zerod.tif
 catch_list_{}.txt
 coordFile_{}.txt
 crosswalk_table_{}.csv
diff --git a/config/deny_branches.lst b/config/deny_branches.lst
index 9629d0317..4616c3717 100644
--- a/config/deny_branches.lst
+++ b/config/deny_branches.lst
@@ -1,12 +1,15 @@
 # List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
+agree_binary_bufgrid.tif
 agree_bufgrid.tif
 agree_bufgrid_allo.tif
 agree_bufgrid_dist.tif
+agree_bufgrid_zerod.tif
 agree_smogrid.tif
 agree_smogrid_allo.tif
 agree_smogrid_dist.tif
+agree_smogrid_zerod.tif
 catch_list_{}.txt
 coordFile_{}.txt
 crosswalk_table_{}.csv
diff --git a/config/deny_unit.lst b/config/deny_unit.lst
index b4680544e..6f0283f76 100644
--- a/config/deny_unit.lst
+++ b/config/deny_unit.lst
@@ -1,5 +1,14 @@
 # List of files for units to delete
 # Use comment to allow list the file
+agree_binary_bufgrid.tif
+agree_bufgrid.tif
+agree_bufgrid_allo.tif
+agree_bufgrid_dist.tif
+agree_bufgrid_zerod.tif
+agree_smogrid.tif
+agree_smogrid_allo.tif
+agree_smogrid_dist.tif
+agree_smogrid_zerod.tif
 #branch_id.lst
 #branch_polygons.gpkg
 #LandSea_subset.gpkg
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e8cb443ef..58d8c98ce 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.0.1 - 2023-03-06 - [PR#841](https://github.com/NOAA-OWP/inundation-mapping/pull/841)
+
+Deletes intermediate files generated by `src/agreedem.py` by adding them to `config/deny_*.lst`
+
+- `config/`
+    - `deny_branch_zero.lst`, `deny_branches.lst`, `deny_branch_unittests.lst`: Added `agree_binary_bufgrid.tif`, `agree_bufgrid_zerod.tif`, and `agree_smogrid_zerod.tif`
+    - `deny_unit.lst`: Added `agree_binary_bufgrid.tif`, `agree_bufgrid.tif`, `agree_bufgrid_allo.tif`, `agree_bufgrid_dist.tif`,  `agree_bufgrid_zerod.tif`, `agree_smogrid.tif`, `agree_smogrid_allo.tif`, `agree_smogrid_dist.tif`, `agree_smogrid_zerod.tif`
+
+<br/><br/>
+
 ## v4.3.0.0 - 2023-02-15 - [PR#814](https://github.com/NOAA-OWP/inundation-mapping/pull/814)
 
 Replaces GRASS with Whitebox. This addresses several issues, including Windows permissions and GRASS projection issues. Whitebox also has a slight performance benefit over GRASS.

From 66853b33fda8819d2d526f84ef5264793527894a Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 10 Mar 2023 17:17:22 +0000
Subject: [PATCH 219/359] v4.3.1.0 Change all occurances of `/data/outputs` to
 `/outputs` (#834)

---
 Dockerfile                                    |  2 ++
 README.md                                     | 19 +++++++++--
 docs/CHANGELOG.md                             | 27 ++++++++++++++++
 fim_pipeline.sh                               |  4 +--
 fim_post_processing.sh                        |  2 +-
 fim_pre_processing.sh                         |  2 +-
 fim_process_unit_wb.sh                        |  2 +-
 src/usgs_gage_crosswalk.py                    | 32 +++++++++++++++----
 src/usgs_gage_unit_setup.py                   |  2 +-
 tools/cache_metrics.py                        |  2 +-
 tools/copy_test_case_folders.py               |  2 +-
 tools/run_test_case.py                        |  8 ++---
 unit_tests/clip_vectors_to_wbd_params.json    | 20 ++++++------
 unit_tests/derive_level_paths_params.json     | 16 +++++-----
 ..._catchments_and_add_attributes_params.json | 12 +++----
 unit_tests/outputs_cleanup_params.json        |  6 ++--
 .../rating_curve_comparison_params.json       | 10 +++---
 unit_tests/split_flows_params.json            | 16 +++++-----
 unit_tests/split_flows_test.py                |  4 +--
 unit_tests/tools/inundate_gms_params.json     |  2 +-
 unit_tests/tools/inundation_params.json       |  8 ++---
 unit_tests/usgs_gage_crosswalk_params.json    | 12 +++----
 22 files changed, 137 insertions(+), 73 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 34c01455d..f8f535cef 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,9 @@ RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d
 # Base Image that has GDAL, PROJ, etc
 FROM osgeo/gdal:ubuntu-full-3.1.2
 ARG dataDir=/data
+# remove below line, and uncomment # ENV projectDir=/foss_fim, & delete this line
 ARG projectDir=/foss_fim
+# ENV projectDir=/foss_fim
 ARG depDir=/dependencies
 ENV inputDataDir=$dataDir/inputs
 ENV outputDataDir=/outputs
diff --git a/README.md b/README.md
index dc5225b8e..8c7c7570a 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,22 @@ This software is configurable via parameters found in the `config` directory. Co
 Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
 - `params_template.env`
 
-This system has an optional tool called the `calibration database tool`. In order to use this system, you will need to install the calibration database service or disable it in the `params_template.env` file. See [calibration tool README](https://github.com/NOAA-OWP/inundation-mapping/blob/dev/tools/calibration-db/README.md) for more details.
+This system has an optional tool called the `calibration database tool`. In order to use this system, you have three options:  
+1.  Install the calibration database service.
+2.  Disable it by providing the `-skipcal` command line option to `fim_pipeline.sh` or `fim_preprocessing.sh`.
+3.  Disable it in the `params_template.env` file. See [calibration tool README](https://github.com/NOAA-OWP/inundation-mapping/blob/dev/tools/calibration-db/README.md) for more details.
+
+### Start/run the Docker Container
+
+Since all of the dependencies are managed in utilizing a Docker container, we must issue the [`docker run`](https://docs.docker.com/engine/reference/run/#clean-up---rm) command to start a container as the run-time environment. The container is launched from a Docker Image which was built in [Installation](#installation) step 2. The correct input file pathing is necessary for the `/data` volume mount (`-v`) for the `<input_path>`. The `<input_path>` should contain a subdirectory named `/inputs` (similar to `s3://noaa-nws-owp-fim/hand_fim`). If the pathing is set correctly, we do not need to adjust the `params_template.env` file, and can use the default file paths provided.
+
+```bash 
+docker run --rm -it --name <your_container_name> -v <path/to/repository>/:/foss_fim -v <desired_output_path>/:/outputs -v <input_path>:/data <image_name>:<tag>
+```
+For example:  
+```bash
+docker run --rm -it --name robs_container -v /home/projects/inundation-mapping/:/foss_fim -v /home/projects/fim/outputs/:/outputs -v /home/projects/fim/inputs/:/data fim_4:dev_20230224_ad87a74
+```
 
 ### Produce HAND Hydrofabric
 ```
@@ -72,7 +87,7 @@ fim_pipeline.sh -u <huc8> -n <name_your_run>
 - Manditory arguments:
     - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited (.lst) file. To run the entire domain of available data use one of the ```/data/inputs/included_huc8.lst``` files or a HUC list file of your choice.  Depending on the performance of your server, especially the number of CPU cores, running the full domain can take multiple days.
     - `-n` is a name of your run (only alphanumeric)
-- Outputs can be found under ```/data/outputs/<name_your_run>```.
+- Outputs can be found under ```/outputs/<name_your_run>```.
 
 Processing of HUC's in FIM4 comes in three pieces. You can run `fim_pipeline.sh` which automatically runs all of three major section, but you can run each of the sections independently if you like. The three sections are:
 - `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 58d8c98ce..0f29bb68d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,33 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.1.0 - 2023-03-10 - [PR#834](https://github.com/NOAA-OWP/inundation-mapping/pull/834)
+
+Change all occurances of /data/outputs to /outputs to honor the correct volume mount directory specified when executing docker run.
+
+### Changes
+
+- `Dockerfile` - updated comments in relation to `projectDir=/foss_fim`
+- `fim_pipeline.sh` - updated comments in relation to `projectDir=/foss_fim`
+- `fim_pre_processing.sh` -updated comments in relation to `projectDir=/foss_fim`
+- `fim_post_processing.sh` - updated comments in relation to `projectDir=/foss_fim`
+- `README.md` - Provide documentation on starting the Docker Container, and update docs to include additional command line option for calibration database tool.   
+
+- `src/` 
+  - `usgs_gage_crosswalk.py` - added newline character to shorten commented example usage
+  - `usgs_gage_unit_setup.py` - `/data/outputs/` => `/outputs/`  
+
+- `tools/` 
+  - `cache_metrics.py` -  `/data/outputs/` => `/outputs/`
+  - `copy_test_case_folders.py`  - `/data/outputs/` => `/outputs/`
+  - `run_test_case.py` - `/data/outputs/` => `/outputs/`  
+
+- `unit_tests/*_params.json`  - `/data/outputs/` => `/outputs/` 
+
+- `unit_tests/split_flows_test.py`  - `/data/outputs/` => `/outputs/` 
+
+<br/><br/>
+
 ## v4.3.0.1 - 2023-03-06 - [PR#841](https://github.com/NOAA-OWP/inundation-mapping/pull/841)
 
 Deletes intermediate files generated by `src/agreedem.py` by adding them to `config/deny_*.lst`
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 0e5a81a18..154f899fe 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -10,7 +10,7 @@ For more details on
     - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited file
     i. To run entire domain of available data use the ```/data/inputs/included_huc8.lst``` file or a huc list file of your choice.
     - `-n` is a name of your run (only alphanumeric)
-- Outputs can be found under ```/data/outputs/<name_your_run>```
+- Outputs can be found under ```/outputs/<name_your_run>```
 
 Processing of HUC''s in FIM4 comes in three pieces. You can run `fim_pipeline.sh` which automatically runs all of three major section, but you can run each of the sections independently if you like. The three sections are:
 - `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
@@ -23,7 +23,7 @@ Running the `fim_pipeline.sh` is a quicker process than running all three steps
 set -e
 
 # TODO
-# upgrade Dockerfile to add this as an env value
+# update Dockerfile to add this as an env value, and delete line below
 projectDir=/foss_fim
 
 # See fim_pre_processing.sh for details of how to use this script. fim_pre_processing.sh
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 0367f24cc..54ad2a5b1 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -41,7 +41,7 @@ in
 done
 
 # TODO
-# upgrade Dockerfile to add this as an env value
+# update Dockerfile to add this as an env value, and delete line below
 projectDir=/foss_fim
 
 
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 1a5b409c4..b11b6084d 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -124,7 +124,7 @@ then
 fi
 
 # TODO
-# upgrade Dockerfile to add this as an env value
+# update Dockerfile to add this as an env value, and delete line below
 projectDir=/foss_fim
 
 # outputDataDir, srcDir and others come from the Dockerfile
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
index 0f3bb8b69..842835a73 100755
--- a/fim_process_unit_wb.sh
+++ b/fim_process_unit_wb.sh
@@ -61,7 +61,7 @@ echo "---- Start of huc processing for $hucNumber"
 
 # outputDataDir, srcDir and others come from the Dockerfile
 
-export outputRunDataDir=/$outputDataDir/$runName
+export outputRunDataDir=$outputDataDir/$runName
 export outputHucDataDir=$outputRunDataDir/$hucNumber
 export outputBranchDataDir=$outputHucDataDir/branches
 export current_branch_id=0
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index a12494754..05ccfa07e 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -140,9 +140,29 @@ def snap_to_line(row):
     gage_crosswalk.run_crosswalk(input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_table_filename)
 
 """
-python /foss_fim/src/usgs_gage_crosswalk.py -gages /data/outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg -flows /data/outputs/carson_gms_bogus/02020005/branches/3246000305/demDerived_reaches_split_filtered_3246000305.gpkg -cat /data/outputs/carson_gms_bogus/02020005/branches/3246000305/gw_catchments_reaches_filtered_addedAttributes_3246000305.gpkg -dem /data/outputs/carson_gms_bogus/02020005/branches/3246000305/dem_meters_3246000305.tif -dem_adj /data/outputs/carson_gms_bogus/02020005/branches/3246000305/dem_thalwegCond_3246000305.tif -outtable /data/outputs/carson_gms_bogus/02020005/branches/3246000305/usgs_elev_table.csv -b 3246000305
-
-python /foss_fim/src/usgs_gage_crosswalk.py -gages /data/outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg -flows /data/outputs/carson_gms_bogus/02020005/branches/3246000257/demDerived_reaches_split_filtered_3246000257.gpkg -cat /data/outputs/carson_gms_bogus/02020005/branches/3246000257/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg -dem /data/outputs/carson_gms_bogus/02020005/branches/3246000257/dem_meters_3246000257.tif -dem_adj /data/outputs/carson_gms_bogus/02020005/branches/3246000257/dem_thalwegCond_3246000257.tif -outtable /data/outputs/carson_gms_bogus/02020005/branches/3246000257/usgs_elev_table.csv -b 3246000257
-
-python /foss_fim/src/usgs_gage_crosswalk.py -gages /data/outputs/carson_gage_test/04130001/usgs_subset_gages.gpkg -flows /data/outputs/carson_gage_test/04130001/branches/9041000030/demDerived_reaches_split_filtered_9041000030.gpkg -cat /data/outputs/carson_gage_test/04130001/branches/9041000030/gw_catchments_reaches_filtered_addedAttributes_9041000030.gpkg -dem /data/outputs/carson_gage_test/04130001/branches/9041000030/dem_meters_9041000030.tif -dem_adj /data/outputs/carson_gage_test/04130001/branches/904100030/dem_thalwegCond_0941000030.tif -outtable /data/outputs/carson_gage_test/04130001/branches/9041000030/usgs_elev_table.csv -b 9041000030
-"""
\ No newline at end of file
+Examples:
+
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg                    \ 
+    -flows /outputs/carson_gms_bogus/02020005/branches/3246000305/demDerived_reaches_split_filtered_3246000305.gpkg             \ 
+    -cat /outputs/carson_gms_bogus/02020005/branches/3246000305/gw_catchments_reaches_filtered_addedAttributes_3246000305.gpkg  \ 
+    -dem /outputs/carson_gms_bogus/02020005/branches/3246000305/dem_meters_3246000305.tif                                       \ 
+    -dem_adj /outputs/carson_gms_bogus/02020005/branches/3246000305/dem_thalwegCond_3246000305.tif                              \ 
+    -outtable /outputs/carson_gms_bogus/02020005/branches/3246000305/usgs_elev_table.csv                                        \ 
+    -b 32460003 05
+
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg                    \ 
+    -flows /outputs/carson_gms_bogus/02020005/branches/3246000257/demDerived_reaches_split_filtered_3246000257.gpkg             \ 
+    -cat /outputs/carson_gms_bogus/02020005/branches/3246000257/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg  \ 
+    -dem /outputs/carson_gms_bogus/02020005/branches/3246000257/dem_meters_3246000257.tif                                       \ 
+    -dem_adj /outputs/carson_gms_bogus/02020005/branches/3246000257/dem_thalwegCond_3246000257.tif                              \ 
+    -outtable /outputs/carson_gms_bogus/02020005/branches/3246000257/usgs_elev_table.csv                                        \ 
+    -b 32460002 57
+
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gage_test/04130001/usgs_subset_gages.gpkg                    \ 
+    -flows /outputs/carson_gage_test/04130001/branches/9041000030/demDerived_reaches_split_filtered_9041000030.gpkg             \ 
+    -cat /outputs/carson_gage_test/04130001/branches/9041000030/gw_catchments_reaches_filtered_addedAttributes_9041000030.gpkg  \ 
+    -dem /outputs/carson_gage_test/04130001/branches/9041000030/dem_meters_9041000030.tif                                       \ 
+    -dem_adj /outputs/carson_gage_test/04130001/branches/904100030/dem_thalwegCond_0941000030.tif                               \ 
+    -outtable /outputs/carson_gage_test/04130001/branches/9041000030/usgs_elev_table.csv                                        \ 
+    -b 90410000 30
+"""
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 187bb3657..075ff961d 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -136,7 +136,7 @@ def filter_gage_branches(fim_inputs_filename):
         This is an easy way to filter fim_inputs so that only branches with gages will run during fim_process_unit_wb.sh.
 
         example:
-        python3 src/usgs_gage_unit_setup.py -gages x -ahps x -nwm x -o x -huc x -ff /data/outputs/test_output/fim_inputs.csv
+        python3 src/usgs_gage_unit_setup.py -gages x -ahps x -nwm x -o x -huc x -ff /outputs/test_output/fim_inputs.csv
         '''
         assert os.path.isfile(filter_fim_inputs)
         Gage2Branch.filter_gage_branches(filter_fim_inputs)
diff --git a/tools/cache_metrics.py b/tools/cache_metrics.py
index 22ef4d9bf..825b8f4fb 100755
--- a/tools/cache_metrics.py
+++ b/tools/cache_metrics.py
@@ -9,7 +9,7 @@
 
 TEST_CASES_DIR = r'/data/test_cases_new/'  #TODO remove "_new"
 PREVIOUS_FIM_DIR = r'/data/previous_fim'
-OUTPUTS_DIR = r'/data/outputs'
+OUTPUTS_DIR = r'/outputs'
 
 
 def process_alpha_test(args):
diff --git a/tools/copy_test_case_folders.py b/tools/copy_test_case_folders.py
index 4e601e224..c2e16fe23 100644
--- a/tools/copy_test_case_folders.py
+++ b/tools/copy_test_case_folders.py
@@ -89,7 +89,7 @@ def copy_folders(folder_name_list,
 #   and it has to be run on each root folder, one at a time (for now. aka.. no wildcards)
 
 # Sample Usage: 
-#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_20220420.lst -s /outputs/copy_test_synth/ -t /data/outputs/copy_test_synth_combined -a
+#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_20220420.lst -s /outputs/copy_test_synth/ -t /outputs/copy_test_synth_combined -a
 
 #  NOTE the 'a' at the end meaning go ahead create the gms_input.csv. This is normally
 # left for the last folder to be copied over.
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 3920f540d..9f7b6c984 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -75,12 +75,12 @@ def __init__(self, test_id, version, archive=True):
                 ID of the test case in huc8_category format, e.g. `12090201_ble`.
             version : str
                 Version of FIM to which this test_case belongs. This should correspond to the fim directory
-                name in either `/data/previous_fim/` or `/data/outputs/`.
+                name in either `/data/previous_fim/` or `/outputs/`.
             archive : bool
                 If true, this test case outputs will be placed into the `official_versions` folder
                 and the FIM model will be read from the `/data/previous_fim` folder.
                 If false, it will be saved to the `testing_versions/` folder and the FIM model 
-                will be read from the `/data/outputs/` folder.
+                will be read from the `/outputs/` folder.
         
         """
         self.test_id = test_id
@@ -123,12 +123,12 @@ def list_all_test_cases(cls, version, archive, benchmark_categories=[]):
             ----------
             version : str
                 Version of FIM to which this test_case belongs. This should correspond to the fim directory
-                name in either `/data/previous_fim/` or `/data/outputs/`.
+                name in either `/data/previous_fim/` or `/outputs/`.
             archive : bool
                 If true, this test case outputs will be placed into the `official_versions` folder
                 and the FIM model will be read from the `/data/previous_fim` folder.
                 If false, it will be saved to the `testing_versions/` folder and the FIM model 
-                will be read from the `/data/outputs/` folder.
+                will be read from the `/outputs/` folder.
         """
         if not benchmark_categories:
             benchmark_categories = list(cls.MAGNITUDE_DICT.keys())
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 62f3dc137..7875dda5c 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -1,26 +1,26 @@
 {
 	"valid_data": {
-		"outputRunDataDir":  "/data/outputs/fim_unit_test_data_do_not_remove",
-		"subset_nwm_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
-		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"outputRunDataDir":  "/outputs/fim_unit_test_data_do_not_remove",
+		"subset_nwm_lakes": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
+		"subset_nwm_streams": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
 		"hucCode": "05030104",
-		"subset_nwm_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
-		"wbd_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
-		"wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
+		"subset_nwm_headwaters": "/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
+		"wbd_buffer_filename": "/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
+		"wbd_filename": "/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
 		"dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
 		"dem_domain" : "/data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg",
 		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
 		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
-		"subset_nwm_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"subset_nwm_catchments": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
 		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
 		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
-		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
+		"subset_landsea": "/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
 		"nwm_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
-		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
+		"subset_nld_lines": "/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
 		"wbd_buffer_distance": 5000,
 		"levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
-		"subset_levee_protected_areas": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LeveeProtectedAreas_subset.gpkg"
+		"subset_levee_protected_areas": "/outputs/fim_unit_test_data_do_not_remove/05030104/LeveeProtectedAreas_subset.gpkg"
 	}
 }
 
diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json
index a465efc1a..48ce1f23d 100644
--- a/unit_tests/derive_level_paths_params.json
+++ b/unit_tests/derive_level_paths_params.json
@@ -1,15 +1,15 @@
 {
 	"valid_data":
 	{
-		"outputRunDataDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
-		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
-		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"outputRunDataDir" : "/outputs/fim_unit_test_data_do_not_remove/",
+		"in_stream_network": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"out_stream_network": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
 		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"out_stream_network_dissolved": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
+		"catchments": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
 		"verbose": true
 	}	
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
index be247ac5a..5b6370373 100644
--- a/unit_tests/filter_catchments_and_add_attributes_params.json
+++ b/unit_tests/filter_catchments_and_add_attributes_params.json
@@ -1,12 +1,12 @@
 {
 	"valid_data":
 	{
-		"outputRunDataDir":  "/data/outputs/fim_unit_test_data_do_not_remove",
-		"input_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
-		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
-		"output_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
-		"output_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
-	    "wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
+		"outputRunDataDir":  "/outputs/fim_unit_test_data_do_not_remove",
+		"input_catchments_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
+		"input_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
+		"output_catchments_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
+		"output_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
+	    "wbd_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
 		"huc_code": "02020005"
 	}
 }
diff --git a/unit_tests/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json
index a545e6753..f63745ad9 100644
--- a/unit_tests/outputs_cleanup_params.json
+++ b/unit_tests/outputs_cleanup_params.json
@@ -1,7 +1,7 @@
 {
 	"valid_specific_branch_data":
 	{
-		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
+		"src_dir": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
 		"deny_list": "/foss_fim/config/deny_branches.lst",
 		"branch_id": "3246000009",
 		"verbose": true
@@ -9,7 +9,7 @@
 
 	"valid_directory_data":
 	{
-		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
+		"src_dir": "/outputs/fim_unit_test_data_do_not_remove",
 		"deny_list": "/foss_fim/config/deny_branches.lst",
 		"branch_id": "0",
 		"verbose": true
@@ -17,7 +17,7 @@
 
 	"skip_clean":
 	{
-		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
+		"src_dir": "/outputs/fim_unit_test_data_do_not_remove",
 		"deny_list": "None",
 		"branch_id": "",
 		"verbose": true
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
index 24949dcb6..dc0ef15bd 100644
--- a/unit_tests/rating_curve_comparison_params.json
+++ b/unit_tests/rating_curve_comparison_params.json
@@ -3,12 +3,12 @@
 	{
 		"02020005":
 		[
-			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_elev_table.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches",
+			"/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_elev_table.csv",
+			"/outputs/fim_unit_test_data_do_not_remove/02020005/branches",
 			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
+			"/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
+			"/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
+			"/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"02020005",
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
index 6bd115d23..06f4118c2 100644
--- a/unit_tests/split_flows_params.json
+++ b/unit_tests/split_flows_params.json
@@ -1,16 +1,16 @@
 {
 	"valid_data":
 	{
-		"outputRunDataDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
+		"outputRunDataDir" : "/outputs/fim_unit_test_data_do_not_remove/",
 		"max_length": 1500,
 		"slope_min": 0.001,
 		"lakes_buffer_input": 20,
-		"flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
-		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-	    "split_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
-		"split_points_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
-		"wbd8_clp_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
-		"lakes_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_lakes_proj_subset.gpkg",
-		"nwm_streams_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_subset_streams_levelPaths.gpkg"
+		"flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
+		"dem_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+	    "split_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
+		"split_points_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
+		"wbd8_clp_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
+		"lakes_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_lakes_proj_subset.gpkg",
+		"nwm_streams_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_subset_streams_levelPaths.gpkg"
 	}
 }
diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py
index 6b94fd2c7..66788a928 100644
--- a/unit_tests/split_flows_test.py
+++ b/unit_tests/split_flows_test.py
@@ -27,8 +27,8 @@ def setUpClass(self):
     def test_split_flows_success(self):
 
         '''
-        The /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
-        /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
+        The /outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
+        /outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
         If the test is successful, these files will be created.
         '''
 
diff --git a/unit_tests/tools/inundate_gms_params.json b/unit_tests/tools/inundate_gms_params.json
index 236614d6e..ec1d3d55f 100644
--- a/unit_tests/tools/inundate_gms_params.json
+++ b/unit_tests/tools/inundate_gms_params.json
@@ -1,7 +1,7 @@
 {
 	"valid_data_inudation_raster_single_huc":
 	{
-		"hydrofabric_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
+		"hydrofabric_dir": "/outputs/fim_unit_test_data_do_not_remove/",
 		"forecast": "/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv",
 		"num_workers": 1,
 		"hucs": "02020005, 05030104",
diff --git a/unit_tests/tools/inundation_params.json b/unit_tests/tools/inundation_params.json
index 6e2b1d800..cc4685bcc 100644
--- a/unit_tests/tools/inundation_params.json
+++ b/unit_tests/tools/inundation_params.json
@@ -1,10 +1,10 @@
 {
 	"valid_data_inundate_branch":
 	{
-		"rem": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
-		"catchment_poly": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
-		"hydro_table": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/hydroTable_3246000001.csv",
+		"rem": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
+		"catchments": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
+		"catchment_poly": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
+		"hydro_table": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/hydroTable_3246000001.csv",
 	    "forecast": "/data/test_cases/usgs_test_cases/validation_data_usgs/02020005/ptvn6/action/ahps_ptvn6_huc_02020005_flows_action.csv",
 		"mask_type": null,
 		"hucs": null,
diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json
index 2d91c01aa..32f51a347 100644
--- a/unit_tests/usgs_gage_crosswalk_params.json
+++ b/unit_tests/usgs_gage_crosswalk_params.json
@@ -1,12 +1,12 @@
 {
 	"valid_data":
 	{
-		"usgs_gages_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_subset_gages.gpkg",
-		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
-		"input_catchment_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
-		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_meters_3246000005.tif",
-		"dem_adj_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-		"output_table_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/usgs_elev_table.csv",
+		"usgs_gages_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_subset_gages.gpkg",
+		"input_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
+		"input_catchment_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
+		"dem_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_meters_3246000005.tif",
+		"dem_adj_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+		"output_table_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/usgs_elev_table.csv",
 		"branch_id": "3246000005"
 	}
 }
\ No newline at end of file

From 2d5c95c7ac47dec30dae57ca10ba666f61f1b0ac Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 17 Mar 2023 11:43:35 -0400
Subject: [PATCH 220/359] 4.3.2.0: Revise masking levee-protected areas from
 inundation (#845)

See Changelog 4.3.2.0 for full details.
---
 .gitignore                                   |   2 +-
 config/deny_branch_zero.lst                  |   2 +-
 config/deny_branches.lst                     |   2 +-
 config/deny_unit.lst                         |   2 +
 config/params_template.env                   |   6 +-
 data/esri.py                                 | 141 +++++++++++++++
 data/nld/levee_download.py                   | 152 ++++++++++++++++
 data/nld/preprocess_levee_protected_areas.py | 181 -------------------
 docs/CHANGELOG.md                            |  40 ++++
 src/associate_levelpaths_with_levees.py      | 129 +++++++++++++
 src/bash_variables.env                       |   4 +-
 src/burn_in_levees.py                        |   2 +-
 src/clip_vectors_to_wbd.py                   |  16 +-
 src/delineate_hydros_and_produce_HAND.sh     |   2 +-
 src/mask_dem.py                              |  66 +++++--
 src/run_by_branch.sh                         |   2 +-
 src/run_unit_wb.sh                           |  49 +++--
 17 files changed, 574 insertions(+), 224 deletions(-)
 create mode 100644 data/esri.py
 create mode 100755 data/nld/levee_download.py
 delete mode 100644 data/nld/preprocess_levee_protected_areas.py
 create mode 100644 src/associate_levelpaths_with_levees.py

diff --git a/.gitignore b/.gitignore
index b34cb48fd..7ce22521c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,4 @@ config/*.swp
 .vscode/
 **/.DS_Store
 **/*_pytest.py
-
+.private/
diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst
index 34caa2694..1da9b4fcc 100644
--- a/config/deny_branch_zero.lst
+++ b/config/deny_branch_zero.lst
@@ -28,7 +28,7 @@ demDerived_streamPixels_ids_{}_dist.tif
 dem_burned_{}.tif
 dem_burned_filled_{}.tif
 dem_lateral_thalweg_adj_{}.tif
-#dem_meters_{}.tif
+dem_meters_{}.tif
 dem_thalwegCond_{}.tif
 flowaccum_d8_burned_filled_{}.tif
 #flowdir_d8_burned_filled_{}.tif
diff --git a/config/deny_branches.lst b/config/deny_branches.lst
index 4616c3717..13af6b41a 100644
--- a/config/deny_branches.lst
+++ b/config/deny_branches.lst
@@ -45,8 +45,8 @@ headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
 LandSea_subset_{}.tif
+levee_levelpaths.csv
 nld_rasterized_elev_{}.tif
-nld_subset_levees_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
diff --git a/config/deny_unit.lst b/config/deny_unit.lst
index 6f0283f76..cb7e76754 100644
--- a/config/deny_unit.lst
+++ b/config/deny_unit.lst
@@ -9,8 +9,10 @@ agree_smogrid.tif
 agree_smogrid_allo.tif
 agree_smogrid_dist.tif
 agree_smogrid_zerod.tif
+#3d_nld_subset_levees_burned.gpkg
 #branch_id.lst
 #branch_polygons.gpkg
+dem_meters.tif
 #LandSea_subset.gpkg
 #LeveeProtectedAreas_subset.gpkg
 #nld_subset_levees.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index 2c8eb2a1b..5c0754368 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -4,6 +4,7 @@
 export negative_burn_value=1000
 export agree_DEM_buffer=70
 export wbd_buffer=5000
+export levee_buffer=1000
 export thalweg_lateral_elev_threshold=3
 
 #### geospatial parameters ####
@@ -26,8 +27,9 @@ export branch_buffer_distance_meters=7000
 export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.
 export branch_zero_id="0"
 
-#### mask levee-protected areas from relative elevation model
-export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from relative elevation model
+#### mask levee-protected areas from DEM
+export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from DEM
+export levee_id_attribute=SYSTEM_ID
 
 #### estimating bankfull stage in SRCs ####
 export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
diff --git a/data/esri.py b/data/esri.py
new file mode 100644
index 000000000..83e26341c
--- /dev/null
+++ b/data/esri.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+import requests
+import pandas as pd
+import geopandas as gpd
+from tqdm import tqdm
+
+class ESRI_REST(object):
+    """
+    This class was built for querying ESRI REST endpoints for the purpose of downloading datasets.
+    See /data/nld/levee_download.py for an example useage.
+    """
+    
+    def __init__(self, query_url, params, verbose=True):
+        self.query_url = query_url
+        self.params = params
+        self.verbose = verbose
+        self.exceededTransferLimit = True
+        self.feature_count = 0
+    
+    @classmethod
+    def query(cls, query_url:str, save_file:str=None, **kwargs):
+        '''
+        Classmethod for easily queries on ESRI feature services. See /data/nld/levee_download.py for example usage.
+
+        Parameters
+        ----------
+        query_url: str
+            URL to query. This should have the layer # and 'query' at the end, e.g.
+            https://ags03.sec.usace.army.mil/server/rest/services/NLD2_PUBLIC/FeatureServer/15/query
+        save_file: str
+            Optional. Location to save the output geopackage. This method will not return
+            a geodataframe if this parameter is set.
+        **kwargs
+            All kwargs get passed to the service query. Here's an example of some standard ones:
+            `f="json", where="1=1", returnGeometry="true", outFields="*", outSR="5070"`
+
+        Returns
+        -------
+        gdf_complete: geopandas.GeoDataFrame
+            GeoDataFrame containing all of the features returned by the query. `None` is returned if the 
+            save_file parameter is set.
+        '''
+        # Query the input URL using the kwargs as URL parameters
+        rest_call = cls(query_url, kwargs)
+        gdf_complete = rest_call._query_rest()
+        # Save geodataframe as geopackage
+        if save_file:
+            gdf_complete.to_file(save_file, driver="GPKG", index=False)
+        else:
+            return gdf_complete
+        
+    def _query_rest(self):
+        '''
+        Method that sets up multiple REST calls when there are more features than the transfer limit
+        set by the feature service and concatenates each response into a single GeoDataFrame.
+
+        Returns
+        -------
+        gdf_complete: geopandas.GeoDataFrame
+            GeoDataFrame containing all of the features returned by the query. 
+        '''
+        gdf_list = []
+        record_count = 0
+        backup_counter = 0
+        if self.verbose:
+            self._meta_query()
+            print("-------------------------------------")
+            print(f"Service name: {self.metadata['name']}")
+            print(f"Features returned: {self.feature_count}")
+            if 'resultRecordCount' in self.params.keys():
+                print(f"Request max record count: {self.params['resultRecordCount']}")
+                print(f"Total API calls: {-(self.feature_count//-self.params['resultRecordCount'])}")
+            else:
+                print(f"Service max record count: {self.metadata['maxRecordCount']}")
+                print(f"Total API calls: {-(self.feature_count//-self.metadata['maxRecordCount'])}")
+        # Call the REST API repeatedly until all of the features have been collected, i.e. the transfer
+        # limit has no longer been exceeded
+        with tqdm(total=self.feature_count, desc='Feature download progress', disable=not self.verbose) as pbar:
+            while (self.exceededTransferLimit) and (backup_counter < 9999):
+                # Set the resultOffset to the number of records that's already been downloaded
+                self.params['resultOffset'] = record_count
+                sub_gdf = self._sub_query(self.params)
+                gdf_list.append(sub_gdf)
+                record_count += len(sub_gdf)
+                backup_counter += 1
+                pbar.update(len(sub_gdf))
+        # Concatenate all responses into a single geodataframe
+        gdf_complete = pd.concat(gdf_list)
+        return gdf_complete
+
+    def _sub_query(self, params):
+        '''
+        This method calls the REST API.
+
+        Parameters
+        ----------
+        params: dict
+            Parameters for the rest query.
+
+        Returns
+        -------
+        sub_gdf: geopandas.GeoDataFrame
+            GeoDataFrame containing features returned by the query. FYI This may not be the complete
+            dataset if the 'exceededTransferLimit' == True
+        '''
+        self._api_call(self.query_url, params)
+        # Set exceededTransferLimit if there need to be another sub_request
+        r_dict = self.response.json()
+        if 'exceededTransferLimit' in r_dict.keys():
+            self.exceededTransferLimit = r_dict['exceededTransferLimit']
+        # This very nondescript error was returned when querying a polygon layer.
+        # Setting resultRecordCount to a lower value fixed the error.
+        elif 'error' in r_dict.keys():
+            print("There was an error with the query. It may have been caused by requesting too many features. Try setting resultRecordCount to a lower value.")
+            raise Exception(r_dict['error']['message'], f"code: {r_dict['error']['code']}")
+        else:
+            self.exceededTransferLimit = False
+        # Read the response into a GeoDataFrame
+        sub_gdf = gpd.read_file(self.response.text)
+        return sub_gdf
+    
+    def _api_call(self, url, params=None):
+        '''
+        Helper method for calling the API and checking that the response is ok.
+
+
+        '''
+        self.response = requests.get(url, params=params)
+        if not self.response.ok:
+            raise Exception(f"The following URL recieved a bad response.\n{self.response.url}")
+    
+    def _meta_query(self):
+
+        # Get the service metadata
+        self._api_call(self.query_url[:self.query_url.rfind('query')], self.params)
+        self.metadata = self.response.json()
+        # Get the record count returned by the query
+        params = self.params.copy()
+        params['returnCountOnly'] = "true"
+        self._api_call(self.query_url, params)
+        self.feature_count = self.response.json()['count']
\ No newline at end of file
diff --git a/data/nld/levee_download.py b/data/nld/levee_download.py
new file mode 100755
index 000000000..c7f1f41e9
--- /dev/null
+++ b/data/nld/levee_download.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import re
+import geopandas as gpd
+from tqdm import tqdm
+from datetime import datetime
+from shapely.geometry import LineString, MultiLineString
+
+sys.path += ['/foss_fim/src', '/foss_fim/data', '/foss_fim/tools']
+from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
+from tools_shared_variables import INPUTS_DIR
+from esri import ESRI_REST
+
+epsg_code = re.search('\d+$', DEFAULT_FIM_PROJECTION_CRS).group()
+today = datetime.now().strftime('%y%m%d')
+nld_vector_output = os.path.join(INPUTS_DIR, 'nld_vectors', f'System_Routes_NLDFS_5070_{today}.gpkg')
+processed_nld_vector = os.path.join(INPUTS_DIR, 'nld_vectors', f'3d_nld_preprocessed_{today}.gpkg')
+nld_protected_areas = os.path.join(INPUTS_DIR, 'nld_vectors', f'Leveed_Areas_NLDFS_5070_{today}.gpkg')
+
+def download_nld_lines():
+    '''
+    First main function call for this module. Downloads levees from the National Levee Database 
+    ESRI service, save the raw output for use in the levee masking algorithm, processes
+    the levees to remove lines and vertices that have no elevation data, and saves the preprocessed
+    levee geopackage for use in levee burning. NOTE: In order for the files generated by this script to be used, update the date (today) in bash_variables.env for input_NLD, input_levees_preprocessed, and input_nld_levee_protected_areas.
+    '''
+
+    # Query REST service to download levee 'system routes'
+    print("Downloading levee lines from the NLD...")
+    nld_url = "https://ags03.sec.usace.army.mil/server/rest/services/NLD2_PUBLIC/FeatureServer/15/query"
+    levees = ESRI_REST.query(nld_url, 
+            f="json", where="1=1", returnGeometry="true", outFields="*", outSR=epsg_code, returnZ="true")
+                   
+    # Write levees to a single geopackage
+    levees.to_file(nld_vector_output, index=False, driver='GPKG')
+    print(f"Levees written to file:\n{nld_vector_output}")
+
+    # Spatial join to huc2
+    print('Spatial join levees to HUC-2')
+    huc2 = gpd.read_file(os.path.join(INPUTS_DIR, 'wbd', 'WBD_National_EPSG_5070.gpkg'), layer='WBDHU2')
+    levees = gpd.sjoin(levees, huc2[['HUC2', 'geometry']], how='left')
+
+    # Preprocess levees to remove features and vertices with no elevation
+    print('Preprocess levees to remove features and vertices with no elevation ')
+    process_levee_lines(levees, out_levees=processed_nld_vector)
+
+
+def process_levee_lines(levee_gdf:gpd.GeoDataFrame, out_levees:str):
+    '''
+    Function for processing levee lines prior to rasterization and burning into 
+    the DEM. NOTE: Do not use the output of this function for the levee protected
+    area masking. This dataset will be incomplete since it filters out some levees
+    that have no z-values.
+
+    Parameters
+    ----------
+    levee_lines: gpd.GeoDataFrame
+        Raw NLD vectors file.
+    out_levees: str
+        Path to right preprocessed levees.
+    '''
+
+    # Filter vertices that have z-values less than the minimum from levee geometry
+    tqdm.pandas(desc='Removing null elevations')
+    levee_gdf['geometry'] = levee_gdf.progress_apply(lambda row: remove_nulls(row.geometry, row.HUC2), axis=1)
+    # Remove levees that have empty geometries resulting from the previous filter
+    levee_gdf = levee_gdf[~levee_gdf.is_empty]
+    levee_gdf.to_file(out_levees, index=False, driver='GPKG')
+    print(f"Preprocessed levees written to \n{out_levees}")
+
+
+def remove_nulls(geom:LineString, huc:str):
+    '''
+    Removes vertices from shapely LineString `geom` if they are less than `min_z`.
+
+    Parameters
+    ----------
+    geom: shapely.geometry.LineString
+        Shapely geometry from which to filter vertices.
+    huc: str
+        HUC. Can be any digit 2 or greater.
+
+    Returns
+    -------
+    out_geom: shapely.geometry.LineString
+        Filtered geometry.
+    '''
+    # Set min z based on HUC2
+    huc2 = huc[:2] # works with any HUC digit code
+    if huc2 in ['01','02','03','12']: # Coastal HUCs may have values near 0
+        min_z = 0.01
+    elif huc2 == '08': # Louisana below sea level
+        min_z = -10.0
+    else:
+        min_z = 1.0 # Default set to 1 ft
+    # Loop through the vertices
+    out_geom = []
+    part_geom = []
+    skipped_vert = 0
+    max_skipped_vert = 5
+    for coord in geom.coords:
+        skip_flag = False
+        if coord[2] > min_z:
+            # Convert units from feet to meters
+            part_geom.append(tuple([coord[0], coord[1], coord[2]*0.3048]))
+        elif skipped_vert < max_skipped_vert:
+            # Allows a few (5) vertices to be skipped without forcing a multipart break.
+            # This enables short sections of roads that cross levees to have the levee elevations
+            # burned in to account for temporary flood walls not captured in the data.
+            skip_flag = True
+            skipped_vert += 1
+        elif (len(part_geom) > 1) and (not skip_flag):   # Create a multipart geometry when there's a break in z-values
+            out_geom.append(LineString(part_geom))
+            part_geom = []
+            skipped_vert = 0
+    # Append the last segment
+    if len(part_geom) > 1:
+        out_geom.append(LineString(part_geom))
+    # Compile LineString geometries into one multipart geometry
+    if len(out_geom) >= 2:
+        return MultiLineString(out_geom)
+    elif (len(out_geom) == 1) and (len(out_geom[0].coords) > 1):
+        return MultiLineString(out_geom)
+    else:
+        return None
+
+def download_nld_poly():
+    '''
+    Second main function call for this module. Downloads levee protected areas from the National Levee Database 
+    ESRI service and saves the raw output for use in the levee masking algorithm.
+    '''
+    # Query REST service to download levee 'system routes'
+    print("Downloading levee protected areas from the NLD...")
+    nld_area_url = "https://ags03.sec.usace.army.mil/server/rest/services/NLD2_PUBLIC/FeatureServer/14/query"
+    # FYI to whomever takes the time to read this code, the resultRecordCount had to be set on this query because
+    # the service was returning an error that turned out to be caused by the size of the request. Running the 
+    # default max record count of 5000 was too large for polygons, so using resultRecordCount=2000 prevents the error.
+    leveed_areas = ESRI_REST.query(nld_area_url, 
+            f="json", where="1=1", returnGeometry="true", outFields="*", outSR=epsg_code, resultRecordCount=2000)
+                   
+    # Write levees to a single geopackage
+    leveed_areas.to_file(nld_protected_areas, index=False, driver='GPKG')
+    print(f"Levees written to file:\n{nld_protected_areas}")
+
+
+if __name__ == '__main__':
+
+    download_nld_lines()
+
+    download_nld_poly()
diff --git a/data/nld/preprocess_levee_protected_areas.py b/data/nld/preprocess_levee_protected_areas.py
deleted file mode 100644
index d3f20a351..000000000
--- a/data/nld/preprocess_levee_protected_areas.py
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import geopandas as gpd
-import logging
-import os
-import pathlib
-import sys
-
-from datetime import datetime
-
-sys.path.append('/foss_fim/src')
-from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
-import utils.shared_functions as sf
-from utils.shared_functions import FIM_Helpers as fh
-
-def preprocess_levee_protected_areas(source_file_name_and_path,
-                                     target_output_folder_path = '',
-                                     target_output_filename = 'levee_protected_areas.gpkg',
-                                     projection_crs_id = DEFAULT_FIM_PROJECTION_CRS,
-                                     overwrite = False):
-                                     
-    '''
-    Overview
-    ----------
-    This takes a geojson file in (assuming it has geometry), converts it to a gkpg,
-    and converts it's projection (id) (defaulting to EPSG:5070)
-    
-    Parameters:
-    ----------
-        - source_file_name_and_path (str):
-            Name and location of the geojson file. ie) /data/input_temp/LeveedArea.geojson
-        - target_output_folder_path (str):
-            Location where the output gkpg will be saved (defaults to input folder)
-        - target_output_filename (str):
-            Output file name (defaults to levee_protected_areas.gpkg)
-        - projection_crs_id (str):
-            CRS ID that it will be reprojected to (defaults to EPSG:5070). 
-            Remember.. projections can and often do get changed during clipping.
-            
-    Returns:
-    ----------
-        None
-    ''' 
-
-    # -------------------
-    # Validation and setup variables
-    
-    # source file
-    if (source_file_name_and_path is None) or (source_file_name_and_path == ''):
-        raise ValueError("source file name and path can not be empty")
-    source_file_name_and_path = source_file_name_and_path.strip()
-    
-    if (not os.path.exists(source_file_name_and_path)):
-        raise FileNotFoundError("levee geojson source file not found.")
-    
-    source_file_extension = pathlib.Path(source_file_name_and_path).suffix
-    if (source_file_extension.lower() != ".geojson"):
-        raise ValueError("source file does not appear to be a geojson file")
-
-    # target file and path
-    if (target_output_folder_path is not None) and (target_output_folder_path != ''):
-        if (not os.path.exists(target_output_folder_path)):
-            os.mkdir(target_output_folder_path)
-    else:
-        target_output_folder_path = os.path.dirname(source_file_name_and_path)
-        
-    if (target_output_filename is not None) and (target_output_filename != ''):
-        target_file_extension = pathlib.Path(target_output_filename).suffix
-        if (target_file_extension.lower() != ".gpkg"):
-            raise ValueError("target file name does not appear to be a gpkg file")
-    else:
-        target_output_filename = 'levee_protected_areas.gpkg'
-
-    target_output_file = os.path.join(target_output_folder_path, target_output_filename)
-    if (not overwrite) and os.path.exists(target_output_file):
-        raise Exception("Target output file of {target_output_file} already exists. "\
-                        "Add overwrite flag or change target path or target file name")
-       
-    # projection crs
-    if (projection_crs_id is None) or (projection_crs_id == ''):
-        projection_crs_id = DEFAULT_FIM_PROJECTION_CRS
-
-    # -------------------
-    # setup logs
-    #start_time = datetime.now()
-    print("=======================")
-    print("Starting preprocessing levee protected area geojson file")
-   
-    __setup_logger(target_output_folder_path)
-    #sf.setup_logger('preprocess_level_protected_areas', target_output_folder_path)
-    logging.info(f"Downloading to {target_output_folder_path}")
-    
-    # -------------------
-    # processing
-    
-    # load geojson into a dataframe
-    # Note:: the logs record a weird line from geopandas
-    #    Skipping field stewardOrgIds: invalid type 1 but seems to be fine.
-    gdf = gpd.read_file(source_file_name_and_path)
-    if (len(gdf) == 0) or (gdf.empty):
-        raise Exception("Source geojson file appears to be empty or invalid")
-   
-    # check current CRS
-    gdf_crs = str(gdf.crs)
-    #print(gdf_crs)
-    
-    # don't change it if the projection's already match    
-    if (gdf_crs.lower() != projection_crs_id.lower()):
-        gdf.to_crs(projection_crs_id, inplace=True)           
-    
-    gdf.to_file(target_output_file, driver="GPKG", index=False)
-    
-    msg = f"Preprocessing Complete: file created at {target_output_file}"
-    print(msg)
-    logging.info(msg)
-    
-    #end_time = datetime.now()
-    #logging.info(fh.print_date_time_duration(start_time, end_time))
-        
-def __setup_logger(output_folder_path):
-    
-    root_file_name = "preprocess_level_protected_areas"
-    start_time = datetime.now()
-    file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
-    log_file_name = f"{root_file_name}-{file_dt_string}.log"
-
-    log_file_path = os.path.join(output_folder_path, log_file_name)
-
-    file_handler = logging.FileHandler(log_file_path)
-    file_handler.setLevel(logging.INFO)
-    console_handler = logging.StreamHandler()
-    console_handler.setLevel(logging.DEBUG)
-
-    logger = logging.getLogger()
-    logger.addHandler(file_handler)
-    logger.setLevel(logging.DEBUG)    
-    
-    logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
-    logging.info("----------------")
-
-        
-
-if __name__ == '__main__':
-    
-    # Acquire Notes as of Nov 8, 2022
-    '''
-    Working with gkpg''s in code is 10 to 20 times faster than working directly with geojson files
-    so we will convert it. We have to download the geojson file by hand for now as USACE doesn't have an API
-    that allows you to download the entire CONUS+. They do have api's that download levee geojson files at a huc
-    level but they don't have geometry. You have to call once for the geojson per huc, then go back for the geometries.
-    For now, we will stay with downloading it by hand.
-    '''
-    
-    # Instructions for Downloading (as of Nov 8, 2022)
-    #  1) go to :https://levees.sec.usace.army.mil/#/levees/search/&viewType=map&resultsType=systems&advanced=true&hideList=false&eventSystem=false
-    #  2) Click on the "Download Data" button in the top right corner and wait for a bit. It takes a bit as it is 200mg but it isn't bad.
-    #     It downloads one big LeevedArea.geojson with geometry.
-
-    # Sample Usage: python3 /foss_fim/data/nld/preprocess_levee_protected_areas.py -s /data/inputs/LeveedArea.geojson
-    
-    parser = argparse.ArgumentParser(description='Preprocess the NLD levee protected areas geojson into a gkpg.')
-
-    parser.add_argument('-s','--source-file-name-and-path', help='location of the geojson that will converted.', 
-                        required=True)
-    
-    parser.add_argument('-t','--target-output-folder-path', help='location of where the gkpg file'\
-                        ' will be saved (defaults to input folder path).', required=False)
-
-    parser.add_argument('-f', '--target-output-filename', help='output gpkg file name'\
-                        ' (defaults to levee_protected_area.gkpg).', required=False)
-
-    parser.add_argument('-c','--projection-crs-id', help='Reproject to CRS ID (default = EPSG:5070).', 
-                        required=False)
-
-    parser.add_argument('-o','--overwrite', help='Overwrite if the file already exists.',
-                        required=False,  action='store_true')
-
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-
-    preprocess_levee_protected_areas(**args)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0f29bb68d..fe848e32d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,46 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.2.0 - 2023-03-15 - [PR#845](https://github.com/NOAA-OWP/inundation-mapping/pull/845)
+
+This merge revises the methodology for masking levee-protected areas from inundation. It accomplishes two major tasks: (1) updates the procedure for acquiring and preprocessing the levee data to be burned into the DEM and (2) revises the way levee-protected areas are masked from branches.
+
+(1) There are now going to be two different levee vector line files in each HUC. One (`nld_subset_levees_burned.gpkg`) for the levee elevation burning and one (`nld_subset_levees.gpkg`) for the levee-level-path assignment and masking workflow.
+
+(2) Levee-protected areas are masked from inundation based on a few methods:
+  - Branch 0: All levee-protected areas are masked.
+  - Other branches: Levee-protected areas are masked from the DEMs of branches for level path(s) that the levee is protecting against by using single-sided buffers alongside each side of the levee to determine which side the levee is protecting against (the side opposite the associated levee-protected area).
+
+### Additions
+
+- `.gitignore`: Adds `.private` folder for unversioned code.
+- `data/`
+    - `esri.py`: Class for querying and downloading ESRI feature services.
+    - `nld/`
+        - `levee_download.py`: Module that handles downloading and preprocessing levee lines and protected areas from the National Levee Database.
+- `src/associate_levelpaths_with_levees.py`: Associates level paths with levees using single-sided levee buffers and writes to CSV to be used by `src/mask_dem.py`
+
+### Changes
+
+- `.config/`
+    - `deny_branch_zero.lst`: Adds `dem_meters_{}.tif`.
+    - `deny_branches.lst`: Adds `levee_levelpaths.csv` and removes `nld_subset_levees_{}.tif`.
+    - `deny_unit.lst`: Adds `dem_meters.tif`.
+    - `params_template.env`: Adds `levee_buffer` parameter for levee buffer size/distance in meters and `levee_id_attribute`.
+- `src/`
+    - `bash_variables.env`: Updates `input_nld_levee_protected_areas` and adds `input_NLD` (moved from `run_unit_wb.sh`) and `input_levees_preprocessed` environment. .variables
+    - `burn_in_levees.py`: Removed the unit conversion from feet to meters because it's now being done in `levee_download.py`.
+    - `clip_vectors_to_wbd.py`: Added the new levee lines for the levee-level-path assignment and masking workflow.
+    - `delineate_hydros_and_produce_HAND.sh`: Updates input arguments.
+    - `mask_dem.py`: Updates to use `levee_levelpaths.csv` (output from `associate_levelpaths_with_levees.py`) to mask branch DEMs.
+    - `run_by_branch.sh`: Clips `dem_meters.tif` to use for branches instead of `dem_meters_0.tif` since branch 0 is already masked.
+    - `run_unit_wb.sh`: Added inputs to `clip_vectors_to_wbd.py`. Added `associate_levelpaths_with_levees.py`. Processes `dem_meters.tif` and then makes a copy for branch 0. Moved `deny_unit.lst` cleanup to after branch processing.
+
+### Removals
+- `data/nld/preprocess_levee_protected_areas.py`: Deprecated.
+
+<br/><br/>
+
 ## v4.3.1.0 - 2023-03-10 - [PR#834](https://github.com/NOAA-OWP/inundation-mapping/pull/834)
 
 Change all occurances of /data/outputs to /outputs to honor the correct volume mount directory specified when executing docker run.
diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py
new file mode 100644
index 000000000..7abcb411c
--- /dev/null
+++ b/src/associate_levelpaths_with_levees.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+
+def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str, leveed_areas_filename:str, levelpaths_filename:str, branch_id_attribute:str, levee_buffer:float, out_filename:str):
+    """
+    Finds the level path associated with each levee. Ignores level paths that cross a levee exactly once.
+
+    Parameters
+    ----------
+    levees_filename: str
+        Path to levees file.
+    levee_id_attribute: str
+        Name of levee ID attribute.
+    leveed_areas_filename: str
+        Path to levee-protected areas file.
+    levelpaths_filename: str
+        Path to level paths file.
+    branch_id_attribute: str
+        Name of branch ID attribute.
+    levee_buffer: float
+        Distance to buffer from levee.
+    out_filename: str
+        Path to write output CSV file.    
+    """
+
+    if os.path.exists(levees_filename):
+        # Read in geodataframes
+        levees = gpd.read_file(levees_filename)
+        leveed_areas = gpd.read_file(leveed_areas_filename)
+        levelpaths = gpd.read_file(levelpaths_filename)
+
+        levees[levee_id_attribute] = levees[levee_id_attribute].astype(int)
+        leveed_areas[levee_id_attribute] = leveed_areas[levee_id_attribute].astype(int)
+        levelpaths[branch_id_attribute] = levelpaths[branch_id_attribute].astype(int)
+
+        # Buffer each side of levee line
+        levees_buffered_left = levees.copy()
+        levees_buffered_right = levees.copy()
+        levees_buffered_left.geometry = levees.buffer(levee_buffer, single_sided=True)
+        levees_buffered_right.geometry = levees.buffer(-levee_buffer, single_sided=True)
+
+        # Intersect leveed areas with single-sided levee buffers
+        leveed_left = gpd.overlay(levees_buffered_left, leveed_areas, how='intersection')
+        leveed_right = gpd.overlay(levees_buffered_right, leveed_areas, how='intersection')
+
+        # Find leveed areas not intersected by either buffer
+        levees_not_found = leveed_areas[~leveed_areas['SYSTEM_ID'].isin(pd.concat([leveed_left['SYSTEM_ID_1'], leveed_right['SYSTEM_ID_1']]).values)]
+
+        # Associate levees and leveed areas
+        matches_left = np.where(leveed_left[f'{levee_id_attribute}_1']==leveed_left[f'{levee_id_attribute}_2'])[0]
+        matches_right = np.where(leveed_right[f'{levee_id_attribute}_1']==leveed_right[f'{levee_id_attribute}_2'])[0]
+
+        leveed_left = leveed_left.loc[matches_left]
+        leveed_right = leveed_right.loc[matches_right]
+
+        # Get area of associated leveed areas
+        leveed_left['leveed_area'] = leveed_left.area
+        leveed_right['leveed_area'] = leveed_right.area
+
+        leveed_left = leveed_left[[f'{levee_id_attribute}_1', 'leveed_area', 'geometry']]
+        leveed_right = leveed_right[[f'{levee_id_attribute}_1', 'leveed_area', 'geometry']]
+
+        # Merge left and right levee protected areas
+        leveed = leveed_left.merge(leveed_right, on=f'{levee_id_attribute}_1', how='outer', suffixes=['_left', '_right'])
+
+        # Set unmatched areas to zero
+        leveed.loc[np.isnan(leveed['leveed_area_left']), 'leveed_area_left'] = 0
+        leveed.loc[np.isnan(leveed['leveed_area_right']), 'leveed_area_right'] = 0
+
+        # Determine which side the levee is protecting (opposite of levee protected area)
+        leveed['levee_side'] = np.where(leveed['leveed_area_left'] < leveed['leveed_area_right'], 'left', 'right')
+
+        # Split into sides
+        left_ids = leveed.loc[leveed['levee_side']=='left', f'{levee_id_attribute}_1']
+        right_ids = leveed.loc[leveed['levee_side']=='right', f'{levee_id_attribute}_1']
+
+        # Associate level paths with levee buffers
+        levee_levelpaths_left = gpd.sjoin(levees_buffered_left, levelpaths)
+        levee_levelpaths_right = gpd.sjoin(levees_buffered_right, levelpaths)
+
+        levee_levelpaths_left = levee_levelpaths_left[[levee_id_attribute, branch_id_attribute]]
+        levee_levelpaths_right = levee_levelpaths_right[[levee_id_attribute, branch_id_attribute]]
+
+        # Select streams on the correct side of levee
+        levee_levelpaths_left = levee_levelpaths_left[levee_levelpaths_left[levee_id_attribute].isin(left_ids)]
+        levee_levelpaths_right = levee_levelpaths_right[levee_levelpaths_right[levee_id_attribute].isin(right_ids)]
+
+        # Add level paths to levees not found
+        levees_not_found.geometry = levees_not_found.buffer(2*levee_buffer)
+        levees_not_found = gpd.sjoin(levees_not_found, levelpaths)
+
+        # Join left and right
+        out_df =  pd.concat([levee_levelpaths_right[[levee_id_attribute, branch_id_attribute]], levee_levelpaths_left[[levee_id_attribute, branch_id_attribute]], levees_not_found[[levee_id_attribute, branch_id_attribute]]]).drop_duplicates().reset_index(drop=True)
+
+        # Remove levelpaths that cross the levee exactly once
+        for j, row in out_df.iterrows():
+            # Intersect levees and levelpaths
+            row_intersections = gpd.overlay(levees[levees[levee_id_attribute] == row[levee_id_attribute]], levelpaths[levelpaths[branch_id_attribute] == row[branch_id_attribute]], how='intersection', keep_geom_type=False)
+
+            # Convert MultiPoint to Point
+            row_intersections = row_intersections.explode()
+
+            # Select Point geometry type
+            row_intersections = row_intersections[row_intersections.geom_type =='Point']
+
+            if len(row_intersections) == 1:
+                out_df = out_df.drop(j)
+                
+        out_df.to_csv(out_filename, columns=[levee_id_attribute, branch_id_attribute], index=False)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Associate level paths with levees')
+    parser.add_argument('-nld','--levees-filename', help='NLD levees filename', required=True, type=str)
+    parser.add_argument('-l','--levee-id-attribute', help='Levee ID attribute name', required=True, type=str)
+    parser.add_argument('-out','--out-filename', help='out CSV filename', required=True, type=str)
+    parser.add_argument('-s', '--levelpaths-filename', help='Level path layer filename', required=True, type=str)
+    parser.add_argument('-b','--branch-id-attribute', help='Level path ID attribute name', required=True, type=str)
+    parser.add_argument('-lpa', '--leveed-areas-filename', help='NLD levee-protected areas filename', required=True, type=str)
+    parser.add_argument('-w', '--levee-buffer', help='Buffer width (in meters)', required=True, type=float)
+
+    args = vars(parser.parse_args())
+
+    associate_levelpaths_with_levees(**args)
diff --git a/src/bash_variables.env b/src/bash_variables.env
index c2ba361ab..8d48ce733 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -4,7 +4,9 @@ export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
 export input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
 export input_DEM_domain=$inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
 export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
-export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Levee_protected_areas.gpkg
+export input_NLD=$inputDataDir/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
+export input_levees_preprocessed=$inputDataDir/nld_vectors/3d_nld_preprocessed_230314.gpkg
+export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
 export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
 export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
 export input_nwm_headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
diff --git a/src/burn_in_levees.py b/src/burn_in_levees.py
index 93acbc1e4..ea8acd92c 100755
--- a/src/burn_in_levees.py
+++ b/src/burn_in_levees.py
@@ -18,7 +18,7 @@ def burn_in_levees(dem_filename,nld_filename,out_dem_filename):
 
     no_data = nld.nodata
 
-    nld_m = np.where(nld_data == int(no_data), -9999.0, (nld_data*0.3048).astype(rasterio.float32))
+    nld_m = np.where(nld_data == int(no_data), -9999.0, (nld_data).astype(rasterio.float32))
 
     dem_profile = dem.profile.copy()
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index f311fc196..5ff1c8c9a 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -4,7 +4,7 @@
 import geopandas as gpd
 import argparse
 import rasterio as rio
-
+import fiona
 from shapely.geometry import MultiPolygon,Polygon
 from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 from utils.shared_functions import getDriver, mem_profile
@@ -22,17 +22,17 @@ def subset_vector_layers(subset_nwm_lakes,
                          nwm_catchments,
                          subset_nwm_catchments,
                          nld_lines,
+                         nld_lines_preprocessed,
                          landsea,
                          nwm_streams,
                          subset_landsea,
                          nwm_headwaters,
                          subset_nld_lines,
+                         subset_nld_lines_preprocessed,
                          wbd_buffer_distance,
                          levee_protected_areas,
                          subset_levee_protected_areas):
         
-    hucUnitLength = len(str(hucCode))
-
     print("Getting Cell Size", flush=True)
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
@@ -95,6 +95,12 @@ def subset_vector_layers(subset_nwm_lakes,
         nld_lines.to_file(subset_nld_lines, driver = getDriver(subset_nld_lines), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del nld_lines
 
+    # Preprocced levee lines for burning
+    nld_lines_preprocessed = gpd.read_file(nld_lines_preprocessed, mask = wbd_buffer)
+    if not nld_lines_preprocessed.empty:
+        nld_lines_preprocessed.to_file(subset_nld_lines_preprocessed, driver = getDriver(subset_nld_lines_preprocessed), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+    del nld_lines_preprocessed
+
     # Subset NWM headwaters
     print("Subsetting NWM Headwater Points", flush=True)
     nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer)
@@ -160,6 +166,8 @@ def subset_vector_layers(subset_nwm_lakes,
                         required=True)
     parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path',
                         required=True)
+    parser.add_argument('-rp','--nld-lines-preprocessed', help='Levee vectors to use for DEM burning',
+                        required=True)
     parser.add_argument('-v','--landsea', help='LandSea - land boundary',
                         required=True)	 
     parser.add_argument('-w','--nwm-streams', help='NWM flowlines',
@@ -170,6 +178,8 @@ def subset_vector_layers(subset_nwm_lakes,
                         required=True)	 
     parser.add_argument('-z','--subset-nld-lines', help='Subset of NLD levee vectors for HUC',
                         required=True)
+    parser.add_argument('-zp','--subset-nld-lines-preprocessed', help='Subset of NLD levee vectors for burning elevations into DEMs',
+                        required=True)
     parser.add_argument('-wb','--wbd-buffer-distance', help='WBD Mask buffer distance', 
                         required=True, type=int)
     parser.add_argument('-lpf','--levee-protected-areas', 
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index f92d39e61..51cbad515 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -11,7 +11,7 @@ if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtec
     echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/mask_dem.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif  -s $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -i $current_branch_id -b0 $branch_zero_id
+    python3 -m memory_profiler $srcDir/mask_dem.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -b $branch_id_attribute -i $current_branch_id -b0 $branch_zero_id -csv $outputHucDataDir/levee_levelpaths.csv -l $levee_id_attribute
     Tcount
 fi
 
diff --git a/src/mask_dem.py b/src/mask_dem.py
index f7310c74c..7cfab2789 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import pandas as pd
 import geopandas as gpd
 import fiona
 import rasterio as rio
@@ -9,20 +10,56 @@
 from utils.shared_functions import mem_profile
 
 @mem_profile
-def mask_dem(dem_filename, nld_filename, out_dem_filename, stream_layer, branch_id_attribute, branch_id, order_attribute, branch_zero_id):
+def mask_dem(dem_filename:str, nld_filename:str, levee_id_attribute:str, out_dem_filename:str, branch_id_attribute:str, branch_id:int, branch_zero_id:int, levee_levelpaths:str):
     """
-    Masks levee-protected areas from DEM in branch 0 or if stream order is at least the minimum order (max - 1)
+    Masks levee-protected areas from DEM in branch 0 or if the level path is associated with a levee (determined in src/associate_levelpaths_with_levees.py).
+
+    Parameters
+    ----------
+    dem_filename: str
+        Path to DEM file.
+    nld_filename: str
+        Path to levee-protected areas file.
+    levee_id_attribute: str
+        Name of levee ID attribute.
+    out_dem_filename: str
+        Path to write masked DEM.
+    branch_id_attribute: str
+        Name of branch ID attribute.
+    branch_id: int
+        Branch ID number
+    branch_zero_id: int
+        Branch 0 ID number
+    levee_levelpaths: str
+        Path to levee-levelpath association file.
     """
-    if os.path.exists(stream_layer):
-        streams_df = gpd.read_file(stream_layer, ignore_geometry=True)
+    
+    # Rasterize if branch zero
+    if (branch_id == branch_zero_id):
+        with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
+            dem_profile = dem.profile.copy()
+
+            geoms = [feature["geometry"] for feature in leveed]
+
+            # Mask out levee-protected areas from DEM
+            out_dem_masked, _ = mask(dem, geoms, invert=True)
 
-        # Rasterize if branch zero or if stream order is at least the minimum order (max - 1)
-        if (branch_id == branch_zero_id) or (streams_df.loc[streams_df[branch_id_attribute].astype(int)==branch_id, order_attribute].max() >= streams_df[order_attribute].max() - 1):
+            with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
+                dest.write(out_dem_masked[0,:,:], indexes=1)
 
-            with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
+    elif os.path.exists(levee_levelpaths):
+        levee_levelpaths = pd.read_csv(levee_levelpaths)
+
+        levee_levelpaths = levee_levelpaths[levee_levelpaths[branch_id_attribute] == branch_id]
+
+        levelpath_levees = list(levee_levelpaths[levee_id_attribute])
+        
+        if len(levelpath_levees) > 0:
+            with rio.open(dem_filename) as dem:#, fiona.open(nld_filename) as leveed:
+                leveed = gpd.read_file(nld_filename)
                 dem_profile = dem.profile.copy()
 
-                geoms = [feature["geometry"] for feature in leveed]
+                geoms = [feature['geometry'] for i, feature in leveed.iterrows() if feature[levee_id_attribute] in levelpath_levees]
 
                 # Mask out levee-protected areas from DEM
                 out_dem_masked, _ = mask(dem, geoms, invert=True)
@@ -31,16 +68,17 @@ def mask_dem(dem_filename, nld_filename, out_dem_filename, stream_layer, branch_
                     dest.write(out_dem_masked[0,:,:], indexes=1)
 
 
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Mask levee-protected areas from DEM')
-    parser.add_argument('-dem','--dem-filename', help='DEM filename', required=True,type=str)
-    parser.add_argument('-nld','--nld-filename', help='NLD filename', required=True,type=str)
-    parser.add_argument('-out','--out-dem-filename', help='out DEM filename', required=True,type=str)
-    parser.add_argument('-s', '--stream-layer', help='Stream layer filename', required=True)
-    parser.add_argument('-b', '--branch-id-attribute', help='Branch ID attribute name', required=False, default='levpa_id')
+    parser.add_argument('-dem','--dem-filename', help='DEM filename', required=True, type=str)
+    parser.add_argument('-nld','--nld-filename', help='NLD levee-protected areas filename', required=True, type=str)
+    parser.add_argument('-l','--levee-id-attribute', help='Levee ID attribute name', required=True,type=str)
+    parser.add_argument('-out','--out-dem-filename', help='out DEM filename', required=True, type=str)
+    parser.add_argument('-b', '--branch-id-attribute', help='Branch ID attribute name', required=True, type=str)
     parser.add_argument('-i', '--branch-id', help='Branch ID', type=int, required='True')
-    parser.add_argument('-a', '--order-attribute', help='Stream order attribute name', required=False, default='order_')
     parser.add_argument('-b0', '--branch-zero-id', help='Branch zero ID', type=int, required=False, default=0)
+    parser.add_argument('-csv', '--levee-levelpaths', help='Levee - levelpath layer filename', type=str, required=True)
 
     args = vars(parser.parse_args())
 
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index 3bd8f8658..6ecb5388d 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -52,7 +52,7 @@ Tcount
 echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $outputBranchDataDir/$branch_zero_id/dem_meters_$branch_zero_id.tif $outputBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $outputCurrentBranchDataDir/dem_meters.tif $outputCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
+$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $outputHucDataDir/dem_meters.tif $outputBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $outputCurrentBranchDataDir/dem_meters.tif $outputCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
 Tcount
 
 ## GET RASTER METADATA
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 13955ea7b..ab41e9919 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -27,8 +27,6 @@ huc4Identifier=${hucNumber:0:4}
 huc2Identifier=${hucNumber:0:2}
 input_NHD_WBHD_layer=WBDHU$hucUnitLength
 
-input_NLD=$inputDataDir/nld_vectors/huc2_levee_lines/nld_preprocessed_"$huc2Identifier".gpkg
-
 # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
 if [[ $huc2Identifier == "04" ]] ; then
   input_LANDSEA=$input_GL_boundaries
@@ -61,11 +59,13 @@ cmd_args+=" -l $input_nwm_lakes"
 cmd_args+=" -m $input_nwm_catchments"
 cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"
 cmd_args+=" -r $input_NLD"
+cmd_args+=" -rp $input_levees_preprocessed"
 cmd_args+=" -v $input_LANDSEA"
 cmd_args+=" -w $input_nwm_flows"
 cmd_args+=" -x $outputHucDataDir/LandSea_subset.gpkg"
 cmd_args+=" -y $input_nwm_headwaters"
 cmd_args+=" -z $outputHucDataDir/nld_subset_levees.gpkg"
+cmd_args+=" -zp $outputHucDataDir/3d_nld_subset_levees_burned.gpkg"
 cmd_args+=" -wb $wbd_buffer"
 cmd_args+=" -lpf $input_nld_levee_protected_areas"
 cmd_args+=" -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg"
@@ -93,6 +93,14 @@ subscript_exit_code=$?
 if [ $subscript_exit_code -ne 0 ]; then exit $subscript_exit_code; fi
 Tcount
 
+## ASSOCIATE LEVEL PATHS WITH LEVEES
+echo -e $startDiv"Associate level paths with levees"
+date -u
+Tstart
+[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
+python3 $srcDir/associate_levelpaths_with_levees.py -nld $outputHucDataDir/nld_subset_levees.gpkg -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -lpa $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputHucDataDir/levee_levelpaths.csv -w $levee_buffer -b $branch_id_attribute -l $levee_id_attribute
+Tcount
+
 ## STREAM BRANCH POLYGONS
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
 date -u
@@ -120,22 +128,22 @@ echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"
 date -u
 Tstart
 [ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \
-gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $outputHucDataDir/dem_meters.tif
 Tcount
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"
 date -u
 Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/dem_meters.tif)
 
 ## RASTERIZE NLD MULTILINES ##
 echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $branch_zero_id"
 date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
-gdal_rasterize -l nld_subset_levees -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/nld_subset_levees.gpkg $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif
+[ -f $outputHucDataDir/3d_nld_subset_levees_burned.gpkg ] && \
+gdal_rasterize -l 3d_nld_subset_levees_burned -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/3d_nld_subset_levees_burned.gpkg $outputCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif
 Tcount
 
 ## BURN LEVEES INTO DEM ##
@@ -144,7 +152,7 @@ date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
 [ -f $outputCurrentBranchDataDir/nld_subset_levees.tif ] && \
-python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -nld $outputCurrentBranchDataDir/nld_subset_levees_$branch_zero_id.tif -out $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputHucDataDir/dem_meters.tif -nld $outputCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif -out $outputHucDataDir/dem_meters.tif
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
@@ -167,7 +175,7 @@ Tcount
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -w $outputCurrentBranchDataDir -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputHucDataDir/dem_meters.tif -w $outputCurrentBranchDataDir -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
 Tcount
 
 ## PIT REMOVE BURNED DEM ##
@@ -184,6 +192,13 @@ Tstart
 mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
 Tcount
 
+## MAKE A COPY OF THE DEM FOR BRANCH 0
+echo -e $startDiv"Copying DEM to Branch 0"
+date -u
+Tstart
+cp $outputHucDataDir/dem_meters.tif $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+Tcount
+
 ## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
 export hucNumber=$hucNumber
 export current_branch_id=$current_branch_id
@@ -223,15 +238,6 @@ echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"
 $srcDir/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branch_zero_list -b $branch_zero_id
 
 
-## REMOVE FILES FROM DENY LIST ##
-if [ -f $deny_unit_list ]; then
-    echo -e $startDiv"Remove files $hucNumber"
-    date -u
-    Tstart
-    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
-    Tcount
-fi
-
 # -------------------
 ## Start the local csv branch list
 $srcDir/generate_branch_list_csv.py -o $branch_list_csv_file -u $hucNumber -b $branch_zero_id
@@ -250,6 +256,15 @@ else
     echo "No level paths exist with this HUC. Processing branch zero only."
 fi
 
+## REMOVE FILES FROM DENY LIST ##
+if [ -f $deny_unit_list ]; then
+    echo -e $startDiv"Remove files $hucNumber"
+    date -u
+    Tstart
+    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
+    Tcount
+fi
+
 # -------------------
 ## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO (but using normal branch deny) ##
 if [ "$has_deny_branch_zero_override" == "1" ]

From 92318a5149984de5de34c33949499cc1946dd314 Mon Sep 17 00:00:00 2001
From: LauraKeys-NOAA <70279180+LauraKeys-NOAA@users.noreply.github.com>
Date: Fri, 17 Mar 2023 10:54:03 -0500
Subject: [PATCH 221/359] 4.3.3.0: Fix for CatFIM sites sharing LID and
 workaround for WRDS slowdown (#835)

See Changelog 4.3.3.0 for full details.
---
 docs/CHANGELOG.md                         |  19 ++
 tools/generate_categorical_fim.py         |  36 ++-
 tools/generate_categorical_fim_flows.py   | 281 +++++++++++++---------
 tools/generate_categorical_fim_mapping.py |   2 +-
 tools/tools_shared_functions.py           |  23 +-
 5 files changed, 236 insertions(+), 125 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index fe848e32d..65edb23d3 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,25 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.3.0 - 2023-03-02 - [PR#831](https://github.com/NOAA-OWP/inundation-mapping/pull/831)
+
+Addresses bug wherein multiple CatFIM sites in the flow-based service were displaying the same NWS LID. This merge also creates a workaround solution for a slowdown that was observed in the WRDS location API, which may be a temporary workaround, until WRDS addresses the slowdown.
+
+### Changes
+
+- `tools/generate_categorical_fim_mapping.py`: resets the list of tifs to format for each LID within the loop that does the map processing, instead of only once before the start of the loop.
+- `tools/tools_shared_functions.py`:
+  - adds a try-except block around code that attempted to iterate on an empty list when the API didn't return relevant metadata for a given feature ID (this is commented out, but may be used in the future once WRDS slowdown is addressed).
+  - Uses a passed NWM flows geodataframe to determine stream order.
+- `/tools/generate_categorical_fim_flows.py`:
+  - Adds multiprocessing to flows generation and uses `nwm_flows.gpkg` instead of the WRDS API to determine stream order of NWM feature_ids.
+  - Adds duration print messages.
+- `/tools/generate_categorical_fim.py`:
+  - Refactor to allow for new NWM filtering scheme.
+  - Bug fix in multiprocessing calls for interval map production.
+  - Adds duration print messages.
+
+<br/><br/>
 
 ## v4.3.2.0 - 2023-03-15 - [PR#845](https://github.com/NOAA-OWP/inundation-mapping/pull/845)
 
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index ee0c5ea03..53f0d5968 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -5,6 +5,7 @@
 import csv
 import traceback
 import sys
+from datetime import datetime
 import time
 from pathlib import Path
 import geopandas as gpd
@@ -29,6 +30,13 @@ def process_generate_categorical_fim(fim_run_dir, job_number_huc, job_number_inu
                                      stage_based, output_folder, overwrite, search, 
                                      lid_to_run, job_number_intervals, past_major_interval_cap):
         
+    print("================================")
+    print("Start generate categorical fim")
+    overall_start_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"started: {dt_string}")
+    print()
+        
     # Check job numbers and raise error if necessary
     total_cpus_requested = job_number_huc * job_number_inundate * job_number_intervals
     total_cpus_available = os.cpu_count() - 1
@@ -88,9 +96,10 @@ def process_generate_categorical_fim(fim_run_dir, job_number_huc, job_number_inu
         fim_dir = ""
         print('Creating flow files using the ' + catfim_method + ' technique...')
         start = time.time()
-        nws_sites_layer = generate_catfim_flows(output_flows_dir, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir)
+        nws_sites_layer = generate_catfim_flows(output_flows_dir, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir, job_number_huc)
         end = time.time()
         elapsed_time = (end-start)/60
+        
         print(f'Finished creating flow files in {elapsed_time} minutes')
         # Generate CatFIM mapping
         print('Begin mapping')
@@ -109,6 +118,17 @@ def process_generate_categorical_fim(fim_run_dir, job_number_huc, job_number_inu
     reformatted_catfim_method = catfim_method.lower().replace('-', '_')
     create_csvs(output_mapping_dir, reformatted_catfim_method)
 
+    print("================================")
+    print("End generate categorical fim")
+
+    end_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"ended: {dt_string}")
+
+    # calculate duration
+    time_duration = end_time - overall_start_time
+    print(f"Duration: {str(time_duration).split('.')[0]}")
+    print()
 
 def create_csvs(output_mapping_dir, reformatted_catfim_method):
     '''
@@ -236,7 +256,7 @@ def mark_complete(site_directory):
     marker_file.touch()
     return
 
-def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, threshold_url, flood_categories, all_lists, past_major_interval_cap, number_of_jobs, number_of_interval_jobs, attributes_dir):
+def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, threshold_url, flood_categories, all_lists, past_major_interval_cap, number_of_jobs, number_of_interval_jobs, attributes_dir, nwm_flows_df):
     missing_huc_files = []
     all_messages = []
     stage_based_att_dict = {}
@@ -423,7 +443,7 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
         
         # Filter segments to be of like stream order.
         desired_order = metadata['nwm_feature_data']['stream_order']
-        segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order)
+        segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df)
         action_stage = stages['action']
         minor_stage = stages['minor']
         moderate_stage = stages['moderate']
@@ -458,9 +478,9 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
                 all_messages.append([f'{lid}:missing some HUC data'])
                 
         # Now that the "official" category maps are made, produce the incremental maps.
-        for interval_stage in interval_list:
+        with ProcessPoolExecutor(max_workers=number_of_interval_jobs) as executor:
             try:
-                with ProcessPoolExecutor(max_workers=number_of_interval_jobs) as executor:
+                for interval_stage in interval_list:
                     # Determine category the stage value belongs with.
                     if action_stage <= interval_stage < minor_stage:
                         category = 'action_' + str(interval_stage).replace('.', 'p') + 'ft'
@@ -535,14 +555,14 @@ def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us
     
     flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
 
-    huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists = generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based=True, fim_dir=fim_dir, lid_to_run=lid_to_run)
+    huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists, nwm_flows_df = generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based=True, fim_dir=fim_dir, lid_to_run=lid_to_run)
     
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
         for huc in huc_dictionary:
             executor.submit(iterate_through_huc_stage_based, workspace, huc, 
                             fim_dir, huc_dictionary, threshold_url, flood_categories,
                             all_lists, past_major_interval_cap, number_of_jobs, 
-                            number_of_interval_jobs, attributes_dir)
+                            number_of_interval_jobs, attributes_dir, nwm_flows_df)
                 
     print('Wrapping up Stage-Based CatFIM...')
     csv_files = os.listdir(attributes_dir)
@@ -677,7 +697,7 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
             
             # Create inundation maps with branch and stage data
             try:
-                print("Running inundation for " + huc + " and branch " + branch)
+                print("Generating stage-based FIM for " + huc + " and branch " + branch)
                 executor.submit(produce_inundation_map_with_stage_and_feature_ids, rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch)
             except Exception:
                 messages.append([f'{lid}:inundation failed at {category}'])
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index 667b3ec9d..3c64b25ac 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -3,8 +3,10 @@
 import pandas as pd
 import numpy as np
 import geopandas as gpd
+from datetime import datetime
 import time
 from tools_shared_functions import aggregate_wbd_hucs, mainstem_nwm_segs, get_thresholds, flow_data, get_metadata, get_nwm_segs, get_datum, ngvd_to_navd_ft, filter_nwm_segments_by_stream_order
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 import argparse
 from dotenv import load_dotenv
 import os
@@ -21,7 +23,114 @@ def get_env_paths():
     return API_BASE_URL, WBD_LAYER
 
 
-def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir=""):
+def process_generate_flows(huc, huc_dictionary, threshold_url, all_lists, workspace, attributes_dir, huc_messages_dir, nwm_flows_df):
+    
+    # Process each huc unit, first define message variable and flood categories.
+    all_messages = []
+    flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
+    
+    print(f'Iterating through {huc}')
+    #Get list of nws_lids
+    nws_lids = huc_dictionary[huc]
+    #Loop through each lid in list to create flow file
+    for lid in nws_lids:            
+        #Convert lid to lower case
+        lid = lid.lower()
+        #Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
+        print("getting thresholds")
+        stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
+        if stages == None or flows == None:
+            print("Likely WRDS error")
+            continue
+        #Check if stages are supplied, if not write message and exit. 
+        if all(stages.get(category, None)==None for category in flood_categories):
+            message = f'{lid}:missing threshold stages'
+            all_messages.append(message)
+            continue
+        #Check if calculated flows are supplied, if not write message and exit.
+        if all(flows.get(category, None) == None for category in flood_categories):
+            message = f'{lid}:missing calculated flows'
+            all_messages.append(message)
+            continue
+        #find lid metadata from master list of metadata dictionaries (line 66).
+        metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
+                                  
+        #Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
+        unfiltered_segments = list(set(get_nwm_segs(metadata)))
+        
+        desired_order = metadata['nwm_feature_data']['stream_order']
+        # Filter segments to be of like stream order.
+        print("filtering segments")
+        start = time.time()
+        segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df)
+        end = time.time()
+        elapsed_time = round(((end-start)/60), 6)
+        print(f'Finished filtering segments in {elapsed_time} minutes')
+        #if no segments, write message and exit out
+        if not segments:
+            message = f'{lid}:missing nwm segments'
+            all_messages.append(message)
+            continue
+        # For each flood category
+        for category in flood_categories:
+            #G et the flow
+            flow = flows[category]
+            # If there is a valid flow value, write a flow file.
+            if flow:
+                #round flow to nearest hundredth
+                flow = round(flow,2)
+                #Create the guts of the flow file.
+                flow_info = flow_data(segments,flow)
+                #Define destination path and create folders
+                output_file = workspace / huc / lid / category / (f'ahps_{lid}_huc_{huc}_flows_{category}.csv') 
+                output_file.parent.mkdir(parents = True, exist_ok = True)
+                #Write flow file to file
+                flow_info.to_csv(output_file, index = False)
+            else:
+                message = f'{lid}:{category} is missing calculated flow'
+                all_messages.append(message)
+        # Get various attributes of the site.
+        lat = float(metadata['nws_preferred']['latitude'])
+        lon = float(metadata['nws_preferred']['longitude'])
+        wfo = metadata['nws_data']['wfo']
+        rfc = metadata['nws_data']['rfc']
+        state = metadata['nws_data']['state']
+        county = metadata['nws_data']['county']
+        name = metadata['nws_data']['name']
+        flow_source = flows['source']
+        stage_source = stages['source']
+        wrds_timestamp = stages['wrds_timestamp']
+        nrldb_timestamp = metadata['nrldb_timestamp']
+        nwis_timestamp = metadata['nwis_timestamp']
+                    
+        # Create a csv with same information as shapefile but with each threshold as new record.
+        csv_df = pd.DataFrame()
+        for threshold in flood_categories:
+            line_df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'magnitude': threshold, 'q':flows[threshold], 'q_uni':flows['units'], 'q_src':flow_source, 'stage':stages[threshold], 'stage_uni':stages['units'], 's_src':stage_source, 'wrds_time':wrds_timestamp, 'nrldb_time':nrldb_timestamp,'nwis_time':nwis_timestamp, 'lat':[lat], 'lon':[lon]})
+            csv_df = csv_df.append(line_df)
+        #Round flow and stage columns to 2 decimal places.
+        csv_df = csv_df.round({'q':2,'stage':2})
+
+        # If a site folder exists (ie a flow file was written) save files containing site attributes.
+        output_dir = workspace / huc / lid
+        if output_dir.exists():
+            #Export DataFrame to csv containing attributes
+            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index = False)
+            message = f'{lid}:flows available'
+            all_messages.append(message)
+        else:
+            message = f'{lid}:missing all calculated flows'
+            all_messages.append(message)
+            
+    # Write all_messages to huc-specific file.
+    print("Writing message file for huc")
+    huc_messages_txt_file = os.path.join(huc_messages_dir, str(huc) + '_messages.txt')
+    with open(huc_messages_txt_file, 'w') as f:
+        for item in all_messages:
+            f.write("%s\n" % item)
+
+
+def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir="", job_number_huc=1):
     '''
     This will create static flow files for all nws_lids and save to the 
     workspace directory with the following format:
@@ -49,7 +158,7 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
     None.
     '''
         
-    all_start = time.time()
+    all_start = datetime.now()
     API_BASE_URL, WBD_LAYER = get_env_paths()
     #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
     workspace = Path(workspace)
@@ -59,10 +168,21 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
     threshold_url = f'{API_BASE_URL}/nws_threshold'
     ###################################################################
     
-    #Create workspace
+    # Create workspace
     workspace.mkdir(parents=True,exist_ok = True)
 
+    # Create HUC message directory to store messages that will be read and joined after multiprocessing
+    huc_messages_dir = os.path.join(workspace, 'huc_messages')
+    if not os.path.exists(huc_messages_dir):
+        os.mkdir(huc_messages_dir)
+    
+    # Open NWM flows geopackage
+    nwm_flows_gpkg = r'/data/inputs/nwm_hydrofabric/nwm_flows.gpkg'
+    nwm_flows_df = gpd.read_file(nwm_flows_gpkg)
+
     print(f'Retrieving metadata for site(s): {lid_to_run}...')
+    start_dt = datetime.now()
+    
     #Get metadata for 'CONUS'
     print(metadata_url)
     if lid_to_run != 'all':
@@ -75,7 +195,15 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
         # Append the dataframes and lists
         all_lists = conus_list + islands_list
     print(len(all_lists))
+    
+    end_dt = datetime.now()
+    time_duration = end_dt - start_dt
+    print(f"Retrieving metadata Duration: {str(time_duration).split('.')[0]}")
+    print()
+    
     print('Determining HUC using WBD layer...')
+    start_dt = datetime.now()    
+    
     # Assign HUCs to all sites using a spatial join of the FIM 3 HUC layer. 
     # Get a dictionary of hucs (key) and sites (values) as well as a GeoDataFrame
     # of all sites used later in script.
@@ -84,107 +212,28 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
     out_gdf = out_gdf.drop(['downstream_nwm_features'], axis=1, errors='ignore')
     out_gdf = out_gdf.drop(['upstream_nwm_features'], axis=1, errors='ignore')
     out_gdf = out_gdf.astype({'metadata_sources': str})
-
-    # Loop through each huc unit, first define message variable and flood categories.
-    all_messages = []
-    flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
+        
+    end_dt = datetime.now()        
+    time_duration = end_dt - start_dt
+    print(f"Determining HUC using WBD layer Duration: {str(time_duration).split('.')[0]}")
+    print()
         
     if stage_based:
-        return huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists
-    
-    for huc in huc_dictionary:
-        print(f'Iterating through {huc}')
-        #Get list of nws_lids
-        nws_lids = huc_dictionary[huc]
-        #Loop through each lid in list to create flow file
-        for lid in nws_lids:            
-            #Convert lid to lower case
-            lid = lid.lower()
-            #Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
-            stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
-            if stages == None or flows == None:
-                print("Likely WRDS error")
-                continue
-            #Check if stages are supplied, if not write message and exit. 
-            if all(stages.get(category, None)==None for category in flood_categories):
-                message = f'{lid}:missing threshold stages'
-                all_messages.append(message)
-                continue
-            #Check if calculated flows are supplied, if not write message and exit.
-            if all(flows.get(category, None) == None for category in flood_categories):
-                message = f'{lid}:missing calculated flows'
-                all_messages.append(message)
-                continue
-            #find lid metadata from master list of metadata dictionaries (line 66).
-            metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
-                                      
-            #Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
-            unfiltered_segments = list(set(get_nwm_segs(metadata)))
-            
-            desired_order = metadata['nwm_feature_data']['stream_order']
-            # Filter segments to be of like stream order.
-            segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order)
-            #if no segments, write message and exit out
-            if not segments:
-                print(f'{lid} no segments')
-                message = f'{lid}:missing nwm segments'
-                all_messages.append(message)
-                continue
-            #For each flood category
-            for category in flood_categories:
-                #Get the flow
-                flow = flows[category]
-                #If there is a valid flow value, write a flow file.
-                if flow:
-                    #round flow to nearest hundredth
-                    flow = round(flow,2)
-                    #Create the guts of the flow file.
-                    flow_info = flow_data(segments,flow)
-                    #Define destination path and create folders
-                    output_file = workspace / huc / lid / category / (f'ahps_{lid}_huc_{huc}_flows_{category}.csv') 
-                    output_file.parent.mkdir(parents = True, exist_ok = True)
-                    #Write flow file to file
-                    flow_info.to_csv(output_file, index = False)
-                else:
-                    message = f'{lid}:{category} is missing calculated flow'
-                    all_messages.append(message)
-            #Get various attributes of the site.
-#            lat = float(metadata['usgs_preferred']['latitude'])
-#            lon = float(metadata['usgs_preferred']['longitude'])
-            lat = float(metadata['nws_preferred']['latitude'])
-            lon = float(metadata['nws_preferred']['longitude'])
-            wfo = metadata['nws_data']['wfo']
-            rfc = metadata['nws_data']['rfc']
-            state = metadata['nws_data']['state']
-            county = metadata['nws_data']['county']
-            name = metadata['nws_data']['name']
-            flow_units = flows['units']
-            flow_source = flows['source']
-            stage_units = stages['units']
-            stage_source = stages['source']
-            wrds_timestamp = stages['wrds_timestamp']
-            nrldb_timestamp = metadata['nrldb_timestamp']
-            nwis_timestamp = metadata['nwis_timestamp']
-                        
-            #Create a csv with same information as shapefile but with each threshold as new record.
-            csv_df = pd.DataFrame()
-            for threshold in flood_categories:
-                line_df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'magnitude': threshold, 'q':flows[threshold], 'q_uni':flows['units'], 'q_src':flow_source, 'stage':stages[threshold], 'stage_uni':stages['units'], 's_src':stage_source, 'wrds_time':wrds_timestamp, 'nrldb_time':nrldb_timestamp,'nwis_time':nwis_timestamp, 'lat':[lat], 'lon':[lon]})
-                csv_df = csv_df.append(line_df)
-            #Round flow and stage columns to 2 decimal places.
-            csv_df = csv_df.round({'q':2,'stage':2})
+        return huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists, nwm_flows_df
 
-            #If a site folder exists (ie a flow file was written) save files containing site attributes.
-            output_dir = workspace / huc / lid
-            if output_dir.exists():
-                #Export DataFrame to csv containing attributes
-                csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index = False)
-                message = f'{lid}:flows available'
-                all_messages.append(message)
-            else:
-                message = f'{lid}:missing all calculated flows'
-                all_messages.append(message)
-    print('Wrapping up...')
+    print("Generating flows for hucs using " + str(job_number_huc) + " jobs...")
+    start_dt = datetime.now()        
+    
+    with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
+        for huc in huc_dictionary:
+            executor.submit(process_generate_flows, huc, huc_dictionary, threshold_url, all_lists, workspace, attributes_dir, huc_messages_dir, nwm_flows_df)
+            
+    end_dt = datetime.now()
+    time_duration = end_dt - start_dt
+    print(f"Generating flows for hucs Duration: {str(time_duration).split('.')[0]}")
+    print()
+            
+    print('Wrapping up flows generation...')
     #Recursively find all *_attributes csv files and append
     csv_files = os.listdir(attributes_dir)
     all_csv_df = pd.DataFrame()
@@ -216,25 +265,39 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
     # 'mapped' column with 'No' if sites did not map.
     viz_out_gdf = viz_out_gdf.merge(lids_df, how = 'left', on = 'nws_lid')    
     viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
-    #Write messages to DataFrame, split into columns, aggregate messages.
-    messages_df  = pd.DataFrame(all_messages, columns = ['message'])
+    
+    # Read all messages for all HUCs TODO
+    huc_message_list = []
+    huc_messages_dir_list = os.listdir(huc_messages_dir)
+    for huc_message_file in huc_messages_dir_list:
+        full_path_file = os.path.join(huc_messages_dir, huc_message_file)
+        with open(full_path_file, 'r') as f:
+            if full_path_file.endswith('.txt'):
+                lines = f.readlines()
+                for line in lines:
+                    huc_message_list.append(line)
+    
+    # Write messages to DataFrame, split into columns, aggregate messages.
+    messages_df  = pd.DataFrame(huc_message_list, columns = ['message'])
     messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
     status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
     
-    #Join messages to populate status field to candidate sites. Assign 
-    #status for null fields.
+    # Join messages to populate status field to candidate sites. Assign 
+    # status for null fields.
     viz_out_gdf = viz_out_gdf.merge(status_df, how = 'left', on = 'nws_lid')
     viz_out_gdf['status'] = viz_out_gdf['status'].fillna('all calculated flows available')
     
-    #Filter out columns and write out to file
+    # Filter out columns and write out to file
 #    viz_out_gdf = viz_out_gdf.filter(['nws_lid','usgs_gage','nwm_seg','HUC8','mapped','status','geometry'])
     nws_lid_layer = os.path.join(workspace, 'nws_lid_sites.gpkg').replace('flows', 'mapping')
 
     viz_out_gdf.to_file(nws_lid_layer, driver='GPKG')
     
-    #time operation
-    all_end = time.time()
-    print(f'total time is {round((all_end - all_start)/60),1} minutes')
+    # time operation
+    all_end = datetime.now()
+    all_time_duration = all_end - all_start
+    print(f"Duration: {str(all_time_duration).split('.')[0]}")
+    print()
     
     return nws_lid_layer
 
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index db9328dbf..85a5fe5f6 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -119,10 +119,10 @@ def run_inundation(magnitude_flows_csv, huc, output_extent_grid, ahps_site, magn
 
 def post_process_huc_level(job_number_tif, ahps_dir_list, huc_dir, attributes_dir, gpkg_dir, fim_version, huc):
     
-    tifs_to_reformat_list = []
 
     # Loop through ahps sites
     for ahps_lid in ahps_dir_list:
+        tifs_to_reformat_list = []
         ahps_lid_dir = os.path.join(huc_dir, ahps_lid)
 
         # Append desired filenames to list.
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index cc317e867..f1ad637ce 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -8,6 +8,7 @@
 import requests
 import numpy as np
 import pathlib
+import time
 from pathlib import Path
 import rasterio.shutil
 from rasterio.warp import calculate_default_transform, reproject, Resampling
@@ -27,7 +28,7 @@ def get_env_paths():
     return API_BASE_URL, WBD_LAYER
 
 
-def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order):
+def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df):
     """
     This function uses the WRDS API to filter out NWM segments from a list if their stream order is different than
     the target stream order.
@@ -39,15 +40,24 @@ def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order):
         filtered_segments (list): A list of NWM feature_id strings, paired down to only those that share the target order.        
     
     """
-    API_BASE_URL, WBD_LAYER = get_env_paths()
+        
+#    API_BASE_URL, WBD_LAYER = get_env_paths()
     #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
-    metadata_url = f'{API_BASE_URL}/metadata'
+#    metadata_url = f'{API_BASE_URL}/metadata'
+        
     
+
+    # feature ID of 0 is getting passed to WRDS and returns empty results,
+    # which can cause failures on next() 
+#    if '0' in unfiltered_segments:
+#        unfiltered_segments = unfiltered_segments.remove('0')
+#    if unfiltered_segments is None:
+#        return filtered_segments
+
     filtered_segments = []
+
     for feature_id in unfiltered_segments:
-        all_lists = get_metadata(metadata_url, select_by = 'nwm_feature_id', selector = [feature_id])
-        feature_id_metadata = next(item for item in all_lists)[0]
-        stream_order = feature_id_metadata['nwm_feature_data']['stream_order']
+        stream_order = nwm_flows_df.loc[nwm_flows_df['ID'] == int(feature_id), 'order_'].values[0]
         if stream_order == desired_order:
             filtered_segments.append(feature_id)
 
@@ -767,7 +777,6 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
 #        metadata_gdf = metadata_gdf[retain_attributes]
     print("Performing spatial and tabular operations on geodataframe...")
     #Perform a spatial join to get the WBD HUC 8 assigned to each AHPS
-    print(metadata_gdf)
     joined_gdf = gpd.sjoin(metadata_gdf, huc8, how = 'inner', op = 'intersects', lsuffix = 'ahps', rsuffix = 'wbd')
     joined_gdf = joined_gdf.drop(columns = 'index_wbd')
 

From e81e1288edba7a232be33400ec88adfac93b30f8 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 22 Mar 2023 11:11:46 -0400
Subject: [PATCH 222/359] v4.3.3.1 Bug fix -- KeyError in
 `src/associate_levelpaths_with_levees.py` (#855)

---
 docs/CHANGELOG.md                       | 10 ++++
 src/associate_levelpaths_with_levees.py | 70 ++++++++++++++++++-------
 2 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 65edb23d3..ccf4610d0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.3.1 - 2023-03-20 - [PR#855](https://github.com/NOAA-OWP/inundation-mapping/pull/855)
+
+Bug fix for KeyError in `src/associate_levelpaths_with_levees.py`
+
+### Changes
+
+- `src/associate_levelpaths_with_levees.py`: Adds check if input files exist and handles empty GeoDataFrame(s) after intersecting levee buffers with leveed areas.
+
+<br/><br/>
+
 ## v4.3.3.0 - 2023-03-02 - [PR#831](https://github.com/NOAA-OWP/inundation-mapping/pull/831)
 
 Addresses bug wherein multiple CatFIM sites in the flow-based service were displaying the same NWS LID. This merge also creates a workaround solution for a slowdown that was observed in the WRDS location API, which may be a temporary workaround, until WRDS addresses the slowdown.
diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py
index 7abcb411c..0dbf7327c 100644
--- a/src/associate_levelpaths_with_levees.py
+++ b/src/associate_levelpaths_with_levees.py
@@ -28,7 +28,7 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
         Path to write output CSV file.    
     """
 
-    if os.path.exists(levees_filename):
+    if os.path.exists(levees_filename) and os.path.exists(leveed_areas_filename) and os.path.exists(levelpaths_filename):
         # Read in geodataframes
         levees = gpd.read_file(levees_filename)
         leveed_areas = gpd.read_file(leveed_areas_filename)
@@ -49,28 +49,54 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
         leveed_right = gpd.overlay(levees_buffered_right, leveed_areas, how='intersection')
 
         # Find leveed areas not intersected by either buffer
-        levees_not_found = leveed_areas[~leveed_areas['SYSTEM_ID'].isin(pd.concat([leveed_left['SYSTEM_ID_1'], leveed_right['SYSTEM_ID_1']]).values)]
+        leveed_intersected = []
+        if not leveed_left.empty:
+            [leveed_intersected.append(x) for x in leveed_left[f'{levee_id_attribute}_1'].values]
 
-        # Associate levees and leveed areas
-        matches_left = np.where(leveed_left[f'{levee_id_attribute}_1']==leveed_left[f'{levee_id_attribute}_2'])[0]
-        matches_right = np.where(leveed_right[f'{levee_id_attribute}_1']==leveed_right[f'{levee_id_attribute}_2'])[0]
+            # Associate levees and leveed areas
+            matches_left = np.where(leveed_left[f'{levee_id_attribute}_1']==leveed_left[f'{levee_id_attribute}_2'])[0]
 
-        leveed_left = leveed_left.loc[matches_left]
-        leveed_right = leveed_right.loc[matches_right]
+            leveed_left = leveed_left.loc[matches_left]
 
-        # Get area of associated leveed areas
-        leveed_left['leveed_area'] = leveed_left.area
-        leveed_right['leveed_area'] = leveed_right.area
+            # Get area of associated leveed areas
+            leveed_left['leveed_area'] = leveed_left.area
 
-        leveed_left = leveed_left[[f'{levee_id_attribute}_1', 'leveed_area', 'geometry']]
-        leveed_right = leveed_right[[f'{levee_id_attribute}_1', 'leveed_area', 'geometry']]
+            leveed_left = leveed_left[[f'{levee_id_attribute}_1', 'leveed_area', 'geometry']]
+
+        if not leveed_right.empty:
+            [leveed_intersected.append(x) for x in leveed_right[f'{levee_id_attribute}_1'].values]
+
+            # Associate levees and leveed areas
+            matches_right = np.where(leveed_right[f'{levee_id_attribute}_1']==leveed_right[f'{levee_id_attribute}_2'])[0]
+
+            leveed_right = leveed_right.loc[matches_right]
+
+            # Get area of associated leveed areas
+            leveed_right['leveed_area'] = leveed_right.area
+
+            leveed_right = leveed_right[[f'{levee_id_attribute}_1', 'leveed_area', 'geometry']]
+
+        if len(leveed_intersected) > 0:
+            levees_not_found = leveed_areas[~leveed_areas[levee_id_attribute].isin(leveed_intersected)]
 
         # Merge left and right levee protected areas
-        leveed = leveed_left.merge(leveed_right, on=f'{levee_id_attribute}_1', how='outer', suffixes=['_left', '_right'])
+        if leveed_left.empty and leveed_right.empty:
+            return
+        
+        elif not leveed_left.empty and not leveed_right.empty:
+            leveed = leveed_left.merge(leveed_right, on=f'{levee_id_attribute}_1', how='outer', suffixes=['_left', '_right'])
 
-        # Set unmatched areas to zero
-        leveed.loc[np.isnan(leveed['leveed_area_left']), 'leveed_area_left'] = 0
-        leveed.loc[np.isnan(leveed['leveed_area_right']), 'leveed_area_right'] = 0
+            # Set unmatched areas to zero
+            leveed.loc[np.isnan(leveed['leveed_area_left']), 'leveed_area_left'] = 0.
+            leveed.loc[np.isnan(leveed['leveed_area_right']), 'leveed_area_right'] = 0.
+
+        elif leveed_left.empty:
+            leveed = leveed_right.rename(columns={'leveed_area': 'leveed_area_right'})
+            leveed['leveed_area_left'] = 0.
+
+        elif leveed_right.empty:
+            leveed = leveed_left.rename(columns={'leveed_area': 'leveed_area_left'})
+            leveed['leveed_area_right'] = 0.
 
         # Determine which side the levee is protecting (opposite of levee protected area)
         leveed['levee_side'] = np.where(leveed['leveed_area_left'] < leveed['leveed_area_right'], 'left', 'right')
@@ -90,12 +116,16 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
         levee_levelpaths_left = levee_levelpaths_left[levee_levelpaths_left[levee_id_attribute].isin(left_ids)]
         levee_levelpaths_right = levee_levelpaths_right[levee_levelpaths_right[levee_id_attribute].isin(right_ids)]
 
+        # Join left and right
+        out_df =  pd.concat([levee_levelpaths_right[[levee_id_attribute, branch_id_attribute]], levee_levelpaths_left[[levee_id_attribute, branch_id_attribute]]]).drop_duplicates().reset_index(drop=True)
+
         # Add level paths to levees not found
-        levees_not_found.geometry = levees_not_found.buffer(2*levee_buffer)
-        levees_not_found = gpd.sjoin(levees_not_found, levelpaths)
+        if len(levees_not_found) > 0:
+            levees_not_found.geometry = levees_not_found.buffer(2*levee_buffer)
+            levees_not_found = gpd.sjoin(levees_not_found, levelpaths)
 
-        # Join left and right
-        out_df =  pd.concat([levee_levelpaths_right[[levee_id_attribute, branch_id_attribute]], levee_levelpaths_left[[levee_id_attribute, branch_id_attribute]], levees_not_found[[levee_id_attribute, branch_id_attribute]]]).drop_duplicates().reset_index(drop=True)
+            # Add to out_df
+            out_df =  pd.concat([out_df[[levee_id_attribute, branch_id_attribute]], levees_not_found[[levee_id_attribute, branch_id_attribute]]]).drop_duplicates().reset_index(drop=True)
 
         # Remove levelpaths that cross the levee exactly once
         for j, row in out_df.iterrows():

From b6da3a2c4ce03ca606da0e95974ee95912635992 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 22 Mar 2023 11:16:44 -0400
Subject: [PATCH 223/359] v4.3.3.2 Bug fix - Change .split() to
 os.path.splitext() (#851)

---
 docs/CHANGELOG.md      |  9 +++++++++
 src/stream_branches.py | 13 +++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ccf4610d0..268d2cd19 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.3.2 - 2023-03-20 - [PR#851](https://github.com/NOAA-OWP/inundation-mapping/pull/851)
+
+Bug fix to change `.split()` to `os.path.splitext()`
+
+### Changes
+- `src/stream_branches.py`: Change 3 occurrences of `.split()` to `os.path.splitext()`
+
+<br/><br/>
+
 ## v4.3.3.1 - 2023-03-20 - [PR#855](https://github.com/NOAA-OWP/inundation-mapping/pull/855)
 
 Bug fix for KeyError in `src/associate_levelpaths_with_levees.py`
diff --git a/src/stream_branches.py b/src/stream_branches.py
index c5e68a512..8a6ebf6cc 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import os
 import geopandas as gpd
 import pandas as pd
 import rasterio
@@ -1072,8 +1073,8 @@ def __find_matching_record(vector,attribute,value,matching='first'):
             out_records += __find_matching_record(vector,branch_id_attribute,bid,matching='all')
 
         if (out_filename_template is not None) & ( len(out_records) != 0):
-                base,ext = out_filename_template.split('.')
-                out_filename = base + "_{}.".format(bid) + ext
+                base, ext = os.path.splitext(out_filename_template)
+                out_filename = base + "_{}".format(bid) + ext
                 
                 with fiona.open(out_filename,'w',**source_meta) as out_file:
                     out_file.writerecords(out_records)
@@ -1134,8 +1135,8 @@ def clip(self,to_clip,out_filename_template=None,branch_id=None,branch_id_attrib
                 if out_filename_template is not None:
                     branch_id = row[self.branch_id_attribute]
 
-                    base,ext = out_filename_template.split('.')
-                    out_filename = base + "_{}.".format(branch_id) + ext
+                    base, ext = os.path.splitext(out_filename_template)
+                    out_filename = base + "_{}".format(branch_id) + ext
                     
                     with rasterio.open(out_filename,'w',**buffered_meta) as out:
                         out.write(buffered_array)
@@ -1152,8 +1153,8 @@ def clip(self,to_clip,out_filename_template=None,branch_id=None,branch_id_attrib
                 return_list += [out]
                 
                 if (out_filename_template is not None) & (not out.empty):
-                    base,ext = out_filename_template.split('.')
-                    out_filename = base + "_{}.".format(branch_id) + ext
+                    base, ext = os.path.splitext(out_filename_template)
+                    out_filename = base + "_{}".format(branch_id) + ext
                     StreamNetwork.write(out,out_filename)
         
         return(return_list)

From a1ea4eb898662310442ffdbd0e6c3fda7d7ea07d Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Wed, 22 Mar 2023 10:22:18 -0500
Subject: [PATCH 224/359] v4.3.3.3 Fixed datatype issue in coord accuracy list
 (#854)

---
 docs/CHANGELOG.md               | 11 +++++++++++
 tools/tools_shared_variables.py |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 268d2cd19..27c0faa90 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,11 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.3.3 - 2023-03-20 - [PR#854](https://github.com/NOAA-OWP/inundation-mapping/pull/854)
+
+At least one site (e.g. TRYM7) was not been getting mapped in Stage-Based CatFIM, despite having all of the acceptable accuracy codes. This was caused by a data type issue in the `acceptable_coord_acc_code_list` in `tools_shared_variables.py` having the accuracy codes of 5 and 1 as a strings instead of an integers.
+
+### Changes
+
+- `/tools/tools_shared_variables.py`: Added integers 5 and 1 to the acceptable_coord_acc_code_list, kept the '5' and '1' strings as well.
+
+<br/><br/>
+
 ## v4.3.3.2 - 2023-03-20 - [PR#851](https://github.com/NOAA-OWP/inundation-mapping/pull/851)
 
 Bug fix to change `.split()` to `os.path.splitext()`
 
 ### Changes
+
 - `src/stream_branches.py`: Change 3 occurrences of `.split()` to `os.path.splitext()`
 
 <br/><br/>
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index e63d1cf80..da86b828c 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -65,7 +65,7 @@
 # USGS gages acceptance criteria. Likely not constants, so not using all caps.
 # ANY CHANGE TO THESE VALUES SHOULD WARRANT A CODE VERSION CHANGE
 # https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html
-acceptable_coord_acc_code_list = ['H','1','5','S','R','B','C','D','E']
+acceptable_coord_acc_code_list = ['H','1','5','S','R','B','C','D','E', 5, 1]
 # https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html
 acceptable_coord_method_code_list = ['C','D','W','X','Y','Z','N','M','L','G','R','F','S']
 # https://help.waterdata.usgs.gov/codes-and-parameters/codes#SI

From 5a201355d79804dbcc8d5ba1345adc7ec73b6ee8 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Wed, 22 Mar 2023 10:54:22 -0500
Subject: [PATCH 225/359] v4.3.3.4 Hotfix for inundate nation projection error
 (#849)

---
 docs/CHANGELOG.md        | 10 ++++++++++
 tools/inundate_nation.py |  5 +++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 27c0faa90..37d4dcd07 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.3.4 - 2023-03-17 - [PR#849](https://github.com/NOAA-OWP/inundation-mapping/pull/849)
+
+This hotfix addresses an error in inundate_nation.py relating to projection CRS.
+
+## Changes
+
+- `tools/inundate_nation.py`: #782 CRS projection change likely causing issue with previous projection configuration
+
+<br/><br/>
+
 ## v4.3.3.3 - 2023-03-20 - [PR#854](https://github.com/NOAA-OWP/inundation-mapping/pull/854)
 
 At least one site (e.g. TRYM7) was not been getting mapped in Stage-Based CatFIM, despite having all of the acceptable accuracy codes. This was caused by a data type issue in the `acceptable_coord_acc_code_list` in `tools_shared_variables.py` having the accuracy codes of 5 and 1 as a strings instead of an integers.
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index a07f9c9d9..efcf0e503 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -11,11 +11,13 @@
 from rasterio.merge import merge
 from osgeo import gdal, ogr
 
-sys.path.append('/foss_fim/src')
 from datetime import datetime
 from mosaic_inundation import Mosaic_inundation
 from inundate_gms import Inundate_gms
 from inundation import inundate
+
+# setting path
+sys.path.append('/foss_fim/src')
 from utils.shared_variables import elev_raster_ndv, PREP_PROJECTION
 from utils.shared_functions import FIM_Helpers as fh
 
@@ -193,7 +195,6 @@ def create_bool_rasters(args):
                 blockxsize=512, 
                 blockysize=512,
                 dtype='int8',
-                crs=PREP_PROJECTION,
                 compress='lzw')
     with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, 'w', **profile) as dst:
         dst.write(array.astype(rasterio.int8))

From 32493de9500f51f17c1af3635c3e97647874e8b2 Mon Sep 17 00:00:00 2001
From: EmilyDeardorff-NOAA <60829052+EmilyDeardorff@users.noreply.github.com>
Date: Fri, 31 Mar 2023 09:31:23 -0700
Subject: [PATCH 226/359] v4.3.3.5 Update the alpha test to accept previous
 runs' metrics CSV (#848)

---
 docs/CHANGELOG.md              |  15 ++-
 tools/inundation.py            |   8 +-
 tools/synthesize_test_cases.py | 174 +++++++++++++++++++++++----------
 3 files changed, 142 insertions(+), 55 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 37d4dcd07..cf9e47f74 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,11 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.3.5 - 2023-03-23 - [PR#848](https://github.com/NOAA-OWP/inundation-mapping/pull/848)
+
+Introduces two new arguments (`-pcsv` and `-pfiles`) and improves the documentation of  `synthesize_test_cases.py`. The new arguments allow the user to provide a CSV of previous metrics (`-pcsv`) and to specity whether or not metrics should pulled from previous directories (`-pfiles`). 
+
+The dtype warning was suppressed through updates to the `read_csv` function in `hydrotable.py` and additional comments were added throughout script to improve readability.
+
+### Changes
+- `tools/inundation.py`: Add data types to the section that reads in the hydrotable (line 483).
+
+- `tools/synthesize_test_cases.py`: Improved formatting, spacing, and added comments. Added two new arguments: `pcsv` and `pfiles` along with checks to verify they are not being called concurrently (lines 388-412). In `create_master_metrics_csv`, creates an `iteration_list` that only contains `['comparison']` if `pfiles` is not true, reads in the previous metric csv `prev_metrics_csv` if it is provided and combine it with the compiled metrics (after it is converted to dataframe), and saves the metrics dataframe (`df_to_write`) to CSV.
+  
+<br/><br/>
+
 ## v4.3.3.4 - 2023-03-17 - [PR#849](https://github.com/NOAA-OWP/inundation-mapping/pull/849)
 
 This hotfix addresses an error in inundate_nation.py relating to projection CRS.
 
-## Changes
+### Changes
 
 - `tools/inundate_nation.py`: #782 CRS projection change likely causing issue with previous projection configuration
 
diff --git a/tools/inundation.py b/tools/inundation.py
index 188f0bce0..c55ade9a7 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -480,7 +480,11 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
                                  hydroTable,
                                  dtype={'HUC':str,'feature_id':str,
                                          'HydroID':str,'stage':float,
-                                         'discharge_cms':float,'LakeID' : int}
+                                         'discharge_cms':float,
+                                         'LakeID' : int,
+                                         'last_updated':object, 
+                                         'submitter':object, 
+                                         'obs_source':object}
                                 )
         huc_error = hydroTable.HUC.unique()
         hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
@@ -624,7 +628,7 @@ def create_src_subset_csv(hydro_table,catchmentStagesDict,src_table):
     src_df = pd.DataFrame.from_dict(catchmentStagesDict, orient='index')
     src_df.reset_index(inplace=True)
     src_df.columns = ['HydroID','stage_inund']
-    df_htable = pd.read_csv(hydro_table,dtype={'HydroID': int})
+    df_htable = pd.read_csv(hydro_table,dtype={'HydroID': int,'HUC': object, 'branch_id':int, 'last_updated':object, 'submitter':object, 'obs_source':object})
     df_htable = df_htable.merge(src_df,how='left',on='HydroID')
     df_htable['find_match'] = (df_htable['stage'] - df_htable['stage_inund']).abs()
     df_htable = df_htable.loc[df_htable.groupby('HydroID')['find_match'].idxmin()].reset_index(drop=True)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 21c5d47d4..fe788871a 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -1,31 +1,31 @@
 #!/usr/bin/env python3
 
 import os, argparse, json, csv, ast, re, sys, traceback, signal
+import pandas as pd
 from datetime import datetime
 from multiprocessing import Pool
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 from tqdm import tqdm
-
 from utils.shared_functions import FIM_Helpers as fh
 from run_test_case import test_case
 from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT
 
-
-def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list):
+def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list, pfiles, prev_metrics_csv):
     """
     This function searches for and collates metrics into a single CSV file that can queried database-style. The
     CSV is an input to eval_plots.py. This function automatically looks for metrics produced for official versions
     and loads them into memory to be written to the output CSV.
     
     Args:
-        master_metrics_csv_output (str): Full path to CSV output. If a file already exists at this path, it will be overwritten.
-        dev_versions_to_include_list (list): A list of non-official FIM version names. If a user supplied information on the command
-                                            line using the -dc flag, then this function will search for metrics in the "testing_versions"
-                                            library of metrics and include them in the CSV output.
-    
+        master_metrics_csv_output (str): Full path to CSV output. If a file already exists at this path, it will be 
+        overwritten.
+        dev_versions_to_include_list (list): A list of non-official FIM version names. If a user supplied information 
+                                            on the command line using the -dc flag, then this function will search for 
+                                            metrics in the "testing_versions" library of metrics and include them in 
+                                            the CSV output.
     """
     
-    # Construct header
+    # Construct header  
     metrics_to_write = ['true_negatives_count',
                         'false_negatives_count',
                         'true_positives_count',
@@ -70,42 +70,50 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                         'masked_area_km2'
                         ]
 
-
+    # Create table header
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
     list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
 
     versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
 
+    # Specify which results to iterate through
     if len(dev_versions_to_include_list) > 0:
-        iteration_list = ['official', 'comparison']
+        iteration_list = ['official', 'comparison'] # iterating through official model results AND comparison model(s)
     else:
-        iteration_list = ['official']
+        iteration_list = ['official'] # only iterating through official model results
 
+    if pfiles != True: 
+        iteration_list = ['comparison'] # not iterating through official model results, just the comparison model(s) 
+
+    # Iterate through 5 benchmark sources 
     for benchmark_source in ['ble', 'nws', 'usgs', 'ifc','ras2fim']:
         benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
         test_cases_list = [d for d in os.listdir(benchmark_test_case_dir) if re.match('\d{8}_\w{3,7}', d)]
+
         if benchmark_source in ['ble', 'ifc','ras2fim']:
-            
             magnitude_list = MAGNITUDE_DICT[benchmark_source]
             
+            # Iterate through available test cases
             for test_case in test_cases_list:
                 try:
+                    # Get HUC id
                     int(test_case.split('_')[0])
-
                     huc = test_case.split('_')[0]
 
+                    # Update filepaths based on whether the official or dev versions should be included
                     for iteration in iteration_list:
-
-                        if iteration == "official":
+                        if iteration == "official": # and str(pfiles) == "True": # "official" refers to previous finalized model versions
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
                             # add in composite of versions
                             composite_versions = [v.replace('_ms', '_comp') for v in versions_to_aggregate if '_ms' in v]
                             versions_to_aggregate += composite_versions
-                        if iteration == "comparison":
+
+                        if iteration == "comparison": # "comparison" refers to the development model version(s) being evaluated
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = dev_versions_to_include_list
 
+                        # Pull version info from filepath
                         for magnitude in magnitude_list:
                             for version in versions_to_aggregate:
                                 if '_ms' in version:
@@ -121,6 +129,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                 version_dir = os.path.join(versions_to_crawl, version)
                                 magnitude_dir = os.path.join(version_dir, magnitude)
 
+                                # Add metrics from file to metrics table ('list_to_write')
                                 if os.path.exists(magnitude_dir):
                                     magnitude_dir_list = os.listdir(magnitude_dir)
                                     for f in magnitude_dir_list:
@@ -143,27 +152,29 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                 except ValueError:
                     pass
         
+        # Iterate through AHPS benchmark data
         if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
             test_cases_list = os.listdir(benchmark_test_case_dir)
 
             for test_case in test_cases_list:
                 try:
+                    # Get HUC id
                     int(test_case.split('_')[0])
-
                     huc = test_case.split('_')[0]
 
+                    # Update filepaths based on whether the official or dev versions should be included
                     for iteration in iteration_list:
-
-                        if iteration == "official":
+                        if iteration == "official" and str(pfiles) == "True": # "official" refers to previous finalized model versions
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
                             versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
                             # add in composite of versions
                             composite_versions = [v.replace('_ms', '_comp') for v in versions_to_aggregate if '_ms' in v]
                             versions_to_aggregate += composite_versions
-                        if iteration == "comparison":
+                        if iteration == "comparison": # "comparison" refers to the development model version(s) being evaluated
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = dev_versions_to_include_list
 
+                        # Pull model info from filepath
                         for magnitude in ['action', 'minor', 'moderate', 'major']:
                             for version in versions_to_aggregate:
                                 if '_ms' in version:
@@ -179,6 +190,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
                                 version_dir = os.path.join(versions_to_crawl, version)
                                 magnitude_dir = os.path.join(version_dir, magnitude)
+                                
                                 if os.path.exists(magnitude_dir):
                                     magnitude_dir_list = os.listdir(magnitude_dir)
                                     for f in magnitude_dir_list:
@@ -189,7 +201,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                             flow = ''
                                             if os.path.exists(full_json_path):
 
-                                                # Get flow used to map.
+                                                # Get flow used to map
                                                 flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
                                                 if os.path.exists(flow_file):
                                                     with open(flow_file, newline='') as csv_file:
@@ -198,6 +210,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                         for row in reader:
                                                             flow = row[1]
 
+                                                # Add metrics from file to metrics table ('list_to_write')
                                                 stats_dict = json.load(open(full_json_path))
                                                 for metric in metrics_to_write:
                                                     sub_list_to_append.append(stats_dict[metric])
@@ -206,14 +219,30 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                 sub_list_to_append.append(benchmark_source)
                                                 sub_list_to_append.append(extent_config)
                                                 sub_list_to_append.append(calibrated)
-
                                                 list_to_write.append(sub_list_to_append)
                 except ValueError:
-                    pass
-    
-    with open(master_metrics_csv_output, 'w', newline='') as csvfile:
-        csv_writer = csv.writer(csvfile)
-        csv_writer.writerows(list_to_write)
+                    pass 
+
+    # If previous metrics are provided: read in previously compiled metrics and join to calcaulated metrics
+    if prev_metrics_csv != None:
+        prev_metrics_df = pd.read_csv(prev_metrics_csv)
+
+        # Put calculated metrics into a dataframe and set the headers
+        df_to_write_calc = pd.DataFrame(list_to_write)
+        df_to_write_calc.columns = df_to_write_calc.iloc[0]
+        df_to_write_calc = df_to_write_calc[1:]
+
+        # Join the calculated metrics and the previous metrics dataframe
+        df_to_write = pd.concat([df_to_write_calc, prev_metrics_df], axis=0)
+
+    else:
+        df_to_write = pd.DataFrame(list_to_write)
+        df_to_write.columns = df_to_write.iloc[0]
+        df_to_write = df_to_write[1:]
+
+
+    # Save aggregated compiled metrics ('df_to_write') as a CSV
+    df_to_write.to_csv(master_metrics_csv_output,index=False)
 
 def progress_bar_handler(executor_dict, verbose, desc):
 
@@ -227,7 +256,6 @@ def progress_bar_handler(executor_dict, verbose, desc):
         except Exception as exc:
             print('{}, {}, {}'.format(executor_dict[future],exc.__class__.__name__,exc))
 
-
 if __name__ == '__main__':
 
     # Sample usage:
@@ -238,13 +266,16 @@ def progress_bar_handler(executor_dict, verbose, desc):
      Notes:
        - fim_input.csv MUST be in the folder suggested.
        - the -v param is the name in the folder in the "outputs/" directory where the test hucs are at.
-           It also becomes the folder names inside the test_case folders when done.
+         It also becomes the folder names inside the test_case folders when done.
        - the -vg param may not be working (will be assessed better on later releases).
        - Find a balance between -jh (number of jobs for hucs) versus -jb (number of jobs for branches)
          on quick tests on a 96 core machine, we tried [1 @ 80], [2 @ 40], and [3 @ 25] (and others).
        -jb 3 -jh 25 was noticably better. You can likely go more jb cores with better success, just
          experiment.  Start times, End Times and duration are now included.
        - The -m can be any path and any name.
+       - Previous metric CSV (-pcsv) and the cycle previous files argument (-pfiles) will return an error 
+         if called at the same time. If neither are used, the alpha test metrics will only be compiled 
+         for the provided dev version to compare. 
     
      To see your outputs in the test_case folder (hard coded path), you can check for outputs using
          (cd .... to your test_case folder), then command becomes  find . -name gms_test_* -type d (Notice the
@@ -271,20 +302,31 @@ def progress_bar_handler(executor_dict, verbose, desc):
 
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=False,default='DEV')
-    parser.add_argument('-l','--calibrated',help='Denotes use of calibrated n values. This should be taken from meta-data from hydrofabric dir',required=False, default=False,action='store_true')
-    parser.add_argument('-e','--model',help='Denotes model used. FR, MS, or GMS allowed. This should be taken from meta-data in hydrofabric dir.', default='GMS', required=False)
-    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
-    parser.add_argument('-jh','--job-number-huc',help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',required=False, default=1,type=int)
-    parser.add_argument('-jb','--job-number-branch',help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',required=False, default=1,type=int)
-    parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
-    parser.add_argument('-b','--benchmark-category',help='A benchmark category to specify. Defaults to process all categories.',required=False, default="all")
-    parser.add_argument('-o','--overwrite',help='Overwrite all metrics or only fill in missing metrics.',required=False, action="store_true")
-    parser.add_argument('-dc', '--dev-version-to-compare', nargs='+', help='Specify the name(s) of a dev (testing) version to include in master metrics CSV. Pass a space-delimited list.',required=False)
-    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=False,default=None)
-    parser.add_argument('-d','--fr-run-dir',help='Name of test case directory containing FIM for FR model',required=False,default=None)
-    parser.add_argument('-vr','--verbose',help='Verbose',required=False,default=None,action='store_true')
-    parser.add_argument('-vg','--gms-verbose',help='GMS Verbose Progress Bar',required=False,default=None,action='store_true')
+    parser.add_argument('-c','--config', help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"', 
+                        required=False, default='DEV')
+    parser.add_argument('-l','--calibrated', help='Denotes use of calibrated n values. This should be taken from meta-data from hydrofabric dir', 
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-e','--model', help='Denotes model used. FR, MS, or GMS allowed. This should be taken from meta-data in hydrofabric dir.', 
+                        default='GMS', required=False)
+    parser.add_argument('-v','--fim-version', help='Name of fim version to cache.', required=False, default="all")
+    parser.add_argument('-jh','--job-number-huc', help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply '\
+                        'to no more than one less than the CPU count of the machine.', required=False, default=1, type=int)
+    parser.add_argument('-jb','--job-number-branch', help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should '\
+                        'multiply to no more than one less than the CPU count of the machine.', required=False, default=1, type=int)
+    parser.add_argument('-s','--special-string', help='Add a special name to the end of the branch.', required=False, default="")
+    parser.add_argument('-b','--benchmark-category', help='A benchmark category to specify. Defaults to process all categories.', required=False, 
+                        default="all")
+    parser.add_argument('-o','--overwrite', help='Overwrite all metrics or only fill in missing metrics.', required=False, action="store_true")
+    parser.add_argument('-dc', '--dev-version-to-compare', nargs='+', help='Specify the name(s) of a dev (testing) version to include in master '\
+                        'metrics CSV. Pass a space-delimited list.', required=False)
+    parser.add_argument('-m','--master-metrics-csv', help='Define path for master metrics CSV file.', required=False, default=None)
+    parser.add_argument('-d','--fr-run-dir', help='Name of test case directory containing FIM for FR model', required=False, default=None)
+    parser.add_argument('-vr','--verbose', help='Verbose', required=False, default=None, action='store_true')
+    parser.add_argument('-vg','--gms-verbose', help='GMS Verbose Progress Bar', required=False, default=None, action='store_true')
+    parser.add_argument('-pcsv', '--previous-metrics-csv', help='Optional: Filepath for a CSV with previous metrics to concatenate with new '\
+                        'metrics to form a final aggregated metrics csv.', required=False, default=None)
+    parser.add_argument('-pfiles', '--cycle-previous-files', help='Optional: Specifies whether previous metrics should be compiled by cycling '\
+                        'through files (True). Cannot be used if a previous metrics CSV is provided.', required=False, action="store_true")
 
     # Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -302,6 +344,8 @@ def progress_bar_handler(executor_dict, verbose, desc):
     model = args['model']
     verbose = bool(args['verbose'])
     gms_verbose = bool(args['gms_verbose'])
+    prev_metrics_csv = args['previous_metrics_csv']
+    pfiles = bool(args['cycle_previous_files'])
 
     print("================================")
     print("Start synthesize test cases")
@@ -324,9 +368,9 @@ def progress_bar_handler(executor_dict, verbose, desc):
     if fim_version != "all":
         previous_fim_list = [fim_version]
     else:
-        if config == 'PREV':
+        if config == 'PREV': # official fim model results
             previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)
-        elif config == 'DEV':
+        elif config == 'DEV': # development fim model results
             previous_fim_list = os.listdir(OUTPUTS_DIR)
 
     # Define whether or not to archive metrics in "official_versions" or "testing_versions" for each test_id.
@@ -341,10 +385,35 @@ def progress_bar_handler(executor_dict, verbose, desc):
     all_test_cases = test_case.list_all_test_cases(version = fim_version, archive = archive_results,
             benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
     
+    # Make sure cycle-previous-files and a previous metric CSV have not been concurrently selected
+    if prev_metrics_csv != None and pfiles == True:
+        print(f"Error: Cycle previous files and previous metric CSV functionality cannot be used concurrently.")
+        sys.exit(1)
+
+    # Check whether a previous metrics CSV has been provided and, if so, make sure the CSV exists
+    if prev_metrics_csv != None:
+        if not os.path.exists(prev_metrics_csv):
+            print(f"Error: File does not exist at {prev_metrics_csv}")
+            sys.exit(1)
+        else: 
+            print(f"Metrics will be combined with previous metric CSV: {prev_metrics_csv}") 
+            print()
+    else:
+        print("ALERT: A previous metric CSV has not been provided (-pcsv).")
+        print()
+
+    # Print whether the previous files will be cycled through
+    if pfiles == True:
+        print("ALERT: Metrics from previous directories will be compiled.")
+        print()
+    else:
+        print("ALERT: Metrics from previous directories will NOT be compiled. (-pfiles)")
+        print()
+
     # Set up multiprocessor
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
 
-        ## Loop through all test cases, build the alpha test arguments, and submit them to the process pool
+        # Loop through all test cases, build the alpha test arguments, and submit them to the process pool
         executor_dict = {}
         for test_case_class in all_test_cases:
             
@@ -374,10 +443,10 @@ def progress_bar_handler(executor_dict, verbose, desc):
         progress_bar_handler(executor_dict, True, f"Running {model} alpha test cases with {job_number_huc} workers")
         #wait(executor_dict.keys())
 
-    ## Composite alpha test run is initiated by a MS `model` and providing a `fr_run_dir`
+    # Composite alpha test run is initiated by a MS `model` and providing a `fr_run_dir`
     if model == 'MS' and fr_run_dir:
 
-        ## Rebuild all test cases list with the FR version, loop through them and apply the alpha test
+        # Rebuild all test cases list with the FR version, loop through them and apply the alpha test
         all_test_cases = test_case.list_all_test_cases(version = fr_run_dir, archive = archive_results,
                 benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
 
@@ -436,9 +505,10 @@ def progress_bar_handler(executor_dict, verbose, desc):
         # Do aggregate_metrics.
         print("Creating master metrics CSV...")
 
-        # this function is not compatible with GMS
+        # Note: This function is not compatible with GMS
         create_master_metrics_csv(master_metrics_csv_output = master_metrics_csv, 
-                                  dev_versions_to_include_list = dev_versions_to_include_list)
+                                  dev_versions_to_include_list = dev_versions_to_include_list,
+                                  pfiles=pfiles, prev_metrics_csv=prev_metrics_csv)
     
     print("================================")
     print("End synthesize test cases")
@@ -447,7 +517,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
     print (f"ended: {dt_string}")
 
-    # calculate duration
+    # Calculate duration
     time_duration = end_time - start_time
     print(f"Duration: {str(time_duration).split('.')[0]}")
     print()

From 97a685235f288a12e005c59e3e9c3cf4a5e02a0d Mon Sep 17 00:00:00 2001
From: EmilyDeardorff-NOAA <60829052+EmilyDeardorff@users.noreply.github.com>
Date: Fri, 31 Mar 2023 09:34:14 -0700
Subject: [PATCH 227/359] v4.3.3.6 Branch should be get cleaned up with an exit
 of code 61 (#859)

---
 docs/CHANGELOG.md     | 11 +++++++++++
 src/process_branch.sh |  1 +
 2 files changed, 12 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index cf9e47f74..0387dc2cf 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.3.6 - 2023-03-30 - [PR#859](https://github.com/NOAA-OWP/inundation-mapping/pull/859)
+
+Addresses the issue of output storage space being taken up by output files from branches that did not run. Updates branch processing to remove the extraneous branch file if a branch gets an error code of 61.
+
+### Changes
+
+- `src/process_branch.sh`: added line 41, which removes the outputs and output folder if Error 61 occurs.
+
+<br/><br/>
+
+=======
 ## v4.3.3.5 - 2023-03-23 - [PR#848](https://github.com/NOAA-OWP/inundation-mapping/pull/848)
 
 Introduces two new arguments (`-pcsv` and `-pfiles`) and improves the documentation of  `synthesize_test_cases.py`. The new arguments allow the user to provide a CSV of previous metrics (`-pcsv`) and to specity whether or not metrics should pulled from previous directories (`-pfiles`). 
diff --git a/src/process_branch.sh b/src/process_branch.sh
index bd1d4d90f..da9897c8d 100755
--- a/src/process_branch.sh
+++ b/src/process_branch.sh
@@ -38,6 +38,7 @@ do
         echo
         err_exists=1
         echo "***** Branch has no valid flowlines *****"
+        rm -rf $outputRunDataDir/$hucNumber/branches/$branchId/
     elif [ $code -ne 0 ]; then
         echo
         err_exists=1

From 32365472e51d6292c62fe82513260dd3cf7ba443 Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 31 Mar 2023 16:37:43 +0000
Subject: [PATCH 228/359] v4.3.3.7 Update PR template (#856)

---
 .github/PULL_REQUEST_TEMPLATE.md | 64 +++++++-------------------------
 docs/CHANGELOG.md                | 12 +++++-
 2 files changed, 25 insertions(+), 51 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 69d40bb97..306c46d21 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,63 +1,27 @@
 [Short description explaining the high-level reason for the pull request]
 
-## Additions
+### Additions  
 
--
+### Changes  
 
-## Removals
+### Removals 
 
--
+### Testing
 
-## Changes
 
--
+### Checklist
 
-## Testing
+_You may update this checklist before and/or after creating the PR. If you're unsure about any of them, please ask, we're here to help! These items are what we are going to look for before merging your code._
 
-1.
-
-## Screenshots
-
-
-## Notes
-
--
-
-## Todos
-
--
-
-## Checklist
-
-- [ ] PR has an informative and human-readable title
+- [ ] Informative and human-readable title, using the format: `[_pt] PR: <description>`
+- [ ] Links are provided if this PR resolves an issue, or depends on another other PR
+- [ ] If submitting a PR to the `dev` branch (the default branch), you have a descriptive [Feature Branch](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow) name using the format: `dev-<description-of-change>` (e.g.: `dev-revise-levee-masking`)
 - [ ] Changes are limited to a single goal (no scope creep)
-- [ ] Code can be automatically merged (no conflicts)
-- [ ] Code follows project standards (link if applicable)
-- [ ] Passes all existing automated tests
+- [ ] The feature branch you're submitting as a PR is up to date (merged) with the latest `dev` branch
+- [ ] Changes adhere to [PEP-8 Style Guidelines](https://peps.python.org/pep-0008/)
 - [ ] Any _change_ in functionality is tested
+- [ ] Passes all unit tests locally (inside interactive Docker container, at `/foss_fim/`, run: `pytest unit_tests/`)
 - [ ] New functions are documented (with a description, list of inputs, and expected output)
 - [ ] Placeholder code is flagged / future todos are captured in comments
-- [ ] Visually tested in supported browsers and devices (see checklist below :point_down:)
-- [ ] Project documentation has been updated (including the "Unreleased" section of the CHANGELOG)
-- [ ] Reviewers requested with the [Reviewers tool](https://help.github.com/articles/requesting-a-pull-request-review/) :arrow_right:
-
-## Testing checklist
-
-### Target Environment support
-
-- [ ] Windows
-- [ ] Linux
-- [ ] Browser
-
-### Accessibility
-
-- [ ] Keyboard friendly
-- [ ] Screen reader friendly
-
-### Other
-
-- [ ] Is useable without CSS
-- [ ] Is useable without JS
-- [ ] Flexible from small to large screens
-- [ ] No linting errors or warnings
-- [ ] JavaScript tests are passing
+- [ ] Project documentation has been updated ([CHANGELOG](/docs/CHANGELOG.md) and/or [README](/README.md))
+- [ ] [Reviewers requested](https://help.github.com/articles/requesting-a-pull-request-review/)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0387dc2cf..cd39da6b0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.3.3.7 - 2023-03-22 - [PR#856](https://github.com/NOAA-OWP/inundation-mapping/pull/856)
+
+Simple update to the `PULL_REQUEST_TEMPLATE.md` to remove unnecessary/outdated boilerplate items, add octothorpe (#) in front of Additions, Changes, Removals to mirror `CHANGELOG.md` format, and clean up the PR Checklist.
+
+### Changes
+- `docs/`  
+  - `PULL_REQUEST_TEMPLATE.md` 
+
+<br/><br/>
+  
 ## v4.3.3.6 - 2023-03-30 - [PR#859](https://github.com/NOAA-OWP/inundation-mapping/pull/859)
 
 Addresses the issue of output storage space being taken up by output files from branches that did not run. Updates branch processing to remove the extraneous branch file if a branch gets an error code of 61.
@@ -11,7 +22,6 @@ Addresses the issue of output storage space being taken up by output files from
 
 <br/><br/>
 
-=======
 ## v4.3.3.5 - 2023-03-23 - [PR#848](https://github.com/NOAA-OWP/inundation-mapping/pull/848)
 
 Introduces two new arguments (`-pcsv` and `-pfiles`) and improves the documentation of  `synthesize_test_cases.py`. The new arguments allow the user to provide a CSV of previous metrics (`-pcsv`) and to specity whether or not metrics should pulled from previous directories (`-pfiles`). 

From ee26f0f41297a015e2a744426c42e47ab7c40d84 Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 31 Mar 2023 18:50:08 +0000
Subject: [PATCH 229/359] v4.3.4.0 Add Temporary Working Directory (#847)

---
 Dockerfile                                    | 11 ++-
 README.md                                     | 22 ++++-
 docs/CHANGELOG.md                             | 39 ++++++++
 fim_pipeline.sh                               | 11 +--
 fim_post_processing.sh                        | 46 ++++-----
 fim_pre_processing.sh                         | 45 ++++-----
 fim_process_unit_wb.sh                        | 48 +++++----
 src/bash_variables.env                        | 24 ++---
 src/check_huc_inputs.py                       |  2 +-
 src/delineate_hydros_and_produce_HAND.sh      | 63 ++++++------
 src/process_branch.sh                         | 15 ++-
 src/run_by_branch.sh                          | 38 ++++----
 src/run_unit_wb.sh                            | 97 ++++++++++---------
 src/utils/shared_functions.py                 |  2 +-
 tools/inundation_wrapper_custom_flow.py       |  2 +-
 tools/inundation_wrapper_nwm_flows.py         |  2 +-
 tools/tools_shared_variables.py               |  2 +-
 unit_tests/README.md                          | 22 +++--
 unit_tests/aggregate_branch_lists_params.json |  4 +-
 unit_tests/check_unit_errors_params.json      |  6 +-
 unit_tests/check_unit_errors_test.py          |  2 +-
 unit_tests/clip_vectors_to_wbd_params.json    | 19 ++--
 unit_tests/derive_level_paths_params.json     | 16 +--
 unit_tests/derive_level_paths_test.py         |  2 +-
 ..._catchments_and_add_attributes_params.json | 12 +--
 .../generate_branch_list_csv_params.json      |  6 +-
 unit_tests/generate_branch_list_params.json   |  4 +-
 unit_tests/outputs_cleanup_params.json        |  6 +-
 .../rating_curve_comparison_params.json       | 10 +-
 unit_tests/shared_functions_params.json       |  4 +-
 unit_tests/shared_functions_test.py           |  8 +-
 unit_tests/split_flows_params.json            | 16 +--
 unit_tests/split_flows_test.py                |  4 +-
 unit_tests/tools/inundate_gms_params.json     |  8 +-
 unit_tests/tools/inundation_params.json       |  8 +-
 unit_tests/usgs_gage_crosswalk_params.json    | 14 +--
 36 files changed, 349 insertions(+), 291 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f8f535cef..4dc93f128 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,13 +49,12 @@ RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d
 # Base Image that has GDAL, PROJ, etc
 FROM osgeo/gdal:ubuntu-full-3.1.2
 ARG dataDir=/data
-# remove below line, and uncomment # ENV projectDir=/foss_fim, & delete this line
-ARG projectDir=/foss_fim
-# ENV projectDir=/foss_fim
+ENV projectDir=/foss_fim
 ARG depDir=/dependencies
-ENV inputDataDir=$dataDir/inputs
-ENV outputDataDir=/outputs
+ENV inputsDir=$dataDir/inputs
+ENV outputsDir=/outputs
 ENV srcDir=$projectDir/src
+ENV workDir=/fim_temp
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
@@ -66,6 +65,8 @@ RUN addgroup --gid $GroupID $GroupName
 ENV GID=$GroupID
 ENV GN=$GroupName
 
+RUN mkdir -p $workDir
+
 RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
diff --git a/README.md b/README.md
index 8c7c7570a..7b86974bf 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,8 @@ aws s3 ls s3://noaa-nws-owp-fim/ --request-payer requester
 
 Download a directory of outputs for a HUC8:
 ```
-aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_0_18_02/12090301 /your_local_folder_name/12090301 --request-payer requester
+aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_0_18_02/12090301 \
+    /your_local_folder_name/12090301 --request-payer requester
 ```
 By adjusting pathing, you can also download entire directories such as the fim_4_0_18_0 folder.
 **Note**: There may be newer editions than fim_4_0_18_0, and it is recommended to adjust the command above for the latest version.
@@ -72,11 +73,19 @@ This system has an optional tool called the `calibration database tool`. In orde
 Since all of the dependencies are managed in utilizing a Docker container, we must issue the [`docker run`](https://docs.docker.com/engine/reference/run/#clean-up---rm) command to start a container as the run-time environment. The container is launched from a Docker Image which was built in [Installation](#installation) step 2. The correct input file pathing is necessary for the `/data` volume mount (`-v`) for the `<input_path>`. The `<input_path>` should contain a subdirectory named `/inputs` (similar to `s3://noaa-nws-owp-fim/hand_fim`). If the pathing is set correctly, we do not need to adjust the `params_template.env` file, and can use the default file paths provided.
 
 ```bash 
-docker run --rm -it --name <your_container_name> -v <path/to/repository>/:/foss_fim -v <desired_output_path>/:/outputs -v <input_path>:/data <image_name>:<tag>
+docker run --rm -it --name <your_container_name> \
+    -v <path/to/repository>/:/foss_fim \
+    -v <desired_output_path>/:/outputs \
+    -v <input_path>:/data \
+    <image_name>:<tag>
 ```
 For example:  
 ```bash
-docker run --rm -it --name robs_container -v /home/projects/inundation-mapping/:/foss_fim -v /home/projects/fim/outputs/:/outputs -v /home/projects/fim/inputs/:/data fim_4:dev_20230224_ad87a74
+docker run --rm -it --name robs_container \
+    -v /home/projects/inundation-mapping/:/foss_fim \
+    -v /home/projects/fim/outputs/:/outputs \
+    -v /home/projects/fim/inputs/:/data \
+    fim_4:dev_20230224_ad87a74
 ```
 
 ### Produce HAND Hydrofabric
@@ -120,7 +129,12 @@ After `fim_pipeline.sh` completes, or combinations of the three major steps desc
 
 To evaluate model skill, run the following:
 ```
-python /foss_fim/tools/synthesize_test_cases.py -c DEV -v <fim_run_name> -m <path/to/output/metrics.csv> -j [num_of_jobs (cores and/or procs)]
+python /foss_fim/tools/synthesize_test_cases.py \
+    -c DEV \
+    -v <fim_run_name> \
+    -m <path/to/output/metrics.csv> \
+    -jh [num of jobs (cores and/or procs) per huc] \
+    -jb [num of jobs (cores and/or procs) per branch]
 ```
 
 More information can be found by running:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index cd39da6b0..2e2956ad3 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,45 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.4.0 - 2023-03-16-23 [PR#847](https://github.com/NOAA-OWP/inundation-mapping/pull/847)
+
+### Changes
+
+Create a 'working directory' in the Docker container to run processes within the container's non-persistent filesystem. Modify variables in scripts that process HUCs and branches to use the temporary working directory, and then copy temporary directory (after trimming un-wanted files) over to output directory (persistent filesystem).  Roll back changes to `unit_tests/` to use `/data/outputs` (contains canned data), as the volume mounted `outputs/` most likely will not contain the necessary unit test data. 
+
+- `Dockerfile` - create a `/fim_temp` working directory, update `projectDir` to an `ENV`, rename inputs and outputs directory variables  
+- `fim_pipeline.sh` - remove `projectDir=/foss_fim`, update path of `logFile`, remove indentation  
+- `fim_pre_processing.sh` - change `$outputRunDataDir` => `$outputDestDir` & add `$tempRunDir`  
+- `fim_post_processing.sh` - change `$outputRunDataDir` => `$outputDestDir`   
+- `fim_process_unit_wb.sh` - change `$outputRunDataDir` => `$outputDestDir`, add vars & export `tempRunDir`, `tempHucDataDir`, & `tempBranchDataDir` to `run_unit_wb.sh`  
+- `README.md` - add linebreaks to codeblocks  
+
+- `src/` 
+  - `bash_variables.env` - `$inputDataDir` => `$inputsDir`  
+  - `check_huc_inputs.py` - `$inputDataDir` => `$inputsDir`  
+  - `delineate_hydros_and_produce_HAND.py` - `$outputHucDataDir` => `$tempHucDataDir`, `$outputCurrentBranchDataDir` => `$tempCurrentBranchDataDir`  
+  - `process_branch.sh` - `$outputRunDataDir` => `$outputsDestDir`  
+  - `run_by_branch.sh` - `$outputCurrentBranchDataDir` => `$tempCurrentBranchDataDir`, `$outputHucDataDir` => `$tempHucDataDir`  
+  - `run_unit_wb.sh` - `$outputRunDataDir` => `$outputDestDir`, `$outputHucDataDir` => `$tempHucDataDir`  
+  - `utils/`  
+    - `shared_functions.py` - `$inputDataDir` => `$inputsDir`  
+
+- `tools/` 
+  - `inundation_wrapper_custom_flow.py` - `$outputDataDir` => `$outputsDir`  
+  - `inundation_wrapper_nwm_flows.py`  - `$outputDataDir` => `$outputsDir`  
+  - `tools_shared_variables.py` - `$outputDataDir` => `$outputsDir`    
+
+- `unit_tests/`
+  - `README.md` - add linebreaks to code blocks, `/outputs/` => `/data/outputs/`  
+  - `*_params.json` - `/outputs/` => `/data/outputs/` & `$outputRunDataDir` => `$outputDestDir`  
+  - `derive_level_paths_test.py` - `$outputRunDataDir` => `$outputDestDir`  
+  - `check_unit_errors_test.py` - `/outputs/` => `/data/outputs/`  
+  - `shared_functions_test.py` - `$outputRunDataDir` => `$outputDestDir`
+  - `split_flows_test.py`  - `/outputs/` => `/data/outputs/`  
+  - `tools/` 
+    - `*_params.json` - `/outputs/` => `/data/outputs/` & `$outputRunDataDir` => `$outputDestDir`  
+
+<br/><br/>
 
 ## v4.3.3.7 - 2023-03-22 - [PR#856](https://github.com/NOAA-OWP/inundation-mapping/pull/856)
 
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 154f899fe..a783ae92b 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -22,10 +22,6 @@ Running the `fim_pipeline.sh` is a quicker process than running all three steps
 
 set -e
 
-# TODO
-# update Dockerfile to add this as an env value, and delete line below
-projectDir=/foss_fim
-
 # See fim_pre_processing.sh for details of how to use this script. fim_pre_processing.sh
 # is a proxy for collecting and validating input.
 
@@ -37,8 +33,7 @@ echo "---- Started: `date -u`"
 source $srcDir/bash_functions.env
 . $projectDir/fim_pre_processing.sh "$@"
 
-
-logFile=$outputRunDataDir/logs/unit/pipeline_summary_unit.log
+logFile=$outputDestDir/logs/unit/pipeline_summary_unit.log
 process_wb_file=$projectDir/fim_process_unit_wb.sh
 
 pipeline_start_time=`date +%s`
@@ -47,13 +42,13 @@ pipeline_start_time=`date +%s`
 # Why an if and else? watch the number of colons
 if [ -f "$hucList" ]; then
     if [ "$jobHucLimit" = "1" ]; then
-                parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList 
+        parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList 
     else
         parallel --eta -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList
     fi
 else 
     if [ "$jobHucLimit" = "1" ]; then
-                parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName ::: $hucList
+        parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName ::: $hucList
     else
         parallel --eta -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file  $runName  ::: $hucList
     fi
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 54ad2a5b1..7ed679be6 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -40,11 +40,6 @@ in
     shift
 done
 
-# TODO
-# update Dockerfile to add this as an env value, and delete line below
-projectDir=/foss_fim
-
-
 # print usage if arguments empty
 if [ "$runName" = "" ]
 then
@@ -52,10 +47,10 @@ then
     usage
 fi
 
-outputRunDataDir=$outputDataDir/$runName
+outputDestDir=$outputsDir/$runName
 
-## Check for run data directory ##
-if [ ! -d "$outputRunDataDir" ]; then 
+## Check for output destination directory ##
+if [ ! -d "$outputDestDir" ]; then 
     echo "Depends on output from units and branches. Please provide an output folder name that has hucs/branches run."
     exit 1
 fi
@@ -63,16 +58,16 @@ fi
 if [ "$jobLimit" = "" ]; then jobLimit=1; fi
 
 # Clean out the other post processing files before starting
-rm -rdf $outputRunDataDir/logs/src_optimization
-rm -f $outputRunDataDir/logs/log_bankfull_indentify.log
-rm -f $outputRunDataDir/logs/subdiv_src_.log
+rm -rdf $outputDestDir/logs/src_optimization
+rm -f $outputDestDir/logs/log_bankfull_indentify.log
+rm -f $outputDestDir/logs/subdiv_src_.log
 
 # load up enviromental information
-args_file=$outputRunDataDir/runtime_args.env
-fim_inputs=$outputRunDataDir/fim_inputs.csv
+args_file=$outputDestDir/runtime_args.env
+fim_inputs=$outputDestDir/fim_inputs.csv
 
 source $args_file
-source $outputRunDataDir/params.env
+source $outputDestDir/params.env
 source $srcDir/bash_functions.env
 source $srcDir/bash_variables.env
 
@@ -87,25 +82,25 @@ post_proc_start_time=`date +%s`
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
 echo -e $startDiv"Start branch aggregation"
-python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $fim_inputs
+python3 $srcDir/aggregate_branch_lists.py -d $outputDestDir -f "branch_ids.csv" -o $fim_inputs
 
 ## GET NON ZERO EXIT CODES FOR UNITS ##
 ## No longer applicable
 
 ## GET NON ZERO EXIT CODES FOR BRANCHES ##
 echo -e $startDiv"Start non-zero exit code checking"
-find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
+find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputDestDir/branch_errors/non_zero_exit_codes.log" &
 
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
 echo "Processing usgs gage aggregation"
-python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -i $fim_inputs
+python3 $srcDir/usgs_gage_aggregate.py -fim $outputDestDir -i $fim_inputs
 
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
     echo -e $startDiv"Estimating bankfull stage in SRCs"
     # Run SRC bankfull estimation routine routine
     Tstart
-    python3 $srcDir/identify_src_bankfull.py -fim_dir $outputRunDataDir -flows $bankfull_flows_file -j $jobLimit
+    python3 $srcDir/identify_src_bankfull.py -fim_dir $outputDestDir -flows $bankfull_flows_file -j $jobLimit
     Tcount
 fi
 
@@ -114,14 +109,15 @@ if [ "$src_subdiv_toggle" = "True" ]; then
     echo -e $startDiv"Performing SRC channel/overbank subdivision routine"
     # Run SRC Subdivision & Variable Roughness routine
     Tstart
-    python3 $srcDir/subdiv_chan_obank_src.py -fim_dir $outputRunDataDir -mann $vmann_input_file -j $jobLimit
+    python3 $srcDir/subdiv_chan_obank_src.py -fim_dir $outputDestDir -mann $vmann_input_file -j $jobLimit
     Tcount
 fi
 
 ## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
 if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
     if [ ! -f $CALB_DB_KEYS_FILE ]; then
-        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file), but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
+        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file),"
+        echo "but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
         exit 1
     else
         source $CALB_DB_KEYS_FILE
@@ -152,7 +148,7 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
         echo "Loading HUC Data"
         echo
 
-        ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+        ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputsDir/wbd/WBD_National.gpkg WBDHU8
 
         echo "Loading Point Data"
         echo
@@ -170,7 +166,7 @@ if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
     echo    
     echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
     # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
-    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputRunDataDir -usgs_rc $inputDataDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputDestDir -usgs_rc $inputsDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
     Tcount
     date -u
 fi
@@ -180,7 +176,7 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [
     Tstart
     echo
     echo -e $startDiv"Performing SRC adjustments using benchmark point database"
-    python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputRunDataDir -j $jobLimit
+    python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputDestDir -j $jobLimit
     Tcount
     date -u
 fi
@@ -189,13 +185,13 @@ echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
 Tstart
-python3 /foss_fim/tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
+python3 /foss_fim/tools/combine_crosswalk_tables.py -d $outputDestDir -o $outputDestDir/crosswalk_table.csv
 Tcount
 date -u
 
 echo
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- Start of fim_post_processing complete"
+echo "---- End of fim_post_processing, fim_post_processing complete"
 echo "---- Ended: `date -u`"
 Calc_Duration $post_proc_start_time
 echo
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index b11b6084d..22f6d704d 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -123,12 +123,9 @@ then
     usage
 fi
 
-# TODO
-# update Dockerfile to add this as an env value, and delete line below
-projectDir=/foss_fim
-
-# outputDataDir, srcDir and others come from the Dockerfile
-outputRunDataDir=$outputDataDir/$runName
+# outputsDir & workDir come from the Dockerfile
+outputDestDir=$outputsDir/$runName
+tempRunDir=$workDir/$runName
 
 # default values
 if [ "$envFile" = "" ]; then envFile=/$projectDir/config/params_template.env; fi
@@ -183,9 +180,9 @@ else
 fi
 
 # Safety feature to avoid accidentaly overwrites
-if [ -d $outputRunDataDir ] && [ $overwrite -eq 0 ]; then
+if [ -d $outputDestDir ] && [ $overwrite -eq 0 ]; then
     echo
-    echo "ERROR: Output dir $outputRunDataDir exists. Use overwrite -o to run."
+    echo "ERROR: Output dir $outputDestDir exists. Use overwrite -o to run."
     echo        
     usage
 fi
@@ -202,34 +199,34 @@ echo
 echo "--- Number of HUCs to process is $num_hucs"
 
 # make dirs
-if [ ! -d $outputRunDataDir ]; then
-    mkdir -p $outputRunDataDir
+if [ ! -d $outputDestDir ]; then
+    mkdir -p $outputDestDir
+    mkdir -p $tempRunDir
 else
     # remove these directories and files on a new or overwrite run
-    rm -rdf $outputRunDataDir/logs
-    rm -rdf $outputRunDataDir/branch_errors
-    rm -rdf $outputRunDataDir/unit_errors
-    rm -rdf $outputRunDataDir/eval
-    rm -f $outputRunDataDir/crosswalk_table.csv
-    rm -f $outputRunDataDir/fim_inputs*
-    rm -f $outputRunDataDir/*.env
+    rm -rdf $outputDestDir/logs
+    rm -rdf $outputDestDir/branch_errors
+    rm -rdf $outputDestDir/unit_errors
+    rm -rdf $outputDestDir/eval
+    rm -f $outputDestDir/crosswalk_table.csv
+    rm -f $outputDestDir/fim_inputs*
+    rm -f $outputDestDir/*.env
 fi
 
-#logFile=$outputRunDataDir/logs/unit/summary_unit.log
-mkdir -p $outputRunDataDir/logs/unit
-mkdir -p $outputRunDataDir/logs/branch
-mkdir -p $outputRunDataDir/unit_errors
-mkdir -p $outputRunDataDir/branch_errors
+mkdir -p $outputDestDir/logs/unit
+mkdir -p $outputDestDir/logs/branch
+mkdir -p $outputDestDir/unit_errors
+mkdir -p $outputDestDir/branch_errors
 
 # copy over config file and rename it (note.. yes, the envFile file can still be
 # loaded from command line and have its own values, it simply gets renamed and saved)
-cp $envFile $outputRunDataDir/params.env
+cp $envFile $outputDestDir/params.env
 
 # create an new .env file on the fly that contains all runtime values
 # that any unit can load it independently (in seperate AWS objects, AWS fargates)
 # or via pipeline. There is likely a more elegent way to do this.
 
-args_file=$outputRunDataDir/runtime_args.env
+args_file=$outputDestDir/runtime_args.env
 
 # the jobHucLimit is not from the args files, only jobBranchLimit
 echo "export runName=$runName" >> $args_file
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
index 842835a73..dc5fc86bd 100755
--- a/fim_process_unit_wb.sh
+++ b/fim_process_unit_wb.sh
@@ -6,8 +6,7 @@
 
 # This file will also catch any and all errors from src/run_unit_wb.sh file, even script aborts from that file
 
-# You really can not call directly to src/run_unit_wb.sh as that file relys on export values
-# from this file.
+# You really can not call directly to src/run_unit_wb.sh as that file relies on export values from this file.
 # run_unit_wb will futher process branches with its own iterator (parallelization).
 
 # Sample Usage: /foss_fim/fim_process_unit_wb.sh rob_test_wb_1 05030104
@@ -59,11 +58,13 @@ fi
 echo "=========================================================================="
 echo "---- Start of huc processing for $hucNumber"
 
-# outputDataDir, srcDir and others come from the Dockerfile
 
-export outputRunDataDir=$outputDataDir/$runName
-export outputHucDataDir=$outputRunDataDir/$hucNumber
-export outputBranchDataDir=$outputHucDataDir/branches
+# outputsDir, srcDir, workDir and others come from the Dockerfile
+export tempRunDir=$workDir/$runName
+export outputDestDir=$outputsDir/$runName
+export tempHucDataDir=$tempRunDir/$hucNumber
+export outputHucDataDir=$outputDestDir/$hucNumber
+export tempBranchDataDir=$tempHucDataDir/branches
 export current_branch_id=0
 
 ## huc data
@@ -72,16 +73,17 @@ if [ -d "$outputHucDataDir" ]; then
 fi
 
 # make outputs directory
-mkdir -p $outputHucDataDir
-mkdir -p $outputBranchDataDir
+mkdir -p $tempHucDataDir
+mkdir -p $tempBranchDataDir
 
 # Clean out previous unit logs and branch logs starting with this huc
-rm -f $outputRunDataDir/logs/unit/"$hucNumber"_unit.log
-rm -f $outputRunDataDir/logs/branch/"$hucNumber"_summary_branch.log
-rm -f $outputRunDataDir/logs/branch/"$hucNumber"*.log
-rm -f $outputRunDataDir/unit_errors/"$hucNumber"*.log
-rm -f $outputRunDataDir/branch_errors/"$hucNumber"*.log
-hucLogFileName=$outputRunDataDir/logs/unit/"$hucNumber"_unit.log
+rm -f $outputDestDir/logs/unit/"$hucNumber"_unit.log
+rm -f $outputDestDir/logs/branch/"$hucNumber"_summary_branch.log
+rm -f $outputDestDir/logs/branch/"$hucNumber"*.log
+rm -f $outputDestDir/unit_errors/"$hucNumber"*.log
+rm -f $outputDestDir/branch_errors/"$hucNumber"*.log
+
+hucLogFileName=$outputDestDir/logs/unit/"$hucNumber"_unit.log
 
 # Process the actual huc
 /usr/bin/time -v $srcDir/run_unit_wb.sh 2>&1 | tee $hucLogFileName
@@ -90,9 +92,6 @@ hucLogFileName=$outputRunDataDir/logs/unit/"$hucNumber"_unit.log
 # and yes.. we can not use the $? as we are messing with exit codes
 return_codes=( "${PIPESTATUS[@]}" )
 
-#echo "huc return codes are:"
-#echo $return_codes
-
 # we do this way instead of working directly with stderr and stdout
 # as they were messing with output logs which we always want.
 err_exists=0
@@ -101,7 +100,7 @@ do
     # Make an extra copy of the unit log into a new folder.
 
     # Note: It was tricky to load in the fim_enum into bash, so we will just 
-    # go with the code for now
+    # go with the exit code for now
     if [ $code -eq 0 ]; then
         echo
         # do nothing
@@ -122,8 +121,17 @@ done
 
 if [ "$err_exists" = "1" ]; then
     # copy the error log over to the unit_errors folder to better isolate it
-    cp $hucLogFileName $outputRunDataDir/unit_errors
+    cp $hucLogFileName $outputDestDir/unit_errors
 fi
-echo "=========================================================================="
+
+# Here we're using the mv command to move the contents of the temp directory into the specified output directory
+echo "============================================================================================="
+echo 
+mv -f $tempHucDataDir $outputHucDataDir
+echo "***** Copied temp directory: $tempHucDataDir to output directory: $outputHucDataDir  *****"
+echo
+echo "============================================================================================="
+
+
 # we always return a success at this point (so we don't stop the loops / iterator)
 exit 0
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 8d48ce733..46403c153 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -1,17 +1,17 @@
 ## Define inputs
-# NOTE: $inputDataDir is defined in Dockerfile
+# NOTE: $inputsDir is defined in Dockerfile
 export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
-export input_DEM=$inputDataDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
-export input_DEM_domain=$inputDataDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
-export input_GL_boundaries=$inputDataDir/landsea/gl_water_polygons.gpkg
-export input_NLD=$inputDataDir/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
-export input_levees_preprocessed=$inputDataDir/nld_vectors/3d_nld_preprocessed_230314.gpkg
-export input_nld_levee_protected_areas=$inputDataDir/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
-export input_nwm_catchments=$inputDataDir/nwm_hydrofabric/nwm_catchments.gpkg
-export input_nwm_flows=$inputDataDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_nwm_headwaters=$inputDataDir/nwm_hydrofabric/nwm_headwaters.gpkg
-export input_nwm_lakes=$inputDataDir/nwm_hydrofabric/nwm_lakes.gpkg
-export input_WBD_gdb=$inputDataDir/wbd/WBD_National.gpkg
+export input_DEM=$inputsDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
+export input_DEM_domain=$inputsDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
+export input_GL_boundaries=$inputsDir/landsea/gl_water_polygons.gpkg
+export input_NLD=$inputsDir/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
+export input_levees_preprocessed=$inputsDir/nld_vectors/3d_nld_preprocessed_230314.gpkg
+export input_nld_levee_protected_areas=$inputsDir/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
+export input_nwm_catchments=$inputsDir/nwm_hydrofabric/nwm_catchments.gpkg
+export input_nwm_flows=$inputsDir/nwm_hydrofabric/nwm_flows.gpkg
+export input_nwm_headwaters=$inputsDir/nwm_hydrofabric/nwm_headwaters.gpkg
+export input_nwm_lakes=$inputsDir/nwm_hydrofabric/nwm_lakes.gpkg
+export input_WBD_gdb=$inputsDir/wbd/WBD_National.gpkg
 
 # Styling
 export startDiv="\n-----------------------------------------------------------------\n"
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index f44725180..a31296390 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -69,7 +69,7 @@ def __check_for_membership(hucs,accepted_hucs_set):
 
 def check_hucs(hucs):
     
-    huc_list_path = os.path.join(os.environ['inputDataDir'],'huc_lists')
+    huc_list_path = os.path.join(os.environ['inputsDir'],'huc_lists')
     accepted_hucs = __read_included_files(huc_list_path)
     list_hucs = __read_input_hucs(hucs)
     __check_for_membership(list_hucs, accepted_hucs)
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 51cbad515..dc6d9482d 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -7,11 +7,11 @@ level=$1
 T_total_start
 
 ## MASK LEVEE-PROTECTED AREAS FROM DEM ##
-if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $outputHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
+if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $tempHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
     echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/mask_dem.py -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -b $branch_id_attribute -i $current_branch_id -b0 $branch_zero_id -csv $outputHucDataDir/levee_levelpaths.csv -l $levee_id_attribute
+    python3 -m memory_profiler $srcDir/mask_dem.py -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $tempHucDataDir/LeveeProtectedAreas_subset.gpkg -out $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -b $branch_id_attribute -i $current_branch_id -b0 $branch_zero_id -csv $tempHucDataDir/levee_levelpaths.csv -l $levee_id_attribute
     Tcount
 fi
 
@@ -19,105 +19,105 @@ fi
 echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/aread8 -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -wg $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif -nc
+$taudemDir/aread8 -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -ad8 $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -wg $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif -nc
 Tcount
 
 # THRESHOLD ACCUMULATIONS ##
 echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/threshold -ssa $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -thresh 1
+$taudemDir/threshold -ssa $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -thresh 1
 Tcount
 
 ## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
 echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif
+python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $tempCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif
 Tcount
 
 ## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
 echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -a $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif -d $outputCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif -t 50 -o $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -th $thalweg_lateral_elev_threshold
+python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -a $tempCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif -d $tempCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif -t 50 -o $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -th $thalweg_lateral_elev_threshold
 Tcount
 
 ## MASK BURNED DEM FOR STREAMS ONLY ###
 echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -B $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif --calc="A/B" --outfile="$outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" --NoDataValue=0
+gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -B $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif --calc="A/B" --outfile="$tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" --NoDataValue=0
 Tcount
 
 ## FLOW CONDITION STREAMS ##
 echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/flowdircond -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif -z $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -zfdc $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
+$taudemDir/flowdircond -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif -z $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -zfdc $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
 Tcount
 
 ## D8 SLOPES ##
 echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"
 date -u
 Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif 
 Tcount
 
 ## STREAMNET FOR REACHES ##
 echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/streamnet -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -fel $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -ad8 $outputCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -ord $outputCurrentBranchDataDir/streamOrder_$current_branch_id.tif -tree $outputCurrentBranchDataDir/treeFile_$current_branch_id.txt -coord $outputCurrentBranchDataDir/coordFile_$current_branch_id.txt -w $outputCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif -net $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
+$taudemDir/streamnet -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -fel $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -ad8 $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -ord $tempCurrentBranchDataDir/streamOrder_$current_branch_id.tif -tree $tempCurrentBranchDataDir/treeFile_$current_branch_id.txt -coord $tempCurrentBranchDataDir/coordFile_$current_branch_id.txt -w $tempCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif -net $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
 Tcount
 
 ## SPLIT DERIVED REACHES ##
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/split_flows.py -f $outputCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -l $outputHucDataDir/nwm_lakes_proj_subset.gpkg -n $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
+$srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg -n $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg 
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
 echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -gw $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif -o $outputCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
+mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -gw $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif -o $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
 Tcount
 
 ## VECTORIZE FEATURE ID CENTROIDS ##
 echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/reachID_grid_to_vector_points.py -r $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -i featureID -p $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
+$srcDir/reachID_grid_to_vector_points.py -r $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -i featureID -p $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
 Tcount
 
 ## GAGE WATERSHED FOR PIXELS ##
 echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"
 date -u
 Tstart
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif -gw $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg -id $outputCurrentBranchDataDir/idFile_$current_branch_id.txt
+mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif -gw $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
 Tcount
 
 # D8 REM ##
 echo -e $startDiv"D8 REM $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/make_rem.py -d $outputCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $outputCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $outputCurrentBranchDataDir/rem_$current_branch_id.tif -t $outputCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
+$srcDir/make_rem.py -d $tempCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $tempCurrentBranchDataDir/rem_$current_branch_id.tif -t $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
 Tcount
 
 ## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
 echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"
 date -u
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/rem_$current_branch_id.tif -B $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
 Tcount
 
 ## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
-if [ -f $outputHucDataDir/LandSea_subset.gpkg ]; then
+if [ -f $tempHucDataDir/LandSea_subset.gpkg ]; then
     echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"
     date -u
     Tstart
-    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/LandSea_subset.gpkg $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
+    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/LandSea_subset.gpkg $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
     Tcount
 fi
 
@@ -125,42 +125,42 @@ fi
 echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_polygonize.py -8 -f GPKG $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
+gdal_polygonize.py -8 -f GPKG $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
 Tcount
 
 ## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
 echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $outputCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $outputHucDataDir/wbd8_clp.gpkg -u $hucNumber
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -u $hucNumber
 Tcount
 
 ## RASTERIZE NEW CATCHMENTS AGAIN ##
 echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
+gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
 Tcount
 
 ## MASK SLOPE TO CATCHMENTS ##
 echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"
 date -u
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif -B $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif --calc="A*(B>0)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif -B $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif --calc="A*(B>0)" --NoDataValue=$ndv --outfile=$tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
 Tcount
 
 ## MAKE CATCHMENT AND STAGE FILES ##
 echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/make_stages_and_catchlist.py -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -c $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/stage_$current_branch_id.txt -a $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
+$srcDir/make_stages_and_catchlist.py -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -s $tempCurrentBranchDataDir/stage_$current_branch_id.txt -a $tempCurrentBranchDataDir/catch_list_$current_branch_id.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
 Tcount
 
 ## MASK REM RASTER TO REMOVE OCEAN AREAS ##
-if  [ -f $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
+if  [ -f $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
     echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"
     date -u
     Tstart
-    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $outputCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$outputCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
     Tcount
 fi
 
@@ -168,7 +168,7 @@ fi
 echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/catchhydrogeo -hand $outputCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -catch $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif -catchlist $outputCurrentBranchDataDir/catch_list_$current_branch_id.txt -slp $outputCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif -h $outputCurrentBranchDataDir/stage_$current_branch_id.txt -table $outputCurrentBranchDataDir/src_base_$current_branch_id.csv
+$taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -catch $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif -catchlist $tempCurrentBranchDataDir/catch_list_$current_branch_id.txt -slp $tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif -h $tempCurrentBranchDataDir/stage_$current_branch_id.txt -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv
 Tcount
 
 ## FINALIZE CATCHMENTS AND MODEL STREAMS ##
@@ -176,12 +176,13 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_bran
 date -u
 Tstart
 if [ "$level" = "branch" ]; then
-    b_arg=$outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
-    z_arg=$outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg
+    b_arg=$tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
+    z_arg=$tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg
 elif [ "$level" = "unit" ]; then
     # Branch zero has a different source for -b and -z arguments
-    b_arg=$outputHucDataDir/nwm_subset_streams.gpkg
-    z_arg=$outputHucDataDir/nwm_catchments_proj_subset.gpkg
+    b_arg=$tempHucDataDir/nwm_subset_streams.gpkg
+    z_arg=$tempHucDataDir/nwm_catchments_proj_subset.gpkg
 fi
-python3 -m memory_profiler $srcDir/add_crosswalk.py -d $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $outputCurrentBranchDataDir/src_base_$current_branch_id.csv -l $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $outputCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $outputCurrentBranchDataDir/src_$current_branch_id.json -x $outputCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $outputCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $outputHucDataDir/wbd8_clp.gpkg -b $b_arg -y $outputCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -k $outputCurrentBranchDataDir/small_segments_$current_branch_id.csv
+
+python3 -m memory_profiler $srcDir/add_crosswalk.py -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $tempCurrentBranchDataDir/src_base_$current_branch_id.csv -l $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $tempCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $tempCurrentBranchDataDir/src_$current_branch_id.json -x $tempCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $tempCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $tempHucDataDir/wbd8_clp.gpkg -b $b_arg -y $tempCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -k $tempCurrentBranchDataDir/small_segments_$current_branch_id.csv
 Tcount
diff --git a/src/process_branch.sh b/src/process_branch.sh
index da9897c8d..4b1ff0a52 100755
--- a/src/process_branch.sh
+++ b/src/process_branch.sh
@@ -13,10 +13,9 @@ runName=$1
 hucNumber=$2
 branchId=$3
 
-# outputDataDir, srcDir and others come from the Dockerfile
-export outputRunDataDir=$outputDataDir/$runName
-branchLogFileName=$outputRunDataDir/logs/branch/"$hucNumber"_branch_"$branchId".log
-branch_list_csv_file=$outputRunDataDir/$hucNumber/branch_ids.csv
+# outputDestDir & tempHucDataDir come from fim_process_unit_wb.sh
+branchLogFileName=$outputDestDir/logs/branch/"$hucNumber"_branch_"$branchId".log
+branch_list_csv_file=$tempHucDataDir/branch_ids.csv
 
 /usr/bin/time -v $srcDir/run_by_branch.sh $hucNumber $branchId 2>&1 | tee $branchLogFileName 
 
@@ -43,16 +42,16 @@ do
         echo
         err_exists=1
         echo "***** An error has occured  *****"
-        cp $branchLogFileName $outputRunDataDir/branch_errors
+        cp $branchLogFileName $outputDestDir/branch_errors
     fi
 done
 
 # Note: For branches, we do not copy over the log file for codes of 60 and 61.
 
+# Only add the huc and branch number to the csv if the branch was successful at processing
 if [ "$err_exists" = "0" ]; then
-    # Only add the huc and branch number to the csv is the branch was successful at processing
-    # We also don't want to include 60's and 61's
     $srcDir/generate_branch_list_csv.py -o $branch_list_csv_file -u $hucNumber -b $branchId
 fi
 
-exit 0  # we always return a success at this point (so we don't stop the loops / iterator)
+# We always return a success at this point (so we don't stop the loops / iterator)
+exit 0  
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index 6ecb5388d..a0f6511cd 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -18,17 +18,15 @@ if [ $current_branch_id = $branch_zero_id ]; then
     exit 0
 fi
 
-outputHucDataDir=$outputRunDataDir/$hucNumber
-outputBranchDataDir=$outputHucDataDir/branches
-outputCurrentBranchDataDir=$outputBranchDataDir/$current_branch_id
+tempCurrentBranchDataDir=$tempBranchDataDir/$current_branch_id
 
 ## OVERWRITE
-if [ -d "$outputCurrentBranchDataDir" ]; then
-    rm -rf $outputCurrentBranchDataDir
+if [ -d "$tempCurrentBranchDataDir" ]; then
+    rm -rf $tempCurrentBranchDataDir
 fi
 
 ## MAKE OUTPUT BRANCH DIRECTORY
-mkdir -p $outputCurrentBranchDataDir
+mkdir -p $tempCurrentBranchDataDir
 
 ## START MESSAGE ##
 echo -e $startDiv"Processing HUC: $hucNumber - branch_id: $current_branch_id"
@@ -39,48 +37,48 @@ echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"
 date -u
 Tstart
 echo -e "Querying NWM streams ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg
 echo -e "Querying NWM catchments ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
 echo -e "Querying NWM Dissolved Levelpaths headwaters ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
 #echo -e "Querying NWM headwaters ..."
-#ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $outputCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $outputHucDataDir/nwm_headwaters.gpkg
+# ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $tempHucDataDir/nwm_headwaters.gpkg
 Tcount
 
 ## GET RASTERS FROM BRANCH ZERO AND CLIP TO CURRENT BRANCH BUFFER ##
 echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $outputHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $outputHucDataDir/dem_meters.tif $outputBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $outputCurrentBranchDataDir/dem_meters.tif $outputCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
+$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $tempHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $tempHucDataDir/dem_meters.tif $tempBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $tempCurrentBranchDataDir/dem_meters.tif $tempCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
 Tcount
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"
 date -u
 Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $outputCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $tempCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
 Tcount
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
 echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $outputCurrentBranchDataDir/headwaters_$current_branch_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif
 Tcount
 
 ## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
 export hucNumber=$hucNumber
 export current_branch_id=$current_branch_id
-export outputCurrentBranchDataDir=$outputCurrentBranchDataDir
-export outputHucDataDir=$outputHucDataDir
+export tempCurrentBranchDataDir=$tempCurrentBranchDataDir
+export tempHucDataDir=$tempHucDataDir
 export ndv=$ndv
 export xmin=$xmin
 export ymin=$ymin
@@ -91,11 +89,11 @@ export nrows=$nrows
 $srcDir/delineate_hydros_and_produce_HAND.sh "branch"
 
 ## USGS CROSSWALK ##
-if [ -f $outputHucDataDir/usgs_subset_gages.gpkg ]; then
+if [ -f $tempHucDataDir/usgs_subset_gages.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $current_branch_id"
     date -u
     Tstart
-    python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$current_branch_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $current_branch_id
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -outtable $tempCurrentBranchDataDir/usgs_elev_table.csv -b $current_branch_id
     Tcount
 fi
 
@@ -104,9 +102,9 @@ if [ -f $deny_branches_list ]; then
     echo -e $startDiv"Remove files $hucNumber $current_branch_id"
     date -u
     Tstart
-    $srcDir/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branches_list -b $current_branch_id
+    $srcDir/outputs_cleanup.py -d $tempCurrentBranchDataDir -l $deny_branches_list -b $current_branch_id
     Tcount
 fi
 
 echo -e $startDiv"End Branch Processing $hucNumber $current_branch_id ..."
-echo
\ No newline at end of file
+echo
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index ab41e9919..7ec0b9f94 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -5,17 +5,17 @@
 
 ## SOURCE FILE AND FUNCTIONS ##
 # load the various enviro files
-args_file=$outputRunDataDir/runtime_args.env
+args_file=$outputDestDir/runtime_args.env
 
 source $args_file
-source $outputRunDataDir/params.env
+source $outputDestDir/params.env
 source $srcDir/bash_functions.env
 source $srcDir/bash_variables.env
 
-branch_list_csv_file=$outputHucDataDir/branch_ids.csv
-branch_list_lst_file=$outputHucDataDir/branch_ids.lst
+branch_list_csv_file=$tempHucDataDir/branch_ids.csv
+branch_list_lst_file=$tempHucDataDir/branch_ids.lst
 
-branchSummaryLogFile=$outputRunDataDir/logs/branch/"$hucNumber"_summary_branch.log
+branchSummaryLogFile=$outputDestDir/logs/branch/"$hucNumber"_summary_branch.log
 
 ## INITIALIZE TOTAL TIME TIMER ##
 T_total_start
@@ -32,14 +32,14 @@ if [[ $huc2Identifier == "04" ]] ; then
   input_LANDSEA=$input_GL_boundaries
   #echo -e "Using $input_LANDSEA for water body mask (Great Lakes)"
 else
-  input_LANDSEA=$inputDataDir/landsea/water_polygons_us.gpkg
+  input_LANDSEA=$inputsDir/landsea/water_polygons_us.gpkg
 fi
 
 ## GET WBD ##
 echo -e $startDiv"Get WBD $hucNumber"
 date -u
 Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS $outputHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS $tempHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
 Tcount
 
 ## Subset Vector Layers ##
@@ -47,28 +47,28 @@ echo -e $startDiv"Get Vector Layers and Subset $hucNumber"
 date -u
 Tstart
 
-cmd_args=" -a $outputHucDataDir/nwm_lakes_proj_subset.gpkg"
-cmd_args+=" -b $outputHucDataDir/nwm_subset_streams.gpkg"
+cmd_args=" -a $tempHucDataDir/nwm_lakes_proj_subset.gpkg"
+cmd_args+=" -b $tempHucDataDir/nwm_subset_streams.gpkg"
 cmd_args+=" -d $hucNumber"
-cmd_args+=" -e $outputHucDataDir/nwm_headwater_points_subset.gpkg"
-cmd_args+=" -f $outputHucDataDir/wbd_buffered.gpkg"
-cmd_args+=" -g $outputHucDataDir/wbd.gpkg"
+cmd_args+=" -e $tempHucDataDir/nwm_headwater_points_subset.gpkg"
+cmd_args+=" -f $tempHucDataDir/wbd_buffered.gpkg"
+cmd_args+=" -g $tempHucDataDir/wbd.gpkg"
 cmd_args+=" -i $input_DEM"
 cmd_args+=" -j $input_DEM_domain"
 cmd_args+=" -l $input_nwm_lakes"
 cmd_args+=" -m $input_nwm_catchments"
-cmd_args+=" -n $outputHucDataDir/nwm_catchments_proj_subset.gpkg"
+cmd_args+=" -n $tempHucDataDir/nwm_catchments_proj_subset.gpkg"
 cmd_args+=" -r $input_NLD"
 cmd_args+=" -rp $input_levees_preprocessed"
 cmd_args+=" -v $input_LANDSEA"
 cmd_args+=" -w $input_nwm_flows"
-cmd_args+=" -x $outputHucDataDir/LandSea_subset.gpkg"
+cmd_args+=" -x $tempHucDataDir/LandSea_subset.gpkg"
 cmd_args+=" -y $input_nwm_headwaters"
-cmd_args+=" -z $outputHucDataDir/nld_subset_levees.gpkg"
-cmd_args+=" -zp $outputHucDataDir/3d_nld_subset_levees_burned.gpkg"
+cmd_args+=" -z $tempHucDataDir/nld_subset_levees.gpkg"
+cmd_args+=" -zp $tempHucDataDir/3d_nld_subset_levees_burned.gpkg"
 cmd_args+=" -wb $wbd_buffer"
 cmd_args+=" -lpf $input_nld_levee_protected_areas"
-cmd_args+=" -lps $outputHucDataDir/LeveeProtectedAreas_subset.gpkg"
+cmd_args+=" -lps $tempHucDataDir/LeveeProtectedAreas_subset.gpkg"
 
 #echo "$cmd_args"
 python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
@@ -78,14 +78,14 @@ Tcount
 echo -e $startDiv"Clip WBD8"
 date -u
 Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -clipsrc $outputHucDataDir/wbd_buffered.gpkg $outputHucDataDir/wbd8_clp.gpkg $inputDataDir/wbd/WBD_National.gpkg WBDHU8
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -clipsrc $tempHucDataDir/wbd_buffered.gpkg $tempHucDataDir/wbd8_clp.gpkg $inputsDir/wbd/WBD_National.gpkg WBDHU8
 Tcount
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"
 date -u
 Tstart
-$srcDir/derive_level_paths.py -i $outputHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $outputHucDataDir/nwm_headwaters.gpkg -c $outputHucDataDir/nwm_catchments_proj_subset.gpkg -t $outputHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $outputHucDataDir/nwm_lakes_proj_subset.gpkg
+$srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $tempHucDataDir/nwm_headwaters.gpkg -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg
 
 # test if we received a non-zero code back from derive_level_paths.py
 subscript_exit_code=$?
@@ -97,53 +97,55 @@ Tcount
 echo -e $startDiv"Associate level paths with levees"
 date -u
 Tstart
-[ -f $outputHucDataDir/nld_subset_levees.gpkg ] && \
-python3 $srcDir/associate_levelpaths_with_levees.py -nld $outputHucDataDir/nld_subset_levees.gpkg -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -lpa $outputHucDataDir/LeveeProtectedAreas_subset.gpkg -out $outputHucDataDir/levee_levelpaths.csv -w $levee_buffer -b $branch_id_attribute -l $levee_id_attribute
+[ -f $tempHucDataDir/nld_subset_levees.gpkg ] && \
+python3 $srcDir/associate_levelpaths_with_levees.py -nld $tempHucDataDir/nld_subset_levees.gpkg -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -lpa $tempHucDataDir/LeveeProtectedAreas_subset.gpkg -out $tempHucDataDir/levee_levelpaths.csv -w $levee_buffer -b $branch_id_attribute -l $levee_id_attribute
 Tcount
 
 ## STREAM BRANCH POLYGONS
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
 date -u
 Tstart
-$srcDir/buffer_stream_branches.py -a $input_DEM_domain -s $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $outputHucDataDir/branch_polygons.gpkg
+$srcDir/buffer_stream_branches.py -a $input_DEM_domain -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $tempHucDataDir/branch_polygons.gpkg
 Tcount
 
 ## CREATE BRANCHID LIST FILE
 echo -e $startDiv"Create list file of branch ids for $hucNumber"
 date -u
 Tstart
-$srcDir/generate_branch_list.py -d $outputHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -o $branch_list_lst_file
+$srcDir/generate_branch_list.py -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -o $branch_list_lst_file
 Tcount
 
 ## CREATE BRANCH ZERO ##
 echo -e $startDiv"Creating branch zero for $hucNumber"
-outputCurrentBranchDataDir=$outputBranchDataDir/$branch_zero_id
+tempCurrentBranchDataDir=$tempBranchDataDir/$branch_zero_id
 
 ## MAKE OUTPUT BRANCH DIRECTORY
-mkdir -p $outputCurrentBranchDataDir
+mkdir -p $tempCurrentBranchDataDir
 
 ## CLIP RASTERS
 echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"
 # Note: don't need to use gdalwarp -cblend as we are using a buffered wbd
 date -u
 Tstart
-[ ! -f $outputCurrentBranchDataDir/dem_meters.tif ] && \
-gdalwarp -cutline $outputHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $outputHucDataDir/dem_meters.tif
+
+[ ! -f $tempCurrentBranchDataDir/dem_meters.tif ] && \
+gdalwarp -cutline $tempHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $tempHucDataDir/dem_meters.tif
+
 Tcount
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"
 date -u
 Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $outputHucDataDir/dem_meters.tif)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $tempHucDataDir/dem_meters.tif)
 
 ## RASTERIZE NLD MULTILINES ##
 echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $branch_zero_id"
 date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $outputHucDataDir/3d_nld_subset_levees_burned.gpkg ] && \
-gdal_rasterize -l 3d_nld_subset_levees_burned -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $outputHucDataDir/3d_nld_subset_levees_burned.gpkg $outputCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif
+[ -f $tempHucDataDir/3d_nld_subset_levees_burned.gpkg ] && \
+gdal_rasterize -l 3d_nld_subset_levees_burned -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $tempHucDataDir/3d_nld_subset_levees_burned.gpkg $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif
 Tcount
 
 ## BURN LEVEES INTO DEM ##
@@ -151,22 +153,22 @@ echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwr
 date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $outputCurrentBranchDataDir/nld_subset_levees.tif ] && \
-python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $outputHucDataDir/dem_meters.tif -nld $outputCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif -out $outputHucDataDir/dem_meters.tif
+[ -f $tempCurrentBranchDataDir/nld_subset_levees.tif ] && \
+python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $tempHucDataDir/dem_meters.tif -nld $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif -out $tempHucDataDir/dem_meters.tif
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $branch_zero_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_subset_streams.gpkg $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_subset_streams.gpkg $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
 Tcount
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
 echo -e $startDiv"Rasterize NWM Headwaters $hucNumber $branch_zero_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $outputHucDataDir/nwm_headwater_points_subset.gpkg $outputCurrentBranchDataDir/headwaters_$branch_zero_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_headwater_points_subset.gpkg $tempCurrentBranchDataDir/headwaters_$branch_zero_id.tif
 Tcount
 
 ## DEM Reconditioning ##
@@ -175,35 +177,35 @@ Tcount
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/agreedem.py -r $outputCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $outputHucDataDir/dem_meters.tif -w $outputCurrentBranchDataDir -o $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+python3 -m memory_profiler $srcDir/agreedem.py -r $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $tempHucDataDir/dem_meters.tif -w $tempCurrentBranchDataDir -o $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
 Tcount
 
 ## PIT REMOVE BURNED DEM ##
 echo -e $startDiv"Pit remove Burned DEM $hucNumber $branch_zero_id"
 date -u
 Tstart
-rd_depression_filling $outputCurrentBranchDataDir/dem_burned_$branch_zero_id.tif $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
+rd_depression_filling $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
 Tcount
 
 ## D8 FLOW DIR ##
 echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $branch_zero_id"
 date -u
 Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $outputCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $outputCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
 Tcount
 
 ## MAKE A COPY OF THE DEM FOR BRANCH 0
 echo -e $startDiv"Copying DEM to Branch 0"
 date -u
 Tstart
-cp $outputHucDataDir/dem_meters.tif $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
+cp $tempHucDataDir/dem_meters.tif $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
 Tcount
 
 ## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
 export hucNumber=$hucNumber
 export current_branch_id=$current_branch_id
-export outputCurrentBranchDataDir=$outputCurrentBranchDataDir
-export outputHucDataDir=$outputHucDataDir
+export tempCurrentBranchDataDir=$tempCurrentBranchDataDir
+export tempHucDataDir=$tempHucDataDir
 export ndv=$ndv
 export xmin=$xmin
 export ymin=$ymin
@@ -216,26 +218,26 @@ export nrows=$nrows
 $srcDir/delineate_hydros_and_produce_HAND.sh "unit"
 
 ## CREATE USGS GAGES FILE
-if [ -f $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
+if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputDataDir/usgs_gages/usgs_gages.gpkg -nwm $outputHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $outputHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputDataDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id
+    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputsDir/usgs_gages/usgs_gages.gpkg -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $tempHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputsDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id
     Tcount
 fi
 
 ## USGS CROSSWALK ##
-if [ -f $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
+if [ -f $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 $srcDir/usgs_gage_crosswalk.py -gages $outputHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $outputCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $outputCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $outputCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $outputCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -outtable $outputCurrentBranchDataDir/usgs_elev_table.csv -b $branch_zero_id
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -outtable $tempCurrentBranchDataDir/usgs_elev_table.csv -b $branch_zero_id
     Tcount
 fi
 
 ## CLEANUP BRANCH ZERO OUTPUTS ##
 echo -e $startDiv"Cleaning up outputs in branch zero $hucNumber"
-$srcDir/outputs_cleanup.py -d $outputCurrentBranchDataDir -l $deny_branch_zero_list -b $branch_zero_id
+$srcDir/outputs_cleanup.py -d $tempCurrentBranchDataDir -l $deny_branch_zero_list -b $branch_zero_id
 
 
 # -------------------
@@ -270,11 +272,11 @@ fi
 if [ "$has_deny_branch_zero_override" == "1" ]
 then
     echo -e $startDiv"Second cleanup of files for branch zero (none default)"
-    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_branch_zero_list -b 0
+    $srcDir/outputs_cleanup.py -d $tempHucDataDir -l $deny_branch_zero_list -b 0
 
 else 
     echo -e $startDiv"Second cleanup of files for branch zero using the default branch deny list"
-    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_branches_list -b 0
+    $srcDir/outputs_cleanup.py -d $tempHucDataDir -l $deny_branches_list -b 0
 fi
 
 echo "---- HUC $hucNumber - branches have now been processed"
@@ -286,4 +288,3 @@ echo "---- HUC processing for $hucNumber is complete"
 Calc_Duration $huc_start_time
 echo
 
-
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index df4ce8224..33f8e1eb2 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -104,7 +104,7 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
 
 
 def get_fossid_from_huc8(huc8_id,foss_id_attribute='fossid',
-                         hucs=os.path.join(os.environ['inputDataDir'],'wbd','WBD_National.gpkg'),
+                         hucs=os.path.join(os.environ['inputsDir'],'wbd','WBD_National.gpkg'),
                          hucs_layerName=None):
 
     hucs = fiona.open(hucs,'r',layer=hucs_layerName)
diff --git a/tools/inundation_wrapper_custom_flow.py b/tools/inundation_wrapper_custom_flow.py
index 72ce20bf0..0ea1b8e2a 100755
--- a/tools/inundation_wrapper_custom_flow.py
+++ b/tools/inundation_wrapper_custom_flow.py
@@ -14,7 +14,7 @@
 
 TEST_CASES_DIR = r'/data/inundation_review/inundation_custom_flow/'  # Will update.
 INPUTS_DIR = r'/data/inputs'
-OUTPUTS_DIR = os.environ['outputDataDir']
+OUTPUTS_DIR = os.environ['outputsDir']
 
 ENDC = '\033[m'
 TGREEN_BOLD = '\033[32;1m'
diff --git a/tools/inundation_wrapper_nwm_flows.py b/tools/inundation_wrapper_nwm_flows.py
index 8af5462f4..fa10dd164 100755
--- a/tools/inundation_wrapper_nwm_flows.py
+++ b/tools/inundation_wrapper_nwm_flows.py
@@ -14,7 +14,7 @@
 
 INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
 INPUTS_DIR = r'/data/inputs'
-OUTPUTS_DIR = os.environ['outputDataDir']
+OUTPUTS_DIR = os.environ['outputsDir']
 
 ENDC = '\033[m'
 TGREEN_BOLD = '\033[32;1m'
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index da86b828c..4c837e094 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -3,7 +3,7 @@
 # Environmental variables and constants.
 TEST_CASES_DIR = r'/data/test_cases/'
 PREVIOUS_FIM_DIR = r'/data/previous_fim'
-OUTPUTS_DIR = os.environ['outputDataDir']
+OUTPUTS_DIR = os.environ['outputsDir']
 INPUTS_DIR = r'/data/inputs'
 AHPS_BENCHMARK_CATEGORIES = ['usgs', 'nws']
 FR_BENCHMARK_CATEGORIES = ['ble', 'ifc']
diff --git a/unit_tests/README.md b/unit_tests/README.md
index 1bf2435c1..a5eabdec3 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -31,11 +31,17 @@ One way is to use the incoming arg parser. Most python files include the code bl
 
 Start a docker container as you normally would for any development. 
 ```bash 
-docker run --rm -it --name <a docker container name> -v /home/<your name>/projects/<folder path>/:/foss_fim {your docker image name}
+docker run --rm -it --name <a docker container name> \
+	-v /home/<your name>/projects/<folder path>/:/foss_fim \
+	{your docker image name}
 ```
 Example:
 ```bash 
-docker run --rm -it --name mytest -v /home/abcd/projects/dev/innudation-mapping/:/foss_fim -v /abcd_share/foss_fim/outputs/:/outputs -v /abcs_share/foss_fim/:/data fim_4:dev_20220208_8eba0ee
+docker run --rm -it --name mytest \
+	-v /home/abcd/projects/dev/innudation-mapping/:/foss_fim \
+	-v /abcd_share/foss_fim/outputs/:/outputs \
+	-v /abcs_share/foss_fim/:/data \
+	fim_4:dev_20220208_8eba0ee
 ```
 
 For unit tests to work, you need to run the following (if not already in place).
@@ -44,7 +50,8 @@ Notice a modified branch "deny_branch_unittests.lst"  (special for unittests)
 Here are the params and args you need if you need to re-run unit and branch
 
 ```bash
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 05030104" -bd /foss_fim/config/deny_branch_unittests.lst -ud None -j 1 -o
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 05030104" \
+	-bd /foss_fim/config/deny_branch_unittests.lst -ud None -j 1 -o
 ```
 
 **NOTICE: the deny file used for fim_pipeline.sh, has a special one for unittests `deny_branch_unittests.lst`.
@@ -52,7 +59,8 @@ fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 05030104" -bd /
 If you need to run inundation tests, fun the following:
 
 ```bash
-python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_not_remove -jh 1 -jb 1 -m /outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
+python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_not_remove \
+	-jh 1 -jb 1 -m /data/outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
 ```
 ### If you'd like to test the whole unit test suite:
 ```
@@ -96,11 +104,13 @@ If one test case is choosen, it will scan all of the test files, and scan for th
 9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist). 
 ie:
 ```bash
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
+	-c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o
 ```
 but ours would be:
 ```bash 
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 -c /foss_fim/config/params_template.env -j 1 -d no_list -o
+fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
+	-c /foss_fim/config/params_template.env -j 1 -d no_list -o
 ```
 
 ## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
diff --git a/unit_tests/aggregate_branch_lists_params.json b/unit_tests/aggregate_branch_lists_params.json
index fa5f36a29..de17dcd0f 100644
--- a/unit_tests/aggregate_branch_lists_params.json
+++ b/unit_tests/aggregate_branch_lists_params.json
@@ -1,8 +1,8 @@
 {
 	"valid_data":
 	{
-		"output_dir": "/outputs/fim_unit_test_data_do_not_remove/",
+		"output_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
 		"file_name": "branch_ids.csv",
-		"output_file_name": "/outputs/fim_unit_test_data_do_not_remove/test.csv"
+		"output_file_name": "/data/outputs/fim_unit_test_data_do_not_remove/test.csv"
 	}
 }
diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json
index d97ec466f..5c1cf2b12 100644
--- a/unit_tests/check_unit_errors_params.json
+++ b/unit_tests/check_unit_errors_params.json
@@ -1,11 +1,11 @@
 {
 	"valid_data":
 	{
-		"fim_dir": "/outputs/fim_unit_test_data_do_not_remove"
+		"fim_dir": "/data/outputs/fim_unit_test_data_do_not_remove"
 	},
 
 	"invalid_path":
 	{
-		"fim_dir": "/outputs/check_errors_example_unit_tests_not_not_valid"
+		"fim_dir": "/data/outputs/check_errors_example_unit_tests_not_not_valid"
 	}	
-}
\ No newline at end of file
+}
diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py
index d10080e96..9c2cc30b0 100644
--- a/unit_tests/check_unit_errors_test.py
+++ b/unit_tests/check_unit_errors_test.py
@@ -182,7 +182,7 @@ def __remove_temp_unit_errors_folder(self, output_folder):
             Note.. it is possible the temp folder does not exist,
             but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
         Input:
-            output_folder: the root output folder (ie. /outputs/fim_unit_test_data_do_not_remove/)
+            output_folder: the root output folder (ie. /data/outputs/fim_unit_test_data_do_not_remove/)
         Returns:
             nothing
         '''
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 7875dda5c..c9198281b 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -1,26 +1,25 @@
 {
 	"valid_data": {
-		"outputRunDataDir":  "/outputs/fim_unit_test_data_do_not_remove",
-		"subset_nwm_lakes": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
-		"subset_nwm_streams": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"subset_nwm_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
+		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
 		"hucCode": "05030104",
-		"subset_nwm_headwaters": "/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
-		"wbd_buffer_filename": "/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
-		"wbd_filename": "/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
+		"subset_nwm_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
+		"wbd_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
+		"wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
 		"dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
 		"dem_domain" : "/data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg",
 		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
 		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
-		"subset_nwm_catchments": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"subset_nwm_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
 		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
 		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
-		"subset_landsea": "/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
+		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
 		"nwm_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
-		"subset_nld_lines": "/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
+		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
 		"wbd_buffer_distance": 5000,
 		"levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
-		"subset_levee_protected_areas": "/outputs/fim_unit_test_data_do_not_remove/05030104/LeveeProtectedAreas_subset.gpkg"
+		"subset_levee_protected_areas": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LeveeProtectedAreas_subset.gpkg"
 	}
 }
 
diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json
index 48ce1f23d..badee64de 100644
--- a/unit_tests/derive_level_paths_params.json
+++ b/unit_tests/derive_level_paths_params.json
@@ -1,15 +1,15 @@
 {
 	"valid_data":
 	{
-		"outputRunDataDir" : "/outputs/fim_unit_test_data_do_not_remove/",
-		"in_stream_network": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
-		"out_stream_network": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"outputDestDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
+		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
 		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
-		"catchments": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
 		"verbose": true
 	}	
diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py
index 8f395e561..2cd05f680 100644
--- a/unit_tests/derive_level_paths_test.py
+++ b/unit_tests/derive_level_paths_test.py
@@ -80,7 +80,7 @@ def test_Derive_level_paths_invalid_input_stream_network(self):
         params["in_stream_network"] = "some bad path"
         
         with pytest.raises(Exception) as e_info:
-            actual = src.Derive_level_paths(in_stream_network = ut_helpers.json_concat(params, "outputRunDataDir", "in_stream_network"),
+            actual = src.Derive_level_paths(in_stream_network = ut_helpers.json_concat(params, "outputDestDir", "in_stream_network"),
                                             out_stream_network = params["out_stream_network"],
                                             branch_id_attribute = params["branch_id_attribute"],
                                             out_stream_network_dissolved = params["out_stream_network_dissolved"],
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
index 5b6370373..2638a2ceb 100644
--- a/unit_tests/filter_catchments_and_add_attributes_params.json
+++ b/unit_tests/filter_catchments_and_add_attributes_params.json
@@ -1,12 +1,12 @@
 {
 	"valid_data":
 	{
-		"outputRunDataDir":  "/outputs/fim_unit_test_data_do_not_remove",
-		"input_catchments_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
-		"input_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
-		"output_catchments_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
-		"output_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
-	    "wbd_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
+		"outputDestDir":  "/outputs/fim_unit_test_data_do_not_remove",
+		"input_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
+		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
+		"output_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
+		"output_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
+	    "wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
 		"huc_code": "02020005"
 	}
 }
diff --git a/unit_tests/generate_branch_list_csv_params.json b/unit_tests/generate_branch_list_csv_params.json
index 004c5719b..123171471 100644
--- a/unit_tests/generate_branch_list_csv_params.json
+++ b/unit_tests/generate_branch_list_csv_params.json
@@ -3,18 +3,18 @@
 	{
 		"huc_id": "05030104",
 		"branch_id": "0",
-		"output_branch_csv": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
+		"output_branch_csv": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
 	},
 	"valid_data_add_branch":
 	{
 		"huc_id": "05030104",
 		"branch_id": "1946000003",
-		"output_branch_csv": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
+		"output_branch_csv": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
 	},	
 	"invalid_bad_file_extension":
 	{
 		"huc_id": "05030104",
 		"branch_id": "1946000003",
-		"output_branch_csv": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids2"
+		"output_branch_csv": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids2"
 	}	
 }
diff --git a/unit_tests/generate_branch_list_params.json b/unit_tests/generate_branch_list_params.json
index b8c9bb4cc..a7c862295 100644
--- a/unit_tests/generate_branch_list_params.json
+++ b/unit_tests/generate_branch_list_params.json
@@ -1,8 +1,8 @@
 {
 	"valid_data":
 	{
-		"stream_network_dissolved": "/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
+		"stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
 		"branch_id_attribute": "levpa_id",
-		"output_branch_list_file": "/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.lst"
+		"output_branch_list_file": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.lst"
 	}
 }
diff --git a/unit_tests/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json
index f63745ad9..a545e6753 100644
--- a/unit_tests/outputs_cleanup_params.json
+++ b/unit_tests/outputs_cleanup_params.json
@@ -1,7 +1,7 @@
 {
 	"valid_specific_branch_data":
 	{
-		"src_dir": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
+		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
 		"deny_list": "/foss_fim/config/deny_branches.lst",
 		"branch_id": "3246000009",
 		"verbose": true
@@ -9,7 +9,7 @@
 
 	"valid_directory_data":
 	{
-		"src_dir": "/outputs/fim_unit_test_data_do_not_remove",
+		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
 		"deny_list": "/foss_fim/config/deny_branches.lst",
 		"branch_id": "0",
 		"verbose": true
@@ -17,7 +17,7 @@
 
 	"skip_clean":
 	{
-		"src_dir": "/outputs/fim_unit_test_data_do_not_remove",
+		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
 		"deny_list": "None",
 		"branch_id": "",
 		"verbose": true
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
index dc0ef15bd..24949dcb6 100644
--- a/unit_tests/rating_curve_comparison_params.json
+++ b/unit_tests/rating_curve_comparison_params.json
@@ -3,12 +3,12 @@
 	{
 		"02020005":
 		[
-			"/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_elev_table.csv",
-			"/outputs/fim_unit_test_data_do_not_remove/02020005/branches",
+			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_elev_table.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches",
 			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
-			"/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
-			"/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
+			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
 			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
 			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
 			"02020005",
diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json
index 9dce603d1..cb2b9be90 100644
--- a/unit_tests/shared_functions_params.json
+++ b/unit_tests/shared_functions_params.json
@@ -1,7 +1,7 @@
 {
 	"append_append_id_to_file_name_single_identifier_valid":
 	{
-		"outputRunDataDir": "/output/fim_unit_test_data_do_not_remove/",
+		"outputDestDir": "/output/fim_unit_test_data_do_not_remove/",
 		"file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
 		"identifier": "8888",
 		"expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_8888.tif"
@@ -9,7 +9,7 @@
 
 	"append_append_id_to_file_name_identifier_list_valid":
 	{
-		"outputRunDataDir": "/output/fim_unit_test_data_do_not_remove/",
+		"outputDestDir": "/output/fim_unit_test_data_do_not_remove/",
 		"file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
 		"identifier": ["7777", "8888"],
 		"expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_7777_8888.tif"
diff --git a/unit_tests/shared_functions_test.py b/unit_tests/shared_functions_test.py
index 3f9de9829..a2a3b0b1a 100644
--- a/unit_tests/shared_functions_test.py
+++ b/unit_tests/shared_functions_test.py
@@ -29,12 +29,12 @@ def test_append_id_to_file_name_single_identifier_success(self):
 
         params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy() 
 
-        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputRunDataDir" , "file_name"),
+        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputDestDir" , "file_name"),
                                                    identifier = params["identifier"])
 
         err_msg = "actual output does not match expected output"
 
-        expected_output = ut_helpers.json_concat(params, "outputRunDataDir" , "expected_output")
+        expected_output = ut_helpers.json_concat(params, "outputDestDir" , "expected_output")
         
         assert  expected_output == actual_output, err_msg
 
@@ -47,12 +47,12 @@ def test_append_id_to_file_name_indentifer_list_success(self):
         
         params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()        
 
-        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputRunDataDir" , "file_name"),
+        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputDestDir" , "file_name"),
                                                    identifier = params["identifier"])
 
         err_msg = "actual output does not match expected output"
         
-        expected_output = ut_helpers.json_concat(params, "outputRunDataDir" , "expected_output")
+        expected_output = ut_helpers.json_concat(params, "outputDestDir" , "expected_output")
         
         assert expected_output == actual_output, err_msg
 
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
index 06f4118c2..a3a8e07de 100644
--- a/unit_tests/split_flows_params.json
+++ b/unit_tests/split_flows_params.json
@@ -1,16 +1,16 @@
 {
 	"valid_data":
 	{
-		"outputRunDataDir" : "/outputs/fim_unit_test_data_do_not_remove/",
+		"outputDestDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
 		"max_length": 1500,
 		"slope_min": 0.001,
 		"lakes_buffer_input": 20,
-		"flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
-		"dem_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-	    "split_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
-		"split_points_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
-		"wbd8_clp_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
-		"lakes_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_lakes_proj_subset.gpkg",
-		"nwm_streams_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_subset_streams_levelPaths.gpkg"
+		"flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
+		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+	    "split_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
+		"split_points_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
+		"wbd8_clp_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
+		"lakes_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_lakes_proj_subset.gpkg",
+		"nwm_streams_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_subset_streams_levelPaths.gpkg"
 	}
 }
diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py
index 66788a928..6b94fd2c7 100644
--- a/unit_tests/split_flows_test.py
+++ b/unit_tests/split_flows_test.py
@@ -27,8 +27,8 @@ def setUpClass(self):
     def test_split_flows_success(self):
 
         '''
-        The /outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
-        /outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
+        The /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
+        /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
         If the test is successful, these files will be created.
         '''
 
diff --git a/unit_tests/tools/inundate_gms_params.json b/unit_tests/tools/inundate_gms_params.json
index ec1d3d55f..13445b514 100644
--- a/unit_tests/tools/inundate_gms_params.json
+++ b/unit_tests/tools/inundate_gms_params.json
@@ -1,15 +1,15 @@
 {
 	"valid_data_inudation_raster_single_huc":
 	{
-		"hydrofabric_dir": "/outputs/fim_unit_test_data_do_not_remove/",
+		"hydrofabric_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
 		"forecast": "/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv",
 		"num_workers": 1,
 		"hucs": "02020005, 05030104",
-	    "inundation_raster": "/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
+	    "inundation_raster": "/data/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
 		"inundation_polygon": null,
 		"depths_raster": null,
 		"verbose": true,
-		"log_file": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_logfile.txt",
-		"output_fileNames": "/outputs/fim_unit_test_data_do_not_remove/logs/inundation_file_list.csv"
+		"log_file": "/data/outputs/fim_unit_test_data_do_not_remove/logs/inundation_logfile.txt",
+		"output_fileNames": "/data/outputs/fim_unit_test_data_do_not_remove/logs/inundation_file_list.csv"
 	}
 }
diff --git a/unit_tests/tools/inundation_params.json b/unit_tests/tools/inundation_params.json
index cc4685bcc..6e2b1d800 100644
--- a/unit_tests/tools/inundation_params.json
+++ b/unit_tests/tools/inundation_params.json
@@ -1,10 +1,10 @@
 {
 	"valid_data_inundate_branch":
 	{
-		"rem": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
-		"catchments": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
-		"catchment_poly": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
-		"hydro_table": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/hydroTable_3246000001.csv",
+		"rem": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
+		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
+		"catchment_poly": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
+		"hydro_table": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/hydroTable_3246000001.csv",
 	    "forecast": "/data/test_cases/usgs_test_cases/validation_data_usgs/02020005/ptvn6/action/ahps_ptvn6_huc_02020005_flows_action.csv",
 		"mask_type": null,
 		"hucs": null,
diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json
index 32f51a347..885c0fdd6 100644
--- a/unit_tests/usgs_gage_crosswalk_params.json
+++ b/unit_tests/usgs_gage_crosswalk_params.json
@@ -1,12 +1,12 @@
 {
 	"valid_data":
 	{
-		"usgs_gages_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_subset_gages.gpkg",
-		"input_flows_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
-		"input_catchment_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
-		"dem_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_meters_3246000005.tif",
-		"dem_adj_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-		"output_table_filename": "/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/usgs_elev_table.csv",
+		"usgs_gages_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_subset_gages.gpkg",
+		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
+		"input_catchment_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
+		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_meters_3246000005.tif",
+		"dem_adj_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+		"output_table_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/usgs_elev_table.csv",
 		"branch_id": "3246000005"
 	}
-}
\ No newline at end of file
+}

From ea8f5d9ca8b158616eea9a804aa2f37ee6a0992e Mon Sep 17 00:00:00 2001
From: LauraKeys-NOAA <70279180+LauraKeys-NOAA@users.noreply.github.com>
Date: Fri, 31 Mar 2023 14:02:37 -0500
Subject: [PATCH 230/359] v4.3.5.0 Shapely upgrade (#857)

---
 Dockerfile                              |   28 +-
 Pipfile                                 |   20 +-
 Pipfile.lock                            | 1707 ++++++++++++++++-------
 docs/CHANGELOG.md                       |   36 +
 src/add_crosswalk.py                    |    2 +-
 src/adjust_headwater_streams.py         |    6 +-
 src/aggregate_vector_inputs.py          |    8 +-
 src/associate_levelpaths_with_levees.py |    9 +-
 src/build_stream_traversal.py           |    2 +-
 src/clip_vectors_to_wbd.py              |    6 +-
 src/derive_headwaters.py                |    2 +-
 src/split_flows.py                      |   19 +-
 src/stream_branches.py                  |    4 +-
 tools/fimr_to_benchmark.py              |    2 +-
 tools/inundation.py                     |    3 +-
 tools/overlapping_inundation.py         |    8 +-
 tools/tools_shared_functions.py         |    6 +-
 17 files changed, 1269 insertions(+), 599 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4dc93f128..7c4441c90 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -92,15 +92,25 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 ## install python 3 modules ##
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
-#RUN pip3 install pipenv==2022.4.8 && pipenv install --system --deploy --ignore-pipfile (too slow to
-#     leave out shapely,pygeos at this time. Likely better after upgrading)
-
-# TEMP FIX as neither shapely or Shapely is staying in the pip list. If we manually add
-# it with pip (not pipenv), it works. Notice case for Shapely versus shapely. 
-# This temp fix works for now until we can reconsile the shapely package,
-# pygeos, geopanda's and possibly others (coming soon)
-RUN pip install shapely==1.7.0
+RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
+
+# ----------------------------------
+# Mar 2023
+# There are some nuances in the whitebox python downloads in that the first time it loads
+# it goes to the internet and downloads the latest/greatest WBT (whiteboxtools) engine which is
+# required for the whitebox python library to work. We don't want to have FIM attempting a download
+# each time a container is opened and the whitebox engine is called.
+# Instead we will setup the WBT engine at time of docker build (same as Taudem and AWS).
+# Whitebox code detects that the engine it there and makes no attempt to update it.
+# We download and unzip it to the same file folder that pip deployed the whitebox library.
+# Whitebox also attempts to always download a folder called testdata regardless of use. 
+# We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
+RUN wbox_path=/usr/local/lib/python3.8/dist-packages/whitebox/ && \
+wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
+unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
+cp $wbox_path/WBT/whitebox_tools $wbox_path && \
+mkdir $wbox_path/testdata
+# ----------------------------------
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
 ADD ./src/entrypoint.sh /
diff --git a/Pipfile b/Pipfile
index a84391528..3ced6075e 100755
--- a/Pipfile
+++ b/Pipfile
@@ -8,15 +8,14 @@ ipython = "*"
 
 [packages]
 fiona = "==1.8.17"
-geopandas = "==0.8.1"
-numba = "==0.50.1"
-numpy = "==1.22.4"
-pandas = "==1.0.5"
-pygeos = "==0.7.1"
+geopandas = "==0.12.2"
+numba = "==0.56.4"
+numpy = "==1.23.5"
+pandas = "==1.1"
 rasterio = "==1.1.5"
-rasterstats = "==0.15.0"
+rasterstats = "==0.18.0"
 richdem = "==0.3.4"
-tqdm = "==4.48.0"
+tqdm = "==4.65.0"
 seaborn = "==0.11.0"
 python-dotenv = "*"
 natsort = "*"
@@ -32,8 +31,11 @@ jupyter = "*"
 jupyterlab = "*"
 ipympl = "*"
 pytest = "*"
-whitebox = "*"
-shapely = "==1.7.0"
+whitebox = "==2.3.1"
+shapely = "==2.0.1"
+pyarrow = "==11.0.0"
+rtree = "==1.0.1"
+py7zr = "==0.20.4"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 03826cafa..ea78e73b3 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "4ef68f0d8f3ea64518ebd604722e5283603cf6ffe3401320f3db3be365224ac0"
+            "sha256": "375fa6bf31e8485eaae40981f64ce07fddc91e3b86de727846fb6a3ce9e63654"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "markers": "python_version >= '3.7' and python_version < '4'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -108,11 +108,11 @@
         },
         "babel": {
             "hashes": [
-                "sha256:1ad3eca1c885218f6dce2ab67291178944f810a10a9b5f3cb8382a5a232b64fe",
-                "sha256:5ef4b3226b0180dedded4229651c8b0e1a3a6a2837d45a073272f313e4cf97f6"
+                "sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610",
+                "sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.11.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.12.1"
         },
         "backcall": {
             "hashes": [
@@ -123,11 +123,11 @@
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:0e79446b10b3ecb499c1556f7e228a53e64a2bfcebd455f370d8927cb5b59e39",
-                "sha256:bc4bdda6717de5a2987436fb8d72f45dc90dd856bdfd512a1314ce90349a0106"
+                "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591",
+                "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==4.11.2"
+            "version": "==4.12.0"
         },
         "bleach": {
             "hashes": [
@@ -139,19 +139,107 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027",
-                "sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729"
+                "sha256:043f8981d10c4e7c48736df4381dac557b46c5b369b0a450d8f3d7f5fdd24db5",
+                "sha256:b00f416832bc59863b96175045d2ebe067d9222289bce677c48fd72c006eaaad"
             ],
             "index": "pypi",
-            "version": "==1.26.76"
+            "version": "==1.26.102"
         },
         "botocore": {
             "hashes": [
-                "sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7",
-                "sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7"
+                "sha256:4bae8f502507da18ff37c61cb18745cfb11d87a61dd0ea27e346adadff92aa3f",
+                "sha256:58b11c630d2044ea732ba4c403d29fab51e954465f9b3f7099cbf5ac0ce7ab47"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.29.76"
+            "version": "==1.29.102"
+        },
+        "brotli": {
+            "hashes": [
+                "sha256:02177603aaca36e1fd21b091cb742bb3b305a569e2402f1ca38af471777fb019",
+                "sha256:11d3283d89af7033236fa4e73ec2cbe743d4f6a81d41bd234f24bf63dde979df",
+                "sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d",
+                "sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8",
+                "sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b",
+                "sha256:19598ecddd8a212aedb1ffa15763dd52a388518c4550e615aed88dc3753c0f0c",
+                "sha256:1c48472a6ba3b113452355b9af0a60da5c2ae60477f8feda8346f8fd48e3e87c",
+                "sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70",
+                "sha256:269a5743a393c65db46a7bb982644c67ecba4b8d91b392403ad8a861ba6f495f",
+                "sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181",
+                "sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130",
+                "sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19",
+                "sha256:3148362937217b7072cf80a2dcc007f09bb5ecb96dae4617316638194113d5be",
+                "sha256:330e3f10cd01da535c70d09c4283ba2df5fb78e915bea0a28becad6e2ac010be",
+                "sha256:336b40348269f9b91268378de5ff44dc6fbaa2268194f85177b53463d313842a",
+                "sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa",
+                "sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429",
+                "sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126",
+                "sha256:3b8b09a16a1950b9ef495a0f8b9d0a87599a9d1f179e2d4ac014b2ec831f87e7",
+                "sha256:3c1306004d49b84bd0c4f90457c6f57ad109f5cc6067a9664e12b7b79a9948ad",
+                "sha256:3ffaadcaeafe9d30a7e4e1e97ad727e4f5610b9fa2f7551998471e3736738679",
+                "sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4",
+                "sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0",
+                "sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b",
+                "sha256:495ba7e49c2db22b046a53b469bbecea802efce200dffb69b93dd47397edc9b6",
+                "sha256:4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438",
+                "sha256:503fa6af7da9f4b5780bb7e4cbe0c639b010f12be85d02c99452825dd0feef3f",
+                "sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389",
+                "sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6",
+                "sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26",
+                "sha256:5bf37a08493232fbb0f8229f1824b366c2fc1d02d64e7e918af40acd15f3e337",
+                "sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7",
+                "sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14",
+                "sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2",
+                "sha256:68715970f16b6e92c574c30747c95cf8cf62804569647386ff032195dc89a430",
+                "sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296",
+                "sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12",
+                "sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f",
+                "sha256:73fd30d4ce0ea48010564ccee1a26bfe39323fde05cb34b5863455629db61dc7",
+                "sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d",
+                "sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a",
+                "sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452",
+                "sha256:7ee83d3e3a024a9618e5be64648d6d11c37047ac48adff25f12fa4226cf23d1c",
+                "sha256:854c33dad5ba0fbd6ab69185fec8dab89e13cda6b7d191ba111987df74f38761",
+                "sha256:85f7912459c67eaab2fb854ed2bc1cc25772b300545fe7ed2dc03954da638649",
+                "sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b",
+                "sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea",
+                "sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c",
+                "sha256:8ed6a5b3d23ecc00ea02e1ed8e0ff9a08f4fc87a1f58a2530e71c0f48adf882f",
+                "sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a",
+                "sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031",
+                "sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267",
+                "sha256:97f715cf371b16ac88b8c19da00029804e20e25f30d80203417255d239f228b5",
+                "sha256:9bf919756d25e4114ace16a8ce91eb340eb57a08e2c6950c3cebcbe3dff2a5e7",
+                "sha256:9d12cf2851759b8de8ca5fde36a59c08210a97ffca0eb94c532ce7b17c6a3d1d",
+                "sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c",
+                "sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43",
+                "sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa",
+                "sha256:b1375b5d17d6145c798661b67e4ae9d5496920d9265e2f00f1c2c0b5ae91fbde",
+                "sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17",
+                "sha256:b3523f51818e8f16599613edddb1ff924eeb4b53ab7e7197f85cbc321cdca32f",
+                "sha256:b43775532a5904bc938f9c15b77c613cb6ad6fb30990f3b0afaea82797a402d8",
+                "sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb",
+                "sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb",
+                "sha256:ba72d37e2a924717990f4d7482e8ac88e2ef43fb95491eb6e0d124d77d2a150d",
+                "sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b",
+                "sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4",
+                "sha256:c8e521a0ce7cf690ca84b8cc2272ddaf9d8a50294fd086da67e517439614c755",
+                "sha256:cab1b5964b39607a66adbba01f1c12df2e55ac36c81ec6ed44f2fca44178bf1a",
+                "sha256:cb02ed34557afde2d2da68194d12f5719ee96cfb2eacc886352cb73e3808fc5d",
+                "sha256:cc0283a406774f465fb45ec7efb66857c09ffefbe49ec20b7882eff6d3c86d3a",
+                "sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3",
+                "sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7",
+                "sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1",
+                "sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb",
+                "sha256:e1abbeef02962596548382e393f56e4c94acd286bd0c5afba756cffc33670e8a",
+                "sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91",
+                "sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b",
+                "sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1",
+                "sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806",
+                "sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3",
+                "sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1"
+            ],
+            "markers": "platform_python_implementation == 'CPython'",
+            "version": "==1.0.9"
         },
         "certifi": {
             "hashes": [
@@ -259,97 +347,84 @@
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b",
-                "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42",
-                "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d",
-                "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b",
-                "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a",
-                "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59",
-                "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154",
-                "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1",
-                "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c",
-                "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a",
-                "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d",
-                "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6",
-                "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b",
-                "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b",
-                "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783",
-                "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5",
-                "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918",
-                "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555",
-                "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639",
-                "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786",
-                "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e",
-                "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed",
-                "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820",
-                "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8",
-                "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3",
-                "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541",
-                "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14",
-                "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be",
-                "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e",
-                "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76",
-                "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b",
-                "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c",
-                "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b",
-                "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3",
-                "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc",
-                "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6",
-                "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59",
-                "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4",
-                "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d",
-                "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d",
-                "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3",
-                "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a",
-                "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea",
-                "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6",
-                "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e",
-                "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603",
-                "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24",
-                "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a",
-                "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58",
-                "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678",
-                "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a",
-                "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c",
-                "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6",
-                "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18",
-                "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174",
-                "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317",
-                "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f",
-                "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc",
-                "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837",
-                "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41",
-                "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c",
-                "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579",
-                "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753",
-                "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8",
-                "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291",
-                "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087",
-                "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866",
-                "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3",
-                "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d",
-                "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1",
-                "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca",
-                "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e",
-                "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db",
-                "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72",
-                "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d",
-                "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc",
-                "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539",
-                "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d",
-                "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af",
-                "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b",
-                "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602",
-                "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f",
-                "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478",
-                "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c",
-                "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e",
-                "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479",
-                "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7",
-                "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"
+                "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6",
+                "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1",
+                "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e",
+                "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373",
+                "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62",
+                "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230",
+                "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be",
+                "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c",
+                "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0",
+                "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448",
+                "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f",
+                "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649",
+                "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d",
+                "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0",
+                "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706",
+                "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a",
+                "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59",
+                "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23",
+                "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5",
+                "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb",
+                "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e",
+                "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e",
+                "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c",
+                "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28",
+                "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d",
+                "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41",
+                "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974",
+                "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce",
+                "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f",
+                "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1",
+                "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d",
+                "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8",
+                "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017",
+                "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31",
+                "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7",
+                "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8",
+                "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e",
+                "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14",
+                "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd",
+                "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d",
+                "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795",
+                "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b",
+                "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b",
+                "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b",
+                "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203",
+                "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f",
+                "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19",
+                "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1",
+                "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a",
+                "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac",
+                "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9",
+                "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0",
+                "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137",
+                "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f",
+                "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6",
+                "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5",
+                "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909",
+                "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f",
+                "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0",
+                "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324",
+                "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755",
+                "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb",
+                "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854",
+                "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c",
+                "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60",
+                "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84",
+                "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0",
+                "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b",
+                "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1",
+                "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531",
+                "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1",
+                "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11",
+                "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326",
+                "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df",
+                "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.0.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.1.0"
         },
         "click": {
             "hashes": [
@@ -371,16 +446,16 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "comm": {
             "hashes": [
-                "sha256:3e2f5826578e683999b93716285b3b1f344f157bf75fa9ce0a797564e742f062",
-                "sha256:9f3abf3515112fa7c55a42a6a5ab358735c9dccc8b5910a9d8e3ef5998130666"
+                "sha256:16613c6211e20223f215fc6d3b266a247b6e2641bf4e0a3ad34cb1aff2aa3f37",
+                "sha256:a61efa9daffcfbe66fd643ba966f846a624e4e6d6767eda9cf6e993aadaab93e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.1.2"
+            "version": "==0.1.3"
         },
         "contourpy": {
             "hashes": [
@@ -492,11 +567,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e",
-                "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"
+                "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e",
+                "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.1.0"
+            "version": "==1.1.1"
         },
         "executing": {
             "hashes": [
@@ -507,10 +582,10 @@
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:01e366f25d9047816fe3d288cbfc3e10541daf0af2044763f3d0ade42476da18",
-                "sha256:21f918e8d9a1a4ba9c22e09574ba72267a6762d47822db9add95f6454e51cc1c"
+                "sha256:04fbecc94300436f628517b05741b7ea009506ce8f946d40996567c669318490",
+                "sha256:4a30d6315a68c253cfa8f963b9697246315aa3db89f98b97235e345dedfb0b8e"
             ],
-            "version": "==2.16.2"
+            "version": "==2.16.3"
         },
         "fiona": {
             "hashes": [
@@ -530,11 +605,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:2bb244009f9bf3fa100fc3ead6aeb99febe5985fa20afbfbaa2f8946c2fbdaf1",
-                "sha256:820466f43c8be8c3009aef8b87e785014133508f0de64ec469e4efb643ae54fb"
+                "sha256:64c0c05c337f826183637570ac5ab49ee220eec66cf50248e8df527edfa95aeb",
+                "sha256:9234b9f57b74e31b192c3fc32ef1a40750a8fbc1cd9837a7b7bfc4ca4a5c51d7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==4.38.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.39.3"
         },
         "fqdn": {
             "hashes": [
@@ -545,11 +620,11 @@
         },
         "geopandas": {
             "hashes": [
-                "sha256:e28a729e44ac53c1891b54b1aca60e3bc0bb9e88ad0f2be8e301a03b9510f6e2",
-                "sha256:ef90a7f5ce9337f412c1dab9e014b4076b639fb7fc0edcf8b57c252c91a096c4"
+                "sha256:0a470e4bf6f5367e6fd83ab6b40405e0b805c8174665bbcb7c4077ed90202912",
+                "sha256:0acdacddefa176525e4da6d9aeeece225da26055c4becdc6e97cf40fa97c27f4"
             ],
             "index": "pypi",
-            "version": "==0.8.1"
+            "version": "==0.12.2"
         },
         "h5py": {
             "hashes": [
@@ -577,11 +652,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad",
-                "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"
+                "sha256:43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20",
+                "sha256:ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09"
             ],
-            "markers": "python_version < '3.10'",
-            "version": "==6.0.0"
+            "markers": "python_version < '3.9'",
+            "version": "==6.1.0"
         },
         "importlib-resources": {
             "hashes": [
@@ -591,6 +666,81 @@
             "markers": "python_version < '3.9'",
             "version": "==5.12.0"
         },
+        "inflate64": {
+            "hashes": [
+                "sha256:08c3b03514d4b849901762a32a45eeba7fd5d784fec698eca6975f41cca33672",
+                "sha256:094ef56a87c7b7398d93af7bfe7f24f830f24b6e55b77426f6516cef43e05460",
+                "sha256:09dd0f8d6dee0da467c264dbd9bca8b33f9c915860fc3385f2a633640a65bd10",
+                "sha256:0b0c8aa2fcdb1052d3bc6c5b5b1191b9c708d30e47af98ba0a8117ae1f6c9efc",
+                "sha256:130dfdca4bd38e588ea4f878bf62635e36f83ddf7f2842d1055d1c16a11890cf",
+                "sha256:1749da3a02b53035cde1cf95f885e78e0c2c49b201e97d368b3ba97e0f3d42c3",
+                "sha256:17aaac096f40bd80dd72481831607a0846271d401ba3cd863386b8c244c7ebc1",
+                "sha256:1d861fed6b2098d1862b64db9df650b9bd41fc41caa9fcaeee399079342aa4a8",
+                "sha256:26e8319fd032c520203e2c001f1693c1c03774d85915900427e884011718f41d",
+                "sha256:29946840e6970d68e7739207ca21140c59ffebe7e02d28c7e86348166ce32418",
+                "sha256:35e24ffd8d6225fbfe26c524b45ace1bb8956811bd79e9f3d523a721d51b0d4e",
+                "sha256:3a17e1dd1a5a872edfc02bc4a048868ada4865a3f4ee3ad5d224b192f2e53df7",
+                "sha256:3bacbe9d4b7c185011b59268223a010ed777a28ed8cf40efc74fab1b7262e904",
+                "sha256:3c270d373ca3717dbeb9b171eea53cbf2c9d7471b9b5de1e57f165e60cf58037",
+                "sha256:3c913b679f023f5907a54bfa9a6e438407ed4e40eee23ed19b4118128bdd091c",
+                "sha256:3cf41f82dd4e90e8684c7be4583d7232bd800a561f3ed0241c84e39148861887",
+                "sha256:41504988023042452d2d84e4110c9ef4ff8ebd33cb90ba83e44b92c9a6753c43",
+                "sha256:42a6ef375b3e7059bd52993a0938f2bf97725cb5dc380f0c4dbaa9fc3780e025",
+                "sha256:473e0081c268ffa4b18683586b55170eb96d8b4fc684dd3ed9599c17c512d2e4",
+                "sha256:48fd2527a462374dc19be06301d6aa30a03190532f2f8bddfbc39b7158561750",
+                "sha256:4d807cfa9ddad940401ef04502eb367a77f569850f59c2e71670347d558a3830",
+                "sha256:4e7b0a598adaa11366ffbbb7b3d3110db29edd4b732d9336570891363b22b002",
+                "sha256:525bc309d8533ef9917e006284996ee7a9a71ac6dd19fb57c0f741ad0c805d4f",
+                "sha256:553cd992f02af574d2116c74ca48d7cf10894c6b9ba8159f488f3bfac3c201ae",
+                "sha256:5c5b2eb7e89d550d287774dea7d429ee24ce44ca34499a6cef113a14f108e700",
+                "sha256:5efd55c21b794601fd44b99b8e2f17498744f573116ce27a745bc5e08f0457e1",
+                "sha256:6059eaba5044739ad6424588e845bd856f89a1a18f1addc31b97c49f02f68728",
+                "sha256:664929528047b6b472852a4c0d12b4b9cf6e663059ba64ebd10f08aa56365755",
+                "sha256:67e37d96ea2ee8257b12cde83a09e4f0276950268a7a2f777aee7de60db5ec72",
+                "sha256:67efdfd21d7b99f30a43560b22264c1e580ff08ae9831e78c99445575962dbc7",
+                "sha256:6ff89f94823b2466bae45759fc324bd25bd20c490607a7d8407237cf64ccafa9",
+                "sha256:74ceb9d172ce06572632bc8070d54b963455421e216013575383f991e722bb7d",
+                "sha256:7b7966193f1bf23e050af72b4c4720dffa5f33471de7afea37ba0d0f0195adef",
+                "sha256:7ba954600441eafe8f6f54eadffeac4d1ab2416d5d1a6b0ab403e50284ba457b",
+                "sha256:7c142fbbbfbe0877fe821ff8bc4cc10f96d344b7400721579b3d17deeae28f59",
+                "sha256:7f8346e644de449a4a90dcb22971dea456398b6cc788102013675b11256ae47e",
+                "sha256:80a125dd5cb7b7985c05a78b0bfd7751249d0d84fc330901dbd9faa693e1f53f",
+                "sha256:82393e46b8ba2f8613d030f38c7c492b0896ff8803f7ff870677f25d3e5e7113",
+                "sha256:84287d1d09fd879353d3ccadd43f3d8adea75e830476ddfd46d8849d36d25afe",
+                "sha256:853f3442eceda8035072686533694ab833c4293d10c9d0685147200f0e964356",
+                "sha256:906a4b57df32f903e847766ca685e44ed3e7ee3a960fa94264d5e68b836d446d",
+                "sha256:90f95b92d0f672d11151cb964964d1723e2e3ce3a19d32d24aece1acdec1e287",
+                "sha256:91233b5300bbb7562804c3d07617e9ce2983e8434218991db98ef175491e417f",
+                "sha256:9297115bf144c585e9d6a746e851c64c81d8f1ce8b62da4885babe66c36f7d29",
+                "sha256:9f6737a575c6e7e818963d95a998be4c91484374961734cee97265f3c4c3b979",
+                "sha256:a075b174bace5174828906c7c87019a2af3cc5707025f01ee0395fb4b88fd98e",
+                "sha256:a1b481343f12641b1ae7a19135a70c44ecf020dccb670e02522c2b02db920851",
+                "sha256:a2f4aaa02f9a5ada944960428b6528a0a9d773925efc73485882f34bf42654be",
+                "sha256:a6bec3d2f30f6f2656e1c5a4147181e401c8d7026cd598d86ad5647c616fc618",
+                "sha256:aa7476129e7f81e67a9253470c3085a9fd75ec77e6fae3de61f7795138ce725e",
+                "sha256:ab8f9e14ba6495f440101751ba8aa371e4a52941b5e343c6f3e8c61021e2df5e",
+                "sha256:ac60868745f7bfbcd615329fbdc35997fa36043ce358a1c64d229ef448ebecf0",
+                "sha256:ad4cae5097bdff7e0bb1ab676d86ad08716597baa3b616e5b710a724f5d5cbc4",
+                "sha256:ad84ac611eae17a961124c5fbe754b6982291a3945ab2b9c334a08e2e56f9ccc",
+                "sha256:b52dd8fefd2ba179e5dfa18d6eca7e2fc822584616271c039d5ef1f9ca90c71c",
+                "sha256:b7aa123c740f2f9798f72873e50d7c6d43664d12cad7a1405296079987bdb04a",
+                "sha256:c1faf43890dbfff31195f5d59e37e49824f5ff4be77d67f7144a6b953bbde51c",
+                "sha256:c28cb635ccb9aae399fbc8e82c85b89ea0a7bb2219e7d582bbc007a29fb6e149",
+                "sha256:c71821f93c931ae379cf9c9bbdd7099738fa00802ccf2a5271e2b68bc67a6ab8",
+                "sha256:ced0af509a31dcba0cd98ecdd06cb7c9ce66ebde78e0d99ba3515d4e991e34d0",
+                "sha256:d4e2a337c6c03b0e96ccd79940cbb04fe2063974d56fff6d78f8d57839546c57",
+                "sha256:d71af8b23ac23bc9e9f776451c125be6320ad4589a7d5bcb5ab5e1fc61b4e58f",
+                "sha256:d881b605b7448be451f02c59128dc5fac262dbd0dcff4638e702dc8c7bbb8ef0",
+                "sha256:e1987bbc482aa3e2e7fb72c70b22483cfaed3dbebc5ba6f9ac6f75240794709b",
+                "sha256:e32a78c81afba5699569c3493066ecb38fb45ccdf4c35b3c2232c9c2585b5257",
+                "sha256:f2a4dac4ebc4ad58a4ac911e39cf97cd74906c0c82c16333887aa9f287e98d5b",
+                "sha256:f39b57974db0e85897fff40518da420f4c4012b73515ca6f415a472228fea288",
+                "sha256:fd04764d0bb830414788cae897d082bf6ad92324e571a5511bd7e1de4a0cdc67",
+                "sha256:fde3f85864c84badb26f42d95639360e627fd09c529a76c46a06dbd7a5735c51"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.3.1"
+        },
         "iniconfig": {
             "hashes": [
                 "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
@@ -601,11 +751,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:430d00549b6aaf49bd0f5393150691edb1815afa62d457ee6b1a66b25cb17874",
-                "sha256:6e9213484e4ce1fb14267ee435e18f23cc3a0634e635b9fb4ed4677b84e0fdf8"
+                "sha256:1ae6047c1277508933078163721bbb479c3e7292778a04b4bacf0874550977d6",
+                "sha256:302558b81f1bc22dc259fb2a0c5c7cf2f4c0bdb21b50484348f7bafe7fb71421"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.21.2"
+            "version": "==6.22.0"
         },
         "ipympl": {
             "hashes": [
@@ -617,11 +767,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:b13a1d6c1f5818bd388db53b7107d17454129a70de2b87481d555daede5eb49e",
-                "sha256:b38c31e8fc7eff642fc7c597061fff462537cf2314e3225a19c906b7b0d8a345"
+                "sha256:1c183bf61b148b00bcebfa5d9b39312733ae97f6dad90d7e9b4d86c8647f498c",
+                "sha256:a950236df04ad75b5bc7f816f9af3d74dc118fd42f2ff7e80e8e60ca1f182e2d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.10.0"
+            "version": "==8.12.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -632,11 +782,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:c0005a77a47d77889cafed892b58e33b4a2a96712154404c6548ec22272811ea",
-                "sha256:ebb195e743b16c3947fe8827190fb87b4d00979c0fbf685afe4d2c4927059fa1"
+                "sha256:a60bf8d2528997e05ac83fd19ea2fbe65f2e79fbe1b2b35779bdfc46c2941dcc",
+                "sha256:de7d779f2045d60de9f6c25f653fdae2dba57898e6a1284494b3ba20b6893bb8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.0.4"
+            "version": "==8.0.6"
         },
         "isoduration": {
             "hashes": [
@@ -705,27 +855,27 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:be48ac6bd659cbbddb7a674cf06b3b8afbf53f228253cf58bde604c03bd487b0",
-                "sha256:ed65498bea6d876ef9d8da3e0db3dd33c5d129f5b2645f56ae03993782966bd0"
+                "sha256:3fbab64100a0dcac7701b1e0f1a4412f1ccb45546ff2ad9bc4fcbe4e19804811",
+                "sha256:d5b8e739d7816944be50f81121a109788a3d92732ecf1ad1e4dadebc948818fe"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.0.3"
+            "version": "==8.1.0"
         },
         "jupyter-console": {
             "hashes": [
-                "sha256:5931212d5cbc1f956f6fd615755b5e15f389a8eaa697288dbbe4377017615ecc",
-                "sha256:b32b5cb673a90909911988a2f1906ced005b07e8dadab6c8664d3f5de44a8191"
+                "sha256:309d33409fcc92ffdad25f0bcdf9a4a9daa61b6f341177570fdac03de5352485",
+                "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.6.1"
+            "version": "==6.6.3"
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:1407cdb4c79ee467696c04b76633fc1884015fa109323365a6372c8e890cc83f",
-                "sha256:4bdc2928c37f6917130c667d8b8708f20aee539d8283c6be72aabd2a4b4c83b0"
+                "sha256:6db75be0c83edbf1b7c9f91ec266a9a24ef945da630f3120e1a0046dc13713fc",
+                "sha256:d4201af84559bc8c70cead287e1ab94aeef3c512848dde077b7684b54d67730d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.2.0"
+            "version": "==5.3.0"
         },
         "jupyter-events": {
             "hashes": [
@@ -737,19 +887,19 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:29d6657bfb160b0e39b9030d67f33f918a188f2eba28065314a933b327fef872",
-                "sha256:b15078954120886d580e19d1746e2b62a3dc7bd082cb4716115c25fcd7061b00"
+                "sha256:9fde612791f716fd34d610cd939704a9639643744751ba66e7ee8fdc9cead07e",
+                "sha256:e6bc1e9e96d7c55b9ce9699ff6cb9a910581fe7349e27c40389acb67632e24c0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.3.0"
+            "version": "==2.5.0"
         },
         "jupyter-server-fileid": {
             "hashes": [
-                "sha256:0b580a9d75cc7a7132132d6bb3faa4645e34527bff8760861e9e6de51bec7397",
-                "sha256:340e86b45875d51a60e0e93d8d3bcb609c2bc8d315ec07c003f36d561f637c0e"
+                "sha256:1e0816d0857f490fadea11348570f0cba03f70f315c9842225aecfa45882b6af",
+                "sha256:6092ef114eddccf6cba69c0f0feb612c2f476f2e9467828809edb854c18806bb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.7.0"
+            "version": "==0.8.0"
         },
         "jupyter-server-terminals": {
             "hashes": [
@@ -761,27 +911,27 @@
         },
         "jupyter-server-ydoc": {
             "hashes": [
-                "sha256:18275ff1ce7e93bbda2301ca066273b3951fc50b0d9c8fc33788374134ad7920",
-                "sha256:ab10864708c81fa41ab9f2ed3626b54ff6926eaf14545d1d439714978dad6e9f"
+                "sha256:969a3a1a77ed4e99487d60a74048dc9fa7d3b0dcd32e60885d835bbf7ba7be11",
+                "sha256:a6fe125091792d16c962cc3720c950c2b87fcc8c3ecf0c54c84e9a20b814526c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.6.1"
+            "version": "==0.8.0"
         },
         "jupyter-ydoc": {
             "hashes": [
-                "sha256:3163bd4745eedd46d4bba6df52ab26be3c5c44c3a8aaf247635062486ea8f84f",
-                "sha256:596a9ae5986b59f8776c42430b5ad516405963574078ab801781933c9690be93"
+                "sha256:3ac51abfe378c6aeb62a449e8f0241bede1205f0199b0d27429140cbba950f79",
+                "sha256:98db7785215873c64d7dfcb1b741f41df11994c4b3d7e2957e004b392d6f11ea"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.2.2"
+            "version": "==0.2.3"
         },
         "jupyterlab": {
             "hashes": [
-                "sha256:ad6707dd0149b629d0ed5b56916cfcdb816b376c6af3190337faba09e27ea29e",
-                "sha256:aee98c174180e98a30470297d10b959e8e64f2288970c0de65f0a6d2b4807034"
+                "sha256:6c87c5910f886a14009352bc63f640961b206f73ed650dcf94d65f9dfcb30f95",
+                "sha256:e55bc40c36c2a52b76cf301138507a5488eb769137dd39d9f31a6259a00c6b03"
             ],
             "index": "pypi",
-            "version": "==3.6.1"
+            "version": "==3.6.2"
         },
         "jupyterlab-pygments": {
             "hashes": [
@@ -793,19 +943,19 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:51f6922e34f9f3db875051f4f7b57539a04ddd030f42d9ce6062dedf67bf7f2f",
-                "sha256:9aec21a2183bbedd9f91a86628355449575f1862d88b28ad5f905019d31e6c21"
+                "sha256:b4f5b48eaae1be83e2fd6fb77ac49d9b639be4ca4bd2e05b5368d29632a93725",
+                "sha256:ff1e7a81deb2dcb433215d469000988590fd5a5733574aa2698d643a6c9b3ace"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.19.0"
+            "version": "==2.21.0"
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:a04a42e50231b355b7087e16a818f541e53589f7647144ea0344c4bf16f300e5",
-                "sha256:eeaecdeaf6c03afc960ddae201ced88d5979b4ca9c3891bcb8f6631af705f5ef"
+                "sha256:c3a50ed5bf528a0c7a869096503af54702f86dda1db469aee1c92dc0c01b43ca",
+                "sha256:c73f8370338ec19f1bec47254752d6505b03601cbd5a67e6a0b184532f73a459"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.5"
+            "version": "==3.0.7"
         },
         "kiwisolver": {
             "hashes": [
@@ -883,25 +1033,37 @@
         },
         "llvmlite": {
             "hashes": [
-                "sha256:03f5557f9e52bbaeca60a2e89ae71ca907e2f19a03f75fea73d9d5bbf619654e",
-                "sha256:1a31702cc1383893a4c3d8103226207399848aaa4d16633051d0a25d78cfc726",
-                "sha256:2e83ca852d002d768523251719865223eae693bdf012720eccd0af95aee798b5",
-                "sha256:37b0da8df72d575d57e930efdd8e257c72bb9ceb722403483349fbdb5cadb726",
-                "sha256:4eeb747c5c8acb7bafb42ae7a69cf95ed21b26d54e7165bc8468f9e9c8a1ed5e",
-                "sha256:53981cde265b2dd489b0efccd2e456ac72b849d46a91a4b6f89b0267488edece",
-                "sha256:5d4f8433df3bdb5e008b9766aa6de5854f5c5b29314037d301c92ca12bfb7f1a",
-                "sha256:5f181a0aae3367787291cdfd07b703ddbb8b08f394fad64d555db7ea217d0f24",
-                "sha256:89b9458b1e1c9b3adb48695ba1bf266dffc7cb381265839113eda5a6102e731d",
-                "sha256:9c8aae96f7fba10d9ac864b443d1e8c7ee4765c31569a2b201b3d0b67d8fc596",
-                "sha256:b9ffc8a7c44f1726330ad8a6bc97272728212ef0667105259c26396e94a76fbd",
-                "sha256:c2b71b560555b9ddbdb50425bbecfb1df2d5153c1b0db47e2d23286deb1c3753",
-                "sha256:c5a09728fb98336dd3d0f96aafadb2e3b4ccc3214804f71a02afb9ae40cfcd91",
-                "sha256:cc4b5564c644ba438cf2616de4731766d093199299d9c7573f6fe8aa5b2c07c3",
-                "sha256:f74e8ae96cb82622f17cad04048f8565a906377d61df31bcb7c82038144aded1",
-                "sha256:ff7145e263cccb82e93ec88b30e55b7705ce40cde3bd86de26e83e22734b0632"
+                "sha256:03aee0ccd81735696474dc4f8b6be60774892a2929d6c05d093d17392c237f32",
+                "sha256:1578f5000fdce513712e99543c50e93758a954297575610f48cb1fd71b27c08a",
+                "sha256:16f56eb1eec3cda3a5c526bc3f63594fc24e0c8d219375afeb336f289764c6c7",
+                "sha256:1ec3d70b3e507515936e475d9811305f52d049281eaa6c8273448a61c9b5b7e2",
+                "sha256:22d36591cd5d02038912321d9ab8e4668e53ae2211da5523f454e992b5e13c36",
+                "sha256:3803f11ad5f6f6c3d2b545a303d68d9fabb1d50e06a8d6418e6fcd2d0df00959",
+                "sha256:39dc2160aed36e989610fc403487f11b8764b6650017ff367e45384dff88ffbf",
+                "sha256:3fc14e757bc07a919221f0cbaacb512704ce5774d7fcada793f1996d6bc75f2a",
+                "sha256:4c6ebace910410daf0bebda09c1859504fc2f33d122e9a971c4c349c89cca630",
+                "sha256:50aea09a2b933dab7c9df92361b1844ad3145bfb8dd2deb9cd8b8917d59306fb",
+                "sha256:60f8dd1e76f47b3dbdee4b38d9189f3e020d22a173c00f930b52131001d801f9",
+                "sha256:62c0ea22e0b9dffb020601bb65cb11dd967a095a488be73f07d8867f4e327ca5",
+                "sha256:6546bed4e02a1c3d53a22a0bced254b3b6894693318b16c16c8e43e29d6befb6",
+                "sha256:6717c7a6e93c9d2c3d07c07113ec80ae24af45cde536b34363d4bcd9188091d9",
+                "sha256:7ebf1eb9badc2a397d4f6a6c8717447c81ac011db00064a00408bc83c923c0e4",
+                "sha256:9ffc84ade195abd4abcf0bd3b827b9140ae9ef90999429b9ea84d5df69c9058c",
+                "sha256:a3f331a323d0f0ada6b10d60182ef06c20a2f01be21699999d204c5750ffd0b4",
+                "sha256:b1a0bbdb274fb683f993198775b957d29a6f07b45d184c571ef2a721ce4388cf",
+                "sha256:b43abd7c82e805261c425d50335be9a6c4f84264e34d6d6e475207300005d572",
+                "sha256:c0f158e4708dda6367d21cf15afc58de4ebce979c7a1aa2f6b977aae737e2a54",
+                "sha256:d0bfd18c324549c0fec2c5dc610fd024689de6f27c6cc67e4e24a07541d6e49b",
+                "sha256:ddab526c5a2c4ccb8c9ec4821fcea7606933dc53f510e2a6eebb45a418d3488a",
+                "sha256:e172c73fccf7d6db4bd6f7de963dedded900d1a5c6778733241d878ba613980e",
+                "sha256:e2c00ff204afa721b0bb9835b5bf1ba7fba210eefcec5552a9e05a63219ba0dc",
+                "sha256:e31f4b799d530255aaf0566e3da2df5bfc35d3cd9d6d5a3dcc251663656c27b1",
+                "sha256:e4f212c018db951da3e1dc25c2651abc688221934739721f2dad5ff1dd5f90e7",
+                "sha256:fa9b26939ae553bf30a9f5c4c754db0fb2d2677327f2511e674aa2f5df941789",
+                "sha256:fb62fc7016b592435d3e3a8f680e3ea8897c3c9e62e6e6cc58011e7a4801439e"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.33.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==0.39.1"
         },
         "markupsafe": {
             "hashes": [
@@ -961,50 +1123,50 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:01681566e95b9423021b49dea6a2395c16fa054604eacb87f0f4c439750f9114",
-                "sha256:03eb2c8ff8d85da679b71e14c7c95d16d014c48e0c0bfa14db85f6cdc5c92aad",
-                "sha256:092e6abc80cdf8a95f7d1813e16c0e99ceda8d5b195a3ab859c680f3487b80a2",
-                "sha256:0a776462a4a63c0bfc9df106c15a0897aa2dbab6795c693aa366e8e283958854",
-                "sha256:0dfd4a0cbd151f6439e6d7f8dca5292839ca311e7e650596d073774847ca2e4f",
-                "sha256:111ef351f28fd823ed7177632070a6badd6f475607122bc9002a526f2502a0b5",
-                "sha256:21269450243d6928da81a9bed201f0909432a74e7d0d65db5545b9fa8a0d0223",
-                "sha256:21a8aeac39b4a795e697265d800ce52ab59bdeb6bb23082e2d971f3041074f02",
-                "sha256:21bd4033c40b95abd5b8453f036ed5aa70856e56ecbd887705c37dce007a4c21",
-                "sha256:3493b48e56468c39bd9c1532566dff3b8062952721b7521e1f394eb6791495f4",
-                "sha256:3a10428d4f8d1a478ceabd652e61a175b2fdeed4175ab48da4a7b8deb561e3fa",
-                "sha256:3d1e52365d8d5af699f04581ca191112e1d1220a9ce4386b57d807124d8b55e6",
-                "sha256:3da8b9618188346239e51f1ea6c0f8f05c6e218cfcc30b399dd7dd7f52e8bceb",
-                "sha256:4497d88c559b76da320b7759d64db442178beeea06a52dc0c629086982082dcd",
-                "sha256:46ca923e980f76d34c1c633343a72bb042d6ba690ecc649aababf5317997171d",
-                "sha256:4f640534ec2760e270801056bc0d8a10777c48b30966eef78a7c35d8590915ba",
-                "sha256:51fb664c37714cbaac69c16d6b3719f517a13c96c3f76f4caadd5a0aa7ed0329",
-                "sha256:56b7b79488209041a9bf7ddc34f1b069274489ce69e34dc63ae241d0d6b4b736",
-                "sha256:691ef1f15360e439886186d0db77b5345b24da12cbc4fc57b26c4826db4d6cab",
-                "sha256:71b751d06b2ed1fd017de512d7439c0259822864ea16731522b251a27c0b2ede",
-                "sha256:7d0dcd1a0bf8d56551e8617d6dc3881d8a1c7fb37d14e5ec12cbb293f3e6170a",
-                "sha256:827e78239292e561cfb70abf356a9d7eaf5bf6a85c97877f254009f20b892f89",
-                "sha256:8665855f3919c80551f377bc16df618ceabf3ef65270bc14b60302dce88ca9ab",
-                "sha256:8f6efd313430d7ef70a38a3276281cb2e8646b3a22b3b21eb227da20e15e6813",
-                "sha256:9d85355c48ef8b9994293eb7c00f44aa8a43cad7a297fbf0770a25cdb2244b91",
-                "sha256:a06a6c9822e80f323549c6bc9da96d4f233178212ad9a5f4ab87fd153077a507",
-                "sha256:b51ab8a5d5d3bbd4527af633a638325f492e09e45e78afdf816ef55217a09664",
-                "sha256:c0592ba57217c22987b7322df10f75ef95bc44dce781692b4b7524085de66019",
-                "sha256:c5465735eaaafd1cfaec3fed60aee776aeb3fd3992aa2e49f4635339c931d443",
-                "sha256:c849aa94ff2a70fb71f318f48a61076d1205c6013b9d3885ade7f992093ac434",
-                "sha256:c869b646489c6a94375714032e5cec08e3aa8d3f7d4e8ef2b0fb50a52b317ce6",
-                "sha256:cb52aa97b92acdee090edfb65d1cb84ea60ab38e871ba8321a10bbcebc2a3540",
-                "sha256:cf119eee4e57389fba5ac8b816934e95c256535e55f0b21628b4205737d1de85",
-                "sha256:cf6346644e8fe234dc847e6232145dac199a650d3d8025b3ef65107221584ba4",
-                "sha256:de20eb1247725a2f889173d391a6d9e7e0f2540feda24030748283108b0478ec",
-                "sha256:eb2e76cd429058d8954121c334dddfcd11a6186c6975bca61f3f248c99031b05",
-                "sha256:f336e7014889c38c59029ebacc35c59236a852e4b23836708cfd3f43d1eaeed5",
-                "sha256:f4ddac5f59e78d04b20469bc43853a8e619bb6505c7eac8ffb343ff2c516d72f",
-                "sha256:f910d924da8b9fb066b5beae0b85e34ed1b6293014892baadcf2a51da1c65807",
-                "sha256:f91d35b3ef51d29d9c661069b9e4ba431ce283ffc533b981506889e144b5b40e",
-                "sha256:fb0304c1cd802e9a25743414c887e8a7cd51d96c9ec96d388625d2cd1c137ae3"
+                "sha256:08308bae9e91aca1ec6fd6dda66237eef9f6294ddb17f0d0b3c863169bf82353",
+                "sha256:14645aad967684e92fc349493fa10c08a6da514b3d03a5931a1bac26e6792bd1",
+                "sha256:21e9cff1a58d42e74d01153360de92b326708fb205250150018a52c70f43c290",
+                "sha256:28506a03bd7f3fe59cd3cd4ceb2a8d8a2b1db41afede01f66c42561b9be7b4b7",
+                "sha256:2bf092f9210e105f414a043b92af583c98f50050559616930d884387d0772aba",
+                "sha256:3032884084f541163f295db8a6536e0abb0db464008fadca6c98aaf84ccf4717",
+                "sha256:3a2cb34336110e0ed8bb4f650e817eed61fa064acbefeb3591f1b33e3a84fd96",
+                "sha256:3ba2af245e36990facf67fde840a760128ddd71210b2ab6406e640188d69d136",
+                "sha256:3d7bc90727351fb841e4d8ae620d2d86d8ed92b50473cd2b42ce9186104ecbba",
+                "sha256:438196cdf5dc8d39b50a45cb6e3f6274edbcf2254f85fa9b895bf85851c3a613",
+                "sha256:46a561d23b91f30bccfd25429c3c706afe7d73a5cc64ef2dfaf2b2ac47c1a5dc",
+                "sha256:4cf327e98ecf08fcbb82685acaf1939d3338548620ab8dfa02828706402c34de",
+                "sha256:4f99e1b234c30c1e9714610eb0c6d2f11809c9c78c984a613ae539ea2ad2eb4b",
+                "sha256:544764ba51900da4639c0f983b323d288f94f65f4024dc40ecb1542d74dc0500",
+                "sha256:56d94989191de3fcc4e002f93f7f1be5da476385dde410ddafbb70686acf00ea",
+                "sha256:57bfb8c8ea253be947ccb2bc2d1bb3862c2bccc662ad1b4626e1f5e004557042",
+                "sha256:617f14ae9d53292ece33f45cba8503494ee199a75b44de7717964f70637a36aa",
+                "sha256:6eb88d87cb2c49af00d3bbc33a003f89fd9f78d318848da029383bfc08ecfbfb",
+                "sha256:75d4725d70b7c03e082bbb8a34639ede17f333d7247f56caceb3801cb6ff703d",
+                "sha256:770a205966d641627fd5cf9d3cb4b6280a716522cd36b8b284a8eb1581310f61",
+                "sha256:7b73305f25eab4541bd7ee0b96d87e53ae9c9f1823be5659b806cd85786fe882",
+                "sha256:7c9a4b2da6fac77bcc41b1ea95fadb314e92508bf5493ceff058e727e7ecf5b0",
+                "sha256:81a6b377ea444336538638d31fdb39af6be1a043ca5e343fe18d0f17e098770b",
+                "sha256:83111e6388dec67822e2534e13b243cc644c7494a4bb60584edbff91585a83c6",
+                "sha256:8704726d33e9aa8a6d5215044b8d00804561971163563e6e6591f9dcf64340cc",
+                "sha256:89768d84187f31717349c6bfadc0e0d8c321e8eb34522acec8a67b1236a66332",
+                "sha256:8bf26ade3ff0f27668989d98c8435ce9327d24cffb7f07d24ef609e33d582439",
+                "sha256:8c587963b85ce41e0a8af53b9b2de8dddbf5ece4c34553f7bd9d066148dc719c",
+                "sha256:95cbc13c1fc6844ab8812a525bbc237fa1470863ff3dace7352e910519e194b1",
+                "sha256:97cc368a7268141afb5690760921765ed34867ffb9655dd325ed207af85c7529",
+                "sha256:a867bf73a7eb808ef2afbca03bcdb785dae09595fbe550e1bab0cd023eba3de0",
+                "sha256:b867e2f952ed592237a1828f027d332d8ee219ad722345b79a001f49df0936eb",
+                "sha256:c0bd19c72ae53e6ab979f0ac6a3fafceb02d2ecafa023c5cca47acd934d10be7",
+                "sha256:ce463ce590f3825b52e9fe5c19a3c6a69fd7675a39d589e8b5fbe772272b3a24",
+                "sha256:cf0e4f727534b7b1457898c4f4ae838af1ef87c359b76dcd5330fa31893a3ac7",
+                "sha256:def58098f96a05f90af7e92fd127d21a287068202aa43b2a93476170ebd99e87",
+                "sha256:e99bc9e65901bb9a7ce5e7bb24af03675cbd7c70b30ac670aa263240635999a4",
+                "sha256:eb7d248c34a341cd4c31a06fd34d64306624c8cd8d0def7abb08792a5abfd556",
+                "sha256:f67bfdb83a8232cb7a92b869f9355d677bce24485c460b19d01970b64b2ed476",
+                "sha256:f883a22a56a84dba3b588696a2b8a1ab0d2c3d41be53264115c71b0a942d8fdb",
+                "sha256:fbdeeb58c0cf0595efe89c05c224e0a502d1aa6a8696e68a73c3efc6bc354304"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.7.0"
+            "version": "==3.7.1"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -1029,6 +1191,14 @@
             ],
             "version": "==2.0.5"
         },
+        "multivolumefile": {
+            "hashes": [
+                "sha256:237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678",
+                "sha256:a0648d0aafbc96e59198d5c17e9acad7eb531abea51035d08ce8060dcad709d6"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.2.3"
+        },
         "munch": {
             "hashes": [
                 "sha256:2d735f6f24d4dba3417fa448cae40c6e896ec1fdab6cdb5e6510999758a4dbd2",
@@ -1038,19 +1208,19 @@
         },
         "natsort": {
             "hashes": [
-                "sha256:04fe18fdd2b9e5957f19f687eb117f102ef8dde6b574764e536e91194bed4f5f",
-                "sha256:57f85b72c688b09e053cdac302dd5b5b53df5f73ae20b4874fcbffd8bf783d11"
+                "sha256:517595492dde570a4fd6b6a76f644440c1ba51e2338c8a671d7f0475fda8f9fd",
+                "sha256:d583bc9050dd10538de36297c960b93f873f0cd01671a3c50df5bd86dd391dcb"
             ],
             "index": "pypi",
-            "version": "==8.2.0"
+            "version": "==8.3.1"
         },
         "nbclassic": {
             "hashes": [
-                "sha256:40f11bbcc59e8956c3d5ef132dec8e5a853e893ecf831e791d54da0d8a50d79d",
-                "sha256:6403a996562dadefa7fee9c49e17b663b5fd508241de5df655b90011cf3342d9"
+                "sha256:312b3f7d7ff2e6c261d51799bb12e6493498ab47d3469d3a01015d5533fd4d2b",
+                "sha256:df78e4ba143f35084ad060deec16cb1d4839dde47bfdbc4232beb7071a6d70ea"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.5.2"
+            "version": "==0.5.4"
         },
         "nbclient": {
             "hashes": [
@@ -1062,19 +1232,19 @@
         },
         "nbconvert": {
             "hashes": [
-                "sha256:495638c5e06005f4a5ce828d8a81d28e34f95c20f4384d5d7a22254b443836e7",
-                "sha256:a42c3ac137c64f70cbe4d763111bf358641ea53b37a01a5c202ed86374af5234"
+                "sha256:8eed67bd8314f3ec87c4351c2f674af3a04e5890ab905d6bd927c05aec1cf27d",
+                "sha256:e41118f81698d3d59b3c7c2887937446048f741aba6c367c1c1a77810b3e2d08"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.2.9"
+            "version": "==7.2.10"
         },
         "nbformat": {
             "hashes": [
-                "sha256:22a98a6516ca216002b0a34591af5bcb8072ca6c63910baffc901cfa07fefbf0",
-                "sha256:4b021fca24d3a747bf4e626694033d792d594705829e5e35b14ee3369f9f6477"
+                "sha256:46dac64c781f1c34dfd8acba16547024110348f9fc7eab0f31981c2a3dc48d1f",
+                "sha256:d910082bd3e0bffcf07eabf3683ed7dda0727a326c446eeb2922abe102e65162"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.7.3"
+            "version": "==5.8.0"
         },
         "nest-asyncio": {
             "hashes": [
@@ -1127,11 +1297,11 @@
         },
         "notebook": {
             "hashes": [
-                "sha256:c1897e5317e225fc78b45549a6ab4b668e4c996fd03a04e938fe5e7af2bfffd0",
-                "sha256:e04f9018ceb86e4fa841e92ea8fb214f8d23c1cedfde530cc96f92446924f0e4"
+                "sha256:50a334ad9d60b30cb759405168ef6fc3d60350ab5439fb1631544bb09dcb2cce",
+                "sha256:b12bee3292211d85dd7e588a790ddce30cb3e8fbcfa1e803522a207f60819e05"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.5.2"
+            "version": "==6.5.3"
         },
         "notebook-shim": {
             "hashes": [
@@ -1143,28 +1313,37 @@
         },
         "numba": {
             "hashes": [
-                "sha256:24852c21fbf7edf9e000eeec9fbd1b24d1ca17c86ae449b06a3707bcdec95479",
-                "sha256:2e17eadf3c5c3cae525ce6d1a71d021dd67a980ce766484a68018d3c7d7cfcab",
-                "sha256:38146c10f8457705e74e2e02f0d0d339f4a5143b92cccaa1ef18ae240bcdae4f",
-                "sha256:3a4114dc1b9af491235ce30517913afe0a0a226117a924e7361d626f55e0e054",
-                "sha256:404809fca5fd71b20096a6eea201c5bcf5255337818d3939f1631ba190a06d1f",
-                "sha256:44407fd45655d887b3bb12b9282c0d994c2e7d59ba537e7429349403de6eb8c0",
-                "sha256:5848d6bc5604664823a1681b17eb934147b24cfcb4df672be33315ff3f9f7afe",
-                "sha256:5caf68ac45eddafc6c4275cc8bae67bdbc246536284821f7894c5a8fabb0c132",
-                "sha256:753f400038e74b685ef21edf8218c524a782c269819e074c83fa241e6c6cde0e",
-                "sha256:77eba26deda636cf67c1554e2af70063a603b952e181f0570caa3ff1f101c950",
-                "sha256:89e81b51b880f9b18c82b7095beaccc6856fcf84ba29c4f0ced42e4e5748a3a7",
-                "sha256:925580ec493370206abe9143f96a3e843a1982dbae2f33151a71614ab9aca39d",
-                "sha256:943ddf7192a90823485028039cdef5358fddd0f55922c6ab46fe6b74b7335f16",
-                "sha256:c98b09d78144da2e5b8c7b690f80d12076a6b8e9af6e3c7f772a08cf04821cc1",
-                "sha256:d42e0bdfdb920db7cfc68d49908aba25f51789a9b2497dc4dc889d8df234691d",
-                "sha256:d9a8089b9c6905fab7465700caf27757fc32b8011df9c4ea48a261cdb58c1dad",
-                "sha256:ddc73deb0637699df4ed678f48d1d73f9397e86b134f6f47868c823241578706",
-                "sha256:df5217e05e5612f4df7033a1588b9e55c76dead00faf40ed168f1c62b21d64ad",
-                "sha256:e4c0abd4c75b3da824d9601989d97666db6e94011e26e83eb78ac8e723224460"
+                "sha256:0240f9026b015e336069329839208ebd70ec34ae5bfbf402e4fcc8e06197528e",
+                "sha256:03634579d10a6129181129de293dd6b5eaabee86881369d24d63f8fe352dd6cb",
+                "sha256:03fe94cd31e96185cce2fae005334a8cc712fc2ba7756e52dff8c9400718173f",
+                "sha256:0611e6d3eebe4cb903f1a836ffdb2bda8d18482bcd0a0dcc56e79e2aa3fefef5",
+                "sha256:0da583c532cd72feefd8e551435747e0e0fbb3c0530357e6845fcc11e38d6aea",
+                "sha256:14dbbabf6ffcd96ee2ac827389afa59a70ffa9f089576500434c34abf9b054a4",
+                "sha256:32d9fef412c81483d7efe0ceb6cf4d3310fde8b624a9cecca00f790573ac96ee",
+                "sha256:3a993349b90569518739009d8f4b523dfedd7e0049e6838c0e17435c3e70dcc4",
+                "sha256:3cb1a07a082a61df80a468f232e452d818f5ae254b40c26390054e4e868556e0",
+                "sha256:42f9e1be942b215df7e6cc9948cf9c15bb8170acc8286c063a9e57994ef82fd1",
+                "sha256:4373da9757049db7c90591e9ec55a2e97b2b36ba7ae3bf9c956a513374077470",
+                "sha256:4e08e203b163ace08bad500b0c16f6092b1eb34fd1fce4feaf31a67a3a5ecf3b",
+                "sha256:553da2ce74e8862e18a72a209ed3b6d2924403bdd0fb341fa891c6455545ba7c",
+                "sha256:720886b852a2d62619ae3900fe71f1852c62db4f287d0c275a60219e1643fc04",
+                "sha256:85dbaed7a05ff96492b69a8900c5ba605551afb9b27774f7f10511095451137c",
+                "sha256:8a95ca9cc77ea4571081f6594e08bd272b66060634b8324e99cd1843020364f9",
+                "sha256:91f021145a8081f881996818474ef737800bcc613ffb1e618a655725a0f9e246",
+                "sha256:9f62672145f8669ec08762895fe85f4cf0ead08ce3164667f2b94b2f62ab23c3",
+                "sha256:a12ef323c0f2101529d455cfde7f4135eaa147bad17afe10b48634f796d96abd",
+                "sha256:c602d015478b7958408d788ba00a50272649c5186ea8baa6cf71d4a1c761bba1",
+                "sha256:c75e8a5f810ce80a0cfad6e74ee94f9fde9b40c81312949bf356b7304ef20740",
+                "sha256:d0ae9270a7a5cc0ede63cd234b4ff1ce166c7a749b91dbbf45e0000c56d3eade",
+                "sha256:d69ad934e13c15684e7887100a8f5f0f61d7a8e57e0fd29d9993210089a5b531",
+                "sha256:dbcc847bac2d225265d054993a7f910fda66e73d6662fe7156452cac0325b073",
+                "sha256:e64d338b504c9394a4a34942df4627e1e6cb07396ee3b49fe7b8d6420aa5104f",
+                "sha256:f4cfc3a19d1e26448032049c79fc60331b104f694cf570a9e94f4e2c9d0932bb",
+                "sha256:fbfb45e7b297749029cb28694abf437a78695a100e7c2033983d69f0ba2698d4",
+                "sha256:fcdf84ba3ed8124eb7234adfbb8792f311991cbf8aed1cad4b1b1a7ee08380c1"
             ],
             "index": "pypi",
-            "version": "==0.50.1"
+            "version": "==0.56.4"
         },
         "numexpr": {
             "hashes": [
@@ -1204,31 +1383,37 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207",
-                "sha256:1ce7ab2053e36c0a71e7a13a7475bd3b1f54750b4b433adc96313e127b870887",
-                "sha256:2d487e06ecbf1dc2f18e7efce82ded4f705f4bd0cd02677ffccfb39e5c284c7e",
-                "sha256:37431a77ceb9307c28382c9773da9f306435135fae6b80b62a11c53cfedd8802",
-                "sha256:3e1ffa4748168e1cc8d3cde93f006fe92b5421396221a02f2274aab6ac83b077",
-                "sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af",
-                "sha256:43a8ca7391b626b4c4fe20aefe79fec683279e31e7c79716863b4b25021e0e74",
-                "sha256:4c6036521f11a731ce0648f10c18ae66d7143865f19f7299943c985cdc95afb5",
-                "sha256:59d55e634968b8f77d3fd674a3cf0b96e85147cd6556ec64ade018f27e9479e1",
-                "sha256:64f56fc53a2d18b1924abd15745e30d82a5782b2cab3429aceecc6875bd5add0",
-                "sha256:7228ad13744f63575b3a972d7ee4fd61815b2879998e70930d4ccf9ec721dce0",
-                "sha256:9ce7df0abeabe7fbd8ccbf343dc0db72f68549856b863ae3dd580255d009648e",
-                "sha256:a911e317e8c826ea632205e63ed8507e0dc877dcdc49744584dfc363df9ca08c",
-                "sha256:b89bf9b94b3d624e7bb480344e91f68c1c6c75f026ed6755955117de00917a7c",
-                "sha256:ba9ead61dfb5d971d77b6c131a9dbee62294a932bf6a356e48c75ae684e635b3",
-                "sha256:c1d937820db6e43bec43e8d016b9b3165dcb42892ea9f106c70fb13d430ffe72",
-                "sha256:cc7f00008eb7d3f2489fca6f334ec19ca63e31371be28fd5dad955b16ec285bd",
-                "sha256:d4c5d5eb2ec8da0b4f50c9a843393971f31f1d60be87e0fb0917a49133d257d6",
-                "sha256:e96d7f3096a36c8754207ab89d4b3282ba7b49ea140e4973591852c77d09eb76",
-                "sha256:f0725df166cf4785c0bc4cbfb320203182b1ecd30fee6e541c8752a92df6aa32",
-                "sha256:f3eb268dbd5cfaffd9448113539e44e2dd1c5ca9ce25576f7c04a5453edc26fa",
-                "sha256:fb7a980c81dd932381f8228a426df8aeb70d59bbcda2af075b627bbc50207cba"
+                "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d",
+                "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07",
+                "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df",
+                "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9",
+                "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d",
+                "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a",
+                "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719",
+                "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2",
+                "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280",
+                "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa",
+                "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387",
+                "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1",
+                "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43",
+                "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f",
+                "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398",
+                "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63",
+                "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de",
+                "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8",
+                "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481",
+                "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0",
+                "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d",
+                "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e",
+                "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96",
+                "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb",
+                "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6",
+                "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d",
+                "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a",
+                "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
             ],
             "index": "pypi",
-            "version": "==1.22.4"
+            "version": "==1.23.5"
         },
         "packaging": {
             "hashes": [
@@ -1240,25 +1425,25 @@
         },
         "pandas": {
             "hashes": [
-                "sha256:02f1e8f71cd994ed7fcb9a35b6ddddeb4314822a0e09a9c5b2d278f8cb5d4096",
-                "sha256:13f75fb18486759da3ff40f5345d9dd20e7d78f2a39c5884d013456cec9876f0",
-                "sha256:35b670b0abcfed7cad76f2834041dcf7ae47fd9b22b63622d67cdc933d79f453",
-                "sha256:4c73f373b0800eb3062ffd13d4a7a2a6d522792fa6eb204d67a4fad0a40f03dc",
-                "sha256:5759edf0b686b6f25a5d4a447ea588983a33afc8a0081a0954184a4a87fd0dd7",
-                "sha256:5a7cf6044467c1356b2b49ef69e50bf4d231e773c3ca0558807cdba56b76820b",
-                "sha256:69c5d920a0b2a9838e677f78f4dde506b95ea8e4d30da25859db6469ded84fa8",
-                "sha256:8778a5cc5a8437a561e3276b85367412e10ae9fff07db1eed986e427d9a674f8",
-                "sha256:9871ef5ee17f388f1cb35f76dc6106d40cb8165c562d573470672f4cdefa59ef",
-                "sha256:9c31d52f1a7dd2bb4681d9f62646c7aa554f19e8e9addc17e8b1b20011d7522d",
-                "sha256:ab8173a8efe5418bbe50e43f321994ac6673afc5c7c4839014cf6401bbdd0705",
-                "sha256:ae961f1f0e270f1e4e2273f6a539b2ea33248e0e3a11ffb479d757918a5e03a9",
-                "sha256:b3c4f93fcb6e97d993bf87cdd917883b7dab7d20c627699f360a8fb49e9e0b91",
-                "sha256:c9410ce8a3dee77653bc0684cfa1535a7f9c291663bd7ad79e39f5ab58f67ab3",
-                "sha256:f69e0f7b7c09f1f612b1f8f59e2df72faa8a6b41c5a436dde5b615aaf948f107",
-                "sha256:faa42a78d1350b02a7d2f0dbe3c80791cf785663d6997891549d0f86dc49125e"
+                "sha256:0210f8fe19c2667a3817adb6de2c4fd92b1b78e1975ca60c0efa908e0985cbdb",
+                "sha256:0227e3a6e3a22c0e283a5041f1e3064d78fbde811217668bb966ed05386d8a7e",
+                "sha256:0bc440493cf9dc5b36d5d46bbd5508f6547ba68b02a28234cd8e81fdce42744d",
+                "sha256:16504f915f1ae424052f1e9b7cd2d01786f098fbb00fa4e0f69d42b22952d798",
+                "sha256:182a5aeae319df391c3df4740bb17d5300dcd78034b17732c12e62e6dd79e4a4",
+                "sha256:35db623487f00d9392d8af44a24516d6cb9f274afaf73cfcfe180b9c54e007d2",
+                "sha256:40ec0a7f611a3d00d3c666c4cceb9aa3f5bf9fbd81392948a93663064f527203",
+                "sha256:47a03bfef80d6812c91ed6fae43f04f2fa80a4e1b82b35aa4d9002e39529e0b8",
+                "sha256:4b21d46728f8a6be537716035b445e7ef3a75dbd30bd31aa1b251323219d853e",
+                "sha256:4d1a806252001c5db7caecbe1a26e49a6c23421d85a700960f6ba093112f54a1",
+                "sha256:60e20a4ab4d4fec253557d0fc9a4e4095c37b664f78c72af24860c8adcd07088",
+                "sha256:9f61cca5262840ff46ef857d4f5f65679b82188709d0e5e086a9123791f721c8",
+                "sha256:a15835c8409d5edc50b4af93be3377b5dd3eb53517e7f785060df1f06f6da0e2",
+                "sha256:b39508562ad0bb3f384b0db24da7d68a2608b9ddc85b1d931ccaaa92d5e45273",
+                "sha256:ed60848caadeacecefd0b1de81b91beff23960032cded0ac1449242b506a3b3f",
+                "sha256:fc714895b6de6803ac9f661abb316853d0cd657f5d23985222255ad76ccedc25"
             ],
             "index": "pypi",
-            "version": "==1.0.5"
+            "version": "==1.1"
         },
         "pandocfilters": {
             "hashes": [
@@ -1384,11 +1569,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9",
-                "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"
+                "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08",
+                "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.0"
+            "version": "==3.2.0"
         },
         "pluggy": {
             "hashes": [
@@ -1408,10 +1593,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:6a2948ec427dfcc7c983027b1044b355db6aaa8be374f54ad2015471f7d81c5b"
+                "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b",
+                "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.37"
+            "version": "==3.0.38"
         },
         "psutil": {
             "hashes": [
@@ -1430,7 +1616,7 @@
                 "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7",
                 "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "sys_platform != 'cygwin'",
             "version": "==5.9.4"
         },
         "psycopg2-binary": {
@@ -1489,6 +1675,129 @@
             ],
             "version": "==0.2.2"
         },
+        "py7zr": {
+            "hashes": [
+                "sha256:1d01f98ea1e1f5c49940358691b2076f9a5848056426541e783de33834f59e21",
+                "sha256:94d0c24217f6582741813ee94490a4ca82bd5f9bf35e4f8610cb588cf7445764"
+            ],
+            "index": "pypi",
+            "version": "==0.20.4"
+        },
+        "pyarrow": {
+            "hashes": [
+                "sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc",
+                "sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c",
+                "sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06",
+                "sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841",
+                "sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2",
+                "sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835",
+                "sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854",
+                "sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2",
+                "sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e",
+                "sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97",
+                "sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d",
+                "sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418",
+                "sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba",
+                "sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70",
+                "sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85",
+                "sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00",
+                "sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56",
+                "sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689",
+                "sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c",
+                "sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4",
+                "sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841",
+                "sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b",
+                "sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca",
+                "sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5",
+                "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"
+            ],
+            "index": "pypi",
+            "version": "==11.0.0"
+        },
+        "pybcj": {
+            "hashes": [
+                "sha256:023082fd677f67ebd36fe96322a4a45ac33a2b340d49010d88e1867c76744c50",
+                "sha256:05fad9a905772774aacc96cb174571ac1f5afa80b9f54c6ec414d369865d305c",
+                "sha256:09872b32edad4e3653d5b357b244d267ca58fe52d4e1dd3cdff816d3bb9d9f7c",
+                "sha256:0eaa90639992b6096afb1485380fae7f084483db6b92867847a3bfdf22cc4efc",
+                "sha256:10182725b0e6aa944d13a10a4a9cb5208bafe0016b4326253340948153de4bc0",
+                "sha256:10961ea10ae930b9348132707b9dd3cf3e71a41ef1df7656fbc4f14a71f10747",
+                "sha256:15edd1786617127ecfda4274bbb04f09ae299c474ada86e369bcf050d5cb88dd",
+                "sha256:1a5365edcaa82dc47e7757ba2efb48f96b9b352e3811a2aaa90084802479ddbe",
+                "sha256:1c0e1657c233f9f4070ab578951e03d569f1b645042ce661341091f50e41b541",
+                "sha256:1e6434a46f852cd3e6929633b43537887bd381bc614dbf5c4a128fdde4966b3a",
+                "sha256:20fc0d8f67e2d9747e0c31082d5f64b112258ae602a85aa5c7e6bf5a7cad287b",
+                "sha256:21098001200273c3c9fd90e7bf909fb905a8e1c102c80b604cb7c6a3103ef7e0",
+                "sha256:225a0addf4b3d580bf4eae583b5168dac0125a703c53ded8b3f120882e1e0312",
+                "sha256:262f53e27bca6096e3424c63e5e59948b10985eee4b03a5d70c3f3f6161a79e7",
+                "sha256:2d6b34ec233fcf5a83ccfbf422fef22256947eaa7077aaa012e5961d15aa302c",
+                "sha256:2e1859d36c073231737956fbeb5bbcfa8dba880e1b66bfbd001466718d6d89dc",
+                "sha256:358dba3dc39a07cded6897b9f99bb5b951a0ad95d567eda535b44861caa02f5b",
+                "sha256:3847387b43af47d9677952b8a22d9c2d8a544c2175b6d5304c200669c05d39e1",
+                "sha256:393d95f83e47976d137bcec7b66986f51282dcb2091933f88983dd7eb89e59c4",
+                "sha256:39dd836134e261ec769cd5aa9ae7a3a330a7dac81efb66eb5504643abd8235df",
+                "sha256:421ed75e54ebecd79c80178c1df5bdbe1e0e3e10e7efef5f011b5f0be6a9a12f",
+                "sha256:421f211fb15aeb836b4ba61174cb409fc82222ab3b2486deb4953ae863e6507b",
+                "sha256:43e8bc75773ca06ee7a64602b799613171e4edf4d9d8fd38fa5c49f1cdbb4407",
+                "sha256:46b82fe50eb8171ee2205e935f3fd5900e31beb5e54e10c88f23a5420902467d",
+                "sha256:4bc8720f3a224c27bd413a930b9bec5f225fda050641258967b1ebb252a053fb",
+                "sha256:4d10dd75fad48555e9530c5565c7ccf13754adad2fe331feefb263055cdca7b3",
+                "sha256:534b8b253dbdb746c06bab28383db31d7e2b42aa9b33ed4e7836319622dcd75b",
+                "sha256:570a3cf4e016dcb0fc561991833e5170a2a0bc6ee88fe5667591f356bd7b7895",
+                "sha256:5de90f8b6c7fc1d28dbe74c29b1d5053a7a8703cbc2c6f4f112907ffd7529f8e",
+                "sha256:603daa737579cf69efb368fab716cdce18d0b2615af77bb623f5f42aa546b3d8",
+                "sha256:6485c6b091504c0e6431a9495309271626eaa9ecb23276903486824f94f4c551",
+                "sha256:6ca6ddae1302477879099d4c4efc65790f4610d71ceff7fbe8f8b60f6ac6dcff",
+                "sha256:6df9eccc99a0d7bc091b58cff2f507b89f076d657253975fa2ca9eb42dbb4733",
+                "sha256:6f589af70286ec6565e3415145a03abc3c14a23ed7ed198ac741de81af332f26",
+                "sha256:731800acfc6112132aa2b7d08f9d6fe49a0c0071b30985809d084e238af98dac",
+                "sha256:74d34340323996b70dbd73e9530cca71c05ff7c97e30fe4d32aeea2f877836ca",
+                "sha256:7801ee9a9fcd47b92d4d90ff9a28cfdc23195cad72bd8032938ab3c794942b43",
+                "sha256:795dff9229dc024e54bd0f618f5a3adb269ee0cccd7ac9a0bef29df388beed23",
+                "sha256:7b3773a77ae3b18778c9bf22c7ba6478a0e5416f84b7d2ac6d764001f6d0d985",
+                "sha256:872697d8bff2572e4225ed8cbce17be338faac28ec1ab3c00419aaef2f56dd3c",
+                "sha256:8b682ed08caabfb7c042d4be083e28ddc692afb1deff5567111f8855071b75c3",
+                "sha256:8e846a8272bf02202794fe22beaf389ed27c2d8ebf59aafb43af4935feac0389",
+                "sha256:8efed581f2ee74f1e0ec04a10e97881b93abc258d13b15ef966aee71732ac152",
+                "sha256:970dc23ca1c64611d35a3abe76a059cf551da53d62faefd84c5bf3e0af1602d1",
+                "sha256:9b56eeff51efa556ecc186260ac486a4ddd79ad37bc88d669e96c45190f3c0da",
+                "sha256:9fc313b1a5547c5416982853f2de1454980704f3ab3dbcad18dacdc565a2eafc",
+                "sha256:a74e70bf3fd50a413fdce4264e037b8e8f34cb8d9207ac364167b6eb076c14ec",
+                "sha256:a77796b4c5370cedd4fad2264b6d7a78cb40229c7fa3cbcab24df3adea768962",
+                "sha256:a81f14f213a75597f9be44feb97740a51adda558465fb159114472dc2ab39ef8",
+                "sha256:acfc4a02ddf22f6df7184441b39f38c31e95aa8af41de4d2f825821ab1fb85c6",
+                "sha256:b3861996b06b8238f799b4f1bd9542d1a8ae8e4765adbdde25ed011c3bda11df",
+                "sha256:b901f12380e988da07f21bb6b75da7f91fd9feffb43fcf70fad698e40a2ef3a7",
+                "sha256:b99f4291e59dcbe548be5a1e8c6a1a19a860184526c2d14fc374ec687b98ad7d",
+                "sha256:bb378b0f133e19d437eca4327bac7c3f38e30950c5c604092c72b18cba839bc2",
+                "sha256:bbb49772fc3896850a704215160df8316db89e5e8876b2d8af6c6c15b4e0f6ea",
+                "sha256:bc1684b9f7ec92d2ae94a137ec311bd2227f684429521061af7ceed4952c7f72",
+                "sha256:bdabbe7fd66886943393ecf98318d7801dd40183af80314acd4464bccdd44d53",
+                "sha256:bf87f2a7f827656bc6e1d9888d47931aa0ae35cdc4ff33b1cec70d8d462590b3",
+                "sha256:c1e02170f8d358a8ddc716606760c73d55eea6bdb0cca2d97b86447e9524708b",
+                "sha256:c72ff262613c9a6f20e80bcf1e8bbc000b78b95a7fa301164ab3e3bd23bd936c",
+                "sha256:c854a206d8c3a5a959b803405760f3627bb4878450e2f36b5d35af09c89152fc",
+                "sha256:d5b327df02761c42399c878cd6c37f885bf0639befbd4d1ab763cd44ba1e0552",
+                "sha256:d5c4ca6faff0af4b5f3e7d88d13ec76f8cac36c9bcc814b8c84d9f3f951b2cf9",
+                "sha256:d61f287f820787d3acf60d113c5ce6e506870d9d3103bc37a74373e72ce9d7a6",
+                "sha256:dc23f2ac2c1ded250f1aa66fbd1a3d823f76de549978b61eed4fb34affc11338",
+                "sha256:dc79ed4773cd35328377a8fedbbdcafb3a9d242ee63b96863c0692c81faefab8",
+                "sha256:df75707f466ab6fa086f164bff2df75fd16543c8d43ca43a268f938c1144e792",
+                "sha256:e6a74cb618da93ac1322d6a548a4508e76eb4c388ed1c80560bc25d8764cf272",
+                "sha256:e89a814f1727be7d543ac6910f0d94131f43a337e811ab684606d42dbc22b701",
+                "sha256:efe75e3b8768c4f9d454d3c1b2b2a67e757f2b00d638146d3a4cddb38460fc3a",
+                "sha256:f2f950ca403ffaa808a017e40e3371115bcb0b4b1061772b03e7d842555132ac",
+                "sha256:f4428b6808d781f4b605a27f53fc10a3ca343d1cd901c691b9ba2e4ed85a5fc7",
+                "sha256:f46ba61c942ee64198444c9562c5cf089eaf97f17b413e15fa1c0614df304734",
+                "sha256:f472da992a6ba58381c0314b994c01d20e522ff8836417ef1c0975bdae142406",
+                "sha256:f58e489e43c9a1688c7d5ceb7455b44952d87f183b7b9c915b301478a2b3bfbe",
+                "sha256:f8576a1dcf445ef064bf8c3b2cdc1d6353e41cb4b366329946883e285dcbcec0",
+                "sha256:fa787b414c4dc6b6cd75338fac18a7dbb53a09443dd863020a2d2bda76940ca6",
+                "sha256:fda423836d7d69cba6a6f99e7a34c2e5fe3621e5e945cd25ea9ba60a96223254"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.1"
+        },
         "pycparser": {
             "hashes": [
                 "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
@@ -1496,27 +1805,44 @@
             ],
             "version": "==2.21"
         },
-        "pygeos": {
-            "hashes": [
-                "sha256:0dbb816ebede8bb87f4e4c87c2303b9ed9e4281f8c4cac2263e7e2ac2271a8dc",
-                "sha256:2761fcbaaa22bb71787bf5c87fd4172c519a8b785859c83e50e52a0558f8e396",
-                "sha256:3bd0e326ab9081038719deaa44f117c8d396f560df371fac160403ebbafa7648",
-                "sha256:440c61514dba22a0775c76ffb8b10cca6ce1cf659f8ff917fa2d434d65770068",
-                "sha256:5513839e85662f2dca01e58dae2e6f0635d7ce5d6818284504b3414d053c2783",
-                "sha256:75bbb025e798146e21d676bbaacc98f6206715d64140bd0d48b383c51d741803",
-                "sha256:79bf18d5e9b3f7b58f7b10d2e6277a5c5002855f42b549d796dca495698e19ed",
-                "sha256:7a7bb256071e78eb7c293ed2455f29f1ca88939a2262b48361617c5945e9d6fb",
-                "sha256:7d0c6c336f835aafc822861c23c59c6b50a53505d0fd6ac5f37cfb7bac0c63c1",
-                "sha256:9170f0b2c2ca5e4f9f73060ec9e390d7494d41513acdbe42a5becf3ba4034104",
-                "sha256:a7124af0af8b53c751204daef7dc22720ca3e1f1465556766598f1d6028e06d5",
-                "sha256:b5d1e7ed1905c2c6a1e54d216b1f9f37cd8c8ed00eeb53cdd9d878375d74009b",
-                "sha256:bc0e707750f813b7460a61e6ce33361e00845fe9ee0ceacbeaa2e892ace68330",
-                "sha256:d2d549f2ffba5ed29cdf499f7e6b390603f964021892c94e567810c5ee3296a1",
-                "sha256:dd39660031fd6b489fde5d46481d927da0228f80a75438a0b27f4049d0fd9ae6",
-                "sha256:ffacef4bb16873dd00fa243b68b8f04817814a247dfc2e04bd75572b3d8b609e"
+        "pycryptodomex": {
+            "hashes": [
+                "sha256:0af93aad8d62e810247beedef0261c148790c52f3cd33643791cc6396dd217c1",
+                "sha256:12056c38e49d972f9c553a3d598425f8a1c1d35b2e4330f89d5ff1ffb70de041",
+                "sha256:23d83b610bd97704f0cd3acc48d99b76a15c8c1540d8665c94d514a49905bad7",
+                "sha256:2d4d395f109faba34067a08de36304e846c791808524614c731431ee048fe70a",
+                "sha256:32e764322e902bbfac49ca1446604d2839381bbbdd5a57920c9daaf2e0b778df",
+                "sha256:3c2516b42437ae6c7a29ef3ddc73c8d4714e7b6df995b76be4695bbe4b3b5cd2",
+                "sha256:40e8a11f578bd0851b02719c862d55d3ee18d906c8b68a9c09f8c564d6bb5b92",
+                "sha256:4b51e826f0a04d832eda0790bbd0665d9bfe73e5a4d8ea93b6a9b38beeebe935",
+                "sha256:4c4674f4b040321055c596aac926d12f7f6859dfe98cd12f4d9453b43ab6adc8",
+                "sha256:55eed98b4150a744920597c81b3965b632038781bab8a08a12ea1d004213c600",
+                "sha256:599bb4ae4bbd614ca05f49bd4e672b7a250b80b13ae1238f05fd0f09d87ed80a",
+                "sha256:5c23482860302d0d9883404eaaa54b0615eefa5274f70529703e2c43cc571827",
+                "sha256:64b876d57cb894b31056ad8dd6a6ae1099b117ae07a3d39707221133490e5715",
+                "sha256:67a3648025e4ddb72d43addab764336ba2e670c8377dba5dd752e42285440d31",
+                "sha256:6feedf4b0e36b395329b4186a805f60f900129cdf0170e120ecabbfcb763995d",
+                "sha256:78f0ddd4adc64baa39b416f3637aaf99f45acb0bcdc16706f0cc7ebfc6f10109",
+                "sha256:7a6651a07f67c28b6e978d63aa3a3fccea0feefed9a8453af3f7421a758461b7",
+                "sha256:7a8dc3ee7a99aae202a4db52de5a08aa4d01831eb403c4d21da04ec2f79810db",
+                "sha256:7cc28dd33f1f3662d6da28ead4f9891035f63f49d30267d3b41194c8778997c8",
+                "sha256:7fa0b52df90343fafe319257b31d909be1d2e8852277fb0376ba89d26d2921db",
+                "sha256:88b0d5bb87eaf2a31e8a759302b89cf30c97f2f8ca7d83b8c9208abe8acb447a",
+                "sha256:a4fa037078e92c7cc49f6789a8bac3de06856740bb2038d05f2d9a2e4b165d59",
+                "sha256:a57e3257bacd719769110f1f70dd901c5b6955e9596ad403af11a3e6e7e3311c",
+                "sha256:ab33c2d9f275e05e235dbca1063753b5346af4a5cac34a51fa0da0d4edfb21d7",
+                "sha256:c84689c73358dfc23f9fdcff2cb9e7856e65e2ce3b5ed8ff630d4c9bdeb1867b",
+                "sha256:c92537b596bd5bffb82f8964cabb9fef1bca8a28a9e0a69ffd3ec92a4a7ad41b",
+                "sha256:caa937ff29d07a665dfcfd7a84f0d4207b2ebf483362fa9054041d67fdfacc20",
+                "sha256:d38ab9e53b1c09608ba2d9b8b888f1e75d6f66e2787e437adb1fecbffec6b112",
+                "sha256:d4cf0128da167562c49b0e034f09e9cedd733997354f2314837c2fa461c87bb1",
+                "sha256:db23d7341e21b273d2440ec6faf6c8b1ca95c8894da612e165be0b89a8688340",
+                "sha256:ee8bf4fdcad7d66beb744957db8717afc12d176e3fd9c5d106835133881a049b",
+                "sha256:f854c8476512cebe6a8681cc4789e4fcff6019c17baa0fd72b459155dc605ab4",
+                "sha256:fd29d35ac80755e5c0a99d96b44fb9abbd7e871849581ea6a4cb826d24267537"
             ],
-            "index": "pypi",
-            "version": "==0.7.1"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==3.17"
         },
         "pygments": {
             "hashes": [
@@ -1534,6 +1860,90 @@
             "markers": "python_full_version >= '3.6.8'",
             "version": "==3.0.9"
         },
+        "pyppmd": {
+            "hashes": [
+                "sha256:024f714ebb8ddf59dae164adc3c220c24555d470f4adb5bd022abc50298cfff3",
+                "sha256:05950d8a39fd9bf6c64572d69a6dd0a1af3fadf8d4a2a0bb62f5b04c0a618300",
+                "sha256:075c9bd297e3b0a87dd7aeabca7fee668218acbe69ecc1c6511064558de8840f",
+                "sha256:07e067e114f05918c8a4ab1fa6a070e2c7a9e497aa73fbf6d87a90e7a6e62a57",
+                "sha256:09668aa43e4f02b8725e6233dfc66e532c72f0e69fa1b34dd814a9f7200e0496",
+                "sha256:0e9c001719527dbafdd7fd8709b98bd63c173451c2eddbaa77abf62486a13da0",
+                "sha256:10c8a41093952cde52b6d89488dc601ee7b10f6c95c430488f68987393777b46",
+                "sha256:12a783a0e3c76484a1bc93783867a36ab9a60de5b5298d57c9fe7348e848346e",
+                "sha256:12be01e919a34c6944568592b35451acf7c98ed18e005bb4b1c046ed520aff7f",
+                "sha256:1658714d012a5f9a8a3e67f3a9ede3519a2558064ccbd3163c39aca0cfd2412b",
+                "sha256:18d7cf4d0a9ced96ff1fa44de9ee3d65f5b06278c8f9a61c3edeb660f12f146b",
+                "sha256:18f863d58c4451e00765137be731c2b2150aff829468f59de4169e052429e1fd",
+                "sha256:1c0fd06aaf782e65b7b5bbc47f8a9dbe050c1ba18474ccbe0a2b37f57a8d8c72",
+                "sha256:1ec00b07c6b68feb402d6596f3575a7892ad69e4f455deee7b5301df703e60dd",
+                "sha256:2858471a291b51fab49242d78bd67c2b7719368618a02e4aa995de8c855da73c",
+                "sha256:2c31e5b331923f3b3b2cfbc66a60ecfd73db1a19a646bd1faf25bfde709a80d0",
+                "sha256:31a09fd1b10518342ff442b57dd8c890b9bfea6bbdbb785c729f0d139092e42e",
+                "sha256:385a0b1341ebdfd7cb101c43eea130546830073c01bdb5036bca45c033ee633e",
+                "sha256:3bc75ed4e969b09fd1a766dd79cb3d5efe56edc11c86ac0b357b5648c7181ce2",
+                "sha256:3ea6a0d26db17027805a804d013cf761d732df5bce9d6b314cd1c727fe347277",
+                "sha256:3ecb83e0cc92960f959111518ea208b51a58e8cc303ff959e9cd2cc56dd36a63",
+                "sha256:47ea218f7dfa94d15286c25d60db3091db1082ba958fa0a32ccaaaeaca7fc712",
+                "sha256:5331a7780d3444d7029e15e68385c94d6a26f688c1e87a5a9ee2e836ea6e4559",
+                "sha256:556b6a3af3fca2b41ca25f51c481e5df8df4da842fc5a567da7bb099cfa52423",
+                "sha256:5805c73590fb8f0ceb3e6cb115774b66a6f4700ae84b31d962ad69667e05dfbd",
+                "sha256:5847c47127ff9ea323f5910c62b9f136c3fab181a5144bfe72be13f051047357",
+                "sha256:5d90d87377d83d909eafbf23301057fe16e6662c98ffea738159a234d9000a68",
+                "sha256:6059ea0c9acc3b52b2961412ac75d1da72656f8b69bb8fc3d92eec6776176011",
+                "sha256:61acfeee5ed59796037499119edd3159bf6b8c5fcaef17e295a2ca4103112d60",
+                "sha256:6279f1c4b6aefacb95df49db2f2e232530592d1849c37b73478a4f26eb405d12",
+                "sha256:62f970173baf80aad9472c7c6edca4a021ae7965174b1c5d6f400c9571e92efc",
+                "sha256:63ddd5a81d6aaed9373cd9fc4de9529f10fa052aaf064ab283dc6218418cc5b5",
+                "sha256:68184b7246ea73a92a764e16cc18b74ccf3c8d6bfc438bbace57aeb1914118a7",
+                "sha256:6a0c524be57698fe61fff893d485a9af21e6bc0aa2d385b71a63ff951921d4b6",
+                "sha256:6e2f5ff5071e4e43c92065f383753d4ad59778816485a01ee7b29e2a1ff48140",
+                "sha256:6f8a3b9192714b3e4773fc49c100ca13defa2502cb38e56205eb5a131ccf555d",
+                "sha256:703c4fbc9b5e1454f403fb1d6b4a6c4c729f72eef14690146deecd2166429d6d",
+                "sha256:706d33cec3601d894f8a4a158bc652b7a3f01cd9e92c2da5d8711efeb9755835",
+                "sha256:71f994f281439705cb04c497adc2863551fa5813606af6fb26c673a44a36c4e3",
+                "sha256:74bd56b165283bb5586ff9ac7a896b217b3c94effe144b768279807840142bb4",
+                "sha256:7ae419f71afa88784d53dd2449882af982bbd0328fa22a7e6a339221f3143918",
+                "sha256:7e8d3c309061ae7fb40e4a26d30f8982b367abc562b9b8621cb79932cb3b94d9",
+                "sha256:7f1e7a1747518b5822eb755f3715d88bd1459e24de828aed86b7c1aa35e3ed76",
+                "sha256:8049c19af4b78b400b2347bff4514763257b55516c359144e9d8091991ed12e8",
+                "sha256:8680008b1b1e9e77f3337a1a53c1b32541cac9f93f79ae12d34de050585999ac",
+                "sha256:8a90b98f9d501eaedaca4d0e82f9e771bd2d780d71effcdeacc9fc6180a00e07",
+                "sha256:8aafe6fc436a782e6d424a0ac00de08a1559b6d6ddd08031adbe791ff4e54c90",
+                "sha256:93d0d6ed97046ce25d64427ec493e06c23f32838972258bf11d603c9c998d6b3",
+                "sha256:a5fbec6f39a307818593508d8623d9328baf494137d191fc98e11f47e058ceee",
+                "sha256:a63adeeb9dc4afd6d377ac1c9801f9539f9a81430e9c96d332023bf2ad6c04a1",
+                "sha256:a7240c00083527cf0b1bbdc92f6967e522efb9ad6968c953be174c390b091b3e",
+                "sha256:a7f83970a057157c88d4a53a40431d07d8d3f38029ad2eae621422f955bd243b",
+                "sha256:aadc63d0ac83f8c5744eb34ea47a70ff7bfab519b293482d7ccb09946c374dc7",
+                "sha256:ab4e29f774e064af09baf8478acd967684524e566b78fcc4f6f669757f0a2ab5",
+                "sha256:ac19ec1b6e3a0aadc1537466f537017189373593e23fe254df050fdd01f4a722",
+                "sha256:ac1aeba466617cf975cd6719070ca9721bcd83a1a84bd8cf74c3a2808724481e",
+                "sha256:aee9c52a6f232f3f7c683b87213aa3a9eacd281ab31187e784290ba1c05024fe",
+                "sha256:b0a87399ade5820f787758449976e758604c7739eb5f79ed9e594b5fa3a6a1bc",
+                "sha256:b6b6c01e46fcf785ad6c272be400ebcbcb434a1d91150614e10de8cc569b8bff",
+                "sha256:b8eee08c615ae9edd7bf1f214a377cac3d27417f22112685e581d4bab43029b0",
+                "sha256:bae08176e0d3ed0a5cbd838ff1ac557dfa088a652af633ab1905ab35bb9d7bc4",
+                "sha256:c60031de93834e5cd262db4b27272101d04a9a18c4cc49f81d483221211a97c8",
+                "sha256:ca6a926d229a6dbf2ccdb0d4e692d81ff927459b59a1cec14ef522522df6d757",
+                "sha256:ca8a842b4ff671642b63ed4edd4e1ff7dc0ba0a7af4135758233f056ab992fca",
+                "sha256:ccdfc8b2a1d73b2186850b9a5bd96106d5fd4419a620d344b0ab8bf630680cf8",
+                "sha256:cd227b8c292ac43d3297a91055fab51c27894dba39d04ccc774a72d9e6f85752",
+                "sha256:cecf0859b461bcf04439f32bcfb6e081016fa6204c92b5950d19d248fd1aad6b",
+                "sha256:cfb716a4a07ccbef84ed9fc31d012cef3b38404a6510e24d307cf64025999b21",
+                "sha256:d2c3c16f644afb1b3caa4f6c717682030f7c3f54a12af8b1416b21877f0b5226",
+                "sha256:d5c6c40f15b9fdea10bf966e5b07ee0a0ebcb8cf188ed9a466029c894816b303",
+                "sha256:e17b08a5c283faf48b4ee888f8fa53f919cd8afd0930eae4d59f719f6be519fb",
+                "sha256:ea4b1a326afe2055304740a03a233f7389f615179b9f6377264b306f619bfb11",
+                "sha256:f488164e8876d213b0627a7a6cb798081eaf84fd9ec6dde5a1668296b15e1a6c",
+                "sha256:f751882dd529328ca43af8018f79cdd02ed707fcda30a2fa9acb1ee5c48261a6",
+                "sha256:f79ebcd7312b541d3520e1a0d4c362731e24403e2f9f6761679b2ad819d5c706",
+                "sha256:f7a1b08612627d5280ef2dad1fadb0b1a10c70df0c484f9091eff5fab5e4c84e",
+                "sha256:f8dbe3076fe20c4d65cb1d1b51eeb17a1c177402b83100017a55daad888e198e",
+                "sha256:f9a3782f5accab4186d68c86defc61fcc7d0146e9cdc5b54e18656852c71db16"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.0"
+        },
         "pyproj": {
             "hashes": [
                 "sha256:04c185102e659439c5bd428ac5473d36ef795fca8e225bbbe78e20643d804ec0",
@@ -1594,11 +2004,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5",
-                "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"
+                "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e",
+                "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"
             ],
             "index": "pypi",
-            "version": "==7.2.1"
+            "version": "==7.2.2"
         },
         "python-dateutil": {
             "hashes": [
@@ -1610,11 +2020,11 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49",
-                "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"
+                "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba",
+                "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"
             ],
             "index": "pypi",
-            "version": "==0.21.1"
+            "version": "==1.0.0"
         },
         "python-json-logger": {
             "hashes": [
@@ -1626,10 +2036,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0",
-                "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"
+                "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588",
+                "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"
             ],
-            "version": "==2022.7.1"
+            "version": "==2023.3"
         },
         "pyyaml": {
             "hashes": [
@@ -1679,102 +2089,207 @@
         },
         "pyzmq": {
             "hashes": [
-                "sha256:00c94fd4c9dd3c95aace0c629a7fa713627a5c80c1819326b642adf6c4b8e2a2",
-                "sha256:01d53958c787cfea34091fcb8ef36003dbb7913b8e9f8f62a0715234ebc98b70",
-                "sha256:0282bba9aee6e0346aa27d6c69b5f7df72b5a964c91958fc9e0c62dcae5fdcdc",
-                "sha256:02f5cb60a7da1edd5591a15efa654ffe2303297a41e1b40c3c8942f8f11fc17c",
-                "sha256:0645b5a2d2a06fd8eb738018490c514907f7488bf9359c6ee9d92f62e844b76f",
-                "sha256:0a154ef810d44f9d28868be04641f837374a64e7449df98d9208e76c260c7ef1",
-                "sha256:0a90b2480a26aef7c13cff18703ba8d68e181facb40f78873df79e6d42c1facc",
-                "sha256:0e8d00228db627ddd1b418c7afd81820b38575f237128c9650365f2dd6ac3443",
-                "sha256:17e1cb97d573ea84d7cd97188b42ca6f611ab3ee600f6a75041294ede58e3d20",
-                "sha256:183e18742be3621acf8908903f689ec520aee3f08449bfd29f583010ca33022b",
-                "sha256:1f6116991568aac48b94d6d8aaed6157d407942ea385335a6ed313692777fb9d",
-                "sha256:20638121b0bdc80777ce0ec8c1f14f1ffec0697a1f88f0b564fa4a23078791c4",
-                "sha256:2754fa68da08a854f4816e05160137fa938a2347276471103d31e04bcee5365c",
-                "sha256:28bcb2e66224a7ac2843eb632e4109d6b161479e7a2baf24e37210461485b4f1",
-                "sha256:293a7c2128690f496057f1f1eb6074f8746058d13588389981089ec45d8fdc77",
-                "sha256:2a73af6504e0d2805e926abf136ebf536735a13c22f709be7113c2ec65b4bec3",
-                "sha256:2d05d904f03ddf1e0d83d97341354dfe52244a619b5a1440a5f47a5b3451e84e",
-                "sha256:2e7b87638ee30ab13230e37ce5331b3e730b1e0dda30120b9eeec3540ed292c8",
-                "sha256:3100dddcada66ec5940ed6391ebf9d003cc3ede3d320748b2737553019f58230",
-                "sha256:31e523d067ce44a04e876bed3ff9ea1ff8d1b6636d16e5fcace9d22f8c564369",
-                "sha256:3594c0ff604e685d7e907860b61d0e10e46c74a9ffca168f6e9e50ea934ee440",
-                "sha256:3670e8c5644768f214a3b598fe46378a4a6f096d5fb82a67dfd3440028460565",
-                "sha256:4046d03100aca266e70d54a35694cb35d6654cfbef633e848b3c4a8d64b9d187",
-                "sha256:4725412e27612f0d7d7c2f794d89807ad0227c2fc01dd6146b39ada49c748ef9",
-                "sha256:484c2c4ee02c1edc07039f42130bd16e804b1fe81c4f428e0042e03967f40c20",
-                "sha256:487305c2a011fdcf3db1f24e8814bb76d23bc4d2f46e145bc80316a59a9aa07d",
-                "sha256:4a1bc30f0c18444d51e9b0d0dd39e3a4e7c53ee74190bebef238cd58de577ea9",
-                "sha256:4c25c95416133942280faaf068d0fddfd642b927fb28aaf4ab201a738e597c1e",
-                "sha256:4cbb885f347eba7ab7681c450dee5b14aed9f153eec224ec0c3f299273d9241f",
-                "sha256:4d3d604fe0a67afd1aff906e54da557a5203368a99dcc50a70eef374f1d2abef",
-                "sha256:4e295f7928a31ae0f657e848c5045ba6d693fe8921205f408ca3804b1b236968",
-                "sha256:5049e75cc99db65754a3da5f079230fb8889230cf09462ec972d884d1704a3ed",
-                "sha256:5050f5c50b58a6e38ccaf9263a356f74ef1040f5ca4030225d1cb1a858c5b7b6",
-                "sha256:526f884a27e8bba62fe1f4e07c62be2cfe492b6d432a8fdc4210397f8cf15331",
-                "sha256:531866c491aee5a1e967c286cfa470dffac1e2a203b1afda52d62b58782651e9",
-                "sha256:5605621f2181f20b71f13f698944deb26a0a71af4aaf435b34dd90146092d530",
-                "sha256:58fc3ad5e1cfd2e6d24741fbb1e216b388115d31b0ca6670f894187f280b6ba6",
-                "sha256:60ecbfe7669d3808ffa8a7dd1487d6eb8a4015b07235e3b723d4b2a2d4de7203",
-                "sha256:610d2d112acd4e5501fac31010064a6c6efd716ceb968e443cae0059eb7b86de",
-                "sha256:6136bfb0e5a9cf8c60c6ac763eb21f82940a77e6758ea53516c8c7074f4ff948",
-                "sha256:62b9e80890c0d2408eb42d5d7e1fc62a5ce71be3288684788f74cf3e59ffd6e2",
-                "sha256:656281d496aaf9ca4fd4cea84e6d893e3361057c4707bd38618f7e811759103c",
-                "sha256:66509c48f7446b640eeae24b60c9c1461799a27b1b0754e438582e36b5af3315",
-                "sha256:6bf3842af37af43fa953e96074ebbb5315f6a297198f805d019d788a1021dbc8",
-                "sha256:731b208bc9412deeb553c9519dca47136b5a01ca66667cafd8733211941b17e4",
-                "sha256:75243e422e85a62f0ab7953dc315452a56b2c6a7e7d1a3c3109ac3cc57ed6b47",
-                "sha256:7877264aa851c19404b1bb9dbe6eed21ea0c13698be1eda3784aab3036d1c861",
-                "sha256:81f99fb1224d36eb91557afec8cdc2264e856f3464500b55749020ce4c848ef2",
-                "sha256:8539216173135e9e89f6b1cc392e74e6b935b91e8c76106cf50e7a02ab02efe5",
-                "sha256:85456f0d8f3268eecd63dede3b99d5bd8d3b306310c37d4c15141111d22baeaf",
-                "sha256:866eabf7c1315ef2e93e34230db7cbf672e0d7c626b37c11f7e870c8612c3dcc",
-                "sha256:926236ca003aec70574754f39703528947211a406f5c6c8b3e50eca04a9e87fc",
-                "sha256:930e6ad4f2eaac31a3d0c2130619d25db754b267487ebc186c6ad18af2a74018",
-                "sha256:94f0a7289d0f5c80807c37ebb404205e7deb737e8763eb176f4770839ee2a287",
-                "sha256:9a2d5e419bd39a1edb6cdd326d831f0120ddb9b1ff397e7d73541bf393294973",
-                "sha256:9ca6db34b26c4d3e9b0728841ec9aa39484eee272caa97972ec8c8e231b20c7e",
-                "sha256:9f72ea279b2941a5203e935a4588b9ba8a48aeb9a926d9dfa1986278bd362cb8",
-                "sha256:a0e7ef9ac807db50b4eb6f534c5dcc22f998f5dae920cc28873d2c1d080a4fc9",
-                "sha256:a1cd4a95f176cdc0ee0a82d49d5830f13ae6015d89decbf834c273bc33eeb3d3",
-                "sha256:a9c464cc508177c09a5a6122b67f978f20e2954a21362bf095a0da4647e3e908",
-                "sha256:ac97e7d647d5519bcef48dd8d3d331f72975afa5c4496c95f6e854686f45e2d9",
-                "sha256:af1fbfb7ad6ac0009ccee33c90a1d303431c7fb594335eb97760988727a37577",
-                "sha256:b055a1cddf8035966ad13aa51edae5dc8f1bba0b5d5e06f7a843d8b83dc9b66b",
-                "sha256:b6f75b4b8574f3a8a0d6b4b52606fc75b82cb4391471be48ab0b8677c82f9ed4",
-                "sha256:b90bb8dfbbd138558f1f284fecfe328f7653616ff9a972433a00711d9475d1a9",
-                "sha256:be05504af0619d1cffa500af1e0ede69fb683f301003851f5993b5247cc2c576",
-                "sha256:c21a5f4e54a807df5afdef52b6d24ec1580153a6bcf0607f70a6e1d9fa74c5c3",
-                "sha256:c48f257da280b3be6c94e05bd575eddb1373419dbb1a72c3ce64e88f29d1cd6d",
-                "sha256:cac602e02341eaaf4edfd3e29bd3fdef672e61d4e6dfe5c1d065172aee00acee",
-                "sha256:ccb3e1a863222afdbda42b7ca8ac8569959593d7abd44f5a709177d6fa27d266",
-                "sha256:e1081d7030a1229c8ff90120346fb7599b54f552e98fcea5170544e7c6725aab",
-                "sha256:e14df47c1265356715d3d66e90282a645ebc077b70b3806cf47efcb7d1d630cb",
-                "sha256:e4bba04ea779a3d7ef25a821bb63fd0939142c88e7813e5bd9c6265a20c523a2",
-                "sha256:e99629a976809fe102ef73e856cf4b2660acd82a412a51e80ba2215e523dfd0a",
-                "sha256:f330a1a2c7f89fd4b0aa4dcb7bf50243bf1c8da9a2f1efc31daf57a2046b31f2",
-                "sha256:f3f96d452e9580cb961ece2e5a788e64abaecb1232a80e61deffb28e105ff84a",
-                "sha256:fc7c1421c5b1c916acf3128bf3cc7ea7f5018b58c69a6866d70c14190e600ce9"
+                "sha256:032f5c8483c85bf9c9ca0593a11c7c749d734ce68d435e38c3f72e759b98b3c9",
+                "sha256:08bfcc21b5997a9be4fefa405341320d8e7f19b4d684fb9c0580255c5bd6d695",
+                "sha256:1a843d26a8da1b752c74bc019c7b20e6791ee813cd6877449e6a1415589d22ff",
+                "sha256:1f124cb73f1aa6654d31b183810febc8505fd0c597afa127c4f40076be4574e0",
+                "sha256:1f82906a2d8e4ee310f30487b165e7cc8ed09c009e4502da67178b03083c4ce0",
+                "sha256:21ec0bf4831988af43c8d66ba3ccd81af2c5e793e1bf6790eb2d50e27b3c570a",
+                "sha256:24683285cc6b7bf18ad37d75b9db0e0fefe58404e7001f1d82bf9e721806daa7",
+                "sha256:24abbfdbb75ac5039205e72d6c75f10fc39d925f2df8ff21ebc74179488ebfca",
+                "sha256:25e6873a70ad5aa31e4a7c41e5e8c709296edef4a92313e1cd5fc87bbd1874e2",
+                "sha256:269968f2a76c0513490aeb3ba0dc3c77b7c7a11daa894f9d1da88d4a0db09835",
+                "sha256:26b0358e8933990502f4513c991c9935b6c06af01787a36d133b7c39b1df37fa",
+                "sha256:28fdb9224a258134784a9cf009b59265a9dde79582fb750d4e88a6bcbc6fa3dc",
+                "sha256:2b9c9cc965cdf28381e36da525dcb89fc1571d9c54800fdcd73e3f73a2fc29bd",
+                "sha256:2da6813b7995b6b1d1307329c73d3e3be2fd2d78e19acfc4eff2e27262732388",
+                "sha256:3059a6a534c910e1d5d068df42f60d434f79e6cc6285aa469b384fa921f78cf8",
+                "sha256:312b3f0f066b4f1d17383aae509bacf833ccaf591184a1f3c7a1661c085063ae",
+                "sha256:34a6fddd159ff38aa9497b2e342a559f142ab365576284bc8f77cb3ead1f79c5",
+                "sha256:374b55516393bfd4d7a7daa6c3b36d6dd6a31ff9d2adad0838cd6a203125e714",
+                "sha256:38d9f78d69bcdeec0c11e0feb3bc70f36f9b8c44fc06e5d06d91dc0a21b453c7",
+                "sha256:4a31992a8f8d51663ebf79df0df6a04ffb905063083d682d4380ab8d2c67257c",
+                "sha256:4a4b4261eb8f9ed71f63b9eb0198dd7c934aa3b3972dac586d0ef502ba9ab08b",
+                "sha256:510d8e55b3a7cd13f8d3e9121edf0a8730b87d925d25298bace29a7e7bc82810",
+                "sha256:531e36d9fcd66f18de27434a25b51d137eb546931033f392e85674c7a7cea853",
+                "sha256:54a96cf77684a3a537b76acfa7237b1e79a8f8d14e7f00e0171a94b346c5293e",
+                "sha256:56a94ab1d12af982b55ca96c6853db6ac85505e820d9458ac76364c1998972f4",
+                "sha256:5c5fbb229e40a89a2fe73d0c1181916f31e30f253cb2d6d91bea7927c2e18413",
+                "sha256:5d496815074e3e3d183fe2c7fcea2109ad67b74084c254481f87b64e04e9a471",
+                "sha256:5eaeae038c68748082137d6896d5c4db7927e9349237ded08ee1bbd94f7361c9",
+                "sha256:62ec8d979f56c0053a92b2b6a10ff54b9ec8a4f187db2b6ec31ee3dd6d3ca6e2",
+                "sha256:64812f29d6eee565e129ca14b0c785744bfff679a4727137484101b34602d1a7",
+                "sha256:6526d097b75192f228c09d48420854d53dfbc7abbb41b0e26f363ccb26fbc177",
+                "sha256:659e62e1cbb063151c52f5b01a38e1df6b54feccfa3e2509d44c35ca6d7962ee",
+                "sha256:65c19a63b4a83ae45d62178b70223adeee5f12f3032726b897431b6553aa25af",
+                "sha256:67da1c213fbd208906ab3470cfff1ee0048838365135a9bddc7b40b11e6d6c89",
+                "sha256:6a821a506822fac55d2df2085a52530f68ab15ceed12d63539adc32bd4410f6e",
+                "sha256:6a979e59d2184a0c8f2ede4b0810cbdd86b64d99d9cc8a023929e40dce7c86cc",
+                "sha256:6b8c1bbb70e868dc88801aa532cae6bd4e3b5233784692b786f17ad2962e5149",
+                "sha256:6fadc60970714d86eff27821f8fb01f8328dd36bebd496b0564a500fe4a9e354",
+                "sha256:715cff7644a80a7795953c11b067a75f16eb9fc695a5a53316891ebee7f3c9d5",
+                "sha256:77942243ff4d14d90c11b2afd8ee6c039b45a0be4e53fb6fa7f5e4fd0b59da39",
+                "sha256:7b504ae43d37e282301da586529e2ded8b36d4ee2cd5e6db4386724ddeaa6bbc",
+                "sha256:827bf60e749e78acb408a6c5af6688efbc9993e44ecc792b036ec2f4b4acf485",
+                "sha256:8280ada89010735a12b968ec3ea9a468ac2e04fddcc1cede59cb7f5178783b9c",
+                "sha256:83d822e8687621bed87404afc1c03d83fa2ce39733d54c2fd52d8829edb8a7ff",
+                "sha256:8560756318ec7c4c49d2c341012167e704b5a46d9034905853c3d1ade4f55bee",
+                "sha256:85762712b74c7bd18e340c3639d1bf2f23735a998d63f46bb6584d904b5e401d",
+                "sha256:88649b19ede1cab03b96b66c364cbbf17c953615cdbc844f7f6e5f14c5e5261c",
+                "sha256:9a2e5fe42dfe6b73ca120b97ac9f34bfa8414feb15e00e37415dbd51cf227ef6",
+                "sha256:9af0bb0277e92f41af35e991c242c9c71920169d6aa53ade7e444f338f4c8128",
+                "sha256:9bdc40efb679b9dcc39c06d25629e55581e4c4f7870a5e88db4f1c51ce25e20d",
+                "sha256:9e1d2f2d86fc75ed7f8845a992c5f6f1ab5db99747fb0d78b5e4046d041164d2",
+                "sha256:a2e92ff20ad5d13266bc999a29ed29a3b5b101c21fdf4b2cf420c09db9fb690e",
+                "sha256:a35960c8b2f63e4ef67fd6731851030df68e4b617a6715dd11b4b10312d19fef",
+                "sha256:a6f6ae12478fdc26a6d5fdb21f806b08fa5403cd02fd312e4cb5f72df078f96f",
+                "sha256:a9b5eeb5278a8a636bb0abdd9ff5076bcbb836cd2302565df53ff1fa7d106d54",
+                "sha256:ab046e9cb902d1f62c9cc0eca055b1d11108bdc271caf7c2171487298f229b56",
+                "sha256:ab2c056ac503f25a63f6c8c6771373e2a711b98b304614151dfb552d3d6c81f6",
+                "sha256:abbce982a17c88d2312ec2cf7673985d444f1beaac6e8189424e0a0e0448dbb3",
+                "sha256:ac178e666c097c8d3deb5097b58cd1316092fc43e8ef5b5fdb259b51da7e7315",
+                "sha256:ad761cfbe477236802a7ab2c080d268c95e784fe30cafa7e055aacd1ca877eb0",
+                "sha256:affec1470351178e892121b3414c8ef7803269f207bf9bef85f9a6dd11cde264",
+                "sha256:b164cc3c8acb3d102e311f2eb6f3c305865ecb377e56adc015cb51f721f1dda6",
+                "sha256:b48616a09d7df9dbae2f45a0256eee7b794b903ddc6d8657a9948669b345f220",
+                "sha256:b491998ef886662c1f3d49ea2198055a9a536ddf7430b051b21054f2a5831800",
+                "sha256:b733076ff46e7db5504c5e7284f04a9852c63214c74688bdb6135808531755a3",
+                "sha256:c8fedc3ccd62c6b77dfe6f43802057a803a411ee96f14e946f4a76ec4ed0e117",
+                "sha256:cb1f69a0a2a2b1aae8412979dd6293cc6bcddd4439bf07e4758d864ddb112354",
+                "sha256:cca8524b61c0eaaa3505382dc9b9a3bc8165f1d6c010fdd1452c224225a26689",
+                "sha256:cfb9f7eae02d3ac42fbedad30006b7407c984a0eb4189a1322241a20944d61e5",
+                "sha256:d4427b4a136e3b7f85516c76dd2e0756c22eec4026afb76ca1397152b0ca8145",
+                "sha256:d488c5c8630f7e782e800869f82744c3aca4aca62c63232e5d8c490d3d66956a",
+                "sha256:dd771a440effa1c36d3523bc6ba4e54ff5d2e54b4adcc1e060d8f3ca3721d228",
+                "sha256:ed15e3a2c3c2398e6ae5ce86d6a31b452dfd6ad4cd5d312596b30929c4b6e182",
+                "sha256:edbbf06cc2719889470a8d2bf5072bb00f423e12de0eb9ffec946c2c9748e149",
+                "sha256:eef2a0b880ab40aca5a878933376cb6c1ec483fba72f7f34e015c0f675c90b20",
+                "sha256:f7c8b8368e84381ae7c57f1f5283b029c888504aaf4949c32e6e6fb256ec9bf0",
+                "sha256:ffc71111433bd6ec8607a37b9211f4ef42e3d3b271c6d76c813669834764b248"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==25.0.0"
+            "version": "==25.0.2"
+        },
+        "pyzstd": {
+            "hashes": [
+                "sha256:00c9e9aa800b7ded58bdddd90930f949205475c17d1e5923ab936321b805ede0",
+                "sha256:01e723cafb6e681dccba134524f9f012c5f04078d6cb05e03b2f5d871716e0db",
+                "sha256:0820d84c272865e996696fc84c8c63cf6898d66bfda8ac4109880e24272ac13a",
+                "sha256:0d742edd7340e4b8ba4cbe240e873bbc88c197c0b3838212eb5c4885094f191b",
+                "sha256:15b84a04edb257cc7d75c208be43dc4c66e8bd2a44ad9e2af523a829386f0e4c",
+                "sha256:1659da1d388a85c8297083772fdd16543a43305ec0213f1d6f5a6b65702c8402",
+                "sha256:16eb2b2b206489bf8a7b690cf3b06b24352a16ee25717cb38c4381aefb3a94ce",
+                "sha256:1e138c585fc2d07fb20e61f6cbac77fdc77497299199a610a1fca28933128948",
+                "sha256:1e53d12e026605b254569a384191850b6e9f2fcd0fba8b3c80e09b9683fc86af",
+                "sha256:2146d6ab27f3b5dd02b093d748151897e3c929ec009768f20b7a5f3627de7d9c",
+                "sha256:21b7f72717d58b39487c903854556fc363be3329b94aa696e9111291a13d714c",
+                "sha256:229626e1ee592fd8a922935b997e9b91b8b8b874a72565fe1839041cf3ae1a54",
+                "sha256:23c07257fa5df97ebde0a01a54e7c4ae295dc684908f8b1c4ac2f407b881d91b",
+                "sha256:242467cc26e2e4b344028887c934fce3e48354e5eddbc4bb4e695f2c011d3755",
+                "sha256:2589c484c59a0ef2d52d1fd7ab3efd648ef38a396ef22b892c79c2cb878487e3",
+                "sha256:27e1b4a3a6fdab664f1d415c3bbc63700a85f8ec46ad2bf7287e6a86bbea0581",
+                "sha256:2988e5282c807634911d9336ff4797de84e308d94a1f57bfab4223d927e992c5",
+                "sha256:29f692a74f3d9778c71ad6899a1c7578fd44dfb5b6187fe37abdfc7cc034e9d3",
+                "sha256:2e25838c7410245201a455ff000372c65654c781a7a168b2249432dae4ce260f",
+                "sha256:33d42c56fde20a2a98bd200d91d09e44957ce782ffeef5041fc4c544df7640d1",
+                "sha256:39fa99cac5abe0cdb34afda54f54735593646d8c57de1b734d1fe3c9761e0575",
+                "sha256:3b0d384a48592683edf812c1a3d75a0cf5af253b1629596567ce3a2991d6ccbc",
+                "sha256:448d4412462d00bbb7c4857f9a4bedc879f9834f01e2723c23f9de4edde8bde3",
+                "sha256:452e1e774fd7e693f2f5065de61f8cf42eb449ea5614e4cd15ae458e70977d98",
+                "sha256:46e95ec6b499571eca1683c078d2f6cfe3815a5103bf4f488a4edbb1607be652",
+                "sha256:4b41ca76a847286cd54ebabbfd37fa732a68dc2daf5178aede34ee377598cfa5",
+                "sha256:4bf88010b15470f00810e8fc3cb46608af7dceb0d8ca696552d635be652bdfd7",
+                "sha256:4e18f986325eedf0b5d86feb87fad70a0430e3da0bff1a1c33ddce8c70fc371e",
+                "sha256:4f4a96bdffa773f8820e065ec840c5d3f252b0611951b73567121757388f171c",
+                "sha256:55a368a2195bf9cabf8f20ea2841c6a7463c21d8345c8c7d50b809635fec262e",
+                "sha256:55dd0cf038e9053147e58fd4d0509fc1b735f63ce276b88151560bae271a25af",
+                "sha256:56cb56e24e9d33ef571c23b0157593c44c052ffe5810599a76bc6c60ec77f67a",
+                "sha256:5d774cf6406c33c9af6e1fde10a1b0aab7e1f573b2510c491ed24c6f4dbad677",
+                "sha256:5ec07186d5150a6129327e906c0c1f15fd505d4f5538957b2481913a959121ee",
+                "sha256:60079134dd0967fc4c81ea50a7deedd99c52bb263a46750326cba9f53b03f93c",
+                "sha256:649f23ab5c203dade1594f593c6b65247e848f99ba5b11ef26b74541e1f0b605",
+                "sha256:65dc2ac46389823214dd21141a05009cc2fc8d139d8a0aae5f7eb6830ffe5ded",
+                "sha256:67ee34825be593151ba0da78d6914914afce9a7ddc77683319d9406bf422c578",
+                "sha256:6be764774531007ce83c32d24379c32474329655bcc177f1b154e2d2e7e4dba1",
+                "sha256:7048a34197205cb4459492a1d397059ec35a758e29cee226ffa8e593148300af",
+                "sha256:72bf7435dad00fe8a24815f67fcd9fddf44265aa8155a507ac79472cd13c97c1",
+                "sha256:7422e119a9dd10695dee636707881fa6e1af872df5a4f5f1ae6feec34e3cdab6",
+                "sha256:74750bee4fd2b584bf502cdc03861c9decb0875a7febcb2d629cf72a0470188d",
+                "sha256:7740d39f74f9154a7f42d15c1b16f87ba062ac16439d62525f2c85165ea9ebc2",
+                "sha256:78a5e30f90f499f6961e56263e8f029e228b92550aa4e8a5649ed220f8f19230",
+                "sha256:7aea5a1474cacdd0285f16bc5392271645147806986e17be8b100ef750520548",
+                "sha256:7da76175bbbec45ecd35e611eaa50e40c0f29a05e86a8cd50b1b20b5a52cd218",
+                "sha256:821555ce9bc7f50130bf52dc7b7d342913e00435c68a5cac564c5f2e57f33cbf",
+                "sha256:8576440b3212d648fbff49d1127e2bffe0f74aa8be40c3ec8b783f276135cb30",
+                "sha256:88e9f435a68ab9d04a4a90d5b11c587080295ab4057a403730990944b98d0b4e",
+                "sha256:8a7f8be2493356bf55a1261b7a9bb476324738ced6210c3f876f452df8288785",
+                "sha256:912345726aa45dba8fb4930e172149f558caf606bf22c219cde9deb984a1523a",
+                "sha256:91cbd6d8a2fa0b18f9b50076a46e837b90e9f1974578483fcc4b094b43cbd89b",
+                "sha256:929f894d0b1439826f45a248d7569f8c9ceb43ff3d5739e7ac9ff264acdcb070",
+                "sha256:94302b3a36c562a31f791a5e0112bcc129e0245c6f462d86627a4cefda0f5764",
+                "sha256:9860aeb47f88e61f8bd615f4f3e8372990c8e680c52d8a2f893157ebdb15e8a2",
+                "sha256:9aed2fda29875d42a7f96b741582e6e9adb48f3903760439f60b0f8fb8f822f1",
+                "sha256:9b433e0495b0231c20733b4649d33c5ab51f3670db4b47e0964b0b4dd240e03f",
+                "sha256:9c681938be80e4ceedc2f43b5d7c08e8e3719492a8250cc742382633c7176919",
+                "sha256:9db032965b8fabf0bac13b75706031444ebfa7214794afe80be41799d8a349fb",
+                "sha256:a3e11b0534f958668dbcac6d3d943af93d1f5c4dfa7857147589398b405a9ee6",
+                "sha256:a6250aa075408753cde58d79a80923c29c1791f32c4b3e58f25701ae5dbf1678",
+                "sha256:aa9ab4ca7e32c219ad0d420786897cdb3c09f1ffc3fd1a2785e7652ace5684e2",
+                "sha256:ab21cc20922de335feef32245a9329bc3cb443346dcda243a711aab15bc130cc",
+                "sha256:ab55575f1c6c12ca65918dba8196bfb281df86038a9f9e376fb6ac5a501ac724",
+                "sha256:ab88c8123e2c8ad298d1ac79f5b5aeab68ef27a0afe968e5c5be1a2f24577e2e",
+                "sha256:b06eabacbefaf440785ee1f499582c79083822fb3d2640fb74dd5d3391bbc2ae",
+                "sha256:b28e6f095fd56ac373844ee76ab74b011dbd128e4113a033911675421742ce91",
+                "sha256:b4833842d5e6660c1b6cd316f102ffddb95566f12ec1f89c5630d4b511e7e084",
+                "sha256:b5f921cf4ba10cf2f1dbaaba3b9e7afe876a0962cf43f16455c4e5052c1dc0bd",
+                "sha256:b73b18d8ebf5caba863bfccb6ad422b9322c7a806b50a555bb44f51f16b7af5e",
+                "sha256:bca4d7285363b73966e8c6d8af8da515d23bcaf8eb91293e866d5d4615ea5227",
+                "sha256:be4e4bc65de7093e4781d98016b15224311f52eacb220f67b1cbffcae3ac5497",
+                "sha256:be71183e8e7a03b298d4e547f58f68ee1664889bef4e7460464fdf1e70349b66",
+                "sha256:bf63ad569ebc08fee48bc1080c1eb98c9620af4d52ecc56b2a54d6f01797d7be",
+                "sha256:c03e1bfc6c391a05ea04dfff485f4884b07805a16f198c1d5b4e24f4df1e1723",
+                "sha256:c173226b2fab75bfcf9c0315d551e9b484e50f4d84869676bcf33d080f5b43f8",
+                "sha256:c2c6f976e45e63a4f6cbaf20240bb0842ed2ed65703718d54a6fad8642d3c230",
+                "sha256:c2e35d5a169ef9107760bd276974331e0be8fafe2e26ec824020c4999013dc40",
+                "sha256:ca8f520618a3a1c66ee0f6bc3a6b4fc6f9d55b7b1ce80a8f8a422c82a2f79418",
+                "sha256:cb03d3fca1fab2b33b4d0de9b80ab0efc9bb872eb28575eeb73af0d9aac08322",
+                "sha256:d1bcfb77bf00ab5e134321ddf9123845c80b1f49500d499998a17881aa644e28",
+                "sha256:d1da1bcbac16a075675609865e2705d9ec90a2312450d4d99b65a003fdef84f7",
+                "sha256:d62f3e5ca97150ddd32e83275170bcb7ea462f271df03d39de434e8e02889b09",
+                "sha256:d6f2bbd863e674875882de8dd6f54ba98916cfeb89d9224393e9846c9e9aea1d",
+                "sha256:d9069da3bb94cac8af3fd2d7b622c7a78a248d3295a478587d880553287d56e6",
+                "sha256:ddfbaad47edf70932a6e0fba37348ca240a34408e674c056ff16631fa07ea5fd",
+                "sha256:de07ac54f57642f186732075cdce2be3d4a30228c3b17a6d8c6053765dc6eec8",
+                "sha256:df80324759cb74a10818f5ea4dfb0332b6064a31305246f66b043c4f73b37918",
+                "sha256:e111d203153837fca01a3c6ed48167af4b590f148998a2d8edefd98338615006",
+                "sha256:e1e7e43714b0646fb1d61b6b9bbf6e6ede6c77c89667f2addb086f75196ce389",
+                "sha256:e3b4ee242e00029c68fc5d8cddc3e01f6cdf7ac466d844a1440ebde4ae1f9e79",
+                "sha256:e7361f4e5a354715d00c43ad3472188cb9eab83d2e75a2ab7ca75734779579bf",
+                "sha256:e884faae04b2f52cc6aa5aa18de2631ab75786440fd8a708f6b886d75aade937",
+                "sha256:e9074eff37261effb5c7312911e11f0a4c26475ab55bae803d3b095fa291e631",
+                "sha256:e96731ecfff34aeefb5e0f93222c9831880e8a94ea700f4432d4ee45995badc1",
+                "sha256:eb300366e9bf62776ef8c5c5fca57d5b18fdf6a62d055244e9f9f71ea263597e",
+                "sha256:f437e316cd24f21ea13ad635d7e24550458217af75182554963f461bae0bc189",
+                "sha256:f4d47ea4a9a3a43a556da63ee59bf941aecfb529bb85957960e91339c6d6cf3d"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==0.15.4"
         },
         "qtconsole": {
             "hashes": [
-                "sha256:57748ea2fd26320a0b77adba20131cfbb13818c7c96d83fafcb110ff55f58b35",
-                "sha256:be13560c19bdb3b54ed9741a915aa701a68d424519e8341ac479a91209e694b2"
+                "sha256:bae8c7e10170cdcdcaf7e6d53ad7d6a7412249b9b8310a0eaa6b6f3b260f32db",
+                "sha256:f67a03f40f722e13261791280f73068dbaf9dafcc335cbba644ccc8f892640e5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.0"
+            "version": "==5.4.1"
         },
         "qtpy": {
             "hashes": [
-                "sha256:0603c9c83ccc035a4717a12908bf6bc6cb22509827ea2ec0e94c2da7c9ed57c5",
-                "sha256:8d6d544fc20facd27360ea189592e6135c614785f0dec0b4f083289de6beb408"
+                "sha256:5193d20e0b16e4d9d3bc2c642d04d9f4e2c892590bd1b9c92bfe38a95d5a2e12",
+                "sha256:a8c74982d6d172ce124d80cafd39653df78989683f760f2281ba91a6e7b9de8b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.3.0"
+            "version": "==2.3.1"
         },
         "rasterio": {
             "hashes": [
@@ -1794,18 +2309,18 @@
         },
         "rasterstats": {
             "hashes": [
-                "sha256:3b9c8de46125ac262187099962fa1b987797fca4e34180aabf8a688a6aa15044",
-                "sha256:d554fbc7e749e5649dfcdbd7ea8e21c7388437bc66ef13fde9ef3662a258b277"
+                "sha256:00014f422160f6047d3431c9132ebe3d356021ca8e22c6f40d04764f98a60795",
+                "sha256:29389bfcbeac1a4206aba6e1d795058ec8a64d560efad48156c27fad97c2e09a"
             ],
             "index": "pypi",
-            "version": "==0.15.0"
+            "version": "==0.18.0"
         },
         "requests": {
             "hashes": [
                 "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa",
                 "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "markers": "python_version >= '3.7' and python_version < '4'",
             "version": "==2.28.2"
         },
         "rfc3339-validator": {
@@ -1843,6 +2358,57 @@
             "index": "pypi",
             "version": "==0.3.4"
         },
+        "rtree": {
+            "hashes": [
+                "sha256:004e131b570dc360a49e7f3b60e7bc6517943a54df056587964d1cb903889e7e",
+                "sha256:015df09e1bc55ddf7c88799bf1515d058cd0ee78eacf4cd443a32876d3b3a863",
+                "sha256:0d68a81ad419d5c2ea5fecc677e6c178666c057e2c7b24100a6c48392196f1e9",
+                "sha256:11d16f51cf9205cd6995af36e24efe8f184270f667fb49bb69b09fc46b97e7d4",
+                "sha256:157207191aebdacbbdbb369e698cfbfebce53bc97114e96c8af5bed3126475f1",
+                "sha256:16900ee02cf5c198a42b03635268a80f606aa102f3f7618b89f75023d406da1c",
+                "sha256:18ce7e4d04b85c48f2d364835620b3b20e38e199639746e7b12f07a2303e18ff",
+                "sha256:1a213e5d385278ca7668bc5b27083f8d6e39996a9bd59b6528f3a30009dae4ed",
+                "sha256:1a94e2f4bf74bd202ea8b67ea3d7c71e763ad41f79be1d6b72aa2c8d5a8e92c4",
+                "sha256:1e894112cef4de6c518bdea0b43eada65f12888c3645cc437c3a677aa023039f",
+                "sha256:222121699c303a64065d849bf7038b1ecabc37b65c7fa340bedb38ef0e805429",
+                "sha256:273ee61783de3a1664e5f868feebf5eea4629447137751bfa4087b0f82093082",
+                "sha256:296203e933b6ec0dd07f6a7456c4f1492def95b6993f20cc61c92b0fee0aecc5",
+                "sha256:2ee7165e9872a026ccb868c021711eba39cedf7d1820763c9de52d5324691a92",
+                "sha256:3573cbb0de872f54d0a0c29596a84e8ac3939c47ca3bece4a82e92775730a0d0",
+                "sha256:50b658a6707f215a0056d52e9f83a97148c0af62dea07cf29b3789a2c429e78a",
+                "sha256:57128293dd625cb1f07726f32208097953e8854d70ab1fc55d6858733618b9ed",
+                "sha256:582854252b8fd5c8472478af060635434931fb55edd269bac128cbf2eef43620",
+                "sha256:5b20f69e040a05503b22297af223f336fe7047909b57e4b207b98292f33a229f",
+                "sha256:62f38020af47b765adc6b0bc7c4e810c6c3d1eab44ba339b592ff25a4c0dc0a7",
+                "sha256:656b148589c0b5bab4a7db4d033634329f42a5feaac10ca40aceeca109d83c1f",
+                "sha256:6792de0e3c2fd3ad7e069445027603bec7a47000432f49c80246886311f4f152",
+                "sha256:698de8ce6c62e159d93b35bacf64bcf3619077b5367bc88cd2cff5e0bc36169b",
+                "sha256:6ce4a6fdb63254a4c1efebe7a4f7a59b1c333c703bde4ae715d9ad88c833e10b",
+                "sha256:6db6a0a93e41594ffc14b053f386dd414ab5a82535bbd9aedafa6ac8dc0650d8",
+                "sha256:77908cd7acdd519a731979ebf5baff8afd102109c2f52864c1e6ee75d3ea2d87",
+                "sha256:784efa6b7be9e99b33613ae8495931032689441eabb6120c9b3eb91188c33794",
+                "sha256:7b2c15f9373ba314c83a8df5cb6d99b4e3af23c376c6b1317add995432dd0970",
+                "sha256:7e3d5f0e7b28250afbb290ab88b49aa0f121c9714d0da2080581783690347507",
+                "sha256:8de99f28af0f1783eefb80918959903b4b18112f6a12b48f296ecb162804e69d",
+                "sha256:93c5e0bf31e76b4f92a6eec3d2891e938408774c75a8ed6ac3d2c8db04a2be33",
+                "sha256:9855b8f11cdad99c56eb361b7b632a4fbd3d8cbe3f2081426b445f0cfb7fdca9",
+                "sha256:ad9912faeddb1ddcec5e26b33089166d58a107af6862d8b7f1bb2b7c0002ab39",
+                "sha256:b31fd22d214160859d038da7cb2aaa27acb71efc24a7bcc75c84b5e502721549",
+                "sha256:b54057e8a8ad92c1d8e9eaa5cf32aad70dde454abbf9b638e9d6024520a52c02",
+                "sha256:becd711fe97c2e09b1b7969e83080a3c8012bce2d30f6db879aade255fcba5c1",
+                "sha256:c2973b76f61669a85e160b4ad09879c4089fc0e3f20fd99adf161ca298fe8374",
+                "sha256:c5fb3671a8d440c24b1dd29ec621d4345ced7185e26f02abe98e85a6629fcb50",
+                "sha256:c6e29e5eb3083ad12ac5c1ce6e37465ea3428d894d3466cc9c9e2ee4bf768e53",
+                "sha256:cfa8cffec5cb9fed494c4bb335ebdb69b3c26178b0b685f67f79296c6b3d800c",
+                "sha256:d5abe5a19d943a88bea14901970e4c53e4579fc2662404cdea6163bf4c04d49a",
+                "sha256:e4335e131a58952635560a003458011d97f9ea6f3c010dc24906050b42ee2c03",
+                "sha256:e7ca5d743f6a1dc62653dfac8ee7ce2e1ba91be7cf97916a7f60b7cbe48fb48d",
+                "sha256:e898d7409ab645c25e06d4e058f99271182601d70b2887aba3351bf08e09a0c6",
+                "sha256:f5120da3a1b96f3a7a17dd6af0afdd4e6f3cc9baa87e9ee0a272882f01f980bb"
+            ],
+            "index": "pypi",
+            "version": "==1.0.1"
+        },
         "s3transfer": {
             "hashes": [
                 "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd",
@@ -1895,115 +2461,146 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330",
-                "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251"
+                "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a",
+                "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==67.4.0"
+            "version": "==67.6.1"
         },
         "shapely": {
             "hashes": [
-                "sha256:11090bd5b5f11d54e1924a11198226971dab6f392c2e5a3c74514857f764b971",
-                "sha256:1af407c58e7898a511ad01dc6e7c2099493071d939340553686b27513db6478e",
-                "sha256:2154b9f25c5f13785cb05ce80b2c86e542bc69671193743f29c9f4c791c35db3",
-                "sha256:234c5424d61d8b263d6d20045f5f32437819627ca57c1ea0c08368013b49824b",
-                "sha256:29be7767a32df19e2186288cee63e539b386a35139524dc22eeceb244d0b092b",
-                "sha256:2a2d37105c1d6d936f829de6c1c4ec8d43484d7b8bae8493bdd4267140dce650",
-                "sha256:3793b09cbd86fe297193b365cbaf58b2f7d1ddeb273213185b2ddbab360e54ae",
-                "sha256:4acbd566544c33bbc58c7dd264638ff3b91a57d9b162693c37520ea60d13668d",
-                "sha256:50f96eb9993b6d841aac0addb84ea5f9da81c3fa97e1ec67c11964c8bb4fa0a5",
-                "sha256:640e8a82b5f69ccd14e7520dd66d1247cf362096586e663ef9b8098cc0cb272b",
-                "sha256:7554b1acd64a34d78189ab2f691bac967e0d9b38a4f345044552f9dcf3f92149",
-                "sha256:7e9b01e89712fd988f931721fa36298e06a02eedf87fe7a7fd704d08f74c00f1",
-                "sha256:9c62a9f7adceaa3110f2ec359c70dddd1640191609e91029e4d307e63fc8a5af",
-                "sha256:a6c07b3b87455d107b0e4097889e9aba80a0812abf32a322a133af819b85d68a",
-                "sha256:ae9a2da2b30c0b42029337854f78c71c28d285d254efd5f3be3700d997bfd18e",
-                "sha256:b845a97f8366cc4aca197c3b04304cc301d9250518123155732da6a0e0575b49",
-                "sha256:cc0fb1851b59473d2fa2f257f1e35740875af3f402c4575b4115028234e6f2eb",
-                "sha256:e21a9fe1a416463ff11ae037766fe410526c95700b9e545372475d2361cc951e",
-                "sha256:f7eb83fb36755edcbeb76fb367104efdf980307536c38ef610cb2e1a321defe0"
+                "sha256:01224899ff692a62929ef1a3f5fe389043e262698a708ab7569f43a99a48ae82",
+                "sha256:05c51a29336e604c084fb43ae5dbbfa2c0ef9bd6fedeae0a0d02c7b57a56ba46",
+                "sha256:09d6c7763b1bee0d0a2b84bb32a4c25c6359ad1ac582a62d8b211e89de986154",
+                "sha256:193a398d81c97a62fc3634a1a33798a58fd1dcf4aead254d080b273efbb7e3ff",
+                "sha256:1a34a23d6266ca162499e4a22b79159dc0052f4973d16f16f990baa4d29e58b6",
+                "sha256:2569a4b91caeef54dd5ae9091ae6f63526d8ca0b376b5bb9fd1a3195d047d7d4",
+                "sha256:33403b8896e1d98aaa3a52110d828b18985d740cc9f34f198922018b1e0f8afe",
+                "sha256:3ad81f292fffbd568ae71828e6c387da7eb5384a79db9b4fde14dd9fdeffca9a",
+                "sha256:3cb256ae0c01b17f7bc68ee2ffdd45aebf42af8992484ea55c29a6151abe4386",
+                "sha256:45b4833235b90bc87ee26c6537438fa77559d994d2d3be5190dd2e54d31b2820",
+                "sha256:4641325e065fd3e07d55677849c9ddfd0cf3ee98f96475126942e746d55b17c8",
+                "sha256:502e0a607f1dcc6dee0125aeee886379be5242c854500ea5fd2e7ac076b9ce6d",
+                "sha256:66a6b1a3e72ece97fc85536a281476f9b7794de2e646ca8a4517e2e3c1446893",
+                "sha256:70a18fc7d6418e5aea76ac55dce33f98e75bd413c6eb39cfed6a1ba36469d7d4",
+                "sha256:7d3bbeefd8a6a1a1017265d2d36f8ff2d79d0162d8c141aa0d37a87063525656",
+                "sha256:83a8ec0ee0192b6e3feee9f6a499d1377e9c295af74d7f81ecba5a42a6b195b7",
+                "sha256:865bc3d7cc0ea63189d11a0b1120d1307ed7a64720a8bfa5be2fde5fc6d0d33f",
+                "sha256:90cfa4144ff189a3c3de62e2f3669283c98fb760cfa2e82ff70df40f11cadb39",
+                "sha256:91575d97fd67391b85686573d758896ed2fc7476321c9d2e2b0c398b628b961c",
+                "sha256:9a6ac34c16f4d5d3c174c76c9d7614ec8fe735f8f82b6cc97a46b54f386a86bf",
+                "sha256:a529218e72a3dbdc83676198e610485fdfa31178f4be5b519a8ae12ea688db14",
+                "sha256:a70a614791ff65f5e283feed747e1cc3d9e6c6ba91556e640636bbb0a1e32a71",
+                "sha256:ac1dfc397475d1de485e76de0c3c91cc9d79bd39012a84bb0f5e8a199fc17bef",
+                "sha256:b06d031bc64149e340448fea25eee01360a58936c89985cf584134171e05863f",
+                "sha256:b4f0711cc83734c6fad94fc8d4ec30f3d52c1787b17d9dca261dc841d4731c64",
+                "sha256:b50c401b64883e61556a90b89948297f1714dbac29243d17ed9284a47e6dd731",
+                "sha256:b519cf3726ddb6c67f6a951d1bb1d29691111eaa67ea19ddca4d454fbe35949c",
+                "sha256:bca57b683e3d94d0919e2f31e4d70fdfbb7059650ef1b431d9f4e045690edcd5",
+                "sha256:c43755d2c46b75a7b74ac6226d2cc9fa2a76c3263c5ae70c195c6fb4e7b08e79",
+                "sha256:c7eed1fb3008a8a4a56425334b7eb82651a51f9e9a9c2f72844a2fb394f38a6c",
+                "sha256:c8b0d834b11be97d5ab2b4dceada20ae8e07bcccbc0f55d71df6729965f406ad",
+                "sha256:ce88ec79df55430e37178a191ad8df45cae90b0f6972d46d867bf6ebbb58cc4d",
+                "sha256:d173d24e85e51510e658fb108513d5bc11e3fd2820db6b1bd0522266ddd11f51",
+                "sha256:d8f55f355be7821dade839df785a49dc9f16d1af363134d07eb11e9207e0b189",
+                "sha256:da71de5bf552d83dcc21b78cc0020e86f8d0feea43e202110973987ffa781c21",
+                "sha256:e55698e0ed95a70fe9ff9a23c763acfe0bf335b02df12142f74e4543095e9a9b",
+                "sha256:f32a748703e7bf6e92dfa3d2936b2fbfe76f8ce5f756e24f49ef72d17d26ad02",
+                "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"
             ],
             "index": "pypi",
-            "version": "==1.7.0"
+            "version": "==2.0.1"
         },
         "simplejson": {
             "hashes": [
-                "sha256:04a4b9a297cccbc9e1d66fe652fbffd55b36d6579c43132e821d315957302194",
-                "sha256:063db62a9251e61ea0c17e49c3e7bed465bfcc5359655abcb8c0bc6130a4e0d4",
-                "sha256:070ab073ce72f1624107dfd6d095c87ac32aafe7ba54a5c5055a3dd83cb06e51",
-                "sha256:099bbd3b5b4ea83159a980348cd481a34984dee5fe1b9fac31a9137158f46960",
-                "sha256:0baf8c60efef74944ed4adb034d14bcf737731576f0e4c3c56fb875ea256af69",
-                "sha256:0e7c3fae6c9540064e06a653780b4f263675cd69ca6841345029fee3e27e9bb5",
-                "sha256:141782a0a25c1792627575b37b4951583358ccc7137623aa45947f8425ee8d96",
-                "sha256:14b35fb90083218e59df5dba733c7086655f2938f3fcabe36ad849623941d660",
-                "sha256:169c2c7446ef33439c304a6aa5b7b5a2dbc938c9c2dd882dd3f2553f9518ebf6",
-                "sha256:16cc750d19852fa5ebafd55da86fa357f87991e07b4e2afb37a5975dfdde0153",
-                "sha256:1907d49d70c75530976119c13785db91168d2599288debaca7d25da9cd2f3747",
-                "sha256:1b79e2607ac5ba98381c2e068727acc1e4dd385a6d216914c0613f8f568a06a5",
-                "sha256:1e49c84df6e71e3c23169d3df481565dd607cbee4aa1e0af15c493cccad7c745",
-                "sha256:23fce984045804194f513a2739dcd82be350198470d5ade5058da019a48cf3f8",
-                "sha256:24823364fee93bab141621b3a2e10612e31be7ca58788bf9b2cd2b1ce37ab07d",
-                "sha256:290bbcdcbb37af3f7e43378f592ab7a9168fca640da6af63d42cdb535f96bbf2",
-                "sha256:2a1b3222bc8f6ac91b5ebe3263111c7dc4dc4b01c52f0153f5bb1f3ef3bf0023",
-                "sha256:2b0f6de11f5ce4b80f51bc49d08b898602e190547f8efe4e44af8ae3cda7779d",
-                "sha256:2be75f4cb9951efeb2616e16f944ee4f9a09768475a3f5c40a6ac4dc5ee68dfd",
-                "sha256:2c7ee643ee93684bf76196e2d84a2090c6df8f01737a016e869b579593827b6e",
-                "sha256:37bdef13412c0bc338db2993a38f3911d5bd2a0ba8d00b3bc66d1063edd7c33e",
-                "sha256:3bab9ea49ff477c926c5787f79ec47cf51c7ffb15c9d8dd0f09e728807d44f4b",
-                "sha256:44d6c52d4f5c0c087a6e88a92bf9f94234321d21be32c6471ba39856e304bbe3",
-                "sha256:4b8d4d958c5ab3489d1174917a7fad82da642560c39ce559a624e63deaaa36b1",
-                "sha256:4de9fed1166aeedee44150fa83bc059aca6b612940281f8b5a39374781f16196",
-                "sha256:502d86fbfe914263642479b87ed61af3b27b9e039df77acd2416cfccfc892e68",
-                "sha256:508342d7227ed66beecfbba7a38b46e1a713faeb034216f43f03ec5c175e0622",
-                "sha256:50f4b6d52f3a2d1cffd11834a1fe7f9516f0e3f20cbe78027aa88ff990fad7d6",
-                "sha256:52465a5578cfc2c5e374a574df14dfb75e04c6cb6a100b7abc8bf6c89bea8f5e",
-                "sha256:55aa983575b0aef143845f5bfbb35075475eccaebf7d4b30f4037a2fe8414666",
-                "sha256:55df3dfd8777bf134e1078d2f195352432a77f23ccb90b92b08218123d56adc9",
-                "sha256:56f186d44a9f625b5e5d9ba4b9551e263604000a7df60cb373b3e789ca603b2a",
-                "sha256:5780e3929435a8d39671537174f8ce0ccafb4f6e0c748ffe139916ffbdca39d3",
-                "sha256:59a629240cfbc5b4f390a8578dca74ae77ab617de971862acb946822d2eb1b11",
-                "sha256:5b009342e712026ffabe8a471d5b4a4ff2a038687387e74eae601574c04dae33",
-                "sha256:62628ea5df8c830d00a7417d5ecd949a1b24a8d0a5063a2a77f7ec7522110a0f",
-                "sha256:694332fd6fd10fe8868c2508583220d1a1a7be9ff049dab5bd6b9aedfb9edc50",
-                "sha256:6a49665169c18f27a0fc10935466332ee7406ee14ced8dc0a1b4d465547299aa",
-                "sha256:6b997739fdbc9b7030ff490fc8e5f8c144b8ec80f3605eff643983672bb8cfde",
-                "sha256:6bd81d10cb3384f64242316da8a2b2f88618776bc1ef38bcc79f1afe8ad36616",
-                "sha256:6c4c56c5abb82e22877b913186e5c0fd7d9eef0c930719e28fa451d3f11defb4",
-                "sha256:6fe1173b4146641c872bafa6f9a21f3a2012f502d54fbb523a76e6320024fae9",
-                "sha256:75eb555dc349d0cbe2c95ea2be665b306c6ac6d5b64e3a3920af9b805ecdb5f7",
-                "sha256:7c26fe63755ecc59c502ddde8e58ce8b765bf4fdd3f5858d2b7c8ab28bc2a9c8",
-                "sha256:7e73d9d6af3c29b60a92e28b3144d951110f59a3d05fc402c3f6c5248b883400",
-                "sha256:7ff65b475091084e5bdb7f26e9c555956be7355b573ce494fa96f9f8e34541ac",
-                "sha256:8209c40279ed9b2cd5fbe2d617a29a074e90ea97fce7c07a0128a01cb3e8afc5",
-                "sha256:88f59a07873dc1f06fd9e6712dd71286f1b297a066ad2fd9110ad080d3cb011c",
-                "sha256:96ade36640734b54176c4765d00a60767bd7fae5b7a5b3574accc055ac18e34c",
-                "sha256:9cf299fbb7d476676dfea372a3262654af98694bd1df35b060ce0fe1b68087f1",
-                "sha256:a2960b95f3ba822d077d1afa7e1fea9799cfb2990028cf010e666f64195ecb5a",
-                "sha256:a80bd9a3db88a76a401155c64e3499376c702307c2206cb381cc2a8dd9cc4f1f",
-                "sha256:aad323e92cb1bd3b1db6f57c007dca964d13c52247ad844203ce381e94066601",
-                "sha256:ab5bdf0b8d07f7fd603b2d0c1982412cd9f8ade997088ddced251f7e656c7fd4",
-                "sha256:b0352428b35da859a98770949e7353866ae65463026f1c8e4c89a6395d4b5fd7",
-                "sha256:b2c4e8b65987f3c6529149495d28e23efe213e94dc3659176c4ab22d18a9ee4a",
-                "sha256:bcd9eac304a133ee4af58e68c5ded4c5ba663d3ee4602e8613359b776a1f8c8f",
-                "sha256:c3b696770b504f881f271f97b94a687487ec1ef20bfbd5f20d92bbab7a85952d",
-                "sha256:c4514675f6571da8190fea52a110bca686fa844972e8b2b3bc07ace9e632ee4f",
-                "sha256:c98fddc374468158778a8afb3fd7296412a2b2fc34cebba64212ac3e018e7382",
-                "sha256:cde5a3ff5e0bd5d6da676314dfae86c9e99bff77bca03d30223c9718a58f9e83",
-                "sha256:cf7168b2046db0eceb83d8ed2ee31c0847ce18b2d8baf3e93de9560f3921a8c3",
-                "sha256:d774782159347d66563cd7ac18b9dd37010438a825160cde4818caa18110a746",
-                "sha256:d990ea42ba908cb57a3df97d283aa26c1822f10a0a60e250b54ee21cd08c48d0",
-                "sha256:e762e9d8556fa9f3a99f8a278eeba50a35b5f554b82deeb282ddbdd85816e638",
-                "sha256:e8a4750e8db92109e6f1f7783a7faae4254d6d5dc28a41ff7eff7d2265f0586b",
-                "sha256:eb81cfef0c0039010f0212f4e5eb6909641b8a54c761584054ac97fd7bd0c21a",
-                "sha256:ebb53837c5ffcb6100646018565d3f1afed6f4b185b14b2c9cbccf874fe40157",
-                "sha256:efa70fd9b6c7b57b048ecadb909683acd535cddebc5b22f3c05ba3b369739caf",
-                "sha256:f73bae5e315adf7bc8cb7f0a13a1e9e33bead42e8ce174be83ac9ecc2513c86a",
-                "sha256:f89f078114cacedb9a3392615cc099cf02a51efa7507f90e2006bf7ec38c880d",
-                "sha256:f9f72d2b539512f382a48cc9ad6cea2d3a572e71e92c40e03d2140041eeaa233",
-                "sha256:fc8df5831b645e96a318ea51a66ce6e2bb869eebc3fa9a860bbf67aecd270055"
+                "sha256:01f426ee9e3a2d205aa4c22c3da996b51f2de75c4199ef703258a28b304dea8c",
+                "sha256:03de1ec4ad734f28ca49b0a758b997d752be0d089ed30360157c4e8811999c8f",
+                "sha256:041dd69026284d10f035cefb4a75026d2cfcef31f31e62585eeb2b7776e7e047",
+                "sha256:05a668d4a93816fb8a644e90e7987aa3beeb9d2112ca50a474d41e6acb5bb88a",
+                "sha256:094275b1b8f003afce1167c8a674cd1ee2fd48c566632dac5d149901d5012ff8",
+                "sha256:097e48686e49026836ef384c7c10ca670acc023cb16a976a689c2eb6c1852df4",
+                "sha256:0cc9a47bf8cde85c99db5f4a919bb756e62427ade0f2e875a6ec89ae8492d486",
+                "sha256:0cdb5069870f7d26a34e5adc30672d0a7b26e652720530a023bb3a8d8a42e37f",
+                "sha256:0dcc54e7cfbd9674ec4ca181e26eaa5b038446601faeaa6c83d146ddef2f2652",
+                "sha256:16fbebfc38ad4285c256d2430797fd669b0437d090e985c6d443521d4303b133",
+                "sha256:1844d7782652f859d9648531778582d4842d80cfff8d334eb23bb8da0d22a1b0",
+                "sha256:2362c66d2c633925d90f2f177f05e0570d320d986130d34dff9ad6edbf7be8ac",
+                "sha256:29a86bc9c8a913a4e0ffab85c563a7505cdf4bd13fba05342f8314facc0b7586",
+                "sha256:2a6e5c0e0817fb20dbb880c83caebbd4ef39f1901f6f8e53b73a3c74de4e5172",
+                "sha256:2d1b47f768e1f4c1c8a9457effabed735939401e85c0ddcdf68444c88a9242e6",
+                "sha256:30e381471158290ccb79bd31e7bbda4c8f2cf7e1a5f6b557c1b97d6036ccd05b",
+                "sha256:313dfd911723dc3022fed7050a7b315d5d0681cd56eee08e44e2cbd39fd9ad81",
+                "sha256:32de1672f91a789cc9e1c36c406b2d75457a242d64e9e73a70b9b814ef00095e",
+                "sha256:340b7d085b4a5063aacb8664b1250e4a7426c16e1cc80705c548a229153af147",
+                "sha256:34d95ad8e27754f0d91917600d6ea273e05c82a71021f168c45be48637d9502f",
+                "sha256:3d549efc7e8f9a180c59462b124991b690ff25c235d5cf495c3246c66a7679cd",
+                "sha256:3dbfaa79b1c0efdb768392a19110f1aff793f3e8d43f57e292f46734b8affb45",
+                "sha256:44058bea97429cfa0d6fb1d8eb0736a77022f34a326d5bc64fd6fed8d9304571",
+                "sha256:46b8cc86204b51eddcf157cbaf3c44a20f24393030442af0909eeb961186cb67",
+                "sha256:47509775a5c41ec2a6cd17c9c00fc14965cad8e6670059663872ba5e39332f57",
+                "sha256:4b5df4ee48403885046c6f4fd8adc84c4ac0adec69482f22a17bd4ba52876341",
+                "sha256:544e5607142d66a469ecf78a3154ec0f915834dc3b8cfdb2677a78ca58319ad6",
+                "sha256:56d36f47bc7c7684504f0f18feb161a0b1162546b3622e45aa6155f8285180ac",
+                "sha256:5b217201efc007166e24e9a282007cc208a2d059350a7c5bd0b0303460ad3019",
+                "sha256:5c4f59dd358c3a99efa46d62dc1583be3a1c37171f5240c4cbdc2d5838870902",
+                "sha256:5d45ed9452a42064805143480397b586ea2ea322f4b8b69034c51181e7f38342",
+                "sha256:6197cfebe659ac802a686b5408494115a7062b45cdf37679c4d6a9d4f39649b7",
+                "sha256:65de5876e34780b43f92d9d2539de16ecc56d16f56e56e59b34adfa1cebe064f",
+                "sha256:676e8c182f8079851f12ae1cee2fcebe04def2da2a5703a9d747ab125af47732",
+                "sha256:682b202f56d9d9e1bb22eaca3e37321002223fd5ddef7189b9233e3c14079917",
+                "sha256:695da62e494e4689ab78fae173a78390a175b6a5ccc4292277ce0f8dba3945d5",
+                "sha256:6d65ea4582b47d77e9094c22eb0aeded0ebd96c1df86e988870b40c6514c6e21",
+                "sha256:706a7fc81ceeb321a1040d008b134056012188f95a5c31ad94fb03153b35cc84",
+                "sha256:7339bd6203351555c1e728acd601ba95ebce0f6041ebdb386e025f00af3f1769",
+                "sha256:7701a289d45fdfeb37f1d15cf638801cea439df667a613379443772a86e82936",
+                "sha256:7f27a079cb009ba569983061a50a9270b7e1d35f81e4eeaf0e26f8924027e550",
+                "sha256:827ddc3b3603f7d0421b054388da6face7871d800c4b3bbedeedc8778e4085ea",
+                "sha256:8ccc982197982cdda19e3e5ba4ef7f6ad6bed3eb39bb423bfbf7fa2cd29488ab",
+                "sha256:8f381747c2edebe3c750a571e55103bfcc33b2707a9b91ae033ab9ba718d976a",
+                "sha256:93ba80fbf959b5852554f23201a5f4b30885930c303546ffa883859a435ea3cf",
+                "sha256:9a50a9da1cf93e35f26c4ddee162abf3184a340339ec2d4001c34607b87e71b4",
+                "sha256:a1163bfe5d043c20adeb5c4c8e89dd1dd39b375c8ca6f1c1e35ec537ad7a12e7",
+                "sha256:a2285609b4edbf9957440642493788ebef6583042b3fb96217c2e71f29bc6d80",
+                "sha256:a255d30cda6334ba780eb40a56e8134efd3453948b995d3966e45212e34bf018",
+                "sha256:a3bba99178f1b25878752a8bc6da2f93fbae754ebd4914d2ac4b869b9fb24102",
+                "sha256:a56005332d70b8d02d476d4a85818b27b01e51dac1a21d5c1a1d8a5df2efb4a6",
+                "sha256:a89d7fe994b115f0a792e6673f387af3db812a1760d594abad51e0ea11d3e470",
+                "sha256:a8ac155e3fd3b54a63040df024e57e62c130b15a2fc66eff3c2a946f42beed52",
+                "sha256:ab5941e1fd509fc151258477ef4b663fe14c94f8faf3581827bf4b02080fd4ba",
+                "sha256:ab64f087c5863ac621b42e227e5a43bd9b28de581afe7be12ad96562b9be8203",
+                "sha256:ad37f25fd8dfbed80815c3281b82a165be2a74e663856b9a50099d18789987bc",
+                "sha256:b17026f3f349a6e87818cd3531e3bbb5cc78a6f4b2b6718f574a8e0512d71e08",
+                "sha256:b43d3c2e204d709af955bdb904ae127fe137363ace87fbf7dc8fe6017f7f8449",
+                "sha256:b482d1fdd8f860e743c7de8cd6dfe54fb9fe8cd6ccba29e2966912ac89e17b2f",
+                "sha256:b6c6cfc492710d8f0303705fa1ff7bb3d6a145f523384e45a6f3b13ada37021f",
+                "sha256:b9893852c559998f667e6434d2c2474518d4cdfd1b9cec8e57b3c9d577ba55c1",
+                "sha256:bd694c465cc61fa8e599355e535b6eb561279834d9883aeef08d0e86c44c300c",
+                "sha256:c0444423129df448788edc66a129bc7560ad7d6a661d74f0900959c0b44349a1",
+                "sha256:c37b092d29741096c4723f48924a80b1d3de62ca1de254ce88178fa083dd520c",
+                "sha256:ced906b172bfad62736a27cfafcb6e24bc9938533b0529ff8150f7926fe35b54",
+                "sha256:d0d3b9f7cee233368d92c89746dde74313abafaa3ec1f0c06a3f4f164dc27bcc",
+                "sha256:d5f67bffa6fc68e391b2250e1feb43d534ded64a7b918eb89cf7e3e679759d94",
+                "sha256:d9f0dfde448611f4f818da05f9b544a78f29355dc39151b0dad8e7c65c513e4f",
+                "sha256:da6dc0cb00ef1e1a8daf285074ca8b2bb89591170c42ceab0c37bcdb9adc802c",
+                "sha256:dc74a9ef4d61e18ee6f1886b6ef1fe285b1f432885288afacfb7402f7d469448",
+                "sha256:deb71e6166e4f1264174d78b5b88abd52b14c6649e6eabaf9cf93cb1c7362850",
+                "sha256:e042ae053e05fe193514d51d6b0f0243729961901e9a75f8b596bfaf69522c52",
+                "sha256:e2f87a483c4ab0bb2a9adc9ca09173e7f7cf3696e4fa67bd45a6b33181e57921",
+                "sha256:e2fa1ee5ca34ab2ecfbe3f7a7e952a1ecaebb5b4818f002b5b146324912ac3d5",
+                "sha256:e7d3f7cd57ce0c6a5bb8133f8ed5c3d1be0473a88b7d91a300626298f12d0999",
+                "sha256:edb334cab35dcd90eb563fdacb085f10e5dd0b1acb57fa43f8933308b42a8f88",
+                "sha256:efae49d0148ec68b6e012f1b9e19bd530f4dced378ba919e3e906ae2b829cc31",
+                "sha256:f1085cadec0f7e76377951d7a87744628c90ac6cc634fc97eecce0c4d41ec563",
+                "sha256:f15f56b3119fb71fa57eb4613bcd87eb7df6c2f3547de7d341853d3e50cef97e",
+                "sha256:f1b425a857ce52e651739314e4118fc68bd702ef983148b8fd5cb6f68bb6a020",
+                "sha256:f31e126204ec38f92dee119af87cf881044ef7dea6f7477ef774ed3d84199c24",
+                "sha256:f5e0a03e533313eee9437ccc6c4eab47369f17bc919b57df4a20ccd8bc85d8fd",
+                "sha256:f85d87986ca375b8305b5c4f166783b8db383a6469e8b99b8dba22878388f234",
+                "sha256:fa6fe8fa94a831886ee164ac03514f361e1387a62a1b9da32fde5c0c1f27fa8d",
+                "sha256:fb0f8b35c11fd8e4b924f974d331b20fa54555282451db7f2a3b24bd2d33cc11"
             ],
             "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.18.3"
+            "version": "==3.18.4"
         },
         "six": {
             "hashes": [
@@ -2081,6 +2678,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.17.1"
         },
+        "texttable": {
+            "hashes": [
+                "sha256:290348fb67f7746931bcdfd55ac7584ecd4e5b0846ab164333f0794b121760f2",
+                "sha256:b7b68139aa8a6339d2c320ca8b1dc42d13a7831a346b446cb9eb385f0c76310c"
+            ],
+            "version": "==1.6.7"
+        },
         "tinycss2": {
             "hashes": [
                 "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847",
@@ -2116,11 +2720,11 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:6baa75a88582b1db6d34ce4690da5501d2a1cb65c34664840a456b2c9f794d29",
-                "sha256:fcb7cb5b729b60a27f300b15c1ffd4744f080fb483b88f31dc8654b082cc8ea5"
+                "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5",
+                "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"
             ],
             "index": "pypi",
-            "version": "==4.48.0"
+            "version": "==4.65.0"
         },
         "traitlets": {
             "hashes": [
@@ -2130,6 +2734,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==5.9.0"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb",
+                "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
+            ],
+            "markers": "python_version < '3.10'",
+            "version": "==4.5.0"
+        },
         "uri-template": {
             "hashes": [
                 "sha256:934e4d09d108b70eb8a24410af8615294d09d279ce0e7cbcdaef1bd21f932b06",
@@ -2139,11 +2751,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
-                "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
+                "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305",
+                "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.14"
+            "version": "==1.26.15"
         },
         "wcwidth": {
             "hashes": [
@@ -2154,10 +2766,10 @@
         },
         "webcolors": {
             "hashes": [
-                "sha256:16d043d3a08fd6a1b1b7e3e9e62640d09790dce80d2bdd4792a175b35fe794a9",
-                "sha256:d98743d81d498a2d3eaf165196e65481f0d2ea85281463d856b1e51b09f62dce"
+                "sha256:29bc7e8752c0a1bd4a1f03c14d6e6a72e93d82193738fa860cbff59d0fcc11bf",
+                "sha256:c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a"
             ],
-            "version": "==1.12"
+            "version": "==1.13"
         },
         "webencodings": {
             "hashes": [
@@ -2176,19 +2788,19 @@
         },
         "whitebox": {
             "hashes": [
-                "sha256:0455cf727233adb53abe1bbe221fe0ac597d3c2609a3f7e74c1243f8ffb4b190",
-                "sha256:8afc78a2f59e1b58e1cae2509f9489b7fc9f2105a5ee7d84a4005e5905dcea01"
+                "sha256:38becfa894091c0adcebb162ffe7eb8b735e9a52f91376d36cb1a554bd10544d",
+                "sha256:6b4080bc8181f0876cda6c9093bcb463faa4ab8a8c2c631c06b579d3ec1131ba"
             ],
             "index": "pypi",
-            "version": "==2.2.0"
+            "version": "==2.3.1"
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:003f716d930d385be3fd9de42dd9bf008e30053f73bddde235d14fbeaeff19af",
-                "sha256:eaaaf434fb9b08bd197b2a14ffe45ddb5ac3897593d43c69287091e5f3147bf7"
+                "sha256:be3228a73bbab189a16be2d4a3cd89ecbd4e31948bfdc64edac17dcdee3cd99c",
+                "sha256:ea67c17a7cd4ae358f8f46c3b304c40698bc0423732e3f273321ee141232c8be"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.0.5"
+            "version": "==4.0.7"
         },
         "xarray": {
             "hashes": [
@@ -2279,11 +2891,11 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:188834565033387710d046e3fe96acfc9b5e86cbca7f39ff69cf21a4128198b7",
-                "sha256:9e5421e176ef5ab4c0ad896624e87a7b2f07aca746c9b2aa305952800cb8eecb"
+                "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b",
+                "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.14.0"
+            "version": "==3.15.0"
         }
     },
     "develop": {
@@ -2318,11 +2930,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:b13a1d6c1f5818bd388db53b7107d17454129a70de2b87481d555daede5eb49e",
-                "sha256:b38c31e8fc7eff642fc7c597061fff462537cf2314e3225a19c906b7b0d8a345"
+                "sha256:1c183bf61b148b00bcebfa5d9b39312733ae97f6dad90d7e9b4d86c8647f498c",
+                "sha256:a950236df04ad75b5bc7f816f9af3d74dc118fd42f2ff7e80e8e60ca1f182e2d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.10.0"
+            "version": "==8.12.0"
         },
         "jedi": {
             "hashes": [
@@ -2365,10 +2977,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:6a2948ec427dfcc7c983027b1044b355db6aaa8be374f54ad2015471f7d81c5b"
+                "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b",
+                "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.37"
+            "version": "==3.0.38"
         },
         "ptyprocess": {
             "hashes": [
@@ -2416,6 +3029,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==5.9.0"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb",
+                "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
+            ],
+            "markers": "python_version < '3.10'",
+            "version": "==4.5.0"
+        },
         "wcwidth": {
             "hashes": [
                 "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2e2956ad3..24f1a476e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,42 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.5.0 - 2023-03-02 - [PR#857](https://github.com/NOAA-OWP/inundation-mapping/pull/857)
+
+Addresses changes to function calls needed to run upgraded Shapely library plus other related library upgrades. Upgraded libraries include:
+- shapely
+- geopandas
+- pandas
+- numba
+- rasterstats
+- numpy
+- rtree
+- tqdm
+- pyarrow
+- py7zr
+
+Pygeos is removed because its functionality is incorporated into the upgraded shapely library.
+
+### Changes
+
+- `Dockerfile`
+- `Pipfile and Pipfile.lock`
+- `src/`
+	- `associate_levelpaths_with_levees.py`
+    - `build_stream_traversal.py`
+	- `add_crosswalk.py`
+	- `adjust_headwater_streams.py`
+	- `aggregate_vector_inputs.py`
+	- `clip_vectors_to_wbd.py`
+	- `derive_headwaters.py`
+	- `stream_branches.py`
+	- `split_flows.py`
+- `tools/`
+	- `fimr_to_benchmark.py`
+	- `tools_shared_functions.py`
+
+<br/><br/>
+
 ## v4.3.4.0 - 2023-03-16-23 [PR#847](https://github.com/NOAA-OWP/inundation-mapping/pull/847)
 
 ### Changes
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 15dfe2efc..6a5034853 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -97,7 +97,7 @@ def add_crosswalk(input_catchments_fileName,
         input_flows_midpoint = input_flows_midpoint.set_index('HydroID')
 
         # Create crosswalk
-        crosswalk = gpd.sjoin(input_flows_midpoint, input_nwmcat, how='left', op='within').reset_index()
+        crosswalk = gpd.sjoin(input_flows_midpoint, input_nwmcat, how='left', predicate='within').reset_index()
         crosswalk = crosswalk.rename(columns={"index_right": "feature_id"})
 
         # fill in missing ms
diff --git a/src/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
index 300e542b0..afea3f337 100644
--- a/src/adjust_headwater_streams.py
+++ b/src/adjust_headwater_streams.py
@@ -24,7 +24,7 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
     # Identify true headwater segments
     nhd_streams_adj = nhd_streams.loc[(nhd_streams.headwaters_id > 0) & (nhd_streams.downstream_of_headwater == False),:].copy()
-    nhd_streams_adj = nhd_streams_adj.explode()
+    nhd_streams_adj = nhd_streams_adj.explode(index_parts=True)
     nhd_streams_adj = nhd_streams_adj.reset_index(drop=True)
 
     if nwm_headwaters["site_id"].dtype != 'int': nwm_headwaters["site_id"] = nwm_headwaters["site_id"].astype(int)
@@ -66,7 +66,7 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
                 closest_stream = nhd_streams.loc[nhd_streams["nws_lid"]==point[headwater_id]]
 
             try: # Seeing inconsistent geometry objects even after exploding nhd_streams_adj; not sure why this is
-                closest_stream =closest_stream.explode()
+                closest_stream =closest_stream.explode(index_parts=True)
             except:
                 pass
 
@@ -143,7 +143,7 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
 
         # Identify ajusted nhd headwaters
         nhd_headwater_streams_adj = nhd_streams.loc[nhd_streams['is_headwater'],:]
-        nhd_headwater_streams_adj = nhd_headwater_streams_adj.explode()
+        nhd_headwater_streams_adj = nhd_headwater_streams_adj.explode(index_parts=True)
 
         hw_points = np.zeros(len(nhd_headwater_streams_adj),dtype=object)
         for index,lineString in enumerate(nhd_headwater_streams_adj.geometry):
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
index 32b25739e..6ebb91d45 100755
--- a/src/aggregate_vector_inputs.py
+++ b/src/aggregate_vector_inputs.py
@@ -34,7 +34,7 @@ def identify_nwm_ms_streams(nwm_streams_filename,ahps_filename,nwm_streams_all_f
     # Subset NHDPlus HR
     nwm_streams['is_relevant_stream'] = nwm_streams['is_headwater'].copy()
 
-    nwm_streams = nwm_streams.explode()
+    nwm_streams = nwm_streams.explode(index_parts=True)
 
     # Trace down from headwaters
     nwm_streams.set_index('ID',inplace=True,drop=False)
@@ -93,7 +93,7 @@ def find_nwm_incoming_streams(nwm_streams_,wbd,huc_unit):
         col_name = f"HUC{huc_unit}"
         huc = row[col_name]
         huc_mask = wbd.loc[wbd[col_name]==str(huc)]
-        huc_mask = huc_mask.explode()
+        huc_mask = huc_mask.explode(index_parts=True)
         huc_mask = huc_mask.reset_index(drop=True)
 
         # Input nwm streams
@@ -104,7 +104,7 @@ def find_nwm_incoming_streams(nwm_streams_,wbd,huc_unit):
         else:
             raise TypeError("Pass dataframe or filepath for nwm streams")
 
-        nwm_streams = nwm_streams.explode()
+        nwm_streams = nwm_streams.explode(index_parts=True)
         nwm_streams = nwm_streams.reset_index(drop=True)
 
         for index, polygon in enumerate(huc_mask.geometry):
@@ -239,7 +239,7 @@ def subset_stream_networks(args, huc):
     del wbd8
 
     huc_mask = selected_wbd4.loc[selected_wbd4.HUC4.str.startswith(huc)]
-    huc_mask = huc_mask.explode()
+    huc_mask = huc_mask.explode(index_parts=True)
     huc_mask = huc_mask.reset_index(drop=True)
 
     if len(selected_wbd8.HUC8) > 0:
diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py
index 0dbf7327c..58bd58910 100644
--- a/src/associate_levelpaths_with_levees.py
+++ b/src/associate_levelpaths_with_levees.py
@@ -2,6 +2,7 @@
 
 import os
 import argparse
+import warnings
 import numpy as np
 import pandas as pd
 import geopandas as gpd
@@ -45,8 +46,10 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
         levees_buffered_right.geometry = levees.buffer(-levee_buffer, single_sided=True)
 
         # Intersect leveed areas with single-sided levee buffers
-        leveed_left = gpd.overlay(levees_buffered_left, leveed_areas, how='intersection')
-        leveed_right = gpd.overlay(levees_buffered_right, leveed_areas, how='intersection')
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            leveed_left = gpd.overlay(levees_buffered_left, leveed_areas, how='intersection')
+            leveed_right = gpd.overlay(levees_buffered_right, leveed_areas, how='intersection')
 
         # Find leveed areas not intersected by either buffer
         leveed_intersected = []
@@ -133,7 +136,7 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
             row_intersections = gpd.overlay(levees[levees[levee_id_attribute] == row[levee_id_attribute]], levelpaths[levelpaths[branch_id_attribute] == row[branch_id_attribute]], how='intersection', keep_geom_type=False)
 
             # Convert MultiPoint to Point
-            row_intersections = row_intersections.explode()
+            row_intersections = row_intersections.explode(index_parts=True)
 
             # Select Point geometry type
             row_intersections = row_intersections[row_intersections.geom_type =='Point']
diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py
index b3639ac95..8952a973c 100644
--- a/src/build_stream_traversal.py
+++ b/src/build_stream_traversal.py
@@ -48,7 +48,7 @@ def execute(self, streams, wbd8, hydro_id):
                     stream_midpoint = stream_midpoint + [lineString.interpolate(0.5,normalized=True)]
 
                 stream_md_gpd = gpd.GeoDataFrame({'geometry':stream_midpoint}, crs=streams.crs, geometry='geometry')
-                stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', op='within')
+                stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', predicate='within')
                 stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "midpoint", "index_right": "HUC8id"})
                 streams = streams.join(stream_wbdjoin).drop(columns=['midpoint'])
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 5ff1c8c9a..c462d0ea6 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -80,11 +80,11 @@ def subset_vector_layers(subset_nwm_lakes,
 
     if not nwm_lakes.empty:
         # Perform fill process to remove holes/islands in the NWM lake polygons
-        nwm_lakes = nwm_lakes.explode()
+        nwm_lakes = nwm_lakes.explode(index_parts=True)
         nwm_lakes_fill_holes = MultiPolygon(Polygon(p.exterior) for p in nwm_lakes['geometry']) # remove donut hole geometries
         # Loop through the filled polygons and insert the new geometry
-        for i in range(len(nwm_lakes_fill_holes)):
-            nwm_lakes.loc[i, 'geometry'] = nwm_lakes_fill_holes[i]
+        for i in range(len(nwm_lakes_fill_holes.geoms)):
+            nwm_lakes.loc[i, 'geometry'] = nwm_lakes_fill_holes.geoms[i]
         nwm_lakes.to_file(subset_nwm_lakes, driver = getDriver(subset_nwm_lakes), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     del nwm_lakes
 
diff --git a/src/derive_headwaters.py b/src/derive_headwaters.py
index 1bae7209e..b105d2c67 100644
--- a/src/derive_headwaters.py
+++ b/src/derive_headwaters.py
@@ -8,7 +8,7 @@
 
 def findHeadWaterPoints(flows):
 
-    flows = flows.explode()
+    flows = flows.explode(index_parts=True)
     headwater_points = []
     starting_points = set() ; end_points = set()
     for i,g in enumerate(flows.geometry):
diff --git a/src/split_flows.py b/src/split_flows.py
index 100be57b1..e774bb939 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -62,7 +62,8 @@ def split_flows(max_length,
 
     wbd8 = wbd8.filter(items=[FIM_ID, 'geometry'])
     wbd8 = wbd8.set_index(FIM_ID)
-    flows = flows.explode()
+    # don't index parts because the new index format causes problems later on
+    flows = flows.explode(index_parts=False)
 
     # temp
     flows = flows.to_crs(wbd8.crs)
@@ -76,7 +77,7 @@ def split_flows(max_length,
     if (os.path.split(os.path.dirname(nwm_streams_filename))[1] != '0'):
         print ('trimming DEM stream to NWM branch terminus')
         # read in nwm lines, explode to ensure linestrings are the only geometry
-        levelpath_lines = gpd.read_file(nwm_streams_filename).explode()
+        levelpath_lines = gpd.read_file(nwm_streams_filename).explode(index_parts=True)
 
         # Dissolve the linestring (how much faith should I hold that these are digitized with flow?)
         linestring_geo = levelpath_lines.iloc[0]['geometry']
@@ -85,12 +86,12 @@ def split_flows(max_length,
 
         # Identify the end vertex (most downstream, should be last), transform into geodataframe
         terminal_nwm_point = []
-        first, last = linestring_geo.boundary
-        terminal_nwm_point.append({'ID':'teminal','geometry':last})
+        last = Point(linestring_geo.coords[-1])
+        terminal_nwm_point.append({'ID':'terminal','geometry':last})
         snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(levelpath_lines.crs)
 
         # Snap to DEM flows
-        snapped_point['geometry'] = snapped_point.apply(lambda row: flows.interpolate(flows.project( row.geometry)), axis=1)
+        snapped_point['geometry'] = flows.interpolate(flows.project(snapped_point.geometry))
 
         # Trim flows to snapped point
         # buffer here because python precision issues, print(demDerived_reaches.distance(snapped_point) < 1e-8)
@@ -98,7 +99,7 @@ def split_flows(max_length,
         # Edge cases: line string not split?, nothing is returned, split does not preserve linestring order?
         # Note to dear reader: last here is really the most upstream segmennt (see crevats above).  When we split we should get 3 segments, the most downstream one
         # the tiny 1 meter segment that falls within the snapped point buffer, and the most upstream one.  We want that last one which is why we trimmed_line[len(trimmed_line)-1]
-        last_line_segment = pd.DataFrame({'id':['first'],'geometry':[trimmed_line[len(trimmed_line)-1].wkt]})
+        last_line_segment = pd.DataFrame({'id':['first'],'geometry':[trimmed_line.geoms[len(trimmed_line.geoms)-1].wkt]})
         last_line_segment['geometry'] = last_line_segment['geometry'].apply(wkt.loads) # can be last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment) when we update geopandas verisons
         last_line_segment_geodataframe = gpd.GeoDataFrame(last_line_segment).set_crs(flows.crs)
 
@@ -107,7 +108,7 @@ def split_flows(max_length,
 
     # split at HUC8 boundaries
     print ('splitting stream segments at HUC8 boundaries')
-    flows = gpd.overlay(flows, wbd8, how='union').explode().reset_index(drop=True)
+    flows = gpd.overlay(flows, wbd8, how='union',keep_geom_type=True).explode(index_parts=True).reset_index(drop=True)
     flows = flows[~flows.is_empty]
 
     if (len(flows) == 0):
@@ -122,7 +123,7 @@ def split_flows(max_length,
           #create splits at lake boundaries
           lakes = lakes.filter(items=['newID', 'geometry'])
           lakes = lakes.set_index('newID')
-          flows = gpd.overlay(flows, lakes, how='union').explode().reset_index(drop=True)
+          flows = gpd.overlay(flows, lakes, how='union',keep_geom_type=True).explode(index_parts=True).reset_index(drop=True)
           lakes_buffer = lakes.copy()
           lakes_buffer['geometry'] = lakes.buffer(lakes_buffer_input) # adding X meter buffer for spatial join comparison (currently using 20meters)
 
@@ -215,7 +216,7 @@ def split_flows(max_length,
     split_flows_gdf = gpd.GeoDataFrame({'S0' : slopes ,'geometry':split_flows}, crs=flows.crs, geometry='geometry')
     split_flows_gdf['LengthKm'] = split_flows_gdf.geometry.length * toMetersConversion
     if lakes is not None:
-        split_flows_gdf = gpd.sjoin(split_flows_gdf, lakes_buffer, how='left', op='within') #options: intersects, within, contains, crosses
+        split_flows_gdf = gpd.sjoin(split_flows_gdf, lakes_buffer, how='left', predicate='within') #options: intersects, within, contains, crosses
         split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
     else:
         split_flows_gdf['LakeID'] = -999
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 8a6ebf6cc..7102981fc 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -550,7 +550,7 @@ def remove_branches_in_waterbodies(self,
 
         if isinstance(waterbodies,gpd.GeoDataFrame):
             # Find branches in waterbodies
-            sjoined = gpd.sjoin(self, waterbodies, op='within')
+            sjoined = gpd.sjoin(self, waterbodies, predicate='within')
             self.drop(sjoined.index, inplace=True)
 
             if out_vector_files is not None:
@@ -910,7 +910,7 @@ def explode_to_points(self,reach_id_attribute='ID', sampling_size=None,
 
         points_gdf['geometry'] = all_exploded_points
         
-        points_gdf = points_gdf.explode()
+        points_gdf = points_gdf.explode(index_parts=True)
         points_gdf.reset_index(inplace=True,drop=True)
 
         return(points_gdf)
diff --git a/tools/fimr_to_benchmark.py b/tools/fimr_to_benchmark.py
index 0d4184815..90639c4fa 100644
--- a/tools/fimr_to_benchmark.py
+++ b/tools/fimr_to_benchmark.py
@@ -93,7 +93,7 @@ def fimr_to_benchmark(fimr_path, output_path):
     correct_flow = joined_flowlines_fimr.loc[joined_flowlines_fimr['order_'] == stream_order_float]
     
     # Exploding the fimr inundaton polygon
-    exploded_fimr = fimr.explode()
+    exploded_fimr = fimr.explode(index_parts=True)
     # Taking only the fimr polygons that overlap the flowlines to minimize noise
     clean_fimr = gpd.sjoin(exploded_fimr, correct_flow)
     # Renaming columns to prevent duplicate column names in overlay
diff --git a/tools/inundation.py b/tools/inundation.py
index c55ade9a7..455c6f6f9 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -484,7 +484,8 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
                                          'LakeID' : int,
                                          'last_updated':object, 
                                          'submitter':object, 
-                                         'obs_source':object}
+                                         'obs_source':object},
+                                 low_memory=False
                                 )
         huc_error = hydroTable.HUC.unique()
         hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py
index a99abdeec..003b12457 100644
--- a/tools/overlapping_inundation.py
+++ b/tools/overlapping_inundation.py
@@ -238,9 +238,7 @@ def read_rst_data(self,
                               window[-1][0],
                               window[-1][1],
                               window[0][0],
-                              transform=self.depth_rsts[ds].transform,
-                              height=window_height,
-                              width=window_width)
+                              transform=self.depth_rsts[ds].transform)
 
             bnds.append(bnd)
 
@@ -255,9 +253,7 @@ def read_rst_data(self,
                                  window[-1][0],
                                  window[-1][1],
                                  window[0][0],
-                                 transform=meta['transform'],
-                                 height=window_height,
-                                 width=window_width)
+                                 transform=meta['transform'])
 
         return [final_bnds, bnds, data]
 
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index f1ad637ce..d0f50caca 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -777,7 +777,7 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
 #        metadata_gdf = metadata_gdf[retain_attributes]
     print("Performing spatial and tabular operations on geodataframe...")
     #Perform a spatial join to get the WBD HUC 8 assigned to each AHPS
-    joined_gdf = gpd.sjoin(metadata_gdf, huc8, how = 'inner', op = 'intersects', lsuffix = 'ahps', rsuffix = 'wbd')
+    joined_gdf = gpd.sjoin(metadata_gdf, huc8, how = 'inner', predicate = 'intersects', lsuffix = 'ahps', rsuffix = 'wbd')
     joined_gdf = joined_gdf.drop(columns = 'index_wbd')
 
     #Remove all Alaska HUCS (Not in NWM v2.0 domain)
@@ -1340,7 +1340,7 @@ def process_extent(extent, profile, output_raster = False):
         
     #Convert extent to feature and explode geometry
     poly_extent = raster_to_feature(extent, profile, footprint_only = True)
-    poly_extent = poly_extent.explode()
+    poly_extent = poly_extent.explode(index_parts=True)
     
     #Fill holes in extent
     poly_extent_fill_holes=MultiPolygon(Polygon(p.exterior) for p in poly_extent['geometry'])
@@ -1351,7 +1351,7 @@ def process_extent(extent, profile, output_raster = False):
     #Dissolve filled holes with main map and explode
     poly_extent['dissolve_field'] = 1
     poly_extent = poly_extent.dissolve(by = 'dissolve_field')
-    poly_extent = poly_extent.explode()
+    poly_extent = poly_extent.explode(index_parts=True)
     poly_extent = poly_extent.reset_index()
     
     #Convert filled polygon back to raster

From a71a985ec5b675703918a5f26cc82c9b48bed100 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 3 Apr 2023 10:17:15 -0500
Subject: [PATCH 231/359] v4.3.5.1 HUC outputs not being cleaned and erroring
 out (#867)

---
 docs/CHANGELOG.md      | 14 ++++++++++++++
 fim_process_unit_wb.sh |  2 +-
 src/run_unit_wb.sh     |  2 +-
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 24f1a476e..3e0aa0db1 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.3.5.1 - 2023-04-01 - [PR#867](https://github.com/NOAA-OWP/inundation-mapping/pull/867)
+
+outputs_cleanup.py was throwing an error saying that the HUC source directory (to be cleaned up), did not exist. This was confirmed in a couple of environments. The src path in run_unit_wb.sh was sending in the "outputs" directory and not the "fim_temp" directory. This might have been a merge issue.
+
+The log file was moved to the unit_errors folder to validate the error, as expected.
+
+### Changes  
+
+- `src/run_unit_wb.sh`: Change the source path being submitted to `outputs_cleanup.py` from the `outputs` HUC directory to the `fim_temp` HUC directory.
+- `fim_process_unit_wb.sh`: Updated the phrase "Copied temp directory" to "Moved temp directory"
+
+<br/><br/>
+
 ## v4.3.5.0 - 2023-03-02 - [PR#857](https://github.com/NOAA-OWP/inundation-mapping/pull/857)
 
 Addresses changes to function calls needed to run upgraded Shapely library plus other related library upgrades. Upgraded libraries include:
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
index dc5fc86bd..e73f4d295 100755
--- a/fim_process_unit_wb.sh
+++ b/fim_process_unit_wb.sh
@@ -128,7 +128,7 @@ fi
 echo "============================================================================================="
 echo 
 mv -f $tempHucDataDir $outputHucDataDir
-echo "***** Copied temp directory: $tempHucDataDir to output directory: $outputHucDataDir  *****"
+echo "***** Moved temp directory: $tempHucDataDir to output directory: $outputHucDataDir  *****"
 echo
 echo "============================================================================================="
 
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 7ec0b9f94..aba913457 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -263,7 +263,7 @@ if [ -f $deny_unit_list ]; then
     echo -e $startDiv"Remove files $hucNumber"
     date -u
     Tstart
-    $srcDir/outputs_cleanup.py -d $outputHucDataDir -l $deny_unit_list -b $hucNumber
+    $srcDir/outputs_cleanup.py -d $tempHucDataDir -l $deny_unit_list -b $hucNumber
     Tcount
 fi
 

From 45f7f2d623475877aece815d36e67469e2d49898 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 14 Apr 2023 14:50:38 -0400
Subject: [PATCH 232/359] v4.3.6.0 Clip WBD to DEM domain (#858)

---
 data/wbd/preprocess_wbd.py | 54 ++++++++++++++++++++++++++++++++++++++
 docs/CHANGELOG.md          | 14 ++++++++++
 src/clip_vectors_to_wbd.py |  4 ---
 3 files changed, 68 insertions(+), 4 deletions(-)
 create mode 100755 data/wbd/preprocess_wbd.py

diff --git a/data/wbd/preprocess_wbd.py b/data/wbd/preprocess_wbd.py
new file mode 100755
index 000000000..3bf1b1da6
--- /dev/null
+++ b/data/wbd/preprocess_wbd.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import os
+from pathlib import Path
+import argparse
+import geopandas as gpd
+from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
+
+
+def clip_wbd_to_dem_domain(dem:str, wbd_in:str, wbd_out:str, huc_level:int):
+    """
+    Clips Watershed Boundary Dataset (WBD) to DEM domain
+
+    Parameters
+    ----------
+    dem: str
+        Path to DEM domain file
+    wbd_in: str
+        Path to WBD file input
+    wbd_out: str
+        Path to WBD file output
+    huc_level: int
+        HUC level
+    """
+
+    # Erase area outside 3DEP domain
+    if Path(wbd_in).is_file() and Path(dem).is_file():
+        layer = f'WBDHU{huc_level}'
+
+        # Read input files
+        wbd = gpd.read_file(wbd_in, layer=layer)
+        dem_domain = gpd.read_file(dem)
+
+        wbd = gpd.clip(wbd, dem_domain)
+
+        # Write output file
+        wbd.to_file(wbd_out, layer=layer, crs=DEFAULT_FIM_PROJECTION_CRS, driver='GPKG')
+    
+
+if __name__ == '__main__':
+    
+    parser = argparse.ArgumentParser(description='Clip WBD to DEM domain')
+    parser.add_argument('-d', '--dem', help='Path to DEM', type=str, required=True)
+    parser.add_argument('-w', '--wbd-in', help='Input WBD filename', type=str, required=True)
+    parser.add_argument('-o', '--wbd-out', help='Output WBD filename', type=str, required=True)
+    parser.add_argument('-l', '--huc-level', help='HUC level', type=int, required=True)
+
+    args = vars(parser.parse_args())
+
+    clip_wbd_to_dem_domain(**args)
+
+    # Example:
+    # preprocess_wbd.py -d /data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg -w /data/inputs/wbd/WBD_National_EPSG_5070.gpkg -o /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -l 8
+
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3e0aa0db1..4ef23d100 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.6.0 - 2023-03-23 - [PR#803](https://github.com/NOAA-OWP/inundation-mapping/pull/803)
+
+Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. Essentially, this is a wrapper for `geopandas.clip()` and moves clipping from `src/clip_vectors_to_wbd.py` to `data/wbd/preprocess_wbd.py`.
+
+### Additions
+
+- `data/wbd/preprocess_wbd.py`: Clips WBD to DEM domain polygon
+
+### Changes
+
+- `src/`
+    - `bash_variables.env`: Updates `input_WBD_gdb` environment variable
+    - `clip_vectors_to_wbd.py`: Removes clipping to DEM domain
+
 
 ## v4.3.5.1 - 2023-04-01 - [PR#867](https://github.com/NOAA-OWP/inundation-mapping/pull/867)
 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index c462d0ea6..a4f5a9576 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -37,12 +37,8 @@ def subset_vector_layers(subset_nwm_lakes,
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
 
-    # Erase area outside 3DEP domain
-    print("Erase area outside 3DEP domain", flush=True)
     wbd = gpd.read_file(wbd_filename)
     dem_domain = gpd.read_file(dem_domain)
-    wbd = gpd.clip(wbd, dem_domain)
-    wbd.to_file(wbd_filename, layer='WBDHU8', crs=DEFAULT_FIM_PROJECTION_CRS)
 
     # Get wbd buffer
     print("Create wbd buffer", flush=True)

From 83c23873488a06ee8993a84a88338085ace38ba4 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 14 Apr 2023 13:56:48 -0500
Subject: [PATCH 233/359] v4.3.7.0 post processing aggregate branch hydrotables
 to huc (#868)

---
 docs/CHANGELOG.md            |  17 +++
 fim_pipeline.sh              |   4 +-
 fim_post_processing.sh       |  51 ++++----
 src/add_crosswalk.py         |  52 ++++----
 src/aggregate_by_huc.py      | 227 +++++++++++++++++++++++++++++++++++
 src/subdiv_chan_obank_src.py |   4 +-
 src/usgs_gage_aggregate.py   |  98 ---------------
 tools/inundate_gms.py        |  18 ++-
 tools/inundation.py          |  23 ++--
 9 files changed, 331 insertions(+), 163 deletions(-)
 create mode 100644 src/aggregate_by_huc.py
 delete mode 100644 src/usgs_gage_aggregate.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4ef23d100..42b38aae4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.7.0 - 2023-03-02 - [PR#868](https://github.com/NOAA-OWP/inundation-mapping/pull/868)
+
+This pull request adds a new feature to `fim_post_processing.sh` to aggregate all of the hydrotables for a given HUC into a single HUC-level `hydrotable.csv` file. Note that the aggregation step happens near the end of `fim_post_processing.sh` (after the subdivision and calibration routines), and the branch hydrotable files are preserved in the branch directories for the time being.
+
+### Changes  
+
+- `fim_pipeline.sh`: created a new variable `$jobMaxLimit` that multiplies the `$jobHucLimit` and the `$jobBranchLimit`
+- `fim_post_processing.sh`: added new aggregation/concatenation step after the SRC calibration routines; passing the new `$jobMaxLimit` to the commands that accept a multiprocessing job number input; added `$skipcal` argument to the USGS rating curve calibration routine
+- `src/add_crosswalk.py`: changed the default value for `calb_applied` variable to be a boolean
+- `src/aggregate_by_huc.py`: file renamed (previous name: `src/usgs_gage_aggregate.py`); updated to perform branch to huc file aggregation for `hydroTable_{branch_id}.csv` and `src_full_crosswalked_{branch_id}.csv` files; note that the input arguments ask you to specify which file types to aggregate using the flags: `-elev`, `-htable`, and `-src`
+- `tools/inundate_gms.py`: added check to use the aggregated HUC-level `hydrotable.csv` if it exists, otherwise continue to use the branch hydroTable files
+- `tools/inundation.py`: added `usecols` argument to the `pd.read_csv` commands to improve read time for hydrotables
+- `src/subdiv_chan_obank_src.py`: add dtype to hydrotable pd.read_csv to resolve pandas dtype interpretation warnings
+
+<br/><br/>
+
 ## v4.3.6.0 - 2023-03-23 - [PR#803](https://github.com/NOAA-OWP/inundation-mapping/pull/803)
 
 Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. Essentially, this is a wrapper for `geopandas.clip()` and moves clipping from `src/clip_vectors_to_wbd.py` to `data/wbd/preprocess_wbd.py`.
@@ -15,6 +31,7 @@ Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. E
     - `bash_variables.env`: Updates `input_WBD_gdb` environment variable
     - `clip_vectors_to_wbd.py`: Removes clipping to DEM domain
 
+<br/><br/>
 
 ## v4.3.5.1 - 2023-04-01 - [PR#867](https://github.com/NOAA-OWP/inundation-mapping/pull/867)
 
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index a783ae92b..fe1ddeb69 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -32,6 +32,7 @@ echo "---- Started: `date -u`"
 ## LOAD AND VALIDATE INCOMING ARGUMENTS
 source $srcDir/bash_functions.env
 . $projectDir/fim_pre_processing.sh "$@"
+jobMaxLimit=$(( $jobHucLimit * $jobBranchLimit ))
 
 logFile=$outputDestDir/logs/unit/pipeline_summary_unit.log
 process_wb_file=$projectDir/fim_process_unit_wb.sh
@@ -60,8 +61,7 @@ date -u
 
 ## POST PROCESSING
 
-# TODO: multiply the two job limits together for the limit here ??
-. $projectDir/fim_post_processing.sh -n $runName -j $jobHucLimit
+. $projectDir/fim_post_processing.sh -n $runName -j $jobMaxLimit
 
 echo
 echo "======================== End of fim_pipeline.sh =========================="
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 7ed679be6..fce7a2b21 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -93,7 +93,7 @@ find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "
 
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
 echo "Processing usgs gage aggregation"
-python3 $srcDir/usgs_gage_aggregate.py -fim $outputDestDir -i $fim_inputs
+python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev
 
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
@@ -113,6 +113,17 @@ if [ "$src_subdiv_toggle" = "True" ]; then
     Tcount
 fi
 
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
+if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$skipcal" = "0" ]; then
+    Tstart
+    echo    
+    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
+    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
+    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputDestDir -usgs_rc $inputsDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+    Tcount
+    date -u
+fi
+
 ## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
 if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
     if [ ! -f $CALB_DB_KEYS_FILE ]; then
@@ -122,15 +133,13 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
     else
         source $CALB_DB_KEYS_FILE
 
-        : '
-        This makes the local variables from the calb_db_keys files
-        into global variables that can be used in other files, including python.
+        ## This makes the local variables from the calb_db_keys files
+        ## into global variables that can be used in other files, including python.
 
-        Why not just leave the word export in front of each of the keys in the
-        calb_db_keys.env? Becuase that file is used against docker-compose
-        when we start up that part of the sytem and it does not like the word
-        export.
-        '
+        ## Why not just leave the word export in front of each of the keys in the
+        ## calb_db_keys.env? Because that file is used against docker-compose        
+        ## when we start up that part of the sytem and it does not like the word
+        ## export.
 
         # Pick up the docker parent host machine name and override the one coming from the config file (aws only)
         if [ "$isAWS" = "1" ]; then
@@ -144,9 +153,8 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
         export DEFAULT_FIM_PROJECTION_CRS=$DEFAULT_FIM_PROJECTION_CRS
 
         Tstart
-        echo "Populate PostgrSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
-        echo "Loading HUC Data"
         echo
+        echo -e $startDiv"Populate PostgreSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"        echo
 
         ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputsDir/wbd/WBD_National.gpkg WBDHU8
 
@@ -160,17 +168,6 @@ else
     echo "Skipping Populate PostgrSQL database"
 fi
 
-## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
-if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ]; then
-    Tstart
-    echo    
-    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
-    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
-    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputDestDir -usgs_rc $inputsDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
-    Tcount
-    date -u
-fi
-
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
 if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [ "$skipcal" = "0" ]; then
     Tstart
@@ -181,6 +178,16 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [
     date -u
 fi
 
+## AGGREGATE BRANCH TABLES ##
+# TODO: How do we skip aggregation if there is a branch error
+# maybe against the non_zero logs above
+echo 
+echo -e $startDiv"Aggregating branch hydrotables"
+Tstart
+python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -htable
+Tcount
+date -u
+
 echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 6a5034853..f75da3618 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -260,7 +260,7 @@ def add_crosswalk(input_catchments_fileName,
     output_hydro_table['channel_n'] = pd.NA
     output_hydro_table['subdiv_discharge_cms'] = pd.NA
     ## Placeholder vars for the calibration routine
-    output_hydro_table['calb_applied'] = pd.NA
+    output_hydro_table['calb_applied'] = False
     output_hydro_table['last_updated'] = pd.NA
     output_hydro_table['submitter'] = pd.NA
     output_hydro_table['obs_source'] = pd.NA
@@ -285,33 +285,29 @@ def add_crosswalk(input_catchments_fileName,
     if output_hydro_table.feature_id.dtype != 'int': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(int)
     if output_hydro_table.feature_id.dtype != 'str': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(str)
 
-    # write out based on mode
-    if calibration_mode == True:
-        output_hydro_table.to_csv(output_hydro_table_fileName,index=False)
-    else:
-        # make src json
-        output_src_json = dict()
-        hydroID_list = unique(output_src['HydroID'])
-
-        for hid in hydroID_list:
-            indices_of_hid = output_src['HydroID'] == hid
-            stage_list = output_src['Stage'][indices_of_hid].astype(float)
-            q_list = output_src['Discharge (m3s-1)'][indices_of_hid].astype(float)
-
-            stage_list = stage_list.tolist()
-            q_list = q_list.tolist()
-
-            output_src_json[str(hid)] = { 'q_list' : q_list , 'stage_list' : stage_list }
-
-        # write out
-        output_catchments.to_file(output_catchments_fileName,driver=getDriver(output_catchments_fileName),index=False)
-        output_flows.to_file(output_flows_fileName,driver=getDriver(output_flows_fileName),index=False)
-        output_src.to_csv(output_src_fileName,index=False)
-        output_crosswalk.to_csv(output_crosswalk_fileName,index=False)
-        output_hydro_table.to_csv(output_hydro_table_fileName,index=False)
-
-        with open(output_src_json_fileName,'w') as f:
-            json.dump(output_src_json,f,sort_keys=True,indent=2)
+    # make src json
+    output_src_json = dict()
+    hydroID_list = unique(output_src['HydroID'])
+
+    for hid in hydroID_list:
+        indices_of_hid = output_src['HydroID'] == hid
+        stage_list = output_src['Stage'][indices_of_hid].astype(float)
+        q_list = output_src['Discharge (m3s-1)'][indices_of_hid].astype(float)
+
+        stage_list = stage_list.tolist()
+        q_list = q_list.tolist()
+
+        output_src_json[str(hid)] = { 'q_list' : q_list , 'stage_list' : stage_list }
+
+    # write out
+    output_catchments.to_file(output_catchments_fileName,driver=getDriver(output_catchments_fileName),index=False)
+    output_flows.to_file(output_flows_fileName,driver=getDriver(output_flows_fileName),index=False)
+    output_src.to_csv(output_src_fileName,index=False)
+    output_crosswalk.to_csv(output_crosswalk_fileName,index=False)
+    output_hydro_table.to_csv(output_hydro_table_fileName,index=False)
+
+    with open(output_src_json_fileName,'w') as f:
+        json.dump(output_src_json,f,sort_keys=True,indent=2)
 
 
 if __name__ == '__main__':
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
new file mode 100644
index 000000000..261b04280
--- /dev/null
+++ b/src/aggregate_by_huc.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+
+import os
+from os.path import join
+import pandas as pd
+import re
+import argparse
+
+class HucDirectory(object):
+
+    def __init__(self, path, limit_branches=[]):
+
+        self.dir = path
+        self.name = os.path.basename(path)
+        self.limit_branches = limit_branches
+
+        self.usgs_dtypes = {'location_id':str,
+                            'nws_lid':str,
+                            'feature_id':int,
+                            'HydroID':int,
+                            'levpa_id':str,
+                            'dem_elevation':float,
+                            'dem_adj_elevation':float,
+                            'order_':str,
+                            'LakeID':object,
+                            'HUC8':str,
+                            'snap_distance':float}
+        self.agg_usgs_elev_table = pd.DataFrame(columns=list(self.usgs_dtypes.keys()))
+
+        self.hydrotable_dtypes = {'HydroID':int,
+                                  'branch_id':int,
+                                  'feature_id':int,
+                                  'NextDownID':int,
+                                  'order_':int,
+                                  'Number of Cells':int,
+                                  'SurfaceArea (m2)':float,
+                                  'BedArea (m2)':float,
+                                  'TopWidth (m)':float,
+                                  'LENGTHKM':float,
+                                  'AREASQKM':float,
+                                  'WettedPerimeter (m)':float,
+                                  'HydraulicRadius (m)':float,
+                                  'WetArea (m2)':float,
+                                  'Volume (m3)':float,
+                                  'SLOPE':float,
+                                  'ManningN':float,
+                                  'stage':float,
+                                  'default_discharge_cms':float,
+                                  'default_Volume (m3)':float,
+                                  'default_WetArea (m2)':float,
+                                  'default_HydraulicRadius (m)':float,
+                                  'default_ManningN':float,
+                                  'calb_applied':bool,
+                                  'last_updated':str,
+                                  'submitter':str,
+                                  'obs_source':str,
+                                  'precalb_discharge_cms':float,
+                                  'calb_coef_usgs':float,
+                                  'calb_coef_spatial':float,
+                                  'calb_coef_final':float,
+                                  'HUC':int,
+                                  'LakeID':int,
+                                  'subdiv_applied':bool,
+                                  'channel_n':float,
+                                  'overbank_n':float,
+                                  'subdiv_discharge_cms':float,
+                                  'discharge_cms':float}
+        self.agg_hydrotable = pd.DataFrame(columns=list(self.hydrotable_dtypes.keys()))
+
+        self.src_crosswalked_dtypes = {'branch_id':int,
+                                    'HydroID':int,
+                                    'feature_id':int,
+                                    'Stage':float,
+                                    'Number of Cells':int,
+                                    'SurfaceArea (m2)':float,
+                                    'BedArea (m2)':float,
+                                    'Volume (m3)':float,
+                                    'SLOPE':float,
+                                    'LENGTHKM':float,
+                                    'AREASQKM':float,
+                                    'ManningN':float,
+                                    'NextDownID':int,
+                                    'order_':int,
+                                    'TopWidth (m)':float,
+                                    'WettedPerimeter (m)':float,
+                                    'WetArea (m2)':float,
+                                    'HydraulicRadius (m)':float,
+                                    'Discharge (m3s-1)':float,
+                                    'bankfull_flow':float,
+                                    'Stage_bankfull':float,
+                                    'BedArea_bankfull':float,
+                                    'Volume_bankfull':float,
+                                    'HRadius_bankfull':float,
+                                    'SurfArea_bankfull':float,
+                                    'bankfull_proxy':str,
+                                    'Volume_chan (m3)':float,
+                                    'BedArea_chan (m2)':float,
+                                    'WettedPerimeter_chan (m)':float,
+                                    'Volume_obank (m3)':float,
+                                    'BedArea_obank (m2)':float,
+                                    'WettedPerimeter_obank (m)':float,
+                                    'channel_n':float,
+                                    'overbank_n':float,
+                                    'subdiv_applied':bool,
+                                    'WetArea_chan (m2)':float,
+                                    'HydraulicRadius_chan (m)':float,
+                                    'Discharge_chan (m3s-1)':float,
+                                    'Velocity_chan (m/s)':float,
+                                    'WetArea_obank (m2)':float,
+                                    'HydraulicRadius_obank (m)':float,
+                                    'Discharge_obank (m3s-1)':float,
+                                    'Velocity_obank (m/s)':float,
+                                    'Discharge (m3s-1)_subdiv':float}
+        self.agg_src_cross = pd.DataFrame(columns=list(self.src_crosswalked_dtypes.keys()))
+
+    def iter_branches(self):
+
+        if self.limit_branches:
+            for branch in self.limit_branches:
+                yield (branch, join(self.dir, 'branches', branch))
+
+        else:
+            for branch in os.listdir(join(self.dir, 'branches')):
+                yield (branch, join(self.dir, 'branches', branch))
+
+    def usgs_elev_table(self, branch_path):
+
+        usgs_elev_filename = join(branch_path, 'usgs_elev_table.csv')
+        if not os.path.isfile(usgs_elev_filename):
+            return
+
+        usgs_elev_table = pd.read_csv(usgs_elev_filename, dtype=self.usgs_dtypes)
+        self.agg_usgs_elev_table = self.agg_usgs_elev_table.append(usgs_elev_table)
+
+
+    def aggregate_hydrotables(self, branch_path, branch_id):
+
+        hydrotable_filename = join(branch_path, f'hydroTable_{branch_id}.csv')
+        if not os.path.isfile(hydrotable_filename):
+            return
+
+        hydrotable = pd.read_csv(hydrotable_filename, dtype=self.hydrotable_dtypes)
+        hydrotable['branch_id'] = branch_id
+        hydrotable[['calb_applied']] = hydrotable[['calb_applied']].fillna(value=False)
+        self.agg_hydrotable = self.agg_hydrotable.append(hydrotable)
+
+    def aggregate_src_full_crosswalk(self, branch_path, branch_id):
+
+        src_cross_filename = join(branch_path, f'src_full_crosswalked_{branch_id}.csv')
+        if not os.path.isfile(src_cross_filename):
+            return
+
+        src_cross = pd.read_csv(src_cross_filename, dtype=self.src_crosswalked_dtypes)
+        src_cross['branch_id'] = branch_id
+        self.agg_src_cross = self.agg_src_cross.append(src_cross)
+
+
+    def agg_function(self,usgs_elev_flag,hydro_table_flag,src_cross_flag):
+
+        for branch_id, branch_path in self.iter_branches():
+            if usgs_elev_flag:
+                self.usgs_elev_table(branch_path)
+
+            ## Other aggregate funtions can go here
+            if hydro_table_flag:
+                self.aggregate_hydrotables(branch_path, branch_id)
+            if src_cross_flag:
+                self.aggregate_src_full_crosswalk(branch_path, branch_id)
+
+        ## After all of the branches are visited, the code below will write the aggregates
+        if usgs_elev_flag:
+            usgs_elev_table_file = join(self.dir, 'usgs_elev_table.csv')
+            if os.path.isfile(usgs_elev_table_file):
+                os.remove(usgs_elev_table_file)
+
+            if not self.agg_usgs_elev_table.empty:
+                self.agg_usgs_elev_table.to_csv(usgs_elev_table_file, index=False)
+
+        if hydro_table_flag:
+            hydrotable_file = join(self.dir, 'hydrotable.csv')
+            if os.path.isfile(hydrotable_file):
+                os.remove(hydrotable_file)
+
+            if not self.agg_hydrotable.empty:
+                self.agg_hydrotable.to_csv(hydrotable_file, index=False)
+
+        if src_cross_flag:
+            src_crosswalk_file = join(self.dir, 'src_full_crosswalked.csv')
+            if os.path.isfile(src_crosswalk_file):
+                os.remove(src_crosswalk_file)
+
+            if not self.agg_src_cross.empty:
+                self.agg_src_cross.to_csv(src_crosswalk_file, index=False)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Aggregates usgs_elev_table.csv at the HUC level')
+    parser.add_argument('-fim','--fim_directory', help='Input FIM Directory', required=True)
+    parser.add_argument('-i','--fim_inputs', help='Input fim_inputs CSV file', required=False)
+    parser.add_argument('-elev','--usgs_elev', help='Perform aggregate on branch usgs elev tables', required=False, default=False, action='store_true')
+    parser.add_argument('-htable','--hydro_table', help='Perform aggregate on branch hydrotables', required=False, default=False, action='store_true')
+    parser.add_argument('-src','--src_cross', help='Perform aggregate on branch src crosswalk files', required=False, default=False, action='store_true')
+
+    args = vars(parser.parse_args())
+
+    fim_directory = args['fim_directory']
+    fim_inputs = args['fim_inputs']
+    usgs_elev_flag = args['usgs_elev']
+    hydro_table_flag = args['hydro_table']
+    src_cross_flag = args['src_cross']
+    assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
+
+    if fim_inputs:
+        fim_inputs = pd.read_csv(fim_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
+
+        for huc in fim_inputs.huc.unique():
+
+            branches = fim_inputs.loc[fim_inputs.huc == huc, 'levpa_id'].tolist()
+            huc = HucDirectory(join(fim_directory, huc), limit_branches=branches)
+            huc.agg_function(usgs_elev_flag,hydro_table_flag,src_cross_flag)
+
+    else:
+        for huc_dir in [d for d in os.listdir(fim_directory) if re.match('\d{8}', d)]:
+
+            huc = HucDirectory(join(fim_directory, huc_dir))
+            huc.agg_function(usgs_elev_flag,hydro_table_flag,src_cross_flag)
\ No newline at end of file
diff --git a/src/subdiv_chan_obank_src.py b/src/subdiv_chan_obank_src.py
index ae09508df..6f1d41133 100755
--- a/src/subdiv_chan_obank_src.py
+++ b/src/subdiv_chan_obank_src.py
@@ -90,7 +90,7 @@ def variable_mannings_calc(args):
             df_src_trim = df_src[['HydroID','Stage','subdiv_applied','channel_n','overbank_n','Discharge (m3s-1)_subdiv']]
             df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_subdiv': 'subdiv_discharge_cms'})
             df_src_trim['discharge_cms'] = df_src_trim['subdiv_discharge_cms'] # create a copy of vmann modified discharge (used to track future changes)
-            df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
+            df_htable = pd.read_csv(htable_filename,dtype={'HUC': str,'last_updated':object,'submitter':object,'obs_source':object})
 
             ## drop the previously modified discharge column to be replaced with updated version
             df_htable.drop(['subdiv_applied','discharge_cms','overbank_n','channel_n','subdiv_discharge_cms'], axis=1, errors='ignore', inplace=True) 
@@ -191,7 +191,7 @@ def multi_process(variable_mannings_calc, procs_list, log_file, number_of_jobs,
         number_of_jobs = available_cores - 2
         print("Provided job number exceeds the number of available cores. " + str(number_of_jobs) + " max jobs will be used instead.")
 
-    print(f"Computing subdivided SRC and applying variable Manning's n to channel/overbank for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    print(f"Computing subdivided SRC and applying variable Manning's n to channel/overbank for {len(procs_list)} branches using {number_of_jobs} jobs")
     with Pool(processes=number_of_jobs) as pool:
         if verbose:
             map_output = tqdm(pool.imap(variable_mannings_calc, procs_list), total=len(procs_list))
diff --git a/src/usgs_gage_aggregate.py b/src/usgs_gage_aggregate.py
deleted file mode 100644
index 1a8993e83..000000000
--- a/src/usgs_gage_aggregate.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-from os.path import join
-import pandas as pd
-import re
-import argparse
-
-class HucDirectory(object):
-
-    def __init__(self, path, limit_branches=[]):
-
-        self.dir = path
-        self.name = os.path.basename(path)
-        self.limit_branches = limit_branches
-
-        self.usgs_dtypes = {'location_id':str,
-                            'nws_lid':str,
-                            'feature_id':int,
-                            'HydroID':int,
-                            'levpa_id':str,
-                            'dem_elevation':float,
-                            'dem_adj_elevation':float,
-                            'order_':str,
-                            'LakeID':object,
-                            'HUC8':str,
-                            'snap_distance':float}
-        self.agg_usgs_elev_table = pd.DataFrame(columns=list(self.usgs_dtypes.keys()))
-
-    def iter_branches(self):
-
-        if self.limit_branches:
-            for branch in self.limit_branches:
-                yield (branch, join(self.dir, 'branches', branch))
-
-        else:
-            for branch in os.listdir(join(self.dir, 'branches')):
-                yield (branch, join(self.dir, 'branches', branch))
-
-    def usgs_elev_table(self, branch_path):
-
-        usgs_elev_filename = join(branch_path, 'usgs_elev_table.csv')
-        if not os.path.isfile(usgs_elev_filename):
-            return
-
-        usgs_elev_table = pd.read_csv(usgs_elev_filename, dtype=self.usgs_dtypes)
-        self.agg_usgs_elev_table = self.agg_usgs_elev_table.append(usgs_elev_table)
-
-
-    def agg_function(self):
-
-        for branch_id, branch_path in self.iter_branches():
-
-            self.usgs_elev_table(branch_path)
-
-            ## Other aggregate funtions can go here
-        
-        ## After all of the branches are visited, the code below will write the aggregates
-        if os.path.isfile(join(self.dir, 'usgs_elev_table.csv')):
-            os.remove(join(self.dir, 'usgs_elev_table.csv'))
-
-        if not self.agg_usgs_elev_table.empty:
-            self.agg_usgs_elev_table.to_csv(join(self.dir, 'usgs_elev_table.csv'), index=False)
-        
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Aggregates usgs_elev_table.csv at the HUC level')
-    parser.add_argument('-fim','--fim_directory', help='Input FIM Directory', required=True)
-    parser.add_argument('-i','--fim_inputs', help='Input fim_inputs CSV file', required=False)
-
-    args = vars(parser.parse_args())
-
-    fim_directory = args['fim_directory']
-    fim_inputs = args['fim_inputs']
-    assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
-
-    if fim_inputs:
-        fim_inputs = pd.read_csv(fim_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
-
-        for huc in fim_inputs.huc.unique():
-
-            branches = fim_inputs.loc[fim_inputs.huc == huc, 'levpa_id'].tolist()
-            huc = HucDirectory(join(fim_directory, huc), limit_branches=branches)
-            huc.agg_function()
-
-    else:
-        for huc_dir in [d for d in os.listdir(fim_directory) if re.match('\d{8}', d)]:
-
-            huc = HucDirectory(join(fim_directory, huc_dir))
-            huc.agg_function()
-
-
-        
-
-
-
diff --git a/tools/inundate_gms.py b/tools/inundate_gms.py
index e1a90fa67..b0f5363f6 100755
--- a/tools/inundate_gms.py
+++ b/tools/inundate_gms.py
@@ -171,8 +171,22 @@ def __inundate_gms_generator( hucs_branches,
 
         catchments_file_name = f'gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif'
         catchments_branch = os.path.join( huc_dir, branch_id, catchments_file_name )
-
-        hydroTable_branch = os.path.join( huc_dir, branch_id, 'hydroTable_{}.csv'.format(branch_id) )
+        
+        # FIM versions > 4.3.5 use an aggregated hydrotable file rather than individual branch hydrotables
+        hydroTable_huc = os.path.join( huc_dir, 'hydrotable.csv' )
+        if os.path.isfile(hydroTable_huc):
+            htable_req_cols = ['HUC','branch_id','feature_id','HydroID','stage','discharge_cms','LakeID']
+            hydroTable_all = pd.read_csv(
+                                 hydroTable_huc,
+                                 dtype={'HUC':str,'branch_id':int,'feature_id':str,
+                                         'HydroID':str,'stage':float,
+                                         'discharge_cms':float,'LakeID' : int},
+                                usecols=htable_req_cols
+                                )
+            hydroTable_branch = hydroTable_all.loc[hydroTable_all['branch_id'] == int(branch_id)]
+        else:
+            # Earlier FIM4 versions only have branch level hydrotables
+            hydroTable_branch = os.path.join( huc_dir, branch_id, 'hydroTable_{}.csv'.format(branch_id) )
 
         xwalked_file_name = f'gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg'
         catchment_poly = os.path.join( huc_dir, branch_id, xwalked_file_name )
diff --git a/tools/inundation.py b/tools/inundation.py
index 455c6f6f9..66633bde2 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -476,16 +476,20 @@ def __append_huc_code_to_file_name(fileName,hucCode):
 def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
 
     if isinstance(hydroTable,str):
+        htable_req_cols = ['HUC','feature_id','HydroID','stage','discharge_cms','LakeID']
         hydroTable = pd.read_csv(
                                  hydroTable,
-                                 dtype={'HUC':str,'feature_id':str,
-                                         'HydroID':str,'stage':float,
-                                         'discharge_cms':float,
-                                         'LakeID' : int,
-                                         'last_updated':object, 
-                                         'submitter':object, 
-                                         'obs_source':object},
-                                 low_memory=False
+                                 dtype={'HUC':str,
+                                        'feature_id':str,
+                                        'HydroID':str,
+                                        'stage':float,
+                                        'discharge_cms':float,
+                                        'LakeID' : int,
+                                        'last_updated':object, 
+                                        'submitter':object, 
+                                        'obs_source':object},
+                                 low_memory=False,
+                                 usecols=htable_req_cols
                                 )
         huc_error = hydroTable.HUC.unique()
         hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
@@ -629,7 +633,8 @@ def create_src_subset_csv(hydro_table,catchmentStagesDict,src_table):
     src_df = pd.DataFrame.from_dict(catchmentStagesDict, orient='index')
     src_df.reset_index(inplace=True)
     src_df.columns = ['HydroID','stage_inund']
-    df_htable = pd.read_csv(hydro_table,dtype={'HydroID': int,'HUC': object, 'branch_id':int, 'last_updated':object, 'submitter':object, 'obs_source':object})
+    htable_req_cols = ['HUC','feature_id','HydroID','stage','discharge_cms','LakeID']
+    df_htable = pd.read_csv(hydro_table,dtype={'HydroID': int,'HUC': object, 'branch_id':int, 'last_updated':object, 'submitter':object, 'obs_source':object},usecols=htable_req_cols)
     df_htable = df_htable.merge(src_df,how='left',on='HydroID')
     df_htable['find_match'] = (df_htable['stage'] - df_htable['stage_inund']).abs()
     df_htable = df_htable.loc[df_htable.groupby('HydroID')['find_match'].idxmin()].reset_index(drop=True)

From 6029629d26e78a81383a88fa7b21aeee4017140e Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 14 Apr 2023 14:05:58 -0500
Subject: [PATCH 234/359] v4.3.7.1 Hotfix for branch exit code 61 failing with
 temp directories (#874)

---
 docs/CHANGELOG.md     | 11 +++++++++++
 fim_pipeline.sh       |  4 ++++
 src/process_branch.sh |  2 +-
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 42b38aae4..f9176a636 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.7.1 - 2023-04-06 - [PR#874](https://github.com/NOAA-OWP/inundation-mapping/pull/874)
+
+Hotfix to `process_branch.sh` because it wasn't removing code-61 branches on exit. Also removes the current run from the new fim_temp directory.
+
+### Changes  
+
+- `fim_pipeline.sh`: removal of current run from fim_temp directory
+- `src/process_branch.sh`: switched the exit 61 block to use the temp directory instead of the outputs directory
+
+<br/><br/>
+
 ## v4.3.7.0 - 2023-03-02 - [PR#868](https://github.com/NOAA-OWP/inundation-mapping/pull/868)
 
 This pull request adds a new feature to `fim_post_processing.sh` to aggregate all of the hydrotables for a given HUC into a single HUC-level `hydrotable.csv` file. Note that the aggregation step happens near the end of `fim_post_processing.sh` (after the subdivision and calibration routines), and the branch hydrotable files are preserved in the branch directories for the time being.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index fe1ddeb69..ca935cde7 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -61,6 +61,10 @@ date -u
 
 ## POST PROCESSING
 
+# Remove run from the fim_temp directory
+rm -d $workDir/$runName
+
+# Pipe into post processing 
 . $projectDir/fim_post_processing.sh -n $runName -j $jobMaxLimit
 
 echo
diff --git a/src/process_branch.sh b/src/process_branch.sh
index 4b1ff0a52..3cf0365de 100755
--- a/src/process_branch.sh
+++ b/src/process_branch.sh
@@ -37,7 +37,7 @@ do
         echo
         err_exists=1
         echo "***** Branch has no valid flowlines *****"
-        rm -rf $outputRunDataDir/$hucNumber/branches/$branchId/
+        rm -rf $tempHucDataDir/branches/$branchId/
     elif [ $code -ne 0 ]; then
         echo
         err_exists=1

From 2b9256ce0fd7e7edfc5de21c8066434310a9f6e5 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 14 Apr 2023 15:07:24 -0400
Subject: [PATCH 235/359] v4.3.7.2 Replace os.environ with input arguments
 (#879)

---
 docs/CHANGELOG.md                        | 15 +++++++++++++++
 fim_pre_processing.sh                    |  2 +-
 src/add_crosswalk.py                     |  8 ++++++--
 src/check_huc_inputs.py                  |  5 +++--
 src/delineate_hydros_and_produce_HAND.sh |  4 ++--
 src/split_flows.py                       | 13 ++++++++-----
 6 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f9176a636..d06bfdd7b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -44,6 +44,21 @@ Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. E
 
 <br/><br/>
 
+## v4.3.7.2 - 2023-04-06 - [PR#879](https://github.com/NOAA-OWP/inundation-mapping/pull/879)
+
+Replaces `os.environ` with input arguments in Python files that are called from bash scripts. The bash scripts now access the environment variables and pass them to the Python files as input arguments. In addition to adapting some Python scripts to a more modular structure which allows them to be run individually, it also allows Visual Studio Code debugger to work properly. Closes #875.
+
+### Changes
+
+- `fim_pre_processing.sh`: Added `-i $inputsDir` input argument to `src/check_huc_inputs.py`
+- `src/`
+    - `add_crosswalk.py`: Changed `min_catchment_area` and `min_stream_length` environment variables to input arguments
+    - `check_huc_inputs.py`: Changed `inputsDir` environment variable to input argument
+    - `delineate_hydros_and_produce_HAND.sh`: Added `-m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters` input arguments to `src/split_flows.py`
+    - `split_flows.py`: Changed `max_split_distance_meters`, `slope_min`, and `lakes_buffer_dist_meters` from environment variables to input arguments
+
+<br/><br/>
+
 ## v4.3.5.1 - 2023-04-01 - [PR#867](https://github.com/NOAA-OWP/inundation-mapping/pull/867)
 
 outputs_cleanup.py was throwing an error saying that the HUC source directory (to be cleaned up), did not exist. This was confirmed in a couple of environments. The src path in run_unit_wb.sh was sending in the "outputs" directory and not the "fim_temp" directory. This might have been a merge issue.
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 22f6d704d..3fd2fbda7 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -194,7 +194,7 @@ source $srcDir/bash_functions.env
 export runName=$runName
 export jobHucLimit=$jobHucLimit
 
-num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList)
+num_hucs=$(python3 $srcDir/check_huc_inputs.py -u $hucList -i $inputsDir)
 echo
 echo "--- Number of HUCs to process is $num_hucs"
 
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index f75da3618..b550d97b4 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -35,14 +35,16 @@ def add_crosswalk(input_catchments_fileName,
                   input_nwmcat_fileName,
                   extent,
                   small_segments_filename,
+                  min_catchment_area,
+                  min_stream_length,
                   calibration_mode=False):
 
     input_catchments = gpd.read_file(input_catchments_fileName)
     input_flows = gpd.read_file(input_flows_fileName)
     input_huc = gpd.read_file(input_huc_fileName)
     input_nwmflows = gpd.read_file(input_nwmflows_fileName)
-    min_catchment_area = float(os.environ['min_catchment_area']) #0.25#
-    min_stream_length = float(os.environ['min_stream_length']) #0.5#
+    min_catchment_area = float(min_catchment_area) #0.25#
+    min_stream_length = float(min_stream_length) #0.5#
 
     if extent == 'FR':
         ## crosswalk using majority catchment method
@@ -330,6 +332,8 @@ def add_crosswalk(input_catchments_fileName,
     parser.add_argument('-p','--extent',help='GMS only for now', default='GMS', required=False)
     parser.add_argument('-k','--small-segments-filename',help='output list of short segments',required=True)
     parser.add_argument('-c','--calibration-mode',help='Mannings calibration flag',required=False,action='store_true')
+    parser.add_argument('-e', '--min-catchment-area', help='Minimum catchment area', required=True)
+    parser.add_argument('-g', '--min-stream-length', help='Minimum stream length', required=True)
 
     args = vars(parser.parse_args())
 
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index a31296390..d1f4d2708 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -67,9 +67,9 @@ def __check_for_membership(hucs,accepted_hucs_set):
             raise KeyError("HUC {} not found in available inputs. Edit HUC inputs or acquire datasets and try again".format(huc))
 
 
-def check_hucs(hucs):
+def check_hucs(hucs, inputsDir):
     
-    huc_list_path = os.path.join(os.environ['inputsDir'],'huc_lists')
+    huc_list_path = os.path.join(inputsDir, 'huc_lists')
     accepted_hucs = __read_included_files(huc_list_path)
     list_hucs = __read_input_hucs(hucs)
     __check_for_membership(list_hucs, accepted_hucs)
@@ -87,6 +87,7 @@ def check_hucs(hucs):
     # parse arguments
     parser = argparse.ArgumentParser(description='Checks input hucs for availability within inputs')
     parser.add_argument('-u','--hucs',help='Line-delimited file or list of HUCs to check availibility for',required=True, nargs='+')
+    parser.add_argument('-i', '--inputsDir', help='Inputs directory', required=True)
 
     # extract to dictionary
     args = vars(parser.parse_args())
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index dc6d9482d..6209328c4 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -75,7 +75,7 @@ Tcount
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg -n $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg 
+$srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg -n $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg -m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
@@ -184,5 +184,5 @@ elif [ "$level" = "unit" ]; then
     z_arg=$tempHucDataDir/nwm_catchments_proj_subset.gpkg
 fi
 
-python3 -m memory_profiler $srcDir/add_crosswalk.py -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $tempCurrentBranchDataDir/src_base_$current_branch_id.csv -l $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $tempCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $tempCurrentBranchDataDir/src_$current_branch_id.json -x $tempCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $tempCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $tempHucDataDir/wbd8_clp.gpkg -b $b_arg -y $tempCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -k $tempCurrentBranchDataDir/small_segments_$current_branch_id.csv
+python3 -m memory_profiler $srcDir/add_crosswalk.py -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $tempCurrentBranchDataDir/src_base_$current_branch_id.csv -l $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $tempCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $tempCurrentBranchDataDir/src_$current_branch_id.json -x $tempCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $tempCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $tempHucDataDir/wbd8_clp.gpkg -b $b_arg -y $tempCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -k $tempCurrentBranchDataDir/small_segments_$current_branch_id.csv -e $min_catchment_area -g $min_stream_length
 Tcount
diff --git a/src/split_flows.py b/src/split_flows.py
index e774bb939..7c34dca4b 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -275,10 +275,6 @@ def split_flows(max_length,
 
 
 if __name__ == '__main__':
-    max_length             = float(environ['max_split_distance_meters'])
-    slope_min              = float(environ['slope_min'])
-    lakes_buffer_input     = float(environ['lakes_buffer_dist_meters'])
-    
     # Parse arguments.
     parser = argparse.ArgumentParser(description='splitflows.py')
     parser.add_argument('-f', '--flows-filename', help='flows-filename',required=True)
@@ -288,8 +284,15 @@ def split_flows(max_length,
     parser.add_argument('-w', '--wbd8-clp-filename', help='wbd8-clp-filename',required=True)
     parser.add_argument('-l', '--lakes-filename', help='lakes-filename',required=True)
     parser.add_argument('-n', '--nwm-streams-filename', help='nwm-streams-filename',required=True)
+    parser.add_argument('-m', '--max-length', help='Maximum split distance (meters)', required=True)
+    parser.add_argument('-t', '--slope-min', help='Slope minimum', required=True)
+    parser.add_argument('-b', '--lakes-buffer-input', help='Lakes buffer distance (meters)', required=True)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
 
-    split_flows(max_length, slope_min, lakes_buffer_input, **args)
+    args['max_length'] = float(args['max_length'])
+    args['slope_min'] = float(args['slope_min'])
+    args['lakes_buffer_input'] = float(args['lakes_buffer_input'])
+
+    split_flows(**args)

From c60111a046c5f5fed093753b1ce6d56ec828ce00 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 14 Apr 2023 14:13:09 -0500
Subject: [PATCH 236/359] v4.3.7.2 Hotfix to address nrmse error (#880)

---
 docs/CHANGELOG.md                | 40 ++++++++++++++++++++------------
 tools/rating_curve_comparison.py |  5 ++--
 2 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d06bfdd7b..9c22b9df1 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,31 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.7.3 - 2023-04-14 - [PR#880](https://github.com/NOAA-OWP/inundation-mapping/pull/880)
+
+Hotfix for addressing an error during the NRMSE calculation/aggregation step within `tools/rating_curve_comparison.py`. Also added the "n" variable to the agg_nwm_recurr_flow_elev_stats table. Addresses #878 
+
+### Changes  
+
+- `tools/rating_curve_comparison.py`: address error for computing nrmse when n=1; added the "n" variable (sample size) to the output metrics table
+
+<br/><br/>
+
+## v4.3.7.2 - 2023-04-06 - [PR#879](https://github.com/NOAA-OWP/inundation-mapping/pull/879)
+
+Replaces `os.environ` with input arguments in Python files that are called from bash scripts. The bash scripts now access the environment variables and pass them to the Python files as input arguments. In addition to adapting some Python scripts to a more modular structure which allows them to be run individually, it also allows Visual Studio Code debugger to work properly. Closes #875.
+
+### Changes
+
+- `fim_pre_processing.sh`: Added `-i $inputsDir` input argument to `src/check_huc_inputs.py`
+- `src/`
+    - `add_crosswalk.py`: Changed `min_catchment_area` and `min_stream_length` environment variables to input arguments
+    - `check_huc_inputs.py`: Changed `inputsDir` environment variable to input argument
+    - `delineate_hydros_and_produce_HAND.sh`: Added `-m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters` input arguments to `src/split_flows.py`
+    - `split_flows.py`: Changed `max_split_distance_meters`, `slope_min`, and `lakes_buffer_dist_meters` from environment variables to input arguments
+
+<br/><br/>
+
 ## v4.3.7.1 - 2023-04-06 - [PR#874](https://github.com/NOAA-OWP/inundation-mapping/pull/874)
 
 Hotfix to `process_branch.sh` because it wasn't removing code-61 branches on exit. Also removes the current run from the new fim_temp directory.
@@ -44,21 +69,6 @@ Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. E
 
 <br/><br/>
 
-## v4.3.7.2 - 2023-04-06 - [PR#879](https://github.com/NOAA-OWP/inundation-mapping/pull/879)
-
-Replaces `os.environ` with input arguments in Python files that are called from bash scripts. The bash scripts now access the environment variables and pass them to the Python files as input arguments. In addition to adapting some Python scripts to a more modular structure which allows them to be run individually, it also allows Visual Studio Code debugger to work properly. Closes #875.
-
-### Changes
-
-- `fim_pre_processing.sh`: Added `-i $inputsDir` input argument to `src/check_huc_inputs.py`
-- `src/`
-    - `add_crosswalk.py`: Changed `min_catchment_area` and `min_stream_length` environment variables to input arguments
-    - `check_huc_inputs.py`: Changed `inputsDir` environment variable to input argument
-    - `delineate_hydros_and_produce_HAND.sh`: Added `-m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters` input arguments to `src/split_flows.py`
-    - `split_flows.py`: Changed `max_split_distance_meters`, `slope_min`, and `lakes_buffer_dist_meters` from environment variables to input arguments
-
-<br/><br/>
-
 ## v4.3.5.1 - 2023-04-01 - [PR#867](https://github.com/NOAA-OWP/inundation-mapping/pull/867)
 
 outputs_cleanup.py was throwing an error saying that the HUC source directory (to be cleaned up), did not exist. This was confirmed in a couple of environments. The src path in run_unit_wb.sh was sending in the "outputs" directory and not the "fim_temp" directory. This might have been a merge issue.
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 53ed5c97c..4b6a5e7ab 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -303,7 +303,6 @@ def aggregate_metrics(output_dir,procs_list,stat_groups):
         #         usgs_recurr_stats.to_csv(agg_usgs_interp_elev_stats,index=False, mode='a',header=False)
         #     else:
         #         usgs_recurr_stats.to_csv(agg_usgs_interp_elev_stats,index=False)
-
         if os.path.isfile(huc[4]):
             nwm_recurr_data = pd.read_csv(huc[4],dtype={'location_id': str,
                                                         'feature_id': str})
@@ -661,7 +660,7 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
 
     # Calculate nrmse
     def NRMSE(x):
-        if x['n'][0] == 1:      # when n==1, NRME equation will return an `inf` 
+        if x['n'].values == 1:      # when n==1, NRME equation will return an `inf` 
             return x['sum_y_diff'] ** 0.5
         else:
             return ((x['sum_y_diff'] / x['n']) ** 0.5) / (x['y_max'] - x['y_min'])
@@ -681,7 +680,7 @@ def NRMSE(x):
     percent_bias = station_rc.apply(lambda x: 100 * (x["yhat_minus_y"].sum() / x[usgs_elev].sum()))\
         .reset_index(stat_groups, drop = False).rename({0: "percent_bias"}, axis=1)
 
-    rc_stat_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [nrmse, mean_abs_y_diff, mean_y_diff, percent_bias])
+    rc_stat_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [n, nrmse, mean_abs_y_diff, mean_y_diff, percent_bias])
 
     return rc_stat_table
 

From 45095d28941c6c40141d1956710c8f4255b01afe Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 14 Apr 2023 15:16:35 -0400
Subject: [PATCH 237/359] v4.3.7.4 Check for empty output catchments (#882)

---
 docs/CHANGELOG.md                           | 78 +++++++++++++++++++++
 src/filter_catchments_and_add_attributes.py | 14 ++--
 2 files changed, 88 insertions(+), 4 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9c22b9df1..00513cfbf 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -69,6 +69,84 @@ Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. E
 
 <br/><br/>
 
+## v4.3.7.4 - 2023-04-10 - [PR#882](https://github.com/NOAA-OWP/inundation-mapping/pull/882)
+
+Bug fix for empty `output_catchments` in `src/filter_catchments_and_add_attributes.py`
+
+### Changes
+
+- `src/filter_catchments_and_add_attributes.py`: Adds check for empty `output_catchments` and exits with Status 61 if empty.
+
+<br/><br/>
+
+## v4.3.7.3 - 2023-04-14 - [PR#880](https://github.com/NOAA-OWP/inundation-mapping/pull/880)
+
+Hotfix for addressing an error during the NRMSE calculation/aggregation step within `tools/rating_curve_comparison.py`. Also added the "n" variable to the agg_nwm_recurr_flow_elev_stats table. Addresses #878 
+
+### Changes  
+
+- `tools/rating_curve_comparison.py`: address error for computing nrmse when n=1; added the "n" variable (sample size) to the output metrics table
+
+<br/><br/>
+
+## v4.3.7.2 - 2023-04-06 - [PR#879](https://github.com/NOAA-OWP/inundation-mapping/pull/879)
+
+Replaces `os.environ` with input arguments in Python files that are called from bash scripts. The bash scripts now access the environment variables and pass them to the Python files as input arguments. In addition to adapting some Python scripts to a more modular structure which allows them to be run individually, it also allows Visual Studio Code debugger to work properly. Closes #875.
+
+### Changes
+
+- `fim_pre_processing.sh`: Added `-i $inputsDir` input argument to `src/check_huc_inputs.py`
+- `src/`
+    - `add_crosswalk.py`: Changed `min_catchment_area` and `min_stream_length` environment variables to input arguments
+    - `check_huc_inputs.py`: Changed `inputsDir` environment variable to input argument
+    - `delineate_hydros_and_produce_HAND.sh`: Added `-m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters` input arguments to `src/split_flows.py`
+    - `split_flows.py`: Changed `max_split_distance_meters`, `slope_min`, and `lakes_buffer_dist_meters` from environment variables to input arguments
+
+<br/><br/>
+
+## v4.3.7.1 - 2023-04-06 - [PR#874](https://github.com/NOAA-OWP/inundation-mapping/pull/874)
+
+Hotfix to `process_branch.sh` because it wasn't removing code-61 branches on exit. Also removes the current run from the new fim_temp directory.
+
+### Changes  
+
+- `fim_pipeline.sh`: removal of current run from fim_temp directory
+- `src/process_branch.sh`: switched the exit 61 block to use the temp directory instead of the outputs directory
+
+<br/><br/>
+
+## v4.3.7.0 - 2023-03-02 - [PR#868](https://github.com/NOAA-OWP/inundation-mapping/pull/868)
+
+This pull request adds a new feature to `fim_post_processing.sh` to aggregate all of the hydrotables for a given HUC into a single HUC-level `hydrotable.csv` file. Note that the aggregation step happens near the end of `fim_post_processing.sh` (after the subdivision and calibration routines), and the branch hydrotable files are preserved in the branch directories for the time being.
+
+### Changes  
+
+- `fim_pipeline.sh`: created a new variable `$jobMaxLimit` that multiplies the `$jobHucLimit` and the `$jobBranchLimit`
+- `fim_post_processing.sh`: added new aggregation/concatenation step after the SRC calibration routines; passing the new `$jobMaxLimit` to the commands that accept a multiprocessing job number input; added `$skipcal` argument to the USGS rating curve calibration routine
+- `src/add_crosswalk.py`: changed the default value for `calb_applied` variable to be a boolean
+- `src/aggregate_by_huc.py`: file renamed (previous name: `src/usgs_gage_aggregate.py`); updated to perform branch to huc file aggregation for `hydroTable_{branch_id}.csv` and `src_full_crosswalked_{branch_id}.csv` files; note that the input arguments ask you to specify which file types to aggregate using the flags: `-elev`, `-htable`, and `-src`
+- `tools/inundate_gms.py`: added check to use the aggregated HUC-level `hydrotable.csv` if it exists, otherwise continue to use the branch hydroTable files
+- `tools/inundation.py`: added `usecols` argument to the `pd.read_csv` commands to improve read time for hydrotables
+- `src/subdiv_chan_obank_src.py`: add dtype to hydrotable pd.read_csv to resolve pandas dtype interpretation warnings
+
+<br/><br/>
+
+## v4.3.6.0 - 2023-03-23 - [PR#803](https://github.com/NOAA-OWP/inundation-mapping/pull/803)
+
+Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. Essentially, this is a wrapper for `geopandas.clip()` and moves clipping from `src/clip_vectors_to_wbd.py` to `data/wbd/preprocess_wbd.py`.
+
+### Additions
+
+- `data/wbd/preprocess_wbd.py`: Clips WBD to DEM domain polygon
+
+### Changes
+
+- `src/`
+    - `bash_variables.env`: Updates `input_WBD_gdb` environment variable
+    - `clip_vectors_to_wbd.py`: Removes clipping to DEM domain
+
+<br/><br/>
+
 ## v4.3.5.1 - 2023-04-01 - [PR#867](https://github.com/NOAA-OWP/inundation-mapping/pull/867)
 
 outputs_cleanup.py was throwing an error saying that the HUC source directory (to be cleaned up), did not exist. This was confirmed in a couple of environments. The src path in run_unit_wb.sh was sending in the "outputs" directory and not the "fim_temp" directory. This might have been a merge issue.
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index 7801b613d..0b17be3d7 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -51,13 +51,19 @@ def filter_catchments_and_add_attributes(input_catchments_filename,
         # add geometry column
         output_catchments['areasqkm'] = output_catchments.geometry.area/(1000**2)
 
-        try:
-            output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
-            output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
-        except ValueError:
+        if not output_catchments.empty:
+            try:
+                output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
+                output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
+            except ValueError:
+                # this is not an exception, but a custom exit code that can be trapped
+                print("There are no flowlines in the HUC after stream order filtering.")
+                sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+        else:
             # this is not an exception, but a custom exit code that can be trapped
             print("There are no flowlines in the HUC after stream order filtering.")
             sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+
     else:
         # this is not an exception, but a custom exit code that can be trapped
         print("There are no flowlines in the HUC after stream order filtering.")

From ac65cb5105691c61479741878b09a5abfc3bfa74 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Mon, 17 Apr 2023 09:20:20 -0500
Subject: [PATCH 238/359] Update CHANGELOG.md (#885)

---
 docs/CHANGELOG.md | 67 -----------------------------------------------
 1 file changed, 67 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 00513cfbf..599d54278 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,73 +1,6 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-## v4.3.7.3 - 2023-04-14 - [PR#880](https://github.com/NOAA-OWP/inundation-mapping/pull/880)
-
-Hotfix for addressing an error during the NRMSE calculation/aggregation step within `tools/rating_curve_comparison.py`. Also added the "n" variable to the agg_nwm_recurr_flow_elev_stats table. Addresses #878 
-
-### Changes  
-
-- `tools/rating_curve_comparison.py`: address error for computing nrmse when n=1; added the "n" variable (sample size) to the output metrics table
-
-<br/><br/>
-
-## v4.3.7.2 - 2023-04-06 - [PR#879](https://github.com/NOAA-OWP/inundation-mapping/pull/879)
-
-Replaces `os.environ` with input arguments in Python files that are called from bash scripts. The bash scripts now access the environment variables and pass them to the Python files as input arguments. In addition to adapting some Python scripts to a more modular structure which allows them to be run individually, it also allows Visual Studio Code debugger to work properly. Closes #875.
-
-### Changes
-
-- `fim_pre_processing.sh`: Added `-i $inputsDir` input argument to `src/check_huc_inputs.py`
-- `src/`
-    - `add_crosswalk.py`: Changed `min_catchment_area` and `min_stream_length` environment variables to input arguments
-    - `check_huc_inputs.py`: Changed `inputsDir` environment variable to input argument
-    - `delineate_hydros_and_produce_HAND.sh`: Added `-m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters` input arguments to `src/split_flows.py`
-    - `split_flows.py`: Changed `max_split_distance_meters`, `slope_min`, and `lakes_buffer_dist_meters` from environment variables to input arguments
-
-<br/><br/>
-
-## v4.3.7.1 - 2023-04-06 - [PR#874](https://github.com/NOAA-OWP/inundation-mapping/pull/874)
-
-Hotfix to `process_branch.sh` because it wasn't removing code-61 branches on exit. Also removes the current run from the new fim_temp directory.
-
-### Changes  
-
-- `fim_pipeline.sh`: removal of current run from fim_temp directory
-- `src/process_branch.sh`: switched the exit 61 block to use the temp directory instead of the outputs directory
-
-<br/><br/>
-
-## v4.3.7.0 - 2023-03-02 - [PR#868](https://github.com/NOAA-OWP/inundation-mapping/pull/868)
-
-This pull request adds a new feature to `fim_post_processing.sh` to aggregate all of the hydrotables for a given HUC into a single HUC-level `hydrotable.csv` file. Note that the aggregation step happens near the end of `fim_post_processing.sh` (after the subdivision and calibration routines), and the branch hydrotable files are preserved in the branch directories for the time being.
-
-### Changes  
-
-- `fim_pipeline.sh`: created a new variable `$jobMaxLimit` that multiplies the `$jobHucLimit` and the `$jobBranchLimit`
-- `fim_post_processing.sh`: added new aggregation/concatenation step after the SRC calibration routines; passing the new `$jobMaxLimit` to the commands that accept a multiprocessing job number input; added `$skipcal` argument to the USGS rating curve calibration routine
-- `src/add_crosswalk.py`: changed the default value for `calb_applied` variable to be a boolean
-- `src/aggregate_by_huc.py`: file renamed (previous name: `src/usgs_gage_aggregate.py`); updated to perform branch to huc file aggregation for `hydroTable_{branch_id}.csv` and `src_full_crosswalked_{branch_id}.csv` files; note that the input arguments ask you to specify which file types to aggregate using the flags: `-elev`, `-htable`, and `-src`
-- `tools/inundate_gms.py`: added check to use the aggregated HUC-level `hydrotable.csv` if it exists, otherwise continue to use the branch hydroTable files
-- `tools/inundation.py`: added `usecols` argument to the `pd.read_csv` commands to improve read time for hydrotables
-- `src/subdiv_chan_obank_src.py`: add dtype to hydrotable pd.read_csv to resolve pandas dtype interpretation warnings
-
-<br/><br/>
-
-## v4.3.6.0 - 2023-03-23 - [PR#803](https://github.com/NOAA-OWP/inundation-mapping/pull/803)
-
-Clips Watershed Boundary Dataset (WBD) to DEM domain for increased efficiency. Essentially, this is a wrapper for `geopandas.clip()` and moves clipping from `src/clip_vectors_to_wbd.py` to `data/wbd/preprocess_wbd.py`.
-
-### Additions
-
-- `data/wbd/preprocess_wbd.py`: Clips WBD to DEM domain polygon
-
-### Changes
-
-- `src/`
-    - `bash_variables.env`: Updates `input_WBD_gdb` environment variable
-    - `clip_vectors_to_wbd.py`: Removes clipping to DEM domain
-
-<br/><br/>
 
 ## v4.3.7.4 - 2023-04-10 - [PR#882](https://github.com/NOAA-OWP/inundation-mapping/pull/882)
 

From 70533fc7537a970e3428cf93f14f8db5c1d1514f Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 21 Apr 2023 14:24:03 -0400
Subject: [PATCH 239/359] v4.3.8.0 Clip branch 0 DEM-derived flowlines to
 terminal NWM streamlines (#881)

---
 docs/CHANGELOG.md                        |  12 +++
 src/delineate_hydros_and_produce_HAND.sh |  19 ++--
 src/split_flows.py                       | 107 ++++++++++++++++-------
 3 files changed, 98 insertions(+), 40 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 599d54278..dda92467c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.8.0 - 2023-04-07 - [PR#881](https://github.com/NOAA-OWP/inundation-mapping/pull/881)
+
+Clips branch 0 to terminal segments of NWM streams using the `to` attribute of NWM streams (where `to=0`).
+
+### Changes
+
+- `src/`
+    - `delineate_hydros_and_produce_HAND.sh`: Added input arguments to `src/split_flows.py`
+    - `split_flows.py`: Added functionality to snap and trim branch 0 flows to terminal NWM streamlines
+
+<br/><br/>
+
 ## v4.3.7.4 - 2023-04-10 - [PR#882](https://github.com/NOAA-OWP/inundation-mapping/pull/882)
 
 Bug fix for empty `output_catchments` in `src/filter_catchments_and_add_attributes.py`
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 6209328c4..167db7da0 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -3,6 +3,15 @@
 ## Level is equal to the parent script: 'unit' or 'branch'
 level=$1
 
+if [ "$level" = "branch" ]; then
+    b_arg=$tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
+    z_arg=$tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg
+elif [ "$level" = "unit" ]; then
+    # Branch zero has a different source for -b and -z arguments
+    b_arg=$tempHucDataDir/nwm_subset_streams.gpkg
+    z_arg=$tempHucDataDir/nwm_catchments_proj_subset.gpkg
+fi
+
 ## INITIALIZE TOTAL TIME TIMER ##
 T_total_start
 
@@ -75,7 +84,7 @@ Tcount
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg -n $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg -m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters
+$srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg -n $b_arg -m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
@@ -175,14 +184,6 @@ Tcount
 echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"
 date -u
 Tstart
-if [ "$level" = "branch" ]; then
-    b_arg=$tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg
-    z_arg=$tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg
-elif [ "$level" = "unit" ]; then
-    # Branch zero has a different source for -b and -z arguments
-    b_arg=$tempHucDataDir/nwm_subset_streams.gpkg
-    z_arg=$tempHucDataDir/nwm_catchments_proj_subset.gpkg
-fi
 
 python3 -m memory_profiler $srcDir/add_crosswalk.py -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $tempCurrentBranchDataDir/src_base_$current_branch_id.csv -l $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $tempCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $tempCurrentBranchDataDir/src_$current_branch_id.json -x $tempCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $tempCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $tempHucDataDir/wbd8_clp.gpkg -b $b_arg -y $tempCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -k $tempCurrentBranchDataDir/small_segments_$current_branch_id.csv -e $min_catchment_area -g $min_stream_length
 Tcount
diff --git a/src/split_flows.py b/src/split_flows.py
index 7c34dca4b..ff7f4fbc6 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -31,17 +31,60 @@
 from utils.fim_enums import FIM_exit_codes
 
 @mem_profile
-def split_flows(max_length, 
-                slope_min, 
-                lakes_buffer_input, 
-                flows_filename,
+def split_flows(flows_filename,
                 dem_filename, 
                 split_flows_filename, 
                 split_points_filename, 
                 wbd8_clp_filename, 
                 lakes_filename,
-                nwm_streams_filename):
+                nwm_streams_filename,
+                max_length, 
+                slope_min, 
+                lakes_buffer_input):
+
+    def snap_and_trim_flow(snapped_point, flows):
+        # Find nearest flow line
+        if len(flows) > 1:
+            sjoin_nearest = gpd.sjoin_nearest(snapped_point, flows, max_distance=100)
+            if sjoin_nearest.empty:
+                return flows
+            
+            if len(sjoin_nearest) > 1:
+                sjoin_nearest = sjoin_nearest[sjoin_nearest['LINKNO'].isin(sjoin_nearest['DSLINKNO'])]
+            
+            nearest_index = int(sjoin_nearest['LINKNO'])
+            flow = flows[flows['LINKNO']==nearest_index]
+            flow.index = [0]
+
+        else:
+            flow = flows
+            nearest_index = None
+
+        # Snap to DEM flows
+        snapped_point['geometry'] = flow.interpolate(flow.project(snapped_point.geometry))[0]
+
+        # Trim flows to snapped point
+        # buffer here because python precision issues, print(demDerived_reaches.distance(snapped_point) < 1e-8)
+        trimmed_line = shapely_ops_split(flow.iloc[0]['geometry'], snapped_point.iloc[0]['geometry'].buffer(1))
+        # Edge cases: line string not split?, nothing is returned, split does not preserve linestring order?
+        # Note to dear reader: last here is really the most upstream segment (see crevats above).  When we split we should get 3 segments, the most downstream one
+        # the tiny 1 meter segment that falls within the snapped point buffer, and the most upstream one.  We want that last one which is why we trimmed_line[len(trimmed_line)-1]
+        last_line_segment = pd.DataFrame({'id':['first'],'geometry':[trimmed_line.geoms[len(trimmed_line.geoms)-1].wkt]})
+        last_line_segment['geometry'] = last_line_segment['geometry'].apply(wkt.loads) # can be last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment) # when we update geopandas verisons
+        last_line_segment_geodataframe = gpd.GeoDataFrame(last_line_segment).set_crs(flow.crs)
+
+        # replace geometry in merged flowine
+        flow_geometry = last_line_segment_geodataframe.iloc[0]['geometry']
 
+        if nearest_index is not None:
+            # Update geometry of line closest to snapped_point
+            flows.loc[flows['LINKNO']==nearest_index, 'geometry'] = flow_geometry
+        else:
+            flows['geometry'] = flow_geometry
+
+        return flows
+
+        
     toMetersConversion = 1e-3
 
     print('Loading data ...')
@@ -73,38 +116,40 @@ def split_flows(max_length,
     hydro_id = 'HydroID'
 
     # If loop addressing: https://github.com/NOAA-OWP/inundation-mapping/issues/560
-    # if we are processing branch 0, skip this step
-    if (os.path.split(os.path.dirname(nwm_streams_filename))[1] != '0'):
-        print ('trimming DEM stream to NWM branch terminus')
-        # read in nwm lines, explode to ensure linestrings are the only geometry
-        levelpath_lines = gpd.read_file(nwm_streams_filename).explode(index_parts=True)
-
-        # Dissolve the linestring (how much faith should I hold that these are digitized with flow?)
-        linestring_geo = levelpath_lines.iloc[0]['geometry']
-        if (len(levelpath_lines) > 1):
-            linestring_geo = ops.linemerge(levelpath_lines.dissolve(by='levpa_id').iloc[0]['geometry'])
+    print ('trimming DEM stream to NWM branch terminus')
+    # read in nwm lines, explode to ensure linestrings are the only geometry
+    nwm_streams = gpd.read_file(nwm_streams_filename).explode(index_parts=True)
+
+    # Dissolve levelpath if not branch 0
+    if 'levpa_id' in nwm_streams.columns:
+        if (len(nwm_streams) > 1):
+            # Dissolve the linestring (how much faith should I hold that these are digitized with flow?)
+            linestring_geo = ops.linemerge(nwm_streams.dissolve(by='levpa_id').iloc[0]['geometry'])
+        else:
+            linestring_geo = nwm_streams.iloc[0]['geometry']
 
         # Identify the end vertex (most downstream, should be last), transform into geodataframe
         terminal_nwm_point = []
         last = Point(linestring_geo.coords[-1])
         terminal_nwm_point.append({'ID':'terminal','geometry':last})
-        snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(levelpath_lines.crs)
+        snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(nwm_streams.crs)
 
-        # Snap to DEM flows
-        snapped_point['geometry'] = flows.interpolate(flows.project(snapped_point.geometry))
+        flows = snap_and_trim_flow(snapped_point, flows)
 
-        # Trim flows to snapped point
-        # buffer here because python precision issues, print(demDerived_reaches.distance(snapped_point) < 1e-8)
-        trimmed_line = shapely_ops_split(flows.iloc[0]['geometry'], snapped_point.iloc[0]['geometry'].buffer(1))
-        # Edge cases: line string not split?, nothing is returned, split does not preserve linestring order?
-        # Note to dear reader: last here is really the most upstream segmennt (see crevats above).  When we split we should get 3 segments, the most downstream one
-        # the tiny 1 meter segment that falls within the snapped point buffer, and the most upstream one.  We want that last one which is why we trimmed_line[len(trimmed_line)-1]
-        last_line_segment = pd.DataFrame({'id':['first'],'geometry':[trimmed_line.geoms[len(trimmed_line.geoms)-1].wkt]})
-        last_line_segment['geometry'] = last_line_segment['geometry'].apply(wkt.loads) # can be last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment) when we update geopandas verisons
-        last_line_segment_geodataframe = gpd.GeoDataFrame(last_line_segment).set_crs(flows.crs)
+    # If branch 0: loop over NWM terminal segments
+    else:
+        nwm_streams_terminal = nwm_streams[nwm_streams['to']==0]
+        if not nwm_streams_terminal.empty:
+            for i, row in nwm_streams_terminal.iterrows():
+                linestring_geo = row['geometry']
+                # Identify the end vertex (most downstream, should be last), transform into geodataframe
+                terminal_nwm_point = []
+                last = Point(linestring_geo.coords[-1])
+                terminal_nwm_point.append({'ID':'terminal','geometry':last})
+                snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(nwm_streams.crs)
+
+                flows = snap_and_trim_flow(snapped_point, flows)
 
-        # replace geometry in merged flowine
-        flows['geometry'] = last_line_segment_geodataframe.iloc[0]['geometry']
 
     # split at HUC8 boundaries
     print ('splitting stream segments at HUC8 boundaries')
@@ -276,7 +321,7 @@ def split_flows(max_length,
 
 if __name__ == '__main__':
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='splitflows.py')
+    parser = argparse.ArgumentParser(description='split_flows.py')
     parser.add_argument('-f', '--flows-filename', help='flows-filename',required=True)
     parser.add_argument('-d', '--dem-filename', help='dem-filename',required=True)
     parser.add_argument('-s', '--split-flows-filename', help='split-flows-filename',required=True)
@@ -285,7 +330,7 @@ def split_flows(max_length,
     parser.add_argument('-l', '--lakes-filename', help='lakes-filename',required=True)
     parser.add_argument('-n', '--nwm-streams-filename', help='nwm-streams-filename',required=True)
     parser.add_argument('-m', '--max-length', help='Maximum split distance (meters)', required=True)
-    parser.add_argument('-t', '--slope-min', help='Slope minimum', required=True)
+    parser.add_argument('-t', '--slope-min', help='Minimum slope', required=True)
     parser.add_argument('-b', '--lakes-buffer-input', help='Lakes buffer distance (meters)', required=True)
 
     # Extract to dictionary and assign to variables.

From 082431393bb6cd575a18f43aa8eed7441ac2aeee Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 21 Apr 2023 14:28:41 -0400
Subject: [PATCH 240/359] v4.3.9.0 Update base GDAL Docker image and other
 Python packages (#889)

---
 Dockerfile                    |   14 +-
 Pipfile                       |   38 +-
 Pipfile.lock                  | 1217 +++++++++++++++++++--------------
 docs/CHANGELOG.md             |   22 +
 fim_post_processing.sh        |    3 +-
 src/add_crosswalk.py          |    2 +-
 src/aggregate_by_huc.py       |    6 +-
 src/build_stream_traversal.py |    2 +-
 src/getRasterInfoNative.py    |    2 +-
 src/src_adjust_usgs_rating.py |    2 +-
 src/stream_branches.py        |    6 +-
 src/usgs_gage_unit_setup.py   |    2 +-
 tools/eval_plots.py           |    4 +-
 tools/inundation.py           |    3 -
 14 files changed, 766 insertions(+), 557 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7c4441c90..2fc6f8993 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 ## Temporary image to build the libraries and only save the needed artifacts
-FROM osgeo/gdal:ubuntu-full-3.1.2 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.4.3 AS builder
 WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
@@ -47,7 +47,7 @@ RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d
 
 
 # Base Image that has GDAL, PROJ, etc
-FROM osgeo/gdal:ubuntu-full-3.1.2
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.4.3
 ARG dataDir=/data
 ENV projectDir=/foss_fim
 ARG depDir=/dependencies
@@ -71,7 +71,7 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 RUN apt update --fix-missing 
-RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync tzdata
 
 RUN apt auto-remove
 
@@ -106,10 +106,10 @@ RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --syste
 # Whitebox also attempts to always download a folder called testdata regardless of use. 
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
 RUN wbox_path=/usr/local/lib/python3.8/dist-packages/whitebox/ && \
-wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
-unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
-cp $wbox_path/WBT/whitebox_tools $wbox_path && \
-mkdir $wbox_path/testdata
+    wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
+    unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
+    cp $wbox_path/WBT/whitebox_tools $wbox_path && \
+    mkdir $wbox_path/testdata
 # ----------------------------------
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
diff --git a/Pipfile b/Pipfile
index 3ced6075e..8704dd35e 100755
--- a/Pipfile
+++ b/Pipfile
@@ -4,38 +4,38 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-ipython = "*"
+ipython = "==8.12.0"
 
 [packages]
-fiona = "==1.8.17"
+fiona = "==1.8.22"
 geopandas = "==0.12.2"
 numba = "==0.56.4"
 numpy = "==1.23.5"
-pandas = "==1.1"
-rasterio = "==1.1.5"
+pandas = "==2.0.0"
+rasterio = "==1.3.6"
 rasterstats = "==0.18.0"
 richdem = "==0.3.4"
 tqdm = "==4.65.0"
-seaborn = "==0.11.0"
-python-dotenv = "*"
-natsort = "*"
-memory-profiler = "*"
-xarray = "==0.19.0"
-netcdf4 = "==1.5.7"
-tables = "==3.6.1"
-h5py = "==3.4.0"
-pyproj = "==3.1.0"
-psycopg2-binary = "==2.8.6"
-boto3 = "*"
-jupyter = "*"
-jupyterlab = "*"
-ipympl = "*"
-pytest = "*"
+seaborn = "==0.12.2"
+python-dotenv = "==1.0.0"
+natsort = "==8.3.1"
+memory-profiler = "==0.61.0"
+xarray = "==2023.1.0"
+netcdf4 = "==1.6.3"
+tables = "==3.8.0"
+pyproj = "==3.5.0"
+psycopg2-binary = "==2.9.6"
+boto3 = "==1.26.109"
+jupyter = "==1.0.0"
+jupyterlab = "==3.6.3"
+ipympl = "==0.9.3"
+pytest = "==7.3.0"
 whitebox = "==2.3.1"
 shapely = "==2.0.1"
 pyarrow = "==11.0.0"
 rtree = "==1.0.1"
 py7zr = "==0.20.4"
+scipy = "==1.10.1"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index ea78e73b3..738f09b88 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "375fa6bf31e8485eaae40981f64ce07fddc91e3b86de727846fb6a3ce9e63654"
+            "sha256": "1f4c512cbfa21c9f52ab6461f0d0d26f955d9e0b21ec612fc4d0b47b8a7820b4"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -123,11 +123,11 @@
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591",
-                "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"
+                "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
+                "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==4.12.0"
+            "version": "==4.12.2"
         },
         "bleach": {
             "hashes": [
@@ -137,21 +137,48 @@
             "markers": "python_version >= '3.7'",
             "version": "==6.0.0"
         },
+        "blosc2": {
+            "hashes": [
+                "sha256:0497793f55db0b75de08eb4c047a0bc5b96dbe5e405b53803dd3368e36336188",
+                "sha256:0ec63269428aa3fb45f7d4881b2d11b428c4cb62e854caf54a767a64da4df83e",
+                "sha256:1fbecd7ef4876811719aa58d84e4b414f430f329162c18578b870f5e77b59864",
+                "sha256:32c365bf744353103ed91dc1f03889de03b986588181601594aa7ee773818cb4",
+                "sha256:4085e5c1df186e1747d8a8578b0cc1c8b7668391d635e9f89e17156912fba85a",
+                "sha256:54c5614b18f9f01473758fa64e3bc699adbe31b307a45eca0e07fa2204e4d4a1",
+                "sha256:5c193959a984e7814833c8be6b9026d7744d2cff4d450476561583a87152e13e",
+                "sha256:6545d6d7e7365a2a3533d4bdf7095856443aed7d5ddc577ecd0e78083790bff1",
+                "sha256:659e18d5e606a0ca4d80766f86d87e640818051911d01679eed11022243a7e4f",
+                "sha256:8eb02f67d4ed8ac8f0ce5f3c8cafc0059255bb6899fd35127e4076925640f239",
+                "sha256:96dd63eb7641594208e6a865fd60a0bdde24568a180180beb8af4d6608796f3a",
+                "sha256:98ab1cd57f9d7422f1636a6b290f2940113ee8be26bfe3823e8c011826972b9c",
+                "sha256:c58990ab2bcd0f412496acf1d05c65d955d963197bbaa57b10b2ace31c29181a",
+                "sha256:c5c52649837d514669107c77e8f172e9e5ecfa030eef0d378bb47ce1689921c9",
+                "sha256:cec4c0570543921ce6b8c9ffdbf9f2170de37ecaf8e2b213e867e3130b81f205",
+                "sha256:d98e850f0de55e15c402c6e27105ba850f8954e784e30a7f8bde89eb70a08574",
+                "sha256:ea3396de7757092d502fb297a44a8d019d92e750e5aebcd9d39a157fde8785b3",
+                "sha256:ef018926b55799cf23345127dde8f29356b4451b3e067e1b07f0d186213bd821",
+                "sha256:f19b0b3674f6c825b490f00d8264b0c540c2cdc11ec7e81178d38b83c57790a1",
+                "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
+                "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
+            ],
+            "markers": "python_version >= '3.8' and python_version < '4'",
+            "version": "==2.0.0"
+        },
         "boto3": {
             "hashes": [
-                "sha256:043f8981d10c4e7c48736df4381dac557b46c5b369b0a450d8f3d7f5fdd24db5",
-                "sha256:b00f416832bc59863b96175045d2ebe067d9222289bce677c48fd72c006eaaad"
+                "sha256:a1c48674c2ff25ddced599c51d14ba7a6dbe9fd51641e8d63a6887bebd9712d7",
+                "sha256:d388cb7f54f1a3056f91ffcfb5cf18b226454204e5df7a5c10774718c3fbb166"
             ],
             "index": "pypi",
-            "version": "==1.26.102"
+            "version": "==1.26.109"
         },
         "botocore": {
             "hashes": [
-                "sha256:4bae8f502507da18ff37c61cb18745cfb11d87a61dd0ea27e346adadff92aa3f",
-                "sha256:58b11c630d2044ea732ba4c403d29fab51e954465f9b3f7099cbf5ac0ce7ab47"
+                "sha256:2e449525f0ccedb31fdb962a77caac48b4c486c23515b84c5989a39a1823a024",
+                "sha256:cf43dddb7e2ba5425fe19fad68b10043307b61d9103d06566f1ab6034e38b8db"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.29.102"
+            "version": "==1.29.109"
         },
         "brotli": {
             "hashes": [
@@ -428,11 +455,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
-                "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==7.1.2"
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.3"
         },
         "click-plugins": {
             "hashes": [
@@ -526,28 +553,75 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.11.0"
         },
+        "cython": {
+            "hashes": [
+                "sha256:03daae07f8cbf797506446adae512c3dd86e7f27a62a541fa1ee254baf43e32c",
+                "sha256:0963266dad685812c1dbb758fcd4de78290e3adc7db271c8664dcde27380b13e",
+                "sha256:0ab3cbf3d62b0354631a45dc93cfcdf79098663b1c65a6033af4a452b52217a7",
+                "sha256:0e9032cd650b0cb1d2c2ef2623f5714c14d14c28d7647d589c3eeed0baf7428e",
+                "sha256:11b1b278b8edef215caaa5250ad65a10023bfa0b5a93c776552248fc6f60098d",
+                "sha256:1909688f5d7b521a60c396d20bba9e47a1b2d2784bfb085401e1e1e7d29a29a8",
+                "sha256:1d6c809e2f9ce5950bbc52a1d2352ef3d4fc56186b64cb0d50c8c5a3c1d17661",
+                "sha256:21b88200620d80cfe193d199b259cdad2b9af56f916f0f7f474b5a3631ca0caa",
+                "sha256:308c8f1e58bf5e6e8a1c4dcf8abbd2d13d0f9b1e582f4d9ae8b89857342d8bb5",
+                "sha256:44733366f1604b0c327613b6918469284878d2f5084297d10d26072fc6948d51",
+                "sha256:459994d1de0f99bb18fad9f2325f760c4b392b1324aef37bcc1cd94922dfce41",
+                "sha256:4a2723447d1334484681d5aede34184f2da66317891f94b80e693a2f96a8f1a7",
+                "sha256:56866323f1660cecb4d5ff3a1fba92a56b91b7cfae0a8253777aa4bdb3bdf9a8",
+                "sha256:5718319a01489688fdd22ddebb8e2fcbbd60be5f30de4336ea7063c3ae29fbe5",
+                "sha256:5a8de3e793a576e40ca9b4f5518610cd416273c7dc5e254115656b6e4ec70663",
+                "sha256:5c121dc185040f4333bfded68963b4529698e1b6d994da56be32c97a90c896b6",
+                "sha256:60969d38e6a456a67e7ef8ae20668eff54e32ba439d4068ccf2854a44275a30f",
+                "sha256:67b850cf46b861bc27226d31e1d87c0e69869a02f8d3cc5d5bef549764029879",
+                "sha256:742544024ddb74314e2d597accdb747ed76bd126e61fcf49940a5b5be0a8f381",
+                "sha256:7595d29eaee95633dd8060f50f0e54b27472d01587659557ebcfe39da3ea946b",
+                "sha256:7879992487d9060a61393eeefe00d299210256928dce44d887b6be313d342bac",
+                "sha256:8c3cd8bb8e880a3346f5685601004d96e0a2221e73edcaeea57ea848618b4ac6",
+                "sha256:9489de5b2044dcdfd9d6ca8242a02d560137b3c41b1f5ae1c4f6707d66d6e44d",
+                "sha256:a0f4229df10bc4545ebbeaaf96ebb706011d8b333e54ed202beb03f2bee0a50e",
+                "sha256:a8ad755f9364e720f10a36734a1c7a5ced5c679446718b589259261438a517c9",
+                "sha256:b6149f7cc5b31bccb158c5b968e5a8d374fdc629792e7b928a9b66e08b03fca5",
+                "sha256:bdb3285660e3068438791ace7dd7b1efd6b442a10b5c8d7a4f0c9d184d08c8ed",
+                "sha256:be4f6b7be75a201c290c8611c0978549c60353890204573078e865423dbe3c83",
+                "sha256:ccb223b5f0fd95d8d27561efc0c14502c0945f1a32274835831efa5d5baddfc1",
+                "sha256:cfb2302ef617d647ee590a4c0a00ba3c2da05f301dcefe7721125565d2e51351",
+                "sha256:d7ef5f68f4c5baa93349ea54a352f8716d18bee9a37f3e93eff38a5d4e9b7262",
+                "sha256:d8f822fb6ecd5d88c42136561f82960612421154fc5bf23c57103a367bb91356",
+                "sha256:dbd79221869ee9a6ccc4953b2c8838bb6ae08ab4d50ea4b60d7894f03739417b",
+                "sha256:dce0a36d163c05ae8b21200059511217d79b47baf2b7b0f926e8367bd7a3cc24",
+                "sha256:e40cf86aadc29ecd1cb6de67b0d9488705865deea4fc185c7ad56d7a6fc78703",
+                "sha256:e4401270b0dc464c23671e2e9d52a60985f988318febaf51b047190e855bbe7d",
+                "sha256:e6ef7879668214d80ea3914c17e7d4e1ebf4242e0dd4dabe95ca5ccbe75589a5",
+                "sha256:e971db8aeb12e7c0697cefafe65eefcc33ff1224ae3d8c7f83346cbc42c6c270",
+                "sha256:f674ceb5f722d364395f180fbac273072fc1a266aab924acc9cfd5afc645aae1",
+                "sha256:fd1ea21f1cebf33ae288caa0f3e9b5563a709f4df8925d53bad99be693fc0d9b"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.29.34"
+        },
         "debugpy": {
             "hashes": [
-                "sha256:0ea1011e94416e90fb3598cc3ef5e08b0a4dd6ce6b9b33ccd436c1dffc8cd664",
-                "sha256:23363e6d2a04d726bbc1400bd4e9898d54419b36b2cdf7020e3e215e1dcd0f8e",
-                "sha256:23c29e40e39ad7d869d408ded414f6d46d82f8a93b5857ac3ac1e915893139ca",
-                "sha256:549ae0cb2d34fc09d1675f9b01942499751d174381b6082279cf19cdb3c47cbe",
-                "sha256:70ab53918fd907a3ade01909b3ed783287ede362c80c75f41e79596d5ccacd32",
-                "sha256:72687b62a54d9d9e3fb85e7a37ea67f0e803aaa31be700e61d2f3742a5683917",
-                "sha256:78739f77c58048ec006e2b3eb2e0cd5a06d5f48c915e2fc7911a337354508110",
-                "sha256:7aa7e103610e5867d19a7d069e02e72eb2b3045b124d051cfd1538f1d8832d1b",
-                "sha256:87755e173fcf2ec45f584bb9d61aa7686bb665d861b81faa366d59808bbd3494",
-                "sha256:9b5d1b13d7c7bf5d7cf700e33c0b8ddb7baf030fcf502f76fc061ddd9405d16c",
-                "sha256:a771739902b1ae22a120dbbb6bd91b2cae6696c0e318b5007c5348519a4211c6",
-                "sha256:b9c2130e1c632540fbf9c2c88341493797ddf58016e7cba02e311de9b0a96b67",
-                "sha256:be596b44448aac14eb3614248c91586e2bc1728e020e82ef3197189aae556115",
-                "sha256:c05349890804d846eca32ce0623ab66c06f8800db881af7a876dc073ac1c2225",
-                "sha256:de4a045fbf388e120bb6ec66501458d3134f4729faed26ff95de52a754abddb1",
-                "sha256:dff595686178b0e75580c24d316aa45a8f4d56e2418063865c114eef651a982e",
-                "sha256:f6383c29e796203a0bba74a250615ad262c4279d398e89d895a69d3069498305"
+                "sha256:0679b7e1e3523bd7d7869447ec67b59728675aadfc038550a63a362b63029d2c",
+                "sha256:279d64c408c60431c8ee832dfd9ace7c396984fd7341fa3116aee414e7dcd88d",
+                "sha256:33edb4afa85c098c24cc361d72ba7c21bb92f501104514d4ffec1fb36e09c01a",
+                "sha256:38ed626353e7c63f4b11efad659be04c23de2b0d15efff77b60e4740ea685d07",
+                "sha256:5224eabbbeddcf1943d4e2821876f3e5d7d383f27390b82da5d9558fd4eb30a9",
+                "sha256:53f7a456bc50706a0eaabecf2d3ce44c4d5010e46dfc65b6b81a518b42866267",
+                "sha256:9cd10cf338e0907fdcf9eac9087faa30f150ef5445af5a545d307055141dd7a4",
+                "sha256:aaf6da50377ff4056c8ed470da24632b42e4087bc826845daad7af211e00faad",
+                "sha256:b3e7ac809b991006ad7f857f016fa92014445085711ef111fdc3f74f66144096",
+                "sha256:bae1123dff5bfe548ba1683eb972329ba6d646c3a80e6b4c06cd1b1dd0205e9b",
+                "sha256:c0ff93ae90a03b06d85b2c529eca51ab15457868a377c4cc40a23ab0e4e552a3",
+                "sha256:c4c2f0810fa25323abfdfa36cbbbb24e5c3b1a42cb762782de64439c575d67f2",
+                "sha256:d71b31117779d9a90b745720c0eab54ae1da76d5b38c8026c654f4a066b0130a",
+                "sha256:dbe04e7568aa69361a5b4c47b4493d5680bfa3a911d1e105fbea1b1f23f3eb45",
+                "sha256:de86029696e1b3b4d0d49076b9eba606c226e33ae312a57a46dca14ff370894d",
+                "sha256:e3876611d114a18aafef6383695dfc3f1217c98a9168c1aaf1a02b01ec7d8d1e",
+                "sha256:ed6d5413474e209ba50b1a75b2d9eecf64d41e6e4501977991cdc755dc83ab0f",
+                "sha256:f90a2d4ad9a035cee7331c06a4cf2245e38bd7c89554fe3b616d90ab8aab89cc"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.6.6"
+            "version": "==1.6.7"
         },
         "decorator": {
             "hashes": [
@@ -589,19 +663,28 @@
         },
         "fiona": {
             "hashes": [
-                "sha256:2563d9a9f21390a7376556c82d9cef90a20a362949a9c7cb66f4ce63d90b66cd",
-                "sha256:4743fd04e3c1d88619e3c1e1266b826427c4e3fcf1286abbda8d3480028781bf",
-                "sha256:5558d925d67f5b08d0bdb3d58f3611b2e9d4f070cb9b026bbdd52b6d487d5e03",
-                "sha256:716201c21246587f374785bec6d6a20a984fe1f6c2b0e83bf15127eb8f724d0c",
-                "sha256:7f101957734898d2ad694be8d2bdd3f3e4f722387d3972c89dcedcae6b6c4bb7",
-                "sha256:84131992cc664d531c93d1d4860c4b7ceac0516a2f31b9cd71823889c21fa67d",
-                "sha256:c9adb0c4245aec2d642727e027b79b862be4ffa13bdcc65b7ea5f9405cec492c",
-                "sha256:cfeed61eb411ddc30f5048c43f3dfc0c03a97603dfa2bdc2ee16d5d063ff24b4",
-                "sha256:d38a6ef59087b5a20ad7298608c5392e37705ff14d27b44435e04072bbf6632c",
-                "sha256:fcfd8b67403de9b1cc53c045c72542e9f30cb15e617c89c41b928046a9b27daa"
+                "sha256:18649326a7724611b16b648e14fd094089d517413b95ac91d0cdb0adc5fcb8de",
+                "sha256:3f26c8b6ea9bc92cbd52a4dd83ffd44472450bf92f4e3d4ef2341adc2f35a54d",
+                "sha256:59a3800bc09ebee3516d64d02a8a6818d07ab1573c6096f3ef3468bf9f8f95f8",
+                "sha256:6ba2294bc6adcbc36229862667aac6b98e6c306e1958caf53b8bfcf9a3b8c77a",
+                "sha256:75924f69c51db6e258c91308780546278028c509db12aa33a47692a0266c9667",
+                "sha256:89cfcc3bdb4aba7bba1eb552b3866b851334693ab694529803122b21f5927960",
+                "sha256:904793b17aee70ca9c3d582dbf01623eccfdeacd00c5e1a8e421be41f2e43d67",
+                "sha256:a82a99ce9b3e7825740157c45c9fb2259d4e92f0a886aaac25f0db40ffe1eea3",
+                "sha256:b5cad3424b7473eb0e19f17ee45abec92133a694a4b452a278f02e3b8d0f810f",
+                "sha256:b88e2e6548a41c1dfa3f96c8275ff472a3edca729e14a641c0fa5b2e146a8ab5",
+                "sha256:c28d9ffa5d230a1d9eaf571529fa9eb7573d39613354c090ad077ad153a37ee1",
+                "sha256:c4aafdd565b3a30bdd78cafae35d4945f6741eef31401c1bb1e166b6262d7539",
+                "sha256:ce9a22c9883cc5d11c05ba3fb9db5082044a07c6b299753ea5bb8e178b8ba53b",
+                "sha256:d0df3e105ad7f0cca5f16b441c232fd693ef6c4adf2c1b6271aaaa1cdc06164d",
+                "sha256:d47777890aa1d715025abc7a6d6b2a6bb8d2a37cc94c44ce95940b80eda21444",
+                "sha256:df34c980cd7396adfbc89bbb363bdd6e358c76f91969fc98c9dfc076dd11638d",
+                "sha256:e33860aaf70bbd2726cff12fd3857bd832b6dc2ad3ce4b27e7563bd68abdc26f",
+                "sha256:e3ed1c0c1c60f710a612aaeb294de54214d228c4ef40e0c1dc159e46f86a9446",
+                "sha256:ed75dd29c89e0e455e3a322f28cd92f192bcb8fced16e2bfb6422a7f95ffe5e9"
             ],
             "index": "pypi",
-            "version": "==1.8.17"
+            "version": "==1.8.22"
         },
         "fonttools": {
             "hashes": [
@@ -626,22 +709,6 @@
             "index": "pypi",
             "version": "==0.12.2"
         },
-        "h5py": {
-            "hashes": [
-                "sha256:0b0f002f5f341afe7d3d7e15198e80d9021da24a4d182d88068d79bfc91fba86",
-                "sha256:1edf33e722d47c6eb3878d51173b23dd848939f006f41b498bafceff87fb4cbd",
-                "sha256:46917f20021dde02865572a5fd2bb620945f7b7cd268bdc8e3f5720c32b38140",
-                "sha256:708ddff49af12c01d77e0f9782bb1a0364d96459ec0d1f85d90baea6d203764b",
-                "sha256:8745e5159830d7975a9cf38690455f22601509cda04de29b7e88b3fbdc747611",
-                "sha256:8e809149f95d9a3a33b1279bfbf894c78635a5497e8d5ac37420fa5ec0cf4f29",
-                "sha256:aa511bd05a9174c3008becdc93bd5785e254d34a6ab5f0425e6b2fbbc88afa6d",
-                "sha256:bb4ce46095e3b16c872aaf62adad33f40039fecae04674eb62c035386affcb91",
-                "sha256:be2a545f09074546f73305e0db6d36aaf1fb6ea2fcf1add2ce306b9c7f78e55a",
-                "sha256:ee1c683d91ab010d5e85cb61e8f9e7ee0d8eab545bf3dd50a9618f1d0e8f615e"
-            ],
-            "index": "pypi",
-            "version": "==3.4.0"
-        },
         "idna": {
             "hashes": [
                 "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
@@ -652,11 +719,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20",
-                "sha256:ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09"
+                "sha256:23c2bcae4762dfb0bbe072d358faec24957901d75b6c4ab11172c0c982532402",
+                "sha256:8f8bd2af397cf33bd344d35cfe7f489219b7d14fc79a3f854b75b8417e9226b0"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==6.1.0"
+            "version": "==6.3.0"
         },
         "importlib-resources": {
             "hashes": [
@@ -895,11 +962,11 @@
         },
         "jupyter-server-fileid": {
             "hashes": [
-                "sha256:1e0816d0857f490fadea11348570f0cba03f70f315c9842225aecfa45882b6af",
-                "sha256:6092ef114eddccf6cba69c0f0feb612c2f476f2e9467828809edb854c18806bb"
+                "sha256:171538b7c7d08d11dbc57d4e6da196e0c258e4c2cd29249ef1e032bb423677f8",
+                "sha256:5b489c6fe6783c41174a728c7b81099608518387e53c3d53451a67f46a0cb7b0"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.8.0"
+            "version": "==0.9.0"
         },
         "jupyter-server-terminals": {
             "hashes": [
@@ -927,11 +994,11 @@
         },
         "jupyterlab": {
             "hashes": [
-                "sha256:6c87c5910f886a14009352bc63f640961b206f73ed650dcf94d65f9dfcb30f95",
-                "sha256:e55bc40c36c2a52b76cf301138507a5488eb769137dd39d9f31a6259a00c6b03"
+                "sha256:373e9cfb8a72edd294be14f16662563a220cecf0fb26de7aab1af9a29b689b82",
+                "sha256:6aba0caa771697d02fbf409f9767b2fdb4ee32ce935940e3b9a0d5d48d994d0f"
             ],
             "index": "pypi",
-            "version": "==3.6.2"
+            "version": "==3.6.3"
         },
         "jupyterlab-pygments": {
             "hashes": [
@@ -943,11 +1010,11 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:b4f5b48eaae1be83e2fd6fb77ac49d9b639be4ca4bd2e05b5368d29632a93725",
-                "sha256:ff1e7a81deb2dcb433215d469000988590fd5a5733574aa2698d643a6c9b3ace"
+                "sha256:0f9f6752b0c534a7b22a6542b984fa6a2c18ab4d4e0a4c79f191138506a9a75f",
+                "sha256:f4a7263ada89958854631a64bed45285caeac482925233159709f643c5871490"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.21.0"
+            "version": "==2.22.0"
         },
         "jupyterlab-widgets": {
             "hashes": [
@@ -1191,6 +1258,74 @@
             ],
             "version": "==2.0.5"
         },
+        "msgpack": {
+            "hashes": [
+                "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164",
+                "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b",
+                "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c",
+                "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf",
+                "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd",
+                "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d",
+                "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c",
+                "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a",
+                "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e",
+                "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd",
+                "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025",
+                "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5",
+                "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705",
+                "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a",
+                "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d",
+                "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb",
+                "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11",
+                "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f",
+                "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c",
+                "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d",
+                "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea",
+                "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba",
+                "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87",
+                "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a",
+                "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c",
+                "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080",
+                "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198",
+                "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9",
+                "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a",
+                "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b",
+                "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f",
+                "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437",
+                "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f",
+                "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7",
+                "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2",
+                "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0",
+                "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48",
+                "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898",
+                "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0",
+                "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57",
+                "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8",
+                "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282",
+                "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1",
+                "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82",
+                "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc",
+                "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb",
+                "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6",
+                "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7",
+                "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9",
+                "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c",
+                "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1",
+                "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed",
+                "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c",
+                "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c",
+                "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77",
+                "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81",
+                "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a",
+                "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3",
+                "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086",
+                "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9",
+                "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f",
+                "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b",
+                "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"
+            ],
+            "version": "==1.0.5"
+        },
         "multivolumefile": {
             "hashes": [
                 "sha256:237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678",
@@ -1216,27 +1351,27 @@
         },
         "nbclassic": {
             "hashes": [
-                "sha256:312b3f7d7ff2e6c261d51799bb12e6493498ab47d3469d3a01015d5533fd4d2b",
-                "sha256:df78e4ba143f35084ad060deec16cb1d4839dde47bfdbc4232beb7071a6d70ea"
+                "sha256:47791b04dbcb89bf7fde910a3d848fd4793a4248a8936202453631a87da37d51",
+                "sha256:d2c91adc7909b0270c73e3e253d3687a6704b4f0a94bc156a37c85eba09f4d37"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.5.4"
+            "version": "==0.5.5"
         },
         "nbclient": {
             "hashes": [
-                "sha256:884a3f4a8c4fc24bb9302f263e0af47d97f0d01fe11ba714171b320c8ac09547",
-                "sha256:d97ac6257de2794f5397609df754fcbca1a603e94e924eb9b99787c031ae2e7c"
+                "sha256:26e41c6dca4d76701988bc34f64e1bfc2413ae6d368f13d7b5ac407efb08c755",
+                "sha256:8fa96f7e36693d5e83408f5e840f113c14a45c279befe609904dbe05dad646d1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.7.2"
+            "version": "==0.7.3"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:8eed67bd8314f3ec87c4351c2f674af3a04e5890ab905d6bd927c05aec1cf27d",
-                "sha256:e41118f81698d3d59b3c7c2887937446048f741aba6c367c1c1a77810b3e2d08"
+                "sha256:78685362b11d2e8058e70196fe83b09abed8df22d3e599cf271f4d39fdc48b9e",
+                "sha256:d2e95904666f1ff77d36105b9de4e0801726f93b862d5b28f69e93d99ad3b19c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.2.10"
+            "version": "==7.3.1"
         },
         "nbformat": {
             "hashes": [
@@ -1256,52 +1391,44 @@
         },
         "netcdf4": {
             "hashes": [
-                "sha256:02c2317f377f23005e447e4b187ebce3cdeae76b9d0a443d9d6f5156a3980f04",
-                "sha256:087d09594c0739b03a18ed058d6eca8dc80cc672961b231473bdf2c728112ec8",
-                "sha256:0bf6b4375d99ef6aa9b5d82bdd2a92113c130821461e10bf9831609783f43a2a",
-                "sha256:1dd21bff8e29476decfe42602ab40694f10be84b1a585bc5d28c8b08c19f7283",
-                "sha256:1ee78d5c129fcfeafd6d4a6339984d6108dbabfb8a2cec651f9dc2132c792c78",
-                "sha256:206af8b5d110b89132ddea9b7b162efe82c26db37bc7a7633110750685243195",
-                "sha256:2106c5eaacb8e4dd1ad9837cb020fdf1b9c9e5630cd6648a6ac760690f098eed",
-                "sha256:34355a0129f05e44b26be22e352af183dee7dcd4c6504b398c10ef89ca9f9303",
-                "sha256:35f527f9d22b23c93cbb6b20e07fe1e3e27ed5ac80a699c2b1f2d150d5db70ee",
-                "sha256:393f9c7e92f03e08f8093100396dddb8619a43cb37ad2ac6beb70aaa71226dd5",
-                "sha256:41c0ff220a16c894bf147ce2b1c0266610a5b94bf7aed66c5ee2e5aeaa6bfb20",
-                "sha256:41ff33837732c94a9446ebc1562bf5d954c43e1009f1429423da9151bee5bac1",
-                "sha256:43c95d05458b730a496bbfa52080f4133ec60f07b3994a5b7f45034c65c6d31b",
-                "sha256:4933f948dba82f4f95dcb4d0c4227ab4e085c35e33048009c77ec446a86b2d63",
-                "sha256:50821324f804cd99043b7273d15e65e204cadbf7ca8af27059fade765fb2c430",
-                "sha256:56808afaf4f8afd6d7f6dcdba5ee39d37160028066abba140e0210bdb1d50712",
-                "sha256:5eb58e4b4ad9b79b72fceb8a806b5236b6ce4d5053f7020cff367fcc0bebc765",
-                "sha256:5f520457cf48541edb83655567b5b17cdb2df19a305703f015a85e7f2aceb560",
-                "sha256:7d31d1b736d402b2779f703479d0fbdee59fc6e1e880d1af6eb36fded934a982",
-                "sha256:8164915dc40baa26c65d0e5661d96cd8cbc87157563cf364167d9ec82a6ef7f6",
-                "sha256:8bb9e8b073e25c44cdcc42947e945a5b950ea93a519ba28fec642a92a1c983a9",
-                "sha256:8f4352717660ea48df8c21920f29c708ba69b7002f609dc89036507913b82608",
-                "sha256:a0a47694a123647ff7ef4c16a863510ad640fd44e75f8e2ee5042578ad2a0d67",
-                "sha256:ad705a74449bcefeebb6ae41a5e5987ab29942f508e092151dac7508803e12e8",
-                "sha256:aff200ee07495c73f4257f55d19d13f3678d7f3171d48a85481706e06a6db0e9",
-                "sha256:c06f59f54f73aab86b123838fabe68c728503be981cad9f28283dcfb627f7023",
-                "sha256:c270e4e5161955a35a152348a659270540c85fb2f238c13601c8649cd992032f",
-                "sha256:c3056cc36669abd35b152b978a0bc54b622995d9d7165f8d380c134fa60e27d6",
-                "sha256:c6b0c6ae9f9a050e9878f700e87405044fb2ee9a730559d6feb87f2361b361c6",
-                "sha256:ca8f6f519b290b68892186cb966ee8d2287faec8463128fe6aaec188a0c5ae96",
-                "sha256:cb374f259d0a6d0e436294607d4bc5999f20f61e1a7d365b50e07898ea43a7de",
-                "sha256:d145f9c12da29da3922d8b8aafea2a2a89501bcb28a219a46b7b828b57191594",
-                "sha256:e8d64208e69cd5e67112eb4378066946124dcd1fbb7c2aa3d12d488f4d024759",
-                "sha256:f7b8b19971ff5e1dc5267f1f917667214c6614816f879ff6620bc152132c2629",
-                "sha256:fc489ecca6fb2e5ab738eedf962f9213dcde005b6f9e6f9b2d3d9d6513811759"
+                "sha256:1194e88dbf5b35dc344a1e520c19140a0a51cc328c32f35d04c5a17ffc623614",
+                "sha256:1831f99bb84d8db5901a4ecb7e382bb8d93847269a48a56b3b3e18cd2b564fd8",
+                "sha256:269f3817604bfbd08c7c8acccb6a5863a0a0222425203d9548ad42cbb554254b",
+                "sha256:3d6a4f7da760ee713aa1197e073c9f066476699dc0cd277428bd5256ebb3878a",
+                "sha256:4f7735871ffc0c8fef710a6b8c5f5af04ed480fee2dc23e737a378f9630d9475",
+                "sha256:5de65543a325451c1ea23b1361fef19da42f7e9874a92a4475c290a045db90b4",
+                "sha256:628d48a31e4b094e807252423e5d1c5b39d4d173fde5cdbe7a18da7626cb606c",
+                "sha256:6fc24dc9d39fee9206710f660b12eb3529d6817583a9de8391e62d4f9f4367fb",
+                "sha256:729544be6ca6a4507d4b3fdd46af578d17c834e1bb53719bebb90ac108035f6a",
+                "sha256:85384e575ddc7e329ca409d380a2d92bb52da5917a171055cf49435f0b6ce07e",
+                "sha256:8c98a3a8cda06920ee8bd24a71226ddf0328c22bd838b0afca9cb45fb4580d99",
+                "sha256:969470f70c6fb51f9fe851cc55d29769c798aeb2de2cb97200f8dac498f299b4",
+                "sha256:9dfab2b84b3f29902515897c6b2b5a92327a78abbfde6ac3917dae80dd17835b",
+                "sha256:a006a912ca204f74f7ec625b1b8d8b06e2f8366fd5be46bacb5e20760684a852",
+                "sha256:a02a8cd53311a447e0c5c185a0d79c3b5d57d49cb1743459417b5e8ca18561bb",
+                "sha256:b46e8404e3526047070f88ad3a65acb813ffccf41d8ff12ff823320be730ba66",
+                "sha256:bbfc767980f87c184f6d6fc9d5a164caa0895c0a6b1820c779f7ec7789c01b0e",
+                "sha256:c94f95ac1ff5590aeb1793eee10519422a9a02da0ece1daf7efa597eabb4e246",
+                "sha256:cb1647d2878a081b4a83fe6d5d5792d8befa59b2b18487bee93aae4b8efa0762",
+                "sha256:ccb1524eb4ea9ec1c4360070b12840784c4afa5e539d509b1f2a921f26e49f39",
+                "sha256:ce9c0b86f603fdaf4723fb92f728265f456f9090544cd62abe27794046bee507",
+                "sha256:d7c45f7f729cfdca8cb7b80373085036a08c88bd6f9d8bfcea559056206f3c3d",
+                "sha256:ddd889dff168baa2fe4777e9117175ecd127e224304950786499744c8956d877",
+                "sha256:e62554a197a344858a526c0cc8340b56d4ab7708feab08528bb602150b8139b1",
+                "sha256:eb06c09bd5a6b44d65d38f544b62f503ede05dacdc9d5cfa6da25b6f738da1bb",
+                "sha256:f6b6d88480052efb3e8b0dd56f773064ed00edcf6ab028ef8fe79a9a06179f43",
+                "sha256:f94a89db78f34fdf68342840efb064fe1474310e8359dffce42e90a9ddf88f2f"
             ],
             "index": "pypi",
-            "version": "==1.5.7"
+            "version": "==1.6.3"
         },
         "notebook": {
             "hashes": [
-                "sha256:50a334ad9d60b30cb759405168ef6fc3d60350ab5439fb1631544bb09dcb2cce",
-                "sha256:b12bee3292211d85dd7e588a790ddce30cb3e8fbcfa1e803522a207f60819e05"
+                "sha256:517209568bd47261e2def27a140e97d49070602eea0d226a696f42a7f16c9a4e",
+                "sha256:dd17e78aefe64c768737b32bf171c1c766666a21cc79a44d37a1700771cab56f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.5.3"
+            "version": "==6.5.4"
         },
         "notebook-shim": {
             "hashes": [
@@ -1425,25 +1552,34 @@
         },
         "pandas": {
             "hashes": [
-                "sha256:0210f8fe19c2667a3817adb6de2c4fd92b1b78e1975ca60c0efa908e0985cbdb",
-                "sha256:0227e3a6e3a22c0e283a5041f1e3064d78fbde811217668bb966ed05386d8a7e",
-                "sha256:0bc440493cf9dc5b36d5d46bbd5508f6547ba68b02a28234cd8e81fdce42744d",
-                "sha256:16504f915f1ae424052f1e9b7cd2d01786f098fbb00fa4e0f69d42b22952d798",
-                "sha256:182a5aeae319df391c3df4740bb17d5300dcd78034b17732c12e62e6dd79e4a4",
-                "sha256:35db623487f00d9392d8af44a24516d6cb9f274afaf73cfcfe180b9c54e007d2",
-                "sha256:40ec0a7f611a3d00d3c666c4cceb9aa3f5bf9fbd81392948a93663064f527203",
-                "sha256:47a03bfef80d6812c91ed6fae43f04f2fa80a4e1b82b35aa4d9002e39529e0b8",
-                "sha256:4b21d46728f8a6be537716035b445e7ef3a75dbd30bd31aa1b251323219d853e",
-                "sha256:4d1a806252001c5db7caecbe1a26e49a6c23421d85a700960f6ba093112f54a1",
-                "sha256:60e20a4ab4d4fec253557d0fc9a4e4095c37b664f78c72af24860c8adcd07088",
-                "sha256:9f61cca5262840ff46ef857d4f5f65679b82188709d0e5e086a9123791f721c8",
-                "sha256:a15835c8409d5edc50b4af93be3377b5dd3eb53517e7f785060df1f06f6da0e2",
-                "sha256:b39508562ad0bb3f384b0db24da7d68a2608b9ddc85b1d931ccaaa92d5e45273",
-                "sha256:ed60848caadeacecefd0b1de81b91beff23960032cded0ac1449242b506a3b3f",
-                "sha256:fc714895b6de6803ac9f661abb316853d0cd657f5d23985222255ad76ccedc25"
+                "sha256:0778ab54c8f399d83d98ffb674d11ec716449956bc6f6821891ab835848687f2",
+                "sha256:24472cfc7ced511ac90608728b88312be56edc8f19b9ed885a7d2e47ffaf69c0",
+                "sha256:2d1d138848dd71b37e3cbe7cd952ff84e2ab04d8988972166e18567dcc811245",
+                "sha256:3bb9d840bf15656805f6a3d87eea9dcb7efdf1314a82adcf7f00b820427c5570",
+                "sha256:425705cee8be54db2504e8dd2a730684790b15e5904b750c367611ede49098ab",
+                "sha256:4f3320bb55f34af4193020158ef8118ee0fb9aec7cc47d2084dbfdd868a0a24f",
+                "sha256:4ffb14f50c74ee541610668137830bb93e9dfa319b1bef2cedf2814cd5ac9c70",
+                "sha256:52c858de9e9fc422d25e67e1592a6e6135d7bcf9a19fcaf4d0831a0be496bf21",
+                "sha256:57c34b79c13249505e850d0377b722961b99140f81dafbe6f19ef10239f6284a",
+                "sha256:6ded51f7e3dd9b4f8b87f2ceb7bd1a8df2491f7ee72f7074c6927a512607199e",
+                "sha256:70db5c278bbec0306d32bf78751ff56b9594c05a5098386f6c8a563659124f91",
+                "sha256:78425ca12314b23356c28b16765639db10ebb7d8983f705d6759ff7fe41357fa",
+                "sha256:8318de0f886e4dcb8f9f36e45a3d6a6c3d1cfdc508354da85e739090f0222991",
+                "sha256:8f987ec26e96a8490909bc5d98c514147236e49830cba7df8690f6087c12bbae",
+                "sha256:9253edfd015520ce77a9343eb7097429479c039cd3ebe81d7810ea11b4b24695",
+                "sha256:977326039bd1ded620001a1889e2ed4798460a6bc5a24fbaebb5f07a41c32a55",
+                "sha256:a4f789b7c012a608c08cda4ff0872fd979cb18907a37982abe884e6f529b8793",
+                "sha256:b3ba8f5dd470d8bfbc4259829589f4a32881151c49e36384d9eb982b35a12020",
+                "sha256:b5337c87c4e963f97becb1217965b6b75c6fe5f54c4cf09b9a5ac52fc0bd03d3",
+                "sha256:bbb2c5e94d6aa4e632646a3bacd05c2a871c3aa3e85c9bec9be99cb1267279f2",
+                "sha256:c24c7d12d033a372a9daf9ff2c80f8b0af6f98d14664dbb0a4f6a029094928a7",
+                "sha256:cda9789e61b44463c1c4fe17ef755de77bcd13b09ba31c940d20f193d63a5dc8",
+                "sha256:d08e41d96bc4de6f500afe80936c68fce6099d5a434e2af7c7fd8e7c72a3265d",
+                "sha256:d93b7fcfd9f3328072b250d6d001dcfeec5d3bb66c1b9c8941e109a46c0c01a8",
+                "sha256:fcd471c9d9f60926ab2f15c6c29164112f458acb42280365fbefa542d0c2fc74"
             ],
             "index": "pypi",
-            "version": "==1.1"
+            "version": "==2.0.0"
         },
         "pandocfilters": {
             "hashes": [
@@ -1478,86 +1614,75 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:013016af6b3a12a2f40b704677f8b51f72cb007dac785a9933d5c86a72a7fe33",
-                "sha256:0845adc64fe9886db00f5ab68c4a8cd933ab749a87747555cec1c95acea64b0b",
-                "sha256:0884ba7b515163a1a05440a138adeb722b8a6ae2c2b33aea93ea3118dd3a899e",
-                "sha256:09b89ddc95c248ee788328528e6a2996e09eaccddeeb82a5356e92645733be35",
-                "sha256:0dd4c681b82214b36273c18ca7ee87065a50e013112eea7d78c7a1b89a739153",
-                "sha256:0e51f608da093e5d9038c592b5b575cadc12fd748af1479b5e858045fff955a9",
-                "sha256:0f3269304c1a7ce82f1759c12ce731ef9b6e95b6df829dccd9fe42912cc48569",
-                "sha256:16a8df99701f9095bea8a6c4b3197da105df6f74e6176c5b410bc2df2fd29a57",
-                "sha256:19005a8e58b7c1796bc0167862b1f54a64d3b44ee5d48152b06bb861458bc0f8",
-                "sha256:1b4b4e9dda4f4e4c4e6896f93e84a8f0bcca3b059de9ddf67dac3c334b1195e1",
-                "sha256:28676836c7796805914b76b1837a40f76827ee0d5398f72f7dcc634bae7c6264",
-                "sha256:2968c58feca624bb6c8502f9564dd187d0e1389964898f5e9e1fbc8533169157",
-                "sha256:3f4cc516e0b264c8d4ccd6b6cbc69a07c6d582d8337df79be1e15a5056b258c9",
-                "sha256:3fa1284762aacca6dc97474ee9c16f83990b8eeb6697f2ba17140d54b453e133",
-                "sha256:43521ce2c4b865d385e78579a082b6ad1166ebed2b1a2293c3be1d68dd7ca3b9",
-                "sha256:451f10ef963918e65b8869e17d67db5e2f4ab40e716ee6ce7129b0cde2876eab",
-                "sha256:46c259e87199041583658457372a183636ae8cd56dbf3f0755e0f376a7f9d0e6",
-                "sha256:46f39cab8bbf4a384ba7cb0bc8bae7b7062b6a11cfac1ca4bc144dea90d4a9f5",
-                "sha256:519e14e2c49fcf7616d6d2cfc5c70adae95682ae20f0395e9280db85e8d6c4df",
-                "sha256:53dcb50fbdc3fb2c55431a9b30caeb2f7027fcd2aeb501459464f0214200a503",
-                "sha256:54614444887e0d3043557d9dbc697dbb16cfb5a35d672b7a0fcc1ed0cf1c600b",
-                "sha256:575d8912dca808edd9acd6f7795199332696d3469665ef26163cd090fa1f8bfa",
-                "sha256:5dd5a9c3091a0f414a963d427f920368e2b6a4c2f7527fdd82cde8ef0bc7a327",
-                "sha256:5f532a2ad4d174eb73494e7397988e22bf427f91acc8e6ebf5bb10597b49c493",
-                "sha256:60e7da3a3ad1812c128750fc1bc14a7ceeb8d29f77e0a2356a8fb2aa8925287d",
-                "sha256:653d7fb2df65efefbcbf81ef5fe5e5be931f1ee4332c2893ca638c9b11a409c4",
-                "sha256:6663977496d616b618b6cfa43ec86e479ee62b942e1da76a2c3daa1c75933ef4",
-                "sha256:6abfb51a82e919e3933eb137e17c4ae9c0475a25508ea88993bb59faf82f3b35",
-                "sha256:6c6b1389ed66cdd174d040105123a5a1bc91d0aa7059c7261d20e583b6d8cbd2",
-                "sha256:6d9dfb9959a3b0039ee06c1a1a90dc23bac3b430842dcb97908ddde05870601c",
-                "sha256:765cb54c0b8724a7c12c55146ae4647e0274a839fb6de7bcba841e04298e1011",
-                "sha256:7a21222644ab69ddd9967cfe6f2bb420b460dae4289c9d40ff9a4896e7c35c9a",
-                "sha256:7ac7594397698f77bce84382929747130765f66406dc2cd8b4ab4da68ade4c6e",
-                "sha256:7cfc287da09f9d2a7ec146ee4d72d6ea1342e770d975e49a8621bf54eaa8f30f",
-                "sha256:83125753a60cfc8c412de5896d10a0a405e0bd88d0470ad82e0869ddf0cb3848",
-                "sha256:847b114580c5cc9ebaf216dd8c8dbc6b00a3b7ab0131e173d7120e6deade1f57",
-                "sha256:87708d78a14d56a990fbf4f9cb350b7d89ee8988705e58e39bdf4d82c149210f",
-                "sha256:8a2b5874d17e72dfb80d917213abd55d7e1ed2479f38f001f264f7ce7bae757c",
-                "sha256:8f127e7b028900421cad64f51f75c051b628db17fb00e099eb148761eed598c9",
-                "sha256:94cdff45173b1919350601f82d61365e792895e3c3a3443cf99819e6fbf717a5",
-                "sha256:99d92d148dd03fd19d16175b6d355cc1b01faf80dae93c6c3eb4163709edc0a9",
-                "sha256:9a3049a10261d7f2b6514d35bbb7a4dfc3ece4c4de14ef5876c4b7a23a0e566d",
-                "sha256:9d9a62576b68cd90f7075876f4e8444487db5eeea0e4df3ba298ee38a8d067b0",
-                "sha256:9e5f94742033898bfe84c93c831a6f552bb629448d4072dd312306bab3bd96f1",
-                "sha256:a1c2d7780448eb93fbcc3789bf3916aa5720d942e37945f4056680317f1cd23e",
-                "sha256:a2e0f87144fcbbe54297cae708c5e7f9da21a4646523456b00cc956bd4c65815",
-                "sha256:a4dfdae195335abb4e89cc9762b2edc524f3c6e80d647a9a81bf81e17e3fb6f0",
-                "sha256:a96e6e23f2b79433390273eaf8cc94fec9c6370842e577ab10dabdcc7ea0a66b",
-                "sha256:aabdab8ec1e7ca7f1434d042bf8b1e92056245fb179790dc97ed040361f16bfd",
-                "sha256:b222090c455d6d1a64e6b7bb5f4035c4dff479e22455c9eaa1bdd4c75b52c80c",
-                "sha256:b52ff4f4e002f828ea6483faf4c4e8deea8d743cf801b74910243c58acc6eda3",
-                "sha256:b70756ec9417c34e097f987b4d8c510975216ad26ba6e57ccb53bc758f490dab",
-                "sha256:b8c2f6eb0df979ee99433d8b3f6d193d9590f735cf12274c108bd954e30ca858",
-                "sha256:b9b752ab91e78234941e44abdecc07f1f0d8f51fb62941d32995b8161f68cfe5",
-                "sha256:ba6612b6548220ff5e9df85261bddc811a057b0b465a1226b39bfb8550616aee",
-                "sha256:bd752c5ff1b4a870b7661234694f24b1d2b9076b8bf337321a814c612665f343",
-                "sha256:c3c4ed2ff6760e98d262e0cc9c9a7f7b8a9f61aa4d47c58835cdaf7b0b8811bb",
-                "sha256:c5c1362c14aee73f50143d74389b2c158707b4abce2cb055b7ad37ce60738d47",
-                "sha256:cb362e3b0976dc994857391b776ddaa8c13c28a16f80ac6522c23d5257156bed",
-                "sha256:d197df5489004db87d90b918033edbeee0bd6df3848a204bca3ff0a903bef837",
-                "sha256:d3b56206244dc8711f7e8b7d6cad4663917cd5b2d950799425076681e8766286",
-                "sha256:d5b2f8a31bd43e0f18172d8ac82347c8f37ef3e0b414431157718aa234991b28",
-                "sha256:d7081c084ceb58278dd3cf81f836bc818978c0ccc770cbbb202125ddabec6628",
-                "sha256:db74f5562c09953b2c5f8ec4b7dfd3f5421f31811e97d1dbc0a7c93d6e3a24df",
-                "sha256:df41112ccce5d47770a0c13651479fbcd8793f34232a2dd9faeccb75eb5d0d0d",
-                "sha256:e1339790c083c5a4de48f688b4841f18df839eb3c9584a770cbd818b33e26d5d",
-                "sha256:e621b0246192d3b9cb1dc62c78cfa4c6f6d2ddc0ec207d43c0dedecb914f152a",
-                "sha256:e8c5cf126889a4de385c02a2c3d3aba4b00f70234bfddae82a5eaa3ee6d5e3e6",
-                "sha256:e9d7747847c53a16a729b6ee5e737cf170f7a16611c143d95aa60a109a59c336",
-                "sha256:eaef5d2de3c7e9b21f1e762f289d17b726c2239a42b11e25446abf82b26ac132",
-                "sha256:ed3e4b4e1e6de75fdc16d3259098de7c6571b1a6cc863b1a49e7d3d53e036070",
-                "sha256:ef21af928e807f10bf4141cad4746eee692a0dd3ff56cfb25fce076ec3cc8abe",
-                "sha256:f09598b416ba39a8f489c124447b007fe865f786a89dbfa48bb5cf395693132a",
-                "sha256:f0caf4a5dcf610d96c3bd32932bfac8aee61c96e60481c2a0ea58da435e25acd",
-                "sha256:f6e78171be3fb7941f9910ea15b4b14ec27725865a73c15277bc39f5ca4f8391",
-                "sha256:f715c32e774a60a337b2bb8ad9839b4abf75b267a0f18806f6f4f5f1688c4b5a",
-                "sha256:fb5c1ad6bad98c57482236a21bf985ab0ef42bd51f7ad4e4538e89a997624e12"
+                "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1",
+                "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba",
+                "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a",
+                "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799",
+                "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51",
+                "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb",
+                "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5",
+                "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270",
+                "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6",
+                "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47",
+                "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf",
+                "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e",
+                "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b",
+                "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66",
+                "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865",
+                "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec",
+                "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c",
+                "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1",
+                "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38",
+                "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906",
+                "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705",
+                "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef",
+                "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc",
+                "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f",
+                "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf",
+                "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392",
+                "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d",
+                "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe",
+                "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32",
+                "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5",
+                "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7",
+                "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44",
+                "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d",
+                "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3",
+                "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625",
+                "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e",
+                "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829",
+                "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089",
+                "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3",
+                "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78",
+                "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96",
+                "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964",
+                "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597",
+                "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99",
+                "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a",
+                "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140",
+                "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7",
+                "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16",
+                "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903",
+                "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1",
+                "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296",
+                "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572",
+                "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115",
+                "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a",
+                "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd",
+                "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4",
+                "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1",
+                "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb",
+                "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa",
+                "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a",
+                "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569",
+                "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c",
+                "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf",
+                "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082",
+                "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062",
+                "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==9.4.0"
+            "version": "==9.5.0"
         },
         "pkgutil-resolve-name": {
             "hashes": [
@@ -1616,49 +1741,76 @@
                 "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7",
                 "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"
             ],
-            "markers": "sys_platform != 'cygwin'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==5.9.4"
         },
         "psycopg2-binary": {
             "hashes": [
-                "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c",
-                "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67",
-                "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0",
-                "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6",
-                "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db",
-                "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94",
-                "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52",
-                "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056",
-                "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b",
-                "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd",
-                "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550",
-                "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679",
-                "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83",
-                "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77",
-                "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2",
-                "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77",
-                "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2",
-                "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd",
-                "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859",
-                "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1",
-                "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25",
-                "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152",
-                "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf",
-                "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f",
-                "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729",
-                "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71",
-                "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66",
-                "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4",
-                "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449",
-                "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da",
-                "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a",
-                "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c",
-                "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb",
-                "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4",
-                "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5"
+                "sha256:02c0f3757a4300cf379eb49f543fb7ac527fb00144d39246ee40e1df684ab514",
+                "sha256:02c6e3cf3439e213e4ee930308dc122d6fb4d4bea9aef4a12535fbd605d1a2fe",
+                "sha256:0645376d399bfd64da57148694d78e1f431b1e1ee1054872a5713125681cf1be",
+                "sha256:0892ef645c2fabb0c75ec32d79f4252542d0caec1d5d949630e7d242ca4681a3",
+                "sha256:0d236c2825fa656a2d98bbb0e52370a2e852e5a0ec45fc4f402977313329174d",
+                "sha256:0e0f754d27fddcfd74006455b6e04e6705d6c31a612ec69ddc040a5468e44b4e",
+                "sha256:15e2ee79e7cf29582ef770de7dab3d286431b01c3bb598f8e05e09601b890081",
+                "sha256:1876843d8e31c89c399e31b97d4b9725a3575bb9c2af92038464231ec40f9edb",
+                "sha256:1f64dcfb8f6e0c014c7f55e51c9759f024f70ea572fbdef123f85318c297947c",
+                "sha256:2ab652e729ff4ad76d400df2624d223d6e265ef81bb8aa17fbd63607878ecbee",
+                "sha256:30637a20623e2a2eacc420059be11527f4458ef54352d870b8181a4c3020ae6b",
+                "sha256:34b9ccdf210cbbb1303c7c4db2905fa0319391bd5904d32689e6dd5c963d2ea8",
+                "sha256:38601cbbfe600362c43714482f43b7c110b20cb0f8172422c616b09b85a750c5",
+                "sha256:441cc2f8869a4f0f4bb408475e5ae0ee1f3b55b33f350406150277f7f35384fc",
+                "sha256:498807b927ca2510baea1b05cc91d7da4718a0f53cb766c154c417a39f1820a0",
+                "sha256:4ac30da8b4f57187dbf449294d23b808f8f53cad6b1fc3623fa8a6c11d176dd0",
+                "sha256:4c727b597c6444a16e9119386b59388f8a424223302d0c06c676ec8b4bc1f963",
+                "sha256:4d67fbdaf177da06374473ef6f7ed8cc0a9dc640b01abfe9e8a2ccb1b1402c1f",
+                "sha256:4dfb4be774c4436a4526d0c554af0cc2e02082c38303852a36f6456ece7b3503",
+                "sha256:4ea29fc3ad9d91162c52b578f211ff1c931d8a38e1f58e684c45aa470adf19e2",
+                "sha256:51537e3d299be0db9137b321dfb6a5022caaab275775680e0c3d281feefaca6b",
+                "sha256:61b047a0537bbc3afae10f134dc6393823882eb263088c271331602b672e52e9",
+                "sha256:6460c7a99fc939b849431f1e73e013d54aa54293f30f1109019c56a0b2b2ec2f",
+                "sha256:65bee1e49fa6f9cf327ce0e01c4c10f39165ee76d35c846ade7cb0ec6683e303",
+                "sha256:65c07febd1936d63bfde78948b76cd4c2a411572a44ac50719ead41947d0f26b",
+                "sha256:71f14375d6f73b62800530b581aed3ada394039877818b2d5f7fc77e3bb6894d",
+                "sha256:7a40c00dbe17c0af5bdd55aafd6ff6679f94a9be9513a4c7e071baf3d7d22a70",
+                "sha256:7e13a5a2c01151f1208d5207e42f33ba86d561b7a89fca67c700b9486a06d0e2",
+                "sha256:7f0438fa20fb6c7e202863e0d5ab02c246d35efb1d164e052f2f3bfe2b152bd0",
+                "sha256:8122cfc7cae0da9a3077216528b8bb3629c43b25053284cc868744bfe71eb141",
+                "sha256:8338a271cb71d8da40b023a35d9c1e919eba6cbd8fa20a54b748a332c355d896",
+                "sha256:84d2222e61f313c4848ff05353653bf5f5cf6ce34df540e4274516880d9c3763",
+                "sha256:8a6979cf527e2603d349a91060f428bcb135aea2be3201dff794813256c274f1",
+                "sha256:8a76e027f87753f9bd1ab5f7c9cb8c7628d1077ef927f5e2446477153a602f2c",
+                "sha256:964b4dfb7c1c1965ac4c1978b0f755cc4bd698e8aa2b7667c575fb5f04ebe06b",
+                "sha256:9972aad21f965599ed0106f65334230ce826e5ae69fda7cbd688d24fa922415e",
+                "sha256:a8c28fd40a4226b4a84bdf2d2b5b37d2c7bd49486b5adcc200e8c7ec991dfa7e",
+                "sha256:ae102a98c547ee2288637af07393dd33f440c25e5cd79556b04e3fca13325e5f",
+                "sha256:af335bac6b666cc6aea16f11d486c3b794029d9df029967f9938a4bed59b6a19",
+                "sha256:afe64e9b8ea66866a771996f6ff14447e8082ea26e675a295ad3bdbffdd72afb",
+                "sha256:b4b24f75d16a89cc6b4cdff0eb6a910a966ecd476d1e73f7ce5985ff1328e9a6",
+                "sha256:b6c8288bb8a84b47e07013bb4850f50538aa913d487579e1921724631d02ea1b",
+                "sha256:b83456c2d4979e08ff56180a76429263ea254c3f6552cd14ada95cff1dec9bb8",
+                "sha256:bfb13af3c5dd3a9588000910178de17010ebcccd37b4f9794b00595e3a8ddad3",
+                "sha256:c3dba7dab16709a33a847e5cd756767271697041fbe3fe97c215b1fc1f5c9848",
+                "sha256:c48d8f2db17f27d41fb0e2ecd703ea41984ee19362cbce52c097963b3a1b4365",
+                "sha256:c7e62ab8b332147a7593a385d4f368874d5fe4ad4e341770d4983442d89603e3",
+                "sha256:c83a74b68270028dc8ee74d38ecfaf9c90eed23c8959fca95bd703d25b82c88e",
+                "sha256:cacbdc5839bdff804dfebc058fe25684cae322987f7a38b0168bc1b2df703fb1",
+                "sha256:cf4499e0a83b7b7edcb8dabecbd8501d0d3a5ef66457200f77bde3d210d5debb",
+                "sha256:cfec476887aa231b8548ece2e06d28edc87c1397ebd83922299af2e051cf2827",
+                "sha256:d26e0342183c762de3276cca7a530d574d4e25121ca7d6e4a98e4f05cb8e4df7",
+                "sha256:d4e6036decf4b72d6425d5b29bbd3e8f0ff1059cda7ac7b96d6ac5ed34ffbacd",
+                "sha256:d57c3fd55d9058645d26ae37d76e61156a27722097229d32a9e73ed54819982a",
+                "sha256:dfa74c903a3c1f0d9b1c7e7b53ed2d929a4910e272add6700c38f365a6002820",
+                "sha256:e3ed340d2b858d6e6fb5083f87c09996506af483227735de6964a6100b4e6a54",
+                "sha256:e78e6e2a00c223e164c417628572a90093c031ed724492c763721c2e0bc2a8df",
+                "sha256:e9182eb20f41417ea1dd8e8f7888c4d7c6e805f8a7c98c1081778a3da2bee3e4",
+                "sha256:e99e34c82309dd78959ba3c1590975b5d3c862d6f279f843d47d26ff89d7d7e1",
+                "sha256:f6a88f384335bb27812293fdb11ac6aee2ca3f51d3c7820fe03de0a304ab6249",
+                "sha256:f81e65376e52f03422e1fb475c9514185669943798ed019ac50410fb4c4df232",
+                "sha256:ffe9dc0a884a8848075e576c1de0290d85a533a9f6e9c4e564f19adf8f6e54a7"
             ],
             "index": "pypi",
-            "version": "==2.8.6"
+            "version": "==2.9.6"
         },
         "ptyprocess": {
             "hashes": [
@@ -1675,6 +1827,13 @@
             ],
             "version": "==0.2.2"
         },
+        "py-cpuinfo": {
+            "hashes": [
+                "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690",
+                "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"
+            ],
+            "version": "==9.0.0"
+        },
         "py7zr": {
             "hashes": [
                 "sha256:1d01f98ea1e1f5c49940358691b2076f9a5848056426541e783de33834f59e21",
@@ -1846,11 +2005,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297",
-                "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"
+                "sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094",
+                "sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.14.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.15.0"
         },
         "pyparsing": {
             "hashes": [
@@ -1946,28 +2105,44 @@
         },
         "pyproj": {
             "hashes": [
-                "sha256:04c185102e659439c5bd428ac5473d36ef795fca8e225bbbe78e20643d804ec0",
-                "sha256:10dad599b9f7ce2194996dc25f1000e0aa15754ecef9db46b624713959c67957",
-                "sha256:1e88ebc4e08e661e9011b5c1ebfb32f0d311963a9824a6effb4168c7e07918b1",
-                "sha256:4f3ad09cf3352bf5664794042b28d98781362ec8d9774ad73f28a1a0101a27f1",
-                "sha256:5f8a8d982bde211e65dc2de1f8f36cf162f9cc7fcd8a7625046ea265284e5e65",
-                "sha256:67b94f4e694ae33fc90dfb7da0e6b5ed5f671dd0acc2f6cf46e9c39d56e16e1a",
-                "sha256:808f5992320e9631b2e45444028a65cd6ba3ee40229292934178ef07020a5ffd",
-                "sha256:8eda240225971b5cd0bac2d399ed6222068f0598ee92d5f6e847bd2019d2c8b0",
-                "sha256:911d773da9fa4d4f3f7580173858c391e3ee0b61acaf0be303baab323d2eae78",
-                "sha256:9cc464a1c51baad28ffb7a233116e8d4ce4c560b32039fa986d0f992ac3c431f",
-                "sha256:a162ed199cd2ec392cffe20b2fa3381b68e7a166d55f3f060eceb8d517e4f46d",
-                "sha256:aa87df0982aa0f4477478899d9c930cc0f97cd6d8a4ce84c43ac88ccf86d1da7",
-                "sha256:ae237492767e0225f99b53a0fd7110fde2b7e7cabc105bbc243c151a7497de88",
-                "sha256:ae5534fa7a3b74f20534694d297fce6f7483890ff6ca404394ecf372f3c589d4",
-                "sha256:b635e7e21fea5af74e90fc9e54d1a4c27078efdce6f214101c98dd93afae599a",
-                "sha256:b6c74bbec679199746a3e02c0e0fad093c3652df96dd63e086a2fbf2afe9dc0e",
-                "sha256:c4193e1069d165476b2d0f7d882b7712b3eab6e2e6fe2a0a78ef40de825a1f28",
-                "sha256:da88abc5e2f6a8fb07533855a57ca2a31845f58901a87f821b68b0db6b023978",
-                "sha256:ebbba7707fe83a01e54bce8e3e7342feb0b3e0d74ff8c28df12f8bc59b76827c"
+                "sha256:052c49fce8b5d55943a35c36ccecb87350c68b48ba95bc02a789770c374ef819",
+                "sha256:1507138ea28bf2134d31797675380791cc1a7156a3aeda484e65a78a4aba9b62",
+                "sha256:1798ff7d65d9057ebb2d017ffe8403268b8452f24d0428b2140018c25c7fa1bc",
+                "sha256:1b7c2113c4d11184a238077ec85e31eda1dcc58ffeb9a4429830e0a7036e787d",
+                "sha256:2b708fd43453b985642b737d4a6e7f1d6a0ab1677ffa4e14cc258537b49224b0",
+                "sha256:385b0341861d3ebc8cad98337a738821dcb548d465576527399f4955ca24b6ed",
+                "sha256:38862fe07316ae12b79d82d298e390973a4f00b684f3c2d037238e20e00610ba",
+                "sha256:5674923351e76222e2c10c58b5e1ac119d7a46b270d822c463035971b06f724b",
+                "sha256:5c4b85ac10d733c42d73a2e6261c8d6745bf52433a31848dd1b6561c9a382da3",
+                "sha256:61e4ad57d89b03a7b173793b31bca8ee110112cde1937ef0f42a70b9120c827d",
+                "sha256:621d78a9d8bf4d06e08bef2471021fbcb1a65aa629ad4a20c22e521ce729cc20",
+                "sha256:6475ce653880938468a1a1b7321267243909e34b972ba9e53d5982c41d555918",
+                "sha256:6a87b419a2a352413fbf759ecb66da9da50bd19861c8f26db6a25439125b27b9",
+                "sha256:6f316a66031a14e9c5a88c91f8b77aa97f5454895674541ed6ab630b682be35d",
+                "sha256:71b65f2a38cd9e16883dbb0f8ae82bdf8f6b79b1b02975c78483ab8428dbbf2f",
+                "sha256:73f7960a97225812f9b1d7aeda5fb83812f38de9441e3476fcc8abb3e2b2f4de",
+                "sha256:7572983134e310e0ca809c63f1722557a040fe9443df5f247bf11ba887eb1229",
+                "sha256:788a5dadb532644a64efe0f5f01bf508c821eb7e984f13a677d56002f1e8a67a",
+                "sha256:7bdd2021bb6f7f346bfe1d2a358aa109da017d22c4704af2d994e7c7ee0a7a53",
+                "sha256:8fe6bb1b68a35d07378d38be77b5b2f8dd2bea5910c957bfcc7bee55988d3910",
+                "sha256:97ed199033c2c770e7eea2ef80ff5e6413426ec2d7ec985b869792f04ab95d05",
+                "sha256:9859d1591c1863414d875ae0759e72c2cffc01ab989dc64137fbac572cc81bf6",
+                "sha256:a730f5b4c98c8a0f312437873e6e34dbd4cc6dc23d5afd91a6691c62724b1f68",
+                "sha256:b60d93a200639e8367c6542a964fd0aa2dbd152f256c1831dc18cd5aa470fb8a",
+                "sha256:b752b7d9c4b08181c7e8c0d9c7f277cbefff42227f34d3310696a87c863d9dd3",
+                "sha256:b937215bfbaf404ec8f03ca741fc3f9f2c4c2c5590a02ccddddd820ae3c71331",
+                "sha256:c02742ef3d846401861a878a61ef7ad911ea7539d6cc4619ddb52dbdf7b45aee",
+                "sha256:cd5e2b6aa255023c4acd0b977590f1f7cc801ba21b4d806fcf6dfac3474ebb83",
+                "sha256:d711517a8487ef3245b08dc82f781a906df9abb3b6cb0ce0486f0eeb823ca570",
+                "sha256:d9a024370e917c899bff9171f03ea6079deecdc7482a146a2c565f3b9df134ea",
+                "sha256:e08db25b61cf024648d55973cc3d1c3f1d0818fabf594d5f5a8e2318103d2aa0",
+                "sha256:e97573de0ab3bbbcb4c7748bc41f4ceb6da10b45d35b1a294b5820701e7c25f0",
+                "sha256:eccb417b91d0be27805dfc97550bfb8b7db94e9fe1db5ebedb98f5b88d601323",
+                "sha256:f7c2f4d9681e810cf40239caaca00079930a6d9ee6591139b88d592d36051d82",
+                "sha256:fde5ece4d2436b5a57c8f5f97b49b5de06a856d03959f836c957d3e609f2de7e"
             ],
             "index": "pypi",
-            "version": "==3.1.0"
+            "version": "==3.5.0"
         },
         "pyrsistent": {
             "hashes": [
@@ -2004,11 +2179,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e",
-                "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"
+                "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d",
+                "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"
             ],
             "index": "pypi",
-            "version": "==7.2.2"
+            "version": "==7.3.0"
         },
         "python-dateutil": {
             "hashes": [
@@ -2172,116 +2347,122 @@
         },
         "pyzstd": {
             "hashes": [
-                "sha256:00c9e9aa800b7ded58bdddd90930f949205475c17d1e5923ab936321b805ede0",
-                "sha256:01e723cafb6e681dccba134524f9f012c5f04078d6cb05e03b2f5d871716e0db",
-                "sha256:0820d84c272865e996696fc84c8c63cf6898d66bfda8ac4109880e24272ac13a",
-                "sha256:0d742edd7340e4b8ba4cbe240e873bbc88c197c0b3838212eb5c4885094f191b",
-                "sha256:15b84a04edb257cc7d75c208be43dc4c66e8bd2a44ad9e2af523a829386f0e4c",
-                "sha256:1659da1d388a85c8297083772fdd16543a43305ec0213f1d6f5a6b65702c8402",
-                "sha256:16eb2b2b206489bf8a7b690cf3b06b24352a16ee25717cb38c4381aefb3a94ce",
-                "sha256:1e138c585fc2d07fb20e61f6cbac77fdc77497299199a610a1fca28933128948",
-                "sha256:1e53d12e026605b254569a384191850b6e9f2fcd0fba8b3c80e09b9683fc86af",
-                "sha256:2146d6ab27f3b5dd02b093d748151897e3c929ec009768f20b7a5f3627de7d9c",
-                "sha256:21b7f72717d58b39487c903854556fc363be3329b94aa696e9111291a13d714c",
-                "sha256:229626e1ee592fd8a922935b997e9b91b8b8b874a72565fe1839041cf3ae1a54",
-                "sha256:23c07257fa5df97ebde0a01a54e7c4ae295dc684908f8b1c4ac2f407b881d91b",
-                "sha256:242467cc26e2e4b344028887c934fce3e48354e5eddbc4bb4e695f2c011d3755",
-                "sha256:2589c484c59a0ef2d52d1fd7ab3efd648ef38a396ef22b892c79c2cb878487e3",
-                "sha256:27e1b4a3a6fdab664f1d415c3bbc63700a85f8ec46ad2bf7287e6a86bbea0581",
-                "sha256:2988e5282c807634911d9336ff4797de84e308d94a1f57bfab4223d927e992c5",
-                "sha256:29f692a74f3d9778c71ad6899a1c7578fd44dfb5b6187fe37abdfc7cc034e9d3",
-                "sha256:2e25838c7410245201a455ff000372c65654c781a7a168b2249432dae4ce260f",
-                "sha256:33d42c56fde20a2a98bd200d91d09e44957ce782ffeef5041fc4c544df7640d1",
-                "sha256:39fa99cac5abe0cdb34afda54f54735593646d8c57de1b734d1fe3c9761e0575",
-                "sha256:3b0d384a48592683edf812c1a3d75a0cf5af253b1629596567ce3a2991d6ccbc",
-                "sha256:448d4412462d00bbb7c4857f9a4bedc879f9834f01e2723c23f9de4edde8bde3",
-                "sha256:452e1e774fd7e693f2f5065de61f8cf42eb449ea5614e4cd15ae458e70977d98",
-                "sha256:46e95ec6b499571eca1683c078d2f6cfe3815a5103bf4f488a4edbb1607be652",
-                "sha256:4b41ca76a847286cd54ebabbfd37fa732a68dc2daf5178aede34ee377598cfa5",
-                "sha256:4bf88010b15470f00810e8fc3cb46608af7dceb0d8ca696552d635be652bdfd7",
-                "sha256:4e18f986325eedf0b5d86feb87fad70a0430e3da0bff1a1c33ddce8c70fc371e",
-                "sha256:4f4a96bdffa773f8820e065ec840c5d3f252b0611951b73567121757388f171c",
-                "sha256:55a368a2195bf9cabf8f20ea2841c6a7463c21d8345c8c7d50b809635fec262e",
-                "sha256:55dd0cf038e9053147e58fd4d0509fc1b735f63ce276b88151560bae271a25af",
-                "sha256:56cb56e24e9d33ef571c23b0157593c44c052ffe5810599a76bc6c60ec77f67a",
-                "sha256:5d774cf6406c33c9af6e1fde10a1b0aab7e1f573b2510c491ed24c6f4dbad677",
-                "sha256:5ec07186d5150a6129327e906c0c1f15fd505d4f5538957b2481913a959121ee",
-                "sha256:60079134dd0967fc4c81ea50a7deedd99c52bb263a46750326cba9f53b03f93c",
-                "sha256:649f23ab5c203dade1594f593c6b65247e848f99ba5b11ef26b74541e1f0b605",
-                "sha256:65dc2ac46389823214dd21141a05009cc2fc8d139d8a0aae5f7eb6830ffe5ded",
-                "sha256:67ee34825be593151ba0da78d6914914afce9a7ddc77683319d9406bf422c578",
-                "sha256:6be764774531007ce83c32d24379c32474329655bcc177f1b154e2d2e7e4dba1",
-                "sha256:7048a34197205cb4459492a1d397059ec35a758e29cee226ffa8e593148300af",
-                "sha256:72bf7435dad00fe8a24815f67fcd9fddf44265aa8155a507ac79472cd13c97c1",
-                "sha256:7422e119a9dd10695dee636707881fa6e1af872df5a4f5f1ae6feec34e3cdab6",
-                "sha256:74750bee4fd2b584bf502cdc03861c9decb0875a7febcb2d629cf72a0470188d",
-                "sha256:7740d39f74f9154a7f42d15c1b16f87ba062ac16439d62525f2c85165ea9ebc2",
-                "sha256:78a5e30f90f499f6961e56263e8f029e228b92550aa4e8a5649ed220f8f19230",
-                "sha256:7aea5a1474cacdd0285f16bc5392271645147806986e17be8b100ef750520548",
-                "sha256:7da76175bbbec45ecd35e611eaa50e40c0f29a05e86a8cd50b1b20b5a52cd218",
-                "sha256:821555ce9bc7f50130bf52dc7b7d342913e00435c68a5cac564c5f2e57f33cbf",
-                "sha256:8576440b3212d648fbff49d1127e2bffe0f74aa8be40c3ec8b783f276135cb30",
-                "sha256:88e9f435a68ab9d04a4a90d5b11c587080295ab4057a403730990944b98d0b4e",
-                "sha256:8a7f8be2493356bf55a1261b7a9bb476324738ced6210c3f876f452df8288785",
-                "sha256:912345726aa45dba8fb4930e172149f558caf606bf22c219cde9deb984a1523a",
-                "sha256:91cbd6d8a2fa0b18f9b50076a46e837b90e9f1974578483fcc4b094b43cbd89b",
-                "sha256:929f894d0b1439826f45a248d7569f8c9ceb43ff3d5739e7ac9ff264acdcb070",
-                "sha256:94302b3a36c562a31f791a5e0112bcc129e0245c6f462d86627a4cefda0f5764",
-                "sha256:9860aeb47f88e61f8bd615f4f3e8372990c8e680c52d8a2f893157ebdb15e8a2",
-                "sha256:9aed2fda29875d42a7f96b741582e6e9adb48f3903760439f60b0f8fb8f822f1",
-                "sha256:9b433e0495b0231c20733b4649d33c5ab51f3670db4b47e0964b0b4dd240e03f",
-                "sha256:9c681938be80e4ceedc2f43b5d7c08e8e3719492a8250cc742382633c7176919",
-                "sha256:9db032965b8fabf0bac13b75706031444ebfa7214794afe80be41799d8a349fb",
-                "sha256:a3e11b0534f958668dbcac6d3d943af93d1f5c4dfa7857147589398b405a9ee6",
-                "sha256:a6250aa075408753cde58d79a80923c29c1791f32c4b3e58f25701ae5dbf1678",
-                "sha256:aa9ab4ca7e32c219ad0d420786897cdb3c09f1ffc3fd1a2785e7652ace5684e2",
-                "sha256:ab21cc20922de335feef32245a9329bc3cb443346dcda243a711aab15bc130cc",
-                "sha256:ab55575f1c6c12ca65918dba8196bfb281df86038a9f9e376fb6ac5a501ac724",
-                "sha256:ab88c8123e2c8ad298d1ac79f5b5aeab68ef27a0afe968e5c5be1a2f24577e2e",
-                "sha256:b06eabacbefaf440785ee1f499582c79083822fb3d2640fb74dd5d3391bbc2ae",
-                "sha256:b28e6f095fd56ac373844ee76ab74b011dbd128e4113a033911675421742ce91",
-                "sha256:b4833842d5e6660c1b6cd316f102ffddb95566f12ec1f89c5630d4b511e7e084",
-                "sha256:b5f921cf4ba10cf2f1dbaaba3b9e7afe876a0962cf43f16455c4e5052c1dc0bd",
-                "sha256:b73b18d8ebf5caba863bfccb6ad422b9322c7a806b50a555bb44f51f16b7af5e",
-                "sha256:bca4d7285363b73966e8c6d8af8da515d23bcaf8eb91293e866d5d4615ea5227",
-                "sha256:be4e4bc65de7093e4781d98016b15224311f52eacb220f67b1cbffcae3ac5497",
-                "sha256:be71183e8e7a03b298d4e547f58f68ee1664889bef4e7460464fdf1e70349b66",
-                "sha256:bf63ad569ebc08fee48bc1080c1eb98c9620af4d52ecc56b2a54d6f01797d7be",
-                "sha256:c03e1bfc6c391a05ea04dfff485f4884b07805a16f198c1d5b4e24f4df1e1723",
-                "sha256:c173226b2fab75bfcf9c0315d551e9b484e50f4d84869676bcf33d080f5b43f8",
-                "sha256:c2c6f976e45e63a4f6cbaf20240bb0842ed2ed65703718d54a6fad8642d3c230",
-                "sha256:c2e35d5a169ef9107760bd276974331e0be8fafe2e26ec824020c4999013dc40",
-                "sha256:ca8f520618a3a1c66ee0f6bc3a6b4fc6f9d55b7b1ce80a8f8a422c82a2f79418",
-                "sha256:cb03d3fca1fab2b33b4d0de9b80ab0efc9bb872eb28575eeb73af0d9aac08322",
-                "sha256:d1bcfb77bf00ab5e134321ddf9123845c80b1f49500d499998a17881aa644e28",
-                "sha256:d1da1bcbac16a075675609865e2705d9ec90a2312450d4d99b65a003fdef84f7",
-                "sha256:d62f3e5ca97150ddd32e83275170bcb7ea462f271df03d39de434e8e02889b09",
-                "sha256:d6f2bbd863e674875882de8dd6f54ba98916cfeb89d9224393e9846c9e9aea1d",
-                "sha256:d9069da3bb94cac8af3fd2d7b622c7a78a248d3295a478587d880553287d56e6",
-                "sha256:ddfbaad47edf70932a6e0fba37348ca240a34408e674c056ff16631fa07ea5fd",
-                "sha256:de07ac54f57642f186732075cdce2be3d4a30228c3b17a6d8c6053765dc6eec8",
-                "sha256:df80324759cb74a10818f5ea4dfb0332b6064a31305246f66b043c4f73b37918",
-                "sha256:e111d203153837fca01a3c6ed48167af4b590f148998a2d8edefd98338615006",
-                "sha256:e1e7e43714b0646fb1d61b6b9bbf6e6ede6c77c89667f2addb086f75196ce389",
-                "sha256:e3b4ee242e00029c68fc5d8cddc3e01f6cdf7ac466d844a1440ebde4ae1f9e79",
-                "sha256:e7361f4e5a354715d00c43ad3472188cb9eab83d2e75a2ab7ca75734779579bf",
-                "sha256:e884faae04b2f52cc6aa5aa18de2631ab75786440fd8a708f6b886d75aade937",
-                "sha256:e9074eff37261effb5c7312911e11f0a4c26475ab55bae803d3b095fa291e631",
-                "sha256:e96731ecfff34aeefb5e0f93222c9831880e8a94ea700f4432d4ee45995badc1",
-                "sha256:eb300366e9bf62776ef8c5c5fca57d5b18fdf6a62d055244e9f9f71ea263597e",
-                "sha256:f437e316cd24f21ea13ad635d7e24550458217af75182554963f461bae0bc189",
-                "sha256:f4d47ea4a9a3a43a556da63ee59bf941aecfb529bb85957960e91339c6d6cf3d"
+                "sha256:033e1235438ebb9e02f33f6e9632e1cf74df340a2412704b93409ced861d265a",
+                "sha256:035f2d2c3ef28012e31a719337d5bb2bae20e9fcda0bf291cc933342bbaf97c9",
+                "sha256:061c7a9f653a5eb3bba776d817e14bb563933e118d0e154c83a34058dd07b8a4",
+                "sha256:12cff189a3aafddf8e3825e9676a545d8bf29e8de4aaa3bc903fd2ccdbbc26f3",
+                "sha256:1bf4144fabcd58c4477b1161f619ad4b1e9760bfde89aa35079b39a4737a2d35",
+                "sha256:1dbce07991e3f188bacffe7ec03d3cc2dcb280857de6e80a3141c0b794434c11",
+                "sha256:1f9b60357a3e34ecf7376f2af425bee1ed7160025180902fc0ebbd8e278a17e5",
+                "sha256:220c55a25be08ebf066e5bf9df44e559145a6a2ff45de2b7d7011b0193d281fa",
+                "sha256:229dbdaa7f6c54407b6e974176c70db2cbd23688175e9b46e992af73e1c7012e",
+                "sha256:22c79f0e8d978e53ca894872e115873e872aac7026f4258da6a06838c1d62fa8",
+                "sha256:25bd3025e97d0f645bb2f211aca505915f4a6ea78fbcbf376231bd3163bdb911",
+                "sha256:2caa249172491f535ecc220f37d9f921b83cfb7a61ea8c4a15694c4b10a51b45",
+                "sha256:301bf12e70a69ce66ca5517b48cd61cb010cead67a86e8967f40b28b21710b58",
+                "sha256:32a1b67d5340d8df381e718a788417455edd76bed7e8a4cbd259acdc30b5e17e",
+                "sha256:32d4dc68afdb63a897f994899b897b1df159f60104146a99cecd230b890ef890",
+                "sha256:3379c1c2ee05957221571fe3508cd390319c1acf66e31f9067bc4ac7dd256175",
+                "sha256:35d690fa6113e506984c546d37190d1182b24a9fbc972d5de4dfeefcb768ba36",
+                "sha256:388563dd99db5f79dc370be41702ab13414bc90381555af38e5a6dddab904e39",
+                "sha256:3bffef33e39e396a9ea705cc14ecc23db286d74427bf10984cc630e401fb32b1",
+                "sha256:3d34d263cfa14c8e6e8c711f2707bce534a6c77260015e5f379e753c5ea375df",
+                "sha256:408d98887b40608be9feaa5b0547a1480d5af82ef841668b58142c8b741a7114",
+                "sha256:417001d1587ad87ca03430e4b9e46c3a4fc34a43af4fca62d47eca2a8e3b2318",
+                "sha256:44bc449e3ddef015eb572c8377ebc04712bdd672122ccb5754abc44455d1bda0",
+                "sha256:46cdb38b6e5c2bee4390d5512bbb7890fcd6ea15a54537a3cd3746916715ba17",
+                "sha256:49869a1780d4a78a86cd5580df17de88c503f8159f944c354bff247cda659ad1",
+                "sha256:4a16066313affecc6113f0b0053461809c65a472e7f1d88b74f68f97f8d461e3",
+                "sha256:4ca6193b745fc9e15bf24716a4ca8623387936b9f68f7bb333c0f735890fd888",
+                "sha256:4db7248c3b179cdf79fb9ddbdeed499a137ebd0d0c3004728fb1f17454e77763",
+                "sha256:4e421146153343d80586618202d9bbd4bab47274b54084319a7bc56afc0161f0",
+                "sha256:4ebe360d014e588adc98f22c8f2ab891b8f4611b3030062fa2b6501fdc47b6fc",
+                "sha256:4eefd13c72243d63e4b81c2199b73e937a8860eb8e9d270d6e210003d6bc7ed7",
+                "sha256:5058b50ca268a5baafcadeac4ad9b24993a35b0e4c7636075a74c69c356635c4",
+                "sha256:509413892de1b8008104897c98ff1e8ff258b29de16e39a3e9fc153de06579c6",
+                "sha256:54f0fabfb5db31b4d8e1a7bff3297a82cb44729a7cbd5f353cd67f70cec4dd88",
+                "sha256:576a6d6c75c3afc4a40eea23826b475f63df3a93ade94fd40aa9ad7e9574bb26",
+                "sha256:59ebec36f3b8e57501c6db0310d1059b161035571a0b54437891a52d58e96268",
+                "sha256:61949834ceaa5dd90e9154d429f501d75b3bf832c2eb08613ce708b243b6f26f",
+                "sha256:61a7a36e0c19f8d34148733758e7ae6a5286fb2a04a48b6df44eef55ad09db27",
+                "sha256:65ff4cf6f0ffb2f32f9d5e49f6cde305eaf67c3a9abe398b80fb108a1f7ca7ec",
+                "sha256:69d6a9c83e65a6cc2c6f87f031fd75c92459c0afbf4eb440b4f35569e9c58a7a",
+                "sha256:6d08f6d079f6c669217d8a18e9133e2f9c7bf904fbf5aed6de48bdd12e4819e5",
+                "sha256:6f56459ce65c31ed4817ee13723d08e45a8b4793dc59cc4ff49772d1dcfa19e5",
+                "sha256:74a80e3e21a27b270e3c9ff021f1abefe4f3ab92d2dac06072512fc225b07877",
+                "sha256:78057e5fdfdcc1ecfb97144df9b58a9989d3b67650dc01f0b79dd798afb1c197",
+                "sha256:7a3e172eb95caded48bb1565d93486e13fb0f3f63bc61b38859e6332714d3366",
+                "sha256:7b829cc81d3b6e43f923cd9479ac35a97893c9c2c5ba1be5bd12bd0211ef0e30",
+                "sha256:7bf21dd201e6b46ec01b6d189c083d68c2df71d537b444286ebaa772de52f6ed",
+                "sha256:7c859ca552f41245e6cbcf35e1ab4425ed72d1215569192e70ac4d391640c034",
+                "sha256:7e2beea67072d1aec5b5b1a0372736ddb268a65dedf3ab327c8bb0b1bb482a8e",
+                "sha256:81c82d7fc925c22b6bcf22bbdf28d61b12848d29fac69eed045fc0b3cdfaaf30",
+                "sha256:86e2b4387bd7849c6898ac5788e23e3f452bc157bfdc72cdc19c10e78125cb50",
+                "sha256:8ba3eaf9a449f3f5ca555739b30aa9f7aaf93139f5d4fa332ac604c8c263ade7",
+                "sha256:8e9911ce37bcc1d95b41449901b3bbe6eb993120f2ed13604dc5943e82539dc7",
+                "sha256:934735c1b59bd0dece877bd3aefc8a09b997d26bc1e778911e26fa204578dc93",
+                "sha256:93ee8738431cc0319c8ec770e2415606032eb99dcfe79a48a421b138fa676518",
+                "sha256:9740bb14e9f6b26a66f9de28882424689436a2686674078d3272fdba61ca18f8",
+                "sha256:98cfa7f1cace5256c8aca5fa4b43df64b4e88791980d18fd768f3de13e31a2f1",
+                "sha256:9913023dd0adde5bd480b803efba11e8f6cb179ba8a361b74a4f8fbe9dfe24bf",
+                "sha256:9a20a08f6810f4c4d2e94ec190fb631eadcdfdbcc6e0259fbb604eb717ae4160",
+                "sha256:9b6b39ab621c84f6b2f66bc1c3fe743b3511c7c0067b11bd6c65fa05b0be08d0",
+                "sha256:9ed90a7a6f113ea251daf8701bd7e19a22af1ae1a2695d386ae7a71ede65d51d",
+                "sha256:a195b0aa444acfaf9e2cb8c6b7df0a47d81a64b7390a7f4d677e2f83d62de156",
+                "sha256:a2bddc90a19178e4e9491bf819dce73fe3567cf72815f1abe47e6c297e121520",
+                "sha256:a47c3ba07ec3c6f72e2846066630a2fd2ad96744579f0b182811b34987bed6f2",
+                "sha256:ac9602131b5d0897cfdafe10d850c5fb808b0449d9b6c904c723c68388c7f101",
+                "sha256:b08a93c5c1a42e8a9ab2cd6ffa635ccf7134d87992c9da67ada5488c86bda07a",
+                "sha256:b478b5c5db1f34f44c0cf1a4258fbf11ffc16b30474784a84c7c97ccf35ebe81",
+                "sha256:baa60b607acb3ac73c43976247dcf68ea614d78ecc50a71aa67f1d56ade0cf2a",
+                "sha256:be92561b26d009a4e505e40ebd9104687e41bf847027836be3d1bee7b8bbe12f",
+                "sha256:bf10839517db6c21b5c93d80b000b60896ca07a07a6519a3ec4b48c7a5557f8a",
+                "sha256:c0ec20c3cd345b05f5b5db1a3fa1994593ef7277cfe4dcf7c0581d76e62ecdb0",
+                "sha256:c1de2b822c452dd2e803bf2fa608caa20a8794c1c18eb008012d722e3ce5d52c",
+                "sha256:c3cc11c4f42aca08cb2de5353e70345eb12c00cafcb4bb2adde95e1a06afe524",
+                "sha256:c713006bd866721b4d4c4b741e1cdbe896465334dbbc1462e5645067e59242b0",
+                "sha256:c81d7cf1e872474ac49a3ff9a0a6456440c0dfc5f32fc45286b201b8cf8f3829",
+                "sha256:c8d636e52ce414f582a376e245f81cad45f3b210589a0d63df373c86c6781b2b",
+                "sha256:cd7900f827d11fa649b9fb5e2cf3401536295461c059c75f5496ff999ac3c417",
+                "sha256:cff62a2d68ccd53e6d74471c6d0b44a37ef73844137aba3fb898bc16a68d255c",
+                "sha256:d0ede594db3fc377f623fa6a5031bc6b8dca3026df06838a78f92074dd46dd14",
+                "sha256:d1580a65dfed84925cdc9ac941de7b72eafdc27edc6fad6fd02e2cbf2c403ba1",
+                "sha256:d26932cabaecd0d54ac0e793f464f42dc4f59cfacf1fa7f2b1726f87a4e958be",
+                "sha256:d3793d414c3fe92778b23c860a9495e25c5450f6fb2f29acc0b2aa330f9e2e17",
+                "sha256:d4a557bf018c7e3712261dea0ac86387479fa6a099e5777f58dca3ed65189b0d",
+                "sha256:d611d20513732dfaa2d01d31d4332d1243d6d60cd0b55191b356013cadb5b5cd",
+                "sha256:d71f6949182af74125e3dc2eaa809a7b296d0299116d1d8f9c27dc27aa128dbd",
+                "sha256:d82df1c6c4c180e7d40bb42fd81b8573790e6d352c09e3f5f3ca4e5b89921436",
+                "sha256:d88dfadfa5f57903f3222ad319529d64aaa9e33133c5cd4b0269bb6abdb64bbf",
+                "sha256:d8fdcdc335a8ef022a92ea8f68072d8920305bb87184af2be2621dc38151d6a3",
+                "sha256:dd2f48b1b209d2156bd01d96f98096ef9e9bf33df67da57103afbdccac09800a",
+                "sha256:de00167af7af51de31cc5b2e0c798db96525291b69b29efc4c3234d699c33576",
+                "sha256:de434e906b399b43f552a794879b6108a636b47d816775c094990ea816f0247d",
+                "sha256:e2a6378ca49c7bc94ee6d9b29f7a169125741cb8b653585d8882acf8409d50df",
+                "sha256:e3d65304b9d959ce5ac8406e3a0a837a3615d92e621e63db0a05682c2689f9ba",
+                "sha256:e54688a0b26d4d383b61b3a66677c477055048db01815daa61a868eaca129c51",
+                "sha256:e64d58de7b21f2a121505a2e37c6dabb8b21329e5e63350a77411996c03eba05",
+                "sha256:e91ddc8503d5979f269fbf86702ba1a62c44ab1ce510608bc2a510562a9235d9",
+                "sha256:eac42b4635b9a69d02c07e6472ec418e12765e80e648441b9e1ad5f66db49c8a",
+                "sha256:ed2efc34162c837782c2a36bff81ddf3bd15adaf2561681049aa50a3550af52c",
+                "sha256:eece0a403120b5db50ea8744275a00f9cce497702dfb101eb2cb04582a81a6dc",
+                "sha256:f355fdea18c32450d3b352b9c6e449c3ba21535e4c6889792d75b1483c850adc",
+                "sha256:f646c6b6120b1a4455fa23ba5b8b0c7eb6f5e5dbb262e85c3c511bd2cceba5f3",
+                "sha256:f7496a2fc9e24892416a6f5003776013a9b2620bf235d788699e853194cd96fa",
+                "sha256:f83939ae9d4c560fce3b27b7122cc5dc2ba597b649d36d24d81cb560355e2490",
+                "sha256:fb471fa96772d311057a1eb97165887f127387c71d7f904bcff59df468043779",
+                "sha256:feae3801216957dbfd4f5baf2d433dc014499405722974f5180189083df609cc"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.15.4"
+            "version": "==0.15.6"
         },
         "qtconsole": {
             "hashes": [
-                "sha256:bae8c7e10170cdcdcaf7e6d53ad7d6a7412249b9b8310a0eaa6b6f3b260f32db",
-                "sha256:f67a03f40f722e13261791280f73068dbaf9dafcc335cbba644ccc8f892640e5"
+                "sha256:30975c6a7d7941dd646d00a23e5982db49beaa60c3920bb243727d43da805f12",
+                "sha256:dc935780da276a2ab31a7a08a8cf327a2ea47fa27e21d485073251a7eeb16167"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.1"
+            "version": "==5.4.2"
         },
         "qtpy": {
             "hashes": [
@@ -2293,19 +2474,26 @@
         },
         "rasterio": {
             "hashes": [
-                "sha256:03c8bb7be4bb2d808be81ade0391b222dd870c593aebffd361b168d3b8015615",
-                "sha256:39269f8f22a040301d7e7f335bce806aa4f8246cdc5be39ec51ed11389b4b3eb",
-                "sha256:5663bbe5fff1684d031c86e99f4610bc068ad3d9f48b79993ac177e6013200b4",
-                "sha256:6fc016a95ec207fb1091365dd33e207cc26321316d98b3b110d4338d64cd5567",
-                "sha256:754e6653bd6b7998af9f530dfd2d26da33bfe5c7ea6f4cc6c6d38152952c35cb",
-                "sha256:7ffd1e03706f47587a9a6e580cc8f814a83aa8bebafaaeffe15fa0ab664ff815",
-                "sha256:b9b753748a634140609171b4a8009e85cf859c4815d58f76a44b94a661d35364",
-                "sha256:cf68a574500ec1f7d29c9ea353d0d0536255c4a805a17d80db2cc5fe597c312e",
-                "sha256:ebe75c71f9257c780615caaec8ef81fa4602702cf9290a65c213e1639284acc9",
-                "sha256:ecc314d110a7d6185a58097cf2b02fc918196ed441d764df2afe72b221259c22"
+                "sha256:0883a38bd32e6a3d8d85bac67e3b75a2f04f7de265803585516883223ddbb8d1",
+                "sha256:1321372c653a36928b4e5e11cbe7f851903fb76608b8e48a860168b248d5f8e6",
+                "sha256:23a8d10ba17301029962a5667915381a8b4711ed80b712eb71cf68834cb5f946",
+                "sha256:50785004d7adf66cf96c9c3498cf530ec91292e9349e66e8d1f1183085ee93b1",
+                "sha256:69fdc712e9c79e82d00d783d23034bb16ca8faa18856e83e297bb7e4d7e3e277",
+                "sha256:76b6bd4b566cd733f0ddd05ba88bea3f96705ff74e2e5fab73ead2a26cbc5979",
+                "sha256:83f764c2b30e3d07bea5626392f1ce5481e61d5583256ab66f3a610a2f40dec7",
+                "sha256:8a584fedd92953a0580e8de3f41ce9f33a3205ba79ea58fff8f90ba5d14a0c04",
+                "sha256:92f0f92254fcce57d25d5f60ef2cf649297f8a1e1fa279b32795bde20f11ff41",
+                "sha256:9f3f901097c3f306f1143d6fdc503440596c66a2c39054e25604bdf3f4eaaff3",
+                "sha256:a732f8d314b7d9cb532b1969e968d08bf208886f04309662a5d16884af39bb4a",
+                "sha256:b72fc032ddca55d73de87ef3872530b7384989378a1bc66d77c69cedafe7feaf",
+                "sha256:c8b90eb10e16102d1ab0334a7436185f295de1c07f0d197e206d1c005fc33905",
+                "sha256:cb3288add5d55248f5d48815f9d509819ba8985cd0302d2e8dd743f83c5ec96d",
+                "sha256:d03e2fcd8f3aafb0ea1fa27a021fecc385655630a46c70d6ba693675c6cc3830",
+                "sha256:e73339e8fb9b9091a4a0ffd9f84725b2d1f118cf51c35fb0d03b94e82e1736a3",
+                "sha256:eaaeb2e661d1ffc07a7ae4fd997bb326d3561f641178126102842d608a010cc3"
             ],
             "index": "pypi",
-            "version": "==1.1.5"
+            "version": "==1.3.6"
         },
         "rasterstats": {
             "hashes": [
@@ -2441,16 +2629,16 @@
                 "sha256:e7354fd7527a4b0377ce55f286805b34e8c54b91be865bac273f527e1b839019",
                 "sha256:fae8a7b898c42dffe3f7361c40d5952b6bf32d10c4569098d276b4c547905ee1"
             ],
-            "markers": "python_version < '3.12' and python_version >= '3.8'",
+            "index": "pypi",
             "version": "==1.10.1"
         },
         "seaborn": {
             "hashes": [
-                "sha256:390f8437b14f5ce845062f2865ad51656464c306d09bb97d7764c6cba1dd607c",
-                "sha256:62439a38482decdb263a8339f54ecb9823995ad8716abc830e91ca0753201e70"
+                "sha256:374645f36509d0dcab895cba5b47daf0586f77bfe3b36c97c607db7da5be0139",
+                "sha256:ebf15355a4dba46037dfd65b7350f014ceb1f13c05e814eda2c9f5fd731afc08"
             ],
             "index": "pypi",
-            "version": "==0.11.0"
+            "version": "==0.12.2"
         },
         "send2trash": {
             "hashes": [
@@ -2513,94 +2701,94 @@
         },
         "simplejson": {
             "hashes": [
-                "sha256:01f426ee9e3a2d205aa4c22c3da996b51f2de75c4199ef703258a28b304dea8c",
-                "sha256:03de1ec4ad734f28ca49b0a758b997d752be0d089ed30360157c4e8811999c8f",
-                "sha256:041dd69026284d10f035cefb4a75026d2cfcef31f31e62585eeb2b7776e7e047",
-                "sha256:05a668d4a93816fb8a644e90e7987aa3beeb9d2112ca50a474d41e6acb5bb88a",
-                "sha256:094275b1b8f003afce1167c8a674cd1ee2fd48c566632dac5d149901d5012ff8",
-                "sha256:097e48686e49026836ef384c7c10ca670acc023cb16a976a689c2eb6c1852df4",
-                "sha256:0cc9a47bf8cde85c99db5f4a919bb756e62427ade0f2e875a6ec89ae8492d486",
-                "sha256:0cdb5069870f7d26a34e5adc30672d0a7b26e652720530a023bb3a8d8a42e37f",
-                "sha256:0dcc54e7cfbd9674ec4ca181e26eaa5b038446601faeaa6c83d146ddef2f2652",
-                "sha256:16fbebfc38ad4285c256d2430797fd669b0437d090e985c6d443521d4303b133",
-                "sha256:1844d7782652f859d9648531778582d4842d80cfff8d334eb23bb8da0d22a1b0",
-                "sha256:2362c66d2c633925d90f2f177f05e0570d320d986130d34dff9ad6edbf7be8ac",
-                "sha256:29a86bc9c8a913a4e0ffab85c563a7505cdf4bd13fba05342f8314facc0b7586",
-                "sha256:2a6e5c0e0817fb20dbb880c83caebbd4ef39f1901f6f8e53b73a3c74de4e5172",
-                "sha256:2d1b47f768e1f4c1c8a9457effabed735939401e85c0ddcdf68444c88a9242e6",
-                "sha256:30e381471158290ccb79bd31e7bbda4c8f2cf7e1a5f6b557c1b97d6036ccd05b",
-                "sha256:313dfd911723dc3022fed7050a7b315d5d0681cd56eee08e44e2cbd39fd9ad81",
-                "sha256:32de1672f91a789cc9e1c36c406b2d75457a242d64e9e73a70b9b814ef00095e",
-                "sha256:340b7d085b4a5063aacb8664b1250e4a7426c16e1cc80705c548a229153af147",
-                "sha256:34d95ad8e27754f0d91917600d6ea273e05c82a71021f168c45be48637d9502f",
-                "sha256:3d549efc7e8f9a180c59462b124991b690ff25c235d5cf495c3246c66a7679cd",
-                "sha256:3dbfaa79b1c0efdb768392a19110f1aff793f3e8d43f57e292f46734b8affb45",
-                "sha256:44058bea97429cfa0d6fb1d8eb0736a77022f34a326d5bc64fd6fed8d9304571",
-                "sha256:46b8cc86204b51eddcf157cbaf3c44a20f24393030442af0909eeb961186cb67",
-                "sha256:47509775a5c41ec2a6cd17c9c00fc14965cad8e6670059663872ba5e39332f57",
-                "sha256:4b5df4ee48403885046c6f4fd8adc84c4ac0adec69482f22a17bd4ba52876341",
-                "sha256:544e5607142d66a469ecf78a3154ec0f915834dc3b8cfdb2677a78ca58319ad6",
-                "sha256:56d36f47bc7c7684504f0f18feb161a0b1162546b3622e45aa6155f8285180ac",
-                "sha256:5b217201efc007166e24e9a282007cc208a2d059350a7c5bd0b0303460ad3019",
-                "sha256:5c4f59dd358c3a99efa46d62dc1583be3a1c37171f5240c4cbdc2d5838870902",
-                "sha256:5d45ed9452a42064805143480397b586ea2ea322f4b8b69034c51181e7f38342",
-                "sha256:6197cfebe659ac802a686b5408494115a7062b45cdf37679c4d6a9d4f39649b7",
-                "sha256:65de5876e34780b43f92d9d2539de16ecc56d16f56e56e59b34adfa1cebe064f",
-                "sha256:676e8c182f8079851f12ae1cee2fcebe04def2da2a5703a9d747ab125af47732",
-                "sha256:682b202f56d9d9e1bb22eaca3e37321002223fd5ddef7189b9233e3c14079917",
-                "sha256:695da62e494e4689ab78fae173a78390a175b6a5ccc4292277ce0f8dba3945d5",
-                "sha256:6d65ea4582b47d77e9094c22eb0aeded0ebd96c1df86e988870b40c6514c6e21",
-                "sha256:706a7fc81ceeb321a1040d008b134056012188f95a5c31ad94fb03153b35cc84",
-                "sha256:7339bd6203351555c1e728acd601ba95ebce0f6041ebdb386e025f00af3f1769",
-                "sha256:7701a289d45fdfeb37f1d15cf638801cea439df667a613379443772a86e82936",
-                "sha256:7f27a079cb009ba569983061a50a9270b7e1d35f81e4eeaf0e26f8924027e550",
-                "sha256:827ddc3b3603f7d0421b054388da6face7871d800c4b3bbedeedc8778e4085ea",
-                "sha256:8ccc982197982cdda19e3e5ba4ef7f6ad6bed3eb39bb423bfbf7fa2cd29488ab",
-                "sha256:8f381747c2edebe3c750a571e55103bfcc33b2707a9b91ae033ab9ba718d976a",
-                "sha256:93ba80fbf959b5852554f23201a5f4b30885930c303546ffa883859a435ea3cf",
-                "sha256:9a50a9da1cf93e35f26c4ddee162abf3184a340339ec2d4001c34607b87e71b4",
-                "sha256:a1163bfe5d043c20adeb5c4c8e89dd1dd39b375c8ca6f1c1e35ec537ad7a12e7",
-                "sha256:a2285609b4edbf9957440642493788ebef6583042b3fb96217c2e71f29bc6d80",
-                "sha256:a255d30cda6334ba780eb40a56e8134efd3453948b995d3966e45212e34bf018",
-                "sha256:a3bba99178f1b25878752a8bc6da2f93fbae754ebd4914d2ac4b869b9fb24102",
-                "sha256:a56005332d70b8d02d476d4a85818b27b01e51dac1a21d5c1a1d8a5df2efb4a6",
-                "sha256:a89d7fe994b115f0a792e6673f387af3db812a1760d594abad51e0ea11d3e470",
-                "sha256:a8ac155e3fd3b54a63040df024e57e62c130b15a2fc66eff3c2a946f42beed52",
-                "sha256:ab5941e1fd509fc151258477ef4b663fe14c94f8faf3581827bf4b02080fd4ba",
-                "sha256:ab64f087c5863ac621b42e227e5a43bd9b28de581afe7be12ad96562b9be8203",
-                "sha256:ad37f25fd8dfbed80815c3281b82a165be2a74e663856b9a50099d18789987bc",
-                "sha256:b17026f3f349a6e87818cd3531e3bbb5cc78a6f4b2b6718f574a8e0512d71e08",
-                "sha256:b43d3c2e204d709af955bdb904ae127fe137363ace87fbf7dc8fe6017f7f8449",
-                "sha256:b482d1fdd8f860e743c7de8cd6dfe54fb9fe8cd6ccba29e2966912ac89e17b2f",
-                "sha256:b6c6cfc492710d8f0303705fa1ff7bb3d6a145f523384e45a6f3b13ada37021f",
-                "sha256:b9893852c559998f667e6434d2c2474518d4cdfd1b9cec8e57b3c9d577ba55c1",
-                "sha256:bd694c465cc61fa8e599355e535b6eb561279834d9883aeef08d0e86c44c300c",
-                "sha256:c0444423129df448788edc66a129bc7560ad7d6a661d74f0900959c0b44349a1",
-                "sha256:c37b092d29741096c4723f48924a80b1d3de62ca1de254ce88178fa083dd520c",
-                "sha256:ced906b172bfad62736a27cfafcb6e24bc9938533b0529ff8150f7926fe35b54",
-                "sha256:d0d3b9f7cee233368d92c89746dde74313abafaa3ec1f0c06a3f4f164dc27bcc",
-                "sha256:d5f67bffa6fc68e391b2250e1feb43d534ded64a7b918eb89cf7e3e679759d94",
-                "sha256:d9f0dfde448611f4f818da05f9b544a78f29355dc39151b0dad8e7c65c513e4f",
-                "sha256:da6dc0cb00ef1e1a8daf285074ca8b2bb89591170c42ceab0c37bcdb9adc802c",
-                "sha256:dc74a9ef4d61e18ee6f1886b6ef1fe285b1f432885288afacfb7402f7d469448",
-                "sha256:deb71e6166e4f1264174d78b5b88abd52b14c6649e6eabaf9cf93cb1c7362850",
-                "sha256:e042ae053e05fe193514d51d6b0f0243729961901e9a75f8b596bfaf69522c52",
-                "sha256:e2f87a483c4ab0bb2a9adc9ca09173e7f7cf3696e4fa67bd45a6b33181e57921",
-                "sha256:e2fa1ee5ca34ab2ecfbe3f7a7e952a1ecaebb5b4818f002b5b146324912ac3d5",
-                "sha256:e7d3f7cd57ce0c6a5bb8133f8ed5c3d1be0473a88b7d91a300626298f12d0999",
-                "sha256:edb334cab35dcd90eb563fdacb085f10e5dd0b1acb57fa43f8933308b42a8f88",
-                "sha256:efae49d0148ec68b6e012f1b9e19bd530f4dced378ba919e3e906ae2b829cc31",
-                "sha256:f1085cadec0f7e76377951d7a87744628c90ac6cc634fc97eecce0c4d41ec563",
-                "sha256:f15f56b3119fb71fa57eb4613bcd87eb7df6c2f3547de7d341853d3e50cef97e",
-                "sha256:f1b425a857ce52e651739314e4118fc68bd702ef983148b8fd5cb6f68bb6a020",
-                "sha256:f31e126204ec38f92dee119af87cf881044ef7dea6f7477ef774ed3d84199c24",
-                "sha256:f5e0a03e533313eee9437ccc6c4eab47369f17bc919b57df4a20ccd8bc85d8fd",
-                "sha256:f85d87986ca375b8305b5c4f166783b8db383a6469e8b99b8dba22878388f234",
-                "sha256:fa6fe8fa94a831886ee164ac03514f361e1387a62a1b9da32fde5c0c1f27fa8d",
-                "sha256:fb0f8b35c11fd8e4b924f974d331b20fa54555282451db7f2a3b24bd2d33cc11"
+                "sha256:081ea6305b3b5e84ae7417e7f45956db5ea3872ec497a584ec86c3260cda049e",
+                "sha256:08be5a241fdf67a8e05ac7edbd49b07b638ebe4846b560673e196b2a25c94b92",
+                "sha256:0c16ec6a67a5f66ab004190829eeede01c633936375edcad7cbf06d3241e5865",
+                "sha256:0ccb2c1877bc9b25bc4f4687169caa925ffda605d7569c40e8e95186e9a5e58b",
+                "sha256:17a963e8dd4d81061cc05b627677c1f6a12e81345111fbdc5708c9f088d752c9",
+                "sha256:199a0bcd792811c252d71e3eabb3d4a132b3e85e43ebd93bfd053d5b59a7e78b",
+                "sha256:1cb19eacb77adc5a9720244d8d0b5507421d117c7ed4f2f9461424a1829e0ceb",
+                "sha256:203412745fed916fc04566ecef3f2b6c872b52f1e7fb3a6a84451b800fb508c1",
+                "sha256:2098811cd241429c08b7fc5c9e41fcc3f59f27c2e8d1da2ccdcf6c8e340ab507",
+                "sha256:22b867205cd258050c2625325fdd9a65f917a5aff22a23387e245ecae4098e78",
+                "sha256:23fbb7b46d44ed7cbcda689295862851105c7594ae5875dce2a70eeaa498ff86",
+                "sha256:2541fdb7467ef9bfad1f55b6c52e8ea52b3ce4a0027d37aff094190a955daa9d",
+                "sha256:3231100edee292da78948fa0a77dee4e5a94a0a60bcba9ed7a9dc77f4d4bb11e",
+                "sha256:344a5093b71c1b370968d0fbd14d55c9413cb6f0355fdefeb4a322d602d21776",
+                "sha256:37724c634f93e5caaca04458f267836eb9505d897ab3947b52f33b191bf344f3",
+                "sha256:3844305bc33d52c4975da07f75b480e17af3558c0d13085eaa6cc2f32882ccf7",
+                "sha256:390f4a8ca61d90bcf806c3ad644e05fa5890f5b9a72abdd4ca8430cdc1e386fa",
+                "sha256:3a4480e348000d89cf501b5606415f4d328484bbb431146c2971123d49fd8430",
+                "sha256:3b652579c21af73879d99c8072c31476788c8c26b5565687fd9db154070d852a",
+                "sha256:3e0902c278243d6f7223ba3e6c5738614c971fd9a887fff8feaa8dcf7249c8d4",
+                "sha256:412e58997a30c5deb8cab5858b8e2e5b40ca007079f7010ee74565cc13d19665",
+                "sha256:44cdb4e544134f305b033ad79ae5c6b9a32e7c58b46d9f55a64e2a883fbbba01",
+                "sha256:46133bc7dd45c9953e6ee4852e3de3d5a9a4a03b068bd238935a5c72f0a1ce34",
+                "sha256:46e89f58e4bed107626edce1cf098da3664a336d01fc78fddcfb1f397f553d44",
+                "sha256:4710806eb75e87919b858af0cba4ffedc01b463edc3982ded7b55143f39e41e1",
+                "sha256:476c8033abed7b1fd8db62a7600bf18501ce701c1a71179e4ce04ac92c1c5c3c",
+                "sha256:48600a6e0032bed17c20319d91775f1797d39953ccfd68c27f83c8d7fc3b32cb",
+                "sha256:4d3025e7e9ddb48813aec2974e1a7e68e63eac911dd5e0a9568775de107ac79a",
+                "sha256:547ea86ca408a6735335c881a2e6208851027f5bfd678d8f2c92a0f02c7e7330",
+                "sha256:54fca2b26bcd1c403146fd9461d1da76199442297160721b1d63def2a1b17799",
+                "sha256:5673d27806085d2a413b3be5f85fad6fca4b7ffd31cfe510bbe65eea52fff571",
+                "sha256:58ee5e24d6863b22194020eb62673cf8cc69945fcad6b283919490f6e359f7c5",
+                "sha256:5ca922c61d87b4c38f37aa706520328ffe22d7ac1553ef1cadc73f053a673553",
+                "sha256:5db86bb82034e055257c8e45228ca3dbce85e38d7bfa84fa7b2838e032a3219c",
+                "sha256:6277f60848a7d8319d27d2be767a7546bc965535b28070e310b3a9af90604a4c",
+                "sha256:6424d8229ba62e5dbbc377908cfee9b2edf25abd63b855c21f12ac596cd18e41",
+                "sha256:65dafe413b15e8895ad42e49210b74a955c9ae65564952b0243a18fb35b986cc",
+                "sha256:66389b6b6ee46a94a493a933a26008a1bae0cfadeca176933e7ff6556c0ce998",
+                "sha256:66d780047c31ff316ee305c3f7550f352d87257c756413632303fc59fef19eac",
+                "sha256:69a8b10a4f81548bc1e06ded0c4a6c9042c0be0d947c53c1ed89703f7e613950",
+                "sha256:6a561320485017ddfc21bd2ed5de2d70184f754f1c9b1947c55f8e2b0163a268",
+                "sha256:6aa7ca03f25b23b01629b1c7f78e1cd826a66bfb8809f8977a3635be2ec48f1a",
+                "sha256:6b79642a599740603ca86cf9df54f57a2013c47e1dd4dd2ae4769af0a6816900",
+                "sha256:6e7c70f19405e5f99168077b785fe15fcb5f9b3c0b70b0b5c2757ce294922c8c",
+                "sha256:70128fb92932524c89f373e17221cf9535d7d0c63794955cc3cd5868e19f5d38",
+                "sha256:73d0904c2471f317386d4ae5c665b16b5c50ab4f3ee7fd3d3b7651e564ad74b1",
+                "sha256:74bf802debe68627227ddb665c067eb8c73aa68b2476369237adf55c1161b728",
+                "sha256:79c748aa61fd8098d0472e776743de20fae2686edb80a24f0f6593a77f74fe86",
+                "sha256:79d46e7e33c3a4ef853a1307b2032cfb7220e1a079d0c65488fbd7118f44935a",
+                "sha256:7e78d79b10aa92f40f54178ada2b635c960d24fc6141856b926d82f67e56d169",
+                "sha256:8090e75653ea7db75bc21fa5f7bcf5f7bdf64ea258cbbac45c7065f6324f1b50",
+                "sha256:87b190e6ceec286219bd6b6f13547ca433f977d4600b4e81739e9ac23b5b9ba9",
+                "sha256:889328873c35cb0b2b4c83cbb83ec52efee5a05e75002e2c0c46c4e42790e83c",
+                "sha256:8f8d179393e6f0cf6c7c950576892ea6acbcea0a320838c61968ac7046f59228",
+                "sha256:919bc5aa4d8094cf8f1371ea9119e5d952f741dc4162810ab714aec948a23fe5",
+                "sha256:926957b278de22797bfc2f004b15297013843b595b3cd7ecd9e37ccb5fad0b72",
+                "sha256:93f5ac30607157a0b2579af59a065bcfaa7fadeb4875bf927a8f8b6739c8d910",
+                "sha256:96ade243fb6f3b57e7bd3b71e90c190cd0f93ec5dce6bf38734a73a2e5fa274f",
+                "sha256:9f14ecca970d825df0d29d5c6736ff27999ee7bdf5510e807f7ad8845f7760ce",
+                "sha256:a755f7bfc8adcb94887710dc70cc12a69a454120c6adcc6f251c3f7b46ee6aac",
+                "sha256:a79b439a6a77649bb8e2f2644e6c9cc0adb720fc55bed63546edea86e1d5c6c8",
+                "sha256:aa9d614a612ad02492f704fbac636f666fa89295a5d22b4facf2d665fc3b5ea9",
+                "sha256:ad071cd84a636195f35fa71de2186d717db775f94f985232775794d09f8d9061",
+                "sha256:b0e9a5e66969f7a47dc500e3dba8edc3b45d4eb31efb855c8647700a3493dd8a",
+                "sha256:b438e5eaa474365f4faaeeef1ec3e8d5b4e7030706e3e3d6b5bee6049732e0e6",
+                "sha256:b46aaf0332a8a9c965310058cf3487d705bf672641d2c43a835625b326689cf4",
+                "sha256:c39fa911e4302eb79c804b221ddec775c3da08833c0a9120041dd322789824de",
+                "sha256:ca56a6c8c8236d6fe19abb67ef08d76f3c3f46712c49a3b6a5352b6e43e8855f",
+                "sha256:cb502cde018e93e75dc8fc7bb2d93477ce4f3ac10369f48866c61b5e031db1fd",
+                "sha256:cd4d50a27b065447c9c399f0bf0a993bd0e6308db8bbbfbc3ea03b41c145775a",
+                "sha256:d125e754d26c0298715bdc3f8a03a0658ecbe72330be247f4b328d229d8cf67f",
+                "sha256:d300773b93eed82f6da138fd1d081dc96fbe53d96000a85e41460fe07c8d8b33",
+                "sha256:d396b610e77b0c438846607cd56418bfc194973b9886550a98fd6724e8c6cfec",
+                "sha256:d61482b5d18181e6bb4810b4a6a24c63a490c3a20e9fbd7876639653e2b30a1a",
+                "sha256:d9f2c27f18a0b94107d57294aab3d06d6046ea843ed4a45cae8bd45756749f3a",
+                "sha256:dc2b3f06430cbd4fac0dae5b2974d2bf14f71b415fb6de017f498950da8159b1",
+                "sha256:dc935d8322ba9bc7b84f99f40f111809b0473df167bf5b93b89fb719d2c4892b",
+                "sha256:e333c5b62e93949f5ac27e6758ba53ef6ee4f93e36cc977fe2e3df85c02f6dc4",
+                "sha256:e765b1f47293dedf77946f0427e03ee45def2862edacd8868c6cf9ab97c8afbd",
+                "sha256:ed18728b90758d171f0c66c475c24a443ede815cf3f1a91e907b0db0ebc6e508",
+                "sha256:eff87c68058374e45225089e4538c26329a13499bc0104b52b77f8428eed36b2",
+                "sha256:f05d05d99fce5537d8f7a0af6417a9afa9af3a6c4bb1ba7359c53b6257625fcb",
+                "sha256:f253edf694ce836631b350d758d00a8c4011243d58318fbfbe0dd54a6a839ab4",
+                "sha256:f41915a4e1f059dfad614b187bc06021fefb5fc5255bfe63abf8247d2f7a646a",
+                "sha256:f96def94576f857abf58e031ce881b5a3fc25cbec64b2bc4824824a8a4367af9"
             ],
             "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.18.4"
+            "version": "==3.19.1"
         },
         "six": {
             "hashes": [
@@ -2642,33 +2830,26 @@
         },
         "tables": {
             "hashes": [
-                "sha256:169450bd11959c0e1c43137e768cf8b60b2a4f3b2ebf9a620e21865dc0c2d059",
-                "sha256:361da30289ecdcb39b7648c786d8185f9ab08879c0a58a3fc56dab026e122d8e",
-                "sha256:4628e762a8aacfa038cdae118d2d1df9a9ddd9b4a82d6993f4bcbfa7744a9f8a",
-                "sha256:49a972b8a7c27a8a173aeb05f67acb45fe608b64cd8e9fa667c0962a60b71b49",
-                "sha256:6055dd1d3ec03fd25c60bb93a4be396464f0640fd5845884230dae1deb7e6cc6",
-                "sha256:6e13a3eaf86f9eba582a04b44929ee1585a05dd539d207a10a22374b7e4552ca",
-                "sha256:7acbf0e2fb7132a40f441ebb53b53c97cee05fb88ce743afdd97c681d1d377d7",
-                "sha256:8c96c5d4a9ebe34b72b918b3189954e2d5b6f87cb211d4244b7c001661d8a861",
-                "sha256:8ea87231788cfd5773ffbe33f149f778f9ef4ab681149dec00cb88e1681bd299",
-                "sha256:94d7ccac04277089e3bb466bf5c8f7038dd53bb8f19ea9679b7fea62c5c3ae8f",
-                "sha256:950167d56b45ece117f79d839d5d55f0cb45bfca20290fa9dcd70255282f969e",
-                "sha256:9d06c5fda6657698bae4fbe841204625b501ddf2e2a77131c23f3d3ac072db82",
-                "sha256:acb3f905c63e437023071833744b3e5a83376dc457f413f0840d8d50dd5d402b",
-                "sha256:bfdbcacffec122ce8d1b0dd6ffc3c6051bedd6081e20264fa96165d43fc78f52",
-                "sha256:c0b97a7363941d9518573c217cb5bfe4b2b456748aac1e9420d3979f7d5e82d2",
-                "sha256:d95faa1174653a738ac8183a95f050a29a3f69efac6e71f70cde8d717e31af17",
-                "sha256:da9e1ee83c01ed4d1382c7b186d77b4c0ef80b340a48d11a66346e30342c5929",
-                "sha256:db163df08ded7804d596dee14d88397f6c55cdf4671b3992cb885c0b3890a54d",
-                "sha256:dedb959c00ac9e84562a69e80fa858d7aa06d91f96c6cb8cccbbbaf7a879436b",
-                "sha256:ea4b41ed95953ad588bcd6e557577414e50754011430c27934daf5dbd2d52251",
-                "sha256:eed1e030bb077476d585697e37f2b8e37db4157ff93b485b43f374254cff8698",
-                "sha256:f1327aeef8b6c0fec5aae9f5f5a57b2d8ec98c08495fd09471b749ea46de9eb0",
-                "sha256:f9c88511483c8fd39e7841fc60bc7038c96eeb87fe776092439172e1e6330f49",
-                "sha256:fd63c94960f8208cb13d41033a3114c0242e7737cb578f2454c6a087c5d246ec"
+                "sha256:01e82e40f9845f71de137b4472210909e35c440bbcd0858bdd2871715daef4c7",
+                "sha256:117cf0f73ee2a5cba5c2b04e4aca375779aec66045aa63128e043dc608f2023b",
+                "sha256:239f15fa9881c257b5c0d9fb4cb8832778af1c5c8c1db6f6722466f8f26541e2",
+                "sha256:254a4d5c2009c7ebe4293b02b8d91ea60837bff85a3c0a40cd075b8f12b1e6c3",
+                "sha256:2861cd3ef9eb95eead7530e4de49fd130954871e7e6d2e288012797cb9d7c2e8",
+                "sha256:3375bfafc6cf305d13617a572ab3fffc51fae2fbe0f6efce9407a41f79970b62",
+                "sha256:34f3fa2366ce20b18f1df573a77c1d27306ce1f2a41d9f9eff621b5192ea8788",
+                "sha256:70a3585a268beee6d0e71bfc9abec98da84d168182f350a2ffa1ae5e42798c18",
+                "sha256:72da9404094ef8277bf62fce8873e8dc141cee9a8763ec8e7080b2d0de206094",
+                "sha256:7e9bdbfbe025b6c751976382123c5f5cbd8fab6956aed776b0e8c889669e90d3",
+                "sha256:a5ccb80651c5fad6ac744e2a756b28cfac78eab3b8503f4a2320ee6653b3bee9",
+                "sha256:b9370c2a4dc0051aad6b71de4f1f9b0b8b60d30b662df5c742434f2b5c6a005e",
+                "sha256:c83a74cac3c0629a0e83570d465f88843ef3609ef56a8ef9a49ee85ab3b8f02f",
+                "sha256:da3c96456c473fb977cf6dbca9e889710ac020df1fa5b9ebb7f676e83996337d",
+                "sha256:db185d855afd45a7259ddd0b53e5f2f8993bb134b370002c6c19532f27ce92ac",
+                "sha256:e19686fad4e8f5a91c3dc1eb4b7ea928838e86fefa474c63c5787a125ea79fc7",
+                "sha256:f0821007048f2af8c1a21eb3d832072046c5df366e39587a7c7e4afad14e73fc"
             ],
             "index": "pypi",
-            "version": "==3.6.1"
+            "version": "==3.8.0"
         },
         "terminado": {
             "hashes": [
@@ -2742,6 +2923,14 @@
             "markers": "python_version < '3.10'",
             "version": "==4.5.0"
         },
+        "tzdata": {
+            "hashes": [
+                "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a",
+                "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"
+            ],
+            "markers": "python_version >= '2'",
+            "version": "==2023.3"
+        },
         "uri-template": {
             "hashes": [
                 "sha256:934e4d09d108b70eb8a24410af8615294d09d279ce0e7cbcdaef1bd21f932b06",
@@ -2804,11 +2993,11 @@
         },
         "xarray": {
             "hashes": [
-                "sha256:3a365ce09127fc841ba88baa63f37ca61376ffe389a6c5e66d52f2c88c23a62b",
-                "sha256:da0f0d7719b5ee95143a34804e3afb82cb8abbe5c10f9750a9dbed156ad30f00"
+                "sha256:7bee552751ff1b29dab8b7715726e5ecb56691ac54593cf4881dff41978ce0cd",
+                "sha256:7e530b1deafdd43e5c2b577d0944e6b528fbe88045fd849e49a8d11871ecd522"
             ],
             "index": "pypi",
-            "version": "==0.19.0"
+            "version": "==2023.1.0"
         },
         "y-py": {
             "hashes": [
@@ -3000,11 +3189,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297",
-                "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"
+                "sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094",
+                "sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.14.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.15.0"
         },
         "six": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index dda92467c..7bc7f90ce 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,28 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.9.0 - 2023-04-19 - [PR#889](https://github.com/NOAA-OWP/inundation-mapping/pull/889)
+
+Updates GDAL in base Docker image from 3.1.2 to 3.4.3 and updates all Python packages to latest versions, including Pandas v.2.0.0. Fixes resulting errors caused by deprecation and/or other changes in dependencies. 
+
+NOTE: Although the most current GDAL is version 3.6.3, something in 3.5 causes an issue in TauDEM `aread8` (this has been submitted as https://github.com/dtarb/TauDEM/issues/254)
+
+### Changes
+
+- `Dockerfile`: Upgrade package versions and fix `tzdata`
+- `fim_post_processing.sh`: Fix typo
+- `Pipfile` and `Pipfile.lock`: Update Python versions
+- `src/`
+    - `add_crosswalk.py`, `aggregate_by_huc.py`, `src_adjust_usgs_rating.py`, and `usgs_gage_unit_setup.py`: Change `df1.append(df2)` (deprecated) to `pd.concat([df1, df2])`
+    - `build_stream_traversal.py`: Add `dropna=True` to address change in NaN handling
+    - `getRasterInfoNative.py`: Replace `import gdal` (deprecated) with `from osgeo import gdal`
+    - `stream_branches.py`: Change deprecated indexing to `.iloc[0]` and avoid `groupby.max()` over geometry
+- `tools`
+    - `inundation.py`: Cleans unused `from gdal`
+    - `eval_plots.py`: deprecated dataframe.append fixed and deprecated python query pattern fixed.
+
+<br/><br/>
+
 ## v4.3.8.0 - 2023-04-07 - [PR#881](https://github.com/NOAA-OWP/inundation-mapping/pull/881)
 
 Clips branch 0 to terminal segments of NWM streams using the `to` attribute of NWM streams (where `to=0`).
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index fce7a2b21..32ae37a5d 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -154,7 +154,8 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
 
         Tstart
         echo
-        echo -e $startDiv"Populate PostgreSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"        echo
+        echo -e $startDiv"Populate PostgreSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
+        echo
 
         ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputsDir/wbd/WBD_National.gpkg WBDHU8
 
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index b550d97b4..d75586e32 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -190,7 +190,7 @@ def add_crosswalk(input_catchments_fileName,
                 update_id = output_flows.loc[output_flows.HydroID==short_id]['HydroID'].item()
 
             str_order = output_flows.loc[output_flows.HydroID==short_id]['order_'].item()
-            sml_segs = sml_segs.append({'short_id':short_id, 'update_id':update_id, 'str_order':str_order}, ignore_index=True)
+            sml_segs = pd.concat([sml_segs, pd.DataFrame({'short_id':[short_id], 'update_id':[update_id], 'str_order':[str_order]})], ignore_index=True)
 
     print("Number of short reaches [{} < {} and {} < {}] = {}".format("areasqkm", min_catchment_area, "LengthKm", min_stream_length, len(sml_segs)))
 
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index 261b04280..61628eea3 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -130,7 +130,7 @@ def usgs_elev_table(self, branch_path):
             return
 
         usgs_elev_table = pd.read_csv(usgs_elev_filename, dtype=self.usgs_dtypes)
-        self.agg_usgs_elev_table = self.agg_usgs_elev_table.append(usgs_elev_table)
+        self.agg_usgs_elev_table = pd.concat([self.agg_usgs_elev_table, usgs_elev_table])
 
 
     def aggregate_hydrotables(self, branch_path, branch_id):
@@ -142,7 +142,7 @@ def aggregate_hydrotables(self, branch_path, branch_id):
         hydrotable = pd.read_csv(hydrotable_filename, dtype=self.hydrotable_dtypes)
         hydrotable['branch_id'] = branch_id
         hydrotable[['calb_applied']] = hydrotable[['calb_applied']].fillna(value=False)
-        self.agg_hydrotable = self.agg_hydrotable.append(hydrotable)
+        self.agg_hydrotable = pd.concat([self.agg_hydrotable, hydrotable])
 
     def aggregate_src_full_crosswalk(self, branch_path, branch_id):
 
@@ -152,7 +152,7 @@ def aggregate_src_full_crosswalk(self, branch_path, branch_id):
 
         src_cross = pd.read_csv(src_cross_filename, dtype=self.src_crosswalked_dtypes)
         src_cross['branch_id'] = branch_id
-        self.agg_src_cross = self.agg_src_cross.append(src_cross)
+        self.agg_src_cross = pd.concat([self.agg_src_cross, src_cross])
 
 
     def agg_function(self,usgs_elev_flag,hydro_table_flag,src_cross_flag):
diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py
index 8952a973c..c73d896bf 100644
--- a/src/build_stream_traversal.py
+++ b/src/build_stream_traversal.py
@@ -52,7 +52,7 @@ def execute(self, streams, wbd8, hydro_id):
                 stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "midpoint", "index_right": "HUC8id"})
                 streams = streams.join(stream_wbdjoin).drop(columns=['midpoint'])
 
-                streams['seqID'] = (streams.groupby('HUC8id').cumcount(ascending=True)+1).astype('str').str.zfill(4)
+                streams['seqID'] = (streams.groupby('HUC8id', dropna=False).cumcount(ascending=True)+1).astype('str').str.zfill(4)
                 streams = streams.loc[streams['HUC8id'].notna(),:]
                 if streams.HUC8id.dtype != 'str': streams.HUC8id = streams.HUC8id.astype(str)
                 if streams.seqID.dtype != 'str': streams.seqID = streams.seqID.astype(str)
diff --git a/src/getRasterInfoNative.py b/src/getRasterInfoNative.py
index 11e93e322..3164ce6d0 100755
--- a/src/getRasterInfoNative.py
+++ b/src/getRasterInfoNative.py
@@ -3,7 +3,7 @@
 
 #TODO standardize this script
 
-import gdal
+from osgeo import gdal
 import os.path
 import sys
 from osgeo import osr
diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py
index 1db6f3eec..58ab34f12 100644
--- a/src/src_adjust_usgs_rating.py
+++ b/src/src_adjust_usgs_rating.py
@@ -107,7 +107,7 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
         calc_df.rename(columns={interval+"_0_year":'nwm_recur_flow_cms'}, inplace=True)
         # Subset calc_df for final output
         calc_df = calc_df[['location_id','hydroid','feature_id','levpa_id','huc','hand','discharge_cms','check_variance','nwm_recur_flow_cms','nwm_recur','layer']]
-        final_df = final_df.append(calc_df, ignore_index=True)
+        final_df = pd.concat([final_df, calc_df], ignore_index=True)
         # Log any negative HAND elev values and remove from database
         log_text += ('Warning: Negative HAND stage values -->\n')
         log_text += (calc_df[calc_df['hand']<0].to_string() +'\n')
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 7102981fc..5fc0673e5 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -479,7 +479,7 @@ def find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
 
             for downstream_ID in downstream_IDs:
                 # Stop if lowest reach is not in a lake
-                if int(tmp_self.Lake[tmp_self.From_Node.astype(int)==downstream_ID]) == -9999:
+                if tmp_self.Lake[tmp_self.From_Node.astype(int)==downstream_ID].iloc[0] == -9999:
                     continue
                 else:
                     # Remove reach from tmp_self
@@ -497,7 +497,7 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
             
             for upstream_ID in upstream_IDs:
                 # Stop if uppermost reach is not in a lake
-                if int(tmp_self.Lake[tmp_self.From_Node.astype(int)==upstream_ID]) == -9999:
+                if tmp_self.Lake[tmp_self.From_Node.astype(int)==upstream_ID].iloc[0] == -9999:
                     continue
                 else:
                     if int(tmp_self.To_Node[tmp_self.From_Node.astype(int)==upstream_ID]) in nonlake_reaches:
@@ -799,7 +799,7 @@ def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_exclude
         self['bids_temp'] = self.loc[:,branch_id_attribute].copy()
 
         # ensure the new stream order has the order from it's highest child
-        max_stream_order = self.groupby(branch_id_attribute).max()['order_'].copy()        
+        max_stream_order = self[[branch_id_attribute, 'order_']].groupby(branch_id_attribute).max()['order_'].copy()
 
         self = self.dissolve(by=branch_id_attribute)
         self.rename(columns={'bids_temp' : branch_id_attribute},inplace=True)
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 075ff961d..a6260483a 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -32,7 +32,7 @@ def load_gages(self):
             ahps_sites.rename(columns={'nwm_feature_id':'feature_id',
                                 'usgs_site_code':'location_id'}, inplace=True)
             ahps_sites = ahps_sites[ahps_sites.location_id.isna()] # Filter sites that are already in the USGS dataset
-            self.gages = self.gages.append(ahps_sites[['feature_id', 'nws_lid', 'location_id', 'HUC8', 'name', 'states','geometry']])
+            self.gages = pd.concat([self.gages, ahps_sites[['feature_id', 'nws_lid', 'location_id', 'HUC8', 'name', 'states','geometry']]])
 
         # Create gages attribute
         self.gages.location_id.fillna(usgs_gages.nws_lid, inplace=True)
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 256342015..f64082744 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -341,7 +341,7 @@ def filter_dataframe(dataframe, unique_field):
         #Query filtered dataframe and only include data associated with the common sites for that magnitude
         filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
         #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
-        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)
+        final_filtered_dataframe = pd.concat([final_filtered_dataframe,  filtered_common_sites], ignore_index = True)
 
     return final_filtered_dataframe, all_unique_sites
 ##############################################################################
@@ -465,7 +465,7 @@ def eval_plots(metrics_csv, workspace, versions = [],
     # If versions are supplied then filter out
     if versions:
         #Filter out versions based on supplied version list
-        metrics = csv_df.query('version.str.startswith(tuple(@versions))', engine='python')
+        metrics = csv_df.query("version == @versions")
     else:
         metrics = csv_df
 
diff --git a/tools/inundation.py b/tools/inundation.py
index 66633bde2..471035d60 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -4,7 +4,6 @@
 import pandas as pd
 from numba import njit, typed, types
 from concurrent.futures import ThreadPoolExecutor,as_completed
-from subprocess import run
 from os.path import splitext
 import rasterio
 import fiona
@@ -14,9 +13,7 @@
 from collections import OrderedDict
 import argparse
 from warnings import warn
-from gdal import BuildVRT
 import geopandas as gpd
-import sys
 import xarray as xr
 
 class hydroTableHasOnlyLakes(Exception): 

From 5fcbc0bf20315dec98c0eae68057042bb9b70c3b Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Thu, 11 May 2023 09:26:11 -0500
Subject: [PATCH 241/359] Update README.md and CITATION.cff

---
 CITATION.cff | 13 +++++--------
 README.md    | 23 ++++++++++++-----------
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 6d018da0c..029d8c02a 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,11 +1,8 @@
 cff-version: 1.2.0
 message: "If you use this software, please cite it as below."
 authors:
-- family-names: "Aristizabal"
-  given-names: "Fernando"
-  orcid: "https://orcid.org/0000-0003-2525-4712"
-title: "Generalized Mainstems (GMS): An improvement of the Height Above Nearest Drainage Method (HAND) for flood inundation mapping (FIM) that uses processing units of unit stream order"
-version: alpha
-doi: None
-date-released: pending
-url: "https://github.com/NOAA-OWP/fim_4"
+- family-names: "NOAA Office of Water Prediction"
+title: "Inundation Mapping"
+url: "https://github.com/NOAA-OWP/inundation-mapping"
+version: 4.3.9.0
+date-released: 2023
diff --git a/README.md b/README.md
index 7b86974bf..27312796f 100644
--- a/README.md
+++ b/README.md
@@ -173,7 +173,7 @@ If you are on a machine that has a particularly slow internet connection, you ma
 ----
 ## Citing This Work
 
-Please cite this work in your research and projects according to the CITATION.cff file found in the root of this repository.
+Please cite this work in your research and projects according to the `CITATION.cff` file found in the root of this repository.
 
 ----
 ### Known Issues & Getting Help
@@ -190,13 +190,14 @@ NOAA's National Water Center welcomes anyone to contribute to the Inundation Map
 
 ### Credits and References
 1. [Office of Water Prediction (OWP)](https://water.noaa.gov/)
-2. [National Flood Interoperability Experiment(NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)
-3. Garousi‐Nejad, I., Tarboton, D. G.,Aboutalebi, M., & Torres‐Rua, A.(2019). Terrain analysis enhancements to the Height Above Nearest Drainage flood inundation mapping method. Water Resources Research, 55 , 7983–8009.
-4. [Zheng, X., D.G. Tarboton, D.R. Maidment, Y.Y. Liu, and P. Passalacqua. 2018. “River Channel Geometry and Rating Curve Estimation Using Height above the Nearest Drainage.” Journal of the American Water Resources Association 54 (4): 785–806.](https://doi.org/10.1111/1752-1688.12661)
-5. [Liu, Y. Y., D. R. Maidment, D. G. Tarboton, X. Zheng and S. Wang, (2018), "A CyberGIS Integration and Computation Framework for High-Resolution Continental-Scale Flood Inundation Mapping," JAWRA Journal of the American Water Resources Association, 54(4): 770-784.](https://doi.org/10.1111/1752-1688.12660)
-6. [Barnes, Richard. 2016. RichDEM: Terrain Analysis Software](http://github.com/r-barnes/richdem)
-7. [TauDEM](https://github.com/dtarb/TauDEM)
-8. [Federal Emergency Management Agency (FEMA) Base Level Engineering (BLE)](https://webapps.usgs.gov/infrm/estBFE/)
-9. [Verdin, James; Verdin, Kristine; Mathis, Melissa; Magadzire, Tamuka; Kabuchanga, Eric; Woodbury, Mark; and Gadain, Hussein, 2016, A software tool for rapid flood inundation mapping: U.S. Geological Survey Open-File Report 2016–1038, 26](http://dx.doi.org/10.3133/ofr20161038)
-10. [United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR)](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution)
-11. [Esri Arc Hydro](https://www.esri.com/library/fliers/pdfs/archydro.pdf)
+2. [Fernando Aristizabal, Fernando Salas, Gregory Petrochenkov, Trevor Grout, Brian Avant, Bradford Bates, Ryan Spies, Nick Chadwick, Zachary Wills, Jasmeet Judge. 2023. "Extending Height Above Nearest Drainage to Model Multiple Fluvial Sources in Flood Inundation Mapping Applications for the U.S. National Water Model.' Water Resources Research](https://agupubs.onlinelibrary.wiley.com/doi/full/10.1029/2022WR032039).
+3. [National Flood Interoperability Experiment (NFIE)](https://web.corral.tacc.utexas.edu/nfiedata/)
+4. Garousi‐Nejad, I., Tarboton, D. G.,Aboutalebi, M., & Torres‐Rua, A.(2019). Terrain analysis enhancements to the Height Above Nearest Drainage flood inundation mapping method. Water Resources Research, 55 , 7983–8009.
+5. [Zheng, X., D.G. Tarboton, D.R. Maidment, Y.Y. Liu, and P. Passalacqua. 2018. “River Channel Geometry and Rating Curve Estimation Using Height above the Nearest Drainage.” Journal of the American Water Resources Association 54 (4): 785–806.](https://doi.org/10.1111/1752-1688.12661)
+6. [Liu, Y. Y., D. R. Maidment, D. G. Tarboton, X. Zheng and S. Wang, (2018), "A CyberGIS Integration and Computation Framework for High-Resolution Continental-Scale Flood Inundation Mapping," JAWRA Journal of the American Water Resources Association, 54(4): 770-784.](https://doi.org/10.1111/1752-1688.12660)
+7. [Barnes, Richard. 2016. RichDEM: Terrain Analysis Software](http://github.com/r-barnes/richdem)
+8. [TauDEM](https://github.com/dtarb/TauDEM)
+9. [Federal Emergency Management Agency (FEMA) Base Level Engineering (BLE)](https://webapps.usgs.gov/infrm/estBFE/)
+10. [Verdin, James; Verdin, Kristine; Mathis, Melissa; Magadzire, Tamuka; Kabuchanga, Eric; Woodbury, Mark; and Gadain, Hussein, 2016, A software tool for rapid flood inundation mapping: U.S. Geological Survey Open-File Report 2016–1038, 26](http://dx.doi.org/10.3133/ofr20161038)
+11. [United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR)](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution)
+12. [Esri Arc Hydro](https://www.esri.com/library/fliers/pdfs/archydro.pdf)

From 9f971f887c5aedca2a8f791d8a83590012635c07 Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 12 May 2023 14:16:55 -0600
Subject: [PATCH 242/359] Fix and update unit tests

---
 docs/CHANGELOG.md                          | 16 ++++++++++++
 unit_tests/README.md                       | 30 +++++++++++++---------
 unit_tests/check_unit_errors_test.py       | 17 +++++-------
 unit_tests/clip_vectors_to_wbd_params.json |  2 ++
 unit_tests/clip_vectors_to_wbd_test.py     |  2 ++
 unit_tests/pyproject.toml                  |  3 ++-
 unit_tests/usgs_gage_crosswalk_test.py     |  7 +++--
 7 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7bc7f90ce..9d69cb273 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.9.1 - 2023-05-12 - [PR#893](https://github.com/NOAA-OWP/inundation-mapping/pull/893)
+
+Fix existing unit tests, remove unwanted behavior in `check_unit_errors_test.py`, update `unit_tests/README.md`
+
+### Changes  
+
+- `unit_tests/`
+    - `README.md` : Split up headings for setting up unit tests/running unit tests & re-formatted code block.
+    - `check_unit_errors_test.py`: Fixed unwanted behavior of test leaving behind `sample_n.txt` files in `unit_errors/`
+    - `clip_vectors_to_wbd_params.json`: Update parameters
+    - `clip_vectors_to_wbd_test.py`: Update arguments
+    - `pyproject.toml`: Ignore RuntimeWarning, to suppress pytest failure. 
+    - `usgs_gage_crosswalk_test.py`: Enhance readability of arguments in `gage_crosswalk.run_crosswalk` call
+
+<br/><br/>
+
 ## v4.3.9.0 - 2023-04-19 - [PR#889](https://github.com/NOAA-OWP/inundation-mapping/pull/889)
 
 Updates GDAL in base Docker image from 3.1.2 to 3.4.3 and updates all Python packages to latest versions, including Pandas v.2.0.0. Fixes resulting errors caused by deprecation and/or other changes in dependencies. 
diff --git a/unit_tests/README.md b/unit_tests/README.md
index a5eabdec3..bea445a23 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -1,4 +1,4 @@
-## Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
+# Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
 
 Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
 
@@ -27,7 +27,7 @@ One way is to use the incoming arg parser. Most python files include the code bl
 * Find/replace all single quotes to double quotes then cleanup the left tab formatting.
 
 
-## Running unit tests
+## Setting up unit test data
 
 Start a docker container as you normally would for any development. 
 ```bash 
@@ -40,7 +40,8 @@ Example:
 docker run --rm -it --name mytest \
 	-v /home/abcd/projects/dev/innudation-mapping/:/foss_fim \
 	-v /abcd_share/foss_fim/outputs/:/outputs \
-	-v /abcs_share/foss_fim/:/data \
+	-v /abcd_share/foss_fim/:/data \
+	-v /abcd_share/foss_fim/outputs_temp/:/fim_temp \
 	fim_4:dev_20220208_8eba0ee
 ```
 
@@ -62,6 +63,9 @@ If you need to run inundation tests, fun the following:
 python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_not_remove \
 	-jh 1 -jb 1 -m /data/outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
 ```
+
+## Running unit tests
+
 ### If you'd like to test the whole unit test suite:
 ```
 pytest /foss_fim/unit_tests
@@ -103,15 +107,17 @@ If one test case is choosen, it will scan all of the test files, and scan for th
 
 9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist). 
 ie:
-```bash
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
-	-c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o
-```
-but ours would be:
-```bash 
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
-	-c /foss_fim/config/params_template.env -j 1 -d no_list -o
-```
+	```bash
+	fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
+		-c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o
+	```
+	
+	 but ours would be:
+	
+	```bash 
+	fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
+		-c /foss_fim/config/params_template.env -j 1 -d no_list -o
+	```
 
 ## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
 
diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py
index 9c2cc30b0..a3b75ce9c 100644
--- a/unit_tests/check_unit_errors_test.py
+++ b/unit_tests/check_unit_errors_test.py
@@ -48,17 +48,16 @@ def test_check_unit_errors_success_below_min_errors(self):
         params = self.params["valid_data"].copy()
         
         num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
-        ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"], 
-                                                                        num_dummy_files_reqd)
+
+        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
+
         expected_output = 0
         actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
         
         err_msg = "Number of dummy files IS NOT less than the overall min number of error files."
         assert expected_output == actual_output, err_msg
         
-        if (ue_folder_existed):
-            self.__remove_temp_unit_errors_folder(params["fim_dir"])
-        
+        self.__remove_temp_unit_errors_folder(params["fim_dir"])
             
     def test_check_unit_errors_fail_above_min_errors(self):
 
@@ -91,8 +90,7 @@ def test_check_unit_errors_success_above_percent_errors(self):
         
         num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
             
-        ue_folder_existed = self.__create_temp_unit_errors_folder_files(params["fim_dir"],
-                                                                            num_dummy_files_reqd)
+        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
             
         num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
         expected_output = 0
@@ -101,10 +99,7 @@ def test_check_unit_errors_success_above_percent_errors(self):
         err_msg = "Number of dummy files IS NOT more than the overall min number of error files"
         assert expected_output == actual_output, err_msg
             
-        if (ue_folder_existed):
-            self.__remove_temp_unit_errors_folder(params["fim_dir"]) 
-            
-        # We have to put the unit_errors folders back to the way is was.
+        # We have to put the unit_errors folders back to the way is was.           
         self.__remove_temp_unit_errors_folder(params["fim_dir"])
 
 
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index c9198281b..64461dcb8 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -12,11 +12,13 @@
 		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
 		"subset_nwm_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
 		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
+		"nld_lines_preprocessed": "/data/inputs/nld_vectors/3d_nld_preprocessed_230314.gpkg",
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
 		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
 		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
 		"nwm_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
 		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
+		"subset_nld_lines_preprocessed": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/3d_nld_subset_levees_burned.gpkg",
 		"wbd_buffer_distance": 5000,
 		"levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
 		"subset_levee_protected_areas": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LeveeProtectedAreas_subset.gpkg"
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
index 66857c1a8..3cc35c743 100644
--- a/unit_tests/clip_vectors_to_wbd_test.py
+++ b/unit_tests/clip_vectors_to_wbd_test.py
@@ -52,11 +52,13 @@ def test_subset_vector_layers_success(self):
                                      nwm_catchments = params["nwm_catchments"],
                                      subset_nwm_catchments = params["subset_nwm_catchments"],
                                      nld_lines = params["nld_lines"],
+                                     nld_lines_preprocessed = params["nld_lines_preprocessed"],
                                      landsea = params["landsea"],
                                      nwm_streams = params["nwm_streams"],
                                      subset_landsea = params["subset_landsea"],
                                      nwm_headwaters = params["nwm_headwaters"],
                                      subset_nld_lines = params["subset_nld_lines"],
+                                     subset_nld_lines_preprocessed = params["subset_nld_lines_preprocessed"],
                                      wbd_buffer_distance = params["wbd_buffer_distance"],
                                      levee_protected_areas = params["levee_protected_areas"],
                                      subset_levee_protected_areas = params["subset_levee_protected_areas"])
diff --git a/unit_tests/pyproject.toml b/unit_tests/pyproject.toml
index 142fbd053..655af66aa 100644
--- a/unit_tests/pyproject.toml
+++ b/unit_tests/pyproject.toml
@@ -3,5 +3,6 @@ filterwarnings = [
     "error",
     "ignore::UserWarning",
     "ignore::DeprecationWarning",
-    "ignore::PendingDeprecationWarning"
+    "ignore::PendingDeprecationWarning",
+    "ignore::RuntimeWarning"
 ]
diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py
index 4b9bedc1a..fccc0117c 100644
--- a/unit_tests/usgs_gage_crosswalk_test.py
+++ b/unit_tests/usgs_gage_crosswalk_test.py
@@ -46,8 +46,11 @@ def test_GageCrosswalk_success(self):
         gage_crosswalk = GageCrosswalk(params["usgs_gages_filename"], params["branch_id"])
 
         # Run crosswalk
-        gage_crosswalk.run_crosswalk(params["input_catchment_filename"], params["input_flows_filename"], 
-                            params["dem_filename"], params["dem_adj_filename"], params["output_table_filename"])
+        gage_crosswalk.run_crosswalk(params["input_catchment_filename"], \
+                                     params["input_flows_filename"], \
+                                     params["dem_filename"], \
+                                     params["dem_adj_filename"], \
+                                     params["output_table_filename"])
 
         # Make sure that the usgs_elev_table.csv was written
         msg = f'{params["output_table_filename"]} does not exist'

From caeaf23dbcd7c117963f1d715ec9367ad5731582 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 12 May 2023 15:26:14 -0500
Subject: [PATCH 243/359] Fix to Stage-Based CatFIM

---
 docs/CHANGELOG.md                 |  9 +++++++++
 tools/generate_categorical_fim.py | 15 +++++++++++----
 tools/tools_shared_functions.py   | 10 +++++-----
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9d69cb273..c918a5935 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.9.2 - 2023-05-12 - [PR#902](https://github.com/NOAA-OWP/inundation-mapping/pull/902)
+
+This merge fixes several sites in Stage-Based CatFIM sites that showed overinundation. The cause was found to be the result of Stage-Based CatFIM code pulling the wrong value from the `usgs_elev_table.csv`. Priority is intended to go to the `dem_adj_elevation` value that is not from branch 0, however there was a flaw in the prioritization logic. Also includes a change to `requests` usage that is in response to an apparent IT SSL change. This latter change was necessary in order to run CatFIM. Also added a check to make sure the `dem_adj_thalweg` is not too far off the official elevation, and continues if it is.
+
+### Changes  
+- `/tools/generate_categorical_fim.py`: fixed pandas bug where the non-branch zero `dem_adj_elevation` value was not being properly indexed. Also added a check to make sure the `dem_adj_thalweg` is not too far off the official elevation, and continues if it is.
+- ` /tools/tools_shared_functions.py`: added `verify=False` to `requests` library calls because connections to WRDS was being refused (likely because of new IT protocols).
+
+<br/><br/>
 
 ## v4.3.9.1 - 2023-05-12 - [PR#893](https://github.com/NOAA-OWP/inundation-mapping/pull/893)
 
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index 53f0d5968..31f4c32ed 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -327,7 +327,7 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
             matching_rows = acceptable_usgs_elev_df.loc[acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation']
 
             if len(matching_rows) == 2:  # It means there are two level paths, use the one that is not 0
-                lid_usgs_elev = acceptable_usgs_elev_df.loc[(acceptable_usgs_elev_df['nws_lid'] == lid.upper()) & ('levpa_id' != 0), 'dem_adj_elevation'].values[0]
+                lid_usgs_elev = acceptable_usgs_elev_df.loc[(acceptable_usgs_elev_df['nws_lid'] == lid.upper()) & (acceptable_usgs_elev_df['levpa_id'] != 0), 'dem_adj_elevation'].values[0]
             else:
                 lid_usgs_elev = acceptable_usgs_elev_df.loc[acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'].values[0]
         except IndexError:  # Occurs when LID is missing from table
@@ -454,7 +454,14 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
             all_messages.append([f'{lid}:no stage values available'])
             continue
         interval_list = np.arange(min(stage_list), max(stage_list) + past_major_interval_cap, 1.0)  # Go an extra 10 ft beyond the max stage, arbitrary
-        # For each flood category
+        
+        # Check for large discrepancies between the elevation values from WRDS and HAND. Otherwise this causes bad mapping.
+        elevation_diff = lid_usgs_elev - (lid_altitude*0.3048)
+        if abs(elevation_diff) > 10:
+            all_messages.append([f'{lid}:large discrepancy in elevation estimates from gage and HAND'])
+            continue
+        
+            # For each flood category
         for category in flood_categories:
             
             # Pull stage value and confirm it's valid, then process
@@ -477,6 +484,7 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
             if huc in missing_huc_files:
                 all_messages.append([f'{lid}:missing some HUC data'])
                 
+        
         # Now that the "official" category maps are made, produce the incremental maps.
         with ProcessPoolExecutor(max_workers=number_of_interval_jobs) as executor:
             try:
@@ -523,7 +531,7 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
                                         'dadj_w_m': stage_based_att_dict[lid][threshold]['datum_adj_wse_m'],
                                         'lid_alt_ft': stage_based_att_dict[lid][threshold]['lid_alt_ft'],
                                         'lid_alt_m': stage_based_att_dict[lid][threshold]['lid_alt_m']})
-                csv_df = csv_df.append(line_df)
+                csv_df = pd.concat([csv_df, line_df])
                 
             except Exception as e:
                 print(e)
@@ -752,7 +760,6 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
 
 
 if __name__ == '__main__':
-    
     # Parse arguments
     parser = argparse.ArgumentParser(description = 'Run Categorical FIM')
     parser.add_argument('-f', '--fim_run_dir', help='Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_0_9_2',
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index d0f50caca..307dac081 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -658,7 +658,7 @@ def get_metadata(metadata_url, select_by, selector, must_include = None, upstrea
     params['upstream_trace_distance'] = upstream_trace_distance
     params['downstream_trace_distance'] = downstream_trace_distance
     #Request data from url
-    response = requests.get(url, params = params)
+    response = requests.get(url, params = params, verify=False)
 #    print(response)
 #    print(url)
     if response.ok:
@@ -767,7 +767,7 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
             #Reproject to huc 8 crs
             site_gdf = site_gdf.to_crs(huc8.crs)
             #Append site geodataframe to metadata geodataframe
-            metadata_gdf = metadata_gdf.append(site_gdf, ignore_index = True)
+            metadata_gdf = pd.concat([metadata_gdf, site_gdf], ignore_index = True)
     
     #Trim metadata to only have certain fields.
     if not retain_attributes:
@@ -927,7 +927,7 @@ def get_thresholds(threshold_url, select_by, selector, threshold = 'all'):
     params = {}
     params['threshold'] = threshold
     url = f'{threshold_url}/{select_by}/{selector}'
-    response = requests.get(url, params = params)
+    response = requests.get(url, params = params, verify=False)
     if response.ok:
         thresholds_json = response.json()
         #Get metadata
@@ -1141,7 +1141,7 @@ def ngvd_to_navd_ft(datum_info, region = 'contiguous'):
     params['tar_vertical_unit'] = 'm' #Target vertical height
     
     #Call the API
-    response = requests.get(datum_url, params = params)
+    response = requests.get(datum_url, params = params, verify=False)
 
     #If successful get the navd adjustment
     if response:
@@ -1184,7 +1184,7 @@ def get_rating_curve(rating_curve_url, location_ids):
     url = f'{rating_curve_url}/{joined_location_ids}'
     
     #Call the API 
-    response = requests.get(url)
+    response = requests.get(url, verify=False)
 
     #If successful
     if response.ok:

From c35ce7ebdbeca63a09800ac3d82acb52b239ef41 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 12 May 2023 15:31:33 -0500
Subject: [PATCH 244/359] Fix speed for aggregate_by_huc

---
 docs/CHANGELOG.md            |  16 +++
 fim_pipeline.sh              |   1 +
 fim_post_processing.sh       |  25 ++--
 src/aggregate_by_huc.py      | 265 ++++++++++++++++++++++++++---------
 src/identify_src_bankfull.py |   2 +-
 5 files changed, 233 insertions(+), 76 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c918a5935..914d24cff 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.3.10.0 - 2023-05-12 - [PR#888](https://github.com/NOAA-OWP/inundation-mapping/pull/888)
+
+`aggregate_by_huc.py` was taking a long time to process. Most HUCs can aggregate their branches into one merged hydrotable.csv in just 22 seconds, but a good handful took over 2 mins and a few took over 7 mins. When multiplied by 2,138 HUCs it was super slow. Multi-proc has not been added and it now takes appx 40 mins at 80 cores. 
+
+An error logging system was also added to track errors that may have occurred during processing. 
+
+### Changes  
+- `fim_pipeline.sh` - added a duration counter at the end of processing HUCs
+- `fim_post_processing.sh` - added a job limit (number of procs), did a little cleanup, and added a warning note about usage of job limits in this script, 
+- `src`
+    - `aggregate_by_huc.py`: Added multi proc, made it useable for non external script calls, added a logging system for errors only.
+    - `indentify_src_bankful.py`: typo fix.
+
+<br/><br/>
+
 ## v4.3.9.2 - 2023-05-12 - [PR#902](https://github.com/NOAA-OWP/inundation-mapping/pull/902)
 
 This merge fixes several sites in Stage-Based CatFIM sites that showed overinundation. The cause was found to be the result of Stage-Based CatFIM code pulling the wrong value from the `usgs_elev_table.csv`. Priority is intended to go to the `dem_adj_elevation` value that is not from branch 0, however there was a flaw in the prioritization logic. Also includes a change to `requests` usage that is in response to an apparent IT SSL change. This latter change was necessary in order to run CatFIM. Also added a check to make sure the `dem_adj_thalweg` is not too far off the official elevation, and continues if it is.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index ca935cde7..df68a06c0 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -58,6 +58,7 @@ fi
 echo
 echo "---- Unit (HUC) processing is complete"
 date -u
+Calc_Duration $pipeline_start_time
 
 ## POST PROCESSING
 
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 32ae37a5d..632055553 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -55,8 +55,20 @@ if [ ! -d "$outputDestDir" ]; then
     exit 1
 fi
 
+
+#############################################
+#
+# PLEASE DO NOT USE the job limits coming in from the runtime_args.env
+# Most of the time, post processing will not be run on the same servers
+# that is running fim_process_unit_wb.sh and the processing power
+# used to run fim_post_processing.sh will be different (hence.. different job limit)
+#
+#############################################
+
 if [ "$jobLimit" = "" ]; then jobLimit=1; fi
 
+
+
 # Clean out the other post processing files before starting
 rm -rdf $outputDestDir/logs/src_optimization
 rm -f $outputDestDir/logs/log_bankfull_indentify.log
@@ -84,16 +96,13 @@ post_proc_start_time=`date +%s`
 echo -e $startDiv"Start branch aggregation"
 python3 $srcDir/aggregate_branch_lists.py -d $outputDestDir -f "branch_ids.csv" -o $fim_inputs
 
-## GET NON ZERO EXIT CODES FOR UNITS ##
-## No longer applicable
-
 ## GET NON ZERO EXIT CODES FOR BRANCHES ##
 echo -e $startDiv"Start non-zero exit code checking"
 find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputDestDir/branch_errors/non_zero_exit_codes.log" &
 
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
-echo "Processing usgs gage aggregation"
-python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev
+echo "Processing usgs gage aggregation"   
+python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev -j $jobLimit
 
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
@@ -180,12 +189,10 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [
 fi
 
 ## AGGREGATE BRANCH TABLES ##
-# TODO: How do we skip aggregation if there is a branch error
-# maybe against the non_zero logs above
 echo 
 echo -e $startDiv"Aggregating branch hydrotables"
 Tstart
-python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -htable
+python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -htable -j $jobLimit
 Tcount
 date -u
 
@@ -199,7 +206,7 @@ date -u
 
 echo
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- End of fim_post_processing, fim_post_processing complete"
+echo "---- End of fim_post_processing"
 echo "---- Ended: `date -u`"
 Calc_Duration $post_proc_start_time
 echo
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index 61628eea3..40cc37e32 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -1,17 +1,21 @@
 #!/usr/bin/env python3
 
-import os
+import os, glob, re, traceback
+from datetime import datetime
 from os.path import join
-import pandas as pd
-import re
+
 import argparse
+import pandas as pd
+
+from concurrent.futures import ProcessPoolExecutor
+from utils.shared_functions import progress_bar_handler
 
 class HucDirectory(object):
 
-    def __init__(self, path, limit_branches=[]):
+    def __init__(self, fim_directory, huc_id, limit_branches=[]):
 
-        self.dir = path
-        self.name = os.path.basename(path)
+        self.fim_directory = fim_directory
+        self.huc_dir_path = join(fim_directory, huc_id)
         self.limit_branches = limit_branches
 
         self.usgs_dtypes = {'location_id':str,
@@ -117,11 +121,11 @@ def iter_branches(self):
 
         if self.limit_branches:
             for branch in self.limit_branches:
-                yield (branch, join(self.dir, 'branches', branch))
+                yield (branch, join(self.huc_dir_path, 'branches', branch))
 
         else:
-            for branch in os.listdir(join(self.dir, 'branches')):
-                yield (branch, join(self.dir, 'branches', branch))
+            for branch in os.listdir(join(self.huc_dir_path, 'branches')):
+                yield (branch, join(self.huc_dir_path, 'branches', branch))
 
     def usgs_elev_table(self, branch_path):
 
@@ -144,6 +148,7 @@ def aggregate_hydrotables(self, branch_path, branch_id):
         hydrotable[['calb_applied']] = hydrotable[['calb_applied']].fillna(value=False)
         self.agg_hydrotable = pd.concat([self.agg_hydrotable, hydrotable])
 
+
     def aggregate_src_full_crosswalk(self, branch_path, branch_id):
 
         src_cross_filename = join(branch_path, f'src_full_crosswalked_{branch_id}.csv')
@@ -154,74 +159,202 @@ def aggregate_src_full_crosswalk(self, branch_path, branch_id):
         src_cross['branch_id'] = branch_id
         self.agg_src_cross = pd.concat([self.agg_src_cross, src_cross])
 
-
-    def agg_function(self,usgs_elev_flag,hydro_table_flag,src_cross_flag):
-
-        for branch_id, branch_path in self.iter_branches():
+    def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, huc_id):
+        
+        try:
+            #try catch and its own log file output in error only.
+            for branch_id, branch_path in self.iter_branches():
+                if usgs_elev_flag:
+                    self.usgs_elev_table(branch_path)
+
+                ## Other aggregate funtions can go here
+                if hydro_table_flag:
+                    self.aggregate_hydrotables(branch_path, branch_id)
+                if src_cross_flag:
+                    self.aggregate_src_full_crosswalk(branch_path, branch_id)
+
+            ## After all of the branches are visited, the code below will write the aggregates
             if usgs_elev_flag:
-                self.usgs_elev_table(branch_path)
-
-            ## Other aggregate funtions can go here
-            if hydro_table_flag:
-                self.aggregate_hydrotables(branch_path, branch_id)
-            if src_cross_flag:
-                self.aggregate_src_full_crosswalk(branch_path, branch_id)
-
-        ## After all of the branches are visited, the code below will write the aggregates
-        if usgs_elev_flag:
-            usgs_elev_table_file = join(self.dir, 'usgs_elev_table.csv')
-            if os.path.isfile(usgs_elev_table_file):
-                os.remove(usgs_elev_table_file)
-
-            if not self.agg_usgs_elev_table.empty:
-                self.agg_usgs_elev_table.to_csv(usgs_elev_table_file, index=False)
-
-        if hydro_table_flag:
-            hydrotable_file = join(self.dir, 'hydrotable.csv')
-            if os.path.isfile(hydrotable_file):
-                os.remove(hydrotable_file)
+                usgs_elev_table_file = join(self.huc_dir_path, 'usgs_elev_table.csv')
+                if os.path.isfile(usgs_elev_table_file):
+                    os.remove(usgs_elev_table_file)
 
-            if not self.agg_hydrotable.empty:
-                self.agg_hydrotable.to_csv(hydrotable_file, index=False)
+                if not self.agg_usgs_elev_table.empty:
+                    self.agg_usgs_elev_table.to_csv(usgs_elev_table_file, index=False)
 
-        if src_cross_flag:
-            src_crosswalk_file = join(self.dir, 'src_full_crosswalked.csv')
-            if os.path.isfile(src_crosswalk_file):
-                os.remove(src_crosswalk_file)
+            if hydro_table_flag:
+                hydrotable_file = join(self.huc_dir_path, 'hydrotable.csv')
+                if os.path.isfile(hydrotable_file):
+                    os.remove(hydrotable_file)
 
-            if not self.agg_src_cross.empty:
-                self.agg_src_cross.to_csv(src_crosswalk_file, index=False)
+                if not self.agg_hydrotable.empty:
+                    self.agg_hydrotable.to_csv(hydrotable_file, index=False)
 
+            if src_cross_flag:
+                src_crosswalk_file = join(self.huc_dir_path, 'src_full_crosswalked.csv')
+                if os.path.isfile(src_crosswalk_file):
+                    os.remove(src_crosswalk_file)
+
+                if not self.agg_src_cross.empty:
+                    self.agg_src_cross.to_csv(src_crosswalk_file, index=False)
+
+            #print(f"agg_by_huc for huc id {huc_id} is done")
+            
+        except Exception:
+            
+            errMsg = "--------------------------------------" \
+                     f"\n huc_id {huc_id} has an error - outside multi proc\n"
+            errMsg = errMsg + traceback.format_exc()
+            print(errMsg, flush=True)
+            log_error(self.fim_directory, usgs_elev_flag,
+                        hydro_table_flag, src_cross_flag, huc_id, errMsg)
+           
+
+
+# ==============================
+# This is done independantly in each worker and does not attempt to write to a shared file
+# as those can collide with multi proc
+def log_error(fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, huc_id, errMsg):
+    
+    file_name = f"agg_by_huc_{huc_id}"
+    if (usgs_elev_flag): file_name+= "_elev"
+    if (hydro_table_flag): file_name+= "_hydro"
+    if (src_cross_flag): file_name+= "_src_cross"
+    file_name+= "_error.log"
+    
+    log_path = os.path.join(fim_directory, "logs", "agg_by_huc_errors")
+    file_path =  os.path.join(log_path, file_name)
+    
+    f = open(file_path, "a")
+    f.write(errMsg)
+    f.close()
+
+
+def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
+                     hydro_table_flag, src_cross_flag, num_job_workers):
+    
+    assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
 
+    # -------------------
+    # Validation
+    total_cpus_available = os.cpu_count() - 2
+    if num_job_workers > total_cpus_available:
+        raise ValueError('The number of jobs {number_of_jobs}'\
+                          'exceeds your machine\'s available CPU count minus two. '\
+                          'Please lower the number of jobs '\
+                          'values accordingly.'.format(num_job_workers)
+                        )
+   
+    # create log folder, might end up empty but at least create the folder
+    # Yes.. this is duplicate in the log function
+    log_folder = os.path.join(fim_directory, "logs", "agg_by_huc_errors")
+    if (os.path.exists(log_folder) == False):
+        os.mkdir(log_folder)
+    else:
+        # empty only ones with this type (we want to keep others that 
+        # might have been called with different types. aka.. once for -elev
+        # and another for -hydro)
+        agg_type = ""
+        if (usgs_elev_flag): agg_type+= "_elev"
+        if (hydro_table_flag): agg_type+= "_hydro"
+        if (src_cross_flag): agg_type+= "_src_cross"
+        filelist = glob.glob(os.path.join(log_folder, f"*{agg_type}*"))
+        for f in filelist:
+            os.remove(f)
+
+    start_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"started: {dt_string}")
+   
+    # Set up multiprocessor
+    with ProcessPoolExecutor(max_workers = num_job_workers) as executor:
+
+        # Loop through all applicable HUCs, build the agg_function arguments, and submit them to the process pool
+        executor_dict = {}
+    
+        try:    
+            if fim_inputs:
+                fim_inputs_csv = pd.read_csv(fim_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
+                huc_list = fim_inputs_csv.huc.unique()
+    
+                # with multi proc, it won't be 100% in order as different hucs
+                # process faster, but it does help a little
+                huc_list_sorted = sorted(huc_list) 
+                for huc_id in huc_list_sorted:
+
+                    branches = fim_inputs_csv.loc[fim_inputs_csv.huc == huc_id, 'levpa_id'].tolist()
+                    huc_dir = HucDirectory(fim_directory, huc_id, limit_branches=branches)
+
+                    args_agg = { 'usgs_elev_flag': usgs_elev_flag,
+                                 'hydro_table_flag': hydro_table_flag,
+                                 'src_cross_flag': src_cross_flag,
+                                 'huc_id': huc_id}       
+
+                    future = executor.submit(huc_dir.agg_function, **args_agg)
+                    executor_dict[future] = huc_id
+
+            else:
+                huc_list = [d for d in os.listdir(fim_directory) if re.match('\d{8}', d)]
+                
+                # with multi proc, it won't be 100% in order as different hucs
+                # process faster, but it does help a little
+                huc_list_sorted = sorted(huc_list)
+                for huc_id in huc_list_sorted:
+
+                    if (huc_id.isnumeric() == False):
+                        continue;
+                    
+                    huc_dir = HucDirectory(fim_directory, huc_id)
+                    
+                    args_agg = { 'usgs_elev_flag': usgs_elev_flag,
+                                'hydro_table_flag': hydro_table_flag,
+                                'src_cross_flag': src_cross_flag,
+                                'huc_id': huc_id}       
+                    future = executor.submit(huc_dir.agg_function, **args_agg)
+                    executor_dict[future] = huc_id
+
+        except Exception:
+            errMsg = "--------------------------------------" \
+                     f"\n huc_id {huc_id} has an error - outside multi proc\n"
+            errMsg = errMsg + traceback.format_exc()
+            print(errMsg, flush=True)
+            log_error(fim_directory, usgs_elev_flag,
+                        hydro_table_flag, src_cross_flag, huc_id, errMsg)
+            #sys.exit(1)
+    
+        # Send the executor to the progress bar and wait for all MS tasks to finish
+        progress_bar_handler(executor_dict, f"Running aggregate_by_huc with {num_job_workers} workers")      
+
+
+    end_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print (f"ended: {dt_string}")
+
+    # Calculate duration
+    time_duration = end_time - start_time
+    print(f"Duration: {str(time_duration).split('.')[0]}")
+    print()
+
+
+    
 if __name__ == '__main__':
 
+    # Note: This processing is done here for all hucs and branches at once instead of 
+    # at each individual huc processing due to some hydrotable's being adjusted 
+    # in post processing.
+
     parser = argparse.ArgumentParser(description='Aggregates usgs_elev_table.csv at the HUC level')
     parser.add_argument('-fim','--fim_directory', help='Input FIM Directory', required=True)
     parser.add_argument('-i','--fim_inputs', help='Input fim_inputs CSV file', required=False)
-    parser.add_argument('-elev','--usgs_elev', help='Perform aggregate on branch usgs elev tables', required=False, default=False, action='store_true')
-    parser.add_argument('-htable','--hydro_table', help='Perform aggregate on branch hydrotables', required=False, default=False, action='store_true')
-    parser.add_argument('-src','--src_cross', help='Perform aggregate on branch src crosswalk files', required=False, default=False, action='store_true')
+    parser.add_argument('-elev','--usgs_elev_flag', help='Perform aggregate on branch usgs elev tables',
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-htable','--hydro_table_flag', help='Perform aggregate on branch hydrotables',
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-src','--src_cross_flag', help='Perform aggregate on branch src crosswalk files',
+                        required=False, default=False, action='store_true')
+    parser.add_argument('-j','--num_job_workers', help='Number of processes to use',
+                        required=False, default=1, type=int)
 
     args = vars(parser.parse_args())
 
-    fim_directory = args['fim_directory']
-    fim_inputs = args['fim_inputs']
-    usgs_elev_flag = args['usgs_elev']
-    hydro_table_flag = args['hydro_table']
-    src_cross_flag = args['src_cross']
-    assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
-
-    if fim_inputs:
-        fim_inputs = pd.read_csv(fim_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
-
-        for huc in fim_inputs.huc.unique():
-
-            branches = fim_inputs.loc[fim_inputs.huc == huc, 'levpa_id'].tolist()
-            huc = HucDirectory(join(fim_directory, huc), limit_branches=branches)
-            huc.agg_function(usgs_elev_flag,hydro_table_flag,src_cross_flag)
-
-    else:
-        for huc_dir in [d for d in os.listdir(fim_directory) if re.match('\d{8}', d)]:
-
-            huc = HucDirectory(join(fim_directory, huc_dir))
-            huc.agg_function(usgs_elev_flag,hydro_table_flag,src_cross_flag)
\ No newline at end of file
+    aggregate_by_huc(**args)
\ No newline at end of file
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index 585e6ae74..747506aee 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -176,7 +176,7 @@ def multi_process(src_bankfull_lookup, procs_list, log_file, number_of_jobs, ver
         number_of_jobs = available_cores - 2
         print("Provided job number exceeds the number of available cores. " + str(number_of_jobs) + " max jobs will be used instead.")
 
-    print(f"Identifying bankfull stage for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    print(f"Identifying bankfull stage for {len(procs_list)} branches using {number_of_jobs} jobs")
     with Pool(processes=number_of_jobs) as pool:
         #progress_bar = tqdm(total=len(procs_list[0]))
         if verbose:

From a505c328949c34a9031cb3838f60fd6df035150f Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 12 May 2023 16:54:52 -0500
Subject: [PATCH 245/359] Separate hydroconditioning for branch flow direction
 generation

---
 config/deny_branch_zero.lst |  4 +-
 config/deny_unit.lst        |  6 ++-
 docs/CHANGELOG.md           | 18 +++++++++
 src/clip_vectors_to_wbd.py  |  4 ++
 src/derive_level_paths.py   | 13 +++++--
 src/run_by_branch.sh        |  4 +-
 src/run_unit_wb.sh          | 74 +++++++++++++++++++++++++++----------
 src/utils/fim_enums.py      |  3 +-
 8 files changed, 98 insertions(+), 28 deletions(-)

diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst
index 1da9b4fcc..3d7a8cd9c 100644
--- a/config/deny_branch_zero.lst
+++ b/config/deny_branch_zero.lst
@@ -31,7 +31,7 @@ dem_lateral_thalweg_adj_{}.tif
 dem_meters_{}.tif
 dem_thalwegCond_{}.tif
 flowaccum_d8_burned_filled_{}.tif
-#flowdir_d8_burned_filled_{}.tif
+flowdir_d8_burned_filled_{}.tif
 flowdir_d8_burned_filled_flows_{}.tif
 flows_grid_boolean_{}.tif
 flows_points_pixels_{}.gpkg
@@ -44,7 +44,7 @@ gw_catchments_reaches_{}.tif
 headwaters_{}.tif
 #hydroTable_{}.csv
 idFile_{}.txt
-#LandSea_subset_{}.tif
+LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
diff --git a/config/deny_unit.lst b/config/deny_unit.lst
index cb7e76754..752719f7b 100644
--- a/config/deny_unit.lst
+++ b/config/deny_unit.lst
@@ -12,7 +12,11 @@ agree_smogrid_zerod.tif
 #3d_nld_subset_levees_burned.gpkg
 #branch_id.lst
 #branch_polygons.gpkg
+dem_burned.tif
+dem_burned_filled.tif
 dem_meters.tif
+flowdir_d8_burned_filled.tif
+flows_grid_boolean.tif
 #LandSea_subset.gpkg
 #LeveeProtectedAreas_subset.gpkg
 #nld_subset_levees.gpkg
@@ -29,4 +33,4 @@ nwm_headwaters.gpkg
 #wbd.gpkg
 #wbd8_clp.gpkg
 wbd_buffered.gpkg
-#wbd_buffered_streams.gpkg
+wbd_buffered_streams.gpkg
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 914d24cff..eb444f8e2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,24 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.11.0 - 2023-05-12 - [PR#903](https://github.com/NOAA-OWP/inundation-mapping/pull/903)
+
+These changes address some known issues where the DEM derived flowlines follow the incorrect flow path (address issues with stream order 1 and 2 only). The revised code adds a new workflow to generate a new flow direction raster separately for input to the `run_by_branch.sh` workflow (branch 0 remains unchanged). This modification helps ensure that the DEM derived flowlines follow the desired NWM flow line when generating the DEM derived flowlines at the branch level. 
+
+### Changes  
+- `config/deny_branch_zero.lst`: removed `LandSea_subset_{}.tif` and `flowdir_d8_burned_filled_{}.tif` from the "keep" list as these files are now kept in the huc root folder.
+- `config/deny_unit.lst`: added file cleanups for newly generated branch input files stored in the huc root folder (`dem_burned.tif`, `dem_burned_filled.tif`, `flowdir_d8_burned_filled.tif`, `flows_grid_boolean.tif`, `wbd_buffered_streams.gpkg`)
+- `src/clip_vectors_to_wbd.py`: saving the `wbd_streams_buffer` as an output gpkg for input to `derive_level_paths.py`
+- `src/derive_level_paths.py`: added a new step to clip the `out_stream_network_dissolved` with the `buffer_wbd_streams` polygon. this resolves errors with the edge case scenarios where a NWM flow line intersects the WBD buffer polygon
+- `src/run_unit_wb.sh`: Introduce new processing steps to generate separate outputs for input to branch 0 vs. all other branches. Remove the branch zero `outputs_cleanup.py` as the branches are no longer pointing to files stored in the branch 0 directory (stored in huc directory)
+   - Rasterize reach boolean (1 & 0) for all branches (not branch 0): using the `nwm_subset_streams_levelPaths_dissolved.gpkg` to define the branch levelpath flow lines
+   - AGREEDEM reconditioning for all branches (not branch 0)
+   - Pit remove burned DEM for all branches (not branch 0)
+   - D8 flow direction generation for all branches (not branch 0)
+- `src/run_by_branch.sh`: changed `clip_rasters_to_branches.py` input file location for `$tempHucDataDir/flowdir_d8_burned_filled.tif` (newly created file)
+
+<br/><br/>
+
 ## v4.3.10.0 - 2023-05-12 - [PR#888](https://github.com/NOAA-OWP/inundation-mapping/pull/888)
 
 `aggregate_by_huc.py` was taking a long time to process. Most HUCs can aggregate their branches into one merged hydrotable.csv in just 22 seconds, but a good handful took over 2 mins and a few took over 7 mins. When multiplied by 2,138 HUCs it was super slow. Multi-proc has not been added and it now takes appx 40 mins at 80 cores. 
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index a4f5a9576..5d80da537 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -15,6 +15,7 @@ def subset_vector_layers(subset_nwm_lakes,
                          hucCode,
                          subset_nwm_headwaters,
                          wbd_buffer_filename,
+                         wbd_streams_buffer_filename,
                          wbd_filename,
                          dem_filename,
                          dem_domain,
@@ -53,6 +54,7 @@ def subset_vector_layers(subset_nwm_lakes,
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
     wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+    wbd_streams_buffer.to_file(wbd_streams_buffer_filename, driver=getDriver(wbd_streams_buffer_filename), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
 
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea, mask=wbd_buffer)
@@ -152,6 +154,8 @@ def subset_vector_layers(subset_nwm_lakes,
                         required=True, default=None)
     parser.add_argument('-f','--wbd_buffer_filename', help='Buffered HUC boundary', 
                         required=True)
+    parser.add_argument('-s','--wbd_streams_buffer_filename', help='Buffered HUC boundary (streams)', 
+                        required=True)
     parser.add_argument('-g','--wbd-filename', help='HUC boundary', required=True)
     parser.add_argument('-i','--dem-filename', help='DEM filename', required=True)
     parser.add_argument('-j','--dem-domain', help='DEM domain polygon', required=True)
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index f34dc494c..90f09b0a1 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -8,7 +8,7 @@
 from stream_branches import StreamNetwork
 from utils.fim_enums import FIM_exit_codes
 
-def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribute,
+def Derive_level_paths(in_stream_network, buffer_wbd_streams, out_stream_network, branch_id_attribute,
                        out_stream_network_dissolved=None, huc_id=None,
                        headwaters_outfile=None, catchments=None, waterbodies=None,
                        catchments_outfile=None,
@@ -38,7 +38,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         # Throw an exception with valid text. This will show up in the non-zero exit codes and explain why an error.
         # Later, we can look at creating custom sys exit codes 
         # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file.")
-        print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
+        print("Sorry, no streams exist and processing can not continue. This could be an empty file.")
         sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # values_exluded of 1 and 2 mean where are dropping stream orders 1 and 2. We are leaving those
@@ -47,7 +47,8 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
 
     # if there are no reaches at this point (due to filtering)
     if (len(stream_network) == 0):
-        print("No branches exist but branch zero processing will continue. This could be due to stream order filtering.")
+        print("No branches exist but branch zero processing will continue (Exit 63). This could be due to stream order filtering.")
+        #sys.exit(FIM_exit_codes.NO_BRANCH_LEVELPATHS_EXIST.value)  # will send a 63 back
         return
                                                  
     inlets_attribute = 'inlet_id'
@@ -152,6 +153,11 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
         stream_network = stream_network.remove_branches_in_waterbodies(waterbodies=waterbodies,
                                                                        out_vector_files=out_stream_network_dissolved,
                                                                        verbose=False)
+        # clip dissolved stream network to the wbd_buffered domain (avoids issues with reaches that extend outside buffer)
+        wbd_buffer = gpd.read_file(buffer_wbd_streams)
+        stream_network_out = gpd.read_file(out_stream_network_dissolved)
+        stream_network_out = gpd.clip(stream_network_out,wbd_buffer)
+        stream_network_out.to_file(out_stream_network_dissolved, index=True, driver='GPKG')
                                        
     if branch_inlets_outfile is not None:
         branch_inlets = stream_network.derive_inlet_points_by_feature(feature_attribute=branch_id_attribute,
@@ -167,6 +173,7 @@ def Derive_level_paths(in_stream_network, out_stream_network, branch_id_attribut
 
     parser = argparse.ArgumentParser(description='Create stream network level paths')
     parser.add_argument('-i','--in-stream-network', help='Input stream network', required=True)
+    parser.add_argument('-s','--buffer-wbd-streams', help='Input wbd buffer for stream network', required=True)
     parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
     parser.add_argument('-u','--huc-id', help='Current HUC ID', required=False, default=None)
     parser.add_argument('-r','--reach-id-attribute', help='Reach ID attribute to use in source file', required=False, default='HydroID')
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index a0f6511cd..4a82c84d3 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -46,11 +46,11 @@ ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="
 # ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $tempHucDataDir/nwm_headwaters.gpkg
 Tcount
 
-## GET RASTERS FROM BRANCH ZERO AND CLIP TO CURRENT BRANCH BUFFER ##
+## GET RASTERS FROM ROOT HUC DIRECTORY AND CLIP TO CURRENT BRANCH BUFFER ##
 echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $tempHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $tempHucDataDir/dem_meters.tif $tempBranchDataDir/$branch_zero_id/flowdir_d8_burned_filled_$branch_zero_id.tif -c $tempCurrentBranchDataDir/dem_meters.tif $tempCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
+$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $tempHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $tempHucDataDir/dem_meters.tif $tempHucDataDir/flowdir_d8_burned_filled.tif -c $tempCurrentBranchDataDir/dem_meters.tif $tempCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
 Tcount
 
 ## GET RASTER METADATA
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index aba913457..90f9f895b 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -52,6 +52,7 @@ cmd_args+=" -b $tempHucDataDir/nwm_subset_streams.gpkg"
 cmd_args+=" -d $hucNumber"
 cmd_args+=" -e $tempHucDataDir/nwm_headwater_points_subset.gpkg"
 cmd_args+=" -f $tempHucDataDir/wbd_buffered.gpkg"
+cmd_args+=" -s $tempHucDataDir/wbd_buffered_streams.gpkg"
 cmd_args+=" -g $tempHucDataDir/wbd.gpkg"
 cmd_args+=" -i $input_DEM"
 cmd_args+=" -j $input_DEM_domain"
@@ -85,12 +86,17 @@ Tcount
 echo -e $startDiv"Generating Level Paths for $hucNumber"
 date -u
 Tstart
-$srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg -b $branch_id_attribute -r "ID" -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $tempHucDataDir/nwm_headwaters.gpkg -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg
+$srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg -s $tempHucDataDir/wbd_buffered_streams.gpkg -b $branch_id_attribute -r "ID" -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $tempHucDataDir/nwm_headwaters.gpkg -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg
 
 # test if we received a non-zero code back from derive_level_paths.py
-subscript_exit_code=$?
+#subscript_exit_code=$?
+
 # we have to retrow it if it is not a zero (but it will stop further execution in this script)
-if [ $subscript_exit_code -ne 0 ]; then exit $subscript_exit_code; fi
+#if [ $subscript_exit_code -ne 0 ] && [ $subscript_exit_code -ne 62 ] && [ $subscript_exit_code -eq 63 ]; then exit $subscript_exit_code; fi
+
+# check if level paths exists
+levelpaths_exist=1
+if [ ! -f $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg ]; then levelpaths_exist=0; fi
 Tcount
 
 ## ASSOCIATE LEVEL PATHS WITH LEVEES
@@ -157,13 +163,23 @@ Tstart
 python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $tempHucDataDir/dem_meters.tif -nld $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif -out $tempHucDataDir/dem_meters.tif
 Tcount
 
-## RASTERIZE REACH BOOLEAN (1 & 0) ##
+## RASTERIZE REACH BOOLEAN (1 & 0) - BRANCH 0 (include all NWM streams) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $branch_zero_id"
 date -u
 Tstart
 gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_subset_streams.gpkg $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
 Tcount
 
+## RASTERIZE REACH BOOLEAN (1 & 0) - BRANCHES (Not 0) (NWM levelpath streams) ##
+if [ "$levelpaths_exist" = "1" ] 
+then 
+    echo -e $startDiv"Rasterize Reach Boolean $hucNumber (Branches)"
+    date -u
+    Tstart
+    gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg $tempHucDataDir/flows_grid_boolean.tif
+    Tcount
+fi
+
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
 echo -e $startDiv"Rasterize NWM Headwaters $hucNumber $branch_zero_id"
 date -u
@@ -171,7 +187,7 @@ Tstart
 gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_headwater_points_subset.gpkg $tempCurrentBranchDataDir/headwaters_$branch_zero_id.tif
 Tcount
 
-## DEM Reconditioning ##
+## DEM Reconditioning - BRANCH 0 (include all NWM streams) ##
 # Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
 # This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
@@ -180,20 +196,52 @@ Tstart
 python3 -m memory_profiler $srcDir/agreedem.py -r $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $tempHucDataDir/dem_meters.tif -w $tempCurrentBranchDataDir -o $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
 Tcount
 
-## PIT REMOVE BURNED DEM ##
+## DEM Reconditioning - BRANCHES (NOT 0) (NWM levelpath streams) ##
+# Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
+# This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
+if [ "$levelpaths_exist" = "1" ] 
+then 
+    echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber (Branches)"
+    date -u
+    Tstart
+    python3 -m memory_profiler $srcDir/agreedem.py -r $tempHucDataDir/flows_grid_boolean.tif -d $tempHucDataDir/dem_meters.tif -w $tempHucDataDir -o $tempHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+    Tcount
+fi
+
+## PIT REMOVE BURNED DEM - BRANCH 0 (include all NWM streams) ##
 echo -e $startDiv"Pit remove Burned DEM $hucNumber $branch_zero_id"
 date -u
 Tstart
 rd_depression_filling $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
 Tcount
 
-## D8 FLOW DIR ##
+## PIT REMOVE BURNED DEM - BRANCHES (NOT 0) (NWM levelpath streams) ##
+if [ "$levelpaths_exist" = "1" ] 
+then 
+    echo -e $startDiv"Pit remove Burned DEM $hucNumber (Branches)"
+    date -u
+    Tstart
+    rd_depression_filling $tempHucDataDir/dem_burned.tif $tempHucDataDir/dem_burned_filled.tif
+    Tcount
+fi
+
+## D8 FLOW DIR - BRANCH 0 (include all NWM streams) ##
 echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $branch_zero_id"
 date -u
 Tstart
 mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
 Tcount
 
+## D8 FLOW DIR - BRANCHES (NOT 0) (NWM levelpath streams) ##
+if [ "$levelpaths_exist" = "1" ] 
+then 
+    echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber (Branches)"
+    date -u
+    Tstart
+    mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $tempHucDataDir/dem_burned_filled.tif -p $tempHucDataDir/flowdir_d8_burned_filled.tif
+    Tcount
+fi
+
 ## MAKE A COPY OF THE DEM FOR BRANCH 0
 echo -e $startDiv"Copying DEM to Branch 0"
 date -u
@@ -267,18 +315,6 @@ if [ -f $deny_unit_list ]; then
     Tcount
 fi
 
-# -------------------
-## REMOVE FILES FROM DENY LIST FOR BRANCH ZERO (but using normal branch deny) ##
-if [ "$has_deny_branch_zero_override" == "1" ]
-then
-    echo -e $startDiv"Second cleanup of files for branch zero (none default)"
-    $srcDir/outputs_cleanup.py -d $tempHucDataDir -l $deny_branch_zero_list -b 0
-
-else 
-    echo -e $startDiv"Second cleanup of files for branch zero using the default branch deny list"
-    $srcDir/outputs_cleanup.py -d $tempHucDataDir -l $deny_branches_list -b 0
-fi
-
 echo "---- HUC $hucNumber - branches have now been processed"
 Calc_Duration $branch_processing_start_time
 echo
diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py
index 5af4d8451..f5692aba6 100644
--- a/src/utils/fim_enums.py
+++ b/src/utils/fim_enums.py
@@ -25,5 +25,6 @@ class FIM_exit_codes(Enum):
 
     UNIT_NO_BRANCHES = 60
     NO_FLOWLINES_EXIST = 61
-    EXCESS_UNIT_ERRORS = 62    
+    EXCESS_UNIT_ERRORS = 62
+    NO_BRANCH_LEVELPATHS_EXIST = 63    
             
\ No newline at end of file

From 625cb3f9690cd29a229488b494cdefd243280ec9 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Tue, 16 May 2023 09:18:43 -0500
Subject: [PATCH 246/359] Update CITATION.cff

v4.3.11.0
---
 CITATION.cff | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CITATION.cff b/CITATION.cff
index 029d8c02a..f95808091 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.3.9.0
+version: 4.3.11.0
 date-released: 2023

From b9bd379d77c338968e8472bdb21db2995bcf8821 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Thu, 18 May 2023 10:08:07 -0400
Subject: [PATCH 247/359] 4.3.11.1 Update instances of pandas.append
 (deprecated) to pandas.concat

---
 CITATION.cff                                  |   2 +-
 data/nws/preprocess_ahps_nws.py               |   2 +-
 data/usgs/acquire_and_preprocess_3dep_dems.py |   3 +-
 data/usgs/preprocess_ahps_usgs.py             |   2 +-
 docs/CHANGELOG.md                             |  37 ++
 src/add_crosswalk.py                          |   2 -
 src/adjust_headwater_streams.py               |   4 +-
 src/aggregate_vector_inputs.py                |   4 +-
 src/reset_mannings.py                         |   2 +-
 tools/aggregate_mannings_calibration.py       |   2 +-
 tools/eval_plots.py                           |   2 +-
 tools/generate_categorical_fim.py             |   2 +-
 tools/generate_categorical_fim_flows.py       |   4 +-
 tools/plots/eval_plots.py                     |   4 +-
 tools/plots/utils/shared_functions.py         |   2 +-
 tools/rating_curve_comparison.py              |  16 +-
 tools/rating_curve_get_usgs_curves.py         |   6 +-
 tools/thalweg_drop_check.py                   | 383 ------------------
 tools/tools_shared_functions.py               |   2 +-
 19 files changed, 67 insertions(+), 414 deletions(-)
 delete mode 100644 tools/thalweg_drop_check.py

diff --git a/CITATION.cff b/CITATION.cff
index f95808091..893916853 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.3.11.0
+version: 4.3.11.1
 date-released: 2023
diff --git a/data/nws/preprocess_ahps_nws.py b/data/nws/preprocess_ahps_nws.py
index 9bba37573..8bc6c8eb6 100644
--- a/data/nws/preprocess_ahps_nws.py
+++ b/data/nws/preprocess_ahps_nws.py
@@ -355,7 +355,7 @@ def preprocess_nws(source_dir, destination, reference_raster):
     all_attributes = pd.DataFrame()
     for i in attribute_files:
         attribute_df = pd.read_csv(i, dtype={'huc':str})
-        all_attributes = all_attributes.append(attribute_df)
+        all_attributes = pd.concat([all_attributes, attribute_df])
     
     if not all_attributes.empty:
         all_attributes.to_csv(destination / 'attributes.csv', index = False)    
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index d69b0eea3..65bef1bcf 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -7,6 +7,7 @@
 import subprocess
 import sys
 import traceback
+import pandas as pd
 import geopandas as gpd
 
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait
@@ -309,7 +310,7 @@ def polygonize(target_output_folder_path):
         if n == 0:
             dem_gpkgs = gdf
         else:
-            dem_gpkgs = dem_gpkgs.append(gdf)
+            dem_gpkgs = pd.concat([dem_gpkgs, gdf])
 
         os.remove(edge_tif)
         
diff --git a/data/usgs/preprocess_ahps_usgs.py b/data/usgs/preprocess_ahps_usgs.py
index 1c8b78be3..500bdffe7 100644
--- a/data/usgs/preprocess_ahps_usgs.py
+++ b/data/usgs/preprocess_ahps_usgs.py
@@ -450,7 +450,7 @@ def preprocess_usgs(source_dir, destination, reference_raster):
     all_attributes = pd.DataFrame()
     for i in attribute_files:
         attribute_df = pd.read_csv(i, dtype={'huc':str})
-        all_attributes = all_attributes.append(attribute_df)
+        all_attributes = pd.concat([all_attributes, attribute_df])
     if not all_attributes.empty:
         all_attributes.to_csv(destination / 'attributes.csv', index = False)
             
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index eb444f8e2..f8bfb999d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,43 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.11.1 - 2023-05-16 - [PR#904](https://github.com/NOAA-OWP/inundation-mapping/pull/904)
+
+`pandas.append` was deprecated in our last Pandas upgrade (v4.3.9.0). This PR updates the remaining instances of `pandas.append` to `pandas.concat`.
+
+The file `tools/thalweg_drop_check.py` had an instance of `pandas.append` but was deleted as it is no longer used or necessary.
+
+### Changes
+
+The following files had instances of `pandas.append` changed to `pandas.concat`:
+- `data/`
+    - `nws/preprocess_ahps_nws.py`
+    - `usgs/`
+        - `acquire_and_preprocess_3dep_dems.py`
+        - `preprocess_ahps_usgs.py`
+- `src/`
+    - `add_crosswalk.py`
+    - `adjust_headwater_streams.py`
+    - `aggregate_vector_inputs.py`
+    - `reset_mannings.py`
+- `tools/`
+    - `aggregate_mannings_calibration.py`
+    - `eval_plots.py`
+    - `generate_categorical_fim.py`
+    - `generate_categorical_fim_flows.py`
+    - `plots/`
+        - `eval_plots.py`
+        - `utils/shared_functions.py`
+    - `rating_curve_comparison.py`
+    - `rating_curve_get_usgs_curves.py`
+    - `tools_shared_functions.py`
+
+### Removals
+
+- `tools/thalweg_drop_check.py`
+
+<br/><br/>
+
 ## v4.3.11.0 - 2023-05-12 - [PR#903](https://github.com/NOAA-OWP/inundation-mapping/pull/903)
 
 These changes address some known issues where the DEM derived flowlines follow the incorrect flow path (address issues with stream order 1 and 2 only). The revised code adds a new workflow to generate a new flow direction raster separately for input to the `run_by_branch.sh` workflow (branch 0 remains unchanged). This modification helps ensure that the DEM derived flowlines follow the desired NWM flow line when generating the DEM derived flowlines at the branch level. 
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index d75586e32..a6a30da8e 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -9,8 +9,6 @@
 import argparse
 import sys
 from utils.shared_functions import getDriver
-# sys.path.append('/foss_fim/src')
-# sys.path.append('/foss_fim/config')
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
 from memory_profiler import profile
diff --git a/src/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
index afea3f337..fdd803e5d 100644
--- a/src/adjust_headwater_streams.py
+++ b/src/adjust_headwater_streams.py
@@ -41,7 +41,7 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
     #     print (f"nws lid(s) {missing_nws_lids} missing from aggregate dataset in huc {huc}")
 
     # Combine NWM headwaters and AHPS sites to be snapped to NHDPlus HR segments
-    headwater_pts = headwater_limited.append(nws_lid_limited)
+    headwater_pts = pd.concat([headwater_limited, nws_lid_limited])
     headwater_pts = headwater_pts.reset_index(drop=True)
 
     if headwater_pts is not None:
@@ -159,7 +159,7 @@ def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
         del nhd_headwater_streams_adj
 
         try:
-            combined_pts = snapped_ahps_points.append(nhd_headwater_points_adj)
+            combined_pts = pd.concat([snapped_ahps_points, nhd_headwater_points_adj])
         except:
             combined_pts = nhd_headwater_points_adj.copy()
 
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
index 6ebb91d45..3b2bdef97 100755
--- a/src/aggregate_vector_inputs.py
+++ b/src/aggregate_vector_inputs.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 import os
-# sys.path.append('/foss_fim/src')
+import pandas as pd
 import geopandas as gpd
 from utils.shared_variables import PREP_PROJECTION
 from utils.shared_functions import getDriver
@@ -282,7 +282,7 @@ def subset_stream_networks(args, huc):
             nhd_huc8_intersections['pt_type'] = 'nhd_huc8_intersections'
             nhd_huc8_intersections = nhd_huc8_intersections.rename(columns={"NHDPlusID": headwater_pts_id})
             nhd_huc8_intersections = nhd_huc8_intersections[column_order]
-            adj_nhd_headwater_points_all = adj_nhd_headwater_points.append(nhd_huc8_intersections)
+            adj_nhd_headwater_points_all = pd.concat([adj_nhd_headwater_points, nhd_huc8_intersections])
             adj_nhd_headwater_points_all = adj_nhd_headwater_points_all.reset_index(drop=True)
 
             adj_nhd_streams_all_fileName = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
diff --git a/src/reset_mannings.py b/src/reset_mannings.py
index 888e156e3..06990f412 100755
--- a/src/reset_mannings.py
+++ b/src/reset_mannings.py
@@ -165,7 +165,7 @@ def identify_small_reaches(stream_network,min_catchment_area=None,min_stream_len
                 update_id = stream_network.loc[stream_network.HydroID==short_id]['HydroID'].item()
 
             str_order = stream_network.loc[stream_network.HydroID==short_id]['order_'].item()
-            sml_segs = sml_segs.append({'short_id':short_id, 'update_id':update_id, 'str_order':str_order}, ignore_index=True)
+            sml_segs = pd.concat([sml_segs, {'short_id':short_id, 'update_id':update_id, 'str_order':str_order}], ignore_index=True)
 
     #print("Number of short reaches [{} < {} and {} < {}] = {}".format("areasqkm", min_catchment_area, "LengthKm", min_stream_length, len(sml_segs)))
     
diff --git a/tools/aggregate_mannings_calibration.py b/tools/aggregate_mannings_calibration.py
index c57b17776..ff9565850 100755
--- a/tools/aggregate_mannings_calibration.py
+++ b/tools/aggregate_mannings_calibration.py
@@ -32,7 +32,7 @@ def aggregate_parameter_sets(huc_list_path,calibration_stat_folder,summary_file,
                     total_area_stats = total_area_stats.loc[['true_positives_count', 'true_negatives_count', 'false_positives_count', 'false_negatives_count','masked_count', 'cell_area_m2', 'CSI'],:]
                     total_area_stats = total_area_stats.reset_index()
                     total_area_stats_table = pd.DataFrame({'metric': total_area_stats.iloc[:,0], 'value': total_area_stats.iloc[:,1], 'stream_order': stream_order, 'mannings_n': mannings_value, 'huc': huc, 'interval': flood_recurrence})
-                    mannings_summary_table = mannings_summary_table.append(total_area_stats_table, ignore_index=True)
+                    mannings_summary_table = pd.concat([mannings_summary_table, total_area_stats_table], ignore_index=True)
 
     mannings_summary_table.to_csv(summary_file,index=False)
 
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index f64082744..3d5268eaf 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -460,7 +460,7 @@ def eval_plots(metrics_csv, workspace, versions = [],
         #Set extent configuration to MS (instead of FR)
         fim_1_rows['extent_config'] = 'MS'
         #Append duplicate FIM 1 rows to original dataframe
-        csv_df = csv_df.append(fim_1_rows, ignore_index = True)
+        csv_df = pd.concat([csv_df, fim_1_rows], ignore_index = True)
 
     # If versions are supplied then filter out
     if versions:
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index 31f4c32ed..a51f8df0a 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -581,7 +581,7 @@ def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us
         # HUC has to be read in as string to preserve leading zeros.
         try:
             temp_df = pd.read_csv(full_csv_path, dtype={'huc':str})
-            all_csv_df = all_csv_df.append(temp_df, ignore_index = True)
+            all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index = True)
             refined_csv_files_list.append(csv_file)
         except Exception:  # Happens if a file is empty (i.e. no mapping)
             pass
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index 3c64b25ac..2716e0647 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -107,7 +107,7 @@ def process_generate_flows(huc, huc_dictionary, threshold_url, all_lists, worksp
         csv_df = pd.DataFrame()
         for threshold in flood_categories:
             line_df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'magnitude': threshold, 'q':flows[threshold], 'q_uni':flows['units'], 'q_src':flow_source, 'stage':stages[threshold], 'stage_uni':stages['units'], 's_src':stage_source, 'wrds_time':wrds_timestamp, 'nrldb_time':nrldb_timestamp,'nwis_time':nwis_timestamp, 'lat':[lat], 'lon':[lon]})
-            csv_df = csv_df.append(line_df)
+            csv_df = pd.concat([csv_df, line_df])
         #Round flow and stage columns to 2 decimal places.
         csv_df = csv_df.round({'q':2,'stage':2})
 
@@ -241,7 +241,7 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
         full_csv_path = os.path.join(attributes_dir, csv)
         # Huc has to be read in as string to preserve leading zeros.
         temp_df = pd.read_csv(full_csv_path, dtype={'huc':str})
-        all_csv_df = all_csv_df.append(temp_df, ignore_index = True)
+        all_csv_df = pd.concat(all_csv_df, temp_df], ignore_index = True)
     # Write to file
     all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index = False)
    
diff --git a/tools/plots/eval_plots.py b/tools/plots/eval_plots.py
index 9c29087e1..443a2ba32 100755
--- a/tools/plots/eval_plots.py
+++ b/tools/plots/eval_plots.py
@@ -103,7 +103,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
         #Set extent configuration to MS (instead of FR)
         fim_1_rows['extent_config'] = 'MS'
         #Append duplicate FIM 1 rows to original dataframe
-        csv_df = csv_df.append(fim_1_rows, ignore_index = True)
+        csv_df = pd.concat([csv_df, fim_1_rows], ignore_index = True)
         
     #If versions are supplied then filter out    
     if versions:
@@ -282,7 +282,7 @@ def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'
                 #Join to dataset
                 dataset_with_subset = dataset.merge(subset, on = 'nws_lid')
                 #Append rows to final_join dataframe
-                final_join = final_join.append(dataset_with_subset)
+                final_join = pd.concat([final_join, dataset_with_subset])
         
         #Modify version field
         final_join['version'] = final_join.version.str.split('_nws|_usgs').str[0]
diff --git a/tools/plots/utils/shared_functions.py b/tools/plots/utils/shared_functions.py
index 60342059e..e2664173f 100644
--- a/tools/plots/utils/shared_functions.py
+++ b/tools/plots/utils/shared_functions.py
@@ -315,7 +315,7 @@ def filter_dataframe(dataframe, unique_field):
         #Query filtered dataframe and only include data associated with the common sites for that magnitude
         filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
         #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
-        final_filtered_dataframe = final_filtered_dataframe.append(filtered_common_sites, ignore_index = True)            
+        final_filtered_dataframe = pd.concat([final_filtered_dataframe, filtered_common_sites], ignore_index = True)            
     
     return final_filtered_dataframe, all_unique_sites
 
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 4b6a5e7ab..ad5914a6f 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -107,7 +107,7 @@ def generate_rating_curve_metrics(args):
             if hydrotable.empty:
                 hydrotable = branch_hydrotable
             else:
-                hydrotable = hydrotable.append(branch_hydrotable)
+                hydrotable = pd.concat([hydrotable, branch_hydrotable])
 
         # Join rating curves with elevation data
         #elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
@@ -136,10 +136,10 @@ def generate_rating_curve_metrics(args):
                 limited_hydrotable_default = hydrotable.filter(items=['location_id','elevation_ft', 'default_discharge_cfs','HydroID', 'levpa_id', 'dem_adj_elevation'])
                 limited_hydrotable_default['discharge_cfs'] = limited_hydrotable_default.default_discharge_cfs
                 limited_hydrotable_default['source'] = "FIM_default"
-                rating_curves = limited_hydrotable.append(select_usgs_gages)
-                rating_curves = rating_curves.append(limited_hydrotable_default)
+                rating_curves = pd.concat([limited_hydrotable, select_usgs_gages])
+                rating_curves = pd.concat([rating_curves, limited_hydrotable_default])
             else:
-                rating_curves = limited_hydrotable.append(select_usgs_gages)
+                rating_curves = pd.concat([limited_hydrotable, select_usgs_gages])
 
             # Add stream order
             stream_orders = hydrotable.filter(items=['location_id','order_']).drop_duplicates()
@@ -164,7 +164,7 @@ def generate_rating_curve_metrics(args):
 
             # Append catfim data (already set up in format similar to nwm_recurr_intervals_all)
             cat_fim = pd.read_csv(catfim_flows_filename, dtype={'feature_id':str})
-            nwm_recurr_intervals_all = nwm_recurr_intervals_all.append(cat_fim)
+            nwm_recurr_intervals_all = pd.concat([nwm_recurr_intervals_all, cat_fim])
 
             # Convert discharge to cfs and filter
             nwm_recurr_intervals_all['discharge_cfs'] = nwm_recurr_intervals_all.discharge_cms * 35.3147
@@ -229,7 +229,7 @@ def generate_rating_curve_metrics(args):
 
                 # Melt dataframe
                 usgs_pred_elev = pd.melt(usgs_pred_elev, id_vars=['location_id','feature_id','recurr_interval','discharge_cfs','HUC','HUC4','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
-                nwm_recurr_data_table = nwm_recurr_data_table.append(usgs_pred_elev)
+                nwm_recurr_data_table = pd.concat([nwm_recurr_data_table, usgs_pred_elev])
 
                 # Interpolate FIM elevation at USGS observations
                 # fim_rc = fim_rc.merge(usgs_crosswalk, on="location_id")
@@ -247,7 +247,7 @@ def generate_rating_curve_metrics(args):
                 # usgs_rc = pd.melt(usgs_rc, id_vars=['location_id','discharge_cfs','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
                 #
                 # if not usgs_rc.empty:
-                #     usgs_recurr_data = usgs_recurr_data.append(usgs_rc)
+                #     usgs_recurr_data = pd.concat([usgs_recurr_data, usgs_rc])
 
             # Generate stats for all sites in huc
             # if not usgs_recurr_data.empty:
@@ -794,7 +794,7 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
 
     # Combine all dataframes into one
     all_results = sierra_results[0]
-    all_results = all_results.append(sierra_results[1:])
+    all_results = pd.concat([all_results, sierra_results[1:]])
 
     # Melt results for boxplotting
     all_results_melted = all_results.melt(id_vars=["location_id", '_version'], 
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
index 2c152850b..ba84689b8 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -119,7 +119,7 @@ def write_categorical_flow_files(metadata, workspace):
                 data['location_id'] = usgs_code
                 data = data.rename(columns = {'discharge':'discharge_cms'})
                 #Append site data to master DataFrame
-                all_data = all_data.append(data, ignore_index = True)
+                all_data = pd.concat([all_data, data], ignore_index = True)
     
     #Write CatFIM flows to file
     final_data = all_data[['feature_id','discharge_cms', 'recurr_interval']]
@@ -207,7 +207,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
                 chunk_list, chunk_df = get_metadata(metadata_url, select_by, chunk, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None )
                 #Append chunk data to metadata_list/df
                 metadata_list.extend(chunk_list)
-                metadata_df = metadata_df.append(chunk_df)
+                metadata_df = pd.concat([metadata_df, chunk_df])
         else:
             #If selector has less than max sites, then get metadata.
             metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None )
@@ -274,7 +274,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
         curve['navd88_datum'] = navd88_datum
         curve['elevation_navd88'] = curve['stage'] + navd88_datum
         #Append all rating curves to a dataframe
-        all_rating_curves = all_rating_curves.append(curve)
+        all_rating_curves = pd.concat([all_rating_curves, curve])
 
     #Rename columns and add attribute indicating if rating curve exists
     sites_gdf.rename(columns = {'nwm_feature_id':'feature_id','usgs_site_code':'location_id'}, inplace = True)
diff --git a/tools/thalweg_drop_check.py b/tools/thalweg_drop_check.py
deleted file mode 100644
index 319c8bcd5..000000000
--- a/tools/thalweg_drop_check.py
+++ /dev/null
@@ -1,383 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import geopandas as gpd
-sys.path.append('/foss_fim/src')
-from shapely.geometry import Point, LineString
-import rasterio
-import numpy as np
-import argparse
-import matplotlib.pyplot as plt
-import seaborn as sns
-from collections import deque
-from os.path import join
-from multiprocessing import Pool
-from utils.shared_functions import getDriver
-from rasterio import features
-from reachID_grid_to_vector_points import convert_grid_cells_to_points
-import warnings
-warnings.simplefilter(action='ignore', category=FutureWarning)
-
-"""
-    Plot Rating Curves and Compare to USGS Gages
-
-    Parameters
-    ----------
-    fim_dir : str
-        Directory containing FIM output folders.
-    output_dir : str
-        Stream layer to be evaluated.
-    stream_type : str
-        File name of USGS rating curves.
-    point_density : str
-        Elevation sampling density.
-    number_of_jobs : str
-        Number of jobs.
-"""
-
-
-def compare_thalweg(args):
-
-    huc_dir                             = args[0]
-    stream_type                         = args[1]
-    point_density                       = args[2]
-    huc                                 = args[3]
-    dem_meters_filename                 = args[4]
-    dem_lateral_thalweg_adj_filename    = args[5]
-    dem_thalwegCond_filename            = args[6]
-    profile_plots_filename              = args[7]
-    profile_gpkg_filename               = args[8]
-    profile_table_filename              = args[9]
-    flows_grid_boolean_filename         = args[10]
-
-    if stream_type == 'derived':
-
-        dem_derived_reaches_filename = os.path.join(huc_dir,'demDerived_reaches_split.gpkg')
-        streams = gpd.read_file(dem_derived_reaches_filename)
-        nhd_headwater_filename = os.path.join(huc_dir,'nhd_headwater_points_subset.gpkg')
-        wbd_filename = os.path.join(huc_dir,'wbd.gpkg')
-        wbd = gpd.read_file(wbd_filename)
-        headwaters_layer = gpd.read_file(nhd_headwater_filename,mask=wbd)
-        headwater_list = headwaters_layer.loc[headwaters_layer.pt_type == 'nws_lid']
-        stream_id = 'HydroID'
-
-    elif stream_type == 'burnline':
-
-        nhd_reaches_filename = os.path.join(huc_dir,'NHDPlusBurnLineEvent_subset.gpkg')
-        nhd_reaches = gpd.read_file(nhd_reaches_filename)
-        streams = nhd_reaches.copy()
-        headwaters_layer = None
-
-        # Get lists of all complete reaches using headwater attributes
-        headwater_list = streams.loc[streams.nws_lid!=''].nws_lid
-        stream_id = 'NHDPlusID'
-
-    headwater_col = 'is_headwater'
-    streams[headwater_col] = False
-    headwater_list = headwater_list.reset_index(drop=True)
-
-    if stream_type == 'derived':
-        streams['nws_lid'] = ''
-
-        if streams.NextDownID.dtype != 'int': streams.NextDownID = streams.NextDownID.astype(int)
-
-        min_dist = np.empty(len(headwater_list))
-        streams['min_dist'] = 1000
-
-        for i, point in headwater_list.iterrows():
-            streams['min_dist'] = [point.geometry.distance(line) for line in streams.geometry]
-            streams.loc[streams.min_dist==np.min(streams.min_dist),'nws_lid'] = point.site_id
-
-        headwater_list = headwater_list.site_id
-
-    streams.set_index(stream_id,inplace=True,drop=False)
-
-    # Collect headwater streams
-    single_stream_paths = []
-    dem_meters = rasterio.open(dem_meters_filename,'r')
-    index_option = 'reachID'
-    for index, headwater_site in enumerate(headwater_list):
-        stream_path = get_downstream_segments(streams.copy(),'nws_lid', headwater_site,'downstream',stream_id,stream_type)
-        stream_path = stream_path.reset_index(drop=True)
-        stream_path = stream_path.sort_values(by=['downstream_count'])
-        stream_path = stream_path.loc[stream_path.downstream==True]
-        if stream_type == 'burnline':
-            geom_value = []
-            for index, segment in stream_path.iterrows():
-                lineString = LineString(segment.geometry.coords[::-1])
-                geom_value = geom_value + [(lineString, segment.downstream_count)]
-            nhd_reaches_raster = features.rasterize(shapes=geom_value , out_shape=[dem_meters.height, dem_meters.width],fill=dem_meters.nodata,transform=dem_meters.transform, all_touched=True, dtype=np.float32)
-            flow_bool = rasterio.open(flows_grid_boolean_filename)
-            flow_bool_data = flow_bool.read(1)
-            nhd_reaches_raster = np.where(flow_bool_data == int(0), -9999.0, (nhd_reaches_raster).astype(rasterio.float32))
-            out_dem_filename = os.path.join(huc_dir,'NHDPlusBurnLineEvent_raster.tif')
-            with rasterio.open(out_dem_filename, "w", **dem_meters.profile, BIGTIFF='YES') as dest:
-                dest.write(nhd_reaches_raster, indexes = 1)
-            stream_path = convert_grid_cells_to_points(out_dem_filename,index_option)
-        stream_path["headwater_path"] = headwater_site
-        single_stream_paths = single_stream_paths + [stream_path]
-        print(f"length of {headwater_site} path: {len(stream_path)}")
-
-    # Collect elevation values from multiple grids along each individual reach point
-    dem_lateral_thalweg_adj = rasterio.open(dem_lateral_thalweg_adj_filename,'r')
-    dem_thalwegCond = rasterio.open(dem_thalwegCond_filename,'r')
-    thalweg_points = gpd.GeoDataFrame()
-    for path in single_stream_paths:
-        split_points = []
-        stream_ids = []
-        dem_m_elev = []
-        dem_burned_filled_elev = []
-        dem_lat_thal_adj_elev = []
-        dem_thal_adj_elev = []
-        headwater_path = []
-        index_count = []
-        for index, segment in path.iterrows():
-            if stream_type == 'derived':
-                linestring = segment.geometry
-                if point_density == 'midpoints':
-                    midpoint = linestring.interpolate(0.5,normalized=True)
-                    stream_ids = stream_ids + [segment[stream_id]]
-                    split_points = split_points + [midpoint]
-                    index_count = index_count + [segment.downstream_count]
-                    dem_m_elev = dem_m_elev + [np.array(list(dem_meters.sample((Point(midpoint).coords), indexes=1))).item()]
-                    dem_lat_thal_adj_elev = dem_lat_thal_adj_elev + [np.array(list(dem_lateral_thalweg_adj.sample((Point(midpoint).coords), indexes=1))).item()]
-                    dem_thal_adj_elev = dem_thal_adj_elev + [np.array(list(dem_thalwegCond.sample((Point(midpoint).coords), indexes=1))).item()]
-                    headwater_path = headwater_path + [segment.headwater_path]
-                elif point_density == 'all_points':
-                    count=0
-                    for point in zip(*linestring.coords.xy):
-                        stream_ids = stream_ids + [segment[stream_id]]
-                        split_points = split_points + [Point(point)]
-                        count = count + 1
-                        index_count = index_count + [segment.downstream_count*1000 + count]
-                        dem_m_elev = dem_m_elev + [np.array(list(dem_meters.sample((Point(point).coords), indexes=1))).item()]
-                        dem_lat_thal_adj_elev = dem_lat_thal_adj_elev + [np.array(list(dem_lateral_thalweg_adj.sample((Point(point).coords), indexes=1))).item()]
-                        dem_thal_adj_elev = dem_thal_adj_elev + [np.array(list(dem_thalwegCond.sample((Point(point).coords), indexes=1))).item()]
-                        headwater_path = headwater_path + [segment.headwater_path]
-            elif stream_type == 'burnline':
-                stream_ids = stream_ids + [segment['id']]
-                split_points = split_points + [Point(segment.geometry)]
-                index_count = index_count + [segment['id']]
-                dem_m_elev = dem_m_elev + [np.array(list(dem_meters.sample((Point(segment.geometry).coords), indexes=1))).item()]
-                dem_lat_thal_adj_elev = dem_lat_thal_adj_elev + [np.array(list(dem_lateral_thalweg_adj.sample((Point(segment.geometry).coords), indexes=1))).item()]
-                dem_thal_adj_elev = dem_thal_adj_elev + [np.array(list(dem_thalwegCond.sample((Point(segment.geometry).coords), indexes=1))).item()]
-                headwater_path = headwater_path + [segment.headwater_path]
-        # gpd.GeoDataFrame({**data, "source": "dem_m"})
-        dem_m_pts = gpd.GeoDataFrame({'stream_id': stream_ids, 'source': 'dem_m', 'elevation_m': dem_m_elev, 'headwater_path': headwater_path, 'index_count': index_count, 'geometry': split_points}, crs=path.crs, geometry='geometry')
-        dem_lat_thal_adj_pts = gpd.GeoDataFrame({'stream_id': stream_ids, 'source': 'dem_lat_thal_adj', 'elevation_m': dem_lat_thal_adj_elev, 'headwater_path': headwater_path, 'index_count': index_count, 'geometry': split_points}, crs=path.crs, geometry='geometry')
-        dem_thal_adj_pts = gpd.GeoDataFrame({'stream_id': stream_ids, 'source': 'thal_adj_dem', 'elevation_m': dem_thal_adj_elev, 'headwater_path': headwater_path, 'index_count': index_count, 'geometry': split_points}, crs=path.crs, geometry='geometry')
-        for raster in [dem_m_pts,dem_lat_thal_adj_pts,dem_thal_adj_pts]:
-            raster = raster.sort_values(by=['index_count'])
-            raster.set_index('index_count',inplace=True,drop=True)
-            raster = raster.reset_index(drop=True)
-            raster.index.names = ['index_count']
-            raster = raster.reset_index(drop=False)
-            thalweg_points = thalweg_points.append(raster,ignore_index = True)
-            del raster
-        del dem_m_pts,dem_lat_thal_adj_pts,dem_thal_adj_pts
-
-    del dem_lateral_thalweg_adj,dem_thalwegCond,dem_meters
-
-    try:
-        # Remove nodata_pts and convert elevation to ft
-        thalweg_points = thalweg_points.loc[thalweg_points.elevation_m > 0.0]
-        thalweg_points.elevation_m =  np.round(thalweg_points.elevation_m,3)
-        thalweg_points['elevation_ft'] =  np.round(thalweg_points.elevation_m*3.28084,3)
-
-        # Plot thalweg profile
-        plot_profile(thalweg_points, profile_plots_filename)
-
-        # Filter final thalweg ajdusted layer
-        thal_adj_points = thalweg_points.loc[thalweg_points.source=='thal_adj_dem'].copy()
-        # thal_adj_points.to_file(profile_gpkg_filename,driver=getDriver(profile_gpkg_filename))
-
-        # Identify significant rises/drops in elevation
-        thal_adj_points['elev_change'] = thal_adj_points.groupby(['headwater_path', 'source'])['elevation_m'].apply(lambda x: x - x.shift())
-        elev_changes = thal_adj_points.loc[(thal_adj_points.elev_change<=-lateral_elevation_threshold) | (thal_adj_points.elev_change>0.0)]
-
-        if not elev_changes.empty:
-            # elev_changes.to_csv(profile_table_filename,index=False)
-            elev_changes.to_file(profile_gpkg_filename,index=False,driver=getDriver(profile_gpkg_filename))
-
-
-        # Zoom in to plot only areas with steep elevation changes
-        # select_streams = elev_changes.stream_id.to_list()
-        # downstream_segments = [index + 1 for index in select_streams]
-        # upstream_segments = [index - 1 for index in select_streams]
-        # select_streams = list(set(upstream_segments + downstream_segments + select_streams))
-        # thal_adj_points_select = thal_adj_points.loc[thal_adj_points.stream_id.isin(select_streams)]
-        # plot_profile(thal_adj_points_select, profile_plots_filename_zoom)
-
-    except:
-        print(f"huc {huc} has {len(thalweg_points)} thalweg points")
-
-def get_downstream_segments(streams, headwater_col,headwater_id,flag_column,stream_id,stream_type):
-
-    streams[flag_column] = False
-    streams['downstream_count'] = -9
-    streams.loc[streams[headwater_col]==headwater_id,flag_column] = True
-    streams.loc[streams[headwater_col]==headwater_id,'downstream_count'] = 0
-    count = 0
-
-    Q = deque(streams.loc[streams[headwater_col]==headwater_id,stream_id].tolist())
-    visited = set()
-
-    while Q:
-        q = Q.popleft()
-
-        if q in visited:
-            continue
-
-        visited.add(q)
-
-        count = count + 1
-        if stream_type == 'burnline':
-
-            toNode,DnLevelPat = streams.loc[q,['ToNode','DnLevelPat']]
-            downstream_ids = streams.loc[streams['FromNode'] == toNode,:].index.tolist()
-
-            # If multiple downstream_ids are returned select the ids that are along the main flow path (i.e. exclude segments that are diversions)
-            if len(set(downstream_ids)) > 1: # special case: remove duplicate NHDPlusIDs
-
-                relevant_ids = [segment for segment in downstream_ids if DnLevelPat == streams.loc[segment,'LevelPathI']]
-
-            else:
-
-                relevant_ids = downstream_ids
-
-        elif stream_type == 'derived':
-
-            toNode = streams.loc[q,['NextDownID']].item()
-            relevant_ids = streams.loc[streams[stream_id] == toNode,:].index.tolist()
-
-        streams.loc[relevant_ids,flag_column] = True
-        streams.loc[relevant_ids,'downstream_count'] = count
-
-        for i in relevant_ids:
-
-            if i not in visited:
-                Q.append(i)
-
-    streams = streams.loc[streams[flag_column],:]
-
-    return streams
-
-
-def plot_profile(elevation_table,profile_plots_filename):
-    num_plots = len(elevation_table.headwater_path.unique())
-    unique_rasters = elevation_table.source.unique()
-    if num_plots > 3:
-        columns = int(np.ceil(num_plots / 3))
-    else:
-        columns = 1
-    # palette = dict(zip(unique_rasters, sns.color_palette(n_colors=len(unique_rasters))))
-    # palette.update({'dem_m':'gray'})
-    sns.set(style="ticks")
-    if len(unique_rasters) > 1:
-        g = sns.FacetGrid(elevation_table, col="headwater_path", hue="source", hue_order=['dem_m', 'dem_lat_thal_adj', 'thal_adj_dem'], sharex=False, sharey=False,col_wrap=columns)
-    else:
-        g = sns.FacetGrid(elevation_table, col="headwater_path", hue="source", sharex=False, sharey=False,col_wrap=columns)
-    g.map(sns.lineplot, "index_count", "elevation_ft", palette="tab20c")
-    g.set_axis_labels(x_var="Longitudinal Profile (index)", y_var="Elevation (ft)")
-    # Iterate thorugh each axis to get individual y-axis bounds
-    for ax in g.axes.flat:
-        mins = []
-        maxes = []
-        for line in ax.lines:
-            mins = mins + [min(line.get_ydata())]
-            maxes = maxes + [max(line.get_ydata())]
-        min_y = min(mins) - (max(maxes) - min(mins))/10
-        max_y = max(maxes) + (max(maxes) - min(mins))/10
-        ax.set_ylim(min_y,max_y)
-    # if len(unique_rasters) > 1:
-    #     ax.lines[0].set_linestyle("--")
-    #     ax.lines[0].set_color('gray')
-    # box = ax.get_position()
-    # ax.set_position([box.x0, box.y0 + box.height * 0.1,box.width, box.height * 0.9])
-    # Adjust the arrangement of the plots
-    # g.fig.tight_layout(w_pad=5) #w_pad=2
-    g.add_legend()
-    # plt.legend(bbox_to_anchor=(1.04,0.5), loc="center left", borderaxespad=0)
-    plt.subplots_adjust(bottom=0.25)
-    plt.savefig(profile_plots_filename)
-    plt.close()
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='generate rating curve plots and tables for FIM and USGS gages')
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-output_dir','--output-dir', help='rating curves output folder', required=True,type=str)
-    # parser.add_argument('-rasters','--raster-list',help='list of rasters to be evaluated',required=True,type=str)
-    parser.add_argument('-stream_type','--stream-type',help='stream layer to be evaluated',required=True,type=str,choices=['derived','burnline'])
-    parser.add_argument('-point_density','--point-density',help='elevation sampling density',required=True,type=str,choices=['midpoints','all_points'])
-    parser.add_argument('-th','--elevation_threshold',help='significant elevation drop threshold in meters.',required=True)
-    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-
-    args = vars(parser.parse_args())
-
-    fim_dir = args['fim_dir']
-    output_dir = args['output_dir']
-    # raster_list = args['raster_list']
-    stream_type = args['stream_type']
-    point_density = args['point_density']
-    number_of_jobs = args['number_of_jobs']
-
-    # dem_meters_dir = os.environ.get('dem_meters')
-
-    plots_dir = join(output_dir,'plots')
-    os.makedirs(plots_dir, exist_ok=True)
-    spatial_dir = os.path.join(output_dir,'spatial_layers')
-    os.makedirs(spatial_dir, exist_ok=True)
-
-    # Open log file
-    sys.__stdout__ = sys.stdout
-    log_file = open(join(output_dir,'thalweg_profile_comparison.log'),"w")
-    sys.stdout = log_file
-
-    procs_list = []
-    huc_list  = os.listdir(fim_dir)
-    for huc in huc_list:
-        if huc != 'logs':
-
-            huc_dir = os.path.join(fim_dir,huc)
-            flows_grid_boolean_filename = os.path.join(huc_dir,'flows_grid_boolean.tif')
-            dem_meters_filename = os.path.join(huc_dir,'dem_meters.tif')
-            dem_lateral_thalweg_adj_filename = os.path.join(huc_dir,'dem_lateral_thalweg_adj.tif')
-            dem_thalwegCond_filename = os.path.join(huc_dir,'dem_thalwegCond.tif')
-            profile_plots_filename = os.path.join(plots_dir,f"profile_drop_plots_{huc}_{point_density}_{stream_type}.png")
-            profile_gpkg_filename = os.path.join(spatial_dir,f"thalweg_elevation_changes_{huc}_{point_density}_{stream_type}.gpkg")
-            profile_table_filename = os.path.join(spatial_dir,f"thalweg_elevation_changes_{huc}_{point_density}_{stream_type}.csv")
-
-            procs_list.append([huc_dir,stream_type,point_density,huc,dem_meters_filename,dem_lateral_thalweg_adj_filename,dem_thalwegCond_filename,profile_plots_filename,profile_gpkg_filename,profile_table_filename,flows_grid_boolean_filename])
-
-    # Initiate multiprocessing
-    print(f"Generating thalweg elevation profiles for {len(procs_list)} hucs using {number_of_jobs} jobs")
-    with Pool(processes=number_of_jobs) as pool:
-        # Get elevation values along thalweg for each headwater stream path
-        pool.map(compare_thalweg, procs_list)
-
-    # Append all elevation change spatial layers to a single gpkg
-    spatial_list  = os.listdir(spatial_dir)
-    agg_thalweg_elevations_gpkg_fileName = os.path.join(output_dir, f"agg_thalweg_elevation_changes_{point_density}_{stream_type}.gpkg")
-    agg_thalweg_elevation_table_fileName = os.path.join(output_dir, f"agg_thalweg_elevation_changes_{point_density}_{stream_type}.csv")
-    for layer in spatial_list:
-
-        huc_gpd = gpd.read_file(os.path.join(spatial_dir,layer))
-
-        # Write aggregate layer
-        if os.path.isfile(agg_thalweg_elevations_gpkg_fileName):
-            huc_gpd.to_file(agg_thalweg_elevations_gpkg_fileName,driver=getDriver(agg_thalweg_elevations_gpkg_fileName),index=False, mode='a')
-        else:
-            huc_gpd.to_file(agg_thalweg_elevations_gpkg_fileName,driver=getDriver(agg_thalweg_elevations_gpkg_fileName),index=False)
-
-        del huc_gpd
-
-    # Create csv of elevation table
-    huc_table = gpd.read_file(agg_thalweg_elevations_gpkg_fileName)
-    huc_table.to_csv(agg_thalweg_elevation_table_fileName,index=False)
-
-    # Close log file
-    sys.stdout = sys.__stdout__
-    log_file.close()
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 307dac081..d2068f9b7 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -1210,7 +1210,7 @@ def get_rating_curve(rating_curve_url, location_ids):
                 curve_df['wrds_timestamp'] = response.headers['Date']
 
                 #Append rating curve to DataFrame containing all curves
-                all_curves = all_curves.append(curve_df)
+                all_curves = pd.concat([all_curves, curve_df])
             else:
                 continue
 

From 03ce7a384a5abdeb3531a34893309fee074d7ef6 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 19 May 2023 14:36:37 -0500
Subject: [PATCH 248/359] v4.3.11.2 Fix to address levee DEM burn skipping

---
 docs/CHANGELOG.md  | 9 +++++++++
 src/run_unit_wb.sh | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f8bfb999d..a3a05ad4b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.11.2 - 2023-05-19 - [PR#918](https://github.com/NOAA-OWP/inundation-mapping/pull/918)
+
+This fix addresses a bug that was preventing `burn_in_levees.py` from running. The if statement in run_unit_wb.sh preceeding `burn_in_levees.py` was checking for the existence of a filepath that doesn't exist.
+
+### Changes  
+- `src/run_unit_wb.sh`: fixed the if statement filepath to check for the presence of levee features to burn into the DEM
+
+<br/><br/>
+
 ## v4.3.11.1 - 2023-05-16 - [PR#904](https://github.com/NOAA-OWP/inundation-mapping/pull/904)
 
 `pandas.append` was deprecated in our last Pandas upgrade (v4.3.9.0). This PR updates the remaining instances of `pandas.append` to `pandas.concat`.
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 90f9f895b..f05e427bf 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -159,7 +159,7 @@ echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwr
 date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
-[ -f $tempCurrentBranchDataDir/nld_subset_levees.tif ] && \
+[ -f $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif ] && \
 python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $tempHucDataDir/dem_meters.tif -nld $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif -out $tempHucDataDir/dem_meters.tif
 Tcount
 

From c0b7f9a95f8e0547a2c006add85dc5ab700ae1d5 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 26 May 2023 10:44:42 -0500
Subject: [PATCH 249/359] v4.3.11.3 Fixes append statement in CatFIM

---
 CITATION.cff                            |  2 +-
 docs/CHANGELOG.md                       | 10 ++++++++++
 tools/generate_categorical_fim_flows.py |  2 +-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 893916853..4c7b2d356 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.3.11.1
+version: 4.3.11.3
 date-released: 2023
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a3a05ad4b..a53a3639b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.11.3 - 2023-05-25 - [PR#920](https://github.com/NOAA-OWP/inundation-mapping/pull/920)
+
+Fixes a bug in CatFIM script where a bracket was missing on a pandas `concat` statement.
+
+### Changes  
+- `/tools/generate_categorical_fim.py`: fixes `concat` statement where bracket was missing.
+
+
+<br/><br/>
+
 ## v4.3.11.2 - 2023-05-19 - [PR#918](https://github.com/NOAA-OWP/inundation-mapping/pull/918)
 
 This fix addresses a bug that was preventing `burn_in_levees.py` from running. The if statement in run_unit_wb.sh preceeding `burn_in_levees.py` was checking for the existence of a filepath that doesn't exist.
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index 2716e0647..1acf87449 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -241,7 +241,7 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
         full_csv_path = os.path.join(attributes_dir, csv)
         # Huc has to be read in as string to preserve leading zeros.
         temp_df = pd.read_csv(full_csv_path, dtype={'huc':str})
-        all_csv_df = pd.concat(all_csv_df, temp_df], ignore_index = True)
+        all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index = True)
     # Write to file
     all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index = False)
    

From 8f4a209dd3ab1b1154c7db918a536b3d4aa4aa74 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 26 May 2023 10:48:03 -0500
Subject: [PATCH 250/359] v4.3.11.4 Simplified script to push up file to HV

---
 CITATION.cff                           |   2 +-
 data/aws/push-hv-data-support-files.sh | 169 +++++++++++++++++++++++++
 docs/CHANGELOG.md                      |  13 ++
 3 files changed, 183 insertions(+), 1 deletion(-)
 create mode 100644 data/aws/push-hv-data-support-files.sh

diff --git a/CITATION.cff b/CITATION.cff
index 4c7b2d356..45601e54c 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.3.11.3
+version: 4.3.11.4
 date-released: 2023
diff --git a/data/aws/push-hv-data-support-files.sh b/data/aws/push-hv-data-support-files.sh
new file mode 100644
index 000000000..6f4e84e69
--- /dev/null
+++ b/data/aws/push-hv-data-support-files.sh
@@ -0,0 +1,169 @@
+#!/bin/bash -e
+
+#This files pushs all support files required for a HV production release,
+
+#When running this script, ensure the docker container is mounted to the dev_fim_share folders
+
+#Note: This files is rather unpolished and as it is Git, it must not included the
+#base HV S3 folder name or AWS credentials.
+
+usage ()
+{
+    echo 'Copying key FIM production run output files to HV.'
+    echo 'Does not included the filtered HAND files from FIM.'
+    echo 'Does included key files such as catfim, fim performance and sierra test files.'
+    echo 'Note: Please look at the code as there are assumed directories and pathing.'
+    echo '      Also.. for now, please pre-run the aws configure script for creds.'    
+    echo
+    echo 'Usage : push-hv-data-support-files.sh [REQ: -n <run name> ]'
+    echo '                                      [REQ: -hv <HV S3 bucket name>'
+    echo ''
+    echo 'REQUIRED:'
+    echo '  -n/--runName       : A name of HAND output release.'
+    echo '  -hv/--HV_S3_bucket : The root HV S3 bucket name. ie) s3://{hv....}' 
+    echo ''
+    echo 'OPTIONS:'
+    echo '  -h/--help       : help file'
+    echo
+    exit
+}
+
+while [ "$1" != "" ]; do
+case $1
+in
+    -n|--runName)
+        shift
+        runName=$1
+        ;;
+    -hv|--HV_S3_bucket)
+        shift
+        HV_S3_bucket=$1
+        ;;
+    -h|--help)
+        shift
+        usage
+        ;;
+    *) ;;
+    esac
+    shift
+done
+
+# print usage if arguments empty
+if [ "$runName" = "" ]
+then
+    echo "ERROR: Missing -n  -- run time name (fim version / folder name) argument"
+    usage
+fi
+
+if [ "$HV_S3_bucket" = "" ]
+then
+    echo "ERROR: Missing -hv  -- the HV s3 bucket name (ie: s3://{hv...})"
+    usage
+fi
+
+# FOR NOW, run the aws configure script by hand outside this tool
+read -p "Did you run aws configure and load creds already (y/n): " resp
+
+case $resp in
+    y ) echo ok, we wil proceed;;
+    n ) echo exiting...;
+         exit;;
+    * ) echo invalid response;
+         exit 1;;
+
+esac
+
+echo "Loading files to $HV_S3_bucket"
+
+echo "Copying catfim files"
+
+aws_cmd_cp="aws s3 cp"
+
+# ----------------------
+# Catfim Stage Based
+aws_src="/data/catfim/"$runName"_stage_based/mapping/stage_based_catfim.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+
+aws_src="/data/catfim/"$runName"_stage_based/mapping/stage_based_catfim_sites.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+# ----------------------
+# Catfim Flow Based
+aws_src="/data/catfim/"$runName"_flow_based/mapping/flow_based_catfim.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+
+aws_src="/data/catfim/"$runName"_flow_based/mapping/flow_based_catfim_sites.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+# ----------------------
+# Fim Performance
+aws_src="/data/fim_performance/"$runName"/fim_performance_polys.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+
+aws_src="/data/fim_performance/"$runName"/fim_performance_points.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+
+aws_src="/data/fim_performance/"$runName"/fim_performance_catchments.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+# ----------------------
+# Sierra Test (Rating Curve Comparison Metrics)
+aws_src="/data/fim_performance/"$runName"/rating_curve_comparison/agg_nwm_recurr_flow_elev_stats_location_id.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+
+aws_src="/data/inputs/usgs_gages/usgs_rating_curves.csv"
+aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
+
+aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
+echo "command is '"$aws_cmd"'"
+eval "${aws_cmd}"
+echo 
+
+
+echo "done coping files and folders to S3"
+echo
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a53a3639b..05c2bf31b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.11.4 - 2023-05-18 - [PR#917](https://github.com/NOAA-OWP/inundation-mapping/pull/917)
+
+There is a growing number of files that need to be pushed up to HydroVis S3 during a production release, counting the new addition of rating curve comparison reports.
+
+Earlier, we were running a number of aws cli scripts one at a time. This tool simplies it and pushes all of the QA and supporting files. Note: the HAND files from a release, will continue to be pushed by `/data/aws/s3.py` as it filters out files to be sent to HV s3.
+
+### Additions  
+
+- `data\aws`
+     - `push-hv-data-support-files.sh`: As described above. See file for command args.
+
+<br/><br/>
+
 
 ## v4.3.11.3 - 2023-05-25 - [PR#920](https://github.com/NOAA-OWP/inundation-mapping/pull/920)
 

From 41998f4fe0a06e934d4b9fd6ed5950af80a6a86f Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 2 Jun 2023 10:53:52 -0500
Subject: [PATCH 251/359] v4.3.11.5 Hotfix for synthesize metrics using PREV
 versions

---
 docs/CHANGELOG.md              | 10 ++++++
 tools/synthesize_test_cases.py | 66 +++++++++++++++++-----------------
 2 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 05c2bf31b..9590ceabb 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.11.5 - 2023-05-30 - [PR#911](https://github.com/NOAA-OWP/inundation-mapping/pull/911)
+
+This fix addresses bugs found when using the recently added functionality in `tools/synthesize_test_cases.py` along with the `PREV` argument. The `-pfiles` argument now performs as expected for both `DEV` and `PREV` processing. Addresses #871 
+
+### Changes  
+`tools/synthesize_test_cases.py`: multiple changes to enable all expected functionality with the `-pfiles` and `-pcsv` arguments
+
+
+<br/><br/>
+
 ## v4.3.11.4 - 2023-05-18 - [PR#917](https://github.com/NOAA-OWP/inundation-mapping/pull/917)
 
 There is a growing number of files that need to be pushed up to HydroVis S3 during a production release, counting the new addition of rating curve comparison reports.
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index fe788871a..cc8de54a3 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -10,7 +10,7 @@
 from run_test_case import test_case
 from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT
 
-def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list, pfiles, prev_metrics_csv):
+def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list, prev_versions_to_include_list, iteration_list, pfiles, prev_metrics_csv):
     """
     This function searches for and collates metrics into a single CSV file that can queried database-style. The
     CSV is an input to eval_plots.py. This function automatically looks for metrics produced for official versions
@@ -74,16 +74,10 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
     list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
 
-    versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
-
-    # Specify which results to iterate through
-    if len(dev_versions_to_include_list) > 0:
-        iteration_list = ['official', 'comparison'] # iterating through official model results AND comparison model(s)
-    else:
-        iteration_list = ['official'] # only iterating through official model results
-
-    if pfiles != True: 
-        iteration_list = ['comparison'] # not iterating through official model results, just the comparison model(s) 
+    # add in composite of versions (used for previous FIM3 versions)
+    if "official" in iteration_list:
+        composite_versions = [v.replace('_ms', '_comp') for v in prev_versions_to_include_list if '_ms' in v]
+        prev_versions_to_include_list += composite_versions
 
     # Iterate through 5 benchmark sources 
     for benchmark_source in ['ble', 'nws', 'usgs', 'ifc','ras2fim']:
@@ -104,12 +98,9 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                     for iteration in iteration_list:
                         if iteration == "official": # and str(pfiles) == "True": # "official" refers to previous finalized model versions
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-                            versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
-                            # add in composite of versions
-                            composite_versions = [v.replace('_ms', '_comp') for v in versions_to_aggregate if '_ms' in v]
-                            versions_to_aggregate += composite_versions
-
-                        if iteration == "comparison": # "comparison" refers to the development model version(s) being evaluated
+                            versions_to_aggregate = prev_versions_to_include_list
+                            
+                        if iteration == "testing": # "testing" refers to the development model version(s) being evaluated
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = dev_versions_to_include_list
 
@@ -164,13 +155,11 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
                     # Update filepaths based on whether the official or dev versions should be included
                     for iteration in iteration_list:
-                        if iteration == "official" and str(pfiles) == "True": # "official" refers to previous finalized model versions
+                        if iteration == "official": # "official" refers to previous finalized model versions
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
-                            versions_to_aggregate = os.listdir(PREVIOUS_FIM_DIR)
-                            # add in composite of versions
-                            composite_versions = [v.replace('_ms', '_comp') for v in versions_to_aggregate if '_ms' in v]
-                            versions_to_aggregate += composite_versions
-                        if iteration == "comparison": # "comparison" refers to the development model version(s) being evaluated
+                            versions_to_aggregate = prev_versions_to_include_list
+
+                        if iteration == "testing": # "testing" refers to the development model version(s) being evaluated
                             versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
                             versions_to_aggregate = dev_versions_to_include_list
 
@@ -303,7 +292,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
     parser.add_argument('-c','--config', help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"', 
-                        required=False, default='DEV')
+                        required=True, default='DEV')
     parser.add_argument('-l','--calibrated', help='Denotes use of calibrated n values. This should be taken from meta-data from hydrofabric dir', 
                         required=False, default=False, action='store_true')
     parser.add_argument('-e','--model', help='Denotes model used. FR, MS, or GMS allowed. This should be taken from meta-data in hydrofabric dir.', 
@@ -365,13 +354,17 @@ def progress_bar_handler(executor_dict, verbose, desc):
                         )
 
     # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
-    if fim_version != "all":
-        previous_fim_list = [fim_version]
-    else:
+    prev_versions_to_include_list = []
+    dev_versions_to_include_list = []
+    if fim_version != "all" and pfiles == False:
         if config == 'PREV': # official fim model results
-            previous_fim_list = os.listdir(PREVIOUS_FIM_DIR)
+            prev_versions_to_include_list = [fim_version]
         elif config == 'DEV': # development fim model results
-            previous_fim_list = os.listdir(OUTPUTS_DIR)
+            dev_versions_to_include_list = [fim_version]
+    else:
+        prev_versions_to_include_list = os.listdir(PREVIOUS_FIM_DIR)
+        if config == 'DEV': # development fim model results
+            dev_versions_to_include_list = [fim_version]
 
     # Define whether or not to archive metrics in "official_versions" or "testing_versions" for each test_id.
     if config == 'PREV':
@@ -399,7 +392,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
             print(f"Metrics will be combined with previous metric CSV: {prev_metrics_csv}") 
             print()
     else:
-        print("ALERT: A previous metric CSV has not been provided (-pcsv).")
+        print("ALERT: A previous metric CSV has not been provided (-pcsv) - this is optional.")
         print()
 
     # Print whether the previous files will be cycled through
@@ -407,7 +400,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
         print("ALERT: Metrics from previous directories will be compiled.")
         print()
     else:
-        print("ALERT: Metrics from previous directories will NOT be compiled. (-pfiles)")
+        print("ALERT: Metrics from previous directories will NOT be compiled (-pfiles not provided) - this is optional")
         print()
 
     # Set up multiprocessor
@@ -496,10 +489,15 @@ def progress_bar_handler(executor_dict, verbose, desc):
             # Send the executor to the progress bar
             progress_bar_handler(executor_dict, verbose, f"Compositing test cases with {job_number_huc} workers")
 
+    ## if using DEV version, include the testing versions the user included with the "-dc" flag
     if dev_versions_to_compare != None:
-        dev_versions_to_include_list = dev_versions_to_compare + previous_fim_list
+        dev_versions_to_include_list += dev_versions_to_compare
+    
+    # Specify which results to iterate through
+    if config == 'DEV':
+        iteration_list = ['official', 'testing'] # iterating through official model results AND testing model(s)
     else:
-        dev_versions_to_include_list = previous_fim_list
+        iteration_list = ['official'] # only iterating through official model results
 
     if master_metrics_csv is not None:
         # Do aggregate_metrics.
@@ -508,6 +506,8 @@ def progress_bar_handler(executor_dict, verbose, desc):
         # Note: This function is not compatible with GMS
         create_master_metrics_csv(master_metrics_csv_output = master_metrics_csv, 
                                   dev_versions_to_include_list = dev_versions_to_include_list,
+                                  prev_versions_to_include_list = prev_versions_to_include_list,
+                                  iteration_list = iteration_list,
                                   pfiles=pfiles, prev_metrics_csv=prev_metrics_csv)
     
     print("================================")

From c252c782f522dc35dda2b635c814497095d3f166 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 2 Jun 2023 10:59:13 -0500
Subject: [PATCH 252/359] v4.3.11.6 Bump requests from 2.28.2 to 2.31.0

---
 CITATION.cff      |   2 +-
 Pipfile.lock      | 563 +++++++++++++++++++++++-----------------------
 docs/CHANGELOG.md |  11 +-
 3 files changed, 293 insertions(+), 283 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 45601e54c..802d32183 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.3.11.4
+version: 4.3.11.6
 date-released: 2023
diff --git a/Pipfile.lock b/Pipfile.lock
index 738f09b88..0467c9a7c 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -34,11 +34,11 @@
         },
         "aiosqlite": {
             "hashes": [
-                "sha256:c3511b841e3a2c5614900ba1d179f366826857586f78abd75e7cbeb88e75a557",
-                "sha256:faa843ef5fb08bafe9a9b3859012d3d9d6f77ce3637899de20606b7fc39aa213"
+                "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d",
+                "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.18.0"
+            "version": "==0.19.0"
         },
         "anyio": {
             "hashes": [
@@ -100,11 +100,11 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836",
-                "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"
+                "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
+                "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==22.2.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==23.1.0"
         },
         "babel": {
             "hashes": [
@@ -161,7 +161,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4'",
+            "markers": "python_version < '4' and python_full_version >= '3.8.0'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -174,11 +174,11 @@
         },
         "botocore": {
             "hashes": [
-                "sha256:2e449525f0ccedb31fdb962a77caac48b4c486c23515b84c5989a39a1823a024",
-                "sha256:cf43dddb7e2ba5425fe19fad68b10043307b61d9103d06566f1ab6034e38b8db"
+                "sha256:31edc237088c104f7a05887646bbec31d7459dd2e108fd90cbffa315902817e2",
+                "sha256:3d145f30d10a9c712acee48e7ce906c9456bb25fe50d477c9312c702ccfa50d1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.29.109"
+            "version": "==1.29.138"
         },
         "brotli": {
             "hashes": [
@@ -270,11 +270,11 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
-                "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
+                "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7",
+                "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2022.12.7"
+            "version": "==2023.5.7"
         },
         "cffi": {
             "hashes": [
@@ -542,7 +542,7 @@
                 "sha256:fc1464c97579da9f3ab16763c32e5c5d5bb5fa1ec7ce509a4ca6108b61b84fab",
                 "sha256:fd7dc0e6812b799a34f6d12fcb1000539098c249c8da54f3566c6a6461d0dbad"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==1.0.7"
         },
         "cycler": {
@@ -656,10 +656,10 @@
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:04fbecc94300436f628517b05741b7ea009506ce8f946d40996567c669318490",
-                "sha256:4a30d6315a68c253cfa8f963b9697246315aa3db89f98b97235e345dedfb0b8e"
+                "sha256:4b90b252628ca695280924d863fe37234eebadc29c5360d322571233dc9746e0",
+                "sha256:f4eeb8a77cef54861dbf7424ac8ce71306f12cbb086c45131bcba2c6a4f726e3"
             ],
-            "version": "==2.16.3"
+            "version": "==2.17.1"
         },
         "fiona": {
             "hashes": [
@@ -688,11 +688,11 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:64c0c05c337f826183637570ac5ab49ee220eec66cf50248e8df527edfa95aeb",
-                "sha256:9234b9f57b74e31b192c3fc32ef1a40750a8fbc1cd9837a7b7bfc4ca4a5c51d7"
+                "sha256:106caf6167c4597556b31a8d9175a3fdc0356fdcd70ab19973c3b0d4c893c461",
+                "sha256:dba8d7cdb8e2bac1b3da28c5ed5960de09e59a2fe7e63bb73f5a59e57b0430d2"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==4.39.3"
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==4.39.4"
         },
         "fqdn": {
             "hashes": [
@@ -719,11 +719,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:23c2bcae4762dfb0bbe072d358faec24957901d75b6c4ab11172c0c982532402",
-                "sha256:8f8bd2af397cf33bd344d35cfe7f489219b7d14fc79a3f854b75b8417e9226b0"
+                "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed",
+                "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==6.3.0"
+            "version": "==6.6.0"
         },
         "importlib-resources": {
             "hashes": [
@@ -818,11 +818,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:1ae6047c1277508933078163721bbb479c3e7292778a04b4bacf0874550977d6",
-                "sha256:302558b81f1bc22dc259fb2a0c5c7cf2f4c0bdb21b50484348f7bafe7fb71421"
+                "sha256:1aba0ae8453e15e9bc6b24e497ef6840114afcdb832ae597f32137fa19d42a6f",
+                "sha256:77aeffab056c21d16f1edccdc9e5ccbf7d96eb401bd6703610a21be8b068aadc"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==6.22.0"
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==6.23.1"
         },
         "ipympl": {
             "hashes": [
@@ -834,11 +834,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:1c183bf61b148b00bcebfa5d9b39312733ae97f6dad90d7e9b4d86c8647f498c",
-                "sha256:a950236df04ad75b5bc7f816f9af3d74dc118fd42f2ff7e80e8e60ca1f182e2d"
+                "sha256:c7b80eb7f5a855a88efc971fda506ff7a91c280b42cdae26643e0f601ea281ea",
+                "sha256:ea8801f15dfe4ffb76dea1b09b847430ffd70d827b41735c64a0638a04103bfc"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==8.12.0"
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==8.12.2"
         },
         "ipython-genutils": {
             "hashes": [
@@ -888,10 +888,10 @@
         },
         "json5": {
             "hashes": [
-                "sha256:1aa54b80b5e507dfe31d12b7743a642e2ffa6f70bf73b8e3d7d1d5fba83d99bd",
-                "sha256:4f1e196acc55b83985a51318489f345963c7ba84aa37607e49073066c562e99b"
+                "sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f",
+                "sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02"
             ],
-            "version": "==0.9.11"
+            "version": "==0.9.14"
         },
         "jsonpointer": {
             "hashes": [
@@ -922,11 +922,11 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:3fbab64100a0dcac7701b1e0f1a4412f1ccb45546ff2ad9bc4fcbe4e19804811",
-                "sha256:d5b8e739d7816944be50f81121a109788a3d92732ecf1ad1e4dadebc948818fe"
+                "sha256:9fe233834edd0e6c0aa5f05ca2ab4bdea1842bfd2d8a932878212fc5301ddaf0",
+                "sha256:b18219aa695d39e2ad570533e0d71fb7881d35a873051054a84ee2a17c4b7389"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==8.1.0"
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==8.2.0"
         },
         "jupyter-console": {
             "hashes": [
@@ -941,7 +941,7 @@
                 "sha256:6db75be0c83edbf1b7c9f91ec266a9a24ef945da630f3120e1a0046dc13713fc",
                 "sha256:d4201af84559bc8c70cead287e1ab94aeef3c512848dde077b7684b54d67730d"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==5.3.0"
         },
         "jupyter-events": {
@@ -957,7 +957,7 @@
                 "sha256:9fde612791f716fd34d610cd939704a9639643744751ba66e7ee8fdc9cead07e",
                 "sha256:e6bc1e9e96d7c55b9ce9699ff6cb9a910581fe7349e27c40389acb67632e24c0"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==2.5.0"
         },
         "jupyter-server-fileid": {
@@ -973,7 +973,7 @@
                 "sha256:57ab779797c25a7ba68e97bcfb5d7740f2b5e8a83b5e8102b10438041a7eac5d",
                 "sha256:75779164661cec02a8758a5311e18bb8eb70c4e86c6b699403100f1585a12a36"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==0.4.4"
         },
         "jupyter-server-ydoc": {
@@ -986,11 +986,11 @@
         },
         "jupyter-ydoc": {
             "hashes": [
-                "sha256:3ac51abfe378c6aeb62a449e8f0241bede1205f0199b0d27429140cbba950f79",
-                "sha256:98db7785215873c64d7dfcb1b741f41df11994c4b3d7e2957e004b392d6f11ea"
+                "sha256:a3f670a69135e90493ffb91d6788efe2632bf42c6cc42a25f25c2e6eddd55a0e",
+                "sha256:d1a51c73ead6f6417bec0450f53c577a66abe8d43e9c2d8a1acaf7a17259f843"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.2.3"
+            "version": "==0.2.4"
         },
         "jupyterlab": {
             "hashes": [
@@ -1010,11 +1010,11 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:0f9f6752b0c534a7b22a6542b984fa6a2c18ab4d4e0a4c79f191138506a9a75f",
-                "sha256:f4a7263ada89958854631a64bed45285caeac482925233159709f643c5871490"
+                "sha256:1c8eb55c7cd70a50a51fef42a7b4e26ef2f7fc48728f0290604bd89b1dd156e6",
+                "sha256:dfaaf898af84b9d01ae9583b813f378b96ee90c3a66f24c5186ea5d1bbdb2089"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.22.0"
+            "version": "==2.22.1"
         },
         "jupyterlab-widgets": {
             "hashes": [
@@ -1232,7 +1232,7 @@
                 "sha256:f883a22a56a84dba3b588696a2b8a1ab0d2c3d41be53264115c71b0a942d8fdb",
                 "sha256:fbdeeb58c0cf0595efe89c05c224e0a502d1aa6a8696e68a73c3efc6bc354304"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==3.7.1"
         },
         "matplotlib-inline": {
@@ -1336,10 +1336,11 @@
         },
         "munch": {
             "hashes": [
-                "sha256:2d735f6f24d4dba3417fa448cae40c6e896ec1fdab6cdb5e6510999758a4dbd2",
-                "sha256:6f44af89a2ce4ed04ff8de41f70b226b984db10a91dcc7b9ac2efc1c77022fdd"
+                "sha256:0e4108418cfea898dcad01ff9569c30ff58f01d6f699331d04364f51623627c0",
+                "sha256:5284603030c00906d9d64d8108728c004fbeb91fc1c1e4caca342bc48f2a6dfd"
             ],
-            "version": "==2.5.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.0"
         },
         "natsort": {
             "hashes": [
@@ -1351,27 +1352,27 @@
         },
         "nbclassic": {
             "hashes": [
-                "sha256:47791b04dbcb89bf7fde910a3d848fd4793a4248a8936202453631a87da37d51",
-                "sha256:d2c91adc7909b0270c73e3e253d3687a6704b4f0a94bc156a37c85eba09f4d37"
+                "sha256:0ae11eb2319455d805596bf320336cda9554b41d99ab9a3c31bf8180bffa30e3",
+                "sha256:f99e4769b4750076cd4235c044b61232110733322384a94a63791d2e7beacc66"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.5.5"
+            "version": "==1.0.0"
         },
         "nbclient": {
             "hashes": [
-                "sha256:26e41c6dca4d76701988bc34f64e1bfc2413ae6d368f13d7b5ac407efb08c755",
-                "sha256:8fa96f7e36693d5e83408f5e840f113c14a45c279befe609904dbe05dad646d1"
+                "sha256:25e861299e5303a0477568557c4045eccc7a34c17fc08e7959558707b9ebe548",
+                "sha256:f9b179cd4b2d7bca965f900a2ebf0db4a12ebff2f36a711cb66861e4ae158e55"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.7.3"
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==0.8.0"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:78685362b11d2e8058e70196fe83b09abed8df22d3e599cf271f4d39fdc48b9e",
-                "sha256:d2e95904666f1ff77d36105b9de4e0801726f93b862d5b28f69e93d99ad3b19c"
+                "sha256:51b6c77b507b177b73f6729dba15676e42c4e92bcb00edc8cc982ee72e7d89d7",
+                "sha256:af5064a9db524f9f12f4e8be7f0799524bd5b14c1adea37e34e83c95127cc818"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.3.1"
+            "version": "==7.4.0"
         },
         "nbformat": {
             "hashes": [
@@ -1432,11 +1433,11 @@
         },
         "notebook-shim": {
             "hashes": [
-                "sha256:090e0baf9a5582ff59b607af523ca2db68ff216da0c69956b62cab2ef4fc9c3f",
-                "sha256:9c6c30f74c4fbea6fce55c1be58e7fd0409b1c681b075dcedceb005db5026949"
+                "sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7",
+                "sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.2.2"
+            "version": "==0.2.3"
         },
         "numba": {
             "hashes": [
@@ -1544,11 +1545,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2",
-                "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"
+                "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
+                "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.0"
+            "version": "==23.1"
         },
         "pandas": {
             "hashes": [
@@ -1694,11 +1695,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08",
-                "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"
+                "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f",
+                "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.2.0"
+            "version": "==3.5.1"
         },
         "pluggy": {
             "hashes": [
@@ -1726,23 +1727,23 @@
         },
         "psutil": {
             "hashes": [
-                "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff",
-                "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1",
-                "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62",
-                "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549",
-                "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08",
-                "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7",
-                "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e",
-                "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe",
-                "sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24",
-                "sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad",
-                "sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94",
-                "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8",
-                "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7",
-                "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.4"
+                "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d",
+                "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217",
+                "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4",
+                "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c",
+                "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f",
+                "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da",
+                "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4",
+                "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42",
+                "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5",
+                "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4",
+                "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9",
+                "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f",
+                "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
+                "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
+            ],
+            "markers": "sys_platform != 'cygwin'",
+            "version": "==5.9.5"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -1966,50 +1967,49 @@
         },
         "pycryptodomex": {
             "hashes": [
-                "sha256:0af93aad8d62e810247beedef0261c148790c52f3cd33643791cc6396dd217c1",
-                "sha256:12056c38e49d972f9c553a3d598425f8a1c1d35b2e4330f89d5ff1ffb70de041",
-                "sha256:23d83b610bd97704f0cd3acc48d99b76a15c8c1540d8665c94d514a49905bad7",
-                "sha256:2d4d395f109faba34067a08de36304e846c791808524614c731431ee048fe70a",
-                "sha256:32e764322e902bbfac49ca1446604d2839381bbbdd5a57920c9daaf2e0b778df",
-                "sha256:3c2516b42437ae6c7a29ef3ddc73c8d4714e7b6df995b76be4695bbe4b3b5cd2",
-                "sha256:40e8a11f578bd0851b02719c862d55d3ee18d906c8b68a9c09f8c564d6bb5b92",
-                "sha256:4b51e826f0a04d832eda0790bbd0665d9bfe73e5a4d8ea93b6a9b38beeebe935",
-                "sha256:4c4674f4b040321055c596aac926d12f7f6859dfe98cd12f4d9453b43ab6adc8",
-                "sha256:55eed98b4150a744920597c81b3965b632038781bab8a08a12ea1d004213c600",
-                "sha256:599bb4ae4bbd614ca05f49bd4e672b7a250b80b13ae1238f05fd0f09d87ed80a",
-                "sha256:5c23482860302d0d9883404eaaa54b0615eefa5274f70529703e2c43cc571827",
-                "sha256:64b876d57cb894b31056ad8dd6a6ae1099b117ae07a3d39707221133490e5715",
-                "sha256:67a3648025e4ddb72d43addab764336ba2e670c8377dba5dd752e42285440d31",
-                "sha256:6feedf4b0e36b395329b4186a805f60f900129cdf0170e120ecabbfcb763995d",
-                "sha256:78f0ddd4adc64baa39b416f3637aaf99f45acb0bcdc16706f0cc7ebfc6f10109",
-                "sha256:7a6651a07f67c28b6e978d63aa3a3fccea0feefed9a8453af3f7421a758461b7",
-                "sha256:7a8dc3ee7a99aae202a4db52de5a08aa4d01831eb403c4d21da04ec2f79810db",
-                "sha256:7cc28dd33f1f3662d6da28ead4f9891035f63f49d30267d3b41194c8778997c8",
-                "sha256:7fa0b52df90343fafe319257b31d909be1d2e8852277fb0376ba89d26d2921db",
-                "sha256:88b0d5bb87eaf2a31e8a759302b89cf30c97f2f8ca7d83b8c9208abe8acb447a",
-                "sha256:a4fa037078e92c7cc49f6789a8bac3de06856740bb2038d05f2d9a2e4b165d59",
-                "sha256:a57e3257bacd719769110f1f70dd901c5b6955e9596ad403af11a3e6e7e3311c",
-                "sha256:ab33c2d9f275e05e235dbca1063753b5346af4a5cac34a51fa0da0d4edfb21d7",
-                "sha256:c84689c73358dfc23f9fdcff2cb9e7856e65e2ce3b5ed8ff630d4c9bdeb1867b",
-                "sha256:c92537b596bd5bffb82f8964cabb9fef1bca8a28a9e0a69ffd3ec92a4a7ad41b",
-                "sha256:caa937ff29d07a665dfcfd7a84f0d4207b2ebf483362fa9054041d67fdfacc20",
-                "sha256:d38ab9e53b1c09608ba2d9b8b888f1e75d6f66e2787e437adb1fecbffec6b112",
-                "sha256:d4cf0128da167562c49b0e034f09e9cedd733997354f2314837c2fa461c87bb1",
-                "sha256:db23d7341e21b273d2440ec6faf6c8b1ca95c8894da612e165be0b89a8688340",
-                "sha256:ee8bf4fdcad7d66beb744957db8717afc12d176e3fd9c5d106835133881a049b",
-                "sha256:f854c8476512cebe6a8681cc4789e4fcff6019c17baa0fd72b459155dc605ab4",
-                "sha256:fd29d35ac80755e5c0a99d96b44fb9abbd7e871849581ea6a4cb826d24267537"
+                "sha256:160a39a708c36fa0b168ab79386dede588e62aec06eb505add870739329aecc6",
+                "sha256:192306cf881fe3467dda0e174a4f47bb3a8bb24b90c9cdfbdc248eec5fc0578c",
+                "sha256:1949e09ea49b09c36d11a951b16ff2a05a0ffe969dda1846e4686ee342fe8646",
+                "sha256:215be2980a6b70704c10796dd7003eb4390e7be138ac6fb8344bf47e71a8d470",
+                "sha256:27072a494ce621cc7a9096bbf60ed66826bb94db24b49b7359509e7951033e74",
+                "sha256:2dc4eab20f4f04a2d00220fdc9258717b82d31913552e766d5f00282c031b70a",
+                "sha256:302a8f37c224e7b5d72017d462a2be058e28f7be627bdd854066e16722d0fc0c",
+                "sha256:3d9314ac785a5b75d5aaf924c5f21d6ca7e8df442e5cf4f0fefad4f6e284d422",
+                "sha256:3e3ecb5fe979e7c1bb0027e518340acf7ee60415d79295e5251d13c68dde576e",
+                "sha256:4d9379c684efea80fdab02a3eb0169372bca7db13f9332cb67483b8dc8b67c37",
+                "sha256:50308fcdbf8345e5ec224a5502b4215178bdb5e95456ead8ab1a69ffd94779cb",
+                "sha256:5594a125dae30d60e94f37797fc67ce3c744522de7992c7c360d02fdb34918f8",
+                "sha256:58fc0aceb9c961b9897facec9da24c6a94c5db04597ec832060f53d4d6a07196",
+                "sha256:6421d23d6a648e83ba2670a352bcd978542dad86829209f59d17a3f087f4afef",
+                "sha256:6875eb8666f68ddbd39097867325bd22771f595b4e2b0149739b5623c8bf899b",
+                "sha256:6ed3606832987018615f68e8ed716a7065c09a0fe94afd7c9ca1b6777f0ac6eb",
+                "sha256:71687eed47df7e965f6e0bf3cadef98f368d5221f0fb89d2132effe1a3e6a194",
+                "sha256:73d64b32d84cf48d9ec62106aa277dbe99ab5fbfd38c5100bc7bddd3beb569f7",
+                "sha256:75672205148bdea34669173366df005dbd52be05115e919551ee97171083423d",
+                "sha256:76f0a46bee539dae4b3dfe37216f678769349576b0080fdbe431d19a02da42ff",
+                "sha256:8ff129a5a0eb5ff16e45ca4fa70a6051da7f3de303c33b259063c19be0c43d35",
+                "sha256:ac614363a86cc53d8ba44b6c469831d1555947e69ab3276ae8d6edc219f570f7",
+                "sha256:ba95abd563b0d1b88401658665a260852a8e6c647026ee6a0a65589287681df8",
+                "sha256:bbdcce0a226d9205560a5936b05208c709b01d493ed8307792075dedfaaffa5f",
+                "sha256:bec6c80994d4e7a38312072f89458903b65ec99bed2d65aa4de96d997a53ea7a",
+                "sha256:c2953afebf282a444c51bf4effe751706b4d0d63d7ca2cc51db21f902aa5b84e",
+                "sha256:d35a8ffdc8b05e4b353ba281217c8437f02c57d7233363824e9d794cf753c419",
+                "sha256:d56c9ec41258fd3734db9f5e4d2faeabe48644ba9ca23b18e1839b3bdf093222",
+                "sha256:d84e105787f5e5d36ec6a581ff37a1048d12e638688074b2a00bcf402f9aa1c2",
+                "sha256:e00a4bacb83a2627e8210cb353a2e31f04befc1155db2976e5e239dd66482278",
+                "sha256:f237278836dda412a325e9340ba2e6a84cb0f56b9244781e5b61f10b3905de88",
+                "sha256:f9ab5ef0718f6a8716695dea16d83b671b22c45e9c0c78fd807c32c0192e54b5"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==3.17"
+            "version": "==3.18.0"
         },
         "pygments": {
             "hashes": [
-                "sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094",
-                "sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500"
+                "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
+                "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.15.0"
+            "version": "==2.15.1"
         },
         "pyparsing": {
             "hashes": [
@@ -2347,122 +2347,122 @@
         },
         "pyzstd": {
             "hashes": [
-                "sha256:033e1235438ebb9e02f33f6e9632e1cf74df340a2412704b93409ced861d265a",
-                "sha256:035f2d2c3ef28012e31a719337d5bb2bae20e9fcda0bf291cc933342bbaf97c9",
-                "sha256:061c7a9f653a5eb3bba776d817e14bb563933e118d0e154c83a34058dd07b8a4",
-                "sha256:12cff189a3aafddf8e3825e9676a545d8bf29e8de4aaa3bc903fd2ccdbbc26f3",
-                "sha256:1bf4144fabcd58c4477b1161f619ad4b1e9760bfde89aa35079b39a4737a2d35",
-                "sha256:1dbce07991e3f188bacffe7ec03d3cc2dcb280857de6e80a3141c0b794434c11",
-                "sha256:1f9b60357a3e34ecf7376f2af425bee1ed7160025180902fc0ebbd8e278a17e5",
-                "sha256:220c55a25be08ebf066e5bf9df44e559145a6a2ff45de2b7d7011b0193d281fa",
-                "sha256:229dbdaa7f6c54407b6e974176c70db2cbd23688175e9b46e992af73e1c7012e",
-                "sha256:22c79f0e8d978e53ca894872e115873e872aac7026f4258da6a06838c1d62fa8",
-                "sha256:25bd3025e97d0f645bb2f211aca505915f4a6ea78fbcbf376231bd3163bdb911",
-                "sha256:2caa249172491f535ecc220f37d9f921b83cfb7a61ea8c4a15694c4b10a51b45",
-                "sha256:301bf12e70a69ce66ca5517b48cd61cb010cead67a86e8967f40b28b21710b58",
-                "sha256:32a1b67d5340d8df381e718a788417455edd76bed7e8a4cbd259acdc30b5e17e",
-                "sha256:32d4dc68afdb63a897f994899b897b1df159f60104146a99cecd230b890ef890",
-                "sha256:3379c1c2ee05957221571fe3508cd390319c1acf66e31f9067bc4ac7dd256175",
-                "sha256:35d690fa6113e506984c546d37190d1182b24a9fbc972d5de4dfeefcb768ba36",
-                "sha256:388563dd99db5f79dc370be41702ab13414bc90381555af38e5a6dddab904e39",
-                "sha256:3bffef33e39e396a9ea705cc14ecc23db286d74427bf10984cc630e401fb32b1",
-                "sha256:3d34d263cfa14c8e6e8c711f2707bce534a6c77260015e5f379e753c5ea375df",
-                "sha256:408d98887b40608be9feaa5b0547a1480d5af82ef841668b58142c8b741a7114",
-                "sha256:417001d1587ad87ca03430e4b9e46c3a4fc34a43af4fca62d47eca2a8e3b2318",
-                "sha256:44bc449e3ddef015eb572c8377ebc04712bdd672122ccb5754abc44455d1bda0",
-                "sha256:46cdb38b6e5c2bee4390d5512bbb7890fcd6ea15a54537a3cd3746916715ba17",
-                "sha256:49869a1780d4a78a86cd5580df17de88c503f8159f944c354bff247cda659ad1",
-                "sha256:4a16066313affecc6113f0b0053461809c65a472e7f1d88b74f68f97f8d461e3",
-                "sha256:4ca6193b745fc9e15bf24716a4ca8623387936b9f68f7bb333c0f735890fd888",
-                "sha256:4db7248c3b179cdf79fb9ddbdeed499a137ebd0d0c3004728fb1f17454e77763",
-                "sha256:4e421146153343d80586618202d9bbd4bab47274b54084319a7bc56afc0161f0",
-                "sha256:4ebe360d014e588adc98f22c8f2ab891b8f4611b3030062fa2b6501fdc47b6fc",
-                "sha256:4eefd13c72243d63e4b81c2199b73e937a8860eb8e9d270d6e210003d6bc7ed7",
-                "sha256:5058b50ca268a5baafcadeac4ad9b24993a35b0e4c7636075a74c69c356635c4",
-                "sha256:509413892de1b8008104897c98ff1e8ff258b29de16e39a3e9fc153de06579c6",
-                "sha256:54f0fabfb5db31b4d8e1a7bff3297a82cb44729a7cbd5f353cd67f70cec4dd88",
-                "sha256:576a6d6c75c3afc4a40eea23826b475f63df3a93ade94fd40aa9ad7e9574bb26",
-                "sha256:59ebec36f3b8e57501c6db0310d1059b161035571a0b54437891a52d58e96268",
-                "sha256:61949834ceaa5dd90e9154d429f501d75b3bf832c2eb08613ce708b243b6f26f",
-                "sha256:61a7a36e0c19f8d34148733758e7ae6a5286fb2a04a48b6df44eef55ad09db27",
-                "sha256:65ff4cf6f0ffb2f32f9d5e49f6cde305eaf67c3a9abe398b80fb108a1f7ca7ec",
-                "sha256:69d6a9c83e65a6cc2c6f87f031fd75c92459c0afbf4eb440b4f35569e9c58a7a",
-                "sha256:6d08f6d079f6c669217d8a18e9133e2f9c7bf904fbf5aed6de48bdd12e4819e5",
-                "sha256:6f56459ce65c31ed4817ee13723d08e45a8b4793dc59cc4ff49772d1dcfa19e5",
-                "sha256:74a80e3e21a27b270e3c9ff021f1abefe4f3ab92d2dac06072512fc225b07877",
-                "sha256:78057e5fdfdcc1ecfb97144df9b58a9989d3b67650dc01f0b79dd798afb1c197",
-                "sha256:7a3e172eb95caded48bb1565d93486e13fb0f3f63bc61b38859e6332714d3366",
-                "sha256:7b829cc81d3b6e43f923cd9479ac35a97893c9c2c5ba1be5bd12bd0211ef0e30",
-                "sha256:7bf21dd201e6b46ec01b6d189c083d68c2df71d537b444286ebaa772de52f6ed",
-                "sha256:7c859ca552f41245e6cbcf35e1ab4425ed72d1215569192e70ac4d391640c034",
-                "sha256:7e2beea67072d1aec5b5b1a0372736ddb268a65dedf3ab327c8bb0b1bb482a8e",
-                "sha256:81c82d7fc925c22b6bcf22bbdf28d61b12848d29fac69eed045fc0b3cdfaaf30",
-                "sha256:86e2b4387bd7849c6898ac5788e23e3f452bc157bfdc72cdc19c10e78125cb50",
-                "sha256:8ba3eaf9a449f3f5ca555739b30aa9f7aaf93139f5d4fa332ac604c8c263ade7",
-                "sha256:8e9911ce37bcc1d95b41449901b3bbe6eb993120f2ed13604dc5943e82539dc7",
-                "sha256:934735c1b59bd0dece877bd3aefc8a09b997d26bc1e778911e26fa204578dc93",
-                "sha256:93ee8738431cc0319c8ec770e2415606032eb99dcfe79a48a421b138fa676518",
-                "sha256:9740bb14e9f6b26a66f9de28882424689436a2686674078d3272fdba61ca18f8",
-                "sha256:98cfa7f1cace5256c8aca5fa4b43df64b4e88791980d18fd768f3de13e31a2f1",
-                "sha256:9913023dd0adde5bd480b803efba11e8f6cb179ba8a361b74a4f8fbe9dfe24bf",
-                "sha256:9a20a08f6810f4c4d2e94ec190fb631eadcdfdbcc6e0259fbb604eb717ae4160",
-                "sha256:9b6b39ab621c84f6b2f66bc1c3fe743b3511c7c0067b11bd6c65fa05b0be08d0",
-                "sha256:9ed90a7a6f113ea251daf8701bd7e19a22af1ae1a2695d386ae7a71ede65d51d",
-                "sha256:a195b0aa444acfaf9e2cb8c6b7df0a47d81a64b7390a7f4d677e2f83d62de156",
-                "sha256:a2bddc90a19178e4e9491bf819dce73fe3567cf72815f1abe47e6c297e121520",
-                "sha256:a47c3ba07ec3c6f72e2846066630a2fd2ad96744579f0b182811b34987bed6f2",
-                "sha256:ac9602131b5d0897cfdafe10d850c5fb808b0449d9b6c904c723c68388c7f101",
-                "sha256:b08a93c5c1a42e8a9ab2cd6ffa635ccf7134d87992c9da67ada5488c86bda07a",
-                "sha256:b478b5c5db1f34f44c0cf1a4258fbf11ffc16b30474784a84c7c97ccf35ebe81",
-                "sha256:baa60b607acb3ac73c43976247dcf68ea614d78ecc50a71aa67f1d56ade0cf2a",
-                "sha256:be92561b26d009a4e505e40ebd9104687e41bf847027836be3d1bee7b8bbe12f",
-                "sha256:bf10839517db6c21b5c93d80b000b60896ca07a07a6519a3ec4b48c7a5557f8a",
-                "sha256:c0ec20c3cd345b05f5b5db1a3fa1994593ef7277cfe4dcf7c0581d76e62ecdb0",
-                "sha256:c1de2b822c452dd2e803bf2fa608caa20a8794c1c18eb008012d722e3ce5d52c",
-                "sha256:c3cc11c4f42aca08cb2de5353e70345eb12c00cafcb4bb2adde95e1a06afe524",
-                "sha256:c713006bd866721b4d4c4b741e1cdbe896465334dbbc1462e5645067e59242b0",
-                "sha256:c81d7cf1e872474ac49a3ff9a0a6456440c0dfc5f32fc45286b201b8cf8f3829",
-                "sha256:c8d636e52ce414f582a376e245f81cad45f3b210589a0d63df373c86c6781b2b",
-                "sha256:cd7900f827d11fa649b9fb5e2cf3401536295461c059c75f5496ff999ac3c417",
-                "sha256:cff62a2d68ccd53e6d74471c6d0b44a37ef73844137aba3fb898bc16a68d255c",
-                "sha256:d0ede594db3fc377f623fa6a5031bc6b8dca3026df06838a78f92074dd46dd14",
-                "sha256:d1580a65dfed84925cdc9ac941de7b72eafdc27edc6fad6fd02e2cbf2c403ba1",
-                "sha256:d26932cabaecd0d54ac0e793f464f42dc4f59cfacf1fa7f2b1726f87a4e958be",
-                "sha256:d3793d414c3fe92778b23c860a9495e25c5450f6fb2f29acc0b2aa330f9e2e17",
-                "sha256:d4a557bf018c7e3712261dea0ac86387479fa6a099e5777f58dca3ed65189b0d",
-                "sha256:d611d20513732dfaa2d01d31d4332d1243d6d60cd0b55191b356013cadb5b5cd",
-                "sha256:d71f6949182af74125e3dc2eaa809a7b296d0299116d1d8f9c27dc27aa128dbd",
-                "sha256:d82df1c6c4c180e7d40bb42fd81b8573790e6d352c09e3f5f3ca4e5b89921436",
-                "sha256:d88dfadfa5f57903f3222ad319529d64aaa9e33133c5cd4b0269bb6abdb64bbf",
-                "sha256:d8fdcdc335a8ef022a92ea8f68072d8920305bb87184af2be2621dc38151d6a3",
-                "sha256:dd2f48b1b209d2156bd01d96f98096ef9e9bf33df67da57103afbdccac09800a",
-                "sha256:de00167af7af51de31cc5b2e0c798db96525291b69b29efc4c3234d699c33576",
-                "sha256:de434e906b399b43f552a794879b6108a636b47d816775c094990ea816f0247d",
-                "sha256:e2a6378ca49c7bc94ee6d9b29f7a169125741cb8b653585d8882acf8409d50df",
-                "sha256:e3d65304b9d959ce5ac8406e3a0a837a3615d92e621e63db0a05682c2689f9ba",
-                "sha256:e54688a0b26d4d383b61b3a66677c477055048db01815daa61a868eaca129c51",
-                "sha256:e64d58de7b21f2a121505a2e37c6dabb8b21329e5e63350a77411996c03eba05",
-                "sha256:e91ddc8503d5979f269fbf86702ba1a62c44ab1ce510608bc2a510562a9235d9",
-                "sha256:eac42b4635b9a69d02c07e6472ec418e12765e80e648441b9e1ad5f66db49c8a",
-                "sha256:ed2efc34162c837782c2a36bff81ddf3bd15adaf2561681049aa50a3550af52c",
-                "sha256:eece0a403120b5db50ea8744275a00f9cce497702dfb101eb2cb04582a81a6dc",
-                "sha256:f355fdea18c32450d3b352b9c6e449c3ba21535e4c6889792d75b1483c850adc",
-                "sha256:f646c6b6120b1a4455fa23ba5b8b0c7eb6f5e5dbb262e85c3c511bd2cceba5f3",
-                "sha256:f7496a2fc9e24892416a6f5003776013a9b2620bf235d788699e853194cd96fa",
-                "sha256:f83939ae9d4c560fce3b27b7122cc5dc2ba597b649d36d24d81cb560355e2490",
-                "sha256:fb471fa96772d311057a1eb97165887f127387c71d7f904bcff59df468043779",
-                "sha256:feae3801216957dbfd4f5baf2d433dc014499405722974f5180189083df609cc"
+                "sha256:04eb56c932bf7ad9a22bdfde995c2bb22909ae00218dcb3e174f76375389a339",
+                "sha256:07150e8f018df72c32cbc7958eacae8d18a14064a3b18dda00ef443b12804ad0",
+                "sha256:07c3863c968eaeab8c4ebd32c3ddf9ca3e5762d77e20da21a3d9a7578bd0b6c4",
+                "sha256:0802d7a01ab7bd73dd0dd5c3d9143a7fe77e6c98ab33fdf4b14b20fe5ca090a7",
+                "sha256:0d4fd161f173c330291f54de5ab8e849e8686ad24c76f45c8045ce7d48aa0c33",
+                "sha256:110e35df5837e4ca5506760dc6e354104eefb7c3e4cbc9f8a53bb7813778601f",
+                "sha256:12f1d451ba9d1bd4f8e8585d59f09fee6fc6ed978b27dd55c2915a37bfa63dfc",
+                "sha256:13c653dac81e95e966759a9b8b47d6c0fdf0a6c51a743397dee9091e62c7bfa8",
+                "sha256:1ab0309a5a36ebe6bbf6b33f200aa9faa71493b842924e85840d78ecd72ce3cd",
+                "sha256:1c489982b59bce1d7a21dd71204ddae96dd34bcbdb50050cf3e3a1dc18f144d0",
+                "sha256:1d4dc50cc7d4a8763454ba3e12fbf558bcf7b0764eb7591fb5dfe9ac35b8162d",
+                "sha256:1e5472aa34e798d3a8ae306521a711e69ed1e3af72970c2f76312c795e547770",
+                "sha256:2129572d541e86561d4dc0f6be10788de6c19ff25a8ed83231f1825b05154cf2",
+                "sha256:2192f5a3b115274a9c68f75ff2c21cfb4006f0e5eff637f856e622871f0e7e26",
+                "sha256:298b24dfe7e94aa7857ab78059948158565a659f900e21645fffd55fc8736a84",
+                "sha256:2e17e38ea19c21a65026405e511c3cf9b26fe503c492e6a9db469763818e33c5",
+                "sha256:30ca96e20dcda02665c241a70b577202d3b9443416aaffefc490f6ff9d78f83f",
+                "sha256:30ea45ecd108aea9aec504192a7b18dce34e8dcb48503659a7a598c623d08437",
+                "sha256:34661ecc88dfed442b7be22dbf481f3bc80111bb44b225f4520c73ed380b2424",
+                "sha256:371a0523d40f560d1352ca91e532acaa2b331500ded3cf3ad699eee88d76af60",
+                "sha256:38ca4cc998b93bcc062a1cb01f41761b7ca54cad135cd8dd50074623449d5424",
+                "sha256:39a3ca5e44d9ba036f2515a3aec1fdb27849bd55affd205c578447fc2e2d2503",
+                "sha256:3a29a44f06ddd3f57a8656326044c57ec05e5162e97ffeed70b376b309c1bfb0",
+                "sha256:3bdf506a8b3b17b3cfa9b9b0b69d5fb542058fa3bb7121d5e5c4ec553ff39801",
+                "sha256:3d109b2321f9d2df7e17fe49791379f831cc8cfd7ad29069e1687faf99095188",
+                "sha256:3e03065608eff491466bcfb7fa1c48d3f81bfac1926db68f13f3362a1b8bc2db",
+                "sha256:3ec72c95dc86430d156643a41ebc467677f45907a6dd44c013377c3bbc4e68d0",
+                "sha256:414ec9fff19a5da1ed4c6322129eed2a8b1dd77ce6d55cf2b99013e374368489",
+                "sha256:443844f59416327abc0074d7d91a1cbeaaca270e79b71cb5acb824c8a6472b9d",
+                "sha256:4460309364de3a6139a6ac337a9bb454101e8280fed834970c3dd04a00d0677e",
+                "sha256:492301b5234acc0d04443bdb00bf7a1ad601caf82bc837f8a4c137c7eef8932e",
+                "sha256:52461114f78380b7432f9a31b0249933930aa4968507ecc9a27a345a905abac6",
+                "sha256:525bbe1db05d8cb657cb793aa994372f84f8fa22a009a67751e3c170f78b8812",
+                "sha256:5361782c9ff6c5a40d39637258dd64e109f7c535976e9c2db8ae17a7c6c2b3cb",
+                "sha256:53958a8c20e4344a86bad6e9cd0b886710bfac2bc581102b852bb24c2c09084c",
+                "sha256:54a7fa73ae22ee477ae3a5e3c6cafba4f1463363929f9e5e9b471f2d2278c649",
+                "sha256:55e503f28f5a9d225ce9d0639e3f5b1801bacace5aea926ec2998e73c5150fe7",
+                "sha256:57317f65d68dc217d6f7e354a63f349a6f08fd25c962e54c99df1520dce87b53",
+                "sha256:5b6d9e1be9fdaef16c53ce0b2bfd880ae21ca54bc244a1de1c28d0cefeee8760",
+                "sha256:5b6dbd6b01544a82836d32dffa4dc4fa63fed520c24409856372801bc65204a9",
+                "sha256:5dacf851e2a2cba837dcdf007af07dad7c860f70061e0c39094e500fc70a6e50",
+                "sha256:5e5058ec639d754267fbccb51f50f752c1e17e0a9a407c7eb2c3cdce8eb6f571",
+                "sha256:6c355d04dac84f9dcaee1cd2fb9df0236afadbc90e6ebf1285376c480523d8b1",
+                "sha256:701db75f49f1eca26f07f6b08751c43d5979e9ff39360a40ba165aebccbb2199",
+                "sha256:718f6cef753f8cc9275f5e790178151a59fae5f8587f53ee6f36e3a939466bd4",
+                "sha256:71e7c2fe631f06fd2ebc5870640bc0e19cdcd3a26ad2e2a663fe4b5aa2bc3ea2",
+                "sha256:72b0037a683a344c118c51a2cc353de03d1442792a8c1fd9adb64592f5794cd2",
+                "sha256:758b4b8261032991849fc1652feab2fdc432255a01898e15fe4957f91c49f5a9",
+                "sha256:7edd705d185b16399ff1143f84dcb34a8e8ec73229ecc187b32d916158541ada",
+                "sha256:7fa6ace75ad05303967908321d9b02f7011a1ea5103f9f72342dba820fa47b1a",
+                "sha256:827717b9a9f28cef58ba1018879f9ba777b9f287675107baf7d92409ac9d908d",
+                "sha256:86716c3f2c8bc82fd677f0fe1064833daf73148d168249c5fd5d6a8d1c3a6921",
+                "sha256:867d909f633a39add6f62fa19b61520abccb9af527194db0c490ebd2e910200d",
+                "sha256:8718cf7d96eae80e01334c5a3e171d698bfbcb0d357a27f5421be0de5a096a47",
+                "sha256:877ccd7ce58737fb672f17681d3ff5e78ab5731e7096662ae7b36305d95eb82c",
+                "sha256:8af59e31d1e51ccd998142f8ec7347e5ba7a801572539a8bffd9009956a76343",
+                "sha256:8e444c7198e531d236a5791eb6d1045249c93aa1dfd036b8275e9b0f36633c79",
+                "sha256:8ecd7c61ac4694b89c906d94f465324cb22772c5b54dacc947e45a927cfb5784",
+                "sha256:92d0370d2b0806aa652d2c1240d6c858a5b2a30baa892dce0f2638260590b038",
+                "sha256:933928a57c4445c4039e0dd4c75a9bfcab5bf0b1bb23b4869e340e28bc7aeb1e",
+                "sha256:97d408496c7f2d263533a1be2798fca13d1ae5eb625f556b3f3412cfd9894e01",
+                "sha256:98b8f16815eac222f6092e27c65f2811e7ad2300661f72224f02b28ec361baae",
+                "sha256:9a374e990d365b403723be368a054c086d168c65a8f2fe74bdf5e1344da37a82",
+                "sha256:9ba445c38aff01c41175dcc621b88357907f4deb1f147ff140e691e74c589029",
+                "sha256:9d8382fa1e672baf15d5fb3d13e0a1ca0f2b38730eab0611944bb5e0c7df9bc6",
+                "sha256:9e067a28b7935f1087b77b89f4cff7c84f0673bcf756aa6dbf21a36a32065ef2",
+                "sha256:a5983053242dab7daf36bfea5c246329249f28e71afc2ab8b4dbe8ac3c9287dd",
+                "sha256:ac8e5ab21c1d6baa1b2f6778ffa8b4970f5277bfa60c9fedbb03fc483ab21ee7",
+                "sha256:aed59809b3c5c021c7ae9f380da6f0835062cfd91a81a3125e7d4eb4e394fc15",
+                "sha256:b0b6a55ed8ae86d6a809bdf0552753f8bc11ce1a8b4ccb2a55f9a4026ecde01c",
+                "sha256:b1034413b31fa2f74213e19c109dcf2d4e5aac49336efc0640c7fbea9713e0b0",
+                "sha256:b3c9cedaf2f79fdc43940f29c9ff0fe5f135ed1ea9ad8f3be18bd4b77847889d",
+                "sha256:b4b4cd04b4f02693811015f9356945caffe7c5ded7c897df88d87550778d5c58",
+                "sha256:b855ce93692c9d598e21e0a28a32776c0ec63749f32125ba885eefbb5d669258",
+                "sha256:c3a9266cf6038d8af82cc4067f38dd5de84ebb6a152cfc25b9ef0b44c8ce84e9",
+                "sha256:c8408c499ef67dcec49be00b7007e0ceb7cbb1211cdaeb390294d8a0e9ceab75",
+                "sha256:d05909397f155b12a0a9b7c36cf1fb8a96df5b7caa9bc69c47ed36ab73e1aba9",
+                "sha256:d05d0ee19a1a37f995568ce05208b985683681d7f49073618559a44a4a568d44",
+                "sha256:d313de63feda18974c406cb05a963b27234aa0ad604561ed435ff726eaca1160",
+                "sha256:d3fab381de4e0bf8dfa8e356ee04c0ac18d5b603344cca98c0f6106760e979a8",
+                "sha256:d779c3e8faa92b241e58600f7b2126f3c055d75d3968ed5d856e2c5e2ec847ce",
+                "sha256:d865b3a330bafe0891d8414a87f1a56509baa19980150c3ba75f8b8e966a8655",
+                "sha256:dfdddfea962e039f8397a0e892eccf0fc25378ec3eb1ca81fdf2658f20272f0e",
+                "sha256:e25b652b67077b3d37451a7471b8975d478ec87d7f96f8f4673ed2d6b97355df",
+                "sha256:e34142d94a8f423687c940fb64031cfe099ca31dee463ecfcc784060d58929ff",
+                "sha256:e4d0abeaed6e6162d4506e7cc3e9df40f5589d9946922635bdf03821ff969380",
+                "sha256:e5504321223125495b9ae78cd49c7a4495ae8e547d09f704aad0c43a9fa8c5f2",
+                "sha256:e733ff332afa44737b3d1e433a023b8f588881e5d2ae7698293560437aaa1879",
+                "sha256:ec9bf97d6959112fe15c04daf1e94451a78a28d2573ef22f1d215db00204db20",
+                "sha256:ee64beaa98c3917fe21262670bc536663ab69cd07ab96c160e01ff30d654a880",
+                "sha256:eed9cbe732bdcd8cbf23f33b5d942a2111e91bf0573d1d28d49b626b63b9db52",
+                "sha256:ef7f2f8a95fdb755047015894e1859a49a5495bd6d1b197898b72e2c3abeca92",
+                "sha256:effda4ab1426a2b3384ea354f1d873865f87ae0e946508765dfe1b52c6dea6b8",
+                "sha256:f30cbf469eb11f39dbdca787bf6b3b287898e10202c4371863fa56844b4220f0",
+                "sha256:f4483359b6ebaf64ad5381c6f590e7f297d251dd6de019992e9481123c55c790",
+                "sha256:f4b87bf782306c27e9dc97ae148a72207b5dc018d5ad05768aeda6fbf262be1d",
+                "sha256:f5a2ed163bd9f00c494f70f387803277592bf5788f8a07a67eebda3ad1c11b7d",
+                "sha256:f5d03b13d32608a72301ee78afac0105b3a14bf63f2a9361c3742b37d85b38d7",
+                "sha256:f60f561ebd5ef2acfa971173ddf7f0987be3489d03aeb57a066f3daaeaa2b50e",
+                "sha256:f6b8443be8d4bd58d2f4d33a41fafddc012214e890d5c3634ba14e048287a982",
+                "sha256:f98470a2e8a7c11692deafe586b31bf87f5481d25ceec34fe49319b9c5de6ad1",
+                "sha256:f9d44e8ac7a788d60589e22cd27e072cf480100c4fa68fab6bcbc5927a162e18",
+                "sha256:fa29986a7ed07f321878445d10a896a0a3f1a556c39b3d2750091759b0f252e2",
+                "sha256:fc340a9159ead0ffaac5893569e124e975958e31ad381357e65765296882b81e",
+                "sha256:fd134893881bbbb1ede80e1c69a1ce8590018bee627be9b325ae2bdfe42b2e6b"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.15.6"
+            "version": "==0.15.7"
         },
         "qtconsole": {
             "hashes": [
-                "sha256:30975c6a7d7941dd646d00a23e5982db49beaa60c3920bb243727d43da805f12",
-                "sha256:dc935780da276a2ab31a7a08a8cf327a2ea47fa27e21d485073251a7eeb16167"
+                "sha256:35fd6e87b1f6d1fd41801b07e69339f8982e76afd4fa8ef35595bc6036717189",
+                "sha256:5e4082a86a201796b2a5cfd4298352d22b158b51b57736531824715fc2a979dd"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.2"
+            "version": "==5.4.3"
         },
         "qtpy": {
             "hashes": [
@@ -2505,11 +2505,11 @@
         },
         "requests": {
             "hashes": [
-                "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa",
-                "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"
+                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
+                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4'",
-            "version": "==2.28.2"
+            "index": "pypi",
+            "version": "==2.31.0"
         },
         "rfc3339-validator": {
             "hashes": [
@@ -2599,11 +2599,11 @@
         },
         "s3transfer": {
             "hashes": [
-                "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd",
-                "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"
+                "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346",
+                "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.6.0"
+            "version": "==0.6.1"
         },
         "scipy": {
             "hashes": [
@@ -2642,18 +2642,19 @@
         },
         "send2trash": {
             "hashes": [
-                "sha256:d2c24762fd3759860a0aff155e45871447ea58d2be6bdd39b5c8f966a0c99c2d",
-                "sha256:f20eaadfdb517eaca5ce077640cb261c7d2698385a6a0f072a4a5447fd49fa08"
+                "sha256:a384719d99c07ce1eefd6905d2decb6f8b7ed054025bb0e618919f945de4f679",
+                "sha256:c132d59fa44b9ca2b1699af5c86f57ce9f4c5eb56629d5d55fbb7a35f84e2312"
             ],
-            "version": "==1.8.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.8.2"
         },
         "setuptools": {
             "hashes": [
-                "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a",
-                "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"
+                "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f",
+                "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==67.6.1"
+            "version": "==67.8.0"
         },
         "shapely": {
             "hashes": [
@@ -2815,11 +2816,11 @@
         },
         "soupsieve": {
             "hashes": [
-                "sha256:49e5368c2cda80ee7e84da9dbe3e110b70a4575f196efb74e51b94549d921955",
-                "sha256:e28dba9ca6c7c00173e34e4ba57448f0688bb681b7c5e8bf4971daafc093d69a"
+                "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8",
+                "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.4"
+            "version": "==2.4.1"
         },
         "stack-data": {
             "hashes": [
@@ -2884,20 +2885,20 @@
         },
         "tornado": {
             "hashes": [
-                "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca",
-                "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72",
-                "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23",
-                "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8",
-                "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b",
-                "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9",
-                "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13",
-                "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75",
-                "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac",
-                "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e",
-                "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==6.2"
+                "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4",
+                "sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf",
+                "sha256:29e71c847a35f6e10ca3b5c2990a52ce38b233019d8e858b755ea6ce4dcdd19d",
+                "sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba",
+                "sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe",
+                "sha256:6a0848f1aea0d196a7c4f6772197cbe2abc4266f836b0aac76947872cd29b411",
+                "sha256:7efcbcc30b7c654eb6a8c9c9da787a851c18f8ccd4a5a3a95b05c7accfa068d2",
+                "sha256:834ae7540ad3a83199a8da8f9f2d383e3c3d5130a328889e4cc991acc81e87a0",
+                "sha256:b46a6ab20f5c7c1cb949c72c1994a4585d2eaa0be4853f50a03b5031e964fc7c",
+                "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f",
+                "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829"
+            ],
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==6.3.2"
         },
         "tqdm": {
             "hashes": [
@@ -2917,11 +2918,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb",
-                "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
+                "sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223",
+                "sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.5.0"
+            "version": "==4.6.0"
         },
         "tzdata": {
             "hashes": [
@@ -2969,11 +2970,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:3f09e6d8230892547132177f575a4e3e73cfdf06526e20cc02aa1c3b47184d40",
-                "sha256:cdf5877568b7e83aa7cf2244ab56a3213de587bbe0ce9d8b9600fc77b455d89e"
+                "sha256:c7d67c13b928645f259d9b847ab5b57fd2d127213ca41ebd880de1f553b7c23b",
+                "sha256:f8c64e28cd700e7ba1f04350d66422b6833b82a796b525a51e740b8cc8dab4b1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.5.1"
+            "version": "==1.5.2"
         },
         "whitebox": {
             "hashes": [
@@ -3119,11 +3120,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:1c183bf61b148b00bcebfa5d9b39312733ae97f6dad90d7e9b4d86c8647f498c",
-                "sha256:a950236df04ad75b5bc7f816f9af3d74dc118fd42f2ff7e80e8e60ca1f182e2d"
+                "sha256:c7b80eb7f5a855a88efc971fda506ff7a91c280b42cdae26643e0f601ea281ea",
+                "sha256:ea8801f15dfe4ffb76dea1b09b847430ffd70d827b41735c64a0638a04103bfc"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==8.12.0"
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==8.12.2"
         },
         "jedi": {
             "hashes": [
@@ -3189,11 +3190,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094",
-                "sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500"
+                "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
+                "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.15.0"
+            "version": "==2.15.1"
         },
         "six": {
             "hashes": [
@@ -3220,11 +3221,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb",
-                "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
+                "sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223",
+                "sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.5.0"
+            "version": "==4.6.0"
         },
         "wcwidth": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9590ceabb..b15e70402 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.11.6 - 2023-05-26 - [PR#919](https://github.com/NOAA-OWP/inundation-mapping/pull/919)
+
+Auto Bot asked for the python package of `requests` be upgraded from 2.28.2 to 2.31.0. This has triggered a number of packages to upgrade.
+
+### Changes  
+- `Pipfile.lock`: as described.
+
+<br/><br/>
+
 ## v4.3.11.5 - 2023-05-30 - [PR#911](https://github.com/NOAA-OWP/inundation-mapping/pull/911)
 
 This fix addresses bugs found when using the recently added functionality in `tools/synthesize_test_cases.py` along with the `PREV` argument. The `-pfiles` argument now performs as expected for both `DEV` and `PREV` processing. Addresses #871 
@@ -8,7 +17,6 @@ This fix addresses bugs found when using the recently added functionality in `to
 ### Changes  
 `tools/synthesize_test_cases.py`: multiple changes to enable all expected functionality with the `-pfiles` and `-pcsv` arguments
 
-
 <br/><br/>
 
 ## v4.3.11.4 - 2023-05-18 - [PR#917](https://github.com/NOAA-OWP/inundation-mapping/pull/917)
@@ -35,6 +43,7 @@ Fixes a bug in CatFIM script where a bracket was missing on a pandas `concat` st
 
 <br/><br/>
 
+
 ## v4.3.11.2 - 2023-05-19 - [PR#918](https://github.com/NOAA-OWP/inundation-mapping/pull/918)
 
 This fix addresses a bug that was preventing `burn_in_levees.py` from running. The if statement in run_unit_wb.sh preceeding `burn_in_levees.py` was checking for the existence of a filepath that doesn't exist.

From b422a5f59318b629ed2a0e52476d2a1957304735 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 9 Jun 2023 12:05:55 -0400
Subject: [PATCH 253/359] Fix for AWS S3 in README.md

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 27312796f..ed12c3839 100644
--- a/README.md
+++ b/README.md
@@ -29,18 +29,18 @@ This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket.
 
 #### Examples
 
-List bucket folder structure:
+The available versions can be found by running
 ```
-aws s3 ls s3://noaa-nws-owp-fim/ --request-payer requester
+aws s3 ls s3://noaa-nws-owp-fim/hand_fim/ --request-payer requester
 ```
 
-Download a directory of outputs for a HUC8:
+Download a directory of outputs for a single HUC8:
 ```
-aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_0_18_02/12090301 \
+aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_3_11_0/12090301 \
     /your_local_folder_name/12090301 --request-payer requester
 ```
-By adjusting pathing, you can also download entire directories such as the fim_4_0_18_0 folder.
-**Note**: There may be newer editions than fim_4_0_18_0, and it is recommended to adjust the command above for the latest version.
+By adjusting pathing, you can also download entire directories such as the fim_4_3_11_0 folder.
+**Note**: There may be newer editions than fim_4_3_11_0, and it is recommended to adjust the command above for the latest version.
 
 
 ## Running the Code

From 1358a3d945c2d49ba840250f6b40025321f838ea Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 16 Jun 2023 10:26:55 -0600
Subject: [PATCH 254/359] v4.3.11.7 Add csv file to capture processing time for
 each HUC & Update: PR Template, Unit Tests

---
 .github/PULL_REQUEST_TEMPLATE.md           | 13 +++++++---
 CITATION.cff                               |  2 +-
 docs/CHANGELOG.md                          | 19 ++++++++++++++
 src/bash_functions.env                     | 30 +++++++++++++++-------
 src/check_huc_inputs.py                    |  3 ++-
 src/run_unit_wb.sh                         |  3 +++
 unit_tests/clip_vectors_to_wbd_params.json |  3 ++-
 unit_tests/clip_vectors_to_wbd_test.py     |  1 +
 unit_tests/derive_level_paths_params.json  |  2 +-
 unit_tests/derive_level_paths_test.py      |  1 +
 10 files changed, 61 insertions(+), 16 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 306c46d21..350a5d6ee 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -9,13 +9,13 @@
 ### Testing
 
 
-### Checklist
+### Issuer Checklist (For developer use)
 
 _You may update this checklist before and/or after creating the PR. If you're unsure about any of them, please ask, we're here to help! These items are what we are going to look for before merging your code._
 
 - [ ] Informative and human-readable title, using the format: `[_pt] PR: <description>`
 - [ ] Links are provided if this PR resolves an issue, or depends on another other PR
-- [ ] If submitting a PR to the `dev` branch (the default branch), you have a descriptive [Feature Branch](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow) name using the format: `dev-<description-of-change>` (e.g.: `dev-revise-levee-masking`)
+- [ ] If submitting a PR to the `dev` branch (the default branch), you have a descriptive [Feature Branch](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow) name using the format: `dev-<description-of-change>` (e.g. `dev-revise-levee-masking`)
 - [ ] Changes are limited to a single goal (no scope creep)
 - [ ] The feature branch you're submitting as a PR is up to date (merged) with the latest `dev` branch
 - [ ] Changes adhere to [PEP-8 Style Guidelines](https://peps.python.org/pep-0008/)
@@ -23,5 +23,12 @@ _You may update this checklist before and/or after creating the PR. If you're un
 - [ ] Passes all unit tests locally (inside interactive Docker container, at `/foss_fim/`, run: `pytest unit_tests/`)
 - [ ] New functions are documented (with a description, list of inputs, and expected output)
 - [ ] Placeholder code is flagged / future todos are captured in comments
-- [ ] Project documentation has been updated ([CHANGELOG](/docs/CHANGELOG.md) and/or [README](/README.md))
+- [ ] [CHANGELOG](/docs/CHANGELOG.md) updated with template version number, e.g. `4.x.x.x` 
 - [ ] [Reviewers requested](https://help.github.com/articles/requesting-a-pull-request-review/)
+- [ ] Add yourself as an [assignee](https://docs.github.com/en/issues/tracking-your-work-with-issues/assigning-issues-and-pull-requests-to-other-github-users) in the PR  as well as the FIM Technical Lead
+
+### Merge Checklist (For Technical Lead use only)
+
+- [ ] Update [CHANGELOG](/docs/CHANGELOG.md) with latest version number
+- [ ] Update the [Citation.cff](/CITATION.cff) file to reflect the latest version number in the [CHANGELOG](/docs/CHANGELOG.md)
+- [ ] If applicable, update [README](/README.md) with major alterations
diff --git a/CITATION.cff b/CITATION.cff
index 802d32183..3eb8c7de6 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.3.11.6
+version: 4.3.11.7
 date-released: 2023
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b15e70402..297d08d5c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,25 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.11.7 - 2023-06-12 - [PR#932](https://github.com/NOAA-OWP/inundation-mapping/pull/932)
+
+Write to a csv file with processing time of `run_unit_wb.sh`, update PR Template, add/update bash functions in `bash_functions.env`, and modify error handling in `src/check_huc_inputs.py`. Update unit tests to throw no failures, `25 passed, 3 skipped`.
+
+### Changes
+- `.github/`
+    - `PULL_REQUEST_TEMPLATE.md` : Update PR Checklist into Issuer Checklist and Merge Checklist  
+- `src/`
+    - `run_unit_wb.sh`: Add line to log processing time to `$outputDestDir/logs/unit/total_duration_run_by_unit_all_HUCs.csv`
+    - `check_huc_inputs.py`: Modify error handling. Correctly print HUC number if it is not valid (within `included_huc*.lst`)
+    - `bash_functions.env`: Add `Calc_Time` function, add `local` keyword to functionally scoped variables in `Calc_Duration`
+- `unit_tests/`
+    - `derive_level_paths_test.py`: Update - new parameter (`buffer_wbd_streams`)
+    - `derive_level_paths_params.json`: Add new parameter (`buffer_wbd_streams`)
+    - `clip_vectors_to_wbd_test.py`: Update - new parameter (`wbd_streams_buffer_filename`)
+    - `clip_vectors_to_wbd_params.json`: Add new parameter (`wbd_streams_buffer_filename`) & Fix pathing for `nwm_headwaters`
+
+<br/><br/>
+
 ## v4.3.11.6 - 2023-05-26 - [PR#919](https://github.com/NOAA-OWP/inundation-mapping/pull/919)
 
 Auto Bot asked for the python package of `requests` be upgraded from 2.28.2 to 2.31.0. This has triggered a number of packages to upgrade.
diff --git a/src/bash_functions.env b/src/bash_functions.env
index 11ed8ea46..44e3bb4cb 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-## functions
-# timing
+## Timing functions
+
 T_total_start () {
     t0=`date +%s`
 }
@@ -12,8 +12,6 @@ Tstart () {
 
 Tcount () {
     t2=`date +%s`
-    #echo "Time  = `expr $t2 \- $t1`sec"
-    #echo "Cumulative_Time = `expr $t2 \- $t0`sec"
 
     local total_sec=$(( $t2 - $t1))
     local dur_min=$((total_sec / 60))
@@ -22,15 +20,29 @@ Tcount () {
 }
 
 Calc_Duration() {
-    start_time=$1
-    end_time=`date +%s`
+    local start_time=$1
+    local end_time=`date +%s`
 
-    total_sec=$(( $end_time - $start_time ))
-    dur_min=$((total_sec / 60))
-    dur_remainder_sec=$((total_sec % 60))
+    local total_sec=$(( $end_time - $start_time ))
+    local dur_min=$((total_sec / 60))
+    local dur_remainder_sec=$((total_sec % 60))
     echo "Duration = $dur_min min(s) and $dur_remainder_sec sec"
 }
 
+Calc_Time() {
+    local start_time=$1 
+    local end_time=`date +%s`
+    local total_sec=$(( $end_time - $start_time ))
+    local dur_min=$((total_sec / 60))
+    local dur_remainder_sec=$((total_sec % 60))
+    
+    if (( $dur_remainder_sec < 10 )); then
+        echo "$dur_min:0$dur_remainder_sec"
+    else
+        echo "$dur_min:$dur_remainder_sec"
+    fi
+}
+
 export -f T_total_start
 export -f Tstart
 export -f Tcount
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index d1f4d2708..7d659928c 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -64,7 +64,8 @@ def __check_for_membership(hucs,accepted_hucs_set):
             raise KeyError(msg)
         
         if huc not in accepted_hucs_set:
-            raise KeyError("HUC {} not found in available inputs. Edit HUC inputs or acquire datasets and try again".format(huc))
+            msg = f"HUC {huc} not found in available inputs. Edit HUC inputs or acquire datasets and try again."
+            raise KeyError(msg)
 
 
 def check_hucs(hucs, inputsDir):
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index f05e427bf..228262982 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -319,6 +319,9 @@ echo "---- HUC $hucNumber - branches have now been processed"
 Calc_Duration $branch_processing_start_time
 echo
 
+# WRITE TO LOG FILE CONTAINING ALL HUC PROCESSING TIMES
+echo "$hucNumber, $(Calc_Time $huc_start_time)" >> "$outputDestDir/logs/unit/total_duration_run_by_unit_all_HUCs.csv"
+
 date -u
 echo "---- HUC processing for $hucNumber is complete"
 Calc_Duration $huc_start_time
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 64461dcb8..336e88086 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -5,6 +5,7 @@
 		"hucCode": "05030104",
 		"subset_nwm_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
 		"wbd_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
+		"wbd_streams_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered_streams.gpkg",
 		"wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
 		"dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
 		"dem_domain" : "/data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg",
@@ -16,7 +17,7 @@
 		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
 		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
 		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
-		"nwm_headwaters": "/data/inputs/nhdplus_vectors_aggregate/agg_nhd_headwaters_adj.gpkg",
+		"nwm_headwaters": "/data/inputs/nwm_hydrofabric/nwm_headwaters.gpkg",
 		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
 		"subset_nld_lines_preprocessed": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/3d_nld_subset_levees_burned.gpkg",
 		"wbd_buffer_distance": 5000,
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
index 3cc35c743..dfe223d2c 100644
--- a/unit_tests/clip_vectors_to_wbd_test.py
+++ b/unit_tests/clip_vectors_to_wbd_test.py
@@ -45,6 +45,7 @@ def test_subset_vector_layers_success(self):
                                      hucCode = params["hucCode"],
                                      subset_nwm_headwaters = params["subset_nwm_headwaters"],                               
                                      wbd_buffer_filename = params["wbd_buffer_filename"],
+                                     wbd_streams_buffer_filename = params["wbd_streams_buffer_filename"],
                                      wbd_filename = params["wbd_filename"],
                                      dem_filename = params["dem_filename"],
                                      dem_domain = params["dem_domain"],
diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json
index badee64de..e3ff0420e 100644
--- a/unit_tests/derive_level_paths_params.json
+++ b/unit_tests/derive_level_paths_params.json
@@ -3,6 +3,7 @@
 	{
 		"outputDestDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
 		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"buffer_wbd_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered_streams.gpkg",
 		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
 		"branch_id_attribute": "levpa_id",
 		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
@@ -15,4 +16,3 @@
 	}	
 }
 
-
diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py
index 2cd05f680..59f423872 100644
--- a/unit_tests/derive_level_paths_test.py
+++ b/unit_tests/derive_level_paths_test.py
@@ -42,6 +42,7 @@ def test_Derive_level_paths_success_all_params(self):
         # so we skip them here.
         # Returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
         actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
+                                           buffer_wbd_streams = params["buffer_wbd_streams"],
                                            out_stream_network = params["out_stream_network"],
                                            branch_id_attribute = params["branch_id_attribute"],
                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],

From a7240515ccb2a59c04fccd537b41f0834f456d95 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 23 Jun 2023 10:48:31 -0600
Subject: [PATCH 255/359] Update to README.md

---
 README.md | 117 +++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 76 insertions(+), 41 deletions(-)

diff --git a/README.md b/README.md
index ed12c3839..d6ed398e0 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 # FIM Version 4 
 
 ## Accessing Data through ESIP S3 Bucket
-The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools.  You will need permission from ESIP to access this data. Please contact Carson Pruitt (carson.pruitt@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) for assistance.
+The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools. Please contact Carson Pruitt (carson.pruitt@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) if you experience issues with permissions.
 
 AWS Region: `US East (N. Virginia) us-east-1`
 
@@ -27,25 +27,47 @@ AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
 
 This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If you are using compute resources in the same region as the S3 Bucket, then there is no cost.
 
-#### Examples
+### Examples
 
-The available versions can be found by running
+**Note:** All examples are based on linux pathing. Also, for each sample below, remove the line breaks [backslash(s) "\"] before running the command.
+
+The available inputs, test cases, and versioned FIM outputs can be found by running:
 ```
 aws s3 ls s3://noaa-nws-owp-fim/hand_fim/ --request-payer requester
 ```
 
-Download a directory of outputs for a single HUC8:
+Download a directory of sample outputs for a single HUC8:
 ```
 aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_3_11_0/12090301 \
     /your_local_folder_name/12090301 --request-payer requester
 ```
-By adjusting pathing, you can also download entire directories such as the fim_4_3_11_0 folder.
-**Note**: There may be newer editions than fim_4_3_11_0, and it is recommended to adjust the command above for the latest version.
+By adjusting pathing, you can also download entire directories such as the `fim_4_3_11_0` folder. An entire output FIM set (e.g. `fim_4_3_11_0`) is approximately 1.1 TB.
 
+**Note**: There may be newer editions than `fim_4_3_11_0`, and it is recommended to adjust the command above for the latest version.
 
-## Running the Code
-### Input Data
-Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP S3 Bucket" section above). All necessary non-publicly available files are in this S3 bucket, as well as sample input data for HUCs 1204 and 1209.
+## Setting up your Environment
+
+### Folder Structure
+You are welcome to set up your folder structure in any pattern you like. For example purposes, we will use a folder structure shown below.
+Starting with a base folder, e.g `/home/projects/` add the following folders:
+- `fim`
+   - `code`
+   - `data`
+      - `inputs`
+      - `outputs`
+      - `outputs_temp`
+### Getting FIM source code
+(based on sample pathing described above)
+
+``` path to your "code" folder. e.g.
+cd /home/projects/fim/code
+```
+
+``` download the current FIM inundation mapping code
+git clone https://github.com/NOAA-OWP/inundation-mapping.git
+```
+
+Git will auto create a subfolder named `inundation-mapping` where the code will be. Your Docker mounts should include this `inundation-mapping` folder. 
 
 ### Dependencies
 [Docker](https://docs.docker.com/get-docker/)
@@ -58,72 +80,84 @@ Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
     - Linux: `chgrp -R fim <path/to/repository>`
 
+### Input Data
+Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP S3 Bucket" section above). The FIM inputs directory can be found at `s3://noaa-nws-owp-fim/hand_fim/inputs`. It is appx 400GB and it needs to be in your `data` folder.
+
+```
+aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/inputs /home/projects/fim/data/inputs --request-payer requester --dry-run
+```
+**Note**: When you include the `--dry-run` argument in the command, a large list will be returned showing you exactly which files are to be downloaded and where they will be saved. We recommend including this argument the first time you run the command, then quickly aborting it (CTRL-C) so you don't get the full list. However, you can see that your chosen target path on your machine is correct.  When you are happy with the pathing, run the `aws s3` command again and leave off the `--dry-run` argument.
+
+The S3 inputs directory has all of the folders and files you need to run FIM. It includes some publicly available and some non-publicly availible data.
+
+## Running the Code
+
 ### Configuration
-This software is configurable via parameters found in the `config` directory. Copy files before editing and remove "template" pattern from the filename.
-Make sure to set the config folder group to 'fim' recursively using the chown command. Each development version will include a calibrated parameter set of manning’s n values.
-- `params_template.env`
+
+There are two ways, which can be used together, to configure the system and/or data processing. Some configuration is based on input arguments when running `fim_pipeline.sh` described below in the "Produce HAND Hydrofabric" section. Another configuration option is based on using a file named `params_template.env`, found in the `config` directory. To use this latter technique, copy the `params_template.env` file before editing and remove the word "template" from the filename. The `params.env` file includes, among other options, a calibrated parameters set of Manning’s n values. The new `params.env` becomes one of the arguments submitted when running `fim_pipeline.sh`.
+
+Make sure to set the config folder group to `fim` recursively using the chown command.
 
 This system has an optional tool called the `calibration database tool`. In order to use this system, you have three options:  
-1.  Install the calibration database service.
+1.  Install the calibration database service, found in the [calibration tools README.md](https://github.com/NOAA-OWP/inundation-mapping/tree/dev/tools/calibration-db).
 2.  Disable it by providing the `-skipcal` command line option to `fim_pipeline.sh` or `fim_preprocessing.sh`.
-3.  Disable it in the `params_template.env` file. See [calibration tool README](https://github.com/NOAA-OWP/inundation-mapping/blob/dev/tools/calibration-db/README.md) for more details.
+3.  Disable it in the `params.env` file. See [calibration tool README](https://github.com/NOAA-OWP/inundation-mapping/blob/dev/tools/calibration-db/README.md) for more details.
 
 ### Start/run the Docker Container
 
-Since all of the dependencies are managed in utilizing a Docker container, we must issue the [`docker run`](https://docs.docker.com/engine/reference/run/#clean-up---rm) command to start a container as the run-time environment. The container is launched from a Docker Image which was built in [Installation](#installation) step 2. The correct input file pathing is necessary for the `/data` volume mount (`-v`) for the `<input_path>`. The `<input_path>` should contain a subdirectory named `/inputs` (similar to `s3://noaa-nws-owp-fim/hand_fim`). If the pathing is set correctly, we do not need to adjust the `params_template.env` file, and can use the default file paths provided.
+Since all of the dependencies are managed by utilizing a Docker container, we must issue the [`docker run`](https://docs.docker.com/engine/reference/commandline/run/) command to start a container as the run-time environment. The container is launched from a Docker image which was built in [Installation](#installation). The `-v <input_path>:/data` must contain a subdirectory named `inputs` (similar to `s3://noaa-nws-owp-fim/hand_fim`). If the pathing is set correctly, we do not need to adjust the `params_template.env` file, and can use the default file paths provided.
+
 
 ```bash 
 docker run --rm -it --name <your_container_name> \
     -v <path/to/repository>/:/foss_fim \
     -v <desired_output_path>/:/outputs \
+    -v <desired_outputs_temp_path>/:/fim_temp \
     -v <input_path>:/data \
     <image_name>:<tag>
 ```
 For example:  
 ```bash
 docker run --rm -it --name robs_container \
-    -v /home/projects/inundation-mapping/:/foss_fim \
-    -v /home/projects/fim/outputs/:/outputs \
-    -v /home/projects/fim/inputs/:/data \
-    fim_4:dev_20230224_ad87a74
+    -v /home/projects/fim/code/inundation-mapping/:/foss_fim \
+    -v /home/projects/fim/data/outputs/:/outputs \
+    -v /home/projects/fim/data/outputs_temp/:/fim_temp \
+    -v /home/projects/fim/data/:/data \
+    fim_4:dev_20230620
 ```
 
 ### Produce HAND Hydrofabric
 ```
-fim_pipeline.sh -u <huc8> -n <name_your_run>
+fim_pipeline.sh -u <huc8> -n <name_your_run> -o
 ```
 - There are a wide number of options and defaulted values, for details run ```fim_pipeline.sh -h```.
-- Manditory arguments:
+- Mandatory arguments:
     - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited (.lst) file. To run the entire domain of available data use one of the ```/data/inputs/included_huc8.lst``` files or a HUC list file of your choice.  Depending on the performance of your server, especially the number of CPU cores, running the full domain can take multiple days.
-    - `-n` is a name of your run (only alphanumeric)
+    - `-n` is a name of your run (only alphanumeric). This becomes the name of the folder in your `outputs` folder.
+    - `-o` is an optional param but means "overwrite". Add this argument if you want to allow the command to overwrite the folder created as part of the `-n` (name) argument.
+    - While not mandatory, if you override the `params_template.env` file, you may want to use the `-c` argument to point to your adjusted file.
 - Outputs can be found under ```/outputs/<name_your_run>```.
 
-Processing of HUC's in FIM4 comes in three pieces. You can run `fim_pipeline.sh` which automatically runs all of three major section, but you can run each of the sections independently if you like. The three sections are:
-- `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
-- `fim_process_unit_wb.sh` : This script processes one and exactly one HUC8 plus all of it's related branches. While it can only process one, you can run this script multiple times, each with different HUC (or overwriting a HUC). When you run `fim_pipeline.sh`, it automatically iterates when more than one HUC number has been supplied either by command line arguments or via a HUC list. For each HUC provided, `fim_pipeline.sh` will `fim_process_unit_wb.sh`. Using the `fim_process_unit_wb.sh`  script allows for a run / rerun of a HUC, or running other HUCs at different times / days or even different docker containers.
-- `fim_post_processing.sh` : This section takes all of the HUCs that have been processed, aggregates key information from each HUC directory and looks for errors across all HUC folders. It also processes the group in sub-steps such as usgs guages processesing, rating curve adjustments and more. Naturally, running or re-running this script can only be done after running `fim_pre_processing.sh` and at least one run of `fim_process_unit_wb.sh`.
+Processing of HUCs in FIM4 occurs in three sections. 
+You can run `fim_pipeline.sh` which automatically runs all of three major section, 
+OR you can run each of the sections independently if you like. 
 
-Running the `fim_pipeline.sh` is a quicker process than running all three steps independently.
+The three sections are:
+1. `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
+2. `fim_process_unit_wb.sh` : This script processes one and exactly one HUC8 plus all of it's related branches. While it can only process one, you can run this script multiple times, each with different HUC (or overwriting a HUC). When you run `fim_pipeline.sh`, it automatically iterates when more than one HUC number has been supplied either by command line arguments or via a HUC list. For each HUC provided, `fim_pipeline.sh` will run  `fim_process_unit_wb.sh`. Using the `fim_process_unit_wb.sh`  script allows for a run / rerun of a HUC, or running other HUCs at different times / days or even different docker containers.
+3. `fim_post_processing.sh` : This section takes all of the HUCs that have been processed, aggregates key information from each HUC directory and looks for errors across all HUC folders. It also processes the HUC group in sub-steps such as usgs guages processesing, rating curve adjustments and more. Naturally, running or re-running this script can only be done after running `fim_pre_processing.sh` and at least one run of `fim_process_unit_wb.sh`.
+
+Running the `fim_pipeline.sh` is a quicker process than running all three steps independently, but you can run some sections more than once if you like.
 
 ### Testing in Other HUCs
 To test in HUCs other than the provided HUCs, the following processes can be followed to acquire and preprocess additional NHDPlus rasters and vectors. After these steps are run, the "Produce HAND Hydrofabric" step can be run for the new HUCs.
 
 ```
-/foss_fim/src/acquire_and_preprocess_inputs.py -u <huc4s_to_process>
+/foss_fim/src/acquire_and_preprocess_inputs.py -u <huc8s_to_process>
 ```
-    Note: This tool is deprecated, updates will be coming soon.
-
-- `-u` can be a single HUC4, series of HUC4s (e.g. 1209 1210), path to line-delimited file with HUC4s.
-- Please run `/foss_fim/src/acquire_and_preprocess_inputs.py --help` for more information.
-- See United States Geological Survey (USGS) National Hydrography Dataset Plus High Resolution (NHDPlusHR) [site](https://www.usgs.gov/core-science-systems/ngp/national-hydrography/nhdplus-high-resolution) for more information
+    Sorry, this tool is deprecated, updates will be coming soon.
 
-#### Reproject NHDPlus High-Res Rasters and Convert to Meters.
-```
-/foss_fim/src/preprocess_rasters.py
-```
-    Note: This tool is deprecated, updates will be coming soon.
 
-----
 ### Evaluating Inundation Map Performance
 After `fim_pipeline.sh` completes, or combinations of the three major steps described above, you can evaluate the model's skill. The evaluation benchmark datasets are available through ESIP in the `test_cases` directory.
 
@@ -151,7 +185,7 @@ When you execute `docker build` from the `Installation` section above, all of th
 
 While very rare, you may want to add more dependencies. You can follow the following steps:
 
-- From inside your docker container, run the following command:
+- From inside your docker container, run the following command from your root directory in your docker container :
     ```bash
     pipenv install <your package name> --dev
     ```
@@ -163,7 +197,8 @@ While very rare, you may want to add more dependencies. You can follow the follo
     pipenv lock
     ```
 
-    This will update the `Pipfile.lock`. Copy the new updated `Pipfile` and `Pipfile.lock` in the source directory and include both in your git commits. The docker image installs the environment from the lock file. 
+    This will update the `Pipfile.lock`. Copy the new updated `Pipfile` and `Pipfile.lock` in the FIM source directory and include both in your git commits. The docker image installs the environment from the lock file. 
+
     
 **Make sure you test it heavily including create new docker images and that it continues to work with the code.**
 

From 85da99fb39090ad5cbbc8778a9d479a56aa1be73 Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 7 Jul 2023 10:24:01 -0600
Subject: [PATCH 256/359] v4.3.12.0 Refactor Calibration Database to use
 .parquet files (#940)

---
 .gitignore                                    |   1 +
 README.md                                     |  11 +-
 config/calb_db_keys_template.env              |  12 -
 config/params_template.env                    |   2 -
 data/write_parquet_from_calib_pts.py          | 307 ++++++++++++++++++
 docs/CHANGELOG.md                             |  31 ++
 fim_post_processing.sh                        |  69 +---
 fim_pre_processing.sh                         |   8 +-
 src/bash_variables.env                        |  28 +-
 src/src_adjust_spatial_obs.py                 | 235 +++++---------
 src/src_roughness_optimization.py             |  18 +-
 tools/calibration-db/README.md                |  83 -----
 tools/calibration-db/docker-compose.yml       |  18 -
 .../docker-entrypoint-initdb.d/init-db.sh     |   6 -
 14 files changed, 469 insertions(+), 360 deletions(-)
 delete mode 100755 config/calb_db_keys_template.env
 create mode 100644 data/write_parquet_from_calib_pts.py
 delete mode 100755 tools/calibration-db/README.md
 delete mode 100755 tools/calibration-db/docker-compose.yml
 delete mode 100755 tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh

diff --git a/.gitignore b/.gitignore
index 7ce22521c..89bae3092 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,4 @@ config/*.swp
 **/.DS_Store
 **/*_pytest.py
 .private/
+
diff --git a/README.md b/README.md
index d6ed398e0..96ed32ced 100644
--- a/README.md
+++ b/README.md
@@ -94,14 +94,13 @@ The S3 inputs directory has all of the folders and files you need to run FIM. It
 
 ### Configuration
 
-There are two ways, which can be used together, to configure the system and/or data processing. Some configuration is based on input arguments when running `fim_pipeline.sh` described below in the "Produce HAND Hydrofabric" section. Another configuration option is based on using a file named `params_template.env`, found in the `config` directory. To use this latter technique, copy the `params_template.env` file before editing and remove the word "template" from the filename. The `params.env` file includes, among other options, a calibrated parameters set of Manning’s n values. The new `params.env` becomes one of the arguments submitted when running `fim_pipeline.sh`.
+There are two ways, which can be used together, to configure the system and/or data processing. Some configuration is based on input arguments when running `fim_pipeline.sh` described below in the "Produce HAND Hydrofabric" section. Another configuration option is based on using a file named `params_template.env`, found in the `config` directory. To use this latter technique, copy the `params_template.env` file before editing and remove the word "template" from the filename. The `params_template.env` file includes, among other options, a calibrated parameters set of Manning’s n values. The new `params.env` becomes one of the arguments submitted when running `fim_pipeline.sh`.
 
 Make sure to set the config folder group to `fim` recursively using the chown command.
 
-This system has an optional tool called the `calibration database tool`. In order to use this system, you have three options:  
-1.  Install the calibration database service, found in the [calibration tools README.md](https://github.com/NOAA-OWP/inundation-mapping/tree/dev/tools/calibration-db).
-2.  Disable it by providing the `-skipcal` command line option to `fim_pipeline.sh` or `fim_preprocessing.sh`.
-3.  Disable it in the `params.env` file. See [calibration tool README](https://github.com/NOAA-OWP/inundation-mapping/blob/dev/tools/calibration-db/README.md) for more details.
+This application has an default optional tool called the `calibration points tool`. In order to disable its' use, you can:  
+1.  Disable it by providing the `-skipcal` command line option to `fim_pipeline.sh` or `fim_pre_processing.sh`.
+2.  Disable it in the [`params_template.env`](/config/params_template.env) file by setting `src_adjust_spatial="FALSE"`.
 
 ### Start/run the Docker Container
 
@@ -208,7 +207,7 @@ If you are on a machine that has a particularly slow internet connection, you ma
 ----
 ## Citing This Work
 
-Please cite this work in your research and projects according to the `CITATION.cff` file found in the root of this repository.
+Please cite this work in your research and projects according to the [`CITATION.cff`](/CITATION.cff) file.
 
 ----
 ### Known Issues & Getting Help
diff --git a/config/calb_db_keys_template.env b/config/calb_db_keys_template.env
deleted file mode 100755
index a6287bb5b..000000000
--- a/config/calb_db_keys_template.env
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# SENSITIVE INFO!!! NOT FOR PUBLIC DISSEMENATION
-
-# Notes:
-#  - The DB_Host name is the name of the server running the calibration db.
-#  - More than one computer can call across the network to one calibraion db server.
-#  - You can change the name of the database if you like, but not recommended.
-CALIBRATION_DB_HOST=
-CALIBRATION_DB_NAME=calibration
-CALIBRATION_DB_USER_NAME=
-CALIBRATION_DB_PASS=
diff --git a/config/params_template.env b/config/params_template.env
index 5c0754368..991ecaad9 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -47,8 +47,6 @@ export nwm_recur_file="/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recur
 #### apply SRC adjustments using observed FIM/flow point database ####
 export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
 export fim_obs_pnt_data="/data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
-#### path to env file with sensitive paths for accessing postgres database ####
-export CALB_DB_KEYS_FILE="/data/config/calb_db_keys.env"
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py
new file mode 100644
index 000000000..ab65c191f
--- /dev/null
+++ b/data/write_parquet_from_calib_pts.py
@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+
+import os
+import time
+import datetime as dt
+import geopandas as gpd
+import argparse
+import logging
+
+from pathlib import Path
+from dotenv import load_dotenv
+from multiprocessing import Pool
+
+######################################################################################################
+#                                                                                                    #
+#    Overview:                                                                                       #
+#      Read two .gpkg files into two separate GeoDataFrames                                          #
+#      Create a new GeoDataFrame per HUC8 with all associated calibration point data                 #
+#      Write each new GeoDataFrame with HUC id and calibration points to a .parquet file             #
+#    Usage:                                                                                          #
+#      This script must be run in a Docker container with the correct volume mount to the /data      #
+#      directory containing the default input files, and optionally, the output path.                #
+#          eg: -v /efs-drives/fim-dev-efs/fim-data/:/data                                            #
+#                                                                                                    #
+######################################################################################################
+
+load_dotenv('/foss_fim/src/bash_variables.env')
+DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
+input_WBD_gdb              = os.getenv('input_WBD_gdb')
+input_calib_points_dir     = os.getenv('input_calib_points_dir')
+
+def __setup_logger(output_dir):
+    # Set logging to file and stderr
+    
+    # The log file will include the date, so be mindful the logs when running this script multiple times on the same day
+    curr_date = dt.datetime.now().strftime("%m_%d_%Y")
+
+    log_file_name = f"write_parquet_from_calib_pts_{curr_date}.log"
+
+    log_file_path = os.path.join(output_dir, log_file_name)
+
+    file_handler = logging.FileHandler(log_file_path)
+    file_handler.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.INFO)
+
+    logger = logging.getLogger()
+    logger.addHandler(file_handler)
+    logger.setLevel(logging.INFO)
+
+    # Print start time
+    dt_string = dt.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    logging.info(f'==========================================================================')
+    logging.info(f"\n write_parquet_from_calib_pts.py")
+    logging.info(f"\n \t Started: {dt_string} \n")
+
+
+def load_WBD_gpkg_into_GDF(WBD_National_gpkg_file):
+    '''
+    The function reads .gpkg file and returns a GeoDataFrame containing HUC boundaries.
+
+    Inputs
+    - WBD_National_gpkg_file:          .gpkg file containing HUC8 boundary polygons
+
+    Outputs
+    - huc_polygons_df:                 GeoDataFrame
+    '''
+    
+    huc_polygons_df = gpd.read_file(WBD_National_gpkg_file, layer='WBDHU8',
+                                    ignore_fields=['tnmid', 'metasourceid', 'sourcedatadesc', 'sourceoriginator',
+                                                    'sourcefeatureid', 'loaddate', 'referencegnis_ids', 'areaacres',
+                                                    'areasqkm', 'states', 'name', 'globalid', 'shape_Length',
+                                                    'shape_Area', 'fimid', 'fossid'])
+
+    return huc_polygons_df
+
+
+def load_fim_obs_points_into_GDF(fim_obs_points_data_file):
+    '''
+    The function reads .gpkg file and returns a GeoDataFrame containing calibration points.
+
+    Inputs
+    - fim_obs_points_data_file:        .gpkg file containing calibration points
+
+    Outputs
+    - fim_obs_point_df:                GeoDataFrame 
+    '''
+    
+    fim_obs_point_df = gpd.read_file(fim_obs_points_data_file, layer='usgs_nws_benchmark_points',
+                                     ignore_fields=['Join_Count', 'TARGET_FID', 'DN', 'ORIG_FID',
+                                                    'ID', 'AreaSqKM', 'Shape_Leng', 'Shape_Area'] )
+
+    return fim_obs_point_df
+
+
+def create_single_huc_gdf_and_write_parquet_file(args):
+    '''
+    The function writes a .parquet file for the 'huc' argument provided.
+
+    Inputs
+    - huc:                  HUC8 to write a parquet file (<HUC8>.parquet)
+    - output_dir:           Directory in which to write the file.
+    - wbd_GDF:              GeoDataFrame containing watershed (HUC8) boundaries.
+    - pnt_GDF:              GeoDataFrame containing points.
+
+    Processing
+    - Find the HUC8 polygon from the wbd_GDF containing all HUC8 boundary polygons.
+    - Create a new GeoDataFrame with all of the calibration points contained within a HUC8 boundary.
+    - Drop index_right created by join and set the CRS projection as set in src/bash_variables.env.
+    - Write the new GeoDataFrame containing all of the points and a new HUC8 column to a .parquet file.
+
+    Outputs
+    - none:                 While this function doesn't 'return' any output, it does write a new file.
+    '''
+    
+    # We have to explicitly unpack the args from pool.map()
+    huc        = args[0]
+    output_dir = args[1]
+    wbd_GDF    = args[2]
+    pnt_GDF    = args[3]
+    
+    # Get the row within the wbd_GDF containing the watershed boundary polygon
+    one_huc = wbd_GDF.loc[wbd_GDF['HUC8'] == huc]
+
+    # Create a new GeoDataFrame with an inner spatial join of all points with a HUC8 boundary.
+    huc_with_points_gdf = pnt_GDF.sjoin(one_huc, how='inner', predicate='within')
+    
+    # If there are no calibration points within the HUC boundary, skip current HUC
+    if len(huc_with_points_gdf) == 0: 
+        logging.info(f'HUC # {huc} does not contain any calibration points, skipping...')
+        time.sleep(.1)
+        return
+    
+    # Drop index_right column created by join
+    huc_with_points_gdf.drop(['index_right'], axis=1, inplace=True)
+    
+    # Set the crs projection
+    huc_gdf = huc_with_points_gdf.set_crs(DEFAULT_FIM_PROJECTION_CRS, allow_override=True)
+
+    # Set filepath and write file
+    parquet_filepath = os.path.join(output_dir, f'{huc}.parquet')
+    huc_gdf.to_parquet(parquet_filepath, index=False)
+    
+    logging.info(f'HUC # {huc} calibration points written to file: \n' \
+        f' \t {output_dir}/{huc}.parquet')
+
+
+def create_parquet_directory(output_dir):
+    '''
+    The function creates an output directory to write the .parquet files if the filepath provided as an argument doesn't exist.
+
+    Inputs
+    - output_dir:               Filepath to create a directory.
+
+    Outputs
+    - none:                     While no return value, a new directory is created if it does not exist.
+    '''
+    
+    if os.path.isdir(output_dir) == False:
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        logging.info(f"Created directory: {output_dir}, .parquet files will be located there.")
+    elif os.path.isdir(output_dir) == True:
+        logging.info(f"Output Direcrtory: {output_dir} exists, .parquet files will be located there.")
+
+
+def create_parquet_files(points_data_file_name,
+                            number_of_jobs,
+                            output_dir=input_calib_points_dir,
+                            wbd_layer=input_WBD_gdb,
+                            huc_list=None,
+                            all_hucs=False):
+    '''
+    The function is the main driver of the program to iterate and parallelize writing HUC8.parquet files.
+
+    Inputs
+    - points_data_file_name:            Filepath to a .gpkg file containing calibration points.
+    - number_of_jobs:                   Amount of cpus used for parallelization.
+    - output_dir:                       Directory to write .parquet files.
+    - wbd_layer:                        Filepath to a .gpkg containing HUC8 boundary polygons.
+    - huc_list:                         Optional: list of hucs to write .parquet files based on points.
+    - all_hucs:                         Optional: use all HUC8s located wbd_layer argument.
+
+    Processing
+    - Validate number_of_jobs based on cpus availabe on current system.
+    - Set up logging.
+    - Create output directory.
+    - Decipher what HUCs to use for the hucs_to_parquet_list based on provided arguments.
+    - Build arguments via procs_list to each parallel call to create_single_huc_gdf_and_write_parquet_file.
+    - Parallelize the processing of GeoDataFrame creation and .parquet writing of the hucs_to_parquet_list 
+        using multiprocessing.Pool.
+
+    Outputs
+    - parquet files:       
+    - log file
+    '''
+    
+    # Validation
+    total_cpus_available = os.cpu_count()
+    if number_of_jobs > total_cpus_available:
+        logging.info(f'Provided: -j {number_of_jobs}, which is greater than than amount of available cpus -1: ' \
+                        f'{total_cpus_available -1} will be used instead.')
+        number_of_jobs = total_cpus_available -1
+    
+    # Set start_time and setup logger
+    start_time = dt.datetime.now()
+    __setup_logger(output_dir)
+    
+    create_parquet_directory(output_dir)
+    
+    logging.info(f'Loading .gpkg files into GeoDataFrames....')
+    huc_polygons_df  = load_WBD_gpkg_into_GDF(wbd_layer)
+    fim_obs_point_df = load_fim_obs_points_into_GDF(points_data_file_name)
+    
+    # Verify same projection
+    if (huc_polygons_df.crs == fim_obs_point_df.crs) is not True:
+        raise ValueError(f'Provided: -p {points_data_file_name} crs & -wbd {wbd_layer} crs do not match, ' \
+                        f'please make adjustments and re-run.')
+    
+    # Print timing
+    load_gdf_end_time = dt.datetime.now()
+    load_duration = load_gdf_end_time - start_time
+    logging.info("Finished loading input .gpkg files into GeoDataFrames.")
+    print(f"\t TIME: {str(load_duration).split('.')[0]} ")
+    
+    # Only use provided HUCs
+    if huc_list is not None:
+        hucs_to_parquet_list = list(huc_list.split(','))
+        
+    # Use all HUCS in WBD file 
+    elif all_hucs:
+        # Define the list of HUCS based on all HUCS in WBD File
+        all_hucs_in_WBD = huc_polygons_df.iloc[:,0]
+        hucs_to_parquet_list = list(all_hucs_in_WBD)
+        
+    # Default HUC list comes from calibration point .parquet files that are pre-existing in current /inputs location
+    else:
+        # Get a list of all files in output_dir
+        current_hucs_in_output_dir = os.listdir(output_dir)
+        # Slice off the .parquet file format to get just the HUC number of all files with the .parquet extension
+        hucs_to_parquet_list = [i[:-8] for i in current_hucs_in_output_dir if i.endswith('.parquet')]
+    
+    logging.info(f"Submitting {len(hucs_to_parquet_list)} HUCS.")
+        
+    # Build arguments (procs_list) for each process (create_single_huc_gdf_and_write_parquet_file)
+    procs_list = []
+    for huc in hucs_to_parquet_list:
+        procs_list.append([huc, output_dir, huc_polygons_df, fim_obs_point_df])
+    
+    # Parallelize each huc in hucs_to_parquet_list
+    logging.info(f'Parallelizing HUC level GeoDataFrame creation and .parquet file writes')
+    with Pool(processes=number_of_jobs) as pool:
+                pool.map(create_single_huc_gdf_and_write_parquet_file, procs_list)
+    
+    # Get time metrics
+    end_time = dt.datetime.now()
+    dt_string = dt.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    logging.info(f"\n Ended: {dt_string} \n")
+
+    # Calculate duration
+    time_duration = end_time - start_time
+    logging.info(f'==========================================================================')
+    logging.info(f"\t Completed writing all .parquet files \n" \
+        f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}")
+    logging.info(f'==========================================================================')
+
+
+if __name__ == '__main__':
+    '''
+    Sample Usage:   python3 /foss_fim/data/write_parquet_from_calib_pts.py                                       \
+                        -p /data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg  \
+                        -j 6                                                                                     \
+                        -o /data/inputs/rating_curve/water_edge_database/calibration_points                      \
+                        -wbd /data/inputs/wbd/WBD_National.gpkg                                                  \
+                        -u "12040103,04100003"
+                        
+                    python3 /foss_fim/data/write_parquet_from_calib_pts.py                                       \
+                        -p /data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg  \
+                        -a
+    '''
+
+    parser = argparse.ArgumentParser(description='Create a parquet file/files with calibration points.')
+    
+    parser.add_argument('-p','--points_data_file_name', help='REQUIRED: Complete relative filepath of a .gpkg file with fim calibration points.', 
+        type=str, required=True)
+    
+    parser.add_argument('-o', '--output_dir', help=f'OPTIONAL: path to send .parquet file/files. Default location specifed as the "input_calib_points_dir" ' \
+        'variable in src/bash_variables.env', type=str, required=False, default=input_calib_points_dir)
+        
+    parser.add_argument('-j','--number_of_jobs', help='OPTIONAL: number of cores/processes (default=4). This is a memory intensive '                         \
+        'script, and the multiprocessing will crash if too many CPUs are used. It is recommended to provide half the amount'                                 \
+        'of available CPUs.', type=int, required=False, default=4)
+
+    parser.add_argument('-wbd','--wbd_layer', help='OPTIONAL: A directory of where a .gpkg file exists, containing HUC boundary polygons. '                  \
+        'Default filepath is specified as the "input_WBD_gdb" variable in src/bash_variables.env', type=str, required=False, default=input_WBD_gdb)
+    
+    parser.add_argument('-u','--huc_list', help='OPTIONAL: HUC list - String with comma seperated HUC numbers. *DO NOT include spaces*.'                     \
+        'Provide certain HUCs if points were added/updated only within a few known HUCs.', type=str, required=False, default=None)
+    
+    parser.add_argument('-a', '--all_hucs', help='OPTIONAL: Provide the -a flag if new calibration points were added to many HUCs which currently'           \
+        'do not have .parquet files.  All HUC polygons in the file provided to -wbd argument will be checked for calibration points contained in '           \
+        '<points_data_file_name>. If not provided, either the HUCs in the huc_list argument (-u) or the current HUCs with files in <output_dir>'             \
+        'will be used.', required=False, action='store_true')
+    
+    args = vars(parser.parse_args())
+
+
+    create_parquet_files(**args)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 297d08d5c..65fb58099 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,37 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.12.0 - 2023-07-05 - [PR#940](https://github.com/NOAA-OWP/inundation-mapping/pull/940)
+
+Refactor Point Calibration Database for synthetic rating curve adjustment to use `.parquet` files instead of a PostgreSQL database. 
+
+### Additions
+- `data/`
+    -`write_parquet_from_calib_pts.py`: Script to write `.parquet` files based on calibration points contained in a .gpkg file. 
+
+### Changes  
+- `src/`
+    - `src_adjust_spatial_obs.py`: Refactor to remove PostgreSQL and use `.parquet` files.
+    - `src_roughness_optimization.py`: Line up comments and add newline at EOF. 
+    - `bash_variables.env`: Update formatting, and add `{}` to inherited `.env` variables for proper variable expansion in Python scripts.  
+- `/config`
+    - `params_template.env`: Update comment.
+- `fim_pre_processing.sh`: In usage statement, remove references to PostGRES calibration tool.
+- `fim_post_processing.sh`: Remove connection to and loading of PostgreSQL database. 
+- `.gitignore`: Add newline.
+- `README.md`: Remove references to PostGRES calibration tool.
+
+### Removals
+- `config/` 
+    - `calb_db_keys_template.env`: No longer necessary without PostGRES Database.
+
+- `/tools/calibration-db` : Removed directory including files below. 
+    - `README.md`
+    - `docker-compose.yml`
+    - `docker-entrypoint-enitdb.d/init-db.sh`
+
+<br/><br/>
+
 ## v4.3.11.7 - 2023-06-12 - [PR#932](https://github.com/NOAA-OWP/inundation-mapping/pull/932)
 
 Write to a csv file with processing time of `run_unit_wb.sh`, update PR Template, add/update bash functions in `bash_functions.env`, and modify error handling in `src/check_huc_inputs.py`. Update unit tests to throw no failures, `25 passed, 3 skipped`.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 632055553..c3eb4415c 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -55,20 +55,17 @@ if [ ! -d "$outputDestDir" ]; then
     exit 1
 fi
 
-
-#############################################
-#
-# PLEASE DO NOT USE the job limits coming in from the runtime_args.env
-# Most of the time, post processing will not be run on the same servers
-# that is running fim_process_unit_wb.sh and the processing power
-# used to run fim_post_processing.sh will be different (hence.. different job limit)
-#
-#############################################
+#########################################################################################
+#                                                                                       #
+# PLEASE DO NOT USE the job limits coming in from the runtime_args.env                  #
+# Most of the time, post processing will not be run on the same servers                 #
+# that is running fim_process_unit_wb.sh and the processing power                       #
+# used to run fim_post_processing.sh will be different (hence.. different job limit)    #
+#                                                                                       #
+#########################################################################################
 
 if [ "$jobLimit" = "" ]; then jobLimit=1; fi
 
-
-
 # Clean out the other post processing files before starting
 rm -rdf $outputDestDir/logs/src_optimization
 rm -f $outputDestDir/logs/log_bankfull_indentify.log
@@ -83,7 +80,6 @@ source $outputDestDir/params.env
 source $srcDir/bash_functions.env
 source $srcDir/bash_variables.env
 
-
 echo
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 echo "---- Start of fim_post_processing"
@@ -133,56 +129,11 @@ if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$s
     date -u
 fi
 
-## CONNECT TO CALIBRATION POSTGRESQL DATABASE (OPTIONAL) ##
-if [ "$src_adjust_spatial" = "True" ] && [ "$skipcal" = "0" ]; then
-    if [ ! -f $CALB_DB_KEYS_FILE ]; then
-        echo "ERROR! - the src_adjust_spatial parameter in the params_template.env (or equiv) is set to "True" (see parameter file),"
-        echo "but the provided calibration database access keys file does not exist: $CALB_DB_KEYS_FILE"
-        exit 1
-    else
-        source $CALB_DB_KEYS_FILE
-
-        ## This makes the local variables from the calb_db_keys files
-        ## into global variables that can be used in other files, including python.
-
-        ## Why not just leave the word export in front of each of the keys in the
-        ## calb_db_keys.env? Because that file is used against docker-compose        
-        ## when we start up that part of the sytem and it does not like the word
-        ## export.
-
-        # Pick up the docker parent host machine name and override the one coming from the config file (aws only)
-        if [ "$isAWS" = "1" ]; then
-            CALIBRATION_DB_HOST=$(curl http://169.254.169.254/latest/meta-data/local-ipv4 -s)
-        fi
-
-        export CALIBRATION_DB_HOST=$CALIBRATION_DB_HOST
-        export CALIBRATION_DB_NAME=$CALIBRATION_DB_NAME
-        export CALIBRATION_DB_USER_NAME=$CALIBRATION_DB_USER_NAME
-        export CALIBRATION_DB_PASS=$CALIBRATION_DB_PASS
-        export DEFAULT_FIM_PROJECTION_CRS=$DEFAULT_FIM_PROJECTION_CRS
-
-        Tstart
-        echo
-        echo -e $startDiv"Populate PostgreSQL database with benchmark FIM extent points and HUC attributes (the calibration database)"
-        echo
-
-        ogr2ogr -overwrite -nln hucs -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $inputsDir/wbd/WBD_National.gpkg WBDHU8
-
-        echo "Loading Point Data"
-        echo
-        ogr2ogr -overwrite -t_srs $DEFAULT_FIM_PROJECTION_CRS -f PostgreSQL PG:"host=$CALIBRATION_DB_HOST dbname=$CALIBRATION_DB_NAME user=$CALIBRATION_DB_USER_NAME password=$CALIBRATION_DB_PASS" $fim_obs_pnt_data usgs_nws_benchmark_points -nln points
-
-        Tcount
-    fi
-else
-    echo "Skipping Populate PostgrSQL database"
-fi
-
-## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINT DATABASE (POSTGRESQL) ##
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINTS (.parquet files) ##
 if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [ "$skipcal" = "0" ]; then
     Tstart
     echo
-    echo -e $startDiv"Performing SRC adjustments using benchmark point database"
+    echo -e $startDiv"Performing SRC adjustments using benchmark point .parquet files"
     python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputDestDir -j $jobLimit
     Tcount
     date -u
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 3fd2fbda7..05f87da10 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -11,13 +11,13 @@ usage ()
     echo '          -zd <branch zero deny list file>'
     echo '          -jh <number of jobs for HUCs>'
     echo '          -jb <number of jobs for branches>'
-    echo '          -skipcal <skip the PostGres calibration process>'
+    echo '          -skipcal <skip the calibration points process>'
     echo '          -isaws ]'
     echo ''
     echo 'REQUIRED:'
-    echo '  -u/--hucList    : HUC8s to run or multiple passed in quotes (space delimited) file.'
+    echo '  -u/--hucList    : HUC8s to run; more than one HUC8 should be passed in quotes (space delimited).'
     echo '                    A line delimited file, with a .lst extension, is also acceptable.'
-    echo '                    HUCs must present in inputs directory.'
+    echo '                    HUC8s must present in inputs directory.'
     echo '  -n/--runName    : A name to tag the output directories and log files.'
     echo 
     echo 'OPTIONS:'
@@ -49,7 +49,7 @@ usage ()
     echo '                         NOTE: Make sure that the multiplication of jh and jb subtract 2 (jh x jb -2) does not'
     echo '                         exceed the total number of cores available.'
     echo '  -o               : Overwrite outputs if already exist'
-    echo '  -skipcal         : If this param is included, post gres calibration system will be skipped'    
+    echo '  -skipcal         : If this param is included, updating the S.R.C. via the calibration points will be skipped'    
     echo '  -isaws           : If this param is included, the code will use AWS objects where possible'
     echo '                   : Note: This feature is not yet implemented'
     echo
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 46403c153..b581dc439 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -1,18 +1,20 @@
 ## Define inputs
+
 # NOTE: $inputsDir is defined in Dockerfile
-export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
-export input_DEM=$inputsDir/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
-export input_DEM_domain=$inputsDir/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
-export input_GL_boundaries=$inputsDir/landsea/gl_water_polygons.gpkg
-export input_NLD=$inputsDir/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
-export input_levees_preprocessed=$inputsDir/nld_vectors/3d_nld_preprocessed_230314.gpkg
-export input_nld_levee_protected_areas=$inputsDir/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
-export input_nwm_catchments=$inputsDir/nwm_hydrofabric/nwm_catchments.gpkg
-export input_nwm_flows=$inputsDir/nwm_hydrofabric/nwm_flows.gpkg
-export input_nwm_headwaters=$inputsDir/nwm_hydrofabric/nwm_headwaters.gpkg
-export input_nwm_lakes=$inputsDir/nwm_hydrofabric/nwm_lakes.gpkg
-export input_WBD_gdb=$inputsDir/wbd/WBD_National.gpkg
+export      DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
+export                       input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
+export                input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
+export             input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
+export                       input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
+export       input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg
+export input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
+export            input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg
+export                 input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
+export            input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
+export                 input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
+export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National.gpkg
+export          input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
 
 # Styling
 export startDiv="\n-----------------------------------------------------------------\n"
-export stopDiv="\n-----------------------------------------------------------------\n"
+export  stopDiv="\n-----------------------------------------------------------------\n"
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index 1f2d08f1d..df41e40cb 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -3,32 +3,24 @@
 import argparse
 import datetime as dt
 import geopandas as gpd
-#import json
 import multiprocessing
-import numpy as np
 import os
-import pandas as pd
-import psycopg2 # python package for connecting to postgres
 import rasterio
 import sys
-import time
 
-from collections import deque
 from dotenv import load_dotenv
-from geopandas.tools import sjoin
 from multiprocessing import Pool
 from src_roughness_optimization import update_rating_curve
 from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH, DEFAULT_FIM_PROJECTION_CRS
 
-#import variables from .env file
-load_dotenv()
-CALIBRATION_DB_HOST = os.getenv("CALIBRATION_DB_HOST")
-CALIBRATION_DB_NAME = os.getenv("CALIBRATION_DB_NAME")
-CALIBRATION_DB_USER_NAME = os.getenv("CALIBRATION_DB_USER_NAME")
-CALIBRATION_DB_PASS = os.getenv("CALIBRATION_DB_PASS")
+# Import variables from .env file
+load_dotenv('/foss_fim/src/bash_variables.env')
+outputsDir             = os.getenv("outputsDir")
+input_calib_points_dir = os.getenv("input_calib_points_dir")
 
 '''
-The script imports a PostgreSQL database containing observed FIM extent points and associated flow data. This script attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow.
+The script imports .parquet files per HUC8 containing observed FIM extent points and associated flow data. 
+This script attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow.
 
 Processing
 - Define CRS to use for initial geoprocessing and read wbd_path and points_layer.
@@ -50,9 +42,9 @@
 '''
 
 def process_points(args):
-
     '''
-    The function ingests geodataframe and attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow
+    This function ingests geodataframe and attributes the point data with its hydroid and HAND values before passing 
+    a dataframe to the src_roughness_optimization.py workflow.
 
     Processing
     - Extract x,y coordinates from geometry
@@ -74,7 +66,7 @@ def process_points(args):
     ## Define coords variable to be used in point raster value attribution.
     coords = [(x,y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
 
-    water_edge_df.to_crs(DEFAULT_FIM_PROJECTION_CRS)
+    water_edge_df = water_edge_df.to_crs(DEFAULT_FIM_PROJECTION_CRS)
     
     ## Use point geometry to determine HAND raster pixel values.
     with rasterio.open(hand_path) as hand_src, rasterio.open(catchments_path) as catchments_src:
@@ -114,6 +106,7 @@ def process_points(args):
         log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc, 
                                        branch_id, catchments_poly_path, optional_outputs, 
                                        source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
+        
         ## Still testing: use code below to print out any exceptions.
         '''
         try:
@@ -125,109 +118,91 @@ def process_points(args):
     return(log_text)
 
 
-def find_points_in_huc(huc_id, conn):
-    # Point data in the database is already attributed with HUC8 id
+def find_points_in_huc(huc_id):
     '''
-    The function queries the PostgreSQL database and returns all points attributed with the input huc id.
+    This function loads the .parquet file containing all points attributed with the input huc id into a GeoDataFrame.
 
     Processing
-    - Query the PostgreSQL database for points attributed with huc id.
-    - Reads the filtered database result into a pandas geodataframe
+    - Query the <input_calib_points_dir> directory for a <HUC8>.parquet file containing calibration points.
+    - Reads points contained in .parquet file into a pandas geodataframe
 
     Inputs
-    - conn:         connection to PostgreSQL db
-    - huc_id:       HUC id to query the db
+    - huc_id:        HUC id to find in <input_calib_points_dir>
 
     Outputs
     - water_edge_df: geodataframe with point data
     '''
 
-    huc_pt_query = """SELECT ST_X(P.geom), ST_Y(P.geom), P.submitter, P.flow, P.coll_time, P.flow_unit, P.layer, P.geom 
-    FROM points P 
-    JOIN hucs H ON ST_Contains(H.geom, P.geom)
-    WHERE H.huc8 = %s """
+    # The CRS Projection is set when initially writing .parquet files (call to script: write_parquet_from_calib_pts.py)
+    water_edge_filepath = os.path.join(input_calib_points_dir, f'{huc_id}.parquet')
     
-    # Use EPSG:5070 instead of the default ESRI:102039 (gdal pyproj throws an error with crs 102039)
-    # Appears that EPSG:5070 is functionally equivalent to ESRI:102039: https://gis.stackexchange.com/questions/329123/crs-interpretation-in-qgis
-    water_edge_df = gpd.GeoDataFrame.from_postgis(huc_pt_query, con=conn, 
-                                                  params=[huc_id], crs=DEFAULT_FIM_PROJECTION_CRS,
-                                                  parse_dates=['coll_time'])
-    water_edge_df = water_edge_df.drop(columns=['st_x','st_y'])
+    water_edge_df = gpd.read_parquet(water_edge_filepath)
     
     return water_edge_df
 
-def find_hucs_with_points(conn, fim_out_huc_list):
-    '''
-    The function queries the PostgreSQL database and returns a list of all the HUCs that contain calb point data.
-
-    Processing
-    - Query the PostgreSQL database for all unique huc ids
 
-    Inputs
-    - conn:         connection to PostgreSQL db
-
-    Outputs
-    - hucs_wpoints: list with all unique huc ids
+def find_hucs_with_points(points_file_dir, fim_out_huc_list):
     '''
-
-    cursor = conn.cursor()
+    This function queries a directory with .parquet files of HUCs containing calibration points (generated from /data/write_parquet_from_calib_pts.py)
+    and returns a list of all the HUCs in <fim_out_huc_list> that contain calibration point data.
     '''
-    cursor.execute("""
-        SELECT DISTINCT H.huc8
-        FROM points P JOIN hucs H ON ST_Contains(H.geom, P.geom);
-    """)
-    '''
-    cursor.execute("SELECT DISTINCT H.huc8 FROM points P JOIN hucs H ON ST_Contains(H.geom, P.geom) WHERE H.huc8 = ANY(%s);", (fim_out_huc_list,))
-    hucs_fetch = cursor.fetchall() # list with tuple with the attributes defined above (need to convert to df?)
-    hucs_wpoints = []
-    for huc in hucs_fetch:
-        hucs_wpoints.append(huc[0])
-    cursor.close()
+
+    files_in_points_file_dir = os.listdir(points_file_dir)
+    
+    # Use list comprehension to slice .parquet off filename, and also prune non-parquet files in directory 
+    hucs_in_points_file_dir = [i[:-8] for i in files_in_points_file_dir if i.endswith('.parquet')]
+    
+    # Use list comprehension to only keep hucs in both the points_file_dir & fim_out_huc_list
+    hucs_wpoints = [x for x in hucs_in_points_file_dir if x in fim_out_huc_list]
+    
     return hucs_wpoints
 
+
 def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_file):
     '''
-    The function obtains all points within a given huc, locates the corresponding FIM output files for each huc (confirms all necessary files exist), and then passes a proc list of huc organized data to process_points function.
-
-    Processing
-    - Query the PostgreSQL database for all unique huc ids that have calb points
-    - Loop through all HUCs with calb data and locate necessary fim output files to pass to calb workflow
+    The function obtains all points within a given huc, locates the corresponding FIM output files for each huc
+    (confirms all necessary files exist), and then passes a proc list of huc organized data to process_points function.
 
     Inputs
     - fim_directory:        parent directory of fim ouputs (contains HUC directories)
     - job_number:           number of multiprocessing jobs to use for processing hucs
     - debug_outputs_option: optional flag to output intermediate files
+    - log_file:             where stdout/stderr will be logged
 
-    Outputs
+    Processing
+    - Query the <input_calib_points_dir> for all unique huc ids that have calb points
+    - Loop through all HUCs with calib data and locate necessary fim output files to pass to calib workflow
+    
     - procs_list:           passes multiprocessing list of input args for process_points function input
+
+    Outputs
+    - log_file:             where stdout/stderr will be logged
     '''
-    
-    log_file.write('Connecting to database via host\n')    
-    conn = connect() # Connect to the PostgreSQL db once
-    
-    if (conn is None):
-        msg = "unable to connect to calibration db\n"
-        print(msg)
-        log_file.write(msg)
-        return
-    
-    log_file.write('Connected to database via host\n')
+
     print("Finding all fim_output hucs that contain calibration points...")
     fim_out_huc_list  = [ item for item in os.listdir(fim_directory) if os.path.isdir(os.path.join(fim_directory, item)) ]
 
+    # Remove logs, unit_errors, and branch_errors folders if they exist in <fim_directory>
     fim_out_huc_list.remove('logs')
+    if 'unit_errors' in fim_out_huc_list:
+        fim_out_huc_list.remove('unit_errors')
+    if 'branch_errors' in fim_out_huc_list:
+        fim_out_huc_list.remove('branch_errors')
+    
     ## Record run time and close log file
     run_time_start = dt.datetime.now()
     log_file.write('Finding all hucs that contain calibration points...' + '\n')
-    huc_list_db = find_hucs_with_points(conn, fim_out_huc_list)
+    huc_list_db = find_hucs_with_points(input_calib_points_dir, fim_out_huc_list)
+    
     run_time_end = dt.datetime.now()
     task_run_time = run_time_end - run_time_start
+    
     log_file.write('HUC SEARCH TASK RUN TIME: ' + str(task_run_time) + '\n')
-    print(f"{len(huc_list_db)} hucs found in point database" + '\n')
-    log_file.write(f"{len(huc_list_db)} hucs found in point database" + '\n')
+    print(f"{len(huc_list_db)} hucs found in point file directory" + '\n')
+    log_file.write(f"{len(huc_list_db)} hucs found in point file directory" + '\n')
     log_file.write('#########################################################\n')
 
-    ## Ensure HUC id is either HUC8
+    # Ensure HUC id has 8 characters
     huc_list = []
     for huc in huc_list_db:
         ## zfill to the appropriate scale to ensure leading zeros are present, if necessary.
@@ -237,34 +212,39 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
             huc_list.append(huc)
             log_file.write(str(huc) + '\n')
 
-    procs_list = []  # Initialize proc list for mulitprocessing.
+    # Initialize process list for multiprocessing.
+    procs_list = []
 
-    #huc_list = ['12040103']
+    # huc_list = ['12040103'] # Uncomment for testing
+    # Sort huc_list for helping track progress in future print statments
+    huc_list.sort() 
     ## Define paths to relevant HUC HAND data.
-    huc_list.sort() # sort huc_list for helping track progress in future print statments
     for huc in huc_list:
-        huc_branches_dir = os.path.join(fim_directory, huc,'branches')
-        water_edge_df = find_points_in_huc(huc, conn).reset_index()
+        huc_branches_dir = os.path.join(fim_directory, huc, 'branches')
+        water_edge_df = find_points_in_huc(huc)
         print(f"{len(water_edge_df)} points found in " + str(huc))
         log_file.write(f"{len(water_edge_df)} points found in " + str(huc) + '\n')
 
         ## Create X and Y location columns by extracting from geometry.
-        water_edge_df['X'] = water_edge_df['geom'].x
-        water_edge_df['Y'] = water_edge_df['geom'].y
+        water_edge_df['X'] = water_edge_df['geometry'].x
+        water_edge_df['Y'] = water_edge_df['geometry'].y
 
         ## Check to make sure the HUC directory exists in the current fim_directory
         if not os.path.exists(os.path.join(fim_directory, huc)):
             log_file.write("FIM Directory for huc: " + str(huc) + " does not exist --> skipping SRC adjustments for this HUC (obs points found)\n")
-
+        
         ## Intermediate output for debugging
         if debug_outputs_option:
             huc_debug_pts_out = os.path.join(fim_directory, huc, 'debug_water_edge_df_' + huc + '.csv')
             water_edge_df.to_csv(huc_debug_pts_out)
             huc_debug_pts_out_gpkg = os.path.join(fim_directory, huc, 'export_water_edge_df_' + huc + '.gpkg')
             water_edge_df.to_file(huc_debug_pts_out_gpkg, driver='GPKG', index=False)
+            # write parquet file using ".to_parquet() method" 
+            parquet_filepath = os.path.join(fim_directory, huc, 'debug_water_edge_df_' + huc + '.parquet')
+            water_edge_df.to_parquet(parquet_filepath, index=False)
         
         for branch_id in os.listdir(huc_branches_dir):
-            branch_dir = os.path.join(huc_branches_dir,branch_id)
+            branch_dir = os.path.join(huc_branches_dir, branch_id)
             ## Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
             hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
             catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
@@ -290,58 +270,18 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
                 log_file.writelines(["%s\n" % item  for item in log_output])
                 
     log_file.write('#########################################################\n')
-    disconnect(conn) # move this to happen at the end of the huc looping
 
-def connect():
-    """ Connect to the PostgreSQL database server """
-
-    print('Connecting to the PostgreSQL database...')
-    conn = None
-    not_connected = True
-    fail_ctr = 0
-    while not_connected and fail_ctr < 6:
-        try:
-
-            # connect to the PostgreSQL server
-            conn = psycopg2.connect(
-                host=CALIBRATION_DB_HOST,
-                database=CALIBRATION_DB_NAME,
-                user=CALIBRATION_DB_USER_NAME,
-                password=CALIBRATION_DB_PASS)
-
-            # create a cursor
-            cur = conn.cursor()
-
-            # execute a statement
-            print('Host name: ' + CALIBRATION_DB_HOST)
-            print('PostgreSQL database version:')
-            cur.execute('SELECT version()')
-
-            # display the PostgreSQL database server version
-            db_version = cur.fetchone()
-            print(db_version)
-
-               # close the communication with the PostgreSQL
-            cur.close()
-            not_connected = False
-            print("Connected to database\n\n")
-            
-        except (Exception, psycopg2.DatabaseError) as error:
-            print("Waiting for database to come online")
-            fail_ctr += 1
-            time.sleep(5)
-
-    return conn
-
-def disconnect(conn):
-    """ Disconnect from the PostgreSQL database server """
-
-    if conn is not None:
-        conn.close()
-        print('Database connection closed.')
 
 def run_prep(fim_directory, debug_outputs_option, ds_thresh_override, DOWNSTREAM_THRESHOLD, job_number):
+    '''
+    Main function to call the processing functions defined above, with validation, logging, and timing
     
+    Validation:
+        - fim_directory exists
+        - job_number does not exceed available cpus. 
+        - ds_thresh_override value is different than defualy and warn user
+    '''
+
     assert os.path.isdir(fim_directory), 'ERROR: could not find the input fim_dir location: '\
         + str(fim_directory)
 
@@ -386,29 +326,28 @@ def run_prep(fim_directory, debug_outputs_option, ds_thresh_override, DOWNSTREAM
     sys.stdout = sys.__stdout__
     log_file.close()
 
+
 if __name__ == '__main__':
     ## Parse arguments.
-    parser = argparse.ArgumentParser(description='Adjusts rating curve given a shapefile containing points of known water boundary.')
-    #parser.add_argument('-db','--points-layer',help='Path to points layer containing known water boundary locations',required=True)
-    parser.add_argument('-fim_dir','--fim-directory',
+    parser = argparse.ArgumentParser(description=f'Adjusts rating curve based on files in {input_calib_points_dir}, containing points of known water boundary.')
+    parser.add_argument('-fim_dir', '--fim-directory',
                         help='Parent directory of FIM-required datasets.', required=True)
-    parser.add_argument('-debug','--extra-outputs',
+    parser.add_argument('-debug', '--extra-outputs',
                         help='OPTIONAL flag: Use this to keep intermediate output files for debugging/testing',
                         default=False, required=False, action='store_true')
-    parser.add_argument('-dthresh','--downstream-thresh',
+    parser.add_argument('-dthresh', '--downstream-thresh',
                         help='OPTIONAL Override: distance in km to propogate modified roughness values downstream', 
                         default=DOWNSTREAM_THRESHOLD, required=False)
-    parser.add_argument('-j','--job-number',
-                        help='OPTIONAL: Number of jobs to use', required=False,
+    parser.add_argument('-j', '--job-number',
+                        help='OPTIONAL: Number of jobs to use', type=int, required=False,
                         default=2)
 
     ## Assign variables from arguments.
-    args = vars(parser.parse_args())
-    #points_layer = args['points_layer']
-    fim_directory = args['fim_directory']
+    args                 = vars(parser.parse_args())
+    fim_directory        = args['fim_directory']
     debug_outputs_option = args['extra_outputs']
-    ds_thresh_override = args['downstream_thresh']
-    job_number = int(args['job_number'])
+    ds_thresh_override   = args['downstream_thresh']
+    job_number           = args['job_number']
 
     run_prep(fim_directory, debug_outputs_option, ds_thresh_override, 
              DOWNSTREAM_THRESHOLD, job_number)
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index 9be4b397e..356b69826 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -326,12 +326,12 @@ def group_manningn_calc(df_nmerge, down_dist_thresh):
     ## Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg
     #df_nmerge.sort_values(by=['NextDownID'], inplace=True)
     dist_accum = 0; hyid_count = 0; hyid_accum_count = 0; 
-    run_accum_mann = 0; group_calb_coef = 0; branch_start = 1                                        # initialize counter and accumulation variables
+    run_accum_mann = 0; group_calb_coef = 0; branch_start = 1                                       # initialize counter and accumulation variables
     lid_count = 0; prev_lid = 'x'
     for index, row in df_nmerge.iterrows():                                                         # loop through the df (parse by hydroid)
         if int(df_nmerge.loc[index,'branch_id']) != branch_start:                                   # check if start of new branch
             dist_accum = 0; hyid_count = 0; hyid_accum_count = 0;                                   # initialize counter vars
-            run_accum_mann = 0; group_calb_coef = 0                                                  # initialize counter vars
+            run_accum_mann = 0; group_calb_coef = 0                                                 # initialize counter vars
             branch_start = int(df_nmerge.loc[index,'branch_id'])                                    # reassign the branch_start var to evaluate on next iteration
             # use the code below to withold downstream hydroid_calb_coef values (use this for downstream evaluation tests)
             '''
@@ -346,14 +346,14 @@ def group_manningn_calc(df_nmerge, down_dist_thresh):
                 lid_count = 1
             prev_lid = df_nmerge.loc[index,'ahps_lid']
             '''
-        if np.isnan(df_nmerge.loc[index,'hydroid_calb_coef']):                                       # check if the hydroid_calb_coef value is nan (indicates a non-calibrated hydroid)
+        if np.isnan(df_nmerge.loc[index,'hydroid_calb_coef']):                                      # check if the hydroid_calb_coef value is nan (indicates a non-calibrated hydroid)
             df_nmerge.loc[index,'accum_dist'] = row['LENGTHKM'] + dist_accum                        # calculate accumulated river distance
             dist_accum += row['LENGTHKM']                                                           # add hydroid length to the dist_accum var
             hyid_count = 0                                                                          # reset the hydroid counter to 0
             df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hydroid accum counter
-            if dist_accum < down_dist_thresh:                                                   # check if the accum distance is less than Xkm downstream from valid hydroid_calb_coef group value
+            if dist_accum < down_dist_thresh:                                                       # check if the accum distance is less than Xkm downstream from valid hydroid_calb_coef group value
                 if hyid_accum_count > 1:                                                            # only apply the group_calb_coef if there are 2 or more valid hydorids that contributed to the upstream group_calb_coef
-                    df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                          # output the group_calb_coef var
+                    df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                        # output the group_calb_coef var
             else:
                 run_avg_mann = 0                                                                    # reset the running average manningn variable (greater than 10km downstream)
         else:                                                                                       # performs the following for hydroids that have a valid hydroid_calb_coef value
@@ -361,10 +361,10 @@ def group_manningn_calc(df_nmerge, down_dist_thresh):
             df_nmerge.loc[index,'accum_dist'] = 0                                                   # output the accum_dist value (set to 0)
             if hyid_count == 1:                                                                     # checks if this the first in a series of valid hydroid_calb_coef values
                 run_accum_mann = 0; hyid_accum_count = 0                                            # initialize counter and running accumulated manningN value
-            group_calb_coef = (row['hydroid_calb_coef'] + run_accum_mann)/float(hyid_count)           # calculate the group_calb_coef (NOTE: this will continue to change as more hydroid values are accumulated in the "group" moving downstream)
-            df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                                  # output the group_calb_coef var 
+            group_calb_coef = (row['hydroid_calb_coef'] + run_accum_mann)/float(hyid_count)         # calculate the group_calb_coef (NOTE: this will continue to change as more hydroid values are accumulated in the "group" moving downstream)
+            df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                                # output the group_calb_coef var 
             df_nmerge.loc[index,'hyid_count'] = hyid_count                                          # output the hyid_count var 
-            run_accum_mann += row['hydroid_calb_coef']                                               # add current hydroid manningn value to the running accum mann var
+            run_accum_mann += row['hydroid_calb_coef']                                              # add current hydroid manningn value to the running accum mann var
             hyid_accum_count += 1                                                                   # increase the # of hydroid accum counter
             df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hyid_accum_count var
 
@@ -372,4 +372,4 @@ def group_manningn_calc(df_nmerge, down_dist_thresh):
     if 'hyid_count' in df_nmerge.columns:
         df_nmerge.drop(['hyid_count','accum_dist','hyid_accum_count'], axis=1, inplace=True, errors='ignore') # drop hydroid counter if it exists
     #df_nmerge.drop(['accum_dist','hyid_accum_count'], axis=1, inplace=True) # drop accum vars from group calc
-    return(df_nmerge)
\ No newline at end of file
+    return(df_nmerge)
diff --git a/tools/calibration-db/README.md b/tools/calibration-db/README.md
deleted file mode 100755
index 6235d877c..000000000
--- a/tools/calibration-db/README.md
+++ /dev/null
@@ -1,83 +0,0 @@
-## The calibration database service
-
-The **calibration-db tool** is an optional use tool. When started, it creates a service, via a docker container, that creates a postgres database. The calibration-db tool will make minor updates to hydrotables.csv's, but only to applicable HUCs. The applicable HUCs generally have AHPS library data or good USGS guage data.
-
--------------------------------
-To setup the system, it needs to be done in two parts.
-
-**A)** Update the `/config/params_template.env` file and set `src_adjust_spatial` to "True".  If you are not using the tool, set the flag to "False".
-
-**B)** Setup the calibration database service. See steps below.
-
--------------------------------
-## Creating the calibration database service
-
-When you start the service, it will start up a new docker container named "fim_calibration_db", if enabled in the params_template.env.
-
-To run commands, use a bash terminal window.
-
-Steps:
-1. Copy `/config/calb_db_keys_template.env` and rename it to a name of your choice (ie: `calb_db_keys.env` or whatever). We recommend saving the file outside the fim source code folder for security purposes. 
-
-2. Update the values in the new .env file. 
-	Note: The `CALIBRATION_DB_HOST` is the computer/server where the docker container is physically running. This may be your server name, but can also be a different server if you like. One calibration database service can be shared by multiple other servers, each with their own FIM source code folders (a "client" if you will). Each "client" server can have it's own calb_db_keys.env file and/or can share the .env with other servers.  ie) dev-1 and prod-1 can share a .env file if they have a shared drive.
-	
-3. Start the service from the /tools/calibration-db directory of the server that will run the calibration database service. 
-   Using a bash terminal window, run
-	  `docker-compose --env-file {path to .env file}/{your key file name} up --build -d` 
-	  ie) docker-compose --env-file /my_server/config/calb_db_keys.env up --build -d
-
-	   - If you get an error of permissions denied, you might need to upgrade the permissions to your calibration-db folder. From your /tools directory, run `chmod -R 777 calibration-db` (yes.. 777).  You may need to add the word "sudo" at the front of the command, depending on your system configuration.
-	   - If you get an error saying `command not found: docker compose`, you may need to install it via `sudo apt install docker-compose` or check your system configuration.
-
-4. You should be able to see a container named `fim_calibration_db` via a bash terminal command of `docker stats --no-stream`.
-
--------------------------------
-## Destroying the database (from the /tools/calibration-db directory)
-
-Using a bash terminal window, run
-    `docker-compose down`
-    `rm -rf pgdata`
-  
-If that command errors out, use `docker-compose down --rmi local -v --remove-orphans`. 
- 
--------------------------------
-## Debugging Postgres DB
-
-Use the following command to connect to the postgres DB (using values from your .env file):
-
-`docker run -it --rm postgis/postgis psql -h CALIBRATION_DB_HOST -U CALIBRATION_DB_USER_NAME` (and follow prompt to enter password (CALIBRATION_DB_PASS))
-ie) `docker run -it --rm postgis/postgis psql -h my_server_name -U fim_postgres` (and follow prompt to enter password (CALIBRATION_DB_PASS))
-
-### View names of databases (you can have more than one database in a postgres database server. ie) calibation and calibration-test.
-
-`\l`
-
-    #### Note: FIM will create a database as it is being used. If you have not yet processed any HUCs, there will not be a database yet.
-
-### Connect to calibration DB  
-
-`\c calibration`  (or the name of the database you gave in the calib_db_keys.env (CALIBRATION_DB_NAME))
-
-### View tables in DB (now connected)
-
-`\dt`
-
-### View columns in tables
-
-`\d+ hucs`
-`\d+ points`
-
-### View number of rows in a table
-
-'select count(*) from table_name;`  (you need the ";" on the end)
-
-Postgres uses psql. See [https://www.postgresql.org/docs/current/sql-commands.html]9https://www.postgresql.org/docs/current/sql-commands.html) for details on commands.
-
-
-## System Notes
-
-The `docker-entrypoint-initdb.d` scripts only run once, when the DB is initially created. To rerun these scripts, you will need to destroy the database.
-The `docker-entrypoint-initdb.d` scripts do not create the tables for the database, but rather simply enables the PostGIS extension.
-If you are trying to connect to a database that is on the same server that you are working on, you have to use the full servername and not `localhost`.
-
diff --git a/tools/calibration-db/docker-compose.yml b/tools/calibration-db/docker-compose.yml
deleted file mode 100755
index 36c962140..000000000
--- a/tools/calibration-db/docker-compose.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-version: '3.5'
-services:
-  calibration-db:
-    image: postgis/postgis
-    container_name: fim_calibration_db
-    restart: always
-    environment:
-      - POSTGRES_USER=${CALIBRATION_DB_USER_NAME}
-      - POSTGRES_PASSWORD=${CALIBRATION_DB_PASS}
-      - POSTGRES_DB=${CALIBRATION_DB_NAME}
-      - PGDATA=/var/lib/postgresql/data/fim/pgdata
-    ports:
-      - "5432:5432"
-    volumes:
-      - calibration_db_data:/var/lib/postgresql/data/fim
-      - ./docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
-volumes:
-    calibration_db_data:
diff --git a/tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh b/tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh
deleted file mode 100755
index c60ee4d18..000000000
--- a/tools/calibration-db/docker-entrypoint-initdb.d/init-db.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-set -e
-
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-  CREATE EXTENSION postgis;
-EOSQL

From 2e99cc09caf9496dd325992e0f004e48f9552bfb Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 2 Aug 2023 18:25:31 -0400
Subject: [PATCH 257/359] 4.3.12.1 Fix for aggregated hydrotable inundation
 (#950)

---
 docs/CHANGELOG.md                          |  16 +++
 tools/inundate_gms.py                      | 132 ++++++++++-----------
 unit_tests/tools/inundate_gms_params.json  |   8 +-
 unit_tests/tools/inundate_gms_test.py      |  59 ++++++---
 unit_tests/tools/inundate_gms_unittests.py |  83 -------------
 5 files changed, 131 insertions(+), 167 deletions(-)
 delete mode 100644 unit_tests/tools/inundate_gms_unittests.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 65fb58099..0d48be9fa 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.12.1 - 2023-07-21 - [PR#950](https://github.com/NOAA-OWP/inundation-mapping/pull/950)
+
+Fixes a couple of bugs that prevented inundation using HUC-level hydrotables. Update associated unit tests.
+
+### Changes
+
+- `tools/inundate_gms.py`: Fixes a file path error and Pandas DataFrame indexing error.
+- `unit_tests/tools/inundate_gms_test.py`: Do not skip this test, refactor to check that all branch inundation rasters exist.
+- `unit_tests/tools/inundate_gms_params.json`: Only test 1 HUC, update forecast filepath, use 4 'workers'.
+
+### Removals
+
+- `unit_tests/tools/inundate_gms_unittests.py`: No longer used. Holdover from legacy unit tests.
+
+<br/><br/>
+
 ## v4.3.12.0 - 2023-07-05 - [PR#940](https://github.com/NOAA-OWP/inundation-mapping/pull/940)
 
 Refactor Point Calibration Database for synthetic rating curve adjustment to use `.parquet` files instead of a PostgreSQL database. 
diff --git a/tools/inundate_gms.py b/tools/inundate_gms.py
index b0f5363f6..89e6d9cee 100755
--- a/tools/inundate_gms.py
+++ b/tools/inundate_gms.py
@@ -8,17 +8,17 @@
 
 from tqdm import tqdm
 from inundation import inundate
-from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor,as_completed
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from inundation import hydroTableHasOnlyLakes, NoForecastFound
 from utils.shared_functions import FIM_Helpers as fh
 
-def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
-                  hucs = None,
-                  inundation_raster = None,
-                  inundation_polygon = None, depths_raster = None,
-                  verbose = False,
-                  log_file = None,
-                  output_fileNames = None ):
+def Inundate_gms( hydrofabric_dir, forecast, num_workers=1,
+                  hucs=None,
+                  inundation_raster=None,
+                  inundation_polygon=None, depths_raster=None,
+                  verbose=False,
+                  log_file=None,
+                  output_fileNames=None ):
 
     # input handling
     if hucs is not None:
@@ -27,7 +27,7 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
         except TypeError:
             raise ValueError("hucs argument must be an iterable")
     
-    if isinstance(hucs,str):
+    if isinstance(hucs, str):
         hucs = [hucs]
 
     num_workers = int(num_workers)
@@ -38,27 +38,26 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
             os.remove(log_file)
 
         if verbose :
-            print('HUC8,BranchID,Exception',file=open(log_file,'w'))
+            print('HUC8,BranchID,Exception', file=open(log_file, 'w'))
     #if log_file:
         #logging.basicConfig(filename=log_file, level=logging.INFO)
         #logging.info('HUC8,BranchID,Exception')
 
     # load fim inputs
-    hucs_branches = pd.read_csv( os.path.join(hydrofabric_dir,'fim_inputs.csv'),
+    hucs_branches = pd.read_csv( os.path.join(hydrofabric_dir, 'fim_inputs.csv'),
                                  header=None,
-                                 dtype= {0:str,1:str} )
+                                 dtype= {0:str, 1:str} )
     
     if hucs is not None:
         hucs = set(hucs)
-        huc_indices = hucs_branches.loc[:,0].isin(hucs)
-        hucs_branches = hucs_branches.loc[huc_indices,:]
+        huc_indices = hucs_branches.loc[:, 0].isin(hucs)
+        hucs_branches = hucs_branches.loc[huc_indices, :]
 
     # get number of branches
     number_of_branches = len(hucs_branches)
     
     # make inundate generator
     inundate_input_generator = __inundate_gms_generator( hucs_branches,
-                                                         number_of_branches,
                                                          hydrofabric_dir,
                                                          inundation_raster,
                                                          inundation_polygon,
@@ -68,7 +67,7 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
 
     # start up process pool
     # better results with Process pool
-    executor = ProcessPoolExecutor(max_workers = num_workers)
+    executor = ProcessPoolExecutor(max_workers=num_workers)
 
     # collect output filenames
     inundation_raster_fileNames = [None] * number_of_branches
@@ -78,12 +77,12 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
     branch_ids = [None] * number_of_branches
        
     executor_generator = { 
-                executor.submit(inundate,**inp) : ids for inp,ids in inundate_input_generator 
+                executor.submit(inundate, **inp) : ids for inp, ids in inundate_input_generator 
                 }          
     idx = 0
     for future in tqdm(as_completed(executor_generator),
                        total=len(executor_generator),
-                       desc="Inundating branches with {} workers".format(num_workers),
+                       desc=f"Inundating branches with {num_workers} workers",
                        disable=(not verbose) ):
         
         hucCode, branch_id = executor_generator[future]
@@ -94,21 +93,21 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
         except NoForecastFound as exc:
             if log_file is not None:
                 print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
-                      file=open(log_file,'a'))
+                      file=open(log_file, 'a'))
             elif verbose:
                 print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
 
         except hydroTableHasOnlyLakes as exc:
             if log_file is not None:
                 print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
-                      file=open(log_file,'a'))
+                      file=open(log_file, 'a'))
             elif verbose:
                 print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
         
         except Exception as exc:
             if log_file is not None:
                 print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
-                      file=open(log_file,'a'))
+                      file=open(log_file, 'a'))
             else:
                 print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
         else:
@@ -138,76 +137,76 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers = 1,
     executor.shutdown(wait=True)
 
     # make filename dataframe
-    output_fileNames_df = pd.DataFrame( { 'huc8' : hucCodes,
-                                          'branchID' : branch_ids,
-                                          'inundation_rasters' : inundation_raster_fileNames,
-                                          'depths_rasters' : depths_raster_fileNames,
-                                          'inundation_polygons' : inundation_polygon_fileNames } )
+    output_fileNames_df = pd.DataFrame( { 'huc8': hucCodes,
+                                          'branchID': branch_ids,
+                                          'inundation_rasters': inundation_raster_fileNames,
+                                          'depths_rasters': depths_raster_fileNames,
+                                          'inundation_polygons': inundation_polygon_fileNames } )
 
     if output_fileNames is not None:
-        output_fileNames_df.to_csv(output_fileNames,index=False)
+        output_fileNames_df.to_csv(output_fileNames, index=False)
 
     return(output_fileNames_df)
 
 
 def __inundate_gms_generator( hucs_branches,
-                              number_of_branches,
                               hydrofabric_dir,
                               inundation_raster,
                               inundation_polygon,
                               depths_raster,
-                              forecast, verbose = False ):
+                              forecast, verbose=False ):
 
-    # iterate over branches
-    for idx,row in hucs_branches.iterrows():
+    # Iterate over branches
+    for idx, row in hucs_branches.iterrows():
         
         huc = str(row[0])
         branch_id = str(row[1])
 
-        huc_dir = os.path.join(hydrofabric_dir, huc, 'branches')
+        huc_dir = os.path.join(hydrofabric_dir, huc)
+        branch_dir = os.path.join(huc_dir, 'branches', branch_id)
 
-        rem_file_name = 'rem_zeroed_masked_{}.tif'.format(branch_id)
-        rem_branch = os.path.join( huc_dir, branch_id, rem_file_name )
+        rem_file_name = f'rem_zeroed_masked_{branch_id}.tif'
+        rem_branch = os.path.join( branch_dir, rem_file_name )
 
         catchments_file_name = f'gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif'
-        catchments_branch = os.path.join( huc_dir, branch_id, catchments_file_name )
+        catchments_branch = os.path.join( branch_dir, catchments_file_name )
         
         # FIM versions > 4.3.5 use an aggregated hydrotable file rather than individual branch hydrotables
-        hydroTable_huc = os.path.join( huc_dir, 'hydrotable.csv' )
+        hydroTable_huc = os.path.join(huc_dir, 'hydrotable.csv')
         if os.path.isfile(hydroTable_huc):
-            htable_req_cols = ['HUC','branch_id','feature_id','HydroID','stage','discharge_cms','LakeID']
+            htable_req_cols = ['HUC', 'branch_id', 'feature_id', 'HydroID', 'stage', 'discharge_cms', 'LakeID']
             hydroTable_all = pd.read_csv(
                                  hydroTable_huc,
-                                 dtype={'HUC':str,'branch_id':int,'feature_id':str,
-                                         'HydroID':str,'stage':float,
-                                         'discharge_cms':float,'LakeID' : int},
+                                 dtype={'HUC':str, 'branch_id':int, 'feature_id':str,
+                                         'HydroID':str, 'stage':float,
+                                         'discharge_cms':float, 'LakeID': int},
                                 usecols=htable_req_cols
                                 )
+            hydroTable_all.set_index(['HUC', 'feature_id', 'HydroID'], inplace=True)
             hydroTable_branch = hydroTable_all.loc[hydroTable_all['branch_id'] == int(branch_id)]
         else:
             # Earlier FIM4 versions only have branch level hydrotables
-            hydroTable_branch = os.path.join( huc_dir, branch_id, 'hydroTable_{}.csv'.format(branch_id) )
+            hydroTable_branch = os.path.join( branch_dir, f'hydroTable_{branch_id}.csv' )
 
         xwalked_file_name = f'gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg'
-        catchment_poly = os.path.join( huc_dir, branch_id, xwalked_file_name )
-        
+        catchment_poly = os.path.join( branch_dir, xwalked_file_name )
     
         # branch output
         # Some other functions that call in here already added a huc, so only add it if not yet there
         if (inundation_raster is not None) and (huc not in inundation_raster):
-            inundation_branch_raster = fh.append_id_to_file_name(inundation_raster,[huc,branch_id])
+            inundation_branch_raster = fh.append_id_to_file_name(inundation_raster, [huc, branch_id])
         else:
-            inundation_branch_raster = fh.append_id_to_file_name(inundation_raster,branch_id)
+            inundation_branch_raster = fh.append_id_to_file_name(inundation_raster, branch_id)
 
         if (inundation_polygon is not None) and (huc not in inundation_polygon):
-            inundation_branch_polygon = fh.append_id_to_file_name(inundation_polygon,[huc,branch_id])
+            inundation_branch_polygon = fh.append_id_to_file_name(inundation_polygon, [huc, branch_id])
         else:
-            inundation_branch_polygon = fh.append_id_to_file_name(inundation_polygon,branch_id)
+            inundation_branch_polygon = fh.append_id_to_file_name(inundation_polygon, branch_id)
 
         if (depths_raster is not None) and (huc not in depths_raster):
-            depths_branch_raster = fh.append_id_to_file_name(depths_raster,[huc,branch_id])
+            depths_branch_raster = fh.append_id_to_file_name(depths_raster, [huc, branch_id])
         else:
-            depths_branch_raster = fh.append_id_to_file_name(depths_raster,branch_id)
+            depths_branch_raster = fh.append_id_to_file_name(depths_raster, branch_id)
 
         # identifiers
         identifiers = (huc,branch_id)
@@ -215,26 +214,27 @@ def __inundate_gms_generator( hucs_branches,
         #print(f"inundation_branch_raster is {inundation_branch_raster}")
 
         # inundate input
-        inundate_input = {  'rem' : rem_branch, 
-                            'catchments' : catchments_branch, 
-                            'catchment_poly' : catchment_poly,
-                            'hydro_table' : hydroTable_branch,
-                            'forecast' : forecast,
-                            'mask_type' : 'filter',
-                            'hucs' : None,
-                            'hucs_layerName' : None,
-                            'subset_hucs' : None, 
-                            'num_workers' : 1,
-                            'aggregate' : False,
-                            'inundation_raster' : inundation_branch_raster,
-                            'inundation_polygon' : inundation_branch_polygon,
-                            'depths' : depths_branch_raster,
-                            'out_raster_profile' : None,
-                            'out_vector_profile' : None,
-                            'quiet' : not verbose }
+        inundate_input = {  'rem': rem_branch, 
+                            'catchments': catchments_branch, 
+                            'catchment_poly': catchment_poly,
+                            'hydro_table': hydroTable_branch,
+                            'forecast': forecast,
+                            'mask_type': 'filter',
+                            'hucs': None,
+                            'hucs_layerName': None,
+                            'subset_hucs': None, 
+                            'num_workers': 1,
+                            'aggregate': False,
+                            'inundation_raster': inundation_branch_raster,
+                            'inundation_polygon': inundation_branch_polygon,
+                            'depths': depths_branch_raster,
+                            'out_raster_profile': None,
+                            'out_vector_profile': None,
+                            'quiet': not verbose }
         
         yield (inundate_input, identifiers)
 
+
 if __name__ == '__main__':
 
     # parse arguments
diff --git a/unit_tests/tools/inundate_gms_params.json b/unit_tests/tools/inundate_gms_params.json
index 13445b514..4b52ba45e 100644
--- a/unit_tests/tools/inundate_gms_params.json
+++ b/unit_tests/tools/inundate_gms_params.json
@@ -2,10 +2,10 @@
 	"valid_data_inudation_raster_single_huc":
 	{
 		"hydrofabric_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
-		"forecast": "/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv",
-		"num_workers": 1,
-		"hucs": "02020005, 05030104",
-	    "inundation_raster": "/data/outputs/fim_unit_test_data_do_not_remove/inundation_unittest_02020005_gms.tif",
+		"forecast": "/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_2_0_cms.csv",
+		"num_workers": 4,
+		"hucs": "02020005",
+		"inundation_raster": "/data/outputs/fim_unit_test_data_do_not_remove/inundation_extent_02020005.tif",
 		"inundation_polygon": null,
 		"depths_raster": null,
 		"verbose": true,
diff --git a/unit_tests/tools/inundate_gms_test.py b/unit_tests/tools/inundate_gms_test.py
index 9e0f195d6..a44b79858 100644
--- a/unit_tests/tools/inundate_gms_test.py
+++ b/unit_tests/tools/inundate_gms_test.py
@@ -5,12 +5,13 @@
 
 import json
 import unittest
+import pandas as pd
 import pytest
 
 sys.path.append('/foss_fim/unit_tests/')
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-sys.path.append('/foss_fim/tools/gms_tools')
+sys.path.append('/foss_fim/tools')
 import inundate_gms as src
 
 
@@ -27,18 +28,25 @@ def setUpClass(self):
             self.params = json.load(params_file)
 
 # Test Cases:
-    @pytest.mark.skip(reason="Inundate_gms will be rebuilt in the future, so this test will be left broken.")
     def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self):
-
         '''
-        Test for creating a gms inundation raster, not a depth raster and no
+        Test for creating a gms inundation rasters, not a depth raster and no
         inundation_polygons.
-        This test is based on creating a raster based on a single huc and its branches
-        within a gms output folder.
+        
+        This test is essentially testing the Inundate_gms function, and the creation of a 
+        the "output_fileNames" .csv file.
         '''
-
-        params = self.params["valid_data_inudation_raster_single_huc"].copy()         
         
+        params = self.params["valid_data_inudation_raster_single_huc"].copy()         
+
+        # Clear previous outputs, if they exist.
+        for file in os.listdir(params["hydrofabric_dir"]):
+            if file.endswith(".tif"):
+                os.remove(os.path.join(params["hydrofabric_dir"], file))
+                
+        os.remove(params["output_fileNames"])
+                
+        # Test the Inundate_gms function
         output_fileNames_df = src.Inundate_gms(hydrofabric_dir = params["hydrofabric_dir"],
                                                forecast = params["forecast"],
                                                num_workers = params["num_workers"],
@@ -47,15 +55,38 @@ def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self
                                                inundation_polygon = params["inundation_polygon"],
                                                depths_raster = params["depths_raster"],
                                                verbose = params["verbose"], 
-                                            #    log_file = None,
-                                            #    output_fileNames = params["output_fileNames"]
+                                               log_file = None,
+                                               output_fileNames = params["output_fileNames"]
                                                )
 
-        # check if output files df has records.
+        # Check if output files df has records.
         assert len(output_fileNames_df) > 0, "Expected as least one dataframe record"
 
-        # also check output log file and output raster. Can't... there will be multiple outputs
-        assert os.path.exists(params["inundation_raster"]), "Inundation Raster does not exist"
+    def test_Inundate_gms_create_inundation_rasters(self):
+
+        '''
+        Test for creating a gms inundation rasters, not a depth raster and no
+        inundation_polygons.
+        This test is based on creating a raster based on a single huc and its branches
+        within the output folder.
+        '''
+        
+        params = self.params["valid_data_inudation_raster_single_huc"].copy() 
+        
+        # Check all output rasters exist.
+        csv_out = pd.read_csv(params["output_fileNames"], skipinitialspace=True)
+        
+        csv_rasters = csv_out.inundation_rasters
+        
+        actual_rasters = []
+        for file in os.listdir(params["hydrofabric_dir"]):
+            if file.endswith(".tif"):
+                actual_rasters.append(file)
+        
+        for csv_raster, actual_raster in zip(csv_rasters, actual_rasters):
+            assert os.path.exists(os.path.join(params["hydrofabric_dir"], actual_raster)), f"Inundation Raster {csv_raster} does not exist"
+
 
-        assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
+    # Logging removed from tools/inundate_gms.py as of 8/1/23
+    # assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
 
diff --git a/unit_tests/tools/inundate_gms_unittests.py b/unit_tests/tools/inundate_gms_unittests.py
deleted file mode 100644
index ade2bfa30..000000000
--- a/unit_tests/tools/inundate_gms_unittests.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import os
-import sys
-
-import json
-import warnings
-import unittest
-
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-sys.path.append('/foss_fim/tools')
-import inundate_gms as src
-
-# NOTE: This goes directly to the function.
-# Ultimately, it should emulate going through command line (not import -> direct function call)
-class test_inundate_gms(unittest.TestCase):
-
-    '''
-    Allows the params to be loaded one and used for all test methods
-    '''
-    @classmethod
-    def setUpClass(self):
-
-        warnings.simplefilter('ignore')
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-
-    def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self):
-
-        '''
-        Test for creating a gms inundation raster, not a depth raster and no
-        inundation_polygons.
-        This test is based on creating a raster based on a single huc and its branches
-        within a gms output folder.
-        '''
-
-        params = self.params["valid_data_inudation_raster_single_huc"].copy()         
-        
-        output_fileNames_df = src.Inundate_gms(hydrofabric_dir = params["hydrofabric_dir"],
-                                               forecast = params["forecast"],
-                                               num_workers = params["num_workers"],
-                                               hucs = params["hucs"],
-                                               inundation_raster = params["inundation_raster"],
-                                               inundation_polygon = params["inundation_polygon"],
-                                               depths_raster = params["depths_raster"],
-                                               verbose = params["verbose"], 
-                                               log_file = params["log_file"],
-                                               output_fileNames = params["output_fileNames"])
-
-        # check if output files df has records.
-        assert len(output_fileNames_df) > 0, "Expected as least one dataframe record"
-
-        # also check output log file and output raster. Can't... there will be multiple outputs
-        #assert os.path.exists(params["inundation_raster"]), "Inundation Raster does not exist"
-
-        assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
-       
-       
-        print(f"Test Success: {inspect.currentframe().f_code.co_name}")
-        print("*************************************************************")
-
-        
-    # ***********************
-
-
-if __name__ == '__main__':
-
-    script_file_name = os.path.basename(__file__)
-
-    print("*****************************")
-    print(f"Start of {script_file_name} tests")
-    print()
-   
-    unittest.main()
-    
-    print()    
-    print(f"End of {script_file_name} tests")
-    

From 17f92971db57a09af4df89419920e553ab46dc44 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 2 Aug 2023 18:29:36 -0400
Subject: [PATCH 258/359] v4.3.13.0 Manually calibrate synthetic rating curves
 (#952)

---
 config/params_template.env    |   4 +
 docs/CHANGELOG.md             |  17 ++++
 fim_post_processing.sh        |  10 +++
 src/src_manual_calibration.py | 146 ++++++++++++++++++++++++++++++++++
 4 files changed, 177 insertions(+)
 create mode 100644 src/src_manual_calibration.py

diff --git a/config/params_template.env b/config/params_template.env
index 991ecaad9..19f8dd4a4 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -48,6 +48,10 @@ export nwm_recur_file="/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recur
 export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
 export fim_obs_pnt_data="/data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 
+#### apply manual calibration ####
+export manual_calb_toggle="True" # Toggle to run manual calibration routine (True=on; False=off)
+export man_calb_file="/data/inputs/rating_curve/manual_calibration_coefficients.csv" # input file location with HUC, nwm feature_id and manual calibration coefficients
+
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
 export ncores_fd=1 # mpi number of cores for flow directions
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0d48be9fa..4f8ea5d9f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.13.0 - 2023-07-26 - [PR#952](https://github.com/NOAA-OWP/inundation-mapping/pull/952)
+
+Adds a feature to manually calibrate rating curves for specified NWM `feature_id`s using a CSV of manual coefficients to output a new rating curve. Manual calibration is applied after any/all other calibrations. Coefficient values between 0 and 1 increase the discharge value (and decrease inundation) for each stage in the rating curve while values greater than 1 decrease the discharge value (and increase inundation).
+
+Manual calibration is performed if `manual_calb_toggle="True"` and the file specified by `man_calb_file` (with `HUC8`, `feature_id`, and `calb_coef_manual` fields) exists. The original HUC-level `hydrotable.csv` (after calibration) is saved with a suffix of `_pre-manual` before the new rating curve is written.
+
+### Additions
+
+- `src/src_manual_calibration.py`: Adds functionality for manual calibration by CSV file
+
+### Changes
+
+- `config/params_template.env`: Adds `manual_calb_toggle` and `man_calb_file` parameters
+- `fim_post_processing.sh`: Adds check for toggle and if `man_calb_file` exists before running manual calibration
+
+<br/><br/>
+
 ## v4.3.12.1 - 2023-07-21 - [PR#950](https://github.com/NOAA-OWP/inundation-mapping/pull/950)
 
 Fixes a couple of bugs that prevented inundation using HUC-level hydrotables. Update associated unit tests.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index c3eb4415c..921c4b7d2 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -147,6 +147,16 @@ python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -htable -
 Tcount
 date -u
 
+## PERFORM MANUAL CALIBRATION
+if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
+    echo
+    echo -e $startDiv"Performing manual calibration"
+    Tstart
+    python3 $srcDir/src_manual_calibration.py -fim_dir $outputDestDir -calb_file $man_calb_file
+    Tcount
+    date -u
+fi
+
 echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
diff --git a/src/src_manual_calibration.py b/src/src_manual_calibration.py
new file mode 100644
index 000000000..6067229b2
--- /dev/null
+++ b/src/src_manual_calibration.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import numpy as np
+import pandas as pd
+
+htable_dtypes = {'HydroID':int,
+                'branch_id':int,
+                'feature_id':int,
+                'NextDownID':int,
+                'order_':int,
+                'Number of Cells':int,
+                'SurfaceArea (m2)':float,
+                'BedArea (m2)':float,
+                'TopWidth (m)':float,
+                'LENGTHKM':float,
+                'AREASQKM':float,
+                'WettedPerimeter (m)':float,
+                'HydraulicRadius (m)':float,
+                'WetArea (m2)':float,
+                'Volume (m3)':float,
+                'SLOPE':float,
+                'ManningN':float,
+                'stage':float,
+                'default_discharge_cms':float,
+                'default_Volume (m3)':float,
+                'default_WetArea (m2)':float,
+                'default_HydraulicRadius (m)':float,
+                'default_ManningN':float,
+                'calb_applied':bool,
+                'last_updated':str,
+                'submitter':str,
+                'obs_source':str,
+                'precalb_discharge_cms':float,
+                'calb_coef_usgs':float,
+                'calb_coef_spatial':float,
+                'calb_coef_final':float,
+                'HUC':str,
+                'LakeID':int,
+                'subdiv_applied':bool,
+                'channel_n':float,
+                'overbank_n':float,
+                'subdiv_discharge_cms':float,
+                'discharge_cms':float}
+
+def manual_calibration(fim_directory:str, calibration_file:str):
+    """
+    Perform manual calibration.
+     
+    Use a CSV of coefficients to output a new rating curve. 
+    Coefficient values between 0 and 1 increase the discharge value 
+    (and decrease inundation) for each stage in the rating curve while 
+    values greater than 1 decrease the discharge value (and increase 
+    inundation).
+    
+    The original rating curve is saved with a suffix of '_pre-manual' 
+    before the new rating curve is written.
+
+    Parameters
+    ----------
+    fim_directory : str
+        Path to the parent directory of FIM-required datasets.
+    calibration_file : str
+        Path to the manual calibration file. This file should be a CSV 
+        with the following columns:
+            HUC8: str
+                HUC8 code
+            feature_id: int
+                NWM feature_id
+            calb_coef_manual: float
+                Manual calibration coefficient for each HUC8 and 
+                feature_id combination.
+    """
+
+    # Get list of HUCs
+    fim_inputs = pd.read_csv(os.path.join(fim_directory, 'fim_inputs.csv'), 
+                             header=None,
+                             names=['HUC', 'branch_id'],
+                             dtype={'HUC':str, 'branch_id':int})
+
+    fim_inputs_hucs = fim_inputs['HUC'].unique()
+
+    # Read manual calibration table
+    if os.path.exists(calibration_file):
+        manual_calib_df = pd.read_csv(calibration_file, dtype={'HUC8':str,
+                                                               'feature_id':int,
+                                                               'calb_coef_manual':float})
+
+        if manual_calib_df['calb_coef_manual'].min() >= 0:
+            # Find HUCs with manual calibration coefficients
+            calib_hucs = manual_calib_df['HUC8'].unique()
+            hucs = np.intersect1d(fim_inputs_hucs, calib_hucs)
+
+            for huc in hucs:
+                print(f'Updating hydrotable for {huc}')
+
+                # Read hydrotable
+                htable_file = os.path.join(fim_directory, huc, 'hydrotable.csv')
+                htable_file_split = os.path.splitext(htable_file)
+                htable_file_original = htable_file_split[0] + '_pre-manual' + htable_file_split[1]
+
+                if not os.path.exists(htable_file_original):
+                    # Save a copy of the original hydrotable
+                    os.rename(htable_file, htable_file_original)
+
+                df_htable = pd.read_csv(htable_file_original, dtype=htable_dtypes)
+
+                df_htable = df_htable.rename(columns={'discharge_cms':'postcalb_discharge_cms'})
+
+                df_htable = df_htable.merge(manual_calib_df, how='left', on='feature_id')
+                df_htable.drop(columns=['HUC8'], inplace=True)
+
+                # Calculate new discharge_cms with manual calibration coefficient
+                df_htable['discharge_cms'] = np.where(df_htable['calb_coef_manual'].isnull(), 
+                                                      df_htable['postcalb_discharge_cms'], 
+                                                      df_htable['postcalb_discharge_cms']/df_htable['calb_coef_manual'])
+
+                # Write new hydroTable.csv rating curve (overwrites the previous file)
+                df_htable.to_csv(htable_file, index=False)
+
+        else:
+            raise ValueError(f'Manual calibration coefficients must be greater than 0. Minimum value found: {manual_calib_df["calb_coef_manual"].min()}')
+
+    else:
+        raise FileNotFoundError(f'No calibration file found at {calibration_file}. Skipping manual calibration.')
+
+
+if __name__ == '__main__':
+    ## Parse arguments.
+    parser = argparse.ArgumentParser(description=f'Manually calibrate rating curve')
+    parser.add_argument('-fim_dir', '--fim-directory',
+                        help='Parent directory of FIM-required datasets.', required=True)
+    parser.add_argument('-calb_file', '--calibration-file',
+                        help='Path to manual calibration file', required=True)
+
+    # args = parser.parse_args()
+    args                 = vars(parser.parse_args())
+    fim_directory        = args['fim_directory']
+    calibration_file     = args['calibration_file']
+
+    manual_calibration(fim_directory, calibration_file)
+
+    # Usage example:
+    # python /foss_fim/src/src_manual_calibration.py -fim_dir /outputs/dev-manual-calibration \
+    # -calb_file /data/inputs/rating_curve/manual_calibration_coefficients.csv

From f3c0dc6284172363bd13289f91c3e704c2dbd8f5 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Thu, 3 Aug 2023 15:23:27 -0500
Subject: [PATCH 259/359] v4.3.14.0 Bathymetry adjustment routine (#953)

---
 config/params_template.env               |   3 +
 data/bathymetry/preprocess_bathymetry.py | 130 ++++++++++++
 docs/CHANGELOG.md                        |  18 ++
 fim_post_processing.sh                   |   9 +
 src/add_crosswalk.py                     |  15 +-
 src/aggregate_by_huc.py                  |   4 +-
 src/bathymetric_adjustment.py            | 243 +++++++++++++++++++++++
 src/identify_src_bankfull.py             |   2 +-
 src/subdiv_chan_obank_src.py             |   3 +-
 9 files changed, 415 insertions(+), 12 deletions(-)
 create mode 100644 data/bathymetry/preprocess_bathymetry.py
 create mode 100644 src/bathymetric_adjustment.py

diff --git a/config/params_template.env b/config/params_template.env
index 19f8dd4a4..a4c1c9ac6 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -31,6 +31,9 @@ export branch_zero_id="0"
 export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from DEM
 export levee_id_attribute=SYSTEM_ID
 
+#### apply bathymetry adjustment to rating curve ####
+export bathymetry_adjust=True
+
 #### estimating bankfull stage in SRCs ####
 export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
 export bankfull_flows_file="/data/inputs/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv" # input file location with nwm feature_id and recurrence flow values
diff --git a/data/bathymetry/preprocess_bathymetry.py b/data/bathymetry/preprocess_bathymetry.py
new file mode 100644
index 000000000..560ef24f2
--- /dev/null
+++ b/data/bathymetry/preprocess_bathymetry.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+import rasterio
+import pandas as pd
+import geopandas as gpd
+import numpy as np
+from rasterstats import zonal_stats
+from osgeo import gdal
+from osgeo import gdal_array
+from argparse import ArgumentParser
+import os
+
+"""
+To use this script you will need to create a tif file from the eHydro depth survey tin file.
+    To create depth tif:
+        1. Download eHydro survey data from USACE Hydrographic Surveys and open the tin file in Q-GIS.
+        2. Run rasterize mesh tool on tin file, e.g.
+        processing.run("native:meshrasterize", {'INPUT':'ESRI_TIN:"C:/Users/riley.mcdermott/
+        Documents/Bathymetry_PA/AL_LP_EMS_20150809_CS_005_065_SORT/AL_LP_EMS_20150809_CS_005_065_SORT_tin/
+        tdenv9.adf"','DATASET_GROUPS':[0],'DATASET_TIME':{'type': 'static'},'EXTENT':None,'PIXEL_SIZE':1,
+        'CRS_OUTPUT':QgsCoordinateReferenceSystem('EPSG:5070'),'OUTPUT':'C:/Users/riley.mcdermott/Documents
+        /Bathymetry_PA/Allegheny_USACE_Depth_ft.tif'})
+"""
+def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_threshold):
+    """ Function for calculating missing reach averaged cross sectional 
+    area and wetted perimeter by NWM feature_id.
+
+        Parameters
+        ----------
+        tif : str
+            Path to depth tif file from rasterized USACE eHydro survey tin, e.g.
+            "/data/inputs/bathymetry/HU_01_TRY_20211108_CS_5043_30.tif".
+        bathy_bounds : str
+            Name of survey extent vector layer within given geodatabase, 
+            defaults to 'SurveyJob'.
+        survey_gdb : str
+            Path to survey geodatabase containing vector survey extent layer, e.g.
+            "/data/inputs/bathymetry/HU_01_TRY_20211108_CS_5043_30.gdb".
+        output : str
+            Path to output location for NWM flows geopackage, e.g.
+            "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
+        min_depth_threshold : int
+            Highest value allowed for the minimum depth of the survey 
+            in feet (Checks for data referenced to a datum).
+
+    """
+    with rasterio.open(tif) as bathy_ft:
+        bathy_affine = bathy_ft.transform
+        bathy_ft = bathy_ft.read(1)
+        bathy_ft[np.where(bathy_ft == -9999.)] = np.nan
+    survey_min_depth = np.nanmin(bathy_ft)
+    assert survey_min_depth < min_depth_threshold, f"The minimum depth value of the survey is {survey_min_depth} which exceeds the minimum depth threshold. This may indicate depth values are based on a datum." 
+    assert survey_min_depth > 0, f"The minimum depth value of the survey is {survey_min_depth}, negative values may indicate a problem with the datum conversion or survey."
+    bathy_m = bathy_ft/3.28084
+    bathy_gdal = gdal_array.OpenArray(bathy_m)
+
+    # Read in shapefiles
+    bathy_bounds = gpd.read_file(survey_gdb, layer = bathy_bounds)
+    nwm_streams = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_flows.gpkg", mask = bathy_bounds)
+    nwm_catchments = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", mask = bathy_bounds)
+    bathy_bounds = bathy_bounds.to_crs(nwm_streams.crs)
+
+    # Find missing volume from depth tif
+    zs_area = zonal_stats(nwm_catchments, bathy_m, stats = ["sum"], affine= bathy_affine, geojson_out = True, nodata=np.nan)
+    zs_area = gpd.GeoDataFrame.from_features(zs_area)
+    zs_area = zs_area.set_crs(nwm_streams.crs)
+    zs_area = zs_area.rename(columns = {"sum":"missing_volume_m3"})
+
+    # Derive slope tif
+    output_slope_tif = os.path.join(os.path.dirname(tif), 'bathy_slope.tif')
+    slope_tif = gdal.DEMProcessing(output_slope_tif, bathy_gdal , 'slope', format = 'GTiff')
+    slope_tif = slope_tif.GetRasterBand(1).ReadAsArray()
+    os.remove(output_slope_tif)
+    slope_tif[np.where(slope_tif == -9999.)] = np.nan
+    missing_bed_area = (1 / np.cos(slope_tif*np.pi/180)) - 1
+
+    # Find missing bed area from slope tif
+    zs_slope = zonal_stats(nwm_catchments, missing_bed_area, stats = ["sum"], affine= bathy_affine, geojson_out = True, nodata=np.nan)
+    zs_slope = gpd.GeoDataFrame.from_features(zs_slope)
+    zs_slope = zs_slope.rename(columns = {"sum":"missing_bed_area_m2"})
+
+    # Clip streams to survey bounds and find new reach lengths
+    nwm_streams_clip = nwm_streams.clip(bathy_bounds)
+    nwm_streams_clip["Length"] = nwm_streams_clip.length
+
+    # Merge data into nwm streams file and remove extremely small reaches
+    bathy_nwm_streams = nwm_streams_clip.merge(zs_area[['missing_volume_m3', 'ID']], on='ID')
+    bathy_nwm_streams = bathy_nwm_streams.merge(zs_slope[['missing_bed_area_m2', 'ID']], on='ID')
+    max_order = bathy_nwm_streams["order_"].max()
+    bathy_nwm_streams = bathy_nwm_streams.loc[bathy_nwm_streams['order_'] >= (max_order-1)]
+
+    # Calculate wetted perimeter and cross sectional area
+    bathy_nwm_streams['missing_xs_area_m2'] = bathy_nwm_streams['missing_volume_m3']/bathy_nwm_streams['Length']
+    bathy_nwm_streams['missing_wet_perimeter_m'] = bathy_nwm_streams['missing_bed_area_m2']/bathy_nwm_streams['Length']
+
+    # Add survey meta data
+    bathy_nwm_streams['SurveyDateStamp'] = bathy_bounds.loc[0, 'SurveyDateStamp']
+    bathy_nwm_streams['SurveyId'] = bathy_bounds.loc[0, 'SurveyId']
+    bathy_nwm_streams['Sheet_Name'] = bathy_bounds.loc[0, 'Sheet_Name']
+    bathy_nwm_streams.assign(Bathymetry_source='USACE eHydro')
+
+    # Export geopackage with bathymetry
+    num_streams = len(bathy_nwm_streams)
+    bathy_nwm_streams = bathy_nwm_streams.to_crs(epsg=5070)
+    if os.path.exists(output):
+        print(f"{output} already exists. Concatinating now...")
+        existing_bathy_file = gpd.read_file(output)
+        bathy_nwm_streams = pd.concat([existing_bathy_file, bathy_nwm_streams])
+    bathy_nwm_streams.to_file(output, index = False)
+    print(f"Added {num_streams} new NWM features")
+
+if __name__ == '__main__':
+    
+    parser = ArgumentParser(description="Preprocessed Bathymetry")
+    parser.add_argument('-tif','--tif', help='Survey Depth Raster', required=True,type=str)
+    parser.add_argument('-survey_gdb','--survey_gdb', help='Survey Geodatabase Ex. AL_LP_EMS_20150809_CS_005_065_SORT.gbd', required=True,type=str)
+    parser.add_argument('-bathy_bounds','--bathy_bounds', help='Survey Bounds Layer Ex. SurveyJob.gpkg', default = 'SurveyJob', required=False,type=str)
+    parser.add_argument('-output','--output', help='output geopackage location', required=True,type=str)
+    parser.add_argument('-min_depth_threshold','--min_depth_threshold', help='minimum expected depth value', required=False,type=int, default= 10)
+
+    args = vars(parser.parse_args())
+
+    tif = args['tif']
+    survey_gdb = args['survey_gdb']
+    bathy_bounds = args['bathy_bounds']
+    output = args['output']
+    min_depth_threshold = args['min_depth_threshold']
+
+    preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_threshold)
+    print("success :)")
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4f8ea5d9f..f589e871d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.14.0 - 2023-08-03 - [PR#953](https://github.com/NOAA-OWP/inundation-mapping/pull/953)
+
+The enhancements in this PR include the new modules for pre-processing bathymetric data from the USACE eHydro dataset and integrating the missing hydraulic geometry into the HAND synthetic rating curves.
+
+### Additions  
+
+- `data/bathymetry/preprocess_bathymetry.py`: preprocesses the eHydro datasets.
+- `src/bathymetric_adjustment.py`: adjusts synthetic rating curves for HUCs where preprocessed bathymetry is available.
+
+### Changes  
+
+- `config/params_template.env`: added a toggle for the bathymetric adjustment routine: `bathymetry_adjust`
+- `fim_post_processing.sh`: added the new `bathymetric_adjustment.py` to the postprocessing lineup
+- `src/`
+    - `add_crosswalk.py`, `aggregate_by_huc.py`, & `subdiv_chan_obank_src.py`: accounting for the new Bathymetry_source field in SRCs
+
+<br/><br/>
+
 ## v4.3.13.0 - 2023-07-26 - [PR#952](https://github.com/NOAA-OWP/inundation-mapping/pull/952)
 
 Adds a feature to manually calibrate rating curves for specified NWM `feature_id`s using a CSV of manual coefficients to output a new rating curve. Manual calibration is applied after any/all other calibrations. Coefficient values between 0 and 1 increase the discharge value (and decrease inundation) for each stage in the rating curve while values greater than 1 decrease the discharge value (and increase inundation).
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 921c4b7d2..0ba647bfc 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -100,6 +100,15 @@ find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "
 echo "Processing usgs gage aggregation"   
 python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev -j $jobLimit
 
+## RUN BATHYMETRY ADJUSTMENT ROUTINE ##
+if [ "$bathymetry_adjust" = "True" ]; then
+    echo -e $startDiv"Performing Bathymetry Adjustment routine"
+    # Run bathymetry adjustment routine
+    Tstart
+    python3 $srcDir/bathymetric_adjustment.py -fim_dir $outputDestDir -bathy $inputsDir/bathymetry/bathymetry_adjustment_data.gpkg -buffer $wbd_buffer -wbd $inputsDir/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -j $jobLimit
+    Tcount
+fi
+
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
     echo -e $startDiv"Estimating bankfull stage in SRCs"
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index a6a30da8e..188162737 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -211,8 +211,9 @@ def add_crosswalk(input_catchments_fileName,
 
     # set nans to 0
     input_src_base.loc[input_src_base['Stage']==0,['Discharge (m3s-1)']] = 0
+    input_src_base['Bathymetry_source'] = pd.NA
 
-    output_src = input_src_base.drop(columns=['CatchId'])
+    output_src = input_src_base.drop(columns=['CatchId']).copy()
     if output_src.HydroID.dtype != 'int': output_src.HydroID = output_src.HydroID.astype(int)
 
     # update rating curves
@@ -238,14 +239,10 @@ def add_crosswalk(input_catchments_fileName,
     output_crosswalk = output_src[['HydroID','feature_id']]
     output_crosswalk = output_crosswalk.drop_duplicates(ignore_index=True)
 
-    ## bathy estimation integration in synthetic rating curve calculations
-    #if (bathy_src_calc == True and extent == 'MS'):
-    #    output_src = bathy_rc_lookup(output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName)
-    #else:
-    #    print('Note: NOT using bathy estimation approach to modify the SRC...')
-
     # make hydroTable
-    output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)','BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)','Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
+    output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)',
+        'BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)',
+        'Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
     output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
     ## Set placeholder variables to be replaced in post-processing (as needed). Create here to ensure consistent column vars
     ## These variables represent the original unmodified values
@@ -254,6 +251,8 @@ def add_crosswalk(input_catchments_fileName,
     output_hydro_table['default_WetArea (m2)'] = output_src['WetArea (m2)']
     output_hydro_table['default_HydraulicRadius (m)'] = output_src['HydraulicRadius (m)']
     output_hydro_table['default_ManningN'] = output_src['ManningN']
+    ## Placeholder vars for BARC
+    output_hydro_table['Bathymetry_source'] = pd.NA
     ## Placeholder vars for subdivision routine
     output_hydro_table['subdiv_applied'] = False
     output_hydro_table['overbank_n'] = pd.NA
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index 40cc37e32..4a0cd4fc3 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -239,10 +239,10 @@ def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
     # Validation
     total_cpus_available = os.cpu_count() - 2
     if num_job_workers > total_cpus_available:
-        raise ValueError('The number of jobs {number_of_jobs}'\
+        raise ValueError(f'The number of jobs {num_job_workers}'\
                           'exceeds your machine\'s available CPU count minus two. '\
                           'Please lower the number of jobs '\
-                          'values accordingly.'.format(num_job_workers)
+                          'values accordingly.'
                         )
    
     # create log folder, might end up empty but at least create the folder
diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py
new file mode 100644
index 000000000..8d9243292
--- /dev/null
+++ b/src/bathymetric_adjustment.py
@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+
+import sys, os, traceback, re
+from os.path import join
+import pandas as pd
+import geopandas as gpd
+import datetime as dt
+from argparse import ArgumentParser
+from concurrent.futures import ProcessPoolExecutor
+sys.path.append('/foss_fim/tools')
+from synthesize_test_cases import progress_bar_handler
+
+
+def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
+    """Function for correcting synthetic rating curves. It will correct each branch's
+    SRCs in serial based on the feature_ids in the input bathy_file.
+
+        Parameters
+        ----------
+        fim_dir : str
+            Directory path for fim_pipeline output.
+        huc : str
+            HUC-8 string.
+        bathy_file : str
+            Path to bathymetric adjustment geopackage, e.g. 
+            "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
+        verbose : bool
+            Verbose printing.
+
+        Returns
+        ----------
+        log_text : str
+            
+    """
+
+    log_text = f'Calculating bathymetry adjustment: {huc}\n'
+
+    # Load wbd and use it as a mask to pull the bathymetry data
+    fim_huc_dir = join(fim_dir, huc)
+    wbd8_clp = gpd.read_file(join(fim_huc_dir, 'wbd8_clp.gpkg'))
+    bathy_data = gpd.read_file(bathy_file, mask=wbd8_clp)
+    bathy_data = bathy_data.rename(columns={'ID':'feature_id'})
+
+    # Get src_full from each branch
+    src_all_branches = []
+    branches = os.listdir(join(fim_huc_dir, 'branches'))
+    for branch in branches:
+        src_full = join(fim_huc_dir, 'branches', str(branch), f'src_full_crosswalked_{branch}.csv')
+        if os.path.isfile(src_full):
+            src_all_branches.append(src_full)
+
+    # Update src parameters with bathymetric data
+    for src in src_all_branches:
+
+        src_df = pd.read_csv(src)
+        if 'Bathymetry_source' in src_df.columns:
+            src_df = src_df.drop(columns='Bathymetry_source')
+        branch = re.search('branches/(\d{10}|0)/', src).group()[9:-1]
+        log_text += f'  Branch: {branch}\n'
+
+        if bathy_data.empty:
+            log_text += '  There were no bathymetry feature_ids for this branch'
+            src_df['Bathymetry_source'] = [""]* len(src_df)
+            src_df.to_csv(src, index=False)
+            return log_text
+    
+        # Merge in missing bathy data and fill Nans
+        try:
+            src_df = src_df.merge(bathy_data[['feature_id', 'missing_xs_area_m2', 'missing_wet_perimeter_m', 'Bathymetry_source']], 
+                                  on='feature_id', how='left', validate='many_to_one')
+        # If there's more than one feature_id in the bathy data, just take the mean
+        except pd.errors.MergeError:
+            reconciled_bathy_data = bathy_data.groupby('feature_id')[['missing_xs_area_m2', 'missing_wet_perimeter_m']].mean()
+            reconciled_bathy_data['Bathymetry_source'] = bathy_data.groupby('feature_id')['Bathymetry_source'].first()
+            src_df = src_df.merge(reconciled_bathy_data, on='feature_id', how='left', validate='many_to_one')
+        # Exit if there are no recalculations to be made
+        if ~src_df['Bathymetry_source'].any(axis=None):
+            log_text += f'    No matching feature_ids in this branch\n'
+            continue
+
+        src_df['missing_xs_area_m2'] = src_df['missing_xs_area_m2'].fillna(0.0)
+        src_df['missing_wet_perimeter_m'] = src_df['missing_wet_perimeter_m'].fillna(0.0)
+        # Add missing hydraulic geometry into base parameters
+        src_df['Volume (m3)'] = src_df['Volume (m3)'] + (src_df['missing_xs_area_m2'] * (src_df['LENGTHKM'] * 1000))
+        src_df['BedArea (m2)'] = src_df['BedArea (m2)'] + (src_df['missing_wet_perimeter_m'] * (src_df['LENGTHKM'] * 1000))
+        # Recalc discharge with adjusted geometries
+        src_df['WettedPerimeter (m)'] = src_df['WettedPerimeter (m)'] + src_df['missing_wet_perimeter_m']
+        src_df['WetArea (m2)'] = src_df['WetArea (m2)'] + src_df['missing_xs_area_m2']
+        src_df['HydraulicRadius (m)'] = src_df['WetArea (m2)']/src_df['WettedPerimeter (m)']
+        src_df['HydraulicRadius (m)'] = src_df['HydraulicRadius (m)'].fillna(0)
+        src_df['Discharge (m3s-1)'] = src_df['WetArea (m2)']* \
+            pow(src_df['HydraulicRadius (m)'],2.0/3)* \
+            pow(src_df['SLOPE'],0.5)/src_df['ManningN']
+        # Force zero stage to have zero discharge
+        src_df.loc[src_df['Stage']==0,['Discharge (m3s-1)']] = 0
+        # Calculate number of adjusted HydroIDs
+        count = len(src_df.loc[(src_df['Stage']==0) & (src_df['Bathymetry_source'] == 'USACE eHydro')])
+
+        # Write src back to file
+        src_df.to_csv(src, index=False)
+        log_text += f'    Successfully recalculated {count} HydroIDs\n'
+    return log_text
+
+
+def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, number_of_jobs, verbose):
+    """Function for correcting synthetic rating curves. It will correct each branch's
+    SRCs in serial based on the feature_ids in the input bathy_file.
+
+        Parameters
+        ----------
+        fim_dir : str
+            Directory path for fim_pipeline output.
+        bathy_file : str
+            Path to bathymetric adjustment geopackage, e.g. 
+            "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
+        wbd_buffer : int
+            Distance in meters to buffer wbd dataset when searching for relevant HUCs.
+        wbd : str
+            Path to wbd input data, e.g. 
+            "/data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg".
+        output_suffix : str
+            Output filename suffix. 
+        number_of_jobs : int
+            Number of CPU cores to parallelize HUC processing.
+        verbose : bool
+            Verbose printing.
+
+    """
+
+    # Set up log file
+    print('Writing progress to log file here: ' + str(join(fim_dir,'logs','bathymetric_adjustment' + output_suffix + '.log')))
+    print('This may take a few minutes...')
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+
+    ## Initiate log file
+    log_file = open(join(fim_dir,'logs','bathymetric_adjustment' + output_suffix + '.log'),"w")
+    log_file.write('START TIME: ' + str(begin_time) + '\n')
+    log_file.write('#########################################################\n\n')
+
+    # Exit program if the bathymetric data doesn't exist
+    if not os.path.exists(bathy_file):
+        statement = f'The input bathymetry file {bathy_file} does not exist. Exiting...'
+        log_file.write(statement)
+        print(statement)
+        sys.exit(0)
+
+    # Find applicable HUCs to apply bathymetric adjustment
+    # NOTE: This block can be removed if we have estimated bathymetry data for
+    # the whole domain later.
+    fim_hucs = [h for h in os.listdir(fim_dir) if re.match('\d{8}', h)]
+    bathy_gdf = gpd.read_file(bathy_file)
+    buffered_bathy = bathy_gdf.geometry.buffer(wbd_buffer)    # We buffer the bathymetric data to get adjacent 
+    wbd = gpd.read_file(wbd, mask=buffered_bathy)             # HUCs that could also have bathymetric reaches included
+    hucs_with_bathy = wbd.HUC8.to_list()
+    hucs = [h for h in fim_hucs if h in hucs_with_bathy]
+    log_file.write(f"Identified {len(hucs)} HUCs that have bathymetric data: {hucs}\n")
+    print(f"Identified {len(hucs)} HUCs that have bathymetric data\n")
+
+    # Set up multiprocessor
+    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
+
+        # Loop through all test cases, build the alpha test arguments, and submit them to the process pool
+        executor_dict = {}
+        for huc in hucs:
+            
+            arg_keeper = { 
+                        'fim_dir': fim_dir,
+                        'huc': huc,
+                        'bathy_file': bathy_file,
+                        'verbose': verbose,
+                        }
+            future = executor.submit(correct_rating_for_bathymetry, **arg_keeper)
+            executor_dict[future] = huc
+
+        # Send the executor to the progress bar and wait for all tasks to finish
+        progress_bar_handler(executor_dict, True, f"Running BARC on {len(hucs)} HUCs")
+        # Get the returned logs and write to the log file
+        for future in executor_dict.keys():
+            try:
+                log_file.write(future.result())
+            except Exception as ex:
+                print(f"WARNING: {executor_dict[future]} BARC failed for some reason")
+                log_file.write( f"ERROR --> {executor_dict[future]} BARC failed (details: *** {ex} )\n")
+                traceback.print_exc(file=log_file)
+
+    ## Record run time and close log file
+    end_time = dt.datetime.now()
+    log_file.write('END TIME: ' + str(end_time) + '\n')
+    tot_run_time = end_time - begin_time
+    log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+    log_file.close()
+
+if __name__ == '__main__':
+
+    """
+    Parameters
+    ----------
+    fim_dir : str
+        Directory path for fim_pipeline output. Log file will be placed in 
+        fim_dir/logs/bathymetric_adjustment.log.
+    bathy_file : str
+        Path to bathymetric adjustment geopackage, e.g. 
+        "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
+    wbd_buffer : int
+        Distance in meters to buffer wbd dataset when searching for relevant HUCs.
+    wbd : str
+        Path to wbd input data, e.g. 
+        "/data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg".
+    output_suffix : str
+        Optional. Output filename suffix. Defaults to no suffix. 
+    number_of_jobs : int
+        Optional. Number of CPU cores to parallelize HUC processing. Defaults to 8.
+    verbose : bool
+        Optional flag for enabling verbose printing.
+
+    Sample Usage
+    ----------
+    python3 /foss_fim/src/bathymetric_adjustment.py -fim_dir /outputs/fim_run_dir -bathy /data/inputs/bathymetry/bathymetry_adjustment_data.gpkg 
+        -buffer 5000 -wbd /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -j $jobLimit
+
+    """
+
+    parser = ArgumentParser(description="Bathymetric Adjustment")
+    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
+    parser.add_argument('-bathy','--bathy_file',help="Path to geopackage with preprocessed bathymetic data",required=True,type=str)
+    parser.add_argument('-buffer','--wbd-buffer',help="Buffer to apply to bathymetry data to find applicable HUCs",required=True,type=int)
+    parser.add_argument('-wbd','--wbd',help="Buffer to apply to bathymetry data to find applicable HUCs",required=True,type=str)
+    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",default="",required=False,type=str)
+    parser.add_argument('-j','--number-of-jobs',help='OPTIONAL: number of workers (default=8)',required=False,default=8,type=int)
+    parser.add_argument('-vb','--verbose',help='OPTIONAL: verbose progress bar',required=False,default=None,action='store_true')
+
+    args = vars(parser.parse_args())
+
+    fim_dir = args['fim_dir']
+    bathy_file = args['bathy_file']
+    wbd_buffer = int(args['wbd_buffer'])
+    wbd = args['wbd']
+    output_suffix = args['output_suffix']
+    number_of_jobs = args['number_of_jobs']
+    verbose = bool(args['verbose'])
+
+    multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, number_of_jobs, verbose)
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index 747506aee..0547aa7fe 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -206,7 +206,7 @@ def run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_opti
     log_file.write('#########################################################\n\n')
 
     ## List of columns in SRC_full_crosswalk to read in (ignores other columns that may have been added by previous post-proccessing runs)
-    src_usecols=['Stage','Number of Cells','SurfaceArea (m2)','BedArea (m2)','Volume (m3)','SLOPE','LENGTHKM','AREASQKM','ManningN','HydroID','NextDownID','order_','TopWidth (m)','WettedPerimeter (m)','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)','feature_id']
+    src_usecols=['Stage','Number of Cells','SurfaceArea (m2)','BedArea (m2)','Volume (m3)','SLOPE','LENGTHKM','AREASQKM','ManningN','HydroID','NextDownID','order_','TopWidth (m)','WettedPerimeter (m)','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)','feature_id', 'Bathymetry_source']
 
     df_bflows = pd.read_csv(bankfull_flow_filepath,dtype={'feature_id': int})
     huc_list  = os.listdir(fim_dir)
diff --git a/src/subdiv_chan_obank_src.py b/src/subdiv_chan_obank_src.py
index 6f1d41133..1e9f2aab1 100755
--- a/src/subdiv_chan_obank_src.py
+++ b/src/subdiv_chan_obank_src.py
@@ -87,7 +87,7 @@ def variable_mannings_calc(args):
             df_src.to_csv(in_src_bankfull_filename,index=False)
 
             ## Output new hydroTable with updated discharge and ManningN column
-            df_src_trim = df_src[['HydroID','Stage','subdiv_applied','channel_n','overbank_n','Discharge (m3s-1)_subdiv']]
+            df_src_trim = df_src[['HydroID','Stage', 'Bathymetry_source', 'subdiv_applied','channel_n','overbank_n','Discharge (m3s-1)_subdiv']]
             df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_subdiv': 'subdiv_discharge_cms'})
             df_src_trim['discharge_cms'] = df_src_trim['subdiv_discharge_cms'] # create a copy of vmann modified discharge (used to track future changes)
             df_htable = pd.read_csv(htable_filename,dtype={'HUC': str,'last_updated':object,'submitter':object,'obs_source':object})
@@ -158,6 +158,7 @@ def subdiv_mannings_eq(df_src):
     ## Calcuate the total of the subdivided discharge (channel + overbank)
     df_src.drop(['Discharge (m3s-1)_subdiv'], axis=1, inplace=True, errors='ignore') # drop these cols (in case subdiv was previously performed)
     df_src['Discharge (m3s-1)_subdiv'] = df_src['Discharge_chan (m3s-1)'] + df_src['Discharge_obank (m3s-1)']
+    df_src.loc[df_src['Stage']==0,['Discharge (m3s-1)_subdiv']] = 0
     return(df_src)
 
 def generate_src_plot(df_src, plt_out_dir):

From bf29c70899033838696e6e91c97e15371f929cbf Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 4 Aug 2023 10:55:12 -0600
Subject: [PATCH 260/359] v4.3.14.1 Fix acquire for ras2fim use (#946)

---
 README.md                                     | 10 +--
 data/usgs/acquire_and_preprocess_3dep_dems.py | 64 ++++++++++++-------
 docs/CHANGELOG.md                             | 21 ++++++
 fim_pipeline.sh                               |  2 +
 fim_pre_processing.sh                         |  3 +-
 fim_process_unit_wb.sh                        |  7 +-
 6 files changed, 75 insertions(+), 32 deletions(-)

diff --git a/README.md b/README.md
index 96ed32ced..de3502665 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
 
 ### Accessing Data using the AWS CLI
 
-This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If you are using compute resources in the same region as the S3 Bucket, then there is no cost.
+This S3 Bucket (`s3://noaa-nws-owp-fim`) is set to avoid having AWS credentials loaded. Read more about the use of the `--no-sign-request` [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html#:~:text=%2D%2Dno%2Dsign%2Drequest,prevents%20credentials%20from%20being%20loaded.). There should be no cost associated with using this S3 Bucket.
 
 ### Examples
 
@@ -33,13 +33,13 @@ This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket.
 
 The available inputs, test cases, and versioned FIM outputs can be found by running:
 ```
-aws s3 ls s3://noaa-nws-owp-fim/hand_fim/ --request-payer requester
+aws s3 ls s3://noaa-nws-owp-fim/hand_fim/ --no-sign-request
 ```
 
 Download a directory of sample outputs for a single HUC8:
 ```
 aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_3_11_0/12090301 \
-    /your_local_folder_name/12090301 --request-payer requester
+    /your_local_folder_name/12090301 --no-sign-request
 ```
 By adjusting pathing, you can also download entire directories such as the `fim_4_3_11_0` folder. An entire output FIM set (e.g. `fim_4_3_11_0`) is approximately 1.1 TB.
 
@@ -84,9 +84,9 @@ Git will auto create a subfolder named `inundation-mapping` where the code will
 Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP S3 Bucket" section above). The FIM inputs directory can be found at `s3://noaa-nws-owp-fim/hand_fim/inputs`. It is appx 400GB and it needs to be in your `data` folder.
 
 ```
-aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/inputs /home/projects/fim/data/inputs --request-payer requester --dry-run
+aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/inputs /home/projects/fim/data/inputs --no-sign-request --dryrun
 ```
-**Note**: When you include the `--dry-run` argument in the command, a large list will be returned showing you exactly which files are to be downloaded and where they will be saved. We recommend including this argument the first time you run the command, then quickly aborting it (CTRL-C) so you don't get the full list. However, you can see that your chosen target path on your machine is correct.  When you are happy with the pathing, run the `aws s3` command again and leave off the `--dry-run` argument.
+**Note**: When you include the `--dryrun` argument in the command, a large list will be returned showing you exactly which files are to be downloaded and where they will be saved. We recommend including this argument the first time you run the command, then quickly aborting it (CTRL-C) so you don't get the full list. However, you can see that your chosen target path on your machine is correct.  When you are happy with the pathing, run the `aws s3` command again and leave off the `--dryrun` argument.
 
 The S3 inputs directory has all of the folders and files you need to run FIM. It includes some publicly available and some non-publicly availible data.
 
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index 65bef1bcf..0a0d38e16 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -20,6 +20,13 @@
 
 from utils.shared_functions import FIM_Helpers as fh
 
+'''
+TODO:
+    - Change to change resolution (which means URL, block size also ahve to be parameterize)
+    - if creating polygons, take out some of the hardcoding.
+'''
+
+
 # local constants (until changed to input param)
 # This URL is part of a series of vrt data available from USGS via an S3 Bucket.
 # for more info see: "http://prd-tnm.s3.amazonaws.com/index.html?prefix=StagedProducts/Elevation/". The odd folder numbering is
@@ -30,14 +37,15 @@
 def acquire_and_preprocess_3dep_dems(extent_file_path,
                                      target_output_folder_path = '', 
                                      number_of_jobs = 1, 
-                                     retry = False):
+                                     retry = False,
+                                     skip_polygons = False):
     
     '''
     Overview
     ----------
     This will download 3dep rasters from USGS using USGS vrts.
     By default USGS 3Dep stores all their rasters in lat/long (northing and easting).
-    By us downloading the rasters using WBD HUC4 clips and gdal, we an accomplish a few extra
+    By us downloading the rasters using WBD HUC6 clips and gdal, we an accomplish a few extra
     steps.
         1) Ensure the projection types that are downloaded are consistant and controlled.
            We are going to download them as NAD83 basic (espg: 4269) which is consistant
@@ -47,15 +55,18 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
         3) Create the 3dep rasters in the size we want (default at HUC4 for now)
         
     Notes:
-        - As this is a very low use tool, all values such as the USGS vrt path, output
-          folder paths, huc unit level (huc4), etc are all hardcoded
+        - It really can be used for any huc size.
+        - As this is a very low use tool, most values such as the USGS vrt path, output
+          folder paths, etc are all hardcoded
+        - You can really submit any polys of any HUC size as it as long as it is 10m
+          resolution for now.
         
     Parameters
     ----------
         - extent_file_path (str):
             Location of where the extent files that are to be used as clip extent against
             the USGS 3Dep vrt url.
-            ie) \data\inputs\wbd\HUC4
+            ie) \data\inputs\wbd\HUC6
             
         - target_output_folder_path (str):
             The output location of the new 3dep dem files. When the param is not submitted,
@@ -67,6 +78,10 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
         - retry (True / False):
             If retry is True and the file exists (either the raw downloaded DEM and/or)
             the projected one, then skip it
+            
+        - skip_polygons (bool)
+             If True, then we will not attempt to create polygon files for each dem file.
+
     '''
     # -------------------
     # Validation
@@ -110,11 +125,16 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
     # download dems, setting projection, block size, etc
     __download_usgs_dems(extent_file_names, target_output_folder_path, number_of_jobs, retry)
 
-    polygonize(target_output_folder_path)
+    if (skip_polygons == False):
+        polygonize(target_output_folder_path)
     
     end_time = datetime.now()
     fh.print_end_header('Loading 3dep dems', start_time, end_time)
-    print(f'---- NOTE: Remember to scan the log file for any failures')
+    
+    print()
+    print('---- NOTE: Remember to scan the log file for any failures. If you find errors in the'\
+          ' log file, delete the output file and retry')
+    print()
     logging.info(fh.print_date_time_duration(start_time, end_time))
 
 
@@ -148,7 +168,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     base_cmd += ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
     base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
     base_cmd += ' -t_srs {3} -cblend 6'
-   
+        
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
 
         executor_dict = {}
@@ -218,7 +238,7 @@ def download_usgs_dem_file(extent_file,
                 base_cmd +=  ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
                 base_cmd +=  ' -t_srs {3} -cblend 6'
         - retry (bool)
-             If True, and the file exists (and is over 0k), downloading will be skipped.
+             If True, and the file exists, downloading will be skipped.
         
     '''
     
@@ -227,22 +247,15 @@ def download_usgs_dem_file(extent_file,
     target_path_raw = os.path.join(output_folder_path,
                                     target_file_name_raw)
     
-    # File might exist from a previous failed run. If it was aborted or failed
-    # on a previous attempt, it's size less than 1mg, so delete it.
-    # 
-    # IMPORTANT:
-    #
-    # it might be compromised on a previous run but GREATER 1mg (part written).
-    # That scenerio is not handled as we can not tell if it completed.
+    # It does happen where the final output size can be very small (or all no-data)
+    # which is related to to the spatial extents of the dem and the vrt combined.
+    # so, super small .tifs are correct.
     
     if (retry) and (os.path.exists(target_path_raw)):
-        if (os.stat(target_path_raw).st_size < 1000000):
-            os.remove(target_path_raw)
-        else:
-            msg = f" - Downloading -- {target_file_name_raw} - Skipped (already exists (see retry flag))"
-            print(msg)  
-            logging.info(msg)
-            return
+        msg = f" - Downloading -- {target_file_name_raw} - Skipped (already exists (see retry flag))"
+        print(msg)  
+        logging.info(msg)
+        return
     
     msg = f" - Downloading -- {target_file_name_raw} - Started"
     print(msg)
@@ -364,6 +377,8 @@ def __setup_logger(output_folder_path):
     #     as/if needed (command line params)
     #   - The output path can be adjusted in case of a test reload of newer data for 3dep.
     #     The default is /data/input/usgs/3dep_dems/10m/
+    #   - Each output file will be the name of the input poly plus "_dem.tif". ie) if the wbd gpkg
+    #     is named named "HUC8_12090301", then the output file name will be "HUC8_12090301_dem.tif"
     #   - While you can (and should use more than one job number (if manageable by your server)),
     #     this tool is memory intensive and needs more RAM then it needs cores / cpus. Go ahead and 
     #     anyways and increase the job number so you are getting the most out of your RAM. Or
@@ -392,6 +407,9 @@ def __setup_logger(output_folder_path):
 
     parser.add_argument('-t','--target_output_folder_path', help='location of where the 3dep files'\
                         ' will be saved', required=False, default='')
+    
+    parser.add_argument('-sp','--skip_polygons', help='If this flag is included, polygons of the dems'\
+                        ' will not be made', required=False, action='store_true', default=False)    
 
 
     # Extract to dictionary and assign to variables.
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f589e871d..8a6436d1b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,27 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.14.1 - 2023-07-13 - [PR#946](https://github.com/NOAA-OWP/inundation-mapping/pull/946)
+
+ras2fim product had a need to run the acquire 3dep script to pull down some HUC8 DEMs. The old script was geared to HUC6 but could handle HUC8's but needed a few enhancements. ras2fim also did not need polys made from the DEMs, so a switch was added for that.
+
+The earlier version on the "retry" feature would check the file size and if it was smaller than a particular size, it would attempt to reload it.  The size test has now been removed. If a file fails to download, the user will need to look at the log out, then remove the file before attempting again. Why? So the user can see why it failed and decide action from there.
+
+Note: later, as needed, we might upgrade it to handle more than just 10m (which it is hardcoded against).
+
+Additional changes to README to reflect how users can access ESIP's S3 as well as a one line addition to change file permissions in fim_process_unit_wb.sh.
+
+### Changes  
+- `data`
+    - `usgs`
+        - `acquire_and_preprocess_3dep_dems.py`:  As described above.
+ - `fim_pipeline.sh`:  a minor styling fix (added a couple of lines for readability)
+ - `fim_pre_processing.sh`: a user message was incorrect & chmod 777 $outputDestDir. 
+ - `fim_process_unit_wb.sh`: chmod 777 for /output/<run_name> directory.
+ - `README.md`: --no-sign-request instead of --request-payer requester for ESIP S3 access.
+
+<br/><br/>
+
 ## v4.3.14.0 - 2023-08-03 - [PR#953](https://github.com/NOAA-OWP/inundation-mapping/pull/953)
 
 The enhancements in this PR include the new modules for pre-processing bathymetric data from the USACE eHydro dataset and integrating the missing hydraulic geometry into the HAND synthetic rating curves.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index df68a06c0..87768b482 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -14,7 +14,9 @@ For more details on
 
 Processing of HUC''s in FIM4 comes in three pieces. You can run `fim_pipeline.sh` which automatically runs all of three major section, but you can run each of the sections independently if you like. The three sections are:
 - `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
+
 - `fim_process_unit_wb.sh` : This script processes one and exactly one HUC8 plus all of it''s related branches. While it can only process one, you can run this script multiple times, each with different HUC (or overwriting a HUC). When you run `fim_pipeline.sh`, it automatically iterates when more than one HUC number has been supplied either by command line arguments or via a HUC list. For each HUC provided, `fim_pipeline.sh` will `fim_process_unit_wb.sh`. Using the `fim_process_unit_wb.sh`  script allows for a run / rerun of a HUC, or running other HUCs at different times / days or even different docker containers.
+
 - `fim_post_processing.sh` : This section takes all of the HUCs that have been processed, aggregates key information from each HUC directory and looks for errors across all HUC folders. It also processes the group in sub-steps such as usgs guages processesing, rating curve adjustments and more. Naturally, running or re-running this script can only be done after running `fim_pre_processing.sh` and at least one run of `fim_process_unit_wb.sh`.
 
 Running the `fim_pipeline.sh` is a quicker process than running all three steps independently.
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 05f87da10..29c2c42e1 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -46,7 +46,7 @@ usage ()
     echo '                    use the word NONE as this value for this parameter.'    
     echo '  -jh/--jobLimit   : Max number of concurrent HUC jobs to run. Default 1 job at time.'
     echo '  -jb/--jobBranchLimit : Max number of concurrent Branch jobs to run. Default 1 job at time.' 
-    echo '                         NOTE: Make sure that the multiplication of jh and jb subtract 2 (jh x jb -2) does not'
+    echo '                         NOTE: Make sure that the multiplication of jh and jb plus 2 (jh x jb + 2) does not'
     echo '                         exceed the total number of cores available.'
     echo '  -o               : Overwrite outputs if already exist'
     echo '  -skipcal         : If this param is included, updating the S.R.C. via the calibration points will be skipped'    
@@ -201,6 +201,7 @@ echo "--- Number of HUCs to process is $num_hucs"
 # make dirs
 if [ ! -d $outputDestDir ]; then
     mkdir -p $outputDestDir
+    chmod 777 $outputDestDir
     mkdir -p $tempRunDir
 else
     # remove these directories and files on a new or overwrite run
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
index e73f4d295..2144337a1 100755
--- a/fim_process_unit_wb.sh
+++ b/fim_process_unit_wb.sh
@@ -124,14 +124,15 @@ if [ "$err_exists" = "1" ]; then
     cp $hucLogFileName $outputDestDir/unit_errors
 fi
 
-# Here we're using the mv command to move the contents of the temp directory into the specified output directory
+# Move the contents of the temp directory into the outputs directory and update file permissions
+mv -f $tempHucDataDir $outputHucDataDir
+find $outputHucDataDir -type d -exec chmod 777 {} +
+
 echo "============================================================================================="
 echo 
-mv -f $tempHucDataDir $outputHucDataDir
 echo "***** Moved temp directory: $tempHucDataDir to output directory: $outputHucDataDir  *****"
 echo
 echo "============================================================================================="
 
-
 # we always return a success at this point (so we don't stop the loops / iterator)
 exit 0

From 6361781fc65d063a326e10aa665f3004adf248e3 Mon Sep 17 00:00:00 2001
From: RileyMcDermott-NOAA
 <139250904+RileyMcDermott-NOAA@users.noreply.github.com>
Date: Fri, 11 Aug 2023 11:35:43 -0500
Subject: [PATCH 261/359] v4.3.14.2 Bathymetry adjustment hotfix (#959)

---
 config/params_template.env               |  1 +
 data/bathymetry/preprocess_bathymetry.py |  2 +-
 docs/CHANGELOG.md                        | 11 +++++++++++
 fim_post_processing.sh                   |  2 +-
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/config/params_template.env b/config/params_template.env
index a4c1c9ac6..e22783f88 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -33,6 +33,7 @@ export levee_id_attribute=SYSTEM_ID
 
 #### apply bathymetry adjustment to rating curve ####
 export bathymetry_adjust=True
+export bathymetry_file="/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg"
 
 #### estimating bankfull stage in SRCs ####
 export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
diff --git a/data/bathymetry/preprocess_bathymetry.py b/data/bathymetry/preprocess_bathymetry.py
index 560ef24f2..2fe0f4bf5 100644
--- a/data/bathymetry/preprocess_bathymetry.py
+++ b/data/bathymetry/preprocess_bathymetry.py
@@ -97,7 +97,7 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
     bathy_nwm_streams['SurveyDateStamp'] = bathy_bounds.loc[0, 'SurveyDateStamp']
     bathy_nwm_streams['SurveyId'] = bathy_bounds.loc[0, 'SurveyId']
     bathy_nwm_streams['Sheet_Name'] = bathy_bounds.loc[0, 'Sheet_Name']
-    bathy_nwm_streams.assign(Bathymetry_source='USACE eHydro')
+    bathy_nwm_streams["Bathymetry_source"] = 'USACE eHydro'
 
     # Export geopackage with bathymetry
     num_streams = len(bathy_nwm_streams)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8a6436d1b..85407f01e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.14.2 - 2023-08-08 - [PR#959](https://github.com/NOAA-OWP/inundation-mapping/pull/959)
+
+The enhancements in this PR include the new modules for pre-processing bathymetric data from the USACE eHydro dataset and integrating the missing hydraulic geometry into the HAND synthetic rating curves.
+
+### Changes  
+- `data/bathymetry/preprocess_bathymetry.py`: added data source column to output geopackage attribute table.
+- `fim_post_processing.sh`: changed -bathy input reference location.
+- `config/params_template.env`: added export to bathymetry_file
+
+<br/><br/>
+
 ## v4.3.14.1 - 2023-07-13 - [PR#946](https://github.com/NOAA-OWP/inundation-mapping/pull/946)
 
 ras2fim product had a need to run the acquire 3dep script to pull down some HUC8 DEMs. The old script was geared to HUC6 but could handle HUC8's but needed a few enhancements. ras2fim also did not need polys made from the DEMs, so a switch was added for that.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 0ba647bfc..a1ba00fff 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -105,7 +105,7 @@ if [ "$bathymetry_adjust" = "True" ]; then
     echo -e $startDiv"Performing Bathymetry Adjustment routine"
     # Run bathymetry adjustment routine
     Tstart
-    python3 $srcDir/bathymetric_adjustment.py -fim_dir $outputDestDir -bathy $inputsDir/bathymetry/bathymetry_adjustment_data.gpkg -buffer $wbd_buffer -wbd $inputsDir/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -j $jobLimit
+    python3 $srcDir/bathymetric_adjustment.py -fim_dir $outputDestDir -bathy $bathymetry_file -buffer $wbd_buffer -wbd $inputsDir/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -j $jobLimit
     Tcount
 fi
 

From 6162ab7f83f145f1d9fafd47f185c0a07bab7613 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 11 Aug 2023 11:38:59 -0500
Subject: [PATCH 262/359] Bump certifi from 2023.5.7 to 2023.7.22 (#951)

---
 Pipfile.lock | 1721 +++++++++++++++++++++++++++-----------------------
 1 file changed, 920 insertions(+), 801 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 0467c9a7c..2a22fc93b 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4'",
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -42,11 +42,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421",
-                "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"
+                "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780",
+                "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"
             ],
-            "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.6.2"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.7.1"
         },
         "argon2-cffi": {
             "hashes": [
@@ -161,7 +161,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version < '4' and python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8' and python_version < '4.0'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -174,11 +174,11 @@
         },
         "botocore": {
             "hashes": [
-                "sha256:31edc237088c104f7a05887646bbec31d7459dd2e108fd90cbffa315902817e2",
-                "sha256:3d145f30d10a9c712acee48e7ce906c9456bb25fe50d477c9312c702ccfa50d1"
+                "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df",
+                "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.29.138"
+            "version": "==1.29.165"
         },
         "brotli": {
             "hashes": [
@@ -270,11 +270,11 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7",
-                "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"
+                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
+                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2023.5.7"
+            "index": "pypi",
+            "version": "==2023.7.22"
         },
         "cffi": {
             "hashes": [
@@ -374,92 +374,92 @@
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6",
-                "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1",
-                "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e",
-                "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373",
-                "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62",
-                "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230",
-                "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be",
-                "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c",
-                "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0",
-                "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448",
-                "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f",
-                "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649",
-                "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d",
-                "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0",
-                "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706",
-                "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a",
-                "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59",
-                "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23",
-                "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5",
-                "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb",
-                "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e",
-                "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e",
-                "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c",
-                "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28",
-                "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d",
-                "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41",
-                "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974",
-                "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce",
-                "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f",
-                "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1",
-                "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d",
-                "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8",
-                "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017",
-                "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31",
-                "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7",
-                "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8",
-                "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e",
-                "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14",
-                "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd",
-                "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d",
-                "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795",
-                "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b",
-                "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b",
-                "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b",
-                "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203",
-                "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f",
-                "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19",
-                "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1",
-                "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a",
-                "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac",
-                "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9",
-                "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0",
-                "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137",
-                "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f",
-                "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6",
-                "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5",
-                "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909",
-                "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f",
-                "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0",
-                "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324",
-                "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755",
-                "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb",
-                "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854",
-                "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c",
-                "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60",
-                "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84",
-                "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0",
-                "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b",
-                "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1",
-                "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531",
-                "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1",
-                "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11",
-                "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326",
-                "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df",
-                "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"
+                "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
+                "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
+                "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
+                "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
+                "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
+                "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
+                "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
+                "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
+                "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
+                "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
+                "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
+                "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
+                "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
+                "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
+                "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
+                "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
+                "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
+                "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
+                "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
+                "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
+                "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
+                "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
+                "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
+                "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
+                "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
+                "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
+                "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
+                "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
+                "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
+                "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
+                "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
+                "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
+                "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
+                "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
+                "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
+                "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
+                "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
+                "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
+                "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
+                "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
+                "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
+                "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
+                "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
+                "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
+                "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
+                "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
+                "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
+                "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
+                "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
+                "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
+                "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
+                "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
+                "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
+                "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
+                "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
+                "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
+                "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
+                "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
+                "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
+                "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
+                "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
+                "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
+                "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
+                "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
+                "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
+                "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
+                "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
+                "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
+                "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
+                "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
+                "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
+                "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
+                "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
+                "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
+                "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.0"
+            "version": "==3.2.0"
         },
         "click": {
             "hashes": [
-                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
-                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
+                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
+                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.3"
+            "version": "==8.1.6"
         },
         "click-plugins": {
             "hashes": [
@@ -473,7 +473,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
             "version": "==0.7.2"
         },
         "comm": {
@@ -486,64 +486,48 @@
         },
         "contourpy": {
             "hashes": [
-                "sha256:031154ed61f7328ad7f97662e48660a150ef84ee1bc8876b6472af88bf5a9b98",
-                "sha256:0f9d350b639db6c2c233d92c7f213d94d2e444d8e8fc5ca44c9706cf72193772",
-                "sha256:130230b7e49825c98edf0b428b7aa1125503d91732735ef897786fe5452b1ec2",
-                "sha256:152fd8f730c31fd67fe0ffebe1df38ab6a669403da93df218801a893645c6ccc",
-                "sha256:1c71fdd8f1c0f84ffd58fca37d00ca4ebaa9e502fb49825484da075ac0b0b803",
-                "sha256:24847601071f740837aefb730e01bd169fbcaa610209779a78db7ebb6e6a7051",
-                "sha256:2e9ebb4425fc1b658e13bace354c48a933b842d53c458f02c86f371cecbedecc",
-                "sha256:30676ca45084ee61e9c3da589042c24a57592e375d4b138bd84d8709893a1ba4",
-                "sha256:31a55dccc8426e71817e3fe09b37d6d48ae40aae4ecbc8c7ad59d6893569c436",
-                "sha256:366a0cf0fc079af5204801786ad7a1c007714ee3909e364dbac1729f5b0849e5",
-                "sha256:38e2e577f0f092b8e6774459317c05a69935a1755ecfb621c0a98f0e3c09c9a5",
-                "sha256:3c184ad2433635f216645fdf0493011a4667e8d46b34082f5a3de702b6ec42e3",
-                "sha256:3caea6365b13119626ee996711ab63e0c9d7496f65641f4459c60a009a1f3e80",
-                "sha256:3e927b3868bd1e12acee7cc8f3747d815b4ab3e445a28d2e5373a7f4a6e76ba1",
-                "sha256:4ee3ee247f795a69e53cd91d927146fb16c4e803c7ac86c84104940c7d2cabf0",
-                "sha256:54d43960d809c4c12508a60b66cb936e7ed57d51fb5e30b513934a4a23874fae",
-                "sha256:57119b0116e3f408acbdccf9eb6ef19d7fe7baf0d1e9aaa5381489bc1aa56556",
-                "sha256:58569c491e7f7e874f11519ef46737cea1d6eda1b514e4eb5ac7dab6aa864d02",
-                "sha256:5a011cf354107b47c58ea932d13b04d93c6d1d69b8b6dce885e642531f847566",
-                "sha256:5caeacc68642e5f19d707471890f037a13007feba8427eb7f2a60811a1fc1350",
-                "sha256:5dd34c1ae752515318224cba7fc62b53130c45ac6a1040c8b7c1a223c46e8967",
-                "sha256:60835badb5ed5f4e194a6f21c09283dd6e007664a86101431bf870d9e86266c4",
-                "sha256:62398c80ef57589bdbe1eb8537127321c1abcfdf8c5f14f479dbbe27d0322e66",
-                "sha256:6381fa66866b0ea35e15d197fc06ac3840a9b2643a6475c8fff267db8b9f1e69",
-                "sha256:64757f6460fc55d7e16ed4f1de193f362104285c667c112b50a804d482777edd",
-                "sha256:69f8ff4db108815addd900a74df665e135dbbd6547a8a69333a68e1f6e368ac2",
-                "sha256:6c180d89a28787e4b73b07e9b0e2dac7741261dbdca95f2b489c4f8f887dd810",
-                "sha256:71b0bf0c30d432278793d2141362ac853859e87de0a7dee24a1cea35231f0d50",
-                "sha256:769eef00437edf115e24d87f8926955f00f7704bede656ce605097584f9966dc",
-                "sha256:7f6979d20ee5693a1057ab53e043adffa1e7418d734c1532e2d9e915b08d8ec2",
-                "sha256:87f4d8941a9564cda3f7fa6a6cd9b32ec575830780677932abdec7bcb61717b0",
-                "sha256:89ba9bb365446a22411f0673abf6ee1fea3b2cf47b37533b970904880ceb72f3",
-                "sha256:8acf74b5d383414401926c1598ed77825cd530ac7b463ebc2e4f46638f56cce6",
-                "sha256:9056c5310eb1daa33fc234ef39ebfb8c8e2533f088bbf0bc7350f70a29bde1ac",
-                "sha256:95c3acddf921944f241b6773b767f1cbce71d03307270e2d769fd584d5d1092d",
-                "sha256:9e20e5a1908e18aaa60d9077a6d8753090e3f85ca25da6e25d30dc0a9e84c2c6",
-                "sha256:a1e97b86f73715e8670ef45292d7cc033548266f07d54e2183ecb3c87598888f",
-                "sha256:a877ada905f7d69b2a31796c4b66e31a8068b37aa9b78832d41c82fc3e056ddd",
-                "sha256:a9d7587d2fdc820cc9177139b56795c39fb8560f540bba9ceea215f1f66e1566",
-                "sha256:abf298af1e7ad44eeb93501e40eb5a67abbf93b5d90e468d01fc0c4451971afa",
-                "sha256:ae90d5a8590e5310c32a7630b4b8618cef7563cebf649011da80874d0aa8f414",
-                "sha256:b6d0f9e1d39dbfb3977f9dd79f156c86eb03e57a7face96f199e02b18e58d32a",
-                "sha256:b8d587cc39057d0afd4166083d289bdeff221ac6d3ee5046aef2d480dc4b503c",
-                "sha256:c5210e5d5117e9aec8c47d9156d1d3835570dd909a899171b9535cb4a3f32693",
-                "sha256:cc331c13902d0f50845099434cd936d49d7a2ca76cb654b39691974cb1e4812d",
-                "sha256:ce41676b3d0dd16dbcfabcc1dc46090aaf4688fd6e819ef343dbda5a57ef0161",
-                "sha256:d8165a088d31798b59e91117d1f5fc3df8168d8b48c4acc10fc0df0d0bdbcc5e",
-                "sha256:e7281244c99fd7c6f27c1c6bfafba878517b0b62925a09b586d88ce750a016d2",
-                "sha256:e96a08b62bb8de960d3a6afbc5ed8421bf1a2d9c85cc4ea73f4bc81b4910500f",
-                "sha256:ed33433fc3820263a6368e532f19ddb4c5990855e4886088ad84fd7c4e561c71",
-                "sha256:efb8f6d08ca7998cf59eaf50c9d60717f29a1a0a09caa46460d33b2924839dbd",
-                "sha256:efe99298ba37e37787f6a2ea868265465410822f7bea163edcc1bd3903354ea9",
-                "sha256:f99e9486bf1bb979d95d5cffed40689cb595abb2b841f2991fc894b3452290e8",
-                "sha256:fc1464c97579da9f3ab16763c32e5c5d5bb5fa1ec7ce509a4ca6108b61b84fab",
-                "sha256:fd7dc0e6812b799a34f6d12fcb1000539098c249c8da54f3566c6a6461d0dbad"
-            ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==1.0.7"
+                "sha256:052cc634bf903c604ef1a00a5aa093c54f81a2612faedaa43295809ffdde885e",
+                "sha256:084eaa568400cfaf7179b847ac871582199b1b44d5699198e9602ecbbb5f6104",
+                "sha256:0b6616375d7de55797d7a66ee7d087efe27f03d336c27cf1f32c02b8c1a5ac70",
+                "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882",
+                "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f",
+                "sha256:17cfaf5ec9862bc93af1ec1f302457371c34e688fbd381f4035a06cd47324f48",
+                "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e",
+                "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a",
+                "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37",
+                "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a",
+                "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2",
+                "sha256:25ae46595e22f93592d39a7eac3d638cda552c3e1160255258b695f7b58e5655",
+                "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545",
+                "sha256:2b836d22bd2c7bb2700348e4521b25e077255ebb6ab68e351ab5aa91ca27e027",
+                "sha256:30f511c05fab7f12e0b1b7730ebdc2ec8deedcfb505bc27eb570ff47c51a8f15",
+                "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94",
+                "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439",
+                "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d",
+                "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa",
+                "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae",
+                "sha256:62013a2cf68abc80dadfd2307299bfa8f5aa0dcaec5b2954caeb5fa094171103",
+                "sha256:89f06eff3ce2f4b3eb24c1055a26981bffe4e7264acd86f15b97e40530b794bc",
+                "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa",
+                "sha256:911ff4fd53e26b019f898f32db0d4956c9d227d51338fb3b03ec72ff0084ee5f",
+                "sha256:9382a1c0bc46230fb881c36229bfa23d8c303b889b788b939365578d762b5c18",
+                "sha256:9f2931ed4741f98f74b410b16e5213f71dcccee67518970c42f64153ea9313b9",
+                "sha256:a67259c2b493b00e5a4d0f7bfae51fb4b3371395e47d079a4446e9b0f4d70e76",
+                "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493",
+                "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9",
+                "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed",
+                "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4",
+                "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f",
+                "sha256:dffcc2ddec1782dd2f2ce1ef16f070861af4fb78c69862ce0aab801495dda6a3",
+                "sha256:e53046c3863828d21d531cc3b53786e6580eb1ba02477e8681009b6aa0870b21",
+                "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e",
+                "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1",
+                "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a",
+                "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002",
+                "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.1.0"
         },
         "cycler": {
             "hashes": [
@@ -555,49 +539,67 @@
         },
         "cython": {
             "hashes": [
-                "sha256:03daae07f8cbf797506446adae512c3dd86e7f27a62a541fa1ee254baf43e32c",
-                "sha256:0963266dad685812c1dbb758fcd4de78290e3adc7db271c8664dcde27380b13e",
-                "sha256:0ab3cbf3d62b0354631a45dc93cfcdf79098663b1c65a6033af4a452b52217a7",
-                "sha256:0e9032cd650b0cb1d2c2ef2623f5714c14d14c28d7647d589c3eeed0baf7428e",
-                "sha256:11b1b278b8edef215caaa5250ad65a10023bfa0b5a93c776552248fc6f60098d",
-                "sha256:1909688f5d7b521a60c396d20bba9e47a1b2d2784bfb085401e1e1e7d29a29a8",
-                "sha256:1d6c809e2f9ce5950bbc52a1d2352ef3d4fc56186b64cb0d50c8c5a3c1d17661",
-                "sha256:21b88200620d80cfe193d199b259cdad2b9af56f916f0f7f474b5a3631ca0caa",
-                "sha256:308c8f1e58bf5e6e8a1c4dcf8abbd2d13d0f9b1e582f4d9ae8b89857342d8bb5",
-                "sha256:44733366f1604b0c327613b6918469284878d2f5084297d10d26072fc6948d51",
-                "sha256:459994d1de0f99bb18fad9f2325f760c4b392b1324aef37bcc1cd94922dfce41",
-                "sha256:4a2723447d1334484681d5aede34184f2da66317891f94b80e693a2f96a8f1a7",
-                "sha256:56866323f1660cecb4d5ff3a1fba92a56b91b7cfae0a8253777aa4bdb3bdf9a8",
-                "sha256:5718319a01489688fdd22ddebb8e2fcbbd60be5f30de4336ea7063c3ae29fbe5",
-                "sha256:5a8de3e793a576e40ca9b4f5518610cd416273c7dc5e254115656b6e4ec70663",
-                "sha256:5c121dc185040f4333bfded68963b4529698e1b6d994da56be32c97a90c896b6",
-                "sha256:60969d38e6a456a67e7ef8ae20668eff54e32ba439d4068ccf2854a44275a30f",
-                "sha256:67b850cf46b861bc27226d31e1d87c0e69869a02f8d3cc5d5bef549764029879",
-                "sha256:742544024ddb74314e2d597accdb747ed76bd126e61fcf49940a5b5be0a8f381",
-                "sha256:7595d29eaee95633dd8060f50f0e54b27472d01587659557ebcfe39da3ea946b",
-                "sha256:7879992487d9060a61393eeefe00d299210256928dce44d887b6be313d342bac",
-                "sha256:8c3cd8bb8e880a3346f5685601004d96e0a2221e73edcaeea57ea848618b4ac6",
-                "sha256:9489de5b2044dcdfd9d6ca8242a02d560137b3c41b1f5ae1c4f6707d66d6e44d",
-                "sha256:a0f4229df10bc4545ebbeaaf96ebb706011d8b333e54ed202beb03f2bee0a50e",
-                "sha256:a8ad755f9364e720f10a36734a1c7a5ced5c679446718b589259261438a517c9",
-                "sha256:b6149f7cc5b31bccb158c5b968e5a8d374fdc629792e7b928a9b66e08b03fca5",
-                "sha256:bdb3285660e3068438791ace7dd7b1efd6b442a10b5c8d7a4f0c9d184d08c8ed",
-                "sha256:be4f6b7be75a201c290c8611c0978549c60353890204573078e865423dbe3c83",
-                "sha256:ccb223b5f0fd95d8d27561efc0c14502c0945f1a32274835831efa5d5baddfc1",
-                "sha256:cfb2302ef617d647ee590a4c0a00ba3c2da05f301dcefe7721125565d2e51351",
-                "sha256:d7ef5f68f4c5baa93349ea54a352f8716d18bee9a37f3e93eff38a5d4e9b7262",
-                "sha256:d8f822fb6ecd5d88c42136561f82960612421154fc5bf23c57103a367bb91356",
-                "sha256:dbd79221869ee9a6ccc4953b2c8838bb6ae08ab4d50ea4b60d7894f03739417b",
-                "sha256:dce0a36d163c05ae8b21200059511217d79b47baf2b7b0f926e8367bd7a3cc24",
-                "sha256:e40cf86aadc29ecd1cb6de67b0d9488705865deea4fc185c7ad56d7a6fc78703",
-                "sha256:e4401270b0dc464c23671e2e9d52a60985f988318febaf51b047190e855bbe7d",
-                "sha256:e6ef7879668214d80ea3914c17e7d4e1ebf4242e0dd4dabe95ca5ccbe75589a5",
-                "sha256:e971db8aeb12e7c0697cefafe65eefcc33ff1224ae3d8c7f83346cbc42c6c270",
-                "sha256:f674ceb5f722d364395f180fbac273072fc1a266aab924acc9cfd5afc645aae1",
-                "sha256:fd1ea21f1cebf33ae288caa0f3e9b5563a709f4df8925d53bad99be693fc0d9b"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.29.34"
+                "sha256:06db81b1a01858fcc406616f8528e686ffb6cf7c3d78fb83767832bfecea8ad8",
+                "sha256:06fcb4628ccce2ba5abc8630adbeaf4016f63a359b4c6c3827b2d80e0673981c",
+                "sha256:090256c687106932339f87f888b95f0d69c617bc9b18801555545b695d29d8ab",
+                "sha256:090e24cfa31c926d0b13d8bb2ef48175acdd061ae1413343c94a2b12a4a4fa6f",
+                "sha256:0d2c1e172f1c81bafcca703093608e10dc16e3e2d24c5644c17606c7fdb1792c",
+                "sha256:0e1e4b7e4bfbf22fecfa5b852f0e499c442d4853b7ebd33ae37cdec9826ed5d8",
+                "sha256:204690be60f0ff32eb70b04f28ef0d1e50ffd7b3f77ba06a7dc2389ee3b848e0",
+                "sha256:20c604e974832aaf8b7a1f5455ee7274b34df62a35ee095cd7d2ed7e818e6c53",
+                "sha256:254ed1f03a6c237fa64f0c6e44862058de65bfa2e6a3b48ca3c205492e0653aa",
+                "sha256:296c53b6c0030cf82987eef163444e8d7631cc139d995f9d58679d9fd1ddbf31",
+                "sha256:2d8158277c8942c0b20ff4c074fe6a51c5b89e6ac60cef606818de8c92773596",
+                "sha256:2fadde1da055944f5e1e17625055f54ddd11f451889110278ef30e07bd5e1695",
+                "sha256:30a4bd2481e59bd7ab2539f835b78edc19fc455811e476916f56026b93afd28b",
+                "sha256:30f10e79393b411af7677c270ea69807acb9fc30205c8ff25561f4deef780ec1",
+                "sha256:33c900d1ca9f622b969ac7d8fc44bdae140a4a6c7d8819413b51f3ccd0586a09",
+                "sha256:350b18f9673e63101dbbfcf774ee2f57c20ac4636d255741d76ca79016b1bd82",
+                "sha256:35abcf07b8277ec95bbe49a07b5c8760a2d941942ccfe759a94c8d2fe5602e9f",
+                "sha256:3b71b399b10b038b056ad12dce1e317a8aa7a96e99de7e4fa2fa5d1c9415cfb9",
+                "sha256:3e234e2549e808d9259fdb23ebcfd145be30c638c65118326ec33a8d29248dc2",
+                "sha256:4123c8d03167803df31da6b39de167cb9c04ac0aa4e35d4e5aa9d08ad511b84d",
+                "sha256:4965f2ebade17166f21a508d66dd60d2a0b3a3b90abe3f72003baa17ae020dd6",
+                "sha256:4cd00f2158dc00f7f93a92444d0f663eda124c9c29bbbd658964f4e89c357fe8",
+                "sha256:4dc6bbe7cf079db37f1ebb9b0f10d0d7f29e293bb8688e92d50b5ea7a91d82f3",
+                "sha256:4e212237b7531759befb92699c452cd65074a78051ae4ee36ff8b237395ecf3d",
+                "sha256:53328a8af0806bebbdb48a4191883b11ee9d9dfb084d84f58fa5a8ab58baefc9",
+                "sha256:54e34f99b2a8c1e11478541b2822e6408c132b98b6b8f5ed89411e5e906631ea",
+                "sha256:5962e70b15e863e72bed6910e8c6ffef77d36cc98e2b31c474378f3b9e49b0e3",
+                "sha256:5b4cc896d49ce2bae8d6a030f9a4c64965b59c38acfbf4617685e17f7fcf1731",
+                "sha256:609777d3a7a0a23b225e84d967af4ad2485c8bdfcacef8037cf197e87d431ca0",
+                "sha256:626a4a6ef4b7ced87c348ea805488e4bd39dad9d0b39659aa9e1040b62bbfedf",
+                "sha256:650d03ddddc08b051b4659778733f0f173ca7d327415755c05d265a6c1ba02fb",
+                "sha256:6b00df42cdd1a285a64491ba23de08ab14169d3257c840428d40eb7e8e9979af",
+                "sha256:7c7d728e1a49ad01d41181e3a9ea80b8d14e825f4679e4dd837cbf7bca7998a5",
+                "sha256:7f4a6dfd42ae0a45797f50fc4f6add702abf46ab3e7cd61811a6c6a97a40e1a2",
+                "sha256:829c8333195100448a23863cf64a07e1334fae6a275aefe871458937911531b6",
+                "sha256:84176bd04ce9f3cc8799b47ec6d1959fa1ea5e71424507df7bbf0b0915bbedef",
+                "sha256:877d1c8745df59dd2061a0636c602729e9533ba13f13aa73a498f68662e1cbde",
+                "sha256:8abb8915eb2e57fa53d918afe641c05d1bcc6ed1913682ec1f28de71f4e3f398",
+                "sha256:8d86651347bbdbac1aca1824696c5e4c0a3b162946c422edcca2be12a03744d1",
+                "sha256:93a34e1ca8afa4b7075b02ed14a7e4969256297029fb1bfd4cbe48f7290dbcff",
+                "sha256:9e69139f4e60ab14c50767a568612ea64d6907e9c8e0289590a170eb495e005f",
+                "sha256:9f29307463eba53747b31f71394ed087e3e3e264dcc433e62de1d51f5c0c966c",
+                "sha256:a1cdd01ce45333bc264a218c6e183700d6b998f029233f586a53c9b13455c2d2",
+                "sha256:a44d6b9a29b2bff38bb648577b2fcf6a68cf8b1783eee89c2eb749f69494b98d",
+                "sha256:a65bc50dc1bc2faeafd9425defbdef6a468974f5c4192497ff7f14adccfdcd32",
+                "sha256:aa606675c6bd23478b1d174e2a84e3c5a2c660968f97dc455afe0fae198f9d3d",
+                "sha256:bb1165ca9e78823f9ad1efa5b3d83156f868eabd679a615d140a3021bb92cd65",
+                "sha256:bc816d8eb3686d6f8d165f4156bac18c1147e1035dc28a76742d0b7fb5b7c032",
+                "sha256:c40bdbcb2286f0aeeb5df9ce53d45da2d2a9b36a16b331cd0809d212d22a8fc7",
+                "sha256:c85fd2b1cbd9400d60ebe074795bb9a9188752f1612be3b35b0831a24879b91f",
+                "sha256:c93634845238645ce7abf63a56b1c5b6248189005c7caff898fd4a0dac1c5e1e",
+                "sha256:cec2a67a0a7d9d4399758c0657ca03e5912e37218859cfbf046242cc532bfb3b",
+                "sha256:d3355e6f690184f984eeb108b0f5bbc4bcf8b9444f8168933acf79603abf7baf",
+                "sha256:e28763e75e380b8be62b02266a7995a781997c97c119efbdccb8fb954bcd7574",
+                "sha256:ecee663d2d50ca939fc5db81f2f8a219c2417b4651ad84254c50a03a9cb1aadd",
+                "sha256:edae615cb4af51d5173e76ba9aea212424d025c57012e9cdf2f131f774c5ba71",
+                "sha256:f42f304c097cc53e9eb5f1a1d150380353d5018a3191f1b77f0de353c762181e",
+                "sha256:ff1aef1a03cfe293237c7a86ae9625b0411b2df30c53d1a7f29a8d381f38a1df"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==3.0.0"
         },
         "debugpy": {
             "hashes": [
@@ -641,11 +643,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e",
-                "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"
+                "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5",
+                "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.1.1"
+            "version": "==1.1.2"
         },
         "executing": {
             "hashes": [
@@ -656,10 +658,10 @@
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:4b90b252628ca695280924d863fe37234eebadc29c5360d322571233dc9746e0",
-                "sha256:f4eeb8a77cef54861dbf7424ac8ce71306f12cbb086c45131bcba2c6a4f726e3"
+                "sha256:128039912a11a807068a7c87d0da36660afbfd7202780db26c4aa7153cfdc799",
+                "sha256:e820349dd16f806e4bd1467a138dced9def4bc7d6213a34295272a6cac95b5bd"
             ],
-            "version": "==2.17.1"
+            "version": "==2.18.0"
         },
         "fiona": {
             "hashes": [
@@ -688,11 +690,43 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:106caf6167c4597556b31a8d9175a3fdc0356fdcd70ab19973c3b0d4c893c461",
-                "sha256:dba8d7cdb8e2bac1b3da28c5ed5960de09e59a2fe7e63bb73f5a59e57b0430d2"
-            ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==4.39.4"
+                "sha256:1df1b6f4c7c4bc8201eb47f3b268adbf2539943aa43c400f84556557e3e109c0",
+                "sha256:2a22b2c425c698dcd5d6b0ff0b566e8e9663172118db6fd5f1941f9b8063da9b",
+                "sha256:33191f062549e6bb1a4782c22a04ebd37009c09360e2d6686ac5083774d06d95",
+                "sha256:38cdecd8f1fd4bf4daae7fed1b3170dfc1b523388d6664b2204b351820aa78a7",
+                "sha256:3ae64303ba670f8959fdaaa30ba0c2dabe75364fdec1caeee596c45d51ca3425",
+                "sha256:3d1f9471134affc1e3b1b806db6e3e2ad3fa99439e332f1881a474c825101096",
+                "sha256:4e3334d51f0e37e2c6056e67141b2adabc92613a968797e2571ca8a03bd64773",
+                "sha256:4edc795533421e98f60acee7d28fc8d941ff5ac10f44668c9c3635ad72ae9045",
+                "sha256:547ab36a799dded58a46fa647266c24d0ed43a66028cd1cd4370b246ad426cac",
+                "sha256:59eba8b2e749a1de85760da22333f3d17c42b66e03758855a12a2a542723c6e7",
+                "sha256:704bccd69b0abb6fab9f5e4d2b75896afa48b427caa2c7988792a2ffce35b441",
+                "sha256:73ef0bb5d60eb02ba4d3a7d23ada32184bd86007cb2de3657cfcb1175325fc83",
+                "sha256:7763316111df7b5165529f4183a334aa24c13cdb5375ffa1dc8ce309c8bf4e5c",
+                "sha256:849ec722bbf7d3501a0e879e57dec1fc54919d31bff3f690af30bb87970f9784",
+                "sha256:891cfc5a83b0307688f78b9bb446f03a7a1ad981690ac8362f50518bc6153975",
+                "sha256:952cb405f78734cf6466252fec42e206450d1a6715746013f64df9cbd4f896fa",
+                "sha256:a7bbb290d13c6dd718ec2c3db46fe6c5f6811e7ea1e07f145fd8468176398224",
+                "sha256:a9b3cc10dc9e0834b6665fd63ae0c6964c6bc3d7166e9bc84772e0edd09f9fa2",
+                "sha256:aaaef294d8e411f0ecb778a0aefd11bb5884c9b8333cc1011bdaf3b58ca4bd75",
+                "sha256:afce2aeb80be72b4da7dd114f10f04873ff512793d13ce0b19d12b2a4c44c0f0",
+                "sha256:b0938ebbeccf7c80bb9a15e31645cf831572c3a33d5cc69abe436e7000c61b14",
+                "sha256:b2d1ee95be42b80d1f002d1ee0a51d7a435ea90d36f1a5ae331be9962ee5a3f1",
+                "sha256:b927e5f466d99c03e6e20961946314b81d6e3490d95865ef88061144d9f62e38",
+                "sha256:bdd729744ae7ecd7f7311ad25d99da4999003dcfe43b436cf3c333d4e68de73d",
+                "sha256:c2071267deaa6d93cb16288613419679c77220543551cbe61da02c93d92df72f",
+                "sha256:cac73bbef7734e78c60949da11c4903ee5837168e58772371bd42a75872f4f82",
+                "sha256:da2c2964bdc827ba6b8a91dc6de792620be4da3922c4cf0599f36a488c07e2b2",
+                "sha256:e16a9449f21a93909c5be2f5ed5246420f2316e94195dbfccb5238aaa38f9751",
+                "sha256:e5c2b0a95a221838991e2f0e455dec1ca3a8cc9cd54febd68cc64d40fdb83669",
+                "sha256:ec453a45778524f925a8f20fd26a3326f398bfc55d534e37bab470c5e415caa1",
+                "sha256:edee0900cf0eedb29d17c7876102d6e5a91ee333882b1f5abc83e85b934cadb5",
+                "sha256:f14f3ccea4cc7dd1b277385adf3c3bf18f9860f87eab9c2fb650b0af16800f55",
+                "sha256:f240d9adf0583ac8fc1646afe7f4ac039022b6f8fa4f1575a2cfa53675360b69",
+                "sha256:f48602c0b3fd79cd83a34c40af565fe6db7ac9085c8823b552e6e751e3a5b8be"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.41.1"
         },
         "fqdn": {
             "hashes": [
@@ -719,19 +753,19 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed",
-                "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"
+                "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb",
+                "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==6.6.0"
+            "version": "==6.8.0"
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6",
-                "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"
+                "sha256:4cf94875a8368bd89531a756df9a9ebe1f150e0f885030b461237bc7f2d905f2",
+                "sha256:d952faee11004c045f785bb5636e8f885bed30dc3c940d5d42798a2a4541c185"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==5.12.0"
+            "version": "==6.0.0"
         },
         "inflate64": {
             "hashes": [
@@ -818,11 +852,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:1aba0ae8453e15e9bc6b24e497ef6840114afcdb832ae597f32137fa19d42a6f",
-                "sha256:77aeffab056c21d16f1edccdc9e5ccbf7d96eb401bd6703610a21be8b068aadc"
+                "sha256:e342ce84712861be4b248c4a73472be4702c1b0dd77448bfd6bcfb3af9d5ddf9",
+                "sha256:f0042e867ac3f6bca1679e6a88cbd6a58ed93a44f9d0866aecde6efe8de76659"
             ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==6.23.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==6.25.0"
         },
         "ipympl": {
             "hashes": [
@@ -837,7 +871,7 @@
                 "sha256:c7b80eb7f5a855a88efc971fda506ff7a91c280b42cdae26643e0f601ea281ea",
                 "sha256:ea8801f15dfe4ffb76dea1b09b847430ffd70d827b41735c64a0638a04103bfc"
             ],
-            "markers": "python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8'",
             "version": "==8.12.2"
         },
         "ipython-genutils": {
@@ -849,11 +883,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:a60bf8d2528997e05ac83fd19ea2fbe65f2e79fbe1b2b35779bdfc46c2941dcc",
-                "sha256:de7d779f2045d60de9f6c25f653fdae2dba57898e6a1284494b3ba20b6893bb8"
+                "sha256:50ace0a8886e9a0d68b980db82f94c25d55d21ff2340ed36f802dd9365e94acf",
+                "sha256:e0aed0c95a1e55b6a123f64305245578bdc09e52965a34941c2b6a578b8c64a0"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.0.6"
+            "version": "==8.0.7"
         },
         "isoduration": {
             "hashes": [
@@ -895,21 +929,29 @@
         },
         "jsonpointer": {
             "hashes": [
-                "sha256:51801e558539b4e9cd268638c078c6c5746c9ac96bc38152d443400e4f3793e9",
-                "sha256:97cba51526c829282218feb99dab1b1e6bdf8efd1c43dc9d57be093c0d69c99a"
+                "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a",
+                "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"
             ],
-            "version": "==2.3"
+            "version": "==2.4"
         },
         "jsonschema": {
             "extras": [
                 "format-nongpl"
             ],
             "hashes": [
-                "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d",
-                "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"
+                "sha256:971be834317c22daaa9132340a51c01b50910724082c2c1a2ac87eeec153a3fe",
+                "sha256:fb3642735399fa958c0d2aad7057901554596c63349f4f6b283c493cf692a25d"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==4.17.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.18.4"
+        },
+        "jsonschema-specifications": {
+            "hashes": [
+                "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1",
+                "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.7.1"
         },
         "jupyter": {
             "hashes": [
@@ -922,11 +964,11 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:9fe233834edd0e6c0aa5f05ca2ab4bdea1842bfd2d8a932878212fc5301ddaf0",
-                "sha256:b18219aa695d39e2ad570533e0d71fb7881d35a873051054a84ee2a17c4b7389"
+                "sha256:3af69921fe99617be1670399a0b857ad67275eefcfa291e2c81a160b7b650f5f",
+                "sha256:7441af0c0672edc5d28035e92ba5e32fadcfa8a4e608a434c228836a89df6158"
             ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==8.2.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==8.3.0"
         },
         "jupyter-console": {
             "hashes": [
@@ -938,11 +980,11 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:6db75be0c83edbf1b7c9f91ec266a9a24ef945da630f3120e1a0046dc13713fc",
-                "sha256:d4201af84559bc8c70cead287e1ab94aeef3c512848dde077b7684b54d67730d"
+                "sha256:5ba5c7938a7f97a6b0481463f7ff0dbac7c15ba48cf46fa4035ca6e838aa1aba",
+                "sha256:ae9036db959a71ec1cac33081eeb040a79e681f08ab68b0883e9a676c7a90dce"
             ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==5.3.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.3.1"
         },
         "jupyter-events": {
             "hashes": [
@@ -954,11 +996,11 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:9fde612791f716fd34d610cd939704a9639643744751ba66e7ee8fdc9cead07e",
-                "sha256:e6bc1e9e96d7c55b9ce9699ff6cb9a910581fe7349e27c40389acb67632e24c0"
+                "sha256:36da0a266d31a41ac335a366c88933c17dfa5bb817a48f5c02c16d303bc9477f",
+                "sha256:6a77912aff643e53fa14bdb2634884b52b784a4be77ce8e93f7283faed0f0849"
             ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==2.5.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.7.0"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -973,7 +1015,7 @@
                 "sha256:57ab779797c25a7ba68e97bcfb5d7740f2b5e8a83b5e8102b10438041a7eac5d",
                 "sha256:75779164661cec02a8758a5311e18bb8eb70c4e86c6b699403100f1585a12a36"
             ],
-            "markers": "python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8'",
             "version": "==0.4.4"
         },
         "jupyter-server-ydoc": {
@@ -986,11 +1028,11 @@
         },
         "jupyter-ydoc": {
             "hashes": [
-                "sha256:a3f670a69135e90493ffb91d6788efe2632bf42c6cc42a25f25c2e6eddd55a0e",
-                "sha256:d1a51c73ead6f6417bec0450f53c577a66abe8d43e9c2d8a1acaf7a17259f843"
+                "sha256:5759170f112c70320a84217dd98d287699076ae65a7f88d458d57940a9f2b882",
+                "sha256:5a02ca7449f0d875f73e8cb8efdf695dddef15a8e71378b1f4eda6b7c90f5382"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.2.4"
+            "version": "==0.2.5"
         },
         "jupyterlab": {
             "hashes": [
@@ -1010,19 +1052,19 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:1c8eb55c7cd70a50a51fef42a7b4e26ef2f7fc48728f0290604bd89b1dd156e6",
-                "sha256:dfaaf898af84b9d01ae9583b813f378b96ee90c3a66f24c5186ea5d1bbdb2089"
+                "sha256:4e6f99e0a5579bbbc32e449c4dbb039561d4f1a7827d5733273ed56738f21f07",
+                "sha256:5f077e142bb8dc9b843d960f940c513581bceca3793a0d80f9c67d9522c4e876"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.22.1"
+            "version": "==2.24.0"
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:c3a50ed5bf528a0c7a869096503af54702f86dda1db469aee1c92dc0c01b43ca",
-                "sha256:c73f8370338ec19f1bec47254752d6505b03601cbd5a67e6a0b184532f73a459"
+                "sha256:4715912d6ceab839c9db35953c764b3214ebbc9161c809f6e0510168845dfdf5",
+                "sha256:d428ab97b8d87cc7c54cbf37644d6e0f0e662f23876e05fa460a73ec3257252a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.7"
+            "version": "==3.0.8"
         },
         "kiwisolver": {
             "hashes": [
@@ -1134,106 +1176,106 @@
         },
         "markupsafe": {
             "hashes": [
-                "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed",
-                "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc",
-                "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2",
-                "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460",
-                "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7",
-                "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0",
-                "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1",
-                "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa",
-                "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03",
-                "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323",
-                "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65",
-                "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013",
-                "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036",
-                "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f",
-                "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4",
-                "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419",
-                "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2",
-                "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619",
-                "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a",
-                "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a",
-                "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd",
-                "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7",
-                "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666",
-                "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65",
-                "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859",
-                "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625",
-                "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff",
-                "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156",
-                "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd",
-                "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba",
-                "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f",
-                "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1",
-                "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094",
-                "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a",
-                "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513",
-                "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed",
-                "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d",
-                "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3",
-                "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147",
-                "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c",
-                "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603",
-                "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601",
-                "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a",
-                "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1",
-                "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d",
-                "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3",
-                "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54",
-                "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2",
-                "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6",
-                "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"
+                "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e",
+                "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e",
+                "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431",
+                "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686",
+                "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559",
+                "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc",
+                "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c",
+                "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0",
+                "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4",
+                "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9",
+                "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575",
+                "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba",
+                "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d",
+                "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3",
+                "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00",
+                "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155",
+                "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac",
+                "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52",
+                "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f",
+                "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8",
+                "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b",
+                "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24",
+                "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea",
+                "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198",
+                "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0",
+                "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee",
+                "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be",
+                "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2",
+                "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707",
+                "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6",
+                "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58",
+                "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779",
+                "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636",
+                "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c",
+                "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad",
+                "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee",
+                "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc",
+                "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2",
+                "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48",
+                "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7",
+                "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e",
+                "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b",
+                "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa",
+                "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5",
+                "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e",
+                "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb",
+                "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9",
+                "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57",
+                "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc",
+                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.1.2"
+            "version": "==2.1.3"
         },
         "matplotlib": {
             "hashes": [
-                "sha256:08308bae9e91aca1ec6fd6dda66237eef9f6294ddb17f0d0b3c863169bf82353",
-                "sha256:14645aad967684e92fc349493fa10c08a6da514b3d03a5931a1bac26e6792bd1",
-                "sha256:21e9cff1a58d42e74d01153360de92b326708fb205250150018a52c70f43c290",
-                "sha256:28506a03bd7f3fe59cd3cd4ceb2a8d8a2b1db41afede01f66c42561b9be7b4b7",
-                "sha256:2bf092f9210e105f414a043b92af583c98f50050559616930d884387d0772aba",
-                "sha256:3032884084f541163f295db8a6536e0abb0db464008fadca6c98aaf84ccf4717",
-                "sha256:3a2cb34336110e0ed8bb4f650e817eed61fa064acbefeb3591f1b33e3a84fd96",
-                "sha256:3ba2af245e36990facf67fde840a760128ddd71210b2ab6406e640188d69d136",
-                "sha256:3d7bc90727351fb841e4d8ae620d2d86d8ed92b50473cd2b42ce9186104ecbba",
-                "sha256:438196cdf5dc8d39b50a45cb6e3f6274edbcf2254f85fa9b895bf85851c3a613",
-                "sha256:46a561d23b91f30bccfd25429c3c706afe7d73a5cc64ef2dfaf2b2ac47c1a5dc",
-                "sha256:4cf327e98ecf08fcbb82685acaf1939d3338548620ab8dfa02828706402c34de",
-                "sha256:4f99e1b234c30c1e9714610eb0c6d2f11809c9c78c984a613ae539ea2ad2eb4b",
-                "sha256:544764ba51900da4639c0f983b323d288f94f65f4024dc40ecb1542d74dc0500",
-                "sha256:56d94989191de3fcc4e002f93f7f1be5da476385dde410ddafbb70686acf00ea",
-                "sha256:57bfb8c8ea253be947ccb2bc2d1bb3862c2bccc662ad1b4626e1f5e004557042",
-                "sha256:617f14ae9d53292ece33f45cba8503494ee199a75b44de7717964f70637a36aa",
-                "sha256:6eb88d87cb2c49af00d3bbc33a003f89fd9f78d318848da029383bfc08ecfbfb",
-                "sha256:75d4725d70b7c03e082bbb8a34639ede17f333d7247f56caceb3801cb6ff703d",
-                "sha256:770a205966d641627fd5cf9d3cb4b6280a716522cd36b8b284a8eb1581310f61",
-                "sha256:7b73305f25eab4541bd7ee0b96d87e53ae9c9f1823be5659b806cd85786fe882",
-                "sha256:7c9a4b2da6fac77bcc41b1ea95fadb314e92508bf5493ceff058e727e7ecf5b0",
-                "sha256:81a6b377ea444336538638d31fdb39af6be1a043ca5e343fe18d0f17e098770b",
-                "sha256:83111e6388dec67822e2534e13b243cc644c7494a4bb60584edbff91585a83c6",
-                "sha256:8704726d33e9aa8a6d5215044b8d00804561971163563e6e6591f9dcf64340cc",
-                "sha256:89768d84187f31717349c6bfadc0e0d8c321e8eb34522acec8a67b1236a66332",
-                "sha256:8bf26ade3ff0f27668989d98c8435ce9327d24cffb7f07d24ef609e33d582439",
-                "sha256:8c587963b85ce41e0a8af53b9b2de8dddbf5ece4c34553f7bd9d066148dc719c",
-                "sha256:95cbc13c1fc6844ab8812a525bbc237fa1470863ff3dace7352e910519e194b1",
-                "sha256:97cc368a7268141afb5690760921765ed34867ffb9655dd325ed207af85c7529",
-                "sha256:a867bf73a7eb808ef2afbca03bcdb785dae09595fbe550e1bab0cd023eba3de0",
-                "sha256:b867e2f952ed592237a1828f027d332d8ee219ad722345b79a001f49df0936eb",
-                "sha256:c0bd19c72ae53e6ab979f0ac6a3fafceb02d2ecafa023c5cca47acd934d10be7",
-                "sha256:ce463ce590f3825b52e9fe5c19a3c6a69fd7675a39d589e8b5fbe772272b3a24",
-                "sha256:cf0e4f727534b7b1457898c4f4ae838af1ef87c359b76dcd5330fa31893a3ac7",
-                "sha256:def58098f96a05f90af7e92fd127d21a287068202aa43b2a93476170ebd99e87",
-                "sha256:e99bc9e65901bb9a7ce5e7bb24af03675cbd7c70b30ac670aa263240635999a4",
-                "sha256:eb7d248c34a341cd4c31a06fd34d64306624c8cd8d0def7abb08792a5abfd556",
-                "sha256:f67bfdb83a8232cb7a92b869f9355d677bce24485c460b19d01970b64b2ed476",
-                "sha256:f883a22a56a84dba3b588696a2b8a1ab0d2c3d41be53264115c71b0a942d8fdb",
-                "sha256:fbdeeb58c0cf0595efe89c05c224e0a502d1aa6a8696e68a73c3efc6bc354304"
-            ],
-            "markers": "python_full_version >= '3.8.0'",
-            "version": "==3.7.1"
+                "sha256:070f8dddd1f5939e60aacb8fa08f19551f4b0140fab16a3669d5cd6e9cb28fc8",
+                "sha256:0c3cca3e842b11b55b52c6fb8bd6a4088693829acbfcdb3e815fa9b7d5c92c1b",
+                "sha256:0f506a1776ee94f9e131af1ac6efa6e5bc7cb606a3e389b0ccb6e657f60bb676",
+                "sha256:12f01b92ecd518e0697da4d97d163b2b3aa55eb3eb4e2c98235b3396d7dad55f",
+                "sha256:152ee0b569a37630d8628534c628456b28686e085d51394da6b71ef84c4da201",
+                "sha256:1c308b255efb9b06b23874236ec0f10f026673ad6515f602027cc8ac7805352d",
+                "sha256:1cd120fca3407a225168238b790bd5c528f0fafde6172b140a2f3ab7a4ea63e9",
+                "sha256:20f844d6be031948148ba49605c8b96dfe7d3711d1b63592830d650622458c11",
+                "sha256:23fb1750934e5f0128f9423db27c474aa32534cec21f7b2153262b066a581fd1",
+                "sha256:2699f7e73a76d4c110f4f25be9d2496d6ab4f17345307738557d345f099e07de",
+                "sha256:26bede320d77e469fdf1bde212de0ec889169b04f7f1179b8930d66f82b30cbc",
+                "sha256:2ecb5be2b2815431c81dc115667e33da0f5a1bcf6143980d180d09a717c4a12e",
+                "sha256:2f8e4a49493add46ad4a8c92f63e19d548b2b6ebbed75c6b4c7f46f57d36cdd1",
+                "sha256:305e3da477dc8607336ba10bac96986d6308d614706cae2efe7d3ffa60465b24",
+                "sha256:30e1409b857aa8a747c5d4f85f63a79e479835f8dffc52992ac1f3f25837b544",
+                "sha256:318c89edde72ff95d8df67d82aca03861240512994a597a435a1011ba18dbc7f",
+                "sha256:35d74ebdb3f71f112b36c2629cf32323adfbf42679e2751252acd468f5001c07",
+                "sha256:50e0a55ec74bf2d7a0ebf50ac580a209582c2dd0f7ab51bc270f1b4a0027454e",
+                "sha256:5dea00b62d28654b71ca92463656d80646675628d0828e08a5f3b57e12869e13",
+                "sha256:60c521e21031632aa0d87ca5ba0c1c05f3daacadb34c093585a0be6780f698e4",
+                "sha256:6515e878f91894c2e4340d81f0911857998ccaf04dbc1bba781e3d89cbf70608",
+                "sha256:6d2ff3c984b8a569bc1383cd468fc06b70d7b59d5c2854ca39f1436ae8394117",
+                "sha256:71667eb2ccca4c3537d9414b1bc00554cb7f91527c17ee4ec38027201f8f1603",
+                "sha256:717157e61b3a71d3d26ad4e1770dc85156c9af435659a25ee6407dc866cb258d",
+                "sha256:71f7a8c6b124e904db550f5b9fe483d28b896d4135e45c4ea381ad3b8a0e3256",
+                "sha256:936bba394682049919dda062d33435b3be211dc3dcaa011e09634f060ec878b2",
+                "sha256:a1733b8e84e7e40a9853e505fe68cc54339f97273bdfe6f3ed980095f769ddc7",
+                "sha256:a2c1590b90aa7bd741b54c62b78de05d4186271e34e2377e0289d943b3522273",
+                "sha256:a7e28d6396563955f7af437894a36bf2b279462239a41028323e04b85179058b",
+                "sha256:a8035ba590658bae7562786c9cc6ea1a84aa49d3afab157e414c9e2ea74f496d",
+                "sha256:a8cdb91dddb04436bd2f098b8fdf4b81352e68cf4d2c6756fcc414791076569b",
+                "sha256:ac60daa1dc83e8821eed155796b0f7888b6b916cf61d620a4ddd8200ac70cd64",
+                "sha256:af4860132c8c05261a5f5f8467f1b269bf1c7c23902d75f2be57c4a7f2394b3e",
+                "sha256:bc221ffbc2150458b1cd71cdd9ddd5bb37962b036e41b8be258280b5b01da1dd",
+                "sha256:ce55289d5659b5b12b3db4dc9b7075b70cef5631e56530f14b2945e8836f2d20",
+                "sha256:d9881356dc48e58910c53af82b57183879129fa30492be69058c5b0d9fddf391",
+                "sha256:dbcf59334ff645e6a67cd5f78b4b2cdb76384cdf587fa0d2dc85f634a72e1a3e",
+                "sha256:ebf577c7a6744e9e1bd3fee45fc74a02710b214f94e2bde344912d85e0c9af7c",
+                "sha256:f081c03f413f59390a80b3e351cc2b2ea0205839714dbc364519bcf51f4b56ca",
+                "sha256:fdbb46fad4fb47443b5b8ac76904b2e7a66556844f33370861b4788db0f8816a",
+                "sha256:fdcd28360dbb6203fb5219b1a5658df226ac9bebc2542a9e8f457de959d713d0"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.7.2"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -1253,10 +1295,11 @@
         },
         "mistune": {
             "hashes": [
-                "sha256:0246113cb2492db875c6be56974a7c893333bf26cd92891c85f63151cee09d34",
-                "sha256:bad7f5d431886fcbaf5f758118ecff70d31f75231b34024a1341120340a65ce8"
+                "sha256:b9b3e438efbb57c62b5beb5e134dab664800bdf1284a7ee09e8b12b13eb1aac6",
+                "sha256:e912116c13aa0944f9dc530db38eb88f6a77087ab128f49f84a48f4c05ea163c"
             ],
-            "version": "==2.0.5"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.0.1"
         },
         "msgpack": {
             "hashes": [
@@ -1336,11 +1379,11 @@
         },
         "munch": {
             "hashes": [
-                "sha256:0e4108418cfea898dcad01ff9569c30ff58f01d6f699331d04364f51623627c0",
-                "sha256:5284603030c00906d9d64d8108728c004fbeb91fc1c1e4caca342bc48f2a6dfd"
+                "sha256:542cb151461263216a4e37c3fd9afc425feeaf38aaa3025cd2a981fadb422235",
+                "sha256:71033c45db9fb677a0b7eb517a4ce70ae09258490e419b0e7f00d1e386ecb1b4"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==3.0.0"
+            "version": "==4.0.0"
         },
         "natsort": {
             "hashes": [
@@ -1363,24 +1406,24 @@
                 "sha256:25e861299e5303a0477568557c4045eccc7a34c17fc08e7959558707b9ebe548",
                 "sha256:f9b179cd4b2d7bca965f900a2ebf0db4a12ebff2f36a711cb66861e4ae158e55"
             ],
-            "markers": "python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8'",
             "version": "==0.8.0"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:51b6c77b507b177b73f6729dba15676e42c4e92bcb00edc8cc982ee72e7d89d7",
-                "sha256:af5064a9db524f9f12f4e8be7f0799524bd5b14c1adea37e34e83c95127cc818"
+                "sha256:3022adadff3f86578a47fab7c2228bb3ca9c56a24345642a22f917f6168b48fc",
+                "sha256:4a5996bf5f3cd16aa0431897ba1aa4c64842c2079f434b3dc6b8c4b252ef3355"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==7.4.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==7.7.3"
         },
         "nbformat": {
             "hashes": [
-                "sha256:46dac64c781f1c34dfd8acba16547024110348f9fc7eab0f31981c2a3dc48d1f",
-                "sha256:d910082bd3e0bffcf07eabf3683ed7dda0727a326c446eeb2922abe102e65162"
+                "sha256:3a7f52d040639cbd8a3890218c8b0ffb93211588c57446c90095e32ba5881b5d",
+                "sha256:b7968ebf4811178a4108ee837eae1442e3f054132100f0359219e9ed1ce3ca45"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==5.8.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.9.1"
         },
         "nest-asyncio": {
             "hashes": [
@@ -1543,6 +1586,14 @@
             "index": "pypi",
             "version": "==1.23.5"
         },
+        "overrides": {
+            "hashes": [
+                "sha256:6187d8710a935d09b0bcef8238301d6ee2569d2ac1ae0ec39a8c7924e27f58ca",
+                "sha256:8b97c6c1e1681b78cbc9424b138d880f0803c2254c5ebaabdde57bb6c62093f2"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==7.3.1"
+        },
         "packaging": {
             "hashes": [
                 "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
@@ -1615,75 +1666,65 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1",
-                "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba",
-                "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a",
-                "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799",
-                "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51",
-                "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb",
-                "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5",
-                "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270",
-                "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6",
-                "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47",
-                "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf",
-                "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e",
-                "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b",
-                "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66",
-                "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865",
-                "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec",
-                "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c",
-                "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1",
-                "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38",
-                "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906",
-                "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705",
-                "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef",
-                "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc",
-                "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f",
-                "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf",
-                "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392",
-                "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d",
-                "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe",
-                "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32",
-                "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5",
-                "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7",
-                "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44",
-                "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d",
-                "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3",
-                "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625",
-                "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e",
-                "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829",
-                "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089",
-                "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3",
-                "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78",
-                "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96",
-                "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964",
-                "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597",
-                "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99",
-                "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a",
-                "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140",
-                "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7",
-                "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16",
-                "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903",
-                "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1",
-                "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296",
-                "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572",
-                "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115",
-                "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a",
-                "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd",
-                "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4",
-                "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1",
-                "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb",
-                "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa",
-                "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a",
-                "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569",
-                "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c",
-                "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf",
-                "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082",
-                "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062",
-                "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==9.5.0"
+                "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5",
+                "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530",
+                "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d",
+                "sha256:1ce91b6ec08d866b14413d3f0bbdea7e24dfdc8e59f562bb77bc3fe60b6144ca",
+                "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891",
+                "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992",
+                "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7",
+                "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3",
+                "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba",
+                "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3",
+                "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3",
+                "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f",
+                "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538",
+                "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3",
+                "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d",
+                "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c",
+                "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017",
+                "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3",
+                "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223",
+                "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e",
+                "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3",
+                "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6",
+                "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640",
+                "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334",
+                "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1",
+                "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba",
+                "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa",
+                "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0",
+                "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396",
+                "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d",
+                "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485",
+                "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf",
+                "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43",
+                "sha256:bc2ec7c7b5d66b8ec9ce9f720dbb5fa4bace0f545acd34870eff4a369b44bf37",
+                "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2",
+                "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd",
+                "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86",
+                "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967",
+                "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629",
+                "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568",
+                "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed",
+                "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f",
+                "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551",
+                "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3",
+                "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614",
+                "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff",
+                "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d",
+                "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883",
+                "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684",
+                "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0",
+                "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de",
+                "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b",
+                "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3",
+                "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199",
+                "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51",
+                "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==10.0.0"
         },
         "pkgutil-resolve-name": {
             "hashes": [
@@ -1695,35 +1736,35 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f",
-                "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5"
+                "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421",
+                "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.5.1"
+            "version": "==3.9.1"
         },
         "pluggy": {
             "hashes": [
-                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
-                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+                "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849",
+                "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.0.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.2.0"
         },
         "prometheus-client": {
             "hashes": [
-                "sha256:0836af6eb2c8f4fed712b2f279f6c0a8bbab29f9f4aa15276b91c7cb0d1616ab",
-                "sha256:a03e35b359f14dd1630898543e2120addfdeacd1a6069c1367ae90fd93ad3f48"
+                "sha256:21e674f39831ae3f8acde238afd9a27a37d0d2fb5a28ea094f0ce25d2cbf2091",
+                "sha256:e537f37160f6807b8202a6fc4764cdd19bac5480ddd3e0d463c3002b34462101"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.16.0"
+            "version": "==0.17.1"
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b",
-                "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"
+                "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
+                "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.38"
+            "version": "==3.0.39"
         },
         "psutil": {
             "hashes": [
@@ -1742,7 +1783,7 @@
                 "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
                 "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
             ],
-            "markers": "sys_platform != 'cygwin'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==5.9.5"
         },
         "psycopg2-binary": {
@@ -2144,39 +2185,6 @@
             "index": "pypi",
             "version": "==3.5.0"
         },
-        "pyrsistent": {
-            "hashes": [
-                "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8",
-                "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440",
-                "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a",
-                "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c",
-                "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3",
-                "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393",
-                "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9",
-                "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da",
-                "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf",
-                "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64",
-                "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a",
-                "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3",
-                "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98",
-                "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2",
-                "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8",
-                "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf",
-                "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc",
-                "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7",
-                "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28",
-                "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2",
-                "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b",
-                "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a",
-                "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64",
-                "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19",
-                "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1",
-                "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9",
-                "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.19.3"
-        },
         "pytest": {
             "hashes": [
                 "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d",
@@ -2218,243 +2226,243 @@
         },
         "pyyaml": {
             "hashes": [
-                "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf",
-                "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293",
-                "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b",
-                "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57",
-                "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b",
-                "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4",
-                "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07",
-                "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba",
-                "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9",
-                "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287",
-                "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513",
-                "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0",
-                "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782",
-                "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0",
-                "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92",
-                "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f",
-                "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2",
-                "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc",
-                "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1",
-                "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c",
-                "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86",
-                "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4",
-                "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c",
-                "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34",
-                "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b",
-                "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d",
-                "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c",
-                "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb",
-                "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7",
-                "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737",
-                "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3",
-                "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d",
-                "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358",
-                "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53",
-                "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78",
-                "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803",
-                "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a",
-                "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f",
-                "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174",
-                "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"
+                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
+                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
+                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
+                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
+                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
+                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
+                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
+                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
+                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
+                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
+                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
+                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
+                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
+                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
+                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
+                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
+                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
+                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
+                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
+                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
+                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
+                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
+                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
+                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
+                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
+                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
+                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
+                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
+                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
+                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
+                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
+                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
+                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
+                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
+                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
+                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
+                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
+                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
+                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
+                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==6.0"
+            "version": "==6.0.1"
         },
         "pyzmq": {
             "hashes": [
-                "sha256:032f5c8483c85bf9c9ca0593a11c7c749d734ce68d435e38c3f72e759b98b3c9",
-                "sha256:08bfcc21b5997a9be4fefa405341320d8e7f19b4d684fb9c0580255c5bd6d695",
-                "sha256:1a843d26a8da1b752c74bc019c7b20e6791ee813cd6877449e6a1415589d22ff",
-                "sha256:1f124cb73f1aa6654d31b183810febc8505fd0c597afa127c4f40076be4574e0",
-                "sha256:1f82906a2d8e4ee310f30487b165e7cc8ed09c009e4502da67178b03083c4ce0",
-                "sha256:21ec0bf4831988af43c8d66ba3ccd81af2c5e793e1bf6790eb2d50e27b3c570a",
-                "sha256:24683285cc6b7bf18ad37d75b9db0e0fefe58404e7001f1d82bf9e721806daa7",
-                "sha256:24abbfdbb75ac5039205e72d6c75f10fc39d925f2df8ff21ebc74179488ebfca",
-                "sha256:25e6873a70ad5aa31e4a7c41e5e8c709296edef4a92313e1cd5fc87bbd1874e2",
-                "sha256:269968f2a76c0513490aeb3ba0dc3c77b7c7a11daa894f9d1da88d4a0db09835",
-                "sha256:26b0358e8933990502f4513c991c9935b6c06af01787a36d133b7c39b1df37fa",
-                "sha256:28fdb9224a258134784a9cf009b59265a9dde79582fb750d4e88a6bcbc6fa3dc",
-                "sha256:2b9c9cc965cdf28381e36da525dcb89fc1571d9c54800fdcd73e3f73a2fc29bd",
-                "sha256:2da6813b7995b6b1d1307329c73d3e3be2fd2d78e19acfc4eff2e27262732388",
-                "sha256:3059a6a534c910e1d5d068df42f60d434f79e6cc6285aa469b384fa921f78cf8",
-                "sha256:312b3f0f066b4f1d17383aae509bacf833ccaf591184a1f3c7a1661c085063ae",
-                "sha256:34a6fddd159ff38aa9497b2e342a559f142ab365576284bc8f77cb3ead1f79c5",
-                "sha256:374b55516393bfd4d7a7daa6c3b36d6dd6a31ff9d2adad0838cd6a203125e714",
-                "sha256:38d9f78d69bcdeec0c11e0feb3bc70f36f9b8c44fc06e5d06d91dc0a21b453c7",
-                "sha256:4a31992a8f8d51663ebf79df0df6a04ffb905063083d682d4380ab8d2c67257c",
-                "sha256:4a4b4261eb8f9ed71f63b9eb0198dd7c934aa3b3972dac586d0ef502ba9ab08b",
-                "sha256:510d8e55b3a7cd13f8d3e9121edf0a8730b87d925d25298bace29a7e7bc82810",
-                "sha256:531e36d9fcd66f18de27434a25b51d137eb546931033f392e85674c7a7cea853",
-                "sha256:54a96cf77684a3a537b76acfa7237b1e79a8f8d14e7f00e0171a94b346c5293e",
-                "sha256:56a94ab1d12af982b55ca96c6853db6ac85505e820d9458ac76364c1998972f4",
-                "sha256:5c5fbb229e40a89a2fe73d0c1181916f31e30f253cb2d6d91bea7927c2e18413",
-                "sha256:5d496815074e3e3d183fe2c7fcea2109ad67b74084c254481f87b64e04e9a471",
-                "sha256:5eaeae038c68748082137d6896d5c4db7927e9349237ded08ee1bbd94f7361c9",
-                "sha256:62ec8d979f56c0053a92b2b6a10ff54b9ec8a4f187db2b6ec31ee3dd6d3ca6e2",
-                "sha256:64812f29d6eee565e129ca14b0c785744bfff679a4727137484101b34602d1a7",
-                "sha256:6526d097b75192f228c09d48420854d53dfbc7abbb41b0e26f363ccb26fbc177",
-                "sha256:659e62e1cbb063151c52f5b01a38e1df6b54feccfa3e2509d44c35ca6d7962ee",
-                "sha256:65c19a63b4a83ae45d62178b70223adeee5f12f3032726b897431b6553aa25af",
-                "sha256:67da1c213fbd208906ab3470cfff1ee0048838365135a9bddc7b40b11e6d6c89",
-                "sha256:6a821a506822fac55d2df2085a52530f68ab15ceed12d63539adc32bd4410f6e",
-                "sha256:6a979e59d2184a0c8f2ede4b0810cbdd86b64d99d9cc8a023929e40dce7c86cc",
-                "sha256:6b8c1bbb70e868dc88801aa532cae6bd4e3b5233784692b786f17ad2962e5149",
-                "sha256:6fadc60970714d86eff27821f8fb01f8328dd36bebd496b0564a500fe4a9e354",
-                "sha256:715cff7644a80a7795953c11b067a75f16eb9fc695a5a53316891ebee7f3c9d5",
-                "sha256:77942243ff4d14d90c11b2afd8ee6c039b45a0be4e53fb6fa7f5e4fd0b59da39",
-                "sha256:7b504ae43d37e282301da586529e2ded8b36d4ee2cd5e6db4386724ddeaa6bbc",
-                "sha256:827bf60e749e78acb408a6c5af6688efbc9993e44ecc792b036ec2f4b4acf485",
-                "sha256:8280ada89010735a12b968ec3ea9a468ac2e04fddcc1cede59cb7f5178783b9c",
-                "sha256:83d822e8687621bed87404afc1c03d83fa2ce39733d54c2fd52d8829edb8a7ff",
-                "sha256:8560756318ec7c4c49d2c341012167e704b5a46d9034905853c3d1ade4f55bee",
-                "sha256:85762712b74c7bd18e340c3639d1bf2f23735a998d63f46bb6584d904b5e401d",
-                "sha256:88649b19ede1cab03b96b66c364cbbf17c953615cdbc844f7f6e5f14c5e5261c",
-                "sha256:9a2e5fe42dfe6b73ca120b97ac9f34bfa8414feb15e00e37415dbd51cf227ef6",
-                "sha256:9af0bb0277e92f41af35e991c242c9c71920169d6aa53ade7e444f338f4c8128",
-                "sha256:9bdc40efb679b9dcc39c06d25629e55581e4c4f7870a5e88db4f1c51ce25e20d",
-                "sha256:9e1d2f2d86fc75ed7f8845a992c5f6f1ab5db99747fb0d78b5e4046d041164d2",
-                "sha256:a2e92ff20ad5d13266bc999a29ed29a3b5b101c21fdf4b2cf420c09db9fb690e",
-                "sha256:a35960c8b2f63e4ef67fd6731851030df68e4b617a6715dd11b4b10312d19fef",
-                "sha256:a6f6ae12478fdc26a6d5fdb21f806b08fa5403cd02fd312e4cb5f72df078f96f",
-                "sha256:a9b5eeb5278a8a636bb0abdd9ff5076bcbb836cd2302565df53ff1fa7d106d54",
-                "sha256:ab046e9cb902d1f62c9cc0eca055b1d11108bdc271caf7c2171487298f229b56",
-                "sha256:ab2c056ac503f25a63f6c8c6771373e2a711b98b304614151dfb552d3d6c81f6",
-                "sha256:abbce982a17c88d2312ec2cf7673985d444f1beaac6e8189424e0a0e0448dbb3",
-                "sha256:ac178e666c097c8d3deb5097b58cd1316092fc43e8ef5b5fdb259b51da7e7315",
-                "sha256:ad761cfbe477236802a7ab2c080d268c95e784fe30cafa7e055aacd1ca877eb0",
-                "sha256:affec1470351178e892121b3414c8ef7803269f207bf9bef85f9a6dd11cde264",
-                "sha256:b164cc3c8acb3d102e311f2eb6f3c305865ecb377e56adc015cb51f721f1dda6",
-                "sha256:b48616a09d7df9dbae2f45a0256eee7b794b903ddc6d8657a9948669b345f220",
-                "sha256:b491998ef886662c1f3d49ea2198055a9a536ddf7430b051b21054f2a5831800",
-                "sha256:b733076ff46e7db5504c5e7284f04a9852c63214c74688bdb6135808531755a3",
-                "sha256:c8fedc3ccd62c6b77dfe6f43802057a803a411ee96f14e946f4a76ec4ed0e117",
-                "sha256:cb1f69a0a2a2b1aae8412979dd6293cc6bcddd4439bf07e4758d864ddb112354",
-                "sha256:cca8524b61c0eaaa3505382dc9b9a3bc8165f1d6c010fdd1452c224225a26689",
-                "sha256:cfb9f7eae02d3ac42fbedad30006b7407c984a0eb4189a1322241a20944d61e5",
-                "sha256:d4427b4a136e3b7f85516c76dd2e0756c22eec4026afb76ca1397152b0ca8145",
-                "sha256:d488c5c8630f7e782e800869f82744c3aca4aca62c63232e5d8c490d3d66956a",
-                "sha256:dd771a440effa1c36d3523bc6ba4e54ff5d2e54b4adcc1e060d8f3ca3721d228",
-                "sha256:ed15e3a2c3c2398e6ae5ce86d6a31b452dfd6ad4cd5d312596b30929c4b6e182",
-                "sha256:edbbf06cc2719889470a8d2bf5072bb00f423e12de0eb9ffec946c2c9748e149",
-                "sha256:eef2a0b880ab40aca5a878933376cb6c1ec483fba72f7f34e015c0f675c90b20",
-                "sha256:f7c8b8368e84381ae7c57f1f5283b029c888504aaf4949c32e6e6fb256ec9bf0",
-                "sha256:ffc71111433bd6ec8607a37b9211f4ef42e3d3b271c6d76c813669834764b248"
+                "sha256:01f06f33e12497dca86353c354461f75275a5ad9eaea181ac0dc1662da8074fa",
+                "sha256:0b6b42f7055bbc562f63f3df3b63e3dd1ebe9727ff0f124c3aa7bcea7b3a00f9",
+                "sha256:0c4fc2741e0513b5d5a12fe200d6785bbcc621f6f2278893a9ca7bed7f2efb7d",
+                "sha256:108c96ebbd573d929740d66e4c3d1bdf31d5cde003b8dc7811a3c8c5b0fc173b",
+                "sha256:13bbe36da3f8aaf2b7ec12696253c0bf6ffe05f4507985a8844a1081db6ec22d",
+                "sha256:154bddda2a351161474b36dba03bf1463377ec226a13458725183e508840df89",
+                "sha256:19d0383b1f18411d137d891cab567de9afa609b214de68b86e20173dc624c101",
+                "sha256:1a6169e69034eaa06823da6a93a7739ff38716142b3596c180363dee729d713d",
+                "sha256:1fc56a0221bdf67cfa94ef2d6ce5513a3d209c3dfd21fed4d4e87eca1822e3a3",
+                "sha256:2a21fec5c3cea45421a19ccbe6250c82f97af4175bc09de4d6dd78fb0cb4c200",
+                "sha256:2b15247c49d8cbea695b321ae5478d47cffd496a2ec5ef47131a9e79ddd7e46c",
+                "sha256:2f5efcc29056dfe95e9c9db0dfbb12b62db9c4ad302f812931b6d21dd04a9119",
+                "sha256:2f666ae327a6899ff560d741681fdcdf4506f990595201ed39b44278c471ad98",
+                "sha256:332616f95eb400492103ab9d542b69d5f0ff628b23129a4bc0a2fd48da6e4e0b",
+                "sha256:33d5c8391a34d56224bccf74f458d82fc6e24b3213fc68165c98b708c7a69325",
+                "sha256:3575699d7fd7c9b2108bc1c6128641a9a825a58577775ada26c02eb29e09c517",
+                "sha256:3830be8826639d801de9053cf86350ed6742c4321ba4236e4b5568528d7bfed7",
+                "sha256:3a522510e3434e12aff80187144c6df556bb06fe6b9d01b2ecfbd2b5bfa5c60c",
+                "sha256:3bed53f7218490c68f0e82a29c92335daa9606216e51c64f37b48eb78f1281f4",
+                "sha256:414b8beec76521358b49170db7b9967d6974bdfc3297f47f7d23edec37329b00",
+                "sha256:442d3efc77ca4d35bee3547a8e08e8d4bb88dadb54a8377014938ba98d2e074a",
+                "sha256:47b915ba666c51391836d7ed9a745926b22c434efa76c119f77bcffa64d2c50c",
+                "sha256:48e5e59e77c1a83162ab3c163fc01cd2eebc5b34560341a67421b09be0891287",
+                "sha256:4a82faae00d1eed4809c2f18b37f15ce39a10a1c58fe48b60ad02875d6e13d80",
+                "sha256:4a983c8694667fd76d793ada77fd36c8317e76aa66eec75be2653cef2ea72883",
+                "sha256:4c2fc7aad520a97d64ffc98190fce6b64152bde57a10c704b337082679e74f67",
+                "sha256:4cb27ef9d3bdc0c195b2dc54fcb8720e18b741624686a81942e14c8b67cc61a6",
+                "sha256:4d67609b37204acad3d566bb7391e0ecc25ef8bae22ff72ebe2ad7ffb7847158",
+                "sha256:5482f08d2c3c42b920e8771ae8932fbaa0a67dff925fc476996ddd8155a170f3",
+                "sha256:5489738a692bc7ee9a0a7765979c8a572520d616d12d949eaffc6e061b82b4d1",
+                "sha256:5693dcc4f163481cf79e98cf2d7995c60e43809e325b77a7748d8024b1b7bcba",
+                "sha256:58416db767787aedbfd57116714aad6c9ce57215ffa1c3758a52403f7c68cff5",
+                "sha256:5873d6a60b778848ce23b6c0ac26c39e48969823882f607516b91fb323ce80e5",
+                "sha256:5af31493663cf76dd36b00dafbc839e83bbca8a0662931e11816d75f36155897",
+                "sha256:5e7fbcafa3ea16d1de1f213c226005fea21ee16ed56134b75b2dede5a2129e62",
+                "sha256:65346f507a815a731092421d0d7d60ed551a80d9b75e8b684307d435a5597425",
+                "sha256:6581e886aec3135964a302a0f5eb68f964869b9efd1dbafdebceaaf2934f8a68",
+                "sha256:69511d604368f3dc58d4be1b0bad99b61ee92b44afe1cd9b7bd8c5e34ea8248a",
+                "sha256:7018289b402ebf2b2c06992813523de61d4ce17bd514c4339d8f27a6f6809492",
+                "sha256:71c7b5896e40720d30cd77a81e62b433b981005bbff0cb2f739e0f8d059b5d99",
+                "sha256:75217e83faea9edbc29516fc90c817bc40c6b21a5771ecb53e868e45594826b0",
+                "sha256:7e23a8c3b6c06de40bdb9e06288180d630b562db8ac199e8cc535af81f90e64b",
+                "sha256:80c41023465d36280e801564a69cbfce8ae85ff79b080e1913f6e90481fb8957",
+                "sha256:831ba20b660b39e39e5ac8603e8193f8fce1ee03a42c84ade89c36a251449d80",
+                "sha256:851fb2fe14036cfc1960d806628b80276af5424db09fe5c91c726890c8e6d943",
+                "sha256:8751f9c1442624da391bbd92bd4b072def6d7702a9390e4479f45c182392ff78",
+                "sha256:8b45d722046fea5a5694cba5d86f21f78f0052b40a4bbbbf60128ac55bfcc7b6",
+                "sha256:8b697774ea8273e3c0460cf0bba16cd85ca6c46dfe8b303211816d68c492e132",
+                "sha256:90146ab578931e0e2826ee39d0c948d0ea72734378f1898939d18bc9c823fcf9",
+                "sha256:9301cf1d7fc1ddf668d0abbe3e227fc9ab15bc036a31c247276012abb921b5ff",
+                "sha256:95bd3a998d8c68b76679f6b18f520904af5204f089beebb7b0301d97704634dd",
+                "sha256:968b0c737797c1809ec602e082cb63e9824ff2329275336bb88bd71591e94a90",
+                "sha256:97d984b1b2f574bc1bb58296d3c0b64b10e95e7026f8716ed6c0b86d4679843f",
+                "sha256:9e68ae9864d260b18f311b68d29134d8776d82e7f5d75ce898b40a88df9db30f",
+                "sha256:adecf6d02b1beab8d7c04bc36f22bb0e4c65a35eb0b4750b91693631d4081c70",
+                "sha256:af56229ea6527a849ac9fb154a059d7e32e77a8cba27e3e62a1e38d8808cb1a5",
+                "sha256:b324fa769577fc2c8f5efcd429cef5acbc17d63fe15ed16d6dcbac2c5eb00849",
+                "sha256:b5a07c4f29bf7cb0164664ef87e4aa25435dcc1f818d29842118b0ac1eb8e2b5",
+                "sha256:bad172aba822444b32eae54c2d5ab18cd7dee9814fd5c7ed026603b8cae2d05f",
+                "sha256:bdca18b94c404af6ae5533cd1bc310c4931f7ac97c148bbfd2cd4bdd62b96253",
+                "sha256:be24a5867b8e3b9dd5c241de359a9a5217698ff616ac2daa47713ba2ebe30ad1",
+                "sha256:be86a26415a8b6af02cd8d782e3a9ae3872140a057f1cadf0133de685185c02b",
+                "sha256:c66b7ff2527e18554030319b1376d81560ca0742c6e0b17ff1ee96624a5f1afd",
+                "sha256:c8398a1b1951aaa330269c35335ae69744be166e67e0ebd9869bdc09426f3871",
+                "sha256:cad9545f5801a125f162d09ec9b724b7ad9b6440151b89645241d0120e119dcc",
+                "sha256:cb6d161ae94fb35bb518b74bb06b7293299c15ba3bc099dccd6a5b7ae589aee3",
+                "sha256:d40682ac60b2a613d36d8d3a0cd14fbdf8e7e0618fbb40aa9fa7b796c9081584",
+                "sha256:d6128d431b8dfa888bf51c22a04d48bcb3d64431caf02b3cb943269f17fd2994",
+                "sha256:dbc466744a2db4b7ca05589f21ae1a35066afada2f803f92369f5877c100ef62",
+                "sha256:ddbef8b53cd16467fdbfa92a712eae46dd066aa19780681a2ce266e88fbc7165",
+                "sha256:e21cc00e4debe8f54c3ed7b9fcca540f46eee12762a9fa56feb8512fd9057161",
+                "sha256:eb52e826d16c09ef87132c6e360e1879c984f19a4f62d8a935345deac43f3c12",
+                "sha256:f0d9e7ba6a815a12c8575ba7887da4b72483e4cfc57179af10c9b937f3f9308f",
+                "sha256:f1e931d9a92f628858a50f5bdffdfcf839aebe388b82f9d2ccd5d22a38a789dc",
+                "sha256:f45808eda8b1d71308c5416ef3abe958f033fdbb356984fabbfc7887bed76b3f",
+                "sha256:f6d39e42a0aa888122d1beb8ec0d4ddfb6c6b45aecb5ba4013c27e2f28657765",
+                "sha256:fc34fdd458ff77a2a00e3c86f899911f6f269d393ca5675842a6e92eea565bae"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==25.0.2"
+            "version": "==25.1.0"
         },
         "pyzstd": {
             "hashes": [
-                "sha256:04eb56c932bf7ad9a22bdfde995c2bb22909ae00218dcb3e174f76375389a339",
-                "sha256:07150e8f018df72c32cbc7958eacae8d18a14064a3b18dda00ef443b12804ad0",
-                "sha256:07c3863c968eaeab8c4ebd32c3ddf9ca3e5762d77e20da21a3d9a7578bd0b6c4",
-                "sha256:0802d7a01ab7bd73dd0dd5c3d9143a7fe77e6c98ab33fdf4b14b20fe5ca090a7",
-                "sha256:0d4fd161f173c330291f54de5ab8e849e8686ad24c76f45c8045ce7d48aa0c33",
-                "sha256:110e35df5837e4ca5506760dc6e354104eefb7c3e4cbc9f8a53bb7813778601f",
-                "sha256:12f1d451ba9d1bd4f8e8585d59f09fee6fc6ed978b27dd55c2915a37bfa63dfc",
-                "sha256:13c653dac81e95e966759a9b8b47d6c0fdf0a6c51a743397dee9091e62c7bfa8",
-                "sha256:1ab0309a5a36ebe6bbf6b33f200aa9faa71493b842924e85840d78ecd72ce3cd",
-                "sha256:1c489982b59bce1d7a21dd71204ddae96dd34bcbdb50050cf3e3a1dc18f144d0",
-                "sha256:1d4dc50cc7d4a8763454ba3e12fbf558bcf7b0764eb7591fb5dfe9ac35b8162d",
-                "sha256:1e5472aa34e798d3a8ae306521a711e69ed1e3af72970c2f76312c795e547770",
-                "sha256:2129572d541e86561d4dc0f6be10788de6c19ff25a8ed83231f1825b05154cf2",
-                "sha256:2192f5a3b115274a9c68f75ff2c21cfb4006f0e5eff637f856e622871f0e7e26",
-                "sha256:298b24dfe7e94aa7857ab78059948158565a659f900e21645fffd55fc8736a84",
-                "sha256:2e17e38ea19c21a65026405e511c3cf9b26fe503c492e6a9db469763818e33c5",
-                "sha256:30ca96e20dcda02665c241a70b577202d3b9443416aaffefc490f6ff9d78f83f",
-                "sha256:30ea45ecd108aea9aec504192a7b18dce34e8dcb48503659a7a598c623d08437",
-                "sha256:34661ecc88dfed442b7be22dbf481f3bc80111bb44b225f4520c73ed380b2424",
-                "sha256:371a0523d40f560d1352ca91e532acaa2b331500ded3cf3ad699eee88d76af60",
-                "sha256:38ca4cc998b93bcc062a1cb01f41761b7ca54cad135cd8dd50074623449d5424",
-                "sha256:39a3ca5e44d9ba036f2515a3aec1fdb27849bd55affd205c578447fc2e2d2503",
-                "sha256:3a29a44f06ddd3f57a8656326044c57ec05e5162e97ffeed70b376b309c1bfb0",
-                "sha256:3bdf506a8b3b17b3cfa9b9b0b69d5fb542058fa3bb7121d5e5c4ec553ff39801",
-                "sha256:3d109b2321f9d2df7e17fe49791379f831cc8cfd7ad29069e1687faf99095188",
-                "sha256:3e03065608eff491466bcfb7fa1c48d3f81bfac1926db68f13f3362a1b8bc2db",
-                "sha256:3ec72c95dc86430d156643a41ebc467677f45907a6dd44c013377c3bbc4e68d0",
-                "sha256:414ec9fff19a5da1ed4c6322129eed2a8b1dd77ce6d55cf2b99013e374368489",
-                "sha256:443844f59416327abc0074d7d91a1cbeaaca270e79b71cb5acb824c8a6472b9d",
-                "sha256:4460309364de3a6139a6ac337a9bb454101e8280fed834970c3dd04a00d0677e",
-                "sha256:492301b5234acc0d04443bdb00bf7a1ad601caf82bc837f8a4c137c7eef8932e",
-                "sha256:52461114f78380b7432f9a31b0249933930aa4968507ecc9a27a345a905abac6",
-                "sha256:525bbe1db05d8cb657cb793aa994372f84f8fa22a009a67751e3c170f78b8812",
-                "sha256:5361782c9ff6c5a40d39637258dd64e109f7c535976e9c2db8ae17a7c6c2b3cb",
-                "sha256:53958a8c20e4344a86bad6e9cd0b886710bfac2bc581102b852bb24c2c09084c",
-                "sha256:54a7fa73ae22ee477ae3a5e3c6cafba4f1463363929f9e5e9b471f2d2278c649",
-                "sha256:55e503f28f5a9d225ce9d0639e3f5b1801bacace5aea926ec2998e73c5150fe7",
-                "sha256:57317f65d68dc217d6f7e354a63f349a6f08fd25c962e54c99df1520dce87b53",
-                "sha256:5b6d9e1be9fdaef16c53ce0b2bfd880ae21ca54bc244a1de1c28d0cefeee8760",
-                "sha256:5b6dbd6b01544a82836d32dffa4dc4fa63fed520c24409856372801bc65204a9",
-                "sha256:5dacf851e2a2cba837dcdf007af07dad7c860f70061e0c39094e500fc70a6e50",
-                "sha256:5e5058ec639d754267fbccb51f50f752c1e17e0a9a407c7eb2c3cdce8eb6f571",
-                "sha256:6c355d04dac84f9dcaee1cd2fb9df0236afadbc90e6ebf1285376c480523d8b1",
-                "sha256:701db75f49f1eca26f07f6b08751c43d5979e9ff39360a40ba165aebccbb2199",
-                "sha256:718f6cef753f8cc9275f5e790178151a59fae5f8587f53ee6f36e3a939466bd4",
-                "sha256:71e7c2fe631f06fd2ebc5870640bc0e19cdcd3a26ad2e2a663fe4b5aa2bc3ea2",
-                "sha256:72b0037a683a344c118c51a2cc353de03d1442792a8c1fd9adb64592f5794cd2",
-                "sha256:758b4b8261032991849fc1652feab2fdc432255a01898e15fe4957f91c49f5a9",
-                "sha256:7edd705d185b16399ff1143f84dcb34a8e8ec73229ecc187b32d916158541ada",
-                "sha256:7fa6ace75ad05303967908321d9b02f7011a1ea5103f9f72342dba820fa47b1a",
-                "sha256:827717b9a9f28cef58ba1018879f9ba777b9f287675107baf7d92409ac9d908d",
-                "sha256:86716c3f2c8bc82fd677f0fe1064833daf73148d168249c5fd5d6a8d1c3a6921",
-                "sha256:867d909f633a39add6f62fa19b61520abccb9af527194db0c490ebd2e910200d",
-                "sha256:8718cf7d96eae80e01334c5a3e171d698bfbcb0d357a27f5421be0de5a096a47",
-                "sha256:877ccd7ce58737fb672f17681d3ff5e78ab5731e7096662ae7b36305d95eb82c",
-                "sha256:8af59e31d1e51ccd998142f8ec7347e5ba7a801572539a8bffd9009956a76343",
-                "sha256:8e444c7198e531d236a5791eb6d1045249c93aa1dfd036b8275e9b0f36633c79",
-                "sha256:8ecd7c61ac4694b89c906d94f465324cb22772c5b54dacc947e45a927cfb5784",
-                "sha256:92d0370d2b0806aa652d2c1240d6c858a5b2a30baa892dce0f2638260590b038",
-                "sha256:933928a57c4445c4039e0dd4c75a9bfcab5bf0b1bb23b4869e340e28bc7aeb1e",
-                "sha256:97d408496c7f2d263533a1be2798fca13d1ae5eb625f556b3f3412cfd9894e01",
-                "sha256:98b8f16815eac222f6092e27c65f2811e7ad2300661f72224f02b28ec361baae",
-                "sha256:9a374e990d365b403723be368a054c086d168c65a8f2fe74bdf5e1344da37a82",
-                "sha256:9ba445c38aff01c41175dcc621b88357907f4deb1f147ff140e691e74c589029",
-                "sha256:9d8382fa1e672baf15d5fb3d13e0a1ca0f2b38730eab0611944bb5e0c7df9bc6",
-                "sha256:9e067a28b7935f1087b77b89f4cff7c84f0673bcf756aa6dbf21a36a32065ef2",
-                "sha256:a5983053242dab7daf36bfea5c246329249f28e71afc2ab8b4dbe8ac3c9287dd",
-                "sha256:ac8e5ab21c1d6baa1b2f6778ffa8b4970f5277bfa60c9fedbb03fc483ab21ee7",
-                "sha256:aed59809b3c5c021c7ae9f380da6f0835062cfd91a81a3125e7d4eb4e394fc15",
-                "sha256:b0b6a55ed8ae86d6a809bdf0552753f8bc11ce1a8b4ccb2a55f9a4026ecde01c",
-                "sha256:b1034413b31fa2f74213e19c109dcf2d4e5aac49336efc0640c7fbea9713e0b0",
-                "sha256:b3c9cedaf2f79fdc43940f29c9ff0fe5f135ed1ea9ad8f3be18bd4b77847889d",
-                "sha256:b4b4cd04b4f02693811015f9356945caffe7c5ded7c897df88d87550778d5c58",
-                "sha256:b855ce93692c9d598e21e0a28a32776c0ec63749f32125ba885eefbb5d669258",
-                "sha256:c3a9266cf6038d8af82cc4067f38dd5de84ebb6a152cfc25b9ef0b44c8ce84e9",
-                "sha256:c8408c499ef67dcec49be00b7007e0ceb7cbb1211cdaeb390294d8a0e9ceab75",
-                "sha256:d05909397f155b12a0a9b7c36cf1fb8a96df5b7caa9bc69c47ed36ab73e1aba9",
-                "sha256:d05d0ee19a1a37f995568ce05208b985683681d7f49073618559a44a4a568d44",
-                "sha256:d313de63feda18974c406cb05a963b27234aa0ad604561ed435ff726eaca1160",
-                "sha256:d3fab381de4e0bf8dfa8e356ee04c0ac18d5b603344cca98c0f6106760e979a8",
-                "sha256:d779c3e8faa92b241e58600f7b2126f3c055d75d3968ed5d856e2c5e2ec847ce",
-                "sha256:d865b3a330bafe0891d8414a87f1a56509baa19980150c3ba75f8b8e966a8655",
-                "sha256:dfdddfea962e039f8397a0e892eccf0fc25378ec3eb1ca81fdf2658f20272f0e",
-                "sha256:e25b652b67077b3d37451a7471b8975d478ec87d7f96f8f4673ed2d6b97355df",
-                "sha256:e34142d94a8f423687c940fb64031cfe099ca31dee463ecfcc784060d58929ff",
-                "sha256:e4d0abeaed6e6162d4506e7cc3e9df40f5589d9946922635bdf03821ff969380",
-                "sha256:e5504321223125495b9ae78cd49c7a4495ae8e547d09f704aad0c43a9fa8c5f2",
-                "sha256:e733ff332afa44737b3d1e433a023b8f588881e5d2ae7698293560437aaa1879",
-                "sha256:ec9bf97d6959112fe15c04daf1e94451a78a28d2573ef22f1d215db00204db20",
-                "sha256:ee64beaa98c3917fe21262670bc536663ab69cd07ab96c160e01ff30d654a880",
-                "sha256:eed9cbe732bdcd8cbf23f33b5d942a2111e91bf0573d1d28d49b626b63b9db52",
-                "sha256:ef7f2f8a95fdb755047015894e1859a49a5495bd6d1b197898b72e2c3abeca92",
-                "sha256:effda4ab1426a2b3384ea354f1d873865f87ae0e946508765dfe1b52c6dea6b8",
-                "sha256:f30cbf469eb11f39dbdca787bf6b3b287898e10202c4371863fa56844b4220f0",
-                "sha256:f4483359b6ebaf64ad5381c6f590e7f297d251dd6de019992e9481123c55c790",
-                "sha256:f4b87bf782306c27e9dc97ae148a72207b5dc018d5ad05768aeda6fbf262be1d",
-                "sha256:f5a2ed163bd9f00c494f70f387803277592bf5788f8a07a67eebda3ad1c11b7d",
-                "sha256:f5d03b13d32608a72301ee78afac0105b3a14bf63f2a9361c3742b37d85b38d7",
-                "sha256:f60f561ebd5ef2acfa971173ddf7f0987be3489d03aeb57a066f3daaeaa2b50e",
-                "sha256:f6b8443be8d4bd58d2f4d33a41fafddc012214e890d5c3634ba14e048287a982",
-                "sha256:f98470a2e8a7c11692deafe586b31bf87f5481d25ceec34fe49319b9c5de6ad1",
-                "sha256:f9d44e8ac7a788d60589e22cd27e072cf480100c4fa68fab6bcbc5927a162e18",
-                "sha256:fa29986a7ed07f321878445d10a896a0a3f1a556c39b3d2750091759b0f252e2",
-                "sha256:fc340a9159ead0ffaac5893569e124e975958e31ad381357e65765296882b81e",
-                "sha256:fd134893881bbbb1ede80e1c69a1ce8590018bee627be9b325ae2bdfe42b2e6b"
+                "sha256:00c188704141c709da96cc4a79f058d51f5318e839d6f904c7cc9badcf78e98e",
+                "sha256:013321ddaff083b24e43a8b06303446771978343b488ed73adf56c70a46e2783",
+                "sha256:0a4334e972109bdd17fb40dbdd9fcca6137648cab416fca505a2dcd186f50533",
+                "sha256:12668ceb8329aaa908b4d907d3a77bb748ff28b309c3b105c995a8715d535d2b",
+                "sha256:14121a4d95070f54bdc9a80dab1dd8fd9093907a1e687926447ca69b5b40a4d5",
+                "sha256:1b9cda5314982d64c856f9298be0d9bf69fbff0ca514d1651037616354b473ff",
+                "sha256:1cbf212253abd65e6451acdfb608adafe98ad8f05462fb9a054ddab816545caa",
+                "sha256:1dbe76b6d8fe75f6dbec24793fc07b1d1ae9464de9941138d5b9668f7670e6b0",
+                "sha256:209a92fbe892bd69cde58ffcb4861468e2c3c2d0626763e16e122bb55cb1fb1a",
+                "sha256:20f2dd56d46441cd9277077060c34c0b9ce3469412665ea5ccd506dd2708d994",
+                "sha256:23695dabdfd5081beab25754dc0105b42fbd2085a7c293901bcb45045969c5ec",
+                "sha256:250dad90140a6faea4cef555f339b6ceaad5cf03ed1127b8d06de214ff0db2e7",
+                "sha256:289e25871fe232d2482c0985a75a1faa7c92e10a6c3e3914d165f62d005d0aa6",
+                "sha256:2919afd114fd12309ed2f831ef6e95730ebf13c2a92d258ad055769d00ef4d7a",
+                "sha256:29e452caaf0de9cc17319225921d8c28cdc7a879948e990ff1e7735e7f976517",
+                "sha256:305c232462dbe80d0ee5ec91b1b0ec9153ec6ba6393d5348741af5d30b07ef52",
+                "sha256:31f60f01884350aec24e7a68f3ad089151b7a636490203c41a1a7c8e0cddd9b8",
+                "sha256:3351ad2feb51dcbb936defd47cab00d6f114214f224636503ed08298f30164c9",
+                "sha256:346f835e368e1051f8ea187ad9b49759cf6249c9ebf2f2a3861e435a568104b8",
+                "sha256:370b34a7c2f9c53cee494028daa5a7264690e1756a89c3855fd0be5ad298ec30",
+                "sha256:3bc0e7e2cccf78e562ab416daf68448b6552a5b6450a1ff3e15cabfc19254883",
+                "sha256:3f0fe2ef7ebc6e9b347585e414c4fefd32270ba8bdf9eb82496f3030cbdca465",
+                "sha256:3f72f310b10b730cddfb654006ae497e7706c81e6a7642d3da7fd2439df7d88d",
+                "sha256:40bdb468281a5cd525e2e990b97344f0974e0589bd1b395501c25471fcd7edda",
+                "sha256:4358dd80b315c82d760b44c6df7857c9c898d04e7b0c14abb0eb3692354e9379",
+                "sha256:441078bfd3b508597415338af667c3575980364f1286eedde58291558b9c2832",
+                "sha256:47c2a4c319300c381f194274203f47b12c433e1fd86b90ecdc7fb258c630f93b",
+                "sha256:49c57ae18f138a4b66480b2364fe6a0f2345ada919e93fc729c95c6b17ec73a4",
+                "sha256:4a0dcb32ac4d1d67a77ae6a2d60ea0921af7e682b3427202d8acb8e86642391c",
+                "sha256:4ed01beb31d5177456ec2c4b66591a0df83dbc72df29f05f40502bfefe47bbe4",
+                "sha256:50ccbaafee80b4f1c5c55bbe07f80871b9b8fe3499bf7357dde2c23fb1c2ac0e",
+                "sha256:51607d7d44f94a364ef0e3ccf9a92390def0faf6e7572eef082f15c657b5d03a",
+                "sha256:5345c7a697327e2fa7c37534bb2968ea84595d8ec7fc8c4a60216ec1be6e65bd",
+                "sha256:542808d88464d538f5d2c6b48b545a7fe15f0d20c7fa703b469d039a08c9fa10",
+                "sha256:5819d502dacd54114c30bc24efcb76e723b93f8f528be70851056a396a792c46",
+                "sha256:5aed5fc86d0bfc5f16e871cbb35ec93df61476d7fde4c1c6081015a075ecfbc1",
+                "sha256:5d9ec8634ab0cbfbcff535ac07555ebdae0282ad66762f0471fad11c16181e33",
+                "sha256:5fb00c706d0b59c53124f982bd84b7d46866a8ea2a7670aaaa1ab4dbe6001b50",
+                "sha256:5fd7cf79949174d1018b896638f88aea1ff2a969f87a6199ea23b25b506e26c5",
+                "sha256:606b2452d78f0f731566d392f8d83cd012c2ffadb2cb2e2903fdd360c1faac8a",
+                "sha256:6128cb653d011f3781554b70ce1f1f388cd516820fbaf8fd03ee245ecaa48349",
+                "sha256:639935b5b3d9ed3911493504581254b76cb578279302f7f340924ac5bfca4090",
+                "sha256:64564f4c175c5bb8e744de5816d69ee0b940e472160a5e665f30adc412b694f3",
+                "sha256:69f12ce4866a3725138e97f22f2c4cb21d3ae18cd422906cd57ed12a9ffd86c5",
+                "sha256:6a60ee6836599363a24367cf780ad45446b07eba49ec72d19bad761d5414aca7",
+                "sha256:6b9af8d62c087354abd071e01d9445ea51b31779c8a4a0d5c14ee12caee3d18f",
+                "sha256:6c456882baab2a48a5bfabe458a557af25d0768ff29acbe200461e84c0f697d5",
+                "sha256:6f281cc2f096530c339f122e0d9866545f5592dd9bffe0fade565c2771130a45",
+                "sha256:73877eebbdcb8259cf0099665f8c8274d4273b361371405a611fb6bd9f4d64f6",
+                "sha256:74455bd918e7bc9883e3178a1a8fe796308670f0ee4488c80a0d9514e13807a1",
+                "sha256:7452ae7e6d80e697d78d3f56d1b4d2a350286eea229afb35f55ab88b934b6acd",
+                "sha256:77294f0f797c97a46ffb3daff1fe097c9d5aa9f96867333978e6791286963e50",
+                "sha256:7ac886e04f253960ae82e38ded8352085c61d78de99412d178a94ecf475b5e5f",
+                "sha256:7c420878726d677da7484f6021dbe7e1f9345a791b155de632c6ce36678fb621",
+                "sha256:836f1d85a4b5d3689d455aeb1dc6c42acb96aaf8e5282825c00ccf2545ad5630",
+                "sha256:866ba6ce85f337fa1677516217b6f10fc25e19acb6e17a501d5822e66396bdd5",
+                "sha256:86e0e65e205793b337d62d9764700dfd02b5f83b01e26ad345736e7ac0554ebd",
+                "sha256:87a1a4ca93da414f3b6da8131e61aca6d48a4e837fb0b1cbde05ae9d13332317",
+                "sha256:8d3a1b6fa71a0ae7abc320d9db91b5a96a71eef1dbee0d62a6232b71c97af962",
+                "sha256:8f9eb97fb6fd4551ff9d5012b4fcee9abeea9c8af6b9e3ebc3c76cc2bd0a43a7",
+                "sha256:91453ce9476363d777b2ea2e9c6dccecd2073cf35697e048de2e8d47e1f36c7c",
+                "sha256:9596aeb8c71192f4fba1ca25cec420da195219398d2df811d5082559efd9561f",
+                "sha256:960ab83a977a44284c4ffab2820ccd6c9b332571a3d622fefa4b29b0a5de72b0",
+                "sha256:97e05f66c5847e6889594508298d78ddb84a0115e9234d598415dc5a06d3a4a7",
+                "sha256:9ac634753f6d26cba503cea7bb5b350aec7c5366f44fa68c79e9c90be9fd0ebc",
+                "sha256:9e1097d8b57f64878a3f176f4cd6b9a1bbe9fb2d236f1a85a4357722626d8f25",
+                "sha256:a1b81cc86b69ff530d45e735ed479e14704999f534ad28a39f04be4a8fe2b91f",
+                "sha256:a4f786f1b1ab39a0908db04ebe5b2c7cbc6f1ce07a27d3a12eb980bffd7fea7d",
+                "sha256:a594795ef89bd83297c860ff585f2d25580ce9805eb9cc44c831d311e7f1951a",
+                "sha256:a708b9e6ff1826504940beb6b5c2c9dfd4e3b55c16ab88a4572f5b9dbb64cc56",
+                "sha256:a90b901ccfd24b028faea19c927ff03f3cfefe82ba0b931fbb8da4ef0664911b",
+                "sha256:ae3d0575721a372c20130681bfaf873225fd9e1c290b7d56b7e0c14f413318f6",
+                "sha256:afef9eb882cf3b395eef9c85b737a4acd09528975e6a5d9faedf28874ca65f52",
+                "sha256:aff1b469187f6c789cdf17cd95c9b24e87396dc86953b1cf38b9a05cea873c80",
+                "sha256:b2ae8993f3863632d31ca8921c8a5dc9ecc5551c7b88895cefb5a26d17643391",
+                "sha256:b2dd39e12f7467a7422ce50711524759d4d22016714cbae6a7096b954bc2fa32",
+                "sha256:b4de7741d542a477387299bf9450e8be3e768c352d6b3438254eb02af1e59462",
+                "sha256:b5b517fbbc5d223fc36041673e7c2a0d3a82be6a5464a5f0599069330b76f97d",
+                "sha256:bdc09de97b1b3f6c3d87fec04d6fe29dd4fefe6b354ad2d822fc369b8aa0942b",
+                "sha256:c249741b10eb714578d765487b767e0e7fcc2ac84a299209a6073566e730dbea",
+                "sha256:c2b093a74b10232c70b5d29814fcee6544bb6f30e2d922d26db9ab4b4cd00c04",
+                "sha256:c31f6dd5bd60688d51487a3f5e2ae29ed1948926e44d7a2316b193b083f80d5d",
+                "sha256:c41e5457f4de5d38a270bc44619873589bbe6fe251225deec583ed20199df0f3",
+                "sha256:c46e77c2ad614a0399503dc675d72436cbf6332a20d49a0e5bad03058d6cbfad",
+                "sha256:cbfdde6c5768ffa5d2f14127bbc1d7c3c2d03c0ceaeb0736946197e06275ccc7",
+                "sha256:cd6a8d43a0c294918e3afb7e4b1d8c04d2e4c3ea9ddf05475fdaf366c7e5b3a6",
+                "sha256:cffaab46f9e04856dc3daa6097bfb3d3bea0b1771237e869c57b13f3dcc2c238",
+                "sha256:d0929302d187bfeca335b7f710f774f1b2ea3f610b2a80e8a1ac2da216cd9766",
+                "sha256:d44a7d4586f02b630658298c089ff755e74d0677b93c71e09d33dd35bdd4987a",
+                "sha256:d7ddbf234c9adc72189bb552d830e9a0c2c4401b5baf7b003eacd5c552ddcc00",
+                "sha256:dca286c6c1ca5febf13f5f2ae7e8aa7536e49bd07f4232796651a43ff741ceca",
+                "sha256:dcb2172ca8b62f82af9d1f8db80c21c64c5ba3991935caefde88bb378f0afb51",
+                "sha256:e4e00c1600022b47ef0e9e1f893cb0c2322209ec6c1581a3e3f63ed78330ddf0",
+                "sha256:e789e19095b818f7126180b4387c0f01700c3ad2378a4e7649b2ddf4bf47ffbc",
+                "sha256:e79babb67b415aa54abb213897ceaa011515a5f3e146a2a97f4e6486b9743af4",
+                "sha256:e8f75e839ee253af60b03d9957182fdd069dfaebb62b4e999bd74016f4e120bb",
+                "sha256:ef3399e0544b46d31c2a8ff14ae1fb3c3571ae1153bbbc5ddf0d242c67bde624",
+                "sha256:f169e166774587227255f6ffe71f5b3303ea73cde0e2c6d52e53b9e12c03d787",
+                "sha256:f1d8b58f00137ccbe8b828a5ede92be3f0115cef75e6bed88d4d0bd1e7a0b1fc",
+                "sha256:f2839c13e486e4a23b19b1d2dc4624565cec6c228bbf803c066be1106515966b",
+                "sha256:f66790e4b2dcfcabc0aa54dd89317ea5671cabf06aa93cbef7cbdd4d2fdb7ee3",
+                "sha256:f6d8a881b50bb2015e9bdba5edb0331e85d41ff44ab33cde551047480b98d748",
+                "sha256:f7cfc683d320402d61205a196ace77f15dcfd16b5771f8b9ffaf406868c98e78",
+                "sha256:f9c5fc29a5b9d61a8f0a3494172107e0e6cf23d0cb800d6285c6722ba7fc3535",
+                "sha256:fc92a718bccb8ce5c9eb63fca743c38f3fa4c4e47f58f0c4ada51b2474668184"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.15.7"
+            "version": "==0.15.9"
         },
         "qtconsole": {
             "hashes": [
@@ -2503,12 +2511,20 @@
             "index": "pypi",
             "version": "==0.18.0"
         },
+        "referencing": {
+            "hashes": [
+                "sha256:47237742e990457f7512c7d27486394a9aadaf876cbfaa4be65b27b4f4d47c6b",
+                "sha256:c257b08a399b6c2f5a3510a50d28ab5dbc7bbde049bcaf954d43c446f83ab548"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.30.0"
+        },
         "requests": {
             "hashes": [
                 "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
                 "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
             ],
-            "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==2.31.0"
         },
         "rfc3339-validator": {
@@ -2546,6 +2562,109 @@
             "index": "pypi",
             "version": "==0.3.4"
         },
+        "rpds-py": {
+            "hashes": [
+                "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f",
+                "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238",
+                "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f",
+                "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f",
+                "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c",
+                "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298",
+                "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260",
+                "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1",
+                "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d",
+                "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7",
+                "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f",
+                "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876",
+                "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe",
+                "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be",
+                "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32",
+                "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3",
+                "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18",
+                "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d",
+                "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620",
+                "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b",
+                "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae",
+                "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496",
+                "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1",
+                "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67",
+                "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f",
+                "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764",
+                "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196",
+                "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e",
+                "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846",
+                "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b",
+                "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d",
+                "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26",
+                "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e",
+                "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044",
+                "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c",
+                "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d",
+                "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad",
+                "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d",
+                "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab",
+                "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920",
+                "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e",
+                "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872",
+                "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3",
+                "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611",
+                "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4",
+                "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c",
+                "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193",
+                "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af",
+                "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10",
+                "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd",
+                "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f",
+                "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b",
+                "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945",
+                "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752",
+                "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c",
+                "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387",
+                "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8",
+                "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d",
+                "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931",
+                "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03",
+                "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502",
+                "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f",
+                "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55",
+                "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82",
+                "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798",
+                "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a",
+                "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b",
+                "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa",
+                "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f",
+                "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192",
+                "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020",
+                "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7",
+                "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1",
+                "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386",
+                "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90",
+                "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f",
+                "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe",
+                "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596",
+                "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f",
+                "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387",
+                "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16",
+                "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e",
+                "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b",
+                "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6",
+                "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1",
+                "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de",
+                "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0",
+                "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3",
+                "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468",
+                "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e",
+                "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd",
+                "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324",
+                "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c",
+                "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535",
+                "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55",
+                "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6",
+                "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.9.2"
+        },
         "rtree": {
             "hashes": [
                 "sha256:004e131b570dc360a49e7f3b60e7bc6517943a54df056587964d1cb903889e7e",
@@ -2650,11 +2769,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f",
-                "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"
+                "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f",
+                "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==67.8.0"
+            "version": "==68.0.0"
         },
         "shapely": {
             "hashes": [
@@ -2897,7 +3016,7 @@
                 "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f",
                 "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829"
             ],
-            "markers": "python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8'",
             "version": "==6.3.2"
         },
         "tqdm": {
@@ -2918,11 +3037,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223",
-                "sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768"
+                "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
+                "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.6.0"
+            "version": "==4.7.1"
         },
         "tzdata": {
             "hashes": [
@@ -2934,18 +3053,18 @@
         },
         "uri-template": {
             "hashes": [
-                "sha256:934e4d09d108b70eb8a24410af8615294d09d279ce0e7cbcdaef1bd21f932b06",
-                "sha256:f1699c77b73b925cf4937eae31ab282a86dc885c333f2e942513f08f691fc7db"
+                "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7",
+                "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"
             ],
-            "version": "==1.2.0"
+            "version": "==1.3.0"
         },
         "urllib3": {
             "hashes": [
-                "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305",
-                "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"
+                "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f",
+                "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.15"
+            "version": "==1.26.16"
         },
         "wcwidth": {
             "hashes": [
@@ -2970,11 +3089,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:c7d67c13b928645f259d9b847ab5b57fd2d127213ca41ebd880de1f553b7c23b",
-                "sha256:f8c64e28cd700e7ba1f04350d66422b6833b82a796b525a51e740b8cc8dab4b1"
+                "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd",
+                "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.5.2"
+            "version": "==1.6.1"
         },
         "whitebox": {
             "hashes": [
@@ -2986,11 +3105,11 @@
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:be3228a73bbab189a16be2d4a3cd89ecbd4e31948bfdc64edac17dcdee3cd99c",
-                "sha256:ea67c17a7cd4ae358f8f46c3b304c40698bc0423732e3f273321ee141232c8be"
+                "sha256:2e37f0ce9da11651056280c7efe96f2db052fe8fc269508e3724f5cbd6c93018",
+                "sha256:9ec291ba87c2dfad42c3d5b6f68713fa18be1acd7476569516b2431682315c17"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.0.7"
+            "version": "==4.0.8"
         },
         "xarray": {
             "hashes": [
@@ -3002,90 +3121,90 @@
         },
         "y-py": {
             "hashes": [
-                "sha256:05f805b58422d5d7c8e7e8e2141d1c3cac4daaa4557ae6a9b84b141fe8d6289e",
-                "sha256:065f90501cf008375d70be6ce72dd41745e09d088f0b545f5f914d2c3f04f7ae",
-                "sha256:0c0e333c20b0a6ce4a5851203d45898ab93f16426c342420b931e190c5b71d3d",
-                "sha256:13b9d2959d9a26536b6ad118fb026ff19bd79da52e4addf6f3a562e7c01d516e",
-                "sha256:1906f13e8d5ebfbd9c7948f57bc6f6f53b451b19c99350f42a0f648147a8acfe",
-                "sha256:1f54625b9ed4e787872c45d3044dcfd04c0da4258d9914f3d32308830b35246c",
-                "sha256:202b2a3e42e0a1eaedee26f8a3bc73cd9f994c4c2b15511ea56b9838178eb380",
-                "sha256:2532ea5aefb223fd688c93860199d348a7601d814aac9e8784d816314588ddeb",
-                "sha256:25637e3d011ca6f877a24f3083ff2549d1d619406d7e8a1455c445527205046c",
-                "sha256:2692c808bf28f797f8d693f45dc86563ac3b1626579f67ce9546dca69644d687",
-                "sha256:27c1e9a866146d250e9e16d99fe22a40c82f5b592ab85da97e5679fc3841c7ce",
-                "sha256:2ffebe5e62cbfee6e24593927dedba77dc13ac4cfb9c822074ab566b1fb63d59",
-                "sha256:50cfa0532bcee27edb8c64743b49570e28bb76a00cd384ead1d84b6f052d9368",
-                "sha256:55098440e32339c2dc3d652fb36bb77a4927dee5fd4ab0cb1fe12fdd163fd4f5",
-                "sha256:5dbd8d177ec7b9fef4a7b6d22eb2f8d5606fd5aac31cf2eab0dc18f0b3504c7c",
-                "sha256:63ef8e5b76cd54578a7fd5f72d8c698d9ccd7c555c7900ebfd38a24d397c3b15",
-                "sha256:73200c59bb253b880825466717941ac57267f2f685b053e183183cb6fe82874d",
-                "sha256:7353af0e9c1f42fbf0ab340e253eeb333d58c890fa91d3eadb1b9adaf9336732",
-                "sha256:742c486d5b792c4ad76e09426161302edddca85efe826fa01dcee50907326cd7",
-                "sha256:753aaae817d658a1e9d271663439d8e83d9d8effa45590ecdcadc600c7cf77e3",
-                "sha256:76b3480e7037ac9390c450e2aff9e46e2c9e61520c0d88afe228110ec728adc5",
-                "sha256:800e73d2110b97a74c52db2c8ce03a78e96f0d66a7e0c87d8254170a67c2db0e",
-                "sha256:85585e669d7679126e4a04e4bc0a063a641175a74eecfe47539e8da3e5b1da6e",
-                "sha256:8d4dfc276f988175baaa4ab321c3321a16ce33db3356c9bc5f4dea0db3de55aa",
-                "sha256:91be189fae8ba242528333e266e38d65cae3d9a09fe45867fab8578a3ddf2ea2",
-                "sha256:9484a3fc33f812234e58a5ee834b42bb0a628054d61b5c06c323aa56c12e557d",
-                "sha256:9513ae81fcc805671ae134c4c7421ca322acf92ce8b33817e1775ea8c0176973",
-                "sha256:95d13b38c9055d607565b77cbae12e2bf0c1671c5cb8f2ee2e1230d41d2d6d34",
-                "sha256:9983e99e3a61452b39ffce98206c7e4c6d260f4e917c8fe53fb54aaf25df89a3",
-                "sha256:9a59603cf42c20d02ee5add2e3d0ce48e89c480a2a02f642fb77f142c4f37958",
-                "sha256:a57d81260e048caacf43a2f851766687f53e8a8356df6947fb0eee7336a7e2de",
-                "sha256:a7977eeaceaeb0dfffcc5643c985c337ebc33a0b1d792ae0a9b1331cdd97366f",
-                "sha256:add793f5f5c7c7a3eb1b09ffc771bdaae10a0bd482a370bf696b83f8dee8d1b4",
-                "sha256:ae82a6d9cbaff8cb7505e81b5b7f9cd7756bb7e7110aef7914375fe56b012a90",
-                "sha256:af6df5ec1d66ee2d962026635d60e84ad35fc01b2a1e36b993360c0ce60ae349",
-                "sha256:afa9a11aa2880dd8689894f3269b653e6d3bd1956963d5329be9a5bf021dab62",
-                "sha256:b0ed760e6aa5316227a0ba2d5d29634a4ef2d72c8bc55169ac01664e17e4b536",
-                "sha256:b44473bb32217c78e18db66f497f6c8be33e339bab5f52398bb2468c904d5140",
-                "sha256:b67dad339f9b6701f74ff7a6e901c7909eca4eea02cf955b28d87a42650bd1be",
-                "sha256:bc9052a814e8b7ec756371a191f38de68b956437e0bb429c2dd503e658f298f9",
-                "sha256:c1f5f287cc7ae127ed6a2fb1546e631b316a41d087d7d2db9caa3e5f59906dcf",
-                "sha256:c3ae6d22b7cc599220a26b06da6ead9fd582eea5fdb6273b06fa3f060d0a26a7",
-                "sha256:c42f3a6cd20153925b00c49af855a3277989d411bb8ea849095be943ee160821",
-                "sha256:c7ca64a2a97f708569dcabd55865915943e30267bf6d26c4d212d005951efe62",
-                "sha256:caf9b1feb69379d424a1d3d7c899b8e0389a3fb3131d39c3c03dcc3d4a93dbdc",
-                "sha256:cb68445414940efe547291340e91604c7b8379b60822678ef29f4fc2a0e11c62",
-                "sha256:cc8e5f38842a4b043c9592bfa9a740147ddb8fac2d7a5b7bf6d52466c090ec23",
-                "sha256:cd6f373dbf592ad83aaf95c16abebc8678928e49bd509ebd593259e1908345ae",
-                "sha256:d2da2a9e28dceab4832945a745cad507579f52b4d0c9e2f54ae156eb56875861",
-                "sha256:d373c6bb8e21d5f7ec0833b76fa1ab480086ada602ef5bbf4724a25a21a00b6a",
-                "sha256:d722d6a27230c1f395535da5cee6a9a16497c6343afd262c846090075c083009",
-                "sha256:db1ac7f2d1862eb4c448cf76183399d555a63dbe2452bafecb1c2f691e36d687",
-                "sha256:df78a0409dca11554a4b6442d7a8e61f762c3cfc78d55d98352392869a6b9ae0",
-                "sha256:e30fe2491d095c6d695a2c96257967fd3e2497f0f777030c8492d03c18d46e2a",
-                "sha256:e370ce076781adea161b04d2f666e8b4f89bc7e8927ef842fbb0283d3bfa73e0",
-                "sha256:ecd3cb0d13ac92e7b9235d1024dba9af0788161246f12dcf1f635d634ccb206a",
-                "sha256:ed0fd5265905cc7e23709479bc152d69f4972dec32fa322d20cb77f749707e78",
-                "sha256:f6d87d0c2e87990bc00c049742d36a5dbbb1510949459af17198728890ee748a",
-                "sha256:f7434c77cd23592973ed63341b8d337e6aebaba5ed40d7f22e2d43dfd0c3a56e",
-                "sha256:f8b67ae37af8aac6160fda66c0f73bcdf65c06da9022eb76192c3fc45cfab994",
-                "sha256:f8f238144a302f17eb26b122cad9382fcff5ec6653b8a562130b9a5e44010098",
-                "sha256:fa685f7e43ce490dfb1e392ac48f584b75cd21f05dc526c160d15308236ce8a0",
-                "sha256:fce5feb57f6231376eb10d1fb68c60da106ffa0b520b3129471c466eff0304cc",
-                "sha256:fdafb93bfd5532b13a53c4090675bcd31724160017ecc73e492dc1211bc0377a",
-                "sha256:fe70d0134fe2115c08866f0cac0eb5c0788093872b5026eb438a74e1ebafd659",
-                "sha256:ff3ddedaa95284f4f22a92b362f658f3d92f272d8c0fa009051bd5490c4d5a04"
-            ],
-            "version": "==0.5.9"
+                "sha256:035ba7ce31bb87bd7b5977eee71ee2ff71e54d347a35e2079362b1c23731dccd",
+                "sha256:0624a5adf29d29330a336eecdf15874871f559d50944d542012665e1c3a18265",
+                "sha256:11345294820908d5b8af9c6616ea908dda8b3e554ee6f6d50be6a2e15940f63e",
+                "sha256:1667b8a67ace756c04f03778e86fc359028c98905212f8686afb48c26c252bda",
+                "sha256:19b7c3eaf65b162e59486a48bea5dd2035937952f15e008a14813e8cb7c24d7b",
+                "sha256:1ea7d796bb55d08dd1a60736beb724004f2cbdc207592b5f301a5ff314b17137",
+                "sha256:1eefb6371cd6e072cf467b897f85bd0d7575f3a3e944fb8675f84fb59aedd071",
+                "sha256:1fb03947937b0fcb09eb2b94eb08d8e8030ef0ed70af777684ab670bd369bc3c",
+                "sha256:281535bb4f18fe09e5517a63b8206dd6f26ad6fb7e7c25c62bf785e594adab4d",
+                "sha256:2c230bc01b96081550b7583b77d00404fd39825657f4064b919a10515f660cdf",
+                "sha256:304e88a3deaff9906faa7ba514cf82f4ca4bad1ea88728206ff906e66179abd3",
+                "sha256:30b9337e4f3d541879a8187af121be1bd42ea110372a21895a1a3f800a6bd1c3",
+                "sha256:374ffef1939c42286ea18e2a413c9974430226227f8f1480bbee469933aa675b",
+                "sha256:37b9f24b00972e5685d0b9bbd01413d9c33d124145343fb92667f0e076f040ad",
+                "sha256:391a232c328c2be1de4cb152ed3e9427826e4cbd9d645feacb3dbb344b122e10",
+                "sha256:4181b28f736cae3bb4517090ae5eeca318c075c0106466f13a4ed6682265fc8a",
+                "sha256:418aaa796a22b0102de09b36b6c6294d0a485f04bc8866c3b28f17e7022c44ba",
+                "sha256:46836169f7dc2957df8513cfe4bc2009175b3a473e630af421a8e75ee1c48f98",
+                "sha256:479da40ef1205de52d87209534bf8e713a782e01eeed3df8dff44d21085e3f63",
+                "sha256:47b3604c874d25616a097adaaabcad6e77729e23c5d029092b8149af1a08b2a5",
+                "sha256:49adf7e25c3b3bac9f19bee181ef5253659ebe5747a7141860692015222b2007",
+                "sha256:4c16d50d0728abd915bd9e2e0c3ce982005ba78b60e4b6666aadc592d9982c79",
+                "sha256:4cac9259839b32706336b3f521cacfd16fc7cefee609bd9c2b5123099328d696",
+                "sha256:4f025c50301d9ddbbc2384f98d3ff1dbfe43606146b747e23a17774a02faffe9",
+                "sha256:513a2fe1318c247fc3b3c3ad208488e870a216784f2a3e6dbe2688c92f671c86",
+                "sha256:51f1997dae6d77b12b50502871c7a9aae22e84048e83b64fe6d4f18dec2e4700",
+                "sha256:5743e94c982585f05e02d9a3345dd9b1f28d90fa128df9f60b0eb357a76d2c32",
+                "sha256:6377e3cbab8f5b8b918130e9f924358f98ca1bea12a8096d3fadea191f7137f1",
+                "sha256:69e05e01594e99c934562124b159720533b7ad887dde7762d460916aac47a8e4",
+                "sha256:6a5a882591c8e1b1d6fbdb7ab43884907cef2b6a18e36c7ae85589e5f55371e5",
+                "sha256:71f7689c25bd7608e1e7a76a13138cb202455fac165018693a3e8e5675f54b82",
+                "sha256:74d5ebb5f9ef0c4c1f7bdd9ab5e53b9d8be4c7464905f39761b22b6ce0d327d3",
+                "sha256:76e2b14004cadb237499a8a068fd7a8b805b5c1fd0508530473e087c7dd25163",
+                "sha256:83115cbbd4f6d3b38ebe06d80b1d0dbf1b10e53947f71df16f6145a4f0d14716",
+                "sha256:8f143fdcda7a6a89bf96d9b359142a7ca3315e8a9018aa46b0abbdeb47d7192e",
+                "sha256:9242f3a5c6293e634817d9984c60523ffb34cf5b41501c5958681a75745946e6",
+                "sha256:95083c4cdbd593497a695e841b2ad050c0b9a8a9e374f8496aa478cebfcf9cc9",
+                "sha256:97812f9443fd846012d60ecacffa2a11992d02ad9f8618d4faae8e596736c646",
+                "sha256:9a920bf096d1eecb0f30afc38ee56bfcb9e2c863c33db96fc9d30d4ac0dbee58",
+                "sha256:9dad6af2d83a2b0618ba3c1a2fc6657c5303cf4e9f1a65cc3fea40ffbcc552e2",
+                "sha256:9f56888aeb07ca76a5cd552581bb3735fcd2d8c18165b946fdb6e4507b10e76c",
+                "sha256:a027c39296c925f0b81e28a0fefab8c5964a0ea2b50fa05cbddf5e5ab167a380",
+                "sha256:a20a4d10c8f0ee2b6df265d182d0be0ecd2ba7348c0a20b9df7d4d39df895801",
+                "sha256:a4b488be17d83173acb7f07c7e3430d2c66d0bd55b821683089311699562b58b",
+                "sha256:a752ba8875ed2038dfc7d62738536cb22b4e308951cb925a7fe8fef782c6db08",
+                "sha256:aaaec9718f8a23924c95294d41d87829b113bc9a606a3667dfb995afc45c9920",
+                "sha256:b44fdd64598e9ed4008158e5e60be5e1e2daeed6fae0ab2bf0002461e960709d",
+                "sha256:b5f5975c1a8c2ca99980571b8811d151db8590de9cc96346572a81e0f6f1e30e",
+                "sha256:b6273d84605ee55b3ac52742018f94602dab9b0457f29e6f787021c473b02fed",
+                "sha256:b71cd495d322da25a53a6a830b591a2c0c46db22bb0b3556fca0bbdb1d45a18e",
+                "sha256:b75c2199a125ef8926f3216fb324c3bcd8b1b4b6c0b428888cc753ee4c85f81f",
+                "sha256:c0505e2ca36408b754774a2bb20d93b5c7def3873406c13e1855de6f007f8a94",
+                "sha256:c276a7eb3ae3360f5a2fc503f1e4535d4a2f1c8cfc22af4595ad752e9a94fd77",
+                "sha256:cca539c3804a580992304b18a33f1980282d9097a723f0bd01971477cb365b28",
+                "sha256:d1301bfeaa26f78f4b0e5f96e0f22761b38cc407713f70550a1be490945fd6d7",
+                "sha256:e48b5b30242c7d517be85b48246b21e4e26540505a1ffe4fe473e239a8ec56d3",
+                "sha256:e5126786f914ff53ea2f04f9da790db168db172521cc4f114d5501badd2f6b96",
+                "sha256:e5f89cf9ef1daf12f438a075415a02f227594e4b0494c78d3b83cb83651631f5",
+                "sha256:e76be7258010ce8cbb93a841f78f52901bba1253a51213d3535972d13aa4e89e",
+                "sha256:eb60fe68774117378efdbd368ef83cf1417e61d4bc39c6be8e7f4ee91fb7428a",
+                "sha256:ebbebc4f6a9e0c89c7b57035f91043b038e804dd1953845d8a66066f4526c853",
+                "sha256:eccf67d09a4df42a7be2a5427c1b2e0b89bec862f519ded754bd452df516b380",
+                "sha256:ef0f08edb2094869e4d12346ee68d5154cb3d23bc3b1e7679222fae12228261c",
+                "sha256:efb3225b58dc67152c004da3c26ae5bad0afebbb3c7509d853bdd87eaa655137",
+                "sha256:f79ef7303e332e91d738e66e9bb7fce0243d0407a02631a58ebc0bf2fb8743d0",
+                "sha256:fc48db294d327a5cc10ee49f73f1fa1478240cc827c9029e0871106e327353ac"
+            ],
+            "version": "==0.6.0"
         },
         "ypy-websocket": {
             "hashes": [
-                "sha256:491b2cc4271df4dde9be83017c15f4532b597dc43148472eb20c5aeb838a5b46",
-                "sha256:9049d5a7d61c26c2b5a39757c9ffcbe2274bf3553adeea8de7fe1c04671d4145"
+                "sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff",
+                "sha256:b1ba0dfcc9762f0ca168d2378062d3ca1299d39076b0f145d961359121042be5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.8.2"
+            "version": "==0.8.4"
         },
         "zipp": {
             "hashes": [
-                "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b",
-                "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"
+                "sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0",
+                "sha256:ebc15946aa78bd63458992fc81ec3b6f7b1e92d51c35e6de1c3804e73b799147"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.15.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==3.16.2"
         }
     },
     "develop": {
@@ -3123,7 +3242,7 @@
                 "sha256:c7b80eb7f5a855a88efc971fda506ff7a91c280b42cdae26643e0f601ea281ea",
                 "sha256:ea8801f15dfe4ffb76dea1b09b847430ffd70d827b41735c64a0638a04103bfc"
             ],
-            "markers": "python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8'",
             "version": "==8.12.2"
         },
         "jedi": {
@@ -3167,11 +3286,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b",
-                "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"
+                "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
+                "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.38"
+            "version": "==3.0.39"
         },
         "ptyprocess": {
             "hashes": [
@@ -3221,11 +3340,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223",
-                "sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768"
+                "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
+                "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.6.0"
+            "version": "==4.7.1"
         },
         "wcwidth": {
             "hashes": [

From 24297b2c7b5f0acc1808a20d156b887328ee1edb Mon Sep 17 00:00:00 2001
From: Gregory Petrochenkov
 <30809186+GregoryPetrochenkov-NOAA@users.noreply.github.com>
Date: Fri, 11 Aug 2023 09:54:13 -0700
Subject: [PATCH 263/359] v4.3.15.0 Integrate GVAL In Synthesize Cases (#956)

---
 Dockerfile                            |   1 +
 Pipfile                               |   4 +-
 Pipfile.lock                          | 958 ++++++++++++++++++--------
 docs/CHANGELOG.md                     |  14 +
 tools/run_test_case.py                |  38 +-
 tools/synthesize_test_cases.py        |   9 +-
 tools/tools_shared_functions.py       | 666 ++++++++----------
 unit_tests/tools/inundate_gms_test.py |   5 +-
 8 files changed, 998 insertions(+), 697 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2fc6f8993..069da2abd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -90,6 +90,7 @@ ENV PATH="$projectDir:${PATH}"
 ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 
 ## install python 3 modules ##
+
 COPY Pipfile .
 COPY Pipfile.lock .
 RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
diff --git a/Pipfile b/Pipfile
index 8704dd35e..9e62fe41d 100755
--- a/Pipfile
+++ b/Pipfile
@@ -7,11 +7,12 @@ verify_ssl = true
 ipython = "==8.12.0"
 
 [packages]
+certifi = "==2023.7.22"
 fiona = "==1.8.22"
 geopandas = "==0.12.2"
 numba = "==0.56.4"
 numpy = "==1.23.5"
-pandas = "==2.0.0"
+pandas = "==2.0.2"
 rasterio = "==1.3.6"
 rasterstats = "==0.18.0"
 richdem = "==0.3.4"
@@ -36,6 +37,7 @@ pyarrow = "==11.0.0"
 rtree = "==1.0.1"
 py7zr = "==0.20.4"
 scipy = "==1.10.1"
+gval = "==0.1.2"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 2a22fc93b..070d13bd0 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "1f4c512cbfa21c9f52ab6461f0d0d26f955d9e0b21ec612fc4d0b47b8a7820b4"
+            "sha256": "6ea9602b1a3842fbcfde2289f3b414315cbf2d38875ec5cbd738830fb7fe19fc"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -48,6 +48,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==3.7.1"
         },
+        "appdirs": {
+            "hashes": [
+                "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
+                "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
+            ],
+            "version": "==1.4.4"
+        },
         "argon2-cffi": {
             "hashes": [
                 "sha256:8c976986f2c5c0e5000919e6de187906cfd81fb1c72bf9d88c01177e77da7f80",
@@ -126,7 +133,7 @@
                 "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
                 "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==4.12.2"
         },
         "bleach": {
@@ -161,7 +168,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4.0'",
+            "markers": "python_version >= '3.8' and python_version < '4'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -268,6 +275,14 @@
             "markers": "platform_python_implementation == 'CPython'",
             "version": "==1.0.9"
         },
+        "cachetools": {
+            "hashes": [
+                "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590",
+                "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==5.3.1"
+        },
         "certifi": {
             "hashes": [
                 "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
@@ -450,7 +465,8 @@
                 "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
                 "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
+
             "version": "==3.2.0"
         },
         "click": {
@@ -473,16 +489,40 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' and python_version < '4'",
             "version": "==0.7.2"
         },
+        "cloudpickle": {
+            "hashes": [
+                "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f",
+                "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.2.1"
+        },
+        "colorcet": {
+            "hashes": [
+                "sha256:51455a20353d12fac91f953772d8409f2474e6a0db1af3fa4f7005f405a2480b",
+                "sha256:8daff01824ee9935fdf762d15c444a67d3e361ad4f8b738ad59ac9bf38f30600"
+            ],
+            "markers": "python_version >= '2.7'",
+            "version": "==3.0.1"
+        },
         "comm": {
             "hashes": [
-                "sha256:16613c6211e20223f215fc6d3b266a247b6e2641bf4e0a3ad34cb1aff2aa3f37",
-                "sha256:a61efa9daffcfbe66fd643ba966f846a624e4e6d6767eda9cf6e993aadaab93e"
+                "sha256:354e40a59c9dd6db50c5cc6b4acc887d82e9603787f83b68c01a80a923984d15",
+                "sha256:6d52794cba11b36ed9860999cd10fd02d6b2eac177068fdd585e1e2f8a96e67a"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.1.4"
+        },
+        "contextily": {
+            "hashes": [
+                "sha256:3428aedbd4a7e3e1f6caf56f578d7685469072b5bd7541d92e4b54751d51163a",
+                "sha256:c7f0ddf62c87fd20e46a435aaa0551707a552b776f16eb545df5a61d32d80d75"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.1.3"
+            "version": "==1.3.0"
         },
         "contourpy": {
             "hashes": [
@@ -601,6 +641,28 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==3.0.0"
         },
+        "dask": {
+            "hashes": [
+                "sha256:32b34986519b7ddc0947c8ca63c2fc81b964e4c208dfb5cbf9f4f8aec92d152b",
+                "sha256:4f4c28ac406e81b8f21b5be4b31b21308808f3e0e7c7e2f4a914f16476d9941b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.5.0"
+        },
+        "datashader": {
+            "hashes": [
+                "sha256:2eea890cd8ea74bdb8d5601fd844b875f298931225bea8d1fc9db786c112f045",
+                "sha256:be481f3bd58628cb9a2f94b173031058a1ba1446312e045b92f7ed524330838b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.15.1"
+        },
+        "datashape": {
+            "hashes": [
+                "sha256:2356ea690c3cf003c1468a243a9063144235de45b080b3652de4f3d44e57d783"
+            ],
+            "version": "==0.5.2"
+        },
         "debugpy": {
             "hashes": [
                 "sha256:0679b7e1e3523bd7d7869447ec67b59728675aadfc038550a63a362b63029d2c",
@@ -641,6 +703,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.7.1"
         },
+        "entrypoints": {
+            "hashes": [
+                "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4",
+                "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.4"
+        },
         "exceptiongroup": {
             "hashes": [
                 "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5",
@@ -690,43 +760,44 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:1df1b6f4c7c4bc8201eb47f3b268adbf2539943aa43c400f84556557e3e109c0",
-                "sha256:2a22b2c425c698dcd5d6b0ff0b566e8e9663172118db6fd5f1941f9b8063da9b",
-                "sha256:33191f062549e6bb1a4782c22a04ebd37009c09360e2d6686ac5083774d06d95",
-                "sha256:38cdecd8f1fd4bf4daae7fed1b3170dfc1b523388d6664b2204b351820aa78a7",
-                "sha256:3ae64303ba670f8959fdaaa30ba0c2dabe75364fdec1caeee596c45d51ca3425",
-                "sha256:3d1f9471134affc1e3b1b806db6e3e2ad3fa99439e332f1881a474c825101096",
-                "sha256:4e3334d51f0e37e2c6056e67141b2adabc92613a968797e2571ca8a03bd64773",
-                "sha256:4edc795533421e98f60acee7d28fc8d941ff5ac10f44668c9c3635ad72ae9045",
-                "sha256:547ab36a799dded58a46fa647266c24d0ed43a66028cd1cd4370b246ad426cac",
-                "sha256:59eba8b2e749a1de85760da22333f3d17c42b66e03758855a12a2a542723c6e7",
-                "sha256:704bccd69b0abb6fab9f5e4d2b75896afa48b427caa2c7988792a2ffce35b441",
-                "sha256:73ef0bb5d60eb02ba4d3a7d23ada32184bd86007cb2de3657cfcb1175325fc83",
-                "sha256:7763316111df7b5165529f4183a334aa24c13cdb5375ffa1dc8ce309c8bf4e5c",
-                "sha256:849ec722bbf7d3501a0e879e57dec1fc54919d31bff3f690af30bb87970f9784",
-                "sha256:891cfc5a83b0307688f78b9bb446f03a7a1ad981690ac8362f50518bc6153975",
-                "sha256:952cb405f78734cf6466252fec42e206450d1a6715746013f64df9cbd4f896fa",
-                "sha256:a7bbb290d13c6dd718ec2c3db46fe6c5f6811e7ea1e07f145fd8468176398224",
-                "sha256:a9b3cc10dc9e0834b6665fd63ae0c6964c6bc3d7166e9bc84772e0edd09f9fa2",
-                "sha256:aaaef294d8e411f0ecb778a0aefd11bb5884c9b8333cc1011bdaf3b58ca4bd75",
-                "sha256:afce2aeb80be72b4da7dd114f10f04873ff512793d13ce0b19d12b2a4c44c0f0",
-                "sha256:b0938ebbeccf7c80bb9a15e31645cf831572c3a33d5cc69abe436e7000c61b14",
-                "sha256:b2d1ee95be42b80d1f002d1ee0a51d7a435ea90d36f1a5ae331be9962ee5a3f1",
-                "sha256:b927e5f466d99c03e6e20961946314b81d6e3490d95865ef88061144d9f62e38",
-                "sha256:bdd729744ae7ecd7f7311ad25d99da4999003dcfe43b436cf3c333d4e68de73d",
-                "sha256:c2071267deaa6d93cb16288613419679c77220543551cbe61da02c93d92df72f",
-                "sha256:cac73bbef7734e78c60949da11c4903ee5837168e58772371bd42a75872f4f82",
-                "sha256:da2c2964bdc827ba6b8a91dc6de792620be4da3922c4cf0599f36a488c07e2b2",
-                "sha256:e16a9449f21a93909c5be2f5ed5246420f2316e94195dbfccb5238aaa38f9751",
-                "sha256:e5c2b0a95a221838991e2f0e455dec1ca3a8cc9cd54febd68cc64d40fdb83669",
-                "sha256:ec453a45778524f925a8f20fd26a3326f398bfc55d534e37bab470c5e415caa1",
-                "sha256:edee0900cf0eedb29d17c7876102d6e5a91ee333882b1f5abc83e85b934cadb5",
-                "sha256:f14f3ccea4cc7dd1b277385adf3c3bf18f9860f87eab9c2fb650b0af16800f55",
-                "sha256:f240d9adf0583ac8fc1646afe7f4ac039022b6f8fa4f1575a2cfa53675360b69",
-                "sha256:f48602c0b3fd79cd83a34c40af565fe6db7ac9085c8823b552e6e751e3a5b8be"
+                "sha256:01cfe02416b6d416c5c8d15e30315cbcd3e97d1b50d3b34b0ce59f742ef55258",
+                "sha256:0a1466713e54bdbf5521f2f73eebfe727a528905ff5ec63cda40961b4b1eea95",
+                "sha256:0df8ef75ba5791e873c9eac2262196497525e3f07699a2576d3ab9ddf41cb619",
+                "sha256:10dac980f2b975ef74532e2a94bb00e97a95b4595fb7f98db493c474d5f54d0e",
+                "sha256:150122ed93127a26bc3670ebab7e2add1e0983d30927733aec327ebf4255b072",
+                "sha256:1f81ed9065b4bd3f4f3ce8e4873cd6a6b3f4e92b1eddefde35d332c6f414acc3",
+                "sha256:27ec3246a088555629f9f0902f7412220c67340553ca91eb540cf247aacb1983",
+                "sha256:2d6dc3fa91414ff4daa195c05f946e6a575bd214821e26d17ca50f74b35b0fe4",
+                "sha256:329341ba3d86a36e482610db56b30705384cb23bd595eac8cbb045f627778e9d",
+                "sha256:3fb2a69870bfe143ec20b039a1c8009e149dd7780dd89554cc8a11f79e5de86b",
+                "sha256:4655c480a1a4d706152ff54f20e20cf7609084016f1df3851cce67cef768f40a",
+                "sha256:48e82d776d2e93f88ca56567509d102266e7ab2fb707a0326f032fe657335238",
+                "sha256:57b68eab183fafac7cd7d464a7bfa0fcd4edf6c67837d14fb09c1c20516cf20b",
+                "sha256:58c1165f9b2662645de9b19a8c8bdd636b36294ccc07e1b0163856b74f10bafc",
+                "sha256:614b1283dca88effd20ee48160518e6de275ce9b5456a3134d5f235523fc5065",
+                "sha256:685a4dd6cf31593b50d6d441feb7781a4a7ef61e19551463e14ed7c527b86f9f",
+                "sha256:6bd7e4777bff1dcb7c4eff4786998422770f3bfbef8be401c5332895517ba3fa",
+                "sha256:703101eb0490fae32baf385385d47787b73d9ea55253df43b487c89ec767e0d7",
+                "sha256:83b98be5d291e08501bd4fc0c4e0f8e6e05b99f3924068b17c5c9972af6fff84",
+                "sha256:8ece1886d12bb36c48c00b2031518877f41abae317e3a55620d38e307d799b7e",
+                "sha256:9c456d1f23deff64ffc8b5b098718e149279abdea4d8692dba69172fb6a0d597",
+                "sha256:9cd2363ea7728496827658682d049ffb2e98525e2247ca64554864a8cc945568",
+                "sha256:a9b55d2a3b360e0c7fc5bd8badf1503ca1c11dd3a1cd20f2c26787ffa145a9c7",
+                "sha256:ae7df0ae9ee2f3f7676b0ff6f4ebe48ad0acaeeeaa0b6839d15dbf0709f2c5ef",
+                "sha256:ae881e484702efdb6cf756462622de81d4414c454edfd950b137e9a7352b3cb9",
+                "sha256:b8600ae7dce6ec3ddfb201abb98c9d53abbf8064d7ac0c8a0d8925e722ccf2a0",
+                "sha256:c36c904ce0322df01e590ba814d5d69e084e985d7e4c2869378671d79662a7d4",
+                "sha256:c8bf88f9e3ce347c716921804ef3a8330cb128284eb6c0b6c4b3574f3c580023",
+                "sha256:d40673b2e927f7cd0819c6f04489dfbeb337b4a7b10fc633c89bf4f34ecb9620",
+                "sha256:d54e600a2bcfa5cdaa860237765c01804a03b08404d6affcd92942fa7315ffba",
+                "sha256:dfe7fa7e607f7e8b58d0c32501a3a7cac148538300626d1b930082c90ae7f6bd",
+                "sha256:e35bed436726194c5e6e094fdfb423fb7afaa0211199f9d245e59e11118c576c",
+                "sha256:f0290ea7f9945174bd4dfd66e96149037441eb2008f3649094f056201d99e293",
+                "sha256:fae4e801b774cc62cecf4a57b1eae4097903fced00c608d9e2bc8f84cd87b54a"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.41.1"
+            "version": "==4.42.0"
+
         },
         "fqdn": {
             "hashes": [
@@ -735,6 +806,30 @@
             ],
             "version": "==1.5.1"
         },
+        "fsspec": {
+            "hashes": [
+                "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a",
+                "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.6.0"
+        },
+        "geocube": {
+            "hashes": [
+                "sha256:35d446f61e8ce97d0bcc3c692d003b355df19d7f59c8cd8bd61d2f9e3c7f72c8",
+                "sha256:b50ee83e4dcf465ccc54e62abe0e6458a26e8faf1e7b15b27604f936781c8a04"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.3.3"
+        },
+        "geographiclib": {
+            "hashes": [
+                "sha256:6b7225248e45ff7edcee32becc4e0a1504c606ac5ee163a5656d482e0cd38734",
+                "sha256:f7f41c85dc3e1c2d3d935ec86660dc3b2c848c83e17f9a9e51ba9d5146a15859"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0"
+        },
         "geopandas": {
             "hashes": [
                 "sha256:0a470e4bf6f5367e6fd83ab6b40405e0b805c8174665bbcb7c4077ed90202912",
@@ -743,6 +838,22 @@
             "index": "pypi",
             "version": "==0.12.2"
         },
+        "geopy": {
+            "hashes": [
+                "sha256:228cd53b6eef699b2289d1172e462a90d5057779a10388a7366291812601187f",
+                "sha256:4a29a16d41d8e56ba8e07310802a1cbdf098eeb6069cc3d6d3068fc770629ffc"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.3.0"
+        },
+        "gval": {
+            "hashes": [
+                "sha256:4074fce7cb2cda26f5cc610b88da0d9d4c9559d52d75074154948b1dbe4b9bac",
+                "sha256:880d0bb3cdb1d366d9e18423d4478b74f72010d120a2b53cb53603163c790bc0"
+            ],
+            "index": "pypi",
+            "version": "==0.1.2"
+        },
         "idna": {
             "hashes": [
                 "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
@@ -761,11 +872,11 @@
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:4cf94875a8368bd89531a756df9a9ebe1f150e0f885030b461237bc7f2d905f2",
-                "sha256:d952faee11004c045f785bb5636e8f885bed30dc3c940d5d42798a2a4541c185"
+                "sha256:134832a506243891221b88b4ae1213327eea96ceb4e407a00d790bb0626f45cf",
+                "sha256:4359457e42708462b9626a04657c6208ad799ceb41e5c58c57ffa0e6a098a5d4"
             ],
-            "markers": "python_version < '3.9'",
-            "version": "==6.0.0"
+            "markers": "python_version < '3.10'",
+            "version": "==6.0.1"
         },
         "inflate64": {
             "hashes": [
@@ -839,7 +950,7 @@
                 "sha256:fd04764d0bb830414788cae897d082bf6ad92324e571a5511bd7e1de4a0cdc67",
                 "sha256:fde3f85864c84badb26f42d95639360e627fd09c529a76c46a06dbd7a5735c51"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version > '3.6'",
             "version": "==0.3.1"
         },
         "iniconfig": {
@@ -852,11 +963,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:e342ce84712861be4b248c4a73472be4702c1b0dd77448bfd6bcfb3af9d5ddf9",
-                "sha256:f0042e867ac3f6bca1679e6a88cbd6a58ed93a44f9d0866aecde6efe8de76659"
+                "sha256:050391364c0977e768e354bdb60cbbfbee7cbb943b1af1618382021136ffd42f",
+                "sha256:c8a2430b357073b37c76c21c52184db42f6b4b0e438e1eb7df3c4440d120497c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.25.0"
+            "version": "==6.25.1"
         },
         "ipympl": {
             "hashes": [
@@ -883,11 +994,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:50ace0a8886e9a0d68b980db82f94c25d55d21ff2340ed36f802dd9365e94acf",
-                "sha256:e0aed0c95a1e55b6a123f64305245578bdc09e52965a34941c2b6a578b8c64a0"
+                "sha256:6c8396cc7b8c95dfb4e9ab0054f48c002f045e7e5d7ae523f559d64e525a98ab",
+                "sha256:ce97dd90525b3066fd00094690964e7eac14cf9b7745d35565b5eeac20cce687"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.0.7"
+            "version": "==8.1.0"
         },
         "isoduration": {
             "hashes": [
@@ -898,11 +1009,11 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
-                "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
+                "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
+                "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.2"
+            "version": "==0.19.0"
         },
         "jinja2": {
             "hashes": [
@@ -920,6 +1031,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.0.1"
         },
+        "joblib": {
+            "hashes": [
+                "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3",
+                "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.1"
+        },
         "json5": {
             "hashes": [
                 "sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f",
@@ -935,15 +1054,12 @@
             "version": "==2.4"
         },
         "jsonschema": {
-            "extras": [
-                "format-nongpl"
-            ],
             "hashes": [
-                "sha256:971be834317c22daaa9132340a51c01b50910724082c2c1a2ac87eeec153a3fe",
-                "sha256:fb3642735399fa958c0d2aad7057901554596c63349f4f6b283c493cf692a25d"
+                "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb",
+                "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.18.4"
+            "version": "==4.19.0"
         },
         "jsonschema-specifications": {
             "hashes": [
@@ -964,11 +1080,11 @@
         },
         "jupyter-client": {
             "hashes": [
-                "sha256:3af69921fe99617be1670399a0b857ad67275eefcfa291e2c81a160b7b650f5f",
-                "sha256:7441af0c0672edc5d28035e92ba5e32fadcfa8a4e608a434c228836a89df6158"
+                "sha256:214668aaea208195f4c13d28eb272ba79f945fc0cf3f11c7092c20b2ca1980e7",
+                "sha256:52be28e04171f07aed8f20e1616a5a552ab9fee9cbbe6c1896ae170c3880d392"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==8.3.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==7.4.9"
         },
         "jupyter-console": {
             "hashes": [
@@ -988,11 +1104,11 @@
         },
         "jupyter-events": {
             "hashes": [
-                "sha256:57a2749f87ba387cd1bfd9b22a0875b889237dbf2edc2121ebb22bde47036c17",
-                "sha256:9a6e9995f75d1b7146b436ea24d696ce3a35bfa8bfe45e0c33c334c79464d0b3"
+                "sha256:4753da434c13a37c3f3c89b500afa0c0a6241633441421f6adafe2fb2e2b924e",
+                "sha256:7be27f54b8388c03eefea123a4f79247c5b9381c49fb1cd48615ee191eb12615"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.6.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.7.0"
         },
         "jupyter-server": {
             "hashes": [
@@ -1174,6 +1290,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.39.1"
         },
+        "locket": {
+            "hashes": [
+                "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632",
+                "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.0.0"
+        },
         "markupsafe": {
             "hashes": [
                 "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e",
@@ -1232,50 +1356,51 @@
         },
         "matplotlib": {
             "hashes": [
-                "sha256:070f8dddd1f5939e60aacb8fa08f19551f4b0140fab16a3669d5cd6e9cb28fc8",
-                "sha256:0c3cca3e842b11b55b52c6fb8bd6a4088693829acbfcdb3e815fa9b7d5c92c1b",
-                "sha256:0f506a1776ee94f9e131af1ac6efa6e5bc7cb606a3e389b0ccb6e657f60bb676",
-                "sha256:12f01b92ecd518e0697da4d97d163b2b3aa55eb3eb4e2c98235b3396d7dad55f",
-                "sha256:152ee0b569a37630d8628534c628456b28686e085d51394da6b71ef84c4da201",
-                "sha256:1c308b255efb9b06b23874236ec0f10f026673ad6515f602027cc8ac7805352d",
-                "sha256:1cd120fca3407a225168238b790bd5c528f0fafde6172b140a2f3ab7a4ea63e9",
-                "sha256:20f844d6be031948148ba49605c8b96dfe7d3711d1b63592830d650622458c11",
-                "sha256:23fb1750934e5f0128f9423db27c474aa32534cec21f7b2153262b066a581fd1",
-                "sha256:2699f7e73a76d4c110f4f25be9d2496d6ab4f17345307738557d345f099e07de",
-                "sha256:26bede320d77e469fdf1bde212de0ec889169b04f7f1179b8930d66f82b30cbc",
-                "sha256:2ecb5be2b2815431c81dc115667e33da0f5a1bcf6143980d180d09a717c4a12e",
-                "sha256:2f8e4a49493add46ad4a8c92f63e19d548b2b6ebbed75c6b4c7f46f57d36cdd1",
-                "sha256:305e3da477dc8607336ba10bac96986d6308d614706cae2efe7d3ffa60465b24",
-                "sha256:30e1409b857aa8a747c5d4f85f63a79e479835f8dffc52992ac1f3f25837b544",
-                "sha256:318c89edde72ff95d8df67d82aca03861240512994a597a435a1011ba18dbc7f",
-                "sha256:35d74ebdb3f71f112b36c2629cf32323adfbf42679e2751252acd468f5001c07",
-                "sha256:50e0a55ec74bf2d7a0ebf50ac580a209582c2dd0f7ab51bc270f1b4a0027454e",
-                "sha256:5dea00b62d28654b71ca92463656d80646675628d0828e08a5f3b57e12869e13",
-                "sha256:60c521e21031632aa0d87ca5ba0c1c05f3daacadb34c093585a0be6780f698e4",
-                "sha256:6515e878f91894c2e4340d81f0911857998ccaf04dbc1bba781e3d89cbf70608",
-                "sha256:6d2ff3c984b8a569bc1383cd468fc06b70d7b59d5c2854ca39f1436ae8394117",
-                "sha256:71667eb2ccca4c3537d9414b1bc00554cb7f91527c17ee4ec38027201f8f1603",
-                "sha256:717157e61b3a71d3d26ad4e1770dc85156c9af435659a25ee6407dc866cb258d",
-                "sha256:71f7a8c6b124e904db550f5b9fe483d28b896d4135e45c4ea381ad3b8a0e3256",
-                "sha256:936bba394682049919dda062d33435b3be211dc3dcaa011e09634f060ec878b2",
-                "sha256:a1733b8e84e7e40a9853e505fe68cc54339f97273bdfe6f3ed980095f769ddc7",
-                "sha256:a2c1590b90aa7bd741b54c62b78de05d4186271e34e2377e0289d943b3522273",
-                "sha256:a7e28d6396563955f7af437894a36bf2b279462239a41028323e04b85179058b",
-                "sha256:a8035ba590658bae7562786c9cc6ea1a84aa49d3afab157e414c9e2ea74f496d",
-                "sha256:a8cdb91dddb04436bd2f098b8fdf4b81352e68cf4d2c6756fcc414791076569b",
-                "sha256:ac60daa1dc83e8821eed155796b0f7888b6b916cf61d620a4ddd8200ac70cd64",
-                "sha256:af4860132c8c05261a5f5f8467f1b269bf1c7c23902d75f2be57c4a7f2394b3e",
-                "sha256:bc221ffbc2150458b1cd71cdd9ddd5bb37962b036e41b8be258280b5b01da1dd",
-                "sha256:ce55289d5659b5b12b3db4dc9b7075b70cef5631e56530f14b2945e8836f2d20",
-                "sha256:d9881356dc48e58910c53af82b57183879129fa30492be69058c5b0d9fddf391",
-                "sha256:dbcf59334ff645e6a67cd5f78b4b2cdb76384cdf587fa0d2dc85f634a72e1a3e",
-                "sha256:ebf577c7a6744e9e1bd3fee45fc74a02710b214f94e2bde344912d85e0c9af7c",
-                "sha256:f081c03f413f59390a80b3e351cc2b2ea0205839714dbc364519bcf51f4b56ca",
-                "sha256:fdbb46fad4fb47443b5b8ac76904b2e7a66556844f33370861b4788db0f8816a",
-                "sha256:fdcd28360dbb6203fb5219b1a5658df226ac9bebc2542a9e8f457de959d713d0"
+                "sha256:08308bae9e91aca1ec6fd6dda66237eef9f6294ddb17f0d0b3c863169bf82353",
+                "sha256:14645aad967684e92fc349493fa10c08a6da514b3d03a5931a1bac26e6792bd1",
+                "sha256:21e9cff1a58d42e74d01153360de92b326708fb205250150018a52c70f43c290",
+                "sha256:28506a03bd7f3fe59cd3cd4ceb2a8d8a2b1db41afede01f66c42561b9be7b4b7",
+                "sha256:2bf092f9210e105f414a043b92af583c98f50050559616930d884387d0772aba",
+                "sha256:3032884084f541163f295db8a6536e0abb0db464008fadca6c98aaf84ccf4717",
+                "sha256:3a2cb34336110e0ed8bb4f650e817eed61fa064acbefeb3591f1b33e3a84fd96",
+                "sha256:3ba2af245e36990facf67fde840a760128ddd71210b2ab6406e640188d69d136",
+                "sha256:3d7bc90727351fb841e4d8ae620d2d86d8ed92b50473cd2b42ce9186104ecbba",
+                "sha256:438196cdf5dc8d39b50a45cb6e3f6274edbcf2254f85fa9b895bf85851c3a613",
+                "sha256:46a561d23b91f30bccfd25429c3c706afe7d73a5cc64ef2dfaf2b2ac47c1a5dc",
+                "sha256:4cf327e98ecf08fcbb82685acaf1939d3338548620ab8dfa02828706402c34de",
+                "sha256:4f99e1b234c30c1e9714610eb0c6d2f11809c9c78c984a613ae539ea2ad2eb4b",
+                "sha256:544764ba51900da4639c0f983b323d288f94f65f4024dc40ecb1542d74dc0500",
+                "sha256:56d94989191de3fcc4e002f93f7f1be5da476385dde410ddafbb70686acf00ea",
+                "sha256:57bfb8c8ea253be947ccb2bc2d1bb3862c2bccc662ad1b4626e1f5e004557042",
+                "sha256:617f14ae9d53292ece33f45cba8503494ee199a75b44de7717964f70637a36aa",
+                "sha256:6eb88d87cb2c49af00d3bbc33a003f89fd9f78d318848da029383bfc08ecfbfb",
+                "sha256:75d4725d70b7c03e082bbb8a34639ede17f333d7247f56caceb3801cb6ff703d",
+                "sha256:770a205966d641627fd5cf9d3cb4b6280a716522cd36b8b284a8eb1581310f61",
+                "sha256:7b73305f25eab4541bd7ee0b96d87e53ae9c9f1823be5659b806cd85786fe882",
+                "sha256:7c9a4b2da6fac77bcc41b1ea95fadb314e92508bf5493ceff058e727e7ecf5b0",
+                "sha256:81a6b377ea444336538638d31fdb39af6be1a043ca5e343fe18d0f17e098770b",
+                "sha256:83111e6388dec67822e2534e13b243cc644c7494a4bb60584edbff91585a83c6",
+                "sha256:8704726d33e9aa8a6d5215044b8d00804561971163563e6e6591f9dcf64340cc",
+                "sha256:89768d84187f31717349c6bfadc0e0d8c321e8eb34522acec8a67b1236a66332",
+                "sha256:8bf26ade3ff0f27668989d98c8435ce9327d24cffb7f07d24ef609e33d582439",
+                "sha256:8c587963b85ce41e0a8af53b9b2de8dddbf5ece4c34553f7bd9d066148dc719c",
+                "sha256:95cbc13c1fc6844ab8812a525bbc237fa1470863ff3dace7352e910519e194b1",
+                "sha256:97cc368a7268141afb5690760921765ed34867ffb9655dd325ed207af85c7529",
+                "sha256:a867bf73a7eb808ef2afbca03bcdb785dae09595fbe550e1bab0cd023eba3de0",
+                "sha256:b867e2f952ed592237a1828f027d332d8ee219ad722345b79a001f49df0936eb",
+                "sha256:c0bd19c72ae53e6ab979f0ac6a3fafceb02d2ecafa023c5cca47acd934d10be7",
+                "sha256:ce463ce590f3825b52e9fe5c19a3c6a69fd7675a39d589e8b5fbe772272b3a24",
+                "sha256:cf0e4f727534b7b1457898c4f4ae838af1ef87c359b76dcd5330fa31893a3ac7",
+                "sha256:def58098f96a05f90af7e92fd127d21a287068202aa43b2a93476170ebd99e87",
+                "sha256:e99bc9e65901bb9a7ce5e7bb24af03675cbd7c70b30ac670aa263240635999a4",
+                "sha256:eb7d248c34a341cd4c31a06fd34d64306624c8cd8d0def7abb08792a5abfd556",
+                "sha256:f67bfdb83a8232cb7a92b869f9355d677bce24485c460b19d01970b64b2ed476",
+                "sha256:f883a22a56a84dba3b588696a2b8a1ab0d2c3d41be53264115c71b0a942d8fdb",
+                "sha256:fbdeeb58c0cf0595efe89c05c224e0a502d1aa6a8696e68a73c3efc6bc354304"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.7.2"
+            "version": "==3.7.1"
+
         },
         "matplotlib-inline": {
             "hashes": [
@@ -1293,6 +1418,13 @@
             "index": "pypi",
             "version": "==0.61.0"
         },
+        "mercantile": {
+            "hashes": [
+                "sha256:30f457a73ee88261aab787b7069d85961a5703bb09dc57a170190bc042cd023f",
+                "sha256:fa3c6db15daffd58454ac198b31887519a19caccee3f9d63d17ae7ff61b3b56b"
+            ],
+            "version": "==1.2.1"
+        },
         "mistune": {
             "hashes": [
                 "sha256:b9b3e438efbb57c62b5beb5e134dab664800bdf1284a7ee09e8b12b13eb1aac6",
@@ -1301,6 +1433,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==3.0.1"
         },
+        "morecantile": {
+            "hashes": [
+                "sha256:610f1dcc3ae0a99f0a0e6c05e18508d0e9e26b53279e06a65150939e2f54963b",
+                "sha256:aa683415f0e6d07f804912ddf15f5a4a96e23281205776e12be5b323b75ca447"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.3.0"
+        },
         "msgpack": {
             "hashes": [
                 "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164",
@@ -1369,6 +1509,21 @@
             ],
             "version": "==1.0.5"
         },
+        "multimethod": {
+            "hashes": [
+                "sha256:1589bf52ca294667fd15527ea830127c763f5bfc38562e3642591ffd0fd9d56f",
+                "sha256:52f8f1f2b9d5a4c7adfdcc114dbeeebe3245a4420801e8807e26522a79fb6bc2"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.9.1"
+        },
+        "multipledispatch": {
+            "hashes": [
+                "sha256:0c53cd8b077546da4e48869f49b13164bebafd0c2a5afceb6bb6a316e7fb46e4",
+                "sha256:5c839915465c68206c3e9c473357908216c28383b425361e5d144594bf85a7e0"
+            ],
+            "version": "==1.0.0"
+        },
         "multivolumefile": {
             "hashes": [
                 "sha256:237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678",
@@ -1385,6 +1540,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==4.0.0"
         },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d",
+                "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.0.0"
+        },
         "natsort": {
             "hashes": [
                 "sha256:517595492dde570a4fd6b6a76f644440c1ba51e2338c8a671d7f0475fda8f9fd",
@@ -1419,19 +1582,19 @@
         },
         "nbformat": {
             "hashes": [
-                "sha256:3a7f52d040639cbd8a3890218c8b0ffb93211588c57446c90095e32ba5881b5d",
-                "sha256:b7968ebf4811178a4108ee837eae1442e3f054132100f0359219e9ed1ce3ca45"
+                "sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9",
+                "sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.9.1"
+            "version": "==5.9.2"
         },
         "nest-asyncio": {
             "hashes": [
-                "sha256:b9a953fb40dceaa587d109609098db21900182b16440652454a146cffb06e8b8",
-                "sha256:d267cc1ff794403f7df692964d1d2a3fa9418ffea2a3f6859a439ff482fef290"
+                "sha256:5301c82941b550b3123a1ea772ba9a1c80bad3a182be8c1a5ae6ad3be57a9657",
+                "sha256:6a80f7b98f24d9083ed24608977c09dd608d83f91cccc24c9d2cba6d10e01c10"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==1.5.6"
+            "version": "==1.5.7"
         },
         "netcdf4": {
             "hashes": [
@@ -1468,11 +1631,11 @@
         },
         "notebook": {
             "hashes": [
-                "sha256:517209568bd47261e2def27a140e97d49070602eea0d226a696f42a7f16c9a4e",
-                "sha256:dd17e78aefe64c768737b32bf171c1c766666a21cc79a44d37a1700771cab56f"
+                "sha256:171039245a5b1a8f8233165091210632c21250ce2a652daed38fe8f94389984f",
+                "sha256:457caa1fa1c647395420945b2b7559f603eedbc9aeb2a59a0c286c8029e31efa"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.5.4"
+            "version": "==6.5.5"
         },
         "notebook-shim": {
             "hashes": [
@@ -1518,39 +1681,39 @@
         },
         "numexpr": {
             "hashes": [
-                "sha256:059546e8f6283ccdb47c683101a890844f667fa6d56258d48ae2ecf1b3875957",
-                "sha256:17ac9cfe6d0078c5fc06ba1c1bbd20b8783f28c6f475bbabd3cad53683075cab",
-                "sha256:3f039321d1c17962c33079987b675fb251b273dbec0f51aac0934e932446ccc3",
-                "sha256:5538b30199bfc68886d2be18fcef3abd11d9271767a7a69ff3688defe782800a",
-                "sha256:655d84eb09adfee3c09ecf4a89a512225da153fdb7de13c447404b7d0523a9a7",
-                "sha256:6931b1e9d4f629f43c14b21d44f3f77997298bea43790cfcdb4dd98804f90783",
-                "sha256:6c368aa35ae9b18840e78b05f929d3a7b3abccdba9630a878c7db74ca2368339",
-                "sha256:6ee9db7598dd4001138b482342b96d78110dd77cefc051ec75af3295604dde6a",
-                "sha256:77898fdf3da6bb96aa8a4759a8231d763a75d848b2f2e5c5279dad0b243c8dfe",
-                "sha256:7bca95f4473b444428061d4cda8e59ac564dc7dc6a1dea3015af9805c6bc2946",
-                "sha256:7d71add384adc9119568d7e9ffa8a35b195decae81e0abf54a2b7779852f0637",
-                "sha256:845a6aa0ed3e2a53239b89c1ebfa8cf052d3cc6e053c72805e8153300078c0b1",
-                "sha256:90f12cc851240f7911a47c91aaf223dba753e98e46dff3017282e633602e76a7",
-                "sha256:9400781553541f414f82eac056f2b4c965373650df9694286b9bd7e8d413f8d8",
-                "sha256:9e34931089a6bafc77aaae21f37ad6594b98aa1085bb8b45d5b3cd038c3c17d9",
-                "sha256:9f096d707290a6a00b6ffdaf581ee37331109fb7b6c8744e9ded7c779a48e517",
-                "sha256:a38664e699526cb1687aefd9069e2b5b9387da7feac4545de446141f1ef86f46",
-                "sha256:a6d2d7740ae83ba5f3531e83afc4b626daa71df1ef903970947903345c37bd03",
-                "sha256:a75967d46b6bd56455dd32da6285e5ffabe155d0ee61eef685bbfb8dafb2e484",
-                "sha256:b076db98ca65eeaf9bd224576e3ac84c05e451c0bd85b13664b7e5f7b62e2c70",
-                "sha256:b318541bf3d8326682ebada087ba0050549a16d8b3fa260dd2585d73a83d20a7",
-                "sha256:b96334fc1748e9ec4f93d5fadb1044089d73fb08208fdb8382ed77c893f0be01",
-                "sha256:c867cc36cf815a3ec9122029874e00d8fbcef65035c4a5901e9b120dd5d626a2",
-                "sha256:d5432537418d18691b9115d615d6daa17ee8275baef3edf1afbbf8bc69806147",
-                "sha256:db93cf1842f068247de631bfc8af20118bf1f9447cd929b531595a5e0efc9346",
-                "sha256:df35324666b693f13a016bc7957de7cc4d8801b746b81060b671bf78a52b9037",
-                "sha256:df3a1f6b24214a1ab826e9c1c99edf1686c8e307547a9aef33910d586f626d01",
-                "sha256:eaec59e9bf70ff05615c34a8b8d6c7bd042bd9f55465d7b495ea5436f45319d0",
-                "sha256:f3a920bfac2645017110b87ddbe364c9c7a742870a4d2f6120b8786c25dc6db3",
-                "sha256:ff5835e8af9a212e8480003d731aad1727aaea909926fd009e8ae6a1cba7f141"
+                "sha256:1510da20e6f5f45333610b1ded44c566e2690c6c437c84f2a212ca09627c7e01",
+                "sha256:178b85ad373c6903e55d75787d61b92380439b70d94b001cb055a501b0821335",
+                "sha256:183d5430db76826e54465c69db93a3c6ecbf03cda5aa1bb96eaad0147e9b68dc",
+                "sha256:283ce8609a7ccbadf91a68f3484558b3e36d27c93c98a41ec205efb0ab43c872",
+                "sha256:34af2a0e857d02a4bc5758bc037a777d50dacb13bcd57c7905268a3e44994ed6",
+                "sha256:39ce106f92ccea5b07b1d6f2f3c4370f05edf27691dc720a63903484a2137e48",
+                "sha256:3c00be69f747f44a631830215cab482f0f77f75af2925695adff57c1cc0f9a68",
+                "sha256:45ed41e55a0abcecf3d711481e12a5fb7a904fe99d42bc282a17cc5f8ea510be",
+                "sha256:4ed0e1c1ef5f34381448539f1fe9015906d21c9cfa2797c06194d4207dadb465",
+                "sha256:51f3ab160c3847ebcca93cd88f935a7802b54a01ab63fe93152994a64d7a6cf2",
+                "sha256:558390fea6370003ac749ed9d0f38d708aa096f5dcb707ddb6e0ca5a0dd37da1",
+                "sha256:55983806815035eb63c5039520688c49536bb7f3cc3fc1d7d64c6a00cf3f353e",
+                "sha256:578fe4008e4d5d6ff01bbeb2d7b7ba1ec658a5cda9c720cd26a9a8325f8ef438",
+                "sha256:5a8dad2bfaad5a5c34a2e8bbf62b9df1dfab266d345fda1feb20ff4e264b347a",
+                "sha256:62b4faf8e0627673b0210a837792bddd23050ecebc98069ab23eb0633ff1ef5f",
+                "sha256:6df184d40d4cf9f21c71f429962f39332f7398147762588c9f3a5c77065d0c06",
+                "sha256:783324ba40eb804ecfc9ebae86120a1e339ab112d0ab8a1f0d48a26354d5bf9b",
+                "sha256:894b027438b8ec88dea32a19193716c79f4ff8ddb92302dcc9731b51ba3565a8",
+                "sha256:9e8b5bf7bcb4e8dcd66522d8fc96e1db7278f901cb4fd2e155efbe62a41dde08",
+                "sha256:aea6ab45c87c0a7041183c08a798f0ad4d7c5eccbce20cfe79ce6f1a45ef3702",
+                "sha256:b594dc9e2d6291a0bc5c065e6d9caf3eee743b5663897832e9b17753c002947a",
+                "sha256:b93f5a866cd13a808bc3d3a9c487d94cd02eec408b275ff0aa150f2e8e5191f8",
+                "sha256:bf85ba1327eb87ec82ae7936f13c8850fb969a0ca34f3ba9fa3897c09d5c80d7",
+                "sha256:c46350dcdb93e32f033eea5a21269514ffcaf501d9abd6036992d37e48a308b0",
+                "sha256:cbfd833ee5fdb0efb862e152aee7e6ccea9c596d5c11d22604c2e6307bff7cad",
+                "sha256:db5c65417d69414f1ab31302ea01d3548303ef31209c38b4849d145be4e1d1ba",
+                "sha256:dd57ab1a3d3aaa9274aff1cefbf93b8ddacc7973afef5b125905f6bf18fabab0",
+                "sha256:de29c77f674e4eb8f0846525a475cab64008c227c8bc4ba5153ab3f72441cc63",
+                "sha256:eb36ffcfa1606e41aa08d559b4277bcad0e16b83941d1a4fee8d2bd5a34f8e0e",
+                "sha256:ef621b4ee366a5c6a484f6678c9259f5b826569f8bfa0b89ba2306d5055468bb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.8.4"
+            "version": "==2.8.5"
         },
         "numpy": {
             "hashes": [
@@ -1586,13 +1749,21 @@
             "index": "pypi",
             "version": "==1.23.5"
         },
+        "odc-geo": {
+            "hashes": [
+                "sha256:0b04d0835e783685f128453e4b4169d86d322632887bfee9c7dbde364a3c324d",
+                "sha256:e03074229181a3ff6115cec6a822be66e916ad644febd0f47ab96b2804a1b45c"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.4.1"
+        },
         "overrides": {
             "hashes": [
-                "sha256:6187d8710a935d09b0bcef8238301d6ee2569d2ac1ae0ec39a8c7924e27f58ca",
-                "sha256:8b97c6c1e1681b78cbc9424b138d880f0803c2254c5ebaabdde57bb6c62093f2"
+                "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d",
+                "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==7.3.1"
+            "version": "==7.4.0"
         },
         "packaging": {
             "hashes": [
@@ -1604,34 +1775,42 @@
         },
         "pandas": {
             "hashes": [
-                "sha256:0778ab54c8f399d83d98ffb674d11ec716449956bc6f6821891ab835848687f2",
-                "sha256:24472cfc7ced511ac90608728b88312be56edc8f19b9ed885a7d2e47ffaf69c0",
-                "sha256:2d1d138848dd71b37e3cbe7cd952ff84e2ab04d8988972166e18567dcc811245",
-                "sha256:3bb9d840bf15656805f6a3d87eea9dcb7efdf1314a82adcf7f00b820427c5570",
-                "sha256:425705cee8be54db2504e8dd2a730684790b15e5904b750c367611ede49098ab",
-                "sha256:4f3320bb55f34af4193020158ef8118ee0fb9aec7cc47d2084dbfdd868a0a24f",
-                "sha256:4ffb14f50c74ee541610668137830bb93e9dfa319b1bef2cedf2814cd5ac9c70",
-                "sha256:52c858de9e9fc422d25e67e1592a6e6135d7bcf9a19fcaf4d0831a0be496bf21",
-                "sha256:57c34b79c13249505e850d0377b722961b99140f81dafbe6f19ef10239f6284a",
-                "sha256:6ded51f7e3dd9b4f8b87f2ceb7bd1a8df2491f7ee72f7074c6927a512607199e",
-                "sha256:70db5c278bbec0306d32bf78751ff56b9594c05a5098386f6c8a563659124f91",
-                "sha256:78425ca12314b23356c28b16765639db10ebb7d8983f705d6759ff7fe41357fa",
-                "sha256:8318de0f886e4dcb8f9f36e45a3d6a6c3d1cfdc508354da85e739090f0222991",
-                "sha256:8f987ec26e96a8490909bc5d98c514147236e49830cba7df8690f6087c12bbae",
-                "sha256:9253edfd015520ce77a9343eb7097429479c039cd3ebe81d7810ea11b4b24695",
-                "sha256:977326039bd1ded620001a1889e2ed4798460a6bc5a24fbaebb5f07a41c32a55",
-                "sha256:a4f789b7c012a608c08cda4ff0872fd979cb18907a37982abe884e6f529b8793",
-                "sha256:b3ba8f5dd470d8bfbc4259829589f4a32881151c49e36384d9eb982b35a12020",
-                "sha256:b5337c87c4e963f97becb1217965b6b75c6fe5f54c4cf09b9a5ac52fc0bd03d3",
-                "sha256:bbb2c5e94d6aa4e632646a3bacd05c2a871c3aa3e85c9bec9be99cb1267279f2",
-                "sha256:c24c7d12d033a372a9daf9ff2c80f8b0af6f98d14664dbb0a4f6a029094928a7",
-                "sha256:cda9789e61b44463c1c4fe17ef755de77bcd13b09ba31c940d20f193d63a5dc8",
-                "sha256:d08e41d96bc4de6f500afe80936c68fce6099d5a434e2af7c7fd8e7c72a3265d",
-                "sha256:d93b7fcfd9f3328072b250d6d001dcfeec5d3bb66c1b9c8941e109a46c0c01a8",
-                "sha256:fcd471c9d9f60926ab2f15c6c29164112f458acb42280365fbefa542d0c2fc74"
+                "sha256:02755de164da6827764ceb3bbc5f64b35cb12394b1024fdf88704d0fa06e0e2f",
+                "sha256:0a1e0576611641acde15c2322228d138258f236d14b749ad9af498ab69089e2d",
+                "sha256:1eb09a242184092f424b2edd06eb2b99d06dc07eeddff9929e8667d4ed44e181",
+                "sha256:30a89d0fec4263ccbf96f68592fd668939481854d2ff9da709d32a047689393b",
+                "sha256:50e451932b3011b61d2961b4185382c92cc8c6ee4658dcd4f320687bb2d000ee",
+                "sha256:51a93d422fbb1bd04b67639ba4b5368dffc26923f3ea32a275d2cc450f1d1c86",
+                "sha256:598e9020d85a8cdbaa1815eb325a91cfff2bb2b23c1442549b8a3668e36f0f77",
+                "sha256:66d00300f188fa5de73f92d5725ced162488f6dc6ad4cecfe4144ca29debe3b8",
+                "sha256:69167693cb8f9b3fc060956a5d0a0a8dbfed5f980d9fd2c306fb5b9c855c814c",
+                "sha256:6d6d10c2142d11d40d6e6c0a190b1f89f525bcf85564707e31b0a39e3b398e08",
+                "sha256:713f2f70abcdade1ddd68fc91577cb090b3544b07ceba78a12f799355a13ee44",
+                "sha256:7376e13d28eb16752c398ca1d36ccfe52bf7e887067af9a0474de6331dd948d2",
+                "sha256:77550c8909ebc23e56a89f91b40ad01b50c42cfbfab49b3393694a50549295ea",
+                "sha256:7b21cb72958fc49ad757685db1919021d99650d7aaba676576c9e88d3889d456",
+                "sha256:9ebb9f1c22ddb828e7fd017ea265a59d80461d5a79154b49a4207bd17514d122",
+                "sha256:a18e5c72b989ff0f7197707ceddc99828320d0ca22ab50dd1b9e37db45b010c0",
+                "sha256:a6b5f14cd24a2ed06e14255ff40fe2ea0cfaef79a8dd68069b7ace74bd6acbba",
+                "sha256:b42b120458636a981077cfcfa8568c031b3e8709701315e2bfa866324a83efa8",
+                "sha256:c4af689352c4fe3d75b2834933ee9d0ccdbf5d7a8a7264f0ce9524e877820c08",
+                "sha256:c7319b6e68de14e6209460f72a8d1ef13c09fb3d3ef6c37c1e65b35d50b5c145",
+                "sha256:cf3f0c361a4270185baa89ec7ab92ecaa355fe783791457077473f974f654df5",
+                "sha256:dd46bde7309088481b1cf9c58e3f0e204b9ff9e3244f441accd220dd3365ce7c",
+                "sha256:dd5476b6c3fe410ee95926873f377b856dbc4e81a9c605a0dc05aaccc6a7c6c6",
+                "sha256:e69140bc2d29a8556f55445c15f5794490852af3de0f609a24003ef174528b79",
+                "sha256:f908a77cbeef9bbd646bd4b81214cbef9ac3dda4181d5092a4aa9797d1bc7774"
             ],
             "index": "pypi",
-            "version": "==2.0.0"
+            "version": "==2.0.2"
+        },
+        "pandera": {
+            "hashes": [
+                "sha256:1cc70f5f182f5b4bd2e248f49372fcd88eba8423b8604ea9e1ebd64222730a41",
+                "sha256:1cf13c06068d64edb1630b0e50d6588243b7c202d7ce00d968fe3d6bef7684f8"
+            ],
+            "markers": "python_version < '3.12' and python_version >= '3.7'",
+            "version": "==0.15.1"
         },
         "pandocfilters": {
             "hashes": [
@@ -1641,6 +1820,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.5.0"
         },
+        "param": {
+            "hashes": [
+                "sha256:59d55048d42a85e148a69837df42bd11c3391d47fad15ba57d118e145f001ef2",
+                "sha256:a2e3b7b07ca7dd1adaa4fb3020a3ef4fe434f27ede453a9d94194c5155677e30"
+            ],
+            "markers": "python_version >= '2.7'",
+            "version": "==1.13.0"
+        },
         "parso": {
             "hashes": [
                 "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0",
@@ -1649,6 +1836,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.8.3"
         },
+        "partd": {
+            "hashes": [
+                "sha256:7a63529348cf0dff14b986db641cd1b83c16b5cb9fc647c2851779db03282ef8",
+                "sha256:aa0ff35dbbcc807ae374db56332f4c1b39b46f67bf2975f5151e0b4186aed0d5"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.4.0"
+        },
         "pexpect": {
             "hashes": [
                 "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
@@ -1736,11 +1931,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421",
-                "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f"
+                "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d",
+                "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.9.1"
+            "version": "==3.10.0"
         },
         "pluggy": {
             "hashes": [
@@ -1763,7 +1958,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.39"
         },
         "psutil": {
@@ -1859,7 +2054,6 @@
                 "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35",
                 "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"
             ],
-            "markers": "os_name != 'nt'",
             "version": "==0.7.0"
         },
         "pure-eval": {
@@ -2044,21 +2238,71 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==3.18.0"
         },
+        "pyct": {
+            "hashes": [
+                "sha256:a4038a8885059ab8cac6f946ea30e0b5e6bdbe0b92b6723f06737035f9d65e8c",
+                "sha256:dd9f4ac5cbd8e37c352c04036062d3c5f67efec76d404761ef16b0cbf26aa6a0"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.5.0"
+        },
+        "pydantic": {
+            "hashes": [
+                "sha256:20a3b30fd255eeeb63caa9483502ba96b7795ce5bf895c6a179b3d909d9f53a6",
+                "sha256:2b71bd504d1573b0b722ae536e8ffb796bedeef978979d076bf206e77dcc55a5",
+                "sha256:3403a090db45d4027d2344859d86eb797484dfda0706cf87af79ace6a35274ef",
+                "sha256:37ebddef68370e6f26243acc94de56d291e01227a67b2ace26ea3543cf53dd5f",
+                "sha256:3b8d5bd97886f9eb59260594207c9f57dce14a6f869c6ceea90188715d29921a",
+                "sha256:409b810f387610cc7405ab2fa6f62bdf7ea485311845a242ebc0bd0496e7e5ac",
+                "sha256:4870f13a4fafd5bc3e93cff3169222534fad867918b188e83ee0496452978437",
+                "sha256:566a04ba755e8f701b074ffb134ddb4d429f75d5dced3fbd829a527aafe74c71",
+                "sha256:67b3714b97ff84b2689654851c2426389bcabfac9080617bcf4306c69db606f6",
+                "sha256:6dab5219659f95e357d98d70577b361383057fb4414cfdb587014a5f5c595f7b",
+                "sha256:748d10ab6089c5d196e1c8be9de48274f71457b01e59736f7a09c9dc34f51887",
+                "sha256:762aa598f79b4cac2f275d13336b2dd8662febee2a9c450a49a2ab3bec4b385f",
+                "sha256:7a26841be620309a9697f5b1ffc47dce74909e350c5315ccdac7a853484d468a",
+                "sha256:7a7db03339893feef2092ff7b1afc9497beed15ebd4af84c3042a74abce02d48",
+                "sha256:7aa75d1bd9cc275cf9782f50f60cddaf74cbaae19b6ada2a28e737edac420312",
+                "sha256:86936c383f7c38fd26d35107eb669c85d8f46dfceae873264d9bab46fe1c7dde",
+                "sha256:88546dc10a40b5b52cae87d64666787aeb2878f9a9b37825aedc2f362e7ae1da",
+                "sha256:8c40964596809eb616d94f9c7944511f620a1103d63d5510440ed2908fc410af",
+                "sha256:990027e77cda6072a566e433b6962ca3b96b4f3ae8bd54748e9d62a58284d9d7",
+                "sha256:9965e49c6905840e526e5429b09e4c154355b6ecc0a2f05492eda2928190311d",
+                "sha256:9f62a727f5c590c78c2d12fda302d1895141b767c6488fe623098f8792255fe5",
+                "sha256:a2d5be50ac4a0976817144c7d653e34df2f9436d15555189f5b6f61161d64183",
+                "sha256:a5939ec826f7faec434e2d406ff5e4eaf1716eb1f247d68cd3d0b3612f7b4c8a",
+                "sha256:aac218feb4af73db8417ca7518fb3bade4534fcca6e3fb00f84966811dd94450",
+                "sha256:adad1ee4ab9888f12dac2529276704e719efcf472e38df7813f5284db699b4ec",
+                "sha256:b69f9138dec566962ec65623c9d57bee44412d2fc71065a5f3ebb3820bdeee96",
+                "sha256:c41bbaae89e32fc582448e71974de738c055aef5ab474fb25692981a08df808a",
+                "sha256:c62376890b819bebe3c717a9ac841a532988372b7e600e76f75c9f7c128219d5",
+                "sha256:ce937a2a2c020bcad1c9fde02892392a1123de6dda906ddba62bfe8f3e5989a2",
+                "sha256:db4c7f7e60ca6f7d6c1785070f3e5771fcb9b2d88546e334d2f2c3934d949028",
+                "sha256:e0014e29637125f4997c174dd6167407162d7af0da73414a9340461ea8573252",
+                "sha256:e088e3865a2270ecbc369924cd7d9fbc565667d9158e7f304e4097ebb9cf98dd",
+                "sha256:ea9eebc2ebcba3717e77cdeee3f6203ffc0e78db5f7482c68b1293e8cc156e5e",
+                "sha256:edfdf0a5abc5c9bf2052ebaec20e67abd52e92d257e4f2d30e02c354ed3e6030",
+                "sha256:f3d4ee957a727ccb5a36f1b0a6dbd9fad5dedd2a41eada99a8df55c12896e18d",
+                "sha256:f79db3652ed743309f116ba863dae0c974a41b688242482638b892246b7db21d"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.10.10"
+        },
         "pygments": {
             "hashes": [
-                "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
-                "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
+                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
+                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.15.1"
+            "version": "==2.16.1"
         },
         "pyparsing": {
             "hashes": [
-                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
-                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
+                "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb",
+                "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"
             ],
             "markers": "python_full_version >= '3.6.8'",
-            "version": "==3.0.9"
+            "version": "==3.1.1"
         },
         "pyppmd": {
             "hashes": [
@@ -2198,7 +2442,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==2.8.2"
         },
         "python-dotenv": {
@@ -2272,86 +2516,84 @@
         },
         "pyzmq": {
             "hashes": [
-                "sha256:01f06f33e12497dca86353c354461f75275a5ad9eaea181ac0dc1662da8074fa",
-                "sha256:0b6b42f7055bbc562f63f3df3b63e3dd1ebe9727ff0f124c3aa7bcea7b3a00f9",
-                "sha256:0c4fc2741e0513b5d5a12fe200d6785bbcc621f6f2278893a9ca7bed7f2efb7d",
-                "sha256:108c96ebbd573d929740d66e4c3d1bdf31d5cde003b8dc7811a3c8c5b0fc173b",
-                "sha256:13bbe36da3f8aaf2b7ec12696253c0bf6ffe05f4507985a8844a1081db6ec22d",
-                "sha256:154bddda2a351161474b36dba03bf1463377ec226a13458725183e508840df89",
-                "sha256:19d0383b1f18411d137d891cab567de9afa609b214de68b86e20173dc624c101",
-                "sha256:1a6169e69034eaa06823da6a93a7739ff38716142b3596c180363dee729d713d",
-                "sha256:1fc56a0221bdf67cfa94ef2d6ce5513a3d209c3dfd21fed4d4e87eca1822e3a3",
-                "sha256:2a21fec5c3cea45421a19ccbe6250c82f97af4175bc09de4d6dd78fb0cb4c200",
-                "sha256:2b15247c49d8cbea695b321ae5478d47cffd496a2ec5ef47131a9e79ddd7e46c",
-                "sha256:2f5efcc29056dfe95e9c9db0dfbb12b62db9c4ad302f812931b6d21dd04a9119",
-                "sha256:2f666ae327a6899ff560d741681fdcdf4506f990595201ed39b44278c471ad98",
-                "sha256:332616f95eb400492103ab9d542b69d5f0ff628b23129a4bc0a2fd48da6e4e0b",
-                "sha256:33d5c8391a34d56224bccf74f458d82fc6e24b3213fc68165c98b708c7a69325",
-                "sha256:3575699d7fd7c9b2108bc1c6128641a9a825a58577775ada26c02eb29e09c517",
-                "sha256:3830be8826639d801de9053cf86350ed6742c4321ba4236e4b5568528d7bfed7",
-                "sha256:3a522510e3434e12aff80187144c6df556bb06fe6b9d01b2ecfbd2b5bfa5c60c",
-                "sha256:3bed53f7218490c68f0e82a29c92335daa9606216e51c64f37b48eb78f1281f4",
-                "sha256:414b8beec76521358b49170db7b9967d6974bdfc3297f47f7d23edec37329b00",
-                "sha256:442d3efc77ca4d35bee3547a8e08e8d4bb88dadb54a8377014938ba98d2e074a",
-                "sha256:47b915ba666c51391836d7ed9a745926b22c434efa76c119f77bcffa64d2c50c",
-                "sha256:48e5e59e77c1a83162ab3c163fc01cd2eebc5b34560341a67421b09be0891287",
-                "sha256:4a82faae00d1eed4809c2f18b37f15ce39a10a1c58fe48b60ad02875d6e13d80",
-                "sha256:4a983c8694667fd76d793ada77fd36c8317e76aa66eec75be2653cef2ea72883",
-                "sha256:4c2fc7aad520a97d64ffc98190fce6b64152bde57a10c704b337082679e74f67",
-                "sha256:4cb27ef9d3bdc0c195b2dc54fcb8720e18b741624686a81942e14c8b67cc61a6",
-                "sha256:4d67609b37204acad3d566bb7391e0ecc25ef8bae22ff72ebe2ad7ffb7847158",
-                "sha256:5482f08d2c3c42b920e8771ae8932fbaa0a67dff925fc476996ddd8155a170f3",
-                "sha256:5489738a692bc7ee9a0a7765979c8a572520d616d12d949eaffc6e061b82b4d1",
-                "sha256:5693dcc4f163481cf79e98cf2d7995c60e43809e325b77a7748d8024b1b7bcba",
-                "sha256:58416db767787aedbfd57116714aad6c9ce57215ffa1c3758a52403f7c68cff5",
-                "sha256:5873d6a60b778848ce23b6c0ac26c39e48969823882f607516b91fb323ce80e5",
-                "sha256:5af31493663cf76dd36b00dafbc839e83bbca8a0662931e11816d75f36155897",
-                "sha256:5e7fbcafa3ea16d1de1f213c226005fea21ee16ed56134b75b2dede5a2129e62",
-                "sha256:65346f507a815a731092421d0d7d60ed551a80d9b75e8b684307d435a5597425",
-                "sha256:6581e886aec3135964a302a0f5eb68f964869b9efd1dbafdebceaaf2934f8a68",
-                "sha256:69511d604368f3dc58d4be1b0bad99b61ee92b44afe1cd9b7bd8c5e34ea8248a",
-                "sha256:7018289b402ebf2b2c06992813523de61d4ce17bd514c4339d8f27a6f6809492",
-                "sha256:71c7b5896e40720d30cd77a81e62b433b981005bbff0cb2f739e0f8d059b5d99",
-                "sha256:75217e83faea9edbc29516fc90c817bc40c6b21a5771ecb53e868e45594826b0",
-                "sha256:7e23a8c3b6c06de40bdb9e06288180d630b562db8ac199e8cc535af81f90e64b",
-                "sha256:80c41023465d36280e801564a69cbfce8ae85ff79b080e1913f6e90481fb8957",
-                "sha256:831ba20b660b39e39e5ac8603e8193f8fce1ee03a42c84ade89c36a251449d80",
-                "sha256:851fb2fe14036cfc1960d806628b80276af5424db09fe5c91c726890c8e6d943",
-                "sha256:8751f9c1442624da391bbd92bd4b072def6d7702a9390e4479f45c182392ff78",
-                "sha256:8b45d722046fea5a5694cba5d86f21f78f0052b40a4bbbbf60128ac55bfcc7b6",
-                "sha256:8b697774ea8273e3c0460cf0bba16cd85ca6c46dfe8b303211816d68c492e132",
-                "sha256:90146ab578931e0e2826ee39d0c948d0ea72734378f1898939d18bc9c823fcf9",
-                "sha256:9301cf1d7fc1ddf668d0abbe3e227fc9ab15bc036a31c247276012abb921b5ff",
-                "sha256:95bd3a998d8c68b76679f6b18f520904af5204f089beebb7b0301d97704634dd",
-                "sha256:968b0c737797c1809ec602e082cb63e9824ff2329275336bb88bd71591e94a90",
-                "sha256:97d984b1b2f574bc1bb58296d3c0b64b10e95e7026f8716ed6c0b86d4679843f",
-                "sha256:9e68ae9864d260b18f311b68d29134d8776d82e7f5d75ce898b40a88df9db30f",
-                "sha256:adecf6d02b1beab8d7c04bc36f22bb0e4c65a35eb0b4750b91693631d4081c70",
-                "sha256:af56229ea6527a849ac9fb154a059d7e32e77a8cba27e3e62a1e38d8808cb1a5",
-                "sha256:b324fa769577fc2c8f5efcd429cef5acbc17d63fe15ed16d6dcbac2c5eb00849",
-                "sha256:b5a07c4f29bf7cb0164664ef87e4aa25435dcc1f818d29842118b0ac1eb8e2b5",
-                "sha256:bad172aba822444b32eae54c2d5ab18cd7dee9814fd5c7ed026603b8cae2d05f",
-                "sha256:bdca18b94c404af6ae5533cd1bc310c4931f7ac97c148bbfd2cd4bdd62b96253",
-                "sha256:be24a5867b8e3b9dd5c241de359a9a5217698ff616ac2daa47713ba2ebe30ad1",
-                "sha256:be86a26415a8b6af02cd8d782e3a9ae3872140a057f1cadf0133de685185c02b",
-                "sha256:c66b7ff2527e18554030319b1376d81560ca0742c6e0b17ff1ee96624a5f1afd",
-                "sha256:c8398a1b1951aaa330269c35335ae69744be166e67e0ebd9869bdc09426f3871",
-                "sha256:cad9545f5801a125f162d09ec9b724b7ad9b6440151b89645241d0120e119dcc",
-                "sha256:cb6d161ae94fb35bb518b74bb06b7293299c15ba3bc099dccd6a5b7ae589aee3",
-                "sha256:d40682ac60b2a613d36d8d3a0cd14fbdf8e7e0618fbb40aa9fa7b796c9081584",
-                "sha256:d6128d431b8dfa888bf51c22a04d48bcb3d64431caf02b3cb943269f17fd2994",
-                "sha256:dbc466744a2db4b7ca05589f21ae1a35066afada2f803f92369f5877c100ef62",
-                "sha256:ddbef8b53cd16467fdbfa92a712eae46dd066aa19780681a2ce266e88fbc7165",
-                "sha256:e21cc00e4debe8f54c3ed7b9fcca540f46eee12762a9fa56feb8512fd9057161",
-                "sha256:eb52e826d16c09ef87132c6e360e1879c984f19a4f62d8a935345deac43f3c12",
-                "sha256:f0d9e7ba6a815a12c8575ba7887da4b72483e4cfc57179af10c9b937f3f9308f",
-                "sha256:f1e931d9a92f628858a50f5bdffdfcf839aebe388b82f9d2ccd5d22a38a789dc",
-                "sha256:f45808eda8b1d71308c5416ef3abe958f033fdbb356984fabbfc7887bed76b3f",
-                "sha256:f6d39e42a0aa888122d1beb8ec0d4ddfb6c6b45aecb5ba4013c27e2f28657765",
-                "sha256:fc34fdd458ff77a2a00e3c86f899911f6f269d393ca5675842a6e92eea565bae"
+                "sha256:0108358dab8c6b27ff6b985c2af4b12665c1bc659648284153ee501000f5c107",
+                "sha256:07bec1a1b22dacf718f2c0e71b49600bb6a31a88f06527dfd0b5aababe3fa3f7",
+                "sha256:0e8f482c44ccb5884bf3f638f29bea0f8dc68c97e38b2061769c4cb697f6140d",
+                "sha256:0ec91f1bad66f3ee8c6deb65fa1fe418e8ad803efedd69c35f3b5502f43bd1dc",
+                "sha256:0f14cffd32e9c4c73da66db97853a6aeceaac34acdc0fae9e5bbc9370281864c",
+                "sha256:15975747462ec49fdc863af906bab87c43b2491403ab37a6d88410635786b0f4",
+                "sha256:1724117bae69e091309ffb8255412c4651d3f6355560d9af312d547f6c5bc8b8",
+                "sha256:1a7c280185c4da99e0cc06c63bdf91f5b0b71deb70d8717f0ab870a43e376db8",
+                "sha256:1b7928bb7580736ffac5baf814097be342ba08d3cfdfb48e52773ec959572287",
+                "sha256:2032d9cb994ce3b4cba2b8dfae08c7e25bc14ba484c770d4d3be33c27de8c45b",
+                "sha256:20e7eeb1166087db636c06cae04a1ef59298627f56fb17da10528ab52a14c87f",
+                "sha256:216f5d7dbb67166759e59b0479bca82b8acf9bed6015b526b8eb10143fb08e77",
+                "sha256:28b119ba97129d3001673a697b7cce47fe6de1f7255d104c2f01108a5179a066",
+                "sha256:3104f4b084ad5d9c0cb87445cc8cfd96bba710bef4a66c2674910127044df209",
+                "sha256:3e6192dbcefaaa52ed81be88525a54a445f4b4fe2fffcae7fe40ebb58bd06bfd",
+                "sha256:42d4f97b9795a7aafa152a36fe2ad44549b83a743fd3e77011136def512e6c2a",
+                "sha256:44e706bac34e9f50779cb8c39f10b53a4d15aebb97235643d3112ac20bd577b4",
+                "sha256:47b11a729d61a47df56346283a4a800fa379ae6a85870d5a2e1e4956c828eedc",
+                "sha256:4854f9edc5208f63f0841c0c667260ae8d6846cfa233c479e29fdc85d42ebd58",
+                "sha256:48f721f070726cd2a6e44f3c33f8ee4b24188e4b816e6dd8ba542c8c3bb5b246",
+                "sha256:52afb0ac962963fff30cf1be775bc51ae083ef4c1e354266ab20e5382057dd62",
+                "sha256:54d8b9c5e288362ec8595c1d98666d36f2070fd0c2f76e2b3c60fbad9bd76227",
+                "sha256:5bd3d7dfd9cd058eb68d9a905dec854f86649f64d4ddf21f3ec289341386c44b",
+                "sha256:613010b5d17906c4367609e6f52e9a2595e35d5cc27d36ff3f1b6fa6e954d944",
+                "sha256:624321120f7e60336be8ec74a172ae7fba5c3ed5bf787cc85f7e9986c9e0ebc2",
+                "sha256:65c94410b5a8355cfcf12fd600a313efee46ce96a09e911ea92cf2acf6708804",
+                "sha256:6640f83df0ae4ae1104d4c62b77e9ef39be85ebe53f636388707d532bee2b7b8",
+                "sha256:687700f8371643916a1d2c61f3fdaa630407dd205c38afff936545d7b7466066",
+                "sha256:77c2713faf25a953c69cf0f723d1b7dd83827b0834e6c41e3fb3bbc6765914a1",
+                "sha256:78068e8678ca023594e4a0ab558905c1033b2d3e806a0ad9e3094e231e115a33",
+                "sha256:7a23ccc1083c260fa9685c93e3b170baba45aeed4b524deb3f426b0c40c11639",
+                "sha256:7abddb2bd5489d30ffeb4b93a428130886c171b4d355ccd226e83254fcb6b9ef",
+                "sha256:80093b595921eed1a2cead546a683b9e2ae7f4a4592bb2ab22f70d30174f003a",
+                "sha256:8242543c522d84d033fe79be04cb559b80d7eb98ad81b137ff7e0a9020f00ace",
+                "sha256:838812c65ed5f7c2bd11f7b098d2e5d01685a3f6d1f82849423b570bae698c00",
+                "sha256:83ea1a398f192957cb986d9206ce229efe0ee75e3c6635baff53ddf39bd718d5",
+                "sha256:8421aa8c9b45ea608c205db9e1c0c855c7e54d0e9c2c2f337ce024f6843cab3b",
+                "sha256:858375573c9225cc8e5b49bfac846a77b696b8d5e815711b8d4ba3141e6e8879",
+                "sha256:86de64468cad9c6d269f32a6390e210ca5ada568c7a55de8e681ca3b897bb340",
+                "sha256:87f7ac99b15270db8d53f28c3c7b968612993a90a5cf359da354efe96f5372b4",
+                "sha256:8bad8210ad4df68c44ff3685cca3cda448ee46e20d13edcff8909eba6ec01ca4",
+                "sha256:8bb4af15f305056e95ca1bd086239b9ebc6ad55e9f49076d27d80027f72752f6",
+                "sha256:8c78bfe20d4c890cb5580a3b9290f700c570e167d4cdcc55feec07030297a5e3",
+                "sha256:8f3f3154fde2b1ff3aa7b4f9326347ebc89c8ef425ca1db8f665175e6d3bd42f",
+                "sha256:94010bd61bc168c103a5b3b0f56ed3b616688192db7cd5b1d626e49f28ff51b3",
+                "sha256:941fab0073f0a54dc33d1a0460cb04e0d85893cb0c5e1476c785000f8b359409",
+                "sha256:9dca7c3956b03b7663fac4d150f5e6d4f6f38b2462c1e9afd83bcf7019f17913",
+                "sha256:a180dbd5ea5d47c2d3b716d5c19cc3fb162d1c8db93b21a1295d69585bfddac1",
+                "sha256:a2712aee7b3834ace51738c15d9ee152cc5a98dc7d57dd93300461b792ab7b43",
+                "sha256:a435ef8a3bd95c8a2d316d6e0ff70d0db524f6037411652803e118871d703333",
+                "sha256:abb756147314430bee5d10919b8493c0ccb109ddb7f5dfd2fcd7441266a25b75",
+                "sha256:abe6eb10122f0d746a0d510c2039ae8edb27bc9af29f6d1b05a66cc2401353ff",
+                "sha256:acbd0a6d61cc954b9f535daaa9ec26b0a60a0d4353c5f7c1438ebc88a359a47e",
+                "sha256:ae08ac90aa8fa14caafc7a6251bd218bf6dac518b7bff09caaa5e781119ba3f2",
+                "sha256:ae61446166983c663cee42c852ed63899e43e484abf080089f771df4b9d272ef",
+                "sha256:afe1f3bc486d0ce40abb0a0c9adb39aed3bbac36ebdc596487b0cceba55c21c1",
+                "sha256:b946da90dc2799bcafa682692c1d2139b2a96ec3c24fa9fc6f5b0da782675330",
+                "sha256:b947e264f0e77d30dcbccbb00f49f900b204b922eb0c3a9f0afd61aaa1cedc3d",
+                "sha256:bb5635c851eef3a7a54becde6da99485eecf7d068bd885ac8e6d173c4ecd68b0",
+                "sha256:bcbebd369493d68162cddb74a9c1fcebd139dfbb7ddb23d8f8e43e6c87bac3a6",
+                "sha256:c31805d2c8ade9b11feca4674eee2b9cce1fec3e8ddb7bbdd961a09dc76a80ea",
+                "sha256:c8840f064b1fb377cffd3efeaad2b190c14d4c8da02316dae07571252d20b31f",
+                "sha256:ccb94342d13e3bf3ffa6e62f95b5e3f0bc6bfa94558cb37f4b3d09d6feb536ff",
+                "sha256:d66689e840e75221b0b290b0befa86f059fb35e1ee6443bce51516d4d61b6b99",
+                "sha256:dabf1a05318d95b1537fd61d9330ef4313ea1216eea128a17615038859da3b3b",
+                "sha256:db03704b3506455d86ec72c3358a779e9b1d07b61220dfb43702b7b668edcd0d",
+                "sha256:de4217b9eb8b541cf2b7fde4401ce9d9a411cc0af85d410f9d6f4333f43640be",
+                "sha256:df0841f94928f8af9c7a1f0aaaffba1fb74607af023a152f59379c01c53aee58",
+                "sha256:dfb992dbcd88d8254471760879d48fb20836d91baa90f181c957122f9592b3dc",
+                "sha256:e7e66b4e403c2836ac74f26c4b65d8ac0ca1eef41dfcac2d013b7482befaad83",
+                "sha256:e8012bce6836d3f20a6c9599f81dfa945f433dab4dbd0c4917a6fb1f998ab33d",
+                "sha256:f01de4ec083daebf210531e2cca3bdb1608dbbbe00a9723e261d92087a1f6ebc",
+                "sha256:f0d945a85b70da97ae86113faf9f1b9294efe66bd4a5d6f82f2676d567338b66",
+                "sha256:fa0ae3275ef706c0309556061185dd0e4c4cd3b7d6f67ae617e4e677c7a41e2e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==25.1.0"
+            "version": "==24.0.1"
+
         },
         "pyzstd": {
             "hashes": [
@@ -2375,6 +2617,7 @@
                 "sha256:3351ad2feb51dcbb936defd47cab00d6f114214f224636503ed08298f30164c9",
                 "sha256:346f835e368e1051f8ea187ad9b49759cf6249c9ebf2f2a3861e435a568104b8",
                 "sha256:370b34a7c2f9c53cee494028daa5a7264690e1756a89c3855fd0be5ad298ec30",
+                "sha256:3a26df749589d898cd3253d2139eb85b867ddffc49286059c8bdb3cb9ce9b545",
                 "sha256:3bc0e7e2cccf78e562ab416daf68448b6552a5b6450a1ff3e15cabfc19254883",
                 "sha256:3f0fe2ef7ebc6e9b347585e414c4fefd32270ba8bdf9eb82496f3030cbdca465",
                 "sha256:3f72f310b10b730cddfb654006ae497e7706c81e6a7642d3da7fd2439df7d88d",
@@ -2410,6 +2653,7 @@
                 "sha256:7ac886e04f253960ae82e38ded8352085c61d78de99412d178a94ecf475b5e5f",
                 "sha256:7c420878726d677da7484f6021dbe7e1f9345a791b155de632c6ce36678fb621",
                 "sha256:836f1d85a4b5d3689d455aeb1dc6c42acb96aaf8e5282825c00ccf2545ad5630",
+                "sha256:84aa6eecba967bdac167451501dcaceec548d8b8c4ca7fa41ceda4dbfc279297",
                 "sha256:866ba6ce85f337fa1677516217b6f10fc25e19acb6e17a501d5822e66396bdd5",
                 "sha256:86e0e65e205793b337d62d9764700dfd02b5f83b01e26ad345736e7ac0554ebd",
                 "sha256:87a1a4ca93da414f3b6da8131e61aca6d48a4e837fb0b1cbde05ae9d13332317",
@@ -2439,6 +2683,8 @@
                 "sha256:c31f6dd5bd60688d51487a3f5e2ae29ed1948926e44d7a2316b193b083f80d5d",
                 "sha256:c41e5457f4de5d38a270bc44619873589bbe6fe251225deec583ed20199df0f3",
                 "sha256:c46e77c2ad614a0399503dc675d72436cbf6332a20d49a0e5bad03058d6cbfad",
+                "sha256:c9589cb79d4e401630481755c92b072aa7ba5505ec81dec865ef43932ec037e4",
+                "sha256:ca19213785f864781848e0216cba07e97f563f60a50bbc7885b54461d8c64873",
                 "sha256:cbfdde6c5768ffa5d2f14127bbc1d7c3c2d03c0ceaeb0736946197e06275ccc7",
                 "sha256:cd6a8d43a0c294918e3afb7e4b1d8c04d2e4c3ea9ddf05475fdaf366c7e5b3a6",
                 "sha256:cffaab46f9e04856dc3daa6097bfb3d3bea0b1771237e869c57b13f3dcc2c238",
@@ -2451,6 +2697,7 @@
                 "sha256:e789e19095b818f7126180b4387c0f01700c3ad2378a4e7649b2ddf4bf47ffbc",
                 "sha256:e79babb67b415aa54abb213897ceaa011515a5f3e146a2a97f4e6486b9743af4",
                 "sha256:e8f75e839ee253af60b03d9957182fdd069dfaebb62b4e999bd74016f4e120bb",
+                "sha256:e9934277abdddf9c733267e4dcc4886de8a3302d28f390237d447e215e8ce47d",
                 "sha256:ef3399e0544b46d31c2a8ff14ae1fb3c3571ae1153bbbc5ddf0d242c67bde624",
                 "sha256:f169e166774587227255f6ffe71f5b3303ea73cde0e2c6d52e53b9e12c03d787",
                 "sha256:f1d8b58f00137ccbe8b828a5ede92be3f0115cef75e6bed88d4d0bd1e7a0b1fc",
@@ -2513,11 +2760,11 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:47237742e990457f7512c7d27486394a9aadaf876cbfaa4be65b27b4f4d47c6b",
-                "sha256:c257b08a399b6c2f5a3510a50d28ab5dbc7bbde049bcaf954d43c446f83ab548"
+                "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf",
+                "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.30.0"
+            "version": "==0.30.2"
         },
         "requests": {
             "hashes": [
@@ -2562,6 +2809,22 @@
             "index": "pypi",
             "version": "==0.3.4"
         },
+        "rio-cogeo": {
+            "hashes": [
+                "sha256:33e4743a7c8837b0be3600590dfd91cac833a27d3459ed1a83ea59f1dc97e7cb",
+                "sha256:b2fddd7ce76fa8435ac2dda9195a95d06bffe469c31566869111b636a5894eaf"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.0.0"
+        },
+        "rioxarray": {
+            "hashes": [
+                "sha256:0cad24ad2c3c5ee181a0cfad2b8c2152a609b7eb118a3430034aec171e9cf14f",
+                "sha256:56eef711d9817d3c729c1a267c940e7dff66bfc874a0b24ed3604ea2f958dfb2"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.13.4"
+        },
         "rpds-py": {
             "hashes": [
                 "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f",
@@ -2907,7 +3170,7 @@
                 "sha256:f41915a4e1f059dfad614b187bc06021fefb5fc5255bfe63abf8247d2f7a646a",
                 "sha256:f96def94576f857abf58e031ce881b5a3fc25cbec64b2bc4824824a8a4367af9"
             ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==3.19.1"
         },
         "six": {
@@ -2915,7 +3178,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.16.0"
         },
         "sniffio": {
@@ -3002,6 +3265,14 @@
             "markers": "python_version < '3.11'",
             "version": "==2.0.1"
         },
+        "toolz": {
+            "hashes": [
+                "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f",
+                "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==0.12.0"
+        },
         "tornado": {
             "hashes": [
                 "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4",
@@ -3035,14 +3306,29 @@
             "markers": "python_version >= '3.7'",
             "version": "==5.9.0"
         },
+        "typeguard": {
+            "hashes": [
+                "sha256:94a8a8974eca6b980c0fd05a032f283914426ad9f2958b342075358e2744226a",
+                "sha256:b05a54bb0276eefd28880df42e004a71e699c8081fcb9d0536b2ceb01019f60c"
+            ],
+            "markers": "python_full_version >= '3.7.4'",
+            "version": "==4.1.0"
+        },
         "typing-extensions": {
             "hashes": [
                 "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
                 "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
-            "markers": "python_version < '3.10'",
+            "markers": "python_version >= '3.7'",
             "version": "==4.7.1"
         },
+        "typing-inspect": {
+            "hashes": [
+                "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f",
+                "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78"
+            ],
+            "version": "==0.9.0"
+        },
         "tzdata": {
             "hashes": [
                 "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a",
@@ -3111,6 +3397,87 @@
             "markers": "python_version >= '3.7'",
             "version": "==4.0.8"
         },
+        "wrapt": {
+            "hashes": [
+                "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0",
+                "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420",
+                "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a",
+                "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c",
+                "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079",
+                "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923",
+                "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f",
+                "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1",
+                "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8",
+                "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86",
+                "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0",
+                "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364",
+                "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e",
+                "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c",
+                "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e",
+                "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c",
+                "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727",
+                "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff",
+                "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e",
+                "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29",
+                "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7",
+                "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72",
+                "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475",
+                "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a",
+                "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317",
+                "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2",
+                "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd",
+                "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640",
+                "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98",
+                "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248",
+                "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e",
+                "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d",
+                "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec",
+                "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1",
+                "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e",
+                "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9",
+                "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92",
+                "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb",
+                "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094",
+                "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46",
+                "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29",
+                "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd",
+                "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705",
+                "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8",
+                "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975",
+                "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb",
+                "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e",
+                "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b",
+                "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418",
+                "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019",
+                "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1",
+                "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba",
+                "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6",
+                "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2",
+                "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3",
+                "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7",
+                "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752",
+                "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416",
+                "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f",
+                "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1",
+                "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc",
+                "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145",
+                "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee",
+                "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a",
+                "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7",
+                "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b",
+                "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653",
+                "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0",
+                "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90",
+                "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29",
+                "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6",
+                "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034",
+                "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09",
+                "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559",
+                "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.15.0"
+        },
         "xarray": {
             "hashes": [
                 "sha256:7bee552751ff1b29dab8b7715726e5ecb56691ac54593cf4881dff41978ce0cd",
@@ -3119,6 +3486,22 @@
             "index": "pypi",
             "version": "==2023.1.0"
         },
+        "xarray-spatial": {
+            "hashes": [
+                "sha256:8a356ac66a61ff9522453194db5d184e1eccee8ab890c7ec723bb48c6eafd392",
+                "sha256:fd7e3b447236f6e8f8c5110684a7ca50f97149c806f56ebf4ed1eeef1dd681ce"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.3.5"
+        },
+        "xyzservices": {
+            "hashes": [
+                "sha256:0ec928742227d6f5d4367ea7b457fcfed943429f4de2949b5b02a82cdf5569d6",
+                "sha256:88e9cbf22b31a2f9c1b242e2b18690f5c705f0e539c9bfd37a10399e1037731b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.7.0"
+        },
         "y-py": {
             "hashes": [
                 "sha256:035ba7ce31bb87bd7b5977eee71ee2ff71e54d347a35e2079362b1c23731dccd",
@@ -3247,11 +3630,11 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
-                "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
+                "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
+                "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.2"
+            "version": "==0.19.0"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -3289,7 +3672,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.39"
         },
         "ptyprocess": {
@@ -3297,7 +3680,6 @@
                 "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35",
                 "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"
             ],
-            "markers": "os_name != 'nt'",
             "version": "==0.7.0"
         },
         "pure-eval": {
@@ -3309,18 +3691,18 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
-                "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
+                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
+                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.15.1"
+            "version": "==2.16.1"
         },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.16.0"
         },
         "stack-data": {
@@ -3343,7 +3725,7 @@
                 "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
                 "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
-            "markers": "python_version < '3.10'",
+            "markers": "python_version >= '3.7'",
             "version": "==4.7.1"
         },
         "wcwidth": {
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 85407f01e..4ba66abbc 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.15.0 - 2023-08-08 - [PR##956](https://github.com/NOAA-OWP/inundation-mapping/pull/956)
+
+Integrating GVAL in to the evaluation of agreement maps and contingency tables.
+
+- `Dockerfile`: Add dependencies for GVAL
+- `Pipfile`: Add GVAL and update related dependencies
+- `Pipfile.lock`: Setup for Docker Image builds
+- `run_test_case.py`: Remove unused arguments and cleanup
+- `synthesize_test_cases.py`: Fix None comparisons and cleanup
+- `tools/shared_functions.py`: Add GVAL crosswalk function, add rework create_stats_from_raster, create and create_stats_from_contingency_table
+- `unit_tests/tools/inundate_gms_test.py`: Bug fix
+
+<br/><br/>
+
 ## v4.3.14.2 - 2023-08-08 - [PR#959](https://github.com/NOAA-OWP/inundation-mapping/pull/959)
 
 The enhancements in this PR include the new modules for pre-processing bathymetric data from the USACE eHydro dataset and integrating the missing hydraulic geometry into the HAND synthetic rating curves.
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 9f7b6c984..534209099 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -10,6 +10,7 @@
 from tools_shared_functions import compute_contingency_stats_from_rasters
 from utils.shared_functions import FIM_Helpers as fh
 
+
 class benchmark(object):
 
     AHPS_BENCHMARK_CATEGORIES = AHPS_BENCHMARK_CATEGORIES
@@ -310,27 +311,34 @@ def _inundate_and_compute(self,
             # FIM v3 and before
             else:
                 fh.vprint("Begin FIM v3 (or earlier) Inundation", verbose)
-                inundate_result = inundate(self.rem, self.catchments, self.catchment_poly, self.hydro_table, benchmark_flows,
-                    self.mask_type,hucs=self.hucs,hucs_layerName=self.hucs_layerName,
-                    subset_hucs=self.huc,num_workers=1,aggregate=False,
-                    inundation_raster=inundation_path,inundation_polygon=None,
-                    depths=None,out_raster_profile=None,out_vector_profile=None,
-                    quiet=True)
+                inundate_result = inundate(self.rem,
+                                           self.catchments,
+                                           self.catchment_poly,
+                                           self.hydro_table,
+                                           benchmark_flows,
+                                           self.mask_type,
+                                           hucs=self.hucs,
+                                           hucs_layerName=self.hucs_layerName,
+                                           subset_hucs=self.huc,
+                                           num_workers=1,
+                                           aggregate=False,
+                                           inundation_raster=inundation_path,
+                                           inundation_polygon=None,
+                                           depths=None,out_raster_profile=None,
+                                           out_vector_profile=None,
+                                           quiet=True)
                 if inundate_result != 0:
                     return inundate_result
 
         # Create contingency rasters and stats
         fh.vprint("Begin creating contingency rasters and stats", verbose)
         if os.path.isfile(predicted_raster_path):
-            compute_contingency_stats_from_rasters( predicted_raster_path,
-                                                    benchmark_rast,
-                                                    agreement_raster,
-                                                    stats_csv=stats_csv,
-                                                    stats_json=stats_json,
-                                                    mask_values=[],
-                                                    stats_modes_list=self.stats_modes_list,
-                                                    test_id=self.test_id,
-                                                    mask_dict=mask_dict_indiv )
+            compute_contingency_stats_from_rasters(predicted_raster_path,
+                                                   benchmark_rast,
+                                                   agreement_raster,
+                                                   stats_csv=stats_csv,
+                                                   stats_json=stats_json,
+                                                   mask_dict=mask_dict_indiv)
         return
 
 
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index cc8de54a3..3092b7420 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -84,7 +84,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
         test_cases_list = [d for d in os.listdir(benchmark_test_case_dir) if re.match('\d{8}_\w{3,7}', d)]
 
-        if benchmark_source in ['ble', 'ifc','ras2fim']:
+        if benchmark_source in ['ble', 'ifc', 'ras2fim']:
             magnitude_list = MAGNITUDE_DICT[benchmark_source]
             
             # Iterate through available test cases
@@ -377,6 +377,8 @@ def progress_bar_handler(executor_dict, verbose, desc):
     # Create a list of all test_cases for which we have validation data
     all_test_cases = test_case.list_all_test_cases(version = fim_version, archive = archive_results,
             benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
+
+    # print('all test cases', all_test_cases)
     
     # Make sure cycle-previous-files and a previous metric CSV have not been concurrently selected
     if prev_metrics_csv != None and pfiles == True:
@@ -384,7 +386,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
         sys.exit(1)
 
     # Check whether a previous metrics CSV has been provided and, if so, make sure the CSV exists
-    if prev_metrics_csv != None:
+    if prev_metrics_csv is not None:
         if not os.path.exists(prev_metrics_csv):
             print(f"Error: File does not exist at {prev_metrics_csv}")
             sys.exit(1)
@@ -408,6 +410,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
 
         # Loop through all test cases, build the alpha test arguments, and submit them to the process pool
         executor_dict = {}
+
         for test_case_class in all_test_cases:
             
             if not os.path.exists(test_case_class.fim_dir):
@@ -420,7 +423,7 @@ def progress_bar_handler(executor_dict, verbose, desc):
                                 'model': model,
                                 'mask_type': 'huc',
                                 'overwrite': overwrite,
-                                'verbose':gms_verbose if model == 'GMS' else verbose,
+                                'verbose': gms_verbose if model == 'GMS' else verbose,
                                 'gms_workers': job_number_branch
                                 }
 
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index d2068f9b7..bb755bc29 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -2,14 +2,14 @@
 
 import os
 import json
-import rasterio
-import pandas as pd
-import geopandas as gpd
-import requests
-import numpy as np
 import pathlib
-import time
 from pathlib import Path
+
+import requests
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+import rasterio
 import rasterio.shutil
 from rasterio.warp import calculate_default_transform, reproject, Resampling
 import rasterio.crs
@@ -18,6 +18,10 @@
 from shapely.geometry import Polygon
 from shapely.geometry import MultiPolygon
 from dotenv import load_dotenv
+import rioxarray as rxr
+import xarray as xr
+from geocube.api.core import make_geocube
+from gval import CatStats
 
 
 def get_env_paths():
@@ -80,58 +84,58 @@ def check_for_regression(stats_json_to_test, previous_version, previous_version_
     return difference_dict
 
 
-def compute_contingency_stats_from_rasters(predicted_raster_path, benchmark_raster_path, agreement_raster=None, stats_csv=None, stats_json=None, mask_values=None, stats_modes_list=['total_area'], test_id='', mask_dict={}):
+def compute_contingency_stats_from_rasters(predicted_raster_path: str, benchmark_raster_path: str,
+                                           agreement_raster: str = None, stats_csv: str = None,
+                                           stats_json: str = None, mask_dict: dict = {}):
     """
-    This function contains FIM-specific logic to prepare raster datasets for use in the generic get_contingency_table_from_binary_rasters() function.
-    This function also calls the generic compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
+    This function contains FIM-specific logic to prepare raster datasets for use in the generic
+    get_stats_table_from_binary_rasters() function. This function also calls the generic
+    compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
 
-    Args:
-        predicted_raster_path (str): The path to the predicted, or modeled, FIM extent raster.
-        benchmark_raster_path (str): The path to the benchmark, or truth, FIM extent raster.
-        agreement_raster (str): Optional. An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive, 3: True Positive.
-        stats_csv (str): Optional. Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
-        stats_json (str): Optional. Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary in other processes.
+    Parameters
+    ----------
+    predicted_raster_path: str
+        The path to the predicted, or modeled, FIM extent raster.
+    benchmark_raster_path: str
+        The path to the benchmark, or truth, FIM extent raster.
+    agreement_raster: str, optional
+        An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive,
+        3: True Positive.
+    stats_csv: str, optional
+        Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
+    stats_json: str, optional
+        Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary
+        in other processes.
+    mask_dict: dict
+        Dictionary with inclusionary and/or exclusionary masks asn options.
 
-    Returns:
-        stats_dictionary (dict): A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and statistic values are the values.
+    Returns
+    -------
+    dict
+        A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and
+        statistic values are the values.
     """
 
-    # Get cell size of benchmark raster.
-    raster = rasterio.open(predicted_raster_path)
-    t = raster.transform
-    cell_x = t[0]
-    cell_y = t[4]
-    cell_area = abs(cell_x*cell_y)
-
-    # Get contingency table from two rasters.
-    contingency_table_dictionary = get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster, mask_values=mask_values, mask_dict=mask_dict)
-
-    stats_dictionary = {}
+    # Get statistics table from two rasters.
+    stats_dictionary = get_stats_table_from_binary_rasters(benchmark_raster_path,
+                                                                 predicted_raster_path,
+                                                                 agreement_raster,
+                                                                 mask_dict=mask_dict)
 
-    for stats_mode in contingency_table_dictionary:
-        true_negatives = contingency_table_dictionary[stats_mode]['true_negatives']
-        false_negatives = contingency_table_dictionary[stats_mode]['false_negatives']
-        false_positives = contingency_table_dictionary[stats_mode]['false_positives']
-        true_positives = contingency_table_dictionary[stats_mode]['true_positives']
-        masked_count = contingency_table_dictionary[stats_mode]['masked_count']
-        file_handle = contingency_table_dictionary[stats_mode]['file_handle']
-
-        # Produce statistics from continency table and assign to dictionary. cell_area argument optional (defaults to None).
-        mode_stats_dictionary = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area, masked_count)
+    for stats_mode in stats_dictionary:
 
         # Write the mode_stats_dictionary to the stats_csv.
         if stats_csv != None:
-            stats_csv = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.csv')
-            df = pd.DataFrame.from_dict(mode_stats_dictionary, orient="index", columns=['value'])
+            stats_csv = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.csv')
+            df = pd.DataFrame.from_dict(stats_dictionary[stats_mode], orient="index", columns=['value'])
             df.to_csv(stats_csv)
 
         # Write the mode_stats_dictionary to the stats_json.
         if stats_json != None:
-            stats_json = os.path.join(os.path.split(stats_csv)[0], file_handle + '_stats.json')
+            stats_json = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.json')
             with open(stats_json, "w") as outfile:
-                json.dump(mode_stats_dictionary, outfile)
+                json.dump(stats_dictionary[stats_mode], outfile)
 
-        stats_dictionary.update({stats_mode: mode_stats_dictionary})
 
     return stats_dictionary
 
@@ -170,7 +174,8 @@ def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
     return archive_dictionary
 
 
-def compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=None, masked_count=None):
+def compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives,
+                                         cell_area=None, masked_count=None):
     """
     This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.
     Much of the calculations below were taken from older Python files. This is evident in the inconsistent use of case.
@@ -189,278 +194,178 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
 
     """
 
-    import numpy as np
-
-    total_population = true_negatives + false_negatives + false_positives + true_positives
+    vals, keys = CatStats.process_statistics(func_names="all", tp=true_positives, tn=true_negatives, fp=false_positives,
+                                             fn=false_negatives)
+    alt_keys = ['band', 'tn', 'fn', 'fp', 'tp']
+    alt_vals = [1, true_negatives, false_negatives, false_positives, true_positives]
 
-    # Basic stats.
-#    Percent_correct = ((true_positives + true_negatives) / total_population) * 100
-#    pod             = true_positives / (true_positives + false_negatives)
+    for k, v in zip(alt_keys[::-1], alt_vals[::-1]):
+        keys.insert(0, k)
+        vals.insert(0, v)
 
-    try:
-        FAR = false_positives / (true_positives + false_positives)
-    except ZeroDivisionError:
-        FAR = "NA"
+    metrics_table = pd.DataFrame({x: [y] for x, y in zip(keys, vals)})
 
-    try:
-        CSI = true_positives / (true_positives + false_positives + false_negatives)
-    except ZeroDivisionError:
-        CSI = "NA"
+    return cross_walk_gval_fim(metric_df=metrics_table,
+                               cell_area=cell_area,
+                               masked_count=masked_count)
 
-    try:
-        BIAS = (true_positives + false_positives) / (true_positives + false_negatives)
-    except ZeroDivisionError:
-        BIAS = "NA"
 
-    # Compute equitable threat score (ETS) / Gilbert Score.
-    try:
-        a_ref = ((true_positives + false_positives)*(true_positives + false_negatives)) / total_population
-        EQUITABLE_THREAT_SCORE = (true_positives - a_ref) / (true_positives - a_ref + false_positives + false_negatives)
-    except ZeroDivisionError:
-        EQUITABLE_THREAT_SCORE = "NA"
+def cross_walk_gval_fim(metric_df: pd.DataFrame, cell_area: int, masked_count: int) -> dict:
+    """
+    Crosswalks metrics made from GVAL to standard FIM names and conventions
+    
+    Parameters
+    ----------
+    metric_df: pd.DataFrame
+        Dataframe for getting
+    cell_area: int
+        Area in meters of squared resolution
+    masked_count: int
+        How many pixels are masked
 
-    if total_population == 0:
-        TP_perc, FP_perc, TN_perc, FN_perc = "NA", "NA", "NA", "NA"
-    else:
-        TP_perc = (true_positives / total_population) * 100
-        FP_perc = (false_positives / total_population) * 100
-        TN_perc = (true_negatives / total_population) * 100
-        FN_perc = (false_negatives / total_population) * 100
-
-    predPositive = true_positives + false_positives
-    predNegative = true_negatives + false_negatives
-    obsPositive = true_positives + false_negatives
-    obsNegative = true_negatives + false_positives
-
-    TP = float(true_positives)
-    TN = float(true_negatives)
-    FN = float(false_negatives)
-    FP = float(false_positives)
-    try:
-        MCC = (TP*TN - FP*FN)/ np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))
-    except ZeroDivisionError:
-        MCC = "NA"
-
-    if masked_count != None:
-        total_pop_and_mask_pop = total_population + masked_count
-        if total_pop_and_mask_pop == 0:
-            masked_perc = "NA"
-        else:
-            masked_perc = (masked_count / total_pop_and_mask_pop) * 100
-    else:
-        masked_perc = None
+    Returns
+    -------
+    dict
+        Dictionary of statistical metrics
+    """
 
-    # This checks if a cell_area has been provided, thus making areal calculations possible.
+    # Remove band entry
+    metric_df = metric_df.iloc[:, 1:]
+
+    # Dictionary to crosswalk column names
+    crosswalk = {'tn': 'true_negatives_count',
+                 'fn': 'false_negatives_count',
+                 'fp': 'false_positives_count',
+                 'tp': 'true_positives_count',
+                 'accuracy': 'ACC',
+                 'balanced_accuracy': 'Bal_ACC',
+                 'critical_success_index': 'CSI',
+                 'equitable_threat_score': 'EQUITABLE_THREAT_SCORE',
+                 'f_score': 'F1_SCORE',
+                 'false_discovery_rate': 'FAR',
+                 'false_negative_rate': 'PND',
+                 'false_omission_rate': 'FALSE_OMISSION_RATE',
+                 'false_positive_rate': 'FALSE_POSITIVE_RATE',
+                 'fowlkes_mallows_index': 'FOWLKES_MALLOW_INDEX',
+                 'matthews_correlation_coefficient': 'MCC',
+                 'negative_likelihood_ratio': 'NEGATIVE_LIKELIHOOD_RATIO',
+                 'negative_predictive_value': 'NPV',
+                 'overall_bias': 'BIAS',
+                 'positive_likelihood_ratio': 'POSITIVE_LIKELIHOOD_RATIO',
+                 'positive_predictive_value': 'PPV',
+                 'prevalence': 'PREVALENCE',
+                 'prevalence_threshold': 'PREVALENCE_THRESHOLD',
+                 'true_negative_rate': 'TNR',
+                 'true_positive_rate': 'TPR'
+                 }
+
+    metric_df.columns = [crosswalk[x] for x in metric_df.columns]
+
+    # Build
+    tn, fn, tp, fp = (metric_df['true_negatives_count'].values[0], metric_df['false_negatives_count'].values[0],
+                      metric_df['true_positives_count'].values[0], metric_df['false_positives_count'].values[0])
+    total_population = tn + fn + tp + fp
+    metric_df['contingency_tot_count'] = total_population
+
+    metric_df['TP_perc'] = (tp / total_population) * 100 if total_population > 0 else "NA"
+    metric_df['FP_perc'] = (fp / total_population) * 100 if total_population > 0 else "NA"
+    metric_df['TN_perc'] = (tn / total_population) * 100 if total_population > 0 else "NA"
+    metric_df['FN_perc'] = (fn / total_population) * 100 if total_population > 0 else "NA"
+
+    predPositive = tp + fp
+    predNegative = tn + fn
+    obsPositive = tp + fn
+    obsNegative = tn + fp
+
+    metric_df['cell_area_m2'] = cell_area
     sq_km_converter = 1000000
 
-    if cell_area != None:
-        TP_area = (true_positives * cell_area) / sq_km_converter
-        FP_area = (false_positives * cell_area) / sq_km_converter
-        TN_area = (true_negatives * cell_area) / sq_km_converter
-        FN_area = (false_negatives * cell_area) / sq_km_converter
-        area = (total_population * cell_area) / sq_km_converter
-
-        predPositive_area = (predPositive * cell_area) / sq_km_converter
-        predNegative_area = (predNegative * cell_area) / sq_km_converter
-        obsPositive_area =  (obsPositive * cell_area) / sq_km_converter
-        obsNegative_area =  (obsNegative * cell_area) / sq_km_converter
-        positiveDiff_area = predPositive_area - obsPositive_area
-
-        if masked_count != None:
-            masked_area = (masked_count * cell_area) / sq_km_converter
-        else:
-            masked_area = None
-
-    # If no cell_area is provided, then the contingeny tables are likely not derived from areal analysis.
-    else:
-        TP_area = None
-        FP_area = None
-        TN_area = None
-        FN_area = None
-        area = None
-
-        predPositive_area = None
-        predNegative_area = None
-        obsPositive_area =  None
-        obsNegative_area =  None
-        positiveDiff_area = None
-        MCC = None
-
-    if total_population == 0:
-        predPositive_perc, predNegative_perc, obsPositive_perc, obsNegative_perc , positiveDiff_perc = "NA", "NA", "NA", "NA", "NA"
-    else:
-        predPositive_perc = (predPositive / total_population) * 100
-        predNegative_perc = (predNegative / total_population) * 100
-        obsPositive_perc = (obsPositive / total_population) * 100
-        obsNegative_perc = (obsNegative / total_population) * 100
-
-        positiveDiff_perc = predPositive_perc - obsPositive_perc
-
-    if total_population == 0:
-        prevalence = "NA"
-    else:
-        prevalence = (true_positives + false_negatives) / total_population
-
-    try:
-        PPV = true_positives / predPositive
-    except ZeroDivisionError:
-        PPV = "NA"
-
-    try:
-        NPV = true_negatives / predNegative
-    except ZeroDivisionError:
-        NPV = "NA"
-
-    try:
-        TNR = true_negatives / obsNegative
-    except ZeroDivisionError:
-        TNR = "NA"
-
-    try:
-        TPR = true_positives / obsPositive
-
-    except ZeroDivisionError:
-        TPR = "NA"
-
-    try:
-        Bal_ACC = np.mean([TPR,TNR])
-    except TypeError:
-        Bal_ACC = "NA"
-
-    if total_population == 0:
-        ACC = "NA"
-    else:
-        ACC = (true_positives + true_negatives) / total_population
-
-    try:
-        F1_score = (2*true_positives) / (2*true_positives + false_positives + false_negatives)
-    except ZeroDivisionError:
-        F1_score = "NA"
-
-    if TPR == 'NA':
-        PND = 'NA'
-    else:
-        PND = 1.0 - TPR  # Probability Not Detected (PND)
-
-    stats_dictionary = {'true_negatives_count': int(true_negatives),
-                        'false_negatives_count': int(false_negatives),
-                        'true_positives_count': int(true_positives),
-                        'false_positives_count': int(false_positives),
-                        'contingency_tot_count': int(total_population),
-                        'cell_area_m2': cell_area,
-
-                        'TP_area_km2': TP_area,
-                        'FP_area_km2': FP_area,
-                        'TN_area_km2': TN_area,
-                        'FN_area_km2': FN_area,
-
-                        'contingency_tot_area_km2': area,
-                        'predPositive_area_km2': predPositive_area,
-                        'predNegative_area_km2': predNegative_area,
-                        'obsPositive_area_km2': obsPositive_area,
-                        'obsNegative_area_km2': obsNegative_area,
-                        'positiveDiff_area_km2': positiveDiff_area,
-
-                        'CSI': CSI,
-                        'FAR': FAR,
-                        'TPR': TPR,
-                        'TNR': TNR,
-                        'PND': PND,
-
-                        'PPV': PPV,
-                        'NPV': NPV,
-                        'ACC': ACC,
-                        'Bal_ACC': Bal_ACC,
-                        'MCC': MCC,
-                        'EQUITABLE_THREAT_SCORE': EQUITABLE_THREAT_SCORE,
-                        'PREVALENCE': prevalence,
-                        'BIAS': BIAS,
-                        'F1_SCORE': F1_score,
-
-                        'TP_perc': TP_perc,
-                        'FP_perc': FP_perc,
-                        'TN_perc': TN_perc,
-                        'FN_perc': FN_perc,
-                        'predPositive_perc': predPositive_perc,
-                        'predNegative_perc': predNegative_perc,
-                        'obsPositive_perc': obsPositive_perc,
-                        'obsNegative_perc': obsNegative_perc,
-                        'positiveDiff_perc': positiveDiff_perc,
-
-                        'masked_count': int(masked_count),
-                        'masked_perc': masked_perc,
-                        'masked_area_km2': masked_area,
-
-                        }
-
-    return stats_dictionary
-
-
-def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_raster_path, agreement_raster=None, mask_values=None, mask_dict={}):
+    # This checks if a cell_area has been provided, thus making areal calculations possible.
+    metric_df['TP_area_km2'] = (tp * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['FP_area_km2'] = (fp * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['TN_area_km2'] = (tn * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['FN_area_km2'] = (fn * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['contingency_tot_area_km2'] = (total_population * cell_area) \
+                                            / sq_km_converter if cell_area is not None else None
+
+    metric_df['predPositive_area_km2'] = (predPositive * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['predNegative_area_km2'] = (predNegative * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['obsPositive_area_km2'] = (obsPositive * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['obsNegative_area_km2'] = (obsNegative * cell_area) / sq_km_converter if cell_area is not None else None
+    metric_df['positiveDiff_area_km2'] = (metric_df['predPositive_area_km2'] - metric_df['obsPositive_area_km2'])[
+        0] if cell_area is not None else None
+
+    total_pop_and_mask_pop = total_population + masked_count if masked_count > 0 else None
+    metric_df['masked_count'] = masked_count if masked_count > 0 else 0
+    metric_df['masked_perc'] = (masked_count / total_pop_and_mask_pop) * 100 if masked_count > 0 else 0
+    metric_df['masked_area_km2'] = (masked_count * cell_area) / sq_km_converter if masked_count > 0 else 0
+    metric_df['predPositive_perc'] = (predPositive / total_population) * 100 if total_population > 0 else "NA"
+    metric_df['predNegative_perc'] = (predNegative / total_population) * 100 if total_population > 0 else "NA"
+    metric_df['obsPositive_perc'] = (obsPositive / total_population) * 100 if total_population > 0 else "NA"
+    metric_df['obsNegative_perc'] = (obsNegative / total_population) * 100 if total_population > 0 else "NA"
+    metric_df['positiveDiff_perc'] = metric_df['predPositive_perc'].values[0] - metric_df['obsPositive_perc'].values[
+        0] if total_population > 0 else "NA"
+
+    return {x: y for x, y in zip(metric_df.columns, metric_df.values[0])}
+
+
+def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
+                                        candidate_raster_path: str,
+                                        agreement_raster: str = None,
+                                        mask_dict: dict = {}):
     """
-    Produces contingency table from 2 rasters and returns it. Also exports an agreement raster classified as:
+    Produces categorical statistics table from 2 rasters and returns it. Also exports an agreement raster classified as:
         0: True Negatives
         1: False Negative
         2: False Positive
         3: True Positive
+        4: Masked
+        10: Nodata
 
-    Args:
-        benchmark_raster_path (str): Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
-        predicted_raster_path (str): Path to the predicted raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
+    Parameters
+    ----------
+    benchmark_raster_path: str
+        Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
+    candidate_raster_path: str
+        Path to the predicted raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
+    agreement_raster: str, default = None.
+        Path to save agreement raster/s
+    mask_dict : dict, default = {}
+        Dictionary with inclusionary and/or exclusionary masks asn options.
 
-    Returns:
-        contingency_table_dictionary (dict): A Python dictionary of a contingency table. Key/value pair formatted as:
-                                            {true_negatives: int, false_negatives: int, false_positives: int, true_positives: int}
+    Returns
+    -------
+    dict
+        A Python dictionary of a contingency table. Key/value pair formatted as:
+        {true_negatives: int, false_negatives: int, false_positives: int, true_positives: int}
 
     """
-    from rasterio.warp import reproject, Resampling
-    import rasterio
-    import numpy as np
-    import os
-    import rasterio.mask
-    import geopandas as gpd
-    from shapely.geometry import box
-
-#    print("-----> Evaluating performance across the total area...")
-    # Load rasters.
-    benchmark_src = rasterio.open(benchmark_raster_path)
-    predicted_src = rasterio.open(predicted_raster_path)
-    predicted_array = predicted_src.read(1)
-
-    benchmark_array_original = benchmark_src.read(1)
-
-    if benchmark_array_original.shape != predicted_array.shape:
-        benchmark_array = np.empty(predicted_array.shape, dtype=np.int8)
-
-        reproject(benchmark_array_original,
-              destination = benchmark_array,
-              src_transform = benchmark_src.transform,
-              src_crs = benchmark_src.crs,
-              src_nodata = benchmark_src.nodata,
-              dst_transform = predicted_src.transform,
-              dst_crs = predicted_src.crs,
-              dst_nodata = benchmark_src.nodata,
-              dst_resolution = predicted_src.res,
-              resampling = Resampling.nearest)
-
-    predicted_array_raw = predicted_src.read(1)
-
-    # Align the benchmark domain to the modeled domain.
-    benchmark_array = np.where(predicted_array==predicted_src.nodata, 10, benchmark_array)
-
-    # Ensure zeros and ones for binary comparison. Assume that positive values mean flooding and 0 or negative values mean dry.
-    predicted_array = np.where(predicted_array==predicted_src.nodata, 10, predicted_array)  # Reclassify NoData to 10
-    predicted_array = np.where(predicted_array<0, 0, predicted_array)
-    predicted_array = np.where(predicted_array>0, 1, predicted_array)
-
-    benchmark_array = np.where(benchmark_array==benchmark_src.nodata, 10, benchmark_array)  # Reclassify NoData to 10
 
-    agreement_array = np.add(benchmark_array, 2*predicted_array)
-    agreement_array = np.where(agreement_array>4, 10, agreement_array)
+    # Load benchmark and candidate data
+    benchmark_raster = rxr.open_rasterio(benchmark_raster_path, mask_and_scale=True)
+    cell_area = np.abs(np.prod(benchmark_raster.rio.resolution()))
+    candidate_raster = rxr.open_rasterio(candidate_raster_path, mask_and_scale=True)
+    candidate_raster.data = xr.where(candidate_raster >= 0, 1, candidate_raster)
+    candidate_raster.data = xr.where(candidate_raster < 0, 0, candidate_raster)
+
+    pairing_dictionary = {
+        (0, 0): 0,
+        (0, 1): 1,
+        (0, np.nan): np.nan,
+        (1, 0): 2,
+        (1, 1): 3,
+        (1, np.nan): np.nan,
+        (4, 0): 4,
+        (4, 1): 4,
+        (4, np.nan): np.nan,
+        (np.nan, 0): np.nan,
+        (np.nan, 1): np.nan,
+        (np.nan, np.nan): np.nan
+    }
 
-    del benchmark_src, benchmark_array, predicted_array, predicted_array_raw
 
     # Loop through exclusion masks and mask the agreement_array.
+    rasterized_mask_list = []
     if mask_dict != {}:
         for poly_layer in mask_dict:
 
@@ -471,52 +376,49 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                 poly_path = mask_dict[poly_layer]['path']
                 buffer_val = mask_dict[poly_layer]['buffer']
 
-                reference = predicted_src
+                # Read mask bounds with candidate boundary box
+                poly_all = gpd.read_file(poly_path, bbox=candidate_raster.rio.bounds())
 
-                bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
-                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).
-                poly_all = gpd.read_file(poly_path, bbox = bounding_box)
-
-                # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
+                # Make sure features are present in bounding box area before projecting.
+                # Continue to next layer if features are absent.
                 if poly_all.empty:
+                    del poly_all
                     continue
 
-#                print("-----> Masking at " + poly_layer + "...")
-                #Project layer to reference crs.
-                poly_all_proj = poly_all.to_crs(reference.crs)
-                # check if there are any lakes within our reference raster extent.
-                if poly_all_proj.empty:
-                    #If no features within reference raster extent, create a zero array of same shape as reference raster.
-                    poly_mask = np.zeros(reference.shape)
-                else:
-                    #Check if a buffer value is passed to function.
-                    if buffer_val is None:
-                        #If features are present and no buffer is passed, assign geometry to variable.
-                        geometry = poly_all_proj.geometry
-                    else:
-                        #If  features are present and a buffer is passed, assign buffered geometry to variable.
-                        geometry = poly_all_proj.buffer(buffer_val)
+                # Project layer to reference crs.
+                poly_all_proj = poly_all.to_crs(candidate_raster.rio.crs)
+
+                # Buffer if buffer val exists
+                poly_all_proj = poly_all_proj.buffer(buffer_val) if buffer_val else poly_all_proj
+
+                poly_all_proj['mask'] = 4
+
+                rasterized_mask_list.append(make_geocube(poly_all_proj, ['mask'], like=candidate_raster))
 
-                    #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
-                    in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
-                    #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
-                    poly_mask = np.where(in_poly == True, 1,0)
+                del poly_all, poly_all_proj
 
-                    # Perform mask.
-                    masked_agreement_array = np.where(poly_mask == 1, 4, agreement_array)
+    og_data = candidate_raster.data
+    if len(rasterized_mask_list) > 0:
+        all_masks = xr.merge(rasterized_mask_list).to_array()
+        # Hold on to original data
 
-                    # Get rid of masked values outside of the modeled domain.
-                    agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
+        candidate_raster.data = xr.where((all_masks.data == 4) & (candidate_raster.isnull() == 0), 4, candidate_raster)
 
-    contingency_table_dictionary = {}  # Initialize empty dictionary.
+    stats_table_dictionary = {}  # Initialize empty dictionary.
+
+    (agreement_map,
+     crosstab_table,
+     metrics_table) = candidate_raster.gval.categorical_compare(benchmark_map=benchmark_raster,
+                                                                positive_categories=[1],
+                                                                target_map="candidate",
+                                                                negative_categories=[0],
+                                                                comparison_function='pairing_dict',
+                                                                pairing_dict=pairing_dictionary)
 
     # Only write the agreement raster if user-specified.
     if agreement_raster != None:
-        with rasterio.Env():
-            profile = predicted_src.profile
-            profile.update(nodata=10)
-            with rasterio.open(agreement_raster, 'w', **profile) as dst:
-                dst.write(agreement_array, 1)
+        agreement_map = agreement_map.rio.write_nodata(10, encoded=True)
+        agreement_map.rio.to_raster(agreement_raster, dtype=np.int32, driver="COG")
 
         # Write legend text file
         legend_txt = os.path.join(os.path.split(agreement_raster)[0], 'read_me.txt')
@@ -531,18 +433,16 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
             f.write("%s\n" % '1: False Negative')
             f.write("%s\n" % '2: False Positive')
             f.write("%s\n" % '3: True Positive')
-            f.write("%s\n" % '4: Masked area (excluded from contingency table analysis). Mask layers: {mask_dict}'.format(mask_dict=mask_dict))
+            f.write("%s\n" % '4: Masked area (excluded from contingency table analysis). '
+                             'Mask layers: {mask_dict}'.format(mask_dict=mask_dict))
             f.write("%s\n" % 'Results produced at: {current_time}'.format(current_time=current_time))
 
     # Store summed pixel counts in dictionary.
-    contingency_table_dictionary.update({'total_area':{'true_negatives': int((agreement_array == 0).sum()),
-                                                      'false_negatives': int((agreement_array == 1).sum()),
-                                                      'false_positives': int((agreement_array == 2).sum()),
-                                                      'true_positives': int((agreement_array == 3).sum()),
-                                                      'masked_count': int((agreement_array == 4).sum()),
-                                                      'file_handle': 'total_area'
+    stats_table_dictionary.update({'total_area': cross_walk_gval_fim(metric_df=metrics_table,
+                                                                     cell_area=cell_area,
+                                                                     masked_count=np.sum(agreement_map.data == 4))})
 
-                                                      }})
+    del agreement_map, crosstab_table, metrics_table, all_masks, rasterized_mask_list
 
     # After agreement_array is masked with default mask layers, check for inclusion masks in mask_dict.
     if mask_dict != {}:
@@ -555,67 +455,57 @@ def get_contingency_table_from_binary_rasters(benchmark_raster_path, predicted_r
                 poly_path = mask_dict[poly_layer]['path']
                 buffer_val = mask_dict[poly_layer]['buffer']
 
-                reference = predicted_src
-
-                bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
-                #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).
-                poly_all = gpd.read_file(poly_path, bbox = bounding_box)
+                # Read mask bounds with candidate boundary box
+                poly_all = gpd.read_file(poly_path, bbox=candidate_raster.rio.bounds())
 
-                # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
+                # Make sure features are present in bounding box area before projecting.
+                # Continue to next layer if features are absent.
                 if poly_all.empty:
+                    del poly_all
                     continue
 
-#                print("-----> Evaluating performance at " + poly_layer + "...")
-                #Project layer to reference crs.
-                poly_all_proj = poly_all.to_crs(reference.crs)
-                # check if there are any lakes within our reference raster extent.
-                if poly_all_proj.empty:
-                    #If no features within reference raster extent, create a zero array of same shape as reference raster.
-                    poly_mask = np.zeros(reference.shape)
-                else:
-                    #Check if a buffer value is passed to function.
-                    if buffer_val is None:
-                        #If features are present and no buffer is passed, assign geometry to variable.
-                        geometry = poly_all_proj.geometry
-                    else:
-                        #If  features are present and a buffer is passed, assign buffered geometry to variable.
-                        geometry = poly_all_proj.buffer(buffer_val)
-
-                    #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
-                    in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
-                    #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
-                    poly_mask = np.where(in_poly == True, 1, 0)
+                poly_all_proj = poly_all.to_crs(candidate_raster.rio.crs)
 
-                    # Perform mask.
-                    masked_agreement_array = np.where(poly_mask == 0, 4, agreement_array)  # Changed to poly_mask == 0
+                # Buffer if buffer val exists
+                poly_all_proj = poly_all_proj.buffer(buffer_val) if buffer_val else poly_all_proj
 
-                    # Get rid of masked values outside of the modeled domain.
-                    temp_agreement_array = np.where(agreement_array == 10, 10, masked_agreement_array)
+                poly_all_proj['mask'] = 4
 
-                    if buffer_val == None:  # The buffer used is added to filename, and 0 is easier to read than None.
-                        buffer_val = 0
+                mask = make_geocube(poly_all_proj, ['mask'], like=candidate_raster).to_array()
+                candidate_raster.data = og_data
+                candidate_raster.data = xr.where((mask.data != 4) & (candidate_raster.isnull() == 0), 4, candidate_raster)
 
-                    poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
+                poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
 
+                # Do analysis on inclusion masked area
+                (agreement_map,
+                 crosstab_table,
+                 metrics_table) = candidate_raster.gval.categorical_compare(benchmark_map=benchmark_raster,
+                                                                            positive_categories=[1],
+                                                                            target_map="candidate",
+                                                                            negative_categories=[0],
+                                                                            comparison_function='pairing_dict',
+                                                                            pairing_dict=pairing_dictionary)
+                if agreement_raster:
                     # Write the layer_agreement_raster.
-                    layer_agreement_raster = os.path.join(os.path.split(agreement_raster)[0], poly_handle + '_agreement.tif')
-                    with rasterio.Env():
-                        profile = predicted_src.profile
-                        profile.update(nodata=10)
-                        with rasterio.open(layer_agreement_raster, 'w', **profile) as dst:
-                            dst.write(temp_agreement_array, 1)
-
-
-                    # Store summed pixel counts in dictionary.
-                    contingency_table_dictionary.update({poly_handle:{'true_negatives': int((temp_agreement_array == 0).sum()),
-                                                                     'false_negatives': int((temp_agreement_array == 1).sum()),
-                                                                     'false_positives': int((temp_agreement_array == 2).sum()),
-                                                                     'true_positives': int((temp_agreement_array == 3).sum()),
-                                                                     'masked_count': int((temp_agreement_array == 4).sum()),
-                                                                     'file_handle': poly_handle
-                                                                      }})
-
-    return contingency_table_dictionary
+                    layer_agreement_raster = os.path.join(os.path.split(agreement_raster)[0],
+                                                          poly_handle + '_agreement.tif')
+                    agreement_map = agreement_map.rio.write_nodata(10, encoded=True)
+                    agreement_map.rio.to_raster(layer_agreement_raster, dtype=np.int32, driver="COG")
+
+                # Update stats table dictionary
+                stats_table_dictionary.update({poly_handle: cross_walk_gval_fim(metric_df=metrics_table,
+                                                                                cell_area=cell_area,
+                                                                                masked_count=np.sum(
+                                                                                    agreement_map.data == 4
+                                                                                ))})
+
+                del poly_all, poly_all_proj, agreement_map, metrics_table, crosstab_table
+
+    del candidate_raster, benchmark_raster, og_data
+
+    return stats_table_dictionary
+
 ########################################################################
 ########################################################################
 #Functions related to categorical fim and ahps evaluation
diff --git a/unit_tests/tools/inundate_gms_test.py b/unit_tests/tools/inundate_gms_test.py
index a44b79858..cdb8b77e8 100644
--- a/unit_tests/tools/inundate_gms_test.py
+++ b/unit_tests/tools/inundate_gms_test.py
@@ -43,8 +43,9 @@ def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self
         for file in os.listdir(params["hydrofabric_dir"]):
             if file.endswith(".tif"):
                 os.remove(os.path.join(params["hydrofabric_dir"], file))
-                
-        os.remove(params["output_fileNames"])
+
+        if os.path.isfile(params["output_fileNames"]):
+            os.remove(params["output_fileNames"])
                 
         # Test the Inundate_gms function
         output_fileNames_df = src.Inundate_gms(hydrofabric_dir = params["hydrofabric_dir"],

From 090a44ab5741fc38f2eaefcba37aa8682a3f74fd Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 11 Aug 2023 10:00:13 -0700
Subject: [PATCH 264/359] v4.3.15.1 Update BLE benchmark data (#960)

---
 data/ble/ble_benchmark/Dockerfile             |   9 +
 data/ble/ble_benchmark/Pipfile                |  15 +
 data/ble/ble_benchmark/Pipfile.lock           | 343 ++++++++++++++++++
 data/ble/ble_benchmark/README.md              |  59 +++
 .../ble/ble_benchmark/create_ble_benchmark.py | 221 +++++++++++
 .../create_flow_forecast_file.py              | 191 ++++++++++
 .../ble_benchmark}/preprocess_benchmark.py    |   2 +-
 docs/CHANGELOG.md                             |  18 +
 tools/create_flow_forecast_file.py            |  98 -----
 9 files changed, 857 insertions(+), 99 deletions(-)
 create mode 100644 data/ble/ble_benchmark/Dockerfile
 create mode 100755 data/ble/ble_benchmark/Pipfile
 create mode 100644 data/ble/ble_benchmark/Pipfile.lock
 create mode 100644 data/ble/ble_benchmark/README.md
 create mode 100644 data/ble/ble_benchmark/create_ble_benchmark.py
 create mode 100755 data/ble/ble_benchmark/create_flow_forecast_file.py
 rename {tools => data/ble/ble_benchmark}/preprocess_benchmark.py (99%)
 delete mode 100755 tools/create_flow_forecast_file.py

diff --git a/data/ble/ble_benchmark/Dockerfile b/data/ble/ble_benchmark/Dockerfile
new file mode 100644
index 000000000..bb35c7c88
--- /dev/null
+++ b/data/ble/ble_benchmark/Dockerfile
@@ -0,0 +1,9 @@
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.0
+
+RUN apt-get update && apt-get install -y \
+    python3-pip
+
+## install python 3 modules ##
+COPY Pipfile .
+COPY Pipfile.lock .
+RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
diff --git a/data/ble/ble_benchmark/Pipfile b/data/ble/ble_benchmark/Pipfile
new file mode 100755
index 000000000..edb4c575e
--- /dev/null
+++ b/data/ble/ble_benchmark/Pipfile
@@ -0,0 +1,15 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pandas = "*"
+rasterio = "*"
+geopandas = "*"
+openpyxl = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"
diff --git a/data/ble/ble_benchmark/Pipfile.lock b/data/ble/ble_benchmark/Pipfile.lock
new file mode 100644
index 000000000..f4df3ba79
--- /dev/null
+++ b/data/ble/ble_benchmark/Pipfile.lock
@@ -0,0 +1,343 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "43b6d8b600067ac2becfb2d00935884984da473d94e96a01d4943d31d3091e80"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.10"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "affine": {
+            "hashes": [
+                "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92",
+                "sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.4.0"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
+                "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==23.1.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7",
+                "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2023.5.7"
+        },
+        "click": {
+            "hashes": [
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.3"
+        },
+        "click-plugins": {
+            "hashes": [
+                "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b",
+                "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"
+            ],
+            "version": "==1.1.1"
+        },
+        "cligj": {
+            "hashes": [
+                "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
+                "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "version": "==0.7.2"
+        },
+        "et-xmlfile": {
+            "hashes": [
+                "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c",
+                "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.1.0"
+        },
+        "fiona": {
+            "hashes": [
+                "sha256:1a585002a6385cc8ab0f66ddf3caf18711f531901906abd011a67a0cc89ab7b0",
+                "sha256:1da8b954f6f222c3c782bc285586ea8dd9d7e55e1bc7861da9cd772bca671660",
+                "sha256:285f3dd4f96aa0a3955ed469f0543375b20989731b2dddc85124453f11ac62bc",
+                "sha256:2c2c7b09eecee3bb074ef8aa518cd6ab30eb663c6fdd0eff3c88d454a9746eaa",
+                "sha256:39baf11ff0e4318397e2b2197de427b4eebdc49d4a9a7c1366f8a7ed682978a4",
+                "sha256:450561d308d3ce7c7e30294822b1de3f4f942033b703ddd4a91a7f7f5f506ca0",
+                "sha256:5679d3f7e0d513035eb72e59527bb90486859af4405755dfc739138633106120",
+                "sha256:71b023ef5248ebfa5524e7a875033f7db3bbfaf634b1b5c1ae36958d1eb82083",
+                "sha256:74511d3755695d75cea0f4ff6f5e0c6c5d5be8e0d46dafff124c6a219e99b1eb",
+                "sha256:7bfb1f49e0e53f6cd7ad64ae809d72646266b37a7b9881205977408b443a8d79",
+                "sha256:a670ea4262cb9140445bcfc97cbfd2f508a058be342f4a97e966b8ce7696601f",
+                "sha256:b0387cae39e27f338fd948b3b50b6e6ce198cc4cec257fc91660849697c69dc3",
+                "sha256:b633a2e550e083805c638d2ab8059c283ca112aaea8241e170c012d2ee0aa905",
+                "sha256:c1faa625d5202b8403471bbc9f9c96b1bf9099cfcb0ee02a80a3641d3d02383e",
+                "sha256:c671d8832287cda397621d79c5a635d52e4631f33a8f0e6fdc732a79a93cb96c",
+                "sha256:d6483a20037db2209c8e9a0c6f1e552f807d03c8f42ed0c865ab500945a37c4d",
+                "sha256:d93c993265f6378b23f47708c83bddb3377ca6814a1f0b5a0ae0bee9c8d72cf8",
+                "sha256:dbe158947099a83ad16f9acd3a21f50ff01114c64e2de67805e382e6b6e0083a",
+                "sha256:ea7c44c15b3a653452b9b3173181490b7afc5f153b0473c145c43c0fbf90448b",
+                "sha256:f5da66b723a876142937e683431bbaa5c3d81bb2ed3ec98941271bc99b7f8cd0"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.9.4.post1"
+        },
+        "geopandas": {
+            "hashes": [
+                "sha256:101cfd0de54bcf9e287a55b5ea17ebe0db53a5e25a28bacf100143d0507cabd9",
+                "sha256:e5b56d9c20800c77bcc0c914db3f27447a37b23b2cd892be543f5001a694a968"
+            ],
+            "index": "pypi",
+            "version": "==0.13.2"
+        },
+        "numpy": {
+            "hashes": [
+                "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187",
+                "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812",
+                "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7",
+                "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4",
+                "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6",
+                "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0",
+                "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4",
+                "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570",
+                "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4",
+                "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f",
+                "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80",
+                "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289",
+                "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385",
+                "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078",
+                "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c",
+                "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463",
+                "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3",
+                "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950",
+                "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155",
+                "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7",
+                "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c",
+                "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096",
+                "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17",
+                "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf",
+                "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4",
+                "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02",
+                "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c",
+                "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.24.3"
+        },
+        "openpyxl": {
+            "hashes": [
+                "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184",
+                "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"
+            ],
+            "index": "pypi",
+            "version": "==3.1.2"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
+                "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==23.1"
+        },
+        "pandas": {
+            "hashes": [
+                "sha256:02755de164da6827764ceb3bbc5f64b35cb12394b1024fdf88704d0fa06e0e2f",
+                "sha256:0a1e0576611641acde15c2322228d138258f236d14b749ad9af498ab69089e2d",
+                "sha256:1eb09a242184092f424b2edd06eb2b99d06dc07eeddff9929e8667d4ed44e181",
+                "sha256:30a89d0fec4263ccbf96f68592fd668939481854d2ff9da709d32a047689393b",
+                "sha256:50e451932b3011b61d2961b4185382c92cc8c6ee4658dcd4f320687bb2d000ee",
+                "sha256:51a93d422fbb1bd04b67639ba4b5368dffc26923f3ea32a275d2cc450f1d1c86",
+                "sha256:598e9020d85a8cdbaa1815eb325a91cfff2bb2b23c1442549b8a3668e36f0f77",
+                "sha256:66d00300f188fa5de73f92d5725ced162488f6dc6ad4cecfe4144ca29debe3b8",
+                "sha256:69167693cb8f9b3fc060956a5d0a0a8dbfed5f980d9fd2c306fb5b9c855c814c",
+                "sha256:6d6d10c2142d11d40d6e6c0a190b1f89f525bcf85564707e31b0a39e3b398e08",
+                "sha256:713f2f70abcdade1ddd68fc91577cb090b3544b07ceba78a12f799355a13ee44",
+                "sha256:7376e13d28eb16752c398ca1d36ccfe52bf7e887067af9a0474de6331dd948d2",
+                "sha256:77550c8909ebc23e56a89f91b40ad01b50c42cfbfab49b3393694a50549295ea",
+                "sha256:7b21cb72958fc49ad757685db1919021d99650d7aaba676576c9e88d3889d456",
+                "sha256:9ebb9f1c22ddb828e7fd017ea265a59d80461d5a79154b49a4207bd17514d122",
+                "sha256:a18e5c72b989ff0f7197707ceddc99828320d0ca22ab50dd1b9e37db45b010c0",
+                "sha256:a6b5f14cd24a2ed06e14255ff40fe2ea0cfaef79a8dd68069b7ace74bd6acbba",
+                "sha256:b42b120458636a981077cfcfa8568c031b3e8709701315e2bfa866324a83efa8",
+                "sha256:c4af689352c4fe3d75b2834933ee9d0ccdbf5d7a8a7264f0ce9524e877820c08",
+                "sha256:c7319b6e68de14e6209460f72a8d1ef13c09fb3d3ef6c37c1e65b35d50b5c145",
+                "sha256:cf3f0c361a4270185baa89ec7ab92ecaa355fe783791457077473f974f654df5",
+                "sha256:dd46bde7309088481b1cf9c58e3f0e204b9ff9e3244f441accd220dd3365ce7c",
+                "sha256:dd5476b6c3fe410ee95926873f377b856dbc4e81a9c605a0dc05aaccc6a7c6c6",
+                "sha256:e69140bc2d29a8556f55445c15f5794490852af3de0f609a24003ef174528b79",
+                "sha256:f908a77cbeef9bbd646bd4b81214cbef9ac3dda4181d5092a4aa9797d1bc7774"
+            ],
+            "index": "pypi",
+            "version": "==2.0.2"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
+                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
+            ],
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.0.9"
+        },
+        "pyproj": {
+            "hashes": [
+                "sha256:00fab048596c17572fa8980014ef117dbb2a445e6f7ba3b9ddfcc683efc598e7",
+                "sha256:08dfc5c9533c78a97afae9d53b99b810a4a8f97c3be9eb2b8f323b726c736403",
+                "sha256:1283d3c1960edbb74828f5f3405b27578a9a27f7766ab6a3956f4bd851f08239",
+                "sha256:137a07404f937f264b11b7130cd4cfa00002dbe4333b222e8056db84849c2ea4",
+                "sha256:18a8bdb87aeb41b60a2e91d32f623227de3569fb83b4c64b174c3a7c5b0ed3ae",
+                "sha256:23787460fab85ba2f857ee60ffb2e8e21fd9bd5db9833c51c1c05b2a6d9f0be5",
+                "sha256:2799499a4045e4fb73e44c31bdacab0593a253a7a4b6baae6fdd27d604cf9bc2",
+                "sha256:3a4d2d438b007cb1f8d5f6f308d53d7ff9a2508cff8f9da6e2a93b76ffd98aaf",
+                "sha256:4d8a9773503085eada59b6892c96ddf686ab8cf64cfdc18ad744d13ee76dfa6f",
+                "sha256:557e6592855111c84eda176ddf6b130f55d5e2b9cb1c017b8c91b69f37f474f5",
+                "sha256:595376e4d3bb72b7dceeccbce0f4c43053d47561f17a1ad0224407e9980ee849",
+                "sha256:78276c6b0c831255c97c56dff7313a3571f327a284d8ac63d6a56437a72ed0e0",
+                "sha256:830e6de7cfe43853967afee5ef908dfd5aa72d1ec12af9b9e3fecc179886e346",
+                "sha256:8fbac2eb9a0e425d7d6b7c6f4ebacd675cf3bdef0c59887057b8b4b0374e7c12",
+                "sha256:95120d65cbc5983dfd877076f28dbc18b9b329cbee38ca6e217bb7a5a043c099",
+                "sha256:9de1aab71234bfd3fd648a1152519b5ee152c43113d7d8ea52590a0140129501",
+                "sha256:a5b111865b3f0f8b77b3983f2fbe4dd6248fc09d3730295949977c8dcd988062",
+                "sha256:ba5e7c8ddd6ed5a3f9fcf95ea80ba44c931913723de2ece841c94bb38b200c4a",
+                "sha256:d7f6cd045df29aae960391dfe06a575c110af598f1dea5add8be6ca42332b0f5",
+                "sha256:de6288b6ceabdeeac01abf627c74414822d322d8f55dc8efe4d29dedd27c5719",
+                "sha256:dfe392dfc0eba2248dc08c976a72f52ff9da2bddfddfd9ff5dcf18e8e88200c7",
+                "sha256:e342b3010b2b20134671564ff9a8c476e5e512bf589477480aded1a5813af7c8",
+                "sha256:e427ccdbb1763872416549bdfa9fa1f5f169054653c4daf674e71480cc39cf11",
+                "sha256:e600f6a2771d3b41aeb2cc1efd96771ae9a01451013da1dd48ff272e7c6e34ef",
+                "sha256:f04f6297c615c3b17f835df2556ac8fb9b4f51f281e960437eaf0cd80e7ae26a"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==3.6.0"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.8.2"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588",
+                "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"
+            ],
+            "version": "==2023.3"
+        },
+        "rasterio": {
+            "hashes": [
+                "sha256:075731165d26ef33e33e3ab903a0ecabb643893794bea1c78dd6c631713d1820",
+                "sha256:106e6519f5d2a0d35f0e963a8d07dc313ba7f6595a4c89fe9d6dcd8662a86617",
+                "sha256:41f233b7c85a74d66da4e31e37cfabe3c1aa116260063519d49f266e278bd4b3",
+                "sha256:5247affa7122f3c10a82d8cfa56b89fb60bfb981e4a22ebad8889d1caa138215",
+                "sha256:58d3d42cf06645f694911ea2707e6204868006784e0df3540e05e4615209b9c3",
+                "sha256:a8013826cdf540fb4f1507f92bb4ef440b594a4a8895d9038e25498e5b135f52",
+                "sha256:a97610a2997d9dc246ccd3f1361fdd63ae0b23a676e68af4f018f03e8c8a7592",
+                "sha256:abfdcb8f10210b8fad939f40d545d6c47e9e3b5cf4a43773ca8dd11c58204304",
+                "sha256:b4ad780117fefcd0fb81e2459d409a722f5964d1f655252278c7c56eb18f7ae6",
+                "sha256:baab6f3ba468d3b3ada05b282574af0f3110830f50888948d50c1f16aa6130af",
+                "sha256:c04a4932807ae64642f18d7c3f7cf504cb6703ec35496d07f2e5679ec5b9c4d3",
+                "sha256:c2ddc9cb2ccb0d296ca8c9da8af73b4d1c03c887a7dabbb4b4a6fa1e7450c1e7",
+                "sha256:cd21347b58fb60f2f0a8f7eb95ce58680590e205dee80562a67e868eebaaeea5",
+                "sha256:ce334df60b9d6ea66751b03afd1d138a551d27f2588e95ac6d7479856bb575a8",
+                "sha256:d6fa64dfda2ce1fe4c8b4e61d6e476ab3f389ac679e49b5386bb83708939a1b2",
+                "sha256:e3c06f1a62b28437c908851f0fc67abc6b31fce4a535e88e92311e73b8709996",
+                "sha256:feebf547b6785d57330f9ef69af54534a9548b24510c4e2fe87e5b9355112c28"
+            ],
+            "index": "pypi",
+            "version": "==1.3.7"
+        },
+        "setuptools": {
+            "hashes": [
+                "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f",
+                "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==67.8.0"
+        },
+        "shapely": {
+            "hashes": [
+                "sha256:01224899ff692a62929ef1a3f5fe389043e262698a708ab7569f43a99a48ae82",
+                "sha256:05c51a29336e604c084fb43ae5dbbfa2c0ef9bd6fedeae0a0d02c7b57a56ba46",
+                "sha256:09d6c7763b1bee0d0a2b84bb32a4c25c6359ad1ac582a62d8b211e89de986154",
+                "sha256:193a398d81c97a62fc3634a1a33798a58fd1dcf4aead254d080b273efbb7e3ff",
+                "sha256:1a34a23d6266ca162499e4a22b79159dc0052f4973d16f16f990baa4d29e58b6",
+                "sha256:2569a4b91caeef54dd5ae9091ae6f63526d8ca0b376b5bb9fd1a3195d047d7d4",
+                "sha256:33403b8896e1d98aaa3a52110d828b18985d740cc9f34f198922018b1e0f8afe",
+                "sha256:3ad81f292fffbd568ae71828e6c387da7eb5384a79db9b4fde14dd9fdeffca9a",
+                "sha256:3cb256ae0c01b17f7bc68ee2ffdd45aebf42af8992484ea55c29a6151abe4386",
+                "sha256:45b4833235b90bc87ee26c6537438fa77559d994d2d3be5190dd2e54d31b2820",
+                "sha256:4641325e065fd3e07d55677849c9ddfd0cf3ee98f96475126942e746d55b17c8",
+                "sha256:502e0a607f1dcc6dee0125aeee886379be5242c854500ea5fd2e7ac076b9ce6d",
+                "sha256:66a6b1a3e72ece97fc85536a281476f9b7794de2e646ca8a4517e2e3c1446893",
+                "sha256:70a18fc7d6418e5aea76ac55dce33f98e75bd413c6eb39cfed6a1ba36469d7d4",
+                "sha256:7d3bbeefd8a6a1a1017265d2d36f8ff2d79d0162d8c141aa0d37a87063525656",
+                "sha256:83a8ec0ee0192b6e3feee9f6a499d1377e9c295af74d7f81ecba5a42a6b195b7",
+                "sha256:865bc3d7cc0ea63189d11a0b1120d1307ed7a64720a8bfa5be2fde5fc6d0d33f",
+                "sha256:90cfa4144ff189a3c3de62e2f3669283c98fb760cfa2e82ff70df40f11cadb39",
+                "sha256:91575d97fd67391b85686573d758896ed2fc7476321c9d2e2b0c398b628b961c",
+                "sha256:9a6ac34c16f4d5d3c174c76c9d7614ec8fe735f8f82b6cc97a46b54f386a86bf",
+                "sha256:a529218e72a3dbdc83676198e610485fdfa31178f4be5b519a8ae12ea688db14",
+                "sha256:a70a614791ff65f5e283feed747e1cc3d9e6c6ba91556e640636bbb0a1e32a71",
+                "sha256:ac1dfc397475d1de485e76de0c3c91cc9d79bd39012a84bb0f5e8a199fc17bef",
+                "sha256:b06d031bc64149e340448fea25eee01360a58936c89985cf584134171e05863f",
+                "sha256:b4f0711cc83734c6fad94fc8d4ec30f3d52c1787b17d9dca261dc841d4731c64",
+                "sha256:b50c401b64883e61556a90b89948297f1714dbac29243d17ed9284a47e6dd731",
+                "sha256:b519cf3726ddb6c67f6a951d1bb1d29691111eaa67ea19ddca4d454fbe35949c",
+                "sha256:bca57b683e3d94d0919e2f31e4d70fdfbb7059650ef1b431d9f4e045690edcd5",
+                "sha256:c43755d2c46b75a7b74ac6226d2cc9fa2a76c3263c5ae70c195c6fb4e7b08e79",
+                "sha256:c7eed1fb3008a8a4a56425334b7eb82651a51f9e9a9c2f72844a2fb394f38a6c",
+                "sha256:c8b0d834b11be97d5ab2b4dceada20ae8e07bcccbc0f55d71df6729965f406ad",
+                "sha256:ce88ec79df55430e37178a191ad8df45cae90b0f6972d46d867bf6ebbb58cc4d",
+                "sha256:d173d24e85e51510e658fb108513d5bc11e3fd2820db6b1bd0522266ddd11f51",
+                "sha256:d8f55f355be7821dade839df785a49dc9f16d1af363134d07eb11e9207e0b189",
+                "sha256:da71de5bf552d83dcc21b78cc0020e86f8d0feea43e202110973987ffa781c21",
+                "sha256:e55698e0ed95a70fe9ff9a23c763acfe0bf335b02df12142f74e4543095e9a9b",
+                "sha256:f32a748703e7bf6e92dfa3d2936b2fbfe76f8ce5f756e24f49ef72d17d26ad02",
+                "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.1"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.16.0"
+        },
+        "snuggs": {
+            "hashes": [
+                "sha256:501cf113fe3892e14e2fee76da5cd0606b7e149c411c271898e6259ebde2617b",
+                "sha256:988dde5d4db88e9d71c99457404773dabcc7a1c45971bfbe81900999942d9f07"
+            ],
+            "version": "==1.4.7"
+        },
+        "tzdata": {
+            "hashes": [
+                "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a",
+                "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"
+            ],
+            "markers": "python_version >= '2'",
+            "version": "==2023.3"
+        }
+    },
+    "develop": {}
+}
diff --git a/data/ble/ble_benchmark/README.md b/data/ble/ble_benchmark/README.md
new file mode 100644
index 000000000..7d2a900a2
--- /dev/null
+++ b/data/ble/ble_benchmark/README.md
@@ -0,0 +1,59 @@
+## Update BLE benchmark data
+    
+Downloads and preprocesses FEMA [Base Level Engineering](https://webapps.usgs.gov/infrm/estbfe/) (BLE) benchmark datasets for purposes of evaluating FIM output. A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference dataset. The outputs will be a CSV table of flows with [`feature_id`, `discharge`] columns and a boolean (True/False) TIF inundation extent raster raster with inundated areas (True or 1) and dry areas (False or 0).
+
+As the `reference_raster` is required for preprocessing, it is assumed that `fim_pipeline.py` has previously been run for the HUCs being processed.
+    
+## Installation
+    
+1. A Dockerfile is provided in the root of the `/foss_fim/data/ble/ble_benchmark` directory. The Docker image can be built using the following command from `/foss_fim/data/ble/ble_benchmark`:
+```
+docker build --rm --no-cache --force-rm -t dev:ble_benchmark .
+```
+
+2. The Docker image can be run using the following command:
+```
+docker run --rm -it -v ~/git/inundation-mapping:/foss_fim -v ~/efs/fim_data/fim_4_0_13_1:/data \
+-v ~/efs/fim_data/fim_4_0_13_1/outputs:/outputs --name ble_benchmark dev:ble_benchmark
+```
+
+## Usage
+```
+Create BLE benchmark files
+
+options:
+  -h, --help            show this help message and exit
+  -i INPUT_FILE, --input-file INPUT_FILE
+                        Input file (.xlsx or .csv) a version resides in inputs/ble/ble_benchmark
+  -s SAVE_FOLDER, --save-folder SAVE_FOLDER
+                        Output folder
+  -r REFERENCE_FOLDER, --reference-folder REFERENCE_FOLDER
+                        Reference folder
+  -o BENCHMARK_FOLDER, --benchmark-folder BENCHMARK_FOLDER
+                        Benchmark folder
+  -n NWM_GEOPACKAGE, --nwm-geopackage NWM_GEOPACKAGE
+                        NWM flows geopackage
+  -u HUC, --huc HUC     Run a single HUC. If not supplied, it will run all HUCs in the input file.
+  -xs BLE_XS_LAYER_NAME, --ble-xs-layer-name BLE_XS_LAYER_NAME
+                        BLE cross section layer. Default layer is "XS" (sometimes it is "XS_1D").
+  -l NWM_STREAM_LAYER_NAME, --nwm-stream-layer-name NWM_STREAM_LAYER_NAME
+                        NWM streams layer. Default layer is "nwm_streams".
+  -id NWM_FEATURE_ID_FIELD, --nwm-feature-id-field NWM_FEATURE_ID_FIELD
+                        ID field for NWM streams. Not required if NWM v2.1 is used (default id field is "ID").
+```
+
+## Example
+```
+python /foss_fim/data/ble/ble_benchmark/create_ble_benchmark.py \
+-i /data/inputs/ble/ble_benchmark/EBFE_urls_20230608.xlsx \
+-s /data/temp/ble_benchmark \
+-r /data/outputs/fim_4_3_12_0 \
+-o /data/test_cases/ble_test_cases/validation_data_ble \
+-n /data/inputs/nwm_hydrofabric/nwm_flows.gpkg \
+-l nwm_streams \
+-u 12090301
+```
+
+## Notes
+
+If this is run without the `-u` option, it will run all HUCs in the input file and download nearly 1TB of data. If this is done, please notify Florence Thompson (fethomps@usgs.gov) so that the download usage can be attributed.
diff --git a/data/ble/ble_benchmark/create_ble_benchmark.py b/data/ble/ble_benchmark/create_ble_benchmark.py
new file mode 100644
index 000000000..516d3725e
--- /dev/null
+++ b/data/ble/ble_benchmark/create_ble_benchmark.py
@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+
+import os
+from datetime import datetime
+import argparse
+import numpy as np
+import pandas as pd
+from osgeo import gdal
+from urllib.request import urlopen
+from io import BytesIO
+from zipfile import ZipFile
+from glob import glob
+from preprocess_benchmark import preprocess_benchmark_static
+from create_flow_forecast_file import create_flow_forecast_file
+
+
+def create_ble_benchmark(input_file: str, 
+                         save_folder: str,
+                         reference_folder: str,
+                         benchmark_folder: str,
+                         nwm_geopackage: str,
+                         ble_xs_layer_name: str,
+                         nwm_stream_layer_name: str,
+                         nwm_feature_id_field: str,
+                         huc: str = None):
+    """
+    Downloads and preprocesses BLE benchmark datasets for purposes of evaluating FIM output. 
+    A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference dataset. 
+    The benchmark raster will also be converted to a boolean (True/False) raster with inundated areas (True or 1) and dry areas (False or 0).
+
+    As the reference_raster is required for preprocessing, it is assumed that fim_pipeline.py has previously been run for the HUCs being processed.
+    
+    Parameters
+    ----------
+    input_file: str
+        Path to input file (XLSX or CSV) with list of URL(s) to files to download (e.g. EBFE_urls_20230608.xlsx)
+    save_folder: str
+        Path to save folder where the downloaded ZIP files and extracted depth rasters will be saved (e.g., /data/temp/ble_downloads). 
+        This folder will be created if it doesn't exist.
+    reference_folder: str
+        Path to reference raster (e.g., /data/outputs/fim_4_3_12_0). This folder must exist (e.g., created by running fim_pipeline.py).
+    benchmark_folder: str
+        Path to the benchmark folder (e.g., /data/test_cases/ble_test_cases/validation_data_ble). This folder will be created if it doesn't exist.
+
+    Returns
+    -------
+    None
+    """
+
+    if not os.path.exists(save_folder):
+        os.makedirs(save_folder)
+
+    # EBFE_urls_20230608.xlsx acquired from FEMA (fethomps@usgs.gov)
+    ext = os.path.splitext(input_file)[1]
+    if ext == '.xlsx':
+        data = pd.read_excel(input_file, header=None, names=['size', 'units', 'URL'])
+    elif ext == '.csv':
+        data = pd.read_csv(input_file, header=None, names=['size', 'units', 'URL'])
+
+    # Subset Spatial Data URLs
+    spatial_df = data[data['URL'].str.contains('SpatialData')]
+    if huc is not None:
+        spatial_df = spatial_df[spatial_df['URL'].str.contains(huc)]
+
+    spatial_df = spatial_df.reset_index()
+
+    # Convert size to MiB
+    spatial_df['MiB'] = np.where(spatial_df['units']=='GiB', spatial_df['size'] * 1000, spatial_df['size'])
+
+    spatial_df, ble_geodatabase = download_and_extract_rasters(spatial_df, save_folder)
+
+    for i, row in spatial_df.iterrows():
+        huc = row['HUC']
+        # reference_raster is used to set the metadata for benchmark_raster
+        reference_raster = os.path.join(reference_folder, f'{huc}/branches/0/rem_zeroed_masked_0.tif') 
+        for benchmark_raster in row['rasters']:
+            magnitude = '100yr' if 'BLE_DEP01PCT' in benchmark_raster else '500yr'
+
+            out_raster_dir = os.path.join(benchmark_folder, huc, magnitude)
+            if not os.path.exists(out_raster_dir):
+                os.makedirs(out_raster_dir)
+                
+            out_raster_path = os.path.join(out_raster_dir, f"ble_huc_{huc}_extent_{magnitude}.tif")
+            
+            # Make benchmark inundation raster
+            preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_path)
+
+            create_flow_forecast_file(huc, ble_geodatabase, nwm_geopackage, benchmark_folder, 
+                                      ble_xs_layer_name, nwm_stream_layer_name, nwm_feature_id_field)
+
+
+def download_and_extract_rasters(spatial_df: pd.DataFrame, save_folder: str):
+    """
+    Download and extract rasters from URLs in spatial_df. Extracted rasters will be saved to save_folder.
+
+    Parameters
+    ----------
+    spatial_df: pandas.DataFrame
+        DataFrame containing URLs to download and extract
+    save_folder: str
+        Path to save folder
+
+    Returns
+    -------
+    spatial_df: pandas.DataFrame
+        Updated DataFrame containing HUC and HUC_Name
+    out_files: list
+        List of paths to extracted rasters
+    """
+    
+    depth_rasters = ['BLE_DEP0_2PCT', 'BLE_DEP01PCT']
+
+    # Download and unzip each file
+    hucs = []
+    huc_names = []
+    out_files = []
+    out_list = []
+    for i, row in spatial_df.iterrows():
+        # Extract HUC and HUC Name from URL
+        url = row['URL']
+        huc, huc_name = os.path.basename(os.path.dirname(url)).split('_')
+        hucs.append(huc)
+        huc_names.append(huc_name)
+
+        # Download and unzip file
+        save_file = os.path.join(save_folder, os.path.basename(url))
+        if not os.path.exists(save_file):
+            http_response = urlopen(url)
+            zipfile = ZipFile(BytesIO(http_response.read()))
+            zipfile.extractall(path=save_file)
+
+        gdb_list = glob(os.path.join(save_file, '**', '*.gdb'), recursive=True)
+            
+        if len(gdb_list) == 1:
+            ble_geodatabase = gdb_list[0]
+            src_ds = gdal.Open(ble_geodatabase, gdal.GA_ReadOnly)
+            subdatasets = src_ds.GetSubDatasets()
+
+            # Find depth rasters
+            for depth_raster in depth_rasters:
+                out_file = os.path.join(save_folder, f'{huc}_{depth_raster}.tif')
+
+                if not os.path.exists(out_file):
+                    # Read raster data from GDB
+                    print(f'Reading {depth_raster} for {huc}')
+                    depth_raster_path = [item[0] for item in subdatasets if depth_raster in item[1]][0]
+
+                    extract_raster(depth_raster_path, out_file)
+
+                out_list.append(out_file)
+
+        if len(out_list) > 0:
+            out_files.append(out_list)
+
+    spatial_df['HUC'] = hucs
+    spatial_df['HUC_Name'] = huc_names
+    spatial_df['rasters'] = out_files
+
+    return spatial_df, ble_geodatabase
+
+
+def extract_raster(in_raster: str, out_raster: str):
+    """
+    Extract raster from GDB and save to out_raster
+
+    Parameters
+    ----------
+    in_raster: str
+        Path to input raster
+    out_raster: str
+        Path to output raster
+
+    Returns
+    -------
+    None
+    """
+    
+    data_ds = gdal.Open(in_raster, gdal.GA_ReadOnly)
+    data = data_ds.ReadAsArray()
+    nodata = data_ds.GetRasterBand(1).GetNoDataValue()
+
+    driver = gdal.GetDriverByName('GTiff')
+    out_ds = driver.Create(out_raster, data.shape[1], data.shape[0], 1, gdal.GDT_Float32)
+    out_ds.SetGeoTransform(data_ds.GetGeoTransform())
+    out_ds.SetProjection(data_ds.GetProjection())
+    out_ds.GetRasterBand(1).WriteArray(data)
+    out_ds.GetRasterBand(1).SetNoDataValue(nodata)
+    out_ds = None
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create BLE benchmark files', 
+                                     usage='''python3 /foss_fim/data/ble/ble_benchmark/create_ble_benchmark.py
+                                                -i /data/inputs/ble/ble_benchmark/EBFE_urls_20230608.xlsx
+                                                -s /data/temp/ble_benchmark
+                                                -r /data/outputs/my_run/
+                                                -o /data/test_cases/ble_test_cases/validation_data_ble
+                                                -n /data/inputs/nwm_hydrofabric/nwm_flows.gpkg
+                                                -u 12030105
+                                            ''')
+    parser.add_argument('-i', '--input-file', type=str, help='Input file', required=True)
+    parser.add_argument('-s', '--save-folder', type=str, help='Output folder', required=True)
+    parser.add_argument('-r', '--reference-folder', type=str, help='Reference folder', required=True)
+    parser.add_argument('-o', '--benchmark-folder', type=str, help='Benchmark folder', required=True)
+    parser.add_argument('-n', '--nwm-geopackage', type=str, help='NWM streams geopackage', required=True)
+    parser.add_argument('-u', '--huc', type=str, help='Run a single HUC. If not supplied, it will run all HUCs in the input file', required=False)
+    parser.add_argument('-xs', '--ble-xs-layer-name', help='BLE cross section layer. Default layer is "XS" (sometimes it is "XS_1D").', required=False, default='XS')
+    parser.add_argument('-l', '--nwm-stream-layer-name', help='NWM streams layer. Default layer is "nwm_streams"', required=False, default='nwm_streams')
+    parser.add_argument('-id', '--nwm-feature-id-field', help='id field for nwm streams. Not required if NWM v2.1 is used (default id field is "ID")', required=False, default='ID')
+
+    args = vars(parser.parse_args())
+
+    start_time = datetime.now()
+    print('Pulling BLE Benchmark data...  \n')
+    
+    create_ble_benchmark(**args)
+
+    end_time = datetime.now()
+    print('\n Finished Pulling BLE Benchmark data \n',
+          'Total Time: ', end_time - start_time)
+
diff --git a/data/ble/ble_benchmark/create_flow_forecast_file.py b/data/ble/ble_benchmark/create_flow_forecast_file.py
new file mode 100755
index 000000000..d3e91502b
--- /dev/null
+++ b/data/ble/ble_benchmark/create_flow_forecast_file.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+import os
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+import argparse
+import fiona
+
+def create_flow_forecast_file(huc,
+                              ble_geodatabase,
+                              nwm_geodatabase,
+                              output_parent_dir, 
+                              ble_xs_layer_name = 'XS', 
+                              nwm_stream_layer_name = 'nwm_streams', 
+                              nwm_feature_id_field ='ID'):
+    '''
+    This function will create a forecast flow file using ble data. 
+    It will import the ble XS layer and will intersect it with a nwm streams layer. 
+    It will the perform an intersection with the BLE XS layer and the nwm river layer.
+    It will then calculate the median flow for the 100 and 500 year events for each nwm segment and convert flow from CFS to CMS. 
+    Individual forecast files (.csv) will be created  with two columns (nwm segment ID and Flow (CMS)) for the 100 year and the 500 year event. 
+    Flow field names are set to the default field names within the BLE submittal. 
+    As differing versions of the NWM river layer have different names for the ID field, 
+    this field will need to be specified as an argument to the function. 
+    Output flow forecast files will be automatically named to be compatible with current FIM code,
+    and will be written out in specific folder structures.
+
+    Parameters
+    ----------
+    ble_geodatabase : STRING
+        Path to BLE geodatabase.
+    nwm_geodatabase : STRING
+        Path to nwm geodatabase.
+    output_parent_dir : STRING
+        Output parent directory of output. Flow files will be output to subdirectories within parent directory.
+    ble_xs_layer_name : STRING
+       The cross section layer in the ble geodatabase to be imported. Default is 'XS' (sometimes it is 'XS_1D')
+    ble_huc_layer_name : STRING
+       The huc layer in the ble geodatabase.  Default is 'S_HUC_Ar' (sometimes it is 'S_HUC_ar' )
+    nwm_stream_layer_name : STRING
+       The stream centerline layer name (or partial layer name) for the NWM geodatabase.  Default is 'RouteLink_FL_2020_04_07'.
+    nwm_feature_id_field : STRING
+       The feature id of the nwm segments.  Default is 'ID' (applicable if nwmv2.1 is used)
+    Returns
+    -------
+    None.
+
+    '''
+
+    def fill_missing_flows(forecast: pd.DataFrame, nwm_river_layer: pd.DataFrame):
+        """
+        This function will fill in missing flows in the forecast dataframe. 
+        It will do this by finding segments that don't intersect with the forecast dataframe, and
+        then finding the upstream and downstream segments. 
+        It will then assign the average of the two to the missing segment.
+
+        Parameters
+        ----------
+        forecast: pandas.DataFrame
+            Dataframe containing forecast flows.
+        nwm_river_layer: pandas.DataFrame
+            Dataframe containing NWM river layer.
+
+        Returns
+        -------
+        forecast: pandas.DataFrame
+            Dataframe containing forecast flows with missing flows filled in.
+        """
+        n = 1
+        while True:
+            # Merge flows with nwm_river_layer. This will be used to find upstream and downstream flow.
+            merged = nwm_river_layer.merge(forecast, left_on='ID', right_on='feature_id', how='left') 
+
+            nonintersects = merged[merged['discharge'].isna()]
+
+            print(f'Iteration {n}. {len(nonintersects)} segments remaining.')
+
+            updated = False
+            for i, row in nonintersects.iterrows():
+                new_flow = []
+
+                # Find sum of upstream flow(s)
+                upstreams = merged.loc[merged['to']==row['ID']]
+                # if all upstream segments have flows
+                if len(upstreams) > 0 and not upstreams['discharge'].isnull().values.any(): 
+                    new_flow.append(upstreams['discharge'].sum())
+
+                # Find downstream flow(s)
+                downstream = merged.loc[merged['ID']==row['to']]
+
+                if len(downstream) > 0:
+                    downstream_upstreams = merged.loc[merged['to']==int(downstream['ID'].iloc[0])]
+                
+                    # all segments have flow except for the one being filled in
+                    if (downstream['discharge'].isnull().sum() + downstream_upstreams['discharge'].isnull().sum()) == 1: 
+                        new_flow.append(downstream['discharge'].iloc[0] - downstream_upstreams['discharge'].sum())
+
+                # Add row data to forecast
+                if len(new_flow) > 0:
+                    forecast = pd.concat([forecast, pd.DataFrame({'feature_id': row['ID'], 
+                                                                  'discharge': np.mean(new_flow)}, 
+                                                                 index=[0])]).reset_index(drop=True)
+                    updated = True
+
+            if not updated:
+                break
+            else:
+                n += 1
+
+        forecast = forecast[forecast['discharge'] > 0]
+
+        return forecast
+
+
+    # Read the ble xs layer into a geopandas dataframe.
+    layers = fiona.listlayers(ble_geodatabase)
+    if ble_xs_layer_name not in layers:
+        ble_xs_layer_name = 'XS_1D'
+    
+    xs_layer = gpd.read_file(ble_geodatabase,layer = ble_xs_layer_name)
+
+    # Read in the NWM stream layer into a geopandas dataframe using the bounding box option based on the extents of the BLE XS layer.
+    nwm_river_layer = gpd.read_file(nwm_geodatabase, bbox = xs_layer, layer=nwm_stream_layer_name)
+
+    # Make sure xs_layer is in same projection as nwm_river_layer.
+    xs_layer_proj = xs_layer.to_crs(nwm_river_layer.crs)
+
+    # Perform an intersection of the BLE layers and the NWM layers, using the keep_geom_type set to False produces a point output.
+    intersection = gpd.overlay(xs_layer_proj, nwm_river_layer, how='intersection', keep_geom_type = False)
+
+    ## Create the flow forecast files
+    # Define fields containing flow (typically these won't change for BLE)
+    flow_fields = ['E_Q_01PCT', 'E_Q_0_2PCT']
+
+    # Define return period associated with flow_fields (in same order as flow_fields). These will also serve as subdirectory names.
+    return_period = ['100yr', '500yr']
+
+    # Conversion factor from CFS to CMS
+    dischargeMultiplier = 0.3048 ** 3
+
+    # Write individual flow csv files
+    for i,flow in enumerate(flow_fields):
+
+        # Write dataframe with just ID and single flow event
+        forecast = intersection[[nwm_feature_id_field,flow]]
+
+        # Rename field names and re-define datatypes
+        forecast = forecast.rename(columns={nwm_feature_id_field:'feature_id', flow: 'discharge'})
+        forecast = forecast.astype({'feature_id': int, 'discharge': float})
+
+        # Calculate median flow per feature id
+        forecast = forecast.groupby('feature_id').median()
+        forecast = forecast.reset_index(level=0)
+
+        # Convert CFS to CMS
+        forecast['discharge'] = forecast['discharge'] * dischargeMultiplier
+
+        # Assign flow to segments missing flows
+        print(f'Filling in missing flows for {return_period[i]} flow.')
+        forecast = fill_missing_flows(forecast, nwm_river_layer)
+
+        # Set paths and write file
+        output_dir = os.path.join(output_parent_dir, huc)
+        dir_of_csv = os.path.join(output_dir,return_period[i])
+        os.makedirs(dir_of_csv,exist_ok = True)
+        path_to_csv = os.path.join(dir_of_csv,"ble_huc_{}_flows_{}.csv".format(huc,return_period[i]))
+        forecast.to_csv(path_to_csv,index=False)
+
+
+if __name__ == '__main__':
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Produce forecast flow files from BLE datasets')
+    parser.add_argument('-b', '--ble-geodatabase', help='BLE geodatabase (.gdb file extension).                               \
+        Will look for layer with "XS" in name. It is assumed the 100 year flow field is "E_Q_01PCT"                           \
+        and the 500 year flow field is "E_Q_0_2_PCT" as these are the default field names.', required=True)
+    parser.add_argument('-n', '--nwm-geodatabase',  help='NWM geodatabase (.gdb file extension).', required=True)
+    parser.add_argument('-o', '--output-parent-dir', help='Output directory where forecast files will be stored.              \
+        Two subdirectories are created (100yr and 500yr) and in each subdirectory a forecast file is written', required=True)
+    parser.add_argument('-xs', '--ble-xs-layer-name', help='BLE cross section layer.                                          \
+        Default layer is "XS" (sometimes it is "XS_1D").', required = False, default='XS')
+    parser.add_argument('-hu', '--huc', help = 'HUC', required=True)
+    parser.add_argument('-l', '--nwm-stream-layer-name', help='NWM streams layer. Default layer is "nwm_streams")',           \
+        required=False, default='nwm_streams')
+    parser.add_argument('-f', '--nwm-feature-id-field', help='id field for nwm streams.                                       \
+        Not required if NWM v2.1 is used (default id field is "ID")', required=False, default='ID')
+    
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+    # Run create_flow_forecast_file
+    create_flow_forecast_file(**args)
diff --git a/tools/preprocess_benchmark.py b/data/ble/ble_benchmark/preprocess_benchmark.py
similarity index 99%
rename from tools/preprocess_benchmark.py
rename to data/ble/ble_benchmark/preprocess_benchmark.py
index 81a65db2d..d17c931d4 100755
--- a/tools/preprocess_benchmark.py
+++ b/data/ble/ble_benchmark/preprocess_benchmark.py
@@ -61,7 +61,7 @@ def preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_p
     profile.update(transform = new_transform)
     profile.update(dtype = rasterio.int8)
     profile.update(nodata = 2) #Update NODATA to some integer so we can keep int8 datatype. There are no NODATA in the raster dataset
-    profile.update (width = new_width)
+    profile.update(width = new_width)
     profile.update(height = new_height)
 
     # Write out preprocessed benchmark array to raster if path is supplied
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4ba66abbc..76e4462e6 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.15.1 - 2023-08-08 - [PR#960](https://github.com/NOAA-OWP/inundation-mapping/pull/960)
+
+Provides a scripted procedure for updating BLE benchmark data including downloading, extracting, and processing raw BLE data into benchmark inundation files (inundation rasters and discharge tables).
+
+### Additions
+
+- `data/ble/ble_benchmark/`
+    - `Dockerfile`, `Pipfile`, and `Pipfile.lock`: creates a new Docker image with necessary Python packages
+    - `README.md`: contains installation and usage information
+    - `create_ble_benchmark.py`: main script to generate BLE benchmark data
+
+### Changes
+
+- `data/ble/ble_benchmark/`
+    - `create_flow_forecast_file.py` and `preprocess_benchmark.py`: moved from /tools
+
+<br/><br/>
+
 ## v4.3.15.0 - 2023-08-08 - [PR##956](https://github.com/NOAA-OWP/inundation-mapping/pull/956)
 
 Integrating GVAL in to the evaluation of agreement maps and contingency tables.
diff --git a/tools/create_flow_forecast_file.py b/tools/create_flow_forecast_file.py
deleted file mode 100755
index bb8833343..000000000
--- a/tools/create_flow_forecast_file.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import geopandas as gpd
-import argparse
-
-def create_flow_forecast_file(ble_geodatabase, nwm_geodatabase, output_parent_dir, ble_xs_layer_name = 'XS', ble_huc_layer_name = 'S_HUC_Ar', ble_huc_id_field = 'HUC_CODE', nwm_stream_layer_name = 'RouteLink_FL_2020_04_07', nwm_feature_id_field ='ID'):
-    '''
-    This function will create a forecast flow file using ble data. It will import the ble XS layer and will intersect it with a nwm streams layer. It will the perform an intersection with the BLE XS layer and the nwm river layer. It will then calculate the median flow for the 100 and 500 year events for each nwm segment and convert flow from CFS to CMS. Individual forecast files (.csv) will be created  with two columns (nwm segment ID and Flow (CMS)) for the 100 year and the 500 year event. Flow field names are set to the default field names within the BLE submittal. As differing versions of the NWM river layer have different names for the ID field, this field will need to be specified as an argument to the function. Output flow forecast files will be automatically named to be compatible with current FIM code and will be written out in specific folder structures.
-
-    Parameters
-    ----------
-    ble_geodatabase : STRING
-        Path to BLE geodatabase.
-    nwm_geodatabase : STRING
-        Path to nwm geodatabase.
-    output_parent_dir : STRING
-        Output parent directory of output. Flow files will be output to subdirectories within parent directory.
-    ble_xs_layer_name : STRING
-       The cross section layer in the ble geodatabase to be imported. Default is 'XS' (sometimes it is 'XS_1D')
-    ble_huc_layer_name : STRING
-       The huc layer in the ble geodatabase.  Default is 'S_HUC_Ar' (sometimes it is 'S_HUC_ar' )
-    ble_huc_id_field : STRING
-       The attribute field within the ble_huc_layer_name containing the huc code. Default is 'HUC_CODE'. Assumes only 1 unique code.
-    nwm_stream_layer_name : STRING
-       The stream centerline layer name (or partial layer name) for the NWM geodatabase.  Default is 'RouteLink_FL_2020_04_07'.
-    nwm_feature_id_field : STRING
-       The feature id of the nwm segments.  Default is 'ID' (applicable if nwmv2.1 is used)
-    Returns
-    -------
-    None.
-
-    '''
-    # Read the ble xs layer into a geopandas dataframe.
-    xs_layer = gpd.read_file(ble_geodatabase,layer = ble_xs_layer_name)
-
-    # Read ble huc layer into a geopandas dataframe and extract the huc code. By default it assumes only one HUC in the layer (typically always the case).
-    huc_layer = gpd.read_file(ble_geodatabase, layer = ble_huc_layer_name)
-    [huc] = huc_layer[ble_huc_id_field].unique()
-
-    # Read in the NWM stream layer into a geopandas dataframe using the bounding box option based on the extents of the BLE XS layer.
-    nwm_river_layer = gpd.read_file(nwm_geodatabase, bbox = xs_layer, layer = nwm_stream_layer_name)
-
-    # Make sure xs_layer is in same projection as nwm_river_layer.
-    xs_layer_proj = xs_layer.to_crs(nwm_river_layer.crs)
-
-    # Perform an intersection of the BLE layers and the NWM layers, using the keep_geom_type set to False produces a point output.
-    intersection = gpd.overlay(xs_layer_proj, nwm_river_layer, how = 'intersection', keep_geom_type = False)
-
-    ## Create the flow forecast files
-    # Define fields containing flow (typically these won't change for BLE)
-    flow_fields = ['E_Q_01PCT','E_Q_0_2PCT']
-
-    # Define return period associated with flow_fields (in same order as flow_fields). These will also serve as subdirectory names.
-    return_period = ['100yr','500yr']
-
-    # Conversion factor from CFS to CMS
-    dischargeMultiplier = 0.3048 ** 3
-
-    # Write individual flow csv files
-    for i,flow in enumerate(flow_fields):
-
-        # Write dataframe with just ID and single flow event
-        forecast = intersection[[nwm_feature_id_field,flow]]
-
-        # Rename field names and re-define datatypes
-        forecast = forecast.rename(columns={nwm_feature_id_field :'feature_id',flow : 'discharge'})
-        forecast = forecast.astype({'feature_id' : int , 'discharge' : float})
-
-        #Calculate median flow per feature id
-        forecast = forecast.groupby('feature_id').median()
-        forecast = forecast.reset_index(level=0)
-
-        # Convert CFS to CMS
-        forecast['discharge'] = forecast['discharge'] * dischargeMultiplier
-
-        # Set paths and write file
-        output_dir = os.path.join(output_parent_dir, huc)
-        dir_of_csv = os.path.join(output_dir,return_period[i])
-        os.makedirs(dir_of_csv,exist_ok = True)
-        path_to_csv = os.path.join(dir_of_csv,"ble_huc_{}_flows_{}.csv".format(huc,return_period[i]))
-        forecast.to_csv(path_to_csv,index=False)
-
-if __name__ == '__main__':
-    # Parse arguments
-    parser = argparse.ArgumentParser(description = 'Produce forecast flow files from BLE datasets')
-    parser.add_argument('-b', '--ble-geodatabase', help = 'BLE geodatabase (.gdb file extension). Will look for layer with "XS" in name. It is assumed the 100 year flow field is "E_Q_01PCT" and the 500 year flow field is "E_Q_0_2_PCT" as these are the default field names.', required = True)
-    parser.add_argument('-n', '--nwm-geodatabase',  help = 'NWM geodatabase (.gdb file extension).', required = True)
-    parser.add_argument('-o', '--output-parent-dir', help = 'Output directory where forecast files will be stored. Two subdirectories are created (100yr and 500yr) and in each subdirectory a forecast file is written', required = True)
-    parser.add_argument('-xs', '--ble-xs-layer-name', help = 'BLE cross section layer. Default layer is "XS" (sometimes it is "XS_1D").', required = False, default = 'XS')
-    parser.add_argument('-hu', '--ble-huc-layer-name', help = 'BLE huc layer. Default layer is "S_HUC_Ar" (sometimes it is "S_HUC_ar").', required = False, default = 'S_HUC_Ar')
-    parser.add_argument('-huid', '--ble-huc-id-field', help = 'BLE id field in the ble-huc-layer-name. Default field is "HUC_CODE".', required = False, default = 'HUC_CODE')
-    parser.add_argument('-l', '--nwm-stream-layer-name', help = 'NWM streams layer. Default layer is "RouteLink_FL_2020_04_07")', required = False, default = 'RouteLink_FL_2020_04_07')
-    parser.add_argument('-f', '--nwm-feature-id-field', help = 'id field for nwm streams. Not required if NWM v2.1 is used (default id field is "ID")', required = False, default = 'ID')
-    # Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-    # Run create_flow_forecast_file
-    create_flow_forecast_file(**args)

From 831c603fcfa247dddb2bd9bb8fa919dafce971b7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 17 Aug 2023 12:49:53 -0500
Subject: [PATCH 265/359] Bump certifi from 2023.5.7 to 2023.7.22 in
 /data/ble/ble_benchmark (#963)

---
 data/ble/ble_benchmark/Pipfile.lock | 83 ++++++++++++++---------------
 1 file changed, 40 insertions(+), 43 deletions(-)

diff --git a/data/ble/ble_benchmark/Pipfile.lock b/data/ble/ble_benchmark/Pipfile.lock
index f4df3ba79..5e258cf1b 100644
--- a/data/ble/ble_benchmark/Pipfile.lock
+++ b/data/ble/ble_benchmark/Pipfile.lock
@@ -34,19 +34,19 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7",
-                "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"
+                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
+                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2023.5.7"
+            "index": "pypi",
+            "version": "==2023.7.22"
         },
         "click": {
             "hashes": [
-                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
-                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
+                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
+                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.3"
+            "version": "==8.1.6"
         },
         "click-plugins": {
             "hashes": [
@@ -107,37 +107,34 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187",
-                "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812",
-                "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7",
-                "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4",
-                "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6",
-                "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0",
-                "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4",
-                "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570",
-                "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4",
-                "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f",
-                "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80",
-                "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289",
-                "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385",
-                "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078",
-                "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c",
-                "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463",
-                "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3",
-                "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950",
-                "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155",
-                "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7",
-                "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c",
-                "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096",
-                "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17",
-                "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf",
-                "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4",
-                "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02",
-                "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c",
-                "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"
+                "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
+                "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55",
+                "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf",
+                "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01",
+                "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca",
+                "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901",
+                "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d",
+                "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4",
+                "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf",
+                "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380",
+                "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044",
+                "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545",
+                "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f",
+                "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f",
+                "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3",
+                "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364",
+                "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9",
+                "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418",
+                "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f",
+                "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295",
+                "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3",
+                "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187",
+                "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926",
+                "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357",
+                "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.24.3"
+            "markers": "python_version >= '3.9'",
+            "version": "==1.25.2"
         },
         "openpyxl": {
             "hashes": [
@@ -188,11 +185,11 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
-                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
+                "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb",
+                "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"
             ],
             "markers": "python_full_version >= '3.6.8'",
-            "version": "==3.0.9"
+            "version": "==3.1.1"
         },
         "pyproj": {
             "hashes": [
@@ -265,11 +262,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f",
-                "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"
+                "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f",
+                "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==67.8.0"
+            "version": "==68.0.0"
         },
         "shapely": {
             "hashes": [

From b63524154458377425fb2a66e9d9c1622d18437c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 17 Aug 2023 12:58:09 -0500
Subject: [PATCH 266/359] Bump tornado from 6.3.2 to 6.3.3 (#966)

---
 Pipfile.lock | 107 +++++++++++++++++++++++++--------------------------
 1 file changed, 53 insertions(+), 54 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 070d13bd0..3bbb3a49f 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "markers": "python_version >= '3.7' and python_version < '4'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -133,7 +133,7 @@
                 "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
                 "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
             ],
-            "markers": "python_full_version >= '3.6.0'",
+            "markers": "python_version >= '3.6'",
             "version": "==4.12.2"
         },
         "bleach": {
@@ -465,8 +465,7 @@
                 "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
                 "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
             ],
-            "markers": "python_full_version >= '3.7.0'",
-
+            "markers": "python_version >= '3.7'",
             "version": "==3.2.0"
         },
         "click": {
@@ -489,7 +488,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -665,27 +664,27 @@
         },
         "debugpy": {
             "hashes": [
-                "sha256:0679b7e1e3523bd7d7869447ec67b59728675aadfc038550a63a362b63029d2c",
-                "sha256:279d64c408c60431c8ee832dfd9ace7c396984fd7341fa3116aee414e7dcd88d",
-                "sha256:33edb4afa85c098c24cc361d72ba7c21bb92f501104514d4ffec1fb36e09c01a",
-                "sha256:38ed626353e7c63f4b11efad659be04c23de2b0d15efff77b60e4740ea685d07",
-                "sha256:5224eabbbeddcf1943d4e2821876f3e5d7d383f27390b82da5d9558fd4eb30a9",
-                "sha256:53f7a456bc50706a0eaabecf2d3ce44c4d5010e46dfc65b6b81a518b42866267",
-                "sha256:9cd10cf338e0907fdcf9eac9087faa30f150ef5445af5a545d307055141dd7a4",
-                "sha256:aaf6da50377ff4056c8ed470da24632b42e4087bc826845daad7af211e00faad",
-                "sha256:b3e7ac809b991006ad7f857f016fa92014445085711ef111fdc3f74f66144096",
-                "sha256:bae1123dff5bfe548ba1683eb972329ba6d646c3a80e6b4c06cd1b1dd0205e9b",
-                "sha256:c0ff93ae90a03b06d85b2c529eca51ab15457868a377c4cc40a23ab0e4e552a3",
-                "sha256:c4c2f0810fa25323abfdfa36cbbbb24e5c3b1a42cb762782de64439c575d67f2",
-                "sha256:d71b31117779d9a90b745720c0eab54ae1da76d5b38c8026c654f4a066b0130a",
-                "sha256:dbe04e7568aa69361a5b4c47b4493d5680bfa3a911d1e105fbea1b1f23f3eb45",
-                "sha256:de86029696e1b3b4d0d49076b9eba606c226e33ae312a57a46dca14ff370894d",
-                "sha256:e3876611d114a18aafef6383695dfc3f1217c98a9168c1aaf1a02b01ec7d8d1e",
-                "sha256:ed6d5413474e209ba50b1a75b2d9eecf64d41e6e4501977991cdc755dc83ab0f",
-                "sha256:f90a2d4ad9a035cee7331c06a4cf2245e38bd7c89554fe3b616d90ab8aab89cc"
+                "sha256:038c51268367c9c935905a90b1c2d2dbfe304037c27ba9d19fe7409f8cdc710c",
+                "sha256:1093a5c541af079c13ac8c70ab8b24d1d35c8cacb676306cf11e57f699c02926",
+                "sha256:3370ef1b9951d15799ef7af41f8174194f3482ee689988379763ef61a5456426",
+                "sha256:38651c3639a4e8bbf0ca7e52d799f6abd07d622a193c406be375da4d510d968d",
+                "sha256:3de5d0f97c425dc49bce4293df6a04494309eedadd2b52c22e58d95107e178d9",
+                "sha256:4b9eba71c290852f959d2cf8a03af28afd3ca639ad374d393d53d367f7f685b2",
+                "sha256:65b28435a17cba4c09e739621173ff90c515f7b9e8ea469b92e3c28ef8e5cdfb",
+                "sha256:72f5d2ecead8125cf669e62784ef1e6300f4067b0f14d9f95ee00ae06fc7c4f7",
+                "sha256:85969d864c45f70c3996067cfa76a319bae749b04171f2cdeceebe4add316155",
+                "sha256:890f7ab9a683886a0f185786ffbda3b46495c4b929dab083b8c79d6825832a52",
+                "sha256:903bd61d5eb433b6c25b48eae5e23821d4c1a19e25c9610205f5aeaccae64e32",
+                "sha256:92b6dae8bfbd497c90596bbb69089acf7954164aea3228a99d7e43e5267f5b36",
+                "sha256:973a97ed3b434eab0f792719a484566c35328196540676685c975651266fccf9",
+                "sha256:d16882030860081e7dd5aa619f30dec3c2f9a421e69861125f83cc372c94e57d",
+                "sha256:d4ac7a4dba28801d184b7fc0e024da2635ca87d8b0a825c6087bb5168e3c0d28",
+                "sha256:eea8d8cfb9965ac41b99a61f8e755a8f50e9a20330938ad8271530210f54e09c",
+                "sha256:f0851403030f3975d6e2eaa4abf73232ab90b98f041e3c09ba33be2beda43fcf",
+                "sha256:fe87ec0182ef624855d05e6ed7e0b7cb1359d2ffa2a925f8ec2d22e98b75d0ca"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.6.7"
+            "version": "==1.6.7.post1"
         },
         "decorator": {
             "hashes": [
@@ -713,11 +712,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5",
-                "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"
+                "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9",
+                "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.1.2"
+            "version": "==1.1.3"
         },
         "executing": {
             "hashes": [
@@ -797,7 +796,6 @@
             ],
             "markers": "python_version >= '3.8'",
             "version": "==4.42.0"
-
         },
         "fqdn": {
             "hashes": [
@@ -950,7 +948,7 @@
                 "sha256:fd04764d0bb830414788cae897d082bf6ad92324e571a5511bd7e1de4a0cdc67",
                 "sha256:fde3f85864c84badb26f42d95639360e627fd09c529a76c46a06dbd7a5735c51"
             ],
-            "markers": "python_version > '3.6'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.3.1"
         },
         "iniconfig": {
@@ -1033,11 +1031,11 @@
         },
         "joblib": {
             "hashes": [
-                "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3",
-                "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915"
+                "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1",
+                "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.3.1"
+            "version": "==1.3.2"
         },
         "json5": {
             "hashes": [
@@ -1054,6 +1052,9 @@
             "version": "==2.4"
         },
         "jsonschema": {
+            "extras": [
+                "format-nongpl"
+            ],
             "hashes": [
                 "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb",
                 "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"
@@ -1400,7 +1401,6 @@
             ],
             "markers": "python_version >= '3.8'",
             "version": "==3.7.1"
-
         },
         "matplotlib-inline": {
             "hashes": [
@@ -1958,7 +1958,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.39"
         },
         "psutil": {
@@ -1978,7 +1978,7 @@
                 "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
                 "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "sys_platform != 'cygwin'",
             "version": "==5.9.5"
         },
         "psycopg2-binary": {
@@ -2442,7 +2442,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.2"
         },
         "python-dotenv": {
@@ -2593,7 +2593,6 @@
             ],
             "markers": "python_version >= '3.6'",
             "version": "==24.0.1"
-
         },
         "pyzstd": {
             "hashes": [
@@ -3170,7 +3169,7 @@
                 "sha256:f41915a4e1f059dfad614b187bc06021fefb5fc5255bfe63abf8247d2f7a646a",
                 "sha256:f96def94576f857abf58e031ce881b5a3fc25cbec64b2bc4824824a8a4367af9"
             ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==3.19.1"
         },
         "six": {
@@ -3178,7 +3177,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "sniffio": {
@@ -3275,20 +3274,20 @@
         },
         "tornado": {
             "hashes": [
-                "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4",
-                "sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf",
-                "sha256:29e71c847a35f6e10ca3b5c2990a52ce38b233019d8e858b755ea6ce4dcdd19d",
-                "sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba",
-                "sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe",
-                "sha256:6a0848f1aea0d196a7c4f6772197cbe2abc4266f836b0aac76947872cd29b411",
-                "sha256:7efcbcc30b7c654eb6a8c9c9da787a851c18f8ccd4a5a3a95b05c7accfa068d2",
-                "sha256:834ae7540ad3a83199a8da8f9f2d383e3c3d5130a328889e4cc991acc81e87a0",
-                "sha256:b46a6ab20f5c7c1cb949c72c1994a4585d2eaa0be4853f50a03b5031e964fc7c",
-                "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f",
-                "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829"
+                "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f",
+                "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5",
+                "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d",
+                "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3",
+                "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2",
+                "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a",
+                "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16",
+                "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a",
+                "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17",
+                "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0",
+                "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==6.3.2"
+            "index": "pypi",
+            "version": "==6.3.3"
         },
         "tqdm": {
             "hashes": [
@@ -3672,7 +3671,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.39"
         },
         "ptyprocess": {
@@ -3702,7 +3701,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "stack-data": {

From c585dec276f1ae953a708962539cd0ee836403d0 Mon Sep 17 00:00:00 2001
From: Brad <bradford.bates@noaa.gov>
Date: Fri, 18 Aug 2023 16:40:43 +0000
Subject: [PATCH 267/359] v4.3.15.2 Adds inundation and mosaic wrapper with
 polygon merging functionality (#948)

---
 docs/CHANGELOG.md                |  13 +++
 tools/inundate_gms.py            |   5 +-
 tools/inundate_mosaic_wrapper.py | 143 +++++++++++++++++++++++++++++++
 tools/mosaic_inundation.py       |  34 ++++++++
 4 files changed, 191 insertions(+), 4 deletions(-)
 create mode 100644 tools/inundate_mosaic_wrapper.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 76e4462e6..3d7376191 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.3.15.2 - 2023-07-18 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/948)
+
+Adds a script to produce inundation maps (extent TIFs, polygons, and depth grids) given a flow file and hydrofabric outputs. This is meant to make it easier to team members and external collaborators to produce inundation maps.
+
+### Additions
+- `data/`
+    - `/tools/inundate_mosaic_wrapper.py`: The script that performs the inundation and mosaicking processes.
+    - `/tools/mosaic_inundation.py`: Add function (mosaic_final_inundation_extent_to_poly).
+  
+<br/><br/>
+
 ## v4.3.15.1 - 2023-08-08 - [PR#960](https://github.com/NOAA-OWP/inundation-mapping/pull/960)
 
 Provides a scripted procedure for updating BLE benchmark data including downloading, extracting, and processing raw BLE data into benchmark inundation files (inundation rasters and discharge tables).
@@ -116,6 +128,7 @@ Fixes a couple of bugs that prevented inundation using HUC-level hydrotables. Up
 
 <br/><br/>
 
+
 ## v4.3.12.0 - 2023-07-05 - [PR#940](https://github.com/NOAA-OWP/inundation-mapping/pull/940)
 
 Refactor Point Calibration Database for synthetic rating curve adjustment to use `.parquet` files instead of a PostgreSQL database. 
diff --git a/tools/inundate_gms.py b/tools/inundate_gms.py
index 89e6d9cee..48f7aa5b9 100755
--- a/tools/inundate_gms.py
+++ b/tools/inundate_gms.py
@@ -249,7 +249,4 @@ def __inundate_gms_generator( hucs_branches,
     parser.add_argument('-o','--output-fileNames',help='Output CSV file with filenames for inundation rasters, inundation polygons, and depth rasters',required=False,default=None)
     parser.add_argument('-w','--num-workers', help='Number of Workers', required=False,default=1)
     parser.add_argument('-v','--verbose',help='Verbose printing',required=False,default=None,action='store_true')
-    
-    
-    # extract to dictionary and run
-    Inundate_gms( **vars(parser.parse_args()) )
+    
\ No newline at end of file
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
new file mode 100644
index 000000000..9178ea5b5
--- /dev/null
+++ b/tools/inundate_mosaic_wrapper.py
@@ -0,0 +1,143 @@
+import os, sys
+import argparse
+import errno
+import rasterio
+from rasterio.features import shapes
+import geopandas as gpd
+import numpy as np
+from shapely.geometry.polygon import Polygon
+from shapely.geometry.multipolygon import MultiPolygon
+from timeit import default_timer as timer
+sys.path.append('/foss_fim/tools')
+from mosaic_inundation import Mosaic_inundation, mosaic_final_inundation_extent_to_poly
+from inundate_gms import Inundate_gms
+
+
+def inundate_and_mosaic(hydrofabric_dir, huc, flow_file, inundation_raster, depths_raster, log_file, output_fileNames, 
+                        num_workers, remove_intermediate, verbose, mosaic_attribute, huc_dir):
+    
+    """
+    Helper function. Refer to produce_mosaicked_inundation for arg descriptions.
+    """
+    
+    map_file = Inundate_gms(  hydrofabric_dir = hydrofabric_dir, 
+                                     forecast = flow_file, 
+                                     num_workers = num_workers,
+                                     hucs = huc,
+                                     inundation_raster = inundation_raster,
+                                     inundation_polygon = None,
+                                     depths_raster = depths_raster,
+                                     verbose = verbose,
+                                     log_file = None,
+                                     output_fileNames = None )
+
+    print("Mosaicking extent for " + huc + "...")
+    # Call Mosaic_inundation
+    Mosaic_inundation( map_file,
+                        mosaic_attribute = mosaic_attribute,
+                        mosaic_output = inundation_raster if depths_raster == None else depths_raster,
+                        mask = os.path.join(huc_dir,'wbd.gpkg'),
+                        unit_attribute_name = 'huc8',
+                        nodata = -9999,
+                        workers = 1,
+                        remove_inputs = remove_intermediate,
+                        subset = None,
+                        verbose = verbose )
+    
+
+def produce_mosaicked_inundation(hydrofabric_dir, huc, flow_file, inundation_raster, inundation_polygon, depths_raster,
+                                 log_file, output_fileNames, num_workers, remove_intermediate, verbose):
+    """
+    This function calls Inundate_gms and Mosaic_inundation to produce inundation maps. Possible outputs include inundation rasters
+    encoded by HydroID (negative HydroID for dry and positive HydroID for wet), polygons depicting extent, and depth rasters. The
+    function requires a flow file organized by NWM feature_id and discharge in cms. "feature_id" and "discharge" columns MUST be
+    present in the flow file.
+    
+    Args:
+        hydrofabric_dir (str): Directory path to hydrofabric directory where FIM outputs were written by fim_pipeline.
+        huc (str): The HUC for which to produce mosaicked inundation files.
+        flow_file (str): Directory path to flow file to be used for inundation. feature_ids in flow_file should be present in supplied HUC.
+        inundation_raster (str): Full path to output inundation raster (encoded by positive and negative HydroIDs).
+        inuntation_polygon (str): Full path to output inundation polygon. Optional.
+        depths_raster (str): Full path to output depths_raster. Pixel values will be in meters. Optional.
+        log_file (str): Full path to log file to write logs to. Optional. Not tested.
+        output_fileNames (str): Full path to CSV containing paths to output file names. Optional. Not tested.
+        num_workers (int): Number of parallel jobs to run.
+        keep_intermediate (bool): Option to keep intermediate files. Not tested.
+        verbose (bool): Print verbose messages to screen. Not tested.
+        
+    """
+    
+    # Check that output directory exists. Notify user that output directory will be created if not.
+    for output_file in [inundation_raster, inundation_polygon, depths_raster]:
+        if output_file == None:
+            continue
+        parent_dir = os.path.split(output_file)[0]
+        if not os.path.exists(parent_dir):
+            print("Parent directory for " + os.path.split(output_file)[1] + " does not exist. The parent directory will be produced.")
+            os.makedirs(parent_dir)
+    
+    # Check that hydrofabric_dir exists
+    if not os.path.exists(hydrofabric_dir):
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), hydrofabric_dir)
+        
+    # Check that huc folder exists in the hydrofabric_dir.
+    if not os.path.exists(os.path.join(hydrofabric_dir, huc)):
+        raise FileNotFoundError((errno.ENOENT, os.strerror(errno.ENOENT), os.path.join(hydrofabric_dir, huc)))
+        
+    # Check that flow file exists
+    if not os.path.exists(flow_file):
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), flow_file)
+    
+    # Check job numbers and raise error if necessary
+    total_cpus_available = os.cpu_count() - 1
+    if num_workers > total_cpus_available:
+        raise ValueError('The number of workers (-w), {}, '\
+                          'exceeds your machine\'s available CPU count minus one ({}). '\
+                          'Please lower the num_workers.'.format(num_workers, total_cpus_available))
+    
+    huc_dir = os.path.join(hydrofabric_dir, huc)
+    print("Running inundate for " + huc + "...")
+    # Call Inundate_gms
+    fn_inundation_raster, fn_depths_raster, mosaic_attribute = inundation_raster, None, 'inundation_rasters'
+    
+    inundate_and_mosaic(hydrofabric_dir, huc, flow_file, fn_inundation_raster, fn_depths_raster,
+                                 log_file, output_fileNames, num_workers, remove_intermediate, verbose, mosaic_attribute, huc_dir)
+    
+    # Produce depths if instructed
+    if depths_raster != None:
+        print("Computing depths for " + huc + "...")
+        fn_inundation_raster, fn_depths_raster, mosaic_attribute = None, depths_raster, 'depths_rasters'
+
+        inundate_and_mosaic(hydrofabric_dir, huc, flow_file, fn_inundation_raster, fn_depths_raster,
+                                     log_file, output_fileNames, num_workers, remove_intermediate, verbose, mosaic_attribute, huc_dir)
+    else:
+        pass
+    
+    if inundation_polygon != None:
+        mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon)
+                
+    print("Mosaicking complete.")
+    
+if __name__ == '__main__':
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Helpful utility to produce mosaicked inundation extents (raster and poly) and depths.')
+    parser.add_argument('-y','--hydrofabric_dir', help='Directory path to FIM hydrofabric by processing unit.', required=True)
+    parser.add_argument('-u','--huc',help='List of HUCS to run',required=True,default="",type=str,)
+    parser.add_argument('-f','--flow_file',help='Discharges in CMS as CSV file. "feature_id" and "discharge" columns MUST be supplied.',required=True)
+    parser.add_argument('-i','--inundation-raster',help='Inundation raster output.',required=True,default=None)
+    parser.add_argument('-p','--inundation-polygon',help='Inundation polygon output. Only writes if designated.',required=False,default=None)
+    parser.add_argument('-d','--depths-raster',help='Depths raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
+    parser.add_argument('-l','--log-file',help='Log-file to store level-path exceptions. Not tested.',required=False,default=None)
+    parser.add_argument('-o','--output-fileNames',help='Output CSV file with filenames for inundation rasters, inundation polygons, and depth rasters. Not tested.',required=False,default=None)
+    parser.add_argument('-w','--num-workers', help='Number of workers.', required=False,default=1, type=int)
+    parser.add_argument('-r','--remove-intermediate',help='Keep intermediate products, i.e. individual branch inundation.',required=False,default=False,action='store_true')
+    parser.add_argument('-v','--verbose',help='Verbose printing. Not tested.',required=False,default=False,action='store_true')
+    
+    start = timer()
+
+    # Extract to dictionary and run
+    produce_mosaicked_inundation( **vars(parser.parse_args()) )
+
+    print(f'Completed in {round((timer() - start)/60, 2)} minutes.')
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 215e6511c..4ec312655 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -6,12 +6,14 @@
 import pandas as pd
 import sys
 
+
 from glob import glob
 from overlapping_inundation import OverlapWindowMerge
 from tqdm import tqdm
 from utils.shared_variables import elev_raster_ndv
 from utils.shared_functions import FIM_Helpers as fh
 
+
 def Mosaic_inundation( map_file,
                        mosaic_attribute = 'inundation_rasters',
                        mosaic_output = None,
@@ -132,7 +134,39 @@ def mosaic_by_unit(inundation_maps_list,
             if inun_map is not None:
                 if os.path.isfile(inun_map):
                     os.remove(inun_map)
+                    
+
+def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon):
+    import numpy as np
+    import rasterio
+    from rasterio.features import shapes
+    from shapely.geometry.polygon import Polygon
+    from shapely.geometry.multipolygon import MultiPolygon
+    import geopandas as gpd
+    
+    with rasterio.open(inundation_raster) as src:
+        # Open inundation_raster using rasterio.
+        image = src.read(1)
+        mask = image > 0
+        print("Producing merged polygon...")
+    
+        # Use numpy.where operation to reclassify depth_array on the condition that the pixel values are > 0.
+        reclass_inundation_array = np.where((image>0) & (image != src.nodata), 1, 0).astype('uint8')
+
+        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
 
+        # Aggregate shapes
+        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(reclass_inundation_array, mask=reclass_inundation_array>0,transform=src.transform)))
+
+        # Convert list of shapes to polygon, then dissolve
+        extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=src.crs)
+        extent_poly_diss = extent_poly.dissolve(by='extent')
+        extent_poly_diss["geometry"] = [MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in extent_poly_diss["geometry"]]
+        
+        # Write polygon
+        extent_poly_diss.to_file(inundation_polygon, driver='GPKG')
+
+    
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Mosaic GMS Inundation Rasters')

From dc2e84388a777e20783a249ca7e5b3b78c03b3c3 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 1 Sep 2023 10:16:59 -0500
Subject: [PATCH 268/359] v4.3.15.3 Hotfix for stacked bar Eval Plots (#983)

---
 docs/CHANGELOG.md              |   9 +
 tools/eval_plots_stackedbar.py | 734 ++++++++++++++++++++++++---------
 2 files changed, 549 insertions(+), 194 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3d7376191..7894adb07 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.15.3 - 2023-09-01 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/983)
+
+### Changes  
+
+This hotfix addresses some bugs introduced in the pandas upgrade.
+
+- `/tools/eval_plots_stackedbar.py`: 2 lines were changed to work with the pandas upgrade. Added an argument for a `groupby` median call and fixed a bug with the pandas `query`. Also updated with Black compliance.
+
+<br/><br/>
 
 ## v4.3.15.2 - 2023-07-18 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/948)
 
diff --git a/tools/eval_plots_stackedbar.py b/tools/eval_plots_stackedbar.py
index 8cf8705fc..533516a26 100644
--- a/tools/eval_plots_stackedbar.py
+++ b/tools/eval_plots_stackedbar.py
@@ -4,321 +4,667 @@
 import sys
 import argparse
 from pathlib import Path
-sys.path.append('/foss_fim/tools/')
+
+sys.path.append("/foss_fim/tools/")
 from tools_shared_variables import BAD_SITES
 
 from matplotlib.patches import Patch
 from matplotlib.lines import Line2D
 import mpl_toolkits.axisartist as AA
-mag_dict = {'action':'1_act', 'minor':'2_min', 'moderate':'3_mod', 'major':'4_maj',
-           '100yr':100, '500yr':500, '2yr':2,'5yr':5, 
-            '10yr':10, '25yr':25, '50yr':50, '200yr':200}
+
+mag_dict = {
+    "action": "1_act",
+    "minor": "2_min",
+    "moderate": "3_mod",
+    "major": "4_maj",
+    "100yr": 100,
+    "500yr": 500,
+    "2yr": 2,
+    "5yr": 5,
+    "10yr": 10,
+    "25yr": 25,
+    "50yr": 50,
+    "200yr": 200,
+}
 inverted_mag = dict((v, k) for k, v in mag_dict.items())
 
+
 def eval_plot_stack_data_prep(metric_csv, versions=[]):
-    
     # Load in FIM 4 CSV and select the proper version metrics
-    metrics = pd.read_csv(metric_csv, dtype={'huc':str})
+    metrics = pd.read_csv(metric_csv, dtype={"huc": str})
     if versions:
         versions = list(versions)
         # Check to make sure requested versions are in the metrics file
         for v in versions:
-            assert v in metrics.version.unique(), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
+            assert (
+                v in metrics.version.unique()
+            ), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
         # Filter the metrics to the requested versions
         metrics = metrics.loc[metrics.version.isin(versions)]
     # Change the magnitudes to make them more sort-friendly for the plot below
     metrics.magnitude = metrics.magnitude.apply(lambda m: mag_dict[m])
     # Fill the non-AHPS sites with values for nws_lid
-    metrics.nws_lid.fillna(metrics.huc +'_'+ metrics.benchmark_source, inplace=True)
+    metrics.nws_lid.fillna(metrics.huc + "_" + metrics.benchmark_source, inplace=True)
     # Create multi-index for easy merging and plotting
-    metrics.set_index(['benchmark_source', 'nws_lid', 'magnitude'], inplace=True, drop=False)
-    
+    metrics.set_index(
+        ["benchmark_source", "nws_lid", "magnitude"], inplace=True, drop=False
+    )
+
     # Normalize data to the total of TP + FN
-    metrics['TP_FN_sum'] = metrics['false_negatives_count'] + metrics['true_positives_count']
-    metrics['TP_norm'] = metrics['true_positives_count'] / metrics['TP_FN_sum']
-    metrics['FN_norm'] = metrics['false_negatives_count'] / metrics['TP_FN_sum']
-    metrics['FP_norm'] = metrics['false_positives_count'] / metrics['TP_FN_sum']
-    metrics['FP_norm'].replace([np.inf, -np.inf], np.nan, inplace=True) # some bad sites will divide by 0
-    
+    metrics["TP_FN_sum"] = (
+        metrics["false_negatives_count"] + metrics["true_positives_count"]
+    )
+    metrics["TP_norm"] = metrics["true_positives_count"] / metrics["TP_FN_sum"]
+    metrics["FN_norm"] = metrics["false_negatives_count"] / metrics["TP_FN_sum"]
+    metrics["FP_norm"] = metrics["false_positives_count"] / metrics["TP_FN_sum"]
+    metrics["FP_norm"].replace(
+        [np.inf, -np.inf], np.nan, inplace=True
+    )  # some bad sites will divide by 0
+
     return metrics
 
-def eval_plot_stack_indiv(metric_csv, versions, outfig, category):
 
+def eval_plot_stack_indiv(metric_csv, versions, outfig, category):
     # Check inputs
-    assert category in ('nws','usgs','ble','ifc','ras2fim'), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
-    
+    assert category in (
+        "nws",
+        "usgs",
+        "ble",
+        "ifc",
+        "ras2fim",
+    ), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
+
     metrics_df = eval_plot_stack_data_prep(metric_csv, versions)
 
     # Filter the plotting data to the selected category
     data = metrics_df.loc[(category)]
     num_subplots = len(data.nws_lid.unique())
-    xmax = data['FP_norm'].max() + 1
+    xmax = data["FP_norm"].max() + 1
     # Save version input order for plotting
-    version_dict = {v:i for i,v in enumerate(versions)}
-    data = data.assign(plot_order=data['version'])
-    data['plot_order'] = data.plot_order.map(version_dict)
-    data['magnitude_'] = data.magnitude
+    version_dict = {v: i for i, v in enumerate(versions)}
+    data = data.assign(plot_order=data["version"])
+    data["plot_order"] = data.plot_order.map(version_dict)
+    data["magnitude_"] = data.magnitude
     # Create the plot
-    fig, ax = plt.subplots(num_subplots, 1, figsize=(8, len(data)*0.25), dpi=100, facecolor='white', subplot_kw={'axes_class':AA.Axes})
+    fig, ax = plt.subplots(
+        num_subplots,
+        1,
+        figsize=(8, len(data) * 0.25),
+        dpi=100,
+        facecolor="white",
+        subplot_kw={"axes_class": AA.Axes},
+    )
     # Create a subplot for every site (nws_lid)
     for i, site in enumerate(sorted(data.nws_lid.unique())):
         subplot_data = data.loc[(site)]
-        subplot_data = subplot_data.sort_values(['magnitude_','plot_order'], ascending=False)
-        new_y = [j*1.25 for j in range(len(subplot_data))]
+        subplot_data = subplot_data.sort_values(
+            ["magnitude_", "plot_order"], ascending=False
+        )
+        new_y = [j * 1.25 for j in range(len(subplot_data))]
         # Stacked horizontal bar plots
-        ax[i].barh(y=new_y, width='TP_norm', left=0.0, color='#2c7bb6',
-                   linewidth=0.5, data=subplot_data)
-        ax[i].barh(y=new_y, width='FN_norm', left='TP_norm', color='#fdae61',
-                   linewidth=0.5, data=subplot_data)
-        ax[i].barh(y=new_y, width='FP_norm', left=1.0, color='#d7191c',
-                   linewidth=0.5, data=subplot_data)
+        ax[i].barh(
+            y=new_y,
+            width="TP_norm",
+            left=0.0,
+            color="#2c7bb6",
+            linewidth=0.5,
+            data=subplot_data,
+        )
+        ax[i].barh(
+            y=new_y,
+            width="FN_norm",
+            left="TP_norm",
+            color="#fdae61",
+            linewidth=0.5,
+            data=subplot_data,
+        )
+        ax[i].barh(
+            y=new_y,
+            width="FP_norm",
+            left=1.0,
+            color="#d7191c",
+            linewidth=0.5,
+            data=subplot_data,
+        )
         # Plot the CSI and MCC scores
-        ax[i].scatter(y=new_y, x=subplot_data['CSI'].array, c='k', s=15, marker=r'x', zorder=3, linewidth=0.75)
-        ax[i].scatter(y=new_y, x=subplot_data['MCC'].array, s=15, marker=r'o', zorder=3, linewidths=.75, facecolor='None', edgecolors='k')
+        ax[i].scatter(
+            y=new_y,
+            x=subplot_data["CSI"].array,
+            c="k",
+            s=15,
+            marker=r"x",
+            zorder=3,
+            linewidth=0.75,
+        )
+        ax[i].scatter(
+            y=new_y,
+            x=subplot_data["MCC"].array,
+            s=15,
+            marker=r"o",
+            zorder=3,
+            linewidths=0.75,
+            facecolor="None",
+            edgecolors="k",
+        )
         # Various axis customizations
-        ax[i].set_yticks(new_y, labels=subplot_data.version + ' ' + subplot_data.magnitude.map(inverted_mag).astype(str))
-        ax[i].axis["left"].label.set(visible=True, text=site, rotation=90, pad=10, ha='right')
+        ax[i].set_yticks(
+            new_y,
+            labels=subplot_data.version
+            + " "
+            + subplot_data.magnitude.map(inverted_mag).astype(str),
+        )
+        ax[i].axis["left"].label.set(
+            visible=True, text=site, rotation=90, pad=10, ha="right"
+        )
         ax[i].axis["left"].major_ticks.set(tick_out=True)
-        ax[i].axis["left"].major_ticklabels.set(ha='left')
+        ax[i].axis["left"].major_ticklabels.set(ha="left")
         ax[i].set_xlim(0, xmax)
-        ax[i].set_ylim(-1, new_y[-1]+1)
+        ax[i].set_ylim(-1, new_y[-1] + 1)
         num_mags = len(subplot_data.magnitude.unique())
-        hlines = [j for j in np.linspace(new_y[-1]/num_mags, new_y[-1], num_mags-1, endpoint=False)]
-        ax[i].hlines(hlines, xmin=0, xmax=xmax, colors=['0.9' if i % 2 == 0 else 'w']*(num_mags-1), linestyles='dotted', zorder=0)
-        ax[i].grid(axis='x', color='0.8', zorder=0)
-        ax[i].axis['bottom'].set_visible(True if i == num_subplots-1 else False)
-        ax[i].axis['top'].set_visible(True if i == 0 else False)
-        ax[i].axis['top'].major_ticklabels.set_visible(True if i == 0 else False)
-        ax[i].set_facecolor('w' if i % 2 == 0 else '0.9')
+        hlines = [
+            j
+            for j in np.linspace(
+                new_y[-1] / num_mags, new_y[-1], num_mags - 1, endpoint=False
+            )
+        ]
+        ax[i].hlines(
+            hlines,
+            xmin=0,
+            xmax=xmax,
+            colors=["0.9" if i % 2 == 0 else "w"] * (num_mags - 1),
+            linestyles="dotted",
+            zorder=0,
+        )
+        ax[i].grid(axis="x", color="0.8", zorder=0)
+        ax[i].axis["bottom"].set_visible(True if i == num_subplots - 1 else False)
+        ax[i].axis["top"].set_visible(True if i == 0 else False)
+        ax[i].axis["top"].major_ticklabels.set_visible(True if i == 0 else False)
+        ax[i].set_facecolor("w" if i % 2 == 0 else "0.9")
         # Label sites that have been identified as "Bad"
         if site and site in BAD_SITES:
-            ax[i].text(xmax/2, new_y[-1]/2, '--BAD SITE--', horizontalalignment='center', verticalalignment='center')
-            ax[i].set_facecolor('0.67')
+            ax[i].text(
+                xmax / 2,
+                new_y[-1] / 2,
+                "--BAD SITE--",
+                horizontalalignment="center",
+                verticalalignment="center",
+            )
+            ax[i].set_facecolor("0.67")
     plt.subplots_adjust(wspace=0, hspace=0)
-    ax[0].set_title(f'{category.upper()} FIM Evaluation | Individual Sites', loc='center', pad=40)
-    TP_patch = Patch(color='#2c7bb6', linewidth=0.5, label=f'True Positive')
-    FN_patch = Patch(color='#fdae61', linewidth=0.5, label=f'False Negative')
-    FP_patch = Patch(color='#d7191c', linewidth=0.5, label=f'False Positive')
-    x_marker = Line2D([0], [0], marker='x', color='None', markeredgecolor='k', markerfacecolor='k', label='CSI Score', markersize=5, linewidth=0.5)
-    o_marker = Line2D([0], [0], marker='o', color='None', markeredgecolor='k', markerfacecolor='None', label='MCC Score', markersize=5, linewidth=10)
+    ax[0].set_title(
+        f"{category.upper()} FIM Evaluation | Individual Sites", loc="center", pad=40
+    )
+    TP_patch = Patch(color="#2c7bb6", linewidth=0.5, label=f"True Positive")
+    FN_patch = Patch(color="#fdae61", linewidth=0.5, label=f"False Negative")
+    FP_patch = Patch(color="#d7191c", linewidth=0.5, label=f"False Positive")
+    x_marker = Line2D(
+        [0],
+        [0],
+        marker="x",
+        color="None",
+        markeredgecolor="k",
+        markerfacecolor="k",
+        label="CSI Score",
+        markersize=5,
+        linewidth=0.5,
+    )
+    o_marker = Line2D(
+        [0],
+        [0],
+        marker="o",
+        color="None",
+        markeredgecolor="k",
+        markerfacecolor="None",
+        label="MCC Score",
+        markersize=5,
+        linewidth=10,
+    )
     # Get the height of the figure in pixels so we can put the legend in a consistent position
-    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    ax_pixel_height = (
+        ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    )
     handles = [TP_patch, FN_patch, FP_patch, x_marker, o_marker]
-    ax[0].legend(loc='center', ncol=len(handles), handles=handles, fontsize=8, columnspacing=1,
-                bbox_to_anchor=(0.5, (0.3+ax_pixel_height)/ax_pixel_height))
-    plt.savefig(outfig, bbox_inches='tight')
-    
+    ax[0].legend(
+        loc="center",
+        ncol=len(handles),
+        handles=handles,
+        fontsize=8,
+        columnspacing=1,
+        bbox_to_anchor=(0.5, (0.3 + ax_pixel_height) / ax_pixel_height),
+    )
+    plt.savefig(outfig, bbox_inches="tight")
+
     return metrics_df
 
-def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
 
+def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
     # Check inputs
-    assert category in ('nws','usgs','ble','ifc','ras2fim'), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
-    
+    assert category in (
+        "nws",
+        "usgs",
+        "ble",
+        "ifc",
+        "ras2fim",
+    ), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
+
     metrics_df = eval_plot_stack_data_prep(metric_csv, versions)
     metrics_df = metrics_df.loc[~metrics_df.nws_lid.isin(BAD_SITES)]
-    grouped = metrics_df.reset_index(drop=True).groupby(['benchmark_source','version','magnitude'], sort=False)
-    count_df = grouped.count()['CSI']
-    metrics_df = grouped.median()
-    metrics_df['TP_norm_q1'] = metrics_df['TP_norm'] - grouped['TP_norm'].quantile(0.25)
-    metrics_df['TP_norm_q3'] = grouped['TP_norm'].quantile(0.75) - metrics_df['TP_norm']
-    metrics_df['FP_norm_q1'] = metrics_df['FP_norm'] - grouped['FP_norm'].quantile(0.25)
-    metrics_df['FP_norm_q3'] = grouped['FP_norm'].quantile(0.75) - metrics_df['FP_norm']
-    
+    grouped = metrics_df.reset_index(drop=True).groupby(
+        ["benchmark_source", "version", "magnitude"], sort=False
+    )
+    count_df = grouped.count()["CSI"]
+    metrics_df = grouped.median(numeric_only=True)
+    metrics_df["TP_norm_q1"] = metrics_df["TP_norm"] - grouped["TP_norm"].quantile(0.25)
+    metrics_df["TP_norm_q3"] = grouped["TP_norm"].quantile(0.75) - metrics_df["TP_norm"]
+    metrics_df["FP_norm_q1"] = metrics_df["FP_norm"] - grouped["FP_norm"].quantile(0.25)
+    metrics_df["FP_norm_q3"] = grouped["FP_norm"].quantile(0.75) - metrics_df["FP_norm"]
+
     # Filter the plotting data to the selected category
     data = metrics_df.loc[(category)].swaplevel()
     num_subplots = len(data.reset_index().magnitude.unique())
     num_ver = len(data.reset_index().version.unique())
-    xmax = 1+ data['FP_norm'].max() + data['FP_norm_q3'][data['FP_norm'].idxmax()]
+    xmax = 1 + data["FP_norm"].max() + data["FP_norm_q3"][data["FP_norm"].idxmax()]
     # Save version input order for plotting
-    version_dict = {v:i for i,v in enumerate(versions)}
-    data = data.assign(plot_order=data.index.get_level_values('version'))
-    data['plot_order'] = data.plot_order.map(version_dict)
-    
+    version_dict = {v: i for i, v in enumerate(versions)}
+    data = data.assign(plot_order=data.index.get_level_values("version"))
+    data["plot_order"] = data.plot_order.map(version_dict)
+
     # Create the plot
-    fig, ax = plt.subplots(num_subplots, 1, figsize=(8, len(data)*0.25), dpi=100, facecolor='white', subplot_kw={'axes_class':AA.Axes})
+    fig, ax = plt.subplots(
+        num_subplots,
+        1,
+        figsize=(8, len(data) * 0.25),
+        dpi=100,
+        facecolor="white",
+        subplot_kw={"axes_class": AA.Axes},
+    )
     # Create a subplot for every flow (nws_lid)
     for i, mag in enumerate(sorted(data.reset_index().magnitude.unique())):
         subplot_data = data.loc[(mag)]
-        subplot_data = subplot_data.sort_values(['plot_order'], ascending=False)
-        new_y = [j*1.25 for j in range(len(subplot_data))]
+        subplot_data = subplot_data.sort_values(["plot_order"], ascending=False)
+        new_y = [j * 1.25 for j in range(len(subplot_data))]
         # Stacked horizontal bar plots
-        ax[i].barh(y=new_y, width='TP_norm', left=0.0, color='#2c7bb6', 
-                   linewidth=0.5, data=subplot_data,
-                   xerr=[subplot_data['TP_norm_q1'], subplot_data['TP_norm_q3']] if show_iqr else None,
-                   error_kw=dict(elinewidth=1))
-        ax[i].barh(y=new_y, width='FN_norm', left='TP_norm', color='#fdae61', 
-                   linewidth=0.5, data=subplot_data)
-        ax[i].barh(y=new_y, width='FP_norm', left=1.0, color='#d7191c',
-                   linewidth=0.5, data=subplot_data, 
-                   xerr=[subplot_data['FP_norm_q1'], subplot_data['FP_norm_q3']] if show_iqr else None,
-                   error_kw=dict(elinewidth=1))
+        ax[i].barh(
+            y=new_y,
+            width="TP_norm",
+            left=0.0,
+            color="#2c7bb6",
+            linewidth=0.5,
+            data=subplot_data,
+            xerr=[subplot_data["TP_norm_q1"], subplot_data["TP_norm_q3"]]
+            if show_iqr
+            else None,
+            error_kw=dict(elinewidth=1),
+        )
+        ax[i].barh(
+            y=new_y,
+            width="FN_norm",
+            left="TP_norm",
+            color="#fdae61",
+            linewidth=0.5,
+            data=subplot_data,
+        )
+        ax[i].barh(
+            y=new_y,
+            width="FP_norm",
+            left=1.0,
+            color="#d7191c",
+            linewidth=0.5,
+            data=subplot_data,
+            xerr=[subplot_data["FP_norm_q1"], subplot_data["FP_norm_q3"]]
+            if show_iqr
+            else None,
+            error_kw=dict(elinewidth=1),
+        )
         # Plot the CSI and MCC scores
-        ax[i].scatter(y=new_y, x=subplot_data['CSI'].array, c='k', s=15, marker=r'x', zorder=3, linewidth=0.75)
-        ax[i].scatter(y=new_y, x=subplot_data['MCC'].array, s=15, marker=r'o', zorder=3, linewidths=.75, facecolor='None', edgecolors='k')
+        ax[i].scatter(
+            y=new_y,
+            x=subplot_data["CSI"].array,
+            c="k",
+            s=15,
+            marker=r"x",
+            zorder=3,
+            linewidth=0.75,
+        )
+        ax[i].scatter(
+            y=new_y,
+            x=subplot_data["MCC"].array,
+            s=15,
+            marker=r"o",
+            zorder=3,
+            linewidths=0.75,
+            facecolor="None",
+            edgecolors="k",
+        )
         # Various axis customizations
-        ax[i].set_yticks(new_y, labels=subplot_data.index)#, ha='left')
-        ax[i].axis["left"].label.set(visible=True, text=inverted_mag[mag], rotation=90, pad=10, ha='right')
+        ax[i].set_yticks(new_y, labels=subplot_data.index)  # , ha='left')
+        ax[i].axis["left"].label.set(
+            visible=True, text=inverted_mag[mag], rotation=90, pad=10, ha="right"
+        )
         ax[i].axis["left"].major_ticks.set(tick_out=True)
-        ax[i].axis["left"].major_ticklabels.set(ha='left')
+        ax[i].axis["left"].major_ticklabels.set(ha="left")
         n = count_df.loc[(category, versions[0], mag)]
-        ax[i].axis["right"].label.set(visible=True, text=f'n={n}', rotation=270, pad=5, ha='left')
+        ax[i].axis["right"].label.set(
+            visible=True, text=f"n={n}", rotation=270, pad=5, ha="left"
+        )
         ax[i].set_xlim(0, xmax)
-        ax[i].set_ylim(-1.25, num_ver+1.25)
-        ax[i].grid(axis='x', color='0.8', zorder=0)
-        ax[i].axis['bottom'].set_visible(True if i == num_subplots-1 else False)
-        ax[i].axis['top'].set_visible(True if i == 0 else False)
-        ax[i].axis['top'].major_ticklabels.set_visible(True if i == 0 else False)
-        ax[i].set_facecolor('w' if i % 2 == 0 else '0.9')
+        ax[i].set_ylim(-1.25, num_ver + 1.25)
+        ax[i].grid(axis="x", color="0.8", zorder=0)
+        ax[i].axis["bottom"].set_visible(True if i == num_subplots - 1 else False)
+        ax[i].axis["top"].set_visible(True if i == 0 else False)
+        ax[i].axis["top"].major_ticklabels.set_visible(True if i == 0 else False)
+        ax[i].set_facecolor("w" if i % 2 == 0 else "0.9")
     plt.subplots_adjust(wspace=0, hspace=0)
-    ax[0].set_title(f'{category.upper()} FIM Evaluation', loc='center', pad=35)
-    TP_patch = Patch(color='#2c7bb6', linewidth=0.5, label=f'True Positive')
-    FN_patch = Patch(color='#fdae61', linewidth=0.5, label=f'False Negative')
-    FP_patch = Patch(color='#d7191c', linewidth=0.5, label=f'False Positive')
-    x_marker = Line2D([0], [0], marker='x', color='None', markeredgecolor='k', markerfacecolor='k', label='CSI Score', markersize=5, linewidth=10)
-    o_marker = Line2D([0], [0], marker='o', color='None', markeredgecolor='k', markerfacecolor='None', label='MCC Score', markersize=5, linewidth=10)
+    ax[0].set_title(f"{category.upper()} FIM Evaluation", loc="center", pad=35)
+    TP_patch = Patch(color="#2c7bb6", linewidth=0.5, label=f"True Positive")
+    FN_patch = Patch(color="#fdae61", linewidth=0.5, label=f"False Negative")
+    FP_patch = Patch(color="#d7191c", linewidth=0.5, label=f"False Positive")
+    x_marker = Line2D(
+        [0],
+        [0],
+        marker="x",
+        color="None",
+        markeredgecolor="k",
+        markerfacecolor="k",
+        label="CSI Score",
+        markersize=5,
+        linewidth=10,
+    )
+    o_marker = Line2D(
+        [0],
+        [0],
+        marker="o",
+        color="None",
+        markeredgecolor="k",
+        markerfacecolor="None",
+        label="MCC Score",
+        markersize=5,
+        linewidth=10,
+    )
     # Get the height of the figure in pixels so we can put the legend in a consistent position
-    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    ax_pixel_height = (
+        ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    )
     handles = [TP_patch, FN_patch, FP_patch, x_marker, o_marker]
-    ax[0].legend(loc='center', ncol=len(handles), handles=handles, fontsize=8, columnspacing=1,
-                bbox_to_anchor=(0.5, (0.3+ax_pixel_height)/ax_pixel_height))
-    plt.savefig(outfig, bbox_inches='tight')
-    
+    ax[0].legend(
+        loc="center",
+        ncol=len(handles),
+        handles=handles,
+        fontsize=8,
+        columnspacing=1,
+        bbox_to_anchor=(0.5, (0.3 + ax_pixel_height) / ax_pixel_height),
+    )
+    plt.savefig(outfig, bbox_inches="tight")
+
     return data
 
-def diff_bar_plots(versions, metric_csv, category, outfig, stat='CSI'): 
-    
+
+def diff_bar_plots(versions, metric_csv, category, outfig, stat="CSI"):
     # Check inputs
-    assert category in ('nws','usgs','ble','ifc','ras2fim'), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
-    assert stat in ('CSI', 'MCC', 'TPR', 'PND', 'FAR'), f"stat must be one of ('CSI', 'MCC', 'TPR', 'PND', 'FAR'), not {stat}"
-    assert len(versions) == 2, f"You can only produce the comparison plots when comparing 2 fim versions. You have input {len(versions)}."
+    assert category in (
+        "nws",
+        "usgs",
+        "ble",
+        "ifc",
+        "ras2fim",
+    ), f"category must be one of ('nws','usgs','ble','ifc','ras2fim'), not {category}"
+    assert stat in (
+        "CSI",
+        "MCC",
+        "TPR",
+        "PND",
+        "FAR",
+    ), f"stat must be one of ('CSI', 'MCC', 'TPR', 'PND', 'FAR'), not {stat}"
+    assert (
+        len(versions) == 2
+    ), f"You can only produce the comparison plots when comparing 2 fim versions. You have input {len(versions)}."
     # CSI, MCC, and TPR scores are better when closer to 1, however PND and FAR scores are better when closer to 0
-    better_score = 'positive' if stat in ('CSI', 'MCC', 'TPR') else 'negative' 
-    
+    better_score = "positive" if stat in ("CSI", "MCC", "TPR") else "negative"
+
     # Load in FIM 4 CSV and select the proper version metrics
-    metrics = pd.read_csv(metric_csv, dtype={'huc':str})
+    metrics = pd.read_csv(metric_csv, dtype={"huc": str})
     # Check to make sure requested versions are in the metrics file
     for v in versions:
-        assert v in metrics.version.unique(), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
+        assert (
+            v in metrics.version.unique()
+        ), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
     # Change the magnitudes to make them more sort-friendly for the plot below
     metrics.magnitude = metrics.magnitude.apply(lambda m: mag_dict[m])
     # Fill the non-AHPS sites with values for nws_lid
-    metrics.nws_lid.fillna(metrics.huc +'_'+ metrics.benchmark_source, inplace=True)
+    metrics.nws_lid.fillna(metrics.huc + "_" + metrics.benchmark_source, inplace=True)
     # Create multi-index for easy merging and plotting
-    metrics.set_index(['benchmark_source', 'nws_lid', 'magnitude'], inplace=True, drop=False)
+    metrics.set_index(
+        ["benchmark_source", "nws_lid", "magnitude"], inplace=True, drop=False
+    )
 
     # Separate versions into 2 dataframes
     metrics_base = metrics.loc[metrics.version == versions[0]].copy()
-    metrics_new  = metrics.loc[metrics.version == versions[1]].copy()
-    
+    metrics_new = metrics.loc[metrics.version == versions[1]].copy()
+
     # Create a new column in the base dataframe and merge it into the comparison dataframe
-    metrics_base[f'BASE_{stat}'] = metrics_base[stat]
+    metrics_base[f"BASE_{stat}"] = metrics_base[stat]
     metrics_base.drop(columns=stat, inplace=True)
-    fim_compare = metrics_base.merge(metrics_new[stat], left_index=True, right_index=True, how='inner').sort_index(level=0,sort_remaining=True)
-    fim_compare[f'{stat}_diff'] = fim_compare[stat] - fim_compare[f'BASE_{stat}']
+    fim_compare = metrics_base.merge(
+        metrics_new[stat], left_index=True, right_index=True, how="inner"
+    ).sort_index(level=0, sort_remaining=True)
+    fim_compare[f"{stat}_diff"] = fim_compare[stat] - fim_compare[f"BASE_{stat}"]
     # Color the boxes according to improved vs regressed scores. See definition of 'better_score' above for details.
-    if better_score == 'positive':
-        fim_compare['color'] = fim_compare.apply(lambda row: 'None' if row[f'{stat}_diff'] < 0 else 'g', axis=1)
-        fim_compare['edge_color'] = fim_compare.apply(lambda row: 'r' if row[f'{stat}_diff'] < 0 else 'g', axis=1)
-    elif better_score == 'negative':
-        fim_compare['color'] = fim_compare.apply(lambda row: 'None' if row[f'{stat}_diff'] > 0 else 'g', axis=1)
-        fim_compare['edge_color'] = fim_compare.apply(lambda row: 'r' if row[f'{stat}_diff'] > 0 else 'g', axis=1)
-    
+    if better_score == "positive":
+        fim_compare["color"] = fim_compare.apply(
+            lambda row: "None" if row[f"{stat}_diff"] < 0 else "g", axis=1
+        )
+        fim_compare["edge_color"] = fim_compare.apply(
+            lambda row: "r" if row[f"{stat}_diff"] < 0 else "g", axis=1
+        )
+    elif better_score == "negative":
+        fim_compare["color"] = fim_compare.apply(
+            lambda row: "None" if row[f"{stat}_diff"] > 0 else "g", axis=1
+        )
+        fim_compare["edge_color"] = fim_compare.apply(
+            lambda row: "r" if row[f"{stat}_diff"] > 0 else "g", axis=1
+        )
+
     # Filter the plotting data to the selected category
     data = fim_compare.loc[(category)]
     num_subplots = len(data.nws_lid.unique())
     num_mags = len(data.magnitude.unique())
     # Create the plot
-    fig, ax = plt.subplots(num_subplots, 1, figsize=(8, len(data)*0.18), dpi=100, facecolor='white')
+    fig, ax = plt.subplots(
+        num_subplots, 1, figsize=(8, len(data) * 0.18), dpi=100, facecolor="white"
+    )
     # Create a subplot for every site (nws_lid)
     for i, site in enumerate(data.nws_lid.unique()):
         subplot_data = data.loc[(site)]
-        ax[i].barh(y=np.arange(len(subplot_data)), width=f'{stat}_diff', left=f'BASE_{stat}', color='color',
-                   edgecolor='edge_color', linewidth=0.5, data=subplot_data, zorder=3)
-        ax[i].set_yticks(np.arange(len(subplot_data)), labels=subplot_data.index.map(inverted_mag).astype(str))
-        ax[i].set_xticks(np.linspace(0,1,11))
-        ax[i].set_ylabel(site, rotation='horizontal', labelpad=40)
+        ax[i].barh(
+            y=np.arange(len(subplot_data)),
+            width=f"{stat}_diff",
+            left=f"BASE_{stat}",
+            color="color",
+            edgecolor="edge_color",
+            linewidth=0.5,
+            data=subplot_data,
+            zorder=3,
+        )
+        ax[i].set_yticks(
+            np.arange(len(subplot_data)),
+            labels=subplot_data.index.map(inverted_mag).astype(str),
+        )
+        ax[i].set_xticks(np.linspace(0, 1, 11))
+        ax[i].set_ylabel(site, rotation="horizontal", labelpad=40)
         ax[i].set_xlim(0, 1)
-        ax[i].set_ylim(-0.5, num_mags-0.25)
-        ax[i].grid(axis='x', color='0.8', zorder=0)
-        ax[i].spines['bottom'].set_color('0' if i == num_subplots-1 else 'None') 
-        ax[i].spines['top'].set_color('0' if i == 0 else '0.8') 
-        ax[i].set_facecolor('w' if i % 2 == 0 else '0.9')
-        ax[i].tick_params(axis="x", 
-                          bottom=True if i == num_subplots-1 else False, 
-                          top=True if i == 0 else False, 
-                          labelbottom=True if i == num_subplots-1 else False, 
-                          labeltop=True if i == 0 else False)
+        ax[i].set_ylim(-0.5, num_mags - 0.25)
+        ax[i].grid(axis="x", color="0.8", zorder=0)
+        ax[i].spines["bottom"].set_color("0" if i == num_subplots - 1 else "None")
+        ax[i].spines["top"].set_color("0" if i == 0 else "0.8")
+        ax[i].set_facecolor("w" if i % 2 == 0 else "0.9")
+        ax[i].tick_params(
+            axis="x",
+            bottom=True if i == num_subplots - 1 else False,
+            top=True if i == 0 else False,
+            labelbottom=True if i == num_subplots - 1 else False,
+            labeltop=True if i == 0 else False,
+        )
         # Label sites that have been identified as "Bad"
         if site and site in BAD_SITES:
-            ax[i].text(0.1, 1.5, '--BAD SITE--', horizontalalignment='center', verticalalignment='center')
-            ax[i].set_facecolor('0.67')
+            ax[i].text(
+                0.1,
+                1.5,
+                "--BAD SITE--",
+                horizontalalignment="center",
+                verticalalignment="center",
+            )
+            ax[i].set_facecolor("0.67")
     plt.subplots_adjust(wspace=0, hspace=0)
-    ax[0].set_title(f'{category.upper()} {stat} Comparison\n{versions[0]}  v  {versions[1]}', loc='center', pad=45)
-    g_patch = Patch(color='g', linewidth=0.5, label=f'{stat} Score Improvement')
-    r_patch = Patch(facecolor='None', edgecolor='r', linewidth=0.5, label=f'{stat} Score Regression')
+    ax[0].set_title(
+        f"{category.upper()} {stat} Comparison\n{versions[0]}  v  {versions[1]}",
+        loc="center",
+        pad=45,
+    )
+    g_patch = Patch(color="g", linewidth=0.5, label=f"{stat} Score Improvement")
+    r_patch = Patch(
+        facecolor="None", edgecolor="r", linewidth=0.5, label=f"{stat} Score Regression"
+    )
     # Get the height of the figure in pixels so we can put the legend in a consistent position
-    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
-    ax[0].legend(loc='center', ncol=2, handles=[g_patch, r_patch], fontsize=8,
-                bbox_to_anchor=(0.5, (0.4+ax_pixel_height)/ax_pixel_height))
-    plt.savefig(outfig, bbox_inches='tight')
-    
+    ax_pixel_height = (
+        ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
+    )
+    ax[0].legend(
+        loc="center",
+        ncol=2,
+        handles=[g_patch, r_patch],
+        fontsize=8,
+        bbox_to_anchor=(0.5, (0.4 + ax_pixel_height) / ax_pixel_height),
+    )
+    plt.savefig(outfig, bbox_inches="tight")
+
     return fim_compare
 
-def iter_benchmarks(metric_csv, workspace, versions=[], individual_plots=False, show_iqr=False, diff_stat=False):
-    
+
+def iter_benchmarks(
+    metric_csv,
+    workspace,
+    versions=[],
+    individual_plots=False,
+    show_iqr=False,
+    diff_stat=False,
+):
     # Import metrics csv as DataFrame and initialize all_datasets dictionary
-    csv_df = pd.read_csv(metric_csv, dtype = {'huc':str})
+    csv_df = pd.read_csv(metric_csv, dtype={"huc": str})
 
     # If versions are supplied then filter out
     if versions:
-        #Filter out versions based on supplied version list
-        metrics = csv_df.query('version.str.startswith(tuple(@versions))', engine='python')
+        # Filter out versions based on supplied version list
+        metrics = csv_df.query("version == @versions")
     else:
         metrics = csv_df
 
     # Group by benchmark source
-    benchmark_by_source = metrics.groupby(['benchmark_source', 'extent_config'])
-    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:
-
+    benchmark_by_source = metrics.groupby(["benchmark_source", "extent_config"])
+    for (
+        benchmark_source,
+        extent_configuration,
+    ), benchmark_metrics in benchmark_by_source:
         # Define and create the output workspace as a subfolder within the supplied workspace
-        output_workspace = Path(workspace) / benchmark_source / extent_configuration.lower()
-        output_workspace.mkdir(parents = True, exist_ok = True)
-        output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar{'_indiv'if individual_plots else ''}.png"
+        output_workspace = (
+            Path(workspace) / benchmark_source / extent_configuration.lower()
+        )
+        output_workspace.mkdir(parents=True, exist_ok=True)
+        output_png = (
+            Path(output_workspace)
+            / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar{'_indiv'if individual_plots else ''}.png"
+        )
         if diff_stat:
-            output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_diff_plot_{diff_stat}.png"
-            diff_bar_plots(versions, metric_csv, category=benchmark_source, outfig=output_png, stat=diff_stat)
+            output_png = (
+                Path(output_workspace)
+                / f"{benchmark_source}_{extent_configuration.lower()}_diff_plot_{diff_stat}.png"
+            )
+            diff_bar_plots(
+                versions,
+                metric_csv,
+                category=benchmark_source,
+                outfig=output_png,
+                stat=diff_stat,
+            )
         elif individual_plots:
-            output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar_indiv.png"
-            eval_plot_stack_indiv(metric_csv=metric_csv, versions=versions, category=benchmark_source, outfig=output_png)
+            output_png = (
+                Path(output_workspace)
+                / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar_indiv.png"
+            )
+            eval_plot_stack_indiv(
+                metric_csv=metric_csv,
+                versions=versions,
+                category=benchmark_source,
+                outfig=output_png,
+            )
         else:
-            output_png = Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar.png"
-            eval_plot_stack(metric_csv=metric_csv, versions=versions, category=benchmark_source, show_iqr=show_iqr, outfig=output_png)
+            output_png = (
+                Path(output_workspace)
+                / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar.png"
+            )
+            eval_plot_stack(
+                metric_csv=metric_csv,
+                versions=versions,
+                category=benchmark_source,
+                show_iqr=show_iqr,
+                outfig=output_png,
+            )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     # Parse arguments
-    parser = argparse.ArgumentParser(description = f'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
-    parser.add_argument('-m','--metric_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
-    parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
-    parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
-    parser.add_argument('-i', '--site-plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
-    parser.add_argument('-iqr', '--show-iqr', help = 'If enabled, inter-quartile range error bars will be added.', action = 'store_true', required = False)
-    parser.add_argument('-d', '--diff-stat', help = 'This option creates diff plots instead of stacked bar plots. Only 2 versions can be used with this option. ' + \
-        "Input one of the following statistics to be used for comparison: ('CSI', 'MCC', 'TPR', 'PND', 'FAR').", default = False, required = False)
+    parser = argparse.ArgumentParser(
+        description=f"Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)"
+    )
+    parser.add_argument(
+        "-m",
+        "--metric_csv",
+        help="Metrics csv created from synthesize test cases.",
+        required=True,
+    )
+    parser.add_argument("-w", "--workspace", help="Output workspace", required=True)
+    parser.add_argument(
+        "-v",
+        "--versions",
+        help='List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.',
+        nargs="+",
+        default=[],
+    )
+    parser.add_argument(
+        "-i",
+        "--site-plots",
+        help="If enabled individual barplots for each site are created.",
+        action="store_true",
+        required=False,
+    )
+    parser.add_argument(
+        "-iqr",
+        "--show-iqr",
+        help="If enabled, inter-quartile range error bars will be added.",
+        action="store_true",
+        required=False,
+    )
+    parser.add_argument(
+        "-d",
+        "--diff-stat",
+        help="This option creates diff plots instead of stacked bar plots. Only 2 versions can be used with this option. "
+        + "Input one of the following statistics to be used for comparison: ('CSI', 'MCC', 'TPR', 'PND', 'FAR').",
+        default=False,
+        required=False,
+    )
 
     # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
 
     # Finalize Variables
-    m = args['metric_csv']
-    w = args['workspace']
-    v = args['versions']
-    i = args['site_plots']
-    iqr = args['show_iqr']
-    d = args['diff_stat']
+    m = args["metric_csv"]
+    w = args["workspace"]
+    v = args["versions"]
+    i = args["site_plots"]
+    iqr = args["show_iqr"]
+    d = args["diff_stat"]
 
     # Run eval_plots function
-    print('The following AHPS sites are considered "BAD_SITES":  ' + ', '.join(BAD_SITES))
+    print(
+        'The following AHPS sites are considered "BAD_SITES":  ' + ", ".join(BAD_SITES)
+    )
     iter_benchmarks(m, w, v, i, iqr, d)
-

From 8c747f3fde12a4ff54c3b9abc37a6a585ae01c9f Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 1 Sep 2023 11:22:42 -0400
Subject: [PATCH 269/359] v4.3.15.4 Bugfix: Correct `nodata` value in
 `src/mask_dem.py` (#977)

---
 docs/CHANGELOG.md             | 14 +++++++
 src/buffer_stream_branches.py | 73 +++++++++++++++++++++++------------
 src/burn_in_levees.py         | 45 ++++++++++-----------
 src/generate_branch_list.py   | 42 ++++++++++----------
 src/mask_dem.py               |  5 ++-
 5 files changed, 108 insertions(+), 71 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7894adb07..726af5955 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.15.4 - 2023-08-28 - [PR#977](https://github.com/NOAA-OWP/inundation-mapping/pull/977)
+
+Fixes incorrect `nodata` value in `src/burn_in_levees.py` that was responsible for missing branches (Exit code: 61). Also cleans up related files.
+
+### Changes
+
+- `src/`
+    - `buffer_stream_branches.py`: Moves script functionality into a function.
+    - `burn_in_levees.py`: Corrects `nodata` value. Adds context managers for reading rasters.
+    - `generate_branch_list.py`: Removes unused imports.
+    - `mask_dem.py`: Removes commented code.
+
+<br/><br/>
+
 ## v4.3.15.3 - 2023-09-01 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/983)
 
 ### Changes  
diff --git a/src/buffer_stream_branches.py b/src/buffer_stream_branches.py
index 06313f5ad..530998bde 100755
--- a/src/buffer_stream_branches.py
+++ b/src/buffer_stream_branches.py
@@ -7,35 +7,58 @@
 import argparse
 
 
-if __name__ == '__main__':
-
-    # parse arguments
-    parser = argparse.ArgumentParser(description='Generates branch polygons')
-    parser.add_argument('-a', '--dem-domain', help='DEM domain file', required=False, type=str)
-    parser.add_argument('-s','--streams', help='Streams file to branch', required=True)
-    parser.add_argument('-i','--branch-id', help='Attribute with branch ids', required=True)
-    parser.add_argument('-d','--buffer-distance', help='Distance to buffer branches to create branch polygons', required=True,type=int)
-    parser.add_argument('-b','--branches', help='Branch polygons out file name', required=False,default=None)
-    parser.add_argument('-v','--verbose', help='Verbose printing', required=False,default=None,action='store_true')
-
-    # extract to dictionary
-    args = vars(parser.parse_args())
-
-    streams_file, branch_id_attribute, buffer_distance, stream_polygons_file, dem_domain, verbose = args["streams"], args["branch_id"] , args["buffer_distance"], args["branches"], args['dem_domain'] , args["verbose"]
-    
+def buffer_stream_branches(
+    streams_file: str,
+    branch_id_attribute: str,
+    buffer_distance: int,
+    stream_polygons_file: str = None,
+    dem_domain: str = None,
+    verbose: bool = False,
+):
     if os.path.exists(streams_file):
         # load file
-        stream_network = StreamNetwork.from_file( filename=streams_file,branch_id_attribute=branch_id_attribute,
-                                                values_excluded=None, attribute_excluded=None, verbose = verbose)
-        
-        # make stream polygons 
-        stream_polys = StreamBranchPolygons.buffer_stream_branches( stream_network,
-                                                                    buffer_distance=buffer_distance,
-                                                                    verbose=verbose                  )
+        stream_network = StreamNetwork.from_file(
+            filename=streams_file,
+            branch_id_attribute=branch_id_attribute,
+            values_excluded=None,
+            attribute_excluded=None,
+            verbose=verbose,
+        )
+
+        # make stream polygons
+        stream_polys = StreamBranchPolygons.buffer_stream_branches(
+            stream_network, buffer_distance=buffer_distance, verbose=verbose
+        )
 
         # Clip to DEM domain
         if os.path.exists(dem_domain):
             dem_domain = gpd.read_file(dem_domain)
             stream_polys.geometry = gpd.clip(stream_polys, dem_domain).geometry
-        
-        stream_polys.write(stream_polygons_file,verbose=verbose)
\ No newline at end of file
+
+        stream_polys.write(stream_polygons_file, verbose=verbose)
+
+
+if __name__ == "__main__":
+    # parse arguments
+    parser = argparse.ArgumentParser(description="Generates branch polygons")
+    parser.add_argument("-a", "--dem-domain", help="DEM domain file", required=False, type=str)
+    parser.add_argument("-s", "--streams-file", help="Streams file to branch", required=True)
+    parser.add_argument("-i", "--branch-id-attribute", help="Attribute with branch ids", required=True)
+    parser.add_argument(
+        "-d",
+        "--buffer-distance",
+        help="Distance to buffer branches to create branch polygons",
+        required=True,
+        type=int,
+    )
+    parser.add_argument(
+        "-b", "--stream-polygons-file", help="Branch polygons out file name", required=False, default=None
+    )
+    parser.add_argument(
+        "-v", "--verbose", help="Verbose printing", required=False, default=None, action="store_true"
+    )
+
+    # extract to dictionary
+    args = vars(parser.parse_args())
+
+    buffer_stream_branches(**args)
diff --git a/src/burn_in_levees.py b/src/burn_in_levees.py
index ea8acd92c..28b83585b 100755
--- a/src/burn_in_levees.py
+++ b/src/burn_in_levees.py
@@ -7,39 +7,34 @@
 
 
 @mem_profile
-def burn_in_levees(dem_filename,nld_filename,out_dem_filename):
+def burn_in_levees(dem_filename, nld_filename, out_dem_filename):
+    # TODO Document this code
+    with rasterio.open(dem_filename) as dem, rasterio.open(nld_filename) as nld:
+        dem_data = dem.read(1)
+        nld_data = nld.read(1)
 
-    #TODO Document this code
-    dem = rasterio.open(dem_filename)
-    nld = rasterio.open(nld_filename)
+        no_data = nld.nodata
 
-    dem_data = dem.read(1)
-    nld_data = nld.read(1)
+        nld_m = np.where(nld_data == int(no_data), no_data, nld_data.astype(rasterio.float32))
 
-    no_data = nld.nodata
+        dem_profile = dem.profile.copy()
 
-    nld_m = np.where(nld_data == int(no_data), -9999.0, (nld_data).astype(rasterio.float32))
+        dem_nld_burn = np.maximum(dem_data, nld_m)
 
-    dem_profile = dem.profile.copy()
+    with rasterio.open(out_dem_filename, "w", **dem_profile, BIGTIFF="YES") as dest:
+        dest.write(dem_nld_burn, indexes=1)
 
-    dem_nld_burn = np.maximum(dem_data, nld_m)
 
-
-    with rasterio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
-        dest.write(dem_nld_burn, indexes = 1)
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Burn in NLD elevations')
-    parser.add_argument('-dem','--dem-filename', help='DEM filename', required=True,type=str)
-    parser.add_argument('-nld','--nld-filename', help='NLD filename', required=True,type=str)
-    parser.add_argument('-out','--out-dem-filename', help='out DEM filename', required=True,type=str)
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Burn in NLD elevations")
+    parser.add_argument("-dem", "--dem-filename", help="DEM filename", required=True, type=str)
+    parser.add_argument("-nld", "--nld-filename", help="NLD filename", required=True, type=str)
+    parser.add_argument("-out", "--out-dem-filename", help="out DEM filename", required=True, type=str)
 
     args = vars(parser.parse_args())
 
-    dem_filename = args['dem_filename']
-    nld_filename = args['nld_filename']
-    out_dem_filename = args['out_dem_filename']
+    dem_filename = args["dem_filename"]
+    nld_filename = args["nld_filename"]
+    out_dem_filename = args["out_dem_filename"]
 
-    burn_in_levees(dem_filename,nld_filename,out_dem_filename)
+    burn_in_levees(dem_filename, nld_filename, out_dem_filename)
diff --git a/src/generate_branch_list.py b/src/generate_branch_list.py
index 2529a223e..dc4a8e094 100755
--- a/src/generate_branch_list.py
+++ b/src/generate_branch_list.py
@@ -2,20 +2,17 @@
 
 import os
 import argparse
-import pandas as pd
-import sys
 
 from stream_branches import StreamNetwork
 
-def generate_branch_list(stream_network_dissolved, branch_id_attribute,
-                         output_branch_list_file):
 
-    '''
+def generate_branch_list(stream_network_dissolved, branch_id_attribute, output_branch_list_file):
+    """
     Processing:
         This create a branch_ids.lst file which is required at the very start of processing
         hucs.  This becomes the list that run_unit_wb.sh needs to iterate over branches
-        
-        Note: The .csv twin to this is appended to each time a branch completes, 
+
+        Note: The .csv twin to this is appended to each time a branch completes,
         resulting in a list that only contains successfully processed branches.
     Params:
         - stream_network_dissolved (str): the gkpg that contains the list of disolved branch ids
@@ -24,26 +21,31 @@ def generate_branch_list(stream_network_dissolved, branch_id_attribute,
         - output_branch_list_file (str): file name and path of the list to be created.
     Output:
         - create a file (likely a .lst file) with branch ids (not including branch zero)
-    '''
-    
+    """
+
     if os.path.exists(stream_network_dissolved):
         # load stream network
-        stream_network_dissolved = StreamNetwork.from_file( stream_network_dissolved,
-                                                            branch_id_attribute=branch_id_attribute )
+        stream_network_dissolved = StreamNetwork.from_file(
+            stream_network_dissolved, branch_id_attribute=branch_id_attribute
+        )
         # reduce to branch id attribute and convert to pandas df
-        stream_network_dissolved = stream_network_dissolved.loc[:,branch_id_attribute]
-        
+        stream_network_dissolved = stream_network_dissolved.loc[:, branch_id_attribute]
+
         # write out the list version (just branch numbers)
-        stream_network_dissolved.to_csv(output_branch_list_file, sep= " ", index=False, header=False)
+        stream_network_dissolved.to_csv(output_branch_list_file, sep=" ", index=False, header=False)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Create branch list")
+    parser.add_argument("-d", "--stream-network-dissolved", help="Dissolved stream network", required=True)
+    parser.add_argument(
+        "-b",
+        "--branch-id-attribute",
+        help="Branch ID attribute to use in dissolved stream network",
+        required=True,
+    )
+    parser.add_argument("-o", "--output-branch-list-file", help="Output branch list", required=True)
 
-    parser = argparse.ArgumentParser(description='Create branch list')
-    parser.add_argument('-d','--stream-network-dissolved', help='Dissolved stream network', required=True)
-    parser.add_argument('-b','--branch-id-attribute', help='Branch ID attribute to use in dissolved stream network', required=True)
-    parser.add_argument('-o','--output-branch-list-file', help='Output branch list', required=True)    
-    
     args = vars(parser.parse_args())
 
     generate_branch_list(**args)
diff --git a/src/mask_dem.py b/src/mask_dem.py
index 7cfab2789..6f2f52c70 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -50,15 +50,18 @@ def mask_dem(dem_filename:str, nld_filename:str, levee_id_attribute:str, out_dem
     elif os.path.exists(levee_levelpaths):
         levee_levelpaths = pd.read_csv(levee_levelpaths)
 
+        # Select levees associated with branch
         levee_levelpaths = levee_levelpaths[levee_levelpaths[branch_id_attribute] == branch_id]
 
+        # Get levee IDs
         levelpath_levees = list(levee_levelpaths[levee_id_attribute])
         
         if len(levelpath_levees) > 0:
-            with rio.open(dem_filename) as dem:#, fiona.open(nld_filename) as leveed:
+            with rio.open(dem_filename) as dem:
                 leveed = gpd.read_file(nld_filename)
                 dem_profile = dem.profile.copy()
 
+                # Get geometries of levee protected areas associated with levelpath
                 geoms = [feature['geometry'] for i, feature in leveed.iterrows() if feature[levee_id_attribute] in levelpath_levees]
 
                 # Mask out levee-protected areas from DEM

From fdf3e8ee196fdfd4d5fd051887ec96f764e4e835 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 1 Sep 2023 11:32:14 -0400
Subject: [PATCH 270/359] v4.3.15.5 Fix stream outlet clipping (#970)

---
 docs/CHANGELOG.md                         |   17 +
 src/clip_vectors_to_wbd.py                |   13 +-
 src/derive_level_paths.py                 |  362 ++++---
 src/run_unit_wb.sh                        |    3 +-
 src/stream_branches.py                    | 1201 ++++++++++++---------
 unit_tests/derive_level_paths_params.json |    2 +-
 unit_tests/derive_level_paths_test.py     |  108 +-
 7 files changed, 999 insertions(+), 707 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 726af5955..368646a16 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.3.15.5 - 2023-08-17 - [PR#970](https://github.com/NOAA-OWP/inundation-mapping/pull/970)
+
+Fixes an issue where the stream network was clipped inside the DEM resulting in a burned stream channel that was then filled by the DEM depression filling process so that all pixels in the burned channel had the same elevation which was the elevation at the spill point (which wasn't necessarily at the HUC outlet). The stream network is now extended from the WBD to the buffered WBD and all streams except the outlet are clipped to the streams buffer inside the WBD (WBD - (3 x cell_size)). This also prevents reverse flow issues.
+
+### Changes
+
+- `src/`
+    - `clip_vectors_to_wbd.py`: Clip NWM streams to buffered WBD and clip non-outlet streams to WBD streams buffer (WBD - (3 x cell_size)).
+    - `derive_level_paths.py`: Add WBD input argument
+    - `run_unit_wb.py`: Add WBD input argument
+    - `src_stream_branches.py`: Ignore branches outside HUC
+- `unit_tests/`
+    - `derive_level_paths_params.json`: Add WBD parameter value
+    - `derive_level_paths_test.py`: Add WBD parameter
+ 
+<br/><br/>
+
 ## v4.3.15.4 - 2023-08-28 - [PR#977](https://github.com/NOAA-OWP/inundation-mapping/pull/977)
 
 Fixes incorrect `nodata` value in `src/burn_in_levees.py` that was responsible for missing branches (Exit code: 61). Also cleans up related files.
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 5d80da537..3901a3eb2 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 
 import sys
+import pandas as pd
 import geopandas as gpd
 import argparse
 import rasterio as rio
-import fiona
 from shapely.geometry import MultiPolygon,Polygon
 from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 from utils.shared_functions import getDriver, mem_profile
@@ -124,13 +124,18 @@ def subset_vector_layers(subset_nwm_lakes,
     # Subset nwm streams
     print("Subsetting NWM Streams", flush=True)
 
-    nwm_streams = gpd.read_file(nwm_streams, mask = wbd)
+    nwm_streams = gpd.read_file(nwm_streams, mask = wbd_buffer)
 
-     # NWM can have duplicate records, but appear to always be identical duplicates
+    # NWM can have duplicate records, but appear to always be identical duplicates
     nwm_streams.drop_duplicates(subset="ID", keep="first", inplace=True)
 
+    nwm_streams_outlets = nwm_streams[~nwm_streams['to'].isin(nwm_streams['ID'])]
+    nwm_streams_nonoutlets = nwm_streams[nwm_streams['to'].isin(nwm_streams['ID'])]
+
     if len(nwm_streams) > 0:
-        nwm_streams = gpd.clip(nwm_streams, wbd_streams_buffer)
+        nwm_streams_nonoutlets = gpd.clip(nwm_streams_nonoutlets, wbd_streams_buffer)
+
+        nwm_streams = pd.concat([nwm_streams_nonoutlets, nwm_streams_outlets])
 
         nwm_streams.to_file(subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
     else:
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index 90f09b0a1..d78a794f0 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -8,185 +8,305 @@
 from stream_branches import StreamNetwork
 from utils.fim_enums import FIM_exit_codes
 
-def Derive_level_paths(in_stream_network, buffer_wbd_streams, out_stream_network, branch_id_attribute,
-                       out_stream_network_dissolved=None, huc_id=None,
-                       headwaters_outfile=None, catchments=None, waterbodies=None,
-                       catchments_outfile=None,
-                       branch_inlets_outfile=None,
-                       toNode_attribute='To_Node', fromNode_attribute='From_Node',
-                       reach_id_attribute='HydroID', verbose=False):
-   
-    
+
+def Derive_level_paths(
+    in_stream_network,
+    wbd,
+    buffer_wbd_streams,
+    out_stream_network,
+    branch_id_attribute,
+    out_stream_network_dissolved=None,
+    huc_id=None,
+    headwaters_outfile=None,
+    catchments=None,
+    waterbodies=None,
+    catchments_outfile=None,
+    branch_inlets_outfile=None,
+    toNode_attribute="To_Node",
+    fromNode_attribute="From_Node",
+    reach_id_attribute="HydroID",
+    verbose=False,
+):
     if verbose:
         print("Deriving level paths ...")
 
     # load file
     if verbose:
         print("Loading stream network ...")
-        
+
     if os.path.exists(in_stream_network):
         stream_network = StreamNetwork.from_file(filename=in_stream_network)
     else:
-        print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
+        print(
+            "Sorry, no branches exist and processing can not continue. This could be an empty file."
+        )
         sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # if there are no reaches at this point
-    if (len(stream_network) == 0):
+    if len(stream_network) == 0:
         # This is technically not an error but we need to have it logged so the user know what
-        # happened to it and we need the huc to not be included in future processing. 
+        # happened to it and we need the huc to not be included in future processing.
         # We need it to be not included in the fim_input.csv at the end of the unit processing.
         # Throw an exception with valid text. This will show up in the non-zero exit codes and explain why an error.
-        # Later, we can look at creating custom sys exit codes 
+        # Later, we can look at creating custom sys exit codes
         # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file.")
-        print("Sorry, no streams exist and processing can not continue. This could be an empty file.")
+        print(
+            "Sorry, no streams exist and processing can not continue. This could be an empty file."
+        )
         sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # values_exluded of 1 and 2 mean where are dropping stream orders 1 and 2. We are leaving those
     # for branch zero.
-    stream_network = stream_network.exclude_attribute_values(branch_id_attribute="order_", values_excluded=[1,2] )
+    stream_network = stream_network.exclude_attribute_values(
+        branch_id_attribute="order_", values_excluded=[1, 2]
+    )
 
     # if there are no reaches at this point (due to filtering)
-    if (len(stream_network) == 0):
-        print("No branches exist but branch zero processing will continue (Exit 63). This could be due to stream order filtering.")
-        #sys.exit(FIM_exit_codes.NO_BRANCH_LEVELPATHS_EXIST.value)  # will send a 63 back
+    if len(stream_network) == 0:
+        print(
+            "No branches exist but branch zero processing will continue (Exit 63). This could be due to stream order filtering."
+        )
+        # sys.exit(FIM_exit_codes.NO_BRANCH_LEVELPATHS_EXIST.value)  # will send a 63 back
         return
-                                                 
-    inlets_attribute = 'inlet_id'
-    outlets_attribute = 'outlet_id'
+
+    inlets_attribute = "inlet_id"
+    outlets_attribute = "outlet_id"
     outlet_linestring_index = -1
 
     # converts multi-linestrings to linestrings
     stream_network = stream_network.multilinestrings_to_linestrings()
 
     # derive nodes
-    stream_network = stream_network.derive_nodes(toNode_attribute=toNode_attribute,
-                                                 fromNode_attribute=fromNode_attribute,
-                                                 reach_id_attribute=reach_id_attribute,
-                                                 outlet_linestring_index=outlet_linestring_index,
-                                                 node_prefix=None,
-                                                 verbose=verbose)
-    
+    stream_network = stream_network.derive_nodes(
+        toNode_attribute=toNode_attribute,
+        fromNode_attribute=fromNode_attribute,
+        reach_id_attribute=reach_id_attribute,
+        outlet_linestring_index=outlet_linestring_index,
+        node_prefix=None,
+        verbose=verbose,
+    )
+
     # derive outlets and inlets
-    stream_network = stream_network.derive_outlets(toNode_attribute,
-                                                   fromNode_attribute,
-                                                   outlets_attribute=outlets_attribute,
-                                                   verbose=verbose)
-    
-    stream_network = stream_network.derive_inlets(toNode_attribute,
-                                                  fromNode_attribute,
-                                                  inlets_attribute=inlets_attribute,
-                                                  verbose=verbose
-                                                  )    # derive up and downstream networks
+    stream_network = stream_network.derive_outlets(
+        toNode_attribute,
+        fromNode_attribute,
+        outlets_attribute=outlets_attribute,
+        verbose=verbose,
+    )
+
+    stream_network = stream_network.derive_inlets(
+        toNode_attribute,
+        fromNode_attribute,
+        inlets_attribute=inlets_attribute,
+        verbose=verbose,
+    )  # derive up and downstream networks
     upstreams, downstreams = stream_network.make_up_and_downstream_dictionaries(
-                                                                         reach_id_attribute=reach_id_attribute,
-                                                                         toNode_attribute=toNode_attribute,
-                                                                         fromNode_attribute=fromNode_attribute,
-                                                                         verbose=True)
+        reach_id_attribute=reach_id_attribute,
+        toNode_attribute=toNode_attribute,
+        fromNode_attribute=fromNode_attribute,
+        verbose=True,
+    )
 
     # derive arbolate sum
-    stream_network = stream_network.get_arbolate_sum(arbolate_sum_attribute='arbolate_sum',
-                                                     inlets_attribute=inlets_attribute,
-                                                     reach_id_attribute=reach_id_attribute,
-                                                     upstreams=upstreams,
-                                                     downstreams=downstreams,
-                                                     length_conversion_factor_to_km = 0.001,
-                                                     verbose=verbose)
+    stream_network = stream_network.get_arbolate_sum(
+        arbolate_sum_attribute="arbolate_sum",
+        inlets_attribute=inlets_attribute,
+        reach_id_attribute=reach_id_attribute,
+        upstreams=upstreams,
+        downstreams=downstreams,
+        length_conversion_factor_to_km=0.001,
+        verbose=verbose,
+    )
 
     # derive stream branches
-    stream_network = stream_network.derive_stream_branches(toNode_attribute=toNode_attribute,
-                                                           fromNode_attribute=fromNode_attribute,
-                                                           upstreams=upstreams,
-                                                           branch_id_attribute=branch_id_attribute,
-                                                           reach_id_attribute=reach_id_attribute,
-                                                           comparison_attributes=['arbolate_sum', 'order_'],
-                                                           comparison_function=max,
-                                                           verbose=verbose)
-    
+    stream_network = stream_network.derive_stream_branches(
+        toNode_attribute=toNode_attribute,
+        fromNode_attribute=fromNode_attribute,
+        upstreams=upstreams,
+        branch_id_attribute=branch_id_attribute,
+        reach_id_attribute=reach_id_attribute,
+        comparison_attributes=["arbolate_sum", "order_"],
+        comparison_function=max,
+        verbose=verbose,
+    )
+
     # filter out streams without catchments
     if (catchments is not None) & (catchments_outfile is not None):
         catchments = gpd.read_file(catchments)
 
-        stream_network = stream_network.remove_branches_without_catchments( 
-                                                                catchments,
-                                                                reach_id_attribute=reach_id_attribute,
-                                                                branch_id_attribute=branch_id_attribute,
-                                                                reach_id_attribute_in_catchments=reach_id_attribute,
-                                                                verbose=verbose)
+        stream_network = stream_network.remove_branches_without_catchments(
+            catchments,
+            reach_id_attribute=reach_id_attribute,
+            branch_id_attribute=branch_id_attribute,
+            reach_id_attribute_in_catchments=reach_id_attribute,
+            verbose=verbose,
+        )
 
         # subset which columns to merge
-        stream_network_to_merge = stream_network.filter(items = [reach_id_attribute,inlets_attribute,
-                                                                 outlets_attribute,branch_id_attribute])
+        stream_network_to_merge = stream_network.filter(
+            items=[
+                reach_id_attribute,
+                inlets_attribute,
+                outlets_attribute,
+                branch_id_attribute,
+            ]
+        )
 
-        catchments = catchments.merge(stream_network_to_merge,how='inner',
-                                      left_on=reach_id_attribute,
-                                      right_on=reach_id_attribute)
+        catchments = catchments.merge(
+            stream_network_to_merge,
+            how="inner",
+            left_on=reach_id_attribute,
+            right_on=reach_id_attribute,
+        )
 
         catchments.reset_index(drop=True, inplace=True)
 
-        catchments.to_file(catchments_outfile, index=False, driver='GPKG')
+        catchments.to_file(catchments_outfile, index=False, driver="GPKG")
 
     # derive headwaters
-    if (headwaters_outfile is not None):
+    if headwaters_outfile is not None:
         headwaters = stream_network.derive_headwater_points_with_inlets(
-                                                        fromNode_attribute=fromNode_attribute,
-                                                        inlets_attribute=inlets_attribute,
-                                                        outlet_linestring_index=outlet_linestring_index)
+            fromNode_attribute=fromNode_attribute,
+            inlets_attribute=inlets_attribute,
+            outlet_linestring_index=outlet_linestring_index,
+        )
         # headwaters write
-        headwaters.to_file(headwaters_outfile, index=False, driver='GPKG')
+        headwaters.to_file(headwaters_outfile, index=False, driver="GPKG")
 
     if out_stream_network is not None:
         if verbose:
             print("Writing stream branches ...")
         stream_network.write(out_stream_network, index=True)
-    
+
     if out_stream_network_dissolved is not None:
-        stream_network = stream_network.trim_branches_in_waterbodies(branch_id_attribute=branch_id_attribute,
-                                                    verbose=verbose)
+        stream_network = stream_network.trim_branches_in_waterbodies(
+            branch_id_attribute=branch_id_attribute, verbose=verbose
+        )
 
         # dissolve by levelpath
-        stream_network = stream_network.dissolve_by_branch(branch_id_attribute=branch_id_attribute,
-                                                           attribute_excluded=None, #'order_',
-                                                           values_excluded=None, #[1,2],
-                                                           out_vector_files=out_stream_network_dissolved,
-                                                           verbose=verbose)
-
-        stream_network = stream_network.remove_branches_in_waterbodies(waterbodies=waterbodies,
-                                                                       out_vector_files=out_stream_network_dissolved,
-                                                                       verbose=False)
-        # clip dissolved stream network to the wbd_buffered domain (avoids issues with reaches that extend outside buffer)
-        wbd_buffer = gpd.read_file(buffer_wbd_streams)
-        stream_network_out = gpd.read_file(out_stream_network_dissolved)
-        stream_network_out = gpd.clip(stream_network_out,wbd_buffer)
-        stream_network_out.to_file(out_stream_network_dissolved, index=True, driver='GPKG')
-                                       
+        stream_network = stream_network.dissolve_by_branch(
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=None,  #'order_',
+            values_excluded=None,  # [1,2],
+            out_vector_files=out_stream_network_dissolved,
+            verbose=verbose,
+        )
+
+        stream_network = stream_network.remove_branches_in_waterbodies(
+            waterbodies=waterbodies,
+            out_vector_files=out_stream_network_dissolved,
+            verbose=False,
+        )
+        stream_network = stream_network.select_branches_intersecting_huc(
+            wbd=wbd,
+            buffer_wbd_streams=buffer_wbd_streams,
+            out_vector_files=out_stream_network_dissolved,
+            verbose=False,
+        )
+
     if branch_inlets_outfile is not None:
-        branch_inlets = stream_network.derive_inlet_points_by_feature(feature_attribute=branch_id_attribute,
-                                                                    outlet_linestring_index=outlet_linestring_index)
-
-        branch_inlets.to_file(branch_inlets_outfile, index=False, driver='GPKG')
-
-    return(stream_network)
-    
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Create stream network level paths')
-    parser.add_argument('-i','--in-stream-network', help='Input stream network', required=True)
-    parser.add_argument('-s','--buffer-wbd-streams', help='Input wbd buffer for stream network', required=True)
-    parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
-    parser.add_argument('-u','--huc-id', help='Current HUC ID', required=False, default=None)
-    parser.add_argument('-r','--reach-id-attribute', help='Reach ID attribute to use in source file', required=False, default='HydroID')
-    parser.add_argument('-c','--catchments', help='NWM catchments to append level path data to', required=False, default=None)
-    parser.add_argument('-t','--catchments-outfile', help='NWM catchments outfile with appended level path data', required=False, default=None)
-    parser.add_argument('-w','--waterbodies', help='NWM waterbodies to eliminate branches from', required=False, default=None)
-    parser.add_argument('-n','--branch_inlets_outfile', help='Output level paths inlets', required=False, default=None)
-    parser.add_argument('-o','--out-stream-network', help='Output stream network', required=False, default=None)
-    parser.add_argument('-e','--headwaters-outfile', help='Output stream network headwater points', required=False, default=None)
-    parser.add_argument('-d','--out-stream-network-dissolved', help='Dissolved output stream network', required=False, default=None)
-    parser.add_argument('-v','--verbose', help='Verbose output', required=False, default=False, action='store_true')
-    
+        branch_inlets = stream_network.derive_inlet_points_by_feature(
+            feature_attribute=branch_id_attribute,
+            outlet_linestring_index=outlet_linestring_index,
+        )
+
+        branch_inlets.to_file(branch_inlets_outfile, index=False, driver="GPKG")
+
+    return stream_network
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Create stream network level paths")
+    parser.add_argument(
+        "-i", "--in-stream-network", help="Input stream network", required=True
+    )
+    parser.add_argument(
+        "-s",
+        "--buffer-wbd-streams",
+        help="Input wbd buffer for stream network",
+        required=True,
+    )
+    parser.add_argument(
+        "-wbd",
+        "--wbd",
+        help="Input watershed boundary (HUC) dataset",
+        required=True,
+        default=None,
+    )
+    parser.add_argument(
+        "-b",
+        "--branch-id-attribute",
+        help="Name of the branch attribute desired",
+        required=True,
+    )
+    parser.add_argument(
+        "-u", "--huc-id", help="Current HUC ID", required=False, default=None
+    )
+    parser.add_argument(
+        "-r",
+        "--reach-id-attribute",
+        help="Reach ID attribute to use in source file",
+        required=False,
+        default="HydroID",
+    )
+    parser.add_argument(
+        "-c",
+        "--catchments",
+        help="NWM catchments to append level path data to",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-t",
+        "--catchments-outfile",
+        help="NWM catchments outfile with appended level path data",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-w",
+        "--waterbodies",
+        help="NWM waterbodies to eliminate branches from",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-n",
+        "--branch_inlets_outfile",
+        help="Output level paths inlets",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-o",
+        "--out-stream-network",
+        help="Output stream network",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-e",
+        "--headwaters-outfile",
+        help="Output stream network headwater points",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-d",
+        "--out-stream-network-dissolved",
+        help="Dissolved output stream network",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Verbose output",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+
     args = vars(parser.parse_args())
 
     Derive_level_paths(**args)
-
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 228262982..373562605 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -86,7 +86,7 @@ Tcount
 echo -e $startDiv"Generating Level Paths for $hucNumber"
 date -u
 Tstart
-$srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg -s $tempHucDataDir/wbd_buffered_streams.gpkg -b $branch_id_attribute -r "ID" -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $tempHucDataDir/nwm_headwaters.gpkg -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg
+$srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg -s $tempHucDataDir/wbd_buffered_streams.gpkg -b $branch_id_attribute -r "ID" -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $tempHucDataDir/nwm_headwaters.gpkg -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg -wbd $tempHucDataDir/wbd.gpkg
 
 # test if we received a non-zero code back from derive_level_paths.py
 #subscript_exit_code=$?
@@ -326,4 +326,3 @@ date -u
 echo "---- HUC processing for $hucNumber is complete"
 Calc_Duration $huc_start_time
 echo
-
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 5fc0673e5..43a1be1ac 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -19,242 +19,258 @@
 from scipy.stats import mode
 from utils.shared_variables import PREP_CRS
 
+
 class StreamNetwork(gpd.GeoDataFrame):
 
-    '''
+    """
     Notes:
         - Many of the methods support two attributes called branch_id_attribute and values_excluded.
-          This can be used to filter out records. 
+          This can be used to filter out records.
             ie) When calling the nwm_subset_streams.gpkg, you can filter some records like this:
                 StreamNetwork.from_file(filename=outputs/<some huc>/nwm_subset_streams.gpkg,
                                                  branch_id_attribute="order_",
                                                  values_excluded=[1,2]
                 (which means drop all records that have an order_ of 1 or 2.
-                
+
         - Note: from_file is using its branch_id_attribute and values_excluded as intended but other
              methods may not be incomplete and not filtering as expected.
-    '''
-    
-    geom_name = 'geometry' # geometry attribute name
-    branch_id_attribute = None # branch id attribute name
+    """
+
+    geom_name = "geometry"  # geometry attribute name
+    branch_id_attribute = None  # branch id attribute name
     values_excluded = None
     attribute_excluded = None
 
-    def __init__(self,*args,**kwargs):
-        
+    def __init__(self, *args, **kwargs):
         if kwargs:
-            branch_id_attribute = kwargs.pop("branch_id_attribute",None)
-            values_excluded = kwargs.pop("values_excluded",None)
-            attribute_excluded = kwargs.pop("attribute_excluded",None)
-        
-        super().__init__(*args,**kwargs)
-        
+            branch_id_attribute = kwargs.pop("branch_id_attribute", None)
+            values_excluded = kwargs.pop("values_excluded", None)
+            attribute_excluded = kwargs.pop("attribute_excluded", None)
+
+        super().__init__(*args, **kwargs)
+
         self.branch_id_attribute = branch_id_attribute
         self.values_excluded = values_excluded
         self.attribute_excluded = attribute_excluded
 
-
     @classmethod
-    def from_file(cls, filename, branch_id_attribute=None, values_excluded=None, 
-                  attribute_excluded=None, verbose=False, *args, **kwargs):
-
-        """ loads stream network from file to streamnetwork geopandas """
+    def from_file(
+        cls,
+        filename,
+        branch_id_attribute=None,
+        values_excluded=None,
+        attribute_excluded=None,
+        verbose=False,
+        *args,
+        **kwargs,
+    ):
+        """loads stream network from file to streamnetwork geopandas"""
 
         if kwargs:
-            inputs = {  'branch_id_attribute' : kwargs.pop("branch_id_attribute",None) , 
-                        'values_excluded' :kwargs.pop("values_excluded",None) ,
-                        'attribute_excluded' : kwargs.pop("attribute_excluded",None)      }
-            
-            verbose = kwargs.pop('verbose',None)
+            inputs = {
+                "branch_id_attribute": kwargs.pop("branch_id_attribute", None),
+                "values_excluded": kwargs.pop("values_excluded", None),
+                "attribute_excluded": kwargs.pop("attribute_excluded", None),
+            }
+
+            verbose = kwargs.pop("verbose", None)
         else:
-            inputs = {  'branch_id_attribute' : branch_id_attribute , 
-                        'values_excluded' : values_excluded ,
-                        'attribute_excluded' : attribute_excluded      }
-            
-        if verbose: 
-            print('Loading file')
-            
-        raw_df = gpd.read_file(filename,*args,**kwargs)
+            inputs = {
+                "branch_id_attribute": branch_id_attribute,
+                "values_excluded": values_excluded,
+                "attribute_excluded": attribute_excluded,
+            }
+
+        if verbose:
+            print("Loading file")
+
+        raw_df = gpd.read_file(filename, *args, **kwargs)
 
         # Reproject
         if raw_df.crs.to_authority() != PREP_CRS.to_authority():
             raw_df.to_crs(PREP_CRS)
 
         filtered_df = gpd.GeoDataFrame()
-                      
+
         if (branch_id_attribute is not None) and (values_excluded is not None):
             filtered_df = raw_df[~raw_df[branch_id_attribute].isin(values_excluded)]
         else:
             filtered_df = raw_df
-       
-        if verbose:         
-             print("======" + filename)
-             print("Number of df rows = " + str(filtered_df.shape[0]))
-        
-        return(cls(filtered_df,**inputs))
 
+        if verbose:
+            print("======" + filename)
+            print("Number of df rows = " + str(filtered_df.shape[0]))
 
-    def write(self,fileName,layer=None,index=True,verbose=False):
+        return cls(filtered_df, **inputs)
 
-        """ Gets driver Name from file extension for Geopandas writing """
+    def write(self, fileName, layer=None, index=True, verbose=False):
+        """Gets driver Name from file extension for Geopandas writing"""
 
         if verbose:
             print("Writing to {}".format(fileName))
 
         # sets driver
-        driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
+        driverDictionary = {
+            ".gpkg": "GPKG",
+            ".geojson": "GeoJSON",
+            ".shp": "ESRI Shapefile",
+        }
         driver = driverDictionary[splitext(fileName)[1]]
-    
+
         self.to_file(fileName, driver=driver, layer=layer, index=index)
 
-    def apply(self,*args,**kwargs):
-        
+    def apply(self, *args, **kwargs):
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
         values_excluded = self.values_excluded
         crs = self.crs
-        
-        self = super().apply(*args,**kwargs)
+
+        self = super().apply(*args, **kwargs)
         self = self.set_crs(crs)
-        
-        self = StreamNetwork( self,
-                              branch_id_attribute=branch_id_attribute,
-                              attribute_excluded=attribute_excluded,
-                              values_excluded=values_excluded
-                            )
 
-        return(self)
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
 
+        return self
 
     def multilinestrings_to_linestrings(self):
-        
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
         values_excluded = self.values_excluded
-        
+
         def convert_to_linestring(row):
+            geometry = row["geometry"]
 
-            geometry = row['geometry']
+            if isinstance(geometry, MultiLineString):
+                linestring = LineString(sum([list(item.coords) for item in list(geometry.geoms)], []))
+                row["geometry"] = linestring
 
-            if isinstance(geometry,MultiLineString):
-                linestring = LineString(sum([list(item.coords) for item in list(geometry.geoms)],[]))
-                row['geometry'] = linestring
-            
-            return(row)
+            return row
 
-        
-        self = StreamNetwork( self.apply(convert_to_linestring,axis=1),
-                              branch_id_attribute=branch_id_attribute,
-                              attribute_excluded=attribute_excluded,
-                              values_excluded=values_excluded
-                            )
-        
-        return(self)
+        self = StreamNetwork(
+            self.apply(convert_to_linestring, axis=1),
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
 
+        return self
 
-    def explode(self,**kwargs):
-        
+    def explode(self, **kwargs):
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
         values_excluded = self.values_excluded
 
-        self = StreamNetwork( super().explode(**kwargs),
-                             branch_id_attribute=branch_id_attribute,
-                             attribute_excluded=attribute_excluded,
-                             values_excluded=values_excluded
-                            )
-
-        return(self)
-
+        self = StreamNetwork(
+            super().explode(**kwargs),
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
 
-    def to_df(self,*args,**kwargs):
+        return self
 
-        """ Converts back to dataframe """
-        
-        self = pd.DataFrame(self,*args,**kwargs)
+    def to_df(self, *args, **kwargs):
+        """Converts back to dataframe"""
 
-        return(self)
+        self = pd.DataFrame(self, *args, **kwargs)
 
+        return self
 
-    def merge(self,*args,**kwargs):
+    def merge(self, *args, **kwargs):
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
         values_excluded = self.values_excluded
 
-        self = super().merge(*args,**kwargs)
-
-        self = StreamNetwork(self,branch_id_attribute=branch_id_attribute,
-                             attribute_excluded=attribute_excluded,
-                             values_excluded=values_excluded)
-
-        return(self)
-
+        self = super().merge(*args, **kwargs)
 
-    def merge_stream_branches(self,stream_branch_dataset,on='ID',branch_id_attribute='LevelPathI',attributes='StreamOrde',stream_branch_layer_name=None):
-
-        """ Merges stream branch id attribute from another vector file """
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+
+        return self
+
+    def merge_stream_branches(
+        self,
+        stream_branch_dataset,
+        on="ID",
+        branch_id_attribute="LevelPathI",
+        attributes="StreamOrde",
+        stream_branch_layer_name=None,
+    ):
+        """Merges stream branch id attribute from another vector file"""
 
         # load vaas
-        if isinstance(stream_branch_dataset,str):
-            stream_branch_dataset = gpd.read_file(stream_branch_dataset,layer=stream_branch_layer_name)
-        elif isinstance(stream_branch_dataset,gpd.GeoDataFrame):
+        if isinstance(stream_branch_dataset, str):
+            stream_branch_dataset = gpd.read_file(stream_branch_dataset, layer=stream_branch_layer_name)
+        elif isinstance(stream_branch_dataset, gpd.GeoDataFrame):
             pass
         else:
-            raise TypeError('Pass stream_branch_dataset argument as filepath or GeoDataframe')
-    
+            raise TypeError("Pass stream_branch_dataset argument as filepath or GeoDataframe")
+
         # merge and drop duplicate columns
-        if isinstance(attributes,list):
-            what = [on] + [branch_id_attribute] + attributes 
-        elif isinstance(attributes,str):
-            what = [on] + [branch_id_attribute] +[attributes]
+        if isinstance(attributes, list):
+            what = [on] + [branch_id_attribute] + attributes
+        elif isinstance(attributes, str):
+            what = [on] + [branch_id_attribute] + [attributes]
 
-        self = self.merge(stream_branch_dataset[what],on=on, how='inner')
-        
-        # make sure it's the correct object type
-        self = StreamNetwork(self,branch_id_attribute=branch_id_attribute)
+        self = self.merge(stream_branch_dataset[what], on=on, how="inner")
 
-        return(self)
+        # make sure it's the correct object type
+        self = StreamNetwork(self, branch_id_attribute=branch_id_attribute)
 
+        return self
 
     @staticmethod
     def flip_inlet_outlet_linestring_index(linestring_index):
-
         # returns -1 for 0 and 0 for -1
-        inlet_outlet_linestring_index_dict = { 0 : -1, -1 : 0 }
-        
+        inlet_outlet_linestring_index_dict = {0: -1, -1: 0}
+
         try:
-            return( inlet_outlet_linestring_index_dict[linestring_index] )
+            return inlet_outlet_linestring_index_dict[linestring_index]
         except KeyError:
-            raise ValueError('Linestring index should be 0 or -1')
-
-
-    def derive_nodes(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',reach_id_attribute='ID',
-                     outlet_linestring_index=0,node_prefix=None,max_node_digits=8,verbose=False):
-        
+            raise ValueError("Linestring index should be 0 or -1")
+
+    def derive_nodes(
+        self,
+        toNode_attribute="ToNode",
+        fromNode_attribute="FromNode",
+        reach_id_attribute="ID",
+        outlet_linestring_index=0,
+        node_prefix=None,
+        max_node_digits=8,
+        verbose=False,
+    ):
         if verbose:
             print("Deriving nodes ...")
 
         inlet_linestring_index = StreamNetwork.flip_inlet_outlet_linestring_index(outlet_linestring_index)
-        
+
         # set node prefix to string
         if node_prefix is None:
-            node_prefix=''
-        
+            node_prefix = ""
+
         # handle digits and values for node ids
         max_post_node_digits = max_node_digits - len(node_prefix)
-        max_node_value = int('9' * max_post_node_digits)
+        max_node_value = int("9" * max_post_node_digits)
 
         # sets index of stream branches as reach id attribute
-        #if self.index.name != reach_id_attribute:
-            #self.set_index(reach_id_attribute,drop=True,inplace=True)
-        
+        # if self.index.name != reach_id_attribute:
+        # self.set_index(reach_id_attribute,drop=True,inplace=True)
+
         inlet_coordinates, outlet_coordinates = dict(), dict()
         node_coordinates = dict()
-        toNodes, fromNodes = [None] * len(self),[None] * len(self)
-        current_node_id = '1'.zfill(max_post_node_digits)
+        toNodes, fromNodes = [None] * len(self), [None] * len(self)
+        current_node_id = "1".zfill(max_post_node_digits)
 
-        for i,(_,row) in enumerate(self.iterrows()):
-            
+        for i, (_, row) in enumerate(self.iterrows()):
             reach_id = row[reach_id_attribute]
 
             # get foss id for node_prefix
@@ -262,21 +278,21 @@ def derive_nodes(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',re
                 current_node_prefix = node_prefix
             else:
                 current_node_prefix = str(reach_id)[0:4]
-            
+
             # makes list of coordinates. Merges multi-part geoms
-            reach_coordinates = list(row['geometry'].coords)
+            reach_coordinates = list(row["geometry"].coords)
 
             inlet_coordinate = reach_coordinates[inlet_linestring_index]
             outlet_coordinate = reach_coordinates[outlet_linestring_index]
-            
+
             if inlet_coordinate not in node_coordinates:
                 current_node_id_with_prefix = current_node_prefix + current_node_id
                 node_coordinates[inlet_coordinate] = current_node_id_with_prefix
                 fromNodes[i] = current_node_id_with_prefix
-                
-                current_node_id = int(current_node_id.lstrip('0')) + 1
+
+                current_node_id = int(current_node_id.lstrip("0")) + 1
                 if current_node_id > max_node_value:
-                    raise ValueError('Current Node ID exceeding max. Look at source code to change.')
+                    raise ValueError("Current Node ID exceeding max. Look at source code to change.")
                 current_node_id = str(current_node_id).zfill(max_post_node_digits)
 
             else:
@@ -287,44 +303,48 @@ def derive_nodes(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',re
                 node_coordinates[outlet_coordinate] = current_node_id_with_prefix
                 toNodes[i] = current_node_id_with_prefix
 
-                current_node_id = int(current_node_id.lstrip('0')) + 1
+                current_node_id = int(current_node_id.lstrip("0")) + 1
                 if current_node_id > max_node_value:
-                    raise ValueError('Current Node ID exceeding max. Look at source code to change.')
+                    raise ValueError("Current Node ID exceeding max. Look at source code to change.")
                 current_node_id = str(current_node_id).zfill(max_post_node_digits)
 
             else:
                 toNodes[i] = node_coordinates[outlet_coordinate]
 
-        self.loc[:,fromNode_attribute] = fromNodes
-        self.loc[:,toNode_attribute] = toNodes
-        
-        return(self)
+        self.loc[:, fromNode_attribute] = fromNodes
+        self.loc[:, toNode_attribute] = toNodes
 
+        return self
 
-    def derive_outlets(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',outlets_attribute='outlet_id',
-                       verbose=False):
-
+    def derive_outlets(
+        self,
+        toNode_attribute="ToNode",
+        fromNode_attribute="FromNode",
+        outlets_attribute="outlet_id",
+        verbose=False,
+    ):
         if verbose:
             print("Deriving outlets ...")
 
         fromNodes = set(i for i in self[fromNode_attribute])
 
         outlets = [-1] * len(self)
-        
-        for i,tn in enumerate(self[toNode_attribute]):
+
+        for i, tn in enumerate(self[toNode_attribute]):
             if tn not in fromNodes:
                 outlets[i] = i + 1
 
         self[outlets_attribute] = outlets
 
-        return(self)
-
-
-    def derive_inlets(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',
-                      inlets_attribute='inlet_id',
-                      verbose=False
-                     ):
+        return self
 
+    def derive_inlets(
+        self,
+        toNode_attribute="ToNode",
+        fromNode_attribute="FromNode",
+        inlets_attribute="inlet_id",
+        verbose=False,
+    ):
         if verbose:
             print("Deriving inlets ...")
 
@@ -332,159 +352,164 @@ def derive_inlets(self,toNode_attribute='ToNode',fromNode_attribute='FromNode',
 
         inlets = [-1] * len(self)
 
-        for i,fn in enumerate(self[fromNode_attribute]):
+        for i, fn in enumerate(self[fromNode_attribute]):
             if fn not in toNodes:
                 inlets[i] = i + 1
 
         self[inlets_attribute] = inlets
 
-        return(self)
-
+        return self
 
-    def derive_inlet_points_by_feature( self,feature_attribute, outlet_linestring_index):
-        
-        """ Finds the upstream point of every feature in the stream network """
+    def derive_inlet_points_by_feature(self, feature_attribute, outlet_linestring_index):
+        """Finds the upstream point of every feature in the stream network"""
 
         inlet_linestring_index = StreamNetwork.flip_inlet_outlet_linestring_index(outlet_linestring_index)
 
         feature_inlet_points_gdf = gpd.GeoDataFrame(self.copy())
 
         for idx, row in self.iterrows():
-            feature_inlet_point = Point(row.geometry.coords[inlet_linestring_index])  
+            feature_inlet_point = Point(row.geometry.coords[inlet_linestring_index])
 
-            feature_inlet_points_gdf.loc[idx,'geometry'] = feature_inlet_point
+            feature_inlet_points_gdf.loc[idx, "geometry"] = feature_inlet_point
 
-        return(feature_inlet_points_gdf)
+        return feature_inlet_points_gdf
 
+    def derive_headwater_points_with_inlets(
+        self,
+        inlets_attribute="inlet_id",
+        fromNode_attribute="FromNode",
+        outlet_linestring_index=0,
+    ):
+        """Derives headwater points file given inlets"""
 
-    def derive_headwater_points_with_inlets( self,inlets_attribute='inlet_id',
-                                             fromNode_attribute='FromNode',
-                                             outlet_linestring_index=0
-                                           ):
-    
-        """ Derives headwater points file given inlets """
-        
         # get inlet linestring index
         inlet_linestring_index = StreamNetwork.flip_inlet_outlet_linestring_index(outlet_linestring_index)
 
-        inlet_indices = self.loc[:,inlets_attribute] != -1
+        inlet_indices = self.loc[:, inlets_attribute] != -1
 
-        inlets = self.loc[inlet_indices,:].reset_index(drop=True)
+        inlets = self.loc[inlet_indices, :].reset_index(drop=True)
 
         headwater_points_gdf = gpd.GeoDataFrame(inlets.copy())
-        
-        for idx,row in inlets.iterrows():
+
+        for idx, row in inlets.iterrows():
             headwater_point = row.geometry.coords[inlet_linestring_index]
 
             headwater_point = Point(headwater_point)
 
-            headwater_points_gdf.loc[idx,'geometry'] = headwater_point
-
-        return(headwater_points_gdf)
+            headwater_points_gdf.loc[idx, "geometry"] = headwater_point
 
+        return headwater_points_gdf
 
     def exclude_attribute_values(self, branch_id_attribute=None, values_excluded=None, verbose=False):
-                      
         if (branch_id_attribute is not None) and (values_excluded is not None):
-            self = StreamNetwork(self[~self[branch_id_attribute].isin(values_excluded)], branch_id_attribute=branch_id_attribute)
-       
-        if verbose:         
-             print("Number of df rows = " + str(self.shape[0]))
-        
-        return(self)
+            self = StreamNetwork(
+                self[~self[branch_id_attribute].isin(values_excluded)],
+                branch_id_attribute=branch_id_attribute,
+            )
 
+        if verbose:
+            print("Number of df rows = " + str(self.shape[0]))
 
-    def remove_stream_segments_without_catchments( self,
-                                                   catchments,
-                                                   reach_id_attribute='ID',
-                                                   reach_id_attribute_in_catchments='ID',
-                                                   verbose=False
-                                                 ):
+        return self
 
+    def remove_stream_segments_without_catchments(
+        self,
+        catchments,
+        reach_id_attribute="ID",
+        reach_id_attribute_in_catchments="ID",
+        verbose=False,
+    ):
         if verbose:
             print("Removing stream segments without catchments ...")
 
         # load catchments
-        if isinstance(catchments,gpd.GeoDataFrame):
+        if isinstance(catchments, gpd.GeoDataFrame):
             pass
-        elif isinstance(catchments,str):
+        elif isinstance(catchments, str):
             catchments = gpd.read_file(catchments)
         else:
             raise TypeError("Catchments needs to be GeoDataFame or path to vector file")
-        
-        self = self.merge(catchments.loc[:,reach_id_attribute_in_catchments],
-                          left_on=reach_id_attribute,
-                          right_on=reach_id_attribute_in_catchments,
-                          how='inner')
-
-        return(self)
-
-
-    def remove_branches_without_catchments(self,
-                                           catchments,
-                                           reach_id_attribute='ID',
-                                           branch_id_attribute='branchID',
-                                           reach_id_attribute_in_catchments='ID',
-                                           verbose=False
-                                           ):
 
+        self = self.merge(
+            catchments.loc[:, reach_id_attribute_in_catchments],
+            left_on=reach_id_attribute,
+            right_on=reach_id_attribute_in_catchments,
+            how="inner",
+        )
+
+        return self
+
+    def remove_branches_without_catchments(
+        self,
+        catchments,
+        reach_id_attribute="ID",
+        branch_id_attribute="branchID",
+        reach_id_attribute_in_catchments="ID",
+        verbose=False,
+    ):
         if verbose:
             print("Removing stream branches without catchments ...")
 
         # load catchments
         if isinstance(catchments, gpd.GeoDataFrame):
             pass
-        elif isinstance(catchments,str):
+        elif isinstance(catchments, str):
             catchments = gpd.read_file(catchments)
         else:
             raise TypeError("Catchments needs to be GeoDataFame or path to vector file")
-        
-        unique_stream_branches = self.loc[:,branch_id_attribute].unique()
-        unique_catchments = set(catchments.loc[:,reach_id_attribute_in_catchments].unique())
+
+        unique_stream_branches = self.loc[:, branch_id_attribute].unique()
+        unique_catchments = set(catchments.loc[:, reach_id_attribute_in_catchments].unique())
 
         current_index_name = self.index.name
         self.set_index(branch_id_attribute, drop=False, inplace=True)
 
         for usb in unique_stream_branches:
-
             try:
-                reach_ids_in_branch = set(self.loc[usb,reach_id_attribute].unique())
+                reach_ids_in_branch = set(self.loc[usb, reach_id_attribute].unique())
             except AttributeError:
-                reach_ids_in_branch = set( [ self.loc[usb,reach_id_attribute] ] )
+                reach_ids_in_branch = set([self.loc[usb, reach_id_attribute]])
 
-            if len( reach_ids_in_branch & unique_catchments) == 0:
-                #print(f'Dropping {usb}')
-                self.drop(usb,inplace=True)
+            if len(reach_ids_in_branch & unique_catchments) == 0:
+                # print(f'Dropping {usb}')
+                self.drop(usb, inplace=True)
 
         if current_index_name is None:
             self.reset_index(drop=True, inplace=True)
         else:
             self.set_index(current_index_name, drop=True, inplace=True)
 
-        return(self)
+        return self
 
-
-    def trim_branches_in_waterbodies(self,
-                                       branch_id_attribute,
-                                       verbose=False
-                                       ):
+    def trim_branches_in_waterbodies(self, branch_id_attribute, verbose=False):
         """
-        Recursively trims the reaches from the ends of the branches if they are in a 
+        Recursively trims the reaches from the ends of the branches if they are in a
         waterbody (determined by the Lake attribute).
         """
 
         def find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
             # Find lowest reach(es)
-            downstream_IDs = [int(x) for x in tmp_self.From_Node[~tmp_self.To_Node.isin(tmp_self.From_Node)]] # IDs of most downstream reach(es)
+            downstream_IDs = [
+                int(x) for x in tmp_self.From_Node[~tmp_self.To_Node.isin(tmp_self.From_Node)]
+            ]  # IDs of most downstream reach(es)
 
             for downstream_ID in downstream_IDs:
                 # Stop if lowest reach is not in a lake
-                if tmp_self.Lake[tmp_self.From_Node.astype(int)==downstream_ID].iloc[0] == -9999:
+                if tmp_self.Lake[tmp_self.From_Node.astype(int) == downstream_ID].iloc[0] == -9999:
                     continue
                 else:
                     # Remove reach from tmp_self
                     tmp_IDs.append(downstream_ID)
-                    tmp_self.drop(tmp_self[tmp_self.From_Node.astype(int).isin([downstream_ID,])].index, inplace=True)
+                    tmp_self.drop(
+                        tmp_self[
+                            tmp_self.From_Node.astype(int).isin(
+                                [
+                                    downstream_ID,
+                                ]
+                            )
+                        ].index,
+                        inplace=True,
+                    )
                     # Repeat for next lowest downstream reach
                     if downstream_ID in tmp_self.To_Node.astype(int).values:
                         return find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
@@ -492,19 +517,35 @@ def find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
 
         def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
             # Find highest reach(es)
-            upstream_IDs = [int(x) for x in tmp_self.From_Node[~tmp_self.From_Node.isin(tmp_self.To_Node)]] # IDs of most upstream reach(es)
-            nonlake_reaches = [int(x) for x in tmp_self.From_Node[tmp_self.Lake  == -9999]] # IDs of most  reach(es) that are not designated as lake reaches
-            
+            upstream_IDs = [
+                int(x) for x in tmp_self.From_Node[~tmp_self.From_Node.isin(tmp_self.To_Node)]
+            ]  # IDs of most upstream reach(es)
+            nonlake_reaches = [
+                int(x) for x in tmp_self.From_Node[tmp_self.Lake == -9999]
+            ]  # IDs of most  reach(es) that are not designated as lake reaches
+
             for upstream_ID in upstream_IDs:
                 # Stop if uppermost reach is not in a lake
-                if tmp_self.Lake[tmp_self.From_Node.astype(int)==upstream_ID].iloc[0] == -9999:
+                if tmp_self.Lake[tmp_self.From_Node.astype(int) == upstream_ID].iloc[0] == -9999:
                     continue
                 else:
-                    if int(tmp_self.To_Node[tmp_self.From_Node.astype(int)==upstream_ID]) in nonlake_reaches:
+                    if (
+                        int(tmp_self.To_Node[tmp_self.From_Node.astype(int) == upstream_ID])
+                        in nonlake_reaches
+                    ):
                         continue
                     # Remove reach from tmp_self
                     tmp_IDs.append(upstream_ID)
-                    tmp_self.drop(tmp_self[tmp_self.From_Node.astype(int).isin([upstream_ID,])].index, inplace=True)
+                    tmp_self.drop(
+                        tmp_self[
+                            tmp_self.From_Node.astype(int).isin(
+                                [
+                                    upstream_ID,
+                                ]
+                            )
+                        ].index,
+                        inplace=True,
+                    )
                     # Repeat for next highest upstream reach
                     return find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
             return tmp_IDs
@@ -513,12 +554,12 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
             print("Trimming stream branches in waterbodies ...")
 
         for branch in self[branch_id_attribute].astype(int).unique():
-            tmp_self = self[self[branch_id_attribute].astype(int)==branch]
+            tmp_self = self[self[branch_id_attribute].astype(int) == branch]
 
             # If entire branch is in waterbody
             if all(tmp_self.Lake.values != -9999):
                 tmp_IDs = tmp_self.From_Node.astype(int)
-                
+
             else:
                 # Find bottom up
                 tmp_IDs = find_downstream_reaches_in_waterbodies(tmp_self)
@@ -529,117 +570,154 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
             if len(tmp_IDs) > 0:
                 self.drop(self[self.From_Node.astype(int).isin(tmp_IDs)].index, inplace=True)
 
-        return(self)
-
+        return self
 
-    def remove_branches_in_waterbodies(self,
-                                       waterbodies,
-                                       out_vector_files=None,
-                                       verbose=False
-                                       ):
+    def remove_branches_in_waterbodies(self, waterbodies, out_vector_files=None, verbose=False):
         """
         Removes branches completely in waterbodies
         """
 
         if verbose:
-            print('Removing branches in waterbodies')
+            print("Removing branches in waterbodies")
 
         # load waterbodies
-        if isinstance(waterbodies,str) and isfile(waterbodies):
+        if isinstance(waterbodies, str) and isfile(waterbodies):
             waterbodies = gpd.read_file(waterbodies)
 
-        if isinstance(waterbodies,gpd.GeoDataFrame):
+        if isinstance(waterbodies, gpd.GeoDataFrame):
             # Find branches in waterbodies
-            sjoined = gpd.sjoin(self, waterbodies, predicate='within')
+            sjoined = gpd.sjoin(self, waterbodies, predicate="within")
             self.drop(sjoined.index, inplace=True)
 
             if out_vector_files is not None:
-                
                 if verbose:
                     print("Writing pruned branches ...")
-                
+
                 self.write(out_vector_files, index=False)
 
-        return(self)
+        return self
+
+    def select_branches_intersecting_huc(self, wbd, buffer_wbd_streams, out_vector_files, verbose=False):
+        """
+        Select branches that intersect the HUC
+        """
+
+        if verbose:
+            print("Selecting branches that intersect the HUC")
+
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+
+        # Check if required input files exist
+        for filename in [buffer_wbd_streams, wbd]:
+            if not isfile(buffer_wbd_streams):
+                raise FileNotFoundError(f"{filename} does not exist")
+
+        # load waterbodies
+        if isinstance(wbd, str):
+            wbd = gpd.read_file(wbd)
 
+        if isinstance(wbd, gpd.GeoDataFrame):
+            # Find branches intersecting HUC
+            sjoined = gpd.sjoin(self, wbd, predicate="intersects")
+
+            self = self[self.index.isin(sjoined.index.values)]
+
+            self = StreamNetwork(
+                self,
+                branch_id_attribute=branch_id_attribute,
+                attribute_excluded=attribute_excluded,
+                values_excluded=values_excluded,
+            )
+
+            if out_vector_files is not None:
+                if verbose:
+                    print("Writing selected branches ...")
 
-    def derive_stream_branches(self,toNode_attribute='ToNode',
-                               fromNode_attribute='FromNode',
-                               upstreams=None,
-                               outlet_attribute='outlet_id',
-                               branch_id_attribute='branchID',
-                               reach_id_attribute='ID',
-                               comparison_attributes='StreamOrde',
-                               comparison_function=max,
-                               max_branch_id_digits=6,
-                               verbose=False):
+                self.write(out_vector_files, index=False)
 
-        """ Derives stream branches """
+        return self
+
+    def derive_stream_branches(
+        self,
+        toNode_attribute="ToNode",
+        fromNode_attribute="FromNode",
+        upstreams=None,
+        outlet_attribute="outlet_id",
+        branch_id_attribute="branchID",
+        reach_id_attribute="ID",
+        comparison_attributes="StreamOrde",
+        comparison_function=max,
+        max_branch_id_digits=6,
+        verbose=False,
+    ):
+        """Derives stream branches"""
 
         # checks inputs
-        allowed_comparison_function = {max,min}
+        allowed_comparison_function = {max, min}
         if comparison_function not in allowed_comparison_function:
             raise ValueError(f"Only {allowed_comparison_function} comparison functions allowed")
 
         # sets index of stream branches as reach id attribute
         reset_index = False
         if self.index.name != reach_id_attribute:
-            self.set_index(reach_id_attribute,drop=True,inplace=True)
+            self.set_index(reach_id_attribute, drop=True, inplace=True)
             reset_index = True
 
         # make upstream and downstream dictionaries if none are passed
         if upstreams is None:
-            upstreams,_ = self.make_up_and_downstream_dictionaries(reach_id_attribute=reach_id_attribute,
-                                                                   toNode_attribute=toNode_attribute,
-                                                                   fromNode_attribute=fromNode_attribute,
-                                                                   verbose=verbose)
-        
+            upstreams, _ = self.make_up_and_downstream_dictionaries(
+                reach_id_attribute=reach_id_attribute,
+                toNode_attribute=toNode_attribute,
+                fromNode_attribute=fromNode_attribute,
+                verbose=verbose,
+            )
+
         # initialize empty queue, visited set, branch attribute column, and all toNodes set
         Q = deque()
         visited = set()
         self[branch_id_attribute] = [-1] * len(self)
 
         # progress bar
-        progress = tqdm(total=len(self),disable=(not verbose),desc='Stream branches')
+        progress = tqdm(total=len(self), disable=(not verbose), desc="Stream branches")
 
         outlet_boolean_mask = self[outlet_attribute] >= 0
         outlet_reach_ids = self.index[outlet_boolean_mask].tolist()
 
-        branch_ids = [ str(h)[0:4] + str(b+1).zfill(max_branch_id_digits) for b,h in enumerate(outlet_reach_ids) ]
+        branch_ids = [
+            str(h)[0:4] + str(b + 1).zfill(max_branch_id_digits) for b, h in enumerate(outlet_reach_ids)
+        ]
 
-        self.loc[outlet_reach_ids,branch_id_attribute] = branch_ids
+        self.loc[outlet_reach_ids, branch_id_attribute] = branch_ids
         Q = deque(outlet_reach_ids)
         visited = set(outlet_reach_ids)
-        bid = int(branch_ids[-1][-max_branch_id_digits:].lstrip('0')) + 1
-        progress.update(bid-1)
+        bid = int(branch_ids[-1][-max_branch_id_digits:].lstrip("0")) + 1
+        progress.update(bid - 1)
 
         # breath-first traversal
         # while queue contains reaches
         while Q:
-
             # pop current reach id from queue
             current_reach_id = Q.popleft()
-            
+
             # update progress
             progress.update(1)
 
             # get current reach stream order and branch id
             # current_reach_comparison_value = self.at[current_reach_id,comparison_attributes]
-            current_reach_branch_id = self.at[current_reach_id,branch_id_attribute]
+            current_reach_branch_id = self.at[current_reach_id, branch_id_attribute]
 
             # get upstream ids
             upstream_ids = upstreams[current_reach_id]
 
             # identify upstreams by finding if fromNode exists in set of all toNodes
             if upstream_ids:
-                
                 # determine if each upstream has been visited or not
-                not_visited_upstream_ids = [] # list to save not visited upstreams
+                not_visited_upstream_ids = []  # list to save not visited upstreams
                 for us in upstream_ids:
-
                     # if upstream id has not been visited
                     if us not in visited:
-
                         # add to visited set and to queue
                         visited.add(us)
                         Q.append(us)
@@ -647,32 +725,43 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
 
                 # if upstreams that are not visited exist
                 if not_visited_upstream_ids:
-
-                    # find 
-                    upstream_reaches_compare_values = self.loc[not_visited_upstream_ids,comparison_attributes]
+                    # find
+                    upstream_reaches_compare_values = self.loc[
+                        not_visited_upstream_ids, comparison_attributes
+                    ]
                     # matching_value = comparison_function(upstream_reaches_compare_values)
 
-                    #==================================================================================
+                    # ==================================================================================
                     # If the two stream orders aren't the same, then follow the highest order, otherwise use arbolate sum
-                    if upstream_reaches_compare_values.idxmax()['order_'] == upstream_reaches_compare_values.idxmin()['order_']:
-                        decision_attribute = 'arbolate_sum'
+                    if (
+                        upstream_reaches_compare_values.idxmax()["order_"]
+                        == upstream_reaches_compare_values.idxmin()["order_"]
+                    ):
+                        decision_attribute = "arbolate_sum"
                     else:
-                        decision_attribute = 'order_'
+                        decision_attribute = "order_"
                     # Continue the current branch up the larger stream
                     continue_id = upstream_reaches_compare_values.idxmax()[decision_attribute]
-                    self.loc[continue_id,branch_id_attribute] = current_reach_branch_id
+                    self.loc[continue_id, branch_id_attribute] = current_reach_branch_id
                     # Create a new level path for the smaller tributary(ies)
-                    if len(not_visited_upstream_ids) == 1: continue # only create a new branch if there are 2 upstreams
-                    new_upstream_branches = upstream_reaches_compare_values.loc[~upstream_reaches_compare_values.index.isin([continue_id,])]
+                    if len(not_visited_upstream_ids) == 1:
+                        continue  # only create a new branch if there are 2 upstreams
+                    new_upstream_branches = upstream_reaches_compare_values.loc[
+                        ~upstream_reaches_compare_values.index.isin(
+                            [
+                                continue_id,
+                            ]
+                        )
+                    ]
                     for new_up_id in new_upstream_branches.index:
                         branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
-                        self.loc[new_up_id,branch_id_attribute] = branch_id
+                        self.loc[new_up_id, branch_id_attribute] = branch_id
                         bid += 1
-                    #==================================================================================
-                    ''' NOTE: The above logic uses stream order to override arbolate sum. Use the commented section
-                     below if this turns out to be a bad idea!'''
-                    #matches = 0 # if upstream matches are more than 1, limits to only one match
-                    #for usrcv,nvus in zip(upstream_reaches_compare_values,not_visited_upstream_ids):
+                    # ==================================================================================
+                    """ NOTE: The above logic uses stream order to override arbolate sum. Use the commented section
+                     below if this turns out to be a bad idea!"""
+                    # matches = 0 # if upstream matches are more than 1, limits to only one match
+                    # for usrcv,nvus in zip(upstream_reaches_compare_values,not_visited_upstream_ids):
                     #    if (usrcv == matching_value) & (matches == 0):
                     #        self.at[nvus,branch_id_attribute] = current_reach_branch_id
                     #        matches += 1
@@ -680,57 +769,70 @@ def derive_stream_branches(self,toNode_attribute='ToNode',
                     #        branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
                     #        self.at[nvus,branch_id_attribute] = branch_id
                     #        bid += 1
-        
+
         progress.close()
 
         if reset_index:
-            self.reset_index(drop=False,inplace=True)
+            self.reset_index(drop=False, inplace=True)
 
-        return(self)
+        return self
 
-
-    def make_up_and_downstream_dictionaries(self,reach_id_attribute='ID',
-                                            toNode_attribute='ToNode',
-                                            fromNode_attribute='FromNode',
-                                            verbose=False):
-        
+    def make_up_and_downstream_dictionaries(
+        self,
+        reach_id_attribute="ID",
+        toNode_attribute="ToNode",
+        fromNode_attribute="FromNode",
+        verbose=False,
+    ):
         # sets index of stream branches as reach id attribute
-        #if self.index.name != reach_id_attribute:
+        # if self.index.name != reach_id_attribute:
         #    self.set_index(reach_id_attribute,drop=True,inplace=True)
 
         # find upstream and downstream dictionaries
-        upstreams,downstreams = dict(),dict()
-        
-        for _, row in tqdm(self.iterrows(),disable=(not verbose),
-                           total=len(self), desc='Upstream and downstream dictionaries'):
-            
+        upstreams, downstreams = dict(), dict()
+
+        for _, row in tqdm(
+            self.iterrows(),
+            disable=(not verbose),
+            total=len(self),
+            desc="Upstream and downstream dictionaries",
+        ):
             reach_id = row[reach_id_attribute]
-            downstreams[reach_id] = self.loc[ self[fromNode_attribute] == row[toNode_attribute] , reach_id_attribute].tolist() 
-            upstreams[reach_id] = self.loc[ self[toNode_attribute] == row[fromNode_attribute] , reach_id_attribute].tolist()
-        
-        return(upstreams,downstreams)
-
-
-    def get_arbolate_sum(self,arbolate_sum_attribute='arbolate_sum',inlets_attribute='inlet_id',
-                         reach_id_attribute='ID',length_conversion_factor_to_km = 0.001,
-                         upstreams=None, downstreams=None,
-                         toNode_attribute='ToNode',
-                         fromNode_attribute='FromNode',
-                         verbose=False
-                        ):
-        
+            downstreams[reach_id] = self.loc[
+                self[fromNode_attribute] == row[toNode_attribute], reach_id_attribute
+            ].tolist()
+            upstreams[reach_id] = self.loc[
+                self[toNode_attribute] == row[fromNode_attribute], reach_id_attribute
+            ].tolist()
+
+        return (upstreams, downstreams)
+
+    def get_arbolate_sum(
+        self,
+        arbolate_sum_attribute="arbolate_sum",
+        inlets_attribute="inlet_id",
+        reach_id_attribute="ID",
+        length_conversion_factor_to_km=0.001,
+        upstreams=None,
+        downstreams=None,
+        toNode_attribute="ToNode",
+        fromNode_attribute="FromNode",
+        verbose=False,
+    ):
         # sets index of stream branches as reach id attribute
         reset_index = False
         if self.index.name != reach_id_attribute:
-            self.set_index(reach_id_attribute,drop=True,inplace=True)
+            self.set_index(reach_id_attribute, drop=True, inplace=True)
             reset_index = True
 
         # make upstream and downstream dictionaries if none are passed
         if (upstreams is None) | (downstreams is None):
-            upstreams, downstreams = self.make_up_and_downstream_dictionaries(reach_id_attribute=reach_id_attribute,
-                                                                              toNode_attribute=toNode_attribute,
-                                                                              fromNode_attribute=fromNode_attribute,
-                                                                              verbose=verbose)
+            upstreams, downstreams = self.make_up_and_downstream_dictionaries(
+                reach_id_attribute=reach_id_attribute,
+                toNode_attribute=toNode_attribute,
+                fromNode_attribute=fromNode_attribute,
+                verbose=verbose,
+            )
 
         # initialize queue, visited set, with inlet reach ids
         inlet_reach_ids = self.index[self[inlets_attribute] >= 0].tolist()
@@ -738,20 +840,19 @@ def get_arbolate_sum(self,arbolate_sum_attribute='arbolate_sum',inlets_attribute
         visited = set()
 
         # initialize arbolate sum, make length km column, make all from nodes set
-        self[arbolate_sum_attribute] = self.geometry.length * length_conversion_factor_to_km 
-        
+        self[arbolate_sum_attribute] = self.geometry.length * length_conversion_factor_to_km
+
         # progress bar
-        progress = tqdm(total=len(self),disable=(not verbose), desc= "Arbolate sums")
+        progress = tqdm(total=len(self), disable=(not verbose), desc="Arbolate sums")
 
         # depth-first traversal
         # while stack contains reaches
         while S:
-            
             # pop current reach id from queue
             current_reach_id = S.pop()
-            
+
             # current arbolate sum
-            current_reach_arbolate_sum = self.at[current_reach_id,arbolate_sum_attribute]
+            current_reach_arbolate_sum = self.at[current_reach_id, arbolate_sum_attribute]
 
             # if current reach id is not visited mark as visited
             if current_reach_id not in visited:
@@ -760,32 +861,34 @@ def get_arbolate_sum(self,arbolate_sum_attribute='arbolate_sum',inlets_attribute
 
             # get downstream ids
             downstream_ids = downstreams[current_reach_id]
-            
+
             if downstream_ids:
-                
                 for ds in downstream_ids:
-                    
                     # figure out of all upstream reaches of ds have been visited
                     upstream_of_ds_ids = set(upstreams[ds])
                     all_upstream_ids_of_ds_are_visited = upstream_of_ds_ids.issubset(visited)
-                    
+
                     # append downstream to stack
                     if all_upstream_ids_of_ds_are_visited:
                         S.append(ds)
-                        
-                    self.loc[ds,arbolate_sum_attribute] += current_reach_arbolate_sum
-        
-        progress.close()
-    
-        if reset_index:
-            self.reset_index(drop=False,inplace=True)
 
-        return(self)
+                    self.loc[ds, arbolate_sum_attribute] += current_reach_arbolate_sum
 
+        progress.close()
 
-    def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_excluded='StreamOrde',
-                           values_excluded=[1,2], out_vector_files=None, verbose=False):
-        
+        if reset_index:
+            self.reset_index(drop=False, inplace=True)
+
+        return self
+
+    def dissolve_by_branch(
+        self,
+        branch_id_attribute="LevelPathI",
+        attribute_excluded="StreamOrde",
+        values_excluded=[1, 2],
+        out_vector_files=None,
+        verbose=False,
+    ):
         if verbose:
             print("Dissolving by branch ...")
 
@@ -793,215 +896,235 @@ def dissolve_by_branch(self, branch_id_attribute='LevelPathI', attribute_exclude
         if (attribute_excluded is not None) & (values_excluded is not None):
             values_excluded = set(values_excluded)
             exclude_indices = [False if i in values_excluded else True for i in self[attribute_excluded]]
-            self = self.loc[exclude_indices,:]
+            self = self.loc[exclude_indices, :]
 
         # dissolve lines
-        self['bids_temp'] = self.loc[:,branch_id_attribute].copy()
+        self["bids_temp"] = self.loc[:, branch_id_attribute].copy()
 
         # ensure the new stream order has the order from it's highest child
-        max_stream_order = self[[branch_id_attribute, 'order_']].groupby(branch_id_attribute).max()['order_'].copy()
+        max_stream_order = (
+            self[[branch_id_attribute, "order_"]].groupby(branch_id_attribute).max()["order_"].copy()
+        )
 
         self = self.dissolve(by=branch_id_attribute)
-        self.rename(columns={'bids_temp' : branch_id_attribute},inplace=True)
-        
+        self.rename(columns={"bids_temp": branch_id_attribute}, inplace=True)
+
         self["order_"] = max_stream_order.values
 
         # merges each multi-line string to a singular linestring
-        for lpid,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc="Merging mult-part geoms"):
-            if isinstance(row.geometry,MultiLineString):
+        for lpid, row in tqdm(
+            self.iterrows(),
+            total=len(self),
+            disable=(not verbose),
+            desc="Merging mult-part geoms",
+        ):
+            if isinstance(row.geometry, MultiLineString):
                 merged_line = linemerge(row.geometry)
-                #self.loc[lpid,'geometry'] = merged_line
+                # self.loc[lpid,'geometry'] = merged_line
                 try:
-                    self.loc[lpid,'geometry'] = merged_line
+                    self.loc[lpid, "geometry"] = merged_line
                 except ValueError:
                     merged_line = list(merged_line.geoms)[0]
-                    self.loc[lpid,'geometry'] = merged_line
-        
-        #self[branch_id_attribute] = bids
-        self = StreamNetwork(self, branch_id_attribute=branch_id_attribute,
-                             attribute_excluded=attribute_excluded,
-                             values_excluded=values_excluded)
+                    self.loc[lpid, "geometry"] = merged_line
+
+        # self[branch_id_attribute] = bids
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
 
         if out_vector_files is not None:
-            
             # base_file_path,extension = splitext(out_vector_files)
-            
+
             if verbose:
                 print("Writing dissolved branches ...")
-            
-            #for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
-                #out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
-                
-                #bid_indices = self.loc[:,branch_id_attribute] == bid
-                #current_stream_network = StreamNetwork(self.loc[bid_indices,:])
-
-                #current_stream_network.write(out_vector_file,index=False)
-            self.write(out_vector_files, index=False)
 
-        return(self)
+            # for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
+            # out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
 
+            # bid_indices = self.loc[:,branch_id_attribute] == bid
+            # current_stream_network = StreamNetwork(self.loc[bid_indices,:])
 
-    def derive_segments(self,inlets_attribute='inlet_id', reach_id_attribute='ID'):
-        pass
+            # current_stream_network.write(out_vector_file,index=False)
+            self.write(out_vector_files, index=False)
+
+        return self
 
+    def derive_segments(self, inlets_attribute="inlet_id", reach_id_attribute="ID"):
+        pass
 
-    def conflate_branches(self,target_stream_network,branch_id_attribute_left='branch_id',
-                          branch_id_attribute_right='branch_id', left_order_attribute='order_',
-                          right_order_attribute='order_',
-                          crosswalk_attribute='crosswalk_id', verbose=False):
-        
+    def conflate_branches(
+        self,
+        target_stream_network,
+        branch_id_attribute_left="branch_id",
+        branch_id_attribute_right="branch_id",
+        left_order_attribute="order_",
+        right_order_attribute="order_",
+        crosswalk_attribute="crosswalk_id",
+        verbose=False,
+    ):
         # get unique stream orders
-        orders = self.loc[:,right_order_attribute].unique()
+        orders = self.loc[:, right_order_attribute].unique()
 
         # make a dictionary of STR trees for every stream order
-        trees = { o:STRtree(target_stream_network.geometry.tolist()) for o in orders }
+        trees = {o: STRtree(target_stream_network.geometry.tolist()) for o in orders}
 
         # make the crosswalk id attribute and set index
-        self.loc[:,crosswalk_attribute] = [None] * len(self)
-        self.set_index(branch_id_attribute_left,inplace=True)
+        self.loc[:, crosswalk_attribute] = [None] * len(self)
+        self.set_index(branch_id_attribute_left, inplace=True)
 
         # loop through rows of self
-        for idx,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc="Conflating branches"):
-
-            g = row['geometry']
+        for idx, row in tqdm(
+            self.iterrows(),
+            total=len(self),
+            disable=(not verbose),
+            desc="Conflating branches",
+        ):
+            g = row["geometry"]
             o = row[left_order_attribute]
-            
+
             tree = trees[o]
 
             # find nearest geom in target and its index
             matching_geom = tree.nearest(g)
             match_idx = target_stream_network.geometry == matching_geom
-            
+
             # get the branch ids
-            right_branch_id = int(target_stream_network.loc[match_idx,branch_id_attribute_left])
+            right_branch_id = int(target_stream_network.loc[match_idx, branch_id_attribute_left])
             left_branch_id = idx
 
-            # save the target matching branch id 
-            self.loc[left_branch_id,crosswalk_attribute] = right_branch_id
+            # save the target matching branch id
+            self.loc[left_branch_id, crosswalk_attribute] = right_branch_id
 
         # reset indices
-        self.reset_index(inplace=True,drop=False)
+        self.reset_index(inplace=True, drop=False)
 
-        return(self)
-        
+        return self
 
-    def explode_to_points(self,reach_id_attribute='ID', sampling_size=None,
-                          verbose=False):
-        
+    def explode_to_points(self, reach_id_attribute="ID", sampling_size=None, verbose=False):
         points_gdf = self.copy()
-        points_gdf.reset_index(inplace=True,drop=True)
+        points_gdf.reset_index(inplace=True, drop=True)
 
         all_exploded_points = [None] * len(points_gdf)
-        for idx,row in tqdm(self.iterrows(),total=len(self),disable=(not verbose),desc='Exploding Points'):
-            
-            geom = row['geometry']
-            
+        for idx, row in tqdm(
+            self.iterrows(),
+            total=len(self),
+            disable=(not verbose),
+            desc="Exploding Points",
+        ):
+            geom = row["geometry"]
+
             exploded_points = [p for p in iter(geom.coords)]
 
             if sampling_size is None:
                 exploded_points = MultiPoint(exploded_points)
             else:
                 try:
-                    exploded_points = MultiPoint( sample(exploded_points,sampling_size) )
+                    exploded_points = MultiPoint(sample(exploded_points, sampling_size))
                 except ValueError:
-                    exploded_points = MultiPoint( exploded_points )
-            
+                    exploded_points = MultiPoint(exploded_points)
+
             all_exploded_points[idx] = exploded_points
 
-        points_gdf['geometry'] = all_exploded_points
-        
+        points_gdf["geometry"] = all_exploded_points
+
         points_gdf = points_gdf.explode(index_parts=True)
-        points_gdf.reset_index(inplace=True,drop=True)
+        points_gdf.reset_index(inplace=True, drop=True)
 
-        return(points_gdf)
+        return points_gdf
 
     @staticmethod
-    def conflate_points(source_points,target_points,source_reach_id_attribute,target_reach_id_attribute,verbose=False):
-
+    def conflate_points(
+        source_points,
+        target_points,
+        source_reach_id_attribute,
+        target_reach_id_attribute,
+        verbose=False,
+    ):
         tree = STRtree(target_points.geometry.tolist())
 
         # find matching geometry
-        matches_dict = dict.fromkeys(source_points.loc[:,source_reach_id_attribute].astype(int).tolist(),[])
-        for idx,row in tqdm(source_points.iterrows(),total=len(source_points),disable=(not verbose),desc="Conflating points"):
-
-            geom = row['geometry']
+        matches_dict = dict.fromkeys(source_points.loc[:, source_reach_id_attribute].astype(int).tolist(), [])
+        for idx, row in tqdm(
+            source_points.iterrows(),
+            total=len(source_points),
+            disable=(not verbose),
+            desc="Conflating points",
+        ):
+            geom = row["geometry"]
             nearest_target_point = tree.nearest(geom)
             match_idx = target_points.index[target_points.geometry == nearest_target_point].tolist()
-            
+
             if len(match_idx) > 1:
                 match_idx = match_idx[0]
             else:
                 match_idx = match_idx[0]
 
-            matched_id = int(target_points.loc[match_idx,target_reach_id_attribute])
+            matched_id = int(target_points.loc[match_idx, target_reach_id_attribute])
             source_id = int(row[source_reach_id_attribute])
             matches_dict[source_id] = matches_dict[source_id] + [matched_id]
 
-            #if len(matches_dict[source_id])>1:
+            # if len(matches_dict[source_id])>1:
             #    print(matches_dict[source_id])
 
         # get mode of matches
         if verbose:
             print("Finding mode of matches ...")
 
-        for source_id,matches in matches_dict.items():
+        for source_id, matches in matches_dict.items():
             majority = mode(matches).mode
             matches_dict[source_id] = majority[0]
-        
-        
+
         # make dataframe
         if verbose:
             print("Generating crosswalk table ...")
 
-        crosswalk_table = pd.DataFrame.from_dict(matches_dict,orient='index',
-                                                 columns=[target_reach_id_attribute])
+        crosswalk_table = pd.DataFrame.from_dict(
+            matches_dict, orient="index", columns=[target_reach_id_attribute]
+        )
         crosswalk_table.index.name = source_reach_id_attribute
 
+        return crosswalk_table
 
-        return(crosswalk_table)
-
-
-    def clip(self,mask,keep_geom_type=False,verbose=False):
-
+    def clip(self, mask, keep_geom_type=False, verbose=False):
         if verbose:
             print("Clipping streams to mask ...")
-        
+
         # load mask
-        if isinstance(mask,gpd.GeoDataFrame):
+        if isinstance(mask, gpd.GeoDataFrame):
             pass
-        elif isinstance(mask,str):
+        elif isinstance(mask, str):
             mask = gpd.read_file(mask)
         else:
             raise TypeError("mask needs to be GeoDataFame or path to vector file")
 
-
-        branch_id_attribute = self.branch_id_attribute 
+        branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
         values_excluded = self.values_excluded
-    
+
         self = StreamNetwork(
-                             gpd.clip(self,mask,keep_geom_type).reset_index(drop=True),
-                             branch_id_attribute=branch_id_attribute,
-                             attribute_excluded=attribute_excluded,
-                             values_excluded=values_excluded)
+            gpd.clip(self, mask, keep_geom_type).reset_index(drop=True),
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
 
-        return(self)
+        return self
 
 
 class StreamBranchPolygons(StreamNetwork):
+    # branch_id_attribute = None
+    # values_excluded = None
+    # attribute_excluded = None
 
-    #branch_id_attribute = None
-    #values_excluded = None
-    #attribute_excluded = None
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
 
-    def __init__(self,*args,**kwargs):
-        super().__init__(*args,**kwargs)
-     
-   
     @classmethod
-    def buffer_stream_branches(cls,stream_network,buffer_distance,verbose=True):
-
-        """ Buffers stream branches by distance """
+    def buffer_stream_branches(cls, stream_network, buffer_distance, verbose=True):
+        """Buffers stream branches by distance"""
 
         if verbose:
             print("Buffering stream branches to polygons")
@@ -1009,127 +1132,141 @@ def buffer_stream_branches(cls,stream_network,buffer_distance,verbose=True):
         buffer_distance = int(buffer_distance)
 
         # buffer lines
-        new_bids = [None] *len(stream_network) ; new_geoms = new_bids.copy()
-        i=0
-   
-        for _,row in tqdm(stream_network.iterrows(),disable=(not verbose),total=len(stream_network)):
+        new_bids = [None] * len(stream_network)
+        new_geoms = new_bids.copy()
+        i = 0
+
+        for _, row in tqdm(stream_network.iterrows(), disable=(not verbose), total=len(stream_network)):
             new_geoms[i] = row[stream_network.geometry.name].buffer(buffer_distance)
             new_bids[i] = i + 1
             i += 1
-        
+
         # create polys gpd
         polys = stream_network.copy()
 
         # assign to StreamBranchPolys
-        #polys[stream_network.branch_id_attribute] = new_bids
+        # polys[stream_network.branch_id_attribute] = new_bids
         polys[stream_network.geom_name] = new_geoms
         polys.set_geometry(stream_network.geom_name)
-        
+
         # assign class and attributes
-        polys = cls(polys,branch_id_attribute=stream_network.branch_id_attribute,
-                    attribute_excluded=stream_network.attribute_excluded,
-                    values_excluded=stream_network.values_excluded)
+        polys = cls(
+            polys,
+            branch_id_attribute=stream_network.branch_id_attribute,
+            attribute_excluded=stream_network.attribute_excluded,
+            values_excluded=stream_network.values_excluded,
+        )
 
-        return(polys)
+        return polys
 
     @staticmethod
-    def query_vectors_by_branch(vector,branch_ids,branch_id_attribute,out_filename_template=None,vector_layer=None):
-        
+    def query_vectors_by_branch(
+        vector,
+        branch_ids,
+        branch_id_attribute,
+        out_filename_template=None,
+        vector_layer=None,
+    ):
         # load vaas
-        if isinstance(vector,str):
+        if isinstance(vector, str):
             vector_filename = vector
-            #vector = gpd.read_file(vector_filename,layer=vector_layer)
-            vector = fiona.open(vector_filename,'r',layer=vector_layer)
-        elif isinstance(vector,fiona.Collection):
+            # vector = gpd.read_file(vector_filename,layer=vector_layer)
+            vector = fiona.open(vector_filename, "r", layer=vector_layer)
+        elif isinstance(vector, fiona.Collection):
             pass
         else:
-            raise TypeError('Pass vector argument as filepath or fiona collection')
-
+            raise TypeError("Pass vector argument as filepath or fiona collection")
 
-        def __find_matching_record(vector,attribute,value,matching='first'):
+        def __find_matching_record(vector, attribute, value, matching="first"):
+            if matching not in ("first", "all"):
+                raise ValueError("matching needs to be 'first' or 'all'")
 
-            if matching not in ('first','all'):
-                raise ValueError("matching needs to be \'first\' or \'all\'")
-            
             matches = []
             for rec in vector:
-               if rec['properties'][attribute] == value:
-                   if matching == 'first':
-                       matches = [ rec ]
-                       break
-                   elif matching == 'all':
-                       matches += [ rec ] 
-
-            return(matches)
+                if rec["properties"][attribute] == value:
+                    if matching == "first":
+                        matches = [rec]
+                        break
+                    elif matching == "all":
+                        matches += [rec]
 
+            return matches
 
         # get source information
         source_meta = vector.meta
 
         # out records
         out_records = []
-        
+
         for bid in branch_ids:
-            out_records += __find_matching_record(vector,branch_id_attribute,bid,matching='all')
-
-        if (out_filename_template is not None) & ( len(out_records) != 0):
-                base, ext = os.path.splitext(out_filename_template)
-                out_filename = base + "_{}".format(bid) + ext
-                
-                with fiona.open(out_filename,'w',**source_meta) as out_file:
-                    out_file.writerecords(out_records)
-                
+            out_records += __find_matching_record(vector, branch_id_attribute, bid, matching="all")
+
+        if (out_filename_template is not None) & (len(out_records) != 0):
+            base, ext = os.path.splitext(out_filename_template)
+            out_filename = base + "_{}".format(bid) + ext
+
+            with fiona.open(out_filename, "w", **source_meta) as out_file:
+                out_file.writerecords(out_records)
+
         # close
         vector.close()
 
-        return(out_records)
-
+        return out_records
 
-    def clip(self,to_clip,out_filename_template=None,branch_id=None,branch_id_attribute=None):
+    def clip(
+        self,
+        to_clip,
+        out_filename_template=None,
+        branch_id=None,
+        branch_id_attribute=None,
+    ):
+        """Clips a raster or vector to the stream branch polygons"""
 
-        """ Clips a raster or vector to the stream branch polygons """
+        fileType = "raster"  # default
 
-        fileType = "raster" #default
-        
         # load raster or vector file
-        if isinstance(to_clip,DatasetReader): #if already rasterio dataset
+        if isinstance(to_clip, DatasetReader):  # if already rasterio dataset
             pass
-        elif isinstance(to_clip,str): # if a string
-
-            try: # tries to open as rasterio first then geopanda
-                to_clip = rasterio.open(to_clip,'r')
+        elif isinstance(to_clip, str):  # if a string
+            try:  # tries to open as rasterio first then geopanda
+                to_clip = rasterio.open(to_clip, "r")
             except rasterio.errors.RasterioIOError:
                 try:
                     to_clip = gpd.read_file(to_clip)
                     fileType = "vector"
-                except DriverError: 
+                except DriverError:
                     raise IOError("{} file not found".format(to_clip))
 
-        elif isinstance(to_clip,gpd.GeoDataFrame): # if a geopanda dataframe
+        elif isinstance(to_clip, gpd.GeoDataFrame):  # if a geopanda dataframe
             fileType = "vector"
         else:
-            raise TypeError("Pass rasterio dataset,geopandas GeoDataFrame, or filepath to raster or vector file")
+            raise TypeError(
+                "Pass rasterio dataset,geopandas GeoDataFrame, or filepath to raster or vector file"
+            )
 
         # generator to iterate
         if branch_id is not None:
-            #print(iter(tuple([0,self.loc[self.loc[:,branch_id_attribute]==branch_id,:].squeeze()])))
-            generator_to_iterate = enumerate( [self.loc[self.loc[:,branch_id_attribute]==branch_id,:].squeeze()] ) 
+            # print(iter(tuple([0,self.loc[self.loc[:,branch_id_attribute]==branch_id,:].squeeze()])))
+            generator_to_iterate = enumerate(
+                [self.loc[self.loc[:, branch_id_attribute] == branch_id, :].squeeze()]
+            )
         else:
             generator_to_iterate = self.iterrows()
 
-        return_list = [] # list to return rasterio objects or gdf's
-        
+        return_list = []  # list to return rasterio objects or gdf's
+
         if fileType == "raster":
             buffered_meta = to_clip.meta.copy()
             buffered_meta.update(blockxsize=256, blockysize=256, tiled=True)
 
-            for i,row in generator_to_iterate:
-                buffered_array,buffered_transform = mask(to_clip,[row[self.geom_name]],crop=True)
+            for i, row in generator_to_iterate:
+                buffered_array, buffered_transform = mask(to_clip, [row[self.geom_name]], crop=True)
 
-                buffered_meta.update(height = buffered_array.shape[1],
-                                     width = buffered_array.shape[2],
-                                     transform = buffered_transform
-                                    )
+                buffered_meta.update(
+                    height=buffered_array.shape[1],
+                    width=buffered_array.shape[2],
+                    transform=buffered_transform,
+                )
 
                 # write out files
                 if out_filename_template is not None:
@@ -1137,24 +1274,24 @@ def clip(self,to_clip,out_filename_template=None,branch_id=None,branch_id_attrib
 
                     base, ext = os.path.splitext(out_filename_template)
                     out_filename = base + "_{}".format(branch_id) + ext
-                    
-                    with rasterio.open(out_filename,'w',**buffered_meta) as out:
+
+                    with rasterio.open(out_filename, "w", **buffered_meta) as out:
                         out.write(buffered_array)
-            
+
                 # return files in list
                 return_list += [out]
 
             out.close()
 
         if fileType == "vector":
-            for i,row in generator_to_iterate:
+            for i, row in generator_to_iterate:
                 branch_id = row[self.branch_id_attribute]
-                out = gpd.clip(to_clip,row[self.geom_name],keep_geom_type=True)
+                out = gpd.clip(to_clip, row[self.geom_name], keep_geom_type=True)
                 return_list += [out]
-                
+
                 if (out_filename_template is not None) & (not out.empty):
                     base, ext = os.path.splitext(out_filename_template)
                     out_filename = base + "_{}".format(branch_id) + ext
-                    StreamNetwork.write(out,out_filename)
-        
-        return(return_list)
+                    StreamNetwork.write(out, out_filename)
+
+        return return_list
diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json
index e3ff0420e..629d39860 100644
--- a/unit_tests/derive_level_paths_params.json
+++ b/unit_tests/derive_level_paths_params.json
@@ -3,6 +3,7 @@
 	{
 		"outputDestDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
 		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
+		"wbd": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
 		"buffer_wbd_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered_streams.gpkg",
 		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
 		"branch_id_attribute": "levpa_id",
@@ -15,4 +16,3 @@
 		"verbose": true
 	}	
 }
-
diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py
index 59f423872..ff37ed87d 100644
--- a/unit_tests/derive_level_paths_test.py
+++ b/unit_tests/derive_level_paths_test.py
@@ -9,7 +9,7 @@
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 # # importing python folders in other direcories
-sys.path.append('/foss_fim/src/')
+sys.path.append("/foss_fim/src/")
 import derive_level_paths as src
 import stream_branches
 from utils.fim_enums import FIM_exit_codes as fec
@@ -17,79 +17,93 @@
 
 class test_Derive_level_paths(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-    
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-
-# Test Cases:
+    # Test Cases:
 
     def test_Derive_level_paths_success_all_params(self):
-        '''
+        """
         This test includes all params with many optional parms being set to the default value of the function
-        '''
-      
+        """
+
         params = self.params["valid_data"].copy()
 
         # Notes:
         # Other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__ ,
         # so we skip them here.
         # Returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = src.Derive_level_paths(in_stream_network = params["in_stream_network"],
-                                           buffer_wbd_streams = params["buffer_wbd_streams"],
-                                           out_stream_network = params["out_stream_network"],
-                                           branch_id_attribute = params["branch_id_attribute"],
-                                           out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                           headwaters_outfile = params["headwaters_outfile"],
-                                           catchments = params["catchments"],
-                                           catchments_outfile = params["catchments_outfile"],
-                                           branch_inlets_outfile = params["branch_inlets_outfile"],
-                                           reach_id_attribute = params["reach_id_attribute"],
-                                           verbose = params["verbose"])
+        actual_df = src.Derive_level_paths(
+            in_stream_network=params["in_stream_network"],
+            buffer_wbd_streams=params["buffer_wbd_streams"],
+            wbd=params["wbd"],
+            out_stream_network=params["out_stream_network"],
+            branch_id_attribute=params["branch_id_attribute"],
+            out_stream_network_dissolved=params["out_stream_network_dissolved"],
+            headwaters_outfile=params["headwaters_outfile"],
+            catchments=params["catchments"],
+            catchments_outfile=params["catchments_outfile"],
+            branch_inlets_outfile=params["branch_inlets_outfile"],
+            reach_id_attribute=params["reach_id_attribute"],
+            verbose=params["verbose"],
+        )
 
         # test data type being return is as expected. Downstream code might to know that type
         assert isinstance(actual_df, stream_branches.StreamNetwork)
-        
 
-        #**** NOTE: Based on 05030104         
+        # **** NOTE: Based on 05030104
         # Test row count for dissolved level path GeoDataframe which is returned.
-        actual_row_count = len(actual_df) 
+        actual_row_count = len(actual_df)
         expected_row_count = 4
         assert actual_row_count == expected_row_count
-        
+
         # Test that output files exist as expected
-        assert os.path.exists(params["out_stream_network"]) == True, f"Expected file {params['out_stream_network']} but it does not exist."
-        assert os.path.exists(params["out_stream_network_dissolved"]) == True, f"Expected file {params['out_stream_network_dissolved']} but it does not exist."
-        assert os.path.exists(params["headwaters_outfile"]) == True, f"Expected file {params['headwaters_outfile']} but it does not exist."
-        assert os.path.exists(params["catchments_outfile"]) == True, f"Expected file {params['catchments_outfile']} but it does not exist."
-        assert os.path.exists(params["catchments_outfile"]) == True, f"Expected file {params['catchments_outfile']} but it does not exist."
-        assert os.path.exists(params["branch_inlets_outfile"]) == True, f"Expected file {params['branch_inlets_outfile']} but it does not exist."
-
-    
+        assert (
+            os.path.exists(params["out_stream_network"]) == True
+        ), f"Expected file {params['out_stream_network']} but it does not exist."
+        assert (
+            os.path.exists(params["out_stream_network_dissolved"]) == True
+        ), f"Expected file {params['out_stream_network_dissolved']} but it does not exist."
+        assert (
+            os.path.exists(params["headwaters_outfile"]) == True
+        ), f"Expected file {params['headwaters_outfile']} but it does not exist."
+        assert (
+            os.path.exists(params["catchments_outfile"]) == True
+        ), f"Expected file {params['catchments_outfile']} but it does not exist."
+        assert (
+            os.path.exists(params["catchments_outfile"]) == True
+        ), f"Expected file {params['catchments_outfile']} but it does not exist."
+        assert (
+            os.path.exists(params["branch_inlets_outfile"]) == True
+        ), f"Expected file {params['branch_inlets_outfile']} but it does not exist."
+
     # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
     def test_Derive_level_paths_invalid_input_stream_network(self):
-
         # NOTE: As we are expecting an exception, we use pytest.raises(Exception).
-                
+
         params = self.params["valid_data"].copy()
         params["in_stream_network"] = "some bad path"
-        
-        with pytest.raises(Exception) as e_info:
-            actual = src.Derive_level_paths(in_stream_network = ut_helpers.json_concat(params, "outputDestDir", "in_stream_network"),
-                                            out_stream_network = params["out_stream_network"],
-                                            branch_id_attribute = params["branch_id_attribute"],
-                                            out_stream_network_dissolved = params["out_stream_network_dissolved"],
-                                            huc_id = params["huc_id"],
-                                            headwaters_outfile = params["headwaters_outfile"],
-                                            catchments = params["catchments"],
-                                            catchments_outfile = params["catchments_outfile"],
-                                            branch_inlets_outfile = params["branch_inlets_outfile"],
-                                            reach_id_attribute = params["reach_id_attribute"],
-                                            verbose = params["verbose"] )       
 
+        with pytest.raises(Exception) as e_info:
+            actual = src.Derive_level_paths(
+                in_stream_network=ut_helpers.json_concat(
+                    params, "outputDestDir", "in_stream_network"
+                ),
+                out_stream_network=params["out_stream_network"],
+                branch_id_attribute=params["branch_id_attribute"],
+                out_stream_network_dissolved=params["out_stream_network_dissolved"],
+                huc_id=params["huc_id"],
+                headwaters_outfile=params["headwaters_outfile"],
+                catchments=params["catchments"],
+                catchments_outfile=params["catchments_outfile"],
+                branch_inlets_outfile=params["branch_inlets_outfile"],
+                reach_id_attribute=params["reach_id_attribute"],
+                verbose=params["verbose"],
+            )

From fa0aa48750b9ac19683b971981be0ddc9e95525a Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 1 Sep 2023 11:40:05 -0400
Subject: [PATCH 271/359] v4.3.15.6 Extend inundate_mosaic_wrapper (#972)

---
 config/params_template.env       |  12 +-
 docs/CHANGELOG.md                |  56 +++
 tools/composite_inundation.py    | 485 ++++++++++++----------
 tools/inundate_gms.py            | 319 +++++++++------
 tools/inundate_mosaic_wrapper.py | 300 +++++++++-----
 tools/inundate_nation.py         | 288 +++++++------
 tools/mosaic_inundation.py       | 308 +++++++++-----
 tools/overlapping_inundation.py  | 352 ++++++++--------
 tools/run_test_case.py           | 670 +++++++++++++++++++------------
 tools/tools_shared_variables.py  | 167 +++++---
 10 files changed, 1755 insertions(+), 1202 deletions(-)

diff --git a/config/params_template.env b/config/params_template.env
index e22783f88..03b15760b 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -33,28 +33,28 @@ export levee_id_attribute=SYSTEM_ID
 
 #### apply bathymetry adjustment to rating curve ####
 export bathymetry_adjust=True
-export bathymetry_file="/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg"
+export bathymetry_file="$inputsDir/bathymetry/bathymetry_adjustment_data.gpkg"
 
 #### estimating bankfull stage in SRCs ####
 export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
-export bankfull_flows_file="/data/inputs/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv" # input file location with nwm feature_id and recurrence flow values
+export bankfull_flows_file="$inputsDir/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv" # input file location with nwm feature_id and recurrence flow values
 
 #### applying channel/overbank subdivision routine to SRCs ####
 export src_subdiv_toggle="True" # Toggle to run composite roughness src routine (True=on; False=off)
 export vrough_suffix="" # text to append to output log and hydrotable file names (use for testing/debugging)
-export vmann_input_file="/data/inputs/rating_curve/variable_roughness/mannings_global_06_12.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
+export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
 
 #### apply SRC adjustments using USGS rating curve database (calibration db) ####
 export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; False=off)
-export nwm_recur_file="/data/inputs/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
+export nwm_recur_file="$inputsDir/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
 
 #### apply SRC adjustments using observed FIM/flow point database ####
 export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
-export fim_obs_pnt_data="/data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
+export fim_obs_pnt_data="$inputsDir/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 
 #### apply manual calibration ####
 export manual_calb_toggle="True" # Toggle to run manual calibration routine (True=on; False=off)
-export man_calb_file="/data/inputs/rating_curve/manual_calibration_coefficients.csv" # input file location with HUC, nwm feature_id and manual calibration coefficients
+export man_calb_file="$inputsDir/rating_curve/manual_calibration_coefficients.csv" # input file location with HUC, nwm feature_id and manual calibration coefficients
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 368646a16..957767795 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -42,6 +42,62 @@ This hotfix addresses some bugs introduced in the pandas upgrade.
 
 <br/><br/>
 
+## v4.3.15.6 - 2023-09-01 - [PR#972](https://github.com/NOAA-OWP/inundation-mapping/pull/972)
+
+Adds functionality to `tools/inundate_mosaic_wrapper.py` and incorporates functionality into existing `inundation-mapping` scripts.
+
+### Changes
+
+- `tools/`
+    - `inundate_mosaic_wrapper.py`: Refactors to call `Inundate_gms` only once; adds functionality to produce a mosaicked polygon from `depths_raster` without needing to generate the `inundation_raster`; removes `log_file` and `output_fileNames` as variables and input arguments; updates the help description for `keep_intermediate`.
+    - `composite_inundation.py`, 'inundate_nation.py`, and `run_test_case.py`: Implements `produce_mosaicked_inundation()` from `tools/inundate_mosaic_wrapper.py`.
+    - `inundate_gms.py`: Adds back `Inundate_gms(**vars(parser.parse_args()))` command-line function call.
+    - `mosaic_inundation.py` and `overlapping_inundation.py`: Removes unused import(s).
+    - `tools_shared_variables.py`: Changes hardcoded `INPUT_DIR` to environment variable.
+
+<br/><br/>
+
+## v4.3.15.5 - 2023-08-17 - [PR#970](https://github.com/NOAA-OWP/inundation-mapping/pull/970)
+
+Fixes an issue where the stream network was clipped inside the DEM resulting in a burned stream channel that was then filled by the DEM depression filling process so that all pixels in the burned channel had the same elevation which was the elevation at the spill point (which wasn't necessarily at the HUC outlet). The stream network is now extended from the WBD to the buffered WBD and all streams except the outlet are clipped to the streams buffer inside the WBD (WBD - (3 x cell_size)). This also prevents reverse flow issues.
+
+### Changes
+
+- `src/`
+    - `clip_vectors_to_wbd.py`: Clip NWM streams to buffered WBD and clip non-outlet streams to WBD streams buffer (WBD - (3 x cell_size)).
+    - `derive_level_paths.py`: Add WBD input argument
+    - `run_unit_wb.py`: Add WBD input argument
+    - `src_stream_branches.py`: Ignore branches outside HUC
+- `unit_tests/`
+    - `derive_level_paths_params.json`: Add WBD parameter value
+    - `derive_level_paths_test.py`: Add WBD parameter
+ 
+<br/><br/>
+
+## v4.3.15.4 - 2023-08-28 - [PR#977](https://github.com/NOAA-OWP/inundation-mapping/pull/977)
+
+Fixes incorrect `nodata` value in `src/burn_in_levees.py` that was responsible for missing branches (Exit code: 61). Also cleans up related files.
+
+### Changes
+
+- `src/`
+    - `buffer_stream_branches.py`: Moves script functionality into a function.
+    - `burn_in_levees.py`: Corrects `nodata` value. Adds context managers for reading rasters.
+    - `generate_branch_list.py`: Removes unused imports.
+    - `mask_dem.py`: Removes commented code.
+
+<br/><br/>
+
+## v4.3.15.3 - 2023-09-01 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/983)
+
+This hotfix addresses some bugs introduced in the pandas upgrade.
+
+### Changes  
+
+- `/tools/eval_plots_stackedbar.py`: 2 lines were changed to work with the pandas upgrade. Added an argument for a `groupby` median call and fixed a bug with the pandas `query`. Also updated with Black compliance.
+
+<br/><br/>
+
 ## v4.3.15.2 - 2023-07-18 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/948)
 
 Adds a script to produce inundation maps (extent TIFs, polygons, and depth grids) given a flow file and hydrofabric outputs. This is meant to make it easier to team members and external collaborators to produce inundation maps.
diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py
index d5784594b..915b96e3e 100644
--- a/tools/composite_inundation.py
+++ b/tools/composite_inundation.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-import os, argparse, copy, sys
+import os, argparse, copy
 import json
 import rasterio
 import numpy as np
@@ -7,49 +7,49 @@
 
 from datetime import datetime
 from multiprocessing import Pool
-#from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
+
+# from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
 import concurrent.futures as cf
 from tqdm import tqdm
 
 from inundation import inundate
 from mosaic_inundation import Mosaic_inundation
-from inundate_gms import Inundate_gms
+from inundate_mosaic_wrapper import produce_mosaicked_inundation
 from utils.shared_functions import FIM_Helpers as fh
 from utils.shared_variables import elev_raster_ndv
 
-########################################################
-'''
-Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-   a significant review and/or upgrade.
-'''
-
-########################################################
-
 
 class InundateModel_HUC(object):
-
-    def __init__(self, source_directory, huc):
-
+    def __init__(self, model, source_directory, huc):
+        self.model = model
         self.source_directory = source_directory
-        self.huc = huc        
-
-    def inundate_huc(self, flows_file, composite_output_dir, output_name, log_file_path,
-                     num_workers_branches, no_cleanup, verbose):
-        '''
+        self.huc = huc
+
+    def inundate_huc(
+        self,
+        flows_file,
+        composite_output_dir,
+        output_name,
+        log_file_path,
+        num_workers_branches,
+        no_cleanup,
+        verbose,
+    ):
+        """
         Processing:
-            Will inundate a single huc directory and if gms, will create an aggregate mosiac per huc
+            Will inundate a single huc directory and if gms, will create an aggregate mosaic per huc
 
         Returns:
             The map file of the inundated raster.
-        '''
+        """
 
         source_huc_dir = os.path.join(self.source_directory, self.huc)
 
         # hucs do not need to exist as the list might have one huc from one
         # source directory that is not in another. TODO: how to log this.
-        if (not os.path.exists(source_huc_dir)): 
+        if not os.path.exists(source_huc_dir):
             print(f"HUC {self.huc} does not exist in {self.source_directory}")
-            return None 
+            return None
 
         output_huc_dir = os.path.join(composite_output_dir, self.huc)
         # Create output directory if it does not exist
@@ -59,45 +59,46 @@ def inundate_huc(self, flows_file, composite_output_dir, output_name, log_file_p
         inundation_map_file = None
 
         output_raster_name = os.path.join(output_huc_dir, output_name)
-        output_raster_name = fh.append_id_to_file_name(output_raster_name, [self.huc,self.model])
-
-        # adjust to add model and huc number
-        log_file = None
-        inundation_list_file = None        
-        if (log_file_path != None):
-            log_file = os.path.join(log_file_path, f"{self.huc}_error_logs.txt") 
-            inundation_list_file = os.path.join(log_file_path, 
-                                               f"{self.huc}_inundation_file_list.csv") 
+        output_raster_name = fh.append_id_to_file_name(output_raster_name, [self.huc, self.model])
 
-        if (verbose):
-            print(f'... Creating an inundation map for the FIM4'\
-                    f' configuration for HUC {self.huc}...')
+        if verbose:
+            print(f"... Creating an inundation map for the FIM4" f" configuration for HUC {self.huc}...")
 
         if self.model in ["fr", "ms"]:
-
-            rem = os.path.join(source_huc_dir, 'rem_zeroed_masked.tif')
-            catchments = os.path.join(source_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
-            catchment_poly = os.path.join(source_huc_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-            hydro_table = os.path.join(source_huc_dir, 'hydroTable.csv')
+            if self.model == "ms":
+                extent_friendly = "mainstem (MS)"
+            elif self.model == "fr":
+                extent_friendly = "full-resolution (FR)"
+            rem = os.path.join(source_huc_dir, "rem_zeroed_masked.tif")
+            catchments = os.path.join(source_huc_dir, "gw_catchments_reaches_filtered_addedAttributes.tif")
+            catchment_poly = os.path.join(
+                source_huc_dir,
+                "gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg",
+            )
+            hydro_table = os.path.join(source_huc_dir, "hydroTable.csv")
 
             # Ensure that all of the required files exist in the huc directory
             for file in (rem, catchments, catchment_poly, hydro_table):
                 if not os.path.exists(file):
-                    raise Exception(f"The following file does not exist within the supplied FIM directory:\n{file}")
+                    raise Exception(
+                        f"The following file does not exist within the supplied FIM directory:\n{file}"
+                    )
 
             # Run inundation()
             # Must set workers to one as we are only processing one huc.
-            map_file = inundate(rem, 
-                                catchments, 
-                                catchment_poly, 
-                                hydro_table,
-                                flows_file, 
-                                mask_type = None, 
-                                num_workers = 1,
-                                inundation_raster = output_raster_name,
-                                quiet = not verbose)
-
-            #if verbose:
+            map_file = inundate(
+                rem,
+                catchments,
+                catchment_poly,
+                hydro_table,
+                flows_file,
+                mask_type=None,
+                num_workers=1,
+                inundation_raster=output_raster_name,
+                quiet=not verbose,
+            )
+
+            # if verbose:
             #    print("Inundation Response:")
             #    print(map_file)
 
@@ -105,71 +106,61 @@ def inundate_huc(self, flows_file, composite_output_dir, output_name, log_file_p
                 raise Exception(f"Failed to inundate {extent_friendly} using the provided flows.")
 
             mosaic_file_path = map_file[0][0]
-            inundation_map_file = [self.model, self.huc, mosaic_file_path]            
+            inundation_map_file = [self.model, self.huc, mosaic_file_path]
 
         else:  # gms
-            # we are doing each huc one at a time
-            map_file = Inundate_gms( hydrofabric_dir = self.source_directory,
-                                    forecast = flows_file, 
-                                    num_workers = num_workers_branches,
-                                    hucs = self.huc,
-                                    inundation_raster = output_raster_name,
-                                    verbose = verbose,
-                                    log_file = log_file,
-                                    output_fileNames = inundation_list_file )
-            
-            mask_path_gms = os.path.join(source_huc_dir, 'wbd.gpkg')
-
-            # we are going to mosaic the gms files per huc
-            # NOTE: Leave workers as 1, it fails to composite correctly if more than one.
-            mosaic_file_path = Mosaic_inundation( map_file, 
-                                                  mosaic_attribute = 'inundation_rasters',
-                                                  mosaic_output = output_raster_name,
-                                                  mask = mask_path_gms, 
-                                                  unit_attribute_name = 'huc8',
-                                                  nodata = elev_raster_ndv,
-                                                  workers = 1,
-                                                  remove_inputs = not no_cleanup,
-                                                  subset = None,
-                                                  verbose = verbose )
+            mosaic_file_path = produce_mosaicked_inundation(
+                self.source_directory,
+                [self.huc],
+                flows_file,
+                inundation_raster=output_raster_name,
+                num_workers=num_workers_branches,
+                remove_intermediate=not no_cleanup,
+                verbose=verbose,
+            )
 
             inundation_map_file = [self.model, self.huc, mosaic_file_path]
-            
-            if verbose: print(f"Inundation for HUC {self.huc} is complete")
+
+            if verbose:
+                print(f"Inundation for HUC {self.huc} is complete")
 
         return inundation_map_file
 
-class Composite_HUC(object):
 
+class Composite_HUC(object):
     # Composites two source directories for a single huc
     # Note: The huc does not need to exist in both source directories
     @classmethod
     def composite_huc(self, args):
-
         huc = args["current_huc"]
-        print(f"Processing huc {huc}")        
+        print(f"Processing huc {huc}")
         composite_model_map_files = []
         for model in args["models"]:
-
             # setup original fim processed directory
-            if model == "ms" : source_dir = args["fim_dir_ms"]
-            elif model == "fr" : source_dir = args["fim_dir_fr"]
-            else: source_dir = args["gms_dir"]
-            
+            if model == "ms":
+                source_dir = args["fim_dir_ms"]
+            elif model == "fr":
+                source_dir = args["fim_dir_fr"]
+            else:
+                source_dir = args["gms_dir"]
+
             ci = InundateModel_HUC(model, source_dir, huc)
-            map_file = ci.inundate_huc(args["flows_file"],
-                                        args["composite_output_dir"], 
-                                        args["output_name"],
-                                        args["log_file_path"],
-                                        args["num_workers_branches"],
-                                        args["no_cleanup"],
-                                        args["verbose"])
+            map_file = ci.inundate_huc(
+                args["flows_file"],
+                args["composite_output_dir"],
+                args["output_name"],
+                args["log_file_path"],
+                args["num_workers_branches"],
+                args["no_cleanup"],
+                args["verbose"],
+            )
             if map_file is not None:
                 composite_model_map_files.append(map_file)
 
         # Composite the two final model outputs
-        inundation_map_file_df = pd.DataFrame(composite_model_map_files,
-                                              columns = ['model', 'huc8', 'inundation_rasters'])
+        inundation_map_file_df = pd.DataFrame(
+            composite_model_map_files, columns=["model", "huc8", "inundation_rasters"]
+        )
 
         if args["verbose"]:
             print("inundation_map_file_df")
@@ -181,35 +172,35 @@ def composite_huc(self, args):
         # NOTE: Leave workers as 1, it fails to composite correctly if more than one.
         #    - Also. by adding the is_mosaic_for_gms_branches = False, Mosaic_inudation
         #      will not auto add the HUC into the output name (its default behaviour)
-        mosaic_file_path = Mosaic_inundation( inundation_map_file_df,
-                                            mosaic_attribute='inundation_rasters',
-                                            mosaic_output = composite_file_output,
-                                            mask = None,
-                                            unit_attribute_name = 'huc8',
-                                            nodata = elev_raster_ndv,
-                                            workers = 1,
-                                            remove_inputs = not args["no_cleanup"],
-                                            subset = None,
-                                            verbose = args["verbose"] )
+        Mosaic_inundation(
+            inundation_map_file_df,
+            mosaic_attribute="inundation_rasters",
+            mosaic_output=composite_file_output,
+            mask=None,
+            unit_attribute_name="huc8",
+            nodata=elev_raster_ndv,
+            workers=1,
+            remove_inputs=not args["no_cleanup"],
+            subset=None,
+            verbose=args["verbose"],
+        )
 
         if args["is_bin_raster"]:
-            if args["verbose"]: print("Converting to binary")
+            if args["verbose"]:
+                print("Converting to binary")
             CompositeInundation.hydroid_to_binary(composite_file_output)
 
 
 class CompositeInundation(object):
-
     @classmethod
     def run_composite(self, args):
-
         def __validate_args(args):
-
             """
             Runs `inundate()` on any two of the following:
                 1) FIM 3.X mainstem (MS)
                 2) FIM 3.X full-resolution (FR)
                 3) FIM 4.x (gms)
-                All three need to have outputs and composites results. Assumes that all products 
+                All three need to have outputs and composites results. Assumes that all products
             necessary for `inundate()` are in each huc8 folder.
 
             Parameters
@@ -223,7 +214,7 @@ def __validate_args(args):
             huc: str, optional
                 If this value comes in, it shoudl be a single huc value. If it does not exist,
                 we use all hucs in the give source directories.
-            flows_file : str : 
+            flows_file : str :
                 Can be a single file path to a forecast csv or a comma-separated list of files.
             composite_output_dir : str
                 Folder path to write outputs. It will be created if it does not exist.
@@ -240,38 +231,38 @@ def __validate_args(args):
             verbose : bool, optional
                 show extra output.
             """
-            
+
             if (args["fim_dir_ms"]) and (args["fim_dir_ms"].lower() == "none"):
                 args["fim_dir_ms"] = None
             if (args["fim_dir_fr"]) and (args["fim_dir_fr"].lower() == "none"):
-                args["fim_dir_fr"] = None        
+                args["fim_dir_fr"] = None
             if (args["gms_dir"]) and (args["gms_dir"].lower() == "none"):
                 args["gms_dir"] = None
-            
+
             # count number of input dir types and ensure their are no duplciates.
-            dir_list_lowercase = [] # we use a forced lowercase to help ensure dups (might be mixed case)
+            dir_list_lowercase = []  # we use a forced lowercase to help ensure dups (might be mixed case)
             dir_list_raw = []
-            missing_dir_msg = "{} directory of {} does not exist"    
+            missing_dir_msg = "{} directory of {} does not exist"
             args["models"] = []
-            if (args["fim_dir_ms"] != None):
+            if args["fim_dir_ms"] is not None:
                 args["models"].append("ms")
                 assert os.path.isdir(args["fim_dir_ms"]), missing_dir_msg.format("ms", args["fim_dir_ms"])
                 dir_list_raw.append(args["fim_dir_ms"])
                 dir_list_lowercase.append(args["fim_dir_ms"].lower())
 
-            if (args["fim_dir_fr"] != None):
+            if args["fim_dir_fr"] is not None:
                 args["models"].append("fr")
                 assert os.path.isdir(args["fim_dir_fr"]), missing_dir_msg.format("fr", args["fim_dir_fr"])
                 dir_list_raw.append(args["fim_dir_fr"])
-                dir_list_lowercase.append(args["fim_dir_fr"].lower())        
+                dir_list_lowercase.append(args["fim_dir_fr"].lower())
 
-            if (args["gms_dir"] != None):
+            if args["gms_dir"] is not None:
                 args["models"].append("gms")
                 assert os.path.isdir(args["gms_dir"]), missing_dir_msg.format("gms", args["gms_dir"])
                 dir_list_raw.append(args["gms_dir"])
-                dir_list_lowercase.append(args["gms_dir"].lower())                
+                dir_list_lowercase.append(args["gms_dir"].lower())
 
-            if (len(args["models"]) != 2):
+            if not len(args["models"]) != 2 or (len(args["models"]) == 1 and "gms" not in args["models"]):
                 raise ValueError("Must submit exactly two directories (ms, fr and/or gms")
 
             # check for duplicate dir names (submitting same dir for two args)
@@ -288,172 +279,233 @@ def __validate_args(args):
             total_cpus_requested = args["num_workers_huc"] * args["num_workers_branches"]
             total_cpus_available = os.cpu_count()
             if total_cpus_requested > (total_cpus_available - 1):
-                raise ValueError('The HUC job num of workers, {}, multiplied by the branch workers number, {}, '\
-                                'exceeds your machine\'s available CPU count of {} minus one. '\
-                                'Please lower the num_workers_huc or num_workers_branches'\
-                                'values accordingly.'.format(args["num_workers_huc"], 
-                                total_cpus_available, args["num_workers_branches"]) )
-            
+                raise ValueError(
+                    "The HUC job num of workers, {}, multiplied by the branch workers number, {}, "
+                    "exceeds your machine's available CPU count of {} minus one. "
+                    "Please lower the num_workers_huc or num_workers_branches"
+                    "values accordingly.".format(
+                        args["num_workers_huc"],
+                        total_cpus_available,
+                        args["num_workers_branches"],
+                    )
+                )
+
             # Create output directory if it does not exist
             if not os.path.isdir(args["composite_output_dir"]):
                 os.mkdir(args["composite_output_dir"])
 
-            # If no output name supplied, create one using the flows file name 
+            # If no output name supplied, create one using the flows file name
             # the output_name is the final composite output file.
             # we also extract the basic file name without extension for use as the log file name
             if not args["output_name"]:
                 flows_root = os.path.splitext(os.path.basename(args["flows_file"]))
-                root_output_file_name = f'inundation_composite_{flows_root[0]}'
+                root_output_file_name = f"inundation_composite_{flows_root[0]}"
                 args["output_name"] = f"{root_output_file_name}.tif"
             else:
                 # see if the file name has a path or not, fail if it does
                 output_file_name_split = os.path.split(args["output_name"])
                 if args["output_name"] == output_file_name_split[0]:
-                    raise ValueError("""If submitting the -n (output file name), please ensure 
-                                        it has no pathing. You can also leave it blank if you like.""")
+                    raise ValueError(
+                        """If submitting the -n (output file name), please ensure 
+                                        it has no pathing. You can also leave it blank if you like."""
+                    )
 
                 root_output_file_name = os.path.splitext(args["output_name"])[0]
 
             # setup log file and its directory
             # Note: Log files are only created at this time if verbose\
             args["log_file_path"] = None
-            if (args["verbose"]):
+            if args["verbose"]:
                 args["log_file_path"] = os.path.join(args["composite_output_dir"], "mosaic_logs")
                 if not os.path.isdir(args["log_file_path"]):
                     os.mkdir(args["log_file_path"])
 
             # Save run parameters up to this point
-            args_file = os.path.join(args["composite_output_dir"], root_output_file_name + '_args.json')
-            with open(args_file, 'w') as json_file:
+            args_file = os.path.join(args["composite_output_dir"], root_output_file_name + "_args.json")
+            with open(args_file, "w") as json_file:
                 json.dump(args, json_file)
                 print(f"Args printed to file at {args_file}")
 
             # make combined huc list, NOTE: not all source dirs will have the same huc folders
             huc_list = set()
-            if (args["huc"] != None):
+            if args["huc"] != None:
                 if (len(args["huc"]) != 8) or (not args["huc"].isnumeric()):
                     raise ValueError("Single huc value (-u arg) was submitted but appears invalid")
                 else:
                     huc_list.add(args["huc"])
             else:
                 for dir in dir_list_raw:
-                    sub_dirs = [ item for item in os.listdir(dir) if os.path.isdir(os.path.join(dir, item)) ]
-                    # Some directories may not be hucs (log folders, etc)            
-                    huc_folders = [ item for item in sub_dirs if item.isnumeric() ]
+                    sub_dirs = [item for item in os.listdir(dir) if os.path.isdir(os.path.join(dir, item))]
+                    # Some directories may not be hucs (log folders, etc)
+                    huc_folders = [item for item in sub_dirs if item.isnumeric()]
                     huc_set = set(huc_folders)
-                    huc_list.update(huc_set) # will ensure no dups
+                    huc_list.update(huc_set)  # will ensure no dups
 
             args["huc_list"] = huc_list
 
             return args
 
-        args =  __validate_args(args)
+        args = __validate_args(args)
 
-        huc_list = args["huc_list"] 
+        huc_list = args["huc_list"]
         number_huc_workers = args["num_workers_huc"]
-        #if len(huc_list == 1): # skip iterator
-        if (number_huc_workers == 1):
+        # if len(huc_list == 1): # skip iterator
+        if number_huc_workers == 1:
             for huc in sorted(huc_list):
-                Composite_HUC.composite_huc(huc, args)
+                args["current_huc"] = huc
+                Composite_HUC.composite_huc(args)
         else:
-
             print(f"Processing {len(huc_list)} hucs")
             args_list = []
-            #sorted_hucs = sorted(huc_list)
-            #params_items = [(args, huc) for huc in huc_list]
+            # sorted_hucs = sorted(huc_list)
+            # params_items = [(args, huc) for huc in huc_list]
             for huc in sorted(huc_list):
                 huc_args = copy.deepcopy(args)
                 huc_args["current_huc"] = huc
                 args_list.append(huc_args)
 
-            with cf.ProcessPoolExecutor(max_workers = number_huc_workers) as executor:
-                executor_gen = { executor.submit(Composite_HUC.composite_huc, params): params for params in args_list }
-
-                for future in tqdm(cf.as_completed(executor_gen),
-                                total = len(executor_gen),
-                                desc = f"Running composite inundation with {number_huc_workers} workers" ):
+            with cf.ProcessPoolExecutor(max_workers=number_huc_workers) as executor:
+                executor_gen = {
+                    executor.submit(Composite_HUC.composite_huc, params): params for params in args_list
+                }
 
+                for future in tqdm(
+                    cf.as_completed(executor_gen),
+                    total=len(executor_gen),
+                    desc=f"Running composite inundation with {number_huc_workers} workers",
+                ):
                     executor_gen[future]
 
                 try:
                     future.result()
                 except Exception as exc:
-                    print('{}, {}, {}'.format(hucCode,exc.__class__.__name__,exc))
-
+                    print("{}, {}, {}".format(huc, exc.__class__.__name__, exc))
 
-            print("All hucs have been processed")            
+            print("All hucs have been processed")
 
     @staticmethod
     def hydroid_to_binary(hydroid_raster_filename):
-        #Converts hydroid positive/negative grid to 1/0
-        #to_bin = lambda x: np.where(x > 0, 1, np.where(x == 0, -9999, 0))
+        # Converts hydroid positive/negative grid to 1/0
+        # to_bin = lambda x: np.where(x > 0, 1, np.where(x == 0, -9999, 0))
         to_bin = lambda x: np.where(x > 0, 1, np.where(x != -9999, 0, -9999))
         hydroid_raster = rasterio.open(hydroid_raster_filename)
-        profile = hydroid_raster.profile # get profile for new raster creation later on
-        profile['nodata'] = -9999
-        bin_raster = to_bin(hydroid_raster.read(1)) # converts neg/pos to 0/1
+        profile = hydroid_raster.profile  # get profile for new raster creation later on
+        profile["nodata"] = -9999
+        bin_raster = to_bin(hydroid_raster.read(1))  # converts neg/pos to 0/1
         # Overwrite inundation raster
         with rasterio.open(hydroid_raster_filename, "w", **profile) as out_raster:
-            out_raster.write(bin_raster.astype(hydroid_raster.profile['dtype']), 1)
-        del hydroid_raster,profile,bin_raster
-    
+            out_raster.write(bin_raster.astype(hydroid_raster.profile["dtype"]), 1)
+        del hydroid_raster, profile, bin_raster
 
 
-if __name__ == '__main__':
-    
-    '''
-        Runs inundate and compositing on any and exactly two of the following:
-            1) FIM 3.X mainstem (MS)
-            2) FIM 3.X full-resolution (FR)
-            3) FIM 4.x (gms)
+if __name__ == "__main__":
+    """
+    Runs inundate and compositing on any and exactly two of the following:
+        1) FIM 3.X mainstem (MS)
+        2) FIM 3.X full-resolution (FR)
+        3) FIM 4.x (gms)
 
-        Examples of usage
-        Notice: arg keys and values for some of the variants
-        --------
-        a) ms and fr (single huc)
-        python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -fr /outputs/inundation_test_1_FIM3_fr -u /data/inputs/huc_lists/include_huc8.lst -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
+    Examples of usage
+    Notice: arg keys and values for some of the variants
+    --------
+    a) ms and fr (single huc)
+    python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -fr /outputs/inundation_test_1_FIM3_fr -u /data/inputs/huc_lists/include_huc8.lst -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
 
-        a) ms and gms (all hucs in each folder)
-        python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -gms /outputs/inundation_test_1_gms -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -c -jh 3 -jb 20
+    a) ms and gms (all hucs in each folder)
+    python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -gms /outputs/inundation_test_1_gms -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -c -jh 3 -jb 20
 
-        b) fr and gms (single huc)
-        python3 /foss_fim/tools/composite_inundation.py -fr /outputs/inundation_test_1_FIM3_fr -gms /outputs/inundation_test_1_gms -u 13090001 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
-    '''
+    b) fr and gms (single huc)
+    python3 /foss_fim/tools/composite_inundation.py -fr /outputs/inundation_test_1_FIM3_fr -gms /outputs/inundation_test_1_gms -u 13090001 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
+    """
 
     # parse arguments
-    parser = argparse.ArgumentParser(description="""Inundate FIM 3 full resolution
-                and mainstem outputs using a flow file and composite the results.""")
-    parser.add_argument('-ms','--fim-dir-ms', help='Source directory that contains MS FIM outputs.',
-                        required=False, default=None)
-    parser.add_argument('-fr','--fim-dir-fr', help='Source directory that contains FR FIM outputs.', 
-                        required=False, default=None)
-    parser.add_argument('-gms','--gms-dir', help='Source directory that contains FIM4 GMS outputs.', 
-                        required=False, default=None)
-    parser.add_argument('-u','--huc', 
-                        help="""(Optional) If a single HUC is provided, only that HUC will be processed.
+    parser = argparse.ArgumentParser(
+        description="""Inundate FIM 3 full resolution
+                and mainstem outputs using a flow file and composite the results."""
+    )
+    parser.add_argument(
+        "-ms",
+        "--fim-dir-ms",
+        help="Source directory that contains MS FIM outputs.",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-fr",
+        "--fim-dir-fr",
+        help="Source directory that contains FR FIM outputs.",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-gms",
+        "--gms-dir",
+        help="Source directory that contains FIM4 GMS outputs.",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-u",
+        "--huc",
+        help="""(Optional) If a single HUC is provided, only that HUC will be processed.
                         If not submitted, all HUCs in the source directories will be used.""",
-                        required=False, default=None)
-    parser.add_argument('-f','--flows-file', 
-                        help='File path of flows csv.',
-                        required=True)
-    parser.add_argument('-o','--composite-output-dir', help='Folder to write Composite Raster output.',
-                        required=True)
-    parser.add_argument('-n','--output-name', help='File name for output(s).', 
-                        default=None, required=False)
-    parser.add_argument('-b','--is-bin-raster', 
-                        help="If flag is included, the output raster will be changed to wet/dry.", 
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-jh', '--num-workers-huc',
-                        help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',
-                        required=False, default=1, type=int)
-    parser.add_argument('-jb', '--num-workers-branches',
-                        help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.',
-                        required=False, default=1, type=int)
-    parser.add_argument('-c','--no_cleanup', 
-                        help='If flag used, intermediate rasters are NOT cleaned up.',
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-v','--verbose', help='Show additional outputs.',
-                        required=False, default=False, action='store_true')
+        required=False,
+        default=None,
+    )
+    parser.add_argument("-f", "--flows-file", help="File path of flows csv.", required=True)
+    parser.add_argument(
+        "-o",
+        "--composite-output-dir",
+        help="Folder to write Composite Raster output.",
+        required=True,
+    )
+    parser.add_argument(
+        "-n",
+        "--output-name",
+        help="File name for output(s).",
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        "-b",
+        "--is-bin-raster",
+        help="If flag is included, the output raster will be changed to wet/dry.",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "-jh",
+        "--num-workers-huc",
+        help="Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.",
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        "-jb",
+        "--num-workers-branches",
+        help="Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.",
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        "-c",
+        "--no_cleanup",
+        help="If flag used, intermediate rasters are NOT cleaned up.",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Show additional outputs.",
+        required=False,
+        default=False,
+        action="store_true",
+    )
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
@@ -461,7 +513,7 @@ def hydroid_to_binary(hydroid_raster_filename):
     start_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
     print("================================")
-    print(f"Start composite inundation - {dt_string}")    
+    print(f"Start composite inundation - {dt_string}")
     print()
 
     ci = CompositeInundation()
@@ -476,4 +528,3 @@ def hydroid_to_binary(hydroid_raster_filename):
     time_duration = end_time - start_time
     print(f"Duration: {str(time_duration).split('.')[0]}")
     print()
-
diff --git a/tools/inundate_gms.py b/tools/inundate_gms.py
index 48f7aa5b9..6cf939667 100755
--- a/tools/inundate_gms.py
+++ b/tools/inundate_gms.py
@@ -2,6 +2,7 @@
 
 import os
 import argparse
+
 # import logging
 
 import pandas as pd
@@ -12,21 +13,26 @@
 from inundation import hydroTableHasOnlyLakes, NoForecastFound
 from utils.shared_functions import FIM_Helpers as fh
 
-def Inundate_gms( hydrofabric_dir, forecast, num_workers=1,
-                  hucs=None,
-                  inundation_raster=None,
-                  inundation_polygon=None, depths_raster=None,
-                  verbose=False,
-                  log_file=None,
-                  output_fileNames=None ):
 
+def Inundate_gms(
+    hydrofabric_dir,
+    forecast,
+    num_workers=1,
+    hucs=None,
+    inundation_raster=None,
+    inundation_polygon=None,
+    depths_raster=None,
+    verbose=False,
+    log_file=None,
+    output_fileNames=None,
+):
     # input handling
     if hucs is not None:
         try:
             _ = (i for i in hucs)
         except TypeError:
             raise ValueError("hucs argument must be an iterable")
-    
+
     if isinstance(hucs, str):
         hucs = [hucs]
 
@@ -37,17 +43,19 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers=1,
         if os.path.exists(log_file):
             os.remove(log_file)
 
-        if verbose :
-            print('HUC8,BranchID,Exception', file=open(log_file, 'w'))
-    #if log_file:
-        #logging.basicConfig(filename=log_file, level=logging.INFO)
-        #logging.info('HUC8,BranchID,Exception')
+        if verbose:
+            print("HUC8,BranchID,Exception", file=open(log_file, "w"))
+    # if log_file:
+    # logging.basicConfig(filename=log_file, level=logging.INFO)
+    # logging.info('HUC8,BranchID,Exception')
 
     # load fim inputs
-    hucs_branches = pd.read_csv( os.path.join(hydrofabric_dir, 'fim_inputs.csv'),
-                                 header=None,
-                                 dtype= {0:str, 1:str} )
-    
+    hucs_branches = pd.read_csv(
+        os.path.join(hydrofabric_dir, "fim_inputs.csv"),
+        header=None,
+        dtype={0: str, 1: str},
+    )
+
     if hucs is not None:
         hucs = set(hucs)
         huc_indices = hucs_branches.loc[:, 0].isin(hucs)
@@ -55,15 +63,17 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers=1,
 
     # get number of branches
     number_of_branches = len(hucs_branches)
-    
+
     # make inundate generator
-    inundate_input_generator = __inundate_gms_generator( hucs_branches,
-                                                         hydrofabric_dir,
-                                                         inundation_raster,
-                                                         inundation_polygon,
-                                                         depths_raster,
-                                                         forecast,
-                                                         verbose=False )
+    inundate_input_generator = __inundate_gms_generator(
+        hucs_branches,
+        hydrofabric_dir,
+        inundation_raster,
+        inundation_polygon,
+        depths_raster,
+        forecast,
+        verbose=False,
+    )
 
     # start up process pool
     # better results with Process pool
@@ -75,48 +85,52 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers=1,
     depths_raster_fileNames = [None] * number_of_branches
     hucCodes = [None] * number_of_branches
     branch_ids = [None] * number_of_branches
-       
-    executor_generator = { 
-                executor.submit(inundate, **inp) : ids for inp, ids in inundate_input_generator 
-                }          
+
+    executor_generator = {executor.submit(inundate, **inp): ids for inp, ids in inundate_input_generator}
     idx = 0
-    for future in tqdm(as_completed(executor_generator),
-                       total=len(executor_generator),
-                       desc=f"Inundating branches with {num_workers} workers",
-                       disable=(not verbose) ):
-        
+    for future in tqdm(
+        as_completed(executor_generator),
+        total=len(executor_generator),
+        desc=f"Inundating branches with {num_workers} workers",
+        disable=(not verbose),
+    ):
         hucCode, branch_id = executor_generator[future]
 
         try:
             future.result()
-        
+
         except NoForecastFound as exc:
             if log_file is not None:
-                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
-                      file=open(log_file, 'a'))
+                print(
+                    f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}",
+                    file=open(log_file, "a"),
+                )
             elif verbose:
-                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
+                print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}")
 
         except hydroTableHasOnlyLakes as exc:
             if log_file is not None:
-                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
-                      file=open(log_file, 'a'))
+                print(
+                    f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}",
+                    file=open(log_file, "a"),
+                )
             elif verbose:
-                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
-        
+                print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}")
+
         except Exception as exc:
             if log_file is not None:
-                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}',
-                      file=open(log_file, 'a'))
+                print(
+                    f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}",
+                    file=open(log_file, "a"),
+                )
             else:
-                print(f'{hucCode},{branch_id},{exc.__class__.__name__}, {exc}')
+                print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}")
         else:
-            
             hucCodes[idx] = hucCode
             branch_ids[idx] = branch_id
 
             try:
-                #print(hucCode,branch_id,future.result()[0][0])                
+                # print(hucCode,branch_id,future.result()[0][0])
                 inundation_raster_fileNames[idx] = future.result()[0][0]
             except TypeError:
                 pass
@@ -131,66 +145,85 @@ def Inundate_gms( hydrofabric_dir, forecast, num_workers=1,
             except TypeError:
                 pass
 
-            idx += 1 
-    
+            idx += 1
+
     # power down pool
     executor.shutdown(wait=True)
 
     # make filename dataframe
-    output_fileNames_df = pd.DataFrame( { 'huc8': hucCodes,
-                                          'branchID': branch_ids,
-                                          'inundation_rasters': inundation_raster_fileNames,
-                                          'depths_rasters': depths_raster_fileNames,
-                                          'inundation_polygons': inundation_polygon_fileNames } )
+    output_fileNames_df = pd.DataFrame(
+        {
+            "huc8": hucCodes,
+            "branchID": branch_ids,
+            "inundation_rasters": inundation_raster_fileNames,
+            "depths_rasters": depths_raster_fileNames,
+            "inundation_polygons": inundation_polygon_fileNames,
+        }
+    )
 
     if output_fileNames is not None:
         output_fileNames_df.to_csv(output_fileNames, index=False)
 
-    return(output_fileNames_df)
-
+    return output_fileNames_df
 
-def __inundate_gms_generator( hucs_branches,
-                              hydrofabric_dir,
-                              inundation_raster,
-                              inundation_polygon,
-                              depths_raster,
-                              forecast, verbose=False ):
 
+def __inundate_gms_generator(
+    hucs_branches,
+    hydrofabric_dir,
+    inundation_raster,
+    inundation_polygon,
+    depths_raster,
+    forecast,
+    verbose=False,
+):
     # Iterate over branches
     for idx, row in hucs_branches.iterrows():
-        
         huc = str(row[0])
         branch_id = str(row[1])
 
         huc_dir = os.path.join(hydrofabric_dir, huc)
-        branch_dir = os.path.join(huc_dir, 'branches', branch_id)
+        branch_dir = os.path.join(huc_dir, "branches", branch_id)
+
+        rem_file_name = f"rem_zeroed_masked_{branch_id}.tif"
+        rem_branch = os.path.join(branch_dir, rem_file_name)
 
-        rem_file_name = f'rem_zeroed_masked_{branch_id}.tif'
-        rem_branch = os.path.join( branch_dir, rem_file_name )
+        catchments_file_name = f"gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif"
+        catchments_branch = os.path.join(branch_dir, catchments_file_name)
 
-        catchments_file_name = f'gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif'
-        catchments_branch = os.path.join( branch_dir, catchments_file_name )
-        
         # FIM versions > 4.3.5 use an aggregated hydrotable file rather than individual branch hydrotables
-        hydroTable_huc = os.path.join(huc_dir, 'hydrotable.csv')
+        hydroTable_huc = os.path.join(huc_dir, "hydrotable.csv")
         if os.path.isfile(hydroTable_huc):
-            htable_req_cols = ['HUC', 'branch_id', 'feature_id', 'HydroID', 'stage', 'discharge_cms', 'LakeID']
+            htable_req_cols = [
+                "HUC",
+                "branch_id",
+                "feature_id",
+                "HydroID",
+                "stage",
+                "discharge_cms",
+                "LakeID",
+            ]
             hydroTable_all = pd.read_csv(
-                                 hydroTable_huc,
-                                 dtype={'HUC':str, 'branch_id':int, 'feature_id':str,
-                                         'HydroID':str, 'stage':float,
-                                         'discharge_cms':float, 'LakeID': int},
-                                usecols=htable_req_cols
-                                )
-            hydroTable_all.set_index(['HUC', 'feature_id', 'HydroID'], inplace=True)
-            hydroTable_branch = hydroTable_all.loc[hydroTable_all['branch_id'] == int(branch_id)]
+                hydroTable_huc,
+                dtype={
+                    "HUC": str,
+                    "branch_id": int,
+                    "feature_id": str,
+                    "HydroID": str,
+                    "stage": float,
+                    "discharge_cms": float,
+                    "LakeID": int,
+                },
+                usecols=htable_req_cols,
+            )
+            hydroTable_all.set_index(["HUC", "feature_id", "HydroID"], inplace=True)
+            hydroTable_branch = hydroTable_all.loc[hydroTable_all["branch_id"] == int(branch_id)]
         else:
             # Earlier FIM4 versions only have branch level hydrotables
-            hydroTable_branch = os.path.join( branch_dir, f'hydroTable_{branch_id}.csv' )
+            hydroTable_branch = os.path.join(branch_dir, f"hydroTable_{branch_id}.csv")
+
+        xwalked_file_name = f"gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg"
+        catchment_poly = os.path.join(branch_dir, xwalked_file_name)
 
-        xwalked_file_name = f'gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg'
-        catchment_poly = os.path.join( branch_dir, xwalked_file_name )
-    
         # branch output
         # Some other functions that call in here already added a huc, so only add it if not yet there
         if (inundation_raster is not None) and (huc not in inundation_raster):
@@ -209,44 +242,96 @@ def __inundate_gms_generator( hucs_branches,
             depths_branch_raster = fh.append_id_to_file_name(depths_raster, branch_id)
 
         # identifiers
-        identifiers = (huc,branch_id)
+        identifiers = (huc, branch_id)
 
-        #print(f"inundation_branch_raster is {inundation_branch_raster}")
+        # print(f"inundation_branch_raster is {inundation_branch_raster}")
 
         # inundate input
-        inundate_input = {  'rem': rem_branch, 
-                            'catchments': catchments_branch, 
-                            'catchment_poly': catchment_poly,
-                            'hydro_table': hydroTable_branch,
-                            'forecast': forecast,
-                            'mask_type': 'filter',
-                            'hucs': None,
-                            'hucs_layerName': None,
-                            'subset_hucs': None, 
-                            'num_workers': 1,
-                            'aggregate': False,
-                            'inundation_raster': inundation_branch_raster,
-                            'inundation_polygon': inundation_branch_polygon,
-                            'depths': depths_branch_raster,
-                            'out_raster_profile': None,
-                            'out_vector_profile': None,
-                            'quiet': not verbose }
-        
-        yield (inundate_input, identifiers)
+        inundate_input = {
+            "rem": rem_branch,
+            "catchments": catchments_branch,
+            "catchment_poly": catchment_poly,
+            "hydro_table": hydroTable_branch,
+            "forecast": forecast,
+            "mask_type": "filter",
+            "hucs": None,
+            "hucs_layerName": None,
+            "subset_hucs": None,
+            "num_workers": 1,
+            "aggregate": False,
+            "inundation_raster": inundation_branch_raster,
+            "inundation_polygon": inundation_branch_polygon,
+            "depths": depths_branch_raster,
+            "out_raster_profile": None,
+            "out_vector_profile": None,
+            "quiet": not verbose,
+        }
 
+        yield (inundate_input, identifiers)
 
-if __name__ == '__main__':
 
+if __name__ == "__main__":
     # parse arguments
-    parser = argparse.ArgumentParser(description='Inundate FIM')
-    parser.add_argument('-y','--hydrofabric_dir', help='Directory path to FIM hydrofabric by processing unit', required=True)
-    parser.add_argument('-u','--hucs',help='List of HUCS to run',required=False,default=None,type=str,nargs='+')
-    parser.add_argument('-f','--forecast',help='Forecast discharges in CMS as CSV file',required=True)
-    parser.add_argument('-i','--inundation-raster',help='Inundation Raster output. Only writes if designated.',required=False,default=None)
-    parser.add_argument('-p','--inundation-polygon',help='Inundation polygon output. Only writes if designated.',required=False,default=None)
-    parser.add_argument('-d','--depths-raster',help='Depths raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
-    parser.add_argument('-l','--log-file',help='Log-file to store level-path exceptions',required=False,default=None)
-    parser.add_argument('-o','--output-fileNames',help='Output CSV file with filenames for inundation rasters, inundation polygons, and depth rasters',required=False,default=None)
-    parser.add_argument('-w','--num-workers', help='Number of Workers', required=False,default=1)
-    parser.add_argument('-v','--verbose',help='Verbose printing',required=False,default=None,action='store_true')
-    
\ No newline at end of file
+    parser = argparse.ArgumentParser(description="Inundate FIM")
+    parser.add_argument(
+        "-y",
+        "--hydrofabric_dir",
+        help="Directory path to FIM hydrofabric by processing unit",
+        required=True,
+    )
+    parser.add_argument(
+        "-u",
+        "--hucs",
+        help="List of HUCS to run",
+        required=False,
+        default=None,
+        type=str,
+        nargs="+",
+    )
+    parser.add_argument("-f", "--forecast", help="Forecast discharges in CMS as CSV file", required=True)
+    parser.add_argument(
+        "-i",
+        "--inundation-raster",
+        help="Inundation Raster output. Only writes if designated.",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-p",
+        "--inundation-polygon",
+        help="Inundation polygon output. Only writes if designated.",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-d",
+        "--depths-raster",
+        help="Depths raster output. Only writes if designated. Appends HUC code in batch mode.",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-l",
+        "--log-file",
+        help="Log-file to store level-path exceptions",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "-o",
+        "--output-fileNames",
+        help="Output CSV file with filenames for inundation rasters, inundation polygons, and depth rasters",
+        required=False,
+        default=None,
+    )
+    parser.add_argument("-w", "--num-workers", help="Number of Workers", required=False, default=1)
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Verbose printing",
+        required=False,
+        default=None,
+        action="store_true",
+    )
+
+    Inundate_gms(**vars(parser.parse_args()))
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
index 9178ea5b5..50cf12503 100644
--- a/tools/inundate_mosaic_wrapper.py
+++ b/tools/inundate_mosaic_wrapper.py
@@ -1,58 +1,36 @@
 import os, sys
 import argparse
 import errno
-import rasterio
-from rasterio.features import shapes
-import geopandas as gpd
-import numpy as np
-from shapely.geometry.polygon import Polygon
-from shapely.geometry.multipolygon import MultiPolygon
 from timeit import default_timer as timer
-sys.path.append('/foss_fim/tools')
-from mosaic_inundation import Mosaic_inundation, mosaic_final_inundation_extent_to_poly
+
+sys.path.append("/foss_fim/tools")
+from mosaic_inundation import Mosaic_inundation
 from inundate_gms import Inundate_gms
+from utils.shared_variables import elev_raster_ndv
+from utils.shared_functions import FIM_Helpers as fh
 
 
-def inundate_and_mosaic(hydrofabric_dir, huc, flow_file, inundation_raster, depths_raster, log_file, output_fileNames, 
-                        num_workers, remove_intermediate, verbose, mosaic_attribute, huc_dir):
-    
-    """
-    Helper function. Refer to produce_mosaicked_inundation for arg descriptions.
-    """
-    
-    map_file = Inundate_gms(  hydrofabric_dir = hydrofabric_dir, 
-                                     forecast = flow_file, 
-                                     num_workers = num_workers,
-                                     hucs = huc,
-                                     inundation_raster = inundation_raster,
-                                     inundation_polygon = None,
-                                     depths_raster = depths_raster,
-                                     verbose = verbose,
-                                     log_file = None,
-                                     output_fileNames = None )
-
-    print("Mosaicking extent for " + huc + "...")
-    # Call Mosaic_inundation
-    Mosaic_inundation( map_file,
-                        mosaic_attribute = mosaic_attribute,
-                        mosaic_output = inundation_raster if depths_raster == None else depths_raster,
-                        mask = os.path.join(huc_dir,'wbd.gpkg'),
-                        unit_attribute_name = 'huc8',
-                        nodata = -9999,
-                        workers = 1,
-                        remove_inputs = remove_intermediate,
-                        subset = None,
-                        verbose = verbose )
-    
-
-def produce_mosaicked_inundation(hydrofabric_dir, huc, flow_file, inundation_raster, inundation_polygon, depths_raster,
-                                 log_file, output_fileNames, num_workers, remove_intermediate, verbose):
+def produce_mosaicked_inundation(
+    hydrofabric_dir,
+    hucs,
+    flow_file,
+    inundation_raster=None,
+    inundation_polygon=None,
+    depths_raster=None,
+    map_filename=None,
+    mask=None,
+    unit_attribute_name="huc8",
+    num_workers=1,
+    remove_intermediate=True,
+    verbose=False,
+    is_mosaic_for_branches=False,
+):
     """
     This function calls Inundate_gms and Mosaic_inundation to produce inundation maps. Possible outputs include inundation rasters
     encoded by HydroID (negative HydroID for dry and positive HydroID for wet), polygons depicting extent, and depth rasters. The
     function requires a flow file organized by NWM feature_id and discharge in cms. "feature_id" and "discharge" columns MUST be
     present in the flow file.
-    
+
     Args:
         hydrofabric_dir (str): Directory path to hydrofabric directory where FIM outputs were written by fim_pipeline.
         huc (str): The HUC for which to produce mosaicked inundation files.
@@ -60,84 +38,212 @@ def produce_mosaicked_inundation(hydrofabric_dir, huc, flow_file, inundation_ras
         inundation_raster (str): Full path to output inundation raster (encoded by positive and negative HydroIDs).
         inuntation_polygon (str): Full path to output inundation polygon. Optional.
         depths_raster (str): Full path to output depths_raster. Pixel values will be in meters. Optional.
-        log_file (str): Full path to log file to write logs to. Optional. Not tested.
-        output_fileNames (str): Full path to CSV containing paths to output file names. Optional. Not tested.
         num_workers (int): Number of parallel jobs to run.
-        keep_intermediate (bool): Option to keep intermediate files. Not tested.
+        keep_intermediate (bool): Option to keep intermediate files.
         verbose (bool): Print verbose messages to screen. Not tested.
-        
     """
-    
+
+    # Check that inundation_raster or depths_raster is supplied
+    if inundation_raster is None and depths_raster is None:
+        raise ValueError("Must supply either inundation_raster or depths_raster.")
+
     # Check that output directory exists. Notify user that output directory will be created if not.
     for output_file in [inundation_raster, inundation_polygon, depths_raster]:
         if output_file == None:
             continue
         parent_dir = os.path.split(output_file)[0]
         if not os.path.exists(parent_dir):
-            print("Parent directory for " + os.path.split(output_file)[1] + " does not exist. The parent directory will be produced.")
+            fh.vprint(
+                "Parent directory for "
+                + os.path.split(output_file)[1]
+                + " does not exist. The parent directory will be produced.",
+                verbose,
+            )
             os.makedirs(parent_dir)
-    
+
     # Check that hydrofabric_dir exists
     if not os.path.exists(hydrofabric_dir):
         raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), hydrofabric_dir)
-        
+
     # Check that huc folder exists in the hydrofabric_dir.
-    if not os.path.exists(os.path.join(hydrofabric_dir, huc)):
-        raise FileNotFoundError((errno.ENOENT, os.strerror(errno.ENOENT), os.path.join(hydrofabric_dir, huc)))
-        
+    for huc in hucs:
+        if not os.path.exists(os.path.join(hydrofabric_dir, huc)):
+            raise FileNotFoundError(
+                (
+                    errno.ENOENT,
+                    os.strerror(errno.ENOENT),
+                    os.path.join(hydrofabric_dir, huc),
+                )
+            )
+
     # Check that flow file exists
     if not os.path.exists(flow_file):
         raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), flow_file)
-    
+
     # Check job numbers and raise error if necessary
     total_cpus_available = os.cpu_count() - 1
     if num_workers > total_cpus_available:
-        raise ValueError('The number of workers (-w), {}, '\
-                          'exceeds your machine\'s available CPU count minus one ({}). '\
-                          'Please lower the num_workers.'.format(num_workers, total_cpus_available))
-    
-    huc_dir = os.path.join(hydrofabric_dir, huc)
-    print("Running inundate for " + huc + "...")
+        raise ValueError(
+            "The number of workers (-w), {}, "
+            "exceeds your machine's available CPU count minus one ({}). "
+            "Please lower the num_workers.".format(num_workers, total_cpus_available)
+        )
+
+    fh.vprint("Running inundate for " + huc + "...", verbose)
+
     # Call Inundate_gms
-    fn_inundation_raster, fn_depths_raster, mosaic_attribute = inundation_raster, None, 'inundation_rasters'
-    
-    inundate_and_mosaic(hydrofabric_dir, huc, flow_file, fn_inundation_raster, fn_depths_raster,
-                                 log_file, output_fileNames, num_workers, remove_intermediate, verbose, mosaic_attribute, huc_dir)
-    
-    # Produce depths if instructed
-    if depths_raster != None:
-        print("Computing depths for " + huc + "...")
-        fn_inundation_raster, fn_depths_raster, mosaic_attribute = None, depths_raster, 'depths_rasters'
-
-        inundate_and_mosaic(hydrofabric_dir, huc, flow_file, fn_inundation_raster, fn_depths_raster,
-                                     log_file, output_fileNames, num_workers, remove_intermediate, verbose, mosaic_attribute, huc_dir)
-    else:
-        pass
-    
-    if inundation_polygon != None:
-        mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon)
-                
-    print("Mosaicking complete.")
-    
-if __name__ == '__main__':
+    map_file = Inundate_gms(
+        hydrofabric_dir=hydrofabric_dir,
+        forecast=flow_file,
+        num_workers=num_workers,
+        hucs=hucs,
+        inundation_raster=inundation_raster,
+        depths_raster=depths_raster,
+        verbose=verbose,
+    )
+
+    # Write map file if designated
+    if map_filename is not None:
+        if not os.path.isdir(os.path.dirname(map_filename)):
+            os.makedirs(os.path.dirname(map_filename))
+
+        map_file.to_csv(map_filename, index=False)
+
+    fh.vprint("Mosaicking extent...", verbose)
+
+    for mosaic_attribute in ["depths_rasters", "inundation_rasters"]:
+        mosaic_output = None
+        if mosaic_attribute == "inundation_rasters":
+            if inundation_raster is not None:
+                mosaic_output = inundation_raster
+        elif mosaic_attribute == "depths_rasters":
+            if depths_raster is not None:
+                mosaic_output = depths_raster
 
+    if mosaic_output is not None:
+        # Call Mosaic_inundation
+        mosaic_file_path = Mosaic_inundation(
+            map_file.copy(),
+            mosaic_attribute=mosaic_attribute,
+            mosaic_output=mosaic_output,
+            mask=mask,
+            unit_attribute_name=unit_attribute_name,
+            nodata=elev_raster_ndv,
+            remove_inputs=remove_intermediate,
+            verbose=verbose,
+            is_mosaic_for_branches=is_mosaic_for_branches,
+            inundation_polygon=inundation_polygon,
+        )
+
+    fh.vprint("Mosaicking complete.", verbose)
+
+    return mosaic_file_path
+
+
+if __name__ == "__main__":
     # Parse arguments
-    parser = argparse.ArgumentParser(description='Helpful utility to produce mosaicked inundation extents (raster and poly) and depths.')
-    parser.add_argument('-y','--hydrofabric_dir', help='Directory path to FIM hydrofabric by processing unit.', required=True)
-    parser.add_argument('-u','--huc',help='List of HUCS to run',required=True,default="",type=str,)
-    parser.add_argument('-f','--flow_file',help='Discharges in CMS as CSV file. "feature_id" and "discharge" columns MUST be supplied.',required=True)
-    parser.add_argument('-i','--inundation-raster',help='Inundation raster output.',required=True,default=None)
-    parser.add_argument('-p','--inundation-polygon',help='Inundation polygon output. Only writes if designated.',required=False,default=None)
-    parser.add_argument('-d','--depths-raster',help='Depths raster output. Only writes if designated. Appends HUC code in batch mode.',required=False,default=None)
-    parser.add_argument('-l','--log-file',help='Log-file to store level-path exceptions. Not tested.',required=False,default=None)
-    parser.add_argument('-o','--output-fileNames',help='Output CSV file with filenames for inundation rasters, inundation polygons, and depth rasters. Not tested.',required=False,default=None)
-    parser.add_argument('-w','--num-workers', help='Number of workers.', required=False,default=1, type=int)
-    parser.add_argument('-r','--remove-intermediate',help='Keep intermediate products, i.e. individual branch inundation.',required=False,default=False,action='store_true')
-    parser.add_argument('-v','--verbose',help='Verbose printing. Not tested.',required=False,default=False,action='store_true')
-    
+    parser = argparse.ArgumentParser(
+        description="Helpful utility to produce mosaicked inundation extents (raster and poly) and depths."
+    )
+    parser.add_argument(
+        "-y",
+        "--hydrofabric_dir",
+        help="Directory path to FIM hydrofabric by processing unit.",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        "-u",
+        "--hucs",
+        help="List of HUCS to run",
+        required=True,
+        default="",
+        type=str,
+        nargs="+",
+    )
+    parser.add_argument(
+        "-f",
+        "--flow_file",
+        help='Discharges in CMS as CSV file. "feature_id" and "discharge" columns MUST be supplied.',
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        "-i",
+        "--inundation-raster",
+        help="Inundation raster output.",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-p",
+        "--inundation-polygon",
+        help="Inundation polygon output. Only writes if designated.",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-d",
+        "--depths-raster",
+        help="Depths raster output. Only writes if designated. Appends HUC code in batch mode.",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-m",
+        "--map-filename",
+        help="Path to write output map file CSV (optional). Default is None.",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-k",
+        "--mask",
+        help="Name of mask file.",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-a",
+        "--unit_attribute_name",
+        help='Name of attribute column in map_file. Default is "huc8".',
+        required=False,
+        default="huc8",
+        type=str,
+    )
+    parser.add_argument(
+        "-w",
+        "--num-workers",
+        help="Number of workers.",
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        "-r",
+        "--remove-intermediate",
+        help="Remove intermediate products, i.e. individual branch inundation.",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Verbose printing. Not tested.",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+
     start = timer()
 
     # Extract to dictionary and run
-    produce_mosaicked_inundation( **vars(parser.parse_args()) )
+    produce_mosaicked_inundation(**vars(parser.parse_args()))
 
-    print(f'Completed in {round((timer() - start)/60, 2)} minutes.')
+    print(f"Completed in {round((timer() - start)/60, 2)} minutes.")
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index efcf0e503..db166442a 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -12,82 +12,88 @@
 from osgeo import gdal, ogr
 
 from datetime import datetime
-from mosaic_inundation import Mosaic_inundation
-from inundate_gms import Inundate_gms
-from inundation import inundate
+from inundate_mosaic_wrapper import produce_mosaicked_inundation
 
 # setting path
-sys.path.append('/foss_fim/src')
+sys.path.append("/foss_fim/src")
 from utils.shared_variables import elev_raster_ndv, PREP_PROJECTION
 from utils.shared_functions import FIM_Helpers as fh
 
 
-#INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
-#INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
-#INPUTS_DIR = r'/data/inputs'
-#OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/
-#DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
+# INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
+# INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
+# INPUTS_DIR = r'/data/inputs'
+# OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/
+# DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
 
 
-def inundate_nation(fim_run_dir, output_dir, magnitude_key, 
-                    flow_file, inc_mosaic, job_number):
+def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, inc_mosaic, job_number):
+    assert os.path.isdir(fim_run_dir), f"ERROR: could not find the input fim_dir location: {fim_run_dir}"
 
-    
-    assert os.path.isdir(fim_run_dir), f'ERROR: could not find the input fim_dir location: {fim_run_dir}'
-    
-    assert os.path.exists(flow_file), f'ERROR: could not find the flow file: {flow_file}'
+    assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}"
 
     if job_number > available_cores:
         job_number = available_cores - 1
-        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+        print(
+            "Provided job number exceeds the number of available cores. "
+            + str(job_number)
+            + " max jobs will be used instead."
+        )
 
     fim_version = os.path.basename(os.path.normpath(fim_run_dir))
-    logging.info(f'Using fim version: {fim_version}')
+    logging.info(f"Using fim version: {fim_version}")
     output_base_file_name = magnitude_key + "_" + fim_version
-    #print(output_base_file_name)
-    
-    __setup_logger(output_dir, output_base_file_name )
+    # print(output_base_file_name)
+
+    __setup_logger(output_dir, output_base_file_name)
 
     start_dt = datetime.now()
 
-    logging.info(f'Input FIM Directory: {fim_run_dir}')
-    logging.info(f'output_dir: {output_dir}')
-    logging.info(f'magnitude_key: {magnitude_key}')
-    logging.info(f'flow_file: {flow_file}')
-    logging.info(f'inc_mosaic: {str(inc_mosaic)}')
+    logging.info(f"Input FIM Directory: {fim_run_dir}")
+    logging.info(f"output_dir: {output_dir}")
+    logging.info(f"magnitude_key: {magnitude_key}")
+    logging.info(f"flow_file: {flow_file}")
+    logging.info(f"inc_mosaic: {str(inc_mosaic)}")
 
     print("Preparing to generate inundation outputs for magnitude: " + magnitude_key)
     print("Input flow file: " + flow_file)
-    
+
     magnitude_output_dir = os.path.join(output_dir, output_base_file_name)
 
     if not os.path.exists(magnitude_output_dir):
-        print('Creating new output directory for raw mosaic files: ' + magnitude_output_dir)
+        print("Creating new output directory for raw mosaic files: " + magnitude_output_dir)
         os.mkdir(magnitude_output_dir)
     else:
         # we need to empty it. we will kill it and remake it (using rmtree to force it)
         shutil.rmtree(magnitude_output_dir, ignore_errors=True)
         os.mkdir(magnitude_output_dir)
-        
+
     huc_list = []
     for huc in os.listdir(fim_run_dir):
-        
-        #if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
-        if re.match('\d{8}', huc):    
+        # if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
+        if re.match("\d{8}", huc):
             huc_list.append(huc)
 
-    print('Inundation raw mosaic outputs here: ' + magnitude_output_dir)
-   
-    run_inundation([fim_run_dir, huc_list, magnitude_key,
-                    magnitude_output_dir, flow_file, job_number])
-                
+    print("Inundation raw mosaic outputs here: " + magnitude_output_dir)
+
+    run_inundation(
+        [
+            fim_run_dir,
+            huc_list,
+            magnitude_key,
+            magnitude_output_dir,
+            flow_file,
+            job_number,
+        ]
+    )
+
     # Perform mosaic operation
     if inc_mosaic:
         fh.print_current_date_time()
         logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))
         print("Performing bool mosaic process...")
         logging.info("Performing bool mosaic process...")
-        
+
         output_bool_dir = os.path.join(output_dir, "bool_temp")
         if not os.path.exists(output_bool_dir):
             os.mkdir(output_bool_dir)
@@ -98,8 +104,8 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key,
 
         procs_list = []
         for rasfile in os.listdir(magnitude_output_dir):
-            if rasfile.endswith('.tif') and "extent" in rasfile:
-                #p = magnitude_output_dir + rasfile
+            if rasfile.endswith(".tif") and "extent" in rasfile:
+                # p = magnitude_output_dir + rasfile
                 procs_list.append([magnitude_output_dir, rasfile, output_bool_dir])
 
         # Multiprocess --> create boolean inundation rasters for all hucs
@@ -107,19 +113,16 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key,
             with Pool(processes=job_number) as pool:
                 pool.map(create_bool_rasters, procs_list)
         else:
-            msg = f'Did not find any valid FIM extent rasters: {magnitude_output_dir}'
+            msg = f"Did not find any valid FIM extent rasters: {magnitude_output_dir}"
             print(msg)
             logging.info(msg)
 
-        # Perform VRT creation and final mosaic using boolean rasters
-        vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name)
-        
         # now cleanup the raw mosiac directories
         shutil.rmtree(output_bool_dir, ignore_errors=True)
 
     # now cleanup the raw mosiac directories
     shutil.rmtree(magnitude_output_dir, ignore_errors=True)
-        
+
     fh.print_current_date_time()
     logging.info(logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S")))
     end_time = datetime.now()
@@ -129,48 +132,36 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key,
 def run_inundation(args):
     """
     This script is a wrapper for the inundate function and is designed for multiprocessing.
-    
+
     Args:
         args (list): [fim_run_dir (str), huc_list (list), magnitude (str), magnitude_output_dir (str), forecast (str), job_number (int)]
-    
+
     """
-    
-    fim_run_dir = args[0]  
+
+    fim_run_dir = args[0]
     huc_list = args[1]
     magnitude = args[2]
     magnitude_output_dir = args[3]
     forecast = args[4]
     job_number = args[5]
-   
+
     # Define file paths for use in inundate().
 
-    inundation_raster = os.path.join(magnitude_output_dir, magnitude + '_inund_extent.tif')
-    
+    inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif")
+
     print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude))
 
-    map_file = Inundate_gms( hydrofabric_dir = fim_run_dir, 
-                             forecast = forecast, 
-                             num_workers = job_number,
-                             hucs = huc_list,
-                             inundation_raster = inundation_raster,
-                             inundation_polygon = None,
-                             depths_raster = None,
-                             verbose = True,
-                             log_file = None,
-                             output_fileNames = None )
-    
-    Mosaic_inundation( map_file,
-                       mosaic_attribute = 'inundation_rasters',
-                       mosaic_output = inundation_raster,
-                       #mask = os.path.join(fim_run_dir,huc8,'wbd.gpkg'),
-                       mask = None,
-                       unit_attribute_name = 'huc8',
-                       nodata = elev_raster_ndv,
-                       workers = 1,
-                       remove_inputs = True,
-                       subset = None,
-                       verbose = True,
-                       is_mosaic_for_branches = True )
+    produce_mosaicked_inundation(
+        fim_run_dir,
+        huc_list,
+        forecast,
+        inundation_raster=inundation_raster,
+        num_workers=job_number,
+        remove_intermediate=True,
+        verbose=True,
+        is_mosaic_for_branches=True,
+    )
+
 
 def create_bool_rasters(args):
     in_raster_dir = args[0]
@@ -183,71 +174,41 @@ def create_bool_rasters(args):
     profile = raster.profile
     array = raster.read()
     del raster
-    array[array>0] = 1
-    array[array<=0] = 0
+    array[array > 0] = 1
+    array[array <= 0] = 0
     # And then change the band count to 1, set the
     # dtype to uint8, and specify LZW compression.
-    profile.update(driver="GTiff",
-                height=array.shape[1],
-                width=array.shape[2],
-                tiled=True,
-                nodata=0,
-                blockxsize=512, 
-                blockysize=512,
-                dtype='int8',
-                compress='lzw')
-    with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, 'w', **profile) as dst:
+    profile.update(
+        driver="GTiff",
+        height=array.shape[1],
+        width=array.shape[2],
+        tiled=True,
+        nodata=0,
+        blockxsize=512,
+        blockysize=512,
+        dtype="int8",
+        compress="lzw",
+    )
+    with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, "w", **profile) as dst:
         dst.write(array.astype(rasterio.int8))
 
 
-def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag):
-    
-  
-    # NOTE: Oct 2022.. we no longer need the VRT, only the large mosaic'd raster. 
-    # this code is about to be deprecated, so we will leave it as is.
-    
-    rasters_to_mosaic = []
-    for rasfile in os.listdir(output_bool_dir):
-        if rasfile.endswith('.tif') and "extent" in rasfile:
-            p = output_bool_dir + os.sep + rasfile
-            print("Processing: " + p)
-            rasters_to_mosaic.append(p)
-
-    logging.info(fh.print_current_date_time())
-    output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt")
-    print("Creating virtual raster: " + output_mosiac_vrt)
-    logging.info("Creating virtual raster: " + output_mosiac_vrt)
-    vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic)
-    
-    output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif")    
-    print("Building raster mosaic: " + output_mosiac_raster)
-    logging.info("Building raster mosaic: " + output_mosiac_raster)
-    print("This can take a number of hours, watch 'docker stats' cpu value to ensure the process"\
-        "to ensure the process is still working")
-    gdal.Translate(output_mosiac_raster, vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2'])
-    vrt = None        
-
-
 def __setup_logger(output_folder_path, log_file_name_key):
-
     start_time = datetime.now()
     file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
     log_file_name = f"{log_file_name_key}-{file_dt_string}.log"
 
     log_file_path = os.path.join(output_folder_path, log_file_name)
-    
-    logging.basicConfig(filename=log_file_path, 
-                        level=logging.DEBUG,
-                        format='%(message)s')
+
+    logging.basicConfig(filename=log_file_path, level=logging.DEBUG, format="%(message)s")
 
     # yes.. this can do console logs as well, but it can be a bit unstable and ugly
-    
+
     logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
     logging.info("----------------")
 
 
-if __name__ == '__main__':
-
+if __name__ == "__main__":
     # Sample usage: python3 /foss_fim/tools/inundate_nation.py -r /outputs/fim_4_0_9_2 -m 100_0 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_100_0_cms.csv -s -j 10
     # outputs become /data/inundation_review/inundate_nation/100_0_fim_4_0_9_2_mosiac.tif (.log, etc)
 
@@ -255,33 +216,66 @@ def __setup_logger(output_folder_path, log_file_name_key):
     # outputs become /data/inundation_review/inundate_nation/hw_fim_4_0_9_2_mosiac.tif (.log, etc)
 
     # if run on UCS2, you can map docker as -v /dev_fim_share... etc/:/data -v /local....outputs:/outputs -v foss_fim as normal.
-    
+
     available_cores = multiprocessing.cpu_count()
 
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow '\
-        'recurrence interflow data. Inundation outputs are stored in the '\
-        '/inundation_review/inundation_nwm_recurr/ directory.')
-        
-    parser.add_argument('-r', '--fim-run-dir', help='Name of directory containing outputs '\
-        'of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)', required=True)
-        
-    parser.add_argument('-o', '--output-dir', help='Optional: The path to a directory to write the '\
-        'outputs. If not used, the inundation_nation directory is used by default '\
-        'ie) /data/inundation_review/inundate_nation/',
-        default='/data/inundation_review/inundate_nation/', required=False)
-       
-    parser.add_argument('-m', '--magnitude_key', help = 'used in output folders names and temp files, '\
-        'added to output_file_name_key ie 100_0, 2_0, hw, etc)', required = True)
-        
-    parser.add_argument('-f', '--flow_file', help = 'the path and flow file to be used. '\
-        'ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv', required = True)
-        
-    parser.add_argument('-s', '--inc_mosaic',help='Optional flag to produce mosaic of FIM extent rasters',
-                        action='store_true')
-                        
-    parser.add_argument('-j', '--job-number', help='The number of jobs', required=False, default=1, type=int)
-        
+    parser = argparse.ArgumentParser(
+        description="Inundation mapping for FOSS FIM using streamflow "
+        "recurrence interflow data. Inundation outputs are stored in the "
+        "/inundation_review/inundation_nwm_recurr/ directory."
+    )
+
+    parser.add_argument(
+        "-r",
+        "--fim-run-dir",
+        help="Name of directory containing outputs "
+        "of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        help="Optional: The path to a directory to write the "
+        "outputs. If not used, the inundation_nation directory is used by default "
+        "ie) /data/inundation_review/inundate_nation/",
+        default="/data/inundation_review/inundate_nation/",
+        required=False,
+    )
+
+    parser.add_argument(
+        "-m",
+        "--magnitude_key",
+        help="used in output folders names and temp files, "
+        "added to output_file_name_key ie 100_0, 2_0, hw, etc)",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-f",
+        "--flow_file",
+        help="the path and flow file to be used. "
+        "ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-s",
+        "--inc_mosaic",
+        help="Optional flag to produce mosaic of FIM extent rasters",
+        action="store_true",
+    )
+
+    parser.add_argument(
+        "-j",
+        "--job-number",
+        help="The number of jobs",
+        required=False,
+        default=1,
+        type=int,
+    )
+
     args = vars(parser.parse_args())
 
-    inundate_nation(**args)
\ No newline at end of file
+    inundate_nation(**args)
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 4ec312655..368fc2978 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -4,60 +4,56 @@
 import argparse
 import os
 import pandas as pd
-import sys
-
-
-from glob import glob
 from overlapping_inundation import OverlapWindowMerge
 from tqdm import tqdm
 from utils.shared_variables import elev_raster_ndv
 from utils.shared_functions import FIM_Helpers as fh
 
 
-def Mosaic_inundation( map_file,
-                       mosaic_attribute = 'inundation_rasters',
-                       mosaic_output = None,
-                       mask = None,
-                       unit_attribute_name = 'huc8',
-                       nodata = elev_raster_ndv,
-                       workers = 1,
-                       remove_inputs = False,
-                       subset = None,
-                       verbose = True,
-                       is_mosaic_for_branches = False ):
-    
-    # Notes:
-    #    - If is_mosaic_for_branches is true, the mosaic output name
-    #      will add the HUC into the output name for overwrite resons.
+def Mosaic_inundation(
+    map_file,
+    mosaic_attribute,
+    mosaic_output=None,
+    mask=None,
+    unit_attribute_name="huc8",
+    nodata=elev_raster_ndv,
+    workers=1,
+    remove_inputs=False,
+    subset=None,
+    verbose=True,
+    is_mosaic_for_branches=False,
+    inundation_polygon=None,
+):
+    if not os.path.isdir(os.path.dirname(mosaic_output)):
+        os.makedirs(os.path.dirname(mosaic_output))
 
     # check input
-    if mosaic_attribute not in ('inundation_rasters','depths_rasters'):
-        raise ValueError('Pass inundation or depths for mosaic_attribute argument') 
+    if mosaic_attribute not in ("inundation_rasters", "depths_rasters"):
+        raise ValueError("Pass inundation_rasters or depths_raster for mosaic_attribute argument")
 
     # load file
-    if isinstance(map_file,pd.DataFrame):
+    if isinstance(map_file, pd.DataFrame):
         inundation_maps_df = map_file
         del map_file
-    elif isinstance(map_file,str):
-        inundation_maps_df = pd.read_csv(map_file,
-                                         dtype={unit_attribute_name:str,'branchID':str})
+    elif isinstance(map_file, str):
+        inundation_maps_df = pd.read_csv(map_file, dtype={unit_attribute_name: str, "branchID": str})
     else:
-        raise TypeError('Pass Pandas Dataframe or file path string to csv for map_file argument')
+        raise TypeError("Pass Pandas Dataframe or file path string to csv for map_file argument")
 
     # remove NaNs
-    inundation_maps_df.dropna(axis=0, how='all', inplace=True)
+    inundation_maps_df.dropna(axis=0, how="all", inplace=True)
 
     # subset
     if subset is not None:
-        subset_mask = inundation_maps_df.loc[:,unit_attribute_name].isin(subset)
-        inundation_maps_df = inundation_maps_df.loc[subset_mask,:]
-    
+        subset_mask = inundation_maps_df.loc[:, unit_attribute_name].isin(subset)
+        inundation_maps_df = inundation_maps_df.loc[subset_mask, :]
+
     # unique aggregation units
-    aggregation_units = inundation_maps_df.loc[:,unit_attribute_name].unique()
+    aggregation_units = inundation_maps_df.loc[:, unit_attribute_name].unique()
 
     inundation_maps_df.set_index(unit_attribute_name, drop=True, inplace=True)
 
-    # decide upon wheter to display 
+    # decide upon whether to display
     if verbose & len(aggregation_units) == 1:
         tqdm_disable = False
     elif verbose:
@@ -66,134 +62,228 @@ def Mosaic_inundation( map_file,
         tqdm_disable = True
 
     ag_mosaic_output = ""
+    remove_at_end = []
 
-    for ag in tqdm(aggregation_units, disable = tqdm_disable, desc = 'Mosaicing FIMs'):
-
+    for ag in tqdm(aggregation_units, disable=tqdm_disable, desc="Mosaicing FIMs"):
         try:
-            inundation_maps_list = inundation_maps_df.loc[ag,mosaic_attribute].tolist()
+            inundation_maps_list = inundation_maps_df.loc[ag, mosaic_attribute].tolist()
         except AttributeError:
-            inundation_maps_list = [ inundation_maps_df.loc[ag,mosaic_attribute] ]
+            inundation_maps_list = [inundation_maps_df.loc[ag, mosaic_attribute]]
 
-        # Some processes may have already added the ag value (if it is a huc) to 
+        # Some processes may have already added the ag value (if it is a huc) to
         # the file name, so don't re-add it.
-        # Only add the huc into the name if branches are being processed, as 
-        # sometimes the mosiac is not for gms branches but maybe mosaic of an
+        # Only add the huc into the name if branches are being processed, as
+        # sometimes the mosaic is not for gms branches but maybe mosaic of an
         # fr set with a gms composite map.
 
         ag_mosaic_output = mosaic_output
         if (is_mosaic_for_branches) and (ag not in mosaic_output):
-            ag_mosaic_output = fh.append_id_to_file_name(mosaic_output, ag) # change it
+            ag_mosaic_output = fh.append_id_to_file_name(mosaic_output, ag)  # change it
+
+        remove_list = mosaic_by_unit(
+            inundation_maps_list,
+            ag_mosaic_output,
+            nodata,
+            workers=1,
+            remove_inputs=remove_inputs,
+            mask=mask,
+            verbose=verbose,
+        )
 
-        mosaic_by_unit(inundation_maps_list, 
-                      ag_mosaic_output,
-                      nodata,
-                      workers = workers, 
-                      remove_inputs = remove_inputs,
-                      mask = mask,
-                      verbose = verbose)
+        if remove_list is not None:
+            remove_at_end.extend(remove_list)
 
+    # # inundation maps
+    # inundation_maps_df.reset_index(drop=True)
 
-    # inundation maps
-    inundation_maps_df.reset_index(drop=True)
+    if inundation_polygon is not None:
+        mosaic_final_inundation_extent_to_poly(ag_mosaic_output, inundation_polygon)
+
+    if remove_inputs:
+        fh.vprint("Removing inputs ...", verbose)
+
+        for remove_file in remove_at_end:
+            os.remove(remove_file)
 
     # Return file name and path of the final mosaic output file.
     # Might be empty.
     return ag_mosaic_output
 
 
-# Note: This uses threading and not processes. If the number of workers is more than 
+# Note: This uses threading and not processes. If the number of workers is more than
 # the number of possible threads, no results will be returned. But it is usually
 # pretty fast anyways. This needs to be fixed.
-def mosaic_by_unit(inundation_maps_list,
-                   mosaic_output,
-                   nodata = elev_raster_ndv,
-                   workers = 1,
-                   remove_inputs = False,
-                   mask = None,
-                   verbose = False):
-
+def mosaic_by_unit(
+    inundation_maps_list,
+    mosaic_output,
+    nodata=elev_raster_ndv,
+    workers=1,
+    remove_inputs=False,
+    mask=None,
+    verbose=False,
+):
     # overlap object instance
-    overlap = OverlapWindowMerge( inundation_maps_list, (30, 30) )
+    overlap = OverlapWindowMerge(inundation_maps_list, (30, 30))
 
     if mosaic_output is not None:
         if workers > 1:
             threaded = True
         else:
-            threaded= False
-        
-        overlap.merge_rasters(mosaic_output, threaded=threaded, workers=workers, nodata=nodata)
+            threaded = False
+
+        overlap.merge_rasters(mosaic_output, threaded=threaded, workers=1, nodata=nodata)
 
         if mask:
-            fh.vprint("Masking ...", verbose)                
+            fh.vprint("Masking ...", verbose)
             overlap.mask_mosaic(mosaic_output, mask, outfile=mosaic_output)
-    
+
     if remove_inputs:
         fh.vprint("Removing inputs ...", verbose)
 
+        remove_list = []
         for inun_map in inundation_maps_list:
-            if inun_map is not None:
-                if os.path.isfile(inun_map):
-                    os.remove(inun_map)
-                    
+            if inun_map is not None and os.path.isfile(inun_map):
+                remove_list.append(inun_map)
+
+        return remove_list
 
-def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon):
+
+def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon, driver="GPKG"):
     import numpy as np
     import rasterio
     from rasterio.features import shapes
     from shapely.geometry.polygon import Polygon
     from shapely.geometry.multipolygon import MultiPolygon
     import geopandas as gpd
-    
+
     with rasterio.open(inundation_raster) as src:
         # Open inundation_raster using rasterio.
         image = src.read(1)
-        mask = image > 0
         print("Producing merged polygon...")
-    
-        # Use numpy.where operation to reclassify depth_array on the condition that the pixel values are > 0.
-        reclass_inundation_array = np.where((image>0) & (image != src.nodata), 1, 0).astype('uint8')
 
-        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
+        # Use numpy.where operation to reclassify depth_array on the condition that the pixel values are > 0.
+        reclass_inundation_array = np.where((image > 0) & (image != src.nodata), 1, 0).astype("uint8")
 
         # Aggregate shapes
-        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(reclass_inundation_array, mask=reclass_inundation_array>0,transform=src.transform)))
+        results = (
+            {"properties": {"extent": 1}, "geometry": s}
+            for i, (s, v) in enumerate(
+                shapes(
+                    reclass_inundation_array,
+                    mask=reclass_inundation_array > 0,
+                    transform=src.transform,
+                )
+            )
+        )
 
         # Convert list of shapes to polygon, then dissolve
         extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=src.crs)
-        extent_poly_diss = extent_poly.dissolve(by='extent')
-        extent_poly_diss["geometry"] = [MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in extent_poly_diss["geometry"]]
-        
+        extent_poly_diss = extent_poly.dissolve(by="extent")
+        extent_poly_diss["geometry"] = [
+            MultiPolygon([feature]) if type(feature) == Polygon else feature
+            for feature in extent_poly_diss["geometry"]
+        ]
+
         # Write polygon
-        extent_poly_diss.to_file(inundation_polygon, driver='GPKG')
-
-    
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Mosaic GMS Inundation Rasters')
-    parser.add_argument('-i','--map-file', 
-                        help='List of file paths to inundation/depth maps to mosaic',
-                        required=True)
-    parser.add_argument('-a','--mask', 
-                        help='File path to vector polygon mask to clip mosaic too',
-                        required=False, default=None)
-    parser.add_argument('-s','--subset', help='Subset units', 
-                        required=False, default=None, type=str, nargs='+')
-    parser.add_argument('-n','--nodata', help='Inundation Maps',
-                        required=False, default=elev_raster_ndv)
-    parser.add_argument('-w','--workers', help='Number of Workers', 
-                        required=False, default=4, type=int)
-    parser.add_argument('-t','--mosaic-attribute', help='Mosaiced inundation Maps', 
-                        required=False, default=None)
-    parser.add_argument('-m','--mosaic-output', help='Mosaiced inundation Maps file name',
-                        required=False, default=None)
-    parser.add_argument('-r','--remove-inputs', help='Remove original input inundation Maps',
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-v','--verbose', help='Remove original input inundation Maps', 
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-g','--is-mosaic-for-branches', 
-                        help='If the mosaic is for branchs, include this arg',
-                        required=False, default=False, action='store_true')
+        extent_poly_diss.to_file(inundation_polygon, driver=driver)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Mosaic GMS Inundation Rasters")
+    parser.add_argument(
+        "-m",
+        "--map-file",
+        help="Pandas Dataframe or file path string to CSV of inundation/depth maps to mosaic.",
+        required=True,
+    )
+    parser.add_argument(
+        "-t",
+        "--mosaic-attribute",
+        help="Attribute name: should be either inundation_rasters or depths_rasters.",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        "-a",
+        "--mask",
+        help="File path to vector polygon mask used to clip mosaic (optional). Default is None",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-u",
+        "--unit-attribute-name",
+        help="Unit attribute name (optional). Default is huc8",
+        required=False,
+        default="huc8",
+        type=str,
+    )
+    parser.add_argument(
+        "-s",
+        "--subset",
+        help="Value(s) of unit attribute name used to subset (optional)",
+        required=False,
+        default=None,
+        type=str,
+        nargs="+",
+    )
+    parser.add_argument(
+        "-n",
+        "--nodata",
+        help="NODATA value for output raster",
+        required=False,
+        default=elev_raster_ndv,
+    )
+    parser.add_argument(
+        "-w",
+        "--workers",
+        help="Number of Workers (optional). Default value is 1.",
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        "-o",
+        "--mosaic-output",
+        help="Mosaiced inundation Maps file name",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-i",
+        "--inundation-polygon",
+        help="Filename of the final inundation extent polygon (optional). Default is None.",
+        required=False,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-r",
+        "--remove-inputs",
+        help="Remove original input inundation Maps (optional). Default is False",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Print out messages (optional). Default is False",
+        required=False,
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "-g",
+        "--is-mosaic-for-branches",
+        help="If the mosaic is for branchs, include this arg. If is_mosaic_for_branches is true, the mosaic output name will add the HUC into the output name for overwrite reasons.",
+        required=False,
+        default=False,
+        action="store_true",
+    )
 
     args = vars(parser.parse_args())
-    
+
     Mosaic_inundation(**args)
diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py
index 003b12457..7ed30b20c 100644
--- a/tools/overlapping_inundation.py
+++ b/tools/overlapping_inundation.py
@@ -12,15 +12,11 @@
 from numba import njit
 import geopandas as gpd
 from rasterio.mask import mask
-import sys
 import warnings
 
-class OverlapWindowMerge:
 
-    def __init__(self,
-                 inundation_rsts,
-                 num_partitions=None,
-                 window_xy_size=None):
+class OverlapWindowMerge:
+    def __init__(self, inundation_rsts, num_partitions=None, window_xy_size=None):
         """
         Initialize the object
 
@@ -30,40 +26,44 @@ def __init__(self,
         """
 
         # sort for largest spanning dataset (todo: handle mismatched resolutions)
-        size_func = lambda x: np.abs(x.bounds.left - x.bounds.right) * \
-                              np.abs(x.bounds.top - x.bounds.bottom)
-        key_sort_func = lambda x: x['size']
+        size_func = lambda x: np.abs(x.bounds.left - x.bounds.right) * np.abs(x.bounds.top - x.bounds.bottom)
+        key_sort_func = lambda x: x["size"]
         datasets = [rasterio.open(ds) for ds in inundation_rsts]
-        ds_dict = [{'dataset': ds, 'size': size_func(ds)} for ds in datasets]
+        ds_dict = [{"dataset": ds, "size": size_func(ds)} for ds in datasets]
         ds_dict.sort(key=key_sort_func, reverse=True)
 
         # load sample overlapping inundation depth rasters
-        self.depth_rsts = [x['dataset'] for x in ds_dict]
+        self.depth_rsts = [x["dataset"] for x in ds_dict]
         del ds_dict
 
         self.rst_dims = [[x.height, x.width] for x in self.depth_rsts]
 
-        self.res = self.depth_rsts[0].meta['transform'][0]
-        self.depth_bounds = np.array([[[x.bounds.top,
-                                        x.bounds.left],
-                                       [x.bounds.bottom,
-                                        x.bounds.right]] for x in self.depth_rsts]) / self.res
-
+        self.res = self.depth_rsts[0].meta["transform"][0]
+        self.depth_bounds = (
+            np.array(
+                [[[x.bounds.top, x.bounds.left], [x.bounds.bottom, x.bounds.right]] for x in self.depth_rsts]
+            )
+            / self.res
+        )
 
         # get transform, width, height and bounds
-        self.proc_unit_transform, self.proc_unit_width, \
-        self.proc_unit_height, final_bounds = \
-        self.get_final_dims()
+        (
+            self.proc_unit_transform,
+            self.proc_unit_width,
+            self.proc_unit_height,
+            final_bounds,
+        ) = self.get_final_dims()
 
-        self.proc_unit_bounds = np.array([[final_bounds['top'],
-                                           final_bounds['left']],
-                                          [final_bounds['bottom'],
-                                           final_bounds['right']]])
+        self.proc_unit_bounds = np.array(
+            [[final_bounds["top"], final_bounds["left"]], [final_bounds["bottom"], final_bounds["right"]]]
+        )
 
         self.proc_unit_bounds = self.proc_unit_bounds / self.res
 
-        self.lat_lon_sign = [np.sign(self.proc_unit_bounds[1, 0] - self.proc_unit_bounds[0, 0]),
-                             np.sign(self.proc_unit_bounds[1, 1] - self.proc_unit_bounds[0, 1])]
+        self.lat_lon_sign = [
+            np.sign(self.proc_unit_bounds[1, 0] - self.proc_unit_bounds[0, 0]),
+            np.sign(self.proc_unit_bounds[1, 1] - self.proc_unit_bounds[0, 1]),
+        ]
 
         self.partitions = num_partitions
         self.window_sizes = window_xy_size
@@ -96,21 +96,13 @@ def get_final_dims(self):
         left = newton(self.get_res_bbox_min, left, args=(left, right, self.res))
         bottom = newton(self.get_res_bbox_min, bottom, args=(bottom, top, self.res))
 
-        transform = self.depth_rsts[0].meta['transform']
+        transform = self.depth_rsts[0].meta["transform"]
 
         width = int(np.abs(right - left) / self.res)
         height = int(np.abs(top - bottom) / self.res)
-        new_transform = Affine(transform[0],
-                               transform[1],
-                               left,
-                               transform[3],
-                               transform[4],
-                               top)
-
-        return new_transform, width, height, {'left': left,
-                                              'top': top,
-                                              'right': right,
-                                              'bottom': bottom}
+        new_transform = Affine(transform[0], transform[1], left, transform[3], transform[4], top)
+
+        return new_transform, width, height, {"left": left, "top": top, "right": right, "bottom": bottom}
 
     def get_window_coords(self):
         """
@@ -127,7 +119,7 @@ def get_window_coords(self):
         elif self.window_sizes is not None:
             x_res, y_res = self.window_sizes
         else:
-            raise('in bran crunch')
+            raise ("in bran crunch")
 
         # Get window widths (both normal and edge windows)
         window_width1 = np.repeat(int(self.proc_unit_width / x_res), x_res) * self.lat_lon_sign[1]
@@ -140,20 +132,18 @@ def get_window_coords(self):
         window_height2[-1] += self.proc_unit_height - window_height1[0] * y_res * self.lat_lon_sign[0]
 
         # Get window sizes (both normal and edge windows)
-        window_bounds1 = np.flip(np.array(np.meshgrid(window_width1,
-                                                      window_height1)).T.reshape(-1, 2),
-                               axis=1).astype(np.int)
-        window_bounds2 = np.flip(np.array(np.meshgrid(window_width2,
-                                                      window_height2)).T.reshape(-1, 2),
-                               axis=1).astype(np.int)
+        window_bounds1 = np.flip(
+            np.array(np.meshgrid(window_width1, window_height1)).T.reshape(-1, 2), axis=1
+        ).astype(np.int)
+        window_bounds2 = np.flip(
+            np.array(np.meshgrid(window_width2, window_height2)).T.reshape(-1, 2), axis=1
+        ).astype(np.int)
 
-        window_idx = np.array(np.unravel_index(np.arange(y_res * x_res), (y_res, x_res), order='F'))
+        window_idx = np.array(np.unravel_index(np.arange(y_res * x_res), (y_res, x_res), order="F"))
 
         return [window_bounds1, window_bounds2], window_idx
 
-    def create_lat_lons(self,
-                        window_bounds,
-                        window_idx):
+    def create_lat_lons(self, window_bounds, window_idx):
         """
         Return bbox of window and list of latitudes and longitudes
 
@@ -162,29 +152,32 @@ def create_lat_lons(self,
         :return: list of float latitudes, list of float longitudes, list of window bbox, list of ul/br coords for window
         """
 
-        upper_left = (window_idx.T * window_bounds[0])
+        upper_left = window_idx.T * window_bounds[0]
         lower_right = upper_left + window_bounds[1]
 
         # Merge point arrays, convert back to original units, and get drawable path for each window
         bbox = np.hstack([upper_left, lower_right])
-        scaled_path_points = [np.array(np.meshgrid([st[0], st[2]], [st[1], st[3]])).T.reshape(-1, 2) for st in bbox]
+        scaled_path_points = [
+            np.array(np.meshgrid([st[0], st[2]], [st[1], st[3]])).T.reshape(-1, 2) for st in bbox
+        ]
         path_points = (scaled_path_points + self.proc_unit_bounds[0]) * self.res
 
         # Create arange of latitudes and longitudes and add half of window size
-        latitudes = np.arange(self.proc_unit_bounds[0, 0],
-                              self.proc_unit_bounds[1, 0] + self.lat_lon_sign[0],
-                              window_bounds[1][0][0])[:-1] + (window_bounds[1][0][0] / 2)
-        longitudes = np.arange(self.proc_unit_bounds[0, 1],
-                               self.proc_unit_bounds[1, 1] + self.lat_lon_sign[1],
-                               window_bounds[1][0][1])[:-1] + (window_bounds[1][0][1] / 2)
+        latitudes = np.arange(
+            self.proc_unit_bounds[0, 0],
+            self.proc_unit_bounds[1, 0] + self.lat_lon_sign[0],
+            window_bounds[1][0][0],
+        )[:-1] + (window_bounds[1][0][0] / 2)
+        longitudes = np.arange(
+            self.proc_unit_bounds[0, 1],
+            self.proc_unit_bounds[1, 1] + self.lat_lon_sign[1],
+            window_bounds[1][0][1],
+        )[:-1] + (window_bounds[1][0][1] / 2)
 
         return latitudes, longitudes, path_points, bbox
 
     @staticmethod
-    def get_window_idx(latitudes,
-                       longitudes,
-                       coords,
-                       partitions):
+    def get_window_idx(latitudes, longitudes, coords, partitions):
         """
         Return raveled window indices
 
@@ -204,14 +197,9 @@ def get_window_idx(latitudes,
         # Create mesh grid for each possible set of coords and ravel to get window idx
         grid = np.array(np.meshgrid(lat_range, lon_range)).T.reshape(-1, 2)
         del lon_range, lat_range, lat_dif, lon_dif
-        return np.ravel_multi_index([grid[:, 0], grid[:, 1]], partitions, order='F')
-
-    def read_rst_data(self,
-                      win_idx,
-                      datasets,
-                      path_points,
-                      bbox,
-                      meta):
+        return np.ravel_multi_index([grid[:, 0], grid[:, 1]], partitions, order="F")
+
+    def read_rst_data(self, win_idx, datasets, path_points, bbox, meta):
         """
         Return data windows and final bounds of window
 
@@ -226,38 +214,38 @@ def read_rst_data(self,
         """
         # Get window bounding box and get final array output dimensions
         window = path_points[win_idx]
-        window_height, window_width = np.array([np.abs(bbox[win_idx][2] - bbox[win_idx][0]),
-                                                np.abs(bbox[win_idx][3] - bbox[win_idx][1])]).astype(np.int)
+        window_height, window_width = np.array(
+            [np.abs(bbox[win_idx][2] - bbox[win_idx][0]), np.abs(bbox[win_idx][3] - bbox[win_idx][1])]
+        ).astype(np.int)
 
         bnds = []
         data = []
         for ds in datasets:
             # Get rasterio window for each pair of window bounds and depth dataset
 
-            bnd = from_bounds(window[0][1],
-                              window[-1][0],
-                              window[-1][1],
-                              window[0][0],
-                              transform=self.depth_rsts[ds].transform)
+            bnd = from_bounds(
+                window[0][1],
+                window[-1][0],
+                window[-1][1],
+                window[0][0],
+                transform=self.depth_rsts[ds].transform,
+            )
 
             bnds.append(bnd)
 
             # Read raster data with window
             read_data = self.depth_rsts[ds].read(1, window=bnd).astype(np.float32)
             # Convert all no data to nan values
-            read_data[read_data == np.float32(self.depth_rsts[ds].meta['nodata'])] = np.nan
+            read_data[read_data == np.float32(self.depth_rsts[ds].meta["nodata"])] = np.nan
             data.append(read_data)
             del bnd
 
-        final_bnds = from_bounds(window[0][1],
-                                 window[-1][0],
-                                 window[-1][1],
-                                 window[0][0],
-                                 transform=meta['transform'])
+        final_bnds = from_bounds(
+            window[0][1], window[-1][0], window[-1][1], window[0][0], transform=meta["transform"]
+        )
 
         return [final_bnds, bnds, data]
 
-
     def merge_rasters(self, out_fname, nodata=-9999, threaded=False, workers=4):
         """
         Merge multiple raster datasets
@@ -267,14 +255,12 @@ def merge_rasters(self, out_fname, nodata=-9999, threaded=False, workers=4):
         """
 
         window_bounds, window_idx = self.get_window_coords()
-        latitudes, longitudes, path_points, bbox = self.create_lat_lons(window_bounds,
-                                                                        window_idx)
-        
-        windows = [self.get_window_idx(latitudes,
-                                       longitudes,
-                                       coords,
-                                       self.partitions)
-                   for coords in self.depth_bounds]
+        latitudes, longitudes, path_points, bbox = self.create_lat_lons(window_bounds, window_idx)
+
+        windows = [
+            self.get_window_idx(latitudes, longitudes, coords, self.partitions)
+            for coords in self.depth_bounds
+        ]
 
         # Create dict with window idx key and dataset idx vals
         data_dict = {}
@@ -289,114 +275,104 @@ def merge_rasters(self, out_fname, nodata=-9999, threaded=False, workers=4):
 
         meta = self.depth_rsts[0].meta
 
-        meta.update(transform=self.proc_unit_transform,
-                    width=self.proc_unit_width,
-                    height=self.proc_unit_height,
-                    nodata=nodata,blockxsize=256,
-                    blockysize=256, tiled=True,
-                    compress='lzw')
+        meta.update(
+            transform=self.proc_unit_transform,
+            width=self.proc_unit_width,
+            height=self.proc_unit_height,
+            nodata=nodata,
+            blockxsize=256,
+            blockysize=256,
+            tiled=True,
+            compress="lzw",
+        )
 
         final_windows, data_windows, data = [], [], []
 
-        def __data_generator(data_dict,path_points,bbox,meta):
-
+        def __data_generator(data_dict, path_points, bbox, meta):
             for key, val in data_dict.items():
-
-                f_window, window, dat = self.read_rst_data(key,
-                                                           val,
-                                                           path_points,
-                                                           bbox,
-                                                           meta
-                                                          )
-                yield(dat, window, f_window, val)
-                #final_windows.append(f_window)
-                #data_windows.append(window)
-                #data.append(dat)
-                #del f_window, window, dat
+                f_window, window, dat = self.read_rst_data(key, val, path_points, bbox, meta)
+                yield (dat, window, f_window, val)
+                # final_windows.append(f_window)
+                # data_windows.append(window)
+                # data.append(dat)
+                # del f_window, window, dat
 
         # create data generator
-        dgen = __data_generator(data_dict,path_points,bbox,meta)
-        
+        dgen = __data_generator(data_dict, path_points, bbox, meta)
+
         lock = Lock()
 
-        with rasterio.open(out_fname, 'w', **meta) as rst:
+        with rasterio.open(out_fname, "w", **meta) as rst:
+            merge_partial = partial(
+                merge_data,
+                rst=rst,
+                lock=lock,
+                dtype=meta["dtype"],
+                agg_function=agg_function,
+                nodata=meta["nodata"],
+                rst_dims=self.rst_dims,
+            )
 
-            merge_partial = partial(merge_data,
-                             rst=rst,
-                             lock=lock,
-                             dtype=meta['dtype'],
-                             agg_function=agg_function,
-                             nodata=meta['nodata'],
-                             rst_dims=self.rst_dims)
-            
             if not threaded:
-                #for d, dw, fw, ddict in zip(data,
+                # for d, dw, fw, ddict in zip(data,
                 #                            data_windows,
                 #                            final_windows,
                 #                            data_dict.values()):
                 for d, dw, fw, ddict in dgen:
                     merge_partial(d, dw, fw, ddict)
             else:
-                with concurrent.futures.ThreadPoolExecutor(
-                        max_workers=workers
-                ) as executor:
-                    executor.map(merge_partial,
-                                 data,
-                                 data_windows,
-                                 final_windows,
-                                 data_dict.values()
-                                 )
-
-    def mask_mosaic(self,mosaic,polys,polys_layer=None,outfile=None):
-        
-        #rem_array,window_transform = mask(rem,[shape(huc['geometry'])],crop=True,indexes=1)
-        
+                with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+                    executor.map(merge_partial, data, data_windows, final_windows, data_dict.values())
+
+    def mask_mosaic(self, mosaic, polys, polys_layer=None, outfile=None):
+        # rem_array,window_transform = mask(rem,[shape(huc['geometry'])],crop=True,indexes=1)
+
         # input rem
-        if isinstance(mosaic,str):
+        if isinstance(mosaic, str):
             mosaic = rasterio.open(mosaic)
-        elif isinstance(mosaic,rasterio.DatasetReader):
+        elif isinstance(mosaic, rasterio.DatasetReader):
             pass
         else:
             raise TypeError("Pass rasterio dataset or filepath for mosaic")
 
-        if isinstance(polys,str):
-            polys=gpd.read_file(polys,layer=polys_layer)
-        elif isinstance(polys,gpd.GeoDataFrame):
+        if isinstance(polys, str):
+            polys = gpd.read_file(polys, layer=polys_layer)
+        elif isinstance(polys, gpd.GeoDataFrame):
             pass
         else:
             raise TypeError("Pass geopandas dataset or filepath for catchment polygons")
 
-        #fossid = huc['properties']['fossid']
-        #if polys.HydroID.dtype != 'str': polys.HydroID = polys.HydroID.astype(str)
-        #polys=polys[polys.HydroID.str.startswith(fossid)]
-        mosaic_array, window_transform = mask(mosaic,polys['geometry'],crop=True,indexes=1)
-        
+        # fossid = huc['properties']['fossid']
+        # if polys.HydroID.dtype != 'str': polys.HydroID = polys.HydroID.astype(str)
+        # polys=polys[polys.HydroID.str.startswith(fossid)]
+        mosaic_array, window_transform = mask(mosaic, polys["geometry"], crop=True, indexes=1)
+
         if outfile:
             out_profile = mosaic.profile
-            out_profile.update(height=mosaic_array.shape[0],width=mosaic_array.shape[1],
-                                      transform = window_transform, driver= 'GTiff',
-                                      blockxsize=256, blockysize=256, tiled=True, compress='lzw')
+            out_profile.update(
+                height=mosaic_array.shape[0],
+                width=mosaic_array.shape[1],
+                transform=window_transform,
+                driver="GTiff",
+                blockxsize=256,
+                blockysize=256,
+                tiled=True,
+                compress="lzw",
+            )
+
+            with rasterio.open(outfile, "w", **out_profile) as otfi:
+                otfi.write(mosaic_array, indexes=1)
+
+        return (mosaic_array, out_profile)
 
-            with rasterio.open(outfile,'w',**out_profile) as otfi:
-                otfi.write(mosaic_array,indexes=1)
-         
-        return(mosaic_array,out_profile)
 
 # Quasi multi write
 # Throughput achieved assuming processing time is not identical between windows
 # and queued datasets, preferably approx N/2 threads for 9 windows
 # @njit
-def merge_data(rst_data,
-               window_bnds,
-               final_window,
-               datasets,
-               dtype,
-               rst,
-               lock,
-               agg_function,
-               nodata,
-               rst_dims
-               ):
+def merge_data(
+    rst_data, window_bnds, final_window, datasets, dtype, rst, lock, agg_function, nodata, rst_dims
+):
     """
     Merge data in to final dataset (multi threaded)
 
@@ -418,32 +394,30 @@ def merge_data(rst_data,
     for data, bnds, idx in zip(rst_data, window_bnds, datasets):
         # Get indices to apply to base
 
-        col_slice = slice(int(np.max([0,
-                                      np.ceil(bnds.col_off * -1)])),
-                          int(np.min([bnds.width,
-                                      rst_dims[idx][1] - bnds.col_off])))
+        col_slice = slice(
+            int(np.max([0, np.ceil(bnds.col_off * -1)])),
+            int(np.min([bnds.width, rst_dims[idx][1] - bnds.col_off])),
+        )
 
-        row_slice = slice(int(np.max([0,
-                                      np.ceil(bnds.row_off * -1)])),
-                          int(np.min([bnds.height,
-                                      rst_dims[idx][0] - bnds.row_off])))
+        row_slice = slice(
+            int(np.max([0, np.ceil(bnds.row_off * -1)])),
+            int(np.min([bnds.height, rst_dims[idx][0] - bnds.row_off])),
+        )
 
         win_shape = window_data[row_slice, col_slice].shape
 
         if not np.all(np.sign(np.array(win_shape) - np.array(data.shape)) > 0):
-            data = data[:win_shape[0], :win_shape[1]]
+            data = data[: win_shape[0], : win_shape[1]]
         # Assign the data to the base array with aggregate function
-        merge = [window_data[row_slice,
-                             col_slice],
-                 data]
+        merge = [window_data[row_slice, col_slice], data]
 
         del data
-        
+
         with warnings.catch_warnings():
             # This `with` block supresses the RuntimeWarning thrown by numpy when aggregating nan values
             warnings.simplefilter("ignore", category=RuntimeWarning)
             window_data[row_slice, col_slice] = agg_function(merge)
-        
+
         window_data[np.isnan(window_data)] = nodata
         del merge
 
@@ -456,8 +430,9 @@ def merge_data(rst_data,
     del window_data
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import time
+
     # import tracemalloc
     import glob
 
@@ -472,7 +447,7 @@ def merge_data(rst_data,
     # print('end', time.localtime())
 
     # tracemalloc.start()
-    print('start', time.localtime())
+    print("start", time.localtime())
     # project_path = r'../documentation/data'
     # project_path = '*/mosaicing_data/1_fr_ms_composite'
     # overlap = OverlapWindowMerge([project_path + '/inundation_extent_12090301_FR.tif',
@@ -482,7 +457,7 @@ def merge_data(rst_data,
     # overlap.merge_rasters(project_path + '/merged_final5.tif', threaded=True, workers=4, nodata=0)
 
     # tracemalloc.start()
-    print('start', time.localtime())
+    print("start", time.localtime())
     # project_path = r'../documentation/data'
     # project_path = '*/mosaicing_data/2_gms'
     # a = glob.glob(project_path + '/inundation*.tif')
@@ -493,11 +468,10 @@ def merge_data(rst_data,
     # print(f"Current memory usage is {current / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB")
     # tracemalloc.stop()
 
-    project_path = '*'
-    overlap = OverlapWindowMerge([project_path + '/nwm_resampled.tif',
-                                  project_path + '/rnr_inundation_031403_2020092000.tif'
-                                 ],
-                                 (1, 1))
-    overlap.merge_rasters(project_path + '/merged_final5.tif', threaded=False, workers=4)
+    project_path = "*"
+    overlap = OverlapWindowMerge(
+        [project_path + "/nwm_resampled.tif", project_path + "/rnr_inundation_031403_2020092000.tif"], (1, 1)
+    )
+    overlap.merge_rasters(project_path + "/merged_final5.tif", threaded=False, workers=4)
 
-    print('end', time.localtime())
+    print("end", time.localtime())
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 534209099..574aadb10 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -3,49 +3,64 @@
 import os, re, shutil, json, sys
 import pandas as pd
 
-from tools_shared_variables import TEST_CASES_DIR, INPUTS_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT, elev_raster_ndv
+from tools_shared_variables import (
+    TEST_CASES_DIR,
+    INPUTS_DIR,
+    PREVIOUS_FIM_DIR,
+    OUTPUTS_DIR,
+    AHPS_BENCHMARK_CATEGORIES,
+    MAGNITUDE_DICT,
+    elev_raster_ndv,
+)
 from inundation import inundate
-from mosaic_inundation import Mosaic_inundation
-from inundate_gms import Inundate_gms
+from inundate_mosaic_wrapper import produce_mosaicked_inundation
 from tools_shared_functions import compute_contingency_stats_from_rasters
 from utils.shared_functions import FIM_Helpers as fh
 
 
 class benchmark(object):
-
     AHPS_BENCHMARK_CATEGORIES = AHPS_BENCHMARK_CATEGORIES
     MAGNITUDE_DICT = MAGNITUDE_DICT
-    
+
     def __init__(self, category):
         """Class that handles benchmark data.
 
-            Parameters
-            ----------
-            category : str
-                Category of the benchmark site. Should be one of ['ble', 'ifc', 'nws', 'usgs', 'ras2fim'].
+        Parameters
+        ----------
+        category : str
+            Category of the benchmark site. Should be one of ['ble', 'ifc', 'nws', 'usgs', 'ras2fim'].
         """
-        
+
         self.category = category.lower()
-        assert category in list(self.MAGNITUDE_DICT.keys()), f"Category must be one of {list(self.MAGNITUDE_DICT.keys())}"
-        self.validation_data = os.path.join(TEST_CASES_DIR, f'{self.category}_test_cases', f'validation_data_{self.category}')
-        self.is_ahps = True if self.category in self.AHPS_BENCHMARK_CATEGORIES else False
-    
+        assert category in list(
+            self.MAGNITUDE_DICT.keys()
+        ), f"Category must be one of {list(self.MAGNITUDE_DICT.keys())}"
+        self.validation_data = os.path.join(
+            TEST_CASES_DIR,
+            f"{self.category}_test_cases",
+            f"validation_data_{self.category}",
+        )
+        self.is_ahps = (
+            True if self.category in self.AHPS_BENCHMARK_CATEGORIES else False
+        )
+
     def magnitudes(self):
-        '''Returns the magnitudes associated with the benchmark category.'''
+        """Returns the magnitudes associated with the benchmark category."""
         return self.MAGNITUDE_DICT[self.category]
-    
+
     def huc_data(self):
-        '''Returns a dict of HUC8, magnitudes, and sites.'''
+        """Returns a dict of HUC8, magnitudes, and sites."""
         huc_mags = {}
         for huc in os.listdir(self.validation_data):
-            if not re.match('\d{8}', huc): continue
+            if not re.match("\d{8}", huc):
+                continue
             huc_mags[huc] = self.data(huc)
         return huc_mags
-    
+
     def data(self, huc):
-        '''Returns a dict of magnitudes and sites for a given huc. Sites will be AHPS lids for 
-           AHPS sites and empty strings for non-AHPS sites.
-        '''
+        """Returns a dict of magnitudes and sites for a given huc. Sites will be AHPS lids for
+        AHPS sites and empty strings for non-AHPS sites.
+        """
         huc_dir = os.path.join(self.validation_data, huc)
         if not os.path.isdir(huc_dir):
             return {}
@@ -55,7 +70,9 @@ def data(self, huc):
             mag_dict = {}
             for lid in lids:
                 lid_dir = os.path.join(huc_dir, lid)
-                for mag in [file for file in os.listdir(lid_dir) if file in self.magnitudes()]:
+                for mag in [
+                    file for file in os.listdir(lid_dir) if file in self.magnitudes()
+                ]:
                     if mag in mag_dict:
                         mag_dict[mag].append(lid)
                     else:
@@ -63,148 +80,189 @@ def data(self, huc):
             return mag_dict
         else:
             mags = list(os.listdir(huc_dir))
-            return {mag:[''] for mag in mags}
+            return {mag: [""] for mag in mags}
 
-class test_case(benchmark):
 
+class test_case(benchmark):
     def __init__(self, test_id, version, archive=True):
         """Class that handles test cases, specifically running the alpha test.
 
-            Parameters
-            ----------
-            test_id : str
-                ID of the test case in huc8_category format, e.g. `12090201_ble`.
-            version : str
-                Version of FIM to which this test_case belongs. This should correspond to the fim directory
-                name in either `/data/previous_fim/` or `/outputs/`.
-            archive : bool
-                If true, this test case outputs will be placed into the `official_versions` folder
-                and the FIM model will be read from the `/data/previous_fim` folder.
-                If false, it will be saved to the `testing_versions/` folder and the FIM model 
-                will be read from the `/outputs/` folder.
-        
+        Parameters
+        ----------
+        test_id : str
+            ID of the test case in huc8_category format, e.g. `12090201_ble`.
+        version : str
+            Version of FIM to which this test_case belongs. This should correspond to the fim directory
+            name in either `/data/previous_fim/` or `/outputs/`.
+        archive : bool
+            If true, this test case outputs will be placed into the `official_versions` folder
+            and the FIM model will be read from the `/data/previous_fim` folder.
+            If false, it will be saved to the `testing_versions/` folder and the FIM model
+            will be read from the `/outputs/` folder.
+
         """
         self.test_id = test_id
-        self.huc, self.benchmark_cat = test_id.split('_')
+        self.huc, self.benchmark_cat = test_id.split("_")
         super().__init__(self.benchmark_cat)
         self.version = version
         self.archive = archive
         # FIM run directory path - uses HUC 6 for FIM 1 & 2
-        self.fim_dir = os.path.join(PREVIOUS_FIM_DIR if archive else OUTPUTS_DIR, 
-                                    self.version, 
-                                    self.huc if not re.search('^fim_[1,2]', version, re.IGNORECASE) else self.huc[:6])
+        self.fim_dir = os.path.join(
+            PREVIOUS_FIM_DIR if archive else OUTPUTS_DIR,
+            self.version,
+            self.huc
+            if not re.search("^fim_[1,2]", version, re.IGNORECASE)
+            else self.huc[:6],
+        )
         # Test case directory path
-        self.dir = os.path.join(TEST_CASES_DIR, f'{self.benchmark_cat}_test_cases', 
-                                test_id,
-                                'official_versions' if archive else 'testing_versions',
-                                version)
+        self.dir = os.path.join(
+            TEST_CASES_DIR,
+            f"{self.benchmark_cat}_test_cases",
+            test_id,
+            "official_versions" if archive else "testing_versions",
+            version,
+        )
         if not os.path.exists(self.dir):
             os.makedirs(self.dir)
         # Benchmark data path
         self.benchmark_dir = os.path.join(self.validation_data, self.huc)
 
         # Create list of shapefile paths to use as exclusion areas.
-        self.mask_dict = {'levees':
-                        {'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg',
-                        'buffer': None,
-                        'operation': 'exclude'
-                        },
-                    'waterbodies':
-                        {'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg',
-                        'buffer': None,
-                        'operation': 'exclude',
-                        },
-                    }
+        self.mask_dict = {
+            "levees": {
+                "path": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
+                "buffer": None,
+                "operation": "exclude",
+            },
+            "waterbodies": {
+                "path": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
+                "buffer": None,
+                "operation": "exclude",
+            },
+        }
 
     @classmethod
     def list_all_test_cases(cls, version, archive, benchmark_categories=[]):
         """Returns a complete list of all benchmark category test cases as classes.
 
-            Parameters
-            ----------
-            version : str
-                Version of FIM to which this test_case belongs. This should correspond to the fim directory
-                name in either `/data/previous_fim/` or `/outputs/`.
-            archive : bool
-                If true, this test case outputs will be placed into the `official_versions` folder
-                and the FIM model will be read from the `/data/previous_fim` folder.
-                If false, it will be saved to the `testing_versions/` folder and the FIM model 
-                will be read from the `/outputs/` folder.
+        Parameters
+        ----------
+        version : str
+            Version of FIM to which this test_case belongs. This should correspond to the fim directory
+            name in either `/data/previous_fim/` or `/outputs/`.
+        archive : bool
+            If true, this test case outputs will be placed into the `official_versions` folder
+            and the FIM model will be read from the `/data/previous_fim` folder.
+            If false, it will be saved to the `testing_versions/` folder and the FIM model
+            will be read from the `/outputs/` folder.
         """
         if not benchmark_categories:
             benchmark_categories = list(cls.MAGNITUDE_DICT.keys())
-            
+
         test_case_list = []
         for bench_cat in benchmark_categories:
-
             benchmark_class = benchmark(bench_cat)
             benchmark_data = benchmark_class.huc_data()
 
             for huc in benchmark_data.keys():
-                test_case_list.append(cls(f'{huc}_{bench_cat}', version, archive))
+                test_case_list.append(cls(f"{huc}_{bench_cat}", version, archive))
 
         return test_case_list
 
-    def alpha_test(self, calibrated=False, model='', mask_type='huc', inclusion_area='',
-                inclusion_area_buffer=0, overwrite=True, verbose=False, gms_workers=1):
-        '''Compares a FIM directory with benchmark data from a variety of sources.
-        
-            Parameters
-            ----------
-            calibrated : bool
-                Whether or not this FIM version is calibrated.
-            model : str
-                MS or FR extent of the model. This value will be written to the eval_metadata.json.
-            mask_type : str
-                Mask type to feed into inundation.py.
-            inclusion_area : int
-                Area to include in agreement analysis.
-            inclusion_area_buffer : int
-                Buffer distance in meters to include outside of the model's domain.
-            overwrite : bool
-                If True, overwites pre-existing test cases within the test_cases directory.
-            verbose : bool
-                If True, prints out all pertinent data.
-            gms_workers : int
-                Number of worker processes assigned to GMS processing.
-        '''
-        
+    def alpha_test(
+        self,
+        calibrated=False,
+        model="",
+        mask_type="huc",
+        inclusion_area="",
+        inclusion_area_buffer=0,
+        overwrite=True,
+        verbose=False,
+        gms_workers=1,
+    ):
+        """Compares a FIM directory with benchmark data from a variety of sources.
+
+        Parameters
+        ----------
+        calibrated : bool
+            Whether or not this FIM version is calibrated.
+        model : str
+            MS or FR extent of the model. This value will be written to the eval_metadata.json.
+        mask_type : str
+            Mask type to feed into inundation.py.
+        inclusion_area : int
+            Area to include in agreement analysis.
+        inclusion_area_buffer : int
+            Buffer distance in meters to include outside of the model's domain.
+        overwrite : bool
+            If True, overwites pre-existing test cases within the test_cases directory.
+        verbose : bool
+            If True, prints out all pertinent data.
+        gms_workers : int
+            Number of worker processes assigned to GMS processing.
+        """
+
         try:
             if not overwrite and os.path.isdir(self.dir):
-                print(f"Metrics for {self.dir} already exist. Use overwrite flag (-o) to overwrite metrics.")
+                print(
+                    f"Metrics for {self.dir} already exist. Use overwrite flag (-o) to overwrite metrics."
+                )
                 return
 
             fh.vprint(f"Starting alpha test for {self.dir}", verbose)
-            
-            self.stats_modes_list = ['total_area']
+
+            self.stats_modes_list = ["total_area"]
 
             # Create paths to fim_run outputs for use in inundate()
-            if model != 'GMS':
-                self.rem = os.path.join(self.fim_dir, 'rem_zeroed_masked.tif')
+            if model != "GMS":
+                self.rem = os.path.join(self.fim_dir, "rem_zeroed_masked.tif")
                 if not os.path.exists(self.rem):
-                    self.rem = os.path.join(self.fim_dir, 'rem_clipped_zeroed_masked.tif')
-                self.catchments = os.path.join(self.fim_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+                    self.rem = os.path.join(
+                        self.fim_dir, "rem_clipped_zeroed_masked.tif"
+                    )
+                self.catchments = os.path.join(
+                    self.fim_dir, "gw_catchments_reaches_filtered_addedAttributes.tif"
+                )
                 if not os.path.exists(self.catchments):
-                    self.catchments = os.path.join(self.fim_dir, 'gw_catchments_reaches_clipped_addedAttributes.tif')
+                    self.catchments = os.path.join(
+                        self.fim_dir,
+                        "gw_catchments_reaches_clipped_addedAttributes.tif",
+                    )
                 self.mask_type = mask_type
-                if mask_type == 'huc':
-                    self.catchment_poly = ''
+                if mask_type == "huc":
+                    self.catchment_poly = ""
                 else:
-                    self.catchment_poly = os.path.join(self.fim_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-                self.hydro_table = os.path.join(self.fim_dir, 'hydroTable.csv')
+                    self.catchment_poly = os.path.join(
+                        self.fim_dir,
+                        "gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg",
+                    )
+                self.hydro_table = os.path.join(self.fim_dir, "hydroTable.csv")
 
             # Map necessary inputs for inundate().
-            self.hucs, self.hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
-
-            if inclusion_area != '':
-                inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
-                self.mask_dict.update({inclusion_area_name: {'path': inclusion_area,
-                                                        'buffer': int(inclusion_area_buffer),
-                                                        'operation': 'include'}})
+            self.hucs, self.hucs_layerName = (
+                os.path.join(INPUTS_DIR, "wbd", "WBD_National.gpkg"),
+                "WBDHU8",
+            )
+
+            if inclusion_area != "":
+                inclusion_area_name = os.path.split(inclusion_area)[1].split(".")[
+                    0
+                ]  # Get layer name
+                self.mask_dict.update(
+                    {
+                        inclusion_area_name: {
+                            "path": inclusion_area,
+                            "buffer": int(inclusion_area_buffer),
+                            "operation": "include",
+                        }
+                    }
+                )
                 # Append the concatenated inclusion_area_name and buffer.
                 if inclusion_area_buffer == None:
                     inclusion_area_buffer = 0
-                self.stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm')
+                self.stats_modes_list.append(
+                    inclusion_area_name + "_b" + str(inclusion_area_buffer) + "m"
+                )
 
             # Delete the directory if it exists
             if os.path.exists(self.dir):
@@ -214,9 +272,17 @@ def alpha_test(self, calibrated=False, model='', mask_type='huc', inclusion_area
             # Get the magnitudes and lids for the current huc and loop through them
             validation_data = self.data(self.huc)
             for magnitude in validation_data:
-                for instance in validation_data[magnitude]:      # instance will be the lid for AHPS sites and '' for other sites
+                for instance in validation_data[
+                    magnitude
+                ]:  # instance will be the lid for AHPS sites and '' for other sites
                     # For each site, inundate the REM and compute aggreement raster with stats
-                    self._inundate_and_compute(magnitude, instance, model=model, verbose=verbose, gms_workers=gms_workers)
+                    self._inundate_and_compute(
+                        magnitude,
+                        instance,
+                        model=model,
+                        verbose=verbose,
+                        gms_workers=gms_workers,
+                    )
 
                 # Clean up 'total_area' outputs from AHPS sites
                 if self.is_ahps:
@@ -231,149 +297,176 @@ def alpha_test(self, calibrated=False, model='', mask_type='huc', inclusion_area
         except Exception as ex:
             print(ex)
             sys.exit(1)
-        
-    def _inundate_and_compute(self, 
-                              magnitude, 
-                              lid, 
-                              compute_only = False, 
-                              model = '',
-                              verbose = False,
-                              gms_workers = 1):
-        '''Method for inundating and computing contingency rasters as part of the alpha_test.
-           Used by both the alpha_test() and composite() methods.
-
-            Parameters
-            ----------
-            magnitude : str
-                Magnitude of the current benchmark site.
-            lid : str
-                lid of the current benchmark site. For non-AHPS sites, this should be an empty string ('').
-            compute_only : bool
-                If true, skips inundation and only computes contingency stats.
-        '''
+
+    def _inundate_and_compute(
+        self, magnitude, lid, compute_only=False, model="", verbose=False, gms_workers=1
+    ):
+        """Method for inundating and computing contingency rasters as part of the alpha_test.
+        Used by both the alpha_test() and composite() methods.
+
+         Parameters
+         ----------
+         magnitude : str
+             Magnitude of the current benchmark site.
+         lid : str
+             lid of the current benchmark site. For non-AHPS sites, this should be an empty string ('').
+         compute_only : bool
+             If true, skips inundation and only computes contingency stats.
+        """
         # Output files
         fh.vprint("Creating output files", verbose)
 
-        test_case_out_dir     = os.path.join(self.dir, magnitude)
-        inundation_prefix     = lid + '_' if lid else ''
-        inundation_path       = os.path.join(test_case_out_dir, f'{inundation_prefix}inundation_extent.tif')
-        predicted_raster_path = inundation_path.replace('.tif', f'_{self.huc}.tif')
-        agreement_raster      = os.path.join(test_case_out_dir, (f'ahps_{lid}' if lid else '') +'total_area_agreement.tif')
-        stats_json            = os.path.join(test_case_out_dir, 'stats.json')
-        stats_csv             = os.path.join(test_case_out_dir, 'stats.csv')
+        test_case_out_dir = os.path.join(self.dir, magnitude)
+        inundation_prefix = lid + "_" if lid else ""
+        inundation_path = os.path.join(
+            test_case_out_dir, f"{inundation_prefix}inundation_extent.tif"
+        )
+        predicted_raster_path = inundation_path.replace(".tif", f"_{self.huc}.tif")
+        agreement_raster = os.path.join(
+            test_case_out_dir,
+            (f"ahps_{lid}" if lid else "") + "total_area_agreement.tif",
+        )
+        stats_json = os.path.join(test_case_out_dir, "stats.json")
+        stats_csv = os.path.join(test_case_out_dir, "stats.csv")
 
         # Create directory
         if not os.path.isdir(test_case_out_dir):
             os.mkdir(test_case_out_dir)
 
         # Benchmark raster and flow files
-        benchmark_rast = (f'ahps_{lid}' if lid else self.benchmark_cat) + f'_huc_{self.huc}_extent_{magnitude}.tif'
-        benchmark_rast = os.path.join(self.benchmark_dir, lid, magnitude, benchmark_rast)
-        benchmark_flows = benchmark_rast.replace(f'_extent_{magnitude}.tif', f'_flows_{magnitude}.csv')
+        benchmark_rast = (
+            f"ahps_{lid}" if lid else self.benchmark_cat
+        ) + f"_huc_{self.huc}_extent_{magnitude}.tif"
+        benchmark_rast = os.path.join(
+            self.benchmark_dir, lid, magnitude, benchmark_rast
+        )
+        benchmark_flows = benchmark_rast.replace(
+            f"_extent_{magnitude}.tif", f"_flows_{magnitude}.csv"
+        )
         mask_dict_indiv = self.mask_dict.copy()
-        if self.is_ahps: # add domain shapefile to mask for AHPS sites
-            domain = os.path.join(self.benchmark_dir, lid, f'{lid}_domain.shp')
-            mask_dict_indiv.update({lid:
-                            {'path': domain,
-                            'buffer': None,
-                            'operation': 'include'}
-                                })
+        if self.is_ahps:  # add domain shapefile to mask for AHPS sites
+            domain = os.path.join(self.benchmark_dir, lid, f"{lid}_domain.shp")
+            mask_dict_indiv.update(
+                {lid: {"path": domain, "buffer": None, "operation": "include"}}
+            )
         # Check to make sure all relevant files exist
-        if not os.path.isfile(benchmark_rast) or not os.path.isfile(benchmark_flows) or (self.is_ahps and not os.path.isfile(domain)):
+        if (
+            not os.path.isfile(benchmark_rast)
+            or not os.path.isfile(benchmark_flows)
+            or (self.is_ahps and not os.path.isfile(domain))
+        ):
             return -1
 
         # Inundate REM
-        if not compute_only:             # composite alpha tests don't need to be inundated
-            if model == 'GMS':
-                fh.vprint("Begin FIM4 Inundation", verbose)
-                map_file = Inundate_gms( hydrofabric_dir = os.path.dirname(self.fim_dir), 
-                                         forecast = benchmark_flows, 
-                                         num_workers = gms_workers,
-                                         hucs = self.huc,
-                                         inundation_raster = predicted_raster_path,
-                                         inundation_polygon = None,
-                                         depths_raster = None,
-                                         verbose = verbose,
-                                         log_file = None,
-                                         output_fileNames = None )
-                #if (len(map_file) > 0):
-                fh.vprint("Begin FIM4 Mosaic", verbose)
-                Mosaic_inundation( map_file,
-                                    mosaic_attribute = 'inundation_rasters',
-                                    mosaic_output = predicted_raster_path,
-                                    mask = os.path.join(self.fim_dir,'wbd.gpkg'),
-                                    unit_attribute_name = 'huc8',
-                                    nodata = elev_raster_ndv,
-                                    workers = 1,
-                                    remove_inputs = True,
-                                    subset = None,
-                                    verbose = verbose )
+        if not compute_only:  # composite alpha tests don't need to be inundated
+            if model == "GMS":
+                produce_mosaicked_inundation(
+                    os.path.dirname(self.fim_dir),
+                    self.huc,
+                    benchmark_flows,
+                    inundation_raster=predicted_raster_path,
+                    mask=os.path.join(self.fim_dir, "wbd.gpkg"),
+                    verbose=verbose,
+                )
+
             # FIM v3 and before
             else:
                 fh.vprint("Begin FIM v3 (or earlier) Inundation", verbose)
-                inundate_result = inundate(self.rem,
-                                           self.catchments,
-                                           self.catchment_poly,
-                                           self.hydro_table,
-                                           benchmark_flows,
-                                           self.mask_type,
-                                           hucs=self.hucs,
-                                           hucs_layerName=self.hucs_layerName,
-                                           subset_hucs=self.huc,
-                                           num_workers=1,
-                                           aggregate=False,
-                                           inundation_raster=inundation_path,
-                                           inundation_polygon=None,
-                                           depths=None,out_raster_profile=None,
-                                           out_vector_profile=None,
-                                           quiet=True)
+                inundate_result = inundate(
+                    self.rem,
+                    self.catchments,
+                    self.catchment_poly,
+                    self.hydro_table,
+                    benchmark_flows,
+                    self.mask_type,
+                    hucs=self.hucs,
+                    hucs_layerName=self.hucs_layerName,
+                    subset_hucs=self.huc,
+                    num_workers=1,
+                    aggregate=False,
+                    inundation_raster=inundation_path,
+                    inundation_polygon=None,
+                    depths=None,
+                    out_raster_profile=None,
+                    out_vector_profile=None,
+                    quiet=True,
+                )
                 if inundate_result != 0:
                     return inundate_result
 
         # Create contingency rasters and stats
         fh.vprint("Begin creating contingency rasters and stats", verbose)
         if os.path.isfile(predicted_raster_path):
-            compute_contingency_stats_from_rasters(predicted_raster_path,
-                                                   benchmark_rast,
-                                                   agreement_raster,
-                                                   stats_csv=stats_csv,
-                                                   stats_json=stats_json,
-                                                   mask_dict=mask_dict_indiv)
+            compute_contingency_stats_from_rasters(
+                predicted_raster_path,
+                benchmark_rast,
+                agreement_raster,
+                stats_csv=stats_csv,
+                stats_json=stats_json,
+                mask_dict=mask_dict_indiv,
+            )
         return
 
-
     @classmethod
-    def run_alpha_test(cls, version, test_id, magnitude, calibrated, model, archive_results=False, 
-                       mask_type='huc', inclusion_area='', inclusion_area_buffer=0, light_run=False, overwrite=True, verbose=False, gms_workers=1):
-        '''Class method for instantiating the test_case class and running alpha_test directly'''
+    def run_alpha_test(
+        cls,
+        version,
+        test_id,
+        magnitude,
+        calibrated,
+        model,
+        archive_results=False,
+        mask_type="huc",
+        inclusion_area="",
+        inclusion_area_buffer=0,
+        light_run=False,
+        overwrite=True,
+        verbose=False,
+        gms_workers=1,
+    ):
+        """Class method for instantiating the test_case class and running alpha_test directly"""
 
         alpha_class = cls(test_id, version, archive_results)
-        alpha_class.alpha_test(calibrated, model, mask_type, inclusion_area,
-                inclusion_area_buffer, overwrite, verbose, gms_workers)
-
-    def composite(self, version_2, calibrated = False, overwrite = True, verbose = False):
-        '''Class method for compositing MS and FR inundation and creating an agreement raster with stats
-
-            Parameters
-            ----------
-            version_2 : str
-                Version with which to composite.
-            calibrated : bool
-                Whether or not this FIM version is calibrated.
-            overwrite : bool
-                If True, overwites pre-existing test cases within the test_cases directory.
-        '''
-
-        if re.match(r'(.*)(_ms|_fr)', self.version):
-            composite_version_name = re.sub(r'(.*)(_ms|_fr)', r'\1_comp', self.version, count=1)
+        alpha_class.alpha_test(
+            calibrated,
+            model,
+            mask_type,
+            inclusion_area,
+            inclusion_area_buffer,
+            overwrite,
+            verbose,
+            gms_workers,
+        )
+
+    def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
+        """Class method for compositing MS and FR inundation and creating an agreement raster with stats
+
+        Parameters
+        ----------
+        version_2 : str
+            Version with which to composite.
+        calibrated : bool
+            Whether or not this FIM version is calibrated.
+        overwrite : bool
+            If True, overwites pre-existing test cases within the test_cases directory.
+        """
+
+        if re.match(r"(.*)(_ms|_fr)", self.version):
+            composite_version_name = re.sub(
+                r"(.*)(_ms|_fr)", r"\1_comp", self.version, count=1
+            )
         else:
-            composite_version_name = re.sub(r'(.*)(_ms|_fr)', r'\1_comp', version_2, count=1)
+            composite_version_name = re.sub(
+                r"(.*)(_ms|_fr)", r"\1_comp", version_2, count=1
+            )
 
         fh.vprint(f"Begin composite for version : {composite_version_name}", verbose)
 
-        composite_test_case = test_case(self.test_id, composite_version_name, self.archive)
+        composite_test_case = test_case(
+            self.test_id, composite_version_name, self.archive
+        )
         input_test_case_2 = test_case(self.test_id, version_2, self.archive)
-        composite_test_case.stats_modes_list = ['total_area']
+        composite_test_case.stats_modes_list = ["total_area"]
 
         if not overwrite and os.path.isdir(composite_test_case.dir):
             return
@@ -384,66 +477,113 @@ def composite(self, version_2, calibrated = False, overwrite = True, verbose = F
 
         validation_data = composite_test_case.data(composite_test_case.huc)
         for magnitude in validation_data:
-            for instance in validation_data[magnitude]:      # instance will be the lid for AHPS sites and '' for other sites (ble/ifc/ras2fim)
-                inundation_prefix = instance + '_' if instance else ''
-
-                input_inundation = os.path.join(self.dir, magnitude, f'{inundation_prefix}inundation_extent_{self.huc}.tif')
-                input_inundation_2 = os.path.join(input_test_case_2.dir, magnitude, f'{inundation_prefix}inundation_extent_{input_test_case_2.huc}.tif')
-                output_inundation = os.path.join(composite_test_case.dir, magnitude, f'{inundation_prefix}inundation_extent.tif')
-
-                if os.path.isfile(input_inundation) and os.path.isfile(input_inundation_2):
-                    inundation_map_file = pd.DataFrame({ 
-                                            'huc8' : [composite_test_case.huc] * 2,
-                                            'branchID' : [None] * 2,
-                                            'inundation_rasters' : [input_inundation,input_inundation_2],
-                                            'depths_rasters' : [None] * 2,
-                                            'inundation_polygons' : [None] * 2
-                                        })
+            for instance in validation_data[
+                magnitude
+            ]:  # instance will be the lid for AHPS sites and '' for other sites (ble/ifc/ras2fim)
+                inundation_prefix = instance + "_" if instance else ""
+
+                input_inundation = os.path.join(
+                    self.dir,
+                    magnitude,
+                    f"{inundation_prefix}inundation_extent_{self.huc}.tif",
+                )
+                input_inundation_2 = os.path.join(
+                    input_test_case_2.dir,
+                    magnitude,
+                    f"{inundation_prefix}inundation_extent_{input_test_case_2.huc}.tif",
+                )
+                output_inundation = os.path.join(
+                    composite_test_case.dir,
+                    magnitude,
+                    f"{inundation_prefix}inundation_extent.tif",
+                )
+
+                if os.path.isfile(input_inundation) and os.path.isfile(
+                    input_inundation_2
+                ):
+                    inundation_map_file = pd.DataFrame(
+                        {
+                            "huc8": [composite_test_case.huc] * 2,
+                            "branchID": [None] * 2,
+                            "inundation_rasters": [
+                                input_inundation,
+                                input_inundation_2,
+                            ],
+                            "depths_rasters": [None] * 2,
+                            "inundation_polygons": [None] * 2,
+                        }
+                    )
                     os.makedirs(os.path.dirname(output_inundation), exist_ok=True)
 
-                    fh.vprint(f"Begin mosaic inundation for version : {composite_version_name}", verbose)
-                    Mosaic_inundation(inundation_map_file,mosaic_attribute='inundation_rasters',
-                                            mosaic_output=output_inundation, mask=None, unit_attribute_name='huc8',
-                                            nodata=elev_raster_ndv, workers=1, remove_inputs=False, subset=None, verbose=False
-                                            )
-                    composite_test_case._inundate_and_compute(magnitude, instance, compute_only=True)
+                    fh.vprint(
+                        f"Begin mosaic inundation for version : {composite_version_name}",
+                        verbose,
+                    )
+                    Mosaic_inundation(
+                        inundation_map_file,
+                        mosaic_attribute="inundation_rasters",
+                        mosaic_output=output_inundation,
+                        mask=None,
+                        unit_attribute_name="huc8",
+                        nodata=elev_raster_ndv,
+                        workers=1,
+                        remove_inputs=False,
+                        subset=None,
+                        verbose=False,
+                    )
+                    composite_test_case._inundate_and_compute(
+                        magnitude, instance, compute_only=True
+                    )
 
-                elif os.path.isfile(input_inundation) or os.path.isfile(input_inundation_2): 
+                elif os.path.isfile(input_inundation) or os.path.isfile(
+                    input_inundation_2
+                ):
                     # If only one model (MS or FR) has inundation, simply copy over all files as the composite
-                    single_test_case = self if os.path.isfile(input_inundation) else input_test_case_2
-                    shutil.copytree(single_test_case.dir, re.sub(r'(.*)(_ms|_fr)', r'\1_comp', single_test_case.dir, count=1))
-                    composite_test_case.write_metadata(calibrated, 'COMP')
+                    single_test_case = (
+                        self if os.path.isfile(input_inundation) else input_test_case_2
+                    )
+                    shutil.copytree(
+                        single_test_case.dir,
+                        re.sub(
+                            r"(.*)(_ms|_fr)", r"\1_comp", single_test_case.dir, count=1
+                        ),
+                    )
+                    composite_test_case.write_metadata(calibrated, "COMP")
                     return
 
             # Clean up 'total_area' outputs from AHPS sites
             if composite_test_case.is_ahps:
-                composite_test_case.clean_ahps_outputs(os.path.join(composite_test_case.dir, magnitude))
+                composite_test_case.clean_ahps_outputs(
+                    os.path.join(composite_test_case.dir, magnitude)
+                )
 
-        composite_test_case.write_metadata(calibrated, 'COMP')
+        composite_test_case.write_metadata(calibrated, "COMP")
 
     def write_metadata(self, calibrated, model):
-        '''Writes metadata files for a test_case directory.'''
-        with open(os.path.join(self.dir,'eval_metadata.json'),'w') as meta:
-            eval_meta = { 'calibrated' : calibrated , 'model' : model }
-            meta.write( 
-                        json.dumps(eval_meta,indent=2) 
-                    )
+        """Writes metadata files for a test_case directory."""
+        with open(os.path.join(self.dir, "eval_metadata.json"), "w") as meta:
+            eval_meta = {"calibrated": calibrated, "model": model}
+            meta.write(json.dumps(eval_meta, indent=2))
+
     def clean_ahps_outputs(self, magnitude_directory):
-        '''Cleans up `total_area` files from an input AHPS magnitude directory.'''
-        output_file_list = [os.path.join(magnitude_directory, of) for of in os.listdir(magnitude_directory)]
+        """Cleans up `total_area` files from an input AHPS magnitude directory."""
+        output_file_list = [
+            os.path.join(magnitude_directory, of)
+            for of in os.listdir(magnitude_directory)
+        ]
         for output_file in output_file_list:
             if "total_area" in output_file:
                 os.remove(output_file)
 
     def get_current_agreements(self):
-        '''Returns a list of all agreement rasters currently existing for the test_case.'''
+        """Returns a list of all agreement rasters currently existing for the test_case."""
         agreement_list = []
         for mag in os.listdir(self.dir):
             mag_dir = os.path.join(self.dir, mag)
-            if not os.path.isdir(mag_dir): continue
+            if not os.path.isdir(mag_dir):
+                continue
 
             for f in os.listdir(mag_dir):
-                if 'agreement.tif' in f:
+                if "agreement.tif" in f:
                     agreement_list.append(os.path.join(mag_dir, f))
         return agreement_list
-
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index 4c837e094..a257b827e 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -1,76 +1,133 @@
 import os
 
 # Environmental variables and constants.
-TEST_CASES_DIR = r'/data/test_cases/'
-PREVIOUS_FIM_DIR = r'/data/previous_fim'
-OUTPUTS_DIR = os.environ['outputsDir']
-INPUTS_DIR = r'/data/inputs'
-AHPS_BENCHMARK_CATEGORIES = ['usgs', 'nws']
-FR_BENCHMARK_CATEGORIES = ['ble', 'ifc']
-BLE_MAGNITUDE_LIST = ['100yr', '500yr']
-IFC_MAGNITUDE_LIST = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr', '200yr', '500yr']
-AHPS_MAGNITUDE_LIST = ['action', 'minor', 'moderate', 'major']
-RAS2FIM_MAGNITUDE_LIST = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr']
+TEST_CASES_DIR = r"/data/test_cases/"
+PREVIOUS_FIM_DIR = r"/data/previous_fim"
+OUTPUTS_DIR = os.environ["outputsDir"]
+INPUTS_DIR = os.environ["inputsDir"]
+AHPS_BENCHMARK_CATEGORIES = ["usgs", "nws"]
+FR_BENCHMARK_CATEGORIES = ["ble", "ifc"]
+BLE_MAGNITUDE_LIST = ["100yr", "500yr"]
+IFC_MAGNITUDE_LIST = ["2yr", "5yr", "10yr", "25yr", "50yr", "100yr", "200yr", "500yr"]
+AHPS_MAGNITUDE_LIST = ["action", "minor", "moderate", "major"]
+RAS2FIM_MAGNITUDE_LIST = ["2yr", "5yr", "10yr", "25yr", "50yr", "100yr"]
 
-MAGNITUDE_DICT = {'ble': BLE_MAGNITUDE_LIST, 'ifc': IFC_MAGNITUDE_LIST, 'usgs': AHPS_MAGNITUDE_LIST, 'nws': AHPS_MAGNITUDE_LIST, 'ras2fim': RAS2FIM_MAGNITUDE_LIST}
-PRINTWORTHY_STATS = ['CSI', 'TPR', 'TNR', 'FAR', 'MCC', 'TP_area_km2', 'FP_area_km2', 'TN_area_km2', 'FN_area_km2', 'contingency_tot_area_km2', 'TP_perc', 'FP_perc', 'TN_perc', 'FN_perc']
-GO_UP_STATS = ['CSI', 'TPR', 'MCC', 'TN_area_km2', 'TP_area_km2', 'TN_perc', 'TP_perc', 'TNR']
-GO_DOWN_STATS = ['FAR', 'FN_area_km2', 'FP_area_km2', 'FP_perc', 'FN_perc', 'PND']
+MAGNITUDE_DICT = {
+    "ble": BLE_MAGNITUDE_LIST,
+    "ifc": IFC_MAGNITUDE_LIST,
+    "usgs": AHPS_MAGNITUDE_LIST,
+    "nws": AHPS_MAGNITUDE_LIST,
+    "ras2fim": RAS2FIM_MAGNITUDE_LIST,
+}
+PRINTWORTHY_STATS = [
+    "CSI",
+    "TPR",
+    "TNR",
+    "FAR",
+    "MCC",
+    "TP_area_km2",
+    "FP_area_km2",
+    "TN_area_km2",
+    "FN_area_km2",
+    "contingency_tot_area_km2",
+    "TP_perc",
+    "FP_perc",
+    "TN_perc",
+    "FN_perc",
+]
+GO_UP_STATS = [
+    "CSI",
+    "TPR",
+    "MCC",
+    "TN_area_km2",
+    "TP_area_km2",
+    "TN_perc",
+    "TP_perc",
+    "TNR",
+]
+GO_DOWN_STATS = ["FAR", "FN_area_km2", "FP_area_km2", "FP_perc", "FN_perc", "PND"]
 
 # Variables for eval_plots.py
 BAD_SITES = [
-            'baki3', #USGS: ratio of evaluated vs domain is very low
-            'cpei3', #USGS: mainstems does not extend sufficiently upstream (~35%), significant masking upstream
-            'eagi1', #NWS:  ratio of evaluated vs domain is very low
-            'efdn7', #NWS:  mainstems does not extend sufficiently upstream (~30%)
-            'erwn6', #NWS:  ratio of evaluated vs domain is very low
-            'grfi2', #USGS: incorrect location
-            'hohn4', #Both: no mainstems in vicinity
-            'kcdm7', #USGS: incorrect location
-            'kilo1', #Both: mainstems does not extend sufficiently upstream (~20%)
-            'ksdm7', #USGS: masked
-            'levk1', #NWS:  Incorrect feature_id assigned from WRDS, this has been corrected
-            'lkcm7', #NWS:  masked 
-            'loun7', #NWS:  benchmark is not consistent between thresholds
-            'lrlm7', #NWS:  masked
-            'mcri2', #USGS: incorrect location
-            'monv1', #NWS:  mainstems does not extend sufficiently upstream (~30%)
-            'mtao1', #USGS: ratio of evaluated vs domain is very low
-            'nhso1', #USGS: mainstems does not extend sufficiently upstream (45%)
-            'nmso1', #Both: mainstems does not extend sufficiently upstream
-            'pori3', #USGS: mainstems does not extend sufficiently upstream
-            'ptvn6', #Both: mainstems does not extend sufficiently upstream (50%)
-            'roun6', #USGS: ratio of evaluated vs domain is very low
-            'rwdn4', #Both: no mainstems in vicinity
-            'selt2', #NWS:  mainstems does not extend sufficiently upstream (~30%)
-            'sweg1', #Both: mainstems does not extend sufficiently upstream (~30%)
-            'vcni3', #USGS: ratio of evaluated vs domain is very low
-            'watw3', #NWS:  mainstems does not extend sufficiently upstream (~30%)
-            'weat2', #NWS:  mainstems does not extend sufficiently upstream (~50%)
-            'wkew3' #NWS:  mainstems does not extend sufficiently upstream (~45%)    
-            ]
+    "baki3",  # USGS: ratio of evaluated vs domain is very low
+    "cpei3",  # USGS: mainstems does not extend sufficiently upstream (~35%), significant masking upstream
+    "eagi1",  # NWS:  ratio of evaluated vs domain is very low
+    "efdn7",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
+    "erwn6",  # NWS:  ratio of evaluated vs domain is very low
+    "grfi2",  # USGS: incorrect location
+    "hohn4",  # Both: no mainstems in vicinity
+    "kcdm7",  # USGS: incorrect location
+    "kilo1",  # Both: mainstems does not extend sufficiently upstream (~20%)
+    "ksdm7",  # USGS: masked
+    "levk1",  # NWS:  Incorrect feature_id assigned from WRDS, this has been corrected
+    "lkcm7",  # NWS:  masked
+    "loun7",  # NWS:  benchmark is not consistent between thresholds
+    "lrlm7",  # NWS:  masked
+    "mcri2",  # USGS: incorrect location
+    "monv1",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
+    "mtao1",  # USGS: ratio of evaluated vs domain is very low
+    "nhso1",  # USGS: mainstems does not extend sufficiently upstream (45%)
+    "nmso1",  # Both: mainstems does not extend sufficiently upstream
+    "pori3",  # USGS: mainstems does not extend sufficiently upstream
+    "ptvn6",  # Both: mainstems does not extend sufficiently upstream (50%)
+    "roun6",  # USGS: ratio of evaluated vs domain is very low
+    "rwdn4",  # Both: no mainstems in vicinity
+    "selt2",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
+    "sweg1",  # Both: mainstems does not extend sufficiently upstream (~30%)
+    "vcni3",  # USGS: ratio of evaluated vs domain is very low
+    "watw3",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
+    "weat2",  # NWS:  mainstems does not extend sufficiently upstream (~50%)
+    "wkew3",  # NWS:  mainstems does not extend sufficiently upstream (~45%)
+]
 DISCARD_AHPS_QUERY = "not flow.isnull() & masked_perc<97 & not nws_lid in @BAD_SITES"
 
 elev_raster_ndv = -9999.0
 
 # Colors.
-ENDC = '\033[m'
-TGREEN_BOLD = '\033[32;1m'
-TGREEN = '\033[32m'
-TRED_BOLD = '\033[31;1m'
-TWHITE = '\033[37m'
-WHITE_BOLD = '\033[37;1m'
-CYAN_BOLD = '\033[36;1m'
+ENDC = "\033[m"
+TGREEN_BOLD = "\033[32;1m"
+TGREEN = "\033[32m"
+TRED_BOLD = "\033[31;1m"
+TWHITE = "\033[37m"
+WHITE_BOLD = "\033[37;1m"
+CYAN_BOLD = "\033[36;1m"
 
 # USGS gages acceptance criteria. Likely not constants, so not using all caps.
 # ANY CHANGE TO THESE VALUES SHOULD WARRANT A CODE VERSION CHANGE
 # https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html
-acceptable_coord_acc_code_list = ['H','1','5','S','R','B','C','D','E', 5, 1]
+acceptable_coord_acc_code_list = ["H", "1", "5", "S", "R", "B", "C", "D", "E", 5, 1]
 # https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html
-acceptable_coord_method_code_list = ['C','D','W','X','Y','Z','N','M','L','G','R','F','S']
+acceptable_coord_method_code_list = [
+    "C",
+    "D",
+    "W",
+    "X",
+    "Y",
+    "Z",
+    "N",
+    "M",
+    "L",
+    "G",
+    "R",
+    "F",
+    "S",
+]
 # https://help.waterdata.usgs.gov/codes-and-parameters/codes#SI
 acceptable_alt_acc_thresh = 1.0
 # https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html
-acceptable_alt_meth_code_list = ['A','D','F','I','J','L','N','R','W','X','Y','Z']
+acceptable_alt_meth_code_list = [
+    "A",
+    "D",
+    "F",
+    "I",
+    "J",
+    "L",
+    "N",
+    "R",
+    "W",
+    "X",
+    "Y",
+    "Z",
+]
 # https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html
-acceptable_site_type_list = ['ST']
+acceptable_site_type_list = ["ST"]

From 24f763ead4ea60212c5c28f2eb4f3c3f8824dc10 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 1 Sep 2023 10:51:53 -0500
Subject: [PATCH 272/359] v4.4.0.0 SRC calibration with ras2fim xsection rating
 data (#965)

---
 config/deny_branches.lst          |   1 +
 config/deny_unit.lst              |   1 +
 config/params_template.env        |  15 +-
 docs/CHANGELOG.md                 |  55 +++----
 fim_post_processing.sh            |  21 ++-
 src/add_crosswalk.py              |   1 +
 src/aggregate_by_huc.py           |  49 +++++-
 src/run_by_branch.sh              |   2 +-
 src/run_unit_wb.sh                |   4 +-
 src/src_adjust_ras2fim_rating.py  | 262 ++++++++++++++++++++++++++++++
 src/src_roughness_optimization.py |  16 +-
 src/subdiv_chan_obank_src.py      |   2 +-
 src/usgs_gage_crosswalk.py        |  34 ++--
 src/usgs_gage_unit_setup.py       |  46 +++++-
 tools/inundate_nation.py          | 101 +++++-------
 15 files changed, 472 insertions(+), 138 deletions(-)
 create mode 100644 src/src_adjust_ras2fim_rating.py

diff --git a/config/deny_branches.lst b/config/deny_branches.lst
index 13af6b41a..9e7d3924b 100644
--- a/config/deny_branches.lst
+++ b/config/deny_branches.lst
@@ -50,6 +50,7 @@ nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
+#ras_elev_table.csv
 rem_{}.tif
 #rem_zeroed_masked_{}.tif
 slopes_d8_dem_meters_{}.tif
diff --git a/config/deny_unit.lst b/config/deny_unit.lst
index 752719f7b..b2a12713d 100644
--- a/config/deny_unit.lst
+++ b/config/deny_unit.lst
@@ -28,6 +28,7 @@ nwm_headwaters.gpkg
 #nwm_subset_streams_levelPaths.gpkg
 #nwm_subset_streams_levelPaths_dissolved.gpkg
 #nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+#ras_elev_table.csv
 #usgs_elev_table.csv
 #usgs_subset_gages.gpkg
 #wbd.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index 03b15760b..d1f072fa4 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -44,12 +44,19 @@ export src_subdiv_toggle="True" # Toggle to run composite roughness src routine
 export vrough_suffix="" # text to append to output log and hydrotable file names (use for testing/debugging)
 export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
 
-#### apply SRC adjustments using USGS rating curve database (calibration db) ####
-export src_adjust_usgs="True" # Toggle to run src adjustment routine (True=on; False=off)
+#### SRC calibration variables
 export nwm_recur_file="$inputsDir/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
 
-#### apply SRC adjustments using observed FIM/flow point database ####
-export src_adjust_spatial="True" # Toggle to run src adjustment routine (True=on; False=off)
+#### apply SRC adjustments using USGS rating curve database ####
+export src_adjust_usgs="True" # Toggle to run src adjustment routine with USGS rating data (True=on; False=off)
+export usgs_rating_curve_csv="$inputsDir/usgs_gages/usgs_rating_curves.csv" # input file location with usgs rating curve database
+
+#### apply SRC adjustments using ras2fim rating curve database ####
+export src_adjust_ras2fim="True" # Toggle to run src adjustment routine with ras2fim data (True=on; False=off)
+export ras_rating_curve_csv="$inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table.csv" # input file location with nwm feature_id and recurrence flow values
+
+#### apply SRC adjustments using observed FIM/flow point database (parquet files)####
+export src_adjust_spatial="True" # Toggle to run src adjustment routine with flood edge point db (True=on; False=off)
 export fim_obs_pnt_data="$inputsDir/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 
 #### apply manual calibration ####
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 957767795..d2c1eb2b8 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,44 +1,31 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-## v4.3.15.5 - 2023-08-17 - [PR#970](https://github.com/NOAA-OWP/inundation-mapping/pull/970)
+## v4.4.0.0 - 2023-09-01 - [PR#965](https://github.com/NOAA-OWP/inundation-mapping/pull/965)
 
-Fixes an issue where the stream network was clipped inside the DEM resulting in a burned stream channel that was then filled by the DEM depression filling process so that all pixels in the burned channel had the same elevation which was the elevation at the spill point (which wasn't necessarily at the HUC outlet). The stream network is now extended from the WBD to the buffered WBD and all streams except the outlet are clipped to the streams buffer inside the WBD (WBD - (3 x cell_size)). This also prevents reverse flow issues.
-
-### Changes
-
-- `src/`
-    - `clip_vectors_to_wbd.py`: Clip NWM streams to buffered WBD and clip non-outlet streams to WBD streams buffer (WBD - (3 x cell_size)).
-    - `derive_level_paths.py`: Add WBD input argument
-    - `run_unit_wb.py`: Add WBD input argument
-    - `src_stream_branches.py`: Ignore branches outside HUC
-- `unit_tests/`
-    - `derive_level_paths_params.json`: Add WBD parameter value
-    - `derive_level_paths_test.py`: Add WBD parameter
- 
-<br/><br/>
+This feature branch includes new functionality to perform an additional layer of HAND SRC calibration using ras2fim rating curve and point data. The calibration workflow for ras2fim data follows the same general logic as the existing USGS rating curve calibration routine. 
 
-## v4.3.15.4 - 2023-08-28 - [PR#977](https://github.com/NOAA-OWP/inundation-mapping/pull/977)
+### Additions
 
-Fixes incorrect `nodata` value in `src/burn_in_levees.py` that was responsible for missing branches (Exit code: 61). Also cleans up related files.
+- `src/src_adjust_ras2fim_rating.py`: New python script to perform the data prep steps for running the SRC calibration routine: 
+1) merge the `ras_elev_table.csv` data and the ras2fim cross section rating curve data (`reformat_ras_rating_curve_table.csv`) 
+2) sample the ras2fim rating curve at NWM recurrence flow intervals (2, 5, 10, 25, 50, 100yr)
+3) pass inputs to the `src_roughness_optimization.py` workflow
 
 ### Changes
 
-- `src/`
-    - `buffer_stream_branches.py`: Moves script functionality into a function.
-    - `burn_in_levees.py`: Corrects `nodata` value. Adds context managers for reading rasters.
-    - `generate_branch_list.py`: Removes unused imports.
-    - `mask_dem.py`: Removes commented code.
-
-<br/><br/>
-
-## v4.3.15.3 - 2023-09-01 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/983)
-
-### Changes  
-
-This hotfix addresses some bugs introduced in the pandas upgrade.
-
-- `/tools/eval_plots_stackedbar.py`: 2 lines were changed to work with the pandas upgrade. Added an argument for a `groupby` median call and fixed a bug with the pandas `query`. Also updated with Black compliance.
+- `config/deny_branches.lst`: Added `ras_elev_table.csv` to keep list. Needed for `fim_post_processing.sh`
+- `config/deny_unit.lst`: Added `ras_elev_table.csv` to keep list. Needed for `fim_post_processing.sh`
+- `config/params_template.env`: Added new block for ras2fim SRC calibration parameters (can turn on/off each of the three SRC calibration routines individually); also reconfigured docstrings for calibration parameters)
+- `fim_post_processing.sh`: Added routines to create ras2fim calibration data and then run the SRC calibration workflow with ras2fim data
+- `src/add_crosswalk.py`: Added placeholder variable (`calb_coef_ras2fim`) in all `hydrotable.csv` files
+- `src/aggregate_by_huc.py`: Added new blocks to perform huc-branch aggregation for all `ras_elev_table.csv` files
+- `src/run_by_branch.sh`: Revised input variable (changed from csv file to directory) for `usgs_gage_crosswalk.py` to facilitate both `usgs_elev_table.csv` and ras_elev_table.csv` outputs
+- `src/run_unit_wb.sh`: Revised inputs and output variables for `usgs_gage_unit_setup.py` and `usgs_gage_crosswalk.py`
+- `src/src_roughness_optimization.py`: Added code blocks to ingest ras2fim rating curve data; added new attributes/renamed output variables to catchments gpkg output
+- `src/usgs_gage_crosswalk.py`: Added code block to process ras2fim point locations alongside existing USGS gage point locations; outputs a separate csv if ras2fim points exist within the huc
+- `src/usgs_gage_unit_setup.py`: Added code block to ingest and process raw ras2fim point locations gpkg file (same general workflow to usgs gages); all valid points (USGS and RAS2FIM) are exported to the huc level `usgs_subset_gages.gpkg`
+- `tools/inundate_nation.py`: Added functionality to allow user to pass in a single HUC for faster spot checking of NWM recurr inundation maps
 
 <br/><br/>
 
@@ -57,7 +44,7 @@ Adds functionality to `tools/inundate_mosaic_wrapper.py` and incorporates functi
 
 <br/><br/>
 
-## v4.3.15.5 - 2023-08-17 - [PR#970](https://github.com/NOAA-OWP/inundation-mapping/pull/970)
+## v4.3.15.5 - 2023-09-01 - [PR#970](https://github.com/NOAA-OWP/inundation-mapping/pull/970)
 
 Fixes an issue where the stream network was clipped inside the DEM resulting in a burned stream channel that was then filled by the DEM depression filling process so that all pixels in the burned channel had the same elevation which was the elevation at the spill point (which wasn't necessarily at the HUC outlet). The stream network is now extended from the WBD to the buffered WBD and all streams except the outlet are clipped to the streams buffer inside the WBD (WBD - (3 x cell_size)). This also prevents reverse flow issues.
 
@@ -74,7 +61,7 @@ Fixes an issue where the stream network was clipped inside the DEM resulting in
  
 <br/><br/>
 
-## v4.3.15.4 - 2023-08-28 - [PR#977](https://github.com/NOAA-OWP/inundation-mapping/pull/977)
+## v4.3.15.4 - 2023-09-01 - [PR#977](https://github.com/NOAA-OWP/inundation-mapping/pull/977)
 
 Fixes incorrect `nodata` value in `src/burn_in_levees.py` that was responsible for missing branches (Exit code: 61). Also cleans up related files.
 
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index a1ba00fff..ceacfb0db 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -97,8 +97,8 @@ echo -e $startDiv"Start non-zero exit code checking"
 find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputDestDir/branch_errors/non_zero_exit_codes.log" &
 
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
-echo "Processing usgs gage aggregation"   
-python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev -j $jobLimit
+echo "Processing usgs & ras2fim elev table aggregation"   
+python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev -ras -j $jobLimit
 
 ## RUN BATHYMETRY ADJUSTMENT ROUTINE ##
 if [ "$bathymetry_adjust" = "True" ]; then
@@ -119,7 +119,7 @@ if [ "$src_bankfull_toggle" = "True" ]; then
 fi
 
 ## RUN SYNTHETIC RATING SUBDIVISION ROUTINE ##
-if [ "$src_subdiv_toggle" = "True" ]; then
+if [ "$src_subdiv_toggle" = "True" ] && [ "$src_bankfull_toggle" = "True" ]; then
     echo -e $startDiv"Performing SRC channel/overbank subdivision routine"
     # Run SRC Subdivision & Variable Roughness routine
     Tstart
@@ -132,8 +132,19 @@ if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$s
     Tstart
     echo    
     echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
-    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow thresholds)
-    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputDestDir -usgs_rc $inputsDir/usgs_gages/usgs_rating_curves.csv -nwm_recur $nwm_recur_file -j $jobLimit
+    # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow values)
+    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputDestDir -usgs_rc $usgs_rating_curve_csv -nwm_recur $nwm_recur_file -j $jobLimit
+    Tcount
+    date -u
+fi
+
+## RUN SYNTHETIC RATING CURVE CALIBRATION W/ RAS2FIM CROSS SECTION RATING CURVES ##
+if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$skipcal" = "0" ]; then
+    Tstart
+    echo    
+    echo -e $startDiv"Performing SRC adjustments using ras2fim rating curve database"
+    # Run SRC Optimization routine using ras2fim rating curve data (WSE and flow @ NWM recur flow values)
+    python3 $srcDir/src_adjust_ras2fim_rating.py -run_dir $outputDestDir -ras_rc $ras_rating_curve_csv -nwm_recur $nwm_recur_file -j $jobLimit
     Tcount
     date -u
 fi
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 188162737..21536bfe2 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -265,6 +265,7 @@ def add_crosswalk(input_catchments_fileName,
     output_hydro_table['obs_source'] = pd.NA
     output_hydro_table['precalb_discharge_cms'] = pd.NA
     output_hydro_table['calb_coef_usgs'] = pd.NA
+    output_hydro_table['calb_coef_ras2fim'] = pd.NA
     output_hydro_table['calb_coef_spatial'] = pd.NA
     output_hydro_table['calb_coef_final'] = pd.NA
 
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index 4a0cd4fc3..74f0b57b9 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -117,6 +117,20 @@ def __init__(self, fim_directory, huc_id, limit_branches=[]):
                                     'Discharge (m3s-1)_subdiv':float}
         self.agg_src_cross = pd.DataFrame(columns=list(self.src_crosswalked_dtypes.keys()))
 
+        self.ras_dtypes = {'location_id':str,
+                            'nws_lid':str,
+                            'feature_id':int,
+                            'HydroID':int,
+                            'levpa_id':str,
+                            'dem_elevation':float,
+                            'dem_adj_elevation':float,
+                            'order_':str,
+                            'LakeID':object,
+                            'HUC8':str,
+                            'snap_distance':float}
+        self.agg_ras_elev_table = pd.DataFrame(columns=list(self.ras_dtypes.keys()))
+        
+
     def iter_branches(self):
 
         if self.limit_branches:
@@ -159,13 +173,24 @@ def aggregate_src_full_crosswalk(self, branch_path, branch_id):
         src_cross['branch_id'] = branch_id
         self.agg_src_cross = pd.concat([self.agg_src_cross, src_cross])
 
-    def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, huc_id):
+    def ras_elev_table(self, branch_path):
+
+        ras_elev_filename = join(branch_path, 'ras_elev_table.csv')
+        if not os.path.isfile(ras_elev_filename):
+            return
+
+        ras_elev_table = pd.read_csv(ras_elev_filename, dtype=self.ras_dtypes)
+        self.agg_ras_elev_table = pd.concat([self.agg_ras_elev_table, ras_elev_table])
+
+    def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id):
         
         try:
             #try catch and its own log file output in error only.
             for branch_id, branch_path in self.iter_branches():
                 if usgs_elev_flag:
                     self.usgs_elev_table(branch_path)
+                if ras_elev_flag:
+                    self.ras_elev_table(branch_path)
 
                 ## Other aggregate funtions can go here
                 if hydro_table_flag:
@@ -198,6 +223,14 @@ def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, huc_id)
                 if not self.agg_src_cross.empty:
                     self.agg_src_cross.to_csv(src_crosswalk_file, index=False)
 
+            if ras_elev_flag:
+                ras_elev_table_file = join(self.huc_dir_path, 'ras_elev_table.csv')
+                if os.path.isfile(ras_elev_table_file):
+                    os.remove(ras_elev_table_file)
+
+                if not self.agg_ras_elev_table.empty:
+                    self.agg_ras_elev_table.to_csv(ras_elev_table_file, index=False)
+
             #print(f"agg_by_huc for huc id {huc_id} is done")
             
         except Exception:
@@ -207,19 +240,20 @@ def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, huc_id)
             errMsg = errMsg + traceback.format_exc()
             print(errMsg, flush=True)
             log_error(self.fim_directory, usgs_elev_flag,
-                        hydro_table_flag, src_cross_flag, huc_id, errMsg)
+                        hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg)
            
 
 
 # ==============================
 # This is done independantly in each worker and does not attempt to write to a shared file
 # as those can collide with multi proc
-def log_error(fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, huc_id, errMsg):
+def log_error(fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg):
     
     file_name = f"agg_by_huc_{huc_id}"
     if (usgs_elev_flag): file_name+= "_elev"
     if (hydro_table_flag): file_name+= "_hydro"
     if (src_cross_flag): file_name+= "_src_cross"
+    if (ras_elev_flag): file_name+= "_ras"
     file_name+= "_error.log"
     
     log_path = os.path.join(fim_directory, "logs", "agg_by_huc_errors")
@@ -231,7 +265,7 @@ def log_error(fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, h
 
 
 def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
-                     hydro_table_flag, src_cross_flag, num_job_workers):
+                     hydro_table_flag, src_cross_flag, ras_elev_flag, num_job_workers):
     
     assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
 
@@ -258,6 +292,7 @@ def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
         if (usgs_elev_flag): agg_type+= "_elev"
         if (hydro_table_flag): agg_type+= "_hydro"
         if (src_cross_flag): agg_type+= "_src_cross"
+        if (ras_elev_flag): agg_type+= "_ras"
         filelist = glob.glob(os.path.join(log_folder, f"*{agg_type}*"))
         for f in filelist:
             os.remove(f)
@@ -288,6 +323,7 @@ def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
                     args_agg = { 'usgs_elev_flag': usgs_elev_flag,
                                  'hydro_table_flag': hydro_table_flag,
                                  'src_cross_flag': src_cross_flag,
+                                 'ras_elev_flag': ras_elev_flag,
                                  'huc_id': huc_id}       
 
                     future = executor.submit(huc_dir.agg_function, **args_agg)
@@ -309,6 +345,7 @@ def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
                     args_agg = { 'usgs_elev_flag': usgs_elev_flag,
                                 'hydro_table_flag': hydro_table_flag,
                                 'src_cross_flag': src_cross_flag,
+                                'ras_elev_flag': ras_elev_flag,
                                 'huc_id': huc_id}       
                     future = executor.submit(huc_dir.agg_function, **args_agg)
                     executor_dict[future] = huc_id
@@ -319,7 +356,7 @@ def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
             errMsg = errMsg + traceback.format_exc()
             print(errMsg, flush=True)
             log_error(fim_directory, usgs_elev_flag,
-                        hydro_table_flag, src_cross_flag, huc_id, errMsg)
+                        hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg)
             #sys.exit(1)
     
         # Send the executor to the progress bar and wait for all MS tasks to finish
@@ -352,6 +389,8 @@ def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
                         required=False, default=False, action='store_true')
     parser.add_argument('-src','--src_cross_flag', help='Perform aggregate on branch src crosswalk files',
                         required=False, default=False, action='store_true')
+    parser.add_argument('-ras','--ras_elev_flag', help='Perform aggregate on branch ras2fim elev tables',
+                        required=False, default=False, action='store_true')
     parser.add_argument('-j','--num_job_workers', help='Number of processes to use',
                         required=False, default=1, type=int)
 
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index 4a82c84d3..b7ad2f844 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -93,7 +93,7 @@ if [ -f $tempHucDataDir/usgs_subset_gages.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $current_branch_id"
     date -u
     Tstart
-    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -outtable $tempCurrentBranchDataDir/usgs_elev_table.csv -b $current_branch_id
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -out $tempCurrentBranchDataDir -b $current_branch_id
     Tcount
 fi
 
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 373562605..37ffc7fe7 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -270,7 +270,7 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputsDir/usgs_gages/usgs_gages.gpkg -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -o $tempHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputsDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id
+    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputsDir/usgs_gages/usgs_gages.gpkg -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -ras $inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points.gpkg -o $tempHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputsDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id 
     Tcount
 fi
 
@@ -279,7 +279,7 @@ if [ -f $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -outtable $tempCurrentBranchDataDir/usgs_elev_table.csv -b $branch_zero_id
+    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -out $tempCurrentBranchDataDir -b $branch_zero_id
     Tcount
 fi
 
diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
new file mode 100644
index 000000000..0a810bab9
--- /dev/null
+++ b/src/src_adjust_ras2fim_rating.py
@@ -0,0 +1,262 @@
+import argparse
+import os
+import pandas as pd
+import sys
+import json
+import datetime as dt
+from pathlib import Path
+from collections import deque
+import multiprocessing
+from multiprocessing import Pool
+from utils.shared_functions import check_file_age, concat_huc_csv
+from src_roughness_optimization import update_rating_curve
+'''
+The script ingests a RAS2FIM rating curve csv and a NWM flow recurrence interval database. The gage location will be associated to the corresponding hydroID and attributed with the HAND elevation value
+
+Processing
+- Read in RAS2FIM rating curve from csv and convert WSE navd88 values to meters
+- Read in the aggregate RAS elev table csv from the HUC fim directory (output from ras_gage_crosswalk.py)
+- Filter null entries and convert RAS2FIM flow from cfs to cms
+- Calculate HAND elevation value for each gage location (NAVD88 elevation - NHD DEM thalweg elevation)
+- Read in the NWM recurr csv file and convert flow to cfs
+- Calculate the closest SRC discharge value to the NWM flow value
+- Create dataframe with crosswalked RAS2FIM flow and NWM recurr flow and assign metadata attributes
+- Calculate flow difference (variance) to check for large discrepancies btw NWM flow and RAS2FIM closest flow
+- Log any signifant differences (or negative HAND values) btw the NWM flow value and closest RAS2FIM rating flow
+- Produce log file
+- Call update_rating_curve() to perform the rating curve calibration.
+
+Inputs
+- branch_dir:        fim directory containing individual HUC output dirs
+- ras_rc_filepath:     RAS2FIM rating curve database (produced by reformat_ras_rating_curve.py in ras2fim repo)
+- nwm_recurr_filepath:  NWM flow recurrence interval dataset
+- debug_outputs_option: optional flag to output intermediate files for reviewing/debugging
+- job_number:           number of multi-processing jobs to use
+
+Outputs
+- water_edge_median_ds: dataframe containing 'location_id','hydroid','feature_id','huc','hand','discharge_cms','nwm_recur_flow_cms','nwm_recur','layer'
+'''
+
+def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir):
+    start_time = dt.datetime.now()
+    print('Reading RAS2FIM rating curves from csv...')
+    log_text = 'Processing database for RAS2FIM flow/WSE at NWM flow recur intervals...\n'
+    col_filter = ["fid_xs", "flow", "wse"]
+    ras_rc_df = pd.read_csv(ras_rc_filepath, dtype={'fid_xs': object}, usecols=col_filter)#, nrows=30000)
+    ras_rc_df.rename(columns={'fid_xs':'location_id'}, inplace=True)
+    #ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
+    print('Duration (read ras_rc_csv): {}'.format(dt.datetime.now() - start_time))
+    
+    # convert WSE navd88 values to meters
+    ras_rc_df.rename(columns={'wse':'wse_navd88_m','flow':'discharge_cms'}, inplace=True) # assume ras2fim elevation data in feet
+    
+    # read in the aggregate RAS elev table csv
+    start_time = dt.datetime.now()
+    cross_df = ras_elev_df[["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_adj_elevation", "source"]].copy()
+    cross_df.rename(columns={'dem_adj_elevation':'hand_datum', 'HydroID':'hydroid', 'HUC8':'huc'}, inplace=True)
+    
+    # filter null location_id rows from cross_df 
+    cross_df = cross_df[cross_df.location_id.notnull()]
+    
+    # merge ras2fim ratings with crosswalk attributes
+    ras_rc_df = ras_rc_df.merge(cross_df, how='left', on='location_id')
+    ras_rc_df = ras_rc_df[ras_rc_df['hydroid'].notna()]
+    
+    # calculate hand elevation
+    ras_rc_df['hand'] = ras_rc_df['wse_navd88_m'] - ras_rc_df['hand_datum']
+    ras_rc_df = ras_rc_df[['location_id','feature_id','hydroid','levpa_id','huc','hand','discharge_cms','source']]
+    ras_rc_df['feature_id'] = ras_rc_df['feature_id'].astype(int)
+    
+    # read in the NWM recurr csv file
+    nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
+    nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
+    nwm_recur_df.rename(columns={'2_0_year_recurrence_flow_17C':'2_0_year','5_0_year_recurrence_flow_17C':'5_0_year','10_0_year_recurrence_flow_17C':'10_0_year','25_0_year_recurrence_flow_17C':'25_0_year','50_0_year_recurrence_flow_17C':'50_0_year','100_0_year_recurrence_flow_17C':'100_0_year'}, inplace=True)
+    
+    #convert cfs to cms (x 0.028317)
+    nwm_recur_df.loc[:, ['2_0_year','5_0_year','10_0_year','25_0_year','50_0_year','100_0_year']] *= 0.028317
+    
+    # merge nwm recurr with ras_rc_df
+    merge_df = ras_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
+    
+    # NWM recurr intervals
+    recurr_intervals = ["2","5","10","25","50","100"] # "2","5","10","25","50","100"
+    final_df = pd.DataFrame() # create empty dataframe to append flow interval dataframes
+    for interval in recurr_intervals:
+        log_text += ('\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n')
+        print('Processing: ' + str(interval) + '-year NWM recurr intervals')
+        ## Calculate the closest SRC discharge value to the NWM flow value
+        merge_df['Q_find'] = (merge_df['discharge_cms'] - merge_df[interval+"_0_year"]).abs()
+        
+        ## Check for any missing/null entries in the input SRC
+        if merge_df['Q_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
+            log_text += 'HUC: ' + str(merge_df['huc']) + ' : feature_id' + str(merge_df['feature_id']) + ' --> Null values found in "Q_find" calc. These will be filled with 999999 () \n'
+            ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
+            merge_df['Q_find'] = merge_df['Q_find'].fillna(999999)
+        if merge_df['hydroid'].isnull().values.any():
+            log_text += 'HUC: ' + str(merge_df['huc']) + ' --> Null values found in "hydroid"... \n'
+        
+        # Create dataframe with crosswalked RAS2FIM flow and NWM recurr flow
+        calc_df = merge_df.loc[merge_df.groupby(['location_id','levpa_id'])['Q_find'].idxmin()].reset_index(drop=True) # find the index of the Q_1_5_find (closest matching flow)
+        # Calculate flow difference (variance) to check for large discrepancies btw NWM flow and RAS2FIM rating closest flow
+        calc_df['check_variance'] = ((calc_df['discharge_cms'] - calc_df[interval+"_0_year"])/calc_df['discharge_cms']).abs()
+        # Assign new metadata attributes
+        calc_df['nwm_recur'] = interval+"_0_year"
+        calc_df['layer'] = '_ras2fim-gage____' + interval+"-year"
+        calc_df.rename(columns={interval+"_0_year":'nwm_recur_flow_cms'}, inplace=True)
+        # Subset calc_df for final output
+        calc_df = calc_df[['location_id','hydroid','feature_id','levpa_id','huc','hand','discharge_cms','check_variance','nwm_recur_flow_cms','nwm_recur','layer','source']]
+        final_df = pd.concat([final_df, calc_df], ignore_index=True)
+        # Log any negative HAND elev values and remove from database
+        log_text += ('Warning: Negative HAND stage values -->\n')
+        log_text += (calc_df[calc_df['hand']<0].to_string() +'\n')
+        final_df = final_df[final_df['hand']>0]
+        # Log any signifant differences btw the NWM flow value and closest RAS2FIM rating flow (this ensures that we consistently sample the RAS2FIM rating curves at known intervals - NWM recur flow)
+        log_text += ('Warning: Large variance (>10%) between NWM flow and closest RAS2FIM flow -->\n')
+        log_text += (calc_df[calc_df['check_variance']>0.1].to_string() +'\n')
+        final_df = final_df[final_df['check_variance']<0.1]
+        # Get datestamp from ras2fim rating curve file to use as coll_time attribute in hydroTable.csv
+        datestamp = check_file_age(ras_rc_filepath)
+        final_df['coll_time'] = str(datestamp)[:15]
+
+    # Rename attributes (for ingest to update_rating_curve) and output csv with the RAS2FIM RC database
+    final_df.rename(columns={'discharge_cms':'flow','source':'submitter'}, inplace=True)  
+    final_df.to_csv(os.path.join(log_dir,"ras2fim_rc_nwm_recurr.csv"),index=False)
+
+    # Output log text to log file
+    log_text += ('#########\nTotal entries per RAS2FIM point location (feature_id) -->\n')
+    loc_id_df = final_df.groupby(['location_id']).size().reset_index(name='count') 
+    log_text += (loc_id_df.to_string() +'\n')
+    log_text += ('#########\nTotal entries per NWM recur value -->\n')
+    recur_count_df = final_df.groupby(['nwm_recur']).size().reset_index(name='count') 
+    log_text += (recur_count_df.to_string() +'\n')
+    log_ras_db = open(os.path.join(log_dir,'log_ras2fim_rc_database.log'),"w")
+    log_ras_db.write(log_text)
+    log_ras_db.close()
+    return(final_df)
+
+def branch_proc_list(ras_df,run_dir,debug_outputs_option,log_file):
+    procs_list = []  # Initialize list for mulitprocessing.
+
+    # loop through all unique level paths that have a USGS gage
+    huc_branch_dict = ras_df.groupby('huc')['levpa_id'].apply(set).to_dict()
+
+    for huc in sorted(huc_branch_dict.keys()): # sort huc_list for helping track progress in future print statments
+        branch_set = huc_branch_dict[huc]
+        for branch_id in branch_set: 
+            # Define paths to branch HAND data.
+            # Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
+            # Assumes outputs are for HUC8 (not HUC6)
+            branch_dir = os.path.join(run_dir,huc,'branches',branch_id)
+            hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
+            catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
+            catchments_poly_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
+            htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
+            water_edge_median_ds = ras_df[(ras_df['huc']==huc) & (ras_df['levpa_id']==branch_id)]
+
+            # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
+            if not os.path.exists(hand_path):
+                print("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+            elif not os.path.exists(catchments_path):
+                print("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')        
+            elif not os.path.exists(htable_path):
+                print("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
+                log_file.write("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+            else:
+                ## Additional arguments for src_roughness_optimization
+                source_tag = 'ras2fim_rating' # tag to use in source attribute field
+                merge_prev_adj = True # merge in previous SRC adjustment calculations
+
+                print('Will perform SRC adjustments for huc: ' + str(huc) + ' - branch-id: ' + str(branch_id))
+                procs_list.append([branch_dir, water_edge_median_ds, htable_path, huc, branch_id, catchments_poly_path, debug_outputs_option, source_tag, merge_prev_adj])
+
+    # multiprocess all available branches
+    print(f"Calculating new SRCs for {len(procs_list)} branches using {job_number} jobs...")
+    with Pool(processes=job_number) as pool:
+            log_output = pool.starmap(update_rating_curve, procs_list)
+            log_file.writelines(["%s\n" % item  for item in log_output])
+    # try statement for debugging 
+    # try:
+    #     with Pool(processes=job_number) as pool:
+    #         log_output = pool.starmap(update_rating_curve, procs_list)
+    #         log_file.writelines(["%s\n" % item  for item in log_output])
+    # except Exception as e:
+    #     print(str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e))
+    #     log_file.write('ERROR!!!: HUC ' + str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e) + '\n')
+
+def run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number):
+    ## Check input args are valid
+    assert os.path.isdir(run_dir), 'ERROR: could not find the input fim_dir location: ' + str(run_dir)
+
+    ## Create an aggregate dataframe with all ras_elev_table.csv entries for hucs in fim_dir
+    print('Reading RAS2FIM point loc HAND elevation from ras_elev_table csv files...')
+    csv_name = 'ras_elev_table.csv' # file name to search for ras location data (in the huc/branch dirs)
+    
+    available_cores = multiprocessing.cpu_count()
+    if job_number > available_cores:
+        job_number = available_cores - 1
+        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+
+    ## Create output dir for log and ras2fim rc database
+    log_dir = os.path.join(run_dir,"logs","src_optimization")
+    print("Log file output here: " + str(log_dir))
+    if not os.path.isdir(log_dir):
+        os.makedirs(log_dir)
+
+    ## Create a time var to log run time
+    begin_time = dt.datetime.now()
+    # Create log file for processing records
+    log_file = open(os.path.join(log_dir,'log_ras2fim_rc_src_adjust.log'),"w")
+    log_file.write('START TIME: ' + str(begin_time) + '\n')
+    log_file.write('#########################################################\n\n')
+   
+    ras_elev_df = concat_huc_csv(run_dir,csv_name)
+
+    if ras_elev_df is None:
+        warn_err = 'WARNING: ras_elev_df not created - check that ' + csv_name + ' files exist in fim_dir!'
+        print(warn_err)
+        log_file.write(warn_err)
+        
+    elif ras_elev_df.empty:
+        warn_err = 'WARNING: ras_elev_df is empty - check that ' + csv_name + ' files exist in fim_dir!'
+        print(warn_err)
+        log_file.write(warn_err)
+
+    else:
+        print('This may take a few minutes...')
+        log_file.write("starting create RAS2FIM rating db")
+        ras_df = create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir)
+
+        ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
+        branch_proc_list(ras_df,run_dir,debug_outputs_option,log_file)
+
+    ## Record run time and close log file
+    log_file.write('#########################################################\n\n')
+    end_time = dt.datetime.now()
+    log_file.write('END TIME: ' + str(end_time) + '\n')
+    tot_run_time = end_time - begin_time
+    log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
+    sys.stdout = sys.__stdout__
+    log_file.close()
+
+if __name__ == '__main__':
+    ## Parse arguments.
+    parser = argparse.ArgumentParser(description='Adjusts rating curve with database of RAS2FIM reach average rating curves (calculated WSE/flow).')
+    parser.add_argument('-run_dir','--run-dir',help='Parent directory of FIM run.',required=True)
+    parser.add_argument('-ras_rc','--ras2fim-ratings',help='Path to RAS2FIM rating curve (reach avg) csv file',required=True)
+    parser.add_argument('-nwm_recur','--nwm_recur',help='Path to NWM recur file (multiple NWM flow intervals). NOTE: assumes flow units are cfs!!',required=True)
+    parser.add_argument('-debug','--extra-outputs',help='Optional flag: Use this to keep intermediate output files for debugging/testing',default=False,required=False, action='store_true')
+    parser.add_argument('-j','--job-number',help='Number of jobs to use',required=False,default=1)
+
+    ## Assign variables from arguments.
+    args = vars(parser.parse_args())
+    run_dir = args['run_dir']
+    ras_rc_filepath = args['ras2fim_ratings']
+    nwm_recurr_filepath = args['nwm_recur']
+    debug_outputs_option = args['extra_outputs']
+    job_number = int(args['job_number'])
+
+    ## Prepare/check inputs, create log file, and spin up the proc list
+    run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number)
+    
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index 356b69826..db667e42e 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -58,8 +58,8 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     - output_src_json:      src.json file with new SRC discharge values
 
     '''
-    #print("Processing huc --> " + str(huc))
-    log_text = "\nProcessing huc --> " + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+    print("Processing " + str(source_tag) + " calibration for huc --> " + str(huc) + '  branch id: ' + str(branch_id))
+    log_text = "\nProcessing " + str(source_tag) + " calibration for huc --> " + str(huc) + '  branch id: ' + str(branch_id) + '\n'
     log_text += "DOWNSTREAM_THRESHOLD: " + str(down_dist_thresh) + 'km\n'
     log_text += "Merge Previous Adj Values: " + str(merge_prev_adj) + '\n'
     df_nvalues = water_edge_median_df.copy()
@@ -68,8 +68,12 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     ## Determine calibration data type for naming calb dataframe column
     if source_tag == 'point_obs':
         calb_type = 'calb_coef_spatial'
-    if source_tag == 'usgs_rating':
+    elif source_tag == 'usgs_rating':
         calb_type = 'calb_coef_usgs'
+    elif source_tag == 'ras2fim_rating':
+        calb_type = 'calb_coef_ras2fim'
+    else:
+        log_text += "ERROR - unknown calibration data source type: " + str(source_tag) + '\n'
 
     ## Read in the hydroTable.csv and check wether it has previously been updated (rename default columns if needed)
     df_htable = pd.read_csv(htable_path, dtype={'HUC': object, 'last_updated':object, 'submitter':object, 'obs_source':object})
@@ -93,7 +97,7 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
         df_prev_adj_htable.rename(columns={'submitter':'submitter_prev','last_updated':'last_updated_prev','calb_coef_final':'calb_coef_final_prev','obs_source':'obs_source_prev'}, inplace=True)
         df_prev_adj_htable = df_prev_adj_htable.groupby(["HydroID"]).first()
         # Only keep previous USGS rating curve adjustments (previous spatial obs adjustments are not retained)
-        df_prev_adj = df_prev_adj_htable[df_prev_adj_htable['obs_source_prev'].str.contains("usgs_rating", na=False)] 
+        df_prev_adj = df_prev_adj_htable[df_prev_adj_htable['obs_source_prev'].str.contains("usgs_rating|ras2fim_rating", na=False)] 
         log_text += 'HUC: ' + str(huc) + '  Branch: ' + str(branch_id) + ': found previous hydroTable calibration attributes --> retaining previous calb attributes for blending...\n'
     
     # Delete previous adj columns to prevent duplicate variable issues (if src_roughness_optimization.py was previously applied)
@@ -240,9 +244,9 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
                     input_catchments = gpd.read_file(catchments_poly_path)
                     ## Create new "src_calibrated" column for viz query
                     if 'src_calibrated' in input_catchments.columns: # check if this attribute already exists and drop if needed
-                        input_catchments.drop(['src_calibrated'], axis=1, inplace=True, errors='ignore')
+                        input_catchments.drop(['src_calibrated','obs_source','calb_coef_final'], axis=1, inplace=True, errors='ignore')
                     df_nmerge['src_calibrated'] = np.where(df_nmerge['calb_coef_final'].notnull(), 'True', 'False')
-                    output_catchments = input_catchments.merge(df_nmerge[['HydroID','src_calibrated']], how='left', on='HydroID')
+                    output_catchments = input_catchments.merge(df_nmerge[['HydroID','src_calibrated','obs_source','calb_coef_final']], how='left', on='HydroID')
                     output_catchments['src_calibrated'].fillna('False', inplace=True)
                     output_catchments.to_file(catchments_poly_path,driver="GPKG",index=False) # overwrite the previous layer
                     df_nmerge.drop(['src_calibrated'], axis=1, inplace=True, errors='ignore')
diff --git a/src/subdiv_chan_obank_src.py b/src/subdiv_chan_obank_src.py
index 1e9f2aab1..efe434dd3 100755
--- a/src/subdiv_chan_obank_src.py
+++ b/src/subdiv_chan_obank_src.py
@@ -93,7 +93,7 @@ def variable_mannings_calc(args):
             df_htable = pd.read_csv(htable_filename,dtype={'HUC': str,'last_updated':object,'submitter':object,'obs_source':object})
 
             ## drop the previously modified discharge column to be replaced with updated version
-            df_htable.drop(['subdiv_applied','discharge_cms','overbank_n','channel_n','subdiv_discharge_cms'], axis=1, errors='ignore', inplace=True) 
+            df_htable.drop(['subdiv_applied','discharge_cms','overbank_n','channel_n','subdiv_discharge_cms','Bathymetry_source'], axis=1, errors='ignore', inplace=True) 
             df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
 
             ## Output new hydroTable csv
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 05ccfa07e..c93381aae 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -6,6 +6,7 @@
 import argparse
 import warnings
 from utils.shared_functions import mem_profile
+from os.path import join
 warnings.simplefilter("ignore")
 
 
@@ -23,8 +24,8 @@
         File path of original DEM. i.e. '/data/path/dem_meters_3246000257.tif'
     dem_adj_filename : str
         File path of thalweg adjusted DEM. i.e. '/data/path/dem_thalwegCond_3246000257.tif'
-    output_table_filename : str
-        File path of output table. i.e. '/data/path/usgs_elev_table.csv'
+    output_directory : str
+        Directory to create output table. i.e. '/data/path/'
     branch_id: str
         ID of the current branch i.e. '3246000257'
 '''
@@ -36,12 +37,11 @@ def __init__(self, usgs_subset_gages_filename, branch_id):
         self.branch_id = branch_id
         self.gages = self._load_gages(usgs_subset_gages_filename)
 
-    def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_table_filename):
+    def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory):
         '''Run the gage crosswalk steps: 1) spatial join to branch catchments layer 2) snap sites to 
         the dem-derived flows 3) sample both dems at the snapped points 4) write the crosswalked points
         to usgs_elev_table.csv
         '''
-        
         if self.gages.empty:
             print(f'There are no gages for branch {branch_id}')
             os._exit(0)
@@ -50,6 +50,7 @@ def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_file
         if self.gages.empty:
             print(f'There are no gages for branch {branch_id}')
             os._exit(0)
+
         # Snap to dem derived flow lines
         self.snap_to_dem_derived_flows(input_flows_filename)
         # Sample DEM and thalweg adjusted DEM
@@ -58,14 +59,14 @@ def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_file
         # Write to csv
         num_gages = len(self.gages)
         print(f"{num_gages} gage{'' if num_gages == 1 else 's'} in branch {self.branch_id}")
-        self.write(output_table_filename)
+        self.write(output_directory)
 
 
     def _load_gages(self, gages_filename):
         '''Reads gage geopackage from huc level and filters based on current branch id'''
 
         usgs_gages = gpd.read_file(gages_filename)
-        return  usgs_gages[usgs_gages.levpa_id == self.branch_id]
+        return  usgs_gages[(usgs_gages.levpa_id == self.branch_id)]
 
     def catchment_sjoin(self, input_catchment_filename):
         '''Spatial joins gages to FIM catchments'''
@@ -93,16 +94,25 @@ def sample_dem(self, dem_filename, column_name):
         with rasterio.open(dem_filename) as dem:
             self.gages[column_name] = [x[0] for x in dem.sample(coord_list)]
 
-    def write(self, output_table_filename):
+    def write(self, output_directory):
         '''Write to csv file'''
 
         # Prep and write out file
         elev_table = self.gages.copy()
         elev_table.loc[elev_table['location_id'] == elev_table['nws_lid'], 'location_id'] = None # set location_id to None where there isn't a gage
         elev_table = elev_table[elev_table['location_id'].notna()]
+        elev_table.source = elev_table.source.apply(str.lower)
+
+        # filter for just ras2fim entries
+        ras_elev_table = elev_table[elev_table['source'] == 'ras2fim']
+        ras_elev_table = ras_elev_table[["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_elevation", "dem_adj_elevation","source", "stream_stn"]]
+        if not ras_elev_table.empty:
+            ras_elev_table.to_csv(join(output_directory, 'ras_elev_table.csv'), index=False)
 
-        if not elev_table.empty:
-            elev_table.to_csv(output_table_filename, index=False)
+        # filter for just usgs entries
+        usgs_elev_table = elev_table[elev_table['source'] != 'ras2fim']
+        if not usgs_elev_table.empty:
+            usgs_elev_table.to_csv(join(output_directory, 'usgs_elev_table.csv'), index=False)
 
     @staticmethod
     def snap_to_line(row):
@@ -120,7 +130,7 @@ def snap_to_line(row):
     parser.add_argument('-cat','--input-catchment-filename', help='DEM derived catchments', required=True)
     parser.add_argument('-dem','--dem-filename',help='DEM',required=True)
     parser.add_argument('-dem_adj','--dem-adj-filename', help='Thalweg adjusted DEM', required=True)
-    parser.add_argument('-outtable','--output-table-filename', help='Table to append data', required=True)
+    parser.add_argument('-out','--output-directory', help='Directory where output tables created', required=True)
     parser.add_argument('-b','--branch-id', help='Branch ID used to filter the gages', type=str, required=True)
 
     args = vars(parser.parse_args())
@@ -130,14 +140,14 @@ def snap_to_line(row):
     input_catchment_filename = args['input_catchment_filename']
     dem_filename = args['dem_filename']
     dem_adj_filename = args['dem_adj_filename']
-    output_table_filename = args['output_table_filename']
+    output_directory = args['output_directory']
     branch_id = args['branch_id']
 
     assert os.path.isfile(usgs_gages_filename), f"The input file {usgs_gages_filename} does not exist."
 
     # Instantiate class
     gage_crosswalk = GageCrosswalk(usgs_gages_filename, branch_id)
-    gage_crosswalk.run_crosswalk(input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_table_filename)
+    gage_crosswalk.run_crosswalk(input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory)
 
 """
 Examples:
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index a6260483a..e6186c744 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -7,23 +7,54 @@
 import pandas as pd
 import argparse
 import warnings
+from shapely.geometry import Point
 from utils.shared_functions import mem_profile
+from utils.shared_variables import PREP_CRS
 warnings.simplefilter("ignore")
 
 class Gage2Branch(object):
 
-    def __init__(self, usgs_gage_filename, ahps_filename, huc8):
+    def __init__(self, usgs_gage_filename, ras_locs_filename, ahps_filename, huc8):
 
         self.usgs_gage_filename = usgs_gage_filename
+        self.ras_locs_filename = ras_locs_filename
         self.ahps_filename = ahps_filename
-        self.huc8 = huc8
+        self.huc8 = str(huc8)
         self.load_gages()
 
     def load_gages(self):
 
-        # Filter USGS gages to huc
-        usgs_gages = gpd.read_file(self.usgs_gage_filename)
-        self.gages = usgs_gages[(usgs_gages.HUC8 == self.huc8)]
+        # Read USGS gage file a
+        usgs_gages = gpd.read_file(self.usgs_gage_filename,dtype={'location_id': object})
+        usgs_gages['source'] = 'usgs_gage'
+
+        # Read RAS2FIM point locations file
+        # !!! Geopandas is not honoring the dtype arg with this read_file below (huc8 being read as int64). 
+        # Need the raw data to store the 'huc8' attribute as an object to avoid issues with integers truncating the leading zero from some hucs
+        ras_locs = gpd.read_file(self.ras_locs_filename,dtype={'huc8': 'object'}) 
+        ras_locs = ras_locs[['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'wrds_timestamp', 'geometry']]
+        ras_locs['location_id'] =  ras_locs['fid_xs']
+        
+        # Convert ras locs crs to match usgs gage crs
+        ras_locs.to_crs(usgs_gages.crs, inplace=True)
+        ras_locs.rename(columns={'huc8':'HUC8'}, inplace=True)
+
+        # Convert Multipoint geometry to Point geometry 
+        ras_locs['geometry'] = ras_locs.representative_point()
+
+        # if ras_locs.huc8.dtype == 'int64':
+        #     ras_locs = ras_locs[ras_locs.huc8 == int(self.huc8)]
+        #     ras_locs['HUC8'] = str(self.huc8)
+        #     ras_locs = ras_locs.drop('huc8', axis=1)
+        # elif ras_locs.huc8.dtype == 'int64':
+        #     ras_locs.rename(columns={'huc8':'HUC8'}, inplace=True)
+
+        # Concat USGS points and RAS2FIM points
+        gages_locs = pd.concat([usgs_gages, ras_locs], axis=0, ignore_index=True)
+        #gages_locs.to_crs(PREP_CRS, inplace=True)
+
+        # Filter USGS gages and RAS locations to huc
+        self.gages = gages_locs[(gages_locs.HUC8 == self.huc8)]
         
         # Get AHPS sites within the HUC and add them to the USGS dataset
         if self.ahps_filename:
@@ -32,6 +63,7 @@ def load_gages(self):
             ahps_sites.rename(columns={'nwm_feature_id':'feature_id',
                                 'usgs_site_code':'location_id'}, inplace=True)
             ahps_sites = ahps_sites[ahps_sites.location_id.isna()] # Filter sites that are already in the USGS dataset
+            ahps_sites['source'] = 'ahps_site'
             self.gages = pd.concat([self.gages, ahps_sites[['feature_id', 'nws_lid', 'location_id', 'HUC8', 'name', 'states','geometry']]])
 
         # Create gages attribute
@@ -100,6 +132,7 @@ def filter_gage_branches(fim_inputs_filename):
 
     parser = argparse.ArgumentParser(description='Assign HUC gages to branch and stage for usgs_gage_crosswalk.py')
     parser.add_argument('-gages','--usgs-gages-filename', help='USGS gages', required=True)
+    parser.add_argument('-ras','--ras-locs-filename', help='RAS2FIM rating curve locations', required=True)
     parser.add_argument('-ahps','--nws-lid-filename', help='AHPS gages', required=False)
     parser.add_argument('-nwm','--input-nwm-filename', help='NWM stream subset', required=True)
     parser.add_argument('-o','--output-filename', help='Table to append data', required=True)
@@ -110,6 +143,7 @@ def filter_gage_branches(fim_inputs_filename):
     args = vars(parser.parse_args())
 
     usgs_gages_filename = args['usgs_gages_filename']
+    ras_locs_filename = args['ras_locs_filename']
     nws_lid_filename = args['nws_lid_filename']
     input_nwm_filename = args['input_nwm_filename']
     output_filename = args['output_filename']
@@ -119,7 +153,7 @@ def filter_gage_branches(fim_inputs_filename):
 
     if not filter_fim_inputs:
 
-        usgs_gage_subset = Gage2Branch(usgs_gages_filename, nws_lid_filename, huc8)
+        usgs_gage_subset = Gage2Branch(usgs_gages_filename, ras_locs_filename, nws_lid_filename, huc8)
         if usgs_gage_subset.gages.empty:
             print(f'There are no gages identified for {huc8}')
             os._exit(0)
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index db166442a..62856f730 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -27,8 +27,9 @@
 # DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
 
 
-def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, inc_mosaic, job_number):
-    assert os.path.isdir(fim_run_dir), f"ERROR: could not find the input fim_dir location: {fim_run_dir}"
+def inundate_nation(fim_run_dir, output_dir, magnitude_key, 
+                    flow_file, huc_list, inc_mosaic, job_number):
+
 
     assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}"
 
@@ -68,11 +69,16 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, inc_mosai
         shutil.rmtree(magnitude_output_dir, ignore_errors=True)
         os.mkdir(magnitude_output_dir)
 
-    huc_list = []
-    for huc in os.listdir(fim_run_dir):
-        # if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
-        if re.match("\d{8}", huc):
-            huc_list.append(huc)
+    if huc_list == None:    
+        huc_list = []
+        for huc in os.listdir(fim_run_dir):
+            
+            #if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
+            if re.match('\d{8}', huc):    
+                huc_list.append(huc)
+    else:
+        for huc in huc_list:
+            assert os.path.isdir(fim_run_dir + os.sep + huc), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}'
 
     print("Inundation raw mosaic outputs here: " + magnitude_output_dir)
 
@@ -220,61 +226,32 @@ def __setup_logger(output_folder_path, log_file_name_key):
     available_cores = multiprocessing.cpu_count()
 
     # Parse arguments.
-    parser = argparse.ArgumentParser(
-        description="Inundation mapping for FOSS FIM using streamflow "
-        "recurrence interflow data. Inundation outputs are stored in the "
-        "/inundation_review/inundation_nwm_recurr/ directory."
-    )
-
-    parser.add_argument(
-        "-r",
-        "--fim-run-dir",
-        help="Name of directory containing outputs "
-        "of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)",
-        required=True,
-    )
-
-    parser.add_argument(
-        "-o",
-        "--output-dir",
-        help="Optional: The path to a directory to write the "
-        "outputs. If not used, the inundation_nation directory is used by default "
-        "ie) /data/inundation_review/inundate_nation/",
-        default="/data/inundation_review/inundate_nation/",
-        required=False,
-    )
-
-    parser.add_argument(
-        "-m",
-        "--magnitude_key",
-        help="used in output folders names and temp files, "
-        "added to output_file_name_key ie 100_0, 2_0, hw, etc)",
-        required=True,
-    )
-
-    parser.add_argument(
-        "-f",
-        "--flow_file",
-        help="the path and flow file to be used. "
-        "ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv",
-        required=True,
-    )
-
-    parser.add_argument(
-        "-s",
-        "--inc_mosaic",
-        help="Optional flag to produce mosaic of FIM extent rasters",
-        action="store_true",
-    )
-
-    parser.add_argument(
-        "-j",
-        "--job-number",
-        help="The number of jobs",
-        required=False,
-        default=1,
-        type=int,
-    )
+    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow '\
+        'recurrence interflow data. Inundation outputs are stored in the '\
+        '/inundation_review/inundation_nwm_recurr/ directory.')
+        
+    parser.add_argument('-r', '--fim-run-dir', help='Name of directory containing outputs '\
+        'of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)', required=True)
+        
+    parser.add_argument('-o', '--output-dir', help='Optional: The path to a directory to write the '\
+        'outputs. If not used, the inundation_nation directory is used by default '\
+        'ie) /data/inundation_review/inundate_nation/',
+        default='/data/inundation_review/inundate_nation/', required=False)
+       
+    parser.add_argument('-m', '--magnitude_key', help = 'used in output folders names and temp files, '\
+        'added to output_file_name_key ie 100_0, 2_0, hw, etc)', required = True)
+        
+    parser.add_argument('-f', '--flow_file', help = 'the path and flow file to be used. '\
+        'ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv', required = True)
+    
+    parser.add_argument('-l', '--huc-list', help = 'OPTIONAL: HUC list to run specified HUC(s). Specifiy multiple hucs single space delimited'\
+        '--> 12090301 12090302. Default (no huc list provided) will use hucs found in -r directory', required = False, default='all',nargs='+')
+        
+    parser.add_argument('-s', '--inc_mosaic',help='Optional flag to produce mosaic of FIM extent rasters',
+                        action='store_true')
+                        
+    parser.add_argument('-j', '--job-number', help='The number of jobs', required=False, default=1, type=int)
+        
 
     args = vars(parser.parse_args())
 

From d009cd688444e34c6fc4754e5944b96e3266c0c2 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 6 Sep 2023 13:51:12 -0400
Subject: [PATCH 273/359] v4.4.0.1 Bug fix for `src/usgs_gage_unit_setup.py`
 (#987)

---
 CITATION.cff                |  2 +-
 docs/CHANGELOG.md           | 10 ++++++++++
 src/usgs_gage_unit_setup.py |  2 +-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 3eb8c7de6..902a93280 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.3.11.7
+version: 4.4.0.1
 date-released: 2023
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d2c1eb2b8..4ab8f1635 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.0.1 - 2023-09-06 - [PR#987](https://github.com/NOAA-OWP/inundation-mapping/pull/987)
+
+Corrects a bug in `src/usgs_gage_unit_setup.py` that causes incorrect values to populate a table, generating an error in `src/usgs_gage_crosswalk.py`.
+
+### Changes
+
+- `src/usgs_gage_unit_setup.py`: Changes `self.gages.location_id.fillna(usgs_gages.nws_lid, inplace=True)` to `self.gages.location_id.fillna(self.gages.nws_lid, inplace=True)`
+
+<br/><br/>
+
 ## v4.4.0.0 - 2023-09-01 - [PR#965](https://github.com/NOAA-OWP/inundation-mapping/pull/965)
 
 This feature branch includes new functionality to perform an additional layer of HAND SRC calibration using ras2fim rating curve and point data. The calibration workflow for ras2fim data follows the same general logic as the existing USGS rating curve calibration routine. 
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index e6186c744..355ffc3a7 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -67,7 +67,7 @@ def load_gages(self):
             self.gages = pd.concat([self.gages, ahps_sites[['feature_id', 'nws_lid', 'location_id', 'HUC8', 'name', 'states','geometry']]])
 
         # Create gages attribute
-        self.gages.location_id.fillna(usgs_gages.nws_lid, inplace=True)
+        self.gages.location_id.fillna(self.gages.nws_lid, inplace=True)
         self.gages.loc[self.gages['nws_lid'] == 'Bogus_ID', 'nws_lid'] = None
 
     def sort_into_branch(self, nwm_subset_streams_levelPaths):

From 318ec8dfa10ba4705276cf7eb76e0e82ef0dc479 Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Mon, 18 Sep 2023 12:41:31 -0600
Subject: [PATCH 274/359] v4.4.1.0 Format and Style Repository (#988)

---
 .gitattributes                                |    2 +-
 .github/ISSUE_TEMPLATE.md                     |    3 -
 .github/PULL_REQUEST_TEMPLATE.md              |   12 +-
 .gitignore                                    |    1 -
 .isort.cfg                                    |    3 -
 .pre-commit-config.yaml                       |   36 +
 CITATION.cff                                  |    2 +-
 CONTRIBUTING.md                               |   70 +-
 Dockerfile                                    |    6 +-
 Pipfile                                       |    5 +
 Pipfile.lock                                  |  809 +++++++-----
 README.md                                     |   30 +-
 config/aws_s3_put_fim3_hydrovis_whitelist.lst |    2 +-
 config/aws_s3_put_fim4_hydrovis_whitelist.lst |    2 +-
 config/deny_branches.lst                      |   14 +-
 config/deny_unit.lst                          |   36 +-
 config/params_template.env                    |   39 +-
 data/aws/.gitignore                           |    2 +-
 data/aws/aws_base.py                          |   75 +-
 data/aws/aws_creds_template.env               |    2 +-
 data/aws/push-hv-data-support-files.sh        |   30 +-
 data/aws/s3.py                                |  192 ++-
 data/bathymetry/preprocess_bathymetry.py      |  107 +-
 data/ble/ble_benchmark/README.md              |   22 +-
 .../ble/ble_benchmark/create_ble_benchmark.py |  128 +-
 .../create_flow_forecast_file.py              |  155 ++-
 .../ble/ble_benchmark/preprocess_benchmark.py |   92 +-
 data/create_vrt_file.py                       |   61 +-
 data/esri.py                                  |   38 +-
 data/nld/levee_download.py                    |   75 +-
 data/nws/.gitignore                           |    4 +
 data/nws/preprocess_ahps_nws.py               |  385 ------
 data/usgs/acquire_and_preprocess_3dep_dems.py |  370 +++---
 data/usgs/preprocess_ahps_usgs.py             |  547 ++++----
 data/usgs/preprocess_download_usgs_grids.py   |  119 +-
 data/wbd/preprocess_wbd.py                    |   10 +-
 data/write_parquet_from_calib_pts.py          |  316 +++--
 docs/CHANGELOG.md                             |  372 +++---
 docs/SECURITY.md                              |   13 +-
 fim_pipeline.sh                               |  110 +-
 fim_post_processing.sh                        |   93 +-
 fim_pre_processing.sh                         |  107 +-
 fim_process_unit_wb.sh                        |   61 +-
 pyproject.toml                                |   88 ++
 src/add_crosswalk.py                          |  486 ++++---
 src/adjust_headwater_streams.py               |  203 ---
 src/adjust_thalweg_lateral.py                 |  146 ++-
 src/aggregate_branch_lists.py                 |   30 +-
 src/aggregate_by_huc.py                       |  457 ++++---
 src/aggregate_fim_outputs.py                  |  159 ++-
 src/aggregate_vector_inputs.py                |  355 ------
 src/agreedem.py                               |  209 +--
 src/associate_levelpaths_with_levees.py       |  115 +-
 src/bash_functions.env                        |    4 +-
 src/bathy_rc_adjust.py                        |  328 +++--
 src/bathy_src_adjust_topwidth.py              |  595 ++++++---
 src/bathymetric_adjustment.py                 |  164 ++-
 src/buffer_stream_branches.py                 |    7 +-
 src/build_stream_traversal.py                 |  139 +-
 src/burn_in_levees.py                         |    6 +-
 src/check_huc_inputs.py                       |   51 +-
 src/check_unit_errors.py                      |   62 +-
 src/clip_rasters_to_branches.py               |   62 +-
 src/clip_vectors_to_wbd.py                    |  229 ++--
 src/crosswalk_nwm_demDerived.py               |  203 +--
 src/delineate_hydros_and_produce_HAND.sh      |  163 ++-
 src/derive_headwaters.py                      |   57 +-
 src/derive_level_paths.py                     |  112 +-
 src/edit_points.py                            |   56 +-
 src/filter_catchments_and_add_attributes.py   |   48 +-
 src/filter_inputs_by_huc.py                   |   24 +-
 src/finalize_srcs.py                          |   90 +-
 src/generate_branch_list.py                   |    2 +-
 src/generate_branch_list_csv.py               |   36 +-
 src/getRasterInfoNative.py                    |  147 ++-
 src/identify_src_bankfull.py                  |  366 ++++--
 src/make_rem.py                               |  133 +-
 src/make_stages_and_catchlist.py              |   48 +-
 src/mask_dem.py                               |   58 +-
 src/outputs_cleanup.py                        |   55 +-
 src/process_branch.sh                         |   16 +-
 src/query_vectors_by_branch_polygons.py       |   83 +-
 src/reachID_grid_to_vector_points.py          |   80 +-
 src/reset_mannings.py                         |  263 ++--
 src/run_by_branch.sh                          |   46 +-
 src/run_unit_wb.sh                            |  138 +-
 src/split_flows.py                            |  185 +--
 src/src_adjust_ras2fim_rating.py              |  344 +++--
 src/src_adjust_spatial_obs.py                 |  320 +++--
 src/src_adjust_usgs_rating.py                 |  341 +++--
 src/src_manual_calibration.py                 |  158 +--
 src/src_roughness_optimization.py             |  739 ++++++++---
 src/stream_branches.py                        |  110 +-
 src/subdiv_chan_obank_src.py                  |  481 +++++--
 src/subset_catch_list_by_branch_id.py         |   72 +-
 src/toDo.md                                   |    4 +-
 src/unique_pixel_and_allocation.py            |   36 +-
 src/usgs_gage_crosswalk.py                    |  129 +-
 src/usgs_gage_unit_setup.py                   |  136 +-
 src/utils/fim_enums.py                        |    9 +-
 src/utils/reproject_dem.py                    |   56 -
 src/utils/shared_functions.py                 |  350 ++---
 src/utils/shared_variables.py                 |  121 +-
 tools/adjust_rc_with_feedback.py              |  241 ++--
 tools/aggregate_csv_files.py                  |   88 +-
 tools/aggregate_mannings_calibration.py       |  216 +++-
 tools/aggregate_metrics.py                    |  210 +--
 tools/cache_metrics.py                        |   67 +-
 tools/check_deep_flooding.py                  |  102 +-
 tools/code_standardizer/Dockerfile            |   12 -
 tools/code_standardizer/entrypoint.sh         |   17 -
 tools/code_standardizer/requirements.txt      |    3 -
 tools/combine_crosswalk_tables.py             |   25 +-
 tools/compare_ms_and_non_ms_metrics.py        |   34 +-
 tools/compile_comp_stats.py                   |   92 --
 tools/compile_computational_stats.py          |   48 -
 tools/composite_inundation.py                 |   94 +-
 tools/consolidate_metrics.py                  |  339 -----
 tools/copy_test_case_folders.py               |  117 --
 tools/create_flow_forecast_file.py            |  171 +++
 tools/cygnss_preprocessing.py                 |  242 ----
 tools/eval_alt_catfim.py                      |  290 +++--
 tools/eval_plots.py                           |  785 ++++++++----
 tools/eval_plots_stackedbar.py                |  254 ++--
 tools/evaluate_continuity.py                  |  114 +-
 tools/fim_completion_check.py                 |   10 +-
 tools/fimr_to_benchmark.py                    |  172 ++-
 tools/find_max_catchment_breadth.py           |   43 +-
 tools/find_test_case_folders.py               |   79 +-
 tools/generate_categorical_fim.py             |  883 ++++++++-----
 tools/generate_categorical_fim_flows.py       |  328 +++--
 tools/generate_categorical_fim_mapping.py     |  298 +++--
 tools/generate_nws_lid.py                     |  226 ++--
 tools/hash_compare.py                         |  200 +++
 tools/inundate_events.py                      |  120 +-
 tools/inundate_gms.py                         |   60 +-
 tools/inundate_mosaic_wrapper.py              |   76 +-
 tools/inundate_nation.py                      |  158 ++-
 tools/inundation.py                           |  756 ++++++-----
 tools/inundation_wrapper_custom_flow.py       |  142 ++-
 tools/inundation_wrapper_nwm_flows.py         |  171 ++-
 tools/make_boxes_from_bounds.py               |   87 +-
 tools/mosaic_inundation.py                    |   23 +-
 tools/nesdis_preprocessing.py                 |  157 ---
 tools/overlapping_inundation.py               |   34 +-
 tools/pixel_counter.py                        |  125 +-
 tools/pixel_counter_functions.py              |   58 +-
 tools/pixel_counter_wrapper.py                |   80 +-
 tools/plots.py                                |  307 +++--
 tools/plots/eval_plots.py                     |  343 -----
 tools/plots/utils/__init__.py                 |    0
 tools/plots/utils/shared_functions.py         |  321 -----
 tools/rating_curve_comparison.py              | 1084 +++++++++++-----
 tools/rating_curve_get_usgs_curves.py         |  397 +++---
 tools/run_test_case.py                        |  165 +--
 tools/synthesize_test_cases.py                |  554 +++++---
 tools/test_case_by_hydro_id.py                |  316 +++--
 tools/tools_shared_functions.py               | 1135 +++++++++--------
 tools/tools_shared_variables.py               |   43 +-
 tools/vary_mannings_n_composite.py            |  416 ++++--
 unit_tests/README.md                          |   92 +-
 unit_tests/__template.json                    |   19 +-
 unit_tests/__template.py                      |   93 +-
 unit_tests/aggregate_branch_lists_params.json |   11 +-
 unit_tests/aggregate_branch_lists_test.py     |   22 +-
 unit_tests/check_unit_errors_params.json      |   15 +-
 unit_tests/check_unit_errors_test.py          |  142 +--
 unit_tests/clip_vectors_to_wbd_params.json    |   51 +-
 unit_tests/clip_vectors_to_wbd_test.py        |   76 +-
 unit_tests/derive_level_paths_params.json     |   25 +-
 unit_tests/derive_level_paths_test.py         |   48 +-
 ..._catchments_and_add_attributes_params.json |   19 +-
 ...lter_catchments_and_add_attributes_test.py |   58 +-
 .../generate_branch_list_csv_params.json      |   33 +-
 unit_tests/generate_branch_list_csv_test.py   |   64 +-
 unit_tests/generate_branch_list_params.json   |   11 +-
 unit_tests/generate_branch_list_test.py       |   24 +-
 unit_tests/outputs_cleanup_params.json        |   43 +-
 unit_tests/outputs_cleanup_test.py            |  178 +--
 unit_tests/pyproject.toml                     |    2 +-
 .../rating_curve_comparison_params.json       |   34 +-
 unit_tests/rating_curve_comparison_test.py    |   48 +-
 unit_tests/shared_functions_params.json       |   32 +-
 unit_tests/shared_functions_test.py           |   71 +-
 unit_tests/split_flows_params.json            |   27 +-
 unit_tests/split_flows_test.py                |   68 +-
 unit_tests/tools/inundate_gms_params.json     |   11 +-
 unit_tests/tools/inundate_gms_test.py         |   79 +-
 unit_tests/tools/inundation_params.json       |   19 +-
 unit_tests/tools/inundation_test.py           |   76 +-
 unit_tests/unit_tests_utils.py                |   18 +-
 unit_tests/usgs_gage_crosswalk_params.json    |   19 +-
 unit_tests/usgs_gage_crosswalk_test.py        |   44 +-
 193 files changed, 16280 insertions(+), 12488 deletions(-)
 delete mode 100755 .isort.cfg
 create mode 100644 .pre-commit-config.yaml
 create mode 100644 data/nws/.gitignore
 delete mode 100644 data/nws/preprocess_ahps_nws.py
 create mode 100644 pyproject.toml
 delete mode 100644 src/adjust_headwater_streams.py
 delete mode 100755 src/aggregate_vector_inputs.py
 delete mode 100755 src/utils/reproject_dem.py
 delete mode 100755 tools/code_standardizer/Dockerfile
 delete mode 100755 tools/code_standardizer/entrypoint.sh
 delete mode 100755 tools/code_standardizer/requirements.txt
 delete mode 100755 tools/compile_comp_stats.py
 delete mode 100755 tools/compile_computational_stats.py
 delete mode 100755 tools/consolidate_metrics.py
 delete mode 100644 tools/copy_test_case_folders.py
 create mode 100755 tools/create_flow_forecast_file.py
 delete mode 100755 tools/cygnss_preprocessing.py
 create mode 100755 tools/hash_compare.py
 delete mode 100755 tools/nesdis_preprocessing.py
 delete mode 100755 tools/plots/eval_plots.py
 delete mode 100644 tools/plots/utils/__init__.py
 delete mode 100644 tools/plots/utils/shared_functions.py

diff --git a/.gitattributes b/.gitattributes
index 72db7b08c..6033e76e7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,2 @@
 # Set the default behavior for end of lines.
-* text eol=lf
\ No newline at end of file
+* text eol=lf
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 63a2ff5ad..44d746703 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -12,6 +12,3 @@ Short description explaining the high-level reason for the new issue.
 
 
 ## Screenshots
-
-
-
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 350a5d6ee..4cb221383 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,10 +1,10 @@
 [Short description explaining the high-level reason for the pull request]
 
-### Additions  
+### Additions
 
-### Changes  
+### Changes
 
-### Removals 
+### Removals
 
 ### Testing
 
@@ -18,17 +18,17 @@ _You may update this checklist before and/or after creating the PR. If you're un
 - [ ] If submitting a PR to the `dev` branch (the default branch), you have a descriptive [Feature Branch](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow) name using the format: `dev-<description-of-change>` (e.g. `dev-revise-levee-masking`)
 - [ ] Changes are limited to a single goal (no scope creep)
 - [ ] The feature branch you're submitting as a PR is up to date (merged) with the latest `dev` branch
-- [ ] Changes adhere to [PEP-8 Style Guidelines](https://peps.python.org/pep-0008/)
+- [ ] `pre-commit` hooks were run locally
 - [ ] Any _change_ in functionality is tested
 - [ ] Passes all unit tests locally (inside interactive Docker container, at `/foss_fim/`, run: `pytest unit_tests/`)
 - [ ] New functions are documented (with a description, list of inputs, and expected output)
 - [ ] Placeholder code is flagged / future todos are captured in comments
-- [ ] [CHANGELOG](/docs/CHANGELOG.md) updated with template version number, e.g. `4.x.x.x` 
+- [ ] [CHANGELOG](/docs/CHANGELOG.md) updated with template version number, e.g. `4.x.x.x`
 - [ ] [Reviewers requested](https://help.github.com/articles/requesting-a-pull-request-review/)
 - [ ] Add yourself as an [assignee](https://docs.github.com/en/issues/tracking-your-work-with-issues/assigning-issues-and-pull-requests-to-other-github-users) in the PR  as well as the FIM Technical Lead
 
 ### Merge Checklist (For Technical Lead use only)
 
-- [ ] Update [CHANGELOG](/docs/CHANGELOG.md) with latest version number
+- [ ] Update [CHANGELOG](/docs/CHANGELOG.md) with latest version number and merge date
 - [ ] Update the [Citation.cff](/CITATION.cff) file to reflect the latest version number in the [CHANGELOG](/docs/CHANGELOG.md)
 - [ ] If applicable, update [README](/README.md) with major alterations
diff --git a/.gitignore b/.gitignore
index 89bae3092..7ce22521c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,4 +15,3 @@ config/*.swp
 **/.DS_Store
 **/*_pytest.py
 .private/
-
diff --git a/.isort.cfg b/.isort.cfg
deleted file mode 100755
index aa7e7666d..000000000
--- a/.isort.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-[tool.isort]
-profile = "black"
-multi_line_output = 3
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..5d4df4c20
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,36 @@
+# For more information: https://pre-commit.com
+# Additional hooks: https://pre-commit.com/hooks.html
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+        -   id: trailing-whitespace
+            exclude: CHANGELOG.md
+        -   id: end-of-file-fixer
+            exclude: Pipfile.lock
+        -   id: check-added-large-files
+            args: ['--maxkb=5000']
+        -   id: check-json
+-   repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+        -   id: flake8
+            entry: pflake8
+            additional_dependencies: [pyproject-flake8]
+-   repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+        -   id: black
+            args: ['--line-length=110', '--skip-magic-trailing-comma', '--skip-string-normalization']
+            exclude: |
+                (?x)^(
+                    src/add_crosswalk.py|
+                    src/bathymetric_adjustment.py|
+                    src/identify_src_bankfull.py
+                )$
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
diff --git a/CITATION.cff b/CITATION.cff
index 902a93280..8498f274d 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.4.0.1
+version: 4.4.1.0
 date-released: 2023
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0f109b26f..5cb51d7b5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -7,8 +7,8 @@
 
 
 There are two primary ways to help:
- - Using the issue tracker, and
- - Changing the code-base.
+ - Using the issue tracker (Bug Reporting, Submitting Feedback)
+ - Changing the code-base (Bug fixes, Feature Implementation & Improving Documentation)
 
 
 ## Using the issue tracker
@@ -26,7 +26,71 @@ guidance below.
 
 Generally speaking, you should fork this repository, make changes in your
 own fork, and then submit a pull request. All new code should have associated
-unit tests that validate implemented features and the presence or lack of defects.
+unit tests (added to `/unit_tests`) that validate implemented features and the presence or lack of defects.
 Additionally, the code should follow any stylistic and architectural guidelines
 prescribed by the project. In the absence of such guidelines, mimic the styles
 and patterns in the existing code-base.
+
+### Guidelines
+
+If you would like to contribute, please follow these steps:
+
+1. Fork the project on the GitHub webpage.
+2. Clone your fork:
+    ```
+    $ git clone https://github.com/<github username>/inundation-mapping.git
+    $ cd inundation-mapping
+    ```
+3. Create a local branch:
+    ```
+    git checkout -b <dev-your-bugfix-or-feature>
+    ```
+4. [pre-commit](https://pre-commit.com/) is used to run auto-formatting and linting tools. Please follow the steps in the link provided to install `pre-commit` locally.  `pre-commit` can be installed as a git hook to verify your staged changes adhere to the project's style and format requirements (configuration defined in [pyproject.toml](/pyproject.toml)).
+5. After you've installed `pre-commit` via `pip` , `homebrew`, or `conda`, check the version to verify installation:
+    ```
+    $ pre-commit --version
+    ```
+
+6. Initialize the pre-commit hooks included within the root directory of `inundation-mapping`:
+
+    ```
+    $ pre-commit install
+    ```
+7. At this point, you should be set up with `pre-commit`. When a commit is made it will run the pre-commit hooks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). For reference, you may run any of the pre-commit hooks manually before issuing the `git commit` command (see below). Some tools used by the pre commit git hook (`isort`, `flake8`, & `black`) are also available as command line executables within the Docker container, however, it is recommend to run them through `pre-commit` outside of the container, as it picks up the correct configuration.
+
+    ```
+    # Check only the staged changes
+    pre-commit run
+
+    # Check all files in the repo
+    pre-commit run -a
+
+    # Run only the black formatting tool
+    pre-commit run -a flake8
+    ```
+
+8. Build the Docker container:
+    ```
+    Docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>
+    ```
+9. [Within the container](README.md#startrun-the-docker-container), ensure sure unit tests pass ([instructions here](/unit_tests/README.md)).
+    ```
+    pytest unit_tests/
+    ```
+
+10. Outside of the Docker container, commit your changes:
+    ```
+    git commit -m <descriptive sentence or two changes>
+    ```
+    This will invoke pre-commit hooks mentioned in step 7 that will lint & format the code. In many cases non-compliant code will be rectified automatically, but in some cases manual changes will be necessary. Make sure all of these checks pass, if not, make necessary changes and re-issue `git commit -m <...>`.
+
+9. Push to your forked branch:
+    ```
+    git push -u origin
+    ```
+    or if the branch is not pushed up yet:
+    ```
+    git push --set-upstream origin <your branch>
+    ```
+
+10. Submit a pull request on [inundation-mapping's GitHub page](https://github.com/NOAA-OWP/inundation-mapping) (please review checklist in [PR template](/.github/PULL_REQUEST_TEMPLATE.md) for additional PR guidance).
diff --git a/Dockerfile b/Dockerfile
index 069da2abd..101fcb2e0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -70,7 +70,7 @@ RUN mkdir -p $workDir
 RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
-RUN apt update --fix-missing 
+RUN apt update --fix-missing
 RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync tzdata
 
 RUN apt auto-remove
@@ -87,7 +87,7 @@ ENV PYTHONUNBUFFERED=TRUE
 
 ## ADD TO PATHS ##
 ENV PATH="$projectDir:${PATH}"
-ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
+ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/unit_tests:$projectDir/tools
 
 ## install python 3 modules ##
 
@@ -104,7 +104,7 @@ RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --syste
 # Instead we will setup the WBT engine at time of docker build (same as Taudem and AWS).
 # Whitebox code detects that the engine it there and makes no attempt to update it.
 # We download and unzip it to the same file folder that pip deployed the whitebox library.
-# Whitebox also attempts to always download a folder called testdata regardless of use. 
+# Whitebox also attempts to always download a folder called testdata regardless of use.
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
 RUN wbox_path=/usr/local/lib/python3.8/dist-packages/whitebox/ && \
     wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
diff --git a/Pipfile b/Pipfile
index 9e62fe41d..3a4e3c353 100755
--- a/Pipfile
+++ b/Pipfile
@@ -38,6 +38,11 @@ rtree = "==1.0.1"
 py7zr = "==0.20.4"
 scipy = "==1.10.1"
 gval = "==0.1.2"
+flake8 = "==6.0.0"
+black = "==23.7.0"
+pyproject-flake8 = "==6.0.0.post1"
+pre-commit = "==3.3.3"
+isort = "==5.12.0"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 3bbb3a49f..9937efe2c 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "6ea9602b1a3842fbcfde2289f3b414315cbf2d38875ec5cbd738830fb7fe19fc"
+            "sha256": "32dc0ff07bc71eeec7a7bc1c64d580d05ce81cceaa3ecfc2e3b4f067e4f5c396"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4'",
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -57,11 +57,11 @@
         },
         "argon2-cffi": {
             "hashes": [
-                "sha256:8c976986f2c5c0e5000919e6de187906cfd81fb1c72bf9d88c01177e77da7f80",
-                "sha256:d384164d944190a7dd7ef22c6aa3ff197da12962bd04b17f64d4e93d934dba5b"
+                "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08",
+                "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==21.3.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==23.1.0"
         },
         "argon2-cffi-bindings": {
             "hashes": [
@@ -136,6 +136,34 @@
             "markers": "python_version >= '3.6'",
             "version": "==4.12.2"
         },
+        "black": {
+            "hashes": [
+                "sha256:01ede61aac8c154b55f35301fac3e730baf0c9cf8120f65a9cd61a81cfb4a0c3",
+                "sha256:022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb",
+                "sha256:25cc308838fe71f7065df53aedd20327969d05671bac95b38fdf37ebe70ac087",
+                "sha256:27eb7a0c71604d5de083757fbdb245b1a4fae60e9596514c6ec497eb63f95320",
+                "sha256:327a8c2550ddc573b51e2c352adb88143464bb9d92c10416feb86b0f5aee5ff6",
+                "sha256:47e56d83aad53ca140da0af87678fb38e44fd6bc0af71eebab2d1f59b1acf1d3",
+                "sha256:501387a9edcb75d7ae8a4412bb8749900386eaef258f1aefab18adddea1936bc",
+                "sha256:552513d5cd5694590d7ef6f46e1767a4df9af168d449ff767b13b084c020e63f",
+                "sha256:5c4bc552ab52f6c1c506ccae05681fab58c3f72d59ae6e6639e8885e94fe2587",
+                "sha256:642496b675095d423f9b8448243336f8ec71c9d4d57ec17bf795b67f08132a91",
+                "sha256:6d1c6022b86f83b632d06f2b02774134def5d4d4f1dac8bef16d90cda18ba28a",
+                "sha256:7f3bf2dec7d541b4619b8ce526bda74a6b0bffc480a163fed32eb8b3c9aed8ad",
+                "sha256:831d8f54c3a8c8cf55f64d0422ee875eecac26f5f649fb6c1df65316b67c8926",
+                "sha256:8417dbd2f57b5701492cd46edcecc4f9208dc75529bcf76c514864e48da867d9",
+                "sha256:86cee259349b4448adb4ef9b204bb4467aae74a386bce85d56ba4f5dc0da27be",
+                "sha256:893695a76b140881531062d48476ebe4a48f5d1e9388177e175d76234ca247cd",
+                "sha256:9fd59d418c60c0348505f2ddf9609c1e1de8e7493eab96198fc89d9f865e7a96",
+                "sha256:ad0014efc7acf0bd745792bd0d8857413652979200ab924fbf239062adc12491",
+                "sha256:b5b0ee6d96b345a8b420100b7d71ebfdd19fab5e8301aff48ec270042cd40ac2",
+                "sha256:c333286dc3ddca6fdff74670b911cccedacb4ef0a60b34e491b8a67c833b343a",
+                "sha256:f9062af71c59c004cd519e2fb8f5d25d39e46d3af011b41ab43b9c74e27e236f",
+                "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995"
+            ],
+            "index": "pypi",
+            "version": "==23.7.0"
+        },
         "bleach": {
             "hashes": [
                 "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414",
@@ -168,7 +196,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4'",
+            "markers": "python_version >= '3.8' and python_version < '4.0'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -360,6 +388,14 @@
             ],
             "version": "==1.15.1"
         },
+        "cfgv": {
+            "hashes": [
+                "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9",
+                "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.4.0"
+        },
         "cftime": {
             "hashes": [
                 "sha256:055d5d60a756c6c1857cf84d77655bb707057bb6c4a4fbb104a550e76c40aad9",
@@ -470,11 +506,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
-                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
+                "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
+                "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.6"
+            "version": "==8.1.7"
         },
         "click-plugins": {
             "hashes": [
@@ -488,7 +524,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -578,67 +614,67 @@
         },
         "cython": {
             "hashes": [
-                "sha256:06db81b1a01858fcc406616f8528e686ffb6cf7c3d78fb83767832bfecea8ad8",
-                "sha256:06fcb4628ccce2ba5abc8630adbeaf4016f63a359b4c6c3827b2d80e0673981c",
-                "sha256:090256c687106932339f87f888b95f0d69c617bc9b18801555545b695d29d8ab",
-                "sha256:090e24cfa31c926d0b13d8bb2ef48175acdd061ae1413343c94a2b12a4a4fa6f",
-                "sha256:0d2c1e172f1c81bafcca703093608e10dc16e3e2d24c5644c17606c7fdb1792c",
-                "sha256:0e1e4b7e4bfbf22fecfa5b852f0e499c442d4853b7ebd33ae37cdec9826ed5d8",
-                "sha256:204690be60f0ff32eb70b04f28ef0d1e50ffd7b3f77ba06a7dc2389ee3b848e0",
-                "sha256:20c604e974832aaf8b7a1f5455ee7274b34df62a35ee095cd7d2ed7e818e6c53",
-                "sha256:254ed1f03a6c237fa64f0c6e44862058de65bfa2e6a3b48ca3c205492e0653aa",
-                "sha256:296c53b6c0030cf82987eef163444e8d7631cc139d995f9d58679d9fd1ddbf31",
-                "sha256:2d8158277c8942c0b20ff4c074fe6a51c5b89e6ac60cef606818de8c92773596",
-                "sha256:2fadde1da055944f5e1e17625055f54ddd11f451889110278ef30e07bd5e1695",
-                "sha256:30a4bd2481e59bd7ab2539f835b78edc19fc455811e476916f56026b93afd28b",
-                "sha256:30f10e79393b411af7677c270ea69807acb9fc30205c8ff25561f4deef780ec1",
-                "sha256:33c900d1ca9f622b969ac7d8fc44bdae140a4a6c7d8819413b51f3ccd0586a09",
-                "sha256:350b18f9673e63101dbbfcf774ee2f57c20ac4636d255741d76ca79016b1bd82",
-                "sha256:35abcf07b8277ec95bbe49a07b5c8760a2d941942ccfe759a94c8d2fe5602e9f",
-                "sha256:3b71b399b10b038b056ad12dce1e317a8aa7a96e99de7e4fa2fa5d1c9415cfb9",
-                "sha256:3e234e2549e808d9259fdb23ebcfd145be30c638c65118326ec33a8d29248dc2",
-                "sha256:4123c8d03167803df31da6b39de167cb9c04ac0aa4e35d4e5aa9d08ad511b84d",
-                "sha256:4965f2ebade17166f21a508d66dd60d2a0b3a3b90abe3f72003baa17ae020dd6",
-                "sha256:4cd00f2158dc00f7f93a92444d0f663eda124c9c29bbbd658964f4e89c357fe8",
-                "sha256:4dc6bbe7cf079db37f1ebb9b0f10d0d7f29e293bb8688e92d50b5ea7a91d82f3",
-                "sha256:4e212237b7531759befb92699c452cd65074a78051ae4ee36ff8b237395ecf3d",
-                "sha256:53328a8af0806bebbdb48a4191883b11ee9d9dfb084d84f58fa5a8ab58baefc9",
-                "sha256:54e34f99b2a8c1e11478541b2822e6408c132b98b6b8f5ed89411e5e906631ea",
-                "sha256:5962e70b15e863e72bed6910e8c6ffef77d36cc98e2b31c474378f3b9e49b0e3",
-                "sha256:5b4cc896d49ce2bae8d6a030f9a4c64965b59c38acfbf4617685e17f7fcf1731",
-                "sha256:609777d3a7a0a23b225e84d967af4ad2485c8bdfcacef8037cf197e87d431ca0",
-                "sha256:626a4a6ef4b7ced87c348ea805488e4bd39dad9d0b39659aa9e1040b62bbfedf",
-                "sha256:650d03ddddc08b051b4659778733f0f173ca7d327415755c05d265a6c1ba02fb",
-                "sha256:6b00df42cdd1a285a64491ba23de08ab14169d3257c840428d40eb7e8e9979af",
-                "sha256:7c7d728e1a49ad01d41181e3a9ea80b8d14e825f4679e4dd837cbf7bca7998a5",
-                "sha256:7f4a6dfd42ae0a45797f50fc4f6add702abf46ab3e7cd61811a6c6a97a40e1a2",
-                "sha256:829c8333195100448a23863cf64a07e1334fae6a275aefe871458937911531b6",
-                "sha256:84176bd04ce9f3cc8799b47ec6d1959fa1ea5e71424507df7bbf0b0915bbedef",
-                "sha256:877d1c8745df59dd2061a0636c602729e9533ba13f13aa73a498f68662e1cbde",
-                "sha256:8abb8915eb2e57fa53d918afe641c05d1bcc6ed1913682ec1f28de71f4e3f398",
-                "sha256:8d86651347bbdbac1aca1824696c5e4c0a3b162946c422edcca2be12a03744d1",
-                "sha256:93a34e1ca8afa4b7075b02ed14a7e4969256297029fb1bfd4cbe48f7290dbcff",
-                "sha256:9e69139f4e60ab14c50767a568612ea64d6907e9c8e0289590a170eb495e005f",
-                "sha256:9f29307463eba53747b31f71394ed087e3e3e264dcc433e62de1d51f5c0c966c",
-                "sha256:a1cdd01ce45333bc264a218c6e183700d6b998f029233f586a53c9b13455c2d2",
-                "sha256:a44d6b9a29b2bff38bb648577b2fcf6a68cf8b1783eee89c2eb749f69494b98d",
-                "sha256:a65bc50dc1bc2faeafd9425defbdef6a468974f5c4192497ff7f14adccfdcd32",
-                "sha256:aa606675c6bd23478b1d174e2a84e3c5a2c660968f97dc455afe0fae198f9d3d",
-                "sha256:bb1165ca9e78823f9ad1efa5b3d83156f868eabd679a615d140a3021bb92cd65",
-                "sha256:bc816d8eb3686d6f8d165f4156bac18c1147e1035dc28a76742d0b7fb5b7c032",
-                "sha256:c40bdbcb2286f0aeeb5df9ce53d45da2d2a9b36a16b331cd0809d212d22a8fc7",
-                "sha256:c85fd2b1cbd9400d60ebe074795bb9a9188752f1612be3b35b0831a24879b91f",
-                "sha256:c93634845238645ce7abf63a56b1c5b6248189005c7caff898fd4a0dac1c5e1e",
-                "sha256:cec2a67a0a7d9d4399758c0657ca03e5912e37218859cfbf046242cc532bfb3b",
-                "sha256:d3355e6f690184f984eeb108b0f5bbc4bcf8b9444f8168933acf79603abf7baf",
-                "sha256:e28763e75e380b8be62b02266a7995a781997c97c119efbdccb8fb954bcd7574",
-                "sha256:ecee663d2d50ca939fc5db81f2f8a219c2417b4651ad84254c50a03a9cb1aadd",
-                "sha256:edae615cb4af51d5173e76ba9aea212424d025c57012e9cdf2f131f774c5ba71",
-                "sha256:f42f304c097cc53e9eb5f1a1d150380353d5018a3191f1b77f0de353c762181e",
-                "sha256:ff1aef1a03cfe293237c7a86ae9625b0411b2df30c53d1a7f29a8d381f38a1df"
+                "sha256:011c4e0b75baee1843334562487eb4fbc0c59ddb2cc32a978b972a81eedcbdcc",
+                "sha256:05cb2a73810f045d328b7579cf98f550a9e601df5e282d1fea0512d8ad589011",
+                "sha256:067b2b9eb487bd61367b296f11b7c1c70a084b3eb7d5a572f607cd1fc5ca5586",
+                "sha256:08d67c7225a09eeb77e090c8d4f60677165b052ccf76e3a57d8237064e5c2de2",
+                "sha256:10cbfb37f31938371a6213cc8b5459c639954aed053efeded3c012d4c5915db9",
+                "sha256:147cc1d3dda8b06de9d86df5e59cdf15f0a522620168b7349a5ec88b48104d7d",
+                "sha256:1b12a8f23270675b537d1c3b988f845bea4bbcc66ae0468857f5ede0526d4522",
+                "sha256:213ff9f95de319e54b520bf31edd6aa7a1fa4fbf617c2beb0f92362595e6476a",
+                "sha256:22ba78e48bdb65977928ecb275ac8c82df7b0eefa075078a1363a5af4606b42e",
+                "sha256:2f84bd6cefa5130750c492038170c44f1cbd6f42e9ed85e168fd9cb453f85160",
+                "sha256:302281b927409b3e0ef8cd9251eab782cf1acd2578eab305519fbae5d184b7e9",
+                "sha256:34f7b014ebce5d325c8084e396c81cdafbd8d82be56780dffe6b67b28c891f1b",
+                "sha256:38085523fa7a299638d051ae08144222785639882f6291bd275c0b12db1034ff",
+                "sha256:477cd3549597f09a1608da7b05e16ba641e9aedd171b868533a5a07790ed886f",
+                "sha256:4bebbca13078125a35937966137af4bd0300a0c66fd7ae4ce36adc049b13bdf3",
+                "sha256:4db017b104f47b1185237702f6ed2651839c8124614683efa7c489f3fa4e19d9",
+                "sha256:4dca13c86d6cd523c7d8bbf8db1b2bbf8faedd0addedb229158d8015ad1819e1",
+                "sha256:4fe806d154b6b7f0ab746dac36c022889e2e7cf47546ff9afdc29a62cfa692d0",
+                "sha256:54d41a1dfbaab74449873e7f8e6cd4239850fe7a50f7f784dd99a560927f3bac",
+                "sha256:550b3fbe9b3c555b44ded934f4822f9fcc04dfcee512167ebcbbd370ccede20e",
+                "sha256:5682293d344b7dbad97ce6eceb9e887aca6e53499709db9da726ca3424e5559d",
+                "sha256:62dd78afdf748a58dae9c9b9c42a1519ae30787b28ce5f84a0e1bb54144142ca",
+                "sha256:6b75e9c9d7ad7c9dd85d45241d1d4e3c5f66079c1f84eec91689c26d98bc3349",
+                "sha256:75a2395cc7b78cff59be6e9b7f92bbb5d7b8d25203f6d3fb6f72bdb7d3f49777",
+                "sha256:77ec0134fc1b10aebef2013936a91c07bff2498ec283bc2eca099ee0cb94d12e",
+                "sha256:786b6034a91e886116bb562fe42f8bf0f97c3e00c02e56791d02675959ed65b1",
+                "sha256:78e2853d484643c6b7ac3bdb48392753442da1c71b689468fa3176b619bebe54",
+                "sha256:7e08ff5da5f5b969639784b1bffcd880a0c0f048d182aed7cba9945ee8b367c2",
+                "sha256:7f43c4d3ecd9e3b8b7afe834e519f55cf4249b1088f96d11b96f02c55cbaeff7",
+                "sha256:809617cf4825b2138ce0ec827e1f28e39668743c81ac8286373f8d148c05f088",
+                "sha256:832bbee87bca760efeae248ddf19ccd77f9a2355cb6f8a64f20cc377e56957b3",
+                "sha256:8850269ff59f77a1629e26d0576701925360d732011d6d3516ccdc5b2c2bc310",
+                "sha256:8948504338d7a140ce588333177dcabf0743a68dbc83b0174f214f5b959634d5",
+                "sha256:8ccb91d2254e34724f1541b2a6fcdfacdb88284185b0097ae84e0ddf476c7a38",
+                "sha256:8f1c9e4b8e413da211dd7942440cf410ff0eafb081309e04e81f4fafbb146bf2",
+                "sha256:9594818dca8bb22ae6580c5222da2bc5cc32334350bd2d294a00d8669bcc61b5",
+                "sha256:989787fc24a95100a26918b6577d06e15a8868a3ed267009c5cfcf1a906179ac",
+                "sha256:9e625eec8c5c9a8cb062a318b257cc469d301bed952c7daf86e38bbd3afe7c91",
+                "sha256:a1c3675394b81024aaf56e4f53c2b4f81d9a116c7049e9d4706f810899c9134e",
+                "sha256:a49dde9f9e29ea82f29aaf3bb1a270b6eb90b75d627c7ff2f5dd3764540ae646",
+                "sha256:a51efba0e136b2af358e5a347bae09678b17460c35cf1eab24f0476820348991",
+                "sha256:ae453cfa933b919c0a19d2cc5dc9fb28486268e95dc2ab7a11ab7f99cf8c3883",
+                "sha256:b032cb0c69082f0665b2c5fb416d041157062f1538336d0edf823b9ee500e39c",
+                "sha256:b1f023d36a3829069ed11017c670128be3f135a9c17bd64c35d3b3442243b05c",
+                "sha256:bc1c8013fad0933f5201186eccc5f2be223cafd6a8dcd586d3f7bb6ba84dc845",
+                "sha256:c298b1589205ecaaed0457ad05e0c8a43e7db2053607f48ed4a899cb6aa114df",
+                "sha256:c5e722732e9aa9bde667ed6d87525234823eb7766ca234cfb19d7e0c095a2ef4",
+                "sha256:c90eeb94395315e65fd758a2f86b92904fce7b50060b4d45a878ef6767f9276e",
+                "sha256:cc9d173ab8b167cae674f6deed8c65ba816574797a2bd6d8aa623277d1fa81ca",
+                "sha256:d0d0cc4ecc05f41c5e02af14ac0083552d22efed976f79eb7bade55fed63b25d",
+                "sha256:d21801981db44b7e9f9768f121317946461d56b51de1e6eff3c42e8914048696",
+                "sha256:d5e5587128e8c2423aefcffa4ded4ddf60d44898938fbb7c0f236636a750a94f",
+                "sha256:dab6a923e21e212aa3dc6dde9b22a190f5d7c449315a94e57ddc019ea74a979b",
+                "sha256:dd30826ca8b27b2955a63c8ffe8aacc9f0779582b4bd154cf7b441ac10dae2cb",
+                "sha256:e486331a29e7700b1ad5f4f753bef483c81412a5e64a873df46d6cb66f9a65de",
+                "sha256:e663c237579c033deaa2cb362b74651da7712f56e441c11382510a8c4c4f2dd7",
+                "sha256:e825e682cef76d0c33384f38b56b7e87c76152482a914dfc78faed6ff66ce05a",
+                "sha256:f37e4287f520f3748a06ad5eaae09ba4ac68f52e155d70de5f75780d83575c43"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.0"
+            "version": "==3.0.2"
         },
         "dask": {
             "hashes": [
@@ -650,11 +686,11 @@
         },
         "datashader": {
             "hashes": [
-                "sha256:2eea890cd8ea74bdb8d5601fd844b875f298931225bea8d1fc9db786c112f045",
-                "sha256:be481f3bd58628cb9a2f94b173031058a1ba1446312e045b92f7ed524330838b"
+                "sha256:9539529379287d69c10e9ab4e0b2900e1a16004d6ff06de0d84aa62c4833b1bb",
+                "sha256:9af96d76bc7011c60b4e76177e283efe20e060278ecd81ce7005a4d80a4dc69e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.15.1"
+            "version": "==0.15.2"
         },
         "datashape": {
             "hashes": [
@@ -702,6 +738,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.7.1"
         },
+        "distlib": {
+            "hashes": [
+                "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057",
+                "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"
+            ],
+            "version": "==0.3.7"
+        },
         "entrypoints": {
             "hashes": [
                 "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4",
@@ -732,6 +775,14 @@
             ],
             "version": "==2.18.0"
         },
+        "filelock": {
+            "hashes": [
+                "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81",
+                "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.12.2"
+        },
         "fiona": {
             "hashes": [
                 "sha256:18649326a7724611b16b648e14fd094089d517413b95ac91d0cdb0adc5fcb8de",
@@ -757,45 +808,53 @@
             "index": "pypi",
             "version": "==1.8.22"
         },
+        "flake8": {
+            "hashes": [
+                "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7",
+                "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181"
+            ],
+            "index": "pypi",
+            "version": "==6.0.0"
+        },
         "fonttools": {
             "hashes": [
-                "sha256:01cfe02416b6d416c5c8d15e30315cbcd3e97d1b50d3b34b0ce59f742ef55258",
-                "sha256:0a1466713e54bdbf5521f2f73eebfe727a528905ff5ec63cda40961b4b1eea95",
-                "sha256:0df8ef75ba5791e873c9eac2262196497525e3f07699a2576d3ab9ddf41cb619",
-                "sha256:10dac980f2b975ef74532e2a94bb00e97a95b4595fb7f98db493c474d5f54d0e",
-                "sha256:150122ed93127a26bc3670ebab7e2add1e0983d30927733aec327ebf4255b072",
-                "sha256:1f81ed9065b4bd3f4f3ce8e4873cd6a6b3f4e92b1eddefde35d332c6f414acc3",
-                "sha256:27ec3246a088555629f9f0902f7412220c67340553ca91eb540cf247aacb1983",
-                "sha256:2d6dc3fa91414ff4daa195c05f946e6a575bd214821e26d17ca50f74b35b0fe4",
-                "sha256:329341ba3d86a36e482610db56b30705384cb23bd595eac8cbb045f627778e9d",
-                "sha256:3fb2a69870bfe143ec20b039a1c8009e149dd7780dd89554cc8a11f79e5de86b",
-                "sha256:4655c480a1a4d706152ff54f20e20cf7609084016f1df3851cce67cef768f40a",
-                "sha256:48e82d776d2e93f88ca56567509d102266e7ab2fb707a0326f032fe657335238",
-                "sha256:57b68eab183fafac7cd7d464a7bfa0fcd4edf6c67837d14fb09c1c20516cf20b",
-                "sha256:58c1165f9b2662645de9b19a8c8bdd636b36294ccc07e1b0163856b74f10bafc",
-                "sha256:614b1283dca88effd20ee48160518e6de275ce9b5456a3134d5f235523fc5065",
-                "sha256:685a4dd6cf31593b50d6d441feb7781a4a7ef61e19551463e14ed7c527b86f9f",
-                "sha256:6bd7e4777bff1dcb7c4eff4786998422770f3bfbef8be401c5332895517ba3fa",
-                "sha256:703101eb0490fae32baf385385d47787b73d9ea55253df43b487c89ec767e0d7",
-                "sha256:83b98be5d291e08501bd4fc0c4e0f8e6e05b99f3924068b17c5c9972af6fff84",
-                "sha256:8ece1886d12bb36c48c00b2031518877f41abae317e3a55620d38e307d799b7e",
-                "sha256:9c456d1f23deff64ffc8b5b098718e149279abdea4d8692dba69172fb6a0d597",
-                "sha256:9cd2363ea7728496827658682d049ffb2e98525e2247ca64554864a8cc945568",
-                "sha256:a9b55d2a3b360e0c7fc5bd8badf1503ca1c11dd3a1cd20f2c26787ffa145a9c7",
-                "sha256:ae7df0ae9ee2f3f7676b0ff6f4ebe48ad0acaeeeaa0b6839d15dbf0709f2c5ef",
-                "sha256:ae881e484702efdb6cf756462622de81d4414c454edfd950b137e9a7352b3cb9",
-                "sha256:b8600ae7dce6ec3ddfb201abb98c9d53abbf8064d7ac0c8a0d8925e722ccf2a0",
-                "sha256:c36c904ce0322df01e590ba814d5d69e084e985d7e4c2869378671d79662a7d4",
-                "sha256:c8bf88f9e3ce347c716921804ef3a8330cb128284eb6c0b6c4b3574f3c580023",
-                "sha256:d40673b2e927f7cd0819c6f04489dfbeb337b4a7b10fc633c89bf4f34ecb9620",
-                "sha256:d54e600a2bcfa5cdaa860237765c01804a03b08404d6affcd92942fa7315ffba",
-                "sha256:dfe7fa7e607f7e8b58d0c32501a3a7cac148538300626d1b930082c90ae7f6bd",
-                "sha256:e35bed436726194c5e6e094fdfb423fb7afaa0211199f9d245e59e11118c576c",
-                "sha256:f0290ea7f9945174bd4dfd66e96149037441eb2008f3649094f056201d99e293",
-                "sha256:fae4e801b774cc62cecf4a57b1eae4097903fced00c608d9e2bc8f84cd87b54a"
+                "sha256:0eb79a2da5eb6457a6f8ab904838454accc7d4cccdaff1fd2bd3a0679ea33d64",
+                "sha256:113337c2d29665839b7d90b39f99b3cac731f72a0eda9306165a305c7c31d341",
+                "sha256:12a7c247d1b946829bfa2f331107a629ea77dc5391dfd34fdcd78efa61f354ca",
+                "sha256:179737095eb98332a2744e8f12037b2977f22948cf23ff96656928923ddf560a",
+                "sha256:19b7db825c8adee96fac0692e6e1ecd858cae9affb3b4812cdb9d934a898b29e",
+                "sha256:37983b6bdab42c501202500a2be3a572f50d4efe3237e0686ee9d5f794d76b35",
+                "sha256:3a35981d90feebeaef05e46e33e6b9e5b5e618504672ca9cd0ff96b171e4bfff",
+                "sha256:46a0ec8adbc6ff13494eb0c9c2e643b6f009ce7320cf640de106fb614e4d4360",
+                "sha256:4aa79366e442dbca6e2c8595645a3a605d9eeabdb7a094d745ed6106816bef5d",
+                "sha256:515607ec756d7865f23070682622c49d922901943697871fc292277cf1e71967",
+                "sha256:53eb5091ddc8b1199330bb7b4a8a2e7995ad5d43376cadce84523d8223ef3136",
+                "sha256:5d18fc642fd0ac29236ff88ecfccff229ec0386090a839dd3f1162e9a7944a40",
+                "sha256:5fb289b7a815638a7613d46bcf324c9106804725b2bb8ad913c12b6958ffc4ec",
+                "sha256:62f481ac772fd68901573956231aea3e4b1ad87b9b1089a61613a91e2b50bb9b",
+                "sha256:689508b918332fb40ce117131633647731d098b1b10d092234aa959b4251add5",
+                "sha256:68a02bbe020dc22ee0540e040117535f06df9358106d3775e8817d826047f3fd",
+                "sha256:6ed2662a3d9c832afa36405f8748c250be94ae5dfc5283d668308391f2102861",
+                "sha256:7286aed4ea271df9eab8d7a9b29e507094b51397812f7ce051ecd77915a6e26b",
+                "sha256:7cc7d685b8eeca7ae69dc6416833fbfea61660684b7089bca666067cb2937dcf",
+                "sha256:8708b98c278012ad267ee8a7433baeb809948855e81922878118464b274c909d",
+                "sha256:9398f244e28e0596e2ee6024f808b06060109e33ed38dcc9bded452fd9bbb853",
+                "sha256:9e36344e48af3e3bde867a1ca54f97c308735dd8697005c2d24a86054a114a71",
+                "sha256:a398bdadb055f8de69f62b0fc70625f7cbdab436bbb31eef5816e28cab083ee8",
+                "sha256:acb47f6f8680de24c1ab65ebde39dd035768e2a9b571a07c7b8da95f6c8815fd",
+                "sha256:be24fcb80493b2c94eae21df70017351851652a37de514de553435b256b2f249",
+                "sha256:c391cd5af88aacaf41dd7cfb96eeedfad297b5899a39e12f4c2c3706d0a3329d",
+                "sha256:c95b0724a6deea2c8c5d3222191783ced0a2f09bd6d33f93e563f6f1a4b3b3a4",
+                "sha256:c9b1ce7a45978b821a06d375b83763b27a3a5e8a2e4570b3065abad240a18760",
+                "sha256:db372213d39fa33af667c2aa586a0c1235e88e9c850f5dd5c8e1f17515861868",
+                "sha256:db55cbaea02a20b49fefbd8e9d62bd481aaabe1f2301dabc575acc6b358874fa",
+                "sha256:ed1a13a27f59d1fc1920394a7f596792e9d546c9ca5a044419dca70c37815d7c",
+                "sha256:f2b82f46917d8722e6b5eafeefb4fb585d23babd15d8246c664cd88a5bddd19c",
+                "sha256:f2f806990160d1ce42d287aa419df3ffc42dfefe60d473695fb048355fe0c6a0",
+                "sha256:f720fa82a11c0f9042376fd509b5ed88dab7e3cd602eee63a1af08883b37342b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.42.0"
+            "version": "==4.42.1"
         },
         "fqdn": {
             "hashes": [
@@ -838,11 +897,11 @@
         },
         "geopy": {
             "hashes": [
-                "sha256:228cd53b6eef699b2289d1172e462a90d5057779a10388a7366291812601187f",
-                "sha256:4a29a16d41d8e56ba8e07310802a1cbdf098eeb6069cc3d6d3068fc770629ffc"
+                "sha256:a59392bf17adb486b25dbdd71fbed27733bdf24a2dac588047a619de56695e36",
+                "sha256:d2639a46d0ce4c091e9688b750ba94348a14b898a1e55c68f4b4a07e7d1afa20"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.3.0"
+            "version": "==2.4.0"
         },
         "gval": {
             "hashes": [
@@ -852,6 +911,14 @@
             "index": "pypi",
             "version": "==0.1.2"
         },
+        "identify": {
+            "hashes": [
+                "sha256:7243800bce2f58404ed41b7c002e53d4d22bcf3ae1b7900c2d7aefd95394bf7f",
+                "sha256:c22a8ead0d4ca11f1edd6c9418c3220669b3b7533ada0a0ffa6cc0ef85cf9b54"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.5.26"
+        },
         "idna": {
             "hashes": [
                 "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
@@ -1005,6 +1072,14 @@
             ],
             "version": "==20.11.0"
         },
+        "isort": {
+            "hashes": [
+                "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504",
+                "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"
+            ],
+            "index": "pypi",
+            "version": "==5.12.0"
+        },
         "jedi": {
             "hashes": [
                 "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
@@ -1113,11 +1188,11 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:36da0a266d31a41ac335a366c88933c17dfa5bb817a48f5c02c16d303bc9477f",
-                "sha256:6a77912aff643e53fa14bdb2634884b52b784a4be77ce8e93f7283faed0f0849"
+                "sha256:98a375347b580e837e7016007c24680a4261ed8ad7cd35196ac087d229f48e5a",
+                "sha256:d64fb4e593907290e5df916e3c9399c15ab2cd7bdb71cbcd1d36452dbfb30523"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==2.7.0"
+            "index": "pypi",
+            "version": "==2.7.2"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1185,77 +1260,113 @@
         },
         "kiwisolver": {
             "hashes": [
-                "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b",
-                "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166",
-                "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c",
-                "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c",
-                "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0",
-                "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4",
-                "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9",
-                "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286",
-                "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767",
-                "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c",
-                "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6",
-                "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b",
-                "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004",
-                "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf",
-                "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494",
-                "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac",
-                "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626",
-                "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766",
-                "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514",
-                "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6",
-                "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f",
-                "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d",
-                "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191",
-                "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d",
-                "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51",
-                "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f",
-                "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8",
-                "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454",
-                "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb",
-                "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da",
-                "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8",
-                "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de",
-                "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a",
-                "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9",
-                "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008",
-                "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3",
-                "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32",
-                "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938",
-                "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1",
-                "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9",
-                "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d",
-                "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824",
-                "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b",
-                "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd",
-                "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2",
-                "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5",
-                "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69",
-                "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3",
-                "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae",
-                "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597",
-                "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e",
-                "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955",
-                "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca",
-                "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a",
-                "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea",
-                "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede",
-                "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4",
-                "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6",
-                "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686",
-                "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408",
-                "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871",
-                "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29",
-                "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750",
-                "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897",
-                "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0",
-                "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2",
-                "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09",
-                "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.4.4"
+                "sha256:00bd361b903dc4bbf4eb165f24d1acbee754fce22ded24c3d56eec268658a5cf",
+                "sha256:040c1aebeda72197ef477a906782b5ab0d387642e93bda547336b8957c61022e",
+                "sha256:05703cf211d585109fcd72207a31bb170a0f22144d68298dc5e61b3c946518af",
+                "sha256:06f54715b7737c2fecdbf140d1afb11a33d59508a47bf11bb38ecf21dc9ab79f",
+                "sha256:0dc9db8e79f0036e8173c466d21ef18e1befc02de8bf8aa8dc0813a6dc8a7046",
+                "sha256:0f114aa76dc1b8f636d077979c0ac22e7cd8f3493abbab152f20eb8d3cda71f3",
+                "sha256:11863aa14a51fd6ec28688d76f1735f8f69ab1fabf388851a595d0721af042f5",
+                "sha256:11c7de8f692fc99816e8ac50d1d1aef4f75126eefc33ac79aac02c099fd3db71",
+                "sha256:11d011a7574eb3b82bcc9c1a1d35c1d7075677fdd15de527d91b46bd35e935ee",
+                "sha256:146d14bebb7f1dc4d5fbf74f8a6cb15ac42baadee8912eb84ac0b3b2a3dc6ac3",
+                "sha256:15568384086b6df3c65353820a4473575dbad192e35010f622c6ce3eebd57af9",
+                "sha256:19df6e621f6d8b4b9c4d45f40a66839294ff2bb235e64d2178f7522d9170ac5b",
+                "sha256:1b04139c4236a0f3aff534479b58f6f849a8b351e1314826c2d230849ed48985",
+                "sha256:210ef2c3a1f03272649aff1ef992df2e724748918c4bc2d5a90352849eb40bea",
+                "sha256:2270953c0d8cdab5d422bee7d2007f043473f9d2999631c86a223c9db56cbd16",
+                "sha256:2400873bccc260b6ae184b2b8a4fec0e4082d30648eadb7c3d9a13405d861e89",
+                "sha256:2a40773c71d7ccdd3798f6489aaac9eee213d566850a9533f8d26332d626b82c",
+                "sha256:2c5674c4e74d939b9d91dda0fae10597ac7521768fec9e399c70a1f27e2ea2d9",
+                "sha256:3195782b26fc03aa9c6913d5bad5aeb864bdc372924c093b0f1cebad603dd712",
+                "sha256:31a82d498054cac9f6d0b53d02bb85811185bcb477d4b60144f915f3b3126342",
+                "sha256:32d5cf40c4f7c7b3ca500f8985eb3fb3a7dfc023215e876f207956b5ea26632a",
+                "sha256:346f5343b9e3f00b8db8ba359350eb124b98c99efd0b408728ac6ebf38173958",
+                "sha256:378a214a1e3bbf5ac4a8708304318b4f890da88c9e6a07699c4ae7174c09a68d",
+                "sha256:39b42c68602539407884cf70d6a480a469b93b81b7701378ba5e2328660c847a",
+                "sha256:3a2b053a0ab7a3960c98725cfb0bf5b48ba82f64ec95fe06f1d06c99b552e130",
+                "sha256:3aba7311af82e335dd1e36ffff68aaca609ca6290c2cb6d821a39aa075d8e3ff",
+                "sha256:3cd32d6c13807e5c66a7cbb79f90b553642f296ae4518a60d8d76243b0ad2898",
+                "sha256:3edd2fa14e68c9be82c5b16689e8d63d89fe927e56debd6e1dbce7a26a17f81b",
+                "sha256:4c380469bd3f970ef677bf2bcba2b6b0b4d5c75e7a020fb863ef75084efad66f",
+                "sha256:4e66e81a5779b65ac21764c295087de82235597a2293d18d943f8e9e32746265",
+                "sha256:53abb58632235cd154176ced1ae8f0d29a6657aa1aa9decf50b899b755bc2b93",
+                "sha256:5794cf59533bc3f1b1c821f7206a3617999db9fbefc345360aafe2e067514929",
+                "sha256:59415f46a37f7f2efeec758353dd2eae1b07640d8ca0f0c42548ec4125492635",
+                "sha256:59ec7b7c7e1a61061850d53aaf8e93db63dce0c936db1fda2658b70e4a1be709",
+                "sha256:59edc41b24031bc25108e210c0def6f6c2191210492a972d585a06ff246bb79b",
+                "sha256:5a580c91d686376f0f7c295357595c5a026e6cbc3d77b7c36e290201e7c11ecb",
+                "sha256:5b94529f9b2591b7af5f3e0e730a4e0a41ea174af35a4fd067775f9bdfeee01a",
+                "sha256:5c7b3b3a728dc6faf3fc372ef24f21d1e3cee2ac3e9596691d746e5a536de920",
+                "sha256:5c90ae8c8d32e472be041e76f9d2f2dbff4d0b0be8bd4041770eddb18cf49a4e",
+                "sha256:5e7139af55d1688f8b960ee9ad5adafc4ac17c1c473fe07133ac092310d76544",
+                "sha256:5ff5cf3571589b6d13bfbfd6bcd7a3f659e42f96b5fd1c4830c4cf21d4f5ef45",
+                "sha256:620ced262a86244e2be10a676b646f29c34537d0d9cc8eb26c08f53d98013390",
+                "sha256:6512cb89e334e4700febbffaaa52761b65b4f5a3cf33f960213d5656cea36a77",
+                "sha256:6c08e1312a9cf1074d17b17728d3dfce2a5125b2d791527f33ffbe805200a355",
+                "sha256:6c3bd3cde54cafb87d74d8db50b909705c62b17c2099b8f2e25b461882e544ff",
+                "sha256:6ef7afcd2d281494c0a9101d5c571970708ad911d028137cd558f02b851c08b4",
+                "sha256:7269d9e5f1084a653d575c7ec012ff57f0c042258bf5db0954bf551c158466e7",
+                "sha256:72d40b33e834371fd330fb1472ca19d9b8327acb79a5821d4008391db8e29f20",
+                "sha256:74d1b44c6cfc897df648cc9fdaa09bc3e7679926e6f96df05775d4fb3946571c",
+                "sha256:74db36e14a7d1ce0986fa104f7d5637aea5c82ca6326ed0ec5694280942d1162",
+                "sha256:763773d53f07244148ccac5b084da5adb90bfaee39c197554f01b286cf869228",
+                "sha256:76c6a5964640638cdeaa0c359382e5703e9293030fe730018ca06bc2010c4437",
+                "sha256:76d9289ed3f7501012e05abb8358bbb129149dbd173f1f57a1bf1c22d19ab7cc",
+                "sha256:7931d8f1f67c4be9ba1dd9c451fb0eeca1a25b89e4d3f89e828fe12a519b782a",
+                "sha256:7b8b454bac16428b22560d0a1cf0a09875339cab69df61d7805bf48919415901",
+                "sha256:7e5bab140c309cb3a6ce373a9e71eb7e4873c70c2dda01df6820474f9889d6d4",
+                "sha256:83d78376d0d4fd884e2c114d0621624b73d2aba4e2788182d286309ebdeed770",
+                "sha256:852542f9481f4a62dbb5dd99e8ab7aedfeb8fb6342349a181d4036877410f525",
+                "sha256:85267bd1aa8880a9c88a8cb71e18d3d64d2751a790e6ca6c27b8ccc724bcd5ad",
+                "sha256:88a2df29d4724b9237fc0c6eaf2a1adae0cdc0b3e9f4d8e7dc54b16812d2d81a",
+                "sha256:88b9f257ca61b838b6f8094a62418421f87ac2a1069f7e896c36a7d86b5d4c29",
+                "sha256:8ab3919a9997ab7ef2fbbed0cc99bb28d3c13e6d4b1ad36e97e482558a91be90",
+                "sha256:92dea1ffe3714fa8eb6a314d2b3c773208d865a0e0d35e713ec54eea08a66250",
+                "sha256:9407b6a5f0d675e8a827ad8742e1d6b49d9c1a1da5d952a67d50ef5f4170b18d",
+                "sha256:9408acf3270c4b6baad483865191e3e582b638b1654a007c62e3efe96f09a9a3",
+                "sha256:955e8513d07a283056b1396e9a57ceddbd272d9252c14f154d450d227606eb54",
+                "sha256:9db8ea4c388fdb0f780fe91346fd438657ea602d58348753d9fb265ce1bca67f",
+                "sha256:9eaa8b117dc8337728e834b9c6e2611f10c79e38f65157c4c38e9400286f5cb1",
+                "sha256:a51a263952b1429e429ff236d2f5a21c5125437861baeed77f5e1cc2d2c7c6da",
+                "sha256:a6aa6315319a052b4ee378aa171959c898a6183f15c1e541821c5c59beaa0238",
+                "sha256:aa12042de0171fad672b6c59df69106d20d5596e4f87b5e8f76df757a7c399aa",
+                "sha256:aaf7be1207676ac608a50cd08f102f6742dbfc70e8d60c4db1c6897f62f71523",
+                "sha256:b0157420efcb803e71d1b28e2c287518b8808b7cf1ab8af36718fd0a2c453eb0",
+                "sha256:b3f7e75f3015df442238cca659f8baa5f42ce2a8582727981cbfa15fee0ee205",
+                "sha256:b9098e0049e88c6a24ff64545cdfc50807818ba6c1b739cae221bbbcbc58aad3",
+                "sha256:ba55dce0a9b8ff59495ddd050a0225d58bd0983d09f87cfe2b6aec4f2c1234e4",
+                "sha256:bb86433b1cfe686da83ce32a9d3a8dd308e85c76b60896d58f082136f10bffac",
+                "sha256:bbea0db94288e29afcc4c28afbf3a7ccaf2d7e027489c449cf7e8f83c6346eb9",
+                "sha256:bbf1d63eef84b2e8c89011b7f2235b1e0bf7dacc11cac9431fc6468e99ac77fb",
+                "sha256:c7940c1dc63eb37a67721b10d703247552416f719c4188c54e04334321351ced",
+                "sha256:c9bf3325c47b11b2e51bca0824ea217c7cd84491d8ac4eefd1e409705ef092bd",
+                "sha256:cdc8a402aaee9a798b50d8b827d7ecf75edc5fb35ea0f91f213ff927c15f4ff0",
+                "sha256:ceec1a6bc6cab1d6ff5d06592a91a692f90ec7505d6463a88a52cc0eb58545da",
+                "sha256:cfe6ab8da05c01ba6fbea630377b5da2cd9bcbc6338510116b01c1bc939a2c18",
+                "sha256:d099e745a512f7e3bbe7249ca835f4d357c586d78d79ae8f1dcd4d8adeb9bda9",
+                "sha256:d0ef46024e6a3d79c01ff13801cb19d0cad7fd859b15037aec74315540acc276",
+                "sha256:d2e5a98f0ec99beb3c10e13b387f8db39106d53993f498b295f0c914328b1333",
+                "sha256:da4cfb373035def307905d05041c1d06d8936452fe89d464743ae7fb8371078b",
+                "sha256:da802a19d6e15dffe4b0c24b38b3af68e6c1a68e6e1d8f30148c83864f3881db",
+                "sha256:dced8146011d2bc2e883f9bd68618b8247387f4bbec46d7392b3c3b032640126",
+                "sha256:dfdd7c0b105af050eb3d64997809dc21da247cf44e63dc73ff0fd20b96be55a9",
+                "sha256:e368f200bbc2e4f905b8e71eb38b3c04333bddaa6a2464a6355487b02bb7fb09",
+                "sha256:e391b1f0a8a5a10ab3b9bb6afcfd74f2175f24f8975fb87ecae700d1503cdee0",
+                "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec",
+                "sha256:e5d706eba36b4c4d5bc6c6377bb6568098765e990cfc21ee16d13963fab7b3e7",
+                "sha256:ec20916e7b4cbfb1f12380e46486ec4bcbaa91a9c448b97023fde0d5bbf9e4ff",
+                "sha256:f1d072c2eb0ad60d4c183f3fb44ac6f73fb7a8f16a2694a91f988275cbf352f9",
+                "sha256:f846c260f483d1fd217fe5ed7c173fb109efa6b1fc8381c8b7552c5781756192",
+                "sha256:f91de7223d4c7b793867797bacd1ee53bfe7359bd70d27b7b58a04efbb9436c8",
+                "sha256:faae4860798c31530dd184046a900e652c95513796ef51a12bc086710c2eec4d",
+                "sha256:fc579bf0f502e54926519451b920e875f433aceb4624a3646b3252b5caa9e0b6",
+                "sha256:fcc700eadbbccbf6bc1bcb9dbe0786b4b1cb91ca0dcda336eef5c2beed37b797",
+                "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892",
+                "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.4.5"
         },
         "llvmlite": {
             "hashes": [
@@ -1410,6 +1521,14 @@
             "markers": "python_version >= '3.5'",
             "version": "==0.1.6"
         },
+        "mccabe": {
+            "hashes": [
+                "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
+                "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.7.0"
+        },
         "memory-profiler": {
             "hashes": [
                 "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84",
@@ -1574,11 +1693,11 @@
         },
         "nbconvert": {
             "hashes": [
-                "sha256:3022adadff3f86578a47fab7c2228bb3ca9c56a24345642a22f917f6168b48fc",
-                "sha256:4a5996bf5f3cd16aa0431897ba1aa4c64842c2079f434b3dc6b8c4b252ef3355"
+                "sha256:aec605e051fa682ccc7934ccc338ba1e8b626cfadbab0db592106b630f63f0f2",
+                "sha256:f5bc15a1247e14dd41ceef0c0a3bc70020e016576eb0578da62f1c5b4f950479"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.7.3"
+            "version": "==7.8.0"
         },
         "nbformat": {
             "hashes": [
@@ -1629,6 +1748,14 @@
             "index": "pypi",
             "version": "==1.6.3"
         },
+        "nodeenv": {
+            "hashes": [
+                "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2",
+                "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+            "version": "==1.8.0"
+        },
         "notebook": {
             "hashes": [
                 "sha256:171039245a5b1a8f8233165091210632c21250ce2a652daed38fe8f94389984f",
@@ -1844,6 +1971,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.4.0"
         },
+        "pathspec": {
+            "hashes": [
+                "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20",
+                "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.11.2"
+        },
         "pexpect": {
             "hashes": [
                 "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
@@ -1939,11 +2074,19 @@
         },
         "pluggy": {
             "hashes": [
-                "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849",
-                "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"
+                "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
+                "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.2.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.3.0"
+        },
+        "pre-commit": {
+            "hashes": [
+                "sha256:10badb65d6a38caff29703362271d7dca483d01da88f9d7e05d0b97171c136cb",
+                "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023"
+            ],
+            "index": "pypi",
+            "version": "==3.3.3"
         },
         "prometheus-client": {
             "hashes": [
@@ -2193,6 +2336,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.0.1"
         },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053",
+                "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.10.0"
+        },
         "pycparser": {
             "hashes": [
                 "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
@@ -2288,6 +2439,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.10.10"
         },
+        "pyflakes": {
+            "hashes": [
+                "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf",
+                "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.1"
+        },
         "pygments": {
             "hashes": [
                 "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
@@ -2429,6 +2588,14 @@
             "index": "pypi",
             "version": "==3.5.0"
         },
+        "pyproject-flake8": {
+            "hashes": [
+                "sha256:bdc7ca9b967b9724983903489b8943b72c668178fb69f03e8774ec74f6a13782",
+                "sha256:d43421caca0ef8a672874405fe63c722b0333e3c22c41648c6df60f21bab2b6b"
+            ],
+            "index": "pypi",
+            "version": "==6.0.0.post1"
+        },
         "pytest": {
             "hashes": [
                 "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d",
@@ -2470,7 +2637,9 @@
         },
         "pyyaml": {
             "hashes": [
+                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
                 "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
+                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
                 "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
                 "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
                 "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
@@ -2478,7 +2647,10 @@
                 "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
                 "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
                 "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
+                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
+                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
                 "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
+                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
                 "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
                 "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
                 "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
@@ -2486,9 +2658,12 @@
                 "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
                 "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
                 "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
+                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
                 "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
                 "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
                 "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
+                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
+                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
                 "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
                 "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
                 "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
@@ -2503,7 +2678,9 @@
                 "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
                 "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
                 "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
+                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
                 "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
+                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
                 "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
                 "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
                 "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
@@ -2720,11 +2897,11 @@
         },
         "qtpy": {
             "hashes": [
-                "sha256:5193d20e0b16e4d9d3bc2c642d04d9f4e2c892590bd1b9c92bfe38a95d5a2e12",
-                "sha256:a8c74982d6d172ce124d80cafd39653df78989683f760f2281ba91a6e7b9de8b"
+                "sha256:4d4f045a41e09ac9fa57fcb47ef05781aa5af294a0a646acc1b729d14225e741",
+                "sha256:db2d508167aa6106781565c8da5c6f1487debacba33519cedc35fa8997d424d4"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.3.1"
+            "version": "==2.4.0"
         },
         "rasterio": {
             "hashes": [
@@ -2826,106 +3003,106 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f",
-                "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238",
-                "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f",
-                "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f",
-                "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c",
-                "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298",
-                "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260",
-                "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1",
-                "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d",
-                "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7",
-                "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f",
-                "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876",
-                "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe",
-                "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be",
-                "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32",
-                "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3",
-                "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18",
-                "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d",
-                "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620",
-                "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b",
-                "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae",
-                "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496",
-                "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1",
-                "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67",
-                "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f",
-                "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764",
-                "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196",
-                "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e",
-                "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846",
-                "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b",
-                "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d",
-                "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26",
-                "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e",
-                "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044",
-                "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c",
-                "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d",
-                "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad",
-                "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d",
-                "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab",
-                "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920",
-                "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e",
-                "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872",
-                "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3",
-                "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611",
-                "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4",
-                "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c",
-                "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193",
-                "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af",
-                "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10",
-                "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd",
-                "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f",
-                "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b",
-                "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945",
-                "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752",
-                "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c",
-                "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387",
-                "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8",
-                "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d",
-                "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931",
-                "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03",
-                "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502",
-                "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f",
-                "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55",
-                "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82",
-                "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798",
-                "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a",
-                "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b",
-                "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa",
-                "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f",
-                "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192",
-                "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020",
-                "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7",
-                "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1",
-                "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386",
-                "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90",
-                "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f",
-                "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe",
-                "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596",
-                "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f",
-                "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387",
-                "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16",
-                "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e",
-                "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b",
-                "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6",
-                "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1",
-                "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de",
-                "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0",
-                "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3",
-                "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468",
-                "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e",
-                "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd",
-                "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324",
-                "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c",
-                "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535",
-                "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55",
-                "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6",
-                "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"
+                "sha256:00215f6a9058fbf84f9d47536902558eb61f180a6b2a0fa35338d06ceb9a2e5a",
+                "sha256:0028eb0967942d0d2891eae700ae1a27b7fd18604cfcb16a1ef486a790fee99e",
+                "sha256:0155c33af0676fc38e1107679be882077680ad1abb6303956b97259c3177e85e",
+                "sha256:063411228b852fb2ed7485cf91f8e7d30893e69b0acb207ec349db04cccc8225",
+                "sha256:0700c2133ba203c4068aaecd6a59bda22e06a5e46255c9da23cbf68c6942215d",
+                "sha256:08e08ccf5b10badb7d0a5c84829b914c6e1e1f3a716fdb2bf294e2bd01562775",
+                "sha256:0d292cabd7c8335bdd3237ded442480a249dbcdb4ddfac5218799364a01a0f5c",
+                "sha256:15932ec5f224b0e35764dc156514533a4fca52dcfda0dfbe462a1a22b37efd59",
+                "sha256:18f87baa20e02e9277ad8960cd89b63c79c05caf106f4c959a9595c43f2a34a5",
+                "sha256:1a6420a36975e0073acaeee44ead260c1f6ea56812cfc6c31ec00c1c48197173",
+                "sha256:1b401e8b9aece651512e62c431181e6e83048a651698a727ea0eb0699e9f9b74",
+                "sha256:1d7b7b71bcb82d8713c7c2e9c5f061415598af5938666beded20d81fa23e7640",
+                "sha256:23750a9b8a329844ba1fe267ca456bb3184984da2880ed17ae641c5af8de3fef",
+                "sha256:23a059143c1393015c68936370cce11690f7294731904bdae47cc3e16d0b2474",
+                "sha256:26d9fd624649a10e4610fab2bc820e215a184d193e47d0be7fe53c1c8f67f370",
+                "sha256:291c9ce3929a75b45ce8ddde2aa7694fc8449f2bc8f5bd93adf021efaae2d10b",
+                "sha256:298e8b5d8087e0330aac211c85428c8761230ef46a1f2c516d6a2f67fb8803c5",
+                "sha256:2c7c4266c1b61eb429e8aeb7d8ed6a3bfe6c890a1788b18dbec090c35c6b93fa",
+                "sha256:2d68a8e8a3a816629283faf82358d8c93fe5bd974dd2704152394a3de4cec22a",
+                "sha256:344b89384c250ba6a4ce1786e04d01500e4dac0f4137ceebcaad12973c0ac0b3",
+                "sha256:3455ecc46ea443b5f7d9c2f946ce4017745e017b0d0f8b99c92564eff97e97f5",
+                "sha256:3d544a614055b131111bed6edfa1cb0fb082a7265761bcb03321f2dd7b5c6c48",
+                "sha256:3e5c26905aa651cc8c0ddc45e0e5dea2a1296f70bdc96af17aee9d0493280a17",
+                "sha256:3f5cc8c7bc99d2bbcd704cef165ca7d155cd6464c86cbda8339026a42d219397",
+                "sha256:4992266817169997854f81df7f6db7bdcda1609972d8ffd6919252f09ec3c0f6",
+                "sha256:4d55528ef13af4b4e074d067977b1f61408602f53ae4537dccf42ba665c2c7bd",
+                "sha256:576da63eae7809f375932bfcbca2cf20620a1915bf2fedce4b9cc8491eceefe3",
+                "sha256:58fc4d66ee349a23dbf08c7e964120dc9027059566e29cf0ce6205d590ed7eca",
+                "sha256:5b9bf77008f2c55dabbd099fd3ac87009471d223a1c7ebea36873d39511b780a",
+                "sha256:5e7996aed3f65667c6dcc8302a69368435a87c2364079a066750a2eac75ea01e",
+                "sha256:5f7487be65b9c2c510819e744e375bd41b929a97e5915c4852a82fbb085df62c",
+                "sha256:6388e4e95a26717b94a05ced084e19da4d92aca883f392dffcf8e48c8e221a24",
+                "sha256:65af12f70355de29e1092f319f85a3467f4005e959ab65129cb697169ce94b86",
+                "sha256:668d2b45d62c68c7a370ac3dce108ffda482b0a0f50abd8b4c604a813a59e08f",
+                "sha256:71333c22f7cf5f0480b59a0aef21f652cf9bbaa9679ad261b405b65a57511d1e",
+                "sha256:7150b83b3e3ddaac81a8bb6a9b5f93117674a0e7a2b5a5b32ab31fdfea6df27f",
+                "sha256:748e472345c3a82cfb462d0dff998a7bf43e621eed73374cb19f307e97e08a83",
+                "sha256:75dbfd41a61bc1fb0536bf7b1abf272dc115c53d4d77db770cd65d46d4520882",
+                "sha256:7618a082c55cf038eede4a918c1001cc8a4411dfe508dc762659bcd48d8f4c6e",
+                "sha256:780fcb855be29153901c67fc9c5633d48aebef21b90aa72812fa181d731c6b00",
+                "sha256:78d10c431073dc6ebceed35ab22948a016cc2b5120963c13a41e38bdde4a7212",
+                "sha256:7a3a3d3e4f1e3cd2a67b93a0b6ed0f2499e33f47cc568e3a0023e405abdc0ff1",
+                "sha256:7b6975d3763d0952c111700c0634968419268e6bbc0b55fe71138987fa66f309",
+                "sha256:80772e3bda6787510d9620bc0c7572be404a922f8ccdfd436bf6c3778119464c",
+                "sha256:80992eb20755701753e30a6952a96aa58f353d12a65ad3c9d48a8da5ec4690cf",
+                "sha256:841128a22e6ac04070a0f84776d07e9c38c4dcce8e28792a95e45fc621605517",
+                "sha256:861d25ae0985a1dd5297fee35f476b60c6029e2e6e19847d5b4d0a43a390b696",
+                "sha256:872f3dcaa8bf2245944861d7311179d2c0c9b2aaa7d3b464d99a7c2e401f01fa",
+                "sha256:87c93b25d538c433fb053da6228c6290117ba53ff6a537c133b0f2087948a582",
+                "sha256:8856aa76839dc234d3469f1e270918ce6bec1d6a601eba928f45d68a15f04fc3",
+                "sha256:885e023e73ce09b11b89ab91fc60f35d80878d2c19d6213a32b42ff36543c291",
+                "sha256:899b5e7e2d5a8bc92aa533c2d4e55e5ebba095c485568a5e4bedbc163421259a",
+                "sha256:8ce8caa29ebbdcde67e5fd652c811d34bc01f249dbc0d61e5cc4db05ae79a83b",
+                "sha256:8e1c68303ccf7fceb50fbab79064a2636119fd9aca121f28453709283dbca727",
+                "sha256:8e7e2b3577e97fa43c2c2b12a16139b2cedbd0770235d5179c0412b4794efd9b",
+                "sha256:92f05fc7d832e970047662b3440b190d24ea04f8d3c760e33e7163b67308c878",
+                "sha256:97f5811df21703446b42303475b8b855ee07d6ab6cdf8565eff115540624f25d",
+                "sha256:9affee8cb1ec453382c27eb9043378ab32f49cd4bc24a24275f5c39bf186c279",
+                "sha256:a2da4a8c6d465fde36cea7d54bf47b5cf089073452f0e47c8632ecb9dec23c07",
+                "sha256:a6903cdca64f1e301af9be424798328c1fe3b4b14aede35f04510989fc72f012",
+                "sha256:a8ab1adf04ae2d6d65835995218fd3f3eb644fe20655ca8ee233e2c7270ff53b",
+                "sha256:a8edd467551c1102dc0f5754ab55cd0703431cd3044edf8c8e7d9208d63fa453",
+                "sha256:ac00c41dd315d147b129976204839ca9de699d83519ff1272afbe4fb9d362d12",
+                "sha256:ad277f74b1c164f7248afa968700e410651eb858d7c160d109fb451dc45a2f09",
+                "sha256:ae46a50d235f1631d9ec4670503f7b30405103034830bc13df29fd947207f795",
+                "sha256:afe6b5a04b2ab1aa89bad32ca47bf71358e7302a06fdfdad857389dca8fb5f04",
+                "sha256:b1cb078f54af0abd835ca76f93a3152565b73be0f056264da45117d0adf5e99c",
+                "sha256:b25136212a3d064a8f0b9ebbb6c57094c5229e0de76d15c79b76feff26aeb7b8",
+                "sha256:b3226b246facae14909b465061ddcfa2dfeadb6a64f407f24300d42d69bcb1a1",
+                "sha256:b98e75b21fc2ba5285aef8efaf34131d16af1c38df36bdca2f50634bea2d3060",
+                "sha256:bbd7b24d108509a1b9b6679fcc1166a7dd031dbef1f3c2c73788f42e3ebb3beb",
+                "sha256:bed57543c99249ab3a4586ddc8786529fbc33309e5e8a1351802a06ca2baf4c2",
+                "sha256:c0583f69522732bdd79dca4cd3873e63a29acf4a299769c7541f2ca1e4dd4bc6",
+                "sha256:c1e0e9916301e3b3d970814b1439ca59487f0616d30f36a44cead66ee1748c31",
+                "sha256:c651847545422c8131660704c58606d841e228ed576c8f1666d98b3d318f89da",
+                "sha256:c7853f27195598e550fe089f78f0732c66ee1d1f0eaae8ad081589a5a2f5d4af",
+                "sha256:cbae50d352e4717ffc22c566afc2d0da744380e87ed44a144508e3fb9114a3f4",
+                "sha256:cdbed8f21204398f47de39b0a9b180d7e571f02dfb18bf5f1b618e238454b685",
+                "sha256:d08395595c42bcd82c3608762ce734504c6d025eef1c06f42326a6023a584186",
+                "sha256:d4639111e73997567343df6551da9dd90d66aece1b9fc26c786d328439488103",
+                "sha256:d63787f289944cc4bde518ad2b5e70a4f0d6e2ce76324635359c74c113fd188f",
+                "sha256:d6d5f061f6a2aa55790b9e64a23dfd87b6664ab56e24cd06c78eb43986cb260b",
+                "sha256:d7865df1fb564092bcf46dac61b5def25342faf6352e4bc0e61a286e3fa26a3d",
+                "sha256:db6585b600b2e76e98131e0ac0e5195759082b51687ad0c94505970c90718f4a",
+                "sha256:e36d7369363d2707d5f68950a64c4e025991eb0177db01ccb6aa6facae48b69f",
+                "sha256:e7947d9a6264c727a556541b1630296bbd5d0a05068d21c38dde8e7a1c703ef0",
+                "sha256:eb2d59bc196e6d3b1827c7db06c1a898bfa0787c0574af398e65ccf2e97c0fbe",
+                "sha256:ee9c2f6ca9774c2c24bbf7b23086264e6b5fa178201450535ec0859739e6f78d",
+                "sha256:f4760e1b02173f4155203054f77a5dc0b4078de7645c922b208d28e7eb99f3e2",
+                "sha256:f70bec8a14a692be6dbe7ce8aab303e88df891cbd4a39af091f90b6702e28055",
+                "sha256:f869e34d2326e417baee430ae998e91412cc8e7fdd83d979277a90a0e79a5b47",
+                "sha256:f8b9a7cd381970e64849070aca7c32d53ab7d96c66db6c2ef7aa23c6e803f514",
+                "sha256:f99d74ddf9d3b6126b509e81865f89bd1283e3fc1b568b68cd7bd9dfa15583d7",
+                "sha256:f9e7e493ded7042712a374471203dd43ae3fff5b81e3de1a0513fa241af9fd41",
+                "sha256:fc72ae476732cdb7b2c1acb5af23b478b8a0d4b6fcf19b90dd150291e0d5b26b",
+                "sha256:fccbf0cd3411719e4c9426755df90bf3449d9fc5a89f077f4a7f1abd4f70c910",
+                "sha256:ffcf18ad3edf1c170e27e88b10282a2c449aa0358659592462448d71b2000cfc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.9.2"
+            "version": "==0.10.0"
         },
         "rtree": {
             "hashes": [
@@ -2980,11 +3157,11 @@
         },
         "s3transfer": {
             "hashes": [
-                "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346",
-                "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"
+                "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084",
+                "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.6.1"
+            "version": "==0.6.2"
         },
         "scipy": {
             "hashes": [
@@ -3031,11 +3208,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f",
-                "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"
+                "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d",
+                "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==68.0.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==68.1.2"
         },
         "shapely": {
             "hashes": [
@@ -3286,7 +3463,7 @@
                 "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0",
                 "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe"
             ],
-            "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==6.3.3"
         },
         "tqdm": {
@@ -3307,18 +3484,18 @@
         },
         "typeguard": {
             "hashes": [
-                "sha256:94a8a8974eca6b980c0fd05a032f283914426ad9f2958b342075358e2744226a",
-                "sha256:b05a54bb0276eefd28880df42e004a71e699c8081fcb9d0536b2ceb01019f60c"
+                "sha256:5b7453b1e3b35fcfe2d62fa4ec500d05e6f2f2eb46f4126ae964677fcc384fff",
+                "sha256:7d4264cd631ac1157c5bb5ec992281b4f1e2ba7a35db91bc15f442235e244803"
             ],
-            "markers": "python_full_version >= '3.7.4'",
-            "version": "==4.1.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.1.3"
         },
         "typing-extensions": {
             "hashes": [
                 "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
                 "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version < '3.10'",
             "version": "==4.7.1"
         },
         "typing-inspect": {
@@ -3351,6 +3528,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==1.26.16"
         },
+        "virtualenv": {
+            "hashes": [
+                "sha256:95a6e9398b4967fbcb5fef2acec5efaf9aa4972049d9ae41f95e0972a683fd02",
+                "sha256:e5c3b4ce817b0b328af041506a2a299418c98747c4b1e68cb7527e74ced23efc"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==20.24.3"
+        },
         "wcwidth": {
             "hashes": [
                 "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
@@ -3374,11 +3559,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd",
-                "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"
+                "sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa",
+                "sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.6.2"
         },
         "whitebox": {
             "hashes": [
@@ -3724,7 +3909,7 @@
                 "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
                 "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version < '3.10'",
             "version": "==4.7.1"
         },
         "wcwidth": {
diff --git a/README.md b/README.md
index de3502665..fb6872d86 100644
--- a/README.md
+++ b/README.md
@@ -4,11 +4,11 @@ This repository includes flood inundation mapping software configured to work wi
 
 This software uses the Height Above Nearest Drainage (HAND) method to generate Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids. This repository also includes functionality to generate flood inundation maps (FIMs) and evaluate FIM accuracy.
 
-#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/cahaba/wiki).
+#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
 
 ---
 
-# FIM Version 4 
+# FIM Version 4
 
 ## Accessing Data through ESIP S3 Bucket
 The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools. Please contact Carson Pruitt (carson.pruitt@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) if you experience issues with permissions.
@@ -67,7 +67,7 @@ cd /home/projects/fim/code
 git clone https://github.com/NOAA-OWP/inundation-mapping.git
 ```
 
-Git will auto create a subfolder named `inundation-mapping` where the code will be. Your Docker mounts should include this `inundation-mapping` folder. 
+Git will auto create a subfolder named `inundation-mapping` where the code will be. Your Docker mounts should include this `inundation-mapping` folder.
 
 ### Dependencies
 [Docker](https://docs.docker.com/get-docker/)
@@ -98,7 +98,7 @@ There are two ways, which can be used together, to configure the system and/or d
 
 Make sure to set the config folder group to `fim` recursively using the chown command.
 
-This application has an default optional tool called the `calibration points tool`. In order to disable its' use, you can:  
+This application has an default optional tool called the `calibration points tool`. In order to disable its' use, you can:
 1.  Disable it by providing the `-skipcal` command line option to `fim_pipeline.sh` or `fim_pre_processing.sh`.
 2.  Disable it in the [`params_template.env`](/config/params_template.env) file by setting `src_adjust_spatial="FALSE"`.
 
@@ -107,7 +107,7 @@ This application has an default optional tool called the `calibration points too
 Since all of the dependencies are managed by utilizing a Docker container, we must issue the [`docker run`](https://docs.docker.com/engine/reference/commandline/run/) command to start a container as the run-time environment. The container is launched from a Docker image which was built in [Installation](#installation). The `-v <input_path>:/data` must contain a subdirectory named `inputs` (similar to `s3://noaa-nws-owp-fim/hand_fim`). If the pathing is set correctly, we do not need to adjust the `params_template.env` file, and can use the default file paths provided.
 
 
-```bash 
+```bash
 docker run --rm -it --name <your_container_name> \
     -v <path/to/repository>/:/foss_fim \
     -v <desired_output_path>/:/outputs \
@@ -115,7 +115,7 @@ docker run --rm -it --name <your_container_name> \
     -v <input_path>:/data \
     <image_name>:<tag>
 ```
-For example:  
+For example:
 ```bash
 docker run --rm -it --name robs_container \
     -v /home/projects/fim/code/inundation-mapping/:/foss_fim \
@@ -137,12 +137,12 @@ fim_pipeline.sh -u <huc8> -n <name_your_run> -o
     - While not mandatory, if you override the `params_template.env` file, you may want to use the `-c` argument to point to your adjusted file.
 - Outputs can be found under ```/outputs/<name_your_run>```.
 
-Processing of HUCs in FIM4 occurs in three sections. 
-You can run `fim_pipeline.sh` which automatically runs all of three major section, 
-OR you can run each of the sections independently if you like. 
+Processing of HUCs in FIM4 occurs in three sections.
+You can run `fim_pipeline.sh` which automatically runs all of three major section,
+OR you can run each of the sections independently if you like.
 
 The three sections are:
-1. `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
+1. `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections.
 2. `fim_process_unit_wb.sh` : This script processes one and exactly one HUC8 plus all of it's related branches. While it can only process one, you can run this script multiple times, each with different HUC (or overwriting a HUC). When you run `fim_pipeline.sh`, it automatically iterates when more than one HUC number has been supplied either by command line arguments or via a HUC list. For each HUC provided, `fim_pipeline.sh` will run  `fim_process_unit_wb.sh`. Using the `fim_process_unit_wb.sh`  script allows for a run / rerun of a HUC, or running other HUCs at different times / days or even different docker containers.
 3. `fim_post_processing.sh` : This section takes all of the HUCs that have been processed, aggregates key information from each HUC directory and looks for errors across all HUC folders. It also processes the HUC group in sub-steps such as usgs guages processesing, rating curve adjustments and more. Naturally, running or re-running this script can only be done after running `fim_pre_processing.sh` and at least one run of `fim_process_unit_wb.sh`.
 
@@ -178,9 +178,9 @@ python /foss_fim/tools/synthesize_test_cases.py --help
 ----
 ### Managing Dependencies
 
-Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/). 
+Dependencies are managed via [Pipenv](https://pipenv.pypa.io/en/latest/).
 
-When you execute `docker build` from the `Installation` section above, all of the dependencies you need are included. This includes dependencies for you to work in JupyterLab for testing purposes. 
+When you execute `docker build` from the `Installation` section above, all of the dependencies you need are included. This includes dependencies for you to work in JupyterLab for testing purposes.
 
 While very rare, you may want to add more dependencies. You can follow the following steps:
 
@@ -189,16 +189,16 @@ While very rare, you may want to add more dependencies. You can follow the follo
     pipenv install <your package name> --dev
     ```
     The `--dev` flag adds development dependencies, omit it if you want to add a production dependency.
-    
+
     This will automatically update the Pipfile in the root of your docker container directory. If the environment looks goods after adding dependencies, lock it with:
 
     ```bash
     pipenv lock
     ```
 
-    This will update the `Pipfile.lock`. Copy the new updated `Pipfile` and `Pipfile.lock` in the FIM source directory and include both in your git commits. The docker image installs the environment from the lock file. 
+    This will update the `Pipfile.lock`. Copy the new updated `Pipfile` and `Pipfile.lock` in the FIM source directory and include both in your git commits. The docker image installs the environment from the lock file.
+
 
-    
 **Make sure you test it heavily including create new docker images and that it continues to work with the code.**
 
 If you are on a machine that has a particularly slow internet connection, you may need to increase the timeout of pipenv. To do this simply add `PIPENV_INSTALL_TIMEOUT=10000000` in front of any of your pipenv commands.
diff --git a/config/aws_s3_put_fim3_hydrovis_whitelist.lst b/config/aws_s3_put_fim3_hydrovis_whitelist.lst
index b81404832..5d8ed5ca7 100644
--- a/config/aws_s3_put_fim3_hydrovis_whitelist.lst
+++ b/config/aws_s3_put_fim3_hydrovis_whitelist.lst
@@ -1,4 +1,4 @@
 hydroTable{}.csv
 gw_catchments_reaches_filtered_addedAttributes{}.tif
 rem_zeroed_masked{}.tif
-usgs_elev_table.csv
\ No newline at end of file
+usgs_elev_table.csv
diff --git a/config/aws_s3_put_fim4_hydrovis_whitelist.lst b/config/aws_s3_put_fim4_hydrovis_whitelist.lst
index 0b1345ef3..f3726de7e 100644
--- a/config/aws_s3_put_fim4_hydrovis_whitelist.lst
+++ b/config/aws_s3_put_fim4_hydrovis_whitelist.lst
@@ -2,4 +2,4 @@ hydroTable{}.csv
 gw_catchments_reaches_filtered_addedAttributes{}.tif
 rem_zeroed_masked{}.tif
 usgs_elev_table.csv
-crosswalk_table.csv
\ No newline at end of file
+crosswalk_table.csv
diff --git a/config/deny_branches.lst b/config/deny_branches.lst
index 9e7d3924b..8cce17df4 100644
--- a/config/deny_branches.lst
+++ b/config/deny_branches.lst
@@ -19,7 +19,7 @@ demDerived_reaches_{}.shp
 demDerived_reaches_{}.shx
 demDerived_reaches_split_{}.gpkg
 demDerived_reaches_split_filtered_{}.gpkg
-#demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
+# demDerived_reaches_split_filtered_addedAttributes_crosswalked_{}.gpkg
 demDerived_reaches_split_points_{}.gpkg
 demDerived_streamPixels_{}.tif
 demDerived_streamPixels_ids_{}.tif
@@ -39,10 +39,10 @@ gw_catchments_pixels_{}.tif
 gw_catchments_reaches_{}.gpkg
 gw_catchments_reaches_{}.tif
 gw_catchments_reaches_filtered_addedAttributes_{}.gpkg
-#gw_catchments_reaches_filtered_addedAttributes_{}.tif
-#gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
+# gw_catchments_reaches_filtered_addedAttributes_{}.tif
+# gw_catchments_reaches_filtered_addedAttributes_crosswalked_{}.gpkg
 headwaters_{}.tif
-#hydroTable_{}.csv
+# hydroTable_{}.csv
 idFile_{}.txt
 LandSea_subset_{}.tif
 levee_levelpaths.csv
@@ -52,14 +52,14 @@ nwm_subset_streams_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
 #ras_elev_table.csv
 rem_{}.tif
-#rem_zeroed_masked_{}.tif
+# rem_zeroed_masked_{}.tif
 slopes_d8_dem_meters_{}.tif
 slopes_d8_dem_meters_masked_{}.tif
 sn_catchments_reaches_{}.tif
 src_{}.json
 src_base_{}.csv
-#src_full_crosswalked_{}.csv
+# src_full_crosswalked_{}.csv
 stage_{}.txt
 streamOrder_{}.tif
 treeFile_{}.txt
-#usgs_elev_table.csv
+# usgs_elev_table.csv
diff --git a/config/deny_unit.lst b/config/deny_unit.lst
index b2a12713d..9cf751d6a 100644
--- a/config/deny_unit.lst
+++ b/config/deny_unit.lst
@@ -9,29 +9,29 @@ agree_smogrid.tif
 agree_smogrid_allo.tif
 agree_smogrid_dist.tif
 agree_smogrid_zerod.tif
-#3d_nld_subset_levees_burned.gpkg
-#branch_id.lst
-#branch_polygons.gpkg
+# 3d_nld_subset_levees_burned.gpkg
+# branch_id.lst
+# branch_polygons.gpkg
 dem_burned.tif
 dem_burned_filled.tif
 dem_meters.tif
 flowdir_d8_burned_filled.tif
 flows_grid_boolean.tif
-#LandSea_subset.gpkg
-#LeveeProtectedAreas_subset.gpkg
-#nld_subset_levees.gpkg
-#nwm_catchments_proj_subset.gpkg
-#nwm_catchments_proj_subset_levelPaths.gpkg
+# LandSea_subset.gpkg
+# LeveeProtectedAreas_subset.gpkg
+# nld_subset_levees.gpkg
+# nwm_catchments_proj_subset.gpkg
+# nwm_catchments_proj_subset_levelPaths.gpkg
 nwm_headwaters.gpkg
-#nwm_lakes_proj_subset.gpkg
-#nwm_subset_streams.gpkg
-#nwm_subset_streams_levelPaths.gpkg
-#nwm_subset_streams_levelPaths_dissolved.gpkg
-#nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
-#ras_elev_table.csv
-#usgs_elev_table.csv
-#usgs_subset_gages.gpkg
-#wbd.gpkg
-#wbd8_clp.gpkg
+# nwm_lakes_proj_subset.gpkg
+# nwm_subset_streams.gpkg
+# nwm_subset_streams_levelPaths.gpkg
+# nwm_subset_streams_levelPaths_dissolved.gpkg
+# nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+# ras_elev_table.csv
+# usgs_elev_table.csv
+# usgs_subset_gages.gpkg
+# wbd.gpkg
+# wbd8_clp.gpkg
 wbd_buffered.gpkg
 wbd_buffered_streams.gpkg
diff --git a/config/params_template.env b/config/params_template.env
index d1f072fa4..6c2cb1687 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -36,32 +36,45 @@ export bathymetry_adjust=True
 export bathymetry_file="$inputsDir/bathymetry/bathymetry_adjustment_data.gpkg"
 
 #### estimating bankfull stage in SRCs ####
-export src_bankfull_toggle="True" # Toggle to run identify_bankfull routine (True=on; False=off)
-export bankfull_flows_file="$inputsDir/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv" # input file location with nwm feature_id and recurrence flow values
+# Toggle to run identify_bankfull routine (True=on; False=off)
+export src_bankfull_toggle="True"
+# input file location with nwm feature_id and recurrence flow values
+export bankfull_flows_file="$inputsDir/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv"
 
 #### applying channel/overbank subdivision routine to SRCs ####
-export src_subdiv_toggle="True" # Toggle to run composite roughness src routine (True=on; False=off)
-export vrough_suffix="" # text to append to output log and hydrotable file names (use for testing/debugging)
-export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv" # input file location with nwm feature_id and channel roughness and overbank roughness attributes
+# Toggle to run composite roughness src routine (True=on; False=off)
+export src_subdiv_toggle="True"
+# text to append to output log and hydrotable file names (use for testing/debugging)
+export vrough_suffix=""
+# input file location with nwm feature_id and channel roughness and overbank roughness attributes
+export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv"
 
 #### SRC calibration variables
-export nwm_recur_file="$inputsDir/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv" # input file location with nwm feature_id and recurrence flow values
+# input file location with nwm feature_id and recurrence flow values
+export nwm_recur_file="$inputsDir/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv"
 
 #### apply SRC adjustments using USGS rating curve database ####
-export src_adjust_usgs="True" # Toggle to run src adjustment routine with USGS rating data (True=on; False=off)
-export usgs_rating_curve_csv="$inputsDir/usgs_gages/usgs_rating_curves.csv" # input file location with usgs rating curve database
+# Toggle to run src adjustment routine with USGS rating data (True=on; False=off)
+export src_adjust_usgs="True"
+# input file location with usgs rating curve database
+export usgs_rating_curve_csv="$inputsDir/usgs_gages/usgs_rating_curves.csv"
 
 #### apply SRC adjustments using ras2fim rating curve database ####
-export src_adjust_ras2fim="True" # Toggle to run src adjustment routine with ras2fim data (True=on; False=off)
-export ras_rating_curve_csv="$inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table.csv" # input file location with nwm feature_id and recurrence flow values
+# Toggle to run src adjustment routine with ras2fim data (True=on; False=off)
+export src_adjust_ras2fim="True"
+# input file location with nwm feature_id and recurrence flow values
+export ras_rating_curve_csv="$inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table.csv"
 
 #### apply SRC adjustments using observed FIM/flow point database (parquet files)####
-export src_adjust_spatial="True" # Toggle to run src adjustment routine with flood edge point db (True=on; False=off)
+# Toggle to run src adjustment routine with flood edge point db (True=on; False=off)
+export src_adjust_spatial="True"
 export fim_obs_pnt_data="$inputsDir/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 
 #### apply manual calibration ####
-export manual_calb_toggle="True" # Toggle to run manual calibration routine (True=on; False=off)
-export man_calb_file="$inputsDir/rating_curve/manual_calibration_coefficients.csv" # input file location with HUC, nwm feature_id and manual calibration coefficients
+# Toggle to run manual calibration routine (True=on; False=off)
+export manual_calb_toggle="True"
+# Input file location with HUC, nwm feature_id and manual calibration coefficients
+export man_calb_file="$inputsDir/rating_curve/manual_calibration_coefficients.csv"
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/data/aws/.gitignore b/data/aws/.gitignore
index a3ad1a24c..11e0775a6 100644
--- a/data/aws/.gitignore
+++ b/data/aws/.gitignore
@@ -1,2 +1,2 @@
 *.env
-!aws_creds_template.env
\ No newline at end of file
+!aws_creds_template.env
diff --git a/data/aws/aws_base.py b/data/aws/aws_base.py
index 0dadce969..b0f6befad 100644
--- a/data/aws/aws_base.py
+++ b/data/aws/aws_base.py
@@ -1,32 +1,28 @@
 #!/usr/bin/env python3
 
-# standard library imports (https://docs.python.org/3/library/
-# (imports first, then "froms", in alpha order)
 import os
 import subprocess
 import sys
 
-sys.path.append('/foss_fim/src')
+from dotenv import load_dotenv
+
 from utils.shared_functions import FIM_Helpers as fh
 
-# third party imports
-# (imports first, then "froms", in alpha order)
-from dotenv import load_dotenv
 
-'''
-This implements all common variables related when communicating to AWS
-'''
 class AWS_Base(object):
-    
-    def __init__(self, path_to_cred_env_file, *args, **kwargs):
 
+    '''
+    This class implements all common variables related when communicating to AWS
+    '''
+
+    def __init__(self, path_to_cred_env_file, *args, **kwargs):
         '''
         Overview
         ----------
         This will load the aws credentials enviroment file.
         For now, we will feed it via an env file. Eventuallly, it should be
         changed to ~/.aws/credentials (and maybe profiles)
-        
+
         The aws_credentials_file will be loaded and an aws client
         will be created ready for use.
 
@@ -39,55 +35,52 @@ def __init__(self, path_to_cred_env_file, *args, **kwargs):
             If True, then debugging output will be included
 
         '''
-        
-        if (not os.path.exists(path_to_cred_env_file)):
+
+        if not os.path.exists(path_to_cred_env_file):
             raise FileNotFoundError("AWS credentials file not found")
-            
+
         load_dotenv(path_to_cred_env_file)
-        
+
         if kwargs:
-            is_verbose = kwargs.pop("is_verbose",None)
+            is_verbose = kwargs.pop("is_verbose", None)
 
         self.is_verbose = is_verbose
-        
+
         # TODO: validate service name with AWS (which will help
         # validate the connection)
-    
-    
+
     def get_aws_cli_credentials(self):
-        
         '''
         Overview
         ----------
         To run aws cli commands (subprocess), aws creds need to be set up
         in the command environment. This method will take care of that
         via bash "exports"
-        
+
         Returns
         -----------
         A string that can be concatenated to the front of a subprocess cmd
         and includes the export creds.
-        
+
         '''
-        
+
         fh.vprint("getting aws credential string", self.is_verbose, True)
-        
+
         cmd = "export AWS_ACCESS_KEY_ID=" + os.getenv('AWS_ACCESS_KEY')
         cmd += " && export AWS_SECRET_ACCESS_KEY=" + os.getenv('AWS_SECRET_ACCESS_KEY')
         cmd += " && export AWS_DEFAULT_REGION=" + os.getenv('AWS_REGION')
-        
+
         return cmd
-   
-   
+
     def create_aws_cli_include_argument(self, whitelist_file_names):
         '''
         Overview
         ----------
         Creates a string valid for aws_cli include commands.
-        
+
         When using an "include", this string will automatically add
         --exclude "*" , without it, the includes will not work.
-        
+
         If the whitelist_file_names is empty, then an empty string will be returned.
 
         Parameters
@@ -98,27 +91,27 @@ def create_aws_cli_include_argument(self, whitelist_file_names):
 
         Returns
         ----------
-        A string that can be added straight into a aws cli command. 
-        
-        example: export AWS_ACCESS_KEY_ID=A{somekey}Q && export 
-        AWS_SECRET_ACCESS_KEY=KW(examplekey)80 && 
+        A string that can be added straight into a aws cli command.
+
+        example: export AWS_ACCESS_KEY_ID=A{somekey}Q && export
+        AWS_SECRET_ACCESS_KEY=KW(examplekey)80 &&
         export AWS_DEFAULT_REGION=us-west-1
         '''
-        
+
         if (whitelist_file_names is None) or (len(whitelist_file_names) == 0):
             return ""  # empty string
-        
+
         # For there to be "includes", for aws cli, you must have exclude "all"
-        cli_whitelist = '--exclude "*"' 
-        
+        cli_whitelist = '--exclude "*"'
+
         for whitelist_file_name in whitelist_file_names:
             if not whitelist_file_name.startswith("*"):
                 whitelist_file_name = "*" + whitelist_file_name
-            
+
             whitelist_file_name = whitelist_file_name.replace("{}", "*")
 
             cli_whitelist += f' --include "{whitelist_file_name}"'
-        
+
         fh.vprint(f"cli include string is {cli_whitelist}", self.is_verbose, True)
-        
+
         return cli_whitelist
diff --git a/data/aws/aws_creds_template.env b/data/aws/aws_creds_template.env
index b236ef3f1..6cc8c1a8b 100644
--- a/data/aws/aws_creds_template.env
+++ b/data/aws/aws_creds_template.env
@@ -1,3 +1,3 @@
 AWS_REGION=
 AWS_ACCESS_KEY=
-AWS_SECRET_ACCESS_KEY=
\ No newline at end of file
+AWS_SECRET_ACCESS_KEY=
diff --git a/data/aws/push-hv-data-support-files.sh b/data/aws/push-hv-data-support-files.sh
index 6f4e84e69..921da624a 100644
--- a/data/aws/push-hv-data-support-files.sh
+++ b/data/aws/push-hv-data-support-files.sh
@@ -1,11 +1,11 @@
 #!/bin/bash -e
 
-#This files pushs all support files required for a HV production release,
+# This script pushes all support files required for a HV production release,
 
-#When running this script, ensure the docker container is mounted to the dev_fim_share folders
+# When running this script, ensure the docker container is mounted to the dev_fim_share folders
 
-#Note: This files is rather unpolished and as it is Git, it must not included the
-#base HV S3 folder name or AWS credentials.
+# Note: This script does not contain base HV S3 folder name or AWS credentials as it is in Git.
+# Therefore, it is rather unpolished.
 
 usage ()
 {
@@ -13,14 +13,14 @@ usage ()
     echo 'Does not included the filtered HAND files from FIM.'
     echo 'Does included key files such as catfim, fim performance and sierra test files.'
     echo 'Note: Please look at the code as there are assumed directories and pathing.'
-    echo '      Also.. for now, please pre-run the aws configure script for creds.'    
+    echo '      Also.. for now, please pre-run the aws configure script for creds.'
     echo
     echo 'Usage : push-hv-data-support-files.sh [REQ: -n <run name> ]'
     echo '                                      [REQ: -hv <HV S3 bucket name>'
     echo ''
     echo 'REQUIRED:'
     echo '  -n/--runName       : A name of HAND output release.'
-    echo '  -hv/--HV_S3_bucket : The root HV S3 bucket name. ie) s3://{hv....}' 
+    echo '  -hv/--HV_S3_bucket : The root HV S3 bucket name. ie) s3://{hv....}'
     echo ''
     echo 'OPTIONS:'
     echo '  -h/--help       : help file'
@@ -87,7 +87,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 
 aws_src="/data/catfim/"$runName"_stage_based/mapping/stage_based_catfim_sites.csv"
@@ -96,7 +96,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 # ----------------------
 # Catfim Flow Based
@@ -106,7 +106,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 
 aws_src="/data/catfim/"$runName"_flow_based/mapping/flow_based_catfim_sites.csv"
@@ -115,7 +115,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 # ----------------------
 # Fim Performance
@@ -125,7 +125,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 
 aws_src="/data/fim_performance/"$runName"/fim_performance_points.csv"
@@ -134,7 +134,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 
 aws_src="/data/fim_performance/"$runName"/fim_performance_catchments.csv"
@@ -143,7 +143,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 # ----------------------
 # Sierra Test (Rating Curve Comparison Metrics)
@@ -153,7 +153,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 
 aws_src="/data/inputs/usgs_gages/usgs_rating_curves.csv"
@@ -162,7 +162,7 @@ aws_target=" "$HV_S3_bucket"/"$runName"/qa_datasets/"
 aws_cmd="${aws_cmd_cp} ${aws_src} ${aws_target}"
 echo "command is '"$aws_cmd"'"
 eval "${aws_cmd}"
-echo 
+echo
 
 
 echo "done coping files and folders to S3"
diff --git a/data/aws/s3.py b/data/aws/s3.py
index adc2f23b0..d208b3c78 100644
--- a/data/aws/s3.py
+++ b/data/aws/s3.py
@@ -1,22 +1,17 @@
 #!/usr/bin/env python3
 
-# standard library imports (https://docs.python.org/3/library/
-# (imports first, then "froms", in alpha order)
-from operator import truediv
+
+import argparse
 import os
-from pickle import FALSE
+import subprocess
 import sys
 from datetime import datetime
 
-# third party imports
-# (imports first, then "froms", in alpha order)
-import argparse
-import subprocess
+from aws_base import AWS_Base
 
-sys.path.append('/foss_fim/src')
-from aws_base import *
 from utils.shared_functions import FIM_Helpers as fh
 
+
 '''
 This file is for communicating to any AWS S3 buckets.
 For now, it can only push to a bucket (such as hydroviz)
@@ -26,70 +21,66 @@
 Later, this will be changed to aws credentials profiles (or ~/aws/credentials (folders))
 '''
 
+
 class S3(AWS_Base):
-    
-    def put_to_bucket(self, src_folder_path,
-                      aws_target_path, whitelist_file_path = None):
-        
+    def put_to_bucket(self, src_folder_path, aws_target_path, whitelist_file_path=None):
         '''
         Overview
         ----------
         Push a data folder to hydroviz visualization (primarily for now)
-        
-        If the aws_target_folder_path folder does not exist, it will be created. If it does exist, files will
-        be just added or overwritten (no pre-clean)
-       
+
+        If the aws_target_folder_path folder does not exist, it will be created. If it does exist,
+        files will be just added or overwritten (no pre-clean)
+
         The source folder will copy all contents (files and subfolders)
         ie) aws_target_path = s3://some-address-us-east-1/test_uploads/220818
             src_folder_path = /outputs/fim_0_34_1_fr/(some files and folders)
-            
+
             Becomes: some-aws-address/test_uploads/220818/(some files and folders)
-            
+
         Note:
             - This is build on aws cli as boto3 can push up single files but is very slow.
               As of Aug 2022, they do not have a bulk boto3 recursive folder load
 
         Parameters
         ----------
-       
+
         - src_folder_path : str
             folder path of files to be copied up
-             
+
         - aws_target_path : str
-            A s3 bucket and folder locations in AWS (can not be blank) 
-             
+            A s3 bucket and folder locations in AWS (can not be blank)
+
         - whitelist_file_path : str
             A file with a set of line delimited file names that can be copied up to S3.
             Note: make sure the list file is unix encoded and not windows encoded.
             If None, then all files / folders will be pushed up
             Note: wildcard variables of * is available.
-            
+
         '''
-        whitelist_file_names = [] # if blank, it is assumed to load all
+        whitelist_file_names = []  # if blank, it is assumed to load all
 
         # --- Validate incoming values
         # test the whitelist path and load it into a collection
         if (whitelist_file_path is not None) and (len(whitelist_file_path.strip()) > 1):
-            
-            # if this list has items, then we load those items only 
+            # if this list has items, then we load those items only
             whitelist_file_names = fh.load_list_file(whitelist_file_path.strip())
-            
+
         if (self.is_verbose) and (len(whitelist_file_names) > 0):
             print("whitelist entries")
             print(whitelist_file_names)
-    
+
         # test src_folder_path
-        if (not os.path.exists(src_folder_path)):
+        if not os.path.exists(src_folder_path):
             raise ValueError("Invalid local_folder_path (src directory)")
-        src_folder_path = src_folder_path.rstrip('/') # trailing slashes
-        
+        src_folder_path = src_folder_path.rstrip('/')  # trailing slashes
+
         # test aws target folder
         if (aws_target_path is None) or (len(aws_target_path.strip()) == 0):
             raise ValueError("aws target folder path not supplied")
-        
-        aws_target_path = aws_target_path.strip() # spaces
-        aws_target_path = aws_target_path.strip('/') # leading/trailing slashes
-        
+
+        aws_target_path = aws_target_path.strip()  # spaces
+        aws_target_path = aws_target_path.strip('/')  # leading/trailing slashes
 
         # --- Upload the files
         print("************************************")
@@ -97,36 +88,34 @@ def put_to_bucket(self, src_folder_path,
         fh.print_current_date_time()
         start_dt = datetime.now()
 
-        self.bulk_upload(src_folder_path, aws_target_path, whitelist_file_names)        
+        self.bulk_upload(src_folder_path, aws_target_path, whitelist_file_names)
 
-        print("--- End uploading files")        
+        print("--- End uploading files")
         fh.print_current_date_time()
         end_dt = datetime.now()
         fh.print_date_time_duration(start_dt, end_dt)
 
-        print("************************************")        
-
+        print("************************************")
 
-    def bulk_upload(self, src_folder_path, aws_target_path, whitelist_file_names = []):
-        
+    def bulk_upload(self, src_folder_path, aws_target_path, whitelist_file_names=[]):
         '''
         Overview
         ----------
         Files will be loaded keeping their folder structure and will be loaded
         via Bash AWS CLI. It is preferred to load via boto3 but as of Aug 2022, it does
         not have a bulk loader, only a single file loader and it is very slow.
-        
+
         With CLI, we have a bit of trouble with outputs, but will track it as best we can
-        
+
         Parameters
         ----------
         - src_folder_path : str
             fully pathed location of where the files are being copied from
-            
+
         - aws_target_path : str
             The full url including subfolder path
             For example: s3://example-us-east-1/test_upload/test_2
-        
+
         - whitelist_file_names : list (but can be empty)
             A list of file names to be included in the transfer. If the file name includes
             a {}, it will be adjusted for aws cli format automatically.
@@ -140,84 +129,91 @@ def bulk_upload(self, src_folder_path, aws_target_path, whitelist_file_names = [
         is_copy_successful = False
 
         cmd = self.get_aws_cli_credentials()
-        
+
         # used cp (copy and replace. could have used 'sync' which looks for updates only)
         cmd += f' && aws s3 cp --recursive {src_folder_path}'
-        
+
         if not aws_target_path.startswith("s3://"):
             aws_target_path = "s3://" + aws_target_path
-        
+
         cmd += f' {aws_target_path}'
-        
+
         if (whitelist_file_names is not None) and (len(whitelist_file_names) > 1):
             cmd += f' {self.create_aws_cli_include_argument(whitelist_file_names)} '
 
         fh.vprint(f"cmd is {cmd}", self.is_verbose, True)
         print("")
-        
-        process = subprocess.Popen(cmd, shell = True, bufsize = 1,
-                                stdout = subprocess.PIPE, 
-                                stderr = subprocess.STDOUT,
-                                errors = 'replace'
-                                ) 
-        
+
+        process = subprocess.Popen(
+            cmd, shell=True, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, errors='replace'
+        )
+
         while True:
-            
             realtime_output = process.stdout.readline()
             if realtime_output == '' and process.poll() is not None:
                 break
             if realtime_output:
                 # AWS spits out a tons of "completes"
-                if (not realtime_output.startswith("Completed")):
+                if not realtime_output.startswith("Completed"):
                     print(realtime_output.strip(), flush=False)
                 sys.stdout.flush()
-                    
+
         is_copy_successful = True  # catching of correct error not quite working
-            
+
         return is_copy_successful
 
 
 if __name__ == '__main__':
+    '''
+    Sample Usage
+       python3 /foss_fim/data/aws/s3.py
+           -a put
+           -c /data/config/aws_creds.env
+           -s /data/previous_fim/fim_4_0_13_1
+           -t "s3://example-us-east-1/fim_4_0_13_1"
+
+    You can leave the -w flag off to load all files/folders from a directory
+    but default is to -w /foss_fim/config/aws_s3_put_fim4_hydrovis_whitelist.lst
+
+    this works for hydroviz but can work with other s3 sites as well (ie esip)
+    '''
 
-    # Sample Usage
-    #python3 /foss_fim/data/aws/s3.py -a put -c /data/config/aws_creds.env -s /data/previous_fim/fim_4_0_13_1 -t "s3://example-us-east-1/fim_4_0_13_1" 
-    
-    # You can leave the -w flag off to load all files/folders from a directory
-    # but default is to -w /foss_fim/config/aws_s3_put_fim4_hydrovis_whitelist.lst
-    
-    # this works for hydroviz but can work with other s3 sites as well (ie esip)
-            
     parser = argparse.ArgumentParser(description='Communication with aws s3 data services')
-    parser.add_argument('-a','--action_type', 
-                        help='value of get or put (defaults to put)',
-                        default="put", required=False)
-    parser.add_argument('-c','--aws_cred_env_file', 
-                        help='path to aws credentials env file', required=True)
-    parser.add_argument('-s','--local_folder_path',
-                        help='folder path of all files to be saved to or from', required=True)
-    parser.add_argument('-t','--aws_target_path',
-                        help='s3 bucket address and folder', required=True)
-    parser.add_argument('-w','--whitelist_file_path', 
-                        help='A file with the last of file names to be copied up (line delimited)',
-                        default='/foss_fim/config/aws_s3_put_fim4_hydrovis_whitelist.lst',
-                        required=False)
-    parser.add_argument('-v','--is_verbose', 
-                        help='Adding this flag will give additional tracing output',
-                        required=False, default=False, action='store_true')
+    parser.add_argument(
+        '-a', '--action_type', help='value of get or put (defaults to put)', default="put", required=False
+    )
+    parser.add_argument('-c', '--aws_cred_env_file', help='path to aws credentials env file', required=True)
+    parser.add_argument(
+        '-s', '--local_folder_path', help='folder path of all files to be saved to or from', required=True
+    )
+    parser.add_argument('-t', '--aws_target_path', help='s3 bucket address and folder', required=True)
+    parser.add_argument(
+        '-w',
+        '--whitelist_file_path',
+        help='A file with the last of file names to be copied up (line delimited)',
+        default='/foss_fim/config/aws_s3_put_fim4_hydrovis_whitelist.lst',
+        required=False,
+    )
+    parser.add_argument(
+        '-v',
+        '--is_verbose',
+        help='Adding this flag will give additional tracing output',
+        required=False,
+        default=False,
+        action='store_true',
+    )
 
     args = vars(parser.parse_args())
-    
-    if (args['action_type'] == "put"):
-        
-        s3 = S3(path_to_cred_env_file = args['aws_cred_env_file'],
-                is_verbose = args['is_verbose'])
-        s3.put_to_bucket(src_folder_path = args['local_folder_path'],
-                         aws_target_path = args['aws_target_path'],
-                         whitelist_file_path = args['whitelist_file_path'])
-        
-    elif (args["action_type"] == "get"):
+
+    if args['action_type'] == "put":
+        s3 = S3(path_to_cred_env_file=args['aws_cred_env_file'], is_verbose=args['is_verbose'])
+        s3.put_to_bucket(
+            src_folder_path=args['local_folder_path'],
+            aws_target_path=args['aws_target_path'],
+            whitelist_file_path=args['whitelist_file_path'],
+        )
+
+    elif args["action_type"] == "get":
         raise Exception("Error: get method not yet implemented or available")
     else:
         raise Exception("Error: action type value invalid. Current options are: 'put' (more coming soon) ")
-
-
diff --git a/data/bathymetry/preprocess_bathymetry.py b/data/bathymetry/preprocess_bathymetry.py
index 2fe0f4bf5..f4788c996 100644
--- a/data/bathymetry/preprocess_bathymetry.py
+++ b/data/bathymetry/preprocess_bathymetry.py
@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 
-import rasterio
-import pandas as pd
+import os
+from argparse import ArgumentParser
+
 import geopandas as gpd
 import numpy as np
+import pandas as pd
+import rasterio
+from osgeo import gdal, gdal_array
 from rasterstats import zonal_stats
-from osgeo import gdal
-from osgeo import gdal_array
-from argparse import ArgumentParser
-import os
+
 
 """
 To use this script you will need to create a tif file from the eHydro depth survey tin file.
@@ -21,8 +22,10 @@
         'CRS_OUTPUT':QgsCoordinateReferenceSystem('EPSG:5070'),'OUTPUT':'C:/Users/riley.mcdermott/Documents
         /Bathymetry_PA/Allegheny_USACE_Depth_ft.tif'})
 """
+
+
 def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_threshold):
-    """ Function for calculating missing reach averaged cross sectional 
+    """Function for calculating missing reach averaged cross sectional
     area and wetted perimeter by NWM feature_id.
 
         Parameters
@@ -31,7 +34,7 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
             Path to depth tif file from rasterized USACE eHydro survey tin, e.g.
             "/data/inputs/bathymetry/HU_01_TRY_20211108_CS_5043_30.tif".
         bathy_bounds : str
-            Name of survey extent vector layer within given geodatabase, 
+            Name of survey extent vector layer within given geodatabase,
             defaults to 'SurveyJob'.
         survey_gdb : str
             Path to survey geodatabase containing vector survey extent layer, e.g.
@@ -40,44 +43,58 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
             Path to output location for NWM flows geopackage, e.g.
             "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
         min_depth_threshold : int
-            Highest value allowed for the minimum depth of the survey 
+            Highest value allowed for the minimum depth of the survey
             in feet (Checks for data referenced to a datum).
 
     """
+
     with rasterio.open(tif) as bathy_ft:
         bathy_affine = bathy_ft.transform
         bathy_ft = bathy_ft.read(1)
-        bathy_ft[np.where(bathy_ft == -9999.)] = np.nan
+        bathy_ft[np.where(bathy_ft == -9999.0)] = np.nan
     survey_min_depth = np.nanmin(bathy_ft)
-    assert survey_min_depth < min_depth_threshold, f"The minimum depth value of the survey is {survey_min_depth} which exceeds the minimum depth threshold. This may indicate depth values are based on a datum." 
-    assert survey_min_depth > 0, f"The minimum depth value of the survey is {survey_min_depth}, negative values may indicate a problem with the datum conversion or survey."
-    bathy_m = bathy_ft/3.28084
+
+    assert survey_min_depth < min_depth_threshold, (
+        f"The minimum depth value of the survey is {survey_min_depth} which exceeds the minimum depth "
+        "threshold. This may indicate depth values are based on a datum."
+    )
+
+    assert survey_min_depth > 0, (
+        f"The minimum depth value of the survey is {survey_min_depth}, negative values may indicate a "
+        "problem with the datum conversion or survey."
+    )
+
+    bathy_m = bathy_ft / 3.28084
     bathy_gdal = gdal_array.OpenArray(bathy_m)
 
     # Read in shapefiles
-    bathy_bounds = gpd.read_file(survey_gdb, layer = bathy_bounds)
-    nwm_streams = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_flows.gpkg", mask = bathy_bounds)
-    nwm_catchments = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", mask = bathy_bounds)
+    bathy_bounds = gpd.read_file(survey_gdb, layer=bathy_bounds)
+    nwm_streams = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_flows.gpkg", mask=bathy_bounds)
+    nwm_catchments = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", mask=bathy_bounds)
     bathy_bounds = bathy_bounds.to_crs(nwm_streams.crs)
 
     # Find missing volume from depth tif
-    zs_area = zonal_stats(nwm_catchments, bathy_m, stats = ["sum"], affine= bathy_affine, geojson_out = True, nodata=np.nan)
+    zs_area = zonal_stats(
+        nwm_catchments, bathy_m, stats=["sum"], affine=bathy_affine, geojson_out=True, nodata=np.nan
+    )
     zs_area = gpd.GeoDataFrame.from_features(zs_area)
     zs_area = zs_area.set_crs(nwm_streams.crs)
-    zs_area = zs_area.rename(columns = {"sum":"missing_volume_m3"})
+    zs_area = zs_area.rename(columns={"sum": "missing_volume_m3"})
 
     # Derive slope tif
     output_slope_tif = os.path.join(os.path.dirname(tif), 'bathy_slope.tif')
-    slope_tif = gdal.DEMProcessing(output_slope_tif, bathy_gdal , 'slope', format = 'GTiff')
+    slope_tif = gdal.DEMProcessing(output_slope_tif, bathy_gdal, 'slope', format='GTiff')
     slope_tif = slope_tif.GetRasterBand(1).ReadAsArray()
     os.remove(output_slope_tif)
-    slope_tif[np.where(slope_tif == -9999.)] = np.nan
-    missing_bed_area = (1 / np.cos(slope_tif*np.pi/180)) - 1
+    slope_tif[np.where(slope_tif == -9999.0)] = np.nan
+    missing_bed_area = (1 / np.cos(slope_tif * np.pi / 180)) - 1
 
     # Find missing bed area from slope tif
-    zs_slope = zonal_stats(nwm_catchments, missing_bed_area, stats = ["sum"], affine= bathy_affine, geojson_out = True, nodata=np.nan)
+    zs_slope = zonal_stats(
+        nwm_catchments, missing_bed_area, stats=["sum"], affine=bathy_affine, geojson_out=True, nodata=np.nan
+    )
     zs_slope = gpd.GeoDataFrame.from_features(zs_slope)
-    zs_slope = zs_slope.rename(columns = {"sum":"missing_bed_area_m2"})
+    zs_slope = zs_slope.rename(columns={"sum": "missing_bed_area_m2"})
 
     # Clip streams to survey bounds and find new reach lengths
     nwm_streams_clip = nwm_streams.clip(bathy_bounds)
@@ -87,11 +104,15 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
     bathy_nwm_streams = nwm_streams_clip.merge(zs_area[['missing_volume_m3', 'ID']], on='ID')
     bathy_nwm_streams = bathy_nwm_streams.merge(zs_slope[['missing_bed_area_m2', 'ID']], on='ID')
     max_order = bathy_nwm_streams["order_"].max()
-    bathy_nwm_streams = bathy_nwm_streams.loc[bathy_nwm_streams['order_'] >= (max_order-1)]
+    bathy_nwm_streams = bathy_nwm_streams.loc[bathy_nwm_streams['order_'] >= (max_order - 1)]
 
     # Calculate wetted perimeter and cross sectional area
-    bathy_nwm_streams['missing_xs_area_m2'] = bathy_nwm_streams['missing_volume_m3']/bathy_nwm_streams['Length']
-    bathy_nwm_streams['missing_wet_perimeter_m'] = bathy_nwm_streams['missing_bed_area_m2']/bathy_nwm_streams['Length']
+    bathy_nwm_streams['missing_xs_area_m2'] = (
+        bathy_nwm_streams['missing_volume_m3'] / bathy_nwm_streams['Length']
+    )
+    bathy_nwm_streams['missing_wet_perimeter_m'] = (
+        bathy_nwm_streams['missing_bed_area_m2'] / bathy_nwm_streams['Length']
+    )
 
     # Add survey meta data
     bathy_nwm_streams['SurveyDateStamp'] = bathy_bounds.loc[0, 'SurveyDateStamp']
@@ -106,17 +127,37 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
         print(f"{output} already exists. Concatinating now...")
         existing_bathy_file = gpd.read_file(output)
         bathy_nwm_streams = pd.concat([existing_bathy_file, bathy_nwm_streams])
-    bathy_nwm_streams.to_file(output, index = False)
+    bathy_nwm_streams.to_file(output, index=False)
     print(f"Added {num_streams} new NWM features")
 
+
 if __name__ == '__main__':
-    
     parser = ArgumentParser(description="Preprocessed Bathymetry")
-    parser.add_argument('-tif','--tif', help='Survey Depth Raster', required=True,type=str)
-    parser.add_argument('-survey_gdb','--survey_gdb', help='Survey Geodatabase Ex. AL_LP_EMS_20150809_CS_005_065_SORT.gbd', required=True,type=str)
-    parser.add_argument('-bathy_bounds','--bathy_bounds', help='Survey Bounds Layer Ex. SurveyJob.gpkg', default = 'SurveyJob', required=False,type=str)
-    parser.add_argument('-output','--output', help='output geopackage location', required=True,type=str)
-    parser.add_argument('-min_depth_threshold','--min_depth_threshold', help='minimum expected depth value', required=False,type=int, default= 10)
+    parser.add_argument('-tif', '--tif', help='Survey Depth Raster', required=True, type=str)
+    parser.add_argument(
+        '-survey_gdb',
+        '--survey_gdb',
+        help='Survey Geodatabase Ex. AL_LP_EMS_20150809_CS_005_065_SORT.gbd',
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-bathy_bounds',
+        '--bathy_bounds',
+        help='Survey Bounds Layer Ex. SurveyJob.gpkg',
+        default='SurveyJob',
+        required=False,
+        type=str,
+    )
+    parser.add_argument('-output', '--output', help='output geopackage location', required=True, type=str)
+    parser.add_argument(
+        '-min_depth_threshold',
+        '--min_depth_threshold',
+        help='minimum expected depth value',
+        required=False,
+        type=int,
+        default=10,
+    )
 
     args = vars(parser.parse_args())
 
diff --git a/data/ble/ble_benchmark/README.md b/data/ble/ble_benchmark/README.md
index 7d2a900a2..b64d484f4 100644
--- a/data/ble/ble_benchmark/README.md
+++ b/data/ble/ble_benchmark/README.md
@@ -1,12 +1,18 @@
 ## Update BLE benchmark data
-    
-Downloads and preprocesses FEMA [Base Level Engineering](https://webapps.usgs.gov/infrm/estbfe/) (BLE) benchmark datasets for purposes of evaluating FIM output. A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference dataset. The outputs will be a CSV table of flows with [`feature_id`, `discharge`] columns and a boolean (True/False) TIF inundation extent raster raster with inundated areas (True or 1) and dry areas (False or 0).
 
-As the `reference_raster` is required for preprocessing, it is assumed that `fim_pipeline.py` has previously been run for the HUCs being processed.
-    
+Downloads and preprocesses FEMA [Base Level Engineering](https://webapps.usgs.gov/infrm/estbfe/) (BLE)
+benchmark datasets for purposes of evaluating FIM output. A benchmark dataset will be transformed using
+properties (CRS, resolution) from an input reference dataset. The outputs will be a CSV table of
+flows with [`feature_id`, `discharge`] columns and a boolean (True/False) TIF inundation extent raster
+with inundated areas (True or 1) and dry areas (False or 0).
+
+As the `reference_raster` is required for preprocessing, it is assumed that `fim_pipeline.py` has previously
+been run for the HUCs being processed.
+
 ## Installation
-    
-1. A Dockerfile is provided in the root of the `/foss_fim/data/ble/ble_benchmark` directory. The Docker image can be built using the following command from `/foss_fim/data/ble/ble_benchmark`:
+
+1. A Dockerfile is provided in the root of the `/foss_fim/data/ble/ble_benchmark` directory.
+The Docker image can be built using the following command from `/foss_fim/data/ble/ble_benchmark`:
 ```
 docker build --rm --no-cache --force-rm -t dev:ble_benchmark .
 ```
@@ -56,4 +62,6 @@ python /foss_fim/data/ble/ble_benchmark/create_ble_benchmark.py \
 
 ## Notes
 
-If this is run without the `-u` option, it will run all HUCs in the input file and download nearly 1TB of data. If this is done, please notify Florence Thompson (fethomps@usgs.gov) so that the download usage can be attributed.
+If this is run without the `-u` option, it will run all HUCs in the input file and download nearly
+1TB of data. If this is done, please notify Florence Thompson (fethomps@usgs.gov) so that the download usage
+can be attributed.
diff --git a/data/ble/ble_benchmark/create_ble_benchmark.py b/data/ble/ble_benchmark/create_ble_benchmark.py
index 516d3725e..1bbd34bb0 100644
--- a/data/ble/ble_benchmark/create_ble_benchmark.py
+++ b/data/ble/ble_benchmark/create_ble_benchmark.py
@@ -1,46 +1,55 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
 from datetime import datetime
-import argparse
+from glob import glob
+from io import BytesIO
+from urllib.request import urlopen
+from zipfile import ZipFile
+
 import numpy as np
 import pandas as pd
+from create_flow_forecast_file import create_flow_forecast_file
 from osgeo import gdal
-from urllib.request import urlopen
-from io import BytesIO
-from zipfile import ZipFile
-from glob import glob
 from preprocess_benchmark import preprocess_benchmark_static
-from create_flow_forecast_file import create_flow_forecast_file
 
 
-def create_ble_benchmark(input_file: str, 
-                         save_folder: str,
-                         reference_folder: str,
-                         benchmark_folder: str,
-                         nwm_geopackage: str,
-                         ble_xs_layer_name: str,
-                         nwm_stream_layer_name: str,
-                         nwm_feature_id_field: str,
-                         huc: str = None):
+def create_ble_benchmark(
+    input_file: str,
+    save_folder: str,
+    reference_folder: str,
+    benchmark_folder: str,
+    nwm_geopackage: str,
+    ble_xs_layer_name: str,
+    nwm_stream_layer_name: str,
+    nwm_feature_id_field: str,
+    huc: str = None,
+):
     """
-    Downloads and preprocesses BLE benchmark datasets for purposes of evaluating FIM output. 
-    A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference dataset. 
-    The benchmark raster will also be converted to a boolean (True/False) raster with inundated areas (True or 1) and dry areas (False or 0).
+    Downloads and preprocesses BLE benchmark datasets for purposes of evaluating FIM output.
+    A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference
+    dataset.  The benchmark raster will also be converted to a boolean (True/False) raster with inundated
+    areas (True or 1) and dry areas (False or 0).
+
+    As the reference_raster is required for preprocessing, it is assumed that fim_pipeline.py has previously
+    been run for the HUCs being processed.
 
-    As the reference_raster is required for preprocessing, it is assumed that fim_pipeline.py has previously been run for the HUCs being processed.
-    
     Parameters
     ----------
     input_file: str
-        Path to input file (XLSX or CSV) with list of URL(s) to files to download (e.g. EBFE_urls_20230608.xlsx)
+        Path to input file (XLSX or CSV) with list of URL(s) to files to download
+        (e.g. EBFE_urls_20230608.xlsx)
     save_folder: str
-        Path to save folder where the downloaded ZIP files and extracted depth rasters will be saved (e.g., /data/temp/ble_downloads). 
+        Path to save folder where the downloaded ZIP files and extracted depth rasters will be saved
+        (e.g., /data/temp/ble_downloads).
         This folder will be created if it doesn't exist.
     reference_folder: str
-        Path to reference raster (e.g., /data/outputs/fim_4_3_12_0). This folder must exist (e.g., created by running fim_pipeline.py).
+        Path to reference raster (e.g., /data/outputs/fim_4_3_12_0). This folder must exist
+        (e.g., created by running fim_pipeline.py).
     benchmark_folder: str
-        Path to the benchmark folder (e.g., /data/test_cases/ble_test_cases/validation_data_ble). This folder will be created if it doesn't exist.
+        Path to the benchmark folder (e.g., /data/test_cases/ble_test_cases/validation_data_ble).
+        This folder will be created if it doesn't exist.
 
     Returns
     -------
@@ -65,28 +74,35 @@ def create_ble_benchmark(input_file: str,
     spatial_df = spatial_df.reset_index()
 
     # Convert size to MiB
-    spatial_df['MiB'] = np.where(spatial_df['units']=='GiB', spatial_df['size'] * 1000, spatial_df['size'])
+    spatial_df['MiB'] = np.where(spatial_df['units'] == 'GiB', spatial_df['size'] * 1000, spatial_df['size'])
 
     spatial_df, ble_geodatabase = download_and_extract_rasters(spatial_df, save_folder)
 
     for i, row in spatial_df.iterrows():
         huc = row['HUC']
         # reference_raster is used to set the metadata for benchmark_raster
-        reference_raster = os.path.join(reference_folder, f'{huc}/branches/0/rem_zeroed_masked_0.tif') 
+        reference_raster = os.path.join(reference_folder, f'{huc}/branches/0/rem_zeroed_masked_0.tif')
         for benchmark_raster in row['rasters']:
             magnitude = '100yr' if 'BLE_DEP01PCT' in benchmark_raster else '500yr'
 
             out_raster_dir = os.path.join(benchmark_folder, huc, magnitude)
             if not os.path.exists(out_raster_dir):
                 os.makedirs(out_raster_dir)
-                
+
             out_raster_path = os.path.join(out_raster_dir, f"ble_huc_{huc}_extent_{magnitude}.tif")
-            
+
             # Make benchmark inundation raster
             preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_path)
 
-            create_flow_forecast_file(huc, ble_geodatabase, nwm_geopackage, benchmark_folder, 
-                                      ble_xs_layer_name, nwm_stream_layer_name, nwm_feature_id_field)
+            create_flow_forecast_file(
+                huc,
+                ble_geodatabase,
+                nwm_geopackage,
+                benchmark_folder,
+                ble_xs_layer_name,
+                nwm_stream_layer_name,
+                nwm_feature_id_field,
+            )
 
 
 def download_and_extract_rasters(spatial_df: pd.DataFrame, save_folder: str):
@@ -107,7 +123,7 @@ def download_and_extract_rasters(spatial_df: pd.DataFrame, save_folder: str):
     out_files: list
         List of paths to extracted rasters
     """
-    
+
     depth_rasters = ['BLE_DEP0_2PCT', 'BLE_DEP01PCT']
 
     # Download and unzip each file
@@ -130,7 +146,7 @@ def download_and_extract_rasters(spatial_df: pd.DataFrame, save_folder: str):
             zipfile.extractall(path=save_file)
 
         gdb_list = glob(os.path.join(save_file, '**', '*.gdb'), recursive=True)
-            
+
         if len(gdb_list) == 1:
             ble_geodatabase = gdb_list[0]
             src_ds = gdal.Open(ble_geodatabase, gdal.GA_ReadOnly)
@@ -174,7 +190,7 @@ def extract_raster(in_raster: str, out_raster: str):
     -------
     None
     """
-    
+
     data_ds = gdal.Open(in_raster, gdal.GA_ReadOnly)
     data = data_ds.ReadAsArray()
     nodata = data_ds.GetRasterBand(1).GetNoDataValue()
@@ -189,33 +205,57 @@ def extract_raster(in_raster: str, out_raster: str):
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Create BLE benchmark files', 
-                                     usage='''python3 /foss_fim/data/ble/ble_benchmark/create_ble_benchmark.py
+    parser = argparse.ArgumentParser(
+        description='Create BLE benchmark files',
+        usage='''python3 /foss_fim/data/ble/ble_benchmark/create_ble_benchmark.py
                                                 -i /data/inputs/ble/ble_benchmark/EBFE_urls_20230608.xlsx
                                                 -s /data/temp/ble_benchmark
                                                 -r /data/outputs/my_run/
                                                 -o /data/test_cases/ble_test_cases/validation_data_ble
                                                 -n /data/inputs/nwm_hydrofabric/nwm_flows.gpkg
                                                 -u 12030105
-                                            ''')
+                                            ''',
+    )
     parser.add_argument('-i', '--input-file', type=str, help='Input file', required=True)
     parser.add_argument('-s', '--save-folder', type=str, help='Output folder', required=True)
     parser.add_argument('-r', '--reference-folder', type=str, help='Reference folder', required=True)
     parser.add_argument('-o', '--benchmark-folder', type=str, help='Benchmark folder', required=True)
     parser.add_argument('-n', '--nwm-geopackage', type=str, help='NWM streams geopackage', required=True)
-    parser.add_argument('-u', '--huc', type=str, help='Run a single HUC. If not supplied, it will run all HUCs in the input file', required=False)
-    parser.add_argument('-xs', '--ble-xs-layer-name', help='BLE cross section layer. Default layer is "XS" (sometimes it is "XS_1D").', required=False, default='XS')
-    parser.add_argument('-l', '--nwm-stream-layer-name', help='NWM streams layer. Default layer is "nwm_streams"', required=False, default='nwm_streams')
-    parser.add_argument('-id', '--nwm-feature-id-field', help='id field for nwm streams. Not required if NWM v2.1 is used (default id field is "ID")', required=False, default='ID')
+    parser.add_argument(
+        '-u',
+        '--huc',
+        type=str,
+        help='Run a single HUC. If not supplied, it will run all HUCs in the input file',
+        required=False,
+    )
+    parser.add_argument(
+        '-xs',
+        '--ble-xs-layer-name',
+        help='BLE cross section layer. Default layer is "XS" (sometimes it is "XS_1D").',
+        required=False,
+        default='XS',
+    )
+    parser.add_argument(
+        '-l',
+        '--nwm-stream-layer-name',
+        help='NWM streams layer. Default layer is "nwm_streams"',
+        required=False,
+        default='nwm_streams',
+    )
+    parser.add_argument(
+        '-id',
+        '--nwm-feature-id-field',
+        help='id field for nwm streams. Not required if NWM v2.1 is used (default id field is "ID")',
+        required=False,
+        default='ID',
+    )
 
     args = vars(parser.parse_args())
 
     start_time = datetime.now()
     print('Pulling BLE Benchmark data...  \n')
-    
+
     create_ble_benchmark(**args)
 
     end_time = datetime.now()
-    print('\n Finished Pulling BLE Benchmark data \n',
-          'Total Time: ', end_time - start_time)
-
+    print('\n Finished Pulling BLE Benchmark data \n', 'Total Time: ', end_time - start_time)
diff --git a/data/ble/ble_benchmark/create_flow_forecast_file.py b/data/ble/ble_benchmark/create_flow_forecast_file.py
index d3e91502b..62bdb7b1e 100755
--- a/data/ble/ble_benchmark/create_flow_forecast_file.py
+++ b/data/ble/ble_benchmark/create_flow_forecast_file.py
@@ -1,28 +1,32 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
+
+import fiona
+import geopandas as gpd
 import numpy as np
 import pandas as pd
-import geopandas as gpd
-import argparse
-import fiona
 
-def create_flow_forecast_file(huc,
-                              ble_geodatabase,
-                              nwm_geodatabase,
-                              output_parent_dir, 
-                              ble_xs_layer_name = 'XS', 
-                              nwm_stream_layer_name = 'nwm_streams', 
-                              nwm_feature_id_field ='ID'):
+
+def create_flow_forecast_file(
+    huc,
+    ble_geodatabase,
+    nwm_geodatabase,
+    output_parent_dir,
+    ble_xs_layer_name='XS',
+    nwm_stream_layer_name='nwm_streams',
+    nwm_feature_id_field='ID',
+):
     '''
-    This function will create a forecast flow file using ble data. 
-    It will import the ble XS layer and will intersect it with a nwm streams layer. 
+    This function will create a forecast flow file using ble data.
+    It will import the ble XS layer and will intersect it with a nwm streams layer.
     It will the perform an intersection with the BLE XS layer and the nwm river layer.
-    It will then calculate the median flow for the 100 and 500 year events for each nwm segment and convert flow from CFS to CMS. 
-    Individual forecast files (.csv) will be created  with two columns (nwm segment ID and Flow (CMS)) for the 100 year and the 500 year event. 
-    Flow field names are set to the default field names within the BLE submittal. 
-    As differing versions of the NWM river layer have different names for the ID field, 
-    this field will need to be specified as an argument to the function. 
+    It will then calculate the median flow for the 100 and 500 year events for each nwm segment and convert flow from CFS to CMS.
+    Individual forecast files (.csv) will be created  with two columns (nwm segment ID and Flow (CMS)) for the 100 year and the 500 year event.
+    Flow field names are set to the default field names within the BLE submittal.
+    As differing versions of the NWM river layer have different names for the ID field,
+    this field will need to be specified as an argument to the function.
     Output flow forecast files will be automatically named to be compatible with current FIM code,
     and will be written out in specific folder structures.
 
@@ -50,9 +54,9 @@ def create_flow_forecast_file(huc,
 
     def fill_missing_flows(forecast: pd.DataFrame, nwm_river_layer: pd.DataFrame):
         """
-        This function will fill in missing flows in the forecast dataframe. 
+        This function will fill in missing flows in the forecast dataframe.
         It will do this by finding segments that don't intersect with the forecast dataframe, and
-        then finding the upstream and downstream segments. 
+        then finding the upstream and downstream segments.
         It will then assign the average of the two to the missing segment.
 
         Parameters
@@ -70,7 +74,7 @@ def fill_missing_flows(forecast: pd.DataFrame, nwm_river_layer: pd.DataFrame):
         n = 1
         while True:
             # Merge flows with nwm_river_layer. This will be used to find upstream and downstream flow.
-            merged = nwm_river_layer.merge(forecast, left_on='ID', right_on='feature_id', how='left') 
+            merged = nwm_river_layer.merge(forecast, left_on='ID', right_on='feature_id', how='left')
 
             nonintersects = merged[merged['discharge'].isna()]
 
@@ -81,26 +85,36 @@ def fill_missing_flows(forecast: pd.DataFrame, nwm_river_layer: pd.DataFrame):
                 new_flow = []
 
                 # Find sum of upstream flow(s)
-                upstreams = merged.loc[merged['to']==row['ID']]
+                upstreams = merged.loc[merged['to'] == row['ID']]
                 # if all upstream segments have flows
-                if len(upstreams) > 0 and not upstreams['discharge'].isnull().values.any(): 
+                if len(upstreams) > 0 and not upstreams['discharge'].isnull().values.any():
                     new_flow.append(upstreams['discharge'].sum())
 
                 # Find downstream flow(s)
-                downstream = merged.loc[merged['ID']==row['to']]
+                downstream = merged.loc[merged['ID'] == row['to']]
 
                 if len(downstream) > 0:
-                    downstream_upstreams = merged.loc[merged['to']==int(downstream['ID'].iloc[0])]
-                
+                    downstream_upstreams = merged.loc[merged['to'] == int(downstream['ID'].iloc[0])]
+
                     # all segments have flow except for the one being filled in
-                    if (downstream['discharge'].isnull().sum() + downstream_upstreams['discharge'].isnull().sum()) == 1: 
-                        new_flow.append(downstream['discharge'].iloc[0] - downstream_upstreams['discharge'].sum())
+                    if (
+                        downstream['discharge'].isnull().sum()
+                        + downstream_upstreams['discharge'].isnull().sum()
+                    ) == 1:
+                        new_flow.append(
+                            downstream['discharge'].iloc[0] - downstream_upstreams['discharge'].sum()
+                        )
 
                 # Add row data to forecast
                 if len(new_flow) > 0:
-                    forecast = pd.concat([forecast, pd.DataFrame({'feature_id': row['ID'], 
-                                                                  'discharge': np.mean(new_flow)}, 
-                                                                 index=[0])]).reset_index(drop=True)
+                    forecast = pd.concat(
+                        [
+                            forecast,
+                            pd.DataFrame(
+                                {'feature_id': row['ID'], 'discharge': np.mean(new_flow)}, index=[0]
+                            ),
+                        ]
+                    ).reset_index(drop=True)
                     updated = True
 
             if not updated:
@@ -112,22 +126,21 @@ def fill_missing_flows(forecast: pd.DataFrame, nwm_river_layer: pd.DataFrame):
 
         return forecast
 
-
     # Read the ble xs layer into a geopandas dataframe.
     layers = fiona.listlayers(ble_geodatabase)
     if ble_xs_layer_name not in layers:
         ble_xs_layer_name = 'XS_1D'
-    
-    xs_layer = gpd.read_file(ble_geodatabase,layer = ble_xs_layer_name)
+
+    xs_layer = gpd.read_file(ble_geodatabase, layer=ble_xs_layer_name)
 
     # Read in the NWM stream layer into a geopandas dataframe using the bounding box option based on the extents of the BLE XS layer.
-    nwm_river_layer = gpd.read_file(nwm_geodatabase, bbox = xs_layer, layer=nwm_stream_layer_name)
+    nwm_river_layer = gpd.read_file(nwm_geodatabase, bbox=xs_layer, layer=nwm_stream_layer_name)
 
     # Make sure xs_layer is in same projection as nwm_river_layer.
     xs_layer_proj = xs_layer.to_crs(nwm_river_layer.crs)
 
     # Perform an intersection of the BLE layers and the NWM layers, using the keep_geom_type set to False produces a point output.
-    intersection = gpd.overlay(xs_layer_proj, nwm_river_layer, how='intersection', keep_geom_type = False)
+    intersection = gpd.overlay(xs_layer_proj, nwm_river_layer, how='intersection', keep_geom_type=False)
 
     ## Create the flow forecast files
     # Define fields containing flow (typically these won't change for BLE)
@@ -137,16 +150,15 @@ def fill_missing_flows(forecast: pd.DataFrame, nwm_river_layer: pd.DataFrame):
     return_period = ['100yr', '500yr']
 
     # Conversion factor from CFS to CMS
-    dischargeMultiplier = 0.3048 ** 3
+    dischargeMultiplier = 0.3048**3
 
     # Write individual flow csv files
-    for i,flow in enumerate(flow_fields):
-
+    for i, flow in enumerate(flow_fields):
         # Write dataframe with just ID and single flow event
-        forecast = intersection[[nwm_feature_id_field,flow]]
+        forecast = intersection[[nwm_feature_id_field, flow]]
 
         # Rename field names and re-define datatypes
-        forecast = forecast.rename(columns={nwm_feature_id_field:'feature_id', flow: 'discharge'})
+        forecast = forecast.rename(columns={nwm_feature_id_field: 'feature_id', flow: 'discharge'})
         forecast = forecast.astype({'feature_id': int, 'discharge': float})
 
         # Calculate median flow per feature id
@@ -162,29 +174,58 @@ def fill_missing_flows(forecast: pd.DataFrame, nwm_river_layer: pd.DataFrame):
 
         # Set paths and write file
         output_dir = os.path.join(output_parent_dir, huc)
-        dir_of_csv = os.path.join(output_dir,return_period[i])
-        os.makedirs(dir_of_csv,exist_ok = True)
-        path_to_csv = os.path.join(dir_of_csv,"ble_huc_{}_flows_{}.csv".format(huc,return_period[i]))
-        forecast.to_csv(path_to_csv,index=False)
+        dir_of_csv = os.path.join(output_dir, return_period[i])
+        os.makedirs(dir_of_csv, exist_ok=True)
+        path_to_csv = os.path.join(dir_of_csv, "ble_huc_{}_flows_{}.csv".format(huc, return_period[i]))
+        forecast.to_csv(path_to_csv, index=False)
 
 
 if __name__ == '__main__':
     # Parse arguments
     parser = argparse.ArgumentParser(description='Produce forecast flow files from BLE datasets')
-    parser.add_argument('-b', '--ble-geodatabase', help='BLE geodatabase (.gdb file extension).                               \
+    parser.add_argument(
+        '-b',
+        '--ble-geodatabase',
+        help='BLE geodatabase (.gdb file extension).                               \
         Will look for layer with "XS" in name. It is assumed the 100 year flow field is "E_Q_01PCT"                           \
-        and the 500 year flow field is "E_Q_0_2_PCT" as these are the default field names.', required=True)
-    parser.add_argument('-n', '--nwm-geodatabase',  help='NWM geodatabase (.gdb file extension).', required=True)
-    parser.add_argument('-o', '--output-parent-dir', help='Output directory where forecast files will be stored.              \
-        Two subdirectories are created (100yr and 500yr) and in each subdirectory a forecast file is written', required=True)
-    parser.add_argument('-xs', '--ble-xs-layer-name', help='BLE cross section layer.                                          \
-        Default layer is "XS" (sometimes it is "XS_1D").', required = False, default='XS')
-    parser.add_argument('-hu', '--huc', help = 'HUC', required=True)
-    parser.add_argument('-l', '--nwm-stream-layer-name', help='NWM streams layer. Default layer is "nwm_streams")',           \
-        required=False, default='nwm_streams')
-    parser.add_argument('-f', '--nwm-feature-id-field', help='id field for nwm streams.                                       \
-        Not required if NWM v2.1 is used (default id field is "ID")', required=False, default='ID')
-    
+        and the 500 year flow field is "E_Q_0_2_PCT" as these are the default field names.',
+        required=True,
+    )
+    parser.add_argument(
+        '-n', '--nwm-geodatabase', help='NWM geodatabase (.gdb file extension).', required=True
+    )
+    parser.add_argument(
+        '-o',
+        '--output-parent-dir',
+        help='Output directory where forecast files will be stored.              \
+        Two subdirectories are created (100yr and 500yr) and in each subdirectory a forecast file is written',
+        required=True,
+    )
+    parser.add_argument(
+        '-xs',
+        '--ble-xs-layer-name',
+        help='BLE cross section layer.                                          \
+        Default layer is "XS" (sometimes it is "XS_1D").',
+        required=False,
+        default='XS',
+    )
+    parser.add_argument('-hu', '--huc', help='HUC', required=True)
+    parser.add_argument(
+        '-l',
+        '--nwm-stream-layer-name',
+        help='NWM streams layer. Default layer is "nwm_streams")',
+        required=False,
+        default='nwm_streams',
+    )
+    parser.add_argument(
+        '-f',
+        '--nwm-feature-id-field',
+        help='id field for nwm streams.                                       \
+        Not required if NWM v2.1 is used (default id field is "ID")',
+        required=False,
+        default='ID',
+    )
+
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
     # Run create_flow_forecast_file
diff --git a/data/ble/ble_benchmark/preprocess_benchmark.py b/data/ble/ble_benchmark/preprocess_benchmark.py
index d17c931d4..f9b1792cc 100755
--- a/data/ble/ble_benchmark/preprocess_benchmark.py
+++ b/data/ble/ble_benchmark/preprocess_benchmark.py
@@ -1,14 +1,19 @@
 #!/usr/bin/env python3
 
+import argparse
+
+import numpy as np
 import rasterio
-from rasterio.warp import calculate_default_transform, reproject, Resampling
 import rasterio.mask
-import numpy as np
-import argparse
+from rasterio.warp import Resampling, calculate_default_transform, reproject
+
 
-def preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_path = None):
+def preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_path=None):
     '''
-    This function will preprocess a benchmark dataset for purposes of evaluating FIM output. A benchmark dataset will be transformed using properties (CRS, resolution) from an input reference dataset. The benchmark raster will also be converted to a boolean (True/False) raster with inundated areas (True or 1) and dry areas (False or 0).
+    This function will preprocess a benchmark dataset for purposes of evaluating FIM output.
+    A benchmark dataset will be transformed using properties (CRS, resolution) from an input
+    reference dataset. The benchmark raster will also be converted to a boolean (True/False) raster
+    with inundated areas (True or 1) and dry areas (False or 0).
 
     Parameters
     ----------
@@ -24,7 +29,8 @@ def preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_p
     boolean_benchmark : Numpy Array
         The preprocessed benchmark array.
     profile : STRING
-        Raster profile information for the preprocessed benchmark array (required for writing to output dataset).
+        Raster profile information for the preprocessed benchmark array
+            (required for writing to output dataset).
 
     '''
     # Open and read raster and benchmark rasters
@@ -32,52 +38,78 @@ def preprocess_benchmark_static(benchmark_raster, reference_raster, out_raster_p
     benchmark = rasterio.open(benchmark_raster)
     benchmark_arr = benchmark.read(1)
 
-    # Set arbitrary no data value that is not possible value of the benchmark dataset. This will be reassigned later
+    # Set arbitrary no data value that is not possible value of the benchmark dataset. It is reassigned later.
     nodata_value = -2147483648
 
     # Determine the new transform and dimensions of reprojected/resampled raster
-    new_transform, new_width, new_height = calculate_default_transform(benchmark.crs, reference.crs, benchmark.width, benchmark.height, *benchmark.bounds, resolution = reference.res)
+    new_transform, new_width, new_height = calculate_default_transform(
+        benchmark.crs,
+        reference.crs,
+        benchmark.width,
+        benchmark.height,
+        *benchmark.bounds,
+        resolution=reference.res,
+    )
 
     # Define an empty array that is same dimensions as output by the "calculate_default_transform" command
-    benchmark_projected = np.empty((new_height,new_width), dtype=np.int32)
+    benchmark_projected = np.empty((new_height, new_width), dtype=np.int32)
 
     # Reproject and resample the benchmark dataset. Bilinear resampling due to continuous depth data
-    reproject(benchmark_arr,
-              destination = benchmark_projected,
-              src_transform = benchmark.transform,
-              src_crs = benchmark.crs,
-              src_nodata = benchmark.nodata,
-              dst_transform = new_transform,
-              dst_crs = reference.crs,
-              dst_nodata = nodata_value,
-              dst_resolution = reference.res,
-              resampling = Resampling.bilinear)
+    reproject(
+        benchmark_arr,
+        destination=benchmark_projected,
+        src_transform=benchmark.transform,
+        src_crs=benchmark.crs,
+        src_nodata=benchmark.nodata,
+        dst_transform=new_transform,
+        dst_crs=reference.crs,
+        dst_nodata=nodata_value,
+        dst_resolution=reference.res,
+        resampling=Resampling.bilinear,
+    )
 
     # Convert entire depth grid to boolean (1 = Flood, 0 = No Flood)
     boolean_benchmark = np.where(benchmark_projected != nodata_value, 1, 0)
 
-    #Update profile (data type, NODATA, transform, width/height)
+    # Update profile (data type, NODATA, transform, width/height)
     profile = reference.profile
-    profile.update(transform = new_transform)
-    profile.update(dtype = rasterio.int8)
-    profile.update(nodata = 2) #Update NODATA to some integer so we can keep int8 datatype. There are no NODATA in the raster dataset
-    profile.update(width = new_width)
-    profile.update(height = new_height)
+    profile.update(transform=new_transform)
+    profile.update(dtype=rasterio.int8)
+    # Update NODATA to some integer so we can keep int8 datatype. There are no NODATA in the raster dataset
+    profile.update(nodata=2)
+    profile.update(width=new_width)
+    profile.update(height=new_height)
 
     # Write out preprocessed benchmark array to raster if path is supplied
     if out_raster_path is not None:
         with rasterio.Env():
             # Write out reassigned values to raster dataset
             with rasterio.open(out_raster_path, 'w', **profile) as dst:
-                dst.write(boolean_benchmark.astype('int8'),1)
+                dst.write(boolean_benchmark.astype('int8'), 1)
     return boolean_benchmark.astype('int8'), profile
 
+
 if __name__ == '__main__':
     # Parse arguments
-    parser = argparse.ArgumentParser(description = 'Preprocess BLE grids (in tiff format) for use in run_test_cast.py. Preprocessing includes reprojecting and converting to boolean raster (1 = Flooding, 0 = No Flooding)')
-    parser.add_argument('-b','--benchmark-raster', help = 'BLE depth or water surface elevation grid (in GTiff format).', required = True)
-    parser.add_argument('-r', '--reference-raster', help = 'Benchmark will use reference raster to set CRS and resolution to reference raster CRS.', required = True)
-    parser.add_argument('-o', '--out-raster-path', help = 'Output raster path (include name and extension).', required = True)
+    parser = argparse.ArgumentParser(
+        description='Preprocess BLE grids (in tiff format) for use in run_test_cast.py. '
+        'Preprocessing includes reprojecting and converting to boolean raster (1 = Flooding, 0 = No Flooding)'
+    )
+    parser.add_argument(
+        '-b',
+        '--benchmark-raster',
+        help='BLE depth or water surface elevation grid (in GTiff format).',
+        required=True,
+    )
+    parser.add_argument(
+        '-r',
+        '--reference-raster',
+        help='Benchmark will use reference raster to set CRS and resolution to reference raster CRS.',
+        required=True,
+    )
+    parser.add_argument(
+        '-o', '--out-raster-path', help='Output raster path (include name and extension).', required=True
+    )
 
     # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
diff --git a/data/create_vrt_file.py b/data/create_vrt_file.py
index 0e3c1a96b..9a6556176 100644
--- a/data/create_vrt_file.py
+++ b/data/create_vrt_file.py
@@ -4,21 +4,20 @@
 import logging
 import os
 import sys
-
 from datetime import datetime
+
 from osgeo import gdal
 
-sys.path.append('/foss_fim/src')
 from utils.shared_functions import FIM_Helpers as fh
 
+
 def create_vrt_file(src_directory, vrt_file_name):
-    
     '''
     Overview
     ----------
     Takes all .tif files in a given directory and creates a vrt from them.
     Note: This assumes all .tifs share a common directory.
-    
+
     Parameters
     ----------
         - src_directory (str):
@@ -27,21 +26,20 @@ def create_vrt_file(src_directory, vrt_file_name):
             The name of the vrt file to be created. Note: it will be in the same
             directory as the tifs.
     '''
-  
+
     # -------------------
     # Validation
-    if (not os.path.exists(src_directory)):
-        raise ValueError(f'src_directory value of {src_directory}'\
-                          ' not set to a valid path')
+    if not os.path.exists(src_directory):
+        raise ValueError(f'src_directory value of {src_directory} not set to a valid path')
 
     if (vrt_file_name is None) or (vrt_file_name == ""):
-        raise ValueError(f'vrt_file_name not defined.')
-        
-    if (not vrt_file_name.endswith(".vrt")):
+        raise ValueError('vrt_file_name not defined.')
+
+    if not vrt_file_name.endswith(".vrt"):
         vrt_file_name += ".vrt"
 
     # -------------------
-    
+
     target_vrt_file_path = os.path.join(src_directory, vrt_file_name)
 
     # -------------------
@@ -54,9 +52,9 @@ def create_vrt_file(src_directory, vrt_file_name):
 
     # -------------------
     # processing
-    
+
     tif_file_names = fh.get_file_names(src_directory, '.tif')
-        
+
     __create_vrt(tif_file_names, target_vrt_file_path)
 
     end_time = datetime.now()
@@ -65,17 +63,15 @@ def create_vrt_file(src_directory, vrt_file_name):
 
 
 def __create_vrt(tif_file_names, target_vrt_file_path):
-    
     logging.info("Files included:")
     for file_name in tif_file_names:
         logging.info(f" - {file_name}")
-    
+
     result = gdal.BuildVRT(target_vrt_file_path, tif_file_names)
     logging.info(result)
 
 
 def __setup_logger(output_folder_path):
-
     start_time = datetime.now()
     file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
     log_file_name = f"vrt_build-{file_dt_string}.log"
@@ -90,27 +86,34 @@ def __setup_logger(output_folder_path):
     logger = logging.getLogger()
     logger.addHandler(file_handler)
     logger.addHandler(console_handler)
-    logger.setLevel(logging.DEBUG)    
-    
+    logger.setLevel(logging.DEBUG)
+
     logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
     logging.info("----------------")
 
 
 if __name__ == '__main__':
-
     # Sample Usage:  python3 /foss_fim/data/create_vrt_file.py -s /data/inputs/3dep_dems/10m_5070/ -n "fim_seamless_3dep_dem_10m_5070.vrt"
 
     parser = argparse.ArgumentParser(description='Create a vrt using all tifs in a given directory')
 
-    parser.add_argument('-s','--src_directory', help='A directory of where the .tif files '\
-                        'files exist. If the -f (tif-file) param is empty then all .tif files '\
-                        'in this directory will be used.', 
-                        required=True)
-    
-    parser.add_argument('-n','--vrt_file_name', help='Name of the vrt file (name only) to be created. ' \
-                        'Note: it will be created in the source directory.', 
-                        required=True)
-    
+    parser.add_argument(
+        '-s',
+        '--src_directory',
+        help='A directory of where the .tif files '
+        'files exist. If the -f (tif-file) param is empty then all .tif files '
+        'in this directory will be used.',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-n',
+        '--vrt_file_name',
+        help='Name of the vrt file (name only) to be created. '
+        'Note: it will be created in the source directory.',
+        required=True,
+    )
+
     args = vars(parser.parse_args())
 
     create_vrt_file(**args)
diff --git a/data/esri.py b/data/esri.py
index 83e26341c..4b3f9fbd9 100644
--- a/data/esri.py
+++ b/data/esri.py
@@ -1,26 +1,28 @@
 #!/usr/bin/env python3
-import requests
-import pandas as pd
 import geopandas as gpd
+import pandas as pd
+import requests
 from tqdm import tqdm
 
+
 class ESRI_REST(object):
     """
     This class was built for querying ESRI REST endpoints for the purpose of downloading datasets.
     See /data/nld/levee_download.py for an example useage.
     """
-    
+
     def __init__(self, query_url, params, verbose=True):
         self.query_url = query_url
         self.params = params
         self.verbose = verbose
         self.exceededTransferLimit = True
         self.feature_count = 0
-    
+
     @classmethod
-    def query(cls, query_url:str, save_file:str=None, **kwargs):
+    def query(cls, query_url: str, save_file: str = None, **kwargs):
         '''
-        Classmethod for easily queries on ESRI feature services. See /data/nld/levee_download.py for example usage.
+        Classmethod for easily queries on ESRI feature services.
+        See /data/nld/levee_download.py for example usage.
 
         Parameters
         ----------
@@ -37,7 +39,7 @@ def query(cls, query_url:str, save_file:str=None, **kwargs):
         Returns
         -------
         gdf_complete: geopandas.GeoDataFrame
-            GeoDataFrame containing all of the features returned by the query. `None` is returned if the 
+            GeoDataFrame containing all of the features returned by the query. `None` is returned if the
             save_file parameter is set.
         '''
         # Query the input URL using the kwargs as URL parameters
@@ -48,7 +50,7 @@ def query(cls, query_url:str, save_file:str=None, **kwargs):
             gdf_complete.to_file(save_file, driver="GPKG", index=False)
         else:
             return gdf_complete
-        
+
     def _query_rest(self):
         '''
         Method that sets up multiple REST calls when there are more features than the transfer limit
@@ -57,7 +59,7 @@ def _query_rest(self):
         Returns
         -------
         gdf_complete: geopandas.GeoDataFrame
-            GeoDataFrame containing all of the features returned by the query. 
+            GeoDataFrame containing all of the features returned by the query.
         '''
         gdf_list = []
         record_count = 0
@@ -75,7 +77,9 @@ def _query_rest(self):
                 print(f"Total API calls: {-(self.feature_count//-self.metadata['maxRecordCount'])}")
         # Call the REST API repeatedly until all of the features have been collected, i.e. the transfer
         # limit has no longer been exceeded
-        with tqdm(total=self.feature_count, desc='Feature download progress', disable=not self.verbose) as pbar:
+        with tqdm(
+            total=self.feature_count, desc='Feature download progress', disable=not self.verbose
+        ) as pbar:
             while (self.exceededTransferLimit) and (backup_counter < 9999):
                 # Set the resultOffset to the number of records that's already been downloaded
                 self.params['resultOffset'] = record_count
@@ -111,14 +115,17 @@ def _sub_query(self, params):
         # This very nondescript error was returned when querying a polygon layer.
         # Setting resultRecordCount to a lower value fixed the error.
         elif 'error' in r_dict.keys():
-            print("There was an error with the query. It may have been caused by requesting too many features. Try setting resultRecordCount to a lower value.")
+            print(
+                "There was an error with the query. It may have been caused by requesting too many features. "
+                "Try setting resultRecordCount to a lower value."
+            )
             raise Exception(r_dict['error']['message'], f"code: {r_dict['error']['code']}")
         else:
             self.exceededTransferLimit = False
         # Read the response into a GeoDataFrame
         sub_gdf = gpd.read_file(self.response.text)
         return sub_gdf
-    
+
     def _api_call(self, url, params=None):
         '''
         Helper method for calling the API and checking that the response is ok.
@@ -128,14 +135,13 @@ def _api_call(self, url, params=None):
         self.response = requests.get(url, params=params)
         if not self.response.ok:
             raise Exception(f"The following URL recieved a bad response.\n{self.response.url}")
-    
-    def _meta_query(self):
 
+    def _meta_query(self):
         # Get the service metadata
-        self._api_call(self.query_url[:self.query_url.rfind('query')], self.params)
+        self._api_call(self.query_url[: self.query_url.rfind('query')], self.params)
         self.metadata = self.response.json()
         # Get the record count returned by the query
         params = self.params.copy()
         params['returnCountOnly'] = "true"
         self._api_call(self.query_url, params)
-        self.feature_count = self.response.json()['count']
\ No newline at end of file
+        self.feature_count = self.response.json()['count']
diff --git a/data/nld/levee_download.py b/data/nld/levee_download.py
index c7f1f41e9..995537f84 100755
--- a/data/nld/levee_download.py
+++ b/data/nld/levee_download.py
@@ -1,38 +1,43 @@
 #!/usr/bin/env python3
 
-import sys
 import os
 import re
-import geopandas as gpd
-from tqdm import tqdm
+import sys
 from datetime import datetime
+
+import geopandas as gpd
+from esri import ESRI_REST
 from shapely.geometry import LineString, MultiLineString
+from tools_shared_variables import INPUTS_DIR
+from tqdm import tqdm
 
-sys.path += ['/foss_fim/src', '/foss_fim/data', '/foss_fim/tools']
 from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
-from tools_shared_variables import INPUTS_DIR
-from esri import ESRI_REST
 
-epsg_code = re.search('\d+$', DEFAULT_FIM_PROJECTION_CRS).group()
+
+epsg_code = re.search(r'\d+$', DEFAULT_FIM_PROJECTION_CRS).group()
 today = datetime.now().strftime('%y%m%d')
 nld_vector_output = os.path.join(INPUTS_DIR, 'nld_vectors', f'System_Routes_NLDFS_5070_{today}.gpkg')
 processed_nld_vector = os.path.join(INPUTS_DIR, 'nld_vectors', f'3d_nld_preprocessed_{today}.gpkg')
 nld_protected_areas = os.path.join(INPUTS_DIR, 'nld_vectors', f'Leveed_Areas_NLDFS_5070_{today}.gpkg')
 
+
 def download_nld_lines():
     '''
-    First main function call for this module. Downloads levees from the National Levee Database 
+    First main function call for this module. Downloads levees from the National Levee Database
     ESRI service, save the raw output for use in the levee masking algorithm, processes
     the levees to remove lines and vertices that have no elevation data, and saves the preprocessed
-    levee geopackage for use in levee burning. NOTE: In order for the files generated by this script to be used, update the date (today) in bash_variables.env for input_NLD, input_levees_preprocessed, and input_nld_levee_protected_areas.
+    levee geopackage for use in levee burning.
+    NOTE: In order for the files generated by this script to be used, update the date (today) in
+        bash_variables.env for input_NLD, input_levees_preprocessed, and input_nld_levee_protected_areas.
     '''
 
     # Query REST service to download levee 'system routes'
     print("Downloading levee lines from the NLD...")
     nld_url = "https://ags03.sec.usace.army.mil/server/rest/services/NLD2_PUBLIC/FeatureServer/15/query"
-    levees = ESRI_REST.query(nld_url, 
-            f="json", where="1=1", returnGeometry="true", outFields="*", outSR=epsg_code, returnZ="true")
-                   
+    levees = ESRI_REST.query(
+        nld_url, f="json", where="1=1", returnGeometry="true", outFields="*", outSR=epsg_code, returnZ="true"
+    )
+
     # Write levees to a single geopackage
     levees.to_file(nld_vector_output, index=False, driver='GPKG')
     print(f"Levees written to file:\n{nld_vector_output}")
@@ -47,9 +52,9 @@ def download_nld_lines():
     process_levee_lines(levees, out_levees=processed_nld_vector)
 
 
-def process_levee_lines(levee_gdf:gpd.GeoDataFrame, out_levees:str):
+def process_levee_lines(levee_gdf: gpd.GeoDataFrame, out_levees: str):
     '''
-    Function for processing levee lines prior to rasterization and burning into 
+    Function for processing levee lines prior to rasterization and burning into
     the DEM. NOTE: Do not use the output of this function for the levee protected
     area masking. This dataset will be incomplete since it filters out some levees
     that have no z-values.
@@ -71,7 +76,7 @@ def process_levee_lines(levee_gdf:gpd.GeoDataFrame, out_levees:str):
     print(f"Preprocessed levees written to \n{out_levees}")
 
 
-def remove_nulls(geom:LineString, huc:str):
+def remove_nulls(geom: LineString, huc: str):
     '''
     Removes vertices from shapely LineString `geom` if they are less than `min_z`.
 
@@ -88,13 +93,13 @@ def remove_nulls(geom:LineString, huc:str):
         Filtered geometry.
     '''
     # Set min z based on HUC2
-    huc2 = huc[:2] # works with any HUC digit code
-    if huc2 in ['01','02','03','12']: # Coastal HUCs may have values near 0
+    huc2 = huc[:2]  # works with any HUC digit code
+    if huc2 in ['01', '02', '03', '12']:  # Coastal HUCs may have values near 0
         min_z = 0.01
-    elif huc2 == '08': # Louisana below sea level
+    elif huc2 == '08':  # Louisana below sea level
         min_z = -10.0
     else:
-        min_z = 1.0 # Default set to 1 ft
+        min_z = 1.0  # Default set to 1 ft
     # Loop through the vertices
     out_geom = []
     part_geom = []
@@ -104,14 +109,16 @@ def remove_nulls(geom:LineString, huc:str):
         skip_flag = False
         if coord[2] > min_z:
             # Convert units from feet to meters
-            part_geom.append(tuple([coord[0], coord[1], coord[2]*0.3048]))
+            part_geom.append(tuple([coord[0], coord[1], coord[2] * 0.3048]))
         elif skipped_vert < max_skipped_vert:
             # Allows a few (5) vertices to be skipped without forcing a multipart break.
             # This enables short sections of roads that cross levees to have the levee elevations
             # burned in to account for temporary flood walls not captured in the data.
             skip_flag = True
             skipped_vert += 1
-        elif (len(part_geom) > 1) and (not skip_flag):   # Create a multipart geometry when there's a break in z-values
+        elif (len(part_geom) > 1) and (
+            not skip_flag
+        ):  # Create a multipart geometry when there's a break in z-values
             out_geom.append(LineString(part_geom))
             part_geom = []
             skipped_vert = 0
@@ -126,27 +133,35 @@ def remove_nulls(geom:LineString, huc:str):
     else:
         return None
 
+
 def download_nld_poly():
     '''
-    Second main function call for this module. Downloads levee protected areas from the National Levee Database 
-    ESRI service and saves the raw output for use in the levee masking algorithm.
+    Second main function call for this module. Downloads levee protected areas from the
+    National Levee Database ESRI service and saves the raw output for use in the levee masking algorithm.
     '''
     # Query REST service to download levee 'system routes'
     print("Downloading levee protected areas from the NLD...")
     nld_area_url = "https://ags03.sec.usace.army.mil/server/rest/services/NLD2_PUBLIC/FeatureServer/14/query"
-    # FYI to whomever takes the time to read this code, the resultRecordCount had to be set on this query because
-    # the service was returning an error that turned out to be caused by the size of the request. Running the 
-    # default max record count of 5000 was too large for polygons, so using resultRecordCount=2000 prevents the error.
-    leveed_areas = ESRI_REST.query(nld_area_url, 
-            f="json", where="1=1", returnGeometry="true", outFields="*", outSR=epsg_code, resultRecordCount=2000)
-                   
+    # FYI to whomever takes the time to read this code, the resultRecordCount had to be set on this query
+    # because the service was returning an error that turned out to be caused by the size of the request.
+    # Running the default max record count of 5000 was too large for polygons, so using resultRecordCount=2000
+    # prevents the error.
+    leveed_areas = ESRI_REST.query(
+        nld_area_url,
+        f="json",
+        where="1=1",
+        returnGeometry="true",
+        outFields="*",
+        outSR=epsg_code,
+        resultRecordCount=2000,
+    )
+
     # Write levees to a single geopackage
     leveed_areas.to_file(nld_protected_areas, index=False, driver='GPKG')
     print(f"Levees written to file:\n{nld_protected_areas}")
 
 
 if __name__ == '__main__':
-
     download_nld_lines()
 
     download_nld_poly()
diff --git a/data/nws/.gitignore b/data/nws/.gitignore
new file mode 100644
index 000000000..5e7d2734c
--- /dev/null
+++ b/data/nws/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
diff --git a/data/nws/preprocess_ahps_nws.py b/data/nws/preprocess_ahps_nws.py
deleted file mode 100644
index 8bc6c8eb6..000000000
--- a/data/nws/preprocess_ahps_nws.py
+++ /dev/null
@@ -1,385 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import geopandas as gpd
-import numpy as np
-import os
-import pandas as pd
-import rasterio
-import sys
-import traceback
-
-from dotenv import load_dotenv
-from pathlib import Path
-sys.path.append('/foss_fim/tools')
-from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
-
-
-########################################################
-'''
-Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-   a significant review and/or upgrade.
-'''
-
-########################################################
-
-
-def get_env_paths():
-    load_dotenv()
-    #import variables from .env file
-    API_BASE_URL = os.getenv("API_BASE_URL")
-    EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
-    WBD_LAYER = os.getenv("WBD_LAYER")
-    return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER
-
-########################################################
-#Preprocess AHPS NWS
-#This script will work on NWS AHPS fim data (some assumptions made about the data structure). 
-#Provide a source directory path (source_dir) where all NWS AHPS FIM data is located. NWS source data was previously downloaded and extracted. Some data is buried through several layers of subfolders in the source data. In general, the downloaded datasets were unzipped and starting from where the folder name was the AHPS code, this was copied and pasted into a new directory which is the source_dir. 
-#Provide a destination directory path (destination) which is where all outputs are located.
-#Provide a reference raster path.
-########################################################
-# source_dir = Path(r'path/to/nws/downloads')
-# destination = Path(r'path/to/preprocessed/nws/data')
-# reference_raster= Path(r'path/to/reference raster') 
-
-
-def preprocess_nws(source_dir, destination, reference_raster):
-    source_dir = Path(source_dir)
-    destination = Path(destination)
-    reference_raster = Path(reference_raster)
-    metadata_url = f'{API_BASE_URL}/metadata' 
-    threshold_url = f'{API_BASE_URL}/nws_threshold'
-    rating_curve_url = f'{API_BASE_URL}/rating_curve'
-    log_file = destination / 'log.txt'
-    
-    #Write a run-time log file
-    destination.mkdir(parents = True, exist_ok = True)
-    log_file = destination / 'log.txt'
-    f = open(log_file, 'a+')
-    
-    #Define distance (in miles) to search for nwm segments
-    nwm_ds_search = 10
-    nwm_us_search = 10
-    #The NWS data was downloaded and unzipped. The ahps folder (with 5 digit code as folder name) was cut and pasted into a separate directory. So the ahps_codes iterates through that parent directory to get all of the AHPS codes that have data. 
-    ahps_codes = [i.name for i in source_dir.glob('*') if i.is_dir() and len(i.name) == 5]
-    #Get mainstems NWM segments
-    #Workaround for sites in 02030103 and 02030104, many are not rfc_forecast_point = True
-    list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
-    ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
-    
-    #Find depth grid subfolder
-    for code in ahps_codes:
-        f.write(f'{code} : Processing\n')
-        print(f'processing {code}')
-        #'mnda2' is in Alaska outside of NWM domain.
-        if code in ['mnda2']:
-            f.write(f'{code} : skipping because outside of NWM domain\n')
-            continue
-           
-        #Get metadata of site and search for NWM segments x miles upstream/x miles downstream
-        select_by = 'nws_lid'
-        selector = [code]    
-        metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
-        metadata = metadata_list[0]  
-    
-        #Assign huc to site using FIM huc layer.
-        dictionary, out_gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = False)
-        [huc] = list(dictionary.keys())
-      
-        #Get thresholds for action, minor, moderate, major. If no threshold data present, exit.
-        #The threshold flows source will dictate what rating curve (and datum) to use as it uses a decision tree (USGS priority then NRLDB)
-        #In multiple instances a USGS ID is given but then no USGS rating curve or in some cases no USGS datum is supplied.
-        select_by = 'nws_lid'
-        selector = code
-        stages, flows =get_thresholds(threshold_url, select_by, selector, threshold = 'all')
-    
-        #Make sure at least one valid threshold is supplied from WRDS.
-        threshold_categories = ['action','minor','moderate','major'] 
-        if not any([stages[threshold] for threshold in threshold_categories]):
-            f.write(f'{code} : skipping because no threshold stages available\n')
-            continue
-        
-        #determine source of interpolated threshold flows, this will be the rating curve that will be used.
-        rating_curve_source = flows.get('source')
-        if rating_curve_source is None:
-            f.write(f'{code} : skipping because no rating curve source\n')
-            continue
-        
-        #Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null). Modifying rating curve source will influence the rating curve and datum retrieved for benchmark determinations.
-        if code == 'bmbp1':
-            rating_curve_source = 'NRLDB'
-        
-        #Get the datum and adjust to NAVD if necessary.
-        nws, usgs = get_datum(metadata)
-        datum_data = {}
-        if rating_curve_source == 'USGS Rating Depot':
-            datum_data = usgs
-        elif rating_curve_source == 'NRLDB':
-            datum_data = nws
-    
-        #If datum not supplied, skip to new site
-        datum = datum_data.get('datum', None)
-        if datum is None:
-            f.write(f'{code} : skipping because site is missing datum\n')
-            continue        
-
-        #Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29 conversion to NAVD88
-        # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7, stkn4, wlln7 
-        # Assumed to be NAD83 (no info from USGS or NWS data): dlrt2, eagi1, eppt2, jffw3, ldot2, rgdt2
-        if code in ['bgwn7', 'dlrt2','eagi1','eppt2','fatw3','jffw3','ldot2','mnvn4','nhpp1','pinn4','rgdt2','rgln4','rssk1','sign4','smfn7','stkn4','wlln7' ]:
-            datum_data.update(crs = 'NAD83')
-        
-        #Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and is actually NAD27. This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and it matches NRLDB coordinates.
-        if code == 'bmbp1':
-            datum_data.update(crs = 'NAD27')
-        
-        #Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure datum reported in NAVD88. If NGVD29 it is converted to NAVD88.
-        #bgwn7, eagi1 vertical datum unknown, assume navd88
-        #fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
-        #wlln7 USGS data indicates vcs is NGVD29 (USGS and NWS info agree on datum value).
-        if code in ['bgwn7','eagi1','fatw3']:
-            datum_data.update(vcs = 'NAVD88')
-        elif code == 'wlln7':
-            datum_data.update(vcs = 'NGVD29')
-        
-        #Adjust datum to NAVD88 if needed
-        if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929', 'NGVD,1929']:
-            #Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
-            datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
-            datum88 = round(datum + datum_adj_ft, 2)
-        else:
-            datum88 = datum
-    
-        #get entire rating curve, same source as interpolated threshold flows (USGS Rating Depot first then NRLDB rating curve).
-        if rating_curve_source == 'NRLDB':
-            site = [code]
-        elif rating_curve_source == 'USGS Rating Depot':
-            site = [metadata.get('identifiers').get('usgs_site_code')]
-            
-        rating_curve = get_rating_curve(rating_curve_url, site)
-    
-        #Add elevation fields to rating curve
-        #Add field with vertical coordinate system
-        vcs = datum_data['vcs']
-        if not vcs:
-            vcs = 'Unspecified, Assumed NAVD88'
-        rating_curve['vcs'] = vcs
-    
-        #Add field with original datum
-        rating_curve['datum'] = datum
-        
-        #If VCS is NGVD29 add rating curve elevation (in NGVD) as well as the NAVD88 datum
-        if vcs in ['NGVD29', 'NGVD 1929']:        
-            #Add field with raw elevation conversion (datum + stage)
-            rating_curve['elevation_ngvd29'] = rating_curve['stage'] + datum
-            #Add field with adjusted NAVD88 datum
-            rating_curve['datum_navd88'] = datum88
-    
-        #Add field with NAVD88 elevation
-        rating_curve['elevation_navd88'] = rating_curve['stage'] + datum88
-    
-        
-        #Search through ahps directory find depth grid folder
-        parent_path = source_dir / code    
-          
-        #Work around for bgwn7 and smit2 where grids were custom created from polygons (bgwn7-no grids, smit2 - no projection and applying projection from polygons had errors)
-        if code in ['bgwn7', 'smit2']:
-            [grids_dir] = [directory for directory in parent_path.glob('*custom*') if directory.is_dir()]
-        else:
-            #Find the directory containing depth grids. Assumes only one directory will be returned.
-            [grids_dir] = [directory for directory in parent_path.glob('*depth_grid*') if directory.is_dir()]        
-            
-        #Get grids (all NWS ESRI grids were converted to Geotiff)
-        grid_paths = [grids for grids in grids_dir.glob('*.tif*') if grids.suffix in ['.tif', '.tiff']]
-        grid_names = [name.stem for name in grid_paths]
-        #If grids are present, interpolate a flow for the grid.
-        if grid_paths:
-            #Construct Dataframe containing grid paths, names, datum, code
-            df = pd.DataFrame({'code': code, 'path':grid_paths, 'name': grid_names, 'datum88': datum88})
-            #Determine elevation from the grid name. All elevations are assumed to be in NAVD88 based on random inspection of AHPS inundation website layers.
-            df['elevation'] = df['name'].str.replace('elev_', '', case = False).str.replace('_','.').astype(float)             
-            # Add a stage column using the datum (in NAVD88). Stage is rounded to the nearest 0.1 ft.
-            df['stage'] = round(df['elevation'] - df['datum88'],1)                                
-            #Sort stage in ascending order
-            df.sort_values(by = 'elevation', ascending = True, inplace = True)
-            #Interpolate flow from the rating curve using the elevation_navd88 values, if value is above or below the rating curve assign nan.
-            df['flow'] = np.interp(df['elevation'], rating_curve['elevation_navd88'], rating_curve['flow'], left = np.nan, right = np.nan)        
-            #Assign flow source to reflect interpolation from rc
-            df['flow_source'] = f'interpolated from {rating_curve_source} rating curve'
-
-        else: 
-            f.write(f'{code} : Site has no benchmark grids\n')
-       
-        #Select the appropriate threshold grid for evaluation. Using the supplied threshold stages and the calculated map stages. 
-        grids,grid_flows = select_grids(df, stages, datum88, 1.1)
-    
-        #workaroud for bigi1 and eag1 which have gridnames based on flows (not elevations)
-        if code in ['eagi1', 'bigi1']:
-            #Elevation is really flows (due to file names), assign this to stage
-            df['flow'] = df['elevation']
-            df['stage'] = df['elevation']
-            #Select grids using flows
-            grids, grid_flows = select_grids(df, flows, datum88, 500)
-            f.write(f'{code} : Site workaround grids names based on flows not elevation\n')
-    
-        #Obtain NWM segments that are on ms to apply flows
-        segments = get_nwm_segs(metadata)
-        site_ms_segs = set(segments).intersection(ms_segs)
-        segments = list(site_ms_segs)
-    
-        #Write out boolean benchmark raster and flow file
-        try:
-            #for each threshold
-            for i in ['action', 'minor', 'moderate', 'major']:
-                #Obtain the flow and grid associated with threshold.
-                flow = grid_flows[i]
-                grid = grids[i]
-                extent = grids['extent']
-                #Make sure that flow and flow grid are valid
-                if not grid in ['No Map', 'No Threshold', 'No Flow']:
-                    #define output directory (to be created later)
-                    outputdir = destination / huc / code / i
-                                  
-                    #Create Binary Grids, first create domain of analysis, then create binary grid
-                    
-                    #Domain extent is largest floodmap in the static library WITH holes filled
-                    filled_domain_raster = outputdir.parent / f'{code}_filled_orig_domain.tif'
-    
-                    #Open benchmark data as a rasterio object.
-                    benchmark = rasterio.open(grid)
-                    benchmark_profile = benchmark.profile   
-    
-                    #Open extent data as rasterio object
-                    domain = rasterio.open(extent)
-                    domain_profile = domain.profile
-    
-                    #if grid doesn't have CRS, then assign CRS using a polygon from the ahps inundation library
-                    if not benchmark.crs:
-                        #Obtain crs of the first polygon inundation layer associated with ahps code. Assumes only one polygon* subdirectory and assumes the polygon directory has at least 1 inundation shapefile.
-                        [ahps_polygons_directory] = [directory for directory in parent_path.glob('*polygon*') if directory.is_dir()]
-                        shapefile_path = list(ahps_polygons_directory.glob('*.shp'))[0]
-                        shapefile = gpd.read_file(shapefile_path)
-                        #Update benchmark and domain profiles with crs from shapefile. Assumed that benchmark/extent have same crs.
-                        benchmark_profile.update(crs = shapefile.crs)  
-                        domain_profile.update(crs = shapefile.crs)
-    
-                    #Create a domain raster if it does not exist.
-                    if not filled_domain_raster.exists():
-                        #Domain should have donut holes removed
-                        process_extent(domain, domain_profile, output_raster = filled_domain_raster)
-    
-    
-                    #Open domain raster as rasterio object
-                    filled_domain = rasterio.open(filled_domain_raster)
-                    filled_domain_profile = filled_domain.profile
-    
-                    #Create the binary benchmark raster                        
-                    boolean_benchmark, boolean_profile = process_grid(benchmark, benchmark_profile, filled_domain, filled_domain_profile, reference_raster)    
-                    
-                    #Output binary benchmark grid and flow file to destination
-                    outputdir.mkdir(parents = True, exist_ok = True)  
-                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_extent_{i}.tif')
-                    
-                    with rasterio.Env():
-                        with rasterio.open(output_raster, 'w', **boolean_profile) as dst:
-                            dst.write(boolean_benchmark,1) 
-                    
-                    #Close datasets
-                    domain.close()
-                    filled_domain.close()
-                    benchmark.close()
-                    
-                    #Create the guts of the flow file.
-                    flow_info = flow_data(segments,flow)
-                    #Write out the flow file to csv
-                    output_flow_file = outputdir / (f'ahps_{code}_huc_{huc}_flows_{i}.csv')
-                    flow_info.to_csv(output_flow_file, index = False)
-                    
-        except Exception as e:
-            f.write(f'{code} : Error preprocessing benchmark\n{repr(e)}\n')
-            f.write(traceback.format_exc())
-            f.write('\n')
-            print(traceback.format_exc())               
-        #Wrapup for ahps sites that were processed.
-        ahps_directory = destination / huc / code
-        if ahps_directory.exists():
-            #Delete original filled domain raster (it is an intermediate file to create benchmark data)
-            orig_domain_grid = ahps_directory / f'{code}_filled_orig_domain.tif'
-            orig_domain_grid.unlink() 
-            #Create domain shapefile from any benchmark grid for site (each benchmark has domain footprint, value = 0).
-            filled_extent = list(ahps_directory.rglob('*_extent_*.tif'))[0]
-            domain_gpd = raster_to_feature(grid = filled_extent, profile_override = False, footprint_only = True)           
-            domain_gpd['nws_lid'] = code
-            domain_gpd.to_file(ahps_directory / f'{code}_domain.shp')
-            #Populate attribute information for site
-            grids_attributes = pd.DataFrame(data=grids.items(), columns = ['magnitude','path'])
-            flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude','grid_flow_cfs'])
-            threshold_attributes = pd.DataFrame(data=stages.items(), columns = ['magnitude','magnitude_stage'])       
-            #merge dataframes
-            attributes = grids_attributes.merge(flows_attributes, on = 'magnitude')
-            attributes = attributes.merge(threshold_attributes, on = 'magnitude')
-            attributes = attributes.merge(df[['path','stage','elevation', 'flow_source']], on = 'path')        
-            #Strip out sensitive paths and convert magnitude stage to elevation
-            attributes['path'] = attributes['path'].apply(lambda x :Path(x).name)
-            attributes['magnitude_elev_navd88']=(datum88 + attributes['magnitude_stage']).astype(float).round(1)
-            #Add general site information
-            attributes['nws_lid'] = code
-            attributes['wfo'] = metadata['nws_data']['wfo']
-            attributes['rfc'] = metadata['nws_data']['rfc']
-            attributes['state'] = metadata['nws_data']['state']
-            attributes['huc'] = huc
-            #Rename and Reorder columns
-            attributes.rename(columns = {'path':'grid_name', 'flow_source':'grid_flow_source','stage':'grid_stage','elevation':'grid_elev_navd88'}, inplace = True)        
-            attributes = attributes[['nws_lid','wfo','rfc','state','huc','magnitude','magnitude_stage','magnitude_elev_navd88','grid_name','grid_stage','grid_elev_navd88', 'grid_flow_cfs','grid_flow_source']]        
-            #Save attributes to csv
-            attributes.to_csv(ahps_directory / f'{code}_attributes.csv', index = False) 
-    
-            #Write the rating curve to a file
-            rating_curve_output = ahps_directory / (f'{code}_rating_curve.csv')
-            rating_curve['lat'] = datum_data['lat']
-            rating_curve['lon'] = datum_data['lon']
-            rating_curve.to_csv(rating_curve_output, index = False)
-            
-            #Write the interpolated flows to file
-            df_output = ahps_directory / (f'{code}_interpolated_flows.csv')
-            df.to_csv(df_output, index = False)
-        
-        else: 
-            f.write(f'{code} : Unable to evaluate site, missing all flows\n')
-    
-    #Close log file.
-    f.close()
-    
-    #Combine all attribute files
-    attribute_files = list(destination.rglob('*_attributes.csv'))
-    all_attributes = pd.DataFrame()
-    for i in attribute_files:
-        attribute_df = pd.read_csv(i, dtype={'huc':str})
-        all_attributes = pd.concat([all_attributes, attribute_df])
-    
-    if not all_attributes.empty:
-        all_attributes.to_csv(destination / 'attributes.csv', index = False)    
-    return
-
-
-if __name__ == '__main__':
-    
-    ########################################################
-    '''
-    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-    a significant review and/or upgrade.
-    '''
-
-    ########################################################
-    
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
-    parser.add_argument('-s', '--source_dir', help = 'Workspace where all source data is located.', required = True)
-    parser.add_argument('-d', '--destination',  help = 'Directory where outputs are to be stored', required = True)
-    parser.add_argument('-r', '--reference_raster', help = 'reference raster used for benchmark raster creation', required = True)
-    args = vars(parser.parse_args())
-    
-
-    #Run get_env_paths and static_flow_lids
-    API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER = get_env_paths()
-    preprocess_nws(**args)
\ No newline at end of file
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index 0a0d38e16..7c3c9e1cd 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -7,19 +7,17 @@
 import subprocess
 import sys
 import traceback
-import pandas as pd
-import geopandas as gpd
-
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 from datetime import datetime
-#from tqdm import tqdm
 
-sys.path.append('/foss_fim/src')
-import utils.shared_variables as sv
-import utils.shared_functions as sf
+import geopandas as gpd
+import pandas as pd
 
+import utils.shared_functions as sf
+import utils.shared_variables as sv
 from utils.shared_functions import FIM_Helpers as fh
 
+
 '''
 TODO:
     - Change to change resolution (which means URL, block size also ahve to be parameterize)
@@ -29,17 +27,17 @@
 
 # local constants (until changed to input param)
 # This URL is part of a series of vrt data available from USGS via an S3 Bucket.
-# for more info see: "http://prd-tnm.s3.amazonaws.com/index.html?prefix=StagedProducts/Elevation/". The odd folder numbering is
-# a translation of arc seconds with 13m  being 1/3 arc second or 10 meters.
-__USGS_3DEP_10M_VRT_URL = r'/vsicurl/https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/TIFF/USGS_Seamless_DEM_13.vrt'  # 10m = 13 (1/3 arc second)
+# for more info see: "http://prd-tnm.s3.amazonaws.com/index.html?prefix=StagedProducts/Elevation/".
+# The odd folder numbering is a translation of arc seconds with 13m  being 1/3 arc second or 10 meters.
+# 10m = 13 (1/3 arc second)
+__USGS_3DEP_10M_VRT_URL = (
+    r'/vsicurl/https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/TIFF/USGS_Seamless_DEM_13.vrt'
+)
 
 
-def acquire_and_preprocess_3dep_dems(extent_file_path,
-                                     target_output_folder_path = '', 
-                                     number_of_jobs = 1, 
-                                     retry = False,
-                                     skip_polygons = False):
-    
+def acquire_and_preprocess_3dep_dems(
+    extent_file_path, target_output_folder_path='', number_of_jobs=1, retry=False, skip_polygons=False
+):
     '''
     Overview
     ----------
@@ -53,32 +51,32 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
            change that as we add the clipped version per HUC8.
         2) ensure we are adjusting blocksizes, compression and other raster params
         3) Create the 3dep rasters in the size we want (default at HUC4 for now)
-        
+
     Notes:
         - It really can be used for any huc size.
         - As this is a very low use tool, most values such as the USGS vrt path, output
           folder paths, etc are all hardcoded
         - You can really submit any polys of any HUC size as it as long as it is 10m
           resolution for now.
-        
+
     Parameters
     ----------
         - extent_file_path (str):
             Location of where the extent files that are to be used as clip extent against
             the USGS 3Dep vrt url.
-            ie) \data\inputs\wbd\HUC6
-            
+            ie) /data/inputs/wbd/HUC6
+
         - target_output_folder_path (str):
             The output location of the new 3dep dem files. When the param is not submitted,
             it will be sent to /data/input/usgs/3dep_dems/10m/.
-    
+
         - number_of_jobs (int):
             This program supports multiple procs if multiple procs/cores are available.
-            
+
         - retry (True / False):
             If retry is True and the file exists (either the raw downloaded DEM and/or)
             the projected one, then skip it
-            
+
         - skip_polygons (bool)
              If True, then we will not attempt to create polygon files for each dem file.
 
@@ -87,138 +85,129 @@ def acquire_and_preprocess_3dep_dems(extent_file_path,
     # Validation
     total_cpus_available = os.cpu_count() - 1
     if number_of_jobs > total_cpus_available:
-        raise ValueError('The number of jobs {number_of_jobs}'\
-                          'exceeds your machine\'s available CPU count minus one. '\
-                          'Please lower the number of jobs '\
-                          'values accordingly.'.format(number_of_jobs)
-                        )
-
-    if (not os.path.exists(extent_file_path)):
-        raise ValueError(f'extent_file_path value of {extent_file_path}'\
-                          ' not set to a valid path')
-    
+        raise ValueError(
+            f'The number of jobs provided: {number_of_jobs} , '
+            f'exceeds your machine\'s available CPU count minus one. '
+            f'Please lower the number of jobs '
+            f'value accordingly.'
+        )
+
+    if not os.path.exists(extent_file_path):
+        raise ValueError(f'extent_file_path value of {extent_file_path}' ' not set to a valid path')
+
     if (target_output_folder_path is None) or (target_output_folder_path == ""):
         target_output_folder_path = os.environ['usgs_3dep_dems_10m']
-    
-    if (not os.path.exists(target_output_folder_path)):
-        raise ValueError(f"Output folder path {target_output_folder_path} does not exist" )
-   
+
+    if not os.path.exists(target_output_folder_path):
+        raise ValueError(f"Output folder path {target_output_folder_path} does not exist")
+
     # -------------------
     # setup logs
     start_time = datetime.now()
     fh.print_start_header('Loading 3dep dems', start_time)
-   
-    #print(f"Downloading to {target_output_folder_path}")
+
+    # print(f"Downloading to {target_output_folder_path}")
     __setup_logger(target_output_folder_path)
     logging.info(f"Downloading to {target_output_folder_path}")
-    
-    
+
     # -------------------
     # processing
-    
+
     # Get the WBD .gpkg files (or clip extent)
     extent_file_names = fh.get_file_names(extent_file_path, 'gpkg')
     msg = f"Extent files coming from {extent_file_path}"
     print(msg)
     logging.info(msg)
-   
+
     # download dems, setting projection, block size, etc
     __download_usgs_dems(extent_file_names, target_output_folder_path, number_of_jobs, retry)
 
-    if (skip_polygons == False):
+    if skip_polygons is False:
         polygonize(target_output_folder_path)
-    
+
     end_time = datetime.now()
     fh.print_end_header('Loading 3dep dems', start_time, end_time)
-    
+
     print()
-    print('---- NOTE: Remember to scan the log file for any failures. If you find errors in the'\
-          ' log file, delete the output file and retry')
+    print(
+        '---- NOTE: Remember to scan the log file for any failures. If you find errors in the'
+        ' log file, delete the output file and retry'
+    )
     print()
     logging.info(fh.print_date_time_duration(start_time, end_time))
 
 
 def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry):
-    
     '''
     Process:
     ----------
     download the actual raw (non reprojected files) from the USGS
     based on stated embedded arguments
-    
+
     Parameters
     ----------
         - fl (object of fim_logger (must have been created))
         - remaining params are defined in acquire_and_preprocess_3dep_dems
-        
+
     Notes
     ----------
         - pixel size set to 10 x 10 (m)
         - block size (256) (sometimes we use 512)
         - cblend 6 add's a small buffer when pulling down the tif (ensuring seamless
-          overlap at the borders.)    
-    
+          overlap at the borders.)
+
     '''
 
-    print(f"==========================================================")
-    print(f"-- Downloading USGS DEMs Starting")
-    
-    base_cmd =  'gdalwarp {0} {1}'
+    print("==========================================================")
+    print("-- Downloading USGS DEMs Starting")
+
+    base_cmd = 'gdalwarp {0} {1}'
     base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
     base_cmd += ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
     base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
     base_cmd += ' -t_srs {3} -cblend 6'
-        
-    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
 
+    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
         executor_dict = {}
-        
+
         for idx, extent_file in enumerate(extent_files):
-            
-            download_dem_args = { 
-                                'extent_file': extent_file,
-                                'output_folder_path': output_folder_path,
-                                'download_url': __USGS_3DEP_10M_VRT_URL,
-                                'base_cmd':base_cmd,
-                                'retry': retry
-                                }
-        
+            download_dem_args = {
+                'extent_file': extent_file,
+                'output_folder_path': output_folder_path,
+                'download_url': __USGS_3DEP_10M_VRT_URL,
+                'base_cmd': base_cmd,
+                'retry': retry,
+            }
+
             try:
                 future = executor.submit(download_usgs_dem_file, **download_dem_args)
                 executor_dict[future] = extent_file
             except Exception as ex:
-                
-                summary = traceback.StackSummary.extract(
-                        traceback.walk_stack(None))
-                print(f"*** {ex}")                
-                print(''.join(summary.format()))    
-                
+                summary = traceback.StackSummary.extract(traceback.walk_stack(None))
+                print(f"*** {ex}")
+                print(''.join(summary.format()))
+
                 logging.critical(f"*** {ex}")
                 logging.critical(''.join(summary.format()))
 
                 sys.exit(1)
-            
+
         # Send the executor to the progress bar and wait for all tasks to finish
-        sf.progress_bar_handler(executor_dict, f"Downloading USGG 3Dep Dems")
-
-    print(f"-- Downloading USGS DEMs Completed")
-    logging.info(f"-- Downloading USGS DEMs Completed")
-    print(f"==========================================================")    
-    
-        
-def download_usgs_dem_file(extent_file, 
-                           output_folder_path, 
-                           download_url,
-                           base_cmd,
-                           retry):
+        sf.progress_bar_handler(executor_dict, "Downloading USGG 3Dep Dems")
+
+    print("-- Downloading USGS DEMs Completed")
+    logging.info("-- Downloading USGS DEMs Completed")
+    print("==========================================================")
 
+
+def download_usgs_dem_file(extent_file, output_folder_path, download_url, base_cmd, retry):
     '''
     Process:
-    ----------    
+    ----------
         Downloads just one dem file from USGS. This is setup as a method
         to allow for multi-processing.
 
-        
+
     Parameters:
     ----------
         - extent_file (str)
@@ -226,7 +215,7 @@ def download_usgs_dem_file(extent_file,
         - output_folder_path (str)
              Location of where the output file will be stored
         - download_url (str)
-             URL for the USGS download site (note: Should include '/vsicurl/' at the 
+             URL for the USGS download site (note: Should include '/vsicurl/' at the
              front of the URL)
         - base_cmd (str)
              The basic GDAL command with string formatting wholes for key values.
@@ -239,52 +228,45 @@ def download_usgs_dem_file(extent_file,
                 base_cmd +=  ' -t_srs {3} -cblend 6'
         - retry (bool)
              If True, and the file exists, downloading will be skipped.
-        
+
     '''
-    
+
     basic_file_name = os.path.basename(extent_file).split('.')[0]
-    target_file_name_raw = f"{basic_file_name}_dem.tif" # as downloaded
-    target_path_raw = os.path.join(output_folder_path,
-                                    target_file_name_raw)
-    
+    target_file_name_raw = f"{basic_file_name}_dem.tif"  # as downloaded
+    target_path_raw = os.path.join(output_folder_path, target_file_name_raw)
+
     # It does happen where the final output size can be very small (or all no-data)
     # which is related to to the spatial extents of the dem and the vrt combined.
     # so, super small .tifs are correct.
-    
+
     if (retry) and (os.path.exists(target_path_raw)):
         msg = f" - Downloading -- {target_file_name_raw} - Skipped (already exists (see retry flag))"
-        print(msg)  
+        print(msg)
         logging.info(msg)
         return
-    
+
     msg = f" - Downloading -- {target_file_name_raw} - Started"
     print(msg)
     logging.info(msg)
-            
-    cmd = base_cmd.format(download_url,
-                            target_path_raw,
-                            extent_file,
-                            sv.DEFAULT_FIM_PROJECTION_CRS)
-    #PREP_PROJECTION_EPSG
-    #fh.vprint(f"cmd is {cmd}", self.is_verbose, True)
-    #print(f"cmd is {cmd}")
-    
+
+    cmd = base_cmd.format(download_url, target_path_raw, extent_file, sv.DEFAULT_FIM_PROJECTION_CRS)
+    # PREP_PROJECTION_EPSG
+    # fh.vprint(f"cmd is {cmd}", self.is_verbose, True)
+    # print(f"cmd is {cmd}")
+
     # didn't use Popen becuase of how it interacts with multi proc
     # was creating some issues. Run worked much better.
-    process = subprocess.run(cmd, shell = True,
-                                stdout = subprocess.PIPE, 
-                                stderr = subprocess.PIPE,
-                                check = True,
-                                universal_newlines=True) 
+    process = subprocess.run(
+        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, universal_newlines=True
+    )
 
     msg = process.stdout
     print(msg)
     logging.info(msg)
-    
-    if (process.stderr != ""):
-        if ("ERROR" in process.stderr.upper()):
-            msg = f" - Downloading -- {target_file_name_raw}"\
-                    f"  ERROR -- details: ({process.stderr})"
+
+    if process.stderr != "":
+        if "ERROR" in process.stderr.upper():
+            msg = f" - Downloading -- {target_file_name_raw}" f"  ERROR -- details: ({process.stderr})"
             print(msg)
             logging.error(msg)
             os.remove(target_path_raw)
@@ -303,7 +285,7 @@ def polygonize(target_output_folder_path):
     msg = f" - Polygonizing -- {dem_domain_file} - Started"
     print(msg)
     logging.info(msg)
-            
+
     dem_files = glob.glob(os.path.join(target_output_folder_path, '*_dem.tif'))
     dem_gpkgs = gpd.GeoDataFrame()
 
@@ -313,7 +295,27 @@ def polygonize(target_output_folder_path):
 
         # Calculate a constant valued raster from valid DEM cells
         if not os.path.exists(edge_tif):
-            subprocess.run(['gdal_calc.py', '-A', dem_file, f'--outfile={edge_tif}', '--calc=where(A > -900, 1, 0)', '--co', 'BIGTIFF=YES', '--co', 'NUM_THREADS=ALL_CPUS', '--co', 'TILED=YES', '--co', 'COMPRESS=LZW', '--co', 'SPARSE_OK=TRUE', '--type=Byte', '--quiet'])
+            subprocess.run(
+                [
+                    'gdal_calc.py',
+                    '-A',
+                    dem_file,
+                    f'--outfile={edge_tif}',
+                    '--calc=where(A > -900, 1, 0)',
+                    '--co',
+                    'BIGTIFF=YES',
+                    '--co',
+                    'NUM_THREADS=ALL_CPUS',
+                    '--co',
+                    'TILED=YES',
+                    '--co',
+                    'COMPRESS=LZW',
+                    '--co',
+                    'SPARSE_OK=TRUE',
+                    '--type=Byte',
+                    '--quiet',
+                ]
+            )
 
         # Polygonize constant valued raster
         subprocess.run(['gdal_polygonize.py', '-8', edge_tif, '-q', '-f', 'GPKG', edge_gpkg])
@@ -326,7 +328,7 @@ def polygonize(target_output_folder_path):
             dem_gpkgs = pd.concat([dem_gpkgs, gdf])
 
         os.remove(edge_tif)
-        
+
     dem_gpkgs['DN'] = 1
     dem_dissolved = dem_gpkgs.dissolve(by='DN')
     dem_dissolved.to_file(dem_domain_file, driver='GPKG')
@@ -342,7 +344,6 @@ def polygonize(target_output_folder_path):
 
 
 def __setup_logger(output_folder_path):
-
     start_time = datetime.now()
     file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
     log_file_name = f"3Dep_downloaded-{file_dt_string}.log"
@@ -356,64 +357,91 @@ def __setup_logger(output_folder_path):
 
     logger = logging.getLogger()
     logger.addHandler(file_handler)
-    logger.setLevel(logging.DEBUG)    
-    
+    logger.setLevel(logging.DEBUG)
+
     logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
     logging.info("----------------")
 
 
 if __name__ == '__main__':
+    '''
+    sample usage (min params):
+        python3 /foss_fim/data/usgs/acquire_and_preprocess_3dep_dems.py
+            -e /data/inputs/wbd/HUC6_ESPG_5070/
+            -t /data/inputs/3dep_dems/10m_5070/
+            -r
+            -j 20
 
-    # Parse arguments.
-    
-    # sample usage (min params):
-    # - python3 /foss_fim/data/usgs/acquire_and_preprocess_3dep_dems.py -e /data/inputs/wbd/HUC6_ESPG_5070/ -t /data/inputs/3dep_dems/10m_5070/ -r -j 20
-    
-    # Notes:
-    #   - This is a very low use tool. So for now, this only can load 10m (1/3 arc second) and is using
-    #     hardcoded paths for the wbd gpkg to be used for clipping (no buffer for now).
-    #     Also hardcoded usgs 3dep urls, etc.  Minor
-    #     upgrades can easily be made for different urls, output folder paths, huc units, etc
-    #     as/if needed (command line params)
-    #   - The output path can be adjusted in case of a test reload of newer data for 3dep.
-    #     The default is /data/input/usgs/3dep_dems/10m/
-    #   - Each output file will be the name of the input poly plus "_dem.tif". ie) if the wbd gpkg
-    #     is named named "HUC8_12090301", then the output file name will be "HUC8_12090301_dem.tif"
-    #   - While you can (and should use more than one job number (if manageable by your server)),
-    #     this tool is memory intensive and needs more RAM then it needs cores / cpus. Go ahead and 
-    #     anyways and increase the job number so you are getting the most out of your RAM. Or
-    #     depending on your machine performance, maybe half of your cpus / cores. This tool will
-    #     not fail or freeze depending on the number of jobs / cores you select.
-        
-        
-    # IMPORTANT: 
-    # (Sept 2022): we do not process HUC2 of 22 (misc US pacific islands).
-    # We left in HUC2 of 19 (alaska) as we hope to get there in the semi near future
-    # They need to be removed from the input src clip directory in the first place.
-    # They can not be reliably removed in code.
-       
-    parser = argparse.ArgumentParser(description='Acquires and preprocesses USGS 3Dep dems')
-
-    parser.add_argument('-e','--extent_file_path', help='location the gpkg files that will'\
-                        ' are being used as clip regions (aka.. huc4_*.gpkg or whatever).'\
-                        ' All gpkgs in this folder will be used.', required=True)
-
-    parser.add_argument('-j','--number_of_jobs', help='Number of (jobs) cores/processes to used.', 
-                        required=False, default=1, type=int)
-
-    parser.add_argument('-r','--retry', help='If included, it will skip files that already exist.'\
-                        ' Default is all will be loaded/reloaded.', 
-                        required=False, action='store_true', default=False)
+    Notes:
+      - This is a very low use tool. So for now, this only can load 10m (1/3 arc second) and is using
+        hardcoded paths for the wbd gpkg to be used for clipping (no buffer for now).
+        Also hardcoded usgs 3dep urls, etc.  Minor
+        upgrades can easily be made for different urls, output folder paths, huc units, etc
+        as/if needed (command line params)
+      - The output path can be adjusted in case of a test reload of newer data for 3dep.
+        The default is /data/input/usgs/3dep_dems/10m/
+      - Each output file will be the name of the input poly plus "_dem.tif". ie) if the wbd gpkg
+        is named named "HUC8_12090301", then the output file name will be "HUC8_12090301_dem.tif"
+      - While you can (and should use more than one job number (if manageable by your server)),
+        this tool is memory intensive and needs more RAM then it needs cores / cpus. Go ahead and
+        anyways and increase the job number so you are getting the most out of your RAM. Or
+        depending on your machine performance, maybe half of your cpus / cores. This tool will
+        not fail or freeze depending on the number of jobs / cores you select.
+
+    IMPORTANT:
+    (Sept 2022): we do not process HUC2 of 22 (misc US pacific islands).
+    We left in HUC2 of 19 (alaska) as we hope to get there in the semi near future
+    They need to be removed from the input src clip directory in the first place.
+    They can not be reliably removed in code.
+    '''
 
-    parser.add_argument('-t','--target_output_folder_path', help='location of where the 3dep files'\
-                        ' will be saved', required=False, default='')
-    
-    parser.add_argument('-sp','--skip_polygons', help='If this flag is included, polygons of the dems'\
-                        ' will not be made', required=False, action='store_true', default=False)    
+    parser = argparse.ArgumentParser(description='Acquires and preprocesses USGS 3Dep dems')
 
+    parser.add_argument(
+        '-e',
+        '--extent_file_path',
+        help='location the gpkg files that will'
+        ' are being used as clip regions (ie: huc4_*.gpkg).'
+        ' All gpkgs in this folder will be used.',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-j',
+        '--number_of_jobs',
+        help='Number of (jobs) cores/processes to used.',
+        required=False,
+        default=1,
+        type=int,
+    )
+
+    parser.add_argument(
+        '-r',
+        '--retry',
+        help='If included, it will skip files that already exist.' ' Default is all will be loaded/reloaded.',
+        required=False,
+        action='store_true',
+        default=False,
+    )
+
+    parser.add_argument(
+        '-t',
+        '--target_output_folder_path',
+        help='location of where the 3dep files' ' will be saved',
+        required=False,
+        default='',
+    )
+
+    parser.add_argument(
+        '-sp',
+        '--skip_polygons',
+        help='If this flag is included, polygons of the dems' ' will not be made',
+        required=False,
+        action='store_true',
+        default=False,
+    )
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
 
     acquire_and_preprocess_3dep_dems(**args)
-
diff --git a/data/usgs/preprocess_ahps_usgs.py b/data/usgs/preprocess_ahps_usgs.py
index 500bdffe7..dddbf2c61 100644
--- a/data/usgs/preprocess_ahps_usgs.py
+++ b/data/usgs/preprocess_ahps_usgs.py
@@ -1,34 +1,49 @@
 #!/usr/bin/env python3
 import argparse
-import numpy as np
 import os
-import pandas as pd
-import rasterio
-import requests
 import sys
 import traceback
+from pathlib import Path
 
+import numpy as np
+import pandas as pd
+import rasterio
+import requests
 from dotenv import load_dotenv
-from pathlib import Path
+from tools_shared_functions import (
+    aggregate_wbd_hucs,
+    flow_data,
+    get_datum,
+    get_metadata,
+    get_nwm_segs,
+    get_rating_curve,
+    get_thresholds,
+    mainstem_nwm_segs,
+    ngvd_to_navd_ft,
+    process_extent,
+    process_grid,
+    raster_to_feature,
+    select_grids,
+)
 
-sys.path.append('/foss_fim/tools')
-from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 
 def get_env_paths():
     load_dotenv()
-    #import variables from .env file
+    # import variables from .env file
     API_BASE_URL = os.getenv("API_BASE_URL")
     EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
     WBD_LAYER = os.getenv("WBD_LAYER")
-    USGS_METADATA_URL = os.getenv("USGS_METADATA_URL")   
+    USGS_METADATA_URL = os.getenv("USGS_METADATA_URL")
     return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER, USGS_METADATA_URL
+
+
 ###############################################################################
-#Get USGS Site metadata
+# Get USGS Site metadata
 ###############################################################################
 def usgs_site_metadata(code):
     '''
     Retrieves site metadata from USGS API and saves output as dictionary. Information used includes shortname and site number.
-    
+
     Parameters
     ----------
     code : STR
@@ -45,20 +60,21 @@ def usgs_site_metadata(code):
     code = code.lower()
     # Get site metadata from USGS API using ahps code
     site_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/sites/MapServer/0/query?where=AHPS_ID+%3D+%27{code}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
-    #Get data from API
+    # Get data from API
     response = requests.get(site_url)
-    #If response is valid, then get metadata and save to dictionary
+    # If response is valid, then get metadata and save to dictionary
     if response.ok:
         response_json = response.json()
         site_metadata = response_json['features'][0]['attributes']
     return site_metadata
 
+
 ###############################################################################
-#Get USGS grid metadata
+# Get USGS grid metadata
 ###############################################################################
-def usgs_grid_metadata(code, has_grid_override = False):    
+def usgs_grid_metadata(code, has_grid_override=False):
     '''
-    Given an ahps code, retrieve the site metadata (using usgs_site_metadata) and then use that information to obtain metadata about available grids. Information includes elevation, stage, and flow for each grid. 
+    Given an ahps code, retrieve the site metadata (using usgs_site_metadata) and then use that information to obtain metadata about available grids. Information includes elevation, stage, and flow for each grid.
 
     Parameters
     ----------
@@ -70,53 +86,53 @@ def usgs_grid_metadata(code, has_grid_override = False):
     appended_dictionary : DICT
         Dictionary of metadata for each available inundation grid including grid id, flows, elevations, grid name for each inundation grid.
     '''
-    #Make sure code is in lower case
-    code = code.lower()   
+    # Make sure code is in lower case
+    code = code.lower()
     # Get site_metadata
-    site_metadata = usgs_site_metadata(code)    
-    #From site metadata get the SHORT_NAME, SITE_NO, and 'MULTI_SITE', 'HAS_GRIDS' key values
+    site_metadata = usgs_site_metadata(code)
+    # From site metadata get the SHORT_NAME, SITE_NO, and 'MULTI_SITE', 'HAS_GRIDS' key values
     short_name = site_metadata['SHORT_NAME']
     site_no = site_metadata['SITE_NO']
     has_grids = site_metadata['HAS_GRIDS']
-    #There is at least one site (kilo1) that doesn't have grids but polygons are available which have been converted grids.
+    # There is at least one site (kilo1) that doesn't have grids but polygons are available which have been converted grids.
     if has_grid_override:
         has_grids = 1
-    multi_site = site_metadata['MULTI_SITE']    
-    #Grid metadata located at one of three URLs
+    multi_site = site_metadata['MULTI_SITE']
+    # Grid metadata located at one of three URLs
     if multi_site == 0 and has_grids == 1:
         grids_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/floodExtents/MapServer/0/query?where=USGSID+%3D+%27{site_no}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
     elif multi_site > 0 and multi_site < 3 and has_grids == 1:
         grids_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/floodExtentsMulti/MapServer/0/query?where=USGSID_1+%3D+%27{site_no}%27+OR+USGSID_2+%3D+%27{site_no}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
     elif multi_site == 3 and has_grids == 1:
-        grids_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/floodExtentsThreeSites/MapServer/0/query?where=USGSID_1+%3D+%27{site_no}%27+OR+USGSID_2+%3D+%27{site_no}%27+OR+USGSID_3+%3D+%27{site_no}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'            
-    #Only get metadata on grids if site has grids available
+        grids_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/floodExtentsThreeSites/MapServer/0/query?where=USGSID_1+%3D+%27{site_no}%27+OR+USGSID_2+%3D+%27{site_no}%27+OR+USGSID_3+%3D+%27{site_no}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
+    # Only get metadata on grids if site has grids available
     if has_grids == 1:
-        #Get data from API
+        # Get data from API
         response = requests.get(grids_url)
-        #If response is valid then combine metadata on all grids into a single dictionary and write out to DataFrame.
+        # If response is valid then combine metadata on all grids into a single dictionary and write out to DataFrame.
         if response.ok:
-            response_json =response.json()
+            response_json = response.json()
             metadata = response_json['features']
             appended_dictionary = {}
             for i in metadata:
                 dictionary = i['attributes']
                 gridname = short_name + '_' + str(dictionary['GRIDID']).zfill(4)
                 appended_dictionary[gridname] = dictionary
-    else: 
+    else:
         appended_dictionary = {}
     return appended_dictionary
 
 
 ########################################################
-#Preprocess USGS FIM
-#This script will work on USGS FIM datasets. 
-#Provide source directory path (source_dir) where all USGS FIM data is located. This data was previously downloaded from USGS urls.
-#Provide a destination directory path (destination) where all outputs are located.
-#Provide a reference raster path.
+# Preprocess USGS FIM
+# This script will work on USGS FIM datasets.
+# Provide source directory path (source_dir) where all USGS FIM data is located. This data was previously downloaded from USGS urls.
+# Provide a destination directory path (destination) where all outputs are located.
+# Provide a reference raster path.
 ########################################################
-#source_dir = Path(r'path/to/usgs/downloads')
-#destination = Path(r'path/to/preprocessed/usgs/data')
-#reference_raster= Path(r'path/to/reference raster') 
+# source_dir = Path(r'path/to/usgs/downloads')
+# destination = Path(r'path/to/preprocessed/usgs/data')
+# reference_raster= Path(r'path/to/reference raster')
 def preprocess_usgs(source_dir, destination, reference_raster):
     '''
     Preprocess USGS AHPS datasets.
@@ -135,336 +151,409 @@ def preprocess_usgs(source_dir, destination, reference_raster):
     None.
 
     '''
-    
+
     source_dir = Path(source_dir)
     destination = Path(destination)
     reference_raster = Path(reference_raster)
-    metadata_url = f'{API_BASE_URL}/metadata' 
+    metadata_url = f'{API_BASE_URL}/metadata'
     threshold_url = f'{API_BASE_URL}/nws_threshold'
     rating_curve_url = f'{API_BASE_URL}/rating_curve'
-    
-    #Write log file
-    destination.mkdir(parents=True, exist_ok = True)
+
+    # Write log file
+    destination.mkdir(parents=True, exist_ok=True)
     log_file = destination / 'log.txt'
     f = open(log_file, 'a+')
-    
-    #Define distance (in miles) to search for nwm segments
+
+    # Define distance (in miles) to search for nwm segments
     nwm_ds_search = 10
     nwm_us_search = 10
-    #Need a list of AHPS codes. See "ahps_dictionaries" for method of getting this list.
+    # Need a list of AHPS codes. See "ahps_dictionaries" for method of getting this list.
     ahps_codes = [folder.name for folder in source_dir.glob('*') if len(folder.name) == 5]
-    
-    #Get mainstems NWM segments
-    #Workaround for sites in 02030103 and 02030104, many are not rfc_forecast_point = True
-    #Import list of evaluated sites
+
+    # Get mainstems NWM segments
+    # Workaround for sites in 02030103 and 02030104, many are not rfc_forecast_point = True
+    # Import list of evaluated sites
     list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
     ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
-    
+
     for code in ahps_codes:
         f.write(f'{code} : Processing\n')
         print(f'processing {code}')
-        #For a given code, find all inundation grids under that code.
+        # For a given code, find all inundation grids under that code.
         code = code.lower()
-          
-        #Get metadata of site and search for NWM segments x miles upstream/x miles downstream
+
+        # Get metadata of site and search for NWM segments x miles upstream/x miles downstream
         select_by = 'nws_lid'
-        selector = [code]    
-        metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        selector = [code]
+        metadata_list, metadata_df = get_metadata(
+            metadata_url,
+            select_by,
+            selector,
+            must_include=None,
+            upstream_trace_distance=nwm_us_search,
+            downstream_trace_distance=nwm_ds_search,
+        )
         metadata = metadata_list[0]
-        
-        #Assign huc to site using FIM huc layer.
-        dictionary, out_gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = False)
+
+        # Assign huc to site using FIM huc layer.
+        dictionary, out_gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes=False)
         [huc] = list(dictionary.keys())
-    
-        #There are 12 sites with special issues such as these don't have any crs coordinates and grid/polygon data don't align or missing grid data but polygons are available.
-        #Sites with no grid data but polygon data --> cfmm8, kilo1
-        #Sites with no projection assigned to grid and polygon/grid don't align --> stak1, nmso1, nori3, sasi3
-        #Sites with reprojection issues using rasterio (manually reprojected with ESRI) --> kcdm7, knym7, mcri2, ptvn6, tmai4
-        #Sites with incomplete grids (used polys to convert to grids) --> 'roun6'
+
+        # There are 12 sites with special issues such as these don't have any crs coordinates and grid/polygon data don't align or missing grid data but polygons are available.
+        # Sites with no grid data but polygon data --> cfmm8, kilo1
+        # Sites with no projection assigned to grid and polygon/grid don't align --> stak1, nmso1, nori3, sasi3
+        # Sites with reprojection issues using rasterio (manually reprojected with ESRI) --> kcdm7, knym7, mcri2, ptvn6, tmai4
+        # Sites with incomplete grids (used polys to convert to grids) --> 'roun6'
         ahps_dir = source_dir / code / 'depth_grids'
-        if code in ['cfmm8','kilo1','stak1', 'sasi3', 'nori3', 'nmso1', 'kcdm7', 'knym7', 'mcri2','ptvn6','tmai4', 'roun6']:        
-            f.write(f'{code} : Custom workaround related to benchmark data (mismatch crs, no grid data, etc)\n')
+        if code in [
+            'cfmm8',
+            'kilo1',
+            'stak1',
+            'sasi3',
+            'nori3',
+            'nmso1',
+            'kcdm7',
+            'knym7',
+            'mcri2',
+            'ptvn6',
+            'tmai4',
+            'roun6',
+        ]:
+            f.write(
+                f'{code} : Custom workaround related to benchmark data (mismatch crs, no grid data, etc)\n'
+            )
             ahps_dir = source_dir / code / 'custom'
-    
-        #Get thresholds (action/minor/moderate/major flows and stages), if not available exit.
-        #For USGS many sites may not have rating curves but the threshold stages are available.
-    
+
+        # Get thresholds (action/minor/moderate/major flows and stages), if not available exit.
+        # For USGS many sites may not have rating curves but the threshold stages are available.
+
         select_by = 'nws_lid'
         selector = code
-        stages, flows =get_thresholds(threshold_url, select_by, selector, threshold = 'all')
-       
-        #Make sure at least one valid threshold is supplied from WRDS.
-        threshold_categories = ['action','minor','moderate','major'] 
+        stages, flows = get_thresholds(threshold_url, select_by, selector, threshold='all')
+
+        # Make sure at least one valid threshold is supplied from WRDS.
+        threshold_categories = ['action', 'minor', 'moderate', 'major']
         if not any([stages[threshold] for threshold in threshold_categories]):
             f.write(f'{code} : Skipping because no threshold stages available\n')
             continue
-    
-        #We need to adjust stages to elevations using the datum adjustment. This next section finds the datum adjustment.
-        #determine primary source for interpolated threshold flows (USGS first then NRLDB). This will dictate what rating curve to pull.
+
+        # We need to adjust stages to elevations using the datum adjustment. This next section finds the datum adjustment.
+        # determine primary source for interpolated threshold flows (USGS first then NRLDB). This will dictate what rating curve to pull.
         rating_curve_source = flows['source']
-        #Workaround for sites that don't have rating curve but do have flows specified (USGS only). Assign rating_curve_source to 'USGS Rating Depot' manually inspected all of these sites and USGS datum is available and will be used.
-        if code in ['bdxt1','ccti3', 'fnnm7', 'mtao1', 'nfsi3', 'omot1' , 'sbrn1', 'vron4', 'watv1']:
-            rating_curve_source = 'USGS Rating Depot'         
-    
-        #Get the datum and adjust to NAVD if necessary.
+        # Workaround for sites that don't have rating curve but do have flows specified (USGS only). Assign rating_curve_source to 'USGS Rating Depot' manually inspected all of these sites and USGS datum is available and will be used.
+        if code in ['bdxt1', 'ccti3', 'fnnm7', 'mtao1', 'nfsi3', 'omot1', 'sbrn1', 'vron4', 'watv1']:
+            rating_curve_source = 'USGS Rating Depot'
+
+        # Get the datum and adjust to NAVD if necessary.
         nws, usgs = get_datum(metadata)
         datum_data = {}
         if rating_curve_source == 'USGS Rating Depot':
             datum_data = usgs
         elif rating_curve_source == 'NRLDB':
             datum_data = nws
-        
-        #If datum not supplied, skip to new site
+
+        # If datum not supplied, skip to new site
         datum = datum_data.get('datum', None)
         if datum is None:
             f.write(f'{code}: Skipping because of missing datum\n')
-            continue        
+            continue
 
-        #Custom workaround, some sites have poorly defined crs. CRS requuired for ngvd to navd conversions
+        # Custom workaround, some sites have poorly defined crs. CRS requuired for ngvd to navd conversions
         # Assumed NAVD88 (no info from USGS or NWS metadata): kynm7, ksvm7, yesg1
         # Assigned NAVD88 because USGS metadata indicates NAD83: arnm7, grfi2, kctm7, nast1, nhri3, roun6, vllm7
         # Assigned NAVD88 (reported NAVD 1988): cmtl1
-        if code in ['arnm7', 'cmtl1','grfi2','kctm7','knym7','ksvm7','nast1','nhri3','roun6','vllm7','yesg1']:
-            #Update crs to NAD83 (some are assumed, others have USGS info indicating NAD83 crs)
-            datum_data.update(crs = 'NAD83')
-        
-        #Adjust datum to NAVD88 if needed (Assumes that if vcs not NGVD29 or NGVD 1929 it is in NAVD88)
+        if code in [
+            'arnm7',
+            'cmtl1',
+            'grfi2',
+            'kctm7',
+            'knym7',
+            'ksvm7',
+            'nast1',
+            'nhri3',
+            'roun6',
+            'vllm7',
+            'yesg1',
+        ]:
+            # Update crs to NAD83 (some are assumed, others have USGS info indicating NAD83 crs)
+            datum_data.update(crs='NAD83')
+
+        # Adjust datum to NAVD88 if needed (Assumes that if vcs not NGVD29 or NGVD 1929 it is in NAVD88)
         if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929']:
-            #Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
-            datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
+            # Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
+            datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
             datum88 = round(datum + datum_adj_ft, 2)
         else:
             datum88 = datum
-    
-    
-        #Set Grid override flag, if set to True then the 'has_grids' property is ignored. Allows for custom workaround.
-        #Special exception for kilo1, where it has attribute (has_grids == 0) yet there is grid metadata and polygons were converted to grids.
+
+        # Set Grid override flag, if set to True then the 'has_grids' property is ignored. Allows for custom workaround.
+        # Special exception for kilo1, where it has attribute (has_grids == 0) yet there is grid metadata and polygons were converted to grids.
         if code == 'kilo1':
             grid_override = True
             f.write(f'{code} : Custom workaround related to "has_grids" attribute')
-        else: 
-            grid_override = False        
-        #get grid metadata (metadata includes, elevation/stage/flow and etc for each site). If empty exit.
+        else:
+            grid_override = False
+        # get grid metadata (metadata includes, elevation/stage/flow and etc for each site). If empty exit.
         grid_metadata = usgs_grid_metadata(code, has_grid_override=grid_override)
         if not grid_metadata:
             f.write(f'{code} : Skipping because no grid metadata available\n')
             continue
-         
-        #Get paths of all grids that have been downloaded, if no grids available for site then exit.
+
+        # Get paths of all grids that have been downloaded, if no grids available for site then exit.
         grid_paths = [grids for grids in ahps_dir.glob('*.tif*') if grids.suffix in ['.tif', '.tiff']]
         if not grid_paths:
             f.write(f'{code} : Skipping because no benchmark grids available\n')
             continue
-        
+
         # Iterate through grid_metadata and add the path to the dictionary as well as an indicator of whether the path exists.
-        for key in grid_metadata:        
-            #When USGS grid data was downloaded, grid was saved with the 'key' name. Update the grid_metadata to include the path.
+        for key in grid_metadata:
+            # When USGS grid data was downloaded, grid was saved with the 'key' name. Update the grid_metadata to include the path.
             path = ahps_dir / (key + '.tif')
-            grid_metadata[key]['path'] = path                
-            #Verify that the path exists (in some instances the grid should be available but it isn't) and add as subkey
+            grid_metadata[key]['path'] = path
+            # Verify that the path exists (in some instances the grid should be available but it isn't) and add as subkey
             if path.is_file():
                 grid_metadata[key]['path_exist'] = True
             else:
                 grid_metadata[key]['path_exist'] = False
-        
-        #Convert grid metadata information to a DataFrame
-        df = pd.DataFrame.from_dict(grid_metadata, orient = 'index')
-        #Filter out rows where grids do not exist
+
+        # Convert grid metadata information to a DataFrame
+        df = pd.DataFrame.from_dict(grid_metadata, orient='index')
+        # Filter out rows where grids do not exist
         df = df.query('path_exist == True')
-        #Prior to renaming columns do a check to make sure single site (will add functionality for multi-sites later)
-        if not 'QCFS' in df.columns:
+        # Prior to renaming columns do a check to make sure single site (will add functionality for multi-sites later)
+        if 'QCFS' not in df.columns:
             f.write(f'{code} : Skipping because multisite\n')
             continue
-        #Rename columns to match NWS AHPS data structure, this only applies to single USGS sites, if a multisite the columns are different from QCFS.
-        df.rename(columns = {'QCFS':'flow', 'STAGE':'stage', 'ELEV':'elevation'}, inplace=True)
-        #Many USGS maps have elevations to numerous decimal places. Round to nearest tenth. 
-        #NWS has maps to nearest tenth, for example HARP1 is both USGS and NWS, the USGS maps are to the hundredth of foot and NWS are to tenth.
-        df['elevation'] = round(df['elevation'],1)
-        #Assume flow source is supplied, if it is interpolated, this column will be changed later on.
+        # Rename columns to match NWS AHPS data structure, this only applies to single USGS sites, if a multisite the columns are different from QCFS.
+        df.rename(columns={'QCFS': 'flow', 'STAGE': 'stage', 'ELEV': 'elevation'}, inplace=True)
+        # Many USGS maps have elevations to numerous decimal places. Round to nearest tenth.
+        # NWS has maps to nearest tenth, for example HARP1 is both USGS and NWS, the USGS maps are to the hundredth of foot and NWS are to tenth.
+        df['elevation'] = round(df['elevation'], 1)
+        # Assume flow source is supplied, if it is interpolated, this column will be changed later on.
         df['flow_source'] = 'supplied by USGS'
-        #Accomodate for vdsg1 (upon inspection WRDS API reports thresholds in elevation instead of stage for this site)
+        # Accomodate for vdsg1 (upon inspection WRDS API reports thresholds in elevation instead of stage for this site)
         if code == 'vdsg1':
             df['stage'] = df['elevation']
             f.write(f'{code} : Custom workaround because thresholds are reported as elevations\n')
-        
-        #Define rating curve as empty dataframe, populate if needed.
-        rating_curve = pd.DataFrame()        
-        #If flows are missing from the grid metadata, then interpolate flows using NWS or USGS rating curve
-        if df['flow'].isnull().all():        
-            #get entire rating curve, same source as interpolated threshold flows (USGS Rating Depot first then NRLDB rating curve).
+
+        # Define rating curve as empty dataframe, populate if needed.
+        rating_curve = pd.DataFrame()
+        # If flows are missing from the grid metadata, then interpolate flows using NWS or USGS rating curve
+        if df['flow'].isnull().all():
+            # get entire rating curve, same source as interpolated threshold flows (USGS Rating Depot first then NRLDB rating curve).
             if rating_curve_source == 'NRLDB':
                 site = [code]
             elif rating_curve_source == 'USGS Rating Depot':
                 site = [metadata.get('identifiers').get('usgs_site_code')]
-            
+
             rating_curve = get_rating_curve(rating_curve_url, site)
-    
-            #If rating curve is not present, skip site
+
+            # If rating curve is not present, skip site
             if rating_curve.empty:
                 f.write(f'{code} : Skipping because no rating curve\n')
                 continue
-            #Add elevation fields to rating curve
-            #Add field with vertical coordinate system
+            # Add elevation fields to rating curve
+            # Add field with vertical coordinate system
             vcs = datum_data['vcs']
             if not vcs:
                 vcs = 'Unspecified, Assumed NAVD88'
             rating_curve['vcs'] = vcs
-    
-            #Add field with original datum
+
+            # Add field with original datum
             rating_curve['datum'] = datum
-            
-            #If VCS is NGVD29 add rating curve elevation (in NGVD) as well as the NAVD88 datum
-            if vcs in ['NGVD29', 'NGVD 1929']:        
-                #Add field with raw elevation conversion (datum + stage)
+
+            # If VCS is NGVD29 add rating curve elevation (in NGVD) as well as the NAVD88 datum
+            if vcs in ['NGVD29', 'NGVD 1929']:
+                # Add field with raw elevation conversion (datum + stage)
                 rating_curve['elevation_ngvd29'] = rating_curve['stage'] + datum
-                #Add field with adjusted NAVD88 datum
+                # Add field with adjusted NAVD88 datum
                 rating_curve['datum_navd88'] = datum88
-            #Add field with NAVD88 elevation
+            # Add field with NAVD88 elevation
             rating_curve['elevation_navd88'] = rating_curve['stage'] + datum88
             # sort inundation grids in ascending order based on stage
-            df.sort_values(by = 'elevation', ascending = True, inplace = True)        
-            #interpolate based on stage (don't need elevation because we have stage of floodgrid)
-            df['flow'] = np.interp(df['elevation'], rating_curve['elevation_navd88'], rating_curve['flow'], left = np.nan, right = np.nan)
-            #Overwrite flow source to reflect interpolation from rc
+            df.sort_values(by='elevation', ascending=True, inplace=True)
+            # interpolate based on stage (don't need elevation because we have stage of floodgrid)
+            df['flow'] = np.interp(
+                df['elevation'],
+                rating_curve['elevation_navd88'],
+                rating_curve['flow'],
+                left=np.nan,
+                right=np.nan,
+            )
+            # Overwrite flow source to reflect interpolation from rc
             df['flow_source'] = f'interpolated from {rating_curve_source} rating curve'
-            
-        #Select the appropriate threshold grid for evaluation. Using the supplied threshold stages and the calculated map stages. 
-        grids,grid_flows = select_grids(df, stages, datum88, 1.1)
-    
-        #Obtain NWM segments that are on ms to apply flows
+
+        # Select the appropriate threshold grid for evaluation. Using the supplied threshold stages and the calculated map stages.
+        grids, grid_flows = select_grids(df, stages, datum88, 1.1)
+
+        # Obtain NWM segments that are on ms to apply flows
         segments = get_nwm_segs(metadata)
         site_ms_segs = set(segments).intersection(ms_segs)
         segments = list(site_ms_segs)
-        #Preprocess grids and export to file and create flow file.
+        # Preprocess grids and export to file and create flow file.
         try:
-            #for each threshold
+            # for each threshold
             for i in threshold_categories:
-                #Obtain the flow and grid associated with threshold as well as extent grid which serves as the domain.
+                # Obtain the flow and grid associated with threshold as well as extent grid which serves as the domain.
                 flow = grid_flows[i]
                 grid = grids[i]
                 extent = grids['extent']
-                #Make sure that flow and flow grid are valid
-                if not grid in ['No Map', 'No Threshold', 'No Flow']:
-                    #Define output directory (to be created later)
-                    outputdir = destination / huc / code / i                    
-                    
-                    #Create Binary Grids, first create domain of analysis, then create binary grid
-                    
-                    #Domain extent is largest floodmap in the static library WITH holes filled
+                # Make sure that flow and flow grid are valid
+                if grid not in ['No Map', 'No Threshold', 'No Flow']:
+                    # Define output directory (to be created later)
+                    outputdir = destination / huc / code / i
+
+                    # Create Binary Grids, first create domain of analysis, then create binary grid
+
+                    # Domain extent is largest floodmap in the static library WITH holes filled
                     filled_domain_raster = outputdir.parent / f'{code}_filled_orig_domain.tif'
-                    #Create a domain raster if it does not exist.
+                    # Create a domain raster if it does not exist.
                     if not filled_domain_raster.exists():
-                        #Open extent data as rasterio object
+                        # Open extent data as rasterio object
                         domain = rasterio.open(extent)
                         domain_profile = domain.profile
-                        #Domain should have donut holes removed
-                        process_extent(domain, domain_profile, output_raster = filled_domain_raster)
-    
-                    #Open domain raster as rasterio object
+                        # Domain should have donut holes removed
+                        process_extent(domain, domain_profile, output_raster=filled_domain_raster)
+
+                    # Open domain raster as rasterio object
                     filled_domain = rasterio.open(filled_domain_raster)
                     filled_domain_profile = filled_domain.profile
-    
-                    #Open benchmark data as a rasterio object.
+
+                    # Open benchmark data as a rasterio object.
                     benchmark = rasterio.open(grid)
-                    benchmark_profile = benchmark.profile                
-    
-                    #Create the binary benchmark raster                        
-                    boolean_benchmark, boolean_profile = process_grid(benchmark, benchmark_profile, filled_domain, filled_domain_profile, reference_raster)    
-                                    
-                    #Output binary benchmark grid and flow file to destination
-                    outputdir.mkdir(parents = True, exist_ok = True)
+                    benchmark_profile = benchmark.profile
+
+                    # Create the binary benchmark raster
+                    boolean_benchmark, boolean_profile = process_grid(
+                        benchmark, benchmark_profile, filled_domain, filled_domain_profile, reference_raster
+                    )
+
+                    # Output binary benchmark grid and flow file to destination
+                    outputdir.mkdir(parents=True, exist_ok=True)
                     output_raster = outputdir / (f'ahps_{code}_huc_{huc}_extent_{i}.tif')
                     with rasterio.Env():
                         with rasterio.open(output_raster, 'w', **boolean_profile) as dst:
-                            dst.write(boolean_benchmark,1)
-                    
-                    #Close datasets
+                            dst.write(boolean_benchmark, 1)
+
+                    # Close datasets
                     domain.close()
                     filled_domain.close()
                     benchmark.close()
-                    
-                    #Create the guts of the flow file.
-                    flow_info = flow_data(segments,flow) 
-                    #Write out the flow file to csv
+
+                    # Create the guts of the flow file.
+                    flow_info = flow_data(segments, flow)
+                    # Write out the flow file to csv
                     output_flow_file = outputdir / (f'ahps_{code}_huc_{huc}_flows_{i}.csv')
-                    flow_info.to_csv(output_flow_file, index = False)
-                    
+                    flow_info.to_csv(output_flow_file, index=False)
+
         except Exception as e:
             f.write(f'{code} : Error preprocessing benchmark\n{repr(e)}\n')
             f.write(traceback.format_exc())
             f.write('\n')
             print(traceback.format_exc())
-        #Wrapup for ahps sites that were processed.
+        # Wrapup for ahps sites that were processed.
         ahps_directory = destination / huc / code
         if ahps_directory.exists():
-            #Delete original filled domain raster (it is an intermediate file to create benchmark data)
+            # Delete original filled domain raster (it is an intermediate file to create benchmark data)
             orig_domain_grid = ahps_directory / f'{code}_filled_orig_domain.tif'
-            orig_domain_grid.unlink()            
-            #Create domain shapefile from any benchmark grid for site (each benchmark has domain footprint, value = 0).
+            orig_domain_grid.unlink()
+            # Create domain shapefile from any benchmark grid for site (each benchmark has domain footprint, value = 0).
             filled_extent = list(ahps_directory.rglob('*_extent_*.tif'))[0]
-            domain_gpd = raster_to_feature(grid = filled_extent, profile_override = False, footprint_only = True)           
+            domain_gpd = raster_to_feature(grid=filled_extent, profile_override=False, footprint_only=True)
             domain_gpd['nws_lid'] = code
-            domain_gpd.to_file(ahps_directory / f'{code}_domain.shp')   
-            #Populate attribute information for site
-            grids_attributes = pd.DataFrame(data=grids.items(), columns = ['magnitude','path'])
-            flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude','grid_flow_cfs'])
-            threshold_attributes = pd.DataFrame(data=stages.items(), columns = ['magnitude','magnitude_stage'])       
-            #merge dataframes
-            attributes = grids_attributes.merge(flows_attributes, on = 'magnitude')
-            attributes = attributes.merge(threshold_attributes, on = 'magnitude')
-            attributes = attributes.merge(df[['path','stage','elevation', 'flow_source']], on = 'path')        
-            #Strip out sensitive paths and convert magnitude stage to elevation
-            attributes['path'] = attributes['path'].apply(lambda x :Path(x).name)
-            attributes['magnitude_elev_navd88']=(datum88 + attributes['magnitude_stage']).astype(float).round(1)
-            #Add general site information
+            domain_gpd.to_file(ahps_directory / f'{code}_domain.shp')
+            # Populate attribute information for site
+            grids_attributes = pd.DataFrame(data=grids.items(), columns=['magnitude', 'path'])
+            flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude', 'grid_flow_cfs'])
+            threshold_attributes = pd.DataFrame(data=stages.items(), columns=['magnitude', 'magnitude_stage'])
+            # merge dataframes
+            attributes = grids_attributes.merge(flows_attributes, on='magnitude')
+            attributes = attributes.merge(threshold_attributes, on='magnitude')
+            attributes = attributes.merge(df[['path', 'stage', 'elevation', 'flow_source']], on='path')
+            # Strip out sensitive paths and convert magnitude stage to elevation
+            attributes['path'] = attributes['path'].apply(lambda x: Path(x).name)
+            attributes['magnitude_elev_navd88'] = (
+                (datum88 + attributes['magnitude_stage']).astype(float).round(1)
+            )
+            # Add general site information
             attributes['nws_lid'] = code
             attributes['wfo'] = metadata['nws_data']['wfo']
             attributes['rfc'] = metadata['nws_data']['rfc']
             attributes['state'] = metadata['nws_data']['state']
             attributes['huc'] = huc
-            #Rename and Reorder columns
-            attributes.rename(columns = {'path':'grid_name', 'flow_source':'grid_flow_source','stage':'grid_stage','elevation':'grid_elev_navd88'}, inplace = True)        
-            attributes = attributes[['nws_lid','wfo','rfc','state','huc','magnitude','magnitude_stage','magnitude_elev_navd88','grid_name','grid_stage','grid_elev_navd88', 'grid_flow_cfs','grid_flow_source']]        
-            #Save attributes to csv
-            attributes.to_csv(ahps_directory / f'{code}_attributes.csv', index = False) 
-                
-            #if rating_curve generated, write the rating curve to a file
+            # Rename and Reorder columns
+            attributes.rename(
+                columns={
+                    'path': 'grid_name',
+                    'flow_source': 'grid_flow_source',
+                    'stage': 'grid_stage',
+                    'elevation': 'grid_elev_navd88',
+                },
+                inplace=True,
+            )
+            attributes = attributes[
+                [
+                    'nws_lid',
+                    'wfo',
+                    'rfc',
+                    'state',
+                    'huc',
+                    'magnitude',
+                    'magnitude_stage',
+                    'magnitude_elev_navd88',
+                    'grid_name',
+                    'grid_stage',
+                    'grid_elev_navd88',
+                    'grid_flow_cfs',
+                    'grid_flow_source',
+                ]
+            ]
+            # Save attributes to csv
+            attributes.to_csv(ahps_directory / f'{code}_attributes.csv', index=False)
+
+            # if rating_curve generated, write the rating curve to a file
             if not rating_curve.empty:
                 rating_curve_output = ahps_directory / (f'{code}_rating_curve.csv')
                 rating_curve['lat'] = datum_data['lat']
                 rating_curve['lon'] = datum_data['lon']
-                rating_curve.to_csv(rating_curve_output, index = False)
+                rating_curve.to_csv(rating_curve_output, index=False)
                 f.write(f'{code} : Rating curve needed to interpolate flow\n')
-            
-            #Write the interpolated flows to file
+
+            # Write the interpolated flows to file
             df_output = ahps_directory / (f'{code}_flows.csv')
-            df.to_csv(df_output, index = False)
-            
-        else: 
+            df.to_csv(df_output, index=False)
+
+        else:
             f.write(f'{code} : Unable to evaluate site, missing all flows\n')
-    
+
     f.close()
-    
-    #Combine all attribute files
+
+    # Combine all attribute files
     attribute_files = list(destination.rglob('*_attributes.csv'))
     all_attributes = pd.DataFrame()
     for i in attribute_files:
-        attribute_df = pd.read_csv(i, dtype={'huc':str})
+        attribute_df = pd.read_csv(i, dtype={'huc': str})
         all_attributes = pd.concat([all_attributes, attribute_df])
     if not all_attributes.empty:
-        all_attributes.to_csv(destination / 'attributes.csv', index = False)
-            
-    return 
+        all_attributes.to_csv(destination / 'attributes.csv', index=False)
+
+    return
+
 
 if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
-    parser.add_argument('-s', '--source_dir', help = 'Workspace where all source data is located.', required = True)
-    parser.add_argument('-d', '--destination',  help = 'Directory where outputs are to be stored', required = True)
-    parser.add_argument('-r', '--reference_raster', help = 'reference raster used for benchmark raster creation', required = True)
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description='Create preprocessed USGS benchmark datasets at AHPS locations.'
+    )
+    parser.add_argument(
+        '-s', '--source_dir', help='Workspace where all source data is located.', required=True
+    )
+    parser.add_argument('-d', '--destination', help='Directory where outputs are to be stored', required=True)
+    parser.add_argument(
+        '-r', '--reference_raster', help='reference raster used for benchmark raster creation', required=True
+    )
     args = vars(parser.parse_args())
-    
 
-    #Run get_env_paths and static_flow_lids
+    # Run get_env_paths and static_flow_lids
     API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER, USGS_METADATA_URL = get_env_paths()
-    preprocess_usgs(**args)
\ No newline at end of file
+    preprocess_usgs(**args)
diff --git a/data/usgs/preprocess_download_usgs_grids.py b/data/usgs/preprocess_download_usgs_grids.py
index 8d85a6e1e..b114f8dd1 100644
--- a/data/usgs/preprocess_download_usgs_grids.py
+++ b/data/usgs/preprocess_download_usgs_grids.py
@@ -1,22 +1,25 @@
 #!/usr/bin/env python3
 import argparse
 import os
-import pandas as pd
-import requests
 import urllib
-import urllib.request 
-
+import urllib.request
 from collections import defaultdict
-from dotenv import load_dotenv
 from pathlib import Path
 
+import pandas as pd
+import requests
+from dotenv import load_dotenv
+
+
 load_dotenv()
 USGS_DOWNLOAD_URL = os.getenv("USGS_DOWNLOAD_URL")
-USGS_METADATA_URL = os.getenv("USGS_METADATA_URL")   
+USGS_METADATA_URL = os.getenv("USGS_METADATA_URL")
 EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+
+
+###############################################################################
+# Get all usgs grids available for download. This step is required because the grid metadata API returns gridID as an integer and truncates leading zeros found in grid names.
 ###############################################################################
-#Get all usgs grids available for download. This step is required because the grid metadata API returns gridID as an integer and truncates leading zeros found in grid names.
-###############################################################################            
 def get_all_usgs_gridnames():
     '''
     Retrieve all the available grids for download from USGS. This is necessary as the grid metadata available from USGS API doesn't preserve leading zeros.
@@ -26,37 +29,39 @@ def get_all_usgs_gridnames():
     grid_lookup : collections.defaultdict
         Dictionary with shortname as the key and a list of gridnames associated with a given shortname as values.
     '''
-    
-    #Grid names are split between 4 websites
+
+    # Grid names are split between 4 websites
     sites = ['grids_1', 'grids_2', 'grids_3', 'grids_4']
-    #Append all grid names to this variable
+    # Append all grid names to this variable
     grid_names = []
-    #loop through each site and append the grid name to a list.
+    # loop through each site and append the grid name to a list.
     for i in sites:
-        #Get gridnames
+        # Get gridnames
         url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/{i}/MapServer?f=pjson'
         response = requests.get(url)
         site_json = response.json()
         info = site_json['layers']
-        #Loop through all grid info and extract the grid name.
+        # Loop through all grid info and extract the grid name.
         for i in info:
             grid_name = i['name']
             grid_names.append(grid_name)
-    #Create dictionary with key of shortname and values being list of grids available.    
+    # Create dictionary with key of shortname and values being list of grids available.
     grid_lookup = defaultdict(list)
     for i in grid_names:
-        #define key (shortname) and value (gridname)
+        # define key (shortname) and value (gridname)
         key = i.split('_')[0]
-        value = i   
+        value = i
         grid_lookup[key].append(value)
     return grid_lookup
+
+
 ###############################################################################
-#Get USGS Site metadata
+# Get USGS Site metadata
 ###############################################################################
 def usgs_site_metadata(code):
     '''
     Retrieves site metadata from USGS API and saves output as dictionary. Information used includes shortname and site number.
-    
+
     Parameters
     ----------
     code : STR
@@ -73,15 +78,17 @@ def usgs_site_metadata(code):
     code = code.lower()
     # Get site metadata from USGS API using ahps code
     site_url = f'{USGS_METADATA_URL}/server/rest/services/FIMMapper/sites/MapServer/0/query?where=AHPS_ID+%3D+%27{code}%27&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=false&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=&parameterValues=&rangeValues=&quantizationParameters=&f=pjson'
-    #Get data from API
+    # Get data from API
     response = requests.get(site_url)
-    #If response is valid, then get metadata and save to dictionary
+    # If response is valid, then get metadata and save to dictionary
     if response.ok:
         response_json = response.json()
         site_metadata = response_json['features'][0]['attributes']
     return site_metadata
+
+
 ########################################################################
-#Get USGS Benchmark Grids
+# Get USGS Benchmark Grids
 ########################################################################
 def obtain_usgs_data(workspace):
     '''
@@ -97,67 +104,65 @@ def obtain_usgs_data(workspace):
     None.
 
     '''
-    
-    
-    #Define workspace where output data is downloaded to
+
+    # Define workspace where output data is downloaded to
     workspace = Path(workspace)
-    #Get all names of grids available for download from USGS website.
+    # Get all names of grids available for download from USGS website.
     grid_lookup = get_all_usgs_gridnames()
-    #List of target ahps codes. In "ahps_dictionary.py" we defined a dictionary (ahps_lookup) that contains all ahps codes and their sources.
+    # List of target ahps codes. In "ahps_dictionary.py" we defined a dictionary (ahps_lookup) that contains all ahps codes and their sources.
     target_ahps_codes = pd.read_csv(EVALUATED_SITES_CSV)
     target_ahps_codes = target_ahps_codes.query('Source in ["Both","USGS"]')['Total_List'].to_list()
-    #Loop through all codes in the target_ahps_codes list.
+    # Loop through all codes in the target_ahps_codes list.
     all_messages = []
     for code in target_ahps_codes:
-        #Get metadata information related to ahps code from USGS API.
+        # Get metadata information related to ahps code from USGS API.
         code_metadata = usgs_site_metadata(code)
-        #From code_metadata get the shortname and site_no associated with the code.
+        # From code_metadata get the shortname and site_no associated with the code.
         shortname = code_metadata['SHORT_NAME']
-        site_no = code_metadata['SITE_NO']     
-        #Define the output location for all grids and create if it doesn't exist.
+        # site_no = code_metadata['SITE_NO']
+        # Define the output location for all grids and create if it doesn't exist.
         dest_dir = workspace / code.lower() / 'depth_grids'
-        dest_dir.mkdir(parents = True, exist_ok = True)    
-        #Get list of all available grids for download using the grid_lookup dictionary
-        gridnames = grid_lookup[shortname]    
-        #Loop through all available grids for download, download them, and save to defined location.
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        # Get list of all available grids for download using the grid_lookup dictionary
+        gridnames = grid_lookup[shortname]
+        # Loop through all available grids for download, download them, and save to defined location.
         for gridname in gridnames:
             print(f'working on {gridname}')
             gridid = gridname.split('_')[1]
-            #Define a filled gridID that has leading zeros out to 4 digits.
-            filled_gridid = gridid.zfill(4)       
-            #Download gridded data from the USGS s3 website. The files will be copied specified directory and the GRIDID will have 4 digits with leading zeros.
-            base_url = f'{USGS_DOWNLOAD_URL}/FIM/tmp1/fimgrids2iwrss/'       
-            #Each grid dataset has these file extensions. Download each file
+            # Define a filled gridID that has leading zeros out to 4 digits.
+            filled_gridid = gridid.zfill(4)
+            # Download gridded data from the USGS s3 website. The files will be copied specified directory and the GRIDID will have 4 digits with leading zeros.
+            base_url = f'{USGS_DOWNLOAD_URL}/FIM/tmp1/fimgrids2iwrss/'
+            # Each grid dataset has these file extensions. Download each file
             extensions = ['.tif', '.tfw', '.tif.aux.xml', '.tif.ovr', '.tif.xml']
-            #Loop through each extension type and download.
+            # Loop through each extension type and download.
             for gridext in extensions:
-                #Define the url associated with each grid
-                url = base_url + gridname + gridext       
-                #Define the output file path of the grid. The grid filename uses the filled gridID. This resolves issues down the road of USGS grid metadata information storing the gridid as a number and truncating leading zeros from the gridname.
+                # Define the url associated with each grid
+                url = base_url + gridname + gridext
+                # Define the output file path of the grid. The grid filename uses the filled gridID. This resolves issues down the road of USGS grid metadata information storing the gridid as a number and truncating leading zeros from the gridname.
                 saved_grid_path = dest_dir / (f'{shortname}_{filled_gridid}{gridext}')
-                #Check to see if file has already been downloaded
+                # Check to see if file has already been downloaded
                 if not saved_grid_path.is_file():
-                    #If file hasn't been downloaded, download it. If there was an error downloading, make note.
+                    # If file hasn't been downloaded, download it. If there was an error downloading, make note.
                     try:
                         urllib.request.urlretrieve(url, saved_grid_path)
                         message = f'{gridname} downloaded'
                         all_messages.append(message)
-                    except: 
-                        message = f'{gridname} error downloading'
+                    except Exception as e:
+                        message = f'{gridname} error downloading. Exception: \n {repr(e)} \n'
                         all_messages.append(message)
-                #If file exists make note of it.
+                # If file exists make note of it.
                 else:
                     message = f'skipping {gridname}, exists on file'
                     all_messages.append(message)
     return
 
+
 if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Download Grid data associated with USGS FIM studies.')
-    parser.add_argument('-w', '--workspace', help = 'Workspace where all outputs will be saved.', required = True)
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Download Grid data associated with USGS FIM studies.')
+    parser.add_argument('-w', '--workspace', help='Workspace where all outputs will be saved.', required=True)
     args = vars(parser.parse_args())
-    
-    #Download datasets
-    obtain_usgs_data(**args)
-
 
+    # Download datasets
+    obtain_usgs_data(**args)
diff --git a/data/wbd/preprocess_wbd.py b/data/wbd/preprocess_wbd.py
index 3bf1b1da6..a61a8010c 100755
--- a/data/wbd/preprocess_wbd.py
+++ b/data/wbd/preprocess_wbd.py
@@ -1,13 +1,15 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
 from pathlib import Path
-import argparse
+
 import geopandas as gpd
+
 from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 
 
-def clip_wbd_to_dem_domain(dem:str, wbd_in:str, wbd_out:str, huc_level:int):
+def clip_wbd_to_dem_domain(dem: str, wbd_in: str, wbd_out: str, huc_level: int):
     """
     Clips Watershed Boundary Dataset (WBD) to DEM domain
 
@@ -35,10 +37,9 @@ def clip_wbd_to_dem_domain(dem:str, wbd_in:str, wbd_out:str, huc_level:int):
 
         # Write output file
         wbd.to_file(wbd_out, layer=layer, crs=DEFAULT_FIM_PROJECTION_CRS, driver='GPKG')
-    
+
 
 if __name__ == '__main__':
-    
     parser = argparse.ArgumentParser(description='Clip WBD to DEM domain')
     parser.add_argument('-d', '--dem', help='Path to DEM', type=str, required=True)
     parser.add_argument('-w', '--wbd-in', help='Input WBD filename', type=str, required=True)
@@ -51,4 +52,3 @@ def clip_wbd_to_dem_domain(dem:str, wbd_in:str, wbd_out:str, huc_level:int):
 
     # Example:
     # preprocess_wbd.py -d /data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg -w /data/inputs/wbd/WBD_National_EPSG_5070.gpkg -o /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -l 8
-
diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py
index ab65c191f..63ac4c666 100644
--- a/data/write_parquet_from_calib_pts.py
+++ b/data/write_parquet_from_calib_pts.py
@@ -1,15 +1,15 @@
 #!/usr/bin/env python3
 
-import os
-import time
-import datetime as dt
-import geopandas as gpd
 import argparse
+import datetime as dt
 import logging
+import os
+import time
+from multiprocessing import Pool
 
-from pathlib import Path
+import geopandas as gpd
 from dotenv import load_dotenv
-from multiprocessing import Pool
+
 
 ######################################################################################################
 #                                                                                                    #
@@ -26,13 +26,15 @@
 
 load_dotenv('/foss_fim/src/bash_variables.env')
 DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
-input_WBD_gdb              = os.getenv('input_WBD_gdb')
-input_calib_points_dir     = os.getenv('input_calib_points_dir')
+input_WBD_gdb = os.getenv('input_WBD_gdb')
+input_calib_points_dir = os.getenv('input_calib_points_dir')
+
 
 def __setup_logger(output_dir):
     # Set logging to file and stderr
-    
-    # The log file will include the date, so be mindful the logs when running this script multiple times on the same day
+
+    # The log file will include the date, so be mindful the logs when running this script
+    # multiple times on the same day
     curr_date = dt.datetime.now().strftime("%m_%d_%Y")
 
     log_file_name = f"write_parquet_from_calib_pts_{curr_date}.log"
@@ -50,8 +52,8 @@ def __setup_logger(output_dir):
 
     # Print start time
     dt_string = dt.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    logging.info(f'==========================================================================')
-    logging.info(f"\n write_parquet_from_calib_pts.py")
+    logging.info('==========================================================================')
+    logging.info("\n write_parquet_from_calib_pts.py")
     logging.info(f"\n \t Started: {dt_string} \n")
 
 
@@ -65,12 +67,29 @@ def load_WBD_gpkg_into_GDF(WBD_National_gpkg_file):
     Outputs
     - huc_polygons_df:                 GeoDataFrame
     '''
-    
-    huc_polygons_df = gpd.read_file(WBD_National_gpkg_file, layer='WBDHU8',
-                                    ignore_fields=['tnmid', 'metasourceid', 'sourcedatadesc', 'sourceoriginator',
-                                                    'sourcefeatureid', 'loaddate', 'referencegnis_ids', 'areaacres',
-                                                    'areasqkm', 'states', 'name', 'globalid', 'shape_Length',
-                                                    'shape_Area', 'fimid', 'fossid'])
+
+    huc_polygons_df = gpd.read_file(
+        WBD_National_gpkg_file,
+        layer='WBDHU8',
+        ignore_fields=[
+            'tnmid',
+            'metasourceid',
+            'sourcedatadesc',
+            'sourceoriginator',
+            'sourcefeatureid',
+            'loaddate',
+            'referencegnis_ids',
+            'areaacres',
+            'areasqkm',
+            'states',
+            'name',
+            'globalid',
+            'shape_Length',
+            'shape_Area',
+            'fimid',
+            'fossid',
+        ],
+    )
 
     return huc_polygons_df
 
@@ -83,12 +102,23 @@ def load_fim_obs_points_into_GDF(fim_obs_points_data_file):
     - fim_obs_points_data_file:        .gpkg file containing calibration points
 
     Outputs
-    - fim_obs_point_df:                GeoDataFrame 
+    - fim_obs_point_df:                GeoDataFrame
     '''
-    
-    fim_obs_point_df = gpd.read_file(fim_obs_points_data_file, layer='usgs_nws_benchmark_points',
-                                     ignore_fields=['Join_Count', 'TARGET_FID', 'DN', 'ORIG_FID',
-                                                    'ID', 'AreaSqKM', 'Shape_Leng', 'Shape_Area'] )
+
+    fim_obs_point_df = gpd.read_file(
+        fim_obs_points_data_file,
+        layer='usgs_nws_benchmark_points',
+        ignore_fields=[
+            'Join_Count',
+            'TARGET_FID',
+            'DN',
+            'ORIG_FID',
+            'ID',
+            'AreaSqKM',
+            'Shape_Leng',
+            'Shape_Area',
+        ],
+    )
 
     return fim_obs_point_df
 
@@ -112,42 +142,42 @@ def create_single_huc_gdf_and_write_parquet_file(args):
     Outputs
     - none:                 While this function doesn't 'return' any output, it does write a new file.
     '''
-    
+
     # We have to explicitly unpack the args from pool.map()
-    huc        = args[0]
+    huc = args[0]
     output_dir = args[1]
-    wbd_GDF    = args[2]
-    pnt_GDF    = args[3]
-    
+    wbd_GDF = args[2]
+    pnt_GDF = args[3]
+
     # Get the row within the wbd_GDF containing the watershed boundary polygon
     one_huc = wbd_GDF.loc[wbd_GDF['HUC8'] == huc]
 
     # Create a new GeoDataFrame with an inner spatial join of all points with a HUC8 boundary.
     huc_with_points_gdf = pnt_GDF.sjoin(one_huc, how='inner', predicate='within')
-    
+
     # If there are no calibration points within the HUC boundary, skip current HUC
-    if len(huc_with_points_gdf) == 0: 
+    if len(huc_with_points_gdf) == 0:
         logging.info(f'HUC # {huc} does not contain any calibration points, skipping...')
-        time.sleep(.1)
+        time.sleep(0.1)
         return
-    
+
     # Drop index_right column created by join
     huc_with_points_gdf.drop(['index_right'], axis=1, inplace=True)
-    
+
     # Set the crs projection
     huc_gdf = huc_with_points_gdf.set_crs(DEFAULT_FIM_PROJECTION_CRS, allow_override=True)
 
     # Set filepath and write file
     parquet_filepath = os.path.join(output_dir, f'{huc}.parquet')
     huc_gdf.to_parquet(parquet_filepath, index=False)
-    
-    logging.info(f'HUC # {huc} calibration points written to file: \n' \
-        f' \t {output_dir}/{huc}.parquet')
+
+    logging.info(f'HUC # {huc} calibration points written to file: \n' f' \t {output_dir}/{huc}.parquet')
 
 
 def create_parquet_directory(output_dir):
     '''
-    The function creates an output directory to write the .parquet files if the filepath provided as an argument doesn't exist.
+    The function creates an output directory to write the .parquet files,
+    if the filepath provided as an argument doesn't exist.
 
     Inputs
     - output_dir:               Filepath to create a directory.
@@ -155,20 +185,22 @@ def create_parquet_directory(output_dir):
     Outputs
     - none:                     While no return value, a new directory is created if it does not exist.
     '''
-    
-    if os.path.isdir(output_dir) == False:
-        Path(output_dir).mkdir(parents=True, exist_ok=True)
-        logging.info(f"Created directory: {output_dir}, .parquet files will be located there.")
-    elif os.path.isdir(output_dir) == True:
-        logging.info(f"Output Direcrtory: {output_dir} exists, .parquet files will be located there.")
-
-
-def create_parquet_files(points_data_file_name,
-                            number_of_jobs,
-                            output_dir=input_calib_points_dir,
-                            wbd_layer=input_WBD_gdb,
-                            huc_list=None,
-                            all_hucs=False):
+
+    if os.path.isdir(output_dir) is False:
+        os.mkdir(output_dir, exist_ok=True)
+        logging.info(f"Created directory: {output_dir}, .parquet files will be written there.")
+    elif os.path.isdir(output_dir) is True:
+        logging.info(f"Output Direcrtory: {output_dir} exists, .parquet files will be written there.")
+
+
+def create_parquet_files(
+    points_data_file_name,
+    number_of_jobs,
+    output_dir=input_calib_points_dir,
+    wbd_layer=input_WBD_gdb,
+    huc_list=None,
+    all_hucs=False,
+):
     '''
     The function is the main driver of the program to iterate and parallelize writing HUC8.parquet files.
 
@@ -186,71 +218,79 @@ def create_parquet_files(points_data_file_name,
     - Create output directory.
     - Decipher what HUCs to use for the hucs_to_parquet_list based on provided arguments.
     - Build arguments via procs_list to each parallel call to create_single_huc_gdf_and_write_parquet_file.
-    - Parallelize the processing of GeoDataFrame creation and .parquet writing of the hucs_to_parquet_list 
+    - Parallelize the processing of GeoDataFrame creation and .parquet writing of the hucs_to_parquet_list
         using multiprocessing.Pool.
 
     Outputs
-    - parquet files:       
+    - parquet files:
     - log file
     '''
-    
+
     # Validation
     total_cpus_available = os.cpu_count()
     if number_of_jobs > total_cpus_available:
-        logging.info(f'Provided: -j {number_of_jobs}, which is greater than than amount of available cpus -1: ' \
-                        f'{total_cpus_available -1} will be used instead.')
-        number_of_jobs = total_cpus_available -1
-    
+        logging.info(
+            f'Provided: -j {number_of_jobs}, which is greater than than amount of available cpus -1: '
+            f'{total_cpus_available -1} will be used instead.'
+        )
+        number_of_jobs = total_cpus_available - 1
+
     # Set start_time and setup logger
     start_time = dt.datetime.now()
     __setup_logger(output_dir)
-    
+
     create_parquet_directory(output_dir)
-    
-    logging.info(f'Loading .gpkg files into GeoDataFrames....')
-    huc_polygons_df  = load_WBD_gpkg_into_GDF(wbd_layer)
+
+    logging.info('Loading .gpkg files into GeoDataFrames....')
+    huc_polygons_df = load_WBD_gpkg_into_GDF(wbd_layer)
     fim_obs_point_df = load_fim_obs_points_into_GDF(points_data_file_name)
-    
+
     # Verify same projection
     if (huc_polygons_df.crs == fim_obs_point_df.crs) is not True:
-        raise ValueError(f'Provided: -p {points_data_file_name} crs & -wbd {wbd_layer} crs do not match, ' \
-                        f'please make adjustments and re-run.')
-    
+        raise ValueError(
+            f'Provided: -p {points_data_file_name} crs & -wbd {wbd_layer} crs do not match, '
+            'please make adjustments and re-run.'
+        )
+
     # Print timing
     load_gdf_end_time = dt.datetime.now()
     load_duration = load_gdf_end_time - start_time
     logging.info("Finished loading input .gpkg files into GeoDataFrames.")
+    logging.info(f"\t TIME: {str(load_duration).split('.')[0]} ")
+
+    print("Finished loading input .gpkg files into GeoDataFrames.")
     print(f"\t TIME: {str(load_duration).split('.')[0]} ")
-    
+
     # Only use provided HUCs
     if huc_list is not None:
         hucs_to_parquet_list = list(huc_list.split(','))
-        
-    # Use all HUCS in WBD file 
+
+    # Use all HUCS in WBD file
     elif all_hucs:
         # Define the list of HUCS based on all HUCS in WBD File
-        all_hucs_in_WBD = huc_polygons_df.iloc[:,0]
+        all_hucs_in_WBD = huc_polygons_df.iloc[:, 0]
         hucs_to_parquet_list = list(all_hucs_in_WBD)
-        
-    # Default HUC list comes from calibration point .parquet files that are pre-existing in current /inputs location
+
+    # Default HUC list comes from calibration point .parquet files that are pre-existing in current
+    # /inputs location
     else:
         # Get a list of all files in output_dir
         current_hucs_in_output_dir = os.listdir(output_dir)
-        # Slice off the .parquet file format to get just the HUC number of all files with the .parquet extension
+        # Slice off the .parquet from filename to get the HUC number of all files with the .parquet extension
         hucs_to_parquet_list = [i[:-8] for i in current_hucs_in_output_dir if i.endswith('.parquet')]
-    
+
     logging.info(f"Submitting {len(hucs_to_parquet_list)} HUCS.")
-        
+
     # Build arguments (procs_list) for each process (create_single_huc_gdf_and_write_parquet_file)
     procs_list = []
     for huc in hucs_to_parquet_list:
         procs_list.append([huc, output_dir, huc_polygons_df, fim_obs_point_df])
-    
+
     # Parallelize each huc in hucs_to_parquet_list
-    logging.info(f'Parallelizing HUC level GeoDataFrame creation and .parquet file writes')
+    logging.info('Parallelizing HUC level GeoDataFrame creation and .parquet file writes')
     with Pool(processes=number_of_jobs) as pool:
-                pool.map(create_single_huc_gdf_and_write_parquet_file, procs_list)
-    
+        pool.map(create_single_huc_gdf_and_write_parquet_file, procs_list)
+
     # Get time metrics
     end_time = dt.datetime.now()
     dt_string = dt.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
@@ -258,50 +298,94 @@ def create_parquet_files(points_data_file_name,
 
     # Calculate duration
     time_duration = end_time - start_time
-    logging.info(f'==========================================================================')
-    logging.info(f"\t Completed writing all .parquet files \n" \
-        f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}")
-    logging.info(f'==========================================================================')
+    logging.info('==========================================================================')
+    logging.info(
+        f"\t Completed writing all .parquet files \n"
+        f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}"
+    )
+    logging.info('==========================================================================')
+
+    print("\t Completed writing all .parquet files \n")
+    print(f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}")
 
 
 if __name__ == '__main__':
     '''
-    Sample Usage:   python3 /foss_fim/data/write_parquet_from_calib_pts.py                                       \
-                        -p /data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg  \
-                        -j 6                                                                                     \
-                        -o /data/inputs/rating_curve/water_edge_database/calibration_points                      \
-                        -wbd /data/inputs/wbd/WBD_National.gpkg                                                  \
-                        -u "12040103,04100003"
-                        
-                    python3 /foss_fim/data/write_parquet_from_calib_pts.py                                       \
-                        -p /data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg  \
-                        -a
+    Sample Usage:
+        python3 /foss_fim/data/write_parquet_from_calib_pts.py
+            -p /data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
+            -j 6
+            -o /data/inputs/rating_curve/water_edge_database/calibration_points
+            -wbd /data/inputs/wbd/WBD_National.gpkg
+            -u "12040103,04100003"
+
+        python3 /foss_fim/data/write_parquet_from_calib_pts.py
+            -p /data/inputs/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
+            -a
     '''
 
     parser = argparse.ArgumentParser(description='Create a parquet file/files with calibration points.')
-    
-    parser.add_argument('-p','--points_data_file_name', help='REQUIRED: Complete relative filepath of a .gpkg file with fim calibration points.', 
-        type=str, required=True)
-    
-    parser.add_argument('-o', '--output_dir', help=f'OPTIONAL: path to send .parquet file/files. Default location specifed as the "input_calib_points_dir" ' \
-        'variable in src/bash_variables.env', type=str, required=False, default=input_calib_points_dir)
-        
-    parser.add_argument('-j','--number_of_jobs', help='OPTIONAL: number of cores/processes (default=4). This is a memory intensive '                         \
-        'script, and the multiprocessing will crash if too many CPUs are used. It is recommended to provide half the amount'                                 \
-        'of available CPUs.', type=int, required=False, default=4)
-
-    parser.add_argument('-wbd','--wbd_layer', help='OPTIONAL: A directory of where a .gpkg file exists, containing HUC boundary polygons. '                  \
-        'Default filepath is specified as the "input_WBD_gdb" variable in src/bash_variables.env', type=str, required=False, default=input_WBD_gdb)
-    
-    parser.add_argument('-u','--huc_list', help='OPTIONAL: HUC list - String with comma seperated HUC numbers. *DO NOT include spaces*.'                     \
-        'Provide certain HUCs if points were added/updated only within a few known HUCs.', type=str, required=False, default=None)
-    
-    parser.add_argument('-a', '--all_hucs', help='OPTIONAL: Provide the -a flag if new calibration points were added to many HUCs which currently'           \
-        'do not have .parquet files.  All HUC polygons in the file provided to -wbd argument will be checked for calibration points contained in '           \
-        '<points_data_file_name>. If not provided, either the HUCs in the huc_list argument (-u) or the current HUCs with files in <output_dir>'             \
-        'will be used.', required=False, action='store_true')
-    
-    args = vars(parser.parse_args())
 
+    parser.add_argument(
+        '-p',
+        '--points_data_file_name',
+        help='REQUIRED: Complete relative filepath of a .gpkg file with fim calibration points.',
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        '-o',
+        '--output_dir',
+        help='OPTIONAL: path to send .parquet file/files. Default location specifed as the'
+        '"input_calib_points_dir" variable in src/bash_variables.env',
+        type=str,
+        required=False,
+        default=input_calib_points_dir,
+    )
+
+    parser.add_argument(
+        '-j',
+        '--number_of_jobs',
+        help='OPTIONAL: number of cores/processes (default=4). This is a memory intensive '
+        'script, and the multiprocessing will crash if too many CPUs are used. It is recommended to provide '
+        'half the amount of available CPUs.',
+        type=int,
+        required=False,
+        default=4,
+    )
+
+    parser.add_argument(
+        '-wbd',
+        '--wbd_layer',
+        help='OPTIONAL: A directory of where a .gpkg file exists, containing HUC boundary polygons. '
+        'Default filepath is specified as the "input_WBD_gdb" variable in src/bash_variables.env',
+        type=str,
+        required=False,
+        default=input_WBD_gdb,
+    )
+
+    parser.add_argument(
+        '-u',
+        '--huc_list',
+        help='OPTIONAL: HUC list - String with comma seperated HUC numbers. *DO NOT include spaces*.'
+        'Provide certain HUCs if points were added/updated only within a few known HUCs.',
+        type=str,
+        required=False,
+        default=None,
+    )
+
+    parser.add_argument(
+        '-a',
+        '--all_hucs',
+        help='OPTIONAL: Provide the -a flag if new calibration points were added to many HUCs which currently'
+        'do not have .parquet files.  All HUC polygons in the file provided to -wbd argument will be checked '
+        'for calibration points contained in <points_data_file_name>. If not provided, either the HUCs in '
+        'the huc_list argument (-u) or the current HUCs with files in <output_dir> will be used.',
+        required=False,
+        action='store_true',
+    )
+
+    args = vars(parser.parse_args())
 
     create_parquet_files(**args)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4ab8f1635..dc1631156 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,62 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.1.0 - 2023-09-18 - [PR#988](https://github.com/NOAA-OWP/inundation-mapping/pull/988)
+
+Format code using `black` formatter, incorporate `isort` package to sort import statements,
+and adhere all code to PEP8 Style Guide using the `flake8` package. Remove deprecated files.
+Set up git pre-commit hooks.
+
+Not all files were modified, however, to avoid individually listing each file here, the `/*` convention
+is used to denote that almost every file in those directories were formatted and linted.
+
+### Changes
+
+- `.gitattributes`: Add newline at EOF.
+- `.github/*`: 
+- `.gitignore`: Trim extra last line.
+- `CONTRIBUTING.md`: Update contributing guidelines.
+- `Dockerfile`: Update PYTHONPATH to point to correct `unit_tests` directory.
+- `Pipfile`: Add flake8, black, pyproject-flake8, pre-commit, isort packages
+- `Pipfile.lock`: Update to correspond with new packages in Pipfile 
+- `README.md` : Update link to wiki, trim whitespace.
+- `config/*`
+- `data/*`
+- `docs/*`
+- `fim_pipeline.sh` : Clean up usage statement
+- `fim_post_processing.sh`: Update usage statement
+- `fim_pre_processing.sh`: Update usage statement.
+- `fim_process_unit_wb.sh`: Make usage functional, combine usage and comments.
+- `src/*`
+- `tools/*`
+- `unit_tests/*`: The directory name where the unit test data must reside was changed from
+`fim_unit_test_data_do_not_remove` => `unit_test_data`
+
+### Additions
+
+- `pyproject.toml`: Configuration file
+- `.pre-commit-config.yaml`: Initialize git pre-commit hooks
+- `tools/hash_compare.py`: Carson's hash compare script added to compare files or directories 
+in which we do not expect any changes.
+
+### Removals
+
+- `data/nws/preprocess_ahps_nws.py`
+- `src/adjust_headwater_streams.py`
+- `src/aggregate_vector_inputs.py`
+- `src/utils/reproject_dem.py`
+- `tools/code_standardizer/*`: Incorporated "code_standardizer" into base level Dockerfile.
+- `tools/compile_comp_stats.py`
+- `tools/compile_computational_stats.py`
+- `tools/consolidate_metrics.py`
+- `tools/copy_test_case_folders.py`
+- `tools/cygnss_preprocessing.py`
+- `tools/nesdis_preprocessing.py`
+- `tools/plots/*`: Duplicate and unused directory.
+- `.isort.cfg`: Incorporated into `pyproject.toml`
+
+<br/><br/>
+
 ## v4.4.0.1 - 2023-09-06 - [PR#987](https://github.com/NOAA-OWP/inundation-mapping/pull/987)
 
 Corrects a bug in `src/usgs_gage_unit_setup.py` that causes incorrect values to populate a table, generating an error in `src/usgs_gage_crosswalk.py`.
@@ -13,12 +69,12 @@ Corrects a bug in `src/usgs_gage_unit_setup.py` that causes incorrect values to
 
 ## v4.4.0.0 - 2023-09-01 - [PR#965](https://github.com/NOAA-OWP/inundation-mapping/pull/965)
 
-This feature branch includes new functionality to perform an additional layer of HAND SRC calibration using ras2fim rating curve and point data. The calibration workflow for ras2fim data follows the same general logic as the existing USGS rating curve calibration routine. 
+This feature branch includes new functionality to perform an additional layer of HAND SRC calibration using ras2fim rating curve and point data. The calibration workflow for ras2fim data follows the same general logic as the existing USGS rating curve calibration routine.
 
 ### Additions
 
-- `src/src_adjust_ras2fim_rating.py`: New python script to perform the data prep steps for running the SRC calibration routine: 
-1) merge the `ras_elev_table.csv` data and the ras2fim cross section rating curve data (`reformat_ras_rating_curve_table.csv`) 
+- `src/src_adjust_ras2fim_rating.py`: New python script to perform the data prep steps for running the SRC calibration routine:
+1) merge the `ras_elev_table.csv` data and the ras2fim cross section rating curve data (`reformat_ras_rating_curve_table.csv`)
 2) sample the ras2fim rating curve at NWM recurrence flow intervals (2, 5, 10, 25, 50, 100yr)
 3) pass inputs to the `src_roughness_optimization.py` workflow
 
@@ -68,7 +124,7 @@ Fixes an issue where the stream network was clipped inside the DEM resulting in
 - `unit_tests/`
     - `derive_level_paths_params.json`: Add WBD parameter value
     - `derive_level_paths_test.py`: Add WBD parameter
- 
+
 <br/><br/>
 
 ## v4.3.15.4 - 2023-09-01 - [PR#977](https://github.com/NOAA-OWP/inundation-mapping/pull/977)
@@ -85,11 +141,11 @@ Fixes incorrect `nodata` value in `src/burn_in_levees.py` that was responsible f
 
 <br/><br/>
 
-## v4.3.15.3 - 2023-09-01 - [PR#948](https://github.com/NOAA-OWP/inundation-mapping/pull/983)
+## v4.3.15.3 - 2023-09-01 - [PR#983](https://github.com/NOAA-OWP/inundation-mapping/pull/983)
 
 This hotfix addresses some bugs introduced in the pandas upgrade.
 
-### Changes  
+### Changes
 
 - `/tools/eval_plots_stackedbar.py`: 2 lines were changed to work with the pandas upgrade. Added an argument for a `groupby` median call and fixed a bug with the pandas `query`. Also updated with Black compliance.
 
@@ -103,7 +159,7 @@ Adds a script to produce inundation maps (extent TIFs, polygons, and depth grids
 - `data/`
     - `/tools/inundate_mosaic_wrapper.py`: The script that performs the inundation and mosaicking processes.
     - `/tools/mosaic_inundation.py`: Add function (mosaic_final_inundation_extent_to_poly).
-  
+
 <br/><br/>
 
 ## v4.3.15.1 - 2023-08-08 - [PR#960](https://github.com/NOAA-OWP/inundation-mapping/pull/960)
@@ -124,7 +180,7 @@ Provides a scripted procedure for updating BLE benchmark data including download
 
 <br/><br/>
 
-## v4.3.15.0 - 2023-08-08 - [PR##956](https://github.com/NOAA-OWP/inundation-mapping/pull/956)
+## v4.3.15.0 - 2023-08-08 - [PR#956](https://github.com/NOAA-OWP/inundation-mapping/pull/956)
 
 Integrating GVAL in to the evaluation of agreement maps and contingency tables.
 
@@ -142,7 +198,7 @@ Integrating GVAL in to the evaluation of agreement maps and contingency tables.
 
 The enhancements in this PR include the new modules for pre-processing bathymetric data from the USACE eHydro dataset and integrating the missing hydraulic geometry into the HAND synthetic rating curves.
 
-### Changes  
+### Changes
 - `data/bathymetry/preprocess_bathymetry.py`: added data source column to output geopackage attribute table.
 - `fim_post_processing.sh`: changed -bathy input reference location.
 - `config/params_template.env`: added export to bathymetry_file
@@ -159,12 +215,12 @@ Note: later, as needed, we might upgrade it to handle more than just 10m (which
 
 Additional changes to README to reflect how users can access ESIP's S3 as well as a one line addition to change file permissions in fim_process_unit_wb.sh.
 
-### Changes  
+### Changes
 - `data`
     - `usgs`
         - `acquire_and_preprocess_3dep_dems.py`:  As described above.
  - `fim_pipeline.sh`:  a minor styling fix (added a couple of lines for readability)
- - `fim_pre_processing.sh`: a user message was incorrect & chmod 777 $outputDestDir. 
+ - `fim_pre_processing.sh`: a user message was incorrect & chmod 777 $outputDestDir.
  - `fim_process_unit_wb.sh`: chmod 777 for /output/<run_name> directory.
  - `README.md`: --no-sign-request instead of --request-payer requester for ESIP S3 access.
 
@@ -174,12 +230,12 @@ Additional changes to README to reflect how users can access ESIP's S3 as well a
 
 The enhancements in this PR include the new modules for pre-processing bathymetric data from the USACE eHydro dataset and integrating the missing hydraulic geometry into the HAND synthetic rating curves.
 
-### Additions  
+### Additions
 
 - `data/bathymetry/preprocess_bathymetry.py`: preprocesses the eHydro datasets.
 - `src/bathymetric_adjustment.py`: adjusts synthetic rating curves for HUCs where preprocessed bathymetry is available.
 
-### Changes  
+### Changes
 
 - `config/params_template.env`: added a toggle for the bathymetric adjustment routine: `bathymetry_adjust`
 - `fim_post_processing.sh`: added the new `bathymetric_adjustment.py` to the postprocessing lineup
@@ -224,29 +280,29 @@ Fixes a couple of bugs that prevented inundation using HUC-level hydrotables. Up
 
 ## v4.3.12.0 - 2023-07-05 - [PR#940](https://github.com/NOAA-OWP/inundation-mapping/pull/940)
 
-Refactor Point Calibration Database for synthetic rating curve adjustment to use `.parquet` files instead of a PostgreSQL database. 
+Refactor Point Calibration Database for synthetic rating curve adjustment to use `.parquet` files instead of a PostgreSQL database.
 
 ### Additions
 - `data/`
-    -`write_parquet_from_calib_pts.py`: Script to write `.parquet` files based on calibration points contained in a .gpkg file. 
+    -`write_parquet_from_calib_pts.py`: Script to write `.parquet` files based on calibration points contained in a .gpkg file.
 
-### Changes  
+### Changes
 - `src/`
     - `src_adjust_spatial_obs.py`: Refactor to remove PostgreSQL and use `.parquet` files.
-    - `src_roughness_optimization.py`: Line up comments and add newline at EOF. 
-    - `bash_variables.env`: Update formatting, and add `{}` to inherited `.env` variables for proper variable expansion in Python scripts.  
+    - `src_roughness_optimization.py`: Line up comments and add newline at EOF.
+    - `bash_variables.env`: Update formatting, and add `{}` to inherited `.env` variables for proper variable expansion in Python scripts.
 - `/config`
     - `params_template.env`: Update comment.
 - `fim_pre_processing.sh`: In usage statement, remove references to PostGRES calibration tool.
-- `fim_post_processing.sh`: Remove connection to and loading of PostgreSQL database. 
+- `fim_post_processing.sh`: Remove connection to and loading of PostgreSQL database.
 - `.gitignore`: Add newline.
 - `README.md`: Remove references to PostGRES calibration tool.
 
 ### Removals
-- `config/` 
+- `config/`
     - `calb_db_keys_template.env`: No longer necessary without PostGRES Database.
 
-- `/tools/calibration-db` : Removed directory including files below. 
+- `/tools/calibration-db` : Removed directory including files below.
     - `README.md`
     - `docker-compose.yml`
     - `docker-entrypoint-enitdb.d/init-db.sh`
@@ -259,7 +315,7 @@ Write to a csv file with processing time of `run_unit_wb.sh`, update PR Template
 
 ### Changes
 - `.github/`
-    - `PULL_REQUEST_TEMPLATE.md` : Update PR Checklist into Issuer Checklist and Merge Checklist  
+    - `PULL_REQUEST_TEMPLATE.md` : Update PR Checklist into Issuer Checklist and Merge Checklist
 - `src/`
     - `run_unit_wb.sh`: Add line to log processing time to `$outputDestDir/logs/unit/total_duration_run_by_unit_all_HUCs.csv`
     - `check_huc_inputs.py`: Modify error handling. Correctly print HUC number if it is not valid (within `included_huc*.lst`)
@@ -276,16 +332,16 @@ Write to a csv file with processing time of `run_unit_wb.sh`, update PR Template
 
 Auto Bot asked for the python package of `requests` be upgraded from 2.28.2 to 2.31.0. This has triggered a number of packages to upgrade.
 
-### Changes  
+### Changes
 - `Pipfile.lock`: as described.
 
 <br/><br/>
 
 ## v4.3.11.5 - 2023-05-30 - [PR#911](https://github.com/NOAA-OWP/inundation-mapping/pull/911)
 
-This fix addresses bugs found when using the recently added functionality in `tools/synthesize_test_cases.py` along with the `PREV` argument. The `-pfiles` argument now performs as expected for both `DEV` and `PREV` processing. Addresses #871 
+This fix addresses bugs found when using the recently added functionality in `tools/synthesize_test_cases.py` along with the `PREV` argument. The `-pfiles` argument now performs as expected for both `DEV` and `PREV` processing. Addresses #871
 
-### Changes  
+### Changes
 `tools/synthesize_test_cases.py`: multiple changes to enable all expected functionality with the `-pfiles` and `-pcsv` arguments
 
 <br/><br/>
@@ -296,7 +352,7 @@ There is a growing number of files that need to be pushed up to HydroVis S3 duri
 
 Earlier, we were running a number of aws cli scripts one at a time. This tool simplies it and pushes all of the QA and supporting files. Note: the HAND files from a release, will continue to be pushed by `/data/aws/s3.py` as it filters out files to be sent to HV s3.
 
-### Additions  
+### Additions
 
 - `data\aws`
      - `push-hv-data-support-files.sh`: As described above. See file for command args.
@@ -308,7 +364,7 @@ Earlier, we were running a number of aws cli scripts one at a time. This tool si
 
 Fixes a bug in CatFIM script where a bracket was missing on a pandas `concat` statement.
 
-### Changes  
+### Changes
 - `/tools/generate_categorical_fim.py`: fixes `concat` statement where bracket was missing.
 
 
@@ -319,7 +375,7 @@ Fixes a bug in CatFIM script where a bracket was missing on a pandas `concat` st
 
 This fix addresses a bug that was preventing `burn_in_levees.py` from running. The if statement in run_unit_wb.sh preceeding `burn_in_levees.py` was checking for the existence of a filepath that doesn't exist.
 
-### Changes  
+### Changes
 - `src/run_unit_wb.sh`: fixed the if statement filepath to check for the presence of levee features to burn into the DEM
 
 <br/><br/>
@@ -363,9 +419,9 @@ The following files had instances of `pandas.append` changed to `pandas.concat`:
 
 ## v4.3.11.0 - 2023-05-12 - [PR#903](https://github.com/NOAA-OWP/inundation-mapping/pull/903)
 
-These changes address some known issues where the DEM derived flowlines follow the incorrect flow path (address issues with stream order 1 and 2 only). The revised code adds a new workflow to generate a new flow direction raster separately for input to the `run_by_branch.sh` workflow (branch 0 remains unchanged). This modification helps ensure that the DEM derived flowlines follow the desired NWM flow line when generating the DEM derived flowlines at the branch level. 
+These changes address some known issues where the DEM derived flowlines follow the incorrect flow path (address issues with stream order 1 and 2 only). The revised code adds a new workflow to generate a new flow direction raster separately for input to the `run_by_branch.sh` workflow (branch 0 remains unchanged). This modification helps ensure that the DEM derived flowlines follow the desired NWM flow line when generating the DEM derived flowlines at the branch level.
 
-### Changes  
+### Changes
 - `config/deny_branch_zero.lst`: removed `LandSea_subset_{}.tif` and `flowdir_d8_burned_filled_{}.tif` from the "keep" list as these files are now kept in the huc root folder.
 - `config/deny_unit.lst`: added file cleanups for newly generated branch input files stored in the huc root folder (`dem_burned.tif`, `dem_burned_filled.tif`, `flowdir_d8_burned_filled.tif`, `flows_grid_boolean.tif`, `wbd_buffered_streams.gpkg`)
 - `src/clip_vectors_to_wbd.py`: saving the `wbd_streams_buffer` as an output gpkg for input to `derive_level_paths.py`
@@ -381,13 +437,13 @@ These changes address some known issues where the DEM derived flowlines follow t
 
 ## v4.3.10.0 - 2023-05-12 - [PR#888](https://github.com/NOAA-OWP/inundation-mapping/pull/888)
 
-`aggregate_by_huc.py` was taking a long time to process. Most HUCs can aggregate their branches into one merged hydrotable.csv in just 22 seconds, but a good handful took over 2 mins and a few took over 7 mins. When multiplied by 2,138 HUCs it was super slow. Multi-proc has not been added and it now takes appx 40 mins at 80 cores. 
+`aggregate_by_huc.py` was taking a long time to process. Most HUCs can aggregate their branches into one merged hydrotable.csv in just 22 seconds, but a good handful took over 2 mins and a few took over 7 mins. When multiplied by 2,138 HUCs it was super slow. Multi-proc has not been added and it now takes appx 40 mins at 80 cores.
 
-An error logging system was also added to track errors that may have occurred during processing. 
+An error logging system was also added to track errors that may have occurred during processing.
 
-### Changes  
+### Changes
 - `fim_pipeline.sh` - added a duration counter at the end of processing HUCs
-- `fim_post_processing.sh` - added a job limit (number of procs), did a little cleanup, and added a warning note about usage of job limits in this script, 
+- `fim_post_processing.sh` - added a job limit (number of procs), did a little cleanup, and added a warning note about usage of job limits in this script,
 - `src`
     - `aggregate_by_huc.py`: Added multi proc, made it useable for non external script calls, added a logging system for errors only.
     - `indentify_src_bankful.py`: typo fix.
@@ -398,7 +454,7 @@ An error logging system was also added to track errors that may have occurred du
 
 This merge fixes several sites in Stage-Based CatFIM sites that showed overinundation. The cause was found to be the result of Stage-Based CatFIM code pulling the wrong value from the `usgs_elev_table.csv`. Priority is intended to go to the `dem_adj_elevation` value that is not from branch 0, however there was a flaw in the prioritization logic. Also includes a change to `requests` usage that is in response to an apparent IT SSL change. This latter change was necessary in order to run CatFIM. Also added a check to make sure the `dem_adj_thalweg` is not too far off the official elevation, and continues if it is.
 
-### Changes  
+### Changes
 - `/tools/generate_categorical_fim.py`: fixed pandas bug where the non-branch zero `dem_adj_elevation` value was not being properly indexed. Also added a check to make sure the `dem_adj_thalweg` is not too far off the official elevation, and continues if it is.
 - ` /tools/tools_shared_functions.py`: added `verify=False` to `requests` library calls because connections to WRDS was being refused (likely because of new IT protocols).
 
@@ -408,21 +464,21 @@ This merge fixes several sites in Stage-Based CatFIM sites that showed overinund
 
 Fix existing unit tests, remove unwanted behavior in `check_unit_errors_test.py`, update `unit_tests/README.md`
 
-### Changes  
+### Changes
 
 - `unit_tests/`
     - `README.md` : Split up headings for setting up unit tests/running unit tests & re-formatted code block.
     - `check_unit_errors_test.py`: Fixed unwanted behavior of test leaving behind `sample_n.txt` files in `unit_errors/`
     - `clip_vectors_to_wbd_params.json`: Update parameters
     - `clip_vectors_to_wbd_test.py`: Update arguments
-    - `pyproject.toml`: Ignore RuntimeWarning, to suppress pytest failure. 
+    - `pyproject.toml`: Ignore RuntimeWarning, to suppress pytest failure.
     - `usgs_gage_crosswalk_test.py`: Enhance readability of arguments in `gage_crosswalk.run_crosswalk` call
 
 <br/><br/>
 
 ## v4.3.9.0 - 2023-04-19 - [PR#889](https://github.com/NOAA-OWP/inundation-mapping/pull/889)
 
-Updates GDAL in base Docker image from 3.1.2 to 3.4.3 and updates all Python packages to latest versions, including Pandas v.2.0.0. Fixes resulting errors caused by deprecation and/or other changes in dependencies. 
+Updates GDAL in base Docker image from 3.1.2 to 3.4.3 and updates all Python packages to latest versions, including Pandas v.2.0.0. Fixes resulting errors caused by deprecation and/or other changes in dependencies.
 
 NOTE: Although the most current GDAL is version 3.6.3, something in 3.5 causes an issue in TauDEM `aread8` (this has been submitted as https://github.com/dtarb/TauDEM/issues/254)
 
@@ -466,9 +522,9 @@ Bug fix for empty `output_catchments` in `src/filter_catchments_and_add_attribut
 
 ## v4.3.7.3 - 2023-04-14 - [PR#880](https://github.com/NOAA-OWP/inundation-mapping/pull/880)
 
-Hotfix for addressing an error during the NRMSE calculation/aggregation step within `tools/rating_curve_comparison.py`. Also added the "n" variable to the agg_nwm_recurr_flow_elev_stats table. Addresses #878 
+Hotfix for addressing an error during the NRMSE calculation/aggregation step within `tools/rating_curve_comparison.py`. Also added the "n" variable to the agg_nwm_recurr_flow_elev_stats table. Addresses #878
 
-### Changes  
+### Changes
 
 - `tools/rating_curve_comparison.py`: address error for computing nrmse when n=1; added the "n" variable (sample size) to the output metrics table
 
@@ -493,7 +549,7 @@ Replaces `os.environ` with input arguments in Python files that are called from
 
 Hotfix to `process_branch.sh` because it wasn't removing code-61 branches on exit. Also removes the current run from the new fim_temp directory.
 
-### Changes  
+### Changes
 
 - `fim_pipeline.sh`: removal of current run from fim_temp directory
 - `src/process_branch.sh`: switched the exit 61 block to use the temp directory instead of the outputs directory
@@ -504,7 +560,7 @@ Hotfix to `process_branch.sh` because it wasn't removing code-61 branches on exi
 
 This pull request adds a new feature to `fim_post_processing.sh` to aggregate all of the hydrotables for a given HUC into a single HUC-level `hydrotable.csv` file. Note that the aggregation step happens near the end of `fim_post_processing.sh` (after the subdivision and calibration routines), and the branch hydrotable files are preserved in the branch directories for the time being.
 
-### Changes  
+### Changes
 
 - `fim_pipeline.sh`: created a new variable `$jobMaxLimit` that multiplies the `$jobHucLimit` and the `$jobBranchLimit`
 - `fim_post_processing.sh`: added new aggregation/concatenation step after the SRC calibration routines; passing the new `$jobMaxLimit` to the commands that accept a multiprocessing job number input; added `$skipcal` argument to the USGS rating curve calibration routine
@@ -538,7 +594,7 @@ outputs_cleanup.py was throwing an error saying that the HUC source directory (t
 
 The log file was moved to the unit_errors folder to validate the error, as expected.
 
-### Changes  
+### Changes
 
 - `src/run_unit_wb.sh`: Change the source path being submitted to `outputs_cleanup.py` from the `outputs` HUC directory to the `fim_temp` HUC directory.
 - `fim_process_unit_wb.sh`: Updated the phrase "Copied temp directory" to "Moved temp directory"
@@ -585,39 +641,39 @@ Pygeos is removed because its functionality is incorporated into the upgraded sh
 
 ### Changes
 
-Create a 'working directory' in the Docker container to run processes within the container's non-persistent filesystem. Modify variables in scripts that process HUCs and branches to use the temporary working directory, and then copy temporary directory (after trimming un-wanted files) over to output directory (persistent filesystem).  Roll back changes to `unit_tests/` to use `/data/outputs` (contains canned data), as the volume mounted `outputs/` most likely will not contain the necessary unit test data. 
+Create a 'working directory' in the Docker container to run processes within the container's non-persistent filesystem. Modify variables in scripts that process HUCs and branches to use the temporary working directory, and then copy temporary directory (after trimming un-wanted files) over to output directory (persistent filesystem).  Roll back changes to `unit_tests/` to use `/data/outputs` (contains canned data), as the volume mounted `outputs/` most likely will not contain the necessary unit test data.
 
-- `Dockerfile` - create a `/fim_temp` working directory, update `projectDir` to an `ENV`, rename inputs and outputs directory variables  
-- `fim_pipeline.sh` - remove `projectDir=/foss_fim`, update path of `logFile`, remove indentation  
-- `fim_pre_processing.sh` - change `$outputRunDataDir` => `$outputDestDir` & add `$tempRunDir`  
-- `fim_post_processing.sh` - change `$outputRunDataDir` => `$outputDestDir`   
-- `fim_process_unit_wb.sh` - change `$outputRunDataDir` => `$outputDestDir`, add vars & export `tempRunDir`, `tempHucDataDir`, & `tempBranchDataDir` to `run_unit_wb.sh`  
-- `README.md` - add linebreaks to codeblocks  
+- `Dockerfile` - create a `/fim_temp` working directory, update `projectDir` to an `ENV`, rename inputs and outputs directory variables
+- `fim_pipeline.sh` - remove `projectDir=/foss_fim`, update path of `logFile`, remove indentation
+- `fim_pre_processing.sh` - change `$outputRunDataDir` => `$outputDestDir` & add `$tempRunDir`
+- `fim_post_processing.sh` - change `$outputRunDataDir` => `$outputDestDir`
+- `fim_process_unit_wb.sh` - change `$outputRunDataDir` => `$outputDestDir`, add vars & export `tempRunDir`, `tempHucDataDir`, & `tempBranchDataDir` to `run_unit_wb.sh`
+- `README.md` - add linebreaks to codeblocks
 
-- `src/` 
-  - `bash_variables.env` - `$inputDataDir` => `$inputsDir`  
-  - `check_huc_inputs.py` - `$inputDataDir` => `$inputsDir`  
-  - `delineate_hydros_and_produce_HAND.py` - `$outputHucDataDir` => `$tempHucDataDir`, `$outputCurrentBranchDataDir` => `$tempCurrentBranchDataDir`  
-  - `process_branch.sh` - `$outputRunDataDir` => `$outputsDestDir`  
-  - `run_by_branch.sh` - `$outputCurrentBranchDataDir` => `$tempCurrentBranchDataDir`, `$outputHucDataDir` => `$tempHucDataDir`  
-  - `run_unit_wb.sh` - `$outputRunDataDir` => `$outputDestDir`, `$outputHucDataDir` => `$tempHucDataDir`  
-  - `utils/`  
-    - `shared_functions.py` - `$inputDataDir` => `$inputsDir`  
+- `src/`
+  - `bash_variables.env` - `$inputDataDir` => `$inputsDir`
+  - `check_huc_inputs.py` - `$inputDataDir` => `$inputsDir`
+  - `delineate_hydros_and_produce_HAND.py` - `$outputHucDataDir` => `$tempHucDataDir`, `$outputCurrentBranchDataDir` => `$tempCurrentBranchDataDir`
+  - `process_branch.sh` - `$outputRunDataDir` => `$outputsDestDir`
+  - `run_by_branch.sh` - `$outputCurrentBranchDataDir` => `$tempCurrentBranchDataDir`, `$outputHucDataDir` => `$tempHucDataDir`
+  - `run_unit_wb.sh` - `$outputRunDataDir` => `$outputDestDir`, `$outputHucDataDir` => `$tempHucDataDir`
+  - `utils/`
+    - `shared_functions.py` - `$inputDataDir` => `$inputsDir`
 
-- `tools/` 
-  - `inundation_wrapper_custom_flow.py` - `$outputDataDir` => `$outputsDir`  
-  - `inundation_wrapper_nwm_flows.py`  - `$outputDataDir` => `$outputsDir`  
-  - `tools_shared_variables.py` - `$outputDataDir` => `$outputsDir`    
+- `tools/`
+  - `inundation_wrapper_custom_flow.py` - `$outputDataDir` => `$outputsDir`
+  - `inundation_wrapper_nwm_flows.py`  - `$outputDataDir` => `$outputsDir`
+  - `tools_shared_variables.py` - `$outputDataDir` => `$outputsDir`
 
 - `unit_tests/`
-  - `README.md` - add linebreaks to code blocks, `/outputs/` => `/data/outputs/`  
-  - `*_params.json` - `/outputs/` => `/data/outputs/` & `$outputRunDataDir` => `$outputDestDir`  
-  - `derive_level_paths_test.py` - `$outputRunDataDir` => `$outputDestDir`  
-  - `check_unit_errors_test.py` - `/outputs/` => `/data/outputs/`  
+  - `README.md` - add linebreaks to code blocks, `/outputs/` => `/data/outputs/`
+  - `*_params.json` - `/outputs/` => `/data/outputs/` & `$outputRunDataDir` => `$outputDestDir`
+  - `derive_level_paths_test.py` - `$outputRunDataDir` => `$outputDestDir`
+  - `check_unit_errors_test.py` - `/outputs/` => `/data/outputs/`
   - `shared_functions_test.py` - `$outputRunDataDir` => `$outputDestDir`
-  - `split_flows_test.py`  - `/outputs/` => `/data/outputs/`  
-  - `tools/` 
-    - `*_params.json` - `/outputs/` => `/data/outputs/` & `$outputRunDataDir` => `$outputDestDir`  
+  - `split_flows_test.py`  - `/outputs/` => `/data/outputs/`
+  - `tools/`
+    - `*_params.json` - `/outputs/` => `/data/outputs/` & `$outputRunDataDir` => `$outputDestDir`
 
 <br/><br/>
 
@@ -626,11 +682,11 @@ Create a 'working directory' in the Docker container to run processes within the
 Simple update to the `PULL_REQUEST_TEMPLATE.md` to remove unnecessary/outdated boilerplate items, add octothorpe (#) in front of Additions, Changes, Removals to mirror `CHANGELOG.md` format, and clean up the PR Checklist.
 
 ### Changes
-- `docs/`  
-  - `PULL_REQUEST_TEMPLATE.md` 
+- `docs/`
+  - `PULL_REQUEST_TEMPLATE.md`
 
 <br/><br/>
-  
+
 ## v4.3.3.6 - 2023-03-30 - [PR#859](https://github.com/NOAA-OWP/inundation-mapping/pull/859)
 
 Addresses the issue of output storage space being taken up by output files from branches that did not run. Updates branch processing to remove the extraneous branch file if a branch gets an error code of 61.
@@ -643,7 +699,7 @@ Addresses the issue of output storage space being taken up by output files from
 
 ## v4.3.3.5 - 2023-03-23 - [PR#848](https://github.com/NOAA-OWP/inundation-mapping/pull/848)
 
-Introduces two new arguments (`-pcsv` and `-pfiles`) and improves the documentation of  `synthesize_test_cases.py`. The new arguments allow the user to provide a CSV of previous metrics (`-pcsv`) and to specity whether or not metrics should pulled from previous directories (`-pfiles`). 
+Introduces two new arguments (`-pcsv` and `-pfiles`) and improves the documentation of  `synthesize_test_cases.py`. The new arguments allow the user to provide a CSV of previous metrics (`-pcsv`) and to specity whether or not metrics should pulled from previous directories (`-pfiles`).
 
 The dtype warning was suppressed through updates to the `read_csv` function in `hydrotable.py` and additional comments were added throughout script to improve readability.
 
@@ -651,7 +707,7 @@ The dtype warning was suppressed through updates to the `read_csv` function in `
 - `tools/inundation.py`: Add data types to the section that reads in the hydrotable (line 483).
 
 - `tools/synthesize_test_cases.py`: Improved formatting, spacing, and added comments. Added two new arguments: `pcsv` and `pfiles` along with checks to verify they are not being called concurrently (lines 388-412). In `create_master_metrics_csv`, creates an `iteration_list` that only contains `['comparison']` if `pfiles` is not true, reads in the previous metric csv `prev_metrics_csv` if it is provided and combine it with the compiled metrics (after it is converted to dataframe), and saves the metrics dataframe (`df_to_write`) to CSV.
-  
+
 <br/><br/>
 
 ## v4.3.3.4 - 2023-03-17 - [PR#849](https://github.com/NOAA-OWP/inundation-mapping/pull/849)
@@ -764,20 +820,20 @@ Change all occurances of /data/outputs to /outputs to honor the correct volume m
 - `fim_pipeline.sh` - updated comments in relation to `projectDir=/foss_fim`
 - `fim_pre_processing.sh` -updated comments in relation to `projectDir=/foss_fim`
 - `fim_post_processing.sh` - updated comments in relation to `projectDir=/foss_fim`
-- `README.md` - Provide documentation on starting the Docker Container, and update docs to include additional command line option for calibration database tool.   
+- `README.md` - Provide documentation on starting the Docker Container, and update docs to include additional command line option for calibration database tool.
 
-- `src/` 
+- `src/`
   - `usgs_gage_crosswalk.py` - added newline character to shorten commented example usage
-  - `usgs_gage_unit_setup.py` - `/data/outputs/` => `/outputs/`  
+  - `usgs_gage_unit_setup.py` - `/data/outputs/` => `/outputs/`
 
-- `tools/` 
+- `tools/`
   - `cache_metrics.py` -  `/data/outputs/` => `/outputs/`
   - `copy_test_case_folders.py`  - `/data/outputs/` => `/outputs/`
-  - `run_test_case.py` - `/data/outputs/` => `/outputs/`  
+  - `run_test_case.py` - `/data/outputs/` => `/outputs/`
 
-- `unit_tests/*_params.json`  - `/data/outputs/` => `/outputs/` 
+- `unit_tests/*_params.json`  - `/data/outputs/` => `/outputs/`
 
-- `unit_tests/split_flows_test.py`  - `/data/outputs/` => `/outputs/` 
+- `unit_tests/split_flows_test.py`  - `/data/outputs/` => `/outputs/`
 
 <br/><br/>
 
@@ -834,7 +890,7 @@ Other file changes:
 - `data`
     - `usgs`
         - `acquire_and_preprocess_3dep_dems.py`: Added text on data input URL source.
-        
+
 <br/><br/>
 
 ## v4.2.0.1 - 2023-02-16 - [PR#827](https://github.com/NOAA-OWP/inundation-mapping/pull/827)
@@ -963,7 +1019,7 @@ An update was required to adjust host name when in the AWS environment
 
 ### Changes
 
-- `fim_post_processing.sh`: Added an "if isAWS" flag system based on the input command args from fim_pipeline.sh or 
+- `fim_post_processing.sh`: Added an "if isAWS" flag system based on the input command args from fim_pipeline.sh or
 
 - `tools/calibration-db`
     - `README.md`: Minor text correction.
@@ -979,7 +1035,7 @@ Add `pytest` package and refactor existing unit tests. Update parameters to unit
 - `/unit_tests`
   - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
   - `pyproject.toml`  - used to specify which warnings are excluded/filtered.
-  - `/gms`  
+  - `/gms`
     - `__init__.py` - needed for `pytest` command line executable to pick up tests.
   - `/tools`
     - `__init__.py`  - needed for `pytest` command line executable to pick up tests.
@@ -990,8 +1046,8 @@ Add `pytest` package and refactor existing unit tests. Update parameters to unit
 
 ### Removals
 
-- `/unit_tests/tools/gms_tools/` directory removed, and files moved up into `/unit_tests/tools`    
- 
+- `/unit_tests/tools/gms_tools/` directory removed, and files moved up into `/unit_tests/tools`
+
 ### Changes
 
 - `Pipfile` - updated to include pytest as a dependency
@@ -999,30 +1055,30 @@ Add `pytest` package and refactor existing unit tests. Update parameters to unit
 
 - `/config`
   - `params_template.env` - leading slash added to paths
-  
-- `/unit_tests/` - All of the `*_test.py` files were refactored to follow the `pytest` paradigm.  
+
+- `/unit_tests/` - All of the `*_test.py` files were refactored to follow the `pytest` paradigm.
   - `*_params.json` - valid paths on `fim-dev1` provided
-  - `README.md`  - updated to include documentation on pytest.  
-  - `unit_tests_utils.py`  
+  - `README.md`  - updated to include documentation on pytest.
+  - `unit_tests_utils.py`
   - `__template_unittests.py` -> `__template.py` - exclude the `_test` suffix to remove from test suite. Updated example on new format for pytest.
-  - `check_unit_errors_test.py`  
-  - `clip_vectors_to_wbd_test.py`  
-  - `filter_catchments_and_add_attributes_test.py`  
-  - `rating_curve_comparison_test.py`  
-  - `shared_functions_test.py`  
-  - `split_flow_test.py`  
-  - `usgs_gage_crosswalk_test.py`  
-  - `aggregate_branch_lists_test.py`  
-  - `generate_branch_list_test.py`  
-  - `generate_branch_list_csv_test.py`  
-  - `aggregate_branch_lists_test.py`  
-  - `generate_branch_list_csv_test.py`  
-  - `generate_branch_list_test.py`  
-    - `/gms`  
-      - `derive_level_paths_test.py`  
+  - `check_unit_errors_test.py`
+  - `clip_vectors_to_wbd_test.py`
+  - `filter_catchments_and_add_attributes_test.py`
+  - `rating_curve_comparison_test.py`
+  - `shared_functions_test.py`
+  - `split_flow_test.py`
+  - `usgs_gage_crosswalk_test.py`
+  - `aggregate_branch_lists_test.py`
+  - `generate_branch_list_test.py`
+  - `generate_branch_list_csv_test.py`
+  - `aggregate_branch_lists_test.py`
+  - `generate_branch_list_csv_test.py`
+  - `generate_branch_list_test.py`
+    - `/gms`
+      - `derive_level_paths_test.py`
       - `outputs_cleanup_test.py`
     - `/tools`
-      - `inundate_unittests.py` -> `inundation_test.py`  
+      - `inundate_unittests.py` -> `inundation_test.py`
       - `inundate_gms_test.py`
 
 
@@ -1051,15 +1107,15 @@ As we move to Amazon Web Service, AWS, we need to change our processing system.
 Our new processing system, starting with this PR,  is to allow each HUC to process it's own branches.
 
 A further requirement was to split up the overall processing flow to independent steps, with each step being able to process itself without relying on "export" variables from other files. Note: There are still a few exceptions.  The basic flow now becomes
-- `fim_pre_processing.sh`, 
+- `fim_pre_processing.sh`,
 - one or more calls to `fim_process_unit_wb.sh` (calling this file for each single HUC to be processed).
-- followed by a call to `fim_post_processing.sh`. 
+- followed by a call to `fim_post_processing.sh`.
 
 
-Note: This is a very large, complex PR with alot of critical details. Please read the details at [PR 806](https://github.com/NOAA-OWP/inundation-mapping/pull/806). 
+Note: This is a very large, complex PR with alot of critical details. Please read the details at [PR 806](https://github.com/NOAA-OWP/inundation-mapping/pull/806).
 
 ### CRITICAL NOTE
-The new `fim_pipeline.sh` and by proxy `fim_pre_processing.sh` has two new key input args, one named **-jh** (job HUCs) and one named **-jb** (job branches).  You can assign the number of cores/CPU's are used for processing a HUC versus the number of branches.  For the -jh number arg, it only is used against the `fim_pipeline.sh` file when it is processing more than one HUC or a list of HUCs as it is the iterator for HUCs.   The -jb flag says how many cores/CPU's can be used when processing branches (note.. the average HUC has 26 branches). 
+The new `fim_pipeline.sh` and by proxy `fim_pre_processing.sh` has two new key input args, one named **-jh** (job HUCs) and one named **-jb** (job branches).  You can assign the number of cores/CPU's are used for processing a HUC versus the number of branches.  For the -jh number arg, it only is used against the `fim_pipeline.sh` file when it is processing more than one HUC or a list of HUCs as it is the iterator for HUCs.   The -jb flag says how many cores/CPU's can be used when processing branches (note.. the average HUC has 26 branches).
 
 BUT.... you have to be careful not to overload your system.  **You need to multiply the -jh and the -jb values together, but only when using the `fim_pipeline.sh` script.**  Why? _If you have 16 CPU's available on your machine, and you assign -jh as 10 and -jb as 26, you are actually asking for 126 cores (10 x 26) but your machine only has 16 cores._   If you are not using `fim_pipeline.sh` but using the three processing steps independently, then the -jh value has not need to be anything but the number of 1 as each actual HUC can only be processed one at a time. (aka.. no iterator).
 </br>
@@ -1067,13 +1123,13 @@ BUT.... you have to be careful not to overload your system.  **You need to multi
 ### Additions
 
 - `fim_pipeline.sh` :  The wrapper for the three new major "FIM" processing steps. This script allows processing in one command, same as the current tool of `gms_pipeline.sh`.
-- `fim_pre_processing.sh`: This file handles all argument input from the user, validates those inputs and sets up or cleans up folders. It also includes a new system of taking most input parameters and some key enviro variables and writing them out to a files called `runtime_args.env`.  Future processing steps need minimal input arguments as it can read most values it needs from this new `runtime_args.env`. This allows the three major steps to work independently from each other. Someone can now come in, run `fim_pre_processing.sh`, then run `fim_process_unit_wb.sh`, each with one HUC, as many time as they like, each adding just its own HUC folder to the output runtime folder. 
-- `fim_post_processing.sh`: Scans all HUC folders inside the runtime folders to handle a number of processing steps which include (to name a few): 
+- `fim_pre_processing.sh`: This file handles all argument input from the user, validates those inputs and sets up or cleans up folders. It also includes a new system of taking most input parameters and some key enviro variables and writing them out to a files called `runtime_args.env`.  Future processing steps need minimal input arguments as it can read most values it needs from this new `runtime_args.env`. This allows the three major steps to work independently from each other. Someone can now come in, run `fim_pre_processing.sh`, then run `fim_process_unit_wb.sh`, each with one HUC, as many time as they like, each adding just its own HUC folder to the output runtime folder.
+- `fim_post_processing.sh`: Scans all HUC folders inside the runtime folders to handle a number of processing steps which include (to name a few):
     - aggregating errors
     - aggregating to create a single list (gms_inputs.csv) for all valid HUCs and their branch ids
     - usgs gage aggregation
     - adjustments to SRV's
-    - and more    
+    - and more
 - `fim_process_unit_wb.sh`: Accepts only input args of runName and HUC number. It then sets up global variable, folders, etc to process just the one HUC. The logic for processing the HUC is in `run_unit_wb.sh` but managed by this `fim_process_unit_wb.sh` file including all error trapping.
 - `src`
     - `aggregate_branch_lists.py`:  When each HUC is being processed, it creates it's own .csv file with its branch id's. In post processing we need one master csv list and this file aggregates them. Note: This is a similar file already in the `src/gms` folder but that version operates a bit different and will be deprecated soon.
@@ -1084,7 +1140,7 @@ BUT.... you have to be careful not to overload your system.  **You need to multi
 - `unit_tests`
     - `aggregate_branch_lists_unittests.py' and `aggregate_branch_lists_params.json`  (based on the newer `src` directory edition of `aggregate_branch_lists.py`).
     - `generate_branch_list_unittest.py` and `generate_branch_list_params.json` (based on the newer `src` directory edition of `generate_branch_list.py`).
-    -  `generate_branch_list_csv_unittest.py` and `generate_branch_list_csv_params.json` 
+    -  `generate_branch_list_csv_unittest.py` and `generate_branch_list_csv_params.json`
 
 ### Changes
 
@@ -1101,7 +1157,7 @@ BUT.... you have to be careful not to overload your system.  **You need to multi
     - `clip_vectors_to_wbd.py`: Cleaned up some print statements for better output traceability.
     - `check_huc_inputs.py`: Added logic to ensure the file was an .lst file. Other file formats were not be handled correctly.
     - `gms`
-        - `delineate_hydros_and_produce_HAND.sh`: Removed all `stopDiv` variable to reduce log and screen output. 
+        - `delineate_hydros_and_produce_HAND.sh`: Removed all `stopDiv` variable to reduce log and screen output.
         - `run_by_branch.sh`: Removed an unnecessary test for overriding outputs.
 
 ### Removed
@@ -1125,7 +1181,7 @@ When running tools/test_case_by_hydroid.py, it throws an error of local variable
 
 ## v4.0.19.4 - 2023-01-25 - [PR#802](https://github.com/NOAA-OWP/inundation-mapping/pull/802)
 
-This revision includes a slight alteration to the filtering technique used to trim/remove lakeid nwm_reaches that exist at the upstream end of each branch network. By keeping a single lakeid reach at the branch level, we can avoid issues with the branch headwater point starting at a lake boundary. This ensures the headwater catchments for some branches are properly identified as a lake catchment (no inundation produced). 
+This revision includes a slight alteration to the filtering technique used to trim/remove lakeid nwm_reaches that exist at the upstream end of each branch network. By keeping a single lakeid reach at the branch level, we can avoid issues with the branch headwater point starting at a lake boundary. This ensures the headwater catchments for some branches are properly identified as a lake catchment (no inundation produced).
 
 ### Changes
 
@@ -1326,7 +1382,7 @@ Added rysnc to docker images. rysnc can now be used inside the images to move da
 
 ### Changes
 
-- `Dockerfile` : added rsync 
+- `Dockerfile` : added rsync
 
 <br/><br/>
 
@@ -1337,7 +1393,7 @@ Added rysnc to docker images. rysnc can now be used inside the images to move da
 Other minor changes include:
 - Removing the system where a user could override `DropStreamOrders` where they could process streams with stream orders 1 and 2 independently like other GMS branches.  This option is now removed, so it will only allow stream orders 3 and higher as gms branches and SO 1 and 2 will always be in branch zero.
 
-- The `retry` flag on the three gms*.sh files has been removed. It did not work correctly and was not being used. Usage of it would have created unreliable results. 
+- The `retry` flag on the three gms*.sh files has been removed. It did not work correctly and was not being used. Usage of it would have created unreliable results.
 
 ### Additions
 
@@ -1365,12 +1421,12 @@ Other minor changes include:
    - Updated for newer reusable output date/time/duration system.
 
 - `src`
-    - `bash_functions.env`:  Added a new method to make it easier / simpler to calculation and display duration time. 
+    - `bash_functions.env`:  Added a new method to make it easier / simpler to calculation and display duration time.
     - `filter_catchments_and_add_attributes.py`:  Remove "dropLowStreamOrders" system.
     - `split_flows.py`: Remove "dropLowStreamOrders" system.
     - `usgs_gage_unit_setup.py`:  Remove "dropLowStreamOrders" system.
 
-- `gms`  
+- `gms`
     - `delineate_hydros_and_produced_HAND.sh` : Remove "dropLowStreamOrders" system.
     - `derive_level_paths.py`: Remove "dropLowStreamOrders" system and some small style updates.
     - `run_by_unit.sh`: Remove "dropLowStreamOrders" system.
@@ -1531,7 +1587,7 @@ This feature branch introduces a new methodology for computing Manning's equatio
 - `tools/vary_mannings_n_composite.py`: *moved this script from /src to /tools*; updated this code from FIM3 to work with FIM4 structure; however, it is not currently implemented (the subdivision routine replaces this)
 - `tools/aggregate_csv_files.py`: helper tool to search for csv files by name/wildcard and concatenate all found files into one csv (used for aggregating previous calibrated roughness values)
 - `tools/eval_plots.py`: updated list of metrics to plot to also include equitable threat score and mathews correlation coefficient (MCC)
-- `tools/synthesize_test_cases.py`: updated the list of FIM version metrics that the `PREV` flag will use to create the final aggregated metrics csv; this change will combine the dev versions provided with the `-dc` flag along with the existing `previous_fim_list` 
+- `tools/synthesize_test_cases.py`: updated the list of FIM version metrics that the `PREV` flag will use to create the final aggregated metrics csv; this change will combine the dev versions provided with the `-dc` flag along with the existing `previous_fim_list`
 
 <br/><br/>
 
@@ -1554,7 +1610,7 @@ Adds capability to produce single rating curve comparison plots for each gage.
 - `tools/rating_curve_comparison.py`
     - Adds generate_single_plot() to make a single rating curve comparison plot for each gage in a given HUC
     - Adds command line switch to generate single plots
-    
+
 <br/><br/>
 
 ## v4.0.11.3 - 2022-11-10 - [PR #739](https://github.com/NOAA-OWP/inundation-mapping/pull/739)
@@ -1687,10 +1743,10 @@ Runs branch 0 on HUCs with no other branches remaining after filtering stream or
     - `generate_branch_list.py`: adds check for `stream_network_dissolved`
 
 <br/><br/>
-    
+
 ## v4.0.10.1 - 2022-10-5 - [PR #695](https://github.com/NOAA-OWP/inundation-mapping/pull/695)
 
-This hotfix address a bug with how the rating curve comparison (sierra test) handles the branch zero synthetic rating curve in the comparison plots. Address #676 
+This hotfix address a bug with how the rating curve comparison (sierra test) handles the branch zero synthetic rating curve in the comparison plots. Address #676
 
 ### Changes
 
@@ -1767,7 +1823,7 @@ This is being added to source control as it might be used again and we don't wan
 
 - `config`
    - `aws_s3_put_fim3_whitelist.lst`
-   
+
 ### Renamed
 
 - `config`
@@ -1828,7 +1884,7 @@ Cleanup Branch Zero output at the end of a processing run. Without this fix, som
 - `gms_run_unit.sh`: changed variables and text to reflect the renamed default `deny_gms_unit_prod.lst` files. Also added a bit of minor output text (styling). Also tells how a user can use the word 'none' for the deny list parameter to skip output cleanup.
 
 - `gms_run_branch.sh`:
-       ... changed variables and text to reflect the renamed default `deny_gms_branches.lst` files. 
+       ... changed variables and text to reflect the renamed default `deny_gms_branches.lst` files.
        ... added a bit of minor output text (styling).
        ... also tells how a user can use the word 'none' for the deny list parameter to skip output cleanup.
        ... added a new section that calls the `outputs_cleanup.py` file and will do post cleanup on branch zero output files.
@@ -1836,13 +1892,13 @@ Cleanup Branch Zero output at the end of a processing run. Without this fix, som
 - `src`
     - `gms`
         - `outputs_cleanup.py`: pretty much rewrote it in its entirety. Now accepts a manditory branch id (can be zero) and can recursively search subdirectories. ie) We can submit a whole output directory with all hucs and ask to cleanup branch 0 folder OR cleanup files in any particular directory as we did before (per branch id).
-          
+
           - `run_by_unit.sh`:  updated to pass in a branch id (or the value of "0" meaning branch zero) to outputs_cleanup.py.
           - `run_by_branch.sh`:  updated to pass in a branch id to outputs_cleanup.py.
-      
+
 - `unit_tests`
     - `README.md`: updated to talk about the specific deny list for unit_testing.
-    - `__template_unittests.py`: updated for the latest code standards for unit tests. 
+    - `__template_unittests.py`: updated for the latest code standards for unit tests.
 
 - `config`
     - `deny_gms_branch_unittest.lst`: Added some new files to be deleted, updated others.
@@ -1890,7 +1946,7 @@ Other changes:
          - `shared_functions.py`:  changes made were
               - Cleanup the "imports" section of the file (including a change to how the utils.shared_variables file is loaded.
               - Added `progress_bar_handler` function which can be re-used by other code files.
-              - Added `get_file_names` which can create a list of files from a given directory matching a given extension. 
+              - Added `get_file_names` which can create a list of files from a given directory matching a given extension.
               - Modified `print_current_date_time` and `print_date_time_duration` and  methods to return the date time strings. These helper methods exist to help with standardization of logging and output console messages.
               - Added `print_start_header` and `print_end_header` to help with standardization of console and logging output messages.
           - `shared_variables.py`: Additions in support of near future functionality of having fim load DEM's from USGS 3DEP instead of NHD rasters.
@@ -1932,7 +1988,7 @@ A couple of changes:
 
 - `utils`
    - `shared_functions.py`:  A couple of new features
-       -  Added a method which accepts a path to a .lst or .txt file with a collection of data and load it into a  python list object. It can be used for a list of HUCS, file paths, or almost anything. 
+       -  Added a method which accepts a path to a .lst or .txt file with a collection of data and load it into a  python list object. It can be used for a list of HUCS, file paths, or almost anything.
        - A new method for quick addition of current date/time in output.
        - A new method for quick calculation and formatting of time duration in hours, min and seconds.
        - A new method for search for a string in a given python list. It was designed with the following in mind, we already have a python list loaded with whitelist of files to be included in an S3 push. As we iterate through files from the file system, we can use this tool to see if the file should be pushed to S3. This tool can easily be used contexts and there is similar functionality in other FIM4 code that might be able to this method.
@@ -1971,7 +2027,7 @@ Trims ends of branches that are in waterbodies; also removes branches if they ar
 
 ## v4.0.7.1 - 2022-08-22 - [PR #665](https://github.com/NOAA-OWP/inundation-mapping/pull/665)
 
-Hotfix for addressing missing input variable when running `gms_run_branch.sh` outside of `gms_pipeline.sh`. 
+Hotfix for addressing missing input variable when running `gms_run_branch.sh` outside of `gms_pipeline.sh`.
 
 ### Changes
 - `gms_run_branch.sh`: defining path to WBD HUC input file directly in ogr2ogr call rather than using the $input_WBD_gdb defined in `gms_run_unit.sh`
@@ -2070,12 +2126,12 @@ Fixes bug that causes [Errno2] No such file or directory error when running synt
 
 ## v4.0.5.3 - 2022-07-27 - [PR #630](https://github.com/NOAA-OWP/inundation-mapping/issues/630)
 
-A file called gms_pipeline.sh already existed but was unusable. This has been updated and now can be used as a "one-command" execution of the fim4/gms run. While you still can run gms_run_unit.sh and gms_run_branch.sh as you did before, you no longer need to. Input arguments were simplified to allow for more default and this simplification was added to `gms_run_unit.sh` and `gms_run_branch.sh` as well. 
+A file called gms_pipeline.sh already existed but was unusable. This has been updated and now can be used as a "one-command" execution of the fim4/gms run. While you still can run gms_run_unit.sh and gms_run_branch.sh as you did before, you no longer need to. Input arguments were simplified to allow for more default and this simplification was added to `gms_run_unit.sh` and `gms_run_branch.sh` as well.
 
 A new feature was added that is being used for `gms_pipeline.sh` which tests the percent and number of errors after hucs are processed before continuing onto branch processing.
 
 New FIM4/gms usability is now just (at a minumum): `gms_pipeline.sh -n <output name> -u <HUC(s) or HUC list path>`
-	
+
 `gms_run_branch.sh` and `gms_run_branch.sh` have also been changed to add the new -a flag and default to dropping stream orders 1 and 2.
 
 ### Additions
@@ -2083,7 +2139,7 @@ New FIM4/gms usability is now just (at a minumum): `gms_pipeline.sh -n <output n
 - `src`
     - `check_unit_errors.py`: as described above.
 - `unit_tests`
-    - `check_unit_errors_unittests.py` and `check_unit_errors_params.json`: to match new file.    
+    - `check_unit_errors_unittests.py` and `check_unit_errors_params.json`: to match new file.
 
 ### Changes
 
@@ -2096,7 +2152,7 @@ New FIM4/gms usability is now just (at a minumum): `gms_pipeline.sh -n <output n
          - `fim_enums.py`:  FIM_system_exit_codes renamed to FIM_exit_codes.
          - `shared_variables.py`: added configurable values for minimum number and percentage of unit errors.
     - `bash_functions.env`:   Update to make the cumulative time screen outputs in mins/secs instead of just seconds.
-    - `check_huc_inputs.py`:  Now returns the number of HUCs being processed, needed by `gms_pipeline.sh` (Note: to get the value back to a bash file, it has to send it back via a "print" line and not a "return" value.  Improved input validation, 
+    - `check_huc_inputs.py`:  Now returns the number of HUCs being processed, needed by `gms_pipeline.sh` (Note: to get the value back to a bash file, it has to send it back via a "print" line and not a "return" value.  Improved input validation,
 - `unit_tests`
    - `README.md`: Misc text and link updates.
 
@@ -2120,16 +2176,16 @@ Updates to unit tests including a minor update for outputs and loading in .json
 - `gms_run_branch.sh`:  Updated help instructions to about using multiple HUCs as command arguments.
 - `gms_run_unit.sh`:  Updated help instructions to about using multiple HUCs as command arguments.
 - `src/utils`
-    - `shared_functions.py`: 
+    - `shared_functions.py`:
        - Added a new function called `vprint` which creates a simpler way (and better readability) for other python files when wanting to include a print line when the verbose flag is on.
-       - Added a new class named `FIM_Helpers` as a wrapper for the new `vprint` method. 
+       - Added a new class named `FIM_Helpers` as a wrapper for the new `vprint` method.
        - With the new `FIM_Helpers` class, a previously existing method named `append_id_to_file_name` was moved into this class making it easier and quicker for usage in other classes.
-       
+
 - `tools`
     - `composite_inundation.py`: Updated its usage of the `append_id_to_file_name` function to now call the`FIM_Helpers` method version of it.
     - `gms_tools`
-       - `inundate_gms.py`: Updated for its adjusted usage of the `append_id_to_file_name` method, also removed its own `def __vprint` function in favour of the `FIM_Helpers.vprint` method. 
-       - `mosaic_inundation.py`: 
+       - `inundate_gms.py`: Updated for its adjusted usage of the `append_id_to_file_name` method, also removed its own `def __vprint` function in favour of the `FIM_Helpers.vprint` method.
+       - `mosaic_inundation.py`:
           - Added adjustments for use of `append_id_to_file_name` and adjustments for `fh.vprint`.
           - Fixed a bug for the variable `ag_mosaic_output` which was not pre-declared and would fail as using an undefined variable in certain conditions.
     - `run_test_case.py`: Ported `test_case` class from FIM 3 and tweaked slightly to allow for GMS FIM. Also added more prints against the new fh.vprint method. Also added a default print line for progress / traceability for all alpha test regardless if the verbose flag is set.
@@ -2179,7 +2235,7 @@ We needed a tool that could composite / mosaic inundation maps for FIM3 FR and F
 
 ### Additions
 
-- `tools/composite_inundation.py`: Technically it is a renamed from composite_ms_fr_inundation.py, and is based on that functionality, but has been heavily modified. It has a number of options, but primarily is designed to take two sets of output directories, inundate the files, then composite them into a single mosiac'd raster per huc. The primary usage is expected to be compositing FIM3 FR with FIM4 / GMS with stream orders 3 and higher. 
+- `tools/composite_inundation.py`: Technically it is a renamed from composite_ms_fr_inundation.py, and is based on that functionality, but has been heavily modified. It has a number of options, but primarily is designed to take two sets of output directories, inundate the files, then composite them into a single mosiac'd raster per huc. The primary usage is expected to be compositing FIM3 FR with FIM4 / GMS with stream orders 3 and higher.
 
 - `unit_tests/gms/inundate_gms_unittests.py and inundate_gms_params.json`: for running unit tests against `tools/gms_tools/inunundate_gms.py`.
 - `unit_tests/shared_functions_unittests.py and shared_functions_params.json`: A new function named `append_id_to_file_name_single_identifier` was added to `src/utils/shared_functions.py` and some unit tests for that function was created.
@@ -2230,7 +2286,7 @@ A few other small adjustments were made for readability and traceability as well
 - `src/gms/aggregate_branch_lists.py`: Adjusted to allow two previously hardcoded values to now be incoming arguments. Now this file can be used by both `gms_run_unit.sh` and `copy_test_case_folders.py`.
 - `tools/synthesize_test_cases.py`: Adjustments for readability and progress status. The embedded progress bars are not working and will be addressed later.
 - `tools/run_test_case.py`: A print statement was added to help with processing progess was added.
-- `gms_run_unit.sh`: This was adjusted to match the new input parameters for `aggregate_branch_lists.py` as well as additions for progress status. It now will show the entire progress period start datetime, end datetime and duration. 
+- `gms_run_unit.sh`: This was adjusted to match the new input parameters for `aggregate_branch_lists.py` as well as additions for progress status. It now will show the entire progress period start datetime, end datetime and duration.
 - `gms_run_branch.sh`: Also was upgraded to show the entire progress period start datetime, end datetime and duration.
 
 <br/><br/>
@@ -2253,16 +2309,16 @@ Fixes were put in place for a couple of new logging behaviors.
    - `utils/`
       - `fim_enums.py`:
          - A new class called `FIM_system_exit_codes` was added. This allows tracking and blocking of duplicate system exit codes when a custom system code is required.
-        
+
 
 ### Changes
 
-- `fim_run.sh`: Added the gms `non-zero-exit-code` system to `fim_run` to help uncover and isolate errors during processing. Errors recorded in log files within in the logs/unit folder are now copied into a new folder called `unit_errors`.  
-    
+- `fim_run.sh`: Added the gms `non-zero-exit-code` system to `fim_run` to help uncover and isolate errors during processing. Errors recorded in log files within in the logs/unit folder are now copied into a new folder called `unit_errors`.
+
 - `gms_run_branch.sh`:
     -  Minor adjustments to how the `non-zero-exit code` logs were created. Testing uncovered that previous versions were not always reliable. This is now stablized and enhanced.
     - In previous versions, only the `gms_unit.sh` was aware that **stream order filtering** was being done. Now all branch processing is also aware that filtering is in place. Processing in child files and classes can now make adjustments as/if required for stream order filtering.
-    - Small output adjustments were made to help with overall screen and log readability.  
+    - Small output adjustments were made to help with overall screen and log readability.
 
 - `gms_run_unit.sh`:
     - Minor adjustments to how the `non-zero-exit-code` logs were created similar to `gms_run_branch.sh.`
@@ -2281,12 +2337,12 @@ Fixes were put in place for a couple of new logging behaviors.
       - Some minor formatting and readability adjustments were added.
       - Additions were made to recognize the same type of errors as being described in other files related to stream order filtering issues.
       - A correction was made to be more precise and more explicit when a gms branch error existed. This was done to ensure that we were not letting other exceptions be trapped that were NOT related to stream flow filtering.
-      
+
    - `time_and_tee_run_by_unit.sh`:
       - The new custom system exit codes was added. Note that the values of 61 (responding system code) are hardcoded instead of using the python based `Fim_system_exit_code` system. This is related to limited communication between python and bash.
 
    - `gms/`
-      - `derive_level_paths.py`:  
+      - `derive_level_paths.py`:
           - Was upgraded to use the new fim_enums.Fim_system_exit_codes system. This occurs when no streams / flows remain after filtering.  Without this upgrade, standard exceptions were being issued with minimal details for the error.
           - Minor adjustments to formatting for readability were made.
 
@@ -2354,9 +2410,9 @@ This PR ports the functionality of `usgs_gage_crosswalk.py` and `rating_curve_co
 - `gms_run_branch.sh`: Added a line to aggregate all `usgs_elev_table.csv` into the HUC directory level using `src/usgs_gage_aggregate.py`.
 - `src/`:
     -  `gms/`
-        - `run_by_branch.sh`: Added a block to run `src/usgs_gage_crosswalk.py`. 
+        - `run_by_branch.sh`: Added a block to run `src/usgs_gage_crosswalk.py`.
         - `run_by_unit.sh`: Added a block to run `src/usgs_gage_unit_setup.py`.
-    - `usgs_gage_crosswalk.py`: Similar to it's functionality in FIM 3, this module snaps USGS gages to the stream network, samples the underlying DEMs, and writes the attributes to `usgs_elev_table.csv`. This CSV is later aggregated to the HUC level and eventually used in `tools/rating_curve_comparison.py`. Addresses #539 
+    - `usgs_gage_crosswalk.py`: Similar to it's functionality in FIM 3, this module snaps USGS gages to the stream network, samples the underlying DEMs, and writes the attributes to `usgs_elev_table.csv`. This CSV is later aggregated to the HUC level and eventually used in `tools/rating_curve_comparison.py`. Addresses #539
 - `tools/rating_curve_comparison.py`: Updated Sierra Test to work with FIM 4 data structure.
 - `unit_tests/`:
     - `rating_curve_comparison_unittests.py` & `rating_curve_comparison_params.json`: Unit test code and parameters for the Sierra Test.
@@ -2369,7 +2425,7 @@ This PR ports the functionality of `usgs_gage_crosswalk.py` and `rating_curve_co
 
 ## v4.0.2.0 - 2022-03-02 - [PR #548](https://github.com/NOAA-OWP/inundation-mapping/pull/548)
 
-Added a new optional system which allows an argument to be added to the `gms_run_unit.sh` command line to filter out stream orders 1 and 2 when calculating branches. 
+Added a new optional system which allows an argument to be added to the `gms_run_unit.sh` command line to filter out stream orders 1 and 2 when calculating branches.
 
 ### Changes
 
@@ -2377,14 +2433,14 @@ Added a new optional system which allows an argument to be added to the `gms_run
 
 - `src/gms`
    - `run_by_unit.sh`: Capture and forward the drop stream orders flag to `derive_level_paths.py`
-	
+
    - `derive_level_paths.py`: Capture the drop stream order flag and working with `stream_branches.py` to include/not include loading nwm stream with stream orders 1 and 2.
-	
+
    - `stream_branchs.py`: A correction was put in place to allow for the filter of branch attributes and values to be excluded. The `from_file` method has the functionality but was incomplete. This was corrected and how could accept the values from `derive_level_paths.py` to use the branch attribute of "order_" (gkpg field) and values excluded of [1,2] when optionally desired.
 
 - `unit_tests/gms`
     - `derive_level_paths_unittests.py` and `derive_level_paths_params.py`: Updated for testing for the new "drop stream orders 1 and 2" feature. Upgrades were also made to earlier existing incomplete test methods to test more output conditions.
-	
+
 <br/><br/>
 
 ## v4.0.1.0 - 2022-02-02 - [PR #525](https://github.com/NOAA-OWP/cahaba/pull/525)
@@ -2403,7 +2459,7 @@ The addition of a very simple and evolving unit test system which has two unit t
 
    - `unit_tests_utils.py`: A python file where methods that are common to all unit tests can be placed.
 
-   - `gms/derive_level_paths_params.json`: A set of default "happy path" values that are expected to pass validation for the derive_level_paths_params.py -> Derive_level_paths (function). 
+   - `gms/derive_level_paths_params.json`: A set of default "happy path" values that are expected to pass validation for the derive_level_paths_params.py -> Derive_level_paths (function).
 
    - `gms/derive_level_paths_unittests.py`: A unit test file for `src/derive_level_paths.py`. Incomplete but evolving.
 
@@ -3309,7 +3365,7 @@ Preprocess MS and FR stream networks
  - Headwater stream segments geometries are adjusted to align with with NWM streams.
  - Incoming streams are selected using intersection points between NWM streams and HUC4 boundaries.
  - `clip_vectors_to_wbd.py` handles local headwaters.
- - Removes NHDPlus features categorized as coastline and underground conduit.  
+ - Removes NHDPlus features categorized as coastline and underground conduit.
  - Added streams layer to production whitelist.
  - Fixed progress bar in `lib/acquire_and_preprocess_inputs.py`.
  - Added `getDriver` to shared `functions.py`.
diff --git a/docs/SECURITY.md b/docs/SECURITY.md
index 0cdd1d99d..10d537362 100644
--- a/docs/SECURITY.md
+++ b/docs/SECURITY.md
@@ -1,6 +1,6 @@
 # Security Policy
 
-There MUST be no unpatched vulnerabilities of medium or higher severity that have been publicly known for more than 60 days. 
+There MUST be no unpatched vulnerabilities of medium or higher severity that have been publicly known for more than 60 days.
 
 The vulnerability must be patched and released by the project itself (patches may be developed elsewhere). A vulnerability becomes publicly known (for this purpose) once it has a CVE with publicly released non-paywalled information (reported, for example, in the <a href="https://nvd.nist.gov/">National Vulnerability Database</a>) or when the project has been informed and the information has been released to the public (possibly by the project). A vulnerability is considered medium or higher severity if its <a href="https://www.first.org/cvss/" >Common Vulnerability Scoring System (CVSS)</a> base qualitative score is medium or higher. In CVSS versions 2.0 through 3.1, this is equivalent to a CVSS score of 4.0 or higher. Projects may use the CVSS score as published in a widely-used vulnerability database (such as the <a href="https://nvd.nist.gov">National Vulnerability Database</a>) using the most-recent version of CVSS reported in that database. Projects may instead calculate the severity themselves using the latest version of <a href="https://www.first.org/cvss/">CVSS</a> at the time of the vulnerability disclosure, if the calculation inputs are publicly revealed once the vulnerability is publicly known. <strong>Note</strong>: this means that users might be left vulnerable to all attackers worldwide for up to 60 days. This criterion is often much easier to meet than what Google recommends in <a href="https://security.googleblog.com/2010/07/rebooting-responsible-disclosure-focus.html">Rebooting responsible disclosure</a>, because Google recommends that the 60-day period start when the project is notified _even_ if the report is not public. Also note that this badge criterion, like other criteria, applies to the individual project. Some projects are part of larger umbrella organizations or larger projects, possibly in multiple layers, and many projects feed their results to other organizations and projects as part of a potentially-complex supply chain. An individual project often cannot control the rest, but an individual project can work to release a vulnerability patch in a timely way. Therefore, we focus solely on the individual project's response time.  Once a patch is available from the individual project, others can determine how to deal with the patch (e.g., they can update to the newer version or they can apply just the patch as a cherry-picked solution).
 
@@ -11,12 +11,11 @@ The public repositories MUST NOT leak any valid private credential (e.g., a work
 Use this section to tell people about which versions of your project are
 currently being supported with security updates.
 
-| Version | Supported          |
-| ------- | ------------------ |
-| 5.1.x   | :white_check_mark: |
-| 5.0.x   | :x:                |
-| 4.0.x   | :white_check_mark: |
-| < 4.0   | :x:                |
+| Version   | Supported          |
+| -------   | ------------------ |
+| > 4.3.15  | :white_check_mark: |
+| 4.3.14.2  | :x:                |
+| < 4.0     | :x:                |
 
 ## Reporting a Vulnerability
 
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 87768b482..8523d21ef 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -1,35 +1,91 @@
 #!/bin/bash -e
 
-: '
-fim_pipeline.sh -u <huc8> -n <name_your_run>
+:
+usage()
+{
+    echo "
+    Processing of HUC's in FIM4 comes in three sections. You can run 'fim_pipeline.sh' which will run
+        the three main scripts: 'fim_pre_processing.sh', 'fim_process_unit_wb.sh' & 'fim_post_processing.sh'.
+
+    Usage : fim_pipeline.sh -u <huc8> -n <name_of_your_run>
+
+    All arguments to this script are passed to 'fim_pre_processing.sh'.
+    REQUIRED:
+      -u/--hucList      : HUC8s to run; more than one HUC8 should be passed in quotes (space delimited).
+                            A line delimited file, with a .lst extension, is also acceptable.
+                            HUC8s must be present in inputs directory.
+      -n/--runName      : A name to tag the output directories and log files (only alphanumeric).
+
+    OPTIONS:
+      -h/--help         : Print usage statement.
+      -c/--config       : Configuration file with bash environment variables to export
+                        - Default: config/params_template.env
+      -ud/--unitDenylist
+                        A file with a line delimited list of files in UNIT (HUC) directories to be
+                            removed upon completion.
+                        - Default: config/deny_unit.lst
+                        - Note: if you want to keep all output files (aka.. no files removed),
+                            use the word NONE as this value for this parameter.
+      -bd/--branchDenylist
+                        A file with a line delimited list of files in BRANCHES directories to be
+                            removed upon completion of branch processing.
+                        - Default: config/deny_branches.lst
+                        - Note: if you want to keep all output files (aka.. no files removed),
+                            use the word NONE as this value for this parameter.
+      -zd/--branchZeroDenylist
+                        A file with a line delimited list of files in BRANCH ZERO directories to
+                            be removed upon completion of branch zero processing.
+                        - Default: config/deny_branch_zero.lst
+                        - Note: If you want to keep all output files (aka.. no files removed),
+                            use the word NONE as this value for this parameter.
+      -jh/--jobLimit    : Max number of concurrent HUC jobs to run. Default 1 job at time.
+      -jb/--jobBranchLimit
+                        Max number of concurrent Branch jobs to run. Default 1 job at time.
+                        - Note: Make sure that the product of jh and jb plus 2 (jh x jb + 2)
+                            does not exceed the total number of cores available.
+      -o                : Overwrite outputs if they already exist.
+      -skipcal          : If this param is included, the S.R.C. will be updated via the calibration points.
+                            will be skipped.
+      -isaws            : If this param is included, AWS objects will be used where possible
+                        - Note: This feature is not yet implemented.
+
+
+    Running 'fim_pipeline.sh' is a quicker process than running all three scripts independently; however,
+        you can run them independently if you like. The three sections are:
+
+            - 'fim_pre_processing.sh' : This section must be run first as it creates the basic output folder
+                for the run. Key files and folders for the next two sections are also created.
+
+            - 'fim_process_unit_wb.sh' : This script processes one and exactly one HUC8 plus all of its
+                related branches. While it can only process one, you can run this script multiple times,
+                each with different HUC (or overwriting a HUC). When you run 'fim_pipeline.sh',
+                when more than one HUC is provided, this script is iterated over, and parallelized.
+                For each HUC provided, 'fim_pipeline.sh' will call 'fim_process_unit_wb.sh'.
+                Using the 'fim_process_unit_wb.sh' script allows for a run / rerun of a HUC, or running other
+                HUCs at different times / days or even in different docker containers.
+
+            - 'fim_post_processing.sh' : This section takes all of the HUCs that have been processed,
+                aggregates key information from each HUC directory and looks for errors across all HUC
+                folders. It also processes the group in sub-steps such as usgs guages processesing,
+                rating curve adjustments and more. Naturally, running or re-running this script can only
+                be done after running 'fim_pre_processing.sh' and at least one run of 'fim_process_unit_wb.sh'.
+
+    "
+    exit
+}
 
-For more details on 
-
-- There are a wide number of options and defaulted values, for details run ```fim_pipeline.sh -h```
-- Manditory arguments:
-    - `-u` can be a single huc, a series passed in quotes space delimited, or a line-delimited file
-    i. To run entire domain of available data use the ```/data/inputs/included_huc8.lst``` file or a huc list file of your choice.
-    - `-n` is a name of your run (only alphanumeric)
-- Outputs can be found under ```/outputs/<name_your_run>```
-
-Processing of HUC''s in FIM4 comes in three pieces. You can run `fim_pipeline.sh` which automatically runs all of three major section, but you can run each of the sections independently if you like. The three sections are:
-- `fim_pre_processing.sh` : This section must be run first as it creates the basic output folder for the run. It also creates a number of key files and folders for the next two sections. 
-
-- `fim_process_unit_wb.sh` : This script processes one and exactly one HUC8 plus all of it''s related branches. While it can only process one, you can run this script multiple times, each with different HUC (or overwriting a HUC). When you run `fim_pipeline.sh`, it automatically iterates when more than one HUC number has been supplied either by command line arguments or via a HUC list. For each HUC provided, `fim_pipeline.sh` will `fim_process_unit_wb.sh`. Using the `fim_process_unit_wb.sh`  script allows for a run / rerun of a HUC, or running other HUCs at different times / days or even different docker containers.
-
-- `fim_post_processing.sh` : This section takes all of the HUCs that have been processed, aggregates key information from each HUC directory and looks for errors across all HUC folders. It also processes the group in sub-steps such as usgs guages processesing, rating curve adjustments and more. Naturally, running or re-running this script can only be done after running `fim_pre_processing.sh` and at least one run of `fim_process_unit_wb.sh`.
-
-Running the `fim_pipeline.sh` is a quicker process than running all three steps independently.
-'
 
 set -e
 
-# See fim_pre_processing.sh for details of how to use this script. fim_pre_processing.sh
-# is a proxy for collecting and validating input.
+# print usage if agrument is '-h' or '--help'
+if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+    usage
+fi
+
 
 echo
 echo "======================= Start of fim_pipeline.sh ========================="
-echo "---- Started: `date -u`" 
+echo "---- Started: `date -u`"
 
 ## LOAD AND VALIDATE INCOMING ARGUMENTS
 source $srcDir/bash_functions.env
@@ -45,11 +101,11 @@ pipeline_start_time=`date +%s`
 # Why an if and else? watch the number of colons
 if [ -f "$hucList" ]; then
     if [ "$jobHucLimit" = "1" ]; then
-        parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList 
+        parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList
     else
         parallel --eta -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList
     fi
-else 
+else
     if [ "$jobHucLimit" = "1" ]; then
         parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName ::: $hucList
     else
@@ -67,7 +123,7 @@ Calc_Duration $pipeline_start_time
 # Remove run from the fim_temp directory
 rm -d $workDir/$runName
 
-# Pipe into post processing 
+# Pipe into post processing
 . $projectDir/fim_post_processing.sh -n $runName -j $jobMaxLimit
 
 echo
@@ -76,3 +132,5 @@ date -u
 Calc_Duration $pipeline_start_time
 echo
 
+# Exit the script
+exit 0
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index ceacfb0db..f8266c94d 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -1,22 +1,23 @@
 #!/bin/bash -e
 
 :
-usage ()
+usage()
 {
-    echo 'Post processing for creating FIM hydrofabric.'
-    echo 'Usage : fim_post_processing.sh [REQ: -n <run name> ]'
-    echo '  	 				         [OPT: -h -j <job limit>]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -n/--runName    : A name to tag the output directories and log files.'
-    echo ''
-    echo 'OPTIONS:'
-    echo '  -h/--help       : help file'
-    echo '  -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs'
-    echo '                    stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest'
-    echo '                    Note: Not the same variable name as fim_pipeline or fim_pre_processing'
-    echo '                    and can be the multiplication of jobHucLimit and jobBranchLimit'
-    echo
+    echo "
+    Post processing for creating FIM hydrofabric.
+
+    Usage : fim_post_processing.sh [REQ: -n <run name> ] [OPT: -h -j <job limit>]
+
+    REQUIRED:
+       -n/--runName    : A name to tag the output directories and log files.
+
+    OPTIONS:
+       -h/--help       : help file
+       -j/--jobLimit   : max number of concurrent jobs to run. Default 1 job at time. 1 outputs
+                         stdout and stderr to terminal and logs. With >1 outputs progress and logs the rest.
+                         Note: Not the same variable name as fim_pipeline or fim_pre_processing
+                         and can be the multiplication of jobHucLimit and jobBranchLimit
+    "
     exit
 }
 
@@ -50,8 +51,9 @@ fi
 outputDestDir=$outputsDir/$runName
 
 ## Check for output destination directory ##
-if [ ! -d "$outputDestDir" ]; then 
-    echo "Depends on output from units and branches. Please provide an output folder name that has hucs/branches run."
+if [ ! -d "$outputDestDir" ]; then
+    echo "Depends on output from units and branches. "
+    echo "Please provide an output folder name that has hucs/branches run."
     exit 1
 fi
 
@@ -83,7 +85,7 @@ source $srcDir/bash_variables.env
 echo
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 echo "---- Start of fim_post_processing"
-echo "---- Started: `date -u`" 
+echo "---- Started: `date -u`"
 T_total_start
 post_proc_start_time=`date +%s`
 
@@ -94,10 +96,12 @@ python3 $srcDir/aggregate_branch_lists.py -d $outputDestDir -f "branch_ids.csv"
 
 ## GET NON ZERO EXIT CODES FOR BRANCHES ##
 echo -e $startDiv"Start non-zero exit code checking"
-find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputDestDir/branch_errors/non_zero_exit_codes.log" &
+find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | \
+    xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > \
+    "$outputDestDir/branch_errors/non_zero_exit_codes.log" &
 
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
-echo "Processing usgs & ras2fim elev table aggregation"   
+echo "Processing usgs & ras2fim elev table aggregation"
 python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev -ras -j $jobLimit
 
 ## RUN BATHYMETRY ADJUSTMENT ROUTINE ##
@@ -105,7 +109,12 @@ if [ "$bathymetry_adjust" = "True" ]; then
     echo -e $startDiv"Performing Bathymetry Adjustment routine"
     # Run bathymetry adjustment routine
     Tstart
-    python3 $srcDir/bathymetric_adjustment.py -fim_dir $outputDestDir -bathy $bathymetry_file -buffer $wbd_buffer -wbd $inputsDir/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -j $jobLimit
+    python3 $srcDir/bathymetric_adjustment.py \
+        -fim_dir $outputDestDir \
+        -bathy $bathymetry_file \
+        -buffer $wbd_buffer \
+        -wbd $inputsDir/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg \
+        -j $jobLimit
     Tcount
 fi
 
@@ -114,7 +123,10 @@ if [ "$src_bankfull_toggle" = "True" ]; then
     echo -e $startDiv"Estimating bankfull stage in SRCs"
     # Run SRC bankfull estimation routine routine
     Tstart
-    python3 $srcDir/identify_src_bankfull.py -fim_dir $outputDestDir -flows $bankfull_flows_file -j $jobLimit
+    python3 $srcDir/identify_src_bankfull.py \
+        -fim_dir $outputDestDir \
+        -flows $bankfull_flows_file \
+        -j $jobLimit
     Tcount
 fi
 
@@ -123,17 +135,24 @@ if [ "$src_subdiv_toggle" = "True" ] && [ "$src_bankfull_toggle" = "True" ]; the
     echo -e $startDiv"Performing SRC channel/overbank subdivision routine"
     # Run SRC Subdivision & Variable Roughness routine
     Tstart
-    python3 $srcDir/subdiv_chan_obank_src.py -fim_dir $outputDestDir -mann $vmann_input_file -j $jobLimit
+    python3 $srcDir/subdiv_chan_obank_src.py \
+        -fim_dir $outputDestDir \
+        -mann $vmann_input_file \
+        -j $jobLimit
     Tcount
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
 if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$skipcal" = "0" ]; then
     Tstart
-    echo    
+    echo
     echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
     # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow values)
-    python3 $srcDir/src_adjust_usgs_rating.py -run_dir $outputDestDir -usgs_rc $usgs_rating_curve_csv -nwm_recur $nwm_recur_file -j $jobLimit
+    python3 $srcDir/src_adjust_usgs_rating.py \
+        -run_dir $outputDestDir \
+        -usgs_rc $usgs_rating_curve_csv \
+        -nwm_recur $nwm_recur_file \
+        -j $jobLimit
     Tcount
     date -u
 fi
@@ -141,10 +160,14 @@ fi
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ RAS2FIM CROSS SECTION RATING CURVES ##
 if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$skipcal" = "0" ]; then
     Tstart
-    echo    
+    echo
     echo -e $startDiv"Performing SRC adjustments using ras2fim rating curve database"
     # Run SRC Optimization routine using ras2fim rating curve data (WSE and flow @ NWM recur flow values)
-    python3 $srcDir/src_adjust_ras2fim_rating.py -run_dir $outputDestDir -ras_rc $ras_rating_curve_csv -nwm_recur $nwm_recur_file -j $jobLimit
+    python3 $srcDir/src_adjust_ras2fim_rating.py \
+        -run_dir $outputDestDir \
+        -ras_rc $ras_rating_curve_csv \
+        -nwm_recur $nwm_recur_file \
+        -j $jobLimit
     Tcount
     date -u
 fi
@@ -160,10 +183,14 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [
 fi
 
 ## AGGREGATE BRANCH TABLES ##
-echo 
+echo
 echo -e $startDiv"Aggregating branch hydrotables"
 Tstart
-python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -htable -j $jobLimit
+python3 $srcDir/aggregate_by_huc.py \
+    -fim $outputDestDir \
+    -i $fim_inputs \
+    -htable \
+    -j $jobLimit
 Tcount
 date -u
 
@@ -172,7 +199,9 @@ if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
     echo
     echo -e $startDiv"Performing manual calibration"
     Tstart
-    python3 $srcDir/src_manual_calibration.py -fim_dir $outputDestDir -calb_file $man_calb_file
+    python3 $srcDir/src_manual_calibration.py \
+        -fim_dir $outputDestDir \
+        -calb_file $man_calb_file
     Tcount
     date -u
 fi
@@ -181,7 +210,9 @@ echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
 Tstart
-python3 /foss_fim/tools/combine_crosswalk_tables.py -d $outputDestDir -o $outputDestDir/crosswalk_table.csv
+python3 /foss_fim/tools/combine_crosswalk_tables.py \
+    -d $outputDestDir \
+    -o $outputDestDir/crosswalk_table.csv
 Tcount
 date -u
 
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 29c2c42e1..61eb3ef1a 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -1,58 +1,53 @@
 #!/bin/bash -e
+
 :
-usage ()
+usage()
 {
-    echo
-    echo 'Produce FIM hydrofabric datasets for unit and branch scale.'
-    echo 'Usage : [REQ: -u <hucs> -n <run name> ]'
-    echo '        [OPT: -h -c <config file> -o'
-    echo '          -ud <unit deny list file>'
-    echo '          -bd <branch deny list file>'
-    echo '          -zd <branch zero deny list file>'
-    echo '          -jh <number of jobs for HUCs>'
-    echo '          -jb <number of jobs for branches>'
-    echo '          -skipcal <skip the calibration points process>'
-    echo '          -isaws ]'
-    echo ''
-    echo 'REQUIRED:'
-    echo '  -u/--hucList    : HUC8s to run; more than one HUC8 should be passed in quotes (space delimited).'
-    echo '                    A line delimited file, with a .lst extension, is also acceptable.'
-    echo '                    HUC8s must present in inputs directory.'
-    echo '  -n/--runName    : A name to tag the output directories and log files.'
-    echo 
-    echo 'OPTIONS:'
-    echo '  -h/--help       : Help file'
-    echo '  -c/--config     : Configuration file with bash environment variables to export'
-    echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
-    echo '  -ud/--unitDenylist'
-    echo '                    A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_unit.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_unit.lst'
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'
-    echo '  -bd/--branchDenylist'
-    echo '                    A file with a line delimited list of files in BRANCHES directories to be removed' 
-    echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_branches.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_branches.lst'   
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'
-    echo '  -zd/--branchZeroDenylist'
-    echo '                    A file with a line delimited list of files in BRANCH ZERO directories to' 
-    echo '                    be removed upon completion of branch zero processing.'
-    echo '                    (see config/deny_branch_zero.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_branch_zero.lst'   
-    echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
-    echo '                    use the word NONE as this value for this parameter.'    
-    echo '  -jh/--jobLimit   : Max number of concurrent HUC jobs to run. Default 1 job at time.'
-    echo '  -jb/--jobBranchLimit : Max number of concurrent Branch jobs to run. Default 1 job at time.' 
-    echo '                         NOTE: Make sure that the multiplication of jh and jb plus 2 (jh x jb + 2) does not'
-    echo '                         exceed the total number of cores available.'
-    echo '  -o               : Overwrite outputs if already exist'
-    echo '  -skipcal         : If this param is included, updating the S.R.C. via the calibration points will be skipped'    
-    echo '  -isaws           : If this param is included, the code will use AWS objects where possible'
-    echo '                   : Note: This feature is not yet implemented'
-    echo
+    echo "
+    This script collects & validates inputs, creates files & folders,
+        as well as loads/sets environment & configuration variables.
+
+    Usage : fim_pre_processing.sh [REQ: -u <hucs> -n <run name> ] [OPT: see below for OPTIONS]
+
+    REQUIRED:
+      -u/--hucList      : HUC8s to run; more than one HUC8 should be passed in quotes (space delimited).
+                            A line delimited file, with a .lst extension, is also acceptable.
+                            HUC8s must be present in inputs directory.
+      -n/--runName      : A name to tag the output directories and log files.
+
+    OPTIONS:
+      -h/--help         : Print usage statement.
+      -c/--config       : Configuration file with bash environment variables to export
+                        - Default: config/params_template.env
+      -ud/--unitDenylist
+                        A file with a line delimited list of files in UNIT (HUC) directories to be
+                            removed upon completion.
+                        - Default: config/deny_unit.lst
+                        - Note: if you want to keep all output files (aka.. no files removed),
+                            use the word NONE as this value for this parameter.
+      -bd/--branchDenylist
+                        A file with a line delimited list of files in BRANCHES directories to be
+                            removed upon completion of branch processing.
+                        - Default: config/deny_branches.lst
+                        - Note: if you want to keep all output files (aka.. no files removed),
+                            use the word NONE as this value for this parameter.
+      -zd/--branchZeroDenylist
+                        A file with a line delimited list of files in BRANCH ZERO directories to
+                            be removed upon completion of branch zero processing.
+                        - Default: config/deny_branch_zero.lst
+                        - Note: If you want to keep all output files (aka.. no files removed),
+                        use the word NONE as this value for this parameter.
+      -jh/--jobLimit    : Max number of concurrent HUC jobs to run. Default 1 job at time.
+      -jb/--jobBranchLimit
+                        Max number of concurrent Branch jobs to run. Default 1 job at time.
+                        - Note: Make sure that the product of jh and jb plus 2 (jh x jb + 2)
+                            does not exceed the total number of cores available.
+      -o                : Overwrite outputs if they already exist.
+      -skipcal          : If this param is included, the S.R.C. will be updated via the calibration points.
+                            will be skipped.
+      -isaws            : If this param is included, AWS objects will be used where possible
+                        - Note: This feature is not yet implemented.
+    "
     exit
 }
 
@@ -80,7 +75,7 @@ in
     -jb|--jobBranchLimit)
         shift
         jobBranchLimit=$1
-        ;;        
+        ;;
     -h|--help)
         shift
         usage
@@ -96,7 +91,7 @@ in
     -zd|--branchZeroDenylist)
         shift
         deny_branch_zero_list=$1
-        ;;        
+        ;;
     -o)
         overwrite=1
         ;;
@@ -173,7 +168,7 @@ then
         usage
     else
         # only if the deny branch zero has been overwritten and file exists
-        has_deny_branch_zero_override=1 
+        has_deny_branch_zero_override=1
     fi
 else
     has_deny_branch_zero_override=1 # it is the value of NONE and is overridden
@@ -183,7 +178,7 @@ fi
 if [ -d $outputDestDir ] && [ $overwrite -eq 0 ]; then
     echo
     echo "ERROR: Output dir $outputDestDir exists. Use overwrite -o to run."
-    echo        
+    echo
     usage
 fi
 
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
index 2144337a1..0d164543d 100755
--- a/fim_process_unit_wb.sh
+++ b/fim_process_unit_wb.sh
@@ -1,41 +1,44 @@
 #!/bin/bash -e
 
-# Why is this file here and it appears to be using duplicate export variables?
-# For AWS, we need to make a direct call to this files with two params, hucNumber first, 
-# then the runName same as the -n flag in fim_pipeline and fim_pre_processing.sh
+:
+usage ()
+{
+    echo "
+    Why is this file here and it appears to be using duplicate export variables?
+    For portability, we can make a direct call to this file with two parameters: HUC Number & Run Name;
+    which correspond to the -n argument in fim_pipeline.sh and fim_pre_processing.sh.
 
-# This file will also catch any and all errors from src/run_unit_wb.sh file, even script aborts from that file
+    This file will catch any and all errors from src/run_unit_wb.sh, even if that script aborts.
 
-# You really can not call directly to src/run_unit_wb.sh as that file relies on export values from this file.
-# run_unit_wb will futher process branches with its own iterator (parallelization).
+    It is not possible to call src/run_unit_wb.sh directly, as it relies on exported values from this file.
+        src/run_unit_wb.sh will futher process branches (src/process_branch.sh) in parallel.
 
-# Sample Usage: /foss_fim/fim_process_unit_wb.sh rob_test_wb_1 05030104
+    Usage: ./fim_process_unit_wb.sh <name_of_your_run> <huc8>
 
-## START MESSAGE ##
+    Produce FIM hydrofabric datasets for a single unit and branch scale.
+    - Note: fim_pre_processing.sh must have been already run. This script does
+        not include post processing (see fim_pipeline.sh).
+        Only a single HUC and its branches will be processed.
 
-echo
+    Arguments:
+        1) run name
+        2) HUC number
+            Example:
 
-usage ()
-{
-    echo
-    echo 'Produce FIM hydrofabric datasets for a single unit and branch scale.'
-    echo 'NOTE: fim_pre_processing must have been already run and this tool'
-    echo '      will not include post processing. Only single independent single'
-    echo '      huc and its branches.'    
-    echo 'Usage : There are no arg keys (aka.. no dashes)'
-    echo '        you need the run name first, then the huc.'
-    echo '        Arguments:'
-    echo '           1) run name'
-    echo '           2) HUC number'
-    echo '        Example:'
-    echo '           /foss_fim/fim_process_unit_wb.sh rob_test_1 05030104'
-    echo
+                ./fim_process_unit_wb.sh test_name 05030104
+    "
     exit
 }
 
+# print usage if agrument is '-h' or '--help'
+if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+    usage
+fi
+
 export runName=$1
 export hucNumber=$2
 
+
 # print usage if arguments empty
 if [ "$runName" = "" ]
 then
@@ -99,7 +102,7 @@ for code in "${return_codes[@]}"
 do
     # Make an extra copy of the unit log into a new folder.
 
-    # Note: It was tricky to load in the fim_enum into bash, so we will just 
+    # Note: It was tricky to load in the fim_enum into bash, so we will just
     # go with the exit code for now
     if [ $code -eq 0 ]; then
         echo
@@ -110,12 +113,12 @@ do
         err_exists=1
     elif [ $code -eq 61 ]; then
         echo
-        echo "***** Unit has no remaining valid flowlines *****"   
-        err_exists=1        
+        echo "***** Unit has no remaining valid flowlines *****"
+        err_exists=1
     else
         echo
         echo "***** An error has occured  *****"
-        err_exists=1        
+        err_exists=1
     fi
 done
 
@@ -129,7 +132,7 @@ mv -f $tempHucDataDir $outputHucDataDir
 find $outputHucDataDir -type d -exec chmod 777 {} +
 
 echo "============================================================================================="
-echo 
+echo
 echo "***** Moved temp directory: $tempHucDataDir to output directory: $outputHucDataDir  *****"
 echo
 echo "============================================================================================="
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..7613ddd00
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,88 @@
+[project]
+name = 'Inundation Mapping'
+
+description = """\
+    Flood inundation mapping software configured to work with the U.S. National Water Model, \
+    operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) \
+    National Water Center (NWC). \
+    This software uses the Height Above Nearest Drainage (HAND) method to generate \
+    Relative Elevation Models (REMs), Synthetic Rating Curves (SRCs), and catchment grids. \
+    This repository also includes functionality to generate flood inundation maps (FIMs) \
+    and evaluate FIM accuracy. \
+    """
+
+maintainers = [
+    {name = "Fernando Salas", email = "fernando.salas@noaa.gov"},
+    {name = "Carson Pruitt", email = "carson.pruitt@noaa.gov"},
+    {name = "Ali Forghani", email = "ali.forghani@noaa.gov"},
+    {name = "Bradford Bates", email = "bradford.bates@noaa.gov"},
+    {name = "Emily Deardorff", email = "emily.deardorff@noaa.gov"},
+    {name = "Fernando Aristizabal", email = "fernando.aristizabal@noaa.gov"},
+    {name = "Greg Petrochenkov", email = "greg.petrochenkov@noaa.gov"},
+    {name = "Hamideh Safa", email = "hamideh.safa@noaa.gov"},
+    {name = "James Coll", email = "james.coll@noaa.gov"},
+    {name = "Matt Luck", email = "matt.luck@noaa.gov"},
+    {name = "Laura Keys", email = "laura.keys@noaa.gov"},
+    {name = "Nick Chadwick", email = "nick.chadwick@noaa.gov"},
+    {name = "Riley McDermott", email = "riley.mcdermott@noaa.gov"},
+    {name = "Rob Gonzalez-Pita", email = "rob.g.pita@noaa.gov"},
+    {name = "Robert Hanna", email = "robert.hanna@noaa.gov"},
+    {name = "Ryan Spies", email = "ryan.spies@noaa.gov"}
+]
+
+requires-python = ">=3.8"
+keywords = ["flood inundation", "mapping"]
+license = {path = "./LICENSE"}
+version = "4.4.0"
+
+[project.urls]
+Source = "https://github.com/NOAA-OWP/inundation-mapping"
+Wiki = "https://github.com/NOAA-OWP/inundation-mapping/wiki"
+
+
+# - Tools -
+
+[tool.black]
+line_length = 110
+skip-string-normalization = true
+skip-magic-trailing-comma = true
+exclude = '''
+/(
+    \.git
+)\
+'''
+
+[tool.isort]
+profile = 'black'
+multi_line_output = 3
+line_length = 110 # It should be the same as `tool.black.line-length` above
+lines_after_imports = 2
+skip_gitignore = true
+
+[tool.flake8]
+count = true
+doctests = true
+max-complexity = 56
+max-line-length = 110  # It should be the same as `tool.black.line-length` above
+extend-ignore = """
+    E203,
+    E266,
+    E501,
+    W503,
+    W391
+    F403,
+    F401,
+    """
+per-file-ignores = """
+    src/subdiv_chan_obank_src.py: E712
+    src/src_roughness_optimization.py: E712, F841
+    src/agreedem.py: E712
+    src/build_stream_traversal.py: E722
+    tools/vary_mannings_n_composite.py: E712
+    tools/tools_shared_functions.py: F821, F841
+    tools/rating_curve_get_usgs_curves.py: F841
+    tools/rating_curve_comparison.py: F821, F841
+    tools/inundation.py: F821
+    tools/eval_alt_catfim.py: F841
+    tools/check_deep_flooding.py: E712
+    """
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 21536bfe2..e326f225a 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -1,75 +1,92 @@
 #!/usr/bin/env python3
 
+import argparse
+import json
 import os
+import sys
+
 import geopandas as gpd
 import pandas as pd
+from memory_profiler import profile
 from numpy import unique
 from rasterstats import zonal_stats
-import json
-import argparse
-import sys
-from utils.shared_functions import getDriver
+
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
-from memory_profiler import profile
 
-# Feb 17, 2023
-# We want to explore using FR methodology as branch zero
 
-@mem_profile
-def add_crosswalk(input_catchments_fileName,
-                  input_flows_fileName,
-                  input_srcbase_fileName,
-                  output_catchments_fileName,
-                  output_flows_fileName,
-                  output_src_fileName,
-                  output_src_json_fileName,
-                  output_crosswalk_fileName,
-                  output_hydro_table_fileName,
-                  input_huc_fileName,
-                  input_nwmflows_fileName,
-                  input_nwmcatras_fileName,
-                  mannings_n,
-                  input_nwmcat_fileName,
-                  extent,
-                  small_segments_filename,
-                  min_catchment_area,
-                  min_stream_length,
-                  calibration_mode=False):
+# TODO - Feb 17, 2023 - We want to explore using FR methodology as branch zero
+
 
+@mem_profile
+def add_crosswalk(
+    input_catchments_fileName,
+    input_flows_fileName,
+    input_srcbase_fileName,
+    output_catchments_fileName,
+    output_flows_fileName,
+    output_src_fileName,
+    output_src_json_fileName,
+    output_crosswalk_fileName,
+    output_hydro_table_fileName,
+    input_huc_fileName,
+    input_nwmflows_fileName,
+    input_nwmcatras_fileName,
+    mannings_n,
+    input_nwmcat_fileName,
+    extent,
+    small_segments_filename,
+    min_catchment_area,
+    min_stream_length,
+    calibration_mode=False,
+):
     input_catchments = gpd.read_file(input_catchments_fileName)
     input_flows = gpd.read_file(input_flows_fileName)
     input_huc = gpd.read_file(input_huc_fileName)
     input_nwmflows = gpd.read_file(input_nwmflows_fileName)
-    min_catchment_area = float(min_catchment_area) #0.25#
-    min_stream_length = float(min_stream_length) #0.5#
+    min_catchment_area = float(min_catchment_area)  # 0.25#
+    min_stream_length = float(min_stream_length)  # 0.5#
 
     if extent == 'FR':
         ## crosswalk using majority catchment method
 
         # calculate majority catchments
-        majority_calc = zonal_stats(input_catchments, input_nwmcatras_fileName, stats=['majority'], geojson_out=True)
+        majority_calc = zonal_stats(
+            input_catchments, input_nwmcatras_fileName, stats=['majority'], geojson_out=True
+        )
         input_majorities = gpd.GeoDataFrame.from_features(majority_calc)
         input_majorities = input_majorities.rename(columns={'majority' : 'feature_id'})
 
         input_majorities = input_majorities[:][input_majorities['feature_id'].notna()]
-        if input_majorities.feature_id.dtype != 'int': input_majorities.feature_id = input_majorities.feature_id.astype(int)
-        if input_majorities.HydroID.dtype != 'int': input_majorities.HydroID = input_majorities.HydroID.astype(int)
-
-        input_nwmflows = input_nwmflows.rename(columns={'ID':'feature_id'})
-        if input_nwmflows.feature_id.dtype != 'int': input_nwmflows.feature_id = input_nwmflows.feature_id.astype(int)
-        relevant_input_nwmflows = input_nwmflows[input_nwmflows['feature_id'].isin(input_majorities['feature_id'])]
-        relevant_input_nwmflows = relevant_input_nwmflows.filter(items=['feature_id','order_'])
-
-        if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
-        output_catchments = input_catchments.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
-        output_catchments = output_catchments.merge(relevant_input_nwmflows[['order_','feature_id']],on='feature_id')
-
-        if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
-        output_flows = input_flows.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
-        if output_flows.HydroID.dtype != 'int': output_flows.HydroID = output_flows.HydroID.astype(int)
-        output_flows = output_flows.merge(relevant_input_nwmflows[['order_','feature_id']],on='feature_id')
-        output_flows = output_flows.merge(output_catchments.filter(items=['HydroID','areasqkm']),on='HydroID')
+        if input_majorities.feature_id.dtype != 'int':
+            input_majorities.feature_id = input_majorities.feature_id.astype(int)
+        if input_majorities.HydroID.dtype != 'int':
+            input_majorities.HydroID = input_majorities.HydroID.astype(int)
+
+        input_nwmflows = input_nwmflows.rename(columns={'ID': 'feature_id'})
+        if input_nwmflows.feature_id.dtype != 'int':
+            input_nwmflows.feature_id = input_nwmflows.feature_id.astype(int)
+        relevant_input_nwmflows = input_nwmflows[
+            input_nwmflows['feature_id'].isin(input_majorities['feature_id'])
+        ]
+        relevant_input_nwmflows = relevant_input_nwmflows.filter(items=['feature_id', 'order_'])
+
+        if input_catchments.HydroID.dtype != 'int':
+            input_catchments.HydroID = input_catchments.HydroID.astype(int)
+        output_catchments = input_catchments.merge(input_majorities[['HydroID', 'feature_id']], on='HydroID')
+        output_catchments = output_catchments.merge(
+            relevant_input_nwmflows[['order_', 'feature_id']], on='feature_id'
+        )
+
+        if input_flows.HydroID.dtype != 'int':
+            input_flows.HydroID = input_flows.HydroID.astype(int)
+        output_flows = input_flows.merge(input_majorities[['HydroID', 'feature_id']], on='HydroID')
+        if output_flows.HydroID.dtype != 'int':
+            output_flows.HydroID = output_flows.HydroID.astype(int)
+        output_flows = output_flows.merge(relevant_input_nwmflows[['order_', 'feature_id']], on='feature_id')
+        output_flows = output_flows.merge(
+            output_catchments.filter(items=['HydroID', 'areasqkm']), on='HydroID'
+        )
 
     elif (extent == 'MS') | (extent == 'GMS'):
         ## crosswalk using stream segment midpoint method
@@ -77,75 +94,96 @@ def add_crosswalk(input_catchments_fileName,
 
         # only reduce nwm catchments to mainstems if running mainstems
         if extent == 'MS':
-            input_nwmcat = input_nwmcat.loc[input_nwmcat.mainstem==1]
+            input_nwmcat = input_nwmcat.loc[input_nwmcat.mainstem == 1]
 
-        input_nwmcat = input_nwmcat.rename(columns={'ID':'feature_id'})
-        if input_nwmcat.feature_id.dtype != 'int': input_nwmcat.feature_id = input_nwmcat.feature_id.astype(int)
-        input_nwmcat=input_nwmcat.set_index('feature_id')
+        input_nwmcat = input_nwmcat.rename(columns={'ID': 'feature_id'})
+        if input_nwmcat.feature_id.dtype != 'int':
+            input_nwmcat.feature_id = input_nwmcat.feature_id.astype(int)
+        input_nwmcat = input_nwmcat.set_index('feature_id')
 
-        input_nwmflows = input_nwmflows.rename(columns={'ID':'feature_id'})
-        if input_nwmflows.feature_id.dtype != 'int': input_nwmflows.feature_id = input_nwmflows.feature_id.astype(int)
+        input_nwmflows = input_nwmflows.rename(columns={'ID': 'feature_id'})
+        if input_nwmflows.feature_id.dtype != 'int':
+            input_nwmflows.feature_id = input_nwmflows.feature_id.astype(int)
 
         # Get stream midpoint
         stream_midpoint = []
         hydroID = []
-        for i,lineString in enumerate(input_flows.geometry):
-            hydroID = hydroID + [input_flows.loc[i,'HydroID']]
-            stream_midpoint = stream_midpoint + [lineString.interpolate(0.5,normalized=True)]
+        for i, lineString in enumerate(input_flows.geometry):
+            hydroID = hydroID + [input_flows.loc[i, 'HydroID']]
+            stream_midpoint = stream_midpoint + [lineString.interpolate(0.5, normalized=True)]
 
-        input_flows_midpoint = gpd.GeoDataFrame({'HydroID':hydroID, 'geometry':stream_midpoint}, crs=input_flows.crs, geometry='geometry')
+        input_flows_midpoint = gpd.GeoDataFrame(
+            {'HydroID': hydroID, 'geometry': stream_midpoint}, crs=input_flows.crs, geometry='geometry'
+        )
         input_flows_midpoint = input_flows_midpoint.set_index('HydroID')
 
         # Create crosswalk
-        crosswalk = gpd.sjoin(input_flows_midpoint, input_nwmcat, how='left', predicate='within').reset_index()
+        crosswalk = gpd.sjoin(
+            input_flows_midpoint, input_nwmcat, how='left', predicate='within'
+        ).reset_index()
         crosswalk = crosswalk.rename(columns={"index_right": "feature_id"})
 
         # fill in missing ms
         crosswalk_missing = crosswalk.loc[crosswalk.feature_id.isna()]
         for index, stream in crosswalk_missing.iterrows():
-
             # find closest nwm catchment by distance
             distances = [stream.geometry.distance(poly) for poly in input_nwmcat.geometry]
             min_dist = min(distances)
-            nwmcat_index=distances.index(min_dist)
+            nwmcat_index = distances.index(min_dist)
 
             # update crosswalk
-            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'feature_id'] = input_nwmcat.iloc[nwmcat_index].name
-            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'AreaSqKM'] = input_nwmcat.iloc[nwmcat_index].AreaSqKM
-            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'Shape_Length'] = input_nwmcat.iloc[nwmcat_index].Shape_Length
-            crosswalk.loc[crosswalk.HydroID==stream.HydroID,'Shape_Area'] = input_nwmcat.iloc[nwmcat_index].Shape_Area
+            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'feature_id'] = input_nwmcat.iloc[
+                nwmcat_index
+            ].name
+            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'AreaSqKM'] = input_nwmcat.iloc[
+                nwmcat_index
+            ].AreaSqKM
+            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'Shape_Length'] = input_nwmcat.iloc[
+                nwmcat_index
+            ].Shape_Length
+            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'Shape_Area'] = input_nwmcat.iloc[
+                nwmcat_index
+            ].Shape_Area
 
         crosswalk = crosswalk.filter(items=['HydroID', 'feature_id'])
-        crosswalk = crosswalk.merge(input_nwmflows[['feature_id','order_']],on='feature_id')
+        crosswalk = crosswalk.merge(input_nwmflows[['feature_id', 'order_']], on='feature_id')
 
         if len(crosswalk) < 1:
-            print ("No relevant streams within HUC boundaries.")
+            print("No relevant streams within HUC boundaries.")
             sys.exit(0)
 
-        if input_catchments.HydroID.dtype != 'int': input_catchments.HydroID = input_catchments.HydroID.astype(int)
-        output_catchments = input_catchments.merge(crosswalk,on='HydroID')
+        if input_catchments.HydroID.dtype != 'int':
+            input_catchments.HydroID = input_catchments.HydroID.astype(int)
+        output_catchments = input_catchments.merge(crosswalk, on='HydroID')
 
-        if input_flows.HydroID.dtype != 'int': input_flows.HydroID = input_flows.HydroID.astype(int)
-        output_flows = input_flows.merge(crosswalk,on='HydroID')
+        if input_flows.HydroID.dtype != 'int':
+            input_flows.HydroID = input_flows.HydroID.astype(int)
+        output_flows = input_flows.merge(crosswalk, on='HydroID')
 
         # added for GMS. Consider adding filter_catchments_and_add_attributes.py to run_by_branch.sh
         if 'areasqkm' not in output_catchments.columns:
-            output_catchments['areasqkm'] = output_catchments.geometry.area/(1000**2)
+            output_catchments['areasqkm'] = output_catchments.geometry.area / (1000**2)
 
-        output_flows = output_flows.merge(output_catchments.filter(items=['HydroID','areasqkm']),on='HydroID')
+        output_flows = output_flows.merge(
+            output_catchments.filter(items=['HydroID', 'areasqkm']), on='HydroID'
+        )
 
     output_flows['ManningN'] = mannings_n
 
-    if output_flows.NextDownID.dtype != 'int': output_flows.NextDownID = output_flows.NextDownID.astype(int)
+    if output_flows.NextDownID.dtype != 'int':
+        output_flows.NextDownID = output_flows.NextDownID.astype(int)
 
     # Adjust short model reach rating curves
-    print("Adjusting model reach rating curves")
+    print('Adjusting model reach rating curves')
     sml_segs = pd.DataFrame()
 
     # replace small segment geometry with neighboring stream
     for stream_index in output_flows.index:
-        if output_flows["areasqkm"][stream_index] < min_catchment_area and output_flows["LengthKm"][stream_index] < min_stream_length and output_flows["LakeID"][stream_index] < 0:
-
+        if (
+            output_flows["areasqkm"][stream_index] < min_catchment_area
+            and output_flows["LengthKm"][stream_index] < min_stream_length
+            and output_flows["LakeID"][stream_index] < 0
+        ):
             short_id = output_flows['HydroID'][stream_index]
             to_node = output_flows['To_Node'][stream_index]
             from_node = output_flows['From_Node'][stream_index]
@@ -153,112 +191,189 @@ def add_crosswalk(input_catchments_fileName,
             # multiple upstream segments
             if len(output_flows.loc[output_flows['NextDownID'] == short_id]['HydroID']) > 1:
                 try:
-                    max_order = max(output_flows.loc[output_flows['NextDownID'] == short_id]['order_']) # drainage area would be better than stream order but we would need to calculate
-                except:
-                    print(f"short_id {short_id} cannot calculate max stream order for multiple upstream segments scenario")
-
-                if len(output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID']) == 1:
-                    update_id = output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID'].item()
+                    # drainage area would be better than stream order but we would need to calculate
+                    max_order = max(output_flows.loc[output_flows['NextDownID'] == short_id]['order_'])
+                except Exception as e:
+                    print(
+                        f"short_id: {short_id} cannot calculate max stream order for "
+                        f"multiple upstream segments scenario. \n Exception: \n {repr(e)} \n"
+                    )
+
+                if (
+                    len(
+                        output_flows.loc[
+                            (output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)
+                        ]['HydroID']
+                    )
+                    == 1
+                ):
+                    update_id = output_flows.loc[
+                        (output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)
+                    ]['HydroID'].item()
 
                 else:
-                    update_id = output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+                    # Get the first one
+                    # (same stream order, without drainage area info, hard to know which is the main channel)
+                    update_id = output_flows.loc[
+                        (output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)
+                    ]['HydroID'].values[0]
 
             # single upstream segments
             elif len(output_flows.loc[output_flows['NextDownID'] == short_id]['HydroID']) == 1:
-                update_id = output_flows.loc[output_flows.To_Node==from_node]['HydroID'].item()
+                update_id = output_flows.loc[output_flows.To_Node == from_node]['HydroID'].item()
 
             # no upstream segments; multiple downstream segments
-            elif len(output_flows.loc[output_flows.From_Node==to_node]['HydroID']) > 1:
+            elif len(output_flows.loc[output_flows.From_Node == to_node]['HydroID']) > 1:
                 try:
-                    max_order = max(output_flows.loc[output_flows.From_Node==to_node]['HydroID']['order_']) # drainage area would be better than stream order but we would need to calculate
-                except:
-                    print(f"To Node {to_node} cannot calculate max stream order for no upstream segments; multiple downstream segments scenario")
-
-                if len(output_flows.loc[(output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)]['HydroID']) == 1:
-                    update_id = output_flows.loc[(output_flows.From_Node==to_node) & (output_flows['order_'] == max_order)]['HydroID'].item()
-
+                    max_order = max(
+                        output_flows.loc[output_flows.From_Node == to_node]['HydroID']['order_']
+                    )  # drainage area would be better than stream order but we would need to calculate
+                except Exception as e:
+                    print(
+                        f"To Node {to_node} cannot calculate max stream order for no upstream segments; "
+                        "multiple downstream segments scenario. "
+                        f"Exception \n {repr(e)} \n"
+                    )
+
+                if (
+                    len(
+                        output_flows.loc[
+                            (output_flows['NextDownID'] == short_id) & (output_flows['order_'] == max_order)
+                        ]['HydroID']
+                    )
+                    == 1
+                ):
+                    update_id = output_flows.loc[
+                        (output_flows.From_Node == to_node) & (output_flows['order_'] == max_order)
+                    ]['HydroID'].item()
+
+                # Get the first one
+                # Same stream order, without drainage area info it is hard to know which is the main channel.
                 else:
-                    update_id = output_flows.loc[(output_flows.From_Node==to_node) & (output_flows['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+                    update_id = output_flows.loc[
+                        (output_flows.From_Node == to_node) & (output_flows['order_'] == max_order)
+                    ]['HydroID'].values[0]
 
             # no upstream segments; single downstream segment
-            elif len(output_flows.loc[output_flows.From_Node==to_node]['HydroID']) == 1:
-                    update_id = output_flows.loc[output_flows.From_Node==to_node]['HydroID'].item()
+            elif len(output_flows.loc[output_flows.From_Node == to_node]['HydroID']) == 1:
+                update_id = output_flows.loc[output_flows.From_Node == to_node]['HydroID'].item()
 
             else:
-                update_id = output_flows.loc[output_flows.HydroID==short_id]['HydroID'].item()
-
-            str_order = output_flows.loc[output_flows.HydroID==short_id]['order_'].item()
-            sml_segs = pd.concat([sml_segs, pd.DataFrame({'short_id':[short_id], 'update_id':[update_id], 'str_order':[str_order]})], ignore_index=True)
-
-    print("Number of short reaches [{} < {} and {} < {}] = {}".format("areasqkm", min_catchment_area, "LengthKm", min_stream_length, len(sml_segs)))
+                update_id = output_flows.loc[output_flows.HydroID == short_id]['HydroID'].item()
+
+            str_order = output_flows.loc[output_flows.HydroID == short_id]['order_'].item()
+            sml_segs = pd.concat(
+                [
+                    sml_segs,
+                    pd.DataFrame(
+                        {'short_id': [short_id], 'update_id': [update_id], 'str_order': [str_order]}
+                    ),
+                ],
+                ignore_index=True,
+            )
+
+    print(
+        f"Number of short reaches [areasqkm < {min_catchment_area} and LengthKm < {min_stream_length}] = "
+        f"{len(sml_segs)}"
+    )
 
     # calculate src_full
-    input_src_base = pd.read_csv(input_srcbase_fileName, dtype= object)
-    if input_src_base.CatchId.dtype != 'int': input_src_base.CatchId = input_src_base.CatchId.astype(int)
+    input_src_base = pd.read_csv(input_srcbase_fileName, dtype=object)
+    if input_src_base.CatchId.dtype != 'int':
+        input_src_base.CatchId = input_src_base.CatchId.astype(int)
 
-    input_src_base = input_src_base.merge(output_flows[['ManningN','HydroID','NextDownID','order_']],left_on='CatchId',right_on='HydroID')
+    input_src_base = input_src_base.merge(
+        output_flows[['ManningN', 'HydroID', 'NextDownID', 'order_']], left_on='CatchId', right_on='HydroID'
+    )
 
     input_src_base = input_src_base.rename(columns=lambda x: x.strip(" "))
-    input_src_base = input_src_base.apply(pd.to_numeric,**{'errors' : 'coerce'})
-    input_src_base['TopWidth (m)'] = input_src_base['SurfaceArea (m2)']/input_src_base['LENGTHKM']/1000
-    input_src_base['WettedPerimeter (m)'] = input_src_base['BedArea (m2)']/input_src_base['LENGTHKM']/1000
-    input_src_base['WetArea (m2)'] = input_src_base['Volume (m3)']/input_src_base['LENGTHKM']/1000
-    input_src_base['HydraulicRadius (m)'] = input_src_base['WetArea (m2)']/input_src_base['WettedPerimeter (m)']
+    input_src_base = input_src_base.apply(pd.to_numeric, **{'errors': 'coerce'})
+    input_src_base['TopWidth (m)'] = input_src_base['SurfaceArea (m2)'] / input_src_base['LENGTHKM'] / 1000
+    input_src_base['WettedPerimeter (m)'] = input_src_base['BedArea (m2)'] / input_src_base['LENGTHKM'] / 1000
+    input_src_base['WetArea (m2)'] = input_src_base['Volume (m3)'] / input_src_base['LENGTHKM'] / 1000
+    input_src_base['HydraulicRadius (m)'] = (
+        input_src_base['WetArea (m2)'] / input_src_base['WettedPerimeter (m)']
+    )
     input_src_base['HydraulicRadius (m)'].fillna(0, inplace=True)
-    input_src_base['Discharge (m3s-1)'] = input_src_base['WetArea (m2)']* \
-    pow(input_src_base['HydraulicRadius (m)'],2.0/3)* \
-    pow(input_src_base['SLOPE'],0.5)/input_src_base['ManningN']
+    input_src_base['Discharge (m3s-1)'] = (
+        input_src_base['WetArea (m2)']
+        * pow(input_src_base['HydraulicRadius (m)'], 2.0 / 3)
+        * pow(input_src_base['SLOPE'], 0.5)
+        / input_src_base['ManningN']
+    )
 
     # set nans to 0
-    input_src_base.loc[input_src_base['Stage']==0,['Discharge (m3s-1)']] = 0
+    input_src_base.loc[input_src_base['Stage'] == 0, ['Discharge (m3s-1)']] = 0
     input_src_base['Bathymetry_source'] = pd.NA
 
     output_src = input_src_base.drop(columns=['CatchId']).copy()
-    if output_src.HydroID.dtype != 'int': output_src.HydroID = output_src.HydroID.astype(int)
+    if output_src.HydroID.dtype != 'int':
+        output_src.HydroID = output_src.HydroID.astype(int)
 
     # update rating curves
     if len(sml_segs) > 0:
-
-        sml_segs.to_csv(small_segments_filename,index=False)
+        sml_segs.to_csv(small_segments_filename, index=False)
         print("Update rating curves for short reaches.")
 
         for index, segment in sml_segs.iterrows():
-
             short_id = segment[0]
-            update_id= segment[1]
+            update_id = segment[1]
             new_values = output_src.loc[output_src['HydroID'] == update_id][['Stage', 'Discharge (m3s-1)']]
 
             for src_index, src_stage in new_values.iterrows():
-                output_src.loc[(output_src['HydroID']== short_id) & (output_src['Stage']== src_stage[0]),['Discharge (m3s-1)']] = src_stage[1]
+                output_src.loc[
+                    (output_src['HydroID'] == short_id) & (output_src['Stage'] == src_stage[0]),
+                    ['Discharge (m3s-1)']] = src_stage[1]
 
     if extent == 'FR':
-        output_src = output_src.merge(input_majorities[['HydroID','feature_id']],on='HydroID')
+        output_src = output_src.merge(input_majorities[['HydroID', 'feature_id']], on='HydroID')
     elif (extent == 'MS') | (extent == 'GMS'):
-        output_src = output_src.merge(crosswalk[['HydroID','feature_id']],on='HydroID')
+        output_src = output_src.merge(crosswalk[['HydroID', 'feature_id']], on='HydroID')
 
-    output_crosswalk = output_src[['HydroID','feature_id']]
+    output_crosswalk = output_src[['HydroID', 'feature_id']]
     output_crosswalk = output_crosswalk.drop_duplicates(ignore_index=True)
 
     # make hydroTable
-    output_hydro_table = output_src.loc[:,['HydroID','feature_id','NextDownID','order_','Number of Cells','SurfaceArea (m2)',
-        'BedArea (m2)','TopWidth (m)','LENGTHKM','AREASQKM','WettedPerimeter (m)','HydraulicRadius (m)','WetArea (m2)',
-        'Volume (m3)','SLOPE','ManningN','Stage','Discharge (m3s-1)']]
-    output_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
-    ## Set placeholder variables to be replaced in post-processing (as needed). Create here to ensure consistent column vars
-    ## These variables represent the original unmodified values
+    output_hydro_table = output_src.loc[
+        :,
+        [
+            'HydroID',
+            'feature_id',
+            'NextDownID',
+            'order_',
+            'Number of Cells',
+            'SurfaceArea (m2)',
+            'BedArea (m2)',
+            'TopWidth (m)',
+            'LENGTHKM',
+            'AREASQKM',
+            'WettedPerimeter (m)',
+            'HydraulicRadius (m)',
+            'WetArea (m2)',
+            'Volume (m3)',
+            'SLOPE',
+            'ManningN',
+            'Stage',
+            'Discharge (m3s-1)'
+        ],
+    ]
+    output_hydro_table.rename(columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'}, inplace=True)
+
+    # Set placeholder variables to be replaced in post-processing (as needed).
+    # Create here to ensure consistent column vars. These variables represent the original unmodified values
     output_hydro_table['default_discharge_cms'] = output_src['Discharge (m3s-1)']
     output_hydro_table['default_Volume (m3)'] = output_src['Volume (m3)']
     output_hydro_table['default_WetArea (m2)'] = output_src['WetArea (m2)']
     output_hydro_table['default_HydraulicRadius (m)'] = output_src['HydraulicRadius (m)']
     output_hydro_table['default_ManningN'] = output_src['ManningN']
-    ## Placeholder vars for BARC
+    # Placeholder vars for BARC
     output_hydro_table['Bathymetry_source'] = pd.NA
-    ## Placeholder vars for subdivision routine
+    # Placeholder vars for subdivision routine
     output_hydro_table['subdiv_applied'] = False
     output_hydro_table['overbank_n'] = pd.NA
     output_hydro_table['channel_n'] = pd.NA
     output_hydro_table['subdiv_discharge_cms'] = pd.NA
-    ## Placeholder vars for the calibration routine
+    # Placeholder vars for the calibration routine
     output_hydro_table['calb_applied'] = False
     output_hydro_table['last_updated'] = pd.NA
     output_hydro_table['submitter'] = pd.NA
@@ -269,21 +384,29 @@ def add_crosswalk(input_catchments_fileName,
     output_hydro_table['calb_coef_spatial'] = pd.NA
     output_hydro_table['calb_coef_final'] = pd.NA
 
-    if output_hydro_table.HydroID.dtype != 'str': output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
-    output_hydro_table[FIM_ID] = output_hydro_table.loc[:,'HydroID'].apply(lambda x : str(x)[0:4])
+    if output_hydro_table.HydroID.dtype != 'str':
+        output_hydro_table.HydroID = output_hydro_table.HydroID.astype(str)
+    output_hydro_table[FIM_ID] = output_hydro_table.loc[:, 'HydroID'].apply(lambda x: str(x)[0:4])
 
-    if input_huc[FIM_ID].dtype != 'str': input_huc[FIM_ID] = input_huc[FIM_ID].astype(str)
-    output_hydro_table = output_hydro_table.merge(input_huc.loc[:,[FIM_ID,'HUC8']],how='left',on=FIM_ID)
+    if input_huc[FIM_ID].dtype != 'str':
+        input_huc[FIM_ID] = input_huc[FIM_ID].astype(str)
+    output_hydro_table = output_hydro_table.merge(input_huc.loc[:, [FIM_ID, 'HUC8']], how='left', on=FIM_ID)
 
-    if output_flows.HydroID.dtype != 'str': output_flows.HydroID = output_flows.HydroID.astype(str)
-    output_hydro_table = output_hydro_table.merge(output_flows.loc[:,['HydroID','LakeID']],how='left',on='HydroID')
+    if output_flows.HydroID.dtype != 'str':
+        output_flows.HydroID = output_flows.HydroID.astype(str)
+    output_hydro_table = output_hydro_table.merge(
+        output_flows.loc[:, ['HydroID', 'LakeID']], how='left', on='HydroID'
+    )
     output_hydro_table['LakeID'] = output_hydro_table['LakeID'].astype(int)
-    output_hydro_table = output_hydro_table.rename(columns={'HUC8':'HUC'})
-    if output_hydro_table.HUC.dtype != 'str': output_hydro_table.HUC = output_hydro_table.HUC.astype(str)
+    output_hydro_table = output_hydro_table.rename(columns={'HUC8': 'HUC'})
+    if output_hydro_table.HUC.dtype != 'str':
+        output_hydro_table.HUC = output_hydro_table.HUC.astype(str)
 
-    output_hydro_table.drop(columns=FIM_ID,inplace=True)
-    if output_hydro_table.feature_id.dtype != 'int': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(int)
-    if output_hydro_table.feature_id.dtype != 'str': output_hydro_table.feature_id = output_hydro_table.feature_id.astype(str)
+    output_hydro_table.drop(columns=FIM_ID, inplace=True)
+    if output_hydro_table.feature_id.dtype != 'int':
+        output_hydro_table.feature_id = output_hydro_table.feature_id.astype(int)
+    if output_hydro_table.feature_id.dtype != 'str':
+        output_hydro_table.feature_id = output_hydro_table.feature_id.astype(str)
 
     # make src json
     output_src_json = dict()
@@ -297,43 +420,62 @@ def add_crosswalk(input_catchments_fileName,
         stage_list = stage_list.tolist()
         q_list = q_list.tolist()
 
-        output_src_json[str(hid)] = { 'q_list' : q_list , 'stage_list' : stage_list }
+        output_src_json[str(hid)] = {'q_list': q_list, 'stage_list': stage_list}
 
     # write out
-    output_catchments.to_file(output_catchments_fileName,driver=getDriver(output_catchments_fileName),index=False)
-    output_flows.to_file(output_flows_fileName,driver=getDriver(output_flows_fileName),index=False)
-    output_src.to_csv(output_src_fileName,index=False)
-    output_crosswalk.to_csv(output_crosswalk_fileName,index=False)
-    output_hydro_table.to_csv(output_hydro_table_fileName,index=False)
+    output_catchments.to_file(
+        output_catchments_fileName, driver=getDriver(output_catchments_fileName), index=False
+    )
+    output_flows.to_file(output_flows_fileName, driver=getDriver(output_flows_fileName), index=False)
+    output_src.to_csv(output_src_fileName, index=False)
+    output_crosswalk.to_csv(output_crosswalk_fileName, index=False)
+    output_hydro_table.to_csv(output_hydro_table_fileName, index=False)
 
-    with open(output_src_json_fileName,'w') as f:
-        json.dump(output_src_json,f,sort_keys=True,indent=2)
+    with open(output_src_json_fileName, 'w') as f:
+        json.dump(output_src_json, f, sort_keys=True, indent=2)
 
 
 if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Crosswalk for MS/FR/GMS networks; calculate synthetic rating curves; update short rating curves')
-    parser.add_argument('-d','--input-catchments-fileName', help='DEM derived catchments', required=True)
-    parser.add_argument('-a','--input-flows-fileName', help='DEM derived streams', required=True)
-    parser.add_argument('-s','--input-srcbase-fileName', help='Base synthetic rating curve table', required=True)
-    parser.add_argument('-l','--output-catchments-fileName', help='Subset crosswalked catchments', required=True)
-    parser.add_argument('-f','--output-flows-fileName', help='Subset crosswalked streams', required=True)
-    parser.add_argument('-r','--output-src-fileName', help='Output crosswalked synthetic rating curve table', required=True)
-    parser.add_argument('-j','--output-src-json-fileName',help='Output synthetic rating curve json',required=True)
-    parser.add_argument('-x','--output-crosswalk-fileName',help='Crosswalk table',required=True)
-    parser.add_argument('-t','--output-hydro-table-fileName',help='Hydrotable',required=True)
-    parser.add_argument('-w','--input-huc-fileName',help='HUC8 boundary',required=True)
-    parser.add_argument('-b','--input-nwmflows-fileName',help='Subest NWM burnlines',required=True)
-    parser.add_argument('-y','--input-nwmcatras-fileName',help='NWM catchment raster',required=False)
-    parser.add_argument('-m','--mannings-n',help='Mannings n. Accepts single parameter set or list of parameter set in calibration mode. Currently input as csv.',required=True)
-    parser.add_argument('-z','--input-nwmcat-fileName',help='NWM catchment polygon',required=True)
-    parser.add_argument('-p','--extent',help='GMS only for now', default='GMS', required=False)
-    parser.add_argument('-k','--small-segments-filename',help='output list of short segments',required=True)
-    parser.add_argument('-c','--calibration-mode',help='Mannings calibration flag',required=False,action='store_true')
-    parser.add_argument('-e', '--min-catchment-area', help='Minimum catchment area', required=True)
-    parser.add_argument('-g', '--min-stream-length', help='Minimum stream length', required=True)
+    parser = argparse.ArgumentParser(
+        description="Crosswalk for MS/FR/GMS networks; calculate synthetic rating curves; update short rating curves"
+    )
+    parser.add_argument("-d", "--input-catchments-fileName", help="DEM derived catchments", required=True)
+    parser.add_argument("-a", "--input-flows-fileName", help="DEM derived streams", required=True)
+    parser.add_argument(
+        "-s", "--input-srcbase-fileName", help="Base synthetic rating curve table", required=True
+    )
+    parser.add_argument(
+        "-l", "--output-catchments-fileName", help="Subset crosswalked catchments", required=True
+    )
+    parser.add_argument("-f", "--output-flows-fileName", help="Subset crosswalked streams", required=True)
+    parser.add_argument(
+        "-r", "--output-src-fileName", help="Output crosswalked synthetic rating curve table", required=True
+    )
+    parser.add_argument(
+        "-j", "--output-src-json-fileName", help="Output synthetic rating curve json", required=True
+    )
+    parser.add_argument("-x", "--output-crosswalk-fileName", help="Crosswalk table", required=True)
+    parser.add_argument("-t", "--output-hydro-table-fileName", help="Hydrotable", required=True)
+    parser.add_argument("-w", "--input-huc-fileName", help="HUC8 boundary", required=True)
+    parser.add_argument("-b", "--input-nwmflows-fileName", help="Subest NWM burnlines", required=True)
+    parser.add_argument("-y", "--input-nwmcatras-fileName", help="NWM catchment raster", required=False)
+    parser.add_argument(
+        "-m",
+        "--mannings-n",
+        help="Mannings n. Accepts single parameter set or list of parameter set in calibration mode. Currently input as csv.",
+        required=True,
+    )
+    parser.add_argument("-z", "--input-nwmcat-fileName", help="NWM catchment polygon", required=True)
+    parser.add_argument("-p", "--extent", help="GMS only for now", default="GMS", required=False)
+    parser.add_argument(
+        "-k", "--small-segments-filename", help="output list of short segments", required=True
+    )
+    parser.add_argument(
+        "-c", "--calibration-mode", help="Mannings calibration flag", required=False, action="store_true"
+    )
+    parser.add_argument("-e", "--min-catchment-area", help="Minimum catchment area", required=True)
+    parser.add_argument("-g", "--min-stream-length", help="Minimum stream length", required=True)
 
     args = vars(parser.parse_args())
 
     add_crosswalk(**args)
-   
diff --git a/src/adjust_headwater_streams.py b/src/adjust_headwater_streams.py
deleted file mode 100644
index fdd803e5d..000000000
--- a/src/adjust_headwater_streams.py
+++ /dev/null
@@ -1,203 +0,0 @@
-#!/usr/bin/env python3
-
-import geopandas as gpd
-import pandas as pd
-import numpy as np
-import argparse
-import pygeos
-from shapely.geometry import Point,LineString
-from shapely.ops import split
-from shapely.wkb import dumps, loads
-from utils.shared_variables import PREP_PROJECTION
-from utils.shared_functions import getDriver
-import warnings
-warnings.simplefilter("ignore")
-
-##################################
-##
-## Likely Deprecated: File not in use. Noticed Jan 16, 2023
-## Might want to be kept for possible re-use at a later time?
-##
-##################################
-
-def adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id):
-
-    # Identify true headwater segments
-    nhd_streams_adj = nhd_streams.loc[(nhd_streams.headwaters_id > 0) & (nhd_streams.downstream_of_headwater == False),:].copy()
-    nhd_streams_adj = nhd_streams_adj.explode(index_parts=True)
-    nhd_streams_adj = nhd_streams_adj.reset_index(drop=True)
-
-    if nwm_headwaters["site_id"].dtype != 'int': nwm_headwaters["site_id"] = nwm_headwaters["site_id"].astype(int)
-    headwater_limited = nwm_headwaters.merge(nhd_streams_adj[["headwaters_id","mainstem"]],left_on="site_id", right_on="headwaters_id",how='right')
-    headwater_limited = headwater_limited.drop(columns=['headwaters_id'])
-
-    nws_lid_limited = nws_lids.merge(nhd_streams[["nws_lid"]],left_on="site_id", right_on="nws_lid",how='right')
-    nws_lid_limited = nws_lid_limited.loc[nws_lid_limited.nws_lid!='']
-    nws_lid_limited = nws_lid_limited.drop(columns=['nws_lid'])
-
-    # Check for issues in nws_lid layer (now this reports back non-headwater nws_lids)
-    # if len(nws_lid_limited) < len(nws_lids):
-    #     missing_nws_lids = list(set(nws_lids.site_id) - set(nws_lid_limited.site_id))
-    #     print (f"nws lid(s) {missing_nws_lids} missing from aggregate dataset in huc {huc}")
-
-    # Combine NWM headwaters and AHPS sites to be snapped to NHDPlus HR segments
-    headwater_pts = pd.concat([headwater_limited, nws_lid_limited])
-    headwater_pts = headwater_pts.reset_index(drop=True)
-
-    if headwater_pts is not None:
-
-        headwaterstreams = []
-        referencedpoints = []
-        snapped_ahps = []
-        nws_lid = []
-        for index, point in headwater_pts.iterrows():
-
-            # Convert headwaterpoint geometries to WKB representation
-            wkb_points = dumps(point.geometry)
-
-            # Create pygeos headwaterpoint geometries from WKB representation
-            pointbin_geom = pygeos.io.from_wkb(wkb_points)
-
-            if point.pt_type == 'nwm_headwater':
-                # Closest segment to headwater
-                closest_stream = nhd_streams_adj.loc[nhd_streams_adj["headwaters_id"]==point[headwater_id]]
-            else:
-                # Closest segment to ahps site
-                closest_stream = nhd_streams.loc[nhd_streams["nws_lid"]==point[headwater_id]]
-
-            try: # Seeing inconsistent geometry objects even after exploding nhd_streams_adj; not sure why this is
-                closest_stream =closest_stream.explode(index_parts=True)
-            except:
-                pass
-
-            try:
-                wkb_closest_stream = dumps(closest_stream.geometry[0])
-            except:
-                wkb_closest_stream = dumps(closest_stream.geometry[0][0])
-
-            streambin_geom = pygeos.io.from_wkb(wkb_closest_stream)
-
-            # Linear reference headwater to closest stream segment
-            pointdistancetoline = pygeos.linear.line_locate_point(streambin_geom, pointbin_geom)
-            referencedpoint = pygeos.linear.line_interpolate_point(streambin_geom, pointdistancetoline)
-
-            # Convert geometries to wkb representation
-            bin_referencedpoint = pygeos.io.to_wkb(referencedpoint)
-
-            # Convert to shapely geometries
-            shply_referencedpoint = loads(bin_referencedpoint)
-            shply_linestring = loads(wkb_closest_stream)
-            headpoint = Point(shply_referencedpoint.coords)
-
-            if point.pt_type == 'nwm_headwater':
-                cumulative_line = []
-                relativedistlst = []
-
-                # Collect all nhd stream segment linestring verticies
-                for point in zip(*shply_linestring.coords.xy):
-                    cumulative_line = cumulative_line + [point]
-                    relativedist = shply_linestring.project(Point(point))
-                    relativedistlst = relativedistlst + [relativedist]
-
-                # Add linear referenced headwater point to closest nhd stream segment
-                if not headpoint in cumulative_line:
-                    cumulative_line = cumulative_line + [headpoint]
-                    relativedist = shply_linestring.project(headpoint)
-                    relativedistlst = relativedistlst + [relativedist]
-
-                # Sort by relative line distance to place headwater point in linestring
-                sortline = pd.DataFrame({'geom' : cumulative_line, 'dist' : relativedistlst}).sort_values('dist')
-                shply_linestring = LineString(sortline.geom.tolist())
-                referencedpoints = referencedpoints + [headpoint]
-
-                # Split the new linestring at the new headwater point
-                try:
-                    line1,line2 = split(shply_linestring, headpoint)
-                    headwaterstreams = headwaterstreams + [LineString(line1)]
-                    nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1)
-                except:
-                    line1 = split(shply_linestring, headpoint)
-                    headwaterstreams = headwaterstreams + [LineString(line1[0])]
-                    nhd_streams.loc[nhd_streams.NHDPlusID==closest_stream.NHDPlusID.values[0],'geometry'] = LineString(line1[0])
-
-                try:
-                    del cumulative_line, relativedistlst
-                except:
-                    print (f"issue deleting adjusted stream variables for huc {huc}")
-
-            else:
-                snapped_ahps = snapped_ahps + [headpoint]
-                nws_lid = nws_lid + [point[headwater_id]]
-
-        nhd_streams = nhd_streams.drop(columns=['is_relevant_stream', 'headwaters_id', 'downstream_of_headwater'])
-
-        try:
-            del nhd_streams_adj, headwater_limited, referencedpoints, headwaterstreams
-        except:
-            print (f"issue deleting adjusted stream variables for huc {huc}")
-
-        # Create snapped ahps sites
-        if len(snapped_ahps) > 0:
-            snapped_ahps_points = gpd.GeoDataFrame({'pt_type': 'nws_lid', headwater_id: nws_lid, 'mainstem': True,
-                                                    'geometry': snapped_ahps},geometry='geometry',crs=PREP_PROJECTION)
-
-        # Identify ajusted nhd headwaters
-        nhd_headwater_streams_adj = nhd_streams.loc[nhd_streams['is_headwater'],:]
-        nhd_headwater_streams_adj = nhd_headwater_streams_adj.explode(index_parts=True)
-
-        hw_points = np.zeros(len(nhd_headwater_streams_adj),dtype=object)
-        for index,lineString in enumerate(nhd_headwater_streams_adj.geometry):
-            hw_point = [point for point in zip(*lineString.coords.xy)][-1]
-            hw_points[index] = Point(*hw_point)
-
-
-        nhd_headwater_points_adj = gpd.GeoDataFrame({'pt_type': 'NHDPlusID', headwater_id: nhd_headwater_streams_adj['NHDPlusID'],
-                                                 'mainstem': False, 'geometry': hw_points},geometry='geometry',crs=PREP_PROJECTION)
-
-        nhd_headwater_points_adj = nhd_headwater_points_adj.reset_index(drop=True)
-
-        del nhd_headwater_streams_adj
-
-        try:
-            combined_pts = pd.concat([snapped_ahps_points, nhd_headwater_points_adj])
-        except:
-            combined_pts = nhd_headwater_points_adj.copy()
-
-        return nhd_streams, combined_pts
-
-
-if __name__ == '__main__':
-
-##################################
-##
-## Likely Deprecated: File not in use. Noticed Jan 16, 2023
-## Might want to be kept for possible re-use at a later time?
-##
-##################################
-
-
-    parser = argparse.ArgumentParser(description='adjust headwater stream geometery based on headwater start points')
-    parser.add_argument('-f','--huc',help='huc number',required=True)
-    parser.add_argument('-l','--nhd-streams',help='NHDPlus HR geodataframe',required=True)
-    parser.add_argument('-p','--nwm-headwaters',help='Headwater points layer',required=True,type=str)
-    parser.add_argument('-s','--subset-nhd-streams-fileName',help='Output streams layer name',required=False,type=str,default=None)
-    parser.add_argument('-a','--adj-headwater-points-fileName',help='Output adj headwater points layer name',required=False,type=str,default=None)
-    parser.add_argument('-g','--headwater-points-fileName',help='Output headwater points layer name',required=False,type=str,default=None)
-    parser.add_argument('-b','--nws-lids',help='NWS lid points',required=True)
-    parser.add_argument('-i','--headwater-id',help='Headwater id column name',required=True)
-
-    args = vars(parser.parse_args())
-
-    #TODO variables below are not defined
-
-    adj_streams_gdf, adj_headwaters_gdf = adjust_headwaters(huc,nhd_streams,nwm_headwaters,nws_lids,headwater_id)
-
-    if subset_nhd_streams_fileName is not None:
-        adj_streams_gdf.to_file(args['subset_nhd_streams_fileName'],driver=getDriver(args['subset_nhd_streams_fileName']))
-
-    if headwater_points_fileName is not None:
-        headwater_points_fileName.to_file(args['headwater_points_fileName'],driver=getDriver(args['headwater_points_fileName']))
-
-    if adj_headwater_points_fileName is not None:
-        adj_headwaters_gdf.to_file(args['adj_headwater_points_fileName'],driver=getDriver(args['adj_headwater_points_fileName']))
-
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index f838a299b..dabfbd488 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -2,45 +2,53 @@
 
 
 import argparse
-from numba import njit, typed, types
-import rasterio
+
 import numpy as np
+import rasterio
+from numba import njit, typed, types
+
 from utils.shared_functions import mem_profile
 
 
 @mem_profile
-def adjust_thalweg_laterally(elevation_raster, stream_raster, allocation_raster, cost_distance_raster, cost_distance_tolerance, dem_lateral_thalweg_adj,lateral_elevation_threshold):
-
-    # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
+def adjust_thalweg_laterally(
+    elevation_raster,
+    stream_raster,
+    allocation_raster,
+    cost_distance_raster,
+    cost_distance_tolerance,
+    dem_lateral_thalweg_adj,
+    lateral_elevation_threshold,
+):
+    # ------------------------------------ Get catchment_min_dict ----------------------------------------- #
     # The following algorithm searches for the zonal minimum elevation in each pixel catchment
     @njit
     def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, cost_tolerance, ndv):
-        for i,elev_m in enumerate(zone_window):
+        for i, elev_m in enumerate(zone_window):
             # If the zone really exists in the dictionary, compare elevation values.
             i = int(i)
             elev_m = int(elev_m)
 
-            if (cost_window[i] <= cost_tolerance):
+            if cost_window[i] <= cost_tolerance:
                 if elevation_window[i] > 0:  # Don't allow bad elevation values
-                    if (elev_m in zone_min_dict):
-
-                        if (elevation_window[i] < zone_min_dict[elev_m]):
-                            # If the elevation_window's elevation value is less than the zone_min_dict min, update the zone_min_dict min.
+                    if elev_m in zone_min_dict:
+                        # If the elevation_window's elevation value is less than the zone_min_dict min,
+                        # update the zone_min_dict min.
+                        if elevation_window[i] < zone_min_dict[elev_m]:
                             zone_min_dict[elev_m] = elevation_window[i]
                     else:
                         zone_min_dict[elev_m] = elevation_window[i]
 
-        return(zone_min_dict)
+        return zone_min_dict
 
-    # ------------------------------------------- Assign zonal min to thalweg ------------------------------------------------ #
+    # ------------------------------------ Assign zonal min to thalweg ------------------------------------ #
     @njit
     def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window):
-
         # Copy elevation values into new array that will store the minimized elevation values.
-        dem_window_to_return = np.empty_like (dem_window)
+        dem_window_to_return = np.empty_like(dem_window)
         dem_window_to_return[:] = dem_window
 
-        for i,elev_m in enumerate(zone_window):
+        for i, elev_m in enumerate(zone_window):
             i = int(i)
             elev_m = int(elev_m)
             thalweg_cell = thalweg_window[i]  # From flows_grid_boolean.tif (0s and 1s)
@@ -51,64 +59,112 @@ def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_w
 
                     elevation_difference = dem_thalweg_elevation - zone_min_elevation
 
-                    if (zone_min_elevation < dem_thalweg_elevation) and (elevation_difference <= lateral_elevation_threshold):
+                    if (zone_min_elevation < dem_thalweg_elevation) and (
+                        elevation_difference <= lateral_elevation_threshold
+                    ):
                         dem_window_to_return[i] = zone_min_elevation
 
-        return(dem_window_to_return)
+        return dem_window_to_return
 
     # Open files.
-    with rasterio.open(elevation_raster) as elevation_raster_object, rasterio.open(allocation_raster) as allocation_zone_raster_object:
-
+    with rasterio.open(elevation_raster) as elevation_raster_object, rasterio.open(
+        allocation_raster
+    ) as allocation_zone_raster_object:
         with rasterio.open(cost_distance_raster) as cost_distance_raster_object:
-
             meta = elevation_raster_object.meta.copy()
             meta['tiled'], meta['compress'] = True, 'lzw'
             ndv = meta['nodata']
 
             # -- Create zone_min_dict -- #
-            zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums
+            zone_min_dict = typed.Dict.empty(
+                types.int32, types.float32
+            )  # Initialize an empty dictionary to store the catchment minimums
             # Update catchment_min_dict with pixel sheds minimum.
-            for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template
-                elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window
-                zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window
+            for ji, window in elevation_raster_object.block_windows(
+                1
+            ):  # Iterate over windows, using elevation_raster_object as template
+                elevation_window = elevation_raster_object.read(
+                    1, window=window
+                ).ravel()  # Define elevation_window
+                zone_window = allocation_zone_raster_object.read(
+                    1, window=window
+                ).ravel()  # Define zone_window
                 cost_window = cost_distance_raster_object.read(1, window=window).ravel()  # Define cost_window
 
                 # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
-                zone_min_dict = make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, int(cost_distance_tolerance), ndv)
-
-            # ------------------------------------------------------------------------------------------------------------------------ #
+                zone_min_dict = make_zone_min_dict(
+                    elevation_window,
+                    zone_min_dict,
+                    zone_window,
+                    cost_window,
+                    int(cost_distance_tolerance),
+                    ndv,
+                )
 
+            # --------------------------------------------------------------------------------------------- #
 
         # Specify raster object metadata.
-        with rasterio.open(stream_raster) as thalweg_object, rasterio.open(dem_lateral_thalweg_adj, 'w', **meta) as dem_lateral_thalweg_adj_object:
-
-            for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
-                dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window
+        with rasterio.open(stream_raster) as thalweg_object, rasterio.open(
+            dem_lateral_thalweg_adj, 'w', **meta
+        ) as dem_lateral_thalweg_adj_object:
+            for ji, window in elevation_raster_object.block_windows(
+                1
+            ):  # Iterate over windows, using dem_rasterio_object as template
+                dem_window = elevation_raster_object.read(1, window=window)  # Define dem_window
                 window_shape = dem_window.shape
                 dem_window = dem_window.ravel()
 
-                zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window
-                thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window
+                zone_window = allocation_zone_raster_object.read(
+                    1, window=window
+                ).ravel()  # Define catchments_window
+                thalweg_window = thalweg_object.read(1, window=window).ravel()  # Define thalweg_window
 
                 # Call numba-optimized function to reassign thalweg cell values to catchment minimum value.
-                minimized_dem_window = minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window)
+                minimized_dem_window = minimize_thalweg_elevation(
+                    dem_window, zone_min_dict, zone_window, thalweg_window
+                )
                 minimized_dem_window = minimized_dem_window.reshape(window_shape).astype(np.float32)
 
                 dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)
 
 
-
 if __name__ == '__main__':
-
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Adjusts the elevation of the thalweg to the lateral zonal minimum.')
-    parser.add_argument('-e','--elevation_raster',help='Raster of elevation.',required=True)
-    parser.add_argument('-s','--stream_raster',help='Raster of thalweg pixels (0=No Thalweg, 1=Thalweg)',required=True)
-    parser.add_argument('-a','--allocation_raster',help='Raster of thalweg allocation zones.',required=True)
-    parser.add_argument('-d','--cost_distance_raster',help='Raster of cost distances for the allocation raster.',required=True)
-    parser.add_argument('-t','--cost_distance_tolerance',help='Tolerance in meters to use when searching for zonal minimum.',required=True)
-    parser.add_argument('-o','--dem_lateral_thalweg_adj',help='Output elevation raster with adjusted thalweg.',required=True)
-    parser.add_argument('-th','--lateral_elevation_threshold',help='Maximum difference between current thalweg elevation and lowest lateral elevation in meters.',required=True,type=int)
+    parser = argparse.ArgumentParser(
+        description='Adjusts the elevation of the thalweg to the lateral zonal minimum.'
+    )
+    parser.add_argument('-e', '--elevation_raster', help='Raster of elevation.', required=True)
+    parser.add_argument(
+        '-s', '--stream_raster', help='Raster of thalweg pixels (0=No Thalweg, 1=Thalweg)', required=True
+    )
+    parser.add_argument(
+        '-a', '--allocation_raster', help='Raster of thalweg allocation zones.', required=True
+    )
+    parser.add_argument(
+        '-d',
+        '--cost_distance_raster',
+        help='Raster of cost distances for the allocation raster.',
+        required=True,
+    )
+    parser.add_argument(
+        '-t',
+        '--cost_distance_tolerance',
+        help='Tolerance in meters to use when searching for zonal minimum.',
+        required=True,
+    )
+    parser.add_argument(
+        '-o',
+        '--dem_lateral_thalweg_adj',
+        help='Output elevation raster with adjusted thalweg.',
+        required=True,
+    )
+    parser.add_argument(
+        '-th',
+        '--lateral_elevation_threshold',
+        help='Maximum difference between current thalweg elevation and lowest lateral elevation in meters.',
+        required=True,
+        type=int,
+    )
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
diff --git a/src/aggregate_branch_lists.py b/src/aggregate_branch_lists.py
index 9cff6bb13..333156a8e 100755
--- a/src/aggregate_branch_lists.py
+++ b/src/aggregate_branch_lists.py
@@ -1,33 +1,33 @@
 #!/usr/bin/env python3
 
-import argparse 
-import pandas as pd
+import argparse
 import sys
-
-from os.path import join
 from glob import glob
+from os.path import join
+
+import pandas as pd
 
-def aggregate_branch_lists(output_dir, file_name, output_file_name):
 
+def aggregate_branch_lists(output_dir, file_name, output_file_name):
     file_names = glob(join(output_dir, '*', file_name))
-    
-    if (len(file_names) == 0):
-        print ("Error: No Branches available to aggregate. Program terminated.", flush=True)
+
+    if len(file_names) == 0:
+        print("Error: No Branches available to aggregate. Program terminated.", flush=True)
         sys.exit(1)
 
-    df_combined = pd.concat([pd.read_csv(f, header=None, dtype ='str') for f in file_names ], ignore_index=True) 
-   
+    df_combined = pd.concat([pd.read_csv(f, header=None, dtype='str') for f in file_names], ignore_index=True)
+
     df_combined.to_csv(output_file_name, index=False, header=False)
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     # This tool takes in a single file name and searchs all directories
     # recusively for the same file name and merges them.
-    
+
     parser = argparse.ArgumentParser(description='Aggregate')
-    parser.add_argument('-d','--output_dir', help='output run data directory', required=True)
-    parser.add_argument('-f','--file_name', help='file name to match', required=True)    
-    parser.add_argument('-o','--output_file_name', help='output file name', required=True)
+    parser.add_argument('-d', '--output_dir', help='output run data directory', required=True)
+    parser.add_argument('-f', '--file_name', help='file name to match', required=True)
+    parser.add_argument('-o', '--output_file_name', help='output file name', required=True)
 
     args = vars(parser.parse_args())
 
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index 74f0b57b9..a6128869b 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -1,138 +1,146 @@
 #!/usr/bin/env python3
 
-import os, glob, re, traceback
+import argparse
+import glob
+import os
+import re
+import traceback
+from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from os.path import join
 
-import argparse
 import pandas as pd
 
-from concurrent.futures import ProcessPoolExecutor
 from utils.shared_functions import progress_bar_handler
 
-class HucDirectory(object):
 
+class HucDirectory(object):
     def __init__(self, fim_directory, huc_id, limit_branches=[]):
-
         self.fim_directory = fim_directory
         self.huc_dir_path = join(fim_directory, huc_id)
         self.limit_branches = limit_branches
 
-        self.usgs_dtypes = {'location_id':str,
-                            'nws_lid':str,
-                            'feature_id':int,
-                            'HydroID':int,
-                            'levpa_id':str,
-                            'dem_elevation':float,
-                            'dem_adj_elevation':float,
-                            'order_':str,
-                            'LakeID':object,
-                            'HUC8':str,
-                            'snap_distance':float}
+        self.usgs_dtypes = {
+            'location_id': str,
+            'nws_lid': str,
+            'feature_id': int,
+            'HydroID': int,
+            'levpa_id': str,
+            'dem_elevation': float,
+            'dem_adj_elevation': float,
+            'order_': str,
+            'LakeID': object,
+            'HUC8': str,
+            'snap_distance': float,
+        }
         self.agg_usgs_elev_table = pd.DataFrame(columns=list(self.usgs_dtypes.keys()))
 
-        self.hydrotable_dtypes = {'HydroID':int,
-                                  'branch_id':int,
-                                  'feature_id':int,
-                                  'NextDownID':int,
-                                  'order_':int,
-                                  'Number of Cells':int,
-                                  'SurfaceArea (m2)':float,
-                                  'BedArea (m2)':float,
-                                  'TopWidth (m)':float,
-                                  'LENGTHKM':float,
-                                  'AREASQKM':float,
-                                  'WettedPerimeter (m)':float,
-                                  'HydraulicRadius (m)':float,
-                                  'WetArea (m2)':float,
-                                  'Volume (m3)':float,
-                                  'SLOPE':float,
-                                  'ManningN':float,
-                                  'stage':float,
-                                  'default_discharge_cms':float,
-                                  'default_Volume (m3)':float,
-                                  'default_WetArea (m2)':float,
-                                  'default_HydraulicRadius (m)':float,
-                                  'default_ManningN':float,
-                                  'calb_applied':bool,
-                                  'last_updated':str,
-                                  'submitter':str,
-                                  'obs_source':str,
-                                  'precalb_discharge_cms':float,
-                                  'calb_coef_usgs':float,
-                                  'calb_coef_spatial':float,
-                                  'calb_coef_final':float,
-                                  'HUC':int,
-                                  'LakeID':int,
-                                  'subdiv_applied':bool,
-                                  'channel_n':float,
-                                  'overbank_n':float,
-                                  'subdiv_discharge_cms':float,
-                                  'discharge_cms':float}
+        self.hydrotable_dtypes = {
+            'HydroID': int,
+            'branch_id': int,
+            'feature_id': int,
+            'NextDownID': int,
+            'order_': int,
+            'Number of Cells': int,
+            'SurfaceArea (m2)': float,
+            'BedArea (m2)': float,
+            'TopWidth (m)': float,
+            'LENGTHKM': float,
+            'AREASQKM': float,
+            'WettedPerimeter (m)': float,
+            'HydraulicRadius (m)': float,
+            'WetArea (m2)': float,
+            'Volume (m3)': float,
+            'SLOPE': float,
+            'ManningN': float,
+            'stage': float,
+            'default_discharge_cms': float,
+            'default_Volume (m3)': float,
+            'default_WetArea (m2)': float,
+            'default_HydraulicRadius (m)': float,
+            'default_ManningN': float,
+            'calb_applied': bool,
+            'last_updated': str,
+            'submitter': str,
+            'obs_source': str,
+            'precalb_discharge_cms': float,
+            'calb_coef_usgs': float,
+            'calb_coef_spatial': float,
+            'calb_coef_final': float,
+            'HUC': int,
+            'LakeID': int,
+            'subdiv_applied': bool,
+            'channel_n': float,
+            'overbank_n': float,
+            'subdiv_discharge_cms': float,
+            'discharge_cms': float,
+        }
         self.agg_hydrotable = pd.DataFrame(columns=list(self.hydrotable_dtypes.keys()))
 
-        self.src_crosswalked_dtypes = {'branch_id':int,
-                                    'HydroID':int,
-                                    'feature_id':int,
-                                    'Stage':float,
-                                    'Number of Cells':int,
-                                    'SurfaceArea (m2)':float,
-                                    'BedArea (m2)':float,
-                                    'Volume (m3)':float,
-                                    'SLOPE':float,
-                                    'LENGTHKM':float,
-                                    'AREASQKM':float,
-                                    'ManningN':float,
-                                    'NextDownID':int,
-                                    'order_':int,
-                                    'TopWidth (m)':float,
-                                    'WettedPerimeter (m)':float,
-                                    'WetArea (m2)':float,
-                                    'HydraulicRadius (m)':float,
-                                    'Discharge (m3s-1)':float,
-                                    'bankfull_flow':float,
-                                    'Stage_bankfull':float,
-                                    'BedArea_bankfull':float,
-                                    'Volume_bankfull':float,
-                                    'HRadius_bankfull':float,
-                                    'SurfArea_bankfull':float,
-                                    'bankfull_proxy':str,
-                                    'Volume_chan (m3)':float,
-                                    'BedArea_chan (m2)':float,
-                                    'WettedPerimeter_chan (m)':float,
-                                    'Volume_obank (m3)':float,
-                                    'BedArea_obank (m2)':float,
-                                    'WettedPerimeter_obank (m)':float,
-                                    'channel_n':float,
-                                    'overbank_n':float,
-                                    'subdiv_applied':bool,
-                                    'WetArea_chan (m2)':float,
-                                    'HydraulicRadius_chan (m)':float,
-                                    'Discharge_chan (m3s-1)':float,
-                                    'Velocity_chan (m/s)':float,
-                                    'WetArea_obank (m2)':float,
-                                    'HydraulicRadius_obank (m)':float,
-                                    'Discharge_obank (m3s-1)':float,
-                                    'Velocity_obank (m/s)':float,
-                                    'Discharge (m3s-1)_subdiv':float}
+        self.src_crosswalked_dtypes = {
+            'branch_id': int,
+            'HydroID': int,
+            'feature_id': int,
+            'Stage': float,
+            'Number of Cells': int,
+            'SurfaceArea (m2)': float,
+            'BedArea (m2)': float,
+            'Volume (m3)': float,
+            'SLOPE': float,
+            'LENGTHKM': float,
+            'AREASQKM': float,
+            'ManningN': float,
+            'NextDownID': int,
+            'order_': int,
+            'TopWidth (m)': float,
+            'WettedPerimeter (m)': float,
+            'WetArea (m2)': float,
+            'HydraulicRadius (m)': float,
+            'Discharge (m3s-1)': float,
+            'bankfull_flow': float,
+            'Stage_bankfull': float,
+            'BedArea_bankfull': float,
+            'Volume_bankfull': float,
+            'HRadius_bankfull': float,
+            'SurfArea_bankfull': float,
+            'bankfull_proxy': str,
+            'Volume_chan (m3)': float,
+            'BedArea_chan (m2)': float,
+            'WettedPerimeter_chan (m)': float,
+            'Volume_obank (m3)': float,
+            'BedArea_obank (m2)': float,
+            'WettedPerimeter_obank (m)': float,
+            'channel_n': float,
+            'overbank_n': float,
+            'subdiv_applied': bool,
+            'WetArea_chan (m2)': float,
+            'HydraulicRadius_chan (m)': float,
+            'Discharge_chan (m3s-1)': float,
+            'Velocity_chan (m/s)': float,
+            'WetArea_obank (m2)': float,
+            'HydraulicRadius_obank (m)': float,
+            'Discharge_obank (m3s-1)': float,
+            'Velocity_obank (m/s)': float,
+            'Discharge (m3s-1)_subdiv': float,
+        }
         self.agg_src_cross = pd.DataFrame(columns=list(self.src_crosswalked_dtypes.keys()))
 
-        self.ras_dtypes = {'location_id':str,
-                            'nws_lid':str,
-                            'feature_id':int,
-                            'HydroID':int,
-                            'levpa_id':str,
-                            'dem_elevation':float,
-                            'dem_adj_elevation':float,
-                            'order_':str,
-                            'LakeID':object,
-                            'HUC8':str,
-                            'snap_distance':float}
+        self.ras_dtypes = {
+            'location_id': str,
+            'nws_lid': str,
+            'feature_id': int,
+            'HydroID': int,
+            'levpa_id': str,
+            'dem_elevation': float,
+            'dem_adj_elevation': float,
+            'order_': str,
+            'LakeID': object,
+            'HUC8': str,
+            'snap_distance': float,
+        }
         self.agg_ras_elev_table = pd.DataFrame(columns=list(self.ras_dtypes.keys()))
-        
 
     def iter_branches(self):
-
         if self.limit_branches:
             for branch in self.limit_branches:
                 yield (branch, join(self.huc_dir_path, 'branches', branch))
@@ -142,7 +150,6 @@ def iter_branches(self):
                 yield (branch, join(self.huc_dir_path, 'branches', branch))
 
     def usgs_elev_table(self, branch_path):
-
         usgs_elev_filename = join(branch_path, 'usgs_elev_table.csv')
         if not os.path.isfile(usgs_elev_filename):
             return
@@ -150,9 +157,7 @@ def usgs_elev_table(self, branch_path):
         usgs_elev_table = pd.read_csv(usgs_elev_filename, dtype=self.usgs_dtypes)
         self.agg_usgs_elev_table = pd.concat([self.agg_usgs_elev_table, usgs_elev_table])
 
-
     def aggregate_hydrotables(self, branch_path, branch_id):
-
         hydrotable_filename = join(branch_path, f'hydroTable_{branch_id}.csv')
         if not os.path.isfile(hydrotable_filename):
             return
@@ -162,9 +167,7 @@ def aggregate_hydrotables(self, branch_path, branch_id):
         hydrotable[['calb_applied']] = hydrotable[['calb_applied']].fillna(value=False)
         self.agg_hydrotable = pd.concat([self.agg_hydrotable, hydrotable])
 
-
     def aggregate_src_full_crosswalk(self, branch_path, branch_id):
-
         src_cross_filename = join(branch_path, f'src_full_crosswalked_{branch_id}.csv')
         if not os.path.isfile(src_cross_filename):
             return
@@ -174,7 +177,6 @@ def aggregate_src_full_crosswalk(self, branch_path, branch_id):
         self.agg_src_cross = pd.concat([self.agg_src_cross, src_cross])
 
     def ras_elev_table(self, branch_path):
-
         ras_elev_filename = join(branch_path, 'ras_elev_table.csv')
         if not os.path.isfile(ras_elev_filename):
             return
@@ -183,9 +185,8 @@ def ras_elev_table(self, branch_path):
         self.agg_ras_elev_table = pd.concat([self.agg_ras_elev_table, ras_elev_table])
 
     def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id):
-        
         try:
-            #try catch and its own log file output in error only.
+            # try catch and its own log file output in error only.
             for branch_id, branch_path in self.iter_branches():
                 if usgs_elev_flag:
                     self.usgs_elev_table(branch_path)
@@ -231,141 +232,168 @@ def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_ele
                 if not self.agg_ras_elev_table.empty:
                     self.agg_ras_elev_table.to_csv(ras_elev_table_file, index=False)
 
-            #print(f"agg_by_huc for huc id {huc_id} is done")
-            
+            # print(f"agg_by_huc for huc id {huc_id} is done")
+
         except Exception:
-            
-            errMsg = "--------------------------------------" \
-                     f"\n huc_id {huc_id} has an error - outside multi proc\n"
+            errMsg = (
+                "--------------------------------------"
+                f"\n huc_id {huc_id} has an error - outside multi proc\n"
+            )
             errMsg = errMsg + traceback.format_exc()
             print(errMsg, flush=True)
-            log_error(self.fim_directory, usgs_elev_flag,
-                        hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg)
-           
+            log_error(
+                self.fim_directory,
+                usgs_elev_flag,
+                hydro_table_flag,
+                src_cross_flag,
+                ras_elev_flag,
+                huc_id,
+                errMsg,
+            )
 
 
 # ==============================
 # This is done independantly in each worker and does not attempt to write to a shared file
 # as those can collide with multi proc
 def log_error(fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg):
-    
     file_name = f"agg_by_huc_{huc_id}"
-    if (usgs_elev_flag): file_name+= "_elev"
-    if (hydro_table_flag): file_name+= "_hydro"
-    if (src_cross_flag): file_name+= "_src_cross"
-    if (ras_elev_flag): file_name+= "_ras"
-    file_name+= "_error.log"
-    
+    if usgs_elev_flag:
+        file_name += "_elev"
+    if hydro_table_flag:
+        file_name += "_hydro"
+    if src_cross_flag:
+        file_name += "_src_cross"
+    if ras_elev_flag:
+        file_name += "_ras"
+    file_name += "_error.log"
+
     log_path = os.path.join(fim_directory, "logs", "agg_by_huc_errors")
-    file_path =  os.path.join(log_path, file_name)
-    
+    file_path = os.path.join(log_path, file_name)
+
     f = open(file_path, "a")
     f.write(errMsg)
     f.close()
 
 
-def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
-                     hydro_table_flag, src_cross_flag, ras_elev_flag, num_job_workers):
-    
+def aggregate_by_huc(
+    fim_directory,
+    fim_inputs,
+    usgs_elev_flag,
+    hydro_table_flag,
+    src_cross_flag,
+    ras_elev_flag,
+    num_job_workers,
+):
     assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
 
     # -------------------
     # Validation
     total_cpus_available = os.cpu_count() - 2
     if num_job_workers > total_cpus_available:
-        raise ValueError(f'The number of jobs {num_job_workers}'\
-                          'exceeds your machine\'s available CPU count minus two. '\
-                          'Please lower the number of jobs '\
-                          'values accordingly.'
-                        )
-   
+        raise ValueError(
+            f'The number of jobs {num_job_workers}'
+            'exceeds your machine\'s available CPU count minus two. '
+            'Please lower the number of jobs '
+            'values accordingly.'
+        )
+
     # create log folder, might end up empty but at least create the folder
     # Yes.. this is duplicate in the log function
     log_folder = os.path.join(fim_directory, "logs", "agg_by_huc_errors")
-    if (os.path.exists(log_folder) == False):
+    if os.path.exists(log_folder) is False:
         os.mkdir(log_folder)
     else:
-        # empty only ones with this type (we want to keep others that 
+        # empty only ones with this type (we want to keep others that
         # might have been called with different types. aka.. once for -elev
         # and another for -hydro)
         agg_type = ""
-        if (usgs_elev_flag): agg_type+= "_elev"
-        if (hydro_table_flag): agg_type+= "_hydro"
-        if (src_cross_flag): agg_type+= "_src_cross"
-        if (ras_elev_flag): agg_type+= "_ras"
+        if usgs_elev_flag:
+            agg_type += "_elev"
+        if hydro_table_flag:
+            agg_type += "_hydro"
+        if src_cross_flag:
+            agg_type += "_src_cross"
+        if ras_elev_flag:
+            agg_type += "_ras"
         filelist = glob.glob(os.path.join(log_folder, f"*{agg_type}*"))
         for f in filelist:
             os.remove(f)
 
     start_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"started: {dt_string}")
-   
-    # Set up multiprocessor
-    with ProcessPoolExecutor(max_workers = num_job_workers) as executor:
+    print(f"started: {dt_string}")
 
-        # Loop through all applicable HUCs, build the agg_function arguments, and submit them to the process pool
+    # Set up multiprocessor
+    with ProcessPoolExecutor(max_workers=num_job_workers) as executor:
+        # Loop through applicable HUCs, build the agg_function arguments, and submit them to the process pool
         executor_dict = {}
-    
-        try:    
+
+        try:
             if fim_inputs:
-                fim_inputs_csv = pd.read_csv(fim_inputs, header=None, names=['huc', 'levpa_id'],dtype=str)
+                fim_inputs_csv = pd.read_csv(fim_inputs, header=None, names=['huc', 'levpa_id'], dtype=str)
                 huc_list = fim_inputs_csv.huc.unique()
-    
+
                 # with multi proc, it won't be 100% in order as different hucs
                 # process faster, but it does help a little
-                huc_list_sorted = sorted(huc_list) 
+                huc_list_sorted = sorted(huc_list)
                 for huc_id in huc_list_sorted:
-
                     branches = fim_inputs_csv.loc[fim_inputs_csv.huc == huc_id, 'levpa_id'].tolist()
                     huc_dir = HucDirectory(fim_directory, huc_id, limit_branches=branches)
 
-                    args_agg = { 'usgs_elev_flag': usgs_elev_flag,
-                                 'hydro_table_flag': hydro_table_flag,
-                                 'src_cross_flag': src_cross_flag,
-                                 'ras_elev_flag': ras_elev_flag,
-                                 'huc_id': huc_id}       
+                    args_agg = {
+                        'usgs_elev_flag': usgs_elev_flag,
+                        'hydro_table_flag': hydro_table_flag,
+                        'src_cross_flag': src_cross_flag,
+                        'ras_elev_flag': ras_elev_flag,
+                        'huc_id': huc_id,
+                    }
 
                     future = executor.submit(huc_dir.agg_function, **args_agg)
                     executor_dict[future] = huc_id
 
             else:
-                huc_list = [d for d in os.listdir(fim_directory) if re.match('\d{8}', d)]
-                
+                huc_list = [d for d in os.listdir(fim_directory) if re.match(r'\d{8}', d)]
+
                 # with multi proc, it won't be 100% in order as different hucs
                 # process faster, but it does help a little
                 huc_list_sorted = sorted(huc_list)
                 for huc_id in huc_list_sorted:
+                    if huc_id.isnumeric() is False:
+                        continue
 
-                    if (huc_id.isnumeric() == False):
-                        continue;
-                    
                     huc_dir = HucDirectory(fim_directory, huc_id)
-                    
-                    args_agg = { 'usgs_elev_flag': usgs_elev_flag,
-                                'hydro_table_flag': hydro_table_flag,
-                                'src_cross_flag': src_cross_flag,
-                                'ras_elev_flag': ras_elev_flag,
-                                'huc_id': huc_id}       
+
+                    args_agg = {
+                        'usgs_elev_flag': usgs_elev_flag,
+                        'hydro_table_flag': hydro_table_flag,
+                        'src_cross_flag': src_cross_flag,
+                        'ras_elev_flag': ras_elev_flag,
+                        'huc_id': huc_id,
+                    }
                     future = executor.submit(huc_dir.agg_function, **args_agg)
                     executor_dict[future] = huc_id
 
         except Exception:
-            errMsg = "--------------------------------------" \
-                     f"\n huc_id {huc_id} has an error - outside multi proc\n"
+            errMsg = (
+                "--------------------------------------"
+                f"\n huc_id {huc_id} has an error - outside multi proc\n"
+            )
             errMsg = errMsg + traceback.format_exc()
             print(errMsg, flush=True)
-            log_error(fim_directory, usgs_elev_flag,
-                        hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg)
-            #sys.exit(1)
-    
+            log_error(
+                fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg
+            )
+            # sys.exit(1)
+
         # Send the executor to the progress bar and wait for all MS tasks to finish
-        progress_bar_handler(executor_dict, f"Running aggregate_by_huc with {num_job_workers} workers")      
+        progress_bar_handler(executor_dict, f"Running aggregate_by_huc with {num_job_workers} workers")
 
+        # Send the executor to the progress bar and wait for all MS tasks to finish
+        progress_bar_handler(executor_dict, f"Running aggregate_by_huc with {num_job_workers} workers")
 
     end_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"ended: {dt_string}")
+    print(f"ended: {dt_string}")
 
     # Calculate duration
     time_duration = end_time - start_time
@@ -373,27 +401,50 @@ def aggregate_by_huc(fim_directory, fim_inputs, usgs_elev_flag,
     print()
 
 
-    
 if __name__ == '__main__':
-
-    # Note: This processing is done here for all hucs and branches at once instead of 
-    # at each individual huc processing due to some hydrotable's being adjusted 
+    # Note: This processing is done here for all hucs and branches at once instead of
+    # at each individual huc processing due to some hydrotable's being adjusted
     # in post processing.
 
     parser = argparse.ArgumentParser(description='Aggregates usgs_elev_table.csv at the HUC level')
-    parser.add_argument('-fim','--fim_directory', help='Input FIM Directory', required=True)
-    parser.add_argument('-i','--fim_inputs', help='Input fim_inputs CSV file', required=False)
-    parser.add_argument('-elev','--usgs_elev_flag', help='Perform aggregate on branch usgs elev tables',
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-htable','--hydro_table_flag', help='Perform aggregate on branch hydrotables',
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-src','--src_cross_flag', help='Perform aggregate on branch src crosswalk files',
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-ras','--ras_elev_flag', help='Perform aggregate on branch ras2fim elev tables',
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-j','--num_job_workers', help='Number of processes to use',
-                        required=False, default=1, type=int)
+    parser.add_argument('-fim', '--fim_directory', help='Input FIM Directory', required=True)
+    parser.add_argument('-i', '--fim_inputs', help='Input fim_inputs CSV file', required=False)
+    parser.add_argument(
+        '-elev',
+        '--usgs_elev_flag',
+        help='Perform aggregate on branch usgs elev tables',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-htable',
+        '--hydro_table_flag',
+        help='Perform aggregate on branch hydrotables',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-src',
+        '--src_cross_flag',
+        help='Perform aggregate on branch src crosswalk files',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-ras',
+        '--ras_elev_flag',
+        help='Perform aggregate on branch ras2fim elev tables',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-j', '--num_job_workers', help='Number of processes to use', required=False, default=1, type=int
+    )
 
     args = vars(parser.parse_args())
 
-    aggregate_by_huc(**args)
\ No newline at end of file
+    aggregate_by_huc(**args)
diff --git a/src/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py
index 7725abc4f..ce3fa2a60 100644
--- a/src/aggregate_fim_outputs.py
+++ b/src/aggregate_fim_outputs.py
@@ -1,49 +1,50 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import csv
+import json
+import os
+import shutil
 from multiprocessing import Pool
+
 import pandas as pd
-import json
 import rasterio
 from rasterio.merge import merge
-from rasterio.warp import calculate_default_transform, reproject, Resampling
-import shutil
-import csv
-from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
+from rasterio.warp import Resampling, calculate_default_transform, reproject
 
+from utils.shared_variables import PREP_PROJECTION, VIZ_PROJECTION
 
-def aggregate_fim_outputs(args):
 
-    fim_out_dir   = args[0]
-    huc6          = args[1]
-    huc_list      = args[2]
+def aggregate_fim_outputs(args):
+    fim_out_dir = args[0]
+    huc6 = args[1]
+    huc_list = args[2]
 
     print(f"aggregating {huc6}")
 
-    huc6_dir = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6))
+    huc6_dir = os.path.join(fim_out_dir, 'aggregate_fim_outputs', str(huc6))
     os.makedirs(huc6_dir, exist_ok=True)
 
     # aggregate file name paths
-    aggregate_hydrotable = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),'hydroTable.csv')
-    aggregate_src = os.path.join(fim_out_dir,'aggregate_fim_outputs',str(huc6),f'rating_curves_{huc6}.json')
+    aggregate_hydrotable = os.path.join(fim_out_dir, 'aggregate_fim_outputs', str(huc6), 'hydroTable.csv')
+    aggregate_src = os.path.join(
+        fim_out_dir, 'aggregate_fim_outputs', str(huc6), f'rating_curves_{huc6}.json'
+    )
 
     for huc in huc_list:
-
         # original file paths
-        hydrotable_filename = os.path.join(fim_out_dir,huc,'hydroTable.csv')
-        src_filename = os.path.join(fim_out_dir,huc,'src.json')
-
-        if len(huc)> 6:
+        hydrotable_filename = os.path.join(fim_out_dir, huc, 'hydroTable.csv')
+        src_filename = os.path.join(fim_out_dir, huc, 'src.json')
 
+        if len(huc) > 6:
             # open hydrotable
             hydrotable = pd.read_csv(hydrotable_filename)
 
             # write/append aggregate hydrotable
             if os.path.isfile(aggregate_hydrotable):
-                hydrotable.to_csv(aggregate_hydrotable,index=False, mode='a',header=False)
+                hydrotable.to_csv(aggregate_hydrotable, index=False, mode='a', header=False)
             else:
-                hydrotable.to_csv(aggregate_hydrotable,index=False)
+                hydrotable.to_csv(aggregate_hydrotable, index=False)
 
             del hydrotable
 
@@ -53,7 +54,6 @@ def aggregate_fim_outputs(args):
 
             # write/append aggregate src
             if os.path.isfile(aggregate_src):
-
                 with open(aggregate_src, "r+") as file:
                     data = json.load(file)
                     data.update(src)
@@ -62,7 +62,7 @@ def aggregate_fim_outputs(args):
                     json.dump(data, outfile)
             else:
                 with open(aggregate_src, 'w') as outfile:
-                        json.dump(src, outfile)
+                    json.dump(src, outfile)
 
             del src
 
@@ -80,7 +80,10 @@ def aggregate_fim_outputs(args):
         csvReader = csv.DictReader(csvf)
 
         for row in csvReader:
-            if row['HydroID'].lstrip('0') in src_data and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]:
+            if (
+                row['HydroID'].lstrip('0') in src_data
+                and 'nwm_feature_id' not in src_data[row['HydroID'].lstrip('0')]
+            ):
                 src_data[row['HydroID'].lstrip('0')]['nwm_feature_id'] = row['feature_id']
 
     # Write src_data to JSON file
@@ -89,44 +92,54 @@ def aggregate_fim_outputs(args):
 
     ## aggregate rasters
     # aggregate file paths
-    rem_mosaic = os.path.join(huc6_dir,f'hand_grid_{huc6}_prepprj.tif')
-    catchment_mosaic = os.path.join(huc6_dir,f'catchments_{huc6}_prepprj.tif')
+    rem_mosaic = os.path.join(huc6_dir, f'hand_grid_{huc6}_prepprj.tif')
+    catchment_mosaic = os.path.join(huc6_dir, f'catchments_{huc6}_prepprj.tif')
 
     if huc6 not in huc_list:
-
         huc6_filter = [path.startswith(huc6) for path in huc_list]
         subset_huc6_list = [i for (i, v) in zip(huc_list, huc6_filter) if v]
 
         # aggregate and mosaic rem
-        rem_list = [os.path.join(fim_out_dir,huc,'rem_zeroed_masked.tif') for huc in subset_huc6_list]
+        rem_list = [os.path.join(fim_out_dir, huc, 'rem_zeroed_masked.tif') for huc in subset_huc6_list]
 
         if len(rem_list) > 1:
-
             rem_files_to_mosaic = []
 
             for rem in rem_list:
-
                 rem_src = rasterio.open(rem)
                 rem_files_to_mosaic.append(rem_src)
 
             mosaic, out_trans = merge(rem_files_to_mosaic)
             out_meta = rem_src.meta.copy()
-            out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
-
-            with rasterio.open(rem_mosaic, "w", **out_meta, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES') as dest:
+            out_meta.update(
+                {
+                    "driver": "GTiff",
+                    "height": mosaic.shape[1],
+                    "width": mosaic.shape[2],
+                    "dtype": str(mosaic.dtype),
+                    "transform": out_trans,
+                    "crs": PREP_PROJECTION,
+                    'compress': 'lzw',
+                }
+            )
+
+            with rasterio.open(
+                rem_mosaic, "w", **out_meta, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES'
+            ) as dest:
                 dest.write(mosaic)
 
-            del rem_files_to_mosaic,rem_src,out_meta,mosaic
-
-        elif len(rem_list)==1:
+            del rem_files_to_mosaic, rem_src, out_meta, mosaic
 
+        elif len(rem_list) == 1:
             shutil.copy(rem_list[0], rem_mosaic)
 
         # aggregate and mosaic catchments
-        catchment_list = [os.path.join(fim_out_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif') for huc in subset_huc6_list]
+        catchment_list = [
+            os.path.join(fim_out_dir, huc, 'gw_catchments_reaches_filtered_addedAttributes.tif')
+            for huc in subset_huc6_list
+        ]
 
         if len(catchment_list) > 1:
-
             cat_files_to_mosaic = []
 
             for cat in catchment_list:
@@ -136,51 +149,62 @@ def aggregate_fim_outputs(args):
             mosaic, out_trans = merge(cat_files_to_mosaic)
             out_meta = cat_src.meta.copy()
 
-            out_meta.update({"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "dtype": str(mosaic.dtype), "transform": out_trans,"crs": PREP_PROJECTION,'compress': 'lzw'})
-
-            with rasterio.open(catchment_mosaic, "w", **out_meta, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES') as dest:
+            out_meta.update(
+                {
+                    "driver": "GTiff",
+                    "height": mosaic.shape[1],
+                    "width": mosaic.shape[2],
+                    "dtype": str(mosaic.dtype),
+                    "transform": out_trans,
+                    "crs": PREP_PROJECTION,
+                    'compress': 'lzw',
+                }
+            )
+
+            with rasterio.open(
+                catchment_mosaic, "w", **out_meta, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES'
+            ) as dest:
                 dest.write(mosaic)
 
-            del cat_files_to_mosaic,cat_src,out_meta,mosaic
-
-        elif len(catchment_list)==1:
+            del cat_files_to_mosaic, cat_src, out_meta, mosaic
 
+        elif len(catchment_list) == 1:
             shutil.copy(catchment_list[0], catchment_mosaic)
 
     else:
         # original file paths
-        rem_filename = os.path.join(fim_out_dir,huc6,'rem_zeroed_masked.tif')
-        catchment_filename = os.path.join(fim_out_dir,huc6,'gw_catchments_reaches_filtered_addedAttributes.tif')
+        rem_filename = os.path.join(fim_out_dir, huc6, 'rem_zeroed_masked.tif')
+        catchment_filename = os.path.join(
+            fim_out_dir, huc6, 'gw_catchments_reaches_filtered_addedAttributes.tif'
+        )
 
         shutil.copy(rem_filename, rem_mosaic)
         shutil.copy(catchment_filename, catchment_mosaic)
 
     ## reproject rasters
-    reproject_raster(rem_mosaic,VIZ_PROJECTION)
+    reproject_raster(rem_mosaic, VIZ_PROJECTION)
     os.remove(rem_mosaic)
 
-    reproject_raster(catchment_mosaic,VIZ_PROJECTION)
+    reproject_raster(catchment_mosaic, VIZ_PROJECTION)
     os.remove(catchment_mosaic)
 
 
-def reproject_raster(raster_name,reprojection):
-
+def reproject_raster(raster_name, reprojection):
     with rasterio.open(raster_name) as src:
         transform, width, height = calculate_default_transform(
-            src.crs, reprojection, src.width, src.height, *src.bounds)
+            src.crs, reprojection, src.width, src.height, *src.bounds
+        )
         kwargs = src.meta.copy()
-        kwargs.update({
-            'crs': reprojection,
-            'transform': transform,
-            'width': width,
-            'height': height,
-            'compress': 'lzw'
-        })
+        kwargs.update(
+            {'crs': reprojection, 'transform': transform, 'width': width, 'height': height, 'compress': 'lzw'}
+        )
 
         raster_proj_rename = os.path.split(raster_name)[1].replace('_prepprj.tif', '.tif')
         raster_proj_dir = os.path.join(os.path.dirname(raster_name), raster_proj_rename)
 
-        with rasterio.open(raster_proj_dir, 'w', **kwargs, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES') as dst:
+        with rasterio.open(
+            raster_proj_dir, 'w', **kwargs, tiled=True, blockxsize=1024, blockysize=1024, BIGTIFF='YES'
+        ) as dst:
             # for i in range(1, src.count + 1):
             reproject(
                 source=rasterio.band(src, 1),
@@ -189,16 +213,22 @@ def reproject_raster(raster_name,reprojection):
                 src_crs=src.crs,
                 dst_transform=transform,
                 dst_crs=reprojection,
-                resampling=Resampling.nearest)
+                resampling=Resampling.nearest,
+            )
     del src, dst
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='Aggregate layers buy HUC6')
-    parser.add_argument('-d','--fim-outputs-directory', help='FIM outputs directory', required=True)
-    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
-
+    parser.add_argument('-d', '--fim-outputs-directory', help='FIM outputs directory', required=True)
+    parser.add_argument(
+        '-j',
+        '--number-of-jobs',
+        help='Number of processes to use. Default is 1.',
+        required=False,
+        default="1",
+        type=int,
+    )
 
     args = vars(parser.parse_args())
 
@@ -213,10 +243,9 @@ def reproject_raster(raster_name,reprojection):
     procs_list = []
 
     for huc6 in huc6_list:
-
         limited_huc_list = [huc for huc in huc_list if huc.startswith(huc6)]
 
-        procs_list.append([fim_outputs_directory,huc6,limited_huc_list])
+        procs_list.append([fim_outputs_directory, huc6, limited_huc_list])
 
     print(f"aggregating {len(huc_list)} hucs to HUC6 scale using {number_of_jobs} jobs")
     with Pool(processes=number_of_jobs) as pool:
diff --git a/src/aggregate_vector_inputs.py b/src/aggregate_vector_inputs.py
deleted file mode 100755
index 3b2bdef97..000000000
--- a/src/aggregate_vector_inputs.py
+++ /dev/null
@@ -1,355 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import pandas as pd
-import geopandas as gpd
-from utils.shared_variables import PREP_PROJECTION
-from utils.shared_functions import getDriver
-from reduce_nhd_stream_density import subset_nhd_network
-from adjust_headwater_streams import adjust_headwaters
-from shapely.geometry import Point
-from concurrent.futures import ProcessPoolExecutor
-from collections import deque
-import numpy as np
-from shapely.wkb import dumps, loads
-import pygeos
-
-
-
-
-def identify_nwm_ms_streams(nwm_streams_filename,ahps_filename,nwm_streams_all_filename):
-
-    # Subset nwm network to ms
-    ahps_headwaters = gpd.read_file(ahps_filename)
-
-    nwm_streams = gpd.read_file(nwm_streams_filename)
-
-    # Remove mainstem column if it already exists
-    nwm_streams = nwm_streams.drop(['mainstem'], axis=1, errors='ignore')
-
-    nwm_streams['is_headwater'] = False
-
-    nwm_streams.loc[nwm_streams.ID.isin(list(ahps_headwaters.nwm_featur)),'is_headwater'] = True
-
-    # Subset NHDPlus HR
-    nwm_streams['is_relevant_stream'] = nwm_streams['is_headwater'].copy()
-
-    nwm_streams = nwm_streams.explode(index_parts=True)
-
-    # Trace down from headwaters
-    nwm_streams.set_index('ID',inplace=True,drop=False)
-
-    Q = deque(nwm_streams.loc[nwm_streams['is_headwater'],'ID'].tolist())
-    visited = set()
-
-    while Q:
-        q = Q.popleft()
-        if q in visited:
-            continue
-
-        visited.add(q)
-        toNode = nwm_streams.loc[q,'to']
-
-        if not toNode == 0:
-
-            nwm_streams.loc[nwm_streams.ID==toNode,'is_relevant_stream'] = True
-
-            if toNode not in visited:
-                Q.append(toNode)
-
-    nwm_streams_ms = nwm_streams.loc[nwm_streams['is_relevant_stream'],:]
-    ms_segments = nwm_streams_ms.ID.to_list()
-
-    nwm_streams.reset_index(drop=True,inplace=True)
-
-    # Add column to FR nwm layer to indicate MS segments
-    nwm_streams['mainstem'] = np.where(nwm_streams.ID.isin(ms_segments), 1, 0)
-
-    nwm_streams = nwm_streams.drop(['is_relevant_stream','is_headwater'], axis=1, errors='ignore')
-
-    nwm_streams.to_file(nwm_streams_all_filename,driver=getDriver(nwm_streams_all_filename),index=False,layer='nwm_streams')
-
-    return ms_segments
-
-
-def find_nwm_incoming_streams(nwm_streams_,wbd,huc_unit):
-
-    # Input wbd
-    if isinstance(wbd,str):
-
-        layer = f"WBDHU{huc_unit}"
-        wbd = gpd.read_file(wbd, layer=layer)
-    elif isinstance(wbd,gpd.GeoDataFrame):
-        pass
-    else:
-        raise TypeError("Pass dataframe or filepath for wbd")
-
-    intersecting_points = []
-    nhdplus_ids = []
-    mainstem_flag = []
-    print (f"iterating through {len(wbd)} hucs")
-    for index, row in wbd.iterrows():
-
-        col_name = f"HUC{huc_unit}"
-        huc = row[col_name]
-        huc_mask = wbd.loc[wbd[col_name]==str(huc)]
-        huc_mask = huc_mask.explode(index_parts=True)
-        huc_mask = huc_mask.reset_index(drop=True)
-
-        # Input nwm streams
-        if isinstance(nwm_streams_,str):
-            nwm_streams = gpd.read_file(nwm_streams_, mask=huc_mask)
-        elif isinstance(nwm_streams_,gpd.GeoDataFrame):
-            nwm_streams = nwm_streams_.copy()
-        else:
-            raise TypeError("Pass dataframe or filepath for nwm streams")
-
-        nwm_streams = nwm_streams.explode(index_parts=True)
-        nwm_streams = nwm_streams.reset_index(drop=True)
-
-        for index, polygon in enumerate(huc_mask.geometry):
-
-            crosses=nwm_streams.crosses(polygon.exterior)
-            nwm_streams_subset =nwm_streams[crosses]
-            nwm_streams_subset = nwm_streams_subset.reset_index(drop=True)
-
-            for index, segment in nwm_streams_subset.iterrows():
-                distances = []
-
-                try:
-                    nhdplus_id = segment.ID
-                except:
-                    nhdplus_id = segment.NHDPlusID
-
-                linestring = segment.geometry
-                mainstem = segment.mainstem
-
-                # Distance to each stream segment
-                for point in zip(*linestring.coords.xy):
-                    distance = Point(point).distance(polygon.exterior)
-                    distances = distances + [distance]
-
-                # Find minimum distance
-                min_index = np.argmin(distances)
-
-                # Closest segment to headwater
-                closest_point = list(linestring.coords)[min_index]
-                last_node = Point(closest_point)
-
-                # Convert geometries to WKB representation
-                wkb_point = dumps(last_node)
-                wkb_poly = dumps(polygon.exterior)
-
-                # Create pygeos geometries from WKB representation
-                stream_point_geom = pygeos.io.from_wkb(wkb_point)
-                polybin_geom = pygeos.io.from_wkb(wkb_poly)
-
-                # Linear reference end node to huc boundary
-                pointdistancetoline = pygeos.linear.line_locate_point(polybin_geom,stream_point_geom)
-                referencedpoint = pygeos.linear.line_interpolate_point(polybin_geom, pointdistancetoline)
-
-                # Convert geometries to wkb representation
-                bin_referencedpoint = pygeos.io.to_wkb(referencedpoint)
-
-                # Convert to shapely geometries
-                shply_referencedpoint = loads(bin_referencedpoint)
-
-                # Collect all nhd stream segment linestring verticies
-                intersecting_points = intersecting_points + [shply_referencedpoint]
-                nhdplus_ids = nhdplus_ids + [nhdplus_id]
-                mainstem_flag = mainstem_flag + [mainstem]
-
-        del huc_mask
-
-    huc_intersection = gpd.GeoDataFrame({'geometry': intersecting_points, 'NHDPlusID': nhdplus_ids,'mainstem': mainstem_flag},crs=nwm_streams.crs,geometry='geometry')
-    huc_intersection = huc_intersection.drop_duplicates()
-
-    del nwm_streams,wbd
-
-    return huc_intersection
-
-
-def collect_stream_attributes(nhdplus_vectors_dir, huc):
-
-    print (f"Starting attribute collection for HUC {huc}",flush=True)
-
-    # Collecting NHDPlus HR attributes
-    burnline_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')
-    vaa_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusFlowLineVAA' + str(huc) + '.gpkg')
-    flowline_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDFlowline' + str(huc) + '.gpkg')
-
-    if os.path.exists(os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '.gpkg')):
-
-        burnline = gpd.read_file(burnline_filename)
-        burnline = burnline[['NHDPlusID','ReachCode','geometry']]
-        flowline = gpd.read_file(flowline_filename)
-        flowline = flowline[['NHDPlusID','FType','FCode']]
-        # flowline = flowline.loc[flowline["FType"].isin([334,420,428,460,558])]
-        flowline = flowline.loc[~flowline["FType"].isin([566,420])]
-
-        nhd_streams_vaa = gpd.read_file(vaa_filename)
-        nhd_streams_vaa = nhd_streams_vaa[['FromNode','ToNode','NHDPlusID','StreamOrde','DnLevelPat','LevelPathI']]
-        nhd_streams = burnline.merge(nhd_streams_vaa,on='NHDPlusID',how='inner')
-        nhd_streams = nhd_streams.merge(flowline,on='NHDPlusID',how='inner')
-
-        del burnline, flowline, nhd_streams_vaa
-
-        nhd_streams = nhd_streams.to_crs(PREP_PROJECTION)
-        nhd_streams = nhd_streams.loc[nhd_streams.geometry!=None,:] # special case: remove segments without geometries
-        nhd_streams['HUC4'] = str(huc)
-
-        # special case; breach in network at Tiber Dam
-        if huc == '1003' and nhd_streams.loc[nhd_streams.NHDPlusID==23001300078682.0,'DnLevelPat'] == 23001300001574.0:
-            nhd_streams = nhd_streams.loc[nhd_streams.NHDPlusID!=23001300009084.0]
-            nhd_streams.loc[nhd_streams.NHDPlusID==23001300078682.0,'DnLevelPat'] = 23001300001566.0
-
-        # Write out NHDPlus HR aggregated
-        nhd_streams_agg_fileName = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
-        nhd_streams.to_file(nhd_streams_agg_fileName,driver=getDriver(nhd_streams_agg_fileName),index=False)
-
-        del nhd_streams
-
-        print (f"finished attribute collection for HUC {huc}",flush=True)
-
-    else:
-        print (f"missing data for HUC {huc}",flush=True)
-
-
-def subset_stream_networks(args, huc):
-
-    nwm_headwaters_filename            = args[0]
-    ahps_filename                      = args[1]
-    wbd4                               = args[2]
-    wbd8                               = args[3]
-    nhdplus_vectors_dir                = args[4]
-    nwm_huc4_intersections_filename    = args[5]
-
-    print(f"starting stream subset for HUC {huc}",flush=True)
-    nwm_headwater_id = 'ID'
-    ahps_headwater_id = 'nws_lid'
-    headwater_pts_id = 'site_id'
-
-    column_order = ['pt_type', headwater_pts_id, 'mainstem', 'geometry']
-    nhd_streams_filename = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
-
-    # Subset to reduce footprint
-    selected_wbd4 = wbd4.loc[wbd4.HUC4.str.startswith(huc)]
-    del wbd4
-    selected_wbd8 = wbd8.loc[wbd8.HUC8.str.startswith(huc)]
-    del wbd8
-
-    huc_mask = selected_wbd4.loc[selected_wbd4.HUC4.str.startswith(huc)]
-    huc_mask = huc_mask.explode(index_parts=True)
-    huc_mask = huc_mask.reset_index(drop=True)
-
-    if len(selected_wbd8.HUC8) > 0:
-
-        selected_wbd8 = selected_wbd8.reset_index(drop=True)
-
-        # Identify FR/NWM headwaters and subset HR network
-        try:
-            nhd_streams_fr = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_filename,nwm_headwaters_filename,nwm_headwater_id,nwm_huc4_intersections_filename)
-        except:
-            print (f"Error subsetting NHD HR network for HUC {huc}",flush=True)
-
-        # Identify nhd mainstem streams
-        try:
-            nhd_streams_all = subset_nhd_network(huc,huc_mask,selected_wbd8,nhd_streams_fr,ahps_filename,ahps_headwater_id,nwm_huc4_intersections_filename,True)
-        except:
-            print (f"Error identifing MS network for HUC {huc}",flush=True)
-
-        # Identify HUC8 intersection points
-        nhd_huc8_intersections = find_nwm_incoming_streams(nhd_streams_all,selected_wbd8,8)
-
-        # Load nwm headwaters
-        nwm_headwaters = gpd.read_file(nwm_headwaters_filename, mask=huc_mask)
-        nwm_headwaters['pt_type'] = 'nwm_headwater'
-        nwm_headwaters = nwm_headwaters.rename(columns={"ID": headwater_pts_id})
-
-        # Load nws lids
-        nws_lids = gpd.read_file(ahps_filename, mask=huc_mask)
-        nws_lids = nws_lids.drop(columns=['name','nwm_feature_id','usgs_site_code','states','HUC8','is_headwater', 'is_colocated'])
-        nws_lids = nws_lids.rename(columns={"nws_lid": headwater_pts_id})
-        nws_lids['pt_type'] = 'nws_lid'
-        nws_lids['mainstem'] = True
-
-        if (len(nwm_headwaters) > 0) or (len(nws_lids) > 0):
-
-            # Adjust FR/NWM headwater segments
-            adj_nhd_streams_all, adj_nhd_headwater_points = adjust_headwaters(huc,nhd_streams_all,nwm_headwaters,nws_lids,headwater_pts_id)
-
-            adj_nhd_headwater_points = adj_nhd_headwater_points[column_order]
-            nhd_huc8_intersections['pt_type'] = 'nhd_huc8_intersections'
-            nhd_huc8_intersections = nhd_huc8_intersections.rename(columns={"NHDPlusID": headwater_pts_id})
-            nhd_huc8_intersections = nhd_huc8_intersections[column_order]
-            adj_nhd_headwater_points_all = pd.concat([adj_nhd_headwater_points, nhd_huc8_intersections])
-            adj_nhd_headwater_points_all = adj_nhd_headwater_points_all.reset_index(drop=True)
-
-            adj_nhd_streams_all_fileName = os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
-            adj_nhd_headwaters_all_fileName = os.path.join(nhdplus_vectors_dir,huc,'nhd' + str(huc) + '_headwaters_adj.gpkg')
-
-            # Write out FR adjusted
-            adj_nhd_streams_all.to_file(adj_nhd_streams_all_fileName,driver=getDriver(adj_nhd_streams_all_fileName),index=False)
-            adj_nhd_headwater_points_all.to_file(adj_nhd_headwaters_all_fileName,driver=getDriver(adj_nhd_headwaters_all_fileName),index=False)
-
-            del adj_nhd_streams_all, adj_nhd_headwater_points_all
-
-        else:
-
-            print (f"skipping headwater adjustments for HUC {huc}")
-
-        del nhd_streams_fr
-
-    print(f"finished stream subset for HUC {huc}",flush=True)
-
-
-def clean_up_intermediate_files(nhdplus_vectors_dir):
-
-    for huc in os.listdir(nhdplus_vectors_dir):
-
-        agg_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_agg.gpkg')
-        streams_adj_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
-        headwater_adj_path= os.path.join(nhdplus_vectors_dir,huc,'nhd' + str(huc) + '_headwaters_adj.gpkg')
-
-        if os.path.exists(agg_path):
-            os.remove(agg_path)
-
-        if os.path.exists(streams_adj_path):
-            os.remove(streams_adj_path)
-
-        if os.path.exists(headwater_adj_path):
-            os.remove(headwater_adj_path)
-
-
-if __name__ == '__main__':
-
-    # # Generate NWM Headwaters
-
-    print ('loading HUC4s')
-    wbd4 = gpd.read_file(wbd_filename, layer='WBDHU4')
-    print ('loading HUC8s')
-    wbd8 = gpd.read_file(wbd_filename, layer='WBDHU8')
-
-    subset_arg_list = (nwm_headwaters_filename,ahps_filename,wbd4,wbd8,nhdplus_vectors_dir,nwm_huc4_intersections_filename)
-    huc_list = os.listdir(nhdplus_vectors_dir)
-
-    missing_subsets = []
-    for huc in os.listdir(nhdplus_vectors_dir):
-        streams_adj_path= os.path.join(nhdplus_vectors_dir,huc,'NHDPlusBurnLineEvent' + str(huc) + '_adj.gpkg')
-        if not os.path.isfile(streams_adj_path):
-            missing_subsets = missing_subsets + [huc]
-
-
-    print (f"Subsetting stream network for {len(missing_subsets)} HUC4s")
-    num_workers=11
-
-    with ProcessPoolExecutor(max_workers=num_workers) as executor:
-        # Preprocess nhd hr and add attributes
-        # collect_attributes = [executor.submit(collect_stream_attributes, nhdplus_vectors_dir, str(huc)) for huc in huc_list]
-        # Subset nhd hr network
-        subset_results = [executor.submit(subset_stream_networks, subset_arg_list, str(huc)) for huc in missing_subsets]
-
-    del wbd4,wbd8
-
-    # Remove intermediate files
-    # clean_up_intermediate_files(nhdplus_vectors_dir)
diff --git a/src/agreedem.py b/src/agreedem.py
index 88e293fcd..2aae80647 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -1,21 +1,35 @@
 #!/usr/bin/env python3
-import rasterio
-import numpy as np
-import os
 import argparse
+import os
+
+import numpy as np
+import rasterio
 import whitebox
+
 from utils.shared_functions import mem_profile
 
 
 @mem_profile
-def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data):
+def agreedem(
+    rivers_raster,
+    dem,
+    output_raster,
+    workspace,
+    buffer_dist,
+    smooth_drop,
+    sharp_drop,
+    delete_intermediate_data,
+):
     '''
-    Produces a hydroconditioned raster using the AGREE DEM methodology as described by Ferdi Hellweger (https://www.caee.utexas.edu/prof/maidment/gishydro/ferdi/research/agree/agree.html). Whiteboxtools is used to calculate intermediate allocation and proximity rasters.
+    Produces a hydroconditioned raster using the AGREE DEM methodology as described by Ferdi Hellweger
+    (https://www.caee.utexas.edu/prof/maidment/gishydro/ferdi/research/agree/agree.html).
+    Whiteboxtools is used to calculate intermediate allocation and proximity rasters.
 
     Parameters
     ----------
     rivers_raster : STR
-        Path to the river raster. River cells = 1 and non-river cells = 0. For example, see flows_grid_boolean.tif.
+        Path to the river raster. River cells = 1 and non-river cells = 0.
+        For example, see flows_grid_boolean.tif.
     dem : STR
         Elevation DEM (units assumed to be in meters). For example, see dem_meters.tif.
     output_raster : STR
@@ -39,8 +53,8 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
     # Set wbt envs
     wbt = whitebox.WhiteboxTools()
     wbt.set_verbose_mode(False)
-    
-    #------------------------------------------------------------------
+
+    # ------------------------------------------------------------------
     # 1. From Hellweger documentation: Compute the vector grid
     # (vectgrid). The cells in the vector grid corresponding to the
     # lines in the vector coverage have data. All other cells have no
@@ -52,25 +66,25 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
 
         # Define smogrid profile and output file
         smo_profile = dem_profile.copy()
-        smo_profile.update(nodata = 0)
-        smo_profile.update(dtype = 'float32')
+        smo_profile.update(nodata=0)
+        smo_profile.update(dtype='float32')
         smo_output = os.path.join(workspace, 'agree_smogrid.tif')
-        vectdist_grid = os.path.join(workspace,'agree_smogrid_dist.tif')
-        vectallo_grid = os.path.join(workspace,'agree_smogrid_allo.tif')
-        
+        vectdist_grid = os.path.join(workspace, 'agree_smogrid_dist.tif')
+        vectallo_grid = os.path.join(workspace, 'agree_smogrid_allo.tif')
+
         # Windowed reading/calculating/writing
         with rasterio.Env():
             with rasterio.open(smo_output, 'w', **smo_profile) as raster:
                 for ji, window in elev.block_windows(1):
                     # read elevation data and mask information
-                    elev_data_window = elev.read(1, window = window)
-                    elev_mask_window = elev.read_masks(1, window = window).astype('bool')
+                    elev_data_window = elev.read(1, window=window)
+                    elev_mask_window = elev.read_masks(1, window=window).astype('bool')
                     # Import boolean river raster and apply same NODATA mask as dem
                     # layer. In case rivers extend beyond valid data regions of DEM.
-                    river_raw_data_window = rivers.read(1, window = window)
+                    river_raw_data_window = rivers.read(1, window=window)
                     river_data_window = np.where(elev_mask_window == True, river_raw_data_window, 0)
 
-                    #---------------------------------------------------------------
+                    # ---------------------------------------------------------------
                     # 2. From Hellweger documentation: Compute the smooth drop/raise
                     # grid (smogrid). The cells in the smooth drop/raise grid
                     # corresponding to the vector lines have an elevation equal to that
@@ -78,13 +92,13 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
                     # (smoothdist). All other cells have no data.
 
                     # Assign smooth distance and calculate the smogrid.
-                    smooth_dist = -1 * smooth_drop # in meters.
-                    smogrid_window = river_data_window*(elev_data_window + smooth_dist)
+                    smooth_dist = -1 * smooth_drop  # in meters.
+                    smogrid_window = river_data_window * (elev_data_window + smooth_dist)
 
                     # Write out raster
-                    raster.write(smogrid_window.astype('float32'), indexes = 1, window = window)
+                    raster.write(smogrid_window.astype('float32'), indexes=1, window=window)
 
-        #------------------------------------------------------------------
+        # ------------------------------------------------------------------
         # 3. From Hellweger documentation: Compute the vector distance grids
         # (vectdist and vectallo). The cells in the vector distance grid
         # (vectdist) store the distance to the closest vector cell. The
@@ -93,11 +107,11 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
 
         # Compute allocation and proximity grid using WhiteboxTools
         smo_output_zerod = os.path.join(workspace, 'agree_smogrid_zerod.tif')
-        wbt.euclidean_distance(rivers_raster,vectdist_grid)
-        wbt.convert_nodata_to_zero(smo_output,smo_output_zerod)
-        wbt.euclidean_allocation(smo_output_zerod,vectallo_grid)
+        wbt.euclidean_distance(rivers_raster, vectdist_grid)
+        wbt.convert_nodata_to_zero(smo_output, smo_output_zerod)
+        wbt.euclidean_allocation(smo_output_zerod, vectallo_grid)
 
-        #------------------------------------------------------------------
+        # ------------------------------------------------------------------
         # 4. From Hellweger documentation: Compute the buffer grid
         # (bufgrid2). The cells in the buffer grid outside the buffer
         # distance (buffer) store the original elevation. The cells in the
@@ -105,37 +119,37 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
 
         # Open distance, allocation, elevation grids.
         with rasterio.open(vectdist_grid) as vectdist:
-
             # Define bufgrid profile and output file.
             buf_output = os.path.join(workspace, 'agree_bufgrid.tif')
-            bufdist_grid = os.path.join(workspace,'agree_bufgrid_dist.tif')
-            bufallo_grid = os.path.join(workspace,'agree_bufgrid_allo.tif')
+            bufdist_grid = os.path.join(workspace, 'agree_bufgrid_dist.tif')
+            bufallo_grid = os.path.join(workspace, 'agree_bufgrid_allo.tif')
             buf_profile = dem_profile.copy()
-            buf_profile.update(dtype = 'float32')
+            buf_profile.update(dtype='float32')
 
             # Windowed reading/calculating/writing
             with rasterio.Env():
                 with rasterio.open(buf_output, 'w', **buf_profile) as raster:
                     for ji, window in elev.block_windows(1):
                         # read distance, allocation, and elevation datasets
-                        vectdist_data_window = vectdist.read(1, window = window)
-                        elev_data_window = elev.read(1, window = window)
+                        vectdist_data_window = vectdist.read(1, window=window)
+                        elev_data_window = elev.read(1, window=window)
 
                         # Define buffer distance and calculate adjustment to compute the
                         # bufgrid.
                         # half_res adjustment equal to half distance of one cell
-                        half_res = elev.res[0]/2
-                        final_buffer = buffer_dist - half_res # assume all units in meters.
+                        half_res = elev.res[0] / 2
+                        final_buffer = buffer_dist - half_res  # assume all units in meters.
 
                         # Calculate bufgrid. Assign NODATA to areas where vectdist_data <=
                         # buffered value.
-                        bufgrid_window = np.where(vectdist_data_window > final_buffer, elev_data_window, dem_profile['nodata'])
+                        bufgrid_window = np.where(
+                            vectdist_data_window > final_buffer, elev_data_window, dem_profile['nodata']
+                        )
 
                         # Write out raster.
-                        raster.write(bufgrid_window.astype('float32'), indexes = 1, window = window)
+                        raster.write(bufgrid_window.astype('float32'), indexes=1, window=window)
 
-        
-            #------------------------------------------------------------------
+            # ------------------------------------------------------------------
             # 5. From Hellweger documentation: Compute the buffer distance grids
             # (bufdist and bufallo). The cells in the buffer distance grid
             # (bufdist) store the distance to the closest valued buffer grid
@@ -147,55 +161,64 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
             with rasterio.open(buf_output) as agree_bufgrid:
                 agree_bufgrid_profile = agree_bufgrid.profile
                 bin_buf_output_profile = agree_bufgrid_profile.copy()
-                bin_buf_output_profile.update(dtype = 'float32')
+                bin_buf_output_profile.update(dtype='float32')
 
                 with rasterio.Env():
                     with rasterio.open(bin_buf_output, 'w', **bin_buf_output_profile) as raster:
                         for ji, window in agree_bufgrid.block_windows(1):
                             # read distance, allocation, and elevation datasets
-                            agree_bufgrid_data_window = agree_bufgrid.read(1, window = window)
+                            agree_bufgrid_data_window = agree_bufgrid.read(1, window=window)
 
                             # Calculate bufgrid. Assign NODATA to areas where vectdist_data <=
-                            agree_bufgrid_data_window = np.where(agree_bufgrid_data_window>-10000, 1, 0)
+                            agree_bufgrid_data_window = np.where(agree_bufgrid_data_window > -10000, 1, 0)
 
                             # Write out raster.
-                            raster.write(agree_bufgrid_data_window.astype('float32'), indexes = 1, window = window)
+                            raster.write(
+                                agree_bufgrid_data_window.astype('float32'), indexes=1, window=window
+                            )
 
             # Compute allocation and proximity grid using WhiteboxTools
             buf_output_zerod = os.path.join(workspace, 'agree_bufgrid_zerod.tif')
-            wbt.euclidean_distance(bin_buf_output,bufdist_grid)
-            wbt.convert_nodata_to_zero(buf_output,buf_output_zerod)
-            wbt.euclidean_allocation(buf_output_zerod,bufallo_grid)
+            wbt.euclidean_distance(bin_buf_output, bufdist_grid)
+            wbt.convert_nodata_to_zero(buf_output, buf_output_zerod)
+            wbt.euclidean_allocation(buf_output_zerod, bufallo_grid)
 
             # Open distance, allocation, elevation grids.
-            with rasterio.open(bufdist_grid) as bufdist, rasterio.open(bufallo_grid) as bufallo, rasterio.open(vectallo_grid) as vectallo:
-
+            with rasterio.open(bufdist_grid) as bufdist, rasterio.open(
+                bufallo_grid
+            ) as bufallo, rasterio.open(vectallo_grid) as vectallo:
                 # Define profile output file.
                 agree_output = output_raster
                 agree_profile = dem_profile.copy()
-                agree_profile.update(dtype = 'float32')
+                agree_profile.update(dtype='float32')
 
                 # Windowed reading/calculating/writing
                 with rasterio.Env():
                     with rasterio.open(agree_output, 'w', **agree_profile) as raster:
                         for ji, window in elev.block_windows(1):
                             # Read elevation data and mask, distance and allocation grids, and river data.
-                            elev_data_window = elev.read(1, window = window)
-                            elev_mask_window = elev.read_masks(1, window = window).astype('bool')
-                            bufdist_data_window = bufdist.read(1, window = window)
-                            bufallo_data_window = bufallo.read(1, window = window)
-                            vectdist_data_window = vectdist.read(1, window = window)
-                            vectallo_data_window = vectallo.read(1, window = window)
-                            river_raw_data_window = rivers.read(1, window = window)
-
-                            bufallo_data_window = np.where(bufallo_data_window == -32768., elev_data_window, bufallo_data_window)
-
-                            vectallo_data_window = np.where(vectallo_data_window == -32768., elev_data_window-10, vectallo_data_window)
+                            elev_data_window = elev.read(1, window=window)
+                            elev_mask_window = elev.read_masks(1, window=window).astype('bool')
+                            bufdist_data_window = bufdist.read(1, window=window)
+                            bufallo_data_window = bufallo.read(1, window=window)
+                            vectdist_data_window = vectdist.read(1, window=window)
+                            vectallo_data_window = vectallo.read(1, window=window)
+                            river_raw_data_window = rivers.read(1, window=window)
+
+                            bufallo_data_window = np.where(
+                                bufallo_data_window == -32768.0, elev_data_window, bufallo_data_window
+                            )
+
+                            vectallo_data_window = np.where(
+                                vectallo_data_window == -32768.0, elev_data_window - 10, vectallo_data_window
+                            )
 
                             river_raw_data_window = river_raw_data_window.astype(np.float32)
 
-                            river_data_window = np.where(elev_mask_window == True, river_raw_data_window, -20.0)
-                            #------------------------------------------------------------------
+                            river_data_window = np.where(
+                                elev_mask_window == True, river_raw_data_window, -20.0
+                            )
+                            # ------------------------------------------------------------------
                             # 6. From Hellweger documentation: Compute the smooth modified
                             # elevation grid (smoelev). The cells in the smooth modified
                             # elevation grid store the results of the smooth surface
@@ -203,9 +226,16 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
                             # equation below assigns the original elevation.
 
                             # Calculate smoelev.
-                            smoelev_window = vectallo_data_window + ((bufallo_data_window - vectallo_data_window)/(bufdist_data_window + vectdist_data_window)) * vectdist_data_window
-
-                            #------------------------------------------------------------------
+                            smoelev_window = (
+                                vectallo_data_window
+                                + (
+                                    (bufallo_data_window - vectallo_data_window)
+                                    / (bufdist_data_window + vectdist_data_window)
+                                )
+                                * vectdist_data_window
+                            )
+
+                            # ------------------------------------------------------------------
                             # 7. From Hellweger documentation: Compute the sharp drop/raise grid
                             # (shagrid). The cells in the sharp drop/raise grid corresponding to
                             # the vector lines have an elevation equal to that of the smooth
@@ -214,10 +244,10 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
 
                             # Define sharp drop distance and calculate the sharp drop grid where
                             # only river cells are dropped by the sharp_dist amount.
-                            sharp_dist = -1 * sharp_drop # in meters.
+                            sharp_dist = -1 * sharp_drop  # in meters.
                             shagrid_window = (smoelev_window + sharp_dist) * river_data_window
 
-                            #------------------------------------------------------------------
+                            # ------------------------------------------------------------------
                             # 8. From Hellweger documentation: Compute the modified elevation
                             # grid (elevgrid). The cells in the modified elevation grid store
                             # the results of the surface reconditioning process. Note that for
@@ -227,12 +257,13 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
                             # Merge sharp drop grid with smoelev grid. Then apply the same
                             # NODATA mask as original elevation grid.
                             elevgrid_window = np.where(river_data_window == 0, smoelev_window, shagrid_window)
-                            agree_dem_window = np.where(elev_mask_window == True, elevgrid_window, dem_profile['nodata'])
+                            agree_dem_window = np.where(
+                                elev_mask_window == True, elevgrid_window, dem_profile['nodata']
+                            )
 
                             # Write out to raster
-                            raster.write(agree_dem_window.astype('float32'), indexes = 1, window = window)
+                            raster.write(agree_dem_window.astype('float32'), indexes=1, window=window)
 
-            
     # If the '-t' flag is called, intermediate data is removed.
     if delete_intermediate_data:
         os.remove(smo_output)
@@ -247,19 +278,20 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
 
 
 if __name__ == '__main__':
-
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Calculate AGREE DEM')
-    parser.add_argument('-r', '--rivers', help = 'flows grid boolean layer', required = True)
-    parser.add_argument('-d', '--dem_m',  help = 'DEM raster in meters', required = True)
-    parser.add_argument('-w', '--workspace', help = 'Workspace', required = True)
-    parser.add_argument('-o',  '--output', help = 'Path to output raster', required = True)
-    parser.add_argument('-b',  '--buffer', help = 'Buffer distance (m) on either side of channel', required = True)
-    parser.add_argument('-sm', '--smooth', help = 'Smooth drop (m)', required = True)
-    parser.add_argument('-sh', '---sharp', help = 'Sharp drop (m)', required = True)
-    parser.add_argument('-t',  '--del',  help = 'Optional flag to delete intermediate datasets', action = 'store_true')
-
-    #Extract to dictionary and assign to variables.
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Calculate AGREE DEM')
+    parser.add_argument('-r', '--rivers', help='flows grid boolean layer', required=True)
+    parser.add_argument('-d', '--dem_m', help='DEM raster in meters', required=True)
+    parser.add_argument('-w', '--workspace', help='Workspace', required=True)
+    parser.add_argument('-o', '--output', help='Path to output raster', required=True)
+    parser.add_argument('-b', '--buffer', help='Buffer distance (m) on either side of channel', required=True)
+    parser.add_argument('-sm', '--smooth', help='Smooth drop (m)', required=True)
+    parser.add_argument('-sh', '---sharp', help='Sharp drop (m)', required=True)
+    parser.add_argument(
+        '-t', '--del', help='Optional flag to delete intermediate datasets', action='store_true'
+    )
+
+    # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
 
     # rename variable inputs
@@ -269,8 +301,17 @@ def agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_d
     output_raster = args['output']
     buffer_dist = float(args['buffer'])
     smooth_drop = float(args['smooth'])
-    sharp_drop =  float(args['sharp'])
+    sharp_drop = float(args['sharp'])
     delete_intermediate_data = args['del']
 
-    #Run agreedem
-    agreedem(rivers_raster, dem, output_raster, workspace, buffer_dist, smooth_drop, sharp_drop, delete_intermediate_data)
+    # Run agreedem
+    agreedem(
+        rivers_raster,
+        dem,
+        output_raster,
+        workspace,
+        buffer_dist,
+        smooth_drop,
+        sharp_drop,
+        delete_intermediate_data,
+    )
diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py
index 58bd58910..181a41d99 100644
--- a/src/associate_levelpaths_with_levees.py
+++ b/src/associate_levelpaths_with_levees.py
@@ -1,13 +1,23 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import os
 import warnings
+
+import geopandas as gpd
 import numpy as np
 import pandas as pd
-import geopandas as gpd
 
-def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str, leveed_areas_filename:str, levelpaths_filename:str, branch_id_attribute:str, levee_buffer:float, out_filename:str):
+
+def associate_levelpaths_with_levees(
+    levees_filename: str,
+    levee_id_attribute: str,
+    leveed_areas_filename: str,
+    levelpaths_filename: str,
+    branch_id_attribute: str,
+    levee_buffer: float,
+    out_filename: str,
+):
     """
     Finds the level path associated with each levee. Ignores level paths that cross a levee exactly once.
 
@@ -26,10 +36,14 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
     levee_buffer: float
         Distance to buffer from levee.
     out_filename: str
-        Path to write output CSV file.    
+        Path to write output CSV file.
     """
 
-    if os.path.exists(levees_filename) and os.path.exists(leveed_areas_filename) and os.path.exists(levelpaths_filename):
+    if (
+        os.path.exists(levees_filename)
+        and os.path.exists(leveed_areas_filename)
+        and os.path.exists(levelpaths_filename)
+    ):
         # Read in geodataframes
         levees = gpd.read_file(levees_filename)
         leveed_areas = gpd.read_file(leveed_areas_filename)
@@ -57,7 +71,9 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
             [leveed_intersected.append(x) for x in leveed_left[f'{levee_id_attribute}_1'].values]
 
             # Associate levees and leveed areas
-            matches_left = np.where(leveed_left[f'{levee_id_attribute}_1']==leveed_left[f'{levee_id_attribute}_2'])[0]
+            matches_left = np.where(
+                leveed_left[f'{levee_id_attribute}_1'] == leveed_left[f'{levee_id_attribute}_2']
+            )[0]
 
             leveed_left = leveed_left.loc[matches_left]
 
@@ -70,7 +86,9 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
             [leveed_intersected.append(x) for x in leveed_right[f'{levee_id_attribute}_1'].values]
 
             # Associate levees and leveed areas
-            matches_right = np.where(leveed_right[f'{levee_id_attribute}_1']==leveed_right[f'{levee_id_attribute}_2'])[0]
+            matches_right = np.where(
+                leveed_right[f'{levee_id_attribute}_1'] == leveed_right[f'{levee_id_attribute}_2']
+            )[0]
 
             leveed_right = leveed_right.loc[matches_right]
 
@@ -85,28 +103,32 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
         # Merge left and right levee protected areas
         if leveed_left.empty and leveed_right.empty:
             return
-        
+
         elif not leveed_left.empty and not leveed_right.empty:
-            leveed = leveed_left.merge(leveed_right, on=f'{levee_id_attribute}_1', how='outer', suffixes=['_left', '_right'])
+            leveed = leveed_left.merge(
+                leveed_right, on=f'{levee_id_attribute}_1', how='outer', suffixes=['_left', '_right']
+            )
 
             # Set unmatched areas to zero
-            leveed.loc[np.isnan(leveed['leveed_area_left']), 'leveed_area_left'] = 0.
-            leveed.loc[np.isnan(leveed['leveed_area_right']), 'leveed_area_right'] = 0.
+            leveed.loc[np.isnan(leveed['leveed_area_left']), 'leveed_area_left'] = 0.0
+            leveed.loc[np.isnan(leveed['leveed_area_right']), 'leveed_area_right'] = 0.0
 
         elif leveed_left.empty:
             leveed = leveed_right.rename(columns={'leveed_area': 'leveed_area_right'})
-            leveed['leveed_area_left'] = 0.
+            leveed['leveed_area_left'] = 0.0
 
         elif leveed_right.empty:
             leveed = leveed_left.rename(columns={'leveed_area': 'leveed_area_left'})
-            leveed['leveed_area_right'] = 0.
+            leveed['leveed_area_right'] = 0.0
 
         # Determine which side the levee is protecting (opposite of levee protected area)
-        leveed['levee_side'] = np.where(leveed['leveed_area_left'] < leveed['leveed_area_right'], 'left', 'right')
+        leveed['levee_side'] = np.where(
+            leveed['leveed_area_left'] < leveed['leveed_area_right'], 'left', 'right'
+        )
 
         # Split into sides
-        left_ids = leveed.loc[leveed['levee_side']=='left', f'{levee_id_attribute}_1']
-        right_ids = leveed.loc[leveed['levee_side']=='right', f'{levee_id_attribute}_1']
+        left_ids = leveed.loc[leveed['levee_side'] == 'left', f'{levee_id_attribute}_1']
+        right_ids = leveed.loc[leveed['levee_side'] == 'right', f'{levee_id_attribute}_1']
 
         # Associate level paths with levee buffers
         levee_levelpaths_left = gpd.sjoin(levees_buffered_left, levelpaths)
@@ -116,45 +138,78 @@ def associate_levelpaths_with_levees(levees_filename:str, levee_id_attribute:str
         levee_levelpaths_right = levee_levelpaths_right[[levee_id_attribute, branch_id_attribute]]
 
         # Select streams on the correct side of levee
-        levee_levelpaths_left = levee_levelpaths_left[levee_levelpaths_left[levee_id_attribute].isin(left_ids)]
-        levee_levelpaths_right = levee_levelpaths_right[levee_levelpaths_right[levee_id_attribute].isin(right_ids)]
+        levee_levelpaths_left = levee_levelpaths_left[
+            levee_levelpaths_left[levee_id_attribute].isin(left_ids)
+        ]
+        levee_levelpaths_right = levee_levelpaths_right[
+            levee_levelpaths_right[levee_id_attribute].isin(right_ids)
+        ]
 
         # Join left and right
-        out_df =  pd.concat([levee_levelpaths_right[[levee_id_attribute, branch_id_attribute]], levee_levelpaths_left[[levee_id_attribute, branch_id_attribute]]]).drop_duplicates().reset_index(drop=True)
+        out_df = (
+            pd.concat(
+                [
+                    levee_levelpaths_right[[levee_id_attribute, branch_id_attribute]],
+                    levee_levelpaths_left[[levee_id_attribute, branch_id_attribute]],
+                ]
+            )
+            .drop_duplicates()
+            .reset_index(drop=True)
+        )
 
         # Add level paths to levees not found
         if len(levees_not_found) > 0:
-            levees_not_found.geometry = levees_not_found.buffer(2*levee_buffer)
+            levees_not_found.geometry = levees_not_found.buffer(2 * levee_buffer)
             levees_not_found = gpd.sjoin(levees_not_found, levelpaths)
 
             # Add to out_df
-            out_df =  pd.concat([out_df[[levee_id_attribute, branch_id_attribute]], levees_not_found[[levee_id_attribute, branch_id_attribute]]]).drop_duplicates().reset_index(drop=True)
+            out_df = (
+                pd.concat(
+                    [
+                        out_df[[levee_id_attribute, branch_id_attribute]],
+                        levees_not_found[[levee_id_attribute, branch_id_attribute]],
+                    ]
+                )
+                .drop_duplicates()
+                .reset_index(drop=True)
+            )
 
         # Remove levelpaths that cross the levee exactly once
         for j, row in out_df.iterrows():
             # Intersect levees and levelpaths
-            row_intersections = gpd.overlay(levees[levees[levee_id_attribute] == row[levee_id_attribute]], levelpaths[levelpaths[branch_id_attribute] == row[branch_id_attribute]], how='intersection', keep_geom_type=False)
+            row_intersections = gpd.overlay(
+                levees[levees[levee_id_attribute] == row[levee_id_attribute]],
+                levelpaths[levelpaths[branch_id_attribute] == row[branch_id_attribute]],
+                how='intersection',
+                keep_geom_type=False,
+            )
 
             # Convert MultiPoint to Point
             row_intersections = row_intersections.explode(index_parts=True)
 
             # Select Point geometry type
-            row_intersections = row_intersections[row_intersections.geom_type =='Point']
+            row_intersections = row_intersections[row_intersections.geom_type == 'Point']
 
             if len(row_intersections) == 1:
                 out_df = out_df.drop(j)
-                
+
         out_df.to_csv(out_filename, columns=[levee_id_attribute, branch_id_attribute], index=False)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Associate level paths with levees')
-    parser.add_argument('-nld','--levees-filename', help='NLD levees filename', required=True, type=str)
-    parser.add_argument('-l','--levee-id-attribute', help='Levee ID attribute name', required=True, type=str)
-    parser.add_argument('-out','--out-filename', help='out CSV filename', required=True, type=str)
-    parser.add_argument('-s', '--levelpaths-filename', help='Level path layer filename', required=True, type=str)
-    parser.add_argument('-b','--branch-id-attribute', help='Level path ID attribute name', required=True, type=str)
-    parser.add_argument('-lpa', '--leveed-areas-filename', help='NLD levee-protected areas filename', required=True, type=str)
+    parser.add_argument('-nld', '--levees-filename', help='NLD levees filename', required=True, type=str)
+    parser.add_argument('-l', '--levee-id-attribute', help='Levee ID attribute name', required=True, type=str)
+    parser.add_argument('-out', '--out-filename', help='out CSV filename', required=True, type=str)
+    parser.add_argument(
+        '-s', '--levelpaths-filename', help='Level path layer filename', required=True, type=str
+    )
+    parser.add_argument(
+        '-b', '--branch-id-attribute', help='Level path ID attribute name', required=True, type=str
+    )
+    parser.add_argument(
+        '-lpa', '--leveed-areas-filename', help='NLD levee-protected areas filename', required=True, type=str
+    )
     parser.add_argument('-w', '--levee-buffer', help='Buffer width (in meters)', required=True, type=float)
 
     args = vars(parser.parse_args())
diff --git a/src/bash_functions.env b/src/bash_functions.env
index 44e3bb4cb..89a21aba7 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -30,12 +30,12 @@ Calc_Duration() {
 }
 
 Calc_Time() {
-    local start_time=$1 
+    local start_time=$1
     local end_time=`date +%s`
     local total_sec=$(( $end_time - $start_time ))
     local dur_min=$((total_sec / 60))
     local dur_remainder_sec=$((total_sec % 60))
-    
+
     if (( $dur_remainder_sec < 10 )); then
         echo "$dur_min:0$dur_remainder_sec"
     else
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
index ebda61044..c99fc3ebe 100755
--- a/src/bathy_rc_adjust.py
+++ b/src/bathy_rc_adjust.py
@@ -1,155 +1,333 @@
 #!/usr/bin/env python3
 
 from os import environ
+
 import geopandas as gpd
-import pandas as pd
 import numpy as np
+import pandas as pd
+
+
+sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag'])  # 10x
+thal_stg_limit = float(environ['thalweg_stg_search_max_limit'])  # 3m
+bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag'])  # 10x
+bathy_xsarea_flag = float(environ['bathy_xs_area_chg_flag'])  # 1x
+thal_hyd_radius_flag = float(environ['thalweg_hyd_radius_flag'])  # 10x
 
-sa_ratio_flag             = float(environ['surf_area_thalweg_ratio_flag']) #10x
-thal_stg_limit            = float(environ['thalweg_stg_search_max_limit']) #3m
-bankful_xs_ratio_flag     = float(environ['bankful_xs_area_ratio_flag']) #10x
-bathy_xsarea_flag         = float(environ['bathy_xs_area_chg_flag']) #1x
-thal_hyd_radius_flag      = float(environ['thalweg_hyd_radius_flag']) #10x
 
-def bathy_rc_lookup(input_src_base,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,):
+def bathy_rc_lookup(
+    input_src_base,
+    input_bathy_fileName,
+    output_bathy_fileName,
+    output_bathy_streamorder_fileName,
+    output_bathy_thalweg_fileName,
+    output_bathy_xs_lookup_fileName,
+):
     ## Convert input_src_base featureid to integer
-    if input_src_base.feature_id.dtype != 'int': input_src_base.feature_id = input_src_base.feature_id.astype(int)
+    if input_src_base.feature_id.dtype != 'int':
+        input_src_base.feature_id = input_src_base.feature_id.astype(int)
 
     ## Read in the bankfull channel geometry text file
-    input_bathy = pd.read_csv(input_bathy_fileName, dtype= {'COMID': int})
+    input_bathy = pd.read_csv(input_bathy_fileName, dtype={'COMID': int})
 
     ## Merge input_bathy and modified_src_base df using feature_id/COMID attributes
-    input_bathy = input_bathy.rename(columns={'COMID':'feature_id','BANKFULL_WIDTH':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA':'BANKFULL_XSEC_AREA (m2)'})
-    modified_src_base = input_src_base.merge(input_bathy.loc[:,['feature_id','BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA (m2)']],how='left',on='feature_id')
+    input_bathy = input_bathy.rename(
+        columns={
+            'COMID': 'feature_id',
+            'BANKFULL_WIDTH': 'BANKFULL_WIDTH (m)',
+            'BANKFULL_XSEC_AREA': 'BANKFULL_XSEC_AREA (m2)',
+        }
+    )
+    modified_src_base = input_src_base.merge(
+        input_bathy.loc[:, ['feature_id', 'BANKFULL_WIDTH (m)', 'BANKFULL_XSEC_AREA (m2)']],
+        how='left',
+        on='feature_id',
+    )
 
     ## Check that the merge process returned matching feature_id entries
     if modified_src_base['BANKFULL_WIDTH (m)'].count() == 0:
-        print('No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC!')
-        return(input_src_base)
+        print(
+            'No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC!'
+        )
+        return input_src_base
     else:
         ## Use SurfaceArea variable to identify thalweg-restricted stage values for each hydroid
         ## Calculate the interrow SurfaceArea ratio n/(n-1)
-        modified_src_base['SA_div'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
+        modified_src_base['SA_div'] = modified_src_base['SurfaceArea (m2)'].div(
+            modified_src_base['SurfaceArea (m2)'].shift(1)
+        )
         ## Mask SA_div when Stage = 0 or when the SA_div value (n / n-1) is > threshold value (i.e. 10x)
-        modified_src_base['SA_div'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
+        modified_src_base['SA_div'].mask(
+            (modified_src_base['Stage'] == 0)
+            | (modified_src_base['SA_div'] < sa_ratio_flag)
+            | (modified_src_base['SurfaceArea (m2)'] == 0),
+            inplace=True,
+        )
         ## Create new df to filter and groupby HydroID
-        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div']]
-        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['Stage']<thal_stg_limit] # assuming thalweg burn-in is less than 3 meters
+        find_thalweg_notch = modified_src_base[['HydroID', 'Stage', 'SurfaceArea (m2)', 'SA_div']]
+        find_thalweg_notch = find_thalweg_notch[
+            find_thalweg_notch['Stage'] < thal_stg_limit
+        ]  # assuming thalweg burn-in is less than 3 meters
         find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div'].notnull()]
-        find_thalweg_notch = find_thalweg_notch.loc[find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()].reset_index(drop=True)
+        find_thalweg_notch = find_thalweg_notch.loc[
+            find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()
+        ].reset_index(drop=True)
         ## Assign thalweg_burn_elev variable to the stage value found in previous step
         find_thalweg_notch['Thalweg_burn_elev'] = find_thalweg_notch['Stage']
         ## Merge the Thalweg_burn_elev value back into the modified SRC --> this is used to mask the discharge after Manning's equation
-        modified_src_base = modified_src_base.merge(find_thalweg_notch.loc[:,['HydroID','Thalweg_burn_elev']],how='left',on='HydroID')
+        modified_src_base = modified_src_base.merge(
+            find_thalweg_notch.loc[:, ['HydroID', 'Thalweg_burn_elev']], how='left', on='HydroID'
+        )
 
         ## Calculate bankfull vs top width difference for each feature_id
-        modified_src_base['Top Width Diff (m)'] = (modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']).abs()
+        modified_src_base['Top Width Diff (m)'] = (
+            modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']
+        ).abs()
         ## Calculate XS Area field (Channel Volume / Stream Length)
-        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (modified_src_base['LENGTHKM'] * 1000)
+        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (
+            modified_src_base['LENGTHKM'] * 1000
+        )
 
         ## Groupby HydroID and find min of Top Width Diff (m)
-        output_bathy = modified_src_base[['feature_id','HydroID','order_','Stage','SurfaceArea (m2)','Thalweg_burn_elev','BANKFULL_WIDTH (m)','TopWidth (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','Top Width Diff (m)']]
+        output_bathy = modified_src_base[
+            [
+                'feature_id',
+                'HydroID',
+                'order_',
+                'Stage',
+                'SurfaceArea (m2)',
+                'Thalweg_burn_elev',
+                'BANKFULL_WIDTH (m)',
+                'TopWidth (m)',
+                'XS Area (m2)',
+                'BANKFULL_XSEC_AREA (m2)',
+                'Top Width Diff (m)',
+            ]
+        ]
         ## filter out stage = 0 rows in SRC (assuming geom at stage 0 is not a valid channel geom)
         output_bathy = output_bathy[output_bathy['Stage'] > 0]
         ## filter SRC rows identified as Thalweg burned
-        output_bathy['Top Width Diff (m)'].mask(output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'],inplace=True)
+        output_bathy['Top Width Diff (m)'].mask(
+            output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'], inplace=True
+        )
         ## ignore hydroid/featureid that did not have a valid Bankfull lookup (areas outside CONUS - i.e. Canada)
         output_bathy = output_bathy[output_bathy['BANKFULL_XSEC_AREA (m2)'].notnull()]
         ## ignore SRC entries with 0 surface area --> handles input SRC artifacts/errors in Great Lakes region
         output_bathy = output_bathy[output_bathy['SurfaceArea (m2)'] > 0]
         ## find index of minimum top width difference --> this will be used as the SRC "bankfull" row for future calcs
-        output_bathy = output_bathy.loc[output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()].reset_index(drop=True)
-        print('Average: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].mean()))
-        print('Minimum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].min()))
-        print('Maximum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].max()))
-        print('STD: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].std()) +'\n' + '#################')
+        output_bathy = output_bathy.loc[
+            output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()
+        ].reset_index(drop=True)
+        print(
+            'Average: bankfull width crosswalk difference (m): '
+            + str(output_bathy['Top Width Diff (m)'].mean())
+        )
+        print(
+            'Minimum: bankfull width crosswalk difference (m): '
+            + str(output_bathy['Top Width Diff (m)'].min())
+        )
+        print(
+            'Maximum: bankfull width crosswalk difference (m): '
+            + str(output_bathy['Top Width Diff (m)'].max())
+        )
+        print(
+            'STD: bankfull width crosswalk difference (m): '
+            + str(output_bathy['Top Width Diff (m)'].std())
+            + '\n'
+            + '#################'
+        )
 
         ## Calculate XS Area difference between SRC and Bankfull database
-        output_bathy['XS Area Diff (m2)'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)'])
-        output_bathy['XS Bankfull Area Ratio'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']).round(2)
+        output_bathy['XS Area Diff (m2)'] = (
+            output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)']
+        )
+        output_bathy['XS Bankfull Area Ratio'] = (
+            output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']
+        ).round(2)
         ## masking negative XS Area Diff and XS Area = 0
-        output_bathy['XS Bankfull Area Ratio'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        output_bathy['XS Bankfull Area Ratio'].mask(
+            (output_bathy['XS Area Diff (m2)'] < 0) | (output_bathy['XS Area (m2)'] == 0), inplace=True
+        )
         ## masking negative XS Area Diff and XS Area = 0
-        output_bathy['XS Area Diff (m2)'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        output_bathy['XS Area Diff (m2)'].mask(
+            (output_bathy['XS Area Diff (m2)'] < 0) | (output_bathy['XS Area (m2)'] == 0), inplace=True
+        )
         ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
-        output_bathy['XS Area Diff (m2)'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        output_bathy['XS Area Diff (m2)'].mask(
+            output_bathy['XS Bankfull Area Ratio'] > bankful_xs_ratio_flag, inplace=True
+        )
         ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
-        output_bathy['XS Bankfull Area Ratio'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        output_bathy['XS Bankfull Area Ratio'].mask(
+            output_bathy['XS Bankfull Area Ratio'] > bankful_xs_ratio_flag, inplace=True
+        )
         ## Print XS Area Diff statistics
-        print('Average: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].mean()))
-        print('Minimum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].min()))
-        print('Maximum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].max()))
-        print('STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std()))
+        print(
+            'Average: bankfull XS Area crosswalk difference (m2): '
+            + str(output_bathy['XS Area Diff (m2)'].mean())
+        )
+        print(
+            'Minimum: bankfull XS Area crosswalk difference (m2): '
+            + str(output_bathy['XS Area Diff (m2)'].min())
+        )
+        print(
+            'Maximum: bankfull XS Area crosswalk difference (m2): '
+            + str(output_bathy['XS Area Diff (m2)'].max())
+        )
+        print(
+            'STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std())
+        )
 
         ## Bin XS Bankfull Area Ratio by stream order
-        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']].copy()
+        stream_order_bathy_ratio = output_bathy[['order_', 'Stage', 'XS Bankfull Area Ratio']].copy()
         ## mask stage values when XS Bankfull Area Ratio is null (need to filter to calculate the median for valid values below)
-        stream_order_bathy_ratio['Stage'].mask(stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(),inplace=True)
-        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(count=('XS Bankfull Area Ratio','count'),mean_xs_area_ratio=('XS Bankfull Area Ratio','mean'),median_stage_bankfull=('Stage','median'))
+        stream_order_bathy_ratio['Stage'].mask(
+            stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(), inplace=True
+        )
+        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(
+            count=('XS Bankfull Area Ratio', 'count'),
+            mean_xs_area_ratio=('XS Bankfull Area Ratio', 'mean'),
+            median_stage_bankfull=('Stage', 'median'),
+        )
         ## fill XS Bankfull Area Ratio and Stage values if no values were found in the grouby calcs
-        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill()+stream_order_bathy_ratio.bfill())/2
+        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill() + stream_order_bathy_ratio.bfill()) / 2
         ## fill first and last stream order values if needed
         stream_order_bathy_ratio = stream_order_bathy_ratio.bfill().ffill()
         ## Get count_total tally of the total number of stream order hydroids in the HUC (not filtering anything out)
-        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage','count'))
-        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(stream_order_bathy_ratio_count,how='left',on='order_')
+        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage', 'count'))
+        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(
+            stream_order_bathy_ratio_count, how='left', on='order_'
+        )
         ## Fill any remaining null values: mean_xs_area_ratio --> 1 median_stage_bankfull --> 0
-        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(),1,inplace=True)
-        stream_order_bathy_ratio['median_stage_bankfull'].mask(stream_order_bathy_ratio['median_stage_bankfull'].isnull(),0,inplace=True)
+        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(
+            stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(), 1, inplace=True
+        )
+        stream_order_bathy_ratio['median_stage_bankfull'].mask(
+            stream_order_bathy_ratio['median_stage_bankfull'].isnull(), 0, inplace=True
+        )
 
         ## Combine SRC df and df of XS Area for each hydroid and matching stage and order from bins above
-        output_bathy = output_bathy.merge(stream_order_bathy_ratio,how='left',on='order_')
-        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio,how='left',on='order_')
+        output_bathy = output_bathy.merge(stream_order_bathy_ratio, how='left', on='order_')
+        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio, how='left', on='order_')
 
         ## Calculate stage vs median_stage_bankfull difference for bankfull lookup
-        modified_src_base['lookup_stage_diff'] = (modified_src_base[['median_stage_bankfull','Thalweg_burn_elev']].max(axis=1) - modified_src_base['Stage']).abs()
+        modified_src_base['lookup_stage_diff'] = (
+            modified_src_base[['median_stage_bankfull', 'Thalweg_burn_elev']].max(axis=1)
+            - modified_src_base['Stage']
+        ).abs()
 
         ## If median_stage_bankfull is null then set lookup_stage_diff to 999 at stage 0 (handles errors for channels outside CONUS)
-        modified_src_base['lookup_stage_diff'].mask((modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),999,inplace=True)
+        modified_src_base['lookup_stage_diff'].mask(
+            (modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),
+            999,
+            inplace=True,
+        )
 
         ## Groupby HydroID again and find min of lookup_stage_diff
-        xs_area_hydroid_lookup = modified_src_base[['HydroID','BANKFULL_XSEC_AREA (m2)','XS Area (m2)','Stage','Thalweg_burn_elev','median_stage_bankfull','lookup_stage_diff','mean_xs_area_ratio']]
-        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()].reset_index(drop=True)
+        xs_area_hydroid_lookup = modified_src_base[
+            [
+                'HydroID',
+                'BANKFULL_XSEC_AREA (m2)',
+                'XS Area (m2)',
+                'Stage',
+                'Thalweg_burn_elev',
+                'median_stage_bankfull',
+                'lookup_stage_diff',
+                'mean_xs_area_ratio',
+            ]
+        ]
+        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[
+            xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()
+        ].reset_index(drop=True)
 
         ## Calculate bathy adjusted XS Area ('XS Area (m2)' mutliplied by mean_xs_area_ratio)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']) - xs_area_hydroid_lookup['XS Area (m2)']
+        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (
+            xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']
+        ) - xs_area_hydroid_lookup['XS Area (m2)']
 
         ## Calculate the ratio btw the lookup SRC XS_Area and the Bankfull_XSEC_AREA --> use this as a flag for potentially bad XS data
-        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'])
+        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (
+            xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)']
+        )
         ## Set bath_cal_xs_area to 0 if the bankfull_XS_ratio_flag is > threshold --> 5x (assuming too large of difference to be a reliable bankfull calculation)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag,xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],inplace=True)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(),0,inplace=True)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(
+            xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] > bathy_xsarea_flag,
+            xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],
+            inplace=True,
+        )
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(
+            xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(), 0, inplace=True
+        )
 
         ## Merge bathy_calc_xs_area to the modified_src_base
-        modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area']],how='left',on='HydroID')
+        modified_src_base = modified_src_base.merge(
+            xs_area_hydroid_lookup.loc[:, ['HydroID', 'bathy_calc_xs_area']], how='left', on='HydroID'
+        )
 
         ## Calculate new bathy adjusted channel geometry variables
-        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'Discharge (m3s-1)_nobathy'})
-        modified_src_base['XS Area (m2)_bathy_adj'] = modified_src_base['XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
-        modified_src_base['Volume (m3)_bathy_adj'] = modified_src_base['XS Area (m2)_bathy_adj'] * modified_src_base['LENGTHKM'] * 1000
-        modified_src_base['WetArea (m2)_bathy_adj'] = modified_src_base['Volume (m3)_bathy_adj']/modified_src_base['LENGTHKM']/1000
-        modified_src_base['HydraulicRadius (m)_bathy_adj'] = modified_src_base['WetArea (m2)_bathy_adj']/modified_src_base['WettedPerimeter (m)']
+        modified_src_base = modified_src_base.rename(
+            columns={'Discharge (m3s-1)': 'Discharge (m3s-1)_nobathy'}
+        )
+        modified_src_base['XS Area (m2)_bathy_adj'] = (
+            modified_src_base['XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
+        )
+        modified_src_base['Volume (m3)_bathy_adj'] = (
+            modified_src_base['XS Area (m2)_bathy_adj'] * modified_src_base['LENGTHKM'] * 1000
+        )
+        modified_src_base['WetArea (m2)_bathy_adj'] = (
+            modified_src_base['Volume (m3)_bathy_adj'] / modified_src_base['LENGTHKM'] / 1000
+        )
+        modified_src_base['HydraulicRadius (m)_bathy_adj'] = (
+            modified_src_base['WetArea (m2)_bathy_adj'] / modified_src_base['WettedPerimeter (m)']
+        )
         modified_src_base['HydraulicRadius (m)_bathy_adj'].fillna(0, inplace=True)
         ## mask out negative top width differences (avoid thalweg burn notch)
-        modified_src_base['HydraulicRadius (m)_bathy_adj'].mask((modified_src_base['HydraulicRadius (m)_bathy_adj']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
+        modified_src_base['HydraulicRadius (m)_bathy_adj'].mask(
+            (modified_src_base['HydraulicRadius (m)_bathy_adj'] > thal_hyd_radius_flag)
+            & (modified_src_base['Stage'] < thal_stg_limit),
+            0,
+            inplace=True,
+        )
 
         ## Calculate Q using Manning's equation
-        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)_bathy_adj']* \
-        pow(modified_src_base['HydraulicRadius (m)_bathy_adj'],2.0/3)* \
-        pow(modified_src_base['SLOPE'],0.5)/modified_src_base['ManningN']
+        modified_src_base['Discharge (m3s-1)'] = (
+            modified_src_base['WetArea (m2)_bathy_adj']
+            * pow(modified_src_base['HydraulicRadius (m)_bathy_adj'], 2.0 / 3)
+            * pow(modified_src_base['SLOPE'], 0.5)
+            / modified_src_base['ManningN']
+        )
         ## mask discharge values for stage = 0 rows in SRC (replace with 0) --> do we need SRC to start at 0??
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0,0,inplace=True)
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'],0,inplace=True)
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'],-999,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0, 0, inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(
+            modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'], 0, inplace=True
+        )
+        modified_src_base['Discharge (m3s-1)'].mask(
+            modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'], -999, inplace=True
+        )
 
         ## Organize bathy calc output variables for csv
-        output_bathy = output_bathy[['HydroID','order_','Stage','SurfaceArea (m2)','TopWidth (m)','BANKFULL_WIDTH (m)','Top Width Diff (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','XS Area Diff (m2)','XS Bankfull Area Ratio','count','median_stage_bankfull','mean_xs_area_ratio']]
+        output_bathy = output_bathy[
+            [
+                'HydroID',
+                'order_',
+                'Stage',
+                'SurfaceArea (m2)',
+                'TopWidth (m)',
+                'BANKFULL_WIDTH (m)',
+                'Top Width Diff (m)',
+                'XS Area (m2)',
+                'BANKFULL_XSEC_AREA (m2)',
+                'XS Area Diff (m2)',
+                'XS Bankfull Area Ratio',
+                'count',
+                'median_stage_bankfull',
+                'mean_xs_area_ratio',
+            ]
+        ]
 
         ## Export bathy/bankful calculation tables for easy viewing
-        output_bathy.to_csv(output_bathy_fileName,index=False)
-        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName,index=True)
-        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName,index=True)
-        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName,index=True)
+        output_bathy.to_csv(output_bathy_fileName, index=False)
+        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName, index=True)
+        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName, index=True)
+        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName, index=True)
 
         print('Completed Bathy Calculations...')
-        return(modified_src_base)
+        return modified_src_base
diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py
index dc2c94dd9..04dd34b24 100644
--- a/src/bathy_src_adjust_topwidth.py
+++ b/src/bathy_src_adjust_topwidth.py
@@ -1,17 +1,19 @@
 #!/usr/bin/env python3
 
-import sys
-import os
-import pandas as pd
-import numpy as np
 import argparse
-import matplotlib.pyplot as plt
-import seaborn as sns
-from multiprocessing import Pool
-from os.path import isfile, join, dirname, isdir
 import datetime as dt
+import os
 import sys
+from multiprocessing import Pool
 from os import environ
+from os.path import dirname, isdir, isfile, join
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+
+
 sns.set_theme(style="whitegrid")
 # from utils.shared_functions import mem_profile
 
@@ -36,196 +38,422 @@
 # bathy_xsarea_flag = 1
 # thal_hyd_radius_flag = 10
 # ignore_streamorder = 10
-sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag']) #10x --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
-thal_stg_limit = float(environ['thalweg_stg_search_max_limit']) #3m --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
-bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag']) #10x --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
-bathy_xsarea_flag = float(environ['bathy_xs_area_chg_flag']) #1x --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
-thal_hyd_radius_flag = float(environ['thalweg_hyd_radius_flag']) #10x --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
-ignore_streamorder = int(environ['ignore_streamorders']) #10 --> Do not perform BARC for streamorders >= provided value
+sa_ratio_flag = float(
+    environ['surf_area_thalweg_ratio_flag']
+)  # 10x --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)
+thal_stg_limit = float(
+    environ['thalweg_stg_search_max_limit']
+)  # 3m --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)
+bankful_xs_ratio_flag = float(
+    environ['bankful_xs_area_ratio_flag']
+)  # 10x --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)
+bathy_xsarea_flag = float(
+    environ['bathy_xs_area_chg_flag']
+)  # 1x --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold
+thal_hyd_radius_flag = float(
+    environ['thalweg_hyd_radius_flag']
+)  # 10x --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0
+ignore_streamorder = int(
+    environ['ignore_streamorders']
+)  # 10 --> Do not perform BARC for streamorders >= provided value
+
 
 def bathy_rc_lookup(args):
-    input_src_fileName                  = args[0]
-    df_bfull_geom                       = args[1]
-    output_bathy_fileName               = args[2]
-    output_bathy_streamorder_fileName   = args[3]
-    output_bathy_thalweg_fileName       = args[4]
-    output_bathy_xs_lookup_fileName     = args[5]
-    input_htable_fileName               = args[6]
-    out_src_filename                    = args[7]
-    huc                                 = args[8]
-    src_plot_option                     = args[9]
-    huc_plot_output_dir                 = args[10]
+    input_src_fileName = args[0]
+    df_bfull_geom = args[1]
+    output_bathy_fileName = args[2]
+    output_bathy_streamorder_fileName = args[3]
+    output_bathy_thalweg_fileName = args[4]
+    output_bathy_xs_lookup_fileName = args[5]
+    input_htable_fileName = args[6]
+    out_src_filename = args[7]
+    huc = args[8]
+    src_plot_option = args[9]
+    huc_plot_output_dir = args[10]
 
     log_text = 'Calculating: ' + str(huc) + '\n'
 
     ## Read in the default src_full_crosswalked.csv
-    input_src_base = pd.read_csv(input_src_fileName, dtype= {'feature_id': int})
+    input_src_base = pd.read_csv(input_src_fileName, dtype={'feature_id': int})
 
     ## Rename input bankfull_geom data columns for consistant referencing
-    df_bfull_geom = df_bfull_geom.rename(columns={'COMID':'feature_id','BANKFULL_WIDTH':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA':'BANKFULL_XSEC_AREA (m2)'})
-    df_bfull_geom = df_bfull_geom.rename(columns={'BANKFULL_TOPWIDTH_q':'BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA_q':'BANKFULL_XSEC_AREA (m2)'})
+    df_bfull_geom = df_bfull_geom.rename(
+        columns={
+            'COMID': 'feature_id',
+            'BANKFULL_WIDTH': 'BANKFULL_WIDTH (m)',
+            'BANKFULL_XSEC_AREA': 'BANKFULL_XSEC_AREA (m2)',
+        }
+    )
+    df_bfull_geom = df_bfull_geom.rename(
+        columns={
+            'BANKFULL_TOPWIDTH_q': 'BANKFULL_WIDTH (m)',
+            'BANKFULL_XSEC_AREA_q': 'BANKFULL_XSEC_AREA (m2)',
+        }
+    )
     ## Merge input_bathy and modified_src_base df using feature_id/COMID attributes
-    modified_src_base = input_src_base.merge(df_bfull_geom.loc[:,['feature_id','BANKFULL_WIDTH (m)','BANKFULL_XSEC_AREA (m2)']],how='left',on='feature_id')
+    modified_src_base = input_src_base.merge(
+        df_bfull_geom.loc[:, ['feature_id', 'BANKFULL_WIDTH (m)', 'BANKFULL_XSEC_AREA (m2)']],
+        how='left',
+        on='feature_id',
+    )
 
     ## Check that the merge process returned matching feature_id entries
     if modified_src_base['BANKFULL_WIDTH (m)'].count() == 0:
-        log_text += 'WARNING: No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC for huc ' + str(huc) + '\n'
+        log_text += (
+            'WARNING: No matching feature_id found between input bathy data and src_base --> No bathy calculations added to SRC for huc '
+            + str(huc)
+            + '\n'
+        )
     else:
         ## Use SurfaceArea variable to identify thalweg-restricted stage values for each hydroid
         ## Calculate the interrow SurfaceArea ratio n/(n-1)
-        modified_src_base['SA_div_flag'] = modified_src_base['SurfaceArea (m2)'].div(modified_src_base['SurfaceArea (m2)'].shift(1))
+        modified_src_base['SA_div_flag'] = modified_src_base['SurfaceArea (m2)'].div(
+            modified_src_base['SurfaceArea (m2)'].shift(1)
+        )
         ## Mask SA_div_flag when Stage = 0 or when the SA_div_flag value (n / n-1) is > threshold value (i.e. 10x)
-        modified_src_base['SA_div_flag'].mask((modified_src_base['Stage']==0) | (modified_src_base['SA_div_flag']<sa_ratio_flag) | (modified_src_base['SurfaceArea (m2)']==0),inplace=True)
+        modified_src_base['SA_div_flag'].mask(
+            (modified_src_base['Stage'] == 0)
+            | (modified_src_base['SA_div_flag'] < sa_ratio_flag)
+            | (modified_src_base['SurfaceArea (m2)'] == 0),
+            inplace=True,
+        )
         ## Create new df to filter and groupby HydroID
-        find_thalweg_notch = modified_src_base[['HydroID','Stage','SurfaceArea (m2)','SA_div_flag']]
-        find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['Stage']<thal_stg_limit] # assuming thalweg burn-in is less than 3 meters
+        find_thalweg_notch = modified_src_base[['HydroID', 'Stage', 'SurfaceArea (m2)', 'SA_div_flag']]
+        find_thalweg_notch = find_thalweg_notch[
+            find_thalweg_notch['Stage'] < thal_stg_limit
+        ]  # assuming thalweg burn-in is less than 3 meters
         find_thalweg_notch = find_thalweg_notch[find_thalweg_notch['SA_div_flag'].notnull()]
-        find_thalweg_notch = find_thalweg_notch.loc[find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()].reset_index(drop=True)
+        find_thalweg_notch = find_thalweg_notch.loc[
+            find_thalweg_notch.groupby('HydroID')['Stage'].idxmax()
+        ].reset_index(drop=True)
         ## Assign thalweg_burn_elev variable to the stage value found in previous step
         find_thalweg_notch['Thalweg_burn_elev'] = find_thalweg_notch['Stage']
         ## Merge the Thalweg_burn_elev value back into the modified SRC --> this is used to mask the discharge after Manning's equation
-        modified_src_base = modified_src_base.merge(find_thalweg_notch.loc[:,['HydroID','Thalweg_burn_elev']],how='left',on='HydroID')
+        modified_src_base = modified_src_base.merge(
+            find_thalweg_notch.loc[:, ['HydroID', 'Thalweg_burn_elev']], how='left', on='HydroID'
+        )
 
         ## Calculate bankfull vs top width difference for each feature_id
-        modified_src_base['Top Width Diff (m)'] = (modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']).abs()
+        modified_src_base['Top Width Diff (m)'] = (
+            modified_src_base['TopWidth (m)'] - modified_src_base['BANKFULL_WIDTH (m)']
+        ).abs()
         ## Calculate XS Area field (Channel Volume / Stream Length)
-        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (modified_src_base['LENGTHKM'] * 1000)
+        modified_src_base['XS Area (m2)'] = modified_src_base['Volume (m3)'] / (
+            modified_src_base['LENGTHKM'] * 1000
+        )
 
         ## Groupby HydroID and find min of Top Width Diff (m)
-        output_bathy = modified_src_base[['feature_id','HydroID','order_','Stage','SurfaceArea (m2)','Thalweg_burn_elev','BANKFULL_WIDTH (m)','TopWidth (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','Top Width Diff (m)']]
+        output_bathy = modified_src_base[
+            [
+                'feature_id',
+                'HydroID',
+                'order_',
+                'Stage',
+                'SurfaceArea (m2)',
+                'Thalweg_burn_elev',
+                'BANKFULL_WIDTH (m)',
+                'TopWidth (m)',
+                'XS Area (m2)',
+                'BANKFULL_XSEC_AREA (m2)',
+                'Top Width Diff (m)',
+            ]
+        ]
         ## filter out stage = 0 rows in SRC (assuming geom at stage 0 is not a valid channel geom)
         output_bathy = output_bathy[output_bathy['Stage'] > 0]
         ## filter SRC rows identified as Thalweg burned
-        output_bathy['Top Width Diff (m)'].mask(output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'],inplace=True)
+        output_bathy['Top Width Diff (m)'].mask(
+            output_bathy['Stage'] <= output_bathy['Thalweg_burn_elev'], inplace=True
+        )
         ## ignore hydroid/featureid that did not have a valid Bankfull lookup (areas outside CONUS - i.e. Canada)
         output_bathy = output_bathy[output_bathy['BANKFULL_XSEC_AREA (m2)'].notnull()]
         ## ignore SRC entries with 0 surface area --> handles input SRC artifacts/errors in Great Lakes region
         output_bathy = output_bathy[output_bathy['SurfaceArea (m2)'] > 0]
         ## find index of minimum top width difference --> this will be used as the SRC "bankfull" row for future calcs
-        output_bathy = output_bathy.loc[output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()].reset_index(drop=True)
-        log_text += ('Average: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].mean())) + '\n'
-        log_text += ('Minimum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].min())) + '\n'
-        log_text += ('Maximum: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].max())) + '\n'
-        log_text += ('STD: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].std())) +'\n'
+        output_bathy = output_bathy.loc[
+            output_bathy.groupby('HydroID')['Top Width Diff (m)'].idxmin()
+        ].reset_index(drop=True)
+        log_text += (
+            'Average: bankfull width crosswalk difference (m): '
+            + str(output_bathy['Top Width Diff (m)'].mean())
+        ) + '\n'
+        log_text += (
+            'Minimum: bankfull width crosswalk difference (m): '
+            + str(output_bathy['Top Width Diff (m)'].min())
+        ) + '\n'
+        log_text += (
+            'Maximum: bankfull width crosswalk difference (m): '
+            + str(output_bathy['Top Width Diff (m)'].max())
+        ) + '\n'
+        log_text += (
+            'STD: bankfull width crosswalk difference (m): ' + str(output_bathy['Top Width Diff (m)'].std())
+        ) + '\n'
 
         ## Calculate XS Area difference between SRC and Bankfull database
-        output_bathy['XS Area Diff (m2)'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)'])
-        output_bathy['XS Bankfull Area Ratio'] = (output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']).round(2)
+        output_bathy['XS Area Diff (m2)'] = (
+            output_bathy['BANKFULL_XSEC_AREA (m2)'] - output_bathy['XS Area (m2)']
+        )
+        output_bathy['XS Bankfull Area Ratio'] = (
+            output_bathy['BANKFULL_XSEC_AREA (m2)'] / output_bathy['XS Area (m2)']
+        ).round(2)
         ## masking negative XS Area Diff and XS Area = 0
-        output_bathy['XS Bankfull Area Ratio'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        output_bathy['XS Bankfull Area Ratio'].mask(
+            (output_bathy['XS Area Diff (m2)'] < 0) | (output_bathy['XS Area (m2)'] == 0), inplace=True
+        )
         ## masking negative XS Area Diff and XS Area = 0
-        output_bathy['XS Area Diff (m2)'].mask((output_bathy['XS Area Diff (m2)']<0) | (output_bathy['XS Area (m2)'] == 0),inplace=True)
+        output_bathy['XS Area Diff (m2)'].mask(
+            (output_bathy['XS Area Diff (m2)'] < 0) | (output_bathy['XS Area (m2)'] == 0), inplace=True
+        )
         ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
-        output_bathy['XS Area Diff (m2)'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        output_bathy['XS Area Diff (m2)'].mask(
+            output_bathy['XS Bankfull Area Ratio'] > bankful_xs_ratio_flag, inplace=True
+        )
         ## remove bogus values where bankfull area ratio > threshold --> 10x (topwidth crosswalk issues or bad bankfull regression data points??)
-        output_bathy['XS Bankfull Area Ratio'].mask(output_bathy['XS Bankfull Area Ratio']>bankful_xs_ratio_flag,inplace=True)
+        output_bathy['XS Bankfull Area Ratio'].mask(
+            output_bathy['XS Bankfull Area Ratio'] > bankful_xs_ratio_flag, inplace=True
+        )
         ## Print XS Area Diff statistics
-        log_text += ('Average: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].mean())) + '\n'
-        log_text += ('Minimum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].min())) + '\n'
-        log_text += ('Maximum: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].max())) + '\n'
-        log_text += ('STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std())) + '\n'
+        log_text += (
+            'Average: bankfull XS Area crosswalk difference (m2): '
+            + str(output_bathy['XS Area Diff (m2)'].mean())
+        ) + '\n'
+        log_text += (
+            'Minimum: bankfull XS Area crosswalk difference (m2): '
+            + str(output_bathy['XS Area Diff (m2)'].min())
+        ) + '\n'
+        log_text += (
+            'Maximum: bankfull XS Area crosswalk difference (m2): '
+            + str(output_bathy['XS Area Diff (m2)'].max())
+        ) + '\n'
+        log_text += (
+            'STD: bankfull XS Area crosswalk difference (m2): ' + str(output_bathy['XS Area Diff (m2)'].std())
+        ) + '\n'
 
         ## Bin XS Bankfull Area Ratio by stream order
-        stream_order_bathy_ratio = output_bathy[['order_','Stage','XS Bankfull Area Ratio']].copy()
+        stream_order_bathy_ratio = output_bathy[['order_', 'Stage', 'XS Bankfull Area Ratio']].copy()
         ## mask stage values when XS Bankfull Area Ratio is null (need to filter to calculate the median for valid values below)
-        stream_order_bathy_ratio['Stage'].mask(stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(),inplace=True)
-        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(count=('XS Bankfull Area Ratio','count'),mean_xs_area_ratio=('XS Bankfull Area Ratio','mean'),median_stage_bankfull=('Stage','median'))
+        stream_order_bathy_ratio['Stage'].mask(
+            stream_order_bathy_ratio['XS Bankfull Area Ratio'].isnull(), inplace=True
+        )
+        stream_order_bathy_ratio = stream_order_bathy_ratio.groupby('order_').agg(
+            count=('XS Bankfull Area Ratio', 'count'),
+            mean_xs_area_ratio=('XS Bankfull Area Ratio', 'mean'),
+            median_stage_bankfull=('Stage', 'median'),
+        )
         ## fill XS Bankfull Area Ratio and Stage values if no values were found in the grouby calcs
-        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill()+stream_order_bathy_ratio.bfill())/2
+        stream_order_bathy_ratio = (stream_order_bathy_ratio.ffill() + stream_order_bathy_ratio.bfill()) / 2
         ## fill first and last stream order values if needed
         stream_order_bathy_ratio = stream_order_bathy_ratio.bfill().ffill()
         ## Get count_total tally of the total number of stream order hydroids in the HUC (not filtering anything out)
-        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage','count'))
-        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(stream_order_bathy_ratio_count,how='left',on='order_')
+        stream_order_bathy_ratio_count = output_bathy.groupby('order_').agg(count_total=('Stage', 'count'))
+        stream_order_bathy_ratio = stream_order_bathy_ratio.merge(
+            stream_order_bathy_ratio_count, how='left', on='order_'
+        )
         ## Fill any remaining null values: mean_xs_area_ratio --> 1 median_stage_bankfull --> 0
-        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(),1,inplace=True)
-        stream_order_bathy_ratio['median_stage_bankfull'].mask(stream_order_bathy_ratio['median_stage_bankfull'].isnull(),0,inplace=True)
+        stream_order_bathy_ratio['mean_xs_area_ratio'].mask(
+            stream_order_bathy_ratio['mean_xs_area_ratio'].isnull(), 1, inplace=True
+        )
+        stream_order_bathy_ratio['median_stage_bankfull'].mask(
+            stream_order_bathy_ratio['median_stage_bankfull'].isnull(), 0, inplace=True
+        )
 
         ## Combine SRC df and df of XS Area for each hydroid and matching stage and order from bins above
-        output_bathy = output_bathy.merge(stream_order_bathy_ratio,how='left',on='order_')
-        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio,how='left',on='order_')
+        output_bathy = output_bathy.merge(stream_order_bathy_ratio, how='left', on='order_')
+        modified_src_base = modified_src_base.merge(stream_order_bathy_ratio, how='left', on='order_')
 
         ## Calculate stage vs median_stage_bankfull difference for bankfull lookup
-        modified_src_base['lookup_stage_diff'] = (modified_src_base[['median_stage_bankfull','Thalweg_burn_elev']].max(axis=1) - modified_src_base['Stage']).abs()
+        modified_src_base['lookup_stage_diff'] = (
+            modified_src_base[['median_stage_bankfull', 'Thalweg_burn_elev']].max(axis=1)
+            - modified_src_base['Stage']
+        ).abs()
 
         ## If median_stage_bankfull is null then set lookup_stage_diff to 999 at stage 0 (handles errors for channels outside CONUS)
-        modified_src_base['lookup_stage_diff'].mask((modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),999,inplace=True)
+        modified_src_base['lookup_stage_diff'].mask(
+            (modified_src_base['Stage'] == 0) & (modified_src_base['median_stage_bankfull'].isnull()),
+            999,
+            inplace=True,
+        )
 
         ## Groupby HydroID again and find min of lookup_stage_diff
-        xs_area_hydroid_lookup = modified_src_base[['HydroID','BANKFULL_XSEC_AREA (m2)','XS Area (m2)','Stage','Thalweg_burn_elev','median_stage_bankfull','lookup_stage_diff','mean_xs_area_ratio']]
-        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()].reset_index(drop=True)
+        xs_area_hydroid_lookup = modified_src_base[
+            [
+                'HydroID',
+                'BANKFULL_XSEC_AREA (m2)',
+                'XS Area (m2)',
+                'Stage',
+                'Thalweg_burn_elev',
+                'median_stage_bankfull',
+                'lookup_stage_diff',
+                'mean_xs_area_ratio',
+            ]
+        ]
+        xs_area_hydroid_lookup = xs_area_hydroid_lookup.loc[
+            xs_area_hydroid_lookup.groupby('HydroID')['lookup_stage_diff'].idxmin()
+        ].reset_index(drop=True)
 
         ## Calculate bathy adjusted XS Area ('XS Area (m2)' mutliplied by mean_xs_area_ratio)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']) - xs_area_hydroid_lookup['XS Area (m2)']
+        xs_area_hydroid_lookup['bathy_calc_xs_area'] = (
+            xs_area_hydroid_lookup['XS Area (m2)'] * xs_area_hydroid_lookup['mean_xs_area_ratio']
+        ) - xs_area_hydroid_lookup['XS Area (m2)']
 
         ## Calculate the ratio btw the lookup SRC XS_Area and the Bankfull_XSEC_AREA --> use this as a flag for potentially bad XS data
-        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'])
+        xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] = (
+            xs_area_hydroid_lookup['bathy_calc_xs_area'] / xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)']
+        )
         ## Set bath_cal_xs_area to 0 if the bankfull_XS_ratio_flag is > threshold --> 5x (assuming too large of difference to be a reliable bankfull calculation)
-        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag']>bathy_xsarea_flag,xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],inplace=True)
-        xs_area_hydroid_lookup['barc_on'] = np.where(xs_area_hydroid_lookup['bathy_calc_xs_area'].isnull(), False, True) # field to identify where vmann is on/off
-        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(),0,inplace=True)
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(
+            xs_area_hydroid_lookup['bankfull_XS_ratio_flag'] > bathy_xsarea_flag,
+            xs_area_hydroid_lookup['BANKFULL_XSEC_AREA (m2)'],
+            inplace=True,
+        )
+        xs_area_hydroid_lookup['barc_on'] = np.where(
+            xs_area_hydroid_lookup['bathy_calc_xs_area'].isnull(), False, True
+        )  # field to identify where vmann is on/off
+        xs_area_hydroid_lookup['bathy_calc_xs_area'].mask(
+            xs_area_hydroid_lookup['bankfull_XS_ratio_flag'].isnull(), 0, inplace=True
+        )
 
         ## Merge bathy_calc_xs_area to the modified_src_base
-        modified_src_base = modified_src_base.merge(xs_area_hydroid_lookup.loc[:,['HydroID','bathy_calc_xs_area','barc_on']],how='left',on='HydroID')
+        modified_src_base = modified_src_base.merge(
+            xs_area_hydroid_lookup.loc[:, ['HydroID', 'bathy_calc_xs_area', 'barc_on']],
+            how='left',
+            on='HydroID',
+        )
 
         ## Mask/null the bathy calculated area for streamorders that the user wants to ignore (set bathy_cals_xs_area = 0 for streamorder = 10)
-        modified_src_base['bathy_calc_xs_area'].mask(modified_src_base['order_'] >= ignore_streamorder,0.0,inplace=True)
+        modified_src_base['bathy_calc_xs_area'].mask(
+            modified_src_base['order_'] >= ignore_streamorder, 0.0, inplace=True
+        )
 
         ## Calculate new bathy adjusted channel geometry variables
-        modified_src_base = modified_src_base.rename(columns={'Discharge (m3s-1)':'orig_Discharge (m3s-1)','XS Area (m2)':'orig_XS Area (m2)','Volume (m3)':'orig_Volume (m3)','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'})
-        modified_src_base['XS Area (m2)'] = modified_src_base['orig_XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
-        modified_src_base['Volume (m3)'] = modified_src_base['XS Area (m2)'] * modified_src_base['LENGTHKM'] * 1000
-        modified_src_base['WetArea (m2)'] = modified_src_base['Volume (m3)']/modified_src_base['LENGTHKM']/1000
-        modified_src_base['HydraulicRadius (m)'] = modified_src_base['WetArea (m2)']/modified_src_base['WettedPerimeter (m)']
+        modified_src_base = modified_src_base.rename(
+            columns={
+                'Discharge (m3s-1)': 'orig_Discharge (m3s-1)',
+                'XS Area (m2)': 'orig_XS Area (m2)',
+                'Volume (m3)': 'orig_Volume (m3)',
+                'WetArea (m2)': 'orig_WetArea (m2)',
+                'HydraulicRadius (m)': 'orig_HydraulicRadius (m)',
+            }
+        )
+        modified_src_base['XS Area (m2)'] = (
+            modified_src_base['orig_XS Area (m2)'] + modified_src_base['bathy_calc_xs_area']
+        )
+        modified_src_base['Volume (m3)'] = (
+            modified_src_base['XS Area (m2)'] * modified_src_base['LENGTHKM'] * 1000
+        )
+        modified_src_base['WetArea (m2)'] = (
+            modified_src_base['Volume (m3)'] / modified_src_base['LENGTHKM'] / 1000
+        )
+        modified_src_base['HydraulicRadius (m)'] = (
+            modified_src_base['WetArea (m2)'] / modified_src_base['WettedPerimeter (m)']
+        )
         modified_src_base['HydraulicRadius (m)'].fillna(0, inplace=True)
         ## mask out negative top width differences (avoid thalweg burn notch)
-        modified_src_base['HydraulicRadius (m)'].mask((modified_src_base['HydraulicRadius (m)']>thal_hyd_radius_flag) & (modified_src_base['Stage']<thal_stg_limit),0,inplace=True)
+        modified_src_base['HydraulicRadius (m)'].mask(
+            (modified_src_base['HydraulicRadius (m)'] > thal_hyd_radius_flag)
+            & (modified_src_base['Stage'] < thal_stg_limit),
+            0,
+            inplace=True,
+        )
 
         ## Calculate Q using Manning's equation
-        modified_src_base['Discharge (m3s-1)'] = modified_src_base['WetArea (m2)']* \
-        pow(modified_src_base['HydraulicRadius (m)'],2.0/3)* \
-        pow(modified_src_base['SLOPE'],0.5)/modified_src_base['ManningN']
+        modified_src_base['Discharge (m3s-1)'] = (
+            modified_src_base['WetArea (m2)']
+            * pow(modified_src_base['HydraulicRadius (m)'], 2.0 / 3)
+            * pow(modified_src_base['SLOPE'], 0.5)
+            / modified_src_base['ManningN']
+        )
         ## mask discharge values for stage = 0 rows in SRC (replace with 0) --> do we need SRC to start at 0??
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0,0,inplace=True)
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'],0,inplace=True)
-        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'],-999,inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(modified_src_base['Stage'] == 0, 0, inplace=True)
+        modified_src_base['Discharge (m3s-1)'].mask(
+            modified_src_base['Stage'] == modified_src_base['Thalweg_burn_elev'], 0, inplace=True
+        )
+        modified_src_base['Discharge (m3s-1)'].mask(
+            modified_src_base['Stage'] < modified_src_base['Thalweg_burn_elev'], -999, inplace=True
+        )
 
         ## Organize bathy calc output variables for csv
-        output_bathy = output_bathy[['HydroID','order_','Stage','SurfaceArea (m2)','TopWidth (m)','BANKFULL_WIDTH (m)','Top Width Diff (m)','XS Area (m2)','BANKFULL_XSEC_AREA (m2)','XS Area Diff (m2)','XS Bankfull Area Ratio','count','median_stage_bankfull','mean_xs_area_ratio']]
+        output_bathy = output_bathy[
+            [
+                'HydroID',
+                'order_',
+                'Stage',
+                'SurfaceArea (m2)',
+                'TopWidth (m)',
+                'BANKFULL_WIDTH (m)',
+                'Top Width Diff (m)',
+                'XS Area (m2)',
+                'BANKFULL_XSEC_AREA (m2)',
+                'XS Area Diff (m2)',
+                'XS Bankfull Area Ratio',
+                'count',
+                'median_stage_bankfull',
+                'mean_xs_area_ratio',
+            ]
+        ]
 
         ## Export bathy/bankful calculation tables for easy viewing
-        output_bathy.to_csv(output_bathy_fileName,index=False)
-        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName,index=True)
-        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName,index=True)
-        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName,index=True)
+        output_bathy.to_csv(output_bathy_fileName, index=False)
+        stream_order_bathy_ratio.to_csv(output_bathy_streamorder_fileName, index=True)
+        find_thalweg_notch.to_csv(output_bathy_thalweg_fileName, index=True)
+        xs_area_hydroid_lookup.to_csv(output_bathy_xs_lookup_fileName, index=True)
 
         ## Output new src_full_crosswalked
-        modified_src_base.to_csv(out_src_filename,index=False)
+        modified_src_base.to_csv(out_src_filename, index=False)
         ## Update the hydroTable
-        modified_hydro_table = modified_src_base.loc[:,['HydroID','Stage','barc_on','Volume (m3)','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)']]
-        modified_hydro_table.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
-        df_htable = pd.read_csv(input_htable_fileName,dtype={'HUC': str})
-        df_htable.drop(['barc_on'], axis=1, inplace=True) # drop the default "barc_on" variable from add_crosswalk.py
-        if not set(['orig_discharge_cms','orig_Volume (m3)','orig_WetArea (m2)','orig_HydraulicRadius (m)']).issubset(df_htable.columns): # check if "orig_" attributes do NOT already exist (likely generated from previous BARC run)
-            df_htable.rename(columns={'discharge_cms':'orig_discharge_cms','Volume (m3)':'orig_Volume (m3)','WetArea (m2)':'orig_WetArea (m2)','HydraulicRadius (m)':'orig_HydraulicRadius (m)'},inplace=True)
+        modified_hydro_table = modified_src_base.loc[
+            :,
+            [
+                'HydroID',
+                'Stage',
+                'barc_on',
+                'Volume (m3)',
+                'WetArea (m2)',
+                'HydraulicRadius (m)',
+                'Discharge (m3s-1)',
+            ],
+        ]
+        modified_hydro_table.rename(
+            columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'}, inplace=True
+        )
+        df_htable = pd.read_csv(input_htable_fileName, dtype={'HUC': str})
+        df_htable.drop(
+            ['barc_on'], axis=1, inplace=True
+        )  # drop the default "barc_on" variable from add_crosswalk.py
+        if not set(
+            ['orig_discharge_cms', 'orig_Volume (m3)', 'orig_WetArea (m2)', 'orig_HydraulicRadius (m)']
+        ).issubset(
+            df_htable.columns
+        ):  # check if "orig_" attributes do NOT already exist (likely generated from previous BARC run)
+            df_htable.rename(
+                columns={
+                    'discharge_cms': 'orig_discharge_cms',
+                    'Volume (m3)': 'orig_Volume (m3)',
+                    'WetArea (m2)': 'orig_WetArea (m2)',
+                    'HydraulicRadius (m)': 'orig_HydraulicRadius (m)',
+                },
+                inplace=True,
+            )
         else:
-            df_htable.drop(['discharge_cms','Volume (m3)','WetArea (m2)','HydraulicRadius (m)'], axis=1, inplace=True) # drop the previously modified columns - to be replaced with updated version
-        df_htable = df_htable.merge(modified_hydro_table, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
-        df_htable.to_csv(input_htable_fileName,index=False)
+            df_htable.drop(
+                ['discharge_cms', 'Volume (m3)', 'WetArea (m2)', 'HydraulicRadius (m)'], axis=1, inplace=True
+            )  # drop the previously modified columns - to be replaced with updated version
+        df_htable = df_htable.merge(
+            modified_hydro_table, how='left', left_on=['HydroID', 'stage'], right_on=['HydroID', 'stage']
+        )
+        df_htable.to_csv(input_htable_fileName, index=False)
         log_text += ('Output new hydroTable and src_full_crosswalked: ') + '\n'
-        log_text += ('Completed Bathy Calculations: ') + str(huc) +'\n#################\n'
+        log_text += ('Completed Bathy Calculations: ') + str(huc) + '\n#################\n'
 
         ## plot rating curves (optional arg)
         if src_plot_option == 'True':
-            if isdir(huc_plot_output_dir) == False:
+            if isdir(huc_plot_output_dir) is False:
                 os.mkdir(huc_plot_output_dir)
             generate_src_plot(df_htable, huc_plot_output_dir)
 
-        return(log_text)
+        return log_text
 
 
 def generate_src_plot(df_src, plt_out_dir):
-
     ## create list of unique hydroids
     hydroids = df_src.HydroID.unique().tolist()
 
@@ -237,29 +465,52 @@ def generate_src_plot(df_src, plt_out_dir):
         f, ax = plt.subplots(figsize=(6.5, 6.5))
         ax.set_title(str(hydroid))
         sns.despine(f, left=True, bottom=True)
-        sns.scatterplot(x='orig_discharge_cms', y='stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
-        sns.scatterplot(x='discharge_cms', y='stage', data=plot_df, label="SRC w/ BARC", ax=ax, color='orange')
-        #sns.lineplot(x='discharge_1_5', y='Stage_1_5', data=plot_df, color='green', ax=ax)
-        #plt.fill_between(plot_df['discharge_1_5'], plot_df['Stage_1_5'],alpha=0.5)
-        #plt.text(plot_df['discharge_1_5'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
+        sns.scatterplot(
+            x='orig_discharge_cms', y='stage', data=plot_df, label="Orig SRC", ax=ax, color='blue'
+        )
+        sns.scatterplot(
+            x='discharge_cms', y='stage', data=plot_df, label="SRC w/ BARC", ax=ax, color='orange'
+        )
+        # sns.lineplot(x='discharge_1_5', y='Stage_1_5', data=plot_df, color='green', ax=ax)
+        # plt.fill_between(plot_df['discharge_1_5'], plot_df['Stage_1_5'],alpha=0.5)
+        # plt.text(plot_df['discharge_1_5'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
         ax.legend()
-        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_barc.png',dpi=175, bbox_inches='tight')
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_barc.png', dpi=175, bbox_inches='tight')
         plt.close()
 
+
 def multi_process(bathy_rc_lookup, procs_list):
     print(f"Applying bathy adjustment calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
     with Pool(processes=number_of_jobs) as pool:
         map_output = pool.map(bathy_rc_lookup, procs_list)
-        #log_file.write(str(map_output))
-    log_file.writelines(["%s\n" % item  for item in map_output])
+        # log_file.write(str(map_output))
+    log_file.writelines(["%s\n" % item for item in map_output])
+
 
 if __name__ == '__main__':
-    #output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName
-    parser = argparse.ArgumentParser(description="Estimate the unaccounted for channel bathymetry using a regression-based estimate of channel XSec Area")
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-bfull_geom','--bankfull-xsec-input',help='Regression dataset w/ bankfull geometry by featureid (topwidth & xsec area)',required=True,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-    parser.add_argument('-plots','--src-plot-option',help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',required=False,default='False',type=str)
+    # output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName
+    parser = argparse.ArgumentParser(
+        description="Estimate the unaccounted for channel bathymetry using a regression-based estimate of channel XSec Area"
+    )
+    parser.add_argument('-fim_dir', '--fim-dir', help='FIM output dir', required=True, type=str)
+    parser.add_argument(
+        '-bfull_geom',
+        '--bankfull-xsec-input',
+        help='Regression dataset w/ bankfull geometry by featureid (topwidth & xsec area)',
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-j', '--number-of-jobs', help='number of workers', required=False, default=1, type=int
+    )
+    parser.add_argument(
+        '-plots',
+        '--src-plot-option',
+        help='Optional (True or False): use this flag to create src plots for all hydroids. WARNING - long runtime',
+        required=False,
+        default='False',
+        type=str,
+    )
 
     args = vars(parser.parse_args())
 
@@ -272,60 +523,112 @@ def multi_process(bathy_rc_lookup, procs_list):
     print('STARTING Bathy Adjusted Rating Curve routine...')
     ## Check that the input bankfull geom filepath exists and then read it to dataframe
     if not isfile(bankfull_regres_filepath):
-        print('!!! Can not find the input bankfull geometry regression file: ' + str(bankfull_regres_filepath))
+        print(
+            '!!! Can not find the input bankfull geometry regression file: ' + str(bankfull_regres_filepath)
+        )
     else:
         ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
         print('Importing the bankfull regression data file: ' + bankfull_regres_filepath)
-        df_bfull_geom = pd.read_csv(bankfull_regres_filepath,dtype= {'COMID': int})
+        df_bfull_geom = pd.read_csv(bankfull_regres_filepath, dtype={'COMID': int})
         if 'COMID' not in df_bfull_geom.columns and 'feature_id' not in df_bfull_geom.columns:
             print('Missing required data column ("feature_id" or "COMID")!!! --> ' + df_bfull_geom)
         else:
             print('Running Bathy Adjust Rating Curve (BARC)...')
 
             ## Print message to user and initiate run clock
-            print('Writing progress to log file here: ' + str(join(fim_dir,'log_BARC.log')))
+            print('Writing progress to log file here: ' + str(join(fim_dir, 'log_BARC.log')))
             print('This may take a few minutes...')
             ## Create a time var to log run time
             begin_time = dt.datetime.now()
 
             ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
-            huc_list  = os.listdir(fim_dir)
+            huc_list = os.listdir(fim_dir)
             huc_pass_list = []
             for huc in huc_list:
                 if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
-                    #output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName
-                    in_src_filename = join(fim_dir,huc,'src_full_crosswalked.csv')
-                    out_src_filename = join(fim_dir,huc,'src_full_crosswalked_BARC.csv')
-                    htable_filename = join(fim_dir,huc,'hydroTable.csv')
-                    output_bath_filename = join(fim_dir,huc,'bathy_crosswalk_calcs.csv')
-                    output_bathy_thalweg_fileName = join(fim_dir,huc,'bathy_thalweg_flag.csv')
-                    output_bathy_streamorder_fileName = join(fim_dir,huc,'bathy_stream_order_calcs.csv')
-                    output_bathy_thalweg_fileName = join(fim_dir,huc,'bathy_thalweg_flag.csv')
-                    output_bathy_xs_lookup_fileName = join(fim_dir,huc,'bathy_xs_area_hydroid_lookup.csv')
-                    huc_plot_output_dir = join(fim_dir,huc,'src_plots')
+                    # output_src,input_bathy_fileName,output_bathy_fileName,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName
+                    in_src_filename = join(fim_dir, huc, 'src_full_crosswalked.csv')
+                    out_src_filename = join(fim_dir, huc, 'src_full_crosswalked_BARC.csv')
+                    htable_filename = join(fim_dir, huc, 'hydroTable.csv')
+                    output_bath_filename = join(fim_dir, huc, 'bathy_crosswalk_calcs.csv')
+                    output_bathy_thalweg_fileName = join(fim_dir, huc, 'bathy_thalweg_flag.csv')
+                    output_bathy_streamorder_fileName = join(fim_dir, huc, 'bathy_stream_order_calcs.csv')
+                    output_bathy_thalweg_fileName = join(fim_dir, huc, 'bathy_thalweg_flag.csv')
+                    output_bathy_xs_lookup_fileName = join(fim_dir, huc, 'bathy_xs_area_hydroid_lookup.csv')
+                    huc_plot_output_dir = join(fim_dir, huc, 'src_plots')
 
                     if isfile(in_src_filename):
                         print(str(huc))
                         huc_pass_list.append(str(huc))
-                        procs_list.append([in_src_filename,df_bfull_geom,output_bath_filename,output_bathy_streamorder_fileName,output_bathy_thalweg_fileName,output_bathy_xs_lookup_fileName,htable_filename,out_src_filename,huc,src_plot_option,huc_plot_output_dir])
+                        procs_list.append(
+                            [
+                                in_src_filename,
+                                df_bfull_geom,
+                                output_bath_filename,
+                                output_bathy_streamorder_fileName,
+                                output_bathy_thalweg_fileName,
+                                output_bathy_xs_lookup_fileName,
+                                htable_filename,
+                                out_src_filename,
+                                huc,
+                                src_plot_option,
+                                huc_plot_output_dir,
+                            ]
+                        )
                     else:
-                        print(str(huc) + ' --> can not find the src_full_crosswalked.csv in the fim output dir: ' + str(join(fim_dir,huc)))
+                        print(
+                            str(huc)
+                            + ' --> can not find the src_full_crosswalked.csv in the fim output dir: '
+                            + str(join(fim_dir, huc))
+                        )
 
             ## initiate log file
-            print(f"Applying bathy adjustment calcs for {len(procs_list)} hucs using {number_of_jobs} jobs...")
+            print(
+                f"Applying bathy adjustment calcs for {len(procs_list)} hucs using {number_of_jobs} jobs..."
+            )
             sys.__stdout__ = sys.stdout
-            log_file = open(join(fim_dir,'logs','log_barc.log'),"w")
+            log_file = open(join(fim_dir, 'logs', 'log_barc.log'), "w")
             sys.stdout = log_file
             log_file.write('START TIME: ' + str(begin_time) + '\n')
-            log_file.writelines(["%s\n" % item  for item in huc_pass_list])
+            log_file.writelines(["%s\n" % item for item in huc_pass_list])
 
             ## Write env variables to log files
-            log_file.write('sa_ratio_flag = ' + str(sa_ratio_flag) + ' --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)' + '\n')
-            log_file.write('thal_stg_limit = ' + str(thal_stg_limit) + ' --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)' + '\n')
-            log_file.write('bankful_xs_ratio_flag = ' + str(bankful_xs_ratio_flag) + ' --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)' + '\n')
-            log_file.write('bathy_xsarea_flag = ' + str(bathy_xsarea_flag) + ' --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold' + '\n')
-            log_file.write('thal_hyd_radius_flag = ' + str(thal_hyd_radius_flag) + ' --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0' + '\n')
-            log_file.write('ignore_streamorder = ' + str(ignore_streamorder) + ' --> Do not perform BARC for streamorders >= provided value' + '\n')
+            log_file.write(
+                'sa_ratio_flag = '
+                + str(sa_ratio_flag)
+                + ' --> Flag: Surface area ratio value to identify possible thalweg notch "jump" (SA x+1 / SA x)'
+                + '\n'
+            )
+            log_file.write(
+                'thal_stg_limit = '
+                + str(thal_stg_limit)
+                + ' --> Threshold: Stage value limit below which to look for the surface area ratio flag (only flag thalweg notch below this threshold)'
+                + '\n'
+            )
+            log_file.write(
+                'bankful_xs_ratio_flag = '
+                + str(bankful_xs_ratio_flag)
+                + ' --> Flag: Identify bogus BARC adjusted values where the regression bankfull XS Area/SRC bankfull area is > threshold (topwidth crosswalk issues or bad bankfull regression data points??)'
+                + '\n'
+            )
+            log_file.write(
+                'bathy_xsarea_flag = '
+                + str(bathy_xsarea_flag)
+                + ' --> Flag: Cross section area limit to cap the amount of bathy XS area added to the SRC. Limits the bathy_calc_xs_area/ BANKFULL_XSEC_AREA to the specified threshold'
+                + '\n'
+            )
+            log_file.write(
+                'thal_hyd_radius_flag = '
+                + str(thal_hyd_radius_flag)
+                + ' --> Flag: Idenitify possible erroneous BARC-adjusted hydraulic radius values. BARC discharge values greater than the specified threshold and within the thal_stg_limit are set to 0'
+                + '\n'
+            )
+            log_file.write(
+                'ignore_streamorder = '
+                + str(ignore_streamorder)
+                + ' --> Do not perform BARC for streamorders >= provided value'
+                + '\n'
+            )
             log_file.write('#########################################################\n\n')
 
             ## Pass huc procs_list to multiprocessing function
diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py
index 8d9243292..b0421bc4f 100644
--- a/src/bathymetric_adjustment.py
+++ b/src/bathymetric_adjustment.py
@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
 
-import sys, os, traceback, re
-from os.path import join
-import pandas as pd
-import geopandas as gpd
 import datetime as dt
+import os
+import re
+import sys
+import traceback
 from argparse import ArgumentParser
 from concurrent.futures import ProcessPoolExecutor
-sys.path.append('/foss_fim/tools')
+from os.path import join
+
+import geopandas as gpd
+import pandas as pd
 from synthesize_test_cases import progress_bar_handler
 
 
@@ -22,7 +25,7 @@ def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
         huc : str
             HUC-8 string.
         bathy_file : str
-            Path to bathymetric adjustment geopackage, e.g. 
+            Path to bathymetric adjustment geopackage, e.g.
             "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
         verbose : bool
             Verbose printing.
@@ -30,7 +33,7 @@ def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
         Returns
         ----------
         log_text : str
-            
+
     """
 
     log_text = f'Calculating bathymetry adjustment: {huc}\n'
@@ -39,7 +42,7 @@ def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
     fim_huc_dir = join(fim_dir, huc)
     wbd8_clp = gpd.read_file(join(fim_huc_dir, 'wbd8_clp.gpkg'))
     bathy_data = gpd.read_file(bathy_file, mask=wbd8_clp)
-    bathy_data = bathy_data.rename(columns={'ID':'feature_id'})
+    bathy_data = bathy_data.rename(columns={'ID': 'feature_id'})
 
     # Get src_full from each branch
     src_all_branches = []
@@ -51,50 +54,66 @@ def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
 
     # Update src parameters with bathymetric data
     for src in src_all_branches:
-
         src_df = pd.read_csv(src)
         if 'Bathymetry_source' in src_df.columns:
             src_df = src_df.drop(columns='Bathymetry_source')
-        branch = re.search('branches/(\d{10}|0)/', src).group()[9:-1]
+        branch = re.search(r'branches/(\d{10}|0)/', src).group()[9:-1]
         log_text += f'  Branch: {branch}\n'
 
         if bathy_data.empty:
             log_text += '  There were no bathymetry feature_ids for this branch'
-            src_df['Bathymetry_source'] = [""]* len(src_df)
+            src_df['Bathymetry_source'] = [""] * len(src_df)
             src_df.to_csv(src, index=False)
             return log_text
-    
+
         # Merge in missing bathy data and fill Nans
         try:
-            src_df = src_df.merge(bathy_data[['feature_id', 'missing_xs_area_m2', 'missing_wet_perimeter_m', 'Bathymetry_source']], 
-                                  on='feature_id', how='left', validate='many_to_one')
+            src_df = src_df.merge(
+                bathy_data[
+                    ['feature_id', 'missing_xs_area_m2', 'missing_wet_perimeter_m', 'Bathymetry_source']
+                ],
+                on='feature_id',
+                how='left',
+                validate='many_to_one'
+            )
         # If there's more than one feature_id in the bathy data, just take the mean
         except pd.errors.MergeError:
-            reconciled_bathy_data = bathy_data.groupby('feature_id')[['missing_xs_area_m2', 'missing_wet_perimeter_m']].mean()
-            reconciled_bathy_data['Bathymetry_source'] = bathy_data.groupby('feature_id')['Bathymetry_source'].first()
+            reconciled_bathy_data = bathy_data.groupby('feature_id')[
+                ['missing_xs_area_m2', 'missing_wet_perimeter_m']
+            ].mean()
+            reconciled_bathy_data['Bathymetry_source'] = bathy_data.groupby('feature_id')[
+                'Bathymetry_source'
+            ].first()
             src_df = src_df.merge(reconciled_bathy_data, on='feature_id', how='left', validate='many_to_one')
         # Exit if there are no recalculations to be made
         if ~src_df['Bathymetry_source'].any(axis=None):
-            log_text += f'    No matching feature_ids in this branch\n'
+            log_text += '    No matching feature_ids in this branch\n'
             continue
 
         src_df['missing_xs_area_m2'] = src_df['missing_xs_area_m2'].fillna(0.0)
         src_df['missing_wet_perimeter_m'] = src_df['missing_wet_perimeter_m'].fillna(0.0)
         # Add missing hydraulic geometry into base parameters
-        src_df['Volume (m3)'] = src_df['Volume (m3)'] + (src_df['missing_xs_area_m2'] * (src_df['LENGTHKM'] * 1000))
-        src_df['BedArea (m2)'] = src_df['BedArea (m2)'] + (src_df['missing_wet_perimeter_m'] * (src_df['LENGTHKM'] * 1000))
+        src_df['Volume (m3)'] = src_df['Volume (m3)'] + (
+            src_df['missing_xs_area_m2'] * (src_df['LENGTHKM'] * 1000)
+        )
+        src_df['BedArea (m2)'] = src_df['BedArea (m2)'] + (
+            src_df['missing_wet_perimeter_m'] * (src_df['LENGTHKM'] * 1000)
+        )
         # Recalc discharge with adjusted geometries
         src_df['WettedPerimeter (m)'] = src_df['WettedPerimeter (m)'] + src_df['missing_wet_perimeter_m']
         src_df['WetArea (m2)'] = src_df['WetArea (m2)'] + src_df['missing_xs_area_m2']
-        src_df['HydraulicRadius (m)'] = src_df['WetArea (m2)']/src_df['WettedPerimeter (m)']
+        src_df['HydraulicRadius (m)'] = src_df['WetArea (m2)'] / src_df['WettedPerimeter (m)']
         src_df['HydraulicRadius (m)'] = src_df['HydraulicRadius (m)'].fillna(0)
-        src_df['Discharge (m3s-1)'] = src_df['WetArea (m2)']* \
-            pow(src_df['HydraulicRadius (m)'],2.0/3)* \
-            pow(src_df['SLOPE'],0.5)/src_df['ManningN']
+        src_df['Discharge (m3s-1)'] = (
+            src_df['WetArea (m2)']
+            * pow(src_df['HydraulicRadius (m)'], 2.0 / 3)
+            * pow(src_df['SLOPE'], 0.5)
+            / src_df['ManningN']
+        )
         # Force zero stage to have zero discharge
-        src_df.loc[src_df['Stage']==0,['Discharge (m3s-1)']] = 0
+        src_df.loc[src_df['Stage'] == 0, ['Discharge (m3s-1)']] = 0
         # Calculate number of adjusted HydroIDs
-        count = len(src_df.loc[(src_df['Stage']==0) & (src_df['Bathymetry_source'] == 'USACE eHydro')])
+        count = len(src_df.loc[(src_df['Stage'] == 0) & (src_df['Bathymetry_source'] == 'USACE eHydro')])
 
         # Write src back to file
         src_df.to_csv(src, index=False)
@@ -111,15 +130,15 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
         fim_dir : str
             Directory path for fim_pipeline output.
         bathy_file : str
-            Path to bathymetric adjustment geopackage, e.g. 
+            Path to bathymetric adjustment geopackage, e.g.
             "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
         wbd_buffer : int
             Distance in meters to buffer wbd dataset when searching for relevant HUCs.
         wbd : str
-            Path to wbd input data, e.g. 
+            Path to wbd input data, e.g.
             "/data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg".
         output_suffix : str
-            Output filename suffix. 
+            Output filename suffix.
         number_of_jobs : int
             Number of CPU cores to parallelize HUC processing.
         verbose : bool
@@ -128,13 +147,16 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
     """
 
     # Set up log file
-    print('Writing progress to log file here: ' + str(join(fim_dir,'logs','bathymetric_adjustment' + output_suffix + '.log')))
+    print(
+        'Writing progress to log file here: '
+        + str(join(fim_dir, 'logs', 'bathymetric_adjustment' + output_suffix + '.log'))
+    )
     print('This may take a few minutes...')
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
 
     ## Initiate log file
-    log_file = open(join(fim_dir,'logs','bathymetric_adjustment' + output_suffix + '.log'),"w")
+    log_file = open(join(fim_dir, 'logs', 'bathymetric_adjustment' + output_suffix + '.log'), "w")
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
 
@@ -148,10 +170,10 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
     # Find applicable HUCs to apply bathymetric adjustment
     # NOTE: This block can be removed if we have estimated bathymetry data for
     # the whole domain later.
-    fim_hucs = [h for h in os.listdir(fim_dir) if re.match('\d{8}', h)]
+    fim_hucs = [h for h in os.listdir(fim_dir) if re.match(r'\d{8}', h)]
     bathy_gdf = gpd.read_file(bathy_file)
-    buffered_bathy = bathy_gdf.geometry.buffer(wbd_buffer)    # We buffer the bathymetric data to get adjacent 
-    wbd = gpd.read_file(wbd, mask=buffered_bathy)             # HUCs that could also have bathymetric reaches included
+    buffered_bathy = bathy_gdf.geometry.buffer(wbd_buffer)  # We buffer the bathymetric data to get adjacent
+    wbd = gpd.read_file(wbd, mask=buffered_bathy)  # HUCs that could also have bathymetric reaches included
     hucs_with_bathy = wbd.HUC8.to_list()
     hucs = [h for h in fim_hucs if h in hucs_with_bathy]
     log_file.write(f"Identified {len(hucs)} HUCs that have bathymetric data: {hucs}\n")
@@ -159,17 +181,10 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
 
     # Set up multiprocessor
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
-
         # Loop through all test cases, build the alpha test arguments, and submit them to the process pool
         executor_dict = {}
         for huc in hucs:
-            
-            arg_keeper = { 
-                        'fim_dir': fim_dir,
-                        'huc': huc,
-                        'bathy_file': bathy_file,
-                        'verbose': verbose,
-                        }
+            arg_keeper = {'fim_dir': fim_dir, 'huc': huc, 'bathy_file': bathy_file, 'verbose': verbose}
             future = executor.submit(correct_rating_for_bathymetry, **arg_keeper)
             executor_dict[future] = huc
 
@@ -181,7 +196,7 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
                 log_file.write(future.result())
             except Exception as ex:
                 print(f"WARNING: {executor_dict[future]} BARC failed for some reason")
-                log_file.write( f"ERROR --> {executor_dict[future]} BARC failed (details: *** {ex} )\n")
+                log_file.write(f"ERROR --> {executor_dict[future]} BARC failed (details: *** {ex} )\n")
                 traceback.print_exc(file=log_file)
 
     ## Record run time and close log file
@@ -191,24 +206,24 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
     log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
     log_file.close()
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     """
     Parameters
     ----------
     fim_dir : str
-        Directory path for fim_pipeline output. Log file will be placed in 
+        Directory path for fim_pipeline output. Log file will be placed in
         fim_dir/logs/bathymetric_adjustment.log.
     bathy_file : str
-        Path to bathymetric adjustment geopackage, e.g. 
+        Path to bathymetric adjustment geopackage, e.g.
         "/data/inputs/bathymetry/bathymetry_adjustment_data.gpkg".
     wbd_buffer : int
         Distance in meters to buffer wbd dataset when searching for relevant HUCs.
     wbd : str
-        Path to wbd input data, e.g. 
+        Path to wbd input data, e.g.
         "/data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg".
     output_suffix : str
-        Optional. Output filename suffix. Defaults to no suffix. 
+        Optional. Output filename suffix. Defaults to no suffix.
     number_of_jobs : int
         Optional. Number of CPU cores to parallelize HUC processing. Defaults to 8.
     verbose : bool
@@ -216,19 +231,58 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
 
     Sample Usage
     ----------
-    python3 /foss_fim/src/bathymetric_adjustment.py -fim_dir /outputs/fim_run_dir -bathy /data/inputs/bathymetry/bathymetry_adjustment_data.gpkg 
+    python3 /foss_fim/src/bathymetric_adjustment.py -fim_dir /outputs/fim_run_dir -bathy /data/inputs/bathymetry/bathymetry_adjustment_data.gpkg
         -buffer 5000 -wbd /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -j $jobLimit
 
     """
 
     parser = ArgumentParser(description="Bathymetric Adjustment")
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-bathy','--bathy_file',help="Path to geopackage with preprocessed bathymetic data",required=True,type=str)
-    parser.add_argument('-buffer','--wbd-buffer',help="Buffer to apply to bathymetry data to find applicable HUCs",required=True,type=int)
-    parser.add_argument('-wbd','--wbd',help="Buffer to apply to bathymetry data to find applicable HUCs",required=True,type=str)
-    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",default="",required=False,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='OPTIONAL: number of workers (default=8)',required=False,default=8,type=int)
-    parser.add_argument('-vb','--verbose',help='OPTIONAL: verbose progress bar',required=False,default=None,action='store_true')
+    parser.add_argument('-fim_dir', '--fim-dir', help='FIM output dir', required=True, type=str)
+    parser.add_argument(
+        '-bathy',
+        '--bathy_file',
+        help="Path to geopackage with preprocessed bathymetic data",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-buffer',
+        '--wbd-buffer',
+        help="Buffer to apply to bathymetry data to find applicable HUCs",
+        required=True,
+        type=int,
+    )
+    parser.add_argument(
+        '-wbd',
+        '--wbd',
+        help="Buffer to apply to bathymetry data to find applicable HUCs",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-suff',
+        '--output-suffix',
+        help="Suffix to append to the output log file (e.g. '_global_06_011')",
+        default="",
+        required=False,
+        type=str,
+    )
+    parser.add_argument(
+        '-j',
+        '--number-of-jobs',
+        help='OPTIONAL: number of workers (default=8)',
+        required=False,
+        default=8,
+        type=int,
+    )
+    parser.add_argument(
+        '-vb',
+        '--verbose',
+        help='OPTIONAL: verbose progress bar',
+        required=False,
+        default=None,
+        action='store_true',
+    )
 
     args = vars(parser.parse_args())
 
diff --git a/src/buffer_stream_branches.py b/src/buffer_stream_branches.py
index 530998bde..6d4e29724 100755
--- a/src/buffer_stream_branches.py
+++ b/src/buffer_stream_branches.py
@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
+
 import geopandas as gpd
-from stream_branches import StreamNetwork
-from stream_branches import StreamBranchPolygons
-import argparse
+
+from stream_branches import StreamBranchPolygons, StreamNetwork
 
 
 def buffer_stream_branches(
diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py
index c73d896bf..590398c12 100644
--- a/src/build_stream_traversal.py
+++ b/src/build_stream_traversal.py
@@ -1,18 +1,24 @@
+import argparse
+import sys
+
+import geopandas as gpd
+
+
 '''
 Description:
-        This tool creates unique IDs for each segment and builds the To_Node, From_Node, and NextDownID columns to traverse the network
+        This tool creates unique IDs for each segment and builds the To_Node, From_Node, and NextDownID
+        columns to traverse the network.
 Required Arguments:
         streams   = stream network
         wbd8          = HUC8 boundary dataset
         hydro_id       = name of ID column (string)
 '''
-import sys
-import argparse
-import geopandas as gpd
 
 
 def trace():
-    import traceback, inspect
+    import inspect
+    import traceback
+
     tb = sys.exc_info()[2]
     tbinfo = traceback.format_tb(tb)[0]
     line = tbinfo.split(", ")[1]
@@ -21,6 +27,7 @@ def trace():
     synerror = traceback.format_exc().splitlines()[-1]
     return line, filename, synerror
 
+
 from_node = "From_Node"
 to_node = "To_Node"
 next_down_id = "NextDownID"
@@ -28,10 +35,13 @@ def trace():
 
 class build_stream_traversal_columns(object):
     '''Tool class for updating the next down IDs of stream features.'''
+
     def __init__(self):
         '''Define tool properties (tool name is the class name).'''
         self.label = 'Find Next Downstream Line'
-        self.description = '''Finds next downstream line, retrieves its HydroID and stores it in the NextDownID field.'''
+        self.description = (
+            '''Finds next downstream line, retrieves its HydroID and stores it in the NextDownID field.'''
+        )
 
     def execute(self, streams, wbd8, hydro_id):
         try:
@@ -39,51 +49,61 @@ def execute(self, streams, wbd8, hydro_id):
             sOK = 'OK'
 
             # check for HydroID; Assign if it doesn't exist
-            if not hydro_id in streams.columns:
-                print ("Required field " + hydro_id + " does not exist in input. Generating..")
+            if hydro_id not in streams.columns:
+                print("Required field " + hydro_id + " does not exist in input. Generating..")
 
                 # Get stream midpoint
                 stream_midpoint = []
-                for i,lineString in enumerate(streams.geometry):
-                    stream_midpoint = stream_midpoint + [lineString.interpolate(0.5,normalized=True)]
+                for i, lineString in enumerate(streams.geometry):
+                    stream_midpoint = stream_midpoint + [lineString.interpolate(0.5, normalized=True)]
 
-                stream_md_gpd = gpd.GeoDataFrame({'geometry':stream_midpoint}, crs=streams.crs, geometry='geometry')
+                stream_md_gpd = gpd.GeoDataFrame(
+                    {'geometry': stream_midpoint}, crs=streams.crs, geometry='geometry'
+                )
                 stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', predicate='within')
-                stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "midpoint", "index_right": "HUC8id"})
+                stream_wbdjoin = stream_wbdjoin.rename(
+                    columns={"geometry": "midpoint", "index_right": "HUC8id"}
+                )
                 streams = streams.join(stream_wbdjoin).drop(columns=['midpoint'])
 
-                streams['seqID'] = (streams.groupby('HUC8id', dropna=False).cumcount(ascending=True)+1).astype('str').str.zfill(4)
-                streams = streams.loc[streams['HUC8id'].notna(),:]
-                if streams.HUC8id.dtype != 'str': streams.HUC8id = streams.HUC8id.astype(str)
-                if streams.seqID.dtype != 'str': streams.seqID = streams.seqID.astype(str)
-
-                streams = streams.assign(hydro_id= lambda x: x.HUC8id + x.seqID)
+                streams['seqID'] = (
+                    (streams.groupby('HUC8id', dropna=False).cumcount(ascending=True) + 1)
+                    .astype('str')
+                    .str.zfill(4)
+                )
+                streams = streams.loc[streams['HUC8id'].notna(), :]
+                if streams.HUC8id.dtype != 'str':
+                    streams.HUC8id = streams.HUC8id.astype(str)
+                if streams.seqID.dtype != 'str':
+                    streams.seqID = streams.seqID.astype(str)
+
+                streams = streams.assign(hydro_id=lambda x: x.HUC8id + x.seqID)
                 streams = streams.rename(columns={"hydro_id": hydro_id}).sort_values(hydro_id)
                 streams = streams.drop(columns=['HUC8id', 'seqID'])
                 streams[hydro_id] = streams[hydro_id].astype(int)
-                print ('Generated ' + hydro_id)
+                print('Generated ' + hydro_id)
 
             # Check for TO/From Nodes; Assign if doesnt exist
             bOK = True
-            if not from_node in streams.columns:
-                print ("Field " + from_node + " does not exist in input ")
+            if from_node not in streams.columns:
+                print("Field " + from_node + " does not exist in input ")
                 bOK = False
-            if not to_node in streams.columns:
-                print ("Field " + to_node + " does not exist in input. Generating..")
+            if to_node not in streams.columns:
+                print("Field " + to_node + " does not exist in input. Generating..")
                 bOK = False
 
-            if(bOK==False):
+            if bOK is False:
                 # Add fields if not they do not exist.
-                if not from_node in streams.columns:
+                if from_node not in streams.columns:
                     streams[from_node] = ''
 
-                if not to_node in streams.columns:
+                if to_node not in streams.columns:
                     streams[to_node] = ''
 
                 streams = streams.sort_values(by=[hydro_id], ascending=True).copy()
 
                 xy_dict = {}
-                bhasnullshape=False
+                bhasnullshape = False
                 for rows in streams[['geometry', from_node, to_node]].iterrows():
                     if rows[1][0]:
                         # From Node
@@ -91,42 +111,41 @@ def execute(self, streams, wbd8, hydro_id):
                         firsty = round(rows[1][0].coords.xy[1][0], 7)
                         from_key = '{},{}'.format(firstx, firsty)
                         if from_key in xy_dict:
-                            streams.at[rows[0], from_node,] = xy_dict[from_key]
+                            streams.at[rows[0], from_node] = xy_dict[from_key]
                         else:
                             xy_dict[from_key] = len(xy_dict) + 1
-                            streams.at[rows[0], from_node,] = xy_dict[from_key]
+                            streams.at[rows[0], from_node] = xy_dict[from_key]
 
                         # To Node
                         lastx = round(rows[1][0].coords.xy[0][-1], 7)
                         lasty = round(rows[1][0].coords.xy[1][-1], 7)
                         to_key = '{},{}'.format(lastx, lasty)
-                        #if xy_dict.has_key(to_key):
+                        # if xy_dict.has_key(to_key):
                         if to_key in xy_dict:
                             streams.at[rows[0], to_node] = xy_dict[to_key]
                         else:
                             xy_dict[to_key] = len(xy_dict) + 1
                             streams.at[rows[0], to_node] = xy_dict[to_key]
                     else:
-                         bhasnullshape=True
+                        bhasnullshape = True
 
-                if bhasnullshape==True:
-                    print ("Some of the input features have a null shape.")
-                    print (from_node + " and " + to_node + " fields cannot be populated for those features.")
+                if bhasnullshape is True:
+                    print("Some of the input features have a null shape.")
+                    print(from_node + " and " + to_node + " fields cannot be populated for those features.")
                 else:
-                    print ('Generated To/From Nodes')
+                    print('Generated To/From Nodes')
 
             # Create NextDownID field
-            if not next_down_id in streams.columns:
+            if next_down_id not in streams.columns:
                 streams[next_down_id] = ''
 
             # Create dict to store from_node values for each HydroID
-            dnodes=dict()
-            lstHydroIDs=[]
-            for row in streams[[from_node,hydro_id]].iterrows():
-
-                if (row[1][0] in dnodes)==False:
-                    lstHydroIDs=[row[1][1]]
-                    dnodes.setdefault(row[1][0],lstHydroIDs)
+            dnodes = dict()
+            lstHydroIDs = []
+            for row in streams[[from_node, hydro_id]].iterrows():
+                if (row[1][0] in dnodes) is False:
+                    lstHydroIDs = [row[1][1]]
+                    dnodes.setdefault(row[1][0], lstHydroIDs)
                 else:
                     lstHydroIDs = dnodes[row[1][0]]
                     lstHydroIDs.append(row[1][1])
@@ -139,51 +158,53 @@ def execute(self, streams, wbd8, hydro_id):
                 try:
                     next_down_ids = dnodes[tonodecol]
                 except:
-                    next_down_ids=[]
+                    next_down_ids = []
                 splits = len(next_down_ids)
-                if splits == 0: # terminal segment
+                if splits == 0:  # terminal segment
                     nextdownIDcol = -1
                 elif splits == 1:
                     nextdownIDcol = next_down_ids[0]
-                else: # either muliple inflows or
+                else:  # either muliple inflows or
                     i = 0
                     for nid in next_down_ids:
                         # Set the split code in the NextDownID field.
-                        if split_code < 0: # set terminal NextDownID field to -1; this should never happen..
+                        if split_code < 0:  # set terminal NextDownID field to -1; this should never happen.
                             if i == 0:
                                 nextdownIDcol = split_code
-                        else: # muliple inflows
+                        else:  # muliple inflows
                             if i == 0:
                                 nextdownIDcol = nid
                         i += 1
                 try:
-                    if next_down_ids:del next_down_ids
+                    if next_down_ids:
+                        del next_down_ids
                 except:
                     pass
-                streams.loc[streams[hydro_id]== hydroIDcol,[next_down_id]] = nextdownIDcol
+
+                streams.loc[streams[hydro_id] == hydroIDcol, [next_down_id]] = nextdownIDcol
 
             tReturns = (sOK, streams)
+
         except Exception:
             sOK = "{}".format(trace())
-            tReturns = (sOK, )
+            tReturns = (sOK,)
         return tReturns
 
 
 if __name__ == '__main__':
     try:
         ap = argparse.ArgumentParser()
-        ap.add_argument("-p", "--parameters", nargs='+', default=[], required=True,
-                    help="list of parameters")
+        ap.add_argument("-p", "--parameters", nargs='+', default=[], required=True, help="list of parameters")
         args = ap.parse_args()
-        streams    = args.parameters[0]
-        wbd8           = args.parameters[1]
-        hydro_id        = args.parameters[2]
+        streams = args.parameters[0]
+        wbd8 = args.parameters[1]
+        hydro_id = args.parameters[2]
 
         oProcessor = build_stream_traversal_columns()
         params = (streams, wbd8, hydro_id)
-        tResults=None
+        tResults = None
         tResults = oProcessor.execute(params)
 
         del oProcessor
-    except:
-        print (str(trace()))
+    except Exception as e:
+        print(repr(e))
diff --git a/src/burn_in_levees.py b/src/burn_in_levees.py
index 28b83585b..b7e08c916 100755
--- a/src/burn_in_levees.py
+++ b/src/burn_in_levees.py
@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
-import rasterio
-import numpy as np
 import argparse
+
+import numpy as np
+import rasterio
+
 from utils.shared_functions import mem_profile
 
 
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index 7d659928c..83e78100c 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -1,93 +1,94 @@
 #!/usr/bin/env python3
 
 
-import os
 import argparse
-import string
+import os
 import pathlib
+import string
 from glob import glob
 from logging import exception
 
+
 def __read_included_files(parent_dir_path):
+    filename_patterns = glob(os.path.join(parent_dir_path, 'included_huc*.lst'))
 
-    filename_patterns = glob(os.path.join(parent_dir_path,'included_huc*.lst'))
-    
     accepted_hucs_set = set()
     for filename in filename_patterns:
-
-        with open(filename,'r') as huc_list_file:
+        with open(filename, 'r') as huc_list_file:
             file_lines = huc_list_file.readlines()
             f_list = [fl.rstrip() for fl in file_lines]
             accepted_hucs_set.update(f_list)
 
-    return(accepted_hucs_set)
+    return accepted_hucs_set
 
 
 def __read_input_hucs(hucs):
-
     huc_list = set()
     if os.path.isfile(hucs[0]):
-        
-        source_file_extension = pathlib.Path(hucs[0]).suffix   
-       
-        if (source_file_extension.lower() != ".lst" ):
+        source_file_extension = pathlib.Path(hucs[0]).suffix
+
+        if source_file_extension.lower() != ".lst":
             raise Exception("Incoming file must be in .lst format if submitting a file name and path.")
-        
-        with open(hucs[0],'r') as hucs_file:
+
+        with open(hucs[0], 'r') as hucs_file:
             file_lines = hucs_file.readlines()
             f_list = [__clean_huc_value(fl) for fl in file_lines]
             huc_list.update(f_list)
     else:
-        if (len(hucs) > 0):
+        if len(hucs) > 0:
             for huc in hucs:
                 huc_list.add(__clean_huc_value(huc))
         else:
             huc_list.add(__clean_huc_value(hucs[0]))
 
-    return(huc_list)
+    return huc_list
+
 
 def __clean_huc_value(huc):
-    
     # Strips the newline character plus
     # single or double quotes (which sometimes happens)
     huc = huc.strip().replace("\"", "")
     huc = huc.replace("\'", "")
     return huc
 
-def __check_for_membership(hucs,accepted_hucs_set):
 
+def __check_for_membership(hucs, accepted_hucs_set):
     for huc in hucs:
         if (type(huc) == string) and (not huc.isnumeric()):
             msg = f"Huc value of {huc} does not appear to be a number."
             msg += "It could be an incorrect value but also could be that the huc list "
             msg += "(if you used one), is not unix encoded."
             raise KeyError(msg)
-        
+
         if huc not in accepted_hucs_set:
-            msg = f"HUC {huc} not found in available inputs. Edit HUC inputs or acquire datasets and try again."
+            msg = f"HUC {huc} not found in available inputs. Edit HUC inputs or acquire datasets & try again."
             raise KeyError(msg)
 
 
 def check_hucs(hucs, inputsDir):
-    
     huc_list_path = os.path.join(inputsDir, 'huc_lists')
     accepted_hucs = __read_included_files(huc_list_path)
     list_hucs = __read_input_hucs(hucs)
     __check_for_membership(list_hucs, accepted_hucs)
-    
+
     # we need to return the number of hucs being used.
     # it is not easy to return a value to bash, except with standard out.
     # so we will just to a print line back (Note: This means there can be no other
-    # print commands in this file, even for debugging, as bash will pick up the 
+    # print commands in this file, even for debugging, as bash will pick up the
     # very first "print"
     print(len(list_hucs))
 
 
 if __name__ == '__main__':
-
     # parse arguments
     parser = argparse.ArgumentParser(description='Checks input hucs for availability within inputs')
-    parser.add_argument('-u','--hucs',help='Line-delimited file or list of HUCs to check availibility for',required=True, nargs='+')
+    parser.add_argument(
+        '-u',
+        '--hucs',
+        help='Line-delimited file or list of HUCs to check availibility for',
+        required=True,
+        nargs='+',
+    )
     parser.add_argument('-i', '--inputsDir', help='Inputs directory', required=True)
 
     # extract to dictionary
diff --git a/src/check_unit_errors.py b/src/check_unit_errors.py
index 5eb467320..1adacc1ea 100644
--- a/src/check_unit_errors.py
+++ b/src/check_unit_errors.py
@@ -5,9 +5,9 @@
 import os
 import sys
 
-from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
-                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
 from utils.fim_enums import FIM_exit_codes
+from utils.shared_variables import UNIT_ERRORS_MIN_NUMBER_THRESHOLD, UNIT_ERRORS_MIN_PERCENT_THRESHOLD
+
 
 """
     Calculates the number of units/hucs that errored during processing.
@@ -15,64 +15,66 @@
     this could respond with an abort processing code. It will also
     only throw that code if a minimum number of error exist.
     There should always be at least one (non_zero_exit_codes.log)
-    
-    Note: The percentage number as a whole number and the min number of 
+
+    Note: The percentage number as a whole number and the min number of
     errors are stored in the utils/shared_variables.py (kinda like constants)
-    
+
     Parameters
     ----------
     fim_dir : str
         Directory containing FIM output folders. (ie output_run_data_dir)
     number_of_input_hucs : int
         Number of hucs originally submitted for processing.
-    
+
     Returns
     ----------
     return_code of 0 (success) or 62 (from fim_enums)
 """
+
+
 def check_unit_errors(fim_dir, number_of_input_hucs):
+    return_code = 0  # default success return code.
 
-    return_code = 0 # default success return code.
-    
-    if (not os.path.isdir(fim_dir)):
+    if not os.path.isdir(fim_dir):
         raise Exception(f"The fim output directory of {fim_dir} does not exist")
 
     unit_errors_dir = os.path.join(fim_dir, "unit_errors")
 
-    if (not os.path.isdir(unit_errors_dir)):
-        raise Exception("The unit errors directory inside the fim output"\
-                          f" directory of {fim_dir} does not exist")
+    if not os.path.isdir(unit_errors_dir):
+        raise Exception(
+            "The unit errors directory inside the fim output" f" directory of {fim_dir} does not exist"
+        )
 
-    error_file_count = 0    
+    error_file_count = 0
     for path in os.listdir(unit_errors_dir):
-        if ( os.path.isfile(os.path.join(unit_errors_dir, path)) and
-            ("non_zero_exit_codes.log" not in path)):
+        if os.path.isfile(os.path.join(unit_errors_dir, path)) and ("non_zero_exit_codes.log" not in path):
             error_file_count += 1
 
     # We will only error out if it is more than the min number of error files.
     # This is done because sometimes during dev, you are expecting a bunch of errors
     # and sometimes, the number of errors is too small to worry about.
-   
-    if (error_file_count > UNIT_ERRORS_MIN_NUMBER_THRESHOLD):
-
-        percentage_of_errors =  error_file_count / number_of_input_hucs * 100
-        
-        if (percentage_of_errors >= UNIT_ERRORS_MIN_PERCENT_THRESHOLD):
-            
-            errMsg = "Too many unit errors exist to continue,"\
-                f" code:{FIM_exit_codes.EXCESS_UNIT_ERRORS.value}"
+
+    if error_file_count > UNIT_ERRORS_MIN_NUMBER_THRESHOLD:
+        percentage_of_errors = error_file_count / number_of_input_hucs * 100
+
+        if percentage_of_errors >= UNIT_ERRORS_MIN_PERCENT_THRESHOLD:
+            errMsg = (
+                "Too many unit errors exist to continue," f" code:{FIM_exit_codes.EXCESS_UNIT_ERRORS.value}"
+            )
             raise Exception(errMsg)
-  
+
     return return_code
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     # parse arguments
     parser = argparse.ArgumentParser(description='Check number of unit errors to determine if continue')
-    parser.add_argument('-f', '--fim_dir',help='root output folder for the process (output + name)',
-                            required=True)
-    parser.add_argument('-n', '--number_of_input_hucs', help = 'Original number of hucs to process',
-                            type=int, required=True)
+    parser.add_argument(
+        '-f', '--fim_dir', help='root output folder for the process (output + name)', required=True
+    )
+    parser.add_argument(
+        '-n', '--number_of_input_hucs', help='Original number of hucs to process', type=int, required=True
+    )
 
     # extract to dictionary
     args = vars(parser.parse_args())
diff --git a/src/clip_rasters_to_branches.py b/src/clip_rasters_to_branches.py
index 7057c830f..4399aeae0 100755
--- a/src/clip_rasters_to_branches.py
+++ b/src/clip_rasters_to_branches.py
@@ -1,37 +1,61 @@
 #!/usr/bin/env python3
 
 
-from stream_branches import StreamNetwork
-from stream_branches import StreamBranchPolygons
 import argparse
+
 from tqdm import tqdm
 
+from stream_branches import StreamBranchPolygons, StreamNetwork
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     # parse arguments
     parser = argparse.ArgumentParser(description='Clips rasters to branch polygons')
-    parser.add_argument('-b','--branches', help='Branch polygons file name', required=True,default=None)
-    parser.add_argument('-d','--branch-id', help='Branch ID attribute', required=True,default=None)
-    parser.add_argument('-i','--branch-id-attribute', help='Branch ID attribute', required=True,default=None)
-    parser.add_argument('-r','--rasters', help='Raster file name to clip', required=True,default=None,nargs="+")
-    parser.add_argument('-c','--clipped-rasters', help='Branch polygons out file name', required=False,default=None,nargs="+")
-    parser.add_argument('-v','--verbose', help='Verbose printing', required=False,default=None,action='store_true')
+    parser.add_argument('-b', '--branches', help='Branch polygons file name', required=True, default=None)
+    parser.add_argument('-d', '--branch-id', help='Branch ID attribute', required=True, default=None)
+    parser.add_argument(
+        '-i', '--branch-id-attribute', help='Branch ID attribute', required=True, default=None
+    )
+    parser.add_argument(
+        '-r', '--rasters', help='Raster file name to clip', required=True, default=None, nargs="+"
+    )
+    parser.add_argument(
+        '-c',
+        '--clipped-rasters',
+        help='Branch polygons out file name',
+        required=False,
+        default=None,
+        nargs="+",
+    )
+    parser.add_argument(
+        '-v', '--verbose', help='Verbose printing', required=False, default=None, action='store_true'
+    )
 
     # extract to dictionary
     args = vars(parser.parse_args())
 
-    stream_polygons_file, branch_id, branch_id_attribute, rasters, clipped_rasters, verbose = args["branches"], args["branch_id"],args["branch_id_attribute"],args["rasters"] , args["clipped_rasters"], args["verbose"]
-    
+    stream_polygons_file, branch_id, branch_id_attribute, rasters, clipped_rasters, verbose = (
+        args["branches"],
+        args["branch_id"],
+        args["branch_id_attribute"],
+        args["rasters"],
+        args["clipped_rasters"],
+        args["verbose"],
+    )
+
     # load file
-    stream_polys = StreamBranchPolygons.from_file( filename=stream_polygons_file, 
-                                                   branch_id_attribute=branch_id_attribute,
-                                                   values_excluded=None,attribute_excluded=None, verbose = verbose)
-    
-    for raster, clipped_raster in tqdm(zip(rasters,clipped_rasters),disable=(not verbose),total=len(rasters)):
+    stream_polys = StreamBranchPolygons.from_file(
+        filename=stream_polygons_file,
+        branch_id_attribute=branch_id_attribute,
+        values_excluded=None,
+        attribute_excluded=None,
+        verbose=verbose,
+    )
+
+    for raster, clipped_raster in tqdm(
+        zip(rasters, clipped_rasters), disable=(not verbose), total=len(rasters)
+    ):
         if verbose:
             print("Clipping \'{}\' to branch polygons ...".format(raster.split('/')[-1].split('.')[0]))
-   
-        stream_polys.clip(raster,clipped_raster,branch_id,branch_id_attribute)
-    
 
+        stream_polys.clip(raster, clipped_raster, branch_id, branch_id_attribute)
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 3901a3eb2..081728506 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -1,39 +1,43 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
-import pandas as pd
+
 import geopandas as gpd
-import argparse
+import pandas as pd
 import rasterio as rio
-from shapely.geometry import MultiPolygon,Polygon
-from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
+from shapely.geometry import MultiPolygon, Polygon
+
 from utils.shared_functions import getDriver, mem_profile
+from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
+
 
 @mem_profile
-def subset_vector_layers(subset_nwm_lakes, 
-                         subset_nwm_streams,
-                         hucCode,
-                         subset_nwm_headwaters,
-                         wbd_buffer_filename,
-                         wbd_streams_buffer_filename,
-                         wbd_filename,
-                         dem_filename,
-                         dem_domain,
-                         nwm_lakes,
-                         nwm_catchments,
-                         subset_nwm_catchments,
-                         nld_lines,
-                         nld_lines_preprocessed,
-                         landsea,
-                         nwm_streams,
-                         subset_landsea,
-                         nwm_headwaters,
-                         subset_nld_lines,
-                         subset_nld_lines_preprocessed,
-                         wbd_buffer_distance,
-                         levee_protected_areas,
-                         subset_levee_protected_areas):
-        
+def subset_vector_layers(
+    subset_nwm_lakes,
+    subset_nwm_streams,
+    hucCode,
+    subset_nwm_headwaters,
+    wbd_buffer_filename,
+    wbd_streams_buffer_filename,
+    wbd_filename,
+    dem_filename,
+    dem_domain,
+    nwm_lakes,
+    nwm_catchments,
+    subset_nwm_catchments,
+    nld_lines,
+    nld_lines_preprocessed,
+    landsea,
+    nwm_streams,
+    subset_landsea,
+    nwm_headwaters,
+    subset_nld_lines,
+    subset_nld_lines_preprocessed,
+    wbd_buffer_distance,
+    levee_protected_areas,
+    subset_levee_protected_areas,
+):
     print("Getting Cell Size", flush=True)
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
@@ -49,54 +53,81 @@ def subset_vector_layers(subset_nwm_lakes,
 
     # Make the streams buffer smaller than the wbd_buffer so streams don't reach the edge of the DEM
     wbd_streams_buffer = wbd_buffer.copy()
-    wbd_streams_buffer.geometry = wbd_streams_buffer.geometry.buffer(-3*dem_cellsize, resolution=32)
+    wbd_streams_buffer.geometry = wbd_streams_buffer.geometry.buffer(-3 * dem_cellsize, resolution=32)
 
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
-    wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
-    wbd_streams_buffer.to_file(wbd_streams_buffer_filename, driver=getDriver(wbd_streams_buffer_filename), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+    wbd_buffer.to_file(
+        wbd_buffer_filename,
+        driver=getDriver(wbd_buffer_filename),
+        index=False,
+        crs=DEFAULT_FIM_PROJECTION_CRS,
+    )
+    wbd_streams_buffer.to_file(
+        wbd_streams_buffer_filename,
+        driver=getDriver(wbd_streams_buffer_filename),
+        index=False,
+        crs=DEFAULT_FIM_PROJECTION_CRS,
+    )
 
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea, mask=wbd_buffer)
     if not landsea.empty:
         print("Create landsea gpkg", flush=True)
-        landsea.to_file(subset_landsea, driver = getDriver(subset_landsea), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        landsea.to_file(
+            subset_landsea, driver=getDriver(subset_landsea), index=False, crs=DEFAULT_FIM_PROJECTION_CRS
+        )
     del landsea
 
     # Clip levee-protected areas polygons for future masking ocean areas (where applicable)
     print("Subsetting Levee Protected Areas", flush=True)
     levee_protected_areas = gpd.read_file(levee_protected_areas, mask=wbd_buffer)
     if not levee_protected_areas.empty:
-        levee_protected_areas.to_file(subset_levee_protected_areas, driver = getDriver
-                                      (subset_levee_protected_areas), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        levee_protected_areas.to_file(
+            subset_levee_protected_areas,
+            driver=getDriver(subset_levee_protected_areas),
+            index=False,
+            crs=DEFAULT_FIM_PROJECTION_CRS,
+        )
     del levee_protected_areas
 
     # Find intersecting lakes and writeout
     print("Subsetting NWM Lakes", flush=True)
-    nwm_lakes = gpd.read_file(nwm_lakes, mask = wbd_buffer)
+    nwm_lakes = gpd.read_file(nwm_lakes, mask=wbd_buffer)
     nwm_lakes = nwm_lakes.loc[nwm_lakes.Shape_Area < 18990454000.0]
 
     if not nwm_lakes.empty:
         # Perform fill process to remove holes/islands in the NWM lake polygons
         nwm_lakes = nwm_lakes.explode(index_parts=True)
-        nwm_lakes_fill_holes = MultiPolygon(Polygon(p.exterior) for p in nwm_lakes['geometry']) # remove donut hole geometries
+        nwm_lakes_fill_holes = MultiPolygon(
+            Polygon(p.exterior) for p in nwm_lakes['geometry']
+        )  # remove donut hole geometries
         # Loop through the filled polygons and insert the new geometry
         for i in range(len(nwm_lakes_fill_holes.geoms)):
             nwm_lakes.loc[i, 'geometry'] = nwm_lakes_fill_holes.geoms[i]
-        nwm_lakes.to_file(subset_nwm_lakes, driver = getDriver(subset_nwm_lakes), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        nwm_lakes.to_file(
+            subset_nwm_lakes, driver=getDriver(subset_nwm_lakes), index=False, crs=DEFAULT_FIM_PROJECTION_CRS
+        )
     del nwm_lakes
 
     # Find intersecting levee lines
     print("Subsetting NLD levee lines", flush=True)
-    nld_lines = gpd.read_file(nld_lines, mask = wbd_buffer)
+    nld_lines = gpd.read_file(nld_lines, mask=wbd_buffer)
     if not nld_lines.empty:
-        nld_lines.to_file(subset_nld_lines, driver = getDriver(subset_nld_lines), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        nld_lines.to_file(
+            subset_nld_lines, driver=getDriver(subset_nld_lines), index=False, crs=DEFAULT_FIM_PROJECTION_CRS
+        )
     del nld_lines
 
     # Preprocced levee lines for burning
-    nld_lines_preprocessed = gpd.read_file(nld_lines_preprocessed, mask = wbd_buffer)
+    nld_lines_preprocessed = gpd.read_file(nld_lines_preprocessed, mask=wbd_buffer)
     if not nld_lines_preprocessed.empty:
-        nld_lines_preprocessed.to_file(subset_nld_lines_preprocessed, driver = getDriver(subset_nld_lines_preprocessed), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        nld_lines_preprocessed.to_file(
+            subset_nld_lines_preprocessed,
+            driver=getDriver(subset_nld_lines_preprocessed),
+            index=False,
+            crs=DEFAULT_FIM_PROJECTION_CRS,
+        )
     del nld_lines_preprocessed
 
     # Subset NWM headwaters
@@ -104,9 +135,14 @@ def subset_vector_layers(subset_nwm_lakes,
     nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer)
 
     if len(nwm_headwaters) > 0:
-        nwm_headwaters.to_file(subset_nwm_headwaters, driver=getDriver(subset_nwm_headwaters), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        nwm_headwaters.to_file(
+            subset_nwm_headwaters,
+            driver=getDriver(subset_nwm_headwaters),
+            index=False,
+            crs=DEFAULT_FIM_PROJECTION_CRS,
+        )
     else:
-        print ("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
+        print("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_headwaters
 
@@ -115,16 +151,21 @@ def subset_vector_layers(subset_nwm_lakes,
     nwm_catchments = gpd.read_file(nwm_catchments, mask=wbd_buffer)
 
     if len(nwm_catchments) > 0:
-        nwm_catchments.to_file(subset_nwm_catchments, driver=getDriver(subset_nwm_catchments), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        nwm_catchments.to_file(
+            subset_nwm_catchments,
+            driver=getDriver(subset_nwm_catchments),
+            index=False,
+            crs=DEFAULT_FIM_PROJECTION_CRS,
+        )
     else:
-        print ("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
+        print("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_catchments
 
     # Subset nwm streams
     print("Subsetting NWM Streams", flush=True)
 
-    nwm_streams = gpd.read_file(nwm_streams, mask = wbd_buffer)
+    nwm_streams = gpd.read_file(nwm_streams, mask=wbd_buffer)
 
     # NWM can have duplicate records, but appear to always be identical duplicates
     nwm_streams.drop_duplicates(subset="ID", keep="first", inplace=True)
@@ -137,61 +178,63 @@ def subset_vector_layers(subset_nwm_lakes,
 
         nwm_streams = pd.concat([nwm_streams_nonoutlets, nwm_streams_outlets])
 
-        nwm_streams.to_file(subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False, crs=DEFAULT_FIM_PROJECTION_CRS)
+        nwm_streams.to_file(
+            subset_nwm_streams,
+            driver=getDriver(subset_nwm_streams),
+            index=False,
+            crs=DEFAULT_FIM_PROJECTION_CRS,
+        )
     else:
-        print ("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
+        print("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_streams
 
 
 if __name__ == '__main__':
-
-    #print(sys.argv)
+    # print(sys.argv)
 
     parser = argparse.ArgumentParser(description='Subset vector layers')
-    parser.add_argument('-a','--subset-nwm-lakes', help='NWM lake subset', 
-                        required=True)
-    parser.add_argument('-b','--subset-nwm-streams', help='NWM streams subset', 
-                        required=True)
-    parser.add_argument('-d','--hucCode', help='HUC boundary ID', required=True,
-                        type=str)
-    parser.add_argument('-e','--subset-nwm-headwaters', help='NWM headwaters subset', 
-                        required=True, default=None)
-    parser.add_argument('-f','--wbd_buffer_filename', help='Buffered HUC boundary', 
-                        required=True)
-    parser.add_argument('-s','--wbd_streams_buffer_filename', help='Buffered HUC boundary (streams)', 
-                        required=True)
-    parser.add_argument('-g','--wbd-filename', help='HUC boundary', required=True)
-    parser.add_argument('-i','--dem-filename', help='DEM filename', required=True)
-    parser.add_argument('-j','--dem-domain', help='DEM domain polygon', required=True)
-    parser.add_argument('-l','--nwm-lakes', help='NWM Lakes', required=True)    
-    parser.add_argument('-m','--nwm-catchments', help='NWM catchments',
-                        required=True)	 
-    parser.add_argument('-n','--subset-nwm-catchments', help='NWM catchments subset', 
-                        required=True)
-    parser.add_argument('-r','--nld-lines', help='Levee vectors to use within project path',
-                        required=True)
-    parser.add_argument('-rp','--nld-lines-preprocessed', help='Levee vectors to use for DEM burning',
-                        required=True)
-    parser.add_argument('-v','--landsea', help='LandSea - land boundary',
-                        required=True)	 
-    parser.add_argument('-w','--nwm-streams', help='NWM flowlines',
-                        required=True)
-    parser.add_argument('-x','--subset-landsea', help='LandSea subset', 
-                        required=True)
-    parser.add_argument('-y','--nwm-headwaters', help='NWM headwaters',
-                        required=True)	 
-    parser.add_argument('-z','--subset-nld-lines', help='Subset of NLD levee vectors for HUC',
-                        required=True)
-    parser.add_argument('-zp','--subset-nld-lines-preprocessed', help='Subset of NLD levee vectors for burning elevations into DEMs',
-                        required=True)
-    parser.add_argument('-wb','--wbd-buffer-distance', help='WBD Mask buffer distance', 
-                        required=True, type=int)
-    parser.add_argument('-lpf','--levee-protected-areas', 
-                        help='Levee-protected areas filename', required=True)    
-    parser.add_argument('-lps','--subset-levee-protected-areas', 
-                        help='Levee-protected areas subset', required=True)
-    
+    parser.add_argument('-a', '--subset-nwm-lakes', help='NWM lake subset', required=True)
+    parser.add_argument('-b', '--subset-nwm-streams', help='NWM streams subset', required=True)
+    parser.add_argument('-d', '--hucCode', help='HUC boundary ID', required=True, type=str)
+    parser.add_argument(
+        '-e', '--subset-nwm-headwaters', help='NWM headwaters subset', required=True, default=None
+    )
+    parser.add_argument('-f', '--wbd_buffer_filename', help='Buffered HUC boundary', required=True)
+    parser.add_argument(
+        '-s', '--wbd_streams_buffer_filename', help='Buffered HUC boundary (streams)', required=True
+    )
+    parser.add_argument('-g', '--wbd-filename', help='HUC boundary', required=True)
+    parser.add_argument('-i', '--dem-filename', help='DEM filename', required=True)
+    parser.add_argument('-j', '--dem-domain', help='DEM domain polygon', required=True)
+    parser.add_argument('-l', '--nwm-lakes', help='NWM Lakes', required=True)
+    parser.add_argument('-m', '--nwm-catchments', help='NWM catchments', required=True)
+    parser.add_argument('-n', '--subset-nwm-catchments', help='NWM catchments subset', required=True)
+    parser.add_argument('-r', '--nld-lines', help='Levee vectors to use within project path', required=True)
+    parser.add_argument(
+        '-rp', '--nld-lines-preprocessed', help='Levee vectors to use for DEM burning', required=True
+    )
+    parser.add_argument('-v', '--landsea', help='LandSea - land boundary', required=True)
+    parser.add_argument('-w', '--nwm-streams', help='NWM flowlines', required=True)
+    parser.add_argument('-x', '--subset-landsea', help='LandSea subset', required=True)
+    parser.add_argument('-y', '--nwm-headwaters', help='NWM headwaters', required=True)
+    parser.add_argument('-z', '--subset-nld-lines', help='Subset of NLD levee vectors for HUC', required=True)
+    parser.add_argument(
+        '-zp',
+        '--subset-nld-lines-preprocessed',
+        help='Subset of NLD levee vectors for burning elevations into DEMs',
+        required=True,
+    )
+    parser.add_argument(
+        '-wb', '--wbd-buffer-distance', help='WBD Mask buffer distance', required=True, type=int
+    )
+    parser.add_argument(
+        '-lpf', '--levee-protected-areas', help='Levee-protected areas filename', required=True
+    )
+    parser.add_argument(
+        '-lps', '--subset-levee-protected-areas', help='Levee-protected areas subset', required=True
+    )
+
     args = vars(parser.parse_args())
 
     subset_vector_layers(**args)
diff --git a/src/crosswalk_nwm_demDerived.py b/src/crosswalk_nwm_demDerived.py
index 55ef7e36e..be267c89b 100755
--- a/src/crosswalk_nwm_demDerived.py
+++ b/src/crosswalk_nwm_demDerived.py
@@ -1,77 +1,86 @@
 #!/usr/bin/env python3
 
+import argparse
+
 import geopandas as gpd
-import pandas as pd
 import numpy as np
-import argparse
-import stream_branches as sb
+import pandas as pd
+from shapely.geometry import MultiLineString
 
+import stream_branches as sb
 from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
-from shapely.geometry import MultiLineString
 
-def Crosswalk_nwm_demDerived(nwm_streams, demDerived, wbd=None, node_prefix=None, sampling_size=None, 
-                             crosswalk_outfile=None, 
-                             demDerived_outfile=None, nwm_outfile=None, verbose=False):
-    
+
+def Crosswalk_nwm_demDerived(
+    nwm_streams,
+    demDerived,
+    wbd=None,
+    node_prefix=None,
+    sampling_size=None,
+    crosswalk_outfile=None,
+    demDerived_outfile=None,
+    nwm_outfile=None,
+    verbose=False,
+):
     # load nwm streams
-    if isinstance(nwm_streams,sb.StreamNetwork):
+    if isinstance(nwm_streams, sb.StreamNetwork):
         pass
-    elif isinstance(nwm_streams,str):
+    elif isinstance(nwm_streams, str):
         nwm_streams = sb.StreamNetwork.from_file(nwm_streams)
     else:
         raise TypeError("For nwm_streams pass file path string or GeoDataFrame object")
 
     # load demDerived
-    if isinstance(demDerived,sb.StreamNetwork):
+    if isinstance(demDerived, sb.StreamNetwork):
         pass
-    elif isinstance(demDerived,str):
+    elif isinstance(demDerived, str):
         demDerived = sb.StreamNetwork.from_file(demDerived)
     else:
         raise TypeError("demDerived pass file path string or GeoDataFrame object")
 
     # clip nwm_streams
     if wbd is not None:
-        nwm_streams = nwm_streams.clip(wbd,keep_geom_type=True,verbose=verbose)
+        nwm_streams = nwm_streams.clip(wbd, keep_geom_type=True, verbose=verbose)
 
-    # build traversal to nwm 
-    nwm_streams = Add_traversal_to_NWM(nwm_streams,node_prefix=node_prefix,
-                                       outfile=nwm_outfile,
-                                       verbose=verbose)
+    # build traversal to nwm
+    nwm_streams = Add_traversal_to_NWM(
+        nwm_streams, node_prefix=node_prefix, outfile=nwm_outfile, verbose=verbose
+    )
 
     # create points for nwm and demDerived networks
-    nwm_points = nwm_streams.explode_to_points(sampling_size=sampling_size, 
-                                               verbose=verbose)
-    demDerived_points = demDerived.explode_to_points(sampling_size=sampling_size, 
-                                                     verbose=verbose)
-    
+    nwm_points = nwm_streams.explode_to_points(sampling_size=sampling_size, verbose=verbose)
+    demDerived_points = demDerived.explode_to_points(sampling_size=sampling_size, verbose=verbose)
+
     # conflate points
-    crosswalk_table = sb.StreamNetwork.conflate_points(demDerived_points, nwm_points,
-                                                         source_reach_id_attribute='HydroID',
-                                                         target_reach_id_attribute='ID',
-                                                         verbose=verbose)
-    
+    crosswalk_table = sb.StreamNetwork.conflate_points(
+        demDerived_points,
+        nwm_points,
+        source_reach_id_attribute='HydroID',
+        target_reach_id_attribute='ID',
+        verbose=verbose,
+    )
+
     # merge crosswalk table
-    crosswalk_table.rename(columns={'ID':'feature_id'},inplace=True)
-    demDerived.drop(columns='feature_id',inplace=True,errors='raise')
+    crosswalk_table.rename(columns={'ID': 'feature_id'}, inplace=True)
+    demDerived.drop(columns='feature_id', inplace=True, errors='raise')
     demDerived['HydroID'] = demDerived['HydroID'].astype(int)
     demDerived = demDerived.merge(crosswalk_table, how='left', left_on='HydroID', right_index=True)
 
     if demDerived_outfile is not None:
-        demDerived.write(demDerived_outfile,index=False,verbose=verbose)
+        demDerived.write(demDerived_outfile, index=False, verbose=verbose)
 
     if crosswalk_outfile is not None:
-        crosswalk_table.to_csv(crosswalk_outfile,index=True)
+        crosswalk_table.to_csv(crosswalk_outfile, index=True)
 
-    #print(demDerived, crosswalk_table)
-    return(demDerived, crosswalk_table)
+    # print(demDerived, crosswalk_table)
+    return (demDerived, crosswalk_table)
 
 
-def Add_traversal_to_NWM(nwm_streams,node_prefix=None,outfile=None,verbose=False):
-    
-    if isinstance(nwm_streams,sb.StreamNetwork):
+def Add_traversal_to_NWM(nwm_streams, node_prefix=None, outfile=None, verbose=False):
+    if isinstance(nwm_streams, sb.StreamNetwork):
         pass
-    elif isinstance(nwm_streams,str):
+    elif isinstance(nwm_streams, str):
         nwm_streams = sb.StreamNetwork.from_file(nwm_streams)
     else:
         raise TypeError("nwm_streams_file pass file path string or GeoDataFrame object")
@@ -79,71 +88,93 @@ def Add_traversal_to_NWM(nwm_streams,node_prefix=None,outfile=None,verbose=False
     # remove multilinestrings if any
     anyMultiLineStrings = np.any(np.array([isinstance(g, MultiLineString) for g in nwm_streams.geometry]))
     if anyMultiLineStrings:
-        nwm_streams = nwm_streams.dissolve_by_branch(branch_id_attribute='ID', attribute_excluded=None,
-                                                     values_excluded=None, verbose=verbose)
-
+        nwm_streams = nwm_streams.dissolve_by_branch(
+            branch_id_attribute='ID', attribute_excluded=None, values_excluded=None, verbose=verbose
+        )
 
     # create stream node ids
-    nwm_streams = nwm_streams.derive_nodes(toNode_attribute='To_Node',fromNode_attribute='From_Node',
-                                           reach_id_attribute='ID',
-                                           outlet_linestring_index=-1,node_prefix=node_prefix,
-                                           max_node_digits=8,verbose=verbose)
+    nwm_streams = nwm_streams.derive_nodes(
+        toNode_attribute='To_Node',
+        fromNode_attribute='From_Node',
+        reach_id_attribute='ID',
+        outlet_linestring_index=-1,
+        node_prefix=node_prefix,
+        max_node_digits=8,
+        verbose=verbose,
+    )
     # inlets and outlets
-    nwm_streams = nwm_streams.derive_outlets(toNode_attribute='To_Node',fromNode_attribute='From_Node',
-                                             outlets_attribute='outlet_id',verbose=verbose)
-    nwm_streams = nwm_streams.derive_inlets(toNode_attribute='To_Node',fromNode_attribute='From_Node',
-                                            inlets_attribute='inlet_id',verbose=verbose)
+    nwm_streams = nwm_streams.derive_outlets(
+        toNode_attribute='To_Node',
+        fromNode_attribute='From_Node',
+        outlets_attribute='outlet_id',
+        verbose=verbose,
+    )
+    nwm_streams = nwm_streams.derive_inlets(
+        toNode_attribute='To_Node',
+        fromNode_attribute='From_Node',
+        inlets_attribute='inlet_id',
+        verbose=verbose,
+    )
 
     # upstream and downstream dictionaries
-    upstreams,downstreams = nwm_streams.make_up_and_downstream_dictionaries(reach_id_attribute='ID',
-                                                                            toNode_attribute='To_Node',
-                                                                            fromNode_attribute='From_Node',
-                                                                            verbose=verbose)
+    upstreams, downstreams = nwm_streams.make_up_and_downstream_dictionaries(
+        reach_id_attribute='ID', toNode_attribute='To_Node', fromNode_attribute='From_Node', verbose=verbose
+    )
 
     # derive arbolate sum
-    nwm_streams = nwm_streams.get_arbolate_sum(arbolate_sum_attribute='arbolate_sum',inlets_attribute='inlet_id',
-                         reach_id_attribute='ID',length_conversion_factor_to_km = 0.001,
-                         upstreams=upstreams, downstreams=downstreams,
-                         toNode_attribute='To_Node',
-                         fromNode_attribute='From_Node',
-                         verbose=verbose
-                        )
+    nwm_streams = nwm_streams.get_arbolate_sum(
+        arbolate_sum_attribute='arbolate_sum',
+        inlets_attribute='inlet_id',
+        reach_id_attribute='ID',
+        length_conversion_factor_to_km=0.001,
+        upstreams=upstreams,
+        downstreams=downstreams,
+        toNode_attribute='To_Node',
+        fromNode_attribute='From_Node',
+        verbose=verbose,
+    )
 
     # derive levelpaths
-    nwm_streams = nwm_streams.derive_stream_branches(toNode_attribute='To_Node',
-                               fromNode_attribute='From_Node',
-                               upstreams=upstreams,
-                               outlet_attribute='outlet_id',
-                               branch_id_attribute='levpa_id',
-                               reach_id_attribute='ID',
-                               comparison_attributes='order_',
-                               comparison_function=max,
-                               max_branch_id_digits=6,
-                               verbose=verbose)
-
-    #nwm_streams = nwm_streams.dissolve_by_branch(branch_id_attribute='levpa_id', attribute_excluded=None,
-    #                                             values_excluded=None, verbose=verbose)
-    
-    nwm_streams.reset_index(drop=True,inplace=True)
+    nwm_streams = nwm_streams.derive_stream_branches(
+        toNode_attribute='To_Node',
+        fromNode_attribute='From_Node',
+        upstreams=upstreams,
+        outlet_attribute='outlet_id',
+        branch_id_attribute='levpa_id',
+        reach_id_attribute='ID',
+        comparison_attributes='order_',
+        comparison_function=max,
+        max_branch_id_digits=6,
+        verbose=verbose,
+    )
+
+    # nwm_streams = nwm_streams.dissolve_by_branch(
+    #     branch_id_attribute='levpa_id', attribute_excluded=None, values_excluded=None, verbose=verbose
+    # )
+
+    nwm_streams.reset_index(drop=True, inplace=True)
 
     if outfile is not None:
-        nwm_streams.write(outfile,index=False,verbose=verbose)
+        nwm_streams.write(outfile, index=False, verbose=verbose)
 
-    return(nwm_streams)
+    return nwm_streams
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='Crosswalking')
-    parser.add_argument('-n','--nwm-streams', help='NWM Streams', required=True)
-    parser.add_argument('-d','--demDerived', help='demDerived Streams', required=True)
-    parser.add_argument('-w','--wbd', help='WBD File', required=False,default=None)
-    parser.add_argument('-p','--node-prefix', help='Node Prefix', required=False,default=None)
-    parser.add_argument('-a','--sampling-size', help='Sample size for Points', required=False,default=None,type=int)
-    parser.add_argument('-c','--crosswalk-outfile', help='Crosswalk Out File', required=False,default=None)
-    parser.add_argument('-e','--demDerived-outfile', help='demDerived Out File', required=False,default=None)
-    parser.add_argument('-m','--nwm-outfile', help='NWM Streams Out File', required=False,default=None)
-    parser.add_argument('-v','--verbose', help='Verbose', required=False,default=False,action='store_true')
+    parser.add_argument('-n', '--nwm-streams', help='NWM Streams', required=True)
+    parser.add_argument('-d', '--demDerived', help='demDerived Streams', required=True)
+    parser.add_argument('-w', '--wbd', help='WBD File', required=False, default=None)
+    parser.add_argument('-p', '--node-prefix', help='Node Prefix', required=False, default=None)
+    parser.add_argument(
+        '-a', '--sampling-size', help='Sample size for Points', required=False, default=None, type=int
+    )
+    parser.add_argument('-c', '--crosswalk-outfile', help='Crosswalk Out File', required=False, default=None)
+    parser.add_argument(
+        '-e', '--demDerived-outfile', help='demDerived Out File', required=False, default=None
+    )
+    parser.add_argument('-m', '--nwm-outfile', help='NWM Streams Out File', required=False, default=None)
+    parser.add_argument('-v', '--verbose', help='Verbose', required=False, default=False, action='store_true')
 
     kwargs = vars(parser.parse_args())
 
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 167db7da0..6c16af193 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -20,7 +20,15 @@ if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $tempHucDataDir/LeveeProtecte
     echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/mask_dem.py -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -nld $tempHucDataDir/LeveeProtectedAreas_subset.gpkg -out $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -b $branch_id_attribute -i $current_branch_id -b0 $branch_zero_id -csv $tempHucDataDir/levee_levelpaths.csv -l $levee_id_attribute
+    python3 -m memory_profiler $srcDir/mask_dem.py \
+        -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
+        -nld $tempHucDataDir/LeveeProtectedAreas_subset.gpkg \
+        -out $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
+        -b $branch_id_attribute \
+        -i $current_branch_id \
+        -b0 $branch_zero_id \
+        -csv $tempHucDataDir/levee_levelpaths.csv \
+        -l $levee_id_attribute
     Tcount
 fi
 
@@ -28,97 +36,155 @@ fi
 echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/aread8 -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -ad8 $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -wg $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif -nc
+$taudemDir/aread8 -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
+    -ad8 $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif \
+    -wg $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif \
+    -nc
 Tcount
 
 # THRESHOLD ACCUMULATIONS ##
 echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/threshold -ssa $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -thresh 1
+$taudemDir/threshold -ssa $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif \
+    -src $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
+    -thresh 1
 Tcount
 
 ## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
 echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -o $tempCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif
+python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py \
+    -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
+    -o $tempCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif
 Tcount
 
 ## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
 echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py -e $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -a $tempCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif -d $tempCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif -t 50 -o $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -th $thalweg_lateral_elev_threshold
+python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py \
+    -e $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
+    -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
+    -a $tempCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif \
+    -d $tempCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_dist.tif \
+    -t 50 \
+    -o $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif \
+    -th $thalweg_lateral_elev_threshold
 Tcount
 
 ## MASK BURNED DEM FOR STREAMS ONLY ###
 echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -B $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif --calc="A/B" --outfile="$tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" --NoDataValue=0
+gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
+    -A $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
+    -B $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
+    --calc="A/B" \
+    --outfile="$tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" \
+    --NoDataValue=0
 Tcount
 
 ## FLOW CONDITION STREAMS ##
 echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/flowdircond -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif -z $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -zfdc $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
+$taudemDir/flowdircond -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif \
+    -z $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif \
+    -zfdc $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
 Tcount
 
 ## D8 SLOPES ##
 echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"
 date -u
 Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif -sd8 $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif 
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir \
+    -fel $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif \
+    -sd8 $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
 Tcount
 
 ## STREAMNET FOR REACHES ##
 echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/streamnet -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -fel $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -ad8 $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif -src $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -ord $tempCurrentBranchDataDir/streamOrder_$current_branch_id.tif -tree $tempCurrentBranchDataDir/treeFile_$current_branch_id.txt -coord $tempCurrentBranchDataDir/coordFile_$current_branch_id.txt -w $tempCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif -net $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
+$taudemDir/streamnet \
+    -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
+    -fel $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
+    -ad8 $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif \
+    -src $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
+    -ord $tempCurrentBranchDataDir/streamOrder_$current_branch_id.tif \
+    -tree $tempCurrentBranchDataDir/treeFile_$current_branch_id.txt \
+    -coord $tempCurrentBranchDataDir/coordFile_$current_branch_id.txt \
+    -w $tempCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif \
+    -net $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
 Tcount
 
 ## SPLIT DERIVED REACHES ##
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg -n $b_arg -m $max_split_distance_meters -t $slope_min -b $lakes_buffer_dist_meters
+$srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp \
+    -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
+    -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg \
+    -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg \
+    -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg \
+    -n $b_arg \
+    -m $max_split_distance_meters \
+    -t $slope_min \
+    -b $lakes_buffer_dist_meters
 Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
 echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"
 date -u
 Tstart
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif -gw $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif -o $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
+mpiexec -n $ncores_gw $taudemDir/gagewatershed \
+    -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
+    -gw $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
+    -o $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg \
+    -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
 Tcount
 
 ## VECTORIZE FEATURE ID CENTROIDS ##
 echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/reachID_grid_to_vector_points.py -r $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif -i featureID -p $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
+$srcDir/reachID_grid_to_vector_points.py \
+    -r $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
+    -i featureID \
+    -p $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
 Tcount
 
 ## GAGE WATERSHED FOR PIXELS ##
 echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"
 date -u
 Tstart
-mpiexec -n $ncores_gw $taudemDir/gagewatershed -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif -gw $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
+mpiexec -n $ncores_gw $taudemDir/gagewatershed \
+    -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif \
+    -gw $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif \
+    -o $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg \
+    -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
 Tcount
 
 # D8 REM ##
 echo -e $startDiv"D8 REM $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/make_rem.py -d $tempCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif -w $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif -o $tempCurrentBranchDataDir/rem_$current_branch_id.tif -t $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
+$srcDir/make_rem.py -d $tempCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif \
+    -w $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif \
+    -o $tempCurrentBranchDataDir/rem_$current_branch_id.tif \
+    -t $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
 Tcount
 
 ## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
 echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"
 date -u
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/rem_$current_branch_id.tif -B $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
+    -A $tempCurrentBranchDataDir/rem_$current_branch_id.tif \
+    -B $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
+    --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv \
+    --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
 Tcount
 
 ## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
@@ -126,7 +192,9 @@ if [ -f $tempHucDataDir/LandSea_subset.gpkg ]; then
     echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"
     date -u
     Tstart
-    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/LandSea_subset.gpkg $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
+    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" \
+        -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/LandSea_subset.gpkg \
+        $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
     Tcount
 fi
 
@@ -134,34 +202,55 @@ fi
 echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_polygonize.py -8 -f GPKG $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
+gdal_polygonize.py -8 -f GPKG $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
+    $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
 Tcount
 
 ## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
 echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py -i $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg -f $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -o $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -w $tempHucDataDir/wbd8_clp.gpkg -u $hucNumber
+python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py \
+    -i $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg \
+    -f $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg \
+    -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
+    -o $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
+    -w $tempHucDataDir/wbd8_clp.gpkg \
+    -u $hucNumber
 Tcount
 
 ## RASTERIZE NEW CATCHMENTS AGAIN ##
 echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
+gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+    -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
+    $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
+    $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
 Tcount
 
 ## MASK SLOPE TO CATCHMENTS ##
 echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"
 date -u
-gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif -B $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif --calc="A*(B>0)" --NoDataValue=$ndv --outfile=$tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
+gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
+    -A $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif \
+    -B $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif \
+    --calc="A*(B>0)" --NoDataValue=$ndv \
+    --outfile=$tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
 Tcount
 
 ## MAKE CATCHMENT AND STAGE FILES ##
 echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/make_stages_and_catchlist.py -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -s $tempCurrentBranchDataDir/stage_$current_branch_id.txt -a $tempCurrentBranchDataDir/catch_list_$current_branch_id.txt -m $stage_min_meters -i $stage_interval_meters -t $stage_max_meters
+$srcDir/make_stages_and_catchlist.py \
+    -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
+    -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
+    -s $tempCurrentBranchDataDir/stage_$current_branch_id.txt \
+    -a $tempCurrentBranchDataDir/catch_list_$current_branch_id.txt \
+    -m $stage_min_meters \
+    -i $stage_interval_meters \
+    -t $stage_max_meters
 Tcount
 
 ## MASK REM RASTER TO REMOVE OCEAN AREAS ##
@@ -169,7 +258,11 @@ if  [ -f $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
     echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"
     date -u
     Tstart
-    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" -A $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -B $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif --calc="(A*B)" --NoDataValue=$ndv --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
+        -A $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
+        -B $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif \
+        --calc="(A*B)" --NoDataValue=$ndv \
+        --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
     Tcount
 fi
 
@@ -177,7 +270,12 @@ fi
 echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif -catch $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif -catchlist $tempCurrentBranchDataDir/catch_list_$current_branch_id.txt -slp $tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif -h $tempCurrentBranchDataDir/stage_$current_branch_id.txt -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv
+$taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
+    -catch $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif \
+    -catchlist $tempCurrentBranchDataDir/catch_list_$current_branch_id.txt \
+    -slp $tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif \
+    -h $tempCurrentBranchDataDir/stage_$current_branch_id.txt \
+    -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv
 Tcount
 
 ## FINALIZE CATCHMENTS AND MODEL STREAMS ##
@@ -185,5 +283,22 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_bran
 date -u
 Tstart
 
-python3 -m memory_profiler $srcDir/add_crosswalk.py -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -s $tempCurrentBranchDataDir/src_base_$current_branch_id.csv -l $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -r $tempCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv -j $tempCurrentBranchDataDir/src_$current_branch_id.json -x $tempCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv -t $tempCurrentBranchDataDir/hydroTable_$current_branch_id.csv -w $tempHucDataDir/wbd8_clp.gpkg -b $b_arg -y $tempCurrentBranchDataDir/nwm_catchments_proj_subset.tif -m $manning_n -z $z_arg -k $tempCurrentBranchDataDir/small_segments_$current_branch_id.csv -e $min_catchment_area -g $min_stream_length
+python3 -m memory_profiler $srcDir/add_crosswalk.py \
+    -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
+    -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
+    -s $tempCurrentBranchDataDir/src_base_$current_branch_id.csv \
+    -l $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg \
+    -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg \
+    -r $tempCurrentBranchDataDir/src_full_crosswalked_$current_branch_id.csv \
+    -j $tempCurrentBranchDataDir/src_$current_branch_id.json \
+    -x $tempCurrentBranchDataDir/crosswalk_table_$current_branch_id.csv \
+    -t $tempCurrentBranchDataDir/hydroTable_$current_branch_id.csv \
+    -w $tempHucDataDir/wbd8_clp.gpkg \
+    -b $b_arg \
+    -y $tempCurrentBranchDataDir/nwm_catchments_proj_subset.tif \
+    -m $manning_n \
+    -z $z_arg \
+    -k $tempCurrentBranchDataDir/small_segments_$current_branch_id.csv \
+    -e $min_catchment_area \
+    -g $min_stream_length
 Tcount
diff --git a/src/derive_headwaters.py b/src/derive_headwaters.py
index b105d2c67..aabcedc66 100644
--- a/src/derive_headwaters.py
+++ b/src/derive_headwaters.py
@@ -1,19 +1,20 @@
 #!/usr/bin/env python3
 
+import argparse
+
 import geopandas as gpd
 from shapely.geometry import Point
+
 from utils.shared_functions import getDriver
-import argparse
 
 
 def findHeadWaterPoints(flows):
-
     flows = flows.explode(index_parts=True)
     headwater_points = []
-    starting_points = set() ; end_points = set()
-    for i,g in enumerate(flows.geometry):
-
-        g_points = [(x,y) for x,y in zip(*g.coords.xy)]
+    starting_points = set()
+    end_points = set()
+    for i, g in enumerate(flows.geometry):
+        g_points = [(x, y) for x, y in zip(*g.coords.xy)]
 
         starting_point = g_points[0]
         end_point = g_points[-1]
@@ -21,33 +22,45 @@ def findHeadWaterPoints(flows):
         starting_points.add(starting_point)
         end_points.add(end_point)
 
-        #line_points = np.append(line_points,g_points)
+        # line_points = np.append(line_points,g_points)
 
-    for i,sp in enumerate(starting_points):
-         #print(sp)
-         if sp not in end_points:
-             headwater_points += [sp]
+    for i, sp in enumerate(starting_points):
+        # print(sp)
+        if sp not in end_points:
+            headwater_points += [sp]
 
-    #print(headwater_points)
+    # print(headwater_points)
     headwater_points_geometries = [Point(*hwp) for hwp in headwater_points]
-    hw_gdf = gpd.GeoDataFrame({'geometry' : headwater_points_geometries},crs=flows.crs,geometry='geometry')
+    hw_gdf = gpd.GeoDataFrame({'geometry': headwater_points_geometries}, crs=flows.crs, geometry='geometry')
 
-    return(hw_gdf)
+    return hw_gdf
 
 
 if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Derive headwater points from flowlines. Linestrings must flow downstream')
-    parser.add_argument('-f','--input-flows',help='Input flowlines. Linestrings must flow downstream',required=True,type=str)
-    parser.add_argument('-l','--input-flows-layer',help='Input layer name',required=False,type=str,default=None)
-    parser.add_argument('-o','--output-headwaters',help='Output headwaters points',required=False,type=str,default=None)
+    parser = argparse.ArgumentParser(
+        description='Derive headwater points from flowlines. Linestrings must flow downstream'
+    )
+    parser.add_argument(
+        '-f',
+        '--input-flows',
+        help='Input flowlines. Linestrings must flow downstream',
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-l', '--input-flows-layer', help='Input layer name', required=False, type=str, default=None
+    )
+    parser.add_argument(
+        '-o', '--output-headwaters', help='Output headwaters points', required=False, type=str, default=None
+    )
 
     args = vars(parser.parse_args())
 
-    flows = gpd.read_file(args['input_flows'],layer=args['input_flows_layer'])
+    flows = gpd.read_file(args['input_flows'], layer=args['input_flows_layer'])
 
     hw_gdf = findHeadWaterPoints(flows)
 
-    #TODO check output_headwaters variable, not defined
+    output_headwaters = args['output_headwaters']
+
     if output_headwaters is not None:
-        hw_gdf.to_file(args['output_headwaters'],driver=getDriver(args['output_headwaters']))
+        hw_gdf.to_file(args['output_headwaters'], driver=getDriver(args['output_headwaters']))
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index d78a794f0..ffa841de6 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
-import geopandas as gpd
+import os
 import sys
 
+import geopandas as gpd
+
 from stream_branches import StreamNetwork
 from utils.fim_enums import FIM_exit_codes
 
@@ -37,9 +38,7 @@ def Derive_level_paths(
     if os.path.exists(in_stream_network):
         stream_network = StreamNetwork.from_file(filename=in_stream_network)
     else:
-        print(
-            "Sorry, no branches exist and processing can not continue. This could be an empty file."
-        )
+        print("Sorry, no branches exist and processing can not continue. This could be an empty file.")
         sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # if there are no reaches at this point
@@ -47,12 +46,12 @@ def Derive_level_paths(
         # This is technically not an error but we need to have it logged so the user know what
         # happened to it and we need the huc to not be included in future processing.
         # We need it to be not included in the fim_input.csv at the end of the unit processing.
-        # Throw an exception with valid text. This will show up in the non-zero exit codes and explain why an error.
+        # Throw an exception with valid text.
+        # This will show up in the non-zero exit codes and explain why an error.
         # Later, we can look at creating custom sys exit codes
-        # raise UserWarning("Sorry, no branches exist and processing can not continue. This could be an empty file.")
-        print(
-            "Sorry, no streams exist and processing can not continue. This could be an empty file."
-        )
+        # raise UserWarning("Sorry, no branches exist and processing can not continue.
+        # This could be an empty file.")
+        print("Sorry, no streams exist and processing can not continue. This could be an empty file.")
         sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
     # values_exluded of 1 and 2 mean where are dropping stream orders 1 and 2. We are leaving those
@@ -64,13 +63,14 @@ def Derive_level_paths(
     # if there are no reaches at this point (due to filtering)
     if len(stream_network) == 0:
         print(
-            "No branches exist but branch zero processing will continue (Exit 63). This could be due to stream order filtering."
+            "No branches exist but branch zero processing will continue (Exit 63)."
+            "This could be due to stream order filtering."
         )
         # sys.exit(FIM_exit_codes.NO_BRANCH_LEVELPATHS_EXIST.value)  # will send a 63 back
         return
 
-    inlets_attribute = "inlet_id"
-    outlets_attribute = "outlet_id"
+    inlets_attribute = 'inlet_id'
+    outlets_attribute = 'outlet_id'
     outlet_linestring_index = -1
 
     # converts multi-linestrings to linestrings
@@ -88,17 +88,11 @@ def Derive_level_paths(
 
     # derive outlets and inlets
     stream_network = stream_network.derive_outlets(
-        toNode_attribute,
-        fromNode_attribute,
-        outlets_attribute=outlets_attribute,
-        verbose=verbose,
+        toNode_attribute, fromNode_attribute, outlets_attribute=outlets_attribute, verbose=verbose
     )
 
     stream_network = stream_network.derive_inlets(
-        toNode_attribute,
-        fromNode_attribute,
-        inlets_attribute=inlets_attribute,
-        verbose=verbose,
+        toNode_attribute, fromNode_attribute, inlets_attribute=inlets_attribute, verbose=verbose
     )  # derive up and downstream networks
     upstreams, downstreams = stream_network.make_up_and_downstream_dictionaries(
         reach_id_attribute=reach_id_attribute,
@@ -144,19 +138,11 @@ def Derive_level_paths(
 
         # subset which columns to merge
         stream_network_to_merge = stream_network.filter(
-            items=[
-                reach_id_attribute,
-                inlets_attribute,
-                outlets_attribute,
-                branch_id_attribute,
-            ]
+            items=[reach_id_attribute, inlets_attribute, outlets_attribute, branch_id_attribute]
         )
 
         catchments = catchments.merge(
-            stream_network_to_merge,
-            how="inner",
-            left_on=reach_id_attribute,
-            right_on=reach_id_attribute,
+            stream_network_to_merge, how="inner", left_on=reach_id_attribute, right_on=reach_id_attribute
         )
 
         catchments.reset_index(drop=True, inplace=True)
@@ -186,16 +172,14 @@ def Derive_level_paths(
         # dissolve by levelpath
         stream_network = stream_network.dissolve_by_branch(
             branch_id_attribute=branch_id_attribute,
-            attribute_excluded=None,  #'order_',
+            attribute_excluded=None,  # 'order_',
             values_excluded=None,  # [1,2],
             out_vector_files=out_stream_network_dissolved,
             verbose=verbose,
         )
 
         stream_network = stream_network.remove_branches_in_waterbodies(
-            waterbodies=waterbodies,
-            out_vector_files=out_stream_network_dissolved,
-            verbose=False,
+            waterbodies=waterbodies, out_vector_files=out_stream_network_dissolved, verbose=False
         )
         stream_network = stream_network.select_branches_intersecting_huc(
             wbd=wbd,
@@ -206,8 +190,7 @@ def Derive_level_paths(
 
     if branch_inlets_outfile is not None:
         branch_inlets = stream_network.derive_inlet_points_by_feature(
-            feature_attribute=branch_id_attribute,
-            outlet_linestring_index=outlet_linestring_index,
+            feature_attribute=branch_id_attribute, outlet_linestring_index=outlet_linestring_index
         )
 
         branch_inlets.to_file(branch_inlets_outfile, index=False, driver="GPKG")
@@ -217,31 +200,17 @@ def Derive_level_paths(
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Create stream network level paths")
+    parser.add_argument("-i", "--in-stream-network", help="Input stream network", required=True)
     parser.add_argument(
-        "-i", "--in-stream-network", help="Input stream network", required=True
+        "-s", "--buffer-wbd-streams", help="Input wbd buffer for stream network", required=True
     )
     parser.add_argument(
-        "-s",
-        "--buffer-wbd-streams",
-        help="Input wbd buffer for stream network",
-        required=True,
+        "-wbd", "--wbd", help="Input watershed boundary (HUC) dataset", required=True, default=None
     )
     parser.add_argument(
-        "-wbd",
-        "--wbd",
-        help="Input watershed boundary (HUC) dataset",
-        required=True,
-        default=None,
-    )
-    parser.add_argument(
-        "-b",
-        "--branch-id-attribute",
-        help="Name of the branch attribute desired",
-        required=True,
-    )
-    parser.add_argument(
-        "-u", "--huc-id", help="Current HUC ID", required=False, default=None
+        "-b", "--branch-id-attribute", help="Name of the branch attribute desired", required=True
     )
+    parser.add_argument("-u", "--huc-id", help="Current HUC ID", required=False, default=None)
     parser.add_argument(
         "-r",
         "--reach-id-attribute",
@@ -250,11 +219,7 @@ def Derive_level_paths(
         default="HydroID",
     )
     parser.add_argument(
-        "-c",
-        "--catchments",
-        help="NWM catchments to append level path data to",
-        required=False,
-        default=None,
+        "-c", "--catchments", help="NWM catchments to append level path data to", required=False, default=None
     )
     parser.add_argument(
         "-t",
@@ -264,25 +229,13 @@ def Derive_level_paths(
         default=None,
     )
     parser.add_argument(
-        "-w",
-        "--waterbodies",
-        help="NWM waterbodies to eliminate branches from",
-        required=False,
-        default=None,
+        "-w", "--waterbodies", help="NWM waterbodies to eliminate branches from", required=False, default=None
     )
     parser.add_argument(
-        "-n",
-        "--branch_inlets_outfile",
-        help="Output level paths inlets",
-        required=False,
-        default=None,
+        "-n", "--branch_inlets_outfile", help="Output level paths inlets", required=False, default=None
     )
     parser.add_argument(
-        "-o",
-        "--out-stream-network",
-        help="Output stream network",
-        required=False,
-        default=None,
+        "-o", "--out-stream-network", help="Output stream network", required=False, default=None
     )
     parser.add_argument(
         "-e",
@@ -299,12 +252,7 @@ def Derive_level_paths(
         default=None,
     )
     parser.add_argument(
-        "-v",
-        "--verbose",
-        help="Verbose output",
-        required=False,
-        default=False,
-        action="store_true",
+        "-v", "--verbose", help="Verbose output", required=False, default=False, action="store_true"
     )
 
     args = vars(parser.parse_args())
diff --git a/src/edit_points.py b/src/edit_points.py
index 400386cc9..a6f078df5 100755
--- a/src/edit_points.py
+++ b/src/edit_points.py
@@ -1,54 +1,70 @@
 #!/usr/bin/env python3
 
-import geopandas as gpd
 import argparse
 
+import geopandas as gpd
 
-def Edit_points(stream_reaches,branch_id_attribute,reach_points,
-                out_reach_points=None,out_pixel_points=None,verbose=False):
 
+def Edit_points(
+    stream_reaches,
+    branch_id_attribute,
+    reach_points,
+    out_reach_points=None,
+    out_pixel_points=None,
+    verbose=False,
+):
     if verbose:
         print("Editing points files ...")
 
     if verbose:
         print("Loading files ...")
     stream_reaches = gpd.read_file(stream_reaches)
-    stream_reaches = stream_reaches.astype({'HydroID':int})
+    stream_reaches = stream_reaches.astype({'HydroID': int})
 
-    reach_points=gpd.read_file(reach_points)
+    reach_points = gpd.read_file(reach_points)
     reach_points['HydroID'] = reach_points['id'].copy()
-    
+
     # merge
     if verbose:
         print("Merging points ...")
-    reach_points = reach_points.merge(stream_reaches.loc[:,["HydroID",branch_id_attribute]],how='inner',on='HydroID')
-    
+    reach_points = reach_points.merge(
+        stream_reaches.loc[:, ["HydroID", branch_id_attribute]], how='inner', on='HydroID'
+    )
+
     # join on HydroID to add branch_id
     if out_reach_points is not None:
-        reach_points.to_file(out_reach_points,driver='GPKG',index=False)
+        reach_points.to_file(out_reach_points, driver='GPKG', index=False)
 
     # make pixel points
     if verbose:
         print("Generating pixel points ...")
 
     pixel_points = reach_points.copy()
-    pixel_points['id'] = list(range(1,len(pixel_points)+1))
+    pixel_points['id'] = list(range(1, len(pixel_points) + 1))
 
     if out_pixel_points is not None:
-        pixel_points.to_file(out_pixel_points,driver='GPKG',index=False)
+        pixel_points.to_file(out_pixel_points, driver='GPKG', index=False)
 
-    return(reach_points,pixel_points)
+    return (reach_points, pixel_points)
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Edit points to include branch ids')
-    parser.add_argument('-i','--stream-reaches', help='Input stream network', required=True)
-    parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
-    parser.add_argument('-r','--reach-points', help='Name of the branch attribute desired', required=True)
-    parser.add_argument('-o','--out-reach-points', help='Output stream network', required=False,default=None)
-    parser.add_argument('-p','--out-pixel-points', help='Dissolved output stream network', required=False,default=None)
-    parser.add_argument('-v','--verbose', help='Verbose output', required=False,default=False,action='store_true')
-    
+    parser.add_argument('-i', '--stream-reaches', help='Input stream network', required=True)
+    parser.add_argument(
+        '-b', '--branch-id-attribute', help='Name of the branch attribute desired', required=True
+    )
+    parser.add_argument('-r', '--reach-points', help='Name of the branch attribute desired', required=True)
+    parser.add_argument(
+        '-o', '--out-reach-points', help='Output stream network', required=False, default=None
+    )
+    parser.add_argument(
+        '-p', '--out-pixel-points', help='Dissolved output stream network', required=False, default=None
+    )
+    parser.add_argument(
+        '-v', '--verbose', help='Verbose output', required=False, default=False, action='store_true'
+    )
+
     args = vars(parser.parse_args())
 
     Edit_points(**args)
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index 0b17be3d7..a638f7a84 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -1,59 +1,60 @@
 #!/usr/bin/env python3
 
 import argparse
+import sys
+
 import geopandas as gpd
 import numpy as np
-import sys
 
-from utils.shared_variables import FIM_ID
-from utils.shared_functions import mem_profile
 from utils.fim_enums import FIM_exit_codes
+from utils.shared_functions import mem_profile
+from utils.shared_variables import FIM_ID
 
-@mem_profile
-def filter_catchments_and_add_attributes(input_catchments_filename,
-                                         input_flows_filename,
-                                         output_catchments_filename,
-                                         output_flows_filename,
-                                         wbd_filename,
-                                         huc_code):
 
+@mem_profile
+def filter_catchments_and_add_attributes(
+    input_catchments_filename,
+    input_flows_filename,
+    output_catchments_filename,
+    output_flows_filename,
+    wbd_filename,
+    huc_code,
+):
     input_catchments = gpd.read_file(input_catchments_filename)
     wbd = gpd.read_file(wbd_filename)
     input_flows = gpd.read_file(input_flows_filename)
 
     # filter segments within huc boundary
-    select_flows = tuple(map(str,map(int,wbd[wbd.HUC8.str.contains(huc_code)][FIM_ID])))
-    
+    select_flows = tuple(map(str, map(int, wbd[wbd.HUC8.str.contains(huc_code)][FIM_ID])))
+
     if input_flows.HydroID.dtype != 'str':
         input_flows.HydroID = input_flows.HydroID.astype(str)
     output_flows = input_flows[input_flows.HydroID.str.startswith(select_flows)].copy()
-    
+
     if output_flows.HydroID.dtype != 'int':
         output_flows.HydroID = output_flows.HydroID.astype(int)
 
     if len(output_flows) > 0:
-
         # merges input flows attributes and filters hydroids
         if input_catchments.HydroID.dtype != 'int':
             input_catchments.HydroID = input_catchments.HydroID.astype(int)
-        output_catchments = input_catchments.merge(output_flows.drop(['geometry'],axis=1),on='HydroID')
+        output_catchments = input_catchments.merge(output_flows.drop(['geometry'], axis=1), on='HydroID')
 
         # filter out smaller duplicate features
-        duplicateFeatures = np.where(np.bincount(output_catchments['HydroID'])>1)[0]
+        duplicateFeatures = np.where(np.bincount(output_catchments['HydroID']) > 1)[0]
 
         for dp in duplicateFeatures:
-
             indices_of_duplicate = np.where(output_catchments['HydroID'] == dp)[0]
-            areas = output_catchments.iloc[indices_of_duplicate,:].geometry.area
+            areas = output_catchments.iloc[indices_of_duplicate, :].geometry.area
             indices_of_smaller_duplicates = indices_of_duplicate[np.where(areas != np.amax(areas))[0]]
             output_catchments = output_catchments.drop(output_catchments.index[indices_of_smaller_duplicates])
 
         # add geometry column
-        output_catchments['areasqkm'] = output_catchments.geometry.area/(1000**2)
+        output_catchments['areasqkm'] = output_catchments.geometry.area / (1000**2)
 
         if not output_catchments.empty:
             try:
-                output_catchments.to_file(output_catchments_filename, driver="GPKG",index=False)
+                output_catchments.to_file(output_catchments_filename, driver="GPKG", index=False)
                 output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
             except ValueError:
                 # this is not an exception, but a custom exit code that can be trapped
@@ -71,16 +72,17 @@ def filter_catchments_and_add_attributes(input_catchments_filename,
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
     parser = argparse.ArgumentParser(description='filter_catchments_and_add_attributes.py')
     parser.add_argument('-i', '--input-catchments-filename', help='input-catchments-filename', required=True)
     parser.add_argument('-f', '--input-flows-filename', help='input-flows-filename', required=True)
-    parser.add_argument('-c', '--output-catchments-filename', help='output-catchments-filename', required=True)
+    parser.add_argument(
+        '-c', '--output-catchments-filename', help='output-catchments-filename', required=True
+    )
     parser.add_argument('-o', '--output-flows-filename', help='output-flows-filename', required=True)
     parser.add_argument('-w', '--wbd-filename', help='wbd-filename', required=True)
     parser.add_argument('-u', '--huc-code', help='huc-code', required=True)
-    
+
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
 
diff --git a/src/filter_inputs_by_huc.py b/src/filter_inputs_by_huc.py
index 4543adacc..0ffb20559 100755
--- a/src/filter_inputs_by_huc.py
+++ b/src/filter_inputs_by_huc.py
@@ -1,19 +1,20 @@
 #!/usr/bin/env python3
 
-import pandas as pd
 import argparse
 
-def filter_inputs_by_huc(fim_inputs, hucs, fim_outputs):
+import pandas as pd
+
 
+def filter_inputs_by_huc(fim_inputs, hucs, fim_outputs):
     try:
         with open(hucs[0]) as hf:
             hucsList = set([str(h).rstrip() for h in hf])
     except FileNotFoundError:
         hucsList = set(hucs)
-    
-    fim_inputs = pd.read_csv(fim_inputs,header=None,dtype=str)
-    fim_inputs_mask = fim_inputs.loc[:,0].isin(hucsList)
-    fim_inputs = fim_inputs.loc[fim_inputs_mask,:]
+
+    fim_inputs = pd.read_csv(fim_inputs, header=None, dtype=str)
+    fim_inputs_mask = fim_inputs.loc[:, 0].isin(hucsList)
+    fim_inputs = fim_inputs.loc[fim_inputs_mask, :]
 
     assert len(fim_inputs) > 0, "Filtered FIM list is empty"
 
@@ -21,12 +22,13 @@ def filter_inputs_by_huc(fim_inputs, hucs, fim_outputs):
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Adjusts the elevation of the thalweg to the lateral zonal minimum.')
-    parser.add_argument('-g','--fim-inputs',help='Raster of elevation.',required=True)
-    parser.add_argument('-u','--hucs',help='Raster of elevation.',required=True,nargs='+')
-    parser.add_argument('-o','--fim-outputs',help='Raster of elevation.',required=True)
+    parser = argparse.ArgumentParser(
+        description='Adjusts the elevation of the thalweg to the lateral zonal minimum.'
+    )
+    parser.add_argument('-g', '--fim-inputs', help='Raster of elevation.', required=True)
+    parser.add_argument('-u', '--hucs', help='Raster of elevation.', required=True, nargs='+')
+    parser.add_argument('-o', '--fim-outputs', help='Raster of elevation.', required=True)
 
     args = vars(parser.parse_args())
 
diff --git a/src/finalize_srcs.py b/src/finalize_srcs.py
index 46b61ca8b..3ca975bdf 100755
--- a/src/finalize_srcs.py
+++ b/src/finalize_srcs.py
@@ -1,66 +1,80 @@
 #!/usr/bin/env python3
 
+import argparse
+import json
+
 import geopandas as gpd
 import pandas as pd
 from numpy import unique
-import json
-import argparse
-from utils.shared_functions import getDriver
 
-def finalize_srcs(srcbase,srcfull,hydrotable,output_srcfull=None,output_hydrotable=None):
+from utils.shared_functions import getDriver
 
 
+def finalize_srcs(srcbase, srcfull, hydrotable, output_srcfull=None, output_hydrotable=None):
     # calculate src_full
-    srcbase = pd.read_csv(srcbase, dtype= {'CatchId': int})
-    srcbase.rename(columns={'CatchId':'HydroID'},inplace=True)
+    srcbase = pd.read_csv(srcbase, dtype={'CatchId': int})
+    srcbase.rename(columns={'CatchId': 'HydroID'}, inplace=True)
     srcbase = srcbase.rename(columns=lambda x: x.strip(" "))
-    
+
     # read and merge in attributes from base hydrofabric src full
     srcfull = pd.read_csv(srcfull, dtype={'CatchId': int})
-    srcfull.rename(columns={'CatchId':'HydroID'},inplace=True)
-    srcfull = srcfull.loc[:,["ManningN","HydroID","feature_id"]].drop_duplicates(subset='HydroID')
-    
-    srcbase = srcbase.merge(srcfull,how='inner',left_on='HydroID',right_on='HydroID')
-
-    srcbase = srcbase.apply(pd.to_numeric,**{'errors' : 'coerce'})
-    srcbase['TopWidth (m)'] = srcbase['SurfaceArea (m2)']/srcbase['LENGTHKM']/1000
-    srcbase['WettedPerimeter (m)'] = srcbase['BedArea (m2)']/srcbase['LENGTHKM']/1000
-    srcbase['WetArea (m2)'] = srcbase['Volume (m3)']/srcbase['LENGTHKM']/1000
-    srcbase['HydraulicRadius (m)'] = srcbase['WetArea (m2)']/srcbase['WettedPerimeter (m)']
+    srcfull.rename(columns={'CatchId': 'HydroID'}, inplace=True)
+    srcfull = srcfull.loc[:, ["ManningN", "HydroID", "feature_id"]].drop_duplicates(subset='HydroID')
+
+    srcbase = srcbase.merge(srcfull, how='inner', left_on='HydroID', right_on='HydroID')
+
+    srcbase = srcbase.apply(pd.to_numeric, **{'errors': 'coerce'})
+    srcbase['TopWidth (m)'] = srcbase['SurfaceArea (m2)'] / srcbase['LENGTHKM'] / 1000
+    srcbase['WettedPerimeter (m)'] = srcbase['BedArea (m2)'] / srcbase['LENGTHKM'] / 1000
+    srcbase['WetArea (m2)'] = srcbase['Volume (m3)'] / srcbase['LENGTHKM'] / 1000
+    srcbase['HydraulicRadius (m)'] = srcbase['WetArea (m2)'] / srcbase['WettedPerimeter (m)']
     srcbase['HydraulicRadius (m)'].fillna(0, inplace=True)
-    srcbase['Discharge (m3s-1)'] = srcbase['WetArea (m2)']* \
-                                          pow(srcbase['HydraulicRadius (m)'],2.0/3)* \
-                                          pow(srcbase['SLOPE'],0.5)/srcbase['ManningN']
+    srcbase['Discharge (m3s-1)'] = (
+        srcbase['WetArea (m2)']
+        * pow(srcbase['HydraulicRadius (m)'], 2.0 / 3)
+        * pow(srcbase['SLOPE'], 0.5)
+        / srcbase['ManningN']
+    )
 
     # set nans to 0
-    srcbase.loc[srcbase['Stage']==0,['Discharge (m3s-1)']] = 0
+    srcbase.loc[srcbase['Stage'] == 0, ['Discharge (m3s-1)']] = 0
 
     if output_srcfull is not None:
-        srcbase.to_csv(output_srcfull,index=False)
+        srcbase.to_csv(output_srcfull, index=False)
 
     hydrotable = pd.read_csv(hydrotable)
-    hydrotable.drop(columns=['stage','discharge_cms'],inplace=True)
-    
-    hydrotable.drop_duplicates(subset='HydroID',inplace=True)
-    #srcfull = srcfull.loc[:,["ManningN","HydroID","feature_id"]].drop_duplicates(subset='HydroID')
-    hydrotable = hydrotable.merge(srcbase.loc[:,['HydroID','Stage','Discharge (m3s-1)']],how='right',left_on='HydroID',right_on='HydroID')
-    hydrotable.rename(columns={'Stage' : 'stage','Discharge (m3s-1)':'discharge_cms'},inplace=True)
-    #hydrotable.drop_duplicates(subset='stage',inplace=True)
-    
+    hydrotable.drop(columns=['stage', 'discharge_cms'], inplace=True)
+
+    hydrotable.drop_duplicates(subset='HydroID', inplace=True)
+    # srcfull = srcfull.loc[:,["ManningN","HydroID","feature_id"]].drop_duplicates(subset='HydroID')
+    hydrotable = hydrotable.merge(
+        srcbase.loc[:, ['HydroID', 'Stage', 'Discharge (m3s-1)']],
+        how='right',
+        left_on='HydroID',
+        right_on='HydroID',
+    )
+    hydrotable.rename(columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'}, inplace=True)
+    # hydrotable.drop_duplicates(subset='stage',inplace=True)
+
     if output_hydrotable is not None:
-        hydrotable.to_csv(output_hydrotable,index=False)
+        hydrotable.to_csv(output_hydrotable, index=False)
 
-    return(srcbase,hydrotable)
+    return (srcbase, hydrotable)
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='')
-    parser.add_argument('-b','--srcbase', help='Base synthetic rating curve table', required=True)
-    parser.add_argument('-f','--srcfull', help='Base synthetic rating curve table', required=True)
-    parser.add_argument('-w','--hydrotable',help='Input Hydro-Table',required=False)
-    parser.add_argument('-r','--output-srcfull', help='Output crosswalked synthetic rating curve table', required=False,default=None)
-    parser.add_argument('-t','--output-hydrotable',help='Hydrotable',required=False,default=None)
+    parser.add_argument('-b', '--srcbase', help='Base synthetic rating curve table', required=True)
+    parser.add_argument('-f', '--srcfull', help='Base synthetic rating curve table', required=True)
+    parser.add_argument('-w', '--hydrotable', help='Input Hydro-Table', required=False)
+    parser.add_argument(
+        '-r',
+        '--output-srcfull',
+        help='Output crosswalked synthetic rating curve table',
+        required=False,
+        default=None,
+    )
+    parser.add_argument('-t', '--output-hydrotable', help='Hydrotable', required=False, default=None)
 
     args = vars(parser.parse_args())
 
diff --git a/src/generate_branch_list.py b/src/generate_branch_list.py
index dc4a8e094..eb738fc73 100755
--- a/src/generate_branch_list.py
+++ b/src/generate_branch_list.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import os
 
 from stream_branches import StreamNetwork
 
diff --git a/src/generate_branch_list_csv.py b/src/generate_branch_list_csv.py
index e355e9f25..be5a67568 100755
--- a/src/generate_branch_list_csv.py
+++ b/src/generate_branch_list_csv.py
@@ -1,55 +1,55 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
-import pandas as pd
+import os
 import pathlib
 
-def generate_branch_list_csv(huc_id, branch_id, output_branch_csv):
+import pandas as pd
+
 
+def generate_branch_list_csv(huc_id, branch_id, output_branch_csv):
     '''
     Processing:
         This create a branch_ids.csv file which is required for various post processing tasks.
         If the csv already, then the new huc, branch id wil be appended.
         If it does not yet exist, a new csv will be created
-        
+
     Params:
         - huc_id
         - branch_id
         - output_branch_csv (str): csv file name and path of the list to be created. (likely branch_list.csv)
-        
+
     Output:
         - create a csv file (assuming the format coming in is a csv
     '''
     # validations
     file_extension = pathlib.Path(output_branch_csv).suffix
-   
-    if (file_extension != ".csv"):
+
+    if file_extension != ".csv":
         raise ValueError("The output branch csv file does not have a .csv extension")
-    
+
     if (len(huc_id) != 8) or (not huc_id.isnumeric()):
         raise ValueError("The huc_id does not appear to be an eight digit number")
-    
-    if (not branch_id.isnumeric()):
+
+    if not branch_id.isnumeric():
         raise ValueError("The branch_id does not appear to be a valid number")
-    
+
     df_csv = None
     new_data = [[huc_id, branch_id]]
-    col_names = ["huc_id","branch_id"]
-    df_csv = pd.DataFrame(new_data, columns=col_names)    
+    col_names = ["huc_id", "branch_id"]
+    df_csv = pd.DataFrame(new_data, columns=col_names)
 
-    if (not os.path.exists(output_branch_csv)):
+    if not os.path.exists(output_branch_csv):
         df_csv.to_csv(output_branch_csv, index=False, header=False)
     else:
         df_csv.to_csv(output_branch_csv, mode='a', index=False, header=False)
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='Create branch list')
-    parser.add_argument('-b','--branch-id', help='Branch ID', required=True)
-    parser.add_argument('-o','--output-branch-csv', help='Output branch csv list', required=True)
-    parser.add_argument('-u','--huc-id', help='HUC number being aggregated', required=True)
+    parser.add_argument('-b', '--branch-id', help='Branch ID', required=True)
+    parser.add_argument('-o', '--output-branch-csv', help='Output branch csv list', required=True)
+    parser.add_argument('-u', '--huc-id', help='HUC number being aggregated', required=True)
     args = vars(parser.parse_args())
 
     generate_branch_list_csv(**args)
diff --git a/src/getRasterInfoNative.py b/src/getRasterInfoNative.py
index 3164ce6d0..e3b8a69dc 100755
--- a/src/getRasterInfoNative.py
+++ b/src/getRasterInfoNative.py
@@ -1,70 +1,74 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8
 
-#TODO standardize this script
+# TODO standardize this script
 
-from osgeo import gdal
 import os.path
 import sys
-from osgeo import osr
+
+from osgeo import gdal, osr
+
 
 """
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy <<< $(./getRasterInfoNative.py <raster.tif>)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy
+    <<< $(./getRasterInfoNative.py <raster.tif>)
 """
 
 
-def GetExtent(gt,cols,rows):
-	''' Return list of corner coordinates from a geotransform
-
-	@type gt:   C{tuple/list}
-	@param gt: geotransform
-	@type cols:   C{int}
-	@param cols: number of columns in the dataset
-	@type rows:   C{int}
-	@param rows: number of rows in the dataset
-	@rtype:    C{[float,...,float]}
-	@return:   coordinates of each corner
-	'''
-	ext=[]
-	xarr=[0,cols]
-	yarr=[0,rows]
-
-	for px in xarr:
-		for py in yarr:
-			x=gt[0]+(px*gt[1])+(py*gt[2])
-			y=gt[3]+(px*gt[4])+(py*gt[5])
-			ext.append([x,y])
-			#print x,y
-		yarr.reverse()
-	return ext
-
-
-def ReprojectCoords(coords,src_srs,tgt_srs):
-	''' Reproject a list of x,y coordinates.
-
-	@type geom:     C{tuple/list}
-	@param geom:    List of [[x,y],...[x,y]] coordinates
-	@type src_srs:  C{osr.SpatialReference}
-	@param src_srs: OSR SpatialReference object
-	@type tgt_srs:  C{osr.SpatialReference}
-	@param tgt_srs: OSR SpatialReference object
-	@rtype:         C{tuple/list}
-	@return:        List of transformed [[x,y],...[x,y]] coordinates
-	'''
-	trans_coords=[]
-	transform = osr.CoordinateTransformation( src_srs, tgt_srs)
-	for x,y in coords:
-		x,y,z = transform.TransformPoint(x,y)
-		trans_coords.append([x,y])
-	return trans_coords
+def GetExtent(gt, cols, rows):
+    '''Return list of corner coordinates from a geotransform
+
+    @type gt:   C{tuple/list}
+    @param gt: geotransform
+    @type cols:   C{int}
+    @param cols: number of columns in the dataset
+    @type rows:   C{int}
+    @param rows: number of rows in the dataset
+    @rtype:    C{[float,...,float]}
+    @return:   coordinates of each corner
+    '''
+    ext = []
+    xarr = [0, cols]
+    yarr = [0, rows]
+
+    for px in xarr:
+        for py in yarr:
+            x = gt[0] + (px * gt[1]) + (py * gt[2])
+            y = gt[3] + (px * gt[4]) + (py * gt[5])
+            ext.append([x, y])
+            # print x,y
+        yarr.reverse()
+    return ext
+
+
+def ReprojectCoords(coords, src_srs, tgt_srs):
+    '''Reproject a list of x,y coordinates.
+
+    @type geom:     C{tuple/list}
+    @param geom:    List of [[x,y],...[x,y]] coordinates
+    @type src_srs:  C{osr.SpatialReference}
+    @param src_srs: OSR SpatialReference object
+    @type tgt_srs:  C{osr.SpatialReference}
+    @param tgt_srs: OSR SpatialReference object
+    @rtype:         C{tuple/list}
+    @return:        List of transformed [[x,y],...[x,y]] coordinates
+    '''
+    trans_coords = []
+    transform = osr.CoordinateTransformation(src_srs, tgt_srs)
+    for x, y in coords:
+        x, y, z = transform.TransformPoint(x, y)
+        trans_coords.append([x, y])
+    return trans_coords
+
 
 # get file size function
 def sizeof_fmt(num, suffix='B'):
-	for unit in ['','K','M','G','T','P','E','Z']:
-		if abs(num) < 1024.0:
-			return "%3.1f%s%s" % (num, unit, suffix)
-		num /= 1024.0
-	return "%.1f%s%s" % (num, 'Y', suffix)
+    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Y', suffix)
+
 
 # open dataset
 ds = gdal.Open(sys.argv[1])
@@ -72,32 +76,43 @@ def sizeof_fmt(num, suffix='B'):
 cols = ds.RasterXSize
 rows = ds.RasterYSize
 nodata = ds.GetRasterBand(1).GetNoDataValue()
-#stats = ds.GetRasterBand(1).GetStatistics(True, True)
+# stats = ds.GetRasterBand(1).GetStatistics(True, True)
 
-gt=ds.GetGeoTransform()
-ext=GetExtent(gt,cols,rows)
+gt = ds.GetGeoTransform()
+ext = GetExtent(gt, cols, rows)
 
-src_srs=osr.SpatialReference()
+src_srs = osr.SpatialReference()
 src_srs.ImportFromWkt(ds.GetProjection())
 tgt_srs = src_srs.CloneGeogCS()
 
-geo_ext=ReprojectCoords(ext,src_srs,tgt_srs)
+geo_ext = ReprojectCoords(ext, src_srs, tgt_srs)
 lon1 = ext[0][0]
 lon2 = ext[2][0]
 lat1 = ext[2][1]
 lat2 = ext[0][1]
 
 # calculate cellsize
-resx = (ext[2][0] - ext[0][0])/cols
-resy = (ext[0][1] - ext[2][1])/rows
+resx = (ext[2][0] - ext[0][0]) / cols
+resy = (ext[0][1] - ext[2][1]) / rows
 
 # print out RasterInfos
-print(fsize,cols,rows,nodata,str(lon1),str(lat1),str(lon2),str(lat2),"{:.15f}".format(resx),"{:.15f}".format(resy))
-#print stats[0],
-#print stats[1],
-#print stats[2],
-#print stats[3],
-#print "\"" + str(lat1) + "," + str(lon1) + "," + str(lat2) + "," + str(lon2) + "\""
+print(
+    fsize,
+    cols,
+    rows,
+    nodata,
+    str(lon1),
+    str(lat1),
+    str(lon2),
+    str(lat2),
+    "{:.15f}".format(resx),
+    "{:.15f}".format(resy),
+)
+# print stats[0],
+# print stats[1],
+# print stats[2],
+# print stats[3],
+# print "\"" + str(lat1) + "," + str(lon1) + "," + str(lat2) + "," + str(lon2) + "\""
 # print str(lon1),
 # print str(lat1),
 # print str(lon2),
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index 0547aa7fe..efef55938 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -1,22 +1,25 @@
 #!/usr/bin/env python3
 
 import argparse
-import os
-import sys
-import pandas as pd
-import matplotlib.pyplot as plt
-import multiprocessing
-import seaborn as sns
-import shutil
-import warnings
 import datetime as dt
+import multiprocessing
+import os
 import re
+import shutil
+import sys
 import traceback
-from pathlib import Path
+import warnings
 from functools import reduce
 from multiprocessing import Pool
-from os.path import isfile, join, dirname, isdir
+from os.path import dirname, isdir, isfile, join
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sns
 from tqdm import tqdm
+
+
 sns.set_theme(style="whitegrid")
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
@@ -35,37 +38,55 @@
         Optional: Flag to create SRC plots for all hydroids (True/False)
 """
 
-def src_bankfull_lookup(args):
 
-    src_full_filename           = args[0]
-    src_usecols                 = args[1]
-    df_bflows                   = args[2]
-    huc                         = args[3]
-    branch_id                   = args[4]
-    src_plot_option             = args[5]
-    huc_output_dir              = args[6]
+def src_bankfull_lookup(args):
+    src_full_filename = args[0]
+    src_usecols = args[1]
+    df_bflows = args[2]
+    huc = args[3]
+    branch_id = args[4]
+    src_plot_option = args[5]
+    huc_output_dir = args[6]
 
     ## Read the src_full_crosswalked.csv
     print('Calculating bankfull: ' + str(huc) + '  branch id: ' + str(branch_id))
     log_text = 'Calculating: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
     try:
-        df_src = pd.read_csv(src_full_filename,usecols=src_usecols,dtype={'HydroID': int,'feature_id': int})
+        df_src = pd.read_csv(
+            src_full_filename, usecols=src_usecols, dtype={'HydroID': int, 'feature_id': int}
+        )
 
         ## NWM recurr rename discharge var
-        df_bflows = df_bflows.rename(columns={'discharge':'bankfull_flow'})
+        df_bflows = df_bflows.rename(columns={'discharge': 'bankfull_flow'})
 
         ## Combine the nwm bankfull estimated flows into the SRC via feature_id
-        df_src = df_src.merge(df_bflows,how='left',on='feature_id')
+        df_src = df_src.merge(df_bflows, how='left', on='feature_id')
 
         ## Check if there are any missing data, negative or zero flow values in the bankfull_flow
         check_null = df_src['bankfull_flow'].isnull().sum()
         if check_null > 0:
-            log_text += 'WARNING: Missing feature_id in crosswalk for huc: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> these featureids will be ignored in bankfull calcs (~' + str(check_null/84) +  ' features) \n'
+            log_text += (
+                'WARNING: Missing feature_id in crosswalk for huc: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' --> these featureids will be ignored in bankfull calcs (~'
+                + str(check_null / 84)
+                + ' features) \n'
+            )
             ## Fill missing/nan nwm bankfull_flow values with -999 to handle later
             df_src['bankfull_flow'] = df_src['bankfull_flow'].fillna(-999)
+
         negative_flows = len(df_src.loc[(df_src.bankfull_flow <= 0) & (df_src.bankfull_flow != -999)])
+
         if negative_flows > 0:
-            log_text += 'WARNING: HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> Negative or zero flow values found in the input bankfull flows csv (posible lakeid loc)\n'
+            log_text += (
+                'WARNING: HUC: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' --> Negative or zero flow values found in the input bankfull flows csv (posible lakeid loc)\n'
+            )
 
         ## Define the channel geometry variable names to use from the src
         hradius_var = 'HydraulicRadius (m)'
@@ -77,47 +98,106 @@ def src_bankfull_lookup(args):
         df_src['Q_bfull_find'] = (df_src['bankfull_flow'] - df_src['Discharge (m3s-1)']).abs()
 
         ## Check for any missing/null entries in the input SRC
-        if df_src['Q_bfull_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
-            log_text += 'WARNING: HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> Null values found in "Q_bfull_find" calc. These will be filled with 999999 () \n'
+        # There may be null values for lake or coastal flow lines
+        # (need to set a value to do groupby idxmin below)
+        if df_src['Q_bfull_find'].isnull().values.any():
+            log_text += (
+                'WARNING: HUC: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' --> Null values found in "Q_bfull_find" calc. These will be filled with 999999 () \n'
+            )
             ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
             df_src['Q_bfull_find'] = df_src['Q_bfull_find'].fillna(999999)
         if df_src['HydroID'].isnull().values.any():
-            log_text += 'WARNING: HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> Null values found in "HydroID"... \n'
-
-        df_bankfull_calc = df_src[['Stage','HydroID',bedarea_var,volume_var,hradius_var,surface_area_var,'Q_bfull_find']] # create new subset df to perform the Q_1_5 lookup
-        df_bankfull_calc = df_bankfull_calc[df_bankfull_calc['Stage'] > 0.0] # Ensure bankfull stage is greater than stage=0
+            log_text += (
+                'WARNING: HUC: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' --> Null values found in "HydroID"... \n'
+            )
+
+        df_bankfull_calc = df_src[
+            ['Stage', 'HydroID', bedarea_var, volume_var, hradius_var, surface_area_var, 'Q_bfull_find']
+        ]  # create new subset df to perform the Q_1_5 lookup
+        df_bankfull_calc = df_bankfull_calc[
+            df_bankfull_calc['Stage'] > 0.0
+        ]  # Ensure bankfull stage is greater than stage=0
         df_bankfull_calc.reset_index(drop=True, inplace=True)
-        df_bankfull_calc = df_bankfull_calc.loc[df_bankfull_calc.groupby('HydroID')['Q_bfull_find'].idxmin()].reset_index(drop=True) # find the index of the Q_bfull_find (closest matching flow)
-        df_bankfull_calc = df_bankfull_calc.rename(columns={'Stage':'Stage_bankfull',bedarea_var:'BedArea_bankfull',volume_var:'Volume_bankfull',hradius_var:'HRadius_bankfull',surface_area_var:'SurfArea_bankfull'}) # rename volume to use later for channel portion calc
-        df_src = df_src.merge(df_bankfull_calc[['Stage_bankfull','HydroID','BedArea_bankfull','Volume_bankfull','HRadius_bankfull','SurfArea_bankfull']],how='left',on='HydroID')
+        # find the index of the Q_bfull_find (closest matching flow)
+        df_bankfull_calc = df_bankfull_calc.loc[
+            df_bankfull_calc.groupby('HydroID')['Q_bfull_find'].idxmin()
+        ].reset_index(
+            drop=True
+        )
+        # rename volume to use later for channel portion calc
+        df_bankfull_calc = df_bankfull_calc.rename(
+            columns={
+                'Stage': 'Stage_bankfull',
+                bedarea_var: 'BedArea_bankfull',
+                volume_var: 'Volume_bankfull',
+                hradius_var: 'HRadius_bankfull',
+                surface_area_var: 'SurfArea_bankfull'
+            }
+        )
+        df_src = df_src.merge(
+            df_bankfull_calc[
+                [
+                    'Stage_bankfull',
+                    'HydroID',
+                    'BedArea_bankfull',
+                    'Volume_bankfull',
+                    'HRadius_bankfull',
+                    'SurfArea_bankfull'
+                ]
+            ],
+            how='left',
+            on='HydroID'
+        )
         df_src.drop(['Q_bfull_find'], axis=1, inplace=True)
 
-        ## The bankfull ratio variables below were previously used for the composite variable roughness routine (not currently implimented)
+        ## The bankfull ratio variables below were previously used for the composite variable roughness routine
+        ##      (not currently implimented)
         # ## Calculate the channel portion of bankfull Volume
         # df_src['chann_volume_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
-        # df_src['chann_volume_ratio'].where(df_src['Stage'] == 0, df_src['Volume_bankfull'] / (df_src[volume_var]),inplace=True)
+        # df_src['chann_volume_ratio'].where(
+        #     df_src['Stage'] == 0, df_src['Volume_bankfull'] / (df_src[volume_var]), inplace=True
+        # )
         # #df_src['chann_volume_ratio'] = df_src['chann_volume_ratio'].clip_upper(1.0)
-        # df_src['chann_volume_ratio'].where(df_src['chann_volume_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
-        # df_src['chann_volume_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True) # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # # set > 1.0 ratio values to 1.0 (these are within the channel)
+        # df_src['chann_volume_ratio'].where(df_src['chann_volume_ratio'] <= 1.0, 1.0, inplace=True)
+        # # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # df_src['chann_volume_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True)
         # #df_src.drop(['Volume_bankfull'], axis=1, inplace=True)
 
         # ## Calculate the channel portion of bankfull Hydraulic Radius
         # df_src['chann_hradius_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
-        # df_src['chann_hradius_ratio'].where(df_src['Stage'] == 0, df_src['HRadius_bankfull'] / (df_src[hradius_var]),inplace=True)
-        # #df_src['chann_hradius_ratio'] = df_src['HRadius_bankfull'] / (df_src[hradius_var]+.0001) # old adding 0.01 to avoid dividing by 0 at stage=0
-        # df_src['chann_hradius_ratio'].where(df_src['chann_hradius_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
-        # df_src['chann_hradius_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True) # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # df_src['chann_hradius_ratio'].where(
+        #     df_src['Stage'] == 0, df_src['HRadius_bankfull'] / (df_src[hradius_var]), inplace=True
+        # )
+        # # old adding 0.01 to avoid dividing by 0 at stage=0
+        # #df_src['chann_hradius_ratio'] = df_src['HRadius_bankfull'] / (df_src[hradius_var]+.0001)
+        # # set > 1.0 ratio values to 1.0 (these are within the channel)
+        # df_src['chann_hradius_ratio'].where(df_src['chann_hradius_ratio'] <= 1.0, 1.0, inplace=True)
+        # # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # df_src['chann_hradius_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True)
         # #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
 
         # ## Calculate the channel portion of bankfull Surface Area
         # df_src['chann_surfarea_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
-        # df_src['chann_surfarea_ratio'].where(df_src['Stage'] == 0, df_src['SurfArea_bankfull'] / (df_src[surface_area_var]),inplace=True)
-        # df_src['chann_surfarea_ratio'].where(df_src['chann_surfarea_ratio'] <= 1.0, 1.0, inplace=True) # set > 1.0 ratio values to 1.0 (these are within the channel)
-        # df_src['chann_surfarea_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True) # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # df_src['chann_surfarea_ratio'].where(
+        #     df_src['Stage'] == 0, df_src['SurfArea_bankfull'] / (df_src[surface_area_var]), inplace=True
+        # )
+        # # set > 1.0 ratio values to 1.0 (these are within the channel)
+        # df_src['chann_surfarea_ratio'].where(df_src['chann_surfarea_ratio'] <= 1.0, 1.0, inplace=True)
+        # # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
+        # df_src['chann_surfarea_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True)
         # #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
 
         ## mask bankfull variables when the bankfull estimated flow value is <= 0
-        df_src['Stage_bankfull'].mask(df_src['bankfull_flow'] <= 0.0,inplace=True)
+        df_src['Stage_bankfull'].mask(df_src['bankfull_flow'] <= 0.0, inplace=True)
 
         ## Create a new column to identify channel/floodplain via the bankfull stage value
         df_src.loc[df_src['Stage'] <= df_src['Stage_bankfull'], 'bankfull_proxy'] = 'channel'
@@ -125,68 +205,92 @@ def src_bankfull_lookup(args):
         df_src['bankfull_proxy'] = df_src['bankfull_proxy'].fillna('channel')
 
         ## Output new SRC with bankfull column
-        df_src.to_csv(src_full_filename,index=False)
+        df_src.to_csv(src_full_filename, index=False)
         log_text += 'Completed: ' + str(huc)
 
         ## plot rating curves (optional arg)
         if src_plot_option:
-            if isdir(huc_output_dir) == False:
+            if isdir(huc_output_dir) is False:
                 os.mkdir(huc_output_dir)
             generate_src_plot(df_src, huc_output_dir)
 
     except Exception as ex:
-        summary = traceback.StackSummary.extract(
-                traceback.walk_stack(None))
+        summary = traceback.StackSummary.extract(traceback.walk_stack(None))
         print(str(huc) + '  branch id: ' + str(branch_id) + " failed for some reason")
-        print(f"*** {ex}")                
-        print(''.join(summary.format()))    
-        log_text += 'ERROR --> ' + str(huc) + '  branch id: ' + str(branch_id) + " failed (details: " + (f"*** {ex}") + (''.join(summary.format())) + '\n'
-    return(log_text)
+        print(f"*** {ex}")
+        print(''.join(summary.format()))
+        log_text += (
+            'ERROR --> '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+            + " failed (details: "
+            + (f"*** {ex}")
+            + (''.join(summary.format()))
+            + '\n'
+        )
+    return log_text
 
-def generate_src_plot(df_src, plt_out_dir):
 
+def generate_src_plot(df_src, plt_out_dir):
     ## create list of unique hydroids
     hydroids = df_src.HydroID.unique().tolist()
-    #hydroids = [17820017]
+    # hydroids = [17820017]
 
     for hydroid in hydroids:
         print("Creating SRC plot: " + str(hydroid))
         plot_df = df_src.loc[df_src['HydroID'] == hydroid]
 
-        fig, axes = plt.subplots(1,2,figsize=(12, 6))
+        fig, axes = plt.subplots(1, 2, figsize=(12, 6))
         fig.suptitle(str(hydroid))
         axes[0].set_title('Rating Curve w/ Bankfull')
         axes[1].set_title('Channel Volume vs. HRadius Ratio')
         sns.despine(fig, left=True, bottom=True)
         sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, ax=axes[0])
         sns.lineplot(x='Discharge (m3s-1)', y='Stage_bankfull', data=plot_df, color='green', ax=axes[0])
-        axes[0].fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'],alpha=0.5)
-        axes[0].text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_bankfull'].median(), "Bankfull Proxy Stage: " + str(plot_df['Stage_bankfull'].median()))
-        sns.scatterplot(x='chann_volume_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_volume_ratio", s=38)
-        sns.scatterplot(x='chann_hradius_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_hradius_ratio", s=12)
-        sns.scatterplot(x='chann_surfarea_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_surfarea_ratio", s=12)
+        axes[0].fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'], alpha=0.5)
+        axes[0].text(
+            plot_df['Discharge (m3s-1)'].median(),
+            plot_df['Stage_bankfull'].median(),
+            "Bankfull Proxy Stage: " + str(plot_df['Stage_bankfull'].median()),
+        )
+        sns.scatterplot(
+            x='chann_volume_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_volume_ratio", s=38
+        )
+        sns.scatterplot(
+            x='chann_hradius_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_hradius_ratio", s=12
+        )
+        sns.scatterplot(
+            x='chann_surfarea_ratio', y='Stage', data=plot_df, ax=axes[1], label="chann_surfarea_ratio", s=12
+        )
         axes[1].legend()
-        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_bankfull.png',dpi=100, bbox_inches='tight')
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_bankfull.png', dpi=100, bbox_inches='tight')
         plt.close()
 
+
 def multi_process(src_bankfull_lookup, procs_list, log_file, number_of_jobs, verbose):
     ## Initiate multiprocessing
     available_cores = multiprocessing.cpu_count()
     if number_of_jobs > available_cores:
         number_of_jobs = available_cores - 2
-        print("Provided job number exceeds the number of available cores. " + str(number_of_jobs) + " max jobs will be used instead.")
+        print(
+            "Provided job number exceeds the number of available cores. "
+            f"{str(number_of_jobs)}"
+            " max jobs will be used instead."
+        )
 
     print(f"Identifying bankfull stage for {len(procs_list)} branches using {number_of_jobs} jobs")
     with Pool(processes=number_of_jobs) as pool:
-        #progress_bar = tqdm(total=len(procs_list[0]))
+        # progress_bar = tqdm(total=len(procs_list[0]))
         if verbose:
             map_output = tqdm(pool.imap(src_bankfull_lookup, procs_list), total=len(procs_list))
             tuple(map_output)  # fetch the lazy results
         else:
             map_output = pool.map(src_bankfull_lookup, procs_list)
-    log_file.writelines(["%s\n" % item  for item in map_output])
+    log_file.writelines(["%s\n" % item for item in map_output])
+
 
-def run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_option):
+def run_prep(fim_dir, bankfull_flow_filepath, number_of_jobs, verbose, src_plot_option):
     procs_list = []
 
     ## Print message to user and initiate run clock
@@ -196,39 +300,81 @@ def run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_opti
     ## Check that the input fim_dir exists
     assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
     ## Check that the bankfull flow filepath exists and read to dataframe
-    assert os.path.isfile(bankfull_flow_filepath), 'ERROR: Can not find the input bankfull flow file: ' + str(bankfull_flow_filepath)
-    
+    assert os.path.isfile(bankfull_flow_filepath), 'ERROR: Can not find the input bankfull flow file: ' + str(
+        bankfull_flow_filepath
+    )
+
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
     ## initiate log file
-    log_file = open(join(fim_dir,'logs','log_bankfull_indentify.log'),"w")
+    log_file = open(join(fim_dir, 'logs', 'log_bankfull_indentify.log'), "w")
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
 
-    ## List of columns in SRC_full_crosswalk to read in (ignores other columns that may have been added by previous post-proccessing runs)
-    src_usecols=['Stage','Number of Cells','SurfaceArea (m2)','BedArea (m2)','Volume (m3)','SLOPE','LENGTHKM','AREASQKM','ManningN','HydroID','NextDownID','order_','TopWidth (m)','WettedPerimeter (m)','WetArea (m2)','HydraulicRadius (m)','Discharge (m3s-1)','feature_id', 'Bathymetry_source']
-
-    df_bflows = pd.read_csv(bankfull_flow_filepath,dtype={'feature_id': int})
-    huc_list  = os.listdir(fim_dir)
-    huc_list.sort() # sort huc_list for helping track progress in future print statments
+    ## List of columns in SRC_full_crosswalk to read in
+    #       (ignores other columns that may have been added by previous post-proccessing runs)
+    src_usecols = [
+        'Stage',
+        'Number of Cells',
+        'SurfaceArea (m2)',
+        'BedArea (m2)',
+        'Volume (m3)',
+        'SLOPE',
+        'LENGTHKM',
+        'AREASQKM',
+        'ManningN',
+        'HydroID',
+        'NextDownID',
+        'order_',
+        'TopWidth (m)',
+        'WettedPerimeter (m)',
+        'WetArea (m2)',
+        'HydraulicRadius (m)',
+        'Discharge (m3s-1)',
+        'feature_id',
+        'Bathymetry_source'
+    ]
+
+    df_bflows = pd.read_csv(bankfull_flow_filepath, dtype={'feature_id': int})
+    huc_list = os.listdir(fim_dir)
+    huc_list.sort()  # sort huc_list for helping track progress in future print statments
     huc_pass_list = []
     for huc in huc_list:
-        #if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
-        if re.match('\d{8}', huc):    
-            huc_branches_dir = os.path.join(fim_dir, huc,'branches')
+        # if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+        if re.match(r'\d{8}', huc):
+            huc_branches_dir = os.path.join(fim_dir, huc, 'branches')
             for branch_id in os.listdir(huc_branches_dir):
-                branch_dir = os.path.join(huc_branches_dir,branch_id)
-                src_orig_full_filename = join(branch_dir,'src_full_crosswalked_' + branch_id + '.csv')
-                huc_output_dir = join(branch_dir,'src_plots')
-                ## check if BARC modified src_full_crosswalked_BARC.csv exists otherwise use the orginial src_full_crosswalked.csv
+                branch_dir = os.path.join(huc_branches_dir, branch_id)
+                src_orig_full_filename = join(branch_dir, 'src_full_crosswalked_' + branch_id + '.csv')
+                huc_output_dir = join(branch_dir, 'src_plots')
+                ## Check if BARC modified src_full_crosswalked_BARC.csv exists otherwise use
+                #   orginial src_full_crosswalked.csv
                 if isfile(src_orig_full_filename):
                     huc_pass_list.append(str(huc) + " --> src_full_crosswalked.csv")
-                    procs_list.append([src_orig_full_filename, src_usecols, df_bflows, huc, branch_id, src_plot_option, huc_output_dir])
+                    procs_list.append(
+                        [
+                            src_orig_full_filename,
+                            src_usecols,
+                            df_bflows,
+                            huc,
+                            branch_id,
+                            src_plot_option,
+                            huc_output_dir
+                        ]
+                    )
                 else:
-                    print('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + 'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
-                    log_file.write('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + 'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
-
-    log_file.writelines(["%s\n" % item  for item in huc_pass_list])
+                    print(
+                        f'HUC: {str(huc)}  branch id: {str(branch_id)}'
+                        'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: '
+                        f' {str(branch_dir)}  - skipping this branch!!!\n'
+                    )
+                    log_file.write(
+                        f'HUC: {str(huc)}  branch id: {str(branch_id)}'
+                        'WARNING --> can not find the SRC crosswalked csv file in the fim output dir: '
+                        f' {str(branch_dir)}  - skipping this branch!!!\n'
+                    )
+
+    log_file.writelines(["%s\n" % item for item in huc_pass_list])
     log_file.write('#########################################################\n\n')
 
     ## Pass huc procs_list to multiprocessing function
@@ -241,14 +387,44 @@ def run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_opti
     log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
     log_file.close()
 
-if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description='Identify bankfull stage for each hydroid synthetic rating curve')
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-flows','--bankfull-flow-input',help='NWM recurrence flows dir (flow units in CMS!!!)',required=True,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='OPTIONAL: number of workers (default=8)',required=False,default=8,type=int)
-    parser.add_argument('-vb','--verbose',help='OPTIONAL: verbose progress bar',required=False,default=None,action='store_true')
-    parser.add_argument('-plots','--src-plot-option',help='OPTIONAL flag: use this flag to create src plots for all hydroids (helpful for evaluating). WARNING - long runtime',default=False,required=False, action='store_true')
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Identify bankfull stage for each hydroid synthetic rating curve'
+    )
+    parser.add_argument('-fim_dir', '--fim-dir', help='FIM output dir', required=True, type=str)
+    parser.add_argument(
+        '-flows',
+        '--bankfull-flow-input',
+        help='NWM recurrence flows dir (flow units in CMS!!!)',
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-j',
+        '--number-of-jobs',
+        help='OPTIONAL: number of workers (default=8)',
+        required=False,
+        default=8,
+        type=int,
+    )
+    parser.add_argument(
+        '-vb',
+        '--verbose',
+        help='OPTIONAL: verbose progress bar',
+        required=False,
+        default=None,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-plots',
+        '--src-plot-option',
+        help='OPTIONAL flag: use this flag to create src plots for all hydroids (helpful for evaluating). '
+        'WARNING - long runtime',
+        default=False,
+        required=False,
+        action='store_true',
+    )
 
     args = vars(parser.parse_args())
 
@@ -257,6 +433,6 @@ def run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_opti
     number_of_jobs = args['number_of_jobs']
     verbose = bool(args['verbose'])
     src_plot_option = args['src_plot_option']
-    
+
     ## Prepare/check inputs, create log file, and spin up the proc list
-    run_prep(fim_dir,bankfull_flow_filepath,number_of_jobs,verbose,src_plot_option)
\ No newline at end of file
+    run_prep(fim_dir, bankfull_flow_filepath, number_of_jobs, verbose, src_plot_option)
diff --git a/src/make_rem.py b/src/make_rem.py
index b90753fee..78ea77ef2 100755
--- a/src/make_rem.py
+++ b/src/make_rem.py
@@ -1,117 +1,134 @@
 #!/usr/bin/env python3
 
-from numba import njit, typeof, typed, types
-import rasterio
-import numpy as np
 import argparse
 import os
-from osgeo import ogr, gdal
+
+import numpy as np
+import rasterio
+from numba import njit, typed, typeof, types
+from osgeo import gdal, ogr
 
 
 def rel_dem(dem_fileName, pixel_watersheds_fileName, rem_fileName, thalweg_raster):
     """
-        Calculates REM/HAND/Detrended DEM
-        
-        Parameters
-        ----------
-        dem_fileName : str
-            File name of pit filled DEM raster.
-        pixel_watersheds_fileName : str
-            File name of stream pixel watersheds raster.
-        rem_fileName : str
-            File name of output relative elevation raster.
+    Calculates REM/HAND/Detrended DEM
+
+    Parameters
+    ----------
+    dem_fileName : str
+        File name of pit filled DEM raster.
+    pixel_watersheds_fileName : str
+        File name of stream pixel watersheds raster.
+    rem_fileName : str
+        File name of output relative elevation raster.
 
     """
 
-    
-    # ------------------------------------------- Get catchment_min_dict --------------------------------------------------- #
-    # The following creates a dictionary of the catchment ids (key) and their elevation along the thalweg (value).
+    # --------------------------------- Get catchment_min_dict --------------------------------------------- #
+    # The following creates a dictionary of the catchment ids (key) and
+    # their elevation along the thalweg (value).
+
     @njit
     def make_catchment_min_dict(flat_dem, catchment_min_dict, flat_catchments, thalweg_window):
-  
-        for i,cm in enumerate(flat_catchments):
+        for i, cm in enumerate(flat_catchments):
             if thalweg_window[i] == 1:  # Only allow reference elevation to be within thalweg.
                 # If the catchment really exists in the dictionary, compare elevation values.
-                if (cm in catchment_min_dict):
-                    if (flat_dem[i] < catchment_min_dict[cm]):
-                        # If the flat_dem's elevation value is less than the catchment_min_dict min, update the catchment_min_dict min.
-                        catchment_min_dict[cm] = flat_dem[i]                                                
+                if cm in catchment_min_dict:
+                    if flat_dem[i] < catchment_min_dict[cm]:
+                        # If the flat_dem's elevation value is less than the catchment_min_dict min,
+                        # update the catchment_min_dict min.
+                        catchment_min_dict[cm] = flat_dem[i]
                 else:
-                    catchment_min_dict[cm] = flat_dem[i]                
-        return(catchment_min_dict)
+                    catchment_min_dict[cm] = flat_dem[i]
+        return catchment_min_dict
 
     # Open the masked gw_catchments_pixels_masked and dem_thalwegCond_masked.
     gw_catchments_pixels_masked_object = rasterio.open(pixel_watersheds_fileName)
     dem_thalwegCond_masked_object = rasterio.open(dem_fileName)
     thalweg_raster_object = rasterio.open(thalweg_raster)
-    
+
     # Specify raster object metadata.
     meta = dem_thalwegCond_masked_object.meta.copy()
     meta['tiled'], meta['compress'] = True, 'lzw'
-    
+
     # -- Create catchment_min_dict -- #
-    catchment_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums.
+    catchment_min_dict = typed.Dict.empty(
+        types.int32, types.float32
+    )  # Initialize an empty dictionary to store the catchment minimums.
     # Update catchment_min_dict with pixel sheds minimum.
-    for ji, window in dem_thalwegCond_masked_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template.
-        dem_window = dem_thalwegCond_masked_object.read(1,window=window).ravel()  # Define dem_window.
-        catchments_window = gw_catchments_pixels_masked_object.read(1,window=window).ravel()  # Define catchments_window.
+    for ji, window in dem_thalwegCond_masked_object.block_windows(
+        1
+    ):  # Iterate over windows, using dem_rasterio_object as template.
+        dem_window = dem_thalwegCond_masked_object.read(1, window=window).ravel()  # Define dem_window.
+        catchments_window = gw_catchments_pixels_masked_object.read(
+            1, window=window
+        ).ravel()  # Define catchments_window.
         thalweg_window = thalweg_raster_object.read(1, window=window).ravel()  # Define cost_window.
-        
+
         # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
-        catchment_min_dict = make_catchment_min_dict(dem_window, catchment_min_dict, catchments_window, thalweg_window)
+        catchment_min_dict = make_catchment_min_dict(
+            dem_window, catchment_min_dict, catchments_window, thalweg_window
+        )
 
     dem_thalwegCond_masked_object.close()
     gw_catchments_pixels_masked_object.close()
     thalweg_raster_object.close()
-    # ------------------------------------------------------------------------------------------------------------------------ #
-    
-    
-    # ------------------------------------------- Produce relative elevation model ------------------------------------------- #
-    @njit
-    def calculate_rem(flat_dem,catchmentMinDict,flat_catchments,ndv):
+    # ------------------------------------------------------------------------------------------------------ #
 
-        rem_window = np.zeros(len(flat_dem),dtype=np.float32)
-        for i,cm in enumerate(flat_catchments):
+    # --------------------------------- Produce relative elevation model ----------------------------------- #
+    @njit
+    def calculate_rem(flat_dem, catchmentMinDict, flat_catchments, ndv):
+        rem_window = np.zeros(len(flat_dem), dtype=np.float32)
+        for i, cm in enumerate(flat_catchments):
             if cm in catchmentMinDict:
                 if catchmentMinDict[cm] == ndv or flat_dem[i] == ndv:
                     rem_window[i] = ndv
                 else:
                     rem_window[i] = flat_dem[i] - catchmentMinDict[cm]
 
-        return(rem_window)
+        return rem_window
 
-    rem_rasterio_object = rasterio.open(rem_fileName,'w',**meta)  # Open rem_rasterio_object for writing to rem_fileName.
-    pixel_catchments_rasterio_object = rasterio.open(pixel_watersheds_fileName)  # Open pixel_catchments_rasterio_object
+    rem_rasterio_object = rasterio.open(
+        rem_fileName, 'w', **meta
+    )  # Open rem_rasterio_object for writing to rem_fileName.
+    pixel_catchments_rasterio_object = rasterio.open(
+        pixel_watersheds_fileName
+    )  # Open pixel_catchments_rasterio_object
     dem_rasterio_object = rasterio.open(dem_fileName)
-        
+
     for ji, window in dem_rasterio_object.block_windows(1):
-        dem_window = dem_rasterio_object.read(1,window=window)
+        dem_window = dem_rasterio_object.read(1, window=window)
         window_shape = dem_window.shape
 
         dem_window = dem_window.ravel()
-        catchments_window = pixel_catchments_rasterio_object.read(1,window=window).ravel()
-        
+        catchments_window = pixel_catchments_rasterio_object.read(1, window=window).ravel()
+
         rem_window = calculate_rem(dem_window, catchment_min_dict, catchments_window, meta['nodata'])
         rem_window = rem_window.reshape(window_shape).astype(np.float32)
-        
+
         rem_rasterio_object.write(rem_window, window=window, indexes=1)
-        
+
     dem_rasterio_object.close()
     pixel_catchments_rasterio_object.close()
     rem_rasterio_object.close()
-    # ------------------------------------------------------------------------------------------------------------------------ #
+    # ------------------------------------------------------------------------------------------------------ #
 
 
 if __name__ == '__main__':
-
     # parse arguments
     parser = argparse.ArgumentParser(description='Relative elevation from pixel based watersheds')
-    parser.add_argument('-d','--dem', help='DEM to use within project path', required=True)
-    parser.add_argument('-w','--watersheds',help='Pixel based watersheds raster to use within project path',required=True)
-    parser.add_argument('-t','--thalweg-raster',help='A binary raster representing the thalweg. 1 for thalweg, 0 for non-thalweg.',required=True)
-    parser.add_argument('-o','--rem',help='Output REM raster',required=True)
-    
-    
+    parser.add_argument('-d', '--dem', help='DEM to use within project path', required=True)
+    parser.add_argument(
+        '-w', '--watersheds', help='Pixel based watersheds raster to use within project path', required=True
+    )
+    parser.add_argument(
+        '-t',
+        '--thalweg-raster',
+        help='A binary raster representing the thalweg. 1 for thalweg, 0 for non-thalweg.',
+        required=True,
+    )
+    parser.add_argument('-o', '--rem', help='Output REM raster', required=True)
+
     # extract to dictionary
     args = vars(parser.parse_args())
 
diff --git a/src/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
index 677de1899..8c2e7e6da 100755
--- a/src/make_stages_and_catchlist.py
+++ b/src/make_stages_and_catchlist.py
@@ -1,23 +1,33 @@
 #!/usr/bin/env python3
 
 import argparse
+import sys
+
 import geopandas as gpd
 import numpy as np
-import sys
+
 from utils.shared_functions import mem_profile
 
 
 @mem_profile
-def make_stages_and_catchlist(flows_filename, catchments_filename, stages_filename, catchlist_filename, stages_min, stages_interval, stages_max):
+def make_stages_and_catchlist(
+    flows_filename,
+    catchments_filename,
+    stages_filename,
+    catchlist_filename,
+    stages_min,
+    stages_interval,
+    stages_max,
+):
     flows = gpd.read_file(flows_filename)
     catchments = gpd.read_file(catchments_filename)
 
     # Reconcile flows and catchments hydroids
-    flows      = flows.merge(catchments[['HydroID']], on='HydroID', how='inner')
+    flows = flows.merge(catchments[['HydroID']], on='HydroID', how='inner')
     catchments = catchments.merge(flows[['HydroID']], on='HydroID', how='inner')
 
     stages_max = stages_max + stages_interval
-    stages = np.round(np.arange(stages_min,stages_max,stages_interval),4)
+    stages = np.round(np.arange(stages_min, stages_max, stages_interval), 4)
 
     hydroIDs = flows['HydroID'].tolist()
     len_of_hydroIDs = len(hydroIDs)
@@ -27,29 +37,31 @@ def make_stages_and_catchlist(flows_filename, catchments_filename, stages_filena
     try:
         areasqkm = catchments['areasqkm'].tolist()
     except KeyError:
-        areasqkm = catchments['geometry'].area/ 10**6
-        
-    with open(stages_filename,'w') as f:
+        areasqkm = catchments['geometry'].area / 10**6
+
+    with open(stages_filename, 'w') as f:
         f.write("Stage\n")
         for stage in stages:
             f.write("{}\n".format(stage))
 
-    with open(catchlist_filename,'w') as f:
+    with open(catchlist_filename, 'w') as f:
         f.write("{}\n".format(len_of_hydroIDs))
-        for h,s,l,a in zip(hydroIDs,slopes,lengthkm,areasqkm):
-            f.write("{} {} {} {}\n".format(h,s,l,a))
+        for h, s, l, a in zip(hydroIDs, slopes, lengthkm, areasqkm):
+            f.write("{} {} {} {}\n".format(h, s, l, a))
+
+    # TODO we need a main block
+
 
-    #TODO we need a main block
 if __name__ == '__main__':
     # Parse arguments.
     parser = argparse.ArgumentParser(description='make_stages_and_catchlist.py')
-    parser.add_argument('-f', '--flows-filename', help='flows-filename',required=True)
-    parser.add_argument('-c', '--catchments-filename', help='catchments-filename',required=True)
-    parser.add_argument('-s', '--stages-filename', help='stages-filename',required=True)
-    parser.add_argument('-a', '--catchlist-filename', help='catchlist-filename',required=True)
-    parser.add_argument('-m', '--stages-min', help='stages-min',required=True,type=float)
-    parser.add_argument('-i', '--stages-interval', help='stages-interval',required=True,type=float)
-    parser.add_argument('-t', '--stages-max', help='stages-max',required=True,type=float)
+    parser.add_argument('-f', '--flows-filename', help='flows-filename', required=True)
+    parser.add_argument('-c', '--catchments-filename', help='catchments-filename', required=True)
+    parser.add_argument('-s', '--stages-filename', help='stages-filename', required=True)
+    parser.add_argument('-a', '--catchlist-filename', help='catchlist-filename', required=True)
+    parser.add_argument('-m', '--stages-min', help='stages-min', required=True, type=float)
+    parser.add_argument('-i', '--stages-interval', help='stages-interval', required=True, type=float)
+    parser.add_argument('-t', '--stages-max', help='stages-max', required=True, type=float)
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
diff --git a/src/mask_dem.py b/src/mask_dem.py
index 6f2f52c70..9f441c3b6 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -1,18 +1,31 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
-import pandas as pd
-import geopandas as gpd
+
 import fiona
+import geopandas as gpd
+import pandas as pd
 import rasterio as rio
 from rasterio.mask import mask
-import argparse
+
 from utils.shared_functions import mem_profile
 
+
 @mem_profile
-def mask_dem(dem_filename:str, nld_filename:str, levee_id_attribute:str, out_dem_filename:str, branch_id_attribute:str, branch_id:int, branch_zero_id:int, levee_levelpaths:str):
+def mask_dem(
+    dem_filename: str,
+    nld_filename: str,
+    levee_id_attribute: str,
+    out_dem_filename: str,
+    branch_id_attribute: str,
+    branch_id: int,
+    branch_zero_id: int,
+    levee_levelpaths: str,
+):
     """
-    Masks levee-protected areas from DEM in branch 0 or if the level path is associated with a levee (determined in src/associate_levelpaths_with_levees.py).
+    Masks levee-protected areas from DEM in branch 0 or if the level path is associated with a levee
+    (determined in src/associate_levelpaths_with_levees.py).
 
     Parameters
     ----------
@@ -33,9 +46,9 @@ def mask_dem(dem_filename:str, nld_filename:str, levee_id_attribute:str, out_dem
     levee_levelpaths: str
         Path to levee-levelpath association file.
     """
-    
+
     # Rasterize if branch zero
-    if (branch_id == branch_zero_id):
+    if branch_id == branch_zero_id:
         with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
             dem_profile = dem.profile.copy()
 
@@ -45,7 +58,7 @@ def mask_dem(dem_filename:str, nld_filename:str, levee_id_attribute:str, out_dem
             out_dem_masked, _ = mask(dem, geoms, invert=True)
 
             with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
-                dest.write(out_dem_masked[0,:,:], indexes=1)
+                dest.write(out_dem_masked[0, :, :], indexes=1)
 
     elif os.path.exists(levee_levelpaths):
         levee_levelpaths = pd.read_csv(levee_levelpaths)
@@ -55,33 +68,42 @@ def mask_dem(dem_filename:str, nld_filename:str, levee_id_attribute:str, out_dem
 
         # Get levee IDs
         levelpath_levees = list(levee_levelpaths[levee_id_attribute])
-        
+
         if len(levelpath_levees) > 0:
             with rio.open(dem_filename) as dem:
                 leveed = gpd.read_file(nld_filename)
                 dem_profile = dem.profile.copy()
 
                 # Get geometries of levee protected areas associated with levelpath
-                geoms = [feature['geometry'] for i, feature in leveed.iterrows() if feature[levee_id_attribute] in levelpath_levees]
+                geoms = [
+                    feature['geometry']
+                    for i, feature in leveed.iterrows()
+                    if feature[levee_id_attribute] in levelpath_levees
+                ]
 
                 # Mask out levee-protected areas from DEM
                 out_dem_masked, _ = mask(dem, geoms, invert=True)
 
                 with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
-                    dest.write(out_dem_masked[0,:,:], indexes=1)
-
+                    dest.write(out_dem_masked[0, :, :], indexes=1)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Mask levee-protected areas from DEM')
-    parser.add_argument('-dem','--dem-filename', help='DEM filename', required=True, type=str)
-    parser.add_argument('-nld','--nld-filename', help='NLD levee-protected areas filename', required=True, type=str)
-    parser.add_argument('-l','--levee-id-attribute', help='Levee ID attribute name', required=True,type=str)
-    parser.add_argument('-out','--out-dem-filename', help='out DEM filename', required=True, type=str)
-    parser.add_argument('-b', '--branch-id-attribute', help='Branch ID attribute name', required=True, type=str)
+    parser.add_argument('-dem', '--dem-filename', help='DEM filename', required=True, type=str)
+    parser.add_argument(
+        '-nld', '--nld-filename', help='NLD levee-protected areas filename', required=True, type=str
+    )
+    parser.add_argument('-l', '--levee-id-attribute', help='Levee ID attribute name', required=True, type=str)
+    parser.add_argument('-out', '--out-dem-filename', help='out DEM filename', required=True, type=str)
+    parser.add_argument(
+        '-b', '--branch-id-attribute', help='Branch ID attribute name', required=True, type=str
+    )
     parser.add_argument('-i', '--branch-id', help='Branch ID', type=int, required='True')
     parser.add_argument('-b0', '--branch-zero-id', help='Branch zero ID', type=int, required=False, default=0)
-    parser.add_argument('-csv', '--levee-levelpaths', help='Levee - levelpath layer filename', type=str, required=True)
+    parser.add_argument(
+        '-csv', '--levee-levelpaths', help='Levee - levelpath layer filename', type=str, required=True
+    )
 
     args = vars(parser.parse_args())
 
diff --git a/src/outputs_cleanup.py b/src/outputs_cleanup.py
index 48d7ba8ea..c791e2bd1 100755
--- a/src/outputs_cleanup.py
+++ b/src/outputs_cleanup.py
@@ -2,18 +2,18 @@
 
 import argparse
 import os
-
 from pathlib import Path
+
 from utils.shared_functions import FIM_Helpers as fh
 
-def remove_deny_list_files(src_dir, deny_list, branch_id, verbose=False):
 
+def remove_deny_list_files(src_dir, deny_list, branch_id, verbose=False):
     '''
     Overview
     ----------
     Delete a set of files in a given directory (and/or subdirectories) based
     on values in the deny list.
-        
+
     Notes:
         - Strange, but.. if you want to skip deleting file, have the value for
           the 'deny_list' param to be the value of "none" (this is a by product
@@ -27,36 +27,36 @@ def remove_deny_list_files(src_dir, deny_list, branch_id, verbose=False):
 
     Parameters
     ----------
-    
+
     - src_dir : str
         Folder path where the files are to be deleted (recursive).
         Will error if does not exist.
-            
+
     - deny_list : str
         If not the value of "none" (any case), the file must exist and it contains
         the list of files to be deleted. Will error if does not exist.
-            
+
     - branch_id : str
-        Needs to have a value and will be subsituted into any {} value. 
+        Needs to have a value and will be subsituted into any {} value.
         Will error if does not exist.
-        
+
     Returns
     ----------
         None
     '''
-    
+
     # Yes.. this is a little strange.
     # if the user submitts the deny list name of "none" (any case)
     # we skip this
-    if (deny_list.upper() == 'NONE'): 
+    if deny_list.upper() == 'NONE':
         print("file clean via the deny list skipped")
         return
 
-    if (not os.path.isdir(src_dir)):
+    if not os.path.isdir(src_dir):
         raise ValueError(f"Sorry, the directory {src_dir} does not exist")
 
-    if (branch_id.strip() == ""):
-        raise ValueError(f"Sorry, branch id value must exist")
+    if branch_id.strip() == "":
+        raise ValueError("Sorry, branch id value must exist")
 
     # Note: some of the deny_file_names might be a comment line
     # this will validate file exists
@@ -68,46 +68,43 @@ def remove_deny_list_files(src_dir, deny_list, branch_id, verbose=False):
     file_removed_count = 0
 
     for deny_file_name in deny_file_names:
-    
         # Only keep lines that do no start with a #
         # aka.. we are only deleting files that do not start a line with #
         deny_file_name = deny_file_name.strip()
-        if (deny_file_name.startswith("#")): 
-            continue 
-        
+        if deny_file_name.startswith("#"):
+            continue
+
         # the file name may / may not have a {} in it . If it does
         # has a {} it in, we will replace it with the branch ID.
         # if the file name does not have a {} in it, that file
         # will be deleted.
         # We will search all directories recursively.
         deny_file_name = deny_file_name.replace("{}", branch_id)
-        
+
         found_files = Path(src_dir).rglob(f"{deny_file_name}")
-        
+
         for found_file in found_files:
             fh.vprint(f"found file: {found_file}", verbose, False)
 
             if os.path.exists(found_file):
                 os.remove(found_file)
-            
-            file_removed_count +=1
+
+            file_removed_count += 1
 
     fh.vprint(f"Removed {file_removed_count} files", verbose, True)
 
 
 if __name__ == '__main__':
-
     # parse arguments
     parser = argparse.ArgumentParser(description='Clean up outputs given file with line delimineted files')
-    parser.add_argument('-d','--src_dir', help='Directory to find files', required=True)
-    parser.add_argument('-l','--deny_list', help='Path to deny list file. Must be line delimited',
-                        required=True)
-    parser.add_argument('-b','--branch_id', help='Branch id value', required=True)
-    parser.add_argument('-v','--verbose', help='Verbose', required=False, 
-                        default=False, action='store_true')
+    parser.add_argument('-d', '--src_dir', help='Directory to find files', required=True)
+    parser.add_argument(
+        '-l', '--deny_list', help='Path to deny list file. Must be line delimited', required=True
+    )
+    parser.add_argument('-b', '--branch_id', help='Branch id value', required=True)
+    parser.add_argument('-v', '--verbose', help='Verbose', required=False, default=False, action='store_true')
 
     # extract to dictionary
     args = vars(parser.parse_args())
 
     remove_deny_list_files(**args)
-
diff --git a/src/process_branch.sh b/src/process_branch.sh
index 3cf0365de..edfa2aa8e 100755
--- a/src/process_branch.sh
+++ b/src/process_branch.sh
@@ -1,13 +1,13 @@
 #!/bin/bash -e
 
-# it is strongly recommended that you do not call directly to src/run_by_branch.sh
-# but call this file and let is call run_by_branch.
-# This file will auto trap any exceptions from run_by_branch.
+# It is strongly recommended that you do not call src/run_by_branch.sh directly.
+# Call this file instead, and let it call run_by_branch.sh.
+# This file will trap any exceptions from run_by_branch.sh.
 
-# also.. remember.. that this file will rarely need to be called (but can be)
+# Also.. remember.. that this file can be called explicitly, but will rarely need to be,
 # as it is usually called through a parallelizing iterator in run_unit_wb.sh
 
-# this also has no named command line arguments, only positional args.
+# This file also has no named command line arguments, only positional args.
 
 runName=$1
 hucNumber=$2
@@ -17,7 +17,7 @@ branchId=$3
 branchLogFileName=$outputDestDir/logs/branch/"$hucNumber"_branch_"$branchId".log
 branch_list_csv_file=$tempHucDataDir/branch_ids.csv
 
-/usr/bin/time -v $srcDir/run_by_branch.sh $hucNumber $branchId 2>&1 | tee $branchLogFileName 
+/usr/bin/time -v $srcDir/run_by_branch.sh $hucNumber $branchId 2>&1 | tee $branchLogFileName
 
 #exit ${PIPESTATUS[0]}
 return_codes=( "${PIPESTATUS[@]}" )
@@ -28,7 +28,7 @@ err_exists=0
 for code in "${return_codes[@]}"
 do
     # Make an extra copy of the branch log in a new folder
-    # Note: It was tricky to load in the fim_enum into bash, so we will just 
+    # Note: It was tricky to load in the fim_enum into bash, so we will just
     # go with the code for now
     if [ $code -eq 0 ]; then
         echo
@@ -54,4 +54,4 @@ if [ "$err_exists" = "0" ]; then
 fi
 
 # We always return a success at this point (so we don't stop the loops / iterator)
-exit 0  
+exit 0
diff --git a/src/query_vectors_by_branch_polygons.py b/src/query_vectors_by_branch_polygons.py
index 27ca1a00e..e0cb7d6fd 100755
--- a/src/query_vectors_by_branch_polygons.py
+++ b/src/query_vectors_by_branch_polygons.py
@@ -1,41 +1,74 @@
 #!/usr/bin/env python3
 
 
-from stream_branches import StreamBranchPolygons
 import argparse
+
 from tqdm import tqdm
 
+from stream_branches import StreamBranchPolygons
+
 
 if __name__ == '__main__':
-
     # parse arguments
     parser = argparse.ArgumentParser(description='Query vectors by unique attribute values')
-    parser.add_argument('-a','--vector-attributes', help='Vector with unique attributs', required=True,default=None)
-    parser.add_argument('-d','--branch-ids', help='Branch ID value', required=True,default=None, nargs='+')
-    parser.add_argument('-i','--attribute-id', help='Attribute Name', required=True,default=None)
-    parser.add_argument('-s','--subset-vectors', help='Vector file names to query by attribute', required=False,default=None,nargs="+")
-    parser.add_argument('-o','--out-files', help='Vector filenames to write to after query', required=False,default=None,nargs="+")
-    parser.add_argument('-v','--verbose', help='Verbose printing', required=False,default=None,action='store_true')
+    parser.add_argument(
+        '-a', '--vector-attributes', help='Vector with unique attributs', required=True, default=None
+    )
+    parser.add_argument('-d', '--branch-ids', help='Branch ID value', required=True, default=None, nargs='+')
+    parser.add_argument('-i', '--attribute-id', help='Attribute Name', required=True, default=None)
+    parser.add_argument(
+        '-s',
+        '--subset-vectors',
+        help='Vector file names to query by attribute',
+        required=False,
+        default=None,
+        nargs="+",
+    )
+    parser.add_argument(
+        '-o',
+        '--out-files',
+        help='Vector filenames to write to after query',
+        required=False,
+        default=None,
+        nargs="+",
+    )
+    parser.add_argument(
+        '-v', '--verbose', help='Verbose printing', required=False, default=None, action='store_true'
+    )
 
     # extract to dictionary
     args = vars(parser.parse_args())
 
-    attributes_vector_file, branch_ids ,attribute, subset_vectors, out_files, verbose = args["vector_attributes"], args["branch_ids"],args["attribute_id"], args["subset_vectors"], args["out_files"], args["verbose"]
-    
+    attributes_vector_file, branch_ids, attribute, subset_vectors, out_files, verbose = (
+        args["vector_attributes"],
+        args["branch_ids"],
+        args["attribute_id"],
+        args["subset_vectors"],
+        args["out_files"],
+        args["verbose"],
+    )
+
     # load file
-    #stream_polys = StreamBranchPolygons.from_file( filename=attributes_vector_file, 
-    #                                               branch_id_attribute=attribute,
-    #                                               values_excluded=None,attribute_excluded=None, verbose = verbose)
-    
-    for subset_vector,out_file in tqdm(zip(subset_vectors,out_files),disable=(not verbose),
-                                       total=len(subset_vectors),
-                                       desc="Query vectors"):
-
-        #if verbose:
-            #print("Query \'{}\' by attribute in \'{}\' ...".format(out_file.split('/')[-1].split('.')[0],
-            #                                                   attributes_vector_file.split('/')[-1].split('.')[0]))
-        StreamBranchPolygons.query_vectors_by_branch(subset_vector,
-                                                     branch_ids=branch_ids,
-                                                     branch_id_attribute=attribute,
-                                                     out_filename_template=out_file)
+    # stream_polys = StreamBranchPolygons.from_file(
+    #     filename=attributes_vector_file,
+    #     branch_id_attribute=attribute,
+    #     values_excluded=None,
+    #     attribute_excluded=None,
+    #     verbose=verbose,
+    # )
 
+    for subset_vector, out_file in tqdm(
+        zip(subset_vectors, out_files), disable=(not verbose), total=len(subset_vectors), desc="Query vectors"
+    ):
+        # if verbose:
+        # print(
+        #     "Query \'{}\' by attribute in \'{}\' ...".format(
+        #         out_file.split('/')[-1].split('.')[0], attributes_vector_file.split('/')[-1].split('.')[0]
+        #     )
+        # )
+        StreamBranchPolygons.query_vectors_by_branch(
+            subset_vector,
+            branch_ids=branch_ids,
+            branch_id_attribute=attribute,
+            out_filename_template=out_file,
+        )
diff --git a/src/reachID_grid_to_vector_points.py b/src/reachID_grid_to_vector_points.py
index 849256953..84ac2eab3 100755
--- a/src/reachID_grid_to_vector_points.py
+++ b/src/reachID_grid_to_vector_points.py
@@ -1,27 +1,23 @@
 #!/usr/bin/env python3
 
-import numpy as np
 import argparse
+
 import geopandas as gpd
-from utils.shared_variables import PREP_PROJECTION
-from shapely.geometry import Point
+import numpy as np
 import rasterio
-from utils.shared_functions import getDriver, mem_profile
+from shapely.geometry import Point
 
-"""
-USAGE:
-./reachID_grid_to_vector_points.py <flows_grid_IDs raster file> <flows_points vector file> <reachID or featureID>
+from utils.shared_functions import getDriver, mem_profile
+from utils.shared_variables import PREP_PROJECTION
 
-"""
 
 @mem_profile
-def convert_grid_cells_to_points(raster,index_option,output_points_filename=False):
-
+def convert_grid_cells_to_points(raster, index_option, output_points_filename=False):
     # Input raster
-    if isinstance(raster,str):
-        raster = rasterio.open(raster,'r')
+    if isinstance(raster, str):
+        raster = rasterio.open(raster, 'r')
 
-    elif isinstance(raster,rasterio.io.DatasetReader):
+    elif isinstance(raster, rasterio.io.DatasetReader):
         pass
 
     else:
@@ -30,36 +26,60 @@ def convert_grid_cells_to_points(raster,index_option,output_points_filename=Fals
     (upper_left_x, x_size, x_rotation, upper_left_y, y_rotation, y_size) = raster.get_transform()
     indices = np.nonzero(raster.read(1) >= 1)
 
-    id =[None] * len(indices[0]);points = [None]*len(indices[0])
+    id = [None] * len(indices[0])
+    points = [None] * len(indices[0])
 
     # Iterate over the Numpy points..
     i = 1
-    for y_index,x_index in zip(*indices):
-        x = x_index * x_size + upper_left_x + (x_size / 2) # add half the cell size
-        y = y_index * y_size + upper_left_y + (y_size / 2) # to center the point
-        points[i-1] = Point(x,y)
+    for y_index, x_index in zip(*indices):
+        x = x_index * x_size + upper_left_x + (x_size / 2)  # add half the cell size
+        y = y_index * y_size + upper_left_y + (y_size / 2)  # to center the point
+        points[i - 1] = Point(x, y)
         if index_option == 'reachID':
-            reachID = np.array(list(raster.sample((Point(x,y).coords), indexes=1))).item() # check this; needs to add raster cell value + index
-            id[i-1] = reachID*10000 + i #reachID + i/100
-        elif (index_option == 'featureID') |(index_option == 'pixelID'):
-            id[i-1] = i
+            reachID = np.array(
+                list(raster.sample((Point(x, y).coords), indexes=1))
+            ).item()  # check this; needs to add raster cell value + index
+            id[i - 1] = reachID * 10000 + i  # reachID + i/100
+        elif (index_option == 'featureID') | (index_option == 'pixelID'):
+            id[i - 1] = i
         i += 1
 
-    pointGDF = gpd.GeoDataFrame({'id' : id, 'geometry' : points},crs=PREP_PROJECTION,geometry='geometry')
+    pointGDF = gpd.GeoDataFrame({'id': id, 'geometry': points}, crs=PREP_PROJECTION, geometry='geometry')
 
-    if output_points_filename == False:
+    if output_points_filename is False:
         return pointGDF
     else:
-        pointGDF.to_file(output_points_filename,driver=getDriver(output_points_filename),index=False)
+        pointGDF.to_file(output_points_filename, driver=getDriver(output_points_filename), index=False)
 
 
 if __name__ == '__main__':
-
     # Parse arguments
+    """
+    USAGE:
+        reachID_grid_to_vector_points.py
+            -r <flows_grid_IDs raster file>
+            -i <reachID or featureID or pixelID>
+            -p <output points filename>
+    """
+
     parser = argparse.ArgumentParser(description='Converts a raster to points')
-    parser.add_argument('-r','--raster',help='Raster to be converted to points',required=True,type=str)
-    parser.add_argument('-i', '--index-option',help='Indexing option',required=True,type=str,choices=['reachID','featureID','pixelID'])
-    parser.add_argument('-p', '--output-points-filename',help='Output points layer filename',required=False,type=str,default=False)
+    parser.add_argument('-r', '--raster', help='Raster to be converted to points', required=True, type=str)
+    parser.add_argument(
+        '-i',
+        '--index-option',
+        help='Indexing option',
+        required=True,
+        type=str,
+        choices=['reachID', 'featureID', 'pixelID'],
+    )
+    parser.add_argument(
+        '-p',
+        '--output-points-filename',
+        help='Output points layer filename',
+        required=False,
+        type=str,
+        default=False,
+    )
 
     args = vars(parser.parse_args())
 
@@ -67,4 +87,4 @@ def convert_grid_cells_to_points(raster,index_option,output_points_filename=Fals
     index_option = args['index_option']
     output_points_filename = args['output_points_filename']
 
-    convert_grid_cells_to_points(raster,index_option,output_points_filename)
+    convert_grid_cells_to_points(raster, index_option, output_points_filename)
diff --git a/src/reset_mannings.py b/src/reset_mannings.py
index 06990f412..ce43b7544 100755
--- a/src/reset_mannings.py
+++ b/src/reset_mannings.py
@@ -1,209 +1,260 @@
 #!/usr/bin/env python3
 
-import pandas as pd
-import numpy as np
 import argparse
 import os
-from glob import iglob,glob
-from stream_branches import StreamNetwork
+from glob import glob, iglob
 
-def Reset_mannings(hydrofabric_dir,mannings_value,overwrite_files=False):
+import numpy as np
+import pandas as pd
+
+from stream_branches import StreamNetwork
 
 
-    src_table_filePaths, hydro_table_filePaths, stream_network_filePaths = make_file_paths_for_inputs(hydrofabric_dir)
+def Reset_mannings(hydrofabric_dir, mannings_value, overwrite_files=False):
+    (src_table_filePaths, hydro_table_filePaths, stream_network_filePaths) = make_file_paths_for_inputs(
+        hydrofabric_dir
+    )
 
     single_stream_network = len(stream_network_filePaths) == 1
 
     if single_stream_network:
         stream_network = StreamNetwork.from_file(stream_network_filePaths[0])
-    
-    
-    for i,(srcFP,hydFP) in enumerate(zip(src_table_filePaths,hydro_table_filePaths)):
 
+    for i, (srcFP, hydFP) in enumerate(zip(src_table_filePaths, hydro_table_filePaths)):
         src_table = load_src_table(srcFP)
         hydro_table = load_hydro_table(hydFP)
-        
-        src_table, hydro_table = reset_mannings_for_a_processing_unit(src_table,hydro_table,mannings_value)
-        
+
+        src_table, hydro_table = reset_mannings_for_a_processing_unit(src_table, hydro_table, mannings_value)
+
         if not single_stream_network:
-            stream_network = StreamNetwork.from_file( stream_network_filePaths[i] )
+            stream_network = StreamNetwork.from_file(stream_network_filePaths[i])
 
-        small_segments = identify_small_reaches(stream_network,min_catchment_area=None,min_stream_length=None)
+        small_segments = identify_small_reaches(
+            stream_network, min_catchment_area=None, min_stream_length=None
+        )
         src_table, hydro_table = replace_discharges_of_small_segments(small_segments, src_table, hydro_table)
 
         if overwrite_files:
-            src_table.to_csv(srcFP,index=False)
-            hydro_table.to_csv(hydFP,index=False)
+            src_table.to_csv(srcFP, index=False)
+            hydro_table.to_csv(hydFP, index=False)
 
-        #yield(src_table, hydro_table)
+        # yield(src_table, hydro_table)
 
 
 def load_hydro_table(hydro_table_filePath):
+    hydro_table = pd.read_csv(
+        hydro_table_filePath,
+        dtype={
+            'HydroID': str,
+            'feature_id': str,
+            'stage': float,
+            'discharge_cms': float,
+            'HUC': str,
+            'LakeID': str,
+        },
+    )
 
-    hydro_table = pd.read_csv( hydro_table_filePath, 
-                               dtype= { 'HydroID' : str,
-                                        'feature_id' : str,
-                                        'stage' : float,
-                                        'discharge_cms': float,
-                                        'HUC' : str,
-                                        'LakeID' : str 
-                                       }
-                             )
-
-    return(hydro_table)
+    return hydro_table
 
 
 def load_src_table(src_table_filePath):
+    src_table = pd.read_csv(
+        src_table_filePath,
+        dtype={
+            'HydroID': str,
+            'feature_id': str,
+            'stage': float,
+            'discharge_cms': float,
+            'HUC': str,
+            'LakeID': str,
+        },
+    )
 
-    src_table = pd.read_csv( src_table_filePath, 
-                             dtype= { 'HydroID' : str,
-                                      'feature_id' : str,
-                                      'stage' : float,
-                                      'discharge_cms': float,
-                                      'HUC' : str,
-                                      'LakeID' : str 
-                                     }
-                           )
-
-    return(src_table)
+    return src_table
 
 
 def make_file_paths_for_inputs(hydrofabric_dir):
+    src_table_filePath_to_glob = os.path.join(hydrofabric_dir, '**', 'src_full_crosswalked*.csv')
+    hydro_table_filePath_to_glob = os.path.join(hydrofabric_dir, '**', 'hydroTable*.csv')
+    stream_network_filePath_to_glob = os.path.join(
+        hydrofabric_dir, '**', 'demDerived_reaches_split_filtered_addedAttributes_crosswalked*.gpkg'
+    )
 
-    src_table_filePath_to_glob = os.path.join(hydrofabric_dir,'**','src_full_crosswalked*.csv')
-    hydro_table_filePath_to_glob = os.path.join(hydrofabric_dir,'**','hydroTable*.csv')
-    stream_network_filePath_to_glob = os.path.join(hydrofabric_dir,'**','demDerived_reaches_split_filtered_addedAttributes_crosswalked*.gpkg')
-
-    src_table_filePaths = iglob(src_table_filePath_to_glob,recursive=True)
-    hydro_table_filePaths = iglob(hydro_table_filePath_to_glob,recursive=True)
-    stream_network_filePaths = glob(stream_network_filePath_to_glob,recursive=True)
-
+    src_table_filePaths = iglob(src_table_filePath_to_glob, recursive=True)
+    hydro_table_filePaths = iglob(hydro_table_filePath_to_glob, recursive=True)
+    stream_network_filePaths = glob(stream_network_filePath_to_glob, recursive=True)
 
-    return(src_table_filePaths,hydro_table_filePaths,stream_network_filePaths)
+    return (src_table_filePaths, hydro_table_filePaths, stream_network_filePaths)
 
 
-def reset_mannings_for_a_processing_unit(src_table,hydro_table,mannings_value):
-
-    src_table = override_mannings(src_table,mannings_value)
+def reset_mannings_for_a_processing_unit(src_table, hydro_table, mannings_value):
+    src_table = override_mannings(src_table, mannings_value)
 
     src_table = calculate_discharge(src_table)
 
     hydro_table["discharge_cms"] = src_table["Discharge (m3s-1)"]
 
-    return(src_table,hydro_table)
-
+    return (src_table, hydro_table)
 
-def override_mannings(table,mannings_value,mannings_attribute="ManningN"):
 
+def override_mannings(table, mannings_value, mannings_attribute="ManningN"):
     table[mannings_attribute] = mannings_value
 
-    return(table)
+    return table
 
 
 def calculate_discharge(src_table):
+    src_table['Discharge (m3s-1)'] = (
+        src_table['WetArea (m2)']
+        * pow(src_table['HydraulicRadius (m)'], 2.0 / 3)
+        * pow(src_table['SLOPE'], 0.5)
+        / src_table['ManningN']
+    )
 
-    src_table['Discharge (m3s-1)'] = src_table['WetArea (m2)']* \
-                                     pow(src_table['HydraulicRadius (m)'],2.0/3)* \
-                                     pow(src_table['SLOPE'],0.5)/src_table['ManningN']
-    
     # set zero stage values to zero discharge
-    src_table.loc[src_table['Stage']==0,['Discharge (m3s-1)']] = 0
-
-    
-    return(src_table)
+    src_table.loc[src_table['Stage'] == 0, ['Discharge (m3s-1)']] = 0
 
+    return src_table
 
-def identify_small_reaches(stream_network,min_catchment_area=None,min_stream_length=None):
 
+def identify_small_reaches(stream_network, min_catchment_area=None, min_stream_length=None):
     # Adjust short model reach rating curves
     sml_segs = pd.DataFrame()
 
     if min_catchment_area is None:
-        min_catchment_area = float(os.environ['min_catchment_area']) #0.25#
+        min_catchment_area = float(os.environ['min_catchment_area'])  # 0.25#
 
     if min_stream_length is None:
-        min_stream_length = float(os.environ['min_stream_length']) #0.5#
+        min_stream_length = float(os.environ['min_stream_length'])  # 0.5#
 
     # replace small segment geometry with neighboring stream
     for stream_index in stream_network.index:
-
-        if stream_network["areasqkm"][stream_index] < min_catchment_area and stream_network["LengthKm"][stream_index] < min_stream_length and stream_network["LakeID"][stream_index] < 0:
-
+        if (
+            stream_network["areasqkm"][stream_index] < min_catchment_area
+            and stream_network["LengthKm"][stream_index] < min_stream_length
+            and stream_network["LakeID"][stream_index] < 0
+        ):
             short_id = stream_network['HydroID'][stream_index]
             to_node = stream_network['To_Node'][stream_index]
             from_node = stream_network['From_Node'][stream_index]
 
             # multiple upstream segments
             if len(stream_network.loc[stream_network['NextDownID'] == short_id]['HydroID']) > 1:
-                max_order = max(stream_network.loc[stream_network['NextDownID'] == short_id]['order_']) # drainage area would be better than stream order but we would need to calculate
-
-                if len(stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID']) == 1:
-                    update_id = stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID'].item()
-
+                max_order = max(
+                    stream_network.loc[stream_network['NextDownID'] == short_id]['order_']
+                )  # drainage area would be better than stream order but we would need to calculate
+
+                if (
+                    len(
+                        stream_network.loc[
+                            (stream_network['NextDownID'] == short_id)
+                            & (stream_network['order_'] == max_order)
+                        ]['HydroID']
+                    )
+                    == 1
+                ):
+                    update_id = stream_network.loc[
+                        (stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)
+                    ]['HydroID'].item()
+
+                # get the first one (same stream order, without drainage area info
+                # it is hard to know which is the main channel)
                 else:
-                    update_id = stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+                    update_id = stream_network.loc[
+                        (stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)
+                    ]['HydroID'].values[0]
 
             # single upstream segments
             elif len(stream_network.loc[stream_network['NextDownID'] == short_id]['HydroID']) == 1:
-                update_id = stream_network.loc[stream_network.To_Node==from_node]['HydroID'].item()
+                update_id = stream_network.loc[stream_network.To_Node == from_node]['HydroID'].item()
 
             # no upstream segments; multiple downstream segments
-            elif len(stream_network.loc[stream_network.From_Node==to_node]['HydroID']) > 1:
-                max_order = max(stream_network.loc[stream_network.From_Node==to_node]['HydroID']['order_']) # drainage area would be better than stream order but we would need to calculate
-
-                if len(stream_network.loc[(stream_network['NextDownID'] == short_id) & (stream_network['order_'] == max_order)]['HydroID']) == 1:
-                    update_id = stream_network.loc[(stream_network.From_Node==to_node) & (stream_network['order_'] == max_order)]['HydroID'].item()
-
+            elif len(stream_network.loc[stream_network.From_Node == to_node]['HydroID']) > 1:
+                max_order = max(
+                    stream_network.loc[stream_network.From_Node == to_node]['HydroID']['order_']
+                )  # drainage area would be better than stream order but we would need to calculate
+
+                if (
+                    len(
+                        stream_network.loc[
+                            (stream_network['NextDownID'] == short_id)
+                            & (stream_network['order_'] == max_order)
+                        ]['HydroID']
+                    )
+                    == 1
+                ):
+                    update_id = stream_network.loc[
+                        (stream_network.From_Node == to_node) & (stream_network['order_'] == max_order)
+                    ]['HydroID'].item()
+
+                # get the first one (same stream order, without drainage area info it is hard to know
+                # which is the main channel)
                 else:
-                    update_id = stream_network.loc[(stream_network.From_Node==to_node) & (stream_network['order_'] == max_order)]['HydroID'].values[0] # get the first one (same stream order, without drainage area info it is hard to know which is the main channel)
+                    update_id = stream_network.loc[
+                        (stream_network.From_Node == to_node) & (stream_network['order_'] == max_order)
+                    ]['HydroID'].values[0]
 
             # no upstream segments; single downstream segment
-            elif len(stream_network.loc[stream_network.From_Node==to_node]['HydroID']) == 1:
-                    update_id = stream_network.loc[stream_network.From_Node==to_node]['HydroID'].item()
+            elif len(stream_network.loc[stream_network.From_Node == to_node]['HydroID']) == 1:
+                update_id = stream_network.loc[stream_network.From_Node == to_node]['HydroID'].item()
 
             else:
-                update_id = stream_network.loc[stream_network.HydroID==short_id]['HydroID'].item()
+                update_id = stream_network.loc[stream_network.HydroID == short_id]['HydroID'].item()
 
-            str_order = stream_network.loc[stream_network.HydroID==short_id]['order_'].item()
-            sml_segs = pd.concat([sml_segs, {'short_id':short_id, 'update_id':update_id, 'str_order':str_order}], ignore_index=True)
+            str_order = stream_network.loc[stream_network.HydroID == short_id]['order_'].item()
+            sml_segs = pd.concat(
+                [sml_segs, {'short_id': short_id, 'update_id': update_id, 'str_order': str_order}],
+                ignore_index=True,
+            )
 
-    #print("Number of short reaches [{} < {} and {} < {}] = {}".format("areasqkm", min_catchment_area, "LengthKm", min_stream_length, len(sml_segs)))
-    
-    return(sml_segs)
+    # print(
+    #     "Number of short reaches [{} < {} and {} < {}] = {}".format(
+    #         "areasqkm", min_catchment_area, "LengthKm", min_stream_length, len(sml_segs)
+    #     )
+    # )
+
+    return sml_segs
 
 
 def replace_discharges_of_small_segments(sml_segs, src_table, hydro_table):
-    
     # update rating curves
     if len(sml_segs) == 0:
-        return(src_table,hydro_table)
+        return (src_table, hydro_table)
 
-    #sml_segs.to_csv(small_segments_filename,index=False)
-    #print("Update rating curves for short reaches.")
+    # sml_segs.to_csv(small_segments_filename,index=False)
+    # print("Update rating curves for short reaches.")
 
     for index, segment in sml_segs.iterrows():
-
         short_id = segment[0]
-        update_id= segment[1]
+        update_id = segment[1]
         new_values = src_table.loc[src_table['HydroID'] == update_id][['Stage', 'Discharge (m3s-1)']]
 
         for src_index, src_stage in new_values.iterrows():
-            src_table.loc[(src_table['HydroID']== short_id) & (src_table['Stage']== src_stage[0]),['Discharge (m3s-1)']] = src_stage[1]
+            src_table.loc[
+                (src_table['HydroID'] == short_id) & (src_table['Stage'] == src_stage[0]),
+                ['Discharge (m3s-1)'],
+            ] = src_stage[1]
 
     hydro_table["discharge_cms"] = src_table["Discharge (m3s-1)"]
 
-    return(src_table,hydro_table)
-
+    return (src_table, hydro_table)
 
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Overwrites mannings n values and recomputes discharge values for SRCs and Hydro-Tables'
+    )
+    parser.add_argument('-y', '--hydrofabric-dir', help='Hydrofabric directory', required=True)
+    parser.add_argument('-n', '--mannings-value', help='Mannings N value to use', required=True, type=float)
+    parser.add_argument(
+        '-o',
+        '--overwrite-files',
+        help='Overwrites original files if used',
+        required=False,
+        default=False,
+        action='store_true',
+    )
 
-    parser = argparse.ArgumentParser(description='Overwrites mannings n values and recomputes discharge values for SRCs and Hydro-Tables')
-    parser.add_argument('-y','--hydrofabric-dir', help='Hydrofabric directory', required=True)
-    parser.add_argument('-n','--mannings-value', help='Mannings N value to use', required=True, type=float)
-    parser.add_argument('-o','--overwrite-files', help='Overwrites original files if used', required=False, default=False,action='store_true')
-    
-    
     args = vars(parser.parse_args())
 
     Reset_mannings(**args)
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index b7ad2f844..c158e3522 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -37,41 +37,62 @@ echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"
 date -u
 Tstart
 echo -e "Querying NWM streams ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+    $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg \
+    $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg
 echo -e "Querying NWM catchments ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+    $tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg \
+    $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
 echo -e "Querying NWM Dissolved Levelpaths headwaters ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+    $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg \
+    $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
 #echo -e "Querying NWM headwaters ..."
-# ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" $tempCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg $tempHucDataDir/nwm_headwaters.gpkg
+# ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+#     $tempCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg \
+#      $tempHucDataDir/nwm_headwaters.gpkg
 Tcount
 
 ## GET RASTERS FROM ROOT HUC DIRECTORY AND CLIP TO CURRENT BRANCH BUFFER ##
 echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"
 date -u
 Tstart
-$srcDir/clip_rasters_to_branches.py -d $current_branch_id -b $tempHucDataDir/branch_polygons.gpkg -i $branch_id_attribute -r $tempHucDataDir/dem_meters.tif $tempHucDataDir/flowdir_d8_burned_filled.tif -c $tempCurrentBranchDataDir/dem_meters.tif $tempCurrentBranchDataDir/flowdir_d8_burned_filled.tif -v
+$srcDir/clip_rasters_to_branches.py -d $current_branch_id \
+    -b $tempHucDataDir/branch_polygons.gpkg \
+    -i $branch_id_attribute \
+    -r $tempHucDataDir/dem_meters.tif $tempHucDataDir/flowdir_d8_burned_filled.tif \
+    -c $tempCurrentBranchDataDir/dem_meters.tif $tempCurrentBranchDataDir/flowdir_d8_burned_filled.tif \
+    -v
 Tcount
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"
 date -u
 Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy\
+<<<$($srcDir/getRasterInfoNative.py $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg $tempCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+    -te $xmin $ymin $xmax $ymax \
+    -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg \
+    $tempCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
 Tcount
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
 echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $current_branch_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+    -te $xmin $ymin $xmax $ymax \
+    -ts $ncols $nrows \
+    $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg \
+    $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif
 Tcount
 
 ## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
@@ -93,7 +114,14 @@ if [ -f $tempHucDataDir/usgs_subset_gages.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $current_branch_id"
     date -u
     Tstart
-    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif -out $tempCurrentBranchDataDir -b $current_branch_id
+    python3 $srcDir/usgs_gage_crosswalk.py \
+        -gages $tempHucDataDir/usgs_subset_gages.gpkg \
+        -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
+        -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg \
+        -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
+        -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
+        -out $tempCurrentBranchDataDir \
+        -b $current_branch_id
     Tcount
 fi
 
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 37ffc7fe7..7baf258e0 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -39,7 +39,9 @@ fi
 echo -e $startDiv"Get WBD $hucNumber"
 date -u
 Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS $tempHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer -where "HUC$hucUnitLength='$hucNumber'"
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS \
+    $tempHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer \
+    -where "HUC$hucUnitLength='$hucNumber'"
 Tcount
 
 ## Subset Vector Layers ##
@@ -79,20 +81,35 @@ Tcount
 echo -e $startDiv"Clip WBD8"
 date -u
 Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -clipsrc $tempHucDataDir/wbd_buffered.gpkg $tempHucDataDir/wbd8_clp.gpkg $inputsDir/wbd/WBD_National.gpkg WBDHU8
+ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS \
+    -clipsrc $tempHucDataDir/wbd_buffered.gpkg $tempHucDataDir/wbd8_clp.gpkg \
+    $inputsDir/wbd/WBD_National.gpkg WBDHU8
 Tcount
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"
 date -u
 Tstart
-$srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg -s $tempHucDataDir/wbd_buffered_streams.gpkg -b $branch_id_attribute -r "ID" -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -e $tempHucDataDir/nwm_headwaters.gpkg -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg -wbd $tempHucDataDir/wbd.gpkg
+$srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg \
+    -s $tempHucDataDir/wbd_buffered_streams.gpkg \
+    -b $branch_id_attribute \
+    -r "ID" \
+    -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
+    -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
+    -e $tempHucDataDir/nwm_headwaters.gpkg \
+    -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg \
+    -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg \
+    -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg \
+    -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg \
+    -wbd $tempHucDataDir/wbd.gpkg
 
 # test if we received a non-zero code back from derive_level_paths.py
 #subscript_exit_code=$?
 
 # we have to retrow it if it is not a zero (but it will stop further execution in this script)
-#if [ $subscript_exit_code -ne 0 ] && [ $subscript_exit_code -ne 62 ] && [ $subscript_exit_code -eq 63 ]; then exit $subscript_exit_code; fi
+# if [ $subscript_exit_code -ne 0 ] && [ $subscript_exit_code -ne 62 ] && [ $subscript_exit_code -eq 63 ]; then
+#     exit $subscript_exit_code
+# fi
 
 # check if level paths exists
 levelpaths_exist=1
@@ -103,22 +120,36 @@ Tcount
 echo -e $startDiv"Associate level paths with levees"
 date -u
 Tstart
+
 [ -f $tempHucDataDir/nld_subset_levees.gpkg ] && \
-python3 $srcDir/associate_levelpaths_with_levees.py -nld $tempHucDataDir/nld_subset_levees.gpkg -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -lpa $tempHucDataDir/LeveeProtectedAreas_subset.gpkg -out $tempHucDataDir/levee_levelpaths.csv -w $levee_buffer -b $branch_id_attribute -l $levee_id_attribute
+python3 $srcDir/associate_levelpaths_with_levees.py -nld $tempHucDataDir/nld_subset_levees.gpkg \
+    -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
+    -lpa $tempHucDataDir/LeveeProtectedAreas_subset.gpkg \
+    -out $tempHucDataDir/levee_levelpaths.csv \
+    -w $levee_buffer \
+    -b $branch_id_attribute \
+    -l $levee_id_attribute
+
 Tcount
 
 ## STREAM BRANCH POLYGONS
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
 date -u
 Tstart
-$srcDir/buffer_stream_branches.py -a $input_DEM_domain -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -i $branch_id_attribute -d $branch_buffer_distance_meters -b $tempHucDataDir/branch_polygons.gpkg
+$srcDir/buffer_stream_branches.py -a $input_DEM_domain \
+    -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
+    -i $branch_id_attribute \
+    -d $branch_buffer_distance_meters \
+    -b $tempHucDataDir/branch_polygons.gpkg
 Tcount
 
 ## CREATE BRANCHID LIST FILE
 echo -e $startDiv"Create list file of branch ids for $hucNumber"
 date -u
 Tstart
-$srcDir/generate_branch_list.py -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg -b $branch_id_attribute -o $branch_list_lst_file
+$srcDir/generate_branch_list.py -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
+    -b $branch_id_attribute \
+    -o $branch_list_lst_file
 Tcount
 
 ## CREATE BRANCH ZERO ##
@@ -135,7 +166,9 @@ date -u
 Tstart
 
 [ ! -f $tempCurrentBranchDataDir/dem_meters.tif ] && \
-gdalwarp -cutline $tempHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $tempHucDataDir/dem_meters.tif
+gdalwarp -cutline $tempHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" \
+    -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" \
+    -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $tempHucDataDir/dem_meters.tif
 
 Tcount
 
@@ -143,7 +176,8 @@ Tcount
 echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"
 date -u
 Tstart
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy<<<$($srcDir/getRasterInfoNative.py $tempHucDataDir/dem_meters.tif)
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy\
+    <<<$($srcDir/getRasterInfoNative.py $tempHucDataDir/dem_meters.tif)
 
 ## RASTERIZE NLD MULTILINES ##
 echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $branch_zero_id"
@@ -151,32 +185,43 @@ date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
 [ -f $tempHucDataDir/3d_nld_subset_levees_burned.gpkg ] && \
-gdal_rasterize -l 3d_nld_subset_levees_burned -3d -at -a_nodata $ndv -te $xmin $ymin $xmax $ymax -ts $ncols $nrows -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" $tempHucDataDir/3d_nld_subset_levees_burned.gpkg $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif
+gdal_rasterize -l 3d_nld_subset_levees_burned -3d -at -a_nodata $ndv \
+    -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
+    -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" \
+    -co "TILED=YES" $tempHucDataDir/3d_nld_subset_levees_burned.gpkg \
+    $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif
 Tcount
 
 ## BURN LEVEES INTO DEM ##
-echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
+echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters"
+echo -e "(*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
 date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
 [ -f $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif ] && \
-python3 -m memory_profiler $srcDir/burn_in_levees.py -dem $tempHucDataDir/dem_meters.tif -nld $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif -out $tempHucDataDir/dem_meters.tif
+python3 -m memory_profiler $srcDir/burn_in_levees.py \
+    -dem $tempHucDataDir/dem_meters.tif \
+    -nld $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif \
+    -out $tempHucDataDir/dem_meters.tif
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) - BRANCH 0 (include all NWM streams) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $branch_zero_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_subset_streams.gpkg $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+    -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
+    $tempHucDataDir/nwm_subset_streams.gpkg $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
 Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) - BRANCHES (Not 0) (NWM levelpath streams) ##
-if [ "$levelpaths_exist" = "1" ] 
-then 
+if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Rasterize Reach Boolean $hucNumber (Branches)"
     date -u
     Tstart
-    gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg $tempHucDataDir/flows_grid_boolean.tif
+    gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+        -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
+        $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg $tempHucDataDir/flows_grid_boolean.tif
     Tcount
 fi
 
@@ -184,7 +229,9 @@ fi
 echo -e $startDiv"Rasterize NWM Headwaters $hucNumber $branch_zero_id"
 date -u
 Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/nwm_headwater_points_subset.gpkg $tempCurrentBranchDataDir/headwaters_$branch_zero_id.tif
+gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+    -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
+    $tempHucDataDir/nwm_headwater_points_subset.gpkg $tempCurrentBranchDataDir/headwaters_$branch_zero_id.tif
 Tcount
 
 ## DEM Reconditioning - BRANCH 0 (include all NWM streams) ##
@@ -193,18 +240,30 @@ Tcount
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/agreedem.py -r $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif -d $tempHucDataDir/dem_meters.tif -w $tempCurrentBranchDataDir -o $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+python3 -m memory_profiler $srcDir/agreedem.py \
+    -r $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif \
+    -d $tempHucDataDir/dem_meters.tif \
+    -w $tempCurrentBranchDataDir \
+    -o $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif \
+    -b $agree_DEM_buffer \
+    -sm 10 \
+    -sh 1000
 Tcount
 
 ## DEM Reconditioning - BRANCHES (NOT 0) (NWM levelpath streams) ##
 # Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
 # This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
-if [ "$levelpaths_exist" = "1" ] 
-then 
+if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber (Branches)"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/agreedem.py -r $tempHucDataDir/flows_grid_boolean.tif -d $tempHucDataDir/dem_meters.tif -w $tempHucDataDir -o $tempHucDataDir/dem_burned.tif -b $agree_DEM_buffer -sm 10 -sh 1000
+    python3 -m memory_profiler $srcDir/agreedem.py -r $tempHucDataDir/flows_grid_boolean.tif \
+        -d $tempHucDataDir/dem_meters.tif \
+        -w $tempHucDataDir \
+        -o $tempHucDataDir/dem_burned.tif \
+        -b $agree_DEM_buffer \
+        -sm 10 \
+        -sh 1000
     Tcount
 fi
 
@@ -212,12 +271,12 @@ fi
 echo -e $startDiv"Pit remove Burned DEM $hucNumber $branch_zero_id"
 date -u
 Tstart
-rd_depression_filling $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
+rd_depression_filling $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif \
+    $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
 Tcount
 
 ## PIT REMOVE BURNED DEM - BRANCHES (NOT 0) (NWM levelpath streams) ##
-if [ "$levelpaths_exist" = "1" ] 
-then 
+if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Pit remove Burned DEM $hucNumber (Branches)"
     date -u
     Tstart
@@ -229,16 +288,19 @@ fi
 echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $branch_zero_id"
 date -u
 Tstart
-mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
+mpiexec -n $ncores_fd $taudemDir2/d8flowdir \
+    -fel $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif \
+    -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
 Tcount
 
 ## D8 FLOW DIR - BRANCHES (NOT 0) (NWM levelpath streams) ##
-if [ "$levelpaths_exist" = "1" ] 
-then 
+if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber (Branches)"
     date -u
     Tstart
-    mpiexec -n $ncores_fd $taudemDir2/d8flowdir -fel $tempHucDataDir/dem_burned_filled.tif -p $tempHucDataDir/flowdir_d8_burned_filled.tif
+    mpiexec -n $ncores_fd $taudemDir2/d8flowdir \
+        -fel $tempHucDataDir/dem_burned_filled.tif \
+        -p $tempHucDataDir/flowdir_d8_burned_filled.tif
     Tcount
 fi
 
@@ -270,7 +332,14 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py -gages $inputsDir/usgs_gages/usgs_gages.gpkg -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg -ras $inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points.gpkg -o $tempHucDataDir/usgs_subset_gages.gpkg -huc $hucNumber -ahps $inputsDir/ahps_sites/nws_lid.gpkg -bzero_id $branch_zero_id 
+    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py \
+        -gages $inputsDir/usgs_gages/usgs_gages.gpkg \
+        -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
+        -ras $inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points.gpkg \
+        -o $tempHucDataDir/usgs_subset_gages.gpkg \
+        -huc $hucNumber \
+        -ahps $inputsDir/ahps_sites/nws_lid.gpkg \
+        -bzero_id $branch_zero_id
     Tcount
 fi
 
@@ -279,7 +348,13 @@ if [ -f $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 $srcDir/usgs_gage_crosswalk.py -gages $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg -dem $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif -out $tempCurrentBranchDataDir -b $branch_zero_id
+    python3 $srcDir/usgs_gage_crosswalk.py \
+        -gages $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg \
+        -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg \
+        -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg \
+        -dem $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif \
+        -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif \
+        -out $tempCurrentBranchDataDir -b $branch_zero_id
     Tcount
 fi
 
@@ -301,7 +376,8 @@ branch_processing_start_time=`date +%s`
 if [ -f $branch_list_lst_file ]; then
     # There may not be a branch_ids.lst if there were no level paths (no stream orders 3+)
     # but there will still be a branch zero
-    parallel --eta --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSummaryLogFile --colsep ',' -- $srcDir/process_branch.sh $runName $hucNumber :::: $branch_list_lst_file
+    parallel --eta --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSummaryLogFile --colsep ',' \
+    -- $srcDir/process_branch.sh $runName $hucNumber :::: $branch_list_lst_file
 else
     echo "No level paths exist with this HUC. Processing branch zero only."
 fi
diff --git a/src/split_flows.py b/src/split_flows.py
index ff7f4fbc6..7b1818994 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -10,50 +10,53 @@
 '''
 
 import argparse
-import build_stream_traversal
-import geopandas as gpd
-import numpy as np
 import os
-import pandas as pd
-import rasterio
 import sys
 import time
-
 from collections import OrderedDict
-from os import remove,environ,path
-from os.path import isfile,split,dirname
+from os import environ, path, remove
+from os.path import dirname, isfile, split
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
 from shapely import ops, wkt
-from shapely.geometry import Point, LineString, MultiPoint
+from shapely.geometry import LineString, MultiPoint, Point
 from shapely.ops import split as shapely_ops_split
 from tqdm import tqdm
+
+import build_stream_traversal
+from utils.fim_enums import FIM_exit_codes
 from utils.shared_functions import getDriver, mem_profile
 from utils.shared_variables import FIM_ID
-from utils.fim_enums import FIM_exit_codes
 
-@mem_profile
-def split_flows(flows_filename,
-                dem_filename, 
-                split_flows_filename, 
-                split_points_filename, 
-                wbd8_clp_filename, 
-                lakes_filename,
-                nwm_streams_filename,
-                max_length, 
-                slope_min, 
-                lakes_buffer_input):
 
+@mem_profile
+def split_flows(
+    flows_filename,
+    dem_filename,
+    split_flows_filename,
+    split_points_filename,
+    wbd8_clp_filename,
+    lakes_filename,
+    nwm_streams_filename,
+    max_length,
+    slope_min,
+    lakes_buffer_input,
+):
     def snap_and_trim_flow(snapped_point, flows):
         # Find nearest flow line
         if len(flows) > 1:
             sjoin_nearest = gpd.sjoin_nearest(snapped_point, flows, max_distance=100)
             if sjoin_nearest.empty:
                 return flows
-            
+
             if len(sjoin_nearest) > 1:
                 sjoin_nearest = sjoin_nearest[sjoin_nearest['LINKNO'].isin(sjoin_nearest['DSLINKNO'])]
-            
+
             nearest_index = int(sjoin_nearest['LINKNO'])
-            flow = flows[flows['LINKNO']==nearest_index]
+            flow = flows[flows['LINKNO'] == nearest_index]
             flow.index = [0]
 
         else:
@@ -65,12 +68,19 @@ def snap_and_trim_flow(snapped_point, flows):
 
         # Trim flows to snapped point
         # buffer here because python precision issues, print(demDerived_reaches.distance(snapped_point) < 1e-8)
-        trimmed_line = shapely_ops_split(flow.iloc[0]['geometry'], snapped_point.iloc[0]['geometry'].buffer(1))
+        trimmed_line = shapely_ops_split(
+            flow.iloc[0]['geometry'], snapped_point.iloc[0]['geometry'].buffer(1)
+        )
         # Edge cases: line string not split?, nothing is returned, split does not preserve linestring order?
-        # Note to dear reader: last here is really the most upstream segment (see crevats above).  When we split we should get 3 segments, the most downstream one
-        # the tiny 1 meter segment that falls within the snapped point buffer, and the most upstream one.  We want that last one which is why we trimmed_line[len(trimmed_line)-1]
-        last_line_segment = pd.DataFrame({'id':['first'],'geometry':[trimmed_line.geoms[len(trimmed_line.geoms)-1].wkt]})
-        last_line_segment['geometry'] = last_line_segment['geometry'].apply(wkt.loads) # can be last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment) # when we update geopandas verisons
+        # Note to dear reader: last here is really the most upstream segment (see crevats above).
+        # When we split we should get 3 segments, the most downstream one
+        # the tiny 1 meter segment that falls within the snapped point buffer, and the most upstream one.
+        # We want that last one which is why we trimmed_line[len(trimmed_line)-1]
+        last_line_segment = pd.DataFrame(
+            {'id': ['first'], 'geometry': [trimmed_line.geoms[len(trimmed_line.geoms) - 1].wkt]}
+        )
+        # when we update geopandas verison: last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment)
+        last_line_segment['geometry'] = last_line_segment['geometry'].apply(wkt.loads)
         last_line_segment_geodataframe = gpd.GeoDataFrame(last_line_segment).set_crs(flow.crs)
 
         # replace geometry in merged flowine
@@ -78,25 +88,24 @@ def snap_and_trim_flow(snapped_point, flows):
 
         if nearest_index is not None:
             # Update geometry of line closest to snapped_point
-            flows.loc[flows['LINKNO']==nearest_index, 'geometry'] = flow_geometry
+            flows.loc[flows['LINKNO'] == nearest_index, 'geometry'] = flow_geometry
         else:
             flows['geometry'] = flow_geometry
 
         return flows
 
-        
     toMetersConversion = 1e-3
 
     print('Loading data ...')
     flows = gpd.read_file(flows_filename)
 
-    if (len(flows) == 0):
+    if len(flows) == 0:
         # this is not an exception, but a custom exit code that can be trapped
         print("No relevant streams within HUC boundaries.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
     wbd8 = gpd.read_file(wbd8_clp_filename)
-    dem = rasterio.open(dem_filename,'r')
+    dem = rasterio.open(dem_filename, 'r')
 
     if isfile(lakes_filename):
         lakes = gpd.read_file(lakes_filename)
@@ -116,13 +125,13 @@ def snap_and_trim_flow(snapped_point, flows):
     hydro_id = 'HydroID'
 
     # If loop addressing: https://github.com/NOAA-OWP/inundation-mapping/issues/560
-    print ('trimming DEM stream to NWM branch terminus')
+    print('trimming DEM stream to NWM branch terminus')
     # read in nwm lines, explode to ensure linestrings are the only geometry
     nwm_streams = gpd.read_file(nwm_streams_filename).explode(index_parts=True)
 
     # Dissolve levelpath if not branch 0
     if 'levpa_id' in nwm_streams.columns:
-        if (len(nwm_streams) > 1):
+        if len(nwm_streams) > 1:
             # Dissolve the linestring (how much faith should I hold that these are digitized with flow?)
             linestring_geo = ops.linemerge(nwm_streams.dissolve(by='levpa_id').iloc[0]['geometry'])
         else:
@@ -131,58 +140,67 @@ def snap_and_trim_flow(snapped_point, flows):
         # Identify the end vertex (most downstream, should be last), transform into geodataframe
         terminal_nwm_point = []
         last = Point(linestring_geo.coords[-1])
-        terminal_nwm_point.append({'ID':'terminal','geometry':last})
+        terminal_nwm_point.append({'ID': 'terminal', 'geometry': last})
         snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(nwm_streams.crs)
 
         flows = snap_and_trim_flow(snapped_point, flows)
 
     # If branch 0: loop over NWM terminal segments
     else:
-        nwm_streams_terminal = nwm_streams[nwm_streams['to']==0]
+        nwm_streams_terminal = nwm_streams[nwm_streams['to'] == 0]
         if not nwm_streams_terminal.empty:
             for i, row in nwm_streams_terminal.iterrows():
                 linestring_geo = row['geometry']
                 # Identify the end vertex (most downstream, should be last), transform into geodataframe
                 terminal_nwm_point = []
                 last = Point(linestring_geo.coords[-1])
-                terminal_nwm_point.append({'ID':'terminal','geometry':last})
+                terminal_nwm_point.append({'ID': 'terminal', 'geometry': last})
                 snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(nwm_streams.crs)
 
                 flows = snap_and_trim_flow(snapped_point, flows)
 
-
     # split at HUC8 boundaries
-    print ('splitting stream segments at HUC8 boundaries')
-    flows = gpd.overlay(flows, wbd8, how='union',keep_geom_type=True).explode(index_parts=True).reset_index(drop=True)
+    print('splitting stream segments at HUC8 boundaries')
+    flows = (
+        gpd.overlay(flows, wbd8, how='union', keep_geom_type=True)
+        .explode(index_parts=True)
+        .reset_index(drop=True)
+    )
     flows = flows[~flows.is_empty]
 
-    if (len(flows) == 0):
+    if len(flows) == 0:
         # this is not an exception, but a custom exit code that can be trapped
         print("No relevant streams within HUC boundaries.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
     # check for lake features
-    if lakes is not None and len(flows) > 0 :
+    if lakes is not None and len(flows) > 0:
         if len(lakes) > 0:
-          print ('splitting stream segments at ' + str(len(lakes)) + ' waterbodies')
-          #create splits at lake boundaries
-          lakes = lakes.filter(items=['newID', 'geometry'])
-          lakes = lakes.set_index('newID')
-          flows = gpd.overlay(flows, lakes, how='union',keep_geom_type=True).explode(index_parts=True).reset_index(drop=True)
-          lakes_buffer = lakes.copy()
-          lakes_buffer['geometry'] = lakes.buffer(lakes_buffer_input) # adding X meter buffer for spatial join comparison (currently using 20meters)
-
-    print ('splitting ' + str(len(flows)) + ' stream segments based on ' + str(max_length) + ' m max length')
+            print('splitting stream segments at ' + str(len(lakes)) + ' waterbodies')
+            # create splits at lake boundaries
+            lakes = lakes.filter(items=['newID', 'geometry'])
+            lakes = lakes.set_index('newID')
+            flows = (
+                gpd.overlay(flows, lakes, how='union', keep_geom_type=True)
+                .explode(index_parts=True)
+                .reset_index(drop=True)
+            )
+            lakes_buffer = lakes.copy()
+            lakes_buffer['geometry'] = lakes.buffer(
+                lakes_buffer_input
+            )  # adding X meter buffer for spatial join comparison (currently using 20meters)
+
+    print('splitting ' + str(len(flows)) + ' stream segments based on ' + str(max_length) + ' m max length')
 
     # remove empty geometries
-    flows = flows.loc[~flows.is_empty,:]
+    flows = flows.loc[~flows.is_empty, :]
 
-    if (len(flows) == 0):
+    if len(flows) == 0:
         # this is not an exception, but a custom exit code that can be trapped
         print("No relevant streams within HUC boundaries.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
-    for i,lineString in tqdm(enumerate(flows.geometry),total=len(flows.geometry)):
+    for i, lineString in tqdm(enumerate(flows.geometry), total=len(flows.geometry)):
         # Reverse geometry order (necessary for BurnLines)
         lineString = LineString(lineString.coords[::-1])
 
@@ -196,8 +214,9 @@ def snap_and_trim_flow(snapped_point, flows):
             line_points = [point for point in zip(*lineString.coords.xy)]
 
             # Calculate channel slope
-            start_point = line_points[0]; end_point = line_points[-1]
-            start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
+            start_point = line_points[0]
+            end_point = line_points[-1]
+            start_elev, end_elev = [i[0] for i in rasterio.sample.sample_gen(dem, [start_point, end_point])]
             slope = float(abs(start_elev - end_elev) / lineString.length)
             if slope < slope_min:
                 slope = slope_min
@@ -213,7 +232,6 @@ def snap_and_trim_flow(snapped_point, flows):
         last_point_in_entire_lineString = list(zip(*lineString.coords.xy))[-1]
 
         for point in zip(*lineString.coords.xy):
-
             cumulative_line = cumulative_line + [point]
             line_points = line_points + [point]
             numberOfPoints_in_cumulative_line = len(cumulative_line)
@@ -227,13 +245,15 @@ def snap_and_trim_flow(snapped_point, flows):
             cumulative_length = LineString(cumulative_line).length
 
             if cumulative_length >= splitLength:
-
                 splitLineString = LineString(cumulative_line)
                 split_flows = split_flows + [splitLineString]
 
                 # Calculate channel slope
-                start_point = cumulative_line[0]; end_point = cumulative_line[-1]
-                start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
+                start_point = cumulative_line[0]
+                end_point = cumulative_line[-1]
+                start_elev, end_elev = [
+                    i[0] for i in rasterio.sample.sample_gen(dem, [start_point, end_point])
+                ]
                 slope = float(abs(start_elev - end_elev) / splitLineString.length)
                 if slope < slope_min:
                     slope = slope_min
@@ -241,7 +261,7 @@ def snap_and_trim_flow(snapped_point, flows):
 
                 last_point = end_point
 
-                if (last_point == last_point_in_entire_lineString):
+                if last_point == last_point_in_entire_lineString:
                     continue
 
                 cumulative_line = []
@@ -251,17 +271,22 @@ def snap_and_trim_flow(snapped_point, flows):
         split_flows = split_flows + [splitLineString]
 
         # Calculate channel slope
-        start_point = cumulative_line[0]; end_point = cumulative_line[-1]
-        start_elev,end_elev = [i[0] for i in rasterio.sample.sample_gen(dem,[start_point,end_point])]
+        start_point = cumulative_line[0]
+        end_point = cumulative_line[-1]
+        start_elev, end_elev = [i[0] for i in rasterio.sample.sample_gen(dem, [start_point, end_point])]
         slope = float(abs(start_elev - end_elev) / splitLineString.length)
         if slope < slope_min:
             slope = slope_min
         slopes = slopes + [slope]
 
-    split_flows_gdf = gpd.GeoDataFrame({'S0' : slopes ,'geometry':split_flows}, crs=flows.crs, geometry='geometry')
+    split_flows_gdf = gpd.GeoDataFrame(
+        {'S0': slopes, 'geometry': split_flows}, crs=flows.crs, geometry='geometry'
+    )
     split_flows_gdf['LengthKm'] = split_flows_gdf.geometry.length * toMetersConversion
     if lakes is not None:
-        split_flows_gdf = gpd.sjoin(split_flows_gdf, lakes_buffer, how='left', predicate='within') #options: intersects, within, contains, crosses
+        split_flows_gdf = gpd.sjoin(
+            split_flows_gdf, lakes_buffer, how='left', predicate='within'
+        )  # options: intersects, within, contains, crosses
         split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
     else:
         split_flows_gdf['LakeID'] = -999
@@ -271,12 +296,12 @@ def snap_and_trim_flow(snapped_point, flows):
 
     # Create Ids and Network Traversal Columns
     addattributes = build_stream_traversal.build_stream_traversal_columns()
-    tResults=None
+    tResults = None
     tResults = addattributes.execute(split_flows_gdf, wbd8, hydro_id)
     if tResults[0] == 'OK':
         split_flows_gdf = tResults[1]
     else:
-        print ('Error: Could not add network attributes to stream segments')
+        print('Error: Could not add network attributes to stream segments')
 
     # remove single node segments
     split_flows_gdf = split_flows_gdf.query("From_Node != To_Node")
@@ -298,7 +323,9 @@ def snap_and_trim_flow(snapped_point, flows):
     hydroIDs_points = [hidp for hidp in split_points.values()]
     split_points = [Point(*point) for point in split_points]
 
-    split_points_gdf = gpd.GeoDataFrame({'id': hydroIDs_points , 'geometry':split_points}, crs=flows.crs, geometry='geometry')
+    split_points_gdf = gpd.GeoDataFrame(
+        {'id': hydroIDs_points, 'geometry': split_points}, crs=flows.crs, geometry='geometry'
+    )
 
     print('Writing outputs ...')
 
@@ -307,28 +334,28 @@ def snap_and_trim_flow(snapped_point, flows):
     if isfile(split_points_filename):
         remove(split_points_filename)
 
-    if (len(split_flows_gdf) == 0):
+    if len(split_flows_gdf) == 0:
         # this is not an exception, but a custom exit code that can be trapped
         print("There are no flowlines after stream order filtering.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
-    split_flows_gdf.to_file(split_flows_filename,driver=getDriver(split_flows_filename),index=False)
+    split_flows_gdf.to_file(split_flows_filename, driver=getDriver(split_flows_filename), index=False)
 
     if len(split_points_gdf) == 0:
         raise Exception("No points exist.")
-    split_points_gdf.to_file(split_points_filename,driver=getDriver(split_points_filename),index=False)
+    split_points_gdf.to_file(split_points_filename, driver=getDriver(split_points_filename), index=False)
 
 
 if __name__ == '__main__':
     # Parse arguments.
     parser = argparse.ArgumentParser(description='split_flows.py')
-    parser.add_argument('-f', '--flows-filename', help='flows-filename',required=True)
-    parser.add_argument('-d', '--dem-filename', help='dem-filename',required=True)
-    parser.add_argument('-s', '--split-flows-filename', help='split-flows-filename',required=True)
-    parser.add_argument('-p', '--split-points-filename', help='split-points-filename',required=True)
-    parser.add_argument('-w', '--wbd8-clp-filename', help='wbd8-clp-filename',required=True)
-    parser.add_argument('-l', '--lakes-filename', help='lakes-filename',required=True)
-    parser.add_argument('-n', '--nwm-streams-filename', help='nwm-streams-filename',required=True)
+    parser.add_argument('-f', '--flows-filename', help='flows-filename', required=True)
+    parser.add_argument('-d', '--dem-filename', help='dem-filename', required=True)
+    parser.add_argument('-s', '--split-flows-filename', help='split-flows-filename', required=True)
+    parser.add_argument('-p', '--split-points-filename', help='split-points-filename', required=True)
+    parser.add_argument('-w', '--wbd8-clp-filename', help='wbd8-clp-filename', required=True)
+    parser.add_argument('-l', '--lakes-filename', help='lakes-filename', required=True)
+    parser.add_argument('-n', '--nwm-streams-filename', help='nwm-streams-filename', required=True)
     parser.add_argument('-m', '--max-length', help='Maximum split distance (meters)', required=True)
     parser.add_argument('-t', '--slope-min', help='Minimum slope', required=True)
     parser.add_argument('-b', '--lakes-buffer-input', help='Lakes buffer distance (meters)', required=True)
diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
index 0a810bab9..7136e617a 100644
--- a/src/src_adjust_ras2fim_rating.py
+++ b/src/src_adjust_ras2fim_rating.py
@@ -1,17 +1,24 @@
+#!/usr/bin/env python3
+
 import argparse
+import datetime as dt
+import json
+import multiprocessing
 import os
-import pandas as pd
 import sys
-import json
-import datetime as dt
-from pathlib import Path
 from collections import deque
-import multiprocessing
 from multiprocessing import Pool
-from utils.shared_functions import check_file_age, concat_huc_csv
+from pathlib import Path
+
+import pandas as pd
+
 from src_roughness_optimization import update_rating_curve
+from utils.shared_functions import check_file_age, concat_huc_csv
+
+
 '''
-The script ingests a RAS2FIM rating curve csv and a NWM flow recurrence interval database. The gage location will be associated to the corresponding hydroID and attributed with the HAND elevation value
+The script ingests a RAS2FIM rating curve csv and a NWM flow recurrence interval database.
+The gage location will be associated to the corresponding hydroID and attributed with the HAND elevation value
 
 Processing
 - Read in RAS2FIM rating curve from csv and convert WSE navd88 values to meters
@@ -22,184 +29,297 @@
 - Calculate the closest SRC discharge value to the NWM flow value
 - Create dataframe with crosswalked RAS2FIM flow and NWM recurr flow and assign metadata attributes
 - Calculate flow difference (variance) to check for large discrepancies btw NWM flow and RAS2FIM closest flow
-- Log any signifant differences (or negative HAND values) btw the NWM flow value and closest RAS2FIM rating flow
+- Log signifant differences (or negative HAND values) btw the NWM flow value and closest RAS2FIM rating flow
 - Produce log file
 - Call update_rating_curve() to perform the rating curve calibration.
 
 Inputs
-- branch_dir:        fim directory containing individual HUC output dirs
-- ras_rc_filepath:     RAS2FIM rating curve database (produced by reformat_ras_rating_curve.py in ras2fim repo)
+- branch_dir:           fim directory containing individual HUC output dirs
+- ras_rc_filepath:      RAS2FIM rating curve database
+                            (produced by reformat_ras_rating_curve.py in ras2fim repo)
 - nwm_recurr_filepath:  NWM flow recurrence interval dataset
 - debug_outputs_option: optional flag to output intermediate files for reviewing/debugging
 - job_number:           number of multi-processing jobs to use
 
 Outputs
-- water_edge_median_ds: dataframe containing 'location_id','hydroid','feature_id','huc','hand','discharge_cms','nwm_recur_flow_cms','nwm_recur','layer'
+- water_edge_median_ds: dataframe containing 'location_id','hydroid','feature_id','huc','hand',
+                        'discharge_cms','nwm_recur_flow_cms','nwm_recur','layer'
 '''
 
+
 def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir):
     start_time = dt.datetime.now()
     print('Reading RAS2FIM rating curves from csv...')
     log_text = 'Processing database for RAS2FIM flow/WSE at NWM flow recur intervals...\n'
     col_filter = ["fid_xs", "flow", "wse"]
-    ras_rc_df = pd.read_csv(ras_rc_filepath, dtype={'fid_xs': object}, usecols=col_filter)#, nrows=30000)
-    ras_rc_df.rename(columns={'fid_xs':'location_id'}, inplace=True)
-    #ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
+    ras_rc_df = pd.read_csv(ras_rc_filepath, dtype={'fid_xs': object}, usecols=col_filter)  # , nrows=30000)
+    ras_rc_df.rename(columns={'fid_xs': 'location_id'}, inplace=True)
+    # ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
     print('Duration (read ras_rc_csv): {}'.format(dt.datetime.now() - start_time))
-    
+
     # convert WSE navd88 values to meters
-    ras_rc_df.rename(columns={'wse':'wse_navd88_m','flow':'discharge_cms'}, inplace=True) # assume ras2fim elevation data in feet
-    
+    ras_rc_df.rename(
+        columns={'wse': 'wse_navd88_m', 'flow': 'discharge_cms'}, inplace=True
+    )  # assume ras2fim elevation data in feet
+
     # read in the aggregate RAS elev table csv
     start_time = dt.datetime.now()
-    cross_df = ras_elev_df[["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_adj_elevation", "source"]].copy()
-    cross_df.rename(columns={'dem_adj_elevation':'hand_datum', 'HydroID':'hydroid', 'HUC8':'huc'}, inplace=True)
-    
-    # filter null location_id rows from cross_df 
+    cross_df = ras_elev_df[
+        ["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_adj_elevation", "source"]
+    ].copy()
+    cross_df.rename(
+        columns={'dem_adj_elevation': 'hand_datum', 'HydroID': 'hydroid', 'HUC8': 'huc'}, inplace=True
+    )
+
+    # filter null location_id rows from cross_df
     cross_df = cross_df[cross_df.location_id.notnull()]
-    
+
     # merge ras2fim ratings with crosswalk attributes
     ras_rc_df = ras_rc_df.merge(cross_df, how='left', on='location_id')
     ras_rc_df = ras_rc_df[ras_rc_df['hydroid'].notna()]
-    
+
     # calculate hand elevation
     ras_rc_df['hand'] = ras_rc_df['wse_navd88_m'] - ras_rc_df['hand_datum']
-    ras_rc_df = ras_rc_df[['location_id','feature_id','hydroid','levpa_id','huc','hand','discharge_cms','source']]
+    ras_rc_df = ras_rc_df[
+        ['location_id', 'feature_id', 'hydroid', 'levpa_id', 'huc', 'hand', 'discharge_cms', 'source']
+    ]
     ras_rc_df['feature_id'] = ras_rc_df['feature_id'].astype(int)
-    
+
     # read in the NWM recurr csv file
     nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
     nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
-    nwm_recur_df.rename(columns={'2_0_year_recurrence_flow_17C':'2_0_year','5_0_year_recurrence_flow_17C':'5_0_year','10_0_year_recurrence_flow_17C':'10_0_year','25_0_year_recurrence_flow_17C':'25_0_year','50_0_year_recurrence_flow_17C':'50_0_year','100_0_year_recurrence_flow_17C':'100_0_year'}, inplace=True)
-    
-    #convert cfs to cms (x 0.028317)
-    nwm_recur_df.loc[:, ['2_0_year','5_0_year','10_0_year','25_0_year','50_0_year','100_0_year']] *= 0.028317
-    
+    nwm_recur_df.rename(
+        columns={
+            '2_0_year_recurrence_flow_17C': '2_0_year',
+            '5_0_year_recurrence_flow_17C': '5_0_year',
+            '10_0_year_recurrence_flow_17C': '10_0_year',
+            '25_0_year_recurrence_flow_17C': '25_0_year',
+            '50_0_year_recurrence_flow_17C': '50_0_year',
+            '100_0_year_recurrence_flow_17C': '100_0_year',
+        },
+        inplace=True,
+    )
+
+    # convert cfs to cms (x 0.028317)
+    nwm_recur_df.loc[
+        :, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year', '100_0_year']
+    ] *= 0.028317
+
     # merge nwm recurr with ras_rc_df
     merge_df = ras_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
-    
+
     # NWM recurr intervals
-    recurr_intervals = ["2","5","10","25","50","100"] # "2","5","10","25","50","100"
-    final_df = pd.DataFrame() # create empty dataframe to append flow interval dataframes
+    recurr_intervals = ["2", "5", "10", "25", "50", "100"]  # "2","5","10","25","50","100"
+    final_df = pd.DataFrame()  # create empty dataframe to append flow interval dataframes
     for interval in recurr_intervals:
-        log_text += ('\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n')
+        log_text += '\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n'
         print('Processing: ' + str(interval) + '-year NWM recurr intervals')
         ## Calculate the closest SRC discharge value to the NWM flow value
-        merge_df['Q_find'] = (merge_df['discharge_cms'] - merge_df[interval+"_0_year"]).abs()
-        
+        merge_df['Q_find'] = (merge_df['discharge_cms'] - merge_df[interval + "_0_year"]).abs()
+
         ## Check for any missing/null entries in the input SRC
-        if merge_df['Q_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
-            log_text += 'HUC: ' + str(merge_df['huc']) + ' : feature_id' + str(merge_df['feature_id']) + ' --> Null values found in "Q_find" calc. These will be filled with 999999 () \n'
+        # there may be null values for lake or coastal flow lines
+        # (need to set a value to do groupby idxmin below)
+        if merge_df['Q_find'].isnull().values.any():
+            log_text += (
+                'HUC: '
+                + str(merge_df['huc'])
+                + ' : feature_id'
+                + str(merge_df['feature_id'])
+                + ' --> Null values found in "Q_find" calc. These will be filled with 999999 () \n'
+            )
             ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
             merge_df['Q_find'] = merge_df['Q_find'].fillna(999999)
         if merge_df['hydroid'].isnull().values.any():
             log_text += 'HUC: ' + str(merge_df['huc']) + ' --> Null values found in "hydroid"... \n'
-        
+
         # Create dataframe with crosswalked RAS2FIM flow and NWM recurr flow
-        calc_df = merge_df.loc[merge_df.groupby(['location_id','levpa_id'])['Q_find'].idxmin()].reset_index(drop=True) # find the index of the Q_1_5_find (closest matching flow)
-        # Calculate flow difference (variance) to check for large discrepancies btw NWM flow and RAS2FIM rating closest flow
-        calc_df['check_variance'] = ((calc_df['discharge_cms'] - calc_df[interval+"_0_year"])/calc_df['discharge_cms']).abs()
+        calc_df = merge_df.loc[merge_df.groupby(['location_id', 'levpa_id'])['Q_find'].idxmin()].reset_index(
+            drop=True
+        )  # find the index of the Q_1_5_find (closest matching flow)
+        # Calculate flow difference (variance) to check for large discrepancies between
+        # NWM flow and RAS2FIM rating closest flow
+        calc_df['check_variance'] = (
+            (calc_df['discharge_cms'] - calc_df[interval + "_0_year"]) / calc_df['discharge_cms']
+        ).abs()
         # Assign new metadata attributes
-        calc_df['nwm_recur'] = interval+"_0_year"
-        calc_df['layer'] = '_ras2fim-gage____' + interval+"-year"
-        calc_df.rename(columns={interval+"_0_year":'nwm_recur_flow_cms'}, inplace=True)
+        calc_df['nwm_recur'] = interval + "_0_year"
+        calc_df['layer'] = '_ras2fim-gage____' + interval + "-year"
+        calc_df.rename(columns={interval + "_0_year": 'nwm_recur_flow_cms'}, inplace=True)
         # Subset calc_df for final output
-        calc_df = calc_df[['location_id','hydroid','feature_id','levpa_id','huc','hand','discharge_cms','check_variance','nwm_recur_flow_cms','nwm_recur','layer','source']]
+        calc_df = calc_df[
+            [
+                'location_id',
+                'hydroid',
+                'feature_id',
+                'levpa_id',
+                'huc',
+                'hand',
+                'discharge_cms',
+                'check_variance',
+                'nwm_recur_flow_cms',
+                'nwm_recur',
+                'layer',
+                'source',
+            ]
+        ]
         final_df = pd.concat([final_df, calc_df], ignore_index=True)
         # Log any negative HAND elev values and remove from database
-        log_text += ('Warning: Negative HAND stage values -->\n')
-        log_text += (calc_df[calc_df['hand']<0].to_string() +'\n')
-        final_df = final_df[final_df['hand']>0]
-        # Log any signifant differences btw the NWM flow value and closest RAS2FIM rating flow (this ensures that we consistently sample the RAS2FIM rating curves at known intervals - NWM recur flow)
-        log_text += ('Warning: Large variance (>10%) between NWM flow and closest RAS2FIM flow -->\n')
-        log_text += (calc_df[calc_df['check_variance']>0.1].to_string() +'\n')
-        final_df = final_df[final_df['check_variance']<0.1]
+        log_text += 'Warning: Negative HAND stage values -->\n'
+        log_text += calc_df[calc_df['hand'] < 0].to_string() + '\n'
+        final_df = final_df[final_df['hand'] > 0]
+        # Log any signifant differences btw the NWM flow value and closest RAS2FIM rating flow
+        # (this ensures that we consistently sample the RAS2FIM rating curves at
+        # known intervals - NWM recur flow)
+        log_text += 'Warning: Large variance (>10%) between NWM flow and closest RAS2FIM flow -->\n'
+        log_text += calc_df[calc_df['check_variance'] > 0.1].to_string() + '\n'
+        final_df = final_df[final_df['check_variance'] < 0.1]
         # Get datestamp from ras2fim rating curve file to use as coll_time attribute in hydroTable.csv
         datestamp = check_file_age(ras_rc_filepath)
         final_df['coll_time'] = str(datestamp)[:15]
 
     # Rename attributes (for ingest to update_rating_curve) and output csv with the RAS2FIM RC database
-    final_df.rename(columns={'discharge_cms':'flow','source':'submitter'}, inplace=True)  
-    final_df.to_csv(os.path.join(log_dir,"ras2fim_rc_nwm_recurr.csv"),index=False)
+    final_df.rename(columns={'discharge_cms': 'flow', 'source': 'submitter'}, inplace=True)
+    final_df.to_csv(os.path.join(log_dir, "ras2fim_rc_nwm_recurr.csv"), index=False)
 
     # Output log text to log file
-    log_text += ('#########\nTotal entries per RAS2FIM point location (feature_id) -->\n')
-    loc_id_df = final_df.groupby(['location_id']).size().reset_index(name='count') 
-    log_text += (loc_id_df.to_string() +'\n')
-    log_text += ('#########\nTotal entries per NWM recur value -->\n')
-    recur_count_df = final_df.groupby(['nwm_recur']).size().reset_index(name='count') 
-    log_text += (recur_count_df.to_string() +'\n')
-    log_ras_db = open(os.path.join(log_dir,'log_ras2fim_rc_database.log'),"w")
+    log_text += '#########\nTotal entries per RAS2FIM point location (feature_id) -->\n'
+    loc_id_df = final_df.groupby(['location_id']).size().reset_index(name='count')
+    log_text += loc_id_df.to_string() + '\n'
+    log_text += '#########\nTotal entries per NWM recur value -->\n'
+    recur_count_df = final_df.groupby(['nwm_recur']).size().reset_index(name='count')
+    log_text += recur_count_df.to_string() + '\n'
+    log_ras_db = open(os.path.join(log_dir, 'log_ras2fim_rc_database.log'), "w")
     log_ras_db.write(log_text)
     log_ras_db.close()
-    return(final_df)
+    return final_df
 
-def branch_proc_list(ras_df,run_dir,debug_outputs_option,log_file):
+
+def branch_proc_list(ras_df, run_dir, debug_outputs_option, log_file):
     procs_list = []  # Initialize list for mulitprocessing.
 
     # loop through all unique level paths that have a USGS gage
     huc_branch_dict = ras_df.groupby('huc')['levpa_id'].apply(set).to_dict()
 
-    for huc in sorted(huc_branch_dict.keys()): # sort huc_list for helping track progress in future print statments
+    for huc in sorted(
+        huc_branch_dict.keys()
+    ):  # sort huc_list for helping track progress in future print statments
         branch_set = huc_branch_dict[huc]
-        for branch_id in branch_set: 
+        for branch_id in branch_set:
             # Define paths to branch HAND data.
             # Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
             # Assumes outputs are for HUC8 (not HUC6)
-            branch_dir = os.path.join(run_dir,huc,'branches',branch_id)
+            branch_dir = os.path.join(run_dir, huc, 'branches', branch_id)
             hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
-            catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
-            catchments_poly_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
+            catchments_path = os.path.join(
+                branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif'
+            )
+            catchments_poly_path = os.path.join(
+                branch_dir,
+                'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg',
+            )
             htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
-            water_edge_median_ds = ras_df[(ras_df['huc']==huc) & (ras_df['levpa_id']==branch_id)]
+            water_edge_median_ds = ras_df[(ras_df['huc'] == huc) & (ras_df['levpa_id'] == branch_id)]
 
             # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
             if not os.path.exists(hand_path):
-                print("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+                print(
+                    "WARNING: HAND grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: HAND grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             elif not os.path.exists(catchments_path):
-                print("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')        
+                print(
+                    "WARNING: Catchments grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: Catchments grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             elif not os.path.exists(htable_path):
-                print("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+                print(
+                    "WARNING: hydroTable does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: hydroTable does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             else:
                 ## Additional arguments for src_roughness_optimization
-                source_tag = 'ras2fim_rating' # tag to use in source attribute field
-                merge_prev_adj = True # merge in previous SRC adjustment calculations
+                source_tag = 'ras2fim_rating'  # tag to use in source attribute field
+                merge_prev_adj = True  # merge in previous SRC adjustment calculations
 
                 print('Will perform SRC adjustments for huc: ' + str(huc) + ' - branch-id: ' + str(branch_id))
-                procs_list.append([branch_dir, water_edge_median_ds, htable_path, huc, branch_id, catchments_poly_path, debug_outputs_option, source_tag, merge_prev_adj])
+                procs_list.append(
+                    [
+                        branch_dir,
+                        water_edge_median_ds,
+                        htable_path,
+                        huc,
+                        branch_id,
+                        catchments_poly_path,
+                        debug_outputs_option,
+                        source_tag,
+                        merge_prev_adj,
+                    ]
+                )
 
     # multiprocess all available branches
     print(f"Calculating new SRCs for {len(procs_list)} branches using {job_number} jobs...")
     with Pool(processes=job_number) as pool:
-            log_output = pool.starmap(update_rating_curve, procs_list)
-            log_file.writelines(["%s\n" % item  for item in log_output])
-    # try statement for debugging 
+        log_output = pool.starmap(update_rating_curve, procs_list)
+        log_file.writelines(["%s\n" % item for item in log_output])
+    # try statement for debugging
     # try:
     #     with Pool(processes=job_number) as pool:
     #         log_output = pool.starmap(update_rating_curve, procs_list)
-    #         log_file.writelines(["%s\n" % item  for item in log_output])
+    #         log_file.writelines(["%s\n" % item for item in log_output])
     # except Exception as e:
     #     print(str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e))
-    #     log_file.write('ERROR!!!: HUC ' + str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e) + '\n')
+    #     log_file.write(
+    #         'ERROR!!!: HUC ' + str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e) + '\n'
+    #     )
+
 
-def run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number):
+def run_prep(run_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number):
     ## Check input args are valid
     assert os.path.isdir(run_dir), 'ERROR: could not find the input fim_dir location: ' + str(run_dir)
 
     ## Create an aggregate dataframe with all ras_elev_table.csv entries for hucs in fim_dir
     print('Reading RAS2FIM point loc HAND elevation from ras_elev_table csv files...')
-    csv_name = 'ras_elev_table.csv' # file name to search for ras location data (in the huc/branch dirs)
-    
+    csv_name = 'ras_elev_table.csv'  # file name to search for ras location data (in the huc/branch dirs)
+
     available_cores = multiprocessing.cpu_count()
     if job_number > available_cores:
         job_number = available_cores - 1
-        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+        print(
+            "Provided job number exceeds the number of available cores. "
+            + str(job_number)
+            + " max jobs will be used instead."
+        )
 
     ## Create output dir for log and ras2fim rc database
-    log_dir = os.path.join(run_dir,"logs","src_optimization")
+    log_dir = os.path.join(run_dir, "logs", "src_optimization")
     print("Log file output here: " + str(log_dir))
     if not os.path.isdir(log_dir):
         os.makedirs(log_dir)
@@ -207,17 +327,17 @@ def run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,jo
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
     # Create log file for processing records
-    log_file = open(os.path.join(log_dir,'log_ras2fim_rc_src_adjust.log'),"w")
+    log_file = open(os.path.join(log_dir, 'log_ras2fim_rc_src_adjust.log'), "w")
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
-   
-    ras_elev_df = concat_huc_csv(run_dir,csv_name)
+
+    ras_elev_df = concat_huc_csv(run_dir, csv_name)
 
     if ras_elev_df is None:
         warn_err = 'WARNING: ras_elev_df not created - check that ' + csv_name + ' files exist in fim_dir!'
         print(warn_err)
         log_file.write(warn_err)
-        
+
     elif ras_elev_df.empty:
         warn_err = 'WARNING: ras_elev_df is empty - check that ' + csv_name + ' files exist in fim_dir!'
         print(warn_err)
@@ -229,7 +349,7 @@ def run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,jo
         ras_df = create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir)
 
         ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
-        branch_proc_list(ras_df,run_dir,debug_outputs_option,log_file)
+        branch_proc_list(ras_df, run_dir, debug_outputs_option, log_file)
 
     ## Record run time and close log file
     log_file.write('#########################################################\n\n')
@@ -240,14 +360,35 @@ def run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,jo
     sys.stdout = sys.__stdout__
     log_file.close()
 
+
 if __name__ == '__main__':
     ## Parse arguments.
-    parser = argparse.ArgumentParser(description='Adjusts rating curve with database of RAS2FIM reach average rating curves (calculated WSE/flow).')
-    parser.add_argument('-run_dir','--run-dir',help='Parent directory of FIM run.',required=True)
-    parser.add_argument('-ras_rc','--ras2fim-ratings',help='Path to RAS2FIM rating curve (reach avg) csv file',required=True)
-    parser.add_argument('-nwm_recur','--nwm_recur',help='Path to NWM recur file (multiple NWM flow intervals). NOTE: assumes flow units are cfs!!',required=True)
-    parser.add_argument('-debug','--extra-outputs',help='Optional flag: Use this to keep intermediate output files for debugging/testing',default=False,required=False, action='store_true')
-    parser.add_argument('-j','--job-number',help='Number of jobs to use',required=False,default=1)
+    parser = argparse.ArgumentParser(
+        description='Adjusts rating curve with database of RAS2FIM reach average rating curves'
+        '(calculated WSE/flow).'
+    )
+    parser.add_argument('-run_dir', '--run-dir', help='Parent directory of FIM run.', required=True)
+    parser.add_argument(
+        '-ras_rc',
+        '--ras2fim-ratings',
+        help='Path to RAS2FIM rating curve (reach avg) csv file',
+        required=True,
+    )
+    parser.add_argument(
+        '-nwm_recur',
+        '--nwm_recur',
+        help='Path to NWM recur file (multiple NWM flow intervals). NOTE: assumes flow units are cfs!!',
+        required=True,
+    )
+    parser.add_argument(
+        '-debug',
+        '--extra-outputs',
+        help='Optional flag: Use this to keep intermediate output files for debugging/testing',
+        default=False,
+        required=False,
+        action='store_true',
+    )
+    parser.add_argument('-j', '--job-number', help='Number of jobs to use', required=False, default=1)
 
     ## Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -258,5 +399,4 @@ def run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,jo
     job_number = int(args['job_number'])
 
     ## Prepare/check inputs, create log file, and spin up the proc list
-    run_prep(run_dir,ras_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number)
-    
+    run_prep(run_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number)
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index df41e40cb..8b39dec45 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -2,49 +2,62 @@
 
 import argparse
 import datetime as dt
-import geopandas as gpd
 import multiprocessing
 import os
-import rasterio
 import sys
+from multiprocessing import Pool
 
+import geopandas as gpd
+import rasterio
 from dotenv import load_dotenv
-from multiprocessing import Pool
+
 from src_roughness_optimization import update_rating_curve
-from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH, DEFAULT_FIM_PROJECTION_CRS
+from utils.shared_variables import (
+    DEFAULT_FIM_PROJECTION_CRS,
+    DOWNSTREAM_THRESHOLD,
+    ROUGHNESS_MAX_THRESH,
+    ROUGHNESS_MIN_THRESH,
+)
+
 
 # Import variables from .env file
 load_dotenv('/foss_fim/src/bash_variables.env')
-outputsDir             = os.getenv("outputsDir")
+outputsDir = os.getenv("outputsDir")
 input_calib_points_dir = os.getenv("input_calib_points_dir")
 
 '''
-The script imports .parquet files per HUC8 containing observed FIM extent points and associated flow data. 
-This script attributes the point data with its hydroid and HAND values before passing a dataframe to the src_roughness_optimization.py workflow.
+The script imports .parquet files per HUC8 containing observed FIM extent points and associated flow data.
+This script attributes the point data with its hydroid and HAND values before passing a dataframe to the
+    src_roughness_optimization.py workflow.
 
 Processing
 - Define CRS to use for initial geoprocessing and read wbd_path and points_layer.
 - Define paths to hydroTable.csv, HAND raster, catchments raster, and synthetic rating curve JSON.
 - Clip the points water_edge_df to the huc cathments polygons (for faster processing?)
-- Define coords variable to be used in point raster value attribution and use point geometry to determine catchment raster pixel values
-- Check that there are valid obs in the water_edge_df (not empty) and convert pandas series to dataframe to pass to update_rating_curve
+- Define coords variable to be used in point raster value attribution and use point geometry to
+    determine catchment raster pixel values
+- Check that there are valid obs in the water_edge_df (not empty) and convert pandas series to dataframe
+    to pass to update_rating_curve
 - Call update_rating_curve() to perform the rating curve calibration.
 
 Inputs
-- points_layer:         .gpkg layer containing observed/truth FIM extent points and associated flow value 
-- fim_directory:        fim directory containing individual HUC output dirs
-- wbd_path:             path the watershed boundary dataset layer (HUC polygon boundaries)
-- job_number:           number of multi-processing jobs to use
-- debug_outputs_option: optional flag to output intermediate files for reviewing/debugging
+- points_layer:            .gpkg layer containing observed/truth FIM extent points and associated flow value
+- fim_directory:           fim directory containing individual HUC output dirs
+- wbd_path:                path the watershed boundary dataset layer (HUC polygon boundaries)
+- job_number:              number of multi-processing jobs to use
+- debug_outputs_option:    optional flag to output intermediate files for reviewing/debugging
 
 Outputs
-- water_edge_median_df: dataframe containing "hydroid", "flow", "submitter", "coll_time", "flow_unit", "layer", and median "HAND" value
+- water_edge_median_df:    dataframe containing:
+                                "hydroid", "flow", "submitter", "coll_time", "flow_unit",
+                                "layer", and median "HAND" value
 '''
 
+
 def process_points(args):
     '''
-    This function ingests geodataframe and attributes the point data with its hydroid and HAND values before passing 
-    a dataframe to the src_roughness_optimization.py workflow.
+    This function ingests geodataframe and attributes the point data with its hydroid and HAND values
+    before passing a dataframe to the src_roughness_optimization.py workflow.
 
     Processing
     - Extract x,y coordinates from geometry
@@ -64,31 +77,41 @@ def process_points(args):
     optional_outputs = args[8]
 
     ## Define coords variable to be used in point raster value attribution.
-    coords = [(x,y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
+    coords = [(x, y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
 
     water_edge_df = water_edge_df.to_crs(DEFAULT_FIM_PROJECTION_CRS)
-    
+
     ## Use point geometry to determine HAND raster pixel values.
     with rasterio.open(hand_path) as hand_src, rasterio.open(catchments_path) as catchments_src:
         water_edge_df['hand'] = [h[0] for h in hand_src.sample(coords)]
         water_edge_df['hydroid'] = [c[0] for c in catchments_src.sample(coords)]
 
-    water_edge_df = water_edge_df[(water_edge_df['hydroid'].notnull()) & (water_edge_df['hand'] > 0) & (water_edge_df['hydroid'] > 0)]
+    water_edge_df = water_edge_df[
+        (water_edge_df['hydroid'].notnull()) & (water_edge_df['hand'] > 0) & (water_edge_df['hydroid'] > 0)
+    ]
 
     ## Check that there are valid obs in the water_edge_df (not empty)
     if water_edge_df.empty:
-        log_text = 'NOTE --> skipping HUC: ' + str(huc) + '  Branch: ' + str(branch_id)\
+        log_text = (
+            'NOTE --> skipping HUC: '
+            + str(huc)
+            + '  Branch: '
+            + str(branch_id)
             + ': no valid observation points found within the branch catchments'
+        )
     else:
         ## Intermediate output for debugging
         if optional_outputs:
-            branch_debug_pts_out_gpkg = os.path.join(branch_dir, 'export_water_edge_df_'
-                                                     + branch_id + '.gpkg')
+            branch_debug_pts_out_gpkg = os.path.join(
+                branch_dir, 'export_water_edge_df_' + branch_id + '.gpkg'
+            )
             water_edge_df.to_file(branch_debug_pts_out_gpkg, driver='GPKG', index=False)
-            
-        #print('Processing points for HUC: ' + str(huc) + '  Branch: ' + str(branch_id))
+
+        # print('Processing points for HUC: ' + str(huc) + '  Branch: ' + str(branch_id))
         ## Get median HAND value for appropriate groups.
-        water_edge_median_ds = water_edge_df.groupby(["hydroid", "flow", "submitter", "coll_time", "flow_unit","layer"])['hand'].median()
+        water_edge_median_ds = water_edge_df.groupby(
+            ["hydroid", "flow", "submitter", "coll_time", "flow_unit", "layer"]
+        )['hand'].median()
 
         ## Write user_supplied_n_vals to CSV for next step.
         pt_n_values_csv = os.path.join(branch_dir, 'user_supplied_n_vals_' + branch_id + '.csv')
@@ -99,28 +122,38 @@ def process_points(args):
         del water_edge_median_ds
 
         ## Additional arguments for src_roughness_optimization
-        source_tag = 'point_obs' # tag to use in source attribute field
-        merge_prev_adj = True # merge in previous SRC adjustment calculations
+        source_tag = 'point_obs'  # tag to use in source attribute field
+        merge_prev_adj = True  # merge in previous SRC adjustment calculations
 
         ## Call update_rating_curve() to perform the rating curve calibration.
-        log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc, 
-                                       branch_id, catchments_poly_path, optional_outputs, 
-                                       source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
-        
+        log_text = update_rating_curve(
+            branch_dir,
+            water_edge_median_df,
+            htable_path,
+            huc,
+            branch_id,
+            catchments_poly_path,
+            optional_outputs,
+            source_tag,
+            merge_prev_adj,
+            DOWNSTREAM_THRESHOLD,
+        )
+
         ## Still testing: use code below to print out any exceptions.
         '''
         try:
-            log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc, catchments_poly_path, optional_outputs, source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
+            log_text = update_rating_curve(branch_dir, water_edge_median_df, htable_path, huc,
+                catchments_poly_path, optional_outputs, source_tag, merge_prev_adj, DOWNSTREAM_THRESHOLD)
         except Exception as e:
             print(str(huc) + ' --> ' + str(e))
             log_text = 'ERROR!!!: HUC ' + str(huc) + ' --> ' + str(e)
         '''
-    return(log_text)
+    return log_text
 
 
 def find_points_in_huc(huc_id):
     '''
-    This function loads the .parquet file containing all points attributed with the input huc id into a GeoDataFrame.
+    This function loads the .parquet file containing points attributed with the input huc id into a GDataFrame
 
     Processing
     - Query the <input_calib_points_dir> directory for a <HUC8>.parquet file containing calibration points.
@@ -133,35 +166,36 @@ def find_points_in_huc(huc_id):
     - water_edge_df: geodataframe with point data
     '''
 
-    # The CRS Projection is set when initially writing .parquet files (call to script: write_parquet_from_calib_pts.py)
     water_edge_filepath = os.path.join(input_calib_points_dir, f'{huc_id}.parquet')
-    
+
     water_edge_df = gpd.read_parquet(water_edge_filepath)
-    
+
     return water_edge_df
 
 
 def find_hucs_with_points(points_file_dir, fim_out_huc_list):
     '''
-    This function queries a directory with .parquet files of HUCs containing calibration points (generated from /data/write_parquet_from_calib_pts.py)
-    and returns a list of all the HUCs in <fim_out_huc_list> that contain calibration point data.
+    This function queries a directory with .parquet files of HUCs containing calibration points
+    (generated from /data/write_parquet_from_calib_pts.py) and returns a list of all the HUCs in
+    <fim_out_huc_list> that contain calibration point data.
     '''
 
     files_in_points_file_dir = os.listdir(points_file_dir)
-    
-    # Use list comprehension to slice .parquet off filename, and also prune non-parquet files in directory 
+
+    # Use list comprehension to slice .parquet off filename, and also prune non-parquet files in directory
     hucs_in_points_file_dir = [i[:-8] for i in files_in_points_file_dir if i.endswith('.parquet')]
-    
+
     # Use list comprehension to only keep hucs in both the points_file_dir & fim_out_huc_list
     hucs_wpoints = [x for x in hucs_in_points_file_dir if x in fim_out_huc_list]
-    
+
     return hucs_wpoints
 
 
 def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_file):
     '''
-    The function obtains all points within a given huc, locates the corresponding FIM output files for each huc
-    (confirms all necessary files exist), and then passes a proc list of huc organized data to process_points function.
+    The function obtains all points within a given huc, locates the corresponding FIM output files
+    for each huc (confirms all necessary files exist), and then passes a proc list of
+    huc organized data to process_points function.
 
     Inputs
     - fim_directory:        parent directory of fim ouputs (contains HUC directories)
@@ -172,7 +206,7 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
     Processing
     - Query the <input_calib_points_dir> for all unique huc ids that have calb points
     - Loop through all HUCs with calib data and locate necessary fim output files to pass to calib workflow
-    
+
     - procs_list:           passes multiprocessing list of input args for process_points function input
 
     Outputs
@@ -180,7 +214,9 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
     '''
 
     print("Finding all fim_output hucs that contain calibration points...")
-    fim_out_huc_list  = [ item for item in os.listdir(fim_directory) if os.path.isdir(os.path.join(fim_directory, item)) ]
+    fim_out_huc_list = [
+        item for item in os.listdir(fim_directory) if os.path.isdir(os.path.join(fim_directory, item))
+    ]
 
     # Remove logs, unit_errors, and branch_errors folders if they exist in <fim_directory>
     fim_out_huc_list.remove('logs')
@@ -188,15 +224,15 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
         fim_out_huc_list.remove('unit_errors')
     if 'branch_errors' in fim_out_huc_list:
         fim_out_huc_list.remove('branch_errors')
-    
+
     ## Record run time and close log file
     run_time_start = dt.datetime.now()
     log_file.write('Finding all hucs that contain calibration points...' + '\n')
     huc_list_db = find_hucs_with_points(input_calib_points_dir, fim_out_huc_list)
-    
+
     run_time_end = dt.datetime.now()
     task_run_time = run_time_end - run_time_start
-    
+
     log_file.write('HUC SEARCH TASK RUN TIME: ' + str(task_run_time) + '\n')
     print(f"{len(huc_list_db)} hucs found in point file directory" + '\n')
     log_file.write(f"{len(huc_list_db)} hucs found in point file directory" + '\n')
@@ -217,7 +253,7 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
 
     # huc_list = ['12040103'] # Uncomment for testing
     # Sort huc_list for helping track progress in future print statments
-    huc_list.sort() 
+    huc_list.sort()
     ## Define paths to relevant HUC HAND data.
     for huc in huc_list:
         huc_branches_dir = os.path.join(fim_directory, huc, 'branches')
@@ -231,88 +267,158 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
 
         ## Check to make sure the HUC directory exists in the current fim_directory
         if not os.path.exists(os.path.join(fim_directory, huc)):
-            log_file.write("FIM Directory for huc: " + str(huc) + " does not exist --> skipping SRC adjustments for this HUC (obs points found)\n")
-        
+            log_file.write(
+                "FIM Directory for huc: "
+                + str(huc)
+                + " does not exist --> skipping SRC adjustments for this HUC (obs points found)\n"
+            )
+
         ## Intermediate output for debugging
         if debug_outputs_option:
             huc_debug_pts_out = os.path.join(fim_directory, huc, 'debug_water_edge_df_' + huc + '.csv')
             water_edge_df.to_csv(huc_debug_pts_out)
             huc_debug_pts_out_gpkg = os.path.join(fim_directory, huc, 'export_water_edge_df_' + huc + '.gpkg')
             water_edge_df.to_file(huc_debug_pts_out_gpkg, driver='GPKG', index=False)
-            # write parquet file using ".to_parquet() method" 
+            # write parquet file using ".to_parquet() method"
             parquet_filepath = os.path.join(fim_directory, huc, 'debug_water_edge_df_' + huc + '.parquet')
             water_edge_df.to_parquet(parquet_filepath, index=False)
-        
+
         for branch_id in os.listdir(huc_branches_dir):
             branch_dir = os.path.join(huc_branches_dir, branch_id)
             ## Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
             hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
-            catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
+            catchments_path = os.path.join(
+                branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif'
+            )
             htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
-            catchments_poly_path = os.path.join(branch_dir, 
-                'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
+            catchments_poly_path = os.path.join(
+                branch_dir,
+                'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg',
+            )
 
             # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
             if not os.path.exists(hand_path):
-                print("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+                print(
+                    "WARNING: HAND grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: HAND grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             elif not os.path.exists(catchments_path):
-                print("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')        
+                print(
+                    "WARNING: Catchments grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: Catchments grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             elif not os.path.exists(htable_path):
-                print("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+                print(
+                    "WARNING: hydroTable does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: hydroTable does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             else:
-                procs_list.append([branch_dir, huc, branch_id, hand_path, catchments_path, catchments_poly_path, water_edge_df, htable_path, debug_outputs_option])
+                procs_list.append(
+                    [
+                        branch_dir,
+                        huc,
+                        branch_id,
+                        hand_path,
+                        catchments_path,
+                        catchments_poly_path,
+                        water_edge_df,
+                        htable_path,
+                        debug_outputs_option,
+                    ]
+                )
 
     with Pool(processes=job_number) as pool:
-                log_output = pool.map(process_points, procs_list)
-                log_file.writelines(["%s\n" % item  for item in log_output])
-                
+        log_output = pool.map(process_points, procs_list)
+        log_file.writelines(["%s\n" % item for item in log_output])
+
     log_file.write('#########################################################\n')
 
 
 def run_prep(fim_directory, debug_outputs_option, ds_thresh_override, DOWNSTREAM_THRESHOLD, job_number):
     '''
     Main function to call the processing functions defined above, with validation, logging, and timing
-    
+
     Validation:
         - fim_directory exists
-        - job_number does not exceed available cpus. 
+        - job_number does not exceed available cpus.
         - ds_thresh_override value is different than defualy and warn user
     '''
 
-    assert os.path.isdir(fim_directory), 'ERROR: could not find the input fim_dir location: '\
-        + str(fim_directory)
+    assert os.path.isdir(fim_directory), 'ERROR: could not find the input fim_dir location: ' + str(
+        fim_directory
+    )
 
     available_cores = multiprocessing.cpu_count()
     if job_number > available_cores:
         job_number = available_cores - 1
-        print("Provided job number exceeds the number of available cores. " \
-            + str(job_number) + " max jobs will be used instead.")
+        print(
+            "Provided job number exceeds the number of available cores. "
+            + str(job_number)
+            + " max jobs will be used instead."
+        )
 
     if ds_thresh_override != DOWNSTREAM_THRESHOLD:
-        print('ALERT!! - Using a downstream distance threshold value ('
-            + str(float(ds_thresh_override)) + 'km) different than the default ('\
-            + str(DOWNSTREAM_THRESHOLD) + 'km) - interpret results accordingly')
+        print(
+            'ALERT!! - Using a downstream distance threshold value ('
+            + str(float(ds_thresh_override))
+            + 'km) different than the default ('
+            + str(DOWNSTREAM_THRESHOLD)
+            + 'km) - interpret results accordingly'
+        )
         DOWNSTREAM_THRESHOLD = float(ds_thresh_override)
 
     ## Create output dir for log file
     output_dir = os.path.join(fim_directory, "logs", "src_optimization")
     if not os.path.isdir(output_dir):
         os.makedirs(output_dir)
-        
+
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
 
     ## Create log file for processing records
     print('This may take a few minutes...')
     sys.__stdout__ = sys.stdout
-    log_file = open(os.path.join(output_dir,'log_spatial_src_adjust.log'),"w")
+    log_file = open(os.path.join(output_dir, 'log_spatial_src_adjust.log'), "w")
     log_file.write('#########################################################\n')
-    log_file.write('Parameter Values:\n' + 'DOWNSTREAM_THRESHOLD = ' + str(DOWNSTREAM_THRESHOLD)\
-        + '\n' + 'ROUGHNESS_MIN_THRESH = ' + str( ROUGHNESS_MIN_THRESH) + '\n'\
-        + 'ROUGHNESS_MAX_THRESH = ' + str(ROUGHNESS_MAX_THRESH) + '\n')
+    log_file.write(
+        'Parameter Values:\n'
+        + 'DOWNSTREAM_THRESHOLD = '
+        + str(DOWNSTREAM_THRESHOLD)
+        + '\n'
+        + 'ROUGHNESS_MIN_THRESH = '
+        + str(ROUGHNESS_MIN_THRESH)
+        + '\n'
+        + 'ROUGHNESS_MAX_THRESH = '
+        + str(ROUGHNESS_MAX_THRESH)
+        + '\n'
+    )
     log_file.write('#########################################################\n\n')
     log_file.write('START TIME: ' + str(begin_time) + '\n')
 
@@ -329,25 +435,37 @@ def run_prep(fim_directory, debug_outputs_option, ds_thresh_override, DOWNSTREAM
 
 if __name__ == '__main__':
     ## Parse arguments.
-    parser = argparse.ArgumentParser(description=f'Adjusts rating curve based on files in {input_calib_points_dir}, containing points of known water boundary.')
-    parser.add_argument('-fim_dir', '--fim-directory',
-                        help='Parent directory of FIM-required datasets.', required=True)
-    parser.add_argument('-debug', '--extra-outputs',
-                        help='OPTIONAL flag: Use this to keep intermediate output files for debugging/testing',
-                        default=False, required=False, action='store_true')
-    parser.add_argument('-dthresh', '--downstream-thresh',
-                        help='OPTIONAL Override: distance in km to propogate modified roughness values downstream', 
-                        default=DOWNSTREAM_THRESHOLD, required=False)
-    parser.add_argument('-j', '--job-number',
-                        help='OPTIONAL: Number of jobs to use', type=int, required=False,
-                        default=2)
+    parser = argparse.ArgumentParser(
+        description=f'Adjusts rating curve based on files in {input_calib_points_dir}, '
+        'containing points of known water boundary.'
+    )
+    parser.add_argument(
+        '-fim_dir', '--fim-directory', help='Parent directory of FIM-required datasets.', required=True
+    )
+    parser.add_argument(
+        '-debug',
+        '--extra-outputs',
+        help='OPTIONAL flag: Use this to keep intermediate output files for debugging/testing',
+        default=False,
+        required=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-dthresh',
+        '--downstream-thresh',
+        help='OPTIONAL Override: distance in km to propogate modified roughness values downstream',
+        default=DOWNSTREAM_THRESHOLD,
+        required=False,
+    )
+    parser.add_argument(
+        '-j', '--job-number', help='OPTIONAL: Number of jobs to use', type=int, required=False, default=2
+    )
 
     ## Assign variables from arguments.
-    args                 = vars(parser.parse_args())
-    fim_directory        = args['fim_directory']
+    args = vars(parser.parse_args())
+    fim_directory = args['fim_directory']
     debug_outputs_option = args['extra_outputs']
-    ds_thresh_override   = args['downstream_thresh']
-    job_number           = args['job_number']
+    ds_thresh_override = args['downstream_thresh']
+    job_number = args['job_number']
 
-    run_prep(fim_directory, debug_outputs_option, ds_thresh_override, 
-             DOWNSTREAM_THRESHOLD, job_number)
+    run_prep(fim_directory, debug_outputs_option, ds_thresh_override, DOWNSTREAM_THRESHOLD, job_number)
diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py
index 58ab34f12..d4684a8e5 100644
--- a/src/src_adjust_usgs_rating.py
+++ b/src/src_adjust_usgs_rating.py
@@ -1,17 +1,22 @@
 import argparse
+import datetime as dt
+import json
+import multiprocessing
 import os
-import pandas as pd
 import sys
-import json
-import datetime as dt
-from pathlib import Path
 from collections import deque
-import multiprocessing
 from multiprocessing import Pool
-from utils.shared_functions import check_file_age, concat_huc_csv
+from pathlib import Path
+
+import pandas as pd
+
 from src_roughness_optimization import update_rating_curve
+from utils.shared_functions import check_file_age, concat_huc_csv
+
+
 '''
-The script ingests a USGS rating curve csv and a NWM flow recurrence interval database. The gage location will be associated to the corresponding hydroID and attributed with the HAND elevation value
+The script ingests a USGS rating curve csv and a NWM flow recurrence interval database.
+The gage location will be associated to the corresponding hydroID and attributed with the HAND elevation value
 
 Processing
 - Read in USGS rating curve from csv and convert WSE navd88 values to meters
@@ -27,186 +32,300 @@
 - Call update_rating_curve() to perform the rating curve calibration.
 
 Inputs
-- branch_dir:        fim directory containing individual HUC output dirs
+- branch_dir:           fim directory containing individual HUC output dirs
 - usgs_rc_filepath:     USGS rating curve database (produced by rating_curve_get_usgs_curves.py)
 - nwm_recurr_filepath:  NWM flow recurrence interval dataset
 - debug_outputs_option: optional flag to output intermediate files for reviewing/debugging
 - job_number:           number of multi-processing jobs to use
 
 Outputs
-- water_edge_median_ds: dataframe containing 'location_id','hydroid','feature_id','huc','hand','discharge_cms','nwm_recur_flow_cms','nwm_recur','layer'
+- water_edge_median_ds: dataframe containing:
+                            'location_id', 'hydroid', 'feature_id', 'huc', 'hand', 'discharge_cms',
+                            'nwm_recur_flow_cms', 'nwm_recur', 'layer'
 '''
 
+
 def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir):
     start_time = dt.datetime.now()
     print('Reading USGS rating curve from csv...')
     log_text = 'Processing database for USGS flow/WSE at NWM flow recur intervals...\n'
     col_usgs = ["location_id", "flow", "stage", "elevation_navd88"]
-    usgs_rc_df = pd.read_csv(usgs_rc_filepath, dtype={'location_id': object}, usecols=col_usgs)#, nrows=30000)
+    usgs_rc_df = pd.read_csv(
+        usgs_rc_filepath, dtype={'location_id': object}, usecols=col_usgs
+    )  # , nrows=30000)
     print('Duration (read usgs_rc_csv): {}'.format(dt.datetime.now() - start_time))
-    
+
     # convert WSE navd88 values to meters
     usgs_rc_df['elevation_navd88_m'] = usgs_rc_df['elevation_navd88'] / 3.28084
-    
+
     # read in the aggregate USGS elev table csv
     start_time = dt.datetime.now()
-    cross_df = usgs_elev_df[["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_adj_elevation"]].copy()
-    cross_df.rename(columns={'dem_adj_elevation':'hand_datum', 'HydroID':'hydroid', 'HUC8':'huc'}, inplace=True)
-    
-    # filter null location_id rows from cross_df (removes ahps lide entries that aren't associated with USGS gage)
+    cross_df = usgs_elev_df[
+        ["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_adj_elevation"]
+    ].copy()
+    cross_df.rename(
+        columns={'dem_adj_elevation': 'hand_datum', 'HydroID': 'hydroid', 'HUC8': 'huc'}, inplace=True
+    )
+
+    # filter null location_id rows from cross_df
+    # (removes ahps lide entries that aren't associated with USGS gage)
     cross_df = cross_df[cross_df.location_id.notnull()]
-    
+
     # convert usgs flow from cfs to cms
     usgs_rc_df['discharge_cms'] = usgs_rc_df.flow / 35.3147
     usgs_rc_df = usgs_rc_df.drop(columns=["flow"])
-    
+
     # merge usgs ratings with crosswalk attributes
     usgs_rc_df = usgs_rc_df.merge(cross_df, how='left', on='location_id')
     usgs_rc_df = usgs_rc_df[usgs_rc_df['hydroid'].notna()]
-    
+
     # calculate hand elevation
     usgs_rc_df['hand'] = usgs_rc_df['elevation_navd88_m'] - usgs_rc_df['hand_datum']
-    usgs_rc_df = usgs_rc_df[['location_id','feature_id','hydroid','levpa_id','huc','hand','discharge_cms']]
+    usgs_rc_df = usgs_rc_df[
+        ['location_id', 'feature_id', 'hydroid', 'levpa_id', 'huc', 'hand', 'discharge_cms']
+    ]
     usgs_rc_df['feature_id'] = usgs_rc_df['feature_id'].astype(int)
-    
+
     # read in the NWM recurr csv file
     nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
     nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
-    nwm_recur_df.rename(columns={'2_0_year_recurrence_flow_17C':'2_0_year','5_0_year_recurrence_flow_17C':'5_0_year','10_0_year_recurrence_flow_17C':'10_0_year','25_0_year_recurrence_flow_17C':'25_0_year','50_0_year_recurrence_flow_17C':'50_0_year','100_0_year_recurrence_flow_17C':'100_0_year'}, inplace=True)
-    
-    #convert cfs to cms (x 0.028317)
-    nwm_recur_df.loc[:, ['2_0_year','5_0_year','10_0_year','25_0_year','50_0_year','100_0_year']] *= 0.028317
-    
+    nwm_recur_df.rename(
+        columns={
+            '2_0_year_recurrence_flow_17C': '2_0_year',
+            '5_0_year_recurrence_flow_17C': '5_0_year',
+            '10_0_year_recurrence_flow_17C': '10_0_year',
+            '25_0_year_recurrence_flow_17C': '25_0_year',
+            '50_0_year_recurrence_flow_17C': '50_0_year',
+            '100_0_year_recurrence_flow_17C': '100_0_year',
+        },
+        inplace=True,
+    )
+
+    # convert cfs to cms (x 0.028317)
+    nwm_recur_df.loc[
+        :, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year', '100_0_year']
+    ] *= 0.028317
+
     # merge nwm recurr with usgs_rc
     merge_df = usgs_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
-    
+
     # NWM recurr intervals
-    recurr_intervals = ("2","5","10","25","50","100")
-    final_df = pd.DataFrame() # create empty dataframe to append flow interval dataframes
+    recurr_intervals = ("2", "5", "10", "25", "50", "100")
+    final_df = pd.DataFrame()  # create empty dataframe to append flow interval dataframes
     for interval in recurr_intervals:
-        log_text += ('\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n')
+        log_text += '\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n'
         print('Processing: ' + str(interval) + '-year NWM recurr intervals')
         ## Calculate the closest SRC discharge value to the NWM flow value
-        merge_df['Q_find'] = (merge_df['discharge_cms'] - merge_df[interval+"_0_year"]).abs()
-        
+        merge_df['Q_find'] = (merge_df['discharge_cms'] - merge_df[interval + "_0_year"]).abs()
+
         ## Check for any missing/null entries in the input SRC
-        if merge_df['Q_find'].isnull().values.any(): # there may be null values for lake or coastal flow lines (need to set a value to do groupby idxmin below)
-            log_text += 'HUC: ' + str(merge_df['huc']) + ' : feature_id' + str(merge_df['feature_id']) + ' --> Null values found in "Q_find" calc. These will be filled with 999999 () \n'
+        # There may be null values for lake or coastal flow lines
+        # (need to set a value to do groupby idxmin below)
+        if merge_df['Q_find'].isnull().values.any():
+            log_text += (
+                'HUC: '
+                + str(merge_df['huc'])
+                + ' : feature_id'
+                + str(merge_df['feature_id'])
+                + ' --> Null values found in "Q_find" calc. These will be filled with 999999 () \n'
+            )
             ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
             merge_df['Q_find'] = merge_df['Q_find'].fillna(999999)
         if merge_df['hydroid'].isnull().values.any():
             log_text += 'HUC: ' + str(merge_df['huc']) + ' --> Null values found in "hydroid"... \n'
-        
+
         # Create dataframe with crosswalked USGS flow and NWM recurr flow
-        calc_df = merge_df.loc[merge_df.groupby(['location_id','levpa_id'])['Q_find'].idxmin()].reset_index(drop=True) # find the index of the Q_1_5_find (closest matching flow)
-        # Calculate flow difference (variance) to check for large discrepancies btw NWM flow and USGS closest flow
-        calc_df['check_variance'] = ((calc_df['discharge_cms'] - calc_df[interval+"_0_year"])/calc_df['discharge_cms']).abs()
+        calc_df = merge_df.loc[merge_df.groupby(['location_id', 'levpa_id'])['Q_find'].idxmin()].reset_index(
+            drop=True
+        )  # find the index of the Q_1_5_find (closest matching flow)
+        # Calculate flow difference (variance) to check for large discrepancies between
+        # NWM flow and USGS closest flow
+        calc_df['check_variance'] = (
+            (calc_df['discharge_cms'] - calc_df[interval + "_0_year"]) / calc_df['discharge_cms']
+        ).abs()
         # Assign new metadata attributes
-        calc_df['nwm_recur'] = interval+"_0_year"
-        calc_df['layer'] = '_usgs-gage____' + interval+"-year"
-        calc_df.rename(columns={interval+"_0_year":'nwm_recur_flow_cms'}, inplace=True)
+        calc_df['nwm_recur'] = interval + "_0_year"
+        calc_df['layer'] = '_usgs-gage____' + interval + "-year"
+        calc_df.rename(columns={interval + "_0_year": 'nwm_recur_flow_cms'}, inplace=True)
         # Subset calc_df for final output
-        calc_df = calc_df[['location_id','hydroid','feature_id','levpa_id','huc','hand','discharge_cms','check_variance','nwm_recur_flow_cms','nwm_recur','layer']]
+        calc_df = calc_df[
+            [
+                'location_id',
+                'hydroid',
+                'feature_id',
+                'levpa_id',
+                'huc',
+                'hand',
+                'discharge_cms',
+                'check_variance',
+                'nwm_recur_flow_cms',
+                'nwm_recur',
+                'layer',
+            ]
+        ]
         final_df = pd.concat([final_df, calc_df], ignore_index=True)
         # Log any negative HAND elev values and remove from database
-        log_text += ('Warning: Negative HAND stage values -->\n')
-        log_text += (calc_df[calc_df['hand']<0].to_string() +'\n')
-        final_df = final_df[final_df['hand']>0]
-        # Log any signifant differences btw the NWM flow value and closest USGS rating flow (this ensures that we consistently sample the USGS rating curves at known intervals - NWM recur flow)
-        log_text += ('Warning: Large variance (>10%) between NWM flow and closest USGS flow -->\n')
-        log_text += (calc_df[calc_df['check_variance']>0.1].to_string() +'\n')
-        final_df = final_df[final_df['check_variance']<0.1]
+        log_text += 'Warning: Negative HAND stage values -->\n'
+        log_text += calc_df[calc_df['hand'] < 0].to_string() + '\n'
+        final_df = final_df[final_df['hand'] > 0]
+        # Log any signifant differences btw the NWM flow value and closest USGS rating flow
+        # This ensures that we consistently sample the USGS rating curves at known intervals - NWM recur flow
+        log_text += 'Warning: Large variance (>10%) between NWM flow and closest USGS flow -->\n'
+        log_text += calc_df[calc_df['check_variance'] > 0.1].to_string() + '\n'
+        final_df = final_df[final_df['check_variance'] < 0.1]
         final_df['submitter'] = 'usgs_rating_wrds_api_' + final_df['location_id']
         # Get datestamp from usgs rating curve file to use as coll_time attribute in hydroTable.csv
         datestamp = check_file_age(usgs_rc_filepath)
         final_df['coll_time'] = str(datestamp)[:15]
 
     # Rename attributes (for ingest to update_rating_curve) and output csv with the USGS RC database
-    final_df.rename(columns={'discharge_cms':'flow'}, inplace=True)  
-    final_df.to_csv(os.path.join(log_dir,"usgs_rc_nwm_recurr.csv"),index=False)
+    final_df.rename(columns={'discharge_cms': 'flow'}, inplace=True)
+    final_df.to_csv(os.path.join(log_dir, "usgs_rc_nwm_recurr.csv"), index=False)
 
     # Output log text to log file
-    log_text += ('#########\nTotal entries per USGS gage location -->\n')
-    loc_id_df = final_df.groupby(['location_id']).size().reset_index(name='count') 
-    log_text += (loc_id_df.to_string() +'\n')
-    log_text += ('#########\nTotal entries per NWM recur value -->\n')
-    recur_count_df = final_df.groupby(['nwm_recur']).size().reset_index(name='count') 
-    log_text += (recur_count_df.to_string() +'\n')
-    log_usgs_db = open(os.path.join(log_dir,'log_usgs_rc_database.log'),"w")
+    log_text += '#########\nTotal entries per USGS gage location -->\n'
+    loc_id_df = final_df.groupby(['location_id']).size().reset_index(name='count')
+    log_text += loc_id_df.to_string() + '\n'
+    log_text += '#########\nTotal entries per NWM recur value -->\n'
+    recur_count_df = final_df.groupby(['nwm_recur']).size().reset_index(name='count')
+    log_text += recur_count_df.to_string() + '\n'
+    log_usgs_db = open(os.path.join(log_dir, 'log_usgs_rc_database.log'), "w")
     log_usgs_db.write(log_text)
     log_usgs_db.close()
-    return(final_df)
+    return final_df
 
-def branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file):
+
+def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
     procs_list = []  # Initialize list for mulitprocessing.
 
     # loop through all unique level paths that have a USGS gage
-    #branch_huc_dict = pd.Series(usgs_df.levpa_id.values,index=usgs_df.huc).to_dict('list')
-    #branch_huc_dict = usgs_df.set_index('huc').T.to_dict('list')
+    # branch_huc_dict = pd.Series(usgs_df.levpa_id.values,index=usgs_df.huc).to_dict('list')
+    # branch_huc_dict = usgs_df.set_index('huc').T.to_dict('list')
     huc_branch_dict = usgs_df.groupby('huc')['levpa_id'].apply(set).to_dict()
 
-    for huc in sorted(huc_branch_dict.keys()): # sort huc_list for helping track progress in future print statments
+    for huc in sorted(
+        huc_branch_dict.keys()
+    ):  # sort huc_list for helping track progress in future print statments
         branch_set = huc_branch_dict[huc]
-        for branch_id in branch_set: 
+        for branch_id in branch_set:
             # Define paths to branch HAND data.
             # Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
             # Assumes outputs are for HUC8 (not HUC6)
-            branch_dir = os.path.join(run_dir,huc,'branches',branch_id)
+            branch_dir = os.path.join(run_dir, huc, 'branches', branch_id)
             hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
-            catchments_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif')
-            catchments_poly_path = os.path.join(branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg')
+            catchments_path = os.path.join(
+                branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif'
+            )
+            catchments_poly_path = os.path.join(
+                branch_dir,
+                'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg',
+            )
             htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
-            water_edge_median_ds = usgs_df[(usgs_df['huc']==huc) & (usgs_df['levpa_id']==branch_id)]
+            water_edge_median_ds = usgs_df[(usgs_df['huc'] == huc) & (usgs_df['levpa_id'] == branch_id)]
 
             # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
             if not os.path.exists(hand_path):
-                print("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: HAND grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+                print(
+                    "WARNING: HAND grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: HAND grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             elif not os.path.exists(catchments_path):
-                print("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: Catchments grid does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')        
+                print(
+                    "WARNING: Catchments grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: Catchments grid does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             elif not os.path.exists(htable_path):
-                print("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id))
-                log_file.write("WARNING: hydroTable does not exist (skipping): " + str(huc) + ' - branch-id: ' + str(branch_id) + '\n')
+                print(
+                    "WARNING: hydroTable does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "WARNING: hydroTable does not exist (skipping): "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
             else:
                 ## Additional arguments for src_roughness_optimization
-                source_tag = 'usgs_rating' # tag to use in source attribute field
-                merge_prev_adj = False # merge in previous SRC adjustment calculations
+                source_tag = 'usgs_rating'  # tag to use in source attribute field
+                merge_prev_adj = False  # merge in previous SRC adjustment calculations
 
                 print('Will perform SRC adjustments for huc: ' + str(huc) + ' - branch-id: ' + str(branch_id))
-                procs_list.append([branch_dir, water_edge_median_ds, htable_path, huc, branch_id, catchments_poly_path, debug_outputs_option, source_tag, merge_prev_adj])
+                procs_list.append(
+                    [
+                        branch_dir,
+                        water_edge_median_ds,
+                        htable_path,
+                        huc,
+                        branch_id,
+                        catchments_poly_path,
+                        debug_outputs_option,
+                        source_tag,
+                        merge_prev_adj,
+                    ]
+                )
 
     # multiprocess all available branches
     print(f"Calculating new SRCs for {len(procs_list)} branches using {job_number} jobs...")
     with Pool(processes=job_number) as pool:
-            log_output = pool.starmap(update_rating_curve, procs_list)
-            log_file.writelines(["%s\n" % item  for item in log_output])
-    # try statement for debugging 
+        log_output = pool.starmap(update_rating_curve, procs_list)
+        log_file.writelines(["%s\n" % item for item in log_output])
+    # try statement for debugging
     # try:
     #     with Pool(processes=job_number) as pool:
     #         log_output = pool.starmap(update_rating_curve, procs_list)
-    #         log_file.writelines(["%s\n" % item  for item in log_output])
+    #         log_file.writelines(["%s\n" % item for item in log_output])
     # except Exception as e:
     #     print(str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e))
-    #     log_file.write('ERROR!!!: HUC ' + str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e) + '\n')
+    #     log_file.write(
+    #         'ERROR!!!: HUC ' + str(huc) + ' --> ' + '  branch id: ' + str(branch_id) + str(e) + '\n'
+    #     )
+
 
-def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number):
+def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number):
     ## Check input args are valid
     assert os.path.isdir(run_dir), 'ERROR: could not find the input fim_dir location: ' + str(run_dir)
 
     ## Create an aggregate dataframe with all usgs_elev_table.csv entries for hucs in fim_dir
     print('Reading USGS gage HAND elevation from usgs_elev_table.csv files...')
-    #usgs_elev_file = os.path.join(branch_dir,'usgs_elev_table.csv')
-    #usgs_elev_df = pd.read_csv(usgs_elev_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int})
+    # usgs_elev_file = os.path.join(branch_dir,'usgs_elev_table.csv')
+    # usgs_elev_df = pd.read_csv(
+    #     usgs_elev_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int}
+    # )
     csv_name = 'usgs_elev_table.csv'
-    
+
     available_cores = multiprocessing.cpu_count()
     if job_number > available_cores:
         job_number = available_cores - 1
-        print("Provided job number exceeds the number of available cores. " + str(job_number) + " max jobs will be used instead.")
+        print(
+            "Provided job number exceeds the number of available cores. "
+            + str(job_number)
+            + " max jobs will be used instead."
+        )
 
     ## Create output dir for log and usgs rc database
-    log_dir = os.path.join(run_dir,"logs","src_optimization")
+    log_dir = os.path.join(run_dir, "logs", "src_optimization")
     print("Log file output here: " + str(log_dir))
     if not os.path.isdir(log_dir):
         os.makedirs(log_dir)
@@ -214,17 +333,19 @@ def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,j
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
     # Create log file for processing records
-    log_file = open(os.path.join(log_dir,'log_usgs_rc_src_adjust.log'),"w")
+    log_file = open(os.path.join(log_dir, 'log_usgs_rc_src_adjust.log'), "w")
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
-   
-    usgs_elev_df = concat_huc_csv(run_dir,csv_name)
+
+    usgs_elev_df = concat_huc_csv(run_dir, csv_name)
 
     if usgs_elev_df is None:
-        warn_err = 'WARNING: usgs_elev_df not created - check that usgs_elev_table.csv files exist in fim_dir!'
+        warn_err = (
+            'WARNING: usgs_elev_df not created - check that usgs_elev_table.csv files exist in fim_dir!'
+        )
         print(warn_err)
         log_file.write(warn_err)
-        
+
     elif usgs_elev_df.empty:
         warn_err = 'WARNING: usgs_elev_df is empty - check that usgs_elev_table.csv files exist in fim_dir!'
         print(warn_err)
@@ -236,7 +357,7 @@ def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,j
         usgs_df = create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir)
 
         ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
-        branch_proc_list(usgs_df,run_dir,debug_outputs_option,log_file)
+        branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file)
 
     ## Record run time and close log file
     log_file.write('#########################################################\n\n')
@@ -247,14 +368,31 @@ def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,j
     sys.stdout = sys.__stdout__
     log_file.close()
 
+
 if __name__ == '__main__':
     ## Parse arguments.
-    parser = argparse.ArgumentParser(description='Adjusts rating curve with database of USGS rating curve (calculated WSE/flow).')
-    parser.add_argument('-run_dir','--run-dir',help='Parent directory of FIM run.',required=True)
-    parser.add_argument('-usgs_rc','--usgs-ratings',help='Path to USGS rating curve csv file',required=True)
-    parser.add_argument('-nwm_recur','--nwm_recur',help='Path to NWM recur file (multiple NWM flow intervals). NOTE: assumes flow units are cfs!!',required=True)
-    parser.add_argument('-debug','--extra-outputs',help='Optional flag: Use this to keep intermediate output files for debugging/testing',default=False,required=False, action='store_true')
-    parser.add_argument('-j','--job-number',help='Number of jobs to use',required=False,default=1)
+    parser = argparse.ArgumentParser(
+        description='Adjusts rating curve with database of USGS rating curve (calculated WSE/flow).'
+    )
+    parser.add_argument('-run_dir', '--run-dir', help='Parent directory of FIM run.', required=True)
+    parser.add_argument(
+        '-usgs_rc', '--usgs-ratings', help='Path to USGS rating curve csv file', required=True
+    )
+    parser.add_argument(
+        '-nwm_recur',
+        '--nwm_recur',
+        help='Path to NWM recur file (multiple NWM flow intervals). NOTE: assumes flow units are cfs!!',
+        required=True,
+    )
+    parser.add_argument(
+        '-debug',
+        '--extra-outputs',
+        help='Optional flag: Use this to keep intermediate output files for debugging/testing',
+        default=False,
+        required=False,
+        action='store_true',
+    )
+    parser.add_argument('-j', '--job-number', help='Number of jobs to use', required=False, default=1)
 
     ## Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -265,5 +403,4 @@ def run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,j
     job_number = int(args['job_number'])
 
     ## Prepare/check inputs, create log file, and spin up the proc list
-    run_prep(run_dir,usgs_rc_filepath,nwm_recurr_filepath,debug_outputs_option,job_number)
-    
+    run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number)
diff --git a/src/src_manual_calibration.py b/src/src_manual_calibration.py
index 6067229b2..ed0012222 100644
--- a/src/src_manual_calibration.py
+++ b/src/src_manual_calibration.py
@@ -1,60 +1,65 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import os
+
 import numpy as np
 import pandas as pd
 
-htable_dtypes = {'HydroID':int,
-                'branch_id':int,
-                'feature_id':int,
-                'NextDownID':int,
-                'order_':int,
-                'Number of Cells':int,
-                'SurfaceArea (m2)':float,
-                'BedArea (m2)':float,
-                'TopWidth (m)':float,
-                'LENGTHKM':float,
-                'AREASQKM':float,
-                'WettedPerimeter (m)':float,
-                'HydraulicRadius (m)':float,
-                'WetArea (m2)':float,
-                'Volume (m3)':float,
-                'SLOPE':float,
-                'ManningN':float,
-                'stage':float,
-                'default_discharge_cms':float,
-                'default_Volume (m3)':float,
-                'default_WetArea (m2)':float,
-                'default_HydraulicRadius (m)':float,
-                'default_ManningN':float,
-                'calb_applied':bool,
-                'last_updated':str,
-                'submitter':str,
-                'obs_source':str,
-                'precalb_discharge_cms':float,
-                'calb_coef_usgs':float,
-                'calb_coef_spatial':float,
-                'calb_coef_final':float,
-                'HUC':str,
-                'LakeID':int,
-                'subdiv_applied':bool,
-                'channel_n':float,
-                'overbank_n':float,
-                'subdiv_discharge_cms':float,
-                'discharge_cms':float}
-
-def manual_calibration(fim_directory:str, calibration_file:str):
+
+htable_dtypes = {
+    'HydroID': int,
+    'branch_id': int,
+    'feature_id': int,
+    'NextDownID': int,
+    'order_': int,
+    'Number of Cells': int,
+    'SurfaceArea (m2)': float,
+    'BedArea (m2)': float,
+    'TopWidth (m)': float,
+    'LENGTHKM': float,
+    'AREASQKM': float,
+    'WettedPerimeter (m)': float,
+    'HydraulicRadius (m)': float,
+    'WetArea (m2)': float,
+    'Volume (m3)': float,
+    'SLOPE': float,
+    'ManningN': float,
+    'stage': float,
+    'default_discharge_cms': float,
+    'default_Volume (m3)': float,
+    'default_WetArea (m2)': float,
+    'default_HydraulicRadius (m)': float,
+    'default_ManningN': float,
+    'calb_applied': bool,
+    'last_updated': str,
+    'submitter': str,
+    'obs_source': str,
+    'precalb_discharge_cms': float,
+    'calb_coef_usgs': float,
+    'calb_coef_spatial': float,
+    'calb_coef_final': float,
+    'HUC': str,
+    'LakeID': int,
+    'subdiv_applied': bool,
+    'channel_n': float,
+    'overbank_n': float,
+    'subdiv_discharge_cms': float,
+    'discharge_cms': float,
+}
+
+
+def manual_calibration(fim_directory: str, calibration_file: str):
     """
     Perform manual calibration.
-     
-    Use a CSV of coefficients to output a new rating curve. 
-    Coefficient values between 0 and 1 increase the discharge value 
-    (and decrease inundation) for each stage in the rating curve while 
-    values greater than 1 decrease the discharge value (and increase 
+
+    Use a CSV of coefficients to output a new rating curve.
+    Coefficient values between 0 and 1 increase the discharge value
+    (and decrease inundation) for each stage in the rating curve while
+    values greater than 1 decrease the discharge value (and increase
     inundation).
-    
-    The original rating curve is saved with a suffix of '_pre-manual' 
+
+    The original rating curve is saved with a suffix of '_pre-manual'
     before the new rating curve is written.
 
     Parameters
@@ -62,30 +67,32 @@ def manual_calibration(fim_directory:str, calibration_file:str):
     fim_directory : str
         Path to the parent directory of FIM-required datasets.
     calibration_file : str
-        Path to the manual calibration file. This file should be a CSV 
+        Path to the manual calibration file. This file should be a CSV
         with the following columns:
             HUC8: str
                 HUC8 code
             feature_id: int
                 NWM feature_id
             calb_coef_manual: float
-                Manual calibration coefficient for each HUC8 and 
+                Manual calibration coefficient for each HUC8 and
                 feature_id combination.
     """
 
     # Get list of HUCs
-    fim_inputs = pd.read_csv(os.path.join(fim_directory, 'fim_inputs.csv'), 
-                             header=None,
-                             names=['HUC', 'branch_id'],
-                             dtype={'HUC':str, 'branch_id':int})
+    fim_inputs = pd.read_csv(
+        os.path.join(fim_directory, 'fim_inputs.csv'),
+        header=None,
+        names=['HUC', 'branch_id'],
+        dtype={'HUC': str, 'branch_id': int},
+    )
 
     fim_inputs_hucs = fim_inputs['HUC'].unique()
 
     # Read manual calibration table
     if os.path.exists(calibration_file):
-        manual_calib_df = pd.read_csv(calibration_file, dtype={'HUC8':str,
-                                                               'feature_id':int,
-                                                               'calb_coef_manual':float})
+        manual_calib_df = pd.read_csv(
+            calibration_file, dtype={'HUC8': str, 'feature_id': int, 'calb_coef_manual': float}
+        )
 
         if manual_calib_df['calb_coef_manual'].min() >= 0:
             # Find HUCs with manual calibration coefficients
@@ -106,38 +113,47 @@ def manual_calibration(fim_directory:str, calibration_file:str):
 
                 df_htable = pd.read_csv(htable_file_original, dtype=htable_dtypes)
 
-                df_htable = df_htable.rename(columns={'discharge_cms':'postcalb_discharge_cms'})
+                df_htable = df_htable.rename(columns={'discharge_cms': 'postcalb_discharge_cms'})
 
                 df_htable = df_htable.merge(manual_calib_df, how='left', on='feature_id')
                 df_htable.drop(columns=['HUC8'], inplace=True)
 
                 # Calculate new discharge_cms with manual calibration coefficient
-                df_htable['discharge_cms'] = np.where(df_htable['calb_coef_manual'].isnull(), 
-                                                      df_htable['postcalb_discharge_cms'], 
-                                                      df_htable['postcalb_discharge_cms']/df_htable['calb_coef_manual'])
+                df_htable['discharge_cms'] = np.where(
+                    df_htable['calb_coef_manual'].isnull(),
+                    df_htable['postcalb_discharge_cms'],
+                    df_htable['postcalb_discharge_cms'] / df_htable['calb_coef_manual'],
+                )
 
                 # Write new hydroTable.csv rating curve (overwrites the previous file)
                 df_htable.to_csv(htable_file, index=False)
 
         else:
-            raise ValueError(f'Manual calibration coefficients must be greater than 0. Minimum value found: {manual_calib_df["calb_coef_manual"].min()}')
+            raise ValueError(
+                'Manual calibration coefficients must be greater than 0. Minimum value found: '
+                f'{manual_calib_df["calb_coef_manual"].min()}'
+            )
 
     else:
-        raise FileNotFoundError(f'No calibration file found at {calibration_file}. Skipping manual calibration.')
+        raise FileNotFoundError(
+            f'No calibration file found at {calibration_file}. Skipping manual calibration.'
+        )
 
 
 if __name__ == '__main__':
     ## Parse arguments.
-    parser = argparse.ArgumentParser(description=f'Manually calibrate rating curve')
-    parser.add_argument('-fim_dir', '--fim-directory',
-                        help='Parent directory of FIM-required datasets.', required=True)
-    parser.add_argument('-calb_file', '--calibration-file',
-                        help='Path to manual calibration file', required=True)
+    parser = argparse.ArgumentParser(description='Manually calibrate rating curve')
+    parser.add_argument(
+        '-fim_dir', '--fim-directory', help='Parent directory of FIM-required datasets.', required=True
+    )
+    parser.add_argument(
+        '-calb_file', '--calibration-file', help='Path to manual calibration file', required=True
+    )
 
     # args = parser.parse_args()
-    args                 = vars(parser.parse_args())
-    fim_directory        = args['fim_directory']
-    calibration_file     = args['calibration_file']
+    args = vars(parser.parse_args())
+    fim_directory = args['fim_directory']
+    calibration_file = args['calibration_file']
 
     manual_calibration(fim_directory, calibration_file)
 
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index db667e42e..e294cd56f 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -1,69 +1,120 @@
 import argparse
-import geopandas as gpd
-from geopandas.tools import sjoin
+import datetime as dt
+import json
+import multiprocessing
 import os
-import rasterio
-import pandas as pd
-import numpy as np
 import sys
-import json
-import datetime as dt
 from collections import deque
-import multiprocessing
 from multiprocessing import Pool
 
-from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MIN_THRESH, ROUGHNESS_MAX_THRESH
-
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
+from geopandas.tools import sjoin
 
-def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, branch_id, catchments_poly_path, debug_outputs_option, source_tag, merge_prev_adj=False, down_dist_thresh=DOWNSTREAM_THRESHOLD):
+from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MAX_THRESH, ROUGHNESS_MIN_THRESH
+
+
+def update_rating_curve(
+    fim_directory,
+    water_edge_median_df,
+    htable_path,
+    huc,
+    branch_id,
+    catchments_poly_path,
+    debug_outputs_option,
+    source_tag,
+    merge_prev_adj=False,
+    down_dist_thresh=DOWNSTREAM_THRESHOLD,
+):
     '''
-    This script ingests a dataframe containing observed data (HAND elevation and flow) and then calculates new SRC roughness values via Manning's equation. The new roughness values are averaged for each HydroID and then progated downstream and a new discharge value is calculated where applicable.
+    This script ingests a dataframe containing observed data (HAND elevation and flow) and
+    calculates new SRC roughness values via Manning's equation.
+    The new roughness values are averaged for each HydroID and then progated downstream and
+    a new discharge value is calculated where applicable.
 
     Processing Steps:
-    - Read in the hydroTable.csv and check whether it has previously been updated (rename default columns if needed)
-    - Loop through the user provided point data --> stage/flow dataframe row by row and copy the corresponding htable values for the matching stage->HAND lookup
+    - Read in the hydroTable.csv and check whether it has previously been updated
+        (rename default columns if needed)
+    - Loop through the user provided point data --> stage/flow dataframe row by row and copy the corresponding
+        htable values for the matching stage->HAND lookup
     - Calculate new HydroID roughness values for input obs data using Manning's equation
-    - Create dataframe to check for erroneous Manning's n values (values set in tools_shared_variables.py: >0.6 or <0.001 --> see input args)
+    - Create dataframe to check for erroneous Manning's n values
+        (values set in tools_shared_variables.py: >0.6 or <0.001 --> see input args)
     - Create magnitude and ahps column by subsetting the "layer" attribute
-    - Create df grouped by hydroid with ahps_lid and huc number and then pivot the magnitude column to display n value for each magnitude at each hydroid
+    - Create df grouped by hydroid with ahps_lid and huc number and then pivot the magnitude column to display
+        n value for each magnitude at each hydroid
     - Create df with the most recent collection time entry and submitter attribs
-    - Cacluate median ManningN to handle cases with multiple hydroid entries and create a df with the median hydroid_ManningN value per feature_id
-    - Rename the original hydrotable variables to allow new calculations to use the primary var name 
-    - Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a single hydroid)
+    - Cacluate median ManningN to handle cases with multiple hydroid entries and create a df with the median
+        hydroid_ManningN value per feature_id
+    - Rename the original hydrotable variables to allow new calculations to use the primary var name
+    - Check for large variabilty in the calculated Manning's N values
+        (for cases with mutliple entries for a single hydroid)
     - Create attributes to traverse the flow network between HydroIDs
-    - Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg)
-    - Create the adjust_ManningN column by combining the hydroid_ManningN with the featid_ManningN (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
-    - Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new adjusted roughness value
+    - Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to
+        non-calb hydroids (constrained to first Xkm of hydroids -
+        set downstream diststance var as input arg)
+    - Create the adjust_ManningN column by combining the hydroid_ManningN with the featid_ManningN
+        (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
+    - Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new adjusted
+        roughness value
     - Update the catchments polygon .gpkg with joined attribute - "src_calibrated"
     - Merge the final ManningN dataframe to the original hydroTable
-    - Create the ManningN column by combining the hydroid_ManningN with the default_ManningN (use modified where available)
+    - Create the ManningN column by combining the hydroid_ManningN with the default_ManningN
+        (use modified where available)
     - Calculate new discharge_cms with new adjusted ManningN
-    - Export a new hydroTable.csv and overwrite the previous version and output new src json (overwrite previous)
+    - Export a new hydroTable.csv and overwrite the previous version and output new src json
+        (overwrite previous)
 
     Inputs:
     - fim_directory:        fim directory containing individual HUC output dirs
-    - water_edge_median_df: dataframe containing observation data (attributes: "hydroid", "flow", "submitter", "coll_time", "flow_unit", "layer", "HAND")
+    - water_edge_median_df: dataframe containing observation data (attributes: "hydroid", "flow", "submitter",
+                                "coll_time", "flow_unit", "layer", "HAND")
     - htable_path:          path to the current HUC hydroTable.csv
     - huc:                  string variable for the HUC id # (huc8 or huc6)
     - branch_id:            string variable for the branch id
-    - catchments_poly_path: path to the current HUC catchments polygon layer .gpkg 
-    - debug_outputs_option: optional input argument to output additional intermediate data files (csv files with SRC calculations)
-    - source_tag:           input text tag used to specify the type/source of the input obs data used for the SRC adjustments (e.g. usgs_rating or point_obs)
-    - merge_prev_adj:       boolean argument to specify when to merge previous SRC adjustments vs. overwrite (default=False)
-    - down_dist_thresh:     optional input argument to override the env variable that controls the downstream distance new roughness values are applied downstream of locations with valid obs data
+    - catchments_poly_path: path to the current HUC catchments polygon layer .gpkg
+    - debug_outputs_option: optional input argument to output additional intermediate data files
+                                (csv files with SRC calculations)
+    - source_tag:           input text tag used to specify the type/source of the input obs data used for the
+                                SRC adjustments (e.g. usgs_rating or point_obs)
+    - merge_prev_adj:       boolean argument to specify when to merge previous SRC adjustments vs. overwrite
+                                (default=False)
+    - down_dist_thresh:     optional input argument to override the env variable that controls the downstream
+                                distance new roughness values are applied downstream of locations with valid
+                                obs data
 
     Ouputs:
-    - output_catchments:    same input "catchments_poly_path" .gpkg with appened attributes for SRC adjustments fields
+    - output_catchments:    same input "catchments_poly_path" .gpkg with appened attributes for SRC
+                                adjustments fields
     - df_htable:            same input "htable_path" --> updated hydroTable.csv with new/modified attributes
     - output_src_json:      src.json file with new SRC discharge values
 
     '''
-    print("Processing " + str(source_tag) + " calibration for huc --> " + str(huc) + '  branch id: ' + str(branch_id))
-    log_text = "\nProcessing " + str(source_tag) + " calibration for huc --> " + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+    print(
+        "Processing "
+        + str(source_tag)
+        + " calibration for huc --> "
+        + str(huc)
+        + '  branch id: '
+        + str(branch_id)
+    )
+    log_text = (
+        "\nProcessing "
+        + str(source_tag)
+        + " calibration for huc --> "
+        + str(huc)
+        + '  branch id: '
+        + str(branch_id)
+        + '\n'
+    )
     log_text += "DOWNSTREAM_THRESHOLD: " + str(down_dist_thresh) + 'km\n'
     log_text += "Merge Previous Adj Values: " + str(merge_prev_adj) + '\n'
     df_nvalues = water_edge_median_df.copy()
-    df_nvalues = df_nvalues[ (df_nvalues.hydroid.notnull()) & (df_nvalues.hydroid > 0) ] # remove null entries that do not have a valid hydroid
+    df_nvalues = df_nvalues[
+        (df_nvalues.hydroid.notnull()) & (df_nvalues.hydroid > 0)
+    ]  # remove null entries that do not have a valid hydroid
 
     ## Determine calibration data type for naming calb dataframe column
     if source_tag == 'point_obs':
@@ -75,10 +126,13 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     else:
         log_text += "ERROR - unknown calibration data source type: " + str(source_tag) + '\n'
 
-    ## Read in the hydroTable.csv and check wether it has previously been updated (rename default columns if needed)
-    df_htable = pd.read_csv(htable_path, dtype={'HUC': object, 'last_updated':object, 'submitter':object, 'obs_source':object})
-    df_prev_adj = pd.DataFrame() # initialize empty df for populating/checking later
-    if 'precalb_discharge_cms' not in df_htable.columns: # need this column to exist before continuing
+    ## Read in the hydroTable.csv and check wether it has previously been updated
+    # (rename default columns if needed)
+    df_htable = pd.read_csv(
+        htable_path, dtype={'HUC': object, 'last_updated': object, 'submitter': object, 'obs_source': object}
+    )
+    df_prev_adj = pd.DataFrame()  # initialize empty df for populating/checking later
+    if 'precalb_discharge_cms' not in df_htable.columns:  # need this column to exist before continuing
         df_htable['calb_applied'] = False
         df_htable['last_updated'] = pd.NA
         df_htable['submitter'] = pd.NA
@@ -87,58 +141,142 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
         df_htable['calb_coef_usgs'] = pd.NA
         df_htable['calb_coef_spatial'] = pd.NA
         df_htable['calb_coef_final'] = pd.NA
-    if df_htable['precalb_discharge_cms'].isnull().values.any(): # check if there are not valid values in the column (True = no previous calibration outputs)
+    if (
+        df_htable['precalb_discharge_cms'].isnull().values.any()
+    ):  # check if there are not valid values in the column (True = no previous calibration outputs)
         df_htable['precalb_discharge_cms'] = df_htable['discharge_cms'].values
 
-    ## The section below allows for previous calibration modifications (i.e. usgs rating calbs) to be available in the final calibration outputs
-    if merge_prev_adj and not df_htable['calb_coef_final'].isnull().all(): # check if the merge_prev_adj setting is True and there are valid 'calb_coef_final' values from previous calibration outputs
+    ## The section below allows for previous calibration modifications (i.e. usgs rating calbs) to be
+    #  available in the final calibration outputs
+    # Check if the merge_prev_adj setting is True and there are valid 'calb_coef_final' values from previous
+    # calibration outputs
+    if merge_prev_adj and not df_htable['calb_coef_final'].isnull().all():
         # Create a subset of hydrotable with previous adjusted SRC attributes
-        df_prev_adj_htable = df_htable.copy()[['HydroID','submitter','last_updated','obs_source','calb_coef_final']] 
-        df_prev_adj_htable.rename(columns={'submitter':'submitter_prev','last_updated':'last_updated_prev','calb_coef_final':'calb_coef_final_prev','obs_source':'obs_source_prev'}, inplace=True)
+        df_prev_adj_htable = df_htable.copy()[
+            ['HydroID', 'submitter', 'last_updated', 'obs_source', 'calb_coef_final']
+        ]
+        df_prev_adj_htable.rename(
+            columns={
+                'submitter': 'submitter_prev',
+                'last_updated': 'last_updated_prev',
+                'calb_coef_final': 'calb_coef_final_prev',
+                'obs_source': 'obs_source_prev',
+            },
+            inplace=True,
+        )
         df_prev_adj_htable = df_prev_adj_htable.groupby(["HydroID"]).first()
         # Only keep previous USGS rating curve adjustments (previous spatial obs adjustments are not retained)
-        df_prev_adj = df_prev_adj_htable[df_prev_adj_htable['obs_source_prev'].str.contains("usgs_rating|ras2fim_rating", na=False)] 
-        log_text += 'HUC: ' + str(huc) + '  Branch: ' + str(branch_id) + ': found previous hydroTable calibration attributes --> retaining previous calb attributes for blending...\n'
-    
-    # Delete previous adj columns to prevent duplicate variable issues (if src_roughness_optimization.py was previously applied)
-    df_htable.drop(['discharge_cms','submitter','last_updated',calb_type,'calb_coef_final','calb_applied','obs_source'], axis=1, inplace=True, errors='ignore') 
-    df_htable.rename(columns={'precalb_discharge_cms':'discharge_cms'}, inplace=True)
+        df_prev_adj = df_prev_adj_htable[
+            df_prev_adj_htable['obs_source_prev'].str.contains("usgs_rating|ras2fim_rating", na=False)
+        ]
+        log_text += (
+            'HUC: '
+            + str(huc)
+            + '  Branch: '
+            + str(branch_id)
+            + ': found previous hydroTable calibration attributes --> '
+            + 'retaining previous calb attributes for blending...\n'
+        )
+
+    # Delete previous adj columns to prevent duplicate variable issues
+    # (if src_roughness_optimization.py was previously applied)
+    df_htable.drop(
+        [
+            'discharge_cms',
+            'submitter',
+            'last_updated',
+            calb_type,
+            'calb_coef_final',
+            'calb_applied',
+            'obs_source',
+        ],
+        axis=1,
+        inplace=True,
+        errors='ignore',
+    )
+    df_htable.rename(columns={'precalb_discharge_cms': 'discharge_cms'}, inplace=True)
 
     ## loop through the user provided point data --> stage/flow dataframe row by row
     for index, row in df_nvalues.iterrows():
         if row.hydroid not in df_htable['HydroID'].values:
-            print('ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid))
-            log_text += 'ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid) + '\n'
+            print(
+                'ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' hydroid: '
+                + str(row.hydroid)
+            )
+            log_text += (
+                'ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' hydroid: '
+                + str(row.hydroid)
+                + '\n'
+            )
         else:
-            df_htable_hydroid = df_htable[(df_htable.HydroID == row.hydroid) & (df_htable.stage > 0)] # filter htable for entries with matching hydroid and ignore stage 0 (first possible stage match at 1ft)
+            # filter htable for entries with matching hydroid and ignore stage 0
+            # (first possible stage match at 1ft)
+            df_htable_hydroid = df_htable[(df_htable.HydroID == row.hydroid) & (df_htable.stage > 0)]
             if df_htable_hydroid.empty:
-                print('ERROR: df_htable_hydroid is empty but expected data: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid))
-                log_text += 'ERROR: df_htable_hydroid is empty but expected data: ' + str(huc) + '  branch id: ' + str(branch_id) + ' hydroid: ' + str(row.hydroid) + '\n'
-                
-            find_src_stage = df_htable_hydroid.loc[df_htable_hydroid['stage'].sub(row.hand).abs().idxmin()] # find closest matching stage to the user provided HAND value
+                print(
+                    'ERROR: df_htable_hydroid is empty but expected data: '
+                    + str(huc)
+                    + '  branch id: '
+                    + str(branch_id)
+                    + ' hydroid: '
+                    + str(row.hydroid)
+                )
+                log_text += (
+                    'ERROR: df_htable_hydroid is empty but expected data: '
+                    + str(huc)
+                    + '  branch id: '
+                    + str(branch_id)
+                    + ' hydroid: '
+                    + str(row.hydroid)
+                    + '\n'
+                )
+
+            find_src_stage = df_htable_hydroid.loc[
+                df_htable_hydroid['stage'].sub(row.hand).abs().idxmin()
+            ]  # find closest matching stage to the user provided HAND value
             ## copy the corresponding htable values for the matching stage->HAND lookup
-            df_nvalues.loc[index,'feature_id'] = find_src_stage.feature_id
-            df_nvalues.loc[index,'LakeID'] = find_src_stage.LakeID
-            df_nvalues.loc[index,'NextDownID'] = find_src_stage.NextDownID
-            df_nvalues.loc[index,'LENGTHKM'] = find_src_stage.LENGTHKM
-            df_nvalues.loc[index,'src_stage'] = find_src_stage.stage
-            df_nvalues.loc[index,'channel_n'] = find_src_stage.channel_n
-            df_nvalues.loc[index,'overbank_n'] = find_src_stage.overbank_n
-            df_nvalues.loc[index,'discharge_cms'] = find_src_stage.discharge_cms
+            df_nvalues.loc[index, 'feature_id'] = find_src_stage.feature_id
+            df_nvalues.loc[index, 'LakeID'] = find_src_stage.LakeID
+            df_nvalues.loc[index, 'NextDownID'] = find_src_stage.NextDownID
+            df_nvalues.loc[index, 'LENGTHKM'] = find_src_stage.LENGTHKM
+            df_nvalues.loc[index, 'src_stage'] = find_src_stage.stage
+            df_nvalues.loc[index, 'channel_n'] = find_src_stage.channel_n
+            df_nvalues.loc[index, 'overbank_n'] = find_src_stage.overbank_n
+            df_nvalues.loc[index, 'discharge_cms'] = find_src_stage.discharge_cms
 
     ## Calculate calibration coefficient
-    df_nvalues.rename(columns={'hydroid':'HydroID'}, inplace=True) # rename the previous ManningN column
-    df_nvalues['hydroid_calb_coef'] = df_nvalues['discharge_cms']/df_nvalues['flow'] # Qobs / Qsrc
+    df_nvalues.rename(columns={'hydroid': 'HydroID'}, inplace=True)  # rename the previous ManningN column
+    df_nvalues['hydroid_calb_coef'] = df_nvalues['discharge_cms'] / df_nvalues['flow']  # Qobs / Qsrc
 
     ## Calcuate a "calibration adjusted" n value using channel and overbank n-values multiplied by calb_coef
-    df_nvalues['channel_n_calb'] = df_nvalues['hydroid_calb_coef']*df_nvalues['channel_n']
-    df_nvalues['overbank_n_calb'] = df_nvalues['hydroid_calb_coef']*df_nvalues['overbank_n']
-
-    ## Create dataframe to check for unrealistic/egregious calibration adjustments by applying the calibration coefficient to the Manning's n values and setting an acceptable range (values set in tools_shared_variables.py --> >0.8 or <0.001)
-    df_nvalues['Mann_flag'] = np.where((df_nvalues['channel_n_calb'] >= ROUGHNESS_MAX_THRESH) | (df_nvalues['overbank_n_calb'] >= ROUGHNESS_MAX_THRESH) | (df_nvalues['channel_n_calb'] <= ROUGHNESS_MIN_THRESH) | (df_nvalues['overbank_n_calb'] <= ROUGHNESS_MIN_THRESH) | (df_nvalues['hydroid_calb_coef'].isnull()),'Fail','Pass')
-    df_mann_flag = df_nvalues[(df_nvalues['Mann_flag'] == 'Fail')][['HydroID','hydroid_calb_coef','channel_n_calb','overbank_n_calb']]
+    df_nvalues['channel_n_calb'] = df_nvalues['hydroid_calb_coef'] * df_nvalues['channel_n']
+    df_nvalues['overbank_n_calb'] = df_nvalues['hydroid_calb_coef'] * df_nvalues['overbank_n']
+
+    ## Create dataframe to check for unrealistic/egregious calibration adjustments by applying the calibration
+    # coefficient to the Manning's n values and setting an acceptable range
+    # (values set in tools_shared_variables.py --> >0.8 or <0.001)
+    df_nvalues['Mann_flag'] = np.where(
+        (df_nvalues['channel_n_calb'] >= ROUGHNESS_MAX_THRESH)
+        | (df_nvalues['overbank_n_calb'] >= ROUGHNESS_MAX_THRESH)
+        | (df_nvalues['channel_n_calb'] <= ROUGHNESS_MIN_THRESH)
+        | (df_nvalues['overbank_n_calb'] <= ROUGHNESS_MIN_THRESH)
+        | (df_nvalues['hydroid_calb_coef'].isnull()),
+        'Fail',
+        'Pass',
+    )
+    df_mann_flag = df_nvalues[(df_nvalues['Mann_flag'] == 'Fail')][
+        ['HydroID', 'hydroid_calb_coef', 'channel_n_calb', 'overbank_n_calb']
+    ]
     if not df_mann_flag.empty:
-        log_text += '!!! Flaged Mannings Roughness values below !!!' +'\n'
+        log_text += '!!! Flaged Mannings Roughness values below !!!' + '\n'
         log_text += df_mann_flag.to_string() + '\n'
 
     ## Create magnitude and ahps column by subsetting the "layer" attribute
@@ -148,16 +286,18 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     df_nvalues.drop(['layer'], axis=1, inplace=True)
 
     ## Create df grouped by hydroid with ahps_lid and huc number
-    df_huc_lid = df_nvalues.groupby(["HydroID"]).first()[['ahps_lid','huc']]
+    df_huc_lid = df_nvalues.groupby(["HydroID"]).first()[['ahps_lid', 'huc']]
     df_huc_lid.columns = pd.MultiIndex.from_product([['info'], df_huc_lid.columns])
 
     ## pivot the magnitude column to display n value for each magnitude at each hydroid
-    df_nvalues_mag = df_nvalues.pivot_table(index='HydroID', columns='magnitude', values=['hydroid_calb_coef'], aggfunc='mean') # if there are multiple entries per hydroid and magnitude - aggregate using mean
-    
+    df_nvalues_mag = df_nvalues.pivot_table(
+        index='HydroID', columns='magnitude', values=['hydroid_calb_coef'], aggfunc='mean'
+    )  # if there are multiple entries per hydroid and magnitude - aggregate using mean
+
     ## Optional: Export csv with the newly calculated Manning's N values
     if debug_outputs_option:
         output_calc_n_csv = os.path.join(fim_directory, calb_type + '_src_calcs_' + branch_id + '.csv')
-        df_nvalues.to_csv(output_calc_n_csv,index=False)
+        df_nvalues.to_csv(output_calc_n_csv, index=False)
 
     ## filter the modified Manning's n dataframe for values out side allowable range
     df_nvalues = df_nvalues[df_nvalues['Mann_flag'] == 'Pass']
@@ -165,41 +305,55 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
     ## Check that there are valid entries in the calculate roughness df after filtering
     if not df_nvalues.empty:
         ## Create df with the most recent collection time entry and submitter attribs
-        df_updated = df_nvalues[['HydroID','coll_time','submitter','ahps_lid']] # subset the dataframe
-        df_updated = df_updated.sort_values('coll_time').drop_duplicates(['HydroID'],keep='last') # sort by collection time and then drop duplicate HydroIDs (keep most recent coll_time per HydroID)
-        df_updated.rename(columns={'coll_time':'last_updated'}, inplace=True)
+        df_updated = df_nvalues[['HydroID', 'coll_time', 'submitter', 'ahps_lid']]  # subset the dataframe
+        df_updated = df_updated.sort_values('coll_time').drop_duplicates(
+            ['HydroID'], keep='last'
+        )  # sort by collection time and then drop duplicate HydroIDs (keep most recent coll_time per HydroID)
+        df_updated.rename(columns={'coll_time': 'last_updated'}, inplace=True)
 
         ## cacluate median ManningN to handle cases with multiple hydroid entries
         df_mann_hydroid = df_nvalues.groupby(["HydroID"])[['hydroid_calb_coef']].median()
 
         ## Create a df with the median hydroid_ManningN value per feature_id
-        #df_mann_featid = df_nvalues.groupby(["feature_id"])[['hydroid_ManningN']].mean()
-        #df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
+        # df_mann_featid = df_nvalues.groupby(["feature_id"])[['hydroid_ManningN']].mean()
+        # df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
 
         ## Rename the original hydrotable variables to allow new calculations to use the primary var name
-        df_htable.rename(columns={'discharge_cms':'precalb_discharge_cms'}, inplace=True)
-
-        ## Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a singel hydroid)
-        df_nrange = df_nvalues.groupby('HydroID').agg({'hydroid_calb_coef': ['median', 'min', 'max', 'std', 'count']})
-        df_nrange['hydroid_calb_coef','range'] = df_nrange['hydroid_calb_coef','max'] - df_nrange['hydroid_calb_coef','min']
-        df_nrange = df_nrange.join(df_nvalues_mag, how='outer') # join the df_nvalues_mag containing hydroid_manningn values per flood magnitude category
-        df_nrange = df_nrange.merge(df_huc_lid, how='outer', on='HydroID') # join the df_huc_lid df to add attributes for lid and huc#
-        log_text += 'Statistics for Modified Roughness Calcs -->' +'\n'
+        df_htable.rename(columns={'discharge_cms': 'precalb_discharge_cms'}, inplace=True)
+
+        ## Check for large variabilty in the calculated Manning's N values
+        # (for cases with mutliple entries for a singel hydroid)
+        df_nrange = df_nvalues.groupby('HydroID').agg(
+            {'hydroid_calb_coef': ['median', 'min', 'max', 'std', 'count']}
+        )
+        df_nrange['hydroid_calb_coef', 'range'] = (
+            df_nrange['hydroid_calb_coef', 'max'] - df_nrange['hydroid_calb_coef', 'min']
+        )
+        df_nrange = df_nrange.join(
+            df_nvalues_mag, how='outer'
+        )  # join the df_nvalues_mag containing hydroid_manningn values per flood magnitude category
+        df_nrange = df_nrange.merge(
+            df_huc_lid, how='outer', on='HydroID'
+        )  # join the df_huc_lid df to add attributes for lid and huc#
+        log_text += 'Statistics for Modified Roughness Calcs -->' + '\n'
         log_text += df_nrange.to_string() + '\n'
         log_text += '----------------------------------------\n'
 
         ## Optional: Output csv with SRC calc stats
         if debug_outputs_option:
-            output_stats_n_csv = os.path.join(fim_directory, calb_type + '_src_coef_vals_stats_' + branch_id + '.csv')
-            df_nrange.to_csv(output_stats_n_csv,index=True)
+            output_stats_n_csv = os.path.join(
+                fim_directory, calb_type + '_src_coef_vals_stats_' + branch_id + '.csv'
+            )
+            df_nrange.to_csv(output_stats_n_csv, index=True)
 
         ## subset the original hydrotable dataframe and subset to one row per HydroID
-        df_nmerge = df_htable[['HydroID','feature_id','NextDownID','LENGTHKM','LakeID','order_']].drop_duplicates(['HydroID'],keep='first') 
+        df_nmerge = df_htable[
+            ['HydroID', 'feature_id', 'NextDownID', 'LENGTHKM', 'LakeID', 'order_']
+        ].drop_duplicates(['HydroID'], keep='first')
 
         ## Need to check that there are non-lake hydroids in the branch hydrotable (prevents downstream error)
         df_htable_check_lakes = df_nmerge.loc[df_nmerge['LakeID'] == -999]
         if not df_htable_check_lakes.empty:
-
             ## Create attributes to traverse the flow network between HydroIDs
             df_nmerge = branch_network_tracer(df_nmerge)
 
@@ -207,173 +361,356 @@ def update_rating_curve(fim_directory, water_edge_median_df, htable_path, huc, b
             df_nmerge = df_nmerge.merge(df_mann_hydroid, how='left', on='HydroID')
             df_nmerge = df_nmerge.merge(df_updated, how='left', on='HydroID')
 
-            ## Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg)
+            ## Calculate group_ManningN (mean calb n for consective hydroids) and apply values downsteam to
+            # non-calb hydroids (constrained to first Xkm of hydroids- set downstream diststance var as input)
             df_nmerge = group_manningn_calc(df_nmerge, down_dist_thresh)
 
             ## Create a df with the median hydroid_calb_coef value per feature_id
             df_mann_featid = df_nmerge.groupby(["feature_id"])[['hydroid_calb_coef']].mean()
-            df_mann_featid.rename(columns={'hydroid_calb_coef':'featid_calb_coef'}, inplace=True)
-            df_mann_featid_attrib = df_nmerge.groupby('feature_id').first() # create a seperate df with attributes to apply to other hydroids that share a featureid
-            df_mann_featid_attrib = df_mann_featid_attrib[df_mann_featid_attrib['submitter'].notna()][['last_updated','submitter']]
+            df_mann_featid.rename(columns={'hydroid_calb_coef': 'featid_calb_coef'}, inplace=True)
+            # create a seperate df with attributes to apply to other hydroids that share a featureid
+            df_mann_featid_attrib = df_nmerge.groupby('feature_id').first()
+            df_mann_featid_attrib = df_mann_featid_attrib[df_mann_featid_attrib['submitter'].notna()][
+                ['last_updated', 'submitter']
+            ]
             df_nmerge = df_nmerge.merge(df_mann_featid, how='left', on='feature_id').set_index('feature_id')
             df_nmerge = df_nmerge.combine_first(df_mann_featid_attrib).reset_index()
-            
-            if not df_nmerge['hydroid_calb_coef'].isnull().all():
 
-                ## Create the calibration coefficient column by combining the hydroid_calb_coef with the featid_calb_coef (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_calb_coef value(s))
-                conditions  = [ (df_nmerge['hydroid_calb_coef'].isnull()) & (df_nmerge['featid_calb_coef'].notnull()), (df_nmerge['hydroid_calb_coef'].isnull()) & (df_nmerge['featid_calb_coef'].isnull()) & (df_nmerge['group_calb_coef'].notnull()) ]
-                choices     = [ df_nmerge['featid_calb_coef'], df_nmerge['group_calb_coef'] ]
+            if not df_nmerge['hydroid_calb_coef'].isnull().all():
+                ## Create the calibration coefficient column by combining the hydroid_calb_coef with the
+                # featid_calb_coef (use feature_id value if the hydroid is in a feature_id that contains
+                # valid hydroid_calb_coef value(s))
+                conditions = [
+                    (df_nmerge['hydroid_calb_coef'].isnull()) & (df_nmerge['featid_calb_coef'].notnull()),
+                    (df_nmerge['hydroid_calb_coef'].isnull())
+                    & (df_nmerge['featid_calb_coef'].isnull())
+                    & (df_nmerge['group_calb_coef'].notnull()),
+                ]
+                choices = [df_nmerge['featid_calb_coef'], df_nmerge['group_calb_coef']]
                 df_nmerge[calb_type] = np.select(conditions, choices, default=df_nmerge['hydroid_calb_coef'])
                 df_nmerge['obs_source'] = np.where(df_nmerge[calb_type].notnull(), source_tag, pd.NA)
-                df_nmerge.drop(['feature_id','NextDownID','LENGTHKM','LakeID','order_'], axis=1, inplace=True, errors='ignore') # drop these columns to avoid duplicates where merging with the full hydroTable df
-                
-                ## Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new adjusted roughness value
+                df_nmerge.drop(
+                    ['feature_id', 'NextDownID', 'LENGTHKM', 'LakeID', 'order_'],
+                    axis=1,
+                    inplace=True,
+                    errors='ignore',
+                )  # drop these columns to avoid duplicates where merging with the full hydroTable df
+
+                ## Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new
+                # adjusted roughness value
                 if not df_prev_adj.empty:
-                    df_nmerge = pd.merge(df_nmerge,df_prev_adj, on='HydroID', how='outer')
-                    df_nmerge['submitter'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['submitter_prev'],df_nmerge['submitter'])
-                    df_nmerge['last_updated'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['last_updated_prev'],df_nmerge['last_updated'])
-                    df_nmerge['obs_source'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['obs_source_prev'],df_nmerge['obs_source'])
-                    df_nmerge['calb_coef_final'] = np.where((df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),df_nmerge['calb_coef_final_prev'],df_nmerge[calb_type])
-                    df_nmerge.drop(['submitter_prev','last_updated_prev','calb_coef_final_prev','obs_source_prev'], axis=1, inplace=True, errors='ignore')
+                    df_nmerge = pd.merge(df_nmerge, df_prev_adj, on='HydroID', how='outer')
+                    df_nmerge['submitter'] = np.where(
+                        (df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),
+                        df_nmerge['submitter_prev'],
+                        df_nmerge['submitter'],
+                    )
+                    df_nmerge['last_updated'] = np.where(
+                        (df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),
+                        df_nmerge['last_updated_prev'],
+                        df_nmerge['last_updated'],
+                    )
+                    df_nmerge['obs_source'] = np.where(
+                        (df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),
+                        df_nmerge['obs_source_prev'],
+                        df_nmerge['obs_source'],
+                    )
+                    df_nmerge['calb_coef_final'] = np.where(
+                        (df_nmerge[calb_type].isnull() & df_nmerge['calb_coef_final_prev'].notnull()),
+                        df_nmerge['calb_coef_final_prev'],
+                        df_nmerge[calb_type],
+                    )
+                    df_nmerge.drop(
+                        ['submitter_prev', 'last_updated_prev', 'calb_coef_final_prev', 'obs_source_prev'],
+                        axis=1,
+                        inplace=True,
+                        errors='ignore',
+                    )
                 else:
                     df_nmerge['calb_coef_final'] = df_nmerge[calb_type]
-                
-                
+
                 ## Update the catchments polygon .gpkg with joined attribute - "src_calibrated"
                 if os.path.isfile(catchments_poly_path):
                     input_catchments = gpd.read_file(catchments_poly_path)
                     ## Create new "src_calibrated" column for viz query
-                    if 'src_calibrated' in input_catchments.columns: # check if this attribute already exists and drop if needed
-                        input_catchments.drop(['src_calibrated','obs_source','calb_coef_final'], axis=1, inplace=True, errors='ignore')
-                    df_nmerge['src_calibrated'] = np.where(df_nmerge['calb_coef_final'].notnull(), 'True', 'False')
-                    output_catchments = input_catchments.merge(df_nmerge[['HydroID','src_calibrated','obs_source','calb_coef_final']], how='left', on='HydroID')
+                    if (
+                        'src_calibrated' in input_catchments.columns
+                    ):  # check if this attribute already exists and drop if needed
+                        input_catchments.drop(
+                            ['src_calibrated', 'obs_source', 'calb_coef_final'],
+                            axis=1,
+                            inplace=True,
+                            errors='ignore',
+                        )
+                    df_nmerge['src_calibrated'] = np.where(
+                        df_nmerge['calb_coef_final'].notnull(), 'True', 'False'
+                    )
+                    output_catchments = input_catchments.merge(
+                        df_nmerge[['HydroID', 'src_calibrated', 'obs_source', 'calb_coef_final']],
+                        how='left',
+                        on='HydroID',
+                    )
                     output_catchments['src_calibrated'].fillna('False', inplace=True)
-                    output_catchments.to_file(catchments_poly_path,driver="GPKG",index=False) # overwrite the previous layer
+                    output_catchments.to_file(
+                        catchments_poly_path, driver="GPKG", index=False
+                    )  # overwrite the previous layer
                     df_nmerge.drop(['src_calibrated'], axis=1, inplace=True, errors='ignore')
-                ## Optional ouputs: 1) merge_n_csv csv with all of the calculated n values and 2) a catchments .gpkg with new joined attributes
+                ## Optional ouputs:
+                #   1) merge_n_csv csv with all of the calculated n values
+                #   2) a catchments .gpkg with new joined attributes
                 if debug_outputs_option:
-                    output_merge_n_csv = os.path.join(fim_directory, calb_type + '_merge_vals_' + branch_id + '.csv')
-                    df_nmerge.to_csv(output_merge_n_csv,index=False)
+                    output_merge_n_csv = os.path.join(
+                        fim_directory, calb_type + '_merge_vals_' + branch_id + '.csv'
+                    )
+                    df_nmerge.to_csv(output_merge_n_csv, index=False)
                     ## output new catchments polygon layer with several new attributes appended
                     if os.path.isfile(catchments_poly_path):
                         input_catchments = gpd.read_file(catchments_poly_path)
-                        output_catchments_fileName = os.path.join(os.path.split(catchments_poly_path)[0],"gw_catchments_src_adjust_" + str(branch_id) + ".gpkg")
+                        output_catchments_fileName = os.path.join(
+                            os.path.split(catchments_poly_path)[0],
+                            "gw_catchments_src_adjust_" + str(branch_id) + ".gpkg",
+                        )
                         output_catchments = input_catchments.merge(df_nmerge, how='left', on='HydroID')
-                        output_catchments.to_file(output_catchments_fileName,driver="GPKG",index=False)
+                        output_catchments.to_file(output_catchments_fileName, driver="GPKG", index=False)
 
                 ## Merge the final ManningN dataframe to the original hydroTable
-                df_nmerge.drop(['ahps_lid','start_catch','route_count','branch_id','hydroid_calb_coef','featid_calb_coef','group_calb_coef',], axis=1, inplace=True, errors='ignore') # drop these columns to avoid duplicates where merging with the full hydroTable df
+                df_nmerge.drop(
+                    [
+                        'ahps_lid',
+                        'start_catch',
+                        'route_count',
+                        'branch_id',
+                        'hydroid_calb_coef',
+                        'featid_calb_coef',
+                        'group_calb_coef',
+                    ],
+                    axis=1,
+                    inplace=True,
+                    errors='ignore',
+                )  # drop these columns to avoid duplicates where merging with the full hydroTable df
                 df_htable = df_htable.merge(df_nmerge, how='left', on='HydroID')
-                df_htable['calb_applied'] = np.where(df_htable['calb_coef_final'].notnull(), 'True', 'False') # create true/false column to clearly identify where new roughness values are applied
+                df_htable['calb_applied'] = np.where(
+                    df_htable['calb_coef_final'].notnull(), 'True', 'False'
+                )  # create true/false column to clearly identify where new roughness values are applied
 
                 ## Calculate new discharge_cms with new adjusted ManningN
-                df_htable['discharge_cms'] = np.where(df_htable['calb_coef_final'].isnull(), df_htable['precalb_discharge_cms'], df_htable['precalb_discharge_cms']/df_htable['calb_coef_final'])
-
-                ## Replace discharge_cms with 0 or -999 if present in the original discharge (carried over from thalweg notch workaround in SRC post-processing)
-                df_htable['discharge_cms'].mask(df_htable['precalb_discharge_cms']==0.0,0.0,inplace=True)
-                df_htable['discharge_cms'].mask(df_htable['precalb_discharge_cms']==-999,-999,inplace=True)
+                df_htable['discharge_cms'] = np.where(
+                    df_htable['calb_coef_final'].isnull(),
+                    df_htable['precalb_discharge_cms'],
+                    df_htable['precalb_discharge_cms'] / df_htable['calb_coef_final'],
+                )
+
+                ## Replace discharge_cms with 0 or -999 if present in the original discharge
+                # (carried over from thalweg notch workaround in SRC post-processing)
+                df_htable['discharge_cms'].mask(df_htable['precalb_discharge_cms'] == 0.0, 0.0, inplace=True)
+                df_htable['discharge_cms'].mask(
+                    df_htable['precalb_discharge_cms'] == -999, -999, inplace=True
+                )
 
                 ## Export a new hydroTable.csv and overwrite the previous version
                 out_htable = os.path.join(fim_directory, 'hydroTable_' + branch_id + '.csv')
-                df_htable.to_csv(out_htable,index=False)
+                df_htable.to_csv(out_htable, index=False)
 
             else:
-                print('ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid hydroid roughness calculations after removing lakeid catchments from consideration')
-                log_text += 'ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid hydroid roughness calculations after removing lakeid catchments from consideration\n'
+                print(
+                    'ALERT!! HUC: '
+                    + str(huc)
+                    + '  branch id: '
+                    + str(branch_id)
+                    + ' --> no valid hydroid roughness calculations after removing lakeid catchments from '
+                    + ' consideration \n'
+                )
+                log_text += (
+                    'ALERT!! HUC: '
+                    + str(huc)
+                    + '  branch id: '
+                    + str(branch_id)
+                    + ' --> no valid hydroid roughness calculations after removing lakeid catchments from '
+                    + ' consideration \n'
+                )
         else:
-                print('WARNING!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> hydrotable is empty after removing lake catchments (will ignore branch)')
-                log_text += 'ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> hydrotable is empty after removing lake catchments (will ignore branch)\n'
+            print(
+                'WARNING!! HUC: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' --> hydrotable is empty after removing lake catchments (will ignore branch)'
+            )
+            log_text += (
+                'ALERT!! HUC: '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + ' --> hydrotable is empty after removing lake catchments (will ignore branch)\n'
+            )
     else:
-        print('ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid roughness calculations - please check point data and src calculations to evaluate')
-        log_text += 'ALERT!! HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + ' --> no valid roughness calculations - please check point data and src calculations to evaluate\n'
-
-    log_text += 'Completed: ' + str(huc) + ' --> branch: ' + str(branch_id) + '\n'
+        print(
+            'ALERT!! HUC: '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+            + ' --> no valid roughness calculations- please check point data and src calculations to evaluate'
+        )
+        log_text += (
+            'ALERT!! HUC: '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+            + ' --> no valid roughness calculations- please check point data and src calculations to evaluate'
+        )
+
+    log_text += '\n Completed: ' + str(huc) + ' --> branch: ' + str(branch_id) + '\n'
     log_text += '#########################################################\n'
     print("Completed huc: " + str(huc) + ' --> branch: ' + str(branch_id))
-    return(log_text)
+    return log_text
+
 
 def branch_network_tracer(df_input_htable):
-    df_input_htable = df_input_htable.astype({'NextDownID': 'int64'}) # ensure attribute has consistent format as int
-    df_input_htable = df_input_htable.loc[df_input_htable['LakeID'] == -999] # remove all hydroids associated with lake/water body (these often have disjoined artifacts in the network)
-    df_input_htable["start_catch"] = ~df_input_htable['HydroID'].isin(df_input_htable['NextDownID']) # define start catchments as hydroids that are not found in the "NextDownID" attribute for all other hydroids
-            
-    df_input_htable.set_index('HydroID',inplace=True,drop=False) # set index to the hydroid
-    branch_heads = deque(df_input_htable[df_input_htable['start_catch'] == True]['HydroID'].tolist()) # create deque of hydroids to define start points in the while loop
-    visited = set() # create set to keep track of all hydroids that have been accounted for
-    branch_count = 0 # start branch id 
+    df_input_htable = df_input_htable.astype(
+        {'NextDownID': 'int64'}
+    )  # ensure attribute has consistent format as int
+    # remove all hydroids associated with lake/water body
+    # (these often have disjoined artifacts in the network)
+    df_input_htable = df_input_htable.loc[df_input_htable['LakeID'] == -999]
+    # define start catchments as hydroids that are not found in the "NextDownID" attribute for all
+    # other hydroids
+    df_input_htable["start_catch"] = ~df_input_htable['HydroID'].isin(df_input_htable['NextDownID'])
+
+    df_input_htable.set_index('HydroID', inplace=True, drop=False)  # set index to the hydroid
+    branch_heads = deque(
+        df_input_htable[df_input_htable['start_catch'] == True]['HydroID'].tolist()
+    )  # create deque of hydroids to define start points in the while loop
+    visited = set()  # create set to keep track of all hydroids that have been accounted for
+    branch_count = 0  # start branch id
     while branch_heads:
-        hid = branch_heads.popleft() # pull off left most hydroid from deque of start hydroids
-        Q = deque(df_input_htable[df_input_htable['HydroID'] == hid]['HydroID'].tolist()) # create a new deque that will be used to populate all relevant downstream hydroids
-        vert_count = 0; branch_count += 1
+        hid = branch_heads.popleft()  # pull off left most hydroid from deque of start hydroids
+        Q = deque(
+            df_input_htable[df_input_htable['HydroID'] == hid]['HydroID'].tolist()
+        )  # create a new deque that will be used to populate all relevant downstream hydroids
+        vert_count = 0
+        branch_count += 1
         while Q:
             q = Q.popleft()
             if q not in visited:
-                df_input_htable.loc[df_input_htable.HydroID==q,'route_count'] = vert_count # assign var with flow order ranking
-                df_input_htable.loc[df_input_htable.HydroID==q,'branch_id'] = branch_count # assign var with current branch id
+                df_input_htable.loc[
+                    df_input_htable.HydroID == q, 'route_count'
+                ] = vert_count  # assign var with flow order ranking
+                df_input_htable.loc[
+                    df_input_htable.HydroID == q, 'branch_id'
+                ] = branch_count  # assign var with current branch id
                 vert_count += 1
                 visited.add(q)
-                nextid = df_input_htable.loc[q,'NextDownID'] # find the id for the next downstream hydroid
-                order = df_input_htable.loc[q,'order_'] # find the streamorder for the current hydroid
-            
+                # find the id for the next downstream hydroid
+                nextid = df_input_htable.loc[q, 'NextDownID']
+                order = df_input_htable.loc[q, 'order_']  # find the streamorder for the current hydroid
+
                 if nextid not in visited and nextid in df_input_htable.HydroID:
-                    check_confluence = (df_input_htable.NextDownID == nextid).sum() > 1 # check if the NextDownID is referenced by more than one hydroid (>1 means this is a confluence)
-                    nextorder = df_input_htable.loc[nextid,'order_'] # find the streamorder for the next downstream hydroid
-                    if nextorder > order and check_confluence == True: # check if the nextdownid streamorder is greater than the current hydroid order and the nextdownid is a confluence (more than 1 upstream hydroid draining to it)
-                        branch_heads.append(nextid) # found a terminal point in the network (append to branch_heads for second pass)
-                        continue # if above conditions are True than stop traversing downstream and move on to next starting hydroid
+                    # check if the NextDownID is referenced by more than one hydroid
+                    # (>1 means this is a confluence)
+                    check_confluence = (df_input_htable.NextDownID == nextid).sum() > 1
+                    nextorder = df_input_htable.loc[
+                        nextid, 'order_'
+                    ]  # find the streamorder for the next downstream hydroid
+                    # check if the nextdownid streamorder is greater than the current hydroid order and the
+                    # nextdownid is a confluence (more than 1 upstream hydroid draining to it)
+                    if nextorder > order and check_confluence == True:
+                        branch_heads.append(
+                            nextid
+                        )  # found a terminal point in the network (append to branch_heads for second pass)
+                        # if above conditions are True than stop traversing downstream and move on to next
+                        # starting hydroid
+                        continue
                     Q.append(nextid)
-    df_input_htable.reset_index(drop=True, inplace=True) # reset index (previously using hydroid as index)
-    df_input_htable.sort_values(['branch_id','route_count'], inplace=True) # sort the dataframe by branch_id and then by route_count (need this ordered to ensure upstream to downstream ranking for each branch)
-    return(df_input_htable)
+    df_input_htable.reset_index(drop=True, inplace=True)  # reset index (previously using hydroid as index)
+    # sort the dataframe by branch_id and then by route_count
+    # (need this ordered to ensure upstream to downstream ranking for each branch)
+    df_input_htable.sort_values(['branch_id', 'route_count'], inplace=True)
+    return df_input_htable
+
 
 def group_manningn_calc(df_nmerge, down_dist_thresh):
-    ## Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg
-    #df_nmerge.sort_values(by=['NextDownID'], inplace=True)
-    dist_accum = 0; hyid_count = 0; hyid_accum_count = 0; 
-    run_accum_mann = 0; group_calb_coef = 0; branch_start = 1                                       # initialize counter and accumulation variables
-    lid_count = 0; prev_lid = 'x'
-    for index, row in df_nmerge.iterrows():                                                         # loop through the df (parse by hydroid)
-        if int(df_nmerge.loc[index,'branch_id']) != branch_start:                                   # check if start of new branch
-            dist_accum = 0; hyid_count = 0; hyid_accum_count = 0;                                   # initialize counter vars
-            run_accum_mann = 0; group_calb_coef = 0                                                 # initialize counter vars
-            branch_start = int(df_nmerge.loc[index,'branch_id'])                                    # reassign the branch_start var to evaluate on next iteration
-            # use the code below to withold downstream hydroid_calb_coef values (use this for downstream evaluation tests)
+    ## Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to
+    # non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg
+    # df_nmerge.sort_values(by=['NextDownID'], inplace=True)
+    dist_accum = 0
+    hyid_count = 0
+    hyid_accum_count = 0
+    run_accum_mann = 0
+    group_calb_coef = 0
+    branch_start = 1  # initialize counter and accumulation variables
+    lid_count = 0
+    prev_lid = 'x'
+    for index, row in df_nmerge.iterrows():  # loop through the df (parse by hydroid)
+        if int(df_nmerge.loc[index, 'branch_id']) != branch_start:  # check if start of new branch
+            dist_accum = 0
+            hyid_count = 0
+            hyid_accum_count = 0
+            # initialize counter vars
+            run_accum_mann = 0
+            group_calb_coef = 0  # initialize counter vars
+            branch_start = int(
+                df_nmerge.loc[index, 'branch_id']
+            )  # reassign the branch_start var to evaluate on next iteration
+            # use the code below to withold downstream hydroid_calb_coef values
+            # (use this for downstream evaluation tests)
             '''
-            lid_count = 0                                                                         
+            lid_count = 0
         if not pd.isna(df_nmerge.loc[index,'ahps_lid']):
             if df_nmerge.loc[index,'ahps_lid'] == prev_lid:
                 lid_count += 1
-                if lid_count > 3: # only keep the first 3 HydroID n values (everything else set to null for downstream application)
+                # only keep the first 3 HydroID n values
+                # (everything else set to null for downstream application)
+                if lid_count > 3:
                     df_nmerge.loc[index,'hydroid_ManningN'] = np.nan
                     df_nmerge.loc[index,'featid_ManningN'] = np.nan
             else:
                 lid_count = 1
             prev_lid = df_nmerge.loc[index,'ahps_lid']
             '''
-        if np.isnan(df_nmerge.loc[index,'hydroid_calb_coef']):                                      # check if the hydroid_calb_coef value is nan (indicates a non-calibrated hydroid)
-            df_nmerge.loc[index,'accum_dist'] = row['LENGTHKM'] + dist_accum                        # calculate accumulated river distance
-            dist_accum += row['LENGTHKM']                                                           # add hydroid length to the dist_accum var
-            hyid_count = 0                                                                          # reset the hydroid counter to 0
-            df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hydroid accum counter
-            if dist_accum < down_dist_thresh:                                                       # check if the accum distance is less than Xkm downstream from valid hydroid_calb_coef group value
-                if hyid_accum_count > 1:                                                            # only apply the group_calb_coef if there are 2 or more valid hydorids that contributed to the upstream group_calb_coef
-                    df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                        # output the group_calb_coef var
+        if np.isnan(
+            df_nmerge.loc[index, 'hydroid_calb_coef']
+        ):  # check if the hydroid_calb_coef value is nan (indicates a non-calibrated hydroid)
+            df_nmerge.loc[index, 'accum_dist'] = (
+                row['LENGTHKM'] + dist_accum
+            )  # calculate accumulated river distance
+            dist_accum += row['LENGTHKM']  # add hydroid length to the dist_accum var
+            hyid_count = 0  # reset the hydroid counter to 0
+            df_nmerge.loc[index, 'hyid_accum_count'] = hyid_accum_count  # output the hydroid accum counter
+            # check if the accum distance is less than Xkm downstream from valid hydroid_calb_coef group value
+            if dist_accum < down_dist_thresh:
+                # only apply the group_calb_coef if there are 2 or more valid hydorids that contributed to the
+                # upstream group_calb_coef
+                if hyid_accum_count > 1:
+                    df_nmerge.loc[
+                        index, 'group_calb_coef'
+                    ] = group_calb_coef  # output the group_calb_coef var
             else:
-                run_avg_mann = 0                                                                    # reset the running average manningn variable (greater than 10km downstream)
-        else:                                                                                       # performs the following for hydroids that have a valid hydroid_calb_coef value
-            dist_accum = 0; hyid_count += 1                                                         # initialize vars
-            df_nmerge.loc[index,'accum_dist'] = 0                                                   # output the accum_dist value (set to 0)
-            if hyid_count == 1:                                                                     # checks if this the first in a series of valid hydroid_calb_coef values
-                run_accum_mann = 0; hyid_accum_count = 0                                            # initialize counter and running accumulated manningN value
-            group_calb_coef = (row['hydroid_calb_coef'] + run_accum_mann)/float(hyid_count)         # calculate the group_calb_coef (NOTE: this will continue to change as more hydroid values are accumulated in the "group" moving downstream)
-            df_nmerge.loc[index,'group_calb_coef'] = group_calb_coef                                # output the group_calb_coef var 
-            df_nmerge.loc[index,'hyid_count'] = hyid_count                                          # output the hyid_count var 
-            run_accum_mann += row['hydroid_calb_coef']                                              # add current hydroid manningn value to the running accum mann var
-            hyid_accum_count += 1                                                                   # increase the # of hydroid accum counter
-            df_nmerge.loc[index,'hyid_accum_count'] = hyid_accum_count                              # output the hyid_accum_count var
+                # reset the running average manningn variable (greater than 10km downstream)
+                run_avg_mann = 0
+        # performs the following for hydroids that have a valid hydroid_calb_coef value
+        else:
+            dist_accum = 0
+            hyid_count += 1  # initialize vars
+            df_nmerge.loc[index, 'accum_dist'] = 0  # output the accum_dist value (set to 0)
+            if hyid_count == 1:  # checks if this the first in a series of valid hydroid_calb_coef values
+                run_accum_mann = 0
+                hyid_accum_count = 0  # initialize counter and running accumulated manningN value
+            # calculate the group_calb_coef (NOTE: this will continue to change as more hydroid values are
+            # accumulated in the "group" moving downstream)
+            group_calb_coef = (row['hydroid_calb_coef'] + run_accum_mann) / float(hyid_count)
+            df_nmerge.loc[index, 'group_calb_coef'] = group_calb_coef  # output the group_calb_coef var
+            df_nmerge.loc[index, 'hyid_count'] = hyid_count  # output the hyid_count var
+            run_accum_mann += row[
+                'hydroid_calb_coef'
+            ]  # add current hydroid manningn value to the running accum mann var
+            hyid_accum_count += 1  # increase the # of hydroid accum counter
+            df_nmerge.loc[index, 'hyid_accum_count'] = hyid_accum_count  # output the hyid_accum_count var
 
     ## Delete unnecessary intermediate outputs
     if 'hyid_count' in df_nmerge.columns:
-        df_nmerge.drop(['hyid_count','accum_dist','hyid_accum_count'], axis=1, inplace=True, errors='ignore') # drop hydroid counter if it exists
-    #df_nmerge.drop(['accum_dist','hyid_accum_count'], axis=1, inplace=True) # drop accum vars from group calc
-    return(df_nmerge)
+        df_nmerge.drop(
+            ['hyid_count', 'accum_dist', 'hyid_accum_count'], axis=1, inplace=True, errors='ignore'
+        )  # drop hydroid counter if it exists
+    ## drop accum vars from group calc
+    # df_nmerge.drop(['accum_dist','hyid_accum_count'], axis=1, inplace=True)
+    return df_nmerge
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 43a1be1ac..0f4b97d51 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -1,22 +1,24 @@
 #!/usr/bin/env python3
 
 import os
+from collections import deque
+from os.path import isfile, splitext
+from random import sample
+
+import fiona
 import geopandas as gpd
+import numpy as np
 import pandas as pd
 import rasterio
-from rasterio.mask import mask
-from rasterio.io import DatasetReader
-from os.path import splitext, isfile
-import fiona
 from fiona.errors import DriverError
-from collections import deque
-import numpy as np
-from tqdm import tqdm
+from rasterio.io import DatasetReader
+from rasterio.mask import mask
+from scipy.stats import mode
+from shapely.geometry import LineString, MultiLineString, MultiPoint, Point
 from shapely.ops import linemerge, unary_union
-from shapely.geometry import MultiLineString, LineString, MultiPoint, Point
 from shapely.strtree import STRtree
-from random import sample
-from scipy.stats import mode
+from tqdm import tqdm
+
 from utils.shared_variables import PREP_CRS
 
 
@@ -110,11 +112,7 @@ def write(self, fileName, layer=None, index=True, verbose=False):
             print("Writing to {}".format(fileName))
 
         # sets driver
-        driverDictionary = {
-            ".gpkg": "GPKG",
-            ".geojson": "GeoJSON",
-            ".shp": "ESRI Shapefile",
-        }
+        driverDictionary = {".gpkg": "GPKG", ".geojson": "GeoJSON", ".shp": "ESRI Shapefile"}
         driver = driverDictionary[splitext(fileName)[1]]
 
         self.to_file(fileName, driver=driver, layer=layer, index=index)
@@ -265,7 +263,7 @@ def derive_nodes(
         # if self.index.name != reach_id_attribute:
         # self.set_index(reach_id_attribute,drop=True,inplace=True)
 
-        inlet_coordinates, outlet_coordinates = dict(), dict()
+        # inlet_coordinates, outlet_coordinates = dict(), dict()
         node_coordinates = dict()
         toNodes, fromNodes = [None] * len(self), [None] * len(self)
         current_node_id = "1".zfill(max_post_node_digits)
@@ -375,10 +373,7 @@ def derive_inlet_points_by_feature(self, feature_attribute, outlet_linestring_in
         return feature_inlet_points_gdf
 
     def derive_headwater_points_with_inlets(
-        self,
-        inlets_attribute="inlet_id",
-        fromNode_attribute="FromNode",
-        outlet_linestring_index=0,
+        self, inlets_attribute="inlet_id", fromNode_attribute="FromNode", outlet_linestring_index=0
     ):
         """Derives headwater points file given inlets"""
 
@@ -413,11 +408,7 @@ def exclude_attribute_values(self, branch_id_attribute=None, values_excluded=Non
         return self
 
     def remove_stream_segments_without_catchments(
-        self,
-        catchments,
-        reach_id_attribute="ID",
-        reach_id_attribute_in_catchments="ID",
-        verbose=False,
+        self, catchments, reach_id_attribute="ID", reach_id_attribute_in_catchments="ID", verbose=False
     ):
         if verbose:
             print("Removing stream segments without catchments ...")
@@ -501,14 +492,7 @@ def find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
                     # Remove reach from tmp_self
                     tmp_IDs.append(downstream_ID)
                     tmp_self.drop(
-                        tmp_self[
-                            tmp_self.From_Node.astype(int).isin(
-                                [
-                                    downstream_ID,
-                                ]
-                            )
-                        ].index,
-                        inplace=True,
+                        tmp_self[tmp_self.From_Node.astype(int).isin([downstream_ID])].index, inplace=True
                     )
                     # Repeat for next lowest downstream reach
                     if downstream_ID in tmp_self.To_Node.astype(int).values:
@@ -537,14 +521,7 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
                     # Remove reach from tmp_self
                     tmp_IDs.append(upstream_ID)
                     tmp_self.drop(
-                        tmp_self[
-                            tmp_self.From_Node.astype(int).isin(
-                                [
-                                    upstream_ID,
-                                ]
-                            )
-                        ].index,
-                        inplace=True,
+                        tmp_self[tmp_self.From_Node.astype(int).isin([upstream_ID])].index, inplace=True
                     )
                     # Repeat for next highest upstream reach
                     return find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
@@ -747,19 +724,15 @@ def derive_stream_branches(
                     if len(not_visited_upstream_ids) == 1:
                         continue  # only create a new branch if there are 2 upstreams
                     new_upstream_branches = upstream_reaches_compare_values.loc[
-                        ~upstream_reaches_compare_values.index.isin(
-                            [
-                                continue_id,
-                            ]
-                        )
+                        ~upstream_reaches_compare_values.index.isin([continue_id])
                     ]
                     for new_up_id in new_upstream_branches.index:
                         branch_id = str(current_reach_branch_id)[0:4] + str(bid).zfill(max_branch_id_digits)
                         self.loc[new_up_id, branch_id_attribute] = branch_id
                         bid += 1
                     # ==================================================================================
-                    """ NOTE: The above logic uses stream order to override arbolate sum. Use the commented section
-                     below if this turns out to be a bad idea!"""
+                    """ NOTE: The above logic uses stream order to override arbolate sum.
+                        Use the commented section below if this turns out to be a bad idea!"""
                     # matches = 0 # if upstream matches are more than 1, limits to only one match
                     # for usrcv,nvus in zip(upstream_reaches_compare_values,not_visited_upstream_ids):
                     #    if (usrcv == matching_value) & (matches == 0):
@@ -778,11 +751,7 @@ def derive_stream_branches(
         return self
 
     def make_up_and_downstream_dictionaries(
-        self,
-        reach_id_attribute="ID",
-        toNode_attribute="ToNode",
-        fromNode_attribute="FromNode",
-        verbose=False,
+        self, reach_id_attribute="ID", toNode_attribute="ToNode", fromNode_attribute="FromNode", verbose=False
     ):
         # sets index of stream branches as reach id attribute
         # if self.index.name != reach_id_attribute:
@@ -913,10 +882,7 @@ def dissolve_by_branch(
 
         # merges each multi-line string to a singular linestring
         for lpid, row in tqdm(
-            self.iterrows(),
-            total=len(self),
-            disable=(not verbose),
-            desc="Merging mult-part geoms",
+            self.iterrows(), total=len(self), disable=(not verbose), desc="Merging mult-part geoms"
         ):
             if isinstance(row.geometry, MultiLineString):
                 merged_line = linemerge(row.geometry)
@@ -977,10 +943,7 @@ def conflate_branches(
 
         # loop through rows of self
         for idx, row in tqdm(
-            self.iterrows(),
-            total=len(self),
-            disable=(not verbose),
-            desc="Conflating branches",
+            self.iterrows(), total=len(self), disable=(not verbose), desc="Conflating branches"
         ):
             g = row["geometry"]
             o = row[left_order_attribute]
@@ -1009,10 +972,7 @@ def explode_to_points(self, reach_id_attribute="ID", sampling_size=None, verbose
 
         all_exploded_points = [None] * len(points_gdf)
         for idx, row in tqdm(
-            self.iterrows(),
-            total=len(self),
-            disable=(not verbose),
-            desc="Exploding Points",
+            self.iterrows(), total=len(self), disable=(not verbose), desc="Exploding Points"
         ):
             geom = row["geometry"]
 
@@ -1037,11 +997,7 @@ def explode_to_points(self, reach_id_attribute="ID", sampling_size=None, verbose
 
     @staticmethod
     def conflate_points(
-        source_points,
-        target_points,
-        source_reach_id_attribute,
-        target_reach_id_attribute,
-        verbose=False,
+        source_points, target_points, source_reach_id_attribute, target_reach_id_attribute, verbose=False
     ):
         tree = STRtree(target_points.geometry.tolist())
 
@@ -1161,11 +1117,7 @@ def buffer_stream_branches(cls, stream_network, buffer_distance, verbose=True):
 
     @staticmethod
     def query_vectors_by_branch(
-        vector,
-        branch_ids,
-        branch_id_attribute,
-        out_filename_template=None,
-        vector_layer=None,
+        vector, branch_ids, branch_id_attribute, out_filename_template=None, vector_layer=None
     ):
         # load vaas
         if isinstance(vector, str):
@@ -1213,13 +1165,7 @@ def __find_matching_record(vector, attribute, value, matching="first"):
 
         return out_records
 
-    def clip(
-        self,
-        to_clip,
-        out_filename_template=None,
-        branch_id=None,
-        branch_id_attribute=None,
-    ):
+    def clip(self, to_clip, out_filename_template=None, branch_id=None, branch_id_attribute=None):
         """Clips a raster or vector to the stream branch polygons"""
 
         fileType = "raster"  # default
diff --git a/src/subdiv_chan_obank_src.py b/src/subdiv_chan_obank_src.py
index efe434dd3..cad5375a8 100755
--- a/src/subdiv_chan_obank_src.py
+++ b/src/subdiv_chan_obank_src.py
@@ -1,28 +1,30 @@
 #!/usr/bin/env python3
-import os
-import sys
 import argparse
 import datetime as dt
-import matplotlib.pyplot as plt
 import multiprocessing
-import numpy as np
-import pandas as pd
+import os
 import re
-import seaborn as sns
 import shutil
+import sys
 import traceback
 import warnings
 from functools import reduce
 from multiprocessing import Pool
-from os.path import isfile, join, dirname, isdir
+from os.path import dirname, isdir, isfile, join
 from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
 from tqdm import tqdm
 
+
 sns.set_theme(style="whitegrid")
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
 """
-    Compute channel geomety and Manning's equation using subdivision method (separate in-channel vs. overbank).
+    Compute channel geomety and Manning's equation using subdivision method (separate in-channel vs. overbank)
     Also apply unique Manning's n-values for channel and overbank using a user provided feature_id csv
 
     Parameters
@@ -30,7 +32,8 @@
     fim_dir : str
         Directory containing FIM output folders.
     mann_n_table : str
-        Path to a csv file containing Manning's n values by feature_id (must contain variables "feature_id", "channel_n", "overbank_n")
+        Path to a csv file containing Manning's n values by feature_id
+        (must contain variables "feature_id", "channel_n", "overbank_n")
     file_suffix : str
         Optional: Suffix to append to the output log file
     number_of_jobs : str
@@ -39,130 +42,267 @@
         Optional (True or False): use this flag to crate src plots for all hydroids (long run time)
 """
 
-def variable_mannings_calc(args):
 
-    in_src_bankfull_filename    = args[0]
-    df_mann                     = args[1]
-    huc                         = args[2]
-    branch_id                   = args[3]
-    htable_filename             = args[4]
-    output_suffix               = args[5]
-    src_plot_option             = args[6]
-    huc_output_dir              = args[7]
+def variable_mannings_calc(args):
+    in_src_bankfull_filename = args[0]
+    df_mann = args[1]
+    huc = args[2]
+    branch_id = args[3]
+    htable_filename = args[4]
+    output_suffix = args[5]
+    src_plot_option = args[6]
+    huc_output_dir = args[7]
 
     ## Read the src_full_crosswalked.csv
     log_text = 'Calculating modified SRC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
     try:
-        df_src_orig = pd.read_csv(in_src_bankfull_filename,dtype={'feature_id': 'int64'})
+        df_src_orig = pd.read_csv(in_src_bankfull_filename, dtype={'feature_id': 'int64'})
 
         ## Check that the channel ratio column the user specified exists in the def
         if 'Stage_bankfull' not in df_src_orig.columns:
-            print('WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified bankfull column: ' + 'Stage_bankfull')
+            print(
+                'WARNING --> '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + in_src_bankfull_filename
+                + ' does not contain the specified bankfull column: '
+                + 'Stage_bankfull'
+            )
             print('Skipping --> ' + str(huc) + '  branch id: ' + str(branch_id))
-            log_text += 'WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified bankfull column: ' + 'Stage_bankfull'  + '\n'
+            log_text += (
+                'WARNING --> '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + in_src_bankfull_filename
+                + ' does not contain the specified bankfull column: '
+                + 'Stage_bankfull'
+                + '\n'
+            )
         else:
-            df_src_orig.drop(['channel_n','overbank_n','subdiv_applied','Discharge (m3s-1)_subdiv','Volume_chan (m3)','Volume_obank (m3)','BedArea_chan (m2)','BedArea_obank (m2)','WettedPerimeter_chan (m)','WettedPerimeter_obank (m)'], axis=1, inplace=True, errors='ignore') # drop these cols (in case vmann was previously performed)
-            
+            df_src_orig.drop(
+                [
+                    'channel_n',
+                    'overbank_n',
+                    'subdiv_applied',
+                    'Discharge (m3s-1)_subdiv',
+                    'Volume_chan (m3)',
+                    'Volume_obank (m3)',
+                    'BedArea_chan (m2)',
+                    'BedArea_obank (m2)',
+                    'WettedPerimeter_chan (m)',
+                    'WettedPerimeter_obank (m)',
+                ],
+                axis=1,
+                inplace=True,
+                errors='ignore',
+            )  # drop these cols (in case vmann was previously performed)
+
             ## Calculate subdiv geometry variables
             print('Calculating subdiv variables for SRC: ' + str(huc) + '  branch id: ' + str(branch_id))
-            log_text = 'Calculating subdiv variables for SRC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+            log_text = (
+                'Calculating subdiv variables for SRC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
+            )
             df_src = subdiv_geometry(df_src_orig)
-            
-            ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the 'Stage_bankfull')
-            df_src = df_src.merge(df_mann,  how='left', on='feature_id')
+
+            ## Merge (crosswalk) the df of Manning's n with the SRC df
+            ##   (using the channel/fplain delination in the 'Stage_bankfull')
+            df_src = df_src.merge(df_mann, how='left', on='feature_id')
             check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
             if check_null > 0:
-                log_text += str(huc) + '  branch id: ' + str(branch_id) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
-
-            ## Check if there are any missing data in the 'Stage_bankfull' column (these are locations where subdiv will not be applied)
-            df_src['subdiv_applied'] = np.where(df_src['Stage_bankfull'].isnull(), False, True) # create field to identify where vmann is applied (True=yes; False=no)        
+                log_text += (
+                    str(huc)
+                    + '  branch id: '
+                    + str(branch_id)
+                    + ' --> '
+                    + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe'
+                    + ' --> missing entries= '
+                    + str(check_null / 84)
+                    + '\n'
+                )
+
+            ## Check if there are any missing data in the 'Stage_bankfull' column
+            ##   (these are locations where subdiv will not be applied)
+            df_src['subdiv_applied'] = np.where(
+                df_src['Stage_bankfull'].isnull(), False, True
+            )  # create field to identify where vmann is applied (True=yes; False=no)
 
             ## Calculate Manning's equation discharge for channel, overbank, and total
             df_src = subdiv_mannings_eq(df_src)
 
             ## Use the default discharge column when vmann is not being applied
-            df_src['Discharge (m3s-1)_subdiv'] = np.where(df_src['subdiv_applied']==False, df_src['Discharge (m3s-1)'], df_src['Discharge (m3s-1)_subdiv']) # reset the discharge value back to the original if vmann=false
+            df_src['Discharge (m3s-1)_subdiv'] = np.where(
+                df_src['subdiv_applied'] == False,
+                df_src['Discharge (m3s-1)'],
+                df_src['Discharge (m3s-1)_subdiv'],
+            )  # reset the discharge value back to the original if vmann=false
 
             ## Output new SRC with bankfull column
-            df_src.to_csv(in_src_bankfull_filename,index=False)
+            df_src.to_csv(in_src_bankfull_filename, index=False)
 
             ## Output new hydroTable with updated discharge and ManningN column
-            df_src_trim = df_src[['HydroID','Stage', 'Bathymetry_source', 'subdiv_applied','channel_n','overbank_n','Discharge (m3s-1)_subdiv']]
-            df_src_trim = df_src_trim.rename(columns={'Stage':'stage','Discharge (m3s-1)_subdiv': 'subdiv_discharge_cms'})
-            df_src_trim['discharge_cms'] = df_src_trim['subdiv_discharge_cms'] # create a copy of vmann modified discharge (used to track future changes)
-            df_htable = pd.read_csv(htable_filename,dtype={'HUC': str,'last_updated':object,'submitter':object,'obs_source':object})
+            df_src_trim = df_src[
+                [
+                    'HydroID',
+                    'Stage',
+                    'Bathymetry_source',
+                    'subdiv_applied',
+                    'channel_n',
+                    'overbank_n',
+                    'Discharge (m3s-1)_subdiv',
+                ]
+            ]
+            df_src_trim = df_src_trim.rename(
+                columns={'Stage': 'stage', 'Discharge (m3s-1)_subdiv': 'subdiv_discharge_cms'}
+            )
+            df_src_trim['discharge_cms'] = df_src_trim[
+                'subdiv_discharge_cms'
+            ]  # create a copy of vmann modified discharge (used to track future changes)
+            df_htable = pd.read_csv(
+                htable_filename,
+                dtype={'HUC': str, 'last_updated': object, 'submitter': object, 'obs_source': object},
+            )
 
             ## drop the previously modified discharge column to be replaced with updated version
-            df_htable.drop(['subdiv_applied','discharge_cms','overbank_n','channel_n','subdiv_discharge_cms','Bathymetry_source'], axis=1, errors='ignore', inplace=True) 
-            df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
+            df_htable.drop(
+                [
+                    'subdiv_applied',
+                    'discharge_cms',
+                    'overbank_n',
+                    'channel_n',
+                    'subdiv_discharge_cms',
+                    'Bathymetry_source',
+                ],
+                axis=1,
+                errors='ignore',
+                inplace=True,
+            )
+            df_htable = df_htable.merge(
+                df_src_trim, how='left', left_on=['HydroID', 'stage'], right_on=['HydroID', 'stage']
+            )
 
             ## Output new hydroTable csv
             if output_suffix != "":
-                htable_filename = os.path.splitext(htable_filename)[0] + output_suffix + '.csv' 
-            df_htable.to_csv(htable_filename,index=False)
+                htable_filename = os.path.splitext(htable_filename)[0] + output_suffix + '.csv'
+            df_htable.to_csv(htable_filename, index=False)
 
             log_text += 'Completed: ' + str(huc)
 
             ## plot rating curves
             if src_plot_option:
-                if isdir(huc_output_dir) == False:
+                if isdir(huc_output_dir) is False:
                     os.mkdir(huc_output_dir)
                 generate_src_plot(df_src, huc_output_dir)
     except Exception as ex:
-        summary = traceback.StackSummary.extract(
-                traceback.walk_stack(None))
-        print('WARNING: ' + str(huc) + '  branch id: ' + str(branch_id) + " subdivision failed for some reason")
-        #print(f"*** {ex}")                
-        #print(''.join(summary.format()))    
-        log_text += 'ERROR --> ' + str(huc) + '  branch id: ' + str(branch_id) + " subdivision failed (details: " + (f"*** {ex}") + (''.join(summary.format())) + '\n'
+        summary = traceback.StackSummary.extract(traceback.walk_stack(None))
+        print(
+            'WARNING: ' + str(huc) + '  branch id: ' + str(branch_id) + " subdivision failed for some reason"
+        )
+        # print(f"*** {ex}")
+        # print(''.join(summary.format()))
+        log_text += (
+            'ERROR --> '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+            + " subdivision failed (details: "
+            + (f"*** {ex}")
+            + (''.join(summary.format()))
+            + '\n'
+        )
+
+    return log_text
 
-    return(log_text)
 
 def subdiv_geometry(df_src):
-
     ## Calculate in-channel volume & bed area
-    df_src['Volume_chan (m3)'] = np.where(df_src['Stage']<=df_src['Stage_bankfull'], df_src['Volume (m3)'], (df_src['Volume_bankfull'] + ((df_src['Stage'] - df_src['Stage_bankfull']) * df_src['SurfArea_bankfull'])))
-    df_src['BedArea_chan (m2)'] = np.where(df_src['Stage']<=df_src['Stage_bankfull'], df_src['BedArea (m2)'], df_src['BedArea_bankfull'])
-    df_src['WettedPerimeter_chan (m)'] = np.where(df_src['Stage']<=df_src['Stage_bankfull'], (df_src['BedArea_chan (m2)']/df_src['LENGTHKM']/1000), (df_src['BedArea_chan (m2)']/df_src['LENGTHKM']/1000) + ((df_src['Stage'] - df_src['Stage_bankfull'])*2))
+    df_src['Volume_chan (m3)'] = np.where(
+        df_src['Stage'] <= df_src['Stage_bankfull'],
+        df_src['Volume (m3)'],
+        (
+            df_src['Volume_bankfull']
+            + ((df_src['Stage'] - df_src['Stage_bankfull']) * df_src['SurfArea_bankfull'])
+        ),
+    )
+    df_src['BedArea_chan (m2)'] = np.where(
+        df_src['Stage'] <= df_src['Stage_bankfull'], df_src['BedArea (m2)'], df_src['BedArea_bankfull']
+    )
+    df_src['WettedPerimeter_chan (m)'] = np.where(
+        df_src['Stage'] <= df_src['Stage_bankfull'],
+        (df_src['BedArea_chan (m2)'] / df_src['LENGTHKM'] / 1000),
+        (df_src['BedArea_chan (m2)'] / df_src['LENGTHKM'] / 1000)
+        + ((df_src['Stage'] - df_src['Stage_bankfull']) * 2),
+    )
 
     ## Calculate overbank volume & bed area
-    df_src['Volume_obank (m3)'] = np.where(df_src['Stage']>df_src['Stage_bankfull'], (df_src['Volume (m3)'] - df_src['Volume_chan (m3)']), 0.0)
-    df_src['BedArea_obank (m2)'] = np.where(df_src['Stage']>df_src['Stage_bankfull'], (df_src['BedArea (m2)'] - df_src['BedArea_chan (m2)']), 0.0)
-    df_src['WettedPerimeter_obank (m)'] = df_src['BedArea_obank (m2)']/df_src['LENGTHKM']/1000
-    return(df_src)
+    df_src['Volume_obank (m3)'] = np.where(
+        df_src['Stage'] > df_src['Stage_bankfull'], (df_src['Volume (m3)'] - df_src['Volume_chan (m3)']), 0.0
+    )
+    df_src['BedArea_obank (m2)'] = np.where(
+        df_src['Stage'] > df_src['Stage_bankfull'],
+        (df_src['BedArea (m2)'] - df_src['BedArea_chan (m2)']),
+        0.0,
+    )
+    df_src['WettedPerimeter_obank (m)'] = df_src['BedArea_obank (m2)'] / df_src['LENGTHKM'] / 1000
+    return df_src
+
 
 def subdiv_mannings_eq(df_src):
     ## Calculate discharge (channel) using Manning's equation
-    df_src.drop(['WetArea_chan (m2)','HydraulicRadius_chan (m)','Discharge_chan (m3s-1)','Velocity_chan (m/s)'], axis=1, inplace=True, errors='ignore') # drop these cols (in case subdiv was previously performed)
-    df_src['WetArea_chan (m2)'] = df_src['Volume_chan (m3)']/df_src['LENGTHKM']/1000
-    df_src['HydraulicRadius_chan (m)'] = df_src['WetArea_chan (m2)']/df_src['WettedPerimeter_chan (m)']
+    df_src.drop(
+        ['WetArea_chan (m2)', 'HydraulicRadius_chan (m)', 'Discharge_chan (m3s-1)', 'Velocity_chan (m/s)'],
+        axis=1,
+        inplace=True,
+        errors='ignore',
+    )  # drop these cols (in case subdiv was previously performed)
+    df_src['WetArea_chan (m2)'] = df_src['Volume_chan (m3)'] / df_src['LENGTHKM'] / 1000
+    df_src['HydraulicRadius_chan (m)'] = df_src['WetArea_chan (m2)'] / df_src['WettedPerimeter_chan (m)']
     df_src['HydraulicRadius_chan (m)'].fillna(0, inplace=True)
-    df_src['Discharge_chan (m3s-1)'] = df_src['WetArea_chan (m2)']* \
-    pow(df_src['HydraulicRadius_chan (m)'],2.0/3)* \
-    pow(df_src['SLOPE'],0.5)/df_src['channel_n']
-    df_src['Velocity_chan (m/s)'] = df_src['Discharge_chan (m3s-1)']/df_src['WetArea_chan (m2)']
+    df_src['Discharge_chan (m3s-1)'] = (
+        df_src['WetArea_chan (m2)']
+        * pow(df_src['HydraulicRadius_chan (m)'], 2.0 / 3)
+        * pow(df_src['SLOPE'], 0.5)
+        / df_src['channel_n']
+    )
+    df_src['Velocity_chan (m/s)'] = df_src['Discharge_chan (m3s-1)'] / df_src['WetArea_chan (m2)']
     df_src['Velocity_chan (m/s)'].fillna(0, inplace=True)
 
     ## Calculate discharge (overbank) using Manning's equation
-    df_src.drop(['WetArea_obank (m2)','HydraulicRadius_obank (m)','Discharge_obank (m3s-1)','Velocity_obank (m/s)'], axis=1, inplace=True, errors='ignore') # drop these cols (in case subdiv was previously performed)
-    df_src['WetArea_obank (m2)'] = df_src['Volume_obank (m3)']/df_src['LENGTHKM']/1000
-    df_src['HydraulicRadius_obank (m)'] = df_src['WetArea_obank (m2)']/df_src['WettedPerimeter_obank (m)']
-    df_src.replace([np.inf, -np.inf], np.nan, inplace=True) # need to replace inf instances (divide by 0)
+    df_src.drop(
+        [
+            'WetArea_obank (m2)',
+            'HydraulicRadius_obank (m)',
+            'Discharge_obank (m3s-1)',
+            'Velocity_obank (m/s)',
+        ],
+        axis=1,
+        inplace=True,
+        errors='ignore',
+    )  # drop these cols (in case subdiv was previously performed)
+    df_src['WetArea_obank (m2)'] = df_src['Volume_obank (m3)'] / df_src['LENGTHKM'] / 1000
+    df_src['HydraulicRadius_obank (m)'] = df_src['WetArea_obank (m2)'] / df_src['WettedPerimeter_obank (m)']
+    df_src.replace([np.inf, -np.inf], np.nan, inplace=True)  # need to replace inf instances (divide by 0)
     df_src['HydraulicRadius_obank (m)'].fillna(0, inplace=True)
-    df_src['Discharge_obank (m3s-1)'] = df_src['WetArea_obank (m2)']* \
-    pow(df_src['HydraulicRadius_obank (m)'],2.0/3)* \
-    pow(df_src['SLOPE'],0.5)/df_src['overbank_n']
-    df_src['Velocity_obank (m/s)'] = df_src['Discharge_obank (m3s-1)']/df_src['WetArea_obank (m2)']
+    df_src['Discharge_obank (m3s-1)'] = (
+        df_src['WetArea_obank (m2)']
+        * pow(df_src['HydraulicRadius_obank (m)'], 2.0 / 3)
+        * pow(df_src['SLOPE'], 0.5)
+        / df_src['overbank_n']
+    )
+    df_src['Velocity_obank (m/s)'] = df_src['Discharge_obank (m3s-1)'] / df_src['WetArea_obank (m2)']
     df_src['Velocity_obank (m/s)'].fillna(0, inplace=True)
 
     ## Calcuate the total of the subdivided discharge (channel + overbank)
-    df_src.drop(['Discharge (m3s-1)_subdiv'], axis=1, inplace=True, errors='ignore') # drop these cols (in case subdiv was previously performed)
+    df_src.drop(
+        ['Discharge (m3s-1)_subdiv'], axis=1, inplace=True, errors='ignore'
+    )  # drop these cols (in case subdiv was previously performed)
     df_src['Discharge (m3s-1)_subdiv'] = df_src['Discharge_chan (m3s-1)'] + df_src['Discharge_obank (m3s-1)']
-    df_src.loc[df_src['Stage']==0,['Discharge (m3s-1)_subdiv']] = 0
-    return(df_src)
+    df_src.loc[df_src['Stage'] == 0, ['Discharge (m3s-1)_subdiv']] = 0
+    return df_src
 
-def generate_src_plot(df_src, plt_out_dir):
 
+def generate_src_plot(df_src, plt_out_dir):
     ## create list of unique hydroids
     hydroids = df_src.HydroID.unique().tolist()
 
@@ -175,76 +315,149 @@ def generate_src_plot(df_src, plt_out_dir):
         ax.set_title(str(hydroid))
         sns.despine(f, left=True, bottom=True)
         sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
-        sns.scatterplot(x='Discharge (m3s-1)_subdiv', y='Stage', data=plot_df, label="SRC w/ Subdiv", ax=ax, color='orange')
-        sns.scatterplot(x='Discharge_chan (m3s-1)', y='Stage', data=plot_df, label="SRC Channel", ax=ax, color='green', s=8)
-        sns.scatterplot(x='Discharge_obank (m3s-1)', y='Stage', data=plot_df, label="SRC Overbank", ax=ax, color='purple', s=8)
+        sns.scatterplot(
+            x='Discharge (m3s-1)_subdiv',
+            y='Stage',
+            data=plot_df,
+            label="SRC w/ Subdiv",
+            ax=ax,
+            color='orange',
+        )
+        sns.scatterplot(
+            x='Discharge_chan (m3s-1)',
+            y='Stage',
+            data=plot_df,
+            label="SRC Channel",
+            ax=ax,
+            color='green',
+            s=8,
+        )
+        sns.scatterplot(
+            x='Discharge_obank (m3s-1)',
+            y='Stage',
+            data=plot_df,
+            label="SRC Overbank",
+            ax=ax,
+            color='purple',
+            s=8,
+        )
         sns.lineplot(x='Discharge (m3s-1)', y='Stage_bankfull', data=plot_df, color='green', ax=ax)
-        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'],alpha=0.5)
-        plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_bankfull'].median(), "NWM Bankfull Approx: " + str(plot_df['Stage_bankfull'].median()))
+        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'], alpha=0.5)
+        plt.text(
+            plot_df['Discharge (m3s-1)'].median(),
+            plot_df['Stage_bankfull'].median(),
+            "NWM Bankfull Approx: " + str(plot_df['Stage_bankfull'].median()),
+        )
         ax.legend()
-        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png',dpi=175, bbox_inches='tight')
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png', dpi=175, bbox_inches='tight')
         plt.close()
 
+
 def multi_process(variable_mannings_calc, procs_list, log_file, number_of_jobs, verbose):
     ## Initiate multiprocessing
     available_cores = multiprocessing.cpu_count()
     if number_of_jobs > available_cores:
         number_of_jobs = available_cores - 2
-        print("Provided job number exceeds the number of available cores. " + str(number_of_jobs) + " max jobs will be used instead.")
-
-    print(f"Computing subdivided SRC and applying variable Manning's n to channel/overbank for {len(procs_list)} branches using {number_of_jobs} jobs")
+        print(
+            "Provided job number exceeds the number of available cores. "
+            + str(number_of_jobs)
+            + " max jobs will be used instead."
+        )
+
+    print(
+        "Computing subdivided SRC and applying variable Manning's n to channel/overbank for "
+        f"{len(procs_list)} branches using {number_of_jobs} jobs"
+    )
     with Pool(processes=number_of_jobs) as pool:
         if verbose:
             map_output = tqdm(pool.imap(variable_mannings_calc, procs_list), total=len(procs_list))
             tuple(map_output)  # fetch the lazy results
         else:
             map_output = pool.map(variable_mannings_calc, procs_list)
-    log_file.writelines(["%s\n" % item  for item in map_output])
+    log_file.writelines(["%s\n" % item for item in map_output])
+
 
-def run_prep(fim_dir,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option):
+def run_prep(fim_dir, mann_n_table, output_suffix, number_of_jobs, verbose, src_plot_option):
     procs_list = []
 
-    print('Writing progress to log file here: ' + str(join(fim_dir,'logs','subdiv_src_' + output_suffix + '.log')))
+    print(f"Writing progress to log file here: {fim_dir}/logs/subdiv_src_{output_suffix}.log")
     print('This may take a few minutes...')
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
 
     ## initiate log file
-    log_file = open(join(fim_dir,'logs','subdiv_src_' + output_suffix + '.log'),"w")
+    log_file = open(join(fim_dir, 'logs', 'subdiv_src_' + output_suffix + '.log'), "w")
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
 
     ## Check that the input fim_dir exists
     assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
     ## Check that the manning's roughness input filepath exists and then read to dataframe
-    assert os.path.isfile(mann_n_table), 'Can not find the input roughness/feature_id file: ' + str(mann_n_table)
-    
+    assert os.path.isfile(mann_n_table), 'Can not find the input roughness/feature_id file: ' + str(
+        mann_n_table
+    )
+
     ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
     print('Importing the Manning roughness data file: ' + mann_n_table)
-    df_mann = pd.read_csv(mann_n_table,dtype={'feature_id': 'int64'})
-    if 'channel_n' not in df_mann.columns or 'overbank_n' not in df_mann.columns or 'feature_id' not in df_mann.columns:
-        print('Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann)
+    df_mann = pd.read_csv(mann_n_table, dtype={'feature_id': 'int64'})
+    if (
+        'channel_n' not in df_mann.columns
+        or 'overbank_n' not in df_mann.columns
+        or 'feature_id' not in df_mann.columns
+    ):
+        print(
+            'Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann
+        )
     else:
         print('Running the variable_mannings_calc function...')
 
         ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
-        huc_list  = os.listdir(fim_dir)
-        huc_list.sort() # sort huc_list for helping track progress in future print statments
+        huc_list = os.listdir(fim_dir)
+        huc_list.sort()  # sort huc_list for helping track progress in future print statments
         for huc in huc_list:
-            #if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
-            if re.match('\d{8}', huc):    
-                huc_branches_dir = os.path.join(fim_dir, huc,'branches')
+            # if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+            if re.match(r'\d{8}', huc):
+                huc_branches_dir = os.path.join(fim_dir, huc, 'branches')
                 for branch_id in os.listdir(huc_branches_dir):
-                    branch_dir = os.path.join(huc_branches_dir,branch_id)
-                    in_src_bankfull_filename = join(branch_dir,'src_full_crosswalked_' + branch_id + '.csv')
-                    htable_filename = join(branch_dir,'hydroTable_' + branch_id + '.csv')
-                    huc_plot_output_dir = join(branch_dir,'src_plots')
+                    branch_dir = os.path.join(huc_branches_dir, branch_id)
+                    in_src_bankfull_filename = join(branch_dir, 'src_full_crosswalked_' + branch_id + '.csv')
+                    htable_filename = join(branch_dir, 'hydroTable_' + branch_id + '.csv')
+                    huc_plot_output_dir = join(branch_dir, 'src_plots')
 
                     if isfile(in_src_bankfull_filename) and isfile(htable_filename):
-                        procs_list.append([in_src_bankfull_filename, df_mann, huc, branch_id, htable_filename, output_suffix, src_plot_option, huc_plot_output_dir])
+                        procs_list.append(
+                            [
+                                in_src_bankfull_filename,
+                                df_mann,
+                                huc,
+                                branch_id,
+                                htable_filename,
+                                output_suffix,
+                                src_plot_option,
+                                huc_plot_output_dir,
+                            ]
+                        )
                     else:
-                        print('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
-                        log_file.write('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+                        print(
+                            'HUC: '
+                            + str(huc)
+                            + '  branch id: '
+                            + str(branch_id)
+                            + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv '
+                            + 'or hydroTable_*.csv) in the fim output dir: '
+                            + str(branch_dir)
+                            + ' - skipping this branch!!!\n'
+                        )
+                        log_file.write(
+                            'HUC: '
+                            + str(huc)
+                            + '  branch id: '
+                            + str(branch_id)
+                            + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv '
+                            + 'or hydroTable_*.csv) in the fim output dir: '
+                            + str(branch_dir)
+                            + ' - skipping this branch!!!\n'
+                        )
 
         ## Pass huc procs_list to multiprocessing function
         multi_process(variable_mannings_calc, procs_list, log_file, number_of_jobs, verbose)
@@ -256,15 +469,53 @@ def run_prep(fim_dir,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_
         log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
         log_file.close()
 
-if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description="Subdivide the default SRC to compute a seperate channel component and overbank component. Impliment user provided Manning's n values for in-channel vs. overbank flow. Recalculate Manning's eq for discharge")
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-mann','--mann-n-table',help="Path to a csv file containing Manning's n values by featureid",required=True,type=str)
-    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",default="",required=False,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='OPTIONAL: number of workers (default=8)',required=False,default=8,type=int)
-    parser.add_argument('-vb','--verbose',help='OPTIONAL: verbose progress bar',required=False,default=None,action='store_true')
-    parser.add_argument('-plots','--src-plot-option',help='OPTIONAL flag: use this flag to create src plots for all hydroids. WARNING - long runtime',default=False,required=False, action='store_true')
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Subdivide the default SRC to compute a seperate channel component and overbank"
+        "component. Impliment user provided Manning's n values for in-channel vs. overbank flow. "
+        "Recalculate Manning's eq for discharge"
+    )
+    parser.add_argument('-fim_dir', '--fim-dir', help='FIM output dir', required=True, type=str)
+    parser.add_argument(
+        '-mann',
+        '--mann-n-table',
+        help="Path to a csv file containing Manning's n values by featureid",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-suff',
+        '--output-suffix',
+        help="Suffix to append to the output log file (e.g. '_global_06_011')",
+        default="",
+        required=False,
+        type=str,
+    )
+    parser.add_argument(
+        '-j',
+        '--number-of-jobs',
+        help='OPTIONAL: number of workers (default=8)',
+        required=False,
+        default=8,
+        type=int,
+    )
+    parser.add_argument(
+        '-vb',
+        '--verbose',
+        help='OPTIONAL: verbose progress bar',
+        required=False,
+        default=None,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-plots',
+        '--src-plot-option',
+        help='OPTIONAL flag: use this flag to create src plots for all hydroids. WARNING - long runtime',
+        default=False,
+        required=False,
+        action='store_true',
+    )
 
     args = vars(parser.parse_args())
 
@@ -275,4 +526,4 @@ def run_prep(fim_dir,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_
     verbose = bool(args['verbose'])
     src_plot_option = args['src_plot_option']
 
-    run_prep(fim_dir,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option)
\ No newline at end of file
+    run_prep(fim_dir, mann_n_table, output_suffix, number_of_jobs, verbose, src_plot_option)
diff --git a/src/subset_catch_list_by_branch_id.py b/src/subset_catch_list_by_branch_id.py
index 606d4e99e..871ad3080 100755
--- a/src/subset_catch_list_by_branch_id.py
+++ b/src/subset_catch_list_by_branch_id.py
@@ -1,70 +1,84 @@
 #!/usr/bin/env python3
 
-import pandas as pd
+import argparse
+from os.path import splitext
+
 import geopandas as gpd
+import pandas as pd
+
 from stream_branches import StreamNetwork
-from os.path import splitext
-import argparse
 
-def Subset_catch_list(catch_list,stream_network,branch_id_attribute,branch_id_list=None,out_catch_list=None,verbose=False):
 
+def Subset_catch_list(
+    catch_list, stream_network, branch_id_attribute, branch_id_list=None, out_catch_list=None, verbose=False
+):
     if verbose:
         print("Loading files ....")
-    
+
     # loading files
-    catch_list = pd.read_csv(catch_list,sep=" ",header=None,skiprows=1)
-    catch_list.rename(columns={0:"HydroID",1:"slopes",2:"lengthKM",3:"areasqkm"},inplace=True)
-    stream_network = StreamNetwork.from_file(stream_network,branch_id_attribute=branch_id_attribute)
-    stream_network = StreamNetwork(stream_network.astype({'HydroID':int}),branch_id_attribute=branch_id_attribute)
+    catch_list = pd.read_csv(catch_list, sep=" ", header=None, skiprows=1)
+    catch_list.rename(columns={0: "HydroID", 1: "slopes", 2: "lengthKM", 3: "areasqkm"}, inplace=True)
+    stream_network = StreamNetwork.from_file(stream_network, branch_id_attribute=branch_id_attribute)
+    stream_network = StreamNetwork(
+        stream_network.astype({'HydroID': int}), branch_id_attribute=branch_id_attribute
+    )
 
     if verbose:
         print("Merging HydroIDs ... ")
-    catch_list = catch_list.merge(stream_network.loc[:,["HydroID",branch_id_attribute]],on='HydroID',how='inner')
+    catch_list = catch_list.merge(
+        stream_network.loc[:, ["HydroID", branch_id_attribute]], on='HydroID', how='inner'
+    )
 
-    unique_branch_ids = catch_list.loc[:,branch_id_attribute].sort_values().unique()
-    base_file_path,extension = splitext(out_catch_list)
+    unique_branch_ids = catch_list.loc[:, branch_id_attribute].sort_values().unique()
+    base_file_path, extension = splitext(out_catch_list)
 
     if branch_id_list:
         # write unique branch ids to file
-        if verbose: 
+        if verbose:
             print("Writing branch id list ...")
-    
-        unique_branch_ids.tofile(branch_id_list,sep="\n")
+
+        unique_branch_ids.tofile(branch_id_list, sep="\n")
 
     if verbose:
         print("Writing catch list subsets ...")
     for bid in unique_branch_ids:
-
         # subsetting to branch id and getting number of hydroids
-        branch_catch_list = catch_list.loc[ catch_list.loc[:,branch_id_attribute] == bid, : ]
+        branch_catch_list = catch_list.loc[catch_list.loc[:, branch_id_attribute] == bid, :]
         num_of_hydroIDs = len(branch_catch_list)
 
         # dropping branch id attribute
         branch_catch_list = branch_catch_list.drop(columns=branch_id_attribute)
 
         # out file name management
-        out_branch_catch_list = "{}_{}{}".format(base_file_path,bid,extension)
+        out_branch_catch_list = "{}_{}{}".format(base_file_path, bid, extension)
 
         # write number of hydroids
-        with open(out_branch_catch_list,'w') as f:
+        with open(out_branch_catch_list, 'w') as f:
             f.write("{}\n".format(num_of_hydroIDs))
 
         # write out catch list in append mode
-        branch_catch_list.to_csv(out_branch_catch_list, mode = 'a',header=False,sep=" ",index=False)
+        branch_catch_list.to_csv(out_branch_catch_list, mode='a', header=False, sep=" ", index=False)
 
-    return(catch_list)
+    return catch_list
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='Subsets catch list to branch scale')
-    parser.add_argument('-c','--catch-list', help='Input catchment list', required=True)
-    parser.add_argument('-s','--stream-network', help='Stream Network with HydroIDs and Branch IDs', required=True)
-    parser.add_argument('-b','--branch-id-attribute', help='Name of the branch attribute desired', required=True)
-    parser.add_argument('-l','--branch-id-list', help='Output the branch id list file desired', required=False,default=None)
-    parser.add_argument('-o','--out-catch-list', help='Output catchment list', required=False,default=None)
-    parser.add_argument('-v','--verbose', help='Verbose output', required=False,default=False,action='store_true')
-    
+    parser.add_argument('-c', '--catch-list', help='Input catchment list', required=True)
+    parser.add_argument(
+        '-s', '--stream-network', help='Stream Network with HydroIDs and Branch IDs', required=True
+    )
+    parser.add_argument(
+        '-b', '--branch-id-attribute', help='Name of the branch attribute desired', required=True
+    )
+    parser.add_argument(
+        '-l', '--branch-id-list', help='Output the branch id list file desired', required=False, default=None
+    )
+    parser.add_argument('-o', '--out-catch-list', help='Output catchment list', required=False, default=None)
+    parser.add_argument(
+        '-v', '--verbose', help='Verbose output', required=False, default=False, action='store_true'
+    )
+
     args = vars(parser.parse_args())
 
     Subset_catch_list(**args)
diff --git a/src/toDo.md b/src/toDo.md
index 53bd35845..d907a670c 100755
--- a/src/toDo.md
+++ b/src/toDo.md
@@ -1,7 +1,7 @@
 # To Do List
 
 ## Eval
-- *Testing Architecture & Modularity:* The test case functionality requires more modularity. There should be a clear set of tools that abstract away the model or test cases used. 
+- *Testing Architecture & Modularity:* The test case functionality requires more modularity. There should be a clear set of tools that abstract away the model or test cases used.
     - User tools should allow any predicted raster and any benchmark raster to be compared given proper encoding.
         - This function should be command line oriented and allow for a list or file to be evaluated using multiple CPUs
     - Eval tools for FIM 3, Lisflood, GMS, etc can then wrap around these core eval tools
@@ -12,7 +12,7 @@
 - *Evaluation extents:* The evaluation extents between FR, MS, GMS are not consistent. While this does have a small effect on CSI, MCC should be immune to it and generally shows a similar trend
 
 
-## GMS 
+## GMS
 - *NWM Divergences:* Levelpath derivation doesn't handle divergences (eg 12020002).
     - it shortens the effective length of levelpaths thus reducing the rating curve height of the most upstream catchment
     - it also creates more levelpaths and likely increases computational time
diff --git a/src/unique_pixel_and_allocation.py b/src/unique_pixel_and_allocation.py
index b100d4c86..3e242a547 100755
--- a/src/unique_pixel_and_allocation.py
+++ b/src/unique_pixel_and_allocation.py
@@ -4,11 +4,13 @@
 
 @author: trevor.grout
 """
+import argparse
 import os
-import rasterio
+
 import numpy as np
-import argparse
+import rasterio
 import whitebox
+
 from utils.shared_functions import mem_profile
 
 
@@ -38,31 +40,31 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels):
 
     workspace = os.path.dirname(unique_stream_pixels)
     base = os.path.basename(unique_stream_pixels)
-    distance_grid = os.path.join(workspace,os.path.splitext(base)[0]+'_dist.tif')
-    allocation_grid = os.path.join(workspace,os.path.splitext(base)[0]+'_allo.tif')
-    
+    distance_grid = os.path.join(workspace, os.path.splitext(base)[0] + '_dist.tif')
+    allocation_grid = os.path.join(workspace, os.path.splitext(base)[0] + '_allo.tif')
+
     # Import stream pixel raster
     with rasterio.open(stream_pixels) as temp:
         streams_profile = temp.profile
         streams = temp.read(1)
 
     # Create array that matches shape of streams raster with unique values for each cell. Dataype is float64.
-    unique_vals = np.arange(streams.size, dtype = 'float64').reshape(*streams.shape)
+    unique_vals = np.arange(streams.size, dtype='float64').reshape(*streams.shape)
 
     # At streams return the unique array value otherwise return 0 values
     stream_pixel_values = np.where(streams == 1, unique_vals, 0)
 
     # Reassign dtype to be float64 (needs to be float64)
-    streams_profile.update(dtype = 'float64')
+    streams_profile.update(dtype='float64')
 
     # Output to raster
     with rasterio.Env():
         with rasterio.open(unique_stream_pixels, 'w', **streams_profile) as raster:
-            raster.write(stream_pixel_values,1)
+            raster.write(stream_pixel_values, 1)
 
     # Compute allocation and proximity grids.
-    wbt.euclidean_distance(stream_pixels,distance_grid)
-    wbt.euclidean_allocation(unique_stream_pixels,allocation_grid)
+    wbt.euclidean_distance(stream_pixels, distance_grid)
+    wbt.euclidean_allocation(unique_stream_pixels, allocation_grid)
 
     with rasterio.open(allocation_grid) as allocation_ds:
         allocation = allocation_ds.read(1)
@@ -74,16 +76,18 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels):
     with rasterio.open(allocation_grid, 'w', **allocation_profile) as allocation_ds:
         allocation_ds.write(allocation, 1)
 
-
     return distance_grid, allocation_grid
 
 
 if __name__ == '__main__':
-
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Produce unique stream pixel values and allocation/proximity grids')
-    parser.add_argument('-s', '--stream', help = 'raster to perform r.grow.distance', required = True)
-    parser.add_argument('-o', '--out', help = 'output raster of unique ids for each stream pixel', required = True)
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description='Produce unique stream pixel values and allocation/proximity grids'
+    )
+    parser.add_argument('-s', '--stream', help='raster to perform r.grow.distance', required=True)
+    parser.add_argument(
+        '-o', '--out', help='output raster of unique ids for each stream pixel', required=True
+    )
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index c93381aae..75af9695a 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -1,12 +1,16 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
+import warnings
+from os.path import join
+
 import geopandas as gpd
 import rasterio
-import argparse
-import warnings
+
 from utils.shared_functions import mem_profile
-from os.path import join
+
+
 warnings.simplefilter("ignore")
 
 
@@ -19,7 +23,8 @@
     input_flows_filename : str
         File path of FIM streams layer. i.e. '/data/path/demDerived_reaches_split_filtered_3246000257.gpkg'
     input_catchment_filename : str
-        File path of FIM catchment layer. i.e. '/data/path/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg'
+        File path of FIM catchment layer. i.e.
+            '/data/path/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg'
     dem_filename : str
         File path of original DEM. i.e. '/data/path/dem_meters_3246000257.tif'
     dem_adj_filename : str
@@ -30,15 +35,16 @@
         ID of the current branch i.e. '3246000257'
 '''
 
-class GageCrosswalk(object):
 
+class GageCrosswalk(object):
     def __init__(self, usgs_subset_gages_filename, branch_id):
-        
         self.branch_id = branch_id
         self.gages = self._load_gages(usgs_subset_gages_filename)
 
-    def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory):
-        '''Run the gage crosswalk steps: 1) spatial join to branch catchments layer 2) snap sites to 
+    def run_crosswalk(
+        self, input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory
+    ):
+        '''Run the gage crosswalk steps: 1) spatial join to branch catchments layer 2) snap sites to
         the dem-derived flows 3) sample both dems at the snapped points 4) write the crosswalked points
         to usgs_elev_table.csv
         '''
@@ -61,36 +67,44 @@ def run_crosswalk(self, input_catchment_filename, input_flows_filename, dem_file
         print(f"{num_gages} gage{'' if num_gages == 1 else 's'} in branch {self.branch_id}")
         self.write(output_directory)
 
-
     def _load_gages(self, gages_filename):
         '''Reads gage geopackage from huc level and filters based on current branch id'''
 
         usgs_gages = gpd.read_file(gages_filename)
-        return  usgs_gages[(usgs_gages.levpa_id == self.branch_id)]
+        return usgs_gages[(usgs_gages.levpa_id == self.branch_id)]
 
     def catchment_sjoin(self, input_catchment_filename):
         '''Spatial joins gages to FIM catchments'''
 
-        input_catchments = gpd.read_file(input_catchment_filename, dtype={'HydroID':int})
+        input_catchments = gpd.read_file(input_catchment_filename, dtype={'HydroID': int})
         self.gages = gpd.sjoin(self.gages, input_catchments[['HydroID', 'LakeID', 'geometry']], how='inner')
 
     def snap_to_dem_derived_flows(self, input_flows_filename):
-        '''Joins to dem derived flow line and produces snap_distance and geometry_snapped for sampling DEMs on the thalweg'''
+        '''
+        Joins to dem derived flow line and produces snap_distance and
+        geometry_snapped for sampling DEMs on the thalweg
+        '''
 
         input_flows = gpd.read_file(input_flows_filename)
         input_flows['geometry_ln'] = input_flows.geometry
         self.gages = self.gages.merge(input_flows[['HydroID', 'geometry_ln']], on='HydroID')
-        
+
         # Snap each point to its feature_id line
-        self.gages['geometry_snapped'], self.gages['snap_distance'] = self.gages.apply(self.snap_to_line, axis=1,result_type='expand').T.values
+        self.gages['geometry_snapped'], self.gages['snap_distance'] = self.gages.apply(
+            self.snap_to_line, axis=1, result_type='expand'
+        ).T.values
         self.gages.geometry_snapped = self.gages.geometry_snapped.astype('geometry')
 
     def sample_dem(self, dem_filename, column_name):
-        '''Sample an input DEM at snapped points. Make sure to run self.gages.set_geometry("geometry_snapped") before runnig
-        this method, otherwise the DEM will be sampled at the actual gage locations.'''
+        '''
+        Sample an input DEM at snapped points. Make sure to run self.gages.set_geometry("geometry_snapped")
+        before running this method, otherwise the DEM will be sampled at the actual gage locations.
+        '''
+
+        coord_list = [
+            (x, y) for x, y in zip(self.gages['geometry_snapped'].x, self.gages['geometry_snapped'].y)
+        ]
 
-        coord_list = [(x,y) for x,y in zip(self.gages['geometry_snapped'].x , self.gages['geometry_snapped'].y)]
-        
         with rasterio.open(dem_filename) as dem:
             self.gages[column_name] = [x[0] for x in dem.sample(coord_list)]
 
@@ -99,13 +113,27 @@ def write(self, output_directory):
 
         # Prep and write out file
         elev_table = self.gages.copy()
-        elev_table.loc[elev_table['location_id'] == elev_table['nws_lid'], 'location_id'] = None # set location_id to None where there isn't a gage
+        elev_table.loc[
+            elev_table['location_id'] == elev_table['nws_lid'], 'location_id'
+        ] = None  # set location_id to None where there isn't a gage
         elev_table = elev_table[elev_table['location_id'].notna()]
         elev_table.source = elev_table.source.apply(str.lower)
 
         # filter for just ras2fim entries
         ras_elev_table = elev_table[elev_table['source'] == 'ras2fim']
-        ras_elev_table = ras_elev_table[["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_elevation", "dem_adj_elevation","source", "stream_stn"]]
+        ras_elev_table = ras_elev_table[
+            [
+                "location_id",
+                "HydroID",
+                "feature_id",
+                "levpa_id",
+                "HUC8",
+                "dem_elevation",
+                "dem_adj_elevation",
+                "source",
+                "stream_stn",
+            ]
+        ]
         if not ras_elev_table.empty:
             ras_elev_table.to_csv(join(output_directory, 'ras_elev_table.csv'), index=False)
 
@@ -123,15 +151,20 @@ def snap_to_line(row):
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='Crosswalk USGS sites to HydroID and get elevations')
-    parser.add_argument('-gages','--usgs-gages-filename', help='USGS gage subset at the huc level', required=True)
-    parser.add_argument('-flows','--input-flows-filename', help='DEM derived streams', required=True)
-    parser.add_argument('-cat','--input-catchment-filename', help='DEM derived catchments', required=True)
-    parser.add_argument('-dem','--dem-filename',help='DEM',required=True)
-    parser.add_argument('-dem_adj','--dem-adj-filename', help='Thalweg adjusted DEM', required=True)
-    parser.add_argument('-out','--output-directory', help='Directory where output tables created', required=True)
-    parser.add_argument('-b','--branch-id', help='Branch ID used to filter the gages', type=str, required=True)
+    parser.add_argument(
+        '-gages', '--usgs-gages-filename', help='USGS gage subset at the huc level', required=True
+    )
+    parser.add_argument('-flows', '--input-flows-filename', help='DEM derived streams', required=True)
+    parser.add_argument('-cat', '--input-catchment-filename', help='DEM derived catchments', required=True)
+    parser.add_argument('-dem', '--dem-filename', help='DEM', required=True)
+    parser.add_argument('-dem_adj', '--dem-adj-filename', help='Thalweg adjusted DEM', required=True)
+    parser.add_argument(
+        '-out', '--output-directory', help='Directory where output tables created', required=True
+    )
+    parser.add_argument(
+        '-b', '--branch-id', help='Branch ID used to filter the gages', type=str, required=True
+    )
 
     args = vars(parser.parse_args())
 
@@ -147,32 +180,34 @@ def snap_to_line(row):
 
     # Instantiate class
     gage_crosswalk = GageCrosswalk(usgs_gages_filename, branch_id)
-    gage_crosswalk.run_crosswalk(input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory)
+    gage_crosswalk.run_crosswalk(
+        input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory
+    )
 
 """
 Examples:
 
-python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg                    \ 
-    -flows /outputs/carson_gms_bogus/02020005/branches/3246000305/demDerived_reaches_split_filtered_3246000305.gpkg             \ 
-    -cat /outputs/carson_gms_bogus/02020005/branches/3246000305/gw_catchments_reaches_filtered_addedAttributes_3246000305.gpkg  \ 
-    -dem /outputs/carson_gms_bogus/02020005/branches/3246000305/dem_meters_3246000305.tif                                       \ 
-    -dem_adj /outputs/carson_gms_bogus/02020005/branches/3246000305/dem_thalwegCond_3246000305.tif                              \ 
-    -outtable /outputs/carson_gms_bogus/02020005/branches/3246000305/usgs_elev_table.csv                                        \ 
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg
+    -flows /outputs/carson_gms_bogus/02020005/branches/3246000305/demDerived_reaches_split_filtered_3246000305.gpkg
+    -cat /outputs/carson_gms_bogus/02020005/branches/3246000305/gw_catchments_reaches_filtered_addedAttributes_3246000305.gpkg
+    -dem /outputs/carson_gms_bogus/02020005/branches/3246000305/dem_meters_3246000305.tif
+    -dem_adj /outputs/carson_gms_bogus/02020005/branches/3246000305/dem_thalwegCond_3246000305.tif
+    -outtable /outputs/carson_gms_bogus/02020005/branches/3246000305/usgs_elev_table.csv
     -b 32460003 05
 
-python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg                    \ 
-    -flows /outputs/carson_gms_bogus/02020005/branches/3246000257/demDerived_reaches_split_filtered_3246000257.gpkg             \ 
-    -cat /outputs/carson_gms_bogus/02020005/branches/3246000257/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg  \ 
-    -dem /outputs/carson_gms_bogus/02020005/branches/3246000257/dem_meters_3246000257.tif                                       \ 
-    -dem_adj /outputs/carson_gms_bogus/02020005/branches/3246000257/dem_thalwegCond_3246000257.tif                              \ 
-    -outtable /outputs/carson_gms_bogus/02020005/branches/3246000257/usgs_elev_table.csv                                        \ 
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gms_bogus/02020005/usgs_subset_gages.gpkg
+    -flows /outputs/carson_gms_bogus/02020005/branches/3246000257/demDerived_reaches_split_filtered_3246000257.gpkg
+    -cat /outputs/carson_gms_bogus/02020005/branches/3246000257/gw_catchments_reaches_filtered_addedAttributes_3246000257.gpkg
+    -dem /outputs/carson_gms_bogus/02020005/branches/3246000257/dem_meters_3246000257.tif
+    -dem_adj /outputs/carson_gms_bogus/02020005/branches/3246000257/dem_thalwegCond_3246000257.tif
+    -outtable /outputs/carson_gms_bogus/02020005/branches/3246000257/usgs_elev_table.csv
     -b 32460002 57
 
-python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gage_test/04130001/usgs_subset_gages.gpkg                    \ 
-    -flows /outputs/carson_gage_test/04130001/branches/9041000030/demDerived_reaches_split_filtered_9041000030.gpkg             \ 
-    -cat /outputs/carson_gage_test/04130001/branches/9041000030/gw_catchments_reaches_filtered_addedAttributes_9041000030.gpkg  \ 
-    -dem /outputs/carson_gage_test/04130001/branches/9041000030/dem_meters_9041000030.tif                                       \ 
-    -dem_adj /outputs/carson_gage_test/04130001/branches/904100030/dem_thalwegCond_0941000030.tif                               \ 
-    -outtable /outputs/carson_gage_test/04130001/branches/9041000030/usgs_elev_table.csv                                        \ 
+python /foss_fim/src/usgs_gage_crosswalk.py -gages /outputs/carson_gage_test/04130001/usgs_subset_gages.gpkg
+    -flows /outputs/carson_gage_test/04130001/branches/9041000030/demDerived_reaches_split_filtered_9041000030.gpkg
+    -cat /outputs/carson_gage_test/04130001/branches/9041000030/gw_catchments_reaches_filtered_addedAttributes_9041000030.gpkg
+    -dem /outputs/carson_gage_test/04130001/branches/9041000030/dem_meters_9041000030.tif
+    -dem_adj /outputs/carson_gage_test/04130001/branches/904100030/dem_thalwegCond_0941000030.tif
+    -outtable /outputs/carson_gage_test/04130001/branches/9041000030/usgs_elev_table.csv
     -b 90410000 30
 """
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 355ffc3a7..3065008b3 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -1,21 +1,24 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
-from posixpath import dirname
 import re
+import warnings
+from posixpath import dirname
+
 import geopandas as gpd
 import pandas as pd
-import argparse
-import warnings
 from shapely.geometry import Point
+
 from utils.shared_functions import mem_profile
 from utils.shared_variables import PREP_CRS
+
+
 warnings.simplefilter("ignore")
 
-class Gage2Branch(object):
 
+class Gage2Branch(object):
     def __init__(self, usgs_gage_filename, ras_locs_filename, ahps_filename, huc8):
-
         self.usgs_gage_filename = usgs_gage_filename
         self.ras_locs_filename = ras_locs_filename
         self.ahps_filename = ahps_filename
@@ -23,23 +26,24 @@ def __init__(self, usgs_gage_filename, ras_locs_filename, ahps_filename, huc8):
         self.load_gages()
 
     def load_gages(self):
-
         # Read USGS gage file a
-        usgs_gages = gpd.read_file(self.usgs_gage_filename,dtype={'location_id': object})
+        usgs_gages = gpd.read_file(self.usgs_gage_filename, dtype={'location_id': object})
         usgs_gages['source'] = 'usgs_gage'
 
         # Read RAS2FIM point locations file
-        # !!! Geopandas is not honoring the dtype arg with this read_file below (huc8 being read as int64). 
+        # !!! Geopandas is not honoring the dtype arg with this read_file below (huc8 being read as int64).
         # Need the raw data to store the 'huc8' attribute as an object to avoid issues with integers truncating the leading zero from some hucs
-        ras_locs = gpd.read_file(self.ras_locs_filename,dtype={'huc8': 'object'}) 
-        ras_locs = ras_locs[['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'wrds_timestamp', 'geometry']]
-        ras_locs['location_id'] =  ras_locs['fid_xs']
-        
+        ras_locs = gpd.read_file(self.ras_locs_filename, dtype={'huc8': 'object'})
+        ras_locs = ras_locs[
+            ['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'wrds_timestamp', 'geometry']
+        ]
+        ras_locs['location_id'] = ras_locs['fid_xs']
+
         # Convert ras locs crs to match usgs gage crs
         ras_locs.to_crs(usgs_gages.crs, inplace=True)
-        ras_locs.rename(columns={'huc8':'HUC8'}, inplace=True)
+        ras_locs.rename(columns={'huc8': 'HUC8'}, inplace=True)
 
-        # Convert Multipoint geometry to Point geometry 
+        # Convert Multipoint geometry to Point geometry
         ras_locs['geometry'] = ras_locs.representative_point()
 
         # if ras_locs.huc8.dtype == 'int64':
@@ -51,34 +55,45 @@ def load_gages(self):
 
         # Concat USGS points and RAS2FIM points
         gages_locs = pd.concat([usgs_gages, ras_locs], axis=0, ignore_index=True)
-        #gages_locs.to_crs(PREP_CRS, inplace=True)
+        # gages_locs.to_crs(PREP_CRS, inplace=True)
 
         # Filter USGS gages and RAS locations to huc
         self.gages = gages_locs[(gages_locs.HUC8 == self.huc8)]
-        
+
         # Get AHPS sites within the HUC and add them to the USGS dataset
         if self.ahps_filename:
             ahps_sites = gpd.read_file(self.ahps_filename)
-            ahps_sites = ahps_sites[ahps_sites.HUC8 == self.huc8] # filter to HUC8
-            ahps_sites.rename(columns={'nwm_feature_id':'feature_id',
-                                'usgs_site_code':'location_id'}, inplace=True)
-            ahps_sites = ahps_sites[ahps_sites.location_id.isna()] # Filter sites that are already in the USGS dataset
+            ahps_sites = ahps_sites[ahps_sites.HUC8 == self.huc8]  # filter to HUC8
+            ahps_sites.rename(
+                columns={'nwm_feature_id': 'feature_id', 'usgs_site_code': 'location_id'}, inplace=True
+            )
+            ahps_sites = ahps_sites[
+                ahps_sites.location_id.isna()
+            ]  # Filter sites that are already in the USGS dataset
             ahps_sites['source'] = 'ahps_site'
-            self.gages = pd.concat([self.gages, ahps_sites[['feature_id', 'nws_lid', 'location_id', 'HUC8', 'name', 'states','geometry']]])
+            self.gages = pd.concat(
+                [
+                    self.gages,
+                    ahps_sites[
+                        ['feature_id', 'nws_lid', 'location_id', 'HUC8', 'name', 'states', 'geometry']
+                    ],
+                ]
+            )
 
         # Create gages attribute
         self.gages.location_id.fillna(self.gages.nws_lid, inplace=True)
         self.gages.loc[self.gages['nws_lid'] == 'Bogus_ID', 'nws_lid'] = None
 
     def sort_into_branch(self, nwm_subset_streams_levelPaths):
-
         nwm_reaches = gpd.read_file(nwm_subset_streams_levelPaths)
-        nwm_reaches.rename(columns={'ID':'feature_id'}, inplace=True)
+        nwm_reaches.rename(columns={'ID': 'feature_id'}, inplace=True)
 
         if not self.gages[self.gages.feature_id.isnull()].empty:
             missing_feature_id = self.gages.loc[self.gages.feature_id.isnull()].copy()
             nwm_reaches_union = nwm_reaches.geometry.unary_union
-            missing_feature_id['feature_id'] = missing_feature_id.apply(lambda row: self.sjoin_nearest_to_nwm(row.geometry, nwm_reaches, nwm_reaches_union), axis=1)
+            missing_feature_id['feature_id'] = missing_feature_id.apply(
+                lambda row: self.sjoin_nearest_to_nwm(row.geometry, nwm_reaches, nwm_reaches_union), axis=1
+            )
 
             self.gages.update(missing_feature_id)
 
@@ -86,22 +101,22 @@ def sort_into_branch(self, nwm_subset_streams_levelPaths):
 
         # Left join gages with NWM streams to get the level path
         self.gages.feature_id = self.gages.feature_id.astype(int)
-        self.gages = self.gages.merge(nwm_reaches[['feature_id','levpa_id','order_']], on='feature_id', how='left')
+        self.gages = self.gages.merge(
+            nwm_reaches[['feature_id', 'levpa_id', 'order_']], on='feature_id', how='left'
+        )
         return self.gages
 
     def branch_zero(self, bzero_id):
-
-        # note that some gages will not have a valid "order_" attribute (not attributed to a level path in the step before - likely a gage on dropped stream order)
+        # note that some gages will not have a valid "order_" attribute
+        #   (not attributed to a level path in the step before - likely a gage on dropped stream order)
         self.gages.levpa_id = bzero_id
         return self.gages
-    
-    def write(self, out_name):
 
-        self.gages.to_file(out_name,driver='GPKG',index=False)
+    def write(self, out_name):
+        self.gages.to_file(out_name, driver='GPKG', index=False)
 
     @staticmethod
     def sjoin_nearest_to_nwm(pnt, lines, union):
-
         snap_geom = union.interpolate(union.project(pnt))
         queried_index = lines.geometry.sindex.query(snap_geom)
         if len(queried_index):
@@ -109,13 +124,12 @@ def sjoin_nearest_to_nwm(pnt, lines, union):
 
     @staticmethod
     def filter_gage_branches(fim_inputs_filename):
-
         fim_dir = os.path.dirname(fim_inputs_filename)
-        fim_inputs = pd.read_csv(fim_inputs_filename, header=None, names=['huc', 'levpa_id'],
-            dtype={'huc':str, 'levpa_id':str})
-
-        for huc_dir in [d for d in os.listdir(fim_dir) if re.search('^\d{8}$', d)]:
+        fim_inputs = pd.read_csv(
+            fim_inputs_filename, header=None, names=['huc', 'levpa_id'], dtype={'huc': str, 'levpa_id': str}
+        )
 
+        for huc_dir in [d for d in os.listdir(fim_dir) if re.search(r'^\d{8}$', d)]:
             gage_file = os.path.join(fim_dir, huc_dir, 'usgs_subset_gages.gpkg')
             if not os.path.isfile(gage_file):
                 fim_inputs.drop(fim_inputs.loc[fim_inputs.huc == huc_dir].index, inplace=True)
@@ -123,22 +137,33 @@ def filter_gage_branches(fim_inputs_filename):
 
             gages = gpd.read_file(gage_file)
             level_paths = gages.levpa_id
-            fim_inputs.drop(fim_inputs.loc[(fim_inputs.huc == huc_dir) & (~fim_inputs.levpa_id.isin(level_paths))].index, inplace=True)
-        
+            fim_inputs.drop(
+                fim_inputs.loc[(fim_inputs.huc == huc_dir) & (~fim_inputs.levpa_id.isin(level_paths))].index,
+                inplace=True,
+            )
+
         fim_inputs.to_csv(fim_inputs_filename, index=False, header=False)
 
 
 if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Assign HUC gages to branch and stage for usgs_gage_crosswalk.py')
-    parser.add_argument('-gages','--usgs-gages-filename', help='USGS gages', required=True)
-    parser.add_argument('-ras','--ras-locs-filename', help='RAS2FIM rating curve locations', required=True)
-    parser.add_argument('-ahps','--nws-lid-filename', help='AHPS gages', required=False)
-    parser.add_argument('-nwm','--input-nwm-filename', help='NWM stream subset', required=True)
-    parser.add_argument('-o','--output-filename', help='Table to append data', required=True)
-    parser.add_argument('-huc','--huc8-id', help='HUC8 ID (to verify gage location huc)', type=str, required=True)
-    parser.add_argument('-bzero_id','--branch-zero-id', help='Branch zero ID value', type=str, required=True)
-    parser.add_argument('-ff','--filter-fim-inputs', help='WARNING: only run this parameter if you know exactly what you are doing', required=False)
+    parser = argparse.ArgumentParser(
+        description='Assign HUC gages to branch and stage for usgs_gage_crosswalk.py'
+    )
+    parser.add_argument('-gages', '--usgs-gages-filename', help='USGS gages', required=True)
+    parser.add_argument('-ras', '--ras-locs-filename', help='RAS2FIM rating curve locations', required=True)
+    parser.add_argument('-ahps', '--nws-lid-filename', help='AHPS gages', required=False)
+    parser.add_argument('-nwm', '--input-nwm-filename', help='NWM stream subset', required=True)
+    parser.add_argument('-o', '--output-filename', help='Table to append data', required=True)
+    parser.add_argument(
+        '-huc', '--huc8-id', help='HUC8 ID (to verify gage location huc)', type=str, required=True
+    )
+    parser.add_argument('-bzero_id', '--branch-zero-id', help='Branch zero ID value', type=str, required=True)
+    parser.add_argument(
+        '-ff',
+        '--filter-fim-inputs',
+        help='WARNING: only run this parameter if you know exactly what you are doing',
+        required=False,
+    )
 
     args = vars(parser.parse_args())
 
@@ -152,7 +177,6 @@ def filter_gage_branches(fim_inputs_filename):
     filter_fim_inputs = args['filter_fim_inputs']
 
     if not filter_fim_inputs:
-
         usgs_gage_subset = Gage2Branch(usgs_gages_filename, ras_locs_filename, nws_lid_filename, huc8)
         if usgs_gage_subset.gages.empty:
             print(f'There are no gages identified for {huc8}')
@@ -161,18 +185,20 @@ def filter_gage_branches(fim_inputs_filename):
         usgs_gage_subset.write(output_filename)
 
         # Create seperate output for branch zero
-        output_filename_zero = os.path.splitext(output_filename)[0] + '_' + bzero_id + os.path.splitext(output_filename)[-1]
+        output_filename_zero = (
+            os.path.splitext(output_filename)[0] + '_' + bzero_id + os.path.splitext(output_filename)[-1]
+        )
         usgs_gage_subset.branch_zero(bzero_id)
         usgs_gage_subset.write(output_filename_zero)
 
     else:
-        ''' 
-        This is an easy way to filter fim_inputs so that only branches with gages will run during fim_process_unit_wb.sh.
+        '''
+        This is an easy way to filter fim_inputs so that only branches with gages will run
+        during fim_process_unit_wb.sh.
 
         example:
-        python3 src/usgs_gage_unit_setup.py -gages x -ahps x -nwm x -o x -huc x -ff /outputs/test_output/fim_inputs.csv
+        python3 src/usgs_gage_unit_setup.py -gages x -ahps x -nwm x -o x -huc x
+            -ff /outputs/test_output/fim_inputs.csv
         '''
         assert os.path.isfile(filter_fim_inputs)
         Gage2Branch.filter_gage_branches(filter_fim_inputs)
-
-    
diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py
index f5692aba6..7524e0277 100644
--- a/src/utils/fim_enums.py
+++ b/src/utils/fim_enums.py
@@ -2,10 +2,11 @@
 
 from enum import Enum
 
+
 class FIM_exit_codes(Enum):
     '''
     This particular enum class allows for special system exit codes to be issued to tell different
-    code layers about what has happened. Currently, most of our code uses just sys.exit(0) and 
+    code layers about what has happened. Currently, most of our code uses just sys.exit(0) and
     sys.exit(1)<which is an error>.
     The list of enums lower (which can grow as needed) allows for more status to be return.
     - Notes about system exit codes:
@@ -20,11 +21,11 @@ class FIM_exit_codes(Enum):
         import utils/fim_enums
         print(FIM_exit_codes.UNIT_NO_BRANCHES.value) -> 60   [this is used in derive_level_paths.py]
 
-    - For more information : https://docs.python.org/3.11/howto/enum.html and https://docs.python.org/3/library/enum.html
+    - For more information : https://docs.python.org/3.11/howto/enum.html &
+                             https://docs.python.org/3/library/enum.html
     '''
 
     UNIT_NO_BRANCHES = 60
     NO_FLOWLINES_EXIST = 61
     EXCESS_UNIT_ERRORS = 62
-    NO_BRANCH_LEVELPATHS_EXIST = 63    
-            
\ No newline at end of file
+    NO_BRANCH_LEVELPATHS_EXIST = 63
diff --git a/src/utils/reproject_dem.py b/src/utils/reproject_dem.py
deleted file mode 100755
index 056602b53..000000000
--- a/src/utils/reproject_dem.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-from osgeo import gdal
-import sys
-sys.path.append('/foss_fim/src')
-from utils.shared_variables import PREP_PROJECTION_CM
-import shutil
-from multiprocessing import Pool
-import argparse
-
-
-def reproject_dem(args):
-
-    raster_dir            = args[0]
-    elev_cm               = args[1]
-    elev_cm_proj          = args[2]
-    reprojection          = args[3]
-    overwrite             = args[4]
-
-    if os.path.exists(elev_cm_proj) & overwrite:
-        os.remove(elev_cm_proj)
-    elif not os.path.exists(elev_cm_proj):
-        pass
-    else:
-        print(f"Skipping {elev_cm_proj}. Raster already exists. Use overwrite option.")
-        return
-
-    shutil.copy(elev_cm, elev_cm_proj)
-
-    print(f"Reprojecting {elev_cm_proj}")
-    gdal.Warp(elev_cm_proj,elev_cm_proj,dstSRS=reprojection)
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Burn in NLD elevations')
-    parser.add_argument('-dem_dir','--dem-dir', help='DEM filename', required=True,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
-
-    args = vars(parser.parse_args())
-
-    dem_dir            = args['dem_dir']
-    number_of_jobs     = args['number_of_jobs']
-
-    reproject_procs_list = []
-
-    for huc in os.listdir(dem_dir):
-        raster_dir = os.path.join(dem_dir,huc)
-        elev_cm = os.path.join(raster_dir, 'elev_cm.tif')
-        elev_cm_proj = os.path.join(raster_dir, 'elev_cm_proj.tif')
-        reproject_procs_list.append([raster_dir, elev_cm, elev_cm_proj, PREP_PROJECTION_CM])
-
-    # Multiprocess reprojection
-    with Pool(processes=number_of_jobs) as pool:
-        pool.map(reproject_dem, reproject_procs_list)
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 33f8e1eb2..7f0eacc37 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -1,34 +1,34 @@
 #!/usr/bin/env python3
 
-import os
 import glob
 import inspect
+import os
 import re
 import sys
-
 from concurrent.futures import as_completed
 from datetime import datetime, timezone
-from pathlib import Path
 from os.path import splitext
+from pathlib import Path
 
 import fiona
 import geopandas as gp
 import numpy as np
 import pandas as pd
 import rasterio
-import utils.shared_variables as sv
-
+from memory_profiler import profile
 from pyproj.crs import CRS
-from rasterio.warp import calculate_default_transform, reproject, Resampling
+from rasterio.warp import Resampling, calculate_default_transform, reproject
 from tqdm import tqdm
 
+import utils.shared_variables as sv
 
-def getDriver(fileName):
 
-    driverDictionary = {'.gpkg' : 'GPKG','.geojson' : 'GeoJSON','.shp' : 'ESRI Shapefile'}
+def getDriver(fileName):
+    driverDictionary = {'.gpkg': 'GPKG', '.geojson': 'GeoJSON', '.shp': 'ESRI Shapefile'}
     driver = driverDictionary[splitext(fileName)[1]]
 
-    return(driver)
+    return driver
+
 
 def pull_file(url, full_pulled_filepath):
     """
@@ -75,8 +75,6 @@ def run_system_command(args):
 
 
 def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
-
-
     print("Subsetting " + wbd_gpkg + "...")
     # Read geopackage into dataframe.
     wbd = gp.read_file(wbd_gpkg)
@@ -84,14 +82,18 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
 
     for index, row in gdf.iterrows():
         state = row["STATES"]
-        if state != None:  # Some polygons are empty in the STATES field.
+        if state is not None:  # Some polygons are empty in the STATES field.
             keep_flag = False  # Default to Fault, i.e. to delete the polygon.
             if state in sv.CONUS_STATE_LIST:
                 keep_flag = True
             # Only split if multiple states present. More efficient this way.
             elif len(state) > 2:
-                for wbd_state in state.split(","):  # Some polygons have multiple states, separated by a comma.
-                    if wbd_state in sv.CONUS_STATE_LIST:  # Check each polygon to make sure it's state abbrev name is allowed.
+                for wbd_state in state.split(
+                    ","
+                ):  # Some polygons have multiple states, separated by a comma.
+                    if (
+                        wbd_state in sv.CONUS_STATE_LIST
+                    ):  # Check each polygon to make sure it's state abbrev name is allowed.
                         keep_flag = True
                         break
             if not keep_flag:
@@ -100,29 +102,30 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
     # Overwrite geopackage.
     layer_name = os.path.split(wbd_gpkg)[1].strip('.gpkg')
     gdf.crs = sv.PREP_PROJECTION
-    gdf.to_file(multilayer_wbd_geopackage, layer=layer_name,driver='GPKG',index=False)
+    gdf.to_file(multilayer_wbd_geopackage, layer=layer_name, driver='GPKG', index=False)
 
 
-def get_fossid_from_huc8(huc8_id,foss_id_attribute='fossid',
-                         hucs=os.path.join(os.environ['inputsDir'],'wbd','WBD_National.gpkg'),
-                         hucs_layerName=None):
-
-    hucs = fiona.open(hucs,'r',layer=hucs_layerName)
+def get_fossid_from_huc8(
+    huc8_id,
+    foss_id_attribute='fossid',
+    hucs=os.path.join(os.environ['inputsDir'], 'wbd', 'WBD_National.gpkg'),
+    hucs_layerName=None,
+):
+    hucs = fiona.open(hucs, 'r', layer=hucs_layerName)
 
     for huc in hucs:
         if huc['properties']['HUC8'] == huc8_id:
-            return(huc['properties'][foss_id_attribute])
+            return huc['properties'][foss_id_attribute]
 
 
 def update_raster_profile(args):
-
-    elev_cm_filename   = args[0]
-    elev_m_filename    = args[1]
-    projection         = args[2]
-    nodata_val         = args[3]
-    blocksize          = args[4]
-    keep_intermediate  = args[5]
-    overwrite  = args[6]
+    elev_cm_filename = args[0]
+    elev_m_filename = args[1]
+    projection = args[2]
+    nodata_val = args[3]
+    blocksize = args[4]
+    keep_intermediate = args[5]
+    overwrite = args[6]
 
     if os.path.exists(elev_m_filename) & overwrite:
         os.remove(elev_m_filename)
@@ -134,9 +137,9 @@ def update_raster_profile(args):
 
     if isinstance(blocksize, int):
         pass
-    elif isinstance(blocksize,str):
+    elif isinstance(blocksize, str):
         blocksize = int(blocksize)
-    elif isinstance(blocksize,float):
+    elif isinstance(blocksize, float):
         blocksize = int(blocksize)
     else:
         raise TypeError("Pass integer for blocksize")
@@ -145,131 +148,59 @@ def update_raster_profile(args):
 
     # Update nodata value and convert from cm to meters
     dem_cm = rasterio.open(elev_cm_filename)
-    
+
     no_data = dem_cm.nodata
-    
+
     dem_m_profile = dem_cm.profile.copy()
-    dem_m_profile.update(driver='GTiff',tiled=True,nodata=nodata_val,
-                         blockxsize=blocksize, blockysize=blocksize,
-                         dtype='float32',crs=projection,compress='lzw',interleave='band')
+    dem_m_profile.update(
+        driver='GTiff',
+        tiled=True,
+        nodata=nodata_val,
+        blockxsize=blocksize,
+        blockysize=blocksize,
+        dtype='float32',
+        crs=projection,
+        compress='lzw',
+        interleave='band',
+    )
 
     dest = rasterio.open(elev_m_filename, "w", **dem_m_profile, BIGTIFF='YES')
 
-    for idx,window in dem_cm.block_windows(1):
-        data = dem_cm.read(1,window=window)
+    for idx, window in dem_cm.block_windows(1):
+        data = dem_cm.read(1, window=window)
 
         # wrote out output of this line as the same variable.
-        data = np.where(data == int(no_data), nodata_val, (data/100).astype(rasterio.float32))
+        data = np.where(data == int(no_data), nodata_val, (data / 100).astype(rasterio.float32))
 
-    # removed this line to avoid having two array copies of data. Kills memory usage
-    #del data
-
-        dest.write(data, indexes = 1, window=window)
+        # removed this line to avoid having two array copies of data. Kills memory usage
+        # del data
 
+        dest.write(data, indexes=1, window=window)
 
     # not necessary
-    #del dem_m
+    # del dem_m
 
     dem_cm.close()
     dest.close()
-    
-    if keep_intermediate == False:
-        os.remove(elev_cm_filename)
-    
-    return(elev_m_filename)
 
-'''
-This function isn't currently used but is the preferred method for
-reprojecting elevation grids.
-
-Several USGS elev_cm.tifs have the crs value in their profile stored as the string "CRS.from_epsg(26904)"
-instead of the actual output of that command.
-
-Rasterio fails to properly read the crs but using gdal retrieves the correct projection.
-Until this issue is resolved use the reproject_dem function in reproject_dem.py instead.
-reproject_dem is not stored in the shared_functions.py because rasterio and
-gdal bindings are not entirely compatible: https://rasterio.readthedocs.io/en/latest/topics/switch.html
-
-'''
-
-def reproject_raster(input_raster_name,
-                     reprojection, 
-                     blocksize=None,
-                     reprojected_raster_name=None):
-
-    # TODO: Might be removed (Nov 8, 2022), not in use
-
-    if blocksize is not None:
-        if isinstance(blocksize, int):
-            pass
-        elif isinstance(blocksize,str):
-            blocksize = int(blocksize)
-        elif isinstance(blocksize,float):
-            blocksize = int(blocksize)
-        else:
-            raise TypeError("Pass integer for blocksize")
-    else:
-        blocksize = 256
-
-    assert input_raster_name.endswith('.tif'), "input raster needs to be a tif"
-
-    reprojection = rasterio.crs.CRS.from_string(reprojection)
-
-    with rasterio.open(input_raster_name) as src:
+    if keep_intermediate is False:
+        os.remove(elev_cm_filename)
 
-        # Check projection
-        if src.crs.to_string() != reprojection:
-            if src.crs.to_string().startswith('EPSG'):
-                epsg = src.crs.to_epsg()
-                proj_crs = CRS.from_epsg(epsg)
-                rio_crs = rasterio.crs.CRS.from_user_input(proj_crs).to_string()
-            else:
-                rio_crs = src.crs.to_string()
-
-            print(f"{input_raster_name} not projected")
-            print(f"Reprojecting from {rio_crs} to {reprojection}")
-
-            transform, width, height = calculate_default_transform(
-                src.crs, reprojection, src.width, src.height, *src.bounds)
-            kwargs = src.meta.copy()
-            kwargs.update({
-                'crs': reprojection,
-                'transform': transform,
-                'width': width,
-                'height': height,
-                'compress': 'lzw'
-            })
-
-            if reprojected_raster_name is None:
-                reprojected_raster_name = input_raster_name
-
-            assert reprojected_raster_name.endswith('.tif'), "output raster needs to be a tif"
-
-            with rasterio.open(reprojected_raster_name, 'w', **kwargs, tiled=True, blockxsize=blocksize, blockysize=blocksize, BIGTIFF='YES') as dst:
-                reproject(
-                    source=rasterio.band(src, 1),
-                    destination=rasterio.band(dst, 1),
-                    src_transform=src.transform,
-                    src_crs=rio_crs,
-                    dst_transform=transform,
-                    dst_crs=reprojection.to_string(),
-                    resampling=Resampling.nearest)
-                del dst
-        del src
+    return elev_m_filename
 
 
 def mem_profile(func):
     def wrapper(*args, **kwargs):
-        if (os.environ.get('mem') == "1"):
+        if os.environ.get('mem') == "1":
             profile(func)(*args, **kwargs)
         else:
             func(*args, **kwargs)
-    return wrapper
 
+    return wrapper
 
 
 ########################################################################
-#Function to check the age of a file (use for flagging potentially outdated input)
+# Function to check the age of a file (use for flagging potentially outdated input)
 ########################################################################
 def check_file_age(file):
     '''
@@ -286,9 +217,9 @@ def check_file_age(file):
 
 
 ########################################################################
-#Function to concatenate huc csv files to a single dataframe/csv
+# Function to concatenate huc csv files to a single dataframe/csv
 ########################################################################
-def concat_huc_csv(fim_dir,csv_name):
+def concat_huc_csv(fim_dir, csv_name):
     '''
     Checks if huc csv file exist, concatenates contents of csv
     Returns
@@ -297,42 +228,40 @@ def concat_huc_csv(fim_dir,csv_name):
     '''
 
     merged_csv = []
-    huc_list  = os.listdir(fim_dir)
+    huc_list = os.listdir(fim_dir)
     for huc in huc_list:
         if huc != 'logs':
-            csv_file = os.path.join(fim_dir,huc,str(csv_name))
+            csv_file = os.path.join(fim_dir, huc, str(csv_name))
             if Path(csv_file).is_file():
-                # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
-                read_csv = pd.read_csv(csv_file, dtype={'HUC8': object, 'location_id': object, 'feature_id': int, 'levpa_id': object})
+                # Aggregate all of the individual huc elev_tables into one for accessing all data in one csv
+                read_csv = pd.read_csv(
+                    csv_file,
+                    dtype={'HUC8': object, 'location_id': object, 'feature_id': int, 'levpa_id': object},
+                )
                 # Add huc field to dataframe
-                read_csv['HUC8'] = huc 
+                read_csv['HUC8'] = huc
                 merged_csv.append(read_csv)
 
     # Create and return a concatenated pd dataframe
     if merged_csv:
-        print(f"Creating aggregate csv")
+        print("Creating aggregate csv")
         concat_df = pd.concat(merged_csv)
         return concat_df
 
 
 # -----------------------------------------------------------
 def progress_bar_handler(executor_dict, desc):
-
-    for future in tqdm(as_completed(executor_dict),
-                    total=len(executor_dict),
-                    desc=desc
-                    ):
+    for future in tqdm(as_completed(executor_dict), total=len(executor_dict), desc=desc):
         try:
             future.result()
         except Exception as exc:
-            print('{}, {}, {}'.format(executor_dict[future],exc.__class__.__name__,exc))
+            print('{}, {}, {}'.format(executor_dict[future], exc.__class__.__name__, exc))
 
 
 # #####################################
 class FIM_Helpers:
-
     # -----------------------------------------------------------
-    @staticmethod    
+    @staticmethod
     def append_id_to_file_name(file_name, identifier):
         '''
         Processing:
@@ -341,35 +270,33 @@ def append_id_to_file_name(file_name, identifier):
             ie) filename = "/output/myfolder/a_raster.tif"
                 indentifer = "13090001"
                 Becomes: "/output/myfolder/a_raster_13090001.tif"
-            Note: 
+            Note:
                 - Can handle a single identifier or a list of identifier
                 ie) identifier = ["13090001", "123000001"]
                 Becomes: "/output/myfolder/a_raster_13090001_123000001.tif"
                 - This allows for file name to not be submitted and will return None
         -------
-        
+
         Inputs:
             file_name: a single file name
-            identifier: a value or list of values to be inserted with an underscore 
+            identifier: a value or list of values to be inserted with an underscore
                 added ahead of the extention
-        
+
         -------
         Output:
             out_file_name: A single name with each identifer added at the end before
                 the extension, each with an underscore in front of the identifier.
 
-        -------                
+        -------
         Usage:
             from utils.shared_functions import FIM_Helpers as fh
             composite_file_output = fh.append_id_to_file_name(composite_file_output, huc)
         '''
 
         if file_name is not None:
-
-            root,extension = os.path.splitext(file_name)
+            root, extension = os.path.splitext(file_name)
 
             if isinstance(identifier, list):
-
                 out_file_name = root
                 for i in identifier:
                     out_file_name += "_{}".format(i)
@@ -379,33 +306,33 @@ def append_id_to_file_name(file_name, identifier):
         else:
             out_file_name = None
 
-        return(out_file_name)
+        return out_file_name
 
     # -----------------------------------------------------------
     @staticmethod
-    def vprint (message, is_verbose, show_caller = False):
+    def vprint(message, is_verbose, show_caller=False):
         '''
-        Processing: Will print a standard output message only when the 
+        Processing: Will print a standard output message only when the
             verbose flag is set to True
         -------
-        
+
         Parameters:
             message : str
                 The message for output
                 Note: this method puts a '...' in front of the message
             is_verbose : bool
-                This exists so the call to vprint always exists and does not 
+                This exists so the call to vprint always exists and does not
                 need a "if verbose: test for inline code
                 If this value is False, this method will simply return
             show_caller : bool
                 Sometimes, it is desired to see the calling function, method or class
-        
+
         -------
         Returns:
             str : the message starting with "... " and optionallly ending with
                 the calling function, method or class name
-        
-        ------- 
+
+        -------
         Usage:
             from utils.shared_functions import FIM_Helpers as fh
             fh.vprint(f"Starting alpha test for {self.dir}", verbose)
@@ -414,13 +341,12 @@ def vprint (message, is_verbose, show_caller = False):
             return
 
         msg = f"... {message}"
-        if (show_caller):
+        if show_caller:
             caller_name = inspect.stack()[1][3]
-            if (caller_name == "<module"):
+            if caller_name == "<module":
                 caller_name = inspect.stack()[1][1]
             msg += f"  [from : {caller_name}]"
-        print (msg)
-
+        print(msg)
 
     # -----------------------------------------------------------
     @staticmethod
@@ -429,43 +355,43 @@ def load_list_file(file_name_and_path):
         Process:
         -------
         Attempts to load a .txt or .lst file of line delimited values into a python list
-        
+
         Parameters:
         -------
         file_name_and_path : str
             path and file name of data to be loaded.
-        
+
         Returns:
         -------
         a Python list
-        
+
         -------
         Usage:
             from utils.shared_functions import FIM_Helpers as fh
             fh.vprint(f"Starting alpha test for {self.dir}", verbose)
-        
+
         '''
-        
-        if (not os.path.isfile(file_name_and_path)):
+
+        if not os.path.isfile(file_name_and_path):
             raise ValueError(f"Sorry, file {file_name_and_path} does not exist. Check name and path.")
-        
+
         line_values = []
-        
+
         with open(file_name_and_path, "r") as data_file:
             data = data_file.read()
-            # replacing end splitting the text 
+            # replacing end splitting the text
             # when newline ('\n') is seen.
             line_values_raw = data.split("\n")
-            line_values_stripped = [i.strip() for i in line_values_raw] # removes extra spaces
-            
+            line_values_stripped = [i.strip() for i in line_values_raw]  # removes extra spaces
+
             # dending on comments in the file or an extra line break at the end, we might
             # get empty entries in the line_values collection. We remove them here
             line_values = [ele for ele in line_values_stripped if ele != ""]
-            
-        if (len(line_values) == 0):
+
+        if len(line_values) == 0:
             raise Exception("Sorry, there are no value were in the list")
-            
-        return line_values       
+
+        return line_values
 
     # -----------------------------------------------------------
     @staticmethod
@@ -474,7 +400,7 @@ def get_file_names(src_folder, file_extension):
         Process
         ----------
         Get a list of file names and paths matching the file extension
-        
+
         Parameters
         ----------
             - src_folder (str)
@@ -483,35 +409,37 @@ def get_file_names(src_folder, file_extension):
             - file_extension (str)
                 All files matching this file_extension will be added to the list.
 
-        
+
         Returns
         ----------
         A list of file names and paths
         '''
 
         if (not file_extension) and (len(file_extension.strip()) == 0):
-            raise ValueError(f"file_extension value not set")
-        
+            raise ValueError("file_extension value not set")
+
         # remove the starting . if it exists
-        if (file_extension.startswith(".")):
+        if file_extension.startswith("."):
             file_extension = file_extension[1:]
-        
+
         # test that folder exists
-        if (not os.path.exists(src_folder)):
+        if not os.path.exists(src_folder):
             raise ValueError(f"{file_extension} src folder of {src_folder} not found")
-        
-        if (not src_folder.endswith("/")):
+
+        if not src_folder.endswith("/"):
             src_folder += "/"
-        
+
         glob_pattern = f"{src_folder}*.{file_extension}"
         file_list = glob.glob(glob_pattern)
-    
-        if (len(file_list) == 0):
-            raise Exception(f"files with the extension of {file_extension} "\
-                            f" in the {src_folder} did not load or do not exist")
-        
+
+        if len(file_list) == 0:
+            raise Exception(
+                f"files with the extension of {file_extension} "
+                f" in the {src_folder} did not load or do not exist"
+            )
+
         file_list.sort()
-        
+
         return file_list
 
     # -----------------------------------------------------------
@@ -521,9 +449,9 @@ def print_current_date_time():
         Process:
         -------
         prints the following:
-        
+
             Current date and time: 2022-08-19 15:22:49
-        
+
         -------
         Usage:
             from utils.shared_functions import FIM_Helpers as fh
@@ -532,13 +460,13 @@ def print_current_date_time():
         -------
         Returns:
             Current date / time as a formatted string
-        
+
         '''
         d1 = datetime.now()
         dt_stamp = "Current date and time : "
         dt_stamp += d1.strftime("%Y-%m-%d %H:%M:%S")
-        print (dt_stamp)
-        
+        print(dt_stamp)
+
         return dt_stamp
 
     # -----------------------------------------------------------
@@ -549,37 +477,36 @@ def print_date_time_duration(start_dt, end_dt):
         -------
         Calcuates the diffenence in time between the start and end time
         and prints is as:
-        
+
             Duration: 4 hours 23 mins 15 secs
-        
+
         -------
         Usage:
             from utils.shared_functions import FIM_Helpers as fh
             fh.print_current_date_time()
-        
+
         -------
         Returns:
             Duration as a formatted string
-            
+
         '''
-        time_delta = (end_dt - start_dt)
+        time_delta = end_dt - start_dt
         total_seconds = int(time_delta.total_seconds())
 
-        total_days, rem_seconds = divmod(total_seconds, 60 * 60 * 24)        
+        total_days, rem_seconds = divmod(total_seconds, 60 * 60 * 24)
         total_hours, rem_seconds = divmod(rem_seconds, 60 * 60)
         total_mins, seconds = divmod(rem_seconds, 60)
 
         time_fmt = f"{total_hours:02d} hours {total_mins:02d} mins {seconds:02d} secs"
-        
+
         duration_msg = "Duration: " + time_fmt
         print(duration_msg)
-        
+
         return duration_msg
 
     # -----------------------------------------------------------
     @staticmethod
     def print_start_header(friendly_program_name, start_time):
-        
         print("================================")
         dt_string = start_time.strftime("%m/%d/%Y %H:%M:%S")
         print(f"Start {friendly_program_name} : {dt_string}")
@@ -588,8 +515,7 @@ def print_start_header(friendly_program_name, start_time):
     # -----------------------------------------------------------
     @staticmethod
     def print_end_header(friendly_program_name, start_time, end_time):
-        
         print("================================")
         dt_string = end_time.strftime("%m/%d/%Y %H:%M:%S")
-        print (f"End {friendly_program_name} : {dt_string}")
+        print(f"End {friendly_program_name} : {dt_string}")
         print()
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index bdf19cec9..c9e364730 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -1,19 +1,53 @@
 import os
-from pyproj import CRS
 
 from dotenv import load_dotenv
+from pyproj import CRS
+
+
 load_dotenv('/foss_fim/src/bash_variables.env')
 
 # -- Projections: EPSG 5070-- #
-PREP_PROJECTION = 'PROJCRS["NAD83 / Conus Albers",BASEGEOGCRS["NAD83",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4269]],CONVERSION["Conus Albers",METHOD["Albers Equal Area",ID["EPSG",9822]],PARAMETER["Latitude of false origin",23,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8821]],PARAMETER["Longitude of false origin",-96,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8822]],PARAMETER["Latitude of 1st standard parallel",29.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8823]],PARAMETER["Latitude of 2nd standard parallel",45.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8824]],PARAMETER["Easting at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8826]],PARAMETER["Northing at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8827]]],CS[Cartesian,2],AXIS["easting (X)",east,ORDER[1],LENGTHUNIT["metre",1]],AXIS["northing (Y)",north,ORDER[2],LENGTHUNIT["metre",1]],USAGE[SCOPE["Data analysis and small scale data presentation for contiguous lower 48 states."],AREA["United States (USA) - CONUS onshore - Alabama; Arizona; Arkansas; California; Colorado; Connecticut; Delaware; Florida; Georgia; Idaho; Illinois; Indiana; Iowa; Kansas; Kentucky; Louisiana; Maine; Maryland; Massachusetts; Michigan; Minnesota; Mississippi; Missouri; Montana; Nebraska; Nevada; New Hampshire; New Jersey; New Mexico; New York; North Carolina; North Dakota; Ohio; Oklahoma; Oregon; Pennsylvania; Rhode Island; South Carolina; South Dakota; Tennessee; Texas; Utah; Vermont; Virginia; Washington; West Virginia; Wisconsin; Wyoming."],BBOX[24.41,-124.79,49.38,-66.91]],ID["EPSG",5070]]' 
+PREP_PROJECTION = (
+    'PROJCRS["NAD83 / Conus Albers",BASEGEOGCRS["NAD83",DATUM["North American Datum 1983",'
+    'ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]]],PRIMEM["Greenwich",'
+    '0,ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4269]],CONVERSION["Conus Albers",'
+    'METHOD["Albers Equal Area",ID["EPSG",9822]],PARAMETER["Latitude of false origin",23,'
+    'ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8821]],PARAMETER["Longitude of false '
+    'origin",-96,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8822]],PARAMETER["Latitude '
+    'of 1st standard parallel",29.5,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8823]],'
+    'PARAMETER["Latitude of 2nd standard parallel",45.5,ANGLEUNIT["degree",0.0174532925199433]'
+    ',ID["EPSG",8824]],PARAMETER["Easting at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",'
+    '8826]],PARAMETER["Northing at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8827]]],'
+    'CS[Cartesian,2],AXIS["easting (X)",east,ORDER[1],LENGTHUNIT["metre",1]],'
+    'AXIS["northing (Y)",north,ORDER[2],LENGTHUNIT["metre",1]],USAGE[SCOPE["Data analysis and '
+    'small scale data presentation for contiguous lower 48 states."],AREA["United States (USA) '
+    '- CONUS onshore - Alabama; Arizona; Arkansas; California; Colorado; Connecticut; Delaware'
+    '; Florida; Georgia; Idaho; Illinois; Indiana; Iowa; Kansas; Kentucky; Louisiana; Maine; '
+    'Maryland; Massachusetts; Michigan; Minnesota; Mississippi; Missouri; Montana; Nebraska; '
+    'Nevada; New Hampshire; New Jersey; New Mexico; New York; North Carolina; North Dakota; '
+    'Ohio; Oklahoma; Oregon; Pennsylvania; Rhode Island; South Carolina; South Dakota; '
+    'Tennessee; Texas; Utah; Vermont; Virginia; Washington; West Virginia; Wisconsin; '
+    'Wyoming."],BBOX[24.41,-124.79,49.38,-66.91]],ID["EPSG",5070]]'
+)
 DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
 PREP_CRS = CRS(PREP_PROJECTION)
-VIZ_PROJECTION ='PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
+VIZ_PROJECTION = (
+    'PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",'
+    'SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",'
+    '0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],'
+    'PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],'
+    'PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
+)
 
 # -- Data URLs-- #
 NHD_URL_PARENT = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/'
-NWM_HYDROFABRIC_URL = r'http://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_channel_hydrofabric.tar.gz'  # Temporary
-WBD_NATIONAL_URL = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip'
+# Temporary
+NWM_HYDROFABRIC_URL = (
+    r'http://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_channel_hydrofabric.tar.gz'
+)
+WBD_NATIONAL_URL = (
+    r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip'
+)
 
 
 # -- Prefixes and Suffixes -- #
@@ -21,15 +55,20 @@
 NHD_RASTER_URL_SUFFIX = '_HU4_RASTER.7z'
 NHD_VECTOR_URL_SUFFIX = '_HU4_GDB.zip'
 
-    #NHD_RASTER_EXTRACTION_PREFIX = 'HRNHDPlusRasters'
-    #NHD_RASTER_EXTRACTION_SUFFIX = 'elev_cm.tif'
+# NHD_RASTER_EXTRACTION_PREFIX = 'HRNHDPlusRasters'
+# NHD_RASTER_EXTRACTION_SUFFIX = 'elev_cm.tif'
 
 NHD_VECTOR_EXTRACTION_PREFIX = 'NHDPLUS_H_'
 NHD_VECTOR_EXTRACTION_SUFFIX = '_HU4_GDB.zip'
 
 ## added here bc prefixs and suffixes just aren't necessary in python3
-nhd_raster_url_template = "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_{}_HU4_RASTER.7z"
-nhd_vector_url_template = "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_{}_HU4_GDB.zip"
+nhd_raster_url_template = (
+    "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/"
+    "NHDPLUS_H_{}_HU4_RASTER.7z"
+)
+nhd_vector_url_template = (
+    "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_{}_HU4_GDB.zip"
+)
 
 # --- Values ---- #
 elev_raster_ndv = -9999.0
@@ -41,17 +80,65 @@
 FIM_ID = 'fimid'
 
 # -- Other -- #
-CONUS_STATE_LIST = {"AL", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA",
-                    "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
-                    "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
-                    "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "PR", "RI", "SC",
-                    "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY"}
+CONUS_STATE_LIST = {
+    "AL",
+    "AZ",
+    "AR",
+    "CA",
+    "CO",
+    "CT",
+    "DC",
+    "DE",
+    "FL",
+    "GA",
+    "HI",
+    "ID",
+    "IL",
+    "IN",
+    "IA",
+    "KS",
+    "KY",
+    "LA",
+    "ME",
+    "MD",
+    "MA",
+    "MI",
+    "MN",
+    "MS",
+    "MO",
+    "MT",
+    "NE",
+    "NV",
+    "NH",
+    "NJ",
+    "NM",
+    "NY",
+    "NC",
+    "ND",
+    "OH",
+    "OK",
+    "OR",
+    "PA",
+    "PR",
+    "RI",
+    "SC",
+    "SD",
+    "TN",
+    "TX",
+    "UT",
+    "VT",
+    "VA",
+    "WA",
+    "WV",
+    "WI",
+    "WY",
+}
 
 OVERWRITE_WBD = 'OVERWRITE_WBD'
 OVERWRITE_NHD = 'OVERWRITE_NHD'
 OVERWRITE_ALL = 'OVERWRITE_ALL'
 
 # Rating Curve Adjustment (local calibration) variables
-DOWNSTREAM_THRESHOLD = 10 # distance in km to propogate new roughness values downstream
-ROUGHNESS_MAX_THRESH = 0.8 # max allowable adjusted roughness value (void values larger than this)
-ROUGHNESS_MIN_THRESH = 0.001 # min allowable adjusted roughness value (void values smaller than this)
+DOWNSTREAM_THRESHOLD = 10  # distance in km to propogate new roughness values downstream
+ROUGHNESS_MAX_THRESH = 0.8  # max allowable adjusted roughness value (void values larger than this)
+ROUGHNESS_MIN_THRESH = 0.001  # min allowable adjusted roughness value (void values smaller than this)
diff --git a/tools/adjust_rc_with_feedback.py b/tools/adjust_rc_with_feedback.py
index 35a924572..429c6da97 100644
--- a/tools/adjust_rc_with_feedback.py
+++ b/tools/adjust_rc_with_feedback.py
@@ -1,66 +1,112 @@
+#!/usr/bin/env python3
+
 import argparse
-import geopandas as gpd
-from geopandas.tools import sjoin
+import json
 import os
-import rasterio
-import pandas as pd
-import numpy as np
 import sys
-import json
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
+from geopandas.tools import sjoin
+
 
 temp_workspace = r''
 HAND_CRS = 'EPSG:3857'
 
+
 def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_file, huc6):
     print("Processing huc --> " + str(huc6))
     log_file.write("\nProcessing huc --> " + str(huc6) + '\n')
-    df_gmed = pd.read_csv(output_csv) # read csv to import as a dataframe
-    df_gmed = df_gmed[df_gmed.hydroid != 0] # remove entries that do not have a valid hydroid
+    df_gmed = pd.read_csv(output_csv)  # read csv to import as a dataframe
+    df_gmed = df_gmed[df_gmed.hydroid != 0]  # remove entries that do not have a valid hydroid
 
-    # Read in the hydroTable.csv and check wether it has previously been updated (rename orig columns if needed)
+    # Read in the hydroTable.csv and check if it has been updated (rename orig columns if needed)
     df_htable = pd.read_csv(htable_path)
     if 'orig_discharge_cms' in df_htable.columns:
-        df_htable = df_htable[['HydroID','feature_id','stage','orig_discharge_cms','HydraulicRadius (m)','WetArea (m2)','SLOPE','default_ManningN','HUC','LakeID']]
-        df_htable.rename(columns={'orig_discharge_cms':'discharge_cms','default_ManningN':'ManningN'}, inplace=True)
+        df_htable = df_htable[
+            [
+                'HydroID',
+                'feature_id',
+                'stage',
+                'orig_discharge_cms',
+                'HydraulicRadius (m)',
+                'WetArea (m2)',
+                'SLOPE',
+                'default_ManningN',
+                'HUC',
+                'LakeID',
+            ]
+        ]
+        df_htable.rename(
+            columns={'orig_discharge_cms': 'discharge_cms', 'default_ManningN': 'ManningN'}, inplace=True
+        )
     else:
-        df_htable = df_htable[['HydroID','feature_id','stage','discharge_cms','HydraulicRadius (m)','WetArea (m2)','SLOPE','ManningN','HUC','LakeID']]
+        df_htable = df_htable[
+            [
+                'HydroID',
+                'feature_id',
+                'stage',
+                'discharge_cms',
+                'HydraulicRadius (m)',
+                'WetArea (m2)',
+                'SLOPE',
+                'ManningN',
+                'HUC',
+                'LakeID',
+            ]
+        ]
 
     # loop through the user provided point data --> stage/flow dataframe row by row
     for index, row in df_gmed.iterrows():
-        df_htable_hydroid = df_htable[df_htable.HydroID == row.hydroid] # filter htable for entries with matching hydroid
-        find_src_stage = df_htable_hydroid.loc[df_htable_hydroid['stage'].sub(row.hand).abs().idxmin()] # find closest matching stage to the user provided HAND value
+        df_htable_hydroid = df_htable[
+            df_htable.HydroID == row.hydroid
+        ]  # filter htable for entries with matching hydroid
+        find_src_stage = df_htable_hydroid.loc[
+            df_htable_hydroid['stage'].sub(row.hand).abs().idxmin()
+        ]  # find closest matching stage to the user provided HAND value
         # copy the corresponding htable values for the matching stage->HAND lookup
-        df_gmed.loc[index,'src_stage'] = find_src_stage.stage
-        df_gmed.loc[index,'ManningN'] = find_src_stage.ManningN
-        df_gmed.loc[index,'SLOPE'] = find_src_stage.SLOPE
-        df_gmed.loc[index,'HydraulicRadius_m'] = find_src_stage['HydraulicRadius (m)']
-        df_gmed.loc[index,'WetArea_m2'] = find_src_stage['WetArea (m2)']
-        df_gmed.loc[index,'discharge_cms'] = find_src_stage.discharge_cms
+        df_gmed.loc[index, 'src_stage'] = find_src_stage.stage
+        df_gmed.loc[index, 'ManningN'] = find_src_stage.ManningN
+        df_gmed.loc[index, 'SLOPE'] = find_src_stage.SLOPE
+        df_gmed.loc[index, 'HydraulicRadius_m'] = find_src_stage['HydraulicRadius (m)']
+        df_gmed.loc[index, 'WetArea_m2'] = find_src_stage['WetArea (m2)']
+        df_gmed.loc[index, 'discharge_cms'] = find_src_stage.discharge_cms
 
     ## Create a df of hydroids and featureids
     df_hydro_feat = df_htable.groupby(["HydroID"])[["feature_id"]].median()
-    #print(df_hydro_feat.to_string())
+    # print(df_hydro_feat.to_string())
 
     ## Calculate roughness using Manning's equation
-    df_gmed.rename(columns={'ManningN':'ManningN_default','hydroid':'HydroID'}, inplace=True) # rename the previous ManningN column
-    df_gmed['hydroid_ManningN'] = df_gmed['WetArea_m2']* \
-    pow(df_gmed['HydraulicRadius_m'],2.0/3)* \
-    pow(df_gmed['SLOPE'],0.5)/df_gmed['flow']
+    df_gmed.rename(
+        columns={'ManningN': 'ManningN_default', 'hydroid': 'HydroID'}, inplace=True
+    )  # rename the previous ManningN column
+    df_gmed['hydroid_ManningN'] = (
+        df_gmed['WetArea_m2']
+        * pow(df_gmed['HydraulicRadius_m'], 2.0 / 3)
+        * pow(df_gmed['SLOPE'], 0.5)
+        / df_gmed['flow']
+    )
     print('Adjusted Mannings N Calculations -->')
     print(df_gmed)
 
     # Create dataframe to check for erroneous Manning's n values (>0.6 or <0.001)
-    df_gmed['Mann_flag'] = np.where((df_gmed['hydroid_ManningN'] >= 0.6) | (df_gmed['hydroid_ManningN'] <= 0.001),'Fail','Pass')
-    df_mann_flag = df_gmed[(df_gmed['hydroid_ManningN'] >= 0.6) | (df_gmed['hydroid_ManningN'] <= 0.001)][['HydroID','hydroid_ManningN']]
+    df_gmed['Mann_flag'] = np.where(
+        (df_gmed['hydroid_ManningN'] >= 0.6) | (df_gmed['hydroid_ManningN'] <= 0.001), 'Fail', 'Pass'
+    )
+    df_mann_flag = df_gmed[(df_gmed['hydroid_ManningN'] >= 0.6) | (df_gmed['hydroid_ManningN'] <= 0.001)][
+        ['HydroID', 'hydroid_ManningN']
+    ]
     print('Here is the df with mann_flag filter:')
     print(df_mann_flag.to_string())
     if not df_mann_flag.empty:
-        log_file.write('!!! Flaged Mannings Roughness values below !!!' +'\n')
+        log_file.write('!!! Flaged Mannings Roughness values below !!!' + '\n')
         log_file.write(df_mann_flag.to_string() + '\n')
 
     # Export csv with the newly calculated Manning's N values
     output_calc_n_csv = os.path.join(fim_directory, huc6, 'calc_src_n_vals_' + huc6 + '.csv')
-    df_gmed.to_csv(output_calc_n_csv,index=False)
+    df_gmed.to_csv(output_calc_n_csv, index=False)
 
     # filter the modified Manning's n dataframe for values out side allowable range
     df_gmed = df_gmed[df_gmed['Mann_flag'] == 'Pass']
@@ -70,7 +116,7 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
 
     # Create df with the most recent collection time entry
     df_updated = df_gmed.groupby(["HydroID"])[['coll_time']].max()
-    df_updated.rename(columns={'coll_time':'last_updated'}, inplace=True)
+    df_updated.rename(columns={'coll_time': 'last_updated'}, inplace=True)
 
     # cacluate median ManningN to handle cases with multiple hydroid entries
     df_mann = df_gmed.groupby(["HydroID"])[['hydroid_ManningN']].median()
@@ -79,14 +125,17 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
 
     # Create a df with the median hydroid_ManningN value per feature_id
     df_mann_featid = df_gmed.groupby(["feature_id"])[['hydroid_ManningN']].median()
-    df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
+    df_mann_featid.rename(columns={'hydroid_ManningN': 'featid_ManningN'}, inplace=True)
 
     # Rename the original hydrotable variables to allow new calculations to use the primary var name
-    df_htable.rename(columns={'ManningN':'default_ManningN','discharge_cms':'orig_discharge_cms'}, inplace=True)
-
-    ## Check for large variabilty in the calculated Manning's N values (for cases with mutliple entries for a singel hydroid)
-    df_nrange = df_gmed.groupby('HydroID').agg({'hydroid_ManningN': ['median', 'min', 'max','count']})
-    log_file.write('Statistics for Modified Roughness Calcs -->' +'\n')
+    df_htable.rename(
+        columns={'ManningN': 'default_ManningN', 'discharge_cms': 'orig_discharge_cms'}, inplace=True
+    )
+
+    # Check for large variabilty in the calculated Manning's N values
+    #   (for cases with mutliple entries for a singel hydroid)
+    df_nrange = df_gmed.groupby('HydroID').agg({'hydroid_ManningN': ['median', 'min', 'max', 'count']})
+    log_file.write('Statistics for Modified Roughness Calcs -->' + '\n')
     log_file.write(df_nrange.to_string() + '\n')
     log_file.write('----------------------------------------\n\n')
 
@@ -95,24 +144,33 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
     df_htable = df_htable.merge(df_mann_featid, how='left', on='feature_id')
     df_htable = df_htable.merge(df_updated, how='left', on='HydroID')
 
-    # Create the modify_ManningN column by combining the hydroid_ManningN with the featid_ManningN (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
-    df_htable['modify_ManningN'] = np.where(df_htable['hydroid_ManningN'].isnull(),df_htable['featid_ManningN'],df_htable['hydroid_ManningN'])
+    # Create the modify_ManningN column by combining the hydroid_ManningN with the featid_ManningN
+    # (use feature_id value if the hydroid is in a feature_id that contains valid hydroid_ManningN value(s))
+    df_htable['modify_ManningN'] = np.where(
+        df_htable['hydroid_ManningN'].isnull(), df_htable['featid_ManningN'], df_htable['hydroid_ManningN']
+    )
 
-    # Create the ManningN column by combining the hydroid_ManningN with the default_ManningN (use modified where available)
-    df_htable['ManningN'] = np.where(df_htable['modify_ManningN'].isnull(),df_htable['default_ManningN'],df_htable['modify_ManningN'])
+    # Create the ManningN column by combining the hydroid_ManningN with the default_ManningN
+    #   (use modified where available)
+    df_htable['ManningN'] = np.where(
+        df_htable['modify_ManningN'].isnull(), df_htable['default_ManningN'], df_htable['modify_ManningN']
+    )
 
     # Calculate new discharge_cms with new ManningN
-    df_htable['discharge_cms'] = df_htable['WetArea (m2)']* \
-    pow(df_htable['HydraulicRadius (m)'],2.0/3)* \
-    pow(df_htable['SLOPE'],0.5)/df_htable['ManningN']
+    df_htable['discharge_cms'] = (
+        df_htable['WetArea (m2)']
+        * pow(df_htable['HydraulicRadius (m)'], 2.0 / 3)
+        * pow(df_htable['SLOPE'], 0.5)
+        / df_htable['ManningN']
+    )
 
     # Replace discharge_cms with 0 or -999 if present in the original discharge
-    df_htable['discharge_cms'].mask(df_htable['orig_discharge_cms']==0.0,0.0,inplace=True)
-    df_htable['discharge_cms'].mask(df_htable['orig_discharge_cms']==-999,-999,inplace=True)
+    df_htable['discharge_cms'].mask(df_htable['orig_discharge_cms'] == 0.0, 0.0, inplace=True)
+    df_htable['discharge_cms'].mask(df_htable['orig_discharge_cms'] == -999, -999, inplace=True)
 
     # Export a new hydroTable.csv and overwrite the previous version
     out_htable = os.path.join(fim_directory, huc6, 'hydroTable.csv')
-    df_htable.to_csv(out_htable,index=False)
+    df_htable.to_csv(out_htable, index=False)
 
     # output new src json (overwrite previous)
     output_src_json = dict()
@@ -124,13 +182,13 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
         q_list = df_htable['discharge_cms'][indices_of_hid].astype(float)
         stage_list = stage_list.tolist()
         q_list = q_list.tolist()
-        output_src_json[str(hid)] = { 'q_list' : q_list , 'stage_list' : stage_list }
+        output_src_json[str(hid)] = {'q_list': q_list, 'stage_list': stage_list}
 
-    with open(output_src_json_file,'w') as f:
-        json.dump(output_src_json,f,sort_keys=True)
+    with open(output_src_json_file, 'w') as f:
+        json.dump(output_src_json, f, sort_keys=True)
 
-def ingest_points_layer(points_layer, fim_directory, wbd_path):
 
+def ingest_points_layer(points_layer, fim_directory, wbd_path):
     # Read wbd_path and points_layer to determine which HUC6 each point is in.
     wbd_huc8_read = gpd.read_file(wbd_path, layer='WBDHU6')
     points_layer_read = gpd.read_file(points_layer)
@@ -158,7 +216,7 @@ def ingest_points_layer(points_layer, fim_directory, wbd_path):
             log_file.write(str(huc6) + '\n')
 
     # Define coords variable to be used in point raster value attribution.
-    coords = [(x,y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
+    coords = [(x, y) for x, y in zip(water_edge_df.X, water_edge_df.Y)]
 
     # Define paths to relevant HUC6 HAND data.
     for huc6 in huc6_list:
@@ -182,7 +240,7 @@ def ingest_points_layer(points_layer, fim_directory, wbd_path):
             print("Rating Curve JSON file for " + huc6 + " does not exist.")
             continue
 
-#        water_edge_df = water_edge_df[water_edge_df['HUC6'] == huc6]
+        #        water_edge_df = water_edge_df[water_edge_df['HUC6'] == huc6]
 
         # Use point geometry to determine pixel values at catchment and HAND grids.
         hand_src = rasterio.open(hand_path)
@@ -190,37 +248,65 @@ def ingest_points_layer(points_layer, fim_directory, wbd_path):
         hand_src.close()
         catchments_src = rasterio.open(catchments_path)
         water_edge_df['hydroid'] = [c[0] for c in catchments_src.sample(coords)]
-        #print(water_edge_df)
+        # print(water_edge_df)
 
         # Get median HAND value for appropriate groups.
-        water_edge_median_ds = water_edge_df.groupby(["hydroid", "flow", "submitter", "coll_time", "flow_unit"])['hand'].median()
+        water_edge_median_ds = water_edge_df.groupby(
+            ["hydroid", "flow", "submitter", "coll_time", "flow_unit"]
+        )['hand'].median()
 
         output_csv = os.path.join(fim_directory, huc6, 'user_supplied_n_vals_' + huc6 + '.csv')
 
         water_edge_median_ds.to_csv(output_csv)
 
-        # 1. Loop and find the corresponding hydroids in the Hydrotable
-        # 2. Grab slope, wetted area, hydraulic radius, and feature_id that correspond with the matching hydroids and HAND value for the nearest stage
-        # 3. Calculate new column for new roughness using the above info
-        #    3b. If multiple flows exist per hydroid, aggregate the resulting Manning Ns
-        #    3c. If range of resulting Manning Ns is high, notify human
-        # 4. Update Hydrotable
-        #    4a. Copy default flow and N columns to new columns with "_default" in the field name
-        #    4b. Overwrite the official flow and N columns with the new calculated values
-        #    4c. Add last_updated column with timestamp where values were changed, also add "submitter" column
-        # 5. What do we do in catchments that match the feature_id?
-        #    5a. If these catchments already have known data, then let it use those. If not, use new calculated Ns.
+        '''
+        1. Loop and find the corresponding hydroids in the Hydrotable
+        2. Grab slope, wetted area, hydraulic radius, and feature_id that correspond with
+          the matching hydroids and HAND value for the nearest stage
+        3. Calculate new column for new roughness using the above info
+           3b. If multiple flows exist per hydroid, aggregate the resulting Manning Ns
+           3c. If range of resulting Manning Ns is high, notify human
+        4. Update Hydrotable
+           4a. Copy default flow and N columns to new columns with "_default" in the field name
+           4b. Overwrite the official flow and N columns with the new calculated values
+           4c. Add last_updated column with timestamp where values were changed, also add "submitter" column
+        5. What do we do in catchments that match the feature_id?
+           5a. If these catchments already have known data, then let it use those.
+                If not, use new calculated Ns.
+        '''
 
         update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_file, huc6)
 
 
-
 if __name__ == '__main__':
+    '''
+    Open catchment, HAND, and point grids and determine pixel values for Hydroid, HAND value,
+       and discharge value, respectively.
+
+    Open rating curve file(s).
+
+    Use three values to determine the hydroid rating curve(s) to update,
+       then update them using a variation of Manning's Equation.
+
+    Ensure the JSON rating curve is updated and saved (overwitten).
+       Consider adding attributes to document what was performed.
+    Close log file
+    '''
+
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Adjusts rating curve given a shapefile containing points of known water boundary.')
-    parser.add_argument('-p','--points-layer',help='Path to points layer containing known water boundary locations',required=True)
-    parser.add_argument('-d','--fim-directory',help='Parent directory of FIM-required datasets.',required=True)
-    parser.add_argument('-w','--wbd-path', help='Path to national HUC6 layer.',required=True)
+    parser = argparse.ArgumentParser(
+        description='Adjusts rating curve given a shapefile containing points of known water boundary.'
+    )
+    parser.add_argument(
+        '-p',
+        '--points-layer',
+        help='Path to points layer containing known water boundary locations',
+        required=True,
+    )
+    parser.add_argument(
+        '-d', '--fim-directory', help='Parent directory of FIM-required datasets.', required=True
+    )
+    parser.add_argument('-w', '--wbd-path', help='Path to national HUC6 layer.', required=True)
 
     # Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -228,21 +314,14 @@ def ingest_points_layer(points_layer, fim_directory, wbd_path):
     fim_directory = args['fim_directory']
     wbd_path = args['wbd_path']
 
-    # Create log file for processing records
     print('This may take a few minutes...')
     sys.__stdout__ = sys.stdout
-    log_file = open(os.path.join(fim_directory,'log_rating_curve_adjust.log'),"w")
-
-    ingest_points_layer(points_layer, fim_directory, wbd_path)
-
-    # Open catchment, HAND, and point grids and determine pixel values for Hydroid, HAND value, and discharge value, respectively.
 
-    # Open rating curve file(s).
-
-    # Use three values to determine the hydroid rating curve(s) to update, then update them using a variation of Manning's Equation.
+    # Create log file for processing records
+    log_file = open(os.path.join(fim_directory, 'log_rating_curve_adjust.log'), "w")
 
-    # Ensure the JSON rating curve is updated and saved (overwitten). Consider adding attributes to document what was performed.
+    ingest_points_layer(points_layer, fim_directory, wbd_path)
 
-    # Close log file
     sys.stdout = sys.__stdout__
+
     log_file.close()
diff --git a/tools/aggregate_csv_files.py b/tools/aggregate_csv_files.py
index cfd531eba..0efeafe59 100644
--- a/tools/aggregate_csv_files.py
+++ b/tools/aggregate_csv_files.py
@@ -1,57 +1,98 @@
-import os, fnmatch
-import pandas as pd
+#!/usr/bin/env python3
+
 import argparse
+import fnmatch
+import os
+
+import pandas as pd
+
 
 def locate(pattern, root_path):
     for path, dirs, files in os.walk(os.path.abspath(root_path)):
         for filename in fnmatch.filter(files, pattern):
             yield os.path.join(path, filename)
 
-def read_csvs_to_df(files_to_merge,head_row):
-    #df = pd.concat((pd.read_csv(f,usecols=["HUC", "feature_id", "HydroID", "last_updated", "submitter", "adjust_ManningN"],dtype={'feature_id': 'int64','HUC': str}) for f in files_to_merge), ignore_index=True)
+
+def read_csvs_to_df(files_to_merge, head_row):
+    # df = pd.concat(
+    #     (
+    #         pd.read_csv(
+    #             f,
+    #             usecols=["HUC", "feature_id", "HydroID", "last_updated", "submitter", "adjust_ManningN"],
+    #             dtype={'feature_id': 'int64', 'HUC': str},
+    #         )
+    #         for f in files_to_merge
+    #     ),
+    #     ignore_index=True,
+    # )
     li = []
     for file_in in files_to_merge:
         print(file_in)
         ## Use below for merging hydroTables for calib n value data
-        #df = pd.read_csv(file_in,usecols=["HUC", "feature_id", "HydroID", "last_updated", "submitter", "adjust_ManningN"],dtype={'feature_id': 'int64','HUC': str}, index_col=None, header=0)
-        #df = df[df['adjust_ManningN'].notna()]
-        #df.drop_duplicates(subset=['HydroID'],inplace=True)
-        df = pd.read_csv(file_in, index_col=None, header=head_row) #dtype={'feature_id': 'int64'}
+        # df = pd.read_csv(
+        #     file_in,
+        #     usecols=["HUC", "feature_id", "HydroID", "last_updated", "submitter", "adjust_ManningN"],
+        #     dtype={'feature_id': 'int64', 'HUC': str},
+        #     index_col=None,
+        #     header=0,
+        # )
+        # df = df[df['adjust_ManningN'].notna()]
+        # df.drop_duplicates(subset=['HydroID'], inplace=True)
+        df = pd.read_csv(file_in, index_col=None, header=head_row)  # dtype={'feature_id': 'int64'}
         df = df[df['Unnamed: 0'] != 'HydroID']
         li.append(df)
 
     frame = pd.concat(li, axis=0, ignore_index=True)
-    return(frame)
+    return frame
+
 
-def write_aggregate(frame,output_file):
+def write_aggregate(frame, output_file):
     print('Writing new csv file: ' + output_file)
-    frame.to_csv(output_file,index=False)
+    frame.to_csv(output_file, index=False)
+
 
 def concat_files(files_to_merge):
     # joining files with concat and read_csv
     print('Concatenating all matching csv files...')
-    df_concat = pd.concat(map(pd.read_csv, files_to_merge), ignore_index=True)
-    
-def run_prep(fim_dir,file_search_str,head_row,output_file):
+    pd.concat(map(pd.read_csv, files_to_merge), ignore_index=True)
+
+
+def run_prep(fim_dir, file_search_str, head_row, output_file):
     assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
 
     files_to_merge = [js for js in locate('*' + file_search_str + '*.csv', fim_dir)]
     if len(files_to_merge) > 0:
         print('Found files: ' + str(len(files_to_merge)))
 
-        aggreg_df = read_csvs_to_df(files_to_merge,head_row)
-        write_aggregate(aggreg_df,output_file)
+        aggreg_df = read_csvs_to_df(files_to_merge, head_row)
+        write_aggregate(aggreg_df, output_file)
 
     else:
         print('Did not find any files using tag: ' + '*' + file_search_str + '*.csv')
 
-if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description="Simple tool to search for csv files (using wildcard text) within a fim output directory and then aggregate all files into a single csv")
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir (e.g. data/outputs/xxxx/', required=True,type=str)
-    parser.add_argument('-search_str','--file-search-str', help='File search string', required=True,type=str)
-    parser.add_argument('-head_row','--header-row', help='Optional: header row to parse (default=0)',default=0, required=False,type=int)
-    parser.add_argument('-out_csv','--output-csv', help='full filepath to write new csv', required=True,type=str)
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Simple tool to search for csv files (using wildcard text) within a fim output directory "
+        "and then aggregate all files into a single csv"
+    )
+    parser.add_argument(
+        '-fim_dir', '--fim-dir', help='FIM output dir (e.g. data/outputs/xxxx/', required=True, type=str
+    )
+    parser.add_argument(
+        '-search_str', '--file-search-str', help='File search string', required=True, type=str
+    )
+    parser.add_argument(
+        '-head_row',
+        '--header-row',
+        help='Optional: header row to parse (default=0)',
+        default=0,
+        required=False,
+        type=int,
+    )
+    parser.add_argument(
+        '-out_csv', '--output-csv', help='full filepath to write new csv', required=True, type=str
+    )
 
     args = vars(parser.parse_args())
     fim_dir = args['fim_dir']
@@ -59,5 +100,4 @@ def run_prep(fim_dir,file_search_str,head_row,output_file):
     head_row = args['header_row']
     output_file = args['output_csv']
 
-    run_prep(fim_dir,file_search_str,head_row,output_file)
-
+    run_prep(fim_dir, file_search_str, head_row, output_file)
diff --git a/tools/aggregate_mannings_calibration.py b/tools/aggregate_mannings_calibration.py
index ff9565850..d447c696e 100755
--- a/tools/aggregate_mannings_calibration.py
+++ b/tools/aggregate_mannings_calibration.py
@@ -1,110 +1,234 @@
 #!/usr/bin/env python3
 
-import os
-import pandas as pd
+import argparse
 import csv
 import json
-import argparse
+import os
 
+import pandas as pd
 
-def aggregate_parameter_sets(huc_list_path,calibration_stat_folder,summary_file, mannings_json):
 
+def aggregate_parameter_sets(huc_list_path, calibration_stat_folder, summary_file, mannings_json):
     outfolder = os.path.dirname(summary_file)
     aggregate_output_dir = os.path.join(outfolder, 'aggregate_metrics')
 
     if not os.path.exists(aggregate_output_dir):
         os.makedirs(aggregate_output_dir)
 
-    mannings_summary_table = pd.DataFrame(columns = ['metric', 'value', 'stream_order', 'mannings_n', 'huc', 'interval'])
+    mannings_summary_table = pd.DataFrame(
+        columns=['metric', 'value', 'stream_order', 'mannings_n', 'huc', 'interval']
+    )
 
     with open(huc_list_path) as f:
         huc_list = [huc.rstrip() for huc in f]
 
     for huc in huc_list:
-        branch_dir = os.path.join('data','test_cases',str(huc) + '_ble','performance_archive','development_versions', calibration_stat_folder)
+        branch_dir = os.path.join(
+            'data',
+            'test_cases',
+            str(huc) + '_ble',
+            'performance_archive',
+            'development_versions',
+            calibration_stat_folder,
+        )
         for stream_order in os.listdir(branch_dir):
             stream_order_dir = os.path.join(branch_dir, stream_order)
             for mannings_value in os.listdir(stream_order_dir):
                 mannings_value_dir = os.path.join(stream_order_dir, mannings_value)
                 for flood_recurrence in os.listdir(mannings_value_dir):
                     flood_recurrence_dir = os.path.join(mannings_value_dir, flood_recurrence)
-                    total_area_stats = pd.read_csv(os.path.join(flood_recurrence_dir, 'total_area_stats.csv'), index_col=0)
-                    total_area_stats = total_area_stats.loc[['true_positives_count', 'true_negatives_count', 'false_positives_count', 'false_negatives_count','masked_count', 'cell_area_m2', 'CSI'],:]
+                    total_area_stats = pd.read_csv(
+                        os.path.join(flood_recurrence_dir, 'total_area_stats.csv'), index_col=0
+                    )
+                    total_area_stats = total_area_stats.loc[
+                        [
+                            'true_positives_count',
+                            'true_negatives_count',
+                            'false_positives_count',
+                            'false_negatives_count',
+                            'masked_count',
+                            'cell_area_m2',
+                            'CSI',
+                        ],
+                        :,
+                    ]
                     total_area_stats = total_area_stats.reset_index()
-                    total_area_stats_table = pd.DataFrame({'metric': total_area_stats.iloc[:,0], 'value': total_area_stats.iloc[:,1], 'stream_order': stream_order, 'mannings_n': mannings_value, 'huc': huc, 'interval': flood_recurrence})
-                    mannings_summary_table = pd.concat([mannings_summary_table, total_area_stats_table], ignore_index=True)
-
-    mannings_summary_table.to_csv(summary_file,index=False)
+                    total_area_stats_table = pd.DataFrame(
+                        {
+                            'metric': total_area_stats.iloc[:, 0],
+                            'value': total_area_stats.iloc[:, 1],
+                            'stream_order': stream_order,
+                            'mannings_n': mannings_value,
+                            'huc': huc,
+                            'interval': flood_recurrence,
+                        }
+                    )
+                    mannings_summary_table = pd.concat(
+                        [mannings_summary_table, total_area_stats_table], ignore_index=True
+                    )
+
+    mannings_summary_table.to_csv(summary_file, index=False)
 
     ## calculate optimal parameter set
     from utils.shared_functions import compute_stats_from_contingency_table
 
-    true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = 0, 0, 0, 0, 0, 0
-
-    list_to_write = [['metric', 'value', 'stream_order', 'mannings_value', 'return_interval']]  # Initialize header.
+    true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = (
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+    )
+
+    list_to_write = [
+        ['metric', 'value', 'stream_order', 'mannings_value', 'return_interval']
+    ]  # Initialize header.
     for stream_order in mannings_summary_table.stream_order.unique():
         for return_interval in mannings_summary_table.interval.unique():
             for mannings_value in mannings_summary_table.mannings_n.unique():
-                true_positives = mannings_summary_table.loc[(mannings_summary_table['interval']==return_interval) & (mannings_summary_table['stream_order']==stream_order) & (mannings_summary_table['mannings_n']==mannings_value) & (mannings_summary_table['metric']=='true_positives_count'),'value'].sum()
-                true_negatives = mannings_summary_table.loc[(mannings_summary_table['interval']==return_interval) & (mannings_summary_table['stream_order']==stream_order) & (mannings_summary_table['mannings_n']==mannings_value) & (mannings_summary_table['metric']=='true_negatives_count'),'value'].sum()
-                false_positives = mannings_summary_table.loc[(mannings_summary_table['interval']==return_interval) & (mannings_summary_table['stream_order']==stream_order) & (mannings_summary_table['mannings_n']==mannings_value) & (mannings_summary_table['metric']=='false_positives_count'),'value'].sum()
-                false_negatives = mannings_summary_table.loc[(mannings_summary_table['interval']==return_interval) & (mannings_summary_table['stream_order']==stream_order) & (mannings_summary_table['mannings_n']==mannings_value) & (mannings_summary_table['metric']=='false_negatives_count'),'value'].sum()
-                masked_count = mannings_summary_table.loc[(mannings_summary_table['interval']==return_interval) & (mannings_summary_table['stream_order']==stream_order) & (mannings_summary_table['mannings_n']==mannings_value) & (mannings_summary_table['metric']=='masked_count'),'value'].sum()
-
-                cell_area = mannings_summary_table.loc[(mannings_summary_table['interval']==return_interval) & (mannings_summary_table['stream_order']==stream_order) & (mannings_summary_table['mannings_n']==mannings_value) & (mannings_summary_table['metric']=='cell_area_m2'),'value'].sum()
+                true_positives = mannings_summary_table.loc[
+                    (mannings_summary_table['interval'] == return_interval)
+                    & (mannings_summary_table['stream_order'] == stream_order)
+                    & (mannings_summary_table['mannings_n'] == mannings_value)
+                    & (mannings_summary_table['metric'] == 'true_positives_count'),
+                    'value',
+                ].sum()
+                true_negatives = mannings_summary_table.loc[
+                    (mannings_summary_table['interval'] == return_interval)
+                    & (mannings_summary_table['stream_order'] == stream_order)
+                    & (mannings_summary_table['mannings_n'] == mannings_value)
+                    & (mannings_summary_table['metric'] == 'true_negatives_count'),
+                    'value',
+                ].sum()
+                false_positives = mannings_summary_table.loc[
+                    (mannings_summary_table['interval'] == return_interval)
+                    & (mannings_summary_table['stream_order'] == stream_order)
+                    & (mannings_summary_table['mannings_n'] == mannings_value)
+                    & (mannings_summary_table['metric'] == 'false_positives_count'),
+                    'value',
+                ].sum()
+                false_negatives = mannings_summary_table.loc[
+                    (mannings_summary_table['interval'] == return_interval)
+                    & (mannings_summary_table['stream_order'] == stream_order)
+                    & (mannings_summary_table['mannings_n'] == mannings_value)
+                    & (mannings_summary_table['metric'] == 'false_negatives_count'),
+                    'value',
+                ].sum()
+                masked_count = mannings_summary_table.loc[
+                    (mannings_summary_table['interval'] == return_interval)
+                    & (mannings_summary_table['stream_order'] == stream_order)
+                    & (mannings_summary_table['mannings_n'] == mannings_value)
+                    & (mannings_summary_table['metric'] == 'masked_count'),
+                    'value',
+                ].sum()
+
+                cell_area = mannings_summary_table.loc[
+                    (mannings_summary_table['interval'] == return_interval)
+                    & (mannings_summary_table['stream_order'] == stream_order)
+                    & (mannings_summary_table['mannings_n'] == mannings_value)
+                    & (mannings_summary_table['metric'] == 'cell_area_m2'),
+                    'value',
+                ].sum()
 
                 # Pass all sums to shared function to calculate metrics.
-                stats_dict = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=cell_area, masked_count=masked_count)
+                stats_dict = compute_stats_from_contingency_table(
+                    true_negatives,
+                    false_negatives,
+                    false_positives,
+                    true_positives,
+                    cell_area=cell_area,
+                    masked_count=masked_count,
+                )
 
                 for stat in stats_dict:
-                    list_to_write.append([stat, stats_dict[stat], stream_order, mannings_value, return_interval])
+                    list_to_write.append(
+                        [stat, stats_dict[stat], stream_order, mannings_value, return_interval]
+                    )
 
     # Map path to output directory for aggregate metrics.
-    output_file = os.path.join(aggregate_output_dir,'aggregate_metrics_mannings_calibration_by_streamorder.csv')
+    output_file = os.path.join(
+        aggregate_output_dir, 'aggregate_metrics_mannings_calibration_by_streamorder.csv'
+    )
 
     with open(output_file, 'w', newline='') as csvfile:
         csv_writer = csv.writer(csvfile)
         csv_writer.writerows(list_to_write)
 
-
     print("Finished aggregating metrics over " + str(len(huc_list)) + " test cases.")
 
-    print ('Writing optimal mannings parameter set')
+    print('Writing optimal mannings parameter set')
 
     manning_dict = {}
     list_to_write_pd = pd.read_csv(output_file)
     for stream_order in list_to_write_pd.stream_order.unique():
-        interval_100 = list_to_write_pd.loc[(list_to_write_pd['stream_order']==stream_order) & (list_to_write_pd['metric']=='CSI') & (list_to_write_pd['return_interval']=='100yr'), 'value'].max()
-        interval_500 = list_to_write_pd.loc[(list_to_write_pd['stream_order']==stream_order) & (list_to_write_pd['metric']=='CSI') & (list_to_write_pd['return_interval']=='500yr'), 'value'].max()
-        mannings_100yr = list_to_write_pd.loc[(list_to_write_pd['stream_order']==stream_order) & (list_to_write_pd['metric']=='CSI') & (list_to_write_pd['return_interval']=='100yr') & (list_to_write_pd['value']==interval_100), 'mannings_value']
-        mannings_500yr = list_to_write_pd.loc[(list_to_write_pd['stream_order']==stream_order) & (list_to_write_pd['metric']=='CSI') & (list_to_write_pd['return_interval']=='500yr') & (list_to_write_pd['value']==interval_500), 'mannings_value']
-        if (len(mannings_100yr)==1) & (len(mannings_500yr)==1):
+        interval_100 = list_to_write_pd.loc[
+            (list_to_write_pd['stream_order'] == stream_order)
+            & (list_to_write_pd['metric'] == 'CSI')
+            & (list_to_write_pd['return_interval'] == '100yr'),
+            'value',
+        ].max()
+        interval_500 = list_to_write_pd.loc[
+            (list_to_write_pd['stream_order'] == stream_order)
+            & (list_to_write_pd['metric'] == 'CSI')
+            & (list_to_write_pd['return_interval'] == '500yr'),
+            'value',
+        ].max()
+        mannings_100yr = list_to_write_pd.loc[
+            (list_to_write_pd['stream_order'] == stream_order)
+            & (list_to_write_pd['metric'] == 'CSI')
+            & (list_to_write_pd['return_interval'] == '100yr')
+            & (list_to_write_pd['value'] == interval_100),
+            'mannings_value',
+        ]
+        mannings_500yr = list_to_write_pd.loc[
+            (list_to_write_pd['stream_order'] == stream_order)
+            & (list_to_write_pd['metric'] == 'CSI')
+            & (list_to_write_pd['return_interval'] == '500yr')
+            & (list_to_write_pd['value'] == interval_500),
+            'mannings_value',
+        ]
+        if (len(mannings_100yr) == 1) & (len(mannings_500yr) == 1):
             if mannings_100yr.iloc[0] == mannings_500yr.iloc[0]:
                 manning_dict[str(stream_order)] = mannings_100yr.iloc[0]
             else:
-                print ('100yr and 500yr optimal mannings vary by ' + str(round(abs(mannings_100yr.iloc[0] - mannings_500yr.iloc[0]),2)) + " for stream order " + str(stream_order))
-                print ('Selecting optimal mannings n for 100yr event')
+                print(
+                    '100yr and 500yr optimal mannings vary by '
+                    + str(round(abs(mannings_100yr.iloc[0] - mannings_500yr.iloc[0]), 2))
+                    + " for stream order "
+                    + str(stream_order)
+                )
+                print('Selecting optimal mannings n for 100yr event')
                 manning_dict[str(stream_order)] = mannings_100yr.iloc[0]
-        elif (len(mannings_100yr)>1) or (len(mannings_500yr)>1):
-            print ('multiple values achieve optimal results '  + " for stream order " + str(stream_order))
-            print ('Selecting optimal mannings n for 100yr event')
+        elif (len(mannings_100yr) > 1) or (len(mannings_500yr) > 1):
+            print('multiple values achieve optimal results ' + " for stream order " + str(stream_order))
+            print('Selecting optimal mannings n for 100yr event')
             manning_dict[str(stream_order)] = mannings_100yr.iloc[0]
 
-
-    for n in range(1,15):
+    for n in range(1, 15):
         if str(n) not in manning_dict:
             manning_dict[str(n)] = 0.06
 
     with open(mannings_json, "w") as outfile:
         json.dump(manning_dict, outfile)
 
-if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description='Aggregates the evaluation statistics of all mannings calibration runs')
-    parser.add_argument('-l','--huc-list-path', help='csv list of HUCs to aggregate', required=True)
-    parser.add_argument('-c','--calibration-stat-folder', help='eval stat column name', required=True)
-    parser.add_argument('-f','--summary-file', help='output file with aggregate mannings calibration stats', required=True)
-    parser.add_argument('-e','--mannings-json', help='file path for optimal mannings n parameter set json', required=False,default="/foss_fim/config/mannings_calibrated.json")
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Aggregates the evaluation statistics of all mannings calibration runs'
+    )
+    parser.add_argument('-l', '--huc-list-path', help='csv list of HUCs to aggregate', required=True)
+    parser.add_argument('-c', '--calibration-stat-folder', help='eval stat column name', required=True)
+    parser.add_argument(
+        '-f', '--summary-file', help='output file with aggregate mannings calibration stats', required=True
+    )
+    parser.add_argument(
+        '-e',
+        '--mannings-json',
+        help='file path for optimal mannings n parameter set json',
+        required=False,
+        default="/foss_fim/config/mannings_calibrated.json",
+    )
 
     args = vars(parser.parse_args())
 
@@ -113,4 +237,4 @@ def aggregate_parameter_sets(huc_list_path,calibration_stat_folder,summary_file,
     summary_file = args['summary_file']
     mannings_json = args['mannings_json']
 
-    aggregate_parameter_sets(huc_list_path,calibration_stat_folder,summary_file,mannings_json)
+    aggregate_parameter_sets(huc_list_path, calibration_stat_folder, summary_file, mannings_json)
diff --git a/tools/aggregate_metrics.py b/tools/aggregate_metrics.py
index bd5a99fdd..0f78cc551 100755
--- a/tools/aggregate_metrics.py
+++ b/tools/aggregate_metrics.py
@@ -1,70 +1,76 @@
 #!/usr/bin/env python3
 
+import argparse
+import csv
 import json
 import os
-import csv
 
-import argparse
+from tools_shared_functions import compute_stats_from_contingency_table
 
-TEST_CASES_DIR = r'/data/test_cases_new/'
+
+TEST_CASES_DIR = r'/data/test_cases/'
 # TEMP = r'/data/temp'
 
 # Search through all previous_versions in test_cases
-from tools_shared_functions import compute_stats_from_contingency_table
 
-def create_master_metrics_csv():
 
+def create_master_metrics_csv(output_csv):
     # Construct header
-    metrics_to_write = ['true_negatives_count',
-                        'false_negatives_count',
-                        'true_positives_count',
-                        'false_positives_count',
-                        'contingency_tot_count',
-                        'cell_area_m2',
-                        'TP_area_km2',
-                        'FP_area_km2',
-                        'TN_area_km2',
-                        'FN_area_km2',
-                        'contingency_tot_area_km2',
-                        'predPositive_area_km2',
-                        'predNegative_area_km2',
-                        'obsPositive_area_km2',
-                        'obsNegative_area_km2',
-                        'positiveDiff_area_km2',
-                        'CSI',
-                        'FAR',
-                        'TPR',
-                        'TNR',
-                        'PPV',
-                        'NPV',
-                        'ACC',
-                        'Bal_ACC',
-                        'MCC',
-                        'EQUITABLE_THREAT_SCORE',
-                        'PREVALENCE',
-                        'BIAS',
-                        'F1_SCORE',
-                        'TP_perc',
-                        'FP_perc',
-                        'TN_perc',
-                        'FN_perc',
-                        'predPositive_perc',
-                        'predNegative_perc',
-                        'obsPositive_perc',
-                        'obsNegative_perc',
-                        'positiveDiff_perc',
-                        'masked_count',
-                        'masked_perc',
-                        'masked_area_km2'
-                        ]
+    metrics_to_write = [
+        'true_negatives_count',
+        'false_negatives_count',
+        'true_positives_count',
+        'false_positives_count',
+        'contingency_tot_count',
+        'cell_area_m2',
+        'TP_area_km2',
+        'FP_area_km2',
+        'TN_area_km2',
+        'FN_area_km2',
+        'contingency_tot_area_km2',
+        'predPositive_area_km2',
+        'predNegative_area_km2',
+        'obsPositive_area_km2',
+        'obsNegative_area_km2',
+        'positiveDiff_area_km2',
+        'CSI',
+        'FAR',
+        'TPR',
+        'TNR',
+        'PPV',
+        'NPV',
+        'ACC',
+        'Bal_ACC',
+        'MCC',
+        'EQUITABLE_THREAT_SCORE',
+        'PREVALENCE',
+        'BIAS',
+        'F1_SCORE',
+        'TP_perc',
+        'FP_perc',
+        'TN_perc',
+        'FN_perc',
+        'predPositive_perc',
+        'predNegative_perc',
+        'obsPositive_perc',
+        'obsNegative_perc',
+        'positiveDiff_perc',
+        'masked_count',
+        'masked_perc',
+        'masked_area_km2',
+    ]
 
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
-    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source']]
+    list_to_write = [
+        additional_header_info_prefix
+        + metrics_to_write
+        + ['full_json_path']
+        + ['flow']
+        + ['benchmark_source']
+    ]
 
     for benchmark_type in ['ble', 'ahps']:
-
         if benchmark_type == 'ble':
-
             test_cases = r'/data/test_cases'
             test_cases_list = os.listdir(test_cases)
             # AHPS test_ids
@@ -75,7 +81,9 @@ def create_master_metrics_csv():
                     int(test_case.split('_')[0])
 
                     huc = test_case.split('_')[0]
-                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
+                    previous_versions = os.path.join(
+                        test_cases, test_case, 'performance_archive', 'previous_versions'
+                    )
 
                     for magnitude in ['100yr', '500yr']:
                         for version in versions_to_aggregate:
@@ -83,7 +91,6 @@ def create_master_metrics_csv():
                             magnitude_dir = os.path.join(version_dir, magnitude)
 
                             if os.path.exists(magnitude_dir):
-
                                 magnitude_dir_list = os.listdir(magnitude_dir)
                                 for f in magnitude_dir_list:
                                     if '.json' in f:
@@ -106,25 +113,31 @@ def create_master_metrics_csv():
                     pass
 
         if benchmark_type == 'ahps':
-
             test_cases = r'/data/test_cases_ahps_testing'
             test_cases_list = os.listdir(test_cases)
             # AHPS test_ids
-            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021',
-                                     'fim_2_x_ms_nws_1_21_2021', 'fim_2_x_ms_usgs_1_21_2021',
-                                     'fim_3_0_0_3_ms_c_nws_1_21_2021', 'fim_3_0_0_3_ms_c_usgs_1_21_2021',
-                                     'ms_xwalk_fill_missing_cal_nws', 'ms_xwalk_fill_missing_cal_usgs']
+            versions_to_aggregate = [
+                'fim_1_0_0_nws_1_21_2021',
+                'fim_1_0_0_usgs_1_21_2021',
+                'fim_2_x_ms_nws_1_21_2021',
+                'fim_2_x_ms_usgs_1_21_2021',
+                'fim_3_0_0_3_ms_c_nws_1_21_2021',
+                'fim_3_0_0_3_ms_c_usgs_1_21_2021',
+                'ms_xwalk_fill_missing_cal_nws',
+                'ms_xwalk_fill_missing_cal_usgs',
+            ]
 
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
 
                     huc = test_case.split('_')[0]
-                    previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
+                    previous_versions = os.path.join(
+                        test_cases, test_case, 'performance_archive', 'previous_versions'
+                    )
 
                     for magnitude in ['action', 'minor', 'moderate', 'major']:
                         for version in versions_to_aggregate:
-
                             if 'nws' in version:
                                 benchmark_source = 'ahps_nws'
                             if 'usgs' in version:
@@ -148,7 +161,20 @@ def create_master_metrics_csv():
                                             if 'nws' in version:
                                                 parent_dir = 'nws_1_21_2021'
 
-                                            flow_file = os.path.join(test_cases, parent_dir, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                                            flow_file = os.path.join(
+                                                test_cases,
+                                                parent_dir,
+                                                huc,
+                                                nws_lid,
+                                                magnitude,
+                                                'ahps_'
+                                                + nws_lid
+                                                + '_huc_'
+                                                + huc
+                                                + '_flows_'
+                                                + magnitude
+                                                + '.csv',
+                                            )
                                             if os.path.exists(flow_file):
                                                 with open(flow_file, newline='') as csv_file:
                                                     reader = csv.reader(csv_file)
@@ -174,9 +200,7 @@ def create_master_metrics_csv():
         csv_writer.writerows(list_to_write)
 
 
-
 def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfolder=""):
-
     # Read hucs into list.
     if hucs != "":
         huc_list = [line.rstrip('\n') for line in open(hucs)]
@@ -192,7 +216,9 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
     # Make directory to store output aggregates.
     if special_string != "":
         special_string = "_" + special_string
-    aggregate_output_dir = os.path.join(outfolder, 'aggregate_metrics', branch + '_aggregate_metrics' + special_string)
+    aggregate_output_dir = os.path.join(
+        outfolder, 'aggregate_metrics', branch + '_aggregate_metrics' + special_string
+    )
     if not os.path.exists(aggregate_output_dir):
         os.makedirs(aggregate_output_dir)
 
@@ -200,16 +226,24 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
 
     for magnitude in ['100yr', '500yr', 'action', 'minor', 'moderate', 'major']:
         huc_path_list = [['huc', 'path']]
-        true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = 0, 0, 0, 0, 0, 0
+        (true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count) = (
+            0,
+            0,
+            0,
+            0,
+            0,
+            0,
+        )
 
         for test_case in test_cases_dir_list:
-
             if test_case not in ['validation_data_ble', 'validation_data_legacy', 'validation_data_ahps']:
-                branch_results_dir = os.path.join(TEST_CASES_DIR, test_case, 'performance_archive', config_version, branch)
+                branch_results_dir = os.path.join(
+                    TEST_CASES_DIR, test_case, 'performance_archive', config_version, branch
+                )
 
                 huc = test_case.split('_')[0]
                 # Check that the huc is in the list of hucs to aggregate.
-                if huc_list != None and huc not in huc_list:
+                if huc_list is not None and huc not in huc_list:
                     continue
 
                 stats_json_path = os.path.join(branch_results_dir, magnitude, 'total_area_stats.json')
@@ -228,12 +262,18 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
 
                     huc_path_list.append([huc, stats_json_path])
 
-
             if cell_area == 0:
                 continue
 
             # Pass all sums to shared function to calculate metrics.
-            stats_dict = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=cell_area, masked_count=masked_count)
+            stats_dict = compute_stats_from_contingency_table(
+                true_negatives,
+                false_negatives,
+                false_positives,
+                true_positives,
+                cell_area=cell_area,
+                masked_count=masked_count,
+            )
 
             list_to_write = [['metric', 'value']]  # Initialize header.
 
@@ -241,7 +281,9 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
                 list_to_write.append([stat, stats_dict[stat]])
 
             # Map path to output directory for aggregate metrics.
-            output_file = os.path.join(aggregate_output_dir, branch + '_aggregate_metrics_' + magnitude + special_string + '.csv')
+            output_file = os.path.join(
+                aggregate_output_dir, branch + '_aggregate_metrics_' + magnitude + special_string + '.csv'
+            )
 
         if cell_area != 0:
             with open(output_file, 'w', newline='') as csvfile:
@@ -251,20 +293,34 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
                 csv_writer.writerows(huc_path_list)
 
             print()
-            print("Finished aggregating for the '" + magnitude + "' magnitude. Aggregated metrics over " + str(len(huc_path_list)-1) + " test cases.")
+            print(
+                "Finished aggregating for the '"
+                + magnitude
+                + "' magnitude. Aggregated metrics over "
+                + str(len(huc_path_list) - 1)
+                + " test cases."
+            )
             print()
             print("Results are at: " + output_file)
             print()
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='Aggregates a metric or metrics for multiple HUC8s.')
-    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=False)
-    parser.add_argument('-b','--branch',help='Name of branch to check all test_cases for and to aggregate.',required=True)
-    parser.add_argument('-u','--hucs',help='HUC8s to restrict the aggregation.',required=False, default="")
-    parser.add_argument('-s','--special_string',help='Special string to add to outputs.',required=False, default="")
-    parser.add_argument('-f','--outfolder',help='output folder',required=True,type=str)
+    parser.add_argument(
+        '-c',
+        '--config',
+        help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',
+        required=False,
+    )
+    parser.add_argument(
+        '-b', '--branch', help='Name of branch to check all test_cases for and to aggregate.', required=True
+    )
+    parser.add_argument('-u', '--hucs', help='HUC8s to restrict the aggregation.', required=False, default="")
+    parser.add_argument(
+        '-s', '--special_string', help='Special string to add to outputs.', required=False, default=""
+    )
+    parser.add_argument('-f', '--outfolder', help='output folder', required=True, type=str)
 
     args = vars(parser.parse_args())
 
diff --git a/tools/cache_metrics.py b/tools/cache_metrics.py
index 825b8f4fb..4402cec83 100755
--- a/tools/cache_metrics.py
+++ b/tools/cache_metrics.py
@@ -1,19 +1,19 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import os
 import traceback
-
-from run_test_case import run_alpha_test
 from multiprocessing import Pool
 
-TEST_CASES_DIR = r'/data/test_cases_new/'  #TODO remove "_new"
+from run_test_case import Test_Case
+
+
+TEST_CASES_DIR = r'/data/test_cases/'
 PREVIOUS_FIM_DIR = r'/data/previous_fim'
 OUTPUTS_DIR = r'/outputs'
 
 
 def process_alpha_test(args):
-
     fim_run_dir = args[0]
     version = args[1]
     test_id = args[2]
@@ -22,26 +22,54 @@ def process_alpha_test(args):
 
     mask_type = 'huc'
 
-    if archive_results == False:
+    if archive_results is False:
         compare_to_previous = True
     else:
         compare_to_previous = False
 
     try:
-        run_alpha_test(fim_run_dir, version, test_id, magnitude, compare_to_previous=compare_to_previous, archive_results=archive_results, mask_type=mask_type)
+        Test_Case.run_alpha_test(
+            fim_run_dir,
+            version,
+            test_id,
+            magnitude,
+            compare_to_previous=compare_to_previous,
+            archive_results=archive_results,
+            mask_type=mask_type,
+        )
     except Exception:
         traceback.print_exc()
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
-    parser.add_argument('-v','--fim-version',help='Name of fim version to cache.',required=False, default="all")
-    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
-    parser.add_argument('-s','--special-string',help='Add a special name to the end of the branch.',required=False, default="")
-    parser.add_argument('-b','--benchmark-category',help='Options include ble or ahps. Defaults to process both.',required=False, default=None)
+    parser.add_argument(
+        '-c',
+        '--config',
+        help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',
+        required=True,
+    )
+    parser.add_argument(
+        '-v', '--fim-version', help='Name of fim version to cache.', required=False, default="all"
+    )
+    parser.add_argument(
+        '-j', '--job-number', help='Number of processes to use. Default is 1.', required=False, default="1"
+    )
+    parser.add_argument(
+        '-s',
+        '--special-string',
+        help='Add a special name to the end of the branch.',
+        required=False,
+        default="",
+    )
+    parser.add_argument(
+        '-b',
+        '--benchmark-category',
+        help='Options include ble or ahps. Defaults to process both.',
+        required=False,
+        default=None,
+    )
 
     test_cases_dir_list = os.listdir(TEST_CASES_DIR)
 
@@ -67,7 +95,7 @@ def process_alpha_test(args):
 
     benchmark_category_list = []
 
-    if benchmark_category == None:
+    if benchmark_category is None:
         for d in test_cases_dir_list:
             if 'test_cases' in d:
                 benchmark_category_list.append(d.replace('_test_cases', ''))
@@ -82,19 +110,18 @@ def process_alpha_test(args):
 
         for test_id in bench_cat_test_case_list:
             if 'validation' not in test_id:
-
                 current_huc = test_id.split('_')[0]
                 if test_id.split('_')[1] in bench_cat:
-
                     for version in previous_fim_list:
-
                         if config == 'DEV':
                             fim_run_dir = os.path.join(OUTPUTS_DIR, version, current_huc)
                         elif config == 'PREV':
                             fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc)
 
                         if not os.path.exists(fim_run_dir):
-                            fim_run_dir = os.path.join(PREVIOUS_FIM_DIR, version, current_huc[:6])  # For previous versions of HAND computed at HUC6 scale
+                            fim_run_dir = os.path.join(
+                                PREVIOUS_FIM_DIR, version, current_huc[:6]
+                            )  # For previous versions of HAND computed at HUC6 scale
 
                         if os.path.exists(fim_run_dir):
                             if special_string != "":
@@ -111,7 +138,9 @@ def process_alpha_test(args):
                             if job_number > 1:
                                 procs_list.append([fim_run_dir, version, test_id, magnitude, archive_results])
                             else:
-                                process_alpha_test([fim_run_dir, version, test_id, magnitude, archive_results])
+                                process_alpha_test(
+                                    [fim_run_dir, version, test_id, magnitude, archive_results]
+                                )
 
     if job_number > 1:
         with Pool(processes=job_number) as pool:
diff --git a/tools/check_deep_flooding.py b/tools/check_deep_flooding.py
index e9bfd746c..daf60aa58 100644
--- a/tools/check_deep_flooding.py
+++ b/tools/check_deep_flooding.py
@@ -1,50 +1,55 @@
+#!/usr/bin/env python3
+
 import argparse
 import os
 from multiprocessing import Pool
+
+import geopandas as gpd
 import numpy as np
-import rasterio.shutil
-from rasterio.warp import calculate_default_transform, reproject, Resampling
-import rasterio.crs
 import rasterio
+import rasterio.crs
 import rasterio.mask
-import geopandas as gpd
+import rasterio.shutil
+from rasterio.warp import Resampling, calculate_default_transform, reproject
 from shapely.geometry import box
 
 
 def check_deep_flooding(args):
-    
     depth_grid_path = args[0]
     shapefile_path = args[1]
     depth_threshold = args[2]
     output_dir = args[3]
-    
+
     print("Checking " + depth_grid_path + "...")
-    
+
     # Open depth_grid_path and shapefile_path and perform np.wheres
     depth_src = rasterio.open(depth_grid_path)
     depth_array = depth_src.read(1)
     reference = depth_src
-    
-    #Read layer using the bbox option. CRS mismatches are handled if bbox is passed a geodataframe (which it is).
+
+    # Read layer using the bbox option.
+    # CRS mismatches are handled if bbox is passed a geodataframe (which it is).
     bounding_box = gpd.GeoDataFrame({'geometry': box(*reference.bounds)}, index=[0], crs=reference.crs)
-    poly_all = gpd.read_file(shapefile_path, bbox = bounding_box)
+    poly_all = gpd.read_file(shapefile_path, bbox=bounding_box)
 
-    # Make sure features are present in bounding box area before projecting. Continue to next layer if features are absent.
+    # Make sure features are present in bounding box area before projecting.
+    # Continue to next layer if features are absent.
     if poly_all.empty:
         return
 
-    #Project layer to reference crs.
+    # Project layer to reference crs.
     poly_all_proj = poly_all.to_crs(reference.crs)
     # check if there are any lakes within our reference raster extent.
+    # If no features within reference raster extent, create a zero array of same shape as reference raster.
     if poly_all_proj.empty:
-        #If no features within reference raster extent, create a zero array of same shape as reference raster.
         poly_mask = np.zeros(reference.shape)
     else:
-        #Perform mask operation on the reference raster and using the previously declared geometry geoseries. Invert set to true as we want areas outside of poly areas to be False and areas inside poly areas to be True.
+        # Perform mask operation on the reference raster and using the previously declared geometry geoseries.
+        # Invert set to true as we want areas outside poly areas => False and areas inside poly areas => True.
         geometry = poly_all_proj.geometry
-        in_poly,transform,c = rasterio.mask.raster_geometry_mask(reference, geometry, invert = True)
-        #Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
-        poly_mask = np.where(in_poly == True, 1,0)
+        in_poly, transform, c = rasterio.mask.raster_geometry_mask(reference, geometry, invert=True)
+        # Write mask array, areas inside polys are set to 1 and areas outside poly are set to 0.
+        poly_mask = np.where(in_poly == True, 1, 0)
 
         # Filter depth_array by depth_threshold
         filtered_depth_array = np.where(depth_array > depth_threshold, depth_array, -1)
@@ -53,11 +58,12 @@ def check_deep_flooding(args):
         masked_depth_array = np.where(poly_mask == 1, filtered_depth_array, -1)
 
         if np.amax(masked_depth_array) > 0:
-
             file_handle = os.path.split(depth_grid_path)[1]
-    
-            checked_depth_raster = os.path.join(output_dir, "checked_" + str(depth_threshold) + "_" + file_handle)
-    
+
+            checked_depth_raster = os.path.join(
+                output_dir, "checked_" + str(depth_threshold) + "_" + file_handle
+            )
+
             print("Writing " + checked_depth_raster + "...")
             # Write output.
             with rasterio.Env():
@@ -68,43 +74,57 @@ def check_deep_flooding(args):
 
 
 if __name__ == '__main__':
-    
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Checks for deep flooding in a specified shapefile. Requires a directory of depth grids and a shapefile.')
-    parser.add_argument('-d','--depth-grid-dir',help='Name of directory containing outputs of depth outputs of inundation.py',required=True)
-    parser.add_argument('-s','--shapefile-path',help='Path to shapefile to be used as the overlay.',required=True)
-    parser.add_argument('-t','--depth-threshold',help='Depth in meters to use as checking threshold.',required=True)
-    parser.add_argument('-o', '--output-dir',help='The path to a directory to write the outputs. If not used, the inundation_review directory is used by default -> type=str',required=True, default="")
-    parser.add_argument('-j', '--job-number',help='The number of jobs',required=False,default=1)
-        
+    parser = argparse.ArgumentParser(
+        description='Checks for deep flooding in a specified shapefile. '
+        'Requires a directory of depth grids and a shapefile.'
+    )
+    parser.add_argument(
+        '-d',
+        '--depth-grid-dir',
+        help='Name of directory containing outputs of depth outputs of inundation.py',
+        required=True,
+    )
+    parser.add_argument(
+        '-s', '--shapefile-path', help='Path to shapefile to be used as the overlay.', required=True
+    )
+    parser.add_argument(
+        '-t', '--depth-threshold', help='Depth in meters to use as checking threshold.', required=True
+    )
+    parser.add_argument(
+        '-o',
+        '--output-dir',
+        help='The path to a directory to write the outputs. '
+        'If not used, the inundation_review directory is used by default -> type=str',
+        required=True,
+        default="",
+    )
+    parser.add_argument('-j', '--job-number', help='The number of jobs', required=False, default=1)
+
     args = vars(parser.parse_args())
-    
+
     depth_grid_dir = args['depth_grid_dir']
     shapefile_path = args['shapefile_path']
     depth_threshold = int(args['depth_threshold'])
     output_dir = args['output_dir']
     job_number = int(args['job_number'])
-    
+
     # Get list of files in depth_grid_dir.
     # Loop through files and determine which ones are depth grids, adding them to a list.
-    
+
     depth_grid_dir_list = os.listdir(depth_grid_dir)
     if not os.path.exists(output_dir):
         os.mkdir(output_dir)
-    
+
     procs_list = []
-    
+
     for f in depth_grid_dir_list:
         if 'depth' in f:
             full_f_path = os.path.join(depth_grid_dir, f)
-            
-#            check_deep_flooding([full_f_path, shapefile_path, depth_threshold, output_dir])
+
+            #            check_deep_flooding([full_f_path, shapefile_path, depth_threshold, output_dir])
             procs_list.append([full_f_path, shapefile_path, depth_threshold, output_dir])
-            
+
     # Multiprocess.
     with Pool(processes=job_number) as pool:
         pool.map(check_deep_flooding, procs_list)
-
-            
-            
-            
\ No newline at end of file
diff --git a/tools/code_standardizer/Dockerfile b/tools/code_standardizer/Dockerfile
deleted file mode 100755
index 0c58dd0a9..000000000
--- a/tools/code_standardizer/Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM python:3.8.5-slim-buster
-
-ENV PYTHONUNBUFFERED 1
-RUN mkdir -p /opt/code_standardizer
-WORKDIR /opt/code_standardizer
-
-COPY requirements.txt .
-RUN pip install -r requirements.txt --no-cache-dir
-COPY . /opt/code_standardizer
-
-RUN chmod +x /opt/code_standardizer/entrypoint.sh
-ENTRYPOINT ["/opt/code_standardizer/entrypoint.sh"]
diff --git a/tools/code_standardizer/entrypoint.sh b/tools/code_standardizer/entrypoint.sh
deleted file mode 100755
index 417a9a08d..000000000
--- a/tools/code_standardizer/entrypoint.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/sh
-
-umask 002
-cd /cahaba
-
-echo "Starting Code Standardizer"
-
-echo "Running Python Black..."
-black .
-
-echo "Running iSort..."
-isort --profile black .
-
-echo "Running Flake8..."
-flake8 .
-
-echo " ALL DONE!"
diff --git a/tools/code_standardizer/requirements.txt b/tools/code_standardizer/requirements.txt
deleted file mode 100755
index c8fd24694..000000000
--- a/tools/code_standardizer/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-black==21.9b0
-flake8==3.9.2
-isort==5.9.3
diff --git a/tools/combine_crosswalk_tables.py b/tools/combine_crosswalk_tables.py
index df2aabcf4..6251966c5 100644
--- a/tools/combine_crosswalk_tables.py
+++ b/tools/combine_crosswalk_tables.py
@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
 
-import os
-import glob
 import argparse
+import glob
+import os
+
 import pandas as pd
 
+
 def combine_crosswalk_tables(data_directory, output_filename):
     """
-    Combines all hydrotables from a run into a single crosswalk table with HUC, BranchID, HydroID, feature_id, and LakeID
+    Combines all hydrotables from a run into a single crosswalk table with:
+        HUC, BranchID, HydroID, feature_id, and LakeID
     """
 
     file_list = sorted(glob.glob(os.path.join(data_directory, '*', 'branches', '*', 'hydroTable_*.csv')))
@@ -15,9 +18,11 @@ def combine_crosswalk_tables(data_directory, output_filename):
     dfs = list()
     for filename in file_list:
         if os.path.exists(filename):
-            file_df = pd.read_csv(filename, usecols=['HUC', 'HydroID', 'feature_id', 'LakeID'], dtype={'HUC':str})
+            file_df = pd.read_csv(
+                filename, usecols=['HUC', 'HydroID', 'feature_id', 'LakeID'], dtype={'HUC': str}
+            )
             file_df.drop_duplicates(inplace=True)
-            file_df.rename(columns={'HUC':'huc8'}, inplace=True)
+            file_df.rename(columns={'HUC': 'huc8'}, inplace=True)
             file_df['BranchID'] = os.path.split(os.path.dirname(filename))[1]
 
             dfs.append(file_df)
@@ -31,10 +36,14 @@ def combine_crosswalk_tables(data_directory, output_filename):
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Combines hydrotables from HUC/branch into a single crosswalk table')
-    parser.add_argument('-d', '--data-directory', help='Data directory (name of run)', type=str, required=True)
+    parser = argparse.ArgumentParser(
+        description='Combines hydrotables from HUC/branch into a single crosswalk table'
+    )
+    parser.add_argument(
+        '-d', '--data-directory', help='Data directory (name of run)', type=str, required=True
+    )
     parser.add_argument('-o', '--output-filename', help='Filename for output', type=str, required=True)
 
     args = vars(parser.parse_args())
 
-    combine_crosswalk_tables(**args)
\ No newline at end of file
+    combine_crosswalk_tables(**args)
diff --git a/tools/compare_ms_and_non_ms_metrics.py b/tools/compare_ms_and_non_ms_metrics.py
index 8edabc0e2..0e90101d7 100644
--- a/tools/compare_ms_and_non_ms_metrics.py
+++ b/tools/compare_ms_and_non_ms_metrics.py
@@ -1,31 +1,33 @@
 #!/usr/bin/env python3
 
-import pandas as pd
-import numpy as np
-import os
-from tqdm import tqdm
 import argparse
+import os
 from concurrent.futures import ProcessPoolExecutor, as_completed
-                                   csi, mcc, tpr, far
-from tools.shared_variables import TEST_CASES_DIR, OUTPUTS_DIR
 from glob import glob
 from itertools import product
 
+import numpy as np
+import pandas as pd
+from tools_shared_functions import csi, far, mcc, tpr
+from tqdm import tqdm
 
-def Compare_ms_and_non_ms_areas():
-    
-    return(None)
-
+from tools.shared_variables import OUTPUTS_DIR, TEST_CASES_DIR
 
 
+def Compare_ms_and_non_ms_areas():
+    return None
 
-if __name__ == '__main__':
 
+if __name__ == "__main__":
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    #parser.add_argument('-c','--config',help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',required=True)
-    
+    parser = argparse.ArgumentParser(description="Caches metrics from previous versions of HAND.")
+    # parser.add_argument(
+    #     '-c',
+    #     '--config',
+    #     help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',
+    #     required=True,
+    # )
+
+    # args = vars(parser.parse_args())
 
-    #args = vars(parser.parse_args())
-    
     Compare_ms_and_non_ms_areas()
diff --git a/tools/compile_comp_stats.py b/tools/compile_comp_stats.py
deleted file mode 100755
index b2bb0fe08..000000000
--- a/tools/compile_comp_stats.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python3
-
-import pandas as pd
-import numpy as np
-from glob import glob, iglob
-import argparse
-import os
-
-########################################################
-'''
-Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-   a significant review and/or upgrade.
-'''
-
-########################################################
-
-
-def Compile_comp_stats(hydrofabric_dirs):
-    
-    all_minutes = []
-    all_models = []
-    all_hucs = []
-
-    file_iterator = get_log_files(hydrofabric_dirs)
-
-    for entry in file_iterator:
-        model, log = entry
-        log_file = open(log,'r')
-        
-        huc8code = log.split('/')[-1][0:8]
-
-        for line in log_file:
-            if 'wall clock' in line:
-                time_string = line.strip().split(' ')[-1]
-
-                time_string_split = time_string.split(':')
-
-                if len(time_string_split) == 2:
-                    minutes, seconds = time_string_split
-                    
-                    total_minutes = float(minutes)+float(seconds)/60
-
-                if len(time_string_split) == 3:
-                    hours, minutes, seconds = time_string_split 
-                
-                    total_minutes = float(hours)*60+float(minutes)+float(seconds)/60
-
-                all_minutes.append(total_minutes)
-                all_models.append(model)
-                all_hucs.append(huc8code)
-
-    df = pd.DataFrame({'Minutes':all_minutes,'Model':all_models, 'HUC': all_hucs})
-
-    total_per_huc = df.pivot_table(values='Minutes',index=['Model','HUC'],aggfunc=sum)
-
-    print(total_per_huc.pivot_table(values='Minutes',index='Model',aggfunc=[np.mean,np.median,np.sum]))
-
-
-
-def get_log_files(hydrofabric_dirs):
-
-    for hydrofabric_dir in hydrofabric_dirs:
-        log_dir = os.path.join(hydrofabric_dir,'logs')
-
-        if os.path.join(log_dir,'branch'):
-            model = 'GMS'
-        if '_MS' in log_dir:
-            model = 'MS'
-        if '_FR' in log_dir:
-            model = 'FR'
-
-        for fn in iglob(os.path.join(log_dir,'**','[0-9]*.log'),recursive=True):
-            yield(model,fn)
-
-
-if __name__ == '__main__':
-
-    ########################################################
-    '''
-    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-    a significant review and/or upgrade.
-    '''
-
-    ########################################################
-
-
-    # parse arguments
-    parser = argparse.ArgumentParser(description='Get Comp Stats')
-    parser.add_argument('-y','--hydrofabric_dirs', help='Directory path to FIM hydrofabric by processing unit', required=True,nargs='+')
-    
-
-    Compile_comp_stats( **vars(parser.parse_args()) )
diff --git a/tools/compile_computational_stats.py b/tools/compile_computational_stats.py
deleted file mode 100755
index e9f10cda4..000000000
--- a/tools/compile_computational_stats.py
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import pandas as pd
-import numpy as np
-from glob import iglob
-from os.path import join
-
-# desired output for branches
-# dataframe columns: HUC, branch_id, exit status, ,time, ram, 
-
-########################################################
-'''
-Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-   a significant review and/or upgrade.
-'''
-
-########################################################
-
-
-def compile_summary(gms_output_dir,ouput=None):
-
-    unit_summary = join(gms_output_dir,logs, 'summary_gms_unit.log')
-    branch_summary = join(gms_output_dir,logs, 'summary_gms_branch.log')
-
-    unit_summary = pd.read_csv(unit_summary,sep='\t')
-    branch_summary = pd.read_csv(branch_summary,sep='\t')
-
-
-if __name__ == '__main__':
-
-    ########################################################
-    '''
-    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-    a significant review and/or upgrade.
-    '''
-
-    ########################################################
-
-
-    parser = argparse.ArgumentParser(description='Create stream network level paths')
-    parser.add_argument('-d','--gms-output-dir', help='Input stream network', required=True)
-    parser.add_argument('-o','--output', help='Input stream network', required=True)
-    
-    args = vars(parser.parse_args())
-
-    compile_summary(**args)
-
diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py
index 915b96e3e..695adb075 100644
--- a/tools/composite_inundation.py
+++ b/tools/composite_inundation.py
@@ -1,20 +1,22 @@
 #!/usr/bin/env python3
-import os, argparse, copy
-import json
-import rasterio
-import numpy as np
-import pandas as pd
-
-from datetime import datetime
-from multiprocessing import Pool
+import argparse
 
 # from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
 import concurrent.futures as cf
-from tqdm import tqdm
+import copy
+import json
+import os
+from datetime import datetime
+from multiprocessing import Pool
 
+import numpy as np
+import pandas as pd
+import rasterio
+from inundate_mosaic_wrapper import produce_mosaicked_inundation
 from inundation import inundate
 from mosaic_inundation import Mosaic_inundation
-from inundate_mosaic_wrapper import produce_mosaicked_inundation
+from tqdm import tqdm
+
 from utils.shared_functions import FIM_Helpers as fh
 from utils.shared_variables import elev_raster_ndv
 
@@ -62,7 +64,7 @@ def inundate_huc(
         output_raster_name = fh.append_id_to_file_name(output_raster_name, [self.huc, self.model])
 
         if verbose:
-            print(f"... Creating an inundation map for the FIM4" f" configuration for HUC {self.huc}...")
+            print(f"... Creating an inundation map for the FIM4 configuration for HUC {self.huc}...")
 
         if self.model in ["fr", "ms"]:
             if self.model == "ms":
@@ -72,8 +74,7 @@ def inundate_huc(
             rem = os.path.join(source_huc_dir, "rem_zeroed_masked.tif")
             catchments = os.path.join(source_huc_dir, "gw_catchments_reaches_filtered_addedAttributes.tif")
             catchment_poly = os.path.join(
-                source_huc_dir,
-                "gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg",
+                source_huc_dir, "gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg"
             )
             hydro_table = os.path.join(source_huc_dir, "hydroTable.csv")
 
@@ -210,7 +211,8 @@ def __validate_args(args):
             fim_dir_fr : str
                 Path to FR FIM directory. This should be an output directory from `fim_run.sh`.
             gms_dir : str
-                Path to FIM4 GMS directory. This should be an output directory from `gms_run_unit, then gms_run_branch`.
+                Path to FIM4 GMS directory. This should be an output directory from
+                    `gms_run_unit, then gms_run_branch`.
             huc: str, optional
                 If this value comes in, it shoudl be a single huc value. If it does not exist,
                 we use all hucs in the give source directories.
@@ -219,7 +221,8 @@ def __validate_args(args):
             composite_output_dir : str
                 Folder path to write outputs. It will be created if it does not exist.
             output_name : str, optional
-                Name for output raster. If not specified, by default the raster will be named 'inundation_composite_{flows_root}.tif'.
+                Name for output raster. If not specified, by default the raster will be named:
+                    'inundation_composite_{flows_root}.tif'.
             is_bin_raster : bool, optional
                 Flag to create binary raster as output.
             num_workers_huc : int, optional
@@ -284,9 +287,7 @@ def __validate_args(args):
                     "exceeds your machine's available CPU count of {} minus one. "
                     "Please lower the num_workers_huc or num_workers_branches"
                     "values accordingly.".format(
-                        args["num_workers_huc"],
-                        total_cpus_available,
-                        args["num_workers_branches"],
+                        args["num_workers_huc"], total_cpus_available, args["num_workers_branches"]
                     )
                 )
 
@@ -306,7 +307,7 @@ def __validate_args(args):
                 output_file_name_split = os.path.split(args["output_name"])
                 if args["output_name"] == output_file_name_split[0]:
                     raise ValueError(
-                        """If submitting the -n (output file name), please ensure 
+                        """If submitting the -n (output file name), please ensure
                                         it has no pathing. You can also leave it blank if you like."""
                     )
 
@@ -328,7 +329,7 @@ def __validate_args(args):
 
             # make combined huc list, NOTE: not all source dirs will have the same huc folders
             huc_list = set()
-            if args["huc"] != None:
+            if args["huc"] is not None:
                 if (len(args["huc"]) != 8) or (not args["huc"].isnumeric()):
                     raise ValueError("Single huc value (-u arg) was submitted but appears invalid")
                 else:
@@ -387,7 +388,10 @@ def __validate_args(args):
     def hydroid_to_binary(hydroid_raster_filename):
         # Converts hydroid positive/negative grid to 1/0
         # to_bin = lambda x: np.where(x > 0, 1, np.where(x == 0, -9999, 0))
-        to_bin = lambda x: np.where(x > 0, 1, np.where(x != -9999, 0, -9999))
+        # to_bin = lambda x: np.where(x > 0, 1, np.where(x != -9999, 0, -9999))
+        def to_bin(x):
+            return np.where(x > 0, 1, np.where(x != -9999, 0, -9999))
+
         hydroid_raster = rasterio.open(hydroid_raster_filename)
         profile = hydroid_raster.profile  # get profile for new raster creation later on
         profile["nodata"] = -9999
@@ -409,13 +413,29 @@ def hydroid_to_binary(hydroid_raster_filename):
     Notice: arg keys and values for some of the variants
     --------
     a) ms and fr (single huc)
-    python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -fr /outputs/inundation_test_1_FIM3_fr -u /data/inputs/huc_lists/include_huc8.lst -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
+    python3 /foss_fim/tools/composite_inundation.py
+        -ms /outputs/inundation_test_1_FIM3_ms
+        -fr /outputs/inundation_test_1_FIM3_fr
+        -u /data/inputs/huc_lists/include_huc8.lst
+        -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv
+        -o /outputs/inundation_test_1_comp/
+        -n test_inundation.tif
 
     a) ms and gms (all hucs in each folder)
-    python3 /foss_fim/tools/composite_inundation.py -ms /outputs/inundation_test_1_FIM3_ms -gms /outputs/inundation_test_1_gms -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv -o /outputs/inundation_test_1_comp/ -c -jh 3 -jb 20
+    python3 /foss_fim/tools/composite_inundation.py
+        -ms /outputs/inundation_test_1_FIM3_ms
+        -gms /outputs/inundation_test_1_gms
+        -f /data/test_cases/nws_test_cases/validation_data_nws/13090001/rgdt2/moderate/ahps_rgdt2_huc_13090001_flows_moderate.csv
+        -o /outputs/inundation_test_1_comp/
+        -c -jh 3 -jb 20
 
     b) fr and gms (single huc)
-    python3 /foss_fim/tools/composite_inundation.py -fr /outputs/inundation_test_1_FIM3_fr -gms /outputs/inundation_test_1_gms -u 13090001 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv -o /outputs/inundation_test_1_comp/ -n test_inundation.tif
+    python3 /foss_fim/tools/composite_inundation.py
+        -fr /outputs/inundation_test_1_FIM3_fr
+        -gms /outputs/inundation_test_1_gms
+        -u 13090001 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv
+        -o /outputs/inundation_test_1_comp/
+        -n test_inundation.tif
     """
 
     # parse arguments
@@ -454,18 +474,9 @@ def hydroid_to_binary(hydroid_raster_filename):
     )
     parser.add_argument("-f", "--flows-file", help="File path of flows csv.", required=True)
     parser.add_argument(
-        "-o",
-        "--composite-output-dir",
-        help="Folder to write Composite Raster output.",
-        required=True,
-    )
-    parser.add_argument(
-        "-n",
-        "--output-name",
-        help="File name for output(s).",
-        default=None,
-        required=False,
+        "-o", "--composite-output-dir", help="Folder to write Composite Raster output.", required=True
     )
+    parser.add_argument("-n", "--output-name", help="File name for output(s).", default=None, required=False)
     parser.add_argument(
         "-b",
         "--is-bin-raster",
@@ -477,7 +488,8 @@ def hydroid_to_binary(hydroid_raster_filename):
     parser.add_argument(
         "-jh",
         "--num-workers-huc",
-        help="Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.",
+        help="Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply"
+        "to no more than one less than the CPU count of the machine.",
         required=False,
         default=1,
         type=int,
@@ -485,7 +497,8 @@ def hydroid_to_binary(hydroid_raster_filename):
     parser.add_argument(
         "-jb",
         "--num-workers-branches",
-        help="Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply to no more than one less than the CPU count of the machine.",
+        help="Number of processes to use for Branch scale operations. HUC and Batch job numbers should multiply"
+        "to no more than one less than the CPU count of the machine.",
         required=False,
         default=1,
         type=int,
@@ -499,12 +512,7 @@ def hydroid_to_binary(hydroid_raster_filename):
         action="store_true",
     )
     parser.add_argument(
-        "-v",
-        "--verbose",
-        help="Show additional outputs.",
-        required=False,
-        default=False,
-        action="store_true",
+        "-v", "--verbose", help="Show additional outputs.", required=False, default=False, action="store_true"
     )
 
     # Extract to dictionary and assign to variables.
diff --git a/tools/consolidate_metrics.py b/tools/consolidate_metrics.py
deleted file mode 100755
index 0b470631d..000000000
--- a/tools/consolidate_metrics.py
+++ /dev/null
@@ -1,339 +0,0 @@
-#!/usr/bin/env python3
-
-import pandas as pd
-import os
-import re
-import json
-import numpy as np
-from glob import glob
-from tqdm import tqdm
-import argparse
-from collections import defaultdict
-from tools_shared_variables import TEST_CASES_DIR,\
-                             PREVIOUS_FIM_DIR,\
-                             OUTPUTS_DIR,\
-                             INPUTS_DIR,\
-                             AHPS_BENCHMARK_CATEGORIES
-from tools_shared_functions import csi,far,tpr,mcc
-
-# set display options
-pd.set_option('display.max_rows', None)
-pd.set_option('display.max_columns', None)
-
-
-########################################################
-'''
-Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-   a significant review and/or upgrade.
-'''
-
-########################################################
-
-def Consolidate_metrics( benchmarks=['all'],versions=['all'],
-                         zones=['total_area'],matching_hucs_only=True,
-                         metrics_output_csv=None,
-                         impute_missing_ms=False
-                       ):
-
-    """ Consolidates metrics into single dataframe """
-
-    if isinstance(benchmarks,str):
-        benchmarks = [benchmarks]
-    elif isinstance(benchmarks,list):
-        pass
-    else:
-        benchmarks = list(benchmarks)
-    
-    if isinstance(versions,str):
-        versions = [versions]
-    elif isinstance(benchmarks,list):
-        pass
-    else:
-        versions = list(versions)
-
-    if isinstance(zones,str):
-        zones = [zones]
-    elif isinstance(zones,list):
-        pass
-    else:
-        zones = list(zones)
-    
-
-    # loop through benchmarks
-    consolidated_metrics_df = [ f for f in return_dataframe_for_benchmark_source(benchmarks,zones) ]
-    
-    # concat
-    consolidated_metrics_df = pd.concat(consolidated_metrics_df,ignore_index=True)
-
-    # find matching rows
-    consolidated_metrics_df = find_matching_rows_by_attribute_value(consolidated_metrics_df,'version',versions)
-
-    if impute_missing_ms:
-        consolidated_metrics_df = impute_missing_MS_with_FR(consolidated_metrics_df)
-
-    if metrics_output_csv is not None:
-        consolidated_metrics_df.to_csv(metrics_output_csv, index=False)
-
-    """
-    #print(consolidated_metrics_df.columns);exit()
-    consolidated_metrics_pivot = pd.pivot_table(
-                                                consolidated_metrics_df,
-                                                values=['FP_area_km2','FN_area_km2','TP_area_km2','contingency_tot_area_km2','obsPositive_area_km2'],
-                                                columns=['extent_config'],
-                                                index=['magnitude'], 
-                                                aggfunc=np.sum
-                                               )
-    print(consolidated_metrics_pivot);exit()"""
-
-    consolidated_secondary_metrics = pivot_metrics(consolidated_metrics_df)
-    print(consolidated_secondary_metrics)
-
-    return(consolidated_metrics_df,consolidated_secondary_metrics)
-
-
-def impute_missing_MS_with_FR(consolidated_metrics_df):
-
-
-    # make sure you have one version per extent_config
-    fr_extent_config = 'FR'
-    ms_extent_config = 'MS'
-    extent_config = [fr_extent_config, ms_extent_config]
-    
-    indices_of_ec = {ec : consolidated_metrics_df.index[consolidated_metrics_df.loc[:,'extent_config'] == ec] for ec in extent_config }
-
-    unique_hucs_dict = dict()
-    for ec in extent_config:
-
-        unique_version = consolidated_metrics_df.loc[ indices_of_ec[ec],'version'].unique()
-
-        if len(unique_version) > 1:
-            raise ValueError(f"{ec} version has more than one extent. Only pass one version per extent configuration.")
-        unique_hucs = set(consolidated_metrics_df.loc[indices_of_ec[ec],'huc'].unique())
-
-        unique_hucs_dict[ec] = unique_hucs
-
-
-    unique_hucs = unique_hucs_dict[fr_extent_config] - unique_hucs_dict[ms_extent_config]
-    
-    # copy those rows over with extent_config set to MS
-    unique_fr_mask = consolidated_metrics_df.loc[ indices_of_ec['FR'],'huc'].isin(unique_hucs)
-    unique_fr_indices = unique_fr_mask.index[unique_fr_mask]
-
-    fr_entries = consolidated_metrics_df.loc[unique_fr_indices,:].reset_index(drop=True)
-    fr_entries.loc[:,'extent_config'] = "MS"
-    fr_entries.loc[:,'version'] = consolidated_metrics_df.loc[ indices_of_ec['MS'], 'version'].unique()
-
-    consolidated_metrics_df = pd.concat((consolidated_metrics_df,fr_entries)).reset_index(drop=True)
-
-    return(consolidated_metrics_df)
-
-
-
-def pivot_metrics(consolidated_metrics_df):
-
-    ''' Pivots metrics to provide summary of results '''
-
-    # pivot out
-    consolidated_metrics_pivot = pd.pivot_table(
-                                                consolidated_metrics_df,
-                                                values=['true_positives_count','false_positives_count',
-                                                        'false_negatives_count','true_negatives_count'],
-                                                index=['magnitude','extent_config','version'], 
-                                                aggfunc=sum
-                                               )
-
-    
-    return( apply_metrics_to_pivot_table(consolidated_metrics_pivot) )
-
-
-
-def apply_metrics_to_pivot_table(consolidated_metrics_pivot):
-
-    # metrics to run
-    metrics_functions = { 'CSI': csi , 'TPR' : tpr, 'FAR' : far, 'MCC' : mcc }
-    #metrics_functions = { 'CSI': csi }
-    
-    def row_wise_function(df,mn,mf):
-        
-        ''' applies function along rows of dataframes '''
-
-        return( 
-                pd.Series(
-                          mf( 
-                             df['true_positives_count'],
-                             df['false_positives_count'],
-                             df['false_negatives_count'],
-                             df['true_negatives_count']
-                            ),
-                          name=mn
-                         )
-              )
-
-    column_generator = (  row_wise_function(consolidated_metrics_pivot,met_name,met_func) 
-                          for met_name,met_func in metrics_functions.items() 
-                       )
-
-    consolidated_secondary_metrics = pd.concat(column_generator,axis=1)
-
-    return(consolidated_secondary_metrics)
-
-
-def find_matching_rows_by_attribute_value(df,attributes,matches):
-    
-    if (len(matches) == 1) & ('all' in matches):
-        return(df)
-    else:
-        df = df.loc[df.loc[:,attributes].isin(matches)]
-
-    return(df)
-
-
-def return_dataframe_for_benchmark_source(benchmarks,zones=['total_area']):
-
-    """ returns a dataframe of results given a name for a benchmark source """
-
-    benchmark_function_dict = { 
-                                'ble' : consolidate_metrics('ble',zones),
-                                'ifc' : consolidate_metrics('ifc',zones)
-                              }
-
-    # if all cycle through all functions, else go through selection
-    if (len(benchmarks)== 1) & ('all' in benchmarks):
-        
-        for f in benchmark_function_dict.values():
-            yield(f)
-
-    else:
-
-        for b in benchmarks:
-            try:
-                yield( benchmark_function_dict[b] )
-            except KeyError: 
-                raise ValueError(f"Benchmark '{b}' not supported. "\
-                                 f"Pass 'all' or one of these"\
-                                 f"{list(benchmark_function_dict.keys())}")
-
-
-def consolidate_metrics(benchmark,zones=['total_area']):
-
-    """ consolidates ble metrics """
-
-    # get filenames for metrics for ble
-    files_to_consolidate = list()
-    for zone in zones:
-        file_pattern_to_glob = os.path.join(TEST_CASES_DIR,f'{benchmark}_test_cases','**',f'{zone}_stats.csv')
-        files_to_consolidate.extend(glob(file_pattern_to_glob, recursive=True))
-
-    # make a metrics dataframe generator
-    metrics_df_generator = metrics_data_frame_generator_from_files(files_to_consolidate)
-
-    # concat said generator of files
-    metrics_df = pd.concat(metrics_df_generator, ignore_index=True)
-
-    # make a meta-data
-    metadata_df = metadata_dataframe_from_file_names_for_ble(files_to_consolidate)
-    
-    # concat metrics and metadata dataframes
-    return(   
-            pd.concat( (metadata_df,metrics_df), axis=1) 
-          )
-
-
-def metrics_data_frame_generator_from_files(file_names):
-
-    """ Reads in metrics dataframes from filenames """
-
-    for f in file_names:
-        yield pd.read_csv(f,index_col=0).T.reset_index(drop=True)
-
-
-def metadata_dataframe_from_file_names_for_ble(file_names):
-    
-    """ Makes a dataframe for meta-data """
-
-    file_name_index_dict = { 
-                            'benchmark_source' : parse_benchmark_source(file_names),
-                            'version' : parse_version_name(file_names),
-                            'huc' : parse_huc(file_names),
-                            'magnitude' : parse_magnitude(file_names),
-                            'extent_config' : parse_eval_metadata(file_names,'model'),
-                            'calibrated' : parse_eval_metadata(file_names,'calibrated')
-                           }
-
-    return( 
-            pd.DataFrame(file_name_index_dict)
-          )
-
-
-def parse_benchmark_source(file_names):
-
-    """ Parses benchmark source """
-
-    for f in file_names:
-        yield f.split('/')[3].split('_')[0]
-
-
-def parse_version_name(file_names):
-
-    """ Parses version name """
-
-    for f in file_names:
-        yield f.split('/')[6]
-
-
-def parse_magnitude(file_names):
-
-    """ Parses magnitude """
-
-    for f in file_names:
-        yield f.split('/')[7]
-
-
-def parse_huc(file_names):
-
-    """ parses huc """
-
-    for f in file_names:
-        yield f.split('/')[4].split('_')[0]
-
-
-def parse_eval_metadata(file_names,metadata_field):
-   
-    """ parsing eval metadata json files """
-
-    for f in file_names:
-        root_dir = os.path.abspath(os.sep)
-        dir_of_metadata = f.split('/')[0:7]
-        eval_metadata_filepath = os.path.join(root_dir,*dir_of_metadata,'eval_metadata.json')
-        
-        # read eval data, if no file write None 
-        try:
-            with open(eval_metadata_filepath) as fObj:
-                eval_metadata = json.load(fObj)
-        except FileNotFoundError:
-            yield None
-        else:
-            yield eval_metadata[metadata_field]
-
-
-if __name__ == '__main__':
-
-    ########################################################
-    '''
-    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
-    a significant review and/or upgrade.
-    '''
-
-    ########################################################
-
-    # Parse arguments.
-    parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-b','--benchmarks',help='Allowed benchmarks', required=False, default='all', nargs="+")
-    parser.add_argument('-v','--versions',help='Allowed versions', required=False, default='all', nargs="+")
-    parser.add_argument('-z','--zones',help='Allowed zones', required=False, default='total_area', nargs="+")
-    parser.add_argument('-o','--metrics-output-csv',help='File path to outputs csv', required=False, default=None)
-    parser.add_argument('-i','--impute-missing_ms',help='Imputes FR metrics in HUCS with no MS. Only supports one version per extent config', required=False, action='store_true',default=False)
-
-    args = vars(parser.parse_args())
-
-    Consolidate_metrics(**args)
diff --git a/tools/copy_test_case_folders.py b/tools/copy_test_case_folders.py
deleted file mode 100644
index c2e16fe23..000000000
--- a/tools/copy_test_case_folders.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import argparse
-import shutil
-import sys
-
-# importing python folders in other direcories
-sys.path.append('/foss_fim/src/')
-import aggregate_branch_lists as agg
-
-#######
-#  Feb 16, 2023: This file is likely now deprecated. If it is used again, the aggregate_branch_lists file will need to 
-#    be reconsiiled as a newer one exists in the src directory with a similar name.
-#######
-
-def copy_folders(folder_name_list,
-                source_dir,
-                target_dir,
-                create_fim_input_list=False):
-
-    '''
-    Summary: Scans the source_directory looking for huc values from the huc list. Once found,
-        the huc directory is copied to the target directory. All recusive files/folders wil be copied.
-        
-        Line items each be on a new line  (ie "\n")
-        
-        ***** Note: The tool will still work, if the list is not a list of hucs. 
-              It can be just a list of folder names.
-    Input:
-        - folder_name_list: A file and path to a .txt or .lst file with a list of line delimited huc numbers.
-        - source_dir: The root folder where the huc (or named) folders reside.
-        - target_dir: The root folder where the huc folders will be copied to. Note. All contents of 
-             each huc folder, including branch folders if applicable, will be copied, in the extact
-             structure as the source directory. Note: The target folder need not pre-exist. 
-        - create_fim_input_list: If this flag is set to True, after coping the folders, the 
-            "aggregate_branch_lists.py" file will be called in order to make the fim_input.csv file.
-            The fim_input.csv is required for futher processing such as reprocessing branchs or set up
-            for test cases.
-    Output:
-        - A copy of huc directories (or named folders) as listed in the folder_name_list.
-    '''
-
-    if (not os.path.exists(folder_name_list)):
-        raise FileNotFoundError(f"Sorry. The file {folder_name_list} does not exist")
-
-    if (not os.path.exists(source_dir)):
-        raise NotADirectoryError(f"Sorry. Source folder of {source_dir} does not exist")
-
-    with open(folder_name_list, "r") as fp:
-        raw_folder_names = fp.read().split("\n")
-
-    # split on new line can add an extra row of a blank value if a newline char exists on the end.
-    # Some lines may have extra spaces, or dups. It is ok if the value is not necessarily a huc
-    folder_names = set() # this will ensure unique names
-    for folder_name in raw_folder_names:
-        folder_name = folder_name.strip()
-        if (folder_name) != '':
-            folder_names.add(folder_name)
-
-    sorted_folder_names = sorted(folder_names)
-
-    print(f"{str(len(sorted_folder_names))} folders to be copied")
-    ctr = 0
-    
-    for folder_name in sorted_folder_names:
-        src_folder = os.path.join(source_dir, folder_name)
-        if not os.path.exists(src_folder):
-            print(f"source folder not found: {src_folder}")
-        else:
-            target_folder = os.path.join(target_dir, folder_name)
-            print(f"coping folder : {target_folder}")
-            shutil.copytree(src_folder, target_folder, dirs_exist_ok=True)
-            ctr+=1
-
-    print(f"{str(ctr)} folders have been copied to {target_dir}")
-
-    if create_fim_input_list == True:
-        # call this code, which scans each huc (unit) directory looking for the branch_id.lst
-        # and adds them together to create the fim_inputs.csv file
-        # Note: folder_name_list needs to be a huc list to work)
-        agg.aggregate_branch_lists(folder_name_list, target_dir, "fim_inputs.csv")
-        print("fim_inputs.csv created")
-    
-
-if __name__ == '__main__':
-
-# Remember: This is for pulling out only folders that are related to a huc list (such as an alpha test list)
-#   and it has to be run on each root folder, one at a time (for now. aka.. no wildcards)
-
-# Sample Usage: 
-#python /foss_fim/tools/copy_test_case_folders.py -f /data/inputs/huc_lists/huc_list_for_alpha_tests_20220420.lst -s /outputs/copy_test_synth/ -t /outputs/copy_test_synth_combined -a
-
-#  NOTE the 'a' at the end meaning go ahead create the gms_input.csv. This is normally
-# left for the last folder to be copied over.
-
-    parser = argparse.ArgumentParser(description='Based on the huc list parameter, ' \
-                        'find and copy the full huc (or other) directories.')
-
-    parser.add_argument('-f','--folder_name_list',
-                            help='file and path to the huc list. Note: The list does not ' \
-                                'necessarily be a list of huc, just a list of unique values',
-                            required=True)                            
-    parser.add_argument('-s','--source_dir', 
-                            help='Source folder to be scanned for unique folders',
-                            required=True)
-    parser.add_argument('-t','--target_dir',
-                            help='Target folder where the folders will be copied to',
-                            required=True)
-
-    parser.add_argument('-a','--create_fim_input_list',
-                            help='Create the fim_input.csv list after copying',
-                            required=False, default=False, action='store_true')
-
-    args = vars(parser.parse_args())
-
-    copy_folders(**args)
diff --git a/tools/create_flow_forecast_file.py b/tools/create_flow_forecast_file.py
new file mode 100755
index 000000000..3512074e7
--- /dev/null
+++ b/tools/create_flow_forecast_file.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+
+import geopandas as gpd
+
+
+def create_flow_forecast_file(
+    ble_geodatabase,
+    nwm_geodatabase,
+    output_parent_dir,
+    ble_xs_layer_name='XS',
+    ble_huc_layer_name='S_HUC_Ar',
+    ble_huc_id_field='HUC_CODE',
+    nwm_stream_layer_name='RouteLink_FL_2020_04_07',
+    nwm_feature_id_field='ID',
+):
+    '''
+    This function will create a forecast flow file using ble data.
+    It will import the ble XS layer and will intersect it with a nwm streams layer.
+    It will the perform an intersection with the BLE XS layer and the nwm river layer.
+    It will then calculate the median flow for the 100 and 500 year events for each nwm segment and
+        convert flow from CFS to CMS.
+    Individual forecast files (.csv) will be created  with two columns (nwm segment ID and Flow (CMS))
+        for the 100 year and the 500 year event.
+    Flow field names are set to the default field names within the BLE submittal.
+    As differing versions of the NWM river layer have different names for the ID field, this field will need
+        to be specified as an argument to the function.
+    Output flow forecast files will be automatically named to be compatible with current FIM code and
+        will be written out in specific folder structures.
+
+    Parameters
+    ----------
+    ble_geodatabase : STRING
+        Path to BLE geodatabase.
+    nwm_geodatabase : STRING
+        Path to nwm geodatabase.
+    output_parent_dir : STRING
+        Output parent directory of output. Flow files will be output to subdirectories within parent directory.
+    ble_xs_layer_name : STRING
+       The cross section layer in the ble geodatabase to be imported. Default is 'XS' (sometimes it is 'XS_1D')
+    ble_huc_layer_name : STRING
+       The huc layer in the ble geodatabase.  Default is 'S_HUC_Ar' (sometimes it is 'S_HUC_ar' )
+    ble_huc_id_field : STRING
+       The attribute field within the ble_huc_layer_name containing the huc code. Default is 'HUC_CODE'.
+       Assumes only 1 unique code.
+    nwm_stream_layer_name : STRING
+       The stream centerline layer name (or partial layer name) for the NWM geodatabase.
+       Default is 'RouteLink_FL_2020_04_07'.
+    nwm_feature_id_field : STRING
+       The feature id of the nwm segments.  Default is 'ID' (applicable if nwmv2.1 is used)
+    Returns
+    -------
+    None.
+
+    '''
+    # Read the ble xs layer into a geopandas dataframe.
+    xs_layer = gpd.read_file(ble_geodatabase, layer=ble_xs_layer_name)
+
+    # Read ble huc layer into a geopandas dataframe and extract the huc code.
+    # By default it assumes only one HUC in the layer (typically always the case).
+    huc_layer = gpd.read_file(ble_geodatabase, layer=ble_huc_layer_name)
+    [huc] = huc_layer[ble_huc_id_field].unique()
+
+    # Read in the NWM stream layer into a geopandas dataframe using the bounding box option based on
+    #   the extents of the BLE XS layer.
+    nwm_river_layer = gpd.read_file(nwm_geodatabase, bbox=xs_layer, layer=nwm_stream_layer_name)
+
+    # Make sure xs_layer is in same projection as nwm_river_layer.
+    xs_layer_proj = xs_layer.to_crs(nwm_river_layer.crs)
+
+    # Perform an intersection of the BLE layers and the NWM layers, using the keep_geom_type set to
+    #   False produces a point output.
+    intersection = gpd.overlay(xs_layer_proj, nwm_river_layer, how='intersection', keep_geom_type=False)
+
+    ## Create the flow forecast files
+    # Define fields containing flow (typically these won't change for BLE)
+    flow_fields = ['E_Q_01PCT', 'E_Q_0_2PCT']
+
+    # Define return period associated with flow_fields (in same order as flow_fields).
+    # These will also serve as subdirectory names.
+    return_period = ['100yr', '500yr']
+
+    # Conversion factor from CFS to CMS
+    dischargeMultiplier = 0.3048**3
+
+    # Write individual flow csv files
+    for i, flow in enumerate(flow_fields):
+        # Write dataframe with just ID and single flow event
+        forecast = intersection[[nwm_feature_id_field, flow]]
+
+        # Rename field names and re-define datatypes
+        forecast = forecast.rename(columns={nwm_feature_id_field: 'feature_id', flow: 'discharge'})
+        forecast = forecast.astype({'feature_id': int, 'discharge': float})
+
+        # Calculate median flow per feature id
+        forecast = forecast.groupby('feature_id').median()
+        forecast = forecast.reset_index(level=0)
+
+        # Convert CFS to CMS
+        forecast['discharge'] = forecast['discharge'] * dischargeMultiplier
+
+        # Set paths and write file
+        output_dir = os.path.join(output_parent_dir, huc)
+        dir_of_csv = os.path.join(output_dir, return_period[i])
+        os.makedirs(dir_of_csv, exist_ok=True)
+        path_to_csv = os.path.join(dir_of_csv, "ble_huc_{}_flows_{}.csv".format(huc, return_period[i]))
+        forecast.to_csv(path_to_csv, index=False)
+
+
+if __name__ == '__main__':
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Produce forecast flow files from BLE datasets')
+    parser.add_argument(
+        '-b',
+        '--ble-geodatabase',
+        help='BLE geodatabase (.gdb file extension). Will look for layer with "XS" in name.'
+        'It is assumed the 100 year flow field is "E_Q_01PCT" and the 500 year flow field is "E_Q_0_2_PCT"'
+        'as these are the default field names.',
+        required=True,
+    )
+    parser.add_argument(
+        '-n', '--nwm-geodatabase', help='NWM geodatabase (.gdb file extension).', required=True
+    )
+    parser.add_argument(
+        '-o',
+        '--output-parent-dir',
+        help='Output directory where forecast files will be stored. Two subdirectories are created (100yr and 500yr)'
+        'and in each subdirectory a forecast file is written',
+        required=True,
+    )
+    parser.add_argument(
+        '-xs',
+        '--ble-xs-layer-name',
+        help='BLE cross section layer. Default layer is "XS" (sometimes it is "XS_1D").',
+        required=False,
+        default='XS',
+    )
+    parser.add_argument(
+        '-hu',
+        '--ble-huc-layer-name',
+        help='BLE huc layer. Default layer is "S_HUC_Ar" (sometimes it is "S_HUC_ar").',
+        required=False,
+        default='S_HUC_Ar',
+    )
+    parser.add_argument(
+        '-huid',
+        '--ble-huc-id-field',
+        help='BLE id field in the ble-huc-layer-name. Default field is "HUC_CODE".',
+        required=False,
+        default='HUC_CODE',
+    )
+    parser.add_argument(
+        '-l',
+        '--nwm-stream-layer-name',
+        help='NWM streams layer. Default layer is "RouteLink_FL_2020_04_07")',
+        required=False,
+        default='RouteLink_FL_2020_04_07',
+    )
+    parser.add_argument(
+        '-f',
+        '--nwm-feature-id-field',
+        help='id field for nwm streams. Not required if NWM v2.1 is used (default id field is "ID")',
+        required=False,
+        default='ID',
+    )
+    # Extract to dictionary and assign to variables.
+    args = vars(parser.parse_args())
+    # Run create_flow_forecast_file
+    create_flow_forecast_file(**args)
diff --git a/tools/cygnss_preprocessing.py b/tools/cygnss_preprocessing.py
deleted file mode 100755
index 5d4450834..000000000
--- a/tools/cygnss_preprocessing.py
+++ /dev/null
@@ -1,242 +0,0 @@
-#!/usr/bin/env python3
-
-from shapely.geometry import box
-import datetime
-import numpy as np
-import pandas as pd
-import geopandas as gpd
-import argparse
-from datetime import datetime
-from make_boxes_from_bounds import find_hucs_of_bounding_boxes
-import requests
-from concurrent.futures import ThreadPoolExecutor,as_completed
-import os
-from tqdm import tqdm
-from foss_fim.tools.inundation import read_nwm_forecast_file
-
-
-##################################
-##
-## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
-## Might want to be kept for possible re-use at a later time?
-##
-##################################
-
-
-def cygnss_preprocessing(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',forecast_output_file=None,retrieve=True,workers=6,download_directory=None, daily_mean_forecast_files=None):
-
-    _, bounding_boxes = find_hucs_of_bounding_boxes(bounding_boxes_file,wbd=wbd,projection_of_boxes=projection_of_boxes,wbd_layer=wbd_layer)
-    
-    # load bounding box file
-    bounding_boxes['event_date'] = pd.to_datetime(bounding_boxes['event_date'],utc=True)
-    bounding_boxes.reset_index(drop=True,inplace=True)
-    
-    wbdcol_name = 'HUC'+wbd_layer[-1]
-
-    # expand dates
-    datetime_indices = bounding_boxes.apply(lambda df:pd.date_range(df['event_date'],periods=24,closed=None,freq='H',tz='UTC'),axis=1)
-
-    datetime_indices.name = 'date_time'
-    datetime_indices=pd.DataFrame(datetime_indices)
-    datetime_indices = datetime_indices.join(bounding_boxes[['Name',wbdcol_name]])
-
-    # append columns to expanded dates
-    forecast_df = pd.DataFrame()
-    for idx,row in datetime_indices.iterrows():
-        dt_df = row['date_time'].to_frame(index=False,name='date_time')
-        
-        row = row.drop('date_time')
-
-        dt_df = dt_df.join(pd.concat([pd.DataFrame(row).T]*len(dt_df),ignore_index=True))
-
-        forecast_df = pd.concat((forecast_df,dt_df),ignore_index=True)
-
-    # add extra dry date 
-    #additional_date_df = forecast_df[['Name',wbdcol_name]].merge(bounding_boxes[['additional_date',wbdcol_name]],left_on=wbdcol_name,right_on=wbdcol_name).drop_duplicates(ignore_index=True)
-
-    #forecast_df = pd.concat((forecast_df,additional_date_df.rename(columns={'additional_date':'date_time'})),ignore_index=True)
-
-    forecast_df = forecast_df.sort_values(['Name',wbdcol_name],ignore_index=True)
-
-
-    #forecast_df['date_time'] = forecast_df.apply(lambda df : df['date_time'].replace(hour=18,minute=0),axis=1)
-
-    forecast_df = forecast_df.rename(columns={wbdcol_name:'huc'})
-
-    forecast_df = construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=retrieve,workers=workers)
-   
-
-    # take daily means
-    def get_forecast(forecast_row):
-        
-        try:
-            forecast_table = read_nwm_forecast_file(forecast_row['forecast_file'])
-        except FileNotFoundError:
-            print(f"Skipping file {forecast_row['forecast_file']}")
-            return None
-
-        return(forecast_table)
-
-    # filter out 2021
-    years = forecast_df['date_time'].dt.year < 2021
-    forecast_df = forecast_df.loc[years,:]
-
-    # unique dates and hourly samples
-    unique_forecast_df = forecast_df.drop(columns='huc').drop_duplicates()
-    unique_forecast_df.reset_index(inplace=True,drop=True)
-
-    dates = unique_forecast_df["date_time"].map(lambda t: t.date())
-    unique_dates, hourly_samples_per_date = np.unique(dates,return_counts=True)
-
-    unique_dict = dict(zip(unique_dates,hourly_samples_per_date))
-    
-    final_forecast_df = forecast_df.groupby(pd.Grouper(key='date_time',freq='d')).first().dropna()
-    final_forecast_df.set_index(pd.to_datetime(final_forecast_df.index.date,utc=True),drop=True,inplace=True)
-    forecast_df_dates = forecast_df.copy()
-    forecast_df_dates.date_time = forecast_df.date_time.dt.date
-    final_forecast_df = final_forecast_df.merge(forecast_df,left_index=True, right_on='date_time')
-
-    final_forecast_df.reset_index(drop=True,inplace=True)
-    final_forecast_df['date_time'] = final_forecast_df.date_time.dt.date
-
-    final_forecast_df.drop(columns={'huc_x','Name_x','forecast_file_x','forecast_url_x','forecast_url_y'},inplace=True)
-
-    final_forecast_df.rename(columns={'Name_y':'Name','huc_y':'huc','forecast_file_y':'forecast_file'},inplace=True)
-    
-    def update_daily(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df):
-        daily_mean = daily_nwm_forecast_df.mean(axis=1).rename('discharge')
-        
-        current_date_string = current_date.strftime(format='%Y%m%d')
-        filename,ext = os.path.basename(daily_mean_forecast_files).split('.')
-        outfile = os.path.join(os.path.dirname(daily_mean_forecast_files),filename+'_'+ current_date_string + '.'+ext)
-        daily_mean.to_csv(outfile,index=True)
-        
-        final_forecast_df.loc[final_forecast_df['date_time'] == current_date,'forecast_file'] = outfile
-        
-        daily_nwm_forecast_df = None
-
-        return(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df)
-
-
-    daily_nwm_forecast_df = None
-    ii = 0
-    current_date_time = unique_forecast_df.loc[0,'date_time']
-    current_date = current_date_time.date()
-    for i,row in tqdm(unique_forecast_df.iterrows(),total=len(unique_forecast_df),desc='Daily Means'):
-        
-        current_nwm_forecast_df = get_forecast(row)
-        
-        if ii == unique_dict[current_date]:
-
-            daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df = update_daily(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df)
-
-        if current_nwm_forecast_df is not None:
-            
-            if daily_nwm_forecast_df is None:
-                ii = 0
-                daily_nwm_forecast_df = pd.DataFrame(np.empty((len(current_nwm_forecast_df),unique_dict[current_date])))
-                daily_nwm_forecast_df.set_index(current_nwm_forecast_df.index,inplace=True,drop=True)
-            
-            daily_nwm_forecast_df.loc[:,ii] = current_nwm_forecast_df.discharge
-            #daily_nwm_forecast_df.rename(columns={ii:current_date_time})
-            
-            ii += 1
-        
-        current_date_time = row['date_time']
-        current_date = current_date_time.date()
-
-    daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df = update_daily(daily_nwm_forecast_df,current_date,daily_mean_forecast_files,final_forecast_df)
-       
-
-    if forecast_output_file is not None:
-        #final_forecast_df.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d %H:%M:%S%Z')
-        final_forecast_df.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d')
-
-
-
-def construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=True,workers=1):
-    # make forecast file names for NWM and retrieve
-
-    #construct url
-    #url = f'{year}/{year}{month}{day}{time}.CHRTOUT_DOMAIN1.comp'
-
-    make_url = lambda df:  "https://noaa-nwm-retrospective-2-1-pds.s3.amazonaws.com/model_output/" + \
-                                          str(df['date_time'].year) + '/' + str(df['date_time'].year) + \
-                                           str(df['date_time'].month).zfill(2) + \
-                                           str(df['date_time'].day).zfill(2) + \
-                                           str(df['date_time'].hour).zfill(2) + \
-                                           str(df['date_time'].minute).zfill(2) +\
-                                           ".CHRTOUT_DOMAIN1.comp"
-        
-    def make_file_names(df,download_directory):
-        # assumes that the last segment after the / represents the file name
-        url = df['forecast_url']
-        file_name_start_pos = url.rfind("/") + 1
-        file_name = url[file_name_start_pos:]
-
-        file_name = os.path.join(download_directory,file_name)
-
-        return(file_name)
-
-
-    urls = forecast_df.apply(make_url,axis=1)
-    forecast_df['forecast_url'] = urls
-
-    file_names = forecast_df.apply(lambda df: make_file_names(df,download_directory),axis=1)
-    forecast_df['forecast_file'] = file_names
-
-    if not retrieve:
-        return(forecast_df)
-
-    download_df = forecast_df[['forecast_url','forecast_file']].drop_duplicates()
-
-
-    def download_url(url,file_name):
-        r = requests.get(url, stream=True)
-        if r.status_code == requests.codes.ok:
-             with open(file_name, 'wb') as f:
-                for data in r:
-                    f.write(data)
-        return url
-
-    pool = ThreadPoolExecutor(max_workers=workers)
-    
-    results = { pool.submit(download_url,*(url,file_name)) : (url,file_name) for idx,(url,file_name) in download_df.iterrows() }
-
-    for future in tqdm(as_completed(results),total=len(download_df),desc='Acquiring NWM forecasts'):
-
-        url,file_name = results[future]
-
-        try:
-            future.result()
-        except Exception as exc:
-            print('error',exc,url)
-
-    pool.shutdown(wait=True)
-    
-    return(forecast_df)
-
-
-if __name__ == '__main__':
-
-
-    ##################################
-    ##
-    ## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
-    ## Might want to be kept for possible re-use at a later time?
-    ##
-    ##################################
-
-    # parse arguments
-    parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
-    parser.add_argument('-b','--bounding-boxes-file', help='Bounding box file', required=True)
-    parser.add_argument('-w','--wbd', help='WBD file', required=True)
-    parser.add_argument('-f','--forecast-output-file', help='Forecast file', required=False,default=None)
-    parser.add_argument('-r','--retrieve', help='Forecast file', required=False,default=False,action='store_true')
-    parser.add_argument('-j','--workers', help='Forecast file', required=False,default=1,type=int)
-    parser.add_argument('-d','--download_directory', help='Forecast file', required=False,default=1)
-    parser.add_argument('-m','--daily-mean-forecast-files', help='Daily Mean Forecast file', required=False,default=None)
-
-    args=vars(parser.parse_args())
-
-    cygnss_preprocessing(**args)
diff --git a/tools/eval_alt_catfim.py b/tools/eval_alt_catfim.py
index d5ce867d8..32c7eb766 100644
--- a/tools/eval_alt_catfim.py
+++ b/tools/eval_alt_catfim.py
@@ -1,79 +1,85 @@
+#!/usr/bin/env python3
 
-import os
 import argparse
-from multiprocessing import Pool
 import csv
 import json
+import os
+from multiprocessing import Pool
 
-from tools_shared_variables import TEST_CASES_DIR
 from tools_shared_functions import compute_contingency_stats_from_rasters
+from tools_shared_variables import TEST_CASES_DIR
 
 
 def create_master_metrics_csv_alt(master_metrics_csv_output, json_list, version):
-
     # Construct header
-    metrics_to_write = ['true_negatives_count',
-                        'false_negatives_count',
-                        'true_positives_count',
-                        'false_positives_count',
-                        'contingency_tot_count',
-                        'cell_area_m2',
-                        'TP_area_km2',
-                        'FP_area_km2',
-                        'TN_area_km2',
-                        'FN_area_km2',
-                        'contingency_tot_area_km2',
-                        'predPositive_area_km2',
-                        'predNegative_area_km2',
-                        'obsPositive_area_km2',
-                        'obsNegative_area_km2',
-                        'positiveDiff_area_km2',
-                        'CSI',
-                        'FAR',
-                        'TPR',
-                        'TNR',
-                        'PPV',
-                        'NPV',
-                        'ACC',
-                        'Bal_ACC',
-                        'MCC',
-                        'EQUITABLE_THREAT_SCORE',
-                        'PREVALENCE',
-                        'BIAS',
-                        'F1_SCORE',
-                        'TP_perc',
-                        'FP_perc',
-                        'TN_perc',
-                        'FN_perc',
-                        'predPositive_perc',
-                        'predNegative_perc',
-                        'obsPositive_perc',
-                        'obsNegative_perc',
-                        'positiveDiff_perc',
-                        'masked_count',
-                        'masked_perc',
-                        'masked_area_km2'
-                        ]
+    metrics_to_write = [
+        'true_negatives_count',
+        'false_negatives_count',
+        'true_positives_count',
+        'false_positives_count',
+        'contingency_tot_count',
+        'cell_area_m2',
+        'TP_area_km2',
+        'FP_area_km2',
+        'TN_area_km2',
+        'FN_area_km2',
+        'contingency_tot_area_km2',
+        'predPositive_area_km2',
+        'predNegative_area_km2',
+        'obsPositive_area_km2',
+        'obsNegative_area_km2',
+        'positiveDiff_area_km2',
+        'CSI',
+        'FAR',
+        'TPR',
+        'TNR',
+        'PPV',
+        'NPV',
+        'ACC',
+        'Bal_ACC',
+        'MCC',
+        'EQUITABLE_THREAT_SCORE',
+        'PREVALENCE',
+        'BIAS',
+        'F1_SCORE',
+        'TP_perc',
+        'FP_perc',
+        'TN_perc',
+        'FN_perc',
+        'predPositive_perc',
+        'predNegative_perc',
+        'obsPositive_perc',
+        'obsNegative_perc',
+        'positiveDiff_perc',
+        'masked_count',
+        'masked_perc',
+        'masked_area_km2',
+    ]
 
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
-    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
-
-
+    list_to_write = [
+        additional_header_info_prefix
+        + metrics_to_write
+        + ['full_json_path']
+        + ['flow']
+        + ['benchmark_source']
+        + ['extent_config']
+        + ["calibrated"]
+    ]
 
     for full_json_path in json_list:
-         
-         # Parse variables from json path.
+        # Parse variables from json path.
         split_json_handle = os.path.split(full_json_path)[1].split('_')
-        
+
         benchmark_source = split_json_handle[2]
         huc = split_json_handle[1]
         nws_lid = split_json_handle[0]
         magnitude = split_json_handle[3].replace('.json', '')
-                  
+
         real_json_path = os.path.join(os.path.split(full_json_path)[0], nws_lid + '_b0m_stats.json')
-        
+
         sub_list_to_append = [version, nws_lid, magnitude, huc]
-        
+
         stats_dict = json.load(open(real_json_path))
         for metric in metrics_to_write:
             sub_list_to_append.append(stats_dict[metric])
@@ -82,9 +88,8 @@ def create_master_metrics_csv_alt(master_metrics_csv_output, json_list, version)
         sub_list_to_append.append(benchmark_source)
         sub_list_to_append.append('MS')
         sub_list_to_append.append('yes')
-        
-        list_to_write.append(sub_list_to_append)
 
+        list_to_write.append(sub_list_to_append)
 
     with open(master_metrics_csv_output, 'w', newline='') as csvfile:
         csv_writer = csv.writer(csvfile)
@@ -92,7 +97,6 @@ def create_master_metrics_csv_alt(master_metrics_csv_output, json_list, version)
 
 
 def process_alt_comparison(args):
-    
     predicted_raster_path = args[0]
     benchmark_raster_path = args[1]
     agreement_raster = args[2]
@@ -103,119 +107,181 @@ def process_alt_comparison(args):
     test_id = args[7]
     mask_dict = args[8]
 
-    compute_contingency_stats_from_rasters(predicted_raster_path,
-                                           benchmark_raster_path,
-                                           agreement_raster,
-                                           stats_csv=stats_csv,
-                                           stats_json=stats_json,
-                                           mask_values=[],
-                                           stats_modes_list=stats_modes_list,
-                                           test_id=test_id,
-                                           mask_dict=mask_dict,
-                                           )
+    compute_contingency_stats_from_rasters(
+        predicted_raster_path,
+        benchmark_raster_path,
+        agreement_raster,
+        stats_csv=stats_csv,
+        stats_json=stats_json,
+        mask_values=[],
+        stats_modes_list=stats_modes_list,
+        test_id=test_id,
+        mask_dict=mask_dict,
+    )
 
     print("Finished processing " + agreement_raster)
 
 
 if __name__ == '__main__':
-    
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Produces metrics for alternative CatFIM.')
-    parser.add_argument('-d','--catfim-directory',help='Path to directory storing CatFIM outputs. This is the most parent dir, usually named by a version.',required=True)
-    parser.add_argument('-w','--output-workspace',help='Add a special name to the end of the branch.',required=True, default="")
-    parser.add_argument('-m','--master-metrics-csv',help='Define path for master metrics CSV file.',required=False,default=None)
-    parser.add_argument('-j','--job-number',help='Number of processes to use. Default is 1.',required=False, default="1")
-    
+    parser.add_argument(
+        '-d',
+        '--catfim-directory',
+        help='Path to directory storing CatFIM outputs. This is the most parent dir, usually named by a version.',
+        required=True,
+    )
+    parser.add_argument(
+        '-w',
+        '--output-workspace',
+        help='Add a special name to the end of the branch.',
+        required=True,
+        default="",
+    )
+    parser.add_argument(
+        '-m',
+        '--master-metrics-csv',
+        help='Define path for master metrics CSV file.',
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-j', '--job-number', help='Number of processes to use. Default is 1.', required=False, default="1"
+    )
+
     # Assign variables from arguments.
     args = vars(parser.parse_args())
     catfim_dir = args['catfim_directory']
     output_workspace = args['output_workspace']
     job_number = int(args['job_number'])
     master_metrics_csv = args['master_metrics_csv']
-    
-    if master_metrics_csv == None:
+
+    if master_metrics_csv is None:
         master_metrics_csv = os.path.join(output_workspace, 'master_metrics.csv')
-    
+
     if not os.path.exists(catfim_dir):
         print("CatFIM directory: " + catfim_dir + " does not exist.")
         quit
-    
+
     if not os.path.exists(output_workspace):
         os.mkdir(output_workspace)
-    
+
     catfim_dir_list = os.listdir(catfim_dir)
-    
+
     procs_list = []
     json_list = []
-    
+
     for huc in catfim_dir_list:
         if len(huc) == 8:
-            
             huc_workspace = os.path.join(output_workspace, huc)
             if not os.path.exists(huc_workspace):
                 os.mkdir(huc_workspace)
 
             huc_dir_path = os.path.join(catfim_dir, huc)
-            
+
             # List AHPS sites.
             site_list = os.listdir(huc_dir_path)
-            
+
             # Loop through AHPS sites.
             for site in site_list:
                 site_dir = os.path.join(huc_dir_path, site)
-                
+
                 site_workspace = os.path.join(huc_workspace, site)
                 if not os.path.exists(site_workspace):
                     os.mkdir(site_workspace)
-                
+
                 for category in ['action', 'minor', 'moderate', 'major']:
                     # Presumptiously define inundation grid path.
-                    category_grid_path = os.path.join(site_dir, site + '_' + category + '_extent_' + huc + '.tif')
-                
-                    if os.path.exists(category_grid_path):
+                    category_grid_path = os.path.join(
+                        site_dir, site + '_' + category + '_extent_' + huc + '.tif'
+                    )
 
+                    if os.path.exists(category_grid_path):
                         site_category_workspace = os.path.join(site_workspace, category)
                         if not os.path.exists(site_category_workspace):
                             os.mkdir(site_category_workspace)
-                            
+
                         # Map path to benchmark data, both NWS and USGS.
                         for benchmark_type in ['nws', 'usgs']:
-                            benchmark_grid = os.path.join(TEST_CASES_DIR, benchmark_type + '_test_cases', 'validation_data_' + benchmark_type, huc, site, category, 'ahps_' + site + '_huc_' + huc + '_extent_' + category + '.tif')
- 
+                            benchmark_grid = os.path.join(
+                                TEST_CASES_DIR,
+                                benchmark_type + '_test_cases',
+                                'validation_data_' + benchmark_type,
+                                huc,
+                                site,
+                                category,
+                                'ahps_' + site + '_huc_' + huc + '_extent_' + category + '.tif',
+                            )
+
                             if os.path.exists(benchmark_grid):
-                                
-                                # Create dir in output workspace for results.                                
-                                file_handle = site + '_' + huc + '_' + benchmark_type + '_' + category 
-                                
+                                # Create dir in output workspace for results.
+                                file_handle = site + '_' + huc + '_' + benchmark_type + '_' + category
+
                                 predicted_raster_path = category_grid_path
                                 benchmark_raster_path = benchmark_grid
                                 agreement_raster = os.path.join(site_category_workspace, file_handle + '.tif')
                                 stats_csv = os.path.join(site_category_workspace, file_handle + '.csv')
                                 stats_json = os.path.join(site_category_workspace, file_handle + '.json')
-                                mask_values=None
-                                stats_modes_list=['total_area']
-                                test_id=''
-                                mask_dict={'levees': {'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg', 'buffer': None, 'operation': 'exclude'}, 
-                                            'waterbodies': {'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg', 'buffer': None, 'operation': 'exclude'},
-                                            site: {'path': '/data/test_cases/{benchmark_type}_test_cases/validation_data_{benchmark_type}/{huc}/{site}/{site}_domain.shp'.format(benchmark_type=benchmark_type, site=site, category=category, huc=huc), 'buffer': None, 'operation': 'include'}}
-                                
+                                mask_values = None
+                                stats_modes_list = ['total_area']
+                                test_id = ''
+                                mask_dict = {
+                                    'levees': {
+                                        'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg',
+                                        'buffer': None,
+                                        'operation': 'exclude',
+                                    },
+                                    'waterbodies': {
+                                        'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg',
+                                        'buffer': None,
+                                        'operation': 'exclude',
+                                    },
+                                    site: {
+                                        'path': f'/data/test_cases/{benchmark_type}_test_cases/'
+                                        f'validation_data_{benchmark_type}/'
+                                        f'{huc}/{site}/{site}_domain.shp',
+                                        'buffer': None,
+                                        'operation': 'include',
+                                    },
+                                }
+
                                 json_list.append(stats_json)
-                                
-                                # Either add to list to multiprocess or process serially, depending on user specification.
+
+                                # Either add to list to multiprocess or process serially, dependent on input
                                 if job_number > 1:
-                                    procs_list.append([predicted_raster_path, benchmark_raster_path, agreement_raster,stats_csv,stats_json,mask_values,stats_modes_list,test_id, mask_dict])
+                                    procs_list.append(
+                                        [
+                                            predicted_raster_path,
+                                            benchmark_raster_path,
+                                            agreement_raster,
+                                            stats_csv,
+                                            stats_json,
+                                            mask_values,
+                                            stats_modes_list,
+                                            test_id,
+                                            mask_dict,
+                                        ]
+                                    )
                                 else:
-                                    process_alt_comparison([predicted_raster_path, benchmark_raster_path, agreement_raster,stats_csv,stats_json, mask_values,stats_modes_list,test_id, mask_dict])
+                                    process_alt_comparison(
+                                        [
+                                            predicted_raster_path,
+                                            benchmark_raster_path,
+                                            agreement_raster,
+                                            stats_csv,
+                                            stats_json,
+                                            mask_values,
+                                            stats_modes_list,
+                                            test_id,
+                                            mask_dict,
+                                        ]
+                                    )
 
     # Multiprocess.
     if job_number > 1:
         with Pool(processes=job_number) as pool:
             pool.map(process_alt_comparison, procs_list)
-            
+
     # Merge stats into single file.
     version = os.path.split(output_workspace)[1]
     create_master_metrics_csv_alt(master_metrics_csv, json_list, version)
-
-            
-    
-    
\ No newline at end of file
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 3d5268eaf..dab85e934 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -1,33 +1,47 @@
 #!/usr/bin/env python3
-import pandas as pd
-from pathlib import Path
+
 import argparse
-from natsort import natsorted
-import geopandas as gpd
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-import re
 import glob
 import os
+import re
 import sys
-sys.path.append('/foss_fim/src')
-from utils.shared_variables import VIZ_PROJECTION
+from pathlib import Path
+
+import geopandas as gpd
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
 from dotenv import load_dotenv
+from natsort import natsorted
 from tools_shared_functions import aggregate_wbd_hucs, get_metadata
 from tools_shared_variables import BAD_SITES, DISCARD_AHPS_QUERY
 
-#Get variables from .env file.
+from utils.shared_variables import VIZ_PROJECTION
+
+
+# Get variables from .env file.
 load_dotenv()
 WBD_LAYER = os.getenv("WBD_LAYER")
 API_BASE_URL = os.getenv("API_BASE_URL")
 
+
 #########################################################################
-#Create boxplot
+# Create boxplot
 #########################################################################
-def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue,
-            title_text, fim_configuration, textbox_str = False, 
-            simplify_legend = False, dest_file = False):
+def boxplot(
+    dataframe,
+    x_field,
+    x_order,
+    y_field,
+    hue_field,
+    ordered_hue,
+    title_text,
+    fim_configuration,
+    textbox_str=False,
+    simplify_legend=False,
+    dest_file=False,
+):
     '''
     Create boxplots.
 
@@ -61,26 +75,34 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue,
 
     '''
 
-    #initialize plot
+    # initialize plot
     fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    #Use seaborn to plot the boxplot
-    axes=sns.boxplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
-    #set title of plot
-    axes.set_title(f'{title_text} ({y_field})',fontsize=20, weight = 'bold')
-    #Set yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    for index,ytick in enumerate(axes.get_yticks()):
-        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel('',fontsize=0,weight = 'bold')
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-    axes.legend(markerscale = 2, fontsize =20, loc = 'lower left')
-
-    #If simple legend desired
+    # Use seaborn to plot the boxplot
+    axes = sns.boxplot(
+        x=x_field,
+        y=y_field,
+        order=x_order,
+        hue=hue_field,
+        hue_order=ordered_hue,
+        data=dataframe,
+        palette='bright',
+    )
+    # set title of plot
+    axes.set_title(f'{title_text} ({y_field})', fontsize=20, weight='bold')
+    # Set yticks and background horizontal line.
+    axes.set(ylim=(0.0, 1.0), yticks=np.arange(0, 1.1, 0.1))
+    for index, ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick, color='black', linestyle='--', linewidth=1, alpha=0.1)
+    # Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field}', fontsize='xx-large', weight='bold')
+    axes.set_xlabel('', fontsize=0, weight='bold')
+    # Set sizes of ticks and legend.
+    axes.tick_params(labelsize='xx-large')
+    axes.legend(markerscale=2, fontsize=20, loc='lower left')
+
+    # If simple legend desired
     if simplify_legend:
-        #trim labels to FIM 1, FIM 2, and the FIM 3 version
+        # trim labels to FIM 1, FIM 2, and the FIM 3 version
         handles, org_labels = axes.get_legend_handles_labels()
         label_dict = {}
         for label in org_labels:
@@ -89,38 +111,58 @@ def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue,
             elif 'fim_2' in label:
                 label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
             elif 'fim_3' in label and len(label) < 20:
-                label_dict[label] = re.split('_fr|_ms|_comp', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                label_dict[label] = (
+                    re.split('_fr|_ms|_comp', label)[0].replace('_', '.').replace('fim.', 'FIM ')
+                    + ' '
+                    + fim_configuration.lower()
+                )
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
             elif 'fim_4' in label and len(label) < 20:
-                label_dict[label] = label.replace('_','.').replace('fim.','FIM ')
+                label_dict[label] = label.replace('_', '.').replace('fim.', 'FIM ')
             else:
                 label_dict[label] = label
-        #Define simplified labels as a list.
+        # Define simplified labels as a list.
         new_labels = [label_dict[label] for label in org_labels]
-        #Define legend location. FAR needs to be in different location than CSI/POD.
-        if y_field in ['FAR', 'PND','EQUITABLE_THREAT_SCORE']:
+        # Define legend location. FAR needs to be in different location than CSI/POD.
+        if y_field in ['FAR', 'PND', 'EQUITABLE_THREAT_SCORE']:
             legend_location = 'upper right'
         else:
             legend_location = 'lower left'
-        #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 14, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
-    #Print textbox if supplied
+        # rename legend labels to the simplified labels.
+        axes.legend(
+            handles,
+            new_labels,
+            markerscale=2,
+            fontsize=14,
+            loc=legend_location,
+            ncol=int(np.ceil(len(new_labels) / 7)),
+        )
+    # Print textbox if supplied
     if textbox_str:
         box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=14, verticalalignment='top', bbox=box_props)
-
-    #If figure to be saved to disk, then do so, otherwise return figure
+        axes.text(
+            0.01,
+            0.99,
+            textbox_str,
+            transform=axes.transAxes,
+            fontsize=14,
+            verticalalignment='top',
+            bbox=box_props,
+        )
+
+    # If figure to be saved to disk, then do so, otherwise return figure
     if dest_file:
         fig.savefig(dest_file)
         plt.close(fig)
     else:
         return fig
 
+
 #########################################################################
-#Create scatter plot
+# Create scatter plot
 #########################################################################
-def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):
+def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate=False, dest_file=False):
     '''
     Create boxplots.
 
@@ -147,56 +189,100 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
 
     '''
 
-    #initialize plot
+    # initialize plot
     fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
 
-    #Use seaborn to plot the boxplot
-    axes=sns.scatterplot(data=dataframe, x=x_field, y=y_field, color = 'black', s = 150)
+    # Use seaborn to plot the boxplot
+    axes = sns.scatterplot(data=dataframe, x=x_field, y=y_field, color='black', s=150)
 
-    #Set xticks and yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
+    # Set xticks and yticks and background horizontal line.
+    axes.set(ylim=(0.0, 1.0), yticks=np.arange(0, 1.1, 0.1))
+    axes.set(xlim=(0.0, 1.0), xticks=np.arange(0, 1.1, 0.1))
     axes.grid(visible=True, which='major', axis='both')
 
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
+    # Set sizes of ticks and legend.
+    axes.tick_params(labelsize='xx-large')
 
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel(f'{x_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
+    # Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.replace("_"," ")}', fontsize='xx-large', weight='bold')
+    axes.set_xlabel(f'{x_field.replace("_"," ")}', fontsize='xx-large', weight='bold')
 
-    #Plot diagonal line
-    diag_range = [0,1]
+    # Plot diagonal line
+    diag_range = [0, 1]
     axes.plot(diag_range, diag_range, color='gray', transform=axes.transAxes)
 
-
-    #set title of plot
-    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
+    # set title of plot
+    axes.set_title(f'{title_text}', fontsize=20, weight='bold')
 
     if annotate:
-        #Set text for labels
+        # Set text for labels
         box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
         textbox_str = 'Target Better'
-        axes.text(0.3, 0.6, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+        axes.text(
+            0.3,
+            0.6,
+            textbox_str,
+            transform=axes.transAxes,
+            fontsize=32,
+            color='gray',
+            fontweight='bold',
+            verticalalignment='top',
+            bbox=box_props,
+            rotation=35,
+            rotation_mode='anchor',
+        )
         textbox_str = 'Baseline Better'
-        axes.text(0.5, 0.2, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
+        axes.text(
+            0.5,
+            0.2,
+            textbox_str,
+            transform=axes.transAxes,
+            fontsize=32,
+            color='gray',
+            fontweight='bold',
+            verticalalignment='top',
+            bbox=box_props,
+            rotation=35,
+            rotation_mode='anchor',
+        )
 
     if stats_text:
-        #Add statistics textbox
-        axes.text(0.01, 0.80, stats_text, transform=axes.transAxes, fontsize=24, verticalalignment='top', bbox=box_props)
-
-    #If figure to be saved to disk, then do so, otherwise return fig
+        # Add statistics textbox
+        axes.text(
+            0.01,
+            0.80,
+            stats_text,
+            transform=axes.transAxes,
+            fontsize=24,
+            verticalalignment='top',
+            bbox=box_props,
+        )
+
+    # If figure to be saved to disk, then do so, otherwise return fig
     if dest_file:
         fig.savefig(dest_file)
         plt.close(fig)
     else:
         return fig
+
+
 #########################################################################
-#Create barplot
+# Create barplot
 #########################################################################
-def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, 
-            title_text, fim_configuration, textbox_str = False, 
-            simplify_legend = False, display_values = False, dest_file = False):
+def barplot(
+    dataframe,
+    x_field,
+    x_order,
+    y_field,
+    hue_field,
+    ordered_hue,
+    title_text,
+    fim_configuration,
+    textbox_str=False,
+    simplify_legend=False,
+    display_values=False,
+    dest_file=False,
+):
     '''
     Create barplots.
 
@@ -233,25 +319,33 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue,
 
     '''
 
-    #initialize plot
+    # initialize plot
     fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    #Use seaborn to plot the boxplot
-    axes=sns.barplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
-    #set title of plot
-    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
-    #Set yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    for index,ytick in enumerate(axes.get_yticks()):
-        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field.upper()}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel('',fontsize=0,weight = 'bold')
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-    axes.legend(markerscale = 2, fontsize =20, loc = 'upper right')
-    #If simple legend desired
+    # Use seaborn to plot the boxplot
+    axes = sns.barplot(
+        x=x_field,
+        y=y_field,
+        order=x_order,
+        hue=hue_field,
+        hue_order=ordered_hue,
+        data=dataframe,
+        palette='bright',
+    )
+    # set title of plot
+    axes.set_title(f'{title_text}', fontsize=20, weight='bold')
+    # Set yticks and background horizontal line.
+    axes.set(ylim=(0.0, 1.0), yticks=np.arange(0, 1.1, 0.1))
+    for index, ytick in enumerate(axes.get_yticks()):
+        plt.axhline(y=ytick, color='black', linestyle='--', linewidth=1, alpha=0.1)
+    # Define y axis label and x axis label.
+    axes.set_ylabel(f'{y_field.upper()}', fontsize='xx-large', weight='bold')
+    axes.set_xlabel('', fontsize=0, weight='bold')
+    # Set sizes of ticks and legend.
+    axes.tick_params(labelsize='xx-large')
+    axes.legend(markerscale=2, fontsize=20, loc='upper right')
+    # If simple legend desired
     if simplify_legend:
-        #trim labels to FIM 1, FIM 2, FIM 3
+        # trim labels to FIM 1, FIM 2, FIM 3
         handles, org_labels = axes.get_legend_handles_labels()
         label_dict = {}
         for label in org_labels:
@@ -260,40 +354,64 @@ def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue,
             elif 'fim_2' in label:
                 label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
             elif 'fim_3' in label and len(label) < 20:
-                label_dict[label] = re.split('_fr|_ms|_comp', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
+                label_dict[label] = (
+                    re.split('_fr|_ms|_comp', label)[0].replace('_', '.').replace('fim.', 'FIM ')
+                    + ' '
+                    + fim_configuration.lower()
+                )
                 if label.endswith('_c'):
                     label_dict[label] = label_dict[label] + ' c'
             elif 'fim_4' in label and len(label) < 20:
-                label_dict[label] = label.replace('_','.').replace('fim.','FIM ')
+                label_dict[label] = label.replace('_', '.').replace('fim.', 'FIM ')
             else:
                 label_dict[label] = label
-        #Define simplified labels as a list.
+        # Define simplified labels as a list.
         new_labels = [label_dict[label] for label in org_labels]
-        #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 14, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
-    #Add Textbox
+        # rename legend labels to the simplified labels.
+        axes.legend(
+            handles,
+            new_labels,
+            markerscale=2,
+            fontsize=14,
+            loc='upper right',
+            ncol=int(np.ceil(len(new_labels) / 7)),
+        )
+    # Add Textbox
     if textbox_str:
         box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=18, verticalalignment='top', bbox=box_props)
-
-    #Display Y values above bars
+        axes.text(
+            0.01,
+            0.99,
+            textbox_str,
+            transform=axes.transAxes,
+            fontsize=18,
+            verticalalignment='top',
+            bbox=box_props,
+        )
+
+    # Display Y values above bars
     if display_values:
-        #Add values of bars directly above bar.
+        # Add values of bars directly above bar.
         for patch in axes.patches:
-            value = round(patch.get_height(),3)
-            axes.text(patch.get_x()+patch.get_width()/2.,
-                    patch.get_height(),
-                    '{:1.3f}'.format(value),
-                    ha="center", fontsize=18)
-
-    #If figure to be saved to disk, then do so, otherwise return fig
+            value = round(patch.get_height(), 3)
+            axes.text(
+                patch.get_x() + patch.get_width() / 2.0,
+                patch.get_height(),
+                '{:1.3f}'.format(value),
+                ha="center",
+                fontsize=18,
+            )
+
+    # If figure to be saved to disk, then do so, otherwise return fig
     if dest_file:
         fig.savefig(dest_file)
         plt.close(fig)
     else:
         return fig
+
+
 #######################################################################
-#Filter dataframe generated from csv file from run_test_case aggregation
+# Filter dataframe generated from csv file from run_test_case aggregation
 ########################################################################
 def filter_dataframe(dataframe, unique_field):
     '''
@@ -317,41 +435,56 @@ def filter_dataframe(dataframe, unique_field):
     Returns
     -------
     final_filtered_dataframe : Pandas Dataframe
-        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude.
+        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude.
+        For example, for AHPS all sites which were run for each version for a given magnitude will be kept
+            or for ble, all hucs which ran for all versions for a given magnitude.
     unique_sites: DICT
         The sites that were included in the dataframe for each magnitude.
 
     '''
 
-    #Get lists of sites for each magnitude/version
-    unique_sites = dataframe.groupby(['magnitude','version'])[unique_field].agg('unique')
-    #Get unique magnitudes
+    # Get lists of sites for each magnitude/version
+    unique_sites = dataframe.groupby(['magnitude', 'version'])[unique_field].agg('unique')
+    # Get unique magnitudes
     magnitudes = dataframe.magnitude.unique()
-    #Create new dataframe to hold metrics for the common sites as well as the actual lists of common sites.
+    # Create new dataframe to hold metrics for the common sites as well as the actual lists of common sites.
     final_filtered_dataframe = pd.DataFrame()
     all_unique_sites = {}
-    #Cycle through each magnitude
+    # Cycle through each magnitude
     for magnitude in magnitudes:
-        #Compile a list of sets containing unique lids pertaining to each threshold. List contains 3 unique sets [{fim1:unique lids},{fim2: unique lids},{fim3: unique lids}]
-        sites_per_magnitude=[set(a) for a in unique_sites[magnitude]]
-        #Intersect the sets to get the common lids per threshold then convert to list.
+        # Compile a list of sets containing unique lids pertaining to each threshold.
+        # List contains 3 unique sets [{fim1:unique lids},{fim2: unique lids},{fim3: unique lids}]
+        sites_per_magnitude = [set(a) for a in unique_sites[magnitude]]
+        # Intersect the sets to get the common lids per threshold then convert to list.
         common_sites_per_magnitude = list(set.intersection(*sites_per_magnitude))
-        #Write common sites to dataframe
+        # Write common sites to dataframe
         all_unique_sites[magnitude] = common_sites_per_magnitude
-        #Query filtered dataframe and only include data associated with the common sites for that magnitude
-        filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
-        #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
-        final_filtered_dataframe = pd.concat([final_filtered_dataframe,  filtered_common_sites], ignore_index = True)
+        # Query filtered dataframe and only include data associated with the common sites for that magnitude
+        filtered_common_sites = dataframe.query(
+            f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude'
+        )
+        # Append the data for each magnitude to a final dataframe that will contain data for all common sites
+        #   for all magnitudes.
+        final_filtered_dataframe = pd.concat(
+            [final_filtered_dataframe, filtered_common_sites], ignore_index=True
+        )
 
     return final_filtered_dataframe, all_unique_sites
+
+
 ##############################################################################
 ##############################################################################
-#Main function to analyze metric csv.
+# Main function to analyze metric csv.
 ##############################################################################
-def eval_plots(metrics_csv, workspace, versions = [], 
-               stats = ['CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE'] , spatial = False,
-               fim_1_ms = False, site_barplots = False):
-
+def eval_plots(
+    metrics_csv,
+    workspace,
+    versions=[],
+    stats=['CSI', 'FAR', 'TPR', 'PND', 'MCC', 'EQUITABLE_THREAT_SCORE'],
+    spatial=False,
+    fim_1_ms=False,
+    site_barplots=False,
+):
     '''
     Creates plots and summary statistics using metrics compiled from
     synthesize_test_cases. Required inputs are metrics_csv and workspace.
@@ -451,20 +584,20 @@ def eval_plots(metrics_csv, workspace, versions = [],
     '''
 
     # Import metrics csv as DataFrame and initialize all_datasets dictionary
-    csv_df = pd.read_csv(metrics_csv, dtype = {'huc':str})
+    csv_df = pd.read_csv(metrics_csv, dtype={'huc': str})
 
     # fim_1_ms flag enables FIM 1 to be shown on MS plots/stats
     if fim_1_ms:
-        #Query FIM 1 rows based on version beginning with "fim_1"
+        # Query FIM 1 rows based on version beginning with "fim_1"
         fim_1_rows = csv_df.query('version.str.startswith("fim_1")').copy()
-        #Set extent configuration to MS (instead of FR)
+        # Set extent configuration to MS (instead of FR)
         fim_1_rows['extent_config'] = 'MS'
-        #Append duplicate FIM 1 rows to original dataframe
-        csv_df = pd.concat([csv_df, fim_1_rows], ignore_index = True)
+        # Append duplicate FIM 1 rows to original dataframe
+        csv_df = pd.concat([csv_df, fim_1_rows], ignore_index=True)
 
     # If versions are supplied then filter out
     if versions:
-        #Filter out versions based on supplied version list
+        # Filter out versions based on supplied version list
         metrics = csv_df.query("version == @versions")
     else:
         metrics = csv_df
@@ -481,66 +614,82 @@ def eval_plots(metrics_csv, workspace, versions = [],
 
     all_datasets = {}
     for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:
-
         '''If source is usgs/nws define the base resolution and query
         (use alternate query if passed). Append filtered datasets to
         all_datasets dictionary.'''
 
-        if benchmark_source in ['usgs','nws']:
-
+        if benchmark_source in ['usgs', 'nws']:
             # Set the base processing unit for the ahps runs.
             base_resolution = 'nws_lid'
 
             # Filter the dataset based on query (IMPORTED FROM TOOLS_SHARED_VARIABLES.py)
             ahps_metrics = benchmark_metrics.query(DISCARD_AHPS_QUERY, engine='python')
 
-            # Filter out all instances where the base_resolution doesn't exist across all desired fim versions for a given magnitude
-            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
+            # Filter out all instances where the base_resolution doesn't exist across all desired
+            #   fim versions for a given magnitude
+            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(
+                ahps_metrics, base_resolution
+            )
 
         # If source is 'ble', set base_resolution and append ble dataset to all_datasets dictionary
         elif benchmark_source in ['ble', 'ifc', 'ras2fim']:
-
             # Set the base processing unit for ble runs
             base_resolution = 'huc'
 
-            # Filter out all instances where base_resolution doesn't exist across all desired fim versions for a given magnitude
-            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(benchmark_metrics, base_resolution)
+            # Filter out all instances where base_resolution doesn't exist across all desired fim versions
+            #   for a given magnitude
+            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(
+                benchmark_metrics, base_resolution
+            )
 
     # For each dataset in all_datasets, generate plots and aggregate statistics
-    for (dataset_name,configuration), (dataset, sites) in all_datasets.items():
-
+    for (dataset_name, configuration), (dataset, sites) in all_datasets.items():
         # Define and create the output workspace as a subfolder within the supplied workspace
         output_workspace = Path(workspace) / dataset_name / configuration.lower()
-        output_workspace.mkdir(parents = True, exist_ok = True)
+        output_workspace.mkdir(parents=True, exist_ok=True)
 
         # Write out the filtered dataset and common sites to file
-        dataset.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_analyzed_data.csv'), index = False)
-        sites_pd = pd.DataFrame.from_dict(sites, orient = 'index').transpose()
-        sites_pd.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_common_sites.csv'), index = False)
+        dataset.to_csv(
+            output_workspace / (f'{dataset_name}_{configuration.lower()}_analyzed_data.csv'), index=False
+        )
+        sites_pd = pd.DataFrame.from_dict(sites, orient='index').transpose()
+        sites_pd.to_csv(
+            output_workspace / (f'{dataset_name}_{configuration.lower()}_common_sites.csv'), index=False
+        )
 
         # Set the order of the magnitudes and define base resolution
         if dataset_name == 'ble':
             magnitude_order = ['100yr', '500yr']
             base_resolution = 'huc'
         elif dataset_name == 'ifc':
-            magnitude_order = ['2yr','5yr','10yr','25yr','50yr','100yr','200yr','500yr']
+            magnitude_order = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr', '200yr', '500yr']
             base_resolution = 'huc'
         elif dataset_name == 'ras2fim':
-            magnitude_order = ['2yr','5yr','10yr','25yr','50yr','100yr']
+            magnitude_order = ['2yr', '5yr', '10yr', '25yr', '50yr', '100yr']
             base_resolution = 'huc'
-        elif dataset_name in ['usgs','nws']:
-            magnitude_order = ['action','minor','moderate','major']
+        elif dataset_name in ['usgs', 'nws']:
+            magnitude_order = ['action', 'minor', 'moderate', 'major']
             base_resolution = 'nws_lid'
 
         # Calculate aggregated metrics based on total_sq_km fields
-        dataset_sums = dataset.groupby(['version', 'magnitude'])[['TP_area_km2','FP_area_km2','FN_area_km2']].sum()
-        dataset_sums['csi'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'] + dataset_sums['FN_area_km2'])
-        dataset_sums['far'] = dataset_sums['FP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'])
-        dataset_sums['pod'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FN_area_km2'])
+        dataset_sums = dataset.groupby(['version', 'magnitude'])[
+            ['TP_area_km2', 'FP_area_km2', 'FN_area_km2']
+        ].sum()
+        dataset_sums['csi'] = dataset_sums['TP_area_km2'] / (
+            dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'] + dataset_sums['FN_area_km2']
+        )
+        dataset_sums['far'] = dataset_sums['FP_area_km2'] / (
+            dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2']
+        )
+        dataset_sums['pod'] = dataset_sums['TP_area_km2'] / (
+            dataset_sums['TP_area_km2'] + dataset_sums['FN_area_km2']
+        )
         dataset_sums = dataset_sums.reset_index()
 
         # Write aggregated metrics to file
-        dataset_sums.to_csv(output_workspace / f'aggregate_{dataset_name}_{configuration.lower()}.csv', index = False )
+        dataset_sums.to_csv(
+            output_workspace / f'aggregate_{dataset_name}_{configuration.lower()}.csv', index=False
+        )
 
         ## This section naturally orders analyzed versions which defines the hue order for the generated plots
         # Get all versions in dataset
@@ -555,11 +704,13 @@ def eval_plots(metrics_csv, workspace, versions = [],
             versions_list = versions
         # For each version supplied by the user
         for version in versions_list:
-            #Select all the versions that start with the supplied version.
-            selected_versions = [sel_version for sel_version in all_versions if sel_version.startswith(version)]
-            #Naturally sort selected_versions
+            # Select all the versions that start with the supplied version.
+            selected_versions = [
+                sel_version for sel_version in all_versions if sel_version.startswith(version)
+            ]
+            # Naturally sort selected_versions
             selected_versions = natsorted(selected_versions)
-            #Populate version order based on the sorted subsets.
+            # Populate version order based on the sorted subsets.
             version_order.extend(selected_versions)
 
         # Define textbox which will contain the counts of each magnitude
@@ -572,113 +723,238 @@ def eval_plots(metrics_csv, workspace, versions = [],
 
         # Create aggregate barplot
         aggregate_file = output_workspace / (f'csi_aggr_{dataset_name}_{configuration.lower()}.png')
-        barplot(dataframe = dataset_sums, x_field = 'magnitude', x_order = magnitude_order, y_field = 'csi', hue_field = 'version', ordered_hue = version_order, title_text = f'Aggregate {dataset_name.upper()} FIM Scores', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = aggregate_file)
-
-        #If enabled, write out barplots of CSI for individual sites.
+        barplot(
+            dataframe=dataset_sums,
+            x_field='magnitude',
+            x_order=magnitude_order,
+            y_field='csi',
+            hue_field='version',
+            ordered_hue=version_order,
+            title_text=f'Aggregate {dataset_name.upper()} FIM Scores',
+            fim_configuration=configuration,
+            textbox_str=textbox,
+            simplify_legend=True,
+            dest_file=aggregate_file,
+        )
+
+        # If enabled, write out barplots of CSI for individual sites.
         if site_barplots:
             individual_dirs = output_workspace / 'individual'
-            individual_dirs.mkdir(parents = True, exist_ok = True)
+            individual_dirs.mkdir(parents=True, exist_ok=True)
             subset = dataset.groupby(base_resolution)
             for site_name, site_data in subset:
-                site_file = individual_dirs / f'csi_{str(site_name)}_{dataset_name}_{configuration.lower()}.png'
-                barplot(dataframe = site_data, x_field = 'magnitude', x_order = magnitude_order, y_field = 'CSI', hue_field = 'version', ordered_hue = version_order, title_text = f'{str(site_name).upper()} FIM Scores', fim_configuration = configuration, textbox_str = False, simplify_legend = True, dest_file = site_file)
+                site_file = (
+                    individual_dirs / f'csi_{str(site_name)}_{dataset_name}_{configuration.lower()}.png'
+                )
+                barplot(
+                    dataframe=site_data,
+                    x_field='magnitude',
+                    x_order=magnitude_order,
+                    y_field='CSI',
+                    hue_field='version',
+                    ordered_hue=version_order,
+                    title_text=f'{str(site_name).upper()} FIM Scores',
+                    fim_configuration=configuration,
+                    textbox_str=False,
+                    simplify_legend=True,
+                    dest_file=site_file,
+                )
 
         # Create box plots for each metric in supplied stats
         for stat in stats:
             output_file = output_workspace / (f'{stat.lower()}_{dataset_name}_{configuration.lower()}.png')
-            boxplot(dataframe = dataset, x_field = 'magnitude', x_order = magnitude_order, y_field = stat, hue_field = 'version', ordered_hue = version_order, title_text = f'{dataset_name.upper()} FIM Sites', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = output_file)
+            boxplot(
+                dataframe=dataset,
+                x_field='magnitude',
+                x_order=magnitude_order,
+                y_field=stat,
+                hue_field='version',
+                ordered_hue=version_order,
+                title_text=f'{dataset_name.upper()} FIM Sites',
+                fim_configuration=configuration,
+                textbox_str=textbox,
+                simplify_legend=True,
+                dest_file=output_file,
+            )
 
         # Get the last 2 versions from the version order for scatter plot
         if len(version_order) == 2:
             x_version, y_version = version_order
             for magnitude in magnitude_order:
                 # Scatterplot comparison between last 2 versions
-                x_csi = dataset.query(f'version == "{x_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
-                y_csi = dataset.query(f'version == "{y_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
-                plotdf = pd.merge(x_csi, y_csi, on = base_resolution, suffixes = (f"_{x_version}",f"_{y_version}"))
+                x_csi = dataset.query(f'version == "{x_version}" & magnitude == "{magnitude}"')[
+                    [base_resolution, 'CSI']
+                ]
+                y_csi = dataset.query(f'version == "{y_version}" & magnitude == "{magnitude}"')[
+                    [base_resolution, 'CSI']
+                ]
+                plotdf = pd.merge(
+                    x_csi, y_csi, on=base_resolution, suffixes=(f"_{x_version}", f"_{y_version}")
+                )
                 # Define arguments for scatterplot function
                 title_text = f'CSI {magnitude}'
                 dest_file = output_workspace / f'csi_scatter_{magnitude}_{configuration.lower()}.png'
-                scatterplot(dataframe = plotdf, x_field = f'CSI_{x_version}', y_field = f'CSI_{y_version}', title_text = title_text, annotate = False, dest_file = dest_file)
-                #Write out dataframe used to create scatter plots
+                scatterplot(
+                    dataframe=plotdf,
+                    x_field=f'CSI_{x_version}',
+                    y_field=f'CSI_{y_version}',
+                    title_text=title_text,
+                    annotate=False,
+                    dest_file=dest_file,
+                )
+                # Write out dataframe used to create scatter plots
                 plotdf['Diff (C-B)'] = plotdf[f'CSI_{y_version}'] - plotdf[f'CSI_{x_version}']
-                plotdf.to_csv(output_workspace /  f'csi_scatter_{magnitude}_{configuration.lower()}_data.csv', index = False)
+                plotdf.to_csv(
+                    output_workspace / f'csi_scatter_{magnitude}_{configuration.lower()}_data.csv',
+                    index=False,
+                )
 
     #######################################################################
-    #Create spatial layers with threshold and mapping information
+    # Create spatial layers with threshold and mapping information
     ########################################################################
     if spatial:
         ###############################################################
-        #This section will join ahps metrics to a spatial point layer
+        # This section will join ahps metrics to a spatial point layer
         ###############################################################
-        if (all_datasets.get(('nws','MS')) and all_datasets.get(('usgs','MS'))) or \
-            (all_datasets.get(('nws','COMP')) and all_datasets.get(('usgs','COMP'))): # export composite metrics to shp
-            #Get point data for ahps sites
-            #Get metrics for usgs and nws benchmark sources
+        if (all_datasets.get(('nws', 'MS')) and all_datasets.get(('usgs', 'MS'))) or (
+            all_datasets.get(('nws', 'COMP')) and all_datasets.get(('usgs', 'COMP'))
+        ):  # export composite metrics to shp
+            # Get point data for ahps sites
+            # Get metrics for usgs and nws benchmark sources
             try:
-                usgs_dataset,sites = all_datasets.get(('usgs','MS'))
-                nws_dataset, sites = all_datasets.get(('nws','MS'))
-            except TypeError: # Composite metrics
-                usgs_dataset,sites = all_datasets.get(('usgs','COMP'))
-                nws_dataset, sites = all_datasets.get(('nws','COMP'))
-            #Append usgs/nws dataframes and filter unnecessary columns and rename remaining.
+                usgs_dataset, sites = all_datasets.get(('usgs', 'MS'))
+                nws_dataset, sites = all_datasets.get(('nws', 'MS'))
+            except TypeError:  # Composite metrics
+                usgs_dataset, sites = all_datasets.get(('usgs', 'COMP'))
+                nws_dataset, sites = all_datasets.get(('nws', 'COMP'))
+            # Append usgs/nws dataframes and filter unnecessary columns and rename remaining.
             all_ahps_datasets = pd.concat([usgs_dataset, nws_dataset])
-            all_ahps_datasets = all_ahps_datasets.filter(['huc','nws_lid','version','magnitude','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE','benchmark_source'])
-            all_ahps_datasets.rename(columns = {'benchmark_source':'source'}, inplace = True)
-
-            #Get spatial data from WRDS
-            #Get metadata from WRDS API
+            all_ahps_datasets = all_ahps_datasets.filter(
+                [
+                    'huc',
+                    'nws_lid',
+                    'version',
+                    'magnitude',
+                    'TP_area_km2',
+                    'FP_area_km2',
+                    'TN_area_km2',
+                    'FN_area_km2',
+                    'CSI',
+                    'FAR',
+                    'TPR',
+                    'PND',
+                    'MCC',
+                    'EQUITABLE_THREAT_SCORE',
+                    'benchmark_source',
+                ]
+            )
+            all_ahps_datasets.rename(columns={'benchmark_source': 'source'}, inplace=True)
+
+            # Get spatial data from WRDS
+            # Get metadata from WRDS API
             select_by = 'nws_lid'
             selector = list(all_ahps_datasets.nws_lid.unique())
             metadata_url = f'{API_BASE_URL}/metadata'
             metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector)
-            #Create geospatial data from WRDS output
-            dictionary, gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes = True)
-            #Trim out unecessary columns and rename remaining columns
-            gdf = gdf.filter(['identifiers_nws_lid', 'nws_data_name', 'identifiers_nwm_feature_id','nws_data_wfo','nws_data_state','nws_data_county','geometry'])
-            gdf.rename(columns = {'identifiers_nws_lid':'nws_lid', 'nws_data_name':'lid_name','identifiers_nwm_feature_id':'feature_id','nws_data_wfo':'wfo','nws_data_state':'state','nws_data_county':'county','HUC8':'huc8'}, inplace = True)
-
-            #Join spatial data to metric data
+            # Create geospatial data from WRDS output
+            dictionary, gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes=True)
+            # Trim out unecessary columns and rename remaining columns
+            gdf = gdf.filter(
+                [
+                    'identifiers_nws_lid',
+                    'nws_data_name',
+                    'identifiers_nwm_feature_id',
+                    'nws_data_wfo',
+                    'nws_data_state',
+                    'nws_data_county',
+                    'geometry',
+                ]
+            )
+            gdf.rename(
+                columns={
+                    'identifiers_nws_lid': 'nws_lid',
+                    'nws_data_name': 'lid_name',
+                    'identifiers_nwm_feature_id': 'feature_id',
+                    'nws_data_wfo': 'wfo',
+                    'nws_data_state': 'state',
+                    'nws_data_county': 'county',
+                    'HUC8': 'huc8',
+                },
+                inplace=True,
+            )
+
+            # Join spatial data to metric data
             gdf['nws_lid'] = gdf['nws_lid'].str.lower()
-            joined = gdf.merge(all_ahps_datasets, on = 'nws_lid')
-            #Project to VIZ projection and write to file
+            joined = gdf.merge(all_ahps_datasets, on='nws_lid')
+            # Project to VIZ projection and write to file
             joined = joined.to_crs(VIZ_PROJECTION)
             joined.to_file(Path(workspace) / 'fim_performance_points.shp')
         else:
-            print('NWS/USGS MS datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a MS version.')
+            print(
+                'NWS/USGS MS datasets not analyzed, no spatial data created.\n'
+                'To produce spatial data analyze a MS version.'
+            )
 
         ################################################################
-        #This section joins ble (FR) metrics to a spatial layer of HUCs.
+        # This section joins ble (FR) metrics to a spatial layer of HUCs.
         ################################################################
-        if (all_datasets.get(('ble','FR')) and all_datasets.get(('ifc','FR')) and all_datasets.get(('ras2fim','FR'))) or \
-            (all_datasets.get(('ble','COMP')) and all_datasets.get(('ifc','COMP')) and all_datasets.get(('ras2fim','COMP'))):
-            #Select BLE, FR dataset.
+        if (
+            all_datasets.get(('ble', 'FR'))
+            and all_datasets.get(('ifc', 'FR'))
+            and all_datasets.get(('ras2fim', 'FR'))
+        ) or (
+            all_datasets.get(('ble', 'COMP'))
+            and all_datasets.get(('ifc', 'COMP'))
+            and all_datasets.get(('ras2fim', 'COMP'))
+        ):
+            # Select BLE, FR dataset.
             try:
-                ble_dataset, sites = all_datasets.get(('ble','FR'))
-                ifc_dataset, sites = all_datasets.get(('ifc','FR'))
-                ras2fim_dataset, sites = all_datasets.get(('ras2fim','FR'))
+                ble_dataset, sites = all_datasets.get(('ble', 'FR'))
+                ifc_dataset, sites = all_datasets.get(('ifc', 'FR'))
+                ras2fim_dataset, sites = all_datasets.get(('ras2fim', 'FR'))
             except TypeError:
-                ble_dataset, sites = all_datasets.get(('ble','COMP'))
-                ifc_dataset, sites = all_datasets.get(('ifc','COMP'))
-                ras2fim_dataset, sites = all_datasets.get(('ras2fim','COMP'))
+                ble_dataset, sites = all_datasets.get(('ble', 'COMP'))
+                ifc_dataset, sites = all_datasets.get(('ifc', 'COMP'))
+                ras2fim_dataset, sites = all_datasets.get(('ras2fim', 'COMP'))
 
             huc_datasets = pd.concat([ble_dataset, ifc_dataset, ras2fim_dataset])
-            #Read in HUC spatial layer
-            wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer = 'WBDHU8')
-            #Join metrics to HUC spatial layer
-            wbd_with_metrics = wbd_gdf.merge(huc_datasets, how = 'inner', left_on = 'HUC8', right_on = 'huc')
-            #Filter out unnecessary columns
-            wbd_with_metrics = wbd_with_metrics.filter(['version','magnitude','huc','TP_area_km2','FP_area_km2','TN_area_km2','FN_area_km2','CSI','FAR','TPR','PND','MCC','EQUITABLE_THREAT_SCORE','benchmark_source','geometry'])
-            wbd_with_metrics.rename(columns = {'benchmark_source':'source'}, inplace = True )
-            #Project to VIZ projection
+            # Read in HUC spatial layer
+            wbd_gdf = gpd.read_file(Path(WBD_LAYER), layer='WBDHU8')
+            # Join metrics to HUC spatial layer
+            wbd_with_metrics = wbd_gdf.merge(huc_datasets, how='inner', left_on='HUC8', right_on='huc')
+            # Filter out unnecessary columns
+            wbd_with_metrics = wbd_with_metrics.filter(
+                [
+                    'version',
+                    'magnitude',
+                    'huc',
+                    'TP_area_km2',
+                    'FP_area_km2',
+                    'TN_area_km2',
+                    'FN_area_km2',
+                    'CSI',
+                    'FAR',
+                    'TPR',
+                    'PND',
+                    'MCC',
+                    'EQUITABLE_THREAT_SCORE',
+                    'benchmark_source',
+                    'geometry',
+                ]
+            )
+            wbd_with_metrics.rename(columns={'benchmark_source': 'source'}, inplace=True)
+            # Project to VIZ projection
             wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION)
-            #Write out to file
+            # Write out to file
             wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
         else:
-            print('BLE/IFC/RAS2FIM FR datasets not analyzed, no spatial data created.\nTo produce spatial data analyze a FR version')
-        
+            print(
+                'BLE/IFC/RAS2FIM FR datasets not analyzed, no spatial data created.\n'
+                'To produce spatial data analyze a FR version'
+            )
+
+
 def convert_shapes_to_csv(workspace):
-    
     # Convert any geopackage in the root level of output_mapping_dir to CSV and rename.
     shape_list = glob.glob(os.path.join(workspace, '*.shp'))
     for shape in shape_list:
@@ -687,19 +963,58 @@ def convert_shapes_to_csv(workspace):
         file_name = shape.replace('.shp', '.csv')
         csv_output_path = os.path.join(parent_directory, file_name)
         gdf.to_csv(csv_output_path)
-    
-        
+
+
 #######################################################################
 if __name__ == '__main__':
     # Parse arguments
-    parser = argparse.ArgumentParser(description = f'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
-    parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
-    parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
-    parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
-    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR','PND','MCC','EQUITABLE_THREAT_SCORE'], required = False)
-    parser.add_argument('-sp', '--spatial', help = 'If enabled, creates spatial layers with metrics populated in attribute table.', action = 'store_true', required = False)
-    parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
-    parser.add_argument('-i', '--site_plots', help = 'If enabled individual barplots for each site are created.', action = 'store_true', required = False)
+    parser = argparse.ArgumentParser(
+        description='Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)'
+    )
+    parser.add_argument(
+        '-m', '--metrics_csv', help='Metrics csv created from synthesize test cases.', required=True
+    )
+    parser.add_argument('-w', '--workspace', help='Output workspace', required=True)
+    parser.add_argument(
+        '-v',
+        '--versions',
+        help='List of versions to be plotted/aggregated. Versions are filtered using the "startswith" '
+        'approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" '
+        '(e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". '
+        'An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.',
+        nargs='+',
+        default=[],
+    )
+    parser.add_argument(
+        '-s',
+        '--stats',
+        help='List of statistics (abbrev to 3 letters) to be plotted/aggregated',
+        nargs='+',
+        default=['CSI', 'TPR', 'FAR', 'PND', 'MCC', 'EQUITABLE_THREAT_SCORE'],
+        required=False,
+    )
+    parser.add_argument(
+        '-sp',
+        '--spatial',
+        help='If enabled, creates spatial layers with metrics populated in attribute table.',
+        action='store_true',
+        required=False,
+    )
+    parser.add_argument(
+        '-f',
+        '--fim_1_ms',
+        help='If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that '
+        'fim_1 can be shown on mainstems plots/stats',
+        action='store_true',
+        required=False,
+    )
+    parser.add_argument(
+        '-i',
+        '--site_plots',
+        help='If enabled individual barplots for each site are created.',
+        action='store_true',
+        required=False,
+    )
 
     # Extract to dictionary and assign to variables
     args = vars(parser.parse_args())
@@ -709,15 +1024,15 @@ def convert_shapes_to_csv(workspace):
     w = args['workspace']
     v = args['versions']
     s = args['stats']
-    sp= args['spatial']
+    sp = args['spatial']
     f = args['fim_1_ms']
     i = args['site_plots']
 
     # Run eval_plots function
     print('The following AHPS sites are considered "BAD_SITES":  ' + ', '.join(BAD_SITES))
     print('The following query is used to filter AHPS:  ' + DISCARD_AHPS_QUERY)
-    eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, spatial = sp, fim_1_ms = f, site_barplots = i)
-    
+    eval_plots(metrics_csv=m, workspace=w, versions=v, stats=s, spatial=sp, fim_1_ms=f, site_barplots=i)
+
     # Convert output shapefiles to CSV
     print("Converting to CSVs...")
     convert_shapes_to_csv(w)
diff --git a/tools/eval_plots_stackedbar.py b/tools/eval_plots_stackedbar.py
index 533516a26..1a07c3921 100644
--- a/tools/eval_plots_stackedbar.py
+++ b/tools/eval_plots_stackedbar.py
@@ -1,16 +1,17 @@
-import numpy as np
-import matplotlib.pyplot as plt
-import pandas as pd
-import sys
+#!/usr/bin/env python3
+
 import argparse
+import sys
 from pathlib import Path
 
-sys.path.append("/foss_fim/tools/")
+import matplotlib.pyplot as plt
+import mpl_toolkits.axisartist as AA
+import numpy as np
+import pandas as pd
+from matplotlib.lines import Line2D
+from matplotlib.patches import Patch
 from tools_shared_variables import BAD_SITES
 
-from matplotlib.patches import Patch
-from matplotlib.lines import Line2D
-import mpl_toolkits.axisartist as AA
 
 mag_dict = {
     "action": "1_act",
@@ -36,9 +37,10 @@ def eval_plot_stack_data_prep(metric_csv, versions=[]):
         versions = list(versions)
         # Check to make sure requested versions are in the metrics file
         for v in versions:
-            assert (
-                v in metrics.version.unique()
-            ), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
+            assert v in metrics.version.unique(), (
+                f"{v} is not included in {metric_csv}.\n"
+                f"The available options are {sorted(metrics.version.unique())}"
+            )
         # Filter the metrics to the requested versions
         metrics = metrics.loc[metrics.version.isin(versions)]
     # Change the magnitudes to make them more sort-friendly for the plot below
@@ -46,20 +48,14 @@ def eval_plot_stack_data_prep(metric_csv, versions=[]):
     # Fill the non-AHPS sites with values for nws_lid
     metrics.nws_lid.fillna(metrics.huc + "_" + metrics.benchmark_source, inplace=True)
     # Create multi-index for easy merging and plotting
-    metrics.set_index(
-        ["benchmark_source", "nws_lid", "magnitude"], inplace=True, drop=False
-    )
+    metrics.set_index(["benchmark_source", "nws_lid", "magnitude"], inplace=True, drop=False)
 
     # Normalize data to the total of TP + FN
-    metrics["TP_FN_sum"] = (
-        metrics["false_negatives_count"] + metrics["true_positives_count"]
-    )
+    metrics["TP_FN_sum"] = metrics["false_negatives_count"] + metrics["true_positives_count"]
     metrics["TP_norm"] = metrics["true_positives_count"] / metrics["TP_FN_sum"]
     metrics["FN_norm"] = metrics["false_negatives_count"] / metrics["TP_FN_sum"]
     metrics["FP_norm"] = metrics["false_positives_count"] / metrics["TP_FN_sum"]
-    metrics["FP_norm"].replace(
-        [np.inf, -np.inf], np.nan, inplace=True
-    )  # some bad sites will divide by 0
+    metrics["FP_norm"].replace([np.inf, -np.inf], np.nan, inplace=True)  # some bad sites will divide by 0
 
     return metrics
 
@@ -97,44 +93,17 @@ def eval_plot_stack_indiv(metric_csv, versions, outfig, category):
     # Create a subplot for every site (nws_lid)
     for i, site in enumerate(sorted(data.nws_lid.unique())):
         subplot_data = data.loc[(site)]
-        subplot_data = subplot_data.sort_values(
-            ["magnitude_", "plot_order"], ascending=False
-        )
+        subplot_data = subplot_data.sort_values(["magnitude_", "plot_order"], ascending=False)
         new_y = [j * 1.25 for j in range(len(subplot_data))]
         # Stacked horizontal bar plots
+        ax[i].barh(y=new_y, width="TP_norm", left=0.0, color="#2c7bb6", linewidth=0.5, data=subplot_data)
         ax[i].barh(
-            y=new_y,
-            width="TP_norm",
-            left=0.0,
-            color="#2c7bb6",
-            linewidth=0.5,
-            data=subplot_data,
-        )
-        ax[i].barh(
-            y=new_y,
-            width="FN_norm",
-            left="TP_norm",
-            color="#fdae61",
-            linewidth=0.5,
-            data=subplot_data,
-        )
-        ax[i].barh(
-            y=new_y,
-            width="FP_norm",
-            left=1.0,
-            color="#d7191c",
-            linewidth=0.5,
-            data=subplot_data,
+            y=new_y, width="FN_norm", left="TP_norm", color="#fdae61", linewidth=0.5, data=subplot_data
         )
+        ax[i].barh(y=new_y, width="FP_norm", left=1.0, color="#d7191c", linewidth=0.5, data=subplot_data)
         # Plot the CSI and MCC scores
         ax[i].scatter(
-            y=new_y,
-            x=subplot_data["CSI"].array,
-            c="k",
-            s=15,
-            marker=r"x",
-            zorder=3,
-            linewidth=0.75,
+            y=new_y, x=subplot_data["CSI"].array, c="k", s=15, marker=r"x", zorder=3, linewidth=0.75
         )
         ax[i].scatter(
             y=new_y,
@@ -148,25 +117,15 @@ def eval_plot_stack_indiv(metric_csv, versions, outfig, category):
         )
         # Various axis customizations
         ax[i].set_yticks(
-            new_y,
-            labels=subplot_data.version
-            + " "
-            + subplot_data.magnitude.map(inverted_mag).astype(str),
-        )
-        ax[i].axis["left"].label.set(
-            visible=True, text=site, rotation=90, pad=10, ha="right"
+            new_y, labels=subplot_data.version + " " + subplot_data.magnitude.map(inverted_mag).astype(str)
         )
+        ax[i].axis["left"].label.set(visible=True, text=site, rotation=90, pad=10, ha="right")
         ax[i].axis["left"].major_ticks.set(tick_out=True)
         ax[i].axis["left"].major_ticklabels.set(ha="left")
         ax[i].set_xlim(0, xmax)
         ax[i].set_ylim(-1, new_y[-1] + 1)
         num_mags = len(subplot_data.magnitude.unique())
-        hlines = [
-            j
-            for j in np.linspace(
-                new_y[-1] / num_mags, new_y[-1], num_mags - 1, endpoint=False
-            )
-        ]
+        hlines = [j for j in np.linspace(new_y[-1] / num_mags, new_y[-1], num_mags - 1, endpoint=False)]
         ax[i].hlines(
             hlines,
             xmin=0,
@@ -191,12 +150,10 @@ def eval_plot_stack_indiv(metric_csv, versions, outfig, category):
             )
             ax[i].set_facecolor("0.67")
     plt.subplots_adjust(wspace=0, hspace=0)
-    ax[0].set_title(
-        f"{category.upper()} FIM Evaluation | Individual Sites", loc="center", pad=40
-    )
-    TP_patch = Patch(color="#2c7bb6", linewidth=0.5, label=f"True Positive")
-    FN_patch = Patch(color="#fdae61", linewidth=0.5, label=f"False Negative")
-    FP_patch = Patch(color="#d7191c", linewidth=0.5, label=f"False Positive")
+    ax[0].set_title(f"{category.upper()} FIM Evaluation | Individual Sites", loc="center", pad=40)
+    TP_patch = Patch(color="#2c7bb6", linewidth=0.5, label="True Positive")
+    FN_patch = Patch(color="#fdae61", linewidth=0.5, label="False Negative")
+    FP_patch = Patch(color="#d7191c", linewidth=0.5, label="False Positive")
     x_marker = Line2D(
         [0],
         [0],
@@ -220,9 +177,7 @@ def eval_plot_stack_indiv(metric_csv, versions, outfig, category):
         linewidth=10,
     )
     # Get the height of the figure in pixels so we can put the legend in a consistent position
-    ax_pixel_height = (
-        ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
-    )
+    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
     handles = [TP_patch, FN_patch, FP_patch, x_marker, o_marker]
     ax[0].legend(
         loc="center",
@@ -291,18 +246,11 @@ def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
             color="#2c7bb6",
             linewidth=0.5,
             data=subplot_data,
-            xerr=[subplot_data["TP_norm_q1"], subplot_data["TP_norm_q3"]]
-            if show_iqr
-            else None,
+            xerr=[subplot_data["TP_norm_q1"], subplot_data["TP_norm_q3"]] if show_iqr else None,
             error_kw=dict(elinewidth=1),
         )
         ax[i].barh(
-            y=new_y,
-            width="FN_norm",
-            left="TP_norm",
-            color="#fdae61",
-            linewidth=0.5,
-            data=subplot_data,
+            y=new_y, width="FN_norm", left="TP_norm", color="#fdae61", linewidth=0.5, data=subplot_data
         )
         ax[i].barh(
             y=new_y,
@@ -311,20 +259,12 @@ def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
             color="#d7191c",
             linewidth=0.5,
             data=subplot_data,
-            xerr=[subplot_data["FP_norm_q1"], subplot_data["FP_norm_q3"]]
-            if show_iqr
-            else None,
+            xerr=[subplot_data["FP_norm_q1"], subplot_data["FP_norm_q3"]] if show_iqr else None,
             error_kw=dict(elinewidth=1),
         )
         # Plot the CSI and MCC scores
         ax[i].scatter(
-            y=new_y,
-            x=subplot_data["CSI"].array,
-            c="k",
-            s=15,
-            marker=r"x",
-            zorder=3,
-            linewidth=0.75,
+            y=new_y, x=subplot_data["CSI"].array, c="k", s=15, marker=r"x", zorder=3, linewidth=0.75
         )
         ax[i].scatter(
             y=new_y,
@@ -338,15 +278,11 @@ def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
         )
         # Various axis customizations
         ax[i].set_yticks(new_y, labels=subplot_data.index)  # , ha='left')
-        ax[i].axis["left"].label.set(
-            visible=True, text=inverted_mag[mag], rotation=90, pad=10, ha="right"
-        )
+        ax[i].axis["left"].label.set(visible=True, text=inverted_mag[mag], rotation=90, pad=10, ha="right")
         ax[i].axis["left"].major_ticks.set(tick_out=True)
         ax[i].axis["left"].major_ticklabels.set(ha="left")
         n = count_df.loc[(category, versions[0], mag)]
-        ax[i].axis["right"].label.set(
-            visible=True, text=f"n={n}", rotation=270, pad=5, ha="left"
-        )
+        ax[i].axis["right"].label.set(visible=True, text=f"n={n}", rotation=270, pad=5, ha="left")
         ax[i].set_xlim(0, xmax)
         ax[i].set_ylim(-1.25, num_ver + 1.25)
         ax[i].grid(axis="x", color="0.8", zorder=0)
@@ -356,9 +292,9 @@ def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
         ax[i].set_facecolor("w" if i % 2 == 0 else "0.9")
     plt.subplots_adjust(wspace=0, hspace=0)
     ax[0].set_title(f"{category.upper()} FIM Evaluation", loc="center", pad=35)
-    TP_patch = Patch(color="#2c7bb6", linewidth=0.5, label=f"True Positive")
-    FN_patch = Patch(color="#fdae61", linewidth=0.5, label=f"False Negative")
-    FP_patch = Patch(color="#d7191c", linewidth=0.5, label=f"False Positive")
+    TP_patch = Patch(color="#2c7bb6", linewidth=0.5, label="True Positive")
+    FN_patch = Patch(color="#fdae61", linewidth=0.5, label="False Negative")
+    FP_patch = Patch(color="#d7191c", linewidth=0.5, label="False Positive")
     x_marker = Line2D(
         [0],
         [0],
@@ -382,9 +318,7 @@ def eval_plot_stack(metric_csv, versions, category, outfig, show_iqr=False):
         linewidth=10,
     )
     # Get the height of the figure in pixels so we can put the legend in a consistent position
-    ax_pixel_height = (
-        ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
-    )
+    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
     handles = [TP_patch, FN_patch, FP_patch, x_marker, o_marker]
     ax[0].legend(
         loc="center",
@@ -415,27 +349,28 @@ def diff_bar_plots(versions, metric_csv, category, outfig, stat="CSI"):
         "PND",
         "FAR",
     ), f"stat must be one of ('CSI', 'MCC', 'TPR', 'PND', 'FAR'), not {stat}"
-    assert (
-        len(versions) == 2
-    ), f"You can only produce the comparison plots when comparing 2 fim versions. You have input {len(versions)}."
-    # CSI, MCC, and TPR scores are better when closer to 1, however PND and FAR scores are better when closer to 0
+    assert len(versions) == 2, (
+        "You can only produce the comparison plots when comparing 2 fim versions. "
+        f"You have input {len(versions)}."
+    )
+    # CSI, MCC, and TPR scores are better when closer to 1,
+    # however PND and FAR scores are better when closer to 0
     better_score = "positive" if stat in ("CSI", "MCC", "TPR") else "negative"
 
     # Load in FIM 4 CSV and select the proper version metrics
     metrics = pd.read_csv(metric_csv, dtype={"huc": str})
     # Check to make sure requested versions are in the metrics file
     for v in versions:
-        assert (
-            v in metrics.version.unique()
-        ), f"{v} is not included in {metric_csv}.\nThe available options are {sorted(metrics.version.unique())}"
+        assert v in metrics.version.unique(), (
+            f"{v} is not included in {metric_csv}.\n"
+            f" The available options are {sorted(metrics.version.unique())}"
+        )
     # Change the magnitudes to make them more sort-friendly for the plot below
     metrics.magnitude = metrics.magnitude.apply(lambda m: mag_dict[m])
     # Fill the non-AHPS sites with values for nws_lid
     metrics.nws_lid.fillna(metrics.huc + "_" + metrics.benchmark_source, inplace=True)
     # Create multi-index for easy merging and plotting
-    metrics.set_index(
-        ["benchmark_source", "nws_lid", "magnitude"], inplace=True, drop=False
-    )
+    metrics.set_index(["benchmark_source", "nws_lid", "magnitude"], inplace=True, drop=False)
 
     # Separate versions into 2 dataframes
     metrics_base = metrics.loc[metrics.version == versions[0]].copy()
@@ -448,7 +383,8 @@ def diff_bar_plots(versions, metric_csv, category, outfig, stat="CSI"):
         metrics_new[stat], left_index=True, right_index=True, how="inner"
     ).sort_index(level=0, sort_remaining=True)
     fim_compare[f"{stat}_diff"] = fim_compare[stat] - fim_compare[f"BASE_{stat}"]
-    # Color the boxes according to improved vs regressed scores. See definition of 'better_score' above for details.
+    # Color the boxes according to improved vs regressed scores.
+    # See definition of 'better_score' above for details.
     if better_score == "positive":
         fim_compare["color"] = fim_compare.apply(
             lambda row: "None" if row[f"{stat}_diff"] < 0 else "g", axis=1
@@ -469,9 +405,7 @@ def diff_bar_plots(versions, metric_csv, category, outfig, stat="CSI"):
     num_subplots = len(data.nws_lid.unique())
     num_mags = len(data.magnitude.unique())
     # Create the plot
-    fig, ax = plt.subplots(
-        num_subplots, 1, figsize=(8, len(data) * 0.18), dpi=100, facecolor="white"
-    )
+    fig, ax = plt.subplots(num_subplots, 1, figsize=(8, len(data) * 0.18), dpi=100, facecolor="white")
     # Create a subplot for every site (nws_lid)
     for i, site in enumerate(data.nws_lid.unique()):
         subplot_data = data.loc[(site)]
@@ -486,8 +420,7 @@ def diff_bar_plots(versions, metric_csv, category, outfig, stat="CSI"):
             zorder=3,
         )
         ax[i].set_yticks(
-            np.arange(len(subplot_data)),
-            labels=subplot_data.index.map(inverted_mag).astype(str),
+            np.arange(len(subplot_data)), labels=subplot_data.index.map(inverted_mag).astype(str)
         )
         ax[i].set_xticks(np.linspace(0, 1, 11))
         ax[i].set_ylabel(site, rotation="horizontal", labelpad=40)
@@ -506,28 +439,16 @@ def diff_bar_plots(versions, metric_csv, category, outfig, stat="CSI"):
         )
         # Label sites that have been identified as "Bad"
         if site and site in BAD_SITES:
-            ax[i].text(
-                0.1,
-                1.5,
-                "--BAD SITE--",
-                horizontalalignment="center",
-                verticalalignment="center",
-            )
+            ax[i].text(0.1, 1.5, "--BAD SITE--", horizontalalignment="center", verticalalignment="center")
             ax[i].set_facecolor("0.67")
     plt.subplots_adjust(wspace=0, hspace=0)
     ax[0].set_title(
-        f"{category.upper()} {stat} Comparison\n{versions[0]}  v  {versions[1]}",
-        loc="center",
-        pad=45,
+        f"{category.upper()} {stat} Comparison\n{versions[0]}  v  {versions[1]}", loc="center", pad=45
     )
     g_patch = Patch(color="g", linewidth=0.5, label=f"{stat} Score Improvement")
-    r_patch = Patch(
-        facecolor="None", edgecolor="r", linewidth=0.5, label=f"{stat} Score Regression"
-    )
+    r_patch = Patch(facecolor="None", edgecolor="r", linewidth=0.5, label=f"{stat} Score Regression")
     # Get the height of the figure in pixels so we can put the legend in a consistent position
-    ax_pixel_height = (
-        ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
-    )
+    ax_pixel_height = ax[0].get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
     ax[0].legend(
         loc="center",
         ncol=2,
@@ -541,12 +462,7 @@ def diff_bar_plots(versions, metric_csv, category, outfig, stat="CSI"):
 
 
 def iter_benchmarks(
-    metric_csv,
-    workspace,
-    versions=[],
-    individual_plots=False,
-    show_iqr=False,
-    diff_stat=False,
+    metric_csv, workspace, versions=[], individual_plots=False, show_iqr=False, diff_stat=False
 ):
     # Import metrics csv as DataFrame and initialize all_datasets dictionary
     csv_df = pd.read_csv(metric_csv, dtype={"huc": str})
@@ -560,46 +476,31 @@ def iter_benchmarks(
 
     # Group by benchmark source
     benchmark_by_source = metrics.groupby(["benchmark_source", "extent_config"])
-    for (
-        benchmark_source,
-        extent_configuration,
-    ), benchmark_metrics in benchmark_by_source:
+    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:
         # Define and create the output workspace as a subfolder within the supplied workspace
-        output_workspace = (
-            Path(workspace) / benchmark_source / extent_configuration.lower()
-        )
+        output_workspace = Path(workspace) / benchmark_source / extent_configuration.lower()
         output_workspace.mkdir(parents=True, exist_ok=True)
-        output_png = (
-            Path(output_workspace)
-            / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar{'_indiv'if individual_plots else ''}.png"
+        output_png = Path(output_workspace) / (
+            f"{benchmark_source}_{extent_configuration.lower()}_"
+            f"stackedbar{'_indiv'if individual_plots else ''}.png"
         )
         if diff_stat:
             output_png = (
                 Path(output_workspace)
                 / f"{benchmark_source}_{extent_configuration.lower()}_diff_plot_{diff_stat}.png"
             )
-            diff_bar_plots(
-                versions,
-                metric_csv,
-                category=benchmark_source,
-                outfig=output_png,
-                stat=diff_stat,
-            )
+            diff_bar_plots(versions, metric_csv, category=benchmark_source, outfig=output_png, stat=diff_stat)
         elif individual_plots:
             output_png = (
                 Path(output_workspace)
                 / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar_indiv.png"
             )
             eval_plot_stack_indiv(
-                metric_csv=metric_csv,
-                versions=versions,
-                category=benchmark_source,
-                outfig=output_png,
+                metric_csv=metric_csv, versions=versions, category=benchmark_source, outfig=output_png
             )
         else:
             output_png = (
-                Path(output_workspace)
-                / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar.png"
+                Path(output_workspace) / f"{benchmark_source}_{extent_configuration.lower()}_stackedbar.png"
             )
             eval_plot_stack(
                 metric_csv=metric_csv,
@@ -613,19 +514,19 @@ def iter_benchmarks(
 if __name__ == "__main__":
     # Parse arguments
     parser = argparse.ArgumentParser(
-        description=f"Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)"
+        description="Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)"
     )
     parser.add_argument(
-        "-m",
-        "--metric_csv",
-        help="Metrics csv created from synthesize test cases.",
-        required=True,
+        "-m", "--metric_csv", help="Metrics csv created from synthesize test cases.", required=True
     )
     parser.add_argument("-w", "--workspace", help="Output workspace", required=True)
     parser.add_argument(
         "-v",
         "--versions",
-        help='List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.',
+        help='List of versions to be plotted/aggregated. Versions are filtered using the "startswith" '
+        'approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" '
+        '(e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". '
+        'An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.',
         nargs="+",
         default=[],
     )
@@ -646,8 +547,9 @@ def iter_benchmarks(
     parser.add_argument(
         "-d",
         "--diff-stat",
-        help="This option creates diff plots instead of stacked bar plots. Only 2 versions can be used with this option. "
-        + "Input one of the following statistics to be used for comparison: ('CSI', 'MCC', 'TPR', 'PND', 'FAR').",
+        help="This option creates diff plots instead of stacked bar plots. "
+        "Only 2 versions can be used with this option. Input one of the following statistics "
+        "to be used for comparison: ('CSI', 'MCC', 'TPR', 'PND', 'FAR').",
         default=False,
         required=False,
     )
@@ -664,7 +566,5 @@ def iter_benchmarks(
     d = args["diff_stat"]
 
     # Run eval_plots function
-    print(
-        'The following AHPS sites are considered "BAD_SITES":  ' + ", ".join(BAD_SITES)
-    )
+    print('The following AHPS sites are considered "BAD_SITES":  ' + ", ".join(BAD_SITES))
     iter_benchmarks(m, w, v, i, iqr, d)
diff --git a/tools/evaluate_continuity.py b/tools/evaluate_continuity.py
index b16c6b140..85baffeb9 100644
--- a/tools/evaluate_continuity.py
+++ b/tools/evaluate_continuity.py
@@ -1,34 +1,34 @@
 #!/usr/bin/env python3
 
-import geopandas as gpd
-import pandas as pd
-import numpy as np
 import argparse
+
+import geopandas as gpd
 import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
 
-def evaluate_continuity(stream_network_file,forecast_file,stream_network_outfile=None,
-                        confluences_only=False,plot_file=None):
 
+def evaluate_continuity(
+    stream_network_file, forecast_file, stream_network_outfile=None, confluences_only=False, plot_file=None
+):
     stream_network = gpd.read_file(stream_network_file)
     forecast = pd.read_csv(forecast_file)
-    
-    stream_network = stream_network.merge(forecast,how='left',left_on='feature_id',right_on='feature_id')
-    
+
+    stream_network = stream_network.merge(forecast, how='left', left_on='feature_id', right_on='feature_id')
+
     stream_network['discharge'] = stream_network['discharge'].fillna(0)
 
- 
     toNodes = set(stream_network['To_Node'].tolist())
 
     # upstream dictionary and confluences
-    upstream_dictionary,confluences = {},set()
-    for idx,reach in stream_network.iterrows():
-    
+    upstream_dictionary, confluences = {}, set()
+    for idx, reach in stream_network.iterrows():
         fromNode = reach['From_Node']
-        hydroId= reach['HydroID']
-        
+        hydroId = reach['HydroID']
+
         if fromNode in toNodes:
             upstream_indices = stream_network['To_Node'] == fromNode
-            upstream_discharges = np.array( stream_network.loc[upstream_indices,'discharge'].tolist() )
+            upstream_discharges = np.array(stream_network.loc[upstream_indices, 'discharge'].tolist())
             upstream_dictionary[hydroId] = upstream_discharges
 
             isconfluence = len(upstream_discharges) > 1
@@ -38,15 +38,14 @@ def evaluate_continuity(stream_network_file,forecast_file,stream_network_outfile
     # filter out non-confluences
     if confluences_only:
         hydroIDs = stream_network['HydroID'].tolist()
-        confluence_bool = np.array(  [True if h in confluences else False for h in hydroIDs]  )
-        stream_network = stream_network.loc[confluence_bool,:]
+        confluence_bool = np.array([True if h in confluences else False for h in hydroIDs])
+        stream_network = stream_network.loc[confluence_bool, :]
 
     actual_discharges, expected_discharges = [], []
     expected_dischages_dict = dict()
-    for idx,reach in stream_network.iterrows():
-    
+    for idx, reach in stream_network.iterrows():
         hydroId = reach['HydroID']
-        
+
         try:
             upstream_discharges = upstream_dictionary[hydroId]
         except KeyError:
@@ -57,38 +56,44 @@ def evaluate_continuity(stream_network_file,forecast_file,stream_network_outfile
         expected_discharges += [np.sum(upstream_discharges)]
         expected_dischages_dict[hydroId] = np.sum(upstream_discharges)
 
-
     actual_discharges, expected_discharges = np.array(actual_discharges), np.array(expected_discharges)
 
     # add to stream_network
-    expected_discharges_df = pd.DataFrame.from_dict(expected_dischages_dict,orient='index',
-                                                    columns=['expected_discharges'])
-    stream_network = stream_network.merge(expected_discharges_df, left_on='HydroID', right_index=True, how='left')
-    
+    expected_discharges_df = pd.DataFrame.from_dict(
+        expected_dischages_dict, orient='index', columns=['expected_discharges']
+    )
+    stream_network = stream_network.merge(
+        expected_discharges_df, left_on='HydroID', right_index=True, how='left'
+    )
+
     number_of_reaches = len(stream_network)
-    SMAPE = smape(actual_discharges,expected_discharges)
+    SMAPE = smape(actual_discharges, expected_discharges)
     diff = actual_discharges - expected_discharges
 
-    print("Number of No Flow Reaches: {} out of {}".format((stream_network['discharge']==0).sum(),number_of_reaches))
+    print(
+        "Number of No Flow Reaches: {} out of {}".format(
+            (stream_network['discharge'] == 0).sum(), number_of_reaches
+        )
+    )
     print("SMAPE = {}%".format(SMAPE[0]))
-    print("Diff (<0) = {}".format(np.sum(diff<0)))
-    print("Diff (>0) = {}".format(np.sum(diff>0)))
-    print("Diff (=0) = {}".format(np.sum(diff==0)))
-    print("Diff (>-10,<10) = {}".format(np.sum(np.logical_or(diff>-10,diff<10))))
+    print("Diff (<0) = {}".format(np.sum(diff < 0)))
+    print("Diff (>0) = {}".format(np.sum(diff > 0)))
+    print("Diff (=0) = {}".format(np.sum(diff == 0)))
+    print("Diff (>-10,<10) = {}".format(np.sum(np.logical_or(diff > -10, diff < 10))))
     print("Median diff: {}".format(np.nanmedian(diff)))
     print("Mean diff: {}".format(np.nanmean(diff)))
 
     if confluences_only:
         nbins = 50
-        xlim = (-2000,2000)
-        ylim = (0,50)
+        xlim = (-2000, 2000)
+        ylim = (0, 50)
         title = 'Discharge Errors (CMS) At Confluence Reaches'
     else:
         nbins = 500
-        xlim = (-450,450)
-        ylim = (0,60)
+        xlim = (-450, 450)
+        ylim = (0, 60)
         title = 'Discharge Errors (CMS)'
-    
+
     fig = plt.figure(1)
     ax = plt.subplot(111)
     n, bins, patches = ax.hist(diff, nbins, facecolor='blue', alpha=0.4)
@@ -100,36 +105,41 @@ def evaluate_continuity(stream_network_file,forecast_file,stream_network_outfile
     plt.title(title)
     plt.xlabel('Discharge Errors (CMS) = actual - expected')
     plt.ylabel('Count (Peak not show)')
-    
+
     if plot_file is not None:
         fig.savefig(plot_file)
 
     if stream_network_outfile is not None:
-        stream_network.to_file(stream_network_outfile,index=False,driver='GPKG')
-
-    return(stream_network)
+        stream_network.to_file(stream_network_outfile, index=False, driver='GPKG')
 
+    return stream_network
 
-def smape(predicted,actual):
 
-    assert len(predicted) == len(actual),"Predicted and actual need to have same length"
+def smape(predicted, actual):
+    assert len(predicted) == len(actual), "Predicted and actual need to have same length"
 
-    sape = 100 * (np.abs(predicted-actual) / (np.abs(predicted) + np.abs(actual)))
+    sape = 100 * (np.abs(predicted - actual) / (np.abs(predicted) + np.abs(actual)))
 
-    return (np.nanmean(sape),sape)
+    return (np.nanmean(sape), sape)
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Evaluating continuity')
-    parser.add_argument('-s','--stream-network-file',help='Stream Network',required=True)
-    parser.add_argument('-f','--forecast-file',help='Forecast File',required=True)
-    parser.add_argument('-o','--stream-network-outfile',help='Stream Network Outfile',required=False,default=None)
-    parser.add_argument('-c','--confluences-only',help='Only at confluence reaches',required=False,default=False,action='store_true')
-    parser.add_argument('-p','--plot-file',help='Plot File',required=False,default=None)
-
-
+    parser.add_argument('-s', '--stream-network-file', help='Stream Network', required=True)
+    parser.add_argument('-f', '--forecast-file', help='Forecast File', required=True)
+    parser.add_argument(
+        '-o', '--stream-network-outfile', help='Stream Network Outfile', required=False, default=None
+    )
+    parser.add_argument(
+        '-c',
+        '--confluences-only',
+        help='Only at confluence reaches',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument('-p', '--plot-file', help='Plot File', required=False, default=None)
 
     args = vars(parser.parse_args())
 
diff --git a/tools/fim_completion_check.py b/tools/fim_completion_check.py
index 6623e4f1c..f78350a2f 100755
--- a/tools/fim_completion_check.py
+++ b/tools/fim_completion_check.py
@@ -1,14 +1,13 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import os
 
 
 if __name__ == '__main__':
-
     parser = argparse.ArgumentParser(description='Checks final FIM outputs to identify missing HUCs')
-    parser.add_argument('-i','--huc-list-dir', help='list of HUCs to run', required=True)
-    parser.add_argument('-o','--output-folder', help='directory of HUCs completed', required=True)
+    parser.add_argument('-i', '--huc-list-dir', help='list of HUCs to run', required=True)
+    parser.add_argument('-o', '--output-folder', help='directory of HUCs completed', required=True)
 
     args = vars(parser.parse_args())
 
@@ -18,7 +17,6 @@
     if not os.path.isfile(huc_list_dir):
         huc_list = huc_list_dir.split()
     else:
-
         with open(huc_list_dir) as f:
             huc_list = f.read().splitlines()
 
@@ -33,4 +31,4 @@
     missing_hucs = list(set(huc_list) - set(output_huc_list))
 
     if len(missing_hucs) > 0:
-        print (f"MISSING {len(missing_hucs)} HUCS from outputs: {missing_hucs}")
+        print(f"MISSING {len(missing_hucs)} HUCS from outputs: {missing_hucs}")
diff --git a/tools/fimr_to_benchmark.py b/tools/fimr_to_benchmark.py
index 90639c4fa..f10dac19a 100644
--- a/tools/fimr_to_benchmark.py
+++ b/tools/fimr_to_benchmark.py
@@ -1,37 +1,46 @@
+#!/usr/bin/env python3
 
+import argparse
+import os
+
+# from rasterio import features
+import affine
 import geopandas as gpd
 import pandas as pd
-import os
-from tools_shared_functions import get_metadata
-from dotenv import load_dotenv
-load_dotenv()
-import argparse
 import rasterio
-#from rasterio import features
-import affine
 import rasterio.mask
+from dotenv import load_dotenv
+from tools_shared_functions import get_metadata
+
+
+load_dotenv()
+
 
 def fimr_to_benchmark(fimr_path, output_path):
     '''
-    This function converts USFIMR shapefiles from this website (https://waterserv.ua.edu/datasets/usfimr/) to rasters that c an 
-    be used for FIM Alpha evaluations and calibration. Note only USFIMR data with flow values (q) can be used here. 
+    This function converts USFIMR shapefiles from this website (https://waterserv.ua.edu/datasets/usfimr/)
+    to rasters that can be used for FIM Alpha evaluations and calibration.
+    Note only USFIMR data with flow values (q) can be used here.
 
     Inputs: 1. FIMR Path - Path where th FIMR Shapefile was downloaded from the aformentioned website.
-            2. Output Path_flow file - Path the flow file will be sent to. 
-            3. Output path raster - path the raster will be sent too. There will be a file for each huc with the huc number in it.
+            2. Output Path_flow file - Path the flow file will be sent to.
+            3. Output path raster - path the raster will be sent too. There will be a file for each huc with
+                the huc number in it.
 
-    Outputs: 1. flow_file.csv - This is a flow file for the river. The flow values are from the most upstream usgs gage.
-             2. rasterized{huc number}.tif - This is the fimr data that has been converted to a raster. It is on the huc8 scale so it provides
-                each huc as a separate file. The inundated areas are assigned 1 and the dry areas are assigned 0.
+    Outputs: 1. flow_file.csv - This is a flow file for the river. The flow values are from the most upstream
+                usgs gage.
+             2. rasterized{huc number}.tif - This is the fimr data that has been converted to a raster.
+                    It is on the huc8 scale so it provides each huc as a separate file.
+                    The inundated areas are assigned 1 and the dry areas are assigned 0.
 
     Other Files Used: 1. wbd - The huc 8 boundaries throughout the country
                       2. flowlines - the flowlines along the all the rivers in the country
     '''
     # input validation
-    if (not os.path.exists(fimr_path)):
+    if not os.path.exists(fimr_path):
         raise Exception(f"Sorry. {fimr_path} does not exist")
 
-    if (not os.path.exists(output_path)):
+    if not os.path.exists(output_path):
         raise Exception(f"Sorry. {output_path} does not exist")
 
     API_BASE_URL = os.getenv("API_BASE_URL")
@@ -41,120 +50,143 @@ def fimr_to_benchmark(fimr_path, output_path):
     flowlines_path = '/data/inputs/nwm_hydrofabric/nwm_flows_ms_wrds.gpkg'
 
     # Saving wbd, flowlines, and fimr data as variables
-    wbd = gpd.read_file(wbd_path, layer = 'WBDHU8')
+    wbd = gpd.read_file(wbd_path, layer='WBDHU8')
     flowlines = gpd.read_file(flowlines_path)
     fimr = gpd.read_file(fimr_path)
 
     # Joining the flowlines and fimr to output only the flowlines in the inundated area
     joined_flowlines_fimr = gpd.sjoin(flowlines, fimr)
-    
+
     # Renaming index_right to prevent it from causing an error in the next spatial join
-    joined_flowlines_fimr = joined_flowlines_fimr.rename(columns = {'index_right':'right_index'})
+    joined_flowlines_fimr = joined_flowlines_fimr.rename(columns={'index_right': 'right_index'})
     # Getting a list of all the unique feature Ids from the joined flowlines
     unique_feature_ids = joined_flowlines_fimr.drop_duplicates('ID')
     # Changing cfs to cms
-    unique_feature_ids['Q_at_Image_cms'] = unique_feature_ids['Q_at_Image']*0.028317
+    unique_feature_ids['Q_at_Image_cms'] = unique_feature_ids['Q_at_Image'] * 0.028317
     # Make a discharge dataframe with only the feture ID and the Q values
-    feature_discharge = unique_feature_ids[['ID','Q_at_Image_cms']]
+    feature_discharge = unique_feature_ids[['ID', 'Q_at_Image_cms']]
 
-    #flow_file_path = os.path.join(output_path, 'flow_file.csv')
-    #feature_discharge.to_csv(flow_file_path, header = ['feature_id', 'discharge'],index = False)
-    #print('flow file created at ',flow_file_path)
+    # flow_file_path = os.path.join(output_path, 'flow_file.csv')
+    # feature_discharge.to_csv(flow_file_path, header = ['feature_id', 'discharge'],index = False)
+    # print('flow file created at ',flow_file_path)
 
-    # Takes the huc 8s that overlap the flowlines for the river 
+    # Takes the huc 8s that overlap the flowlines for the river
     huc_joined = gpd.sjoin(wbd, joined_flowlines_fimr)
     huc_list = huc_joined.HUC8.unique()
     # Overlay the hucs
     fimr_hucs = wbd.loc[wbd.HUC8.isin(huc_list)]
-    fimr_huc_union = gpd.overlay(joined_flowlines_fimr, fimr_hucs, how = 'union')
-    fimr_huc_union = fimr_huc_union.rename(columns = {'Shape_Length':'Shapelength'})
+    fimr_huc_union = gpd.overlay(joined_flowlines_fimr, fimr_hucs, how='union')
+    fimr_huc_union = fimr_huc_union.rename(columns={'Shape_Length': 'Shapelength'})
     # Renames Shape_Length as Shapelength to eliminate a duplicate column and let it be exported
-    huc_joined = huc_joined.rename(columns = {'Shape_Length':'Shapelength'})
+    huc_joined = huc_joined.rename(columns={'Shape_Length': 'Shapelength'})
 
-    # Takes the usgs location ids 
+    # Takes the usgs location ids
     gages = fimr['USGS_Gage']
-    split_gages = gages.iloc[0].split( ", ")
+    split_gages = gages.iloc[0].split(", ")
     # Takes the first usgs gauge idea that has the streamflow data for the whole fimr area
-    first_gage = (split_gages[0])
- 
+    first_gage = split_gages[0]
+
     # Get metadata for all usgs_site_codes that are active in the U.S.
-    metadata_url = f'{API_BASE_URL}/metadata' 
+    metadata_url = f'{API_BASE_URL}/metadata'
     # Define arguments to retrieve metadata and then get metadata from WRDS
     select_by = 'usgs_site_code'
-    selector = [first_gage] #change to usgs location id from fimr dataframe
+    selector = [first_gage]  # change to usgs location id from fimr dataframe
     must_include = 'usgs_data.active'
-    metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = None )
-   
+    metadata_list, metadata_df = get_metadata(
+        metadata_url,
+        select_by,
+        selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=None,
+    )
+
     # Takes the stream order at the gauge
     stream_order = metadata_df['nwm_feature_data_stream_order']
     stream_order_float = stream_order[0]
 
     # Gets flow lines where the streasm order is equal to the magnitude of the mainstem
     correct_flow = joined_flowlines_fimr.loc[joined_flowlines_fimr['order_'] == stream_order_float]
-    
+
     # Exploding the fimr inundaton polygon
     exploded_fimr = fimr.explode(index_parts=True)
     # Taking only the fimr polygons that overlap the flowlines to minimize noise
     clean_fimr = gpd.sjoin(exploded_fimr, correct_flow)
     # Renaming columns to prevent duplicate column names in overlay
-    clean_fimr = clean_fimr.rename(columns = {'index_right':'index_from_right'})
+    clean_fimr = clean_fimr.rename(columns={'index_right': 'index_from_right'})
 
     # Overlaying the cleaned fimr data and the hucs it is in
-    fimr_huc_union = gpd.overlay(clean_fimr, fimr_hucs, how = 'union')
-    fimr_huc_union = fimr_huc_union.rename(columns = {'Shape_Length':'Shapelength'})
+    fimr_huc_union = gpd.overlay(clean_fimr, fimr_hucs, how='union')
+    fimr_huc_union = fimr_huc_union.rename(columns={'Shape_Length': 'Shapelength'})
     # Renames Shape_Length as Shapelength to eliminate a duplicate column and let it be exported
-    huc_joined = huc_joined.rename(columns = {'Shape_Length':'Shapelength'})
-    
-    # Looping through the three huc8s, getting the fimr data in each 
+    huc_joined = huc_joined.rename(columns={'Shape_Length': 'Shapelength'})
+
+    # Looping through the three huc8s, getting the fimr data in each
     for huc in huc_list:
-      
         fimr_huc = fimr_huc_union.loc[fimr_huc_union.HUC8 == huc]
         shapes = []
 
-        for index,row in fimr_huc.iterrows():
-
+        for index, row in fimr_huc.iterrows():
             if pd.isna(row.right_index):
                 value = 0
             else:
                 value = 1
-            
+
             shapes.append((row.geometry, value))
 
-        #Outputting Flow File
-    
+        # Outputting Flow File
+
         flow_file_path = os.path.join(output_path, 'fimr_huc_' + huc + '_flows.csv')
-        feature_discharge.to_csv(flow_file_path, header = ['feature_id', 'discharge'],index = False)
-        print('flow file created at',flow_file_path)
-        # output path for the rasters, float causes it to keep all the digits 
-        out_path_raster = os.path.join(output_path,'fimr_huc_' + huc + '_extent.tif')
+        feature_discharge.to_csv(flow_file_path, header=['feature_id', 'discharge'], index=False)
+        print('flow file created at', flow_file_path)
+        # output path for the rasters, float causes it to keep all the digits
+        out_path_raster = os.path.join(output_path, 'fimr_huc_' + huc + '_extent.tif')
 
-        #Rasterizing
+        # Rasterizing
 
         minx, miny, maxx, maxy = fimr_huc.geometry.total_bounds
         transform = affine.Affine(10, 0, minx, 0, -10, maxy)
 
         # Makeing the empty raster
-        raster_shape = (int((maxy - miny)//10), int((maxx - minx)//10))
-        raster = rasterio.features.rasterize(shapes,
-                                     out_shape = raster_shape,
-                                     transform = transform,
-                                     fill = -999)
-    
-        with rasterio.open(out_path_raster, 'w', driver = 'GTiff', height = raster_shape[0], width = raster_shape[1], count = 1, dtype = raster.dtype, nodata = -999,
-            crs = '+proj=aea +lat_0=23 +lon_0=-96 +lat_1=29.5 +lat_2=45.5 +x_0=0 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs',transform = transform) as dst_dataset:
+        raster_shape = (int((maxy - miny) // 10), int((maxx - minx) // 10))
+        raster = rasterio.features.rasterize(shapes, out_shape=raster_shape, transform=transform, fill=-999)
+
+        with rasterio.open(
+            out_path_raster,
+            'w',
+            driver='GTiff',
+            height=raster_shape[0],
+            width=raster_shape[1],
+            count=1,
+            dtype=raster.dtype,
+            nodata=-999,
+            crs=(
+                '+proj=aea +lat_0=23 +lon_0=-96 +lat_1=29.5 +lat_2=45.5 +x_0=0 '
+                '+y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs'
+            ),
+            transform=transform,
+        ) as dst_dataset:
             # writing array into the empty raster
             dst_dataset.write_band(1, raster)
             print('Rasterized fimr file created at ', out_path_raster)
 
-     # Parse Arguments   
-if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description = 'Get FIMR data for the different rivers from the different files')
-    
-    parser.add_argument('-f','--fimr_path', type = str, help = 'the path for the shapefile inputed from the FIMR dastabase', required = True) 
-    parser.add_argument('-o','--output_path', type = str, help = 'output path for the flow file and FIMR rasters.', required = True)
+if __name__ == '__main__':
+    # Parse Arguments
+    parser = argparse.ArgumentParser(
+        description='Get FIMR data for the different rivers from the different files'
+    )
+
+    parser.add_argument(
+        '-f',
+        '--fimr_path',
+        type=str,
+        help='the path for the shapefile inputed from the FIMR dastabase',
+        required=True,
+    )
+    parser.add_argument(
+        '-o', '--output_path', type=str, help='output path for the flow file and FIMR rasters.', required=True
+    )
     args = vars(parser.parse_args())
 
     fimr_to_benchmark(**args)
-
-
diff --git a/tools/find_max_catchment_breadth.py b/tools/find_max_catchment_breadth.py
index e8da34aa8..0f7405399 100755
--- a/tools/find_max_catchment_breadth.py
+++ b/tools/find_max_catchment_breadth.py
@@ -1,48 +1,46 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
-import pandas as pd
+from glob import glob, iglob
+
 import geopandas as gpd
-from glob import glob,iglob
-import argparse
+import pandas as pd
 from shapely.geometry import Point
 
 
 def Find_max_catchment_breadth(hydrofabric_dir):
-
     catchments_fileNames = glob_file_paths_for_catchments(hydrofabric_dir)
 
     list_with_max_lengths_within_files = []
 
-    for idx,cfn in enumerate(catchments_fileNames):
-
+    for idx, cfn in enumerate(catchments_fileNames):
         catchments = gpd.read_file(cfn)
 
-        lengths = catchments.apply(get_length_and_width_of_poly,axis=1)
+        lengths = catchments.apply(get_length_and_width_of_poly, axis=1)
 
         max_length_within_file = lengths.median()
 
         list_with_max_lengths_within_files.append(max_length_within_file)
 
-        print(idx,max_length_within_file)
-
+        print(idx, max_length_within_file)
 
     print(max(list_with_max_lengths_within_files))
 
 
 def glob_file_paths_for_catchments(hydrofabric_dir):
-
-    file_pattern_to_glob = os.path.join(hydrofabric_dir,'**','gw_catchments_reaches_filtered_addedAttributes_crosswalked*.gpkg')
+    file_pattern_to_glob = os.path.join(
+        hydrofabric_dir, '**', 'gw_catchments_reaches_filtered_addedAttributes_crosswalked*.gpkg'
+    )
 
     catchments_fileNames = iglob(file_pattern_to_glob)
 
-    return(catchments_fileNames)
+    return catchments_fileNames
 
 
 def get_length_and_width_of_poly(geodataframe_row):
-
     poly = geodataframe_row['geometry']
-    
+
     # get minimum bounding box around polygon
     box = poly.minimum_rotated_rectangle
 
@@ -50,24 +48,27 @@ def get_length_and_width_of_poly(geodataframe_row):
     x, y = box.exterior.coords.xy
 
     # get length of bounding box edges
-    edge_length = (Point(x[0], y[0]).distance(Point(x[1], y[1])), Point(x[1], y[1]).distance(Point(x[2], y[2])))
+    edge_length = (
+        Point(x[0], y[0]).distance(Point(x[1], y[1])),
+        Point(x[1], y[1]).distance(Point(x[2], y[2])),
+    )
 
     # get length of polygon as the longest edge of the bounding box
     length = max(edge_length)
 
     # get width of polygon as the shortest edge of the bounding box
-    #width = min(edge_length)
+    # width = min(edge_length)
 
-    return(length)
+    return length
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Find the maximum breadth of catchments in given hydrofabric directory')
-    parser.add_argument('-y','--hydrofabric-dir',help='Path to hydrofabric directory',required=True)
+    parser = argparse.ArgumentParser(
+        description='Find the maximum breadth of catchments in given hydrofabric directory'
+    )
+    parser.add_argument('-y', '--hydrofabric-dir', help='Path to hydrofabric directory', required=True)
 
     args = vars(parser.parse_args())
 
     Find_max_catchment_breadth(**args)
-
diff --git a/tools/find_test_case_folders.py b/tools/find_test_case_folders.py
index c71d397e4..6c6b1db31 100644
--- a/tools/find_test_case_folders.py
+++ b/tools/find_test_case_folders.py
@@ -1,55 +1,59 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
+import os
 import re
 
-def create_huc_list(input_root_search_folder,
-                    output_path,
-                    overwrite=False):
 
+def create_huc_list(input_root_search_folder, output_path, overwrite=False):
     '''
     Summary: This scans an input directory, such as test_cases, and looks for each
          unique huc value through those folders. This gives us a single list of all hucs
-         that can be used for alpha or sierra tests. 
+         that can be used for alpha or sierra tests.
          This looks for first level folders using the following convention:
             - *_test_cases  (ie usgs_test_cases)
                 Then subfolders using the following convention:
                 - {8 numbers}_{3 to 7 characters}. ie) 01080005_usgs
     Input:
-        - input_root_search_folder: A fully qualified root path (relative to Docker pathing) to the folder that all hucs
-            are in.
+        - input_root_search_folder: A fully qualified root path (relative to Docker pathing)
+            to the folder that all hucs are in.
         - output_path: a path and file name (preferred as .lst or .txt) where the list of line delimited hucs
             will be copied. The file and path do not need to pre-exist.
-        - overwrite: If the file exists and the overwrite flag is false, an error will be issued. Else, it will
-            be fully overwritten.
+        - overwrite: If the file exists and the overwrite flag is false, an error will be issued.
+            Else, it will be fully overwritten.
     Output:
         - a line delimited list of all huc numbers that have test cases available.
     '''
 
-    if (not os.path.exists(input_root_search_folder)):
+    if not os.path.exists(input_root_search_folder):
         raise NotADirectoryError(f"Sorry. Search_folder of {input_root_search_folder} does not exist")
 
-    if (os.path.exists(output_path) and not overwrite):
+    if os.path.exists(output_path) and not overwrite:
         raise Exception(f"Sorry. The file {output_path} already exists. Use 'overwrite' argument if desired.")
 
     hucs = set()
 
-    for test_case in [test_case for test_case in os.listdir(input_root_search_folder) 
-        if re.match('.*_test_cases', test_case)]:
-            for test_id in [test_id for test_id in os.listdir(os.path.join(input_root_search_folder, test_case))
-                if re.match('\d{8}_\w{3,7}', test_id)]:
-                    hucs.add(test_id[:8])
+    for test_case in [
+        test_case
+        for test_case in os.listdir(input_root_search_folder)
+        if re.match('.*_test_cases', test_case)
+    ]:
+        for test_id in [
+            test_id
+            for test_id in os.listdir(os.path.join(input_root_search_folder, test_case))
+            if re.match(r'\d{8}_\w{3,7}', test_id)
+        ]:
+            hucs.add(test_id[:8])
 
     sorted_hucs = sorted(hucs)
-    #print(sorted_hucs)
+    # print(sorted_hucs)
     print(f"{str(len(sorted_hucs))} hucs found")
-    
+
     os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
     # Save to disk
-    with (open(output_path, 'w')) as output_file:
-        #output_file.writelines(sorted_hucs)
+    with open(output_path, 'w') as output_file:
+        # output_file.writelines(sorted_hucs)
         for huc in sorted_hucs:
             output_file.write(huc)
             output_file.write('\n')
@@ -58,19 +62,34 @@ def create_huc_list(input_root_search_folder,
 
 
 if __name__ == '__main__':
+    '''
+    Sample Usage:
+    python /foss_fim/tools/find_test_case_hucs.py
+        -d /data/test_cases/
+        -f /some_directory/huc_list_for_tests_22020420.lst
+        -o
 
-# Sample Usage: python /foss_fim/tools/find_test_case_hucs.py -d /data/test_cases/ -f /some_directory/huc_list_for_tests_22020420.lst -o
+    '''
 
     parser = argparse.ArgumentParser(description='Finds all unique hucs that have test case data.')
-    parser.add_argument('-d','--input_root_search_folder', 
-                            help='Root folder to be scanned for unique hucs that have test case',
-                            required=True)
-    parser.add_argument('-f','--output_path',
-                            help='Folder path and file name to be saved (.txt or .lst suggested).',
-                            required=True)
-    parser.add_argument('-o','--overwrite', 
-                            help='Overwrite the file if already existing? (default false)',
-                            action='store_true' )
+    parser.add_argument(
+        '-d',
+        '--input_root_search_folder',
+        help='Root folder to be scanned for unique hucs that have test case',
+        required=True,
+    )
+    parser.add_argument(
+        '-f',
+        '--output_path',
+        help='Folder path and file name to be saved (.txt or .lst suggested).',
+        required=True,
+    )
+    parser.add_argument(
+        '-o',
+        '--overwrite',
+        help='Overwrite the file if already existing? (default false)',
+        action='store_true',
+    )
 
     args = vars(parser.parse_args())
 
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index a51f8df0a..3749bb4fe 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -1,92 +1,132 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
 import csv
-import traceback
+import glob
+import os
 import sys
-from datetime import datetime
 import time
+import traceback
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+from datetime import datetime
 from pathlib import Path
+
 import geopandas as gpd
+import numpy as np
 import pandas as pd
 import rasterio
-from rasterio.warp import calculate_default_transform, reproject, Resampling
-import glob
 from generate_categorical_fim_flows import generate_catfim_flows
 from generate_categorical_fim_mapping import manage_catfim_mapping, post_process_cat_fim_for_viz
-from tools_shared_functions import get_thresholds, get_nwm_segs, get_datum, ngvd_to_navd_ft, filter_nwm_segments_by_stream_order
-from concurrent.futures import ProcessPoolExecutor, as_completed, wait
-import numpy as np
+from rasterio.warp import Resampling, calculate_default_transform, reproject
+from tools_shared_functions import (
+    filter_nwm_segments_by_stream_order,
+    get_datum,
+    get_nwm_segs,
+    get_thresholds,
+    ngvd_to_navd_ft,
+)
+from tools_shared_variables import (
+    acceptable_alt_acc_thresh,
+    acceptable_alt_meth_code_list,
+    acceptable_coord_acc_code_list,
+    acceptable_coord_method_code_list,
+    acceptable_site_type_list,
+)
+
 from utils.shared_variables import VIZ_PROJECTION
-from tools_shared_variables import (acceptable_coord_acc_code_list, 
-                                    acceptable_coord_method_code_list, 
-                                    acceptable_alt_acc_thresh, 
-                                    acceptable_alt_meth_code_list, 
-                                    acceptable_site_type_list)
 
 
-def process_generate_categorical_fim(fim_run_dir, job_number_huc, job_number_inundate, 
-                                     stage_based, output_folder, overwrite, search, 
-                                     lid_to_run, job_number_intervals, past_major_interval_cap):
-        
+def process_generate_categorical_fim(
+    fim_run_dir,
+    job_number_huc,
+    job_number_inundate,
+    stage_based,
+    output_folder,
+    overwrite,
+    search,
+    lid_to_run,
+    job_number_intervals,
+    past_major_interval_cap,
+):
     print("================================")
     print("Start generate categorical fim")
     overall_start_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"started: {dt_string}")
+    print(f"started: {dt_string}")
     print()
-        
+
     # Check job numbers and raise error if necessary
     total_cpus_requested = job_number_huc * job_number_inundate * job_number_intervals
     total_cpus_available = os.cpu_count() - 1
     if total_cpus_requested > total_cpus_available:
-        raise ValueError('The HUC job number, {}, multiplied by the inundate job number, {}, '\
-                          'exceeds your machine\'s available CPU count minus one. '\
-                          'Please lower the job_number_huc or job_number_inundate '\
-                          'values accordingly.'.format(job_number_huc, job_number_inundate) )
+        raise ValueError(
+            'The HUC job number, {}, multiplied by the inundate job number, {}, '
+            'exceeds your machine\'s available CPU count minus one. '
+            'Please lower the job_number_huc or job_number_inundate '
+            'values accordingly.'.format(job_number_huc, job_number_inundate)
+        )
 
     # Define default arguments. Modify these if necessary
     fim_version = os.path.split(fim_run_dir)[1]
-    
+
     # Append option configuration (flow_based or stage_based) to output folder name.
     if stage_based:
         file_handle_appendage, catfim_method = "_stage_based", "STAGE-BASED"
     else:
         file_handle_appendage, catfim_method = "_flow_based", "FLOW-BASED"
-    
+
     # Define output directories
     output_catfim_dir_parent = output_folder + file_handle_appendage
     output_flows_dir = os.path.join(output_catfim_dir_parent, 'flows')
     output_mapping_dir = os.path.join(output_catfim_dir_parent, 'mapping')
     attributes_dir = os.path.join(output_catfim_dir_parent, 'attributes')
-    
+
     # Create output directories
-    if not os.path.exists(output_catfim_dir_parent): os.mkdir(output_catfim_dir_parent)
-    if not os.path.exists(output_flows_dir): os.mkdir(output_flows_dir)
-    if not os.path.exists(output_mapping_dir): os.mkdir(output_mapping_dir)
-    if not os.path.exists(attributes_dir): os.mkdir(attributes_dir)
-    
+    if not os.path.exists(output_catfim_dir_parent):
+        os.mkdir(output_catfim_dir_parent)
+    if not os.path.exists(output_flows_dir):
+        os.mkdir(output_flows_dir)
+    if not os.path.exists(output_mapping_dir):
+        os.mkdir(output_mapping_dir)
+    if not os.path.exists(attributes_dir):
+        os.mkdir(attributes_dir)
+
     # Define upstream and downstream search in miles
     nwm_us_search, nwm_ds_search = search, search
     fim_dir = fim_version
-    
+
     # Set up logging
     log_dir = os.path.join(output_catfim_dir_parent, 'logs')
     log_file = os.path.join(log_dir, 'errors.log')
-    
+
     # STAGE-BASED
     if stage_based:
         # Generate Stage-Based CatFIM mapping
-        nws_sites_layer = generate_stage_based_categorical_fim(output_mapping_dir, fim_version, fim_run_dir,
-                                                               nwm_us_search, nwm_ds_search, job_number_inundate,
-                                                               lid_to_run, attributes_dir, job_number_intervals,
-                                                               past_major_interval_cap, job_number_huc)
-        
+        nws_sites_layer = generate_stage_based_categorical_fim(
+            output_mapping_dir,
+            fim_version,
+            fim_run_dir,
+            nwm_us_search,
+            nwm_ds_search,
+            job_number_inundate,
+            lid_to_run,
+            attributes_dir,
+            job_number_intervals,
+            past_major_interval_cap,
+            job_number_huc,
+        )
+
         job_number_tif = job_number_inundate * job_number_intervals
         print("Post-processing TIFs...")
-        post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_mapping_dir, attributes_dir, log_file=log_file, fim_version=fim_version)
-    
+        post_process_cat_fim_for_viz(
+            job_number_huc,
+            job_number_tif,
+            output_mapping_dir,
+            attributes_dir,
+            log_file=log_file,
+            fim_version=fim_version,
+        )
+
         # Updating mapping status
         print('Updating mapping status...')
         update_mapping_status(str(output_mapping_dir), str(output_flows_dir), nws_sites_layer, stage_based)
@@ -96,23 +136,41 @@ def process_generate_categorical_fim(fim_run_dir, job_number_huc, job_number_inu
         fim_dir = ""
         print('Creating flow files using the ' + catfim_method + ' technique...')
         start = time.time()
-        nws_sites_layer = generate_catfim_flows(output_flows_dir, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir, job_number_huc)
+        nws_sites_layer = generate_catfim_flows(
+            output_flows_dir,
+            nwm_us_search,
+            nwm_ds_search,
+            stage_based,
+            fim_dir,
+            lid_to_run,
+            attributes_dir,
+            job_number_huc,
+        )
         end = time.time()
-        elapsed_time = (end-start)/60
-        
+        elapsed_time = (end - start) / 60
+
         print(f'Finished creating flow files in {elapsed_time} minutes')
         # Generate CatFIM mapping
         print('Begin mapping')
         start = time.time()
-        manage_catfim_mapping(fim_run_dir, output_flows_dir, output_mapping_dir, attributes_dir, job_number_huc, job_number_inundate, overwrite, depthtif=False)
+        manage_catfim_mapping(
+            fim_run_dir,
+            output_flows_dir,
+            output_mapping_dir,
+            attributes_dir,
+            job_number_huc,
+            job_number_inundate,
+            overwrite,
+            depthtif=False,
+        )
         end = time.time()
-        elapsed_time = (end-start)/60
+        elapsed_time = (end - start) / 60
         print(f'Finished mapping in {elapsed_time} minutes')
 
         # Updating mapping status
         print('Updating mapping status')
         update_mapping_status(str(output_mapping_dir), str(output_flows_dir), nws_sites_layer, stage_based)
-    
+
     # Create CSV versions of the final geopackages.
     print('Creating CSVs. This may take several minutes.')
     reformatted_catfim_method = catfim_method.lower().replace('-', '_')
@@ -123,13 +181,14 @@ def process_generate_categorical_fim(fim_run_dir, job_number_huc, job_number_inu
 
     end_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"ended: {dt_string}")
+    print(f"ended: {dt_string}")
 
     # calculate duration
     time_duration = end_time - overall_start_time
     print(f"Duration: {str(time_duration).split('.')[0]}")
     print()
 
+
 def create_csvs(output_mapping_dir, reformatted_catfim_method):
     '''
     Produces CSV versions of desired geopackage in the output_mapping_dir.
@@ -146,7 +205,7 @@ def create_csvs(output_mapping_dir, reformatted_catfim_method):
     None.
 
     '''
-    
+
     # Convert any geopackage in the root level of output_mapping_dir to CSV and rename.
     gpkg_list = glob.glob(os.path.join(output_mapping_dir, '*.gpkg'))
     for gpkg in gpkg_list:
@@ -157,7 +216,7 @@ def create_csvs(output_mapping_dir, reformatted_catfim_method):
             file_name = reformatted_catfim_method + '_catfim.csv'
         if 'nws_lid_sites' in gpkg:
             file_name = reformatted_catfim_method + '_catfim_sites.csv'
-        
+
         csv_output_path = os.path.join(parent_directory, file_name)
         gdf.to_csv(csv_output_path)
 
@@ -187,7 +246,7 @@ def update_mapping_status(output_mapping_dir, output_flows_dir, nws_sites_layer,
     empty_nws_lids = [Path(directory).name for directory in subdirs if not list(Path(directory).iterdir())]
 
     # Write list of empty nws_lids to DataFrame, these are sites that failed in inundation.py
-    mapping_df = pd.DataFrame({'nws_lid':empty_nws_lids})
+    mapping_df = pd.DataFrame({'nws_lid': empty_nws_lids})
     mapping_df['did_it_map'] = 'no'
     mapping_df['map_status'] = ' and all categories failed to map'
 
@@ -196,95 +255,126 @@ def update_mapping_status(output_mapping_dir, output_flows_dir, nws_sites_layer,
 
     try:
         # Join failed sites to flows df
-        flows_df = flows_df.merge(mapping_df, how = 'left', on = 'nws_lid')
+        flows_df = flows_df.merge(mapping_df, how='left', on='nws_lid')
 
         # Switch mapped column to no for failed sites and update status
         flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
-        flows_df.loc[flows_df['did_it_map']=='no','status'] = flows_df['status'] + flows_df['map_status']
-        
-    #    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
-    #    if stage_based:
-    #        missing_mapping_hucs = 
-    #    else:
-    #        flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
-    #        mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
-    #        missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
-    #    
-    #    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
-    #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] = flows_df['status'] + ' and all categories failed to map because missing HUC information'
-    #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
+        flows_df.loc[flows_df['did_it_map'] == 'no', 'status'] = flows_df['status'] + flows_df['map_status']
+
+        #    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
+        #    if stage_based:
+        #        missing_mapping_hucs =
+        #    else:
+        #        flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
+        #        mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
+        #        missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
+        #
+        #    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
+        #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] =
+        #           flows_df['status'] + ' and all categories failed to map because missing HUC information'
+        #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
 
         # Clean up GeoDataFrame and rename columns for consistency
-        flows_df = flows_df.drop(columns = ['did_it_map','map_status'])
-        flows_df = flows_df.rename(columns = {'nws_lid':'ahps_lid'})
+        flows_df = flows_df.drop(columns=['did_it_map', 'map_status'])
+        flows_df = flows_df.rename(columns={'nws_lid': 'ahps_lid'})
 
         # Write out to file
         flows_df.to_file(nws_sites_layer)
-    except:
-        print("No LIDs")
+    except Exception as e:
+        print(f"No LIDs, \n Exception: \n {repr(e)} \n")
 
-def produce_inundation_map_with_stage_and_feature_ids(rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch):
-    
+
+def produce_inundation_map_with_stage_and_feature_ids(
+    rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch
+):
     # Open rem_path and catchment_path using rasterio.
     rem_src = rasterio.open(rem_path)
     catchments_src = rasterio.open(catchments_path)
     rem_array = rem_src.read(1)
     catchments_array = catchments_src.read(1)
-    
-    # Use numpy.where operation to reclassify rem_path on the condition that the pixel values are <= to hand_stage and the catchments
-    # value is in the hydroid_list.
-    reclass_rem_array = np.where((rem_array<=hand_stage) & (rem_array != rem_src.nodata), 1, 0).astype('uint8')
-    hydroid_mask = np.isin(catchments_array, hydroid_list)   
-    target_catchments_array = np.where((hydroid_mask == True) & (catchments_array != catchments_src.nodata), 1, 0).astype('uint8')
-    masked_reclass_rem_array = np.where((reclass_rem_array == 1) & (target_catchments_array == 1), 1, 0).astype('uint8')
-        
+
+    # Use numpy.where operation to reclassify rem_path on the condition that the pixel values
+    #   are <= to hand_stage and the catchments value is in the hydroid_list.
+    reclass_rem_array = np.where((rem_array <= hand_stage) & (rem_array != rem_src.nodata), 1, 0).astype(
+        'uint8'
+    )
+    hydroid_mask = np.isin(catchments_array, hydroid_list)
+    target_catchments_array = np.where(
+        (hydroid_mask is True) & (catchments_array != catchments_src.nodata), 1, 0
+    ).astype('uint8')
+    masked_reclass_rem_array = np.where(
+        (reclass_rem_array == 1) & (target_catchments_array == 1), 1, 0
+    ).astype('uint8')
+
     # Save resulting array to new tif with appropriate name. brdc1_record_extent_18060005.tif
     is_all_zero = np.all((masked_reclass_rem_array == 0))
-    
+
     if not is_all_zero:
-        output_tif = os.path.join(lid_directory, lid + '_' + category + '_extent_' + huc + '_' + branch + '.tif')
+        output_tif = os.path.join(
+            lid_directory, lid + '_' + category + '_extent_' + huc + '_' + branch + '.tif'
+        )
         with rasterio.Env():
             profile = rem_src.profile
             profile.update(dtype=rasterio.uint8)
             profile.update(nodata=10)
-            
+
             with rasterio.open(output_tif, 'w', **profile) as dst:
                 dst.write(masked_reclass_rem_array, 1)
-    
+
+
 def mark_complete(site_directory):
     marker_file = Path(site_directory) / 'complete.txt'
     marker_file.touch()
     return
 
-def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, threshold_url, flood_categories, all_lists, past_major_interval_cap, number_of_jobs, number_of_interval_jobs, attributes_dir, nwm_flows_df):
+
+def iterate_through_huc_stage_based(
+    workspace,
+    huc,
+    fim_dir,
+    huc_dictionary,
+    threshold_url,
+    flood_categories,
+    all_lists,
+    past_major_interval_cap,
+    number_of_jobs,
+    number_of_interval_jobs,
+    attributes_dir,
+    nwm_flows_df,
+):
     missing_huc_files = []
     all_messages = []
     stage_based_att_dict = {}
-    
-    print(f'Iterating through {huc}...') 
+
+    print(f'Iterating through {huc}...')
     # Make output directory for huc.
     huc_directory = os.path.join(workspace, huc)
     if not os.path.exists(huc_directory):
         os.mkdir(huc_directory)
-    
+
     # Define paths to necessary HAND and HAND-related files.
     usgs_elev_table = os.path.join(fim_dir, huc, 'usgs_elev_table.csv')
     branch_dir = os.path.join(fim_dir, huc, 'branches')
-    
+
     # Loop through each lid in nws_lids list
     nws_lids = huc_dictionary[huc]
     for lid in nws_lids:
-        lid = lid.lower() # Convert lid to lower case
+        lid = lid.lower()  # Convert lid to lower case
         # -- If necessary files exist, continue -- #
         if not os.path.exists(usgs_elev_table):
-            all_messages.append([f'{lid}:usgs_elev_table missing, likely unacceptable gage datum error--more details to come in future release'])
+            all_messages.append(
+                [
+                    f'{lid}:usgs_elev_table missing, likely unacceptable gage datum error--'
+                    'more details to come in future release'
+                ]
+            )
             continue
         if not os.path.exists(branch_dir):
             all_messages.append([f'{lid}:branch directory missing'])
             continue
         usgs_elev_df = pd.read_csv(usgs_elev_table)
         # Make lid_directory.
-        
+
         lid_directory = os.path.join(huc_directory, lid)
         if not os.path.exists(lid_directory):
             os.mkdir(lid_directory)
@@ -294,72 +384,93 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
                 all_messages.append([f"{lid}: already completed in previous run."])
                 continue
         # Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
-        stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
+        stages, flows = get_thresholds(
+            threshold_url=threshold_url, select_by='nws_lid', selector=lid, threshold='all'
+        )
 
-        if stages == None:
+        if stages is None:
             all_messages.append([f'{lid}:error getting thresholds from WRDS API'])
             continue
-        # Check if stages are supplied, if not write message and exit. 
-        if all(stages.get(category, None)==None for category in flood_categories):
+        # Check if stages are supplied, if not write message and exit.
+        if all(stages.get(category, None) is None for category in flood_categories):
             all_messages.append([f'{lid}:missing threshold stages'])
             continue
 
         try:
             # Drop columns that offend acceptance criteria
-            usgs_elev_df['acceptable_codes'] = (usgs_elev_df['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
-                                        & usgs_elev_df['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
-                                        & usgs_elev_df['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
-                                        & usgs_elev_df['usgs_data_site_type'].isin(acceptable_site_type_list))
-           
+            usgs_elev_df['acceptable_codes'] = (
+                usgs_elev_df['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
+                & usgs_elev_df['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
+                & usgs_elev_df['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
+                & usgs_elev_df['usgs_data_site_type'].isin(acceptable_site_type_list)
+            )
+
             usgs_elev_df = usgs_elev_df.astype({'usgs_data_alt_accuracy_code': float})
-            usgs_elev_df['acceptable_alt_error'] = np.where(usgs_elev_df['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False)
-            
-            acceptable_usgs_elev_df = usgs_elev_df[(usgs_elev_df['acceptable_codes'] == True) & (usgs_elev_df['acceptable_alt_error'] == True)]
+            usgs_elev_df['acceptable_alt_error'] = np.where(
+                usgs_elev_df['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False
+            )
+
+            acceptable_usgs_elev_df = usgs_elev_df[
+                (usgs_elev_df['acceptable_codes'] is True) & (usgs_elev_df['acceptable_alt_error'] is True)
+            ]
         except Exception as e:
             # Not sure any of the sites actually have those USGS-related
             # columns in this particular file, so just assume it's fine to use
 
-            #print("(Various columns related to USGS probably not in this csv)")
+            # print("(Various columns related to USGS probably not in this csv)")
+            print(f"Exception: \n {repr(e)} \n")
             acceptable_usgs_elev_df = usgs_elev_df
 
-        # Get the dem_adj_elevation value from usgs_elev_table.csv. Prioritize the value that is not from branch 0.
+        # Get the dem_adj_elevation value from usgs_elev_table.csv.
+        # Prioritize the value that is not from branch 0.
         try:
-            matching_rows = acceptable_usgs_elev_df.loc[acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation']
+            matching_rows = acceptable_usgs_elev_df.loc[
+                acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'
+            ]
 
             if len(matching_rows) == 2:  # It means there are two level paths, use the one that is not 0
-                lid_usgs_elev = acceptable_usgs_elev_df.loc[(acceptable_usgs_elev_df['nws_lid'] == lid.upper()) & (acceptable_usgs_elev_df['levpa_id'] != 0), 'dem_adj_elevation'].values[0]
+                lid_usgs_elev = acceptable_usgs_elev_df.loc[
+                    (acceptable_usgs_elev_df['nws_lid'] == lid.upper())
+                    & (acceptable_usgs_elev_df['levpa_id'] != 0),
+                    'dem_adj_elevation',
+                ].values[0]
             else:
-                lid_usgs_elev = acceptable_usgs_elev_df.loc[acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'].values[0]
+                lid_usgs_elev = acceptable_usgs_elev_df.loc[
+                    acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'
+                ].values[0]
         except IndexError:  # Occurs when LID is missing from table
-            all_messages.append([f'{lid}:likely unacceptable gage datum error or accuracy code(s); please see acceptance criteria'])
+            all_messages.append(
+                [
+                    f'{lid}:likely unacceptable gage datum error or accuracy code(s); '
+                    'please see acceptance criteria'
+                ]
+            )
             continue
         # Initialize nested dict for lid attributes
-        stage_based_att_dict.update({lid:{}})
-            
+        stage_based_att_dict.update({lid: {}})
+
         # Find lid metadata from master list of metadata dictionaries.
         metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
         lid_altitude = metadata['usgs_data']['altitude']
-   
+
         # Filter out sites that don't have "good" data
         try:
-            if not metadata['usgs_data']['coord_accuracy_code'] in \
-                    acceptable_coord_acc_code_list:
-                print(f"\t{lid}: {metadata['usgs_data']['coord_accuracy_code']} Not in acceptable coord acc codes")
+            if not metadata['usgs_data']['coord_accuracy_code'] in acceptable_coord_acc_code_list:
+                print(
+                    f"\t{lid}: {metadata['usgs_data']['coord_accuracy_code']} "
+                    "Not in acceptable coord acc codes"
+                )
                 continue
-            if not metadata['usgs_data']['coord_method_code'] in \
-                    acceptable_coord_method_code_list:
+            if not metadata['usgs_data']['coord_method_code'] in acceptable_coord_method_code_list:
                 print(f"\t{lid}: Not in acceptable coord method codes")
                 continue
-            if not metadata['usgs_data']['alt_method_code'] in \
-                    acceptable_alt_meth_code_list:
+            if not metadata['usgs_data']['alt_method_code'] in acceptable_alt_meth_code_list:
                 print(f"\t{lid}: Not in acceptable alt method codes")
                 continue
-            if not metadata['usgs_data']['site_type'] in \
-                    acceptable_site_type_list:
+            if not metadata['usgs_data']['site_type'] in acceptable_site_type_list:
                 print(f"\t{lid}: Not in acceptable site type codes")
                 continue
-            if not float(metadata['usgs_data']['alt_accuracy_code']) <= \
-                    acceptable_alt_acc_thresh:
+            if not float(metadata['usgs_data']['alt_accuracy_code']) <= acceptable_alt_acc_thresh:
                 print(f"\t{lid}: Not in acceptable threshold range")
                 continue
         except Exception as e:
@@ -367,7 +478,7 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
             continue
 
         ### --- Do Datum Offset --- ###
-        #determine source of interpolated threshold flows, this will be the rating curve that will be used.
+        # determine source of interpolated threshold flows, this will be the rating curve that will be used.
         rating_curve_source = flows.get('source')
         if rating_curve_source is None:
             all_messages.append([f'{lid}:No source for rating curve'])
@@ -378,69 +489,96 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
             datum_data = usgs_datum_info
         elif rating_curve_source == 'NRLDB':
             datum_data = nws_datum_info
-                    
+
         # If datum not supplied, skip to new site
         datum = datum_data.get('datum', None)
         if datum is None:
             all_messages.append([f'{lid}:datum info unavailable'])
             continue
-        # _________________________________________________________________________________________________________#
-        # SPECIAL CASE: Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null). 
-        # Modifying rating curve source will influence the rating curve and datum retrieved for benchmark determinations.
+        # ___________________________________________________________________________________________________#
+        # SPECIAL CASE: Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null).
+        # Modifying rating curve source will influence the rating curve and
+        #   datum retrieved for benchmark determinations.
         if lid == 'bmbp1':
             rating_curve_source = 'NRLDB'
-        # ___________________________________________________________________#
-        
-        # SPECIAL CASE: Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29 conversion to NAVD88
-        # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7, stkn4, wlln7 
+        # ___________________________________________________________________________________________________#
+
+        # SPECIAL CASE: Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29
+        #   conversion to NAVD88
+        # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7,
+        #   stkn4, wlln7
         # Assumed to be NAD83 (no info from USGS or NWS data): dlrt2, eagi1, eppt2, jffw3, ldot2, rgdt2
-        if lid in ['bgwn7', 'dlrt2','eagi1','eppt2','fatw3','jffw3','ldot2','mnvn4','nhpp1','pinn4','rgdt2','rgln4','rssk1','sign4','smfn7','stkn4','wlln7' ]:
-            datum_data.update(crs = 'NAD83')
-        # ___________________________________________________________________#
-        
-        # SPECIAL CASE: Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and is actually NAD27. 
-        # This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and it matches NRLDB coordinates.
+        if lid in [
+            'bgwn7',
+            'dlrt2',
+            'eagi1',
+            'eppt2',
+            'fatw3',
+            'jffw3',
+            'ldot2',
+            'mnvn4',
+            'nhpp1',
+            'pinn4',
+            'rgdt2',
+            'rgln4',
+            'rssk1',
+            'sign4',
+            'smfn7',
+            'stkn4',
+            'wlln7',
+        ]:
+            datum_data.update(crs='NAD83')
+        # ___________________________________________________________________________________________________#
+
+        # SPECIAL CASE: Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and
+        #   is actually NAD27.
+        # This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and
+        #   it matches NRLDB coordinates.
         if lid == 'bmbp1':
-            datum_data.update(crs = 'NAD27')
-        # ___________________________________________________________________#
-        
-        # SPECIAL CASE: Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure datum reported in NAVD88. 
+            datum_data.update(crs='NAD27')
+        # ___________________________________________________________________________________________________#
+
+        # SPECIAL CASE: Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure
+        #   datum reported in NAVD88.
         # If NGVD29 it is converted to NAVD88.
         # bgwn7, eagi1 vertical datum unknown, assume navd88
         # fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
         # wlln7 USGS data indicates vcs is NGVD29 (USGS and NWS info agree on datum value).
-        if lid in ['bgwn7','eagi1','fatw3']:
-            datum_data.update(vcs = 'NAVD88')
+        if lid in ['bgwn7', 'eagi1', 'fatw3']:
+            datum_data.update(vcs='NAVD88')
         elif lid == 'wlln7':
-            datum_data.update(vcs = 'NGVD29')
-        # _________________________________________________________________________________________________________#
-        
+            datum_data.update(vcs='NGVD29')
+        # ___________________________________________________________________________________________________#
+
         # Adjust datum to NAVD88 if needed
         # Default datum_adj_ft to 0.0
         datum_adj_ft = 0.0
         crs = datum_data.get('crs')
         if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929', 'NGVD,1929', 'NGVD OF 1929', 'NGVD']:
-            # Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
+            # Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously
+            #   removed otherwise the region needs changed.
             try:
-                datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
+                datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
             except Exception as e:
                 e = str(e)
-                if crs == None:
+                if crs is None:
                     all_messages.append([f'{lid}:NOAA VDatum adjustment error, CRS is missing'])
                 if 'HTTPSConnectionPool' in e:
                     time.sleep(10)  # Maybe the API needs a break, so wait 10 seconds
                     try:
-                        datum_adj_ft = ngvd_to_navd_ft(datum_info = datum_data, region = 'contiguous')
+                        datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
                     except Exception:
                         all_messages.append([f'{lid}:NOAA VDatum adjustment error, possible API issue'])
                 if 'Invalid projection' in e:
-                    all_messages.append([f'{lid}:NOAA VDatum adjustment error, invalid projection: crs={crs}'])
+                    all_messages.append(
+                        [f'{lid}:NOAA VDatum adjustment error, invalid projection: crs={crs}']
+                    )
                 continue
-        
+
         ### -- Concluded Datum Offset --- ###
         # Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
         unfiltered_segments = list(set(get_nwm_segs(metadata)))
-        
+
         # Filter segments to be of like stream order.
         desired_order = metadata['nwm_feature_data']['stream_order']
         segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df)
@@ -453,38 +591,58 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
         if stage_list == []:
             all_messages.append([f'{lid}:no stage values available'])
             continue
-        interval_list = np.arange(min(stage_list), max(stage_list) + past_major_interval_cap, 1.0)  # Go an extra 10 ft beyond the max stage, arbitrary
-        
-        # Check for large discrepancies between the elevation values from WRDS and HAND. Otherwise this causes bad mapping.
-        elevation_diff = lid_usgs_elev - (lid_altitude*0.3048)
+        interval_list = np.arange(
+            min(stage_list), max(stage_list) + past_major_interval_cap, 1.0
+        )  # Go an extra 10 ft beyond the max stage, arbitrary
+
+        # Check for large discrepancies between the elevation values from WRDS and HAND.
+        #   Otherwise this causes bad mapping.
+        elevation_diff = lid_usgs_elev - (lid_altitude * 0.3048)
         if abs(elevation_diff) > 10:
             all_messages.append([f'{lid}:large discrepancy in elevation estimates from gage and HAND'])
             continue
-        
+
             # For each flood category
         for category in flood_categories:
-            
             # Pull stage value and confirm it's valid, then process
             stage = stages[category]
-            
-            if stage != None and datum_adj_ft != None and lid_altitude != None:
-                
+
+            if stage is not None and datum_adj_ft is not None and lid_altitude is not None:
                 # Call function to execute mapping of the TIFs.
-                messages, hand_stage, datum_adj_wse, datum_adj_wse_m = produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs)
+                (messages, hand_stage, datum_adj_wse, datum_adj_wse_m) = produce_stage_based_catfim_tifs(
+                    stage,
+                    datum_adj_ft,
+                    branch_dir,
+                    lid_usgs_elev,
+                    lid_altitude,
+                    fim_dir,
+                    segments,
+                    lid,
+                    huc,
+                    lid_directory,
+                    category,
+                    number_of_jobs,
+                )
                 all_messages += messages
-                                                
-                # Extra metadata for alternative CatFIM technique. TODO Revisit because branches complicate things
-                stage_based_att_dict[lid].update({category: {'datum_adj_wse_ft': datum_adj_wse,
-                                                             'datum_adj_wse_m': datum_adj_wse_m,
-                                                             'hand_stage': hand_stage,
-                                                             'datum_adj_ft': datum_adj_ft,
-                                                             'lid_alt_ft': lid_altitude,
-                                                             'lid_alt_m': lid_altitude*0.3048}})
+
+                # Extra metadata for alternative CatFIM technique.
+                # TODO Revisit because branches complicate things
+                stage_based_att_dict[lid].update(
+                    {
+                        category: {
+                            'datum_adj_wse_ft': datum_adj_wse,
+                            'datum_adj_wse_m': datum_adj_wse_m,
+                            'hand_stage': hand_stage,
+                            'datum_adj_ft': datum_adj_ft,
+                            'lid_alt_ft': lid_altitude,
+                            'lid_alt_m': lid_altitude * 0.3048,
+                        }
+                    }
+                )
             # If missing HUC file data, write message
             if huc in missing_huc_files:
                 all_messages.append([f'{lid}:missing some HUC data'])
-                
-        
+
         # Now that the "official" category maps are made, produce the incremental maps.
         with ProcessPoolExecutor(max_workers=number_of_interval_jobs) as executor:
             try:
@@ -498,54 +656,72 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
                         category = 'moderate_' + str(interval_stage).replace('.', 'p') + 'ft'
                     if interval_stage >= major_stage:
                         category = 'major_' + str(interval_stage).replace('.', 'p') + 'ft'
-                    executor.submit(produce_stage_based_catfim_tifs, interval_stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs)
+                    executor.submit(
+                        produce_stage_based_catfim_tifs,
+                        interval_stage,
+                        datum_adj_ft,
+                        branch_dir,
+                        lid_usgs_elev,
+                        lid_altitude,
+                        fim_dir,
+                        segments,
+                        lid,
+                        huc,
+                        lid_directory,
+                        category,
+                        number_of_jobs,
+                    )
             except TypeError:  # sometimes the thresholds are Nonetypes
                 pass
-                    
+
         # Create a csv with same information as geopackage but with each threshold as new record.
         # Probably a less verbose way.
         csv_df = pd.DataFrame()
         for threshold in flood_categories:
             try:
-                line_df = pd.DataFrame({'nws_lid': [lid], 
-                                        'name':metadata['nws_data']['name'], 
-                                        'WFO': metadata['nws_data']['wfo'], 
-                                        'rfc':metadata['nws_data']['rfc'], 
-                                        'huc':[huc], 
-                                        'state':metadata['nws_data']['state'], 
-                                        'county':metadata['nws_data']['county'], 
-                                        'magnitude': threshold, 
-                                        'q':flows[threshold], 
-                                        'q_uni':flows['units'], 
-                                        'q_src':flows['source'], 
-                                        'stage':stages[threshold], 
-                                        'stage_uni':stages['units'], 
-                                        's_src':stages['source'], 
-                                        'wrds_time':stages['wrds_timestamp'], 
-                                        'nrldb_time':metadata['nrldb_timestamp'],
-                                        'nwis_time':metadata['nwis_timestamp'], 
-                                        'lat':[float(metadata['nws_preferred']['latitude'])], 
-                                        'lon':[float(metadata['nws_preferred']['longitude'])],
-                                        'dtm_adj_ft': stage_based_att_dict[lid][threshold]['datum_adj_ft'],
-                                        'dadj_w_ft': stage_based_att_dict[lid][threshold]['datum_adj_wse_ft'],
-                                        'dadj_w_m': stage_based_att_dict[lid][threshold]['datum_adj_wse_m'],
-                                        'lid_alt_ft': stage_based_att_dict[lid][threshold]['lid_alt_ft'],
-                                        'lid_alt_m': stage_based_att_dict[lid][threshold]['lid_alt_m']})
+                line_df = pd.DataFrame(
+                    {
+                        'nws_lid': [lid],
+                        'name': metadata['nws_data']['name'],
+                        'WFO': metadata['nws_data']['wfo'],
+                        'rfc': metadata['nws_data']['rfc'],
+                        'huc': [huc],
+                        'state': metadata['nws_data']['state'],
+                        'county': metadata['nws_data']['county'],
+                        'magnitude': threshold,
+                        'q': flows[threshold],
+                        'q_uni': flows['units'],
+                        'q_src': flows['source'],
+                        'stage': stages[threshold],
+                        'stage_uni': stages['units'],
+                        's_src': stages['source'],
+                        'wrds_time': stages['wrds_timestamp'],
+                        'nrldb_time': metadata['nrldb_timestamp'],
+                        'nwis_time': metadata['nwis_timestamp'],
+                        'lat': [float(metadata['nws_preferred']['latitude'])],
+                        'lon': [float(metadata['nws_preferred']['longitude'])],
+                        'dtm_adj_ft': stage_based_att_dict[lid][threshold]['datum_adj_ft'],
+                        'dadj_w_ft': stage_based_att_dict[lid][threshold]['datum_adj_wse_ft'],
+                        'dadj_w_m': stage_based_att_dict[lid][threshold]['datum_adj_wse_m'],
+                        'lid_alt_ft': stage_based_att_dict[lid][threshold]['lid_alt_ft'],
+                        'lid_alt_m': stage_based_att_dict[lid][threshold]['lid_alt_m'],
+                    }
+                )
                 csv_df = pd.concat([csv_df, line_df])
-                
+
             except Exception as e:
                 print(e)
-          
+
         # Round flow and stage columns to 2 decimal places.
-        csv_df = csv_df.round({'q':2,'stage':2})
+        csv_df = csv_df.round({'q': 2, 'stage': 2})
         # If a site folder exists (ie a flow file was written) save files containing site attributes.
         output_dir = os.path.join(workspace, huc, lid)
         if os.path.exists(output_dir):
             # Export DataFrame to csv containing attributes
-            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index = False)
+            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index=False)
         else:
             all_messages.append([f'{lid}:missing all calculated flows'])
-            
+
         # If it made it to this point (i.e. no continues), there were no major preventers of mapping
         all_messages.append([f'{lid}:OK'])
         mark_complete(output_dir)
@@ -558,20 +734,44 @@ def iterate_through_huc_stage_based(workspace, huc, fim_dir, huc_dictionary, thr
         writer = csv.writer(output_csv)
         writer.writerows(all_messages)
 
-    
-def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us_search, nwm_ds_search, number_of_jobs, lid_to_run, attributes_dir, number_of_interval_jobs, past_major_interval_cap, job_number_huc):
-    
+
+def generate_stage_based_categorical_fim(
+    workspace,
+    fim_version,
+    fim_dir,
+    nwm_us_search,
+    nwm_ds_search,
+    number_of_jobs,
+    lid_to_run,
+    attributes_dir,
+    number_of_interval_jobs,
+    past_major_interval_cap,
+    job_number_huc,
+):
     flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
 
-    huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists, nwm_flows_df = generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based=True, fim_dir=fim_dir, lid_to_run=lid_to_run)
-    
+    (huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists, nwm_flows_df) = generate_catfim_flows(
+        workspace, nwm_us_search, nwm_ds_search, stage_based=True, fim_dir=fim_dir, lid_to_run=lid_to_run
+    )
+
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
         for huc in huc_dictionary:
-            executor.submit(iterate_through_huc_stage_based, workspace, huc, 
-                            fim_dir, huc_dictionary, threshold_url, flood_categories,
-                            all_lists, past_major_interval_cap, number_of_jobs, 
-                            number_of_interval_jobs, attributes_dir, nwm_flows_df)
-                
+            executor.submit(
+                iterate_through_huc_stage_based,
+                workspace,
+                huc,
+                fim_dir,
+                huc_dictionary,
+                threshold_url,
+                flood_categories,
+                all_lists,
+                past_major_interval_cap,
+                number_of_jobs,
+                number_of_interval_jobs,
+                attributes_dir,
+                nwm_flows_df,
+            )
+
     print('Wrapping up Stage-Based CatFIM...')
     csv_files = os.listdir(attributes_dir)
     all_csv_df = pd.DataFrame()
@@ -580,35 +780,42 @@ def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us
         full_csv_path = os.path.join(attributes_dir, csv_file)
         # HUC has to be read in as string to preserve leading zeros.
         try:
-            temp_df = pd.read_csv(full_csv_path, dtype={'huc':str})
-            all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index = True)
+            temp_df = pd.read_csv(full_csv_path, dtype={'huc': str})
+            all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index=True)
             refined_csv_files_list.append(csv_file)
         except Exception:  # Happens if a file is empty (i.e. no mapping)
             pass
     # Write to file
-    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index = False)
-   
+    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index=False)
+
     # This section populates a geopackage of all potential sites and details
     # whether it was mapped or not (mapped field) and if not, why (status field).
-    
+
     # Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
     viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)
-    viz_out_gdf.rename(columns = {'identifiers_nwm_feature_id': 'nwm_seg', 'identifiers_nws_lid':'nws_lid', 'identifiers_usgs_site_code':'usgs_gage'}, inplace = True)
+    viz_out_gdf.rename(
+        columns={
+            'identifiers_nwm_feature_id': 'nwm_seg',
+            'identifiers_nws_lid': 'nws_lid',
+            'identifiers_usgs_site_code': 'usgs_gage',
+        },
+        inplace=True,
+    )
     viz_out_gdf['nws_lid'] = viz_out_gdf['nws_lid'].str.lower()
-    
+
     # Using list of csv_files, populate DataFrame of all nws_lids that had
     # a flow file produced and denote with "mapped" column.
     nws_lids = []
     for csv_file in csv_files:
         nws_lids.append(csv_file.split('_attributes')[0])
-    lids_df = pd.DataFrame(nws_lids, columns = ['nws_lid'])
+    lids_df = pd.DataFrame(nws_lids, columns=['nws_lid'])
     lids_df['mapped'] = 'yes'
-    
-    # Identify what lids were mapped by merging with lids_df. Populate 
+
+    # Identify what lids were mapped by merging with lids_df. Populate
     # 'mapped' column with 'No' if sites did not map.
-    viz_out_gdf = viz_out_gdf.merge(lids_df, how = 'left', on = 'nws_lid')    
+    viz_out_gdf = viz_out_gdf.merge(lids_df, how='left', on='nws_lid')
     viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
-    
+
     # Create list from all messages in messages dir.
     messages_dir = os.path.join(workspace, 'messages')
     all_messages = []
@@ -619,7 +826,7 @@ def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us
             reader = csv.reader(message_file)
             for row in reader:
                 all_messages.append(row)
-    
+
     # Filter out columns and write out to file
     nws_sites_layer = os.path.join(workspace, 'nws_lid_sites.gpkg')
 
@@ -627,19 +834,22 @@ def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us
     # (and this line shouldn't have been reached if we had an interrupted
     # run previously and are picking back up with a restart)
     if not os.path.exists(nws_sites_layer):
-        
         # Write messages to DataFrame, split into columns, aggregate messages.
-        messages_df  = pd.DataFrame(all_messages, columns = ['message'])
+        messages_df = pd.DataFrame(all_messages, columns=['message'])
 
-        messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
+        messages_df = (
+            messages_df['message']
+            .str.split(':', n=1, expand=True)
+            .rename(columns={0: 'nws_lid', 1: 'status'})
+        )
         status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
-        
-        # Join messages to populate status field to candidate sites. Assign 
+
+        # Join messages to populate status field to candidate sites. Assign
         # status for null fields.
-        viz_out_gdf = viz_out_gdf.merge(status_df, how = 'left', on = 'nws_lid')
-        
+        viz_out_gdf = viz_out_gdf.merge(status_df, how='left', on='nws_lid')
+
         #    viz_out_gdf['status'] = viz_out_gdf['status'].fillna('OK')
-        
+
         # Add acceptance criteria to viz_out_gdf before writing
         viz_out_gdf['acceptable_coord_acc_code_list'] = str(acceptable_coord_acc_code_list)
         viz_out_gdf['acceptable_coord_method_code_list'] = str(acceptable_coord_method_code_list)
@@ -648,20 +858,33 @@ def generate_stage_based_categorical_fim(workspace, fim_version, fim_dir, nwm_us
         viz_out_gdf['acceptable_site_type_list'] = str(acceptable_site_type_list)
 
         viz_out_gdf.to_file(nws_sites_layer, driver='GPKG')
-        
+
     return nws_sites_layer
-    
 
-def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_elev, lid_altitude, fim_dir, segments, lid, huc, lid_directory, category, number_of_jobs):
+
+def produce_stage_based_catfim_tifs(
+    stage,
+    datum_adj_ft,
+    branch_dir,
+    lid_usgs_elev,
+    lid_altitude,
+    fim_dir,
+    segments,
+    lid,
+    huc,
+    lid_directory,
+    category,
+    number_of_jobs,
+):
     messages = []
-    
+
     # Determine datum-offset water surface elevation (from above).
     datum_adj_wse = stage + datum_adj_ft + lid_altitude
-    datum_adj_wse_m = datum_adj_wse*0.3048  # Convert ft to m
-    
+    datum_adj_wse_m = datum_adj_wse * 0.3048  # Convert ft to m
+
     # Subtract HAND gage elevation from HAND WSE to get HAND stage.
     hand_stage = datum_adj_wse_m - lid_usgs_elev
-    
+
     # Produce extent tif hand_stage. Multiprocess across branches.
     branches = os.listdir(branch_dir)
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
@@ -669,9 +892,14 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
             # Define paths to necessary files to produce inundation grids.
             full_branch_path = os.path.join(branch_dir, branch)
             rem_path = os.path.join(fim_dir, huc, full_branch_path, 'rem_zeroed_masked_' + branch + '.tif')
-            catchments_path = os.path.join(fim_dir, huc, full_branch_path, 'gw_catchments_reaches_filtered_addedAttributes_' + branch + '.tif')
+            catchments_path = os.path.join(
+                fim_dir,
+                huc,
+                full_branch_path,
+                'gw_catchments_reaches_filtered_addedAttributes_' + branch + '.tif',
+            )
             hydrotable_path = os.path.join(fim_dir, huc, full_branch_path, 'hydroTable_' + branch + '.csv')
-            
+
             if not os.path.exists(rem_path):
                 messages.append([f"{lid}:rem doesn't exist"])
                 continue
@@ -681,11 +909,11 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
             if not os.path.exists(hydrotable_path):
                 messages.append([f"{lid}:hydrotable doesn't exist"])
                 continue
-            
+
             # Use hydroTable to determine hydroid_list from site_ms_segments.
             hydrotable_df = pd.read_csv(hydrotable_path)
             hydroid_list = []
-            
+
             # Determine hydroids at which to perform inundation
             for feature_id in segments:
                 try:
@@ -693,23 +921,35 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
                     hydroid_list += list(subset_hydrotable_df.HydroID.unique())
                 except IndexError:
                     pass
-                
-            if len(hydroid_list) == 0:
-#                messages.append(f"{lid}:no matching hydroids")  # Some branches don't have matching hydroids
-                continue
+
+            # Some branches don't have matching hydroids
+            # if len(hydroid_list) == 0:
+            #     messages.append(f"{lid}:no matching hydroids")
+            #     continue
 
             # If no segments, write message and exit out
             if not segments:
                 messages.append([f'{lid}:missing nwm segments'])
                 continue
-            
+
             # Create inundation maps with branch and stage data
             try:
                 print("Generating stage-based FIM for " + huc + " and branch " + branch)
-                executor.submit(produce_inundation_map_with_stage_and_feature_ids, rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch)
+                executor.submit(
+                    produce_inundation_map_with_stage_and_feature_ids,
+                    rem_path,
+                    catchments_path,
+                    hydroid_list,
+                    hand_stage,
+                    lid_directory,
+                    category,
+                    huc,
+                    lid,
+                    branch,
+                )
             except Exception:
                 messages.append([f'{lid}:inundation failed at {category}'])
-                
+
     # -- MOSAIC -- #
     # Merge all rasters in lid_directory that have the same magnitude/category.
     path_list = []
@@ -719,37 +959,39 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
         if category in f:
             path_list.append(os.path.join(lid_directory, f))
     path_list.sort()  # To force branch 0 first in list, sort
-    
+
     if len(path_list) > 0:
         zero_branch_grid = path_list[0]
         zero_branch_src = rasterio.open(zero_branch_grid)
         zero_branch_array = zero_branch_src.read(1)
         summed_array = zero_branch_array  # Initialize it as the branch zero array
-        
+
         # Loop through remaining items in list and sum them with summed_array
         for remaining_raster in path_list[1:]:
             remaining_raster_src = rasterio.open(remaining_raster)
             remaining_raster_array_original = remaining_raster_src.read(1)
-        
+
             # Reproject non-branch-zero grids so I can sum them with the branch zero grid
             remaining_raster_array = np.empty(zero_branch_array.shape, dtype=np.int8)
-            reproject(remaining_raster_array_original,
-                  destination = remaining_raster_array,
-                  src_transform = remaining_raster_src.transform,
-                  src_crs = remaining_raster_src.crs,
-                  src_nodata = remaining_raster_src.nodata,
-                  dst_transform = zero_branch_src.transform,
-                  dst_crs = zero_branch_src.crs,
-                  dst_nodata = -1,
-                  dst_resolution = zero_branch_src.res,
-                  resampling = Resampling.nearest)
+            reproject(
+                remaining_raster_array_original,
+                destination=remaining_raster_array,
+                src_transform=remaining_raster_src.transform,
+                src_crs=remaining_raster_src.crs,
+                src_nodata=remaining_raster_src.nodata,
+                dst_transform=zero_branch_src.transform,
+                dst_crs=zero_branch_src.crs,
+                dst_nodata=-1,
+                dst_resolution=zero_branch_src.res,
+                resampling=Resampling.nearest,
+            )
             # Sum rasters
             summed_array = summed_array + remaining_raster_array
-            
+
         del zero_branch_array  # Clean up
-        
+
         # Define path to merged file, in same format as expected by post_process_cat_fim_for_viz function
-        output_tif = os.path.join(lid_directory, lid + '_' + category + '_extent.tif')  
+        output_tif = os.path.join(lid_directory, lid + '_' + category + '_extent.tif')
         profile = zero_branch_src.profile
         summed_array = summed_array.astype('uint8')
         with rasterio.open(output_tif, 'w', **profile) as dst:
@@ -761,30 +1003,83 @@ def produce_stage_based_catfim_tifs(stage, datum_adj_ft, branch_dir, lid_usgs_el
 
 if __name__ == '__main__':
     # Parse arguments
-    parser = argparse.ArgumentParser(description = 'Run Categorical FIM')
-    parser.add_argument('-f', '--fim_run_dir', help='Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_0_9_2',
-                        required=True)
-    parser.add_argument('-jh','--job_number_huc',help='Number of processes to use for HUC scale operations.'\
-        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count of the'\
-        ' machine. CatFIM sites generally only have 2-3 branches overlapping a site, so this number can be kept low (2-4)', required=False, default=1, type=int)
-    parser.add_argument('-jn','--job_number_inundate', help='Number of processes to use for inundating'\
-        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count'\
-        ' of the machine.', required=False, default=1, type=int)    
-    parser.add_argument('-a', '--stage_based', help = 'Run stage-based CatFIM instead of flow-based?'\
-        ' NOTE: flow-based CatFIM is the default.', required=False, default=False, action='store_true')
-    parser.add_argument('-t', '--output_folder', help = 'Target: Where the output folder will be',
-                        required = False, default = '/data/catfim/')
-    parser.add_argument('-o','--overwrite', help='Overwrite files', required=False, action="store_true")
-    parser.add_argument('-s','--search', help='Upstream and downstream search in miles. How far up and downstream do you want to go?',
-                        required=False, default='5')
-    parser.add_argument('-l','--lid_to_run', help='NWS LID, lowercase, to produce CatFIM for. Currently only accepts one. Default is all sites',
-                        required=False, default='all')
-    parser.add_argument('-ji','--job_number_intervals', help='Number of processes to use for inundating multiple intervals in stage-based'\
-        ' inundation and interval job numbers should multiply to no more than one less than the CPU count'\
-        ' of the machine.', required=False, default=1, type=int)
-    parser.add_argument('-mc','--past_major_interval_cap', help='Stage-Based Only. How many feet past major do you want to go for the interval FIMs?'\
-        ' of the machine.', required=False, default=5.0, type=float)    
-    
+    parser = argparse.ArgumentParser(description='Run Categorical FIM')
+    parser.add_argument(
+        '-f',
+        '--fim_run_dir',
+        help='Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_0_9_2',
+        required=True,
+    )
+    parser.add_argument(
+        '-jh',
+        '--job_number_huc',
+        help='Number of processes to use for HUC scale operations.'
+        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count of the'
+        ' machine. CatFIM sites generally only have 2-3 branches overlapping a site, so this number can be '
+        'kept low (2-4)',
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        '-jn',
+        '--job_number_inundate',
+        help='Number of processes to use for inundating'
+        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count'
+        ' of the machine.',
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        '-a',
+        '--stage_based',
+        help='Run stage-based CatFIM instead of flow-based?' ' NOTE: flow-based CatFIM is the default.',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-t',
+        '--output_folder',
+        help='Target: Where the output folder will be',
+        required=False,
+        default='/data/catfim/',
+    )
+    parser.add_argument('-o', '--overwrite', help='Overwrite files', required=False, action="store_true")
+    parser.add_argument(
+        '-s',
+        '--search',
+        help='Upstream and downstream search in miles. How far up and downstream do you want to go?',
+        required=False,
+        default='5',
+    )
+    parser.add_argument(
+        '-l',
+        '--lid_to_run',
+        help='NWS LID, lowercase, to produce CatFIM for. Currently only accepts one. Default is all sites',
+        required=False,
+        default='all',
+    )
+    parser.add_argument(
+        '-ji',
+        '--job_number_intervals',
+        help='Number of processes to use for inundating multiple intervals in stage-based'
+        ' inundation and interval job numbers should multiply to no more than one less than the CPU count '
+        'of the machine.',
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        '-mc',
+        '--past_major_interval_cap',
+        help='Stage-Based Only. How many feet past major do you want to go for the interval FIMs?'
+        ' of the machine.',
+        required=False,
+        default=5.0,
+        type=float,
+    )
+
     args = vars(parser.parse_args())
     process_generate_categorical_fim(**args)
-
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index 1acf87449..faf539a0c 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -1,91 +1,105 @@
 #!/usr/bin/env python3
-from pathlib import Path
-import pandas as pd
-import numpy as np
-import geopandas as gpd
-from datetime import datetime
-import time
-from tools_shared_functions import aggregate_wbd_hucs, mainstem_nwm_segs, get_thresholds, flow_data, get_metadata, get_nwm_segs, get_datum, ngvd_to_navd_ft, filter_nwm_segments_by_stream_order
-from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+
 import argparse
-from dotenv import load_dotenv
 import os
 import sys
-sys.path.append('/foss_fim/src')
+import time
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+from datetime import datetime
+from pathlib import Path
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+from dotenv import load_dotenv
+from tools_shared_functions import (
+    aggregate_wbd_hucs,
+    filter_nwm_segments_by_stream_order,
+    flow_data,
+    get_metadata,
+    get_nwm_segs,
+    get_thresholds,
+)
+
 from utils.shared_variables import VIZ_PROJECTION
 
 
 def get_env_paths():
     load_dotenv()
-    #import variables from .env file
+    # import variables from .env file
     API_BASE_URL = os.getenv("API_BASE_URL")
     WBD_LAYER = os.getenv("WBD_LAYER")
     return API_BASE_URL, WBD_LAYER
 
 
-def process_generate_flows(huc, huc_dictionary, threshold_url, all_lists, workspace, attributes_dir, huc_messages_dir, nwm_flows_df):
-    
+def process_generate_flows(
+    huc, huc_dictionary, threshold_url, all_lists, workspace, attributes_dir, huc_messages_dir, nwm_flows_df
+):
     # Process each huc unit, first define message variable and flood categories.
     all_messages = []
     flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
-    
+
     print(f'Iterating through {huc}')
-    #Get list of nws_lids
+    # Get list of nws_lids
     nws_lids = huc_dictionary[huc]
-    #Loop through each lid in list to create flow file
-    for lid in nws_lids:            
-        #Convert lid to lower case
+    # Loop through each lid in list to create flow file
+    for lid in nws_lids:
+        # Convert lid to lower case
         lid = lid.lower()
-        #Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
+        # Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
         print("getting thresholds")
-        stages, flows = get_thresholds(threshold_url = threshold_url, select_by = 'nws_lid', selector = lid, threshold = 'all')
-        if stages == None or flows == None:
+        stages, flows = get_thresholds(
+            threshold_url=threshold_url, select_by='nws_lid', selector=lid, threshold='all'
+        )
+        if stages is None or flows is None:
             print("Likely WRDS error")
             continue
-        #Check if stages are supplied, if not write message and exit. 
-        if all(stages.get(category, None)==None for category in flood_categories):
+        # Check if stages are supplied, if not write message and exit.
+        if all(stages.get(category, None) is None for category in flood_categories):
             message = f'{lid}:missing threshold stages'
             all_messages.append(message)
             continue
-        #Check if calculated flows are supplied, if not write message and exit.
-        if all(flows.get(category, None) == None for category in flood_categories):
+        # Check if calculated flows are supplied, if not write message and exit.
+        if all(flows.get(category, None) is None for category in flood_categories):
             message = f'{lid}:missing calculated flows'
             all_messages.append(message)
             continue
-        #find lid metadata from master list of metadata dictionaries (line 66).
+        # find lid metadata from master list of metadata dictionaries (line 66).
         metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
-                                  
-        #Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
+
+        # Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
         unfiltered_segments = list(set(get_nwm_segs(metadata)))
-        
+
         desired_order = metadata['nwm_feature_data']['stream_order']
         # Filter segments to be of like stream order.
         print("filtering segments")
         start = time.time()
         segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df)
         end = time.time()
-        elapsed_time = round(((end-start)/60), 6)
+        elapsed_time = round(((end - start) / 60), 6)
         print(f'Finished filtering segments in {elapsed_time} minutes')
-        #if no segments, write message and exit out
+        # if no segments, write message and exit out
         if not segments:
             message = f'{lid}:missing nwm segments'
             all_messages.append(message)
             continue
         # For each flood category
         for category in flood_categories:
-            #G et the flow
+            # G et the flow
             flow = flows[category]
             # If there is a valid flow value, write a flow file.
             if flow:
-                #round flow to nearest hundredth
-                flow = round(flow,2)
-                #Create the guts of the flow file.
-                flow_info = flow_data(segments,flow)
-                #Define destination path and create folders
-                output_file = workspace / huc / lid / category / (f'ahps_{lid}_huc_{huc}_flows_{category}.csv') 
-                output_file.parent.mkdir(parents = True, exist_ok = True)
-                #Write flow file to file
-                flow_info.to_csv(output_file, index = False)
+                # round flow to nearest hundredth
+                flow = round(flow, 2)
+                # Create the guts of the flow file.
+                flow_info = flow_data(segments, flow)
+                # Define destination path and create folders
+                output_file = (
+                    workspace / huc / lid / category / (f'ahps_{lid}_huc_{huc}_flows_{category}.csv')
+                )
+                output_file.parent.mkdir(parents=True, exist_ok=True)
+                # Write flow file to file
+                flow_info.to_csv(output_file, index=False)
             else:
                 message = f'{lid}:{category} is missing calculated flow'
                 all_messages.append(message)
@@ -102,26 +116,48 @@ def process_generate_flows(huc, huc_dictionary, threshold_url, all_lists, worksp
         wrds_timestamp = stages['wrds_timestamp']
         nrldb_timestamp = metadata['nrldb_timestamp']
         nwis_timestamp = metadata['nwis_timestamp']
-                    
+
         # Create a csv with same information as shapefile but with each threshold as new record.
         csv_df = pd.DataFrame()
         for threshold in flood_categories:
-            line_df = pd.DataFrame({'nws_lid': [lid], 'name':name, 'WFO': wfo, 'rfc':rfc, 'huc':[huc], 'state':state, 'county':county, 'magnitude': threshold, 'q':flows[threshold], 'q_uni':flows['units'], 'q_src':flow_source, 'stage':stages[threshold], 'stage_uni':stages['units'], 's_src':stage_source, 'wrds_time':wrds_timestamp, 'nrldb_time':nrldb_timestamp,'nwis_time':nwis_timestamp, 'lat':[lat], 'lon':[lon]})
+            line_df = pd.DataFrame(
+                {
+                    'nws_lid': [lid],
+                    'name': name,
+                    'WFO': wfo,
+                    'rfc': rfc,
+                    'huc': [huc],
+                    'state': state,
+                    'county': county,
+                    'magnitude': threshold,
+                    'q': flows[threshold],
+                    'q_uni': flows['units'],
+                    'q_src': flow_source,
+                    'stage': stages[threshold],
+                    'stage_uni': stages['units'],
+                    's_src': stage_source,
+                    'wrds_time': wrds_timestamp,
+                    'nrldb_time': nrldb_timestamp,
+                    'nwis_time': nwis_timestamp,
+                    'lat': [lat],
+                    'lon': [lon],
+                }
+            )
             csv_df = pd.concat([csv_df, line_df])
-        #Round flow and stage columns to 2 decimal places.
-        csv_df = csv_df.round({'q':2,'stage':2})
+        # Round flow and stage columns to 2 decimal places.
+        csv_df = csv_df.round({'q': 2, 'stage': 2})
 
         # If a site folder exists (ie a flow file was written) save files containing site attributes.
         output_dir = workspace / huc / lid
         if output_dir.exists():
-            #Export DataFrame to csv containing attributes
-            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index = False)
+            # Export DataFrame to csv containing attributes
+            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index=False)
             message = f'{lid}:flows available'
             all_messages.append(message)
         else:
             message = f'{lid}:missing all calculated flows'
             all_messages.append(message)
-            
+
     # Write all_messages to huc-specific file.
     print("Writing message file for huc")
     huc_messages_txt_file = os.path.join(huc_messages_dir, str(huc) + '_messages.txt')
@@ -130,17 +166,26 @@ def process_generate_flows(huc, huc_dictionary, threshold_url, all_lists, worksp
             f.write("%s\n" % item)
 
 
-def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based, fim_dir, lid_to_run, attributes_dir="", job_number_huc=1):
+def generate_catfim_flows(
+    workspace,
+    nwm_us_search,
+    nwm_ds_search,
+    stage_based,
+    fim_dir,
+    lid_to_run,
+    attributes_dir="",
+    job_number_huc=1,
+):
     '''
-    This will create static flow files for all nws_lids and save to the 
+    This will create static flow files for all nws_lids and save to the
     workspace directory with the following format:
     huc code
         nws_lid_code
             threshold (action/minor/moderate/major if they exist/are defined by WRDS)
                 flow file (ahps_{lid code}_huc_{huc 8 code}_flows_{threshold}.csv)
-    
-    This will use the WRDS API to get the nwm segments as well as the flow 
-    values for each threshold at each nws_lid and then create the necessary 
+
+    This will use the WRDS API to get the nwm segments as well as the flow
+    values for each threshold at each nws_lid and then create the necessary
     flow file to use for inundation mapping.
     Parameters
     ----------
@@ -152,120 +197,160 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
         Downstream distance (in miles) for walking down NWM network.
     wbd_path : STR
         Location of HUC geospatial data (geopackage).
-        
+
     Returns
     -------
     None.
     '''
-        
+
     all_start = datetime.now()
     API_BASE_URL, WBD_LAYER = get_env_paths()
-    #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
+    # Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
     workspace = Path(workspace)
     nwm_us_search = int(nwm_us_search)
     nwm_ds_search = int(nwm_ds_search)
     metadata_url = f'{API_BASE_URL}/metadata'
     threshold_url = f'{API_BASE_URL}/nws_threshold'
     ###################################################################
-    
+
     # Create workspace
-    workspace.mkdir(parents=True,exist_ok = True)
+    workspace.mkdir(parents=True, exist_ok=True)
 
     # Create HUC message directory to store messages that will be read and joined after multiprocessing
     huc_messages_dir = os.path.join(workspace, 'huc_messages')
     if not os.path.exists(huc_messages_dir):
         os.mkdir(huc_messages_dir)
-    
+
     # Open NWM flows geopackage
     nwm_flows_gpkg = r'/data/inputs/nwm_hydrofabric/nwm_flows.gpkg'
     nwm_flows_df = gpd.read_file(nwm_flows_gpkg)
 
     print(f'Retrieving metadata for site(s): {lid_to_run}...')
     start_dt = datetime.now()
-    
-    #Get metadata for 'CONUS'
+
+    # Get metadata for 'CONUS'
     print(metadata_url)
     if lid_to_run != 'all':
-        all_lists, conus_dataframe = get_metadata(metadata_url, select_by = 'nws_lid', selector = [lid_to_run], must_include = 'nws_data.rfc_forecast_point', upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        all_lists, conus_dataframe = get_metadata(
+            metadata_url,
+            select_by='nws_lid',
+            selector=[lid_to_run],
+            must_include='nws_data.rfc_forecast_point',
+            upstream_trace_distance=nwm_us_search,
+            downstream_trace_distance=nwm_ds_search,
+        )
     else:
         # Get CONUS metadata
-        conus_list, conus_dataframe = get_metadata(metadata_url, select_by = 'nws_lid', selector = ['all'], must_include = 'nws_data.rfc_forecast_point', upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        conus_list, conus_dataframe = get_metadata(
+            metadata_url,
+            select_by='nws_lid',
+            selector=['all'],
+            must_include='nws_data.rfc_forecast_point',
+            upstream_trace_distance=nwm_us_search,
+            downstream_trace_distance=nwm_ds_search,
+        )
         # Get metadata for Islands
-        islands_list, islands_dataframe = get_metadata(metadata_url, select_by = 'state', selector = ['HI','PR'] , must_include = None, upstream_trace_distance = nwm_us_search, downstream_trace_distance = nwm_ds_search)
+        islands_list, islands_dataframe = get_metadata(
+            metadata_url,
+            select_by='state',
+            selector=['HI', 'PR'],
+            must_include=None,
+            upstream_trace_distance=nwm_us_search,
+            downstream_trace_distance=nwm_ds_search,
+        )
         # Append the dataframes and lists
         all_lists = conus_list + islands_list
     print(len(all_lists))
-    
+
     end_dt = datetime.now()
     time_duration = end_dt - start_dt
     print(f"Retrieving metadata Duration: {str(time_duration).split('.')[0]}")
     print()
-    
+
     print('Determining HUC using WBD layer...')
-    start_dt = datetime.now()    
-    
-    # Assign HUCs to all sites using a spatial join of the FIM 3 HUC layer. 
+    start_dt = datetime.now()
+
+    # Assign HUCs to all sites using a spatial join of the FIM 3 HUC layer.
     # Get a dictionary of hucs (key) and sites (values) as well as a GeoDataFrame
     # of all sites used later in script.
-    huc_dictionary, out_gdf = aggregate_wbd_hucs(metadata_list = all_lists, wbd_huc8_path = WBD_LAYER, retain_attributes=True)
+    huc_dictionary, out_gdf = aggregate_wbd_hucs(
+        metadata_list=all_lists, wbd_huc8_path=WBD_LAYER, retain_attributes=True
+    )
     # Drop list fields if invalid
     out_gdf = out_gdf.drop(['downstream_nwm_features'], axis=1, errors='ignore')
     out_gdf = out_gdf.drop(['upstream_nwm_features'], axis=1, errors='ignore')
     out_gdf = out_gdf.astype({'metadata_sources': str})
-        
-    end_dt = datetime.now()        
+
+    end_dt = datetime.now()
     time_duration = end_dt - start_dt
     print(f"Determining HUC using WBD layer Duration: {str(time_duration).split('.')[0]}")
     print()
-        
+
     if stage_based:
         return huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists, nwm_flows_df
 
     print("Generating flows for hucs using " + str(job_number_huc) + " jobs...")
-    start_dt = datetime.now()        
-    
+    start_dt = datetime.now()
+
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
         for huc in huc_dictionary:
-            executor.submit(process_generate_flows, huc, huc_dictionary, threshold_url, all_lists, workspace, attributes_dir, huc_messages_dir, nwm_flows_df)
-            
+            executor.submit(
+                process_generate_flows,
+                huc,
+                huc_dictionary,
+                threshold_url,
+                all_lists,
+                workspace,
+                attributes_dir,
+                huc_messages_dir,
+                nwm_flows_df,
+            )
+
     end_dt = datetime.now()
     time_duration = end_dt - start_dt
     print(f"Generating flows for hucs Duration: {str(time_duration).split('.')[0]}")
     print()
-            
+
     print('Wrapping up flows generation...')
-    #Recursively find all *_attributes csv files and append
+    # Recursively find all *_attributes csv files and append
     csv_files = os.listdir(attributes_dir)
     all_csv_df = pd.DataFrame()
     for csv in csv_files:
         full_csv_path = os.path.join(attributes_dir, csv)
         # Huc has to be read in as string to preserve leading zeros.
-        temp_df = pd.read_csv(full_csv_path, dtype={'huc':str})
-        all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index = True)
+        temp_df = pd.read_csv(full_csv_path, dtype={'huc': str})
+        all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index=True)
     # Write to file
-    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index = False)
-   
+    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index=False)
+
     # This section populates a shapefile of all potential sites and details
     # whether it was mapped or not (mapped field) and if not, why (status field).
-    
+
     # Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
-    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)    
-    viz_out_gdf.rename(columns = {'identifiers_nwm_feature_id': 'nwm_seg', 'identifiers_nws_lid':'nws_lid', 'identifiers_usgs_site_code':'usgs_gage'}, inplace = True)
+    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)
+    viz_out_gdf.rename(
+        columns={
+            'identifiers_nwm_feature_id': 'nwm_seg',
+            'identifiers_nws_lid': 'nws_lid',
+            'identifiers_usgs_site_code': 'usgs_gage',
+        },
+        inplace=True,
+    )
     viz_out_gdf['nws_lid'] = viz_out_gdf['nws_lid'].str.lower()
-    
+
     # Using list of csv_files, populate DataFrame of all nws_lids that had
     # a flow file produced and denote with "mapped" column.
     nws_lids = []
     for csv_file in csv_files:
         nws_lids.append(csv_file.split('_attributes')[0])
-    lids_df = pd.DataFrame(nws_lids, columns = ['nws_lid'])
+    lids_df = pd.DataFrame(nws_lids, columns=['nws_lid'])
     lids_df['mapped'] = 'yes'
-    
-    # Identify what lids were mapped by merging with lids_df. Populate 
+
+    # Identify what lids were mapped by merging with lids_df. Populate
     # 'mapped' column with 'No' if sites did not map.
-    viz_out_gdf = viz_out_gdf.merge(lids_df, how = 'left', on = 'nws_lid')    
+    viz_out_gdf = viz_out_gdf.merge(lids_df, how='left', on='nws_lid')
     viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
-    
+
     # Read all messages for all HUCs TODO
     huc_message_list = []
     huc_messages_dir_list = os.listdir(huc_messages_dir)
@@ -276,41 +361,62 @@ def generate_catfim_flows(workspace, nwm_us_search, nwm_ds_search, stage_based,
                 lines = f.readlines()
                 for line in lines:
                     huc_message_list.append(line)
-    
+
     # Write messages to DataFrame, split into columns, aggregate messages.
-    messages_df  = pd.DataFrame(huc_message_list, columns = ['message'])
-    messages_df = messages_df['message'].str.split(':', n = 1, expand = True).rename(columns={0:'nws_lid', 1:'status'})   
+    messages_df = pd.DataFrame(huc_message_list, columns=['message'])
+    messages_df = (
+        messages_df['message'].str.split(':', n=1, expand=True).rename(columns={0: 'nws_lid', 1: 'status'})
+    )
     status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
-    
-    # Join messages to populate status field to candidate sites. Assign 
+
+    # Join messages to populate status field to candidate sites. Assign
     # status for null fields.
-    viz_out_gdf = viz_out_gdf.merge(status_df, how = 'left', on = 'nws_lid')
+    viz_out_gdf = viz_out_gdf.merge(status_df, how='left', on='nws_lid')
     viz_out_gdf['status'] = viz_out_gdf['status'].fillna('all calculated flows available')
-    
+
     # Filter out columns and write out to file
-#    viz_out_gdf = viz_out_gdf.filter(['nws_lid','usgs_gage','nwm_seg','HUC8','mapped','status','geometry'])
+    # viz_out_gdf = viz_out_gdf.filter(
+    #     ['nws_lid', 'usgs_gage', 'nwm_seg', 'HUC8', 'mapped', 'status', 'geometry']
+    # )
     nws_lid_layer = os.path.join(workspace, 'nws_lid_sites.gpkg').replace('flows', 'mapping')
 
     viz_out_gdf.to_file(nws_lid_layer, driver='GPKG')
-    
+
     # time operation
     all_end = datetime.now()
     all_time_duration = all_end - all_start
     print(f"Duration: {str(all_time_duration).split('.')[0]}")
     print()
-    
+
     return nws_lid_layer
 
-    
+
 if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Create forecast files for all nws_lid sites')
-    parser.add_argument('-w', '--workspace', help = 'Workspace where all data will be stored.', required = True)
-    parser.add_argument('-u', '--nwm_us_search',  help = 'Walk upstream on NWM network this many miles', required = True)
-    parser.add_argument('-d', '--nwm_ds_search', help = 'Walk downstream on NWM network this many miles', required = True)
-    parser.add_argument('-a', '--stage_based', help = 'Run stage-based CatFIM instead of flow-based? NOTE: flow-based CatFIM is the default.', required=False, default=False, action='store_true')
-    parser.add_argument('-f', '--fim-dir', help='Path to FIM outputs directory. Only use this option if you are running in alt-catfim mode.',required=False,default="")
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Create forecast files for all nws_lid sites')
+    parser.add_argument('-w', '--workspace', help='Workspace where all data will be stored.', required=True)
+    parser.add_argument(
+        '-u', '--nwm_us_search', help='Walk upstream on NWM network this many miles', required=True
+    )
+    parser.add_argument(
+        '-d', '--nwm_ds_search', help='Walk downstream on NWM network this many miles', required=True
+    )
+    parser.add_argument(
+        '-a',
+        '--stage_based',
+        help='Run stage-based CatFIM instead of flow-based? NOTE: flow-based CatFIM is the default.',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-f',
+        '--fim-dir',
+        help='Path to FIM outputs directory. Only use this option if you are running in alt-catfim mode.',
+        required=False,
+        default="",
+    )
     args = vars(parser.parse_args())
 
-    #Run get_env_paths and static_flow_lids
+    # Run get_env_paths and static_flow_lids
     generate_catfim_flows(**args)
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index 85a5fe5f6..1fb068d77 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -1,26 +1,27 @@
 #!/usr/bin/env python3
 
-import sys
-import os
-from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 import argparse
+import os
+import sys
 import traceback
-import rasterio
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+
 import geopandas as gpd
 import pandas as pd
+import rasterio
+from inundate_gms import Inundate_gms
+from mosaic_inundation import Mosaic_inundation
 from rasterio.features import shapes
-from shapely.geometry.polygon import Polygon
 from shapely.geometry.multipolygon import MultiPolygon
-sys.path.append('/foss_fim/src')
-from utils.shared_variables import PREP_PROJECTION,VIZ_PROJECTION
-from utils.shared_functions import getDriver
-from mosaic_inundation import Mosaic_inundation
-from inundate_gms import Inundate_gms
+from shapely.geometry.polygon import Polygon
 
+from utils.shared_functions import getDriver
+from utils.shared_variables import PREP_PROJECTION, VIZ_PROJECTION
 
-def generate_categorical_fim(fim_run_dir, source_flow_dir, output_catfim_dir, 
-                             job_number_huc, job_number_inundate, depthtif, log_file):
 
+def generate_categorical_fim(
+    fim_run_dir, source_flow_dir, output_catfim_dir, job_number_huc, job_number_inundate, depthtif, log_file
+):
     source_flow_dir_list = os.listdir(source_flow_dir)
     output_flow_dir_list = os.listdir(fim_run_dir)
 
@@ -34,7 +35,7 @@ def generate_categorical_fim(fim_run_dir, source_flow_dir, output_catfim_dir,
 
     # Loop through matching huc directories in the source_flow directory
     matching_hucs = list(set(output_flow_dir_list) & set(source_flow_dir_list))
-    
+
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
         for huc in matching_hucs:
             if "." in huc:
@@ -64,42 +65,70 @@ def generate_categorical_fim(fim_run_dir, source_flow_dir, output_catfim_dir,
                 for magnitude in thresholds_dir_list:
                     if "." in magnitude:
                         continue
-                    magnitude_flows_csv = os.path.join(ahps_site_parent, magnitude, 'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                    magnitude_flows_csv = os.path.join(
+                        ahps_site_parent,
+                        magnitude,
+                        'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv',
+                    )
                     if os.path.exists(magnitude_flows_csv):
-                        output_extent_grid = os.path.join(cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif')
+                        output_extent_grid = os.path.join(
+                            cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif'
+                        )
                         try:
-                            executor.submit(run_inundation, magnitude_flows_csv, huc, output_extent_grid, ahps_site, magnitude, log_file, fim_run_dir, job_number_inundate)
+                            executor.submit(
+                                run_inundation,
+                                magnitude_flows_csv,
+                                huc,
+                                output_extent_grid,
+                                ahps_site,
+                                magnitude,
+                                log_file,
+                                fim_run_dir,
+                                job_number_inundate,
+                            )
                         except Exception:
                             traceback.print_exc()
                             sys.exit(1)
-                                
 
-def run_inundation(magnitude_flows_csv, huc, output_extent_grid, ahps_site, magnitude, log_file, fim_run_dir, job_number_inundate):
 
+def run_inundation(
+    magnitude_flows_csv,
+    huc,
+    output_extent_grid,
+    ahps_site,
+    magnitude,
+    log_file,
+    fim_run_dir,
+    job_number_inundate,
+):
     huc_dir = os.path.join(fim_run_dir, huc)
     try:
         print("Running Inundate_gms for " + huc)
-        map_file = Inundate_gms(  hydrofabric_dir = fim_run_dir, 
-                                         forecast = magnitude_flows_csv, 
-                                         num_workers = job_number_inundate,
-                                         hucs = huc,
-                                         inundation_raster = output_extent_grid,
-                                         inundation_polygon = None,
-                                         depths_raster = None,
-                                         verbose = False,
-                                         log_file = None,
-                                         output_fileNames = None )
+        map_file = Inundate_gms(
+            hydrofabric_dir=fim_run_dir,
+            forecast=magnitude_flows_csv,
+            num_workers=job_number_inundate,
+            hucs=huc,
+            inundation_raster=output_extent_grid,
+            inundation_polygon=None,
+            depths_raster=None,
+            verbose=False,
+            log_file=None,
+            output_fileNames=None,
+        )
         print("Mosaicking for " + huc)
-        Mosaic_inundation( map_file,
-                            mosaic_attribute = 'inundation_rasters',
-                            mosaic_output = output_extent_grid,
-                            mask = os.path.join(huc_dir,'wbd.gpkg'),
-                            unit_attribute_name = 'huc8',
-                            nodata = -9999,
-                            workers = 1,
-                            remove_inputs = False,
-                            subset = None,
-                            verbose = False )
+        Mosaic_inundation(
+            map_file,
+            mosaic_attribute='inundation_rasters',
+            mosaic_output=output_extent_grid,
+            mask=os.path.join(huc_dir, 'wbd.gpkg'),
+            unit_attribute_name='huc8',
+            nodata=-9999,
+            workers=1,
+            remove_inputs=False,
+            subset=None,
+            verbose=False,
+        )
 
     except Exception:
         # Log errors and their tracebacks
@@ -107,19 +136,19 @@ def run_inundation(magnitude_flows_csv, huc, output_extent_grid, ahps_site, magn
         f.write(f"{output_extent_grid} - inundation error: {traceback.format_exc()}\n")
         f.close()
 
-    #Inundation.py appends the huc code to the supplied output_extent_grid.
-    #Modify output_extent_grid to match inundation.py saved filename.
-    #Search for this file, if it didn't create, send message to log file.
-    base_file_path,extension = os.path.splitext(output_extent_grid)
-    saved_extent_grid_filename = "{}_{}{}".format(base_file_path,huc,extension)
+    # Inundation.py appends the huc code to the supplied output_extent_grid.
+    # Modify output_extent_grid to match inundation.py saved filename.
+    # Search for this file, if it didn't create, send message to log file.
+    base_file_path, extension = os.path.splitext(output_extent_grid)
+    saved_extent_grid_filename = "{}_{}{}".format(base_file_path, huc, extension)
     if not os.path.exists(saved_extent_grid_filename):
         with open(log_file, 'a+') as f:
-            f.write('FAILURE_huc_{}:{}:{} map failed to create\n'.format(huc,ahps_site,magnitude))
-
+            f.write('FAILURE_huc_{}:{}:{} map failed to create\n'.format(huc, ahps_site, magnitude))
 
-def post_process_huc_level(job_number_tif, ahps_dir_list, huc_dir, attributes_dir, gpkg_dir, fim_version, huc):
-    
 
+def post_process_huc_level(
+    job_number_tif, ahps_dir_list, huc_dir, attributes_dir, gpkg_dir, fim_version, huc
+):
     # Loop through ahps sites
     for ahps_lid in ahps_dir_list:
         tifs_to_reformat_list = []
@@ -133,7 +162,7 @@ def post_process_huc_level(job_number_tif, ahps_dir_list, huc_dir, attributes_di
 
         # Stage-Based CatFIM uses attributes from individual CSVs instead of the master CSV.
         nws_lid_attributes_filename = os.path.join(attributes_dir, ahps_lid + '_attributes.csv')
-    
+
         print(f"Reformatting TIFs {ahps_lid} for {huc_dir}")
         with ProcessPoolExecutor(max_workers=job_number_tif) as executor:
             for tif_to_process in tifs_to_reformat_list:
@@ -142,19 +171,34 @@ def post_process_huc_level(job_number_tif, ahps_dir_list, huc_dir, attributes_di
                 try:
                     magnitude = os.path.split(tif_to_process)[1].split('_')[1]
                     try:
-                        interval_stage = float((os.path.split(tif_to_process)[1].split('_')[2]).replace('p', '.').replace("ft", ""))
+                        interval_stage = float(
+                            (os.path.split(tif_to_process)[1].split('_')[2])
+                            .replace('p', '.')
+                            .replace("ft", "")
+                        )
                         if interval_stage == 'extent':
                             interval_stage = None
                     except ValueError:
                         interval_stage = None
-                    executor.submit(reformat_inundation_maps, ahps_lid, tif_to_process, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename, interval_stage)
+                    executor.submit(
+                        reformat_inundation_maps,
+                        ahps_lid,
+                        tif_to_process,
+                        gpkg_dir,
+                        fim_version,
+                        huc,
+                        magnitude,
+                        nws_lid_attributes_filename,
+                        interval_stage,
+                    )
                 except Exception as ex:
                     print(f"*** {ex}")
                     traceback.print_exc()
-                
 
-def post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file="", fim_version=""):
-    
+
+def post_process_cat_fim_for_viz(
+    job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file="", fim_version=""
+):
     print("In post processing...")
     # Create workspace
     gpkg_dir = os.path.join(output_catfim_dir, 'gpkg')
@@ -163,16 +207,14 @@ def post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_catfim_d
 
     # Find the FIM version
     merged_layer = os.path.join(output_catfim_dir, 'catfim_library.gpkg')
-    if not os.path.exists(merged_layer): # prevents appending to existing output        
+    if not os.path.exists(merged_layer):  # prevents appending to existing output
         huc_ahps_dir_list = os.listdir(output_catfim_dir)
-        skip_list=['errors','logs','gpkg','missing_files.txt','messages',merged_layer]
+        skip_list = ['errors', 'logs', 'gpkg', 'missing_files.txt', 'messages', merged_layer]
 
         # Loop through all categories
         print("Building list of TIFs to reformat...")
         with ProcessPoolExecutor(max_workers=job_number_huc) as huc_exector:
-            
             for huc in huc_ahps_dir_list:
-
                 if huc in skip_list:
                     continue
                 huc_dir = os.path.join(output_catfim_dir, huc)
@@ -184,9 +226,18 @@ def post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_catfim_d
                 if ahps_dir_list == []:
                     os.rmdir(huc_dir)
                     continue
-                
-                huc_exector.submit(post_process_huc_level, job_number_tif, ahps_dir_list, huc_dir, attributes_dir, gpkg_dir, fim_version, huc)
-                
+
+                huc_exector.submit(
+                    post_process_huc_level,
+                    job_number_tif,
+                    ahps_dir_list,
+                    huc_dir,
+                    attributes_dir,
+                    gpkg_dir,
+                    fim_version,
+                    huc,
+                )
+
         # Merge all layers
         print(f"Merging {len(os.listdir(gpkg_dir))} layers...")
         for layer in os.listdir(gpkg_dir):
@@ -194,31 +245,42 @@ def post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_catfim_d
             diss_extent_filename = os.path.join(gpkg_dir, layer)
             diss_extent = gpd.read_file(diss_extent_filename)
             diss_extent['viz'] = 'yes'
-    
+
             # Write/append aggregate diss_extent
             print(f"Merging layer: {layer}")
             if os.path.isfile(merged_layer):
-                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False, mode='a')
+                diss_extent.to_file(merged_layer, driver=getDriver(merged_layer), index=False, mode='a')
             else:
-                diss_extent.to_file(merged_layer,driver=getDriver(merged_layer),index=False)
+                diss_extent.to_file(merged_layer, driver=getDriver(merged_layer), index=False)
             del diss_extent
-    
-            #shutil.rmtree(gpkg_dir)  # TODO
+
+            # shutil.rmtree(gpkg_dir)  # TODO
 
     else:
         print(f"{merged_layer} already exists.")
 
 
-def reformat_inundation_maps(ahps_lid, extent_grid, gpkg_dir, fim_version, huc, magnitude, nws_lid_attributes_filename, interval_stage=None):
-
+def reformat_inundation_maps(
+    ahps_lid,
+    extent_grid,
+    gpkg_dir,
+    fim_version,
+    huc,
+    magnitude,
+    nws_lid_attributes_filename,
+    interval_stage=None,
+):
     try:
         # Convert raster to to shapes
         with rasterio.open(extent_grid) as src:
             image = src.read(1)
             mask = image > 0
-            
+
         # Aggregate shapes
-        results = ({'properties': {'extent': 1}, 'geometry': s} for i, (s, v) in enumerate(shapes(image, mask=mask,transform=src.transform)))
+        results = (
+            {'properties': {'extent': 1}, 'geometry': s}
+            for i, (s, v) in enumerate(shapes(image, mask=mask, transform=src.transform))
+        )
 
         # Convert list of shapes to polygon
         extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
@@ -236,21 +298,34 @@ def reformat_inundation_maps(ahps_lid, extent_grid, gpkg_dir, fim_version, huc,
         extent_poly_diss = extent_poly_diss.to_crs(VIZ_PROJECTION)
 
         # Join attributes
-        nws_lid_attributes_table = pd.read_csv(nws_lid_attributes_filename, dtype={'huc':str})
-        nws_lid_attributes_table = nws_lid_attributes_table.loc[(nws_lid_attributes_table.magnitude==magnitude) & (nws_lid_attributes_table.nws_lid==ahps_lid)]
-        extent_poly_diss = extent_poly_diss.merge(nws_lid_attributes_table, left_on=['ahps_lid','magnitude','huc'], right_on=['nws_lid','magnitude','huc'])
+        nws_lid_attributes_table = pd.read_csv(nws_lid_attributes_filename, dtype={'huc': str})
+        nws_lid_attributes_table = nws_lid_attributes_table.loc[
+            (nws_lid_attributes_table.magnitude == magnitude) & (nws_lid_attributes_table.nws_lid == ahps_lid)
+        ]
+        extent_poly_diss = extent_poly_diss.merge(
+            nws_lid_attributes_table,
+            left_on=['ahps_lid', 'magnitude', 'huc'],
+            right_on=['nws_lid', 'magnitude', 'huc'],
+        )
         extent_poly_diss = extent_poly_diss.drop(columns='nws_lid')
         # Save dissolved multipolygon
         handle = os.path.split(extent_grid)[1].replace('.tif', '')
         diss_extent_filename = os.path.join(gpkg_dir, f"{handle}_{huc}_dissolved.gpkg")
-        extent_poly_diss["geometry"] = [MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in extent_poly_diss["geometry"]]
-        
+        extent_poly_diss["geometry"] = [
+            MultiPolygon([feature]) if type(feature) == Polygon else feature
+            for feature in extent_poly_diss["geometry"]
+        ]
+
         if not extent_poly_diss.empty:
-            extent_poly_diss.to_file(diss_extent_filename,driver=getDriver(diss_extent_filename),index=False)
+            extent_poly_diss.to_file(
+                diss_extent_filename, driver=getDriver(diss_extent_filename), index=False
+            )
 
     except Exception:
         pass
         # Log and clean out the gdb so it's not merged in later
+
+
 #        try:
 #            print(e)
 ##            f = open(log_file, 'a+')
@@ -260,8 +335,16 @@ def reformat_inundation_maps(ahps_lid, extent_grid, gpkg_dir, fim_version, huc,
 #            pass
 
 
-def manage_catfim_mapping(fim_run_dir, source_flow_dir, output_catfim_dir, attributes_dir, job_number_huc, job_number_inundate, overwrite, depthtif):
-    
+def manage_catfim_mapping(
+    fim_run_dir,
+    source_flow_dir,
+    output_catfim_dir,
+    attributes_dir,
+    job_number_huc,
+    job_number_inundate,
+    overwrite,
+    depthtif,
+):
     # Create output directory
     if not os.path.exists(output_catfim_dir):
         os.mkdir(output_catfim_dir)
@@ -277,23 +360,64 @@ def manage_catfim_mapping(fim_run_dir, source_flow_dir, output_catfim_dir, attri
     job_number_tif = job_number_inundate
 
     print("Generating Categorical FIM")
-    generate_categorical_fim(fim_run_dir, source_flow_dir, output_catfim_dir, job_number_huc, job_number_inundate, depthtif, log_file)
+    generate_categorical_fim(
+        fim_run_dir,
+        source_flow_dir,
+        output_catfim_dir,
+        job_number_huc,
+        job_number_inundate,
+        depthtif,
+        log_file,
+    )
 
     print("Aggregating Categorical FIM")
     # Get fim_version.
-    fim_version = os.path.basename(os.path.normpath(fim_run_dir)).replace('fim_','').replace('_ms_c', '').replace('_', '.')
-    post_process_cat_fim_for_viz(job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file, fim_version)
+    fim_version = (
+        os.path.basename(os.path.normpath(fim_run_dir))
+        .replace('fim_', '')
+        .replace('_ms_c', '')
+        .replace('_', '.')
+    )
+    post_process_cat_fim_for_viz(
+        job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file, fim_version
+    )
 
 
 if __name__ == '__main__':
-
     # Parse arguments
     parser = argparse.ArgumentParser(description='Categorical inundation mapping for FOSS FIM.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh',required=True)
-    parser.add_argument('-s', '--source-flow-dir',help='Path to directory containing flow CSVs to use to generate categorical FIM.',required=True, default="")
-    parser.add_argument('-o', '--output-catfim-dir',help='Path to directory where categorical FIM outputs will be written.',required=True, default="")
-    parser.add_argument('-j','--number-of-jobs',help='Number of processes to use. Default is 1.',required=False, default="1",type=int)
-    parser.add_argument('-depthtif','--write-depth-tiff',help='Using this option will write depth TIFFs.',required=False, action='store_true')
+    parser.add_argument(
+        '-r', '--fim-run-dir', help='Name of directory containing outputs of fim_run.sh', required=True
+    )
+    parser.add_argument(
+        '-s',
+        '--source-flow-dir',
+        help='Path to directory containing flow CSVs to use to generate categorical FIM.',
+        required=True,
+        default="",
+    )
+    parser.add_argument(
+        '-o',
+        '--output-catfim-dir',
+        help='Path to directory where categorical FIM outputs will be written.',
+        required=True,
+        default="",
+    )
+    parser.add_argument(
+        '-j',
+        '--number-of-jobs',
+        help='Number of processes to use. Default is 1.',
+        required=False,
+        default="1",
+        type=int,
+    )
+    parser.add_argument(
+        '-depthtif',
+        '--write-depth-tiff',
+        help='Using this option will write depth TIFFs.',
+        required=False,
+        action='store_true',
+    )
 
     args = vars(parser.parse_args())
 
@@ -302,5 +426,5 @@ def manage_catfim_mapping(fim_run_dir, source_flow_dir, output_catfim_dir, attri
     output_catfim_dir = args['output_catfim_dir']
     number_of_jobs = int(args['number_of_jobs'])
     depthtif = args['write_depth_tiff']
-    
+
     manage_catfim_mapping(fim_run_dir, source_flow_dir, output_catfim_dir, number_of_jobs, depthtif)
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
index 005befebe..88ca96e60 100644
--- a/tools/generate_nws_lid.py
+++ b/tools/generate_nws_lid.py
@@ -1,29 +1,32 @@
 #!/usr/bin/env python3
 
-from pathlib import Path
-import pandas as pd
-import geopandas as gpd
-from collections import defaultdict
-from tools_shared_functions import aggregate_wbd_hucs, get_metadata
 import argparse
-from dotenv import load_dotenv
 import os
 import sys
-sys.path.append('/foss_fim/src')
+from collections import defaultdict
+from pathlib import Path
+
+import geopandas as gpd
+import pandas as pd
+from dotenv import load_dotenv
+from tools_shared_functions import aggregate_wbd_hucs, get_metadata
+
 from utils.shared_variables import PREP_PROJECTION
 
+
 load_dotenv()
-#import variables from .env file
+# import variables from .env file
 API_BASE_URL = os.getenv("API_BASE_URL")
 EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
 WBD_LAYER = os.getenv("WBD_LAYER")
-#Define path to NWM stream layer
-NWM_FILE='/data/inputs/nwm_hydrofabric/nwm_flows.gpkg'
+# Define path to NWM stream layer
+NWM_FILE = '/data/inputs/nwm_hydrofabric/nwm_flows.gpkg'
 
 
 def generate_nws_lid(workspace):
     '''
-    Generate the nws_lid layer containing all nws_lid points attributed whether site is mainstems and co-located
+    Generate the nws_lid layer containing all nws_lid points attributed
+    whether site is mainstems and co-located
 
     Parameters
     ----------
@@ -35,145 +38,170 @@ def generate_nws_lid(workspace):
     None.
 
     '''
-    
-    ##############################################################################
-    #Get all nws_lid points
+
+    # Get all nws_lid points
     print('Retrieving metadata ..')
-    
+
     metadata_url = f'{API_BASE_URL}/metadata/'
-    #Trace downstream from all rfc_forecast_point.
+    # Trace downstream from all rfc_forecast_point.
     select_by = 'nws_lid'
     selector = ['all']
     must_include = 'nws_data.rfc_forecast_point'
     downstream_trace_distance = 'all'
-    fcst_list, fcst_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
-    
-    #Get list of all evaluated sites not in fcst_list
+    fcst_list, fcst_dataframe = get_metadata(
+        metadata_url=metadata_url,
+        select_by=select_by,
+        selector=selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=downstream_trace_distance,
+    )
+
+    # Get list of all evaluated sites not in fcst_list
     fcst_list_sites = [record.get('identifiers').get('nws_lid').lower() for record in fcst_list]
     evaluated_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].str.lower().to_list()
-    evaluated_sites= list(set(evaluated_sites) - set(fcst_list_sites))
-    
-    #Trace downstream from all evaluated sites not in fcst_list
+    evaluated_sites = list(set(evaluated_sites) - set(fcst_list_sites))
+
+    # Trace downstream from all evaluated sites not in fcst_list
     select_by = 'nws_lid'
     selector = evaluated_sites
     must_include = None
     downstream_trace_distance = 'all'
-    eval_list, eval_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
-    
-    #Trace downstream from all sites in HI/PR.
+    eval_list, eval_dataframe = get_metadata(
+        metadata_url=metadata_url,
+        select_by=select_by,
+        selector=selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=downstream_trace_distance,
+    )
+
+    # Trace downstream from all sites in HI/PR.
     select_by = 'state'
-    selector = ['HI','PR']
+    selector = ['HI', 'PR']
     must_include = None
     downstream_trace_distance = 'all'
-    islands_list, islands_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
-    
-    #Append all lists
+    islands_list, islands_dataframe = get_metadata(
+        metadata_url=metadata_url,
+        select_by=select_by,
+        selector=selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=downstream_trace_distance,
+    )
+
+    # Append all lists
     all_lists = fcst_list + eval_list + islands_list
-    
-    ###############################################################################
-    #Compile NWM segments from all_lists
-    
-    #Get dictionary of downstream segment (key) and target segments (values)
-    #Get dictionary of target segment (key) and site code (value) 
+
+    # Compile NWM segments from all_lists
+
+    # Get dictionary of downstream segment (key) and target segments (values)
+    # Get dictionary of target segment (key) and site code (value)
     downstream = defaultdict(list)
     target = defaultdict(list)
-    #For each lid metadata dictionary in list
+    # For each lid metadata dictionary in list
     for lid in all_lists:
         site = lid.get('identifiers').get('nws_lid')
-        #Get the nwm feature id associated with the location
+        # Get the nwm feature id associated with the location
         location_nwm_seg = lid.get('identifiers').get('nwm_feature_id')
-        #get all downstream segments
+        # get all downstream segments
         downstream_nwm_segs = lid.get('downstream_nwm_features')
-        #If valid location_nwm_segs construct two dictionaries.
-        if location_nwm_seg: 
-            #Dictionary with target segment and site
-            target[int(location_nwm_seg)].append(site)         
-           #Dictionary of key (2nd to last element) and value (target segment)
-           #2nd to last element used because last element is always 0 (ocean) and the 2nd to last allows for us to get the river 'tree' (Mississippi, Colorado, etc)
+        # If valid location_nwm_segs construct two dictionaries.
+        if location_nwm_seg:
+            # Dictionary with target segment and site
+            target[int(location_nwm_seg)].append(site)
+            # Dictionary of key (2nd to last element) and value (target segment)
+            # 2nd to last element used because last element is always 0 (ocean) and the 2nd to last allows
+            # for us to get the river 'tree' (Mississippi, Colorado, etc)
             value = location_nwm_seg
             if not downstream_nwm_segs:
-                #Special case, no downstream nwm segments are returned (PR/VI/HI).
+                # Special case, no downstream nwm segments are returned (PR/VI/HI).
                 key = location_nwm_seg
             elif len(downstream_nwm_segs) == 1:
-                #Special case, the nws_lid is within 1 segment of the ocean (0)
+                # Special case, the nws_lid is within 1 segment of the ocean (0)
                 key = location_nwm_seg
-            elif len(downstream_nwm_segs)>1:
-                #Otherwise, 2nd to last element used to identify proper river system.
-                key = downstream_nwm_segs[-2]            
-            #Dictionary with key of 2nd to last downstream segment and value of site nwm segment 
+            elif len(downstream_nwm_segs) > 1:
+                # Otherwise, 2nd to last element used to identify proper river system.
+                key = downstream_nwm_segs[-2]
+            # Dictionary with key of 2nd to last downstream segment and value of site nwm segment
             downstream[int(key)].append(int(value))
-    ###############################################################################
-    #Walk downstream the network and identify headwater points
+
+    # Walk downstream the network and identify headwater points
     print('Traversing network..')
-    
-    #Import NWM file and create dictionary of network and create the NWM network dictionary.
+
+    # Import NWM file and create dictionary of network and create the NWM network dictionary.
     nwm_gdf = gpd.read_file(NWM_FILE)
     network = nwm_gdf.groupby('ID')['to'].apply(list).to_dict()
-    
-    #Walk through network and find headwater points
+
+    # Walk through network and find headwater points
     all_dicts = {}
-    for tree, targets in downstream.items():    
-        #All targets are assigned headwaters
-        sub_dict = {i:'is_headwater' for i in targets}
-        #Walk downstream of each target
+    for tree, targets in downstream.items():
+        # All targets are assigned headwaters
+        sub_dict = {i: 'is_headwater' for i in targets}
+        # Walk downstream of each target
         for i in targets:
-            #Check to see element is not a headwater
+            # Check to see element is not a headwater
             if sub_dict[i] == 'not_headwater':
                 continue
-            #Get from_node and to_node.
+            # Get from_node and to_node.
             from_node = i
             [to_node] = network[from_node]
-            #Walk downstream from target
-            while to_node>0:
-                #Check if to_node is in targets list
+            # Walk downstream from target
+            while to_node > 0:
+                # Check if to_node is in targets list
                 if to_node in targets:
-                    sub_dict[to_node] = 'not_headwater'  
-                #Assign downstream ID as to_node
+                    sub_dict[to_node] = 'not_headwater'
+                # Assign downstream ID as to_node
                 [to_node] = network[to_node]
-  
-        #Append status to master dictionary
+
+        # Append status to master dictionary
         all_dicts.update(sub_dict)
-    
-    #Create dictionaries of nws_lid (key) and headwater status (value) and nws_lid (key) and co-located with same feature_id(value)
+
+    # Create dictionaries of nws_lid (key) and headwater status (value) and nws_lid (key) and
+    #   co-located with same feature_id(value)
     final_dict = {}
     duplicate_dict = {}
-    for key,status in all_dicts.items():
-        site_list = target[key]       
+    for key, status in all_dicts.items():
+        site_list = target[key]
         for site in site_list:
             final_dict[site] = status
             if len(site_list) > 1:
                 duplicate_dict[site] = 'is_colocated'
             else:
                 duplicate_dict[site] = 'not_colocated'
-    
-    ##############################################################################
-    #Get Spatial data and populate headwater/duplicate attributes
+
+    # Get Spatial data and populate headwater/duplicate attributes
     print('Attributing nws_lid layer..')
-            
-    #Geodataframe from all_lists, reproject, and reset index.
-    trash, nws_lid_gdf = aggregate_wbd_hucs(all_lists, WBD_LAYER, retain_attributes = False)
-    nws_lid_gdf.columns = [name.replace('identifiers_','') for name in nws_lid_gdf.columns]
-    nws_lid_gdf.to_crs(PREP_PROJECTION, inplace = True)
-    nws_lid_gdf.reset_index(drop = True)
-    
-    #Create DataFrames of headwater and duplicates and join.
-    final_dict_pd = pd.DataFrame(list(final_dict.items()), columns = ['nws_lid','is_headwater'])
-    duplicate_dict_pd = pd.DataFrame(list(duplicate_dict.items()),columns = ['nws_lid','is_colocated'])
-    attributes = final_dict_pd.merge(duplicate_dict_pd, on = 'nws_lid')
-    attributes.replace({'is_headwater': True,'is_colocated': True,'not_headwater': False,'not_colocated':False}, inplace = True)
-    
-    #Join attributes, remove sites with no assigned nwm_feature_id and write to file
-    joined = nws_lid_gdf.merge(attributes, on='nws_lid', how = 'left')
-    joined.dropna(subset =['nwm_feature_id'], inplace = True)
-    Path(workspace).mkdir(parents = True, exist_ok = True)
-    joined.to_file(Path(workspace) / 'nws_lid.gpkg', driver = 'GPKG')
+
+    # Geodataframe from all_lists, reproject, and reset index.
+    trash, nws_lid_gdf = aggregate_wbd_hucs(all_lists, WBD_LAYER, retain_attributes=False)
+    nws_lid_gdf.columns = [name.replace('identifiers_', '') for name in nws_lid_gdf.columns]
+    nws_lid_gdf.to_crs(PREP_PROJECTION, inplace=True)
+    nws_lid_gdf.reset_index(drop=True)
+
+    # Create DataFrames of headwater and duplicates and join.
+    final_dict_pd = pd.DataFrame(list(final_dict.items()), columns=['nws_lid', 'is_headwater'])
+    duplicate_dict_pd = pd.DataFrame(list(duplicate_dict.items()), columns=['nws_lid', 'is_colocated'])
+    attributes = final_dict_pd.merge(duplicate_dict_pd, on='nws_lid')
+    attributes.replace(
+        {'is_headwater': True, 'is_colocated': True, 'not_headwater': False, 'not_colocated': False},
+        inplace=True,
+    )
+
+    # Join attributes, remove sites with no assigned nwm_feature_id and write to file
+    joined = nws_lid_gdf.merge(attributes, on='nws_lid', how='left')
+    joined.dropna(subset=['nwm_feature_id'], inplace=True)
+    Path(workspace).mkdir(parents=True, exist_ok=True)
+    joined.to_file(Path(workspace) / 'nws_lid.gpkg', driver='GPKG')
 
 
 if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Create spatial data of nws_lid points attributed with mainstems and colocated.')
-    parser.add_argument('-w', '--workspace', help = 'Workspace where all data will be stored.', required = True)    
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description='Create spatial data of nws_lid points attributed with mainstems and colocated.'
+    )
+    parser.add_argument('-w', '--workspace', help='Workspace where all data will be stored.', required=True)
     args = vars(parser.parse_args())
-    
-    #Run get_env_paths and static_flow_lids
+
+    # Run get_env_paths and static_flow_lids
     generate_nws_lid(**args)
diff --git a/tools/hash_compare.py b/tools/hash_compare.py
new file mode 100755
index 000000000..49946025a
--- /dev/null
+++ b/tools/hash_compare.py
@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import os
+import re
+
+import geopandas
+from geopandas.testing import assert_geodataframe_equal
+
+
+def main(arg1, arg2, image_only, log_file, gpkg):
+    """
+    This tool compares either directories or single files. It will create and compare a hashing for
+    each file to validate that the files are exactly identical in contents.
+
+    Note: There is an option to compare .gpkg files. This option increases runtimes significantly.
+    Please be advised that this feature has not been thoroughly tested to ensure robustness.
+
+    When arg1 and arg2 are single files, the files names need not match but the extensions must match.
+    """
+
+    if log_file is not None:
+        # Default location of the log file will be the current working directory (where script is called)
+        log = os.path.join(os.getcwd(), log_file)
+        # Initialize list to send discrepancies to a log file
+        print(f" \n \t Will write logging to: {log} \n")
+
+    if os.path.isdir(arg1):
+        compare_fim_runs(arg1, arg2, image_only, log_file, gpkg)
+
+    else:
+        if os.path.splitext(arg1)[1] == '.gpkg' and os.path.splitext(arg2)[1] == '.gpkg':
+            compare_gpkg(arg1, arg2, verbose=True)
+        else:
+            # Calling hashfile() function to obtain hashes
+            # of the files, and saving the result
+            # in a variable
+            f1_hash = hashfile(arg1)
+            f2_hash = hashfile(arg2)
+
+            # Doing primitive string comparison to
+            # check whether the two hashes match or not
+            if f1_hash == f2_hash:
+                print("Both files are the same.")
+                print(f"Hash: {f1_hash}")
+
+            else:
+                print("Files are different!")
+                print(f"Hash of File 1: {f1_hash}")
+                print(f"Hash of File 2: {f2_hash}")
+
+
+def compare_fim_runs(fim_run_dir, previous_fim_run_dir, image_only, log_file, gpkg, list_of_failed_files=[]):
+    for huc in [h for h in os.listdir(fim_run_dir) if re.match(r'\d{8}', h)]:
+        print(huc)
+
+        branch_dir = os.path.join(fim_run_dir, huc, 'branches')
+        branches = [os.path.join(branch_dir, b) for b in os.listdir(branch_dir)]
+
+        for branch in branches:
+            print(f'  {os.path.basename(os.path.abspath(branch))}', end='\r')
+            passing_branch = True
+
+            for file in os.listdir(branch):
+                if image_only and (os.path.splitext(file)[1] != '.tif'):
+                    continue
+
+                elif os.path.splitext(file)[1] == '.gpkg' and gpkg:
+                    f1 = os.path.join(branch, file)
+                    f2 = os.path.join(branch.replace(fim_run_dir, previous_fim_run_dir), file)
+                    compare_gpkg(f1, f2, list_of_failed_files, verbose=False)
+                    continue
+
+                # Skip Geopackages by defualt
+                elif os.path.splitext(file)[1] == '.gpkg':
+                    continue
+
+                f1_hash = hashfile(os.path.join(branch, file))
+                f2_hash = hashfile(os.path.join(branch.replace(fim_run_dir, previous_fim_run_dir), file))
+
+                if f1_hash == f2_hash:
+                    continue
+                else:
+                    if passing_branch:
+                        print(f'  {os.path.basename(os.path.abspath(branch))}')
+                        print(f'    -{file} failed hash compare')
+                        passing_branch = False
+                        if log_file is not None:
+                            list_of_failed_files.append(f'{file}')
+
+            if passing_branch:
+                print(f'  {os.path.basename(os.path.abspath(branch))}...passed')
+
+    if log_file is not None:
+        write_log(list_of_failed_files, log_file)
+
+
+def hashfile(file):
+    # A arbitrary (but fixed) buffer size (change accordingly)
+    # 65536 = 65536 bytes = 64 kilobytes
+    # BUF_SIZE = 65536
+    # 1 MB (size of L2 cache for dev1)
+    BUF_SIZE = 1048576
+
+    # Initializing the sha256() method
+    sha256 = hashlib.sha256()
+
+    # Open file provided
+    with open(file, 'rb') as f:
+        while True:
+            # reading data = BUF_SIZE from the file and saving it in a variable
+            data = f.read(BUF_SIZE)
+
+            if not data:
+                break
+
+            # Passing that data to that sh256 hash function (updating the function with that data)
+            sha256.update(data)
+
+    # sha256.hexdigest() hashes all the input data passed to the sha256() via sha256.update()
+    # Acts as a finalize method, after which all the input data gets hashed hexdigest()
+    # hashes the data, and returns the output in hexadecimal format
+    return sha256.hexdigest()
+
+
+def write_log(list_of_failed_files, file):
+    log_file = os.path.join(os.getcwd(), file)
+
+    if len(list_of_failed_files) == 0:
+        with open(log_file, 'w+') as f:
+            f.write("No files failed hash comparison. \n")
+    else:
+        with open(log_file, 'w+') as f:
+            f.write("The following files failed hash comparison: \n")
+            for failed_compare_file in list_of_failed_files:
+                f.write(f"{failed_compare_file} \n")
+
+
+def compare_gpkg(file1, file2, list_of_failed_files=[], verbose=False):
+    f1_gdf = geopandas.read_file(file1)
+    f2_gdf = geopandas.read_file(file2)
+
+    try:
+        assert_geodataframe_equal(f1_gdf, f2_gdf)
+        # We only care about failures when FIM output directories are compared,
+        # however if only two .gpkg files are compared, print success.
+        if verbose:
+            print("\n Both files are the same. \n")
+
+    except AssertionError as e:
+        print(f"\n {str(e)} \n")
+        print("  The following files failed assert_geodataframe_equal: ")
+        print(f"    -{file1.rsplit('/',1)[-1]} ")
+        list_of_failed_files.append(f1_gdf)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='This script will generate a cryptographic hash for either two files, or all files '
+        'within two directories.  If two directories are provided, it will iterate over both folders named '
+        '"branches", and generate a hash for each file, and compare them. This tool is very valuable when '
+        'we want to compare FIM two output directories which should not have changed. '
+        '.gpkg files will be read into GeoDataFrames and compared (if -gpkg is specified).',
+        usage='''
+            hash_compare.py /efs-drives/fim-dev-efs/fim-data/outputs/dev-f-and-s-no-post/05030104/branches/0/
+                demDerived_reaches_split_filtered_addedAttributes_crosswalked_0.gpkg
+                /efs-drives/fim-dev-efs/fim-data/outputs/dev-no-post/05030104/branches/0/
+                demDerived_reaches_split_filtered_addedAttributes_crosswalked_0.gpkg
+            hash_compare.py /efs-drives/fim-dev-efs/fim-data/outputs/dev-f-and-s-no-post
+                /efs-drives/fim-dev-efs/fim-data/outputs/dev-format-and-style-no-post
+                -gpkg
+                -l log.txt
+        ''',
+    )
+    parser.add_argument(
+        'file1', help='File or directory to compare generated hash values. (.gpkg files not supported)'
+    )
+    parser.add_argument(
+        'file2', help='File or directory to compare generated hash values. (.gpkg files not supported)'
+    )
+    parser.add_argument('-i', '--image', help='', action='store_true')
+    parser.add_argument(
+        '-l', '--log_file', help='Optional argument to write stdout to a log file', default=None
+    )
+    parser.add_argument(
+        '-gpkg',
+        '--compare_gpkg',
+        help='Optional argument to compare .gpkg files. This will significantly increase runtime.',
+        action='store_true',
+        default=False,
+    )
+
+    args = vars(parser.parse_args())
+    file1 = args['file1']
+    file2 = args['file2']
+    image_only = args['image']
+    log_file = args['log_file']
+    gpkg = args['compare_gpkg']
+
+    main(file1, file2, image_only, log_file, gpkg)
diff --git a/tools/inundate_events.py b/tools/inundate_events.py
index c3a42efc3..90d415a85 100755
--- a/tools/inundate_events.py
+++ b/tools/inundate_events.py
@@ -1,98 +1,106 @@
 #!/usr/bin/env python3
 
-from shapely.geometry import box
-from concurrent.futures import ThreadPoolExecutor,as_completed
-from tqdm import tqdm
-import pandas as pd
-import geopandas as gpd
 import argparse
-from inundation import inundate
 import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import geopandas as gpd
+import pandas as pd
+from inundation import inundate
+from shapely.geometry import box
+from tqdm import tqdm
 
 
-def inundate_events(hydrofabric_dir,forecast_file,inundation_file,inundation_polygon=None,jobs=1):
+def inundate_events(hydrofabric_dir, forecast_file, inundation_file, inundation_polygon=None, jobs=1):
+    forecast_df = pd.read_csv(
+        forecast_file, infer_datetime_format=True, dtype={'huc': str}, parse_dates=['date_time']
+    )
 
-    forecast_df = pd.read_csv(forecast_file,infer_datetime_format=True,dtype={'huc':str},parse_dates=['date_time'])
+    # list_of_hucs_to_run = { '09020318','09020317' }
+    # forecast_df = forecast_df.loc[ forecast_df.huc.isin(list_of_hucs_to_run),:]
 
-    #list_of_hucs_to_run = { '09020318','09020317' }
-    #forecast_df = forecast_df.loc[ forecast_df.huc.isin(list_of_hucs_to_run),:]
+    inputs = build_fim3_inputs(hydrofabric_dir, forecast_df, inundation_file, inundation_polygon)
 
-    inputs = build_fim3_inputs(hydrofabric_dir,forecast_df,inundation_file,inundation_polygon)
-    
     executor = ThreadPoolExecutor(max_workers=jobs)
 
-    results = {executor.submit(inundate,**kwargs) : (kwargs['hydro_table'],kwargs['forecast']) for kwargs in inputs}
-                         #rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=None,hucs_layerName=None,
-                 #subset_hucs=None,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
-                 #depths=None,out_raster_profile=None,out_vector_profile=None,quiet=False
-
-    for future in tqdm(as_completed(results),total=len(forecast_df)):
+    results = {
+        executor.submit(inundate, **kwargs): (kwargs['hydro_table'], kwargs['forecast']) for kwargs in inputs
+    }
+    # rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=None,hucs_layerName=None,
+    # subset_hucs=None,num_workers=1,aggregate=False,inundation_raster=None,inundation_polygon=None,
+    # depths=None,out_raster_profile=None,out_vector_profile=None,quiet=False
 
+    for future in tqdm(as_completed(results), total=len(forecast_df)):
         try:
             future.result()
         except Exception as exc:
-            print(exc,results[future])
+            print(exc, results[future])
 
     executor.shutdown(wait=True)
 
 
-
-def build_fim3_inputs(hydrofabric_dir,forecast_df,inundation_file=None,inundation_polygons=None):
-
-
-    for idx,row in forecast_df.iterrows():
-
+def build_fim3_inputs(hydrofabric_dir, forecast_df, inundation_file=None, inundation_polygons=None):
+    for idx, row in forecast_df.iterrows():
         huc = row['huc']
-        rem = os.path.join(hydrofabric_dir,huc,'rem_zeroed_masked.tif')
-        catchments_raster = os.path.join(hydrofabric_dir,huc,'gw_catchments_reaches_filtered_addedAttributes.tif')
-        catchment_poly = os.path.join(hydrofabric_dir,huc,'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
-        hydrotable = os.path.join(hydrofabric_dir,huc,'hydroTable.csv')
+        rem = os.path.join(hydrofabric_dir, huc, 'rem_zeroed_masked.tif')
+        catchments_raster = os.path.join(
+            hydrofabric_dir, huc, 'gw_catchments_reaches_filtered_addedAttributes.tif'
+        )
+        catchment_poly = os.path.join(
+            hydrofabric_dir, huc, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg'
+        )
+        hydrotable = os.path.join(hydrofabric_dir, huc, 'hydroTable.csv')
 
         # inundation raster, inundation poly add HUC,date,time to filename
-        inundation_file_for_row = append_meta_data_to_file_names(inundation_file,row)
-        inundation_poly_for_row = append_meta_data_to_file_names(inundation_polygons,row)
+        inundation_file_for_row = append_meta_data_to_file_names(inundation_file, row)
+        inundation_poly_for_row = append_meta_data_to_file_names(inundation_polygons, row)
 
-        kwargs = { 'rem' : rem, 
-                   'catchments' : catchments_raster, 
-                   'catchment_poly' : catchment_poly , 
-                   'hydro_table': hydrotable,
-                   'mask_type' : 'huc',
-                   'forecast' : row['forecast_file'],
-                   'inundation_raster' : inundation_file_for_row,
-                   'inundation_polygon' : inundation_poly_for_row,
-                   'quiet' : True
-                }
-        
-        yield(kwargs)
+        kwargs = {
+            'rem': rem,
+            'catchments': catchments_raster,
+            'catchment_poly': catchment_poly,
+            'hydro_table': hydrotable,
+            'mask_type': 'huc',
+            'forecast': row['forecast_file'],
+            'inundation_raster': inundation_file_for_row,
+            'inundation_polygon': inundation_poly_for_row,
+            'quiet': True,
+        }
 
+        yield (kwargs)
 
-def append_meta_data_to_file_names(file_name,row):
 
+def append_meta_data_to_file_names(file_name, row):
     if file_name is None:
-        return(file_name)
+        return file_name
 
     base_file_path, extension = os.path.splitext(file_name)
-    
+
     hucCode = row['huc']
     site_name = row['Name']
-    date_time = str(row['date_time'].year) + str(row['date_time'].month).zfill(2) + str(row['date_time'].day).zfill(2) + '_' +str(row['date_time'].hour).zfill(2) + str(row['date_time'].minute).zfill(2) + 'Z'
-
+    date_time = (
+        str(row['date_time'].year)
+        + str(row['date_time'].month).zfill(2)
+        + str(row['date_time'].day).zfill(2)
+        + '_'
+        + str(row['date_time'].hour).zfill(2)
+        + str(row['date_time'].minute).zfill(2)
+        + 'Z'
+    )
 
     appended_file_path = f"{base_file_path}_{site_name}_{hucCode}_{date_time}{extension}"
 
-    return(appended_file_path)
-
+    return appended_file_path
 
 
 if __name__ == '__main__':
-
     # parse arguments
     parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
-    parser.add_argument('-y','--hydrofabric-dir', help='Bounding box file', required=True)
-    parser.add_argument('-f','--forecast-file', help='WBD file', required=True)
-    parser.add_argument('-i','--inundation-file', help='WBD file', required=False,default=None)
-    parser.add_argument('-j','--jobs', help='WBD file', required=False,default=None,type=int)
+    parser.add_argument('-y', '--hydrofabric-dir', help='Bounding box file', required=True)
+    parser.add_argument('-f', '--forecast-file', help='WBD file', required=True)
+    parser.add_argument('-i', '--inundation-file', help='WBD file', required=False, default=None)
+    parser.add_argument('-j', '--jobs', help='WBD file', required=False, default=None, type=int)
 
-    args=vars(parser.parse_args())
+    args = vars(parser.parse_args())
 
     inundate_events(**args)
diff --git a/tools/inundate_gms.py b/tools/inundate_gms.py
index 6cf939667..0e72238b2 100755
--- a/tools/inundate_gms.py
+++ b/tools/inundate_gms.py
@@ -1,16 +1,13 @@
 #!/usr/bin/env python3
 
-import os
 import argparse
-
-# import logging
+import os
+from concurrent.futures import ProcessPoolExecutor, as_completed
 
 import pandas as pd
-
+from inundation import NoForecastFound, hydroTableHasOnlyLakes, inundate
 from tqdm import tqdm
-from inundation import inundate
-from concurrent.futures import ProcessPoolExecutor, as_completed
-from inundation import hydroTableHasOnlyLakes, NoForecastFound
+
 from utils.shared_functions import FIM_Helpers as fh
 
 
@@ -46,14 +43,12 @@ def Inundate_gms(
         if verbose:
             print("HUC8,BranchID,Exception", file=open(log_file, "w"))
     # if log_file:
-    # logging.basicConfig(filename=log_file, level=logging.INFO)
-    # logging.info('HUC8,BranchID,Exception')
+    #     logging.basicConfig(filename=log_file, level=logging.INFO)
+    #     logging.info('HUC8,BranchID,Exception')
 
     # load fim inputs
     hucs_branches = pd.read_csv(
-        os.path.join(hydrofabric_dir, "fim_inputs.csv"),
-        header=None,
-        dtype={0: str, 1: str},
+        os.path.join(hydrofabric_dir, "fim_inputs.csv"), header=None, dtype={0: str, 1: str}
     )
 
     if hucs is not None:
@@ -101,28 +96,19 @@ def Inundate_gms(
 
         except NoForecastFound as exc:
             if log_file is not None:
-                print(
-                    f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}",
-                    file=open(log_file, "a"),
-                )
+                print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}", file=open(log_file, "a"))
             elif verbose:
                 print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}")
 
         except hydroTableHasOnlyLakes as exc:
             if log_file is not None:
-                print(
-                    f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}",
-                    file=open(log_file, "a"),
-                )
+                print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}", file=open(log_file, "a"))
             elif verbose:
                 print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}")
 
         except Exception as exc:
             if log_file is not None:
-                print(
-                    f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}",
-                    file=open(log_file, "a"),
-                )
+                print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}", file=open(log_file, "a"))
             else:
                 print(f"{hucCode},{branch_id},{exc.__class__.__name__}, {exc}")
         else:
@@ -274,19 +260,10 @@ def __inundate_gms_generator(
     # parse arguments
     parser = argparse.ArgumentParser(description="Inundate FIM")
     parser.add_argument(
-        "-y",
-        "--hydrofabric_dir",
-        help="Directory path to FIM hydrofabric by processing unit",
-        required=True,
+        "-y", "--hydrofabric_dir", help="Directory path to FIM hydrofabric by processing unit", required=True
     )
     parser.add_argument(
-        "-u",
-        "--hucs",
-        help="List of HUCS to run",
-        required=False,
-        default=None,
-        type=str,
-        nargs="+",
+        "-u", "--hucs", help="List of HUCS to run", required=False, default=None, type=str, nargs="+"
     )
     parser.add_argument("-f", "--forecast", help="Forecast discharges in CMS as CSV file", required=True)
     parser.add_argument(
@@ -311,11 +288,7 @@ def __inundate_gms_generator(
         default=None,
     )
     parser.add_argument(
-        "-l",
-        "--log-file",
-        help="Log-file to store level-path exceptions",
-        required=False,
-        default=None,
+        "-l", "--log-file", help="Log-file to store level-path exceptions", required=False, default=None
     )
     parser.add_argument(
         "-o",
@@ -326,12 +299,7 @@ def __inundate_gms_generator(
     )
     parser.add_argument("-w", "--num-workers", help="Number of Workers", required=False, default=1)
     parser.add_argument(
-        "-v",
-        "--verbose",
-        help="Verbose printing",
-        required=False,
-        default=None,
-        action="store_true",
+        "-v", "--verbose", help="Verbose printing", required=False, default=None, action="store_true"
     )
 
     Inundate_gms(**vars(parser.parse_args()))
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
index 50cf12503..48da26996 100644
--- a/tools/inundate_mosaic_wrapper.py
+++ b/tools/inundate_mosaic_wrapper.py
@@ -1,13 +1,14 @@
-import os, sys
 import argparse
 import errno
+import os
+import sys
 from timeit import default_timer as timer
 
-sys.path.append("/foss_fim/tools")
-from mosaic_inundation import Mosaic_inundation
 from inundate_gms import Inundate_gms
-from utils.shared_variables import elev_raster_ndv
+from mosaic_inundation import Mosaic_inundation
+
 from utils.shared_functions import FIM_Helpers as fh
+from utils.shared_variables import elev_raster_ndv
 
 
 def produce_mosaicked_inundation(
@@ -26,21 +27,25 @@ def produce_mosaicked_inundation(
     is_mosaic_for_branches=False,
 ):
     """
-    This function calls Inundate_gms and Mosaic_inundation to produce inundation maps. Possible outputs include inundation rasters
-    encoded by HydroID (negative HydroID for dry and positive HydroID for wet), polygons depicting extent, and depth rasters. The
-    function requires a flow file organized by NWM feature_id and discharge in cms. "feature_id" and "discharge" columns MUST be
-    present in the flow file.
+    This function calls Inundate_gms and Mosaic_inundation to produce inundation maps.
+    Possible outputs include inundation rasters encoded by HydroID (negative HydroID for dry and positive
+    HydroID for wet), polygons depicting extent, and depth rasters. The function requires a flow file
+    organized by NWM feature_id and discharge in cms. "feature_id" and "discharge" columns MUST be present
+    in the flow file.
 
     Args:
-        hydrofabric_dir (str): Directory path to hydrofabric directory where FIM outputs were written by fim_pipeline.
-        huc (str): The HUC for which to produce mosaicked inundation files.
-        flow_file (str): Directory path to flow file to be used for inundation. feature_ids in flow_file should be present in supplied HUC.
-        inundation_raster (str): Full path to output inundation raster (encoded by positive and negative HydroIDs).
+        hydrofabric_dir (str):    Path to hydrofabric directory where FIM outputs were written by
+                                    fim_pipeline.
+        huc (str):                The HUC for which to produce mosaicked inundation files.
+        flow_file (str):          Path to flow file to be used for inundation.
+                                    feature_ids in flow_file should be present in supplied HUC.
+        inundation_raster (str):  Full path to output inundation raster
+                                    (encoded by positive and negative HydroIDs).
         inuntation_polygon (str): Full path to output inundation polygon. Optional.
-        depths_raster (str): Full path to output depths_raster. Pixel values will be in meters. Optional.
-        num_workers (int): Number of parallel jobs to run.
+        depths_raster (str):      Full path to output depths_raster. Pixel values will be in meters. Optional.
+        num_workers (int):        Number of parallel jobs to run.
         keep_intermediate (bool): Option to keep intermediate files.
-        verbose (bool): Print verbose messages to screen. Not tested.
+        verbose (bool):           Print verbose messages to screen. Not tested.
     """
 
     # Check that inundation_raster or depths_raster is supplied
@@ -49,7 +54,7 @@ def produce_mosaicked_inundation(
 
     # Check that output directory exists. Notify user that output directory will be created if not.
     for output_file in [inundation_raster, inundation_polygon, depths_raster]:
-        if output_file == None:
+        if output_file is None:
             continue
         parent_dir = os.path.split(output_file)[0]
         if not os.path.exists(parent_dir):
@@ -69,11 +74,7 @@ def produce_mosaicked_inundation(
     for huc in hucs:
         if not os.path.exists(os.path.join(hydrofabric_dir, huc)):
             raise FileNotFoundError(
-                (
-                    errno.ENOENT,
-                    os.strerror(errno.ENOENT),
-                    os.path.join(hydrofabric_dir, huc),
-                )
+                (errno.ENOENT, os.strerror(errno.ENOENT), os.path.join(hydrofabric_dir, huc))
             )
 
     # Check that flow file exists
@@ -153,13 +154,7 @@ def produce_mosaicked_inundation(
         type=str,
     )
     parser.add_argument(
-        "-u",
-        "--hucs",
-        help="List of HUCS to run",
-        required=True,
-        default="",
-        type=str,
-        nargs="+",
+        "-u", "--hucs", help="List of HUCS to run", required=True, default="", type=str, nargs="+"
     )
     parser.add_argument(
         "-f",
@@ -169,12 +164,7 @@ def produce_mosaicked_inundation(
         type=str,
     )
     parser.add_argument(
-        "-i",
-        "--inundation-raster",
-        help="Inundation raster output.",
-        required=False,
-        default=None,
-        type=str,
+        "-i", "--inundation-raster", help="Inundation raster output.", required=False, default=None, type=str
     )
     parser.add_argument(
         "-p",
@@ -200,14 +190,7 @@ def produce_mosaicked_inundation(
         default=None,
         type=str,
     )
-    parser.add_argument(
-        "-k",
-        "--mask",
-        help="Name of mask file.",
-        required=False,
-        default=None,
-        type=str,
-    )
+    parser.add_argument("-k", "--mask", help="Name of mask file.", required=False, default=None, type=str)
     parser.add_argument(
         "-a",
         "--unit_attribute_name",
@@ -216,14 +199,7 @@ def produce_mosaicked_inundation(
         default="huc8",
         type=str,
     )
-    parser.add_argument(
-        "-w",
-        "--num-workers",
-        help="Number of workers.",
-        required=False,
-        default=1,
-        type=int,
-    )
+    parser.add_argument("-w", "--num-workers", help="Number of workers.", required=False, default=1, type=int)
     parser.add_argument(
         "-r",
         "--remove-intermediate",
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index 62856f730..fe8680418 100644
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -1,23 +1,22 @@
+#!/usr/bin/env python3
+
 import argparse
 import logging
 import multiprocessing
 import os
-import rasterio
 import re
 import shutil
 import sys
-
+from datetime import datetime
 from multiprocessing import Pool
-from rasterio.merge import merge
-from osgeo import gdal, ogr
 
-from datetime import datetime
+import rasterio
 from inundate_mosaic_wrapper import produce_mosaicked_inundation
+from osgeo import gdal, ogr
+from rasterio.merge import merge
 
-# setting path
-sys.path.append("/foss_fim/src")
-from utils.shared_variables import elev_raster_ndv, PREP_PROJECTION
 from utils.shared_functions import FIM_Helpers as fh
+from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv
 
 
 # INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
@@ -27,10 +26,7 @@
 # DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
 
 
-def inundate_nation(fim_run_dir, output_dir, magnitude_key, 
-                    flow_file, huc_list, inc_mosaic, job_number):
-
-
+def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, inc_mosaic, job_number):
     assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}"
 
     if job_number > available_cores:
@@ -69,29 +65,26 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key,
         shutil.rmtree(magnitude_output_dir, ignore_errors=True)
         os.mkdir(magnitude_output_dir)
 
-    if huc_list == None:    
+    if huc_list is None:
         huc_list = []
         for huc in os.listdir(fim_run_dir):
-            
-            #if huc != 'logs' and huc != 'branch_errors'and huc != 'unit_errors' and os.path.isdir(os.path.join(fim_run_dir, huc)):
-            if re.match('\d{8}', huc):    
+            # if (
+            #     huc != 'logs'
+            #     and huc != 'branch_errors'
+            #     and huc != 'unit_errors'
+            #     and os.path.isdir(os.path.join(fim_run_dir, huc))
+            # ):
+            if re.match(r'\d{8}', huc):
                 huc_list.append(huc)
     else:
         for huc in huc_list:
-            assert os.path.isdir(fim_run_dir + os.sep + huc), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}'
+            assert os.path.isdir(
+                fim_run_dir + os.sep + huc
+            ), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}'
 
     print("Inundation raw mosaic outputs here: " + magnitude_output_dir)
 
-    run_inundation(
-        [
-            fim_run_dir,
-            huc_list,
-            magnitude_key,
-            magnitude_output_dir,
-            flow_file,
-            job_number,
-        ]
-    )
+    run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number])
 
     # Perform mosaic operation
     if inc_mosaic:
@@ -140,7 +133,8 @@ def run_inundation(args):
     This script is a wrapper for the inundate function and is designed for multiprocessing.
 
     Args:
-        args (list): [fim_run_dir (str), huc_list (list), magnitude (str), magnitude_output_dir (str), forecast (str), job_number (int)]
+        args (list): [fim_run_dir (str), huc_list (list), magnitude (str),
+            magnitude_output_dir (str), forecast (str), job_number (int)]
 
     """
 
@@ -215,43 +209,89 @@ def __setup_logger(output_folder_path, log_file_name_key):
 
 
 if __name__ == "__main__":
-    # Sample usage: python3 /foss_fim/tools/inundate_nation.py -r /outputs/fim_4_0_9_2 -m 100_0 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_100_0_cms.csv -s -j 10
-    # outputs become /data/inundation_review/inundate_nation/100_0_fim_4_0_9_2_mosiac.tif (.log, etc)
-
-    # Sample usage: python3 /foss_fim/tools/inundate_nation.py -r /outputs/fim_4_0_9_2 -m hw -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv -s -j 10
-    # outputs become /data/inundation_review/inundate_nation/hw_fim_4_0_9_2_mosiac.tif (.log, etc)
-
-    # if run on UCS2, you can map docker as -v /dev_fim_share... etc/:/data -v /local....outputs:/outputs -v foss_fim as normal.
+    """
+    Sample usage:
+    python3 /foss_fim/tools/inundate_nation.py
+        -r /outputs/fim_4_0_9_2 -m 100_0
+        -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_100_0_cms.csv
+        -s
+        -j 10
+    outputs become /data/inundation_review/inundate_nation/100_0_fim_4_0_9_2_mosiac.tif (.log, etc)
+
+    python3 /foss_fim/tools/inundate_nation.py
+        -r /outputs/fim_4_0_9_2
+        -m hw
+        -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv
+        -s
+        -j 10
+    outputs become /data/inundation_review/inundate_nation/hw_fim_4_0_9_2_mosiac.tif (.log, etc)
+
+    If run on UCS2, you can map docker as -v /dev_fim_share../:/data -v /local...outputs:/outputs
+    -v .../inundation-mapping/:/foss_fim as normal.
+    """
 
     available_cores = multiprocessing.cpu_count()
 
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow '\
-        'recurrence interflow data. Inundation outputs are stored in the '\
-        '/inundation_review/inundation_nwm_recurr/ directory.')
-        
-    parser.add_argument('-r', '--fim-run-dir', help='Name of directory containing outputs '\
-        'of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)', required=True)
-        
-    parser.add_argument('-o', '--output-dir', help='Optional: The path to a directory to write the '\
-        'outputs. If not used, the inundation_nation directory is used by default '\
+    parser = argparse.ArgumentParser(
+        description='Inundation mapping for FOSS FIM using streamflow '
+        'recurrence interflow data. Inundation outputs are stored in the '
+        '/inundation_review/inundation_nwm_recurr/ directory.'
+    )
+
+    parser.add_argument(
+        '-r',
+        '--fim-run-dir',
+        help='Name of directory containing outputs '
+        'of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-o',
+        '--output-dir',
+        help='Optional: The path to a directory to write the '
+        'outputs. If not used, the inundation_nation directory is used by default '
         'ie) /data/inundation_review/inundate_nation/',
-        default='/data/inundation_review/inundate_nation/', required=False)
-       
-    parser.add_argument('-m', '--magnitude_key', help = 'used in output folders names and temp files, '\
-        'added to output_file_name_key ie 100_0, 2_0, hw, etc)', required = True)
-        
-    parser.add_argument('-f', '--flow_file', help = 'the path and flow file to be used. '\
-        'ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv', required = True)
-    
-    parser.add_argument('-l', '--huc-list', help = 'OPTIONAL: HUC list to run specified HUC(s). Specifiy multiple hucs single space delimited'\
-        '--> 12090301 12090302. Default (no huc list provided) will use hucs found in -r directory', required = False, default='all',nargs='+')
-        
-    parser.add_argument('-s', '--inc_mosaic',help='Optional flag to produce mosaic of FIM extent rasters',
-                        action='store_true')
-                        
+        default='/data/inundation_review/inundate_nation/',
+        required=False,
+    )
+
+    parser.add_argument(
+        '-m',
+        '--magnitude_key',
+        help='used in output folders names and temp files, '
+        'added to output_file_name_key ie 100_0, 2_0, hw, etc)',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-f',
+        '--flow_file',
+        help='the path and flow file to be used. '
+        'ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/'
+        'nwm_high_water_threshold_cms.csv',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-l',
+        '--huc-list',
+        help='OPTIONAL: HUC list to run specified HUC(s).Specifiy multiple hucs single space delimited'
+        '--> 12090301 12090302. Default (no huc list provided) will use hucs found in -r directory',
+        required=False,
+        default='all',
+        nargs='+',
+    )
+
+    parser.add_argument(
+        '-s',
+        '--inc_mosaic',
+        help='Optional flag to produce mosaic of FIM extent rasters',
+        action='store_true',
+    )
+
     parser.add_argument('-j', '--job-number', help='The number of jobs', required=False, default=1, type=int)
-        
 
     args = vars(parser.parse_args())
 
diff --git a/tools/inundation.py b/tools/inundation.py
index 471035d60..0babdb6c9 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -1,55 +1,68 @@
 #!/usr/bin/env python3
 
-import numpy as np
-import pandas as pd
-from numba import njit, typed, types
-from concurrent.futures import ThreadPoolExecutor,as_completed
-from os.path import splitext
-import rasterio
-import fiona
-from shapely.geometry import shape
-from rasterio.mask import mask
-from rasterio.io import DatasetReader,DatasetWriter
-from collections import OrderedDict
 import argparse
+from collections import OrderedDict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from os.path import splitext
 from warnings import warn
+
+import fiona
 import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
 import xarray as xr
-
-class hydroTableHasOnlyLakes(Exception): 
-    """ Raised when a Hydro-Table only has lakes """
-    pass
+from numba import njit, typed, types
+from rasterio.io import DatasetReader, DatasetWriter
+from rasterio.mask import mask
+from shapely.geometry import shape
 
 
-class NoForecastFound(Exception):
-    """ Raised when no forecast is available for a given Hydro-Table """
-    pass
+class hydroTableHasOnlyLakes(Exception):
+    """Raised when a Hydro-Table only has lakes"""
 
-class hydroTableHasOnlyLakes(Exception): 
-    """ Raised when a Hydro-Table only has lakes """
     pass
 
 
 class NoForecastFound(Exception):
-    """ Raised when no forecast is available for a given Hydro-Table """
+    """Raised when no forecast is available for a given Hydro-Table"""
+
     pass
 
-def inundate(rem, catchments, catchment_poly, hydro_table, forecast,
-             mask_type, hucs = None, hucs_layerName = None,
-             subset_hucs = None, num_workers = 1, aggregate = False, 
-             inundation_raster = None, inundation_polygon = None,
-             depths = None, out_raster_profile = None, out_vector_profile = None,
-             src_table = None, quiet = False):
+
+def inundate(
+    rem,
+    catchments,
+    catchment_poly,
+    hydro_table,
+    forecast,
+    mask_type,
+    hucs=None,
+    hucs_layerName=None,
+    subset_hucs=None,
+    num_workers=1,
+    aggregate=False,
+    inundation_raster=None,
+    inundation_polygon=None,
+    depths=None,
+    out_raster_profile=None,
+    out_vector_profile=None,
+    src_table=None,
+    quiet=False,
+):
     """
 
     Run inundation on FIM >=3.0 outputs at job-level scale or aggregated scale
 
-    Generate depths raster, inundation raster, and inundation polygon from FIM >=3.0 outputs. Can use the FIM 3.0 outputs at native HUC level or the aggregated products. Be sure to pass a HUCs file to process in batch mode if passing aggregated products.
+    Generate depths raster, inundation raster, and inundation polygon from FIM >=3.0 outputs.
+    Can use the FIM 3.0 outputs at native HUC level or the aggregated products.
+    Be sure to pass a HUCs file to process in batch mode if passing aggregated products.
 
     Parameters
     ----------
     rem : str or rasterio.DatasetReader
-        File path to or rasterio dataset reader of Relative Elevation Model raster. Must have the same CRS as catchments raster.
+        File path to or rasterio dataset reader of Relative Elevation Model raster.
+        Must have the same CRS as catchments raster.
     catchments : str or rasterio.DatasetReader
         File path to or rasterio dataset reader of Catchments raster. Must have the same CRS as REM raster
     hydro_table : str or pandas.DataFrame
@@ -57,23 +70,28 @@ def inundate(rem, catchments, catchment_poly, hydro_table, forecast,
     forecast : str or pandas.DataFrame
         File path to forecast csv or Pandas DataFrame with correct column names.
     hucs : str or fiona.Collection, optional
-        Batch mode only. File path or fiona collection of vector polygons in HUC 4,6,or 8's to inundate on. Must have an attribute named as either "HUC4","HUC6", or "HUC8" with the associated values.
+        Batch mode only. File path or fiona collection of vector polygons in HUC 4,6,or 8's to inundate on.
+        Must have an attribute named as either "HUC4","HUC6", or "HUC8" with the associated values.
     hucs_layerName : str, optional
         Batch mode only. Layer name in hucs to use if multi-layer file is passed.
     subset_hucs : str or list of str, optional
-        Batch mode only. File path to line delimited file, HUC string, or list of HUC strings to further subset hucs file for inundating.
+        Batch mode only. File path to line delimited file, HUC string, or list of HUC strings to
+        further subset hucs file for inundating.
     num_workers : int, optional
         Batch mode only. Number of workers to use in batch mode. Must be 1 or greater.
     aggregate : bool, optional
-        Batch mode only. Aggregates output rasters to VRT mosaic files and merges polygons to single GPKG file. Currently not functional. Raises warning and sets to false. On to-do list.
+        Batch mode only. Aggregates output rasters to VRT mosaic files and merges polygons to single GPKG file
+        Currently not functional. Raises warning and sets to false. On to-do list.
     inundation_raster : str, optional
         Path to optional inundation raster output. Appends HUC number if ran in batch mode.
     inundation_polygon : str, optional
-        Path to optional inundation vector output. Only accepts GPKG right now. Appends HUC number if ran in batch mode.
+        Path to optional inundation vector output. Only accepts GPKG right now.
+        Appends HUC number if ran in batch mode.
     depths : str, optional
         Path to optional depths raster output. Appends HUC number if ran in batch mode.
     out_raster_profile : str or dictionary, optional
-        Override the default raster profile for outputs. See Rasterio profile documentation for more information.
+        Override the default raster profile for outputs.
+        See Rasterio profile documentation for more information.
     out_vector_profile : str or dictionary
         Override the default kwargs passed to fiona.Collection including crs, driver, and schema.
     quiet : bool, optional
@@ -98,7 +116,8 @@ def inundate(rem, catchments, catchment_poly, hydro_table, forecast,
 
     Notes
     -----
-    - Specifying a subset of the domain in rem or catchments to inundate on is achieved by the HUCs file or the forecast file.
+    - Specifying a subset of the domain in rem or catchments to inundate on is achieved by the HUCs file or
+        the forecast file.
 
     """
 
@@ -114,88 +133,109 @@ def inundate(rem, catchments, catchment_poly, hydro_table, forecast,
         warn("Aggregate feature currently not working. Setting to false for now.")
         aggregate = False
     if hucs is None:
-        assert (not aggregate), "Pass HUCs file if aggregation is desired"
+        assert not aggregate, "Pass HUCs file if aggregation is desired"
 
     # bool quiet
     quiet = bool(quiet)
 
     # input rem
-    if isinstance(rem,str):
+    if isinstance(rem, str):
         rem = rasterio.open(rem)
-    elif isinstance(rem,DatasetReader):
+    elif isinstance(rem, DatasetReader):
         pass
     else:
         raise TypeError("Pass rasterio dataset or filepath for rem")
 
     # input catchments grid
-    if isinstance(catchments,str):
+    if isinstance(catchments, str):
         catchments = rasterio.open(catchments)
-    elif isinstance(catchments,DatasetReader):
+    elif isinstance(catchments, DatasetReader):
         pass
     else:
         raise TypeError("Pass rasterio dataset or filepath for catchments")
 
-
     # check for matching number of bands and single band only
-    assert rem.count == catchments.count == 1, "REM and catchments rasters are required to be single band only"
+    assert (
+        rem.count == catchments.count == 1
+    ), "REM and catchments rasters are required to be single band only"
 
     # check for matching raster sizes
-    assert (rem.width == catchments.width) & (rem.height == catchments.height), "REM and catchments rasters required same shape"
+    assert (rem.width == catchments.width) & (
+        rem.height == catchments.height
+    ), "REM and catchments rasters required same shape"
 
     # check for matching projections
-    #assert rem.crs.to_proj4() == catchments.crs.to_proj4(), "REM and Catchment rasters require same CRS definitions"
+    # assert (
+    #     rem.crs.to_proj4() == catchments.crs.to_proj4()
+    # ), "REM and Catchment rasters require same CRS definitions"
 
     # check for matching bounds
-    assert ( (rem.transform*(0,0)) == (catchments.transform*(0,0)) ) & ( (rem.transform* (rem.width,rem.height)) == (catchments.transform*(catchments.width,catchments.height)) ), "REM and catchments rasters require same upper left and lower right extents"
+    assert ((rem.transform * (0, 0)) == (catchments.transform * (0, 0))) & (
+        (rem.transform * (rem.width, rem.height))
+        == (catchments.transform * (catchments.width, catchments.height))
+    ), "REM and catchments rasters require same upper left and lower right extents"
 
     # open hucs
     if hucs is None:
         pass
-    elif isinstance(hucs,str):
-        hucs = fiona.open(hucs,'r',layer=hucs_layerName)
-    elif isinstance(hucs,fiona.Collection):
+    elif isinstance(hucs, str):
+        hucs = fiona.open(hucs, 'r', layer=hucs_layerName)
+    elif isinstance(hucs, fiona.Collection):
         pass
     else:
         raise TypeError("Pass fiona collection or filepath for hucs")
 
     # check for matching projections
-    #assert to_string(hucs.crs) == rem.crs.to_proj4() == catchments.crs.to_proj4(), "REM, Catchment, and HUCS CRS definitions must match"
+    # assert (
+    #     to_string(hucs.crs) == rem.crs.to_proj4() == catchments.crs.to_proj4()
+    # ), "REM, Catchment, and HUCS CRS definitions must match"
 
     # catchment stages dictionary
     if hydro_table is not None:
-        catchmentStagesDict,hucSet = __subset_hydroTable_to_forecast(hydro_table,forecast,subset_hucs)
+        catchmentStagesDict, hucSet = __subset_hydroTable_to_forecast(hydro_table, forecast, subset_hucs)
     else:
         raise TypeError("Pass hydro table csv")
 
-        
     if catchmentStagesDict is not None:
         if src_table is not None:
-            create_src_subset_csv(hydro_table,catchmentStagesDict,src_table)
+            create_src_subset_csv(hydro_table, catchmentStagesDict, src_table)
 
         # make windows generator
-        window_gen = __make_windows_generator(rem, catchments, catchment_poly,
-                                              mask_type, catchmentStagesDict, inundation_raster,
-                                              inundation_polygon, depths, out_raster_profile,
-                                              out_vector_profile, quiet, 
-                                              hucs = hucs, hucSet = hucSet)
+        window_gen = __make_windows_generator(
+            rem,
+            catchments,
+            catchment_poly,
+            mask_type,
+            catchmentStagesDict,
+            inundation_raster,
+            inundation_polygon,
+            depths,
+            out_raster_profile,
+            out_vector_profile,
+            quiet,
+            hucs=hucs,
+            hucSet=hucSet,
+        )
 
         # start up thread pool
         executor = ThreadPoolExecutor(max_workers=num_workers)
 
         # submit jobs
-        results = {executor.submit(__inundate_in_huc,*wg) : wg[6] for wg in window_gen}
+        results = {executor.submit(__inundate_in_huc, *wg): wg[6] for wg in window_gen}
 
-        inundation_rasters = [] ; depth_rasters = [] ; inundation_polys = []
+        inundation_rasters = []
+        depth_rasters = []
+        inundation_polys = []
         for future in as_completed(results):
             try:
                 future.result()
             except Exception as exc:
-                __vprint("Exception {} for {}".format(exc,results[future]),not quiet)
+                __vprint("Exception {} for {}".format(exc, results[future]), not quiet)
             else:
                 if results[future] is not None:
-                    __vprint("... {} complete".format(results[future]),not quiet)
+                    __vprint("... {} complete".format(results[future]), not quiet)
                 else:
-                    __vprint("... complete",not quiet)
+                    __vprint("... complete", not quiet)
 
                 inundation_rasters += [future.result()[0]]
                 depth_rasters += [future.result()[1]]
@@ -208,43 +248,55 @@ def inundate(rem, catchments, catchment_poly, hydro_table, forecast,
     rem.close()
     catchments.close()
 
-    return(inundation_rasters,depth_rasters,inundation_polys)
-
-
-def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profile,catchments_profile,hucCode,
-                      catchmentStagesDict,depths,inundation_raster,inundation_polygon,
-                      out_raster_profile,out_vector_profile,quiet):
-
+    return (inundation_rasters, depth_rasters, inundation_polys)
+
+
+def __inundate_in_huc(
+    rem_array,
+    catchments_array,
+    crs,
+    window_transform,
+    rem_profile,
+    catchments_profile,
+    hucCode,
+    catchmentStagesDict,
+    depths,
+    inundation_raster,
+    inundation_polygon,
+    out_raster_profile,
+    out_vector_profile,
+    quiet,
+):
     # verbose print
     if hucCode is not None:
-        __vprint("Inundating {} ...".format(hucCode),not quiet)
+        __vprint("Inundating {} ...".format(hucCode), not quiet)
 
     # save desired profiles for outputs
     depths_profile = rem_profile
     inundation_profile = catchments_profile
 
     # update output profiles from inputs
-    if isinstance(out_raster_profile,dict):
+    if isinstance(out_raster_profile, dict):
         depths_profile.update(**out_raster_profile)
         inundation_profile.update(**out_raster_profile)
     elif out_raster_profile is None:
-        depths_profile.update(driver= 'GTiff', blockxsize=256, blockysize=256, tiled=True, compress='lzw')
-        inundation_profile.update(driver= 'GTiff',blockxsize=256, blockysize=256, tiled=True, compress='lzw')
+        depths_profile.update(driver='GTiff', blockxsize=256, blockysize=256, tiled=True, compress='lzw')
+        inundation_profile.update(driver='GTiff', blockxsize=256, blockysize=256, tiled=True, compress='lzw')
     else:
         raise TypeError("Pass dictionary for output raster profiles")
 
     # update profiles with width and heights from array sizes
-    depths_profile.update(height=rem_array.shape[0],width=rem_array.shape[1])
-    inundation_profile.update(height=catchments_array.shape[0],width=catchments_array.shape[1])
+    depths_profile.update(height=rem_array.shape[0], width=rem_array.shape[1])
+    inundation_profile.update(height=catchments_array.shape[0], width=catchments_array.shape[1])
 
     # update transforms of outputs with window transform
     depths_profile.update(transform=window_transform)
     inundation_profile.update(transform=window_transform)
     # open output depths
-    if isinstance(depths,str):
-        depths = __append_huc_code_to_file_name(depths,hucCode)
+    if isinstance(depths, str):
+        depths = __append_huc_code_to_file_name(depths, hucCode)
         depths = rasterio.open(depths, "w", **depths_profile)
-    elif isinstance(depths,DatasetWriter):
+    elif isinstance(depths, DatasetWriter):
         pass
     elif depths is None:
         pass
@@ -252,10 +304,10 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
         raise TypeError("Pass rasterio dataset, filepath for output depths, or None.")
 
     # open output inundation raster
-    if isinstance(inundation_raster,str):
-        inundation_raster = __append_huc_code_to_file_name(inundation_raster,hucCode)
-        inundation_raster = rasterio.open(inundation_raster,"w",**inundation_profile)
-    elif isinstance(inundation_raster,DatasetWriter):
+    if isinstance(inundation_raster, str):
+        inundation_raster = __append_huc_code_to_file_name(inundation_raster, hucCode)
+        inundation_raster = rasterio.open(inundation_raster, "w", **inundation_profile)
+    elif isinstance(inundation_raster, DatasetWriter):
         pass
     elif inundation_raster is None:
         pass
@@ -265,16 +317,18 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
     # prepare output inundation polygons schema
     if inundation_polygon is not None:
         if out_vector_profile is None:
-            out_vector_profile = {'crs' : crs , 'driver' : 'GPKG'}
+            out_vector_profile = {'crs': crs, 'driver': 'GPKG'}
 
-        out_vector_profile['schema'] = { 'geometry' : 'Polygon',
-                                         'properties' : OrderedDict([('HydroID' , 'int')]) }
+        out_vector_profile['schema'] = {
+            'geometry': 'Polygon',
+            'properties': OrderedDict([('HydroID', 'int')]),
+        }
 
         # open output inundation polygons
-        if isinstance(inundation_polygon,str):
-            inundation_polygon = __append_huc_code_to_file_name(inundation_polygon,hucCode)
-            inundation_polygon = fiona.open(inundation_polygon,'w',**out_vector_profile)
-        elif isinstance(inundation_polygon,fiona.Collection):
+        if isinstance(inundation_polygon, str):
+            inundation_polygon = __append_huc_code_to_file_name(inundation_polygon, hucCode)
+            inundation_polygon = fiona.open(inundation_polygon, 'w', **out_vector_profile)
+        elif isinstance(inundation_polygon, fiona.Collection):
             pass
         else:
             raise TypeError("Pass fiona collection or file path as inundation_polygon")
@@ -292,42 +346,51 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
 
     # reset output values
     depths_array[depths_array != depths_profile['nodata']] = 0
-    inundation_array[inundation_array != inundation_profile['nodata']] = inundation_array[inundation_array != inundation_profile['nodata']] * -1
+    inundation_array[inundation_array != inundation_profile['nodata']] = (
+        inundation_array[inundation_array != inundation_profile['nodata']] * -1
+    )
 
     # make output arrays
-    inundation_array,depths_array = __go_fast_mapping(rem_array,catchments_array,catchmentStagesDict,inundation_array,depths_array)
+    inundation_array, depths_array = __go_fast_mapping(
+        rem_array, catchments_array, catchmentStagesDict, inundation_array, depths_array
+    )
 
     # reshape output arrays
     inundation_array = inundation_array.reshape(desired_shape)
     depths_array = depths_array.reshape(desired_shape)
 
     # write out inundation and depth rasters
-    if isinstance(inundation_raster,DatasetWriter):
-        inundation_raster.write(inundation_array,indexes=1)
-    if isinstance(depths,DatasetWriter):
-        depths.write(depths_array,indexes=1)
+    if isinstance(inundation_raster, DatasetWriter):
+        inundation_raster.write(inundation_array, indexes=1)
+    if isinstance(depths, DatasetWriter):
+        depths.write(depths_array, indexes=1)
 
     # polygonize inundation
-    if isinstance(inundation_polygon,fiona.Collection):
-
+    if isinstance(inundation_polygon, fiona.Collection):
         # make generator for inundation polygons
-        inundation_polygon_generator = shapes(inundation_array,mask=inundation_array>0,connectivity=8,transform=window_transform)
-        
+        # TODO shapes() method below is "undefined".
+        inundation_polygon_generator = shapes(
+            inundation_array, mask=inundation_array > 0, connectivity=8, transform=window_transform
+        )
+
         # generate records
         records = []
-        for i,(g,h) in enumerate(inundation_polygon_generator):
+        for i, (g, h) in enumerate(inundation_polygon_generator):
             record = dict()
             record['geometry'] = g
-            record['properties'] = {'HydroID' : int(h)}
+            record['properties'] = {'HydroID': int(h)}
             records += [record]
-        
+
         # write out
         inundation_polygon.writerecords(records)
 
-    if isinstance(depths,DatasetWriter): depths.close()
-    if isinstance(inundation_raster,DatasetWriter): inundation_raster.close()
-    if isinstance(inundation_polygon,fiona.Collection): inundation_polygon.close()
-    #if isinstance(hucs,fiona.Collection): inundation_polygon.close()
+    if isinstance(depths, DatasetWriter):
+        depths.close()
+    if isinstance(inundation_raster, DatasetWriter):
+        inundation_raster.close()
+    if isinstance(inundation_polygon, fiona.Collection):
+        inundation_polygon.close()
+    # if isinstance(hucs,fiona.Collection): inundation_polygon.close()
 
     # return file names of outputs for aggregation. Handle Nones
     try:
@@ -344,224 +407,241 @@ def __inundate_in_huc(rem_array,catchments_array,crs,window_transform,rem_profil
         ip_name = inundation_polygon.path
     except AttributeError:
         ip_name = None
-    
-    #print(ir_name)
-    #yield(ir_name,d_name,ip_name)
-    
-    if isinstance(depths,DatasetWriter): depths.close()
-    if isinstance(inundation_raster,DatasetWriter): inundation_raster.close()
-    if isinstance(inundation_polygon,fiona.Collection): inundation_polygon.close()
 
-    return(ir_name,d_name,ip_name)
+    # print(ir_name)
+    # yield(ir_name,d_name,ip_name)
 
+    if isinstance(depths, DatasetWriter):
+        depths.close()
+    if isinstance(inundation_raster, DatasetWriter):
+        inundation_raster.close()
+    if isinstance(inundation_polygon, fiona.Collection):
+        inundation_polygon.close()
+
+    return (ir_name, d_name, ip_name)
 
-@njit
-def __go_fast_mapping(rem,catchments,catchmentStagesDict,inundation,depths):
 
-    for i,(r,cm) in enumerate(zip(rem,catchments)):
+@njit
+def __go_fast_mapping(rem, catchments, catchmentStagesDict, inundation, depths):
+    for i, (r, cm) in enumerate(zip(rem, catchments)):
         if cm in catchmentStagesDict:
             if r >= 0:
                 depth = catchmentStagesDict[cm] - r
-                depths[i] = max(depth,0) # set negative depths to 0
+                depths[i] = max(depth, 0)  # set negative depths to 0
             else:
                 depths[i] = 0
 
-            if depths[i] > 0: # set positive depths to positive
+            if depths[i] > 0:  # set positive depths to positive
                 inundation[i] *= -1
-            #else: # set positive depths to value of positive catchment value
-                #inundation[i] = cm
-
-    return(inundation,depths)
-
-
-def __make_windows_generator(rem, 
-                             catchments,
-                             catchment_poly,
-                             mask_type,
-                             catchmentStagesDict,
-                             inundation_raster,
-                             inundation_polygon,
-                             depths,
-                             out_raster_profile,
-                             out_vector_profile,
-                             quiet,
-                             hucs = None,
-                             hucSet = None):
-
-    
+            # else: # set positive depths to value of positive catchment value
+            # inundation[i] = cm
+
+    return (inundation, depths)
+
+
+def __make_windows_generator(
+    rem,
+    catchments,
+    catchment_poly,
+    mask_type,
+    catchmentStagesDict,
+    inundation_raster,
+    inundation_polygon,
+    depths,
+    out_raster_profile,
+    out_vector_profile,
+    quiet,
+    hucs=None,
+    hucSet=None,
+):
     if hucs is not None:
-
         # get attribute name for HUC column
         for huc in hucs:
             for hucColName in huc['properties'].keys():
                 if 'HUC' in hucColName:
-                    hucSize = int(hucColName[-1])
+                    # hucSize = int(hucColName[-1])
                     break
             break
 
         # make windows
         for huc in hucs:
-
             # returns hucCode if current huc is in hucSet (at least starts with)
-            def __return_huc_in_hucSet(hucCode,hucSet):
-
+            def __return_huc_in_hucSet(hucCode, hucSet):
                 for hs in hucSet:
                     if hs.startswith(hucCode):
-                        return(hucCode)
+                        return hucCode
 
-                return(None)
+                return None
 
-            if  __return_huc_in_hucSet(huc['properties'][hucColName],hucSet) is None:
+            if __return_huc_in_hucSet(huc['properties'][hucColName], hucSet) is None:
                 continue
 
             try:
                 if mask_type == "huc":
-                    #window = geometry_window(rem,shape(huc['geometry']))
-                    rem_array,window_transform = mask(rem,[shape(huc['geometry'])],crop=True,indexes=1)
-                    catchments_array,_ = mask(catchments,[shape(huc['geometry'])],crop=True,indexes=1)
+                    # window = geometry_window(rem,shape(huc['geometry']))
+                    rem_array, window_transform = mask(rem, [shape(huc['geometry'])], crop=True, indexes=1)
+                    catchments_array, _ = mask(catchments, [shape(huc['geometry'])], crop=True, indexes=1)
                 elif mask_type == "filter":
-
                     # input catchments polygon
-                    if isinstance(catchment_poly,str):
-                        catchment_poly=gpd.read_file(catchment_poly)
-                    elif isinstance(catchment_poly,DatasetReader):
+                    if isinstance(catchment_poly, str):
+                        catchment_poly = gpd.read_file(catchment_poly)
+                    elif isinstance(catchment_poly, DatasetReader):
                         pass
                     else:
                         raise TypeError("Pass geopandas dataset or filepath for catchment polygons")
 
                     fossid = huc['properties']['fossid']
-                    if catchment_poly.HydroID.dtype != 'str': catchment_poly.HydroID = catchment_poly.HydroID.astype(str)
-                    catchment_poly=catchment_poly[catchment_poly.HydroID.str.startswith(fossid)]
+                    if catchment_poly.HydroID.dtype != 'str':
+                        catchment_poly.HydroID = catchment_poly.HydroID.astype(str)
+                    catchment_poly = catchment_poly[catchment_poly.HydroID.str.startswith(fossid)]
 
-                    rem_array,window_transform = mask(rem,catchment_poly['geometry'],crop=True,indexes=1)
-                    catchments_array,_ = mask(catchments,catchment_poly['geometry'],crop=True,indexes=1)
+                    rem_array, window_transform = mask(rem, catchment_poly['geometry'], crop=True, indexes=1)
+                    catchments_array, _ = mask(catchments, catchment_poly['geometry'], crop=True, indexes=1)
                     del catchment_poly
                 elif mask_type is None:
                     pass
                 else:
-                    print ("invalid mask type. Options are 'huc' or 'filter'")
-            except ValueError: # shape doesn't overlap raster
-                continue # skip to next HUC
+                    print("invalid mask type. Options are 'huc' or 'filter'")
+            except ValueError:  # shape doesn't overlap raster
+                continue  # skip to next HUC
 
             hucCode = huc['properties'][hucColName]
 
-            yield (rem_array, catchments_array, rem.crs.wkt,
-                   window_transform, rem.profile, catchments.profile, hucCode,
-                   catchmentStagesDict, depths, inundation_raster,
-                   inundation_polygon, out_raster_profile, out_vector_profile, quiet)
+            yield (
+                rem_array,
+                catchments_array,
+                rem.crs.wkt,
+                window_transform,
+                rem.profile,
+                catchments.profile,
+                hucCode,
+                catchmentStagesDict,
+                depths,
+                inundation_raster,
+                inundation_polygon,
+                out_raster_profile,
+                out_vector_profile,
+                quiet,
+            )
 
     else:
         hucCode = None
-       #window = Window(col_off=0,row_off=0,width=rem.width,height=rem.height)
-
-        yield (rem.read(1),catchments.read(1),rem.crs.wkt,
-               rem.transform,rem.profile,catchments.profile,hucCode,
-               catchmentStagesDict,depths,inundation_raster,
-               inundation_polygon,out_raster_profile,out_vector_profile,quiet)
-
-
-def __append_huc_code_to_file_name(fileName,hucCode):
-
+        # window = Window(col_off=0,row_off=0,width=rem.width,height=rem.height)
+
+        yield (
+            rem.read(1),
+            catchments.read(1),
+            rem.crs.wkt,
+            rem.transform,
+            rem.profile,
+            catchments.profile,
+            hucCode,
+            catchmentStagesDict,
+            depths,
+            inundation_raster,
+            inundation_polygon,
+            out_raster_profile,
+            out_vector_profile,
+            quiet,
+        )
+
+
+def __append_huc_code_to_file_name(fileName, hucCode):
     if hucCode is None:
-        return(fileName)
+        return fileName
 
-    base_file_path,extension = splitext(fileName)
+    base_file_path, extension = splitext(fileName)
 
-    return("{}_{}{}".format(base_file_path,hucCode,extension))
+    return "{}_{}{}".format(base_file_path, hucCode, extension)
 
 
-def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
-
-    if isinstance(hydroTable,str):
-        htable_req_cols = ['HUC','feature_id','HydroID','stage','discharge_cms','LakeID']
+def __subset_hydroTable_to_forecast(hydroTable, forecast, subset_hucs=None):
+    if isinstance(hydroTable, str):
+        htable_req_cols = ['HUC', 'feature_id', 'HydroID', 'stage', 'discharge_cms', 'LakeID']
         hydroTable = pd.read_csv(
-                                 hydroTable,
-                                 dtype={'HUC':str,
-                                        'feature_id':str,
-                                        'HydroID':str,
-                                        'stage':float,
-                                        'discharge_cms':float,
-                                        'LakeID' : int,
-                                        'last_updated':object, 
-                                        'submitter':object, 
-                                        'obs_source':object},
-                                 low_memory=False,
-                                 usecols=htable_req_cols
-                                )
-        huc_error = hydroTable.HUC.unique()
-        hydroTable.set_index(['HUC','feature_id','HydroID'],inplace=True)
-
-    elif isinstance(hydroTable,pd.DataFrame):
-        pass #consider checking for correct dtypes, indices, and columns
+            hydroTable,
+            dtype={
+                'HUC': str,
+                'feature_id': str,
+                'HydroID': str,
+                'stage': float,
+                'discharge_cms': float,
+                'LakeID': int,
+                'last_updated': object,
+                'submitter': object,
+                'obs_source': object,
+            },
+            low_memory=False,
+            usecols=htable_req_cols,
+        )
+        # huc_error = hydroTable.HUC.unique()
+        hydroTable.set_index(['HUC', 'feature_id', 'HydroID'], inplace=True)
+
+    elif isinstance(hydroTable, pd.DataFrame):
+        pass  # consider checking for correct dtypes, indices, and columns
     else:
         raise TypeError("Pass path to hydro-table csv or Pandas DataFrame")
 
-    hydroTable = hydroTable[hydroTable["LakeID"] == -999]  # Subset hydroTable to include only non-lake catchments.
+    hydroTable = hydroTable[
+        hydroTable["LakeID"] == -999
+    ]  # Subset hydroTable to include only non-lake catchments.
 
     # raises error if hydroTable is empty due to all segments being lakes
     if hydroTable.empty:
         raise hydroTableHasOnlyLakes("All stream segments in HUC are within lake boundaries.")
 
-
-    if isinstance(forecast,str):
-
+    if isinstance(forecast, str):
         try:
-            forecast = pd.read_csv(
-                                   forecast,
-                                   dtype={'feature_id' : str , 'discharge' : float}
-                                  )
-            forecast.set_index('feature_id',inplace=True)
+            forecast = pd.read_csv(forecast, dtype={'feature_id': str, 'discharge': float})
+            forecast.set_index('feature_id', inplace=True)
         except UnicodeDecodeError:
             forecast = read_nwm_forecast_file(forecast)
-    
-    elif isinstance(forecast,pd.DataFrame):
-        pass # consider checking for dtypes, indices, and columns
+
+    elif isinstance(forecast, pd.DataFrame):
+        pass  # consider checking for dtypes, indices, and columns
     else:
         raise TypeError("Pass path to forecast file csv or Pandas DataFrame")
 
     # susbset hucs if passed
     if subset_hucs is not None:
-        if isinstance(subset_hucs,list):
+        if isinstance(subset_hucs, list):
             if len(subset_hucs) == 1:
                 try:
                     subset_hucs = open(subset_hucs[0]).read().split('\n')
                 except FileNotFoundError:
                     pass
-        elif isinstance(subset_hucs,str):
-                try:
-                    subset_hucs = open(subset_hucs).read().split('\n')
-                except FileNotFoundError:
-                    subset_hucs = [subset_hucs]
+        elif isinstance(subset_hucs, str):
+            try:
+                subset_hucs = open(subset_hucs).read().split('\n')
+            except FileNotFoundError:
+                subset_hucs = [subset_hucs]
 
     if not hydroTable.empty:
-
-        if isinstance(forecast,str):
-            forecast = pd.read_csv(
-                                   forecast,
-                                   dtype={'feature_id' : str , 'discharge' : float}
-                                  )
-            forecast.set_index('feature_id',inplace=True)
-        elif isinstance(forecast,pd.DataFrame):
-            pass # consider checking for dtypes, indices, and columns
+        if isinstance(forecast, str):
+            forecast = pd.read_csv(forecast, dtype={'feature_id': str, 'discharge': float})
+            forecast.set_index('feature_id', inplace=True)
+        elif isinstance(forecast, pd.DataFrame):
+            pass  # consider checking for dtypes, indices, and columns
         else:
             raise TypeError("Pass path to forecast file csv or Pandas DataFrame")
 
         # susbset hucs if passed
         if subset_hucs is not None:
-            if isinstance(subset_hucs,list):
+            if isinstance(subset_hucs, list):
                 if len(subset_hucs) == 1:
                     try:
                         subset_hucs = open(subset_hucs[0]).read().split('\n')
                     except FileNotFoundError:
                         pass
-            elif isinstance(subset_hucs,str):
-                    try:
-                        subset_hucs = open(subset_hucs).read().split('\n')
-                    except FileNotFoundError:
-                        subset_hucs = [subset_hucs]
+            elif isinstance(subset_hucs, str):
+                try:
+                    subset_hucs = open(subset_hucs).read().split('\n')
+                except FileNotFoundError:
+                    subset_hucs = [subset_hucs]
 
             # subsets HUCS
-            subset_hucs_orig = subset_hucs.copy() ; subset_hucs = []
+            subset_hucs_orig = subset_hucs.copy()
+            subset_hucs = []
             for huc in np.unique(hydroTable.index.get_level_values('HUC')):
                 for sh in subset_hucs_orig:
                     if huc.startswith(sh):
@@ -571,124 +651,186 @@ def __subset_hydroTable_to_forecast(hydroTable,forecast,subset_hucs=None):
 
     # join tables
     try:
-        hydroTable = hydroTable.join(forecast,on=['feature_id'],how='inner')
+        hydroTable = hydroTable.join(forecast, on=['feature_id'], how='inner')
     except AttributeError:
-        #print("FORECAST ERROR")
+        # print("FORECAST ERROR")
         raise NoForecastFound("No forecast value found for the passed feature_ids in the Hydro-Table")
 
     else:
-
         # initialize dictionary
-        catchmentStagesDict = typed.Dict.empty(types.int32,types.float64)
+        catchmentStagesDict = typed.Dict.empty(types.int32, types.float64)
 
         # interpolate stages
-        for hid,sub_table in hydroTable.groupby(level='HydroID'):
-
-            interpolated_stage = np.interp(sub_table.loc[:,'discharge'].unique(),sub_table.loc[:,'discharge_cms'],sub_table.loc[:,'stage'])
+        for hid, sub_table in hydroTable.groupby(level='HydroID'):
+            interpolated_stage = np.interp(
+                sub_table.loc[:, 'discharge'].unique(),
+                sub_table.loc[:, 'discharge_cms'],
+                sub_table.loc[:, 'stage'],
+            )
 
             # add this interpolated stage to catchment stages dict
-            h = round(interpolated_stage[0],4)
+            h = round(interpolated_stage[0], 4)
 
-            hid = types.int32(hid) ; h = types.float32(h)
+            hid = types.int32(hid)
+            h = types.float32(h)
             catchmentStagesDict[hid] = h
 
         # huc set
         hucSet = [str(i) for i in hydroTable.index.get_level_values('HUC').unique().to_list()]
 
-        return(catchmentStagesDict,hucSet)
+        return (catchmentStagesDict, hucSet)
 
 
-def read_nwm_forecast_file(forecast_file,rename_headers=True):
-        
-    """ Reads NWM netcdf comp files and converts to forecast data frame """
+def read_nwm_forecast_file(forecast_file, rename_headers=True):
+    """Reads NWM netcdf comp files and converts to forecast data frame"""
+
+    flows_nc = xr.open_dataset(forecast_file, decode_cf='feature_id', engine='netcdf4')
 
-    flows_nc = xr.open_dataset(forecast_file,decode_cf='feature_id',engine='netcdf4')
-    
     flows_df = flows_nc.to_dataframe()
     flows_df.reset_index(inplace=True)
-    
-    flows_df = flows_df[['streamflow','feature_id']]
-    
+
+    flows_df = flows_df[['streamflow', 'feature_id']]
+
     if rename_headers:
         flows_df = flows_df.rename(columns={"streamflow": "discharge"})
 
-    convert_dict = {'feature_id': str,'discharge': float}
+    convert_dict = {'feature_id': str, 'discharge': float}
     flows_df = flows_df.astype(convert_dict)
-    
-    flows_df.set_index('feature_id',inplace=True,drop=True)
+
+    flows_df.set_index('feature_id', inplace=True, drop=True)
 
     flows_df.dropna(inplace=True)
 
-    return(flows_df)
+    return flows_df
 
 
-def __vprint(message,verbose):
+def __vprint(message, verbose):
     if verbose:
         print(message)
 
-def create_src_subset_csv(hydro_table,catchmentStagesDict,src_table):
+
+def create_src_subset_csv(hydro_table, catchmentStagesDict, src_table):
     src_df = pd.DataFrame.from_dict(catchmentStagesDict, orient='index')
     src_df.reset_index(inplace=True)
-    src_df.columns = ['HydroID','stage_inund']
-    htable_req_cols = ['HUC','feature_id','HydroID','stage','discharge_cms','LakeID']
-    df_htable = pd.read_csv(hydro_table,dtype={'HydroID': int,'HUC': object, 'branch_id':int, 'last_updated':object, 'submitter':object, 'obs_source':object},usecols=htable_req_cols)
-    df_htable = df_htable.merge(src_df,how='left',on='HydroID')
+    src_df.columns = ['HydroID', 'stage_inund']
+    htable_req_cols = ['HUC', 'feature_id', 'HydroID', 'stage', 'discharge_cms', 'LakeID']
+    df_htable = pd.read_csv(
+        hydro_table,
+        dtype={
+            'HydroID': int,
+            'HUC': object,
+            'branch_id': int,
+            'last_updated': object,
+            'submitter': object,
+            'obs_source': object,
+        },
+        usecols=htable_req_cols,
+    )
+    df_htable = df_htable.merge(src_df, how='left', on='HydroID')
     df_htable['find_match'] = (df_htable['stage'] - df_htable['stage_inund']).abs()
     df_htable = df_htable.loc[df_htable.groupby('HydroID')['find_match'].idxmin()].reset_index(drop=True)
-    df_htable.to_csv(src_table,index=False)
+    df_htable.to_csv(src_table, index=False)
 
 
 if __name__ == '__main__':
-
     # parse arguments
-    parser = argparse.ArgumentParser(description='Rapid inundation mapping for FOSS FIM. Operates in single-HUC and batch modes.')
-    parser.add_argument('-r', '--rem', 
-                        help='REM raster at job level or mosaic vrt. Must match catchments CRS.', 
-                        required=True)
-    parser.add_argument('-c', '--catchments',
-                        help='Catchments raster at job level or mosaic VRT. Must match rem CRS.',
-                        required=True)
-    parser.add_argument('-b', '--catchment-poly', help='catchment_vector',
-                        required=True)
-    parser.add_argument('-t', '--hydro-table', help='Hydro-table in csv file format', 
-                        required=True)
-    parser.add_argument('-f', '--forecast', help='Forecast discharges in CMS as CSV file',
-                        required=True)
-    parser.add_argument('-u', '--hucs',
-                        help='Batch mode only: HUCs file to process at. Must match CRS of input rasters',required=False, default=None)
-    parser.add_argument('-l', '--hucs-layerName', help='Batch mode only. Layer name in HUCs file to use',
-                        required=False, default=None)
-    parser.add_argument('-j', '--num-workers',help='Batch mode only. Number of concurrent processes',
-                        required=False, default=1, type=int)
-    parser.add_argument('-s', '--subset-hucs', help = """Batch mode only. HUC code, 
-            series of HUC codes (no quotes required), or line delimited of HUCs to run within 
+    parser = argparse.ArgumentParser(
+        description='Rapid inundation mapping for FOSS FIM. Operates in single-HUC and batch modes.'
+    )
+    parser.add_argument(
+        '-r', '--rem', help='REM raster at job level or mosaic vrt. Must match catchments CRS.', required=True
+    )
+    parser.add_argument(
+        '-c',
+        '--catchments',
+        help='Catchments raster at job level or mosaic VRT. Must match rem CRS.',
+        required=True,
+    )
+    parser.add_argument('-b', '--catchment-poly', help='catchment_vector', required=True)
+    parser.add_argument('-t', '--hydro-table', help='Hydro-table in csv file format', required=True)
+    parser.add_argument('-f', '--forecast', help='Forecast discharges in CMS as CSV file', required=True)
+    parser.add_argument(
+        '-u',
+        '--hucs',
+        help='Batch mode only: HUCs file to process at. Must match CRS of input rasters',
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-l',
+        '--hucs-layerName',
+        help='Batch mode only. Layer name in HUCs file to use',
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-j',
+        '--num-workers',
+        help='Batch mode only. Number of concurrent processes',
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        '-s',
+        '--subset-hucs',
+        help="""Batch mode only. HUC code,
+            series of HUC codes (no quotes required), or line delimited of HUCs to run within
             the hucs file that is passed""",
-                        required=False, default=None, nargs='+')
-    parser.add_argument('-m', '--mask-type',
-                        help='Specify huc (FIM < 3) or filter (FIM >= 3) masking method',
-                        required=False, default="huc")
-    parser.add_argument('-a', '--aggregate',
-                        help="""Batch mode only. Aggregate outputs to VRT files.
+        required=False,
+        default=None,
+        nargs='+',
+    )
+    parser.add_argument(
+        '-m',
+        '--mask-type',
+        help='Specify huc (FIM < 3) or filter (FIM >= 3) masking method',
+        required=False,
+        default="huc",
+    )
+    parser.add_argument(
+        '-a',
+        '--aggregate',
+        help="""Batch mode only. Aggregate outputs to VRT files.
                         Currently, raises warning and sets to false if used.""",
-                        required=False, action='store_true')
-    parser.add_argument('-i', '--inundation-raster',
-                        help="""Inundation Raster output. Only writes if designated.
+        required=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-i',
+        '--inundation-raster',
+        help="""Inundation Raster output. Only writes if designated.
                         Appends HUC code in batch mode.""",
-                        required=False, default=None)
-    parser.add_argument('-p', '--inundation-polygon',
-                        help="""Inundation polygon output. Only writes if designated. 
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-p',
+        '--inundation-polygon',
+        help="""Inundation polygon output. Only writes if designated.
                         Appends HUC code in batch mode.""",
-                        required=False, default=None)
-    parser.add_argument('-d', '--depths',
-                        help="""Depths raster output. Only writes if designated.
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-d',
+        '--depths',
+        help="""Depths raster output. Only writes if designated.
                         Appends HUC code in batch mode.""",
-                        required=False, default=None)
-    parser.add_argument('-n', '--src-table',
-                        help="""Output table with the SRC lookup/interpolation.
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-n',
+        '--src-table',
+        help="""Output table with the SRC lookup/interpolation.
                         Only writes if designated. Appends HUC code in batch mode.""",
-                        required=False, default=None)
-    parser.add_argument('-q','--quiet', help='Quiet terminal output',
-                        required=False, default=False, action='store_true')
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-q', '--quiet', help='Quiet terminal output', required=False, default=False, action='store_true'
+    )
 
     # extract to dictionary
     args = vars(parser.parse_args())
diff --git a/tools/inundation_wrapper_custom_flow.py b/tools/inundation_wrapper_custom_flow.py
index 0ea1b8e2a..d0769bd87 100755
--- a/tools/inundation_wrapper_custom_flow.py
+++ b/tools/inundation_wrapper_custom_flow.py
@@ -1,17 +1,19 @@
 #!/usr/bin/env python3
 
+import argparse
+import os
+import shutil
+import sys
+
+from inundation import inundate
+
+
 # Created: 1/11/2021
 # Primary developer(s): ryan.spies@noaa.gov
 # Purpose: This script provides the user to input a customized flow entry to produce
 # inundation outputs using outputs from fim_run. Note that the flow csv must be
 # formatted with "feature_id" & "discharge" columns. Flow must be in cubic m/s
 
-import os
-import sys
-import argparse
-import shutil
-from inundation import inundate
-
 TEST_CASES_DIR = r'/data/inundation_review/inundation_custom_flow/'  # Will update.
 INPUTS_DIR = r'/data/inputs'
 OUTPUTS_DIR = os.environ['outputsDir']
@@ -24,8 +26,8 @@
 WHITE_BOLD = '\033[37;1m'
 CYAN_BOLD = '\033[36;1m'
 
-def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type='huc'):
 
+def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type='huc'):
     # Construct paths to development test results if not existent.
     huc_id_dir_parent = os.path.join(TEST_CASES_DIR, huc_id)
     if not os.path.exists(huc_id_dir_parent):
@@ -59,16 +61,18 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type=
         catchment_poly = ''
         print('Not using the catchment polygon input layer -- FIM version < 3??')
     else:
-        catchment_poly = os.path.join(fim_run_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+        catchment_poly = os.path.join(
+            fim_run_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg'
+        )
         if not os.path.isfile(catchment_poly):
             print('Can not find catchments polygon file: ' + str(catchments))
     hydro_table = os.path.join(fim_run_dir, 'hydroTable.csv')
     if not os.path.isfile(hydro_table):
         print('Can not find hydro_table file: ' + str(hydro_table))
 
-    #benchmark_category = huc_id.split('_')[1]
+    # benchmark_category = huc_id.split('_')[1]
     current_huc = huc_id.split('_')[0]  # Break off HUC ID and assign to variable.
-    wbd_huc = 'WBDHU' + str(len(huc_id)) # check if the input huc is 2,4,6,8 etc
+    wbd_huc = 'WBDHU' + str(len(huc_id))  # check if the input huc is 2,4,6,8 etc
 
     # Map necessary inputs for inundation().
     hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), wbd_huc
@@ -76,27 +80,41 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type=
     if not os.path.exists(branch_test_case_dir_parent):
         os.mkdir(branch_test_case_dir_parent)
 
+    # branch_test_case_dir = os.path.join(branch_test_case_dir_parent)
 
-    #branch_test_case_dir = os.path.join(branch_test_case_dir_parent)
-
-    #os.makedirs(branch_test_case_dir)  # Make output directory for branch.
+    # os.makedirs(branch_test_case_dir)  # Make output directory for branch.
 
     # Define paths to inundation_raster and forecast file.
     inundation_raster = os.path.join(branch_test_case_dir_parent, branch_name + '_inund_extent.tif')
-    forecast = os.path.join(TEST_CASES_DIR,"_input_flow_files", input_flow_csv)
+    forecast = os.path.join(TEST_CASES_DIR, "_input_flow_files", input_flow_csv)
     if not os.path.isfile(forecast):
         print('Can not find input flow file: ' + str(forecast))
 
-    # Copy forecast flow file into the outputs directory to all viewer to reference the flows used to create inundation_raster
-    shutil.copyfile(forecast,os.path.join(branch_test_case_dir_parent,input_flow_csv))
+    # Copy forecast flow file into the outputs directory to all viewer to reference the flows used
+    #   to create inundation_raster
+    shutil.copyfile(forecast, os.path.join(branch_test_case_dir_parent, input_flow_csv))
 
     # Run inundate.
     print("-----> Running inundate() to produce modeled inundation extent for the " + input_flow_csv)
     inundate(
-             rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-             subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-             depths=None,out_raster_profile=None,out_vector_profile=None,quiet=False
-            )
+        rem,
+        catchments,
+        catchment_poly,
+        hydro_table,
+        forecast,
+        mask_type,
+        hucs=hucs,
+        hucs_layerName=hucs_layerName,
+        subset_hucs=current_huc,
+        num_workers=1,
+        aggregate=False,
+        inundation_raster=inundation_raster,
+        inundation_polygon=None,
+        depths=None,
+        out_raster_profile=None,
+        out_vector_profile=None,
+        quiet=False,
+    )
 
     print("-----> Inundation mapping complete.")
     if not os.path.isfile(inundation_raster):
@@ -104,15 +122,48 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type=
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using a user supplied flow data file. Inundation outputs are stored in the /inundation_review/inundation_custom_flow/ directory.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev/12345678)',required=True)
-    parser.add_argument('-b', '--branch-name',help='The name of the working branch in which features are being tested (used to name the output inundation directory) -> type=str',required=True,default="")
-    parser.add_argument('-t', '--huc-id',help='The huc id to use (single huc). Format as: xxxxxxxx, e.g. 12345678',required=True,default="")
-    parser.add_argument('-m', '--mask-type', help='Optional: specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
-    parser.add_argument('-y', '--input-flow-csv',help='Filename of the user generated (customized) csv. Must contain nwm feature ids and flow value(s) (units: cms) --> put this file in the "_input_flow_files" directory',required=True, default="")
-
+    parser = argparse.ArgumentParser(
+        description='Inundation mapping for FOSS FIM using a user supplied flow data file. '
+        'Inundation outputs are stored in the /inundation_review/inundation_custom_flow/ directory.'
+    )
+    parser.add_argument(
+        '-r',
+        '--fim-run-dir',
+        help='Name of directory containing outputs of fim_run.sh '
+        '(e.g. data/ouputs/dev_abc/12345678_dev/12345678)',
+        required=True,
+    )
+    parser.add_argument(
+        '-b',
+        '--branch-name',
+        help='The name of the working branch in which features are being tested '
+        '(used to name the output inundation directory) -> type=str',
+        required=True,
+        default="",
+    )
+    parser.add_argument(
+        '-t',
+        '--huc-id',
+        help='The huc id to use (single huc). Format as: xxxxxxxx, e.g. 12345678',
+        required=True,
+        default="",
+    )
+    parser.add_argument(
+        '-m',
+        '--mask-type',
+        help='Optional: specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method',
+        required=False,
+        default="huc",
+    )
+    parser.add_argument(
+        '-y',
+        '--input-flow-csv',
+        help='Filename of the user generated (customized) csv. Must contain nwm feature ids and '
+        'flow value(s) (units: cms) --> put this file in the "_input_flow_files" directory',
+        required=True,
+        default="",
+    )
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
@@ -124,22 +175,43 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, input_flow_csv, mask_type=
 
     # Ensure fim_run_dir exists.
     if not os.path.exists(args['fim_run_dir']):
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
-        print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. data/outputs/123456/123456." + ENDC)
+        print(
+            TRED_BOLD
+            + "Warning: "
+            + WHITE_BOLD
+            + "The provided fim_run_dir (-r) "
+            + CYAN_BOLD
+            + args['fim_run_dir']
+            + WHITE_BOLD
+            + " could not be located in the 'outputs' directory."
+            + ENDC
+        )
+        print(
+            WHITE_BOLD
+            + "Please provide the parent directory name for fim_run.sh outputs. "
+            + "These outputs are usually written in a subdirectory, e.g. data/outputs/123456/123456."
+            + ENDC
+        )
         print()
         exit_flag = True
 
-
     if args['input_flow_csv'] == '':
-        print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided input_flow_csv (-y) " + CYAN_BOLD + args['input_flow_csv'] + WHITE_BOLD + " is not provided. Please provide a csv file with nwm featureid and flow values" + ENDC)
+        print(
+            TRED_BOLD
+            + "Error: "
+            + WHITE_BOLD
+            + "The provided input_flow_csv (-y) "
+            + CYAN_BOLD
+            + args['input_flow_csv']
+            + WHITE_BOLD
+            + " is not provided. Please provide a csv file with nwm featureid and flow values"
+            + ENDC
+        )
         exit_flag = True
 
-
     if exit_flag:
         print()
         sys.exit()
 
-
     else:
-
         run_recurr_test(**args)
diff --git a/tools/inundation_wrapper_nwm_flows.py b/tools/inundation_wrapper_nwm_flows.py
index fa10dd164..68f4ff27b 100755
--- a/tools/inundation_wrapper_nwm_flows.py
+++ b/tools/inundation_wrapper_nwm_flows.py
@@ -1,19 +1,21 @@
 #!/usr/bin/env python3
 
+import argparse
+import csv
+import os
+import shutil
+import sys
+
+from inundation import inundate
+from tools_shared_variables import INPUTS_DIR
+
+
 # Created: 1/10/2021
 # Primary developer(s): ryan.spies@noaa.gov
 # Purpose: This script provides the user to generate inundation outputs using
 # the NWM Recurrence Interval flow data for 1.5yr, 5yr, & 10yr events.
 
-import os
-import sys
-import csv
-import argparse
-import shutil
-from inundation import inundate
-
-INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # Will update.
-INPUTS_DIR = r'/data/inputs'
+INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'  # TODO - Update.
 OUTPUTS_DIR = os.environ['outputsDir']
 
 ENDC = '\033[m'
@@ -24,14 +26,14 @@
 WHITE_BOLD = '\033[37;1m'
 CYAN_BOLD = '\033[36;1m'
 
-def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc', output_dir=None):
 
+def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc', output_dir=None):
     # Construct paths to development test results if not existent.
     huc_id_dir_parent = os.path.join(INUN_REVIEW_DIR, huc_id)
     if not os.path.exists(huc_id_dir_parent):
         os.mkdir(huc_id_dir_parent)
-        
-    if output_dir == None:
+
+    if output_dir is None:
         branch_test_case_dir_parent = os.path.join(INUN_REVIEW_DIR, huc_id, branch_name)
     else:
         branch_test_case_dir_parent = os.path.join(output_dir, huc_id, branch_name)
@@ -55,13 +57,15 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'
     if mask_type == 'huc':
         catchment_poly = ''
     else:
-        catchment_poly = os.path.join(fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg')
+        catchment_poly = os.path.join(
+            fim_run_parent, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg'
+        )
     hydro_table = os.path.join(fim_run_parent, 'hydroTable.csv')
 
     # Map necessary inputs for inundation().
     hucs, hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
 
-    #benchmark_category = huc_id.split('_')[1]
+    # benchmark_category = huc_id.split('_')[1]
     current_huc = huc_id.split('_')[0]  # Break off HUC ID and assign to variable.
 
     if not os.path.exists(branch_test_case_dir_parent):
@@ -84,26 +88,77 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'
         forecast = os.path.join(INUN_REVIEW_DIR, 'nwm_recurr_flow_data', 'recurr_' + magnitude + '_cms.csv')
 
         # Run inundate.
-        print("-----> Running inundate() to produce modeled inundation extent for the " + magnitude + " magnitude...")
+        print(
+            "-----> Running inundate() to produce modeled inundation extent for the "
+            + magnitude
+            + " magnitude..."
+        )
         inundate(
-                 rem,catchments,catchment_poly,hydro_table,forecast,mask_type,hucs=hucs,hucs_layerName=hucs_layerName,
-                 subset_hucs=current_huc,num_workers=1,aggregate=False,inundation_raster=inundation_raster,inundation_polygon=None,
-                 depths=None,out_raster_profile=None,out_vector_profile=None,quiet=True
-                )
+            rem,
+            catchments,
+            catchment_poly,
+            hydro_table,
+            forecast,
+            mask_type,
+            hucs=hucs,
+            hucs_layerName=hucs_layerName,
+            subset_hucs=current_huc,
+            num_workers=1,
+            aggregate=False,
+            inundation_raster=inundation_raster,
+            inundation_polygon=None,
+            depths=None,
+            out_raster_profile=None,
+            out_vector_profile=None,
+            quiet=True,
+        )
 
         print("-----> Inundation mapping complete.")
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
-    parser = argparse.ArgumentParser(description='Inundation mapping for FOSS FIM using streamflow recurrence interflow data. Inundation outputs are stored in the /inundation_review/inundation_nwm_recurr/ directory.')
-    parser.add_argument('-r','--fim-run-dir',help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)',required=True)
-    parser.add_argument('-b', '--branch-name',help='The name of the working branch in which features are being tested (used to name the output inundation directory) -> type=str',required=True,default="")
-    parser.add_argument('-t', '--huc-id',help='Provide either a single hucid (Format as: xxxxxxxx, e.g. 12345678) or a filepath to a list of hucids',required=True,default="")
-    parser.add_argument('-m', '--mask-type', help='Optional: specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method', required=False,default="huc")
-    parser.add_argument('-y', '--magnitude',help='The magnitude (reccur interval) to run. Leave blank to use default intervals (options: 1_5, 5_0, 10_0).',required=False, default="")
-
+    parser = argparse.ArgumentParser(
+        description='Inundation mapping for FOSS FIM using streamflow recurrence interflow data. '
+        'Inundation outputs are stored in the /inundation_review/inundation_nwm_recurr/ directory.'
+    )
+    parser.add_argument(
+        '-r',
+        '--fim-run-dir',
+        help='Name of directory containing outputs of fim_run.sh (e.g. data/ouputs/dev_abc/12345678_dev_test)',
+        required=True,
+    )
+    parser.add_argument(
+        '-b',
+        '--branch-name',
+        help='The name of the working branch in which features are being tested '
+        '(used to name the output inundation directory) -> type=str',
+        required=True,
+        default="",
+    )
+    parser.add_argument(
+        '-t',
+        '--huc-id',
+        help='Provide either a single hucid (Format as: xxxxxxxx, e.g. 12345678) or '
+        'a filepath to a list of hucids',
+        required=True,
+        default="",
+    )
+    parser.add_argument(
+        '-m',
+        '--mask-type',
+        help='Optional: specify \'huc\' (FIM < 3) or \'filter\' (FIM >= 3) masking method',
+        required=False,
+        default="huc",
+    )
+    parser.add_argument(
+        '-y',
+        '--magnitude',
+        help='The magnitude (reccur interval) to run. Leave blank to use default intervals '
+        '(options: 1_5, 5_0, 10_0).',
+        required=False,
+        default="",
+    )
 
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
@@ -116,12 +171,21 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'
     # check if user provided a single huc_id or a file path to a list of huc ids
     if args['huc_id'].isdigit():
         huc_list = [args['huc_id']]
-    elif os.path.exists(args['huc_id']): # check if provided str is a valid path
-        with open(args['huc_id'],newline='') as list_file:
+    elif os.path.exists(args['huc_id']):  # check if provided str is a valid path
+        with open(args['huc_id'], newline='') as list_file:
             read_list = csv.reader(list_file)
-            huc_list=[i for row in read_list for i in row]
+            huc_list = [i for row in read_list for i in row]
     else:
-        print(TRED_BOLD + "Warning: " + WHITE_BOLD + "Invalid huc-id entry: " + CYAN_BOLD + args['fim_run_dir'] + WHITE_BOLD + " --> check that huc_id number or list file is valid")
+        print(
+            TRED_BOLD
+            + "Warning: "
+            + WHITE_BOLD
+            + "Invalid huc-id entry: "
+            + CYAN_BOLD
+            + args['fim_run_dir']
+            + WHITE_BOLD
+            + " --> check that huc_id number or list file is valid"
+        )
         exit_flag = True
     print(huc_list)
     if exit_flag:
@@ -133,19 +197,50 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'
         # Ensure fim_run_dir exists.
         fim_run_dir = args['fim_run_dir'] + os.sep + huc_id
         if not os.path.exists(fim_run_dir):
-            print(TRED_BOLD + "Warning: " + WHITE_BOLD + "The provided fim_run_dir (-r) " + CYAN_BOLD + fim_run_dir + WHITE_BOLD + " could not be located in the 'outputs' directory." + ENDC)
-            print(WHITE_BOLD + "Please provide the parent directory name for fim_run.sh outputs. These outputs are usually written in a subdirectory, e.g. data/outputs/123456/123456." + ENDC)
+            print(
+                TRED_BOLD
+                + "Warning: "
+                + WHITE_BOLD
+                + "The provided fim_run_dir (-r) "
+                + CYAN_BOLD
+                + fim_run_dir
+                + WHITE_BOLD
+                + " could not be located in the 'outputs' directory."
+                + ENDC
+            )
+            print(
+                WHITE_BOLD
+                + "Please provide the parent directory name for fim_run.sh outputs. "
+                + "These outputs are usually written in a subdirectory, e.g. data/outputs/123456/123456."
+                + ENDC
+            )
             print()
             exit_flag = True
 
         # Ensure valid flow recurr intervals
-        default_flow_intervals = ['1_5','5_0','10_0']
+        default_flow_intervals = ['1_5', '5_0', '10_0']
         if args['magnitude'] == '':
             args['magnitude'] = default_flow_intervals
-            print(TRED_BOLD + "Using default flow reccurence intervals: " + WHITE_BOLD + str(default_flow_intervals)[1:-1])
+            print(
+                TRED_BOLD
+                + "Using default flow reccurence intervals: "
+                + WHITE_BOLD
+                + str(default_flow_intervals)[1:-1]
+            )
         else:
-            if set(default_flow_intervals).issuperset(set(args['magnitude'])) == False:
-                print(TRED_BOLD + "Error: " + WHITE_BOLD + "The provided magnitude (-y) " + CYAN_BOLD + args['magnitude'] + WHITE_BOLD + " is invalid. NWM Recurrence Interval options include: " + str(default_flow_intervals)[1:-1] + ENDC)
+            if set(default_flow_intervals).issuperset(set(args['magnitude'])) is False:
+                print(
+                    TRED_BOLD
+                    + "Error: "
+                    + WHITE_BOLD
+                    + "The provided magnitude (-y) "
+                    + CYAN_BOLD
+                    + args['magnitude']
+                    + WHITE_BOLD
+                    + " is invalid. NWM Recurrence Interval options include: "
+                    + str(default_flow_intervals)[1:-1]
+                    + ENDC
+                )
                 exit_flag = True
 
         if exit_flag:
@@ -153,4 +248,4 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'
             sys.exit()
 
         else:
-            run_recurr_test(fim_run_dir,args['branch_name'],huc_id,args['magnitude'],args['mask_type'])
+            run_recurr_test(fim_run_dir, args['branch_name'], huc_id, args['magnitude'], args['mask_type'])
diff --git a/tools/make_boxes_from_bounds.py b/tools/make_boxes_from_bounds.py
index 69b9b5a8b..4959a21e4 100755
--- a/tools/make_boxes_from_bounds.py
+++ b/tools/make_boxes_from_bounds.py
@@ -1,76 +1,81 @@
 #!/usr/bin/env python3
 
-
-from shapely.geometry import box
-import pandas as pd
-import geopandas as gpd
 import argparse
 from datetime import datetime
-from foss_fim.src.utils.shared_functions import getDriver
 
+import geopandas as gpd
+import pandas as pd
+from shapely.geometry import box
 
-def find_hucs_of_bounding_boxes(bounding_boxes_file, wbd=None, projection_of_boxes='EPSG:4329',
-                                wbd_layer='WBDHU8', huc_output_file=None,
-                                forecast_output_file=None, bounding_boxes_outfile=None):
+from utils.shared_functions import getDriver
 
 
+def find_hucs_of_bounding_boxes(
+    bounding_boxes_file,
+    wbd=None,
+    projection_of_boxes='EPSG:4329',
+    wbd_layer='WBDHU8',
+    huc_output_file=None,
+    forecast_output_file=None,
+    bounding_boxes_outfile=None,
+):
     # load bounding box file
-    bounding_boxes = pd.read_csv(bounding_boxes_file,
-                                 dtype={'minx':float,'miny':float,'maxx':float,'maxy':float},
-                                 comment='#')
+    bounding_boxes = pd.read_csv(
+        bounding_boxes_file, dtype={'minx': float, 'miny': float, 'maxx': float, 'maxy': float}, comment='#'
+    )
 
+    bounding_boxes['geometry'] = bounding_boxes.apply(
+        lambda df: box(df['minx'], df['miny'], df['maxx'], df['maxy']), axis=1
+    )
 
-    make_box_geom = lambda df : box(df['minx'],df['miny'],df['maxx'],df['maxy'])
+    bounding_boxes = gpd.GeoDataFrame(bounding_boxes, crs=projection_of_boxes)
 
-    bounding_boxes['geometry'] = bounding_boxes.apply(make_box_geom,axis=1)
-
-    bounding_boxes = gpd.GeoDataFrame(bounding_boxes,crs=projection_of_boxes)
-
-    wbd_proj = gpd.read_file(wbd,layer=wbd_layer,rows=1).crs
+    wbd_proj = gpd.read_file(wbd, layer=wbd_layer, rows=1).crs
 
     bounding_boxes = bounding_boxes.to_crs(wbd_proj)
 
     if bounding_boxes_outfile is not None:
-        bounding_boxes.to_file(bounding_boxes_outfile,driver=getDriver(bounding_boxes_outfile),index=False)
+        bounding_boxes.to_file(bounding_boxes_outfile, driver=getDriver(bounding_boxes_outfile), index=False)
 
-    wbdcol_name = 'HUC'+wbd_layer[-1]
+    wbdcol_name = 'HUC' + wbd_layer[-1]
 
-    get_intersections = lambda bbdf : gpd.read_file(wbd,layer=wbd_layer,mask=bbdf.geometry)[wbdcol_name]
+    hucs = bounding_boxes.apply(
+        lambda bbdf: gpd.read_file(wbd, layer=wbd_layer, mask=bbdf.geometry)[wbdcol_name], axis=1
+    )
 
-    hucs = bounding_boxes.apply(get_intersections,axis=1)
-        
-    bounding_boxes.drop(columns=['geometry','minx','miny','maxx','maxy'],inplace=True)
+    bounding_boxes.drop(columns=['geometry', 'minx', 'miny', 'maxx', 'maxy'], inplace=True)
 
     hucs_columns = hucs.columns
     bb_columns = bounding_boxes.columns
     bounding_boxes = hucs.join(bounding_boxes)
-    bounding_boxes = pd.melt(bounding_boxes,id_vars=bb_columns,value_vars=hucs_columns,value_name='HUC8')
-    bounding_boxes.drop(columns=['variable'],inplace=True)
+    bounding_boxes = pd.melt(bounding_boxes, id_vars=bb_columns, value_vars=hucs_columns, value_name='HUC8')
+    bounding_boxes.drop(columns=['variable'], inplace=True)
     bounding_boxes.dropna(inplace=True)
-    bounding_boxes.reset_index(drop=True,inplace=True)
+    bounding_boxes.reset_index(drop=True, inplace=True)
 
     hucs_series = pd.Series(hucs.stack().reset_index(drop=True).unique())
-    
+
     if huc_output_file is not None:
-        hucs_series.to_csv(huc_output_file,sep='\n',index=False,header=False)
-    
+        hucs_series.to_csv(huc_output_file, sep='\n', index=False, header=False)
+
     if forecast_output_file is not None:
-        bounding_boxes.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d %H:%M:%S%Z')
-    
-    return(hucs_series,bounding_boxes)
+        bounding_boxes.to_csv(forecast_output_file, index=False, date_format='%Y-%m-%d %H:%M:%S%Z')
 
+    return (hucs_series, bounding_boxes)
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     # parse arguments
     parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
-    parser.add_argument('-b','--bounding-boxes-file', help='Bounding box file', required=True)
-    parser.add_argument('-w','--wbd', help='WBD file', required=True)
-    parser.add_argument('-p','--projection-of-boxes', help='Projection', required=False,default='EPSG:4329')
-    parser.add_argument('-o','--huc-output-file', help='Output file of HUCS', required=False,default=None)
-    parser.add_argument('-f','--forecast-output-file', help='Forecast file', required=False,default=None)
-    parser.add_argument('-u','--bounding-boxes-outfile', help='Bounding boxes outfile', required=False,default=None)
-
-    args=vars(parser.parse_args())
+    parser.add_argument('-b', '--bounding-boxes-file', help='Bounding box file', required=True)
+    parser.add_argument('-w', '--wbd', help='WBD file', required=True)
+    parser.add_argument('-p', '--projection-of-boxes', help='Projection', required=False, default='EPSG:4329')
+    parser.add_argument('-o', '--huc-output-file', help='Output file of HUCS', required=False, default=None)
+    parser.add_argument('-f', '--forecast-output-file', help='Forecast file', required=False, default=None)
+    parser.add_argument(
+        '-u', '--bounding-boxes-outfile', help='Bounding boxes outfile', required=False, default=None
+    )
+
+    args = vars(parser.parse_args())
 
     find_hucs_of_bounding_boxes(**args)
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 368fc2978..601b389e4 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -3,11 +3,13 @@
 
 import argparse
 import os
+
 import pandas as pd
 from overlapping_inundation import OverlapWindowMerge
 from tqdm import tqdm
-from utils.shared_variables import elev_raster_ndv
+
 from utils.shared_functions import FIM_Helpers as fh
+from utils.shared_variables import elev_raster_ndv
 
 
 def Mosaic_inundation(
@@ -149,12 +151,12 @@ def mosaic_by_unit(
 
 
 def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon, driver="GPKG"):
+    import geopandas as gpd
     import numpy as np
     import rasterio
     from rasterio.features import shapes
-    from shapely.geometry.polygon import Polygon
     from shapely.geometry.multipolygon import MultiPolygon
-    import geopandas as gpd
+    from shapely.geometry.polygon import Polygon
 
     with rasterio.open(inundation_raster) as src:
         # Open inundation_raster using rasterio.
@@ -168,11 +170,7 @@ def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon
         results = (
             {"properties": {"extent": 1}, "geometry": s}
             for i, (s, v) in enumerate(
-                shapes(
-                    reclass_inundation_array,
-                    mask=reclass_inundation_array > 0,
-                    transform=src.transform,
-                )
+                shapes(reclass_inundation_array, mask=reclass_inundation_array > 0, transform=src.transform)
             )
         )
 
@@ -229,11 +227,7 @@ def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon
         nargs="+",
     )
     parser.add_argument(
-        "-n",
-        "--nodata",
-        help="NODATA value for output raster",
-        required=False,
-        default=elev_raster_ndv,
+        "-n", "--nodata", help="NODATA value for output raster", required=False, default=elev_raster_ndv
     )
     parser.add_argument(
         "-w",
@@ -278,7 +272,8 @@ def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon
     parser.add_argument(
         "-g",
         "--is-mosaic-for-branches",
-        help="If the mosaic is for branchs, include this arg. If is_mosaic_for_branches is true, the mosaic output name will add the HUC into the output name for overwrite reasons.",
+        help="If the mosaic is for branchs, include this arg. If is_mosaic_for_branches is true, "
+        "the mosaic output name will add the HUC into the output name for overwrite reasons.",
         required=False,
         default=False,
         action="store_true",
diff --git a/tools/nesdis_preprocessing.py b/tools/nesdis_preprocessing.py
deleted file mode 100755
index 45be46247..000000000
--- a/tools/nesdis_preprocessing.py
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/usr/bin/env python3
-
-from shapely.geometry import box
-import pandas as pd
-import geopandas as gpd
-import argparse
-from datetime import datetime
-from make_boxes_from_bounds import find_hucs_of_bounding_boxes
-import requests
-from concurrent.futures import ThreadPoolExecutor,as_completed
-import os
-from tqdm import tqdm
-
-##################################
-##
-## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
-## Might want to be kept for possible re-use at a later time?
-##
-##################################
-
-
-def nesdis_preprocessing(bounding_boxes_file,wbd=None,projection_of_boxes='EPSG:4329',wbd_layer='WBDHU8',forecast_output_file=None,retrieve=True,workers=6,download_directory=None):
-
-    _, bounding_boxes = find_hucs_of_bounding_boxes(bounding_boxes_file,wbd=wbd,projection_of_boxes=projection_of_boxes,wbd_layer=wbd_layer)
-
-    # load bounding box file
-    bounding_boxes['event_start'] = pd.to_datetime(bounding_boxes['event_start'],utc=True)
-    bounding_boxes['event_end'] = pd.to_datetime(bounding_boxes['event_end'],utc=True)
-    bounding_boxes['additional_date'] = pd.to_datetime(bounding_boxes['additional_date'],utc=True)
-
-    bounding_boxes.reset_index(drop=True,inplace=True)
-    
-    wbdcol_name = 'HUC'+wbd_layer[-1]
-
-    # expand dates
-    datetime_indices = bounding_boxes.apply(lambda df:pd.date_range(df['event_start'],df['event_end'],closed=None,freq='D',tz='UTC'),axis=1)
-    
-    datetime_indices.name = 'date_time'
-    datetime_indices=pd.DataFrame(datetime_indices)
-    datetime_indices = datetime_indices.join(bounding_boxes[['Name',wbdcol_name]])
-
-    # append columns to expanded dates
-    forecast_df = pd.DataFrame()
-    for idx,row in datetime_indices.iterrows():
-        dt_df = row['date_time'].to_frame(index=False,name='date_time')
-        
-        row = row.drop('date_time')
-
-        dt_df = dt_df.join(pd.concat([pd.DataFrame(row).T]*len(dt_df),ignore_index=True))
-
-        forecast_df = pd.concat((forecast_df,dt_df),ignore_index=True)
-
-    # add extra dry date 
-    additional_date_df = forecast_df[['Name',wbdcol_name]].merge(bounding_boxes[['additional_date',wbdcol_name]],left_on=wbdcol_name,right_on=wbdcol_name).drop_duplicates(ignore_index=True)
-
-    forecast_df = pd.concat((forecast_df,additional_date_df.rename(columns={'additional_date':'date_time'})),ignore_index=True)
-
-    forecast_df = forecast_df.sort_values(['Name',wbdcol_name],ignore_index=True)
-
-
-    forecast_df['date_time'] = forecast_df.apply(lambda df : df['date_time'].replace(hour=18,minute=0),axis=1)
-
-    forecast_df = forecast_df.rename(columns={wbdcol_name:'huc'})
-
-    forecast_df = construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=retrieve,workers=workers)
-   
-    if forecast_output_file is not None:
-        forecast_df.to_csv(forecast_output_file,index=False,date_format='%Y-%m-%d %H:%M:%S%Z')
-
-
-
-
-def construct_nwm_forecast_filenames_and_retrieve(forecast_df,download_directory,retrieve=True,workers=1):
-    # make forecast file names for NWM and retrieve
-
-    #construct url
-    #url = f'{year}/{year}{month}{day}{time}.CHRTOUT_DOMAIN1.comp'
-
-    make_url = lambda df:  "https://noaa-nwm-retrospective-2-1-pds.s3.amazonaws.com/model_output/" + \
-                                          str(df['date_time'].year) + '/' + str(df['date_time'].year) + \
-                                           str(df['date_time'].month).zfill(2) + \
-                                           str(df['date_time'].day).zfill(2) + \
-                                           str(df['date_time'].hour).zfill(2) + \
-                                           str(df['date_time'].minute).zfill(2) +\
-                                           ".CHRTOUT_DOMAIN1.comp"
-        
-    def make_file_names(df,download_directory):
-        # assumes that the last segment after the / represents the file name
-        url = df['forecast_url']
-        file_name_start_pos = url.rfind("/") + 1
-        file_name = url[file_name_start_pos:]
-
-        file_name = os.path.join(download_directory,file_name)
-
-        return(file_name)
-
-
-    urls = forecast_df.apply(make_url,axis=1)
-    forecast_df['forecast_url'] = urls
-
-    file_names = forecast_df.apply(lambda df: make_file_names(df,download_directory),axis=1)
-    forecast_df['forecast_file'] = file_names
-
-    if not retrieve:
-        return(forecast_df)
-
-    download_df = forecast_df[['forecast_url','forecast_file']].drop_duplicates()
-
-
-    def download_url(url,file_name):
-        r = requests.get(url, stream=True)
-        if r.status_code == requests.codes.ok:
-             with open(file_name, 'wb') as f:
-                for data in r:
-                    f.write(data)
-        return url
-
-    pool = ThreadPoolExecutor(max_workers=workers)
-    
-    results = { pool.submit(download_url,*(url,file_name)) : (url,file_name) for idx,(url,file_name) in download_df.iterrows() }
-
-    for future in tqdm(as_completed(results),total=len(download_df)):
-
-        url,file_name = results[future]
-
-        try:
-            future.result()
-        except Exception as exc:
-            print('error',exc,url)
-
-    pool.shutdown(wait=True)
-    
-    return(forecast_df)
-
-
-if __name__ == '__main__':
-
-    ##################################
-    ##
-    ## Likely Deprecated: File appears to be no longer used. Noticed Jan 16, 2023
-    ## Might want to be kept for possible re-use at a later time?
-    ##
-    ##################################
-
-
-    # parse arguments
-    parser = argparse.ArgumentParser(description='Find hucs for bounding boxes')
-    parser.add_argument('-b','--bounding-boxes-file', help='Bounding box file', required=True)
-    parser.add_argument('-w','--wbd', help='WBD file', required=True)
-    parser.add_argument('-f','--forecast-output-file', help='Forecast file', required=False,default=None)
-    parser.add_argument('-r','--retrieve', help='Forecast file', required=False,default=False,action='store_true')
-    parser.add_argument('-j','--workers', help='Forecast file', required=False,default=1,type=int)
-    parser.add_argument('-d','--download_directory', help='Forecast file', required=False,default=1)
-
-    args=vars(parser.parse_args())
-
-    nesdis_preprocessing(**args)
diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py
index 7ed30b20c..d1ee54516 100644
--- a/tools/overlapping_inundation.py
+++ b/tools/overlapping_inundation.py
@@ -1,18 +1,19 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import rasterio
-from rasterio.windows import from_bounds
-import numpy as np
+import concurrent.futures
+import warnings
 from functools import partial
-from affine import Affine
-from scipy.optimize import newton
 from threading import Lock
-import concurrent.futures
-from numba import njit
+
 import geopandas as gpd
+import numpy as np
+import rasterio
+from affine import Affine
+from numba import njit
 from rasterio.mask import mask
-import warnings
+from rasterio.windows import from_bounds
+from scipy.optimize import newton
 
 
 class OverlapWindowMerge:
@@ -26,8 +27,14 @@ def __init__(self, inundation_rsts, num_partitions=None, window_xy_size=None):
         """
 
         # sort for largest spanning dataset (todo: handle mismatched resolutions)
-        size_func = lambda x: np.abs(x.bounds.left - x.bounds.right) * np.abs(x.bounds.top - x.bounds.bottom)
-        key_sort_func = lambda x: x["size"]
+        # size_func = lambda x: np.abs(x.bounds.left - x.bounds.right) * np.abs(x.bounds.top - x.bounds.bottom)
+        def size_func(x):
+            return np.abs(x.bounds.left - x.bounds.right) * np.abs(x.bounds.top - x.bounds.bottom)
+
+        # key_sort_func = lambda x: x["size"]
+        def key_sort_func(x):
+            return x['size']
+
         datasets = [rasterio.open(ds) for ds in inundation_rsts]
         ds_dict = [{"dataset": ds, "size": size_func(ds)} for ds in datasets]
         ds_dict.sort(key=key_sort_func, reverse=True)
@@ -149,7 +156,8 @@ def create_lat_lons(self, window_bounds, window_idx):
 
         :param window_bounds: tuple or list of partition sizes for x and y
         :param window_idx: int representing index of window
-        :return: list of float latitudes, list of float longitudes, list of window bbox, list of ul/br coords for window
+        :return: list of float latitudes, list of float longitudes, list of window bbox,
+                    list of ul/br coords for window
         """
 
         upper_left = window_idx.T * window_bounds[0]
@@ -431,10 +439,9 @@ def merge_data(
 
 
 if __name__ == "__main__":
-    import time
-
     # import tracemalloc
     import glob
+    import time
 
     # print('start', time.localtime())
     # project_path = r'../documentation/data'
@@ -445,7 +452,6 @@ def merge_data(
     #                              (3, 3))
     # overlap.merge_rasters(project_path + '/merged_overlap.tif', nodata=0)
     # print('end', time.localtime())
-
     # tracemalloc.start()
     print("start", time.localtime())
     # project_path = r'../documentation/data'
diff --git a/tools/pixel_counter.py b/tools/pixel_counter.py
index b1d3afabb..361dc584b 100644
--- a/tools/pixel_counter.py
+++ b/tools/pixel_counter.py
@@ -1,3 +1,29 @@
+#!/usr/bin/env python3
+
+import argparse
+import copy
+import os
+import pathlib
+import sys
+import tempfile
+
+import numpy as np
+import pandas as pd
+
+# Import raster and vector function libraries
+# from types import NoneType
+from osgeo import gdal, ogr
+from osgeo.gdalconst import *
+from pandas import DataFrame
+from pixel_counter_functions import (
+    get_bridge_counts,
+    get_levee_counts,
+    get_mask_value_counts,
+    get_nlcd_counts,
+    get_nlcd_counts_inside_flood,
+)
+
+
 '''Created on 02/21/2022.
 Written by:
 Anuska Narayanan (The University of Alabama Department of Geography, anarayanan1@crimson.ua.edu;
@@ -7,38 +33,20 @@
 Derived from a Python version of a zonal statistics function written by Matthew Perry (@perrygeo).
 
 Description: This script isolates the number of pixels per class of a raster within the outlines of
-one or more polygons and displays them in a table. It accomplishes this by rasterizing the vector file,
-masking out the desired areas of both rasters, and then summarizing them in a dataframe. It makes use
-of the gdal, numpy, and pandas function libraries.
+    one or more polygons and displays them in a table. It accomplishes this by rasterizing the vector file,
+    masking out the desired areas of both rasters, and then summarizing them in a dataframe. It makes use
+    of the gdal, numpy, and pandas function libraries.
 Inputs: one raster file with at least one set of attributes; one vector file containing one or more polygon
-boundaries
+    boundaries
 Output: a dataframe table with rows displayed by each polygon within the vector file, and columns
-displaying the pixel count of each raster attribute class in the polygon
+    displaying the pixel count of each raster attribute class in the polygon
 '''
 
-# Import raster and vector function libraries
-#from types import NoneType
-from osgeo import gdal, ogr
-from osgeo.gdalconst import *
-# Import numerical data library
-import numpy as np
-# Import file management library
-import sys
-import os
-# Import data analysis library
-import pandas as pd
-import argparse
-from pandas import DataFrame
-import copy
-import pathlib
-import tempfile
-
-from pixel_counter_functions import (get_nlcd_counts, get_levee_counts, get_bridge_counts, get_nlcd_counts_inside_flood,get_mask_value_counts)
-
 
 # Set up error handler
 gdal.PushErrorHandler('CPLQuietErrorHandler')
 
+
 # Function to pologonize flood extent
 def make_flood_extent_polygon(flood_extent):
     flood_extent_dataset = gdal.Open(flood_extent)
@@ -89,7 +97,7 @@ def make_flood_extent_polygon(flood_extent):
     # Polygonize
     gdal.Polygonize(band, None, outLayer, 0, [], callback=None)
     outDatasource.Destroy()
-    sourceRaster = None
+    # sourceRaster = None
 
     fullpath = os.path.abspath(outShapefile)
     print(fullpath)
@@ -117,7 +125,6 @@ def bbox_to_pixel_offsets(gt, bbox):
 
 # Main function that determines zonal statistics of raster classes in a polygon area
 def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_extent=False):
-
     stats = []
 
     # Loop through different raster paths in the raster_path_dict and
@@ -126,14 +133,13 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
         raster_path = raster_path_dict[layer]
         if raster_path == "":  # Only process if a raster path is provided
             continue
-        if layer == 'flood_extent' and raster_path_dict["nlcd"]!= "":
+        if layer == 'flood_extent' and raster_path_dict["nlcd"] != "":
             vector_path = make_flood_extent_polygon(flood_extent)
             raster_path = raster_path_dict["nlcd"]
 
-
         # Opens raster file and sets path
         rds = gdal.Open(raster_path)
-        
+
         assert rds
         rb = rds.GetRasterBand(1)
         rgt = rds.GetGeoTransform()
@@ -145,14 +151,16 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
             print('No vector path provided. Continuing to next layer.')
             continue
         # Opens vector file and sets path
-        
+
         try:
             vds = ogr.Open(vector_path)
             vlyr = vds.GetLayer(0)
-        except:
+        except Exception as e:
+            print(repr(e))
             continue
-        
-        # Creates an in-memory numpy array of the source raster data covering the whole extent of the vector layer
+
+        # Creates an in-memory numpy array of the source raster data covering
+        #   the whole extent of the vector layer
         if global_src_extent:
             # use global source extent
             # useful only when disk IO or raster scanning inefficiencies are your limiting factor
@@ -168,7 +176,7 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
                 0.0,
                 (rgt[3] + (src_offset[1] * rgt[5])),
                 0.0,
-                rgt[5]
+                rgt[5],
             )
 
         mem_drv = ogr.GetDriverByName('Memory')
@@ -179,7 +187,6 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
 
         feat = vlyr.GetNextFeature()
         while feat is not None:
-
             if not global_src_extent:
                 # use local source extent
                 # fastest option when you have fast disks and well indexed raster (ie tiled Geotiff)
@@ -195,7 +202,7 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
                     0.0,
                     (rgt[3] + (src_offset[1] * rgt[5])),
                     0.0,
-                    rgt[5]
+                    rgt[5],
                 )
 
             # Create a temporary vector layer in memory
@@ -209,18 +216,15 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
             gdal.RasterizeLayer(rvds, [1], mem_layer, burn_values=[1])
             rv_array = rvds.ReadAsArray()
 
-            # Mask the source data array with our current feature and get statistics (pixel count) of masked areas
+            # Mask the source data array with our current feature and get statistics (pixel count)
+            #   of masked areas
             # we take the logical_not to flip 0<->1 to get the correct mask effect
             # we also mask out nodata values explictly
             if src_array is None:
                 feat = vlyr.GetNextFeature()
                 continue
             masked = np.ma.MaskedArray(
-                src_array,
-                mask=np.logical_or(
-                    src_array == nodata_value,
-                    np.logical_not(rv_array)
-                )
+                src_array, mask=np.logical_or(src_array == nodata_value, np.logical_not(rv_array))
             )
 
             # Call different counter functions depending on the raster's source.
@@ -254,30 +258,17 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
     # opts = {'VECTOR': sys.argv[1:], 'RASTER': sys.argv[2:]}
     # stats = zonal_stats(opts['VECTOR'], opts['RASTER'])
 
-    parser = argparse.ArgumentParser(description='Computes pixel counts for raster classes within a vector area.')
-    parser.add_argument('-v', '--vector',
-                        help='Path to vector file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-n', '--nlcd',
-                        help='Path to National Land Cover Database raster file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-l', '--levees',
-                        help='Path to levees raster file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-b', '--bridges',
-                        help='Path to bridges file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-f', '--flood_extent',
-                        help='Path to flood extent file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-c', '--csv',
-                        help='Path to export csv file.',
-                        required=True)
+    parser = argparse.ArgumentParser(
+        description='Computes pixel counts for raster classes within a vector area.'
+    )
+    parser.add_argument('-v', '--vector', help='Path to vector file.', required=False, default="")
+    parser.add_argument(
+        '-n', '--nlcd', help='Path to National Land Cover Database raster file.', required=False, default=""
+    )
+    parser.add_argument('-l', '--levees', help='Path to levees raster file.', required=False, default="")
+    parser.add_argument('-b', '--bridges', help='Path to bridges file.', required=False, default="")
+    parser.add_argument('-f', '--flood_extent', help='Path to flood extent file.', required=False, default="")
+    parser.add_argument('-c', '--csv', help='Path to export csv file.', required=True)
     # Assign variables from arguments.
     args = vars(parser.parse_args())
     vector = args['vector']
@@ -287,7 +278,7 @@ def zonal_stats(vector_path, raster_path_dict, nodata_value=None, global_src_ext
     flood_extent = args['flood_extent']
 
     csv = args['csv']
-    
+
     raster_path_dict = {'nlcd': nlcd, 'levees': levees, 'bridges': bridges, 'flood_extent': flood_extent}
     stats = zonal_stats(vector, raster_path_dict)
 
diff --git a/tools/pixel_counter_functions.py b/tools/pixel_counter_functions.py
index 63add9861..36fa01f2a 100644
--- a/tools/pixel_counter_functions.py
+++ b/tools/pixel_counter_functions.py
@@ -1,5 +1,7 @@
 import numpy as np
-def get_nlcd_counts_inside_flood(feat,masked):
+
+
+def get_nlcd_counts_inside_flood(feat, masked):
     # Gets NLCD Counds inside flood extent
     feature_stats = {
         'f_HydroID': feat.GetField('HydroID'),
@@ -25,20 +27,25 @@ def get_nlcd_counts_inside_flood(feat,masked):
         'f_lulc_90': np.count_nonzero((masked == [90])),
         'f_lulc_95': np.count_nonzero((masked == [95])),
         'f_lulc_1': np.count_nonzero((masked == [11])) + np.count_nonzero((masked == [12])),
-        'f_lulc_2': np.count_nonzero((masked == [21])) + np.count_nonzero((masked == [22]))
-                  + np.count_nonzero((masked == [23])) + np.count_nonzero((masked == [24])),
+        'f_lulc_2': np.count_nonzero((masked == [21]))
+        + np.count_nonzero((masked == [22]))
+        + np.count_nonzero((masked == [23]))
+        + np.count_nonzero((masked == [24])),
         'f_lulc_3': np.count_nonzero((masked == [31])),
-        'f_lulc_4': np.count_nonzero((masked == [41])) + np.count_nonzero((masked == [42]))
-                  + np.count_nonzero((masked == [43])),
+        'f_lulc_4': np.count_nonzero((masked == [41]))
+        + np.count_nonzero((masked == [42]))
+        + np.count_nonzero((masked == [43])),
         'f_lulc_5': np.count_nonzero((masked == [51])) + np.count_nonzero((masked == [52])),
-        'f_lulc_7': np.count_nonzero((masked == [71])) + np.count_nonzero((masked == [72]))
-                  + np.count_nonzero((masked == [73])) + np.count_nonzero((masked == [74])),
+        'f_lulc_7': np.count_nonzero((masked == [71]))
+        + np.count_nonzero((masked == [72]))
+        + np.count_nonzero((masked == [73]))
+        + np.count_nonzero((masked == [74])),
         'f_lulc_8': np.count_nonzero((masked == [81])) + np.count_nonzero((masked == [82])),
-        'f_lulc_9': np.count_nonzero((masked == [90])) + np.count_nonzero((masked == [95]))
-
+        'f_lulc_9': np.count_nonzero((masked == [90])) + np.count_nonzero((masked == [95])),
     }
     return feature_stats
 
+
 def get_nlcd_counts(feat, masked):
     # Acquires information for table on each raster attribute per poly feature
     feature_stats = {
@@ -66,30 +73,31 @@ def get_nlcd_counts(feat, masked):
         'lulc_90': np.count_nonzero((masked == [90])),
         'lulc_95': np.count_nonzero((masked == [95])),
         'lulc_1': np.count_nonzero((masked == [11])) + np.count_nonzero((masked == [12])),
-        'lulc_2': np.count_nonzero((masked == [21])) + np.count_nonzero((masked == [22]))
-                  + np.count_nonzero((masked == [23])) + np.count_nonzero((masked == [24])),
+        'lulc_2': np.count_nonzero((masked == [21]))
+        + np.count_nonzero((masked == [22]))
+        + np.count_nonzero((masked == [23]))
+        + np.count_nonzero((masked == [24])),
         'lulc_3': np.count_nonzero((masked == [31])),
-        'lulc_4': np.count_nonzero((masked == [41])) + np.count_nonzero((masked == [42]))
-                  + np.count_nonzero((masked == [43])),
+        'lulc_4': np.count_nonzero((masked == [41]))
+        + np.count_nonzero((masked == [42]))
+        + np.count_nonzero((masked == [43])),
         'lulc_5': np.count_nonzero((masked == [51])) + np.count_nonzero((masked == [52])),
-        'lulc_7': np.count_nonzero((masked == [71])) + np.count_nonzero((masked == [72]))
-                  + np.count_nonzero((masked == [73])) + np.count_nonzero((masked == [74])),
+        'lulc_7': np.count_nonzero((masked == [71]))
+        + np.count_nonzero((masked == [72]))
+        + np.count_nonzero((masked == [73]))
+        + np.count_nonzero((masked == [74])),
         'lulc_8': np.count_nonzero((masked == [81])) + np.count_nonzero((masked == [82])),
-        'lulc_9': np.count_nonzero((masked == [90])) + np.count_nonzero((masked == [95]))
-
+        'lulc_9': np.count_nonzero((masked == [90])) + np.count_nonzero((masked == [95])),
     }
     return feature_stats
 
 
 def get_levee_counts(feat, masked):
-
     # Acquires information for table on each levee attribute per catchment
-    feature_stats = {
-        'HydroID': feat.GetField('HydroID'),
-        'TotalLeveePixels': int(masked.count()),
-    }
+    feature_stats = {'HydroID': feat.GetField('HydroID'), 'TotalLeveePixels': int(masked.count())}
     return feature_stats
 
+
 def get_bridge_counts(feat_masked):
     pass
 
@@ -104,7 +112,7 @@ def get_mask_value_counts(feat, masked):
         'fn': np.count_nonzero((masked == [1])),
         'fp': np.count_nonzero((masked == [2])),
         'tp': np.count_nonzero((masked == [3])),
-        'mp': np.count_nonzero((masked == [4]))
-        }
-    
+        'mp': np.count_nonzero((masked == [4])),
+    }
+
     return feature_stats
diff --git a/tools/pixel_counter_wrapper.py b/tools/pixel_counter_wrapper.py
index 575486646..051fb2a4d 100644
--- a/tools/pixel_counter_wrapper.py
+++ b/tools/pixel_counter_wrapper.py
@@ -1,34 +1,38 @@
+#!/usr/bin/env python3
+
 import argparse
-import pandas as pd
 import os
-from multiprocessing import Pool
 import traceback
+from multiprocessing import Pool
 
+import pandas as pd
 from pixel_counter import zonal_stats
 
 
 def queue_zonal_stats(fim_run_dir, raster_path_dict, output_dir, job_number):
     """
     This function sets up multiprocessing of the process_zonal_stats() function.
-    
+
     """
-    
+
     # Make output directory if it doesn't exist
     if not os.path.exists(output_dir):
         os.mkdir(output_dir)
-        
+
     fim_version = os.path.split(fim_run_dir)[1]  # Parse FIM Version
-    
+
     fim_run_dir_list = os.listdir(fim_run_dir)  # List all HUCs in FIM run dir
-    
+
     # Define variables to pass into process_zonal_stats()
     procs_list = []
     for huc in fim_run_dir_list:
-        vector = os.path.join(fim_run_dir, huc, 'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg')
+        vector = os.path.join(
+            fim_run_dir, huc, 'demDerived_reaches_split_filtered_addedAttributes_crosswalked.gpkg'
+        )
         csv = os.path.join(output_dir, fim_version + '_' + huc + '_pixel_counts.csv')
         print(csv)
         procs_list.append([vector, csv, raster_path_dict])
-        
+
     # Initiate multiprocessing
     with Pool(processes=job_number) as pool:
         pool.map(process_zonal_stats, procs_list)
@@ -37,22 +41,22 @@ def queue_zonal_stats(fim_run_dir, raster_path_dict, output_dir, job_number):
 def process_zonal_stats(args):
     """
     This function calls zonal_stats() in multiprocessing mode.
-    
+
     """
     # Extract variables from args
     vector = args[0]
     csv = args[1]
     raster = args[2]
-    
+
     # Do the zonal stats
     try:
-        stats = zonal_stats(vector,raster)  # Call function
-        
+        stats = zonal_stats(vector, raster)  # Call function
+
         # Export CSV
         df = pd.DataFrame(stats)
         df.to_csv(csv)
         print("Finished writing: " + csv)
-        
+
     # Save traceback to error file if error is encountered.
     except Exception:
         error_file = csv.replace('.csv', '_error.txt')
@@ -61,31 +65,23 @@ def process_zonal_stats(args):
 
 
 if __name__ == '__main__':
-    
-    parser = argparse.ArgumentParser(description='Computes pixel counts for raster classes within a vector area.')
-    parser.add_argument('-d', '--fim-run-dir',
-                        help='Path to vector file.',
-                        required=True)
-    parser.add_argument('-n', '--nlcd',
-                        help='Path to National Land Cover Database raster file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-l', '--levees',
-                        help='Path to levees raster file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-b', '--bridges',
-                        help='Path to bridges file.',
-                        required=False,
-                        default="")
-    parser.add_argument('-o', '--output-dir',
-                        help='Path to output directory where CSV files will be written.',
-                        required=False,
-                        default="")
-    parser.add_argument('-j', '--job-number',
-                        help='Number of jobs to use.',
-                        required=False,
-                        default="")
+    parser = argparse.ArgumentParser(
+        description='Computes pixel counts for raster classes within a vector area.'
+    )
+    parser.add_argument('-d', '--fim-run-dir', help='Path to vector file.', required=True)
+    parser.add_argument(
+        '-n', '--nlcd', help='Path to National Land Cover Database raster file.', required=False, default=""
+    )
+    parser.add_argument('-l', '--levees', help='Path to levees raster file.', required=False, default="")
+    parser.add_argument('-b', '--bridges', help='Path to bridges file.', required=False, default="")
+    parser.add_argument(
+        '-o',
+        '--output-dir',
+        help='Path to output directory where CSV files will be written.',
+        required=False,
+        default="",
+    )
+    parser.add_argument('-j', '--job-number', help='Number of jobs to use.', required=False, default="")
 
     # Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -94,9 +90,9 @@ def process_zonal_stats(args):
     bridges = args['bridges']
 
     raster_path_dict = {'nlcd': nlcd, 'levees': levees, 'bridges': bridges}
-    
+
     args = vars(parser.parse_args())
-    
+
     raster_path_dict = {'nlcd': args['nlcd']}
-    
+
     queue_zonal_stats(args['fim_run_dir'], raster_path_dict, args['output_dir'], int(args['job_number']))
diff --git a/tools/plots.py b/tools/plots.py
index 80a1b4fcb..86b7ee6b4 100755
--- a/tools/plots.py
+++ b/tools/plots.py
@@ -1,253 +1,252 @@
 #!/usr/bin/env python3
 
-import seaborn as sns
-import matplotlib.pyplot as plt
-from matplotlib.ticker import FixedLocator, FixedFormatter, NullFormatter, FormatStrFormatter
 import argparse
-import pandas as pd
-import numpy as np
-from statsmodels.robust.robust_linear_model import RLM
 from itertools import product
 
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+from matplotlib.ticker import FixedFormatter, FixedLocator, FormatStrFormatter, NullFormatter
+from statsmodels.robust.robust_linear_model import RLM
 
 
-def gms_box_plots(metrics_table,output_fig=None):
-
-    if isinstance(metrics_table,pd.DataFrame):
+def gms_box_plots(metrics_table, output_fig=None):
+    if isinstance(metrics_table, pd.DataFrame):
         pass
-    elif isinstance(metrics_table,str):
+    elif isinstance(metrics_table, str):
         metrics_table = pd.read_csv(metrics_table)
-    elif isinstance(metrics_table,list):
-        metrics_table = pd.concat([pd.read_csv(mt) for mt in metrics_table],ignore_index=True)
-    else: 
+    elif isinstance(metrics_table, list):
+        metrics_table = pd.concat([pd.read_csv(mt) for mt in metrics_table], ignore_index=True)
+    else:
         ValueError("Pass metrics_table as DataFrame or path to CSV")
 
     metrics_table = preparing_data_for_plotting(metrics_table)
-    
-    facetgrid = sns.catplot( 
-                    data=metrics_table,x='Model',y='Metric Value',
-                    hue='Magnitude',inner='quartile',split=True,
-                    order=['FR','MS','GMS'],
-                    cut=2,
-                    kind='violin',
-                    col='Mannings N',
-                    row='Metric',
-                    margin_titles=False,
-                    sharex=False,
-                    sharey=False,
-                    despine=True,
-                    scale='width',
-                    height=2.75,
-                    aspect=1,
-                    linewidth=1.5,
-                    legend=False,
-                    saturation=0.55
-                  )
-    
-    facetgrid.map_dataframe( sns.regplot,
-                             data=metrics_table,
-                             x='model_integer_encodings',
-                             y='Metric Value',
-                             scatter=False,
-                             ci=None,
-                             robust=True,
-                             color= 'darkgreen',
-                             truncate=False,
-                             line_kws= {'linewidth' : 2},
-                             label='Trend Line'
-                           )
-    
-    #facetgrid.fig.set_size_inches(10,15)
+
+    facetgrid = sns.catplot(
+        data=metrics_table,
+        x='Model',
+        y='Metric Value',
+        hue='Magnitude',
+        inner='quartile',
+        split=True,
+        order=['FR', 'MS', 'GMS'],
+        cut=2,
+        kind='violin',
+        col='Mannings N',
+        row='Metric',
+        margin_titles=False,
+        sharex=False,
+        sharey=False,
+        despine=True,
+        scale='width',
+        height=2.75,
+        aspect=1,
+        linewidth=1.5,
+        legend=False,
+        saturation=0.55,
+    )
+
+    facetgrid.map_dataframe(
+        sns.regplot,
+        data=metrics_table,
+        x='model_integer_encodings',
+        y='Metric Value',
+        scatter=False,
+        ci=None,
+        robust=True,
+        color='darkgreen',
+        truncate=False,
+        line_kws={'linewidth': 2},
+        label='Trend Line',
+    )
+
+    # facetgrid.fig.set_size_inches(10,15)
 
     # legend
-    handles, labels = facetgrid.axes[0,0].get_legend_handles_labels()
-    facetgrid.fig.legend(loc='lower left',ncol=3,bbox_to_anchor=(0.22,-0.06),handles=handles,labels=labels)
+    handles, labels = facetgrid.axes[0, 0].get_legend_handles_labels()
+    facetgrid.fig.legend(
+        loc='lower left', ncol=3, bbox_to_anchor=(0.22, -0.06), handles=handles, labels=labels
+    )
 
     # xlabel
-    plt.annotate('Model',xy=(0.52,0.054),xycoords='figure fraction')
+    plt.annotate('Model', xy=(0.52, 0.054), xycoords='figure fraction')
 
     # override axes params
     facetgrid = set_axes(facetgrid)
 
-    facetgrid.fig.subplots_adjust( wspace=0,
-                                   hspace=0.1,
-                                 )
-    
+    facetgrid.fig.subplots_adjust(wspace=0, hspace=0.1)
+
     # set margins to tight
-    #plt.tight_layout()
-    
+    # plt.tight_layout()
+
     # set rlm metrics
     metrics_table = robust_linear_model(metrics_table)
-    facetgrid = annotate_rlm(facetgrid,metrics_table)
+    facetgrid = annotate_rlm(facetgrid, metrics_table)
 
     # set text sizes
     facetgrid = set_text_sizes(facetgrid)
 
     if output_fig is not None:
-        plt.savefig(output_fig,dpi=300,format='jpg',bbox_inches='tight')
+        plt.savefig(output_fig, dpi=300, format='jpg', bbox_inches='tight')
 
     plt.show()
-    
+
 
 def set_axes(facetgrid):
+    range_by_row = ((0.45, 0.75, 0.05), (0.50, 0.90, 0.05), (0.00, 0.40, 0.05))
 
-    range_by_row = ( (0.45,0.75,0.05),
-                     (0.50,0.90,0.05),
-                     (0.00,0.40,0.05)
-                   )
-    
-    nrows,ncols = facetgrid.axes.shape
+    nrows, ncols = facetgrid.axes.shape
 
     for rowIdx in range(nrows):
-        
         all_ticks = np.arange(*range_by_row[rowIdx])
         major_ticks = all_ticks[1:]
-        #minor_ticks = (all_ticks[0] , all_ticks[-1]+range_by_row[rowIdx][-1])
-        minor_ticks = np.arange( range_by_row[rowIdx][0],
-                                 range_by_row[rowIdx][1]+range_by_row[rowIdx][2],
-                                 range_by_row[rowIdx][2]*2
-                               )
-        major_ticks = np.arange( range_by_row[rowIdx][0]+range_by_row[rowIdx][2],
-                                 range_by_row[rowIdx][1],
-                                 range_by_row[rowIdx][2]*2
-                               )
-        
-        
+        # minor_ticks = (all_ticks[0] , all_ticks[-1]+range_by_row[rowIdx][-1])
+        minor_ticks = np.arange(
+            range_by_row[rowIdx][0],
+            range_by_row[rowIdx][1] + range_by_row[rowIdx][2],
+            range_by_row[rowIdx][2] * 2,
+        )
+        major_ticks = np.arange(
+            range_by_row[rowIdx][0] + range_by_row[rowIdx][2],
+            range_by_row[rowIdx][1],
+            range_by_row[rowIdx][2] * 2,
+        )
+
         for colIdx in range(ncols):
-            
-            facetgrid.axes[rowIdx,colIdx].set_ylim(range_by_row[rowIdx][:-1])
-            facetgrid.axes[rowIdx,colIdx].yaxis.set_major_locator(FixedLocator(major_ticks))
-            facetgrid.axes[rowIdx,colIdx].yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
-            facetgrid.axes[rowIdx,colIdx].yaxis.set_minor_locator(FixedLocator(minor_ticks))
-            facetgrid.axes[rowIdx,colIdx].yaxis.set_minor_formatter(NullFormatter())
-            facetgrid.axes[rowIdx,colIdx].tick_params(labelrotation=45,axis='y')
-            
+            facetgrid.axes[rowIdx, colIdx].set_ylim(range_by_row[rowIdx][:-1])
+            facetgrid.axes[rowIdx, colIdx].yaxis.set_major_locator(FixedLocator(major_ticks))
+            facetgrid.axes[rowIdx, colIdx].yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
+            facetgrid.axes[rowIdx, colIdx].yaxis.set_minor_locator(FixedLocator(minor_ticks))
+            facetgrid.axes[rowIdx, colIdx].yaxis.set_minor_formatter(NullFormatter())
+            facetgrid.axes[rowIdx, colIdx].tick_params(labelrotation=45, axis='y')
+
             # set axis titles
-            current_title = facetgrid.axes[rowIdx,colIdx].get_title()
-            facetgrid.axes[rowIdx,colIdx].set_title('')
+            current_title = facetgrid.axes[rowIdx, colIdx].get_title()
+            facetgrid.axes[rowIdx, colIdx].set_title('')
             values = [val.strip() for val in current_title.split('|')]
             metric = values[0][-3:]
             mannings = values[1][-4:]
 
             # axis border
-            facetgrid.axes[rowIdx,colIdx].axhline(range_by_row[rowIdx][1],color='black')
-            
+            facetgrid.axes[rowIdx, colIdx].axhline(range_by_row[rowIdx][1], color='black')
+
             if colIdx == 1:
-                facetgrid.axes[rowIdx,colIdx].axvline(x=facetgrid.axes[rowIdx,colIdx].get_xlim()[1],color='black')
+                facetgrid.axes[rowIdx, colIdx].axvline(
+                    x=facetgrid.axes[rowIdx, colIdx].get_xlim()[1], color='black'
+                )
 
             if rowIdx == 0:
-                facetgrid.axes[rowIdx,colIdx].set_title(f'Manning\'s N = {mannings}',pad=25)
+                facetgrid.axes[rowIdx, colIdx].set_title(f'Manning\'s N = {mannings}', pad=25)
 
             if colIdx == 0:
-                facetgrid.axes[rowIdx,colIdx].set_ylabel(metric)
+                facetgrid.axes[rowIdx, colIdx].set_ylabel(metric)
 
             # removes x ticks from everything but bottom row
             if rowIdx < (nrows - 1):
-                facetgrid.axes[rowIdx,colIdx].set_xticks([])
-        
+                facetgrid.axes[rowIdx, colIdx].set_xticks([])
+
         # remove ticks from 2nd column
-        facetgrid.axes[rowIdx,1].set(yticklabels=[]) 
+        facetgrid.axes[rowIdx, 1].set(yticklabels=[])
 
-    return(facetgrid)
+    return facetgrid
 
 
 def preparing_data_for_plotting(metrics_table):
+    metrics_table.rename(
+        columns={'extent_config': 'Model', 'magnitude': 'Magnitude', 'TPR': 'POD'}, inplace=True
+    )
 
-    metrics_table.rename(columns= {'extent_config':'Model',
-                                   'magnitude': 'Magnitude',
-                                   'TPR' : 'POD'
-                                  },
-                         inplace=True)
-
-    set_mannings = lambda df : 0.06 if 'n_6' in df['version'] else 0.12
+    # set_mannings = lambda df: 0.06 if 'n_6' in df['version'] else 0.12
+    def set_mannings(df):
+        return 0.06 if 'n_6' in df['version'] else 0.12
 
-    metrics_table['Mannings N'] = metrics_table.apply(set_mannings,axis=1)
+    metrics_table['Mannings N'] = metrics_table.apply(set_mannings, axis=1)
 
-    metrics_table = pd.melt(metrics_table, id_vars=('Model','Magnitude','Mannings N'),
-                            value_vars=('CSI','POD','FAR'),
-                            var_name='Metric',
-                            value_name='Metric Value'
-                            )
+    metrics_table = pd.melt(
+        metrics_table,
+        id_vars=('Model', 'Magnitude', 'Mannings N'),
+        value_vars=('CSI', 'POD', 'FAR'),
+        var_name='Metric',
+        value_name='Metric Value',
+    )
 
-    model_to_value_dict = {'FR' : 0, 'MS': 1, 'GMS': 2}
+    model_to_value_dict = {'FR': 0, 'MS': 1, 'GMS': 2}
 
-    set_model_integer_encodings = lambda df : model_to_value_dict[ df['Model'] ]
+    # set_model_integer_encodings = lambda df: model_to_value_dict[df['Model']]
+    def set_model_integer_encodings(df):
+        return model_to_value_dict[df['Model']]
 
     metrics_table['model_integer_encodings'] = metrics_table.apply(set_model_integer_encodings, axis=1)
-    
-    return(metrics_table)
 
+    return metrics_table
 
-def robust_linear_model(metrics_table):
 
-    metrics = metrics_table.loc[:,'Metric'].unique()
-    mannings = metrics_table.loc[:,'Mannings N'].unique()
+def robust_linear_model(metrics_table):
+    metrics = metrics_table.loc[:, 'Metric'].unique()
+    mannings = metrics_table.loc[:, 'Mannings N'].unique()
 
-    metric_indices = { m : metrics_table.loc[:,'Metric'] == m for m in metrics }
-    mannings_indices = { m : metrics_table.loc[:,'Mannings N'] == m for m in mannings }
+    # metric_indices = {m: metrics_table.loc[:, 'Metric'] == m for m in metrics}
+    # mannings_indices = {m: metrics_table.loc[:, 'Mannings N'] == m for m in mannings}
 
-    metrics_table.set_index(['Metric','Mannings N'],inplace=True,drop=False)
+    metrics_table.set_index(['Metric', 'Mannings N'], inplace=True, drop=False)
     metrics_table.sort_index(inplace=True)
-    for met,man in product(metrics,mannings):
-        
-        y = metrics_table.loc[(met,man),'Metric Value'].to_numpy()
+    for met, man in product(metrics, mannings):
+        y = metrics_table.loc[(met, man), 'Metric Value'].to_numpy()
+
+        X = metrics_table.loc[(met, man), 'model_integer_encodings'].to_numpy()
 
-        X = metrics_table.loc[(met,man),'model_integer_encodings'].to_numpy()
+        X = np.stack((np.ones(X.shape), X), axis=1)
 
-        X = np.stack((np.ones(X.shape),X),axis=1)
-        
-        model = RLM(y,X)
+        model = RLM(y, X)
         results = model.fit()
 
         beta1 = results.params[1]
-        pval_beta1=results.pvalues[1]/2 # two tailed to one tail
+        pval_beta1 = results.pvalues[1] / 2  # two tailed to one tail
 
-        metrics_table.loc[(met,man),'beta1'] = beta1
-        metrics_table.loc[(met,man),'beta1_pvalue'] = pval_beta1
-    
-    metrics_table.reset_index(drop=True,inplace=True)
+        metrics_table.loc[(met, man), 'beta1'] = beta1
+        metrics_table.loc[(met, man), 'beta1_pvalue'] = pval_beta1
 
-    return(metrics_table)
+    metrics_table.reset_index(drop=True, inplace=True)
 
+    return metrics_table
 
-def annotate_rlm(facetgrid,metrics_table):
 
-    metrics_table.set_index(['Metric','Mannings N'],inplace=True,drop=False)
+def annotate_rlm(facetgrid, metrics_table):
+    metrics_table.set_index(['Metric', 'Mannings N'], inplace=True, drop=False)
     metrics_table.sort_index(inplace=True)
 
-    metric_index_dict = {'CSI':0,'POD':1,'FAR':2}
-    mannings_index_dict = {0.06:0 , 0.12:1}
-    
-    for met,man in metrics_table.index.unique():
+    metric_index_dict = {'CSI': 0, 'POD': 1, 'FAR': 2}
+    mannings_index_dict = {0.06: 0, 0.12: 1}
 
+    for met, man in metrics_table.index.unique():
         rowIdx = metric_index_dict[met]
         colIdx = mannings_index_dict[man]
 
-        beta1 = metrics_table.loc[(met,man),'beta1'].unique()[0]
-        beta1_pvalue = metrics_table.loc[(met,man),'beta1_pvalue'].unique()[0]
-
-        facetgrid.axes[rowIdx,colIdx].annotate( r'$\beta_1$  =  {:.4f}  |  p-value  =  {:.3f}'.format(beta1,beta1_pvalue),
-                                                xy=(0.01,1.025),
-                                                xycoords='axes fraction',
-                                                color='darkgreen')
-    
-    return(facetgrid)
+        beta1 = metrics_table.loc[(met, man), 'beta1'].unique()[0]
+        beta1_pvalue = metrics_table.loc[(met, man), 'beta1_pvalue'].unique()[0]
 
+        facetgrid.axes[rowIdx, colIdx].annotate(
+            r'$\beta_1$  =  {:.4f}  |  p-value  =  {:.3f}'.format(beta1, beta1_pvalue),
+            xy=(0.01, 1.025),
+            xycoords='axes fraction',
+            color='darkgreen',
+        )
 
-def set_text_sizes(facetgrid):
+    return facetgrid
 
-    plt.rc('font',size=12)
 
-    return(facetgrid)
+def set_text_sizes(facetgrid):
+    plt.rc('font', size=12)
 
+    return facetgrid
 
 
 if __name__ == '__main__':
-
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-m','--metrics-table',help='Metrics table',required=True,nargs='+')
-    parser.add_argument('-o','--output-fig',help='Output figure',required=False,default=None)
+    parser.add_argument('-m', '--metrics-table', help='Metrics table', required=True, nargs='+')
+    parser.add_argument('-o', '--output-fig', help='Output figure', required=False, default=None)
 
     args = vars(parser.parse_args())
 
diff --git a/tools/plots/eval_plots.py b/tools/plots/eval_plots.py
deleted file mode 100755
index 443a2ba32..000000000
--- a/tools/plots/eval_plots.py
+++ /dev/null
@@ -1,343 +0,0 @@
-#!/usr/bin/env python3
-import pandas as pd
-from pathlib import Path
-import argparse
-from natsort import natsorted
-import geopandas as gpd
-from utils.shared_functions import filter_dataframe, boxplot, scatterplot, barplot
-def eval_plots(metrics_csv, workspace, versions = [], stats = ['CSI','FAR','TPR'] , alternate_ahps_query = False, spatial_ahps = False, fim_1_ms = False):
-
-    '''
-    Creates plots and summary statistics using metrics compiled from 
-    synthesize_test_cases. Required inputs are metrics_csv and workspace. 
-    Outputs include:
-        aggregate_<benchmark source>_<configuration>.csv: this csv 
-            contains the aggregated total statistics (i.e. CSI, FAR, POD)
-            using the summed area_sq_km fields
-        <benchmark source>_<configuration>_common_sites.csv: this csv 
-            contains the unique sites (e.g usgs/nws: nws_lid; ble: huc08) 
-            considered for aggregation/plots for each magnitude. The selected
-            sites occur in all versions analyzed. For example, if FIM 1,
-            FIM 2, FIM 3.0.0.3 were versions analyzed, the common sites 
-            would be those that had data for ALL versions. This 
-            analysis is then redone for each magnitude. As such, the number
-            of sites may vary with magnitude. The number of sites for each
-            magnitude is annotated on generated plots.
-        <benchmark source>_<configuration>_analyzed_data.csv: this is the 
-            dataset used to create plots and aggregate statistics. It is 
-            a subset of the input metrics file and consists of the common
-            sites.
-        csi_aggr_<benchmark source>_<configuration>.png: bar plot of the 
-            aggregated CSI scores. Number of common sites is annotated
-            (see list of sites listed in *_*_common_sites.csv).
-        csi_<benchmark source>_<configuration>.png: box plot of CSI scores 
-            (sites weighted equally). Number of common sites is annotated 
-            (see list of sites listed in *_*_common_sites.csv).
-        far_<benchmark source>_<configuration>*.png: box plot of FAR scores
-            (sites weighted equally). Number of common sites is annotated 
-            (see list of sites listed in *_*_common_sites.csv).
-        tpr_<benchmark source>_<configuration>*.png: box plot of TPR/POD 
-            scores (sites weighted equally). Number of common sites is 
-            annotated (see list of sites listed in *_*_common_sites.csv).
-        csi_scatter_<magnitude>_<configuration>*.png: scatter plot comparing 
-            two versions for a given magnitude. This is only generated if
-            there are exactly two versions analyzed.
-
-    Parameters
-    ----------
-    metrics_csv : STRING
-        Path to csv produced as part of synthesize_test_cases containing
-        all metrics across all versions.
-    workspace : STRING
-        Path to the output workspace. Subdirectories will be created 
-        reflecting the evaluation datasets.
-    versions: LIST
-        A list of versions to be aggregated/plotted. Uses the "startswith" 
-        approach. Versions should be supplied in the order they are to 
-        be plotted. For example: ['fim_', 'fb']; This will evaluate all 
-        versions that start with fim_ (e.g. fim_1, fim_2, fim_3) and any
-        feature branch that starts with "fb". To esbalish version order,
-        the fim versions are naturally sorted and then fb versions 
-        (naturally sorted) are appended. These versions are also used to 
-        filter the input metric csv as only these versions are retained 
-        for analysis. 
-    stats: LIST
-        A list of statistics to be plotted. Must be identical to column 
-        field in metrics_csv. CSI, POD, TPR are currently calculated, if 
-        additional statistics are desired formulas would need to be coded.
-    alternate_ahps_query : STRING, optional
-        The default is false. Currently the default ahps query is same 
-        as done for apg goals. If a different query is desired it can be 
-        supplied and it will supercede the default query. 
-    spatial_ahps : DICTIONARY, optional
-        The default is false. A dictionary with keys as follows: 
-            'static': Path to AHPS point file created during creation of
-                FIM 3 static libraries.
-            'evaluated': Path to extent file created during the creation
-                of the NWS/USGS AHPS preprocessing.
-            'metadata': Path to previously created file that contains 
-                metadata about each site (feature_id, wfo, rfc and etc).
-        No spatial layers will be created if set to False, if a dictionary
-        is supplied then a spatial layer is produced.
-    fim_1_ms: BOOL
-        Default is false. If True then fim_1 rows are duplicated with 
-        extent_config set to MS. This allows for FIM 1 to be included 
-        in MS plots/stats (helpful for nws/usgs ahps comparisons).
-
-    Returns
-    -------
-    all_datasets : DICT
-        Dictionary containing all datasets generated. 
-        Keys: (benchmark_source, extent_config), 
-        Values: (filtered dataframe, common sites)
-
-    '''
-    
-    #Import metrics csv as DataFrame and initialize all_datasets dictionary
-    csv_df = pd.read_csv(metrics_csv)
-
-    #fim_1_ms flag enables FIM 1 to be shown on MS plots/stats
-    if fim_1_ms:
-        #Query FIM 1 rows based on version beginning with "fim_1"
-        fim_1_rows = csv_df.query('version.str.startswith("fim_1")').copy()
-        #Set extent configuration to MS (instead of FR)
-        fim_1_rows['extent_config'] = 'MS'
-        #Append duplicate FIM 1 rows to original dataframe
-        csv_df = pd.concat([csv_df, fim_1_rows], ignore_index = True)
-        
-    #If versions are supplied then filter out    
-    if versions:
-        #Filter out versions based on supplied version list
-        metrics = csv_df.query('version.str.startswith(tuple(@versions))')
-    else:
-        metrics = csv_df
-       
-    #Group by benchmark source
-    benchmark_by_source = metrics.groupby(['benchmark_source', 'extent_config'])
-
-    #Iterate through benchmark_by_source. Pre-filter metrics dataframe 
-    #as needed (e.g. usgs/nws filter query). Then further filtering to 
-    #discard all hucs/nws_lid that are not present across all analyzed 
-    #versions for a given magnitude. The final filtered dataset is written 
-    #to a dictionary with the key (benchmark source, extent config) 
-    #and values (filtered dataframe, common sites).
-    all_datasets = {}
-    for (benchmark_source, extent_configuration), benchmark_metrics in benchmark_by_source:        
-                
-        #If source is usgs/nws define the base resolution and query 
-        #(use alternate query if passed). Append filtered datasets to 
-        #all_datasets dictionary.
-        if benchmark_source in ['usgs','nws']:
-            
-            #Set the base processing unit for the ahps runs.
-            base_resolution = 'nws_lid'
-            
-            #Default query (used for APG) it could be that bad_sites should be modified. If so pass an alternate query using the "alternate_ahps_query"
-            bad_sites = ['grfi2','ksdm7','hohn4','rwdn4']
-            query = "not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites"
-
-            #If alternate ahps evaluation query argument is passed, use that.
-            if alternate_ahps_query:
-                query = alternate_ahps_query
-
-            #Filter the dataset based on query
-            ahps_metrics = benchmark_metrics.query(query)
-            
-            #Filter out all instances where the base_resolution doesn't 
-            #exist across all desired fim versions for a given magnitude.
-            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(ahps_metrics, base_resolution)
-                     
-        #If source is 'ble', set base_resolution and append ble dataset 
-        #to all_datasets dictionary
-        elif benchmark_source == 'ble':
-            
-            #Set the base processing unit for ble runs
-            base_resolution = 'huc'
-            
-            #Filter out all instances where base_resolution doesn't exist 
-            #across all desired fim versions for a given magnitude.
-            all_datasets[(benchmark_source, extent_configuration)] = filter_dataframe(benchmark_metrics, base_resolution)
-            
-    #For each dataset in all_datasets, generate plots and aggregate statistics.
-    for (dataset_name,configuration), (dataset, sites) in all_datasets.items():
-        
-        #Define and create the output workspace as a subfolder within 
-        #the supplied workspace
-        output_workspace = Path(workspace) / dataset_name / configuration.lower()
-        output_workspace.mkdir(parents = True, exist_ok = True)         
-                
-        #Write out the filtered dataset and common sites to file
-        dataset.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_analyzed_data.csv'), index = False)
-        sites_pd = pd.DataFrame.from_dict(sites, orient = 'index').transpose()
-        sites_pd.to_csv(output_workspace / (f'{dataset_name}_{configuration.lower()}_common_sites.csv'), index = False)
-        
-        #set the order of the magnitudes and define base resolution.
-        if dataset_name == 'ble':
-            magnitude_order = ['100yr', '500yr']
-            base_resolution = 'huc'
-        elif dataset_name in ['usgs','nws']:
-            magnitude_order = ['action','minor','moderate','major']
-            base_resolution = 'nws_lid'
-
-        #Calculate aggregated metrics based on total_sq_km fields.
-        dataset_sums = dataset.groupby(['version', 'magnitude'])[['TP_area_km2','FP_area_km2','FN_area_km2']].sum()
-        dataset_sums['csi'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'] + dataset_sums['FN_area_km2'])
-        dataset_sums['far'] = dataset_sums['FP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FP_area_km2'])
-        dataset_sums['pod'] = dataset_sums['TP_area_km2']/(dataset_sums['TP_area_km2'] + dataset_sums['FN_area_km2'])
-        dataset_sums = dataset_sums.reset_index()
-        
-        #Write aggregated metrics to file.
-        dataset_sums.to_csv(output_workspace / f'aggregate_{dataset_name}_{configuration.lower()}.csv', index = False )
-
-        #This section naturally orders analyzed versions which defines 
-        #the hue order for the generated plots.
-        #Get all versions in dataset
-        all_versions = list(dataset.version.unique())        
-        version_order = []        
-        #If versions are not specified then use all available versions 
-        #and assign to versions_list
-        if not versions:
-            versions_list = all_versions
-        #if versions are supplied assign to versions_list
-        else:
-            versions_list = versions        
-        #For each version supplied by the user
-        for version in versions_list:
-            #Select all the versions that start with the supplied version.
-            selected_versions = [sel_version for sel_version in all_versions if sel_version.startswith(version)]
-            #Naturally sort selected_versions
-            selected_versions = natsorted(selected_versions)
-            #Populate version order based on the sorted subsets.
-            version_order.extend(selected_versions)
-            
-        #Define textbox which will contain the counts of each magnitude.
-        textbox = []
-        for magnitude in sites:
-            count = len(sites[magnitude])
-            line_text = f'{magnitude.title()} Sites = {count}'
-            textbox.append(line_text)
-        textbox = '\n'.join(textbox)
-
-        #Create aggregate barplot
-        aggregate_file = output_workspace / (f'csi_aggr_{dataset_name}_{configuration.lower()}.png')        
-        barplot(dataframe = dataset_sums, x_field = 'magnitude', x_order = magnitude_order, y_field = 'csi', hue_field = 'version', ordered_hue = version_order, title_text = f'Aggregate {dataset_name.upper()} FIM Scores', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = aggregate_file)
-        
-        #Create box plots for each metric in supplied stats.
-        for stat in stats:
-            output_file = output_workspace / (f'{stat.lower()}_{dataset_name}_{configuration.lower()}.png')    
-            boxplot(dataframe = dataset, x_field = 'magnitude', x_order = magnitude_order, y_field = stat, hue_field = 'version', ordered_hue = version_order, title_text = f'{dataset_name.upper()} FIM Sites', fim_configuration = configuration, textbox_str = textbox, simplify_legend = True, dest_file = output_file)
-        
-        #Get the last 2 versions from the version order for scatter plot.
-        if len(version_order) == 2:            
-            x_version, y_version = version_order
-            for magnitude in magnitude_order:
-                #Scatterplot comparison between last 2 versions.
-                x_csi = dataset.query(f'version == "{x_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
-                y_csi = dataset.query(f'version == "{y_version}" & magnitude == "{magnitude}"')[[base_resolution, 'CSI']]
-                plotdf = pd.merge(x_csi, y_csi, on = base_resolution, suffixes = (f"_{x_version}",f"_{y_version}"))
-                #Define arguments for scatterplot function.
-                title_text = f'CSI {magnitude}'
-                dest_file = output_workspace / f'csi_scatter_{magnitude}_{configuration.lower()}.png'
-                scatterplot(dataframe = plotdf, x_field = f'CSI_{x_version}', y_field = f'CSI_{y_version}', title_text = title_text, annotate = False, dest_file = dest_file)
-    
-
-    #######################################################################
-    #Create spatial layers with threshold and mapping information
-    ########################################################################
-    if spatial_ahps:
-
-        #Read in supplied shapefile layers
-        #Layer containing metadata for each site (feature_id, wfo, etc). 
-        #Convert nws_lid to lower case.
-        ahps_metadata = gpd.read_file(spatial_ahps['metadata'])
-        ahps_metadata['nws_lid'] = ahps_metadata['nws_lid'].str.lower()
-        metadata_crs = ahps_metadata.crs
-        
-        #Extent layer generated from preprocessing NWS/USGS datasets
-        evaluated_ahps_extent = gpd.read_file(spatial_ahps['evaluated'])
-        
-        #Extent layer generated from static ahps library preprocessing
-        static_library = gpd.read_file(spatial_ahps['static'])
-        
-        #Fields to keep
-        #Get list of fields to keep in merge
-        preserved_static_library_fields = ['nws_lid'] + [i for i in static_library.columns if i.startswith(('Q','S'))]                
-        #Get list of fields to keep in merge.
-        preserved_evaluated_ahps_fields = ['nws_lid', 'source', 'geometry'] + [i for i in evaluated_ahps_extent.columns if i.startswith(('action','minor','moderate','major'))]
-
-        #Join tables to evaluated_ahps_extent
-        evaluated_ahps_extent = evaluated_ahps_extent[preserved_evaluated_ahps_fields]
-        evaluated_ahps_extent = evaluated_ahps_extent.merge(ahps_metadata, on = 'nws_lid')
-        evaluated_ahps_extent['geometry'] = evaluated_ahps_extent['geometry_y']
-        evaluated_ahps_extent.drop(columns = ['geometry_y','geometry_x'], inplace = True)
-        evaluated_ahps_extent = evaluated_ahps_extent.merge(static_library[preserved_static_library_fields], on = 'nws_lid')
-        
-        #Join dataset metrics to evaluated_ahps_extent data. 
-        final_join = pd.DataFrame()
-        for (dataset_name, configuration), (dataset, sites) in all_datasets.items():
-            #Only select ahps from dataset if config is MS
-            if dataset_name in ['usgs','nws'] and configuration == 'MS':
-                #Select records from evaluated_ahps_extent that match the dataset name
-                subset = evaluated_ahps_extent.query(f'source == "{dataset_name}"')                        
-                #Join to dataset
-                dataset_with_subset = dataset.merge(subset, on = 'nws_lid')
-                #Append rows to final_join dataframe
-                final_join = pd.concat([final_join, dataset_with_subset])
-        
-        #Modify version field
-        final_join['version'] = final_join.version.str.split('_nws|_usgs').str[0]
-        
-        #Write geodataframe to file
-        gdf = gpd.GeoDataFrame(final_join, geometry = final_join['geometry'], crs = metadata_crs)
-        output_shapefile = Path(workspace) / 'nws_usgs_site_info.shp'
-        gdf.to_file(output_shapefile) 
-                    
-
-
-#######################################################################
-if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Plot and aggregate statistics for benchmark datasets (BLE/AHPS libraries)')
-    parser.add_argument('-m','--metrics_csv', help = 'Metrics csv created from synthesize test cases.', required = True)
-    parser.add_argument('-w', '--workspace', help = 'Output workspace', required = True)
-    parser.add_argument('-v', '--versions', help = 'List of versions to be plotted/aggregated. Versions are filtered using the "startswith" approach. For example, ["fim_","fb1"] would retain all versions that began with "fim_" (e.g. fim_1..., fim_2..., fim_3...) as well as any feature branch that began with "fb". An other example ["fim_3","fb"] would result in all fim_3 versions being plotted along with the fb.', nargs = '+', default = [])
-    parser.add_argument('-s', '--stats', help = 'List of statistics (abbrev to 3 letters) to be plotted/aggregated', nargs = '+', default = ['CSI','TPR','FAR'], required = False)
-    parser.add_argument('-q', '--alternate_ahps_query',help = 'Alternate filter query for AHPS. Default is: "not nws_lid.isnull() & not flow.isnull() & masked_perc<97 & not nws_lid in @bad_sites" where bad_sites are (grfi2,ksdm7,hohn4,rwdn4)', default = False, required = False)
-    parser.add_argument('-sp', '--spatial_ahps', help = 'If spatial point layer is desired, supply a csv with 3 lines of the following format: metadata, path/to/metadata/shapefile\nevaluated, path/to/evaluated/shapefile\nstatic, path/to/static/shapefile.', default = False, required = False)
-    parser.add_argument('-f', '--fim_1_ms', help = 'If enabled fim_1 rows will be duplicated and extent config assigned "ms" so that fim_1 can be shown on mainstems plots/stats', action = 'store_true', required = False)
-    
-    #Extract to dictionary and assign to variables.
-    args = vars(parser.parse_args())
-    
-    #If errors occur reassign error to True
-    error = False
-    #Create dictionary if file specified for spatial_ahps
-    if args['spatial_ahps']:
-        #Create dictionary
-        spatial_dict = {}
-        with open(args['spatial_ahps']) as file:
-            for line in file:
-                key, value = line.strip('\n').split(',')
-                spatial_dict[key] = Path(value)
-        args['spatial_ahps'] = spatial_dict
-        #Check that all required keys are present and overwrite args with spatial_dict
-        required_keys = set(['metadata', 'evaluated', 'static'])
-        if required_keys - spatial_dict.keys():
-          print('\n Required keys are: metadata, evaluated, static')
-          error = True
-        else:
-            args['spatial_ahps'] = spatial_dict
-
-
-    #Finalize Variables
-    m = args['metrics_csv']
-    w = args['workspace']
-    v = args['versions']
-    s = args['stats']
-    q = args['alternate_ahps_query']
-    sp= args['spatial_ahps']
-    f = args['fim_1_ms']
-
-    #Run eval_plots function
-    if not error:        
-        eval_plots(metrics_csv = m, workspace = w, versions = v, stats = s, alternate_ahps_query = q, spatial_ahps = sp, fim_1_ms = f)
\ No newline at end of file
diff --git a/tools/plots/utils/__init__.py b/tools/plots/utils/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tools/plots/utils/shared_functions.py b/tools/plots/utils/shared_functions.py
deleted file mode 100644
index e2664173f..000000000
--- a/tools/plots/utils/shared_functions.py
+++ /dev/null
@@ -1,321 +0,0 @@
-#!/usr/bin/env python3
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-import re
-#########################################################################
-#Create boxplot
-#########################################################################
-def boxplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, dest_file = False):      
-    '''
-    Create boxplots. 
-
-    Parameters
-    ----------
-    dataframe : DataFrame
-        Pandas dataframe data to be plotted.
-    x_field : STR
-        Field to use for x-axis
-    x_order : List
-        Order to arrange the x-axis.
-    y_field : STR
-        Field to use for the y-axis
-    hue_field : STR
-        Field to use for hue (typically FIM version)
-    title_text : STR
-        Text for plot title.
-    fim_configuration: STR
-        Configuration of FIM (FR or MS or Composite).
-    simplify_legend : BOOL, optional
-        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
-        The default is False.
-    dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
-        no plot is saved to disk. The default is False.
-
-    Returns
-    -------
-    fig : MATPLOTLIB
-        Plot.
-
-    '''
-
-    #initialize plot
-    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    #Use seaborn to plot the boxplot
-    axes=sns.boxplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
-    #set title of plot
-    axes.set_title(f'{title_text} ({y_field})',fontsize=20, weight = 'bold')
-    #Set yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    for index,ytick in enumerate(axes.get_yticks()):
-        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel('',fontsize=0,weight = 'bold')
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-    axes.legend(markerscale = 2, fontsize =20, loc = 'lower left')
-    
-    #If simple legend desired
-    if simplify_legend:
-        #trim labels to FIM 1, FIM 2, and the FIM 3 version    
-        handles, org_labels = axes.get_legend_handles_labels()
-        label_dict = {}
-        for label in org_labels:
-            if 'fim_1' in label:
-                label_dict[label] = 'FIM 1'
-            elif 'fim_2' in label:
-                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()                  
-            elif 'fim_3' in label:
-                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
-                if label.endswith('_c'):
-                    label_dict[label] = label_dict[label] + ' c'
-            else:
-                label_dict[label] = label + ' ' + fim_configuration.lower()
-        #Define simplified labels as a list.
-        new_labels = [label_dict[label] for label in org_labels]
-        #Define legend location. FAR needs to be in different location than CSI/POD.
-        if y_field == 'FAR':
-            legend_location = 'upper right'
-        else:
-            legend_location = 'lower left' 
-        #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = legend_location, ncol = int(np.ceil(len(new_labels)/7)))
-    #Print textbox if supplied
-    if textbox_str:
-        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=14, verticalalignment='top', bbox=box_props)
-
-    #If figure to be saved to disk, then do so, otherwise return figure
-    if dest_file:
-        fig.savefig(dest_file)
-        plt.close(fig)
-    else:
-        return fig
-
-#########################################################################
-#Create scatter plot
-#########################################################################
-def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annotate = False, dest_file = False):      
-    '''
-    Create boxplots. 
-
-    Parameters
-    ----------
-    dataframe : DataFrame
-        Pandas dataframe data to be plotted.
-    x_field : STR
-        Field to use for x-axis (Assumes FIM 2)
-    y_field : STR
-        Field to use for the y-axis (Assumes FIM 3)
-    title_text : STR
-        Text for plot title.    
-    stats_text : STR or BOOL
-        Text for stats to place on chart. Default is false (no stats printed)
-    dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
-        no plot is saved to disk. The default is False.
-
-    Returnsy
-    -------
-    fig : MATPLOTLIB
-        Plot.
-
-    '''
-
-    #initialize plot
-    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    
-    #Use seaborn to plot the boxplot
-    axes=sns.scatterplot(data=dataframe, x=x_field, y=y_field, color = 'black', s = 150)
-    
-    #Set xticks and yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    axes.set(xlim=(0.0,1.0),xticks = np.arange(0,1.1,0.1))
-    axes.grid(b=True, which='major', axis='both')    
-    
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel(f'{x_field.replace("_"," ")}',fontsize='xx-large',weight = 'bold')
-
-    #Plot diagonal line
-    diag_range = [0,1]
-    axes.plot(diag_range, diag_range, color='gray', transform=axes.transAxes)
-
-
-    #set title of plot
-    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
-   
-    if annotate:
-        #Set text for labels
-        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        textbox_str = 'Target Better'
-        axes.text(0.3, 0.6, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
-        textbox_str = 'Baseline Better'
-        axes.text(0.5, 0.2, textbox_str, transform=axes.transAxes, fontsize=32, color = 'gray', fontweight = 'bold', verticalalignment='top', bbox=box_props, rotation = 35, rotation_mode = 'anchor')
-
-    if stats_text:
-        #Add statistics textbox
-        axes.text(0.01, 0.80, stats_text, transform=axes.transAxes, fontsize=24, verticalalignment='top', bbox=box_props)
-
-    #If figure to be saved to disk, then do so, otherwise return fig
-    if dest_file:
-        fig.savefig(dest_file)
-        plt.close(fig)
-    else:
-        return fig
-#########################################################################
-#Create barplot
-#########################################################################
-def barplot(dataframe, x_field, x_order, y_field, hue_field, ordered_hue, title_text, fim_configuration, textbox_str = False, simplify_legend = False, display_values = False, dest_file = False):      
-    '''
-    Create barplots. 
-
-    Parameters
-    ----------
-    dataframe : DataFrame
-        Pandas dataframe data to be plotted.
-    x_field : STR
-        Field to use for x-axis
-    x_order : List
-        Order to arrange the x-axis.
-    y_field : STR
-        Field to use for the y-axis
-    hue_field : STR
-        Field to use for hue (typically FIM version)
-    title_text : STR
-        Text for plot title.
-    fim_configuration: STR
-        Configuration of FIM (FR or MS or Composite).
-    simplify_legend : BOOL, optional
-        If True, it will simplify legend to FIM 1, FIM 2, FIM 3. 
-        Default is False.
-    display_values : BOOL, optional
-        If True, Y values will be displayed above bars. 
-        Default is False.
-    dest_file : STR or BOOL, optional
-        If STR provide the full path to the figure to be saved. If False 
-        no plot is saved to disk. Default is False.
-
-    Returns
-    -------
-    fig : MATPLOTLIB
-        Plot.
-
-    '''
-
-    #initialize plot
-    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(15, 10))
-    #Use seaborn to plot the boxplot
-    axes=sns.barplot(x=x_field, y=y_field, order=x_order, hue=hue_field, hue_order = ordered_hue, data=dataframe, palette='bright')
-    #set title of plot
-    axes.set_title(f'{title_text}',fontsize=20, weight = 'bold')
-    #Set yticks and background horizontal line.
-    axes.set(ylim=(0.0,1.0),yticks = np.arange(0,1.1,0.1))
-    for index,ytick in enumerate(axes.get_yticks()):
-        plt.axhline(y=ytick,color='black',linestyle = '--',linewidth = 1,alpha = 0.1)
-    #Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field.upper()}',fontsize='xx-large',weight = 'bold')
-    axes.set_xlabel('',fontsize=0,weight = 'bold')
-    #Set sizes of ticks and legend.
-    axes.tick_params(labelsize = 'xx-large')
-    axes.legend(markerscale = 2, fontsize =20, loc = 'upper right')
-    #If simple legend desired
-    if simplify_legend:
-        #trim labels to FIM 1, FIM 2, FIM 3    
-        handles, org_labels = axes.get_legend_handles_labels()
-        label_dict = {}
-        for label in org_labels:
-            if 'fim_1' in label:
-                label_dict[label] = 'FIM 1'
-            elif 'fim_2' in label:
-                label_dict[label] = 'FIM 2' + ' ' + fim_configuration.lower()
-            elif 'fim_3' in label:
-                label_dict[label] = re.split('_fr|_ms', label)[0].replace('_','.').replace('fim.','FIM ') + ' ' + fim_configuration.lower()
-                if label.endswith('_c'):
-                    label_dict[label] = label_dict[label] + ' c'
-            else:
-                label_dict[label] = label + ' ' + fim_configuration.lower()
-        #Define simplified labels as a list.
-        new_labels = [label_dict[label] for label in org_labels]
-        #rename legend labels to the simplified labels.
-        axes.legend(handles, new_labels, markerscale = 2, fontsize = 20, loc = 'upper right', ncol = int(np.ceil(len(new_labels)/7)))
-    #Add Textbox
-    if textbox_str:
-        box_props = dict(boxstyle='round', facecolor='white', alpha=0.5)
-        axes.text(0.01, 0.99, textbox_str, transform=axes.transAxes, fontsize=18, verticalalignment='top', bbox=box_props)
-
-    #Display Y values above bars
-    if display_values:
-        #Add values of bars directly above bar.
-        for patch in axes.patches:
-            value = round(patch.get_height(),3)
-            axes.text(patch.get_x()+patch.get_width()/2.,
-                    patch.get_height(),
-                    '{:1.3f}'.format(value),
-                    ha="center", fontsize=18)  
-    
-    #If figure to be saved to disk, then do so, otherwise return fig
-    if dest_file:
-        fig.savefig(dest_file)
-        plt.close(fig)
-    else:
-        return fig
-#######################################################################
-#Filter dataframe generated from csv file from run_test_case aggregation
-########################################################################
-def filter_dataframe(dataframe, unique_field):
-    '''
-
-    This script will filter out the sites (or hucs) which are not consistently 
-    found for all versions for a given magnitude. For example, an AHPS 
-    lid site must have output for all 3 versions (fim1, fim2, fim3) for 
-    a given magnitude (eg action) otherwise that lid is filtered out. 
-    Likewise for a BLE a huc must have output for all 3 versions 
-    (fim1, fim2, fim3) for a given magnitude (eg 100yr) otherwise it is 
-    filtered out.
-
-    Parameters
-    ----------
-    dataframe : Pandas DataFrame
-        Containing the input metrics originating from synthesize_test_cases
-    unique_field : STR
-        base resolution for each benchmark source: 'nws'/'usgs' (nws_lid)
-        ble (huc).
-
-    Returns
-    -------
-    final_filtered_dataframe : Pandas Dataframe
-        Filtered dataframe that contains only common sites (lids or hucs) between versions for each magnitude. For example, for AHPS all sites which were run for each version for a given magnitude will be kept or for ble, all hucs which ran for all versions for a given magnitude. 
-    unique_sites: DICT
-        The sites that were included in the dataframe for each magnitude.
-
-    '''
-    
-    #Get lists of sites for each magnitude/version
-    unique_sites = dataframe.groupby(['magnitude','version'])[unique_field].agg('unique')
-    #Get unique magnitudes
-    magnitudes = dataframe.magnitude.unique()
-    #Create new dataframe to hold metrics for the common sites as well as the actual lists of common sites.
-    final_filtered_dataframe = pd.DataFrame()
-    all_unique_sites = {}
-    #Cycle through each magnitude
-    for magnitude in magnitudes:
-        #Compile a list of sets containing unique lids pertaining to each threshold. List contains 3 unique sets [{fim1:unique lids},{fim2: unique lids},{fim3: unique lids}]
-        sites_per_magnitude=[set(a) for a in unique_sites[magnitude]]
-        #Intersect the sets to get the common lids per threshold then convert to list.
-        common_sites_per_magnitude = list(set.intersection(*sites_per_magnitude))
-        #Write common sites to dataframe
-        all_unique_sites[magnitude] = common_sites_per_magnitude
-        #Query filtered dataframe and only include data associated with the common sites for that magnitude
-        filtered_common_sites = dataframe.query(f'magnitude == "{magnitude}" & {unique_field} in @common_sites_per_magnitude')
-        #Append the data for each magnitude to a final dataframe that will contain data for all common sites for all magnitudes.
-        final_filtered_dataframe = pd.concat([final_filtered_dataframe, filtered_common_sites], ignore_index = True)            
-    
-    return final_filtered_dataframe, all_unique_sites
-
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index ad5914a6f..2364e48f7 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -1,28 +1,32 @@
 #!/usr/bin/env python3
 
+import argparse
+import logging
 import os
-import sys
 import re
-import pandas as pd, geopandas as gpd
-import numpy as np
-import argparse
-import matplotlib.pyplot as plt
-import seaborn as sns
-from functools import reduce
-from multiprocessing import Pool
-from os.path import isfile, join
 import shutil
+import sys
+import time
 import traceback
-import logging
 import warnings
+from functools import reduce
+from multiprocessing import Pool
+from os.path import isfile, join
 from pathlib import Path
-import time
-warnings.simplefilter(action='ignore', category=FutureWarning)
+
+import geopandas as gpd
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
 import rasterio
+import seaborn as sns
 from rasterio import features as riofeatures
 from rasterio import plot as rioplot
 from shapely.geometry import Polygon
 
+
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
 """
     Plot Rating Curves and Compare to USGS Gages
 
@@ -42,6 +46,7 @@
         string of columns to group eval metrics.
 """
 
+
 def check_file_age(file):
     '''
     Checks if file exists, determines the file age, and recommends
@@ -56,52 +61,70 @@ def check_file_age(file):
     if file.is_file():
         modification_time = file.stat().st_mtime
         current_time = time.time()
-        file_age_days = (current_time - modification_time)/86400
+        file_age_days = (current_time - modification_time) / 86400
         if file_age_days > 30:
-            check = f'{file.name} is {int(file_age_days)} days old, consider updating.\nUpdate with rating_curve_get_usgs_curves.py'
+            check = (
+                f'{file.name} is {int(file_age_days)} days old, consider updating.\n'
+                'Update with rating_curve_get_usgs_curves.py'
+            )
         else:
             check = f'{file.name} is {int(file_age_days)} days old.'
 
         return check
 
+
 # recurr_intervals = ['recurr_1_5_cms.csv','recurr_5_0_cms.csv','recurr_10_0_cms.csv']
 
-def generate_rating_curve_metrics(args):
 
-    elev_table_filename         = args[0]
-    branches_folder             = args[1]
-    usgs_gages_filename         = args[2]
-    usgs_recurr_stats_filename  = args[3]
-    nwm_recurr_data_filename    = args[4]
+def generate_rating_curve_metrics(args):
+    elev_table_filename = args[0]
+    branches_folder = args[1]
+    usgs_gages_filename = args[2]
+    usgs_recurr_stats_filename = args[3]
+    nwm_recurr_data_filename = args[4]
     rc_comparison_plot_filename = args[5]
-    nwm_flow_dir                = args[6]
-    catfim_flows_filename       = args[7]
-    huc                         = args[8]
-    alt_plot                    = args[9]
-    single_plot                 = args[10]
+    nwm_flow_dir = args[6]
+    catfim_flows_filename = args[7]
+    huc = args[8]
+    alt_plot = args[9]
+    single_plot = args[10]
 
     logging.info("Generating rating curve metrics for huc: " + str(huc))
-    elev_table = pd.read_csv(elev_table_filename,dtype={'location_id': object, 'feature_id':object,'HydroID':object, 'levpa_id':object})
-    
+    elev_table = pd.read_csv(
+        elev_table_filename,
+        dtype={'location_id': object, 'feature_id': object, 'HydroID': object, 'levpa_id': object},
+    )
+
     # Filter out null and non-integer location_id entries (the crosswalk steps tries to fill AHPS only sites with the nws_lid)
     elev_table.dropna(subset=['location_id'], inplace=True)
     elev_table = elev_table[elev_table['location_id'].apply(lambda x: str(x).isdigit())]
 
     # Read in the USGS gages rating curve database csv
-    usgs_gages = pd.read_csv(usgs_gages_filename,dtype={'location_id': object, 'feature_id':object})
+    usgs_gages = pd.read_csv(usgs_gages_filename, dtype={'location_id': object, 'feature_id': object})
 
     # Aggregate FIM4 hydroTables
     if not elev_table.empty:
         hydrotable = pd.DataFrame()
         for branch in elev_table.levpa_id.unique():
             branch_elev_table = elev_table.loc[elev_table.levpa_id == branch].copy()
-            #branch_elev_table = elev_table.loc[(elev_table.levpa_id == branch) & (elev_table.location_id.notnull())].copy()
-            branch_hydrotable = pd.read_csv(join(branches_folder, str(branch), f'hydroTable_{branch}.csv'),dtype={'HydroID':object,'feature_id':object,'obs_source':object,'last_updated':object,'submitter':object})
+            # branch_elev_table = elev_table.loc[(elev_table.levpa_id == branch) & (elev_table.location_id.notnull())].copy()
+            branch_hydrotable = pd.read_csv(
+                join(branches_folder, str(branch), f'hydroTable_{branch}.csv'),
+                dtype={
+                    'HydroID': object,
+                    'feature_id': object,
+                    'obs_source': object,
+                    'last_updated': object,
+                    'submitter': object,
+                },
+            )
             # Only pull SRC for hydroids that are in this branch
-            branch_hydrotable = branch_hydrotable.loc[branch_hydrotable.HydroID.isin(branch_elev_table.HydroID)]
+            branch_hydrotable = branch_hydrotable.loc[
+                branch_hydrotable.HydroID.isin(branch_elev_table.HydroID)
+            ]
             branch_hydrotable.drop(columns=['order_'], inplace=True)
             # Join SRC with elevation data
-            branch_elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
+            branch_elev_table.rename(columns={'feature_id': 'fim_feature_id'}, inplace=True)
             branch_hydrotable = branch_hydrotable.merge(branch_elev_table, on="HydroID")
             # Append to full rating curve dataframe
             if hydrotable.empty:
@@ -110,8 +133,8 @@ def generate_rating_curve_metrics(args):
                 hydrotable = pd.concat([hydrotable, branch_hydrotable])
 
         # Join rating curves with elevation data
-        #elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
-        #hydrotable = hydrotable.merge(elev_table, on="HydroID")
+        # elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
+        # hydrotable = hydrotable.merge(elev_table, on="HydroID")
         if 'location_id' in hydrotable.columns:
             relevant_gages = list(hydrotable.location_id.unique())
         else:
@@ -120,20 +143,46 @@ def generate_rating_curve_metrics(args):
         usgs_gages = usgs_gages.reset_index(drop=True)
 
         if len(usgs_gages) > 0:
-
             # Adjust rating curve to elevation
-            hydrotable['elevation_ft'] = (hydrotable.stage + hydrotable.dem_adj_elevation) * 3.28084 # convert from m to ft
+            hydrotable['elevation_ft'] = (
+                hydrotable.stage + hydrotable.dem_adj_elevation
+            ) * 3.28084  # convert from m to ft
             # hydrotable['raw_elevation_ft'] = (hydrotable.stage + hydrotable.dem_elevation) * 3.28084 # convert from m to ft
             hydrotable['discharge_cfs'] = hydrotable.discharge_cms * 35.3147
-            usgs_gages = usgs_gages.rename(columns={"flow": "discharge_cfs", "elevation_navd88": "elevation_ft"})
+            usgs_gages = usgs_gages.rename(
+                columns={"flow": "discharge_cfs", "elevation_navd88": "elevation_ft"}
+            )
 
             hydrotable['source'] = "FIM"
             usgs_gages['source'] = "USGS"
-            limited_hydrotable = hydrotable.filter(items=['location_id','elevation_ft','discharge_cfs','source', 'HydroID', 'levpa_id', 'dem_adj_elevation'])
-            select_usgs_gages = usgs_gages.filter(items=['location_id', 'elevation_ft', 'discharge_cfs','source'])
-            if 'default_discharge_cms' in hydrotable.columns: # check if both "FIM" and "FIM_default" SRCs are available
+            limited_hydrotable = hydrotable.filter(
+                items=[
+                    'location_id',
+                    'elevation_ft',
+                    'discharge_cfs',
+                    'source',
+                    'HydroID',
+                    'levpa_id',
+                    'dem_adj_elevation',
+                ]
+            )
+            select_usgs_gages = usgs_gages.filter(
+                items=['location_id', 'elevation_ft', 'discharge_cfs', 'source']
+            )
+            if (
+                'default_discharge_cms' in hydrotable.columns
+            ):  # check if both "FIM" and "FIM_default" SRCs are available
                 hydrotable['default_discharge_cfs'] = hydrotable.default_discharge_cms * 35.3147
-                limited_hydrotable_default = hydrotable.filter(items=['location_id','elevation_ft', 'default_discharge_cfs','HydroID', 'levpa_id', 'dem_adj_elevation'])
+                limited_hydrotable_default = hydrotable.filter(
+                    items=[
+                        'location_id',
+                        'elevation_ft',
+                        'default_discharge_cfs',
+                        'HydroID',
+                        'levpa_id',
+                        'dem_adj_elevation',
+                    ]
+                )
                 limited_hydrotable_default['discharge_cfs'] = limited_hydrotable_default.default_discharge_cfs
                 limited_hydrotable_default['source'] = "FIM_default"
                 rating_curves = pd.concat([limited_hydrotable, select_usgs_gages])
@@ -142,14 +191,13 @@ def generate_rating_curve_metrics(args):
                 rating_curves = pd.concat([limited_hydrotable, select_usgs_gages])
 
             # Add stream order
-            stream_orders = hydrotable.filter(items=['location_id','order_']).drop_duplicates()
+            stream_orders = hydrotable.filter(items=['location_id', 'order_']).drop_duplicates()
             rating_curves = rating_curves.merge(stream_orders, on='location_id')
-            rating_curves['order_'].fillna(0,inplace=True)
+            rating_curves['order_'].fillna(0, inplace=True)
             rating_curves['order_'] = rating_curves['order_'].astype('int')
 
-
             # NWM recurr intervals
-            recurr_intervals = ("2","5","10","25","50","100")
+            recurr_intervals = ("2", "5", "10", "25", "50", "100")
             recurr_dfs = []
             for interval in recurr_intervals:
                 recurr_file = join(nwm_flow_dir, 'nwm21_17C_recurr_{}_0_cms.csv'.format(interval))
@@ -159,67 +207,90 @@ def generate_rating_curve_metrics(args):
                 recurr_dfs.append(df)
 
             # Merge NWM recurr intervals into a single layer
-            nwm_recurr_intervals_all = reduce(lambda x,y: pd.merge(x,y, on='feature_id', how='outer'), recurr_dfs)
-            nwm_recurr_intervals_all = pd.melt(nwm_recurr_intervals_all, id_vars=['feature_id'], value_vars=recurr_intervals, var_name='recurr_interval', value_name='discharge_cms')
+            nwm_recurr_intervals_all = reduce(
+                lambda x, y: pd.merge(x, y, on='feature_id', how='outer'), recurr_dfs
+            )
+            nwm_recurr_intervals_all = pd.melt(
+                nwm_recurr_intervals_all,
+                id_vars=['feature_id'],
+                value_vars=recurr_intervals,
+                var_name='recurr_interval',
+                value_name='discharge_cms',
+            )
 
             # Append catfim data (already set up in format similar to nwm_recurr_intervals_all)
-            cat_fim = pd.read_csv(catfim_flows_filename, dtype={'feature_id':str})
+            cat_fim = pd.read_csv(catfim_flows_filename, dtype={'feature_id': str})
             nwm_recurr_intervals_all = pd.concat([nwm_recurr_intervals_all, cat_fim])
 
             # Convert discharge to cfs and filter
             nwm_recurr_intervals_all['discharge_cfs'] = nwm_recurr_intervals_all.discharge_cms * 35.3147
-            nwm_recurr_intervals_all = nwm_recurr_intervals_all.filter(items=['discharge_cfs', 'recurr_interval','feature_id']).drop_duplicates()
+            nwm_recurr_intervals_all = nwm_recurr_intervals_all.filter(
+                items=['discharge_cfs', 'recurr_interval', 'feature_id']
+            ).drop_duplicates()
 
             # Identify unique gages
             usgs_crosswalk = hydrotable.filter(items=['location_id', 'feature_id']).drop_duplicates()
             usgs_crosswalk.dropna(subset=['location_id'], inplace=True)
 
             nwm_recurr_data_table = pd.DataFrame()
-            usgs_recurr_data = pd.DataFrame()
+            # usgs_recurr_data = pd.DataFrame()
 
             # Interpolate USGS/FIM elevation at each gage
             for index, gage in usgs_crosswalk.iterrows():
-
                 # Interpolate USGS elevation at NWM recurrence intervals
-                usgs_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="USGS")]
-
-                if len(usgs_rc) <1:
-                    logging.info(f"missing USGS rating curve data for usgs station {gage.location_id} in huc {huc}")
+                usgs_rc = rating_curves.loc[
+                    (rating_curves.location_id == gage.location_id) & (rating_curves.source == "USGS")
+                ]
+
+                if len(usgs_rc) < 1:
+                    logging.info(
+                        f"missing USGS rating curve data for usgs station {gage.location_id} in huc {huc}"
+                    )
                     continue
 
                 str_order = np.unique(usgs_rc.order_).item()
                 feature_id = str(gage.feature_id)
 
-                usgs_pred_elev = get_reccur_intervals(usgs_rc, usgs_crosswalk,nwm_recurr_intervals_all)
+                usgs_pred_elev = get_reccur_intervals(usgs_rc, usgs_crosswalk, nwm_recurr_intervals_all)
 
                 # Handle sites missing data
-                if len(usgs_pred_elev) <1:
-                    logging.info(f"WARNING: missing USGS elevation data for usgs station {gage.location_id} in huc {huc}")
+                if len(usgs_pred_elev) < 1:
+                    logging.info(
+                        f"WARNING: missing USGS elevation data for usgs station {gage.location_id} in huc {huc}"
+                    )
                     continue
 
                 # Clean up data
                 usgs_pred_elev['location_id'] = gage.location_id
-                usgs_pred_elev = usgs_pred_elev.filter(items=['location_id','recurr_interval', 'discharge_cfs','pred_elev'])
+                usgs_pred_elev = usgs_pred_elev.filter(
+                    items=['location_id', 'recurr_interval', 'discharge_cfs', 'pred_elev']
+                )
                 usgs_pred_elev = usgs_pred_elev.rename(columns={"pred_elev": "USGS"})
 
                 # Interpolate FIM elevation at NWM recurrence intervals
-                fim_rc = rating_curves.loc[(rating_curves.location_id==gage.location_id) & (rating_curves.source=="FIM")]
-
-                if len(fim_rc) <1:
-                    logging.info(f"missing FIM rating curve data for usgs station {gage.location_id} in huc {huc}")
+                fim_rc = rating_curves.loc[
+                    (rating_curves.location_id == gage.location_id) & (rating_curves.source == "FIM")
+                ]
+
+                if len(fim_rc) < 1:
+                    logging.info(
+                        f"missing FIM rating curve data for usgs station {gage.location_id} in huc {huc}"
+                    )
                     continue
 
-                fim_pred_elev = get_reccur_intervals(fim_rc, usgs_crosswalk,nwm_recurr_intervals_all)
+                fim_pred_elev = get_reccur_intervals(fim_rc, usgs_crosswalk, nwm_recurr_intervals_all)
 
                 # Handle sites missing data
-                if len(fim_pred_elev) <1:
-                    logging.info(f"WARNING: missing FIM elevation data for usgs station {gage.location_id} in huc {huc}")
+                if len(fim_pred_elev) < 1:
+                    logging.info(
+                        f"WARNING: missing FIM elevation data for usgs station {gage.location_id} in huc {huc}"
+                    )
                     continue
 
                 # Clean up data
                 fim_pred_elev = fim_pred_elev.rename(columns={"pred_elev": "FIM"})
-                fim_pred_elev = fim_pred_elev.filter(items=['recurr_interval', 'discharge_cfs','FIM'])
-                usgs_pred_elev = usgs_pred_elev.merge(fim_pred_elev, on=['recurr_interval','discharge_cfs'])
+                fim_pred_elev = fim_pred_elev.filter(items=['recurr_interval', 'discharge_cfs', 'FIM'])
+                usgs_pred_elev = usgs_pred_elev.merge(fim_pred_elev, on=['recurr_interval', 'discharge_cfs'])
 
                 # Add attributes
                 usgs_pred_elev['HUC'] = huc
@@ -228,7 +299,21 @@ def generate_rating_curve_metrics(args):
                 usgs_pred_elev['feature_id'] = feature_id
 
                 # Melt dataframe
-                usgs_pred_elev = pd.melt(usgs_pred_elev, id_vars=['location_id','feature_id','recurr_interval','discharge_cfs','HUC','HUC4','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
+                usgs_pred_elev = pd.melt(
+                    usgs_pred_elev,
+                    id_vars=[
+                        'location_id',
+                        'feature_id',
+                        'recurr_interval',
+                        'discharge_cfs',
+                        'HUC',
+                        'HUC4',
+                        'str_order',
+                    ],
+                    value_vars=['USGS', 'FIM'],
+                    var_name="source",
+                    value_name='elevation_ft',
+                )
                 nwm_recurr_data_table = pd.concat([nwm_recurr_data_table, usgs_pred_elev])
 
                 # Interpolate FIM elevation at USGS observations
@@ -239,12 +324,24 @@ def generate_rating_curve_metrics(args):
                 # usgs_rc = usgs_rc.sort_values('USGS',ascending=True)
                 #
                 # # Interpolate FIM elevation at USGS observations
-                # usgs_rc['FIM'] = np.interp(usgs_rc.discharge_cfs.values, fim_rc['discharge_cfs'], fim_rc['elevation_ft'], left = np.nan, right = np.nan)
+                # usgs_rc['FIM'] = np.interp(
+                #     usgs_rc.discharge_cfs.values,
+                #     fim_rc['discharge_cfs'],
+                #     fim_rc['elevation_ft'],
+                #     left=np.nan,
+                #     right=np.nan,
+                # )
                 # usgs_rc = usgs_rc[usgs_rc['FIM'].notna()]
                 # usgs_rc = usgs_rc.drop(columns=["source"])
                 #
                 # # Melt dataframe
-                # usgs_rc = pd.melt(usgs_rc, id_vars=['location_id','discharge_cfs','str_order'], value_vars=['USGS','FIM'], var_name="source", value_name='elevation_ft')
+                # usgs_rc = pd.melt(
+                #     usgs_rc,
+                #     id_vars=['location_id', 'discharge_cfs', 'str_order'],
+                #     value_vars=['USGS', 'FIM'],
+                #     var_name="source",
+                #     value_name='elevation_ft',
+                # )
                 #
                 # if not usgs_rc.empty:
                 #     usgs_recurr_data = pd.concat([usgs_recurr_data, usgs_rc])
@@ -255,19 +352,26 @@ def generate_rating_curve_metrics(args):
             #     usgs_recurr_stats_table.to_csv(usgs_recurr_stats_filename,index=False)
 
             # # Generate plots (not currently being used)
-            # fim_elev_at_USGS_rc_plot_filename = join(dirname(rc_comparison_plot_filename),'FIM_elevations_at_USGS_rc_' + str(huc) +'.png')
+            # fim_elev_at_USGS_rc_plot_filename = join(
+            #     dirname(rc_comparison_plot_filename), 'FIM_elevations_at_USGS_rc_' + str(huc) + '.png'
+            # )
             # generate_facet_plot(usgs_recurr_data, fim_elev_at_USGS_rc_plot_filename)
 
             if not nwm_recurr_data_table.empty:
-                nwm_recurr_data_table.discharge_cfs = np.round(nwm_recurr_data_table.discharge_cfs,2)
-                nwm_recurr_data_table.elevation_ft = np.round(nwm_recurr_data_table.elevation_ft,2)
-                nwm_recurr_data_table.to_csv(nwm_recurr_data_filename,index=False)
+                nwm_recurr_data_table.discharge_cfs = np.round(nwm_recurr_data_table.discharge_cfs, 2)
+                nwm_recurr_data_table.elevation_ft = np.round(nwm_recurr_data_table.elevation_ft, 2)
+                nwm_recurr_data_table.to_csv(nwm_recurr_data_filename, index=False)
             if 'location_id' not in nwm_recurr_data_table.columns:
-                logging.info(f"WARNING: nwm_recurr_data_table is missing location_id column for gage {relevant_gages} in huc {huc}")
+                logging.info(
+                    f"WARNING: nwm_recurr_data_table is missing location_id column for gage {relevant_gages} "
+                    f"in huc {huc}"
+                )
 
             # plot rating curves
             if alt_plot:
-                generate_rc_and_rem_plots(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table, branches_folder)
+                generate_rc_and_rem_plots(
+                    rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table, branches_folder
+                )
             elif single_plot:
                 generate_single_plot(rating_curves, rc_comparison_plot_filename, nwm_recurr_data_table)
             else:
@@ -277,15 +381,17 @@ def generate_rating_curve_metrics(args):
     else:
         logging.info(f"no valid USGS gages found in huc {huc} (note: may be ahps sites without UGSG gages)")
 
-def aggregate_metrics(output_dir,procs_list,stat_groups):
 
+def aggregate_metrics(output_dir, procs_list, stat_groups):
     # Default stat group to location_id
     if stat_groups is None:
         stat_groups = ['location_id']
 
     # agg_usgs_interp_elev_stats = join(output_dir,'agg_usgs_interp_elev_stats.csv')
-    agg_nwm_recurr_flow_elev = join(output_dir,'agg_nwm_recurr_flow_elevations.csv')
-    agg_nwm_recurr_flow_elev_stats = join(output_dir,f"agg_nwm_recurr_flow_elev_stats_{'_'.join(stat_groups)}.csv")
+    agg_nwm_recurr_flow_elev = join(output_dir, 'agg_nwm_recurr_flow_elevations.csv')
+    agg_nwm_recurr_flow_elev_stats = join(
+        output_dir, f"agg_nwm_recurr_flow_elev_stats_{'_'.join(stat_groups)}.csv"
+    )
 
     # if os.path.isfile(agg_usgs_interp_elev_stats):
     #     os.remove(agg_usgs_interp_elev_stats)
@@ -304,60 +410,96 @@ def aggregate_metrics(output_dir,procs_list,stat_groups):
         #     else:
         #         usgs_recurr_stats.to_csv(agg_usgs_interp_elev_stats,index=False)
         if os.path.isfile(huc[4]):
-            nwm_recurr_data = pd.read_csv(huc[4],dtype={'location_id': str,
-                                                        'feature_id': str})
+            nwm_recurr_data = pd.read_csv(huc[4], dtype={'location_id': str, 'feature_id': str})
 
             # Write/append nwm_recurr_data
             if os.path.isfile(agg_nwm_recurr_flow_elev):
-                nwm_recurr_data.to_csv(agg_nwm_recurr_flow_elev,index=False, mode='a',header=False)
+                nwm_recurr_data.to_csv(agg_nwm_recurr_flow_elev, index=False, mode='a', header=False)
             else:
-                nwm_recurr_data.to_csv(agg_nwm_recurr_flow_elev,index=False)
+                nwm_recurr_data.to_csv(agg_nwm_recurr_flow_elev, index=False)
 
-    agg_stats = pd.read_csv(agg_nwm_recurr_flow_elev,dtype={'location_id': str,
-                                                            'feature_id': str})
+    agg_stats = pd.read_csv(agg_nwm_recurr_flow_elev, dtype={'location_id': str, 'feature_id': str})
 
-    agg_recurr_stats_table = calculate_rc_stats_elev(agg_stats,stat_groups)
+    agg_recurr_stats_table = calculate_rc_stats_elev(agg_stats, stat_groups)
 
-    agg_recurr_stats_table.to_csv(agg_nwm_recurr_flow_elev_stats,index=False)
+    agg_recurr_stats_table.to_csv(agg_nwm_recurr_flow_elev_stats, index=False)
 
     return agg_recurr_stats_table
 
 
 def generate_single_plot(rc, plot_filename, recurr_data_table):
-
     tmp_rc = rc.copy()
-    
+
     # Filter FIM elevation based on USGS data
     for gage in rc.location_id.unique():
-        rc = rc[rc.location_id==gage]
+        rc = rc[rc.location_id == gage]
 
         plot_filename_splitext = os.path.splitext(plot_filename)
-        gage_plot_filename = plot_filename_splitext[0] + '_' + gage +  plot_filename_splitext[1]
+        gage_plot_filename = plot_filename_splitext[0] + '_' + gage + plot_filename_splitext[1]
 
-        #print(recurr_data_table.head)
+        # print(recurr_data_table.head)
         try:
-            min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
-            max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
-            min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
-            max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
-            ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
-
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
-
-            if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
-                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
+            min_elev = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].elevation_ft.min()
+            max_elev = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].elevation_ft.max()
+            min_q = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].discharge_cfs.min()
+            max_q = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].discharge_cfs.max()
+            ri100 = recurr_data_table[
+                (recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')
+            ].discharge_cfs.max()
+
+            rc = rc.drop(
+                rc[
+                    (rc.location_id == gage)
+                    & (rc.source == 'FIM')
+                    & (
+                        ((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100))
+                        & (rc.discharge_cfs > max_q)
+                    )
+                ].index
+            )
+            rc = rc.drop(
+                rc[
+                    (rc.location_id == gage)
+                    & (rc.source == 'FIM')
+                    & (rc.elevation_ft < min_elev - 2)
+                    & (rc.discharge_cfs < min_q)
+                ].index
+            )
+
+            if 'default_discharge_cfs' in rc.columns:  # Plot both "FIM" and "FIM_default" rating curves
+                rc = rc.drop(
+                    rc[
+                        (rc.location_id == gage)
+                        & (rc.source == 'FIM_default')
+                        & (
+                            ((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100))
+                            & (rc.discharge_cfs > max_q)
+                        )
+                    ].index
+                )
+                rc = rc.drop(
+                    rc[
+                        (rc.location_id == gage)
+                        & (rc.source == 'FIM_default')
+                        & (rc.elevation_ft < min_elev - 2)
+                    ].index
+                )
         except Exception as ex:
-            summary = traceback.StackSummary.extract(
-                    traceback.walk_stack(None))
+            summary = traceback.StackSummary.extract(traceback.walk_stack(None))
             logging.info("WARNING: rating curve dataframe not processed correctly...")
+            logging.info(f'Summary: {summary} \n Exception: \n {repr(ex)}')
 
         rc = rc.rename(columns={"location_id": "USGS Gage"})
 
         # split out branch 0 FIM data
-        rc['source_branch'] = np.where((rc.source == 'FIM') & (rc.levpa_id == '0'), 'FIM_b0', np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source))
-        #rc['source_branch'] = np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source)
+        rc['source_branch'] = np.where(
+            (rc.source == 'FIM') & (rc.levpa_id == '0'),
+            'FIM_b0',
+            np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source),
+        )
+        # rc['source_branch'] = np.where(
+        #     (rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source
+        # )
 
         ## Generate rating curve plots
         num_plots = len(rc["USGS Gage"].unique())
@@ -369,16 +511,42 @@ def generate_single_plot(rc, plot_filename, recurr_data_table):
         sns.set(style="ticks")
 
         # Plot both "FIM" and "FIM_default" rating curves
-        if '0' in rc.levpa_id.values: # checks to see if branch zero data exists in the rating curve df
-            hue_order = ['USGS','FIM','FIM_default','FIM_b0','FIM_default_b0'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM','FIM_b0']
-            kw = {'color': ['blue','green','orange','green','orange'], 'linestyle' : ["-","-","-","--","--"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green','green'], 'linestyle' : ["-","-","--"]}
+        if '0' in rc.levpa_id.values:  # checks to see if branch zero data exists in the rating curve df
+            hue_order = (
+                ['USGS', 'FIM', 'FIM_default', 'FIM_b0', 'FIM_default_b0']
+                if 'default_discharge_cfs' in rc.columns
+                else ['USGS', 'FIM', 'FIM_b0']
+            )
+            kw = (
+                {
+                    'color': ['blue', 'green', 'orange', 'green', 'orange'],
+                    'linestyle': ["-", "-", "-", "--", "--"],
+                }
+                if 'default_discharge_cfs' in rc.columns
+                else {'color': ['blue', 'green', 'green'], 'linestyle': ["-", "-", "--"]}
+            )
         else:
-            hue_order = ['USGS','FIM','FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM']
-            kw = {'color': ['blue','green','orange'], 'linestyle' : ["-","-","-"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green'], 'linestyle' : ["-","_"]}
+            hue_order = (
+                ['USGS', 'FIM', 'FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS', 'FIM']
+            )
+            kw = (
+                {'color': ['blue', 'green', 'orange'], 'linestyle': ["-", "-", "-"]}
+                if 'default_discharge_cfs' in rc.columns
+                else {'color': ['blue', 'green'], 'linestyle': ["-", "_"]}
+            )
         # Facet Grid
-        g = sns.FacetGrid(rc, col="USGS Gage", hue="source_branch", hue_order=hue_order,
-                        sharex=False, sharey=False,col_wrap=columns,
-                        height=3.5, aspect=1.65, hue_kws=kw)
+        g = sns.FacetGrid(
+            rc,
+            col="USGS Gage",
+            hue="source_branch",
+            hue_order=hue_order,
+            sharex=False,
+            sharey=False,
+            col_wrap=columns,
+            height=3.5,
+            aspect=1.65,
+            hue_kws=kw,
+        )
         g.map(plt.plot, "discharge_cfs", "elevation_ft", linewidth=2, alpha=0.8)
         g.set_axis_labels(x_var="Discharge (cfs)", y_var="Elevation (ft)")
 
@@ -388,17 +556,27 @@ def generate_single_plot(rc, plot_filename, recurr_data_table):
             ax = axes[gage]
             plt.sca(ax)
             try:
-                recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
-                    .filter(items=['recurr_interval', 'discharge_cfs'])
+                recurr_data = recurr_data_table[
+                    (recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')
+                ].filter(items=['recurr_interval', 'discharge_cfs'])
                 for i, r in recurr_data.iterrows():
-                    if not r.recurr_interval.isnumeric(): continue # skip catfim flows
-                    l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
-                    plt.axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
-                    plt.text(r.discharge_cfs, ax.get_ylim()[1] - (ax.get_ylim()[1]-ax.get_ylim()[0])*0.03, r.recurr_interval, size='small', c='purple')
+                    if not r.recurr_interval.isnumeric():
+                        continue  # skip catfim flows
+                    label = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None  # only label 2 yr
+                    plt.axvline(
+                        x=r.discharge_cfs, c='purple', linewidth=0.5, label=label
+                    )  # plot recurrence intervals
+                    plt.text(
+                        r.discharge_cfs,
+                        ax.get_ylim()[1] - (ax.get_ylim()[1] - ax.get_ylim()[0]) * 0.03,
+                        r.recurr_interval,
+                        size='small',
+                        c='purple',
+                    )
             except Exception as ex:
-                summary = traceback.StackSummary.extract(
-                        traceback.walk_stack(None))
+                summary = traceback.StackSummary.extract(traceback.walk_stack(None))
                 logging.info("WARNING: Could not plot recurrence intervals...")
+                logging.info(f'Summary: {summary} \n Exception: \n {repr(ex)}')
 
         # Adjust the arrangement of the plots
         g.fig.tight_layout(w_pad=1)
@@ -409,34 +587,73 @@ def generate_single_plot(rc, plot_filename, recurr_data_table):
 
         rc = tmp_rc
 
+
 def generate_facet_plot(rc, plot_filename, recurr_data_table):
-    
     # Filter FIM elevation based on USGS data
     for gage in rc.location_id.unique():
-        #print(recurr_data_table.head)
+        # print(recurr_data_table.head)
         try:
-            min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
-            max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
-            min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
-            max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
-            ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
-
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
-
-            if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
-                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-                rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
+            min_elev = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].elevation_ft.min()
+            max_elev = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].elevation_ft.max()
+            min_q = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].discharge_cfs.min()
+            max_q = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].discharge_cfs.max()
+            ri100 = recurr_data_table[
+                (recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')
+            ].discharge_cfs.max()
+
+            rc = rc.drop(
+                rc[
+                    (rc.location_id == gage)
+                    & (rc.source == 'FIM')
+                    & (
+                        ((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100))
+                        & (rc.discharge_cfs > max_q)
+                    )
+                ].index
+            )
+            rc = rc.drop(
+                rc[
+                    (rc.location_id == gage)
+                    & (rc.source == 'FIM')
+                    & (rc.elevation_ft < min_elev - 2)
+                    & (rc.discharge_cfs < min_q)
+                ].index
+            )
+
+            if 'default_discharge_cfs' in rc.columns:  # Plot both "FIM" and "FIM_default" rating curves
+                rc = rc.drop(
+                    rc[
+                        (rc.location_id == gage)
+                        & (rc.source == 'FIM_default')
+                        & (
+                            ((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100))
+                            & (rc.discharge_cfs > max_q)
+                        )
+                    ].index
+                )
+                rc = rc.drop(
+                    rc[
+                        (rc.location_id == gage)
+                        & (rc.source == 'FIM_default')
+                        & (rc.elevation_ft < min_elev - 2)
+                    ].index
+                )
         except Exception as ex:
-            summary = traceback.StackSummary.extract(
-                    traceback.walk_stack(None))
+            summary = traceback.StackSummary.extract(traceback.walk_stack(None))
             logging.info("WARNING: rating curve dataframe not processed correctly...")
+            logging.info(f'Summary: {summary} \n Exception: \n {repr(ex)}')
 
     rc = rc.rename(columns={"location_id": "USGS Gage"})
 
     # split out branch 0 FIM data
-    rc['source_branch'] = np.where((rc.source == 'FIM') & (rc.levpa_id == '0'), 'FIM_b0', np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source))
-    #rc['source_branch'] = np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source)
+    rc['source_branch'] = np.where(
+        (rc.source == 'FIM') & (rc.levpa_id == '0'),
+        'FIM_b0',
+        np.where((rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source),
+    )
+    # rc['source_branch'] = np.where(
+    #     (rc.source == 'FIM_default') & (rc.levpa_id == '0'), 'FIM_default_b0', rc.source
+    # )
 
     ## Generate rating curve plots
     num_plots = len(rc["USGS Gage"].unique())
@@ -448,16 +665,42 @@ def generate_facet_plot(rc, plot_filename, recurr_data_table):
     sns.set(style="ticks")
 
     # Plot both "FIM" and "FIM_default" rating curves
-    if '0' in rc.levpa_id.values: # checks to see if branch zero data exists in the rating curve df
-        hue_order = ['USGS','FIM','FIM_default','FIM_b0','FIM_default_b0'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM','FIM_b0']
-        kw = {'color': ['blue','green','orange','green','orange'], 'linestyle' : ["-","-","-","--","--"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green','green'], 'linestyle' : ["-","-","--"]}
+    if '0' in rc.levpa_id.values:  # checks to see if branch zero data exists in the rating curve df
+        hue_order = (
+            ['USGS', 'FIM', 'FIM_default', 'FIM_b0', 'FIM_default_b0']
+            if 'default_discharge_cfs' in rc.columns
+            else ['USGS', 'FIM', 'FIM_b0']
+        )
+        kw = (
+            {
+                'color': ['blue', 'green', 'orange', 'green', 'orange'],
+                'linestyle': ["-", "-", "-", "--", "--"],
+            }
+            if 'default_discharge_cfs' in rc.columns
+            else {'color': ['blue', 'green', 'green'], 'linestyle': ["-", "-", "--"]}
+        )
     else:
-        hue_order = ['USGS','FIM','FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS','FIM']
-        kw = {'color': ['blue','green','orange'], 'linestyle' : ["-","-","-"]} if 'default_discharge_cfs' in rc.columns else {'color': ['blue','green'], 'linestyle' : ["-","_"]}
+        hue_order = (
+            ['USGS', 'FIM', 'FIM_default'] if 'default_discharge_cfs' in rc.columns else ['USGS', 'FIM']
+        )
+        kw = (
+            {'color': ['blue', 'green', 'orange'], 'linestyle': ["-", "-", "-"]}
+            if 'default_discharge_cfs' in rc.columns
+            else {'color': ['blue', 'green'], 'linestyle': ["-", "_"]}
+        )
     # Facet Grid
-    g = sns.FacetGrid(rc, col="USGS Gage", hue="source_branch", hue_order=hue_order,
-                    sharex=False, sharey=False,col_wrap=columns,
-                    height=3.5, aspect=1.65, hue_kws=kw)
+    g = sns.FacetGrid(
+        rc,
+        col="USGS Gage",
+        hue="source_branch",
+        hue_order=hue_order,
+        sharex=False,
+        sharey=False,
+        col_wrap=columns,
+        height=3.5,
+        aspect=1.65,
+        hue_kws=kw,
+    )
     g.map(plt.plot, "discharge_cfs", "elevation_ft", linewidth=2, alpha=0.8)
     g.set_axis_labels(x_var="Discharge (cfs)", y_var="Elevation (ft)")
 
@@ -467,17 +710,27 @@ def generate_facet_plot(rc, plot_filename, recurr_data_table):
         ax = axes[gage]
         plt.sca(ax)
         try:
-            recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
-                .filter(items=['recurr_interval', 'discharge_cfs'])
+            recurr_data = recurr_data_table[
+                (recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')
+            ].filter(items=['recurr_interval', 'discharge_cfs'])
             for i, r in recurr_data.iterrows():
-                if not r.recurr_interval.isnumeric(): continue # skip catfim flows
-                l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
-                plt.axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
-                plt.text(r.discharge_cfs, ax.get_ylim()[1] - (ax.get_ylim()[1]-ax.get_ylim()[0])*0.03, r.recurr_interval, size='small', c='purple')
+                if not r.recurr_interval.isnumeric():
+                    continue  # skip catfim flows
+                label = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None  # only label 2 yr
+                plt.axvline(
+                    x=r.discharge_cfs, c='purple', linewidth=0.5, label=label
+                )  # plot recurrence intervals
+                plt.text(
+                    r.discharge_cfs,
+                    ax.get_ylim()[1] - (ax.get_ylim()[1] - ax.get_ylim()[0]) * 0.03,
+                    r.recurr_interval,
+                    size='small',
+                    c='purple',
+                )
         except Exception as ex:
-            summary = traceback.StackSummary.extract(
-                    traceback.walk_stack(None))
+            summary = traceback.StackSummary.extract(traceback.walk_stack(None))
             logging.info("WARNING: Could not plot recurrence intervals...")
+            logging.info(f'Summary: {summary} \n Exception: \n {repr(ex)}')
 
     # Adjust the arrangement of the plots
     g.fig.tight_layout(w_pad=1)
@@ -486,14 +739,14 @@ def generate_facet_plot(rc, plot_filename, recurr_data_table):
     plt.savefig(plot_filename)
     plt.close()
 
-def generate_rc_and_rem_plots(rc, plot_filename, recurr_data_table, branches_folder):
 
+def generate_rc_and_rem_plots(rc, plot_filename, recurr_data_table, branches_folder):
     ## Set up figure
     num_plots = len(rc["location_id"].unique())
-    fig = plt.figure(figsize=(6, 2.4*num_plots))
+    fig = plt.figure(figsize=(6, 2.4 * num_plots))
     gs = fig.add_gridspec(num_plots, 2, width_ratios=[2, 3])
     ax = gs.subplots()
-    if ax.ndim == 1:                     # hucs with only one plot will only have one-dimensional axes;
+    if ax.ndim == 1:  # hucs with only one plot will only have one-dimensional axes;
         ax = np.expand_dims(ax, axis=0)  # the axes manipulations below require 2 dimensions
     plt.tight_layout(w_pad=1)
 
@@ -501,54 +754,132 @@ def generate_rc_and_rem_plots(rc, plot_filename, recurr_data_table, branches_fol
     gage_branch_dict = rc.groupby('location_id')['levpa_id'].first().to_dict()
 
     for i, gage in enumerate(gage_branch_dict):
-    ###################################################################################################################
-    # Filter FIM elevation based on USGS data
-
-        min_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.min()
-        max_elev = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].elevation_ft.max()
-        min_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.min()
-        max_q = rc.loc[(rc.location_id==gage) & (rc.source=='USGS')].discharge_cfs.max()
-        ri100 = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')].discharge_cfs.max()
-
-        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-        rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM') & (rc.elevation_ft < min_elev - 2) & (rc.discharge_cfs < min_q)].index)
-
-        if 'default_discharge_cfs' in rc.columns: # Plot both "FIM" and "FIM_default" rating curves
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100)) & (rc.discharge_cfs > max_q))].index)
-            rc = rc.drop(rc[(rc.location_id==gage) & (rc.source=='FIM_default') & (rc.elevation_ft < min_elev - 2)].index)
-
-    ###################################################################################################################
+        #####################################################################################################
+        # Filter FIM elevation based on USGS data
+
+        min_elev = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].elevation_ft.min()
+        max_elev = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].elevation_ft.max()
+        min_q = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].discharge_cfs.min()
+        max_q = rc.loc[(rc.location_id == gage) & (rc.source == 'USGS')].discharge_cfs.max()
+        ri100 = recurr_data_table[
+            (recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')
+        ].discharge_cfs.max()
+
+        rc = rc.drop(
+            rc[
+                (rc.location_id == gage)
+                & (rc.source == 'FIM')
+                & (
+                    ((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100))
+                    & (rc.discharge_cfs > max_q)
+                )
+            ].index
+        )
+        rc = rc.drop(
+            rc[
+                (rc.location_id == gage)
+                & (rc.source == 'FIM')
+                & (rc.elevation_ft < min_elev - 2)
+                & (rc.discharge_cfs < min_q)
+            ].index
+        )
+
+        if 'default_discharge_cfs' in rc.columns:  # Plot both "FIM" and "FIM_default" rating curves
+            rc = rc.drop(
+                rc[
+                    (rc.location_id == gage)
+                    & (rc.source == 'FIM_default')
+                    & (
+                        ((rc.elevation_ft > (max_elev + 2)) | (rc.discharge_cfs > ri100))
+                        & (rc.discharge_cfs > max_q)
+                    )
+                ].index
+            )
+            rc = rc.drop(
+                rc[
+                    (rc.location_id == gage) & (rc.source == 'FIM_default') & (rc.elevation_ft < min_elev - 2)
+                ].index
+            )
+
+        ######################################################################################################
         ## Read in reaches, catchment raster, and rem raster
         branch = gage_branch_dict[gage]
-        if os.path.isfile(os.path.join(branches_folder, branch, f'demDerived_reaches_split_filtered_addedAttributes_crosswalked_{branch}.gpkg')):
-            reaches = gpd.read_file(os.path.join(branches_folder, branch, f'demDerived_reaches_split_filtered_addedAttributes_crosswalked_{branch}.gpkg'))
+        if os.path.isfile(
+            os.path.join(
+                branches_folder,
+                branch,
+                f'demDerived_reaches_split_filtered_addedAttributes_crosswalked_{branch}.gpkg',
+            )
+        ):
+            reaches = gpd.read_file(
+                os.path.join(
+                    branches_folder,
+                    branch,
+                    f'demDerived_reaches_split_filtered_addedAttributes_crosswalked_{branch}.gpkg',
+                )
+            )
             reach = reaches[reaches.HydroID == hydroid]
-        with rasterio.open(os.path.join(branches_folder, branch, f'gw_catchments_reaches_filtered_addedAttributes_{branch}.tif')) as catch_rast:
+        with rasterio.open(
+            os.path.join(
+                branches_folder, branch, f'gw_catchments_reaches_filtered_addedAttributes_{branch}.tif'
+            )
+        ) as catch_rast:
             catchments = catch_rast.read()
         with rasterio.open(os.path.join(branches_folder, branch, f'rem_zeroed_masked_{branch}.tif')) as rem:
             rem_transform = rem.transform
             rem_extent = rioplot.plotting_extent(rem)
             rem_sub25 = rem.read()
             # Set all pixels above the SRC calculation height to nan
-            rem_sub25[np.where(rem_sub25 > 25.3)] = -9999.
-            rem_sub25[np.where(rem_sub25 == -9999.)] = np.nan
-        
+            rem_sub25[np.where(rem_sub25 > 25.3)] = -9999.0
+            rem_sub25[np.where(rem_sub25 == -9999.0)] = np.nan
+
         # Plot the rating curve
-        ax[i,1].plot("discharge_cfs", "elevation_ft", data=rc[(rc.source == 'USGS') & (rc.location_id == gage)], linewidth=2, alpha=0.8, label='USGS')
-        ax[i,1].plot("discharge_cfs", "elevation_ft", data=rc[(rc.source == 'FIM') & (rc.location_id == gage)], linewidth=2, alpha=0.8, label='FIM')
+        ax[i, 1].plot(
+            "discharge_cfs",
+            "elevation_ft",
+            data=rc[(rc.source == 'USGS') & (rc.location_id == gage)],
+            linewidth=2,
+            alpha=0.8,
+            label='USGS',
+        )
+        ax[i, 1].plot(
+            "discharge_cfs",
+            "elevation_ft",
+            data=rc[(rc.source == 'FIM') & (rc.location_id == gage)],
+            linewidth=2,
+            alpha=0.8,
+            label='FIM',
+        )
         if 'default_discharge_cfs' in rc.columns:
-            ax[i,1].plot("default_discharge_cfs", "elevation_ft", data=rc[(rc.source == 'FIM_default') & (rc.location_id == gage)], linewidth=2, alpha=0.8, label='FIM_default')
-        
+            ax[i, 1].plot(
+                "default_discharge_cfs",
+                "elevation_ft",
+                data=rc[(rc.source == 'FIM_default') & (rc.location_id == gage)],
+                linewidth=2,
+                alpha=0.8,
+                label='FIM_default',
+            )
+
         # Plot the recurrence intervals
-        recurr_data = recurr_data_table[(recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')]\
-            .filter(items=['recurr_interval', 'discharge_cfs'])
+        recurr_data = recurr_data_table[
+            (recurr_data_table.location_id == gage) & (recurr_data_table.source == 'FIM')
+        ].filter(items=['recurr_interval', 'discharge_cfs'])
         for _, r in recurr_data.iterrows():
-            if not r.recurr_interval.isnumeric(): continue # skip catfim flows
-            l = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None # only label 2 yr 
-            ax[i,1].axvline(x=r.discharge_cfs, c='purple', linewidth=0.5, label=l) # plot recurrence intervals
-            ax[i,1].text(r.discharge_cfs, ax[i,1].get_ylim()[1] - (ax[i,1].get_ylim()[1]-ax[i,1].get_ylim()[0])*0.06, r.recurr_interval, size='small', c='purple')
-
-        # Get the hydroid    
+            if not r.recurr_interval.isnumeric():
+                continue  # skip catfim flows
+            label = 'NWM 17C\nRecurrence' if r.recurr_interval == '2' else None  # only label 2 yr
+            ax[i, 1].axvline(
+                x=r.discharge_cfs, c='purple', linewidth=0.5, label=label
+            )  # plot recurrence intervals
+            ax[i, 1].text(
+                r.discharge_cfs,
+                ax[i, 1].get_ylim()[1] - (ax[i, 1].get_ylim()[1] - ax[i, 1].get_ylim()[0]) * 0.06,
+                r.recurr_interval,
+                size='small',
+                c='purple',
+            )
+
+        # Get the hydroid
         hydroid = rc[rc.location_id == gage].HydroID.unique()[0]
         if not hydroid:
             logging.info(f'Gage {gage} in HUC {branch} has no HydroID')
@@ -562,76 +893,97 @@ def generate_rc_and_rem_plots(rc, plot_filename, recurr_data_table, branches_fol
         dem_adj_elevation = rc[rc.location_id == gage].dem_adj_elevation.unique()[0]
         catchment_rem = (catchment_rem + dem_adj_elevation) * 3.28084
         max_elev = rc[(rc.source == 'FIM') & (rc.location_id == gage)].elevation_ft.max()
-        catchment_rem[np.where(catchment_rem > max_elev)] = np.nan         # <-- Comment out this line to get the full raster that is
-                                                                           # used in rating curve creation
-        # Create polygon for perimeter/area stats                          
+        catchment_rem[
+            np.where(catchment_rem > max_elev)
+        ] = np.nan  # <-- Comment out this line to get the full raster that is
+        # used in rating curve creation
+        # Create polygon for perimeter/area stats
         catchment_rem_1s = catchment_rem.copy()
         catchment_rem_1s[np.where(~np.isnan(catchment_rem_1s))] = 1
-        features = riofeatures.shapes(catchment_rem_1s, mask=~np.isnan(catchment_rem), transform=rem_transform, connectivity=8)
+        features = riofeatures.shapes(
+            catchment_rem_1s, mask=~np.isnan(catchment_rem), transform=rem_transform, connectivity=8
+        )
         del catchment_rem_1s
         features = [f for f in features]
         geom = [Polygon(f[0]['coordinates'][0]) for f in features]
-        poly = gpd.GeoDataFrame({'geometry':geom})
-        #poly['perimeter'] = poly.length                                        # These lines are calculating perimeter/area stats
-        #poly['area'] = poly.area                                               # and can be removed if there is a separate process
-        #poly['perimeter_area_ratio'] = poly.length/poly.area                   # set up later that calculates these for all hydroids
-        #poly['perimeter_area_ratio_sqrt'] = poly.length/(poly.area**.5)        # within a catchment.
+        poly = gpd.GeoDataFrame({'geometry': geom})
+        # These lines are calculating perimeter/area stats and can be removed if there is
+        # a separate process set up later that calculates these for all hydroids within a catchment.
+        # poly['perimeter'] = poly.length
+        # poly['area'] = poly.area
+        # # poly['perimeter_area_ratio'] = poly.length/poly.area
+        # poly['perimeter_area_ratio_sqrt'] = poly.length/(poly.area**.5)
         bounds = poly.total_bounds
-        bounds = ((bounds[0]-20, bounds[2]+20), (bounds[1]-20, bounds[3]+20))
-        
+        bounds = ((bounds[0] - 20, bounds[2] + 20), (bounds[1] - 20, bounds[3] + 20))
+
         # REM plot
         if 'reach' in locals():
-            reach.plot(ax=ax[i,0], color='#999999', linewidth=0.9)
-        im = ax[i,0].imshow(rasterio.plot.reshape_as_image(catchment_rem), cmap='gnuplot', extent=rem_extent, interpolation='none')
-        plt.colorbar(im, ax=ax[i,0], location='left')
-        ax[i,0].set_xbound(bounds[0]); ax[i,0].set_ybound(bounds[1])
-        ax[i,0].set_xticks([]); ax[i,0].set_yticks([])
-        ax[i,0].set_title(gage)
+            reach.plot(ax=ax[i, 0], color='#999999', linewidth=0.9)
+        im = ax[i, 0].imshow(
+            rasterio.plot.reshape_as_image(catchment_rem),
+            cmap='gnuplot',
+            extent=rem_extent,
+            interpolation='none',
+        )
+        plt.colorbar(im, ax=ax[i, 0], location='left')
+        ax[i, 0].set_xbound(bounds[0])
+        ax[i, 0].set_ybound(bounds[1])
+        ax[i, 0].set_xticks([])
+        ax[i, 0].set_yticks([])
+        ax[i, 0].set_title(gage)
 
     del catchments, rem_sub25, catchment_rem
-    ax[0,1].legend()
+    ax[0, 1].legend()
     plt.savefig(plot_filename, dpi=200)
     plt.close()
 
-def get_reccur_intervals(site_rc, usgs_crosswalk,nwm_recurr_intervals):
 
+def get_reccur_intervals(site_rc, usgs_crosswalk, nwm_recurr_intervals):
     usgs_site = site_rc.merge(usgs_crosswalk, on="location_id")
     nwm_ids = len(usgs_site.feature_id.drop_duplicates())
 
     if nwm_ids > 0:
         try:
-            nwm_recurr_intervals = nwm_recurr_intervals.copy().loc[nwm_recurr_intervals.feature_id==usgs_site.feature_id.drop_duplicates().item()]
-            nwm_recurr_intervals['pred_elev'] = np.interp(nwm_recurr_intervals.discharge_cfs.values, usgs_site['discharge_cfs'], usgs_site['elevation_ft'], left = np.nan, right = np.nan)
+            nwm_recurr_intervals = nwm_recurr_intervals.copy().loc[
+                nwm_recurr_intervals.feature_id == usgs_site.feature_id.drop_duplicates().item()
+            ]
+            nwm_recurr_intervals['pred_elev'] = np.interp(
+                nwm_recurr_intervals.discharge_cfs.values,
+                usgs_site['discharge_cfs'],
+                usgs_site['elevation_ft'],
+                left=np.nan,
+                right=np.nan,
+            )
 
             return nwm_recurr_intervals
         except Exception as ex:
-            summary = traceback.StackSummary.extract(
-                    traceback.walk_stack(None))
-            #logging.info("WARNING: get_recurr_intervals failed for some reason....")
-            #logging.info(f"*** {ex}")                
-            #logging.info(''.join(summary.format()))
+            summary = traceback.StackSummary.extract(traceback.walk_stack(None))
+            # logging.info("WARNING: get_recurr_intervals failed for some reason....")
+            # logging.info(f"*** {ex}")
+            # logging.info(''.join(summary.format()))
+            print(summary, repr(ex))
             return []
 
     else:
         return []
 
 
-def calculate_rc_stats_elev(rc,stat_groups=None):
-
+def calculate_rc_stats_elev(rc, stat_groups=None):
     usgs_elev = "USGS"
     src_elev = "FIM"
 
     # Collect any extra columns not associated with melt
     col_index = list(rc.columns)
-    pivot_vars = ['source','elevation_ft']
+    pivot_vars = ['source', 'elevation_ft']
     col_index = [col for col in col_index if col not in pivot_vars]
 
     # Unmelt elevation/source
-    rc_unmelt = (rc.set_index(col_index)
+    rc_unmelt = (
+        rc.set_index(col_index)
         .pivot(columns="source")['elevation_ft']
         .reset_index()
         .rename_axis(None, axis=1)
-     )
+    )
 
     # Calculate variables for NRMSE
     rc_unmelt["yhat_minus_y"] = rc_unmelt[src_elev] - rc_unmelt[usgs_elev]
@@ -641,49 +993,75 @@ def calculate_rc_stats_elev(rc,stat_groups=None):
     station_rc = rc_unmelt.groupby(stat_groups)
 
     # Calculate variables for NRMSE
-    sum_y_diff = station_rc.apply(lambda x: x["yhat_minus_y_squared"].sum())\
-        .reset_index(stat_groups, drop = False).rename({0: "sum_y_diff"}, axis=1)
+    sum_y_diff = (
+        station_rc.apply(lambda x: x["yhat_minus_y_squared"].sum())
+        .reset_index(stat_groups, drop=False)
+        .rename({0: "sum_y_diff"}, axis=1)
+    )
 
     # Determine number of events that are modeled
-    n = station_rc.apply(lambda x: x[usgs_elev].count())\
-        .reset_index(stat_groups, drop = False).rename({0: "n"}, axis=1)
+    n = (
+        station_rc.apply(lambda x: x[usgs_elev].count())
+        .reset_index(stat_groups, drop=False)
+        .rename({0: "n"}, axis=1)
+    )
 
     # Determine the maximum/minimum USGS elevation
-    y_max = station_rc.apply(lambda x: x[usgs_elev].max())\
-        .reset_index(stat_groups, drop = False).rename({0: "y_max"}, axis=1)
-    y_min = station_rc.apply(lambda x: x[usgs_elev].min())\
-        .reset_index(stat_groups, drop = False).rename({0: "y_min"}, axis=1)
+    y_max = (
+        station_rc.apply(lambda x: x[usgs_elev].max())
+        .reset_index(stat_groups, drop=False)
+        .rename({0: "y_max"}, axis=1)
+    )
+    y_min = (
+        station_rc.apply(lambda x: x[usgs_elev].min())
+        .reset_index(stat_groups, drop=False)
+        .rename({0: "y_min"}, axis=1)
+    )
 
     # Collect variables for NRMSE
-    nrmse_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [sum_y_diff, n, y_max, y_min])
+    nrmse_table = reduce(
+        lambda x, y: pd.merge(x, y, on=stat_groups, how='outer'), [sum_y_diff, n, y_max, y_min]
+    )
     nrmse_table_group = nrmse_table.groupby(stat_groups)
 
     # Calculate nrmse
     def NRMSE(x):
-        if x['n'].values == 1:      # when n==1, NRME equation will return an `inf` 
+        if x['n'].values == 1:  # when n==1, NRME equation will return an `inf`
             return x['sum_y_diff'] ** 0.5
         else:
             return ((x['sum_y_diff'] / x['n']) ** 0.5) / (x['y_max'] - x['y_min'])
 
-    nrmse = nrmse_table_group.apply(NRMSE)\
-        .reset_index(stat_groups, drop = False).rename({0: "nrmse"}, axis=1)
+    nrmse = nrmse_table_group.apply(NRMSE).reset_index(stat_groups, drop=False).rename({0: "nrmse"}, axis=1)
 
     # Calculate Mean Absolute Depth Difference
-    mean_abs_y_diff = station_rc.apply(lambda x: (abs(x["yhat_minus_y"]).mean()))\
-        .reset_index(stat_groups, drop = False).rename({0: "mean_abs_y_diff_ft"}, axis=1)
+    mean_abs_y_diff = (
+        station_rc.apply(lambda x: (abs(x["yhat_minus_y"]).mean()))
+        .reset_index(stat_groups, drop=False)
+        .rename({0: "mean_abs_y_diff_ft"}, axis=1)
+    )
 
     # Calculate Mean Depth Difference (non-absolute value)
-    mean_y_diff     = station_rc.apply(lambda x: (x["yhat_minus_y"].mean()))\
-        .reset_index(stat_groups, drop = False).rename({0: "mean_y_diff_ft"}, axis=1)
+    mean_y_diff = (
+        station_rc.apply(lambda x: (x["yhat_minus_y"].mean()))
+        .reset_index(stat_groups, drop=False)
+        .rename({0: "mean_y_diff_ft"}, axis=1)
+    )
 
     # Calculate Percent Bias
-    percent_bias = station_rc.apply(lambda x: 100 * (x["yhat_minus_y"].sum() / x[usgs_elev].sum()))\
-        .reset_index(stat_groups, drop = False).rename({0: "percent_bias"}, axis=1)
+    percent_bias = (
+        station_rc.apply(lambda x: 100 * (x["yhat_minus_y"].sum() / x[usgs_elev].sum()))
+        .reset_index(stat_groups, drop=False)
+        .rename({0: "percent_bias"}, axis=1)
+    )
 
-    rc_stat_table = reduce(lambda x,y: pd.merge(x,y, on=stat_groups, how='outer'), [n, nrmse, mean_abs_y_diff, mean_y_diff, percent_bias])
+    rc_stat_table = reduce(
+        lambda x, y: pd.merge(x, y, on=stat_groups, how='outer'),
+        [n, nrmse, mean_abs_y_diff, mean_y_diff, percent_bias],
+    )
 
     return rc_stat_table
 
+
 def create_static_gpkg(output_dir, output_gpkg, agg_recurr_stats_table, gages_gpkg_filepath):
     '''
     Merges the output dataframe from aggregate_metrics() with the usgs gages GIS data
@@ -693,9 +1071,8 @@ def create_static_gpkg(output_dir, output_gpkg, agg_recurr_stats_table, gages_gp
     # Merge the stats for all of the recurrance intervals/thresholds
     usgs_gages = usgs_gages.merge(agg_recurr_stats_table, on='location_id')
     # Load in the rating curves file
-    agg_nwm_recurr_flow_elev = join(output_dir,'agg_nwm_recurr_flow_elevations.csv')
-    agg_stats = pd.read_csv(agg_nwm_recurr_flow_elev,dtype={'location_id': str,
-                                                        'feature_id': str})
+    agg_nwm_recurr_flow_elev = join(output_dir, 'agg_nwm_recurr_flow_elevations.csv')
+    agg_stats = pd.read_csv(agg_nwm_recurr_flow_elev, dtype={'location_id': str, 'feature_id': str})
     diff_table = calculate_rc_diff(agg_stats)
     # Merge recurrence interval difference table with points layer
     usgs_gages = usgs_gages.merge(diff_table, on='location_id')
@@ -705,60 +1082,67 @@ def create_static_gpkg(output_dir, output_gpkg, agg_recurr_stats_table, gages_gp
     usgs_gages.to_file(join(output_dir, output_gpkg), driver='GPKG', index=False)
 
     # Create figure
-    usgs_gages.replace(np.inf, np.nan, inplace=True) # replace inf with nan for plotting
+    usgs_gages.replace(np.inf, np.nan, inplace=True)  # replace inf with nan for plotting
     fig, ax = plt.subplots(2, 2, figsize=(18, 10))
 
     # Bin data
     max_bin = usgs_gages['mean_abs_y_diff_ft'].max()
     bins = (0, 1, 3, 6, 9, max_bin if max_bin > 12 else 12)
     usgs_gages['mean_abs_y_diff_ft'] = pd.cut(usgs_gages['mean_abs_y_diff_ft'], bins=bins)
-    
+
     max_bin = usgs_gages['mean_y_diff_ft'].max()
     min_bin = usgs_gages['mean_y_diff_ft'].min()
     bins = (min_bin if min_bin < -12 else -12, -9, -6, -3, -1, 0, 1, 3, 6, 9, max_bin if max_bin > 12 else 12)
     usgs_gages['mean_y_diff_ft'] = pd.cut(usgs_gages['mean_y_diff_ft'], bins=bins)
 
     # Create subplots
-    sns.histplot(ax=ax[0,0], y='nrmse', data=usgs_gages, binwidth=0.2, binrange=(0, 10))
-    sns.countplot(ax=ax[1,0], y='mean_abs_y_diff_ft', data=usgs_gages)
-    sns.countplot(ax=ax[1,1], y='mean_y_diff_ft', data=usgs_gages)
-    sns.boxplot(ax=ax[0,1], data=usgs_gages[['2', '5', '10', '25', '50', '100', 'action', 'minor', 'moderate','major']])
-    ax[0,1].set(ylim=(-12, 12))
+    sns.histplot(ax=ax[0, 0], y='nrmse', data=usgs_gages, binwidth=0.2, binrange=(0, 10))
+    sns.countplot(ax=ax[1, 0], y='mean_abs_y_diff_ft', data=usgs_gages)
+    sns.countplot(ax=ax[1, 1], y='mean_y_diff_ft', data=usgs_gages)
+    sns.boxplot(
+        ax=ax[0, 1],
+        data=usgs_gages[['2', '5', '10', '25', '50', '100', 'action', 'minor', 'moderate', 'major']],
+    )
+    ax[0, 1].set(ylim=(-12, 12))
 
     fig.tight_layout()
     fig.savefig(join(output_dir, f'{output_gpkg}_summary_plots.png'.replace('.gpkg', '')))
 
     return
 
-def calculate_rc_diff(rc):
 
+def calculate_rc_diff(rc):
     usgs_elev = "USGS"
     src_elev = "FIM"
 
     # Collect any extra columns not associated with melt
     col_index = list(rc.columns)
-    pivot_vars = ['source','elevation_ft']
+    pivot_vars = ['source', 'elevation_ft']
     col_index = [col for col in col_index if col not in pivot_vars]
 
     # Unmelt elevation/source
-    rc_unmelt = (rc.set_index(col_index)
+    rc_unmelt = (
+        rc.set_index(col_index)
         .pivot(columns="source")['elevation_ft']
         .reset_index()
         .rename_axis(None, axis=1)
-     )
+    )
 
     # Calculate water surface elevation difference at recurrence intervals
     rc_unmelt["yhat_minus_y"] = rc_unmelt[src_elev] - rc_unmelt[usgs_elev]
-    # Remove duplicate location_id-recurr_interval pairs and pivot 
+    # Remove duplicate location_id-recurr_interval pairs and pivot
     rc_unmelt.set_index(['location_id', 'recurr_interval'], inplace=True, verify_integrity=False)
-    rc_unmelt = (rc_unmelt[~rc_unmelt.index.duplicated(keep='first')]
+    rc_unmelt = (
+        rc_unmelt[~rc_unmelt.index.duplicated(keep='first')]
         .reset_index()
-        .pivot(index='location_id', columns='recurr_interval', values='yhat_minus_y'))
+        .pivot(index='location_id', columns='recurr_interval', values='yhat_minus_y')
+    )
     # Reorder columns
-    rc_unmelt = rc_unmelt[['2', '5', '10', '25', '50', '100', 'action', 'minor', 'moderate','major']]
+    rc_unmelt = rc_unmelt[['2', '5', '10', '25', '50', '100', 'action', 'minor', 'moderate', 'major']]
 
     return rc_unmelt
-    
+
+
 def evaluate_results(sierra_results=[], labels=[], save_location=''):
     '''
     Compares multiple Sierra Test results using a boxplot.
@@ -768,7 +1152,8 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     sierra_results : list
         List of GeoDataFrames with sierra test results.
     labels : list
-        List of strings that will be used as labels for sierra_results. Length must be equal to sierra_results.
+        List of strings that will be used as labels for sierra_results.
+        Length must be equal to sierra_results.
     save_location : str
         Path to save output boxplot figure.
 
@@ -786,7 +1171,7 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     assert len(sierra_results) == len(labels), "Each Sierra Test results must also have a label"
 
     # Define recurrence intervals to plot
-    recurr_intervals = ("2","5","10","25","50","100","action","minor","moderate","major")
+    recurr_intervals = ("2", "5", "10", "25", "50", "100", "action", "minor", "moderate", "major")
 
     # Assign labels to the input sierra test result dataframes
     for df, label in zip(sierra_results, labels):
@@ -797,34 +1182,71 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     all_results = pd.concat([all_results, sierra_results[1:]])
 
     # Melt results for boxplotting
-    all_results_melted = all_results.melt(id_vars=["location_id", '_version'], 
-                                        value_vars=recurr_intervals, 
-                                        var_name='recurr_interval', 
-                                        value_name='error_ft')
+    all_results_melted = all_results.melt(
+        id_vars=["location_id", '_version'],
+        value_vars=recurr_intervals,
+        var_name='recurr_interval',
+        value_name='error_ft',
+    )
 
     # Plot all results in a comparison boxplot
-    fig, ax = plt.subplots(figsize=(10,5))
-    sns.boxplot(x='recurr_interval', y='error_ft', hue='_version', data=all_results_melted, ax=ax, fliersize=3)
+    fig, ax = plt.subplots(figsize=(10, 5))
+    sns.boxplot(
+        x='recurr_interval', y='error_ft', hue='_version', data=all_results_melted, ax=ax, fliersize=3
+    )
     ax.set(ylim=(-30, 30))
     ax.grid()
     ax.legend(bbox_to_anchor=(1, 1), loc='upper right', title='FIM Version')
     ax.set_title('Sierra Test Results Comparison')
     plt.savefig(save_location)
 
-if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description='generate rating curve plots and tables for FIM and USGS gages')
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-output_dir','--output-dir', help='rating curves output folder', required=True,type=str)
-    parser.add_argument('-gages','--usgs-gages-filename',help='USGS rating curves',required=True,type=str)
-    parser.add_argument('-flows','--nwm-flow-dir',help='NWM recurrence flows dir',required=True,type=str)
-    parser.add_argument('-catfim', '--catfim-flows-filename', help='Categorical FIM flows file',required = True,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-    parser.add_argument('-group','--stat-groups',help='column(s) to group stats',required=False,type=str,nargs='+')
-    parser.add_argument('-pnts','--stat-gages',help='takes 2 arguments: 1) file path of input usgs_gages.gpkg and 2) output GPKG name to write USGS gages with joined stats',required=False,type=str,nargs=2)
-    parser.add_argument('-alt','--alt-plot',help='Generate rating curve plots with REM maps',required=False,default=False,action='store_true')
-    parser.add_argument('-eval','--evaluate-results',help='Create a boxplot comparison of multiple input Sierra Test results. \
-        Expects 2 arguments: 1) path to the Sierra Test results for comparison and 2) the corresponding label for the boxplot.',required=False,nargs=2,action='append')
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='generate rating curve plots and tables for FIM and USGS gages'
+    )
+    parser.add_argument('-fim_dir', '--fim-dir', help='FIM output dir', required=True, type=str)
+    parser.add_argument(
+        '-output_dir', '--output-dir', help='rating curves output folder', required=True, type=str
+    )
+    parser.add_argument('-gages', '--usgs-gages-filename', help='USGS rating curves', required=True, type=str)
+    parser.add_argument('-flows', '--nwm-flow-dir', help='NWM recurrence flows dir', required=True, type=str)
+    parser.add_argument(
+        '-catfim', '--catfim-flows-filename', help='Categorical FIM flows file', required=True, type=str
+    )
+    parser.add_argument(
+        '-j', '--number-of-jobs', help='number of workers', required=False, default=1, type=int
+    )
+    parser.add_argument(
+        '-group', '--stat-groups', help='column(s) to group stats', required=False, type=str, nargs='+'
+    )
+    parser.add_argument(
+        '-pnts',
+        '--stat-gages',
+        help='takes 2 arguments: 1) file path of input usgs_gages.gpkg and 2) output GPKG name to write USGS '
+        'gages with joined stats',
+        required=False,
+        type=str,
+        nargs=2,
+    )
+    parser.add_argument(
+        '-alt',
+        '--alt-plot',
+        help='Generate rating curve plots with REM maps',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-eval',
+        '--evaluate-results',
+        help='Create a boxplot comparison of multiple input Sierra Test results. '
+        'Expects 2 arguments: 1) path to the Sierra Test results for comparison and 2) the corresponding '
+        'label for the boxplot.',
+        required=False,
+        nargs=2,
+        action='append',
+    )
     parser.add_argument('-s', '--single-plot', help='Create single plots', action='store_true')
     args = vars(parser.parse_args())
 
@@ -841,57 +1263,81 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     if args['stat_gages']:
         gages_gpkg_filepath = args['stat_gages'][0]
         stat_gages = args['stat_gages'][1]
-        assert (os.path.exists(gages_gpkg_filepath)), f"{gages_gpkg_filepath} does not exist. Please specify a full path to a USGS geopackage (.gpkg)"
+        assert os.path.exists(
+            gages_gpkg_filepath
+        ), f"{gages_gpkg_filepath} does not exist. Please specify a full path to a USGS geopackage (.gpkg)"
     else:
         stat_gages = None
 
     # Make sure that location_id is the only -group when using -pnts
-    assert (not stat_gages or (stat_gages and (not stat_groups or stat_groups == ['location_id']))), \
-        "location_id is the only acceptable stat_groups argument when producting an output GPKG"
+    assert not stat_gages or (
+        stat_gages and (not stat_groups or stat_groups == ['location_id'])
+    ), "location_id is the only acceptable stat_groups argument when producting an output GPKG"
     # Make sure that the -pnts flag is used with the -eval flag
     assert not eval or (eval and stat_gages), "You must use the -pnts flag with the -eval flag"
     procs_list = []
 
-    plots_dir = join(output_dir,'plots')
+    plots_dir = join(output_dir, 'plots')
     os.makedirs(plots_dir, exist_ok=True)
-    tables_dir = join(output_dir,'tables')
+    tables_dir = join(output_dir, 'tables')
     os.makedirs(tables_dir, exist_ok=True)
 
-    #Check age of gages csv and recommend updating if older than 30 days.
+    # Check age of gages csv and recommend updating if older than 30 days.
     print(check_file_age(usgs_gages_filename))
 
     # Open log file
     # Create log output
-    level    = logging.INFO # using WARNING level to avoid benign? info messages ("Failed to auto identify EPSG: 7")
-    format   = '  %(message)s'
-    handlers = [logging.FileHandler(os.path.join(output_dir,'rating_curve_comparison.log')), logging.StreamHandler()]
-    logging.basicConfig(level = level, format = format, handlers = handlers)
+    level = (
+        logging.INFO
+    )  # using WARNING level to avoid benign? info messages ("Failed to auto identify EPSG: 7")
+    format = '  %(message)s'
+    handlers = [
+        logging.FileHandler(os.path.join(output_dir, 'rating_curve_comparison.log')),
+        logging.StreamHandler(),
+    ]
+    logging.basicConfig(level=level, format=format, handlers=handlers)
 
     merged_elev_table = []
-    huc_list  = [huc for huc in os.listdir(fim_dir) if re.search("^\d{6,8}$", huc)]
+    huc_list = [huc for huc in os.listdir(fim_dir) if re.search(r"^\d{6,8}$", huc)]
     for huc in huc_list:
-
-        elev_table_filename = join(fim_dir,huc,'usgs_elev_table.csv')
-        branches_folder = join(fim_dir,huc,'branches')
-        usgs_recurr_stats_filename = join(tables_dir,f"usgs_interpolated_elevation_stats_{huc}.csv")
-        nwm_recurr_data_filename = join(tables_dir,f"nwm_recurrence_flow_elevations_{huc}.csv")
-        rc_comparison_plot_filename = join(plots_dir,f"FIM-USGS_rating_curve_comparison_{huc}.png")
+        elev_table_filename = join(fim_dir, huc, 'usgs_elev_table.csv')
+        branches_folder = join(fim_dir, huc, 'branches')
+        usgs_recurr_stats_filename = join(tables_dir, f"usgs_interpolated_elevation_stats_{huc}.csv")
+        nwm_recurr_data_filename = join(tables_dir, f"nwm_recurrence_flow_elevations_{huc}.csv")
+        rc_comparison_plot_filename = join(plots_dir, f"FIM-USGS_rating_curve_comparison_{huc}.png")
 
         if isfile(elev_table_filename):
-            procs_list.append([elev_table_filename, branches_folder, usgs_gages_filename, 
-                            usgs_recurr_stats_filename, nwm_recurr_data_filename, rc_comparison_plot_filename,
-                            nwm_flow_dir, catfim_flows_filename, huc, alt_plot, single_plot])
-            # Aggregate all of the individual huc elev_tables into one aggregate for accessing all data in one csv
-            read_elev_table = pd.read_csv(elev_table_filename, dtype={'location_id':str, 'HydroID':str, 'huc':str, 'feature_id':int})
+            procs_list.append(
+                [
+                    elev_table_filename,
+                    branches_folder,
+                    usgs_gages_filename,
+                    usgs_recurr_stats_filename,
+                    nwm_recurr_data_filename,
+                    rc_comparison_plot_filename,
+                    nwm_flow_dir,
+                    catfim_flows_filename,
+                    huc,
+                    alt_plot,
+                    single_plot,
+                ]
+            )
+            # Aggregate all of the individual huc elev_tables into one aggregate
+            #      for accessing all data in one csv
+            read_elev_table = pd.read_csv(
+                elev_table_filename, dtype={'location_id': str, 'HydroID': str, 'huc': str, 'feature_id': int}
+            )
             read_elev_table['huc'] = huc
             merged_elev_table.append(read_elev_table)
 
     # Output a concatenated elev_table to_csv
     if merged_elev_table:
-        logging.info(f"Creating aggregate elev table csv")
+        logging.info("Creating aggregate elev table csv")
         concat_elev_table = pd.concat(merged_elev_table)
-        concat_elev_table['thal_burn_depth_meters'] = concat_elev_table['dem_elevation'] - concat_elev_table['dem_adj_elevation']
-        concat_elev_table.to_csv(join(output_dir,'agg_usgs_elev_table.csv'),index=False)
+        concat_elev_table['thal_burn_depth_meters'] = (
+            concat_elev_table['dem_elevation'] - concat_elev_table['dem_adj_elevation']
+        )
+        concat_elev_table.to_csv(join(output_dir, 'agg_usgs_elev_table.csv'), index=False)
 
     # Initiate multiprocessing
     logging.info(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")
@@ -901,12 +1347,12 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     # Create point layer of usgs gages with joined stats attributes
     if stat_gages:
         logging.info("Creating usgs gages GPKG with joined rating curve summary stats")
-        agg_recurr_stats_table = aggregate_metrics(output_dir,procs_list,['location_id'])
+        agg_recurr_stats_table = aggregate_metrics(output_dir, procs_list, ['location_id'])
         create_static_gpkg(output_dir, stat_gages, agg_recurr_stats_table, gages_gpkg_filepath)
-        del agg_recurr_stats_table # memory cleanup
-    else: # if not producing GIS layer, just aggregate metrics
+        del agg_recurr_stats_table  # memory cleanup
+    else:  # if not producing GIS layer, just aggregate metrics
         logging.info(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
-        aggregate_metrics(output_dir,procs_list,stat_groups)
+        aggregate_metrics(output_dir, procs_list, stat_groups)
 
     logging.info('Delete intermediate tables')
     shutil.rmtree(tables_dir, ignore_errors=True)
@@ -921,6 +1367,6 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
         # Read in all sierra test results
         sierra_test_dfs = [gpd.read_file(i) for i in sierra_test_paths]
         # Feed results into evaluation function
-        evaluate_results(sierra_test_dfs,
-                        sierra_test_labels,
-                        join(output_dir, 'Sierra_Test_Eval_boxplot.png'))
+        evaluate_results(
+            sierra_test_dfs, sierra_test_labels, join(output_dir, 'Sierra_Test_Eval_boxplot.png')
+        )
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
index ba84689b8..7068f4eb4 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -1,31 +1,43 @@
 #!/usr/bin/env python3
+import argparse
+import os
+import sys
 import time
-import pandas as pd
-import geopandas as gpd
 from pathlib import Path
-from tools_shared_functions import get_metadata, get_datum, ngvd_to_navd_ft, get_rating_curve, aggregate_wbd_hucs, get_thresholds, flow_data
-from dotenv import load_dotenv
-import os
+
+import geopandas as gpd
 import numpy as np
-import argparse
-import sys
-sys.path.append('/foss_fim/src')
+import pandas as pd
+from dotenv import load_dotenv
+from tools_shared_functions import (
+    aggregate_wbd_hucs,
+    flow_data,
+    get_datum,
+    get_metadata,
+    get_rating_curve,
+    get_thresholds,
+    ngvd_to_navd_ft,
+)
+from tools_shared_variables import (
+    acceptable_alt_acc_thresh,
+    acceptable_alt_meth_code_list,
+    acceptable_coord_acc_code_list,
+    acceptable_coord_method_code_list,
+    acceptable_site_type_list,
+)
+
 from utils.shared_variables import PREP_PROJECTION
-from tools_shared_variables import (acceptable_coord_acc_code_list, 
-                                    acceptable_coord_method_code_list, 
-                                    acceptable_alt_acc_thresh, 
-                                    acceptable_alt_meth_code_list, 
-                                    acceptable_site_type_list)
+
 
 '''
 This script calls the NOAA Tidal API for datum conversions. Experience shows that
-running script outside of business hours seems to be most consistent way
-to avoid API errors. Currently configured to get rating curve data within
-CONUS. Tidal API call may need to be modified to get datum conversions for 
-AK, HI, PR/VI.
+    running script outside of business hours seems to be most consistent way
+    to avoid API errors. Currently configured to get rating curve data within
+    CONUS. Tidal API call may need to be modified to get datum conversions for
+    AK, HI, PR/VI.
 '''
 
-#import variables from .env file
+# import variables from .env file
 load_dotenv()
 API_BASE_URL = os.getenv("API_BASE_URL")
 WBD_LAYER = os.getenv("WBD_LAYER")
@@ -35,7 +47,7 @@
 
 def get_all_active_usgs_sites():
     '''
-    Compile a list of all active usgs gage sites. 
+    Compile a list of all active usgs gage sites.
     Return a GeoDataFrame of all sites.
 
     Returns
@@ -43,25 +55,33 @@ def get_all_active_usgs_sites():
     None.
 
     '''
-    #Get metadata for all usgs_site_codes that are active in the U.S.
-    metadata_url = f'{API_BASE_URL}/metadata' 
-    #Define arguments to retrieve metadata and then get metadata from WRDS
+    # Get metadata for all usgs_site_codes that are active in the U.S.
+    metadata_url = f'{API_BASE_URL}/metadata'
+    # Define arguments to retrieve metadata and then get metadata from WRDS
     select_by = 'usgs_site_code'
     selector = ['all']
     must_include = 'usgs_data.active'
-    metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = None )
-    #Get a geospatial layer (gdf) for all acceptable sites
+    metadata_list, metadata_df = get_metadata(
+        metadata_url,
+        select_by,
+        selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=None,
+    )
+    # Get a geospatial layer (gdf) for all acceptable sites
     print("Aggregating WBD HUCs...")
     dictionary, gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes=True)
-    #Get a list of all sites in gdf
+    # Get a list of all sites in gdf
     list_of_sites = gdf['identifiers_usgs_site_code'].to_list()
-    #Rename gdf fields
-    gdf.columns = gdf.columns.str.replace('identifiers_','')
+    # Rename gdf fields
+    gdf.columns = gdf.columns.str.replace('identifiers_', '')
 
     return gdf, list_of_sites, metadata_list
-            
+
+
 ##############################################################################
-#Generate categorical flows for each category across all sites.
+# Generate categorical flows for each category across all sites.
 ##############################################################################
 def write_categorical_flow_files(metadata, workspace):
     '''
@@ -80,58 +100,61 @@ def write_categorical_flow_files(metadata, workspace):
     None.
 
     '''
-    
+
     threshold_url = f'{API_BASE_URL}/nws_threshold'
     workspace = Path(workspace)
-    workspace.mkdir(parents = True, exist_ok = True)
-    #For each site in metadata 
+    workspace.mkdir(parents=True, exist_ok=True)
+    # For each site in metadata
     all_data = pd.DataFrame()
-    
+
     for site in metadata:
-        #Get the feature_id and usgs_site_code
+        # Get the feature_id and usgs_site_code
         feature_id = site.get('identifiers').get('nwm_feature_id')
         usgs_code = site.get('identifiers').get('usgs_site_code')
         nws_lid = site.get('identifiers').get('nws_lid')
-        
-        #thresholds only provided for valid nws_lid.
+
+        # thresholds only provided for valid nws_lid.
         if nws_lid == 'Bogus_ID':
             continue
-        
-        #if invalid feature_id skip to next site
+
+        # if invalid feature_id skip to next site
         if feature_id is None:
             continue
-        
-        #Get the stages and flows
-        stages, flows = get_thresholds(threshold_url, select_by = 'nws_lid', selector = nws_lid, threshold = 'all')
-        
-        #For each flood category
-        for category in ['action','minor','moderate','major']:
-            #Get flow
+
+        # Get the stages and flows
+        stages, flows = get_thresholds(threshold_url, select_by='nws_lid', selector=nws_lid, threshold='all')
+
+        # For each flood category
+        for category in ['action', 'minor', 'moderate', 'major']:
+            # Get flow
             flow = flows.get(category, None)
-            #If flow or feature id are not valid, skip to next site
+            # If flow or feature id are not valid, skip to next site
             if flow is None:
                 continue
-            #Otherwise, write 'guts' of a flow file and append to a master DataFrame.
+            # Otherwise, write 'guts' of a flow file and append to a master DataFrame.
             else:
-                data = flow_data([feature_id], flow, convert_to_cms = True)
-                data['recurr_interval'] = category            
+                data = flow_data([feature_id], flow, convert_to_cms=True)
+                data['recurr_interval'] = category
                 data['nws_lid'] = nws_lid
                 data['location_id'] = usgs_code
-                data = data.rename(columns = {'discharge':'discharge_cms'})
-                #Append site data to master DataFrame
-                all_data = pd.concat([all_data, data], ignore_index = True)
-    
-    #Write CatFIM flows to file
-    final_data = all_data[['feature_id','discharge_cms', 'recurr_interval']]
-    final_data.to_csv(workspace / f'catfim_flows_cms.csv', index = False)
+                data = data.rename(columns={'discharge': 'discharge_cms'})
+                # Append site data to master DataFrame
+                all_data = pd.concat([all_data, data], ignore_index=True)
+
+    # Write CatFIM flows to file
+    final_data = all_data[['feature_id', 'discharge_cms', 'recurr_interval']]
+    final_data.to_csv(workspace / 'catfim_flows_cms.csv', index=False)
     return all_data
+
+
 ###############################################################################
-           
-def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
+
+
+def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
     '''
 
     Returns rating curves, for a set of sites, adjusted to elevation NAVD.
-    Currently configured to get rating curve data within CONUS. Tidal API 
+    Currently configured to get rating curve data within CONUS. Tidal API
     call may need to be modified to get datum conversions for AK, HI, PR/VI.
     Workflow as follows:
         1a. If 'all' option passed, get metadata for all acceptable USGS sites in CONUS.
@@ -143,115 +166,129 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
         6.  Convert rating curve to absolute elevation (NAVD) and store in DataFrame
         7.  Append all rating curves to a master DataFrame.
 
-    
+
     Outputs, if a workspace is specified, are:
         usgs_rating_curves.csv -- A csv containing USGS rating curve as well
         as datum adjustment and rating curve expressed as an elevation (NAVD88).
-        ONLY SITES IN CONUS ARE CURRENTLY LISTED IN THIS CSV. To get 
+        ONLY SITES IN CONUS ARE CURRENTLY LISTED IN THIS CSV. To get
         additional sites, the Tidal API will need to be reconfigured and tested.
-        
+
         log.csv -- A csv containing runtime messages.
-        
+
         (if all option passed) usgs_gages.gpkg -- a point layer containing ALL USGS gage sites that meet
         certain criteria. In the attribute table is a 'curve' column that will indicate if a rating
         curve is provided in "usgs_rating_curves.csv"
-       
+
     Parameters
     ----------
     list_of_gage_sites : LIST
         List of all gage site IDs. If all acceptable sites in CONUS are desired
         list_of_gage_sites can be passed 'all' and it will use the get_all_active_usgs_sites
         function to filter out sites that meet certain requirements across CONUS.
-        
+
     workspace : STR
         Directory, if specified, where output csv is saved. OPTIONAL, Default is False.
-    
+
     sleep_time: FLOAT
-        Amount of time to rest between API calls. The Tidal API appears to 
+        Amount of time to rest between API calls. The Tidal API appears to
         error out more during business hours. Increasing sleep_time may help.
-        
+
 
     Returns
     -------
     all_rating_curves : Pandas DataFrame
-        DataFrame containing USGS rating curves adjusted to elevation for 
+        DataFrame containing USGS rating curves adjusted to elevation for
         all input sites. Additional metadata also contained in DataFrame
 
     '''
-    
-    #Define URLs for metadata and rating curve
+
+    # Define URLs for metadata and rating curve
     metadata_url = f'{API_BASE_URL}/metadata'
     rating_curve_url = f'{API_BASE_URL}/rating_curve'
 
     # Create workspace directory if it doesn't exist
     if not os.path.exists(workspace):
         os.mkdir(workspace)
-    
-    #If 'all' option passed to list of gages sites, it retrieves all sites within CONUS.
+
+    # If 'all' option passed to list of gages sites, it retrieves all sites within CONUS.
     print('getting metadata for all sites')
     if list_of_gage_sites == ['all']:
         sites_gdf, sites_list, metadata_list = get_all_active_usgs_sites()
-    #Otherwise, if a list of sites is passed, retrieve sites from WRDS.
-    else:        
-        #Define arguments to retrieve metadata and then get metadata from WRDS
+    # Otherwise, if a list of sites is passed, retrieve sites from WRDS.
+    else:
+        # Define arguments to retrieve metadata and then get metadata from WRDS
         select_by = 'usgs_site_code'
-        selector = list_of_gage_sites        
-        #Since there is a limit to number characters in url, split up selector if too many sites.
+        selector = list_of_gage_sites
+        # Since there is a limit to number characters in url, split up selector if too many sites.
         max_sites = 150
-        if len(selector)>max_sites:
-            chunks = [selector[i:i+max_sites] for i in range(0,len(selector),max_sites)]
-            #Get metadata for each chunk
+        if len(selector) > max_sites:
+            chunks = [selector[i : i + max_sites] for i in range(0, len(selector), max_sites)]
+            # Get metadata for each chunk
             metadata_list = []
             metadata_df = pd.DataFrame()
             for chunk in chunks:
-                chunk_list, chunk_df = get_metadata(metadata_url, select_by, chunk, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None )
-                #Append chunk data to metadata_list/df
+                chunk_list, chunk_df = get_metadata(
+                    metadata_url,
+                    select_by,
+                    chunk,
+                    must_include=None,
+                    upstream_trace_distance=None,
+                    downstream_trace_distance=None,
+                )
+                # Append chunk data to metadata_list/df
                 metadata_list.extend(chunk_list)
                 metadata_df = pd.concat([metadata_df, chunk_df])
         else:
-            #If selector has less than max sites, then get metadata.
-            metadata_list, metadata_df = get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None )
-    
-    #Create DataFrame to store all appended rating curves
+            # If selector has less than max sites, then get metadata.
+            metadata_list, metadata_df = get_metadata(
+                metadata_url,
+                select_by,
+                selector,
+                must_include=None,
+                upstream_trace_distance=None,
+                downstream_trace_distance=None,
+            )
+
+    # Create DataFrame to store all appended rating curves
     print('processing metadata')
     all_rating_curves = pd.DataFrame()
-    regular_messages = []    
-    api_failure_messages=[]
-    #For each site in metadata_list
+    regular_messages = []
+    api_failure_messages = []
+    # For each site in metadata_list
     for metadata in metadata_list:
-        
         print("get_datum")
-        #Get datum information for site (only need usgs_data)
-        nws, usgs = get_datum(metadata)        
-        
-        #Filter out sites that are not in contiguous US. If this section is removed be sure to test with datum adjustment section (region will need changed)
+        # Get datum information for site (only need usgs_data)
+        nws, usgs = get_datum(metadata)
+
+        # Filter out sites that are not in contiguous US. If this section is removed be sure to test with
+        #   datum adjustment section (region will need changed)
         if usgs['state'] in ['Alaska', 'Puerto Rico', 'Virgin Islands', 'Hawaii']:
-            continue        
-        #Get rating curve for site
+            continue
+        # Get rating curve for site
         location_ids = usgs['usgs_site_code']
-        if location_ids == None:  # Some sites don't have a value for usgs_site_code, skip them
+        if location_ids is None:  # Some sites don't have a value for usgs_site_code, skip them
             continue
-        curve = get_rating_curve(rating_curve_url, location_ids = [location_ids])
-        #If no rating curve was returned, skip site.
+        curve = get_rating_curve(rating_curve_url, location_ids=[location_ids])
+        # If no rating curve was returned, skip site.
         if curve.empty:
             message = f'{location_ids}: has no rating curve'
             regular_messages.append(message)
             continue
-        
-        #Adjust datum to NAVD88 if needed. If datum unknown, skip site.
+
+        # Adjust datum to NAVD88 if needed. If datum unknown, skip site.
         if usgs['vcs'] == 'NGVD29':
-            #To prevent time-out errors
+            # To prevent time-out errors
             time.sleep(sleep_time)
-            #Get the datum adjustment to convert NGVD to NAVD. Region needs changed if not in CONUS.
-            datum_adj_ft = ngvd_to_navd_ft(datum_info = usgs, region = 'contiguous')
+            # Get the datum adjustment to convert NGVD to NAVD. Region needs changed if not in CONUS.
+            datum_adj_ft = ngvd_to_navd_ft(datum_info=usgs, region='contiguous')
 
-            #If datum API failed, print message and skip site.
+            # If datum API failed, print message and skip site.
             if datum_adj_ft is None:
                 api_message = f"{location_ids}: datum adjustment failed!!"
                 api_failure_messages.append(api_message)
                 print(api_message)
                 continue
-            #If datum adjustment succeeded, calculate datum in NAVD88            
+            # If datum adjustment succeeded, calculate datum in NAVD88
             navd88_datum = round(usgs['datum'] + datum_adj_ft, 2)
             message = f'{location_ids}:succesfully converted NGVD29 to NAVD88'
             regular_messages.append(message)
@@ -265,105 +302,131 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time = 1.0):
             message = f"{location_ids}: datum unknown"
             regular_messages.append(message)
             continue
-        
+
         print("Populating..")
-        #Populate rating curve with metadata and use navd88 datum to convert stage to elevation.
+        # Populate rating curve with metadata and use navd88 datum to convert stage to elevation.
         curve['active'] = usgs['active']
         curve['datum'] = usgs['datum']
         curve['datum_vcs'] = usgs['vcs']
         curve['navd88_datum'] = navd88_datum
         curve['elevation_navd88'] = curve['stage'] + navd88_datum
-        #Append all rating curves to a dataframe
+        # Append all rating curves to a dataframe
         all_rating_curves = pd.concat([all_rating_curves, curve])
 
-    #Rename columns and add attribute indicating if rating curve exists
-    sites_gdf.rename(columns = {'nwm_feature_id':'feature_id','usgs_site_code':'location_id'}, inplace = True)
-    sites_with_data = pd.DataFrame({'location_id':all_rating_curves['location_id'].unique(),'curve':'yes'})
-    sites_gdf = sites_gdf.merge(sites_with_data, on = 'location_id', how = 'left')
-    sites_gdf.fillna({'curve':'no'},inplace = True)    
-    #Add mainstems attribute to acceptable sites
+    # Rename columns and add attribute indicating if rating curve exists
+    sites_gdf.rename(columns={'nwm_feature_id': 'feature_id', 'usgs_site_code': 'location_id'}, inplace=True)
+    sites_with_data = pd.DataFrame({'location_id': all_rating_curves['location_id'].unique(), 'curve': 'yes'})
+    sites_gdf = sites_gdf.merge(sites_with_data, on='location_id', how='left')
+    sites_gdf.fillna({'curve': 'no'}, inplace=True)
+    # Add mainstems attribute to acceptable sites
     print('Attributing mainstems sites')
-    #Import mainstems segments used in run_by_unit.sh
+    # Import mainstems segments used in run_by_unit.sh
     ms_df = gpd.read_file(NWM_FLOWS_MS)
     ms_segs = ms_df.ID.astype(str).to_list()
-    #Populate mainstems attribute field
+    # Populate mainstems attribute field
     sites_gdf['mainstem'] = 'no'
-    sites_gdf.loc[sites_gdf.eval('feature_id in @ms_segs'),'mainstem'] = 'yes' 
+    sites_gdf.loc[sites_gdf.eval('feature_id in @ms_segs'), 'mainstem'] = 'yes'
     sites_gdf.to_csv(os.path.join(workspace, 'acceptable_sites_pre.csv'))
-    
+
     sites_gdf = sites_gdf.drop(['upstream_nwm_features'], axis=1, errors='ignore')
     sites_gdf = sites_gdf.drop(['downstream_nwm_features'], axis=1, errors='ignore')
-    
+
     print("Recasting...")
     sites_gdf = sites_gdf.astype({'metadata_sources': str})
-    
+
     # -- Filter all_rating_curves according to acceptance criteria -- #
-    # -- We only want acceptable gages in the rating curve CSV -- #           
-    sites_gdf['acceptable_codes'] = (sites_gdf['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
-                                    & sites_gdf['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
-                                    & sites_gdf['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
-                                    & sites_gdf['usgs_data_site_type'].isin(acceptable_site_type_list))
-    
+    # -- We only want acceptable gages in the rating curve CSV -- #
+    sites_gdf['acceptable_codes'] = (
+        sites_gdf['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
+        & sites_gdf['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
+        & sites_gdf['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
+        & sites_gdf['usgs_data_site_type'].isin(acceptable_site_type_list)
+    )
+
     sites_gdf = sites_gdf.astype({'usgs_data_alt_accuracy_code': float})
-    sites_gdf['acceptable_alt_error'] = np.where(sites_gdf['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False)
-    
+    sites_gdf['acceptable_alt_error'] = np.where(
+        sites_gdf['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False
+    )
+
     sites_gdf.to_file(os.path.join(workspace, 'sites_bool_flags.gpkg'), driver='GPKG')
-        
+
     # Filter and save filtered file for viewing
-    acceptable_sites_gdf = sites_gdf[(sites_gdf['acceptable_codes'] == True) & (sites_gdf['acceptable_alt_error'] == True)]
+    acceptable_sites_gdf = sites_gdf[
+        (sites_gdf['acceptable_codes'] is True) & (sites_gdf['acceptable_alt_error'] is True)
+    ]
     acceptable_sites_gdf = acceptable_sites_gdf[acceptable_sites_gdf['curve'] == 'yes']
     acceptable_sites_gdf.to_csv(os.path.join(workspace, 'acceptable_sites_for_rating_curves.csv'))
-    acceptable_sites_gdf.to_file(os.path.join(workspace, 'acceptable_sites_for_rating_curves.gpkg'),driver='GPKG')
-    
+    acceptable_sites_gdf.to_file(
+        os.path.join(workspace, 'acceptable_sites_for_rating_curves.gpkg'), driver='GPKG'
+    )
+
     # Make list of acceptable sites
     acceptable_sites_list = acceptable_sites_gdf['location_id'].tolist()
-        
+
     # Filter out all_rating_curves by list
     all_rating_curves = all_rating_curves[all_rating_curves['location_id'].isin(acceptable_sites_list)]
 
-    #If workspace is specified, write data to file.
+    # If workspace is specified, write data to file.
     if workspace:
-        #Write rating curve dataframe to file
-        Path(workspace).mkdir(parents = True, exist_ok = True)
-        all_rating_curves.to_csv(Path(workspace) / 'usgs_rating_curves.csv', index = False)
-        #Save out messages to file.
-        first_line = [f'THERE WERE {len(api_failure_messages)} SITES THAT EXPERIENCED DATUM CONVERSION ISSUES']
+        # Write rating curve dataframe to file
+        Path(workspace).mkdir(parents=True, exist_ok=True)
+        all_rating_curves.to_csv(Path(workspace) / 'usgs_rating_curves.csv', index=False)
+        # Save out messages to file.
+        first_line = [
+            f'THERE WERE {len(api_failure_messages)} SITES THAT EXPERIENCED DATUM CONVERSION ISSUES'
+        ]
         api_failure_messages = first_line + api_failure_messages
-        regular_messages = api_failure_messages + regular_messages                
-        all_messages = pd.DataFrame({'Messages':regular_messages})
-        all_messages.to_csv(Path(workspace) / 'log.csv', index = False)
+        regular_messages = api_failure_messages + regular_messages
+        all_messages = pd.DataFrame({'Messages': regular_messages})
+        all_messages.to_csv(Path(workspace) / 'log.csv', index=False)
         # If 'all' option specified, reproject then write out shapefile of acceptable sites.
-        if list_of_gage_sites == ['all']:            
+        if list_of_gage_sites == ['all']:
             sites_gdf = sites_gdf.to_crs(PREP_PROJECTION)
-            sites_gdf.to_file(Path(workspace) / 'usgs_gages.gpkg', layer = 'usgs_gages', driver = 'GPKG')
-        
-        #Write out flow files for each threshold across all sites
-        all_data = write_categorical_flow_files(metadata_list, workspace)
-    
+            sites_gdf.to_file(Path(workspace) / 'usgs_gages.gpkg', layer='usgs_gages', driver='GPKG')
+
+        # Write out flow files for each threshold across all sites
+        write_categorical_flow_files(metadata_list, workspace)
+
     return all_rating_curves
 
+
 if __name__ == '__main__':
-    #Parse arguments
-    parser = argparse.ArgumentParser(description = 'Retrieve USGS rating curves adjusted to elevation (NAVD88).\nCurrently configured to get rating curves within CONUS.\nRecommend running outside of business hours to reduce API related errors.\nIf error occurs try increasing sleep time (from default of 1).')
-    parser.add_argument('-l', '--list_of_gage_sites',  help = '"all" for all active usgs sites, specify individual sites separated by space, or provide a csv of sites (one per line).', nargs = '+', required = True)
-    parser.add_argument('-w', '--workspace', help = 'Directory where all outputs will be stored.', default = False, required = False)
-    parser.add_argument('-t', '--sleep_timer', help = 'How long to rest between datum API calls', default = 1.0, required = False)
-       
-    #Extract to dictionary and assign to variables.
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description='Retrieve USGS rating curves adjusted to elevation (NAVD88).\n'
+        'Currently configured to get rating curves within CONUS.\n'
+        'Recommend running outside of business hours to reduce API related errors.\n'
+        'If error occurs try increasing sleep time (from default of 1).'
+    )
+    parser.add_argument(
+        '-l',
+        '--list_of_gage_sites',
+        help='"all" for all active usgs sites, specify individual sites separated by space, '
+        'or provide a csv of sites (one per line).',
+        nargs='+',
+        required=True,
+    )
+    parser.add_argument(
+        '-w', '--workspace', help='Directory where all outputs will be stored.', default=False, required=False
+    )
+    parser.add_argument(
+        '-t', '--sleep_timer', help='How long to rest between datum API calls', default=1.0, required=False
+    )
+
+    # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
 
-    #Check if csv is supplied       
-    if args['list_of_gage_sites'][0].endswith('.csv'):        
-        #Convert csv list to python list
+    # Check if csv is supplied
+    if args['list_of_gage_sites'][0].endswith('.csv'):
+        # Convert csv list to python list
         with open(args['list_of_gage_sites']) as f:
             sites = f.read().splitlines()
         args['list_of_gage_sites'] = sites
 
-    l = args['list_of_gage_sites']
-    w = args['workspace'] 
-    t = float(args['sleep_timer'])           
-    
-    #Generate USGS rating curves
+    list_of_gage_sites = args['list_of_gage_sites']
+    workspace = args['workspace']
+    sleep_timer = float(args['sleep_timer'])
+
+    # Generate USGS rating curves
     print("Executing...")
-    usgs_rating_to_elev(list_of_gage_sites = l, workspace=w, sleep_time = t)
-    
\ No newline at end of file
+    usgs_rating_to_elev(list_of_gage_sites=list_of_gage_sites, workspace=workspace, sleep_time=sleep_timer)
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 574aadb10..89fe04da2 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -1,20 +1,26 @@
 #!/usr/bin/env python3
 
-import os, re, shutil, json, sys
-import pandas as pd
+import json
+import os
+import re
+import shutil
+import sys
 
+import pandas as pd
+from inundate_mosaic_wrapper import produce_mosaicked_inundation
+from inundation import inundate
+from mosaic_inundation import Mosaic_inundation
+from tools_shared_functions import compute_contingency_stats_from_rasters
 from tools_shared_variables import (
-    TEST_CASES_DIR,
-    INPUTS_DIR,
-    PREVIOUS_FIM_DIR,
-    OUTPUTS_DIR,
     AHPS_BENCHMARK_CATEGORIES,
+    INPUTS_DIR,
     MAGNITUDE_DICT,
+    OUTPUTS_DIR,
+    PREVIOUS_FIM_DIR,
+    TEST_CASES_DIR,
     elev_raster_ndv,
 )
-from inundation import inundate
-from inundate_mosaic_wrapper import produce_mosaicked_inundation
-from tools_shared_functions import compute_contingency_stats_from_rasters
+
 from utils.shared_functions import FIM_Helpers as fh
 
 
@@ -36,13 +42,9 @@ def __init__(self, category):
             self.MAGNITUDE_DICT.keys()
         ), f"Category must be one of {list(self.MAGNITUDE_DICT.keys())}"
         self.validation_data = os.path.join(
-            TEST_CASES_DIR,
-            f"{self.category}_test_cases",
-            f"validation_data_{self.category}",
-        )
-        self.is_ahps = (
-            True if self.category in self.AHPS_BENCHMARK_CATEGORIES else False
+            TEST_CASES_DIR, f"{self.category}_test_cases", f"validation_data_{self.category}"
         )
+        self.is_ahps = True if self.category in self.AHPS_BENCHMARK_CATEGORIES else False
 
     def magnitudes(self):
         """Returns the magnitudes associated with the benchmark category."""
@@ -52,7 +54,7 @@ def huc_data(self):
         """Returns a dict of HUC8, magnitudes, and sites."""
         huc_mags = {}
         for huc in os.listdir(self.validation_data):
-            if not re.match("\d{8}", huc):
+            if not re.match(r"\d{8}", huc):
                 continue
             huc_mags[huc] = self.data(huc)
         return huc_mags
@@ -70,9 +72,7 @@ def data(self, huc):
             mag_dict = {}
             for lid in lids:
                 lid_dir = os.path.join(huc_dir, lid)
-                for mag in [
-                    file for file in os.listdir(lid_dir) if file in self.magnitudes()
-                ]:
+                for mag in [file for file in os.listdir(lid_dir) if file in self.magnitudes()]:
                     if mag in mag_dict:
                         mag_dict[mag].append(lid)
                     else:
@@ -83,7 +83,7 @@ def data(self, huc):
             return {mag: [""] for mag in mags}
 
 
-class test_case(benchmark):
+class Test_Case(benchmark):
     def __init__(self, test_id, version, archive=True):
         """Class that handles test cases, specifically running the alpha test.
 
@@ -92,7 +92,7 @@ def __init__(self, test_id, version, archive=True):
         test_id : str
             ID of the test case in huc8_category format, e.g. `12090201_ble`.
         version : str
-            Version of FIM to which this test_case belongs. This should correspond to the fim directory
+            Version of FIM to which this Test_Case belongs. This should correspond to the fim directory
             name in either `/data/previous_fim/` or `/outputs/`.
         archive : bool
             If true, this test case outputs will be placed into the `official_versions` folder
@@ -110,9 +110,7 @@ def __init__(self, test_id, version, archive=True):
         self.fim_dir = os.path.join(
             PREVIOUS_FIM_DIR if archive else OUTPUTS_DIR,
             self.version,
-            self.huc
-            if not re.search("^fim_[1,2]", version, re.IGNORECASE)
-            else self.huc[:6],
+            self.huc if not re.search("^fim_[1,2]", version, re.IGNORECASE) else self.huc[:6],
         )
         # Test case directory path
         self.dir = os.path.join(
@@ -204,9 +202,7 @@ def alpha_test(
 
         try:
             if not overwrite and os.path.isdir(self.dir):
-                print(
-                    f"Metrics for {self.dir} already exist. Use overwrite flag (-o) to overwrite metrics."
-                )
+                print(f"Metrics for {self.dir} already exist. Use overwrite flag (-o) to overwrite metrics.")
                 return
 
             fh.vprint(f"Starting alpha test for {self.dir}", verbose)
@@ -217,37 +213,28 @@ def alpha_test(
             if model != "GMS":
                 self.rem = os.path.join(self.fim_dir, "rem_zeroed_masked.tif")
                 if not os.path.exists(self.rem):
-                    self.rem = os.path.join(
-                        self.fim_dir, "rem_clipped_zeroed_masked.tif"
-                    )
+                    self.rem = os.path.join(self.fim_dir, "rem_clipped_zeroed_masked.tif")
                 self.catchments = os.path.join(
                     self.fim_dir, "gw_catchments_reaches_filtered_addedAttributes.tif"
                 )
                 if not os.path.exists(self.catchments):
                     self.catchments = os.path.join(
-                        self.fim_dir,
-                        "gw_catchments_reaches_clipped_addedAttributes.tif",
+                        self.fim_dir, "gw_catchments_reaches_clipped_addedAttributes.tif"
                     )
                 self.mask_type = mask_type
                 if mask_type == "huc":
                     self.catchment_poly = ""
                 else:
                     self.catchment_poly = os.path.join(
-                        self.fim_dir,
-                        "gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg",
+                        self.fim_dir, "gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg"
                     )
                 self.hydro_table = os.path.join(self.fim_dir, "hydroTable.csv")
 
             # Map necessary inputs for inundate().
-            self.hucs, self.hucs_layerName = (
-                os.path.join(INPUTS_DIR, "wbd", "WBD_National.gpkg"),
-                "WBDHU8",
-            )
+            self.hucs, self.hucs_layerName = (os.path.join(INPUTS_DIR, "wbd", "WBD_National.gpkg"), "WBDHU8")
 
             if inclusion_area != "":
-                inclusion_area_name = os.path.split(inclusion_area)[1].split(".")[
-                    0
-                ]  # Get layer name
+                inclusion_area_name = os.path.split(inclusion_area)[1].split(".")[0]  # Get layer name
                 self.mask_dict.update(
                     {
                         inclusion_area_name: {
@@ -258,11 +245,9 @@ def alpha_test(
                     }
                 )
                 # Append the concatenated inclusion_area_name and buffer.
-                if inclusion_area_buffer == None:
+                if inclusion_area_buffer is None:
                     inclusion_area_buffer = 0
-                self.stats_modes_list.append(
-                    inclusion_area_name + "_b" + str(inclusion_area_buffer) + "m"
-                )
+                self.stats_modes_list.append(inclusion_area_name + "_b" + str(inclusion_area_buffer) + "m")
 
             # Delete the directory if it exists
             if os.path.exists(self.dir):
@@ -277,11 +262,7 @@ def alpha_test(
                 ]:  # instance will be the lid for AHPS sites and '' for other sites
                     # For each site, inundate the REM and compute aggreement raster with stats
                     self._inundate_and_compute(
-                        magnitude,
-                        instance,
-                        model=model,
-                        verbose=verbose,
-                        gms_workers=gms_workers,
+                        magnitude, instance, model=model, verbose=verbose, gms_workers=gms_workers
                     )
 
                 # Clean up 'total_area' outputs from AHPS sites
@@ -318,13 +299,10 @@ def _inundate_and_compute(
 
         test_case_out_dir = os.path.join(self.dir, magnitude)
         inundation_prefix = lid + "_" if lid else ""
-        inundation_path = os.path.join(
-            test_case_out_dir, f"{inundation_prefix}inundation_extent.tif"
-        )
+        inundation_path = os.path.join(test_case_out_dir, f"{inundation_prefix}inundation_extent.tif")
         predicted_raster_path = inundation_path.replace(".tif", f"_{self.huc}.tif")
         agreement_raster = os.path.join(
-            test_case_out_dir,
-            (f"ahps_{lid}" if lid else "") + "total_area_agreement.tif",
+            test_case_out_dir, (f"ahps_{lid}" if lid else "") + "total_area_agreement.tif"
         )
         stats_json = os.path.join(test_case_out_dir, "stats.json")
         stats_csv = os.path.join(test_case_out_dir, "stats.csv")
@@ -337,18 +315,12 @@ def _inundate_and_compute(
         benchmark_rast = (
             f"ahps_{lid}" if lid else self.benchmark_cat
         ) + f"_huc_{self.huc}_extent_{magnitude}.tif"
-        benchmark_rast = os.path.join(
-            self.benchmark_dir, lid, magnitude, benchmark_rast
-        )
-        benchmark_flows = benchmark_rast.replace(
-            f"_extent_{magnitude}.tif", f"_flows_{magnitude}.csv"
-        )
+        benchmark_rast = os.path.join(self.benchmark_dir, lid, magnitude, benchmark_rast)
+        benchmark_flows = benchmark_rast.replace(f"_extent_{magnitude}.tif", f"_flows_{magnitude}.csv")
         mask_dict_indiv = self.mask_dict.copy()
         if self.is_ahps:  # add domain shapefile to mask for AHPS sites
             domain = os.path.join(self.benchmark_dir, lid, f"{lid}_domain.shp")
-            mask_dict_indiv.update(
-                {lid: {"path": domain, "buffer": None, "operation": "include"}}
-            )
+            mask_dict_indiv.update({lid: {"path": domain, "buffer": None, "operation": "include"}})
         # Check to make sure all relevant files exist
         if (
             not os.path.isfile(benchmark_rast)
@@ -424,7 +396,7 @@ def run_alpha_test(
         verbose=False,
         gms_workers=1,
     ):
-        """Class method for instantiating the test_case class and running alpha_test directly"""
+        """Class method for instantiating the Test_Case class and running alpha_test directly"""
 
         alpha_class = cls(test_id, version, archive_results)
         alpha_class.alpha_test(
@@ -452,21 +424,15 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
         """
 
         if re.match(r"(.*)(_ms|_fr)", self.version):
-            composite_version_name = re.sub(
-                r"(.*)(_ms|_fr)", r"\1_comp", self.version, count=1
-            )
+            composite_version_name = re.sub(r"(.*)(_ms|_fr)", r"\1_comp", self.version, count=1)
         else:
-            composite_version_name = re.sub(
-                r"(.*)(_ms|_fr)", r"\1_comp", version_2, count=1
-            )
+            composite_version_name = re.sub(r"(.*)(_ms|_fr)", r"\1_comp", version_2, count=1)
 
         fh.vprint(f"Begin composite for version : {composite_version_name}", verbose)
 
-        composite_test_case = test_case(
-            self.test_id, composite_version_name, self.archive
-        )
-        input_test_case_2 = test_case(self.test_id, version_2, self.archive)
-        composite_test_case.stats_modes_list = ["total_area"]
+        composite_test_case = Test_Case(self.test_id, composite_version_name, self.archive)
+        input_test_case_2 = Test_Case(self.test_id, version_2, self.archive)
+        composite_test_case.stats_modes_list = ['total_area']
 
         if not overwrite and os.path.isdir(composite_test_case.dir):
             return
@@ -483,9 +449,7 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
                 inundation_prefix = instance + "_" if instance else ""
 
                 input_inundation = os.path.join(
-                    self.dir,
-                    magnitude,
-                    f"{inundation_prefix}inundation_extent_{self.huc}.tif",
+                    self.dir, magnitude, f"{inundation_prefix}inundation_extent_{self.huc}.tif"
                 )
                 input_inundation_2 = os.path.join(
                     input_test_case_2.dir,
@@ -493,32 +457,22 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
                     f"{inundation_prefix}inundation_extent_{input_test_case_2.huc}.tif",
                 )
                 output_inundation = os.path.join(
-                    composite_test_case.dir,
-                    magnitude,
-                    f"{inundation_prefix}inundation_extent.tif",
+                    composite_test_case.dir, magnitude, f"{inundation_prefix}inundation_extent.tif"
                 )
 
-                if os.path.isfile(input_inundation) and os.path.isfile(
-                    input_inundation_2
-                ):
+                if os.path.isfile(input_inundation) and os.path.isfile(input_inundation_2):
                     inundation_map_file = pd.DataFrame(
                         {
                             "huc8": [composite_test_case.huc] * 2,
                             "branchID": [None] * 2,
-                            "inundation_rasters": [
-                                input_inundation,
-                                input_inundation_2,
-                            ],
+                            "inundation_rasters": [input_inundation, input_inundation_2],
                             "depths_rasters": [None] * 2,
                             "inundation_polygons": [None] * 2,
                         }
                     )
                     os.makedirs(os.path.dirname(output_inundation), exist_ok=True)
 
-                    fh.vprint(
-                        f"Begin mosaic inundation for version : {composite_version_name}",
-                        verbose,
-                    )
+                    fh.vprint(f"Begin mosaic inundation for version : {composite_version_name}", verbose)
                     Mosaic_inundation(
                         inundation_map_file,
                         mosaic_attribute="inundation_rasters",
@@ -531,31 +485,21 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
                         subset=None,
                         verbose=False,
                     )
-                    composite_test_case._inundate_and_compute(
-                        magnitude, instance, compute_only=True
-                    )
+                    composite_test_case._inundate_and_compute(magnitude, instance, compute_only=True)
 
-                elif os.path.isfile(input_inundation) or os.path.isfile(
-                    input_inundation_2
-                ):
+                elif os.path.isfile(input_inundation) or os.path.isfile(input_inundation_2):
                     # If only one model (MS or FR) has inundation, simply copy over all files as the composite
-                    single_test_case = (
-                        self if os.path.isfile(input_inundation) else input_test_case_2
-                    )
+                    single_test_case = self if os.path.isfile(input_inundation) else input_test_case_2
                     shutil.copytree(
                         single_test_case.dir,
-                        re.sub(
-                            r"(.*)(_ms|_fr)", r"\1_comp", single_test_case.dir, count=1
-                        ),
+                        re.sub(r"(.*)(_ms|_fr)", r"\1_comp", single_test_case.dir, count=1),
                     )
                     composite_test_case.write_metadata(calibrated, "COMP")
                     return
 
             # Clean up 'total_area' outputs from AHPS sites
             if composite_test_case.is_ahps:
-                composite_test_case.clean_ahps_outputs(
-                    os.path.join(composite_test_case.dir, magnitude)
-                )
+                composite_test_case.clean_ahps_outputs(os.path.join(composite_test_case.dir, magnitude))
 
         composite_test_case.write_metadata(calibrated, "COMP")
 
@@ -567,10 +511,7 @@ def write_metadata(self, calibrated, model):
 
     def clean_ahps_outputs(self, magnitude_directory):
         """Cleans up `total_area` files from an input AHPS magnitude directory."""
-        output_file_list = [
-            os.path.join(magnitude_directory, of)
-            for of in os.listdir(magnitude_directory)
-        ]
+        output_file_list = [os.path.join(magnitude_directory, of) for of in os.listdir(magnitude_directory)]
         for output_file in output_file_list:
             if "total_area" in output_file:
                 os.remove(output_file)
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index 3092b7420..ceb7e427f 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -1,92 +1,127 @@
 #!/usr/bin/env python3
 
-import os, argparse, json, csv, ast, re, sys, traceback, signal
-import pandas as pd
+import argparse
+import ast
+import csv
+import json
+import os
+import re
+import signal
+import sys
+import traceback
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
 from datetime import datetime
 from multiprocessing import Pool
-from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+
+import pandas as pd
+from run_test_case import Test_Case
+from tools_shared_variables import (
+    AHPS_BENCHMARK_CATEGORIES,
+    MAGNITUDE_DICT,
+    OUTPUTS_DIR,
+    PREVIOUS_FIM_DIR,
+    TEST_CASES_DIR,
+)
 from tqdm import tqdm
+
 from utils.shared_functions import FIM_Helpers as fh
-from run_test_case import test_case
-from tools_shared_variables import TEST_CASES_DIR, PREVIOUS_FIM_DIR, OUTPUTS_DIR, AHPS_BENCHMARK_CATEGORIES, MAGNITUDE_DICT
 
-def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include_list, prev_versions_to_include_list, iteration_list, pfiles, prev_metrics_csv):
+
+def create_master_metrics_csv(
+    master_metrics_csv_output,
+    dev_versions_to_include_list,
+    prev_versions_to_include_list,
+    iteration_list,
+    pfiles,
+    prev_metrics_csv,
+):
     """
-    This function searches for and collates metrics into a single CSV file that can queried database-style. The
-    CSV is an input to eval_plots.py. This function automatically looks for metrics produced for official versions
-    and loads them into memory to be written to the output CSV.
-    
+    This function searches for and collates metrics into a single CSV file that can queried database-style.
+        The CSV is an input to eval_plots.py.
+        This function automatically looks for metrics produced for official versions and loads them into
+            memory to be written to the output CSV.
+
     Args:
-        master_metrics_csv_output (str): Full path to CSV output. If a file already exists at this path, it will be 
-        overwritten.
-        dev_versions_to_include_list (list): A list of non-official FIM version names. If a user supplied information 
-                                            on the command line using the -dc flag, then this function will search for 
-                                            metrics in the "testing_versions" library of metrics and include them in 
-                                            the CSV output.
+        master_metrics_csv_output (str)    : Full path to CSV output.
+                                                If a file already exists at this path, it will be overwritten.
+        dev_versions_to_include_list (list): A list of non-official FIM version names.
+                                                If a user supplied information on the command line using the
+                                                -dc flag, then this function will search for metrics in the
+                                                "testing_versions" library of metrics and include them in
+                                                the CSV output.
     """
-    
-    # Construct header  
-    metrics_to_write = ['true_negatives_count',
-                        'false_negatives_count',
-                        'true_positives_count',
-                        'false_positives_count',
-                        'contingency_tot_count',
-                        'cell_area_m2',
-                        'TP_area_km2',
-                        'FP_area_km2',
-                        'TN_area_km2',
-                        'FN_area_km2',
-                        'contingency_tot_area_km2',
-                        'predPositive_area_km2',
-                        'predNegative_area_km2',
-                        'obsPositive_area_km2',
-                        'obsNegative_area_km2',
-                        'positiveDiff_area_km2',
-                        'CSI',
-                        'FAR',
-                        'TPR',
-                        'TNR',
-                        'PND',
-                        'PPV',
-                        'NPV',
-                        'ACC',
-                        'Bal_ACC',
-                        'MCC',
-                        'EQUITABLE_THREAT_SCORE',
-                        'PREVALENCE',
-                        'BIAS',
-                        'F1_SCORE',
-                        'TP_perc',
-                        'FP_perc',
-                        'TN_perc',
-                        'FN_perc',
-                        'predPositive_perc',
-                        'predNegative_perc',
-                        'obsPositive_perc',
-                        'obsNegative_perc',
-                        'positiveDiff_perc',
-                        'masked_count',
-                        'masked_perc',
-                        'masked_area_km2'
-                        ]
+
+    # Construct header
+    metrics_to_write = [
+        'true_negatives_count',
+        'false_negatives_count',
+        'true_positives_count',
+        'false_positives_count',
+        'contingency_tot_count',
+        'cell_area_m2',
+        'TP_area_km2',
+        'FP_area_km2',
+        'TN_area_km2',
+        'FN_area_km2',
+        'contingency_tot_area_km2',
+        'predPositive_area_km2',
+        'predNegative_area_km2',
+        'obsPositive_area_km2',
+        'obsNegative_area_km2',
+        'positiveDiff_area_km2',
+        'CSI',
+        'FAR',
+        'TPR',
+        'TNR',
+        'PND',
+        'PPV',
+        'NPV',
+        'ACC',
+        'Bal_ACC',
+        'MCC',
+        'EQUITABLE_THREAT_SCORE',
+        'PREVALENCE',
+        'BIAS',
+        'F1_SCORE',
+        'TP_perc',
+        'FP_perc',
+        'TN_perc',
+        'FN_perc',
+        'predPositive_perc',
+        'predNegative_perc',
+        'obsPositive_perc',
+        'obsNegative_perc',
+        'positiveDiff_perc',
+        'masked_count',
+        'masked_perc',
+        'masked_area_km2',
+    ]
 
     # Create table header
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
-    list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source'] + ['extent_config'] + ["calibrated"]]
+    list_to_write = [
+        additional_header_info_prefix
+        + metrics_to_write
+        + ['full_json_path']
+        + ['flow']
+        + ['benchmark_source']
+        + ['extent_config']
+        + ["calibrated"]
+    ]
 
     # add in composite of versions (used for previous FIM3 versions)
     if "official" in iteration_list:
         composite_versions = [v.replace('_ms', '_comp') for v in prev_versions_to_include_list if '_ms' in v]
         prev_versions_to_include_list += composite_versions
 
-    # Iterate through 5 benchmark sources 
-    for benchmark_source in ['ble', 'nws', 'usgs', 'ifc','ras2fim']:
+    # Iterate through 5 benchmark sources
+    for benchmark_source in ['ble', 'nws', 'usgs', 'ifc', 'ras2fim']:
         benchmark_test_case_dir = os.path.join(TEST_CASES_DIR, benchmark_source + '_test_cases')
-        test_cases_list = [d for d in os.listdir(benchmark_test_case_dir) if re.match('\d{8}_\w{3,7}', d)]
+        test_cases_list = [d for d in os.listdir(benchmark_test_case_dir) if re.match(r'\d{8}_\w{3,7}', d)]
 
         if benchmark_source in ['ble', 'ifc', 'ras2fim']:
             magnitude_list = MAGNITUDE_DICT[benchmark_source]
-            
+
             # Iterate through available test cases
             for test_case in test_cases_list:
                 try:
@@ -96,12 +131,20 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
                     # Update filepaths based on whether the official or dev versions should be included
                     for iteration in iteration_list:
-                        if iteration == "official": # and str(pfiles) == "True": # "official" refers to previous finalized model versions
-                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
+                        if (
+                            iteration == "official"
+                        ):  # and str(pfiles) == "True": # "official" refers to previous finalized model versions
+                            versions_to_crawl = os.path.join(
+                                benchmark_test_case_dir, test_case, 'official_versions'
+                            )
                             versions_to_aggregate = prev_versions_to_include_list
-                            
-                        if iteration == "testing": # "testing" refers to the development model version(s) being evaluated
-                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
+
+                        if (
+                            iteration == "testing"
+                        ):  # "testing" refers to the development model version(s) being evaluated
+                            versions_to_crawl = os.path.join(
+                                benchmark_test_case_dir, test_case, 'testing_versions'
+                            )
                             versions_to_aggregate = dev_versions_to_include_list
 
                         # Pull version info from filepath
@@ -142,7 +185,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                 list_to_write.append(sub_list_to_append)
                 except ValueError:
                     pass
-        
+
         # Iterate through AHPS benchmark data
         if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
             test_cases_list = os.listdir(benchmark_test_case_dir)
@@ -155,12 +198,18 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
                     # Update filepaths based on whether the official or dev versions should be included
                     for iteration in iteration_list:
-                        if iteration == "official": # "official" refers to previous finalized model versions
-                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'official_versions')
+                        if iteration == "official":  # "official" refers to previous finalized model versions
+                            versions_to_crawl = os.path.join(
+                                benchmark_test_case_dir, test_case, 'official_versions'
+                            )
                             versions_to_aggregate = prev_versions_to_include_list
 
-                        if iteration == "testing": # "testing" refers to the development model version(s) being evaluated
-                            versions_to_crawl = os.path.join(benchmark_test_case_dir, test_case, 'testing_versions')
+                        if (
+                            iteration == "testing"
+                        ):  # "testing" refers to the development model version(s) being evaluated
+                            versions_to_crawl = os.path.join(
+                                benchmark_test_case_dir, test_case, 'testing_versions'
+                            )
                             versions_to_aggregate = dev_versions_to_include_list
 
                         # Pull model info from filepath
@@ -179,7 +228,7 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
 
                                 version_dir = os.path.join(versions_to_crawl, version)
                                 magnitude_dir = os.path.join(version_dir, magnitude)
-                                
+
                                 if os.path.exists(magnitude_dir):
                                     magnitude_dir_list = os.listdir(magnitude_dir)
                                     for f in magnitude_dir_list:
@@ -189,9 +238,21 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                             full_json_path = os.path.join(magnitude_dir, f)
                                             flow = ''
                                             if os.path.exists(full_json_path):
-
                                                 # Get flow used to map
-                                                flow_file = os.path.join(benchmark_test_case_dir, 'validation_data_' + benchmark_source, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
+                                                flow_file = os.path.join(
+                                                    benchmark_test_case_dir,
+                                                    'validation_data_' + benchmark_source,
+                                                    huc,
+                                                    nws_lid,
+                                                    magnitude,
+                                                    'ahps_'
+                                                    + nws_lid
+                                                    + '_huc_'
+                                                    + huc
+                                                    + '_flows_'
+                                                    + magnitude
+                                                    + '.csv',
+                                                )
                                                 if os.path.exists(flow_file):
                                                     with open(flow_file, newline='') as csv_file:
                                                         reader = csv.reader(csv_file)
@@ -210,10 +271,10 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
                                                 sub_list_to_append.append(calibrated)
                                                 list_to_write.append(sub_list_to_append)
                 except ValueError:
-                    pass 
+                    pass
 
     # If previous metrics are provided: read in previously compiled metrics and join to calcaulated metrics
-    if prev_metrics_csv != None:
+    if prev_metrics_csv is not None:
         prev_metrics_df = pd.read_csv(prev_metrics_csv)
 
         # Put calculated metrics into a dataframe and set the headers
@@ -229,28 +290,31 @@ def create_master_metrics_csv(master_metrics_csv_output, dev_versions_to_include
         df_to_write.columns = df_to_write.iloc[0]
         df_to_write = df_to_write[1:]
 
-
     # Save aggregated compiled metrics ('df_to_write') as a CSV
-    df_to_write.to_csv(master_metrics_csv_output,index=False)
+    df_to_write.to_csv(master_metrics_csv_output, index=False)
 
-def progress_bar_handler(executor_dict, verbose, desc):
 
-    for future in tqdm(as_completed(executor_dict),
-                    total=len(executor_dict),
-                    disable=(not verbose),
-                    desc=desc,
-                    ):
+def progress_bar_handler(executor_dict, verbose, desc):
+    for future in tqdm(
+        as_completed(executor_dict), total=len(executor_dict), disable=(not verbose), desc=desc
+    ):
         try:
             future.result()
         except Exception as exc:
-            print('{}, {}, {}'.format(executor_dict[future],exc.__class__.__name__,exc))
+            print('{}, {}, {}'.format(executor_dict[future], exc.__class__.__name__, exc))
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     # Sample usage:
     '''
      === FOR (FIM 4)
-    python /foss_fim/tools/synthesize_test_cases.py -c DEV -e GMS -v gms_test_synth_combined -jh 2 -jb 40 -m /outputs/gms_test_synth_combined/gms_synth_metrics.csv -vg -o
+    python /foss_fim/tools/synthesize_test_cases.py
+        -c DEV
+        -e GMS
+        -v gms_test_synth_combined
+        -jh 2 -jb 40
+        -m /outputs/gms_test_synth_combined/gms_synth_metrics.csv
+        -vg -o
 
      Notes:
        - fim_input.csv MUST be in the folder suggested.
@@ -262,10 +326,10 @@ def progress_bar_handler(executor_dict, verbose, desc):
        -jb 3 -jh 25 was noticably better. You can likely go more jb cores with better success, just
          experiment.  Start times, End Times and duration are now included.
        - The -m can be any path and any name.
-       - Previous metric CSV (-pcsv) and the cycle previous files argument (-pfiles) will return an error 
-         if called at the same time. If neither are used, the alpha test metrics will only be compiled 
-         for the provided dev version to compare. 
-    
+       - Previous metric CSV (-pcsv) and the cycle previous files argument (-pfiles) will return an error
+         if called at the same time. If neither are used, the alpha test metrics will only be compiled
+         for the provided dev version to compare.
+
      To see your outputs in the test_case folder (hard coded path), you can check for outputs using
          (cd .... to your test_case folder), then command becomes  find . -name gms_test_* -type d (Notice the
          the -name can be a wildcard for your -v param (or the whole -v value))
@@ -274,48 +338,141 @@ def progress_bar_handler(executor_dict, verbose, desc):
     '''
     '''
      === FOR FIM 3
-    python /foss_fim/tools/synthesize_test_cases.py -c DEV -e MS -v dev_fim_3_0_29_1_ms -jh 4 -m /outputs/dev_fim_3_0_29_1_ms/alpha/alpha_master_metrics_fim_3_0_29_1_ms_src_adjust.csv -vg -o
+    python /foss_fim/tools/synthesize_test_cases.py
+        -c DEV
+        -e MS
+        -v dev_fim_3_0_29_1_ms
+        -jh 4
+        -m /outputs/dev_fim_3_0_29_1_ms/alpha/alpha_master_metrics_fim_3_0_29_1_ms_src_adjust.csv
+        -vg -o
 
      Notes:
        - the -v param is the name in the folder in the "outputs/" directory where the test hucs are at.
            It also becomes the folder names inside the test_case folders when done.
        - the -vg param may not be working (will be assessed better on later releases).
        - The -m can be any path and any name.
-    
+
      To see your outputs in the test_case folder (hard coded path), you can check for outputs using
-         (cd .... to your test_case folder), then command becomes  find . -name dev_fim_3_0_29_1_* -type d (Notice the
-         the -name can be a wildcard for your -v param (or the whole -v value))
+         (cd .... to your test_case folder), then command becomes  find . -name dev_fim_3_0_29_1_* -type d
+         (Notice the the -name can be a wildcard for your -v param (or the whole -v value))
      If you want to delete the test outputs, test the outputs as suggest immediately above, but this time your
          command becomes:  find . -name dev_fim_3_0_29_1_* -type d  -exec rm -rdf {} +
     '''
 
     # Parse arguments.
     parser = argparse.ArgumentParser(description='Caches metrics from previous versions of HAND.')
-    parser.add_argument('-c','--config', help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"', 
-                        required=True, default='DEV')
-    parser.add_argument('-l','--calibrated', help='Denotes use of calibrated n values. This should be taken from meta-data from hydrofabric dir', 
-                        required=False, default=False, action='store_true')
-    parser.add_argument('-e','--model', help='Denotes model used. FR, MS, or GMS allowed. This should be taken from meta-data in hydrofabric dir.', 
-                        default='GMS', required=False)
-    parser.add_argument('-v','--fim-version', help='Name of fim version to cache.', required=False, default="all")
-    parser.add_argument('-jh','--job-number-huc', help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply '\
-                        'to no more than one less than the CPU count of the machine.', required=False, default=1, type=int)
-    parser.add_argument('-jb','--job-number-branch', help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should '\
-                        'multiply to no more than one less than the CPU count of the machine.', required=False, default=1, type=int)
-    parser.add_argument('-s','--special-string', help='Add a special name to the end of the branch.', required=False, default="")
-    parser.add_argument('-b','--benchmark-category', help='A benchmark category to specify. Defaults to process all categories.', required=False, 
-                        default="all")
-    parser.add_argument('-o','--overwrite', help='Overwrite all metrics or only fill in missing metrics.', required=False, action="store_true")
-    parser.add_argument('-dc', '--dev-version-to-compare', nargs='+', help='Specify the name(s) of a dev (testing) version to include in master '\
-                        'metrics CSV. Pass a space-delimited list.', required=False)
-    parser.add_argument('-m','--master-metrics-csv', help='Define path for master metrics CSV file.', required=False, default=None)
-    parser.add_argument('-d','--fr-run-dir', help='Name of test case directory containing FIM for FR model', required=False, default=None)
-    parser.add_argument('-vr','--verbose', help='Verbose', required=False, default=None, action='store_true')
-    parser.add_argument('-vg','--gms-verbose', help='GMS Verbose Progress Bar', required=False, default=None, action='store_true')
-    parser.add_argument('-pcsv', '--previous-metrics-csv', help='Optional: Filepath for a CSV with previous metrics to concatenate with new '\
-                        'metrics to form a final aggregated metrics csv.', required=False, default=None)
-    parser.add_argument('-pfiles', '--cycle-previous-files', help='Optional: Specifies whether previous metrics should be compiled by cycling '\
-                        'through files (True). Cannot be used if a previous metrics CSV is provided.', required=False, action="store_true")
+    parser.add_argument(
+        '-c',
+        '--config',
+        help='Save outputs to development_versions or previous_versions? Options: "DEV" or "PREV"',
+        required=True,
+        default='DEV',
+    )
+    parser.add_argument(
+        '-l',
+        '--calibrated',
+        help='Denotes use of calibrated n values. This should be taken from meta-data from hydrofabric dir',
+        required=False,
+        default=False,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-e',
+        '--model',
+        help='Denotes model used. Options: [FR, MS, or GMS]. '
+        'This should be taken from meta-data in hydrofabric dir.',
+        default='GMS',
+        required=False,
+    )
+    parser.add_argument(
+        '-v', '--fim-version', help='Name of fim version to cache.', required=False, default="all"
+    )
+    parser.add_argument(
+        '-jh',
+        '--job-number-huc',
+        help='Number of processes to use for HUC scale operations. HUC and Batch job numbers should multiply '
+        'to no more than one less than the CPU count of the machine.',
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        '-jb',
+        '--job-number-branch',
+        help='Number of processes to use for Branch scale operations. HUC and Batch job numbers should '
+        'multiply to no more than one less than the CPU count of the machine.',
+        required=False,
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        '-s',
+        '--special-string',
+        help='Add a special name to the end of the branch.',
+        required=False,
+        default="",
+    )
+    parser.add_argument(
+        '-b',
+        '--benchmark-category',
+        help='A benchmark category to specify. Defaults to process all categories.',
+        required=False,
+        default="all",
+    )
+    parser.add_argument(
+        '-o',
+        '--overwrite',
+        help='Overwrite all metrics or only fill in missing metrics.',
+        required=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        '-dc',
+        '--dev-version-to-compare',
+        nargs='+',
+        help='Specify the name(s) of a dev (testing) version to include in master '
+        'metrics CSV. Pass a space-delimited list.',
+        required=False,
+    )
+    parser.add_argument(
+        '-m',
+        '--master-metrics-csv',
+        help='Define path for master metrics CSV file.',
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-d',
+        '--fr-run-dir',
+        help='Name of test case directory containing FIM for FR model',
+        required=False,
+        default=None,
+    )
+    parser.add_argument('-vr', '--verbose', help='Verbose', required=False, default=None, action='store_true')
+    parser.add_argument(
+        '-vg',
+        '--gms-verbose',
+        help='GMS Verbose Progress Bar',
+        required=False,
+        default=None,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-pcsv',
+        '--previous-metrics-csv',
+        help='Optional: Filepath for a CSV with previous metrics to concatenate with new '
+        'metrics to form a final aggregated metrics csv.',
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        '-pfiles',
+        '--cycle-previous-files',
+        help='Optional: Specifies whether previous metrics should be compiled by cycling '
+        'through files (True). Cannot be used if a previous metrics CSV is provided.',
+        required=False,
+        action="store_true",
+    )
 
     # Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -340,30 +497,32 @@ def progress_bar_handler(executor_dict, verbose, desc):
     print("Start synthesize test cases")
     start_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"started: {dt_string}")
+    print(f"started: {dt_string}")
     print()
 
     # check job numbers
     total_cpus_requested = job_number_huc * job_number_branch
     total_cpus_available = os.cpu_count() - 1
     if total_cpus_requested > total_cpus_available:
-        raise ValueError('The HUC job number, {}, multiplied by the branch job number, {}, '\
-                          'exceeds your machine\'s available CPU count minus one. '\
-                          'Please lower the job_number_huc or job_number_branch'\
-                          'values accordingly.'.format(job_number_huc,job_number_branch)
-                        )
-
-    # Default to processing all possible versions in PREVIOUS_FIM_DIR. Otherwise, process only the user-supplied version.
+        raise ValueError(
+            'The HUC job number, {}, multiplied by the branch job number, {}, '
+            'exceeds your machine\'s available CPU count minus one. '
+            'Please lower the job_number_huc or job_number_branch'
+            'values accordingly.'.format(job_number_huc, job_number_branch)
+        )
+
+    # Default to processing all possible versions in PREVIOUS_FIM_DIR.
+    # Otherwise, process only the user-supplied version.
     prev_versions_to_include_list = []
     dev_versions_to_include_list = []
-    if fim_version != "all" and pfiles == False:
-        if config == 'PREV': # official fim model results
+    if fim_version != "all" and pfiles is False:
+        if config == 'PREV':  # official fim model results
             prev_versions_to_include_list = [fim_version]
-        elif config == 'DEV': # development fim model results
+        elif config == 'DEV':  # development fim model results
             dev_versions_to_include_list = [fim_version]
     else:
         prev_versions_to_include_list = os.listdir(PREVIOUS_FIM_DIR)
-        if config == 'DEV': # development fim model results
+        if config == 'DEV':  # development fim model results
             dev_versions_to_include_list = [fim_version]
 
     # Define whether or not to archive metrics in "official_versions" or "testing_versions" for each test_id.
@@ -375,14 +534,19 @@ def progress_bar_handler(executor_dict, verbose, desc):
         print('Config (-c) option incorrectly set. Use "DEV" or "PREV"')
 
     # Create a list of all test_cases for which we have validation data
-    all_test_cases = test_case.list_all_test_cases(version = fim_version, archive = archive_results,
-            benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
+    all_test_cases = Test_Case.list_all_test_cases(
+        version=fim_version,
+        archive=archive_results,
+        benchmark_categories=[] if benchmark_category == "all" else [benchmark_category],
+    )
 
     # print('all test cases', all_test_cases)
-    
+
     # Make sure cycle-previous-files and a previous metric CSV have not been concurrently selected
-    if prev_metrics_csv != None and pfiles == True:
-        print(f"Error: Cycle previous files and previous metric CSV functionality cannot be used concurrently.")
+    if prev_metrics_csv is not None and pfiles is True:
+        print(
+            "Error: Cycle previous files and previous metric CSV functionality cannot be used concurrently."
+        )
         sys.exit(1)
 
     # Check whether a previous metrics CSV has been provided and, if so, make sure the CSV exists
@@ -390,42 +554,43 @@ def progress_bar_handler(executor_dict, verbose, desc):
         if not os.path.exists(prev_metrics_csv):
             print(f"Error: File does not exist at {prev_metrics_csv}")
             sys.exit(1)
-        else: 
-            print(f"Metrics will be combined with previous metric CSV: {prev_metrics_csv}") 
+        else:
+            print(f"Metrics will be combined with previous metric CSV: {prev_metrics_csv}")
             print()
     else:
         print("ALERT: A previous metric CSV has not been provided (-pcsv) - this is optional.")
         print()
 
     # Print whether the previous files will be cycled through
-    if pfiles == True:
+    if pfiles is True:
         print("ALERT: Metrics from previous directories will be compiled.")
         print()
     else:
-        print("ALERT: Metrics from previous directories will NOT be compiled (-pfiles not provided) - this is optional")
+        print(
+            "ALERT: Metrics from previous directories will NOT be compiled (-pfiles not provided) \n"
+            "   - pfiles is optional -"
+        )
         print()
 
     # Set up multiprocessor
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
-
         # Loop through all test cases, build the alpha test arguments, and submit them to the process pool
         executor_dict = {}
 
         for test_case_class in all_test_cases:
-            
             if not os.path.exists(test_case_class.fim_dir):
                 continue
 
             fh.vprint(f"test_case_class.test_id is {test_case_class.test_id}", verbose)
 
-            alpha_test_args = { 
-                                'calibrated': calibrated,
-                                'model': model,
-                                'mask_type': 'huc',
-                                'overwrite': overwrite,
-                                'verbose': gms_verbose if model == 'GMS' else verbose,
-                                'gms_workers': job_number_branch
-                                }
+            alpha_test_args = {
+                'calibrated': calibrated,
+                'model': model,
+                'mask_type': 'huc',
+                'overwrite': overwrite,
+                'verbose': gms_verbose if model == 'GMS' else verbose,
+                'gms_workers': job_number_branch,
+            }
 
             try:
                 future = executor.submit(test_case_class.alpha_test, **alpha_test_args)
@@ -436,28 +601,32 @@ def progress_bar_handler(executor_dict, verbose, desc):
                 sys.exit(1)
 
         # Send the executor to the progress bar and wait for all MS tasks to finish
-        progress_bar_handler(executor_dict, True, f"Running {model} alpha test cases with {job_number_huc} workers")
-        #wait(executor_dict.keys())
+        progress_bar_handler(
+            executor_dict, True, f"Running {model} alpha test cases with {job_number_huc} workers"
+        )
+        # wait(executor_dict.keys())
 
     # Composite alpha test run is initiated by a MS `model` and providing a `fr_run_dir`
     if model == 'MS' and fr_run_dir:
-
         # Rebuild all test cases list with the FR version, loop through them and apply the alpha test
-        all_test_cases = test_case.list_all_test_cases(version = fr_run_dir, archive = archive_results,
-                benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
+        all_test_cases = Test_Case.list_all_test_cases(
+            version=fr_run_dir,
+            archive=archive_results,
+            benchmark_categories=[] if benchmark_category == "all" else [benchmark_category],
+        )
 
         with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
             executor_dict = {}
             for test_case_class in all_test_cases:
                 if not os.path.exists(test_case_class.fim_dir):
                     continue
-                alpha_test_args = { 
-                                    'calibrated': calibrated,
-                                    'model': model,
-                                    'mask_type': 'huc',
-                                    'verbose':verbose,
-                                    'overwrite': overwrite
-                                    }
+                alpha_test_args = {
+                    'calibrated': calibrated,
+                    'model': model,
+                    'mask_type': 'huc',
+                    'verbose': verbose,
+                    'overwrite': overwrite,
+                }
                 try:
                     future = executor.submit(test_case_class.alpha_test, **alpha_test_args)
                     executor_dict[future] = test_case_class.test_id
@@ -468,18 +637,19 @@ def progress_bar_handler(executor_dict, verbose, desc):
 
             # Send the executor to the progress bar and wait for all FR tasks to finish
             progress_bar_handler(executor_dict, True, f"Running FR test cases with {job_number_huc} workers")
-            #wait(executor_dict.keys())
+            # wait(executor_dict.keys())
 
-        # Loop through FR test cases, build composite arguments, and submit the composite method to the process pool
+        # Loop through FR test cases, build composite arguments, and
+        #   submit the composite method to the process pool
         with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
             executor_dict = {}
             for test_case_class in all_test_cases:
-                composite_args = { 
-                                    'version_2': fim_version, # this is the MS version name since `all_test_cases` are FR
-                                    'calibrated': calibrated,
-                                    'overwrite': overwrite,
-                                    'verbose': verbose
-                                    }
+                composite_args = {
+                    'version_2': fim_version,  # this is the MS version name since `all_test_cases` are FR
+                    'calibrated': calibrated,
+                    'overwrite': overwrite,
+                    'verbose': verbose,
+                }
 
                 try:
                     future = executor.submit(test_case_class.alpha_test, **alpha_test_args)
@@ -490,35 +660,43 @@ def progress_bar_handler(executor_dict, verbose, desc):
                     sys.exit(1)
 
             # Send the executor to the progress bar
-            progress_bar_handler(executor_dict, verbose, f"Compositing test cases with {job_number_huc} workers")
+            progress_bar_handler(
+                executor_dict, verbose, f"Compositing test cases with {job_number_huc} workers"
+            )
 
     ## if using DEV version, include the testing versions the user included with the "-dc" flag
-    if dev_versions_to_compare != None:
+    if dev_versions_to_compare is not None:
         dev_versions_to_include_list += dev_versions_to_compare
-    
+
     # Specify which results to iterate through
     if config == 'DEV':
-        iteration_list = ['official', 'testing'] # iterating through official model results AND testing model(s)
+        iteration_list = [
+            'official',
+            'testing',
+        ]  # iterating through official model results AND testing model(s)
     else:
-        iteration_list = ['official'] # only iterating through official model results
+        iteration_list = ['official']  # only iterating through official model results
 
     if master_metrics_csv is not None:
         # Do aggregate_metrics.
         print("Creating master metrics CSV...")
 
         # Note: This function is not compatible with GMS
-        create_master_metrics_csv(master_metrics_csv_output = master_metrics_csv, 
-                                  dev_versions_to_include_list = dev_versions_to_include_list,
-                                  prev_versions_to_include_list = prev_versions_to_include_list,
-                                  iteration_list = iteration_list,
-                                  pfiles=pfiles, prev_metrics_csv=prev_metrics_csv)
-    
+        create_master_metrics_csv(
+            master_metrics_csv_output=master_metrics_csv,
+            dev_versions_to_include_list=dev_versions_to_include_list,
+            prev_versions_to_include_list=prev_versions_to_include_list,
+            iteration_list=iteration_list,
+            pfiles=pfiles,
+            prev_metrics_csv=prev_metrics_csv,
+        )
+
     print("================================")
     print("End synthesize test cases")
 
     end_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"ended: {dt_string}")
+    print(f"ended: {dt_string}")
 
     # Calculate duration
     time_duration = end_time - start_time
diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py
index 7ff336cd7..65ca47e5a 100644
--- a/tools/test_case_by_hydro_id.py
+++ b/tools/test_case_by_hydro_id.py
@@ -1,26 +1,26 @@
 #!/usr/bin/env python3
 
-# Import file management library
-import os
-# Import data analysis library
-import pandas as pd
-import geopandas as gpd
 import argparse
+import os
 from datetime import datetime
 
+import geopandas as gpd
+import pandas as pd
 from pixel_counter import zonal_stats
-from tools_shared_functions import compute_stats_from_contingency_table
-from run_test_case import test_case
+from run_test_case import Test_Case
 from shapely.validation import make_valid
+from tools_shared_functions import compute_stats_from_contingency_table
+
 
 #####################################################
 # Perform zonal stats is a funtion stored in pixel_counter.py.
 # The input huc_gpkg is a single huc8 geopackage, the second input argument must be input as a dict.
-# For the purposes of assembling the alpha metrics by hydroid, always use agreement_raster total area agreement tiff.
+# For the purposes of assembling the alpha metrics by hydroid, always use agreement_raster total
+#   area agreement tiff.
 # This function is called automatically. Returns stats dict of pixel counts.
 #####################################################
-def perform_zonal_stats(huc_gpkg,agree_rast):
-    stats = zonal_stats(huc_gpkg,{"agreement_raster":agree_rast}, nodata_value=10)
+def perform_zonal_stats(huc_gpkg, agree_rast):
+    stats = zonal_stats(huc_gpkg, {"agreement_raster": agree_rast}, nodata_value=10)
     return stats
 
 
@@ -31,110 +31,171 @@ def perform_zonal_stats(huc_gpkg,agree_rast):
 # Mag is the magnitude (100y, action, minor etc.) is passed via the directory loop.
 # Bench is the benchmark source.
 #####################################################
-def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
-    in_mem_df = pd.DataFrame(columns=['HydroID', 'huc8','contingency_tot_area_km2',
-                                    'CSI', 'FAR', 'TPR', 'TNR',  'PPV', 'NPV', 'Bal_ACC',
-                                    'MCC', 'EQUITABLE_THREAT_SCORE', 'PREVALENCE', 'BIAS', 'F1_SCORE',
-                                    'masked_perc', 'MAG','BENCH'])
+def assemble_hydro_alpha_for_single_huc(stats, huc8, mag, bench):
+    in_mem_df = pd.DataFrame(
+        columns=[
+            'HydroID',
+            'huc8',
+            'contingency_tot_area_km2',
+            'CSI',
+            'FAR',
+            'TPR',
+            'TNR',
+            'PPV',
+            'NPV',
+            'Bal_ACC',
+            'MCC',
+            'EQUITABLE_THREAT_SCORE',
+            'PREVALENCE',
+            'BIAS',
+            'F1_SCORE',
+            'masked_perc',
+            'MAG',
+            'BENCH',
+        ]
+    )
 
     for dicts in stats:
-        tot_pop = dicts['tn'] +dicts['fn'] + dicts['fp'] + dicts['tp']
+        tot_pop = dicts['tn'] + dicts['fn'] + dicts['fp'] + dicts['tp']
         if tot_pop == 0:
             continue
-        
-        stats_dictionary = compute_stats_from_contingency_table(dicts['tn'], dicts['fn'], dicts['fp'], dicts['tp'], cell_area=100, masked_count= dicts['mp'])
+
+        stats_dictionary = compute_stats_from_contingency_table(
+            dicts['tn'], dicts['fn'], dicts['fp'], dicts['tp'], cell_area=100, masked_count=dicts['mp']
+        )
         # Calls compute_stats_from_contingency_table from run_test_case.py
-                    
+
         hydroid = dicts['HydroID']
         stats_dictionary['HydroID'] = hydroid
-               
+
         contingency_tot_area_km2 = float(stats_dictionary['contingency_tot_area_km2'])
         if contingency_tot_area_km2 != 'NA':
-            contingency_tot_area_km2 = round(contingency_tot_area_km2,2)
-       
+            contingency_tot_area_km2 = round(contingency_tot_area_km2, 2)
+
         CSI = stats_dictionary['CSI']
         if CSI != 'NA':
-            CSI = round(CSI,2)
+            CSI = round(CSI, 2)
 
         FAR = stats_dictionary['FAR']
         if FAR != 'NA':
-            FAR = round(FAR,2)
-        
+            FAR = round(FAR, 2)
+
         TPR = stats_dictionary['TPR']
         if TPR != 'NA':
-            TPR = round(TPR,2)
+            TPR = round(TPR, 2)
 
         TNR = stats_dictionary['TNR']
         if TNR != 'NA':
-            TNR = round(TNR,2)
+            TNR = round(TNR, 2)
 
         PPV = stats_dictionary['PPV']
         if PPV != 'NA':
-            PPV = round(PPV,2)
+            PPV = round(PPV, 2)
 
         NPV = stats_dictionary['NPV']
         if NPV != 'NA':
-            NPV = round(NPV,2)
+            NPV = round(NPV, 2)
 
         Bal_ACC = stats_dictionary['Bal_ACC']
         if Bal_ACC != 'NA':
-            Bal_ACC = round(Bal_ACC,2)
+            Bal_ACC = round(Bal_ACC, 2)
 
         MCC = float(stats_dictionary['MCC'])
         if MCC != 'NA':
-            MCC = round(MCC,2)
+            MCC = round(MCC, 2)
 
         EQUITABLE_THREAT_SCORE = stats_dictionary['EQUITABLE_THREAT_SCORE']
         if EQUITABLE_THREAT_SCORE != 'NA':
-            EQUITABLE_THREAT_SCORE = round(EQUITABLE_THREAT_SCORE,2)
+            EQUITABLE_THREAT_SCORE = round(EQUITABLE_THREAT_SCORE, 2)
 
         PREVALENCE = stats_dictionary['PREVALENCE']
         if PREVALENCE != 'NA':
-            PREVALENCE = round(PREVALENCE,2)
+            PREVALENCE = round(PREVALENCE, 2)
 
         BIAS = stats_dictionary['BIAS']
         if BIAS != 'NA':
-            BIAS = round(BIAS,2)   
+            BIAS = round(BIAS, 2)
 
         F1_SCORE = stats_dictionary['F1_SCORE']
         if F1_SCORE != 'NA':
-            F1_SCORE = round(F1_SCORE,2)    
+            F1_SCORE = round(F1_SCORE, 2)
 
         masked_perc = stats_dictionary['masked_perc']
         if masked_perc != 'NA':
-            masked_perc = round(masked_perc,2)
+            masked_perc = round(masked_perc, 2)
 
         HydroID = stats_dictionary['HydroID']
 
-        dict_with_list_values = {'HydroID': [HydroID],'huc8':[huc8], 'contingency_tot_area_km2': [contingency_tot_area_km2],
-        'CSI': [CSI], 'FAR': [FAR], 'TPR': [TPR], 'TNR': [TNR], 'PPV': [PPV], 'NPV': [NPV],
-        'Bal_ACC': [Bal_ACC], 'MCC': [MCC], 'EQUITABLE_THREAT_SCORE': [EQUITABLE_THREAT_SCORE], 'PREVALENCE': [PREVALENCE],
-        'BIAS': [BIAS], 'F1_SCORE': [F1_SCORE], 'masked_perc': [masked_perc], 'MAG':[mag],'BENCH':[bench]}
-        
-        dict_to_df = pd.DataFrame(dict_with_list_values,columns=['HydroID','huc8', 'contingency_tot_area_km2',
-        'CSI', 'FAR', 'TPR', 'TNR', 'PPV', 'NPV', 'Bal_ACC',
-        'MCC', 'EQUITABLE_THREAT_SCORE', 'PREVALENCE', 'BIAS', 'F1_SCORE', 'masked_perc', 'MAG','BENCH'])
-        
+        dict_with_list_values = {
+            'HydroID': [HydroID],
+            'huc8': [huc8],
+            'contingency_tot_area_km2': [contingency_tot_area_km2],
+            'CSI': [CSI],
+            'FAR': [FAR],
+            'TPR': [TPR],
+            'TNR': [TNR],
+            'PPV': [PPV],
+            'NPV': [NPV],
+            'Bal_ACC': [Bal_ACC],
+            'MCC': [MCC],
+            'EQUITABLE_THREAT_SCORE': [EQUITABLE_THREAT_SCORE],
+            'PREVALENCE': [PREVALENCE],
+            'BIAS': [BIAS],
+            'F1_SCORE': [F1_SCORE],
+            'masked_perc': [masked_perc],
+            'MAG': [mag],
+            'BENCH': [bench],
+        }
+
+        dict_to_df = pd.DataFrame(
+            dict_with_list_values,
+            columns=[
+                'HydroID',
+                'huc8',
+                'contingency_tot_area_km2',
+                'CSI',
+                'FAR',
+                'TPR',
+                'TNR',
+                'PPV',
+                'NPV',
+                'Bal_ACC',
+                'MCC',
+                'EQUITABLE_THREAT_SCORE',
+                'PREVALENCE',
+                'BIAS',
+                'F1_SCORE',
+                'masked_perc',
+                'MAG',
+                'BENCH',
+            ],
+        )
+
         concat_list = [in_mem_df, dict_to_df]
         in_mem_df = pd.concat(concat_list, sort=False)
-        
+
     return in_mem_df
 
 
 if __name__ == "__main__":
-    
     parser = argparse.ArgumentParser(description='Produces alpha metrics by hyrdoid.')
 
-    parser.add_argument('-b', '--benchmark_category',
-                        help='Choice of truth data. Options are: all, ble, ifc, nws, usgs, ras2fim',
-                        required=True)
-    parser.add_argument('-v', '--version',
-                        help='The fim version to use. Should be similar to fim_3_0_24_14_ms',
-                        required=True)
-    parser.add_argument('-g', '--gpkg',
-                        help='filepath and filename to hold exported gpkg (and csv) file. Similar to /data/path/fim_performance_catchments.gpkg Need to use gpkg as output. ',
-                        required=True)
-
+    parser.add_argument(
+        '-b',
+        '--benchmark_category',
+        help='Choice of truth data. Options are: all, ble, ifc, nws, usgs, ras2fim',
+        required=True,
+    )
+    parser.add_argument(
+        '-v', '--version', help='The fim version to use. Should be similar to fim_3_0_24_14_ms', required=True
+    )
+    parser.add_argument(
+        '-g',
+        '--gpkg',
+        help='filepath and filename to hold exported gpkg (and csv) file. '
+        'Similar to /data/path/fim_performance_catchments.gpkg Need to use gpkg as output.',
+        required=True,
+    )
 
     # Assign variables from arguments.
     args = vars(parser.parse_args())
@@ -146,68 +207,94 @@ def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
     print("Start test_case_by_hydroid.py")
     start_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"started: {dt_string}")
+    print(f"started: {dt_string}")
     print()
-        
-    # Execution code
-    csv_output = gpd.GeoDataFrame(columns=['HydroID', 'huc8','contingency_tot_area_km2',
-                                    'CSI', 'FAR', 'TPR', 'TNR',  'PPV', 'NPV', 'Bal_ACC',
-                                    'MCC', 'EQUITABLE_THREAT_SCORE', 'PREVALENCE', 'BIAS', 'F1_SCORE',
-                                    'masked_perc', 'MAG','BENCH','geometry'], geometry = 'geometry')
 
-    # This funtion, relies on the test_case class defined in run_test_case.py to list all available test cases
+    # Execution code
+    csv_output = gpd.GeoDataFrame(
+        columns=[
+            'HydroID',
+            'huc8',
+            'contingency_tot_area_km2',
+            'CSI',
+            'FAR',
+            'TPR',
+            'TNR',
+            'PPV',
+            'NPV',
+            'Bal_ACC',
+            'MCC',
+            'EQUITABLE_THREAT_SCORE',
+            'PREVALENCE',
+            'BIAS',
+            'F1_SCORE',
+            'masked_perc',
+            'MAG',
+            'BENCH',
+            'geometry',
+        ],
+        geometry='geometry',
+    )
+
+    # This funtion, relies on the Test_Case class defined in run_test_case.py to list all available test cases
     print('listing_test_cases_with_updates')
-    all_test_cases = test_case.list_all_test_cases(version=version, archive=True, benchmark_categories=[] if benchmark_category == "all" else [benchmark_category])
-    
-    for test_case_class in all_test_cases:
+    all_test_cases = Test_Case.list_all_test_cases(
+        version=version,
+        archive=True,
+        benchmark_categories=[] if benchmark_category == "all" else [benchmark_category],
+    )
 
+    for test_case_class in all_test_cases:
         if not os.path.exists(test_case_class.fim_dir):
             print(f'{test_case_class.fim_dir} does not exist')
             continue
-            
+
         print(test_case_class.fim_dir, end='\r')
-                   
+
         agreement_dict = test_case_class.get_current_agreements()
-       
+
         for agree_rast in agreement_dict:
-                
-                print(f'performing_zonal_stats for {agree_rast}')
-
-                branches_dir = os.path.join(test_case_class.fim_dir,'branches')
-                for branches in os.listdir(branches_dir):
-                    if branches != "0":
-                        continue
-                    huc_gpkg = os.path.join(branches_dir,branches,)
-                                            
-                    string_manip = "gw_catchments_reaches_filtered_addedAttributes_crosswalked_" + branches + ".gpkg"
-                    
-                    huc_gpkg = os.path.join(huc_gpkg, string_manip)
-                    
-                    define_mag = agree_rast.split(version)
-
-                    define_mag_1 = define_mag[1].split('/')
-                    
-                    mag = define_mag_1[1]
-               
-                    stats = perform_zonal_stats(huc_gpkg,agree_rast)
-                    if stats == []:
-                        continue
-                    
-                    print('assembling_hydroalpha_for_single_huc')
-                    get_geom = gpd.read_file(huc_gpkg)
-                    
-                    get_geom['geometry'] = get_geom.apply(lambda row: make_valid(row.geometry), axis=1)
-                                        
-                    in_mem_df = assemble_hydro_alpha_for_single_huc(stats, test_case_class.huc, mag, test_case_class.benchmark_cat)
-                    
-                    hydro_geom_df = get_geom[["HydroID", "geometry"]]
-                    
-                    geom_output = hydro_geom_df.merge(in_mem_df, on='HydroID', how ='inner')
-                    
-                    concat_df_list = [geom_output, csv_output]
-
-                    csv_output = pd.concat(concat_df_list, sort=False)
-                                           
+            print(f'performing_zonal_stats for {agree_rast}')
+
+            branches_dir = os.path.join(test_case_class.fim_dir, 'branches')
+            for branches in os.listdir(branches_dir):
+                if branches != "0":
+                    continue
+                huc_gpkg = os.path.join(branches_dir, branches)
+
+                string_manip = (
+                    "gw_catchments_reaches_filtered_addedAttributes_crosswalked_" + branches + ".gpkg"
+                )
+
+                huc_gpkg = os.path.join(huc_gpkg, string_manip)
+
+                define_mag = agree_rast.split(version)
+
+                define_mag_1 = define_mag[1].split('/')
+
+                mag = define_mag_1[1]
+
+                stats = perform_zonal_stats(huc_gpkg, agree_rast)
+                if stats == []:
+                    continue
+
+                print('assembling_hydroalpha_for_single_huc')
+                get_geom = gpd.read_file(huc_gpkg)
+
+                get_geom['geometry'] = get_geom.apply(lambda row: make_valid(row.geometry), axis=1)
+
+                in_mem_df = assemble_hydro_alpha_for_single_huc(
+                    stats, test_case_class.huc, mag, test_case_class.benchmark_cat
+                )
+
+                hydro_geom_df = get_geom[["HydroID", "geometry"]]
+
+                geom_output = hydro_geom_df.merge(in_mem_df, on='HydroID', how='inner')
+
+                concat_df_list = [geom_output, csv_output]
+
+                csv_output = pd.concat(concat_df_list, sort=False)
+
     print('projecting to 3857')
     csv_output = csv_output.to_crs('EPSG:3857')
 
@@ -215,27 +302,24 @@ def assemble_hydro_alpha_for_single_huc(stats,huc8,mag,bench):
     csv_path_list = csv.split(".")
     csv_path = csv_path_list[0]
     csv_path_dot = csv_path + ".csv"
-    
+
     print('writing_to_gpkg')
     csv_output.to_file(csv, driver="GPKG")
-    
+
     # Add version information to csv_output dataframe
     csv_output['version'] = version
 
     print('writing_to_csv')
-    csv_output.to_csv(csv_path_dot) # Save to CSV
+    csv_output.to_csv(csv_path_dot)  # Save to CSV
 
     print("================================")
     print("End test_case_by_hydroid.py")
 
     end_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print (f"ended: {dt_string}")
+    print(f"ended: {dt_string}")
 
     # calculate duration
     time_duration = end_time - start_time
     print(f"Duration: {str(time_duration).split('.')[0]}")
     print()
-
-
-  
\ No newline at end of file
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index bb755bc29..08af7ee9e 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -1,32 +1,27 @@
 #!/usr/bin/env python3
-
-import os
 import json
+import os
 import pathlib
 from pathlib import Path
 
-import requests
+import geopandas as gpd
 import numpy as np
 import pandas as pd
-import geopandas as gpd
 import rasterio
-import rasterio.shutil
-from rasterio.warp import calculate_default_transform, reproject, Resampling
 import rasterio.crs
-from rasterio import features
-from shapely.geometry import shape
-from shapely.geometry import Polygon
-from shapely.geometry import MultiPolygon
-from dotenv import load_dotenv
+import rasterio.shutil
+import requests
 import rioxarray as rxr
 import xarray as xr
+from dotenv import load_dotenv
 from geocube.api.core import make_geocube
 from gval import CatStats
+from rasterio.warp import Resampling, calculate_default_transform, reproject
 
 
 def get_env_paths():
     load_dotenv()
-    #import variables from .env file
+    # import variables from .env file
     API_BASE_URL = os.getenv("API_BASE_URL")
     WBD_LAYER = os.getenv("WBD_LAYER")
     return API_BASE_URL, WBD_LAYER
@@ -34,29 +29,28 @@ def get_env_paths():
 
 def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df):
     """
-    This function uses the WRDS API to filter out NWM segments from a list if their stream order is different than
-    the target stream order.
-    
+    This function uses the WRDS API to filter out NWM segments from a list if their stream order is
+    different than the target stream order.
+
     Args:
         unfiltered_segments (list):  A list of NWM feature_id strings.
-        desired_order (str): The desired stream order.
+        desired_order (str):         The desired stream order.
     Returns:
-        filtered_segments (list): A list of NWM feature_id strings, paired down to only those that share the target order.        
-    
+        filtered_segments (list):    A list of NWM feature_id strings, paired down to only those that
+                                        share the target order.
+
     """
-        
-#    API_BASE_URL, WBD_LAYER = get_env_paths()
-    #Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
-#    metadata_url = f'{API_BASE_URL}/metadata'
-        
-    
+
+    #    API_BASE_URL, WBD_LAYER = get_env_paths()
+    # Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
+    #    metadata_url = f'{API_BASE_URL}/metadata'
 
     # feature ID of 0 is getting passed to WRDS and returns empty results,
-    # which can cause failures on next() 
-#    if '0' in unfiltered_segments:
-#        unfiltered_segments = unfiltered_segments.remove('0')
-#    if unfiltered_segments is None:
-#        return filtered_segments
+    # which can cause failures on next()
+    #    if '0' in unfiltered_segments:
+    #        unfiltered_segments = unfiltered_segments.remove('0')
+    #    if unfiltered_segments is None:
+    #        return filtered_segments
 
     filtered_segments = []
 
@@ -68,8 +62,9 @@ def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_
     return filtered_segments
 
 
-def check_for_regression(stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None):
-
+def check_for_regression(
+    stats_json_to_test, previous_version, previous_version_stats_json_path, regression_test_csv=None
+):
     difference_dict = {}
 
     # Compare stats_csv to previous_version_stats_file
@@ -84,13 +79,19 @@ def check_for_regression(stats_json_to_test, previous_version, previous_version_
     return difference_dict
 
 
-def compute_contingency_stats_from_rasters(predicted_raster_path: str, benchmark_raster_path: str,
-                                           agreement_raster: str = None, stats_csv: str = None,
-                                           stats_json: str = None, mask_dict: dict = {}):
+def compute_contingency_stats_from_rasters(
+    predicted_raster_path: str,
+    benchmark_raster_path: str,
+    agreement_raster: str = None,
+    stats_csv: str = None,
+    stats_json: str = None,
+    mask_dict: dict = {},
+):
     """
     This function contains FIM-specific logic to prepare raster datasets for use in the generic
     get_stats_table_from_binary_rasters() function. This function also calls the generic
-    compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
+    compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON,
+    depending on user input.
 
     Parameters
     ----------
@@ -99,44 +100,42 @@ def compute_contingency_stats_from_rasters(predicted_raster_path: str, benchmark
     benchmark_raster_path: str
         The path to the benchmark, or truth, FIM extent raster.
     agreement_raster: str, optional
-        An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive,
-        3: True Positive.
+        An agreement raster will be written to this path. 0: True Negatives, 1: False Negative,
+        2: False Positive, 3: True Positive.
     stats_csv: str, optional
-        Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
+        Performance statistics will be written to this path. CSV allows for readability and
+        other tabular processes.
     stats_json: str, optional
-        Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary
-        in other processes.
+        Performance statistics will be written to this path. JSON allows for quick ingestion into Python
+        dictionary in other processes.
     mask_dict: dict
         Dictionary with inclusionary and/or exclusionary masks asn options.
 
     Returns
     -------
     dict
-        A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and
-        statistic values are the values.
+        A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are
+        keys and statistic values are the values.
     """
 
     # Get statistics table from two rasters.
-    stats_dictionary = get_stats_table_from_binary_rasters(benchmark_raster_path,
-                                                                 predicted_raster_path,
-                                                                 agreement_raster,
-                                                                 mask_dict=mask_dict)
+    stats_dictionary = get_stats_table_from_binary_rasters(
+        benchmark_raster_path, predicted_raster_path, agreement_raster, mask_dict=mask_dict
+    )
 
     for stats_mode in stats_dictionary:
-
         # Write the mode_stats_dictionary to the stats_csv.
-        if stats_csv != None:
+        if stats_csv is not None:
             stats_csv = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.csv')
             df = pd.DataFrame.from_dict(stats_dictionary[stats_mode], orient="index", columns=['value'])
             df.to_csv(stats_csv)
 
         # Write the mode_stats_dictionary to the stats_json.
-        if stats_json != None:
+        if stats_json is not None:
             stats_json = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.json')
             with open(stats_json, "w") as outfile:
                 json.dump(stats_dictionary[stats_mode], outfile)
 
-
     return stats_dictionary
 
 
@@ -145,13 +144,15 @@ def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
     This function searches multiple directories and locates previously produced performance statistics.
 
     Args:
-        archive_to_check (str): The directory path to search.
-        magnitude (str): Because a benchmark dataset may have multiple magnitudes, this argument defines
-                               which magnitude is to be used when searching for previous statistics.
+        archive_to_check (str):    The directory path to search.
+        magnitude (str):           Because a benchmark dataset may have multiple magnitudes,
+                                   this argument defines which magnitude is to be used when searching for
+                                   previous statistics.
     Returns:
-        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain and magnitude.
-                                  {version: {agreement_raster: agreement_raster_path, stats_csv: stats_csv_path, stats_json: stats_json_path}}
-                                  *Will only add the paths to files that exist.
+        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain
+                                   and magnitude. {version: {agreement_raster: agreement_raster_path,
+                                   stats_csv: stats_csv_path, stats_json: stats_json_path}}
+                                   *Will only add the paths to files that exist.
 
     """
 
@@ -174,28 +175,33 @@ def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
     return archive_dictionary
 
 
-def compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives,
-                                         cell_area=None, masked_count=None):
+def compute_stats_from_contingency_table(
+    true_negatives, false_negatives, false_positives, true_positives, cell_area=None, masked_count=None
+):
     """
-    This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.
-    Much of the calculations below were taken from older Python files. This is evident in the inconsistent use of case.
+    This generic function takes contingency table metrics as arguments and returns a dictionary of contingency
+    table statistics. Much of the calculations below were taken from older Python files.
+    This is evident in the inconsistent use of case.
 
     Args:
-        true_negatives (int): The true negatives from a contingency table.
-        false_negatives (int): The false negatives from a contingency table.
-        false_positives (int): The false positives from a contingency table.
-        true_positives (int): The true positives from a contingency table.
-        cell_area (float or None): This optional argument allows for area-based statistics to be calculated, in the case that
-                                   contingency table metrics were derived from areal analysis.
+        true_negatives (int):      The true negatives from a contingency table.
+        false_negatives (int):     The false negatives from a contingency table.
+        false_positives (int):     The false positives from a contingency table.
+        true_positives (int):      The true positives from a contingency table.
+        cell_area (float or None): This optional argument allows for area-based statistics to be calculated,
+                                   in the case that contingency table metrics were derived from areal
+                                   analysis.
 
     Returns:
-        stats_dictionary (dict): A dictionary of statistics. Statistic names are keys and statistic values are the values.
-                                 Refer to dictionary definition in bottom of function for statistic names.
+        stats_dictionary (dict):   A dictionary of statistics. Statistic names are keys and statistic values
+                                   are the values. Refer to dictionary definition in bottom of function for
+                                   statistic names.
 
     """
 
-    vals, keys = CatStats.process_statistics(func_names="all", tp=true_positives, tn=true_negatives, fp=false_positives,
-                                             fn=false_negatives)
+    vals, keys = CatStats.process_statistics(
+        func_names="all", tp=true_positives, tn=true_negatives, fp=false_positives, fn=false_negatives
+    )
     alt_keys = ['band', 'tn', 'fn', 'fp', 'tp']
     alt_vals = [1, true_negatives, false_negatives, false_positives, true_positives]
 
@@ -205,15 +211,13 @@ def compute_stats_from_contingency_table(true_negatives, false_negatives, false_
 
     metrics_table = pd.DataFrame({x: [y] for x, y in zip(keys, vals)})
 
-    return cross_walk_gval_fim(metric_df=metrics_table,
-                               cell_area=cell_area,
-                               masked_count=masked_count)
+    return cross_walk_gval_fim(metric_df=metrics_table, cell_area=cell_area, masked_count=masked_count)
 
 
 def cross_walk_gval_fim(metric_df: pd.DataFrame, cell_area: int, masked_count: int) -> dict:
     """
     Crosswalks metrics made from GVAL to standard FIM names and conventions
-    
+
     Parameters
     ----------
     metric_df: pd.DataFrame
@@ -233,37 +237,42 @@ def cross_walk_gval_fim(metric_df: pd.DataFrame, cell_area: int, masked_count: i
     metric_df = metric_df.iloc[:, 1:]
 
     # Dictionary to crosswalk column names
-    crosswalk = {'tn': 'true_negatives_count',
-                 'fn': 'false_negatives_count',
-                 'fp': 'false_positives_count',
-                 'tp': 'true_positives_count',
-                 'accuracy': 'ACC',
-                 'balanced_accuracy': 'Bal_ACC',
-                 'critical_success_index': 'CSI',
-                 'equitable_threat_score': 'EQUITABLE_THREAT_SCORE',
-                 'f_score': 'F1_SCORE',
-                 'false_discovery_rate': 'FAR',
-                 'false_negative_rate': 'PND',
-                 'false_omission_rate': 'FALSE_OMISSION_RATE',
-                 'false_positive_rate': 'FALSE_POSITIVE_RATE',
-                 'fowlkes_mallows_index': 'FOWLKES_MALLOW_INDEX',
-                 'matthews_correlation_coefficient': 'MCC',
-                 'negative_likelihood_ratio': 'NEGATIVE_LIKELIHOOD_RATIO',
-                 'negative_predictive_value': 'NPV',
-                 'overall_bias': 'BIAS',
-                 'positive_likelihood_ratio': 'POSITIVE_LIKELIHOOD_RATIO',
-                 'positive_predictive_value': 'PPV',
-                 'prevalence': 'PREVALENCE',
-                 'prevalence_threshold': 'PREVALENCE_THRESHOLD',
-                 'true_negative_rate': 'TNR',
-                 'true_positive_rate': 'TPR'
-                 }
+    crosswalk = {
+        'tn': 'true_negatives_count',
+        'fn': 'false_negatives_count',
+        'fp': 'false_positives_count',
+        'tp': 'true_positives_count',
+        'accuracy': 'ACC',
+        'balanced_accuracy': 'Bal_ACC',
+        'critical_success_index': 'CSI',
+        'equitable_threat_score': 'EQUITABLE_THREAT_SCORE',
+        'f_score': 'F1_SCORE',
+        'false_discovery_rate': 'FAR',
+        'false_negative_rate': 'PND',
+        'false_omission_rate': 'FALSE_OMISSION_RATE',
+        'false_positive_rate': 'FALSE_POSITIVE_RATE',
+        'fowlkes_mallows_index': 'FOWLKES_MALLOW_INDEX',
+        'matthews_correlation_coefficient': 'MCC',
+        'negative_likelihood_ratio': 'NEGATIVE_LIKELIHOOD_RATIO',
+        'negative_predictive_value': 'NPV',
+        'overall_bias': 'BIAS',
+        'positive_likelihood_ratio': 'POSITIVE_LIKELIHOOD_RATIO',
+        'positive_predictive_value': 'PPV',
+        'prevalence': 'PREVALENCE',
+        'prevalence_threshold': 'PREVALENCE_THRESHOLD',
+        'true_negative_rate': 'TNR',
+        'true_positive_rate': 'TPR',
+    }
 
     metric_df.columns = [crosswalk[x] for x in metric_df.columns]
 
     # Build
-    tn, fn, tp, fp = (metric_df['true_negatives_count'].values[0], metric_df['false_negatives_count'].values[0],
-                      metric_df['true_positives_count'].values[0], metric_df['false_positives_count'].values[0])
+    tn, fn, tp, fp = (
+        metric_df['true_negatives_count'].values[0],
+        metric_df['false_negatives_count'].values[0],
+        metric_df['true_positives_count'].values[0],
+        metric_df['false_positives_count'].values[0],
+    )
     total_population = tn + fn + tp + fp
     metric_df['contingency_tot_count'] = total_population
 
@@ -285,15 +294,27 @@ def cross_walk_gval_fim(metric_df: pd.DataFrame, cell_area: int, masked_count: i
     metric_df['FP_area_km2'] = (fp * cell_area) / sq_km_converter if cell_area is not None else None
     metric_df['TN_area_km2'] = (tn * cell_area) / sq_km_converter if cell_area is not None else None
     metric_df['FN_area_km2'] = (fn * cell_area) / sq_km_converter if cell_area is not None else None
-    metric_df['contingency_tot_area_km2'] = (total_population * cell_area) \
-                                            / sq_km_converter if cell_area is not None else None
-
-    metric_df['predPositive_area_km2'] = (predPositive * cell_area) / sq_km_converter if cell_area is not None else None
-    metric_df['predNegative_area_km2'] = (predNegative * cell_area) / sq_km_converter if cell_area is not None else None
-    metric_df['obsPositive_area_km2'] = (obsPositive * cell_area) / sq_km_converter if cell_area is not None else None
-    metric_df['obsNegative_area_km2'] = (obsNegative * cell_area) / sq_km_converter if cell_area is not None else None
-    metric_df['positiveDiff_area_km2'] = (metric_df['predPositive_area_km2'] - metric_df['obsPositive_area_km2'])[
-        0] if cell_area is not None else None
+    metric_df['contingency_tot_area_km2'] = (
+        (total_population * cell_area) / sq_km_converter if cell_area is not None else None
+    )
+
+    metric_df['predPositive_area_km2'] = (
+        (predPositive * cell_area) / sq_km_converter if cell_area is not None else None
+    )
+    metric_df['predNegative_area_km2'] = (
+        (predNegative * cell_area) / sq_km_converter if cell_area is not None else None
+    )
+    metric_df['obsPositive_area_km2'] = (
+        (obsPositive * cell_area) / sq_km_converter if cell_area is not None else None
+    )
+    metric_df['obsNegative_area_km2'] = (
+        (obsNegative * cell_area) / sq_km_converter if cell_area is not None else None
+    )
+    metric_df['positiveDiff_area_km2'] = (
+        (metric_df['predPositive_area_km2'] - metric_df['obsPositive_area_km2'])[0]
+        if cell_area is not None
+        else None
+    )
 
     total_pop_and_mask_pop = total_population + masked_count if masked_count > 0 else None
     metric_df['masked_count'] = masked_count if masked_count > 0 else 0
@@ -303,18 +324,21 @@ def cross_walk_gval_fim(metric_df: pd.DataFrame, cell_area: int, masked_count: i
     metric_df['predNegative_perc'] = (predNegative / total_population) * 100 if total_population > 0 else "NA"
     metric_df['obsPositive_perc'] = (obsPositive / total_population) * 100 if total_population > 0 else "NA"
     metric_df['obsNegative_perc'] = (obsNegative / total_population) * 100 if total_population > 0 else "NA"
-    metric_df['positiveDiff_perc'] = metric_df['predPositive_perc'].values[0] - metric_df['obsPositive_perc'].values[
-        0] if total_population > 0 else "NA"
+    metric_df['positiveDiff_perc'] = (
+        metric_df['predPositive_perc'].values[0] - metric_df['obsPositive_perc'].values[0]
+        if total_population > 0
+        else "NA"
+    )
 
     return {x: y for x, y in zip(metric_df.columns, metric_df.values[0])}
 
 
-def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
-                                        candidate_raster_path: str,
-                                        agreement_raster: str = None,
-                                        mask_dict: dict = {}):
+def get_stats_table_from_binary_rasters(
+    benchmark_raster_path: str, candidate_raster_path: str, agreement_raster: str = None, mask_dict: dict = {}
+):
     """
-    Produces categorical statistics table from 2 rasters and returns it. Also exports an agreement raster classified as:
+    Produces categorical statistics table from 2 rasters and returns it.
+    Also exports an agreement raster classified as:
         0: True Negatives
         1: False Negative
         2: False Positive
@@ -325,7 +349,8 @@ def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
     Parameters
     ----------
     benchmark_raster_path: str
-        Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
+        Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present,
+        NoData = NoData.
     candidate_raster_path: str
         Path to the predicted raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
     agreement_raster: str, default = None.
@@ -360,19 +385,16 @@ def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
         (4, np.nan): np.nan,
         (np.nan, 0): np.nan,
         (np.nan, 1): np.nan,
-        (np.nan, np.nan): np.nan
+        (np.nan, np.nan): np.nan,
     }
 
-
     # Loop through exclusion masks and mask the agreement_array.
     rasterized_mask_list = []
     if mask_dict != {}:
         for poly_layer in mask_dict:
-
             operation = mask_dict[poly_layer]['operation']
 
             if operation == 'exclude':
-
                 poly_path = mask_dict[poly_layer]['path']
                 buffer_val = mask_dict[poly_layer]['buffer']
 
@@ -402,21 +424,23 @@ def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
         all_masks = xr.merge(rasterized_mask_list).to_array()
         # Hold on to original data
 
-        candidate_raster.data = xr.where((all_masks.data == 4) & (candidate_raster.isnull() == 0), 4, candidate_raster)
+        candidate_raster.data = xr.where(
+            (all_masks.data == 4) & (candidate_raster.isnull() == 0), 4, candidate_raster
+        )
 
     stats_table_dictionary = {}  # Initialize empty dictionary.
 
-    (agreement_map,
-     crosstab_table,
-     metrics_table) = candidate_raster.gval.categorical_compare(benchmark_map=benchmark_raster,
-                                                                positive_categories=[1],
-                                                                target_map="candidate",
-                                                                negative_categories=[0],
-                                                                comparison_function='pairing_dict',
-                                                                pairing_dict=pairing_dictionary)
+    (agreement_map, crosstab_table, metrics_table) = candidate_raster.gval.categorical_compare(
+        benchmark_map=benchmark_raster,
+        positive_categories=[1],
+        target_map="candidate",
+        negative_categories=[0],
+        comparison_function='pairing_dict',
+        pairing_dict=pairing_dictionary,
+    )
 
     # Only write the agreement raster if user-specified.
-    if agreement_raster != None:
+    if agreement_raster is not None:
         agreement_map = agreement_map.rio.write_nodata(10, encoded=True)
         agreement_map.rio.to_raster(agreement_raster, dtype=np.int32, driver="COG")
 
@@ -433,25 +457,29 @@ def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
             f.write("%s\n" % '1: False Negative')
             f.write("%s\n" % '2: False Positive')
             f.write("%s\n" % '3: True Positive')
-            f.write("%s\n" % '4: Masked area (excluded from contingency table analysis). '
-                             'Mask layers: {mask_dict}'.format(mask_dict=mask_dict))
+            f.write(
+                "%s\n" % '4: Masked area (excluded from contingency table analysis). '
+                'Mask layers: {mask_dict}'.format(mask_dict=mask_dict)
+            )
             f.write("%s\n" % 'Results produced at: {current_time}'.format(current_time=current_time))
 
     # Store summed pixel counts in dictionary.
-    stats_table_dictionary.update({'total_area': cross_walk_gval_fim(metric_df=metrics_table,
-                                                                     cell_area=cell_area,
-                                                                     masked_count=np.sum(agreement_map.data == 4))})
+    stats_table_dictionary.update(
+        {
+            'total_area': cross_walk_gval_fim(
+                metric_df=metrics_table, cell_area=cell_area, masked_count=np.sum(agreement_map.data == 4)
+            )
+        }
+    )
 
     del agreement_map, crosstab_table, metrics_table, all_masks, rasterized_mask_list
 
     # After agreement_array is masked with default mask layers, check for inclusion masks in mask_dict.
     if mask_dict != {}:
         for poly_layer in mask_dict:
-
             operation = mask_dict[poly_layer]['operation']
 
             if operation == 'include':
-
                 poly_path = mask_dict[poly_layer]['path']
                 buffer_val = mask_dict[poly_layer]['buffer']
 
@@ -473,32 +501,39 @@ def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
 
                 mask = make_geocube(poly_all_proj, ['mask'], like=candidate_raster).to_array()
                 candidate_raster.data = og_data
-                candidate_raster.data = xr.where((mask.data != 4) & (candidate_raster.isnull() == 0), 4, candidate_raster)
+                candidate_raster.data = xr.where(
+                    (mask.data != 4) & (candidate_raster.isnull() == 0), 4, candidate_raster
+                )
 
                 poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
 
                 # Do analysis on inclusion masked area
-                (agreement_map,
-                 crosstab_table,
-                 metrics_table) = candidate_raster.gval.categorical_compare(benchmark_map=benchmark_raster,
-                                                                            positive_categories=[1],
-                                                                            target_map="candidate",
-                                                                            negative_categories=[0],
-                                                                            comparison_function='pairing_dict',
-                                                                            pairing_dict=pairing_dictionary)
+                (agreement_map, crosstab_table, metrics_table) = candidate_raster.gval.categorical_compare(
+                    benchmark_map=benchmark_raster,
+                    positive_categories=[1],
+                    target_map="candidate",
+                    negative_categories=[0],
+                    comparison_function='pairing_dict',
+                    pairing_dict=pairing_dictionary,
+                )
                 if agreement_raster:
                     # Write the layer_agreement_raster.
-                    layer_agreement_raster = os.path.join(os.path.split(agreement_raster)[0],
-                                                          poly_handle + '_agreement.tif')
+                    layer_agreement_raster = os.path.join(
+                        os.path.split(agreement_raster)[0], poly_handle + '_agreement.tif'
+                    )
                     agreement_map = agreement_map.rio.write_nodata(10, encoded=True)
                     agreement_map.rio.to_raster(layer_agreement_raster, dtype=np.int32, driver="COG")
 
                 # Update stats table dictionary
-                stats_table_dictionary.update({poly_handle: cross_walk_gval_fim(metric_df=metrics_table,
-                                                                                cell_area=cell_area,
-                                                                                masked_count=np.sum(
-                                                                                    agreement_map.data == 4
-                                                                                ))})
+                stats_table_dictionary.update(
+                    {
+                        poly_handle: cross_walk_gval_fim(
+                            metric_df=metrics_table,
+                            cell_area=cell_area,
+                            masked_count=np.sum(agreement_map.data == 4),
+                        )
+                    }
+                )
 
                 del poly_all, poly_all_proj, agreement_map, metrics_table, crosstab_table
 
@@ -506,11 +541,19 @@ def get_stats_table_from_binary_rasters(benchmark_raster_path: str,
 
     return stats_table_dictionary
 
+
 ########################################################################
 ########################################################################
-#Functions related to categorical fim and ahps evaluation
+# Functions related to categorical fim and ahps evaluation
 ########################################################################
-def get_metadata(metadata_url, select_by, selector, must_include = None, upstream_trace_distance = None, downstream_trace_distance = None ):
+def get_metadata(
+    metadata_url,
+    select_by,
+    selector,
+    must_include=None,
+    upstream_trace_distance=None,
+    downstream_trace_distance=None,
+):
     '''
     Retrieve metadata for a site or list of sites.
 
@@ -538,31 +581,31 @@ def get_metadata(metadata_url, select_by, selector, must_include = None, upstrea
 
     '''
 
-    #Format selector variable in case multiple selectors supplied
+    # Format selector variable in case multiple selectors supplied
     format_selector = '%2C'.join(selector)
-    #Define the url
+    # Define the url
     url = f'{metadata_url}/{select_by}/{format_selector}/'
-    #Assign optional parameters to a dictionary
+    # Assign optional parameters to a dictionary
     params = {}
     params['must_include'] = must_include
     params['upstream_trace_distance'] = upstream_trace_distance
     params['downstream_trace_distance'] = downstream_trace_distance
-    #Request data from url
-    response = requests.get(url, params = params, verify=False)
-#    print(response)
-#    print(url)
+    # Request data from url
+    response = requests.get(url, params=params, verify=False)
+    #    print(response)
+    #    print(url)
     if response.ok:
-        #Convert data response to a json
+        # Convert data response to a json
         metadata_json = response.json()
-        #Get the count of returned records
+        # Get the count of returned records
         location_count = metadata_json['_metrics']['location_count']
-        #Get metadata
+        # Get metadata
         metadata_list = metadata_json['locations']
-        #Add timestamp of WRDS retrieval
+        # Add timestamp of WRDS retrieval
         timestamp = response.headers['Date']
-        #Add timestamp of sources retrieval
+        # Add timestamp of sources retrieval
         timestamp_list = metadata_json['data_sources']['metadata_sources']
-        
+
         # Default timestamps to "Not available" and overwrite with real values if possible.
         nwis_timestamp, nrldb_timestamp = "Not available", "Not available"
         for timestamp in timestamp_list:
@@ -570,31 +613,32 @@ def get_metadata(metadata_url, select_by, selector, must_include = None, upstrea
                 nwis_timestamp = timestamp
             if "NRLDB" in timestamp:
                 nrldb_timestamp = timestamp
-            
-#        nrldb_timestamp, nwis_timestamp = metadata_json['data_sources']['metadata_sources']        
-        #get crosswalk info (always last dictionary in list)
+
+        #        nrldb_timestamp, nwis_timestamp = metadata_json['data_sources']['metadata_sources']
+        # get crosswalk info (always last dictionary in list)
         crosswalk_info = metadata_json['data_sources']
-        #Update each dictionary with timestamp and crosswalk info also save to DataFrame.
+        # Update each dictionary with timestamp and crosswalk info also save to DataFrame.
         for metadata in metadata_list:
             metadata.update({"wrds_timestamp": timestamp})
-            metadata.update({"nrldb_timestamp":nrldb_timestamp})
-            metadata.update({"nwis_timestamp":nwis_timestamp})
+            metadata.update({"nrldb_timestamp": nrldb_timestamp})
+            metadata.update({"nwis_timestamp": nwis_timestamp})
             metadata.update(crosswalk_info)
         metadata_dataframe = pd.json_normalize(metadata_list)
-        #Replace all periods with underscores in column names
-        metadata_dataframe.columns = metadata_dataframe.columns.str.replace('.','_')
+        # Replace all periods with underscores in column names
+        metadata_dataframe.columns = metadata_dataframe.columns.str.replace('.', '_')
     else:
-        #if request was not succesful, print error message.
+        # if request was not succesful, print error message.
         print(f'Code: {response.status_code}\nMessage: {response.reason}\nURL: {response.url}')
-        #Return empty outputs
+        # Return empty outputs
         metadata_list = []
         metadata_dataframe = pd.DataFrame()
     return metadata_list, metadata_dataframe
 
+
 ########################################################################
-#Function to assign HUC code using the WBD spatial layer using a spatial join
+# Function to assign HUC code using the WBD spatial layer using a spatial join
 ########################################################################
-def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
+def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes=False):
     '''
     Assigns the proper FIM HUC 08 code to each site in the input DataFrame.
     Converts input DataFrame to a GeoDataFrame using lat/lon attributes
@@ -622,71 +666,86 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes = False):
         GeoDataFrame of all NWS_LID sites.
 
     '''
-    #Import huc8 layer as geodataframe and retain necessary columns
+    # Import huc8 layer as geodataframe and retain necessary columns
     print("Reading WBD...")
-    huc8 = gpd.read_file(wbd_huc8_path, layer = 'WBDHU8')
+    huc8 = gpd.read_file(wbd_huc8_path, layer='WBDHU8')
     print("WBD read.")
-    huc8 = huc8[['HUC8','name','states', 'geometry']]
-    #Define EPSG codes for possible latlon datum names (default of NAD83 if unassigned)
-    crs_lookup ={'NAD27':'EPSG:4267', 'NAD83':'EPSG:4269', 'WGS84': 'EPSG:4326'}
-    #Create empty geodataframe and define CRS for potential horizontal datums
+    huc8 = huc8[['HUC8', 'name', 'states', 'geometry']]
+    # Define EPSG codes for possible latlon datum names (default of NAD83 if unassigned)
+    crs_lookup = {'NAD27': 'EPSG:4267', 'NAD83': 'EPSG:4269', 'WGS84': 'EPSG:4326'}
+    # Create empty geodataframe and define CRS for potential horizontal datums
     metadata_gdf = gpd.GeoDataFrame()
-    #Iterate through each site
+    # Iterate through each site
     print("Iterating through metadata list...")
     for metadata in metadata_list:
-        #Convert metadata to json
+        # Convert metadata to json
         df = pd.json_normalize(metadata)
-        #Columns have periods due to nested dictionaries
+        # Columns have periods due to nested dictionaries
         df.columns = df.columns.str.replace('.', '_')
-        #Drop any metadata sites that don't have lat/lon populated
-        df.dropna(subset = ['identifiers_nws_lid','usgs_preferred_latitude', 'usgs_preferred_longitude'], inplace = True)
-        #If dataframe still has data
+        # Drop any metadata sites that don't have lat/lon populated
+        df.dropna(
+            subset=['identifiers_nws_lid', 'usgs_preferred_latitude', 'usgs_preferred_longitude'],
+            inplace=True,
+        )
+        # If dataframe still has data
         if not df.empty:
-#            print(df[:5])
-            #Get horizontal datum
+            #            print(df[:5])
+            # Get horizontal datum
             h_datum = df['usgs_preferred_latlon_datum_name'].item()
-            #Look up EPSG code, if not returned Assume NAD83 as default. 
-            dict_crs = crs_lookup.get(h_datum,'EPSG:4269_ Assumed')
-            #We want to know what sites were assumed, hence the split.
+            # Look up EPSG code, if not returned Assume NAD83 as default.
+            dict_crs = crs_lookup.get(h_datum, 'EPSG:4269_ Assumed')
+            # We want to know what sites were assumed, hence the split.
             src_crs, *message = dict_crs.split('_')
-            #Convert dataframe to geodataframe using lat/lon (USGS). Add attribute of assigned crs (label ones that are assumed)
-            site_gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df['usgs_preferred_longitude'], df['usgs_preferred_latitude']), crs =  src_crs)
-            #Field to indicate if a latlon datum was assumed
+            # Convert dataframe to geodataframe using lat/lon (USGS).
+            #   Add attribute of assigned crs (label ones that are assumed)
+            site_gdf = gpd.GeoDataFrame(
+                df,
+                geometry=gpd.points_from_xy(df['usgs_preferred_longitude'], df['usgs_preferred_latitude']),
+                crs=src_crs,
+            )
+            # Field to indicate if a latlon datum was assumed
             site_gdf['assigned_crs'] = src_crs + ''.join(message)
-            
-            #Reproject to huc 8 crs
+
+            # Reproject to huc 8 crs
             site_gdf = site_gdf.to_crs(huc8.crs)
-            #Append site geodataframe to metadata geodataframe
-            metadata_gdf = pd.concat([metadata_gdf, site_gdf], ignore_index = True)
-    
-    #Trim metadata to only have certain fields.
+            # Append site geodataframe to metadata geodataframe
+            metadata_gdf = pd.concat([metadata_gdf, site_gdf], ignore_index=True)
+
+    # Trim metadata to only have certain fields.
     if not retain_attributes:
-        metadata_gdf = metadata_gdf[['identifiers_nwm_feature_id', 'identifiers_nws_lid', 'identifiers_usgs_site_code', 'geometry']]
-    #If a list of attributes is supplied then use that list.
-#    elif isinstance(retain_attributes,list):
-#        metadata_gdf = metadata_gdf[retain_attributes]
+        metadata_gdf = metadata_gdf[
+            ['identifiers_nwm_feature_id', 'identifiers_nws_lid', 'identifiers_usgs_site_code', 'geometry']
+        ]
+    # If a list of attributes is supplied then use that list.
+    #    elif isinstance(retain_attributes,list):
+    #        metadata_gdf = metadata_gdf[retain_attributes]
     print("Performing spatial and tabular operations on geodataframe...")
-    #Perform a spatial join to get the WBD HUC 8 assigned to each AHPS
-    joined_gdf = gpd.sjoin(metadata_gdf, huc8, how = 'inner', predicate = 'intersects', lsuffix = 'ahps', rsuffix = 'wbd')
-    joined_gdf = joined_gdf.drop(columns = 'index_wbd')
+    # Perform a spatial join to get the WBD HUC 8 assigned to each AHPS
+    joined_gdf = gpd.sjoin(
+        metadata_gdf, huc8, how='inner', predicate='intersects', lsuffix='ahps', rsuffix='wbd'
+    )
+    joined_gdf = joined_gdf.drop(columns='index_wbd')
 
-    #Remove all Alaska HUCS (Not in NWM v2.0 domain)
+    # Remove all Alaska HUCS (Not in NWM v2.0 domain)
     joined_gdf = joined_gdf[~joined_gdf.states.str.contains('AK')]
 
-    #Create a dictionary of huc [key] and nws_lid[value]
+    # Create a dictionary of huc [key] and nws_lid[value]
     dictionary = joined_gdf.groupby('HUC8')['identifiers_nws_lid'].apply(list).to_dict()
 
     return dictionary, joined_gdf
 
+
 ########################################################################
 def mainstem_nwm_segs(metadata_url, list_of_sites):
     '''
     Define the mainstems network. Currently a 4 pass approach that probably needs refined.
     Once a final method is decided the code can be shortened. Passes are:
-        1) Search downstream of gages designated as upstream. This is done to hopefully reduce the issue of mapping starting at the nws_lid. 91038 segments
+        1) Search downstream of gages designated as upstream.
+            This is done to hopefully reduce the issue of mapping starting at the nws_lid. 91038 segments
         2) Search downstream of all LID that are rfc_forecast_point = True. Additional 48,402 segments
         3) Search downstream of all evaluated sites (sites with detailed FIM maps) Additional 222 segments
-        4) Search downstream of all sites in HI/PR (locations have no rfc_forecast_point = True) Additional 408 segments
+        4) Search downstream of all sites in HI/PR (locations have no rfc_forecast_point = True)
+            Additional 408 segments
 
     Parameters
     ----------
@@ -702,56 +761,85 @@ def mainstem_nwm_segs(metadata_url, list_of_sites):
 
     '''
 
-    #Define the downstream trace distance
+    # Define the downstream trace distance
     downstream_trace_distance = 'all'
 
-    #Trace downstream from all 'headwater' usgs gages
+    # Trace downstream from all 'headwater' usgs gages
     select_by = 'tag'
     selector = ['usgs_gages_ii_ref_headwater']
     must_include = None
-    gages_list, gages_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
-
-    #Trace downstream from all rfc_forecast_point.
+    gages_list, gages_dataframe = get_metadata(
+        metadata_url=metadata_url,
+        select_by=select_by,
+        selector=selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=downstream_trace_distance,
+    )
+
+    # Trace downstream from all rfc_forecast_point.
     select_by = 'nws_lid'
     selector = ['all']
     must_include = 'nws_data.rfc_forecast_point'
-    fcst_list, fcst_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
-
-    #Trace downstream from all evaluated ahps sites.
+    fcst_list, fcst_dataframe = get_metadata(
+        metadata_url=metadata_url,
+        select_by=select_by,
+        selector=selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=downstream_trace_distance,
+    )
+
+    # Trace downstream from all evaluated ahps sites.
     select_by = 'nws_lid'
     selector = list_of_sites
     must_include = None
-    eval_list, eval_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
-
-    #Trace downstream from all sites in HI/PR.
+    eval_list, eval_dataframe = get_metadata(
+        metadata_url=metadata_url,
+        select_by=select_by,
+        selector=selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=downstream_trace_distance,
+    )
+
+    # Trace downstream from all sites in HI/PR.
     select_by = 'state'
-    selector = ['HI','PR']
+    selector = ['HI', 'PR']
     must_include = None
-    islands_list, islands_dataframe = get_metadata(metadata_url = metadata_url, select_by = select_by, selector = selector, must_include = must_include, upstream_trace_distance = None, downstream_trace_distance = downstream_trace_distance )
-
-    #Combine all lists of metadata dictionaries into a single list.
+    islands_list, islands_dataframe = get_metadata(
+        metadata_url=metadata_url,
+        select_by=select_by,
+        selector=selector,
+        must_include=must_include,
+        upstream_trace_distance=None,
+        downstream_trace_distance=downstream_trace_distance,
+    )
+
+    # Combine all lists of metadata dictionaries into a single list.
     combined_lists = gages_list + fcst_list + eval_list + islands_list
-    #Define list that will contain all segments listed in metadata.
+    # Define list that will contain all segments listed in metadata.
     all_nwm_segments = []
-    #For each lid metadata dictionary in list
+    # For each lid metadata dictionary in list
     for lid in combined_lists:
-        #get all downstream segments
+        # get all downstream segments
         downstream_nwm_segs = lid.get('downstream_nwm_features')
-        #Append downstream segments
+        # Append downstream segments
         if downstream_nwm_segs:
             all_nwm_segments.extend(downstream_nwm_segs)
-        #Get the nwm feature id associated with the location
+        # Get the nwm feature id associated with the location
         location_nwm_seg = lid.get('identifiers').get('nwm_feature_id')
         if location_nwm_seg:
-            #Append nwm segment (conver to list)
+            # Append nwm segment (conver to list)
             all_nwm_segments.extend([location_nwm_seg])
-    #Remove duplicates by assigning to a set.
+    # Remove duplicates by assigning to a set.
     ms_nwm_segs_set = set(all_nwm_segments)
 
     return ms_nwm_segs_set
 
+
 ##############################################################################
-#Function to create list of NWM segments
+# Function to create list of NWM segments
 ###############################################################################
 def get_nwm_segs(metadata):
     '''
@@ -774,23 +862,24 @@ def get_nwm_segs(metadata):
     downstream_nwm_features = metadata.get('downstream_nwm_features')
 
     all_segments = []
-    #Convert NWM feature id segment to a list (this is always a string or empty)
+    # Convert NWM feature id segment to a list (this is always a string or empty)
     if nwm_feature_id:
         nwm_feature_id = [nwm_feature_id]
         all_segments.extend(nwm_feature_id)
-    #Add all upstream segments (always a list or empty)
+    # Add all upstream segments (always a list or empty)
     if upstream_nwm_features:
         all_segments.extend(upstream_nwm_features)
-    #Add all downstream segments (always a list or empty)
+    # Add all downstream segments (always a list or empty)
     if downstream_nwm_features:
         all_segments.extend(downstream_nwm_features)
 
     return all_segments
 
+
 #######################################################################
-#Thresholds
+# Thresholds
 #######################################################################
-def get_thresholds(threshold_url, select_by, selector, threshold = 'all'):
+def get_thresholds(threshold_url, select_by, selector, threshold='all'):
     '''
     Get nws_lid threshold stages and flows (i.e. bankfull, action, minor,
     moderate, major). Returns a dictionary for stages and one for flows.
@@ -817,38 +906,42 @@ def get_thresholds(threshold_url, select_by, selector, threshold = 'all'):
     params = {}
     params['threshold'] = threshold
     url = f'{threshold_url}/{select_by}/{selector}'
-    response = requests.get(url, params = params, verify=False)
+    response = requests.get(url, params=params, verify=False)
     if response.ok:
         thresholds_json = response.json()
-        #Get metadata
+        # Get metadata
         thresholds_info = thresholds_json['value_set']
-        #Initialize stages/flows dictionaries
+        # Initialize stages/flows dictionaries
         stages = {}
         flows = {}
-        #Check if thresholds information is populated. If site is non-existent thresholds info is blank
+        # Check if thresholds information is populated. If site is non-existent thresholds info is blank
         if thresholds_info:
-            #Get all rating sources and corresponding indexes in a dictionary
-            rating_sources = {i.get('calc_flow_values').get('rating_curve').get('source'): index for index, i in enumerate(thresholds_info)}
-            #Get threshold data use USGS Rating Depot (priority) otherwise NRLDB.
+            # Get all rating sources and corresponding indexes in a dictionary
+            rating_sources = {
+                i.get('calc_flow_values').get('rating_curve').get('source'): index
+                for index, i in enumerate(thresholds_info)
+            }
+            # Get threshold data use USGS Rating Depot (priority) otherwise NRLDB.
             if 'USGS Rating Depot' in rating_sources:
                 threshold_data = thresholds_info[rating_sources['USGS Rating Depot']]
             elif 'NRLDB' in rating_sources:
                 threshold_data = thresholds_info[rating_sources['NRLDB']]
-            #If neither USGS or NRLDB is available use first dictionary to get stage values.
+            # If neither USGS or NRLDB is available use first dictionary to get stage values.
             else:
                 threshold_data = thresholds_info[0]
-            #Get stages and flows for each threshold
+            # Get stages and flows for each threshold
             if threshold_data:
                 stages = threshold_data['stage_values']
                 flows = threshold_data['calc_flow_values']
-                #Add source information to stages and flows. Flows source inside a nested dictionary. Remove key once source assigned to flows.
+                # Add source information to stages and flows. Flows source inside a nested dictionary.
+                # Remove key once source assigned to flows.
                 stages['source'] = threshold_data.get('metadata').get('threshold_source')
                 flows['source'] = flows.get('rating_curve', {}).get('source')
                 flows.pop('rating_curve', None)
-                #Add timestamp WRDS data was retrieved.
+                # Add timestamp WRDS data was retrieved.
                 stages['wrds_timestamp'] = response.headers['Date']
                 flows['wrds_timestamp'] = response.headers['Date']
-                #Add Site information
+                # Add Site information
                 stages['nws_lid'] = threshold_data.get('metadata').get('nws_lid')
                 flows['nws_lid'] = threshold_data.get('metadata').get('nws_lid')
                 stages['usgs_site_code'] = threshold_data.get('metadata').get('usgs_site_code')
@@ -858,12 +951,15 @@ def get_thresholds(threshold_url, select_by, selector, threshold = 'all'):
         return stages, flows
     else:
         print("WRDS response error: ")
+
+
 #        print(response)
 
+
 ########################################################################
 # Function to write flow file
 ########################################################################
-def flow_data(segments, flows, convert_to_cms = True):
+def flow_data(segments, flows, convert_to_cms=True):
     '''
     Given a list of NWM segments and a flow value in cfs, convert flow to
     cms and return a DataFrame that is set up for export to a flow file.
@@ -885,30 +981,32 @@ def flow_data(segments, flows, convert_to_cms = True):
 
     '''
     if convert_to_cms:
-        #Convert cfs to cms
+        # Convert cfs to cms
         cfs_to_cms = 0.3048**3
-        flows_cms = round(flows * cfs_to_cms,2)
+        flows_cms = round(flows * cfs_to_cms, 2)
     else:
-        flows_cms = round(flows,2)
+        flows_cms = round(flows, 2)
 
-    flow_data = pd.DataFrame({'feature_id':segments, 'discharge':flows_cms})
-    flow_data = flow_data.astype({'feature_id' : int , 'discharge' : float})
+    flow_data = pd.DataFrame({'feature_id': segments, 'discharge': flows_cms})
+    flow_data = flow_data.astype({'feature_id': int, 'discharge': float})
     return flow_data
+
+
 #######################################################################
-#Function to get datum information
+# Function to get datum information
 #######################################################################
 def get_datum(metadata):
     '''
     Given a record from the metadata endpoint, retrieve important information
     related to the datum and site from both NWS and USGS sources. This information
     is saved to a dictionary with common keys. USGS has more data available so
-    it has more keys. 
+    it has more keys.
 
     Parameters
     ----------
     metadata : DICT
         Single record from the get_metadata function. Must iterate through
-        the get_metadata output list. 
+        the get_metadata output list.
 
     Returns
     -------
@@ -918,7 +1016,7 @@ def get_datum(metadata):
         Dictionary of USGS Data.
 
     '''
-    #Get site and datum information from nws sub-dictionary. Use consistent naming between USGS and NWS sources.
+    # Get site and datum information from nws sub-dictionary. Use consistent naming between USGS and NWS sources.
     nws_datums = {}
     nws_datums['nws_lid'] = metadata['identifiers']['nws_lid']
     nws_datums['usgs_site_code'] = metadata['identifiers']['usgs_site_code']
@@ -929,8 +1027,9 @@ def get_datum(metadata):
     nws_datums['lon'] = metadata['nws_data']['longitude']
     nws_datums['crs'] = metadata['nws_data']['horizontal_datum_name']
     nws_datums['source'] = 'nws_data'
-    
-    #Get site and datum information from usgs_data sub-dictionary. Use consistent naming between USGS and NWS sources.
+
+    # Get site and datum information from usgs_data sub-dictionary.
+    # Use consistent naming between USGS and NWS sources.
     usgs_datums = {}
     usgs_datums['nws_lid'] = metadata['identifiers']['nws_lid']
     usgs_datums['usgs_site_code'] = metadata['identifiers']['usgs_site_code']
@@ -943,15 +1042,17 @@ def get_datum(metadata):
     usgs_datums['lat'] = metadata['usgs_data']['latitude']
     usgs_datums['lon'] = metadata['usgs_data']['longitude']
     usgs_datums['crs'] = metadata['usgs_data']['latlon_datum_name']
-    usgs_datums['source'] = 'usgs_data' 
-    
+    usgs_datums['source'] = 'usgs_data'
+
     return nws_datums, usgs_datums
+
+
 ########################################################################
-#Function to convert horizontal datums
+# Function to convert horizontal datums
 ########################################################################
-def convert_latlon_datum(lat,lon,src_crs,dest_crs):
+def convert_latlon_datum(lat, lon, src_crs, dest_crs):
     '''
-    Converts latitude and longitude datum from a source CRS to a dest CRS 
+    Converts latitude and longitude datum from a source CRS to a dest CRS
     using geopandas and returns the projected latitude and longitude coordinates.
 
     Parameters
@@ -973,24 +1074,26 @@ def convert_latlon_datum(lat,lon,src_crs,dest_crs):
         Reprojected longitude coordinate in dest_crs.
 
     '''
-    
-    #Create a temporary DataFrame containing the input lat/lon.
-    temp_df = pd.DataFrame({'lat':[lat],'lon':[lon]})
-    #Convert dataframe to a GeoDataFrame using the lat/lon coords. Input CRS is assigned.
-    temp_gdf = gpd.GeoDataFrame(temp_df, geometry=gpd.points_from_xy(temp_df.lon, temp_df.lat), crs =  src_crs)
-    #Reproject GeoDataFrame to destination CRS.
+
+    # Create a temporary DataFrame containing the input lat/lon.
+    temp_df = pd.DataFrame({'lat': [lat], 'lon': [lon]})
+    # Convert dataframe to a GeoDataFrame using the lat/lon coords. Input CRS is assigned.
+    temp_gdf = gpd.GeoDataFrame(temp_df, geometry=gpd.points_from_xy(temp_df.lon, temp_df.lat), crs=src_crs)
+    # Reproject GeoDataFrame to destination CRS.
     reproject = temp_gdf.to_crs(dest_crs)
-    #Get new Lat/Lon coordinates from the geometry data.
-    new_lat,new_lon = [reproject.geometry.y.item(), reproject.geometry.x.item()]
+    # Get new Lat/Lon coordinates from the geometry data.
+    new_lat, new_lon = [reproject.geometry.y.item(), reproject.geometry.x.item()]
     return new_lat, new_lon
+
+
 #######################################################################
-#Function to get conversion adjustment NGVD to NAVD in FEET
+# Function to get conversion adjustment NGVD to NAVD in FEET
 #######################################################################
-def ngvd_to_navd_ft(datum_info, region = 'contiguous'):
+def ngvd_to_navd_ft(datum_info, region='contiguous'):
     '''
-    Given the lat/lon, retrieve the adjustment from NGVD29 to NAVD88 in feet. 
+    Given the lat/lon, retrieve the adjustment from NGVD29 to NAVD88 in feet.
     Uses NOAA tidal API to get conversion factor. Requires that lat/lon is
-    in NAD27 crs. If input lat/lon are not NAD27 then these coords are 
+    in NAD27 crs. If input lat/lon are not NAD27 then these coords are
     reprojected to NAD27 and the reproject coords are used to get adjustment.
     There appears to be an issue when region is not in contiguous US.
 
@@ -1007,49 +1110,50 @@ def ngvd_to_navd_ft(datum_info, region = 'contiguous'):
         Vertical adjustment in feet, from NGVD29 to NAVD88, and rounded to nearest hundredth.
 
     '''
-    #If crs is not NAD 27, convert crs to NAD27 and get adjusted lat lon
+    # If crs is not NAD 27, convert crs to NAD27 and get adjusted lat lon
     if datum_info['crs'] != 'NAD27':
-        lat, lon = convert_latlon_datum(datum_info['lat'],datum_info['lon'],datum_info['crs'],'NAD27')
+        lat, lon = convert_latlon_datum(datum_info['lat'], datum_info['lon'], datum_info['crs'], 'NAD27')
     else:
-        #Otherwise assume lat/lon is in NAD27.
+        # Otherwise assume lat/lon is in NAD27.
         lat = datum_info['lat']
         lon = datum_info['lon']
-    
-    #Define url for datum API
-    datum_url = 'https://vdatum.noaa.gov/vdatumweb/api/convert'     
-    
-    #Define parameters. Hard code most parameters to convert NGVD to NAVD.    
+
+    # Define url for datum API
+    datum_url = 'https://vdatum.noaa.gov/vdatumweb/api/convert'
+
+    # Define parameters. Hard code most parameters to convert NGVD to NAVD.
     params = {}
     params['lat'] = lat
     params['lon'] = lon
     params['region'] = region
-    params['s_h_frame'] = 'NAD27'     #Source CRS
-    params['s_v_frame'] = 'NGVD29'    #Source vertical coord datum
-    params['s_vertical_unit'] = 'm'   #Source vertical units
-    params['src_height'] = 0.0        #Source vertical height
-    params['t_v_frame'] = 'NAVD88'    #Target vertical datum
-    params['tar_vertical_unit'] = 'm' #Target vertical height
-    
-    #Call the API
-    response = requests.get(datum_url, params = params, verify=False)
-
-    #If successful get the navd adjustment
+    params['s_h_frame'] = 'NAD27'  # Source CRS
+    params['s_v_frame'] = 'NGVD29'  # Source vertical coord datum
+    params['s_vertical_unit'] = 'm'  # Source vertical units
+    params['src_height'] = 0.0  # Source vertical height
+    params['t_v_frame'] = 'NAVD88'  # Target vertical datum
+    params['tar_vertical_unit'] = 'm'  # Target vertical height
+
+    # Call the API
+    response = requests.get(datum_url, params=params, verify=False)
+
+    # If successful get the navd adjustment
     if response:
         results = response.json()
-        #Get adjustment in meters (NGVD29 to NAVD88)
+        # Get adjustment in meters (NGVD29 to NAVD88)
         adjustment = results['t_z']
-        #convert meters to feet
-        adjustment_ft = round(float(adjustment) * 3.28084,2)                
+        # convert meters to feet
+        adjustment_ft = round(float(adjustment) * 3.28084, 2)
     else:
         adjustment_ft = None
-    return adjustment_ft    
-   
+    return adjustment_ft
+
+
 #######################################################################
-#Function to download rating curve from API
+# Function to download rating curve from API
 #######################################################################
 def get_rating_curve(rating_curve_url, location_ids):
     '''
-    Given list of location_ids (nws_lids, usgs_site_codes, etc) get the 
+    Given list of location_ids (nws_lids, usgs_site_codes, etc) get the
     rating curve from WRDS API and export as a DataFrame.
 
     Parameters
@@ -1065,33 +1169,31 @@ def get_rating_curve(rating_curve_url, location_ids):
         Rating curves from input list as well as other site information.
 
     '''
-    #Define DataFrame to contain all returned curves.
+    # Define DataFrame to contain all returned curves.
     all_curves = pd.DataFrame()
-    
+
     print(location_ids)
-    #Define call to retrieve all rating curve information from WRDS.
+    # Define call to retrieve all rating curve information from WRDS.
     joined_location_ids = '%2C'.join(location_ids)
     url = f'{rating_curve_url}/{joined_location_ids}'
-    
-    #Call the API 
+
+    # Call the API
     response = requests.get(url, verify=False)
 
-    #If successful
+    # If successful
     if response.ok:
-
-        #Write return to json and extract the rating curves
+        # Write return to json and extract the rating curves
         site_json = response.json()
         rating_curves_list = site_json['rating_curves']
 
-        #For each curve returned
+        # For each curve returned
         for curve in rating_curves_list:
-            #Check if a curve was populated (e.g wasn't blank)
+            # Check if a curve was populated (e.g wasn't blank)
             if curve:
+                # Write rating curve to pandas dataframe as well as site attributes
+                curve_df = pd.DataFrame(curve['rating_curve'], dtype=float)
 
-                #Write rating curve to pandas dataframe as well as site attributes
-                curve_df = pd.DataFrame(curve['rating_curve'],dtype=float)
-
-                #Add other information such as site, site type, source, units, and timestamp.
+                # Add other information such as site, site type, source, units, and timestamp.
                 curve_df['location_id'] = curve['metadata']['location_id']
                 curve_df['location_type'] = curve['metadata']['id_type']
                 curve_df['source'] = curve['metadata']['source']
@@ -1099,112 +1201,127 @@ def get_rating_curve(rating_curve_url, location_ids):
                 curve_df['stage_units'] = curve['metadata']['stage_unit']
                 curve_df['wrds_timestamp'] = response.headers['Date']
 
-                #Append rating curve to DataFrame containing all curves
+                # Append rating curve to DataFrame containing all curves
                 all_curves = pd.concat([all_curves, curve_df])
             else:
                 continue
 
     return all_curves
+
+
 #######################################################################
-#Following Functions used for preprocesing of AHPS sites (NWS and USGS)
+# Following Functions used for preprocesing of AHPS sites (NWS and USGS)
 ########################################################################
-    
+
+
 #######################################################################
-#Function to return a correct maps.
+# Function to return a correct maps.
 ########################################################################
 def select_grids(dataframe, stages, datum88, buffer):
     '''
-    Given a DataFrame (in a specific format), and a dictionary of stages, and the datum (in navd88). 
-    loop through the available inundation datasets and find the datasets that are equal to or immediately above the thresholds and only return 1 dataset per threshold (if any).
+    Given a DataFrame (in a specific format), and a dictionary of stages, and the datum (in navd88).
+    loop through the available inundation datasets and find the datasets that are equal to or immediately
+    above the thresholds and only return 1 dataset per threshold (if any).
 
     Parameters
     ----------
     dataframe : DataFrame
-        DataFrame that has to be in a specific format and contains the stages and paths to the inundation datasets.
+        DataFrame that has to be in a specific format and contains the stages and paths to the
+        inundation datasets.
     stages : DICT
         Dictionary of thresholds (key) and stages (values)
     datum88: FLOAT
         The datum associated with the LID that is pre-converted to NAVD88 (if needed)
     buffer: Float
-        Interval which the uppder bound can be assigned. For example, Threshold + buffer = upper bound. Recommended to make buffer 0.1 greater than desired interval as code selects maps < and not <=
+        Interval which the uppder bound can be assigned. For example, Threshold + buffer = upper bound.
+        Recommended to make buffer 0.1 greater than desired interval as code selects maps < and not <=
 
     Returns
     -------
     maps : DICT
         Dictionary of threshold (key) and inundation dataset path (value)
     map_flows: DICT
-        Dictionary of threshold (key) and flows in CFS rounded to the nearest whole number associated with the selected maps (value)
+        Dictionary of threshold (key) and flows in CFS rounded to the nearest whole number associated
+        with the selected maps (value)
 
     '''
-    #Define threshold categories
+    # Define threshold categories
     thresholds = ['action', 'minor', 'moderate', 'major']
     maps = {}
-    map_flows={}
-    #For each threshold, pick the appropriate map for analysis.
-    for i,threshold in enumerate(thresholds):
-        #Check if stage is None
+    map_flows = {}
+    # For each threshold, pick the appropriate map for analysis.
+    for i, threshold in enumerate(thresholds):
+        # Check if stage is None
         if not stages[threshold] is None:
-            #Define the threshold floor elevation (navd88).
-            lower_bound = round((stages[threshold] + datum88),1)
-            #Define the threshold ceiling (navd88)
-            upper_bound = round((stages[threshold] + datum88 + buffer),1)
-            #For thresholds that are action, minor, moderate
+            # Define the threshold floor elevation (navd88).
+            lower_bound = round((stages[threshold] + datum88), 1)
+            # Define the threshold ceiling (navd88)
+            upper_bound = round((stages[threshold] + datum88 + buffer), 1)
+            # For thresholds that are action, minor, moderate
             if threshold in ['action', 'minor', 'moderate']:
-                #Make sure the next threshold has a valid stage
-                if stages[thresholds[i+1]] is None:
+                # Make sure the next threshold has a valid stage
+                if stages[thresholds[i + 1]] is None:
                     next_threshold = upper_bound
                 else:
-                    #Determine what the next threshold elevation is.
-                    next_threshold = round((stages[thresholds[i+1]] + datum88),1)
-               #Make sure the upper_bound is not greater than the next threshold, if it is then reassign upper_bound.
+                    # Determine what the next threshold elevation is.
+                    next_threshold = round((stages[thresholds[i + 1]] + datum88), 1)
+                # Make sure the upper_bound is not greater than the next threshold, if it is then reassign upper_bound.
                 if upper_bound > next_threshold:
-                    upper_bound = next_threshold                
-                #Get the single map which meets the criteria.
-                value = dataframe.query(f'({lower_bound}<=elevation) & (elevation<{upper_bound})')['elevation'].min()
-            #For major threshold
+                    upper_bound = next_threshold
+                # Get the single map which meets the criteria.
+                value = dataframe.query(f'({lower_bound}<=elevation) & (elevation<{upper_bound})')[
+                    'elevation'
+                ].min()
+            # For major threshold
             else:
-                #Get the single map which meets criteria.
-                value = dataframe.query(f'({lower_bound}<=elevation) & (elevation<{upper_bound})')['elevation'].min()
-    
-            #If the selected value is a number
+                # Get the single map which meets criteria.
+                value = dataframe.query(f'({lower_bound}<=elevation) & (elevation<{upper_bound})')[
+                    'elevation'
+                ].min()
+
+            # If the selected value is a number
             if np.isfinite(value):
-                #Get the map path and the flow associated with the map (rounded to nearest whole number)
+                # Get the map path and the flow associated with the map (rounded to nearest whole number)
                 map_path = dataframe.query(f'elevation == {value}')['path'].item()
-                map_flow = round(dataframe.query(f'elevation == {value}')['flow'].item(),0)
-                #Check to see if map_flow is valid (if beyond rating_curve it is nan)
+                map_flow = round(dataframe.query(f'elevation == {value}')['flow'].item(), 0)
+                # Check to see if map_flow is valid (if beyond rating_curve it is nan)
                 if not np.isfinite(map_flow):
                     map_path = 'No Flow'
                     map_flow = 'No Flow'
-                                    
-            #If the selected value is not a number (or interpolated flow is nan caused by elevation of map which is beyond rating curve range), then map_path and map_flows are both set to 'No Map'. 
-            else: 
-                map_path = 'No Map' 
-                map_flow = 'No Map' 
+
+            # If the selected value is not a number
+            #   (or interpolated flow is nan caused by elevation of map which is beyond rating curve range),
+            #   then map_path and map_flows are both set to 'No Map'.
+            else:
+                map_path = 'No Map'
+                map_flow = 'No Map'
         else:
             map_path = 'No Threshold'
             map_flow = 'No Threshold'
-        
-        #Write map paths and flows to dictionary        
+
+        # Write map paths and flows to dictionary
         maps[threshold] = map_path
         map_flows[threshold] = map_flow
-    
-    #Get the maximum inundation map (using elevation) and this will be the domain extent
+
+    # Get the maximum inundation map (using elevation) and this will be the domain extent
     max_value = dataframe['elevation'].max()
     map_path = dataframe.query(f'elevation == {max_value}')['path'].item()
     map_flow = 'Not Used'
     maps['extent'] = map_path
     map_flows['extent'] = map_flow
-    
-    return maps,map_flows
+
+    return maps, map_flows
+
 
 #######################################################################
-#Process AHPS Extent Grid (Fill Holes)
+# Process AHPS Extent Grid (Fill Holes)
 #######################################################################
-def process_extent(extent, profile, output_raster = False):
+def process_extent(extent, profile, output_raster=False):
     '''
-    Convert raster to feature (using raster_to_feature), the footprint is used so all raster values are set to 1 where there is data.
-    fill all donut holes in resulting feature. 
-    Filled geometry is then converted back to raster using same raster properties as input profile. 
+    Convert raster to feature (using raster_to_feature), the footprint is used so all raster values are set
+        to 1 where there is data.
+    fill all donut holes in resulting feature.
+    Filled geometry is then converted back to raster using same raster properties as input profile.
     Output raster will have be encoded as follows:
         filled footprint (wet) = 1
         remaining area in raster domain (dry) = 0
@@ -1227,44 +1344,52 @@ def process_extent(extent, profile, output_raster = False):
         Profile associated with extent_filled_raster
 
     '''
-        
-    #Convert extent to feature and explode geometry
-    poly_extent = raster_to_feature(extent, profile, footprint_only = True)
+
+    # Convert extent to feature and explode geometry
+    poly_extent = raster_to_feature(extent, profile, footprint_only=True)
     poly_extent = poly_extent.explode(index_parts=True)
-    
-    #Fill holes in extent
-    poly_extent_fill_holes=MultiPolygon(Polygon(p.exterior) for p in poly_extent['geometry'])
+
+    # Fill holes in extent
+    poly_extent_fill_holes = MultiPolygon(Polygon(p.exterior) for p in poly_extent['geometry'])
     # loop through the filled polygons and insert the new geometry
     for i in range(len(poly_extent_fill_holes)):
-        poly_extent.loc[i,'geometry'] = poly_extent_fill_holes[i]
-    
-    #Dissolve filled holes with main map and explode
+        poly_extent.loc[i, 'geometry'] = poly_extent_fill_holes[i]
+
+    # Dissolve filled holes with main map and explode
     poly_extent['dissolve_field'] = 1
-    poly_extent = poly_extent.dissolve(by = 'dissolve_field')
+    poly_extent = poly_extent.dissolve(by='dissolve_field')
     poly_extent = poly_extent.explode(index_parts=True)
     poly_extent = poly_extent.reset_index()
-    
-    #Convert filled polygon back to raster
-    extent_filled_raster = features.rasterize(((geometry, 1) for geometry in poly_extent['geometry']), fill = 0, dtype = 'int32',transform = profile['transform'], out_shape = (profile['height'], profile['width']))
-    
-    #Update profile properties (dtype and no data)
-    profile.update(dtype = rasterio.int32)
+
+    # Convert filled polygon back to raster
+    extent_filled_raster = features.rasterize(
+        ((geometry, 1) for geometry in poly_extent['geometry']),
+        fill=0,
+        dtype='int32',
+        transform=profile['transform'],
+        out_shape=(profile['height'], profile['width']),
+    )
+
+    # Update profile properties (dtype and no data)
+    profile.update(dtype=rasterio.int32)
     profile.update(nodata=0)
-    
-    #Check if output raster is specified. If so, the write extent filled raster to disk. 
+
+    # Check if output raster is specified. If so, the write extent filled raster to disk.
     if output_raster:
-        #Create directory
-        Path(output_raster).parent.mkdir(parents = True, exist_ok = True)
+        # Create directory
+        Path(output_raster).parent.mkdir(parents=True, exist_ok=True)
         with rasterio.Env():
             with rasterio.open(output_raster, 'w', **profile) as dst:
                 dst.write(extent_filled_raster, 1)
-    #If no output raster is supplied the return the rasterio array and profile.
+    # If no output raster is supplied the return the rasterio array and profile.
     else:
         return extent_filled_raster, profile
+
+
 ########################################################################
-#Convert raster to polygon
+# Convert raster to polygon
 ########################################################################
-def raster_to_feature(grid, profile_override = False, footprint_only = False):
+def raster_to_feature(grid, profile_override=False, footprint_only=False):
     '''
     Given a grid path, convert to vector, dissolved by grid value, in GeoDataFrame format.
 
@@ -1275,7 +1400,8 @@ def raster_to_feature(grid, profile_override = False, footprint_only = False):
     profile_override: rasterio Profile
         Default is False, If a rasterio Profile is supplied, it will dictate the transform and crs.
     footprint_only: BOOL
-        If true, dataset will be divided by itself to remove all unique values. If False, all values in grid will be carried through on raster to feature conversion. default = False
+        If true, dataset will be divided by itself to remove all unique values. If False, all values in
+        grid will be carried through on raster to feature conversion. default = False
 
     Returns
     -------
@@ -1283,49 +1409,51 @@ def raster_to_feature(grid, profile_override = False, footprint_only = False):
         Dissolved (by gridvalue) vector data in GeoDataFrame.
 
     '''
-    #Determine what format input grid is:
-    #If a pathlib path, open with rasterio
+    # Determine what format input grid is:
+    # If a pathlib path, open with rasterio
     if isinstance(grid, pathlib.PurePath):
         dataset = rasterio.open(grid)
-    #If a rasterio dataset object, assign to dataset
+    # If a rasterio dataset object, assign to dataset
     elif isinstance(grid, rasterio.DatasetReader):
         dataset = grid
 
-    #Get data/mask and profile properties from dataset
+    # Get data/mask and profile properties from dataset
     data = dataset.read(1)
-    msk = dataset.read_masks(1)   
+    msk = dataset.read_masks(1)
     data_transform = dataset.transform
     coord_sys = dataset.crs
 
-    #If a profile override was supplied, use it to get the transform and coordinate system.
+    # If a profile override was supplied, use it to get the transform and coordinate system.
     if profile_override:
         data_transform = profile_override['transform']
         coord_sys = profile_override['crs']
-    
-    #If a footprint of the raster is desired, convert all data values to 1
+
+    # If a footprint of the raster is desired, convert all data values to 1
     if footprint_only:
-        data[msk == 255] = 1   
-    
-    #Convert grid to feature
+        data[msk == 255] = 1
+
+    # Convert grid to feature
     spatial = []
     values = []
-    for geom, val in rasterio.features.shapes(data, mask = msk, transform = data_transform):
+    for geom, val in rasterio.features.shapes(data, mask=msk, transform=data_transform):
         spatial.append(shape(geom))
-        values.append(val)        
-    spatial_geodataframe = gpd.GeoDataFrame({'values': values,'geometry':spatial }, crs = coord_sys)
-    dissolve_geodataframe = spatial_geodataframe.dissolve(by = 'values')
+        values.append(val)
+    spatial_geodataframe = gpd.GeoDataFrame({'values': values, 'geometry': spatial}, crs=coord_sys)
+    dissolve_geodataframe = spatial_geodataframe.dissolve(by='values')
     return dissolve_geodataframe
+
+
 ########################################################################
-#Create AHPS Benchmark Grid
+# Create AHPS Benchmark Grid
 ########################################################################
 def process_grid(benchmark, benchmark_profile, domain, domain_profile, reference_raster):
     '''
-    Given a benchmark grid and profile, a domain rasterio dataset and profile, and a reference raster, 
+    Given a benchmark grid and profile, a domain rasterio dataset and profile, and a reference raster,
     Match the benchmark dataset to the domain extent and create a classified grid convert to:
         0 (no data footprint of domain)
         1 (data footprint of domain)
         2 (data footprint of benchmark)
-    Then reproject classified benchmark grid to match reference grid resolution and crs. 
+    Then reproject classified benchmark grid to match reference grid resolution and crs.
     Output is an array of values and a profile.
 
     Parameters
@@ -1346,123 +1474,126 @@ def process_grid(benchmark, benchmark_profile, domain, domain_profile, reference
     boolean_benchmark : numpy Array
         Array of values for the benchmark_boolean grid.
     profile : rasterio profile
-        Updated, final profile of the boolean_benchmark grid. 
+        Updated, final profile of the boolean_benchmark grid.
 
     '''
-    
-    #Make benchmark have same dimensions as domain (Assume domain has same CRS as benchmark)
-    #Get source CRS (benchmark and domain assumed to be same CRS)
+
+    # Make benchmark have same dimensions as domain (Assume domain has same CRS as benchmark)
+    # Get source CRS (benchmark and domain assumed to be same CRS)
     source_crs = benchmark_profile['crs'].to_wkt()
-    #Get domain data
+    # Get domain data
     domain_arr = domain.read(1)
-    #Get benchmark data
+    # Get benchmark data
     benchmark_arr = benchmark.read(1)
-    #Create empty array with same dimensions as domain
+    # Create empty array with same dimensions as domain
     benchmark_fit_to_domain = np.empty(domain_arr.shape)
-    #Make benchmark have same footprint as domain (Assume domain has same CRS as benchmark)
-    reproject(benchmark_arr, 
-              destination = benchmark_fit_to_domain,
-              src_transform = benchmark.transform, 
-              src_crs = source_crs,
-              src_nodata = benchmark.nodata,
-              dst_transform = domain.transform, 
-              dst_crs = source_crs,
-              dst_nodata = benchmark.nodata,
-              dst_resolution = source_crs,
-              resampling = Resampling.bilinear)    
-    #Convert fitted benchmark dataset to boolean. 0 = NODATA Regions and 1 = Data Regions
-    benchmark_fit_to_domain_bool = np.where(benchmark_fit_to_domain == benchmark.nodata,0,1)
-    #Merge domain datamask and benchmark data mask. New_nodata_value (2) = Domain NO DATA footprint, 0 = NO DATA for benchmark (within data region of domain), 1 = DATA region of benchmark.
+    # Make benchmark have same footprint as domain (Assume domain has same CRS as benchmark)
+    reproject(
+        benchmark_arr,
+        destination=benchmark_fit_to_domain,
+        src_transform=benchmark.transform,
+        src_crs=source_crs,
+        src_nodata=benchmark.nodata,
+        dst_transform=domain.transform,
+        dst_crs=source_crs,
+        dst_nodata=benchmark.nodata,
+        dst_resolution=source_crs,
+        resampling=Resampling.bilinear,
+    )
+    # Convert fitted benchmark dataset to boolean. 0 = NODATA Regions and 1 = Data Regions
+    benchmark_fit_to_domain_bool = np.where(benchmark_fit_to_domain == benchmark.nodata, 0, 1)
+    # Merge domain datamask and benchmark data mask. New_nodata_value (2) = Domain NO DATA footprint,
+    #   0 = NO DATA for benchmark (within data region of domain), 1 = DATA region of benchmark.
     new_nodata_value = 2
-    classified_benchmark = np.where(domain_arr == domain.nodata, new_nodata_value, benchmark_fit_to_domain_bool)
-    
-    ##Reproject classified benchmark to reference raster crs and resolution.
-    #Read in reference raster
+    classified_benchmark = np.where(
+        domain_arr == domain.nodata, new_nodata_value, benchmark_fit_to_domain_bool
+    )
+
+    # Reproject classified benchmark to reference raster crs and resolution.
+    # Read in reference raster
     reference = rasterio.open(reference_raster)
-    #Determine the new transform and dimensions of reprojected/resampled classified benchmark dataset whos width, height, and bounds are same as domain dataset.
-    new_benchmark_transform, new_benchmark_width, new_benchmark_height = calculate_default_transform(source_crs, reference.crs, domain.width, domain.height, *domain.bounds, resolution = reference.res)   
-    #Define an empty array that is same dimensions as output by the "calculate_default_transform" command. 
-    classified_benchmark_projected = np.empty((new_benchmark_height,new_benchmark_width), dtype=np.uint8) 
-    #Reproject and resample the classified benchmark dataset. Nearest Neighbor resampling due to integer values of classified benchmark.
-    reproject(classified_benchmark, 
-              destination = classified_benchmark_projected,
-              src_transform = domain.transform, 
-              src_crs = source_crs,
-              src_nodata = new_nodata_value,
-              dst_transform = new_benchmark_transform, 
-              dst_crs = reference.crs,
-              dst_nodata = new_nodata_value,
-              dst_resolution = reference.res,
-              resampling = Resampling.nearest)
-    
-    #Update profile using reference profile as base (data type, NODATA, transform, width/height).
+    # Determine the new transform and dimensions of reprojected/resampled classified benchmark dataset whose
+    #   width, height, and bounds are same as domain dataset.
+    (new_benchmark_transform, new_benchmark_width, new_benchmark_height) = calculate_default_transform(
+        source_crs, reference.crs, domain.width, domain.height, *domain.bounds, resolution=reference.res
+    )
+    # Define an empty array that is same dimensions as output by the "calculate_default_transform" command.
+    classified_benchmark_projected = np.empty((new_benchmark_height, new_benchmark_width), dtype=np.uint8)
+    # Reproject and resample the classified benchmark dataset. Nearest Neighbor resampling due to integer
+    #   values of classified benchmark.
+    reproject(
+        classified_benchmark,
+        destination=classified_benchmark_projected,
+        src_transform=domain.transform,
+        src_crs=source_crs,
+        src_nodata=new_nodata_value,
+        dst_transform=new_benchmark_transform,
+        dst_crs=reference.crs,
+        dst_nodata=new_nodata_value,
+        dst_resolution=reference.res,
+        resampling=Resampling.nearest,
+    )
+
+    # Update profile using reference profile as base (data type, NODATA, transform, width/height).
     profile = reference.profile
-    profile.update(transform = new_benchmark_transform)
-    profile.update(dtype = rasterio.uint8)
-    profile.update(nodata = new_nodata_value) 
-    profile.update (width = new_benchmark_width)
-    profile.update(height = new_benchmark_height)   
-    
+    profile.update(transform=new_benchmark_transform)
+    profile.update(dtype=rasterio.uint8)
+    profile.update(nodata=new_nodata_value)
+    profile.update(width=new_benchmark_width)
+    profile.update(height=new_benchmark_height)
+
     return classified_benchmark_projected, profile
 
 
 def calculate_metrics_from_agreement_raster(agreement_raster):
+    '''Calculates metrics from an agreement raster'''
 
-    ''' Calculates metrics from an agreement raster '''
-
-    agreement_encoding_digits_to_names = { 0: "TN",
-                                           1: "FN",
-                                           2: "FP",
-                                           3: "TP"
-                                          }
+    agreement_encoding_digits_to_names = {0: "TN", 1: "FN", 2: "FP", 3: "TP"}
 
-
-    if isinstance(agreement_raster,rasterio.DatasetReader):
+    if isinstance(agreement_raster, rasterio.DatasetReader):
         pass
-    elif isinstance(agreement_raster,str):
+    elif isinstance(agreement_raster, str):
         agreement_raster = rasterio.open(agreement_raster)
     else:
         raise TypeError(f"{agreement_raster} is not a Rasterio Dataset Reader or a filepath to a raster")
 
-    # cycle through blocks 
-    totals = dict.from_keys(list(range(4)),0)
-    for idx,wind in agreement_raster.block_windows(1):
-        window_data = agreement_raster.read(1,window=wind)
-        values, counts = np.unique(window_data,return_counts=True)
-        for val,cts in values_counts:
-            totals[val] += cts
+    # cycle through blocks
+    totals = dict.from_keys(list(range(4)), 0)
+    for idx, wind in agreement_raster.block_windows(1):
+        window_data = agreement_raster.read(1, window=wind)
+        values, counts = np.unique(window_data, return_counts=True)
+        # TODO values_counts is not defined, so commented for now...
+        # for val, cts in values_counts:
+        #     totals[val] += cts
 
     results = dict()
-    for digit,count in totals.items():
+    for digit, count in totals.items():
         results[agreement_encoding_digits_to_names[digit]] = count
-   
-    return(results)
+
+    return results
 
 
 # evaluation metric fucntions
 
-def csi(TP,FP,FN,TN=None):
 
-    ''' Critical Success Index '''
+def csi(TP, FP, FN, TN=None):
+    '''Critical Success Index'''
 
     return TP / (FP + FN + TP)
 
-    
-def tpr(TP,FP,FN,TN=None):
 
-    ''' True Positive Rate '''
-    
-    return TP / (TP + FN)
+def tpr(TP, FP, FN, TN=None):
+    '''True Positive Rate'''
 
+    return TP / (TP + FN)
 
-def far(TP,FP,FN,TN=None):
 
-    ''' False Alarm Rate '''
+def far(TP, FP, FN, TN=None):
+    '''False Alarm Rate'''
 
     return FP / (TP + FP)
 
 
-def mcc(TP,FP,FN,TN=None):
-
-    ''' Matthew's Correlation Coefficient '''
-    return (TP*TN - FP*FN) / np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))
+def mcc(TP, FP, FN, TN=None):
+    '''Matthew's Correlation Coefficient'''
+    return (TP * TN - FP * FN) / np.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index a257b827e..fac649f70 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -1,5 +1,6 @@
 import os
 
+
 # Environmental variables and constants.
 TEST_CASES_DIR = r"/data/test_cases/"
 PREVIOUS_FIM_DIR = r"/data/previous_fim"
@@ -35,16 +36,7 @@
     "TN_perc",
     "FN_perc",
 ]
-GO_UP_STATS = [
-    "CSI",
-    "TPR",
-    "MCC",
-    "TN_area_km2",
-    "TP_area_km2",
-    "TN_perc",
-    "TP_perc",
-    "TNR",
-]
+GO_UP_STATS = ["CSI", "TPR", "MCC", "TN_area_km2", "TP_area_km2", "TN_perc", "TP_perc", "TNR"]
 GO_DOWN_STATS = ["FAR", "FN_area_km2", "FP_area_km2", "FP_perc", "FN_perc", "PND"]
 
 # Variables for eval_plots.py
@@ -97,37 +89,10 @@
 # https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html
 acceptable_coord_acc_code_list = ["H", "1", "5", "S", "R", "B", "C", "D", "E", 5, 1]
 # https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html
-acceptable_coord_method_code_list = [
-    "C",
-    "D",
-    "W",
-    "X",
-    "Y",
-    "Z",
-    "N",
-    "M",
-    "L",
-    "G",
-    "R",
-    "F",
-    "S",
-]
+acceptable_coord_method_code_list = ["C", "D", "W", "X", "Y", "Z", "N", "M", "L", "G", "R", "F", "S"]
 # https://help.waterdata.usgs.gov/codes-and-parameters/codes#SI
 acceptable_alt_acc_thresh = 1.0
 # https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html
-acceptable_alt_meth_code_list = [
-    "A",
-    "D",
-    "F",
-    "I",
-    "J",
-    "L",
-    "N",
-    "R",
-    "W",
-    "X",
-    "Y",
-    "Z",
-]
+acceptable_alt_meth_code_list = ["A", "D", "F", "I", "J", "L", "N", "R", "W", "X", "Y", "Z"]
 # https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html
 acceptable_site_type_list = ["ST"]
diff --git a/tools/vary_mannings_n_composite.py b/tools/vary_mannings_n_composite.py
index 450734d18..1a69ab28e 100755
--- a/tools/vary_mannings_n_composite.py
+++ b/tools/vary_mannings_n_composite.py
@@ -1,22 +1,24 @@
 #!/usr/bin/env python3
+import argparse
 import datetime as dt
-import re
 import os
-import sys
-import pandas as pd
-import numpy as np
-import argparse
-import matplotlib.pyplot as plt
-import seaborn as sns
+import re
 import shutil
+import sys
 import traceback
 import warnings
 from functools import reduce
 from multiprocessing import Pool
-from os.path import isfile, join, dirname, isdir
+from os.path import dirname, isdir, isfile, join
 from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
 from tqdm import tqdm
 
+
 sns.set_theme(style="whitegrid")
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
@@ -39,78 +41,147 @@
         Optional (True or False): use this flag to crate src plots for all hydroids
 """
 
-def variable_mannings_calc(args):
 
-    in_src_bankfull_filename    = args[0]
-    channel_ratio_src_column    = args[1]
-    df_mann                     = args[2]
-    huc                         = args[3]
-    branch_id                   = args[4]
-    htable_filename             = args[5]
-    output_suffix               = args[6]
-    src_plot_option             = args[7]
-    huc_output_dir              = args[8]
+def variable_mannings_calc(args):
+    in_src_bankfull_filename = args[0]
+    channel_ratio_src_column = args[1]
+    df_mann = args[2]
+    huc = args[3]
+    branch_id = args[4]
+    htable_filename = args[5]
+    output_suffix = args[6]
+    src_plot_option = args[7]
+    huc_output_dir = args[8]
 
     ## Read the src_full_crosswalked.csv
     print('Calculating variable roughness: ' + str(huc) + '  branch id: ' + str(branch_id))
     log_text = 'Calculating variable roughness: ' + str(huc) + '  branch id: ' + str(branch_id) + '\n'
-    df_src = pd.read_csv(in_src_bankfull_filename,dtype={'feature_id': 'int64'})
+    df_src = pd.read_csv(in_src_bankfull_filename, dtype={'feature_id': 'int64'})
 
     ## Check that the channel ratio column the user specified exists in the def
     if channel_ratio_src_column not in df_src.columns:
-        print('WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column)
+        print(
+            'WARNING --> '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+            + in_src_bankfull_filename
+            + ' does not contain the specified channel ratio column: '
+            + channel_ratio_src_column
+        )
         print('Skipping --> ' + str(huc) + '  branch id: ' + str(branch_id))
-        log_text += 'WARNING --> ' + str(huc) + '  branch id: ' + str(branch_id) + in_src_bankfull_filename + ' does not contain the specified channel ratio column: ' + channel_ratio_src_column  + '\n'
+        log_text += (
+            'WARNING --> '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+            + in_src_bankfull_filename
+            + ' does not contain the specified channel ratio column: '
+            + channel_ratio_src_column
+            + '\n'
+        )
     else:
         try:
             if 'comp_ManningN' in df_src.columns:
-                df_src.drop(['channel_n','overbank_n','comp_ManningN','vmann_on','Discharge (m3s-1)_varMann'], axis=1, inplace=True) # drop these cols (in case vmann was previously performed)
-            
-            ## Merge (crosswalk) the df of Manning's n with the SRC df (using the channel/fplain delination in the channel_ratio_src_column)
-            df_src = df_src.merge(df_mann,  how='left', on='feature_id')
+                df_src.drop(
+                    ['channel_n', 'overbank_n', 'comp_ManningN', 'vmann_on', 'Discharge (m3s-1)_varMann'],
+                    axis=1,
+                    inplace=True,
+                )  # drop these cols (in case vmann was previously performed)
+
+            ## Merge (crosswalk) the df of Manning's n with the SRC df
+            #   (using the channel/fplain delination in the channel_ratio_src_column)
+            df_src = df_src.merge(df_mann, how='left', on='feature_id')
             check_null = df_src['channel_n'].isnull().sum() + df_src['overbank_n'].isnull().sum()
             if check_null > 0:
-                log_text += str(huc) + '  branch id: ' + str(branch_id) + ' --> ' + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe' + ' --> missing entries= ' + str(check_null/84)  + '\n'
-
-            ## Calculate composite Manning's n using the channel geometry ratio attribute given by user (e.g. chann_hradius_ratio or chann_vol_ratio)
-            df_src['comp_ManningN'] = (df_src[channel_ratio_src_column]*df_src['channel_n']) + ((1.0 - df_src[channel_ratio_src_column])*df_src['overbank_n'])
-            #print('Done calculating composite Manning n (' + channel_ratio_src_column + '): ' + str(huc))
+                log_text += (
+                    str(huc)
+                    + '  branch id: '
+                    + str(branch_id)
+                    + ' --> '
+                    + 'Null feature_ids found in crosswalk btw roughness dataframe and src dataframe'
+                    + ' --> missing entries= '
+                    + str(check_null / 84)
+                    + '\n'
+                )
+
+            ## Calculate composite Manning's n using the channel geometry ratio attribute given by user
+            #   (e.g. chann_hradius_ratio or chann_vol_ratio)
+            df_src['comp_ManningN'] = (df_src[channel_ratio_src_column] * df_src['channel_n']) + (
+                (1.0 - df_src[channel_ratio_src_column]) * df_src['overbank_n']
+            )
+            # print('Done calculating composite Manning n (' + channel_ratio_src_column + '): ' + str(huc))
 
             ## Check if there are any missing data in the composite ManningN column
             check_null_comp = df_src['comp_ManningN'].isnull().sum()
             if check_null_comp > 0:
-                log_text += str(huc) + '  branch id: ' + str(branch_id) + ' --> ' + 'Missing values in the comp_ManningN calculation' + ' --> missing entries= ' + str(check_null_comp/84)  + '\n'
-            df_src['vmann_on'] = np.where(df_src['comp_ManningN'].isnull(), False, True) # create field to identify where vmann is applied (True=yes; False=no)
+                log_text += (
+                    str(huc)
+                    + '  branch id: '
+                    + str(branch_id)
+                    + ' --> '
+                    + 'Missing values in the comp_ManningN calculation'
+                    + ' --> missing entries= '
+                    + str(check_null_comp / 84)
+                    + '\n'
+                )
+            df_src['vmann_on'] = np.where(
+                df_src['comp_ManningN'].isnull(), False, True
+            )  # create field to identify where vmann is applied (True=yes; False=no)
 
             ## Define the channel geometry variable names to use from the src
             hydr_radius = 'HydraulicRadius (m)'
             wet_area = 'WetArea (m2)'
 
             ## Calculate Q using Manning's equation
-            #df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True) # rename the previous Discharge column
-            df_src['Discharge (m3s-1)_varMann'] = df_src[wet_area]* \
-            pow(df_src[hydr_radius],2.0/3)* \
-            pow(df_src['SLOPE'],0.5)/df_src['comp_ManningN']
+            # Uncomment below to rename the previous Discharge column
+            # df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True)
+            df_src['Discharge (m3s-1)_varMann'] = (
+                df_src[wet_area]
+                * pow(df_src[hydr_radius], 2.0 / 3)
+                * pow(df_src['SLOPE'], 0.5)
+                / df_src['comp_ManningN']
+            )
 
             ## Set Q values to 0 and -999 for specified criteria (thalweg notch check happens in BARC)
-            # df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == 0,0,inplace=True)
+            # df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == 0, 0, inplace=True)
             # if 'Thalweg_burn_elev' in df_src:
-            #     df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] == df_src['Thalweg_burn_elev'],0,inplace=True)
-            #     df_src['Discharge (m3s-1)_varMann'].mask(df_src['Stage'] < df_src['Thalweg_burn_elev'],-999,inplace=True)
+            #     df_src['Discharge (m3s-1)_varMann'].mask(
+            #         df_src['Stage'] == df_src['Thalweg_burn_elev'], 0, inplace=True
+            #     )
+            #     df_src['Discharge (m3s-1)_varMann'].mask(
+            #         df_src['Stage'] < df_src['Thalweg_burn_elev'], -999, inplace=True
+            #     )
 
             ## Use the default discharge column when vmann is not being applied
-            df_src['Discharge (m3s-1)_varMann'] = np.where(df_src['vmann_on']==False, df_src['Discharge (m3s-1)'], df_src['Discharge (m3s-1)_varMann']) # reset the discharge value back to the original if vmann=false
-            df_src['comp_ManningN'] = np.where(df_src['vmann_on']==False, df_src['ManningN'], df_src['comp_ManningN']) # reset the ManningN value back to the original if vmann=false
+            df_src['Discharge (m3s-1)_varMann'] = np.where(
+                df_src['vmann_on'] == False, df_src['Discharge (m3s-1)'], df_src['Discharge (m3s-1)_varMann']
+            )  # reset the discharge value back to the original if vmann=false
+            df_src['comp_ManningN'] = np.where(
+                df_src['vmann_on'] == False, df_src['ManningN'], df_src['comp_ManningN']
+            )  # reset the ManningN value back to the original if vmann=false
 
             ## Output new SRC with bankfull column
-            df_src.to_csv(in_src_bankfull_filename,index=False)
+            df_src.to_csv(in_src_bankfull_filename, index=False)
 
             ## Output new hydroTable with updated discharge and ManningN column
-            df_src_trim = df_src[['HydroID','Stage','vmann_on','comp_ManningN','Discharge (m3s-1)_varMann']]
-            df_src_trim = df_src_trim.rename(columns={'Stage':'stage','comp_ManningN':'vmann_ManningN','Discharge (m3s-1)_varMann': 'vmann_discharge_cms'})
-            df_src_trim['ManningN'] = df_src_trim['vmann_ManningN'] # create a copy of vmann modified ManningN (used to track future changes)
-            df_src_trim['discharge_cms'] = df_src_trim['vmann_discharge_cms'] # create a copy of vmann modified discharge (used to track future changes)
-            df_htable = pd.read_csv(htable_filename,dtype={'HUC': str})
+            df_src_trim = df_src[
+                ['HydroID', 'Stage', 'vmann_on', 'comp_ManningN', 'Discharge (m3s-1)_varMann']
+            ]
+            df_src_trim = df_src_trim.rename(
+                columns={
+                    'Stage': 'stage',
+                    'comp_ManningN': 'vmann_ManningN',
+                    'Discharge (m3s-1)_varMann': 'vmann_discharge_cms',
+                }
+            )
+            df_src_trim['ManningN'] = df_src_trim[
+                'vmann_ManningN'
+            ]  # create a copy of vmann modified ManningN (used to track future changes)
+            df_src_trim['discharge_cms'] = df_src_trim[
+                'vmann_discharge_cms'
+            ]  # create a copy of vmann modified discharge (used to track future changes)
+            df_htable = pd.read_csv(htable_filename, dtype={'HUC': str})
 
             ## Check if BARC ran
             # if not set(['orig_discharge_cms']).issubset(df_htable.columns):
@@ -119,35 +190,51 @@ def variable_mannings_calc(args):
             # else:
 
             ## drop the previously modified discharge column to be replaced with updated version
-            df_htable.drop(['vmann_on','discharge_cms','ManningN','vmann_discharge_cms','vmann_ManningN'], axis=1, inplace=True) 
-            df_htable = df_htable.merge(df_src_trim, how='left', left_on=['HydroID','stage'], right_on=['HydroID','stage'])
-
-            df_htable['vmann_on'] = np.where(df_htable['LakeID']>0, False, df_htable['vmann_on']) # reset the ManningN value back to the original if vmann=false
+            df_htable.drop(
+                ['vmann_on', 'discharge_cms', 'ManningN', 'vmann_discharge_cms', 'vmann_ManningN'],
+                axis=1,
+                inplace=True,
+            )
+            df_htable = df_htable.merge(
+                df_src_trim, how='left', left_on=['HydroID', 'stage'], right_on=['HydroID', 'stage']
+            )
+
+            df_htable['vmann_on'] = np.where(
+                df_htable['LakeID'] > 0, False, df_htable['vmann_on']
+            )  # reset the ManningN value back to the original if vmann=false
 
             ## Output new hydroTable csv
             if output_suffix != "":
-                htable_filename = os.path.splitext(htable_filename)[0] + output_suffix + '.csv' 
-            df_htable.to_csv(htable_filename,index=False)
+                htable_filename = os.path.splitext(htable_filename)[0] + output_suffix + '.csv'
+            df_htable.to_csv(htable_filename, index=False)
 
             log_text += 'Completed: ' + str(huc)
 
             ## plot rating curves
             if src_plot_option:
-                if isdir(huc_output_dir) == False:
+                if isdir(huc_output_dir) is False:
                     os.mkdir(huc_output_dir)
                 generate_src_plot(df_src, huc_output_dir)
         except Exception as ex:
-            summary = traceback.StackSummary.extract(
-                    traceback.walk_stack(None))
+            summary = traceback.StackSummary.extract(traceback.walk_stack(None))
             print(str(huc) + '  branch id: ' + str(branch_id) + " failed for some reason")
-            print(f"*** {ex}")                
-            print(''.join(summary.format()))    
-            log_text += 'ERROR --> ' + str(huc) + '  branch id: ' + str(branch_id) + " failed (details: " + (f"*** {ex}") + (''.join(summary.format())) + '\n'
+            print(f"*** {ex}")
+            print(''.join(summary.format()))
+            log_text += (
+                'ERROR --> '
+                + str(huc)
+                + '  branch id: '
+                + str(branch_id)
+                + " failed (details: "
+                + (f"*** {ex}")
+                + (''.join(summary.format()))
+                + '\n'
+            )
+
+    return log_text
 
-    return(log_text)
 
 def generate_src_plot(df_src, plt_out_dir):
-
     ## create list of unique hydroids
     hydroids = df_src.HydroID.unique().tolist()
 
@@ -160,84 +247,144 @@ def generate_src_plot(df_src, plt_out_dir):
         ax.set_title(str(hydroid))
         sns.despine(f, left=True, bottom=True)
         sns.scatterplot(x='Discharge (m3s-1)', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
-        sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
+        sns.scatterplot(
+            x='Discharge (m3s-1)_varMann',
+            y='Stage',
+            data=plot_df,
+            label="SRC w/ vMann",
+            ax=ax,
+            color='orange',
+        )
         sns.lineplot(x='Discharge (m3s-1)', y='Stage_bankfull', data=plot_df, color='green', ax=ax)
-        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'],alpha=0.5)
-        plt.text(plot_df['Discharge (m3s-1)'].median(), plot_df['Stage_bankfull'].median(), "NWM Bankfull Approx: " + str(plot_df['Stage_bankfull'].median()))
+        plt.fill_between(plot_df['Discharge (m3s-1)'], plot_df['Stage_bankfull'], alpha=0.5)
+        plt.text(
+            plot_df['Discharge (m3s-1)'].median(),
+            plot_df['Stage_bankfull'].median(),
+            "NWM Bankfull Approx: " + str(plot_df['Stage_bankfull'].median()),
+        )
         ax.legend()
-        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png',dpi=175, bbox_inches='tight')
+        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '_vmann.png', dpi=175, bbox_inches='tight')
         plt.close()
 
-#    for hydroid in hydroids:
-#        print("Creating SRC plot: " + str(hydroid))
-#        plot_df = df_src.loc[df_src['HydroID'] == hydroid]
-#
-#        f, ax = plt.subplots(figsize=(6.5, 6.5))
-#        ax.set_title(str(hydroid))
-#        sns.despine(f, left=True, bottom=True)
-#        sns.scatterplot(x='comp_ManningN', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
-#        #sns.scatterplot(x='Discharge (m3s-1)_varMann', y='Stage', data=plot_df, label="SRC w/ vMann", ax=ax, color='orange')
-#        sns.lineplot(x='comp_ManningN', y='Stage_1_5', data=plot_df, color='green', ax=ax)
-#        plt.fill_between(plot_df['comp_ManningN'], plot_df['Stage_1_5'],alpha=0.5)
-#        plt.text(plot_df['comp_ManningN'].median(), plot_df['Stage_1_5'].median(), "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()))
-#        ax.legend()
-#        plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png',dpi=175, bbox_inches='tight')
-#        plt.close()
+    # for hydroid in hydroids:
+    #     print("Creating SRC plot: " + str(hydroid))
+    #     plot_df = df_src.loc[df_src['HydroID'] == hydroid]
+
+    #     f, ax = plt.subplots(figsize=(6.5, 6.5))
+    #     ax.set_title(str(hydroid))
+    #     sns.despine(f, left=True, bottom=True)
+    #     sns.scatterplot(x='comp_ManningN', y='Stage', data=plot_df, label="Orig SRC", ax=ax, color='blue')
+    #     # sns.scatterplot(
+    #     #     x='Discharge (m3s-1)_varMann',
+    #     #     y='Stage',
+    #     #     data=plot_df,
+    #     #     label="SRC w/ vMann",
+    #     #     ax=ax,
+    #     #     color='orange',
+    #     # )
+    #     sns.lineplot(x='comp_ManningN', y='Stage_1_5', data=plot_df, color='green', ax=ax)
+    #     plt.fill_between(plot_df['comp_ManningN'], plot_df['Stage_1_5'], alpha=0.5)
+    #     plt.text(
+    #         plot_df['comp_ManningN'].median(),
+    #         plot_df['Stage_1_5'].median(),
+    #         "NWM 1.5yr: " + str(plot_df['Stage_1_5'].median()),
+    #     )
+    #     ax.legend()
+    #     plt.savefig(plt_out_dir + os.sep + str(hydroid) + '.png', dpi=175, bbox_inches='tight')
+    #     plt.close()
+
 
 def multi_process(variable_mannings_calc, procs_list, log_file, verbose):
     ## Initiate multiprocessing
-    print(f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs")
+    print(
+        f"Applying variable Manning's n to SRC calcs for {len(procs_list)} hucs using {number_of_jobs} jobs"
+    )
     with Pool(processes=number_of_jobs) as pool:
         if verbose:
             map_output = tqdm(pool.imap(variable_mannings_calc, procs_list), total=len(procs_list))
             tuple(map_output)  # fetch the lazy results
         else:
             map_output = pool.map(variable_mannings_calc, procs_list)
-    log_file.writelines(["%s\n" % item  for item in map_output])
+    log_file.writelines(["%s\n" % item for item in map_output])
+
 
-def run_prep(fim_dir,channel_ratio_src_column,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option):
+def run_prep(
+    fim_dir, channel_ratio_src_column, mann_n_table, output_suffix, number_of_jobs, verbose, src_plot_option
+):
     procs_list = []
 
-    print('Writing progress to log file here: ' + str(join(fim_dir,'log_composite_n' + output_suffix + '.log')))
+    print(
+        'Writing progress to log file here: ' + str(join(fim_dir, 'log_composite_n' + output_suffix + '.log'))
+    )
     print('This may take a few minutes...')
     ## Create a time var to log run time
     begin_time = dt.datetime.now()
 
     ## initiate log file
-    log_file = open(join(fim_dir,'logs','log_composite_n' + output_suffix + '.log'),"w")
+    log_file = open(join(fim_dir, 'logs', 'log_composite_n' + output_suffix + '.log'), "w")
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
 
     ## Check that the input fim_dir exists
     assert os.path.isdir(fim_dir), 'ERROR: could not find the input fim_dir location: ' + str(fim_dir)
     ## Check that the manning's roughness input filepath exists and then read to dataframe
-    assert os.path.isfile(mann_n_table), 'Can not find the input roughness/feature_id file: ' + str(mann_n_table)
-    
+    assert os.path.isfile(mann_n_table), 'Can not find the input roughness/feature_id file: ' + str(
+        mann_n_table
+    )
+
     ## Read the Manning's n csv (ensure that it contains feature_id, channel mannings, floodplain mannings)
     print('Importing the Manning roughness data file: ' + mann_n_table)
-    df_mann = pd.read_csv(mann_n_table,dtype={'feature_id': 'int64'})
-    if 'channel_n' not in df_mann.columns or 'overbank_n' not in df_mann.columns or 'feature_id' not in df_mann.columns:
-        print('Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann)
+    df_mann = pd.read_csv(mann_n_table, dtype={'feature_id': 'int64'})
+    if (
+        'channel_n' not in df_mann.columns
+        or 'overbank_n' not in df_mann.columns
+        or 'feature_id' not in df_mann.columns
+    ):
+        print(
+            'Missing required data column ("feature_id","channel_n", and/or "overbank_n")!!! --> ' + df_mann
+        )
     else:
         print('Running the variable_mannings_calc function...')
 
         ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
-        huc_list  = os.listdir(fim_dir)
+        huc_list = os.listdir(fim_dir)
         for huc in huc_list:
-            #if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
-            if re.match('\d{8}', huc):    
-                huc_branches_dir = os.path.join(fim_dir, huc,'branches')
+            # if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
+            if re.match(r'\d{8}', huc):
+                huc_branches_dir = os.path.join(fim_dir, huc, 'branches')
                 for branch_id in os.listdir(huc_branches_dir):
-                    branch_dir = os.path.join(huc_branches_dir,branch_id)
-                    in_src_bankfull_filename = join(branch_dir,'src_full_crosswalked_' + branch_id + '.csv')
-                    htable_filename = join(branch_dir,'hydroTable_' + branch_id + '.csv')
-                    huc_plot_output_dir = join(branch_dir,'src_plots')
+                    branch_dir = os.path.join(huc_branches_dir, branch_id)
+                    in_src_bankfull_filename = join(branch_dir, 'src_full_crosswalked_' + branch_id + '.csv')
+                    htable_filename = join(branch_dir, 'hydroTable_' + branch_id + '.csv')
+                    huc_plot_output_dir = join(branch_dir, 'src_plots')
 
                     if isfile(in_src_bankfull_filename) and isfile(htable_filename):
-                        procs_list.append([in_src_bankfull_filename, channel_ratio_src_column, df_mann, huc, branch_id, htable_filename, output_suffix, src_plot_option, huc_plot_output_dir])
+                        procs_list.append(
+                            [
+                                in_src_bankfull_filename,
+                                channel_ratio_src_column,
+                                df_mann,
+                                huc,
+                                branch_id,
+                                htable_filename,
+                                output_suffix,
+                                src_plot_option,
+                                huc_plot_output_dir,
+                            ]
+                        )
                     else:
-                        print('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
-                        log_file.write('HUC: ' + str(huc) + '  branch id: ' + str(branch_id) + '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv or hydroTable_*.csv) in the fim output dir: ' + str(branch_dir) + ' - skipping this branch!!!\n')
+                        print(
+                            f'HUC: {huc} branch id: {branch_id}'
+                            '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv '
+                            f'or hydroTable_*.csv) in the fim output dir: {branch_dir}'
+                            ' - skipping this branch!!!\n'
+                        )
+                        log_file.write(
+                            f'HUC: {huc} branch id: {branch_id}'
+                            '\nWARNING --> can not find required file (src_full_crosswalked_bankfull_*.csv '
+                            f'or hydroTable_*.csv) in the fim output dir: {branch_dir}'
+                            ' - skipping this branch!!!\n'
+                        )
 
         ## Pass huc procs_list to multiprocessing function
         multi_process(variable_mannings_calc, procs_list, log_file, verbose)
@@ -249,16 +396,55 @@ def run_prep(fim_dir,channel_ratio_src_column,mann_n_table,output_suffix,number_
         log_file.write('TOTAL RUN TIME: ' + str(tot_run_time))
         log_file.close()
 
-if __name__ == '__main__':
 
-    parser = argparse.ArgumentParser(description="Vary the Manning's n values for in-channel vs. floodplain (recalculate Manning's eq for Discharge)")
-    parser.add_argument('-fim_dir','--fim-dir', help='FIM output dir', required=True,type=str)
-    parser.add_argument('-bc','--channel-ratio-src-column',help='SRC attribute containing the channel vs. overbank geometry ratio (for composite calc)',required=False,type=str,default='chann_hradius_ratio')
-    parser.add_argument('-mann','--mann-n-table',help="Path to a csv file containing Manning's n values by featureid",required=True,type=str)
-    parser.add_argument('-suff','--output-suffix',help="Suffix to append to the output log file (e.g. '_global_06_011')",default="",required=False,type=str)
-    parser.add_argument('-j','--number-of-jobs',help='number of workers',required=False,default=1,type=int)
-    parser.add_argument('-vb','--verbose',help='Optional verbose progress bar',required=False,default=None,action='store_true')
-    parser.add_argument('-plots','--src-plot-option',help='OPTIONAL flag: use this flag to create src plots for all hydroids. WARNING - long runtime',default=False,required=False, action='store_true')
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Vary the Manning's n values for in-channel vs. floodplain "
+        "(recalculate Manning's eq for Discharge)"
+    )
+    parser.add_argument('-fim_dir', '--fim-dir', help='FIM output dir', required=True, type=str)
+    parser.add_argument(
+        '-bc',
+        '--channel-ratio-src-column',
+        help='SRC attribute containing the channel vs. overbank geometry ratio (for composite calc)',
+        required=False,
+        type=str,
+        default='chann_hradius_ratio',
+    )
+    parser.add_argument(
+        '-mann',
+        '--mann-n-table',
+        help="Path to a csv file containing Manning's n values by featureid",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '-suff',
+        '--output-suffix',
+        help="Suffix to append to the output log file (e.g. '_global_06_011')",
+        default="",
+        required=False,
+        type=str,
+    )
+    parser.add_argument(
+        '-j', '--number-of-jobs', help='number of workers', required=False, default=1, type=int
+    )
+    parser.add_argument(
+        '-vb',
+        '--verbose',
+        help='Optional verbose progress bar',
+        required=False,
+        default=None,
+        action='store_true',
+    )
+    parser.add_argument(
+        '-plots',
+        '--src-plot-option',
+        help='OPTIONAL flag: use this flag to create src plots for all hydroids. WARNING - long runtime',
+        default=False,
+        required=False,
+        action='store_true',
+    )
 
     args = vars(parser.parse_args())
 
@@ -270,4 +456,12 @@ def run_prep(fim_dir,channel_ratio_src_column,mann_n_table,output_suffix,number_
     verbose = bool(args['verbose'])
     src_plot_option = args['src_plot_option']
 
-    run_prep(fim_dir,channel_ratio_src_column,mann_n_table,output_suffix,number_of_jobs,verbose,src_plot_option)
\ No newline at end of file
+    run_prep(
+        fim_dir,
+        channel_ratio_src_column,
+        mann_n_table,
+        output_suffix,
+        number_of_jobs,
+        verbose,
+        src_plot_option,
+    )
diff --git a/unit_tests/README.md b/unit_tests/README.md
index bea445a23..f957a4e77 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -4,39 +4,51 @@ Flood inundation mapping software configured to work with the U.S. National Wate
 
 #### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
 
-# This folder (`/unit_tests`) holds files for unit testing python files
+# The `/unit_tests` folder contains files for unit testing python files
 
 ## Creating unit tests
 
-For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and an accompanying json paramerters file. 
+For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and an accompanying json paramerters file.
 
 The files should be named following FIM convention:
 
-{source py file name}_test.py ->     `derive_level_paths_test.py`  
+{source py file name}_test.py ->     `derive_level_paths_test.py`
 {source py file name}_params.json -> `derive_level_paths_params.json`
 
 
 ## Tips to create a new json file for a new python unit test file.
 
-There are multiple way to figure out a set of default json parameters for the new unit test file. 
+There are multiple way to figure out a set of default json parameters for the new unit test file.
 
-One way is to use the incoming arg parser. Most python files include the code block of ` __name__ == '__main__':`, followed by external arg parsing (`args = vars(parser.parse_args()`). 
-* Add a `print(args)` or similar, and get all the values including keys as output.  
-* Copy that into an editor being used to create the json file.  
-* Add a line break after every comma.  
+One way is to use the incoming arg parser. Most python files include the code block of ` __name__ == '__main__':`, followed by external arg parsing (`args = vars(parser.parse_args()`).
+* Add a `print(args)` or similar, and get all the values including keys as output.
+* Copy that into an editor being used to create the json file.
+* Add a line break after every comma.
 * Find/replace all single quotes to double quotes then cleanup the left tab formatting.
 
 
 ## Setting up unit test data
 
-Start a docker container as you normally would for any development. 
-```bash 
+You can either pull the unit test data from s3, or generate it:
+
+1.) To pull the unit test data from ESIP's S3 bucket.
+```bash
+aws s3 cp --recursive \
+	s3://noaa-nws-owp-fim/hand_fim/unit_test_data/ \
+	<data/outputs/unit_test_data> --no-sign-request
+```
+The directory which the data is copied into must be named: `unit_test_data`
+
+2.) To generate the required unit test data, following these steps:
+
+* Start a docker container as you normally would for any development.
+```bash
 docker run --rm -it --name <a docker container name> \
 	-v /home/<your name>/projects/<folder path>/:/foss_fim \
 	{your docker image name}
 ```
 Example:
-```bash 
+```bash
 docker run --rm -it --name mytest \
 	-v /home/abcd/projects/dev/innudation-mapping/:/foss_fim \
 	-v /abcd_share/foss_fim/outputs/:/outputs \
@@ -44,24 +56,24 @@ docker run --rm -it --name mytest \
 	-v /abcd_share/foss_fim/outputs_temp/:/fim_temp \
 	fim_4:dev_20220208_8eba0ee
 ```
-
-For unit tests to work, you need to run the following (if not already in place).
-Notice a modified branch "deny_branch_unittests.lst"  (special for unittests)
-
-Here are the params and args you need if you need to re-run unit and branch
-
+* Call the `fim_pipeline.sh` script with the necessary arguments.
 ```bash
-fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u "02020005 05030104" \
-	-bd /foss_fim/config/deny_branch_unittests.lst -ud None -j 1 -o
+fim_pipeline.sh -n unit_test_data -u "02020005 05030104" \
+	-bd /foss_fim/config/deny_branch_unittests.lst \
+	-ud None -j 1 -o
 ```
 
-**NOTICE: the deny file used for fim_pipeline.sh, has a special one for unittests `deny_branch_unittests.lst`.
+__NOTICE:__ the deny file used for fim_pipeline.sh, has a special one for unittests `deny_branch_unittests.lst`.
 
-If you need to run inundation tests, fun the following:
+* If you need to run inundation tests, fun the following:
 
 ```bash
-python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_not_remove \
-	-jh 1 -jb 1 -m /data/outputs/fim_unit_test_data_do_not_remove/alpha_test_metrics.csv -o
+python3 foss_fim/tools/synthesize_test_cases.py \
+	-c DEV \
+	-v unit_test_data \
+	-jh 1 -jb 1 \
+	-m /data/outputs/unit_test_data/alpha_test_metrics.csv \
+	-o
 ```
 
 ## Running unit tests
@@ -71,13 +83,13 @@ python3 foss_fim/tools/synthesize_test_cases.py -c DEV -v fim_unit_test_data_do_
 pytest /foss_fim/unit_tests
 ```
 
-This is not 100% stable, as accurate paths for the parameters `.json` files are not included in this repository, are not uniform accross machines, and are subject to change. 
- 
+__NOTE:__ This is subject to error, as the downloaded/generated data could potentially have a different path than what is specified in the `/unit_tests/*_params.json` files. The data files are not included in this repository, are potentially not uniform accross machines, and are subject to change.
+
 ### If you want to test just one unit test (from the root terminal window):
 
 ```bash
-pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py 
-						or  
+pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py
+						or
 pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py
 ```
 
@@ -86,14 +98,14 @@ pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py
 pytest -v -s -k test_append_id_to_file_name_single_identifier_success
 ```
 
-If one test case is choosen, it will scan all of the test files, and scan for the method (test case) specified. 
+If one test case is choosen, it will scan all of the test files, and scan for the method (test case) specified.
 
 ## Key Notes for creating new unit tests
 1) All test functions must start with the phrase `test_`. That is how pytest picks them up. The rest of the function name does not have to match the pattern of `function_name_being_tested` but should. Further, the rest of the function name should say what the test is about, ie) `_failed_input_path`.  ie) `test_{some_function_name_from_the_source_code_file}_failed_input_path`. It is fine that the function names get very long (common in the industry).
 
 2) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
 
-3) As of now, you can not control the order that unit tests are run within a unit test file. 
+3) As of now, you can not control the order that unit tests are run within a unit test file.
 
 4) There must be at least one associated `{original py file name}_params.json` file per unit test.
 
@@ -105,38 +117,38 @@ If one test case is choosen, it will scan all of the test files, and scan for th
 
 8) One Python file = one `{original py file name}_test.py` file.
 
-9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist). 
+9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist).
 ie:
 	```bash
-	fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
+	fim_pipeline.sh -n unit_test_data -u 05030104 \
 		-c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o
 	```
-	
+
 	 but ours would be:
-	
-	```bash 
-	fim_pipeline.sh -n fim_unit_test_data_do_not_remove -u 05030104 \
+
+	```bash
+	fim_pipeline.sh -n unit_test_data -u 05030104 \
 		-c /foss_fim/config/params_template.env -j 1 -d no_list -o
 	```
 
 ## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
 
-The `pyproject.toml` file has been added, which contains the build system requirements of Python projects.  This file used to specify which warnings are disabled to pass our unit tests. 
+The `pyproject.toml` file has been added, which contains the build system requirements of Python projects.  This file used to specify which warnings are disabled to pass our unit tests.
 
 A `__init__.py` file has been added to the subdirectory of `/tools` in order for the `pytest` command run in the `/unit_tests` to pick up the tests in those directories as well.
 
 Luckily, `pytest` works well with The Python Standard Library `unittest`. This made the migration of previous unit tests using `unittest` over to `pytest` quite simple. The caveat is that our current unit tests employ elements of both libraries. A full transition to `pytest` will ideally take place at a future date.
 
 ## Testing for failing conditions
-- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for ONLY one variable value change. A "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions. 
+- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for ONLY one variable value change. A "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions.
 
 - It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
 
-- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass. 
+- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass.
 
 An example is in `unit_tests/Derive_level_paths_test.py` -> `test_Derive_level_paths_invalid_input_stream_network` (function). This example gives you the pattern implemented in Pytest.
 
 ## Future Enhancements
-1) Full transition to the `pytest` library, removing classes of `unittest.TestCase` and taking full advantage of available code re-use patterns offered through `pytest`.  
+1) Full transition to the `pytest` library, removing classes of `unittest.TestCase` and taking full advantage of available code re-use patterns offered through `pytest`.
 
-2) Over time, it is expected that python files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each python file which makes unit testing harder and less specific. Generally function will result in at least one "happy path" unit test function. This might require having test unit test outputs, such as sample .tif or small .gpkg files in subfolders in the unit tests folder, but this remains to be seen. Note: The files `/derive_level_paths_test.py` and `clip_vectors_to_wbd_test.py` are not complete as they do not yet test all output from a method.
+2) Over time, it is expected that python files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each python file which makes unit testing harder and less specific. Generally, one function will correlated to at least one "happy path" unit test function. Note: The files `derive_level_paths_test.py` and `clip_vectors_to_wbd_test.py` are not complete as they do not yet test all output from a method.
diff --git a/unit_tests/__template.json b/unit_tests/__template.json
index 20d34f7c4..c7c190673 100644
--- a/unit_tests/__template.json
+++ b/unit_tests/__template.json
@@ -1,11 +1,12 @@
 {
-	"valid_data":
-	{
-		"some param name (suggested to match the method param name)": "some value",
-		"another one and as many lines as you need": "some value",
-		"for 'None', you can just leave this field out, but sometimes it seems like jsut passing the value of None seems to work (not validated) ": "some value",
-		"for numerics, just put the number": 20,
-	    "for lists/collections: ": ["value 1", "value 2"],
-		"inline json comments don't work (let us know if you figure it out)": "sorry"
-	}
+    "valid_data": {
+        "some_param_name__suggested_to_match_the_method_param_name": "some value",
+        "another_one_and_as_many_lines_as_you_need": "some value",
+        "a_number": 20,
+        "a_list": [
+            "value 1",
+            "value 2"
+        ],
+        "inline_json_comments_dont_work": "sorry"
+    }
 }
diff --git a/unit_tests/__template.py b/unit_tests/__template.py
index f3d6d483e..bef8723b1 100644
--- a/unit_tests/__template.py
+++ b/unit_tests/__template.py
@@ -1,77 +1,86 @@
 #!/usr/bin/env python3
 
-import os
 import json
+import os
 import unittest
-import pytest
 
+import pytest
+import Your_original_source_python_file_name as src
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-import <Your original source python file name> as src
 
-class test_<Your original source python file name>(unittest.TestCase):
+class test_Your_original_source_python_file_name(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases:
+    # Test Cases:
 
     # MUST start with the name of "test_"
     # This is the (or one of the) valid test expected to pass
-    def test_<method name you are about to test>_success(self):
-
-        '''
+    def test_method_name_you_will_test(self):
+        """
         < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
           If there is no return value from the method, please say so.>
-        
+
         Dev Notes: (which can be removed after you make this file)
             Remember... You need to validate the method output if there is any. However, if you have time, it
-            is also recommended that you validate other outputs such as writing or updating file on the file 
+            is also recommended that you validate other outputs such as writing or updating file on the file
             system, aka: Does the expected file exist. Don't worry about its contents.
-        '''
+        """
+
+        # global params_file
+        params = self.params[
+            # update "valid_data" value if you need to (aka.. more than one node)
+            "valid_data"
+        ].copy()
 
-        #global params_file
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-        
         # for now we are happy if no exceptions are thrown.
-        
+
         # See the readme.md, clip_vectors_to_wbd_test.py or gms/derive_level_paths_test.py for examples.
         # Replace this stub example with your own.
         # Try to use the same order to make it easier.
-        # Remember, if the method accepts **params, then you can sent that in here as well.
+        # Remember, if the method accepts **params, then you can send that in here as well.
         # ie: my_py_class.my_method(** params)
-        
-        src.subset_vector_layers(hucCode = params["hucCode"],
-                                 nwm_streams_filename = params["nwm_streams"],
-                                 etc, etc for each param)
-       
-        # This is what we are actually testing-
-        # An assert that evaluates as True passes, one that evaluates to False fails.
-        # A message (string) can be added after the assert statement to provide detail on the case being tested, and
-        # why it failed.
-        assert os.path.exists(params["nwm_streams"]) == True, "The nwm_streams file does not exist"
-    
-    
-    # EXAMPLE SUCCESSFUL TEST CASE WHICH CAPTURES AN EXCEPTION (FAILURE)   
-    
+
+        src.subset_vector_layers(
+            hucCode=params["hucCode"],
+            nwm_streams_filename=params["nwm_streams"],
+            etc=params["a_number"],
+            etc2=params["a_list"],
+        )
+
+        # The assert statement is what we are acutally testing, one that evaluates as True passes,
+        # and one that evaluates to False fails.
+        # A message (string) can be added after the assert statement to provide detail on
+        # the case being tested, and why it failed.
+
+        assert os.path.exists(params["nwm_streams"]) is True, "The nwm_streams file does not exist"
+
+    # EXAMPLE OF A SUCCESSFUL TEST CASE WHICH CAPTURES AN EXCEPTION (FAILURE)
     def test_subset_vector_layers_fail_invalid_stream_path(self):
-        '''
+        """
         Notes about what the test is and the expected results (or expected exception if applicable)
-        '''
-
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
+        """
 
-        params["nwm_streams"] = "/some/bad/path/"  
+        params = self.params[
+            # update "valid_data" value if you need to (aka.. more than one node)
+            "valid_data"
+        ].copy()
 
-        with pytest.raises(Exception) as e_info:
-            clip_vectors_to_wbd.subset_vector_layers(hucCode = params["hucCode"],
-                                                     nwm_streams_filename = params["nwm_streams"],
-                                                     etc, etc for each param)
+        params["nwm_streams"] = "/some/bad/path/"
 
+        with pytest.raises(Exception):
+            src.subset_vector_layers(
+                hucCode=params["hucCode"],
+                nwm_streams_filename=params["nwm_streams"],
+                etc=params["a_number"],
+                etc2=params["a_list"],
+            )
diff --git a/unit_tests/aggregate_branch_lists_params.json b/unit_tests/aggregate_branch_lists_params.json
index de17dcd0f..d1090612c 100644
--- a/unit_tests/aggregate_branch_lists_params.json
+++ b/unit_tests/aggregate_branch_lists_params.json
@@ -1,8 +1,7 @@
 {
-	"valid_data":
-	{
-		"output_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
-		"file_name": "branch_ids.csv",
-		"output_file_name": "/data/outputs/fim_unit_test_data_do_not_remove/test.csv"
-	}
+    "valid_data": {
+        "output_dir": "/data/outputs/unit_test_data/",
+        "file_name": "branch_ids.csv",
+        "output_file_name": "/data/outputs/unit_test_data/test.csv"
+    }
 }
diff --git a/unit_tests/aggregate_branch_lists_test.py b/unit_tests/aggregate_branch_lists_test.py
index e54cd5c85..961db481a 100644
--- a/unit_tests/aggregate_branch_lists_test.py
+++ b/unit_tests/aggregate_branch_lists_test.py
@@ -2,8 +2,8 @@
 
 import json
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 import aggregate_branch_lists as src
@@ -11,25 +11,25 @@
 
 class test_aggregate_branch_lists(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases:
+    # Test Cases:
 
     def test_aggregate_branch_lists_success(self):
+        # global params_file
 
-        #global params_file
-        
         params = self.params["valid_data"].copy()
 
-        src.aggregate_branch_lists(output_dir = params["output_dir"],
-                                   file_name = params["file_name"],
-                                   output_file_name = params["output_file_name"])
-
+        src.aggregate_branch_lists(
+            output_dir=params["output_dir"],
+            file_name=params["file_name"],
+            output_file_name=params["output_file_name"],
+        )
diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json
index 5c1cf2b12..33a144fe6 100644
--- a/unit_tests/check_unit_errors_params.json
+++ b/unit_tests/check_unit_errors_params.json
@@ -1,11 +1,8 @@
 {
-	"valid_data":
-	{
-		"fim_dir": "/data/outputs/fim_unit_test_data_do_not_remove"
-	},
-
-	"invalid_path":
-	{
-		"fim_dir": "/data/outputs/check_errors_example_unit_tests_not_not_valid"
-	}	
+    "valid_data": {
+        "fim_dir": "/data/outputs/unit_test_data"
+    },
+    "invalid_path": {
+        "fim_dir": "/data/outputs/check_errors_example_unit_tests_not_not_valid"
+    }
 }
diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py
index a3b75ce9c..2eba1e9d0 100644
--- a/unit_tests/check_unit_errors_test.py
+++ b/unit_tests/check_unit_errors_test.py
@@ -1,34 +1,32 @@
 #!/usr/bin/env python3
 
+import json
 import math
 import os
 import shutil
-import json
 import unittest
-import pytest
 
-from utils.fim_enums import FIM_exit_codes
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-from utils.shared_variables import (UNIT_ERRORS_MIN_NUMBER_THRESHOLD,
-                                    UNIT_ERRORS_MIN_PERCENT_THRESHOLD)
+
 import check_unit_errors as src
+from utils.shared_variables import UNIT_ERRORS_MIN_NUMBER_THRESHOLD, UNIT_ERRORS_MIN_PERCENT_THRESHOLD
 
 
 class test_check_unit_errors(unittest.TestCase):
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
-    
+    """
+
     @classmethod
     def setUpClass(self):
-        
         # get_params_filename function in ./unit_test_utils handles errors
         params_file_path = ut_helpers.get_params_filename(__file__)
 
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-    '''
+    """
     To make most of this unit test, we have to do the following:
     - rename the current unit_errors directory (if it exists)
     - empty the unit_errors directory
@@ -36,106 +34,97 @@ def setUpClass(self):
     - perform the unit test
     - delete the unit_errors directory
     - rename the original unit_test folder back to unit_tests
-    '''
+    """
 
-# Test Cases:
+    # Test Cases:
 
     def test_check_unit_errors_success_below_min_errors(self):
-
         # Expecting no errors.
         # Test to ensure the number of dummy files is less than the overall min number of error files.
 
         params = self.params["valid_data"].copy()
-        
+
         num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
 
         self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
 
         expected_output = 0
         actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-        
+
         err_msg = "Number of dummy files IS NOT less than the overall min number of error files."
         assert expected_output == actual_output, err_msg
-        
+
         self.__remove_temp_unit_errors_folder(params["fim_dir"])
-            
-    def test_check_unit_errors_fail_above_min_errors(self):
 
+    def test_check_unit_errors_fail_above_min_errors(self):
         # Test to ensure the number of dummy files is more than the overall min number of error files.
         # Expecting sys.exit of 62
         # We do expect this to fail and if it fails, it is successful.
         # Here we expect an exception, and are capturing it using pytest.raises(Exception)
-        # To query the exception, or validate that it is the correct one, it is captured in the `e_info` object
 
         params = self.params["valid_data"].copy()
-        
+
         num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD + 1
-        
+
         self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-        
-        with pytest.raises(Exception) as e_info:
+
+        with pytest.raises(Exception):
             src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-          
+
         # We have to put the unit_errors folders back to the way it was
         self.__remove_temp_unit_errors_folder(params["fim_dir"])
 
-
     def test_check_unit_errors_success_above_percent_errors(self):
-
         # Expecting no errors.
         # Test to ensure the number of dummy files is more than the overall min number of error files.
         # We do expect this not to to fail as it is greater than 10 errors but below the percent threshhold.
 
         params = self.params["valid_data"].copy()
-        
+
         num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-            
+
         self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-            
+
         num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
         expected_output = 0
         actual_output = src.check_unit_errors(params["fim_dir"], num_total_units)
-            
+
         err_msg = "Number of dummy files IS NOT more than the overall min number of error files"
         assert expected_output == actual_output, err_msg
-            
-        # We have to put the unit_errors folders back to the way is was.           
-        self.__remove_temp_unit_errors_folder(params["fim_dir"])
 
+        # We have to put the unit_errors folders back to the way is was.
+        self.__remove_temp_unit_errors_folder(params["fim_dir"])
 
     def test_check_unit_errors_fail_below_percent_errors(self):
-
         # Expecting sys.exit of 62
         # Test to ensure the number of dummy files is more than the overall min number of error files.
-        
+
         # We do expect this to fail as it is greater than 10 errors
         # AND below the percent threshhold (more percent errors than the threshold)
         # Here we expect an exception, and are capturing it using pytest.raises(Exception)
-        # To query the exception, or validate that it is the correct one, it is captured in the `e_info` object
 
         params = self.params["valid_data"].copy()
-        
+
         num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-            
+
         self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-            
+
         num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) - 10
 
-        with pytest.raises(Exception) as e_info:
+        with pytest.raises(Exception):
             src.check_unit_errors(params["fim_dir"], num_total_units)
-          
+
         # We have to put the unit_errors folders back to the way it was.
         self.__remove_temp_unit_errors_folder(params["fim_dir"])
-    
 
-# Helper functions:
+    # Helper functions:
 
     def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files):
-        '''
+        """
         Process:
             We want to preserve the original unit_errors folder if it exists,
             so we wil rename it.
-            Then we will make a new unit_errors folder and fill it with a bunch of 
+            Then we will make a new unit_errors folder and fill it with a bunch of
             dummy files.
             A dummy file for non_zero_exit_codes.log will also be created.
         Input:
@@ -144,32 +133,32 @@ def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files)
         Returns:
             True if the 'unit_errors' folder did original exist and needs to be renamed back.
             False if the 'unit_errors' folder never existed in the first place.
-        '''
-        
+        """
+
         ue_folder_preexists = False
-        
-        if (not os.path.isdir(output_folder)):
+
+        if not os.path.isdir(output_folder):
             raise Exception(f"unit test root folder of {output_folder} does not exist")
-        
+
         ue_folder = os.path.join(output_folder, "unit_errors")
         temp_ue_folder = ue_folder + "_temp"
-        if (os.path.isdir(ue_folder)):
+        if os.path.isdir(ue_folder):
             ue_folder_preexists = True
             os.rename(ue_folder, temp_ue_folder)
-        
+
         os.mkdir(ue_folder)
-        
+
         for i in range(0, number_of_files):
             file_name = "sample_" + str(i) + ".txt"
             file_path = os.path.join(ue_folder, file_name)
-            with open(file_path, 'w') as fp:
+            with open(file_path, "w") as fp:
+                fp.close()
                 pass
-        
+
         return ue_folder_preexists
-        
-        
+
     def __remove_temp_unit_errors_folder(self, output_folder):
-        '''
+        """
         Process:
             We want to preserve the original unit_errors folder if it exists,
             so we will delete our temp unit test version of 'unit_errors', and rename
@@ -177,37 +166,33 @@ def __remove_temp_unit_errors_folder(self, output_folder):
             Note.. it is possible the temp folder does not exist,
             but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
         Input:
-            output_folder: the root output folder (ie. /data/outputs/fim_unit_test_data_do_not_remove/)
+            output_folder: the root output folder (ie. /data/outputs/unit_test_data/)
         Returns:
             nothing
-        '''
-
-        ue_folder = os.path.join(output_folder, "unit_errors")        
-        if (os.path.isdir(ue_folder)):
-            shutil.rmtree(ue_folder)       
-        
-        temp_ue_folder = ue_folder + "_temp"        
-        if (os.path.isdir(temp_ue_folder)):
+        """
+
+        ue_folder = os.path.join(output_folder, "unit_errors")
+        if os.path.isdir(ue_folder):
+            shutil.rmtree(ue_folder)
+
+        temp_ue_folder = ue_folder + "_temp"
+        if os.path.isdir(temp_ue_folder):
             os.rename(temp_ue_folder, ue_folder)
-    
 
-# Test Cases for Helper funcitons:
+    # Test Cases for Helper funcitons:
 
     def test_create_temp_unit_errors_folder_files(self):
-
-        # Here we are testing our helper function to see if it raise exceptions appropriately with a bad path. 
-        # In this case, we want the exception to be raised if there is an invalid path. 
+        # Here we are testing our helper function to see if it raise exceptions appropriately with a bad path.
+        # In this case, we want the exception to be raised if there is an invalid path.
 
         params = self.params["invalid_path"]
         invalid_folder = params["fim_dir"]
 
-        with pytest.raises(Exception) as e_info:
+        with pytest.raises(Exception):
             self.__create_temp_unit_errors_folder_files(invalid_folder, 4)
-    
-    
+
     def test_remove_temp_unit_errors_folder(self):
-        
-        # Test of out helper function to see if the temp folder was removed. 
+        # Test of out helper function to see if the temp folder was removed.
 
         params = self.params["valid_data"].copy()
 
@@ -217,5 +202,4 @@ def test_remove_temp_unit_errors_folder(self):
 
         temp_folder_created = os.path.join(params["fim_dir"], "unit_errors") + "_temp"
 
-        assert os.path.exists(temp_folder_created) == False
-
+        assert os.path.exists(temp_folder_created) is False
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
index 336e88086..a6e5ea0d6 100644
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ b/unit_tests/clip_vectors_to_wbd_params.json
@@ -1,28 +1,27 @@
 {
-	"valid_data": {
-		"subset_nwm_lakes": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_lakes_proj_subset.gpkg",
-		"subset_nwm_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
-		"hucCode": "05030104",
-		"subset_nwm_headwaters": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nhd_headwater_points_subset.gpkg",
-		"wbd_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered.gpkg",
-		"wbd_streams_buffer_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered_streams.gpkg",
-		"wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
-		"dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
-		"dem_domain" : "/data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg",
-		"nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
-		"nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
-		"subset_nwm_catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
-		"nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
-		"nld_lines_preprocessed": "/data/inputs/nld_vectors/3d_nld_preprocessed_230314.gpkg",
-		"landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
-		"nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
-		"subset_landsea": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LandSea_subset.gpkg",
-		"nwm_headwaters": "/data/inputs/nwm_hydrofabric/nwm_headwaters.gpkg",
-		"subset_nld_lines": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nld_subset_levees.gpkg",
-		"subset_nld_lines_preprocessed": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/3d_nld_subset_levees_burned.gpkg",
-		"wbd_buffer_distance": 5000,
-		"levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
-		"subset_levee_protected_areas": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/LeveeProtectedAreas_subset.gpkg"
-	}
+    "valid_data": {
+        "subset_nwm_lakes": "/data/outputs/unit_test_data/05030104/nwm_lakes_proj_subset.gpkg",
+        "subset_nwm_streams": "/data/outputs/unit_test_data/05030104/nwm_subset_streams.gpkg",
+        "hucCode": "05030104",
+        "subset_nwm_headwaters": "/data/outputs/unit_test_data/05030104/nhd_headwater_points_subset.gpkg",
+        "wbd_buffer_filename": "/data/outputs/unit_test_data/05030104/wbd_buffered.gpkg",
+        "wbd_streams_buffer_filename": "/data/outputs/unit_test_data/05030104/wbd_buffered_streams.gpkg",
+        "wbd_filename": "/data/outputs/unit_test_data/05030104/wbd.gpkg",
+        "dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
+        "dem_domain": "/data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg",
+        "nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
+        "nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
+        "subset_nwm_catchments": "/data/outputs/unit_test_data/05030104/nwm_catchments_proj_subset.gpkg",
+        "nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
+        "nld_lines_preprocessed": "/data/inputs/nld_vectors/3d_nld_preprocessed_230314.gpkg",
+        "landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
+        "nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
+        "subset_landsea": "/data/outputs/unit_test_data/05030104/LandSea_subset.gpkg",
+        "nwm_headwaters": "/data/inputs/nwm_hydrofabric/nwm_headwaters.gpkg",
+        "subset_nld_lines": "/data/outputs/unit_test_data/05030104/nld_subset_levees.gpkg",
+        "subset_nld_lines_preprocessed": "/data/outputs/unit_test_data/05030104/3d_nld_subset_levees_burned.gpkg",
+        "wbd_buffer_distance": 5000,
+        "levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
+        "subset_levee_protected_areas": "/data/outputs/unit_test_data/05030104/LeveeProtectedAreas_subset.gpkg"
+    }
 }
-
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
index dfe223d2c..015d797ae 100644
--- a/unit_tests/clip_vectors_to_wbd_test.py
+++ b/unit_tests/clip_vectors_to_wbd_test.py
@@ -1,69 +1,69 @@
 #!/usr/bin/env python3
 
-import os 
 import json
+import os
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
 import clip_vectors_to_wbd as src
 
 
 class test_clip_vectors_to_wbd(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases:
+    # Test Cases:
 
-    def test_subset_vector_layers_success(self):
+    # TODO New Test Case to check that files were being created on the file system
 
-        '''
-        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist as expected.
+    def test_subset_vector_layers_success(self):
+        """
+        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist.
         Most of the output test and internal tests with this function will test a wide variety of conditions.
-        For subcalls to other py classes will not exist in this file, but the unittest file for the other python file.
         Only the basic return output value should be tested to ensure it is as expected.
         For now, we are adding the very basic "happy path" test.
-        '''
+        """
 
         params = self.params["valid_data"].copy()
-       
+
         # There is no default return value.
-        # Later we need to check in this function that files were being created on the file system
-        
         # For now we are happy if no exceptions are thrown.
         try:
-            src.subset_vector_layers(subset_nwm_lakes = params["subset_nwm_lakes"],
-                                     subset_nwm_streams = params["subset_nwm_streams"],
-                                     hucCode = params["hucCode"],
-                                     subset_nwm_headwaters = params["subset_nwm_headwaters"],                               
-                                     wbd_buffer_filename = params["wbd_buffer_filename"],
-                                     wbd_streams_buffer_filename = params["wbd_streams_buffer_filename"],
-                                     wbd_filename = params["wbd_filename"],
-                                     dem_filename = params["dem_filename"],
-                                     dem_domain = params["dem_domain"],
-                                     nwm_lakes = params["nwm_lakes"],
-                                     nwm_catchments = params["nwm_catchments"],
-                                     subset_nwm_catchments = params["subset_nwm_catchments"],
-                                     nld_lines = params["nld_lines"],
-                                     nld_lines_preprocessed = params["nld_lines_preprocessed"],
-                                     landsea = params["landsea"],
-                                     nwm_streams = params["nwm_streams"],
-                                     subset_landsea = params["subset_landsea"],
-                                     nwm_headwaters = params["nwm_headwaters"],
-                                     subset_nld_lines = params["subset_nld_lines"],
-                                     subset_nld_lines_preprocessed = params["subset_nld_lines_preprocessed"],
-                                     wbd_buffer_distance = params["wbd_buffer_distance"],
-                                     levee_protected_areas = params["levee_protected_areas"],
-                                     subset_levee_protected_areas = params["subset_levee_protected_areas"])
+            src.subset_vector_layers(
+                subset_nwm_lakes=params["subset_nwm_lakes"],
+                subset_nwm_streams=params["subset_nwm_streams"],
+                hucCode=params["hucCode"],
+                subset_nwm_headwaters=params["subset_nwm_headwaters"],
+                wbd_buffer_filename=params["wbd_buffer_filename"],
+                wbd_streams_buffer_filename=params["wbd_streams_buffer_filename"],
+                wbd_filename=params["wbd_filename"],
+                dem_filename=params["dem_filename"],
+                dem_domain=params["dem_domain"],
+                nwm_lakes=params["nwm_lakes"],
+                nwm_catchments=params["nwm_catchments"],
+                subset_nwm_catchments=params["subset_nwm_catchments"],
+                nld_lines=params["nld_lines"],
+                nld_lines_preprocessed=params["nld_lines_preprocessed"],
+                landsea=params["landsea"],
+                nwm_streams=params["nwm_streams"],
+                subset_landsea=params["subset_landsea"],
+                nwm_headwaters=params["nwm_headwaters"],
+                subset_nld_lines=params["subset_nld_lines"],
+                subset_nld_lines_preprocessed=params["subset_nld_lines_preprocessed"],
+                wbd_buffer_distance=params["wbd_buffer_distance"],
+                levee_protected_areas=params["levee_protected_areas"],
+                subset_levee_protected_areas=params["subset_levee_protected_areas"],
+            )
 
         except (RuntimeError, TypeError, NameError) as e_info:
             pytest.fail("Error in subset_vector_layers function", e_info)
-
diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json
index 629d39860..df64eee5b 100644
--- a/unit_tests/derive_level_paths_params.json
+++ b/unit_tests/derive_level_paths_params.json
@@ -1,18 +1,17 @@
 {
-	"valid_data":
-	{
-		"outputDestDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
-		"in_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams.gpkg",
-		"wbd": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd.gpkg",
-		"buffer_wbd_streams": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/wbd_buffered_streams.gpkg",
-		"out_stream_network": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
+	"valid_data": {
+		"outputDestDir": "/data/outputs/unit_test_data/",
+		"in_stream_network": "/data/outputs/unit_test_data/05030104/nwm_subset_streams.gpkg",
+		"wbd": "/data/outputs/unit_test_data/05030104/wbd.gpkg",
+		"buffer_wbd_streams": "/data/outputs/unit_test_data/05030104/wbd_buffered_streams.gpkg",
+		"out_stream_network": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths.gpkg",
 		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
+		"out_stream_network_dissolved": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths.gpkg",
+		"headwaters_outfile": "/data/outputs/unit_test_data/05030104/nwm_headwaters.gpkg",
+		"catchments": "/data/outputs/unit_test_data/05030104/nwm_catchments_proj_subset.gpkg",
+		"catchments_outfile": "/data/outputs/unit_test_data/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
+		"branch_inlets_outfile": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
 		"reach_id_attribute": "ID",
 		"verbose": true
-	}	
+	}
 }
diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py
index ff37ed87d..acb89a538 100644
--- a/unit_tests/derive_level_paths_test.py
+++ b/unit_tests/derive_level_paths_test.py
@@ -1,18 +1,16 @@
 #!/usr/bin/env python3
 
+import inspect
+import json
 import os
 import sys
-import json
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-# # importing python folders in other direcories
-sys.path.append("/foss_fim/src/")
-import derive_level_paths as src
-import stream_branches
-from utils.fim_enums import FIM_exit_codes as fec
+from derive_level_paths import Derive_level_paths
+from stream_branches import StreamNetwork
 
 
 class test_Derive_level_paths(unittest.TestCase):
@@ -32,15 +30,15 @@ def setUpClass(self):
     def test_Derive_level_paths_success_all_params(self):
         """
         This test includes all params with many optional parms being set to the default value of the function
+        Notes:
+            Other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into
+            __main__ , so we skip them here.
+            Returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
         """
 
         params = self.params["valid_data"].copy()
 
-        # Notes:
-        # Other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into __main__ ,
-        # so we skip them here.
-        # Returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        actual_df = src.Derive_level_paths(
+        actual_df = Derive_level_paths(
             in_stream_network=params["in_stream_network"],
             buffer_wbd_streams=params["buffer_wbd_streams"],
             wbd=params["wbd"],
@@ -55,8 +53,8 @@ def test_Derive_level_paths_success_all_params(self):
             verbose=params["verbose"],
         )
 
-        # test data type being return is as expected. Downstream code might to know that type
-        assert isinstance(actual_df, stream_branches.StreamNetwork)
+        # test data type being return is as expected. Downstream code might to know the type.
+        assert isinstance(actual_df, StreamNetwork)
 
         # **** NOTE: Based on 05030104
         # Test row count for dissolved level path GeoDataframe which is returned.
@@ -66,36 +64,34 @@ def test_Derive_level_paths_success_all_params(self):
 
         # Test that output files exist as expected
         assert (
-            os.path.exists(params["out_stream_network"]) == True
+            os.path.exists(params["out_stream_network"]) is True
         ), f"Expected file {params['out_stream_network']} but it does not exist."
         assert (
-            os.path.exists(params["out_stream_network_dissolved"]) == True
+            os.path.exists(params["out_stream_network_dissolved"]) is True
         ), f"Expected file {params['out_stream_network_dissolved']} but it does not exist."
         assert (
-            os.path.exists(params["headwaters_outfile"]) == True
+            os.path.exists(params["headwaters_outfile"]) is True
         ), f"Expected file {params['headwaters_outfile']} but it does not exist."
         assert (
-            os.path.exists(params["catchments_outfile"]) == True
+            os.path.exists(params["catchments_outfile"]) is True
         ), f"Expected file {params['catchments_outfile']} but it does not exist."
         assert (
-            os.path.exists(params["catchments_outfile"]) == True
+            os.path.exists(params["catchments_outfile"]) is True
         ), f"Expected file {params['catchments_outfile']} but it does not exist."
         assert (
-            os.path.exists(params["branch_inlets_outfile"]) == True
+            os.path.exists(params["branch_inlets_outfile"]) is True
         ), f"Expected file {params['branch_inlets_outfile']} but it does not exist."
 
-    # Invalid Input stream for demo purposes. Normally, you would not have this basic of a test (input validation).
+    # Invalid Input stream for demo purposes.
     def test_Derive_level_paths_invalid_input_stream_network(self):
         # NOTE: As we are expecting an exception, we use pytest.raises(Exception).
 
         params = self.params["valid_data"].copy()
         params["in_stream_network"] = "some bad path"
 
-        with pytest.raises(Exception) as e_info:
-            actual = src.Derive_level_paths(
-                in_stream_network=ut_helpers.json_concat(
-                    params, "outputDestDir", "in_stream_network"
-                ),
+        with pytest.raises(Exception):
+            Derive_level_paths(
+                in_stream_network=ut_helpers.json_concat(params, "outputDestDir", "in_stream_network"),
                 out_stream_network=params["out_stream_network"],
                 branch_id_attribute=params["branch_id_attribute"],
                 out_stream_network_dissolved=params["out_stream_network_dissolved"],
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
index 2638a2ceb..84a4b5b19 100644
--- a/unit_tests/filter_catchments_and_add_attributes_params.json
+++ b/unit_tests/filter_catchments_and_add_attributes_params.json
@@ -1,12 +1,11 @@
 {
-	"valid_data":
-	{
-		"outputDestDir":  "/outputs/fim_unit_test_data_do_not_remove",
-		"input_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
-		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
-		"output_catchments_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
-		"output_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
-	    "wbd_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
-		"huc_code": "02020005"
-	}
+    "valid_data": {
+        "outputDestDir": "/outputs/unit_test_data",
+        "input_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
+        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
+        "output_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
+        "output_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
+        "wbd_filename": "/data/outputs/unit_test_data/02020005/wbd8_clp.gpkg",
+        "huc_code": "02020005"
+    }
 }
diff --git a/unit_tests/filter_catchments_and_add_attributes_test.py b/unit_tests/filter_catchments_and_add_attributes_test.py
index 258ff2349..554f4fce4 100644
--- a/unit_tests/filter_catchments_and_add_attributes_test.py
+++ b/unit_tests/filter_catchments_and_add_attributes_test.py
@@ -1,43 +1,43 @@
 #!/usr/bin/env python3
 
 import inspect
+import json
 import os
 import sys
-
-import json
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
 import filter_catchments_and_add_attributes as src
 
+
 # *************
-# Important: For this to work, when you run gms_run_branch.sh, you have to 
+# Important: For this to work, when you run gms_run_branch.sh, you have to
 # use deny_gms_branches_dev.lst or the word "none" for the deny list arguments
 # (unit and branch deny list parameters). Key files need to exist for this unit test to work.
 class test_filter_catchments_and_add_attributes(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases:
+    # Test Cases:
 
     def test_filter_catchments_and_add_attributes_success(self):
-
-        '''
+        """
         The gw_catchments_reaches_filtered_addedAttributes_<branchID>.gpkg and
         demDerived_reaches_split_filtered_<branchid>.gpkg should not exit prior to this test.
         If the test is successful, these file will be created.
-        '''
+        """
 
-        params = self.params["valid_data"].copy() 
+        params = self.params["valid_data"].copy()
 
         # To setup the test, lets start by deleted the two expected output files to ensure
         # that they are regenerated.
@@ -45,22 +45,22 @@ def test_filter_catchments_and_add_attributes_success(self):
             os.remove(params["output_flows_filename"])
         if os.path.exists(params["output_catchments_filename"]):
             os.remove(params["output_catchments_filename"])
-        
+
         # Test that the files were deleted
-        assert os.path.exists(params["output_flows_filename"]) == False
-
-        assert os.path.exists(params["output_catchments_filename"]) == False
-        
-        src.filter_catchments_and_add_attributes(input_catchments_filename = params["input_catchments_filename"],
-                                                input_flows_filename = params["input_flows_filename"],
-                                                output_catchments_filename = params["output_catchments_filename"],
-                                                output_flows_filename = params["output_flows_filename"],
-                                                wbd_filename = params["wbd_filename"],
-                                                huc_code = params["huc_code"])
-    
-
-        # Test that the files were created by filer_catchments_and_add_attributes       
-        assert os.path.exists(params["output_flows_filename"]) == True
-        
-        assert os.path.exists(params["output_catchments_filename"]) == True
+        assert os.path.exists(params["output_flows_filename"]) is False
+
+        assert os.path.exists(params["output_catchments_filename"]) is False
+
+        src.filter_catchments_and_add_attributes(
+            input_catchments_filename=params["input_catchments_filename"],
+            input_flows_filename=params["input_flows_filename"],
+            output_catchments_filename=params["output_catchments_filename"],
+            output_flows_filename=params["output_flows_filename"],
+            wbd_filename=params["wbd_filename"],
+            huc_code=params["huc_code"],
+        )
+
+        # Test that the files were created by filer_catchments_and_add_attributes
+        assert os.path.exists(params["output_flows_filename"]) is True
 
+        assert os.path.exists(params["output_catchments_filename"]) is True
diff --git a/unit_tests/generate_branch_list_csv_params.json b/unit_tests/generate_branch_list_csv_params.json
index 123171471..0e2f2574a 100644
--- a/unit_tests/generate_branch_list_csv_params.json
+++ b/unit_tests/generate_branch_list_csv_params.json
@@ -1,20 +1,17 @@
 {
-	"valid_data_add_branch_zero":
-	{
-		"huc_id": "05030104",
-		"branch_id": "0",
-		"output_branch_csv": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
-	},
-	"valid_data_add_branch":
-	{
-		"huc_id": "05030104",
-		"branch_id": "1946000003",
-		"output_branch_csv": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.csv"
-	},	
-	"invalid_bad_file_extension":
-	{
-		"huc_id": "05030104",
-		"branch_id": "1946000003",
-		"output_branch_csv": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids2"
-	}	
+    "valid_data_add_branch_zero": {
+        "huc_id": "05030104",
+        "branch_id": "0",
+        "output_branch_csv": "/data/outputs/unit_test_data/05030104/branch_ids.csv"
+    },
+    "valid_data_add_branch": {
+        "huc_id": "05030104",
+        "branch_id": "1946000003",
+        "output_branch_csv": "/data/outputs/unit_test_data/05030104/branch_ids.csv"
+    },
+    "invalid_bad_file_extension": {
+        "huc_id": "05030104",
+        "branch_id": "1946000003",
+        "output_branch_csv": "/data/outputs/unit_test_data/05030104/branch_ids2"
+    }
 }
diff --git a/unit_tests/generate_branch_list_csv_test.py b/unit_tests/generate_branch_list_csv_test.py
index 5d2d75281..020556e97 100644
--- a/unit_tests/generate_branch_list_csv_test.py
+++ b/unit_tests/generate_branch_list_csv_test.py
@@ -1,68 +1,66 @@
 #!/usr/bin/env python3
 
-import os
 import json
+import os
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 import generate_branch_list_csv as src
 
+
 class test_generate_branch_list_csv(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-        # for these tests to work, we have to check if the .csv exists and remove it 
+        # for these tests to work, we have to check if the .csv exists and remove it
         # prior to exections of the tests.
-        
+
         params = self.params["valid_data_add_branch_zero"].copy()
-        if (os.path.exists(params["output_branch_csv"])):
+        if os.path.exists(params["output_branch_csv"]):
             os.remove(params["output_branch_csv"])
 
-# Test Cases:      
+    # Test Cases:
 
     def test_generate_branch_list_csv_valid_data_add_branch_zero_success(self):
-
         # yes.. we know that we can not control the order
-        
-        #global params_file
-        params = self.params["valid_data_add_branch_zero"].copy()
 
-        src.generate_branch_list_csv(huc_id = params["huc_id"],
-                                     branch_id = params["branch_id"],
-                                     output_branch_csv = params["output_branch_csv"])
-       
+        # global params_file
+        params = self.params["valid_data_add_branch_zero"].copy()
 
+        src.generate_branch_list_csv(
+            huc_id=params["huc_id"],
+            branch_id=params["branch_id"],
+            output_branch_csv=params["output_branch_csv"],
+        )
 
     def test_generate_branch_list_csv_valid_data_add_branch_success(self):
-    
-        # yes.. we know that we can not control the order
-        
-        #global params_file
+        # global params_file
         params = self.params["valid_data_add_branch"].copy()
 
-        src.generate_branch_list_csv(huc_id = params["huc_id"],
-                                     branch_id = params["branch_id"],
-                                     output_branch_csv = params["output_branch_csv"])
-       
+        src.generate_branch_list_csv(
+            huc_id=params["huc_id"],
+            branch_id=params["branch_id"],
+            output_branch_csv=params["output_branch_csv"],
+        )
 
     def test_generate_branch_list_csv_invalid_bad_file_extension(self):
-        
-        #global params_file
+        # global params_file
         params = self.params["invalid_bad_file_extension"].copy()
 
-        # we expect this to fail. If it does fail with an exception, then this test is sucessful.
-        with pytest.raises(Exception) as e_info:
-            src.generate_branch_list_csv(huc_id = params["huc_id"],
-                                        branch_id = params["branch_id"],
-                                        output_branch_csv = params["output_branch_csv"])
-
+        # We expect this to fail. If it does fail with an exception, then this test is sucessful.
+        with pytest.raises(Exception):
+            src.generate_branch_list_csv(
+                huc_id=params["huc_id"],
+                branch_id=params["branch_id"],
+                output_branch_csv=params["output_branch_csv"],
+            )
diff --git a/unit_tests/generate_branch_list_params.json b/unit_tests/generate_branch_list_params.json
index a7c862295..88962a708 100644
--- a/unit_tests/generate_branch_list_params.json
+++ b/unit_tests/generate_branch_list_params.json
@@ -1,8 +1,7 @@
 {
-	"valid_data":
-	{
-		"stream_network_dissolved": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
-		"branch_id_attribute": "levpa_id",
-		"output_branch_list_file": "/data/outputs/fim_unit_test_data_do_not_remove/05030104/branch_ids.lst"
-	}
+    "valid_data": {
+        "stream_network_dissolved": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
+        "branch_id_attribute": "levpa_id",
+        "output_branch_list_file": "/data/outputs/unit_test_data/05030104/branch_ids.lst"
+    }
 }
diff --git a/unit_tests/generate_branch_list_test.py b/unit_tests/generate_branch_list_test.py
index d05d322c4..ab654db89 100644
--- a/unit_tests/generate_branch_list_test.py
+++ b/unit_tests/generate_branch_list_test.py
@@ -1,34 +1,34 @@
 #!/usr/bin/env python3
 
-import os
 import json
+import os
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 import generate_branch_list as src
 
+
 class test_Generate_branch_list(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases:
+    # Test Cases:
 
     def test_Generate_branch_list_success(self):
-  
         params = self.params["valid_data"].copy()
 
-        src.generate_branch_list(stream_network_dissolved = params["stream_network_dissolved"],
-                                 branch_id_attribute = params["branch_id_attribute"],
-                                 output_branch_list_file = params["output_branch_list_file"])
-
-
+        src.generate_branch_list(
+            stream_network_dissolved=params["stream_network_dissolved"],
+            branch_id_attribute=params["branch_id_attribute"],
+            output_branch_list_file=params["output_branch_list_file"],
+        )
diff --git a/unit_tests/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json
index a545e6753..516292c3f 100644
--- a/unit_tests/outputs_cleanup_params.json
+++ b/unit_tests/outputs_cleanup_params.json
@@ -1,25 +1,20 @@
 {
-	"valid_specific_branch_data":
-	{
-		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000009",
-		"deny_list": "/foss_fim/config/deny_branches.lst",
-		"branch_id": "3246000009",
-		"verbose": true
-	},
-
-	"valid_directory_data":
-	{
-		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
-		"deny_list": "/foss_fim/config/deny_branches.lst",
-		"branch_id": "0",
-		"verbose": true
-	},
-
-	"skip_clean":
-	{
-		"src_dir": "/data/outputs/fim_unit_test_data_do_not_remove",
-		"deny_list": "None",
-		"branch_id": "",
-		"verbose": true
-	}	
-}
\ No newline at end of file
+    "valid_specific_branch_data": {
+        "src_dir": "/data/outputs/unit_test_data/02020005/branches/3246000009",
+        "deny_list": "/foss_fim/config/deny_branches.lst",
+        "branch_id": "3246000009",
+        "verbose": true
+    },
+    "valid_directory_data": {
+        "src_dir": "/data/outputs/unit_test_data",
+        "deny_list": "/foss_fim/config/deny_branches.lst",
+        "branch_id": "0",
+        "verbose": true
+    },
+    "skip_clean": {
+        "src_dir": "/data/outputs/unit_test_data",
+        "deny_list": "None",
+        "branch_id": "",
+        "verbose": true
+    }
+}
diff --git a/unit_tests/outputs_cleanup_test.py b/unit_tests/outputs_cleanup_test.py
index ab1a46a92..e3c9e5149 100644
--- a/unit_tests/outputs_cleanup_test.py
+++ b/unit_tests/outputs_cleanup_test.py
@@ -1,156 +1,162 @@
 #!/usr/bin/env python3
 
+import io
+import json
 import os
 import sys
-import json
-from pathlib import Path
-import io
-from contextlib import redirect_stdout
 import unittest
-import pytest
+from contextlib import redirect_stdout
+from pathlib import Path
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-from utils.shared_functions import FIM_Helpers as fh
-
 
-sys.path.append('/foss_fim/src/gms') 
-import outputs_cleanup as src
+import src.outputs_cleanup as src
+from utils.shared_functions import FIM_Helpers as fh
 
 
 class test_outputs_cleanup(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-
-# Test Cases: 
+    # Test Cases:
 
     def test_remove_deny_list_files_specific_branch_success(self):
-        '''
-        This validates removal of files for a directory already pointing to a 
+        """
+        This validates removal of files for a directory already pointing to a
         specific branch in a HUC
-        '''
+        """
 
         params = self.params["valid_specific_branch_data"].copy()
 
         # Gather all of the file names into an array from the deny_list
-        deny_files = self.__get_deny_list_filenames(params["src_dir"], params["deny_list"], params["branch_id"])
+        deny_files = self.__get_deny_list_filenames(
+            params["src_dir"], params["deny_list"], params["branch_id"]
+        )
 
-        # Test whether we have a list of files to check 
+        # Test whether we have a list of files to check
         assert len(deny_files) > 0
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
 
-        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) == True
+        src.remove_deny_list_files(
+            src_dir=params["src_dir"],
+            deny_list=params["deny_list"],
+            branch_id=params["branch_id"],
+            verbose=params["verbose"],
+        )
 
+        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) is True
 
     def test_remove_deny_list_files_huc_level_success(self):
-        '''
+        """
         This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but 
+        Normally used for covering all hucs and their branch zeros but
         can be anything
-        '''
+        """
 
         params = self.params["valid_directory_data"].copy()
 
         # Gather all of the file names into an array from the deny_list
-        deny_files = self.__get_deny_list_filenames(params["src_dir"], params["deny_list"], params["branch_id"])
+        deny_files = self.__get_deny_list_filenames(
+            params["src_dir"], params["deny_list"], params["branch_id"]
+        )
 
-        # Test whether we have a list of files to check 
+        # Test whether we have a list of files to check
         assert len(deny_files) > 0
-        
-        src.remove_deny_list_files(src_dir = params["src_dir"],
-                                   deny_list = params["deny_list"],
-                                   branch_id = params["branch_id"],
-                                   verbose = params["verbose"])
-                                   
-        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) == True
 
+        src.remove_deny_list_files(
+            src_dir=params["src_dir"],
+            deny_list=params["deny_list"],
+            branch_id=params["branch_id"],
+            verbose=params["verbose"],
+        )
+
+        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) is True
 
     def test_remove_deny_list_skip_cleaning_success(self):
-        '''
+        """
         This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but 
+        Normally used for covering all hucs and their branch zeros but
         can be anything
-        '''
+        """
 
         params = self.params["skip_clean"].copy()
 
-        deny_files = self.__get_deny_list_filenames(params["src_dir"], params["deny_list"], params["branch_id"])
+        deny_files = self.__get_deny_list_filenames(
+            params["src_dir"], params["deny_list"], params["branch_id"]
+        )
 
-        # Ensure we have a value of "None" for a deny_list value, 
-        # __get_deny_list_filenames returns an empty array if "None" is provided 
+        # Ensure we have a value of "None" for a deny_list value,
+        # __get_deny_list_filenames returns an empty array if "None" is provided
         assert len(deny_files) == 0
-        
-        # This is a little tricky, as we're capturing the stdout (return statement) from remove_deny_list_files,
-        # to verify the function is returning at the correct place, and not removing files when we do not provide
-        # a deny list file. We set f to the io stream, and redirect it using redirect_stdout. 
+
+        # This is tricky, as we're capturing the stdout (return statement) from remove_deny_list_files,
+        # to verify the function is returning at the correct place, and not removing files
+        # when we do not provide a deny list file. We set f to the io stream, and redirect it using
+        # redirect_stdout.
         f = io.StringIO()
         with redirect_stdout(f):
-            src.remove_deny_list_files(src_dir = params["src_dir"],
-                                       deny_list = params["deny_list"],
-                                       branch_id = params["branch_id"],
-                                       verbose = params["verbose"])
-        
+            src.remove_deny_list_files(
+                src_dir=params["src_dir"],
+                deny_list=params["deny_list"],
+                branch_id=params["branch_id"],
+                verbose=params["verbose"],
+            )
+
         # Get the stdout value of remove_deny_list_files and set it to skip_clean_out
-        skip_clean_out = f.getvalue()                           
-        
+        skip_clean_out = f.getvalue()
+
         # This string must match the print statement in /src/gms/outputs_cleanup.py, including the \n newline,
         # which occurs "behind the scenes" with every call to print() in Python
         assert skip_clean_out == "file clean via the deny list skipped\n"
 
-
     def test_remove_deny_list_files_invalid_src_directory(self):
-        '''
+        """
         Double check the src directory exists
-        '''
+        """
 
-        params = self.params["valid_specific_branch_data"].copy()    
+        params = self.params["valid_specific_branch_data"].copy()
         params["src_dir"] = "/data/does_no_exist"
 
         # We want an exception to be thrown here, if so, the test passes.
-        with pytest.raises(Exception) as e_info:
-
-            src.remove_deny_list_files(src_dir = params["src_dir"],
-                                       deny_list = params["deny_list"],
-                                       branch_id = params["branch_id"],
-                                       verbose = params["verbose"])
-
+        with pytest.raises(Exception):
+            src.remove_deny_list_files(
+                src_dir=params["src_dir"],
+                deny_list=params["deny_list"],
+                branch_id=params["branch_id"],
+                verbose=params["verbose"],
+            )
 
     def test_remove_deny_list_files_invalid_deny_list_does_not_exist(self):
-        '''
+        """
         Double check the deny list exists
-        '''
+        """
 
-        params = self.params["valid_specific_branch_data"].copy()    
+        params = self.params["valid_specific_branch_data"].copy()
         params["deny_list"] = "invalid_file_name.txt"
 
         # We want an exception to be thrown here, if so, the test passes.
-        with pytest.raises(Exception) as e_info:
-        
-            src.remove_deny_list_files(src_dir = params["src_dir"],
-                                       deny_list = params["deny_list"],
-                                       branch_id = params["branch_id"],
-                                       verbose = params["verbose"])
+        with pytest.raises(Exception):
+            src.remove_deny_list_files(
+                src_dir=params["src_dir"],
+                deny_list=params["deny_list"],
+                branch_id=params["branch_id"],
+                verbose=params["verbose"],
+            )
 
-
-# Helper Functions:
+    # Helper Functions:
 
     def __get_deny_list_filenames(self, src_dir, deny_list, branch_id):
-
         deny_list_files = []
 
-        if (deny_list == "None"):
+        if deny_list == "None":
             return deny_list_files
 
         # Note: some of the deny_file_names might be a comment line
@@ -158,31 +164,25 @@ def __get_deny_list_filenames(self, src_dir, deny_list, branch_id):
         deny_file_names = fh.load_list_file(deny_list.strip())
 
         for deny_file_name in deny_file_names:
-        
             # Only add files to the list that do not start with a #
             deny_file_name = deny_file_name.strip()
-            if (deny_file_name.startswith("#")):
-                continue 
-            
+            if deny_file_name.startswith("#"):
+                continue
+
             deny_file_name = deny_file_name.replace("{}", branch_id)
 
             deny_list_files.append(deny_file_name)
-                    
-        return deny_list_files
 
+        return deny_list_files
 
     def __check_no_deny_list_files_exist(self, src_dir, deny_array):
-
         found_files = []
 
         for file_name in deny_array:
-
             found_files.append(os.path.join(src_dir, file_name))
 
         for found_file in found_files:
-
             if os.path.exists(found_file):
                 return False
-        
-        return True
 
+        return True
diff --git a/unit_tests/pyproject.toml b/unit_tests/pyproject.toml
index 655af66aa..3ed480446 100644
--- a/unit_tests/pyproject.toml
+++ b/unit_tests/pyproject.toml
@@ -4,5 +4,5 @@ filterwarnings = [
     "ignore::UserWarning",
     "ignore::DeprecationWarning",
     "ignore::PendingDeprecationWarning",
-    "ignore::RuntimeWarning"
+    "ignore::RuntimeWarning",
 ]
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
index 24949dcb6..89c254900 100644
--- a/unit_tests/rating_curve_comparison_params.json
+++ b/unit_tests/rating_curve_comparison_params.json
@@ -1,19 +1,17 @@
 {
-	"valid_data":
-	{
-		"02020005":
-		[
-			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_elev_table.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches",
-			"/data/inputs/usgs_gages/usgs_rating_curves.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
-			"/data/outputs/fim_unit_test_data_do_not_remove/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
-			"/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
-			"/data/inputs/usgs_gages/catfim_flows_cms.csv",
-			"02020005",
-			false,
-			false
-		]
-	}
-}
\ No newline at end of file
+    "valid_data": {
+        "02020005": [
+            "/data/outputs/unit_test_data/02020005/usgs_elev_table.csv",
+            "/data/outputs/unit_test_data/02020005/branches",
+            "/data/inputs/usgs_gages/usgs_rating_curves.csv",
+            "/data/outputs/unit_test_data/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
+            "/data/outputs/unit_test_data/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
+            "/data/outputs/unit_test_data/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
+            "/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
+            "/data/inputs/usgs_gages/catfim_flows_cms.csv",
+            "02020005",
+            false,
+            false
+        ]
+    }
+}
diff --git a/unit_tests/rating_curve_comparison_test.py b/unit_tests/rating_curve_comparison_test.py
index f1f4ec3d3..110d58c8a 100644
--- a/unit_tests/rating_curve_comparison_test.py
+++ b/unit_tests/rating_curve_comparison_test.py
@@ -1,53 +1,55 @@
 #!/usr/bin/env python3
 
-import os
 import json
+import os
 import unittest
-import pytest
 
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+import pytest
 from rating_curve_comparison import generate_rating_curve_metrics
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 
 class test_rating_curve_comparison(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases
-     
-    @pytest.mark.skip(reason="rating_curve has invalid pathing based on files "\
-        "that are normally not in a runtime folder. This test will disabled for now.")
-    def test_generate_rating_curve_metrics_02020005_success(self):
+    # Test Cases
 
-        '''
-        We are testing whether a .png file was created for a FIM-USGS rating curve comparison, 
+    @pytest.mark.skip(
+        reason="rating_curve has invalid pathing based on files "
+        "that are normally not in a runtime folder. This test will disabled for now."
+    )
+    def test_generate_rating_curve_metrics_02020005_success(self):
+        """
+        We are testing whether a .png file was created for a FIM-USGS rating curve comparison,
         for HUC 02020005 using the `generate_rating_curve_metrics` function.
         The 5th index (parameter) for each HUC in `rating_curve_comparison_params.json` specifies
         the FIM-USGS rating curve comparison .png filepath.
-        '''
-        
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-        
-        _indiv_huc_params = params["02020005"] 
+        """
+
+        params = self.params[
+            "valid_data"
+        ].copy()  # update "valid_data" value if you need to (aka.. more than one node)
+
+        _indiv_huc_params = params["02020005"]
 
         # To setup the test, lets start by deleted the expected output file to ensure
         # that it is regenerated.
         if os.path.exists(_indiv_huc_params[5]):
             os.remove(_indiv_huc_params[5])
-        
+
         # Test that the file was deleted
-        assert os.path.exists(_indiv_huc_params[5]) == False
+        assert os.path.exists(_indiv_huc_params[5]) is False
 
         generate_rating_curve_metrics(_indiv_huc_params)
 
-        # Test that the file was created by generate_rating_curve_metrics       
-        assert os.path.exists(_indiv_huc_params[5]) == True
-
+        # Test that the file was created by generate_rating_curve_metrics
+        assert os.path.exists(_indiv_huc_params[5]) is True
diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json
index cb2b9be90..aff840dd5 100644
--- a/unit_tests/shared_functions_params.json
+++ b/unit_tests/shared_functions_params.json
@@ -1,17 +1,17 @@
 {
-	"append_append_id_to_file_name_single_identifier_valid":
-	{
-		"outputDestDir": "/output/fim_unit_test_data_do_not_remove/",
-		"file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
-		"identifier": "8888",
-		"expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_8888.tif"
-	},
-
-	"append_append_id_to_file_name_identifier_list_valid":
-	{
-		"outputDestDir": "/output/fim_unit_test_data_do_not_remove/",
-		"file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
-		"identifier": ["7777", "8888"],
-		"expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_7777_8888.tif"
-	}	
-}
\ No newline at end of file
+    "append_append_id_to_file_name_single_identifier_valid": {
+        "outputDestDir": "/output/unit_test_data/",
+        "file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
+        "identifier": "8888",
+        "expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_8888.tif"
+    },
+    "append_append_id_to_file_name_identifier_list_valid": {
+        "outputDestDir": "/output/unit_test_data/",
+        "file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
+        "identifier": [
+            "7777",
+            "8888"
+        ],
+        "expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_7777_8888.tif"
+    }
+}
diff --git a/unit_tests/shared_functions_test.py b/unit_tests/shared_functions_test.py
index a2a3b0b1a..e5046bb2e 100644
--- a/unit_tests/shared_functions_test.py
+++ b/unit_tests/shared_functions_test.py
@@ -2,72 +2,73 @@
 
 import json
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
 from utils.shared_functions import FIM_Helpers as src
 
+
 class test_shared_functions(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
-    
-# Test Cases:
 
-    def test_append_id_to_file_name_single_identifier_success(self):
+    # Test Cases:
 
-        '''
+    def test_append_id_to_file_name_single_identifier_success(self):
+        """
         Pass in a file name with the single identifier and get the single adjusted file name back
-        '''
+        """
 
-        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy() 
+        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()
 
-        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputDestDir" , "file_name"),
-                                                   identifier = params["identifier"])
+        actual_output = src.append_id_to_file_name(
+            file_name=ut_helpers.json_concat(params, "outputDestDir", "file_name"),
+            identifier=params["identifier"],
+        )
 
         err_msg = "actual output does not match expected output"
 
-        expected_output = ut_helpers.json_concat(params, "outputDestDir" , "expected_output")
-        
-        assert  expected_output == actual_output, err_msg
+        expected_output = ut_helpers.json_concat(params, "outputDestDir", "expected_output")
 
+        assert expected_output == actual_output, err_msg
 
     def test_append_id_to_file_name_indentifer_list_success(self):
+        """
+        Pass in a file name with the list of identifiers and
+        get a file name back with multiple identifers added.
+        """
 
-        '''
-        Pass in a file name with the list of identifiers and get a file name back with multiple identifers added
-        '''
-        
-        params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()        
+        params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()
 
-        actual_output = src.append_id_to_file_name(file_name = ut_helpers.json_concat(params, "outputDestDir" , "file_name"),
-                                                   identifier = params["identifier"])
+        actual_output = src.append_id_to_file_name(
+            file_name=ut_helpers.json_concat(params, "outputDestDir", "file_name"),
+            identifier=params["identifier"],
+        )
 
         err_msg = "actual output does not match expected output"
-        
-        expected_output = ut_helpers.json_concat(params, "outputDestDir" , "expected_output")
-        
-        assert expected_output == actual_output, err_msg
 
+        expected_output = ut_helpers.json_concat(params, "outputDestDir", "expected_output")
 
-    def test_append_id_to_file_name_no_file_name_success(self):
+        assert expected_output == actual_output, err_msg
 
-        '''
+    def test_append_id_to_file_name_no_file_name_success(self):
+        """
         Pass in an non existant file name and get None back
-        '''
-        
-        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()        
-        
-        actual_output = src.append_id_to_file_name(None, identifier = params["identifier"])
+        """
 
-        error_msg = "actual output should not have a value"
+        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()
 
-        assert actual_output == None, error_msg
+        actual_output = src.append_id_to_file_name(None, identifier=params["identifier"])
+
+        error_msg = "actual output should not have a value"
 
+        assert actual_output is None, error_msg
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
index a3a8e07de..e7f7676c9 100644
--- a/unit_tests/split_flows_params.json
+++ b/unit_tests/split_flows_params.json
@@ -1,16 +1,15 @@
 {
-	"valid_data":
-	{
-		"outputDestDir" : "/data/outputs/fim_unit_test_data_do_not_remove/",
-		"max_length": 1500,
-		"slope_min": 0.001,
-		"lakes_buffer_input": 20,
-		"flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
-		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-	    "split_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
-		"split_points_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
-		"wbd8_clp_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/wbd8_clp.gpkg",
-		"lakes_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_lakes_proj_subset.gpkg",
-		"nwm_streams_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/nwm_subset_streams_levelPaths.gpkg"
-	}
+    "valid_data": {
+        "outputDestDir": "/data/outputs/unit_test_data/",
+        "max_length": 1500,
+        "slope_min": 0.001,
+        "lakes_buffer_input": 20,
+        "flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
+        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+        "split_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
+        "split_points_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
+        "wbd8_clp_filename": "/data/outputs/unit_test_data/02020005/wbd8_clp.gpkg",
+        "lakes_filename": "/data/outputs/unit_test_data/02020005/nwm_lakes_proj_subset.gpkg",
+        "nwm_streams_filename": "/data/outputs/unit_test_data/02020005/nwm_subset_streams_levelPaths.gpkg"
+    }
 }
diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py
index 6b94fd2c7..b7ee9b0d8 100644
--- a/unit_tests/split_flows_test.py
+++ b/unit_tests/split_flows_test.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-import os
 import json
+import os
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 import split_flows as src
@@ -11,28 +11,29 @@
 
 class test_split_flows(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases:
+    # Test Cases:
 
-    # Ensure split_flows_filename & split_points_filename are created by the split_flows function 
+    # Ensure split_flows_filename & split_points_filename are created by the split_flows function
     def test_split_flows_success(self):
+        """
+        The /data/outputs/unit_test_data/<huc>/branches/<branchid>/
+            demDerived_reaches_split_<branchid>.gpkg and
+        /data/outputs/unit_test_data/<huc>/branches/<branchid>/
+            demDerived_reaches_split_points_<branchid>.gpkg
+        should not exit prior to this test. If the test is successful, these files will be created.
+        """
 
-        '''
-        The /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_<branchid>.gpkg and
-        /data/outputs/fim_unit_test_data_do_not_remove/<huc>/branches/<branchid>/demDerived_reaches_split_points_<branchid>.gpkg should not exit prior to this test.
-        If the test is successful, these files will be created.
-        '''
-
-        params = self.params["valid_data"].copy() 
+        params = self.params["valid_data"].copy()
 
         # to setup the test, lets start by deleted the two expected output files to ensure
         # that they are regenerated
@@ -42,25 +43,26 @@ def test_split_flows_success(self):
             os.remove(params["split_points_filename"])
 
         error_msg = params["split_flows_filename"] + " does exist, when it should not (post os.remove call)"
-        assert os.path.exists(params["split_flows_filename"]) == False, error_msg 
-        
+        assert os.path.exists(params["split_flows_filename"]) is False, error_msg
+
         error_msg = params["split_points_filename"] + " does exist, when it should not (post os.remove call)"
-        assert os.path.exists(params["split_points_filename"]) == False, error_msg
-       
-        src.split_flows(max_length = params["max_length"],
-                        slope_min = params["slope_min"],
-                        lakes_buffer_input = params["lakes_buffer_input"],
-                        flows_filename = params["flows_filename"],
-                        dem_filename = params["dem_filename"],
-                        split_flows_filename = params["split_flows_filename"],
-                        split_points_filename = params["split_points_filename"],
-                        wbd8_clp_filename = params["wbd8_clp_filename"],
-                        lakes_filename = params["lakes_filename"],
-                        nwm_streams_filename = params["nwm_streams_filename"])
-       
+        assert os.path.exists(params["split_points_filename"]) is False, error_msg
+
+        src.split_flows(
+            max_length=params["max_length"],
+            slope_min=params["slope_min"],
+            lakes_buffer_input=params["lakes_buffer_input"],
+            flows_filename=params["flows_filename"],
+            dem_filename=params["dem_filename"],
+            split_flows_filename=params["split_flows_filename"],
+            split_points_filename=params["split_points_filename"],
+            wbd8_clp_filename=params["wbd8_clp_filename"],
+            lakes_filename=params["lakes_filename"],
+            nwm_streams_filename=params["nwm_streams_filename"],
+        )
+
         error_msg = params["split_flows_filename"] + " does not exist"
-        assert os.path.exists(params["split_flows_filename"]) == True, error_msg 
-        
-        error_msg = params["split_points_filename"] + " does not exist"
-        assert os.path.exists(params["split_points_filename"]) == True, error_msg
+        assert os.path.exists(params["split_flows_filename"]) is True, error_msg
 
+        error_msg = params["split_points_filename"] + " does not exist"
+        assert os.path.exists(params["split_points_filename"]) is True, error_msg
diff --git a/unit_tests/tools/inundate_gms_params.json b/unit_tests/tools/inundate_gms_params.json
index 4b52ba45e..72af7da14 100644
--- a/unit_tests/tools/inundate_gms_params.json
+++ b/unit_tests/tools/inundate_gms_params.json
@@ -1,15 +1,14 @@
 {
-	"valid_data_inudation_raster_single_huc":
-	{
-		"hydrofabric_dir": "/data/outputs/fim_unit_test_data_do_not_remove/",
+	"valid_data_inudation_raster_single_huc": {
+		"hydrofabric_dir": "/data/outputs/unit_test_data/",
 		"forecast": "/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_2_0_cms.csv",
 		"num_workers": 4,
 		"hucs": "02020005",
-		"inundation_raster": "/data/outputs/fim_unit_test_data_do_not_remove/inundation_extent_02020005.tif",
+		"inundation_raster": "/data/outputs/unit_test_data/inundation_extent_02020005.tif",
 		"inundation_polygon": null,
 		"depths_raster": null,
 		"verbose": true,
-		"log_file": "/data/outputs/fim_unit_test_data_do_not_remove/logs/inundation_logfile.txt",
-		"output_fileNames": "/data/outputs/fim_unit_test_data_do_not_remove/logs/inundation_file_list.csv"
+		"log_file": "/data/outputs/unit_test_data/logs/inundation_logfile.txt",
+		"output_fileNames": "/data/outputs/unit_test_data/logs/inundation_file_list.csv"
 	}
 }
diff --git a/unit_tests/tools/inundate_gms_test.py b/unit_tests/tools/inundate_gms_test.py
index cdb8b77e8..b15557a7b 100644
--- a/unit_tests/tools/inundate_gms_test.py
+++ b/unit_tests/tools/inundate_gms_test.py
@@ -1,43 +1,40 @@
 #!/usr/bin/env python3
 
+import json
 import os
 import sys
-
-import json
 import unittest
+
+import inundate_gms as src
 import pandas as pd
 import pytest
 
-sys.path.append('/foss_fim/unit_tests/')
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-sys.path.append('/foss_fim/tools')
-import inundate_gms as src
+from unit_tests.unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 
 class test_inundate_gms(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-# Test Cases:
+    # Test Cases:
     def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self):
-        '''
+        """
         Test for creating a gms inundation rasters, not a depth raster and no
         inundation_polygons.
-        
-        This test is essentially testing the Inundate_gms function, and the creation of a 
+
+        This test is essentially testing the Inundate_gms function, and the creation of a
         the "output_fileNames" .csv file.
-        '''
-        
-        params = self.params["valid_data_inudation_raster_single_huc"].copy()         
+        """
+
+        params = self.params["valid_data_inudation_raster_single_huc"].copy()
 
         # Clear previous outputs, if they exist.
         for file in os.listdir(params["hydrofabric_dir"]):
@@ -46,48 +43,48 @@ def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self
 
         if os.path.isfile(params["output_fileNames"]):
             os.remove(params["output_fileNames"])
-                
+
         # Test the Inundate_gms function
-        output_fileNames_df = src.Inundate_gms(hydrofabric_dir = params["hydrofabric_dir"],
-                                               forecast = params["forecast"],
-                                               num_workers = params["num_workers"],
-                                               hucs = params["hucs"],
-                                               inundation_raster = params["inundation_raster"],
-                                               inundation_polygon = params["inundation_polygon"],
-                                               depths_raster = params["depths_raster"],
-                                               verbose = params["verbose"], 
-                                               log_file = None,
-                                               output_fileNames = params["output_fileNames"]
-                                               )
+        output_fileNames_df = src.Inundate_gms(
+            hydrofabric_dir=params["hydrofabric_dir"],
+            forecast=params["forecast"],
+            num_workers=params["num_workers"],
+            hucs=params["hucs"],
+            inundation_raster=params["inundation_raster"],
+            inundation_polygon=params["inundation_polygon"],
+            depths_raster=params["depths_raster"],
+            verbose=params["verbose"],
+            log_file=None,
+            output_fileNames=params["output_fileNames"],
+        )
 
         # Check if output files df has records.
         assert len(output_fileNames_df) > 0, "Expected as least one dataframe record"
 
     def test_Inundate_gms_create_inundation_rasters(self):
-
-        '''
+        """
         Test for creating a gms inundation rasters, not a depth raster and no
         inundation_polygons.
         This test is based on creating a raster based on a single huc and its branches
         within the output folder.
-        '''
-        
-        params = self.params["valid_data_inudation_raster_single_huc"].copy() 
-        
+        """
+
+        params = self.params["valid_data_inudation_raster_single_huc"].copy()
+
         # Check all output rasters exist.
         csv_out = pd.read_csv(params["output_fileNames"], skipinitialspace=True)
-        
+
         csv_rasters = csv_out.inundation_rasters
-        
+
         actual_rasters = []
         for file in os.listdir(params["hydrofabric_dir"]):
             if file.endswith(".tif"):
                 actual_rasters.append(file)
-        
-        for csv_raster, actual_raster in zip(csv_rasters, actual_rasters):
-            assert os.path.exists(os.path.join(params["hydrofabric_dir"], actual_raster)), f"Inundation Raster {csv_raster} does not exist"
 
+        for csv_raster, actual_raster in zip(csv_rasters, actual_rasters):
+            assert os.path.exists(
+                os.path.join(params["hydrofabric_dir"], actual_raster)
+            ), f"Inundation Raster {csv_raster} does not exist"
 
     # Logging removed from tools/inundate_gms.py as of 8/1/23
     # assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
-
diff --git a/unit_tests/tools/inundation_params.json b/unit_tests/tools/inundation_params.json
index 6e2b1d800..b566789f8 100644
--- a/unit_tests/tools/inundation_params.json
+++ b/unit_tests/tools/inundation_params.json
@@ -1,24 +1,23 @@
 {
-	"valid_data_inundate_branch":
-	{
-		"rem": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
-		"catchments": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
-		"catchment_poly": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
-		"hydro_table": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000001/hydroTable_3246000001.csv",
-	    "forecast": "/data/test_cases/usgs_test_cases/validation_data_usgs/02020005/ptvn6/action/ahps_ptvn6_huc_02020005_flows_action.csv",
+	"valid_data_inundate_branch": {
+		"rem": "/data/outputs/unit_test_data/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
+		"catchments": "/data/outputs/unit_test_data/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
+		"catchment_poly": "/data/outputs/unit_test_data/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
+		"hydro_table": "/data/outputs/unit_test_data/02020005/branches/3246000001/hydroTable_3246000001.csv",
+		"forecast": "/data/test_cases/usgs_test_cases/validation_data_usgs/02020005/ptvn6/action/ahps_ptvn6_huc_02020005_flows_action.csv",
 		"mask_type": null,
 		"hucs": null,
 		"hucs_layerName": null,
 		"subset_hucs": null,
 		"num_workers": 1,
 		"aggregate": false,
-		"inundation_raster": "/data/test_cases/usgs_test_cases/02020005_usgs/testing_versions/fim_unit_test_data_do_not_remove/action/ptvn6_inundation_extent_02020005_3246000001.tif",
+		"inundation_raster": "/data/test_cases/usgs_test_cases/02020005_usgs/testing_versions/unit_test_data/action/ptvn6_inundation_extent_02020005_3246000001.tif",
 		"inundation_polygon": null,
 		"depths": null,
 		"out_raster_profile": null,
 		"out_vector_profile": null,
 		"src_table": null,
 		"quiet": true,
-		"expected_inundation_raster": "/data/" 
+		"expected_inundation_raster": "/data/"
 	}
-}
\ No newline at end of file
+}
diff --git a/unit_tests/tools/inundation_test.py b/unit_tests/tools/inundation_test.py
index 3cb4c6517..c8348ae7e 100644
--- a/unit_tests/tools/inundation_test.py
+++ b/unit_tests/tools/inundation_test.py
@@ -1,66 +1,66 @@
 #!/usr/bin/env python3
 
+import json
 import os
 import sys
-import json
 import unittest
-import pytest
 
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 import inundation as src
+import pytest
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 
 class test_inundate(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
+    # Test Cases:
 
-# Test Cases:
-
-    @pytest.mark.skip(reason="Inundate_gms will be rebuilt in the future, so this test will be left broken.")
+    @pytest.mark.skip(reason="Inundate_gms will be rebuilt in the future, so this test will be skipped.")
     def test_inundate_create_inundation_raster_single_branch_success(self):
-        '''
+        """
         Test for creating a inundation branch raster, no depth raster and no
         inundation_polygons, no subsets, no mask
-        '''
+        """
+
+        params = self.params["valid_data_inundate_branch"].copy()
 
-        params = self.params["valid_data_inundate_branch"].copy()         
-        
         # returns list of rasters and polys
-        in_rasters, depth_rasters, in_polys = src.inundate(rem = params["rem"],
-                                                           catchments = params["catchments"],
-                                                           catchment_poly = params["catchment_poly"],
-                                                           hydro_table = params["hydro_table"],
-                                                           forecast = params["forecast"],
-                                                           mask_type = params["mask_type"],
-                                                           hucs = params["hucs"],
-                                                           hucs_layerName = params["hucs_layerName"],
-                                                           subset_hucs = params["subset_hucs"],
-                                                           num_workers = params["num_workers"],
-                                                           aggregate = params["aggregate"], 
-                                                           inundation_raster = params["inundation_raster"],
-                                                           inundation_polygon = params["inundation_polygon"],
-                                                           depths = params["depths"],
-                                                           out_raster_profile = params["out_raster_profile"],
-                                                           out_vector_profile = params["out_vector_profile"],
-                                                           src_table = params["src_table"],
-                                                           quiet = params["quiet"])
-        
+        in_rasters, depth_rasters, in_polys = src.inundate(
+            rem=params["rem"],
+            catchments=params["catchments"],
+            catchment_poly=params["catchment_poly"],
+            hydro_table=params["hydro_table"],
+            forecast=params["forecast"],
+            mask_type=params["mask_type"],
+            hucs=params["hucs"],
+            hucs_layerName=params["hucs_layerName"],
+            subset_hucs=params["subset_hucs"],
+            num_workers=params["num_workers"],
+            aggregate=params["aggregate"],
+            inundation_raster=params["inundation_raster"],
+            inundation_polygon=params["inundation_polygon"],
+            depths=params["depths"],
+            out_raster_profile=params["out_raster_profile"],
+            out_vector_profile=params["out_vector_profile"],
+            src_table=params["src_table"],
+            quiet=params["quiet"],
+        )
+
         print("in_rasters")
         print(in_rasters)
-        
+
         assert len(in_rasters) == 1, "Expected exactly one inundation raster path records"
-        assert depth_rasters[0] == None, "Expected no depth raster path records"
-        assert in_polys[0] == None, "Expected no inundation_polys records"
-        
-        msg = f"Expected file {params['expected_inundation_raster']} but it does not exist."
-        assert os.path.exists(params["expected_inundation_raster"]) == True, msg
+        assert depth_rasters[0] is None, "Expected no depth raster path records"
+        assert in_polys[0] is None, "Expected no inundation_polys records"
 
+        msg = f"Expected file {params['expected_inundation_raster']} but it does not exist."
+        assert os.path.exists(params["expected_inundation_raster"]) is True, msg
diff --git a/unit_tests/unit_tests_utils.py b/unit_tests/unit_tests_utils.py
index d2c7223eb..847158b0d 100644
--- a/unit_tests/unit_tests_utils.py
+++ b/unit_tests/unit_tests_utils.py
@@ -4,23 +4,21 @@
 import os
 import sys
 
+
 class FIM_unit_test_helpers(object):
-    
     @staticmethod
     def get_params_filename(unit_test_file_name):
-       
         unittest_file_name = os.path.basename(unit_test_file_name)
         params_file_name = unittest_file_name.replace("_test.py", "_params.json")
         params_file_path = os.path.join(os.path.dirname(unit_test_file_name), params_file_name)
-        
-        if (not os.path.exists(params_file_path)):
+
+        if not os.path.exists(params_file_path):
             raise FileNotFoundError(f"{params_file_path} does not exist")
-        
+
         return params_file_path
-    
+
     @staticmethod
     def json_concat(obj, keyone, keytwo):
-        
-        concat_string = obj[keyone] + obj[keytwo] 
-        
-        return concat_string
\ No newline at end of file
+        concat_string = obj[keyone] + obj[keytwo]
+
+        return concat_string
diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json
index 885c0fdd6..33ec7cd41 100644
--- a/unit_tests/usgs_gage_crosswalk_params.json
+++ b/unit_tests/usgs_gage_crosswalk_params.json
@@ -1,12 +1,11 @@
 {
-	"valid_data":
-	{
-		"usgs_gages_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/usgs_subset_gages.gpkg",
-		"input_flows_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
-		"input_catchment_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
-		"dem_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_meters_3246000005.tif",
-		"dem_adj_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-		"output_table_filename": "/data/outputs/fim_unit_test_data_do_not_remove/02020005/branches/3246000005/usgs_elev_table.csv",
-		"branch_id": "3246000005"
-	}
+    "valid_data": {
+        "usgs_gages_filename": "/data/outputs/unit_test_data/02020005/usgs_subset_gages.gpkg",
+        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
+        "input_catchment_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
+        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/dem_meters_3246000005.tif",
+        "dem_adj_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
+        "output_table_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/usgs_elev_table.csv",
+        "branch_id": "3246000005"
+    }
 }
diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py
index fccc0117c..0fe7d41c9 100644
--- a/unit_tests/usgs_gage_crosswalk_test.py
+++ b/unit_tests/usgs_gage_crosswalk_test.py
@@ -1,58 +1,60 @@
 #!/usr/bin/env python3
 
+import json
 import os
 import sys
-import json
 import unittest
-import pytest
 
+import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+
 from usgs_gage_crosswalk import GageCrosswalk
 
 
 class test_usgs_gage_crosswalk(unittest.TestCase):
 
-    '''
+    """
     Allows the params to be loaded one and used for all test methods
-    '''
+    """
+
     @classmethod
     def setUpClass(self):
-
         params_file_path = ut_helpers.get_params_filename(__file__)
         with open(params_file_path) as params_file:
             self.params = json.load(params_file)
 
-
-# Test Cases:
+    # Test Cases:
 
     def test_GageCrosswalk_success(self):
+        """
+        Test whether the GageCrosswalk object can be instantiated, and test that the run_crosswalk method can
+        successfully create the output table (usgs_elev_table.csv).
+        """
 
-        '''
-        Test whether the GageCrosswalk object can be instantiated, and test that the run_crosswalk method can successfully 
-        create the output table (usgs_elev_table.csv).
-        '''
+        params = self.params[
+            "valid_data"
+        ].copy()  # update "valid_data" value if you need to (aka.. more than one node)
 
-        params = self.params["valid_data"].copy()  #update "valid_data" value if you need to (aka.. more than one node)
-       
         # Delete the usgs_elev_table.csv if it exists
         if os.path.exists(params["output_table_filename"]):
             os.remove(params["output_table_filename"])
 
         # Verify the usgs_elev_table.csv was deleted
         msg = f'{params["output_table_filename"]} does exist, when it should have been deleted'
-        assert os.path.exists(params["output_table_filename"]) == False, msg 
+        assert os.path.exists(params["output_table_filename"]) is False, msg
 
         # Instantiate and run GageCrosswalk
         gage_crosswalk = GageCrosswalk(params["usgs_gages_filename"], params["branch_id"])
 
         # Run crosswalk
-        gage_crosswalk.run_crosswalk(params["input_catchment_filename"], \
-                                     params["input_flows_filename"], \
-                                     params["dem_filename"], \
-                                     params["dem_adj_filename"], \
-                                     params["output_table_filename"])
+        gage_crosswalk.run_crosswalk(
+            params["input_catchment_filename"],
+            params["input_flows_filename"],
+            params["dem_filename"],
+            params["dem_adj_filename"],
+            params["output_table_filename"],
+        )
 
         # Make sure that the usgs_elev_table.csv was written
         msg = f'{params["output_table_filename"]} does not exist'
-        assert os.path.exists(params["output_table_filename"]) == True, msg 
-
+        assert os.path.exists(params["output_table_filename"]) is True, msg

From b6ef9b572969fe0d48a5f138dba19fb43265e2a8 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 20 Sep 2023 15:16:15 -0400
Subject: [PATCH 275/359] v4.4.1.1 Bugfix for `MultiLineString` unit error in
 `src/stream_branches.py` (#992)

---
 CITATION.cff           |  2 +-
 docs/CHANGELOG.md      | 12 ++++++++++++
 pyproject.toml         |  9 +++++----
 src/split_flows.py     | 13 ++++++++-----
 src/stream_branches.py | 15 ++++++++++++---
 5 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index 8498f274d..0c47bf8d3 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.4.1.0
+version: 4.4.1.1
 date-released: 2023
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index dc1631156..b47e6720b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.1.1 - 2023-09-20 - [PR#992](https://github.com/NOAA-OWP/inundation-mapping/pull/992)
+
+Fixes errors caused when a GeoDataFrame contains a `MultiLineString` geometry instead of a `LineString`. Update black force-exclude list.
+
+### Changes
+
+- `src/`
+    `split_flows.py` and `stream_branches.py`: Converts `MultiLineString` geometry into `LineString`s.
+- `pyproject.toml` : Add three files in `src/` to exclude list.
+
+<br/><br/>
+
 ## v4.4.1.0 - 2023-09-18 - [PR#988](https://github.com/NOAA-OWP/inundation-mapping/pull/988)
 
 Format code using `black` formatter, incorporate `isort` package to sort import statements,
diff --git a/pyproject.toml b/pyproject.toml
index 7613ddd00..af8b9d9e3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,10 +46,12 @@ Wiki = "https://github.com/NOAA-OWP/inundation-mapping/wiki"
 line_length = 110
 skip-string-normalization = true
 skip-magic-trailing-comma = true
-exclude = '''
-/(
+force-exclude = '''
     \.git
-)\
+    | src/add_crosswalk.py
+    | src/bathymetric_adjustment.py
+    | src/identify_src_bankfull.py
+    |
 '''
 
 [tool.isort]
@@ -57,7 +59,6 @@ profile = 'black'
 multi_line_output = 3
 line_length = 110 # It should be the same as `tool.black.line-length` above
 lines_after_imports = 2
-skip_gitignore = true
 
 [tool.flake8]
 count = true
diff --git a/src/split_flows.py b/src/split_flows.py
index 7b1818994..205c152e7 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -10,19 +10,17 @@
 '''
 
 import argparse
-import os
 import sys
-import time
 from collections import OrderedDict
-from os import environ, path, remove
-from os.path import dirname, isfile, split
+from os import remove
+from os.path import isfile
 
 import geopandas as gpd
 import numpy as np
 import pandas as pd
 import rasterio
 from shapely import ops, wkt
-from shapely.geometry import LineString, MultiPoint, Point
+from shapely.geometry import LineString, Point
 from shapely.ops import split as shapely_ops_split
 from tqdm import tqdm
 
@@ -139,6 +137,11 @@ def snap_and_trim_flow(snapped_point, flows):
 
         # Identify the end vertex (most downstream, should be last), transform into geodataframe
         terminal_nwm_point = []
+
+        if linestring_geo.geom_type == 'MultiLineString':
+            # Get last LineString
+            linestring_geo = linestring_geo.geoms[-1]
+
         last = Point(linestring_geo.coords[-1])
         terminal_nwm_point.append({'ID': 'terminal', 'geometry': last})
         snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(nwm_streams.crs)
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 0f4b97d51..f0e88cf74 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -15,7 +15,7 @@
 from rasterio.mask import mask
 from scipy.stats import mode
 from shapely.geometry import LineString, MultiLineString, MultiPoint, Point
-from shapely.ops import linemerge, unary_union
+from shapely.ops import linemerge
 from shapely.strtree import STRtree
 from tqdm import tqdm
 
@@ -365,8 +365,17 @@ def derive_inlet_points_by_feature(self, feature_attribute, outlet_linestring_in
 
         feature_inlet_points_gdf = gpd.GeoDataFrame(self.copy())
 
-        for idx, row in self.iterrows():
-            feature_inlet_point = Point(row.geometry.coords[inlet_linestring_index])
+        self_copy = self.copy()
+
+        for idx in self_copy.index:
+            row = self_copy.loc[[idx]]
+            if row.geom_type[idx] == "MultiLineString":
+                # Convert MultiLineString to LineString
+                row = row.explode(index_parts=False)
+                row.loc[row["levpa_id"].duplicated(), "levpa_id"] = np.nan
+                row = row.dropna(subset=["levpa_id"])
+
+            feature_inlet_point = Point(row.geometry[0].coords[inlet_linestring_index])
 
             feature_inlet_points_gdf.loc[idx, "geometry"] = feature_inlet_point
 

From cafa6be936fcf99e61ff10a2a043e679afd4bd4c Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 20 Sep 2023 15:25:54 -0400
Subject: [PATCH 276/359] v4.4.2.0 Fix pandas and geopandas warnings (#993)

---
 data/usgs/preprocess_ahps_usgs.py     |   2 +-
 data/write_parquet_from_calib_pts.py  |   2 +-
 docs/CHANGELOG.md                     |  46 +++++++++
 src/add_crosswalk.py                  |   4 +-
 src/bathy_src_adjust_topwidth.py      |  15 ++-
 src/crosswalk_nwm_demDerived.py       |   6 +-
 src/derive_level_paths.py             |   2 +-
 src/finalize_srcs.py                  |  12 +--
 src/identify_src_bankfull.py          |  10 +-
 src/split_flows.py                    |   2 +-
 src/src_adjust_usgs_rating.py         |  16 ++-
 src/src_roughness_optimization.py     |  50 +++++-----
 src/stream_branches.py                | 134 +++++++++++++++++++++-----
 src/subdiv_chan_obank_src.py          |  18 ++--
 src/subset_catch_list_by_branch_id.py |   2 +-
 src/usgs_gage_unit_setup.py           |  17 ++--
 src/utils/shared_functions.py         |   6 +-
 tools/adjust_rc_with_feedback.py      |  16 +--
 tools/aggregate_csv_files.py          |   2 +-
 tools/combine_crosswalk_tables.py     |   4 +-
 tools/inundation.py                   |  14 +--
 tools/make_boxes_from_bounds.py       |   8 +-
 tools/mosaic_inundation.py            |   4 +-
 tools/plots.py                        |  14 +--
 tools/rating_curve_comparison.py      |  12 +--
 tools/vary_mannings_n_composite.py    |  18 ++--
 26 files changed, 272 insertions(+), 164 deletions(-)

diff --git a/data/usgs/preprocess_ahps_usgs.py b/data/usgs/preprocess_ahps_usgs.py
index dddbf2c61..7c58181fc 100644
--- a/data/usgs/preprocess_ahps_usgs.py
+++ b/data/usgs/preprocess_ahps_usgs.py
@@ -325,7 +325,7 @@ def preprocess_usgs(source_dir, destination, reference_raster):
             f.write(f'{code} : Skipping because multisite\n')
             continue
         # Rename columns to match NWS AHPS data structure, this only applies to single USGS sites, if a multisite the columns are different from QCFS.
-        df.rename(columns={'QCFS': 'flow', 'STAGE': 'stage', 'ELEV': 'elevation'}, inplace=True)
+        df = df.rename(columns={'QCFS': 'flow', 'STAGE': 'stage', 'ELEV': 'elevation'})
         # Many USGS maps have elevations to numerous decimal places. Round to nearest tenth.
         # NWS has maps to nearest tenth, for example HARP1 is both USGS and NWS, the USGS maps are to the hundredth of foot and NWS are to tenth.
         df['elevation'] = round(df['elevation'], 1)
diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py
index 63ac4c666..ee8c9fbc2 100644
--- a/data/write_parquet_from_calib_pts.py
+++ b/data/write_parquet_from_calib_pts.py
@@ -162,7 +162,7 @@ def create_single_huc_gdf_and_write_parquet_file(args):
         return
 
     # Drop index_right column created by join
-    huc_with_points_gdf.drop(['index_right'], axis=1, inplace=True)
+    huc_with_points_gdf = huc_with_points_gdf.drop(['index_right'], axis=1)
 
     # Set the crs projection
     huc_gdf = huc_with_points_gdf.set_crs(DEFAULT_FIM_PROJECTION_CRS, allow_override=True)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b47e6720b..eebbb5703 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,52 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.2.0 - 2023-09-20 - [PR#993](https://github.com/NOAA-OWP/inundation-mapping/pull/993)
+
+Resolves the causes of two warnings in pandas and geopandas: (1) `FutureWarning` from taking the `int()` of single-length Series and (2) `SettingWithCopyWarning` resulting from the use of `inplace=True`.
+
+### Changes
+
+Removed `inplace=True` from
+- `data/`
+    - `usgs/preprocess_ahps_usgs.py`
+    - `write_parquet_from_calib_pts.py`
+- `src/`
+    - `add_crosswalk.py`
+    - `bathy_src_adjust_topwidth.py`
+    - `clip_vectors_to_wbd.py`
+    - `crosswalk_nwm_demDerived.py`
+    - `derive_level_paths.py`
+    - `finalize_srcs.py`
+    - `identify_src_bankfull.py`
+    - `src_adjust_usgs_rating.py`
+    - `src_roughness_optimization.py`
+    - `stream_branches.py`
+    - `subdiv_chan_obank_src.py`
+    - `subset_catch_list_by_branch_id.py`
+    - `usgs_gage_unit_setup.py`
+    - `utils/shared_functions.py`
+- `tools/`
+    - `adjust_rc_with_feedback.py`
+    - `aggregate_csv_files.py`
+    - `combine_crosswalk_tables.py`
+    - `eval_plots_stackedbar.py`
+    - `inundation.py`
+    - `make_boxes_from_bounds.py`
+    - `mosaic_inundation.py`
+    - `plots.py`
+    - `rating_curve_comparison.py`
+    - `vary_mannings_n_composite.py`
+
+Fixed single-length Series in
+- `src/`
+    - `split_flows.py`
+    - `stream_branches.py`
+
+- ``src/stream_branches.py``: Fixed class methods
+
+<br/><br/>
+
 ## v4.4.1.1 - 2023-09-20 - [PR#992](https://github.com/NOAA-OWP/inundation-mapping/pull/992)
 
 Fixes errors caused when a GeoDataFrame contains a `MultiLineString` geometry instead of a `LineString`. Update black force-exclude list.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index e326f225a..283f0ff30 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -2,12 +2,10 @@
 
 import argparse
 import json
-import os
 import sys
 
 import geopandas as gpd
 import pandas as pd
-from memory_profiler import profile
 from numpy import unique
 from rasterstats import zonal_stats
 
@@ -402,7 +400,7 @@ def add_crosswalk(
     if output_hydro_table.HUC.dtype != 'str':
         output_hydro_table.HUC = output_hydro_table.HUC.astype(str)
 
-    output_hydro_table.drop(columns=FIM_ID, inplace=True)
+    output_hydro_table = output_hydro_table.drop(columns=FIM_ID)
     if output_hydro_table.feature_id.dtype != 'int':
         output_hydro_table.feature_id = output_hydro_table.feature_id.astype(int)
     if output_hydro_table.feature_id.dtype != 'str':
diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py
index 04dd34b24..72e86441f 100644
--- a/src/bathy_src_adjust_topwidth.py
+++ b/src/bathy_src_adjust_topwidth.py
@@ -412,30 +412,29 @@ def bathy_rc_lookup(args):
                 'Discharge (m3s-1)',
             ],
         ]
-        modified_hydro_table.rename(
-            columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'}, inplace=True
+        modified_hydro_table = modified_hydro_table.rename(
+            columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'}
         )
         df_htable = pd.read_csv(input_htable_fileName, dtype={'HUC': str})
-        df_htable.drop(
-            ['barc_on'], axis=1, inplace=True
+        df_htable = df_htable.drop(
+            ['barc_on'], axis=1
         )  # drop the default "barc_on" variable from add_crosswalk.py
         if not set(
             ['orig_discharge_cms', 'orig_Volume (m3)', 'orig_WetArea (m2)', 'orig_HydraulicRadius (m)']
         ).issubset(
             df_htable.columns
         ):  # check if "orig_" attributes do NOT already exist (likely generated from previous BARC run)
-            df_htable.rename(
+            df_htable = df_htable.rename(
                 columns={
                     'discharge_cms': 'orig_discharge_cms',
                     'Volume (m3)': 'orig_Volume (m3)',
                     'WetArea (m2)': 'orig_WetArea (m2)',
                     'HydraulicRadius (m)': 'orig_HydraulicRadius (m)',
                 },
-                inplace=True,
             )
         else:
-            df_htable.drop(
-                ['discharge_cms', 'Volume (m3)', 'WetArea (m2)', 'HydraulicRadius (m)'], axis=1, inplace=True
+            df_htable = df_htable.drop(
+                ['discharge_cms', 'Volume (m3)', 'WetArea (m2)', 'HydraulicRadius (m)'], axis=1
             )  # drop the previously modified columns - to be replaced with updated version
         df_htable = df_htable.merge(
             modified_hydro_table, how='left', left_on=['HydroID', 'stage'], right_on=['HydroID', 'stage']
diff --git a/src/crosswalk_nwm_demDerived.py b/src/crosswalk_nwm_demDerived.py
index be267c89b..4ef4469b0 100755
--- a/src/crosswalk_nwm_demDerived.py
+++ b/src/crosswalk_nwm_demDerived.py
@@ -62,8 +62,8 @@ def Crosswalk_nwm_demDerived(
     )
 
     # merge crosswalk table
-    crosswalk_table.rename(columns={'ID': 'feature_id'}, inplace=True)
-    demDerived.drop(columns='feature_id', inplace=True, errors='raise')
+    crosswalk_table = crosswalk_table.rename(columns={'ID': 'feature_id'})
+    demDerived = demDerived.drop(columns='feature_id', errors='raise')
     demDerived['HydroID'] = demDerived['HydroID'].astype(int)
     demDerived = demDerived.merge(crosswalk_table, how='left', left_on='HydroID', right_index=True)
 
@@ -152,7 +152,7 @@ def Add_traversal_to_NWM(nwm_streams, node_prefix=None, outfile=None, verbose=Fa
     #     branch_id_attribute='levpa_id', attribute_excluded=None, values_excluded=None, verbose=verbose
     # )
 
-    nwm_streams.reset_index(drop=True, inplace=True)
+    nwm_streams = nwm_streams.reset_index(drop=True)
 
     if outfile is not None:
         nwm_streams.write(outfile, index=False, verbose=verbose)
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index ffa841de6..5236a0d8f 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -145,7 +145,7 @@ def Derive_level_paths(
             stream_network_to_merge, how="inner", left_on=reach_id_attribute, right_on=reach_id_attribute
         )
 
-        catchments.reset_index(drop=True, inplace=True)
+        catchments = catchments.reset_index(drop=True)
 
         catchments.to_file(catchments_outfile, index=False, driver="GPKG")
 
diff --git a/src/finalize_srcs.py b/src/finalize_srcs.py
index 3ca975bdf..33899aeb5 100755
--- a/src/finalize_srcs.py
+++ b/src/finalize_srcs.py
@@ -13,12 +13,12 @@
 def finalize_srcs(srcbase, srcfull, hydrotable, output_srcfull=None, output_hydrotable=None):
     # calculate src_full
     srcbase = pd.read_csv(srcbase, dtype={'CatchId': int})
-    srcbase.rename(columns={'CatchId': 'HydroID'}, inplace=True)
+    srcbase = srcbase.rename(columns={'CatchId': 'HydroID'})
     srcbase = srcbase.rename(columns=lambda x: x.strip(" "))
 
     # read and merge in attributes from base hydrofabric src full
     srcfull = pd.read_csv(srcfull, dtype={'CatchId': int})
-    srcfull.rename(columns={'CatchId': 'HydroID'}, inplace=True)
+    srcfull = srcfull.rename(columns={'CatchId': 'HydroID'})
     srcfull = srcfull.loc[:, ["ManningN", "HydroID", "feature_id"]].drop_duplicates(subset='HydroID')
 
     srcbase = srcbase.merge(srcfull, how='inner', left_on='HydroID', right_on='HydroID')
@@ -43,9 +43,9 @@ def finalize_srcs(srcbase, srcfull, hydrotable, output_srcfull=None, output_hydr
         srcbase.to_csv(output_srcfull, index=False)
 
     hydrotable = pd.read_csv(hydrotable)
-    hydrotable.drop(columns=['stage', 'discharge_cms'], inplace=True)
+    hydrotable = hydrotable.drop(columns=['stage', 'discharge_cms'])
 
-    hydrotable.drop_duplicates(subset='HydroID', inplace=True)
+    hydrotable = hydrotable.drop_duplicates(subset='HydroID')
     # srcfull = srcfull.loc[:,["ManningN","HydroID","feature_id"]].drop_duplicates(subset='HydroID')
     hydrotable = hydrotable.merge(
         srcbase.loc[:, ['HydroID', 'Stage', 'Discharge (m3s-1)']],
@@ -53,8 +53,8 @@ def finalize_srcs(srcbase, srcfull, hydrotable, output_srcfull=None, output_hydr
         left_on='HydroID',
         right_on='HydroID',
     )
-    hydrotable.rename(columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'}, inplace=True)
-    # hydrotable.drop_duplicates(subset='stage',inplace=True)
+    hydrotable = hydrotable.rename(columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'})
+    # hydrotable = hydrotable.drop_duplicates(subset='stage')
 
     if output_hydrotable is not None:
         hydrotable.to_csv(output_hydrotable, index=False)
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index efef55938..8b99e295d 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -125,7 +125,7 @@ def src_bankfull_lookup(args):
         df_bankfull_calc = df_bankfull_calc[
             df_bankfull_calc['Stage'] > 0.0
         ]  # Ensure bankfull stage is greater than stage=0
-        df_bankfull_calc.reset_index(drop=True, inplace=True)
+        df_bankfull_calc = df_bankfull_calc.reset_index(drop=True)
         # find the index of the Q_bfull_find (closest matching flow)
         df_bankfull_calc = df_bankfull_calc.loc[
             df_bankfull_calc.groupby('HydroID')['Q_bfull_find'].idxmin()
@@ -156,7 +156,7 @@ def src_bankfull_lookup(args):
             how='left',
             on='HydroID'
         )
-        df_src.drop(['Q_bfull_find'], axis=1, inplace=True)
+        df_src = df_src.drop(['Q_bfull_find'], axis=1)
 
         ## The bankfull ratio variables below were previously used for the composite variable roughness routine
         ##      (not currently implimented)
@@ -170,7 +170,7 @@ def src_bankfull_lookup(args):
         # df_src['chann_volume_ratio'].where(df_src['chann_volume_ratio'] <= 1.0, 1.0, inplace=True)
         # # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
         # df_src['chann_volume_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True)
-        # #df_src.drop(['Volume_bankfull'], axis=1, inplace=True)
+        # #df_src = df_src.drop(['Volume_bankfull'], axis=1)
 
         # ## Calculate the channel portion of bankfull Hydraulic Radius
         # df_src['chann_hradius_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
@@ -183,7 +183,7 @@ def src_bankfull_lookup(args):
         # df_src['chann_hradius_ratio'].where(df_src['chann_hradius_ratio'] <= 1.0, 1.0, inplace=True)
         # # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
         # df_src['chann_hradius_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True)
-        # #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
+        # #df_src = df_src.drop(['HRadius_bankfull'], axis=1)
 
         # ## Calculate the channel portion of bankfull Surface Area
         # df_src['chann_surfarea_ratio'] = 1.0 # At stage=0 set channel_ratio to 1.0 (avoid div by 0)
@@ -194,7 +194,7 @@ def src_bankfull_lookup(args):
         # df_src['chann_surfarea_ratio'].where(df_src['chann_surfarea_ratio'] <= 1.0, 1.0, inplace=True)
         # # if the bankfull_flow value <= 0 then set channel ratio to 0 (will use global overbank manning n)
         # df_src['chann_surfarea_ratio'].where(df_src['bankfull_flow'] > 0.0, 0.0, inplace=True)
-        # #df_src.drop(['HRadius_bankfull'], axis=1, inplace=True)
+        # #df_src = df_src.drop(['HRadius_bankfull'], axis=1)
 
         ## mask bankfull variables when the bankfull estimated flow value is <= 0
         df_src['Stage_bankfull'].mask(df_src['bankfull_flow'] <= 0.0, inplace=True)
diff --git a/src/split_flows.py b/src/split_flows.py
index 205c152e7..1cd86abee 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -53,7 +53,7 @@ def snap_and_trim_flow(snapped_point, flows):
             if len(sjoin_nearest) > 1:
                 sjoin_nearest = sjoin_nearest[sjoin_nearest['LINKNO'].isin(sjoin_nearest['DSLINKNO'])]
 
-            nearest_index = int(sjoin_nearest['LINKNO'])
+            nearest_index = int(sjoin_nearest['LINKNO'].iloc[0])
             flow = flows[flows['LINKNO'] == nearest_index]
             flow.index = [0]
 
diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py
index d4684a8e5..d93c6848f 100644
--- a/src/src_adjust_usgs_rating.py
+++ b/src/src_adjust_usgs_rating.py
@@ -1,12 +1,9 @@
 import argparse
 import datetime as dt
-import json
 import multiprocessing
 import os
 import sys
-from collections import deque
 from multiprocessing import Pool
-from pathlib import Path
 
 import pandas as pd
 
@@ -63,8 +60,8 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
     cross_df = usgs_elev_df[
         ["location_id", "HydroID", "feature_id", "levpa_id", "HUC8", "dem_adj_elevation"]
     ].copy()
-    cross_df.rename(
-        columns={'dem_adj_elevation': 'hand_datum', 'HydroID': 'hydroid', 'HUC8': 'huc'}, inplace=True
+    cross_df = cross_df.rename(
+        columns={'dem_adj_elevation': 'hand_datum', 'HydroID': 'hydroid', 'HUC8': 'huc'}
     )
 
     # filter null location_id rows from cross_df
@@ -89,7 +86,7 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
     # read in the NWM recurr csv file
     nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
     nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
-    nwm_recur_df.rename(
+    nwm_recur_df = nwm_recur_df.rename(
         columns={
             '2_0_year_recurrence_flow_17C': '2_0_year',
             '5_0_year_recurrence_flow_17C': '5_0_year',
@@ -97,8 +94,7 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
             '25_0_year_recurrence_flow_17C': '25_0_year',
             '50_0_year_recurrence_flow_17C': '50_0_year',
             '100_0_year_recurrence_flow_17C': '100_0_year',
-        },
-        inplace=True,
+        }
     )
 
     # convert cfs to cms (x 0.028317)
@@ -146,7 +142,7 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
         # Assign new metadata attributes
         calc_df['nwm_recur'] = interval + "_0_year"
         calc_df['layer'] = '_usgs-gage____' + interval + "-year"
-        calc_df.rename(columns={interval + "_0_year": 'nwm_recur_flow_cms'}, inplace=True)
+        calc_df = calc_df.rename(columns={interval + "_0_year": 'nwm_recur_flow_cms'})
         # Subset calc_df for final output
         calc_df = calc_df[
             [
@@ -179,7 +175,7 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
         final_df['coll_time'] = str(datestamp)[:15]
 
     # Rename attributes (for ingest to update_rating_curve) and output csv with the USGS RC database
-    final_df.rename(columns={'discharge_cms': 'flow'}, inplace=True)
+    final_df = final_df.rename(columns={'discharge_cms': 'flow'})
     final_df.to_csv(os.path.join(log_dir, "usgs_rc_nwm_recurr.csv"), index=False)
 
     # Output log text to log file
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index e294cd56f..e417753f6 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -155,14 +155,13 @@ def update_rating_curve(
         df_prev_adj_htable = df_htable.copy()[
             ['HydroID', 'submitter', 'last_updated', 'obs_source', 'calb_coef_final']
         ]
-        df_prev_adj_htable.rename(
+        df_prev_adj_htable = df_prev_adj_htable.rename(
             columns={
                 'submitter': 'submitter_prev',
                 'last_updated': 'last_updated_prev',
                 'calb_coef_final': 'calb_coef_final_prev',
                 'obs_source': 'obs_source_prev',
-            },
-            inplace=True,
+            }
         )
         df_prev_adj_htable = df_prev_adj_htable.groupby(["HydroID"]).first()
         # Only keep previous USGS rating curve adjustments (previous spatial obs adjustments are not retained)
@@ -180,7 +179,7 @@ def update_rating_curve(
 
     # Delete previous adj columns to prevent duplicate variable issues
     # (if src_roughness_optimization.py was previously applied)
-    df_htable.drop(
+    df_htable = df_htable.drop(
         [
             'discharge_cms',
             'submitter',
@@ -191,10 +190,9 @@ def update_rating_curve(
             'obs_source',
         ],
         axis=1,
-        inplace=True,
         errors='ignore',
     )
-    df_htable.rename(columns={'precalb_discharge_cms': 'discharge_cms'}, inplace=True)
+    df_htable = df_htable.rename(columns={'precalb_discharge_cms': 'discharge_cms'})
 
     ## loop through the user provided point data --> stage/flow dataframe row by row
     for index, row in df_nvalues.iterrows():
@@ -253,7 +251,7 @@ def update_rating_curve(
             df_nvalues.loc[index, 'discharge_cms'] = find_src_stage.discharge_cms
 
     ## Calculate calibration coefficient
-    df_nvalues.rename(columns={'hydroid': 'HydroID'}, inplace=True)  # rename the previous ManningN column
+    df_nvalues = df_nvalues.rename(columns={'hydroid': 'HydroID'})  # rename the previous ManningN column
     df_nvalues['hydroid_calb_coef'] = df_nvalues['discharge_cms'] / df_nvalues['flow']  # Qobs / Qsrc
 
     ## Calcuate a "calibration adjusted" n value using channel and overbank n-values multiplied by calb_coef
@@ -283,7 +281,7 @@ def update_rating_curve(
     df_nvalues['magnitude'] = df_nvalues['layer'].str.split("_").str[5]
     df_nvalues['ahps_lid'] = df_nvalues['layer'].str.split("_").str[1]
     df_nvalues['huc'] = str(huc)
-    df_nvalues.drop(['layer'], axis=1, inplace=True)
+    df_nvalues = df_nvalues.drop(['layer'], axis=1)
 
     ## Create df grouped by hydroid with ahps_lid and huc number
     df_huc_lid = df_nvalues.groupby(["HydroID"]).first()[['ahps_lid', 'huc']]
@@ -309,17 +307,17 @@ def update_rating_curve(
         df_updated = df_updated.sort_values('coll_time').drop_duplicates(
             ['HydroID'], keep='last'
         )  # sort by collection time and then drop duplicate HydroIDs (keep most recent coll_time per HydroID)
-        df_updated.rename(columns={'coll_time': 'last_updated'}, inplace=True)
+        df_updated = df_updated.rename(columns={'coll_time': 'last_updated'})
 
         ## cacluate median ManningN to handle cases with multiple hydroid entries
         df_mann_hydroid = df_nvalues.groupby(["HydroID"])[['hydroid_calb_coef']].median()
 
         ## Create a df with the median hydroid_ManningN value per feature_id
         # df_mann_featid = df_nvalues.groupby(["feature_id"])[['hydroid_ManningN']].mean()
-        # df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'}, inplace=True)
+        # df_mann_featid = df_mann_featid.rename(columns={'hydroid_ManningN':'featid_ManningN'})
 
         ## Rename the original hydrotable variables to allow new calculations to use the primary var name
-        df_htable.rename(columns={'discharge_cms': 'precalb_discharge_cms'}, inplace=True)
+        df_htable = df_htable.rename(columns={'discharge_cms': 'precalb_discharge_cms'})
 
         ## Check for large variabilty in the calculated Manning's N values
         # (for cases with mutliple entries for a singel hydroid)
@@ -367,7 +365,7 @@ def update_rating_curve(
 
             ## Create a df with the median hydroid_calb_coef value per feature_id
             df_mann_featid = df_nmerge.groupby(["feature_id"])[['hydroid_calb_coef']].mean()
-            df_mann_featid.rename(columns={'hydroid_calb_coef': 'featid_calb_coef'}, inplace=True)
+            df_mann_featid = df_mann_featid.rename(columns={'hydroid_calb_coef': 'featid_calb_coef'})
             # create a seperate df with attributes to apply to other hydroids that share a featureid
             df_mann_featid_attrib = df_nmerge.groupby('feature_id').first()
             df_mann_featid_attrib = df_mann_featid_attrib[df_mann_featid_attrib['submitter'].notna()][
@@ -389,10 +387,9 @@ def update_rating_curve(
                 choices = [df_nmerge['featid_calb_coef'], df_nmerge['group_calb_coef']]
                 df_nmerge[calb_type] = np.select(conditions, choices, default=df_nmerge['hydroid_calb_coef'])
                 df_nmerge['obs_source'] = np.where(df_nmerge[calb_type].notnull(), source_tag, pd.NA)
-                df_nmerge.drop(
+                df_nmerge = df_nmerge.drop(
                     ['feature_id', 'NextDownID', 'LENGTHKM', 'LakeID', 'order_'],
                     axis=1,
-                    inplace=True,
                     errors='ignore',
                 )  # drop these columns to avoid duplicates where merging with the full hydroTable df
 
@@ -420,10 +417,9 @@ def update_rating_curve(
                         df_nmerge['calb_coef_final_prev'],
                         df_nmerge[calb_type],
                     )
-                    df_nmerge.drop(
+                    df_nmerge = df_nmerge.drop(
                         ['submitter_prev', 'last_updated_prev', 'calb_coef_final_prev', 'obs_source_prev'],
                         axis=1,
-                        inplace=True,
                         errors='ignore',
                     )
                 else:
@@ -436,10 +432,9 @@ def update_rating_curve(
                     if (
                         'src_calibrated' in input_catchments.columns
                     ):  # check if this attribute already exists and drop if needed
-                        input_catchments.drop(
+                        input_catchments = input_catchments.drop(
                             ['src_calibrated', 'obs_source', 'calb_coef_final'],
                             axis=1,
-                            inplace=True,
                             errors='ignore',
                         )
                     df_nmerge['src_calibrated'] = np.where(
@@ -454,7 +449,7 @@ def update_rating_curve(
                     output_catchments.to_file(
                         catchments_poly_path, driver="GPKG", index=False
                     )  # overwrite the previous layer
-                    df_nmerge.drop(['src_calibrated'], axis=1, inplace=True, errors='ignore')
+                    df_nmerge = df_nmerge.drop(['src_calibrated'], axis=1, errors='ignore')
                 ## Optional ouputs:
                 #   1) merge_n_csv csv with all of the calculated n values
                 #   2) a catchments .gpkg with new joined attributes
@@ -474,7 +469,7 @@ def update_rating_curve(
                         output_catchments.to_file(output_catchments_fileName, driver="GPKG", index=False)
 
                 ## Merge the final ManningN dataframe to the original hydroTable
-                df_nmerge.drop(
+                df_nmerge = df_nmerge.drop(
                     [
                         'ahps_lid',
                         'start_catch',
@@ -485,7 +480,6 @@ def update_rating_curve(
                         'group_calb_coef',
                     ],
                     axis=1,
-                    inplace=True,
                     errors='ignore',
                 )  # drop these columns to avoid duplicates where merging with the full hydroTable df
                 df_htable = df_htable.merge(df_nmerge, how='left', on='HydroID')
@@ -576,7 +570,7 @@ def branch_network_tracer(df_input_htable):
     # other hydroids
     df_input_htable["start_catch"] = ~df_input_htable['HydroID'].isin(df_input_htable['NextDownID'])
 
-    df_input_htable.set_index('HydroID', inplace=True, drop=False)  # set index to the hydroid
+    df_input_htable = df_input_htable.set_index('HydroID', drop=False)  # set index to the hydroid
     branch_heads = deque(
         df_input_htable[df_input_htable['start_catch'] == True]['HydroID'].tolist()
     )  # create deque of hydroids to define start points in the while loop
@@ -621,17 +615,17 @@ def branch_network_tracer(df_input_htable):
                         # starting hydroid
                         continue
                     Q.append(nextid)
-    df_input_htable.reset_index(drop=True, inplace=True)  # reset index (previously using hydroid as index)
+    df_input_htable = df_input_htable.reset_index(drop=True)  # reset index (previously using hydroid as index)
     # sort the dataframe by branch_id and then by route_count
     # (need this ordered to ensure upstream to downstream ranking for each branch)
-    df_input_htable.sort_values(['branch_id', 'route_count'], inplace=True)
+    df_input_htable = df_input_htable.sort_values(['branch_id', 'route_count'])
     return df_input_htable
 
 
 def group_manningn_calc(df_nmerge, down_dist_thresh):
     ## Calculate group_calb_coef (mean calb n for consective hydroids) and apply values downsteam to
     # non-calb hydroids (constrained to first Xkm of hydroids - set downstream diststance var as input arg
-    # df_nmerge.sort_values(by=['NextDownID'], inplace=True)
+    # df_nmerge = df_nmerge.sort_values(by=['NextDownID'])
     dist_accum = 0
     hyid_count = 0
     hyid_accum_count = 0
@@ -708,9 +702,9 @@ def group_manningn_calc(df_nmerge, down_dist_thresh):
 
     ## Delete unnecessary intermediate outputs
     if 'hyid_count' in df_nmerge.columns:
-        df_nmerge.drop(
-            ['hyid_count', 'accum_dist', 'hyid_accum_count'], axis=1, inplace=True, errors='ignore'
+        df_nmerge = df_nmerge.drop(
+            ['hyid_count', 'accum_dist', 'hyid_accum_count'], axis=1, errors='ignore'
         )  # drop hydroid counter if it exists
     ## drop accum vars from group calc
-    # df_nmerge.drop(['accum_dist','hyid_accum_count'], axis=1, inplace=True)
+    # df_nmerge = df_nmerge.drop(['accum_dist','hyid_accum_count'], axis=1)
     return df_nmerge
diff --git a/src/stream_branches.py b/src/stream_branches.py
index f0e88cf74..3491f40be 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -105,6 +105,23 @@ def from_file(
 
         return cls(filtered_df, **inputs)
 
+    def GeoDataFrame_to_StreamNetwork(self, gdf):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+        crs = self.crs
+
+        self = gdf
+        self = self.set_crs(crs)
+
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+        return self
+
     def write(self, fileName, layer=None, index=True, verbose=False):
         """Gets driver Name from file extension for Geopandas writing"""
 
@@ -117,6 +134,79 @@ def write(self, fileName, layer=None, index=True, verbose=False):
 
         self.to_file(fileName, driver=driver, layer=layer, index=index)
 
+    def set_index(self, reach_id_attribute, drop=True):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+        crs = self.crs
+
+        self = super(gpd.GeoDataFrame, self)
+        self = self.set_index(reach_id_attribute, drop=drop)
+        self = self.set_crs(crs)
+
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+        return self
+
+    def reset_index(self, drop=True):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+        crs = self.crs
+
+        self = super(gpd.GeoDataFrame, self)
+        self = self.reset_index(drop=drop)
+        self = self.set_crs(crs)
+
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+        return self
+
+    def drop(self, labels=None, axis=0):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+        crs = self.crs
+        geometry = self.geometry
+
+        self = super(gpd.GeoDataFrame, self)
+        self = self.drop(labels=labels, axis=axis)
+        self = gpd.GeoDataFrame(self, crs=crs, geometry=geometry)
+
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+        return self
+
+    def dissolve(self, by=None):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+        crs = self.crs
+        geometry = self.geometry
+
+        self = gpd.GeoDataFrame(self, crs=crs, geometry=geometry)
+        self = self.dissolve(by=by)
+
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+        return self
+
     def apply(self, *args, **kwargs):
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
@@ -261,7 +351,7 @@ def derive_nodes(
 
         # sets index of stream branches as reach id attribute
         # if self.index.name != reach_id_attribute:
-        # self.set_index(reach_id_attribute,drop=True,inplace=True)
+        # self = self.set_index(reach_id_attribute,drop=True)
 
         # inlet_coordinates, outlet_coordinates = dict(), dict()
         node_coordinates = dict()
@@ -462,7 +552,7 @@ def remove_branches_without_catchments(
         unique_catchments = set(catchments.loc[:, reach_id_attribute_in_catchments].unique())
 
         current_index_name = self.index.name
-        self.set_index(branch_id_attribute, drop=False, inplace=True)
+        self = self.set_index(branch_id_attribute, drop=False)
 
         for usb in unique_stream_branches:
             try:
@@ -472,12 +562,12 @@ def remove_branches_without_catchments(
 
             if len(reach_ids_in_branch & unique_catchments) == 0:
                 # print(f'Dropping {usb}')
-                self.drop(usb, inplace=True)
+                self = self.drop(usb)
 
         if current_index_name is None:
-            self.reset_index(drop=True, inplace=True)
+            self = self.reset_index(drop=True)
         else:
-            self.set_index(current_index_name, drop=True, inplace=True)
+            self = self.set_index(current_index_name, drop=True)
 
         return self
 
@@ -500,8 +590,8 @@ def find_downstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
                 else:
                     # Remove reach from tmp_self
                     tmp_IDs.append(downstream_ID)
-                    tmp_self.drop(
-                        tmp_self[tmp_self.From_Node.astype(int).isin([downstream_ID])].index, inplace=True
+                    tmp_self = tmp_self.drop(
+                        tmp_self[tmp_self.From_Node.astype(int).isin([downstream_ID])].index
                     )
                     # Repeat for next lowest downstream reach
                     if downstream_ID in tmp_self.To_Node.astype(int).values:
@@ -523,14 +613,14 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
                     continue
                 else:
                     if (
-                        int(tmp_self.To_Node[tmp_self.From_Node.astype(int) == upstream_ID])
+                        int(tmp_self.To_Node[tmp_self.From_Node.astype(int) == upstream_ID].iloc[0])
                         in nonlake_reaches
                     ):
                         continue
                     # Remove reach from tmp_self
                     tmp_IDs.append(upstream_ID)
-                    tmp_self.drop(
-                        tmp_self[tmp_self.From_Node.astype(int).isin([upstream_ID])].index, inplace=True
+                    tmp_self = tmp_self.drop(
+                        tmp_self[tmp_self.From_Node.astype(int).isin([upstream_ID])].index
                     )
                     # Repeat for next highest upstream reach
                     return find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
@@ -554,7 +644,7 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
                 tmp_IDs = find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs)
 
             if len(tmp_IDs) > 0:
-                self.drop(self[self.From_Node.astype(int).isin(tmp_IDs)].index, inplace=True)
+                self = self.drop(self[self.From_Node.astype(int).isin(tmp_IDs)].index)
 
         return self
 
@@ -573,7 +663,7 @@ def remove_branches_in_waterbodies(self, waterbodies, out_vector_files=None, ver
         if isinstance(waterbodies, gpd.GeoDataFrame):
             # Find branches in waterbodies
             sjoined = gpd.sjoin(self, waterbodies, predicate="within")
-            self.drop(sjoined.index, inplace=True)
+            self = self.drop(sjoined.index)
 
             if out_vector_files is not None:
                 if verbose:
@@ -648,7 +738,7 @@ def derive_stream_branches(
         # sets index of stream branches as reach id attribute
         reset_index = False
         if self.index.name != reach_id_attribute:
-            self.set_index(reach_id_attribute, drop=True, inplace=True)
+            self = self.set_index(reach_id_attribute, drop=True)
             reset_index = True
 
         # make upstream and downstream dictionaries if none are passed
@@ -755,7 +845,7 @@ def derive_stream_branches(
         progress.close()
 
         if reset_index:
-            self.reset_index(drop=False, inplace=True)
+            self = self.reset_index(drop=False)
 
         return self
 
@@ -764,7 +854,7 @@ def make_up_and_downstream_dictionaries(
     ):
         # sets index of stream branches as reach id attribute
         # if self.index.name != reach_id_attribute:
-        #    self.set_index(reach_id_attribute,drop=True,inplace=True)
+        #    self = self.set_index(reach_id_attribute,drop=True)
 
         # find upstream and downstream dictionaries
         upstreams, downstreams = dict(), dict()
@@ -800,7 +890,7 @@ def get_arbolate_sum(
         # sets index of stream branches as reach id attribute
         reset_index = False
         if self.index.name != reach_id_attribute:
-            self.set_index(reach_id_attribute, drop=True, inplace=True)
+            self = self.set_index(reach_id_attribute, drop=True)
             reset_index = True
 
         # make upstream and downstream dictionaries if none are passed
@@ -855,7 +945,7 @@ def get_arbolate_sum(
         progress.close()
 
         if reset_index:
-            self.reset_index(drop=False, inplace=True)
+            self = self.reset_index(drop=False)
 
         return self
 
@@ -885,7 +975,7 @@ def dissolve_by_branch(
         )
 
         self = self.dissolve(by=branch_id_attribute)
-        self.rename(columns={"bids_temp": branch_id_attribute}, inplace=True)
+        self = self.rename(columns={"bids_temp": branch_id_attribute})
 
         self["order_"] = max_stream_order.values
 
@@ -948,7 +1038,7 @@ def conflate_branches(
 
         # make the crosswalk id attribute and set index
         self.loc[:, crosswalk_attribute] = [None] * len(self)
-        self.set_index(branch_id_attribute_left, inplace=True)
+        self = self.set_index(branch_id_attribute_left)
 
         # loop through rows of self
         for idx, row in tqdm(
@@ -971,13 +1061,13 @@ def conflate_branches(
             self.loc[left_branch_id, crosswalk_attribute] = right_branch_id
 
         # reset indices
-        self.reset_index(inplace=True, drop=False)
+        self = self.reset_index(drop=False)
 
         return self
 
     def explode_to_points(self, reach_id_attribute="ID", sampling_size=None, verbose=False):
         points_gdf = self.copy()
-        points_gdf.reset_index(inplace=True, drop=True)
+        points_gdf = points_gdf.reset_index(drop=True)
 
         all_exploded_points = [None] * len(points_gdf)
         for idx, row in tqdm(
@@ -1000,7 +1090,7 @@ def explode_to_points(self, reach_id_attribute="ID", sampling_size=None, verbose
         points_gdf["geometry"] = all_exploded_points
 
         points_gdf = points_gdf.explode(index_parts=True)
-        points_gdf.reset_index(inplace=True, drop=True)
+        points_gdf = points_gdf.reset_index(drop=True)
 
         return points_gdf
 
diff --git a/src/subdiv_chan_obank_src.py b/src/subdiv_chan_obank_src.py
index cad5375a8..1a8fa1435 100755
--- a/src/subdiv_chan_obank_src.py
+++ b/src/subdiv_chan_obank_src.py
@@ -81,7 +81,7 @@ def variable_mannings_calc(args):
                 + '\n'
             )
         else:
-            df_src_orig.drop(
+            df_src_orig = df_src_orig.drop(
                 [
                     'channel_n',
                     'overbank_n',
@@ -95,7 +95,6 @@ def variable_mannings_calc(args):
                     'WettedPerimeter_obank (m)',
                 ],
                 axis=1,
-                inplace=True,
                 errors='ignore',
             )  # drop these cols (in case vmann was previously performed)
 
@@ -165,7 +164,7 @@ def variable_mannings_calc(args):
             )
 
             ## drop the previously modified discharge column to be replaced with updated version
-            df_htable.drop(
+            df_htable = df_htable.drop(
                 [
                     'subdiv_applied',
                     'discharge_cms',
@@ -176,7 +175,6 @@ def variable_mannings_calc(args):
                 ],
                 axis=1,
                 errors='ignore',
-                inplace=True,
             )
             df_htable = df_htable.merge(
                 df_src_trim, how='left', left_on=['HydroID', 'stage'], right_on=['HydroID', 'stage']
@@ -250,10 +248,9 @@ def subdiv_geometry(df_src):
 
 def subdiv_mannings_eq(df_src):
     ## Calculate discharge (channel) using Manning's equation
-    df_src.drop(
+    df_src = df_src.drop(
         ['WetArea_chan (m2)', 'HydraulicRadius_chan (m)', 'Discharge_chan (m3s-1)', 'Velocity_chan (m/s)'],
         axis=1,
-        inplace=True,
         errors='ignore',
     )  # drop these cols (in case subdiv was previously performed)
     df_src['WetArea_chan (m2)'] = df_src['Volume_chan (m3)'] / df_src['LENGTHKM'] / 1000
@@ -269,7 +266,7 @@ def subdiv_mannings_eq(df_src):
     df_src['Velocity_chan (m/s)'].fillna(0, inplace=True)
 
     ## Calculate discharge (overbank) using Manning's equation
-    df_src.drop(
+    df_src = df_src.drop(
         [
             'WetArea_obank (m2)',
             'HydraulicRadius_obank (m)',
@@ -277,12 +274,11 @@ def subdiv_mannings_eq(df_src):
             'Velocity_obank (m/s)',
         ],
         axis=1,
-        inplace=True,
         errors='ignore',
     )  # drop these cols (in case subdiv was previously performed)
     df_src['WetArea_obank (m2)'] = df_src['Volume_obank (m3)'] / df_src['LENGTHKM'] / 1000
     df_src['HydraulicRadius_obank (m)'] = df_src['WetArea_obank (m2)'] / df_src['WettedPerimeter_obank (m)']
-    df_src.replace([np.inf, -np.inf], np.nan, inplace=True)  # need to replace inf instances (divide by 0)
+    df_src = df_src.replace([np.inf, -np.inf], np.nan)  # need to replace inf instances (divide by 0)
     df_src['HydraulicRadius_obank (m)'].fillna(0, inplace=True)
     df_src['Discharge_obank (m3s-1)'] = (
         df_src['WetArea_obank (m2)']
@@ -294,8 +290,8 @@ def subdiv_mannings_eq(df_src):
     df_src['Velocity_obank (m/s)'].fillna(0, inplace=True)
 
     ## Calcuate the total of the subdivided discharge (channel + overbank)
-    df_src.drop(
-        ['Discharge (m3s-1)_subdiv'], axis=1, inplace=True, errors='ignore'
+    df_src = df_src.drop(
+        ['Discharge (m3s-1)_subdiv'], axis=1, errors='ignore'
     )  # drop these cols (in case subdiv was previously performed)
     df_src['Discharge (m3s-1)_subdiv'] = df_src['Discharge_chan (m3s-1)'] + df_src['Discharge_obank (m3s-1)']
     df_src.loc[df_src['Stage'] == 0, ['Discharge (m3s-1)_subdiv']] = 0
diff --git a/src/subset_catch_list_by_branch_id.py b/src/subset_catch_list_by_branch_id.py
index 871ad3080..bf80a9da9 100755
--- a/src/subset_catch_list_by_branch_id.py
+++ b/src/subset_catch_list_by_branch_id.py
@@ -17,7 +17,7 @@ def Subset_catch_list(
 
     # loading files
     catch_list = pd.read_csv(catch_list, sep=" ", header=None, skiprows=1)
-    catch_list.rename(columns={0: "HydroID", 1: "slopes", 2: "lengthKM", 3: "areasqkm"}, inplace=True)
+    catch_list = catch_list.rename(columns={0: "HydroID", 1: "slopes", 2: "lengthKM", 3: "areasqkm"})
     stream_network = StreamNetwork.from_file(stream_network, branch_id_attribute=branch_id_attribute)
     stream_network = StreamNetwork(
         stream_network.astype({'HydroID': int}), branch_id_attribute=branch_id_attribute
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 3065008b3..b3ec127c9 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -41,7 +41,7 @@ def load_gages(self):
 
         # Convert ras locs crs to match usgs gage crs
         ras_locs.to_crs(usgs_gages.crs, inplace=True)
-        ras_locs.rename(columns={'huc8': 'HUC8'}, inplace=True)
+        ras_locs = ras_locs.rename(columns={'huc8': 'HUC8'})
 
         # Convert Multipoint geometry to Point geometry
         ras_locs['geometry'] = ras_locs.representative_point()
@@ -51,7 +51,7 @@ def load_gages(self):
         #     ras_locs['HUC8'] = str(self.huc8)
         #     ras_locs = ras_locs.drop('huc8', axis=1)
         # elif ras_locs.huc8.dtype == 'int64':
-        #     ras_locs.rename(columns={'huc8':'HUC8'}, inplace=True)
+        #     ras_locs = ras_locs.rename(columns={'huc8':'HUC8'})
 
         # Concat USGS points and RAS2FIM points
         gages_locs = pd.concat([usgs_gages, ras_locs], axis=0, ignore_index=True)
@@ -64,8 +64,8 @@ def load_gages(self):
         if self.ahps_filename:
             ahps_sites = gpd.read_file(self.ahps_filename)
             ahps_sites = ahps_sites[ahps_sites.HUC8 == self.huc8]  # filter to HUC8
-            ahps_sites.rename(
-                columns={'nwm_feature_id': 'feature_id', 'usgs_site_code': 'location_id'}, inplace=True
+            ahps_sites = ahps_sites.rename(
+                columns={'nwm_feature_id': 'feature_id', 'usgs_site_code': 'location_id'}
             )
             ahps_sites = ahps_sites[
                 ahps_sites.location_id.isna()
@@ -86,7 +86,7 @@ def load_gages(self):
 
     def sort_into_branch(self, nwm_subset_streams_levelPaths):
         nwm_reaches = gpd.read_file(nwm_subset_streams_levelPaths)
-        nwm_reaches.rename(columns={'ID': 'feature_id'}, inplace=True)
+        nwm_reaches = nwm_reaches.rename(columns={'ID': 'feature_id'})
 
         if not self.gages[self.gages.feature_id.isnull()].empty:
             missing_feature_id = self.gages.loc[self.gages.feature_id.isnull()].copy()
@@ -132,14 +132,13 @@ def filter_gage_branches(fim_inputs_filename):
         for huc_dir in [d for d in os.listdir(fim_dir) if re.search(r'^\d{8}$', d)]:
             gage_file = os.path.join(fim_dir, huc_dir, 'usgs_subset_gages.gpkg')
             if not os.path.isfile(gage_file):
-                fim_inputs.drop(fim_inputs.loc[fim_inputs.huc == huc_dir].index, inplace=True)
+                fim_inputs = fim_inputs.drop(fim_inputs.loc[fim_inputs.huc == huc_dir].index)
                 continue
 
             gages = gpd.read_file(gage_file)
             level_paths = gages.levpa_id
-            fim_inputs.drop(
-                fim_inputs.loc[(fim_inputs.huc == huc_dir) & (~fim_inputs.levpa_id.isin(level_paths))].index,
-                inplace=True,
+            fim_inputs = fim_inputs.drop(
+                fim_inputs.loc[(fim_inputs.huc == huc_dir) & (~fim_inputs.levpa_id.isin(level_paths))].index
             )
 
         fim_inputs.to_csv(fim_inputs_filename, index=False, header=False)
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 7f0eacc37..f0b7dfaf5 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -3,8 +3,6 @@
 import glob
 import inspect
 import os
-import re
-import sys
 from concurrent.futures import as_completed
 from datetime import datetime, timezone
 from os.path import splitext
@@ -16,8 +14,6 @@
 import pandas as pd
 import rasterio
 from memory_profiler import profile
-from pyproj.crs import CRS
-from rasterio.warp import Resampling, calculate_default_transform, reproject
 from tqdm import tqdm
 
 import utils.shared_variables as sv
@@ -97,7 +93,7 @@ def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
                         keep_flag = True
                         break
             if not keep_flag:
-                gdf.drop(index, inplace=True)  # Delete from dataframe.
+                gdf = gdf.drop(index)  # Delete from dataframe.
 
     # Overwrite geopackage.
     layer_name = os.path.split(wbd_gpkg)[1].strip('.gpkg')
diff --git a/tools/adjust_rc_with_feedback.py b/tools/adjust_rc_with_feedback.py
index 429c6da97..dfd16229e 100644
--- a/tools/adjust_rc_with_feedback.py
+++ b/tools/adjust_rc_with_feedback.py
@@ -39,8 +39,8 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
                 'LakeID',
             ]
         ]
-        df_htable.rename(
-            columns={'orig_discharge_cms': 'discharge_cms', 'default_ManningN': 'ManningN'}, inplace=True
+        df_htable = df_htable.rename(
+            columns={'orig_discharge_cms': 'discharge_cms', 'default_ManningN': 'ManningN'}
         )
     else:
         df_htable = df_htable[
@@ -79,8 +79,8 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
     # print(df_hydro_feat.to_string())
 
     ## Calculate roughness using Manning's equation
-    df_gmed.rename(
-        columns={'ManningN': 'ManningN_default', 'hydroid': 'HydroID'}, inplace=True
+    df_gmed = df_gmed.rename(
+        columns={'ManningN': 'ManningN_default', 'hydroid': 'HydroID'}
     )  # rename the previous ManningN column
     df_gmed['hydroid_ManningN'] = (
         df_gmed['WetArea_m2']
@@ -116,7 +116,7 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
 
     # Create df with the most recent collection time entry
     df_updated = df_gmed.groupby(["HydroID"])[['coll_time']].max()
-    df_updated.rename(columns={'coll_time': 'last_updated'}, inplace=True)
+    df_updated = df_updated.rename(columns={'coll_time': 'last_updated'})
 
     # cacluate median ManningN to handle cases with multiple hydroid entries
     df_mann = df_gmed.groupby(["HydroID"])[['hydroid_ManningN']].median()
@@ -125,11 +125,11 @@ def update_rating_curve(fim_directory, output_csv, htable_path, output_src_json_
 
     # Create a df with the median hydroid_ManningN value per feature_id
     df_mann_featid = df_gmed.groupby(["feature_id"])[['hydroid_ManningN']].median()
-    df_mann_featid.rename(columns={'hydroid_ManningN': 'featid_ManningN'}, inplace=True)
+    df_mann_featid = df_mann_featid.rename(columns={'hydroid_ManningN': 'featid_ManningN'})
 
     # Rename the original hydrotable variables to allow new calculations to use the primary var name
-    df_htable.rename(
-        columns={'ManningN': 'default_ManningN', 'discharge_cms': 'orig_discharge_cms'}, inplace=True
+    df_htable = df_htable.rename(
+        columns={'ManningN': 'default_ManningN', 'discharge_cms': 'orig_discharge_cms'}
     )
 
     # Check for large variabilty in the calculated Manning's N values
diff --git a/tools/aggregate_csv_files.py b/tools/aggregate_csv_files.py
index 0efeafe59..9373ab4d4 100644
--- a/tools/aggregate_csv_files.py
+++ b/tools/aggregate_csv_files.py
@@ -37,7 +37,7 @@ def read_csvs_to_df(files_to_merge, head_row):
         #     header=0,
         # )
         # df = df[df['adjust_ManningN'].notna()]
-        # df.drop_duplicates(subset=['HydroID'], inplace=True)
+        # df = df.drop_duplicates(subset=['HydroID'], inplace=True)
         df = pd.read_csv(file_in, index_col=None, header=head_row)  # dtype={'feature_id': 'int64'}
         df = df[df['Unnamed: 0'] != 'HydroID']
         li.append(df)
diff --git a/tools/combine_crosswalk_tables.py b/tools/combine_crosswalk_tables.py
index 6251966c5..9acc2292e 100644
--- a/tools/combine_crosswalk_tables.py
+++ b/tools/combine_crosswalk_tables.py
@@ -21,8 +21,8 @@ def combine_crosswalk_tables(data_directory, output_filename):
             file_df = pd.read_csv(
                 filename, usecols=['HUC', 'HydroID', 'feature_id', 'LakeID'], dtype={'HUC': str}
             )
-            file_df.drop_duplicates(inplace=True)
-            file_df.rename(columns={'HUC': 'huc8'}, inplace=True)
+            file_df = file_df.drop_duplicates()
+            file_df = file_df.rename(columns={'HUC': 'huc8'})
             file_df['BranchID'] = os.path.split(os.path.dirname(filename))[1]
 
             dfs.append(file_df)
diff --git a/tools/inundation.py b/tools/inundation.py
index 0babdb6c9..b39d9d191 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -575,7 +575,7 @@ def __subset_hydroTable_to_forecast(hydroTable, forecast, subset_hucs=None):
             usecols=htable_req_cols,
         )
         # huc_error = hydroTable.HUC.unique()
-        hydroTable.set_index(['HUC', 'feature_id', 'HydroID'], inplace=True)
+        hydroTable = hydroTable.set_index(['HUC', 'feature_id', 'HydroID'])
 
     elif isinstance(hydroTable, pd.DataFrame):
         pass  # consider checking for correct dtypes, indices, and columns
@@ -593,7 +593,7 @@ def __subset_hydroTable_to_forecast(hydroTable, forecast, subset_hucs=None):
     if isinstance(forecast, str):
         try:
             forecast = pd.read_csv(forecast, dtype={'feature_id': str, 'discharge': float})
-            forecast.set_index('feature_id', inplace=True)
+            forecast = forecast.set_index('feature_id')
         except UnicodeDecodeError:
             forecast = read_nwm_forecast_file(forecast)
 
@@ -619,7 +619,7 @@ def __subset_hydroTable_to_forecast(hydroTable, forecast, subset_hucs=None):
     if not hydroTable.empty:
         if isinstance(forecast, str):
             forecast = pd.read_csv(forecast, dtype={'feature_id': str, 'discharge': float})
-            forecast.set_index('feature_id', inplace=True)
+            forecast = forecast.set_index('feature_id')
         elif isinstance(forecast, pd.DataFrame):
             pass  # consider checking for dtypes, indices, and columns
         else:
@@ -687,7 +687,7 @@ def read_nwm_forecast_file(forecast_file, rename_headers=True):
     flows_nc = xr.open_dataset(forecast_file, decode_cf='feature_id', engine='netcdf4')
 
     flows_df = flows_nc.to_dataframe()
-    flows_df.reset_index(inplace=True)
+    flows_df = flows_df.reset_index()
 
     flows_df = flows_df[['streamflow', 'feature_id']]
 
@@ -697,9 +697,9 @@ def read_nwm_forecast_file(forecast_file, rename_headers=True):
     convert_dict = {'feature_id': str, 'discharge': float}
     flows_df = flows_df.astype(convert_dict)
 
-    flows_df.set_index('feature_id', inplace=True, drop=True)
+    flows_df = flows_df.set_index('feature_id', drop=True)
 
-    flows_df.dropna(inplace=True)
+    flows_df = flows_df.dropna()
 
     return flows_df
 
@@ -711,7 +711,7 @@ def __vprint(message, verbose):
 
 def create_src_subset_csv(hydro_table, catchmentStagesDict, src_table):
     src_df = pd.DataFrame.from_dict(catchmentStagesDict, orient='index')
-    src_df.reset_index(inplace=True)
+    src_df = src_df.reset_index()
     src_df.columns = ['HydroID', 'stage_inund']
     htable_req_cols = ['HUC', 'feature_id', 'HydroID', 'stage', 'discharge_cms', 'LakeID']
     df_htable = pd.read_csv(
diff --git a/tools/make_boxes_from_bounds.py b/tools/make_boxes_from_bounds.py
index 4959a21e4..2810cac37 100755
--- a/tools/make_boxes_from_bounds.py
+++ b/tools/make_boxes_from_bounds.py
@@ -43,15 +43,15 @@ def find_hucs_of_bounding_boxes(
         lambda bbdf: gpd.read_file(wbd, layer=wbd_layer, mask=bbdf.geometry)[wbdcol_name], axis=1
     )
 
-    bounding_boxes.drop(columns=['geometry', 'minx', 'miny', 'maxx', 'maxy'], inplace=True)
+    bounding_boxes = bounding_boxes.drop(columns=['geometry', 'minx', 'miny', 'maxx', 'maxy'])
 
     hucs_columns = hucs.columns
     bb_columns = bounding_boxes.columns
     bounding_boxes = hucs.join(bounding_boxes)
     bounding_boxes = pd.melt(bounding_boxes, id_vars=bb_columns, value_vars=hucs_columns, value_name='HUC8')
-    bounding_boxes.drop(columns=['variable'], inplace=True)
-    bounding_boxes.dropna(inplace=True)
-    bounding_boxes.reset_index(drop=True, inplace=True)
+    bounding_boxes = bounding_boxes.drop(columns=['variable'])
+    bounding_boxes = bounding_boxes.dropna()
+    bounding_boxes = bounding_boxes.reset_index(drop=True)
 
     hucs_series = pd.Series(hucs.stack().reset_index(drop=True).unique())
 
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 601b389e4..6229b9907 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -43,7 +43,7 @@ def Mosaic_inundation(
         raise TypeError("Pass Pandas Dataframe or file path string to csv for map_file argument")
 
     # remove NaNs
-    inundation_maps_df.dropna(axis=0, how="all", inplace=True)
+    inundation_maps_df = inundation_maps_df.dropna(axis=0, how="all")
 
     # subset
     if subset is not None:
@@ -53,7 +53,7 @@ def Mosaic_inundation(
     # unique aggregation units
     aggregation_units = inundation_maps_df.loc[:, unit_attribute_name].unique()
 
-    inundation_maps_df.set_index(unit_attribute_name, drop=True, inplace=True)
+    inundation_maps_df = inundation_maps_df.set_index(unit_attribute_name, drop=True)
 
     # decide upon whether to display
     if verbose & len(aggregation_units) == 1:
diff --git a/tools/plots.py b/tools/plots.py
index 86b7ee6b4..67aea2415 100755
--- a/tools/plots.py
+++ b/tools/plots.py
@@ -153,8 +153,8 @@ def set_axes(facetgrid):
 
 
 def preparing_data_for_plotting(metrics_table):
-    metrics_table.rename(
-        columns={'extent_config': 'Model', 'magnitude': 'Magnitude', 'TPR': 'POD'}, inplace=True
+    metrics_table = metrics_table.rename(
+        columns={'extent_config': 'Model', 'magnitude': 'Magnitude', 'TPR': 'POD'}
     )
 
     # set_mannings = lambda df: 0.06 if 'n_6' in df['version'] else 0.12
@@ -189,8 +189,8 @@ def robust_linear_model(metrics_table):
     # metric_indices = {m: metrics_table.loc[:, 'Metric'] == m for m in metrics}
     # mannings_indices = {m: metrics_table.loc[:, 'Mannings N'] == m for m in mannings}
 
-    metrics_table.set_index(['Metric', 'Mannings N'], inplace=True, drop=False)
-    metrics_table.sort_index(inplace=True)
+    metrics_table = metrics_table.set_index(['Metric', 'Mannings N'], drop=False)
+    metrics_table = metrics_table.sort_index()
     for met, man in product(metrics, mannings):
         y = metrics_table.loc[(met, man), 'Metric Value'].to_numpy()
 
@@ -207,14 +207,14 @@ def robust_linear_model(metrics_table):
         metrics_table.loc[(met, man), 'beta1'] = beta1
         metrics_table.loc[(met, man), 'beta1_pvalue'] = pval_beta1
 
-    metrics_table.reset_index(drop=True, inplace=True)
+    metrics_table = metrics_table.reset_index(drop=True)
 
     return metrics_table
 
 
 def annotate_rlm(facetgrid, metrics_table):
-    metrics_table.set_index(['Metric', 'Mannings N'], inplace=True, drop=False)
-    metrics_table.sort_index(inplace=True)
+    metrics_table = metrics_table.set_index(['Metric', 'Mannings N'], drop=False)
+    metrics_table = metrics_table.sort_index()
 
     metric_index_dict = {'CSI': 0, 'POD': 1, 'FAR': 2}
     mannings_index_dict = {0.06: 0, 0.12: 1}
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 2364e48f7..2df1c179b 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -96,7 +96,7 @@ def generate_rating_curve_metrics(args):
     )
 
     # Filter out null and non-integer location_id entries (the crosswalk steps tries to fill AHPS only sites with the nws_lid)
-    elev_table.dropna(subset=['location_id'], inplace=True)
+    elev_table = elev_table.dropna(subset=['location_id'])
     elev_table = elev_table[elev_table['location_id'].apply(lambda x: str(x).isdigit())]
 
     # Read in the USGS gages rating curve database csv
@@ -122,9 +122,9 @@ def generate_rating_curve_metrics(args):
             branch_hydrotable = branch_hydrotable.loc[
                 branch_hydrotable.HydroID.isin(branch_elev_table.HydroID)
             ]
-            branch_hydrotable.drop(columns=['order_'], inplace=True)
+            branch_hydrotable = branch_hydrotable.drop(columns=['order_'])
             # Join SRC with elevation data
-            branch_elev_table.rename(columns={'feature_id': 'fim_feature_id'}, inplace=True)
+            branch_elev_table = branch_elev_table.rename(columns={'feature_id': 'fim_feature_id'})
             branch_hydrotable = branch_hydrotable.merge(branch_elev_table, on="HydroID")
             # Append to full rating curve dataframe
             if hydrotable.empty:
@@ -133,7 +133,7 @@ def generate_rating_curve_metrics(args):
                 hydrotable = pd.concat([hydrotable, branch_hydrotable])
 
         # Join rating curves with elevation data
-        # elev_table.rename(columns={'feature_id':'fim_feature_id'}, inplace=True)
+        # elev_table = elev_table.rename(columns={'feature_id':'fim_feature_id'})
         # hydrotable = hydrotable.merge(elev_table, on="HydroID")
         if 'location_id' in hydrotable.columns:
             relevant_gages = list(hydrotable.location_id.unique())
@@ -230,7 +230,7 @@ def generate_rating_curve_metrics(args):
 
             # Identify unique gages
             usgs_crosswalk = hydrotable.filter(items=['location_id', 'feature_id']).drop_duplicates()
-            usgs_crosswalk.dropna(subset=['location_id'], inplace=True)
+            usgs_crosswalk = usgs_crosswalk.dropna(subset=['location_id'])
 
             nwm_recurr_data_table = pd.DataFrame()
             # usgs_recurr_data = pd.DataFrame()
@@ -1131,7 +1131,7 @@ def calculate_rc_diff(rc):
     # Calculate water surface elevation difference at recurrence intervals
     rc_unmelt["yhat_minus_y"] = rc_unmelt[src_elev] - rc_unmelt[usgs_elev]
     # Remove duplicate location_id-recurr_interval pairs and pivot
-    rc_unmelt.set_index(['location_id', 'recurr_interval'], inplace=True, verify_integrity=False)
+    rc_unmelt = rc_unmelt.set_index(['location_id', 'recurr_interval'], verify_integrity=False)
     rc_unmelt = (
         rc_unmelt[~rc_unmelt.index.duplicated(keep='first')]
         .reset_index()
diff --git a/tools/vary_mannings_n_composite.py b/tools/vary_mannings_n_composite.py
index 1a69ab28e..bdc2151a9 100755
--- a/tools/vary_mannings_n_composite.py
+++ b/tools/vary_mannings_n_composite.py
@@ -3,14 +3,10 @@
 import datetime as dt
 import os
 import re
-import shutil
-import sys
 import traceback
 import warnings
-from functools import reduce
 from multiprocessing import Pool
-from os.path import dirname, isdir, isfile, join
-from pathlib import Path
+from os.path import isdir, isfile, join
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -83,10 +79,9 @@ def variable_mannings_calc(args):
     else:
         try:
             if 'comp_ManningN' in df_src.columns:
-                df_src.drop(
+                df_src = df_src.drop(
                     ['channel_n', 'overbank_n', 'comp_ManningN', 'vmann_on', 'Discharge (m3s-1)_varMann'],
                     axis=1,
-                    inplace=True,
                 )  # drop these cols (in case vmann was previously performed)
 
             ## Merge (crosswalk) the df of Manning's n with the SRC df
@@ -135,7 +130,7 @@ def variable_mannings_calc(args):
 
             ## Calculate Q using Manning's equation
             # Uncomment below to rename the previous Discharge column
-            # df_src.rename(columns={'Discharge (m3s-1)'}, inplace=True)
+            # df_src = df_src.rename(columns={'Discharge (m3s-1)'})
             df_src['Discharge (m3s-1)_varMann'] = (
                 df_src[wet_area]
                 * pow(df_src[hydr_radius], 2.0 / 3)
@@ -185,15 +180,14 @@ def variable_mannings_calc(args):
 
             ## Check if BARC ran
             # if not set(['orig_discharge_cms']).issubset(df_htable.columns):
-            #     df_htable.rename(columns={'discharge_cms':'orig_discharge_cms'},inplace=True)
-            #     df_htable.rename(columns={'ManningN':'orig_ManningN'},inplace=True)
+            #     df_htable = df_htable.rename(columns={'discharge_cms':'orig_discharge_cms'})
+            #     df_htable = df_htable.rename(columns={'ManningN':'orig_ManningN'})
             # else:
 
             ## drop the previously modified discharge column to be replaced with updated version
-            df_htable.drop(
+            df_htable = df_htable.drop(
                 ['vmann_on', 'discharge_cms', 'ManningN', 'vmann_discharge_cms', 'vmann_ManningN'],
                 axis=1,
-                inplace=True,
             )
             df_htable = df_htable.merge(
                 df_src_trim, how='left', left_on=['HydroID', 'stage'], right_on=['HydroID', 'stage']

From bd793002c08486fcad26bfe5b0774ea35f956d7f Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 20 Sep 2023 18:34:06 -0400
Subject: [PATCH 277/359] v4.4.2.1 Bug fix: set geometry after
 `GeoDataFrame.update()` (#990)

---
 CITATION.cff                |  2 +-
 docs/CHANGELOG.md           | 10 ++++++++++
 src/usgs_gage_unit_setup.py |  1 +
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/CITATION.cff b/CITATION.cff
index 0c47bf8d3..cfa1f10b7 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.4.1.1
+version: 4.4.2.1
 date-released: 2023
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index eebbb5703..6075991de 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.2.1 - 2023-09-20 - [PR#990](https://github.com/NOAA-OWP/inundation-mapping/pull/990)
+
+Corrects a bug in `src/usgs_gage_unit_setup.py` caused by missing geometry field after `GeoDataFrame.update()`.
+
+### Changes
+
+- `src/usgs_gage_unit_setup.py`: Sets geometry field in `self.gages`.
+
+<br/><br/>
+
 ## v4.4.2.0 - 2023-09-20 - [PR#993](https://github.com/NOAA-OWP/inundation-mapping/pull/993)
 
 Resolves the causes of two warnings in pandas and geopandas: (1) `FutureWarning` from taking the `int()` of single-length Series and (2) `SettingWithCopyWarning` resulting from the use of `inplace=True`.
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index b3ec127c9..246803a02 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -96,6 +96,7 @@ def sort_into_branch(self, nwm_subset_streams_levelPaths):
             )
 
             self.gages.update(missing_feature_id)
+            self.gages = self.gages.set_geometry("geometry")
 
             del nwm_reaches_union
 

From 6ab186d4ec13f58692521e9ff15859502fd01754 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 21 Sep 2023 09:32:53 -0600
Subject: [PATCH 278/359] Update README.md (#996)

---
 README.md | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index fb6872d86..85c071822 100644
--- a/README.md
+++ b/README.md
@@ -25,21 +25,20 @@ AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
 
 ### Accessing Data using the AWS CLI
 
-This S3 Bucket (`s3://noaa-nws-owp-fim`) is set to avoid having AWS credentials loaded. Read more about the use of the `--no-sign-request` [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html#:~:text=%2D%2Dno%2Dsign%2Drequest,prevents%20credentials%20from%20being%20loaded.). There should be no cost associated with using this S3 Bucket.
-
+This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If you are using compute resources in the same region as the S3 Bucket, then there is no cost.
 ### Examples
 
 **Note:** All examples are based on linux pathing. Also, for each sample below, remove the line breaks [backslash(s) "\"] before running the command.
 
 The available inputs, test cases, and versioned FIM outputs can be found by running:
 ```
-aws s3 ls s3://noaa-nws-owp-fim/hand_fim/ --no-sign-request
+aws s3 ls s3://noaa-nws-owp-fim/hand_fim/ --request-payer
 ```
 
 Download a directory of sample outputs for a single HUC8:
 ```
-aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_3_11_0/12090301 \
-    /your_local_folder_name/12090301 --no-sign-request
+aws s3 sync s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_3_11_0/12090301 \
+    /your_local_folder_name/12090301 --request-payer
 ```
 By adjusting pathing, you can also download entire directories such as the `fim_4_3_11_0` folder. An entire output FIM set (e.g. `fim_4_3_11_0`) is approximately 1.1 TB.
 
@@ -84,7 +83,7 @@ Git will auto create a subfolder named `inundation-mapping` where the code will
 Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP S3 Bucket" section above). The FIM inputs directory can be found at `s3://noaa-nws-owp-fim/hand_fim/inputs`. It is appx 400GB and it needs to be in your `data` folder.
 
 ```
-aws s3 cp --recursive s3://noaa-nws-owp-fim/hand_fim/inputs /home/projects/fim/data/inputs --no-sign-request --dryrun
+aws s3 sync s3://noaa-nws-owp-fim/hand_fim/inputs /home/projects/fim/data/inputs --request-payer --dryrun
 ```
 **Note**: When you include the `--dryrun` argument in the command, a large list will be returned showing you exactly which files are to be downloaded and where they will be saved. We recommend including this argument the first time you run the command, then quickly aborting it (CTRL-C) so you don't get the full list. However, you can see that your chosen target path on your machine is correct.  When you are happy with the pathing, run the `aws s3` command again and leave off the `--dryrun` argument.
 

From 99a121013b903db3a7c8a0da387ab071fd8f1237 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Thu, 21 Sep 2023 17:20:11 -0400
Subject: [PATCH 279/359] v4.4.2.2 Bug fix for reindexing error (#997)

---
 docs/CHANGELOG.md      | 10 ++++++++++
 src/stream_branches.py |  3 ---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 6075991de..043e86924 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.2.2 - 2023-09-21 - [PR#997](https://github.com/NOAA-OWP/inundation-mapping/pull/997)
+
+Bug fix for an error related to reindexing in `StreamNetwork.drop()`.
+
+### Changes
+
+- `src/stream_branches.py`: Fixes reindexing error.
+
+<br/><br/>
+
 ## v4.4.2.1 - 2023-09-20 - [PR#990](https://github.com/NOAA-OWP/inundation-mapping/pull/990)
 
 Corrects a bug in `src/usgs_gage_unit_setup.py` caused by missing geometry field after `GeoDataFrame.update()`.
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 3491f40be..fada36a6d 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -174,12 +174,9 @@ def drop(self, labels=None, axis=0):
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
         values_excluded = self.values_excluded
-        crs = self.crs
-        geometry = self.geometry
 
         self = super(gpd.GeoDataFrame, self)
         self = self.drop(labels=labels, axis=axis)
-        self = gpd.GeoDataFrame(self, crs=crs, geometry=geometry)
 
         self = StreamNetwork(
             self,

From 90ed7dded53f9c818509ae031d9d8259d527135f Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Thu, 21 Sep 2023 15:22:20 -0600
Subject: [PATCH 280/359] v4.4.2.3 Remove exclude list for black, re-run black
 on repository (#998)

---
 .pre-commit-config.yaml            |  9 +++------
 docs/CHANGELOG.md                  | 18 ++++++++++++++++++
 pyproject.toml                     |  8 +-------
 src/add_crosswalk.py               |  7 ++++---
 src/bathy_src_adjust_topwidth.py   |  2 +-
 src/bathymetric_adjustment.py      |  2 +-
 src/identify_src_bankfull.py       | 14 ++++++--------
 src/src_roughness_optimization.py  | 12 +++++-------
 tools/vary_mannings_n_composite.py |  3 +--
 9 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5d4df4c20..2aebea2c3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -12,23 +12,20 @@ repos:
         -   id: check-added-large-files
             args: ['--maxkb=5000']
         -   id: check-json
+
 -   repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
         -   id: flake8
             entry: pflake8
             additional_dependencies: [pyproject-flake8]
+
 -   repo: https://github.com/psf/black
     rev: 23.7.0
     hooks:
         -   id: black
             args: ['--line-length=110', '--skip-magic-trailing-comma', '--skip-string-normalization']
-            exclude: |
-                (?x)^(
-                    src/add_crosswalk.py|
-                    src/bathymetric_adjustment.py|
-                    src/identify_src_bankfull.py
-                )$
+
 -   repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 043e86924..dfff11dd7 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.2.3 - 2023-09-21 - [PR#998](https://github.com/NOAA-OWP/inundation-mapping/pull/998)
+
+Removes exclude list for black formatter in `.pre-commit-config.yaml` as well as in `pyproject.toml`. Ran the `black` executable on the 
+whole repository, the re-formatted files in `src/` & `tools/` are included.
+
+### Changes
+
+- `.pre-commit-config.yaml`
+- `pyproject.toml`
+- `src/add_crosswalk.py`
+- `src/bathy_src_adjust_topwidth.py`
+- `src/bathymetric_adjustment.py`
+- `src/identify_src_bankfull.py`
+- `src/src_roughness_optimization.py`
+- `tools/vary_mannings_n_composite.py`
+
+<br/><br/>
+
 ## v4.4.2.2 - 2023-09-21 - [PR#997](https://github.com/NOAA-OWP/inundation-mapping/pull/997)
 
 Bug fix for an error related to reindexing in `StreamNetwork.drop()`.
diff --git a/pyproject.toml b/pyproject.toml
index af8b9d9e3..f6d623ec0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,13 +46,7 @@ Wiki = "https://github.com/NOAA-OWP/inundation-mapping/wiki"
 line_length = 110
 skip-string-normalization = true
 skip-magic-trailing-comma = true
-force-exclude = '''
-    \.git
-    | src/add_crosswalk.py
-    | src/bathymetric_adjustment.py
-    | src/identify_src_bankfull.py
-    |
-'''
+
 
 [tool.isort]
 profile = 'black'
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 283f0ff30..3f0a5f5c0 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -53,7 +53,7 @@ def add_crosswalk(
             input_catchments, input_nwmcatras_fileName, stats=['majority'], geojson_out=True
         )
         input_majorities = gpd.GeoDataFrame.from_features(majority_calc)
-        input_majorities = input_majorities.rename(columns={'majority' : 'feature_id'})
+        input_majorities = input_majorities.rename(columns={'majority': 'feature_id'})
 
         input_majorities = input_majorities[:][input_majorities['feature_id'].notna()]
         if input_majorities.feature_id.dtype != 'int':
@@ -321,7 +321,8 @@ def add_crosswalk(
             for src_index, src_stage in new_values.iterrows():
                 output_src.loc[
                     (output_src['HydroID'] == short_id) & (output_src['Stage'] == src_stage[0]),
-                    ['Discharge (m3s-1)']] = src_stage[1]
+                    ['Discharge (m3s-1)'],
+                ] = src_stage[1]
 
     if extent == 'FR':
         output_src = output_src.merge(input_majorities[['HydroID', 'feature_id']], on='HydroID')
@@ -352,7 +353,7 @@ def add_crosswalk(
             'SLOPE',
             'ManningN',
             'Stage',
-            'Discharge (m3s-1)'
+            'Discharge (m3s-1)',
         ],
     ]
     output_hydro_table.rename(columns={'Stage': 'stage', 'Discharge (m3s-1)': 'discharge_cms'}, inplace=True)
diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py
index 72e86441f..4ce6ccf9c 100644
--- a/src/bathy_src_adjust_topwidth.py
+++ b/src/bathy_src_adjust_topwidth.py
@@ -430,7 +430,7 @@ def bathy_rc_lookup(args):
                     'Volume (m3)': 'orig_Volume (m3)',
                     'WetArea (m2)': 'orig_WetArea (m2)',
                     'HydraulicRadius (m)': 'orig_HydraulicRadius (m)',
-                },
+                }
             )
         else:
             df_htable = df_htable.drop(
diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py
index b0421bc4f..a102d0a80 100644
--- a/src/bathymetric_adjustment.py
+++ b/src/bathymetric_adjustment.py
@@ -74,7 +74,7 @@ def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
                 ],
                 on='feature_id',
                 how='left',
-                validate='many_to_one'
+                validate='many_to_one',
             )
         # If there's more than one feature_id in the bathy data, just take the mean
         except pd.errors.MergeError:
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index 8b99e295d..5fe955ea4 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -129,9 +129,7 @@ def src_bankfull_lookup(args):
         # find the index of the Q_bfull_find (closest matching flow)
         df_bankfull_calc = df_bankfull_calc.loc[
             df_bankfull_calc.groupby('HydroID')['Q_bfull_find'].idxmin()
-        ].reset_index(
-            drop=True
-        )
+        ].reset_index(drop=True)
         # rename volume to use later for channel portion calc
         df_bankfull_calc = df_bankfull_calc.rename(
             columns={
@@ -139,7 +137,7 @@ def src_bankfull_lookup(args):
                 bedarea_var: 'BedArea_bankfull',
                 volume_var: 'Volume_bankfull',
                 hradius_var: 'HRadius_bankfull',
-                surface_area_var: 'SurfArea_bankfull'
+                surface_area_var: 'SurfArea_bankfull',
             }
         )
         df_src = df_src.merge(
@@ -150,11 +148,11 @@ def src_bankfull_lookup(args):
                     'BedArea_bankfull',
                     'Volume_bankfull',
                     'HRadius_bankfull',
-                    'SurfArea_bankfull'
+                    'SurfArea_bankfull',
                 ]
             ],
             how='left',
-            on='HydroID'
+            on='HydroID',
         )
         df_src = df_src.drop(['Q_bfull_find'], axis=1)
 
@@ -332,7 +330,7 @@ def run_prep(fim_dir, bankfull_flow_filepath, number_of_jobs, verbose, src_plot_
         'HydraulicRadius (m)',
         'Discharge (m3s-1)',
         'feature_id',
-        'Bathymetry_source'
+        'Bathymetry_source',
     ]
 
     df_bflows = pd.read_csv(bankfull_flow_filepath, dtype={'feature_id': int})
@@ -359,7 +357,7 @@ def run_prep(fim_dir, bankfull_flow_filepath, number_of_jobs, verbose, src_plot_
                             huc,
                             branch_id,
                             src_plot_option,
-                            huc_output_dir
+                            huc_output_dir,
                         ]
                     )
                 else:
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index e417753f6..fd8d502aa 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -388,9 +388,7 @@ def update_rating_curve(
                 df_nmerge[calb_type] = np.select(conditions, choices, default=df_nmerge['hydroid_calb_coef'])
                 df_nmerge['obs_source'] = np.where(df_nmerge[calb_type].notnull(), source_tag, pd.NA)
                 df_nmerge = df_nmerge.drop(
-                    ['feature_id', 'NextDownID', 'LENGTHKM', 'LakeID', 'order_'],
-                    axis=1,
-                    errors='ignore',
+                    ['feature_id', 'NextDownID', 'LENGTHKM', 'LakeID', 'order_'], axis=1, errors='ignore'
                 )  # drop these columns to avoid duplicates where merging with the full hydroTable df
 
                 ## Merge in previous SRC adjustments (where available) for hydroIDs that do not have a new
@@ -433,9 +431,7 @@ def update_rating_curve(
                         'src_calibrated' in input_catchments.columns
                     ):  # check if this attribute already exists and drop if needed
                         input_catchments = input_catchments.drop(
-                            ['src_calibrated', 'obs_source', 'calb_coef_final'],
-                            axis=1,
-                            errors='ignore',
+                            ['src_calibrated', 'obs_source', 'calb_coef_final'], axis=1, errors='ignore'
                         )
                     df_nmerge['src_calibrated'] = np.where(
                         df_nmerge['calb_coef_final'].notnull(), 'True', 'False'
@@ -615,7 +611,9 @@ def branch_network_tracer(df_input_htable):
                         # starting hydroid
                         continue
                     Q.append(nextid)
-    df_input_htable = df_input_htable.reset_index(drop=True)  # reset index (previously using hydroid as index)
+    df_input_htable = df_input_htable.reset_index(
+        drop=True
+    )  # reset index (previously using hydroid as index)
     # sort the dataframe by branch_id and then by route_count
     # (need this ordered to ensure upstream to downstream ranking for each branch)
     df_input_htable = df_input_htable.sort_values(['branch_id', 'route_count'])
diff --git a/tools/vary_mannings_n_composite.py b/tools/vary_mannings_n_composite.py
index bdc2151a9..e451eb5dd 100755
--- a/tools/vary_mannings_n_composite.py
+++ b/tools/vary_mannings_n_composite.py
@@ -186,8 +186,7 @@ def variable_mannings_calc(args):
 
             ## drop the previously modified discharge column to be replaced with updated version
             df_htable = df_htable.drop(
-                ['vmann_on', 'discharge_cms', 'ManningN', 'vmann_discharge_cms', 'vmann_ManningN'],
-                axis=1,
+                ['vmann_on', 'discharge_cms', 'ManningN', 'vmann_discharge_cms', 'vmann_ManningN'], axis=1
             )
             df_htable = df_htable.merge(
                 df_src_trim, how='left', left_on=['HydroID', 'stage'], right_on=['HydroID', 'stage']

From c0eda055beb8aebe65cb0b34c937b7cc691c1631 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 6 Oct 2023 11:32:26 -0500
Subject: [PATCH 281/359] Bump urllib3 from 1.26.16 to 1.26.17 (#1002)

---
 Pipfile.lock | 1609 ++++++++++++++++++++++++++------------------------
 1 file changed, 838 insertions(+), 771 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 9937efe2c..4e66c9db4 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -42,11 +42,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780",
-                "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"
+                "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f",
+                "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.7.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.0.0"
         },
         "appdirs": {
             "hashes": [
@@ -92,18 +92,18 @@
         },
         "arrow": {
             "hashes": [
-                "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1",
-                "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2"
+                "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80",
+                "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.2.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.3.0"
         },
         "asttokens": {
             "hashes": [
-                "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
-                "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"
+                "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
+                "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
             ],
-            "version": "==2.2.1"
+            "version": "==2.4.0"
         },
         "attrs": {
             "hashes": [
@@ -133,7 +133,7 @@
                 "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
                 "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==4.12.2"
         },
         "black": {
@@ -162,6 +162,7 @@
                 "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==23.7.0"
         },
         "bleach": {
@@ -196,7 +197,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4.0'",
+            "markers": "python_version >= '3.8' and python_version < '4'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -205,6 +206,7 @@
                 "sha256:d388cb7f54f1a3056f91ffcfb5cf18b226454204e5df7a5c10774718c3fbb166"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==1.26.109"
         },
         "botocore": {
@@ -217,91 +219,92 @@
         },
         "brotli": {
             "hashes": [
-                "sha256:02177603aaca36e1fd21b091cb742bb3b305a569e2402f1ca38af471777fb019",
-                "sha256:11d3283d89af7033236fa4e73ec2cbe743d4f6a81d41bd234f24bf63dde979df",
-                "sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d",
-                "sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8",
-                "sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b",
-                "sha256:19598ecddd8a212aedb1ffa15763dd52a388518c4550e615aed88dc3753c0f0c",
-                "sha256:1c48472a6ba3b113452355b9af0a60da5c2ae60477f8feda8346f8fd48e3e87c",
-                "sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70",
-                "sha256:269a5743a393c65db46a7bb982644c67ecba4b8d91b392403ad8a861ba6f495f",
-                "sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181",
-                "sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130",
-                "sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19",
-                "sha256:3148362937217b7072cf80a2dcc007f09bb5ecb96dae4617316638194113d5be",
-                "sha256:330e3f10cd01da535c70d09c4283ba2df5fb78e915bea0a28becad6e2ac010be",
-                "sha256:336b40348269f9b91268378de5ff44dc6fbaa2268194f85177b53463d313842a",
-                "sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa",
-                "sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429",
-                "sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126",
-                "sha256:3b8b09a16a1950b9ef495a0f8b9d0a87599a9d1f179e2d4ac014b2ec831f87e7",
-                "sha256:3c1306004d49b84bd0c4f90457c6f57ad109f5cc6067a9664e12b7b79a9948ad",
-                "sha256:3ffaadcaeafe9d30a7e4e1e97ad727e4f5610b9fa2f7551998471e3736738679",
-                "sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4",
-                "sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0",
-                "sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b",
-                "sha256:495ba7e49c2db22b046a53b469bbecea802efce200dffb69b93dd47397edc9b6",
-                "sha256:4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438",
-                "sha256:503fa6af7da9f4b5780bb7e4cbe0c639b010f12be85d02c99452825dd0feef3f",
-                "sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389",
-                "sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6",
-                "sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26",
-                "sha256:5bf37a08493232fbb0f8229f1824b366c2fc1d02d64e7e918af40acd15f3e337",
-                "sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7",
-                "sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14",
-                "sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2",
-                "sha256:68715970f16b6e92c574c30747c95cf8cf62804569647386ff032195dc89a430",
-                "sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296",
-                "sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12",
-                "sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f",
-                "sha256:73fd30d4ce0ea48010564ccee1a26bfe39323fde05cb34b5863455629db61dc7",
-                "sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d",
-                "sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a",
-                "sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452",
-                "sha256:7ee83d3e3a024a9618e5be64648d6d11c37047ac48adff25f12fa4226cf23d1c",
-                "sha256:854c33dad5ba0fbd6ab69185fec8dab89e13cda6b7d191ba111987df74f38761",
-                "sha256:85f7912459c67eaab2fb854ed2bc1cc25772b300545fe7ed2dc03954da638649",
-                "sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b",
-                "sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea",
-                "sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c",
-                "sha256:8ed6a5b3d23ecc00ea02e1ed8e0ff9a08f4fc87a1f58a2530e71c0f48adf882f",
-                "sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a",
-                "sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031",
-                "sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267",
-                "sha256:97f715cf371b16ac88b8c19da00029804e20e25f30d80203417255d239f228b5",
-                "sha256:9bf919756d25e4114ace16a8ce91eb340eb57a08e2c6950c3cebcbe3dff2a5e7",
-                "sha256:9d12cf2851759b8de8ca5fde36a59c08210a97ffca0eb94c532ce7b17c6a3d1d",
-                "sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c",
-                "sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43",
-                "sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa",
-                "sha256:b1375b5d17d6145c798661b67e4ae9d5496920d9265e2f00f1c2c0b5ae91fbde",
-                "sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17",
-                "sha256:b3523f51818e8f16599613edddb1ff924eeb4b53ab7e7197f85cbc321cdca32f",
-                "sha256:b43775532a5904bc938f9c15b77c613cb6ad6fb30990f3b0afaea82797a402d8",
-                "sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb",
-                "sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb",
-                "sha256:ba72d37e2a924717990f4d7482e8ac88e2ef43fb95491eb6e0d124d77d2a150d",
-                "sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b",
-                "sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4",
-                "sha256:c8e521a0ce7cf690ca84b8cc2272ddaf9d8a50294fd086da67e517439614c755",
-                "sha256:cab1b5964b39607a66adbba01f1c12df2e55ac36c81ec6ed44f2fca44178bf1a",
-                "sha256:cb02ed34557afde2d2da68194d12f5719ee96cfb2eacc886352cb73e3808fc5d",
-                "sha256:cc0283a406774f465fb45ec7efb66857c09ffefbe49ec20b7882eff6d3c86d3a",
-                "sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3",
-                "sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7",
-                "sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1",
-                "sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb",
-                "sha256:e1abbeef02962596548382e393f56e4c94acd286bd0c5afba756cffc33670e8a",
-                "sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91",
-                "sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b",
-                "sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1",
-                "sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806",
-                "sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3",
-                "sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1"
+                "sha256:03d20af184290887bdea3f0f78c4f737d126c74dc2f3ccadf07e54ceca3bf208",
+                "sha256:0541e747cce78e24ea12d69176f6a7ddb690e62c425e01d31cc065e69ce55b48",
+                "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354",
+                "sha256:0b63b949ff929fbc2d6d3ce0e924c9b93c9785d877a21a1b678877ffbbc4423a",
+                "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128",
+                "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c",
+                "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088",
+                "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9",
+                "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a",
+                "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3",
+                "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438",
+                "sha256:22fc2a8549ffe699bfba2256ab2ed0421a7b8fadff114a3d201794e45a9ff578",
+                "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b",
+                "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b",
+                "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68",
+                "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d",
+                "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd",
+                "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409",
+                "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da",
+                "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50",
+                "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0",
+                "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180",
+                "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d",
+                "sha256:4d4a848d1837973bf0f4b5e54e3bec977d99be36a7895c61abb659301b02c112",
+                "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc",
+                "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265",
+                "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327",
+                "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95",
+                "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd",
+                "sha256:5eeb539606f18a0b232d4ba45adccde4125592f3f636a6182b4a8a436548b914",
+                "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0",
+                "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a",
+                "sha256:6172447e1b368dcbc458925e5ddaf9113477b0ed542df258d84fa28fc45ceea7",
+                "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0",
+                "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451",
+                "sha256:7905193081db9bfa73b1219140b3d315831cbff0d8941f22da695832f0dd188f",
+                "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e",
+                "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248",
+                "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91",
+                "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724",
+                "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966",
+                "sha256:890b5a14ce214389b2cc36ce82f3093f96f4cc730c1cffdbefff77a7c71f2a97",
+                "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d",
+                "sha256:8dadd1314583ec0bf2d1379f7008ad627cd6336625d6679cf2f8e67081b83acf",
+                "sha256:901032ff242d479a0efa956d853d16875d42157f98951c0230f69e69f9c09bac",
+                "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951",
+                "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74",
+                "sha256:929811df5462e182b13920da56c6e0284af407d1de637d8e536c5cd00a7daf60",
+                "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c",
+                "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1",
+                "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8",
+                "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d",
+                "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc",
+                "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61",
+                "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460",
+                "sha256:a743e5a28af5f70f9c080380a5f908d4d21d40e8f0e0c8901604d15cfa9ba751",
+                "sha256:a77def80806c421b4b0af06f45d65a136e7ac0bdca3c09d9e2ea4e515367c7e9",
+                "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1",
+                "sha256:ae15b066e5ad21366600ebec29a7ccbc86812ed267e4b28e860b8ca16a2bc474",
+                "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2",
+                "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6",
+                "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9",
+                "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2",
+                "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467",
+                "sha256:cdbc1fc1bc0bff1cef838eafe581b55bfbffaed4ed0318b724d0b71d4d377619",
+                "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf",
+                "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408",
+                "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579",
+                "sha256:d192f0f30804e55db0d0e0a35d83a9fead0e9a359a9ed0285dbacea60cc10a84",
+                "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b",
+                "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59",
+                "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752",
+                "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80",
+                "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0",
+                "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2",
+                "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3",
+                "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64",
+                "sha256:f296c40e23065d0d6650c4aefe7470d2a25fffda489bcc3eb66083f3ac9f6643",
+                "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e",
+                "sha256:f733d788519c7e3e71f0855c96618720f5d3d60c3cb829d8bbb722dddce37985",
+                "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596",
+                "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2",
+                "sha256:fdc3ff3bfccdc6b9cc7c342c03aa2400683f0cb891d46e94b64a197910dc4064"
             ],
             "markers": "platform_python_implementation == 'CPython'",
-            "version": "==1.0.9"
+            "version": "==1.1.0"
         },
         "cachetools": {
             "hashes": [
@@ -317,76 +320,66 @@
                 "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==2023.7.22"
         },
         "cffi": {
             "hashes": [
-                "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5",
-                "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef",
-                "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104",
-                "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426",
-                "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405",
-                "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375",
-                "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a",
-                "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e",
-                "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc",
-                "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf",
-                "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185",
-                "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497",
-                "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3",
-                "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35",
-                "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c",
-                "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83",
-                "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21",
-                "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca",
-                "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984",
-                "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac",
-                "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd",
-                "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee",
-                "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a",
-                "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2",
-                "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192",
-                "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7",
-                "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585",
-                "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f",
-                "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e",
-                "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27",
-                "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b",
-                "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e",
-                "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e",
-                "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d",
-                "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c",
-                "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415",
-                "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82",
-                "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02",
-                "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314",
-                "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325",
-                "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c",
-                "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3",
-                "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914",
-                "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045",
-                "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d",
-                "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9",
-                "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5",
-                "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2",
-                "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c",
-                "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3",
-                "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2",
-                "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8",
-                "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d",
-                "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d",
-                "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9",
-                "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162",
-                "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76",
-                "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4",
-                "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e",
-                "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9",
-                "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6",
-                "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b",
-                "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01",
-                "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"
-            ],
-            "version": "==1.15.1"
+                "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc",
+                "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a",
+                "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417",
+                "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab",
+                "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520",
+                "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36",
+                "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743",
+                "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8",
+                "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed",
+                "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684",
+                "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56",
+                "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324",
+                "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d",
+                "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235",
+                "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e",
+                "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088",
+                "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000",
+                "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7",
+                "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e",
+                "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673",
+                "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c",
+                "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe",
+                "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2",
+                "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098",
+                "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8",
+                "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a",
+                "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0",
+                "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b",
+                "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896",
+                "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e",
+                "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9",
+                "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2",
+                "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b",
+                "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6",
+                "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404",
+                "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f",
+                "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0",
+                "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4",
+                "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc",
+                "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936",
+                "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba",
+                "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872",
+                "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb",
+                "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614",
+                "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1",
+                "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d",
+                "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969",
+                "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b",
+                "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4",
+                "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627",
+                "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956",
+                "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.16.0"
         },
         "cfgv": {
             "hashes": [
@@ -425,84 +418,99 @@
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
-                "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
-                "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
-                "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
-                "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
-                "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
-                "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
-                "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
-                "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
-                "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
-                "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
-                "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
-                "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
-                "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
-                "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
-                "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
-                "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
-                "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
-                "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
-                "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
-                "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
-                "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
-                "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
-                "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
-                "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
-                "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
-                "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
-                "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
-                "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
-                "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
-                "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
-                "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
-                "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
-                "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
-                "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
-                "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
-                "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
-                "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
-                "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
-                "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
-                "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
-                "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
-                "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
-                "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
-                "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
-                "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
-                "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
-                "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
-                "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
-                "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
-                "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
-                "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
-                "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
-                "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
-                "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
-                "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
-                "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
-                "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
-                "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
-                "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
-                "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
-                "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
-                "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
-                "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
-                "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
-                "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
-                "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
-                "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
-                "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
-                "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
-                "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
-                "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
-                "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
-                "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
-                "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.2.0"
+                "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843",
+                "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786",
+                "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e",
+                "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8",
+                "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4",
+                "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa",
+                "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d",
+                "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82",
+                "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7",
+                "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895",
+                "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d",
+                "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a",
+                "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382",
+                "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678",
+                "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b",
+                "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e",
+                "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741",
+                "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4",
+                "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596",
+                "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9",
+                "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69",
+                "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c",
+                "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77",
+                "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13",
+                "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459",
+                "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e",
+                "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7",
+                "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908",
+                "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a",
+                "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f",
+                "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8",
+                "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482",
+                "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d",
+                "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d",
+                "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545",
+                "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34",
+                "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86",
+                "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6",
+                "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe",
+                "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e",
+                "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc",
+                "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7",
+                "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd",
+                "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c",
+                "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557",
+                "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a",
+                "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89",
+                "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078",
+                "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e",
+                "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4",
+                "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403",
+                "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0",
+                "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89",
+                "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115",
+                "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9",
+                "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05",
+                "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a",
+                "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec",
+                "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56",
+                "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38",
+                "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479",
+                "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c",
+                "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e",
+                "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd",
+                "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186",
+                "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455",
+                "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c",
+                "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65",
+                "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78",
+                "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287",
+                "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df",
+                "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43",
+                "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1",
+                "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7",
+                "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989",
+                "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a",
+                "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63",
+                "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884",
+                "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649",
+                "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810",
+                "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828",
+                "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4",
+                "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2",
+                "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd",
+                "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5",
+                "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe",
+                "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293",
+                "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e",
+                "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e",
+                "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"
+            ],
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==3.3.0"
         },
         "click": {
             "hashes": [
@@ -524,7 +532,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -561,56 +569,69 @@
         },
         "contourpy": {
             "hashes": [
-                "sha256:052cc634bf903c604ef1a00a5aa093c54f81a2612faedaa43295809ffdde885e",
-                "sha256:084eaa568400cfaf7179b847ac871582199b1b44d5699198e9602ecbbb5f6104",
-                "sha256:0b6616375d7de55797d7a66ee7d087efe27f03d336c27cf1f32c02b8c1a5ac70",
-                "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882",
-                "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f",
-                "sha256:17cfaf5ec9862bc93af1ec1f302457371c34e688fbd381f4035a06cd47324f48",
-                "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e",
-                "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a",
-                "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37",
-                "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a",
-                "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2",
-                "sha256:25ae46595e22f93592d39a7eac3d638cda552c3e1160255258b695f7b58e5655",
-                "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545",
-                "sha256:2b836d22bd2c7bb2700348e4521b25e077255ebb6ab68e351ab5aa91ca27e027",
-                "sha256:30f511c05fab7f12e0b1b7730ebdc2ec8deedcfb505bc27eb570ff47c51a8f15",
-                "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94",
-                "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439",
-                "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d",
-                "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa",
-                "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae",
-                "sha256:62013a2cf68abc80dadfd2307299bfa8f5aa0dcaec5b2954caeb5fa094171103",
-                "sha256:89f06eff3ce2f4b3eb24c1055a26981bffe4e7264acd86f15b97e40530b794bc",
-                "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa",
-                "sha256:911ff4fd53e26b019f898f32db0d4956c9d227d51338fb3b03ec72ff0084ee5f",
-                "sha256:9382a1c0bc46230fb881c36229bfa23d8c303b889b788b939365578d762b5c18",
-                "sha256:9f2931ed4741f98f74b410b16e5213f71dcccee67518970c42f64153ea9313b9",
-                "sha256:a67259c2b493b00e5a4d0f7bfae51fb4b3371395e47d079a4446e9b0f4d70e76",
-                "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493",
-                "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9",
-                "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed",
-                "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4",
-                "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f",
-                "sha256:dffcc2ddec1782dd2f2ce1ef16f070861af4fb78c69862ce0aab801495dda6a3",
-                "sha256:e53046c3863828d21d531cc3b53786e6580eb1ba02477e8681009b6aa0870b21",
-                "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e",
-                "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1",
-                "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a",
-                "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002",
-                "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"
+                "sha256:059c3d2a94b930f4dafe8105bcdc1b21de99b30b51b5bce74c753686de858cb6",
+                "sha256:0683e1ae20dc038075d92e0e0148f09ffcefab120e57f6b4c9c0f477ec171f33",
+                "sha256:07d6f11dfaf80a84c97f1a5ba50d129d9303c5b4206f776e94037332e298dda8",
+                "sha256:081f3c0880712e40effc5f4c3b08feca6d064cb8cfbb372ca548105b86fd6c3d",
+                "sha256:0e48694d6a9c5a26ee85b10130c77a011a4fedf50a7279fa0bdaf44bafb4299d",
+                "sha256:11b836b7dbfb74e049c302bbf74b4b8f6cb9d0b6ca1bf86cfa8ba144aedadd9c",
+                "sha256:19557fa407e70f20bfaba7d55b4d97b14f9480856c4fb65812e8a05fe1c6f9bf",
+                "sha256:229a25f68046c5cf8067d6d6351c8b99e40da11b04d8416bf8d2b1d75922521e",
+                "sha256:24216552104ae8f3b34120ef84825400b16eb6133af2e27a190fdc13529f023e",
+                "sha256:3b53d5769aa1f2d4ea407c65f2d1d08002952fac1d9e9d307aa2e1023554a163",
+                "sha256:3de23ca4f381c3770dee6d10ead6fff524d540c0f662e763ad1530bde5112532",
+                "sha256:407d864db716a067cc696d61fa1ef6637fedf03606e8417fe2aeed20a061e6b2",
+                "sha256:41339b24471c58dc1499e56783fedc1afa4bb018bcd035cfb0ee2ad2a7501ef8",
+                "sha256:462c59914dc6d81e0b11f37e560b8a7c2dbab6aca4f38be31519d442d6cde1a1",
+                "sha256:46e24f5412c948d81736509377e255f6040e94216bf1a9b5ea1eaa9d29f6ec1b",
+                "sha256:498e53573e8b94b1caeb9e62d7c2d053c263ebb6aa259c81050766beb50ff8d9",
+                "sha256:4ebf42695f75ee1a952f98ce9775c873e4971732a87334b099dde90b6af6a916",
+                "sha256:4f9147051cb8fdb29a51dc2482d792b3b23e50f8f57e3720ca2e3d438b7adf23",
+                "sha256:549174b0713d49871c6dee90a4b499d3f12f5e5f69641cd23c50a4542e2ca1eb",
+                "sha256:560f1d68a33e89c62da5da4077ba98137a5e4d3a271b29f2f195d0fba2adcb6a",
+                "sha256:566f0e41df06dfef2431defcfaa155f0acfa1ca4acbf8fd80895b1e7e2ada40e",
+                "sha256:56de98a2fb23025882a18b60c7f0ea2d2d70bbbcfcf878f9067234b1c4818442",
+                "sha256:66544f853bfa85c0d07a68f6c648b2ec81dafd30f272565c37ab47a33b220684",
+                "sha256:6c06e4c6e234fcc65435223c7b2a90f286b7f1b2733058bdf1345d218cc59e34",
+                "sha256:6d0a8efc258659edc5299f9ef32d8d81de8b53b45d67bf4bfa3067f31366764d",
+                "sha256:70e5a10f8093d228bb2b552beeb318b8928b8a94763ef03b858ef3612b29395d",
+                "sha256:8394e652925a18ef0091115e3cc191fef350ab6dc3cc417f06da66bf98071ae9",
+                "sha256:8636cd2fc5da0fb102a2504fa2c4bea3cbc149533b345d72cdf0e7a924decc45",
+                "sha256:93df44ab351119d14cd1e6b52a5063d3336f0754b72736cc63db59307dabb718",
+                "sha256:96ba37c2e24b7212a77da85004c38e7c4d155d3e72a45eeaf22c1f03f607e8ab",
+                "sha256:a10dab5ea1bd4401c9483450b5b0ba5416be799bbd50fc7a6cc5e2a15e03e8a3",
+                "sha256:a66045af6cf00e19d02191ab578a50cb93b2028c3eefed999793698e9ea768ae",
+                "sha256:a75cc163a5f4531a256f2c523bd80db509a49fc23721b36dd1ef2f60ff41c3cb",
+                "sha256:b04c2f0adaf255bf756cf08ebef1be132d3c7a06fe6f9877d55640c5e60c72c5",
+                "sha256:ba42e3810999a0ddd0439e6e5dbf6d034055cdc72b7c5c839f37a7c274cb4eba",
+                "sha256:bfc8a5e9238232a45ebc5cb3bfee71f1167064c8d382cadd6076f0d51cff1da0",
+                "sha256:c5bd5680f844c3ff0008523a71949a3ff5e4953eb7701b28760805bc9bcff217",
+                "sha256:c84fdf3da00c2827d634de4fcf17e3e067490c4aea82833625c4c8e6cdea0887",
+                "sha256:ca6fab080484e419528e98624fb5c4282148b847e3602dc8dbe0cb0669469887",
+                "sha256:d0c188ae66b772d9d61d43c6030500344c13e3f73a00d1dc241da896f379bb62",
+                "sha256:d6ab42f223e58b7dac1bb0af32194a7b9311065583cc75ff59dcf301afd8a431",
+                "sha256:dfe80c017973e6a4c367e037cb31601044dd55e6bfacd57370674867d15a899b",
+                "sha256:e0c02b75acfea5cab07585d25069207e478d12309557f90a61b5a3b4f77f46ce",
+                "sha256:e30aaf2b8a2bac57eb7e1650df1b3a4130e8d0c66fc2f861039d507a11760e1b",
+                "sha256:eafbef886566dc1047d7b3d4b14db0d5b7deb99638d8e1be4e23a7c7ac59ff0f",
+                "sha256:efe0fab26d598e1ec07d72cf03eaeeba8e42b4ecf6b9ccb5a356fde60ff08b85",
+                "sha256:f08e469821a5e4751c97fcd34bcb586bc243c39c2e39321822060ba902eac49e",
+                "sha256:f1eaac5257a8f8a047248d60e8f9315c6cff58f7803971170d952555ef6344a7",
+                "sha256:f29fb0b3f1217dfe9362ec55440d0743fe868497359f2cf93293f4b2701b8251",
+                "sha256:f44d78b61740e4e8c71db1cf1fd56d9050a4747681c59ec1094750a658ceb970",
+                "sha256:f6aec19457617ef468ff091669cca01fa7ea557b12b59a7908b9474bb9674cf0",
+                "sha256:f9dc7f933975367251c1b34da882c4f0e0b2e24bb35dc906d2f598a40b72bfc7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.1.0"
+            "version": "==1.1.1"
         },
         "cycler": {
             "hashes": [
-                "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3",
-                "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"
+                "sha256:7896994252d006771357777d0251f3e34d266f4fa5f2c572247a80ab01440947",
+                "sha256:8cc3a7b4861f91b1095157f9916f748549a617046e67eb7619abed9b34d2c94a"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.11.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.12.0"
         },
         "cython": {
             "hashes": [
@@ -700,27 +721,27 @@
         },
         "debugpy": {
             "hashes": [
-                "sha256:038c51268367c9c935905a90b1c2d2dbfe304037c27ba9d19fe7409f8cdc710c",
-                "sha256:1093a5c541af079c13ac8c70ab8b24d1d35c8cacb676306cf11e57f699c02926",
-                "sha256:3370ef1b9951d15799ef7af41f8174194f3482ee689988379763ef61a5456426",
-                "sha256:38651c3639a4e8bbf0ca7e52d799f6abd07d622a193c406be375da4d510d968d",
-                "sha256:3de5d0f97c425dc49bce4293df6a04494309eedadd2b52c22e58d95107e178d9",
-                "sha256:4b9eba71c290852f959d2cf8a03af28afd3ca639ad374d393d53d367f7f685b2",
-                "sha256:65b28435a17cba4c09e739621173ff90c515f7b9e8ea469b92e3c28ef8e5cdfb",
-                "sha256:72f5d2ecead8125cf669e62784ef1e6300f4067b0f14d9f95ee00ae06fc7c4f7",
-                "sha256:85969d864c45f70c3996067cfa76a319bae749b04171f2cdeceebe4add316155",
-                "sha256:890f7ab9a683886a0f185786ffbda3b46495c4b929dab083b8c79d6825832a52",
-                "sha256:903bd61d5eb433b6c25b48eae5e23821d4c1a19e25c9610205f5aeaccae64e32",
-                "sha256:92b6dae8bfbd497c90596bbb69089acf7954164aea3228a99d7e43e5267f5b36",
-                "sha256:973a97ed3b434eab0f792719a484566c35328196540676685c975651266fccf9",
-                "sha256:d16882030860081e7dd5aa619f30dec3c2f9a421e69861125f83cc372c94e57d",
-                "sha256:d4ac7a4dba28801d184b7fc0e024da2635ca87d8b0a825c6087bb5168e3c0d28",
-                "sha256:eea8d8cfb9965ac41b99a61f8e755a8f50e9a20330938ad8271530210f54e09c",
-                "sha256:f0851403030f3975d6e2eaa4abf73232ab90b98f041e3c09ba33be2beda43fcf",
-                "sha256:fe87ec0182ef624855d05e6ed7e0b7cb1359d2ffa2a925f8ec2d22e98b75d0ca"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.7.post1"
+                "sha256:125b9a637e013f9faac0a3d6a82bd17c8b5d2c875fb6b7e2772c5aba6d082332",
+                "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0",
+                "sha256:3c6fb41c98ec51dd010d7ed650accfd07a87fe5e93eca9d5f584d0578f28f35f",
+                "sha256:46ab6780159eeabb43c1495d9c84cf85d62975e48b6ec21ee10c95767c0590aa",
+                "sha256:57161629133113c97b387382045649a2b985a348f0c9366e22217c87b68b73c6",
+                "sha256:5d9de202f5d42e62f932507ee8b21e30d49aae7e46d5b1dd5c908db1d7068637",
+                "sha256:60009b132c91951354f54363f8ebdf7457aeb150e84abba5ae251b8e9f29a8a6",
+                "sha256:61eab4a4c8b6125d41a34bad4e5fe3d2cc145caecd63c3fe953be4cc53e65bf8",
+                "sha256:7fb95ca78f7ac43393cd0e0f2b6deda438ec7c5e47fa5d38553340897d2fbdfb",
+                "sha256:8cd0197141eb9e8a4566794550cfdcdb8b3db0818bdf8c49a8e8f8053e56e38b",
+                "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4",
+                "sha256:a64093656c4c64dc6a438e11d59369875d200bd5abb8f9b26c1f5f723622e153",
+                "sha256:a8b7a2fd27cd9f3553ac112f356ad4ca93338feadd8910277aff71ab24d8775f",
+                "sha256:b05a6b503ed520ad58c8dc682749113d2fd9f41ffd45daec16e558ca884008cd",
+                "sha256:bdc5ef99d14b9c0fcb35351b4fbfc06ac0ee576aeab6b2511702e5a648a2e595",
+                "sha256:e3412f9faa9ade82aa64a50b602544efcba848c91384e9f93497a458767e6926",
+                "sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e",
+                "sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.8.0"
         },
         "decorator": {
             "hashes": [
@@ -763,25 +784,25 @@
         },
         "executing": {
             "hashes": [
-                "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc",
-                "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"
+                "sha256:06df6183df67389625f4e763921c6cf978944721abf3e714000200aab95b0657",
+                "sha256:0ff053696fdeef426cda5bd18eacd94f82c91f49823a2e9090124212ceea9b08"
             ],
-            "version": "==1.2.0"
+            "version": "==2.0.0"
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:128039912a11a807068a7c87d0da36660afbfd7202780db26c4aa7153cfdc799",
-                "sha256:e820349dd16f806e4bd1467a138dced9def4bc7d6213a34295272a6cac95b5bd"
+                "sha256:06dc8680d937628e993fa0cd278f196d20449a1adc087640710846b324d422ea",
+                "sha256:aec6a19e9f66e9810ab371cc913ad5f4e9e479b63a7072a2cd060a9369e329a8"
             ],
-            "version": "==2.18.0"
+            "version": "==2.18.1"
         },
         "filelock": {
             "hashes": [
-                "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81",
-                "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"
+                "sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4",
+                "sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.12.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==3.12.4"
         },
         "fiona": {
             "hashes": [
@@ -814,47 +835,56 @@
                 "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181"
             ],
             "index": "pypi",
+            "markers": "python_full_version >= '3.8.1'",
             "version": "==6.0.0"
         },
         "fonttools": {
             "hashes": [
-                "sha256:0eb79a2da5eb6457a6f8ab904838454accc7d4cccdaff1fd2bd3a0679ea33d64",
-                "sha256:113337c2d29665839b7d90b39f99b3cac731f72a0eda9306165a305c7c31d341",
-                "sha256:12a7c247d1b946829bfa2f331107a629ea77dc5391dfd34fdcd78efa61f354ca",
-                "sha256:179737095eb98332a2744e8f12037b2977f22948cf23ff96656928923ddf560a",
-                "sha256:19b7db825c8adee96fac0692e6e1ecd858cae9affb3b4812cdb9d934a898b29e",
-                "sha256:37983b6bdab42c501202500a2be3a572f50d4efe3237e0686ee9d5f794d76b35",
-                "sha256:3a35981d90feebeaef05e46e33e6b9e5b5e618504672ca9cd0ff96b171e4bfff",
-                "sha256:46a0ec8adbc6ff13494eb0c9c2e643b6f009ce7320cf640de106fb614e4d4360",
-                "sha256:4aa79366e442dbca6e2c8595645a3a605d9eeabdb7a094d745ed6106816bef5d",
-                "sha256:515607ec756d7865f23070682622c49d922901943697871fc292277cf1e71967",
-                "sha256:53eb5091ddc8b1199330bb7b4a8a2e7995ad5d43376cadce84523d8223ef3136",
-                "sha256:5d18fc642fd0ac29236ff88ecfccff229ec0386090a839dd3f1162e9a7944a40",
-                "sha256:5fb289b7a815638a7613d46bcf324c9106804725b2bb8ad913c12b6958ffc4ec",
-                "sha256:62f481ac772fd68901573956231aea3e4b1ad87b9b1089a61613a91e2b50bb9b",
-                "sha256:689508b918332fb40ce117131633647731d098b1b10d092234aa959b4251add5",
-                "sha256:68a02bbe020dc22ee0540e040117535f06df9358106d3775e8817d826047f3fd",
-                "sha256:6ed2662a3d9c832afa36405f8748c250be94ae5dfc5283d668308391f2102861",
-                "sha256:7286aed4ea271df9eab8d7a9b29e507094b51397812f7ce051ecd77915a6e26b",
-                "sha256:7cc7d685b8eeca7ae69dc6416833fbfea61660684b7089bca666067cb2937dcf",
-                "sha256:8708b98c278012ad267ee8a7433baeb809948855e81922878118464b274c909d",
-                "sha256:9398f244e28e0596e2ee6024f808b06060109e33ed38dcc9bded452fd9bbb853",
-                "sha256:9e36344e48af3e3bde867a1ca54f97c308735dd8697005c2d24a86054a114a71",
-                "sha256:a398bdadb055f8de69f62b0fc70625f7cbdab436bbb31eef5816e28cab083ee8",
-                "sha256:acb47f6f8680de24c1ab65ebde39dd035768e2a9b571a07c7b8da95f6c8815fd",
-                "sha256:be24fcb80493b2c94eae21df70017351851652a37de514de553435b256b2f249",
-                "sha256:c391cd5af88aacaf41dd7cfb96eeedfad297b5899a39e12f4c2c3706d0a3329d",
-                "sha256:c95b0724a6deea2c8c5d3222191783ced0a2f09bd6d33f93e563f6f1a4b3b3a4",
-                "sha256:c9b1ce7a45978b821a06d375b83763b27a3a5e8a2e4570b3065abad240a18760",
-                "sha256:db372213d39fa33af667c2aa586a0c1235e88e9c850f5dd5c8e1f17515861868",
-                "sha256:db55cbaea02a20b49fefbd8e9d62bd481aaabe1f2301dabc575acc6b358874fa",
-                "sha256:ed1a13a27f59d1fc1920394a7f596792e9d546c9ca5a044419dca70c37815d7c",
-                "sha256:f2b82f46917d8722e6b5eafeefb4fb585d23babd15d8246c664cd88a5bddd19c",
-                "sha256:f2f806990160d1ce42d287aa419df3ffc42dfefe60d473695fb048355fe0c6a0",
-                "sha256:f720fa82a11c0f9042376fd509b5ed88dab7e3cd602eee63a1af08883b37342b"
+                "sha256:030355fbb0cea59cf75d076d04d3852900583d1258574ff2d7d719abf4513836",
+                "sha256:05056a8c9af048381fdb17e89b17d45f6c8394176d01e8c6fef5ac96ea950d38",
+                "sha256:206808f9717c9b19117f461246372a2c160fa12b9b8dbdfb904ab50ca235ba0a",
+                "sha256:20fc43783c432862071fa76da6fa714902ae587bc68441e12ff4099b94b1fcef",
+                "sha256:25620b738d4533cfc21fd2a4f4b667e481f7cb60e86b609799f7d98af657854e",
+                "sha256:33c40a657fb87ff83185828c0323032d63a4df1279d5c1c38e21f3ec56327803",
+                "sha256:3d7adfa342e6b3a2b36960981f23f480969f833d565a4eba259c2e6f59d2674f",
+                "sha256:48078357984214ccd22d7fe0340cd6ff7286b2f74f173603a1a9a40b5dc25afe",
+                "sha256:5056f69a18f3f28ab5283202d1efcfe011585d31de09d8560f91c6c88f041e92",
+                "sha256:52e77f23a9c059f8be01a07300ba4c4d23dc271d33eed502aea5a01ab5d2f4c1",
+                "sha256:57c22e5f9f53630d458830f710424dce4f43c5f0d95cb3368c0f5178541e4db7",
+                "sha256:5aa67d1e720fdd902fde4a59d0880854ae9f19fc958f3e1538bceb36f7f4dc92",
+                "sha256:5f9660e70a2430780e23830476332bc3391c3c8694769e2c0032a5038702a662",
+                "sha256:635658464dccff6fa5c3b43fe8f818ae2c386ee6a9e1abc27359d1e255528186",
+                "sha256:6a530fa28c155538d32214eafa0964989098a662bd63e91e790e6a7a4e9c02da",
+                "sha256:70f021a6b9eb10dfe7a411b78e63a503a06955dd6d2a4e130906d8760474f77c",
+                "sha256:77e5113233a2df07af9dbf493468ce526784c3b179c0e8b9c7838ced37c98b69",
+                "sha256:7c76f32051159f8284f1a5f5b605152b5a530736fb8b55b09957db38dcae5348",
+                "sha256:812142a0e53cc853964d487e6b40963df62f522b1b571e19d1ff8467d7880ceb",
+                "sha256:82d8e687a42799df5325e7ee12977b74738f34bf7fde1c296f8140efd699a213",
+                "sha256:8dfd8edfce34ad135bd69de20c77449c06e2c92b38f2a8358d0987737f82b49e",
+                "sha256:93c5b6d77baf28f306bc13fa987b0b13edca6a39dc2324eaca299a74ccc6316f",
+                "sha256:9d654d3e780e0ceabb1f4eff5a3c042c67d4428d0fe1ea3afd238a721cf171b3",
+                "sha256:a682fb5cbf8837d1822b80acc0be5ff2ea0c49ca836e468a21ffd388ef280fd3",
+                "sha256:a68b71adc3b3a90346e4ac92f0a69ab9caeba391f3b04ab6f1e98f2c8ebe88e3",
+                "sha256:a6a2e99bb9ea51e0974bbe71768df42c6dd189308c22f3f00560c3341b345646",
+                "sha256:ab80e7d6bb01316d5fc8161a2660ca2e9e597d0880db4927bc866c76474472ef",
+                "sha256:ace0fd5afb79849f599f76af5c6aa5e865bd042c811e4e047bbaa7752cc26126",
+                "sha256:ace51902ab67ef5fe225e8b361039e996db153e467e24a28d35f74849b37b7ce",
+                "sha256:af38f5145258e9866da5881580507e6d17ff7756beef175d13213a43a84244e9",
+                "sha256:b3813f57f85bbc0e4011a0e1e9211f9ee52f87f402e41dc05bc5135f03fa51c1",
+                "sha256:b5e760198f0b87e42478bb35a6eae385c636208f6f0d413e100b9c9c5efafb6a",
+                "sha256:b62a53a4ca83c32c6b78cac64464f88d02929779373c716f738af6968c8c821e",
+                "sha256:d08a694b280d615460563a6b4e2afb0b1b9df708c799ec212bf966652b94fc84",
+                "sha256:d27d960e10cf7617d70cf3104c32a69b008dde56f2d55a9bed4ba6e3df611544",
+                "sha256:da78f39b601ed0b4262929403186d65cf7a016f91ff349ab18fdc5a7080af465",
+                "sha256:dcc01cea0a121fb0c009993497bad93cae25e77db7dee5345fec9cce1aaa09cd",
+                "sha256:e3f8acc6ef4a627394021246e099faee4b343afd3ffe2e517d8195b4ebf20289",
+                "sha256:e4bc589d8da09267c7c4ceaaaa4fc01a7908ac5b43b286ac9279afe76407c384",
+                "sha256:e5d53eddaf436fa131042f44a76ea1ead0a17c354ab9de0d80e818f0cb1629f1",
+                "sha256:ee728d5af70f117581712966a21e2e07031e92c687ef1fdc457ac8d281016f64",
+                "sha256:f19c2b1c65d57cbea25cabb80941fea3fbf2625ff0cdcae8900b5fb1c145704f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.42.1"
+            "version": "==4.43.0"
         },
         "fqdn": {
             "hashes": [
@@ -865,11 +895,11 @@
         },
         "fsspec": {
             "hashes": [
-                "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a",
-                "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"
+                "sha256:603dbc52c75b84da501b9b2ec8c11e1f61c25984c4a0dda1f129ef391fbfc9b4",
+                "sha256:80bfb8c70cc27b2178cc62a935ecf242fc6e8c3fb801f9c571fc01b1e715ba7d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.6.0"
+            "version": "==2023.9.2"
         },
         "geocube": {
             "hashes": [
@@ -893,6 +923,7 @@
                 "sha256:0acdacddefa176525e4da6d9aeeece225da26055c4becdc6e97cf40fa97c27f4"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==0.12.2"
         },
         "geopy": {
@@ -909,15 +940,16 @@
                 "sha256:880d0bb3cdb1d366d9e18423d4478b74f72010d120a2b53cb53603163c790bc0"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==0.1.2"
         },
         "identify": {
             "hashes": [
-                "sha256:7243800bce2f58404ed41b7c002e53d4d22bcf3ae1b7900c2d7aefd95394bf7f",
-                "sha256:c22a8ead0d4ca11f1edd6c9418c3220669b3b7533ada0a0ffa6cc0ef85cf9b54"
+                "sha256:afe67f26ae29bab007ec21b03d4114f41316ab9dd15aa8736a167481e108da54",
+                "sha256:f302a4256a15c849b91cfcdcec052a8ce914634b2f77ae87dad29cd749f2d88d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.26"
+            "version": "==2.5.30"
         },
         "idna": {
             "hashes": [
@@ -937,11 +969,11 @@
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:134832a506243891221b88b4ae1213327eea96ceb4e407a00d790bb0626f45cf",
-                "sha256:4359457e42708462b9626a04657c6208ad799ceb41e5c58c57ffa0e6a098a5d4"
+                "sha256:9d48dcccc213325e810fd723e7fbb45ccb39f6cf5c31f00cf2b965f5f10f3cb9",
+                "sha256:aa50258bbfa56d4e33fbd8aa3ef48ded10d1735f11532b8df95388cc6bdb7e83"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==6.0.1"
+            "version": "==6.1.0"
         },
         "inflate64": {
             "hashes": [
@@ -1028,11 +1060,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:050391364c0977e768e354bdb60cbbfbee7cbb943b1af1618382021136ffd42f",
-                "sha256:c8a2430b357073b37c76c21c52184db42f6b4b0e438e1eb7df3c4440d120497c"
+                "sha256:2e2ee359baba19f10251b99415bb39de1e97d04e1fab385646f24f0596510b77",
+                "sha256:f468ddd1f17acb48c8ce67fcfa49ba6d46d4f9ac0438c1f441be7c3d1372230b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.25.1"
+            "version": "==6.25.2"
         },
         "ipympl": {
             "hashes": [
@@ -1044,11 +1076,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:c7b80eb7f5a855a88efc971fda506ff7a91c280b42cdae26643e0f601ea281ea",
-                "sha256:ea8801f15dfe4ffb76dea1b09b847430ffd70d827b41735c64a0638a04103bfc"
+                "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363",
+                "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.12.2"
+            "version": "==8.12.3"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1059,11 +1091,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:6c8396cc7b8c95dfb4e9ab0054f48c002f045e7e5d7ae523f559d64e525a98ab",
-                "sha256:ce97dd90525b3066fd00094690964e7eac14cf9b7745d35565b5eeac20cce687"
+                "sha256:2b88d728656aea3bbfd05d32c747cfd0078f9d7e159cf982433b58ad717eed7f",
+                "sha256:40211efb556adec6fa450ccc2a77d59ca44a060f4f9f136833df59c9f538e6e8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.0"
+            "version": "==8.1.1"
         },
         "isoduration": {
             "hashes": [
@@ -1078,15 +1110,16 @@
                 "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"
             ],
             "index": "pypi",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==5.12.0"
         },
         "jedi": {
             "hashes": [
-                "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
-                "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
+                "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd",
+                "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.19.0"
+            "version": "==0.19.1"
         },
         "jinja2": {
             "hashes": [
@@ -1131,11 +1164,11 @@
                 "format-nongpl"
             ],
             "hashes": [
-                "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb",
-                "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"
+                "sha256:cd5f1f9ed9444e554b38ba003af06c0a8c2868131e56bfbef0550fb450c0330e",
+                "sha256:ec84cc37cfa703ef7cd4928db24f9cb31428a5d0fa77747b8b51a847458e0bbf"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.19.0"
+            "version": "==4.19.1"
         },
         "jsonschema-specifications": {
             "hashes": [
@@ -1172,11 +1205,11 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:5ba5c7938a7f97a6b0481463f7ff0dbac7c15ba48cf46fa4035ca6e838aa1aba",
-                "sha256:ae9036db959a71ec1cac33081eeb040a79e681f08ab68b0883e9a676c7a90dce"
+                "sha256:0c28db6cbe2c37b5b398e1a1a5b22f84fd64cd10afc1f6c05b02fb09481ba45f",
+                "sha256:a4af53c3fa3f6330cebb0d9f658e148725d15652811d1c32dc0f63bb96f2e6d6"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.3.1"
+            "version": "==5.3.2"
         },
         "jupyter-events": {
             "hashes": [
@@ -1188,11 +1221,11 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:98a375347b580e837e7016007c24680a4261ed8ad7cd35196ac087d229f48e5a",
-                "sha256:d64fb4e593907290e5df916e3c9399c15ab2cd7bdb71cbcd1d36452dbfb30523"
+                "sha256:8e4b90380b59d7a1e31086c4692231f2a2ea4cb269f5516e60aba72ce8317fc9",
+                "sha256:d4916c8581c4ebbc534cebdaa8eca2478d9f3bfdd88eae29fcab0120eac57649"
             ],
-            "index": "pypi",
-            "version": "==2.7.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.7.3"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1232,6 +1265,7 @@
                 "sha256:6aba0caa771697d02fbf409f9767b2fdb4ee32ce935940e3b9a0d5d48d994d0f"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==3.6.3"
         },
         "jupyterlab-pygments": {
@@ -1244,19 +1278,19 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:4e6f99e0a5579bbbc32e449c4dbb039561d4f1a7827d5733273ed56738f21f07",
-                "sha256:5f077e142bb8dc9b843d960f940c513581bceca3793a0d80f9c67d9522c4e876"
+                "sha256:77c2f1f282d610f95e496e20d5bf1d2a7706826dfb7b18f3378ae2870d272fb7",
+                "sha256:c9f67a98b295c5dee87f41551b0558374e45d449f3edca153dd722140630dcb2"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.24.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.25.0"
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:4715912d6ceab839c9db35953c764b3214ebbc9161c809f6e0510168845dfdf5",
-                "sha256:d428ab97b8d87cc7c54cbf37644d6e0f0e662f23876e05fa460a73ec3257252a"
+                "sha256:3cf5bdf5b897bf3bccf1c11873aa4afd776d7430200f765e0686bd352487b58d",
+                "sha256:6005a4e974c7beee84060fdfba341a3218495046de8ae3ec64888e5fe19fdb4c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.8"
+            "version": "==3.0.9"
         },
         "kiwisolver": {
             "hashes": [
@@ -1416,8 +1450,11 @@
                 "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e",
                 "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431",
                 "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686",
+                "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c",
                 "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559",
                 "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc",
+                "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb",
+                "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939",
                 "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c",
                 "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0",
                 "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4",
@@ -1425,6 +1462,7 @@
                 "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575",
                 "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba",
                 "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d",
+                "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd",
                 "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3",
                 "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00",
                 "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155",
@@ -1433,6 +1471,7 @@
                 "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f",
                 "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8",
                 "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b",
+                "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007",
                 "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24",
                 "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea",
                 "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198",
@@ -1440,9 +1479,12 @@
                 "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee",
                 "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be",
                 "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2",
+                "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1",
                 "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707",
                 "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6",
+                "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c",
                 "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58",
+                "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823",
                 "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779",
                 "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636",
                 "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c",
@@ -1461,7 +1503,9 @@
                 "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9",
                 "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57",
                 "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc",
-                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"
+                "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc",
+                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2",
+                "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"
             ],
             "markers": "python_version >= '3.7'",
             "version": "==2.1.3"
@@ -1535,6 +1579,7 @@
                 "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.5'",
             "version": "==0.61.0"
         },
         "mercantile": {
@@ -1546,11 +1591,11 @@
         },
         "mistune": {
             "hashes": [
-                "sha256:b9b3e438efbb57c62b5beb5e134dab664800bdf1284a7ee09e8b12b13eb1aac6",
-                "sha256:e912116c13aa0944f9dc530db38eb88f6a77087ab128f49f84a48f4c05ea163c"
+                "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205",
+                "sha256:fc7f93ded930c92394ef2cb6f04a8aabab4117a91449e72dcc8dfa646a508be8"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.1"
+            "version": "==3.0.2"
         },
         "morecantile": {
             "hashes": [
@@ -1562,79 +1607,73 @@
         },
         "msgpack": {
             "hashes": [
-                "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164",
-                "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b",
-                "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c",
-                "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf",
-                "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd",
-                "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d",
-                "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c",
-                "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a",
-                "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e",
-                "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd",
-                "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025",
-                "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5",
-                "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705",
-                "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a",
-                "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d",
-                "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb",
-                "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11",
-                "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f",
-                "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c",
-                "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d",
-                "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea",
-                "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba",
-                "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87",
-                "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a",
-                "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c",
-                "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080",
-                "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198",
-                "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9",
-                "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a",
-                "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b",
-                "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f",
-                "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437",
-                "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f",
-                "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7",
-                "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2",
-                "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0",
-                "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48",
-                "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898",
-                "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0",
-                "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57",
-                "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8",
-                "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282",
-                "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1",
-                "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82",
-                "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc",
-                "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb",
-                "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6",
-                "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7",
-                "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9",
-                "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c",
-                "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1",
-                "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed",
-                "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c",
-                "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c",
-                "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77",
-                "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81",
-                "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a",
-                "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3",
-                "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086",
-                "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9",
-                "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f",
-                "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b",
-                "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"
-            ],
-            "version": "==1.0.5"
+                "sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862",
+                "sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d",
+                "sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3",
+                "sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672",
+                "sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0",
+                "sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9",
+                "sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee",
+                "sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46",
+                "sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524",
+                "sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819",
+                "sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc",
+                "sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc",
+                "sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1",
+                "sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82",
+                "sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81",
+                "sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6",
+                "sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d",
+                "sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2",
+                "sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c",
+                "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87",
+                "sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84",
+                "sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e",
+                "sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95",
+                "sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f",
+                "sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b",
+                "sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93",
+                "sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf",
+                "sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61",
+                "sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c",
+                "sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8",
+                "sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d",
+                "sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c",
+                "sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4",
+                "sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba",
+                "sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415",
+                "sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee",
+                "sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d",
+                "sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9",
+                "sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075",
+                "sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f",
+                "sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7",
+                "sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681",
+                "sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329",
+                "sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1",
+                "sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf",
+                "sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c",
+                "sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5",
+                "sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b",
+                "sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5",
+                "sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e",
+                "sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b",
+                "sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad",
+                "sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd",
+                "sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7",
+                "sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002",
+                "sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.7"
         },
         "multimethod": {
             "hashes": [
-                "sha256:1589bf52ca294667fd15527ea830127c763f5bfc38562e3642591ffd0fd9d56f",
-                "sha256:52f8f1f2b9d5a4c7adfdcc114dbeeebe3245a4420801e8807e26522a79fb6bc2"
+                "sha256:afd84da9c3d0445c84f827e4d63ad42d17c6d29b122427c6dee9032ac2d2a0d4",
+                "sha256:daa45af3fe257f73abb69673fd54ddeaf31df0eb7363ad6e1251b7c9b192d8c5"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.9.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.10"
         },
         "multipledispatch": {
             "hashes": [
@@ -1673,6 +1712,7 @@
                 "sha256:d583bc9050dd10538de36297c960b93f873f0cd01671a3c50df5bd86dd391dcb"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==8.3.1"
         },
         "nbclassic": {
@@ -1688,7 +1728,7 @@
                 "sha256:25e861299e5303a0477568557c4045eccc7a34c17fc08e7959558707b9ebe548",
                 "sha256:f9b179cd4b2d7bca965f900a2ebf0db4a12ebff2f36a711cb66861e4ae158e55"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==0.8.0"
         },
         "nbconvert": {
@@ -1709,11 +1749,11 @@
         },
         "nest-asyncio": {
             "hashes": [
-                "sha256:5301c82941b550b3123a1ea772ba9a1c80bad3a182be8c1a5ae6ad3be57a9657",
-                "sha256:6a80f7b98f24d9083ed24608977c09dd608d83f91cccc24c9d2cba6d10e01c10"
+                "sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb",
+                "sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==1.5.7"
+            "version": "==1.5.8"
         },
         "netcdf4": {
             "hashes": [
@@ -1746,6 +1786,7 @@
                 "sha256:f94a89db78f34fdf68342840efb064fe1474310e8359dffce42e90a9ddf88f2f"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==1.6.3"
         },
         "nodeenv": {
@@ -1758,11 +1799,11 @@
         },
         "notebook": {
             "hashes": [
-                "sha256:171039245a5b1a8f8233165091210632c21250ce2a652daed38fe8f94389984f",
-                "sha256:457caa1fa1c647395420945b2b7559f603eedbc9aeb2a59a0c286c8029e31efa"
+                "sha256:b4625a4b7a597839dd3156b140d5ba2c7123761f98245a3290f67a8b8ee048d9",
+                "sha256:c1e2eb2e3b6079a0552a04974883a48d04c3c05792170d64a4b23d707d453181"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.5.5"
+            "version": "==6.5.6"
         },
         "notebook-shim": {
             "hashes": [
@@ -1804,43 +1845,44 @@
                 "sha256:fcdf84ba3ed8124eb7234adfbb8792f311991cbf8aed1cad4b1b1a7ee08380c1"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.56.4"
         },
         "numexpr": {
             "hashes": [
-                "sha256:1510da20e6f5f45333610b1ded44c566e2690c6c437c84f2a212ca09627c7e01",
-                "sha256:178b85ad373c6903e55d75787d61b92380439b70d94b001cb055a501b0821335",
-                "sha256:183d5430db76826e54465c69db93a3c6ecbf03cda5aa1bb96eaad0147e9b68dc",
-                "sha256:283ce8609a7ccbadf91a68f3484558b3e36d27c93c98a41ec205efb0ab43c872",
-                "sha256:34af2a0e857d02a4bc5758bc037a777d50dacb13bcd57c7905268a3e44994ed6",
-                "sha256:39ce106f92ccea5b07b1d6f2f3c4370f05edf27691dc720a63903484a2137e48",
-                "sha256:3c00be69f747f44a631830215cab482f0f77f75af2925695adff57c1cc0f9a68",
-                "sha256:45ed41e55a0abcecf3d711481e12a5fb7a904fe99d42bc282a17cc5f8ea510be",
-                "sha256:4ed0e1c1ef5f34381448539f1fe9015906d21c9cfa2797c06194d4207dadb465",
-                "sha256:51f3ab160c3847ebcca93cd88f935a7802b54a01ab63fe93152994a64d7a6cf2",
-                "sha256:558390fea6370003ac749ed9d0f38d708aa096f5dcb707ddb6e0ca5a0dd37da1",
-                "sha256:55983806815035eb63c5039520688c49536bb7f3cc3fc1d7d64c6a00cf3f353e",
-                "sha256:578fe4008e4d5d6ff01bbeb2d7b7ba1ec658a5cda9c720cd26a9a8325f8ef438",
-                "sha256:5a8dad2bfaad5a5c34a2e8bbf62b9df1dfab266d345fda1feb20ff4e264b347a",
-                "sha256:62b4faf8e0627673b0210a837792bddd23050ecebc98069ab23eb0633ff1ef5f",
-                "sha256:6df184d40d4cf9f21c71f429962f39332f7398147762588c9f3a5c77065d0c06",
-                "sha256:783324ba40eb804ecfc9ebae86120a1e339ab112d0ab8a1f0d48a26354d5bf9b",
-                "sha256:894b027438b8ec88dea32a19193716c79f4ff8ddb92302dcc9731b51ba3565a8",
-                "sha256:9e8b5bf7bcb4e8dcd66522d8fc96e1db7278f901cb4fd2e155efbe62a41dde08",
-                "sha256:aea6ab45c87c0a7041183c08a798f0ad4d7c5eccbce20cfe79ce6f1a45ef3702",
-                "sha256:b594dc9e2d6291a0bc5c065e6d9caf3eee743b5663897832e9b17753c002947a",
-                "sha256:b93f5a866cd13a808bc3d3a9c487d94cd02eec408b275ff0aa150f2e8e5191f8",
-                "sha256:bf85ba1327eb87ec82ae7936f13c8850fb969a0ca34f3ba9fa3897c09d5c80d7",
-                "sha256:c46350dcdb93e32f033eea5a21269514ffcaf501d9abd6036992d37e48a308b0",
-                "sha256:cbfd833ee5fdb0efb862e152aee7e6ccea9c596d5c11d22604c2e6307bff7cad",
-                "sha256:db5c65417d69414f1ab31302ea01d3548303ef31209c38b4849d145be4e1d1ba",
-                "sha256:dd57ab1a3d3aaa9274aff1cefbf93b8ddacc7973afef5b125905f6bf18fabab0",
-                "sha256:de29c77f674e4eb8f0846525a475cab64008c227c8bc4ba5153ab3f72441cc63",
-                "sha256:eb36ffcfa1606e41aa08d559b4277bcad0e16b83941d1a4fee8d2bd5a34f8e0e",
-                "sha256:ef621b4ee366a5c6a484f6678c9259f5b826569f8bfa0b89ba2306d5055468bb"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.8.5"
+                "sha256:15469dc722b5ceb92324ec8635411355ebc702303db901ae8cc87f47c5e3a124",
+                "sha256:18b1804923cfa3be7bbb45187d01c0540c8f6df4928c22a0f786e15568e9ebc5",
+                "sha256:1967c16f61c27df1cdc43ba3c0ba30346157048dd420b4259832276144d0f64e",
+                "sha256:211804ec25a9f6d188eadf4198dd1a92b2f61d7d20993c6c7706139bc4199c5b",
+                "sha256:27782177a0081bd0aab229be5d37674e7f0ab4264ef576697323dd047432a4cd",
+                "sha256:31cf610c952eec57081171f0b4427f9bed2395ec70ec432bbf45d260c5c0cdeb",
+                "sha256:38b8b90967026bbc36c7aa6e8ca3b8906e1990914fd21f446e2a043f4ee3bc06",
+                "sha256:47b45da5aa25600081a649f5e8b2aa640e35db3703f4631f34bb1f2f86d1b5b4",
+                "sha256:6336f8dba3f456e41a4ffc3c97eb63d89c73589ff6e1707141224b930263260d",
+                "sha256:681812e2e71ff1ba9145fac42d03f51ddf6ba911259aa83041323f68e7458002",
+                "sha256:6d7003497d82ef19458dce380b36a99343b96a3bd5773465c2d898bf8f5a38f9",
+                "sha256:6e884687da8af5955dc9beb6a12d469675c90b8fb38b6c93668c989cfc2cd982",
+                "sha256:80acbfefb68bd92e708e09f0a02b29e04d388b9ae72f9fcd57988aca172a7833",
+                "sha256:84979bf14143351c2db8d9dd7fef8aca027c66ad9df9cb5e75c93bf5f7b5a338",
+                "sha256:8564186aad5a2c88d597ebc79b8171b52fd33e9b085013e1ff2208f7e4b387e3",
+                "sha256:8e3e6f1588d6c03877cb3b3dcc3096482da9d330013b886b29cb9586af5af3eb",
+                "sha256:95b9da613761e4fc79748535b2a1f58cada22500e22713ae7d9571fa88d1c2e2",
+                "sha256:95c09e814b0d6549de98b5ded7cdf7d954d934bb6b505432ff82e83a6d330bda",
+                "sha256:9ef7e8aaa84fce3aba2e65f243d14a9f8cc92aafd5d90d67283815febfe43eeb",
+                "sha256:aa0f661f5f4872fd7350cc9895f5d2594794b2a7e7f1961649a351724c64acc9",
+                "sha256:b5f96c89aa0b1f13685ec32fa3d71028db0b5981bfd99a0bbc271035949136b3",
+                "sha256:c48221b6a85494a7be5a022899764e58259af585dff031cecab337277278cc93",
+                "sha256:c8f37f7a6af3bdd61f2efd1cafcc083a9525ab0aaf5dc641e7ec8fc0ae2d3aa1",
+                "sha256:d126938c2c3784673c9c58d94e00b1570aa65517d9c33662234d442fc9fb5795",
+                "sha256:d36528a33aa9c23743b3ea686e57526a4f71e7128a1be66210e1511b09c4e4e9",
+                "sha256:d6a88d71c166e86b98d34701285d23e3e89d548d9f5ae3f4b60919ac7151949f",
+                "sha256:dee04d72307c09599f786b9231acffb10df7d7a74b2ce3681d74a574880d13ce",
+                "sha256:e640bc0eaf1b59f3dde52bc02bbfda98e62f9950202b0584deba28baf9f36bbb",
+                "sha256:e93d64cd20940b726477c3cb64926e683d31b778a1e18f9079a5088fd0d8e7c8",
+                "sha256:ef6e8896457a60a539cb6ba27da78315a9bb31edb246829b25b5b0304bfcee91"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.8.6"
         },
         "numpy": {
             "hashes": [
@@ -1874,6 +1916,7 @@
                 "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.23.5"
         },
         "odc-geo": {
@@ -1894,11 +1937,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
-                "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
+                "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
+                "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.1"
+            "version": "==23.2"
         },
         "pandas": {
             "hashes": [
@@ -1929,6 +1972,7 @@
                 "sha256:f908a77cbeef9bbd646bd4b81214cbef9ac3dda4181d5092a4aa9797d1bc7774"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==2.0.2"
         },
         "pandera": {
@@ -1965,11 +2009,11 @@
         },
         "partd": {
             "hashes": [
-                "sha256:7a63529348cf0dff14b986db641cd1b83c16b5cb9fc647c2851779db03282ef8",
-                "sha256:aa0ff35dbbcc807ae374db56332f4c1b39b46f67bf2975f5151e0b4186aed0d5"
+                "sha256:27e766663d36c161e2827aa3e28541c992f0b9527d3cca047e13fb3acdb989e6",
+                "sha256:56c25dd49e6fea5727e731203c466c6e092f308d8f0024e199d02f6aa2167f67"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.4.0"
+            "version": "==1.4.1"
         },
         "pathspec": {
             "hashes": [
@@ -1996,65 +2040,63 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5",
-                "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530",
-                "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d",
-                "sha256:1ce91b6ec08d866b14413d3f0bbdea7e24dfdc8e59f562bb77bc3fe60b6144ca",
-                "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891",
-                "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992",
-                "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7",
-                "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3",
-                "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba",
-                "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3",
-                "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3",
-                "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f",
-                "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538",
-                "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3",
-                "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d",
-                "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c",
-                "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017",
-                "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3",
-                "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223",
-                "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e",
-                "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3",
-                "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6",
-                "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640",
-                "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334",
-                "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1",
-                "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba",
-                "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa",
-                "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0",
-                "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396",
-                "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d",
-                "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485",
-                "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf",
-                "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43",
-                "sha256:bc2ec7c7b5d66b8ec9ce9f720dbb5fa4bace0f545acd34870eff4a369b44bf37",
-                "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2",
-                "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd",
-                "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86",
-                "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967",
-                "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629",
-                "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568",
-                "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed",
-                "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f",
-                "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551",
-                "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3",
-                "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614",
-                "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff",
-                "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d",
-                "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883",
-                "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684",
-                "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0",
-                "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de",
-                "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b",
-                "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3",
-                "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199",
-                "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51",
-                "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"
+                "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff",
+                "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f",
+                "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21",
+                "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635",
+                "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a",
+                "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f",
+                "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1",
+                "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d",
+                "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db",
+                "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849",
+                "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7",
+                "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876",
+                "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3",
+                "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317",
+                "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91",
+                "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d",
+                "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b",
+                "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd",
+                "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed",
+                "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500",
+                "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7",
+                "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a",
+                "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a",
+                "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0",
+                "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf",
+                "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f",
+                "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1",
+                "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088",
+                "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971",
+                "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a",
+                "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205",
+                "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54",
+                "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08",
+                "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21",
+                "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d",
+                "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08",
+                "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e",
+                "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf",
+                "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b",
+                "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145",
+                "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2",
+                "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d",
+                "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d",
+                "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf",
+                "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad",
+                "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d",
+                "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1",
+                "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4",
+                "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2",
+                "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19",
+                "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37",
+                "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4",
+                "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68",
+                "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==10.0.0"
+            "version": "==10.0.1"
         },
         "pkgutil-resolve-name": {
             "hashes": [
@@ -2066,11 +2108,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d",
-                "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"
+                "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3",
+                "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.10.0"
+            "version": "==3.11.0"
         },
         "pluggy": {
             "hashes": [
@@ -2086,6 +2128,7 @@
                 "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==3.3.3"
         },
         "prometheus-client": {
@@ -2101,7 +2144,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.39"
         },
         "psutil": {
@@ -2121,7 +2164,7 @@
                 "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
                 "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
             ],
-            "markers": "sys_platform != 'cygwin'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==5.9.5"
         },
         "psycopg2-binary": {
@@ -2190,6 +2233,7 @@
                 "sha256:ffe9dc0a884a8848075e576c1de0290d85a533a9f6e9c4e564f19adf8f6e54a7"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==2.9.6"
         },
         "ptyprocess": {
@@ -2219,6 +2263,7 @@
                 "sha256:94d0c24217f6582741813ee94490a4ca82bd5f9bf35e4f8610cb588cf7445764"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.20.4"
         },
         "pyarrow": {
@@ -2250,6 +2295,7 @@
                 "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==11.0.0"
         },
         "pybcj": {
@@ -2353,41 +2399,41 @@
         },
         "pycryptodomex": {
             "hashes": [
-                "sha256:160a39a708c36fa0b168ab79386dede588e62aec06eb505add870739329aecc6",
-                "sha256:192306cf881fe3467dda0e174a4f47bb3a8bb24b90c9cdfbdc248eec5fc0578c",
-                "sha256:1949e09ea49b09c36d11a951b16ff2a05a0ffe969dda1846e4686ee342fe8646",
-                "sha256:215be2980a6b70704c10796dd7003eb4390e7be138ac6fb8344bf47e71a8d470",
-                "sha256:27072a494ce621cc7a9096bbf60ed66826bb94db24b49b7359509e7951033e74",
-                "sha256:2dc4eab20f4f04a2d00220fdc9258717b82d31913552e766d5f00282c031b70a",
-                "sha256:302a8f37c224e7b5d72017d462a2be058e28f7be627bdd854066e16722d0fc0c",
-                "sha256:3d9314ac785a5b75d5aaf924c5f21d6ca7e8df442e5cf4f0fefad4f6e284d422",
-                "sha256:3e3ecb5fe979e7c1bb0027e518340acf7ee60415d79295e5251d13c68dde576e",
-                "sha256:4d9379c684efea80fdab02a3eb0169372bca7db13f9332cb67483b8dc8b67c37",
-                "sha256:50308fcdbf8345e5ec224a5502b4215178bdb5e95456ead8ab1a69ffd94779cb",
-                "sha256:5594a125dae30d60e94f37797fc67ce3c744522de7992c7c360d02fdb34918f8",
-                "sha256:58fc0aceb9c961b9897facec9da24c6a94c5db04597ec832060f53d4d6a07196",
-                "sha256:6421d23d6a648e83ba2670a352bcd978542dad86829209f59d17a3f087f4afef",
-                "sha256:6875eb8666f68ddbd39097867325bd22771f595b4e2b0149739b5623c8bf899b",
-                "sha256:6ed3606832987018615f68e8ed716a7065c09a0fe94afd7c9ca1b6777f0ac6eb",
-                "sha256:71687eed47df7e965f6e0bf3cadef98f368d5221f0fb89d2132effe1a3e6a194",
-                "sha256:73d64b32d84cf48d9ec62106aa277dbe99ab5fbfd38c5100bc7bddd3beb569f7",
-                "sha256:75672205148bdea34669173366df005dbd52be05115e919551ee97171083423d",
-                "sha256:76f0a46bee539dae4b3dfe37216f678769349576b0080fdbe431d19a02da42ff",
-                "sha256:8ff129a5a0eb5ff16e45ca4fa70a6051da7f3de303c33b259063c19be0c43d35",
-                "sha256:ac614363a86cc53d8ba44b6c469831d1555947e69ab3276ae8d6edc219f570f7",
-                "sha256:ba95abd563b0d1b88401658665a260852a8e6c647026ee6a0a65589287681df8",
-                "sha256:bbdcce0a226d9205560a5936b05208c709b01d493ed8307792075dedfaaffa5f",
-                "sha256:bec6c80994d4e7a38312072f89458903b65ec99bed2d65aa4de96d997a53ea7a",
-                "sha256:c2953afebf282a444c51bf4effe751706b4d0d63d7ca2cc51db21f902aa5b84e",
-                "sha256:d35a8ffdc8b05e4b353ba281217c8437f02c57d7233363824e9d794cf753c419",
-                "sha256:d56c9ec41258fd3734db9f5e4d2faeabe48644ba9ca23b18e1839b3bdf093222",
-                "sha256:d84e105787f5e5d36ec6a581ff37a1048d12e638688074b2a00bcf402f9aa1c2",
-                "sha256:e00a4bacb83a2627e8210cb353a2e31f04befc1155db2976e5e239dd66482278",
-                "sha256:f237278836dda412a325e9340ba2e6a84cb0f56b9244781e5b61f10b3905de88",
-                "sha256:f9ab5ef0718f6a8716695dea16d83b671b22c45e9c0c78fd807c32c0192e54b5"
+                "sha256:09c9401dc06fb3d94cb1ec23b4ea067a25d1f4c6b7b118ff5631d0b5daaab3cc",
+                "sha256:0b2f1982c5bc311f0aab8c293524b861b485d76f7c9ab2c3ac9a25b6f7655975",
+                "sha256:136b284e9246b4ccf4f752d435c80f2c44fc2321c198505de1d43a95a3453b3c",
+                "sha256:1789d89f61f70a4cd5483d4dfa8df7032efab1118f8b9894faae03c967707865",
+                "sha256:2126bc54beccbede6eade00e647106b4f4c21e5201d2b0a73e9e816a01c50905",
+                "sha256:258c4233a3fe5a6341780306a36c6fb072ef38ce676a6d41eec3e591347919e8",
+                "sha256:263de9a96d2fcbc9f5bd3a279f14ea0d5f072adb68ebd324987576ec25da084d",
+                "sha256:50cb18d4dd87571006fd2447ccec85e6cec0136632a550aa29226ba075c80644",
+                "sha256:5b883e1439ab63af976656446fb4839d566bb096f15fc3c06b5a99cde4927188",
+                "sha256:5d73e9fa3fe830e7b6b42afc49d8329b07a049a47d12e0ef9225f2fd220f19b2",
+                "sha256:61056a1fd3254f6f863de94c233b30dd33bc02f8c935b2000269705f1eeeffa4",
+                "sha256:67c8eb79ab33d0fbcb56842992298ddb56eb6505a72369c20f60bc1d2b6fb002",
+                "sha256:6e45bb4635b3c4e0a00ca9df75ef6295838c85c2ac44ad882410cb631ed1eeaa",
+                "sha256:7cb51096a6a8d400724104db8a7e4f2206041a1f23e58924aa3d8d96bcb48338",
+                "sha256:800a2b05cfb83654df80266692f7092eeefe2a314fa7901dcefab255934faeec",
+                "sha256:8df69e41f7e7015a90b94d1096ec3d8e0182e73449487306709ec27379fff761",
+                "sha256:917033016ecc23c8933205585a0ab73e20020fdf671b7cd1be788a5c4039840b",
+                "sha256:a12144d785518f6491ad334c75ccdc6ad52ea49230b4237f319dbb7cef26f464",
+                "sha256:a3866d68e2fc345162b1b9b83ef80686acfe5cec0d134337f3b03950a0a8bf56",
+                "sha256:a588a1cb7781da9d5e1c84affd98c32aff9c89771eac8eaa659d2760666f7139",
+                "sha256:a77b79852175064c822b047fee7cf5a1f434f06ad075cc9986aa1c19a0c53eb0",
+                "sha256:af83a554b3f077564229865c45af0791be008ac6469ef0098152139e6bd4b5b6",
+                "sha256:b801216c48c0886742abf286a9a6b117e248ca144d8ceec1f931ce2dd0c9cb40",
+                "sha256:bfb040b5dda1dff1e197d2ef71927bd6b8bfcb9793bc4dfe0bb6df1e691eaacb",
+                "sha256:c01678aee8ac0c1a461cbc38ad496f953f9efcb1fa19f5637cbeba7544792a53",
+                "sha256:c74eb1f73f788facece7979ce91594dc177e1a9b5d5e3e64697dd58299e5cb4d",
+                "sha256:c9a68a2f7bd091ccea54ad3be3e9d65eded813e6d79fdf4cc3604e26cdd6384f",
+                "sha256:d4dd3b381ff5a5907a3eb98f5f6d32c64d319a840278ceea1dcfcc65063856f3",
+                "sha256:e8e5ecbd4da4157889fce8ba49da74764dd86c891410bfd6b24969fa46edda51",
+                "sha256:eb2fc0ec241bf5e5ef56c8fbec4a2634d631e4c4f616a59b567947a0f35ad83c",
+                "sha256:edbe083c299835de7e02c8aa0885cb904a75087d35e7bab75ebe5ed336e8c3e2",
+                "sha256:ff64fd720def623bf64d8776f8d0deada1cc1bf1ec3c1f9d6f5bb5bd098d034f"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==3.18.0"
+            "version": "==3.19.0"
         },
         "pyct": {
             "hashes": [
@@ -2586,6 +2632,7 @@
                 "sha256:fde5ece4d2436b5a57c8f5f97b49b5de06a856d03959f836c957d3e609f2de7e"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==3.5.0"
         },
         "pyproject-flake8": {
@@ -2594,6 +2641,7 @@
                 "sha256:d43421caca0ef8a672874405fe63c722b0333e3c22c41648c6df60f21bab2b6b"
             ],
             "index": "pypi",
+            "markers": "python_full_version >= '3.8.1'",
             "version": "==6.0.0.post1"
         },
         "pytest": {
@@ -2602,6 +2650,7 @@
                 "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==7.3.0"
         },
         "python-dateutil": {
@@ -2618,6 +2667,7 @@
                 "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.0.0"
         },
         "python-json-logger": {
@@ -2630,10 +2680,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588",
-                "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"
+                "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b",
+                "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"
             ],
-            "version": "==2023.3"
+            "version": "==2023.3.post1"
         },
         "pyyaml": {
             "hashes": [
@@ -2889,11 +2939,11 @@
         },
         "qtconsole": {
             "hashes": [
-                "sha256:35fd6e87b1f6d1fd41801b07e69339f8982e76afd4fa8ef35595bc6036717189",
-                "sha256:5e4082a86a201796b2a5cfd4298352d22b158b51b57736531824715fc2a979dd"
+                "sha256:a3b69b868e041c2c698bdc75b0602f42e130ffb256d6efa48f9aa756c97672aa",
+                "sha256:b7ffb53d74f23cee29f4cdb55dd6fabc8ec312d94f3c46ba38e1dde458693dfb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.3"
+            "version": "==5.4.4"
         },
         "qtpy": {
             "hashes": [
@@ -2924,6 +2974,7 @@
                 "sha256:eaaeb2e661d1ffc07a7ae4fd997bb326d3561f641178126102842d608a010cc3"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.3.6"
         },
         "rasterstats": {
@@ -2932,6 +2983,7 @@
                 "sha256:29389bfcbeac1a4206aba6e1d795058ec8a64d560efad48156c27fad97c2e09a"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.18.0"
         },
         "referencing": {
@@ -3003,106 +3055,106 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:00215f6a9058fbf84f9d47536902558eb61f180a6b2a0fa35338d06ceb9a2e5a",
-                "sha256:0028eb0967942d0d2891eae700ae1a27b7fd18604cfcb16a1ef486a790fee99e",
-                "sha256:0155c33af0676fc38e1107679be882077680ad1abb6303956b97259c3177e85e",
-                "sha256:063411228b852fb2ed7485cf91f8e7d30893e69b0acb207ec349db04cccc8225",
-                "sha256:0700c2133ba203c4068aaecd6a59bda22e06a5e46255c9da23cbf68c6942215d",
-                "sha256:08e08ccf5b10badb7d0a5c84829b914c6e1e1f3a716fdb2bf294e2bd01562775",
-                "sha256:0d292cabd7c8335bdd3237ded442480a249dbcdb4ddfac5218799364a01a0f5c",
-                "sha256:15932ec5f224b0e35764dc156514533a4fca52dcfda0dfbe462a1a22b37efd59",
-                "sha256:18f87baa20e02e9277ad8960cd89b63c79c05caf106f4c959a9595c43f2a34a5",
-                "sha256:1a6420a36975e0073acaeee44ead260c1f6ea56812cfc6c31ec00c1c48197173",
-                "sha256:1b401e8b9aece651512e62c431181e6e83048a651698a727ea0eb0699e9f9b74",
-                "sha256:1d7b7b71bcb82d8713c7c2e9c5f061415598af5938666beded20d81fa23e7640",
-                "sha256:23750a9b8a329844ba1fe267ca456bb3184984da2880ed17ae641c5af8de3fef",
-                "sha256:23a059143c1393015c68936370cce11690f7294731904bdae47cc3e16d0b2474",
-                "sha256:26d9fd624649a10e4610fab2bc820e215a184d193e47d0be7fe53c1c8f67f370",
-                "sha256:291c9ce3929a75b45ce8ddde2aa7694fc8449f2bc8f5bd93adf021efaae2d10b",
-                "sha256:298e8b5d8087e0330aac211c85428c8761230ef46a1f2c516d6a2f67fb8803c5",
-                "sha256:2c7c4266c1b61eb429e8aeb7d8ed6a3bfe6c890a1788b18dbec090c35c6b93fa",
-                "sha256:2d68a8e8a3a816629283faf82358d8c93fe5bd974dd2704152394a3de4cec22a",
-                "sha256:344b89384c250ba6a4ce1786e04d01500e4dac0f4137ceebcaad12973c0ac0b3",
-                "sha256:3455ecc46ea443b5f7d9c2f946ce4017745e017b0d0f8b99c92564eff97e97f5",
-                "sha256:3d544a614055b131111bed6edfa1cb0fb082a7265761bcb03321f2dd7b5c6c48",
-                "sha256:3e5c26905aa651cc8c0ddc45e0e5dea2a1296f70bdc96af17aee9d0493280a17",
-                "sha256:3f5cc8c7bc99d2bbcd704cef165ca7d155cd6464c86cbda8339026a42d219397",
-                "sha256:4992266817169997854f81df7f6db7bdcda1609972d8ffd6919252f09ec3c0f6",
-                "sha256:4d55528ef13af4b4e074d067977b1f61408602f53ae4537dccf42ba665c2c7bd",
-                "sha256:576da63eae7809f375932bfcbca2cf20620a1915bf2fedce4b9cc8491eceefe3",
-                "sha256:58fc4d66ee349a23dbf08c7e964120dc9027059566e29cf0ce6205d590ed7eca",
-                "sha256:5b9bf77008f2c55dabbd099fd3ac87009471d223a1c7ebea36873d39511b780a",
-                "sha256:5e7996aed3f65667c6dcc8302a69368435a87c2364079a066750a2eac75ea01e",
-                "sha256:5f7487be65b9c2c510819e744e375bd41b929a97e5915c4852a82fbb085df62c",
-                "sha256:6388e4e95a26717b94a05ced084e19da4d92aca883f392dffcf8e48c8e221a24",
-                "sha256:65af12f70355de29e1092f319f85a3467f4005e959ab65129cb697169ce94b86",
-                "sha256:668d2b45d62c68c7a370ac3dce108ffda482b0a0f50abd8b4c604a813a59e08f",
-                "sha256:71333c22f7cf5f0480b59a0aef21f652cf9bbaa9679ad261b405b65a57511d1e",
-                "sha256:7150b83b3e3ddaac81a8bb6a9b5f93117674a0e7a2b5a5b32ab31fdfea6df27f",
-                "sha256:748e472345c3a82cfb462d0dff998a7bf43e621eed73374cb19f307e97e08a83",
-                "sha256:75dbfd41a61bc1fb0536bf7b1abf272dc115c53d4d77db770cd65d46d4520882",
-                "sha256:7618a082c55cf038eede4a918c1001cc8a4411dfe508dc762659bcd48d8f4c6e",
-                "sha256:780fcb855be29153901c67fc9c5633d48aebef21b90aa72812fa181d731c6b00",
-                "sha256:78d10c431073dc6ebceed35ab22948a016cc2b5120963c13a41e38bdde4a7212",
-                "sha256:7a3a3d3e4f1e3cd2a67b93a0b6ed0f2499e33f47cc568e3a0023e405abdc0ff1",
-                "sha256:7b6975d3763d0952c111700c0634968419268e6bbc0b55fe71138987fa66f309",
-                "sha256:80772e3bda6787510d9620bc0c7572be404a922f8ccdfd436bf6c3778119464c",
-                "sha256:80992eb20755701753e30a6952a96aa58f353d12a65ad3c9d48a8da5ec4690cf",
-                "sha256:841128a22e6ac04070a0f84776d07e9c38c4dcce8e28792a95e45fc621605517",
-                "sha256:861d25ae0985a1dd5297fee35f476b60c6029e2e6e19847d5b4d0a43a390b696",
-                "sha256:872f3dcaa8bf2245944861d7311179d2c0c9b2aaa7d3b464d99a7c2e401f01fa",
-                "sha256:87c93b25d538c433fb053da6228c6290117ba53ff6a537c133b0f2087948a582",
-                "sha256:8856aa76839dc234d3469f1e270918ce6bec1d6a601eba928f45d68a15f04fc3",
-                "sha256:885e023e73ce09b11b89ab91fc60f35d80878d2c19d6213a32b42ff36543c291",
-                "sha256:899b5e7e2d5a8bc92aa533c2d4e55e5ebba095c485568a5e4bedbc163421259a",
-                "sha256:8ce8caa29ebbdcde67e5fd652c811d34bc01f249dbc0d61e5cc4db05ae79a83b",
-                "sha256:8e1c68303ccf7fceb50fbab79064a2636119fd9aca121f28453709283dbca727",
-                "sha256:8e7e2b3577e97fa43c2c2b12a16139b2cedbd0770235d5179c0412b4794efd9b",
-                "sha256:92f05fc7d832e970047662b3440b190d24ea04f8d3c760e33e7163b67308c878",
-                "sha256:97f5811df21703446b42303475b8b855ee07d6ab6cdf8565eff115540624f25d",
-                "sha256:9affee8cb1ec453382c27eb9043378ab32f49cd4bc24a24275f5c39bf186c279",
-                "sha256:a2da4a8c6d465fde36cea7d54bf47b5cf089073452f0e47c8632ecb9dec23c07",
-                "sha256:a6903cdca64f1e301af9be424798328c1fe3b4b14aede35f04510989fc72f012",
-                "sha256:a8ab1adf04ae2d6d65835995218fd3f3eb644fe20655ca8ee233e2c7270ff53b",
-                "sha256:a8edd467551c1102dc0f5754ab55cd0703431cd3044edf8c8e7d9208d63fa453",
-                "sha256:ac00c41dd315d147b129976204839ca9de699d83519ff1272afbe4fb9d362d12",
-                "sha256:ad277f74b1c164f7248afa968700e410651eb858d7c160d109fb451dc45a2f09",
-                "sha256:ae46a50d235f1631d9ec4670503f7b30405103034830bc13df29fd947207f795",
-                "sha256:afe6b5a04b2ab1aa89bad32ca47bf71358e7302a06fdfdad857389dca8fb5f04",
-                "sha256:b1cb078f54af0abd835ca76f93a3152565b73be0f056264da45117d0adf5e99c",
-                "sha256:b25136212a3d064a8f0b9ebbb6c57094c5229e0de76d15c79b76feff26aeb7b8",
-                "sha256:b3226b246facae14909b465061ddcfa2dfeadb6a64f407f24300d42d69bcb1a1",
-                "sha256:b98e75b21fc2ba5285aef8efaf34131d16af1c38df36bdca2f50634bea2d3060",
-                "sha256:bbd7b24d108509a1b9b6679fcc1166a7dd031dbef1f3c2c73788f42e3ebb3beb",
-                "sha256:bed57543c99249ab3a4586ddc8786529fbc33309e5e8a1351802a06ca2baf4c2",
-                "sha256:c0583f69522732bdd79dca4cd3873e63a29acf4a299769c7541f2ca1e4dd4bc6",
-                "sha256:c1e0e9916301e3b3d970814b1439ca59487f0616d30f36a44cead66ee1748c31",
-                "sha256:c651847545422c8131660704c58606d841e228ed576c8f1666d98b3d318f89da",
-                "sha256:c7853f27195598e550fe089f78f0732c66ee1d1f0eaae8ad081589a5a2f5d4af",
-                "sha256:cbae50d352e4717ffc22c566afc2d0da744380e87ed44a144508e3fb9114a3f4",
-                "sha256:cdbed8f21204398f47de39b0a9b180d7e571f02dfb18bf5f1b618e238454b685",
-                "sha256:d08395595c42bcd82c3608762ce734504c6d025eef1c06f42326a6023a584186",
-                "sha256:d4639111e73997567343df6551da9dd90d66aece1b9fc26c786d328439488103",
-                "sha256:d63787f289944cc4bde518ad2b5e70a4f0d6e2ce76324635359c74c113fd188f",
-                "sha256:d6d5f061f6a2aa55790b9e64a23dfd87b6664ab56e24cd06c78eb43986cb260b",
-                "sha256:d7865df1fb564092bcf46dac61b5def25342faf6352e4bc0e61a286e3fa26a3d",
-                "sha256:db6585b600b2e76e98131e0ac0e5195759082b51687ad0c94505970c90718f4a",
-                "sha256:e36d7369363d2707d5f68950a64c4e025991eb0177db01ccb6aa6facae48b69f",
-                "sha256:e7947d9a6264c727a556541b1630296bbd5d0a05068d21c38dde8e7a1c703ef0",
-                "sha256:eb2d59bc196e6d3b1827c7db06c1a898bfa0787c0574af398e65ccf2e97c0fbe",
-                "sha256:ee9c2f6ca9774c2c24bbf7b23086264e6b5fa178201450535ec0859739e6f78d",
-                "sha256:f4760e1b02173f4155203054f77a5dc0b4078de7645c922b208d28e7eb99f3e2",
-                "sha256:f70bec8a14a692be6dbe7ce8aab303e88df891cbd4a39af091f90b6702e28055",
-                "sha256:f869e34d2326e417baee430ae998e91412cc8e7fdd83d979277a90a0e79a5b47",
-                "sha256:f8b9a7cd381970e64849070aca7c32d53ab7d96c66db6c2ef7aa23c6e803f514",
-                "sha256:f99d74ddf9d3b6126b509e81865f89bd1283e3fc1b568b68cd7bd9dfa15583d7",
-                "sha256:f9e7e493ded7042712a374471203dd43ae3fff5b81e3de1a0513fa241af9fd41",
-                "sha256:fc72ae476732cdb7b2c1acb5af23b478b8a0d4b6fcf19b90dd150291e0d5b26b",
-                "sha256:fccbf0cd3411719e4c9426755df90bf3449d9fc5a89f077f4a7f1abd4f70c910",
-                "sha256:ffcf18ad3edf1c170e27e88b10282a2c449aa0358659592462448d71b2000cfc"
+                "sha256:015de2ce2af1586ff5dc873e804434185199a15f7d96920ce67e50604592cae9",
+                "sha256:061c3ff1f51ecec256e916cf71cc01f9975af8fb3af9b94d3c0cc8702cfea637",
+                "sha256:08a80cf4884920863623a9ee9a285ee04cef57ebedc1cc87b3e3e0f24c8acfe5",
+                "sha256:09362f86ec201288d5687d1dc476b07bf39c08478cde837cb710b302864e7ec9",
+                "sha256:0bb4f48bd0dd18eebe826395e6a48b7331291078a879295bae4e5d053be50d4c",
+                "sha256:106af1653007cc569d5fbb5f08c6648a49fe4de74c2df814e234e282ebc06957",
+                "sha256:11fdd1192240dda8d6c5d18a06146e9045cb7e3ba7c06de6973000ff035df7c6",
+                "sha256:16a472300bc6c83fe4c2072cc22b3972f90d718d56f241adabc7ae509f53f154",
+                "sha256:176287bb998fd1e9846a9b666e240e58f8d3373e3bf87e7642f15af5405187b8",
+                "sha256:177914f81f66c86c012311f8c7f46887ec375cfcfd2a2f28233a3053ac93a569",
+                "sha256:177c9dd834cdf4dc39c27436ade6fdf9fe81484758885f2d616d5d03c0a83bd2",
+                "sha256:187700668c018a7e76e89424b7c1042f317c8df9161f00c0c903c82b0a8cac5c",
+                "sha256:1d9b5ee46dcb498fa3e46d4dfabcb531e1f2e76b477e0d99ef114f17bbd38453",
+                "sha256:22da15b902f9f8e267020d1c8bcfc4831ca646fecb60254f7bc71763569f56b1",
+                "sha256:24cd91a03543a0f8d09cb18d1cb27df80a84b5553d2bd94cba5979ef6af5c6e7",
+                "sha256:255f1a10ae39b52122cce26ce0781f7a616f502feecce9e616976f6a87992d6b",
+                "sha256:271c360fdc464fe6a75f13ea0c08ddf71a321f4c55fc20a3fe62ea3ef09df7d9",
+                "sha256:2ed83d53a8c5902ec48b90b2ac045e28e1698c0bea9441af9409fc844dc79496",
+                "sha256:2f3e1867dd574014253b4b8f01ba443b9c914e61d45f3674e452a915d6e929a3",
+                "sha256:35fbd23c1c8732cde7a94abe7fb071ec173c2f58c0bd0d7e5b669fdfc80a2c7b",
+                "sha256:37d0c59548ae56fae01c14998918d04ee0d5d3277363c10208eef8c4e2b68ed6",
+                "sha256:39d05e65f23a0fe897b6ac395f2a8d48c56ac0f583f5d663e0afec1da89b95da",
+                "sha256:3ad59efe24a4d54c2742929001f2d02803aafc15d6d781c21379e3f7f66ec842",
+                "sha256:3aed39db2f0ace76faa94f465d4234aac72e2f32b009f15da6492a561b3bbebd",
+                "sha256:3bbac1953c17252f9cc675bb19372444aadf0179b5df575ac4b56faaec9f6294",
+                "sha256:40bc802a696887b14c002edd43c18082cb7b6f9ee8b838239b03b56574d97f71",
+                "sha256:42f712b4668831c0cd85e0a5b5a308700fe068e37dcd24c0062904c4e372b093",
+                "sha256:448a66b8266de0b581246ca7cd6a73b8d98d15100fb7165974535fa3b577340e",
+                "sha256:485301ee56ce87a51ccb182a4b180d852c5cb2b3cb3a82f7d4714b4141119d8c",
+                "sha256:485747ee62da83366a44fbba963c5fe017860ad408ccd6cd99aa66ea80d32b2e",
+                "sha256:4cf0855a842c5b5c391dd32ca273b09e86abf8367572073bd1edfc52bc44446b",
+                "sha256:4eca20917a06d2fca7628ef3c8b94a8c358f6b43f1a621c9815243462dcccf97",
+                "sha256:4ed172d0c79f156c1b954e99c03bc2e3033c17efce8dd1a7c781bc4d5793dfac",
+                "sha256:5267cfda873ad62591b9332fd9472d2409f7cf02a34a9c9cb367e2c0255994bf",
+                "sha256:52b5cbc0469328e58180021138207e6ec91d7ca2e037d3549cc9e34e2187330a",
+                "sha256:53d7a3cd46cdc1689296348cb05ffd4f4280035770aee0c8ead3bbd4d6529acc",
+                "sha256:563646d74a4b4456d0cf3b714ca522e725243c603e8254ad85c3b59b7c0c4bf0",
+                "sha256:570cc326e78ff23dec7f41487aa9c3dffd02e5ee9ab43a8f6ccc3df8f9327623",
+                "sha256:5aca759ada6b1967fcfd4336dcf460d02a8a23e6abe06e90ea7881e5c22c4de6",
+                "sha256:5de11c041486681ce854c814844f4ce3282b6ea1656faae19208ebe09d31c5b8",
+                "sha256:5e271dd97c7bb8eefda5cca38cd0b0373a1fea50f71e8071376b46968582af9b",
+                "sha256:642ed0a209ced4be3a46f8cb094f2d76f1f479e2a1ceca6de6346a096cd3409d",
+                "sha256:6446002739ca29249f0beaaf067fcbc2b5aab4bc7ee8fb941bd194947ce19aff",
+                "sha256:691d50c99a937709ac4c4cd570d959a006bd6a6d970a484c84cc99543d4a5bbb",
+                "sha256:69b857a7d8bd4f5d6e0db4086da8c46309a26e8cefdfc778c0c5cc17d4b11e08",
+                "sha256:6ac3fefb0d168c7c6cab24fdfc80ec62cd2b4dfd9e65b84bdceb1cb01d385c33",
+                "sha256:6c9141af27a4e5819d74d67d227d5047a20fa3c7d4d9df43037a955b4c748ec5",
+                "sha256:7170cbde4070dc3c77dec82abf86f3b210633d4f89550fa0ad2d4b549a05572a",
+                "sha256:763ad59e105fca09705d9f9b29ecffb95ecdc3b0363be3bb56081b2c6de7977a",
+                "sha256:77076bdc8776a2b029e1e6ffbe6d7056e35f56f5e80d9dc0bad26ad4a024a762",
+                "sha256:7cd020b1fb41e3ab7716d4d2c3972d4588fdfbab9bfbbb64acc7078eccef8860",
+                "sha256:821392559d37759caa67d622d0d2994c7a3f2fb29274948ac799d496d92bca73",
+                "sha256:829e91f3a8574888b73e7a3feb3b1af698e717513597e23136ff4eba0bc8387a",
+                "sha256:850c272e0e0d1a5c5d73b1b7871b0a7c2446b304cec55ccdb3eaac0d792bb065",
+                "sha256:87d9b206b1bd7a0523375dc2020a6ce88bca5330682ae2fe25e86fd5d45cea9c",
+                "sha256:8bd01ff4032abaed03f2db702fa9a61078bee37add0bd884a6190b05e63b028c",
+                "sha256:8d54bbdf5d56e2c8cf81a1857250f3ea132de77af543d0ba5dce667183b61fec",
+                "sha256:8efaeb08ede95066da3a3e3c420fcc0a21693fcd0c4396d0585b019613d28515",
+                "sha256:8f94fdd756ba1f79f988855d948ae0bad9ddf44df296770d9a58c774cfbcca72",
+                "sha256:95cde244e7195b2c07ec9b73fa4c5026d4a27233451485caa1cd0c1b55f26dbd",
+                "sha256:975382d9aa90dc59253d6a83a5ca72e07f4ada3ae3d6c0575ced513db322b8ec",
+                "sha256:9dd9d9d9e898b9d30683bdd2b6c1849449158647d1049a125879cb397ee9cd12",
+                "sha256:a019a344312d0b1f429c00d49c3be62fa273d4a1094e1b224f403716b6d03be1",
+                "sha256:a4d9bfda3f84fc563868fe25ca160c8ff0e69bc4443c5647f960d59400ce6557",
+                "sha256:a657250807b6efd19b28f5922520ae002a54cb43c2401e6f3d0230c352564d25",
+                "sha256:a771417c9c06c56c9d53d11a5b084d1de75de82978e23c544270ab25e7c066ff",
+                "sha256:aad6ed9e70ddfb34d849b761fb243be58c735be6a9265b9060d6ddb77751e3e8",
+                "sha256:ae87137951bb3dc08c7d8bfb8988d8c119f3230731b08a71146e84aaa919a7a9",
+                "sha256:af247fd4f12cca4129c1b82090244ea5a9d5bb089e9a82feb5a2f7c6a9fe181d",
+                "sha256:b5d4bdd697195f3876d134101c40c7d06d46c6ab25159ed5cbd44105c715278a",
+                "sha256:b9255e7165083de7c1d605e818025e8860636348f34a79d84ec533546064f07e",
+                "sha256:c22211c165166de6683de8136229721f3d5c8606cc2c3d1562da9a3a5058049c",
+                "sha256:c55f9821f88e8bee4b7a72c82cfb5ecd22b6aad04033334f33c329b29bfa4da0",
+                "sha256:c7aed97f2e676561416c927b063802c8a6285e9b55e1b83213dfd99a8f4f9e48",
+                "sha256:cd2163f42868865597d89399a01aa33b7594ce8e2c4a28503127c81a2f17784e",
+                "sha256:ce5e7504db95b76fc89055c7f41e367eaadef5b1d059e27e1d6eabf2b55ca314",
+                "sha256:cff7351c251c7546407827b6a37bcef6416304fc54d12d44dbfecbb717064717",
+                "sha256:d27aa6bbc1f33be920bb7adbb95581452cdf23005d5611b29a12bb6a3468cc95",
+                "sha256:d3b52a67ac66a3a64a7e710ba629f62d1e26ca0504c29ee8cbd99b97df7079a8",
+                "sha256:de61e424062173b4f70eec07e12469edde7e17fa180019a2a0d75c13a5c5dc57",
+                "sha256:e10e6a1ed2b8661201e79dff5531f8ad4cdd83548a0f81c95cf79b3184b20c33",
+                "sha256:e1a0ffc39f51aa5f5c22114a8f1906b3c17eba68c5babb86c5f77d8b1bba14d1",
+                "sha256:e22491d25f97199fc3581ad8dd8ce198d8c8fdb8dae80dea3512e1ce6d5fa99f",
+                "sha256:e626b864725680cd3904414d72e7b0bd81c0e5b2b53a5b30b4273034253bb41f",
+                "sha256:e8c71ea77536149e36c4c784f6d420ffd20bea041e3ba21ed021cb40ce58e2c9",
+                "sha256:e8d0f0eca087630d58b8c662085529781fd5dc80f0a54eda42d5c9029f812599",
+                "sha256:ea65b59882d5fa8c74a23f8960db579e5e341534934f43f3b18ec1839b893e41",
+                "sha256:ea93163472db26ac6043e8f7f93a05d9b59e0505c760da2a3cd22c7dd7111391",
+                "sha256:eab75a8569a095f2ad470b342f2751d9902f7944704f0571c8af46bede438475",
+                "sha256:ed8313809571a5463fd7db43aaca68ecb43ca7a58f5b23b6e6c6c5d02bdc7882",
+                "sha256:ef5fddfb264e89c435be4adb3953cef5d2936fdeb4463b4161a6ba2f22e7b740",
+                "sha256:ef750a20de1b65657a1425f77c525b0183eac63fe7b8f5ac0dd16f3668d3e64f",
+                "sha256:efb9ece97e696bb56e31166a9dd7919f8f0c6b31967b454718c6509f29ef6fee",
+                "sha256:f4c179a7aeae10ddf44c6bac87938134c1379c49c884529f090f9bf05566c836",
+                "sha256:f602881d80ee4228a2355c68da6b296a296cd22bbb91e5418d54577bbf17fa7c",
+                "sha256:fc2200e79d75b5238c8d69f6a30f8284290c777039d331e7340b6c17cad24a5a",
+                "sha256:fcc1ebb7561a3e24a6588f7c6ded15d80aec22c66a070c757559b57b17ffd1cb"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.0"
+            "version": "==0.10.3"
         },
         "rtree": {
             "hashes": [
@@ -3153,6 +3205,7 @@
                 "sha256:f5120da3a1b96f3a7a17dd6af0afdd4e6f3cc9baa87e9ee0a272882f01f980bb"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==1.0.1"
         },
         "s3transfer": {
@@ -3188,6 +3241,7 @@
                 "sha256:fae8a7b898c42dffe3f7361c40d5952b6bf32d10c4569098d276b4c547905ee1"
             ],
             "index": "pypi",
+            "markers": "python_version < '3.12' and python_version >= '3.8'",
             "version": "==1.10.1"
         },
         "seaborn": {
@@ -3196,6 +3250,7 @@
                 "sha256:ebf15355a4dba46037dfd65b7350f014ceb1f13c05e814eda2c9f5fd731afc08"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.12.2"
         },
         "send2trash": {
@@ -3208,11 +3263,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d",
-                "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"
+                "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87",
+                "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==68.1.2"
+            "version": "==68.2.2"
         },
         "shapely": {
             "hashes": [
@@ -3256,6 +3311,7 @@
                 "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==2.0.1"
         },
         "simplejson": {
@@ -3374,18 +3430,18 @@
         },
         "soupsieve": {
             "hashes": [
-                "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8",
-                "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"
+                "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690",
+                "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.4.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.5"
         },
         "stack-data": {
             "hashes": [
-                "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815",
-                "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"
+                "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9",
+                "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"
             ],
-            "version": "==0.6.2"
+            "version": "==0.6.3"
         },
         "tables": {
             "hashes": [
@@ -3408,6 +3464,7 @@
                 "sha256:f0821007048f2af8c1a21eb3d832072046c5df366e39587a7c7e4afad14e73fc"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==3.8.0"
         },
         "terminado": {
@@ -3472,31 +3529,39 @@
                 "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==4.65.0"
         },
         "traitlets": {
             "hashes": [
-                "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8",
-                "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"
+                "sha256:07ab9c5bf8a0499fd7b088ba51be899c90ffc936ffc797d7b6907fc516bcd116",
+                "sha256:db9c4aa58139c3ba850101913915c042bdba86f7c8a0dda1c6f7f92c5da8e542"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==5.9.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.10.1"
         },
         "typeguard": {
             "hashes": [
-                "sha256:5b7453b1e3b35fcfe2d62fa4ec500d05e6f2f2eb46f4126ae964677fcc384fff",
-                "sha256:7d4264cd631ac1157c5bb5ec992281b4f1e2ba7a35db91bc15f442235e244803"
+                "sha256:8923e55f8873caec136c892c3bed1f676eae7be57cdb94819281b3d3bc9c0953",
+                "sha256:ea0a113bbc111bcffc90789ebb215625c963411f7096a7e9062d4e4630c155fd"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.1.3"
+            "version": "==4.1.5"
+        },
+        "types-python-dateutil": {
+            "hashes": [
+                "sha256:1f4f10ac98bb8b16ade9dbee3518d9ace017821d94b057a425b069f834737f4b",
+                "sha256:f977b8de27787639986b4e28963263fd0e5158942b3ecef91b9335c130cb1ce9"
+            ],
+            "version": "==2.8.19.14"
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
-                "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
+                "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0",
+                "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.7.1"
+            "version": "==4.8.0"
         },
         "typing-inspect": {
             "hashes": [
@@ -3522,26 +3587,27 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f",
-                "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"
+                "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21",
+                "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"
             ],
+            "index": "pypi",
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.16"
+            "version": "==1.26.17"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:95a6e9398b4967fbcb5fef2acec5efaf9aa4972049d9ae41f95e0972a683fd02",
-                "sha256:e5c3b4ce817b0b328af041506a2a299418c98747c4b1e68cb7527e74ced23efc"
+                "sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b",
+                "sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.24.3"
+            "version": "==20.24.5"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
-                "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"
+                "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704",
+                "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"
             ],
-            "version": "==0.2.6"
+            "version": "==0.2.8"
         },
         "webcolors": {
             "hashes": [
@@ -3559,11 +3625,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa",
-                "sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537"
+                "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
+                "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.6.2"
+            "version": "==1.6.3"
         },
         "whitebox": {
             "hashes": [
@@ -3575,11 +3641,11 @@
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:2e37f0ce9da11651056280c7efe96f2db052fe8fc269508e3724f5cbd6c93018",
-                "sha256:9ec291ba87c2dfad42c3d5b6f68713fa18be1acd7476569516b2431682315c17"
+                "sha256:3c1f5e46dc1166dfd40a42d685e6a51396fd34ff878742a3e47c6f0cc4a2a385",
+                "sha256:91452ca8445beb805792f206e560c1769284267a30ceb1cec9f5bcc887d15175"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.0.8"
+            "version": "==4.0.9"
         },
         "wrapt": {
             "hashes": [
@@ -3668,6 +3734,7 @@
                 "sha256:7e530b1deafdd43e5c2b577d0944e6b528fbe88045fd849e49a8d11871ecd522"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==2023.1.0"
         },
         "xarray-spatial": {
@@ -3767,20 +3834,20 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0",
-                "sha256:ebc15946aa78bd63458992fc81ec3b6f7b1e92d51c35e6de1c3804e73b799147"
+                "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31",
+                "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.16.2"
+            "version": "==3.17.0"
         }
     },
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
-                "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"
+                "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
+                "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
             ],
-            "version": "==2.2.1"
+            "version": "==2.4.0"
         },
         "backcall": {
             "hashes": [
@@ -3799,26 +3866,26 @@
         },
         "executing": {
             "hashes": [
-                "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc",
-                "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"
+                "sha256:06df6183df67389625f4e763921c6cf978944721abf3e714000200aab95b0657",
+                "sha256:0ff053696fdeef426cda5bd18eacd94f82c91f49823a2e9090124212ceea9b08"
             ],
-            "version": "==1.2.0"
+            "version": "==2.0.0"
         },
         "ipython": {
             "hashes": [
-                "sha256:c7b80eb7f5a855a88efc971fda506ff7a91c280b42cdae26643e0f601ea281ea",
-                "sha256:ea8801f15dfe4ffb76dea1b09b847430ffd70d827b41735c64a0638a04103bfc"
+                "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363",
+                "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==8.12.2"
+            "version": "==8.12.3"
         },
         "jedi": {
             "hashes": [
-                "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
-                "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
+                "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd",
+                "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.19.0"
+            "version": "==0.19.1"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -3856,7 +3923,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.39"
         },
         "ptyprocess": {
@@ -3891,33 +3958,33 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815",
-                "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"
+                "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9",
+                "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"
             ],
-            "version": "==0.6.2"
+            "version": "==0.6.3"
         },
         "traitlets": {
             "hashes": [
-                "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8",
-                "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"
+                "sha256:07ab9c5bf8a0499fd7b088ba51be899c90ffc936ffc797d7b6907fc516bcd116",
+                "sha256:db9c4aa58139c3ba850101913915c042bdba86f7c8a0dda1c6f7f92c5da8e542"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==5.9.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.10.1"
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
-                "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
+                "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0",
+                "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.7.1"
+            "version": "==4.8.0"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
-                "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"
+                "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704",
+                "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"
             ],
-            "version": "==0.2.6"
+            "version": "==0.2.8"
         }
     }
 }

From 4d403cfbaff26986cad5f4396a19fe0ebbdac949 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 13 Oct 2023 12:58:32 -0400
Subject: [PATCH 282/359] 4.4.3.0 Revise stream clipping to WBD (#1005)

---
 docs/CHANGELOG.md          | 14 ++++++++++++++
 src/bash_variables.env     |  2 +-
 src/clip_vectors_to_wbd.py | 18 ++++++++++++++++--
 src/derive_level_paths.py  |  4 +---
 src/stream_branches.py     |  4 +---
 5 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index dfff11dd7..254e6ccb5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.3.0 - 2023-10-13 - [PR#1005](https://github.com/NOAA-OWP/inundation-mapping/pull/1005)
+
+Revise stream clipping to WBD by (1) reducing the buffer to clip streams away from the edge of the DEM (to prevent reverse flow issues) from 3 cells to 8 cells to account for the 70m AGREE buffer; (2) splitting MultiLineStrings formed by NWM streams being clipped by the DEM edge and then re-entering the DEM, and retaining only the lowest segment. Also changes the value of `input_WBD_gdb` to use the WBD clipped to the DEM domain.
+
+### Changes
+
+- `src/`
+    - `bash_variables.env`: Update WBD to the WBD clipped to the DEM domain
+    - `clip_vectors_to_wbd.py`: Decrease stream buffer from 3 to 8 cells inside of the WBD buffer; select the lowest segment of any incoming levelpaths that are split by the DEM edge.
+    - `derive_level_paths.py`: Remove unused argument
+    - `stream_branches.py`: Remove unused argument
+
+<br/><br/>
+
 ## v4.4.2.3 - 2023-09-21 - [PR#998](https://github.com/NOAA-OWP/inundation-mapping/pull/998)
 
 Removes exclude list for black formatter in `.pre-commit-config.yaml` as well as in `pyproject.toml`. Ran the `black` executable on the 
diff --git a/src/bash_variables.env b/src/bash_variables.env
index b581dc439..db6430d4e 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -12,7 +12,7 @@ export            input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchmen
 export                 input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
 export            input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
 export                 input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
-export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National.gpkg
+export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
 export          input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
 
 # Styling
diff --git a/src/clip_vectors_to_wbd.py b/src/clip_vectors_to_wbd.py
index 081728506..a5fd025fc 100644
--- a/src/clip_vectors_to_wbd.py
+++ b/src/clip_vectors_to_wbd.py
@@ -53,7 +53,7 @@ def subset_vector_layers(
 
     # Make the streams buffer smaller than the wbd_buffer so streams don't reach the edge of the DEM
     wbd_streams_buffer = wbd_buffer.copy()
-    wbd_streams_buffer.geometry = wbd_streams_buffer.geometry.buffer(-3 * dem_cellsize, resolution=32)
+    wbd_streams_buffer.geometry = wbd_streams_buffer.geometry.buffer(-8 * dem_cellsize, resolution=32)
 
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
@@ -174,7 +174,21 @@ def subset_vector_layers(
     nwm_streams_nonoutlets = nwm_streams[nwm_streams['to'].isin(nwm_streams['ID'])]
 
     if len(nwm_streams) > 0:
-        nwm_streams_nonoutlets = gpd.clip(nwm_streams_nonoutlets, wbd_streams_buffer)
+        # Address issue where NWM streams exit the HUC boundary and then re-enter, creating a MultiLineString
+        nwm_streams_nonoutlets = (
+            gpd.clip(nwm_streams_nonoutlets, wbd_streams_buffer).explode(index_parts=True).reset_index()
+        )
+
+        # Find and keep the downstream segment of the NWM stream
+        max_parts = nwm_streams_nonoutlets[['level_0', 'level_1']].groupby('level_0').max()
+
+        nwm_streams_nonoutlets = nwm_streams_nonoutlets.merge(max_parts, on='level_0', suffixes=('', '_max'))
+
+        nwm_streams_nonoutlets = nwm_streams_nonoutlets[
+            nwm_streams_nonoutlets['level_1'] == nwm_streams_nonoutlets['level_1_max']
+        ]
+
+        nwm_streams_nonoutlets = nwm_streams_nonoutlets.drop(columns=['level_1_max'])
 
         nwm_streams = pd.concat([nwm_streams_nonoutlets, nwm_streams_outlets])
 
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index 5236a0d8f..01a405437 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -152,9 +152,7 @@ def Derive_level_paths(
     # derive headwaters
     if headwaters_outfile is not None:
         headwaters = stream_network.derive_headwater_points_with_inlets(
-            fromNode_attribute=fromNode_attribute,
-            inlets_attribute=inlets_attribute,
-            outlet_linestring_index=outlet_linestring_index,
+            inlets_attribute=inlets_attribute, outlet_linestring_index=outlet_linestring_index
         )
         # headwaters write
         headwaters.to_file(headwaters_outfile, index=False, driver="GPKG")
diff --git a/src/stream_branches.py b/src/stream_branches.py
index fada36a6d..849d6e92f 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -468,9 +468,7 @@ def derive_inlet_points_by_feature(self, feature_attribute, outlet_linestring_in
 
         return feature_inlet_points_gdf
 
-    def derive_headwater_points_with_inlets(
-        self, inlets_attribute="inlet_id", fromNode_attribute="FromNode", outlet_linestring_index=0
-    ):
+    def derive_headwater_points_with_inlets(self, inlets_attribute="inlet_id", outlet_linestring_index=0):
         """Derives headwater points file given inlets"""
 
         # get inlet linestring index

From 6db3be492b985b9069be14c51cfa34d4c65f52f9 Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 20 Oct 2023 10:52:36 -0600
Subject: [PATCH 283/359] v4.4.4.0 Pre clip wbd8 (#1012)

---
 {src => data/wbd}/clip_vectors_to_wbd.py |  18 +-
 data/wbd/generate_pre_clip_fim_huc8.py   | 406 +++++++++++++++++++++++
 data/write_parquet_from_calib_pts.py     |   2 +-
 docs/CHANGELOG.md                        |  20 +-
 src/bash_variables.env                   |   1 +
 src/run_unit_wb.sh                       |  65 +---
 6 files changed, 439 insertions(+), 73 deletions(-)
 rename {src => data/wbd}/clip_vectors_to_wbd.py (94%)
 mode change 100644 => 100755
 create mode 100755 data/wbd/generate_pre_clip_fim_huc8.py

diff --git a/src/clip_vectors_to_wbd.py b/data/wbd/clip_vectors_to_wbd.py
old mode 100644
new mode 100755
similarity index 94%
rename from src/clip_vectors_to_wbd.py
rename to data/wbd/clip_vectors_to_wbd.py
index a5fd025fc..44e877291
--- a/src/clip_vectors_to_wbd.py
+++ b/data/wbd/clip_vectors_to_wbd.py
@@ -38,7 +38,7 @@ def subset_vector_layers(
     levee_protected_areas,
     subset_levee_protected_areas,
 ):
-    print("Getting Cell Size", flush=True)
+    print(f"Getting Cell Size for {hucCode}", flush=True)
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
 
@@ -46,7 +46,7 @@ def subset_vector_layers(
     dem_domain = gpd.read_file(dem_domain)
 
     # Get wbd buffer
-    print("Create wbd buffer", flush=True)
+    print(f"Create wbd buffer for {hucCode}", flush=True)
     wbd_buffer = wbd.copy()
     wbd_buffer.geometry = wbd_buffer.geometry.buffer(wbd_buffer_distance, resolution=32)
     wbd_buffer = gpd.clip(wbd_buffer, dem_domain)
@@ -73,14 +73,14 @@ def subset_vector_layers(
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea, mask=wbd_buffer)
     if not landsea.empty:
-        print("Create landsea gpkg", flush=True)
+        print(f"Create landsea gpkg for {hucCode}", flush=True)
         landsea.to_file(
             subset_landsea, driver=getDriver(subset_landsea), index=False, crs=DEFAULT_FIM_PROJECTION_CRS
         )
     del landsea
 
     # Clip levee-protected areas polygons for future masking ocean areas (where applicable)
-    print("Subsetting Levee Protected Areas", flush=True)
+    print(f"Subsetting Levee Protected Areas for {hucCode}", flush=True)
     levee_protected_areas = gpd.read_file(levee_protected_areas, mask=wbd_buffer)
     if not levee_protected_areas.empty:
         levee_protected_areas.to_file(
@@ -92,7 +92,7 @@ def subset_vector_layers(
     del levee_protected_areas
 
     # Find intersecting lakes and writeout
-    print("Subsetting NWM Lakes", flush=True)
+    print(f"Subsetting NWM Lakes for {hucCode}", flush=True)
     nwm_lakes = gpd.read_file(nwm_lakes, mask=wbd_buffer)
     nwm_lakes = nwm_lakes.loc[nwm_lakes.Shape_Area < 18990454000.0]
 
@@ -111,7 +111,7 @@ def subset_vector_layers(
     del nwm_lakes
 
     # Find intersecting levee lines
-    print("Subsetting NLD levee lines", flush=True)
+    print(f"Subsetting NLD levee lines for {hucCode}", flush=True)
     nld_lines = gpd.read_file(nld_lines, mask=wbd_buffer)
     if not nld_lines.empty:
         nld_lines.to_file(
@@ -131,7 +131,7 @@ def subset_vector_layers(
     del nld_lines_preprocessed
 
     # Subset NWM headwaters
-    print("Subsetting NWM Headwater Points", flush=True)
+    print(f"Subsetting NWM Headwater Points for {hucCode}", flush=True)
     nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer)
 
     if len(nwm_headwaters) > 0:
@@ -147,7 +147,7 @@ def subset_vector_layers(
     del nwm_headwaters
 
     # Find intersecting nwm_catchments
-    print("Subsetting NWM Catchments", flush=True)
+    print(f"Subsetting NWM Catchments for {hucCode}", flush=True)
     nwm_catchments = gpd.read_file(nwm_catchments, mask=wbd_buffer)
 
     if len(nwm_catchments) > 0:
@@ -163,7 +163,7 @@ def subset_vector_layers(
     del nwm_catchments
 
     # Subset nwm streams
-    print("Subsetting NWM Streams", flush=True)
+    print(f"Subsetting NWM Streams for {hucCode}", flush=True)
 
     nwm_streams = gpd.read_file(nwm_streams, mask=wbd_buffer)
 
diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
new file mode 100755
index 000000000..b0197de74
--- /dev/null
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -0,0 +1,406 @@
+#!/usr/bin/env python3
+
+import argparse
+import datetime as dt
+import logging
+import os
+import shutil
+import subprocess
+from multiprocessing import Pool
+
+from clip_vectors_to_wbd import subset_vector_layers
+from dotenv import load_dotenv
+
+
+'''
+    Overview:
+      This script was created to absolve run_unit_wb.sh from getting the huc level WBD layer, calling
+      clip_vectors_to_wbd.py, and clipping the WBD for every run, which added a significant amount of
+      processing time for each HUC8. Using this script, we generate the necessary pre-clipped .gpkg files
+      for the rest of the processing steps.
+
+      Read in environment variables from src/bash_variabls.env & config/params_template.env.
+      Parallelize the creation of .gpkg files per HUC:
+        Get huc level WBD layer from National, call the subset_vector_layers function, and clip the wbd.
+        A plethora gpkg files per huc are generated (see args to subset_vector_layers)
+        and placed within the output directory specified as the <outputs_dir> argument.
+
+    Usage:
+        generate_pre_clip_fim_huc8.py
+            -wbd /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
+            -n /data/inputs/pre_clip_huc8/24_3_20
+            -u /data/inputs/huc_lists/included_huc8.lst
+            -j 6
+            -o
+
+    Notes:
+      If running this script to generate new data, modify the pre_clip_huc_dir variable in
+      src/bash_variables.env to the corresponding outputs_dir argument after running and testing this script.
+      The newly generated data should be created in a new folder using the format <year_month_day>
+             (i.e. September 26, 2023 would be 23_9_26)
+'''
+
+srcDir = os.getenv('srcDir')
+projectDir = os.getenv('projectDir')
+
+load_dotenv(f'{srcDir}/bash_variables.env')
+load_dotenv(f'{projectDir}/config/params_template.env')
+
+# Variables from src/bash_variables.env
+DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
+inputsDir = os.getenv('inputsDir')
+input_WBD_gdb = os.getenv('input_WBD_gdb')
+input_DEM = os.getenv('input_DEM')
+input_DEM_domain = os.getenv('input_DEM_domain')
+input_nwm_lakes = os.getenv('input_nwm_lakes')
+input_nwm_catchments = os.getenv('input_nwm_catchments')
+input_NLD = os.getenv('input_NLD')
+input_levees_preprocessed = os.getenv('input_levees_preprocessed')
+input_GL_boundaries = os.getenv('input_GL_boundaries')
+input_nwm_flows = os.getenv('input_nwm_flows')
+input_nwm_headwaters = os.getenv('input_nwm_headwaters')
+input_nld_levee_protected_areas = os.getenv('input_nld_levee_protected_areas')
+
+# Variables from config/params_template.env
+wbd_buffer = os.getenv('wbd_buffer')
+wbd_buffer_int = int(wbd_buffer)
+
+
+def __setup_logger(outputs_dir):
+    '''
+    Set up logging to file. Since log file includes the date, it will be overwritten if this
+    script is run more than once on the same day.
+    '''
+    datetime_now = dt.datetime.now(dt.timezone.utc)
+    curr_date = datetime_now.strftime("%m_%d_%Y")
+
+    log_file_name = f"generate_pre_clip_fim_huc8_{curr_date}.log"
+
+    log_file_path = os.path.join(outputs_dir, log_file_name)
+
+    if not os.path.exists(outputs_dir):
+        os.mkdir(outputs_dir)
+
+    if os.path.exists(log_file_path):
+        os.remove(log_file_path)
+
+    file_handler = logging.FileHandler(log_file_path)
+    file_handler.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.INFO)
+
+    logger = logging.getLogger()
+    logger.addHandler(file_handler)
+    logger.setLevel(logging.INFO)
+
+    # Print start time
+    start_time_string = datetime_now.strftime("%m/%d/%Y %H:%M:%S")
+    logging.info('==========================================================================')
+    logging.info("\n generate_pre_clip_fim_huc8.py")
+    logging.info(f"\n \t Started: {start_time_string} \n")
+
+
+def pre_clip_hucs_from_wbd(wbd_file, outputs_dir, huc_list, number_of_jobs, overwrite):
+    '''
+    The function is the main driver of the program to iterate and parallelize writing
+    pre-clipped HUC8 vector files.
+
+    Inputs:
+    - wbd_file:                       Default take from src/bash_variables.env, or provided as argument.
+    - outputs_dir:                    Output directory to stage pre-clipped vectors.
+    - huc_list:                       List of Hucs to generate pre-clipped .gpkg files.
+    - number_of_jobs:                 Amount of cpus used for parallelization.
+    - overwrite:                      Overwrite existing HUC directories containing stage vectors.
+
+
+    Processing:
+    - Validate number_of_jobs based on cpus availabe on current system.
+    - Set up logging.
+    - Read in HUC list.
+    - If HUC level output directory is existant, delete it, if not, create a new directory.
+    - Parallelize the processing of .gpkg creation of the hucs_to_pre_clip_list using multiprocessing.Pool.
+        (call to huc_level_clip_vectors_to_wbd)
+
+    Outputs:
+    - New directory for each HUC, which contains 10-12 .gpkg files
+    - Write to log file in $pre_clip_huc_dir
+    '''
+
+    # Validation
+    total_cpus_available = os.cpu_count()
+    if number_of_jobs > total_cpus_available:
+        print(
+            f'Provided: -j {number_of_jobs}, which is greater than than amount of available cpus -2: '
+            f'{total_cpus_available -2} will be used instead.'
+        )
+        number_of_jobs = total_cpus_available - 2
+
+    # Set up logging and set start_time
+    __setup_logger(outputs_dir)
+    start_time = dt.datetime.now(dt.timezone.utc)
+
+    # Read in huc_list file and turn into a list data structure
+    if os.path.exists(huc_list):
+        hucs_to_pre_clip_list = open(huc_list).read().splitlines()
+    else:
+        logging.info("The huclist is not valid. Please check <huc_list> arguemnt.")
+        raise Exception("The huclist is not valid. Please check <huc_list> arguemnt.")
+
+    if os.path.exists(outputs_dir) and not overwrite:
+        raise Exception(
+            f"The directory: {outputs_dir} already exists. Use 'overwrite' argument if the intent"
+            " is to re-generate all of the data. "
+        )
+
+    # Iterate over the huc_list argument and create a directory for each huc.
+    for huc in hucs_to_pre_clip_list:
+        if os.path.isdir(os.path.join(outputs_dir, huc)):
+            shutil.rmtree(os.path.join(outputs_dir, huc))
+            os.mkdir(os.path.join(outputs_dir, huc))
+            logging.info(
+                f"\n\t Output Directory: {outputs_dir}/{huc} exists.  It will be overwritten, and the "
+                f"newly generated huc level files will be output there. \n"
+            )
+            print(
+                f"\n\t Output Directory: {outputs_dir}/{huc} exists.  It will be overwritten, and the "
+                f"newly generated huc level files will be output there. \n"
+            )
+
+        elif not os.path.isdir(os.path.join(outputs_dir, huc)):
+            os.mkdir(os.path.join(outputs_dir, huc))
+            logging.info(f"Created directory: {outputs_dir}/{huc}, huc level files will be written there.")
+            print(f"Created directory: {outputs_dir}/{huc}, huc level files will be written there.")
+
+    # Build arguments (procs_list) for each process to execute (huc_level_clip_vectors_to_wbd)
+    procs_list = []
+    for huc in hucs_to_pre_clip_list:
+        print(f"Generating vectors for {huc}. ")
+        procs_list.append([huc, outputs_dir, wbd_file])
+
+    # Parallelize each huc in hucs_to_parquet_list
+    logging.info('Parallelizing HUC level wbd pre-clip vector creation. ')
+    print('Parallelizing HUC level wbd pre-clip vector creation. ')
+    with Pool(processes=number_of_jobs) as pool:
+        pool.map(huc_level_clip_vectors_to_wbd, procs_list)
+
+    # Get time metrics
+    end_time = dt.datetime.now(dt.timezone.utc)
+    end_time_string = end_time.strftime("%m/%d/%Y %H:%M:%S")
+    logging.info(f"\n Ended: {end_time_string} \n")
+
+    # Calculate duration
+    time_duration = end_time - start_time
+    logging.info('==========================================================================')
+    logging.info(
+        f"\t Completed writing all huc level files \n"
+        f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}"
+    )
+    logging.info('==========================================================================')
+
+    print("\n\t Completed writing all huc level files \n")
+    print(f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}")
+
+
+def huc_level_clip_vectors_to_wbd(args):
+    '''
+    Create pre-clipped vectors at the huc level. Necessary to have this as an additional
+    function for multiprocessing. This is mostly a wrapper for the subset_vector_layers() method in
+    clip_vectors_to_wbd.py.
+
+    Inputs:
+    - huc:                           Individual HUC to generate vector files for.
+    - outputs_dir:                   Output directory to stage pre-clipped vectors.
+    - input_WBD_filename:            Filename of WBD to generate pre-clipped .gpkg files.
+
+    Processing:
+    - Define (unpack) arguments.
+    - Define LandSea water body mask.
+    - Use subprocess.run to get the WBD for specified HUC - (ogr2ogr called to generate wbd.gpkg)
+    - Subset Vector Layers - (call subset_vector_layers function in data/wbd/clip_vectors_to_wbd.py file)
+    - Clip WBD -
+        Creation of wbd8_clp.gpkg & wbd_buffered.gpkg using the executable: "ogr2ogr .... -clipsrc ..."
+
+    Outputs:
+    - .gpkg files* dependant on HUC's WBD (*differing amount based on individual huc)
+    '''
+
+    # We have to explicitly unpack the args from pool.map()
+    huc = args[0]
+    outputs_dir = args[1]
+    input_WBD_filename = args[2]
+
+    huc_directory = os.path.join(outputs_dir, huc)
+
+    # SET VARIABLES AND FILE INPUTS #
+    hucUnitLength = len(huc)
+    huc2Identifier = huc[:2]
+    input_NHD_WBHD_layer = f"WBDHU{hucUnitLength}"
+
+    # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
+    if huc2Identifier == "04":
+        input_LANDSEA = f"{input_GL_boundaries}"
+        print(f'Using {input_LANDSEA} for water body mask (Great Lakes)')
+    else:
+        input_LANDSEA = f"{inputsDir}/landsea/water_polygons_us.gpkg"
+
+    print(f"\n Get WBD {huc}")
+
+    # TODO: Use Python API (osgeo.ogr) instead of using ogr2ogr executable
+    get_wbd_subprocess = subprocess.run(
+        [
+            'ogr2ogr',
+            '-f',
+            'GPKG',
+            '-t_srs',
+            DEFAULT_FIM_PROJECTION_CRS,
+            f'{huc_directory}/wbd.gpkg',
+            input_WBD_filename,
+            input_NHD_WBHD_layer,
+            '-where',
+            f"HUC{hucUnitLength}='{huc}'",
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=True,
+        universal_newlines=True,
+    )
+
+    msg = get_wbd_subprocess.stdout
+    print(msg)
+    logging.info(msg)
+
+    if get_wbd_subprocess.stderr != "":
+        if "ERROR" in get_wbd_subprocess.stderr.upper():
+            msg = (
+                f" - Creating -- {huc_directory}/wbd.gpkg"
+                f"  ERROR -- details: ({get_wbd_subprocess.stderr})"
+            )
+            print(msg)
+            logging.error(msg)
+    else:
+        msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete \n"
+        print(msg)
+        logging.info(msg)
+
+    msg = f"Get Vector Layers and Subset {huc}"
+    print(msg)
+    logging.info(msg)
+
+    # Subset Vector Layers
+    subset_vector_layers(
+        subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
+        subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
+        hucCode=huc,
+        subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
+        wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
+        wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
+        wbd_filename=f"{huc_directory}/wbd.gpkg",
+        dem_filename=input_DEM,
+        dem_domain=input_DEM_domain,
+        nwm_lakes=input_nwm_lakes,
+        nwm_catchments=input_nwm_catchments,
+        subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
+        nld_lines=input_NLD,
+        nld_lines_preprocessed=input_levees_preprocessed,
+        landsea=input_LANDSEA,
+        nwm_streams=input_nwm_flows,
+        subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
+        nwm_headwaters=input_nwm_headwaters,
+        subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
+        subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
+        wbd_buffer_distance=wbd_buffer_int,
+        levee_protected_areas=input_nld_levee_protected_areas,
+        subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
+    )
+
+    msg = f"\n\t Completing Get Vector Layers and Subset: {huc} \n"
+    print(msg)
+    logging.info(msg)
+
+    ## Clip WBD8 ##
+    print(f" Clip WBD {huc}")
+
+    clip_wbd8_subprocess = subprocess.run(
+        [
+            'ogr2ogr',
+            '-f',
+            'GPKG',
+            '-t_srs',
+            DEFAULT_FIM_PROJECTION_CRS,
+            '-clipsrc',
+            f'{huc_directory}/wbd_buffered.gpkg',
+            f'{huc_directory}/wbd8_clp.gpkg',
+            f'{inputsDir}/wbd/WBD_National.gpkg',
+            'WBDHU8',
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=True,
+        universal_newlines=True,
+    )
+
+    msg = clip_wbd8_subprocess.stdout
+    print(msg)
+    logging.info(msg)
+
+    if clip_wbd8_subprocess.stderr != "":
+        if "ERROR" in clip_wbd8_subprocess.stderr.upper():
+            msg = (
+                f" - Creating -- {huc_directory}/wbd.gpkg"
+                f"  ERROR -- details: ({clip_wbd8_subprocess.stderr})"
+            )
+            print(msg)
+            logging.error(msg)
+    else:
+        msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete"
+        print(msg)
+        logging.info(msg)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='This script gets WBD layer, calls the clip_vectors_to_wbd.py script, and clips the wbd. '
+        'A plethora gpkg files per huc are generated (see args to subset_vector_layers), and placed within '
+        'the output directory specified as the <outputs_dir> argument.',
+        usage='''
+            ./generate_pre_clip_fim_huc8.py
+                -wbd /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
+                -n /data/inputs/pre_clip_huc8/24_3_20
+                -u /data/inputs/huc_lists/included_huc8.lst
+                -j 6
+                -o
+        ''',
+    )
+    parser.add_argument(
+        '-wbd',
+        '--wbd_file',
+        help='.wbd file to clip into individual HUC.gpkg files. Default is $input_WBD_gdb from src/bash_variables.env.',
+        default=input_WBD_gdb,
+    )
+    parser.add_argument(
+        '-n',
+        '--outputs_dir',
+        help='Directory to output all of the HUC level .gpkg files. Use the format: '
+        '<year_month_day> (i.e. September 26, 2023 would be 23_9_26)',
+    )
+    parser.add_argument('-u', '--huc_list', help='List of HUCs to genereate pre-clipped vectors for.')
+    parser.add_argument(
+        '-j',
+        '--number_of_jobs',
+        help='OPTIONAL: number of cores/processes (default=4). This is a memory intensive '
+        'script, and the multiprocessing will crash if too many CPUs are used. It is recommended to provide '
+        'half the amount of available CPUs.',
+        type=int,
+        required=False,
+        default=4,
+    )
+    parser.add_argument(
+        '-o',
+        '--overwrite',
+        help='Overwrite the file if already existing? (default false)',
+        action='store_true',
+    )
+
+    args = vars(parser.parse_args())
+
+    pre_clip_hucs_from_wbd(**args)
diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py
index ee8c9fbc2..5d1f903b3 100644
--- a/data/write_parquet_from_calib_pts.py
+++ b/data/write_parquet_from_calib_pts.py
@@ -190,7 +190,7 @@ def create_parquet_directory(output_dir):
         os.mkdir(output_dir, exist_ok=True)
         logging.info(f"Created directory: {output_dir}, .parquet files will be written there.")
     elif os.path.isdir(output_dir) is True:
-        logging.info(f"Output Direcrtory: {output_dir} exists, .parquet files will be written there.")
+        logging.info(f"Output Directory: {output_dir} exists, .parquet files will be written there.")
 
 
 def create_parquet_files(
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 254e6ccb5..2d0844de4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,7 +1,25 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-## v4.4.3.0 - 2023-10-13 - [PR#1005](https://github.com/NOAA-OWP/inundation-mapping/pull/1005)
+## v4.4.4.0 - 2023-10-20 - [PR#1012](https://github.com/NOAA-OWP/inundation-mapping/pull/1012)
+
+The way in which watershed boundary data (WBD) is generated and processed has been modified. Instead of generating those files "on the fly" for every run, a script has been added that will take a huclist and create the .gpkg files per HUC in a specified directory (`$pre_clip_huc_dir`).  During a `fim_pipeline.sh` run, the pre-clipped staged vectors will be copied over to the containers' working directory. This reduces runtime and the repetitive computation needed to generate those files every run.
+
+### Changes
+
+- `src/`
+    - `bash_variables.env`: Add pre_clip_huc_dir env variable. 
+    - `clip_vectors_to_wbd.py`: Moved to `/data/wbd/clip_vectors_to_wbd.py`.
+    - `src/run_unit_wb.sh`: Remove ogr2ogr calls to get & clip WBD, remove call to clip_vectors_to_wbd.py, and replace with copying staged .gpkg files. 
+
+### Additions
+
+- `data/wbd/`
+    - `generate_pre_clip_fim_huc8.py`: This script generates the pre-clipped vectors at the huc level.
+
+<br/><br/>
+
+## v4.4.3.0 - 2023-10-10 - [PR#1005](https://github.com/NOAA-OWP/inundation-mapping/pull/1005)
 
 Revise stream clipping to WBD by (1) reducing the buffer to clip streams away from the edge of the DEM (to prevent reverse flow issues) from 3 cells to 8 cells to account for the 70m AGREE buffer; (2) splitting MultiLineStrings formed by NWM streams being clipped by the DEM edge and then re-entering the DEM, and retaining only the lowest segment. Also changes the value of `input_WBD_gdb` to use the WBD clipped to the DEM domain.
 
diff --git a/src/bash_variables.env b/src/bash_variables.env
index db6430d4e..01a1cf4f3 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -14,6 +14,7 @@ export            input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwate
 export                 input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
 export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
 export          input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
+export                pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
 
 # Styling
 export startDiv="\n-----------------------------------------------------------------\n"
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 7baf258e0..13be08314 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -21,70 +21,11 @@ branchSummaryLogFile=$outputDestDir/logs/branch/"$hucNumber"_summary_branch.log
 T_total_start
 huc_start_time=`date +%s`
 
-## SET VARIABLES AND FILE INPUTS ##
-hucUnitLength=${#hucNumber}
-huc4Identifier=${hucNumber:0:4}
-huc2Identifier=${hucNumber:0:2}
-input_NHD_WBHD_layer=WBDHU$hucUnitLength
-
-# Define the landsea water body mask using either Great Lakes or Ocean polygon input #
-if [[ $huc2Identifier == "04" ]] ; then
-  input_LANDSEA=$input_GL_boundaries
-  #echo -e "Using $input_LANDSEA for water body mask (Great Lakes)"
-else
-  input_LANDSEA=$inputsDir/landsea/water_polygons_us.gpkg
-fi
 
-## GET WBD ##
-echo -e $startDiv"Get WBD $hucNumber"
-date -u
-Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS \
-    $tempHucDataDir/wbd.gpkg $input_WBD_gdb $input_NHD_WBHD_layer \
-    -where "HUC$hucUnitLength='$hucNumber'"
-Tcount
+## Copy HUC's pre-clipped .gpkg files from $pre_clip_huc_dir (use -a & /. -- only copies folder's contents)
+echo -e $startDiv"Copying staged wbd and .gpkg files from $pre_clip_huc_dir/$hucNumber"
+cp -a $pre_clip_huc_dir/$hucNumber/. $tempHucDataDir
 
-## Subset Vector Layers ##
-echo -e $startDiv"Get Vector Layers and Subset $hucNumber"
-date -u
-Tstart
-
-cmd_args=" -a $tempHucDataDir/nwm_lakes_proj_subset.gpkg"
-cmd_args+=" -b $tempHucDataDir/nwm_subset_streams.gpkg"
-cmd_args+=" -d $hucNumber"
-cmd_args+=" -e $tempHucDataDir/nwm_headwater_points_subset.gpkg"
-cmd_args+=" -f $tempHucDataDir/wbd_buffered.gpkg"
-cmd_args+=" -s $tempHucDataDir/wbd_buffered_streams.gpkg"
-cmd_args+=" -g $tempHucDataDir/wbd.gpkg"
-cmd_args+=" -i $input_DEM"
-cmd_args+=" -j $input_DEM_domain"
-cmd_args+=" -l $input_nwm_lakes"
-cmd_args+=" -m $input_nwm_catchments"
-cmd_args+=" -n $tempHucDataDir/nwm_catchments_proj_subset.gpkg"
-cmd_args+=" -r $input_NLD"
-cmd_args+=" -rp $input_levees_preprocessed"
-cmd_args+=" -v $input_LANDSEA"
-cmd_args+=" -w $input_nwm_flows"
-cmd_args+=" -x $tempHucDataDir/LandSea_subset.gpkg"
-cmd_args+=" -y $input_nwm_headwaters"
-cmd_args+=" -z $tempHucDataDir/nld_subset_levees.gpkg"
-cmd_args+=" -zp $tempHucDataDir/3d_nld_subset_levees_burned.gpkg"
-cmd_args+=" -wb $wbd_buffer"
-cmd_args+=" -lpf $input_nld_levee_protected_areas"
-cmd_args+=" -lps $tempHucDataDir/LeveeProtectedAreas_subset.gpkg"
-
-#echo "$cmd_args"
-python3 $srcDir/clip_vectors_to_wbd.py $cmd_args
-Tcount
-
-## Clip WBD8 ##
-echo -e $startDiv"Clip WBD8"
-date -u
-Tstart
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS \
-    -clipsrc $tempHucDataDir/wbd_buffered.gpkg $tempHucDataDir/wbd8_clp.gpkg \
-    $inputsDir/wbd/WBD_National.gpkg WBDHU8
-Tcount
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"

From c2a6b3bea972a52d3420dcde32940bb7afbbcd6f Mon Sep 17 00:00:00 2001
From: Gregory Petrochenkov
 <30809186+GregoryPetrochenkov-NOAA@users.noreply.github.com>
Date: Thu, 26 Oct 2023 07:56:21 -0700
Subject: [PATCH 284/359] v4.4.4.1 Synthesize testcases hotfix for GVAL
 memory/performance issues (#1007)

---
 Pipfile                          |    2 +-
 Pipfile.lock                     | 1011 ++++++++++++++++--------------
 docs/CHANGELOG.md                |   15 +
 pyproject.toml                   |    3 +-
 tools/inundate_mosaic_wrapper.py |    4 +-
 tools/run_test_case.py           |  181 +++---
 tools/synthesize_test_cases.py   |   24 +-
 tools/tools_shared_functions.py  |  283 +++++----
 8 files changed, 802 insertions(+), 721 deletions(-)

diff --git a/Pipfile b/Pipfile
index 3a4e3c353..636fbe26c 100755
--- a/Pipfile
+++ b/Pipfile
@@ -37,7 +37,7 @@ pyarrow = "==11.0.0"
 rtree = "==1.0.1"
 py7zr = "==0.20.4"
 scipy = "==1.10.1"
-gval = "==0.1.2"
+gval = "==0.2.3"
 flake8 = "==6.0.0"
 black = "==23.7.0"
 pyproject-flake8 = "==6.0.0.post1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 4e66c9db4..78c257106 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "32dc0ff07bc71eeec7a7bc1c64d580d05ce81cceaa3ecfc2e3b4f067e4f5c396"
+            "sha256": "c05e13cee1824291db432c5f40d2ed7eed74349a1bba77ddb2deb852cd533734"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -115,11 +115,11 @@
         },
         "babel": {
             "hashes": [
-                "sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610",
-                "sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455"
+                "sha256:04c3e2d28d2b7681644508f836be388ae49e0cfe91465095340395b60d00f210",
+                "sha256:fbfcae1575ff78e26c7449136f1abbefc3c13ce542eeb13d43d50d8b047216ec"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.12.1"
+            "version": "==2.13.0"
         },
         "backcall": {
             "hashes": [
@@ -167,11 +167,11 @@
         },
         "bleach": {
             "hashes": [
-                "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414",
-                "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"
+                "sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe",
+                "sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==6.0.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==6.1.0"
         },
         "blosc2": {
             "hashes": [
@@ -532,16 +532,16 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
             "hashes": [
-                "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f",
-                "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"
+                "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7",
+                "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.2.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==3.0.0"
         },
         "colorcet": {
             "hashes": [
@@ -627,75 +627,75 @@
         },
         "cycler": {
             "hashes": [
-                "sha256:7896994252d006771357777d0251f3e34d266f4fa5f2c572247a80ab01440947",
-                "sha256:8cc3a7b4861f91b1095157f9916f748549a617046e67eb7619abed9b34d2c94a"
+                "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30",
+                "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.12.0"
+            "version": "==0.12.1"
         },
         "cython": {
             "hashes": [
-                "sha256:011c4e0b75baee1843334562487eb4fbc0c59ddb2cc32a978b972a81eedcbdcc",
-                "sha256:05cb2a73810f045d328b7579cf98f550a9e601df5e282d1fea0512d8ad589011",
-                "sha256:067b2b9eb487bd61367b296f11b7c1c70a084b3eb7d5a572f607cd1fc5ca5586",
-                "sha256:08d67c7225a09eeb77e090c8d4f60677165b052ccf76e3a57d8237064e5c2de2",
-                "sha256:10cbfb37f31938371a6213cc8b5459c639954aed053efeded3c012d4c5915db9",
-                "sha256:147cc1d3dda8b06de9d86df5e59cdf15f0a522620168b7349a5ec88b48104d7d",
-                "sha256:1b12a8f23270675b537d1c3b988f845bea4bbcc66ae0468857f5ede0526d4522",
-                "sha256:213ff9f95de319e54b520bf31edd6aa7a1fa4fbf617c2beb0f92362595e6476a",
-                "sha256:22ba78e48bdb65977928ecb275ac8c82df7b0eefa075078a1363a5af4606b42e",
-                "sha256:2f84bd6cefa5130750c492038170c44f1cbd6f42e9ed85e168fd9cb453f85160",
-                "sha256:302281b927409b3e0ef8cd9251eab782cf1acd2578eab305519fbae5d184b7e9",
-                "sha256:34f7b014ebce5d325c8084e396c81cdafbd8d82be56780dffe6b67b28c891f1b",
-                "sha256:38085523fa7a299638d051ae08144222785639882f6291bd275c0b12db1034ff",
-                "sha256:477cd3549597f09a1608da7b05e16ba641e9aedd171b868533a5a07790ed886f",
-                "sha256:4bebbca13078125a35937966137af4bd0300a0c66fd7ae4ce36adc049b13bdf3",
-                "sha256:4db017b104f47b1185237702f6ed2651839c8124614683efa7c489f3fa4e19d9",
-                "sha256:4dca13c86d6cd523c7d8bbf8db1b2bbf8faedd0addedb229158d8015ad1819e1",
-                "sha256:4fe806d154b6b7f0ab746dac36c022889e2e7cf47546ff9afdc29a62cfa692d0",
-                "sha256:54d41a1dfbaab74449873e7f8e6cd4239850fe7a50f7f784dd99a560927f3bac",
-                "sha256:550b3fbe9b3c555b44ded934f4822f9fcc04dfcee512167ebcbbd370ccede20e",
-                "sha256:5682293d344b7dbad97ce6eceb9e887aca6e53499709db9da726ca3424e5559d",
-                "sha256:62dd78afdf748a58dae9c9b9c42a1519ae30787b28ce5f84a0e1bb54144142ca",
-                "sha256:6b75e9c9d7ad7c9dd85d45241d1d4e3c5f66079c1f84eec91689c26d98bc3349",
-                "sha256:75a2395cc7b78cff59be6e9b7f92bbb5d7b8d25203f6d3fb6f72bdb7d3f49777",
-                "sha256:77ec0134fc1b10aebef2013936a91c07bff2498ec283bc2eca099ee0cb94d12e",
-                "sha256:786b6034a91e886116bb562fe42f8bf0f97c3e00c02e56791d02675959ed65b1",
-                "sha256:78e2853d484643c6b7ac3bdb48392753442da1c71b689468fa3176b619bebe54",
-                "sha256:7e08ff5da5f5b969639784b1bffcd880a0c0f048d182aed7cba9945ee8b367c2",
-                "sha256:7f43c4d3ecd9e3b8b7afe834e519f55cf4249b1088f96d11b96f02c55cbaeff7",
-                "sha256:809617cf4825b2138ce0ec827e1f28e39668743c81ac8286373f8d148c05f088",
-                "sha256:832bbee87bca760efeae248ddf19ccd77f9a2355cb6f8a64f20cc377e56957b3",
-                "sha256:8850269ff59f77a1629e26d0576701925360d732011d6d3516ccdc5b2c2bc310",
-                "sha256:8948504338d7a140ce588333177dcabf0743a68dbc83b0174f214f5b959634d5",
-                "sha256:8ccb91d2254e34724f1541b2a6fcdfacdb88284185b0097ae84e0ddf476c7a38",
-                "sha256:8f1c9e4b8e413da211dd7942440cf410ff0eafb081309e04e81f4fafbb146bf2",
-                "sha256:9594818dca8bb22ae6580c5222da2bc5cc32334350bd2d294a00d8669bcc61b5",
-                "sha256:989787fc24a95100a26918b6577d06e15a8868a3ed267009c5cfcf1a906179ac",
-                "sha256:9e625eec8c5c9a8cb062a318b257cc469d301bed952c7daf86e38bbd3afe7c91",
-                "sha256:a1c3675394b81024aaf56e4f53c2b4f81d9a116c7049e9d4706f810899c9134e",
-                "sha256:a49dde9f9e29ea82f29aaf3bb1a270b6eb90b75d627c7ff2f5dd3764540ae646",
-                "sha256:a51efba0e136b2af358e5a347bae09678b17460c35cf1eab24f0476820348991",
-                "sha256:ae453cfa933b919c0a19d2cc5dc9fb28486268e95dc2ab7a11ab7f99cf8c3883",
-                "sha256:b032cb0c69082f0665b2c5fb416d041157062f1538336d0edf823b9ee500e39c",
-                "sha256:b1f023d36a3829069ed11017c670128be3f135a9c17bd64c35d3b3442243b05c",
-                "sha256:bc1c8013fad0933f5201186eccc5f2be223cafd6a8dcd586d3f7bb6ba84dc845",
-                "sha256:c298b1589205ecaaed0457ad05e0c8a43e7db2053607f48ed4a899cb6aa114df",
-                "sha256:c5e722732e9aa9bde667ed6d87525234823eb7766ca234cfb19d7e0c095a2ef4",
-                "sha256:c90eeb94395315e65fd758a2f86b92904fce7b50060b4d45a878ef6767f9276e",
-                "sha256:cc9d173ab8b167cae674f6deed8c65ba816574797a2bd6d8aa623277d1fa81ca",
-                "sha256:d0d0cc4ecc05f41c5e02af14ac0083552d22efed976f79eb7bade55fed63b25d",
-                "sha256:d21801981db44b7e9f9768f121317946461d56b51de1e6eff3c42e8914048696",
-                "sha256:d5e5587128e8c2423aefcffa4ded4ddf60d44898938fbb7c0f236636a750a94f",
-                "sha256:dab6a923e21e212aa3dc6dde9b22a190f5d7c449315a94e57ddc019ea74a979b",
-                "sha256:dd30826ca8b27b2955a63c8ffe8aacc9f0779582b4bd154cf7b441ac10dae2cb",
-                "sha256:e486331a29e7700b1ad5f4f753bef483c81412a5e64a873df46d6cb66f9a65de",
-                "sha256:e663c237579c033deaa2cb362b74651da7712f56e441c11382510a8c4c4f2dd7",
-                "sha256:e825e682cef76d0c33384f38b56b7e87c76152482a914dfc78faed6ff66ce05a",
-                "sha256:f37e4287f520f3748a06ad5eaae09ba4ac68f52e155d70de5f75780d83575c43"
+                "sha256:0147a31fb73a063bb7b6c69fd843c1a2bad18f326f58048d4ee5bdaef87c9fbf",
+                "sha256:056340c49bf7861eb1eba941423e67620b7c85e264e9a5594163f1d1e8b95acc",
+                "sha256:0630527a8c9e8fed815c38524e418dab713f5d66f6ac9dc2151b41f3a7727304",
+                "sha256:0e1c9385e99eef299396b9a1e39790e81819446c6a83e249f6f0fc71a64f57a0",
+                "sha256:14b898ec2fdeea68f81bd3838b035800b173b59ed532674f65a82724bab35d3b",
+                "sha256:1571b045ec1cb15c152c3949f3bd53ee0fa66d434271ea3d225658d99b7e721a",
+                "sha256:176953a8a2532e34a589625a40c934ff339088f2bf4ddaa2e5cb77b05ca0c25c",
+                "sha256:188705eeae094bb716bc3e3d0da4e13469f0a0de803b65dfd63fe7eb78ec6173",
+                "sha256:18f4fb7cc6ad8e99e8f387ebbcded171a701bfbfd8cd3fd46156bf44bb4fd968",
+                "sha256:1d3416c24a1b7bf3a2d9615a7f9f12b00fac0b94fb2e61449e0c1ecf20d6ed52",
+                "sha256:1ec9e15b821ef7e3c38abe9e4df4e6dda7af159325bc358afd5a3c2d5027ccfe",
+                "sha256:22d268c3023f405e13aa0c1600389794694ab3671614f8e782d89a1055da0858",
+                "sha256:30c1d9bd2bcb9b1a195dd23b359771857df8ebd4a1038fb37dd155d3ea38c09c",
+                "sha256:3192cd780435fca5ae5d79006b48cbf0ea674853b5a7b0055a122045bff9d84e",
+                "sha256:327309301b01f729f173a94511cb2280c87ba03c89ed428e88f913f778245030",
+                "sha256:3cfbd60137f6fca9c29101d7517d4e341e0fd279ffc2489634e5e2dd592457c2",
+                "sha256:3db04801fd15d826174f63ff45878d4b1e62aff27cf1ea96b186581052d24446",
+                "sha256:42b1ff0e19fb4d1fe68b60f55d46942ed246a323f6bbeec302924b78b4c3b637",
+                "sha256:48bae87b657009e5648c21d4a92de9f3dc6fed3e35e92957fa8a07a18cea2313",
+                "sha256:4c8e5afcc19861c3b22faafbe906c7e1b23f0595073ac10e21a80dec9e60e7dd",
+                "sha256:4cc0f7244da06fdc6a4a7240df788805436b6fb7f20edee777eb77777d9d2eb1",
+                "sha256:4e956383e57d00b1fa6449b5ec03b9fa5fce2afd41ef3e518bee8e7c89f1616c",
+                "sha256:51850f277660f67171135515e45edfc8815f723ff20768e39cb9785b2671062f",
+                "sha256:5545d20d7a1c0cf17559152f7f4a465c3d5caace82dd051f82e2d753ae9fd956",
+                "sha256:587d664ff6bd5b03611ddc6ef320b7f8677d824c45d15553f16a69191a643843",
+                "sha256:59bf689409b0e51ef673e3dd0348727aef5b67e40f23f806be64c49cee321de0",
+                "sha256:5d6af87a787d5ce063e28e508fee34755a945e438c68ecda50eb4ea34c30e13f",
+                "sha256:66b7e71c16cab0814945014ffb101ead2b173259098bbb1b8138e7a547da3709",
+                "sha256:6ccde14ddc4b424435cb5722aa1529c254bbf3611e1ad9baea12d25e9c049361",
+                "sha256:6f05889eb1b5a95a7adf97303279c2d13819ff62292e10337e6c940dbf570b5d",
+                "sha256:6f63e959d13775472d37e731b2450d120e8db87e956e2de74475e8f17a89b1fb",
+                "sha256:74ba0f11b384246b7965169f08bf67d426e4957fee5c165571340217a9b43cfc",
+                "sha256:75d42c8423ab299396f3c938445730600e32e4a2f0298f6f9df4d4a698fe8e16",
+                "sha256:77a920ae19fa1db5adb8a618cebb095ca4f56adfbf9fc32cb7008a590607b62b",
+                "sha256:80bd3167e689419cdaf7ede0d20a9f126b9698a43b1f8d3e8f54b970c7a6cd07",
+                "sha256:84084fa05cf9a67a85818fa72a741d1cae2e3096551158730730a3bafc3b2f52",
+                "sha256:845e24ee70c204062e03f813114751387abf454b29410336797582e04abbc07b",
+                "sha256:85073ab414ff432d2a39d36cb49c39ce69f30b53daccc7699bfad0ce3d1b539a",
+                "sha256:8a6a9a2d98758768052e4ac1bea4ebc20fae69b4c19cb2bc5457c9174532d302",
+                "sha256:8e78fc42a6e846941d23aba1aca587520ad38c8970255242f08f9288b0eeba85",
+                "sha256:9296f332523d5c550ebae694483874d255264cff3281372f25ea5f2739b96651",
+                "sha256:94fa403de3a413cd41b8eb4ddb4adcbd66aa0a64f9a84d1c5f696c93572c83aa",
+                "sha256:9eb128fa40305f18eaa4d8dd0980033b92db86aada927181d3c3d561aa0634db",
+                "sha256:9f40b27545d583fd7df0d3c1b76b3bcaf8a72dbd8d83d5486af2384015660de8",
+                "sha256:b50f4f75f89e7eef2ed9c9b60746bc4ab1ba2bc0dff64587133db2b63e068f09",
+                "sha256:be1a679c7ad90813f9206c9d62993f3bd0cba9330668e97bb3f70c87ae94d5f5",
+                "sha256:bff1fec968a6b2ca452ae9bff6d6d0bf8486427d4d791e85543240266b6915e0",
+                "sha256:d0c7b315f6feb75e2c949dc7816da5626cdca097fea1c0d9f4fdb20d2f4ffc2a",
+                "sha256:d12591939af93c59defea6fc5320ca099eb44e4694e3b2cbe72fb24406079b97",
+                "sha256:d49d20db27c9cfcf45bb1fbf68f777bd1e04e4b949e4e5172d9ee8c9419bc792",
+                "sha256:d52ed47edbf48392dd0f419135e7ff59673f6b32d27d3ffc9e61a515571c050d",
+                "sha256:db9d4de4cd6cd3ad1c3f455aae877ad81a92b92b7cbb01dfb32b6306b873932b",
+                "sha256:e3ad109bdf40f55318e001cad12bcc00e8119569b49f72e442c082355617b036",
+                "sha256:e40ac8bd6d11355d354bb4975bb88f6e923ba30f85e38f1f1234b642634e4fc4",
+                "sha256:e729fd633a5225570c5480b36e7c530c8a82e2ab6d2944ddbe1ddfff5bf181b1",
+                "sha256:f0ac9ec822fad010248b4a59ac197975de38c95378d0f13201c181dd9b0a2624",
+                "sha256:f7578b59ffd0d9c95ae6f7ae852309918915998b7fe0ed2f8725a683de8da276",
+                "sha256:fa08259f4d176b86561eeff6954f9924099c0b0c128fc2cbfc18343c068ad8ca"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.2"
+            "version": "==3.0.3"
         },
         "dask": {
             "hashes": [
@@ -838,53 +838,61 @@
             "markers": "python_full_version >= '3.8.1'",
             "version": "==6.0.0"
         },
+        "flox": {
+            "hashes": [
+                "sha256:44b0ce5aea13485126cb69bf4ab1aad0cfe131ddadc577f1f244d48d856fd08d",
+                "sha256:d4f34a9debbde67c9ae5e9b2bef6d5c236879be8b18b48fb1eb3bf71aa154236"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.7.2"
+        },
         "fonttools": {
             "hashes": [
-                "sha256:030355fbb0cea59cf75d076d04d3852900583d1258574ff2d7d719abf4513836",
-                "sha256:05056a8c9af048381fdb17e89b17d45f6c8394176d01e8c6fef5ac96ea950d38",
-                "sha256:206808f9717c9b19117f461246372a2c160fa12b9b8dbdfb904ab50ca235ba0a",
-                "sha256:20fc43783c432862071fa76da6fa714902ae587bc68441e12ff4099b94b1fcef",
-                "sha256:25620b738d4533cfc21fd2a4f4b667e481f7cb60e86b609799f7d98af657854e",
-                "sha256:33c40a657fb87ff83185828c0323032d63a4df1279d5c1c38e21f3ec56327803",
-                "sha256:3d7adfa342e6b3a2b36960981f23f480969f833d565a4eba259c2e6f59d2674f",
-                "sha256:48078357984214ccd22d7fe0340cd6ff7286b2f74f173603a1a9a40b5dc25afe",
-                "sha256:5056f69a18f3f28ab5283202d1efcfe011585d31de09d8560f91c6c88f041e92",
-                "sha256:52e77f23a9c059f8be01a07300ba4c4d23dc271d33eed502aea5a01ab5d2f4c1",
-                "sha256:57c22e5f9f53630d458830f710424dce4f43c5f0d95cb3368c0f5178541e4db7",
-                "sha256:5aa67d1e720fdd902fde4a59d0880854ae9f19fc958f3e1538bceb36f7f4dc92",
-                "sha256:5f9660e70a2430780e23830476332bc3391c3c8694769e2c0032a5038702a662",
-                "sha256:635658464dccff6fa5c3b43fe8f818ae2c386ee6a9e1abc27359d1e255528186",
-                "sha256:6a530fa28c155538d32214eafa0964989098a662bd63e91e790e6a7a4e9c02da",
-                "sha256:70f021a6b9eb10dfe7a411b78e63a503a06955dd6d2a4e130906d8760474f77c",
-                "sha256:77e5113233a2df07af9dbf493468ce526784c3b179c0e8b9c7838ced37c98b69",
-                "sha256:7c76f32051159f8284f1a5f5b605152b5a530736fb8b55b09957db38dcae5348",
-                "sha256:812142a0e53cc853964d487e6b40963df62f522b1b571e19d1ff8467d7880ceb",
-                "sha256:82d8e687a42799df5325e7ee12977b74738f34bf7fde1c296f8140efd699a213",
-                "sha256:8dfd8edfce34ad135bd69de20c77449c06e2c92b38f2a8358d0987737f82b49e",
-                "sha256:93c5b6d77baf28f306bc13fa987b0b13edca6a39dc2324eaca299a74ccc6316f",
-                "sha256:9d654d3e780e0ceabb1f4eff5a3c042c67d4428d0fe1ea3afd238a721cf171b3",
-                "sha256:a682fb5cbf8837d1822b80acc0be5ff2ea0c49ca836e468a21ffd388ef280fd3",
-                "sha256:a68b71adc3b3a90346e4ac92f0a69ab9caeba391f3b04ab6f1e98f2c8ebe88e3",
-                "sha256:a6a2e99bb9ea51e0974bbe71768df42c6dd189308c22f3f00560c3341b345646",
-                "sha256:ab80e7d6bb01316d5fc8161a2660ca2e9e597d0880db4927bc866c76474472ef",
-                "sha256:ace0fd5afb79849f599f76af5c6aa5e865bd042c811e4e047bbaa7752cc26126",
-                "sha256:ace51902ab67ef5fe225e8b361039e996db153e467e24a28d35f74849b37b7ce",
-                "sha256:af38f5145258e9866da5881580507e6d17ff7756beef175d13213a43a84244e9",
-                "sha256:b3813f57f85bbc0e4011a0e1e9211f9ee52f87f402e41dc05bc5135f03fa51c1",
-                "sha256:b5e760198f0b87e42478bb35a6eae385c636208f6f0d413e100b9c9c5efafb6a",
-                "sha256:b62a53a4ca83c32c6b78cac64464f88d02929779373c716f738af6968c8c821e",
-                "sha256:d08a694b280d615460563a6b4e2afb0b1b9df708c799ec212bf966652b94fc84",
-                "sha256:d27d960e10cf7617d70cf3104c32a69b008dde56f2d55a9bed4ba6e3df611544",
-                "sha256:da78f39b601ed0b4262929403186d65cf7a016f91ff349ab18fdc5a7080af465",
-                "sha256:dcc01cea0a121fb0c009993497bad93cae25e77db7dee5345fec9cce1aaa09cd",
-                "sha256:e3f8acc6ef4a627394021246e099faee4b343afd3ffe2e517d8195b4ebf20289",
-                "sha256:e4bc589d8da09267c7c4ceaaaa4fc01a7908ac5b43b286ac9279afe76407c384",
-                "sha256:e5d53eddaf436fa131042f44a76ea1ead0a17c354ab9de0d80e818f0cb1629f1",
-                "sha256:ee728d5af70f117581712966a21e2e07031e92c687ef1fdc457ac8d281016f64",
-                "sha256:f19c2b1c65d57cbea25cabb80941fea3fbf2625ff0cdcae8900b5fb1c145704f"
+                "sha256:10003ebd81fec0192c889e63a9c8c63f88c7d72ae0460b7ba0cd2a1db246e5ad",
+                "sha256:10b3922875ffcba636674f406f9ab9a559564fdbaa253d66222019d569db869c",
+                "sha256:13a9a185259ed144def3682f74fdcf6596f2294e56fe62dfd2be736674500dba",
+                "sha256:17dbc2eeafb38d5d0e865dcce16e313c58265a6d2d20081c435f84dc5a9d8212",
+                "sha256:18a2477c62a728f4d6e88c45ee9ee0229405e7267d7d79ce1f5ce0f3e9f8ab86",
+                "sha256:18eefac1b247049a3a44bcd6e8c8fd8b97f3cad6f728173b5d81dced12d6c477",
+                "sha256:1952c89a45caceedf2ab2506d9a95756e12b235c7182a7a0fff4f5e52227204f",
+                "sha256:1cf9e974f63b1080b1d2686180fc1fbfd3bfcfa3e1128695b5de337eb9075cef",
+                "sha256:1e09da7e8519e336239fbd375156488a4c4945f11c4c5792ee086dd84f784d02",
+                "sha256:2062542a7565091cea4cc14dd99feff473268b5b8afdee564f7067dd9fff5860",
+                "sha256:25d3da8a01442cbc1106490eddb6d31d7dffb38c1edbfabbcc8db371b3386d72",
+                "sha256:34f713dad41aa21c637b4e04fe507c36b986a40f7179dcc86402237e2d39dcd3",
+                "sha256:360201d46165fc0753229afe785900bc9596ee6974833124f4e5e9f98d0f592b",
+                "sha256:3b7ad05b2beeebafb86aa01982e9768d61c2232f16470f9d0d8e385798e37184",
+                "sha256:4c54466f642d2116686268c3e5f35ebb10e49b0d48d41a847f0e171c785f7ac7",
+                "sha256:4d9740e3783c748521e77d3c397dc0662062c88fd93600a3c2087d3d627cd5e5",
+                "sha256:4f88cae635bfe4bbbdc29d479a297bb525a94889184bb69fa9560c2d4834ddb9",
+                "sha256:51669b60ee2a4ad6c7fc17539a43ffffc8ef69fd5dbed186a38a79c0ac1f5db7",
+                "sha256:5db46659cfe4e321158de74c6f71617e65dc92e54980086823a207f1c1c0e24b",
+                "sha256:5f37e31291bf99a63328668bb83b0669f2688f329c4c0d80643acee6e63cd933",
+                "sha256:6bb5ea9076e0e39defa2c325fc086593ae582088e91c0746bee7a5a197be3da0",
+                "sha256:748015d6f28f704e7d95cd3c808b483c5fb87fd3eefe172a9da54746ad56bfb6",
+                "sha256:7bbbf8174501285049e64d174e29f9578495e1b3b16c07c31910d55ad57683d8",
+                "sha256:884ef38a5a2fd47b0c1291647b15f4e88b9de5338ffa24ee52c77d52b4dfd09c",
+                "sha256:8da417431bfc9885a505e86ba706f03f598c85f5a9c54f67d63e84b9948ce590",
+                "sha256:95e974d70238fc2be5f444fa91f6347191d0e914d5d8ae002c9aa189572cc215",
+                "sha256:9648518ef687ba818db3fcc5d9aae27a369253ac09a81ed25c3867e8657a0680",
+                "sha256:9a2f0aa6ca7c9bc1058a9d0b35483d4216e0c1bbe3962bc62ce112749954c7b8",
+                "sha256:9c36da88422e0270fbc7fd959dc9749d31a958506c1d000e16703c2fce43e3d0",
+                "sha256:9c60ecfa62839f7184f741d0509b5c039d391c3aff71dc5bc57b87cc305cff3b",
+                "sha256:9f727c3e3d08fd25352ed76cc3cb61486f8ed3f46109edf39e5a60fc9fecf6ca",
+                "sha256:a7a06f8d95b7496e53af80d974d63516ffb263a468e614978f3899a6df52d4b3",
+                "sha256:ad0b3f6342cfa14be996971ea2b28b125ad681c6277c4cd0fbdb50340220dfb6",
+                "sha256:b2adca1b46d69dce4a37eecc096fe01a65d81a2f5c13b25ad54d5430ae430b13",
+                "sha256:b84a1c00f832feb9d0585ca8432fba104c819e42ff685fcce83537e2e7e91204",
+                "sha256:bb6d2f8ef81ea076877d76acfb6f9534a9c5f31dc94ba70ad001267ac3a8e56f",
+                "sha256:bf11e2cca121df35e295bd34b309046c29476ee739753bc6bc9d5050de319273",
+                "sha256:d21099b411e2006d3c3e1f9aaf339e12037dbf7bf9337faf0e93ec915991f43b",
+                "sha256:d4071bd1c183b8d0b368cc9ed3c07a0f6eb1bdfc4941c4c024c49a35429ac7cd",
+                "sha256:e117a92b07407a061cde48158c03587ab97e74e7d73cb65e6aadb17af191162a",
+                "sha256:f7a58eb5e736d7cf198eee94844b81c9573102ae5989ebcaa1d1a37acd04b33d",
+                "sha256:fe9b1ec799b6086460a7480e0f55c447b1aca0a4eecc53e444f639e967348896"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.43.0"
+            "version": "==4.43.1"
         },
         "fqdn": {
             "hashes": [
@@ -936,12 +944,12 @@
         },
         "gval": {
             "hashes": [
-                "sha256:4074fce7cb2cda26f5cc610b88da0d9d4c9559d52d75074154948b1dbe4b9bac",
-                "sha256:880d0bb3cdb1d366d9e18423d4478b74f72010d120a2b53cb53603163c790bc0"
+                "sha256:d1a560f9e8b9b7727c618a56d7a04f3621ceb71477f18410c811a6dc0bd9b05d",
+                "sha256:d9f26aa88f164b77a47cb7d7eafc1641e48abe4152501657cb173c44dc84a8fd"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.1.2"
+            "version": "==0.2.3"
         },
         "identify": {
             "hashes": [
@@ -1047,7 +1055,7 @@
                 "sha256:fd04764d0bb830414788cae897d082bf6ad92324e571a5511bd7e1de4a0cdc67",
                 "sha256:fde3f85864c84badb26f42d95639360e627fd09c529a76c46a06dbd7a5735c51"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version > '3.6'",
             "version": "==0.3.1"
         },
         "iniconfig": {
@@ -1160,9 +1168,6 @@
             "version": "==2.4"
         },
         "jsonschema": {
-            "extras": [
-                "format-nongpl"
-            ],
             "hashes": [
                 "sha256:cd5f1f9ed9444e554b38ba003af06c0a8c2868131e56bfbef0550fb450c0330e",
                 "sha256:ec84cc37cfa703ef7cd4928db24f9cb31428a5d0fa77747b8b51a847458e0bbf"
@@ -1205,27 +1210,27 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:0c28db6cbe2c37b5b398e1a1a5b22f84fd64cd10afc1f6c05b02fb09481ba45f",
-                "sha256:a4af53c3fa3f6330cebb0d9f658e148725d15652811d1c32dc0f63bb96f2e6d6"
+                "sha256:66e252f675ac04dcf2feb6ed4afb3cd7f68cf92f483607522dc251f32d471571",
+                "sha256:e4b98344bb94ee2e3e6c4519a97d001656009f9cb2b7f2baf15b3c205770011d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.3.2"
+            "version": "==5.4.0"
         },
         "jupyter-events": {
             "hashes": [
-                "sha256:4753da434c13a37c3f3c89b500afa0c0a6241633441421f6adafe2fb2e2b924e",
-                "sha256:7be27f54b8388c03eefea123a4f79247c5b9381c49fb1cd48615ee191eb12615"
+                "sha256:81f07375c7673ff298bfb9302b4a981864ec64edaed75ca0fe6f850b9b045525",
+                "sha256:fda08f0defce5e16930542ce60634ba48e010830d50073c3dfd235759cee77bf"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.7.0"
+            "version": "==0.8.0"
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:8e4b90380b59d7a1e31086c4692231f2a2ea4cb269f5516e60aba72ce8317fc9",
-                "sha256:d4916c8581c4ebbc534cebdaa8eca2478d9f3bfdd88eae29fcab0120eac57649"
+                "sha256:b11e2ba80667c75f55630faf8ac3d5809f8734f9006d65cce117c46a0a516ab8",
+                "sha256:c57270faa6530393ae69783a2d2f1874c718b9f109080581ea076b05713249fa"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.7.3"
+            "version": "==2.8.0"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1733,11 +1738,11 @@
         },
         "nbconvert": {
             "hashes": [
-                "sha256:aec605e051fa682ccc7934ccc338ba1e8b626cfadbab0db592106b630f63f0f2",
-                "sha256:f5bc15a1247e14dd41ceef0c0a3bc70020e016576eb0578da62f1c5b4f950479"
+                "sha256:39fe4b8bdd1b0104fdd86fc8a43a9077ba64c720bda4c6132690d917a0a154ee",
+                "sha256:e56cc7588acc4f93e2bb5a34ec69028e4941797b2bfaf6462f18a41d1cc258c9"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.8.0"
+            "version": "==7.9.2"
         },
         "nbformat": {
             "hashes": [
@@ -1919,6 +1924,12 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.23.5"
         },
+        "numpy-groupies": {
+            "hashes": [
+                "sha256:fd50026552280ca717722f17d4231390667505bb572de564de1362b40487d34a"
+            ],
+            "version": "==0.9.22"
+        },
         "odc-geo": {
             "hashes": [
                 "sha256:0b04d0835e783685f128453e4b4169d86d322632887bfee9c7dbde364a3c324d",
@@ -2040,63 +2051,63 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff",
-                "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f",
-                "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21",
-                "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635",
-                "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a",
-                "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f",
-                "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1",
-                "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d",
-                "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db",
-                "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849",
-                "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7",
-                "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876",
-                "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3",
-                "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317",
-                "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91",
-                "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d",
-                "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b",
-                "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd",
-                "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed",
-                "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500",
-                "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7",
-                "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a",
-                "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a",
-                "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0",
-                "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf",
-                "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f",
-                "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1",
-                "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088",
-                "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971",
-                "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a",
-                "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205",
-                "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54",
-                "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08",
-                "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21",
-                "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d",
-                "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08",
-                "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e",
-                "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf",
-                "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b",
-                "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145",
-                "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2",
-                "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d",
-                "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d",
-                "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf",
-                "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad",
-                "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d",
-                "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1",
-                "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4",
-                "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2",
-                "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19",
-                "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37",
-                "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4",
-                "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68",
-                "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"
+                "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d",
+                "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de",
+                "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616",
+                "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839",
+                "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099",
+                "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a",
+                "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219",
+                "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106",
+                "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b",
+                "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412",
+                "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b",
+                "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7",
+                "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2",
+                "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7",
+                "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14",
+                "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f",
+                "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27",
+                "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57",
+                "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262",
+                "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28",
+                "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610",
+                "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172",
+                "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273",
+                "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e",
+                "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d",
+                "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818",
+                "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f",
+                "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9",
+                "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01",
+                "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7",
+                "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651",
+                "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312",
+                "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80",
+                "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666",
+                "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061",
+                "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b",
+                "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992",
+                "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593",
+                "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4",
+                "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db",
+                "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba",
+                "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd",
+                "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e",
+                "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212",
+                "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb",
+                "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2",
+                "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34",
+                "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256",
+                "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f",
+                "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2",
+                "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38",
+                "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996",
+                "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a",
+                "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==10.0.1"
+            "version": "==10.1.0"
         },
         "pkgutil-resolve-name": {
             "hashes": [
@@ -2149,23 +2160,25 @@
         },
         "psutil": {
             "hashes": [
-                "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d",
-                "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217",
-                "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4",
-                "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c",
-                "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f",
-                "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da",
-                "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4",
-                "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42",
-                "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5",
-                "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4",
-                "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9",
-                "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f",
-                "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
-                "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
+                "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28",
+                "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017",
+                "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602",
+                "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac",
+                "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a",
+                "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9",
+                "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4",
+                "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c",
+                "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c",
+                "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c",
+                "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a",
+                "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c",
+                "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57",
+                "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a",
+                "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d",
+                "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.5"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==5.9.6"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -2658,7 +2671,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==2.8.2"
         },
         "python-dotenv": {
@@ -2825,9 +2838,11 @@
             "hashes": [
                 "sha256:00c188704141c709da96cc4a79f058d51f5318e839d6f904c7cc9badcf78e98e",
                 "sha256:013321ddaff083b24e43a8b06303446771978343b488ed73adf56c70a46e2783",
+                "sha256:02c95d7109052c985b7d90dac6f6010bc0630227f15aec16302162107137bdbc",
                 "sha256:0a4334e972109bdd17fb40dbdd9fcca6137648cab416fca505a2dcd186f50533",
                 "sha256:12668ceb8329aaa908b4d907d3a77bb748ff28b309c3b105c995a8715d535d2b",
                 "sha256:14121a4d95070f54bdc9a80dab1dd8fd9093907a1e687926447ca69b5b40a4d5",
+                "sha256:145ca5ed6240af2cbfc09faa50aada8aacf1e2928ed6dd9da1d6b8ebe39cdc4c",
                 "sha256:1b9cda5314982d64c856f9298be0d9bf69fbff0ca514d1651037616354b473ff",
                 "sha256:1cbf212253abd65e6451acdfb608adafe98ad8f05462fb9a054ddab816545caa",
                 "sha256:1dbe76b6d8fe75f6dbec24793fc07b1d1ae9464de9941138d5b9668f7670e6b0",
@@ -2848,6 +2863,7 @@
                 "sha256:3f0fe2ef7ebc6e9b347585e414c4fefd32270ba8bdf9eb82496f3030cbdca465",
                 "sha256:3f72f310b10b730cddfb654006ae497e7706c81e6a7642d3da7fd2439df7d88d",
                 "sha256:40bdb468281a5cd525e2e990b97344f0974e0589bd1b395501c25471fcd7edda",
+                "sha256:418e9a676cc7ce00edd2fd044ee063c8639fd8cd6897ffda395a152cdc66ec97",
                 "sha256:4358dd80b315c82d760b44c6df7857c9c898d04e7b0c14abb0eb3692354e9379",
                 "sha256:441078bfd3b508597415338af667c3575980364f1286eedde58291558b9c2832",
                 "sha256:47c2a4c319300c381f194274203f47b12c433e1fd86b90ecdc7fb258c630f93b",
@@ -2856,8 +2872,10 @@
                 "sha256:4ed01beb31d5177456ec2c4b66591a0df83dbc72df29f05f40502bfefe47bbe4",
                 "sha256:50ccbaafee80b4f1c5c55bbe07f80871b9b8fe3499bf7357dde2c23fb1c2ac0e",
                 "sha256:51607d7d44f94a364ef0e3ccf9a92390def0faf6e7572eef082f15c657b5d03a",
+                "sha256:52dcae42f32f7a25c6b90bd479f3d04902700e3214e8fffe1bfe70053eb35ccb",
                 "sha256:5345c7a697327e2fa7c37534bb2968ea84595d8ec7fc8c4a60216ec1be6e65bd",
                 "sha256:542808d88464d538f5d2c6b48b545a7fe15f0d20c7fa703b469d039a08c9fa10",
+                "sha256:5453ebe42a2c7462fa532fd03cbf64e5c6baf5508b3089736c78444148d3c593",
                 "sha256:5819d502dacd54114c30bc24efcb76e723b93f8f528be70851056a396a792c46",
                 "sha256:5aed5fc86d0bfc5f16e871cbb35ec93df61476d7fde4c1c6081015a075ecfbc1",
                 "sha256:5d9ec8634ab0cbfbcff535ac07555ebdae0282ad66762f0471fad11c16181e33",
@@ -2876,6 +2894,7 @@
                 "sha256:74455bd918e7bc9883e3178a1a8fe796308670f0ee4488c80a0d9514e13807a1",
                 "sha256:7452ae7e6d80e697d78d3f56d1b4d2a350286eea229afb35f55ab88b934b6acd",
                 "sha256:77294f0f797c97a46ffb3daff1fe097c9d5aa9f96867333978e6791286963e50",
+                "sha256:78c38850af6b990e8ec1bc87b48f73ed5cc633f4baaa7bbc78f9b2f4449cf081",
                 "sha256:7ac886e04f253960ae82e38ded8352085c61d78de99412d178a94ecf475b5e5f",
                 "sha256:7c420878726d677da7484f6021dbe7e1f9345a791b155de632c6ce36678fb621",
                 "sha256:836f1d85a4b5d3689d455aeb1dc6c42acb96aaf8e5282825c00ccf2545ad5630",
@@ -2886,8 +2905,11 @@
                 "sha256:8d3a1b6fa71a0ae7abc320d9db91b5a96a71eef1dbee0d62a6232b71c97af962",
                 "sha256:8f9eb97fb6fd4551ff9d5012b4fcee9abeea9c8af6b9e3ebc3c76cc2bd0a43a7",
                 "sha256:91453ce9476363d777b2ea2e9c6dccecd2073cf35697e048de2e8d47e1f36c7c",
+                "sha256:922f1bb8ef80c42a2fca297ba0b03442c143a9a1f717e83db79f190514888803",
+                "sha256:937f118fdd7a23654886634f650d6502a2dd12c8a8e2bf14beb2fa5fa95058bf",
                 "sha256:9596aeb8c71192f4fba1ca25cec420da195219398d2df811d5082559efd9561f",
                 "sha256:960ab83a977a44284c4ffab2820ccd6c9b332571a3d622fefa4b29b0a5de72b0",
+                "sha256:9638d40ec02a5b194a4c98a5b6e36cdfde4e9d6b721ae6167ef8e57d2e69002f",
                 "sha256:97e05f66c5847e6889594508298d78ddb84a0115e9234d598415dc5a06d3a4a7",
                 "sha256:9ac634753f6d26cba503cea7bb5b350aec7c5366f44fa68c79e9c90be9fd0ebc",
                 "sha256:9e1097d8b57f64878a3f176f4cd6b9a1bbe9fb2d236f1a85a4357722626d8f25",
@@ -2907,16 +2929,20 @@
                 "sha256:c249741b10eb714578d765487b767e0e7fcc2ac84a299209a6073566e730dbea",
                 "sha256:c2b093a74b10232c70b5d29814fcee6544bb6f30e2d922d26db9ab4b4cd00c04",
                 "sha256:c31f6dd5bd60688d51487a3f5e2ae29ed1948926e44d7a2316b193b083f80d5d",
+                "sha256:c36dbbf71480f1fffeaeca901adb31e0c7d59270a239eca63fe26e4647b7aca8",
                 "sha256:c41e5457f4de5d38a270bc44619873589bbe6fe251225deec583ed20199df0f3",
                 "sha256:c46e77c2ad614a0399503dc675d72436cbf6332a20d49a0e5bad03058d6cbfad",
+                "sha256:c8d1966e38c220d5940f8cb6303651af261f0bcfce77218a030b1a24ec986e2f",
                 "sha256:c9589cb79d4e401630481755c92b072aa7ba5505ec81dec865ef43932ec037e4",
                 "sha256:ca19213785f864781848e0216cba07e97f563f60a50bbc7885b54461d8c64873",
                 "sha256:cbfdde6c5768ffa5d2f14127bbc1d7c3c2d03c0ceaeb0736946197e06275ccc7",
                 "sha256:cd6a8d43a0c294918e3afb7e4b1d8c04d2e4c3ea9ddf05475fdaf366c7e5b3a6",
+                "sha256:cfa981cedd54bb8862d9033440a0afac38845db89e7099ceeb4f4d064dffd2f8",
                 "sha256:cffaab46f9e04856dc3daa6097bfb3d3bea0b1771237e869c57b13f3dcc2c238",
                 "sha256:d0929302d187bfeca335b7f710f774f1b2ea3f610b2a80e8a1ac2da216cd9766",
                 "sha256:d44a7d4586f02b630658298c089ff755e74d0677b93c71e09d33dd35bdd4987a",
                 "sha256:d7ddbf234c9adc72189bb552d830e9a0c2c4401b5baf7b003eacd5c552ddcc00",
+                "sha256:da070933d4bcfcbf58472da12ffa77c9fbc90efb39e21a9b74eb04b5af4b412a",
                 "sha256:dca286c6c1ca5febf13f5f2ae7e8aa7536e49bd07f4232796651a43ff741ceca",
                 "sha256:dcb2172ca8b62f82af9d1f8db80c21c64c5ba3991935caefde88bb378f0afb51",
                 "sha256:e4e00c1600022b47ef0e9e1f893cb0c2322209ec6c1581a3e3f63ed78330ddf0",
@@ -2930,6 +2956,7 @@
                 "sha256:f2839c13e486e4a23b19b1d2dc4624565cec6c228bbf803c066be1106515966b",
                 "sha256:f66790e4b2dcfcabc0aa54dd89317ea5671cabf06aa93cbef7cbdd4d2fdb7ee3",
                 "sha256:f6d8a881b50bb2015e9bdba5edb0331e85d41ff44ab33cde551047480b98d748",
+                "sha256:f73821d429bfbb04645b80ec491ab05b35078f031f9fa3273fbf9027d1406233",
                 "sha256:f7cfc683d320402d61205a196ace77f15dcfd16b5771f8b9ffaf406868c98e78",
                 "sha256:f9c5fc29a5b9d61a8f0a3494172107e0e6cf23d0cb800d6285c6722ba7fc3535",
                 "sha256:fc92a718bccb8ce5c9eb63fca743c38f3fa4c4e47f58f0c4ada51b2474668184"
@@ -3055,106 +3082,108 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:015de2ce2af1586ff5dc873e804434185199a15f7d96920ce67e50604592cae9",
-                "sha256:061c3ff1f51ecec256e916cf71cc01f9975af8fb3af9b94d3c0cc8702cfea637",
-                "sha256:08a80cf4884920863623a9ee9a285ee04cef57ebedc1cc87b3e3e0f24c8acfe5",
-                "sha256:09362f86ec201288d5687d1dc476b07bf39c08478cde837cb710b302864e7ec9",
-                "sha256:0bb4f48bd0dd18eebe826395e6a48b7331291078a879295bae4e5d053be50d4c",
-                "sha256:106af1653007cc569d5fbb5f08c6648a49fe4de74c2df814e234e282ebc06957",
-                "sha256:11fdd1192240dda8d6c5d18a06146e9045cb7e3ba7c06de6973000ff035df7c6",
-                "sha256:16a472300bc6c83fe4c2072cc22b3972f90d718d56f241adabc7ae509f53f154",
-                "sha256:176287bb998fd1e9846a9b666e240e58f8d3373e3bf87e7642f15af5405187b8",
-                "sha256:177914f81f66c86c012311f8c7f46887ec375cfcfd2a2f28233a3053ac93a569",
-                "sha256:177c9dd834cdf4dc39c27436ade6fdf9fe81484758885f2d616d5d03c0a83bd2",
-                "sha256:187700668c018a7e76e89424b7c1042f317c8df9161f00c0c903c82b0a8cac5c",
-                "sha256:1d9b5ee46dcb498fa3e46d4dfabcb531e1f2e76b477e0d99ef114f17bbd38453",
-                "sha256:22da15b902f9f8e267020d1c8bcfc4831ca646fecb60254f7bc71763569f56b1",
-                "sha256:24cd91a03543a0f8d09cb18d1cb27df80a84b5553d2bd94cba5979ef6af5c6e7",
-                "sha256:255f1a10ae39b52122cce26ce0781f7a616f502feecce9e616976f6a87992d6b",
-                "sha256:271c360fdc464fe6a75f13ea0c08ddf71a321f4c55fc20a3fe62ea3ef09df7d9",
-                "sha256:2ed83d53a8c5902ec48b90b2ac045e28e1698c0bea9441af9409fc844dc79496",
-                "sha256:2f3e1867dd574014253b4b8f01ba443b9c914e61d45f3674e452a915d6e929a3",
-                "sha256:35fbd23c1c8732cde7a94abe7fb071ec173c2f58c0bd0d7e5b669fdfc80a2c7b",
-                "sha256:37d0c59548ae56fae01c14998918d04ee0d5d3277363c10208eef8c4e2b68ed6",
-                "sha256:39d05e65f23a0fe897b6ac395f2a8d48c56ac0f583f5d663e0afec1da89b95da",
-                "sha256:3ad59efe24a4d54c2742929001f2d02803aafc15d6d781c21379e3f7f66ec842",
-                "sha256:3aed39db2f0ace76faa94f465d4234aac72e2f32b009f15da6492a561b3bbebd",
-                "sha256:3bbac1953c17252f9cc675bb19372444aadf0179b5df575ac4b56faaec9f6294",
-                "sha256:40bc802a696887b14c002edd43c18082cb7b6f9ee8b838239b03b56574d97f71",
-                "sha256:42f712b4668831c0cd85e0a5b5a308700fe068e37dcd24c0062904c4e372b093",
-                "sha256:448a66b8266de0b581246ca7cd6a73b8d98d15100fb7165974535fa3b577340e",
-                "sha256:485301ee56ce87a51ccb182a4b180d852c5cb2b3cb3a82f7d4714b4141119d8c",
-                "sha256:485747ee62da83366a44fbba963c5fe017860ad408ccd6cd99aa66ea80d32b2e",
-                "sha256:4cf0855a842c5b5c391dd32ca273b09e86abf8367572073bd1edfc52bc44446b",
-                "sha256:4eca20917a06d2fca7628ef3c8b94a8c358f6b43f1a621c9815243462dcccf97",
-                "sha256:4ed172d0c79f156c1b954e99c03bc2e3033c17efce8dd1a7c781bc4d5793dfac",
-                "sha256:5267cfda873ad62591b9332fd9472d2409f7cf02a34a9c9cb367e2c0255994bf",
-                "sha256:52b5cbc0469328e58180021138207e6ec91d7ca2e037d3549cc9e34e2187330a",
-                "sha256:53d7a3cd46cdc1689296348cb05ffd4f4280035770aee0c8ead3bbd4d6529acc",
-                "sha256:563646d74a4b4456d0cf3b714ca522e725243c603e8254ad85c3b59b7c0c4bf0",
-                "sha256:570cc326e78ff23dec7f41487aa9c3dffd02e5ee9ab43a8f6ccc3df8f9327623",
-                "sha256:5aca759ada6b1967fcfd4336dcf460d02a8a23e6abe06e90ea7881e5c22c4de6",
-                "sha256:5de11c041486681ce854c814844f4ce3282b6ea1656faae19208ebe09d31c5b8",
-                "sha256:5e271dd97c7bb8eefda5cca38cd0b0373a1fea50f71e8071376b46968582af9b",
-                "sha256:642ed0a209ced4be3a46f8cb094f2d76f1f479e2a1ceca6de6346a096cd3409d",
-                "sha256:6446002739ca29249f0beaaf067fcbc2b5aab4bc7ee8fb941bd194947ce19aff",
-                "sha256:691d50c99a937709ac4c4cd570d959a006bd6a6d970a484c84cc99543d4a5bbb",
-                "sha256:69b857a7d8bd4f5d6e0db4086da8c46309a26e8cefdfc778c0c5cc17d4b11e08",
-                "sha256:6ac3fefb0d168c7c6cab24fdfc80ec62cd2b4dfd9e65b84bdceb1cb01d385c33",
-                "sha256:6c9141af27a4e5819d74d67d227d5047a20fa3c7d4d9df43037a955b4c748ec5",
-                "sha256:7170cbde4070dc3c77dec82abf86f3b210633d4f89550fa0ad2d4b549a05572a",
-                "sha256:763ad59e105fca09705d9f9b29ecffb95ecdc3b0363be3bb56081b2c6de7977a",
-                "sha256:77076bdc8776a2b029e1e6ffbe6d7056e35f56f5e80d9dc0bad26ad4a024a762",
-                "sha256:7cd020b1fb41e3ab7716d4d2c3972d4588fdfbab9bfbbb64acc7078eccef8860",
-                "sha256:821392559d37759caa67d622d0d2994c7a3f2fb29274948ac799d496d92bca73",
-                "sha256:829e91f3a8574888b73e7a3feb3b1af698e717513597e23136ff4eba0bc8387a",
-                "sha256:850c272e0e0d1a5c5d73b1b7871b0a7c2446b304cec55ccdb3eaac0d792bb065",
-                "sha256:87d9b206b1bd7a0523375dc2020a6ce88bca5330682ae2fe25e86fd5d45cea9c",
-                "sha256:8bd01ff4032abaed03f2db702fa9a61078bee37add0bd884a6190b05e63b028c",
-                "sha256:8d54bbdf5d56e2c8cf81a1857250f3ea132de77af543d0ba5dce667183b61fec",
-                "sha256:8efaeb08ede95066da3a3e3c420fcc0a21693fcd0c4396d0585b019613d28515",
-                "sha256:8f94fdd756ba1f79f988855d948ae0bad9ddf44df296770d9a58c774cfbcca72",
-                "sha256:95cde244e7195b2c07ec9b73fa4c5026d4a27233451485caa1cd0c1b55f26dbd",
-                "sha256:975382d9aa90dc59253d6a83a5ca72e07f4ada3ae3d6c0575ced513db322b8ec",
-                "sha256:9dd9d9d9e898b9d30683bdd2b6c1849449158647d1049a125879cb397ee9cd12",
-                "sha256:a019a344312d0b1f429c00d49c3be62fa273d4a1094e1b224f403716b6d03be1",
-                "sha256:a4d9bfda3f84fc563868fe25ca160c8ff0e69bc4443c5647f960d59400ce6557",
-                "sha256:a657250807b6efd19b28f5922520ae002a54cb43c2401e6f3d0230c352564d25",
-                "sha256:a771417c9c06c56c9d53d11a5b084d1de75de82978e23c544270ab25e7c066ff",
-                "sha256:aad6ed9e70ddfb34d849b761fb243be58c735be6a9265b9060d6ddb77751e3e8",
-                "sha256:ae87137951bb3dc08c7d8bfb8988d8c119f3230731b08a71146e84aaa919a7a9",
-                "sha256:af247fd4f12cca4129c1b82090244ea5a9d5bb089e9a82feb5a2f7c6a9fe181d",
-                "sha256:b5d4bdd697195f3876d134101c40c7d06d46c6ab25159ed5cbd44105c715278a",
-                "sha256:b9255e7165083de7c1d605e818025e8860636348f34a79d84ec533546064f07e",
-                "sha256:c22211c165166de6683de8136229721f3d5c8606cc2c3d1562da9a3a5058049c",
-                "sha256:c55f9821f88e8bee4b7a72c82cfb5ecd22b6aad04033334f33c329b29bfa4da0",
-                "sha256:c7aed97f2e676561416c927b063802c8a6285e9b55e1b83213dfd99a8f4f9e48",
-                "sha256:cd2163f42868865597d89399a01aa33b7594ce8e2c4a28503127c81a2f17784e",
-                "sha256:ce5e7504db95b76fc89055c7f41e367eaadef5b1d059e27e1d6eabf2b55ca314",
-                "sha256:cff7351c251c7546407827b6a37bcef6416304fc54d12d44dbfecbb717064717",
-                "sha256:d27aa6bbc1f33be920bb7adbb95581452cdf23005d5611b29a12bb6a3468cc95",
-                "sha256:d3b52a67ac66a3a64a7e710ba629f62d1e26ca0504c29ee8cbd99b97df7079a8",
-                "sha256:de61e424062173b4f70eec07e12469edde7e17fa180019a2a0d75c13a5c5dc57",
-                "sha256:e10e6a1ed2b8661201e79dff5531f8ad4cdd83548a0f81c95cf79b3184b20c33",
-                "sha256:e1a0ffc39f51aa5f5c22114a8f1906b3c17eba68c5babb86c5f77d8b1bba14d1",
-                "sha256:e22491d25f97199fc3581ad8dd8ce198d8c8fdb8dae80dea3512e1ce6d5fa99f",
-                "sha256:e626b864725680cd3904414d72e7b0bd81c0e5b2b53a5b30b4273034253bb41f",
-                "sha256:e8c71ea77536149e36c4c784f6d420ffd20bea041e3ba21ed021cb40ce58e2c9",
-                "sha256:e8d0f0eca087630d58b8c662085529781fd5dc80f0a54eda42d5c9029f812599",
-                "sha256:ea65b59882d5fa8c74a23f8960db579e5e341534934f43f3b18ec1839b893e41",
-                "sha256:ea93163472db26ac6043e8f7f93a05d9b59e0505c760da2a3cd22c7dd7111391",
-                "sha256:eab75a8569a095f2ad470b342f2751d9902f7944704f0571c8af46bede438475",
-                "sha256:ed8313809571a5463fd7db43aaca68ecb43ca7a58f5b23b6e6c6c5d02bdc7882",
-                "sha256:ef5fddfb264e89c435be4adb3953cef5d2936fdeb4463b4161a6ba2f22e7b740",
-                "sha256:ef750a20de1b65657a1425f77c525b0183eac63fe7b8f5ac0dd16f3668d3e64f",
-                "sha256:efb9ece97e696bb56e31166a9dd7919f8f0c6b31967b454718c6509f29ef6fee",
-                "sha256:f4c179a7aeae10ddf44c6bac87938134c1379c49c884529f090f9bf05566c836",
-                "sha256:f602881d80ee4228a2355c68da6b296a296cd22bbb91e5418d54577bbf17fa7c",
-                "sha256:fc2200e79d75b5238c8d69f6a30f8284290c777039d331e7340b6c17cad24a5a",
-                "sha256:fcc1ebb7561a3e24a6588f7c6ded15d80aec22c66a070c757559b57b17ffd1cb"
+                "sha256:023574366002bf1bd751ebaf3e580aef4a468b3d3c216d2f3f7e16fdabd885ed",
+                "sha256:031f76fc87644a234883b51145e43985aa2d0c19b063e91d44379cd2786144f8",
+                "sha256:052a832078943d2b2627aea0d19381f607fe331cc0eb5df01991268253af8417",
+                "sha256:0699ab6b8c98df998c3eacf51a3b25864ca93dab157abe358af46dc95ecd9801",
+                "sha256:0713631d6e2d6c316c2f7b9320a34f44abb644fc487b77161d1724d883662e31",
+                "sha256:0774a46b38e70fdde0c6ded8d6d73115a7c39d7839a164cc833f170bbf539116",
+                "sha256:0898173249141ee99ffcd45e3829abe7bcee47d941af7434ccbf97717df020e5",
+                "sha256:09586f51a215d17efdb3a5f090d7cbf1633b7f3708f60a044757a5d48a83b393",
+                "sha256:102eac53bb0bf0f9a275b438e6cf6904904908562a1463a6fc3323cf47d7a532",
+                "sha256:10f32b53f424fc75ff7b713b2edb286fdbfc94bf16317890260a81c2c00385dc",
+                "sha256:150eec465dbc9cbca943c8e557a21afdcf9bab8aaabf386c44b794c2f94143d2",
+                "sha256:1d7360573f1e046cb3b0dceeb8864025aa78d98be4bb69f067ec1c40a9e2d9df",
+                "sha256:1f36a9d751f86455dc5278517e8b65580eeee37d61606183897f122c9e51cef3",
+                "sha256:24656dc36f866c33856baa3ab309da0b6a60f37d25d14be916bd3e79d9f3afcf",
+                "sha256:25860ed5c4e7f5e10c496ea78af46ae8d8468e0be745bd233bab9ca99bfd2647",
+                "sha256:26857f0f44f0e791f4a266595a7a09d21f6b589580ee0585f330aaccccb836e3",
+                "sha256:2bb2e4826be25e72013916eecd3d30f66fd076110de09f0e750163b416500721",
+                "sha256:2f6da6d842195fddc1cd34c3da8a40f6e99e4a113918faa5e60bf132f917c247",
+                "sha256:30adb75ecd7c2a52f5e76af50644b3e0b5ba036321c390b8e7ec1bb2a16dd43c",
+                "sha256:3339eca941568ed52d9ad0f1b8eb9fe0958fa245381747cecf2e9a78a5539c42",
+                "sha256:34ad87a831940521d462ac11f1774edf867c34172010f5390b2f06b85dcc6014",
+                "sha256:3777cc9dea0e6c464e4b24760664bd8831738cc582c1d8aacf1c3f546bef3f65",
+                "sha256:3953c6926a63f8ea5514644b7afb42659b505ece4183fdaaa8f61d978754349e",
+                "sha256:3c4eff26eddac49d52697a98ea01b0246e44ca82ab09354e94aae8823e8bda02",
+                "sha256:40578a6469e5d1df71b006936ce95804edb5df47b520c69cf5af264d462f2cbb",
+                "sha256:40f93086eef235623aa14dbddef1b9fb4b22b99454cb39a8d2e04c994fb9868c",
+                "sha256:4134aa2342f9b2ab6c33d5c172e40f9ef802c61bb9ca30d21782f6e035ed0043",
+                "sha256:442626328600bde1d09dc3bb00434f5374948838ce75c41a52152615689f9403",
+                "sha256:4a5ee600477b918ab345209eddafde9f91c0acd931f3776369585a1c55b04c57",
+                "sha256:4ce5a708d65a8dbf3748d2474b580d606b1b9f91b5c6ab2a316e0b0cf7a4ba50",
+                "sha256:516a611a2de12fbea70c78271e558f725c660ce38e0006f75139ba337d56b1f6",
+                "sha256:52c215eb46307c25f9fd2771cac8135d14b11a92ae48d17968eda5aa9aaf5071",
+                "sha256:53c43e10d398e365da2d4cc0bcaf0854b79b4c50ee9689652cdc72948e86f487",
+                "sha256:5752b761902cd15073a527b51de76bbae63d938dc7c5c4ad1e7d8df10e765138",
+                "sha256:5e8a78bd4879bff82daef48c14d5d4057f6856149094848c3ed0ecaf49f5aec2",
+                "sha256:5ed505ec6305abd2c2c9586a7b04fbd4baf42d4d684a9c12ec6110deefe2a063",
+                "sha256:5ee97c683eaface61d38ec9a489e353d36444cdebb128a27fe486a291647aff6",
+                "sha256:61fa268da6e2e1cd350739bb61011121fa550aa2545762e3dc02ea177ee4de35",
+                "sha256:64ccc28683666672d7c166ed465c09cee36e306c156e787acef3c0c62f90da5a",
+                "sha256:66414dafe4326bca200e165c2e789976cab2587ec71beb80f59f4796b786a238",
+                "sha256:68fe9199184c18d997d2e4293b34327c0009a78599ce703e15cd9a0f47349bba",
+                "sha256:6a555ae3d2e61118a9d3e549737bb4a56ff0cec88a22bd1dfcad5b4e04759175",
+                "sha256:6bdc11f9623870d75692cc33c59804b5a18d7b8a4b79ef0b00b773a27397d1f6",
+                "sha256:6cf4393c7b41abbf07c88eb83e8af5013606b1cdb7f6bc96b1b3536b53a574b8",
+                "sha256:6eef672de005736a6efd565577101277db6057f65640a813de6c2707dc69f396",
+                "sha256:734c41f9f57cc28658d98270d3436dba65bed0cfc730d115b290e970150c540d",
+                "sha256:73e0a78a9b843b8c2128028864901f55190401ba38aae685350cf69b98d9f7c9",
+                "sha256:775049dfa63fb58293990fc59473e659fcafd953bba1d00fc5f0631a8fd61977",
+                "sha256:7854a207ef77319ec457c1eb79c361b48807d252d94348305db4f4b62f40f7f3",
+                "sha256:78ca33811e1d95cac8c2e49cb86c0fb71f4d8409d8cbea0cb495b6dbddb30a55",
+                "sha256:79edd779cfc46b2e15b0830eecd8b4b93f1a96649bcb502453df471a54ce7977",
+                "sha256:7bf347b495b197992efc81a7408e9a83b931b2f056728529956a4d0858608b80",
+                "sha256:7fde6d0e00b2fd0dbbb40c0eeec463ef147819f23725eda58105ba9ca48744f4",
+                "sha256:81de24a1c51cfb32e1fbf018ab0bdbc79c04c035986526f76c33e3f9e0f3356c",
+                "sha256:879fb24304ead6b62dbe5034e7b644b71def53c70e19363f3c3be2705c17a3b4",
+                "sha256:8e7f2219cb72474571974d29a191714d822e58be1eb171f229732bc6fdedf0ac",
+                "sha256:9164ec8010327ab9af931d7ccd12ab8d8b5dc2f4c6a16cbdd9d087861eaaefa1",
+                "sha256:945eb4b6bb8144909b203a88a35e0a03d22b57aefb06c9b26c6e16d72e5eb0f0",
+                "sha256:99a57006b4ec39dbfb3ed67e5b27192792ffb0553206a107e4aadb39c5004cd5",
+                "sha256:9e9184fa6c52a74a5521e3e87badbf9692549c0fcced47443585876fcc47e469",
+                "sha256:9ff93d3aedef11f9c4540cf347f8bb135dd9323a2fc705633d83210d464c579d",
+                "sha256:a360cfd0881d36c6dc271992ce1eda65dba5e9368575663de993eeb4523d895f",
+                "sha256:a5d7ed104d158c0042a6a73799cf0eb576dfd5fc1ace9c47996e52320c37cb7c",
+                "sha256:ac17044876e64a8ea20ab132080ddc73b895b4abe9976e263b0e30ee5be7b9c2",
+                "sha256:ad857f42831e5b8d41a32437f88d86ead6c191455a3499c4b6d15e007936d4cf",
+                "sha256:b2039f8d545f20c4e52713eea51a275e62153ee96c8035a32b2abb772b6fc9e5",
+                "sha256:b455492cab07107bfe8711e20cd920cc96003e0da3c1f91297235b1603d2aca7",
+                "sha256:b4a9fe992887ac68256c930a2011255bae0bf5ec837475bc6f7edd7c8dfa254e",
+                "sha256:b5a53f5998b4bbff1cb2e967e66ab2addc67326a274567697379dd1e326bded7",
+                "sha256:b788276a3c114e9f51e257f2a6f544c32c02dab4aa7a5816b96444e3f9ffc336",
+                "sha256:bddd4f91eede9ca5275e70479ed3656e76c8cdaaa1b354e544cbcf94c6fc8ac4",
+                "sha256:c0503c5b681566e8b722fe8c4c47cce5c7a51f6935d5c7012c4aefe952a35eed",
+                "sha256:c1b3cd23d905589cb205710b3988fc8f46d4a198cf12862887b09d7aaa6bf9b9",
+                "sha256:c48f3fbc3e92c7dd6681a258d22f23adc2eb183c8cb1557d2fcc5a024e80b094",
+                "sha256:c63c3ef43f0b3fb00571cff6c3967cc261c0ebd14a0a134a12e83bdb8f49f21f",
+                "sha256:c6c45a2d2b68c51fe3d9352733fe048291e483376c94f7723458cfd7b473136b",
+                "sha256:caa1afc70a02645809c744eefb7d6ee8fef7e2fad170ffdeacca267fd2674f13",
+                "sha256:cc435d059f926fdc5b05822b1be4ff2a3a040f3ae0a7bbbe672babb468944722",
+                "sha256:cf693eb4a08eccc1a1b636e4392322582db2a47470d52e824b25eca7a3977b53",
+                "sha256:cf71343646756a072b85f228d35b1d7407da1669a3de3cf47f8bbafe0c8183a4",
+                "sha256:d08f63561c8a695afec4975fae445245386d645e3e446e6f260e81663bfd2e38",
+                "sha256:d29ddefeab1791e3c751e0189d5f4b3dbc0bbe033b06e9c333dca1f99e1d523e",
+                "sha256:d7f5e15c953ace2e8dde9824bdab4bec50adb91a5663df08d7d994240ae6fa31",
+                "sha256:d858532212f0650be12b6042ff4378dc2efbb7792a286bee4489eaa7ba010586",
+                "sha256:d97dd44683802000277bbf142fd9f6b271746b4846d0acaf0cefa6b2eaf2a7ad",
+                "sha256:dcdc88b6b01015da066da3fb76545e8bb9a6880a5ebf89e0f0b2e3ca557b3ab7",
+                "sha256:dd609fafdcdde6e67a139898196698af37438b035b25ad63704fd9097d9a3482",
+                "sha256:defa2c0c68734f4a82028c26bcc85e6b92cced99866af118cd6a89b734ad8e0d",
+                "sha256:e22260a4741a0e7a206e175232867b48a16e0401ef5bce3c67ca5b9705879066",
+                "sha256:e225a6a14ecf44499aadea165299092ab0cba918bb9ccd9304eab1138844490b",
+                "sha256:e3df0bc35e746cce42579826b89579d13fd27c3d5319a6afca9893a9b784ff1b",
+                "sha256:e6fcc026a3f27c1282c7ed24b7fcac82cdd70a0e84cc848c0841a3ab1e3dea2d",
+                "sha256:e782379c2028a3611285a795b89b99a52722946d19fc06f002f8b53e3ea26ea9",
+                "sha256:e8cdd52744f680346ff8c1ecdad5f4d11117e1724d4f4e1874f3a67598821069",
+                "sha256:e9616f5bd2595f7f4a04b67039d890348ab826e943a9bfdbe4938d0eba606971",
+                "sha256:e98c4c07ee4c4b3acf787e91b27688409d918212dfd34c872201273fdd5a0e18",
+                "sha256:ebdab79f42c5961682654b851f3f0fc68e6cc7cd8727c2ac4ffff955154123c1",
+                "sha256:f0f17f2ce0f3529177a5fff5525204fad7b43dd437d017dd0317f2746773443d",
+                "sha256:f4e56860a5af16a0fcfa070a0a20c42fbb2012eed1eb5ceeddcc7f8079214281"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.3"
+            "version": "==0.10.6"
         },
         "rtree": {
             "hashes": [
@@ -3316,101 +3345,114 @@
         },
         "simplejson": {
             "hashes": [
-                "sha256:081ea6305b3b5e84ae7417e7f45956db5ea3872ec497a584ec86c3260cda049e",
-                "sha256:08be5a241fdf67a8e05ac7edbd49b07b638ebe4846b560673e196b2a25c94b92",
-                "sha256:0c16ec6a67a5f66ab004190829eeede01c633936375edcad7cbf06d3241e5865",
-                "sha256:0ccb2c1877bc9b25bc4f4687169caa925ffda605d7569c40e8e95186e9a5e58b",
-                "sha256:17a963e8dd4d81061cc05b627677c1f6a12e81345111fbdc5708c9f088d752c9",
-                "sha256:199a0bcd792811c252d71e3eabb3d4a132b3e85e43ebd93bfd053d5b59a7e78b",
-                "sha256:1cb19eacb77adc5a9720244d8d0b5507421d117c7ed4f2f9461424a1829e0ceb",
-                "sha256:203412745fed916fc04566ecef3f2b6c872b52f1e7fb3a6a84451b800fb508c1",
-                "sha256:2098811cd241429c08b7fc5c9e41fcc3f59f27c2e8d1da2ccdcf6c8e340ab507",
-                "sha256:22b867205cd258050c2625325fdd9a65f917a5aff22a23387e245ecae4098e78",
-                "sha256:23fbb7b46d44ed7cbcda689295862851105c7594ae5875dce2a70eeaa498ff86",
-                "sha256:2541fdb7467ef9bfad1f55b6c52e8ea52b3ce4a0027d37aff094190a955daa9d",
-                "sha256:3231100edee292da78948fa0a77dee4e5a94a0a60bcba9ed7a9dc77f4d4bb11e",
-                "sha256:344a5093b71c1b370968d0fbd14d55c9413cb6f0355fdefeb4a322d602d21776",
-                "sha256:37724c634f93e5caaca04458f267836eb9505d897ab3947b52f33b191bf344f3",
-                "sha256:3844305bc33d52c4975da07f75b480e17af3558c0d13085eaa6cc2f32882ccf7",
-                "sha256:390f4a8ca61d90bcf806c3ad644e05fa5890f5b9a72abdd4ca8430cdc1e386fa",
-                "sha256:3a4480e348000d89cf501b5606415f4d328484bbb431146c2971123d49fd8430",
-                "sha256:3b652579c21af73879d99c8072c31476788c8c26b5565687fd9db154070d852a",
-                "sha256:3e0902c278243d6f7223ba3e6c5738614c971fd9a887fff8feaa8dcf7249c8d4",
-                "sha256:412e58997a30c5deb8cab5858b8e2e5b40ca007079f7010ee74565cc13d19665",
-                "sha256:44cdb4e544134f305b033ad79ae5c6b9a32e7c58b46d9f55a64e2a883fbbba01",
-                "sha256:46133bc7dd45c9953e6ee4852e3de3d5a9a4a03b068bd238935a5c72f0a1ce34",
-                "sha256:46e89f58e4bed107626edce1cf098da3664a336d01fc78fddcfb1f397f553d44",
-                "sha256:4710806eb75e87919b858af0cba4ffedc01b463edc3982ded7b55143f39e41e1",
-                "sha256:476c8033abed7b1fd8db62a7600bf18501ce701c1a71179e4ce04ac92c1c5c3c",
-                "sha256:48600a6e0032bed17c20319d91775f1797d39953ccfd68c27f83c8d7fc3b32cb",
-                "sha256:4d3025e7e9ddb48813aec2974e1a7e68e63eac911dd5e0a9568775de107ac79a",
-                "sha256:547ea86ca408a6735335c881a2e6208851027f5bfd678d8f2c92a0f02c7e7330",
-                "sha256:54fca2b26bcd1c403146fd9461d1da76199442297160721b1d63def2a1b17799",
-                "sha256:5673d27806085d2a413b3be5f85fad6fca4b7ffd31cfe510bbe65eea52fff571",
-                "sha256:58ee5e24d6863b22194020eb62673cf8cc69945fcad6b283919490f6e359f7c5",
-                "sha256:5ca922c61d87b4c38f37aa706520328ffe22d7ac1553ef1cadc73f053a673553",
-                "sha256:5db86bb82034e055257c8e45228ca3dbce85e38d7bfa84fa7b2838e032a3219c",
-                "sha256:6277f60848a7d8319d27d2be767a7546bc965535b28070e310b3a9af90604a4c",
-                "sha256:6424d8229ba62e5dbbc377908cfee9b2edf25abd63b855c21f12ac596cd18e41",
-                "sha256:65dafe413b15e8895ad42e49210b74a955c9ae65564952b0243a18fb35b986cc",
-                "sha256:66389b6b6ee46a94a493a933a26008a1bae0cfadeca176933e7ff6556c0ce998",
-                "sha256:66d780047c31ff316ee305c3f7550f352d87257c756413632303fc59fef19eac",
-                "sha256:69a8b10a4f81548bc1e06ded0c4a6c9042c0be0d947c53c1ed89703f7e613950",
-                "sha256:6a561320485017ddfc21bd2ed5de2d70184f754f1c9b1947c55f8e2b0163a268",
-                "sha256:6aa7ca03f25b23b01629b1c7f78e1cd826a66bfb8809f8977a3635be2ec48f1a",
-                "sha256:6b79642a599740603ca86cf9df54f57a2013c47e1dd4dd2ae4769af0a6816900",
-                "sha256:6e7c70f19405e5f99168077b785fe15fcb5f9b3c0b70b0b5c2757ce294922c8c",
-                "sha256:70128fb92932524c89f373e17221cf9535d7d0c63794955cc3cd5868e19f5d38",
-                "sha256:73d0904c2471f317386d4ae5c665b16b5c50ab4f3ee7fd3d3b7651e564ad74b1",
-                "sha256:74bf802debe68627227ddb665c067eb8c73aa68b2476369237adf55c1161b728",
-                "sha256:79c748aa61fd8098d0472e776743de20fae2686edb80a24f0f6593a77f74fe86",
-                "sha256:79d46e7e33c3a4ef853a1307b2032cfb7220e1a079d0c65488fbd7118f44935a",
-                "sha256:7e78d79b10aa92f40f54178ada2b635c960d24fc6141856b926d82f67e56d169",
-                "sha256:8090e75653ea7db75bc21fa5f7bcf5f7bdf64ea258cbbac45c7065f6324f1b50",
-                "sha256:87b190e6ceec286219bd6b6f13547ca433f977d4600b4e81739e9ac23b5b9ba9",
-                "sha256:889328873c35cb0b2b4c83cbb83ec52efee5a05e75002e2c0c46c4e42790e83c",
-                "sha256:8f8d179393e6f0cf6c7c950576892ea6acbcea0a320838c61968ac7046f59228",
-                "sha256:919bc5aa4d8094cf8f1371ea9119e5d952f741dc4162810ab714aec948a23fe5",
-                "sha256:926957b278de22797bfc2f004b15297013843b595b3cd7ecd9e37ccb5fad0b72",
-                "sha256:93f5ac30607157a0b2579af59a065bcfaa7fadeb4875bf927a8f8b6739c8d910",
-                "sha256:96ade243fb6f3b57e7bd3b71e90c190cd0f93ec5dce6bf38734a73a2e5fa274f",
-                "sha256:9f14ecca970d825df0d29d5c6736ff27999ee7bdf5510e807f7ad8845f7760ce",
-                "sha256:a755f7bfc8adcb94887710dc70cc12a69a454120c6adcc6f251c3f7b46ee6aac",
-                "sha256:a79b439a6a77649bb8e2f2644e6c9cc0adb720fc55bed63546edea86e1d5c6c8",
-                "sha256:aa9d614a612ad02492f704fbac636f666fa89295a5d22b4facf2d665fc3b5ea9",
-                "sha256:ad071cd84a636195f35fa71de2186d717db775f94f985232775794d09f8d9061",
-                "sha256:b0e9a5e66969f7a47dc500e3dba8edc3b45d4eb31efb855c8647700a3493dd8a",
-                "sha256:b438e5eaa474365f4faaeeef1ec3e8d5b4e7030706e3e3d6b5bee6049732e0e6",
-                "sha256:b46aaf0332a8a9c965310058cf3487d705bf672641d2c43a835625b326689cf4",
-                "sha256:c39fa911e4302eb79c804b221ddec775c3da08833c0a9120041dd322789824de",
-                "sha256:ca56a6c8c8236d6fe19abb67ef08d76f3c3f46712c49a3b6a5352b6e43e8855f",
-                "sha256:cb502cde018e93e75dc8fc7bb2d93477ce4f3ac10369f48866c61b5e031db1fd",
-                "sha256:cd4d50a27b065447c9c399f0bf0a993bd0e6308db8bbbfbc3ea03b41c145775a",
-                "sha256:d125e754d26c0298715bdc3f8a03a0658ecbe72330be247f4b328d229d8cf67f",
-                "sha256:d300773b93eed82f6da138fd1d081dc96fbe53d96000a85e41460fe07c8d8b33",
-                "sha256:d396b610e77b0c438846607cd56418bfc194973b9886550a98fd6724e8c6cfec",
-                "sha256:d61482b5d18181e6bb4810b4a6a24c63a490c3a20e9fbd7876639653e2b30a1a",
-                "sha256:d9f2c27f18a0b94107d57294aab3d06d6046ea843ed4a45cae8bd45756749f3a",
-                "sha256:dc2b3f06430cbd4fac0dae5b2974d2bf14f71b415fb6de017f498950da8159b1",
-                "sha256:dc935d8322ba9bc7b84f99f40f111809b0473df167bf5b93b89fb719d2c4892b",
-                "sha256:e333c5b62e93949f5ac27e6758ba53ef6ee4f93e36cc977fe2e3df85c02f6dc4",
-                "sha256:e765b1f47293dedf77946f0427e03ee45def2862edacd8868c6cf9ab97c8afbd",
-                "sha256:ed18728b90758d171f0c66c475c24a443ede815cf3f1a91e907b0db0ebc6e508",
-                "sha256:eff87c68058374e45225089e4538c26329a13499bc0104b52b77f8428eed36b2",
-                "sha256:f05d05d99fce5537d8f7a0af6417a9afa9af3a6c4bb1ba7359c53b6257625fcb",
-                "sha256:f253edf694ce836631b350d758d00a8c4011243d58318fbfbe0dd54a6a839ab4",
-                "sha256:f41915a4e1f059dfad614b187bc06021fefb5fc5255bfe63abf8247d2f7a646a",
-                "sha256:f96def94576f857abf58e031ce881b5a3fc25cbec64b2bc4824824a8a4367af9"
-            ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.19.1"
+                "sha256:0405984f3ec1d3f8777c4adc33eac7ab7a3e629f3b1c05fdded63acc7cf01137",
+                "sha256:0436a70d8eb42bea4fe1a1c32d371d9bb3b62c637969cb33970ad624d5a3336a",
+                "sha256:061e81ea2d62671fa9dea2c2bfbc1eec2617ae7651e366c7b4a2baf0a8c72cae",
+                "sha256:064300a4ea17d1cd9ea1706aa0590dcb3be81112aac30233823ee494f02cb78a",
+                "sha256:08889f2f597ae965284d7b52a5c3928653a9406d88c93e3161180f0abc2433ba",
+                "sha256:0a48679310e1dd5c9f03481799311a65d343748fe86850b7fb41df4e2c00c087",
+                "sha256:0b0a3eb6dd39cce23801a50c01a0976971498da49bc8a0590ce311492b82c44b",
+                "sha256:0d2d5119b1d7a1ed286b8af37357116072fc96700bce3bec5bb81b2e7057ab41",
+                "sha256:0d551dc931638e2102b8549836a1632e6e7cf620af3d093a7456aa642bff601d",
+                "sha256:1018bd0d70ce85f165185d2227c71e3b1e446186f9fa9f971b69eee223e1e3cd",
+                "sha256:11c39fbc4280d7420684494373b7c5904fa72a2b48ef543a56c2d412999c9e5d",
+                "sha256:11cc3afd8160d44582543838b7e4f9aa5e97865322844b75d51bf4e0e413bb3e",
+                "sha256:1537b3dd62d8aae644f3518c407aa8469e3fd0f179cdf86c5992792713ed717a",
+                "sha256:16ca9c90da4b1f50f089e14485db8c20cbfff2d55424062791a7392b5a9b3ff9",
+                "sha256:176a1b524a3bd3314ed47029a86d02d5a95cc0bee15bd3063a1e1ec62b947de6",
+                "sha256:18955c1da6fc39d957adfa346f75226246b6569e096ac9e40f67d102278c3bcb",
+                "sha256:1bb5b50dc6dd671eb46a605a3e2eb98deb4a9af787a08fcdddabe5d824bb9664",
+                "sha256:1c768e7584c45094dca4b334af361e43b0aaa4844c04945ac7d43379eeda9bc2",
+                "sha256:1dd4f692304854352c3e396e9b5f0a9c9e666868dd0bdc784e2ac4c93092d87b",
+                "sha256:25785d038281cd106c0d91a68b9930049b6464288cea59ba95b35ee37c2d23a5",
+                "sha256:287e39ba24e141b046812c880f4619d0ca9e617235d74abc27267194fc0c7835",
+                "sha256:2c1467d939932901a97ba4f979e8f2642415fcf02ea12f53a4e3206c9c03bc17",
+                "sha256:2c433a412e96afb9a3ce36fa96c8e61a757af53e9c9192c97392f72871e18e69",
+                "sha256:2d022b14d7758bfb98405672953fe5c202ea8a9ccf9f6713c5bd0718eba286fd",
+                "sha256:2f98d918f7f3aaf4b91f2b08c0c92b1774aea113334f7cde4fe40e777114dbe6",
+                "sha256:2fc697be37585eded0c8581c4788fcfac0e3f84ca635b73a5bf360e28c8ea1a2",
+                "sha256:3194cd0d2c959062b94094c0a9f8780ffd38417a5322450a0db0ca1a23e7fbd2",
+                "sha256:332c848f02d71a649272b3f1feccacb7e4f7e6de4a2e6dc70a32645326f3d428",
+                "sha256:346820ae96aa90c7d52653539a57766f10f33dd4be609206c001432b59ddf89f",
+                "sha256:3471e95110dcaf901db16063b2e40fb394f8a9e99b3fe9ee3acc6f6ef72183a2",
+                "sha256:3848427b65e31bea2c11f521b6fc7a3145d6e501a1038529da2391aff5970f2f",
+                "sha256:39b6d79f5cbfa3eb63a869639cfacf7c41d753c64f7801efc72692c1b2637ac7",
+                "sha256:3e74355cb47e0cd399ead3477e29e2f50e1540952c22fb3504dda0184fc9819f",
+                "sha256:3f39bb1f6e620f3e158c8b2eaf1b3e3e54408baca96a02fe891794705e788637",
+                "sha256:40847f617287a38623507d08cbcb75d51cf9d4f9551dd6321df40215128325a3",
+                "sha256:4280e460e51f86ad76dc456acdbfa9513bdf329556ffc8c49e0200878ca57816",
+                "sha256:445a96543948c011a3a47c8e0f9d61e9785df2544ea5be5ab3bc2be4bd8a2565",
+                "sha256:4969d974d9db826a2c07671273e6b27bc48e940738d768fa8f33b577f0978378",
+                "sha256:49aaf4546f6023c44d7e7136be84a03a4237f0b2b5fb2b17c3e3770a758fc1a0",
+                "sha256:49e0e3faf3070abdf71a5c80a97c1afc059b4f45a5aa62de0c2ca0444b51669b",
+                "sha256:49f9da0d6cd17b600a178439d7d2d57c5ef01f816b1e0e875e8e8b3b42db2693",
+                "sha256:4a8c3cc4f9dfc33220246760358c8265dad6e1104f25f0077bbca692d616d358",
+                "sha256:4d36081c0b1c12ea0ed62c202046dca11438bee48dd5240b7c8de8da62c620e9",
+                "sha256:4edcd0bf70087b244ba77038db23cd98a1ace2f91b4a3ecef22036314d77ac23",
+                "sha256:554313db34d63eac3b3f42986aa9efddd1a481169c12b7be1e7512edebff8eaf",
+                "sha256:5675e9d8eeef0aa06093c1ff898413ade042d73dc920a03e8cea2fb68f62445a",
+                "sha256:60848ab779195b72382841fc3fa4f71698a98d9589b0a081a9399904487b5832",
+                "sha256:66e5dc13bfb17cd6ee764fc96ccafd6e405daa846a42baab81f4c60e15650414",
+                "sha256:6779105d2fcb7fcf794a6a2a233787f6bbd4731227333a072d8513b252ed374f",
+                "sha256:6ad331349b0b9ca6da86064a3599c425c7a21cd41616e175ddba0866da32df48",
+                "sha256:6f0a0b41dd05eefab547576bed0cf066595f3b20b083956b1405a6f17d1be6ad",
+                "sha256:73a8a4653f2e809049999d63530180d7b5a344b23a793502413ad1ecea9a0290",
+                "sha256:778331444917108fa8441f59af45886270d33ce8a23bfc4f9b192c0b2ecef1b3",
+                "sha256:7cb98be113911cb0ad09e5523d0e2a926c09a465c9abb0784c9269efe4f95917",
+                "sha256:7d74beca677623481810c7052926365d5f07393c72cbf62d6cce29991b676402",
+                "sha256:7f2398361508c560d0bf1773af19e9fe644e218f2a814a02210ac2c97ad70db0",
+                "sha256:8434dcdd347459f9fd9c526117c01fe7ca7b016b6008dddc3c13471098f4f0dc",
+                "sha256:8a390e56a7963e3946ff2049ee1eb218380e87c8a0e7608f7f8790ba19390867",
+                "sha256:92c4a4a2b1f4846cd4364855cbac83efc48ff5a7d7c06ba014c792dd96483f6f",
+                "sha256:9300aee2a8b5992d0f4293d88deb59c218989833e3396c824b69ba330d04a589",
+                "sha256:9453419ea2ab9b21d925d0fd7e3a132a178a191881fab4169b6f96e118cc25bb",
+                "sha256:9652e59c022e62a5b58a6f9948b104e5bb96d3b06940c6482588176f40f4914b",
+                "sha256:972a7833d4a1fcf7a711c939e315721a88b988553fc770a5b6a5a64bd6ebeba3",
+                "sha256:9c1a4393242e321e344213a90a1e3bf35d2f624aa8b8f6174d43e3c6b0e8f6eb",
+                "sha256:9e038c615b3906df4c3be8db16b3e24821d26c55177638ea47b3f8f73615111c",
+                "sha256:9e4c166f743bb42c5fcc60760fb1c3623e8fda94f6619534217b083e08644b46",
+                "sha256:9eb117db8d7ed733a7317c4215c35993b815bf6aeab67523f1f11e108c040672",
+                "sha256:9eb442a2442ce417801c912df68e1f6ccfcd41577ae7274953ab3ad24ef7d82c",
+                "sha256:a3cd18e03b0ee54ea4319cdcce48357719ea487b53f92a469ba8ca8e39df285e",
+                "sha256:a8617625369d2d03766413bff9e64310feafc9fc4f0ad2b902136f1a5cd8c6b0",
+                "sha256:a970a2e6d5281d56cacf3dc82081c95c1f4da5a559e52469287457811db6a79b",
+                "sha256:aad7405c033d32c751d98d3a65801e2797ae77fac284a539f6c3a3e13005edc4",
+                "sha256:adcb3332979cbc941b8fff07181f06d2b608625edc0a4d8bc3ffc0be414ad0c4",
+                "sha256:af9c7e6669c4d0ad7362f79cb2ab6784d71147503e62b57e3d95c4a0f222c01c",
+                "sha256:b01fda3e95d07a6148702a641e5e293b6da7863f8bc9b967f62db9461330562c",
+                "sha256:b8d940fd28eb34a7084877747a60873956893e377f15a32ad445fe66c972c3b8",
+                "sha256:bccb3e88ec26ffa90f72229f983d3a5d1155e41a1171190fa723d4135523585b",
+                "sha256:bcedf4cae0d47839fee7de344f96b5694ca53c786f28b5f773d4f0b265a159eb",
+                "sha256:be893258d5b68dd3a8cba8deb35dc6411db844a9d35268a8d3793b9d9a256f80",
+                "sha256:c0521e0f07cb56415fdb3aae0bbd8701eb31a9dfef47bb57206075a0584ab2a2",
+                "sha256:c594642d6b13d225e10df5c16ee15b3398e21a35ecd6aee824f107a625690374",
+                "sha256:c87c22bd6a987aca976e3d3e23806d17f65426191db36d40da4ae16a6a494cbc",
+                "sha256:c9ac1c2678abf9270e7228133e5b77c6c3c930ad33a3c1dfbdd76ff2c33b7b50",
+                "sha256:d0e5ffc763678d48ecc8da836f2ae2dd1b6eb2d27a48671066f91694e575173c",
+                "sha256:d0f402e787e6e7ee7876c8b05e2fe6464820d9f35ba3f172e95b5f8b699f6c7f",
+                "sha256:d222a9ed082cd9f38b58923775152003765016342a12f08f8c123bf893461f28",
+                "sha256:d94245caa3c61f760c4ce4953cfa76e7739b6f2cbfc94cc46fff6c050c2390c5",
+                "sha256:de9a2792612ec6def556d1dc621fd6b2073aff015d64fba9f3e53349ad292734",
+                "sha256:e2f5a398b5e77bb01b23d92872255e1bcb3c0c719a3be40b8df146570fe7781a",
+                "sha256:e8dd53a8706b15bc0e34f00e6150fbefb35d2fd9235d095b4f83b3c5ed4fa11d",
+                "sha256:e9eb3cff1b7d71aa50c89a0536f469cb8d6dcdd585d8f14fb8500d822f3bdee4",
+                "sha256:ed628c1431100b0b65387419551e822987396bee3c088a15d68446d92f554e0c",
+                "sha256:ef7938a78447174e2616be223f496ddccdbf7854f7bf2ce716dbccd958cc7d13",
+                "sha256:f1c70249b15e4ce1a7d5340c97670a95f305ca79f376887759b43bb33288c973",
+                "sha256:f3c7363a8cb8c5238878ec96c5eb0fc5ca2cb11fc0c7d2379863d342c6ee367a",
+                "sha256:fbbcc6b0639aa09b9649f36f1bcb347b19403fe44109948392fbb5ea69e48c3e",
+                "sha256:febffa5b1eda6622d44b245b0685aff6fb555ce0ed734e2d7b1c3acd018a2cff",
+                "sha256:ff836cd4041e16003549449cc0a5e372f6b6f871eb89007ab0ee18fb2800fded"
+            ],
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==3.19.2"
         },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.16.0"
         },
         "sniffio": {
@@ -3477,10 +3519,10 @@
         },
         "texttable": {
             "hashes": [
-                "sha256:290348fb67f7746931bcdfd55ac7584ecd4e5b0846ab164333f0794b121760f2",
-                "sha256:b7b68139aa8a6339d2c320ca8b1dc42d13a7831a346b446cb9eb385f0c76310c"
+                "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638",
+                "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917"
             ],
-            "version": "==1.6.7"
+            "version": "==1.7.0"
         },
         "tinycss2": {
             "hashes": [
@@ -3534,11 +3576,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:07ab9c5bf8a0499fd7b088ba51be899c90ffc936ffc797d7b6907fc516bcd116",
-                "sha256:db9c4aa58139c3ba850101913915c042bdba86f7c8a0dda1c6f7f92c5da8e542"
+                "sha256:7564b5bf8d38c40fa45498072bf4dc5e8346eb087bbf1e2ae2d8774f6a0f078e",
+                "sha256:98277f247f18b2c5cabaf4af369187754f4fb0e85911d473f72329db8a7f4fae"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.10.1"
+            "version": "==5.11.2"
         },
         "typeguard": {
             "hashes": [
@@ -3590,7 +3632,6 @@
                 "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21",
                 "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"
             ],
-            "index": "pypi",
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==1.26.17"
         },
@@ -3625,11 +3666,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
-                "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
+                "sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24",
+                "sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.6.3"
+            "version": "==1.6.4"
         },
         "whitebox": {
             "hashes": [
@@ -3747,82 +3788,90 @@
         },
         "xyzservices": {
             "hashes": [
-                "sha256:0ec928742227d6f5d4367ea7b457fcfed943429f4de2949b5b02a82cdf5569d6",
-                "sha256:88e9cbf22b31a2f9c1b242e2b18690f5c705f0e539c9bfd37a10399e1037731b"
+                "sha256:70b9910f6c8e46f6ca92dea21e9b8cf89edf0ead35a870198fb59a7d63579525",
+                "sha256:eee203e91955782fd8bfc2f05846830c289139dc0ab4eaf733bfa8f0be71861f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.7.0"
+            "version": "==2023.10.0"
         },
         "y-py": {
             "hashes": [
-                "sha256:035ba7ce31bb87bd7b5977eee71ee2ff71e54d347a35e2079362b1c23731dccd",
-                "sha256:0624a5adf29d29330a336eecdf15874871f559d50944d542012665e1c3a18265",
-                "sha256:11345294820908d5b8af9c6616ea908dda8b3e554ee6f6d50be6a2e15940f63e",
-                "sha256:1667b8a67ace756c04f03778e86fc359028c98905212f8686afb48c26c252bda",
-                "sha256:19b7c3eaf65b162e59486a48bea5dd2035937952f15e008a14813e8cb7c24d7b",
-                "sha256:1ea7d796bb55d08dd1a60736beb724004f2cbdc207592b5f301a5ff314b17137",
-                "sha256:1eefb6371cd6e072cf467b897f85bd0d7575f3a3e944fb8675f84fb59aedd071",
-                "sha256:1fb03947937b0fcb09eb2b94eb08d8e8030ef0ed70af777684ab670bd369bc3c",
-                "sha256:281535bb4f18fe09e5517a63b8206dd6f26ad6fb7e7c25c62bf785e594adab4d",
-                "sha256:2c230bc01b96081550b7583b77d00404fd39825657f4064b919a10515f660cdf",
-                "sha256:304e88a3deaff9906faa7ba514cf82f4ca4bad1ea88728206ff906e66179abd3",
-                "sha256:30b9337e4f3d541879a8187af121be1bd42ea110372a21895a1a3f800a6bd1c3",
-                "sha256:374ffef1939c42286ea18e2a413c9974430226227f8f1480bbee469933aa675b",
-                "sha256:37b9f24b00972e5685d0b9bbd01413d9c33d124145343fb92667f0e076f040ad",
-                "sha256:391a232c328c2be1de4cb152ed3e9427826e4cbd9d645feacb3dbb344b122e10",
-                "sha256:4181b28f736cae3bb4517090ae5eeca318c075c0106466f13a4ed6682265fc8a",
-                "sha256:418aaa796a22b0102de09b36b6c6294d0a485f04bc8866c3b28f17e7022c44ba",
-                "sha256:46836169f7dc2957df8513cfe4bc2009175b3a473e630af421a8e75ee1c48f98",
-                "sha256:479da40ef1205de52d87209534bf8e713a782e01eeed3df8dff44d21085e3f63",
-                "sha256:47b3604c874d25616a097adaaabcad6e77729e23c5d029092b8149af1a08b2a5",
-                "sha256:49adf7e25c3b3bac9f19bee181ef5253659ebe5747a7141860692015222b2007",
-                "sha256:4c16d50d0728abd915bd9e2e0c3ce982005ba78b60e4b6666aadc592d9982c79",
-                "sha256:4cac9259839b32706336b3f521cacfd16fc7cefee609bd9c2b5123099328d696",
-                "sha256:4f025c50301d9ddbbc2384f98d3ff1dbfe43606146b747e23a17774a02faffe9",
-                "sha256:513a2fe1318c247fc3b3c3ad208488e870a216784f2a3e6dbe2688c92f671c86",
-                "sha256:51f1997dae6d77b12b50502871c7a9aae22e84048e83b64fe6d4f18dec2e4700",
-                "sha256:5743e94c982585f05e02d9a3345dd9b1f28d90fa128df9f60b0eb357a76d2c32",
-                "sha256:6377e3cbab8f5b8b918130e9f924358f98ca1bea12a8096d3fadea191f7137f1",
-                "sha256:69e05e01594e99c934562124b159720533b7ad887dde7762d460916aac47a8e4",
-                "sha256:6a5a882591c8e1b1d6fbdb7ab43884907cef2b6a18e36c7ae85589e5f55371e5",
-                "sha256:71f7689c25bd7608e1e7a76a13138cb202455fac165018693a3e8e5675f54b82",
-                "sha256:74d5ebb5f9ef0c4c1f7bdd9ab5e53b9d8be4c7464905f39761b22b6ce0d327d3",
-                "sha256:76e2b14004cadb237499a8a068fd7a8b805b5c1fd0508530473e087c7dd25163",
-                "sha256:83115cbbd4f6d3b38ebe06d80b1d0dbf1b10e53947f71df16f6145a4f0d14716",
-                "sha256:8f143fdcda7a6a89bf96d9b359142a7ca3315e8a9018aa46b0abbdeb47d7192e",
-                "sha256:9242f3a5c6293e634817d9984c60523ffb34cf5b41501c5958681a75745946e6",
-                "sha256:95083c4cdbd593497a695e841b2ad050c0b9a8a9e374f8496aa478cebfcf9cc9",
-                "sha256:97812f9443fd846012d60ecacffa2a11992d02ad9f8618d4faae8e596736c646",
-                "sha256:9a920bf096d1eecb0f30afc38ee56bfcb9e2c863c33db96fc9d30d4ac0dbee58",
-                "sha256:9dad6af2d83a2b0618ba3c1a2fc6657c5303cf4e9f1a65cc3fea40ffbcc552e2",
-                "sha256:9f56888aeb07ca76a5cd552581bb3735fcd2d8c18165b946fdb6e4507b10e76c",
-                "sha256:a027c39296c925f0b81e28a0fefab8c5964a0ea2b50fa05cbddf5e5ab167a380",
-                "sha256:a20a4d10c8f0ee2b6df265d182d0be0ecd2ba7348c0a20b9df7d4d39df895801",
-                "sha256:a4b488be17d83173acb7f07c7e3430d2c66d0bd55b821683089311699562b58b",
-                "sha256:a752ba8875ed2038dfc7d62738536cb22b4e308951cb925a7fe8fef782c6db08",
-                "sha256:aaaec9718f8a23924c95294d41d87829b113bc9a606a3667dfb995afc45c9920",
-                "sha256:b44fdd64598e9ed4008158e5e60be5e1e2daeed6fae0ab2bf0002461e960709d",
-                "sha256:b5f5975c1a8c2ca99980571b8811d151db8590de9cc96346572a81e0f6f1e30e",
-                "sha256:b6273d84605ee55b3ac52742018f94602dab9b0457f29e6f787021c473b02fed",
-                "sha256:b71cd495d322da25a53a6a830b591a2c0c46db22bb0b3556fca0bbdb1d45a18e",
-                "sha256:b75c2199a125ef8926f3216fb324c3bcd8b1b4b6c0b428888cc753ee4c85f81f",
-                "sha256:c0505e2ca36408b754774a2bb20d93b5c7def3873406c13e1855de6f007f8a94",
-                "sha256:c276a7eb3ae3360f5a2fc503f1e4535d4a2f1c8cfc22af4595ad752e9a94fd77",
-                "sha256:cca539c3804a580992304b18a33f1980282d9097a723f0bd01971477cb365b28",
-                "sha256:d1301bfeaa26f78f4b0e5f96e0f22761b38cc407713f70550a1be490945fd6d7",
-                "sha256:e48b5b30242c7d517be85b48246b21e4e26540505a1ffe4fe473e239a8ec56d3",
-                "sha256:e5126786f914ff53ea2f04f9da790db168db172521cc4f114d5501badd2f6b96",
-                "sha256:e5f89cf9ef1daf12f438a075415a02f227594e4b0494c78d3b83cb83651631f5",
-                "sha256:e76be7258010ce8cbb93a841f78f52901bba1253a51213d3535972d13aa4e89e",
-                "sha256:eb60fe68774117378efdbd368ef83cf1417e61d4bc39c6be8e7f4ee91fb7428a",
-                "sha256:ebbebc4f6a9e0c89c7b57035f91043b038e804dd1953845d8a66066f4526c853",
-                "sha256:eccf67d09a4df42a7be2a5427c1b2e0b89bec862f519ded754bd452df516b380",
-                "sha256:ef0f08edb2094869e4d12346ee68d5154cb3d23bc3b1e7679222fae12228261c",
-                "sha256:efb3225b58dc67152c004da3c26ae5bad0afebbb3c7509d853bdd87eaa655137",
-                "sha256:f79ef7303e332e91d738e66e9bb7fce0243d0407a02631a58ebc0bf2fb8743d0",
-                "sha256:fc48db294d327a5cc10ee49f73f1fa1478240cc827c9029e0871106e327353ac"
-            ],
-            "version": "==0.6.0"
+                "sha256:015f7f6c1ce8a83d57955d1dc7ddd57cb633ae00576741a4fc9a0f72ed70007d",
+                "sha256:032365dfe932bfab8e80937ad6093b4c22e67d63ad880096b5fa8768f8d829ba",
+                "sha256:0649a41cd3c98e290c16592c082dbe42c7ffec747b596172eebcafb7fd8767b0",
+                "sha256:0787e85645bb4986c27e271715bc5ce21bba428a17964e5ec527368ed64669bc",
+                "sha256:0cd6213c3cf2b9eee6f2c9867f198c39124c557f4b3b77d04a73f30fd1277a59",
+                "sha256:0f2d881f0f8bf5674f8fe4774a438c545501e40fa27320c73be4f22463af4b05",
+                "sha256:17bce637a89f6e75f0013be68becac3e38dc082e7aefaf38935e89215f0aa64a",
+                "sha256:17edd21eef863d230ea00004ebc6d582cc91d325e7132deb93f0a90eb368c855",
+                "sha256:1d5b544e79ace93fdbd0b36ed329c86e346898153ac7ba2ec62bc9b4c6b745c9",
+                "sha256:1f798165158b76365a463a4f8aa2e3c2a12eb89b1fc092e7020e93713f2ad4dc",
+                "sha256:266ec46ab9f9cb40fbb5e649f55c329fc4620fa0b1a8117bdeefe91595e182dc",
+                "sha256:26cb1307c3ca9e21a3e307ab2c2099677e071ae9c26ec10ddffb3faceddd76b3",
+                "sha256:2a497ebe617bec6a420fc47378856caae40ab0652e756f3ed40c5f1fe2a12220",
+                "sha256:2b4fac4ea2ce27b86d173ae45765ced7f159120687d4410bb6d0846cbdb170a3",
+                "sha256:2cf817a72ffec4295def5c5be615dd8f1e954cdf449d72ebac579ff427951328",
+                "sha256:2d2b054a1a5f4004967532a4b82c6d1a45421ef2a5b41d35b6a8d41c7142aabe",
+                "sha256:316e5e1c40259d482883d1926fd33fa558dc87b2bd2ca53ce237a6fe8a34e473",
+                "sha256:35fcb9def6ce137540fdc0e91b08729677548b9c393c0151a6359fd199da3bd7",
+                "sha256:376c5cc0c177f03267340f36aec23e5eaf19520d41428d87605ca2ca3235d845",
+                "sha256:3ba99d0bdbd9cabd65f914cd07b4fb2e939ce199b54ae5ace1639ce1edf8e0a2",
+                "sha256:3c011303eb2b360695d2bd4bd7ca85f42373ae89fcea48e7fa5b8dc6fc254a98",
+                "sha256:4757a82a50406a0b3a333aa0122019a331bd6f16e49fed67dca423f928b3fd4d",
+                "sha256:47fcc19158150dc4a6ae9a970c5bc12f40b0298a2b7d0c573a510a7b6bead3f3",
+                "sha256:4c28d977f516d4928f6bc0cd44561f6d0fdd661d76bac7cdc4b73e3c209441d9",
+                "sha256:5415083f7f10eac25e1c434c87f07cb9bfa58909a6cad6649166fdad21119fc5",
+                "sha256:613f83713714972886e81d71685403098a83ffdacf616f12344b52bc73705107",
+                "sha256:69cfbcbe0a05f43e780e6a198080ba28034bf2bb4804d7d28f71a0379bfd1b19",
+                "sha256:6c2f2831c5733b404d2f2da4bfd02bb4612ae18d0822e14ae79b0b92436b816d",
+                "sha256:7227f232f2daf130ba786f6834548f2cfcfa45b7ec4f0d449e72560ac298186c",
+                "sha256:72875641a907523d37f4619eb4b303611d17e0a76f2ffc423b62dd1ca67eef41",
+                "sha256:7c7302619fc962e53093ba4a94559281491c045c925e5c4defec5dac358e0568",
+                "sha256:7cbefd4f1060f05768227ddf83be126397b1d430b026c64e0eb25d3cf50c5734",
+                "sha256:80a827e173372682959a57e6b8cc4f6468b1a4495b4bc7a775ef6ca05ae3e8e8",
+                "sha256:82f2e5b31678065e7a7fa089ed974af5a4f076673cf4f414219bdadfc3246a21",
+                "sha256:82f5ca62bedbf35aaf5a75d1f53b4457a1d9b6ff033497ca346e2a0cedf13d14",
+                "sha256:8448da4092265142662bbd3fc46cb8b0796b1e259189c020bc8f738899abd0b5",
+                "sha256:863e175ce5585f9ff3eba2aa16626928387e2a576157f02c8eb247a218ecdeae",
+                "sha256:86422c6090f34906c062fd3e4fdfdccf3934f2922021e979573ae315050b4288",
+                "sha256:898fede446ca1926b8406bdd711617c2aebba8227ee8ec1f0c2f8568047116f7",
+                "sha256:8f5c14d25611b263b876e9ada1701415a13c3e9f02ea397224fbe4ca9703992b",
+                "sha256:8f6071328aad06fdcc0a4acc2dc4839396d645f5916de07584af807eb7c08407",
+                "sha256:932abb560fe739416b50716a72ba6c6c20b219edded4389d1fc93266f3505d4b",
+                "sha256:9b7cafbe946b4cafc1e5709957e6dd5c6259d241d48ed75713ded42a5e8a4663",
+                "sha256:9b8822a5c0fd9a8cffcabfcc0cd7326bad537ee614fc3654e413a03137b6da1a",
+                "sha256:a21148b8ea09a631b752d975f9410ee2a31c0e16796fdc113422a6d244be10e5",
+                "sha256:a3932f53418b408fa03bd002e6dc573a74075c2c092926dde80657c39aa2e054",
+                "sha256:a70aee572da3994238c974694767365f237fc5949a550bee78a650fe16f83184",
+                "sha256:ae80d505aee7b3172cdcc2620ca6e2f85586337371138bb2b71aa377d2c31e9a",
+                "sha256:b2686d7d8ca31531458a48e08b0344a8eec6c402405446ce7d838e2a7e43355a",
+                "sha256:bae1b1ad8d2b8cf938a60313f8f7461de609621c5dcae491b6e54975f76f83c5",
+                "sha256:bd302c6d46a3be57664571a5f0d4224646804be9890a01d73a0b294f2d3bbff1",
+                "sha256:beea5ad9bd9e56aa77a6583b6f4e347d66f1fe7b1a2cb196fff53b7634f9dc84",
+                "sha256:bf6020560584671e76375b7a0539e0d5388fc70fa183c99dc769895f7ef90233",
+                "sha256:c011997f62d0c3b40a617e61b7faaaf6078e4eeff2e95ce4c45838db537816eb",
+                "sha256:c08311db17647a47d4898fc6f8d9c1f0e58b927752c894877ff0c38b3db0d6e1",
+                "sha256:c26bada6cd109095139237a46f50fc4308f861f0d304bc9e70acbc6c4503d158",
+                "sha256:c31240e30d5636ded02a54b7280aa129344fe8e964fd63885e85d9a8a83db206",
+                "sha256:ce0ae49879d10610cf3c40f4f376bb3cc425b18d939966ac63a2a9c73eb6f32a",
+                "sha256:ce15a842c2a0bf46180ae136743b561fa276300dd7fa61fe76daf00ec7dc0c2d",
+                "sha256:ce7c20b9395696d3b5425dccf2706d374e61ccf8f3656bff9423093a6df488f5",
+                "sha256:cfc8381df1f0f873da8969729974f90111cfb61a725ef0a2e0e6215408fe1217",
+                "sha256:d1dca48687f41efd862355e58b0aa31150586219324901dbea2989a506e291d4",
+                "sha256:d3bbe2f925cc587545c8d01587b4523177408edd252a32ce6d61b97113fe234d",
+                "sha256:d917f5bc27b85611ceee4eb85f0e4088b0a03b4eed22c472409933a94ee953cf",
+                "sha256:dab84c52f64e10adc79011a08673eb80286c159b14e8fb455524bf2994f0cb38",
+                "sha256:de9cfafe97c75cd3ea052a24cd4aabf9fb0cfc3c0f9f810f00121cdf123db9e4",
+                "sha256:df35ea436592eb7e30e59c5403ec08ec3a5e7759e270cf226df73c47b3e739f5",
+                "sha256:e13cba03c7af8c8a846c4495875a09d64362cc4caeed495ada5390644411bbe7",
+                "sha256:e1935d12e503780b859d343161a80df65205d23cad7b4f6c3df6e50321e188a3",
+                "sha256:e42258f66ad9f16d9b62e9c9642742982acb1f30b90f5061522048c1cb99814f",
+                "sha256:e794e44fa260300b8850246c6371d94014753c73528f97f6ccb42f5e7ce698ae",
+                "sha256:e8638355ae2f996356f7f281e03a3e3ce31f1259510f9d551465356532e0302c",
+                "sha256:e92878cc05e844c8da937204bc34c2e6caf66709ce5936802fbfb35f04132892",
+                "sha256:ff32548e45e45bf3280ac1d28b3148337a5c6714c28db23aeb0693e33eba257e"
+            ],
+            "version": "==0.6.2"
         },
         "ypy-websocket": {
             "hashes": [
@@ -3953,7 +4002,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.16.0"
         },
         "stack-data": {
@@ -3965,11 +4014,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:07ab9c5bf8a0499fd7b088ba51be899c90ffc936ffc797d7b6907fc516bcd116",
-                "sha256:db9c4aa58139c3ba850101913915c042bdba86f7c8a0dda1c6f7f92c5da8e542"
+                "sha256:7564b5bf8d38c40fa45498072bf4dc5e8346eb087bbf1e2ae2d8774f6a0f078e",
+                "sha256:98277f247f18b2c5cabaf4af369187754f4fb0e85911d473f72329db8a7f4fae"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.10.1"
+            "version": "==5.11.2"
         },
         "typing-extensions": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2d0844de4..1f09873dd 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.4.1 - 2023-10-26 - [PR#1007](https://github.com/NOAA-OWP/inundation-mapping/pull/1007)
+
+Updates GVAL to address memory and performance issues associated with running synthesize test cases.
+
+### Changes
+
+- `tools/tools_shared_functions.py`
+- `Pipfile`
+- `pyproject.toml`
+- `tools/run_test_case.py`
+- `tools/synthesize_test_cases.py`
+- `tools/inundate_mosaic_wrapper`
+
+<br/><br/>
+
 ## v4.4.4.0 - 2023-10-20 - [PR#1012](https://github.com/NOAA-OWP/inundation-mapping/pull/1012)
 
 The way in which watershed boundary data (WBD) is generated and processed has been modified. Instead of generating those files "on the fly" for every run, a script has been added that will take a huclist and create the .gpkg files per HUC in a specified directory (`$pre_clip_huc_dir`).  During a `fim_pipeline.sh` run, the pre-clipped staged vectors will be copied over to the containers' working directory. This reduces runtime and the repetitive computation needed to generate those files every run.
diff --git a/pyproject.toml b/pyproject.toml
index f6d623ec0..0476fe148 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -74,9 +74,10 @@ per-file-ignores = """
     src/agreedem.py: E712
     src/build_stream_traversal.py: E722
     tools/vary_mannings_n_composite.py: E712
-    tools/tools_shared_functions.py: F821, F841
+    tools/tools_shared_functions.py: F821, F841, E711
     tools/rating_curve_get_usgs_curves.py: F841
     tools/rating_curve_comparison.py: F821, F841
+    tools/run_test_case.py: E711
     tools/inundation.py: F821
     tools/eval_alt_catfim.py: F841
     tools/check_deep_flooding.py: E712
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
index 48da26996..1620a7820 100644
--- a/tools/inundate_mosaic_wrapper.py
+++ b/tools/inundate_mosaic_wrapper.py
@@ -1,7 +1,6 @@
 import argparse
 import errno
 import os
-import sys
 from timeit import default_timer as timer
 
 from inundate_gms import Inundate_gms
@@ -70,6 +69,9 @@ def produce_mosaicked_inundation(
     if not os.path.exists(hydrofabric_dir):
         raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), hydrofabric_dir)
 
+    # If the "hucs" argument is really one huc, convert it to a list
+    if type(hucs) == str:
+        hucs = [hucs]
     # Check that huc folder exists in the hydrofabric_dir.
     for huc in hucs:
         if not os.path.exists(os.path.join(hydrofabric_dir, huc)):
diff --git a/tools/run_test_case.py b/tools/run_test_case.py
index 89fe04da2..a7cb7701e 100755
--- a/tools/run_test_case.py
+++ b/tools/run_test_case.py
@@ -5,6 +5,7 @@
 import re
 import shutil
 import sys
+import traceback
 
 import pandas as pd
 from inundate_mosaic_wrapper import produce_mosaicked_inundation
@@ -24,7 +25,7 @@
 from utils.shared_functions import FIM_Helpers as fh
 
 
-class benchmark(object):
+class Benchmark(object):
     AHPS_BENCHMARK_CATEGORIES = AHPS_BENCHMARK_CATEGORIES
     MAGNITUDE_DICT = MAGNITUDE_DICT
 
@@ -42,27 +43,27 @@ def __init__(self, category):
             self.MAGNITUDE_DICT.keys()
         ), f"Category must be one of {list(self.MAGNITUDE_DICT.keys())}"
         self.validation_data = os.path.join(
-            TEST_CASES_DIR, f"{self.category}_test_cases", f"validation_data_{self.category}"
+            TEST_CASES_DIR, f'{self.category}_test_cases', f'validation_data_{self.category}'
         )
         self.is_ahps = True if self.category in self.AHPS_BENCHMARK_CATEGORIES else False
 
     def magnitudes(self):
-        """Returns the magnitudes associated with the benchmark category."""
+        '''Returns the magnitudes associated with the benchmark category.'''
         return self.MAGNITUDE_DICT[self.category]
 
     def huc_data(self):
-        """Returns a dict of HUC8, magnitudes, and sites."""
+        '''Returns a dict of HUC8, magnitudes, and sites.'''
         huc_mags = {}
         for huc in os.listdir(self.validation_data):
-            if not re.match(r"\d{8}", huc):
+            if not re.match(r'\d{8}', huc):
                 continue
             huc_mags[huc] = self.data(huc)
         return huc_mags
 
     def data(self, huc):
-        """Returns a dict of magnitudes and sites for a given huc. Sites will be AHPS lids for
+        '''Returns a dict of magnitudes and sites for a given huc. Sites will be AHPS lids for
         AHPS sites and empty strings for non-AHPS sites.
-        """
+        '''
         huc_dir = os.path.join(self.validation_data, huc)
         if not os.path.isdir(huc_dir):
             return {}
@@ -80,10 +81,10 @@ def data(self, huc):
             return mag_dict
         else:
             mags = list(os.listdir(huc_dir))
-            return {mag: [""] for mag in mags}
+            return {mag: [''] for mag in mags}
 
 
-class Test_Case(benchmark):
+class Test_Case(Benchmark):
     def __init__(self, test_id, version, archive=True):
         """Class that handles test cases, specifically running the alpha test.
 
@@ -92,7 +93,7 @@ def __init__(self, test_id, version, archive=True):
         test_id : str
             ID of the test case in huc8_category format, e.g. `12090201_ble`.
         version : str
-            Version of FIM to which this Test_Case belongs. This should correspond to the fim directory
+            Version of FIM to which this test_case belongs. This should correspond to the fim directory
             name in either `/data/previous_fim/` or `/outputs/`.
         archive : bool
             If true, this test case outputs will be placed into the `official_versions` folder
@@ -102,7 +103,7 @@ def __init__(self, test_id, version, archive=True):
 
         """
         self.test_id = test_id
-        self.huc, self.benchmark_cat = test_id.split("_")
+        self.huc, self.benchmark_cat = test_id.split('_')
         super().__init__(self.benchmark_cat)
         self.version = version
         self.archive = archive
@@ -110,14 +111,14 @@ def __init__(self, test_id, version, archive=True):
         self.fim_dir = os.path.join(
             PREVIOUS_FIM_DIR if archive else OUTPUTS_DIR,
             self.version,
-            self.huc if not re.search("^fim_[1,2]", version, re.IGNORECASE) else self.huc[:6],
+            self.huc if not re.search('^fim_[1,2]', version, re.IGNORECASE) else self.huc[:6],
         )
         # Test case directory path
         self.dir = os.path.join(
             TEST_CASES_DIR,
-            f"{self.benchmark_cat}_test_cases",
+            f'{self.benchmark_cat}_test_cases',
             test_id,
-            "official_versions" if archive else "testing_versions",
+            'official_versions' if archive else 'testing_versions',
             version,
         )
         if not os.path.exists(self.dir):
@@ -127,15 +128,15 @@ def __init__(self, test_id, version, archive=True):
 
         # Create list of shapefile paths to use as exclusion areas.
         self.mask_dict = {
-            "levees": {
-                "path": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
-                "buffer": None,
-                "operation": "exclude",
+            'levees': {
+                'path': '/data/inputs/nld_vectors/Levee_protected_areas.gpkg',
+                'buffer': None,
+                'operation': 'exclude',
             },
-            "waterbodies": {
-                "path": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
-                "buffer": None,
-                "operation": "exclude",
+            'waterbodies': {
+                'path': '/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg',
+                'buffer': None,
+                'operation': 'exclude',
             },
         }
 
@@ -159,26 +160,26 @@ def list_all_test_cases(cls, version, archive, benchmark_categories=[]):
 
         test_case_list = []
         for bench_cat in benchmark_categories:
-            benchmark_class = benchmark(bench_cat)
+            benchmark_class = Benchmark(bench_cat)
             benchmark_data = benchmark_class.huc_data()
 
             for huc in benchmark_data.keys():
-                test_case_list.append(cls(f"{huc}_{bench_cat}", version, archive))
+                test_case_list.append(cls(f'{huc}_{bench_cat}', version, archive))
 
         return test_case_list
 
     def alpha_test(
         self,
         calibrated=False,
-        model="",
-        mask_type="huc",
-        inclusion_area="",
+        model='',
+        mask_type='huc',
+        inclusion_area='',
         inclusion_area_buffer=0,
         overwrite=True,
         verbose=False,
         gms_workers=1,
     ):
-        """Compares a FIM directory with benchmark data from a variety of sources.
+        '''Compares a FIM directory with benchmark data from a variety of sources.
 
         Parameters
         ----------
@@ -198,7 +199,7 @@ def alpha_test(
             If True, prints out all pertinent data.
         gms_workers : int
             Number of worker processes assigned to GMS processing.
-        """
+        '''
 
         try:
             if not overwrite and os.path.isdir(self.dir):
@@ -207,47 +208,47 @@ def alpha_test(
 
             fh.vprint(f"Starting alpha test for {self.dir}", verbose)
 
-            self.stats_modes_list = ["total_area"]
+            self.stats_modes_list = ['total_area']
 
             # Create paths to fim_run outputs for use in inundate()
-            if model != "GMS":
-                self.rem = os.path.join(self.fim_dir, "rem_zeroed_masked.tif")
+            if model != 'GMS':
+                self.rem = os.path.join(self.fim_dir, 'rem_zeroed_masked.tif')
                 if not os.path.exists(self.rem):
-                    self.rem = os.path.join(self.fim_dir, "rem_clipped_zeroed_masked.tif")
+                    self.rem = os.path.join(self.fim_dir, 'rem_clipped_zeroed_masked.tif')
                 self.catchments = os.path.join(
-                    self.fim_dir, "gw_catchments_reaches_filtered_addedAttributes.tif"
+                    self.fim_dir, 'gw_catchments_reaches_filtered_addedAttributes.tif'
                 )
                 if not os.path.exists(self.catchments):
                     self.catchments = os.path.join(
-                        self.fim_dir, "gw_catchments_reaches_clipped_addedAttributes.tif"
+                        self.fim_dir, 'gw_catchments_reaches_clipped_addedAttributes.tif'
                     )
                 self.mask_type = mask_type
-                if mask_type == "huc":
-                    self.catchment_poly = ""
+                if mask_type == 'huc':
+                    self.catchment_poly = ''
                 else:
                     self.catchment_poly = os.path.join(
-                        self.fim_dir, "gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg"
+                        self.fim_dir, 'gw_catchments_reaches_filtered_addedAttributes_crosswalked.gpkg'
                     )
-                self.hydro_table = os.path.join(self.fim_dir, "hydroTable.csv")
+                self.hydro_table = os.path.join(self.fim_dir, 'hydroTable.csv')
 
             # Map necessary inputs for inundate().
-            self.hucs, self.hucs_layerName = (os.path.join(INPUTS_DIR, "wbd", "WBD_National.gpkg"), "WBDHU8")
+            self.hucs, self.hucs_layerName = os.path.join(INPUTS_DIR, 'wbd', 'WBD_National.gpkg'), 'WBDHU8'
 
-            if inclusion_area != "":
-                inclusion_area_name = os.path.split(inclusion_area)[1].split(".")[0]  # Get layer name
+            if inclusion_area != '':
+                inclusion_area_name = os.path.split(inclusion_area)[1].split('.')[0]  # Get layer name
                 self.mask_dict.update(
                     {
                         inclusion_area_name: {
-                            "path": inclusion_area,
-                            "buffer": int(inclusion_area_buffer),
-                            "operation": "include",
+                            'path': inclusion_area,
+                            'buffer': int(inclusion_area_buffer),
+                            'operation': 'include',
                         }
                     }
                 )
                 # Append the concatenated inclusion_area_name and buffer.
-                if inclusion_area_buffer is None:
+                if inclusion_area_buffer == None:
                     inclusion_area_buffer = 0
-                self.stats_modes_list.append(inclusion_area_name + "_b" + str(inclusion_area_buffer) + "m")
+                self.stats_modes_list.append(inclusion_area_name + '_b' + str(inclusion_area_buffer) + 'm')
 
             # Delete the directory if it exists
             if os.path.exists(self.dir):
@@ -277,12 +278,14 @@ def alpha_test(
             sys.exit(1)
         except Exception as ex:
             print(ex)
+            # Temporarily adding stack trace
+            print(f"trace for {self.test_id} -------------\n", traceback.format_exc())
             sys.exit(1)
 
     def _inundate_and_compute(
-        self, magnitude, lid, compute_only=False, model="", verbose=False, gms_workers=1
+        self, magnitude, lid, compute_only=False, model='', verbose=False, gms_workers=1
     ):
-        """Method for inundating and computing contingency rasters as part of the alpha_test.
+        '''Method for inundating and computing contingency rasters as part of the alpha_test.
         Used by both the alpha_test() and composite() methods.
 
          Parameters
@@ -293,19 +296,19 @@ def _inundate_and_compute(
              lid of the current benchmark site. For non-AHPS sites, this should be an empty string ('').
          compute_only : bool
              If true, skips inundation and only computes contingency stats.
-        """
+        '''
         # Output files
         fh.vprint("Creating output files", verbose)
 
         test_case_out_dir = os.path.join(self.dir, magnitude)
-        inundation_prefix = lid + "_" if lid else ""
-        inundation_path = os.path.join(test_case_out_dir, f"{inundation_prefix}inundation_extent.tif")
-        predicted_raster_path = inundation_path.replace(".tif", f"_{self.huc}.tif")
+        inundation_prefix = lid + '_' if lid else ''
+        inundation_path = os.path.join(test_case_out_dir, f'{inundation_prefix}inundation_extent.tif')
+        predicted_raster_path = inundation_path.replace('.tif', f'_{self.huc}.tif')
         agreement_raster = os.path.join(
-            test_case_out_dir, (f"ahps_{lid}" if lid else "") + "total_area_agreement.tif"
+            test_case_out_dir, (f'ahps_{lid}' if lid else '') + 'total_area_agreement.tif'
         )
-        stats_json = os.path.join(test_case_out_dir, "stats.json")
-        stats_csv = os.path.join(test_case_out_dir, "stats.csv")
+        stats_json = os.path.join(test_case_out_dir, 'stats.json')
+        stats_csv = os.path.join(test_case_out_dir, 'stats.csv')
 
         # Create directory
         if not os.path.isdir(test_case_out_dir):
@@ -313,14 +316,14 @@ def _inundate_and_compute(
 
         # Benchmark raster and flow files
         benchmark_rast = (
-            f"ahps_{lid}" if lid else self.benchmark_cat
-        ) + f"_huc_{self.huc}_extent_{magnitude}.tif"
+            f'ahps_{lid}' if lid else self.benchmark_cat
+        ) + f'_huc_{self.huc}_extent_{magnitude}.tif'
         benchmark_rast = os.path.join(self.benchmark_dir, lid, magnitude, benchmark_rast)
-        benchmark_flows = benchmark_rast.replace(f"_extent_{magnitude}.tif", f"_flows_{magnitude}.csv")
+        benchmark_flows = benchmark_rast.replace(f'_extent_{magnitude}.tif', f'_flows_{magnitude}.csv')
         mask_dict_indiv = self.mask_dict.copy()
         if self.is_ahps:  # add domain shapefile to mask for AHPS sites
-            domain = os.path.join(self.benchmark_dir, lid, f"{lid}_domain.shp")
-            mask_dict_indiv.update({lid: {"path": domain, "buffer": None, "operation": "include"}})
+            domain = os.path.join(self.benchmark_dir, lid, f'{lid}_domain.shp')
+            mask_dict_indiv.update({lid: {'path': domain, 'buffer': None, 'operation': 'include'}})
         # Check to make sure all relevant files exist
         if (
             not os.path.isfile(benchmark_rast)
@@ -388,15 +391,15 @@ def run_alpha_test(
         calibrated,
         model,
         archive_results=False,
-        mask_type="huc",
-        inclusion_area="",
+        mask_type='huc',
+        inclusion_area='',
         inclusion_area_buffer=0,
         light_run=False,
         overwrite=True,
         verbose=False,
         gms_workers=1,
     ):
-        """Class method for instantiating the Test_Case class and running alpha_test directly"""
+        '''Class method for instantiating the test_case class and running alpha_test directly'''
 
         alpha_class = cls(test_id, version, archive_results)
         alpha_class.alpha_test(
@@ -411,7 +414,7 @@ def run_alpha_test(
         )
 
     def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
-        """Class method for compositing MS and FR inundation and creating an agreement raster with stats
+        '''Class method for compositing MS and FR inundation and creating an agreement raster with stats
 
         Parameters
         ----------
@@ -421,12 +424,12 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
             Whether or not this FIM version is calibrated.
         overwrite : bool
             If True, overwites pre-existing test cases within the test_cases directory.
-        """
+        '''
 
-        if re.match(r"(.*)(_ms|_fr)", self.version):
-            composite_version_name = re.sub(r"(.*)(_ms|_fr)", r"\1_comp", self.version, count=1)
+        if re.match(r'(.*)(_ms|_fr)', self.version):
+            composite_version_name = re.sub(r'(.*)(_ms|_fr)', r'\1_comp', self.version, count=1)
         else:
-            composite_version_name = re.sub(r"(.*)(_ms|_fr)", r"\1_comp", version_2, count=1)
+            composite_version_name = re.sub(r'(.*)(_ms|_fr)', r'\1_comp', version_2, count=1)
 
         fh.vprint(f"Begin composite for version : {composite_version_name}", verbose)
 
@@ -446,28 +449,28 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
             for instance in validation_data[
                 magnitude
             ]:  # instance will be the lid for AHPS sites and '' for other sites (ble/ifc/ras2fim)
-                inundation_prefix = instance + "_" if instance else ""
+                inundation_prefix = instance + '_' if instance else ''
 
                 input_inundation = os.path.join(
-                    self.dir, magnitude, f"{inundation_prefix}inundation_extent_{self.huc}.tif"
+                    self.dir, magnitude, f'{inundation_prefix}inundation_extent_{self.huc}.tif'
                 )
                 input_inundation_2 = os.path.join(
                     input_test_case_2.dir,
                     magnitude,
-                    f"{inundation_prefix}inundation_extent_{input_test_case_2.huc}.tif",
+                    f'{inundation_prefix}inundation_extent_{input_test_case_2.huc}.tif',
                 )
                 output_inundation = os.path.join(
-                    composite_test_case.dir, magnitude, f"{inundation_prefix}inundation_extent.tif"
+                    composite_test_case.dir, magnitude, f'{inundation_prefix}inundation_extent.tif'
                 )
 
                 if os.path.isfile(input_inundation) and os.path.isfile(input_inundation_2):
                     inundation_map_file = pd.DataFrame(
                         {
-                            "huc8": [composite_test_case.huc] * 2,
-                            "branchID": [None] * 2,
-                            "inundation_rasters": [input_inundation, input_inundation_2],
-                            "depths_rasters": [None] * 2,
-                            "inundation_polygons": [None] * 2,
+                            'huc8': [composite_test_case.huc] * 2,
+                            'branchID': [None] * 2,
+                            'inundation_rasters': [input_inundation, input_inundation_2],
+                            'depths_rasters': [None] * 2,
+                            'inundation_polygons': [None] * 2,
                         }
                     )
                     os.makedirs(os.path.dirname(output_inundation), exist_ok=True)
@@ -475,10 +478,10 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
                     fh.vprint(f"Begin mosaic inundation for version : {composite_version_name}", verbose)
                     Mosaic_inundation(
                         inundation_map_file,
-                        mosaic_attribute="inundation_rasters",
+                        mosaic_attribute='inundation_rasters',
                         mosaic_output=output_inundation,
                         mask=None,
-                        unit_attribute_name="huc8",
+                        unit_attribute_name='huc8',
                         nodata=elev_raster_ndv,
                         workers=1,
                         remove_inputs=False,
@@ -492,32 +495,32 @@ def composite(self, version_2, calibrated=False, overwrite=True, verbose=False):
                     single_test_case = self if os.path.isfile(input_inundation) else input_test_case_2
                     shutil.copytree(
                         single_test_case.dir,
-                        re.sub(r"(.*)(_ms|_fr)", r"\1_comp", single_test_case.dir, count=1),
+                        re.sub(r'(.*)(_ms|_fr)', r'\1_comp', single_test_case.dir, count=1),
                     )
-                    composite_test_case.write_metadata(calibrated, "COMP")
+                    composite_test_case.write_metadata(calibrated, 'COMP')
                     return
 
             # Clean up 'total_area' outputs from AHPS sites
             if composite_test_case.is_ahps:
                 composite_test_case.clean_ahps_outputs(os.path.join(composite_test_case.dir, magnitude))
 
-        composite_test_case.write_metadata(calibrated, "COMP")
+        composite_test_case.write_metadata(calibrated, 'COMP')
 
     def write_metadata(self, calibrated, model):
-        """Writes metadata files for a test_case directory."""
-        with open(os.path.join(self.dir, "eval_metadata.json"), "w") as meta:
-            eval_meta = {"calibrated": calibrated, "model": model}
+        '''Writes metadata files for a test_case directory.'''
+        with open(os.path.join(self.dir, 'eval_metadata.json'), 'w') as meta:
+            eval_meta = {'calibrated': calibrated, 'model': model}
             meta.write(json.dumps(eval_meta, indent=2))
 
     def clean_ahps_outputs(self, magnitude_directory):
-        """Cleans up `total_area` files from an input AHPS magnitude directory."""
+        '''Cleans up `total_area` files from an input AHPS magnitude directory.'''
         output_file_list = [os.path.join(magnitude_directory, of) for of in os.listdir(magnitude_directory)]
         for output_file in output_file_list:
             if "total_area" in output_file:
                 os.remove(output_file)
 
     def get_current_agreements(self):
-        """Returns a list of all agreement rasters currently existing for the test_case."""
+        '''Returns a list of all agreement rasters currently existing for the test_case.'''
         agreement_list = []
         for mag in os.listdir(self.dir):
             mag_dir = os.path.join(self.dir, mag)
@@ -525,6 +528,6 @@ def get_current_agreements(self):
                 continue
 
             for f in os.listdir(mag_dir):
-                if "agreement.tif" in f:
+                if 'agreement.tif' in f:
                     agreement_list.append(os.path.join(mag_dir, f))
         return agreement_list
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index ceb7e427f..dfe0dac7e 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -123,11 +123,11 @@ def create_master_metrics_csv(
             magnitude_list = MAGNITUDE_DICT[benchmark_source]
 
             # Iterate through available test cases
-            for test_case in test_cases_list:
+            for each_test_case in test_cases_list:
                 try:
                     # Get HUC id
-                    int(test_case.split('_')[0])
-                    huc = test_case.split('_')[0]
+                    int(each_test_case.split('_')[0])
+                    huc = each_test_case.split('_')[0]
 
                     # Update filepaths based on whether the official or dev versions should be included
                     for iteration in iteration_list:
@@ -135,7 +135,7 @@ def create_master_metrics_csv(
                             iteration == "official"
                         ):  # and str(pfiles) == "True": # "official" refers to previous finalized model versions
                             versions_to_crawl = os.path.join(
-                                benchmark_test_case_dir, test_case, 'official_versions'
+                                benchmark_test_case_dir, each_test_case, 'official_versions'
                             )
                             versions_to_aggregate = prev_versions_to_include_list
 
@@ -143,7 +143,7 @@ def create_master_metrics_csv(
                             iteration == "testing"
                         ):  # "testing" refers to the development model version(s) being evaluated
                             versions_to_crawl = os.path.join(
-                                benchmark_test_case_dir, test_case, 'testing_versions'
+                                benchmark_test_case_dir, each_test_case, 'testing_versions'
                             )
                             versions_to_aggregate = dev_versions_to_include_list
 
@@ -190,17 +190,17 @@ def create_master_metrics_csv(
         if benchmark_source in AHPS_BENCHMARK_CATEGORIES:
             test_cases_list = os.listdir(benchmark_test_case_dir)
 
-            for test_case in test_cases_list:
+            for each_test_case in test_cases_list:
                 try:
                     # Get HUC id
-                    int(test_case.split('_')[0])
-                    huc = test_case.split('_')[0]
+                    int(each_test_case.split('_')[0])
+                    huc = each_test_case.split('_')[0]
 
                     # Update filepaths based on whether the official or dev versions should be included
                     for iteration in iteration_list:
                         if iteration == "official":  # "official" refers to previous finalized model versions
                             versions_to_crawl = os.path.join(
-                                benchmark_test_case_dir, test_case, 'official_versions'
+                                benchmark_test_case_dir, each_test_case, 'official_versions'
                             )
                             versions_to_aggregate = prev_versions_to_include_list
 
@@ -208,7 +208,7 @@ def create_master_metrics_csv(
                             iteration == "testing"
                         ):  # "testing" refers to the development model version(s) being evaluated
                             versions_to_crawl = os.path.join(
-                                benchmark_test_case_dir, test_case, 'testing_versions'
+                                benchmark_test_case_dir, each_test_case, 'testing_versions'
                             )
                             versions_to_aggregate = dev_versions_to_include_list
 
@@ -448,7 +448,9 @@ def progress_bar_handler(executor_dict, verbose, desc):
         required=False,
         default=None,
     )
-    parser.add_argument('-vr', '--verbose', help='Verbose', required=False, default=None, action='store_true')
+    parser.add_argument(
+        '-vr', '--verbose', help='Verbose output', required=False, default=None, action='store_true'
+    )
     parser.add_argument(
         '-vg',
         '--gms-verbose',
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 08af7ee9e..96a7e24bd 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 import json
 import os
 import pathlib
@@ -16,7 +17,9 @@
 from dotenv import load_dotenv
 from geocube.api.core import make_geocube
 from gval import CatStats
+from rasterio import features
 from rasterio.warp import Resampling, calculate_default_transform, reproject
+from shapely.geometry import MultiPolygon, Polygon, shape
 
 
 def get_env_paths():
@@ -29,15 +32,14 @@ def get_env_paths():
 
 def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df):
     """
-    This function uses the WRDS API to filter out NWM segments from a list if their stream order is
-    different than the target stream order.
+    This function uses the WRDS API to filter out NWM segments from a list if their stream order is different than
+    the target stream order.
 
     Args:
         unfiltered_segments (list):  A list of NWM feature_id strings.
-        desired_order (str):         The desired stream order.
+        desired_order (str): The desired stream order.
     Returns:
-        filtered_segments (list):    A list of NWM feature_id strings, paired down to only those that
-                                        share the target order.
+        filtered_segments (list): A list of NWM feature_id strings, paired down to only those that share the target order.
 
     """
 
@@ -90,8 +92,7 @@ def compute_contingency_stats_from_rasters(
     """
     This function contains FIM-specific logic to prepare raster datasets for use in the generic
     get_stats_table_from_binary_rasters() function. This function also calls the generic
-    compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON,
-    depending on user input.
+    compute_stats_from_contingency_table() function and writes the results to CSV and/or JSON, depending on user input.
 
     Parameters
     ----------
@@ -100,24 +101,26 @@ def compute_contingency_stats_from_rasters(
     benchmark_raster_path: str
         The path to the benchmark, or truth, FIM extent raster.
     agreement_raster: str, optional
-        An agreement raster will be written to this path. 0: True Negatives, 1: False Negative,
-        2: False Positive, 3: True Positive.
+        An agreement raster will be written to this path. 0: True Negatives, 1: False Negative, 2: False Positive,
+        3: True Positive.
     stats_csv: str, optional
-        Performance statistics will be written to this path. CSV allows for readability and
-        other tabular processes.
+        Performance statistics will be written to this path. CSV allows for readability and other tabular processes.
     stats_json: str, optional
-        Performance statistics will be written to this path. JSON allows for quick ingestion into Python
-        dictionary in other processes.
+        Performance statistics will be written to this path. JSON allows for quick ingestion into Python dictionary
+        in other processes.
     mask_dict: dict
         Dictionary with inclusionary and/or exclusionary masks asn options.
 
     Returns
     -------
     dict
-        A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are
-        keys and statistic values are the values.
+        A dictionary of statistics produced by compute_stats_from_contingency_table(). Statistic names are keys and
+        statistic values are the values.
     """
 
+    # print('candidate', predicted_raster_path)
+    # print('benchmark', benchmark_raster_path)
+
     # Get statistics table from two rasters.
     stats_dictionary = get_stats_table_from_binary_rasters(
         benchmark_raster_path, predicted_raster_path, agreement_raster, mask_dict=mask_dict
@@ -125,13 +128,13 @@ def compute_contingency_stats_from_rasters(
 
     for stats_mode in stats_dictionary:
         # Write the mode_stats_dictionary to the stats_csv.
-        if stats_csv is not None:
+        if stats_csv != None:
             stats_csv = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.csv')
             df = pd.DataFrame.from_dict(stats_dictionary[stats_mode], orient="index", columns=['value'])
             df.to_csv(stats_csv)
 
         # Write the mode_stats_dictionary to the stats_json.
-        if stats_json is not None:
+        if stats_json != None:
             stats_json = os.path.join(os.path.split(stats_csv)[0], stats_mode + '_stats.json')
             with open(stats_json, "w") as outfile:
                 json.dump(stats_dictionary[stats_mode], outfile)
@@ -144,15 +147,13 @@ def profile_test_case_archive(archive_to_check, magnitude, stats_mode):
     This function searches multiple directories and locates previously produced performance statistics.
 
     Args:
-        archive_to_check (str):    The directory path to search.
-        magnitude (str):           Because a benchmark dataset may have multiple magnitudes,
-                                   this argument defines which magnitude is to be used when searching for
-                                   previous statistics.
+        archive_to_check (str): The directory path to search.
+        magnitude (str): Because a benchmark dataset may have multiple magnitudes, this argument defines
+                               which magnitude is to be used when searching for previous statistics.
     Returns:
-        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain
-                                   and magnitude. {version: {agreement_raster: agreement_raster_path,
-                                   stats_csv: stats_csv_path, stats_json: stats_json_path}}
-                                   *Will only add the paths to files that exist.
+        archive_dictionary (dict): A dictionary of available statistics for previous versions of the domain and magnitude.
+                                  {version: {agreement_raster: agreement_raster_path, stats_csv: stats_csv_path, stats_json: stats_json_path}}
+                                  *Will only add the paths to files that exist.
 
     """
 
@@ -179,23 +180,30 @@ def compute_stats_from_contingency_table(
     true_negatives, false_negatives, false_positives, true_positives, cell_area=None, masked_count=None
 ):
     """
-    This generic function takes contingency table metrics as arguments and returns a dictionary of contingency
-    table statistics. Much of the calculations below were taken from older Python files.
-    This is evident in the inconsistent use of case.
+    This generic function takes contingency table metrics as arguments and returns a dictionary of contingency table statistics.
+    Much of the calculations below were taken from older Python files. This is evident in the inconsistent use of case.
 
-    Args:
-        true_negatives (int):      The true negatives from a contingency table.
-        false_negatives (int):     The false negatives from a contingency table.
-        false_positives (int):     The false positives from a contingency table.
-        true_positives (int):      The true positives from a contingency table.
-        cell_area (float or None): This optional argument allows for area-based statistics to be calculated,
-                                   in the case that contingency table metrics were derived from areal
-                                   analysis.
+    Parameters
+    ----------
+    true_negatives: int
+        The true negatives from a contingency table.
+    false_negatives: int
+        The false negatives from a contingency table.
+    false_positives: int
+        The false positives from a contingency table.
+    true_positives: int
+        The true positives from a contingency table.
+    cell_area: float, default = None
+        This optional argument allows for area-based statistics to be calculated, in the case that contingency
+        table metrics were derived from areal analysis.
+    masked_count: int, default = None
+        Amount of pixels masked out of array
 
-    Returns:
-        stats_dictionary (dict):   A dictionary of statistics. Statistic names are keys and statistic values
-                                   are the values. Refer to dictionary definition in bottom of function for
-                                   statistic names.
+    Returns
+    -------
+    dict
+        A dictionary of statistics. Statistic names are keys and statistic values are the values.
+        Refer to dictionary definition in bottom of function for statistic names.
 
     """
 
@@ -337,8 +345,7 @@ def get_stats_table_from_binary_rasters(
     benchmark_raster_path: str, candidate_raster_path: str, agreement_raster: str = None, mask_dict: dict = {}
 ):
     """
-    Produces categorical statistics table from 2 rasters and returns it.
-    Also exports an agreement raster classified as:
+    Produces categorical statistics table from 2 rasters and returns it. Also exports an agreement raster classified as:
         0: True Negatives
         1: False Negative
         2: False Positive
@@ -349,8 +356,7 @@ def get_stats_table_from_binary_rasters(
     Parameters
     ----------
     benchmark_raster_path: str
-        Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present,
-        NoData = NoData.
+        Path to the binary benchmark raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
     candidate_raster_path: str
         Path to the predicted raster. 0 = phenomena not present, 1 = phenomena present, NoData = NoData.
     agreement_raster: str, default = None.
@@ -367,36 +373,44 @@ def get_stats_table_from_binary_rasters(
     """
 
     # Load benchmark and candidate data
-    benchmark_raster = rxr.open_rasterio(benchmark_raster_path, mask_and_scale=True)
+    benchmark_raster = rxr.open_rasterio(benchmark_raster_path)
     cell_area = np.abs(np.prod(benchmark_raster.rio.resolution()))
-    candidate_raster = rxr.open_rasterio(candidate_raster_path, mask_and_scale=True)
-    candidate_raster.data = xr.where(candidate_raster >= 0, 1, candidate_raster)
-    candidate_raster.data = xr.where(candidate_raster < 0, 0, candidate_raster)
+    candidate_raster = rxr.open_rasterio(candidate_raster_path)
+    candidate_raster.data = xr.where(
+        (candidate_raster != candidate_raster.rio.nodata) & (candidate_raster >= 0), 1, candidate_raster
+    )
+    candidate_raster.data = xr.where(
+        (candidate_raster != candidate_raster.rio.nodata) & (candidate_raster < 0), 0, candidate_raster
+    )
+    candidate_raster.data = xr.where(candidate_raster == candidate_raster.rio.nodata, 10, candidate_raster)
+    candidate_raster = candidate_raster.rio.write_nodata(10)
+    benchmark_raster.data = xr.where(benchmark_raster == benchmark_raster.rio.nodata, 10, benchmark_raster)
+    benchmark_raster = benchmark_raster.rio.write_nodata(10)
 
     pairing_dictionary = {
         (0, 0): 0,
         (0, 1): 1,
-        (0, np.nan): np.nan,
+        (0, 10): 10,
         (1, 0): 2,
         (1, 1): 3,
-        (1, np.nan): np.nan,
+        (1, 10): 10,
         (4, 0): 4,
         (4, 1): 4,
-        (4, np.nan): np.nan,
-        (np.nan, 0): np.nan,
-        (np.nan, 1): np.nan,
-        (np.nan, np.nan): np.nan,
+        (4, 10): 10,
+        (10, 0): 10,
+        (10, 1): 10,
+        (10, 10): 10,
     }
 
     # Loop through exclusion masks and mask the agreement_array.
-    rasterized_mask_list = []
+    all_masks_df = None
     if mask_dict != {}:
         for poly_layer in mask_dict:
             operation = mask_dict[poly_layer]['operation']
 
             if operation == 'exclude':
                 poly_path = mask_dict[poly_layer]['path']
-                buffer_val = mask_dict[poly_layer]['buffer']
+                buffer_val = 0 if mask_dict[poly_layer]['buffer'] is None else mask_dict[poly_layer]['buffer']
 
                 # Read mask bounds with candidate boundary box
                 poly_all = gpd.read_file(poly_path, bbox=candidate_raster.rio.bounds())
@@ -411,38 +425,48 @@ def get_stats_table_from_binary_rasters(
                 poly_all_proj = poly_all.to_crs(candidate_raster.rio.crs)
 
                 # Buffer if buffer val exists
-                poly_all_proj = poly_all_proj.buffer(buffer_val) if buffer_val else poly_all_proj
-
-                poly_all_proj['mask'] = 4
+                poly_all_proj = poly_all_proj.buffer(buffer_val) if buffer_val != 0 else poly_all_proj
 
-                rasterized_mask_list.append(make_geocube(poly_all_proj, ['mask'], like=candidate_raster))
+                if all_masks_df is not None:
+                    all_masks_df = pd.concat([all_masks_df, poly_all_proj])
+                else:
+                    all_masks_df = poly_all_proj
 
                 del poly_all, poly_all_proj
 
-    og_data = candidate_raster.data
-    if len(rasterized_mask_list) > 0:
-        all_masks = xr.merge(rasterized_mask_list).to_array()
-        # Hold on to original data
+    stats_table_dictionary = {}  # Initialize empty dictionary.
+
+    c_aligned, b_aligned = candidate_raster.gval.homogenize(benchmark_raster, target_map="candidate")
+    del candidate_raster, benchmark_raster
 
-        candidate_raster.data = xr.where(
-            (all_masks.data == 4) & (candidate_raster.isnull() == 0), 4, candidate_raster
+    agreement_map = c_aligned.gval.compute_agreement_map(
+        b_aligned, comparison_function='pairing_dict', pairing_dict=pairing_dictionary
+    )
+    del c_aligned, b_aligned
+
+    agreement_map_og = agreement_map.copy()
+    agreement_map.rio.write_nodata(4, inplace=True)
+
+    # Mask if mask_dict is provided
+    if all_masks_df is not None:
+        agreement_map = agreement_map.rio.clip(all_masks_df['geometry'], invert=True)
+        agreement_map.data = xr.where(
+            agreement_map_og.sel({'x': agreement_map.coords['x'], 'y': agreement_map.coords['y']}) == 10,
+            10,
+            agreement_map,
         )
 
-    stats_table_dictionary = {}  # Initialize empty dictionary.
+    crosstab_table = agreement_map.gval.compute_crosstab()
 
-    (agreement_map, crosstab_table, metrics_table) = candidate_raster.gval.categorical_compare(
-        benchmark_map=benchmark_raster,
-        positive_categories=[1],
-        target_map="candidate",
-        negative_categories=[0],
-        comparison_function='pairing_dict',
-        pairing_dict=pairing_dictionary,
+    metrics_table = crosstab_table.gval.compute_categorical_metrics(
+        positive_categories=[1], negative_categories=[0], metrics="all"
     )
 
     # Only write the agreement raster if user-specified.
-    if agreement_raster is not None:
-        agreement_map = agreement_map.rio.write_nodata(10, encoded=True)
-        agreement_map.rio.to_raster(agreement_raster, dtype=np.int32, driver="COG")
+    if agreement_raster != None:
+        agreement_map_write = agreement_map.rio.write_nodata(10, encoded=True)
+        agreement_map_write.rio.to_raster(agreement_raster, dtype=np.int32, driver="COG")
+        del agreement_map_write
 
         # Write legend text file
         legend_txt = os.path.join(os.path.split(agreement_raster)[0], 'read_me.txt')
@@ -472,7 +496,7 @@ def get_stats_table_from_binary_rasters(
         }
     )
 
-    del agreement_map, crosstab_table, metrics_table, all_masks, rasterized_mask_list
+    del crosstab_table, metrics_table
 
     # After agreement_array is masked with default mask layers, check for inclusion masks in mask_dict.
     if mask_dict != {}:
@@ -481,10 +505,10 @@ def get_stats_table_from_binary_rasters(
 
             if operation == 'include':
                 poly_path = mask_dict[poly_layer]['path']
-                buffer_val = mask_dict[poly_layer]['buffer']
+                buffer_val = 0 if mask_dict[poly_layer]['buffer'] is None else mask_dict[poly_layer]['buffer']
 
                 # Read mask bounds with candidate boundary box
-                poly_all = gpd.read_file(poly_path, bbox=candidate_raster.rio.bounds())
+                poly_all = gpd.read_file(poly_path, bbox=agreement_map.rio.bounds())
 
                 # Make sure features are present in bounding box area before projecting.
                 # Continue to next layer if features are absent.
@@ -492,37 +516,38 @@ def get_stats_table_from_binary_rasters(
                     del poly_all
                     continue
 
-                poly_all_proj = poly_all.to_crs(candidate_raster.rio.crs)
+                poly_all_proj = poly_all.to_crs(agreement_map.rio.crs)
 
                 # Buffer if buffer val exists
-                poly_all_proj = poly_all_proj.buffer(buffer_val) if buffer_val else poly_all_proj
+                poly_all_proj = poly_all_proj.buffer(buffer_val) if buffer_val != 0 else poly_all_proj
 
-                poly_all_proj['mask'] = 4
+                poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
 
-                mask = make_geocube(poly_all_proj, ['mask'], like=candidate_raster).to_array()
-                candidate_raster.data = og_data
-                candidate_raster.data = xr.where(
-                    (mask.data != 4) & (candidate_raster.isnull() == 0), 4, candidate_raster
+                # Do analysis on inclusion masked area
+                agreement_map_include = agreement_map.rio.clip(poly_all_proj['geometry'])
+                agreement_map_include.data = xr.where(
+                    agreement_map_og.sel(
+                        {'x': agreement_map_include.coords['x'], 'y': agreement_map_include.coords['y']}
+                    )
+                    == 10,
+                    10,
+                    agreement_map_include,
                 )
 
-                poly_handle = poly_layer + '_b' + str(buffer_val) + 'm'
+                crosstab_table = agreement_map_include.gval.compute_crosstab()
 
-                # Do analysis on inclusion masked area
-                (agreement_map, crosstab_table, metrics_table) = candidate_raster.gval.categorical_compare(
-                    benchmark_map=benchmark_raster,
-                    positive_categories=[1],
-                    target_map="candidate",
-                    negative_categories=[0],
-                    comparison_function='pairing_dict',
-                    pairing_dict=pairing_dictionary,
+                metrics_table = crosstab_table.gval.compute_categorical_metrics(
+                    positive_categories=[1], negative_categories=[0], metrics="all"
                 )
+
                 if agreement_raster:
                     # Write the layer_agreement_raster.
                     layer_agreement_raster = os.path.join(
                         os.path.split(agreement_raster)[0], poly_handle + '_agreement.tif'
                     )
-                    agreement_map = agreement_map.rio.write_nodata(10, encoded=True)
-                    agreement_map.rio.to_raster(layer_agreement_raster, dtype=np.int32, driver="COG")
+                    agreement_map_write = agreement_map_include.rio.write_nodata(10, encoded=True)
+                    agreement_map_write.rio.to_raster(layer_agreement_raster, dtype=np.int32, driver="COG")
+                    del agreement_map_write
 
                 # Update stats table dictionary
                 stats_table_dictionary.update(
@@ -530,14 +555,15 @@ def get_stats_table_from_binary_rasters(
                         poly_handle: cross_walk_gval_fim(
                             metric_df=metrics_table,
                             cell_area=cell_area,
-                            masked_count=np.sum(agreement_map.data == 4),
+                            masked_count=np.sum(agreement_map_include.data == 4),
                         )
                     }
                 )
+                del agreement_map_include
 
-                del poly_all, poly_all_proj, agreement_map, metrics_table, crosstab_table
+                del poly_all, poly_all_proj, metrics_table, crosstab_table
 
-    del candidate_raster, benchmark_raster, og_data
+    del agreement_map
 
     return stats_table_dictionary
 
@@ -696,8 +722,7 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes=False):
             dict_crs = crs_lookup.get(h_datum, 'EPSG:4269_ Assumed')
             # We want to know what sites were assumed, hence the split.
             src_crs, *message = dict_crs.split('_')
-            # Convert dataframe to geodataframe using lat/lon (USGS).
-            #   Add attribute of assigned crs (label ones that are assumed)
+            # Convert dataframe to geodataframe using lat/lon (USGS). Add attribute of assigned crs (label ones that are assumed)
             site_gdf = gpd.GeoDataFrame(
                 df,
                 geometry=gpd.points_from_xy(df['usgs_preferred_longitude'], df['usgs_preferred_latitude']),
@@ -740,12 +765,10 @@ def mainstem_nwm_segs(metadata_url, list_of_sites):
     '''
     Define the mainstems network. Currently a 4 pass approach that probably needs refined.
     Once a final method is decided the code can be shortened. Passes are:
-        1) Search downstream of gages designated as upstream.
-            This is done to hopefully reduce the issue of mapping starting at the nws_lid. 91038 segments
+        1) Search downstream of gages designated as upstream. This is done to hopefully reduce the issue of mapping starting at the nws_lid. 91038 segments
         2) Search downstream of all LID that are rfc_forecast_point = True. Additional 48,402 segments
         3) Search downstream of all evaluated sites (sites with detailed FIM maps) Additional 222 segments
-        4) Search downstream of all sites in HI/PR (locations have no rfc_forecast_point = True)
-            Additional 408 segments
+        4) Search downstream of all sites in HI/PR (locations have no rfc_forecast_point = True) Additional 408 segments
 
     Parameters
     ----------
@@ -933,8 +956,7 @@ def get_thresholds(threshold_url, select_by, selector, threshold='all'):
             if threshold_data:
                 stages = threshold_data['stage_values']
                 flows = threshold_data['calc_flow_values']
-                # Add source information to stages and flows. Flows source inside a nested dictionary.
-                # Remove key once source assigned to flows.
+                # Add source information to stages and flows. Flows source inside a nested dictionary. Remove key once source assigned to flows.
                 stages['source'] = threshold_data.get('metadata').get('threshold_source')
                 flows['source'] = flows.get('rating_curve', {}).get('source')
                 flows.pop('rating_curve', None)
@@ -1028,8 +1050,7 @@ def get_datum(metadata):
     nws_datums['crs'] = metadata['nws_data']['horizontal_datum_name']
     nws_datums['source'] = 'nws_data'
 
-    # Get site and datum information from usgs_data sub-dictionary.
-    # Use consistent naming between USGS and NWS sources.
+    # Get site and datum information from usgs_data sub-dictionary. Use consistent naming between USGS and NWS sources.
     usgs_datums = {}
     usgs_datums['nws_lid'] = metadata['identifiers']['nws_lid']
     usgs_datums['usgs_site_code'] = metadata['identifiers']['usgs_site_code']
@@ -1220,29 +1241,25 @@ def get_rating_curve(rating_curve_url, location_ids):
 def select_grids(dataframe, stages, datum88, buffer):
     '''
     Given a DataFrame (in a specific format), and a dictionary of stages, and the datum (in navd88).
-    loop through the available inundation datasets and find the datasets that are equal to or immediately
-    above the thresholds and only return 1 dataset per threshold (if any).
+    loop through the available inundation datasets and find the datasets that are equal to or immediately above the thresholds and only return 1 dataset per threshold (if any).
 
     Parameters
     ----------
     dataframe : DataFrame
-        DataFrame that has to be in a specific format and contains the stages and paths to the
-        inundation datasets.
+        DataFrame that has to be in a specific format and contains the stages and paths to the inundation datasets.
     stages : DICT
         Dictionary of thresholds (key) and stages (values)
     datum88: FLOAT
         The datum associated with the LID that is pre-converted to NAVD88 (if needed)
     buffer: Float
-        Interval which the uppder bound can be assigned. For example, Threshold + buffer = upper bound.
-        Recommended to make buffer 0.1 greater than desired interval as code selects maps < and not <=
+        Interval which the uppder bound can be assigned. For example, Threshold + buffer = upper bound. Recommended to make buffer 0.1 greater than desired interval as code selects maps < and not <=
 
     Returns
     -------
     maps : DICT
         Dictionary of threshold (key) and inundation dataset path (value)
     map_flows: DICT
-        Dictionary of threshold (key) and flows in CFS rounded to the nearest whole number associated
-        with the selected maps (value)
+        Dictionary of threshold (key) and flows in CFS rounded to the nearest whole number associated with the selected maps (value)
 
     '''
     # Define threshold categories
@@ -1289,9 +1306,7 @@ def select_grids(dataframe, stages, datum88, buffer):
                     map_path = 'No Flow'
                     map_flow = 'No Flow'
 
-            # If the selected value is not a number
-            #   (or interpolated flow is nan caused by elevation of map which is beyond rating curve range),
-            #   then map_path and map_flows are both set to 'No Map'.
+            # If the selected value is not a number (or interpolated flow is nan caused by elevation of map which is beyond rating curve range), then map_path and map_flows are both set to 'No Map'.
             else:
                 map_path = 'No Map'
                 map_flow = 'No Map'
@@ -1318,8 +1333,7 @@ def select_grids(dataframe, stages, datum88, buffer):
 #######################################################################
 def process_extent(extent, profile, output_raster=False):
     '''
-    Convert raster to feature (using raster_to_feature), the footprint is used so all raster values are set
-        to 1 where there is data.
+    Convert raster to feature (using raster_to_feature), the footprint is used so all raster values are set to 1 where there is data.
     fill all donut holes in resulting feature.
     Filled geometry is then converted back to raster using same raster properties as input profile.
     Output raster will have be encoded as follows:
@@ -1400,8 +1414,7 @@ def raster_to_feature(grid, profile_override=False, footprint_only=False):
     profile_override: rasterio Profile
         Default is False, If a rasterio Profile is supplied, it will dictate the transform and crs.
     footprint_only: BOOL
-        If true, dataset will be divided by itself to remove all unique values. If False, all values in
-        grid will be carried through on raster to feature conversion. default = False
+        If true, dataset will be divided by itself to remove all unique values. If False, all values in grid will be carried through on raster to feature conversion. default = False
 
     Returns
     -------
@@ -1502,25 +1515,22 @@ def process_grid(benchmark, benchmark_profile, domain, domain_profile, reference
     )
     # Convert fitted benchmark dataset to boolean. 0 = NODATA Regions and 1 = Data Regions
     benchmark_fit_to_domain_bool = np.where(benchmark_fit_to_domain == benchmark.nodata, 0, 1)
-    # Merge domain datamask and benchmark data mask. New_nodata_value (2) = Domain NO DATA footprint,
-    #   0 = NO DATA for benchmark (within data region of domain), 1 = DATA region of benchmark.
+    # Merge domain datamask and benchmark data mask. New_nodata_value (2) = Domain NO DATA footprint, 0 = NO DATA for benchmark (within data region of domain), 1 = DATA region of benchmark.
     new_nodata_value = 2
     classified_benchmark = np.where(
         domain_arr == domain.nodata, new_nodata_value, benchmark_fit_to_domain_bool
     )
 
-    # Reproject classified benchmark to reference raster crs and resolution.
+    ## Reproject classified benchmark to reference raster crs and resolution.
     # Read in reference raster
     reference = rasterio.open(reference_raster)
-    # Determine the new transform and dimensions of reprojected/resampled classified benchmark dataset whose
-    #   width, height, and bounds are same as domain dataset.
-    (new_benchmark_transform, new_benchmark_width, new_benchmark_height) = calculate_default_transform(
+    # Determine the new transform and dimensions of reprojected/resampled classified benchmark dataset whos width, height, and bounds are same as domain dataset.
+    new_benchmark_transform, new_benchmark_width, new_benchmark_height = calculate_default_transform(
         source_crs, reference.crs, domain.width, domain.height, *domain.bounds, resolution=reference.res
     )
     # Define an empty array that is same dimensions as output by the "calculate_default_transform" command.
     classified_benchmark_projected = np.empty((new_benchmark_height, new_benchmark_width), dtype=np.uint8)
-    # Reproject and resample the classified benchmark dataset. Nearest Neighbor resampling due to integer
-    #   values of classified benchmark.
+    # Reproject and resample the classified benchmark dataset. Nearest Neighbor resampling due to integer values of classified benchmark.
     reproject(
         classified_benchmark,
         destination=classified_benchmark_projected,
@@ -1562,9 +1572,8 @@ def calculate_metrics_from_agreement_raster(agreement_raster):
     for idx, wind in agreement_raster.block_windows(1):
         window_data = agreement_raster.read(1, window=wind)
         values, counts = np.unique(window_data, return_counts=True)
-        # TODO values_counts is not defined, so commented for now...
-        # for val, cts in values_counts:
-        #     totals[val] += cts
+        for val, cts in values_counts:
+            totals[val] += cts
 
     results = dict()
     for digit, count in totals.items():

From af093f9c89f71f11781299d1ecce69db8ffef0c8 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 26 Oct 2023 09:10:18 -0600
Subject: [PATCH 285/359] v4.4.5.0 Remove memory profile system (#1018)

---
 CONTRIBUTING.md                             |   2 +-
 Pipfile                                     |   2 +-
 Pipfile.lock                                | 474 +++++++++-----------
 data/wbd/clip_vectors_to_wbd.py             |   3 +-
 docs/CHANGELOG.md                           |  41 +-
 src/add_crosswalk.py                        |   3 +-
 src/adjust_thalweg_lateral.py               |   3 -
 src/aggregate_by_huc.py                     |   6 +-
 src/agreedem.py                             |   3 -
 src/bathy_src_adjust_topwidth.py            |   1 -
 src/burn_in_levees.py                       |   3 -
 src/check_huc_inputs.py                     |   7 +-
 src/delineate_hydros_and_produce_HAND.sh    |  10 +-
 src/filter_catchments_and_add_attributes.py |   2 -
 src/make_stages_and_catchlist.py            |   3 -
 src/mask_dem.py                             |   3 -
 src/reachID_grid_to_vector_points.py        |   3 +-
 src/run_unit_wb.sh                          |   8 +-
 src/split_flows.py                          |   3 +-
 src/unique_pixel_and_allocation.py          |   3 -
 src/usgs_gage_crosswalk.py                  |   2 -
 src/usgs_gage_unit_setup.py                 |   1 -
 src/utils/shared_functions.py               |  11 -
 unit_tests/clip_vectors_to_wbd_test.py      |   3 +-
 24 files changed, 283 insertions(+), 317 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5cb51d7b5..f51276063 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -3,7 +3,7 @@
 > All contributions to this project will be released to the public domain.
 > By submitting a pull request or filing a bug, issue, or
 > feature request, you are agreeing to comply with this waiver of copyright interest.
-> Details can be found in our [TERMS](TERMS.md) and [LICENSE](LICENSE).
+> Details can be found in our [LICENSE](LICENSE).
 
 
 There are two primary ways to help:
diff --git a/Pipfile b/Pipfile
index 636fbe26c..85d6d7fe8 100755
--- a/Pipfile
+++ b/Pipfile
@@ -20,7 +20,6 @@ tqdm = "==4.65.0"
 seaborn = "==0.12.2"
 python-dotenv = "==1.0.0"
 natsort = "==8.3.1"
-memory-profiler = "==0.61.0"
 xarray = "==2023.1.0"
 netcdf4 = "==1.6.3"
 tables = "==3.8.0"
@@ -43,6 +42,7 @@ black = "==23.7.0"
 pyproject-flake8 = "==6.0.0.post1"
 pre-commit = "==3.3.3"
 isort = "==5.12.0"
+urllib3 = "==1.26.18"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 78c257106..cdb131185 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c05e13cee1824291db432c5f40d2ed7eed74349a1bba77ddb2deb852cd533734"
+            "sha256": "c7f9ee645e067007d14cfcb885a4a4a4c8c720ed6b17f42c07328378ee8c1121"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "markers": "python_version >= '3.7' and python_version < '4'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -115,11 +115,11 @@
         },
         "babel": {
             "hashes": [
-                "sha256:04c3e2d28d2b7681644508f836be388ae49e0cfe91465095340395b60d00f210",
-                "sha256:fbfcae1575ff78e26c7449136f1abbefc3c13ce542eeb13d43d50d8b047216ec"
+                "sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900",
+                "sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.13.0"
+            "version": "==2.13.1"
         },
         "backcall": {
             "hashes": [
@@ -133,7 +133,7 @@
                 "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
                 "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
             ],
-            "markers": "python_full_version >= '3.6.0'",
+            "markers": "python_version >= '3.6'",
             "version": "==4.12.2"
         },
         "black": {
@@ -162,7 +162,6 @@
                 "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==23.7.0"
         },
         "bleach": {
@@ -206,7 +205,6 @@
                 "sha256:d388cb7f54f1a3056f91ffcfb5cf18b226454204e5df7a5c10774718c3fbb166"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==1.26.109"
         },
         "botocore": {
@@ -308,11 +306,11 @@
         },
         "cachetools": {
             "hashes": [
-                "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590",
-                "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"
+                "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2",
+                "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.1"
+            "version": "==5.3.2"
         },
         "certifi": {
             "hashes": [
@@ -320,7 +318,6 @@
                 "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.6'",
             "version": "==2023.7.22"
         },
         "cffi": {
@@ -391,126 +388,126 @@
         },
         "cftime": {
             "hashes": [
-                "sha256:055d5d60a756c6c1857cf84d77655bb707057bb6c4a4fbb104a550e76c40aad9",
-                "sha256:07fdef2f75a0f0952b0376fa4cd08ef8a1dad3b963976ac07517811d434936b7",
-                "sha256:0955e1f3e1c09a9e0296b50f135ff9719cb2466f81c8ad4a10ef06fa394de984",
-                "sha256:29c18601abea0fd160fbe423e05c7a56fe1d38dd250a6b010de499a132d3fe18",
-                "sha256:2abdac6ca5b8b6102f319122546739dfc42406b816c16f2a98a8f0cd406d3bf0",
-                "sha256:2ba7909a0cd4adcb16797d8d6ab2767e7ddb980b2bf9dbabfc71b3bdd94f072b",
-                "sha256:3042048324b4d6a1066c978ec78101effdd84320e8862bfdbf8122d7ad7588ec",
-                "sha256:455cec3627e6ca8694b0d9201da6581eb4381b58389f1fbcb51a14fa0e2b3d94",
-                "sha256:56d0242fc4990584b265622622b25bb262a178097711d2d95e53ef52a9d23e7e",
-                "sha256:8614c00fb8a5046de304fdd86dbd224f99408185d7b245ac6628d0276596e6d2",
-                "sha256:86fe550b94525c327578a90b2e13418ca5ba6c636d5efe3edec310e631757eea",
-                "sha256:892d5dc38f8b998c83a2a01f131e63896d020586de473e1878f9e85acc70ad44",
-                "sha256:8d49d69c64cee2c175478eed84c3a57fce083da4ceebce16440f72be561a8489",
-                "sha256:93f00f454329c1f2588ebca2650e8edf7607d6189dbdcc81b5f3be2080155cc4",
-                "sha256:acb294fdb80e33545ae54b4421df35c4e578708a5ffce1c00408b2294e70ecef",
-                "sha256:aedfb7a783d19d7a30cb41951310f3bfe98f9f21fffc723c8af08a11962b0b17",
-                "sha256:afb5b38b51b8bc02f1656a9f15c52b0b20a3999adbe1ab9ac57f926e0065b48a",
-                "sha256:b4d2a1920f0aad663f25700b30621ff64af373499e52b544da1148dd8c09409a",
-                "sha256:e83db2fdda900eb154a9f79dfb665ac6190781c61d2e18151996de5ee7ffd8a2",
-                "sha256:eb7f8cd0996640b83020133b5ef6b97fc9216c3129eaeeaca361abdff5d82166",
-                "sha256:ee70fa069802652cf534de1dd3fc590b7d22d4127447bf96ac9849abcdadadf1"
+                "sha256:022dabf1610cdd04a693e730fa8f71d307059717f29dba921e7486e553412bb4",
+                "sha256:0a38eb9f5c733a23e1714bd3ef2762ed5acee34f127670f8fb4ad6464946f6b3",
+                "sha256:2d113a01ab924445e61d65c26bbd95bc08e4a22878d3b947064bba056c884c4a",
+                "sha256:3b86be8c2f254147be4ba88f12099466dde457a4a3a21de6c69d52a7224c13ae",
+                "sha256:3cf6e216a4c06f9a628cdf8e9c9d5e8097fb3eb02dd087dd14ab3b18478a7271",
+                "sha256:4d6fbd5f41b322cfa7b0ac3aaadeceb4450100a164b5bccbbb9e7c5048489a88",
+                "sha256:523b9a6bf03f5e36407979e248381d0fcab2d225b915bbde77d00c6dde192b90",
+                "sha256:5f11685663a6af97418908060492a07663c16d42519c139ca03c2ffb1377fd25",
+                "sha256:80eb1170ce1639016f55760847f4aadd04b0312496c5bac2797e930914bba48d",
+                "sha256:8a14d2c7d22fd2a6dfa6ad563283b6d6679f1df95e0ed8d14b8f284dad402887",
+                "sha256:8d2c01456d9d7b46aa710a41d1c711a50d5ea259aff4a987d0e973d1093bc922",
+                "sha256:9eb177a02db7cd84aa6962278e4bd2d3106a545de82e6aacd9404f1e153661db",
+                "sha256:a98abb1d46d118e52b0611ce668a0b714b407be26177ef0581ecf5e95f894725",
+                "sha256:b62d42546fa5c914dfea5b15a9aaed2087ea1211cc36d08c374502ef95892038",
+                "sha256:bbf782ab4ac0605bdec2b941952c897595613203942b7f8c2fccd17efa5147df",
+                "sha256:bedb577bc8b8f3f10f5336c0792e5dae88605781890f50f36b45bb46907968e8",
+                "sha256:d0a6b29f72a13f08e008b9becff247cc75c84acb213332ede18879c5b6aa4dfd",
+                "sha256:d87dadd0824262bdd7493babd2a44447da0a22175ded8ae9e060a3aebec7c5d7",
+                "sha256:d9b00c2844c7a1701d8ede5336b6321dfee256ceab81a34a1aff0483d56891a6",
+                "sha256:eb6dd70b2ccabfe1a14b7fbb0bbdce0418e71697094373c0d573c880790fa291",
+                "sha256:f9878bfd8c1c3f24184ecbd528f739ba46ebaceaf1c8a24d348d7befb117a285"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.6.3"
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843",
-                "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786",
-                "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e",
-                "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8",
-                "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4",
-                "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa",
-                "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d",
-                "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82",
-                "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7",
-                "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895",
-                "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d",
-                "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a",
-                "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382",
-                "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678",
-                "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b",
-                "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e",
-                "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741",
-                "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4",
-                "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596",
-                "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9",
-                "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69",
-                "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c",
-                "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77",
-                "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13",
-                "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459",
-                "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e",
-                "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7",
-                "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908",
-                "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a",
-                "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f",
-                "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8",
-                "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482",
-                "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d",
-                "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d",
-                "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545",
-                "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34",
-                "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86",
-                "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6",
-                "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe",
-                "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e",
-                "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc",
-                "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7",
-                "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd",
-                "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c",
-                "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557",
-                "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a",
-                "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89",
-                "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078",
-                "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e",
-                "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4",
-                "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403",
-                "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0",
-                "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89",
-                "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115",
-                "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9",
-                "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05",
-                "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a",
-                "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec",
-                "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56",
-                "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38",
-                "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479",
-                "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c",
-                "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e",
-                "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd",
-                "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186",
-                "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455",
-                "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c",
-                "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65",
-                "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78",
-                "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287",
-                "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df",
-                "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43",
-                "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1",
-                "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7",
-                "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989",
-                "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a",
-                "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63",
-                "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884",
-                "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649",
-                "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810",
-                "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828",
-                "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4",
-                "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2",
-                "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd",
-                "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5",
-                "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe",
-                "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293",
-                "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e",
-                "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e",
-                "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"
-            ],
-            "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.3.0"
+                "sha256:06cf46bdff72f58645434d467bf5228080801298fbba19fe268a01b4534467f5",
+                "sha256:0c8c61fb505c7dad1d251c284e712d4e0372cef3b067f7ddf82a7fa82e1e9a93",
+                "sha256:10b8dd31e10f32410751b3430996f9807fc4d1587ca69772e2aa940a82ab571a",
+                "sha256:1171ef1fc5ab4693c5d151ae0fdad7f7349920eabbaca6271f95969fa0756c2d",
+                "sha256:17a866d61259c7de1bdadef418a37755050ddb4b922df8b356503234fff7932c",
+                "sha256:1d6bfc32a68bc0933819cfdfe45f9abc3cae3877e1d90aac7259d57e6e0f85b1",
+                "sha256:1ec937546cad86d0dce5396748bf392bb7b62a9eeb8c66efac60e947697f0e58",
+                "sha256:223b4d54561c01048f657fa6ce41461d5ad8ff128b9678cfe8b2ecd951e3f8a2",
+                "sha256:2465aa50c9299d615d757c1c888bc6fef384b7c4aec81c05a0172b4400f98557",
+                "sha256:28f512b9a33235545fbbdac6a330a510b63be278a50071a336afc1b78781b147",
+                "sha256:2c092be3885a1b7899cd85ce24acedc1034199d6fca1483fa2c3a35c86e43041",
+                "sha256:2c4c99f98fc3a1835af8179dcc9013f93594d0670e2fa80c83aa36346ee763d2",
+                "sha256:31445f38053476a0c4e6d12b047b08ced81e2c7c712e5a1ad97bc913256f91b2",
+                "sha256:31bbaba7218904d2eabecf4feec0d07469284e952a27400f23b6628439439fa7",
+                "sha256:34d95638ff3613849f473afc33f65c401a89f3b9528d0d213c7037c398a51296",
+                "sha256:352a88c3df0d1fa886562384b86f9a9e27563d4704ee0e9d56ec6fcd270ea690",
+                "sha256:39b70a6f88eebe239fa775190796d55a33cfb6d36b9ffdd37843f7c4c1b5dc67",
+                "sha256:3c66df3f41abee950d6638adc7eac4730a306b022570f71dd0bd6ba53503ab57",
+                "sha256:3f70fd716855cd3b855316b226a1ac8bdb3caf4f7ea96edcccc6f484217c9597",
+                "sha256:3f9bc2ce123637a60ebe819f9fccc614da1bcc05798bbbaf2dd4ec91f3e08846",
+                "sha256:3fb765362688821404ad6cf86772fc54993ec11577cd5a92ac44b4c2ba52155b",
+                "sha256:45f053a0ece92c734d874861ffe6e3cc92150e32136dd59ab1fb070575189c97",
+                "sha256:46fb9970aa5eeca547d7aa0de5d4b124a288b42eaefac677bde805013c95725c",
+                "sha256:4cb50a0335382aac15c31b61d8531bc9bb657cfd848b1d7158009472189f3d62",
+                "sha256:4e12f8ee80aa35e746230a2af83e81bd6b52daa92a8afaef4fea4a2ce9b9f4fa",
+                "sha256:4f3100d86dcd03c03f7e9c3fdb23d92e32abbca07e7c13ebd7ddfbcb06f5991f",
+                "sha256:4f6e2a839f83a6a76854d12dbebde50e4b1afa63e27761549d006fa53e9aa80e",
+                "sha256:4f861d94c2a450b974b86093c6c027888627b8082f1299dfd5a4bae8e2292821",
+                "sha256:501adc5eb6cd5f40a6f77fbd90e5ab915c8fd6e8c614af2db5561e16c600d6f3",
+                "sha256:520b7a142d2524f999447b3a0cf95115df81c4f33003c51a6ab637cbda9d0bf4",
+                "sha256:548eefad783ed787b38cb6f9a574bd8664468cc76d1538215d510a3cd41406cb",
+                "sha256:555fe186da0068d3354cdf4bbcbc609b0ecae4d04c921cc13e209eece7720727",
+                "sha256:55602981b2dbf8184c098bc10287e8c245e351cd4fdcad050bd7199d5a8bf514",
+                "sha256:58e875eb7016fd014c0eea46c6fa92b87b62c0cb31b9feae25cbbe62c919f54d",
+                "sha256:5a3580a4fdc4ac05f9e53c57f965e3594b2f99796231380adb2baaab96e22761",
+                "sha256:5b70bab78accbc672f50e878a5b73ca692f45f5b5e25c8066d748c09405e6a55",
+                "sha256:5ceca5876032362ae73b83347be8b5dbd2d1faf3358deb38c9c88776779b2e2f",
+                "sha256:61f1e3fb621f5420523abb71f5771a204b33c21d31e7d9d86881b2cffe92c47c",
+                "sha256:633968254f8d421e70f91c6ebe71ed0ab140220469cf87a9857e21c16687c034",
+                "sha256:63a6f59e2d01310f754c270e4a257426fe5a591dc487f1983b3bbe793cf6bac6",
+                "sha256:63accd11149c0f9a99e3bc095bbdb5a464862d77a7e309ad5938fbc8721235ae",
+                "sha256:6db3cfb9b4fcecb4390db154e75b49578c87a3b9979b40cdf90d7e4b945656e1",
+                "sha256:71ef3b9be10070360f289aea4838c784f8b851be3ba58cf796262b57775c2f14",
+                "sha256:7ae8e5142dcc7a49168f4055255dbcced01dc1714a90a21f87448dc8d90617d1",
+                "sha256:7b6cefa579e1237ce198619b76eaa148b71894fb0d6bcf9024460f9bf30fd228",
+                "sha256:800561453acdecedaac137bf09cd719c7a440b6800ec182f077bb8e7025fb708",
+                "sha256:82ca51ff0fc5b641a2d4e1cc8c5ff108699b7a56d7f3ad6f6da9dbb6f0145b48",
+                "sha256:851cf693fb3aaef71031237cd68699dded198657ec1e76a76eb8be58c03a5d1f",
+                "sha256:854cc74367180beb327ab9d00f964f6d91da06450b0855cbbb09187bcdb02de5",
+                "sha256:87071618d3d8ec8b186d53cb6e66955ef2a0e4fa63ccd3709c0c90ac5a43520f",
+                "sha256:871d045d6ccc181fd863a3cd66ee8e395523ebfbc57f85f91f035f50cee8e3d4",
+                "sha256:8aee051c89e13565c6bd366813c386939f8e928af93c29fda4af86d25b73d8f8",
+                "sha256:8af5a8917b8af42295e86b64903156b4f110a30dca5f3b5aedea123fbd638bff",
+                "sha256:8ec8ef42c6cd5856a7613dcd1eaf21e5573b2185263d87d27c8edcae33b62a61",
+                "sha256:91e43805ccafa0a91831f9cd5443aa34528c0c3f2cc48c4cb3d9a7721053874b",
+                "sha256:9505dc359edb6a330efcd2be825fdb73ee3e628d9010597aa1aee5aa63442e97",
+                "sha256:985c7965f62f6f32bf432e2681173db41336a9c2611693247069288bcb0c7f8b",
+                "sha256:9a74041ba0bfa9bc9b9bb2cd3238a6ab3b7618e759b41bd15b5f6ad958d17605",
+                "sha256:9edbe6a5bf8b56a4a84533ba2b2f489d0046e755c29616ef8830f9e7d9cf5728",
+                "sha256:a15c1fe6d26e83fd2e5972425a772cca158eae58b05d4a25a4e474c221053e2d",
+                "sha256:a66bcdf19c1a523e41b8e9d53d0cedbfbac2e93c649a2e9502cb26c014d0980c",
+                "sha256:ae4070f741f8d809075ef697877fd350ecf0b7c5837ed68738607ee0a2c572cf",
+                "sha256:ae55d592b02c4349525b6ed8f74c692509e5adffa842e582c0f861751701a673",
+                "sha256:b578cbe580e3b41ad17b1c428f382c814b32a6ce90f2d8e39e2e635d49e498d1",
+                "sha256:b891a2f68e09c5ef989007fac11476ed33c5c9994449a4e2c3386529d703dc8b",
+                "sha256:baec8148d6b8bd5cee1ae138ba658c71f5b03e0d69d5907703e3e1df96db5e41",
+                "sha256:bb06098d019766ca16fc915ecaa455c1f1cd594204e7f840cd6258237b5079a8",
+                "sha256:bc791ec3fd0c4309a753f95bb6c749ef0d8ea3aea91f07ee1cf06b7b02118f2f",
+                "sha256:bd28b31730f0e982ace8663d108e01199098432a30a4c410d06fe08fdb9e93f4",
+                "sha256:be4d9c2770044a59715eb57c1144dedea7c5d5ae80c68fb9959515037cde2008",
+                "sha256:c0c72d34e7de5604df0fde3644cc079feee5e55464967d10b24b1de268deceb9",
+                "sha256:c0e842112fe3f1a4ffcf64b06dc4c61a88441c2f02f373367f7b4c1aa9be2ad5",
+                "sha256:c15070ebf11b8b7fd1bfff7217e9324963c82dbdf6182ff7050519e350e7ad9f",
+                "sha256:c2000c54c395d9e5e44c99dc7c20a64dc371f777faf8bae4919ad3e99ce5253e",
+                "sha256:c30187840d36d0ba2893bc3271a36a517a717f9fd383a98e2697ee890a37c273",
+                "sha256:cb7cd68814308aade9d0c93c5bd2ade9f9441666f8ba5aa9c2d4b389cb5e2a45",
+                "sha256:cd805513198304026bd379d1d516afbf6c3c13f4382134a2c526b8b854da1c2e",
+                "sha256:d0bf89afcbcf4d1bb2652f6580e5e55a840fdf87384f6063c4a4f0c95e378656",
+                "sha256:d9137a876020661972ca6eec0766d81aef8a5627df628b664b234b73396e727e",
+                "sha256:dbd95e300367aa0827496fe75a1766d198d34385a58f97683fe6e07f89ca3e3c",
+                "sha256:dced27917823df984fe0c80a5c4ad75cf58df0fbfae890bc08004cd3888922a2",
+                "sha256:de0b4caa1c8a21394e8ce971997614a17648f94e1cd0640fbd6b4d14cab13a72",
+                "sha256:debb633f3f7856f95ad957d9b9c781f8e2c6303ef21724ec94bea2ce2fcbd056",
+                "sha256:e372d7dfd154009142631de2d316adad3cc1c36c32a38b16a4751ba78da2a397",
+                "sha256:ecd26be9f112c4f96718290c10f4caea6cc798459a3a76636b817a0ed7874e42",
+                "sha256:edc0202099ea1d82844316604e17d2b175044f9bcb6b398aab781eba957224bd",
+                "sha256:f194cce575e59ffe442c10a360182a986535fd90b57f7debfaa5c845c409ecc3",
+                "sha256:f5fb672c396d826ca16a022ac04c9dce74e00a1c344f6ad1a0fdc1ba1f332213",
+                "sha256:f6a02a3c7950cafaadcd46a226ad9e12fc9744652cc69f9e5534f98b47f3bbcf",
+                "sha256:fe81b35c33772e56f4b6cf62cf4aedc1762ef7162a31e6ac7fe5e40d0149eb67"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.3.1"
         },
         "click": {
             "hashes": [
@@ -635,67 +632,67 @@
         },
         "cython": {
             "hashes": [
-                "sha256:0147a31fb73a063bb7b6c69fd843c1a2bad18f326f58048d4ee5bdaef87c9fbf",
-                "sha256:056340c49bf7861eb1eba941423e67620b7c85e264e9a5594163f1d1e8b95acc",
-                "sha256:0630527a8c9e8fed815c38524e418dab713f5d66f6ac9dc2151b41f3a7727304",
-                "sha256:0e1c9385e99eef299396b9a1e39790e81819446c6a83e249f6f0fc71a64f57a0",
-                "sha256:14b898ec2fdeea68f81bd3838b035800b173b59ed532674f65a82724bab35d3b",
-                "sha256:1571b045ec1cb15c152c3949f3bd53ee0fa66d434271ea3d225658d99b7e721a",
-                "sha256:176953a8a2532e34a589625a40c934ff339088f2bf4ddaa2e5cb77b05ca0c25c",
-                "sha256:188705eeae094bb716bc3e3d0da4e13469f0a0de803b65dfd63fe7eb78ec6173",
-                "sha256:18f4fb7cc6ad8e99e8f387ebbcded171a701bfbfd8cd3fd46156bf44bb4fd968",
-                "sha256:1d3416c24a1b7bf3a2d9615a7f9f12b00fac0b94fb2e61449e0c1ecf20d6ed52",
-                "sha256:1ec9e15b821ef7e3c38abe9e4df4e6dda7af159325bc358afd5a3c2d5027ccfe",
-                "sha256:22d268c3023f405e13aa0c1600389794694ab3671614f8e782d89a1055da0858",
-                "sha256:30c1d9bd2bcb9b1a195dd23b359771857df8ebd4a1038fb37dd155d3ea38c09c",
-                "sha256:3192cd780435fca5ae5d79006b48cbf0ea674853b5a7b0055a122045bff9d84e",
-                "sha256:327309301b01f729f173a94511cb2280c87ba03c89ed428e88f913f778245030",
-                "sha256:3cfbd60137f6fca9c29101d7517d4e341e0fd279ffc2489634e5e2dd592457c2",
-                "sha256:3db04801fd15d826174f63ff45878d4b1e62aff27cf1ea96b186581052d24446",
-                "sha256:42b1ff0e19fb4d1fe68b60f55d46942ed246a323f6bbeec302924b78b4c3b637",
-                "sha256:48bae87b657009e5648c21d4a92de9f3dc6fed3e35e92957fa8a07a18cea2313",
-                "sha256:4c8e5afcc19861c3b22faafbe906c7e1b23f0595073ac10e21a80dec9e60e7dd",
-                "sha256:4cc0f7244da06fdc6a4a7240df788805436b6fb7f20edee777eb77777d9d2eb1",
-                "sha256:4e956383e57d00b1fa6449b5ec03b9fa5fce2afd41ef3e518bee8e7c89f1616c",
-                "sha256:51850f277660f67171135515e45edfc8815f723ff20768e39cb9785b2671062f",
-                "sha256:5545d20d7a1c0cf17559152f7f4a465c3d5caace82dd051f82e2d753ae9fd956",
-                "sha256:587d664ff6bd5b03611ddc6ef320b7f8677d824c45d15553f16a69191a643843",
-                "sha256:59bf689409b0e51ef673e3dd0348727aef5b67e40f23f806be64c49cee321de0",
-                "sha256:5d6af87a787d5ce063e28e508fee34755a945e438c68ecda50eb4ea34c30e13f",
-                "sha256:66b7e71c16cab0814945014ffb101ead2b173259098bbb1b8138e7a547da3709",
-                "sha256:6ccde14ddc4b424435cb5722aa1529c254bbf3611e1ad9baea12d25e9c049361",
-                "sha256:6f05889eb1b5a95a7adf97303279c2d13819ff62292e10337e6c940dbf570b5d",
-                "sha256:6f63e959d13775472d37e731b2450d120e8db87e956e2de74475e8f17a89b1fb",
-                "sha256:74ba0f11b384246b7965169f08bf67d426e4957fee5c165571340217a9b43cfc",
-                "sha256:75d42c8423ab299396f3c938445730600e32e4a2f0298f6f9df4d4a698fe8e16",
-                "sha256:77a920ae19fa1db5adb8a618cebb095ca4f56adfbf9fc32cb7008a590607b62b",
-                "sha256:80bd3167e689419cdaf7ede0d20a9f126b9698a43b1f8d3e8f54b970c7a6cd07",
-                "sha256:84084fa05cf9a67a85818fa72a741d1cae2e3096551158730730a3bafc3b2f52",
-                "sha256:845e24ee70c204062e03f813114751387abf454b29410336797582e04abbc07b",
-                "sha256:85073ab414ff432d2a39d36cb49c39ce69f30b53daccc7699bfad0ce3d1b539a",
-                "sha256:8a6a9a2d98758768052e4ac1bea4ebc20fae69b4c19cb2bc5457c9174532d302",
-                "sha256:8e78fc42a6e846941d23aba1aca587520ad38c8970255242f08f9288b0eeba85",
-                "sha256:9296f332523d5c550ebae694483874d255264cff3281372f25ea5f2739b96651",
-                "sha256:94fa403de3a413cd41b8eb4ddb4adcbd66aa0a64f9a84d1c5f696c93572c83aa",
-                "sha256:9eb128fa40305f18eaa4d8dd0980033b92db86aada927181d3c3d561aa0634db",
-                "sha256:9f40b27545d583fd7df0d3c1b76b3bcaf8a72dbd8d83d5486af2384015660de8",
-                "sha256:b50f4f75f89e7eef2ed9c9b60746bc4ab1ba2bc0dff64587133db2b63e068f09",
-                "sha256:be1a679c7ad90813f9206c9d62993f3bd0cba9330668e97bb3f70c87ae94d5f5",
-                "sha256:bff1fec968a6b2ca452ae9bff6d6d0bf8486427d4d791e85543240266b6915e0",
-                "sha256:d0c7b315f6feb75e2c949dc7816da5626cdca097fea1c0d9f4fdb20d2f4ffc2a",
-                "sha256:d12591939af93c59defea6fc5320ca099eb44e4694e3b2cbe72fb24406079b97",
-                "sha256:d49d20db27c9cfcf45bb1fbf68f777bd1e04e4b949e4e5172d9ee8c9419bc792",
-                "sha256:d52ed47edbf48392dd0f419135e7ff59673f6b32d27d3ffc9e61a515571c050d",
-                "sha256:db9d4de4cd6cd3ad1c3f455aae877ad81a92b92b7cbb01dfb32b6306b873932b",
-                "sha256:e3ad109bdf40f55318e001cad12bcc00e8119569b49f72e442c082355617b036",
-                "sha256:e40ac8bd6d11355d354bb4975bb88f6e923ba30f85e38f1f1234b642634e4fc4",
-                "sha256:e729fd633a5225570c5480b36e7c530c8a82e2ab6d2944ddbe1ddfff5bf181b1",
-                "sha256:f0ac9ec822fad010248b4a59ac197975de38c95378d0f13201c181dd9b0a2624",
-                "sha256:f7578b59ffd0d9c95ae6f7ae852309918915998b7fe0ed2f8725a683de8da276",
-                "sha256:fa08259f4d176b86561eeff6954f9924099c0b0c128fc2cbfc18343c068ad8ca"
+                "sha256:0021350f6d7022a37f598320460b84b2c0daccf6bb65641bbdbc8b990bdf4ad2",
+                "sha256:003ccc40e0867770db0018274977d1874e4df64983d5e3e36937f107e0b2fdf6",
+                "sha256:0422a40a58dcfbb54c8b4e125461d741031ff046bc678475cc7a6c801d2a7721",
+                "sha256:0612439f810cc281e51fead69de0545c4d9772a1e82149c119d1aafc1f6210ba",
+                "sha256:061dec1be8d8b601b160582609a78eb08324a4ccf21bee0d04853a3e9dfcbefd",
+                "sha256:0650460b5fd6f16da4186e0a769b47db5532601e306f3b5d17941283d5e36d24",
+                "sha256:07d0e69959f267b79ffd18ece8599711ad2f3d3ed1eddd0d4812d2a97de2b912",
+                "sha256:096cb461bf8d913a4327d93ea38d18bc3dbc577a71d805be04754e4b2cc2c45d",
+                "sha256:0aa2a6bb3ff67794d8d1dafaed22913adcbb327e96eca3ac44e2f3ba4a0ae446",
+                "sha256:0fc9e974419cc0393072b1e9a669f71c3b34209636d2005ff8620687daa82b8c",
+                "sha256:0fe227d6d8e2ea030e82abc8a3e361e31447b66849f8c069caa783999e54a8f2",
+                "sha256:10b426adc3027d66303f5c7aa8b254d10ed80827ff5cce9e892d550b708dc044",
+                "sha256:146bfaab567157f4aa34114a37e3f98a3d9c4527ee99d4fd730cab56482bd3cf",
+                "sha256:15d52f7f9d08b264c042aa508bf457f53654b55f533e0262e146002b1c15d1cd",
+                "sha256:19a64bf2591272348ab08bcd4a5f884259cc3186f008c9038b8ec7d01f847fd5",
+                "sha256:207f53893ca22d8c8f5db533f38382eb7ddc2d0b4ab51699bf052423a6febdad",
+                "sha256:28af4e7dff1742cb0f0a4823102c89c62a2d94115b68f718145fcfe0763c6e21",
+                "sha256:28de18f0d07eb34e2dd7b022ac30beab0fdd277846d07b7a08e69e6294f0762b",
+                "sha256:2e379b491ee985d31e5faaf050f79f4a8f59f482835906efe4477b33b4fbe9ff",
+                "sha256:36299ffd5663203c25d3a76980f077e23b6d4f574d142f0f43943f57be445639",
+                "sha256:4355a2cb03b257773c0d2bb6af9818c72e836a9b09986e28f52e323d87b1fc67",
+                "sha256:48b35ab009227ee6188991b5666aae1936b82a944f707c042cef267709de12b5",
+                "sha256:48cda82eb82ad2014d2ad194442ed3c46156366be98e4e02f3e29742cdbf94a0",
+                "sha256:4e8c144e2c5814e46868d1f81e2f4265ca1f314a8187d0420cd76e9563294cf8",
+                "sha256:58cdfdd942cf5ffcee974aabfe9b9e26c0c1538fd31c1b03596d40405f7f4d40",
+                "sha256:5b906997e7b98d7d29b84d10a5318993eba1aaff82ff7e1a0ac00254307913d7",
+                "sha256:6603a287188dcbc36358a73a7be43e8a2ecf0c6a06835bdfdd1b113943afdd6f",
+                "sha256:6619264ed43d8d8819d4f1cdb8a62ab66f87e92f06f3ff3e2533fd95a9945e59",
+                "sha256:6a0b92adfcac68dcf549daddec83c87a86995caa6f87bfb6f72de4797e1a6ad6",
+                "sha256:6da74000a672eac0d7cf02adc140b2f9c7d54eae6c196e615a1b5deb694d9203",
+                "sha256:8529cf09919263a6826adc04c5dde9f1406dd7920929b16be19ee9848110e353",
+                "sha256:861979428f749faa9883dc4e11e8c3fc2c29bd0031cf49661604459b53ea7c66",
+                "sha256:8692249732d62e049df3884fa601b70fad3358703e137aceeb581e5860e7d9b7",
+                "sha256:8a9262408f05eef039981479b38b38252d5b853992e5bc54a2d2dd05a2a0178e",
+                "sha256:8ada3659608795bb36930d9a206b8dd6b865d85e2999a02ce8b34f3195d88301",
+                "sha256:9d31d76ed777a8a85be3f8f7f1cfef09b3bc33f6ec4abee1067dcef107f49778",
+                "sha256:b3ddfc6f05410095ec11491dde05f50973e501567d21cbfcf5832d95f141878a",
+                "sha256:b72c426df1586f967b1c61d2f8236702d75c6bbf34abdc258a59e09155a16414",
+                "sha256:b86871862bd65806ba0d0aa2b9c77fcdcc6cbd8d36196688f4896a34bb626334",
+                "sha256:bc42004f181373cd3b39bbacfb71a5b0606ed6e4c199c940cca2212ba0f79525",
+                "sha256:beb367fd88fc6ba8c204786f680229106d99da72a60f5906c85fc8d73640b01a",
+                "sha256:bf671d712816b48fa2731799017ed68e5e440922d0c7e13dc825c226639ff766",
+                "sha256:c0fb9e7cf9db38918f19a803fab9bc7b2ed3f33a9e8319c616c464a0a8501b8d",
+                "sha256:c214f6e88ecdc8ff5d13f0914890445fdaad21bddc34a90cd14aeb3ad5e55e2e",
+                "sha256:c2215f436ce3cce49e6e318cb8f7253cfc4d3bea690777c2a5dd52ae93342504",
+                "sha256:c7a7dd7c50d07718a5ac2bdea166085565f7217cf1e030cc07c22a8b80a406a7",
+                "sha256:c8e0f98d950987b0f9d5e10c41236bef5cb4fba701c6e680af0b9734faa3a85e",
+                "sha256:c9b1322f0d8ce5445fcc3a625b966f10c4182190026713e217d6f38d29930cb1",
+                "sha256:d40d4135f76fb0ed4caa2d422fdb4231616615698709d3c421ecc733f1ac7ca0",
+                "sha256:d5a55749509c7f9f8a33bf9cc02cf76fd6564fcb38f486e43d2858145d735953",
+                "sha256:e5e2859f97e9cceb8e70b0934c56157038b8b083245898593008162a70536d7e",
+                "sha256:e84988d384dfba678387ea7e4f68786c3703543018d473605d9299c69a07f197",
+                "sha256:ef4b144c5b29b4ea0b40c401458b86df8d75382b2e5d03e9f67f607c05b516a9",
+                "sha256:f124ac9ee41e1bfdfb16f53f1db85de296cd2144a4e8fdee8c3560a8fe9b6d5d",
+                "sha256:f234bc46043856d118ebd94b13ea29df674503bc94ced3d81ca46a1ad5b5b9ae",
+                "sha256:f24114e1777604a28ae1c7a56a2c9964655f1031edecc448ad51e5abb19a279b",
+                "sha256:f7fcd93d15deceb2747b10266a39deccd94f257d610f3bbd52a7e16bc5908eda",
+                "sha256:fc96efa617184b8581a02663e261b41c13a605da8ef4ba1ed735bf46184f815e"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.3"
+            "version": "==3.0.4"
         },
         "dask": {
             "hashes": [
@@ -835,7 +832,6 @@
                 "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181"
             ],
             "index": "pypi",
-            "markers": "python_full_version >= '3.8.1'",
             "version": "==6.0.0"
         },
         "flox": {
@@ -903,11 +899,11 @@
         },
         "fsspec": {
             "hashes": [
-                "sha256:603dbc52c75b84da501b9b2ec8c11e1f61c25984c4a0dda1f129ef391fbfc9b4",
-                "sha256:80bfb8c70cc27b2178cc62a935ecf242fc6e8c3fb801f9c571fc01b1e715ba7d"
+                "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5",
+                "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.9.2"
+            "version": "==2023.10.0"
         },
         "geocube": {
             "hashes": [
@@ -931,7 +927,6 @@
                 "sha256:0acdacddefa176525e4da6d9aeeece225da26055c4becdc6e97cf40fa97c27f4"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==0.12.2"
         },
         "geopy": {
@@ -1068,11 +1063,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:2e2ee359baba19f10251b99415bb39de1e97d04e1fab385646f24f0596510b77",
-                "sha256:f468ddd1f17acb48c8ce67fcfa49ba6d46d4f9ac0438c1f441be7c3d1372230b"
+                "sha256:3ba3dc97424b87b31bb46586b5167b3161b32d7820b9201a9e698c71e271602c",
+                "sha256:553856658eb8430bbe9653ea041a41bff63e9606fc4628873fc92a6cf3abd404"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.25.2"
+            "version": "==6.26.0"
         },
         "ipympl": {
             "hashes": [
@@ -1118,7 +1113,6 @@
                 "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"
             ],
             "index": "pypi",
-            "markers": "python_full_version >= '3.8.0'",
             "version": "==5.12.0"
         },
         "jedi": {
@@ -1226,11 +1220,11 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:b11e2ba80667c75f55630faf8ac3d5809f8734f9006d65cce117c46a0a516ab8",
-                "sha256:c57270faa6530393ae69783a2d2f1874c718b9f109080581ea076b05713249fa"
+                "sha256:21ad1a3d455d5a79ce4bef5201925cd17510c17898cf9d54e3ccfb6b12734948",
+                "sha256:9ba71be4b9c16e479e4c50c929f8ac4b1015baf90237a08681397a98c76c7e5e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.8.0"
+            "version": "==2.9.1"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1270,7 +1264,6 @@
                 "sha256:6aba0caa771697d02fbf409f9767b2fdb4ee32ce935940e3b9a0d5d48d994d0f"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==3.6.3"
         },
         "jupyterlab-pygments": {
@@ -1578,15 +1571,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.7.0"
         },
-        "memory-profiler": {
-            "hashes": [
-                "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84",
-                "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0"
-            ],
-            "index": "pypi",
-            "markers": "python_version >= '3.5'",
-            "version": "==0.61.0"
-        },
         "mercantile": {
             "hashes": [
                 "sha256:30f457a73ee88261aab787b7069d85961a5703bb09dc57a170190bc042cd023f",
@@ -1717,7 +1701,6 @@
                 "sha256:d583bc9050dd10538de36297c960b93f873f0cd01671a3c50df5bd86dd391dcb"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==8.3.1"
         },
         "nbclassic": {
@@ -1733,7 +1716,7 @@
                 "sha256:25e861299e5303a0477568557c4045eccc7a34c17fc08e7959558707b9ebe548",
                 "sha256:f9b179cd4b2d7bca965f900a2ebf0db4a12ebff2f36a711cb66861e4ae158e55"
             ],
-            "markers": "python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8'",
             "version": "==0.8.0"
         },
         "nbconvert": {
@@ -1791,7 +1774,6 @@
                 "sha256:f94a89db78f34fdf68342840efb064fe1474310e8359dffce42e90a9ddf88f2f"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==1.6.3"
         },
         "nodeenv": {
@@ -1850,7 +1832,6 @@
                 "sha256:fcdf84ba3ed8124eb7234adfbb8792f311991cbf8aed1cad4b1b1a7ee08380c1"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==0.56.4"
         },
         "numexpr": {
@@ -1921,7 +1902,6 @@
                 "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==1.23.5"
         },
         "numpy-groupies": {
@@ -1983,7 +1963,6 @@
                 "sha256:f908a77cbeef9bbd646bd4b81214cbef9ac3dda4181d5092a4aa9797d1bc7774"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==2.0.2"
         },
         "pandera": {
@@ -2004,11 +1983,11 @@
         },
         "param": {
             "hashes": [
-                "sha256:59d55048d42a85e148a69837df42bd11c3391d47fad15ba57d118e145f001ef2",
-                "sha256:a2e3b7b07ca7dd1adaa4fb3020a3ef4fe434f27ede453a9d94194c5155677e30"
+                "sha256:0153a06fb0049fab73132d7faf4c468d44ad77ee00d531702b0bf222c069f9b6",
+                "sha256:19823383b0dac5a794152b85336cb4c6b0d0ef61a2b1eb7cd272aae141be0117"
             ],
-            "markers": "python_version >= '2.7'",
-            "version": "==1.13.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.0.0"
         },
         "parso": {
             "hashes": [
@@ -2139,7 +2118,6 @@
                 "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==3.3.3"
         },
         "prometheus-client": {
@@ -2155,7 +2133,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.39"
         },
         "psutil": {
@@ -2177,7 +2155,7 @@
                 "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d",
                 "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "markers": "sys_platform != 'cygwin'",
             "version": "==5.9.6"
         },
         "psycopg2-binary": {
@@ -2246,7 +2224,6 @@
                 "sha256:ffe9dc0a884a8848075e576c1de0290d85a533a9f6e9c4e564f19adf8f6e54a7"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.6'",
             "version": "==2.9.6"
         },
         "ptyprocess": {
@@ -2276,7 +2253,6 @@
                 "sha256:94d0c24217f6582741813ee94490a4ca82bd5f9bf35e4f8610cb588cf7445764"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==0.20.4"
         },
         "pyarrow": {
@@ -2308,7 +2284,6 @@
                 "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==11.0.0"
         },
         "pybcj": {
@@ -2645,7 +2620,6 @@
                 "sha256:fde5ece4d2436b5a57c8f5f97b49b5de06a856d03959f836c957d3e609f2de7e"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==3.5.0"
         },
         "pyproject-flake8": {
@@ -2654,7 +2628,6 @@
                 "sha256:d43421caca0ef8a672874405fe63c722b0333e3c22c41648c6df60f21bab2b6b"
             ],
             "index": "pypi",
-            "markers": "python_full_version >= '3.8.1'",
             "version": "==6.0.0.post1"
         },
         "pytest": {
@@ -2663,7 +2636,6 @@
                 "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==7.3.0"
         },
         "python-dateutil": {
@@ -2680,7 +2652,6 @@
                 "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==1.0.0"
         },
         "python-json-logger": {
@@ -2974,11 +2945,11 @@
         },
         "qtpy": {
             "hashes": [
-                "sha256:4d4f045a41e09ac9fa57fcb47ef05781aa5af294a0a646acc1b729d14225e741",
-                "sha256:db2d508167aa6106781565c8da5c6f1487debacba33519cedc35fa8997d424d4"
+                "sha256:1c1d8c4fa2c884ae742b069151b0abe15b3f70491f3972698c683b8e38de839b",
+                "sha256:a5a15ffd519550a1361bdc56ffc07fda56a6af7292f17c7b395d4083af632987"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.4.0"
+            "version": "==2.4.1"
         },
         "rasterio": {
             "hashes": [
@@ -3001,7 +2972,6 @@
                 "sha256:eaaeb2e661d1ffc07a7ae4fd997bb326d3561f641178126102842d608a010cc3"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==1.3.6"
         },
         "rasterstats": {
@@ -3010,7 +2980,6 @@
                 "sha256:29389bfcbeac1a4206aba6e1d795058ec8a64d560efad48156c27fad97c2e09a"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==0.18.0"
         },
         "referencing": {
@@ -3234,7 +3203,6 @@
                 "sha256:f5120da3a1b96f3a7a17dd6af0afdd4e6f3cc9baa87e9ee0a272882f01f980bb"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==1.0.1"
         },
         "s3transfer": {
@@ -3270,7 +3238,6 @@
                 "sha256:fae8a7b898c42dffe3f7361c40d5952b6bf32d10c4569098d276b4c547905ee1"
             ],
             "index": "pypi",
-            "markers": "python_version < '3.12' and python_version >= '3.8'",
             "version": "==1.10.1"
         },
         "seaborn": {
@@ -3279,7 +3246,6 @@
                 "sha256:ebf15355a4dba46037dfd65b7350f014ceb1f13c05e814eda2c9f5fd731afc08"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==0.12.2"
         },
         "send2trash": {
@@ -3340,7 +3306,6 @@
                 "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==2.0.1"
         },
         "simplejson": {
@@ -3444,7 +3409,7 @@
                 "sha256:febffa5b1eda6622d44b245b0685aff6fb555ce0ed734e2d7b1c3acd018a2cff",
                 "sha256:ff836cd4041e16003549449cc0a5e372f6b6f871eb89007ab0ee18fb2800fded"
             ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==3.19.2"
         },
         "six": {
@@ -3506,7 +3471,6 @@
                 "sha256:f0821007048f2af8c1a21eb3d832072046c5df366e39587a7c7e4afad14e73fc"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==3.8.0"
         },
         "terminado": {
@@ -3571,16 +3535,15 @@
                 "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
             "version": "==4.65.0"
         },
         "traitlets": {
             "hashes": [
-                "sha256:7564b5bf8d38c40fa45498072bf4dc5e8346eb087bbf1e2ae2d8774f6a0f078e",
-                "sha256:98277f247f18b2c5cabaf4af369187754f4fb0e85911d473f72329db8a7f4fae"
+                "sha256:81539f07f7aebcde2e4b5ab76727f53eabf18ad155c6ed7979a681411602fa47",
+                "sha256:833273bf645d8ce31dcb613c56999e2e055b1ffe6d09168a164bcd91c36d5d35"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.11.2"
+            "version": "==5.12.0"
         },
         "typeguard": {
             "hashes": [
@@ -3629,19 +3592,19 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21",
-                "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"
+                "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07",
+                "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.17"
+            "index": "pypi",
+            "version": "==1.26.18"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b",
-                "sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752"
+                "sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af",
+                "sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.24.5"
+            "version": "==20.24.6"
         },
         "wcwidth": {
             "hashes": [
@@ -3775,7 +3738,6 @@
                 "sha256:7e530b1deafdd43e5c2b577d0944e6b528fbe88045fd849e49a8d11871ecd522"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==2023.1.0"
         },
         "xarray-spatial": {
@@ -3972,7 +3934,7 @@
                 "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
                 "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.39"
         },
         "ptyprocess": {
@@ -4014,11 +3976,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:7564b5bf8d38c40fa45498072bf4dc5e8346eb087bbf1e2ae2d8774f6a0f078e",
-                "sha256:98277f247f18b2c5cabaf4af369187754f4fb0e85911d473f72329db8a7f4fae"
+                "sha256:81539f07f7aebcde2e4b5ab76727f53eabf18ad155c6ed7979a681411602fa47",
+                "sha256:833273bf645d8ce31dcb613c56999e2e055b1ffe6d09168a164bcd91c36d5d35"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.11.2"
+            "version": "==5.12.0"
         },
         "typing-extensions": {
             "hashes": [
diff --git a/data/wbd/clip_vectors_to_wbd.py b/data/wbd/clip_vectors_to_wbd.py
index 44e877291..2bf752bbb 100755
--- a/data/wbd/clip_vectors_to_wbd.py
+++ b/data/wbd/clip_vectors_to_wbd.py
@@ -8,11 +8,10 @@
 import rasterio as rio
 from shapely.geometry import MultiPolygon, Polygon
 
-from utils.shared_functions import getDriver, mem_profile
+from utils.shared_functions import getDriver
 from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 
 
-@mem_profile
 def subset_vector_layers(
     subset_nwm_lakes,
     subset_nwm_streams,
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1f09873dd..a6b6c8352 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,46 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
+
+During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times.
+
+This PR removes that system.
+
+A couple of other minor updates were made:
+- Update to the pip files (also carried forward changes from other current PRs)
+- When a huc or huc list is provided to fim_pipeline, it goes to a script, check_huc_inputs.py, to ensure that the incoming HUCs are valid and in that list. In the previous code it looks for all files with the file name pattern of "included_huc*.lst". However, we now only want it to check against the file "included_huc8.list".
+
+### Changes
+- `CONTRIBUTING.md`: Text update.
+- `Pipfile` and `Pipfile.lock`: updated to remove tghe memory-profiler package, update gval to 0.2.3 and update urllib3 to 1.26.18.
+- `data/wbd`
+    - `clip_vectors_to_wbd.py`: remove profiler
+ - `src`
+     - `add_crosswalk.py`: remove profiler
+     - `add_thalweg_lateral.py`: remove profiler.
+     - `aggregate_by_huc.py`: remove profiler and small text correction.
+     - `agreedem.py`: remove profiler.
+     - `bathy_src_adjust_topwidth.py`: remove profiler.
+     - `burn_in_levees.py`: remove profiler.
+     - `check_huc_inputs.py`: changed test pattern to just look against `included_huc8.lst`.
+     - `delineate_hydros_and_produce_HAND.sh`: remove profiler.
+     - `filter_catchments_and_add_attributes.py`: remove profiler.
+     - `make_stages_and_catchlist.py` remove profiler.
+     - `mask_dem.py`: remove profiler.
+     - `reachID_grid_to_vector_points.py`: remove profiler.
+     - `run_unit_wb.sh`: remove profiler.
+     - `split_flows.py`: remove profiler.
+     - `unique_pixel_and_allocation.py`: remove profiler.
+     - `usgs_gage_crosswalk.py`: remove profiler.
+     - `usgs_gage_unit_setup.py`: remove profiler.
+     - `utils`
+         - `shared_functions`: remove profiler.
+      ` unit_tests`
+          - `clip_vectors_to_wbd_tests.py`: Linting tools change order of the imports.
+
+<br/><br/>
+
 ## v4.4.4.1 - 2023-10-26 - [PR#1007](https://github.com/NOAA-OWP/inundation-mapping/pull/1007)
 
 Updates GVAL to address memory and performance issues associated with running synthesize test cases.
@@ -13,7 +53,6 @@ Updates GVAL to address memory and performance issues associated with running sy
 - `tools/run_test_case.py`
 - `tools/synthesize_test_cases.py`
 - `tools/inundate_mosaic_wrapper`
-
 <br/><br/>
 
 ## v4.4.4.0 - 2023-10-20 - [PR#1012](https://github.com/NOAA-OWP/inundation-mapping/pull/1012)
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 3f0a5f5c0..fd7c8c450 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -9,14 +9,13 @@
 from numpy import unique
 from rasterstats import zonal_stats
 
-from utils.shared_functions import getDriver, mem_profile
+from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
 
 
 # TODO - Feb 17, 2023 - We want to explore using FR methodology as branch zero
 
 
-@mem_profile
 def add_crosswalk(
     input_catchments_fileName,
     input_flows_fileName,
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index dabfbd488..5a55b61fc 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -7,10 +7,7 @@
 import rasterio
 from numba import njit, typed, types
 
-from utils.shared_functions import mem_profile
 
-
-@mem_profile
 def adjust_thalweg_laterally(
     elevation_raster,
     stream_raster,
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index a6128869b..5a6912543 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -292,9 +292,9 @@ def aggregate_by_huc(
     if num_job_workers > total_cpus_available:
         raise ValueError(
             f'The number of jobs {num_job_workers}'
-            'exceeds your machine\'s available CPU count minus two. '
-            'Please lower the number of jobs '
-            'values accordingly.'
+            ' exceeds your machine\'s available CPU count minus two.'
+            ' Please lower the number of jobs'
+            ' values accordingly.'
         )
 
     # create log folder, might end up empty but at least create the folder
diff --git a/src/agreedem.py b/src/agreedem.py
index 2aae80647..437d129ac 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -6,10 +6,7 @@
 import rasterio
 import whitebox
 
-from utils.shared_functions import mem_profile
 
-
-@mem_profile
 def agreedem(
     rivers_raster,
     dem,
diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py
index 4ce6ccf9c..c2db145d0 100644
--- a/src/bathy_src_adjust_topwidth.py
+++ b/src/bathy_src_adjust_topwidth.py
@@ -15,7 +15,6 @@
 
 
 sns.set_theme(style="whitegrid")
-# from utils.shared_functions import mem_profile
 
 """
     Estimate feature_id missing bathymetry in the raw channel geometry using input bankfull regression geometry
diff --git a/src/burn_in_levees.py b/src/burn_in_levees.py
index b7e08c916..17fb460d6 100755
--- a/src/burn_in_levees.py
+++ b/src/burn_in_levees.py
@@ -5,10 +5,7 @@
 import numpy as np
 import rasterio
 
-from utils.shared_functions import mem_profile
 
-
-@mem_profile
 def burn_in_levees(dem_filename, nld_filename, out_dem_filename):
     # TODO Document this code
     with rasterio.open(dem_filename) as dem, rasterio.open(nld_filename) as nld:
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index 83e78100c..ef58280d0 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -10,7 +10,12 @@
 
 
 def __read_included_files(parent_dir_path):
-    filename_patterns = glob(os.path.join(parent_dir_path, 'included_huc*.lst'))
+    # TODO: Oct25, 2023: Previously we had this test done against multiple huc lists.
+    # Now in FIM4 we only want it to check against the one file 'included_huc8.lst'.
+    # I have just replaced the pattern, but later we might want to clean this up.
+
+    # filename_patterns = glob(os.path.join(parent_dir_path, 'included_huc*.lst'))
+    filename_patterns = glob(os.path.join(parent_dir_path, 'included_huc8.lst'))
 
     accepted_hucs_set = set()
     for filename in filename_patterns:
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 6c16af193..2b04da494 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -20,7 +20,7 @@ if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $tempHucDataDir/LeveeProtecte
     echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/mask_dem.py \
+    python3 $srcDir/mask_dem.py \
         -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
         -nld $tempHucDataDir/LeveeProtectedAreas_subset.gpkg \
         -out $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
@@ -55,7 +55,7 @@ Tcount
 echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/unique_pixel_and_allocation.py \
+python3 $srcDir/unique_pixel_and_allocation.py \
     -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
     -o $tempCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif
 Tcount
@@ -64,7 +64,7 @@ Tcount
 echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/adjust_thalweg_lateral.py \
+python3 $srcDir/adjust_thalweg_lateral.py \
     -e $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
     -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
     -a $tempCurrentBranchDataDir/demDerived_streamPixels_ids_"$current_branch_id"_allo.tif \
@@ -210,7 +210,7 @@ Tcount
 echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/filter_catchments_and_add_attributes.py \
+python3 $srcDir/filter_catchments_and_add_attributes.py \
     -i $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg \
     -f $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg \
     -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
@@ -283,7 +283,7 @@ echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_bran
 date -u
 Tstart
 
-python3 -m memory_profiler $srcDir/add_crosswalk.py \
+python3 $srcDir/add_crosswalk.py \
     -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
     -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
     -s $tempCurrentBranchDataDir/src_base_$current_branch_id.csv \
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index a638f7a84..bef42fd35 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -7,11 +7,9 @@
 import numpy as np
 
 from utils.fim_enums import FIM_exit_codes
-from utils.shared_functions import mem_profile
 from utils.shared_variables import FIM_ID
 
 
-@mem_profile
 def filter_catchments_and_add_attributes(
     input_catchments_filename,
     input_flows_filename,
diff --git a/src/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
index 8c2e7e6da..c8007ed19 100755
--- a/src/make_stages_and_catchlist.py
+++ b/src/make_stages_and_catchlist.py
@@ -6,10 +6,7 @@
 import geopandas as gpd
 import numpy as np
 
-from utils.shared_functions import mem_profile
 
-
-@mem_profile
 def make_stages_and_catchlist(
     flows_filename,
     catchments_filename,
diff --git a/src/mask_dem.py b/src/mask_dem.py
index 9f441c3b6..a303eb0c2 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -9,10 +9,7 @@
 import rasterio as rio
 from rasterio.mask import mask
 
-from utils.shared_functions import mem_profile
 
-
-@mem_profile
 def mask_dem(
     dem_filename: str,
     nld_filename: str,
diff --git a/src/reachID_grid_to_vector_points.py b/src/reachID_grid_to_vector_points.py
index 84ac2eab3..912bc719f 100755
--- a/src/reachID_grid_to_vector_points.py
+++ b/src/reachID_grid_to_vector_points.py
@@ -7,11 +7,10 @@
 import rasterio
 from shapely.geometry import Point
 
-from utils.shared_functions import getDriver, mem_profile
+from utils.shared_functions import getDriver
 from utils.shared_variables import PREP_PROJECTION
 
 
-@mem_profile
 def convert_grid_cells_to_points(raster, index_option, output_points_filename=False):
     # Input raster
     if isinstance(raster, str):
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 13be08314..4dc8a4898 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -140,7 +140,7 @@ date -u
 Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
 [ -f $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif ] && \
-python3 -m memory_profiler $srcDir/burn_in_levees.py \
+python3 $srcDir/burn_in_levees.py \
     -dem $tempHucDataDir/dem_meters.tif \
     -nld $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif \
     -out $tempHucDataDir/dem_meters.tif
@@ -181,7 +181,7 @@ Tcount
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
 date -u
 Tstart
-python3 -m memory_profiler $srcDir/agreedem.py \
+python3 $srcDir/agreedem.py \
     -r $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif \
     -d $tempHucDataDir/dem_meters.tif \
     -w $tempCurrentBranchDataDir \
@@ -198,7 +198,7 @@ if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber (Branches)"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/agreedem.py -r $tempHucDataDir/flows_grid_boolean.tif \
+    python3 $srcDir/agreedem.py -r $tempHucDataDir/flows_grid_boolean.tif \
         -d $tempHucDataDir/dem_meters.tif \
         -w $tempHucDataDir \
         -o $tempHucDataDir/dem_burned.tif \
@@ -273,7 +273,7 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"
     date -u
     Tstart
-    python3 -m memory_profiler $srcDir/usgs_gage_unit_setup.py \
+    python3 $srcDir/usgs_gage_unit_setup.py \
         -gages $inputsDir/usgs_gages/usgs_gages.gpkg \
         -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
         -ras $inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points.gpkg \
diff --git a/src/split_flows.py b/src/split_flows.py
index 1cd86abee..b44d59deb 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -26,11 +26,10 @@
 
 import build_stream_traversal
 from utils.fim_enums import FIM_exit_codes
-from utils.shared_functions import getDriver, mem_profile
+from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
 
 
-@mem_profile
 def split_flows(
     flows_filename,
     dem_filename,
diff --git a/src/unique_pixel_and_allocation.py b/src/unique_pixel_and_allocation.py
index 3e242a547..e5c71e61d 100755
--- a/src/unique_pixel_and_allocation.py
+++ b/src/unique_pixel_and_allocation.py
@@ -11,10 +11,7 @@
 import rasterio
 import whitebox
 
-from utils.shared_functions import mem_profile
 
-
-@mem_profile
 def stream_pixel_zones(stream_pixels, unique_stream_pixels):
     '''
     This function will assign a unique ID for each stream pixel and writes to file. It then uses this raster to run GRASS r.grow.distance tool to create the allocation and proximity rasters required to complete the lateral thalweg conditioning.
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 75af9695a..6adba78d4 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -8,8 +8,6 @@
 import geopandas as gpd
 import rasterio
 
-from utils.shared_functions import mem_profile
-
 
 warnings.simplefilter("ignore")
 
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 246803a02..8ef766aaa 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -10,7 +10,6 @@
 import pandas as pd
 from shapely.geometry import Point
 
-from utils.shared_functions import mem_profile
 from utils.shared_variables import PREP_CRS
 
 
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index f0b7dfaf5..aacc3b687 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -13,7 +13,6 @@
 import numpy as np
 import pandas as pd
 import rasterio
-from memory_profiler import profile
 from tqdm import tqdm
 
 import utils.shared_variables as sv
@@ -185,16 +184,6 @@ def update_raster_profile(args):
     return elev_m_filename
 
 
-def mem_profile(func):
-    def wrapper(*args, **kwargs):
-        if os.environ.get('mem') == "1":
-            profile(func)(*args, **kwargs)
-        else:
-            func(*args, **kwargs)
-
-    return wrapper
-
-
 ########################################################################
 # Function to check the age of a file (use for flagging potentially outdated input)
 ########################################################################
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
index 015d797ae..2930883db 100644
--- a/unit_tests/clip_vectors_to_wbd_test.py
+++ b/unit_tests/clip_vectors_to_wbd_test.py
@@ -4,11 +4,10 @@
 import os
 import unittest
 
+import clip_vectors_to_wbd as src
 import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-import clip_vectors_to_wbd as src
-
 
 class test_clip_vectors_to_wbd(unittest.TestCase):
 

From c13bff35e8212024dca9153b7b8406ff61a4ac59 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 17 Nov 2023 10:37:46 -0700
Subject: [PATCH 286/359] v4.4.6.0 Add acquire for South Alaska and download
 them (#1031)

---
 data/usgs/acquire_and_preprocess_3dep_dems.py | 212 +++++++++++++-----
 docs/CHANGELOG.md                             |  24 ++
 src/utils/shared_validators.py                |  69 ++++++
 3 files changed, 245 insertions(+), 60 deletions(-)
 create mode 100644 src/utils/shared_validators.py

diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index 7c3c9e1cd..3e08dc14c 100644
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -4,6 +4,7 @@
 import glob
 import logging
 import os
+import shutil
 import subprocess
 import sys
 import traceback
@@ -14,14 +15,13 @@
 import pandas as pd
 
 import utils.shared_functions as sf
-import utils.shared_variables as sv
+import utils.shared_validators as val
 from utils.shared_functions import FIM_Helpers as fh
 
 
 '''
 TODO:
-    - Change to change resolution (which means URL, block size also ahve to be parameterize)
-    - if creating polygons, take out some of the hardcoding.
+    - Add input args for resolution size, which means URL and block size also hve to be parameterized.
 '''
 
 
@@ -36,28 +36,34 @@
 
 
 def acquire_and_preprocess_3dep_dems(
-    extent_file_path, target_output_folder_path='', number_of_jobs=1, retry=False, skip_polygons=False
+    extent_file_path,
+    target_output_folder_path='',
+    number_of_jobs=1,
+    retry=False,
+    skip_polygons=False,
+    target_projection='EPSG:5070',
 ):
     '''
     Overview
     ----------
     This will download 3dep rasters from USGS using USGS vrts.
     By default USGS 3Dep stores all their rasters in lat/long (northing and easting).
-    By us downloading the rasters using WBD HUC6 clips and gdal, we an accomplish a few extra
-    steps.
+    By us downloading the rasters using WBD HUC8 (or whatever domain) clips and gdal, we can
+     accomplish a few extra steps.
         1) Ensure the projection types that are downloaded are consistant and controlled.
-           We are going to download them as NAD83 basic (espg: 4269) which is consistant
-           with other data sources, even though FIM defaults to ESRI:102039. We will
-           change that as we add the clipped version per HUC8.
-        2) ensure we are adjusting blocksizes, compression and other raster params
-        3) Create the 3dep rasters in the size we want (default at HUC4 for now)
+        2) Ensure we are adjusting blocksizes, compression and other raster params.
+        3) Create the 3dep rasters in the size we want.
 
     Notes:
-        - It really can be used for any huc size.
+        - It really can be used for any huc size or any extent poly as long as it is 10m.
         - As this is a very low use tool, most values such as the USGS vrt path, output
-          folder paths, etc are all hardcoded
-        - You can really submit any polys of any HUC size as it as long as it is 10m
-          resolution for now.
+          file names, etc are all hardcoded
+        - Currently there is no tools to extract the WBD HUC8's that are applicable. You will want
+          a gkpg for each download extent (ie. HUC8, HUC6, whatever. Make a folder of each extent
+          file you want. ie) a folder of WBD HUC8's. One gkpg per HUC8 (or whatever size you like)
+        - When we originally used this tool for CONUS+, we loaded them in HUC6's, but now
+          with selected HUC8's for South Alaska, we will use input extent files as WBD HUC8.
+        - We have a separate tool to create a VRT of any folder of rasters.
 
     Parameters
     ----------
@@ -78,7 +84,13 @@ def acquire_and_preprocess_3dep_dems(
             the projected one, then skip it
 
         - skip_polygons (bool)
-             If True, then we will not attempt to create polygon files for each dem file.
+             If True, then we will not attempt to create polygon files for each dem file. If false,
+             an domain gpkg which covers the extent of all included features merged. It will automatically
+             be named DEM_Domain.gkpg and saved in the same folderd as the target_output_folder_path.
+
+        - target_projection (String)
+            Projection of the output DEMS and polygons (if included)
+
 
     '''
     # -------------------
@@ -86,10 +98,10 @@ def acquire_and_preprocess_3dep_dems(
     total_cpus_available = os.cpu_count() - 1
     if number_of_jobs > total_cpus_available:
         raise ValueError(
-            f'The number of jobs provided: {number_of_jobs} , '
-            f'exceeds your machine\'s available CPU count minus one. '
-            f'Please lower the number of jobs '
-            f'value accordingly.'
+            f'The number of jobs provided: {number_of_jobs} ,'
+            ' exceeds your machine\'s available CPU count minus one.'
+            ' Please lower the number of jobs'
+            ' value accordingly.'
         )
 
     if not os.path.exists(extent_file_path):
@@ -99,11 +111,55 @@ def acquire_and_preprocess_3dep_dems(
         target_output_folder_path = os.environ['usgs_3dep_dems_10m']
 
     if not os.path.exists(target_output_folder_path):
-        raise ValueError(f"Output folder path {target_output_folder_path} does not exist")
+        # It is ok if the child diretory does not exist, but the parent folder must
+        # parent directory
+        parent_dir = os.path.abspath(os.path.join(target_output_folder_path, os.pardir))
+        print(parent_dir)
+        if not os.path.exists(parent_dir):
+            raise ValueError(
+                f"For the output path of {target_output_folder_path}, the child directory"
+                " need not exist but the parent folder must."
+            )
+        os.makedirs(target_output_folder_path)
+    else:  # path exists
+        if not retry:
+            file_list = os.listdir(target_output_folder_path)
+            if len(file_list) > 0:
+                print()
+                msg = f"The target output folder of {target_output_folder_path} appears to not be empty.\n\n"
+                "Do you want to empty the folder first?\n"
+                "  -- Type 'overwrite' if you want to empty the folder and continue.\n"
+                "  -- Type any other value to abort and stop the program.\n"
+                "  ?="
+
+                resp = input(msg).lower()
+                if resp == "overwrite":
+                    shutil.rmtree(target_output_folder_path)
+                    os.mkdir(target_output_folder_path)
+                else:
+                    print("Program stopped\n")
+                    sys.exit(0)
+        else:  # might want to retry but the folder isn't there yet
+            # It is ok if the child diretory does not exist, but the parent folder must
+            # parent directory, we want to reset it
+            parent_dir = os.path.abspath(os.path.join(target_output_folder_path, os.pardir))
+            print(parent_dir)
+            if not os.path.exists(parent_dir):
+                raise ValueError(
+                    f"For the output path of {target_output_folder_path}, the child directory"
+                    " need not exist but the parent folder must."
+                )
+            shutil.rmtree(target_output_folder_path)
+            os.mkdir(target_output_folder_path)
+
+    # I don't need the crs_number for now
+    crs_is_valid, err_msg, crs_number = val.is_valid_crs(target_projection)
+    if crs_is_valid is False:
+        raise ValueError(err_msg)
 
     # -------------------
     # setup logs
-    start_time = datetime.now()
+    start_time = datetime.utcnow()
     fh.print_start_header('Loading 3dep dems', start_time)
 
     # print(f"Downloading to {target_output_folder_path}")
@@ -120,12 +176,14 @@ def acquire_and_preprocess_3dep_dems(
     logging.info(msg)
 
     # download dems, setting projection, block size, etc
-    __download_usgs_dems(extent_file_names, target_output_folder_path, number_of_jobs, retry)
+    __download_usgs_dems(
+        extent_file_names, target_output_folder_path, number_of_jobs, retry, target_projection
+    )
 
     if skip_polygons is False:
         polygonize(target_output_folder_path)
 
-    end_time = datetime.now()
+    end_time = datetime.utcnow()
     fh.print_end_header('Loading 3dep dems', start_time, end_time)
 
     print()
@@ -137,7 +195,7 @@ def acquire_and_preprocess_3dep_dems(
     logging.info(fh.print_date_time_duration(start_time, end_time))
 
 
-def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry):
+def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry, target_projection):
     '''
     Process:
     ----------
@@ -167,6 +225,15 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     base_cmd += ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
     base_cmd += ' -t_srs {3} -cblend 6'
 
+    """
+    e.q. gdalwarp
+       /vs/icurl/https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/TIFF/USGS_Seamless_DEM_13.vrt
+       /data/inputs/usgs/3dep_dems/10m/HUC8_12090301_dem.tif
+       -cutline /data/inputs/wbd/HUC8/HUC8_12090301.gpkg
+       -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"
+       -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10 -t_srs ESRI:102039 -cblend 6
+    """
+
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
         executor_dict = {}
 
@@ -175,6 +242,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
                 'extent_file': extent_file,
                 'output_folder_path': output_folder_path,
                 'download_url': __USGS_3DEP_10M_VRT_URL,
+                'target_projection': target_projection,
                 'base_cmd': base_cmd,
                 'retry': retry,
             }
@@ -200,7 +268,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     print("==========================================================")
 
 
-def download_usgs_dem_file(extent_file, output_folder_path, download_url, base_cmd, retry):
+def download_usgs_dem_file(extent_file, output_folder_path, download_url, target_projection, base_cmd, retry):
     '''
     Process:
     ----------
@@ -217,15 +285,10 @@ def download_usgs_dem_file(extent_file, output_folder_path, download_url, base_c
         - download_url (str)
              URL for the USGS download site (note: Should include '/vsicurl/' at the
              front of the URL)
+        - target_projection (str)
+            ie) EPSG:5070 or EPSG:2276, etc
         - base_cmd (str)
              The basic GDAL command with string formatting wholes for key values.
-             See the cmd variable below.
-             ie)
-                base_cmd =  'gdalwarp {0} {1}'
-                base_cmd += ' -cutline {2} -crop_to_cutline -ot Float32 -r bilinear'
-                base_cmd +=  ' -of "GTiff" -overwrite -co "BLOCKXSIZE=256" -co "BLOCKYSIZE=256"'
-                base_cmd +=  ' -co "TILED=YES" -co "COMPRESS=LZW" -co "BIGTIFF=YES" -tr 10 10'
-                base_cmd +=  ' -t_srs {3} -cblend 6'
         - retry (bool)
              If True, and the file exists, downloading will be skipped.
 
@@ -249,44 +312,59 @@ def download_usgs_dem_file(extent_file, output_folder_path, download_url, base_c
     print(msg)
     logging.info(msg)
 
-    cmd = base_cmd.format(download_url, target_path_raw, extent_file, sv.DEFAULT_FIM_PROJECTION_CRS)
-    # PREP_PROJECTION_EPSG
-    # fh.vprint(f"cmd is {cmd}", self.is_verbose, True)
+    cmd = base_cmd.format(download_url, target_path_raw, extent_file, target_projection)
     # print(f"cmd is {cmd}")
 
     # didn't use Popen becuase of how it interacts with multi proc
     # was creating some issues. Run worked much better.
-    process = subprocess.run(
-        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, universal_newlines=True
-    )
 
-    msg = process.stdout
-    print(msg)
-    logging.info(msg)
+    try:
+        process = subprocess.run(
+            cmd,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=True,
+            universal_newlines=True,
+        )
 
-    if process.stderr != "":
-        if "ERROR" in process.stderr.upper():
-            msg = f" - Downloading -- {target_file_name_raw}" f"  ERROR -- details: ({process.stderr})"
-            print(msg)
-            logging.error(msg)
-            os.remove(target_path_raw)
-    else:
-        msg = f" - Downloading -- {target_file_name_raw} - Complete"
+        msg = process.stdout
         print(msg)
         logging.info(msg)
 
+        if process.stderr != "":
+            if "ERROR" in process.stderr.upper():
+                msg = f" - Downloading -- {target_file_name_raw}" f"  ERROR -- details: ({process.stderr})"
+                print(msg)
+                logging.error(msg)
+                os.remove(target_path_raw)
+        else:
+            msg = f" - Downloading -- {target_file_name_raw} - Complete"
+            print(msg)
+            logging.info(msg)
+    except Exception:
+        msg = "An exception occurred while downloading files."
+        print(msg)
+        print(traceback.format_exc())
+        logging.critical(traceback.format_exc())
+        sys.exit(1)
+
 
 def polygonize(target_output_folder_path):
     """
-    Create a polygon of 3DEP domain from individual HUC6 DEMS which are then dissolved into a single polygon
+    Create a polygon of 3DEP domain from individual HUC DEMS which are then dissolved into a single polygon
     """
-    dem_domain_file = os.path.join(target_output_folder_path, 'HUC6_dem_domain.gpkg')
+    dem_domain_file = os.path.join(target_output_folder_path, 'DEM_Domain.gpkg')
 
     msg = f" - Polygonizing -- {dem_domain_file} - Started"
     print(msg)
     logging.info(msg)
 
     dem_files = glob.glob(os.path.join(target_output_folder_path, '*_dem.tif'))
+
+    if len(dem_files) == 0:
+        raise Exception("There are no DEMs to polygonize")
+
     dem_gpkgs = gpd.GeoDataFrame()
 
     for n, dem_file in enumerate(dem_files):
@@ -328,6 +406,7 @@ def polygonize(target_output_folder_path):
             dem_gpkgs = pd.concat([dem_gpkgs, gdf])
 
         os.remove(edge_tif)
+        os.remove(edge_gpkg)
 
     dem_gpkgs['DN'] = 1
     dem_dissolved = dem_gpkgs.dissolve(by='DN')
@@ -344,7 +423,7 @@ def polygonize(target_output_folder_path):
 
 
 def __setup_logger(output_folder_path):
-    start_time = datetime.now()
+    start_time = datetime.utcnow()
     file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
     log_file_name = f"3Dep_downloaded-{file_dt_string}.log"
 
@@ -359,7 +438,7 @@ def __setup_logger(output_folder_path):
     logger.addHandler(file_handler)
     logger.setLevel(logging.DEBUG)
 
-    logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
+    logging.info(f'Started (UTC): {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
     logging.info("----------------")
 
 
@@ -393,6 +472,10 @@ def __setup_logger(output_folder_path):
     We left in HUC2 of 19 (alaska) as we hope to get there in the semi near future
     They need to be removed from the input src clip directory in the first place.
     They can not be reliably removed in code.
+
+    (Update Nov 2023): South Alaska (not all of HUC2 = 19) is now included but not all of Alaska.
+    A separate output directory will be keep for South Alaska and will use EPSG:3338 versus the FIM
+    default of EPSG:5070
     '''
 
     parser = argparse.ArgumentParser(description='Acquires and preprocesses USGS 3Dep dems')
@@ -400,8 +483,8 @@ def __setup_logger(output_folder_path):
     parser.add_argument(
         '-e',
         '--extent_file_path',
-        help='location the gpkg files that will'
-        ' are being used as clip regions (ie: huc4_*.gpkg).'
+        help='REQUIRED: location the gpkg files that will'
+        ' are being used as clip regions (ie: huc8_*.gpkg).'
         ' All gpkgs in this folder will be used.',
         required=True,
     )
@@ -409,7 +492,7 @@ def __setup_logger(output_folder_path):
     parser.add_argument(
         '-j',
         '--number_of_jobs',
-        help='Number of (jobs) cores/processes to used.',
+        help='OPTIONAL: Number of (jobs) cores/processes to used.',
         required=False,
         default=1,
         type=int,
@@ -418,7 +501,8 @@ def __setup_logger(output_folder_path):
     parser.add_argument(
         '-r',
         '--retry',
-        help='If included, it will skip files that already exist.' ' Default is all will be loaded/reloaded.',
+        help='OPTIONAL: If included, it will skip files that already exist.'
+        'Default is all will be loaded/reloaded.',
         required=False,
         action='store_true',
         default=False,
@@ -427,7 +511,7 @@ def __setup_logger(output_folder_path):
     parser.add_argument(
         '-t',
         '--target_output_folder_path',
-        help='location of where the 3dep files' ' will be saved',
+        help='OPTIONAL: location of where the 3dep files will be saved.',
         required=False,
         default='',
     )
@@ -435,12 +519,20 @@ def __setup_logger(output_folder_path):
     parser.add_argument(
         '-sp',
         '--skip_polygons',
-        help='If this flag is included, polygons of the dems' ' will not be made',
+        help='OPTIONAL: If this flag is included, polygons of the dems will not be made.',
         required=False,
         action='store_true',
         default=False,
     )
 
+    parser.add_argument(
+        '-proj',
+        '--target_projection',
+        help='OPTIONAL: Desired output CRS. Defaults to EPSG:5070',
+        required=False,
+        default='EPSG:5070',
+    )
+
     # Extract to dictionary and assign to variables.
     args = vars(parser.parse_args())
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a6b6c8352..7d9369013 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,30 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.6.0 - 2023-11-17 - [PR#1031](https://github.com/NOAA-OWP/inundation-mapping/pull/1031)
+
+Upgrade our acquire 3Dep DEMs script to pull down South Alaska HUCS with its own CRS.
+
+The previous set of DEMs run for FIM and it's related vrt already included all of Alaska, and those have not been re-run. FIM code will be updated in the near future to detect if the HUC starts with a `19` with slight different logic, so it can preserve the CRS of EPSG:3338 all the way to final FIM outputs.  See [792 ](https://github.com/NOAA-OWP/inundation-mapping/issues/792)for new integration into FIM.
+
+A new vrt for the new South Alaska DEMs was also run with no changes required.
+
+This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1028). 
+
+### Additions
+- `src/utils`
+     - `shared_validators.py`: A new script where we can put in code to validate more complex arguments for python scripts. Currently has one for validating CRS values. It does valid if the CRS value is legitimate but does check a bunch of formatting including that it starts with either the name of `EPSG` or `ESRI`
+
+### Changes
+- `data/usgs` 
+    - `aquire_and_preprocess_3dep_dems.py`: Changes include:
+        - Add new input arg for desired target projection and logic to support an incoming CRS.
+        - Updated logic for pre-existing output folders and `on-the-fly` question to users during execution if they want to overwrite the output folder (if applicable).
+        - Changed date/times to utc.
+        - Upgraded error handing for the gdal "processing" call.
+
+<br/><br/>
+
 ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
 
 During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times.
diff --git a/src/utils/shared_validators.py b/src/utils/shared_validators.py
new file mode 100644
index 000000000..b51251283
--- /dev/null
+++ b/src/utils/shared_validators.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+####################################################################
+"""
+This script is for validation of inputs and objects that can be used in py files.
+"""
+
+
+# -------------------------------------------------
+def is_valid_crs(crs):
+    """
+    Processing:
+        - not counting case, the first for chars must be either EPSG or ESRI
+        - crs number must be between 4 and 6 digits
+    Inputs:
+        - crs (str): in pattern such as EPSG:2277
+
+    Returns three values
+        - First is bool (True or False). True meaning it is a valid crs
+        - Second is a string, which might be empty, but if failed, then this will be the reason for failure
+        - Third is the raw crs number in case you need it. ie) 2277 or 107239
+
+        It is up to the calling code to decide if an exception should be raised.
+
+    Usage:
+        valid_crs, err_msg, crs_number = is_valid_crs(arg_crs)
+
+        (if you choose to hand it this way....)
+        if (valid_crs == False):
+            raise ValueError(err_msg)
+
+    """
+
+    err_msg = ""
+
+    if crs == "":
+        err_msg = "The crs value can not be blank or empty"
+        return False, err_msg, ""
+
+    if ":" not in crs:
+        err_msg = "crs appears to be invalid (missing colon)"
+        return False, err_msg, ""
+
+    crs_seg = crs.split(":")
+
+    if len(crs_seg) != 2:
+        err_msg = "crs appears to be invalid"
+        return False, err_msg, ""
+
+    crs_type = str(crs_seg[0]).upper()
+    if (crs_type != "ESRI") and (crs_type != "EPSG"):
+        err_msg = "crs type is not EPSG OR ESRI (not case-senstive)"
+        return False, err_msg, ""
+
+    crs_number = str(crs_seg[1])
+
+    if crs_number.isnumeric() is False:
+        err_msg = "value after the colon is not a number. ie) 2277"
+        return False, err_msg, ""
+
+    if (len(crs_number) < 4) or (len(crs_number) > 6):
+        err_msg = "crs_number portion is expected to be between 4 and 6 digits"
+        return False, err_msg, ""
+
+    if crs_number[0] == "0":
+        err_msg = "crs_number portion can not start with a zero"
+        return False, err_msg, ""
+
+    return True, "", crs_number

From 89c7023b895c59abd7acb12174ecd032ec7608cb Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 17 Nov 2023 10:41:42 -0700
Subject: [PATCH 287/359] v4.4.7.0 Add CI - GitHub Actions Workflow file
 (#1030)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add GitHub Actions workflow file, update files to remove flake8 E721 errors

* Update Changelog, remove pflake8 reference in pre-commit-config.yaml

* Pass all unit_tests, skipping 2

* Conform unit_tests/clip_vectors_to_wbd_test.py to isort standard, update CHANGELOG

* Update CONTRIBUTING.md

* fix encoding issue

* Update unit tests, remove unit_tests/tools directory, update README

* Adhere to isort auto-fix

* Bump pyarrow from 11.0.0 to 14.0.1

Bumps [pyarrow](https://github.com/apache/arrow) from 11.0.0 to 14.0.1.
- [Commits](https://github.com/apache/arrow/compare/go/v11.0.0...apache-arrow-14.0.1)

---
updated-dependencies:
- dependency-name: pyarrow
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

* Update CHANGELOG.md

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: “RobHanna-NOAA” <“Robert.Hanna@NOAA.gov”>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
---
 .github/workflows/lint_and_format.yaml        |   22 +
 .pre-commit-config.yaml                       |    5 +-
 CONTRIBUTING.md                               |    2 +-
 Pipfile                                       |    4 +-
 Pipfile.lock                                  | 1065 ++++++++---------
 data/wbd/generate_pre_clip_fim_huc8.py        |    2 +-
 data/write_parquet_from_calib_pts.py          |    2 +-
 docs/CHANGELOG.md                             |   47 +
 src/check_huc_inputs.py                       |    5 +-
 src/src_adjust_ras2fim_rating.py              |    4 +-
 tools/eval_plots.py                           |    4 +-
 tools/generate_categorical_fim_mapping.py     |    2 +-
 tools/hash_compare.py                         |    2 +-
 tools/inundate_mosaic_wrapper.py              |    3 +-
 tools/inundation_wrapper_nwm_flows.py         |    2 +-
 tools/mosaic_inundation.py                    |    2 +-
 unit_tests/README.md                          |   15 +-
 unit_tests/clip_vectors_to_wbd_test.py        |    9 +-
 ..._catchments_and_add_attributes_params.json |    8 +-
 .../{tools => }/inundate_gms_params.json      |    0
 unit_tests/{tools => }/inundate_gms_test.py   |    3 +-
 unit_tests/{tools => }/inundation_params.json |    0
 unit_tests/{tools => }/inundation_test.py     |    0
 unit_tests/outputs_cleanup_params.json        |    4 +-
 unit_tests/outputs_cleanup_test.py            |    2 +-
 unit_tests/split_flows_params.json            |    8 +-
 unit_tests/tools/__init__.py                  |    0
 unit_tests/usgs_gage_crosswalk_params.json    |   13 +-
 unit_tests/usgs_gage_crosswalk_test.py        |    2 +-
 29 files changed, 638 insertions(+), 599 deletions(-)
 create mode 100644 .github/workflows/lint_and_format.yaml
 mode change 100755 => 100644 Pipfile
 rename unit_tests/{tools => }/inundate_gms_params.json (100%)
 rename unit_tests/{tools => }/inundate_gms_test.py (97%)
 rename unit_tests/{tools => }/inundation_params.json (100%)
 rename unit_tests/{tools => }/inundation_test.py (100%)
 delete mode 100644 unit_tests/tools/__init__.py

diff --git a/.github/workflows/lint_and_format.yaml b/.github/workflows/lint_and_format.yaml
new file mode 100644
index 000000000..0032cd083
--- /dev/null
+++ b/.github/workflows/lint_and_format.yaml
@@ -0,0 +1,22 @@
+name: Lint and Format Using Pre-Commit
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  lint-and-format:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version-file: pyproject.toml
+    - uses: pre-commit/action@v3.0.0
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2aebea2c3..98fbcea77 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,11 +14,10 @@ repos:
         -   id: check-json
 
 -   repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
         -   id: flake8
-            entry: pflake8
-            additional_dependencies: [pyproject-flake8]
+            additional_dependencies: [flake8-pyproject]
 
 -   repo: https://github.com/psf/black
     rev: 23.7.0
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f51276063..940cfe712 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -65,7 +65,7 @@ If you would like to contribute, please follow these steps:
     # Check all files in the repo
     pre-commit run -a
 
-    # Run only the black formatting tool
+    # Run only the flake8 formatting tool
     pre-commit run -a flake8
     ```
 
diff --git a/Pipfile b/Pipfile
old mode 100755
new mode 100644
index 85d6d7fe8..beace0056
--- a/Pipfile
+++ b/Pipfile
@@ -32,14 +32,14 @@ ipympl = "==0.9.3"
 pytest = "==7.3.0"
 whitebox = "==2.3.1"
 shapely = "==2.0.1"
-pyarrow = "==11.0.0"
+pyarrow = "==14.0.1"
 rtree = "==1.0.1"
 py7zr = "==0.20.4"
 scipy = "==1.10.1"
 gval = "==0.2.3"
 flake8 = "==6.0.0"
 black = "==23.7.0"
-pyproject-flake8 = "==6.0.0.post1"
+flake8-pyproject = "==1.2.3"
 pre-commit = "==3.3.3"
 isort = "==5.12.0"
 urllib3 = "==1.26.18"
diff --git a/Pipfile.lock b/Pipfile.lock
index cdb131185..61853dec6 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c7f9ee645e067007d14cfcb885a4a4a4c8c720ed6b17f42c07328378ee8c1121"
+            "sha256": "196cd7e33d0458f7b032017fcd2e1940761158c53a4130adb8915f036ed61b49"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -100,10 +100,10 @@
         },
         "asttokens": {
             "hashes": [
-                "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
-                "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
+                "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24",
+                "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"
             ],
-            "version": "==2.4.0"
+            "version": "==2.4.1"
         },
         "attrs": {
             "hashes": [
@@ -415,99 +415,99 @@
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:06cf46bdff72f58645434d467bf5228080801298fbba19fe268a01b4534467f5",
-                "sha256:0c8c61fb505c7dad1d251c284e712d4e0372cef3b067f7ddf82a7fa82e1e9a93",
-                "sha256:10b8dd31e10f32410751b3430996f9807fc4d1587ca69772e2aa940a82ab571a",
-                "sha256:1171ef1fc5ab4693c5d151ae0fdad7f7349920eabbaca6271f95969fa0756c2d",
-                "sha256:17a866d61259c7de1bdadef418a37755050ddb4b922df8b356503234fff7932c",
-                "sha256:1d6bfc32a68bc0933819cfdfe45f9abc3cae3877e1d90aac7259d57e6e0f85b1",
-                "sha256:1ec937546cad86d0dce5396748bf392bb7b62a9eeb8c66efac60e947697f0e58",
-                "sha256:223b4d54561c01048f657fa6ce41461d5ad8ff128b9678cfe8b2ecd951e3f8a2",
-                "sha256:2465aa50c9299d615d757c1c888bc6fef384b7c4aec81c05a0172b4400f98557",
-                "sha256:28f512b9a33235545fbbdac6a330a510b63be278a50071a336afc1b78781b147",
-                "sha256:2c092be3885a1b7899cd85ce24acedc1034199d6fca1483fa2c3a35c86e43041",
-                "sha256:2c4c99f98fc3a1835af8179dcc9013f93594d0670e2fa80c83aa36346ee763d2",
-                "sha256:31445f38053476a0c4e6d12b047b08ced81e2c7c712e5a1ad97bc913256f91b2",
-                "sha256:31bbaba7218904d2eabecf4feec0d07469284e952a27400f23b6628439439fa7",
-                "sha256:34d95638ff3613849f473afc33f65c401a89f3b9528d0d213c7037c398a51296",
-                "sha256:352a88c3df0d1fa886562384b86f9a9e27563d4704ee0e9d56ec6fcd270ea690",
-                "sha256:39b70a6f88eebe239fa775190796d55a33cfb6d36b9ffdd37843f7c4c1b5dc67",
-                "sha256:3c66df3f41abee950d6638adc7eac4730a306b022570f71dd0bd6ba53503ab57",
-                "sha256:3f70fd716855cd3b855316b226a1ac8bdb3caf4f7ea96edcccc6f484217c9597",
-                "sha256:3f9bc2ce123637a60ebe819f9fccc614da1bcc05798bbbaf2dd4ec91f3e08846",
-                "sha256:3fb765362688821404ad6cf86772fc54993ec11577cd5a92ac44b4c2ba52155b",
-                "sha256:45f053a0ece92c734d874861ffe6e3cc92150e32136dd59ab1fb070575189c97",
-                "sha256:46fb9970aa5eeca547d7aa0de5d4b124a288b42eaefac677bde805013c95725c",
-                "sha256:4cb50a0335382aac15c31b61d8531bc9bb657cfd848b1d7158009472189f3d62",
-                "sha256:4e12f8ee80aa35e746230a2af83e81bd6b52daa92a8afaef4fea4a2ce9b9f4fa",
-                "sha256:4f3100d86dcd03c03f7e9c3fdb23d92e32abbca07e7c13ebd7ddfbcb06f5991f",
-                "sha256:4f6e2a839f83a6a76854d12dbebde50e4b1afa63e27761549d006fa53e9aa80e",
-                "sha256:4f861d94c2a450b974b86093c6c027888627b8082f1299dfd5a4bae8e2292821",
-                "sha256:501adc5eb6cd5f40a6f77fbd90e5ab915c8fd6e8c614af2db5561e16c600d6f3",
-                "sha256:520b7a142d2524f999447b3a0cf95115df81c4f33003c51a6ab637cbda9d0bf4",
-                "sha256:548eefad783ed787b38cb6f9a574bd8664468cc76d1538215d510a3cd41406cb",
-                "sha256:555fe186da0068d3354cdf4bbcbc609b0ecae4d04c921cc13e209eece7720727",
-                "sha256:55602981b2dbf8184c098bc10287e8c245e351cd4fdcad050bd7199d5a8bf514",
-                "sha256:58e875eb7016fd014c0eea46c6fa92b87b62c0cb31b9feae25cbbe62c919f54d",
-                "sha256:5a3580a4fdc4ac05f9e53c57f965e3594b2f99796231380adb2baaab96e22761",
-                "sha256:5b70bab78accbc672f50e878a5b73ca692f45f5b5e25c8066d748c09405e6a55",
-                "sha256:5ceca5876032362ae73b83347be8b5dbd2d1faf3358deb38c9c88776779b2e2f",
-                "sha256:61f1e3fb621f5420523abb71f5771a204b33c21d31e7d9d86881b2cffe92c47c",
-                "sha256:633968254f8d421e70f91c6ebe71ed0ab140220469cf87a9857e21c16687c034",
-                "sha256:63a6f59e2d01310f754c270e4a257426fe5a591dc487f1983b3bbe793cf6bac6",
-                "sha256:63accd11149c0f9a99e3bc095bbdb5a464862d77a7e309ad5938fbc8721235ae",
-                "sha256:6db3cfb9b4fcecb4390db154e75b49578c87a3b9979b40cdf90d7e4b945656e1",
-                "sha256:71ef3b9be10070360f289aea4838c784f8b851be3ba58cf796262b57775c2f14",
-                "sha256:7ae8e5142dcc7a49168f4055255dbcced01dc1714a90a21f87448dc8d90617d1",
-                "sha256:7b6cefa579e1237ce198619b76eaa148b71894fb0d6bcf9024460f9bf30fd228",
-                "sha256:800561453acdecedaac137bf09cd719c7a440b6800ec182f077bb8e7025fb708",
-                "sha256:82ca51ff0fc5b641a2d4e1cc8c5ff108699b7a56d7f3ad6f6da9dbb6f0145b48",
-                "sha256:851cf693fb3aaef71031237cd68699dded198657ec1e76a76eb8be58c03a5d1f",
-                "sha256:854cc74367180beb327ab9d00f964f6d91da06450b0855cbbb09187bcdb02de5",
-                "sha256:87071618d3d8ec8b186d53cb6e66955ef2a0e4fa63ccd3709c0c90ac5a43520f",
-                "sha256:871d045d6ccc181fd863a3cd66ee8e395523ebfbc57f85f91f035f50cee8e3d4",
-                "sha256:8aee051c89e13565c6bd366813c386939f8e928af93c29fda4af86d25b73d8f8",
-                "sha256:8af5a8917b8af42295e86b64903156b4f110a30dca5f3b5aedea123fbd638bff",
-                "sha256:8ec8ef42c6cd5856a7613dcd1eaf21e5573b2185263d87d27c8edcae33b62a61",
-                "sha256:91e43805ccafa0a91831f9cd5443aa34528c0c3f2cc48c4cb3d9a7721053874b",
-                "sha256:9505dc359edb6a330efcd2be825fdb73ee3e628d9010597aa1aee5aa63442e97",
-                "sha256:985c7965f62f6f32bf432e2681173db41336a9c2611693247069288bcb0c7f8b",
-                "sha256:9a74041ba0bfa9bc9b9bb2cd3238a6ab3b7618e759b41bd15b5f6ad958d17605",
-                "sha256:9edbe6a5bf8b56a4a84533ba2b2f489d0046e755c29616ef8830f9e7d9cf5728",
-                "sha256:a15c1fe6d26e83fd2e5972425a772cca158eae58b05d4a25a4e474c221053e2d",
-                "sha256:a66bcdf19c1a523e41b8e9d53d0cedbfbac2e93c649a2e9502cb26c014d0980c",
-                "sha256:ae4070f741f8d809075ef697877fd350ecf0b7c5837ed68738607ee0a2c572cf",
-                "sha256:ae55d592b02c4349525b6ed8f74c692509e5adffa842e582c0f861751701a673",
-                "sha256:b578cbe580e3b41ad17b1c428f382c814b32a6ce90f2d8e39e2e635d49e498d1",
-                "sha256:b891a2f68e09c5ef989007fac11476ed33c5c9994449a4e2c3386529d703dc8b",
-                "sha256:baec8148d6b8bd5cee1ae138ba658c71f5b03e0d69d5907703e3e1df96db5e41",
-                "sha256:bb06098d019766ca16fc915ecaa455c1f1cd594204e7f840cd6258237b5079a8",
-                "sha256:bc791ec3fd0c4309a753f95bb6c749ef0d8ea3aea91f07ee1cf06b7b02118f2f",
-                "sha256:bd28b31730f0e982ace8663d108e01199098432a30a4c410d06fe08fdb9e93f4",
-                "sha256:be4d9c2770044a59715eb57c1144dedea7c5d5ae80c68fb9959515037cde2008",
-                "sha256:c0c72d34e7de5604df0fde3644cc079feee5e55464967d10b24b1de268deceb9",
-                "sha256:c0e842112fe3f1a4ffcf64b06dc4c61a88441c2f02f373367f7b4c1aa9be2ad5",
-                "sha256:c15070ebf11b8b7fd1bfff7217e9324963c82dbdf6182ff7050519e350e7ad9f",
-                "sha256:c2000c54c395d9e5e44c99dc7c20a64dc371f777faf8bae4919ad3e99ce5253e",
-                "sha256:c30187840d36d0ba2893bc3271a36a517a717f9fd383a98e2697ee890a37c273",
-                "sha256:cb7cd68814308aade9d0c93c5bd2ade9f9441666f8ba5aa9c2d4b389cb5e2a45",
-                "sha256:cd805513198304026bd379d1d516afbf6c3c13f4382134a2c526b8b854da1c2e",
-                "sha256:d0bf89afcbcf4d1bb2652f6580e5e55a840fdf87384f6063c4a4f0c95e378656",
-                "sha256:d9137a876020661972ca6eec0766d81aef8a5627df628b664b234b73396e727e",
-                "sha256:dbd95e300367aa0827496fe75a1766d198d34385a58f97683fe6e07f89ca3e3c",
-                "sha256:dced27917823df984fe0c80a5c4ad75cf58df0fbfae890bc08004cd3888922a2",
-                "sha256:de0b4caa1c8a21394e8ce971997614a17648f94e1cd0640fbd6b4d14cab13a72",
-                "sha256:debb633f3f7856f95ad957d9b9c781f8e2c6303ef21724ec94bea2ce2fcbd056",
-                "sha256:e372d7dfd154009142631de2d316adad3cc1c36c32a38b16a4751ba78da2a397",
-                "sha256:ecd26be9f112c4f96718290c10f4caea6cc798459a3a76636b817a0ed7874e42",
-                "sha256:edc0202099ea1d82844316604e17d2b175044f9bcb6b398aab781eba957224bd",
-                "sha256:f194cce575e59ffe442c10a360182a986535fd90b57f7debfaa5c845c409ecc3",
-                "sha256:f5fb672c396d826ca16a022ac04c9dce74e00a1c344f6ad1a0fdc1ba1f332213",
-                "sha256:f6a02a3c7950cafaadcd46a226ad9e12fc9744652cc69f9e5534f98b47f3bbcf",
-                "sha256:fe81b35c33772e56f4b6cf62cf4aedc1762ef7162a31e6ac7fe5e40d0149eb67"
+                "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027",
+                "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087",
+                "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786",
+                "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8",
+                "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09",
+                "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185",
+                "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574",
+                "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e",
+                "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519",
+                "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898",
+                "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269",
+                "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3",
+                "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f",
+                "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6",
+                "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8",
+                "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a",
+                "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73",
+                "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc",
+                "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714",
+                "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2",
+                "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc",
+                "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce",
+                "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d",
+                "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e",
+                "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6",
+                "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269",
+                "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96",
+                "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d",
+                "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a",
+                "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4",
+                "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77",
+                "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d",
+                "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0",
+                "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed",
+                "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068",
+                "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac",
+                "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25",
+                "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8",
+                "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab",
+                "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26",
+                "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2",
+                "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db",
+                "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f",
+                "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5",
+                "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99",
+                "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c",
+                "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d",
+                "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811",
+                "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa",
+                "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a",
+                "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03",
+                "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b",
+                "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04",
+                "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c",
+                "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001",
+                "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458",
+                "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389",
+                "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99",
+                "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985",
+                "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537",
+                "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238",
+                "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f",
+                "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d",
+                "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796",
+                "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a",
+                "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143",
+                "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8",
+                "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c",
+                "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5",
+                "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5",
+                "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711",
+                "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4",
+                "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6",
+                "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c",
+                "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7",
+                "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4",
+                "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b",
+                "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae",
+                "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12",
+                "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c",
+                "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae",
+                "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8",
+                "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887",
+                "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b",
+                "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4",
+                "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f",
+                "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5",
+                "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33",
+                "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
+                "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.3.1"
+            "version": "==3.3.2"
         },
         "click": {
             "hashes": [
@@ -529,7 +529,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -550,11 +550,11 @@
         },
         "comm": {
             "hashes": [
-                "sha256:354e40a59c9dd6db50c5cc6b4acc887d82e9603787f83b68c01a80a923984d15",
-                "sha256:6d52794cba11b36ed9860999cd10fd02d6b2eac177068fdd585e1e2f8a96e67a"
+                "sha256:2da8d9ebb8dd7bfc247adaff99f24dce705638a8042b85cb995066793e391001",
+                "sha256:a517ea2ca28931c7007a7a99c562a0fa5883cfb48963140cf642c41c948498be"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.1.4"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.2.0"
         },
         "contextily": {
             "hashes": [
@@ -632,67 +632,67 @@
         },
         "cython": {
             "hashes": [
-                "sha256:0021350f6d7022a37f598320460b84b2c0daccf6bb65641bbdbc8b990bdf4ad2",
-                "sha256:003ccc40e0867770db0018274977d1874e4df64983d5e3e36937f107e0b2fdf6",
-                "sha256:0422a40a58dcfbb54c8b4e125461d741031ff046bc678475cc7a6c801d2a7721",
-                "sha256:0612439f810cc281e51fead69de0545c4d9772a1e82149c119d1aafc1f6210ba",
-                "sha256:061dec1be8d8b601b160582609a78eb08324a4ccf21bee0d04853a3e9dfcbefd",
-                "sha256:0650460b5fd6f16da4186e0a769b47db5532601e306f3b5d17941283d5e36d24",
-                "sha256:07d0e69959f267b79ffd18ece8599711ad2f3d3ed1eddd0d4812d2a97de2b912",
-                "sha256:096cb461bf8d913a4327d93ea38d18bc3dbc577a71d805be04754e4b2cc2c45d",
-                "sha256:0aa2a6bb3ff67794d8d1dafaed22913adcbb327e96eca3ac44e2f3ba4a0ae446",
-                "sha256:0fc9e974419cc0393072b1e9a669f71c3b34209636d2005ff8620687daa82b8c",
-                "sha256:0fe227d6d8e2ea030e82abc8a3e361e31447b66849f8c069caa783999e54a8f2",
-                "sha256:10b426adc3027d66303f5c7aa8b254d10ed80827ff5cce9e892d550b708dc044",
-                "sha256:146bfaab567157f4aa34114a37e3f98a3d9c4527ee99d4fd730cab56482bd3cf",
-                "sha256:15d52f7f9d08b264c042aa508bf457f53654b55f533e0262e146002b1c15d1cd",
-                "sha256:19a64bf2591272348ab08bcd4a5f884259cc3186f008c9038b8ec7d01f847fd5",
-                "sha256:207f53893ca22d8c8f5db533f38382eb7ddc2d0b4ab51699bf052423a6febdad",
-                "sha256:28af4e7dff1742cb0f0a4823102c89c62a2d94115b68f718145fcfe0763c6e21",
-                "sha256:28de18f0d07eb34e2dd7b022ac30beab0fdd277846d07b7a08e69e6294f0762b",
-                "sha256:2e379b491ee985d31e5faaf050f79f4a8f59f482835906efe4477b33b4fbe9ff",
-                "sha256:36299ffd5663203c25d3a76980f077e23b6d4f574d142f0f43943f57be445639",
-                "sha256:4355a2cb03b257773c0d2bb6af9818c72e836a9b09986e28f52e323d87b1fc67",
-                "sha256:48b35ab009227ee6188991b5666aae1936b82a944f707c042cef267709de12b5",
-                "sha256:48cda82eb82ad2014d2ad194442ed3c46156366be98e4e02f3e29742cdbf94a0",
-                "sha256:4e8c144e2c5814e46868d1f81e2f4265ca1f314a8187d0420cd76e9563294cf8",
-                "sha256:58cdfdd942cf5ffcee974aabfe9b9e26c0c1538fd31c1b03596d40405f7f4d40",
-                "sha256:5b906997e7b98d7d29b84d10a5318993eba1aaff82ff7e1a0ac00254307913d7",
-                "sha256:6603a287188dcbc36358a73a7be43e8a2ecf0c6a06835bdfdd1b113943afdd6f",
-                "sha256:6619264ed43d8d8819d4f1cdb8a62ab66f87e92f06f3ff3e2533fd95a9945e59",
-                "sha256:6a0b92adfcac68dcf549daddec83c87a86995caa6f87bfb6f72de4797e1a6ad6",
-                "sha256:6da74000a672eac0d7cf02adc140b2f9c7d54eae6c196e615a1b5deb694d9203",
-                "sha256:8529cf09919263a6826adc04c5dde9f1406dd7920929b16be19ee9848110e353",
-                "sha256:861979428f749faa9883dc4e11e8c3fc2c29bd0031cf49661604459b53ea7c66",
-                "sha256:8692249732d62e049df3884fa601b70fad3358703e137aceeb581e5860e7d9b7",
-                "sha256:8a9262408f05eef039981479b38b38252d5b853992e5bc54a2d2dd05a2a0178e",
-                "sha256:8ada3659608795bb36930d9a206b8dd6b865d85e2999a02ce8b34f3195d88301",
-                "sha256:9d31d76ed777a8a85be3f8f7f1cfef09b3bc33f6ec4abee1067dcef107f49778",
-                "sha256:b3ddfc6f05410095ec11491dde05f50973e501567d21cbfcf5832d95f141878a",
-                "sha256:b72c426df1586f967b1c61d2f8236702d75c6bbf34abdc258a59e09155a16414",
-                "sha256:b86871862bd65806ba0d0aa2b9c77fcdcc6cbd8d36196688f4896a34bb626334",
-                "sha256:bc42004f181373cd3b39bbacfb71a5b0606ed6e4c199c940cca2212ba0f79525",
-                "sha256:beb367fd88fc6ba8c204786f680229106d99da72a60f5906c85fc8d73640b01a",
-                "sha256:bf671d712816b48fa2731799017ed68e5e440922d0c7e13dc825c226639ff766",
-                "sha256:c0fb9e7cf9db38918f19a803fab9bc7b2ed3f33a9e8319c616c464a0a8501b8d",
-                "sha256:c214f6e88ecdc8ff5d13f0914890445fdaad21bddc34a90cd14aeb3ad5e55e2e",
-                "sha256:c2215f436ce3cce49e6e318cb8f7253cfc4d3bea690777c2a5dd52ae93342504",
-                "sha256:c7a7dd7c50d07718a5ac2bdea166085565f7217cf1e030cc07c22a8b80a406a7",
-                "sha256:c8e0f98d950987b0f9d5e10c41236bef5cb4fba701c6e680af0b9734faa3a85e",
-                "sha256:c9b1322f0d8ce5445fcc3a625b966f10c4182190026713e217d6f38d29930cb1",
-                "sha256:d40d4135f76fb0ed4caa2d422fdb4231616615698709d3c421ecc733f1ac7ca0",
-                "sha256:d5a55749509c7f9f8a33bf9cc02cf76fd6564fcb38f486e43d2858145d735953",
-                "sha256:e5e2859f97e9cceb8e70b0934c56157038b8b083245898593008162a70536d7e",
-                "sha256:e84988d384dfba678387ea7e4f68786c3703543018d473605d9299c69a07f197",
-                "sha256:ef4b144c5b29b4ea0b40c401458b86df8d75382b2e5d03e9f67f607c05b516a9",
-                "sha256:f124ac9ee41e1bfdfb16f53f1db85de296cd2144a4e8fdee8c3560a8fe9b6d5d",
-                "sha256:f234bc46043856d118ebd94b13ea29df674503bc94ced3d81ca46a1ad5b5b9ae",
-                "sha256:f24114e1777604a28ae1c7a56a2c9964655f1031edecc448ad51e5abb19a279b",
-                "sha256:f7fcd93d15deceb2747b10266a39deccd94f257d610f3bbd52a7e16bc5908eda",
-                "sha256:fc96efa617184b8581a02663e261b41c13a605da8ef4ba1ed735bf46184f815e"
+                "sha256:035cb6930a8534f865a3f4616543f78bd27e4de9c3e117b2826e395085ffc4c0",
+                "sha256:039877e57dc10abf0d30d2de2c7476f0881d8ecef1f29bdeed1a6a06a8d89141",
+                "sha256:048fe89c2c753c24e1a7a05496907173dab17e238c8dc3c7cad90b3679b0d846",
+                "sha256:05c4efd36879ff8020af00407e4c14246b894558ea41dc6486f60dd71601fc67",
+                "sha256:063220a6dc0909b576ef068c7e2acf5c608d64423a6d231aacb72d06352cd95b",
+                "sha256:0759868b4a4d103578375e031e89abd578c26774d957ee4f591764ef8003b363",
+                "sha256:077d9a61e6042174dabf68b8e92c0a80f5aff529439ed314aa6e6a233af80b95",
+                "sha256:0a9206b0720f0cad3e70c018722f6d10e81b32e65477e14ffedd3fbfadfaddca",
+                "sha256:0d9fcfc09d67218fce114fe9fd97bba4f9d56add0f775c588d8c626ed47f1aef",
+                "sha256:0f31d02b831d0fa9bf099b1b714b5a8252eabd8db34b7d33c48e7e808a2dabf9",
+                "sha256:115e76fbe9288119526b66963f614042222d1587f1ba5ddb90088215a3d2a25a",
+                "sha256:13491f1bfcf020fd02751c4a55294aa8957e21b7ecd2542b0521a7aa50c58bb2",
+                "sha256:2ad4eb2608661d63fb57c674dafb9955f5141d748d4579c7722c1a3c6b86a0c2",
+                "sha256:2c4d4d92182002b2878adb3329de1ccb7f3f7571d3586f92602e790bfeab45d0",
+                "sha256:2d6bb318ddce8b978c81cf78caf8b3836db84f6235d721952685e87871f506e4",
+                "sha256:34059c3ea6e342ba388cd9774a61761bb4200ca18bd475de65c9cc70ef4e0204",
+                "sha256:3679a6693456f5f7ccca9ab2a91408e99ee257e145024fe380da7c78a07e98b6",
+                "sha256:39318348db488a2f24e7c84e08bdc82f2624853c0fea8b475ea0b70b27176492",
+                "sha256:3cffbba1888f795de2103e6fb1482c8ea8d457e638fa813e090fe747f9e549bb",
+                "sha256:452679284c15a7d5a88bff675e1dd660349360f0665aea50be2d98b7650925f8",
+                "sha256:45277bb54c35b11bcdd6cf0f56eb950eb280b67146db0cb57853fb6334c6d11d",
+                "sha256:45aaceb082ad89365f2f783a40db42359591ad55914fb298841196edf88afdc5",
+                "sha256:485f8a3087392e2287e2869adc0385b439f69b9cfbd262fdf39b00417690c988",
+                "sha256:4db9eea298e982aee7ba12b3432c66eb2e91bb2f5d026bbd57c35698ea0f557f",
+                "sha256:4faf17ea6e8fc3065a862d4b24be84048fd58ed7abe59aa2f9141446a7a72335",
+                "sha256:4fbc8f62b8d50f9a2eef99927a9dcb8d0a42e5a801ab14c2e4aeab622c88f54b",
+                "sha256:530a474a79fa6c2127bb7e3ba00857b1f26a563615863f17b7434331aa3fe404",
+                "sha256:5742ef31e1e2c9a4824ef6b05af0f4949047a6f73af1d4b238ce12935174aede",
+                "sha256:57b44f789794d74c1feddae054dd045b9f601bdffd7288e069b4ca7ed607ec61",
+                "sha256:6ac1cf1f2ed01656b33618352f7e42bf75d027425b83cc96cfe13ce4b6cba5de",
+                "sha256:75206369504fc442c10a86ecf57b91592dca744e4592af22a47e9a774d53dd10",
+                "sha256:77f4d001fb7a03a68b53be20571acd17452d1dda7907d9c325dff0cc704b1ef9",
+                "sha256:808f56d4cd0511723b787c341f3cc995fd72893e608102268298c188f4a4f2e7",
+                "sha256:93502f45948ae8d7f874ba4c113b50cb6fb4ee664caa82e1ddc398500ee0ffb3",
+                "sha256:9c016b3e859b41cf4ce1b8107f364ad5a83c086f75ea4d8d3990b24691f626a1",
+                "sha256:a1cab30c11880f38a27911b569ea38b0bd67fcf32f8a8a8519b613c70562dae2",
+                "sha256:a90f9c7b6635967eacafebe439d518b7dc720aaaf19cb9553f5aad03c13296f4",
+                "sha256:a95ed0e6f481462a3ff2be4c2e4ffffc5d00fc3884d4ccd1fe5b702d4938ec09",
+                "sha256:a993002fe28c840dc29805fde7341c775b7878b311b85f21560fdebf220c247b",
+                "sha256:abb2362783521bd9a22fe69b2141abab4db97237665a36a034b161ddee5b3e72",
+                "sha256:ada4852db0e33dbdd1425322db081d22b9725cb9f5eba42885467b4e2c4f2ac0",
+                "sha256:b37f4b0d983316242b4b9241ecbbe55220aa92af93ff04626441fe0ea90a54f9",
+                "sha256:b77f2b45535bcf3741592fa03183558bd42198b872c1584b896fa0ba5f2ac68d",
+                "sha256:b94f58e05e69e1a43da551c8f532e9fad057df1641f0f8ae8f103d4ede5a80fe",
+                "sha256:ba3f7b433c1721a43674c0889d5fad746bf608416c8f849343859e6d4d3a7113",
+                "sha256:c18e125537a96e76c8c34201e5a9aad8625e3d872dd26a63155573462e54e185",
+                "sha256:c95bd21d87b08c88fe5296381a5f39cd979a775bf1a1d7379a6ff87c703e510b",
+                "sha256:d0d9431101f600d5a557d55989658cbfd02b7c0dcd1e4675dac8ad7e0da8ea5b",
+                "sha256:d9d17a6ceb301c5dbd3820e62c1b10a4ad3a6eea3e07e7afaf736b5f490c2e32",
+                "sha256:db21997270e943aee9cb7694112d24a4702fbe1977fbe53b3cb4db3d02be73d9",
+                "sha256:dd9cab3b862bec2b110886aedb11765e9deda363c4c7ab5ea205f3d8f143c411",
+                "sha256:dee39967168d6326ea2df56ad215a4d5049aa52f44cd5aad45bfb63d5b4fb9e5",
+                "sha256:e3e011fa2ae9e953fe1ab8394329a21bdb54357c7fe509bcfb02b88bc15bffbb",
+                "sha256:ebc901131057c115a8868e14c1df6e56b9190df774b72664c03ebd858296bb81",
+                "sha256:f18c13d5ed6fde5efd3b1c039f6a34296d1a0409bb00fbf45bec6f9bcf63ddf5",
+                "sha256:f687539ead9fbc17f499e33ee20c1dc41598f70ad95edb4990c576447cec9d23",
+                "sha256:f6fcfef825edb44cf3c6ba2c091ad76a83da62ac9c79553e80e0c2a1f75eda2e",
+                "sha256:fcfd2255458a5779dbab813550695331d541b24f0ef831ace83f04f9516ddf26"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.4"
+            "version": "==3.0.5"
         },
         "dask": {
             "hashes": [
@@ -781,10 +781,11 @@
         },
         "executing": {
             "hashes": [
-                "sha256:06df6183df67389625f4e763921c6cf978944721abf3e714000200aab95b0657",
-                "sha256:0ff053696fdeef426cda5bd18eacd94f82c91f49823a2e9090124212ceea9b08"
+                "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147",
+                "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"
             ],
-            "version": "==2.0.0"
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.1"
         },
         "fastjsonschema": {
             "hashes": [
@@ -795,11 +796,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4",
-                "sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd"
+                "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e",
+                "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.12.4"
+            "version": "==3.13.1"
         },
         "fiona": {
             "hashes": [
@@ -834,6 +835,13 @@
             "index": "pypi",
             "version": "==6.0.0"
         },
+        "flake8-pyproject": {
+            "hashes": [
+                "sha256:6249fe53545205af5e76837644dc80b4c10037e73a0e5db87ff562d75fb5bd4a"
+            ],
+            "index": "pypi",
+            "version": "==1.2.3"
+        },
         "flox": {
             "hashes": [
                 "sha256:44b0ce5aea13485126cb69bf4ab1aad0cfe131ddadc577f1f244d48d856fd08d",
@@ -844,51 +852,51 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:10003ebd81fec0192c889e63a9c8c63f88c7d72ae0460b7ba0cd2a1db246e5ad",
-                "sha256:10b3922875ffcba636674f406f9ab9a559564fdbaa253d66222019d569db869c",
-                "sha256:13a9a185259ed144def3682f74fdcf6596f2294e56fe62dfd2be736674500dba",
-                "sha256:17dbc2eeafb38d5d0e865dcce16e313c58265a6d2d20081c435f84dc5a9d8212",
-                "sha256:18a2477c62a728f4d6e88c45ee9ee0229405e7267d7d79ce1f5ce0f3e9f8ab86",
-                "sha256:18eefac1b247049a3a44bcd6e8c8fd8b97f3cad6f728173b5d81dced12d6c477",
-                "sha256:1952c89a45caceedf2ab2506d9a95756e12b235c7182a7a0fff4f5e52227204f",
-                "sha256:1cf9e974f63b1080b1d2686180fc1fbfd3bfcfa3e1128695b5de337eb9075cef",
-                "sha256:1e09da7e8519e336239fbd375156488a4c4945f11c4c5792ee086dd84f784d02",
-                "sha256:2062542a7565091cea4cc14dd99feff473268b5b8afdee564f7067dd9fff5860",
-                "sha256:25d3da8a01442cbc1106490eddb6d31d7dffb38c1edbfabbcc8db371b3386d72",
-                "sha256:34f713dad41aa21c637b4e04fe507c36b986a40f7179dcc86402237e2d39dcd3",
-                "sha256:360201d46165fc0753229afe785900bc9596ee6974833124f4e5e9f98d0f592b",
-                "sha256:3b7ad05b2beeebafb86aa01982e9768d61c2232f16470f9d0d8e385798e37184",
-                "sha256:4c54466f642d2116686268c3e5f35ebb10e49b0d48d41a847f0e171c785f7ac7",
-                "sha256:4d9740e3783c748521e77d3c397dc0662062c88fd93600a3c2087d3d627cd5e5",
-                "sha256:4f88cae635bfe4bbbdc29d479a297bb525a94889184bb69fa9560c2d4834ddb9",
-                "sha256:51669b60ee2a4ad6c7fc17539a43ffffc8ef69fd5dbed186a38a79c0ac1f5db7",
-                "sha256:5db46659cfe4e321158de74c6f71617e65dc92e54980086823a207f1c1c0e24b",
-                "sha256:5f37e31291bf99a63328668bb83b0669f2688f329c4c0d80643acee6e63cd933",
-                "sha256:6bb5ea9076e0e39defa2c325fc086593ae582088e91c0746bee7a5a197be3da0",
-                "sha256:748015d6f28f704e7d95cd3c808b483c5fb87fd3eefe172a9da54746ad56bfb6",
-                "sha256:7bbbf8174501285049e64d174e29f9578495e1b3b16c07c31910d55ad57683d8",
-                "sha256:884ef38a5a2fd47b0c1291647b15f4e88b9de5338ffa24ee52c77d52b4dfd09c",
-                "sha256:8da417431bfc9885a505e86ba706f03f598c85f5a9c54f67d63e84b9948ce590",
-                "sha256:95e974d70238fc2be5f444fa91f6347191d0e914d5d8ae002c9aa189572cc215",
-                "sha256:9648518ef687ba818db3fcc5d9aae27a369253ac09a81ed25c3867e8657a0680",
-                "sha256:9a2f0aa6ca7c9bc1058a9d0b35483d4216e0c1bbe3962bc62ce112749954c7b8",
-                "sha256:9c36da88422e0270fbc7fd959dc9749d31a958506c1d000e16703c2fce43e3d0",
-                "sha256:9c60ecfa62839f7184f741d0509b5c039d391c3aff71dc5bc57b87cc305cff3b",
-                "sha256:9f727c3e3d08fd25352ed76cc3cb61486f8ed3f46109edf39e5a60fc9fecf6ca",
-                "sha256:a7a06f8d95b7496e53af80d974d63516ffb263a468e614978f3899a6df52d4b3",
-                "sha256:ad0b3f6342cfa14be996971ea2b28b125ad681c6277c4cd0fbdb50340220dfb6",
-                "sha256:b2adca1b46d69dce4a37eecc096fe01a65d81a2f5c13b25ad54d5430ae430b13",
-                "sha256:b84a1c00f832feb9d0585ca8432fba104c819e42ff685fcce83537e2e7e91204",
-                "sha256:bb6d2f8ef81ea076877d76acfb6f9534a9c5f31dc94ba70ad001267ac3a8e56f",
-                "sha256:bf11e2cca121df35e295bd34b309046c29476ee739753bc6bc9d5050de319273",
-                "sha256:d21099b411e2006d3c3e1f9aaf339e12037dbf7bf9337faf0e93ec915991f43b",
-                "sha256:d4071bd1c183b8d0b368cc9ed3c07a0f6eb1bdfc4941c4c024c49a35429ac7cd",
-                "sha256:e117a92b07407a061cde48158c03587ab97e74e7d73cb65e6aadb17af191162a",
-                "sha256:f7a58eb5e736d7cf198eee94844b81c9573102ae5989ebcaa1d1a37acd04b33d",
-                "sha256:fe9b1ec799b6086460a7480e0f55c447b1aca0a4eecc53e444f639e967348896"
+                "sha256:05d7c4d2c95b9490e669f3cb83918799bf1c838619ac6d3bad9ea017cfc63f2e",
+                "sha256:0f412954275e594f7a51c16f3b3edd850acb0d842fefc33856b63a17e18499a5",
+                "sha256:22ea8aa7b3712450b42b044702bd3a64fd118006bad09a6f94bd1b227088492e",
+                "sha256:2db63941fee3122e31a21dd0f5b2138ce9906b661a85b63622421d3654a74ae2",
+                "sha256:2e91e19b583961979e2e5a701269d3cfc07418963bee717f8160b0a24332826b",
+                "sha256:31b38528f25bc662401e6ffae14b3eb7f1e820892fd80369a37155e3b636a2f4",
+                "sha256:3d29509f6e05e8d725db59c2d8c076223d793e4e35773040be6632a0349f2f97",
+                "sha256:46c79af80a835410874683b5779b6c1ec1d5a285e11c45b5193e79dd691eb111",
+                "sha256:4e90dd81b6e0d97ebfe52c0d12a17a9ef7f305d6bfbb93081265057d6092f252",
+                "sha256:50d25893885e80a5955186791eed5579f1e75921751539cc1dc3ffd1160b48cf",
+                "sha256:518a945dbfe337744bfff31423c1430303b8813c5275dffb0f2577f0734a1189",
+                "sha256:54efed22b2799a85475e6840e907c402ba49892c614565dc770aa97a53621b2b",
+                "sha256:58af428746fa73a2edcbf26aff33ac4ef3c11c8d75bb200eaea2f7e888d2de4e",
+                "sha256:59b6ad83cce067d10f4790c037a5904424f45bebb5e7be2eb2db90402f288267",
+                "sha256:63a3112f753baef8c6ac2f5f574bb9ac8001b86c8c0c0380039db47a7f512d20",
+                "sha256:66bc6efd829382f7a7e6cf33c2fb32b13edc8a239eb15f32acbf197dce7a0165",
+                "sha256:6999e80a125b0cd8e068d0210b63323f17338038c2ecd2e11b9209ec430fe7f2",
+                "sha256:6d16d9634ff1e5cea2cf4a8cbda9026f766e4b5f30b48f8180f0e99133d3abfc",
+                "sha256:84f308b7a8d28208d54315d11d35f9888d6d607673dd4d42d60b463682ee0400",
+                "sha256:9ee8692e23028564c13d924004495f284df8ac016a19f17a87251210e1f1f928",
+                "sha256:a3da036b016c975c2d8c69005bdc4d5d16266f948a7fab950244e0f58301996a",
+                "sha256:a7aec7f5d14dfcd71fb3ebc299b3f000c21fdc4043079101777ed2042ba5b7c5",
+                "sha256:a8a1fa9a718de0bc026979c93e1e9b55c5efde60d76f91561fd713387573817d",
+                "sha256:a8b99713d3a0d0e876b6aecfaada5e7dc9fe979fcd90ef9fa0ba1d9b9aed03f2",
+                "sha256:b63da598d9cbc52e2381f922da0e94d60c0429f92207bd3fb04d112fc82ea7cb",
+                "sha256:b6e6aa2d066f8dafd06d8d0799b4944b5d5a1f015dd52ac01bdf2895ebe169a0",
+                "sha256:b99fe8ef4093f672d00841569d2d05691e50334d79f4d9c15c1265d76d5580d2",
+                "sha256:b9beb0fa6ff3ea808ad4a6962d68ac0f140ddab080957b20d9e268e4d67fb335",
+                "sha256:b9eab7f9837fdaa2a10a524fbcc2ec24bf60637c044b6e4a59c3f835b90f0fae",
+                "sha256:bca49da868e8bde569ef36f0cc1b6de21d56bf9c3be185c503b629c19a185287",
+                "sha256:c05064f95aacdfc06f21e55096c964b2228d942b8675fa26995a2551f6329d2d",
+                "sha256:c2de1fb18198acd400c45ffe2aef5420c8d55fde903e91cba705596099550f3b",
+                "sha256:c794de4086f06ae609b71ac944ec7deb09f34ecf73316fddc041087dd24bba39",
+                "sha256:d4fa4f4bc8fd86579b8cdbe5e948f35d82c0eda0091c399d009b2a5a6b61c040",
+                "sha256:dab3d00d27b1a79ae4d4a240e8ceea8af0ff049fd45f05adb4f860d93744110d",
+                "sha256:dbac86d83d96099890e731cc2af97976ff2c98f4ba432fccde657c5653a32f1c",
+                "sha256:df40daa6c03b98652ffe8110ae014fe695437f6e1cb5a07e16ea37f40e73ac86",
+                "sha256:e1cd1c6bb097e774d68402499ff66185190baaa2629ae2f18515a2c50b93db0c",
+                "sha256:e8ff7d19a6804bfd561cfcec9b4200dd1788e28f7de4be70189801530c47c1b3",
+                "sha256:eb01c49c8aa035d5346f46630209923d4927ed15c2493db38d31da9f811eb70d",
+                "sha256:f53526668beccdb3409c6055a4ffe50987a7f05af6436fa55d61f5e7bd450219",
+                "sha256:f611c97678604e302b725f71626edea113a5745a7fb557c958b39edb6add87d5"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.43.1"
+            "version": "==4.44.0"
         },
         "fqdn": {
             "hashes": [
@@ -943,16 +951,15 @@
                 "sha256:d9f26aa88f164b77a47cb7d7eafc1641e48abe4152501657cb173c44dc84a8fd"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==0.2.3"
         },
         "identify": {
             "hashes": [
-                "sha256:afe67f26ae29bab007ec21b03d4114f41316ab9dd15aa8736a167481e108da54",
-                "sha256:f302a4256a15c849b91cfcdcec052a8ce914634b2f77ae87dad29cd749f2d88d"
+                "sha256:7736b3c7a28233637e3c36550646fc6389bedd74ae84cb788200cc8e2dd60b75",
+                "sha256:90199cb9e7bd3c5407a9b7e81b4abec4bb9d249991c79439ec8af740afc6293d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.30"
+            "version": "==2.5.31"
         },
         "idna": {
             "hashes": [
@@ -972,86 +979,70 @@
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:9d48dcccc213325e810fd723e7fbb45ccb39f6cf5c31f00cf2b965f5f10f3cb9",
-                "sha256:aa50258bbfa56d4e33fbd8aa3ef48ded10d1735f11532b8df95388cc6bdb7e83"
+                "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a",
+                "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==6.1.0"
+            "version": "==6.1.1"
         },
         "inflate64": {
             "hashes": [
-                "sha256:08c3b03514d4b849901762a32a45eeba7fd5d784fec698eca6975f41cca33672",
-                "sha256:094ef56a87c7b7398d93af7bfe7f24f830f24b6e55b77426f6516cef43e05460",
-                "sha256:09dd0f8d6dee0da467c264dbd9bca8b33f9c915860fc3385f2a633640a65bd10",
-                "sha256:0b0c8aa2fcdb1052d3bc6c5b5b1191b9c708d30e47af98ba0a8117ae1f6c9efc",
-                "sha256:130dfdca4bd38e588ea4f878bf62635e36f83ddf7f2842d1055d1c16a11890cf",
-                "sha256:1749da3a02b53035cde1cf95f885e78e0c2c49b201e97d368b3ba97e0f3d42c3",
-                "sha256:17aaac096f40bd80dd72481831607a0846271d401ba3cd863386b8c244c7ebc1",
-                "sha256:1d861fed6b2098d1862b64db9df650b9bd41fc41caa9fcaeee399079342aa4a8",
-                "sha256:26e8319fd032c520203e2c001f1693c1c03774d85915900427e884011718f41d",
-                "sha256:29946840e6970d68e7739207ca21140c59ffebe7e02d28c7e86348166ce32418",
-                "sha256:35e24ffd8d6225fbfe26c524b45ace1bb8956811bd79e9f3d523a721d51b0d4e",
-                "sha256:3a17e1dd1a5a872edfc02bc4a048868ada4865a3f4ee3ad5d224b192f2e53df7",
-                "sha256:3bacbe9d4b7c185011b59268223a010ed777a28ed8cf40efc74fab1b7262e904",
-                "sha256:3c270d373ca3717dbeb9b171eea53cbf2c9d7471b9b5de1e57f165e60cf58037",
-                "sha256:3c913b679f023f5907a54bfa9a6e438407ed4e40eee23ed19b4118128bdd091c",
-                "sha256:3cf41f82dd4e90e8684c7be4583d7232bd800a561f3ed0241c84e39148861887",
-                "sha256:41504988023042452d2d84e4110c9ef4ff8ebd33cb90ba83e44b92c9a6753c43",
-                "sha256:42a6ef375b3e7059bd52993a0938f2bf97725cb5dc380f0c4dbaa9fc3780e025",
-                "sha256:473e0081c268ffa4b18683586b55170eb96d8b4fc684dd3ed9599c17c512d2e4",
-                "sha256:48fd2527a462374dc19be06301d6aa30a03190532f2f8bddfbc39b7158561750",
-                "sha256:4d807cfa9ddad940401ef04502eb367a77f569850f59c2e71670347d558a3830",
-                "sha256:4e7b0a598adaa11366ffbbb7b3d3110db29edd4b732d9336570891363b22b002",
-                "sha256:525bc309d8533ef9917e006284996ee7a9a71ac6dd19fb57c0f741ad0c805d4f",
-                "sha256:553cd992f02af574d2116c74ca48d7cf10894c6b9ba8159f488f3bfac3c201ae",
-                "sha256:5c5b2eb7e89d550d287774dea7d429ee24ce44ca34499a6cef113a14f108e700",
-                "sha256:5efd55c21b794601fd44b99b8e2f17498744f573116ce27a745bc5e08f0457e1",
-                "sha256:6059eaba5044739ad6424588e845bd856f89a1a18f1addc31b97c49f02f68728",
-                "sha256:664929528047b6b472852a4c0d12b4b9cf6e663059ba64ebd10f08aa56365755",
-                "sha256:67e37d96ea2ee8257b12cde83a09e4f0276950268a7a2f777aee7de60db5ec72",
-                "sha256:67efdfd21d7b99f30a43560b22264c1e580ff08ae9831e78c99445575962dbc7",
-                "sha256:6ff89f94823b2466bae45759fc324bd25bd20c490607a7d8407237cf64ccafa9",
-                "sha256:74ceb9d172ce06572632bc8070d54b963455421e216013575383f991e722bb7d",
-                "sha256:7b7966193f1bf23e050af72b4c4720dffa5f33471de7afea37ba0d0f0195adef",
-                "sha256:7ba954600441eafe8f6f54eadffeac4d1ab2416d5d1a6b0ab403e50284ba457b",
-                "sha256:7c142fbbbfbe0877fe821ff8bc4cc10f96d344b7400721579b3d17deeae28f59",
-                "sha256:7f8346e644de449a4a90dcb22971dea456398b6cc788102013675b11256ae47e",
-                "sha256:80a125dd5cb7b7985c05a78b0bfd7751249d0d84fc330901dbd9faa693e1f53f",
-                "sha256:82393e46b8ba2f8613d030f38c7c492b0896ff8803f7ff870677f25d3e5e7113",
-                "sha256:84287d1d09fd879353d3ccadd43f3d8adea75e830476ddfd46d8849d36d25afe",
-                "sha256:853f3442eceda8035072686533694ab833c4293d10c9d0685147200f0e964356",
-                "sha256:906a4b57df32f903e847766ca685e44ed3e7ee3a960fa94264d5e68b836d446d",
-                "sha256:90f95b92d0f672d11151cb964964d1723e2e3ce3a19d32d24aece1acdec1e287",
-                "sha256:91233b5300bbb7562804c3d07617e9ce2983e8434218991db98ef175491e417f",
-                "sha256:9297115bf144c585e9d6a746e851c64c81d8f1ce8b62da4885babe66c36f7d29",
-                "sha256:9f6737a575c6e7e818963d95a998be4c91484374961734cee97265f3c4c3b979",
-                "sha256:a075b174bace5174828906c7c87019a2af3cc5707025f01ee0395fb4b88fd98e",
-                "sha256:a1b481343f12641b1ae7a19135a70c44ecf020dccb670e02522c2b02db920851",
-                "sha256:a2f4aaa02f9a5ada944960428b6528a0a9d773925efc73485882f34bf42654be",
-                "sha256:a6bec3d2f30f6f2656e1c5a4147181e401c8d7026cd598d86ad5647c616fc618",
-                "sha256:aa7476129e7f81e67a9253470c3085a9fd75ec77e6fae3de61f7795138ce725e",
-                "sha256:ab8f9e14ba6495f440101751ba8aa371e4a52941b5e343c6f3e8c61021e2df5e",
-                "sha256:ac60868745f7bfbcd615329fbdc35997fa36043ce358a1c64d229ef448ebecf0",
-                "sha256:ad4cae5097bdff7e0bb1ab676d86ad08716597baa3b616e5b710a724f5d5cbc4",
-                "sha256:ad84ac611eae17a961124c5fbe754b6982291a3945ab2b9c334a08e2e56f9ccc",
-                "sha256:b52dd8fefd2ba179e5dfa18d6eca7e2fc822584616271c039d5ef1f9ca90c71c",
-                "sha256:b7aa123c740f2f9798f72873e50d7c6d43664d12cad7a1405296079987bdb04a",
-                "sha256:c1faf43890dbfff31195f5d59e37e49824f5ff4be77d67f7144a6b953bbde51c",
-                "sha256:c28cb635ccb9aae399fbc8e82c85b89ea0a7bb2219e7d582bbc007a29fb6e149",
-                "sha256:c71821f93c931ae379cf9c9bbdd7099738fa00802ccf2a5271e2b68bc67a6ab8",
-                "sha256:ced0af509a31dcba0cd98ecdd06cb7c9ce66ebde78e0d99ba3515d4e991e34d0",
-                "sha256:d4e2a337c6c03b0e96ccd79940cbb04fe2063974d56fff6d78f8d57839546c57",
-                "sha256:d71af8b23ac23bc9e9f776451c125be6320ad4589a7d5bcb5ab5e1fc61b4e58f",
-                "sha256:d881b605b7448be451f02c59128dc5fac262dbd0dcff4638e702dc8c7bbb8ef0",
-                "sha256:e1987bbc482aa3e2e7fb72c70b22483cfaed3dbebc5ba6f9ac6f75240794709b",
-                "sha256:e32a78c81afba5699569c3493066ecb38fb45ccdf4c35b3c2232c9c2585b5257",
-                "sha256:f2a4dac4ebc4ad58a4ac911e39cf97cd74906c0c82c16333887aa9f287e98d5b",
-                "sha256:f39b57974db0e85897fff40518da420f4c4012b73515ca6f415a472228fea288",
-                "sha256:fd04764d0bb830414788cae897d082bf6ad92324e571a5511bd7e1de4a0cdc67",
-                "sha256:fde3f85864c84badb26f42d95639360e627fd09c529a76c46a06dbd7a5735c51"
-            ],
-            "markers": "python_version > '3.6'",
-            "version": "==0.3.1"
+                "sha256:022ca1cc928e7365a05f7371ff06af143c6c667144965e2cf9a9236a2ae1c291",
+                "sha256:0c644bf7208e20825ca3bbb5fb1f7f495cfcb49eb01a5f67338796d44a42f2bf",
+                "sha256:0fe481f31695d35a433c3044ac8fd5d9f5069aaad03a0c04b570eb258ce655aa",
+                "sha256:137ca6b315f0157a786c3a755a09395ca69aed8bcf42ad3437cb349f5ebc86d2",
+                "sha256:1616a87ff04f583e9558cc247ec0b72a30d540ee0c17cc77823be175c0ec92f0",
+                "sha256:1facd35319b6a391ee4c3d709c7c650bcada8cd7141d86cd8c2257287f45e6e6",
+                "sha256:228d504239d27958e71fc77e3119a6ac4528127df38468a0c95a5bd3927204b8",
+                "sha256:2be4e01c1b04761874cb44b35b6103ca5846bc36c18fc3ff5e8cbcd8bfc15e9f",
+                "sha256:2cccded63865640d03253897be7232b2bbac295fe43914c61f86a57aa23bb61d",
+                "sha256:3278827b803cf006a1df251f3e13374c7d26db779e5a33329cc11789b804bc2d",
+                "sha256:34de6902c39d9225459583d5034182d371fc694bc3cfd6c0fc89aa62e9809faf",
+                "sha256:35a45f6979ad5874d4d4898c2fc770b136e61b96b850118fdaec5a5af1b9123a",
+                "sha256:36342338e957c790fc630d4afcdcc3926beb2ecaea0b302336079e8fa37e57a0",
+                "sha256:3a70ea2e456c15f7aa7c74b8ab8f20b4f8940ec657604c9f0a9de3342f280fff",
+                "sha256:3e243ea9bd36a035059f2365bd6d156ff59717fbafb0255cb0c75bf151bf6904",
+                "sha256:46792ecf3565d64fd2c519b0a780c03a57e195613c9954ef94e739a057b3fd06",
+                "sha256:4dc392dec1cd11cacda3d2637214ca45e38202e8a4f31d4a4e566d6e90625fc4",
+                "sha256:543f400201f5c101141af3c79c82059e1aa6ef4f1584a7f1fa035fb2e465097f",
+                "sha256:57fe7c14aebf1c5a74fc3b70d355be1280a011521a76aa3895486e62454f4242",
+                "sha256:5ebad4a6cd2a2c1d81be0b09d4006479f3b258803c49a9224ef8ca0b649072fa",
+                "sha256:5f816d1c8a0593375c289e285c96deaee9c2d8742cb0edbd26ee05588a9ae657",
+                "sha256:5ff8bd2a562343fcbc4eea26fdc368904a3b5f6bb8262344274d3d74a1de15bb",
+                "sha256:6823b2c0cff3a8159140f3b17ec64fb8ec0e663b45a6593618ecdde8aeecb5b2",
+                "sha256:6c5775c91f94f5eced9160fb0af12a09f3e030194f91a6a46e706a79350bd056",
+                "sha256:6ceca14f7ec19fb44b047f56c50efb7521b389d222bba2b0a10286a0caeb03fa",
+                "sha256:75448c7b414dadaeeb11dab9f75e022aa1e0ee19b00f570e9f58e933603d71ac",
+                "sha256:8140942d1614bdeb5a9ddd7559348c5c77f884a42424aef7ccf149ccfb93aa08",
+                "sha256:8b402a50eda7ee75f342fc346d33a41bca58edc222a4b17f9be0db1daed459fa",
+                "sha256:8f033b2879696b855200cde5ca4e293132c7499df790acb2c0dacb336d5e83b1",
+                "sha256:92f0dc6af0e8e97324981178dc442956cbff1247a56d1e201af8d865244653f8",
+                "sha256:9373ccf0661cc72ac84a0ad622634144da5ce7d57c9572ed0723d67a149feed2",
+                "sha256:9964a4eaf26a9d36f82a1d9b12c28e35800dd3d99eb340453ed12ac90c2976a8",
+                "sha256:9a270be6b10cde01258c0097a663a307c62d12c78eb8f62f8e29f205335942c9",
+                "sha256:9b65cc701ef33ab20dbfd1d64088ffd89a8c265b356d2c21ba0ec565661645ef",
+                "sha256:9fe3f9051338bb7d07b5e7d88420d666b5109f33ae39aa55ecd1a053c0f22b1b",
+                "sha256:a475e8822f1a74c873e60b8f270773757ade024097ca39e43402d47c049c67d4",
+                "sha256:a90c0bdf4a7ecddd8a64cc977181810036e35807f56b0bcacee9abb0fcfd18dc",
+                "sha256:a982dc93920f9450da4d4f25c5e5c1288ef053b1d618cedc91adb67e035e35f5",
+                "sha256:ae2572e06bcfe15e3bbf77d4e4a6d6c55e2a70d6abceaaf60c5c3653ddb96dfd",
+                "sha256:b559937a42f0c175b4d2dfc7eb53b97bdc87efa9add15ed5549c6abc1e89d02f",
+                "sha256:bf2981b95c1f26242bb084d9a07f3feb0cfe3d6d0a8d90f42389803bc1252c4a",
+                "sha256:c10ca61212a753bbce6d341e7cfa779c161b839281f1f9fdc15cf1f324ce7c5b",
+                "sha256:ca0310b2c55bc40394c5371db2a22f705fd594226cc09432e1eb04d3aed83930",
+                "sha256:d4367480733ac8daf368f6fc704b7c9db85521ee745eb5bd443f4b97d2051acc",
+                "sha256:d491f104fb3701926ebd82b8c9250dfba0ddcab584504e26f1e4adb26730378d",
+                "sha256:d76d205b844d78ce04768060084ef20e64dcc63a3e9166674f857acaf4d140ed",
+                "sha256:d90730165f471d61a1a694a5e354f3ffa938227e8dcecb62d5d728e8069cee94",
+                "sha256:dd6d3e7d47df43210a995fd1f5989602b64de3f2a17cf4cbff553518b3577fd4",
+                "sha256:e4650c6f65011ec57cf5cd96b92d5b7c6f59e502930c86eb8227c93cf02dc270",
+                "sha256:e95044ae55a161144445527a2efad550851fecc699066423d24b2634a6a83710",
+                "sha256:ebafbd813213dc470719cd0a2bcb53aab89d9059f4e75386048b4c4dcdb2fd99",
+                "sha256:f5924499dc8800928c0ee4580fa8eb4ffa880b2cce4431537d0390e503a9c9ee",
+                "sha256:f79542478e49e471e8b23556700e6f688a40dc93e9a746f77a546c13251b59b1"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.0"
         },
         "iniconfig": {
             "hashes": [
@@ -1162,12 +1153,15 @@
             "version": "==2.4"
         },
         "jsonschema": {
+            "extras": [
+                "format-nongpl"
+            ],
             "hashes": [
-                "sha256:cd5f1f9ed9444e554b38ba003af06c0a8c2868131e56bfbef0550fb450c0330e",
-                "sha256:ec84cc37cfa703ef7cd4928db24f9cb31428a5d0fa77747b8b51a847458e0bbf"
+                "sha256:c9ff4d7447eed9592c23a12ccee508baf0dd0d59650615e847feb6cdca74f392",
+                "sha256:eee9e502c788e89cb166d4d37f43084e3b64ab405c795c03d343a4dbc2c810fc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.19.1"
+            "version": "==4.19.2"
         },
         "jsonschema-specifications": {
             "hashes": [
@@ -1204,27 +1198,27 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:66e252f675ac04dcf2feb6ed4afb3cd7f68cf92f483607522dc251f32d471571",
-                "sha256:e4b98344bb94ee2e3e6c4519a97d001656009f9cb2b7f2baf15b3c205770011d"
+                "sha256:880b86053bf298a8724994f95e99b99130659022a4f7f45f563084b6223861d3",
+                "sha256:e11e02cd8ae0a9de5c6c44abf5727df9f2581055afe00b22183f621ba3585805"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.4.0"
+            "version": "==5.5.0"
         },
         "jupyter-events": {
             "hashes": [
-                "sha256:81f07375c7673ff298bfb9302b4a981864ec64edaed75ca0fe6f850b9b045525",
-                "sha256:fda08f0defce5e16930542ce60634ba48e010830d50073c3dfd235759cee77bf"
+                "sha256:81ad2e4bc710881ec274d31c6c50669d71bbaa5dd9d01e600b56faa85700d399",
+                "sha256:d853b3c10273ff9bc8bb8b30076d65e2c9685579db736873de6c2232dde148bf"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.8.0"
+            "version": "==0.9.0"
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:21ad1a3d455d5a79ce4bef5201925cd17510c17898cf9d54e3ccfb6b12734948",
-                "sha256:9ba71be4b9c16e479e4c50c929f8ac4b1015baf90237a08681397a98c76c7e5e"
+                "sha256:47b8f5e63440125cb1bb8957bf12b18453ee5ed9efe42d2f7b2ca66a7019a278",
+                "sha256:dde56c9bc3cb52d7b72cc0f696d15d7163603526f1a758eb4a27405b73eab2a5"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.9.1"
+            "version": "==2.10.0"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1713,19 +1707,19 @@
         },
         "nbclient": {
             "hashes": [
-                "sha256:25e861299e5303a0477568557c4045eccc7a34c17fc08e7959558707b9ebe548",
-                "sha256:f9b179cd4b2d7bca965f900a2ebf0db4a12ebff2f36a711cb66861e4ae158e55"
+                "sha256:4b28c207877cf33ef3a9838cdc7a54c5ceff981194a82eac59d558f05487295e",
+                "sha256:a3a1ddfb34d4a9d17fc744d655962714a866639acd30130e9be84191cd97cd15"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.8.0"
+            "version": "==0.9.0"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:39fe4b8bdd1b0104fdd86fc8a43a9077ba64c720bda4c6132690d917a0a154ee",
-                "sha256:e56cc7588acc4f93e2bb5a34ec69028e4941797b2bfaf6462f18a41d1cc258c9"
+                "sha256:abedc01cf543177ffde0bfc2a69726d5a478f6af10a332fc1bf29fcb4f0cf000",
+                "sha256:d1d417b7f34a4e38887f8da5bdfd12372adf3b80f995d57556cb0972c68909fe"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.9.2"
+            "version": "==7.11.0"
         },
         "nbformat": {
             "hashes": [
@@ -1902,6 +1896,7 @@
                 "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.23.5"
         },
         "numpy-groupies": {
@@ -2122,11 +2117,11 @@
         },
         "prometheus-client": {
             "hashes": [
-                "sha256:21e674f39831ae3f8acde238afd9a27a37d0d2fb5a28ea094f0ce25d2cbf2091",
-                "sha256:e537f37160f6807b8202a6fc4764cdd19bac5480ddd3e0d463c3002b34462101"
+                "sha256:35f7a8c22139e2bb7ca5a698e92d38145bc8dc74c1c0bf56f25cca886a764e17",
+                "sha256:8de3ae2755f890826f4b6479e5571d4f74ac17a81345fe69a6778fdb92579184"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.17.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.18.0"
         },
         "prompt-toolkit": {
             "hashes": [
@@ -2257,118 +2252,93 @@
         },
         "pyarrow": {
             "hashes": [
-                "sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc",
-                "sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c",
-                "sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06",
-                "sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841",
-                "sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2",
-                "sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835",
-                "sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854",
-                "sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2",
-                "sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e",
-                "sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97",
-                "sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d",
-                "sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418",
-                "sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba",
-                "sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70",
-                "sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85",
-                "sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00",
-                "sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56",
-                "sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689",
-                "sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c",
-                "sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4",
-                "sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841",
-                "sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b",
-                "sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca",
-                "sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5",
-                "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"
+                "sha256:0140c7e2b740e08c5a459439d87acd26b747fc408bde0a8806096ee0baaa0c15",
+                "sha256:01e44de9749cddc486169cb632f3c99962318e9dacac7778315a110f4bf8a450",
+                "sha256:05fe7994745b634c5fb16ce5717e39a1ac1fac3e2b0795232841660aa76647cd",
+                "sha256:06ca79080ef89d6529bb8e5074d4b4f6086143b2520494fcb7cf8a99079cde93",
+                "sha256:097828b55321897db0e1dbfc606e3ff8101ae5725673498cbfa7754ee0da80e4",
+                "sha256:0f6f053cb66dc24091f5511e5920e45c83107f954a21032feadc7b9e3a8e7851",
+                "sha256:11e045dfa09855b6d3e7705a37c42e2dc2c71d608fab34d3c23df2e02df9aec3",
+                "sha256:1a8ae88c0038d1bc362a682320112ee6774f006134cd5afc291591ee4bc06505",
+                "sha256:1daab52050a1c48506c029e6fa0944a7b2436334d7e44221c16f6f1b2cc9c510",
+                "sha256:2a145dab9ed7849fc1101bf03bcdc69913547f10513fdf70fc3ab6c0a50c7eee",
+                "sha256:30d8494870d9916bb53b2a4384948491444741cb9a38253c590e21f836b01222",
+                "sha256:323cbe60210173ffd7db78bfd50b80bdd792c4c9daca8843ef3cd70b186649db",
+                "sha256:32542164d905002c42dff896efdac79b3bdd7291b1b74aa292fac8450d0e4dcd",
+                "sha256:33c1f6110c386464fd2e5e4ea3624466055bbe681ff185fd6c9daa98f30a3f9a",
+                "sha256:3c76807540989fe8fcd02285dd15e4f2a3da0b09d27781abec3adc265ddbeba1",
+                "sha256:3f6d5faf4f1b0d5a7f97be987cf9e9f8cd39902611e818fe134588ee99bf0283",
+                "sha256:450e4605e3c20e558485f9161a79280a61c55efe585d51513c014de9ae8d393f",
+                "sha256:470ae0194fbfdfbf4a6b65b4f9e0f6e1fa0ea5b90c1ee6b65b38aecee53508c8",
+                "sha256:4756a2b373a28f6166c42711240643fb8bd6322467e9aacabd26b488fa41ec23",
+                "sha256:58c889851ca33f992ea916b48b8540735055201b177cb0dcf0596a495a667b00",
+                "sha256:6263cffd0c3721c1e348062997babdf0151301f7353010c9c9a8ed47448f82ab",
+                "sha256:78d4a77a46a7de9388b653af1c4ce539350726cd9af62e0831e4f2bd0c95a2f4",
+                "sha256:7a8089d7e77d1455d529dbd7cff08898bbb2666ee48bc4085203af1d826a33cc",
+                "sha256:906b0dc25f2be12e95975722f1e60e162437023f490dbd80d0deb7375baf3171",
+                "sha256:922e8b49b88da8633d6cac0e1b5a690311b6758d6f5d7c2be71acb0f1e14cd61",
+                "sha256:96d64e5ba7dceb519a955e5eeb5c9adcfd63f73a56aea4722e2cc81364fc567a",
+                "sha256:981670b4ce0110d8dcb3246410a4aabf5714db5d8ea63b15686bce1c914b1f83",
+                "sha256:a8eeef015ae69d104c4c3117a6011e7e3ecd1abec79dc87fd2fac6e442f666ee",
+                "sha256:b8b3f4fe8d4ec15e1ef9b599b94683c5216adaed78d5cb4c606180546d1e2ee1",
+                "sha256:be28e1a07f20391bb0b15ea03dcac3aade29fc773c5eb4bee2838e9b2cdde0cb",
+                "sha256:c7331b4ed3401b7ee56f22c980608cf273f0380f77d0f73dd3c185f78f5a6220",
+                "sha256:cf87e2cec65dd5cf1aa4aba918d523ef56ef95597b545bbaad01e6433851aa10",
+                "sha256:d0351fecf0e26e152542bc164c22ea2a8e8c682726fce160ce4d459ea802d69c",
+                "sha256:d264ad13605b61959f2ae7c1d25b1a5b8505b112715c961418c8396433f213ad",
+                "sha256:e592e482edd9f1ab32f18cd6a716c45b2c0f2403dc2af782f4e9674952e6dd27",
+                "sha256:fada8396bc739d958d0b81d291cfd201126ed5e7913cb73de6bc606befc30226"
             ],
             "index": "pypi",
-            "version": "==11.0.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==14.0.1"
         },
         "pybcj": {
             "hashes": [
-                "sha256:023082fd677f67ebd36fe96322a4a45ac33a2b340d49010d88e1867c76744c50",
-                "sha256:05fad9a905772774aacc96cb174571ac1f5afa80b9f54c6ec414d369865d305c",
-                "sha256:09872b32edad4e3653d5b357b244d267ca58fe52d4e1dd3cdff816d3bb9d9f7c",
-                "sha256:0eaa90639992b6096afb1485380fae7f084483db6b92867847a3bfdf22cc4efc",
-                "sha256:10182725b0e6aa944d13a10a4a9cb5208bafe0016b4326253340948153de4bc0",
-                "sha256:10961ea10ae930b9348132707b9dd3cf3e71a41ef1df7656fbc4f14a71f10747",
-                "sha256:15edd1786617127ecfda4274bbb04f09ae299c474ada86e369bcf050d5cb88dd",
-                "sha256:1a5365edcaa82dc47e7757ba2efb48f96b9b352e3811a2aaa90084802479ddbe",
-                "sha256:1c0e1657c233f9f4070ab578951e03d569f1b645042ce661341091f50e41b541",
-                "sha256:1e6434a46f852cd3e6929633b43537887bd381bc614dbf5c4a128fdde4966b3a",
-                "sha256:20fc0d8f67e2d9747e0c31082d5f64b112258ae602a85aa5c7e6bf5a7cad287b",
-                "sha256:21098001200273c3c9fd90e7bf909fb905a8e1c102c80b604cb7c6a3103ef7e0",
-                "sha256:225a0addf4b3d580bf4eae583b5168dac0125a703c53ded8b3f120882e1e0312",
-                "sha256:262f53e27bca6096e3424c63e5e59948b10985eee4b03a5d70c3f3f6161a79e7",
-                "sha256:2d6b34ec233fcf5a83ccfbf422fef22256947eaa7077aaa012e5961d15aa302c",
-                "sha256:2e1859d36c073231737956fbeb5bbcfa8dba880e1b66bfbd001466718d6d89dc",
-                "sha256:358dba3dc39a07cded6897b9f99bb5b951a0ad95d567eda535b44861caa02f5b",
-                "sha256:3847387b43af47d9677952b8a22d9c2d8a544c2175b6d5304c200669c05d39e1",
-                "sha256:393d95f83e47976d137bcec7b66986f51282dcb2091933f88983dd7eb89e59c4",
-                "sha256:39dd836134e261ec769cd5aa9ae7a3a330a7dac81efb66eb5504643abd8235df",
-                "sha256:421ed75e54ebecd79c80178c1df5bdbe1e0e3e10e7efef5f011b5f0be6a9a12f",
-                "sha256:421f211fb15aeb836b4ba61174cb409fc82222ab3b2486deb4953ae863e6507b",
-                "sha256:43e8bc75773ca06ee7a64602b799613171e4edf4d9d8fd38fa5c49f1cdbb4407",
-                "sha256:46b82fe50eb8171ee2205e935f3fd5900e31beb5e54e10c88f23a5420902467d",
-                "sha256:4bc8720f3a224c27bd413a930b9bec5f225fda050641258967b1ebb252a053fb",
-                "sha256:4d10dd75fad48555e9530c5565c7ccf13754adad2fe331feefb263055cdca7b3",
-                "sha256:534b8b253dbdb746c06bab28383db31d7e2b42aa9b33ed4e7836319622dcd75b",
-                "sha256:570a3cf4e016dcb0fc561991833e5170a2a0bc6ee88fe5667591f356bd7b7895",
-                "sha256:5de90f8b6c7fc1d28dbe74c29b1d5053a7a8703cbc2c6f4f112907ffd7529f8e",
-                "sha256:603daa737579cf69efb368fab716cdce18d0b2615af77bb623f5f42aa546b3d8",
-                "sha256:6485c6b091504c0e6431a9495309271626eaa9ecb23276903486824f94f4c551",
-                "sha256:6ca6ddae1302477879099d4c4efc65790f4610d71ceff7fbe8f8b60f6ac6dcff",
-                "sha256:6df9eccc99a0d7bc091b58cff2f507b89f076d657253975fa2ca9eb42dbb4733",
-                "sha256:6f589af70286ec6565e3415145a03abc3c14a23ed7ed198ac741de81af332f26",
-                "sha256:731800acfc6112132aa2b7d08f9d6fe49a0c0071b30985809d084e238af98dac",
-                "sha256:74d34340323996b70dbd73e9530cca71c05ff7c97e30fe4d32aeea2f877836ca",
-                "sha256:7801ee9a9fcd47b92d4d90ff9a28cfdc23195cad72bd8032938ab3c794942b43",
-                "sha256:795dff9229dc024e54bd0f618f5a3adb269ee0cccd7ac9a0bef29df388beed23",
-                "sha256:7b3773a77ae3b18778c9bf22c7ba6478a0e5416f84b7d2ac6d764001f6d0d985",
-                "sha256:872697d8bff2572e4225ed8cbce17be338faac28ec1ab3c00419aaef2f56dd3c",
-                "sha256:8b682ed08caabfb7c042d4be083e28ddc692afb1deff5567111f8855071b75c3",
-                "sha256:8e846a8272bf02202794fe22beaf389ed27c2d8ebf59aafb43af4935feac0389",
-                "sha256:8efed581f2ee74f1e0ec04a10e97881b93abc258d13b15ef966aee71732ac152",
-                "sha256:970dc23ca1c64611d35a3abe76a059cf551da53d62faefd84c5bf3e0af1602d1",
-                "sha256:9b56eeff51efa556ecc186260ac486a4ddd79ad37bc88d669e96c45190f3c0da",
-                "sha256:9fc313b1a5547c5416982853f2de1454980704f3ab3dbcad18dacdc565a2eafc",
-                "sha256:a74e70bf3fd50a413fdce4264e037b8e8f34cb8d9207ac364167b6eb076c14ec",
-                "sha256:a77796b4c5370cedd4fad2264b6d7a78cb40229c7fa3cbcab24df3adea768962",
-                "sha256:a81f14f213a75597f9be44feb97740a51adda558465fb159114472dc2ab39ef8",
-                "sha256:acfc4a02ddf22f6df7184441b39f38c31e95aa8af41de4d2f825821ab1fb85c6",
-                "sha256:b3861996b06b8238f799b4f1bd9542d1a8ae8e4765adbdde25ed011c3bda11df",
-                "sha256:b901f12380e988da07f21bb6b75da7f91fd9feffb43fcf70fad698e40a2ef3a7",
-                "sha256:b99f4291e59dcbe548be5a1e8c6a1a19a860184526c2d14fc374ec687b98ad7d",
-                "sha256:bb378b0f133e19d437eca4327bac7c3f38e30950c5c604092c72b18cba839bc2",
-                "sha256:bbb49772fc3896850a704215160df8316db89e5e8876b2d8af6c6c15b4e0f6ea",
-                "sha256:bc1684b9f7ec92d2ae94a137ec311bd2227f684429521061af7ceed4952c7f72",
-                "sha256:bdabbe7fd66886943393ecf98318d7801dd40183af80314acd4464bccdd44d53",
-                "sha256:bf87f2a7f827656bc6e1d9888d47931aa0ae35cdc4ff33b1cec70d8d462590b3",
-                "sha256:c1e02170f8d358a8ddc716606760c73d55eea6bdb0cca2d97b86447e9524708b",
-                "sha256:c72ff262613c9a6f20e80bcf1e8bbc000b78b95a7fa301164ab3e3bd23bd936c",
-                "sha256:c854a206d8c3a5a959b803405760f3627bb4878450e2f36b5d35af09c89152fc",
-                "sha256:d5b327df02761c42399c878cd6c37f885bf0639befbd4d1ab763cd44ba1e0552",
-                "sha256:d5c4ca6faff0af4b5f3e7d88d13ec76f8cac36c9bcc814b8c84d9f3f951b2cf9",
-                "sha256:d61f287f820787d3acf60d113c5ce6e506870d9d3103bc37a74373e72ce9d7a6",
-                "sha256:dc23f2ac2c1ded250f1aa66fbd1a3d823f76de549978b61eed4fb34affc11338",
-                "sha256:dc79ed4773cd35328377a8fedbbdcafb3a9d242ee63b96863c0692c81faefab8",
-                "sha256:df75707f466ab6fa086f164bff2df75fd16543c8d43ca43a268f938c1144e792",
-                "sha256:e6a74cb618da93ac1322d6a548a4508e76eb4c388ed1c80560bc25d8764cf272",
-                "sha256:e89a814f1727be7d543ac6910f0d94131f43a337e811ab684606d42dbc22b701",
-                "sha256:efe75e3b8768c4f9d454d3c1b2b2a67e757f2b00d638146d3a4cddb38460fc3a",
-                "sha256:f2f950ca403ffaa808a017e40e3371115bcb0b4b1061772b03e7d842555132ac",
-                "sha256:f4428b6808d781f4b605a27f53fc10a3ca343d1cd901c691b9ba2e4ed85a5fc7",
-                "sha256:f46ba61c942ee64198444c9562c5cf089eaf97f17b413e15fa1c0614df304734",
-                "sha256:f472da992a6ba58381c0314b994c01d20e522ff8836417ef1c0975bdae142406",
-                "sha256:f58e489e43c9a1688c7d5ceb7455b44952d87f183b7b9c915b301478a2b3bfbe",
-                "sha256:f8576a1dcf445ef064bf8c3b2cdc1d6353e41cb4b366329946883e285dcbcec0",
-                "sha256:fa787b414c4dc6b6cd75338fac18a7dbb53a09443dd863020a2d2bda76940ca6",
-                "sha256:fda423836d7d69cba6a6f99e7a34c2e5fe3621e5e945cd25ea9ba60a96223254"
+                "sha256:1079ca63ff8da5c936b76863690e0bd2489e8d4e0a3a340e032095dae805dd91",
+                "sha256:198e0b4768b4025eb3309273d7e81dc53834b9a50092be6e0d9b3983cfd35c35",
+                "sha256:21b5f2460629167340403d359289a173e0729ce8e84e3ce99462009d5d5e01a4",
+                "sha256:22a94885723f8362d4cb468e68910eef92d3e2b1293de82b8eacb4198ef6655f",
+                "sha256:2940fb85730b9869254559c491cd83cf777e56c76a8a60df60e4be4f2a4248d7",
+                "sha256:2ed5b3dd9c209fe7b90990dee4ef21870dca39db1cd326553c314ee1b321c1cc",
+                "sha256:3602be737c6e9553c45ae89e6b0e556f64f34dabf27d5260317d1824d31b79d3",
+                "sha256:3b8d7810fb587adbffba025330cf212d9bbed8f29559656d05cb6609673f306a",
+                "sha256:3ffae79ef8a1ea81ea2748ad7b7ad9b882aa88ddf65ce90f9e944df639eccc61",
+                "sha256:493eab2b1f6f546730a6de0c5ceb75ce16f3767154e8ae30e2b70d41b928b7d2",
+                "sha256:5c3171bb95c9b45cbcad25589e1ae4f4ca4ea99dc1724c4e0671eb6b9055514e",
+                "sha256:63dd2ca52a48841f561bfec0fa3f208d375b0a8dcd3d7b236459e683ae29221d",
+                "sha256:746550dc7b5af4d04bb5fa4d065f18d39c925bcb5dee30db75747cd9a58bb6e8",
+                "sha256:75d6d613bae6f27678d5e44e89d61018779726aa6aa950c516d33a04b8af8c59",
+                "sha256:7bff28d97e47047d69a4ac6bf59adda738cf1d00adde8819117fdb65d966bdbc",
+                "sha256:8007371f6f2b462f5aa05d5c2135d0a1bcf5b7bdd9bd15d86c730f588d10b7d3",
+                "sha256:8204a714029784b1a08a3d790430d80b423b68615c5b1e67aabca5bd5419b77d",
+                "sha256:87108181c7a6ac4d3fc1e4551cab5db5eea7f9fdca611175243234cd94bcc59b",
+                "sha256:8ce9b62b6aaa5b08773be8a919ecc4e865396c969f982b685eeca6e80c82abb7",
+                "sha256:9c2b3e60b65c7ac73e44335934e1e122da8d56db87840984601b3c5dc0ae4c19",
+                "sha256:9ea46e2d45469d13b7f25b08efcdb140220bab1ac5a850db0954591715b8caaa",
+                "sha256:a2562ebe5a0abec4da0229f8abb5e90ee97b178f19762eb925c1159be36828b3",
+                "sha256:a29be917fbc99eca204b08407e0971e0205bfdad4b74ec915930675f352b669d",
+                "sha256:a54ebdc8423ba99d75372708a882fcfc3b14d9d52cf195295ad53e5a47dab37f",
+                "sha256:af19bc61ded933001cd68f004ae2042bf1a78eb498a3c685ebd655fa1be90dbe",
+                "sha256:b8f9368036c9e658d8e3b3534086d298a5349c864542b34657cbe57c260daa49",
+                "sha256:bdb4d8ff5cba3e0bd1adee7d20dbb2b4d80cb31ac04d6ea1cd06cfc02d2ecd0d",
+                "sha256:bdf5bcac4f1da36ad43567ea6f6ef404347658dbbe417c87cdb1699f327d6337",
+                "sha256:c7f5bef7f47723c53420e377bc64d2553843bee8bcac5f0ad076ab1524780018",
+                "sha256:ce1c8af7a4761d2b1b531864d84113948daa0c4245775c63bd9874cb955f4662",
+                "sha256:db57f26b8c0162cfddb52b869efb1741b8c5e67fc536994f743074985f714c55",
+                "sha256:e96ae14062bdcddc3197300e6ee4efa6fbc6749be917db934eac66d0daaecb68",
+                "sha256:e9a785eb26884429d9b9f6326e68c3638828c83bf6d42d2463c97ad5385caff2",
+                "sha256:ef55b96b7f2ed823e0b924de902065ec42ade856366c287dbb073fabd6b90ec1",
+                "sha256:f3f4a447800850aba7724a2274ea0a4800724520c1caf38f7d0dabf2f89a5e15",
+                "sha256:f40f3243139d675f43793a4e35c410c370f7b91ccae74e70c8b2f4877869f90e",
+                "sha256:f9a2585e0da9cf343ea27421995b881736a1eb604a7c1d4ca74126af94c3d4a8",
+                "sha256:fa26415b4a118ea790de9d38f244312f2510a9bb5c65e560184d241a6f391a2d",
+                "sha256:fabb2be57e4ca28ea36c13146cdf97d73abd27c51741923fc6ba1e8cd33e255c",
+                "sha256:fdb7cd8271471a5979d84915c1ee57eea7e0a69c893225fc418db66883b0e2a7",
+                "sha256:fde2376b180ae2620c102fbc3ef06638d306feae83964aaa5051ecbdda54845a"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.0.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.2"
         },
         "pycodestyle": {
             "hashes": [
@@ -2622,14 +2592,6 @@
             "index": "pypi",
             "version": "==3.5.0"
         },
-        "pyproject-flake8": {
-            "hashes": [
-                "sha256:bdc7ca9b967b9724983903489b8943b72c668178fb69f03e8774ec74f6a13782",
-                "sha256:d43421caca0ef8a672874405fe63c722b0333e3c22c41648c6df60f21bab2b6b"
-            ],
-            "index": "pypi",
-            "version": "==6.0.0.post1"
-        },
         "pytest": {
             "hashes": [
                 "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d",
@@ -2643,7 +2605,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.2"
         },
         "python-dotenv": {
@@ -2937,11 +2899,11 @@
         },
         "qtconsole": {
             "hashes": [
-                "sha256:a3b69b868e041c2c698bdc75b0602f42e130ffb256d6efa48f9aa756c97672aa",
-                "sha256:b7ffb53d74f23cee29f4cdb55dd6fabc8ec312d94f3c46ba38e1dde458693dfb"
+                "sha256:6b6bcf8f834c6df1579a3e6623c8531b85d3e723997cee3a1156296df14716c8",
+                "sha256:ea8b4a07d7dc915a1b1238fbfe2c9aea570640402557b64615e09a4bc60df47c"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==5.4.4"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.5.0"
         },
         "qtpy": {
             "hashes": [
@@ -3051,108 +3013,108 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:023574366002bf1bd751ebaf3e580aef4a468b3d3c216d2f3f7e16fdabd885ed",
-                "sha256:031f76fc87644a234883b51145e43985aa2d0c19b063e91d44379cd2786144f8",
-                "sha256:052a832078943d2b2627aea0d19381f607fe331cc0eb5df01991268253af8417",
-                "sha256:0699ab6b8c98df998c3eacf51a3b25864ca93dab157abe358af46dc95ecd9801",
-                "sha256:0713631d6e2d6c316c2f7b9320a34f44abb644fc487b77161d1724d883662e31",
-                "sha256:0774a46b38e70fdde0c6ded8d6d73115a7c39d7839a164cc833f170bbf539116",
-                "sha256:0898173249141ee99ffcd45e3829abe7bcee47d941af7434ccbf97717df020e5",
-                "sha256:09586f51a215d17efdb3a5f090d7cbf1633b7f3708f60a044757a5d48a83b393",
-                "sha256:102eac53bb0bf0f9a275b438e6cf6904904908562a1463a6fc3323cf47d7a532",
-                "sha256:10f32b53f424fc75ff7b713b2edb286fdbfc94bf16317890260a81c2c00385dc",
-                "sha256:150eec465dbc9cbca943c8e557a21afdcf9bab8aaabf386c44b794c2f94143d2",
-                "sha256:1d7360573f1e046cb3b0dceeb8864025aa78d98be4bb69f067ec1c40a9e2d9df",
-                "sha256:1f36a9d751f86455dc5278517e8b65580eeee37d61606183897f122c9e51cef3",
-                "sha256:24656dc36f866c33856baa3ab309da0b6a60f37d25d14be916bd3e79d9f3afcf",
-                "sha256:25860ed5c4e7f5e10c496ea78af46ae8d8468e0be745bd233bab9ca99bfd2647",
-                "sha256:26857f0f44f0e791f4a266595a7a09d21f6b589580ee0585f330aaccccb836e3",
-                "sha256:2bb2e4826be25e72013916eecd3d30f66fd076110de09f0e750163b416500721",
-                "sha256:2f6da6d842195fddc1cd34c3da8a40f6e99e4a113918faa5e60bf132f917c247",
-                "sha256:30adb75ecd7c2a52f5e76af50644b3e0b5ba036321c390b8e7ec1bb2a16dd43c",
-                "sha256:3339eca941568ed52d9ad0f1b8eb9fe0958fa245381747cecf2e9a78a5539c42",
-                "sha256:34ad87a831940521d462ac11f1774edf867c34172010f5390b2f06b85dcc6014",
-                "sha256:3777cc9dea0e6c464e4b24760664bd8831738cc582c1d8aacf1c3f546bef3f65",
-                "sha256:3953c6926a63f8ea5514644b7afb42659b505ece4183fdaaa8f61d978754349e",
-                "sha256:3c4eff26eddac49d52697a98ea01b0246e44ca82ab09354e94aae8823e8bda02",
-                "sha256:40578a6469e5d1df71b006936ce95804edb5df47b520c69cf5af264d462f2cbb",
-                "sha256:40f93086eef235623aa14dbddef1b9fb4b22b99454cb39a8d2e04c994fb9868c",
-                "sha256:4134aa2342f9b2ab6c33d5c172e40f9ef802c61bb9ca30d21782f6e035ed0043",
-                "sha256:442626328600bde1d09dc3bb00434f5374948838ce75c41a52152615689f9403",
-                "sha256:4a5ee600477b918ab345209eddafde9f91c0acd931f3776369585a1c55b04c57",
-                "sha256:4ce5a708d65a8dbf3748d2474b580d606b1b9f91b5c6ab2a316e0b0cf7a4ba50",
-                "sha256:516a611a2de12fbea70c78271e558f725c660ce38e0006f75139ba337d56b1f6",
-                "sha256:52c215eb46307c25f9fd2771cac8135d14b11a92ae48d17968eda5aa9aaf5071",
-                "sha256:53c43e10d398e365da2d4cc0bcaf0854b79b4c50ee9689652cdc72948e86f487",
-                "sha256:5752b761902cd15073a527b51de76bbae63d938dc7c5c4ad1e7d8df10e765138",
-                "sha256:5e8a78bd4879bff82daef48c14d5d4057f6856149094848c3ed0ecaf49f5aec2",
-                "sha256:5ed505ec6305abd2c2c9586a7b04fbd4baf42d4d684a9c12ec6110deefe2a063",
-                "sha256:5ee97c683eaface61d38ec9a489e353d36444cdebb128a27fe486a291647aff6",
-                "sha256:61fa268da6e2e1cd350739bb61011121fa550aa2545762e3dc02ea177ee4de35",
-                "sha256:64ccc28683666672d7c166ed465c09cee36e306c156e787acef3c0c62f90da5a",
-                "sha256:66414dafe4326bca200e165c2e789976cab2587ec71beb80f59f4796b786a238",
-                "sha256:68fe9199184c18d997d2e4293b34327c0009a78599ce703e15cd9a0f47349bba",
-                "sha256:6a555ae3d2e61118a9d3e549737bb4a56ff0cec88a22bd1dfcad5b4e04759175",
-                "sha256:6bdc11f9623870d75692cc33c59804b5a18d7b8a4b79ef0b00b773a27397d1f6",
-                "sha256:6cf4393c7b41abbf07c88eb83e8af5013606b1cdb7f6bc96b1b3536b53a574b8",
-                "sha256:6eef672de005736a6efd565577101277db6057f65640a813de6c2707dc69f396",
-                "sha256:734c41f9f57cc28658d98270d3436dba65bed0cfc730d115b290e970150c540d",
-                "sha256:73e0a78a9b843b8c2128028864901f55190401ba38aae685350cf69b98d9f7c9",
-                "sha256:775049dfa63fb58293990fc59473e659fcafd953bba1d00fc5f0631a8fd61977",
-                "sha256:7854a207ef77319ec457c1eb79c361b48807d252d94348305db4f4b62f40f7f3",
-                "sha256:78ca33811e1d95cac8c2e49cb86c0fb71f4d8409d8cbea0cb495b6dbddb30a55",
-                "sha256:79edd779cfc46b2e15b0830eecd8b4b93f1a96649bcb502453df471a54ce7977",
-                "sha256:7bf347b495b197992efc81a7408e9a83b931b2f056728529956a4d0858608b80",
-                "sha256:7fde6d0e00b2fd0dbbb40c0eeec463ef147819f23725eda58105ba9ca48744f4",
-                "sha256:81de24a1c51cfb32e1fbf018ab0bdbc79c04c035986526f76c33e3f9e0f3356c",
-                "sha256:879fb24304ead6b62dbe5034e7b644b71def53c70e19363f3c3be2705c17a3b4",
-                "sha256:8e7f2219cb72474571974d29a191714d822e58be1eb171f229732bc6fdedf0ac",
-                "sha256:9164ec8010327ab9af931d7ccd12ab8d8b5dc2f4c6a16cbdd9d087861eaaefa1",
-                "sha256:945eb4b6bb8144909b203a88a35e0a03d22b57aefb06c9b26c6e16d72e5eb0f0",
-                "sha256:99a57006b4ec39dbfb3ed67e5b27192792ffb0553206a107e4aadb39c5004cd5",
-                "sha256:9e9184fa6c52a74a5521e3e87badbf9692549c0fcced47443585876fcc47e469",
-                "sha256:9ff93d3aedef11f9c4540cf347f8bb135dd9323a2fc705633d83210d464c579d",
-                "sha256:a360cfd0881d36c6dc271992ce1eda65dba5e9368575663de993eeb4523d895f",
-                "sha256:a5d7ed104d158c0042a6a73799cf0eb576dfd5fc1ace9c47996e52320c37cb7c",
-                "sha256:ac17044876e64a8ea20ab132080ddc73b895b4abe9976e263b0e30ee5be7b9c2",
-                "sha256:ad857f42831e5b8d41a32437f88d86ead6c191455a3499c4b6d15e007936d4cf",
-                "sha256:b2039f8d545f20c4e52713eea51a275e62153ee96c8035a32b2abb772b6fc9e5",
-                "sha256:b455492cab07107bfe8711e20cd920cc96003e0da3c1f91297235b1603d2aca7",
-                "sha256:b4a9fe992887ac68256c930a2011255bae0bf5ec837475bc6f7edd7c8dfa254e",
-                "sha256:b5a53f5998b4bbff1cb2e967e66ab2addc67326a274567697379dd1e326bded7",
-                "sha256:b788276a3c114e9f51e257f2a6f544c32c02dab4aa7a5816b96444e3f9ffc336",
-                "sha256:bddd4f91eede9ca5275e70479ed3656e76c8cdaaa1b354e544cbcf94c6fc8ac4",
-                "sha256:c0503c5b681566e8b722fe8c4c47cce5c7a51f6935d5c7012c4aefe952a35eed",
-                "sha256:c1b3cd23d905589cb205710b3988fc8f46d4a198cf12862887b09d7aaa6bf9b9",
-                "sha256:c48f3fbc3e92c7dd6681a258d22f23adc2eb183c8cb1557d2fcc5a024e80b094",
-                "sha256:c63c3ef43f0b3fb00571cff6c3967cc261c0ebd14a0a134a12e83bdb8f49f21f",
-                "sha256:c6c45a2d2b68c51fe3d9352733fe048291e483376c94f7723458cfd7b473136b",
-                "sha256:caa1afc70a02645809c744eefb7d6ee8fef7e2fad170ffdeacca267fd2674f13",
-                "sha256:cc435d059f926fdc5b05822b1be4ff2a3a040f3ae0a7bbbe672babb468944722",
-                "sha256:cf693eb4a08eccc1a1b636e4392322582db2a47470d52e824b25eca7a3977b53",
-                "sha256:cf71343646756a072b85f228d35b1d7407da1669a3de3cf47f8bbafe0c8183a4",
-                "sha256:d08f63561c8a695afec4975fae445245386d645e3e446e6f260e81663bfd2e38",
-                "sha256:d29ddefeab1791e3c751e0189d5f4b3dbc0bbe033b06e9c333dca1f99e1d523e",
-                "sha256:d7f5e15c953ace2e8dde9824bdab4bec50adb91a5663df08d7d994240ae6fa31",
-                "sha256:d858532212f0650be12b6042ff4378dc2efbb7792a286bee4489eaa7ba010586",
-                "sha256:d97dd44683802000277bbf142fd9f6b271746b4846d0acaf0cefa6b2eaf2a7ad",
-                "sha256:dcdc88b6b01015da066da3fb76545e8bb9a6880a5ebf89e0f0b2e3ca557b3ab7",
-                "sha256:dd609fafdcdde6e67a139898196698af37438b035b25ad63704fd9097d9a3482",
-                "sha256:defa2c0c68734f4a82028c26bcc85e6b92cced99866af118cd6a89b734ad8e0d",
-                "sha256:e22260a4741a0e7a206e175232867b48a16e0401ef5bce3c67ca5b9705879066",
-                "sha256:e225a6a14ecf44499aadea165299092ab0cba918bb9ccd9304eab1138844490b",
-                "sha256:e3df0bc35e746cce42579826b89579d13fd27c3d5319a6afca9893a9b784ff1b",
-                "sha256:e6fcc026a3f27c1282c7ed24b7fcac82cdd70a0e84cc848c0841a3ab1e3dea2d",
-                "sha256:e782379c2028a3611285a795b89b99a52722946d19fc06f002f8b53e3ea26ea9",
-                "sha256:e8cdd52744f680346ff8c1ecdad5f4d11117e1724d4f4e1874f3a67598821069",
-                "sha256:e9616f5bd2595f7f4a04b67039d890348ab826e943a9bfdbe4938d0eba606971",
-                "sha256:e98c4c07ee4c4b3acf787e91b27688409d918212dfd34c872201273fdd5a0e18",
-                "sha256:ebdab79f42c5961682654b851f3f0fc68e6cc7cd8727c2ac4ffff955154123c1",
-                "sha256:f0f17f2ce0f3529177a5fff5525204fad7b43dd437d017dd0317f2746773443d",
-                "sha256:f4e56860a5af16a0fcfa070a0a20c42fbb2012eed1eb5ceeddcc7f8079214281"
+                "sha256:0525847f83f506aa1e28eb2057b696fe38217e12931c8b1b02198cfe6975e142",
+                "sha256:05942656cb2cb4989cd50ced52df16be94d344eae5097e8583966a1d27da73a5",
+                "sha256:0831d3ecdea22e4559cc1793f22e77067c9d8c451d55ae6a75bf1d116a8e7f42",
+                "sha256:0853da3d5e9bc6a07b2486054a410b7b03f34046c123c6561b535bb48cc509e1",
+                "sha256:08e6e7ff286254016b945e1ab632ee843e43d45e40683b66dd12b73791366dd1",
+                "sha256:0a38612d07a36138507d69646c470aedbfe2b75b43a4643f7bd8e51e52779624",
+                "sha256:0bedd91ae1dd142a4dc15970ed2c729ff6c73f33a40fa84ed0cdbf55de87c777",
+                "sha256:0c5441b7626c29dbd54a3f6f3713ec8e956b009f419ffdaaa3c80eaf98ddb523",
+                "sha256:0e9e976e0dbed4f51c56db10831c9623d0fd67aac02853fe5476262e5a22acb7",
+                "sha256:0fadfdda275c838cba5102c7f90a20f2abd7727bf8f4a2b654a5b617529c5c18",
+                "sha256:1096ca0bf2d3426cbe79d4ccc91dc5aaa73629b08ea2d8467375fad8447ce11a",
+                "sha256:171d9a159f1b2f42a42a64a985e4ba46fc7268c78299272ceba970743a67ee50",
+                "sha256:188912b22b6c8225f4c4ffa020a2baa6ad8fabb3c141a12dbe6edbb34e7f1425",
+                "sha256:1b4cf9ab9a0ae0cb122685209806d3f1dcb63b9fccdf1424fb42a129dc8c2faa",
+                "sha256:1e04581c6117ad9479b6cfae313e212fe0dfa226ac727755f0d539cd54792963",
+                "sha256:1fa73ed22c40a1bec98d7c93b5659cd35abcfa5a0a95ce876b91adbda170537c",
+                "sha256:2124f9e645a94ab7c853bc0a3644e0ca8ffbe5bb2d72db49aef8f9ec1c285733",
+                "sha256:240687b5be0f91fbde4936a329c9b7589d9259742766f74de575e1b2046575e4",
+                "sha256:25740fb56e8bd37692ed380e15ec734be44d7c71974d8993f452b4527814601e",
+                "sha256:27ccc93c7457ef890b0dd31564d2a05e1aca330623c942b7e818e9e7c2669ee4",
+                "sha256:281c8b219d4f4b3581b918b816764098d04964915b2f272d1476654143801aa2",
+                "sha256:2d34a5450a402b00d20aeb7632489ffa2556ca7b26f4a63c35f6fccae1977427",
+                "sha256:301bd744a1adaa2f6a5e06c98f1ac2b6f8dc31a5c23b838f862d65e32fca0d4b",
+                "sha256:30e5ce9f501fb1f970e4a59098028cf20676dee64fc496d55c33e04bbbee097d",
+                "sha256:33ab498f9ac30598b6406e2be1b45fd231195b83d948ebd4bd77f337cb6a2bff",
+                "sha256:35585a8cb5917161f42c2104567bb83a1d96194095fc54a543113ed5df9fa436",
+                "sha256:389c0e38358fdc4e38e9995e7291269a3aead7acfcf8942010ee7bc5baee091c",
+                "sha256:3acadbab8b59f63b87b518e09c4c64b142e7286b9ca7a208107d6f9f4c393c5c",
+                "sha256:3b7a64d43e2a1fa2dd46b678e00cabd9a49ebb123b339ce799204c44a593ae1c",
+                "sha256:3c8c0226c71bd0ce9892eaf6afa77ae8f43a3d9313124a03df0b389c01f832de",
+                "sha256:429349a510da82c85431f0f3e66212d83efe9fd2850f50f339341b6532c62fe4",
+                "sha256:466030a42724780794dea71eb32db83cc51214d66ab3fb3156edd88b9c8f0d78",
+                "sha256:47aeceb4363851d17f63069318ba5721ae695d9da55d599b4d6fb31508595278",
+                "sha256:48aa98987d54a46e13e6954880056c204700c65616af4395d1f0639eba11764b",
+                "sha256:4b2416ed743ec5debcf61e1242e012652a4348de14ecc7df3512da072b074440",
+                "sha256:4d0a675a7acbbc16179188d8c6d0afb8628604fc1241faf41007255957335a0b",
+                "sha256:4eb74d44776b0fb0782560ea84d986dffec8ddd94947f383eba2284b0f32e35e",
+                "sha256:4f8a1d990dc198a6c68ec3d9a637ba1ce489b38cbfb65440a27901afbc5df575",
+                "sha256:513ccbf7420c30e283c25c82d5a8f439d625a838d3ba69e79a110c260c46813f",
+                "sha256:5210a0018c7e09c75fa788648617ebba861ae242944111d3079034e14498223f",
+                "sha256:54cdfcda59251b9c2f87a05d038c2ae02121219a04d4a1e6fc345794295bdc07",
+                "sha256:56dd500411d03c5e9927a1eb55621e906837a83b02350a9dc401247d0353717c",
+                "sha256:57ec6baec231bb19bb5fd5fc7bae21231860a1605174b11585660236627e390e",
+                "sha256:5f1519b080d8ce0a814f17ad9fb49fb3a1d4d7ce5891f5c85fc38631ca3a8dc4",
+                "sha256:6174d6ad6b58a6bcf67afbbf1723420a53d06c4b89f4c50763d6fa0a6ac9afd2",
+                "sha256:68172622a5a57deb079a2c78511c40f91193548e8ab342c31e8cb0764d362459",
+                "sha256:6915fc9fa6b3ec3569566832e1bb03bd801c12cea030200e68663b9a87974e76",
+                "sha256:6b75b912a0baa033350367a8a07a8b2d44fd5b90c890bfbd063a8a5f945f644b",
+                "sha256:6f5dcb658d597410bb7c967c1d24eaf9377b0d621358cbe9d2ff804e5dd12e81",
+                "sha256:6f8d7fe73d1816eeb5378409adc658f9525ecbfaf9e1ede1e2d67a338b0c7348",
+                "sha256:7036316cc26b93e401cedd781a579be606dad174829e6ad9e9c5a0da6e036f80",
+                "sha256:7188ddc1a8887194f984fa4110d5a3d5b9b5cd35f6bafdff1b649049cbc0ce29",
+                "sha256:761531076df51309075133a6bc1db02d98ec7f66e22b064b1d513bc909f29743",
+                "sha256:7979d90ee2190d000129598c2b0c82f13053dba432b94e45e68253b09bb1f0f6",
+                "sha256:8015835494b21aa7abd3b43fdea0614ee35ef6b03db7ecba9beb58eadf01c24f",
+                "sha256:81c4d1a3a564775c44732b94135d06e33417e829ff25226c164664f4a1046213",
+                "sha256:81cf9d306c04df1b45971c13167dc3bad625808aa01281d55f3cf852dde0e206",
+                "sha256:88857060b690a57d2ea8569bca58758143c8faa4639fb17d745ce60ff84c867e",
+                "sha256:8c567c664fc2f44130a20edac73e0a867f8e012bf7370276f15c6adc3586c37c",
+                "sha256:91bd2b7cf0f4d252eec8b7046fa6a43cee17e8acdfc00eaa8b3dbf2f9a59d061",
+                "sha256:9620650c364c01ed5b497dcae7c3d4b948daeae6e1883ae185fef1c927b6b534",
+                "sha256:9b007c2444705a2dc4a525964fd4dd28c3320b19b3410da6517cab28716f27d3",
+                "sha256:9bf9acce44e967a5103fcd820fc7580c7b0ab8583eec4e2051aec560f7b31a63",
+                "sha256:a239303acb0315091d54c7ff36712dba24554993b9a93941cf301391d8a997ee",
+                "sha256:a2baa6be130e8a00b6cbb9f18a33611ec150b4537f8563bddadb54c1b74b8193",
+                "sha256:a54917b7e9cd3a67e429a630e237a90b096e0ba18897bfb99ee8bd1068a5fea0",
+                "sha256:a689e1ded7137552bea36305a7a16ad2b40be511740b80748d3140614993db98",
+                "sha256:a952ae3eb460c6712388ac2ec706d24b0e651b9396d90c9a9e0a69eb27737fdc",
+                "sha256:aa32205358a76bf578854bf31698a86dc8b2cb591fd1d79a833283f4a403f04b",
+                "sha256:b2287c09482949e0ca0c0eb68b2aca6cf57f8af8c6dfd29dcd3bc45f17b57978",
+                "sha256:b6b0e17d39d21698185097652c611f9cf30f7c56ccec189789920e3e7f1cee56",
+                "sha256:b710bf7e7ae61957d5c4026b486be593ed3ec3dca3e5be15e0f6d8cf5d0a4990",
+                "sha256:b8e11715178f3608874508f08e990d3771e0b8c66c73eb4e183038d600a9b274",
+                "sha256:b92aafcfab3d41580d54aca35a8057341f1cfc7c9af9e8bdfc652f83a20ced31",
+                "sha256:bec29b801b4adbf388314c0d050e851d53762ab424af22657021ce4b6eb41543",
+                "sha256:c694bee70ece3b232df4678448fdda245fd3b1bb4ba481fb6cd20e13bb784c46",
+                "sha256:c6b52b7028b547866c2413f614ee306c2d4eafdd444b1ff656bf3295bf1484aa",
+                "sha256:cb41ad20064e18a900dd427d7cf41cfaec83bcd1184001f3d91a1f76b3fcea4e",
+                "sha256:cd316dbcc74c76266ba94eb021b0cc090b97cca122f50bd7a845f587ff4bf03f",
+                "sha256:ced40cdbb6dd47a032725a038896cceae9ce267d340f59508b23537f05455431",
+                "sha256:d1c562a9bb72244fa767d1c1ab55ca1d92dd5f7c4d77878fee5483a22ffac808",
+                "sha256:d389ff1e95b6e46ebedccf7fd1fadd10559add595ac6a7c2ea730268325f832c",
+                "sha256:d56b1cd606ba4cedd64bb43479d56580e147c6ef3f5d1c5e64203a1adab784a2",
+                "sha256:d72a4315514e5a0b9837a086cb433b004eea630afb0cc129de76d77654a9606f",
+                "sha256:d9e7f29c00577aff6b318681e730a519b235af292732a149337f6aaa4d1c5e31",
+                "sha256:dbc25baa6abb205766fb8606f8263b02c3503a55957fcb4576a6bb0a59d37d10",
+                "sha256:e57919c32ee295a2fca458bb73e4b20b05c115627f96f95a10f9f5acbd61172d",
+                "sha256:e5bbe011a2cea9060fef1bb3d668a2fd8432b8888e6d92e74c9c794d3c101595",
+                "sha256:e6aea5c0eb5b0faf52c7b5c4a47c8bb64437173be97227c819ffa31801fa4e34",
+                "sha256:e888be685fa42d8b8a3d3911d5604d14db87538aa7d0b29b1a7ea80d354c732d",
+                "sha256:eebaf8c76c39604d52852366249ab807fe6f7a3ffb0dd5484b9944917244cdbe",
+                "sha256:efbe0b5e0fd078ed7b005faa0170da4f72666360f66f0bb2d7f73526ecfd99f9",
+                "sha256:efddca2d02254a52078c35cadad34762adbae3ff01c6b0c7787b59d038b63e0d",
+                "sha256:f05450fa1cd7c525c0b9d1a7916e595d3041ac0afbed2ff6926e5afb6a781b7f",
+                "sha256:f12d69d568f5647ec503b64932874dade5a20255736c89936bf690951a5e79f5",
+                "sha256:f45321224144c25a62052035ce96cbcf264667bcb0d81823b1bbc22c4addd194",
+                "sha256:f62581d7e884dd01ee1707b7c21148f61f2febb7de092ae2f108743fcbef5985",
+                "sha256:f8832a4f83d4782a8f5a7b831c47e8ffe164e43c2c148c8160ed9a6d630bc02a",
+                "sha256:fa35ad36440aaf1ac8332b4a4a433d4acd28f1613f0d480995f5cfd3580e90b7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.6"
+            "version": "==0.12.0"
         },
         "rtree": {
             "hashes": [
@@ -3417,7 +3379,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "sniffio": {
@@ -3539,11 +3501,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:81539f07f7aebcde2e4b5ab76727f53eabf18ad155c6ed7979a681411602fa47",
-                "sha256:833273bf645d8ce31dcb613c56999e2e055b1ffe6d09168a164bcd91c36d5d35"
+                "sha256:9b232b9430c8f57288c1024b34a8f0251ddcc47268927367a0dd3eeaca40deb5",
+                "sha256:baf991e61542da48fe8aef8b779a9ea0aa38d8a54166ee250d5af5ecf4486619"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.12.0"
+            "version": "==5.13.0"
         },
         "typeguard": {
             "hashes": [
@@ -3608,10 +3570,10 @@
         },
         "wcwidth": {
             "hashes": [
-                "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704",
-                "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"
+                "sha256:9a929bd8380f6cd9571a968a9c8f4353ca58d7cd812a4822bba831f8d685b223",
+                "sha256:a675d1a4a2d24ef67096a04b85b02deeecd8e226f57b5e3a72dbb9ed99d27da8"
             ],
-            "version": "==0.2.8"
+            "version": "==0.2.9"
         },
         "webcolors": {
             "hashes": [
@@ -3750,11 +3712,11 @@
         },
         "xyzservices": {
             "hashes": [
-                "sha256:70b9910f6c8e46f6ca92dea21e9b8cf89edf0ead35a870198fb59a7d63579525",
-                "sha256:eee203e91955782fd8bfc2f05846830c289139dc0ab4eaf733bfa8f0be71861f"
+                "sha256:091229269043bc8258042edbedad4fcb44684b0473ede027b5672ad40dc9fa02",
+                "sha256:6a4c38d3a9f89d3e77153eff9414b36a8ee0850c9e8b85796fd1b2a85b8dfd68"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.10.0"
+            "version": "==2023.10.1"
         },
         "y-py": {
             "hashes": [
@@ -3855,10 +3817,10 @@
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
-                "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
+                "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24",
+                "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"
             ],
-            "version": "==2.4.0"
+            "version": "==2.4.1"
         },
         "backcall": {
             "hashes": [
@@ -3877,10 +3839,11 @@
         },
         "executing": {
             "hashes": [
-                "sha256:06df6183df67389625f4e763921c6cf978944721abf3e714000200aab95b0657",
-                "sha256:0ff053696fdeef426cda5bd18eacd94f82c91f49823a2e9090124212ceea9b08"
+                "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147",
+                "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"
             ],
-            "version": "==2.0.0"
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.1"
         },
         "ipython": {
             "hashes": [
@@ -3964,7 +3927,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "stack-data": {
@@ -3976,11 +3939,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:81539f07f7aebcde2e4b5ab76727f53eabf18ad155c6ed7979a681411602fa47",
-                "sha256:833273bf645d8ce31dcb613c56999e2e055b1ffe6d09168a164bcd91c36d5d35"
+                "sha256:9b232b9430c8f57288c1024b34a8f0251ddcc47268927367a0dd3eeaca40deb5",
+                "sha256:baf991e61542da48fe8aef8b779a9ea0aa38d8a54166ee250d5af5ecf4486619"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.12.0"
+            "version": "==5.13.0"
         },
         "typing-extensions": {
             "hashes": [
@@ -3992,10 +3955,10 @@
         },
         "wcwidth": {
             "hashes": [
-                "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704",
-                "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"
+                "sha256:9a929bd8380f6cd9571a968a9c8f4353ca58d7cd812a4822bba831f8d685b223",
+                "sha256:a675d1a4a2d24ef67096a04b85b02deeecd8e226f57b5e3a72dbb9ed99d27da8"
             ],
-            "version": "==0.2.8"
+            "version": "==0.2.9"
         }
     }
 }
diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
index b0197de74..71dbbc9b4 100755
--- a/data/wbd/generate_pre_clip_fim_huc8.py
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -131,7 +131,7 @@ def pre_clip_hucs_from_wbd(wbd_file, outputs_dir, huc_list, number_of_jobs, over
     if number_of_jobs > total_cpus_available:
         print(
             f'Provided: -j {number_of_jobs}, which is greater than than amount of available cpus -2: '
-            f'{total_cpus_available -2} will be used instead.'
+            f'{total_cpus_available - 2} will be used instead.'
         )
         number_of_jobs = total_cpus_available - 2
 
diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py
index 5d1f903b3..5ffb7a50b 100644
--- a/data/write_parquet_from_calib_pts.py
+++ b/data/write_parquet_from_calib_pts.py
@@ -231,7 +231,7 @@ def create_parquet_files(
     if number_of_jobs > total_cpus_available:
         logging.info(
             f'Provided: -j {number_of_jobs}, which is greater than than amount of available cpus -1: '
-            f'{total_cpus_available -1} will be used instead.'
+            f'{total_cpus_available - 1} will be used instead.'
         )
         number_of_jobs = total_cpus_available - 1
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7d9369013..a6cbbc5a7 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,52 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.7.0 - 2023-11-13 - [PR#1030](https://github.com/NOAA-OWP/inundation-mapping/pull/1030)
+
+This PR introduces the `.github/workflows/lint_and_format.yaml` file which serves as the first step in developing a Continuous Integration pipeline for this repository. 
+The `flake8-pyproject` dependency is now used, as it works out of the box with the `pre-commit` GitHub Action in the GitHub Hosted Runner environment.
+In switching to this package, a couple of `E721` errors appeared. Modifications were made to the appropriate files to resolve the `flake8` `E721` errors.
+Also, updates to the `unit_tests` were necessary since Branch IDs have changed with the latest code.  
+
+A small fix was also included where `src_adjust_ras2fim_rating.py` which sometimes fails with an encoding error when the ras2fim csv sometimes is created or adjsuted in windows.
+
+### Changes
+- `.pre-commit-config.yaml`: use `flake8-pyproject` package instead of `pyproject-flake8`.
+- `Pipfile` and `Pipfile.lock`: updated to use `flake8-pyproject` package instead of `pyproject-flake8`, upgrade `pyarrow` version.
+- `data`
+    - `/wbd/generate_pre_clip_fim_huc8.py`: Add space between (-) operator line 134.
+    - `write_parquet_from_calib_pts.py`: Add space between (-) operator line 234.
+- `src`
+    - `check_huc_inputs.py`: Change `== string` to `is str`, remove `import string`
+    - `src_adjust_ras2fim_rating.py`: Fixed encoding error.
+- `tools`
+    - `eval_plots.py`: Add space after comma in lines 207 & 208
+    - `generate_categorical_fim_mapping.py`: Use `is` instead of `==`, line 315
+    - `hash_compare.py`: Add space after comma, line 153.
+    - `inundate_mosaic_wrapper.py`: Use `is` instead of `==`, line 73.
+    - `inundation_wrapper_nwm_flows.py`: Use `is not` instead of `!=`, line 76.
+    - `mosaic_inundation.py`: Use `is` instead of `==`, line 181.
+- `unit_tests`
+    - `README.md`: Updated documentation, run `pytest` in `/foss_fim` directory.
+    - `clip_vectors_to_wbd_test.py`: File moved to data/wbd directory, update import statement, skipped this test.
+    - `filter_catchments_and_add_attributes_params.json`: Update Branch ID
+    - `inundate_gms_params.json`: Moved to `unit_tests/` folder.
+    - `inundate_gms_test.py`: Moved to `unit_tests/` folder.
+    - `inundation_params.json`: Moved to `unit_tests/` folder.
+    - `inundation_test.py`: Moved to `unit_tests/` folder.
+    - `outputs_cleanup_params.json`: Update Branch ID
+    - `outputs_cleanup_test.py`: Update import statement
+    - `split_flows_params.json`: Update Branch ID
+    - `usgs_gage_crosswalk_params.json`: Update Branch ID & update argument to gage_crosswalk.run_crosswalk
+    - `usgs_gage_crosswalk_test.py`: Update params to gage_crosswalk.run_crosswalk
+
+### Additions 
+- `.github/workflows/`
+    - `lint_and_format.yaml`: Add GitHub Actions Workflow file for Continuous Integration environment (lint and format test).
+
+<br/><br/>
+
+=======
 ## v4.4.6.0 - 2023-11-17 - [PR#1031](https://github.com/NOAA-OWP/inundation-mapping/pull/1031)
 
 Upgrade our acquire 3Dep DEMs script to pull down South Alaska HUCS with its own CRS.
@@ -23,6 +69,7 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1
         - Changed date/times to utc.
         - Upgraded error handing for the gdal "processing" call.
 
+
 <br/><br/>
 
 ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index ef58280d0..6c6c2fad9 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -4,7 +4,6 @@
 import argparse
 import os
 import pathlib
-import string
 from glob import glob
 from logging import exception
 
@@ -59,8 +58,8 @@ def __clean_huc_value(huc):
 
 def __check_for_membership(hucs, accepted_hucs_set):
     for huc in hucs:
-        if (type(huc) == string) and (not huc.isnumeric()):
-            msg = f"Huc value of {huc} does not appear to be a number."
+        if (type(huc) is str) and (not huc.isnumeric()):
+            msg = f"Huc value of {huc} does not appear to be a number. "
             msg += "It could be an incorrect value but also could be that the huc list "
             msg += "(if you used one), is not unix encoded."
             raise KeyError(msg)
diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
index 7136e617a..22a47b33e 100644
--- a/src/src_adjust_ras2fim_rating.py
+++ b/src/src_adjust_ras2fim_rating.py
@@ -52,7 +52,9 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
     print('Reading RAS2FIM rating curves from csv...')
     log_text = 'Processing database for RAS2FIM flow/WSE at NWM flow recur intervals...\n'
     col_filter = ["fid_xs", "flow", "wse"]
-    ras_rc_df = pd.read_csv(ras_rc_filepath, dtype={'fid_xs': object}, usecols=col_filter)  # , nrows=30000)
+    ras_rc_df = pd.read_csv(
+        ras_rc_filepath, dtype={'fid_xs': object}, usecols=col_filter, encoding="unicode_escape"
+    )  # , nrows=30000)
     ras_rc_df.rename(columns={'fid_xs': 'location_id'}, inplace=True)
     # ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
     print('Duration (read ras_rc_csv): {}'.format(dt.datetime.now() - start_time))
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index dab85e934..75773deec 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -204,8 +204,8 @@ def scatterplot(dataframe, x_field, y_field, title_text, stats_text=False, annot
     axes.tick_params(labelsize='xx-large')
 
     # Define y axis label and x axis label.
-    axes.set_ylabel(f'{y_field.replace("_"," ")}', fontsize='xx-large', weight='bold')
-    axes.set_xlabel(f'{x_field.replace("_"," ")}', fontsize='xx-large', weight='bold')
+    axes.set_ylabel(f'{y_field.replace("_", " ")}', fontsize='xx-large', weight='bold')
+    axes.set_xlabel(f'{x_field.replace("_", " ")}', fontsize='xx-large', weight='bold')
 
     # Plot diagonal line
     diag_range = [0, 1]
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index 1fb068d77..df44a4a52 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -312,7 +312,7 @@ def reformat_inundation_maps(
         handle = os.path.split(extent_grid)[1].replace('.tif', '')
         diss_extent_filename = os.path.join(gpkg_dir, f"{handle}_{huc}_dissolved.gpkg")
         extent_poly_diss["geometry"] = [
-            MultiPolygon([feature]) if type(feature) == Polygon else feature
+            MultiPolygon([feature]) if type(feature) is Polygon else feature
             for feature in extent_poly_diss["geometry"]
         ]
 
diff --git a/tools/hash_compare.py b/tools/hash_compare.py
index 49946025a..8de401be2 100755
--- a/tools/hash_compare.py
+++ b/tools/hash_compare.py
@@ -150,7 +150,7 @@ def compare_gpkg(file1, file2, list_of_failed_files=[], verbose=False):
     except AssertionError as e:
         print(f"\n {str(e)} \n")
         print("  The following files failed assert_geodataframe_equal: ")
-        print(f"    -{file1.rsplit('/',1)[-1]} ")
+        print(f"    {file1.rsplit('/', 1)[-1]} ")
         list_of_failed_files.append(f1_gdf)
 
 
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
index 1620a7820..c75c4ea2e 100644
--- a/tools/inundate_mosaic_wrapper.py
+++ b/tools/inundate_mosaic_wrapper.py
@@ -70,8 +70,9 @@ def produce_mosaicked_inundation(
         raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), hydrofabric_dir)
 
     # If the "hucs" argument is really one huc, convert it to a list
-    if type(hucs) == str:
+    if type(hucs) is str:
         hucs = [hucs]
+
     # Check that huc folder exists in the hydrofabric_dir.
     for huc in hucs:
         if not os.path.exists(os.path.join(hydrofabric_dir, huc)):
diff --git a/tools/inundation_wrapper_nwm_flows.py b/tools/inundation_wrapper_nwm_flows.py
index 68f4ff27b..2b2d73b60 100755
--- a/tools/inundation_wrapper_nwm_flows.py
+++ b/tools/inundation_wrapper_nwm_flows.py
@@ -73,7 +73,7 @@ def run_recurr_test(fim_run_dir, branch_name, huc_id, magnitude, mask_type='huc'
 
     # Check if magnitude is list of magnitudes or single value.
     magnitude_list = magnitude
-    if type(magnitude_list) != list:
+    if type(magnitude_list) is not list:
         magnitude_list = [magnitude_list]
 
     for magnitude in magnitude_list:
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 6229b9907..7baa7f5dc 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -178,7 +178,7 @@ def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon
         extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=src.crs)
         extent_poly_diss = extent_poly.dissolve(by="extent")
         extent_poly_diss["geometry"] = [
-            MultiPolygon([feature]) if type(feature) == Polygon else feature
+            MultiPolygon([feature]) if type(feature) is Polygon else feature
             for feature in extent_poly_diss["geometry"]
         ]
 
diff --git a/unit_tests/README.md b/unit_tests/README.md
index f957a4e77..6612b9761 100644
--- a/unit_tests/README.md
+++ b/unit_tests/README.md
@@ -56,7 +56,7 @@ docker run --rm -it --name mytest \
 	-v /abcd_share/foss_fim/outputs_temp/:/fim_temp \
 	fim_4:dev_20220208_8eba0ee
 ```
-* Call the `fim_pipeline.sh` script with the necessary arguments.
+* Call the `fim_pipeline.sh` script with the necessary arguments (`-n` must be `unit_test_data`).
 ```bash
 fim_pipeline.sh -n unit_test_data -u "02020005 05030104" \
 	-bd /foss_fim/config/deny_branch_unittests.lst \
@@ -79,18 +79,21 @@ python3 foss_fim/tools/synthesize_test_cases.py \
 ## Running unit tests
 
 ### If you'd like to test the whole unit test suite:
+When inside of the container, ensure you are within the root directory of the repository before running the `pytest` command.
 ```
+cd /foss_fim
 pytest /foss_fim/unit_tests
 ```
 
-__NOTE:__ This is subject to error, as the downloaded/generated data could potentially have a different path than what is specified in the `/unit_tests/*_params.json` files. The data files are not included in this repository, are potentially not uniform accross machines, and are subject to change.
+__NOTE:__ This is subject to error, as the downloaded/generated data could potentially have a different path than what is specified in the `/unit_tests/*_params.json` files. The data files are not included in this repository, so are subject to change.
 
 ### If you want to test just one unit test (from the root terminal window):
 
 ```bash
-pytest /foss_fim/unit_tests/gms/derive_level_paths_test.py
+cd /foss_fim
+pytest unit_tests/gms/derive_level_paths_test.py
 						or
-pytest /foss_fim/unit_tests/clip_vectors_to_wbd_test.py
+pytest unit_tests/inundate_gms_test.py
 ```
 
 ### If you'd like to run a particular test, you can, for example:
@@ -133,9 +136,7 @@ ie:
 
 ## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
 
-The `pyproject.toml` file has been added, which contains the build system requirements of Python projects.  This file used to specify which warnings are disabled to pass our unit tests.
-
-A `__init__.py` file has been added to the subdirectory of `/tools` in order for the `pytest` command run in the `/unit_tests` to pick up the tests in those directories as well.
+The `unit_tests/pyproject.toml` file is used to specify which warnings are disabled for our particular unit tests.
 
 Luckily, `pytest` works well with The Python Standard Library `unittest`. This made the migration of previous unit tests using `unittest` over to `pytest` quite simple. The caveat is that our current unit tests employ elements of both libraries. A full transition to `pytest` will ideally take place at a future date.
 
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
index 2930883db..9c817a448 100644
--- a/unit_tests/clip_vectors_to_wbd_test.py
+++ b/unit_tests/clip_vectors_to_wbd_test.py
@@ -4,10 +4,11 @@
 import os
 import unittest
 
-import clip_vectors_to_wbd as src
 import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
+import data.wbd.clip_vectors_to_wbd as src
+
 
 class test_clip_vectors_to_wbd(unittest.TestCase):
 
@@ -24,7 +25,11 @@ def setUpClass(self):
     # Test Cases:
 
     # TODO New Test Case to check that files were being created on the file system
-
+    @pytest.mark.skip(
+        reason="pre clipped vector files are now generated using generate_pre_clip_fim_huc8.py and "
+        "sent to the /inputs directory. Testing the clip_vectors_to_wbd.py is no longer necessary, "
+        "as this script is not run during fim_pipeline.sh execution."
+    )
     def test_subset_vector_layers_success(self):
         """
         This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist.
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
index 84a4b5b19..a68a46fd7 100644
--- a/unit_tests/filter_catchments_and_add_attributes_params.json
+++ b/unit_tests/filter_catchments_and_add_attributes_params.json
@@ -1,10 +1,10 @@
 {
     "valid_data": {
         "outputDestDir": "/outputs/unit_test_data",
-        "input_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/gw_catchments_reaches_3246000006.gpkg",
-        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/demDerived_reaches_split_3246000006.gpkg",
-        "output_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/gw_catchments_reaches_filtered_addedAttributes_3246000006.gpkg",
-        "output_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000006/demDerived_reaches_split_filtered_3246000006.gpkg",
+        "input_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/gw_catchments_reaches_2274000033.gpkg",
+        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/demDerived_reaches_split_2274000033.gpkg",
+        "output_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/gw_catchments_reaches_filtered_addedAttributes_2274000033.gpkg",
+        "output_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/demDerived_reaches_split_filtered_2274000033.gpkg",
         "wbd_filename": "/data/outputs/unit_test_data/02020005/wbd8_clp.gpkg",
         "huc_code": "02020005"
     }
diff --git a/unit_tests/tools/inundate_gms_params.json b/unit_tests/inundate_gms_params.json
similarity index 100%
rename from unit_tests/tools/inundate_gms_params.json
rename to unit_tests/inundate_gms_params.json
diff --git a/unit_tests/tools/inundate_gms_test.py b/unit_tests/inundate_gms_test.py
similarity index 97%
rename from unit_tests/tools/inundate_gms_test.py
rename to unit_tests/inundate_gms_test.py
index b15557a7b..c8cfdab5e 100644
--- a/unit_tests/tools/inundate_gms_test.py
+++ b/unit_tests/inundate_gms_test.py
@@ -8,8 +8,7 @@
 import inundate_gms as src
 import pandas as pd
 import pytest
-
-from unit_tests.unit_tests_utils import FIM_unit_test_helpers as ut_helpers
+from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
 
 class test_inundate_gms(unittest.TestCase):
diff --git a/unit_tests/tools/inundation_params.json b/unit_tests/inundation_params.json
similarity index 100%
rename from unit_tests/tools/inundation_params.json
rename to unit_tests/inundation_params.json
diff --git a/unit_tests/tools/inundation_test.py b/unit_tests/inundation_test.py
similarity index 100%
rename from unit_tests/tools/inundation_test.py
rename to unit_tests/inundation_test.py
diff --git a/unit_tests/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json
index 516292c3f..a3d70bca5 100644
--- a/unit_tests/outputs_cleanup_params.json
+++ b/unit_tests/outputs_cleanup_params.json
@@ -1,8 +1,8 @@
 {
     "valid_specific_branch_data": {
-        "src_dir": "/data/outputs/unit_test_data/02020005/branches/3246000009",
+        "src_dir": "/data/outputs/unit_test_data/02020005/branches/2274000018",
         "deny_list": "/foss_fim/config/deny_branches.lst",
-        "branch_id": "3246000009",
+        "branch_id": "2274000018",
         "verbose": true
     },
     "valid_directory_data": {
diff --git a/unit_tests/outputs_cleanup_test.py b/unit_tests/outputs_cleanup_test.py
index e3c9e5149..d339ff183 100644
--- a/unit_tests/outputs_cleanup_test.py
+++ b/unit_tests/outputs_cleanup_test.py
@@ -11,7 +11,7 @@
 import pytest
 from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
 
-import src.outputs_cleanup as src
+import outputs_cleanup as src
 from utils.shared_functions import FIM_Helpers as fh
 
 
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
index e7f7676c9..a4a17c248 100644
--- a/unit_tests/split_flows_params.json
+++ b/unit_tests/split_flows_params.json
@@ -4,10 +4,10 @@
         "max_length": 1500,
         "slope_min": 0.001,
         "lakes_buffer_input": 20,
-        "flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_3246000005.shp",
-        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-        "split_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_split_3246000005.gpkg",
-        "split_points_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_split_points_3246000005.gpkg",
+        "flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/demDerived_reaches_2274000031.shp",
+        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/dem_thalwegCond_2274000031.tif",
+        "split_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/demDerived_reaches_split_2274000031.gpkg",
+        "split_points_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/demDerived_reaches_split_points_2274000031.gpkg",
         "wbd8_clp_filename": "/data/outputs/unit_test_data/02020005/wbd8_clp.gpkg",
         "lakes_filename": "/data/outputs/unit_test_data/02020005/nwm_lakes_proj_subset.gpkg",
         "nwm_streams_filename": "/data/outputs/unit_test_data/02020005/nwm_subset_streams_levelPaths.gpkg"
diff --git a/unit_tests/tools/__init__.py b/unit_tests/tools/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json
index 33ec7cd41..e65627bed 100644
--- a/unit_tests/usgs_gage_crosswalk_params.json
+++ b/unit_tests/usgs_gage_crosswalk_params.json
@@ -1,11 +1,12 @@
 {
     "valid_data": {
         "usgs_gages_filename": "/data/outputs/unit_test_data/02020005/usgs_subset_gages.gpkg",
-        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/demDerived_reaches_split_filtered_3246000005.gpkg",
-        "input_catchment_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/gw_catchments_reaches_filtered_addedAttributes_3246000005.gpkg",
-        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/dem_meters_3246000005.tif",
-        "dem_adj_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/dem_thalwegCond_3246000005.tif",
-        "output_table_filename": "/data/outputs/unit_test_data/02020005/branches/3246000005/usgs_elev_table.csv",
-        "branch_id": "3246000005"
+        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/demDerived_reaches_split_filtered_2274000028.gpkg",
+        "input_catchment_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/gw_catchments_reaches_filtered_addedAttributes_2274000028.gpkg",
+        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/dem_meters_2274000028.tif",
+        "dem_adj_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/dem_thalwegCond_2274000028.tif",
+        "output_table_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/usgs_elev_table.csv",
+        "output_directory": "/data/outputs/unit_test_data/02020005/branches/2274000028",
+        "branch_id": "2274000028"
     }
 }
diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py
index 0fe7d41c9..43d9fa2de 100644
--- a/unit_tests/usgs_gage_crosswalk_test.py
+++ b/unit_tests/usgs_gage_crosswalk_test.py
@@ -52,7 +52,7 @@ def test_GageCrosswalk_success(self):
             params["input_flows_filename"],
             params["dem_filename"],
             params["dem_adj_filename"],
-            params["output_table_filename"],
+            params["output_directory"],
         )
 
         # Make sure that the usgs_elev_table.csv was written

From 3a453705f723de39922f2472520af75fe8f43490 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 1 Dec 2023 13:24:58 -0700
Subject: [PATCH 288/359] v4.4.7.1 Adjust ras2fim calibration input changes
 (#1036)

---
 config/params_template.env  | 15 ---------------
 docs/CHANGELOG.md           | 22 ++++++++++++++++++++--
 src/bash_variables.env      | 24 ++++++++++++++++++++++++
 src/run_unit_wb.sh          |  2 +-
 src/usgs_gage_crosswalk.py  | 12 +++++++++---
 src/usgs_gage_unit_setup.py |  4 +---
 6 files changed, 55 insertions(+), 24 deletions(-)

diff --git a/config/params_template.env b/config/params_template.env
index 6c2cb1687..ab761336c 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -33,48 +33,33 @@ export levee_id_attribute=SYSTEM_ID
 
 #### apply bathymetry adjustment to rating curve ####
 export bathymetry_adjust=True
-export bathymetry_file="$inputsDir/bathymetry/bathymetry_adjustment_data.gpkg"
 
 #### estimating bankfull stage in SRCs ####
 # Toggle to run identify_bankfull routine (True=on; False=off)
 export src_bankfull_toggle="True"
-# input file location with nwm feature_id and recurrence flow values
-export bankfull_flows_file="$inputsDir/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv"
 
 #### applying channel/overbank subdivision routine to SRCs ####
 # Toggle to run composite roughness src routine (True=on; False=off)
 export src_subdiv_toggle="True"
 # text to append to output log and hydrotable file names (use for testing/debugging)
 export vrough_suffix=""
-# input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv"
 
 #### SRC calibration variables
-# input file location with nwm feature_id and recurrence flow values
-export nwm_recur_file="$inputsDir/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv"
-
 #### apply SRC adjustments using USGS rating curve database ####
 # Toggle to run src adjustment routine with USGS rating data (True=on; False=off)
 export src_adjust_usgs="True"
-# input file location with usgs rating curve database
-export usgs_rating_curve_csv="$inputsDir/usgs_gages/usgs_rating_curves.csv"
 
 #### apply SRC adjustments using ras2fim rating curve database ####
 # Toggle to run src adjustment routine with ras2fim data (True=on; False=off)
 export src_adjust_ras2fim="True"
-# input file location with nwm feature_id and recurrence flow values
-export ras_rating_curve_csv="$inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table.csv"
 
 #### apply SRC adjustments using observed FIM/flow point database (parquet files)####
 # Toggle to run src adjustment routine with flood edge point db (True=on; False=off)
 export src_adjust_spatial="True"
-export fim_obs_pnt_data="$inputsDir/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 
 #### apply manual calibration ####
 # Toggle to run manual calibration routine (True=on; False=off)
 export manual_calb_toggle="True"
-# Input file location with HUC, nwm feature_id and manual calibration coefficients
-export man_calb_file="$inputsDir/rating_curve/manual_calibration_coefficients.csv"
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a6cbbc5a7..09d892805 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,26 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.7.1 - 2023-12-01 - [PR#1036](https://github.com/NOAA-OWP/inundation-mapping/pull/1036)
+
+Quick update to match incoming ras2fim calibration output files being feed into FIM was the initial change.
+
+There is no FIM issue card for this, but this is related to a ras2fim [PR #205](https://github.com/NOAA-OWP/ras2fim/pull/205) which also made changes to ensure compatibility. New copies of both the `reformat_ras_rating_curve_table_rel_101.csv` and `reformat_ras_rating_curve_points_rel_101.gpkg` were generated from ras2fim but retained the version of `rel_101`.
+
+Originally, was planning to update just the two locations for newer versions of the two `reformat_ras_rating_surve...` files. Both had been update to recognize the ras2release version rel_101.
+
+In the process of doing that, we took the opportunity to move all inputs files from params_template.env and put them into bash_variables.env as per precedence set recently.
+
+### Changes
+
+- `config`/`params_template.env`: moved input variables into `src/bash_variables.env`
+- `src`
+    - `bash_variablles.env`: Added all input variables from `params_template.env` to here and added one new one from `run_unit_wb.sh` for ras_rating_curve_points_gpkg.
+    - `run_unit_wb.sh`:   Updated an input param to the usgs_gage_unit_setup.py file to point the -ras param to the updated rel_101 value now in the `src/bash_variables.env`.
+    - `usgs_gage_unit_setup.py`:  Changed to drop a column no longer going to be coming from ras2fim calibration files.
+
+<br/><br/>
+
 ## v4.4.7.0 - 2023-11-13 - [PR#1030](https://github.com/NOAA-OWP/inundation-mapping/pull/1030)
 
 This PR introduces the `.github/workflows/lint_and_format.yaml` file which serves as the first step in developing a Continuous Integration pipeline for this repository. 
@@ -46,7 +66,6 @@ A small fix was also included where `src_adjust_ras2fim_rating.py` which sometim
 
 <br/><br/>
 
-=======
 ## v4.4.6.0 - 2023-11-17 - [PR#1031](https://github.com/NOAA-OWP/inundation-mapping/pull/1031)
 
 Upgrade our acquire 3Dep DEMs script to pull down South Alaska HUCS with its own CRS.
@@ -69,7 +88,6 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1
         - Changed date/times to utc.
         - Upgraded error handing for the gdal "processing" call.
 
-
 <br/><br/>
 
 ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 01a1cf4f3..fe348a63c 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -16,6 +16,30 @@ export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_W
 export          input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
 export                pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
 
+export                 bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
+
+# input file location with nwm feature_id and recurrence flow values
+export             bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
+
+# input file location with nwm feature_id and channel roughness and overbank roughness attributes
+export                vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv
+
+# input file location with nwm feature_id and recurrence flow values
+export                  nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
+
+# input file location with usgs rating curve database
+export           usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
+
+# input file locations for ras2fim locations and rating curve data
+export            ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
+export    ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
+
+export                fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
+
+# Input file location with HUC, nwm feature_id and manual calibration coefficients
+export                   man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv
+
+
 # Styling
 export startDiv="\n-----------------------------------------------------------------\n"
 export  stopDiv="\n-----------------------------------------------------------------\n"
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 4dc8a4898..701246900 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -276,7 +276,7 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     python3 $srcDir/usgs_gage_unit_setup.py \
         -gages $inputsDir/usgs_gages/usgs_gages.gpkg \
         -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
-        -ras $inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points.gpkg \
+        -ras $ras_rating_curve_points_gpkg \
         -o $tempHucDataDir/usgs_subset_gages.gpkg \
         -huc $hucNumber \
         -ahps $inputsDir/ahps_sites/nws_lid.gpkg \
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 6adba78d4..b87dfc430 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -117,8 +117,8 @@ def write(self, output_directory):
         elev_table = elev_table[elev_table['location_id'].notna()]
         elev_table.source = elev_table.source.apply(str.lower)
 
-        # filter for just ras2fim entries
-        ras_elev_table = elev_table[elev_table['source'] == 'ras2fim']
+        # filter for just ras2fim entries (note that source column includes suffix with version number)
+        ras_elev_table = elev_table[elev_table['source'].str.contains('ras2fim')]
         ras_elev_table = ras_elev_table[
             [
                 "location_id",
@@ -134,9 +134,15 @@ def write(self, output_directory):
         ]
         if not ras_elev_table.empty:
             ras_elev_table.to_csv(join(output_directory, 'ras_elev_table.csv'), index=False)
+        else:
+            print(
+                'INFO: there were no ras2fim points located in this huc'
+                ' (note that most hucs do not have ras2fim data)'
+            )
 
         # filter for just usgs entries
-        usgs_elev_table = elev_table[elev_table['source'] != 'ras2fim']
+        # look for source attributes that do not contain "ras2fim"
+        usgs_elev_table = elev_table[~elev_table['source'].str.contains('ras2fim')]
         if not usgs_elev_table.empty:
             usgs_elev_table.to_csv(join(output_directory, 'usgs_elev_table.csv'), index=False)
 
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 8ef766aaa..61e82c797 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -33,9 +33,7 @@ def load_gages(self):
         # !!! Geopandas is not honoring the dtype arg with this read_file below (huc8 being read as int64).
         # Need the raw data to store the 'huc8' attribute as an object to avoid issues with integers truncating the leading zero from some hucs
         ras_locs = gpd.read_file(self.ras_locs_filename, dtype={'huc8': 'object'})
-        ras_locs = ras_locs[
-            ['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'wrds_timestamp', 'geometry']
-        ]
+        ras_locs = ras_locs[['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'geometry']]
         ras_locs['location_id'] = ras_locs['fid_xs']
 
         # Convert ras locs crs to match usgs gage crs

From 5e794f55f2f2ac909795f66bfba06dfec8d28e0f Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 8 Dec 2023 13:40:41 -0700
Subject: [PATCH 289/359] v4.4.7.2 Fix inundation nation pathing errors (#1026)

* Fixes for inundate_nation to create boolean rasters

* fixed bug with default huc lists

* took out debug line, added more prints

* fix some output print errors

* debugging

* rolled most changes back to Ryan's orig fix

* minor adj to temp dir

* update notes in_nation

* Update inundate_nation.py

* Update CHANGELOG.md

* Update params_template.env

* Added vrt raster and final mosaic step; fixed logging

* More fixes for logging info and adding log printing to screen

* fix linting error

* Added multi-threading to gdal.translate

* linting fix

* Revert "linting fix"

This reverts commit 15aad0427198d5bece4f33b6b7629f45f1c86417.

* fixing linting errors

* more linting fixes

---------

Co-authored-by: RyanSpies-NOAA <ryan.spies@noaa.gov>
Co-authored-by: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
---
 docs/CHANGELOG.md                |  15 ++++
 tools/inundate_mosaic_wrapper.py |   2 -
 tools/inundate_nation.py         | 124 +++++++++++++++++++++++--------
 3 files changed, 107 insertions(+), 34 deletions(-)
 mode change 100644 => 100755 tools/inundate_nation.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 09d892805..a1314d95f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.7.2 - 2023-12-08 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026)
+
+A couple of directly related issues were fixed in this PR.
+The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundation-mapping/issues/1025) which was about a pathing issue for the outputs directory. In testing that fix, it exposed a few other pathing and file cleanup issues which are now fixed. We also added more console output to help view variables and pathing.
+
+### Changes
+
+- `config`/`params_template.env`:  Updated for a newer mannings global file. Changed and tested by Ryan Spies.
+- `tools`
+    - `inundate_mosiac_wrapper.py`:  Took out a misleading and non-required print statement.
+    - `inundate_nation.py`: As mentioned above.
+
+<br/><br/>
+
 ## v4.4.7.1 - 2023-12-01 - [PR#1036](https://github.com/NOAA-OWP/inundation-mapping/pull/1036)
 
 Quick update to match incoming ras2fim calibration output files being feed into FIM was the initial change.
@@ -90,6 +104,7 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1
 
 <br/><br/>
 
+
 ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
 
 During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times.
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
index c75c4ea2e..5b274122a 100644
--- a/tools/inundate_mosaic_wrapper.py
+++ b/tools/inundate_mosaic_wrapper.py
@@ -93,8 +93,6 @@ def produce_mosaicked_inundation(
             "Please lower the num_workers.".format(num_workers, total_cpus_available)
         )
 
-    fh.vprint("Running inundate for " + huc + "...", verbose)
-
     # Call Inundate_gms
     map_file = Inundate_gms(
         hydrofabric_dir=hydrofabric_dir,
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
old mode 100644
new mode 100755
index fe8680418..050436670
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -6,17 +6,14 @@
 import os
 import re
 import shutil
-import sys
 from datetime import datetime
 from multiprocessing import Pool
 
 import rasterio
 from inundate_mosaic_wrapper import produce_mosaicked_inundation
-from osgeo import gdal, ogr
-from rasterio.merge import merge
+from osgeo import gdal
 
 from utils.shared_functions import FIM_Helpers as fh
-from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv
 
 
 # INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
@@ -26,6 +23,9 @@
 # DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
 
 
+# TODO: Nov 2023, Logging system appears to be not working correctly.
+
+
 def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, inc_mosaic, job_number):
     assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}"
 
@@ -37,12 +37,14 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
             + " max jobs will be used instead."
         )
 
+    print()
+    print("Inundation Nation script starting...")
+
     fim_version = os.path.basename(os.path.normpath(fim_run_dir))
-    logging.info(f"Using fim version: {fim_version}")
     output_base_file_name = magnitude_key + "_" + fim_version
-    # print(output_base_file_name)
 
     __setup_logger(output_dir, output_base_file_name)
+    logging.info(f"Using fim version: {fim_version}")
 
     start_dt = datetime.now()
 
@@ -52,48 +54,42 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
     logging.info(f"flow_file: {flow_file}")
     logging.info(f"inc_mosaic: {str(inc_mosaic)}")
 
-    print("Preparing to generate inundation outputs for magnitude: " + magnitude_key)
-    print("Input flow file: " + flow_file)
-
     magnitude_output_dir = os.path.join(output_dir, output_base_file_name)
 
     if not os.path.exists(magnitude_output_dir):
-        print("Creating new output directory for raw mosaic files: " + magnitude_output_dir)
+        logging.info(
+            "Removing previous output dir and creating new output dir for inunation wrapper files: "
+            + magnitude_output_dir
+        )
         os.mkdir(magnitude_output_dir)
     else:
         # we need to empty it. we will kill it and remake it (using rmtree to force it)
         shutil.rmtree(magnitude_output_dir, ignore_errors=True)
         os.mkdir(magnitude_output_dir)
 
-    if huc_list is None:
+    if huc_list == 'all' or len(huc_list) == 0:
         huc_list = []
         for huc in os.listdir(fim_run_dir):
-            # if (
-            #     huc != 'logs'
-            #     and huc != 'branch_errors'
-            #     and huc != 'unit_errors'
-            #     and os.path.isdir(os.path.join(fim_run_dir, huc))
-            # ):
             if re.match(r'\d{8}', huc):
                 huc_list.append(huc)
     else:
         for huc in huc_list:
-            assert os.path.isdir(
-                fim_run_dir + os.sep + huc
-            ), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}'
+            huc_path = os.path.join(fim_run_dir, huc)
+            assert os.path.isdir(huc_path), f'ERROR: could not find the input fim_dir location: {huc_path}'
 
-    print("Inundation raw mosaic outputs here: " + magnitude_output_dir)
+    huc_list.sort()
 
+    logging.info(f"Inundation mosaic wrapper outputs will saved here: {magnitude_output_dir}")
     run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number])
 
     # Perform mosaic operation
     if inc_mosaic:
         fh.print_current_date_time()
         logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))
-        print("Performing bool mosaic process...")
         logging.info("Performing bool mosaic process...")
-
         output_bool_dir = os.path.join(output_dir, "bool_temp")
+        logging.info(f"output_bool_dir is {output_bool_dir}")
+
         if not os.path.exists(output_bool_dir):
             os.mkdir(output_bool_dir)
         else:
@@ -105,7 +101,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
         for rasfile in os.listdir(magnitude_output_dir):
             if rasfile.endswith(".tif") and "extent" in rasfile:
                 # p = magnitude_output_dir + rasfile
-                procs_list.append([magnitude_output_dir, rasfile, output_bool_dir])
+                procs_list.append([magnitude_output_dir, rasfile, output_bool_dir, fim_version])
 
         # Multiprocess --> create boolean inundation rasters for all hucs
         if len(procs_list) > 0:
@@ -116,10 +112,17 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
             print(msg)
             logging.info(msg)
 
-        # now cleanup the raw mosiac directories
+        # Perform VRT creation and mosaic all of the huc rasters using boolean rasters
+        vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name, job_number)
+
+        # now cleanup the temp bool directory
         shutil.rmtree(output_bool_dir, ignore_errors=True)
 
+    else:
+        print("Skipping mosiaking")
+
     # now cleanup the raw mosiac directories
+    # comment this out if you want to see the individual huc rasters
     shutil.rmtree(magnitude_output_dir, ignore_errors=True)
 
     fh.print_current_date_time()
@@ -149,7 +152,16 @@ def run_inundation(args):
 
     inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif")
 
-    print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude))
+    logging.info(
+        "Running inundation wrapper for the NWM recurrence intervals for each huc using magnitude: "
+        + str(magnitude)
+    )
+    print(
+        "This will take a long time depending on the number of HUCs. Progress bar may not appear."
+        " Once it gets to boolean/mosiacing (if applicable), screen output will exist. To see if the script has frozen,"
+        " you should be able to watch the file system for some changes."
+    )
+    print()
 
     produce_mosaicked_inundation(
         fim_run_dir,
@@ -167,6 +179,7 @@ def create_bool_rasters(args):
     in_raster_dir = args[0]
     rasfile = args[1]
     output_bool_dir = args[2]
+    fim_version = args[3]
 
     print("Calculating boolean inundate raster: " + rasfile)
     p = in_raster_dir + os.sep + rasfile
@@ -189,23 +202,70 @@ def create_bool_rasters(args):
         dtype="int8",
         compress="lzw",
     )
-    with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, "w", **profile) as dst:
+    with rasterio.open(
+        output_bool_dir + os.sep + rasfile[:-4] + '_' + fim_version + '.tif', "w", **profile
+    ) as dst:
         dst.write(array.astype(rasterio.int8))
 
 
-def __setup_logger(output_folder_path, log_file_name_key):
+def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads):
+    rasters_to_mosaic = []
+    for rasfile in os.listdir(output_bool_dir):
+        if rasfile.endswith('.tif') and "extent" in rasfile:
+            p = output_bool_dir + os.sep + rasfile
+            rasters_to_mosaic.append(p)
+
+    output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt")
+    logging.info("Creating virtual raster: " + output_mosiac_vrt)
+    vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic)
+
+    output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif")
+    logging.info("Building raster mosaic: " + output_mosiac_raster)
+    logging.info("Using " + str(threads) + " threads for parallizing")
+    print("Note: This step can take a number of hours if processing 100s of hucs")
+    gdal.Translate(
+        output_mosiac_raster,
+        vrt,
+        xRes=10,
+        yRes=-10,
+        creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS=' + str(threads)],
+    )
+    vrt = None
+
+
+def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO):
     start_time = datetime.now()
     file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
     log_file_name = f"{log_file_name_key}-{file_dt_string}.log"
 
     log_file_path = os.path.join(output_folder_path, log_file_name)
+    print('Log file created here:' + str(log_file_path))
+
+    # Clear previous logging configuration
+    logging.getLogger().handlers = []
+
+    # Create a StreamHandler and set the level
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(log_level)
+
+    # Create a FileHandler and set the level
+    file_handler = logging.FileHandler(log_file_path)
+    file_handler.setLevel(log_level)
 
-    logging.basicConfig(filename=log_file_path, level=logging.DEBUG, format="%(message)s")
+    # Create a formatter and set the formatter for the handlers
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    console_handler.setFormatter(formatter)
+    file_handler.setFormatter(formatter)
 
-    # yes.. this can do console logs as well, but it can be a bit unstable and ugly
+    # Add the handlers to the logger
+    logger = logging.getLogger()
+    logger.setLevel(log_level)
+    logger.addHandler(console_handler)
+    logger.addHandler(file_handler)
 
-    logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
-    logging.info("----------------")
+    # Log the start time
+    logger.info(f'Started: {start_time.strftime("%m/%d/%Y %H:%M:%S")}')
+    logger.info("----------------")
 
 
 if __name__ == "__main__":

From 9ed73c6a91ac5668a15e3fadcfefa43027e3eeea Mon Sep 17 00:00:00 2001
From: Rob Gonzalez-Pita <rob.g.pita@noaa.gov>
Date: Fri, 8 Dec 2023 13:45:53 -0700
Subject: [PATCH 290/359] v4.4.8.0 Copy files to scratch directory for
 processing, and remove upon completion (#1045)

---
 Pipfile.lock           | 340 +++++++++++++++++++++--------------------
 config/deny_unit.lst   |   4 +
 docs/CHANGELOG.md      |  13 ++
 src/bash_variables.env |  48 +++---
 src/run_unit_wb.sh     |  15 +-
 5 files changed, 223 insertions(+), 197 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 61853dec6..a0e867a67 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "196cd7e33d0458f7b032017fcd2e1940761158c53a4130adb8915f036ed61b49"
+            "sha256": "05bec99d8be9c133d86cc5a13bbb8dd751b89eaa4368df49170d3527a84bb127"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -42,11 +42,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f",
-                "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"
+                "sha256:56a415fbc462291813a94528a779597226619c8e78af7de0507333f700011e5f",
+                "sha256:5a0bec7085176715be77df87fc66d6c9d70626bd752fcc85f57cdbee5b3760da"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.0.0"
+            "version": "==4.1.0"
         },
         "appdirs": {
             "hashes": [
@@ -133,7 +133,7 @@
                 "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
                 "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==4.12.2"
         },
         "black": {
@@ -773,11 +773,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9",
-                "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"
+                "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14",
+                "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.1.3"
+            "version": "==1.2.0"
         },
         "executing": {
             "hashes": [
@@ -789,10 +789,10 @@
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:06dc8680d937628e993fa0cd278f196d20449a1adc087640710846b324d422ea",
-                "sha256:aec6a19e9f66e9810ab371cc913ad5f4e9e479b63a7072a2cd060a9369e329a8"
+                "sha256:b9fd1a2dd6971dbc7fee280a95bd199ae0dd9ce22beb91cc75e9c1c528a5170e",
+                "sha256:e25df6647e1bc4a26070b700897b07b542ec898dd4f1f6ea013e7f6a88417225"
             ],
-            "version": "==2.18.1"
+            "version": "==2.19.0"
         },
         "filelock": {
             "hashes": [
@@ -963,19 +963,19 @@
         },
         "idna": {
             "hashes": [
-                "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
-                "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
+                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
+                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.4"
+            "version": "==3.6"
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb",
-                "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"
+                "sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7",
+                "sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67"
             ],
             "markers": "python_version < '3.9'",
-            "version": "==6.8.0"
+            "version": "==7.0.0"
         },
         "importlib-resources": {
             "hashes": [
@@ -1157,19 +1157,19 @@
                 "format-nongpl"
             ],
             "hashes": [
-                "sha256:c9ff4d7447eed9592c23a12ccee508baf0dd0d59650615e847feb6cdca74f392",
-                "sha256:eee9e502c788e89cb166d4d37f43084e3b64ab405c795c03d343a4dbc2c810fc"
+                "sha256:4f614fd46d8d61258610998997743ec5492a648b33cf478c1ddc23ed4598a5fa",
+                "sha256:ed6231f0429ecf966f5bc8dfef245998220549cbbcf140f913b7464c52c3b6b3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.19.2"
+            "version": "==4.20.0"
         },
         "jsonschema-specifications": {
             "hashes": [
-                "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1",
-                "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"
+                "sha256:9472fc4fea474cd74bea4a2b190daeccb5a9e4db2ea80efcf7a1b582fc9a81b8",
+                "sha256:e74ba7c0a65e8cb49dc26837d6cfe576557084a8b423ed16a420984228104f93"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.7.1"
+            "version": "==2023.11.2"
         },
         "jupyter": {
             "hashes": [
@@ -1214,11 +1214,12 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:47b8f5e63440125cb1bb8957bf12b18453ee5ed9efe42d2f7b2ca66a7019a278",
-                "sha256:dde56c9bc3cb52d7b72cc0f696d15d7163603526f1a758eb4a27405b73eab2a5"
+                "sha256:0c548151b54bcb516ca466ec628f7f021545be137d01b5467877e87f6fff4374",
+                "sha256:0c99f9367b0f24141e527544522430176613f9249849be80504c6d2b955004bb"
             ],
+            "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==2.10.0"
+            "version": "==2.11.2"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1262,11 +1263,11 @@
         },
         "jupyterlab-pygments": {
             "hashes": [
-                "sha256:2405800db07c9f770863bcf8049a529c3dd4d3e28536638bd7c1c01d2748309f",
-                "sha256:7405d7fde60819d905a9fa8ce89e4cd830e318cdad22a0030f7a901da705585d"
+                "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d",
+                "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.2.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.3.0"
         },
         "jupyterlab-server": {
             "hashes": [
@@ -1710,16 +1711,16 @@
                 "sha256:4b28c207877cf33ef3a9838cdc7a54c5ceff981194a82eac59d558f05487295e",
                 "sha256:a3a1ddfb34d4a9d17fc744d655962714a866639acd30130e9be84191cd97cd15"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==0.9.0"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:abedc01cf543177ffde0bfc2a69726d5a478f6af10a332fc1bf29fcb4f0cf000",
-                "sha256:d1d417b7f34a4e38887f8da5bdfd12372adf3b80f995d57556cb0972c68909fe"
+                "sha256:5b6c848194d270cc55fb691169202620d7b52a12fec259508d142ecbe4219310",
+                "sha256:b1564bd89f69a74cd6398b0362da94db07aafb991b7857216a766204a71612c0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.11.0"
+            "version": "==7.12.0"
         },
         "nbformat": {
             "hashes": [
@@ -2093,11 +2094,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3",
-                "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"
+                "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380",
+                "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.11.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.1.0"
         },
         "pluggy": {
             "hashes": [
@@ -2117,11 +2118,11 @@
         },
         "prometheus-client": {
             "hashes": [
-                "sha256:35f7a8c22139e2bb7ca5a698e92d38145bc8dc74c1c0bf56f25cca886a764e17",
-                "sha256:8de3ae2755f890826f4b6479e5571d4f74ac17a81345fe69a6778fdb92579184"
+                "sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1",
+                "sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.18.0"
+            "version": "==0.19.0"
         },
         "prompt-toolkit": {
             "hashes": [
@@ -2226,6 +2227,7 @@
                 "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35",
                 "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"
             ],
+            "markers": "os_name != 'nt'",
             "version": "==0.7.0"
         },
         "pure-eval": {
@@ -2453,11 +2455,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
-                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
+                "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
+                "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.16.1"
+            "version": "==2.17.2"
         },
         "pyparsing": {
             "hashes": [
@@ -2946,11 +2948,11 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf",
-                "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0"
+                "sha256:81a1471c68c9d5e3831c30ad1dd9815c45b558e596653db751a2bfdd17b3b9ec",
+                "sha256:c19c4d006f1757e3dd75c4f784d38f8698d87b649c54f9ace14e5e8c9667c01d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.30.2"
+            "version": "==0.31.1"
         },
         "requests": {
             "hashes": [
@@ -3013,108 +3015,108 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:0525847f83f506aa1e28eb2057b696fe38217e12931c8b1b02198cfe6975e142",
-                "sha256:05942656cb2cb4989cd50ced52df16be94d344eae5097e8583966a1d27da73a5",
-                "sha256:0831d3ecdea22e4559cc1793f22e77067c9d8c451d55ae6a75bf1d116a8e7f42",
-                "sha256:0853da3d5e9bc6a07b2486054a410b7b03f34046c123c6561b535bb48cc509e1",
-                "sha256:08e6e7ff286254016b945e1ab632ee843e43d45e40683b66dd12b73791366dd1",
-                "sha256:0a38612d07a36138507d69646c470aedbfe2b75b43a4643f7bd8e51e52779624",
-                "sha256:0bedd91ae1dd142a4dc15970ed2c729ff6c73f33a40fa84ed0cdbf55de87c777",
-                "sha256:0c5441b7626c29dbd54a3f6f3713ec8e956b009f419ffdaaa3c80eaf98ddb523",
-                "sha256:0e9e976e0dbed4f51c56db10831c9623d0fd67aac02853fe5476262e5a22acb7",
-                "sha256:0fadfdda275c838cba5102c7f90a20f2abd7727bf8f4a2b654a5b617529c5c18",
-                "sha256:1096ca0bf2d3426cbe79d4ccc91dc5aaa73629b08ea2d8467375fad8447ce11a",
-                "sha256:171d9a159f1b2f42a42a64a985e4ba46fc7268c78299272ceba970743a67ee50",
-                "sha256:188912b22b6c8225f4c4ffa020a2baa6ad8fabb3c141a12dbe6edbb34e7f1425",
-                "sha256:1b4cf9ab9a0ae0cb122685209806d3f1dcb63b9fccdf1424fb42a129dc8c2faa",
-                "sha256:1e04581c6117ad9479b6cfae313e212fe0dfa226ac727755f0d539cd54792963",
-                "sha256:1fa73ed22c40a1bec98d7c93b5659cd35abcfa5a0a95ce876b91adbda170537c",
-                "sha256:2124f9e645a94ab7c853bc0a3644e0ca8ffbe5bb2d72db49aef8f9ec1c285733",
-                "sha256:240687b5be0f91fbde4936a329c9b7589d9259742766f74de575e1b2046575e4",
-                "sha256:25740fb56e8bd37692ed380e15ec734be44d7c71974d8993f452b4527814601e",
-                "sha256:27ccc93c7457ef890b0dd31564d2a05e1aca330623c942b7e818e9e7c2669ee4",
-                "sha256:281c8b219d4f4b3581b918b816764098d04964915b2f272d1476654143801aa2",
-                "sha256:2d34a5450a402b00d20aeb7632489ffa2556ca7b26f4a63c35f6fccae1977427",
-                "sha256:301bd744a1adaa2f6a5e06c98f1ac2b6f8dc31a5c23b838f862d65e32fca0d4b",
-                "sha256:30e5ce9f501fb1f970e4a59098028cf20676dee64fc496d55c33e04bbbee097d",
-                "sha256:33ab498f9ac30598b6406e2be1b45fd231195b83d948ebd4bd77f337cb6a2bff",
-                "sha256:35585a8cb5917161f42c2104567bb83a1d96194095fc54a543113ed5df9fa436",
-                "sha256:389c0e38358fdc4e38e9995e7291269a3aead7acfcf8942010ee7bc5baee091c",
-                "sha256:3acadbab8b59f63b87b518e09c4c64b142e7286b9ca7a208107d6f9f4c393c5c",
-                "sha256:3b7a64d43e2a1fa2dd46b678e00cabd9a49ebb123b339ce799204c44a593ae1c",
-                "sha256:3c8c0226c71bd0ce9892eaf6afa77ae8f43a3d9313124a03df0b389c01f832de",
-                "sha256:429349a510da82c85431f0f3e66212d83efe9fd2850f50f339341b6532c62fe4",
-                "sha256:466030a42724780794dea71eb32db83cc51214d66ab3fb3156edd88b9c8f0d78",
-                "sha256:47aeceb4363851d17f63069318ba5721ae695d9da55d599b4d6fb31508595278",
-                "sha256:48aa98987d54a46e13e6954880056c204700c65616af4395d1f0639eba11764b",
-                "sha256:4b2416ed743ec5debcf61e1242e012652a4348de14ecc7df3512da072b074440",
-                "sha256:4d0a675a7acbbc16179188d8c6d0afb8628604fc1241faf41007255957335a0b",
-                "sha256:4eb74d44776b0fb0782560ea84d986dffec8ddd94947f383eba2284b0f32e35e",
-                "sha256:4f8a1d990dc198a6c68ec3d9a637ba1ce489b38cbfb65440a27901afbc5df575",
-                "sha256:513ccbf7420c30e283c25c82d5a8f439d625a838d3ba69e79a110c260c46813f",
-                "sha256:5210a0018c7e09c75fa788648617ebba861ae242944111d3079034e14498223f",
-                "sha256:54cdfcda59251b9c2f87a05d038c2ae02121219a04d4a1e6fc345794295bdc07",
-                "sha256:56dd500411d03c5e9927a1eb55621e906837a83b02350a9dc401247d0353717c",
-                "sha256:57ec6baec231bb19bb5fd5fc7bae21231860a1605174b11585660236627e390e",
-                "sha256:5f1519b080d8ce0a814f17ad9fb49fb3a1d4d7ce5891f5c85fc38631ca3a8dc4",
-                "sha256:6174d6ad6b58a6bcf67afbbf1723420a53d06c4b89f4c50763d6fa0a6ac9afd2",
-                "sha256:68172622a5a57deb079a2c78511c40f91193548e8ab342c31e8cb0764d362459",
-                "sha256:6915fc9fa6b3ec3569566832e1bb03bd801c12cea030200e68663b9a87974e76",
-                "sha256:6b75b912a0baa033350367a8a07a8b2d44fd5b90c890bfbd063a8a5f945f644b",
-                "sha256:6f5dcb658d597410bb7c967c1d24eaf9377b0d621358cbe9d2ff804e5dd12e81",
-                "sha256:6f8d7fe73d1816eeb5378409adc658f9525ecbfaf9e1ede1e2d67a338b0c7348",
-                "sha256:7036316cc26b93e401cedd781a579be606dad174829e6ad9e9c5a0da6e036f80",
-                "sha256:7188ddc1a8887194f984fa4110d5a3d5b9b5cd35f6bafdff1b649049cbc0ce29",
-                "sha256:761531076df51309075133a6bc1db02d98ec7f66e22b064b1d513bc909f29743",
-                "sha256:7979d90ee2190d000129598c2b0c82f13053dba432b94e45e68253b09bb1f0f6",
-                "sha256:8015835494b21aa7abd3b43fdea0614ee35ef6b03db7ecba9beb58eadf01c24f",
-                "sha256:81c4d1a3a564775c44732b94135d06e33417e829ff25226c164664f4a1046213",
-                "sha256:81cf9d306c04df1b45971c13167dc3bad625808aa01281d55f3cf852dde0e206",
-                "sha256:88857060b690a57d2ea8569bca58758143c8faa4639fb17d745ce60ff84c867e",
-                "sha256:8c567c664fc2f44130a20edac73e0a867f8e012bf7370276f15c6adc3586c37c",
-                "sha256:91bd2b7cf0f4d252eec8b7046fa6a43cee17e8acdfc00eaa8b3dbf2f9a59d061",
-                "sha256:9620650c364c01ed5b497dcae7c3d4b948daeae6e1883ae185fef1c927b6b534",
-                "sha256:9b007c2444705a2dc4a525964fd4dd28c3320b19b3410da6517cab28716f27d3",
-                "sha256:9bf9acce44e967a5103fcd820fc7580c7b0ab8583eec4e2051aec560f7b31a63",
-                "sha256:a239303acb0315091d54c7ff36712dba24554993b9a93941cf301391d8a997ee",
-                "sha256:a2baa6be130e8a00b6cbb9f18a33611ec150b4537f8563bddadb54c1b74b8193",
-                "sha256:a54917b7e9cd3a67e429a630e237a90b096e0ba18897bfb99ee8bd1068a5fea0",
-                "sha256:a689e1ded7137552bea36305a7a16ad2b40be511740b80748d3140614993db98",
-                "sha256:a952ae3eb460c6712388ac2ec706d24b0e651b9396d90c9a9e0a69eb27737fdc",
-                "sha256:aa32205358a76bf578854bf31698a86dc8b2cb591fd1d79a833283f4a403f04b",
-                "sha256:b2287c09482949e0ca0c0eb68b2aca6cf57f8af8c6dfd29dcd3bc45f17b57978",
-                "sha256:b6b0e17d39d21698185097652c611f9cf30f7c56ccec189789920e3e7f1cee56",
-                "sha256:b710bf7e7ae61957d5c4026b486be593ed3ec3dca3e5be15e0f6d8cf5d0a4990",
-                "sha256:b8e11715178f3608874508f08e990d3771e0b8c66c73eb4e183038d600a9b274",
-                "sha256:b92aafcfab3d41580d54aca35a8057341f1cfc7c9af9e8bdfc652f83a20ced31",
-                "sha256:bec29b801b4adbf388314c0d050e851d53762ab424af22657021ce4b6eb41543",
-                "sha256:c694bee70ece3b232df4678448fdda245fd3b1bb4ba481fb6cd20e13bb784c46",
-                "sha256:c6b52b7028b547866c2413f614ee306c2d4eafdd444b1ff656bf3295bf1484aa",
-                "sha256:cb41ad20064e18a900dd427d7cf41cfaec83bcd1184001f3d91a1f76b3fcea4e",
-                "sha256:cd316dbcc74c76266ba94eb021b0cc090b97cca122f50bd7a845f587ff4bf03f",
-                "sha256:ced40cdbb6dd47a032725a038896cceae9ce267d340f59508b23537f05455431",
-                "sha256:d1c562a9bb72244fa767d1c1ab55ca1d92dd5f7c4d77878fee5483a22ffac808",
-                "sha256:d389ff1e95b6e46ebedccf7fd1fadd10559add595ac6a7c2ea730268325f832c",
-                "sha256:d56b1cd606ba4cedd64bb43479d56580e147c6ef3f5d1c5e64203a1adab784a2",
-                "sha256:d72a4315514e5a0b9837a086cb433b004eea630afb0cc129de76d77654a9606f",
-                "sha256:d9e7f29c00577aff6b318681e730a519b235af292732a149337f6aaa4d1c5e31",
-                "sha256:dbc25baa6abb205766fb8606f8263b02c3503a55957fcb4576a6bb0a59d37d10",
-                "sha256:e57919c32ee295a2fca458bb73e4b20b05c115627f96f95a10f9f5acbd61172d",
-                "sha256:e5bbe011a2cea9060fef1bb3d668a2fd8432b8888e6d92e74c9c794d3c101595",
-                "sha256:e6aea5c0eb5b0faf52c7b5c4a47c8bb64437173be97227c819ffa31801fa4e34",
-                "sha256:e888be685fa42d8b8a3d3911d5604d14db87538aa7d0b29b1a7ea80d354c732d",
-                "sha256:eebaf8c76c39604d52852366249ab807fe6f7a3ffb0dd5484b9944917244cdbe",
-                "sha256:efbe0b5e0fd078ed7b005faa0170da4f72666360f66f0bb2d7f73526ecfd99f9",
-                "sha256:efddca2d02254a52078c35cadad34762adbae3ff01c6b0c7787b59d038b63e0d",
-                "sha256:f05450fa1cd7c525c0b9d1a7916e595d3041ac0afbed2ff6926e5afb6a781b7f",
-                "sha256:f12d69d568f5647ec503b64932874dade5a20255736c89936bf690951a5e79f5",
-                "sha256:f45321224144c25a62052035ce96cbcf264667bcb0d81823b1bbc22c4addd194",
-                "sha256:f62581d7e884dd01ee1707b7c21148f61f2febb7de092ae2f108743fcbef5985",
-                "sha256:f8832a4f83d4782a8f5a7b831c47e8ffe164e43c2c148c8160ed9a6d630bc02a",
-                "sha256:fa35ad36440aaf1ac8332b4a4a433d4acd28f1613f0d480995f5cfd3580e90b7"
+                "sha256:06d218e4464d31301e943b65b2c6919318ea6f69703a351961e1baaf60347276",
+                "sha256:12ecf89bd54734c3c2c79898ae2021dca42750c7bcfb67f8fb3315453738ac8f",
+                "sha256:15253fff410873ebf3cfba1cc686a37711efcd9b8cb30ea21bb14a973e393f60",
+                "sha256:188435794405c7f0573311747c85a96b63c954a5f2111b1df8018979eca0f2f0",
+                "sha256:1ceebd0ae4f3e9b2b6b553b51971921853ae4eebf3f54086be0565d59291e53d",
+                "sha256:244e173bb6d8f3b2f0c4d7370a1aa341f35da3e57ffd1798e5b2917b91731fd3",
+                "sha256:25b28b3d33ec0a78e944aaaed7e5e2a94ac811bcd68b557ca48a0c30f87497d2",
+                "sha256:25ea41635d22b2eb6326f58e608550e55d01df51b8a580ea7e75396bafbb28e9",
+                "sha256:29d311e44dd16d2434d5506d57ef4d7036544fc3c25c14b6992ef41f541b10fb",
+                "sha256:2a1472956c5bcc49fb0252b965239bffe801acc9394f8b7c1014ae9258e4572b",
+                "sha256:2a7bef6977043673750a88da064fd513f89505111014b4e00fbdd13329cd4e9a",
+                "sha256:2ac26f50736324beb0282c819668328d53fc38543fa61eeea2c32ea8ea6eab8d",
+                "sha256:2e72f750048b32d39e87fc85c225c50b2a6715034848dbb196bf3348aa761fa1",
+                "sha256:31e220a040b89a01505128c2f8a59ee74732f666439a03e65ccbf3824cdddae7",
+                "sha256:35f53c76a712e323c779ca39b9a81b13f219a8e3bc15f106ed1e1462d56fcfe9",
+                "sha256:38d4f822ee2f338febcc85aaa2547eb5ba31ba6ff68d10b8ec988929d23bb6b4",
+                "sha256:38f9bf2ad754b4a45b8210a6c732fe876b8a14e14d5992a8c4b7c1ef78740f53",
+                "sha256:3a44c8440183b43167fd1a0819e8356692bf5db1ad14ce140dbd40a1485f2dea",
+                "sha256:3ab96754d23372009638a402a1ed12a27711598dd49d8316a22597141962fe66",
+                "sha256:3c55d7f2d817183d43220738270efd3ce4e7a7b7cbdaefa6d551ed3d6ed89190",
+                "sha256:46e1ed994a0920f350a4547a38471217eb86f57377e9314fbaaa329b71b7dfe3",
+                "sha256:4a5375c5fff13f209527cd886dc75394f040c7d1ecad0a2cb0627f13ebe78a12",
+                "sha256:4c2d26aa03d877c9730bf005621c92da263523a1e99247590abbbe252ccb7824",
+                "sha256:4c4e314d36d4f31236a545696a480aa04ea170a0b021e9a59ab1ed94d4c3ef27",
+                "sha256:4d0c10d803549427f427085ed7aebc39832f6e818a011dcd8785e9c6a1ba9b3e",
+                "sha256:4dcc5ee1d0275cb78d443fdebd0241e58772a354a6d518b1d7af1580bbd2c4e8",
+                "sha256:51967a67ea0d7b9b5cd86036878e2d82c0b6183616961c26d825b8c994d4f2c8",
+                "sha256:530190eb0cd778363bbb7596612ded0bb9fef662daa98e9d92a0419ab27ae914",
+                "sha256:5379e49d7e80dca9811b36894493d1c1ecb4c57de05c36f5d0dd09982af20211",
+                "sha256:5493569f861fb7b05af6d048d00d773c6162415ae521b7010197c98810a14cab",
+                "sha256:5a4c1058cdae6237d97af272b326e5f78ee7ee3bbffa6b24b09db4d828810468",
+                "sha256:5d75d6d220d55cdced2f32cc22f599475dbe881229aeddba6c79c2e9df35a2b3",
+                "sha256:5d97e9ae94fb96df1ee3cb09ca376c34e8a122f36927230f4c8a97f469994bff",
+                "sha256:5feae2f9aa7270e2c071f488fab256d768e88e01b958f123a690f1cc3061a09c",
+                "sha256:603d5868f7419081d616dab7ac3cfa285296735e7350f7b1e4f548f6f953ee7d",
+                "sha256:61d42d2b08430854485135504f672c14d4fc644dd243a9c17e7c4e0faf5ed07e",
+                "sha256:61dbc1e01dc0c5875da2f7ae36d6e918dc1b8d2ce04e871793976594aad8a57a",
+                "sha256:65cfed9c807c27dee76407e8bb29e6f4e391e436774bcc769a037ff25ad8646e",
+                "sha256:67a429520e97621a763cf9b3ba27574779c4e96e49a27ff8a1aa99ee70beb28a",
+                "sha256:6aadae3042f8e6db3376d9e91f194c606c9a45273c170621d46128f35aef7cd0",
+                "sha256:6ba8858933f0c1a979781272a5f65646fca8c18c93c99c6ddb5513ad96fa54b1",
+                "sha256:6bc568b05e02cd612be53900c88aaa55012e744930ba2eeb56279db4c6676eb3",
+                "sha256:729408136ef8d45a28ee9a7411917c9e3459cf266c7e23c2f7d4bb8ef9e0da42",
+                "sha256:751758d9dd04d548ec679224cc00e3591f5ebf1ff159ed0d4aba6a0746352452",
+                "sha256:76d59d4d451ba77f08cb4cd9268dec07be5bc65f73666302dbb5061989b17198",
+                "sha256:79bf58c08f0756adba691d480b5a20e4ad23f33e1ae121584cf3a21717c36dfa",
+                "sha256:7de12b69d95072394998c622cfd7e8cea8f560db5fca6a62a148f902a1029f8b",
+                "sha256:7f55cd9cf1564b7b03f238e4c017ca4794c05b01a783e9291065cb2858d86ce4",
+                "sha256:80e5acb81cb49fd9f2d5c08f8b74ffff14ee73b10ca88297ab4619e946bcb1e1",
+                "sha256:87a90f5545fd61f6964e65eebde4dc3fa8660bb7d87adb01d4cf17e0a2b484ad",
+                "sha256:881df98f0a8404d32b6de0fd33e91c1b90ed1516a80d4d6dc69d414b8850474c",
+                "sha256:8a776a29b77fe0cc28fedfd87277b0d0f7aa930174b7e504d764e0b43a05f381",
+                "sha256:8c2a61c0e4811012b0ba9f6cdcb4437865df5d29eab5d6018ba13cee1c3064a0",
+                "sha256:8fa6bd071ec6d90f6e7baa66ae25820d57a8ab1b0a3c6d3edf1834d4b26fafa2",
+                "sha256:96f2975fb14f39c5fe75203f33dd3010fe37d1c4e33177feef1107b5ced750e3",
+                "sha256:96fb0899bb2ab353f42e5374c8f0789f54e0a94ef2f02b9ac7149c56622eaf31",
+                "sha256:97163a1ab265a1073a6372eca9f4eeb9f8c6327457a0b22ddfc4a17dcd613e74",
+                "sha256:9c95a1a290f9acf7a8f2ebbdd183e99215d491beea52d61aa2a7a7d2c618ddc6",
+                "sha256:9d94d78418203904730585efa71002286ac4c8ac0689d0eb61e3c465f9e608ff",
+                "sha256:a6ba2cb7d676e9415b9e9ac7e2aae401dc1b1e666943d1f7bc66223d3d73467b",
+                "sha256:aa0379c1935c44053c98826bc99ac95f3a5355675a297ac9ce0dfad0ce2d50ca",
+                "sha256:ac96d67b37f28e4b6ecf507c3405f52a40658c0a806dffde624a8fcb0314d5fd",
+                "sha256:ade2ccb937060c299ab0dfb2dea3d2ddf7e098ed63ee3d651ebfc2c8d1e8632a",
+                "sha256:aefbdc934115d2f9278f153952003ac52cd2650e7313750390b334518c589568",
+                "sha256:b07501b720cf060c5856f7b5626e75b8e353b5f98b9b354a21eb4bfa47e421b1",
+                "sha256:b5267feb19070bef34b8dea27e2b504ebd9d31748e3ecacb3a4101da6fcb255c",
+                "sha256:b5f6328e8e2ae8238fc767703ab7b95785521c42bb2b8790984e3477d7fa71ad",
+                "sha256:b8996ffb60c69f677245f5abdbcc623e9442bcc91ed81b6cd6187129ad1fa3e7",
+                "sha256:b981a370f8f41c4024c170b42fbe9e691ae2dbc19d1d99151a69e2c84a0d194d",
+                "sha256:b9d121be0217787a7d59a5c6195b0842d3f701007333426e5154bf72346aa658",
+                "sha256:bcef4f2d3dc603150421de85c916da19471f24d838c3c62a4f04c1eb511642c1",
+                "sha256:bed0252c85e21cf73d2d033643c945b460d6a02fc4a7d644e3b2d6f5f2956c64",
+                "sha256:bfdfbe6a36bc3059fff845d64c42f2644cf875c65f5005db54f90cdfdf1df815",
+                "sha256:c0095b8aa3e432e32d372e9a7737e65b58d5ed23b9620fea7cb81f17672f1fa1",
+                "sha256:c1f41d32a2ddc5a94df4b829b395916a4b7f103350fa76ba6de625fcb9e773ac",
+                "sha256:c45008ca79bad237cbc03c72bc5205e8c6f66403773929b1b50f7d84ef9e4d07",
+                "sha256:c82bbf7e03748417c3a88c1b0b291288ce3e4887a795a3addaa7a1cfd9e7153e",
+                "sha256:c918621ee0a3d1fe61c313f2489464f2ae3d13633e60f520a8002a5e910982ee",
+                "sha256:d204957169f0b3511fb95395a9da7d4490fb361763a9f8b32b345a7fe119cb45",
+                "sha256:d329896c40d9e1e5c7715c98529e4a188a1f2df51212fd65102b32465612b5dc",
+                "sha256:d3a61e928feddc458a55110f42f626a2a20bea942ccedb6fb4cee70b4830ed41",
+                "sha256:d48db29bd47814671afdd76c7652aefacc25cf96aad6daefa82d738ee87461e2",
+                "sha256:d5593855b5b2b73dd8413c3fdfa5d95b99d657658f947ba2c4318591e745d083",
+                "sha256:d79c159adea0f1f4617f54aa156568ac69968f9ef4d1e5fefffc0a180830308e",
+                "sha256:db09b98c7540df69d4b47218da3fbd7cb466db0fb932e971c321f1c76f155266",
+                "sha256:ddf23960cb42b69bce13045d5bc66f18c7d53774c66c13f24cf1b9c144ba3141",
+                "sha256:e06cfea0ece444571d24c18ed465bc93afb8c8d8d74422eb7026662f3d3f779b",
+                "sha256:e7c564c58cf8f248fe859a4f0fe501b050663f3d7fbc342172f259124fb59933",
+                "sha256:e86593bf8637659e6a6ed58854b6c87ec4e9e45ee8a4adfd936831cef55c2d21",
+                "sha256:eaffbd8814bb1b5dc3ea156a4c5928081ba50419f9175f4fc95269e040eff8f0",
+                "sha256:ee353bb51f648924926ed05e0122b6a0b1ae709396a80eb583449d5d477fcdf7",
+                "sha256:ee6faebb265e28920a6f23a7d4c362414b3f4bb30607141d718b991669e49ddc",
+                "sha256:efe093acc43e869348f6f2224df7f452eab63a2c60a6c6cd6b50fd35c4e075ba",
+                "sha256:f03a1b3a4c03e3e0161642ac5367f08479ab29972ea0ffcd4fa18f729cd2be0a",
+                "sha256:f0d320e70b6b2300ff6029e234e79fe44e9dbbfc7b98597ba28e054bd6606a57",
+                "sha256:f252dfb4852a527987a9156cbcae3022a30f86c9d26f4f17b8c967d7580d65d2",
+                "sha256:f5f4424cb87a20b016bfdc157ff48757b89d2cc426256961643d443c6c277007",
+                "sha256:f8eae66a1304de7368932b42d801c67969fd090ddb1a7a24f27b435ed4bed68f",
+                "sha256:fdb82eb60d31b0c033a8e8ee9f3fc7dfbaa042211131c29da29aea8531b4f18f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.12.0"
+            "version": "==0.13.2"
         },
         "rtree": {
             "hashes": [
@@ -3437,11 +3439,11 @@
         },
         "terminado": {
             "hashes": [
-                "sha256:6ccbbcd3a4f8a25a5ec04991f39a0b8db52dfcd487ea0e578d977e6752380333",
-                "sha256:8650d44334eba354dd591129ca3124a6ba42c3d5b70df5051b6921d506fdaeae"
+                "sha256:1ea08a89b835dd1b8c0c900d92848147cef2537243361b2e3f4dc15df9b6fded",
+                "sha256:87b0d96642d0fe5f5abd7783857b9cab167f221a39ff98e3b9619a788a3c0f2e"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.17.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.18.0"
         },
         "texttable": {
             "hashes": [
@@ -3476,20 +3478,20 @@
         },
         "tornado": {
             "hashes": [
-                "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f",
-                "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5",
-                "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d",
-                "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3",
-                "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2",
-                "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a",
-                "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16",
-                "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a",
-                "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17",
-                "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0",
-                "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe"
+                "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0",
+                "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63",
+                "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263",
+                "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052",
+                "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f",
+                "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee",
+                "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78",
+                "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579",
+                "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212",
+                "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e",
+                "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.3.3"
+            "version": "==6.4"
         },
         "tqdm": {
             "hashes": [
@@ -3501,11 +3503,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:9b232b9430c8f57288c1024b34a8f0251ddcc47268927367a0dd3eeaca40deb5",
-                "sha256:baf991e61542da48fe8aef8b779a9ea0aa38d8a54166ee250d5af5ecf4486619"
+                "sha256:f14949d23829023013c47df20b4a76ccd1a85effb786dc060f34de7948361b33",
+                "sha256:fcdaa8ac49c04dfa0ed3ee3384ef6dfdb5d6f3741502be247279407679296772"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.13.0"
+            "version": "==5.14.0"
         },
         "typeguard": {
             "hashes": [
@@ -3591,11 +3593,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24",
-                "sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df"
+                "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6",
+                "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.6.4"
+            "version": "==1.7.0"
         },
         "whitebox": {
             "hashes": [
diff --git a/config/deny_unit.lst b/config/deny_unit.lst
index 9cf751d6a..f46240f89 100644
--- a/config/deny_unit.lst
+++ b/config/deny_unit.lst
@@ -35,3 +35,7 @@ nwm_headwaters.gpkg
 # wbd8_clp.gpkg
 wbd_buffered.gpkg
 wbd_buffered_streams.gpkg
+HUC6_dem_domain.gpkg
+usgs_gages.gpkg
+reformat_ras_rating_curve_points_rel_101.gpkg
+nws_lid.gpkg
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a1314d95f..8c4b441f5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.8.0 - 2023-12-08 - [PR#1045](https://github.com/NOAA-OWP/inundation-mapping/pull/1045)
+
+In order to avoid file system collisions on AWS, and keep the reads/writes from the same file on disk to a minimum, three files (`HUC6_dem_domain.gpkg`, `nws_lid.gpkg`, `reformat_ras_rating_curve_points_rel_101.gpkg`, & `usgs_gages.gpkg`) are now copied from disk into a scratch directory (temporary working directory), and removed after processing steps are completed.
+
+### Changes
+
+- `config`/`deny_unit.lst`: Add files to remove list - repetitive copies needed for processing step (`run_unit_wb.sh`)
+- `src`
+    - `bash_variables.env`: Add a new variable for the ras rating curve filename. It will be easier to track the filename in the `.env`, and pull into `run_unit_wb.sh`, rather than hardcode it.
+    - `run_unit_wb.sh`: Copy files and update references from `$inputsDir` to `$tempHucDataDir`.
+
+<br/><br/>
+
 ## v4.4.7.2 - 2023-12-08 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026)
 
 A couple of directly related issues were fixed in this PR.
diff --git a/src/bash_variables.env b/src/bash_variables.env
index fe348a63c..67b20ea57 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -1,43 +1,43 @@
 ## Define inputs
 
 # NOTE: $inputsDir is defined in Dockerfile
-export      DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
-export                       input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
-export                input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
-export             input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
-export                       input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
-export       input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg
-export input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
-export            input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg
-export                 input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
-export            input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
-export                 input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
-export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
-export          input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
-export                pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
-
-export                 bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
+export        DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
+export                         input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
+export                  input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
+export               input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
+export                         input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
+export         input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg
+export   input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
+export              input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg
+export                   input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
+export              input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
+export                   input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
+export                     input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
+export            input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
+export                  pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
+export                   bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
-export             bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
+export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
 
 # input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export                vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv
+export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv
 
 # input file location with nwm feature_id and recurrence flow values
-export                  nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
+export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
 
 # input file location with usgs rating curve database
-export           usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
+export             usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
 
 # input file locations for ras2fim locations and rating curve data
-export            ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
-export    ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
+export              ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
+export      ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
+export    ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points_rel_101.gpkg
 
-export                fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
+export                  fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
 
 # Input file location with HUC, nwm feature_id and manual calibration coefficients
-export                   man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv
+export                     man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv
 
 
 # Styling
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 701246900..6c17078fa 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -26,6 +26,13 @@ huc_start_time=`date +%s`
 echo -e $startDiv"Copying staged wbd and .gpkg files from $pre_clip_huc_dir/$hucNumber"
 cp -a $pre_clip_huc_dir/$hucNumber/. $tempHucDataDir
 
+# Copy necessary files from $inputsDir into $tempHucDataDir to avoid File System Collisions
+# For buffer_stream_branches.py
+cp $input_DEM_domain $tempHucDataDir
+# For usgs_gage_unit_setup.py
+cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir
+cp $ras_rating_curve_points_gpkg $tempHucDataDir
+cp $inputsDir/ahps_sites/nws_lid.gpkg $tempHucDataDir
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"
@@ -77,7 +84,7 @@ Tcount
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
 date -u
 Tstart
-$srcDir/buffer_stream_branches.py -a $input_DEM_domain \
+$srcDir/buffer_stream_branches.py -a $tempHucDataDir/HUC6_dem_domain.gpkg \
     -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
     -i $branch_id_attribute \
     -d $branch_buffer_distance_meters \
@@ -274,12 +281,12 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     date -u
     Tstart
     python3 $srcDir/usgs_gage_unit_setup.py \
-        -gages $inputsDir/usgs_gages/usgs_gages.gpkg \
+        -gages $tempHucDataDir/usgs_gages.gpkg \
         -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
-        -ras $ras_rating_curve_points_gpkg \
+        -ras $tempHucDataDir/$ras_rating_curve_gpkg_filename \
         -o $tempHucDataDir/usgs_subset_gages.gpkg \
         -huc $hucNumber \
-        -ahps $inputsDir/ahps_sites/nws_lid.gpkg \
+        -ahps $tempHucDataDir/nws_lid.gpkg \
         -bzero_id $branch_zero_id
     Tcount
 fi

From 79f8a82e20b888fb6291eac80921c185750703ea Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 8 Dec 2023 13:51:01 -0700
Subject: [PATCH 291/359] v4.4.8.1 Upgrade JDK to v.17 (#1047)

---
 Dockerfile        |  4 ++--
 docs/CHANGELOG.md | 10 ++++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 101fcb2e0..512e726dd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,7 @@ RUN git clone https://github.com/dtarb/taudem.git
 RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
 
 RUN apt-get update --fix-missing && apt-get install -y cmake mpich \
-    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/*
+    libgtest-dev libboost-test-dev libnetcdf-dev openjdk-17-jdk && rm -rf /var/lib/apt/lists/*
 
 ## Compile Main taudem repo ##
 RUN mkdir -p taudem/bin
@@ -71,7 +71,7 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 RUN apt update --fix-missing
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync tzdata
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync tzdata openjdk-17-jdk
 
 RUN apt auto-remove
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8c4b441f5..b62378506 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.8.1 - 2023-12-08 - [PR#1047](https://github.com/NOAA-OWP/inundation-mapping/pull/1047)
+
+Upgrades JDK to v.17.0.9 in Docker image to address security vulnerabilities.
+
+### Changes
+
+- `Dockerfile`: Upgrades JDK to v.17.
+
+<br/><br/>
+
 ## v4.4.8.0 - 2023-12-08 - [PR#1045](https://github.com/NOAA-OWP/inundation-mapping/pull/1045)
 
 In order to avoid file system collisions on AWS, and keep the reads/writes from the same file on disk to a minimum, three files (`HUC6_dem_domain.gpkg`, `nws_lid.gpkg`, `reformat_ras_rating_curve_points_rel_101.gpkg`, & `usgs_gages.gpkg`) are now copied from disk into a scratch directory (temporary working directory), and removed after processing steps are completed.

From 5cca27c5f8c63482e5aeb9c157e7435fb635d781 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Wed, 13 Dec 2023 08:08:37 -0600
Subject: [PATCH 292/359] v4.4.8.2 Hotfix for a switched global manning file
 (#1052)

---
 docs/CHANGELOG.md      | 10 ++++++++++
 src/bash_variables.env |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b62378506..45689b40b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.8.2 - 2023-12-12 - [PR#1052](https://github.com/NOAA-OWP/inundation-mapping/pull/1052)
+
+The alpha test for v4.4.8.1 came back with a large degradation in skill and we noticed that the global manning's roughness file was changed in v4.4.7.1 - likely in error.
+
+### Changes
+
+- `src`/`bash_variables.env`: changed the global roughness file to `${inputsDir}/rating_curve/variable_roughness/mannings_global_06_12.csv`
+
+<br/><br/>
+
 ## v4.4.8.1 - 2023-12-08 - [PR#1047](https://github.com/NOAA-OWP/inundation-mapping/pull/1047)
 
 Upgrades JDK to v.17.0.9 in Docker image to address security vulnerabilities.
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 67b20ea57..a853a78eb 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -21,7 +21,7 @@ export                   bathymetry_file=${inputsDir}/bathymetry/bathymetry_adju
 export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
 
 # input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv
+export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_06_12.csv
 
 # input file location with nwm feature_id and recurrence flow values
 export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv

From 4b2762d1578068938584c8e05b75cce050369f71 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 5 Jan 2024 10:31:16 -0700
Subject: [PATCH 293/359] v4.4.8.3 Fix erroneous branch inundation in
 levee-protected areas (#1059)

---
 docs/CHANGELOG.md                        | 14 ++++
 src/delineate_hydros_and_produce_HAND.sh |  1 +
 src/mask_dem.py                          | 89 +++++++++++++++++-------
 3 files changed, 80 insertions(+), 24 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 45689b40b..7fb3bd956 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.8.3 - 2024-01-05 - [PR#1059](https://github.com/NOAA-OWP/inundation-mapping/pull/1059)
+
+Fixes erroneous branch inundation in levee-protected areas.
+
+Levees disrupt the natural hydrology and can create large catchments that contain low-lying areas in levee-protected areas that are subject to being inundated in the REM (HAND) grid. However, these low-lying areas are hydrologically disconnected from the stream associated with the catchment and can be erroneously inundated. Branch inundation in levee-protected areas is now confined to the catchment for the levelpath.
+
+### Changes
+
+- `src/`
+    - `delineate_hydros_and_produce_HAND.sh`: Adds input argument for catchments.
+    - `mask_dem.py`: Adds DEM masking for areas of levee-protected areas that are not in the levelpath catchment.
+
+<br/><br/>
+
 ## v4.4.8.2 - 2023-12-12 - [PR#1052](https://github.com/NOAA-OWP/inundation-mapping/pull/1052)
 
 The alpha test for v4.4.8.1 came back with a large degradation in skill and we noticed that the global manning's roughness file was changed in v4.4.7.1 - likely in error.
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 2b04da494..c8c469858 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -23,6 +23,7 @@ if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $tempHucDataDir/LeveeProtecte
     python3 $srcDir/mask_dem.py \
         -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
         -nld $tempHucDataDir/LeveeProtectedAreas_subset.gpkg \
+        -catchments $z_arg \
         -out $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
         -b $branch_id_attribute \
         -i $current_branch_id \
diff --git a/src/mask_dem.py b/src/mask_dem.py
index a303eb0c2..bf8897ec7 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -5,6 +5,7 @@
 
 import fiona
 import geopandas as gpd
+import numpy as np
 import pandas as pd
 import rasterio as rio
 from rasterio.mask import mask
@@ -14,6 +15,7 @@ def mask_dem(
     dem_filename: str,
     nld_filename: str,
     levee_id_attribute: str,
+    catchments_filename: str,
     out_dem_filename: str,
     branch_id_attribute: str,
     branch_id: int,
@@ -22,7 +24,8 @@ def mask_dem(
 ):
     """
     Masks levee-protected areas from DEM in branch 0 or if the level path is associated with a levee
-    (determined in src/associate_levelpaths_with_levees.py).
+    (determined in src/associate_levelpaths_with_levees.py). Also masks parts of levee-protected areas
+    through which level paths flow that are not in the level path catchment.
 
     Parameters
     ----------
@@ -41,35 +44,39 @@ def mask_dem(
     branch_zero_id: int
         Branch 0 ID number
     levee_levelpaths: str
-        Path to levee-levelpath association file.
+        Path to levee-levelpath association file (generated by src/associate_levelpaths_with_levees.py)
     """
 
-    # Rasterize if branch zero
-    if branch_id == branch_zero_id:
-        with rio.open(dem_filename) as dem, fiona.open(nld_filename) as leveed:
-            dem_profile = dem.profile.copy()
+    assert os.path.exists(dem_filename), f"DEM file {dem_filename} does not exist"
+    assert os.path.exists(nld_filename), f"NLD file {nld_filename} does not exist"
+    assert os.path.exists(catchments_filename), f"Catchments file {catchments_filename} does not exist"
 
-            geoms = [feature["geometry"] for feature in leveed]
+    dem_masked = None
 
-            # Mask out levee-protected areas from DEM
-            out_dem_masked, _ = mask(dem, geoms, invert=True)
+    with rio.open(dem_filename) as dem:
+        dem_profile = dem.profile.copy()
+        nodata = dem.nodata
 
-            with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
-                dest.write(out_dem_masked[0, :, :], indexes=1)
+        if branch_id == branch_zero_id:
+            # Mask if branch zero
+            with fiona.open(nld_filename) as leveed:
+                geoms = [feature["geometry"] for feature in leveed]
 
-    elif os.path.exists(levee_levelpaths):
-        levee_levelpaths = pd.read_csv(levee_levelpaths)
+            if len(geoms) > 0:
+                dem_masked, _ = mask(dem, geoms, invert=True)
 
-        # Select levees associated with branch
-        levee_levelpaths = levee_levelpaths[levee_levelpaths[branch_id_attribute] == branch_id]
+        elif os.path.exists(levee_levelpaths):
+            # Mask levee-protected areas protected against level path
+            levee_levelpaths = pd.read_csv(levee_levelpaths)
 
-        # Get levee IDs
-        levelpath_levees = list(levee_levelpaths[levee_id_attribute])
+            # Select levees associated with branch
+            levee_levelpaths = levee_levelpaths[levee_levelpaths[branch_id_attribute] == branch_id]
 
-        if len(levelpath_levees) > 0:
-            with rio.open(dem_filename) as dem:
+            # Get levee IDs
+            levelpath_levees = list(levee_levelpaths[levee_id_attribute])
+
+            if len(levelpath_levees) > 0:
                 leveed = gpd.read_file(nld_filename)
-                dem_profile = dem.profile.copy()
 
                 # Get geometries of levee protected areas associated with levelpath
                 geoms = [
@@ -78,11 +85,40 @@ def mask_dem(
                     if feature[levee_id_attribute] in levelpath_levees
                 ]
 
-                # Mask out levee-protected areas from DEM
-                out_dem_masked, _ = mask(dem, geoms, invert=True)
+                if len(geoms) > 0:
+                    dem_masked, _ = mask(dem, geoms, invert=True)
+
+            # Mask levee-protected areas not protected against level path
+            catchments = gpd.read_file(catchments_filename)
+            leveed = gpd.read_file(nld_filename)
+
+            leveed_area_catchments = gpd.overlay(catchments, leveed, how="union")
+
+            # Select levee catchments not associated with level path
+            levee_catchments_to_mask = leveed_area_catchments.loc[
+                ~leveed_area_catchments[levee_id_attribute].isna() & leveed_area_catchments['ID'].isna(), :
+            ]
+
+            geoms = [feature["geometry"] for i, feature in levee_catchments_to_mask.iterrows()]
+
+            levee_catchments_masked = None
+            if len(geoms) > 0:
+                levee_catchments_masked, _ = mask(dem, geoms, invert=True)
 
+            out_masked = None
+            if dem_masked is None:
+                if levee_catchments_masked is not None:
+                    out_masked = levee_catchments_masked
+
+            else:
+                if levee_catchments_masked is None:
+                    out_masked = dem_masked
+                else:
+                    out_masked = np.where(levee_catchments_masked == nodata, nodata, dem_masked)
+
+            if out_masked is not None:
                 with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
-                    dest.write(out_dem_masked[0, :, :], indexes=1)
+                    dest.write(out_masked[0, :, :], indexes=1)
 
 
 if __name__ == '__main__':
@@ -91,8 +127,13 @@ def mask_dem(
     parser.add_argument(
         '-nld', '--nld-filename', help='NLD levee-protected areas filename', required=True, type=str
     )
+    parser.add_argument(
+        '-catchments', '--catchments-filename', help='NWM catchments filename', required=True, type=str
+    )
     parser.add_argument('-l', '--levee-id-attribute', help='Levee ID attribute name', required=True, type=str)
-    parser.add_argument('-out', '--out-dem-filename', help='out DEM filename', required=True, type=str)
+    parser.add_argument(
+        '-out', '--out-dem-filename', help='DEM filename to be written', required=True, type=str
+    )
     parser.add_argument(
         '-b', '--branch-id-attribute', help='Branch ID attribute name', required=True, type=str
     )

From 5478554f1ab4988d2c1330c5ebc6c6fc9d6d5b46 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 12 Jan 2024 10:44:35 -0700
Subject: [PATCH 294/359] v4.4.8.4 Tool to evaluate crosswalk accuracy (#1061)

---
 Dockerfile                               |   1 +
 docs/CHANGELOG.md                        |  22 ++
 fim_pipeline.sh                          |   1 +
 fim_post_processing.sh                   |   2 +-
 fim_pre_processing.sh                    |   5 +
 src/add_crosswalk.py                     |   6 +-
 src/delineate_hydros_and_produce_HAND.sh |  17 +-
 tools/evaluate_crosswalk.py              | 291 +++++++++++++++++++++++
 8 files changed, 342 insertions(+), 3 deletions(-)
 create mode 100644 tools/evaluate_crosswalk.py

diff --git a/Dockerfile b/Dockerfile
index 512e726dd..dd579c65e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,6 +54,7 @@ ARG depDir=/dependencies
 ENV inputsDir=$dataDir/inputs
 ENV outputsDir=/outputs
 ENV srcDir=$projectDir/src
+ENV toolsDir=$projectDir/tools
 ENV workDir=/fim_temp
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7fb3bd956..e514eb23c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,28 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.8.4 - 2024-01-12 - [PR#1061](https://github.com/NOAA-OWP/inundation-mapping/pull/1061)
+
+Adds a post-processing tool to compare crosswalked (conflated) `feature_id`s between NWM stream network to DEM-derived reaches. The tool is run if the `-x` flag is added to `fim_pipeline.sh`. Results are computed for branch 0 and saved in a summary file in the HUC output folder.
+
+### Additions
+
+- `tools/evaluate_crosswalk.py`: evaluates crosswalk accuracy using two methods:
+    - intersections: the number of intersections between streamlines
+    - network (or tree): compares the feature_ids of the immediate upstream segments
+
+### Changes
+
+- `Dockerfile`: added `toolsDir` environment variable
+- `fim_pipeline.sh`: added `-x` flag to run crosswalk evaluation tool
+- `fim_post_processing.sh`: changed hardcoded `/foss_fim/tools` to `toolsDir` environment variable
+- `fim_pre_processing.sh`: added `evaluateCrosswalk` environment variable
+- `src/`
+    - `add_crosswalk.py`: fix bug
+    - `delineate_hydros_and_produce_HAND.sh`: added a call to `verify_crosswalk.py` if evaluateCrosswalk is True.
+
+<br/><br/>
+
 ## v4.4.8.3 - 2024-01-05 - [PR#1059](https://github.com/NOAA-OWP/inundation-mapping/pull/1059)
 
 Fixes erroneous branch inundation in levee-protected areas.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 8523d21ef..4e1e76159 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -48,6 +48,7 @@ usage()
                             will be skipped.
       -isaws            : If this param is included, AWS objects will be used where possible
                         - Note: This feature is not yet implemented.
+      -x                : If this param is included, the crosswalk will be evaluated.
 
 
     Running 'fim_pipeline.sh' is a quicker process than running all three scripts independently; however,
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index f8266c94d..4ebb7b5fa 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -210,7 +210,7 @@ echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
 Tstart
-python3 /foss_fim/tools/combine_crosswalk_tables.py \
+python3 $toolsDir/combine_crosswalk_tables.py \
     -d $outputDestDir \
     -o $outputDestDir/crosswalk_table.csv
 Tcount
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 61eb3ef1a..cff84fa23 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -101,6 +101,9 @@ in
     -isaws)
         isAWS=1
         ;;
+    -x)
+        evaluateCrosswalk=1
+        ;;
     *) ;;
     esac
     shift
@@ -129,6 +132,7 @@ if [ "$jobBranchLimit" = "" ]; then jobBranchLimit=1; fi
 if [ -z "$overwrite" ]; then overwrite=0; fi
 if [ -z "$skipcal" ]; then skipcal=0; fi
 if [ -z "$isAWS" ]; then isAWS=0; fi
+if [ -z "$evaluateCrosswalk" ]; then evaluateCrosswalk=0; fi
 
 # validate and set defaults for the deny lists
 if [ "$deny_unit_list" = "" ]
@@ -234,6 +238,7 @@ echo "export deny_branch_zero_list=$deny_branch_zero_list" >> $args_file
 echo "export has_deny_branch_zero_override=$has_deny_branch_zero_override" >> $args_file
 echo "export isAWS=$isAWS" >> $args_file
 echo "export skipcal=$skipcal" >> $args_file
+echo "export evaluateCrosswalk=$evaluateCrosswalk" >> $args_file
 
 echo "--- Pre-processing is complete"
 
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index fd7c8c450..b10453f61 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -258,7 +258,11 @@ def add_crosswalk(
             else:
                 update_id = output_flows.loc[output_flows.HydroID == short_id]['HydroID'].item()
 
-            str_order = output_flows.loc[output_flows.HydroID == short_id]['order_'].item()
+            output_order = output_flows.loc[output_flows.HydroID == short_id]['order_']
+            if len(output_order) == 1:
+                str_order = output_order.item()
+            else:
+                str_order = output_order.max()
             sml_segs = pd.concat(
                 [
                     sml_segs,
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index c8c469858..b97d1912d 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -17,7 +17,7 @@ T_total_start
 
 ## MASK LEVEE-PROTECTED AREAS FROM DEM ##
 if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $tempHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
-    echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
+    echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $current_branch_id"
     date -u
     Tstart
     python3 $srcDir/mask_dem.py \
@@ -303,3 +303,18 @@ python3 $srcDir/add_crosswalk.py \
     -e $min_catchment_area \
     -g $min_stream_length
 Tcount
+
+## EVALUATE CROSSWALK ##
+if [ "$current_branch_id" = "$branch_zero_id" ] && [ "$evaluateCrosswalk" = "1" ] ; then
+    echo -e $startDiv"Evaluate crosswalk $hucNumber $current_branch_id"
+    date -u
+    Tstart
+    python3 $toolsDir/evaluate_crosswalk.py \
+        -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg \
+        -b $b_arg \
+        -c $tempHucDataDir/crosswalk_evaluation_$current_branch_id.csv \
+        -d $tempHucDataDir/nwm_headwater_points_subset.gpkg \
+        -u $hucNumber \
+        -z $current_branch_id
+    Tcount
+fi
diff --git a/tools/evaluate_crosswalk.py b/tools/evaluate_crosswalk.py
new file mode 100644
index 000000000..0051c3600
--- /dev/null
+++ b/tools/evaluate_crosswalk.py
@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+
+import argparse
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+
+
+def evaluate_crosswalk(
+    input_flows_fileName: str,
+    input_nwmflows_fileName: str,
+    input_nwm_headwaters_fileName: str,
+    output_table_fileName: str,
+    huc: str,
+    branch: str,
+):
+    """
+    Tool to check the accuracy of crosswalked attributes using two methods: counting the number of intersections between two stream representations and network, which checks the upstream and downstream connectivity of each stream segment.
+
+    Parameters
+    ----------
+    input_flows_fileName : str
+        Path to DEM derived streams
+    input_nwmflows_fileName : str
+        Path to subset NWM burnlines
+    input_nwm_headwaters_fileName : str
+        Path to subset NWM headwaters
+    output_table_fileName : str
+        Path to output table filename
+    huc : str
+        HUC ID
+    branch : str
+        Branch ID
+
+    Returns
+    -------
+    results : pandas.DataFrame
+
+    Usage
+    -----
+    python evaluate_crosswalk.py -a <input_flows_fileName> -b <input_nwmflows_fileName> -d <input_nwm_headwaters_fileName> -c <output_table_fileName> -u <huc> -z <branch>
+    """
+
+    intersections_results = _evaluate_crosswalk_intersections(input_flows_fileName, input_nwmflows_fileName)
+
+    intersections_correct = intersections_results['crosswalk'].sum()
+    intersections_total = len(intersections_results)
+    intersections_summary = intersections_correct / intersections_total
+
+    network_results = _evaluate_crosswalk_network(
+        input_flows_fileName, input_nwmflows_fileName, input_nwm_headwaters_fileName
+    )
+
+    network_results = network_results[network_results['status'] >= 0]
+    network_correct = len(network_results[network_results['status'] == 0])
+    network_total = len(network_results)
+    network_summary = network_correct / network_total
+
+    results = pd.DataFrame(
+        data={
+            'huc': [huc, huc],
+            'branch': [branch, branch],
+            'method': ['intersections', 'network'],
+            'correct': [intersections_correct, network_correct],
+            'total': [intersections_total, network_total],
+            'proportion': [intersections_summary, network_summary],
+        }
+    )
+
+    results.to_csv(output_table_fileName, index=False)
+
+    return results
+
+
+def _evaluate_crosswalk_intersections(input_flows_fileName: str, input_nwmflows_fileName: str):
+    """
+    Computes the number of intersections between the NWM and DEM-derived flowlines
+
+    Parameters
+    ----------
+    input_flows_fileName : str
+        Path to DEM derived streams
+    input_nwmflows_fileName : str
+        Path to subset NWM burnlines
+
+    Returns
+    -------
+    pandas.DataFrame
+    """
+
+    # Crosswalk check
+    # fh.vprint('Checking for crosswalks between NWM and DEM-derived flowlines', verbose)
+
+    flows = gpd.read_file(input_flows_fileName)
+    nwm_streams = gpd.read_file(input_nwmflows_fileName)
+
+    # Compute the number of intersections between the NWM and DEM-derived flowlines
+    streams = nwm_streams
+    xwalks = []
+    intersects = flows.sjoin(streams)
+
+    for idx in intersects.index:
+        flows_idx = intersects.loc[intersects.index == idx, 'HydroID'].unique()
+
+        if isinstance(intersects.loc[idx, 'ID'], np.int64):
+            streams_idxs = [intersects.loc[idx, 'ID']]
+        else:
+            streams_idxs = intersects.loc[idx, 'ID'].unique()
+
+        for flows_id in flows_idx:
+            for streams_idx in streams_idxs:
+                intersect = gpd.overlay(
+                    flows[flows['HydroID'] == flows_id],
+                    nwm_streams[nwm_streams['ID'] == streams_idx],
+                    keep_geom_type=False,
+                )
+
+                if len(intersect) == 0:
+                    intersect_points = 0
+                    feature_id = flows.loc[flows['HydroID'] == flows_id, 'feature_id']
+                elif intersect.geometry[0].geom_type == 'Point':
+                    intersect_points = 1
+                    feature_id = flows.loc[flows['HydroID'] == flows_id, 'feature_id']
+                else:
+                    intersect_points = len(intersect.geometry[0].geoms)
+                    feature_id = int(flows.loc[flows['HydroID'] == flows_id, 'feature_id'].iloc[0])
+
+                xwalks.append([flows_id, feature_id, streams_idx, intersect_points])
+
+    # Get the maximum number of intersections for each flowline
+    xwalks = pd.DataFrame(xwalks, columns=['HydroID', 'feature_id', 'ID', 'intersect_points'])
+    xwalks['feature_id'] = xwalks['feature_id'].astype(int)
+
+    xwalks['match'] = xwalks['feature_id'] == xwalks['ID']
+
+    xwalks_groupby = xwalks[['HydroID', 'intersect_points']].groupby('HydroID').max()
+
+    xwalks = xwalks.merge(xwalks_groupby, on='HydroID', how='left')
+    xwalks['max'] = xwalks['intersect_points_x'] == xwalks['intersect_points_y']
+
+    xwalks['crosswalk'] = xwalks['match'] == xwalks['max']
+
+    return xwalks
+
+
+def _evaluate_crosswalk_network(
+    input_flows_fileName: str, input_nwmflows_fileName: str, input_nwm_headwaters_fileName: str
+):
+    """
+    Compares the upstream and downstream connectivity of each stream segment
+
+    Parameters
+    ----------
+    input_flows_fileName : str
+        Path to DEM derived streams
+    input_nwmflows_fileName : str
+        Path to subset NWM burnlines
+    input_nwm_headwaters_fileName : str
+        Path to subset NWM headwaters
+    output_table_fileName : str
+        Path to output table filename
+
+    Returns
+    -------
+    pandas.DataFrame
+    """
+
+    # Check for crosswalks between NWM and DEM-derived flowlines
+    # fh.vprint('Checking for crosswalks between NWM and DEM-derived flowlines', verbose)
+
+    flows = gpd.read_file(input_flows_fileName)
+    flows['HydroID'] = flows['HydroID'].astype(int)
+    nwm_streams = gpd.read_file(input_nwmflows_fileName)
+    nwm_streams = nwm_streams.rename(columns={'ID': 'feature_id'})
+    nwm_headwaters = gpd.read_file(input_nwm_headwaters_fileName)
+
+    streams_outlets = nwm_streams.loc[~nwm_streams.to.isin(nwm_streams.feature_id), 'feature_id']
+    flows_outlets = flows.loc[~flows['NextDownID'].isin(flows['HydroID']), 'HydroID']
+
+    nwm_streams_headwaters_list = ~nwm_streams['feature_id'].isin(nwm_streams['to'])
+    # flows_headwaters_list = ~flows['LINKNO'].isin(flows['DSLINKNO'])
+    flows_headwaters_list = ~flows['HydroID'].isin(flows['NextDownID'])
+
+    nwm_streams_headwaters = nwm_streams[nwm_streams_headwaters_list]
+    flows_headwaters = flows[flows_headwaters_list]
+
+    # Map headwater points to DEM-derived reaches
+    flows_headwaters = flows_headwaters.sjoin_nearest(nwm_headwaters)
+    flows_headwaters = flows_headwaters[['HydroID', 'ID']]
+    nwm_streams_headwaters = nwm_streams_headwaters.sjoin_nearest(nwm_headwaters)
+    nwm_streams_headwaters = nwm_streams_headwaters[['feature_id', 'ID']]
+
+    def _hydroid_to_feature_id(df, hydroid, hydroid_attr, feature_id_attr):
+        return df.loc[df[hydroid_attr] == hydroid, feature_id_attr]
+
+    def _get_upstream_data(data, data_headwaters, data_dict, hydroid, hydroid_attr, nextdownid_attr):
+        # Find upstream segments
+        data_dict[hydroid] = list(data.loc[data[nextdownid_attr] == hydroid, hydroid_attr].values)
+
+        for hydroid in data_dict[hydroid]:
+            if hydroid in data_headwaters[hydroid_attr].values:
+                data_dict[hydroid] = data_headwaters.loc[
+                    data_headwaters[hydroid_attr] == hydroid, 'ID'
+                ].values[0]
+            else:
+                data_dict = _get_upstream_data(
+                    data, data_headwaters, data_dict, hydroid, hydroid_attr, nextdownid_attr
+                )
+
+        return data_dict
+
+    flows_dict = dict()
+    streams_dict = dict()
+
+    # Compare hash tables
+    for hydroid in flows_outlets:
+        flows_dict = _get_upstream_data(flows, flows_headwaters, flows_dict, hydroid, 'HydroID', 'NextDownID')
+
+    for feature_id in streams_outlets:
+        streams_dict = _get_upstream_data(
+            nwm_streams, nwm_streams_headwaters, streams_dict, feature_id, 'feature_id', 'to'
+        )
+
+    results = []
+    for flow in flows_dict:
+        fid = _hydroid_to_feature_id(flows, flow, 'HydroID', 'feature_id').iloc[0]
+        upstream_hid = flows_dict[flow]
+
+        upstream_fids = []
+        nwm_fids = streams_dict[fid]
+        out_list = [flow, fid, upstream_fids, nwm_fids]
+
+        if not isinstance(upstream_hid, np.int64):
+            if len(upstream_hid) > 0:
+                for i in upstream_hid:
+                    # Find upstream feature_id(s)
+                    temp_fid = int(_hydroid_to_feature_id(flows, i, 'HydroID', 'feature_id').iloc[0])
+
+                    if isinstance(temp_fid, list):
+                        upstream_fids.append(temp_fid[0])
+                    else:
+                        upstream_fids.append(temp_fid)
+
+                out_list = [flow, fid, upstream_fids, nwm_fids]
+                if isinstance(nwm_fids, np.int64):
+                    nwm_fids = [nwm_fids]
+
+                if fid in upstream_fids:
+                    # Skip duplicate feature_ids
+                    out_list.append(-1)
+                elif set(upstream_fids) == set(nwm_fids):
+                    # 0: Crosswalk is correct
+                    out_list.append(0)
+                else:
+                    # 1: Crosswalk is incorrect
+                    out_list.append(1)
+            else:
+                # 2: Crosswalk is empty
+                out_list.append(2)
+        else:
+            # 3: if upstream is a headwater point
+            out_list.append(3)
+
+        results.append(out_list)
+
+    results = pd.DataFrame(
+        data=results, columns=['HydroID', 'feature_id', 'upstream_fids', 'upstream_nwm_fids', 'status']
+    )
+
+    return results
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Tool to check crosswalk accuracy')
+    parser.add_argument('-a', '--input-flows-fileName', help='DEM derived streams', type=str, required=True)
+    parser.add_argument(
+        '-b', '--input-nwmflows-fileName', help='Subset NWM burnlines', type=str, required=True
+    )
+    parser.add_argument(
+        '-d', '--input-nwm-headwaters-fileName', help='Subset NWM headwaters', type=str, required=True
+    )
+    parser.add_argument(
+        '-c', '--output-table-fileName', help='Output table filename', type=str, required=True
+    )
+    parser.add_argument('-u', '--huc', help='HUC ID', type=str, required=True)
+    parser.add_argument('-z', '--branch', help='Branch ID', type=str, required=True)
+
+    args = vars(parser.parse_args())
+
+    evaluate_crosswalk(**args)

From e7a6b7956322892bbc6a6953b233bde610c6bf5f Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 12 Jan 2024 10:55:12 -0700
Subject: [PATCH 295/359] v4.4.9.0 Upgrade base Docker image to GDAL v3.8.0
 (#1058)

---
 Dockerfile                               |   17 +-
 Pipfile                                  |    3 +-
 Pipfile.lock                             | 1111 ++++++++++------------
 docs/CHANGELOG.md                        |   18 +
 src/accumulate_headwaters.py             |  102 ++
 src/delineate_hydros_and_produce_HAND.sh |   15 +-
 6 files changed, 660 insertions(+), 606 deletions(-)
 create mode 100644 src/accumulate_headwaters.py

diff --git a/Dockerfile b/Dockerfile
index dd579c65e..70f7d336f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 ## Temporary image to build the libraries and only save the needed artifacts
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.4.3 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0 AS builder
 WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
@@ -15,7 +15,7 @@ RUN git clone https://github.com/dtarb/taudem.git
 RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
 
 RUN apt-get update --fix-missing && apt-get install -y cmake mpich \
-    libgtest-dev libboost-test-dev libnetcdf-dev openjdk-17-jdk && rm -rf /var/lib/apt/lists/*
+    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/*
 
 ## Compile Main taudem repo ##
 RUN mkdir -p taudem/bin
@@ -36,18 +36,13 @@ RUN mkdir -p $taudemDir
 RUN mkdir -p $taudemDir2
 
 ## Move needed binaries to the next stage of the image
-RUN cd taudem/bin && mv -t $taudemDir flowdircond aread8 threshold streamnet gagewatershed catchhydrogeo dinfdistdown
+RUN cd taudem/bin && mv -t $taudemDir flowdircond streamnet gagewatershed catchhydrogeo dinfdistdown
 RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d8flowdir dinfflowdir
 
 
-
-
 ###############################################################################################
-
-
-
 # Base Image that has GDAL, PROJ, etc
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.4.3
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0
 ARG dataDir=/data
 ENV projectDir=/foss_fim
 ARG depDir=/dependencies
@@ -72,7 +67,7 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 RUN apt update --fix-missing
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync tzdata openjdk-17-jdk
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata
 
 RUN apt auto-remove
 
@@ -107,7 +102,7 @@ RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --syste
 # We download and unzip it to the same file folder that pip deployed the whitebox library.
 # Whitebox also attempts to always download a folder called testdata regardless of use.
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
-RUN wbox_path=/usr/local/lib/python3.8/dist-packages/whitebox/ && \
+RUN wbox_path=/usr/local/lib/python3.10/dist-packages/whitebox/ && \
     wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
     unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
     cp $wbox_path/WBT/whitebox_tools $wbox_path && \
diff --git a/Pipfile b/Pipfile
index beace0056..043b69288 100644
--- a/Pipfile
+++ b/Pipfile
@@ -43,6 +43,7 @@ flake8-pyproject = "==1.2.3"
 pre-commit = "==3.3.3"
 isort = "==5.12.0"
 urllib3 = "==1.26.18"
+pyflwdir = "*"
 
 [requires]
-python_version = "3.8"
+python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index a0e867a67..b38946cc9 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "05bec99d8be9c133d86cc5a13bbb8dd751b89eaa4368df49170d3527a84bb127"
+            "sha256": "b0e64fe05b2fb6310dcb5e41cb0d131e77067c5bb3c54db67c9f46a14e705d42"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.8"
+            "python_version": "3.10"
         },
         "sources": [
             {
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4'",
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -42,11 +42,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:56a415fbc462291813a94528a779597226619c8e78af7de0507333f700011e5f",
-                "sha256:5a0bec7085176715be77df87fc66d6c9d70626bd752fcc85f57cdbee5b3760da"
+                "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee",
+                "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.1.0"
+            "version": "==4.2.0"
         },
         "appdirs": {
             "hashes": [
@@ -115,25 +115,18 @@
         },
         "babel": {
             "hashes": [
-                "sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900",
-                "sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed"
+                "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363",
+                "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.13.1"
-        },
-        "backcall": {
-            "hashes": [
-                "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e",
-                "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"
-            ],
-            "version": "==0.2.0"
+            "version": "==2.14.0"
         },
         "beautifulsoup4": {
             "hashes": [
                 "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
                 "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
             ],
-            "markers": "python_full_version >= '3.6.0'",
+            "markers": "python_version >= '3.6'",
             "version": "==4.12.2"
         },
         "black": {
@@ -196,7 +189,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4'",
+            "markers": "python_version >= '3.8' and python_version < '4.0'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -529,7 +522,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -566,61 +559,53 @@
         },
         "contourpy": {
             "hashes": [
-                "sha256:059c3d2a94b930f4dafe8105bcdc1b21de99b30b51b5bce74c753686de858cb6",
-                "sha256:0683e1ae20dc038075d92e0e0148f09ffcefab120e57f6b4c9c0f477ec171f33",
-                "sha256:07d6f11dfaf80a84c97f1a5ba50d129d9303c5b4206f776e94037332e298dda8",
-                "sha256:081f3c0880712e40effc5f4c3b08feca6d064cb8cfbb372ca548105b86fd6c3d",
-                "sha256:0e48694d6a9c5a26ee85b10130c77a011a4fedf50a7279fa0bdaf44bafb4299d",
-                "sha256:11b836b7dbfb74e049c302bbf74b4b8f6cb9d0b6ca1bf86cfa8ba144aedadd9c",
-                "sha256:19557fa407e70f20bfaba7d55b4d97b14f9480856c4fb65812e8a05fe1c6f9bf",
-                "sha256:229a25f68046c5cf8067d6d6351c8b99e40da11b04d8416bf8d2b1d75922521e",
-                "sha256:24216552104ae8f3b34120ef84825400b16eb6133af2e27a190fdc13529f023e",
-                "sha256:3b53d5769aa1f2d4ea407c65f2d1d08002952fac1d9e9d307aa2e1023554a163",
-                "sha256:3de23ca4f381c3770dee6d10ead6fff524d540c0f662e763ad1530bde5112532",
-                "sha256:407d864db716a067cc696d61fa1ef6637fedf03606e8417fe2aeed20a061e6b2",
-                "sha256:41339b24471c58dc1499e56783fedc1afa4bb018bcd035cfb0ee2ad2a7501ef8",
-                "sha256:462c59914dc6d81e0b11f37e560b8a7c2dbab6aca4f38be31519d442d6cde1a1",
-                "sha256:46e24f5412c948d81736509377e255f6040e94216bf1a9b5ea1eaa9d29f6ec1b",
-                "sha256:498e53573e8b94b1caeb9e62d7c2d053c263ebb6aa259c81050766beb50ff8d9",
-                "sha256:4ebf42695f75ee1a952f98ce9775c873e4971732a87334b099dde90b6af6a916",
-                "sha256:4f9147051cb8fdb29a51dc2482d792b3b23e50f8f57e3720ca2e3d438b7adf23",
-                "sha256:549174b0713d49871c6dee90a4b499d3f12f5e5f69641cd23c50a4542e2ca1eb",
-                "sha256:560f1d68a33e89c62da5da4077ba98137a5e4d3a271b29f2f195d0fba2adcb6a",
-                "sha256:566f0e41df06dfef2431defcfaa155f0acfa1ca4acbf8fd80895b1e7e2ada40e",
-                "sha256:56de98a2fb23025882a18b60c7f0ea2d2d70bbbcfcf878f9067234b1c4818442",
-                "sha256:66544f853bfa85c0d07a68f6c648b2ec81dafd30f272565c37ab47a33b220684",
-                "sha256:6c06e4c6e234fcc65435223c7b2a90f286b7f1b2733058bdf1345d218cc59e34",
-                "sha256:6d0a8efc258659edc5299f9ef32d8d81de8b53b45d67bf4bfa3067f31366764d",
-                "sha256:70e5a10f8093d228bb2b552beeb318b8928b8a94763ef03b858ef3612b29395d",
-                "sha256:8394e652925a18ef0091115e3cc191fef350ab6dc3cc417f06da66bf98071ae9",
-                "sha256:8636cd2fc5da0fb102a2504fa2c4bea3cbc149533b345d72cdf0e7a924decc45",
-                "sha256:93df44ab351119d14cd1e6b52a5063d3336f0754b72736cc63db59307dabb718",
-                "sha256:96ba37c2e24b7212a77da85004c38e7c4d155d3e72a45eeaf22c1f03f607e8ab",
-                "sha256:a10dab5ea1bd4401c9483450b5b0ba5416be799bbd50fc7a6cc5e2a15e03e8a3",
-                "sha256:a66045af6cf00e19d02191ab578a50cb93b2028c3eefed999793698e9ea768ae",
-                "sha256:a75cc163a5f4531a256f2c523bd80db509a49fc23721b36dd1ef2f60ff41c3cb",
-                "sha256:b04c2f0adaf255bf756cf08ebef1be132d3c7a06fe6f9877d55640c5e60c72c5",
-                "sha256:ba42e3810999a0ddd0439e6e5dbf6d034055cdc72b7c5c839f37a7c274cb4eba",
-                "sha256:bfc8a5e9238232a45ebc5cb3bfee71f1167064c8d382cadd6076f0d51cff1da0",
-                "sha256:c5bd5680f844c3ff0008523a71949a3ff5e4953eb7701b28760805bc9bcff217",
-                "sha256:c84fdf3da00c2827d634de4fcf17e3e067490c4aea82833625c4c8e6cdea0887",
-                "sha256:ca6fab080484e419528e98624fb5c4282148b847e3602dc8dbe0cb0669469887",
-                "sha256:d0c188ae66b772d9d61d43c6030500344c13e3f73a00d1dc241da896f379bb62",
-                "sha256:d6ab42f223e58b7dac1bb0af32194a7b9311065583cc75ff59dcf301afd8a431",
-                "sha256:dfe80c017973e6a4c367e037cb31601044dd55e6bfacd57370674867d15a899b",
-                "sha256:e0c02b75acfea5cab07585d25069207e478d12309557f90a61b5a3b4f77f46ce",
-                "sha256:e30aaf2b8a2bac57eb7e1650df1b3a4130e8d0c66fc2f861039d507a11760e1b",
-                "sha256:eafbef886566dc1047d7b3d4b14db0d5b7deb99638d8e1be4e23a7c7ac59ff0f",
-                "sha256:efe0fab26d598e1ec07d72cf03eaeeba8e42b4ecf6b9ccb5a356fde60ff08b85",
-                "sha256:f08e469821a5e4751c97fcd34bcb586bc243c39c2e39321822060ba902eac49e",
-                "sha256:f1eaac5257a8f8a047248d60e8f9315c6cff58f7803971170d952555ef6344a7",
-                "sha256:f29fb0b3f1217dfe9362ec55440d0743fe868497359f2cf93293f4b2701b8251",
-                "sha256:f44d78b61740e4e8c71db1cf1fd56d9050a4747681c59ec1094750a658ceb970",
-                "sha256:f6aec19457617ef468ff091669cca01fa7ea557b12b59a7908b9474bb9674cf0",
-                "sha256:f9dc7f933975367251c1b34da882c4f0e0b2e24bb35dc906d2f598a40b72bfc7"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.1.1"
+                "sha256:0274c1cb63625972c0c007ab14dd9ba9e199c36ae1a231ce45d725cbcbfd10a8",
+                "sha256:0d7e03c0f9a4f90dc18d4e77e9ef4ec7b7bbb437f7f675be8e530d65ae6ef956",
+                "sha256:11f8d2554e52f459918f7b8e6aa20ec2a3bce35ce95c1f0ef4ba36fbda306df5",
+                "sha256:139d8d2e1c1dd52d78682f505e980f592ba53c9f73bd6be102233e358b401063",
+                "sha256:16a7380e943a6d52472096cb7ad5264ecee36ed60888e2a3d3814991a0107286",
+                "sha256:171f311cb758de7da13fc53af221ae47a5877be5a0843a9fe150818c51ed276a",
+                "sha256:18fc2b4ed8e4a8fe849d18dce4bd3c7ea637758c6343a1f2bae1e9bd4c9f4686",
+                "sha256:1c203f617abc0dde5792beb586f827021069fb6d403d7f4d5c2b543d87edceb9",
+                "sha256:1c2559d6cffc94890b0529ea7eeecc20d6fadc1539273aa27faf503eb4656d8f",
+                "sha256:1c88dfb9e0c77612febebb6ac69d44a8d81e3dc60f993215425b62c1161353f4",
+                "sha256:1e9dc350fb4c58adc64df3e0703ab076f60aac06e67d48b3848c23647ae4310e",
+                "sha256:247b9d16535acaa766d03037d8e8fb20866d054d3c7fbf6fd1f993f11fc60ca0",
+                "sha256:266270c6f6608340f6c9836a0fb9b367be61dde0c9a9a18d5ece97774105ff3e",
+                "sha256:34b9071c040d6fe45d9826cbbe3727d20d83f1b6110d219b83eb0e2a01d79488",
+                "sha256:3d7d1f8871998cdff5d2ff6a087e5e1780139abe2838e85b0b46b7ae6cc25399",
+                "sha256:461e3ae84cd90b30f8d533f07d87c00379644205b1d33a5ea03381edc4b69431",
+                "sha256:464b423bc2a009088f19bdf1f232299e8b6917963e2b7e1d277da5041f33a779",
+                "sha256:491b1917afdd8638a05b611a56d46587d5a632cabead889a5440f7c638bc6ed9",
+                "sha256:4a1b1208102be6e851f20066bf0e7a96b7d48a07c9b0cfe6d0d4545c2f6cadab",
+                "sha256:575bcaf957a25d1194903a10bc9f316c136c19f24e0985a2b9b5608bdf5dbfe0",
+                "sha256:5c6b28956b7b232ae801406e529ad7b350d3f09a4fde958dfdf3c0520cdde0dd",
+                "sha256:5d16edfc3fc09968e09ddffada434b3bf989bf4911535e04eada58469873e28e",
+                "sha256:5fd1810973a375ca0e097dee059c407913ba35723b111df75671a1976efa04bc",
+                "sha256:67b7f17679fa62ec82b7e3e611c43a016b887bd64fb933b3ae8638583006c6d6",
+                "sha256:68ce4788b7d93e47f84edd3f1f95acdcd142ae60bc0e5493bfd120683d2d4316",
+                "sha256:6d3364b999c62f539cd403f8123ae426da946e142312a514162adb2addd8d808",
+                "sha256:6e739530c662a8d6d42c37c2ed52a6f0932c2d4a3e8c1f90692ad0ce1274abe0",
+                "sha256:6fdd887f17c2f4572ce548461e4f96396681212d858cae7bd52ba3310bc6f00f",
+                "sha256:78e6ad33cf2e2e80c5dfaaa0beec3d61face0fb650557100ee36db808bfa6843",
+                "sha256:884c3f9d42d7218304bc74a8a7693d172685c84bd7ab2bab1ee567b769696df9",
+                "sha256:8d8faf05be5ec8e02a4d86f616fc2a0322ff4a4ce26c0f09d9f7fb5330a35c95",
+                "sha256:999c71939aad2780f003979b25ac5b8f2df651dac7b38fb8ce6c46ba5abe6ae9",
+                "sha256:99ad97258985328b4f207a5e777c1b44a83bfe7cf1f87b99f9c11d4ee477c4de",
+                "sha256:9e6c93b5b2dbcedad20a2f18ec22cae47da0d705d454308063421a3b290d9ea4",
+                "sha256:ab459a1cbbf18e8698399c595a01f6dcc5c138220ca3ea9e7e6126232d102bb4",
+                "sha256:b69303ceb2e4d4f146bf82fda78891ef7bcd80c41bf16bfca3d0d7eb545448aa",
+                "sha256:b7caf9b241464c404613512d5594a6e2ff0cc9cb5615c9475cc1d9b514218ae8",
+                "sha256:b95a225d4948b26a28c08307a60ac00fb8671b14f2047fc5476613252a129776",
+                "sha256:bd2f1ae63998da104f16a8b788f685e55d65760cd1929518fd94cd682bf03e41",
+                "sha256:be16975d94c320432657ad2402f6760990cb640c161ae6da1363051805fa8108",
+                "sha256:ce96dd400486e80ac7d195b2d800b03e3e6a787e2a522bfb83755938465a819e",
+                "sha256:dbd50d0a0539ae2e96e537553aff6d02c10ed165ef40c65b0e27e744a0f10af8",
+                "sha256:dd10c26b4eadae44783c45ad6655220426f971c61d9b239e6f7b16d5cdaaa727",
+                "sha256:ebeac59e9e1eb4b84940d076d9f9a6cec0064e241818bcb6e32124cc5c3e377a"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.2.0"
         },
         "cycler": {
             "hashes": [
@@ -632,67 +617,60 @@
         },
         "cython": {
             "hashes": [
-                "sha256:035cb6930a8534f865a3f4616543f78bd27e4de9c3e117b2826e395085ffc4c0",
-                "sha256:039877e57dc10abf0d30d2de2c7476f0881d8ecef1f29bdeed1a6a06a8d89141",
-                "sha256:048fe89c2c753c24e1a7a05496907173dab17e238c8dc3c7cad90b3679b0d846",
-                "sha256:05c4efd36879ff8020af00407e4c14246b894558ea41dc6486f60dd71601fc67",
-                "sha256:063220a6dc0909b576ef068c7e2acf5c608d64423a6d231aacb72d06352cd95b",
-                "sha256:0759868b4a4d103578375e031e89abd578c26774d957ee4f591764ef8003b363",
-                "sha256:077d9a61e6042174dabf68b8e92c0a80f5aff529439ed314aa6e6a233af80b95",
-                "sha256:0a9206b0720f0cad3e70c018722f6d10e81b32e65477e14ffedd3fbfadfaddca",
-                "sha256:0d9fcfc09d67218fce114fe9fd97bba4f9d56add0f775c588d8c626ed47f1aef",
-                "sha256:0f31d02b831d0fa9bf099b1b714b5a8252eabd8db34b7d33c48e7e808a2dabf9",
-                "sha256:115e76fbe9288119526b66963f614042222d1587f1ba5ddb90088215a3d2a25a",
-                "sha256:13491f1bfcf020fd02751c4a55294aa8957e21b7ecd2542b0521a7aa50c58bb2",
-                "sha256:2ad4eb2608661d63fb57c674dafb9955f5141d748d4579c7722c1a3c6b86a0c2",
-                "sha256:2c4d4d92182002b2878adb3329de1ccb7f3f7571d3586f92602e790bfeab45d0",
-                "sha256:2d6bb318ddce8b978c81cf78caf8b3836db84f6235d721952685e87871f506e4",
-                "sha256:34059c3ea6e342ba388cd9774a61761bb4200ca18bd475de65c9cc70ef4e0204",
-                "sha256:3679a6693456f5f7ccca9ab2a91408e99ee257e145024fe380da7c78a07e98b6",
-                "sha256:39318348db488a2f24e7c84e08bdc82f2624853c0fea8b475ea0b70b27176492",
-                "sha256:3cffbba1888f795de2103e6fb1482c8ea8d457e638fa813e090fe747f9e549bb",
-                "sha256:452679284c15a7d5a88bff675e1dd660349360f0665aea50be2d98b7650925f8",
-                "sha256:45277bb54c35b11bcdd6cf0f56eb950eb280b67146db0cb57853fb6334c6d11d",
-                "sha256:45aaceb082ad89365f2f783a40db42359591ad55914fb298841196edf88afdc5",
-                "sha256:485f8a3087392e2287e2869adc0385b439f69b9cfbd262fdf39b00417690c988",
-                "sha256:4db9eea298e982aee7ba12b3432c66eb2e91bb2f5d026bbd57c35698ea0f557f",
-                "sha256:4faf17ea6e8fc3065a862d4b24be84048fd58ed7abe59aa2f9141446a7a72335",
-                "sha256:4fbc8f62b8d50f9a2eef99927a9dcb8d0a42e5a801ab14c2e4aeab622c88f54b",
-                "sha256:530a474a79fa6c2127bb7e3ba00857b1f26a563615863f17b7434331aa3fe404",
-                "sha256:5742ef31e1e2c9a4824ef6b05af0f4949047a6f73af1d4b238ce12935174aede",
-                "sha256:57b44f789794d74c1feddae054dd045b9f601bdffd7288e069b4ca7ed607ec61",
-                "sha256:6ac1cf1f2ed01656b33618352f7e42bf75d027425b83cc96cfe13ce4b6cba5de",
-                "sha256:75206369504fc442c10a86ecf57b91592dca744e4592af22a47e9a774d53dd10",
-                "sha256:77f4d001fb7a03a68b53be20571acd17452d1dda7907d9c325dff0cc704b1ef9",
-                "sha256:808f56d4cd0511723b787c341f3cc995fd72893e608102268298c188f4a4f2e7",
-                "sha256:93502f45948ae8d7f874ba4c113b50cb6fb4ee664caa82e1ddc398500ee0ffb3",
-                "sha256:9c016b3e859b41cf4ce1b8107f364ad5a83c086f75ea4d8d3990b24691f626a1",
-                "sha256:a1cab30c11880f38a27911b569ea38b0bd67fcf32f8a8a8519b613c70562dae2",
-                "sha256:a90f9c7b6635967eacafebe439d518b7dc720aaaf19cb9553f5aad03c13296f4",
-                "sha256:a95ed0e6f481462a3ff2be4c2e4ffffc5d00fc3884d4ccd1fe5b702d4938ec09",
-                "sha256:a993002fe28c840dc29805fde7341c775b7878b311b85f21560fdebf220c247b",
-                "sha256:abb2362783521bd9a22fe69b2141abab4db97237665a36a034b161ddee5b3e72",
-                "sha256:ada4852db0e33dbdd1425322db081d22b9725cb9f5eba42885467b4e2c4f2ac0",
-                "sha256:b37f4b0d983316242b4b9241ecbbe55220aa92af93ff04626441fe0ea90a54f9",
-                "sha256:b77f2b45535bcf3741592fa03183558bd42198b872c1584b896fa0ba5f2ac68d",
-                "sha256:b94f58e05e69e1a43da551c8f532e9fad057df1641f0f8ae8f103d4ede5a80fe",
-                "sha256:ba3f7b433c1721a43674c0889d5fad746bf608416c8f849343859e6d4d3a7113",
-                "sha256:c18e125537a96e76c8c34201e5a9aad8625e3d872dd26a63155573462e54e185",
-                "sha256:c95bd21d87b08c88fe5296381a5f39cd979a775bf1a1d7379a6ff87c703e510b",
-                "sha256:d0d9431101f600d5a557d55989658cbfd02b7c0dcd1e4675dac8ad7e0da8ea5b",
-                "sha256:d9d17a6ceb301c5dbd3820e62c1b10a4ad3a6eea3e07e7afaf736b5f490c2e32",
-                "sha256:db21997270e943aee9cb7694112d24a4702fbe1977fbe53b3cb4db3d02be73d9",
-                "sha256:dd9cab3b862bec2b110886aedb11765e9deda363c4c7ab5ea205f3d8f143c411",
-                "sha256:dee39967168d6326ea2df56ad215a4d5049aa52f44cd5aad45bfb63d5b4fb9e5",
-                "sha256:e3e011fa2ae9e953fe1ab8394329a21bdb54357c7fe509bcfb02b88bc15bffbb",
-                "sha256:ebc901131057c115a8868e14c1df6e56b9190df774b72664c03ebd858296bb81",
-                "sha256:f18c13d5ed6fde5efd3b1c039f6a34296d1a0409bb00fbf45bec6f9bcf63ddf5",
-                "sha256:f687539ead9fbc17f499e33ee20c1dc41598f70ad95edb4990c576447cec9d23",
-                "sha256:f6fcfef825edb44cf3c6ba2c091ad76a83da62ac9c79553e80e0c2a1f75eda2e",
-                "sha256:fcfd2255458a5779dbab813550695331d541b24f0ef831ace83f04f9516ddf26"
+                "sha256:01b94304aab87496e81d1f546e71abf57b430b39be4269df1cd7da9928d70b5b",
+                "sha256:0c636c9ab92c7838231a1ba769e519d953af8294612f3f772a54d3a5250ff23f",
+                "sha256:0d8a98c7d86ac4d05b251c39faf49423780381aab55fbf2e147f6e006a34a58a",
+                "sha256:13211b67b29f6ed8e87c137496c73d93aff0330d97940b4fbed72eae37a4a2a0",
+                "sha256:133057ac45b6fa7fe5d7baada9d3545d09339432f75c0545f556e8c6fecc2932",
+                "sha256:167b3f3894dcc697cefefac1d198304fae8eb4d5860a7b8bc2459d572e838470",
+                "sha256:225f8bba6428b8d711ca2d6c738d2e3a4667f6a2ae40f8a7a5256f69f6a3600e",
+                "sha256:22d2a684122dfb531853d57c8c85c1d5d44be709e12466dca99fa6aee7d8054f",
+                "sha256:23ceac5315fe899c229e874328742154e331fa41337bb03f6f5264636c351c9e",
+                "sha256:2c67105f2c6ccf5b3adbcfaecf3c5c9fa8940f9f97955c9ad7d2542151d97d93",
+                "sha256:30eb2d2938b9195e2c82951713429aff3ad1be9f104437d1536a04eb0cb3dc0e",
+                "sha256:34d51709e10ad6213b4bf094af7be7ff82bab43216b3c92a07d05b451deeca79",
+                "sha256:3a83e04fde663b84905f3a20213a4333d13a07b79434300704b70dc552761f8b",
+                "sha256:3f02c7240abab48d59f0d5fef7064f18f01a2a204616165fa6367a8abf5a8832",
+                "sha256:45319d2471f4dbf19893ca53785a421107266e18b8cccd2054fce1e3f72a85f1",
+                "sha256:4e8bf00ec1dd1d92e9ae74d2e6891f087a939e1dfb40c9c7fa5d8d6a26c94f5a",
+                "sha256:51e8164b1270625ff101e95c3c1c234421520c07a0a3a20ded9e9431d98afce7",
+                "sha256:539ad5a21141e6420035cf616bcba48d999bf878839e52692f97fc7e2f16265c",
+                "sha256:55f93d3822bc196b37a8bdfa4ec6a35232a399e97f2baa714bd5ed8ea9b0ce68",
+                "sha256:5e3a038332973b12e72236e8884dc99601a840334c2c46cfbbb5851cb94166eb",
+                "sha256:612d83fd1eb5aaa5401a755c1f1aafacd9dab404cd350b90d5f404c98b33e4b3",
+                "sha256:6a1859af761977530df2cd5c36e31d54e8d6708ad2c4656e7125c482364dc216",
+                "sha256:79868ec74e4907a8a6e63effe13547c6157f196a162920b1de066da5849ffb8e",
+                "sha256:79f20c61114c7948cf1214585066406cef4b54a9b935160980e0b6e70ada3a69",
+                "sha256:7c8d579d13cb81abe704c8b0908d122b81d6e2623265a19c4a6a7377f440debb",
+                "sha256:812b193c26553f1f375d4f1c50f805c227b24ed2d595bc9cdaf78c992ecc64a4",
+                "sha256:816f5285d596062c7ef22790de7d75354b58d4417a9fc64cba914aeeb900db0b",
+                "sha256:82f27a0134fc6bb46032ca5f728d8af984f3be94a3cb01cb70ff1224e551b9cf",
+                "sha256:848a28ea49166454c3bff927e5a47629eecf1aa755d6fb3290569cba0fc93766",
+                "sha256:861cf254bf5836d47c2aee86aa75dd93d3de00ccd1b077c3c7a2bb22cba358e7",
+                "sha256:8ad7c2303a338b2c0b6c6c68f101a6768725934538756096cf3388a5c07a7525",
+                "sha256:8ea936cf5931297ba07bce121388c4c6266c1b63a9f4d648ae16c92ff090204b",
+                "sha256:931aade65f77cf59f2a702ac1f549a4836ce221107c740502cbad18d6d8e9511",
+                "sha256:936ec37b261b226d7404eff23a9aad284098338150d42a53d6a9af12b18d3892",
+                "sha256:9fcd9a18ee3ac7f460e0841954feb495102ffbdbec0e6c78562f3495cda000dd",
+                "sha256:b1853bc34ced5ff6473e881fcf6de29da83262552c8f268a0df53b49c2b89e2c",
+                "sha256:b227643d8a40b68554dc7d37fcd03fc97b4fb0bd2614aeb5f2e07ab244642d36",
+                "sha256:b65abca78aa5ebc8675c8480b9a53006f6efea9910ad099cf32c9fb5617ef251",
+                "sha256:b9d0dae6dccd349b8ccf197c10ef2d05c711ca36a649c7eddbab1de2c90b63a1",
+                "sha256:cd6ae43ef2e596c9a88dbf2a8895be2e32cc2f5bc3c8ba2e7753b69068fc0b2d",
+                "sha256:e13abb14843397b76d0472c7d33cd260d5f262ab05cc27ed423317e645e29643",
+                "sha256:e1bdf8a107fdf9e174991aa87a0be7504f60de1ec6bfb1ccfb30e33acac818a0",
+                "sha256:e34b4b08d795ccca920fa26b099558f4f1e4e3f794e4ba8d3433c5bc2454d50a",
+                "sha256:e3c0e19bb41de6be9d8afc85795159ca16296be81a586cd9588be0400d44a855",
+                "sha256:ef5bb0268bfe5992da3ef9292463a5a895ed8700b134ed2c00008d5471b3ba6e",
+                "sha256:f2602a5c97a3d618b3b847514204ef3349fb414c59e1126c0c2c708d2c5680f8",
+                "sha256:f3845c4506e0d207c5e268fb02813928f3a1e135de954a379f165ef0d581da47",
+                "sha256:f674be92673e87dd8ee7cfe553d5960ec4effc5ab15063b9a5e265a51585a31a",
+                "sha256:f6d8ff62ad55dc0393686438eac4b457a916e4d1118a0b550746bb52b4c756cc",
+                "sha256:fb299acf3a578573c190c858d49e0cf9d75f4bc49c3f24c5a63804997ef09213",
+                "sha256:fed25959e4025870fdde5f895fcb126196d22affd4f4fad85a2823e0dddc85b0"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.5"
+            "version": "==3.0.7"
         },
         "dask": {
             "hashes": [
@@ -704,17 +682,11 @@
         },
         "datashader": {
             "hashes": [
-                "sha256:9539529379287d69c10e9ab4e0b2900e1a16004d6ff06de0d84aa62c4833b1bb",
-                "sha256:9af96d76bc7011c60b4e76177e283efe20e060278ecd81ce7005a4d80a4dc69e"
+                "sha256:a2cb0f839067bf29cf6cc9c07a1dad35f0e4aed3768387056fcbac8748087bfa",
+                "sha256:ed4c111957578dcb3fcff972d954f77586dafd71a7345fd5cd069d9fb050d0d1"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==0.15.2"
-        },
-        "datashape": {
-            "hashes": [
-                "sha256:2356ea690c3cf003c1468a243a9063144235de45b080b3652de4f3d44e57d783"
-            ],
-            "version": "==0.5.2"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.16.0"
         },
         "debugpy": {
             "hashes": [
@@ -758,10 +730,10 @@
         },
         "distlib": {
             "hashes": [
-                "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057",
-                "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"
+                "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784",
+                "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"
             ],
-            "version": "==0.3.7"
+            "version": "==0.3.8"
         },
         "entrypoints": {
             "hashes": [
@@ -852,51 +824,51 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:05d7c4d2c95b9490e669f3cb83918799bf1c838619ac6d3bad9ea017cfc63f2e",
-                "sha256:0f412954275e594f7a51c16f3b3edd850acb0d842fefc33856b63a17e18499a5",
-                "sha256:22ea8aa7b3712450b42b044702bd3a64fd118006bad09a6f94bd1b227088492e",
-                "sha256:2db63941fee3122e31a21dd0f5b2138ce9906b661a85b63622421d3654a74ae2",
-                "sha256:2e91e19b583961979e2e5a701269d3cfc07418963bee717f8160b0a24332826b",
-                "sha256:31b38528f25bc662401e6ffae14b3eb7f1e820892fd80369a37155e3b636a2f4",
-                "sha256:3d29509f6e05e8d725db59c2d8c076223d793e4e35773040be6632a0349f2f97",
-                "sha256:46c79af80a835410874683b5779b6c1ec1d5a285e11c45b5193e79dd691eb111",
-                "sha256:4e90dd81b6e0d97ebfe52c0d12a17a9ef7f305d6bfbb93081265057d6092f252",
-                "sha256:50d25893885e80a5955186791eed5579f1e75921751539cc1dc3ffd1160b48cf",
-                "sha256:518a945dbfe337744bfff31423c1430303b8813c5275dffb0f2577f0734a1189",
-                "sha256:54efed22b2799a85475e6840e907c402ba49892c614565dc770aa97a53621b2b",
-                "sha256:58af428746fa73a2edcbf26aff33ac4ef3c11c8d75bb200eaea2f7e888d2de4e",
-                "sha256:59b6ad83cce067d10f4790c037a5904424f45bebb5e7be2eb2db90402f288267",
-                "sha256:63a3112f753baef8c6ac2f5f574bb9ac8001b86c8c0c0380039db47a7f512d20",
-                "sha256:66bc6efd829382f7a7e6cf33c2fb32b13edc8a239eb15f32acbf197dce7a0165",
-                "sha256:6999e80a125b0cd8e068d0210b63323f17338038c2ecd2e11b9209ec430fe7f2",
-                "sha256:6d16d9634ff1e5cea2cf4a8cbda9026f766e4b5f30b48f8180f0e99133d3abfc",
-                "sha256:84f308b7a8d28208d54315d11d35f9888d6d607673dd4d42d60b463682ee0400",
-                "sha256:9ee8692e23028564c13d924004495f284df8ac016a19f17a87251210e1f1f928",
-                "sha256:a3da036b016c975c2d8c69005bdc4d5d16266f948a7fab950244e0f58301996a",
-                "sha256:a7aec7f5d14dfcd71fb3ebc299b3f000c21fdc4043079101777ed2042ba5b7c5",
-                "sha256:a8a1fa9a718de0bc026979c93e1e9b55c5efde60d76f91561fd713387573817d",
-                "sha256:a8b99713d3a0d0e876b6aecfaada5e7dc9fe979fcd90ef9fa0ba1d9b9aed03f2",
-                "sha256:b63da598d9cbc52e2381f922da0e94d60c0429f92207bd3fb04d112fc82ea7cb",
-                "sha256:b6e6aa2d066f8dafd06d8d0799b4944b5d5a1f015dd52ac01bdf2895ebe169a0",
-                "sha256:b99fe8ef4093f672d00841569d2d05691e50334d79f4d9c15c1265d76d5580d2",
-                "sha256:b9beb0fa6ff3ea808ad4a6962d68ac0f140ddab080957b20d9e268e4d67fb335",
-                "sha256:b9eab7f9837fdaa2a10a524fbcc2ec24bf60637c044b6e4a59c3f835b90f0fae",
-                "sha256:bca49da868e8bde569ef36f0cc1b6de21d56bf9c3be185c503b629c19a185287",
-                "sha256:c05064f95aacdfc06f21e55096c964b2228d942b8675fa26995a2551f6329d2d",
-                "sha256:c2de1fb18198acd400c45ffe2aef5420c8d55fde903e91cba705596099550f3b",
-                "sha256:c794de4086f06ae609b71ac944ec7deb09f34ecf73316fddc041087dd24bba39",
-                "sha256:d4fa4f4bc8fd86579b8cdbe5e948f35d82c0eda0091c399d009b2a5a6b61c040",
-                "sha256:dab3d00d27b1a79ae4d4a240e8ceea8af0ff049fd45f05adb4f860d93744110d",
-                "sha256:dbac86d83d96099890e731cc2af97976ff2c98f4ba432fccde657c5653a32f1c",
-                "sha256:df40daa6c03b98652ffe8110ae014fe695437f6e1cb5a07e16ea37f40e73ac86",
-                "sha256:e1cd1c6bb097e774d68402499ff66185190baaa2629ae2f18515a2c50b93db0c",
-                "sha256:e8ff7d19a6804bfd561cfcec9b4200dd1788e28f7de4be70189801530c47c1b3",
-                "sha256:eb01c49c8aa035d5346f46630209923d4927ed15c2493db38d31da9f811eb70d",
-                "sha256:f53526668beccdb3409c6055a4ffe50987a7f05af6436fa55d61f5e7bd450219",
-                "sha256:f611c97678604e302b725f71626edea113a5745a7fb557c958b39edb6add87d5"
+                "sha256:084511482dd265bce6dca24c509894062f0117e4e6869384d853f46c0e6d43be",
+                "sha256:1193fb090061efa2f9e2d8d743ae9850c77b66746a3b32792324cdce65784154",
+                "sha256:174995f7b057e799355b393e97f4f93ef1f2197cbfa945e988d49b2a09ecbce8",
+                "sha256:253bb46bab970e8aae254cebf2ae3db98a4ef6bd034707aa68a239027d2b198d",
+                "sha256:2a78dba8c2a1e9d53a0fb5382979f024200dc86adc46a56cbb668a2249862fda",
+                "sha256:2d2404107626f97a221dc1a65b05396d2bb2ce38e435f64f26ed2369f68675d9",
+                "sha256:40bdbe90b33897d9cc4a39f8e415b0fcdeae4c40a99374b8a4982f127ff5c767",
+                "sha256:495369c660e0c27233e3c572269cbe520f7f4978be675f990f4005937337d391",
+                "sha256:4a9a51745c0439516d947480d4d884fa18bd1458e05b829e482b9269afa655bc",
+                "sha256:511482df31cfea9f697930f61520f6541185fa5eeba2fa760fe72e8eee5af88b",
+                "sha256:52c82df66201f3a90db438d9d7b337c7c98139de598d0728fb99dab9fd0495ca",
+                "sha256:562681188c62c024fe2c611b32e08b8de2afa00c0c4e72bed47c47c318e16d5c",
+                "sha256:59a6c8b71a245800e923cb684a2dc0eac19c56493e2f896218fcf2571ed28984",
+                "sha256:5dde0eab40faaa5476133123f6a622a1cc3ac9b7af45d65690870620323308b4",
+                "sha256:61df4dee5d38ab65b26da8efd62d859a1eef7a34dcbc331299a28e24d04c59a7",
+                "sha256:62d8ddb058b8e87018e5dc26f3258e2c30daad4c87262dfeb0e2617dd84750e6",
+                "sha256:66c92ec7f95fd9732550ebedefcd190a8d81beaa97e89d523a0d17198a8bda4d",
+                "sha256:843509ae9b93db5aaf1a6302085e30bddc1111d31e11d724584818f5b698f500",
+                "sha256:854421e328d47d70aa5abceacbe8eef231961b162c71cbe7ff3f47e235e2e5c5",
+                "sha256:97620c4af36e4c849e52661492e31dc36916df12571cb900d16960ab8e92a980",
+                "sha256:9acfa1cdc479e0dde528b61423855913d949a7f7fe09e276228298fef4589540",
+                "sha256:a77a60315c33393b2bd29d538d1ef026060a63d3a49a9233b779261bad9c3f71",
+                "sha256:b4fabb8cc9422efae1a925160083fdcbab8fdc96a8483441eb7457235df625bd",
+                "sha256:bf1810635c00f7c45d93085611c995fc130009cec5abdc35b327156aa191f982",
+                "sha256:c01f409be619a9a0f5590389e37ccb58b47264939f0e8d58bfa1f3ba07d22671",
+                "sha256:c59227d7ba5b232281c26ae04fac2c73a79ad0e236bca5c44aae904a18f14faf",
+                "sha256:c75e19971209fbbce891ebfd1b10c37320a5a28e8d438861c21d35305aedb81c",
+                "sha256:ce0e2c88c8c985b7b9a7efcd06511fb0a1fe3ddd9a6cd2895ef1dbf9059719d7",
+                "sha256:d6477ba902dd2d7adda7f0fd3bfaeb92885d45993c9e1928c9f28fc3961415f7",
+                "sha256:d986b66ff722ef675b7ee22fbe5947a41f60a61a4da15579d5e276d897fbc7fa",
+                "sha256:dd23848f877c3754f53a4903fb7a593ed100924f9b4bff7d5a4e2e8a7001ae11",
+                "sha256:e3f4d61f3a8195eac784f1d0c16c0a3105382c1b9a74d99ac4ba421da39a8826",
+                "sha256:e6b968543fde4119231c12c2a953dcf83349590ca631ba8216a8edf9cd4d36a9",
+                "sha256:e77bdf52185bdaf63d39f3e1ac3212e6cfa3ab07d509b94557a8902ce9c13c82",
+                "sha256:e79f1a3970d25f692bbb8c8c2637e621a66c0d60c109ab48d4a160f50856deff",
+                "sha256:e7a0a8848726956e9d9fb18c977a279013daadf0cbb6725d2015a6dd57527992",
+                "sha256:e869da810ae35afb3019baa0d0306cdbab4760a54909c89ad8904fa629991812",
+                "sha256:e8acf6dd0434b211b3bd30d572d9e019831aae17a54016629fa8224783b22df8",
+                "sha256:e8fa20748de55d0021f83754b371432dca0439e02847962fc4c42a0e444c2d78",
+                "sha256:ea592e6a09b71cb7a7661dd93ac0b877a6228e2d677ebacbad0a4d118494c86d",
+                "sha256:ec13a10715eef0e031858c1c23bfaee6cba02b97558e4a7bfa089dba4a8c2ebf",
+                "sha256:f4da089f6dfdb822293bde576916492cd708c37c2501c3651adde39804630538"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.44.0"
+            "version": "==4.47.0"
         },
         "fqdn": {
             "hashes": [
@@ -907,19 +879,19 @@
         },
         "fsspec": {
             "hashes": [
-                "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5",
-                "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"
+                "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb",
+                "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.10.0"
+            "version": "==2023.12.2"
         },
         "geocube": {
             "hashes": [
-                "sha256:35d446f61e8ce97d0bcc3c692d003b355df19d7f59c8cd8bd61d2f9e3c7f72c8",
-                "sha256:b50ee83e4dcf465ccc54e62abe0e6458a26e8faf1e7b15b27604f936781c8a04"
+                "sha256:43c88cd5dd18f9df9561e642b4840e4bd061ccc8a3e8c419378f6707384213d0",
+                "sha256:5404060d6dcc8d3f251b453588667c6d5e5e60d07793011851f8bc129f779db0"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==0.3.3"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.4.2"
         },
         "geographiclib": {
             "hashes": [
@@ -939,11 +911,11 @@
         },
         "geopy": {
             "hashes": [
-                "sha256:a59392bf17adb486b25dbdd71fbed27733bdf24a2dac588047a619de56695e36",
-                "sha256:d2639a46d0ce4c091e9688b750ba94348a14b898a1e55c68f4b4a07e7d1afa20"
+                "sha256:50283d8e7ad07d89be5cb027338c6365a32044df3ae2556ad3f52f4840b3d0d1",
+                "sha256:ae8b4bc5c1131820f4d75fce9d4aaaca0c85189b3aa5d64c3dcaf5e3b7b882a7"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.4.0"
+            "version": "==2.4.1"
         },
         "gval": {
             "hashes": [
@@ -955,11 +927,11 @@
         },
         "identify": {
             "hashes": [
-                "sha256:7736b3c7a28233637e3c36550646fc6389bedd74ae84cb788200cc8e2dd60b75",
-                "sha256:90199cb9e7bd3c5407a9b7e81b4abec4bb9d249991c79439ec8af740afc6293d"
+                "sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d",
+                "sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.31"
+            "version": "==2.5.33"
         },
         "idna": {
             "hashes": [
@@ -974,17 +946,9 @@
                 "sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7",
                 "sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67"
             ],
-            "markers": "python_version < '3.9'",
+            "markers": "python_version >= '3.8'",
             "version": "==7.0.0"
         },
-        "importlib-resources": {
-            "hashes": [
-                "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a",
-                "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"
-            ],
-            "markers": "python_version < '3.10'",
-            "version": "==6.1.1"
-        },
         "inflate64": {
             "hashes": [
                 "sha256:022ca1cc928e7365a05f7371ff06af143c6c667144965e2cf9a9236a2ae1c291",
@@ -1054,11 +1018,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:3ba3dc97424b87b31bb46586b5167b3161b32d7820b9201a9e698c71e271602c",
-                "sha256:553856658eb8430bbe9653ea041a41bff63e9606fc4628873fc92a6cf3abd404"
+                "sha256:7d5d594b6690654b4d299edba5e872dc17bb7396a8d0609c97cb7b8a1c605de6",
+                "sha256:dab88b47f112f9f7df62236511023c9bdeef67abc73af7c652e4ce4441601686"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.26.0"
+            "version": "==6.27.1"
         },
         "ipympl": {
             "hashes": [
@@ -1070,11 +1034,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363",
-                "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c"
+                "sha256:2f55d59370f59d0d2b2212109fe0e6035cfea436b1c0e6150ad2244746272ec5",
+                "sha256:ac4da4ecf0042fb4e0ce57c60430c2db3c719fa8bdf92f8631d6bd8a5785d1f0"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==8.12.3"
+            "markers": "python_version >= '3.10'",
+            "version": "==8.19.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1116,11 +1080,12 @@
         },
         "jinja2": {
             "hashes": [
-                "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
-                "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
+                "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
+                "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
             ],
+            "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.2"
+            "version": "==3.1.3"
         },
         "jmespath": {
             "hashes": [
@@ -1198,11 +1163,11 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:880b86053bf298a8724994f95e99b99130659022a4f7f45f563084b6223861d3",
-                "sha256:e11e02cd8ae0a9de5c6c44abf5727df9f2581055afe00b22183f621ba3585805"
+                "sha256:1553311a97ccd12936037f36b9ab4d6ae8ceea6ad2d5c90d94a909e752178e40",
+                "sha256:220dfb00c45f0d780ce132bb7976b58263f81a3ada6e90a9b6823785a424f739"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.5.0"
+            "version": "==5.5.1"
         },
         "jupyter-events": {
             "hashes": [
@@ -1214,28 +1179,27 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:0c548151b54bcb516ca466ec628f7f021545be137d01b5467877e87f6fff4374",
-                "sha256:0c99f9367b0f24141e527544522430176613f9249849be80504c6d2b955004bb"
+                "sha256:dc77b7dcc5fc0547acba2b2844f01798008667201eea27c6319ff9257d700a6d",
+                "sha256:fd030dd7be1ca572e4598203f718df6630c12bd28a599d7f1791c4d7938e1010"
             ],
-            "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==2.11.2"
+            "version": "==2.12.1"
         },
         "jupyter-server-fileid": {
             "hashes": [
-                "sha256:171538b7c7d08d11dbc57d4e6da196e0c258e4c2cd29249ef1e032bb423677f8",
-                "sha256:5b489c6fe6783c41174a728c7b81099608518387e53c3d53451a67f46a0cb7b0"
+                "sha256:7486bca3acf9bbaab7ce5127f9f64d2df58f5d2de377609fb833291a7217a6a2",
+                "sha256:76dd05a45b78c7ec0cba0be98ece289984c6bcfc1ca2da216d42930e506a4d68"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.9.0"
+            "version": "==0.9.1"
         },
         "jupyter-server-terminals": {
             "hashes": [
-                "sha256:57ab779797c25a7ba68e97bcfb5d7740f2b5e8a83b5e8102b10438041a7eac5d",
-                "sha256:75779164661cec02a8758a5311e18bb8eb70c4e86c6b699403100f1585a12a36"
+                "sha256:2fc0692c883bfd891f4fba0c4b4a684a37234b0ba472f2e97ed0a3888f46e1e4",
+                "sha256:ebcd68c9afbf98a480a533e6f3266354336e645536953b7abcc7bdeebc0154a3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.4.4"
+            "version": "==0.5.0"
         },
         "jupyter-server-ydoc": {
             "hashes": [
@@ -1271,11 +1235,11 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:77c2f1f282d610f95e496e20d5bf1d2a7706826dfb7b18f3378ae2870d272fb7",
-                "sha256:c9f67a98b295c5dee87f41551b0558374e45d449f3edca153dd722140630dcb2"
+                "sha256:5b1798c9cc6a44f65c757de9f97fc06fc3d42535afbf47d2ace5e964ab447aaf",
+                "sha256:bd0ec7a99ebcedc8bcff939ef86e52c378e44c2707e053fcd81d046ce979ee63"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.25.0"
+            "version": "==2.25.2"
         },
         "jupyterlab-widgets": {
             "hashes": [
@@ -1711,16 +1675,16 @@
                 "sha256:4b28c207877cf33ef3a9838cdc7a54c5ceff981194a82eac59d558f05487295e",
                 "sha256:a3a1ddfb34d4a9d17fc744d655962714a866639acd30130e9be84191cd97cd15"
             ],
-            "markers": "python_full_version >= '3.8.0'",
+            "markers": "python_version >= '3.8'",
             "version": "==0.9.0"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:5b6c848194d270cc55fb691169202620d7b52a12fec259508d142ecbe4219310",
-                "sha256:b1564bd89f69a74cd6398b0362da94db07aafb991b7857216a766204a71612c0"
+                "sha256:2dc8267dbdfeedce2dcd34c9e3f1b51af18f43cb105549d1c5a18189ec23ba85",
+                "sha256:3c50eb2d326478cc90b8759cf2ab9dde3d892c6537cd6a5bc0991db8ef734bcc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.12.0"
+            "version": "==7.13.1"
         },
         "nbformat": {
             "hashes": [
@@ -1831,39 +1795,38 @@
         },
         "numexpr": {
             "hashes": [
-                "sha256:15469dc722b5ceb92324ec8635411355ebc702303db901ae8cc87f47c5e3a124",
-                "sha256:18b1804923cfa3be7bbb45187d01c0540c8f6df4928c22a0f786e15568e9ebc5",
-                "sha256:1967c16f61c27df1cdc43ba3c0ba30346157048dd420b4259832276144d0f64e",
-                "sha256:211804ec25a9f6d188eadf4198dd1a92b2f61d7d20993c6c7706139bc4199c5b",
-                "sha256:27782177a0081bd0aab229be5d37674e7f0ab4264ef576697323dd047432a4cd",
-                "sha256:31cf610c952eec57081171f0b4427f9bed2395ec70ec432bbf45d260c5c0cdeb",
-                "sha256:38b8b90967026bbc36c7aa6e8ca3b8906e1990914fd21f446e2a043f4ee3bc06",
-                "sha256:47b45da5aa25600081a649f5e8b2aa640e35db3703f4631f34bb1f2f86d1b5b4",
-                "sha256:6336f8dba3f456e41a4ffc3c97eb63d89c73589ff6e1707141224b930263260d",
-                "sha256:681812e2e71ff1ba9145fac42d03f51ddf6ba911259aa83041323f68e7458002",
-                "sha256:6d7003497d82ef19458dce380b36a99343b96a3bd5773465c2d898bf8f5a38f9",
-                "sha256:6e884687da8af5955dc9beb6a12d469675c90b8fb38b6c93668c989cfc2cd982",
-                "sha256:80acbfefb68bd92e708e09f0a02b29e04d388b9ae72f9fcd57988aca172a7833",
-                "sha256:84979bf14143351c2db8d9dd7fef8aca027c66ad9df9cb5e75c93bf5f7b5a338",
-                "sha256:8564186aad5a2c88d597ebc79b8171b52fd33e9b085013e1ff2208f7e4b387e3",
-                "sha256:8e3e6f1588d6c03877cb3b3dcc3096482da9d330013b886b29cb9586af5af3eb",
-                "sha256:95b9da613761e4fc79748535b2a1f58cada22500e22713ae7d9571fa88d1c2e2",
-                "sha256:95c09e814b0d6549de98b5ded7cdf7d954d934bb6b505432ff82e83a6d330bda",
-                "sha256:9ef7e8aaa84fce3aba2e65f243d14a9f8cc92aafd5d90d67283815febfe43eeb",
-                "sha256:aa0f661f5f4872fd7350cc9895f5d2594794b2a7e7f1961649a351724c64acc9",
-                "sha256:b5f96c89aa0b1f13685ec32fa3d71028db0b5981bfd99a0bbc271035949136b3",
-                "sha256:c48221b6a85494a7be5a022899764e58259af585dff031cecab337277278cc93",
-                "sha256:c8f37f7a6af3bdd61f2efd1cafcc083a9525ab0aaf5dc641e7ec8fc0ae2d3aa1",
-                "sha256:d126938c2c3784673c9c58d94e00b1570aa65517d9c33662234d442fc9fb5795",
-                "sha256:d36528a33aa9c23743b3ea686e57526a4f71e7128a1be66210e1511b09c4e4e9",
-                "sha256:d6a88d71c166e86b98d34701285d23e3e89d548d9f5ae3f4b60919ac7151949f",
-                "sha256:dee04d72307c09599f786b9231acffb10df7d7a74b2ce3681d74a574880d13ce",
-                "sha256:e640bc0eaf1b59f3dde52bc02bbfda98e62f9950202b0584deba28baf9f36bbb",
-                "sha256:e93d64cd20940b726477c3cb64926e683d31b778a1e18f9079a5088fd0d8e7c8",
-                "sha256:ef6e8896457a60a539cb6ba27da78315a9bb31edb246829b25b5b0304bfcee91"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.8.6"
+                "sha256:0d7bfc8b77d8a7b04cd64ae42b62b3bf824a8c751ca235692bfd5231c6e90127",
+                "sha256:12146521b1730073859a20454e75004e38cd0cb61333e763c58ef5171e101eb2",
+                "sha256:121b049b6909787111daf92919c052c4fd87b5691172e8f19f702b96f20aaafa",
+                "sha256:17104051f0bd83fd350212e268d8b48017d5eff522b09b573fdbcc560c5e7ab3",
+                "sha256:22ccd67c0fbeae091f2c577f5b9c8046de6631d46b1cbe22aad46a08d2b42c2d",
+                "sha256:290f91c7ba7772abaf7107f3cc0601d93d6a3f21c13ee3da93f1b8a9ca3e8d39",
+                "sha256:296dc1f79d386166dec3bdb45f51caba29ffd8dc91db15447108c04d3001d921",
+                "sha256:2ae264c35fa67cd510191ab8144f131fddd0f1d13413af710913ea6fc0c6aa61",
+                "sha256:307b49fd15ef2ca292f381e67759e5b477410341f2f499a377234f1b42f529a6",
+                "sha256:399cb914b41c4027ba88a18f6b8ccfc3af5c32bc3b1758403a7c44c72530618a",
+                "sha256:3f168b4b42d4cb120fe1993676dcf74b77a3e8e45b58855566da037cfd938ca3",
+                "sha256:47c05007cd1c553515492c1a78b5477eaaba9cadc5d7b795d49f7aae53ccdf68",
+                "sha256:4d83a542d9deefb050e389aacaddea0f09d68ec617dd37e45b9a7cfbcba6d729",
+                "sha256:4f01d71db6fdb97a68def5407e2dbd748eaea9d98929db08816de40aa4ae3084",
+                "sha256:5a92f230dd9d6c42803f855970e93677b44290b6dad15cb6796fd85edee171ce",
+                "sha256:6459dc6ed6abcdeab3cd3667c79f29e4a0f0a02c29ad71ee5cff065e880ee9ef",
+                "sha256:76f0f010f9c6318bae213b21c5c0e381c2fc9c9ecb8b35f99f5030e7ac96c9ce",
+                "sha256:7badc50efbb2f1c8b78cd68089031e0fd29cbafa6a9e6d730533f22d88168406",
+                "sha256:85c9f79e346c26aa0d425ecfc9e5de7184567d5e48d0bdb02d468bb927e92525",
+                "sha256:925927cd1f610593e7783d8f2e12e3d800d5928601e077e4910e2b50bde624b6",
+                "sha256:a82d710145b0fbaec919dde9c90ed9df1e6785625cc36d1c71f3a53112b66fc5",
+                "sha256:aab17d65751c039d13ed9d49c9a7517b130ef488c1885c4666af9b5c6ad59520",
+                "sha256:b4649c1dcf9b0c2ae0a7b767dbbbde4e05ee68480c1ba7f06fc7963f1f73acf4",
+                "sha256:bf8c517bbbb82c07c23c17f9d52b4c9f86601f57d48e87c0cbda24af5907f4dd",
+                "sha256:ccef9b09432d59229c2a737882e55de7906006452003323e107576f264cec373",
+                "sha256:cd07793b074cc38e478637cbe738dff7d8eb92b5cf8ffaacff0c4f0bca5270a0",
+                "sha256:dbac846f713b4c82333e6af0814ebea0b4e74dfb2649e76c58953fd4862322dd",
+                "sha256:e76ce4d25372f46170cf7eb1ff14ed5d9c69a0b162a405063cbe481bafe3af34",
+                "sha256:f031ac4e70f9ad867543bfbde8452e9d1a14f0525346b4b8bd4e5c0f1380a11c"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==2.8.8"
         },
         "numpy": {
             "hashes": [
@@ -1897,14 +1860,15 @@
                 "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==1.23.5"
         },
         "numpy-groupies": {
             "hashes": [
-                "sha256:fd50026552280ca717722f17d4231390667505bb572de564de1362b40487d34a"
+                "sha256:24a574ecdd58c0d669749b67ec31dab971d27e04037b402bbc4f6bedaa2623bc",
+                "sha256:f920c4ded899f5975d94fc63d634e7c89622056bbab8cc98a44d4320a0ae8a12"
             ],
-            "version": "==0.9.22"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.10.2"
         },
         "odc-geo": {
             "hashes": [
@@ -1979,11 +1943,11 @@
         },
         "param": {
             "hashes": [
-                "sha256:0153a06fb0049fab73132d7faf4c468d44ad77ee00d531702b0bf222c069f9b6",
-                "sha256:19823383b0dac5a794152b85336cb4c6b0d0ef61a2b1eb7cd272aae141be0117"
+                "sha256:4bfc94c0e4127626fa833e30c71c91ea73e7675b80c26dbdd4a6e5a8f6dc46db",
+                "sha256:7943a04607822efd46e96e1827dc5fa929a2fc3b1fe9fc7b7dca7d17a8031a5b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.0.0"
+            "version": "==2.0.1"
         },
         "parso": {
             "hashes": [
@@ -2003,26 +1967,19 @@
         },
         "pathspec": {
             "hashes": [
-                "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20",
-                "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"
+                "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08",
+                "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.11.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.12.1"
         },
         "pexpect": {
             "hashes": [
-                "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
-                "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"
+                "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523",
+                "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"
             ],
             "markers": "sys_platform != 'win32'",
-            "version": "==4.8.0"
-        },
-        "pickleshare": {
-            "hashes": [
-                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
-                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
-            ],
-            "version": "==0.7.5"
+            "version": "==4.9.0"
         },
         "pillow": {
             "hashes": [
@@ -2084,14 +2041,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==10.1.0"
         },
-        "pkgutil-resolve-name": {
-            "hashes": [
-                "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174",
-                "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"
-            ],
-            "markers": "python_version < '3.9'",
-            "version": "==1.3.10"
-        },
         "platformdirs": {
             "hashes": [
                 "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380",
@@ -2126,33 +2075,33 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
-                "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
+                "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
+                "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.39"
+            "version": "==3.0.43"
         },
         "psutil": {
             "hashes": [
-                "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28",
-                "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017",
-                "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602",
-                "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac",
-                "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a",
-                "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9",
-                "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4",
-                "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c",
-                "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c",
-                "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c",
-                "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a",
-                "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c",
-                "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57",
-                "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a",
-                "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d",
-                "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"
+                "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340",
+                "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6",
+                "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284",
+                "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c",
+                "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7",
+                "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c",
+                "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e",
+                "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6",
+                "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056",
+                "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9",
+                "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68",
+                "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df",
+                "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e",
+                "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414",
+                "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508",
+                "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"
             ],
             "markers": "sys_platform != 'cygwin'",
-            "version": "==5.9.6"
+            "version": "==5.9.7"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -2292,7 +2241,6 @@
                 "sha256:fada8396bc739d958d0b81d291cfd201126ed5e7913cb73de6bc606befc30226"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==14.0.1"
         },
         "pybcj": {
@@ -2359,41 +2307,42 @@
         },
         "pycryptodomex": {
             "hashes": [
-                "sha256:09c9401dc06fb3d94cb1ec23b4ea067a25d1f4c6b7b118ff5631d0b5daaab3cc",
-                "sha256:0b2f1982c5bc311f0aab8c293524b861b485d76f7c9ab2c3ac9a25b6f7655975",
-                "sha256:136b284e9246b4ccf4f752d435c80f2c44fc2321c198505de1d43a95a3453b3c",
-                "sha256:1789d89f61f70a4cd5483d4dfa8df7032efab1118f8b9894faae03c967707865",
-                "sha256:2126bc54beccbede6eade00e647106b4f4c21e5201d2b0a73e9e816a01c50905",
-                "sha256:258c4233a3fe5a6341780306a36c6fb072ef38ce676a6d41eec3e591347919e8",
-                "sha256:263de9a96d2fcbc9f5bd3a279f14ea0d5f072adb68ebd324987576ec25da084d",
-                "sha256:50cb18d4dd87571006fd2447ccec85e6cec0136632a550aa29226ba075c80644",
-                "sha256:5b883e1439ab63af976656446fb4839d566bb096f15fc3c06b5a99cde4927188",
-                "sha256:5d73e9fa3fe830e7b6b42afc49d8329b07a049a47d12e0ef9225f2fd220f19b2",
-                "sha256:61056a1fd3254f6f863de94c233b30dd33bc02f8c935b2000269705f1eeeffa4",
-                "sha256:67c8eb79ab33d0fbcb56842992298ddb56eb6505a72369c20f60bc1d2b6fb002",
-                "sha256:6e45bb4635b3c4e0a00ca9df75ef6295838c85c2ac44ad882410cb631ed1eeaa",
-                "sha256:7cb51096a6a8d400724104db8a7e4f2206041a1f23e58924aa3d8d96bcb48338",
-                "sha256:800a2b05cfb83654df80266692f7092eeefe2a314fa7901dcefab255934faeec",
-                "sha256:8df69e41f7e7015a90b94d1096ec3d8e0182e73449487306709ec27379fff761",
-                "sha256:917033016ecc23c8933205585a0ab73e20020fdf671b7cd1be788a5c4039840b",
-                "sha256:a12144d785518f6491ad334c75ccdc6ad52ea49230b4237f319dbb7cef26f464",
-                "sha256:a3866d68e2fc345162b1b9b83ef80686acfe5cec0d134337f3b03950a0a8bf56",
-                "sha256:a588a1cb7781da9d5e1c84affd98c32aff9c89771eac8eaa659d2760666f7139",
-                "sha256:a77b79852175064c822b047fee7cf5a1f434f06ad075cc9986aa1c19a0c53eb0",
-                "sha256:af83a554b3f077564229865c45af0791be008ac6469ef0098152139e6bd4b5b6",
-                "sha256:b801216c48c0886742abf286a9a6b117e248ca144d8ceec1f931ce2dd0c9cb40",
-                "sha256:bfb040b5dda1dff1e197d2ef71927bd6b8bfcb9793bc4dfe0bb6df1e691eaacb",
-                "sha256:c01678aee8ac0c1a461cbc38ad496f953f9efcb1fa19f5637cbeba7544792a53",
-                "sha256:c74eb1f73f788facece7979ce91594dc177e1a9b5d5e3e64697dd58299e5cb4d",
-                "sha256:c9a68a2f7bd091ccea54ad3be3e9d65eded813e6d79fdf4cc3604e26cdd6384f",
-                "sha256:d4dd3b381ff5a5907a3eb98f5f6d32c64d319a840278ceea1dcfcc65063856f3",
-                "sha256:e8e5ecbd4da4157889fce8ba49da74764dd86c891410bfd6b24969fa46edda51",
-                "sha256:eb2fc0ec241bf5e5ef56c8fbec4a2634d631e4c4f616a59b567947a0f35ad83c",
-                "sha256:edbe083c299835de7e02c8aa0885cb904a75087d35e7bab75ebe5ed336e8c3e2",
-                "sha256:ff64fd720def623bf64d8776f8d0deada1cc1bf1ec3c1f9d6f5bb5bd098d034f"
+                "sha256:011e859026ecbd15b8e720e8992361186e582cf726c50bde6ff8c0c05e820ddf",
+                "sha256:0b42e2743893f386dfb58fe24a4c8be5305c3d1c825d5f23d9e63fd0700d1110",
+                "sha256:0b7154aff2272962355f8941fd514104a88cb29db2d8f43a29af900d6398eb1c",
+                "sha256:0bc4b7bfaac56e6dfd62044847443a3d110c7abea7fcb0d68c1aea64ed3a6697",
+                "sha256:10c2eed4efdfa084b602ab922e699a0a2ba82053baebfc8afcaf27489def7955",
+                "sha256:1c04cfff163c05d033bf28e3c4429d8222796738c7b6c1638b9d7090b904611e",
+                "sha256:23707238b024b36c35dd3428f5af6c1f0c5ef54c21e387a2063633717699b8b2",
+                "sha256:371bbe0be17b4dd8cc0c2f378d75ea33f00d5a39884c09a672016ac40145a5fa",
+                "sha256:39eb1f82ac3ba3e39d866f38e480e8fa53fcdd22260340f05f54a8188d47d510",
+                "sha256:3f3c58971784fba0e014bc3f8aed1197b86719631e1b597d36d7354be5598312",
+                "sha256:5ca98de2e5ac100e57a7116309723360e8f799f722509e376dc396cdf65eec9c",
+                "sha256:62f51a63d73153482729904381dd2de86800b0733a8814ee8f072fa73e5c92fb",
+                "sha256:76414d39df6b45bcc4f38cf1ba2031e0f4b8e99d1ba3c2eee31ffe1b9f039733",
+                "sha256:8dffe067d5fff14dba4d18ff7d459cc2a47576d82dafbff13a8f1199c3353e41",
+                "sha256:96000b837bcd8e3bf86b419924a056c978e45027281e4318650c81c25a3ef6cc",
+                "sha256:9919a1edd2a83c4dfb69f1d8a4c0c5efde7147ef15b07775633372b80c90b5d8",
+                "sha256:aab7941c2ff53eb63cb26252770e4f14386d79ce07baeffbf98a1323c1646545",
+                "sha256:ac562e239d98cfef763866c0aee4586affb0d58c592202f06c87241af99db241",
+                "sha256:ae75eea2e908383fd4c659fdcfe9621a72869e3e3ee73904227e93b7f7b80b54",
+                "sha256:b5c336dc698650283ad06f8c0237a984087d0af9f403ff21d633507335628156",
+                "sha256:beb5f0664f49b6093da179ee8e27c1d670779f50b9ece0886ce491bb8bd63728",
+                "sha256:c1ae2fb8d5d6771670436dcc889b293e363c97647a6d31c21eebc12b7b760010",
+                "sha256:c9332b04bf3f838327087b028f690f4ddb9341eb014a0221e79b9c19a77f7555",
+                "sha256:c9cb88ed323be1aa642b3c17cd5caa1a03c3a8fbad092d48ecefe88e328ffae3",
+                "sha256:d45d0d35a238d838b872598fa865bbfb31aaef9aeeda77c68b04ef79f9a469dc",
+                "sha256:d7a77391fd351ff1bdf8475558ddc6e92950218cb905419ee14aa02f370f1054",
+                "sha256:de5a43901e47e7a6938490fc5de3074f6e35c8b481a75b227c0d24d6099bd41d",
+                "sha256:e94a7e986b117b72e9472f8eafdd81748dafff30815401f9760f759f1debe9ef",
+                "sha256:ed3bdda44cc05dd13eee697ab9bea6928531bb7b218e68e66d0d3eb2ebab043e",
+                "sha256:f24f49fc6bd706d87048654d6be6c7c967d6836d4879e3a7c439275fab9948ad",
+                "sha256:f8a97b1acd36e9ce9d4067d94a8be99c458f0eb8070828639302a95cfcf0770b",
+                "sha256:f8b3d9e7c17c1ffc1fa5b11c0bbab8a5df3de8596bb32ad30281b21e5ede4bf5"
             ],
+            "index": "pypi",
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==3.19.0"
+            "version": "==3.19.1"
         },
         "pyct": {
             "hashes": [
@@ -2453,6 +2402,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==3.0.1"
         },
+        "pyflwdir": {
+            "hashes": [
+                "sha256:07f5499e976e3b45ea5750869dc6c3b85a77aa674ac73c45f7da0a2e9f7e66db",
+                "sha256:be059dcade8bde51102e7e832903e76bec391b8bb460ca571895f511f4c18e4d"
+            ],
+            "index": "pypi",
+            "version": "==0.5.8"
+        },
         "pygments": {
             "hashes": [
                 "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
@@ -2901,11 +2858,11 @@
         },
         "qtconsole": {
             "hashes": [
-                "sha256:6b6bcf8f834c6df1579a3e6623c8531b85d3e723997cee3a1156296df14716c8",
-                "sha256:ea8b4a07d7dc915a1b1238fbfe2c9aea570640402557b64615e09a4bc60df47c"
+                "sha256:8c75fa3e9b4ed884880ff7cea90a1b67451219279ec33deaee1d59e3df1a5d2b",
+                "sha256:a0e806c6951db9490628e4df80caec9669b65149c7ba40f9bf033c025a5b56bc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.5.0"
+            "version": "==5.5.1"
         },
         "qtpy": {
             "hashes": [
@@ -2948,11 +2905,11 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:81a1471c68c9d5e3831c30ad1dd9815c45b558e596653db751a2bfdd17b3b9ec",
-                "sha256:c19c4d006f1757e3dd75c4f784d38f8698d87b649c54f9ace14e5e8c9667c01d"
+                "sha256:689e64fe121843dcfd57b71933318ef1f91188ffb45367332700a86ac8fd6161",
+                "sha256:bdcd3efb936f82ff86f993093f6da7435c7de69a3b3a5a06678a6050184bee99"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.31.1"
+            "version": "==0.32.0"
         },
         "requests": {
             "hashes": [
@@ -3007,116 +2964,116 @@
         },
         "rioxarray": {
             "hashes": [
-                "sha256:0cad24ad2c3c5ee181a0cfad2b8c2152a609b7eb118a3430034aec171e9cf14f",
-                "sha256:56eef711d9817d3c729c1a267c940e7dff66bfc874a0b24ed3604ea2f958dfb2"
+                "sha256:d2a8429a5b6405913c7b6f515ef2992b05139c96eb39a2dc1c9f475ce0848c9c",
+                "sha256:d7c0b2efc21075f77fe04302b916a995320004695f3c31e4f06d9ab40acd4498"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==0.13.4"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.15.0"
         },
         "rpds-py": {
             "hashes": [
-                "sha256:06d218e4464d31301e943b65b2c6919318ea6f69703a351961e1baaf60347276",
-                "sha256:12ecf89bd54734c3c2c79898ae2021dca42750c7bcfb67f8fb3315453738ac8f",
-                "sha256:15253fff410873ebf3cfba1cc686a37711efcd9b8cb30ea21bb14a973e393f60",
-                "sha256:188435794405c7f0573311747c85a96b63c954a5f2111b1df8018979eca0f2f0",
-                "sha256:1ceebd0ae4f3e9b2b6b553b51971921853ae4eebf3f54086be0565d59291e53d",
-                "sha256:244e173bb6d8f3b2f0c4d7370a1aa341f35da3e57ffd1798e5b2917b91731fd3",
-                "sha256:25b28b3d33ec0a78e944aaaed7e5e2a94ac811bcd68b557ca48a0c30f87497d2",
-                "sha256:25ea41635d22b2eb6326f58e608550e55d01df51b8a580ea7e75396bafbb28e9",
-                "sha256:29d311e44dd16d2434d5506d57ef4d7036544fc3c25c14b6992ef41f541b10fb",
-                "sha256:2a1472956c5bcc49fb0252b965239bffe801acc9394f8b7c1014ae9258e4572b",
-                "sha256:2a7bef6977043673750a88da064fd513f89505111014b4e00fbdd13329cd4e9a",
-                "sha256:2ac26f50736324beb0282c819668328d53fc38543fa61eeea2c32ea8ea6eab8d",
-                "sha256:2e72f750048b32d39e87fc85c225c50b2a6715034848dbb196bf3348aa761fa1",
-                "sha256:31e220a040b89a01505128c2f8a59ee74732f666439a03e65ccbf3824cdddae7",
-                "sha256:35f53c76a712e323c779ca39b9a81b13f219a8e3bc15f106ed1e1462d56fcfe9",
-                "sha256:38d4f822ee2f338febcc85aaa2547eb5ba31ba6ff68d10b8ec988929d23bb6b4",
-                "sha256:38f9bf2ad754b4a45b8210a6c732fe876b8a14e14d5992a8c4b7c1ef78740f53",
-                "sha256:3a44c8440183b43167fd1a0819e8356692bf5db1ad14ce140dbd40a1485f2dea",
-                "sha256:3ab96754d23372009638a402a1ed12a27711598dd49d8316a22597141962fe66",
-                "sha256:3c55d7f2d817183d43220738270efd3ce4e7a7b7cbdaefa6d551ed3d6ed89190",
-                "sha256:46e1ed994a0920f350a4547a38471217eb86f57377e9314fbaaa329b71b7dfe3",
-                "sha256:4a5375c5fff13f209527cd886dc75394f040c7d1ecad0a2cb0627f13ebe78a12",
-                "sha256:4c2d26aa03d877c9730bf005621c92da263523a1e99247590abbbe252ccb7824",
-                "sha256:4c4e314d36d4f31236a545696a480aa04ea170a0b021e9a59ab1ed94d4c3ef27",
-                "sha256:4d0c10d803549427f427085ed7aebc39832f6e818a011dcd8785e9c6a1ba9b3e",
-                "sha256:4dcc5ee1d0275cb78d443fdebd0241e58772a354a6d518b1d7af1580bbd2c4e8",
-                "sha256:51967a67ea0d7b9b5cd86036878e2d82c0b6183616961c26d825b8c994d4f2c8",
-                "sha256:530190eb0cd778363bbb7596612ded0bb9fef662daa98e9d92a0419ab27ae914",
-                "sha256:5379e49d7e80dca9811b36894493d1c1ecb4c57de05c36f5d0dd09982af20211",
-                "sha256:5493569f861fb7b05af6d048d00d773c6162415ae521b7010197c98810a14cab",
-                "sha256:5a4c1058cdae6237d97af272b326e5f78ee7ee3bbffa6b24b09db4d828810468",
-                "sha256:5d75d6d220d55cdced2f32cc22f599475dbe881229aeddba6c79c2e9df35a2b3",
-                "sha256:5d97e9ae94fb96df1ee3cb09ca376c34e8a122f36927230f4c8a97f469994bff",
-                "sha256:5feae2f9aa7270e2c071f488fab256d768e88e01b958f123a690f1cc3061a09c",
-                "sha256:603d5868f7419081d616dab7ac3cfa285296735e7350f7b1e4f548f6f953ee7d",
-                "sha256:61d42d2b08430854485135504f672c14d4fc644dd243a9c17e7c4e0faf5ed07e",
-                "sha256:61dbc1e01dc0c5875da2f7ae36d6e918dc1b8d2ce04e871793976594aad8a57a",
-                "sha256:65cfed9c807c27dee76407e8bb29e6f4e391e436774bcc769a037ff25ad8646e",
-                "sha256:67a429520e97621a763cf9b3ba27574779c4e96e49a27ff8a1aa99ee70beb28a",
-                "sha256:6aadae3042f8e6db3376d9e91f194c606c9a45273c170621d46128f35aef7cd0",
-                "sha256:6ba8858933f0c1a979781272a5f65646fca8c18c93c99c6ddb5513ad96fa54b1",
-                "sha256:6bc568b05e02cd612be53900c88aaa55012e744930ba2eeb56279db4c6676eb3",
-                "sha256:729408136ef8d45a28ee9a7411917c9e3459cf266c7e23c2f7d4bb8ef9e0da42",
-                "sha256:751758d9dd04d548ec679224cc00e3591f5ebf1ff159ed0d4aba6a0746352452",
-                "sha256:76d59d4d451ba77f08cb4cd9268dec07be5bc65f73666302dbb5061989b17198",
-                "sha256:79bf58c08f0756adba691d480b5a20e4ad23f33e1ae121584cf3a21717c36dfa",
-                "sha256:7de12b69d95072394998c622cfd7e8cea8f560db5fca6a62a148f902a1029f8b",
-                "sha256:7f55cd9cf1564b7b03f238e4c017ca4794c05b01a783e9291065cb2858d86ce4",
-                "sha256:80e5acb81cb49fd9f2d5c08f8b74ffff14ee73b10ca88297ab4619e946bcb1e1",
-                "sha256:87a90f5545fd61f6964e65eebde4dc3fa8660bb7d87adb01d4cf17e0a2b484ad",
-                "sha256:881df98f0a8404d32b6de0fd33e91c1b90ed1516a80d4d6dc69d414b8850474c",
-                "sha256:8a776a29b77fe0cc28fedfd87277b0d0f7aa930174b7e504d764e0b43a05f381",
-                "sha256:8c2a61c0e4811012b0ba9f6cdcb4437865df5d29eab5d6018ba13cee1c3064a0",
-                "sha256:8fa6bd071ec6d90f6e7baa66ae25820d57a8ab1b0a3c6d3edf1834d4b26fafa2",
-                "sha256:96f2975fb14f39c5fe75203f33dd3010fe37d1c4e33177feef1107b5ced750e3",
-                "sha256:96fb0899bb2ab353f42e5374c8f0789f54e0a94ef2f02b9ac7149c56622eaf31",
-                "sha256:97163a1ab265a1073a6372eca9f4eeb9f8c6327457a0b22ddfc4a17dcd613e74",
-                "sha256:9c95a1a290f9acf7a8f2ebbdd183e99215d491beea52d61aa2a7a7d2c618ddc6",
-                "sha256:9d94d78418203904730585efa71002286ac4c8ac0689d0eb61e3c465f9e608ff",
-                "sha256:a6ba2cb7d676e9415b9e9ac7e2aae401dc1b1e666943d1f7bc66223d3d73467b",
-                "sha256:aa0379c1935c44053c98826bc99ac95f3a5355675a297ac9ce0dfad0ce2d50ca",
-                "sha256:ac96d67b37f28e4b6ecf507c3405f52a40658c0a806dffde624a8fcb0314d5fd",
-                "sha256:ade2ccb937060c299ab0dfb2dea3d2ddf7e098ed63ee3d651ebfc2c8d1e8632a",
-                "sha256:aefbdc934115d2f9278f153952003ac52cd2650e7313750390b334518c589568",
-                "sha256:b07501b720cf060c5856f7b5626e75b8e353b5f98b9b354a21eb4bfa47e421b1",
-                "sha256:b5267feb19070bef34b8dea27e2b504ebd9d31748e3ecacb3a4101da6fcb255c",
-                "sha256:b5f6328e8e2ae8238fc767703ab7b95785521c42bb2b8790984e3477d7fa71ad",
-                "sha256:b8996ffb60c69f677245f5abdbcc623e9442bcc91ed81b6cd6187129ad1fa3e7",
-                "sha256:b981a370f8f41c4024c170b42fbe9e691ae2dbc19d1d99151a69e2c84a0d194d",
-                "sha256:b9d121be0217787a7d59a5c6195b0842d3f701007333426e5154bf72346aa658",
-                "sha256:bcef4f2d3dc603150421de85c916da19471f24d838c3c62a4f04c1eb511642c1",
-                "sha256:bed0252c85e21cf73d2d033643c945b460d6a02fc4a7d644e3b2d6f5f2956c64",
-                "sha256:bfdfbe6a36bc3059fff845d64c42f2644cf875c65f5005db54f90cdfdf1df815",
-                "sha256:c0095b8aa3e432e32d372e9a7737e65b58d5ed23b9620fea7cb81f17672f1fa1",
-                "sha256:c1f41d32a2ddc5a94df4b829b395916a4b7f103350fa76ba6de625fcb9e773ac",
-                "sha256:c45008ca79bad237cbc03c72bc5205e8c6f66403773929b1b50f7d84ef9e4d07",
-                "sha256:c82bbf7e03748417c3a88c1b0b291288ce3e4887a795a3addaa7a1cfd9e7153e",
-                "sha256:c918621ee0a3d1fe61c313f2489464f2ae3d13633e60f520a8002a5e910982ee",
-                "sha256:d204957169f0b3511fb95395a9da7d4490fb361763a9f8b32b345a7fe119cb45",
-                "sha256:d329896c40d9e1e5c7715c98529e4a188a1f2df51212fd65102b32465612b5dc",
-                "sha256:d3a61e928feddc458a55110f42f626a2a20bea942ccedb6fb4cee70b4830ed41",
-                "sha256:d48db29bd47814671afdd76c7652aefacc25cf96aad6daefa82d738ee87461e2",
-                "sha256:d5593855b5b2b73dd8413c3fdfa5d95b99d657658f947ba2c4318591e745d083",
-                "sha256:d79c159adea0f1f4617f54aa156568ac69968f9ef4d1e5fefffc0a180830308e",
-                "sha256:db09b98c7540df69d4b47218da3fbd7cb466db0fb932e971c321f1c76f155266",
-                "sha256:ddf23960cb42b69bce13045d5bc66f18c7d53774c66c13f24cf1b9c144ba3141",
-                "sha256:e06cfea0ece444571d24c18ed465bc93afb8c8d8d74422eb7026662f3d3f779b",
-                "sha256:e7c564c58cf8f248fe859a4f0fe501b050663f3d7fbc342172f259124fb59933",
-                "sha256:e86593bf8637659e6a6ed58854b6c87ec4e9e45ee8a4adfd936831cef55c2d21",
-                "sha256:eaffbd8814bb1b5dc3ea156a4c5928081ba50419f9175f4fc95269e040eff8f0",
-                "sha256:ee353bb51f648924926ed05e0122b6a0b1ae709396a80eb583449d5d477fcdf7",
-                "sha256:ee6faebb265e28920a6f23a7d4c362414b3f4bb30607141d718b991669e49ddc",
-                "sha256:efe093acc43e869348f6f2224df7f452eab63a2c60a6c6cd6b50fd35c4e075ba",
-                "sha256:f03a1b3a4c03e3e0161642ac5367f08479ab29972ea0ffcd4fa18f729cd2be0a",
-                "sha256:f0d320e70b6b2300ff6029e234e79fe44e9dbbfc7b98597ba28e054bd6606a57",
-                "sha256:f252dfb4852a527987a9156cbcae3022a30f86c9d26f4f17b8c967d7580d65d2",
-                "sha256:f5f4424cb87a20b016bfdc157ff48757b89d2cc426256961643d443c6c277007",
-                "sha256:f8eae66a1304de7368932b42d801c67969fd090ddb1a7a24f27b435ed4bed68f",
-                "sha256:fdb82eb60d31b0c033a8e8ee9f3fc7dfbaa042211131c29da29aea8531b4f18f"
+                "sha256:02744236ac1895d7be837878e707a5c35fb8edc5137602f253b63623d7ad5c8c",
+                "sha256:03f9c5875515820633bd7709a25c3e60c1ea9ad1c5d4030ce8a8c203309c36fd",
+                "sha256:044f6f46d62444800402851afa3c3ae50141f12013060c1a3a0677e013310d6d",
+                "sha256:07a2e1d78d382f7181789713cdf0c16edbad4fe14fe1d115526cb6f0eef0daa3",
+                "sha256:082e0e55d73690ffb4da4352d1b5bbe1b5c6034eb9dc8c91aa2a3ee15f70d3e2",
+                "sha256:13152dfe7d7c27c40df8b99ac6aab12b978b546716e99f67e8a67a1d441acbc3",
+                "sha256:13716e53627ad97babf72ac9e01cf9a7d4af2f75dd5ed7b323a7a9520e948282",
+                "sha256:13ff62d3561a23c17341b4afc78e8fcfd799ab67c0b1ca32091d71383a98ba4b",
+                "sha256:1607cda6129f815493a3c184492acb5ae4aa6ed61d3a1b3663aa9824ed26f7ac",
+                "sha256:164fcee32f15d04d61568c9cb0d919e37ff3195919cd604039ff3053ada0461b",
+                "sha256:1c24e30d720c0009b6fb2e1905b025da56103c70a8b31b99138e4ed1c2a6c5b0",
+                "sha256:1e6fcd0a0f62f2997107f758bb372397b8d5fd5f39cc6dcb86f7cb98a2172d6c",
+                "sha256:1fd0f0b1ccd7d537b858a56355a250108df692102e08aa2036e1a094fd78b2dc",
+                "sha256:2181e86d4e1cdf49a7320cb72a36c45efcb7670d0a88f09fd2d3a7967c0540fd",
+                "sha256:2974e6dff38afafd5ccf8f41cb8fc94600b3f4fd9b0a98f6ece6e2219e3158d5",
+                "sha256:2dccc623725d0b298f557d869a68496a2fd2a9e9c41107f234fa5f7a37d278ac",
+                "sha256:2df3d07a16a3bef0917b28cd564778fbb31f3ffa5b5e33584470e2d1b0f248f0",
+                "sha256:2e7e5633577b3bd56bf3af2ef6ae3778bbafb83743989d57f0e7edbf6c0980e4",
+                "sha256:2ee066a64f0d2ba45391cac15b3a70dcb549e968a117bd0500634754cfe0e5fc",
+                "sha256:2f1f295a5c28cfa74a7d48c95acc1c8a7acd49d7d9072040d4b694fe11cd7166",
+                "sha256:2faa97212b0dc465afeedf49045cdd077f97be1188285e646a9f689cb5dfff9e",
+                "sha256:30479a9f1fce47df56b07460b520f49fa2115ec2926d3b1303c85c81f8401ed1",
+                "sha256:337a8653fb11d2fbe7157c961cc78cb3c161d98cf44410ace9a3dc2db4fad882",
+                "sha256:3423007fc0661827e06f8a185a3792c73dda41f30f3421562f210cf0c9e49569",
+                "sha256:373b76eeb79e8c14f6d82cb1d4d5293f9e4059baec6c1b16dca7ad13b6131b39",
+                "sha256:3b79c63d29101cbaa53a517683557bb550462394fb91044cc5998dd2acff7340",
+                "sha256:3bbc89ce2a219662ea142f0abcf8d43f04a41d5b1880be17a794c39f0d609cb0",
+                "sha256:3c11bc5814554b018f6c5d6ae0969e43766f81e995000b53a5d8c8057055e886",
+                "sha256:3cd61e759c4075510052d1eca5cddbd297fe1164efec14ef1fce3f09b974dfe4",
+                "sha256:3d40fb3ca22e3d40f494d577441b263026a3bd8c97ae6ce89b2d3c4b39ac9581",
+                "sha256:3db0c998c92b909d7c90b66c965590d4f3cd86157176a6cf14aa1f867b77b889",
+                "sha256:422b0901878a31ef167435c5ad46560362891816a76cc0d150683f3868a6f0d1",
+                "sha256:46b4f3d47d1033db569173be62365fbf7808c2bd3fb742314d251f130d90d44c",
+                "sha256:485fbdd23becb822804ed05622907ee5c8e8a5f43f6f43894a45f463b2217045",
+                "sha256:53304cc14b1d94487d70086e1cb0cb4c29ec6da994d58ae84a4d7e78c6a6d04d",
+                "sha256:5595c80dd03d7e6c6afb73f3594bf3379a7d79fa57164b591d012d4b71d6ac4c",
+                "sha256:56b51ba29a18e5f5810224bcf00747ad931c0716e3c09a76b4a1edd3d4aba71f",
+                "sha256:580182fa5b269c2981e9ce9764367cb4edc81982ce289208d4607c203f44ffde",
+                "sha256:5e99d6510c8557510c220b865d966b105464740dcbebf9b79ecd4fbab30a13d9",
+                "sha256:5eb05b654a41e0f81ab27a7c3e88b6590425eb3e934e1d533ecec5dc88a6ffff",
+                "sha256:62b292fff4739c6be89e6a0240c02bda5a9066a339d90ab191cf66e9fdbdc193",
+                "sha256:6a5122b17a4faf5d7a6d91fa67b479736c0cacc7afe791ddebb7163a8550b799",
+                "sha256:6a8ff8e809da81363bffca2b965cb6e4bf6056b495fc3f078467d1f8266fe27f",
+                "sha256:6c43e1b89099279cc03eb1c725c5de12af6edcd2f78e2f8a022569efa639ada3",
+                "sha256:709dc11af2f74ba89c68b1592368c6edcbccdb0a06ba77eb28c8fe08bb6997da",
+                "sha256:7e072f5da38d6428ba1fc1115d3cc0dae895df671cb04c70c019985e8c7606be",
+                "sha256:813a65f95bfcb7c8f2a70dd6add9b51e9accc3bdb3e03d0ff7a9e6a2d3e174bf",
+                "sha256:86c01299942b0f4b5b5f28c8701689181ad2eab852e65417172dbdd6c5b3ccc8",
+                "sha256:893e38d0f4319dfa70c0f36381a37cc418985c87b11d9784365b1fff4fa6973b",
+                "sha256:8a5f574b92b3ee7d254e56d56e37ec0e1416acb1ae357c4956d76a1788dc58fb",
+                "sha256:8b9650f92251fdef843e74fc252cdfd6e3c700157ad686eeb0c6d7fdb2d11652",
+                "sha256:8ec464f20fe803ae00419bd1610934e3bda963aeba1e6181dfc9033dc7e8940c",
+                "sha256:8f333bfe782a2d05a67cfaa0cc9cd68b36b39ee6acfe099f980541ed973a7093",
+                "sha256:8ffdeb7dbd0160d4e391e1f857477e4762d00aa2199c294eb95dfb9451aa1d9f",
+                "sha256:911e600e798374c0d86235e7ef19109cf865d1336942d398ff313375a25a93ba",
+                "sha256:9235be95662559141934fced8197de6fee8c58870f36756b0584424b6d708393",
+                "sha256:938518a11780b39998179d07f31a4a468888123f9b00463842cd40f98191f4d3",
+                "sha256:93c18a1696a8e0388ed84b024fe1a188a26ba999b61d1d9a371318cb89885a8c",
+                "sha256:97532802f14d383f37d603a56e226909f825a83ff298dc1b6697de00d2243999",
+                "sha256:98ee201a52a7f65608e5494518932e1473fd43535f12cade0a1b4ab32737fe28",
+                "sha256:9d2ae79f31da5143e020a8d4fc74e1f0cbcb8011bdf97453c140aa616db51406",
+                "sha256:9d38494a8d21c246c535b41ecdb2d562c4b933cf3d68de03e8bc43a0d41be652",
+                "sha256:9d41ebb471a6f064c0d1c873c4f7dded733d16ca5db7d551fb04ff3805d87802",
+                "sha256:9e09d017e3f4d9bd7d17a30d3f59e4d6d9ba2d2ced280eec2425e84112cf623f",
+                "sha256:a6945c2d61c42bb7e818677f43638675b8c1c43e858b67a96df3eb2426a86c9d",
+                "sha256:a72e00826a2b032dda3eb25aa3e3579c6d6773d22d8446089a57a123481cc46c",
+                "sha256:aa1e626c524d2c7972c0f3a8a575d654a3a9c008370dc2a97e46abd0eaa749b9",
+                "sha256:ab095edf1d840a6a6a4307e1a5b907a299a94e7b90e75436ee770b8c35d22a25",
+                "sha256:ac2ac84a4950d627d84b61f082eba61314373cfab4b3c264b62efab02ababe83",
+                "sha256:ac7187bee72384b9cfedf09a29a3b2b6e8815cc64c095cdc8b5e6aec81e9fd5f",
+                "sha256:ae9d83a81b09ce3a817e2cbb23aabc07f86a3abc664c613cd283ce7a03541e95",
+                "sha256:afeabb382c1256a7477b739820bce7fe782bb807d82927102cee73e79b41b38b",
+                "sha256:b2a4cd924d0e2f4b1a68034abe4cadc73d69ad5f4cf02db6481c0d4d749f548f",
+                "sha256:b414ef79f1f06fb90b5165db8aef77512c1a5e3ed1b4807da8476b7e2c853283",
+                "sha256:b4ecbba7efd82bd2a4bb88aab7f984eb5470991c1347bdd1f35fb34ea28dba6e",
+                "sha256:b61d5096e75fd71018b25da50b82dd70ec39b5e15bb2134daf7eb7bbbc103644",
+                "sha256:b629db53fe17e6ce478a969d30bd1d0e8b53238c46e3a9c9db39e8b65a9ef973",
+                "sha256:b70b45a40ad0798b69748b34d508259ef2bdc84fb2aad4048bc7c9cafb68ddb3",
+                "sha256:b88c3ab98556bc351b36d6208a6089de8c8db14a7f6e1f57f82a334bd2c18f0b",
+                "sha256:baf744e5f9d5ee6531deea443be78b36ed1cd36c65a0b95ea4e8d69fa0102268",
+                "sha256:bbc7421cbd28b4316d1d017db338039a7943f945c6f2bb15e1439b14b5682d28",
+                "sha256:c31272c674f725dfe0f343d73b0abe8c878c646967ec1c6106122faae1efc15b",
+                "sha256:c51a899792ee2c696072791e56b2020caff58b275abecbc9ae0cb71af0645c95",
+                "sha256:c61e42b4ceb9759727045765e87d51c1bb9f89987aca1fcc8a040232138cad1c",
+                "sha256:c7cd0841a586b7105513a7c8c3d5c276f3adc762a072d81ef7fae80632afad1e",
+                "sha256:c827a931c6b57f50f1bb5de400dcfb00bad8117e3753e80b96adb72d9d811514",
+                "sha256:d2aa3ca9552f83b0b4fa6ca8c6ce08da6580f37e3e0ab7afac73a1cfdc230c0e",
+                "sha256:d46ee458452727a147d7897bb33886981ae1235775e05decae5d5d07f537695a",
+                "sha256:d64a657de7aae8db2da60dc0c9e4638a0c3893b4d60101fd564a3362b2bfeb34",
+                "sha256:d800a8e2ac62db1b9ea5d6d1724f1a93c53907ca061de4d05ed94e8dfa79050c",
+                "sha256:d9d7ebcd11ea76ba0feaae98485cd8e31467c3d7985210fab46983278214736b",
+                "sha256:dd7d3608589072f63078b4063a6c536af832e76b0b3885f1bfe9e892abe6c207",
+                "sha256:ec19e823b4ccd87bd69e990879acbce9e961fc7aebe150156b8f4418d4b27b7f",
+                "sha256:ee40206d1d6e95eaa2b7b919195e3689a5cf6ded730632de7f187f35a1b6052c",
+                "sha256:f138f550b83554f5b344d6be35d3ed59348510edc3cb96f75309db6e9bfe8210",
+                "sha256:f3e6e2e502c4043c52a99316d89dc49f416acda5b0c6886e0dd8ea7bb35859e8",
+                "sha256:fb10bb720348fe1647a94eb605accb9ef6a9b1875d8845f9e763d9d71a706387",
+                "sha256:fc066395e6332da1e7525d605b4c96055669f8336600bef8ac569d5226a7c76f",
+                "sha256:fc33267d58dfbb2361baed52668c5d8c15d24bc0372cecbb79fed77339b55e0d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.13.2"
+            "version": "==0.15.2"
         },
         "rtree": {
             "hashes": [
@@ -3222,11 +3179,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87",
-                "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"
+                "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2",
+                "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==68.2.2"
+            "version": "==69.0.2"
         },
         "shapely": {
             "hashes": [
@@ -3526,11 +3483,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0",
-                "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"
+                "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783",
+                "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"
             ],
-            "markers": "python_version < '3.10'",
-            "version": "==4.8.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.9.0"
         },
         "typing-inspect": {
             "hashes": [
@@ -3564,18 +3521,18 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af",
-                "sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381"
+                "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3",
+                "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.24.6"
+            "version": "==20.25.0"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:9a929bd8380f6cd9571a968a9c8f4353ca58d7cd812a4822bba831f8d685b223",
-                "sha256:a675d1a4a2d24ef67096a04b85b02deeecd8e226f57b5e3a72dbb9ed99d27da8"
+                "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02",
+                "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"
             ],
-            "version": "==0.2.9"
+            "version": "==0.2.12"
         },
         "webcolors": {
             "hashes": [
@@ -3617,84 +3574,79 @@
         },
         "wrapt": {
             "hashes": [
-                "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0",
-                "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420",
-                "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a",
-                "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c",
-                "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079",
-                "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923",
-                "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f",
-                "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1",
-                "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8",
-                "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86",
-                "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0",
-                "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364",
-                "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e",
-                "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c",
-                "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e",
-                "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c",
-                "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727",
-                "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff",
-                "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e",
-                "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29",
-                "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7",
-                "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72",
-                "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475",
-                "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a",
-                "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317",
-                "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2",
-                "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd",
-                "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640",
-                "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98",
-                "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248",
-                "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e",
-                "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d",
-                "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec",
-                "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1",
-                "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e",
-                "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9",
-                "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92",
-                "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb",
-                "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094",
-                "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46",
-                "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29",
-                "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd",
-                "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705",
-                "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8",
-                "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975",
-                "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb",
-                "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e",
-                "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b",
-                "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418",
-                "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019",
-                "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1",
-                "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba",
-                "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6",
-                "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2",
-                "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3",
-                "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7",
-                "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752",
-                "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416",
-                "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f",
-                "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1",
-                "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc",
-                "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145",
-                "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee",
-                "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a",
-                "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7",
-                "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b",
-                "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653",
-                "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0",
-                "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90",
-                "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29",
-                "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6",
-                "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034",
-                "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09",
-                "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559",
-                "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"
+                "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc",
+                "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81",
+                "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09",
+                "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e",
+                "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca",
+                "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0",
+                "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb",
+                "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487",
+                "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40",
+                "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c",
+                "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060",
+                "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202",
+                "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41",
+                "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9",
+                "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b",
+                "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664",
+                "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d",
+                "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362",
+                "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00",
+                "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc",
+                "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1",
+                "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267",
+                "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956",
+                "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966",
+                "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1",
+                "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228",
+                "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72",
+                "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d",
+                "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292",
+                "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0",
+                "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0",
+                "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36",
+                "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c",
+                "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5",
+                "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f",
+                "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73",
+                "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b",
+                "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2",
+                "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593",
+                "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39",
+                "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389",
+                "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf",
+                "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf",
+                "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89",
+                "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c",
+                "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c",
+                "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f",
+                "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440",
+                "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465",
+                "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136",
+                "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b",
+                "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8",
+                "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3",
+                "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8",
+                "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6",
+                "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e",
+                "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f",
+                "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c",
+                "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e",
+                "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8",
+                "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2",
+                "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020",
+                "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35",
+                "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d",
+                "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3",
+                "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537",
+                "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809",
+                "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d",
+                "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a",
+                "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.15.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==1.16.0"
         },
         "xarray": {
             "hashes": [
@@ -3849,11 +3801,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363",
-                "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c"
+                "sha256:2f55d59370f59d0d2b2212109fe0e6035cfea436b1c0e6150ad2244746272ec5",
+                "sha256:ac4da4ecf0042fb4e0ce57c60430c2db3c719fa8bdf92f8631d6bd8a5785d1f0"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==8.12.3"
+            "markers": "python_version >= '3.10'",
+            "version": "==8.19.0"
         },
         "jedi": {
             "hashes": [
@@ -3881,11 +3833,11 @@
         },
         "pexpect": {
             "hashes": [
-                "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
-                "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"
+                "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523",
+                "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"
             ],
             "markers": "sys_platform != 'win32'",
-            "version": "==4.8.0"
+            "version": "==4.9.0"
         },
         "pickleshare": {
             "hashes": [
@@ -3896,17 +3848,18 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
-                "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
+                "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
+                "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.39"
+            "version": "==3.0.43"
         },
         "ptyprocess": {
             "hashes": [
                 "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35",
                 "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"
             ],
+            "markers": "os_name != 'nt'",
             "version": "==0.7.0"
         },
         "pure-eval": {
@@ -3918,11 +3871,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
-                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
+                "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
+                "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.16.1"
+            "version": "==2.17.2"
         },
         "six": {
             "hashes": [
@@ -3941,26 +3894,18 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:9b232b9430c8f57288c1024b34a8f0251ddcc47268927367a0dd3eeaca40deb5",
-                "sha256:baf991e61542da48fe8aef8b779a9ea0aa38d8a54166ee250d5af5ecf4486619"
+                "sha256:f14949d23829023013c47df20b4a76ccd1a85effb786dc060f34de7948361b33",
+                "sha256:fcdaa8ac49c04dfa0ed3ee3384ef6dfdb5d6f3741502be247279407679296772"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.13.0"
-        },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0",
-                "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"
-            ],
-            "markers": "python_version < '3.10'",
-            "version": "==4.8.0"
+            "version": "==5.14.0"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:9a929bd8380f6cd9571a968a9c8f4353ca58d7cd812a4822bba831f8d685b223",
-                "sha256:a675d1a4a2d24ef67096a04b85b02deeecd8e226f57b5e3a72dbb9ed99d27da8"
+                "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02",
+                "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"
             ],
-            "version": "==0.2.9"
+            "version": "==0.2.12"
         }
     }
 }
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e514eb23c..be5e9c1be 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.9.0 - 2024-01-12 - [PR#1058](https://github.com/NOAA-OWP/inundation-mapping/pull/1058)
+
+Upgrades base Docker image to GDAL v3.8.0. In order to upgrade past GDAL v.3.4.3 (see #1029), TauDEM's `aread8` was replaced with a module from the `pyflwdir` Python package.
+
+### Additions
+
+- `src/accumulate_headwaters.py`: Uses `pyflwdir` to accumulate headwaters and threshold and create stream pixels.
+
+### Changes
+
+- `Dockerfile`: Upgrade GDAL from v.3.4.3 to v.3.8.0; remove JDK 17 and TauDEM `aread8` and `threshold`.
+- `Pipfile` and `Pipfile.lock`: Add `pyflwdir`, `pycryptodomex` and upgrade Python version.
+- `src/delineate_hydros_and_produce_HAND.sh`: Add `src/accumulate_headwaters.py` and remove TauDEM `aread8` and `threshold`
+
+<br/><br/>
+
 ## v4.4.8.4 - 2024-01-12 - [PR#1061](https://github.com/NOAA-OWP/inundation-mapping/pull/1061)
 
 Adds a post-processing tool to compare crosswalked (conflated) `feature_id`s between NWM stream network to DEM-derived reaches. The tool is run if the `-x` flag is added to `fim_pipeline.sh`. Results are computed for branch 0 and saved in a summary file in the HUC output folder.
@@ -37,6 +53,7 @@ Levees disrupt the natural hydrology and can create large catchments that contai
 
 <br/><br/>
 
+
 ## v4.4.8.2 - 2023-12-12 - [PR#1052](https://github.com/NOAA-OWP/inundation-mapping/pull/1052)
 
 The alpha test for v4.4.8.1 came back with a large degradation in skill and we noticed that the global manning's roughness file was changed in v4.4.7.1 - likely in error.
@@ -47,6 +64,7 @@ The alpha test for v4.4.8.1 came back with a large degradation in skill and we n
 
 <br/><br/>
 
+
 ## v4.4.8.1 - 2023-12-08 - [PR#1047](https://github.com/NOAA-OWP/inundation-mapping/pull/1047)
 
 Upgrades JDK to v.17.0.9 in Docker image to address security vulnerabilities.
diff --git a/src/accumulate_headwaters.py b/src/accumulate_headwaters.py
new file mode 100644
index 000000000..4dcb6fb4d
--- /dev/null
+++ b/src/accumulate_headwaters.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+
+import numpy as np
+import pyflwdir
+import rasterio as rio
+
+
+def accumulate_flow(
+    flow_direction_filename,
+    headwaters_filename,
+    flow_accumulation_filename,
+    stream_pixel_filename,
+    flow_accumulation_threshold,
+):
+    """
+    Accumulate headwaters along the flow direction and threshold accumulations to produce stream pixels.
+
+    Parameters
+    ----------
+    flow_direction_filename : str
+        Flow direction filename
+    headwaters_filename : str
+        Headwaters filename
+    flow_accumulation_filename : str
+        Flow accumulation filename
+    stream_pixel_filename : str
+        Stream pixel filename
+    flow_accumulation_threshold : str
+    """
+
+    assert os.path.isfile(flow_direction_filename), 'Flow direction raster does not exist.'
+
+    # Read the flow direction raster
+    with rio.open(flow_direction_filename) as src:
+        data = src.read(1)
+        nodata = src.nodata
+        profile = src.profile
+        # transform = src.transform
+        # crs = src.crs
+        # latlon = crs.to_epsg() == 4326
+
+    # Convert the TauDEM flow direction raster to a pyflwdir flow direction array
+    temp = data.copy()
+
+    temp[data == 1] = 1
+    temp[data == 2] = 128
+    temp[data == 3] = 64
+    temp[data == 4] = 32
+    temp[data == 5] = 16
+    temp[data == 6] = 8
+    temp[data == 7] = 4
+    temp[data == 8] = 2
+    temp[data == nodata] = 247
+
+    temp = temp.astype(np.uint8)
+
+    flw = pyflwdir.from_array(temp, ftype='d8')
+
+    # Read the flow direction raster
+    with rio.open(headwaters_filename) as src:
+        headwaters = src.read(1)
+        nodata = src.nodata
+
+    flowaccum = flw.accuflux(headwaters, nodata=nodata, direction='up')
+
+    stream = np.where(flowaccum > 0, flow_accumulation_threshold, 0)
+
+    # Write the flow accumulation raster
+    profile.update(dtype=flowaccum.dtype)
+    with rio.open(flow_accumulation_filename, 'w', **profile) as dst, rio.open(
+        stream_pixel_filename, 'w', **profile
+    ) as dst2:
+        dst.write(flowaccum, 1)
+        dst2.write(stream, 1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-fd', '--flow-direction-filename', help='Flow direction filename', required=True, type=str
+    )
+    parser.add_argument('-wg', '--headwaters-filename', help='Headwaters filename', required=True, type=str)
+    parser.add_argument(
+        '-fa', '--flow-accumulation-filename', help='Flow accumulation filename', required=True, type=str
+    )
+    parser.add_argument(
+        '-stream', '--stream-pixel-filename', help='Stream pixel filename', required=True, type=str
+    )
+    parser.add_argument(
+        '-thresh',
+        '--flow-accumulation-threshold',
+        help='Flow accumulation threshold',
+        required=True,
+        type=float,
+    )
+
+    args = parser.parse_args()
+
+    accumulate_flow(**vars(args))
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index b97d1912d..b69aa339a 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -37,18 +37,11 @@ fi
 echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"
 date -u
 Tstart
-$taudemDir/aread8 -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
-    -ad8 $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif \
+python3 $srcDir/accumulate_headwaters.py \
+    -fd $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
+    -fa $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif \
     -wg $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif \
-    -nc
-Tcount
-
-# THRESHOLD ACCUMULATIONS ##
-echo -e $startDiv"Threshold Accumulations $hucNumber $current_branch_id"
-date -u
-Tstart
-$taudemDir/threshold -ssa $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif \
-    -src $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
+    -stream $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
     -thresh 1
 Tcount
 

From 85c77e98c04f45041efa775456a369de4d2e6b65 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 2 Feb 2024 17:20:43 +0000
Subject: [PATCH 296/359] v4.4.9.1 Update pipfile and lock for dependabot fixes
 (#1073)

---
 Pipfile           |    3 +-
 Pipfile.lock      | 1127 +++++++++++++++++++++++----------------------
 docs/CHANGELOG.md |   11 +
 3 files changed, 587 insertions(+), 554 deletions(-)

diff --git a/Pipfile b/Pipfile
index 043b69288..97bac2944 100644
--- a/Pipfile
+++ b/Pipfile
@@ -27,7 +27,7 @@ pyproj = "==3.5.0"
 psycopg2-binary = "==2.9.6"
 boto3 = "==1.26.109"
 jupyter = "==1.0.0"
-jupyterlab = "==3.6.3"
+jupyterlab = "==3.6.7"
 ipympl = "==0.9.3"
 pytest = "==7.3.0"
 whitebox = "==2.3.1"
@@ -44,6 +44,7 @@ pre-commit = "==3.3.3"
 isort = "==5.12.0"
 urllib3 = "==1.26.18"
 pyflwdir = "*"
+pillow = "==10.2.0"
 
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index b38946cc9..61248b1d8 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "b0e64fe05b2fb6310dcb5e41cb0d131e77067c5bb3c54db67c9f46a14e705d42"
+            "sha256": "5f29b79268530835205eb8a550b43c847acfcb16ede4de1a6dc1d22c176f5ac1"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -21,7 +21,7 @@
                 "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92",
                 "sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.4.0"
         },
         "aiofiles": {
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "markers": "python_version < '4' and python_full_version >= '3.7.0'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -37,7 +37,7 @@
                 "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d",
                 "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.19.0"
         },
         "anyio": {
@@ -60,7 +60,7 @@
                 "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08",
                 "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==23.1.0"
         },
         "argon2-cffi-bindings": {
@@ -107,27 +107,27 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
-                "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"
+                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
+                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==23.1.0"
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==23.2.0"
         },
         "babel": {
             "hashes": [
                 "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363",
                 "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.14.0"
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
-                "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
+                "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051",
+                "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==4.12.2"
+            "version": "==4.12.3"
         },
         "black": {
             "hashes": [
@@ -189,7 +189,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4.0'",
+            "markers": "python_version >= '3.8' and python_version < '4'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -205,7 +205,7 @@
                 "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df",
                 "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.29.165"
         },
         "brotli": {
@@ -302,7 +302,7 @@
                 "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2",
                 "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==5.3.2"
         },
         "certifi": {
@@ -499,7 +499,7 @@
                 "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
                 "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.3.2"
         },
         "click": {
@@ -507,7 +507,7 @@
                 "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
                 "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==8.1.7"
         },
         "click-plugins": {
@@ -522,7 +522,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -543,11 +543,11 @@
         },
         "comm": {
             "hashes": [
-                "sha256:2da8d9ebb8dd7bfc247adaff99f24dce705638a8042b85cb995066793e391001",
-                "sha256:a517ea2ca28931c7007a7a99c562a0fa5883cfb48963140cf642c41c948498be"
+                "sha256:0bc91edae1344d39d3661dcbc36937181fdaddb304790458f8b044dbc064b89a",
+                "sha256:87928485c0dfc0e7976fd89fc1e187023cf587e7c353e4a9b417555b44adf021"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.2.0"
+            "version": "==0.2.1"
         },
         "contextily": {
             "hashes": [
@@ -617,60 +617,67 @@
         },
         "cython": {
             "hashes": [
-                "sha256:01b94304aab87496e81d1f546e71abf57b430b39be4269df1cd7da9928d70b5b",
-                "sha256:0c636c9ab92c7838231a1ba769e519d953af8294612f3f772a54d3a5250ff23f",
-                "sha256:0d8a98c7d86ac4d05b251c39faf49423780381aab55fbf2e147f6e006a34a58a",
-                "sha256:13211b67b29f6ed8e87c137496c73d93aff0330d97940b4fbed72eae37a4a2a0",
-                "sha256:133057ac45b6fa7fe5d7baada9d3545d09339432f75c0545f556e8c6fecc2932",
-                "sha256:167b3f3894dcc697cefefac1d198304fae8eb4d5860a7b8bc2459d572e838470",
-                "sha256:225f8bba6428b8d711ca2d6c738d2e3a4667f6a2ae40f8a7a5256f69f6a3600e",
-                "sha256:22d2a684122dfb531853d57c8c85c1d5d44be709e12466dca99fa6aee7d8054f",
-                "sha256:23ceac5315fe899c229e874328742154e331fa41337bb03f6f5264636c351c9e",
-                "sha256:2c67105f2c6ccf5b3adbcfaecf3c5c9fa8940f9f97955c9ad7d2542151d97d93",
-                "sha256:30eb2d2938b9195e2c82951713429aff3ad1be9f104437d1536a04eb0cb3dc0e",
-                "sha256:34d51709e10ad6213b4bf094af7be7ff82bab43216b3c92a07d05b451deeca79",
-                "sha256:3a83e04fde663b84905f3a20213a4333d13a07b79434300704b70dc552761f8b",
-                "sha256:3f02c7240abab48d59f0d5fef7064f18f01a2a204616165fa6367a8abf5a8832",
-                "sha256:45319d2471f4dbf19893ca53785a421107266e18b8cccd2054fce1e3f72a85f1",
-                "sha256:4e8bf00ec1dd1d92e9ae74d2e6891f087a939e1dfb40c9c7fa5d8d6a26c94f5a",
-                "sha256:51e8164b1270625ff101e95c3c1c234421520c07a0a3a20ded9e9431d98afce7",
-                "sha256:539ad5a21141e6420035cf616bcba48d999bf878839e52692f97fc7e2f16265c",
-                "sha256:55f93d3822bc196b37a8bdfa4ec6a35232a399e97f2baa714bd5ed8ea9b0ce68",
-                "sha256:5e3a038332973b12e72236e8884dc99601a840334c2c46cfbbb5851cb94166eb",
-                "sha256:612d83fd1eb5aaa5401a755c1f1aafacd9dab404cd350b90d5f404c98b33e4b3",
-                "sha256:6a1859af761977530df2cd5c36e31d54e8d6708ad2c4656e7125c482364dc216",
-                "sha256:79868ec74e4907a8a6e63effe13547c6157f196a162920b1de066da5849ffb8e",
-                "sha256:79f20c61114c7948cf1214585066406cef4b54a9b935160980e0b6e70ada3a69",
-                "sha256:7c8d579d13cb81abe704c8b0908d122b81d6e2623265a19c4a6a7377f440debb",
-                "sha256:812b193c26553f1f375d4f1c50f805c227b24ed2d595bc9cdaf78c992ecc64a4",
-                "sha256:816f5285d596062c7ef22790de7d75354b58d4417a9fc64cba914aeeb900db0b",
-                "sha256:82f27a0134fc6bb46032ca5f728d8af984f3be94a3cb01cb70ff1224e551b9cf",
-                "sha256:848a28ea49166454c3bff927e5a47629eecf1aa755d6fb3290569cba0fc93766",
-                "sha256:861cf254bf5836d47c2aee86aa75dd93d3de00ccd1b077c3c7a2bb22cba358e7",
-                "sha256:8ad7c2303a338b2c0b6c6c68f101a6768725934538756096cf3388a5c07a7525",
-                "sha256:8ea936cf5931297ba07bce121388c4c6266c1b63a9f4d648ae16c92ff090204b",
-                "sha256:931aade65f77cf59f2a702ac1f549a4836ce221107c740502cbad18d6d8e9511",
-                "sha256:936ec37b261b226d7404eff23a9aad284098338150d42a53d6a9af12b18d3892",
-                "sha256:9fcd9a18ee3ac7f460e0841954feb495102ffbdbec0e6c78562f3495cda000dd",
-                "sha256:b1853bc34ced5ff6473e881fcf6de29da83262552c8f268a0df53b49c2b89e2c",
-                "sha256:b227643d8a40b68554dc7d37fcd03fc97b4fb0bd2614aeb5f2e07ab244642d36",
-                "sha256:b65abca78aa5ebc8675c8480b9a53006f6efea9910ad099cf32c9fb5617ef251",
-                "sha256:b9d0dae6dccd349b8ccf197c10ef2d05c711ca36a649c7eddbab1de2c90b63a1",
-                "sha256:cd6ae43ef2e596c9a88dbf2a8895be2e32cc2f5bc3c8ba2e7753b69068fc0b2d",
-                "sha256:e13abb14843397b76d0472c7d33cd260d5f262ab05cc27ed423317e645e29643",
-                "sha256:e1bdf8a107fdf9e174991aa87a0be7504f60de1ec6bfb1ccfb30e33acac818a0",
-                "sha256:e34b4b08d795ccca920fa26b099558f4f1e4e3f794e4ba8d3433c5bc2454d50a",
-                "sha256:e3c0e19bb41de6be9d8afc85795159ca16296be81a586cd9588be0400d44a855",
-                "sha256:ef5bb0268bfe5992da3ef9292463a5a895ed8700b134ed2c00008d5471b3ba6e",
-                "sha256:f2602a5c97a3d618b3b847514204ef3349fb414c59e1126c0c2c708d2c5680f8",
-                "sha256:f3845c4506e0d207c5e268fb02813928f3a1e135de954a379f165ef0d581da47",
-                "sha256:f674be92673e87dd8ee7cfe553d5960ec4effc5ab15063b9a5e265a51585a31a",
-                "sha256:f6d8ff62ad55dc0393686438eac4b457a916e4d1118a0b550746bb52b4c756cc",
-                "sha256:fb299acf3a578573c190c858d49e0cf9d75f4bc49c3f24c5a63804997ef09213",
-                "sha256:fed25959e4025870fdde5f895fcb126196d22affd4f4fad85a2823e0dddc85b0"
+                "sha256:000dc9e135d0eec6ecb2b40a5b02d0868a2f8d2e027a41b0fe16a908a9e6de02",
+                "sha256:05d7eddc668ae7993643f32c7661f25544e791edb745758672ea5b1a82ecffa6",
+                "sha256:0c38c9f0bcce2df0c3347285863621be904ac6b64c5792d871130569d893efd7",
+                "sha256:0cb2dcc565c7851f75d496f724a384a790fab12d1b82461b663e66605bec429a",
+                "sha256:115f0a50f752da6c99941b103b5cb090da63eb206abbc7c2ad33856ffc73f064",
+                "sha256:13c2a5e57a0358da467d97667297bf820b62a1a87ae47c5f87938b9bb593acbd",
+                "sha256:16873d78be63bd38ffb759da7ab82814b36f56c769ee02b1d5859560e4c3ac3c",
+                "sha256:171b27051253d3f9108e9759e504ba59ff06e7f7ba944457f94deaf9c21bf0b6",
+                "sha256:17a642bb01a693e34c914106566f59844b4461665066613913463a719e0dd15d",
+                "sha256:18bfa387d7a7f77d7b2526af69a65dbd0b731b8d941aaff5becff8e21f6d7717",
+                "sha256:1ab75242869ff71e5665fe5c96f3378e79e792fa3c11762641b6c5afbbbbe026",
+                "sha256:1aca1b97e0095b3a9a6c33eada3f661a4ed0d499067d121239b193e5ba3bb4f0",
+                "sha256:289ce7838208211cd166e975865fd73b0649bf118170b6cebaedfbdaf4a37795",
+                "sha256:2cde23c555470db3f149ede78b518e8274853745289c956a0e06ad8d982e4db9",
+                "sha256:2cdfc32252f3b6dc7c94032ab744dcedb45286733443c294d8f909a4854e7f83",
+                "sha256:2f020fa1c0552052e0660790b8153b79e3fc9a15dbd8f1d0b841fe5d204a6ae6",
+                "sha256:314f2355a1f1d06e3c431eaad4708cf10037b5e91e4b231d89c913989d0bdafd",
+                "sha256:3a3d67f079598af49e90ff9655bf85bd358f093d727eb21ca2708f467c489cae",
+                "sha256:45523fdc2b78d79b32834cc1cc12dc2ca8967af87e22a3ee1bff20e77c7f5520",
+                "sha256:4b983c8e6803f016146c26854d9150ddad5662960c804ea7f0c752c9266752f0",
+                "sha256:51d1426263b0e82fb22bda8ea60dc77a428581cc19e97741011b938445d383f1",
+                "sha256:547eb3cdb2f8c6f48e6865d5a741d9dd051c25b3ce076fbca571727977b28ac3",
+                "sha256:5a567d4b9ba70b26db89d75b243529de9e649a2f56384287533cf91512705bee",
+                "sha256:61a237bc9dd23c7faef0fcfce88c11c65d0c9bb73c74ccfa408b3a012073c20e",
+                "sha256:6717c06e9cfc6c1df18543cd31a21f5d8e378a40f70c851fa2d34f0597037abc",
+                "sha256:6c46939c3983217d140999de7c238c3141f56b1ea349e47ca49cae899969aa2c",
+                "sha256:78825a3774211e7d5089730f00cdf7f473042acc9ceb8b9eeebe13ed3a5541de",
+                "sha256:7990ca127e1f1beedaf8fc8bf66541d066ef4723ad7d8d47a7cbf842e0f47580",
+                "sha256:7e8f2454128974905258d86534f4fd4f91d2f1343605657ecab779d80c9d6d5e",
+                "sha256:80fd94c076e1e1b1ee40a309be03080b75f413e8997cddcf401a118879863388",
+                "sha256:8140597a8b5cc4f119a1190f5a2228a84f5ca6d8d9ec386cfce24663f48b2539",
+                "sha256:8333423d8fd5765e7cceea3a9985dd1e0a5dfeb2734629e1a2ed2d6233d39de6",
+                "sha256:85077915a93e359a9b920280d214dc0cf8a62773e1f3d7d30fab8ea4daed670c",
+                "sha256:870d2a0a7e3cbd5efa65aecdb38d715ea337a904ea7bb22324036e78fb7068e7",
+                "sha256:90d3fe31db55685d8cb97d43b0ec39ef614fcf660f83c77ed06aa670cb0e164f",
+                "sha256:96b028f044f5880e3cb18ecdcfc6c8d3ce9d0af28418d5ab464509f26d8adf12",
+                "sha256:97b2a45845b993304f1799664fa88da676ee19442b15fdcaa31f9da7e1acc434",
+                "sha256:9d3f74388db378a3c6fd06e79a809ed98df3f56484d317b81ee762dbf3c263e0",
+                "sha256:9e2be2b340fea46fb849d378f9b80d3c08ff2e81e2bfbcdb656e2e3cd8c6b2dc",
+                "sha256:a1df7a129344b1215c20096d33c00193437df1a8fcca25b71f17c23b1a44f782",
+                "sha256:a846e0a38e2b24e9a5c5dc74b0e54c6e29420d88d1dafabc99e0fc0f3e338636",
+                "sha256:a973268d7ca1a2bdf78575e459a94a78e1a0a9bb62a7db0c50041949a73b02ff",
+                "sha256:aae26f9663e50caf9657148403d9874eea41770ecdd6caf381d177c2b1bb82ba",
+                "sha256:ae7ac561fd8253a9ae96311e91d12af5f701383564edc11d6338a7b60b285a6f",
+                "sha256:baa0b7f3f841fe087410cab66778e2d3fb20ae2d2078a2be3dffe66c6574be39",
+                "sha256:bfabe115deef4ada5d23c87bddb11289123336dcc14347011832c07db616dd93",
+                "sha256:c1949d6aa7bc792554bee2b67a9fe41008acbfe22f4f8df7b6ec7b799613a4b3",
+                "sha256:c26daaeccda072459b48d211415fd1e5507c06bcd976fa0d5b8b9f1063467d7b",
+                "sha256:c8aa05f5e17f8042a3be052c24f2edc013fb8af874b0bf76907d16c51b4e7871",
+                "sha256:c9c0f29246734561c90f36e70ed0506b61aa3d044e4cc4cba559065a2a741fae",
+                "sha256:c9f2c6e1b8f3bcd6cb230bac1843f85114780bb8be8614855b1628b36bb510e0",
+                "sha256:de892422582f5758bd8de187e98ac829330ec1007bc42c661f687792999988a7",
+                "sha256:df8093deabc55f37028190cf5e575c26aad23fc673f34b85d5f45076bc37ce39",
+                "sha256:e24791ddae2324e88e3c902a765595c738f19ae34ee66bfb1a6dac54b1833419",
+                "sha256:e87294e33e40c289c77a135f491cd721bd089f193f956f7b8ed5aa2d0b8c558f",
+                "sha256:f05c0bf9d085c031df8f583f0d506aa3be1692023de18c45d0aaf78685bbb944",
+                "sha256:fa97893d99385386925d00074654aeae3a98867f298d1e12ceaf38a9054a9bae",
+                "sha256:fe81b339cffd87c0069c6049b4d33e28bdd1874625ee515785bf42c9fdff3658"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.7"
+            "version": "==3.0.8"
         },
         "dask": {
             "hashes": [
@@ -761,10 +768,10 @@
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:b9fd1a2dd6971dbc7fee280a95bd199ae0dd9ce22beb91cc75e9c1c528a5170e",
-                "sha256:e25df6647e1bc4a26070b700897b07b542ec898dd4f1f6ea013e7f6a88417225"
+                "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0",
+                "sha256:e3126a94bdc4623d3de4485f8d468a12f02a67921315ddc87836d6e456dc789d"
             ],
-            "version": "==2.19.0"
+            "version": "==2.19.1"
         },
         "filelock": {
             "hashes": [
@@ -824,51 +831,51 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:084511482dd265bce6dca24c509894062f0117e4e6869384d853f46c0e6d43be",
-                "sha256:1193fb090061efa2f9e2d8d743ae9850c77b66746a3b32792324cdce65784154",
-                "sha256:174995f7b057e799355b393e97f4f93ef1f2197cbfa945e988d49b2a09ecbce8",
-                "sha256:253bb46bab970e8aae254cebf2ae3db98a4ef6bd034707aa68a239027d2b198d",
-                "sha256:2a78dba8c2a1e9d53a0fb5382979f024200dc86adc46a56cbb668a2249862fda",
-                "sha256:2d2404107626f97a221dc1a65b05396d2bb2ce38e435f64f26ed2369f68675d9",
-                "sha256:40bdbe90b33897d9cc4a39f8e415b0fcdeae4c40a99374b8a4982f127ff5c767",
-                "sha256:495369c660e0c27233e3c572269cbe520f7f4978be675f990f4005937337d391",
-                "sha256:4a9a51745c0439516d947480d4d884fa18bd1458e05b829e482b9269afa655bc",
-                "sha256:511482df31cfea9f697930f61520f6541185fa5eeba2fa760fe72e8eee5af88b",
-                "sha256:52c82df66201f3a90db438d9d7b337c7c98139de598d0728fb99dab9fd0495ca",
-                "sha256:562681188c62c024fe2c611b32e08b8de2afa00c0c4e72bed47c47c318e16d5c",
-                "sha256:59a6c8b71a245800e923cb684a2dc0eac19c56493e2f896218fcf2571ed28984",
-                "sha256:5dde0eab40faaa5476133123f6a622a1cc3ac9b7af45d65690870620323308b4",
-                "sha256:61df4dee5d38ab65b26da8efd62d859a1eef7a34dcbc331299a28e24d04c59a7",
-                "sha256:62d8ddb058b8e87018e5dc26f3258e2c30daad4c87262dfeb0e2617dd84750e6",
-                "sha256:66c92ec7f95fd9732550ebedefcd190a8d81beaa97e89d523a0d17198a8bda4d",
-                "sha256:843509ae9b93db5aaf1a6302085e30bddc1111d31e11d724584818f5b698f500",
-                "sha256:854421e328d47d70aa5abceacbe8eef231961b162c71cbe7ff3f47e235e2e5c5",
-                "sha256:97620c4af36e4c849e52661492e31dc36916df12571cb900d16960ab8e92a980",
-                "sha256:9acfa1cdc479e0dde528b61423855913d949a7f7fe09e276228298fef4589540",
-                "sha256:a77a60315c33393b2bd29d538d1ef026060a63d3a49a9233b779261bad9c3f71",
-                "sha256:b4fabb8cc9422efae1a925160083fdcbab8fdc96a8483441eb7457235df625bd",
-                "sha256:bf1810635c00f7c45d93085611c995fc130009cec5abdc35b327156aa191f982",
-                "sha256:c01f409be619a9a0f5590389e37ccb58b47264939f0e8d58bfa1f3ba07d22671",
-                "sha256:c59227d7ba5b232281c26ae04fac2c73a79ad0e236bca5c44aae904a18f14faf",
-                "sha256:c75e19971209fbbce891ebfd1b10c37320a5a28e8d438861c21d35305aedb81c",
-                "sha256:ce0e2c88c8c985b7b9a7efcd06511fb0a1fe3ddd9a6cd2895ef1dbf9059719d7",
-                "sha256:d6477ba902dd2d7adda7f0fd3bfaeb92885d45993c9e1928c9f28fc3961415f7",
-                "sha256:d986b66ff722ef675b7ee22fbe5947a41f60a61a4da15579d5e276d897fbc7fa",
-                "sha256:dd23848f877c3754f53a4903fb7a593ed100924f9b4bff7d5a4e2e8a7001ae11",
-                "sha256:e3f4d61f3a8195eac784f1d0c16c0a3105382c1b9a74d99ac4ba421da39a8826",
-                "sha256:e6b968543fde4119231c12c2a953dcf83349590ca631ba8216a8edf9cd4d36a9",
-                "sha256:e77bdf52185bdaf63d39f3e1ac3212e6cfa3ab07d509b94557a8902ce9c13c82",
-                "sha256:e79f1a3970d25f692bbb8c8c2637e621a66c0d60c109ab48d4a160f50856deff",
-                "sha256:e7a0a8848726956e9d9fb18c977a279013daadf0cbb6725d2015a6dd57527992",
-                "sha256:e869da810ae35afb3019baa0d0306cdbab4760a54909c89ad8904fa629991812",
-                "sha256:e8acf6dd0434b211b3bd30d572d9e019831aae17a54016629fa8224783b22df8",
-                "sha256:e8fa20748de55d0021f83754b371432dca0439e02847962fc4c42a0e444c2d78",
-                "sha256:ea592e6a09b71cb7a7661dd93ac0b877a6228e2d677ebacbad0a4d118494c86d",
-                "sha256:ec13a10715eef0e031858c1c23bfaee6cba02b97558e4a7bfa089dba4a8c2ebf",
-                "sha256:f4da089f6dfdb822293bde576916492cd708c37c2501c3651adde39804630538"
+                "sha256:0255dbc128fee75fb9be364806b940ed450dd6838672a150d501ee86523ac61e",
+                "sha256:0a00bd0e68e88987dcc047ea31c26d40a3c61185153b03457956a87e39d43c37",
+                "sha256:0a1d313a415eaaba2b35d6cd33536560deeebd2ed758b9bfb89ab5d97dc5deac",
+                "sha256:0f750037e02beb8b3569fbff701a572e62a685d2a0e840d75816592280e5feae",
+                "sha256:13819db8445a0cec8c3ff5f243af6418ab19175072a9a92f6cc8ca7d1452754b",
+                "sha256:254d9a6f7be00212bf0c3159e0a420eb19c63793b2c05e049eb337f3023c5ecc",
+                "sha256:29495d6d109cdbabe73cfb6f419ce67080c3ef9ea1e08d5750240fd4b0c4763b",
+                "sha256:32ab2e9702dff0dd4510c7bb958f265a8d3dd5c0e2547e7b5f7a3df4979abb07",
+                "sha256:3480eeb52770ff75140fe7d9a2ec33fb67b07efea0ab5129c7e0c6a639c40c70",
+                "sha256:3a808f3c1d1df1f5bf39be869b6e0c263570cdafb5bdb2df66087733f566ea71",
+                "sha256:3b629108351d25512d4ea1a8393a2dba325b7b7d7308116b605ea3f8e1be88df",
+                "sha256:3d71606c9321f6701642bd4746f99b6089e53d7e9817fc6b964e90d9c5f0ecc6",
+                "sha256:3e2b95dce2ead58fb12524d0ca7d63a63459dd489e7e5838c3cd53557f8933e1",
+                "sha256:4a5a5318ba5365d992666ac4fe35365f93004109d18858a3e18ae46f67907670",
+                "sha256:4c811d3c73b6abac275babb8aa439206288f56fdb2c6f8835e3d7b70de8937a7",
+                "sha256:4e743935139aa485fe3253fc33fe467eab6ea42583fa681223ea3f1a93dd01e6",
+                "sha256:4ec558c543609e71b2275c4894e93493f65d2f41c15fe1d089080c1d0bb4d635",
+                "sha256:5465df494f20a7d01712b072ae3ee9ad2887004701b95cb2cc6dcb9c2c97a899",
+                "sha256:5b60e3afa9635e3dfd3ace2757039593e3bd3cf128be0ddb7a1ff4ac45fa5a50",
+                "sha256:63fbed184979f09a65aa9c88b395ca539c94287ba3a364517698462e13e457c9",
+                "sha256:69731e8bea0578b3c28fdb43dbf95b9386e2d49a399e9a4ad736b8e479b08085",
+                "sha256:6dd58cc03016b281bd2c74c84cdaa6bd3ce54c5a7f47478b7657b930ac3ed8eb",
+                "sha256:740947906590a878a4bde7dd748e85fefa4d470a268b964748403b3ab2aeed6c",
+                "sha256:7df26dd3650e98ca45f1e29883c96a0b9f5bb6af8d632a6a108bc744fa0bd9b3",
+                "sha256:7eb7ad665258fba68fd22228a09f347469d95a97fb88198e133595947a20a184",
+                "sha256:7ee48bd9d6b7e8f66866c9090807e3a4a56cf43ffad48962725a190e0dd774c8",
+                "sha256:86e0427864c6c91cf77f16d1fb9bf1bbf7453e824589e8fb8461b6ee1144f506",
+                "sha256:8f57ecd742545362a0f7186774b2d1c53423ed9ece67689c93a1055b236f638c",
+                "sha256:90f898cdd67f52f18049250a6474185ef6544c91f27a7bee70d87d77a8daf89c",
+                "sha256:94208ea750e3f96e267f394d5588579bb64cc628e321dbb1d4243ffbc291b18b",
+                "sha256:a1c154bb85dc9a4cf145250c88d112d88eb414bad81d4cb524d06258dea1bdc0",
+                "sha256:a5d77479fb885ef38a16a253a2f4096bc3d14e63a56d6246bfdb56365a12b20c",
+                "sha256:a86a5ab2873ed2575d0fcdf1828143cfc6b977ac448e3dc616bb1e3d20efbafa",
+                "sha256:ac71e2e201df041a2891067dc36256755b1229ae167edbdc419b16da78732c2f",
+                "sha256:b3e1304e5f19ca861d86a72218ecce68f391646d85c851742d265787f55457a4",
+                "sha256:b8be28c036b9f186e8c7eaf8a11b42373e7e4949f9e9f370202b9da4c4c3f56c",
+                "sha256:c19044256c44fe299d9a73456aabee4b4d06c6b930287be93b533b4737d70aa1",
+                "sha256:d49ce3ea7b7173faebc5664872243b40cf88814ca3eb135c4a3cdff66af71946",
+                "sha256:e040f905d542362e07e72e03612a6270c33d38281fd573160e1003e43718d68d",
+                "sha256:eabae77a07c41ae0b35184894202305c3ad211a93b2eb53837c2a1143c8bc952",
+                "sha256:f791446ff297fd5f1e2247c188de53c1bfb9dd7f0549eba55b73a3c2087a2703",
+                "sha256:f83a4daef6d2a202acb9bf572958f91cfde5b10c8ee7fb1d09a4c81e5d851fd8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.47.0"
+            "version": "==4.47.2"
         },
         "fqdn": {
             "hashes": [
@@ -887,18 +894,18 @@
         },
         "geocube": {
             "hashes": [
-                "sha256:43c88cd5dd18f9df9561e642b4840e4bd061ccc8a3e8c419378f6707384213d0",
-                "sha256:5404060d6dcc8d3f251b453588667c6d5e5e60d07793011851f8bc129f779db0"
+                "sha256:0049913c9ca0e0ff8bd4473ab6a87d185e5c2457ee053a7184abcfe9e9447b57",
+                "sha256:4fbcdcbb187cbf02d9f03bec0c30628f5d5161cd3e9c51aaf0877c5f69539ed7"
             ],
-            "markers": "python_version >= '3.9'",
-            "version": "==0.4.2"
+            "markers": "python_version >= '3.10'",
+            "version": "==0.5.0"
         },
         "geographiclib": {
             "hashes": [
                 "sha256:6b7225248e45ff7edcee32becc4e0a1504c606ac5ee163a5656d482e0cd38734",
                 "sha256:f7f41c85dc3e1c2d3d935ec86660dc3b2c848c83e17f9a9e51ba9d5146a15859"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.0"
         },
         "geopandas": {
@@ -914,7 +921,7 @@
                 "sha256:50283d8e7ad07d89be5cb027338c6365a32044df3ae2556ad3f52f4840b3d0d1",
                 "sha256:ae8b4bc5c1131820f4d75fce9d4aaaca0c85189b3aa5d64c3dcaf5e3b7b882a7"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.4.1"
         },
         "gval": {
@@ -943,11 +950,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7",
-                "sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67"
+                "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e",
+                "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.0.0"
+            "version": "==7.0.1"
         },
         "inflate64": {
             "hashes": [
@@ -1005,7 +1012,7 @@
                 "sha256:f5924499dc8800928c0ee4580fa8eb4ffa880b2cce4431537d0390e503a9c9ee",
                 "sha256:f79542478e49e471e8b23556700e6f688a40dc93e9a746f77a546c13251b59b1"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.0.0"
         },
         "iniconfig": {
@@ -1013,16 +1020,16 @@
                 "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
                 "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.0.0"
         },
         "ipykernel": {
             "hashes": [
-                "sha256:7d5d594b6690654b4d299edba5e872dc17bb7396a8d0609c97cb7b8a1c605de6",
-                "sha256:dab88b47f112f9f7df62236511023c9bdeef67abc73af7c652e4ce4441601686"
+                "sha256:076663ca68492576f051e4af7720d33f34383e655f2be0d544c8b1c9de915b2f",
+                "sha256:b5dd3013cab7b330df712891c96cd1ab868c27a7159e606f762015e9bf8ceb3f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.27.1"
+            "version": "==6.29.0"
         },
         "ipympl": {
             "hashes": [
@@ -1034,11 +1041,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:2f55d59370f59d0d2b2212109fe0e6035cfea436b1c0e6150ad2244746272ec5",
-                "sha256:ac4da4ecf0042fb4e0ce57c60430c2db3c719fa8bdf92f8631d6bd8a5785d1f0"
+                "sha256:1050a3ab8473488d7eee163796b02e511d0735cf43a04ba2a8348bd0f2eaf8a5",
+                "sha256:48fbc236fbe0e138b88773fa0437751f14c3645fb483f1d4c5dee58b37e5ce73"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.19.0"
+            "version": "==8.21.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1052,7 +1059,7 @@
                 "sha256:2b88d728656aea3bbfd05d32c747cfd0078f9d7e159cf982433b58ad717eed7f",
                 "sha256:40211efb556adec6fa450ccc2a77d59ca44a060f4f9f136833df59c9f538e6e8"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==8.1.1"
         },
         "isoduration": {
@@ -1083,8 +1090,7 @@
                 "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
                 "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
             ],
-            "index": "pypi",
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.1.3"
         },
         "jmespath": {
@@ -1092,7 +1098,7 @@
                 "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
                 "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.0.1"
         },
         "joblib": {
@@ -1100,7 +1106,7 @@
                 "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1",
                 "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.3.2"
         },
         "json5": {
@@ -1122,19 +1128,19 @@
                 "format-nongpl"
             ],
             "hashes": [
-                "sha256:4f614fd46d8d61258610998997743ec5492a648b33cf478c1ddc23ed4598a5fa",
-                "sha256:ed6231f0429ecf966f5bc8dfef245998220549cbbcf140f913b7464c52c3b6b3"
+                "sha256:7996507afae316306f9e2290407761157c6f78002dcf7419acb99822143d1c6f",
+                "sha256:85727c00279f5fa6bedbe6238d2aa6403bedd8b4864ab11207d07df3cc1b2ee5"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.20.0"
+            "version": "==4.21.1"
         },
         "jsonschema-specifications": {
             "hashes": [
-                "sha256:9472fc4fea474cd74bea4a2b190daeccb5a9e4db2ea80efcf7a1b582fc9a81b8",
-                "sha256:e74ba7c0a65e8cb49dc26837d6cfe576557084a8b423ed16a420984228104f93"
+                "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc",
+                "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.11.2"
+            "version": "==2023.12.1"
         },
         "jupyter": {
             "hashes": [
@@ -1150,7 +1156,7 @@
                 "sha256:214668aaea208195f4c13d28eb272ba79f945fc0cf3f11c7092c20b2ca1980e7",
                 "sha256:52be28e04171f07aed8f20e1616a5a552ab9fee9cbbe6c1896ae170c3880d392"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==7.4.9"
         },
         "jupyter-console": {
@@ -1158,16 +1164,16 @@
                 "sha256:309d33409fcc92ffdad25f0bcdf9a4a9daa61b6f341177570fdac03de5352485",
                 "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==6.6.3"
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:1553311a97ccd12936037f36b9ab4d6ae8ceea6ad2d5c90d94a909e752178e40",
-                "sha256:220dfb00c45f0d780ce132bb7976b58263f81a3ada6e90a9b6823785a424f739"
+                "sha256:c65c82126453a723a2804aa52409930434598fd9d35091d63dfb919d2b765bb7",
+                "sha256:de61a9d7fc71240f688b2fb5ab659fbb56979458dc66a71decd098e03c79e218"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.5.1"
+            "version": "==5.7.1"
         },
         "jupyter-events": {
             "hashes": [
@@ -1179,34 +1185,34 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:dc77b7dcc5fc0547acba2b2844f01798008667201eea27c6319ff9257d700a6d",
-                "sha256:fd030dd7be1ca572e4598203f718df6630c12bd28a599d7f1791c4d7938e1010"
+                "sha256:0edb626c94baa22809be1323f9770cf1c00a952b17097592e40d03e6a3951689",
+                "sha256:184a0f82809a8522777cfb6b760ab6f4b1bb398664c5860a27cec696cb884923"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.12.1"
+            "version": "==2.12.5"
         },
         "jupyter-server-fileid": {
             "hashes": [
                 "sha256:7486bca3acf9bbaab7ce5127f9f64d2df58f5d2de377609fb833291a7217a6a2",
                 "sha256:76dd05a45b78c7ec0cba0be98ece289984c6bcfc1ca2da216d42930e506a4d68"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.9.1"
         },
         "jupyter-server-terminals": {
             "hashes": [
-                "sha256:2fc0692c883bfd891f4fba0c4b4a684a37234b0ba472f2e97ed0a3888f46e1e4",
-                "sha256:ebcd68c9afbf98a480a533e6f3266354336e645536953b7abcc7bdeebc0154a3"
+                "sha256:1b80c12765da979513c42c90215481bbc39bd8ae7c0350b4f85bc3eb58d0fa80",
+                "sha256:396b5ccc0881e550bf0ee7012c6ef1b53edbde69e67cab1d56e89711b46052e8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.5.0"
+            "version": "==0.5.2"
         },
         "jupyter-server-ydoc": {
             "hashes": [
                 "sha256:969a3a1a77ed4e99487d60a74048dc9fa7d3b0dcd32e60885d835bbf7ba7be11",
                 "sha256:a6fe125091792d16c962cc3720c950c2b87fcc8c3ecf0c54c84e9a20b814526c"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.8.0"
         },
         "jupyter-ydoc": {
@@ -1214,16 +1220,16 @@
                 "sha256:5759170f112c70320a84217dd98d287699076ae65a7f88d458d57940a9f2b882",
                 "sha256:5a02ca7449f0d875f73e8cb8efdf695dddef15a8e71378b1f4eda6b7c90f5382"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.2.5"
         },
         "jupyterlab": {
             "hashes": [
-                "sha256:373e9cfb8a72edd294be14f16662563a220cecf0fb26de7aab1af9a29b689b82",
-                "sha256:6aba0caa771697d02fbf409f9767b2fdb4ee32ce935940e3b9a0d5d48d994d0f"
+                "sha256:2fadeaec161b0d1aec19f17721d8b803aef1d267f89c8b636b703be14f435c8f",
+                "sha256:d92d57d402f53922bca5090654843aa08e511290dff29fdb0809eafbbeb6df98"
             ],
             "index": "pypi",
-            "version": "==3.6.3"
+            "version": "==3.6.7"
         },
         "jupyterlab-pygments": {
             "hashes": [
@@ -1246,7 +1252,7 @@
                 "sha256:3cf5bdf5b897bf3bccf1c11873aa4afd776d7430200f765e0686bd352487b58d",
                 "sha256:6005a4e974c7beee84060fdfba341a3218495046de8ae3ec64888e5fe19fdb4c"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.9"
         },
         "kiwisolver": {
@@ -1356,7 +1362,7 @@
                 "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892",
                 "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.4.5"
         },
         "llvmlite": {
@@ -1390,7 +1396,7 @@
                 "sha256:fa9b26939ae553bf30a9f5c4c754db0fb2d2677327f2511e674aa2f5df941789",
                 "sha256:fb62fc7016b592435d3e3a8f680e3ea8897c3c9e62e6e6cc58011e7a4801439e"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.39.1"
         },
         "locket": {
@@ -1403,69 +1409,69 @@
         },
         "markupsafe": {
             "hashes": [
-                "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e",
-                "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e",
-                "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431",
-                "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686",
-                "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c",
-                "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559",
-                "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc",
-                "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb",
-                "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939",
-                "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c",
-                "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0",
-                "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4",
-                "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9",
-                "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575",
-                "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba",
-                "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d",
-                "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd",
-                "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3",
-                "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00",
-                "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155",
-                "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac",
-                "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52",
-                "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f",
-                "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8",
-                "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b",
-                "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007",
-                "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24",
-                "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea",
-                "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198",
-                "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0",
-                "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee",
-                "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be",
-                "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2",
-                "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1",
-                "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707",
-                "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6",
-                "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c",
-                "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58",
-                "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823",
-                "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779",
-                "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636",
-                "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c",
-                "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad",
-                "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee",
-                "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc",
-                "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2",
-                "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48",
-                "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7",
-                "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e",
-                "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b",
-                "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa",
-                "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5",
-                "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e",
-                "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb",
-                "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9",
-                "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57",
-                "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc",
-                "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc",
-                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2",
-                "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.1.3"
+                "sha256:0042d6a9880b38e1dd9ff83146cc3c9c18a059b9360ceae207805567aacccc69",
+                "sha256:0c26f67b3fe27302d3a412b85ef696792c4a2386293c53ba683a89562f9399b0",
+                "sha256:0fbad3d346df8f9d72622ac71b69565e621ada2ce6572f37c2eae8dacd60385d",
+                "sha256:15866d7f2dc60cfdde12ebb4e75e41be862348b4728300c36cdf405e258415ec",
+                "sha256:1c98c33ffe20e9a489145d97070a435ea0679fddaabcafe19982fe9c971987d5",
+                "sha256:21e7af8091007bf4bebf4521184f4880a6acab8df0df52ef9e513d8e5db23411",
+                "sha256:23984d1bdae01bee794267424af55eef4dfc038dc5d1272860669b2aa025c9e3",
+                "sha256:31f57d64c336b8ccb1966d156932f3daa4fee74176b0fdc48ef580be774aae74",
+                "sha256:3583a3a3ab7958e354dc1d25be74aee6228938312ee875a22330c4dc2e41beb0",
+                "sha256:36d7626a8cca4d34216875aee5a1d3d654bb3dac201c1c003d182283e3205949",
+                "sha256:396549cea79e8ca4ba65525470d534e8a41070e6b3500ce2414921099cb73e8d",
+                "sha256:3a66c36a3864df95e4f62f9167c734b3b1192cb0851b43d7cc08040c074c6279",
+                "sha256:3aae9af4cac263007fd6309c64c6ab4506dd2b79382d9d19a1994f9240b8db4f",
+                "sha256:3ab3a886a237f6e9c9f4f7d272067e712cdb4efa774bef494dccad08f39d8ae6",
+                "sha256:47bb5f0142b8b64ed1399b6b60f700a580335c8e1c57f2f15587bd072012decc",
+                "sha256:49a3b78a5af63ec10d8604180380c13dcd870aba7928c1fe04e881d5c792dc4e",
+                "sha256:4df98d4a9cd6a88d6a585852f56f2155c9cdb6aec78361a19f938810aa020954",
+                "sha256:5045e892cfdaecc5b4c01822f353cf2c8feb88a6ec1c0adef2a2e705eef0f656",
+                "sha256:5244324676254697fe5c181fc762284e2c5fceeb1c4e3e7f6aca2b6f107e60dc",
+                "sha256:54635102ba3cf5da26eb6f96c4b8c53af8a9c0d97b64bdcb592596a6255d8518",
+                "sha256:54a7e1380dfece8847c71bf7e33da5d084e9b889c75eca19100ef98027bd9f56",
+                "sha256:55d03fea4c4e9fd0ad75dc2e7e2b6757b80c152c032ea1d1de487461d8140efc",
+                "sha256:698e84142f3f884114ea8cf83e7a67ca8f4ace8454e78fe960646c6c91c63bfa",
+                "sha256:6aa5e2e7fc9bc042ae82d8b79d795b9a62bd8f15ba1e7594e3db243f158b5565",
+                "sha256:7653fa39578957bc42e5ebc15cf4361d9e0ee4b702d7d5ec96cdac860953c5b4",
+                "sha256:765f036a3d00395a326df2835d8f86b637dbaf9832f90f5d196c3b8a7a5080cb",
+                "sha256:78bc995e004681246e85e28e068111a4c3f35f34e6c62da1471e844ee1446250",
+                "sha256:7a07f40ef8f0fbc5ef1000d0c78771f4d5ca03b4953fc162749772916b298fc4",
+                "sha256:8b570a1537367b52396e53325769608f2a687ec9a4363647af1cded8928af959",
+                "sha256:987d13fe1d23e12a66ca2073b8d2e2a75cec2ecb8eab43ff5624ba0ad42764bc",
+                "sha256:9896fca4a8eb246defc8b2a7ac77ef7553b638e04fbf170bff78a40fa8a91474",
+                "sha256:9e9e3c4020aa2dc62d5dd6743a69e399ce3de58320522948af6140ac959ab863",
+                "sha256:a0b838c37ba596fcbfca71651a104a611543077156cb0a26fe0c475e1f152ee8",
+                "sha256:a4d176cfdfde84f732c4a53109b293d05883e952bbba68b857ae446fa3119b4f",
+                "sha256:a76055d5cb1c23485d7ddae533229039b850db711c554a12ea64a0fd8a0129e2",
+                "sha256:a76cd37d229fc385738bd1ce4cba2a121cf26b53864c1772694ad0ad348e509e",
+                "sha256:a7cc49ef48a3c7a0005a949f3c04f8baa5409d3f663a1b36f0eba9bfe2a0396e",
+                "sha256:abf5ebbec056817057bfafc0445916bb688a255a5146f900445d081db08cbabb",
+                "sha256:b0fe73bac2fed83839dbdbe6da84ae2a31c11cfc1c777a40dbd8ac8a6ed1560f",
+                "sha256:b6f14a9cd50c3cb100eb94b3273131c80d102e19bb20253ac7bd7336118a673a",
+                "sha256:b83041cda633871572f0d3c41dddd5582ad7d22f65a72eacd8d3d6d00291df26",
+                "sha256:b835aba863195269ea358cecc21b400276747cc977492319fd7682b8cd2c253d",
+                "sha256:bf1196dcc239e608605b716e7b166eb5faf4bc192f8a44b81e85251e62584bd2",
+                "sha256:c669391319973e49a7c6230c218a1e3044710bc1ce4c8e6eb71f7e6d43a2c131",
+                "sha256:c7556bafeaa0a50e2fe7dc86e0382dea349ebcad8f010d5a7dc6ba568eaaa789",
+                "sha256:c8f253a84dbd2c63c19590fa86a032ef3d8cc18923b8049d91bcdeeb2581fbf6",
+                "sha256:d18b66fe626ac412d96c2ab536306c736c66cf2a31c243a45025156cc190dc8a",
+                "sha256:d5291d98cd3ad9a562883468c690a2a238c4a6388ab3bd155b0c75dd55ece858",
+                "sha256:d5c31fe855c77cad679b302aabc42d724ed87c043b1432d457f4976add1c2c3e",
+                "sha256:d6e427c7378c7f1b2bef6a344c925b8b63623d3321c09a237b7cc0e77dd98ceb",
+                "sha256:dac1ebf6983148b45b5fa48593950f90ed6d1d26300604f321c74a9ca1609f8e",
+                "sha256:de8153a7aae3835484ac168a9a9bdaa0c5eee4e0bc595503c95d53b942879c84",
+                "sha256:e1a0d1924a5013d4f294087e00024ad25668234569289650929ab871231668e7",
+                "sha256:e7902211afd0af05fbadcc9a312e4cf10f27b779cf1323e78d52377ae4b72bea",
+                "sha256:e888ff76ceb39601c59e219f281466c6d7e66bd375b4ec1ce83bcdc68306796b",
+                "sha256:f06e5a9e99b7df44640767842f414ed5d7bedaaa78cd817ce04bbd6fd86e2dd6",
+                "sha256:f6be2d708a9d0e9b0054856f07ac7070fbe1754be40ca8525d5adccdbda8f475",
+                "sha256:f9917691f410a2e0897d1ef99619fd3f7dd503647c8ff2475bf90c3cf222ad74",
+                "sha256:fc1a75aa8f11b87910ffd98de62b29d6520b6d6e8a3de69a70ca34dea85d2a8a",
+                "sha256:fe8512ed897d5daf089e5bd010c3dc03bb1bdae00b35588c49b98268d4a01e00"
+            ],
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==2.1.4"
         },
         "matplotlib": {
             "hashes": [
@@ -1542,7 +1548,7 @@
                 "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205",
                 "sha256:fc7f93ded930c92394ef2cb6f04a8aabab4117a91449e72dcc8dfa646a508be8"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.2"
         },
         "morecantile": {
@@ -1617,11 +1623,11 @@
         },
         "multimethod": {
             "hashes": [
-                "sha256:afd84da9c3d0445c84f827e4d63ad42d17c6d29b122427c6dee9032ac2d2a0d4",
-                "sha256:daa45af3fe257f73abb69673fd54ddeaf31df0eb7363ad6e1251b7c9b192d8c5"
+                "sha256:e57253f5b6d530b10696843e693c11f0dadbe299a327bd71cdf3edd0019a3853",
+                "sha256:ef40713d84da4015285122a16f57bf5066cc7876b5eaf183262f3c34668c6ad3"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.10"
+            "markers": "python_version >= '3.9'",
+            "version": "==1.11"
         },
         "multipledispatch": {
             "hashes": [
@@ -1667,7 +1673,7 @@
                 "sha256:0ae11eb2319455d805596bf320336cda9554b41d99ab9a3c31bf8180bffa30e3",
                 "sha256:f99e4769b4750076cd4235c044b61232110733322384a94a63791d2e7beacc66"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.0.0"
         },
         "nbclient": {
@@ -1680,11 +1686,11 @@
         },
         "nbconvert": {
             "hashes": [
-                "sha256:2dc8267dbdfeedce2dcd34c9e3f1b51af18f43cb105549d1c5a18189ec23ba85",
-                "sha256:3c50eb2d326478cc90b8759cf2ab9dde3d892c6537cd6a5bc0991db8ef734bcc"
+                "sha256:a7f8808fd4e082431673ac538400218dd45efd076fbeb07cc6e5aa5a3a4e949e",
+                "sha256:db28590cef90f7faf2ebbc71acd402cbecf13d29176df728c0a9025a49345ea1"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.13.1"
+            "version": "==7.14.2"
         },
         "nbformat": {
             "hashes": [
@@ -1696,11 +1702,11 @@
         },
         "nest-asyncio": {
             "hashes": [
-                "sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb",
-                "sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d"
+                "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe",
+                "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==1.5.8"
+            "version": "==1.6.0"
         },
         "netcdf4": {
             "hashes": [
@@ -1748,7 +1754,7 @@
                 "sha256:b4625a4b7a597839dd3156b140d5ba2c7123761f98245a3290f67a8b8ee048d9",
                 "sha256:c1e2eb2e3b6079a0552a04974883a48d04c3c05792170d64a4b23d707d453181"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==6.5.6"
         },
         "notebook-shim": {
@@ -1756,7 +1762,7 @@
                 "sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7",
                 "sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.2.3"
         },
         "numba": {
@@ -1795,38 +1801,38 @@
         },
         "numexpr": {
             "hashes": [
-                "sha256:0d7bfc8b77d8a7b04cd64ae42b62b3bf824a8c751ca235692bfd5231c6e90127",
-                "sha256:12146521b1730073859a20454e75004e38cd0cb61333e763c58ef5171e101eb2",
-                "sha256:121b049b6909787111daf92919c052c4fd87b5691172e8f19f702b96f20aaafa",
-                "sha256:17104051f0bd83fd350212e268d8b48017d5eff522b09b573fdbcc560c5e7ab3",
-                "sha256:22ccd67c0fbeae091f2c577f5b9c8046de6631d46b1cbe22aad46a08d2b42c2d",
-                "sha256:290f91c7ba7772abaf7107f3cc0601d93d6a3f21c13ee3da93f1b8a9ca3e8d39",
-                "sha256:296dc1f79d386166dec3bdb45f51caba29ffd8dc91db15447108c04d3001d921",
-                "sha256:2ae264c35fa67cd510191ab8144f131fddd0f1d13413af710913ea6fc0c6aa61",
-                "sha256:307b49fd15ef2ca292f381e67759e5b477410341f2f499a377234f1b42f529a6",
-                "sha256:399cb914b41c4027ba88a18f6b8ccfc3af5c32bc3b1758403a7c44c72530618a",
-                "sha256:3f168b4b42d4cb120fe1993676dcf74b77a3e8e45b58855566da037cfd938ca3",
-                "sha256:47c05007cd1c553515492c1a78b5477eaaba9cadc5d7b795d49f7aae53ccdf68",
-                "sha256:4d83a542d9deefb050e389aacaddea0f09d68ec617dd37e45b9a7cfbcba6d729",
-                "sha256:4f01d71db6fdb97a68def5407e2dbd748eaea9d98929db08816de40aa4ae3084",
-                "sha256:5a92f230dd9d6c42803f855970e93677b44290b6dad15cb6796fd85edee171ce",
-                "sha256:6459dc6ed6abcdeab3cd3667c79f29e4a0f0a02c29ad71ee5cff065e880ee9ef",
-                "sha256:76f0f010f9c6318bae213b21c5c0e381c2fc9c9ecb8b35f99f5030e7ac96c9ce",
-                "sha256:7badc50efbb2f1c8b78cd68089031e0fd29cbafa6a9e6d730533f22d88168406",
-                "sha256:85c9f79e346c26aa0d425ecfc9e5de7184567d5e48d0bdb02d468bb927e92525",
-                "sha256:925927cd1f610593e7783d8f2e12e3d800d5928601e077e4910e2b50bde624b6",
-                "sha256:a82d710145b0fbaec919dde9c90ed9df1e6785625cc36d1c71f3a53112b66fc5",
-                "sha256:aab17d65751c039d13ed9d49c9a7517b130ef488c1885c4666af9b5c6ad59520",
-                "sha256:b4649c1dcf9b0c2ae0a7b767dbbbde4e05ee68480c1ba7f06fc7963f1f73acf4",
-                "sha256:bf8c517bbbb82c07c23c17f9d52b4c9f86601f57d48e87c0cbda24af5907f4dd",
-                "sha256:ccef9b09432d59229c2a737882e55de7906006452003323e107576f264cec373",
-                "sha256:cd07793b074cc38e478637cbe738dff7d8eb92b5cf8ffaacff0c4f0bca5270a0",
-                "sha256:dbac846f713b4c82333e6af0814ebea0b4e74dfb2649e76c58953fd4862322dd",
-                "sha256:e76ce4d25372f46170cf7eb1ff14ed5d9c69a0b162a405063cbe481bafe3af34",
-                "sha256:f031ac4e70f9ad867543bfbde8452e9d1a14f0525346b4b8bd4e5c0f1380a11c"
+                "sha256:00dab81d49239ea5423861ad627097b44d10d802df5f883d1b00f742139c3349",
+                "sha256:0e2574cafb18373774f351cac45ed23b5b360d9ecd1dbf3c12dac6d6eefefc87",
+                "sha256:0f619e91034b346ea85a4e1856ff06011dcb7dce10a60eda75e74db90120f880",
+                "sha256:1a78b937861d13de67d440d54c85a835faed7572be5a6fd10d4f3bd4e66e157f",
+                "sha256:1fae6828042b70c2f52a132bfcb9139da704274ed11b982fbf537f91c075d2ef",
+                "sha256:2749bce1c48706d58894992634a43b8458c4ba9411191471c4565fa41e9979ec",
+                "sha256:374dc6ca54b2af813cb15c2b34e85092dfeac1f73d51ec358dd81876bd9adcec",
+                "sha256:37a7dd36fd79a2b69c3fd2bc2b51ac8270bebc69cc96e6d78f1148e147fcbfa8",
+                "sha256:3b03a6cf37a72f5b52f2b962d7ac7f565bea8eaba83c3c4e5fcf8fbb6a938153",
+                "sha256:50f57bc333f285e8c46b1ce61c6e94ec9bb74e4ea0d674d1c6c6f4a286f64fe4",
+                "sha256:520e55d75bd99c76e376b6326e35ecf44c5ce2635a5caed72799a3885fc49173",
+                "sha256:549afc1622296cca3478a132c6e0fb5e55a19e08d32bc0d5a415434824a9c157",
+                "sha256:5615497c3f34b637fda9b571f7774b6a82f2367cc1364b7a4573068dd1aabcaa",
+                "sha256:6eae6c0c2d5682c02e8ac9c4287c2232c2443c9148b239df22500eaa3c5d73b7",
+                "sha256:7c618a5895e34db0a364dcdb9960084c080f93f9d377c45b1ca9c394c24b4e77",
+                "sha256:7c77392aea53f0700d60eb270ad63174b4ff10b04f8de92861101ca2129fee51",
+                "sha256:8efd879839572bde5a38a1aa3ac23fd4dd9b956fb969bc5e43d1c403419e1e8c",
+                "sha256:943ba141f3884ffafa3fa1a3ebf3cdda9e9688a67a3c91986e6eae13dc073d43",
+                "sha256:972e29b5cecc21466c5b177e38568372ab66aab1f053ae04690a49cea09e747d",
+                "sha256:9761195526a228e05eba400b8c484c94bbabfea853b9ea35ab8fa1bf415331b1",
+                "sha256:977537f2a1cc843f888fb5f0507626f956ada674e4b3847168214a3f3c7446fa",
+                "sha256:aa6298fb46bd7ec69911b5b80927a00663d066e719b29f48eb952d559bdd8371",
+                "sha256:b04f12a6130094a251e3a8fff40130589c1c83be6d4eb223873bea14d8c8b630",
+                "sha256:bffcbc55dea5a5f5255e2586da08f00929998820e6592ee717273a08ad021eb3",
+                "sha256:c52b4ac54514f5d4d8ead66768810cd5f77aa198e6064213d9b5c7b2e1c97c35",
+                "sha256:d655b6eacc4e81006b662cba014e4615a9ddd96881b8b4db4ad0d7f6d38069af",
+                "sha256:e1c31f621a625c7be602f92b027d90f2d3d60dcbc19b106e77fb04a4362152af",
+                "sha256:ee48acd6339748a65c0e32403b802ebfadd9cb0e3b602ba5889896238eafdd61",
+                "sha256:f21d12f6c432ce349089eb95342babf6629aebb3fddf187a4492d3aadaadaaf0"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==2.8.8"
+            "version": "==2.9.0"
         },
         "numpy": {
             "hashes": [
@@ -1880,18 +1886,18 @@
         },
         "overrides": {
             "hashes": [
-                "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d",
-                "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"
+                "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a",
+                "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==7.4.0"
+            "version": "==7.7.0"
         },
         "packaging": {
             "hashes": [
                 "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
                 "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==23.2"
         },
         "pandas": {
@@ -1930,24 +1936,24 @@
                 "sha256:1cc70f5f182f5b4bd2e248f49372fcd88eba8423b8604ea9e1ebd64222730a41",
                 "sha256:1cf13c06068d64edb1630b0e50d6588243b7c202d7ce00d968fe3d6bef7684f8"
             ],
-            "markers": "python_version < '3.12' and python_version >= '3.7'",
+            "markers": "python_version < '3.12' and python_full_version >= '3.7.0'",
             "version": "==0.15.1"
         },
         "pandocfilters": {
             "hashes": [
-                "sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38",
-                "sha256:33aae3f25fd1a026079f5d27bdd52496f0e0803b3469282162bafdcbdf6ef14f"
+                "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e",
+                "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.5.0"
+            "version": "==1.5.1"
         },
         "param": {
             "hashes": [
-                "sha256:4bfc94c0e4127626fa833e30c71c91ea73e7675b80c26dbdd4a6e5a8f6dc46db",
-                "sha256:7943a04607822efd46e96e1827dc5fa929a2fc3b1fe9fc7b7dca7d17a8031a5b"
+                "sha256:785845a727a588eb94c7666d80551c7e2bb97d4309d3507beab66f95e57f7527",
+                "sha256:b269fd7397886ec609e544f81035fa52e1950da0e76d20080bfeca3d7a0317ca"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.0.1"
+            "version": "==2.0.2"
         },
         "parso": {
             "hashes": [
@@ -1962,7 +1968,7 @@
                 "sha256:27e766663d36c161e2827aa3e28541c992f0b9527d3cca047e13fb3acdb989e6",
                 "sha256:56c25dd49e6fea5727e731203c466c6e092f308d8f0024e199d02f6aa2167f67"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.4.1"
         },
         "pathspec": {
@@ -1983,79 +1989,93 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d",
-                "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de",
-                "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616",
-                "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839",
-                "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099",
-                "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a",
-                "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219",
-                "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106",
-                "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b",
-                "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412",
-                "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b",
-                "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7",
-                "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2",
-                "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7",
-                "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14",
-                "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f",
-                "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27",
-                "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57",
-                "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262",
-                "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28",
-                "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610",
-                "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172",
-                "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273",
-                "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e",
-                "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d",
-                "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818",
-                "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f",
-                "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9",
-                "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01",
-                "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7",
-                "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651",
-                "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312",
-                "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80",
-                "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666",
-                "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061",
-                "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b",
-                "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992",
-                "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593",
-                "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4",
-                "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db",
-                "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba",
-                "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd",
-                "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e",
-                "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212",
-                "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb",
-                "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2",
-                "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34",
-                "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256",
-                "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f",
-                "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2",
-                "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38",
-                "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996",
-                "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a",
-                "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793"
+                "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8",
+                "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39",
+                "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac",
+                "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869",
+                "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e",
+                "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04",
+                "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9",
+                "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e",
+                "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe",
+                "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef",
+                "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56",
+                "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa",
+                "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f",
+                "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f",
+                "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e",
+                "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a",
+                "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2",
+                "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2",
+                "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5",
+                "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a",
+                "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2",
+                "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213",
+                "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563",
+                "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591",
+                "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c",
+                "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2",
+                "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb",
+                "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757",
+                "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0",
+                "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452",
+                "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad",
+                "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01",
+                "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f",
+                "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5",
+                "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61",
+                "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e",
+                "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b",
+                "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068",
+                "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9",
+                "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588",
+                "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483",
+                "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f",
+                "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67",
+                "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7",
+                "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311",
+                "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6",
+                "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72",
+                "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6",
+                "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129",
+                "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13",
+                "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67",
+                "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c",
+                "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516",
+                "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e",
+                "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e",
+                "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364",
+                "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023",
+                "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1",
+                "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04",
+                "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d",
+                "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a",
+                "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7",
+                "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb",
+                "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4",
+                "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e",
+                "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1",
+                "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48",
+                "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==10.1.0"
+            "index": "pypi",
+            "version": "==10.2.0"
         },
         "platformdirs": {
             "hashes": [
-                "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380",
-                "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"
+                "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068",
+                "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.1.0"
+            "version": "==4.2.0"
         },
         "pluggy": {
             "hashes": [
-                "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
-                "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"
+                "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981",
+                "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.3.0"
+            "version": "==1.4.0"
         },
         "pre-commit": {
             "hashes": [
@@ -2078,30 +2098,30 @@
                 "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
                 "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.43"
         },
         "psutil": {
             "hashes": [
-                "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340",
-                "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6",
-                "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284",
-                "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c",
-                "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7",
-                "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c",
-                "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e",
-                "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6",
-                "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056",
-                "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9",
-                "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68",
-                "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df",
-                "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e",
-                "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414",
-                "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508",
-                "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"
+                "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d",
+                "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73",
+                "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8",
+                "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2",
+                "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e",
+                "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36",
+                "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7",
+                "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c",
+                "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee",
+                "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421",
+                "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf",
+                "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81",
+                "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0",
+                "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631",
+                "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4",
+                "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"
             ],
             "markers": "sys_platform != 'cygwin'",
-            "version": "==5.9.7"
+            "version": "==5.9.8"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -2307,49 +2327,48 @@
         },
         "pycryptodomex": {
             "hashes": [
-                "sha256:011e859026ecbd15b8e720e8992361186e582cf726c50bde6ff8c0c05e820ddf",
-                "sha256:0b42e2743893f386dfb58fe24a4c8be5305c3d1c825d5f23d9e63fd0700d1110",
-                "sha256:0b7154aff2272962355f8941fd514104a88cb29db2d8f43a29af900d6398eb1c",
-                "sha256:0bc4b7bfaac56e6dfd62044847443a3d110c7abea7fcb0d68c1aea64ed3a6697",
-                "sha256:10c2eed4efdfa084b602ab922e699a0a2ba82053baebfc8afcaf27489def7955",
-                "sha256:1c04cfff163c05d033bf28e3c4429d8222796738c7b6c1638b9d7090b904611e",
-                "sha256:23707238b024b36c35dd3428f5af6c1f0c5ef54c21e387a2063633717699b8b2",
-                "sha256:371bbe0be17b4dd8cc0c2f378d75ea33f00d5a39884c09a672016ac40145a5fa",
-                "sha256:39eb1f82ac3ba3e39d866f38e480e8fa53fcdd22260340f05f54a8188d47d510",
-                "sha256:3f3c58971784fba0e014bc3f8aed1197b86719631e1b597d36d7354be5598312",
-                "sha256:5ca98de2e5ac100e57a7116309723360e8f799f722509e376dc396cdf65eec9c",
-                "sha256:62f51a63d73153482729904381dd2de86800b0733a8814ee8f072fa73e5c92fb",
-                "sha256:76414d39df6b45bcc4f38cf1ba2031e0f4b8e99d1ba3c2eee31ffe1b9f039733",
-                "sha256:8dffe067d5fff14dba4d18ff7d459cc2a47576d82dafbff13a8f1199c3353e41",
-                "sha256:96000b837bcd8e3bf86b419924a056c978e45027281e4318650c81c25a3ef6cc",
-                "sha256:9919a1edd2a83c4dfb69f1d8a4c0c5efde7147ef15b07775633372b80c90b5d8",
-                "sha256:aab7941c2ff53eb63cb26252770e4f14386d79ce07baeffbf98a1323c1646545",
-                "sha256:ac562e239d98cfef763866c0aee4586affb0d58c592202f06c87241af99db241",
-                "sha256:ae75eea2e908383fd4c659fdcfe9621a72869e3e3ee73904227e93b7f7b80b54",
-                "sha256:b5c336dc698650283ad06f8c0237a984087d0af9f403ff21d633507335628156",
-                "sha256:beb5f0664f49b6093da179ee8e27c1d670779f50b9ece0886ce491bb8bd63728",
-                "sha256:c1ae2fb8d5d6771670436dcc889b293e363c97647a6d31c21eebc12b7b760010",
-                "sha256:c9332b04bf3f838327087b028f690f4ddb9341eb014a0221e79b9c19a77f7555",
-                "sha256:c9cb88ed323be1aa642b3c17cd5caa1a03c3a8fbad092d48ecefe88e328ffae3",
-                "sha256:d45d0d35a238d838b872598fa865bbfb31aaef9aeeda77c68b04ef79f9a469dc",
-                "sha256:d7a77391fd351ff1bdf8475558ddc6e92950218cb905419ee14aa02f370f1054",
-                "sha256:de5a43901e47e7a6938490fc5de3074f6e35c8b481a75b227c0d24d6099bd41d",
-                "sha256:e94a7e986b117b72e9472f8eafdd81748dafff30815401f9760f759f1debe9ef",
-                "sha256:ed3bdda44cc05dd13eee697ab9bea6928531bb7b218e68e66d0d3eb2ebab043e",
-                "sha256:f24f49fc6bd706d87048654d6be6c7c967d6836d4879e3a7c439275fab9948ad",
-                "sha256:f8a97b1acd36e9ce9d4067d94a8be99c458f0eb8070828639302a95cfcf0770b",
-                "sha256:f8b3d9e7c17c1ffc1fa5b11c0bbab8a5df3de8596bb32ad30281b21e5ede4bf5"
+                "sha256:0daad007b685db36d977f9de73f61f8da2a7104e20aca3effd30752fd56f73e1",
+                "sha256:108e5f1c1cd70ffce0b68739c75734437c919d2eaec8e85bffc2c8b4d2794305",
+                "sha256:19764605feea0df966445d46533729b645033f134baeb3ea26ad518c9fdf212c",
+                "sha256:1be97461c439a6af4fe1cf8bf6ca5936d3db252737d2f379cc6b2e394e12a458",
+                "sha256:25cd61e846aaab76d5791d006497134602a9e451e954833018161befc3b5b9ed",
+                "sha256:2a47bcc478741b71273b917232f521fd5704ab4b25d301669879e7273d3586cc",
+                "sha256:59af01efb011b0e8b686ba7758d59cf4a8263f9ad35911bfe3f416cee4f5c08c",
+                "sha256:5dcac11031a71348faaed1f403a0debd56bf5404232284cf8c761ff918886ebc",
+                "sha256:62a5ec91388984909bb5398ea49ee61b68ecb579123694bffa172c3b0a107079",
+                "sha256:645bd4ca6f543685d643dadf6a856cc382b654cc923460e3a10a49c1b3832aeb",
+                "sha256:653b29b0819605fe0898829c8ad6400a6ccde096146730c2da54eede9b7b8baa",
+                "sha256:69138068268127cd605e03438312d8f271135a33140e2742b417d027a0539427",
+                "sha256:6e186342cfcc3aafaad565cbd496060e5a614b441cacc3995ef0091115c1f6c5",
+                "sha256:76bd15bb65c14900d98835fcd10f59e5e0435077431d3a394b60b15864fddd64",
+                "sha256:7805830e0c56d88f4d491fa5ac640dfc894c5ec570d1ece6ed1546e9df2e98d6",
+                "sha256:7a710b79baddd65b806402e14766c721aee8fb83381769c27920f26476276c1e",
+                "sha256:7a7a8f33a1f1fb762ede6cc9cbab8f2a9ba13b196bfaf7bc6f0b39d2ba315a43",
+                "sha256:82ee7696ed8eb9a82c7037f32ba9b7c59e51dda6f105b39f043b6ef293989cb3",
+                "sha256:88afd7a3af7ddddd42c2deda43d53d3dfc016c11327d0915f90ca34ebda91499",
+                "sha256:8af1a451ff9e123d0d8bd5d5e60f8e3315c3a64f3cdd6bc853e26090e195cdc8",
+                "sha256:8ee606964553c1a0bc74057dd8782a37d1c2bc0f01b83193b6f8bb14523b877b",
+                "sha256:91852d4480a4537d169c29a9d104dda44094c78f1f5b67bca76c29a91042b623",
+                "sha256:9c682436c359b5ada67e882fec34689726a09c461efd75b6ea77b2403d5665b7",
+                "sha256:bc3ee1b4d97081260d92ae813a83de4d2653206967c4a0a017580f8b9548ddbc",
+                "sha256:bca649483d5ed251d06daf25957f802e44e6bb6df2e8f218ae71968ff8f8edc4",
+                "sha256:c39778fd0548d78917b61f03c1fa8bfda6cfcf98c767decf360945fe6f97461e",
+                "sha256:cbe71b6712429650e3883dc81286edb94c328ffcd24849accac0a4dbcc76958a",
+                "sha256:d00fe8596e1cc46b44bf3907354e9377aa030ec4cd04afbbf6e899fc1e2a7781",
+                "sha256:d3584623e68a5064a04748fb6d76117a21a7cb5eaba20608a41c7d0c61721794",
+                "sha256:e48217c7901edd95f9f097feaa0388da215ed14ce2ece803d3f300b4e694abea",
+                "sha256:f2e497413560e03421484189a6b65e33fe800d3bd75590e6d78d4dfdb7accf3b",
+                "sha256:ff5c9a67f8a4fba4aed887216e32cbc48f2a6fb2673bb10a99e43be463e15913"
             ],
-            "index": "pypi",
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==3.19.1"
+            "version": "==3.20.0"
         },
         "pyct": {
             "hashes": [
                 "sha256:a4038a8885059ab8cac6f946ea30e0b5e6bdbe0b92b6723f06737035f9d65e8c",
                 "sha256:dd9f4ac5cbd8e37c352c04036062d3c5f67efec76d404761ef16b0cbf26aa6a0"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.5.0"
         },
         "pydantic": {
@@ -2391,7 +2410,7 @@
                 "sha256:f3d4ee957a727ccb5a36f1b0a6dbd9fad5dedd2a41eada99a8df55c12896e18d",
                 "sha256:f79db3652ed743309f116ba863dae0c974a41b688242482638b892246b7db21d"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.10.10"
         },
         "pyflakes": {
@@ -2415,7 +2434,7 @@
                 "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
                 "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.17.2"
         },
         "pyparsing": {
@@ -2585,10 +2604,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b",
-                "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"
+                "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40",
+                "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a"
             ],
-            "version": "==2023.3.post1"
+            "version": "==2023.4"
         },
         "pyyaml": {
             "hashes": [
@@ -2621,6 +2640,7 @@
                 "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
                 "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
                 "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
+                "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
                 "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
                 "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
                 "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
@@ -2869,7 +2889,7 @@
                 "sha256:1c1d8c4fa2c884ae742b069151b0abe15b3f70491f3972698c683b8e38de839b",
                 "sha256:a5a15ffd519550a1361bdc56ffc07fda56a6af7292f17c7b395d4083af632987"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.4.1"
         },
         "rasterio": {
@@ -2905,18 +2925,18 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:689e64fe121843dcfd57b71933318ef1f91188ffb45367332700a86ac8fd6161",
-                "sha256:bdcd3efb936f82ff86f993093f6da7435c7de69a3b3a5a06678a6050184bee99"
+                "sha256:39240f2ecc770258f28b642dd47fd74bc8b02484de54e1882b74b35ebd779bd5",
+                "sha256:c775fedf74bc0f9189c2a3be1c12fd03e8c23f4d371dce795df44e06c5b412f7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.32.0"
+            "version": "==0.33.0"
         },
         "requests": {
             "hashes": [
                 "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
                 "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.31.0"
         },
         "rfc3339-validator": {
@@ -2964,116 +2984,116 @@
         },
         "rioxarray": {
             "hashes": [
-                "sha256:d2a8429a5b6405913c7b6f515ef2992b05139c96eb39a2dc1c9f475ce0848c9c",
-                "sha256:d7c0b2efc21075f77fe04302b916a995320004695f3c31e4f06d9ab40acd4498"
+                "sha256:75c41e5836a9adba2a42a2cdbc62a50b81e01bd4fcc9f55a96f187322182d671",
+                "sha256:c6d30d8b1b12924dffc45c4d55764840039fa0c8b551589d6cc1364e158e6055"
             ],
-            "markers": "python_version >= '3.9'",
-            "version": "==0.15.0"
+            "markers": "python_version >= '3.10'",
+            "version": "==0.15.1"
         },
         "rpds-py": {
             "hashes": [
-                "sha256:02744236ac1895d7be837878e707a5c35fb8edc5137602f253b63623d7ad5c8c",
-                "sha256:03f9c5875515820633bd7709a25c3e60c1ea9ad1c5d4030ce8a8c203309c36fd",
-                "sha256:044f6f46d62444800402851afa3c3ae50141f12013060c1a3a0677e013310d6d",
-                "sha256:07a2e1d78d382f7181789713cdf0c16edbad4fe14fe1d115526cb6f0eef0daa3",
-                "sha256:082e0e55d73690ffb4da4352d1b5bbe1b5c6034eb9dc8c91aa2a3ee15f70d3e2",
-                "sha256:13152dfe7d7c27c40df8b99ac6aab12b978b546716e99f67e8a67a1d441acbc3",
-                "sha256:13716e53627ad97babf72ac9e01cf9a7d4af2f75dd5ed7b323a7a9520e948282",
-                "sha256:13ff62d3561a23c17341b4afc78e8fcfd799ab67c0b1ca32091d71383a98ba4b",
-                "sha256:1607cda6129f815493a3c184492acb5ae4aa6ed61d3a1b3663aa9824ed26f7ac",
-                "sha256:164fcee32f15d04d61568c9cb0d919e37ff3195919cd604039ff3053ada0461b",
-                "sha256:1c24e30d720c0009b6fb2e1905b025da56103c70a8b31b99138e4ed1c2a6c5b0",
-                "sha256:1e6fcd0a0f62f2997107f758bb372397b8d5fd5f39cc6dcb86f7cb98a2172d6c",
-                "sha256:1fd0f0b1ccd7d537b858a56355a250108df692102e08aa2036e1a094fd78b2dc",
-                "sha256:2181e86d4e1cdf49a7320cb72a36c45efcb7670d0a88f09fd2d3a7967c0540fd",
-                "sha256:2974e6dff38afafd5ccf8f41cb8fc94600b3f4fd9b0a98f6ece6e2219e3158d5",
-                "sha256:2dccc623725d0b298f557d869a68496a2fd2a9e9c41107f234fa5f7a37d278ac",
-                "sha256:2df3d07a16a3bef0917b28cd564778fbb31f3ffa5b5e33584470e2d1b0f248f0",
-                "sha256:2e7e5633577b3bd56bf3af2ef6ae3778bbafb83743989d57f0e7edbf6c0980e4",
-                "sha256:2ee066a64f0d2ba45391cac15b3a70dcb549e968a117bd0500634754cfe0e5fc",
-                "sha256:2f1f295a5c28cfa74a7d48c95acc1c8a7acd49d7d9072040d4b694fe11cd7166",
-                "sha256:2faa97212b0dc465afeedf49045cdd077f97be1188285e646a9f689cb5dfff9e",
-                "sha256:30479a9f1fce47df56b07460b520f49fa2115ec2926d3b1303c85c81f8401ed1",
-                "sha256:337a8653fb11d2fbe7157c961cc78cb3c161d98cf44410ace9a3dc2db4fad882",
-                "sha256:3423007fc0661827e06f8a185a3792c73dda41f30f3421562f210cf0c9e49569",
-                "sha256:373b76eeb79e8c14f6d82cb1d4d5293f9e4059baec6c1b16dca7ad13b6131b39",
-                "sha256:3b79c63d29101cbaa53a517683557bb550462394fb91044cc5998dd2acff7340",
-                "sha256:3bbc89ce2a219662ea142f0abcf8d43f04a41d5b1880be17a794c39f0d609cb0",
-                "sha256:3c11bc5814554b018f6c5d6ae0969e43766f81e995000b53a5d8c8057055e886",
-                "sha256:3cd61e759c4075510052d1eca5cddbd297fe1164efec14ef1fce3f09b974dfe4",
-                "sha256:3d40fb3ca22e3d40f494d577441b263026a3bd8c97ae6ce89b2d3c4b39ac9581",
-                "sha256:3db0c998c92b909d7c90b66c965590d4f3cd86157176a6cf14aa1f867b77b889",
-                "sha256:422b0901878a31ef167435c5ad46560362891816a76cc0d150683f3868a6f0d1",
-                "sha256:46b4f3d47d1033db569173be62365fbf7808c2bd3fb742314d251f130d90d44c",
-                "sha256:485fbdd23becb822804ed05622907ee5c8e8a5f43f6f43894a45f463b2217045",
-                "sha256:53304cc14b1d94487d70086e1cb0cb4c29ec6da994d58ae84a4d7e78c6a6d04d",
-                "sha256:5595c80dd03d7e6c6afb73f3594bf3379a7d79fa57164b591d012d4b71d6ac4c",
-                "sha256:56b51ba29a18e5f5810224bcf00747ad931c0716e3c09a76b4a1edd3d4aba71f",
-                "sha256:580182fa5b269c2981e9ce9764367cb4edc81982ce289208d4607c203f44ffde",
-                "sha256:5e99d6510c8557510c220b865d966b105464740dcbebf9b79ecd4fbab30a13d9",
-                "sha256:5eb05b654a41e0f81ab27a7c3e88b6590425eb3e934e1d533ecec5dc88a6ffff",
-                "sha256:62b292fff4739c6be89e6a0240c02bda5a9066a339d90ab191cf66e9fdbdc193",
-                "sha256:6a5122b17a4faf5d7a6d91fa67b479736c0cacc7afe791ddebb7163a8550b799",
-                "sha256:6a8ff8e809da81363bffca2b965cb6e4bf6056b495fc3f078467d1f8266fe27f",
-                "sha256:6c43e1b89099279cc03eb1c725c5de12af6edcd2f78e2f8a022569efa639ada3",
-                "sha256:709dc11af2f74ba89c68b1592368c6edcbccdb0a06ba77eb28c8fe08bb6997da",
-                "sha256:7e072f5da38d6428ba1fc1115d3cc0dae895df671cb04c70c019985e8c7606be",
-                "sha256:813a65f95bfcb7c8f2a70dd6add9b51e9accc3bdb3e03d0ff7a9e6a2d3e174bf",
-                "sha256:86c01299942b0f4b5b5f28c8701689181ad2eab852e65417172dbdd6c5b3ccc8",
-                "sha256:893e38d0f4319dfa70c0f36381a37cc418985c87b11d9784365b1fff4fa6973b",
-                "sha256:8a5f574b92b3ee7d254e56d56e37ec0e1416acb1ae357c4956d76a1788dc58fb",
-                "sha256:8b9650f92251fdef843e74fc252cdfd6e3c700157ad686eeb0c6d7fdb2d11652",
-                "sha256:8ec464f20fe803ae00419bd1610934e3bda963aeba1e6181dfc9033dc7e8940c",
-                "sha256:8f333bfe782a2d05a67cfaa0cc9cd68b36b39ee6acfe099f980541ed973a7093",
-                "sha256:8ffdeb7dbd0160d4e391e1f857477e4762d00aa2199c294eb95dfb9451aa1d9f",
-                "sha256:911e600e798374c0d86235e7ef19109cf865d1336942d398ff313375a25a93ba",
-                "sha256:9235be95662559141934fced8197de6fee8c58870f36756b0584424b6d708393",
-                "sha256:938518a11780b39998179d07f31a4a468888123f9b00463842cd40f98191f4d3",
-                "sha256:93c18a1696a8e0388ed84b024fe1a188a26ba999b61d1d9a371318cb89885a8c",
-                "sha256:97532802f14d383f37d603a56e226909f825a83ff298dc1b6697de00d2243999",
-                "sha256:98ee201a52a7f65608e5494518932e1473fd43535f12cade0a1b4ab32737fe28",
-                "sha256:9d2ae79f31da5143e020a8d4fc74e1f0cbcb8011bdf97453c140aa616db51406",
-                "sha256:9d38494a8d21c246c535b41ecdb2d562c4b933cf3d68de03e8bc43a0d41be652",
-                "sha256:9d41ebb471a6f064c0d1c873c4f7dded733d16ca5db7d551fb04ff3805d87802",
-                "sha256:9e09d017e3f4d9bd7d17a30d3f59e4d6d9ba2d2ced280eec2425e84112cf623f",
-                "sha256:a6945c2d61c42bb7e818677f43638675b8c1c43e858b67a96df3eb2426a86c9d",
-                "sha256:a72e00826a2b032dda3eb25aa3e3579c6d6773d22d8446089a57a123481cc46c",
-                "sha256:aa1e626c524d2c7972c0f3a8a575d654a3a9c008370dc2a97e46abd0eaa749b9",
-                "sha256:ab095edf1d840a6a6a4307e1a5b907a299a94e7b90e75436ee770b8c35d22a25",
-                "sha256:ac2ac84a4950d627d84b61f082eba61314373cfab4b3c264b62efab02ababe83",
-                "sha256:ac7187bee72384b9cfedf09a29a3b2b6e8815cc64c095cdc8b5e6aec81e9fd5f",
-                "sha256:ae9d83a81b09ce3a817e2cbb23aabc07f86a3abc664c613cd283ce7a03541e95",
-                "sha256:afeabb382c1256a7477b739820bce7fe782bb807d82927102cee73e79b41b38b",
-                "sha256:b2a4cd924d0e2f4b1a68034abe4cadc73d69ad5f4cf02db6481c0d4d749f548f",
-                "sha256:b414ef79f1f06fb90b5165db8aef77512c1a5e3ed1b4807da8476b7e2c853283",
-                "sha256:b4ecbba7efd82bd2a4bb88aab7f984eb5470991c1347bdd1f35fb34ea28dba6e",
-                "sha256:b61d5096e75fd71018b25da50b82dd70ec39b5e15bb2134daf7eb7bbbc103644",
-                "sha256:b629db53fe17e6ce478a969d30bd1d0e8b53238c46e3a9c9db39e8b65a9ef973",
-                "sha256:b70b45a40ad0798b69748b34d508259ef2bdc84fb2aad4048bc7c9cafb68ddb3",
-                "sha256:b88c3ab98556bc351b36d6208a6089de8c8db14a7f6e1f57f82a334bd2c18f0b",
-                "sha256:baf744e5f9d5ee6531deea443be78b36ed1cd36c65a0b95ea4e8d69fa0102268",
-                "sha256:bbc7421cbd28b4316d1d017db338039a7943f945c6f2bb15e1439b14b5682d28",
-                "sha256:c31272c674f725dfe0f343d73b0abe8c878c646967ec1c6106122faae1efc15b",
-                "sha256:c51a899792ee2c696072791e56b2020caff58b275abecbc9ae0cb71af0645c95",
-                "sha256:c61e42b4ceb9759727045765e87d51c1bb9f89987aca1fcc8a040232138cad1c",
-                "sha256:c7cd0841a586b7105513a7c8c3d5c276f3adc762a072d81ef7fae80632afad1e",
-                "sha256:c827a931c6b57f50f1bb5de400dcfb00bad8117e3753e80b96adb72d9d811514",
-                "sha256:d2aa3ca9552f83b0b4fa6ca8c6ce08da6580f37e3e0ab7afac73a1cfdc230c0e",
-                "sha256:d46ee458452727a147d7897bb33886981ae1235775e05decae5d5d07f537695a",
-                "sha256:d64a657de7aae8db2da60dc0c9e4638a0c3893b4d60101fd564a3362b2bfeb34",
-                "sha256:d800a8e2ac62db1b9ea5d6d1724f1a93c53907ca061de4d05ed94e8dfa79050c",
-                "sha256:d9d7ebcd11ea76ba0feaae98485cd8e31467c3d7985210fab46983278214736b",
-                "sha256:dd7d3608589072f63078b4063a6c536af832e76b0b3885f1bfe9e892abe6c207",
-                "sha256:ec19e823b4ccd87bd69e990879acbce9e961fc7aebe150156b8f4418d4b27b7f",
-                "sha256:ee40206d1d6e95eaa2b7b919195e3689a5cf6ded730632de7f187f35a1b6052c",
-                "sha256:f138f550b83554f5b344d6be35d3ed59348510edc3cb96f75309db6e9bfe8210",
-                "sha256:f3e6e2e502c4043c52a99316d89dc49f416acda5b0c6886e0dd8ea7bb35859e8",
-                "sha256:fb10bb720348fe1647a94eb605accb9ef6a9b1875d8845f9e763d9d71a706387",
-                "sha256:fc066395e6332da1e7525d605b4c96055669f8336600bef8ac569d5226a7c76f",
-                "sha256:fc33267d58dfbb2361baed52668c5d8c15d24bc0372cecbb79fed77339b55e0d"
+                "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147",
+                "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7",
+                "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2",
+                "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68",
+                "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1",
+                "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382",
+                "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d",
+                "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921",
+                "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38",
+                "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4",
+                "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a",
+                "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d",
+                "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518",
+                "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e",
+                "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d",
+                "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf",
+                "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5",
+                "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba",
+                "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6",
+                "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59",
+                "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253",
+                "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6",
+                "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f",
+                "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3",
+                "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea",
+                "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1",
+                "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76",
+                "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93",
+                "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad",
+                "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad",
+                "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc",
+                "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049",
+                "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d",
+                "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90",
+                "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d",
+                "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd",
+                "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25",
+                "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2",
+                "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f",
+                "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6",
+                "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4",
+                "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c",
+                "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8",
+                "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d",
+                "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b",
+                "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19",
+                "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453",
+                "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9",
+                "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde",
+                "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296",
+                "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58",
+                "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec",
+                "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99",
+                "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a",
+                "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb",
+                "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383",
+                "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d",
+                "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896",
+                "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc",
+                "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6",
+                "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b",
+                "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7",
+                "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22",
+                "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf",
+                "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394",
+                "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0",
+                "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57",
+                "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74",
+                "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83",
+                "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29",
+                "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9",
+                "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f",
+                "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745",
+                "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb",
+                "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811",
+                "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55",
+                "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342",
+                "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23",
+                "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82",
+                "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041",
+                "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb",
+                "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066",
+                "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55",
+                "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6",
+                "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a",
+                "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140",
+                "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b",
+                "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9",
+                "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256",
+                "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c",
+                "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772",
+                "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4",
+                "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae",
+                "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920",
+                "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a",
+                "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b",
+                "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361",
+                "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8",
+                "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.15.2"
+            "version": "==0.17.1"
         },
         "rtree": {
             "hashes": [
@@ -3131,7 +3151,7 @@
                 "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084",
                 "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.6.2"
         },
         "scipy": {
@@ -3179,11 +3199,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2",
-                "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"
+                "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05",
+                "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==69.0.2"
+            "version": "==69.0.3"
         },
         "shapely": {
             "hashes": [
@@ -3346,7 +3366,7 @@
                 "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101",
                 "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.3.0"
         },
         "snuggs": {
@@ -3414,7 +3434,7 @@
                 "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847",
                 "sha256:8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==1.2.1"
         },
         "tomli": {
@@ -3427,11 +3447,11 @@
         },
         "toolz": {
             "hashes": [
-                "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f",
-                "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"
+                "sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85",
+                "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==0.12.0"
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==0.12.1"
         },
         "tornado": {
             "hashes": [
@@ -3460,11 +3480,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:f14949d23829023013c47df20b4a76ccd1a85effb786dc060f34de7948361b33",
-                "sha256:fcdaa8ac49c04dfa0ed3ee3384ef6dfdb5d6f3741502be247279407679296772"
+                "sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74",
+                "sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.14.0"
+            "version": "==5.14.1"
         },
         "typeguard": {
             "hashes": [
@@ -3476,10 +3496,11 @@
         },
         "types-python-dateutil": {
             "hashes": [
-                "sha256:1f4f10ac98bb8b16ade9dbee3518d9ace017821d94b057a425b069f834737f4b",
-                "sha256:f977b8de27787639986b4e28963263fd0e5158942b3ecef91b9335c130cb1ce9"
+                "sha256:1f8db221c3b98e6ca02ea83a58371b22c374f42ae5bbdf186db9c9a76581459f",
+                "sha256:efbbdc54590d0f16152fa103c9879c7d4a00e82078f6e2cf01769042165acaa2"
             ],
-            "version": "==2.8.19.14"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.8.19.20240106"
         },
         "typing-extensions": {
             "hashes": [
@@ -3498,11 +3519,11 @@
         },
         "tzdata": {
             "hashes": [
-                "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a",
-                "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"
+                "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3",
+                "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"
             ],
             "markers": "python_version >= '2'",
-            "version": "==2023.3"
+            "version": "==2023.4"
         },
         "uri-template": {
             "hashes": [
@@ -3524,15 +3545,15 @@
                 "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3",
                 "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==20.25.0"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02",
-                "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"
+                "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859",
+                "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"
             ],
-            "version": "==0.2.12"
+            "version": "==0.2.13"
         },
         "webcolors": {
             "hashes": [
@@ -3569,7 +3590,7 @@
                 "sha256:3c1f5e46dc1166dfd40a42d685e6a51396fd34ff878742a3e47c6f0cc4a2a385",
                 "sha256:91452ca8445beb805792f206e560c1769284267a30ceb1cec9f5bcc887d15175"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==4.0.9"
         },
         "wrapt": {
@@ -3661,7 +3682,7 @@
                 "sha256:8a356ac66a61ff9522453194db5d184e1eccee8ab890c7ec723bb48c6eafd392",
                 "sha256:fd7e3b447236f6e8f8c5110684a7ca50f97149c806f56ebf4ed1eeef1dd681ce"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.3.5"
         },
         "xyzservices": {
@@ -3756,7 +3777,7 @@
                 "sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff",
                 "sha256:b1ba0dfcc9762f0ca168d2378062d3ca1299d39076b0f145d961359121042be5"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==0.8.4"
         },
         "zipp": {
@@ -3801,11 +3822,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:2f55d59370f59d0d2b2212109fe0e6035cfea436b1c0e6150ad2244746272ec5",
-                "sha256:ac4da4ecf0042fb4e0ce57c60430c2db3c719fa8bdf92f8631d6bd8a5785d1f0"
+                "sha256:1050a3ab8473488d7eee163796b02e511d0735cf43a04ba2a8348bd0f2eaf8a5",
+                "sha256:48fbc236fbe0e138b88773fa0437751f14c3645fb483f1d4c5dee58b37e5ce73"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.19.0"
+            "version": "==8.21.0"
         },
         "jedi": {
             "hashes": [
@@ -3851,7 +3872,7 @@
                 "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
                 "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.43"
         },
         "ptyprocess": {
@@ -3874,7 +3895,7 @@
                 "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
                 "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==2.17.2"
         },
         "six": {
@@ -3894,18 +3915,18 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:f14949d23829023013c47df20b4a76ccd1a85effb786dc060f34de7948361b33",
-                "sha256:fcdaa8ac49c04dfa0ed3ee3384ef6dfdb5d6f3741502be247279407679296772"
+                "sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74",
+                "sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.14.0"
+            "version": "==5.14.1"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02",
-                "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"
+                "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859",
+                "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"
             ],
-            "version": "==0.2.12"
+            "version": "==0.2.13"
         }
     }
 }
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index be5e9c1be..d367487a5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.9.1 - 2024-02-02 - [PR#1073](https://github.com/NOAA-OWP/inundation-mapping/pull/1073)
+
+Dependabot requested two fixes. One for an upgrade to pillow [#1068](https://github.com/NOAA-OWP/inundation-mapping/pull/1068) and the other for juypterlab #[1067 ](https://github.com/NOAA-OWP/inundation-mapping/pull/1067)
+
+### Changes
+
+- `src`
+    - `Pipfile` and `Pipfile.lock`: Updated some packages.
+    
+<br/><br/>
+
 ## v4.4.9.0 - 2024-01-12 - [PR#1058](https://github.com/NOAA-OWP/inundation-mapping/pull/1058)
 
 Upgrades base Docker image to GDAL v3.8.0. In order to upgrade past GDAL v.3.4.3 (see #1029), TauDEM's `aread8` was replaced with a module from the `pyflwdir` Python package.

From 682c15bb6e4706ba673bd7a97199d617655db03b Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 2 Feb 2024 09:24:15 -0800
Subject: [PATCH 297/359] v4.4.9.2 Add index to aggregated crosswalk table
 (#1066)

---
 docs/CHANGELOG.md                 | 11 +++++++++++
 tools/combine_crosswalk_tables.py | 12 +++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d367487a5..1c49ad73a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.4.9.2 - 2024-02-02 - [PR#1066](https://github.com/NOAA-OWP/inundation-mapping/pull/1066)
+
+Adds an index to the aggregated `crosswalk_table.csv`. The index is a consecutive integer that starts at 1. Columns have been reordered, renamed, and sorted.
+
+### Changes
+
+`tools/combine_crosswalk_tables.py`: Adds index and sorts and renames columns
+
+<br/><br/>
+
 ## v4.4.9.1 - 2024-02-02 - [PR#1073](https://github.com/NOAA-OWP/inundation-mapping/pull/1073)
 
 Dependabot requested two fixes. One for an upgrade to pillow [#1068](https://github.com/NOAA-OWP/inundation-mapping/pull/1068) and the other for juypterlab #[1067 ](https://github.com/NOAA-OWP/inundation-mapping/pull/1067)
diff --git a/tools/combine_crosswalk_tables.py b/tools/combine_crosswalk_tables.py
index 9acc2292e..79f0d8d90 100644
--- a/tools/combine_crosswalk_tables.py
+++ b/tools/combine_crosswalk_tables.py
@@ -22,7 +22,6 @@ def combine_crosswalk_tables(data_directory, output_filename):
                 filename, usecols=['HUC', 'HydroID', 'feature_id', 'LakeID'], dtype={'HUC': str}
             )
             file_df = file_df.drop_duplicates()
-            file_df = file_df.rename(columns={'HUC': 'huc8'})
             file_df['BranchID'] = os.path.split(os.path.dirname(filename))[1]
 
             dfs.append(file_df)
@@ -32,6 +31,17 @@ def combine_crosswalk_tables(data_directory, output_filename):
     if len(dfs) > 1:
         df = pd.concat(dfs)
 
+        df = df.rename(
+            columns={'HUC': 'huc8', 'HydroID': 'hydro_id', 'LakeID': 'lake_id', 'BranchID': 'branch_id'}
+        )
+
+        df = df.sort_values(by=['feature_id', 'huc8', 'branch_id', 'hydro_id'])
+
+        df = df.reset_index(drop=True)
+        df['hand_id'] = df.index + 1
+
+        df = df[['hand_id', 'feature_id', 'huc8', 'branch_id', 'hydro_id', 'lake_id']]
+
         df.to_csv(output_filename, index=False)
 
 

From a14e5e1cfbe10a4572b0e1f0cddca7506949ce72 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 2 Feb 2024 17:27:24 +0000
Subject: [PATCH 298/359] v4.4.10.0 Updates for acquire, logging and cpu test
 (#1054)

---
 .gitattributes                                |  1 +
 .pre-commit-config.yaml                       |  5 +-
 CONTRIBUTING.md                               | 84 +++++++++++------
 data/usgs/acquire_and_preprocess_3dep_dems.py | 91 +++++++++----------
 data/usgs/preprocess_ahps_usgs.py             |  0
 data/usgs/preprocess_download_usgs_grids.py   |  0
 docs/CHANGELOG.md                             | 28 ++++++
 fim_pipeline.sh                               |  6 +-
 fim_post_processing.sh                        |  2 +
 fim_pre_processing.sh                         | 18 ++--
 fim_process_unit_wb.sh                        |  2 +-
 src/bash_functions.env                        | 28 ++++++
 src/delineate_hydros_and_produce_HAND.sh      | 83 ++---------------
 src/derive_level_paths.py                     |  2 +-
 src/run_by_branch.sh                          | 38 +++-----
 src/run_unit_wb.sh                            | 86 +++---------------
 src/split_flows.py                            |  3 +-
 17 files changed, 214 insertions(+), 263 deletions(-)
 mode change 100644 => 100755 data/usgs/acquire_and_preprocess_3dep_dems.py
 mode change 100644 => 100755 data/usgs/preprocess_ahps_usgs.py
 mode change 100644 => 100755 data/usgs/preprocess_download_usgs_grids.py

diff --git a/.gitattributes b/.gitattributes
index 6033e76e7..76ebc9c6c 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,3 @@
 # Set the default behavior for end of lines.
 * text eol=lf
+docs/CHANGELOG.md merge=union
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 98fbcea77..6c2bb6513 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,9 +6,10 @@ repos:
     rev: v4.4.0
     hooks:
         -   id: trailing-whitespace
-            exclude: CHANGELOG.md
+            # Below is python regex to exclude all .md files
+            exclude: .*md$
         -   id: end-of-file-fixer
-            exclude: Pipfile.lock
+            exclude:  Pipfile.lock
         -   id: check-added-large-files
             args: ['--maxkb=5000']
         -   id: check-json
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 940cfe712..a66e1971f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -45,46 +45,77 @@ If you would like to contribute, please follow these steps:
     ```
     git checkout -b <dev-your-bugfix-or-feature>
     ```
-4. [pre-commit](https://pre-commit.com/) is used to run auto-formatting and linting tools. Please follow the steps in the link provided to install `pre-commit` locally.  `pre-commit` can be installed as a git hook to verify your staged changes adhere to the project's style and format requirements (configuration defined in [pyproject.toml](/pyproject.toml)).
-5. After you've installed `pre-commit` via `pip` , `homebrew`, or `conda`, check the version to verify installation:
-    ```
-    $ pre-commit --version
-    ```
-
-6. Initialize the pre-commit hooks included within the root directory of `inundation-mapping`:
-
+4. Pre-commit installation:
+
+   [pre-commit](https://pre-commit.com/) is used to run auto-formatting and enforce styling rules.
+   It is a critical part of development and is enforced at the 'git commit' step. Key tools are included **inside the docker container** if you want to execute the correctly configured linting and formatting command line executables there. If you intend to execute `flake8`, `black` or `isort` from the command line **outside of the docker container**, additional configuration and installation is required, which will not be described here.
+   
+   **Note: These steps below are similar to another required critical step (pre-commit configuration) later in this document, which also needs to be run**.
+
+   If pre-commit is not already installed on your system:
+   ```
+   pip install pre-commit
+   ```
+   All related tools (git hook scripts) are installed under the `pre-commit install` step, not at this level. See https://pre-commit.com/#install
+   
+   If you get an error message during the install of pre-commit which says:
+   
+   *Installing collected packages: pre-commit
+       WARNING: The script pre-commit is installed in '/home/{your_user_name}/.local/bin' which is not on PATH.
+       Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.`,
+   then you will need to do some additional configuration. You need to adjust your path.*
+   ```
+   (Adjusting the path to be exactly the path you see in the WARNING message above from your console output).
+   export PATH="/home/{your_user_name}/.local/bin:$PATH"
+   ```
+   To test that it installed correctly, is pathed correctly and check the version:
+   ```
+   pre-commit --version
+   ```
+   It should respond with the phrase *pre-commit 3.6.0* (version may not be exact).
+
+
+5. pre-commit configuration:
+   
+   Now, you need to configure your local clone of the repo to honor the pre-commit hooks.
+   The `pre-commit` package is used to pick up the pre-commit hooks which verify your staged changes adhere to the project's style and format requirements (configuration defined in [pyproject.toml](/pyproject.toml)).
+
+   Initialize the pre-commit hooks included within the root directory of this code repository (`inundation-mapping`):
     ```
     $ pre-commit install
     ```
-7. At this point, you should be set up with `pre-commit`. When a commit is made it will run the pre-commit hooks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). For reference, you may run any of the pre-commit hooks manually before issuing the `git commit` command (see below). Some tools used by the pre commit git hook (`isort`, `flake8`, & `black`) are also available as command line executables within the Docker container, however, it is recommend to run them through `pre-commit` outside of the container, as it picks up the correct configuration.
+    
+6. At this point, you should be set up with `pre-commit`. When a commit is made it will run the pre-commit hooks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). For reference, you may run any of the pre-commit hooks manually before issuing the `git commit` command (see below). Some tools used by the pre commit git hook (`isort`, `flake8`, & `black`) are also available as command line executables **within the Docker container***, however, it is recommended to run them through `pre-commit` **outside of the container**, as it picks up the correct configuration.
 
-    ```
-    # Check only the staged changes
-    pre-commit run
-
-    # Check all files in the repo
-    pre-commit run -a
+   ```
+   # Check only the staged changes
+   pre-commit run
 
-    # Run only the flake8 formatting tool
-    pre-commit run -a flake8
-    ```
+   # Check all files in the repo
+   pre-commit run -a
 
-8. Build the Docker container:
+   # Run only the flake8, isort, or black.
+   pre-commit run -a flake8
+   pre-commit run -a isort
+   pre-commit run -a black
+   ```
+7. Build the Docker container:
     ```
     Docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>
     ```
-9. [Within the container](README.md#startrun-the-docker-container), ensure sure unit tests pass ([instructions here](/unit_tests/README.md)).
+ 
+8.  [Within the container](README.md#startrun-the-docker-container), ensure sure unit tests pass ([instructions here](/unit_tests/README.md)).
     ```
     pytest unit_tests/
     ```
 
-10. Outside of the Docker container, commit your changes:
+9. Outside of the Docker container, commit your changes:
     ```
-    git commit -m <descriptive sentence or two changes>
+    git commit -m "<descriptive sentence or two of changes>"
     ```
-    This will invoke pre-commit hooks mentioned in step 7 that will lint & format the code. In many cases non-compliant code will be rectified automatically, but in some cases manual changes will be necessary. Make sure all of these checks pass, if not, make necessary changes and re-issue `git commit -m <...>`.
-
-9. Push to your forked branch:
+    This will invoke pre-commit hooks mentioned in step 6 that will lint & format the code (some others as well). In many cases non-compliant code will be rectified automatically, but in some cases manual changes will be necessary. Make sure all of these checks pass. If not, make necessary changes (`git add <...>`), and re-issue `git commit -m "<...>"`.
+   
+10. Push to your forked branch:
     ```
     git push -u origin
     ```
@@ -93,4 +124,5 @@ If you would like to contribute, please follow these steps:
     git push --set-upstream origin <your branch>
     ```
 
-10. Submit a pull request on [inundation-mapping's GitHub page](https://github.com/NOAA-OWP/inundation-mapping) (please review checklist in [PR template](/.github/PULL_REQUEST_TEMPLATE.md) for additional PR guidance).
+11. Submit a pull request on [inundation-mapping's GitHub page](https://github.com/NOAA-OWP/inundation-mapping) (please review checklist in [PR template](/.github/PULL_REQUEST_TEMPLATE.md) for additional PR guidance).
+   
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
old mode 100644
new mode 100755
index 3e08dc14c..b6ded4198
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -39,7 +39,7 @@ def acquire_and_preprocess_3dep_dems(
     extent_file_path,
     target_output_folder_path='',
     number_of_jobs=1,
-    retry=False,
+    repair=False,
     skip_polygons=False,
     target_projection='EPSG:5070',
 ):
@@ -79,9 +79,11 @@ def acquire_and_preprocess_3dep_dems(
         - number_of_jobs (int):
             This program supports multiple procs if multiple procs/cores are available.
 
-        - retry (True / False):
-            If retry is True and the file exists (either the raw downloaded DEM and/or)
-            the projected one, then skip it
+        - repair (True / False):
+            If repair is True then look for output DEMs that are missing or are too small (under 10mg).
+            This happens often as there can be instabilty when running long running processes.
+            USGS calls and networks can blip and some of the full BED can take many, many hours.
+            It will also look for DEMs that were missed entirely on previous runs.
 
         - skip_polygons (bool)
              If True, then we will not attempt to create polygon files for each dem file. If false,
@@ -120,37 +122,28 @@ def acquire_and_preprocess_3dep_dems(
                 f"For the output path of {target_output_folder_path}, the child directory"
                 " need not exist but the parent folder must."
             )
-        os.makedirs(target_output_folder_path)
+        os.makedir(target_output_folder_path)
+
     else:  # path exists
-        if not retry:
+        if repair is False:
             file_list = os.listdir(target_output_folder_path)
             if len(file_list) > 0:
                 print()
-                msg = f"The target output folder of {target_output_folder_path} appears to not be empty.\n\n"
-                "Do you want to empty the folder first?\n"
-                "  -- Type 'overwrite' if you want to empty the folder and continue.\n"
-                "  -- Type any other value to abort and stop the program.\n"
-                "  ?="
-
-                resp = input(msg).lower()
+                msg = (
+                    f"The target output folder of {target_output_folder_path} appears to not be empty.\n\n"
+                    "Do you want to empty the folder first?\n"
+                    "  -- Type 'overwrite' if you want to empty the folder and continue.\n"
+                    "  -- Type any other value to abort and stop the program.\n"
+                )
+                print(msg)
+                resp = input(" ?=").lower()
                 if resp == "overwrite":
                     shutil.rmtree(target_output_folder_path)
                     os.mkdir(target_output_folder_path)
                 else:
                     print("Program stopped\n")
                     sys.exit(0)
-        else:  # might want to retry but the folder isn't there yet
-            # It is ok if the child diretory does not exist, but the parent folder must
-            # parent directory, we want to reset it
-            parent_dir = os.path.abspath(os.path.join(target_output_folder_path, os.pardir))
-            print(parent_dir)
-            if not os.path.exists(parent_dir):
-                raise ValueError(
-                    f"For the output path of {target_output_folder_path}, the child directory"
-                    " need not exist but the parent folder must."
-                )
-            shutil.rmtree(target_output_folder_path)
-            os.mkdir(target_output_folder_path)
+        # no else:
 
     # I don't need the crs_number for now
     crs_is_valid, err_msg, crs_number = val.is_valid_crs(target_projection)
@@ -177,25 +170,25 @@ def acquire_and_preprocess_3dep_dems(
 
     # download dems, setting projection, block size, etc
     __download_usgs_dems(
-        extent_file_names, target_output_folder_path, number_of_jobs, retry, target_projection
+        extent_file_names, target_output_folder_path, number_of_jobs, repair, target_projection
     )
 
     if skip_polygons is False:
         polygonize(target_output_folder_path)
 
     end_time = datetime.utcnow()
-    fh.print_end_header('Loading 3dep dems', start_time, end_time)
+    fh.print_end_header('Loading 3dep dems complete', start_time, end_time)
 
     print()
     print(
         '---- NOTE: Remember to scan the log file for any failures. If you find errors in the'
-        ' log file, delete the output file and retry'
+        ' log file, delete the output file and repair.'
     )
     print()
     logging.info(fh.print_date_time_duration(start_time, end_time))
 
 
-def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry, target_projection):
+def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, repair, target_projection):
     '''
     Process:
     ----------
@@ -244,7 +237,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
                 'download_url': __USGS_3DEP_10M_VRT_URL,
                 'target_projection': target_projection,
                 'base_cmd': base_cmd,
-                'retry': retry,
+                'repair': repair,
             }
 
             try:
@@ -268,7 +261,9 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
     print("==========================================================")
 
 
-def download_usgs_dem_file(extent_file, output_folder_path, download_url, target_projection, base_cmd, retry):
+def download_usgs_dem_file(
+    extent_file, output_folder_path, download_url, target_projection, base_cmd, repair
+):
     '''
     Process:
     ----------
@@ -289,8 +284,9 @@ def download_usgs_dem_file(extent_file, output_folder_path, download_url, target
             ie) EPSG:5070 or EPSG:2276, etc
         - base_cmd (str)
              The basic GDAL command with string formatting wholes for key values.
-        - retry (bool)
-             If True, and the file exists, downloading will be skipped.
+        - repair (bool)
+             If True, and the file does not exist or is too small (under 10mb),
+             it will attempt to download.
 
     '''
 
@@ -302,11 +298,14 @@ def download_usgs_dem_file(extent_file, output_folder_path, download_url, target
     # which is related to to the spatial extents of the dem and the vrt combined.
     # so, super small .tifs are correct.
 
-    if (retry) and (os.path.exists(target_path_raw)):
-        msg = f" - Downloading -- {target_file_name_raw} - Skipped (already exists (see retry flag))"
-        print(msg)
-        logging.info(msg)
-        return
+    if (repair) and (os.path.exists(target_path_raw)):
+        if os.stat(target_path_raw).st_size < 1000000:
+            os.remove(target_path_raw)
+        else:
+            msg = f" - Downloading -- {target_file_name_raw} - Skipped (already exists (see retry flag))"
+            print(msg)
+            logging.info(msg)
+            return
 
     msg = f" - Downloading -- {target_file_name_raw} - Started"
     print(msg)
@@ -356,7 +355,7 @@ def polygonize(target_output_folder_path):
     """
     dem_domain_file = os.path.join(target_output_folder_path, 'DEM_Domain.gpkg')
 
-    msg = f" - Polygonizing -- {dem_domain_file} - Started"
+    msg = f" - Polygonizing -- {dem_domain_file} - Started (be patient, it can take a while)"
     print(msg)
     logging.info(msg)
 
@@ -368,6 +367,7 @@ def polygonize(target_output_folder_path):
     dem_gpkgs = gpd.GeoDataFrame()
 
     for n, dem_file in enumerate(dem_files):
+        print(f"Polygonizing: {dem_file}")
         edge_tif = f'{os.path.splitext(dem_file)[0]}_edge.tif'
         edge_gpkg = f'{os.path.splitext(edge_tif)[0]}.gpkg'
 
@@ -446,9 +446,8 @@ def __setup_logger(output_folder_path):
     '''
     sample usage (min params):
         python3 /foss_fim/data/usgs/acquire_and_preprocess_3dep_dems.py
-            -e /data/inputs/wbd/HUC6_ESPG_5070/
-            -t /data/inputs/3dep_dems/10m_5070/
-            -r
+            -e /data/inputs/wbd/wbd/HUC8_South_Alaska/
+            -t /data/inputs/3dep_dems/10m_South_Alaska/
             -j 20
 
     Notes:
@@ -499,10 +498,10 @@ def __setup_logger(output_folder_path):
     )
 
     parser.add_argument(
-        '-r',
-        '--retry',
-        help='OPTIONAL: If included, it will skip files that already exist.'
-        'Default is all will be loaded/reloaded.',
+        '-rp',
+        '--repair',
+        help='OPTIONAL: If included, it process only HUCs missing output DEMs or if the output DEM'
+        ' is too small (under 10 MB), which does happen.',
         required=False,
         action='store_true',
         default=False,
diff --git a/data/usgs/preprocess_ahps_usgs.py b/data/usgs/preprocess_ahps_usgs.py
old mode 100644
new mode 100755
diff --git a/data/usgs/preprocess_download_usgs_grids.py b/data/usgs/preprocess_download_usgs_grids.py
old mode 100644
new mode 100755
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1c49ad73a..9760431b2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,34 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.10.0 - 2024-02-02 - [PR#1054](https://github.com/NOAA-OWP/inundation-mapping/pull/1054)
+
+Recent testing exposed a bug with the `acquire_and_preprocess_3dep_dems.py` script. It lost the ability to be re-run and look for files that were unsuccessful earlier attempts and try them again. It may have been lost due to confusion of the word "retry". Now "retry" means restart the entire run. A new flag called "repair"  has been added meaning fix what failed earlier.  This is a key feature it is common for communication failures when calling USGS to download DEMs.  And with some runs taking many hours, this feature becomes important.
+
+Also used the opportunity to fix a couple of other minor issues:
+1) Reduce log output
+2) Add a test for ensuring the user does not submit job numbers (num of cpu requests) to exceed the system max cpus. This test exists in a number of places in the code but way later in the processing stack after alot of processing has been done. Now it is done at the start of the fim pipeline stack.
+3) remove arguments for "isaws" which is no longer in use and has not been for a while.
+4) quick upgrade to the tracker log that keeps track of duration of each unit being processed.
+
+### Changes
+
+
+- `data\usgs\`
+    - `acquire_and_preprocess_3dep_dems.py`: Re-add a feature which allowed for restarting and redo missing outputs or partial outputs. System now named as a "repair" system.
+- `fim_pipeline.sh`:  remove the parallel `--eta` flag to reduce logging. It was not needed, also removed "isaws" flag.
+- `fim_pre_processing.sh`: Added validation tests for maximum CPU requests (job numbers)
+- `fim_post_processing.sh`: Added a permissions updated as output folders were being locked due to permissions.
+- `fim_process_unit_wb.sh`: Fixed a bug with output folders being locked due to permissions, but it was not recursive.
+- `src`
+    - `bash_functions.sh`: Added function so the unit timing logs would also have a time in percentage so it can easily be used to calculate averages.
+    - `delineate_hydros_and_produce_HAND.sh`: Removed some unnecessary logging. Changed a few gdal calls to be less verbose.
+    - `derive_level_paths.py`: Changed verbose to false to reduce  unnecessary logging.
+    - `run_by_branch.sh`: Removed some unnecessary logging. Added a duration system so we know how long the branch took to process.
+    - `run_unit_by_wb.sh`: Removed some unnecessary logging. Changed a few gdal calls to be less verbose.
+    - `split_flows.py`: Removed progress bar which was unnecessary and was adding to logging.
+  
+<br/><br/>
 
 ## v4.4.9.2 - 2024-02-02 - [PR#1066](https://github.com/NOAA-OWP/inundation-mapping/pull/1066)
 
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 4e1e76159..8fc4ae9d7 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -46,8 +46,6 @@ usage()
       -o                : Overwrite outputs if they already exist.
       -skipcal          : If this param is included, the S.R.C. will be updated via the calibration points.
                             will be skipped.
-      -isaws            : If this param is included, AWS objects will be used where possible
-                        - Note: This feature is not yet implemented.
       -x                : If this param is included, the crosswalk will be evaluated.
 
 
@@ -104,13 +102,13 @@ if [ -f "$hucList" ]; then
     if [ "$jobHucLimit" = "1" ]; then
         parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList
     else
-        parallel --eta -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList
+        parallel -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName :::: $hucList
     fi
 else
     if [ "$jobHucLimit" = "1" ]; then
         parallel --verbose --lb -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file $runName ::: $hucList
     else
-        parallel --eta -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file  $runName  ::: $hucList
+        parallel -j $jobHucLimit --colsep ',' --joblog $logFile -- $process_wb_file  $runName  ::: $hucList
     fi
 fi
 
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 4ebb7b5fa..a157db6f6 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -216,6 +216,8 @@ python3 $toolsDir/combine_crosswalk_tables.py \
 Tcount
 date -u
 
+find $outputDestDir -type d -exec chmod -R 777 {} +
+
 echo
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 echo "---- End of fim_post_processing"
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index cff84fa23..f8453c8e3 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -45,8 +45,6 @@ usage()
       -o                : Overwrite outputs if they already exist.
       -skipcal          : If this param is included, the S.R.C. will be updated via the calibration points.
                             will be skipped.
-      -isaws            : If this param is included, AWS objects will be used where possible
-                        - Note: This feature is not yet implemented.
     "
     exit
 }
@@ -98,9 +96,6 @@ in
     -skipcal)
         skipcal=1
         ;;
-    -isaws)
-        isAWS=1
-        ;;
     -x)
         evaluateCrosswalk=1
         ;;
@@ -131,7 +126,6 @@ if [ "$jobHucLimit" = "" ]; then jobHucLimit=1; fi
 if [ "$jobBranchLimit" = "" ]; then jobBranchLimit=1; fi
 if [ -z "$overwrite" ]; then overwrite=0; fi
 if [ -z "$skipcal" ]; then skipcal=0; fi
-if [ -z "$isAWS" ]; then isAWS=0; fi
 if [ -z "$evaluateCrosswalk" ]; then evaluateCrosswalk=0; fi
 
 # validate and set defaults for the deny lists
@@ -186,6 +180,17 @@ if [ -d $outputDestDir ] && [ $overwrite -eq 0 ]; then
     usage
 fi
 
+# Test to ensure we are not overuseing cores
+num_available_cores=$(echo $(grep -c processor /proc/cpuinfo))
+let total_requested_jobs=$jobHucLimit*$jobBranchLimit
+if [[ $total_requested_jobs -gt $num_available_cores ]]; then
+    echo
+    echo "ERROR: There are $num_available_cores available, but -jh (jobHucLimit) * -jb (jobBranchLimit)"\
+         "exceed the number of available cores"
+    echo
+    usage
+fi
+
 ## SOURCE ENV FILE AND FUNCTIONS ##
 source $srcDir/bash_functions.env
 
@@ -236,7 +241,6 @@ echo "export deny_unit_list=$deny_unit_list" >> $args_file
 echo "export deny_branches_list=$deny_branches_list" >> $args_file
 echo "export deny_branch_zero_list=$deny_branch_zero_list" >> $args_file
 echo "export has_deny_branch_zero_override=$has_deny_branch_zero_override" >> $args_file
-echo "export isAWS=$isAWS" >> $args_file
 echo "export skipcal=$skipcal" >> $args_file
 echo "export evaluateCrosswalk=$evaluateCrosswalk" >> $args_file
 
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
index 0d164543d..1c013f777 100755
--- a/fim_process_unit_wb.sh
+++ b/fim_process_unit_wb.sh
@@ -129,7 +129,7 @@ fi
 
 # Move the contents of the temp directory into the outputs directory and update file permissions
 mv -f $tempHucDataDir $outputHucDataDir
-find $outputHucDataDir -type d -exec chmod 777 {} +
+find $outputHucDataDir -type d -exec chmod -R 777 {} +
 
 echo "============================================================================================="
 echo
diff --git a/src/bash_functions.env b/src/bash_functions.env
index 89a21aba7..72a33f450 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -43,6 +43,34 @@ Calc_Time() {
     fi
 }
 
+Calc_Time_Minutes_in_Percent(){
+
+    # git actions didn't like this. It says it returns a exit code of 1 ??? works here.
+    # so we will just copy paste code from Calc_Time
+    #local actual_duration=$(Calc_Time $1)
+
+
+    # split off the seconds and turn it into a a percent.
+    #readarray -d ":" -t time_array <<< "$actual_duration"
+
+    #local dur_min="${time_array[0]}"
+    #local dur_sec="${time_array[1]}"
+    #local num_dur_sec=$((dur_sec))
+
+    local start_time=$1
+    local end_time=`date +%s`
+    local total_sec=$(( $end_time - $start_time ))
+    local dur_min=$((total_sec / 60))
+    local dur_remainder_sec=$((total_sec % 60))
+    local dur_sec_percent=$((100*$dur_remainder_sec/60))
+
+    if (( $dur_sec_percent < 10 )); then
+        echo "$dur_min.$dur_sec_percent"
+    else
+        echo "$dur_min.$dur_sec_percent"
+    fi
+}
+
 export -f T_total_start
 export -f Tstart
 export -f Tcount
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index b69aa339a..276da3741 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -12,14 +12,10 @@ elif [ "$level" = "unit" ]; then
     z_arg=$tempHucDataDir/nwm_catchments_proj_subset.gpkg
 fi
 
-## INITIALIZE TOTAL TIME TIMER ##
-T_total_start
 
 ## MASK LEVEE-PROTECTED AREAS FROM DEM ##
 if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $tempHucDataDir/LeveeProtectedAreas_subset.gpkg ]; then
     echo -e $startDiv"Mask levee-protected areas from DEM (*Overwrite dem_meters.tif output) $hucNumber $current_branch_id"
-    date -u
-    Tstart
     python3 $srcDir/mask_dem.py \
         -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
         -nld $tempHucDataDir/LeveeProtectedAreas_subset.gpkg \
@@ -30,34 +26,27 @@ if [ "$mask_leveed_area_toggle" = "True" ] && [ -f $tempHucDataDir/LeveeProtecte
         -b0 $branch_zero_id \
         -csv $tempHucDataDir/levee_levelpaths.csv \
         -l $levee_id_attribute
-    Tcount
 fi
 
 ## D8 FLOW ACCUMULATIONS ##
 echo -e $startDiv"D8 Flow Accumulations $hucNumber $current_branch_id"
-date -u
-Tstart
 python3 $srcDir/accumulate_headwaters.py \
     -fd $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
     -fa $tempCurrentBranchDataDir/flowaccum_d8_burned_filled_$current_branch_id.tif \
     -wg $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif \
     -stream $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
     -thresh 1
-Tcount
+
 
 ## PREPROCESSING FOR LATERAL THALWEG ADJUSTMENT ###
 echo -e $startDiv"Preprocessing for lateral thalweg adjustment $hucNumber $current_branch_id"
-date -u
-Tstart
 python3 $srcDir/unique_pixel_and_allocation.py \
     -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
     -o $tempCurrentBranchDataDir/demDerived_streamPixels_ids_$current_branch_id.tif
-Tcount
+
 
 ## ADJUST THALWEG MINIMUM USING LATERAL ZONAL MINIMUM ##
 echo -e $startDiv"Performing lateral thalweg adjustment $hucNumber $current_branch_id"
-date -u
-Tstart
 python3 $srcDir/adjust_thalweg_lateral.py \
     -e $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
     -s $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
@@ -66,42 +55,31 @@ python3 $srcDir/adjust_thalweg_lateral.py \
     -t 50 \
     -o $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif \
     -th $thalweg_lateral_elev_threshold
-Tcount
+
 
 ## MASK BURNED DEM FOR STREAMS ONLY ###
 echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id"
-date -u
-Tstart
 gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
     -A $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
     -B $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
     --calc="A/B" \
     --outfile="$tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" \
     --NoDataValue=0
-Tcount
 
 ## FLOW CONDITION STREAMS ##
 echo -e $startDiv"Flow Condition Thalweg $hucNumber $current_branch_id"
-date -u
-Tstart
 $taudemDir/flowdircond -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif \
     -z $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif \
     -zfdc $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif
-Tcount
 
 ## D8 SLOPES ##
 echo -e $startDiv"D8 Slopes from DEM $hucNumber $current_branch_id"
-date -u
-Tstart
 mpiexec -n $ncores_fd $taudemDir2/d8flowdir \
     -fel $tempCurrentBranchDataDir/dem_lateral_thalweg_adj_$current_branch_id.tif \
     -sd8 $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif
-Tcount
 
 ## STREAMNET FOR REACHES ##
 echo -e $startDiv"Stream Net for Reaches $hucNumber $current_branch_id"
-date -u
-Tstart
 $taudemDir/streamnet \
     -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
     -fel $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
@@ -112,12 +90,9 @@ $taudemDir/streamnet \
     -coord $tempCurrentBranchDataDir/coordFile_$current_branch_id.txt \
     -w $tempCurrentBranchDataDir/sn_catchments_reaches_$current_branch_id.tif \
     -net $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp
-Tcount
 
 ## SPLIT DERIVED REACHES ##
 echo -e $startDiv"Split Derived Reaches $hucNumber $current_branch_id"
-date -u
-Tstart
 $srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_branch_id.shp \
     -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
     -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg \
@@ -127,83 +102,60 @@ $srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_
     -m $max_split_distance_meters \
     -t $slope_min \
     -b $lakes_buffer_dist_meters
-Tcount
 
 ## GAGE WATERSHED FOR REACHES ##
 echo -e $startDiv"Gage Watershed for Reaches $hucNumber $current_branch_id"
-date -u
-Tstart
 mpiexec -n $ncores_gw $taudemDir/gagewatershed \
     -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
     -gw $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
     -o $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg \
     -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
-Tcount
 
 ## VECTORIZE FEATURE ID CENTROIDS ##
 echo -e $startDiv"Vectorize Pixel Centroids $hucNumber $current_branch_id"
-date -u
-Tstart
 $srcDir/reachID_grid_to_vector_points.py \
     -r $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
     -i featureID \
     -p $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg
-Tcount
 
 ## GAGE WATERSHED FOR PIXELS ##
 echo -e $startDiv"Gage Watershed for Pixels $hucNumber $current_branch_id"
-date -u
-Tstart
 mpiexec -n $ncores_gw $taudemDir/gagewatershed \
     -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_"$current_branch_id".tif \
     -gw $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif \
     -o $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg \
     -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
-Tcount
 
 # D8 REM ##
 echo -e $startDiv"D8 REM $hucNumber $current_branch_id"
-date -u
-Tstart
 $srcDir/make_rem.py -d $tempCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif \
     -w $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif \
     -o $tempCurrentBranchDataDir/rem_$current_branch_id.tif \
     -t $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif
-Tcount
 
 ## BRING DISTANCE DOWN TO ZERO & MASK TO CATCHMENTS##
 echo -e $startDiv"Bring negative values in REM to zero and mask to catchments $hucNumber $current_branch_id"
-date -u
 gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
     -A $tempCurrentBranchDataDir/rem_$current_branch_id.tif \
     -B $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
     --calc="(A*(A>=0)*(B>0))" --NoDataValue=$ndv \
     --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
-Tcount
 
 ## RASTERIZE LANDSEA (OCEAN AREA) POLYGON (IF APPLICABLE) ##
 if [ -f $tempHucDataDir/LandSea_subset.gpkg ]; then
     echo -e $startDiv"Rasterize filtered/dissolved ocean/Glake polygon $hucNumber $current_branch_id"
-    date -u
-    Tstart
-    gdal_rasterize -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" \
+    gdal_rasterize -q -ot Int32 -burn $ndv -a_nodata $ndv -init 1 -co "COMPRESS=LZW" -co "BIGTIFF=YES" \
         -co "TILED=YES" -te $xmin $ymin $xmax $ymax -ts $ncols $nrows $tempHucDataDir/LandSea_subset.gpkg \
         $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif
-    Tcount
 fi
 
 ## POLYGONIZE REACH WATERSHEDS ##
 echo -e $startDiv"Polygonize Reach Watersheds $hucNumber $current_branch_id"
-date -u
-Tstart
-gdal_polygonize.py -8 -f GPKG $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
+gdal_polygonize.py -q -8 -f GPKG $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
     $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg catchments HydroID
-Tcount
 
 ## PROCESS CATCHMENTS AND MODEL STREAMS STEP 1 ##
 echo -e $startDiv"Process catchments and model streams $hucNumber $current_branch_id"
-date -u
-Tstart
 python3 $srcDir/filter_catchments_and_add_attributes.py \
     -i $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.gpkg \
     -f $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg \
@@ -211,32 +163,24 @@ python3 $srcDir/filter_catchments_and_add_attributes.py \
     -o $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
     -w $tempHucDataDir/wbd8_clp.gpkg \
     -u $hucNumber
-Tcount
 
 ## RASTERIZE NEW CATCHMENTS AGAIN ##
 echo -e $startDiv"Rasterize filtered catchments $hucNumber $current_branch_id"
-date -u
-Tstart
-gdal_rasterize -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+gdal_rasterize -q -ot Int32 -a HydroID -a_nodata 0 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
     -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
     $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
     $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif
-Tcount
 
 ## MASK SLOPE TO CATCHMENTS ##
 echo -e $startDiv"Mask to slopes to catchments $hucNumber $current_branch_id"
-date -u
 gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
     -A $tempCurrentBranchDataDir/slopes_d8_dem_meters_$current_branch_id.tif \
     -B $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif \
     --calc="A*(B>0)" --NoDataValue=$ndv \
     --outfile=$tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif
-Tcount
 
 ## MAKE CATCHMENT AND STAGE FILES ##
 echo -e $startDiv"Generate Catchment List and Stage List Files $hucNumber $current_branch_id"
-date -u
-Tstart
 $srcDir/make_stages_and_catchlist.py \
     -f $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
     -c $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
@@ -245,38 +189,28 @@ $srcDir/make_stages_and_catchlist.py \
     -m $stage_min_meters \
     -i $stage_interval_meters \
     -t $stage_max_meters
-Tcount
 
 ## MASK REM RASTER TO REMOVE OCEAN AREAS ##
 if  [ -f $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
     echo -e $startDiv"Additional masking to REM raster to remove ocean/Glake areas $hucNumber $current_branch_id"
-    date -u
-    Tstart
     gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
         -A $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
         -B $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif \
         --calc="(A*B)" --NoDataValue=$ndv \
         --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
-    Tcount
 fi
 
 ## HYDRAULIC PROPERTIES ##
 echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"
-date -u
-Tstart
 $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
     -catch $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.tif \
     -catchlist $tempCurrentBranchDataDir/catch_list_$current_branch_id.txt \
     -slp $tempCurrentBranchDataDir/slopes_d8_dem_meters_masked_$current_branch_id.tif \
     -h $tempCurrentBranchDataDir/stage_$current_branch_id.txt \
     -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv
-Tcount
 
 ## FINALIZE CATCHMENTS AND MODEL STREAMS ##
 echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"
-date -u
-Tstart
-
 python3 $srcDir/add_crosswalk.py \
     -d $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_$current_branch_id.gpkg \
     -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
@@ -295,13 +229,11 @@ python3 $srcDir/add_crosswalk.py \
     -k $tempCurrentBranchDataDir/small_segments_$current_branch_id.csv \
     -e $min_catchment_area \
     -g $min_stream_length
-Tcount
+
 
 ## EVALUATE CROSSWALK ##
 if [ "$current_branch_id" = "$branch_zero_id" ] && [ "$evaluateCrosswalk" = "1" ] ; then
     echo -e $startDiv"Evaluate crosswalk $hucNumber $current_branch_id"
-    date -u
-    Tstart
     python3 $toolsDir/evaluate_crosswalk.py \
         -a $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg \
         -b $b_arg \
@@ -309,5 +241,4 @@ if [ "$current_branch_id" = "$branch_zero_id" ] && [ "$evaluateCrosswalk" = "1"
         -d $tempHucDataDir/nwm_headwater_points_subset.gpkg \
         -u $hucNumber \
         -z $current_branch_id
-    Tcount
 fi
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index 01a405437..8fc1102dc 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -98,7 +98,7 @@ def Derive_level_paths(
         reach_id_attribute=reach_id_attribute,
         toNode_attribute=toNode_attribute,
         fromNode_attribute=fromNode_attribute,
-        verbose=True,
+        verbose=False,
     )
 
     # derive arbolate sum
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index c158e3522..53db98bee 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -30,12 +30,14 @@ mkdir -p $tempCurrentBranchDataDir
 
 ## START MESSAGE ##
 echo -e $startDiv"Processing HUC: $hucNumber - branch_id: $current_branch_id"
-echo
+
+## INITIALIZE TOTAL BRANCH TIMER ##
+T_total_start
+branch_start_time=`date +%s`
+date -u
 
 ## SUBSET VECTORS
 echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"
-date -u
-Tstart
 echo -e "Querying NWM streams ..."
 ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
     $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg \
@@ -52,48 +54,36 @@ ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="
 # ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
 #     $tempCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg \
 #      $tempHucDataDir/nwm_headwaters.gpkg
-Tcount
 
 ## GET RASTERS FROM ROOT HUC DIRECTORY AND CLIP TO CURRENT BRANCH BUFFER ##
 echo -e $startDiv"Clipping rasters to branches $hucNumber $current_branch_id"
-date -u
-Tstart
 $srcDir/clip_rasters_to_branches.py -d $current_branch_id \
     -b $tempHucDataDir/branch_polygons.gpkg \
     -i $branch_id_attribute \
     -r $tempHucDataDir/dem_meters.tif $tempHucDataDir/flowdir_d8_burned_filled.tif \
-    -c $tempCurrentBranchDataDir/dem_meters.tif $tempCurrentBranchDataDir/flowdir_d8_burned_filled.tif \
-    -v
-Tcount
+    -c $tempCurrentBranchDataDir/dem_meters.tif $tempCurrentBranchDataDir/flowdir_d8_burned_filled.tif
+
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $current_branch_id"
-date -u
-Tstart
 read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy\
 <<<$($srcDir/getRasterInfoNative.py $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif)
-Tcount
+
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"
-date -u
-Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+gdal_rasterize -q -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
     -te $xmin $ymin $xmax $ymax \
     -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg \
     $tempCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
-Tcount
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
 echo -e $startDiv"Rasterize NHD Headwaters $hucNumber $current_branch_id"
-date -u
-Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+gdal_rasterize -q -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
     -te $xmin $ymin $xmax $ymax \
     -ts $ncols $nrows \
     $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg \
     $tempCurrentBranchDataDir/headwaters_$current_branch_id.tif
-Tcount
 
 ## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
 export hucNumber=$hucNumber
@@ -112,8 +102,6 @@ $srcDir/delineate_hydros_and_produce_HAND.sh "branch"
 ## USGS CROSSWALK ##
 if [ -f $tempHucDataDir/usgs_subset_gages.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $current_branch_id"
-    date -u
-    Tstart
     python3 $srcDir/usgs_gage_crosswalk.py \
         -gages $tempHucDataDir/usgs_subset_gages.gpkg \
         -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$current_branch_id.gpkg \
@@ -122,17 +110,15 @@ if [ -f $tempHucDataDir/usgs_subset_gages.gpkg ]; then
         -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
         -out $tempCurrentBranchDataDir \
         -b $current_branch_id
-    Tcount
 fi
 
 ## REMOVE FILES FROM DENY LIST ##
 if [ -f $deny_branches_list ]; then
     echo -e $startDiv"Remove files $hucNumber $current_branch_id"
-    date -u
-    Tstart
     $srcDir/outputs_cleanup.py -d $tempCurrentBranchDataDir -l $deny_branches_list -b $current_branch_id
-    Tcount
 fi
 
 echo -e $startDiv"End Branch Processing $hucNumber $current_branch_id ..."
+date -u
+Calc_Duration $branch_start_time
 echo
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 6c17078fa..ca6c9979a 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -17,10 +17,10 @@ branch_list_lst_file=$tempHucDataDir/branch_ids.lst
 
 branchSummaryLogFile=$outputDestDir/logs/branch/"$hucNumber"_summary_branch.log
 
-## INITIALIZE TOTAL TIME TIMER ##
+## INITIALIZE TOTAL UNIT AND IT'S BRANCHES TIMER ##
 T_total_start
 huc_start_time=`date +%s`
-
+date -u
 
 ## Copy HUC's pre-clipped .gpkg files from $pre_clip_huc_dir (use -a & /. -- only copies folder's contents)
 echo -e $startDiv"Copying staged wbd and .gpkg files from $pre_clip_huc_dir/$hucNumber"
@@ -36,8 +36,6 @@ cp $inputsDir/ahps_sites/nws_lid.gpkg $tempHucDataDir
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"
-date -u
-Tstart
 $srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg \
     -s $tempHucDataDir/wbd_buffered_streams.gpkg \
     -b $branch_id_attribute \
@@ -62,13 +60,9 @@ $srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg \
 # check if level paths exists
 levelpaths_exist=1
 if [ ! -f $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg ]; then levelpaths_exist=0; fi
-Tcount
 
 ## ASSOCIATE LEVEL PATHS WITH LEVEES
 echo -e $startDiv"Associate level paths with levees"
-date -u
-Tstart
-
 [ -f $tempHucDataDir/nld_subset_levees.gpkg ] && \
 python3 $srcDir/associate_levelpaths_with_levees.py -nld $tempHucDataDir/nld_subset_levees.gpkg \
     -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
@@ -78,27 +72,19 @@ python3 $srcDir/associate_levelpaths_with_levees.py -nld $tempHucDataDir/nld_sub
     -b $branch_id_attribute \
     -l $levee_id_attribute
 
-Tcount
-
 ## STREAM BRANCH POLYGONS
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
-date -u
-Tstart
 $srcDir/buffer_stream_branches.py -a $tempHucDataDir/HUC6_dem_domain.gpkg \
     -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
     -i $branch_id_attribute \
     -d $branch_buffer_distance_meters \
     -b $tempHucDataDir/branch_polygons.gpkg
-Tcount
 
 ## CREATE BRANCHID LIST FILE
 echo -e $startDiv"Create list file of branch ids for $hucNumber"
-date -u
-Tstart
 $srcDir/generate_branch_list.py -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
     -b $branch_id_attribute \
     -o $branch_list_lst_file
-Tcount
 
 ## CREATE BRANCH ZERO ##
 echo -e $startDiv"Creating branch zero for $hucNumber"
@@ -110,84 +96,60 @@ mkdir -p $tempCurrentBranchDataDir
 ## CLIP RASTERS
 echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"
 # Note: don't need to use gdalwarp -cblend as we are using a buffered wbd
-date -u
-Tstart
-
 [ ! -f $tempCurrentBranchDataDir/dem_meters.tif ] && \
 gdalwarp -cutline $tempHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" \
     -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" \
-    -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $tempHucDataDir/dem_meters.tif
-
-Tcount
+    -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $tempHucDataDir/dem_meters.tif -q
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"
-date -u
-Tstart
 read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy\
     <<<$($srcDir/getRasterInfoNative.py $tempHucDataDir/dem_meters.tif)
 
 ## RASTERIZE NLD MULTILINES ##
 echo -e $startDiv"Rasterize all NLD multilines using zelev vertices $hucNumber $branch_zero_id"
-date -u
-Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
 [ -f $tempHucDataDir/3d_nld_subset_levees_burned.gpkg ] && \
-gdal_rasterize -l 3d_nld_subset_levees_burned -3d -at -a_nodata $ndv \
+gdal_rasterize -q -l 3d_nld_subset_levees_burned -3d -at -a_nodata $ndv \
     -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
     -ot Float32 -of GTiff -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "COMPRESS=LZW" -co "BIGTIFF=YES" \
     -co "TILED=YES" $tempHucDataDir/3d_nld_subset_levees_burned.gpkg \
     $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif
-Tcount
 
 ## BURN LEVEES INTO DEM ##
 echo -e $startDiv"Burn nld levees into dem & convert nld elev to meters"
 echo -e "(*Overwrite dem_meters.tif output) $hucNumber $branch_zero_id"
-date -u
-Tstart
 # REMAINS UNTESTED FOR AREAS WITH LEVEES
 [ -f $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif ] && \
 python3 $srcDir/burn_in_levees.py \
     -dem $tempHucDataDir/dem_meters.tif \
     -nld $tempCurrentBranchDataDir/nld_rasterized_elev_$branch_zero_id.tif \
     -out $tempHucDataDir/dem_meters.tif
-Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) - BRANCH 0 (include all NWM streams) ##
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $branch_zero_id"
-date -u
-Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+gdal_rasterize -q -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
     -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
     $tempHucDataDir/nwm_subset_streams.gpkg $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif
-Tcount
 
 ## RASTERIZE REACH BOOLEAN (1 & 0) - BRANCHES (Not 0) (NWM levelpath streams) ##
 if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Rasterize Reach Boolean $hucNumber (Branches)"
-    date -u
-    Tstart
-    gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+    gdal_rasterize -q -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
         -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
         $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg $tempHucDataDir/flows_grid_boolean.tif
-    Tcount
 fi
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
 echo -e $startDiv"Rasterize NWM Headwaters $hucNumber $branch_zero_id"
-date -u
-Tstart
-gdal_rasterize -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
+gdal_rasterize -q -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
     -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
     $tempHucDataDir/nwm_headwater_points_subset.gpkg $tempCurrentBranchDataDir/headwaters_$branch_zero_id.tif
-Tcount
 
 ## DEM Reconditioning - BRANCH 0 (include all NWM streams) ##
 # Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
 # This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
 echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber $branch_zero_id"
-date -u
-Tstart
 python3 $srcDir/agreedem.py \
     -r $tempCurrentBranchDataDir/flows_grid_boolean_$branch_zero_id.tif \
     -d $tempHucDataDir/dem_meters.tif \
@@ -196,15 +158,12 @@ python3 $srcDir/agreedem.py \
     -b $agree_DEM_buffer \
     -sm 10 \
     -sh 1000
-Tcount
 
 ## DEM Reconditioning - BRANCHES (NOT 0) (NWM levelpath streams) ##
 # Using AGREE methodology, hydroenforce the DEM so that it is consistent with the supplied stream network.
 # This allows for more realistic catchment delineation which is ultimately reflected in the output FIM mapping.
 if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Creating AGREE DEM using $agree_DEM_buffer meter buffer $hucNumber (Branches)"
-    date -u
-    Tstart
     python3 $srcDir/agreedem.py -r $tempHucDataDir/flows_grid_boolean.tif \
         -d $tempHucDataDir/dem_meters.tif \
         -w $tempHucDataDir \
@@ -212,52 +171,37 @@ if [ "$levelpaths_exist" = "1" ]; then
         -b $agree_DEM_buffer \
         -sm 10 \
         -sh 1000
-    Tcount
 fi
 
 ## PIT REMOVE BURNED DEM - BRANCH 0 (include all NWM streams) ##
 echo -e $startDiv"Pit remove Burned DEM $hucNumber $branch_zero_id"
-date -u
-Tstart
 rd_depression_filling $tempCurrentBranchDataDir/dem_burned_$branch_zero_id.tif \
     $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif
-Tcount
 
 ## PIT REMOVE BURNED DEM - BRANCHES (NOT 0) (NWM levelpath streams) ##
 if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Pit remove Burned DEM $hucNumber (Branches)"
-    date -u
-    Tstart
     rd_depression_filling $tempHucDataDir/dem_burned.tif $tempHucDataDir/dem_burned_filled.tif
-    Tcount
 fi
 
 ## D8 FLOW DIR - BRANCH 0 (include all NWM streams) ##
 echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber $branch_zero_id"
-date -u
-Tstart
 mpiexec -n $ncores_fd $taudemDir2/d8flowdir \
     -fel $tempCurrentBranchDataDir/dem_burned_filled_$branch_zero_id.tif \
     -p $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$branch_zero_id.tif
-Tcount
 
 ## D8 FLOW DIR - BRANCHES (NOT 0) (NWM levelpath streams) ##
 if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"D8 Flow Directions on Burned DEM $hucNumber (Branches)"
-    date -u
-    Tstart
     mpiexec -n $ncores_fd $taudemDir2/d8flowdir \
         -fel $tempHucDataDir/dem_burned_filled.tif \
         -p $tempHucDataDir/flowdir_d8_burned_filled.tif
-    Tcount
 fi
 
 ## MAKE A COPY OF THE DEM FOR BRANCH 0
 echo -e $startDiv"Copying DEM to Branch 0"
-date -u
-Tstart
 cp $tempHucDataDir/dem_meters.tif $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif
-Tcount
+
 
 ## PRODUCE THE REM AND OTHER HAND FILE OUTPUTS ##
 export hucNumber=$hucNumber
@@ -278,8 +222,6 @@ $srcDir/delineate_hydros_and_produce_HAND.sh "unit"
 ## CREATE USGS GAGES FILE
 if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     echo -e $startDiv"Assigning USGS gages to branches for $hucNumber"
-    date -u
-    Tstart
     python3 $srcDir/usgs_gage_unit_setup.py \
         -gages $tempHucDataDir/usgs_gages.gpkg \
         -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
@@ -288,14 +230,11 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
         -huc $hucNumber \
         -ahps $tempHucDataDir/nws_lid.gpkg \
         -bzero_id $branch_zero_id
-    Tcount
 fi
 
 ## USGS CROSSWALK ##
 if [ -f $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"
-    date -u
-    Tstart
     python3 $srcDir/usgs_gage_crosswalk.py \
         -gages $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg \
         -flows $tempCurrentBranchDataDir/demDerived_reaches_split_filtered_$branch_zero_id.gpkg \
@@ -303,7 +242,6 @@ if [ -f $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
         -dem $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif \
         -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif \
         -out $tempCurrentBranchDataDir -b $branch_zero_id
-    Tcount
 fi
 
 ## CLEANUP BRANCH ZERO OUTPUTS ##
@@ -322,10 +260,13 @@ echo "---- Start of branch processing for $hucNumber"
 branch_processing_start_time=`date +%s`
 
 if [ -f $branch_list_lst_file ]; then
+    date -u
+    Tstart
     # There may not be a branch_ids.lst if there were no level paths (no stream orders 3+)
     # but there will still be a branch zero
-    parallel --eta --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSummaryLogFile --colsep ',' \
+    parallel --timeout $branch_timeout -j $jobBranchLimit --joblog $branchSummaryLogFile --colsep ',' \
     -- $srcDir/process_branch.sh $runName $hucNumber :::: $branch_list_lst_file
+    Tcount
 else
     echo "No level paths exist with this HUC. Processing branch zero only."
 fi
@@ -344,7 +285,8 @@ Calc_Duration $branch_processing_start_time
 echo
 
 # WRITE TO LOG FILE CONTAINING ALL HUC PROCESSING TIMES
-echo "$hucNumber, $(Calc_Time $huc_start_time)" >> "$outputDestDir/logs/unit/total_duration_run_by_unit_all_HUCs.csv"
+total_duration_display="$hucNumber,$(Calc_Time $huc_start_time),$(Calc_Time_Minutes_in_Percent $huc_start_time)"
+echo "$total_duration_display" >> "$outputDestDir/logs/unit/total_duration_run_by_unit_all_HUCs.csv"
 
 date -u
 echo "---- HUC processing for $hucNumber is complete"
diff --git a/src/split_flows.py b/src/split_flows.py
index b44d59deb..5c47d4ae2 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -22,7 +22,6 @@
 from shapely import ops, wkt
 from shapely.geometry import LineString, Point
 from shapely.ops import split as shapely_ops_split
-from tqdm import tqdm
 
 import build_stream_traversal
 from utils.fim_enums import FIM_exit_codes
@@ -202,7 +201,7 @@ def snap_and_trim_flow(snapped_point, flows):
         print("No relevant streams within HUC boundaries.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
-    for i, lineString in tqdm(enumerate(flows.geometry), total=len(flows.geometry)):
+    for i, lineString in enumerate(flows.geometry):
         # Reverse geometry order (necessary for BurnLines)
         lineString = LineString(lineString.coords[::-1])
 

From b7f81386d54172b8fda224d47cde7a62c8dd00b2 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 16 Feb 2024 17:21:26 +0000
Subject: [PATCH 299/359] v4.4.10.1 Fix return error status from pre-processing
 and remove CPU test (#1075)

---
 docs/CHANGELOG.md      | 17 +++++++++++++++++
 fim_post_processing.sh |  2 ++
 fim_pre_processing.sh  | 20 ++++++++------------
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9760431b2..d8c074dbb 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.10.1 - 2024-02-16 - [PR#1075](https://github.com/NOAA-OWP/inundation-mapping/pull/1075)
+
+We recently added code to fim_pre_processing.sh that checks the CPU count. Earlier this test was being done in post-processing and was killing a pipeline that had already been running for a while.
+
+Fix:
+- Removed the CPU test from pre-processing. This puts us back to it possibly failing in post-processing but we have to leave it for now. 
+- Exit status codes (non 0) are now returned in pre-processing and post-processing when an error has occurred.
+
+Tested that the a non zero return exit from pre-processing shuts down the AWS step functions.
+
+### Changes
+- `fim_pre_processing.sh`: added non zero exit codes when in error, plus removed CPU test
+- `fim_post_processing.sh`:  added non zero exit codes when in error
+
+<br/><br/>
+
+
 ## v4.4.10.0 - 2024-02-02 - [PR#1054](https://github.com/NOAA-OWP/inundation-mapping/pull/1054)
 
 Recent testing exposed a bug with the `acquire_and_preprocess_3dep_dems.py` script. It lost the ability to be re-run and look for files that were unsuccessful earlier attempts and try them again. It may have been lost due to confusion of the word "retry". Now "retry" means restart the entire run. A new flag called "repair"  has been added meaning fix what failed earlier.  This is a key feature it is common for communication failures when calling USGS to download DEMs.  And with some runs taking many hours, this feature becomes important.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index a157db6f6..9a176e695 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -46,6 +46,7 @@ if [ "$runName" = "" ]
 then
     echo "ERROR: Missing -n run time name argument"
     usage
+    exit 22
 fi
 
 outputDestDir=$outputsDir/$runName
@@ -217,6 +218,7 @@ Tcount
 date -u
 
 find $outputDestDir -type d -exec chmod -R 777 {} +
+find $outputDestDir -type f -exec chmod -R 777 {} +
 
 echo
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index f8453c8e3..0846b522e 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -46,7 +46,6 @@ usage()
       -skipcal          : If this param is included, the S.R.C. will be updated via the calibration points.
                             will be skipped.
     "
-    exit
 }
 
 set -e
@@ -104,16 +103,20 @@ in
     shift
 done
 
+# exit 22 means bad argument
+
 # print usage if arguments empty
 if [ "$hucList" = "" ]
 then
     echo "ERROR: Missing -u Huclist argument"
     usage
+    exit 22
 fi
 if [ "$runName" = "" ]
 then
     echo "ERROR: Missing -n run time name argument"
     usage
+    exit 22
 fi
 
 # outputsDir & workDir come from the Dockerfile
@@ -137,6 +140,7 @@ then
     # NONE is not case sensitive
     echo "Error: The -ud <unit deny file> does not exist and is not the word NONE"
     usage
+    exit 22
 fi
 
 # validate and set defaults for the deny lists
@@ -148,6 +152,7 @@ then
     # NONE is not case sensitive
     echo "Error: The -bd <branch deny file> does not exist and is not the word NONE"
     usage
+    exit 22
 fi
 
 # We do a 1st cleanup of branch zero using branchZeroDenylist (which might be none).
@@ -164,6 +169,7 @@ then
     then
         echo "Error: The -zd <branch zero deny file> does not exist and is not the word NONE"
         usage
+        exit 22
     else
         # only if the deny branch zero has been overwritten and file exists
         has_deny_branch_zero_override=1
@@ -178,17 +184,7 @@ if [ -d $outputDestDir ] && [ $overwrite -eq 0 ]; then
     echo "ERROR: Output dir $outputDestDir exists. Use overwrite -o to run."
     echo
     usage
-fi
-
-# Test to ensure we are not overuseing cores
-num_available_cores=$(echo $(grep -c processor /proc/cpuinfo))
-let total_requested_jobs=$jobHucLimit*$jobBranchLimit
-if [[ $total_requested_jobs -gt $num_available_cores ]]; then
-    echo
-    echo "ERROR: There are $num_available_cores available, but -jh (jobHucLimit) * -jb (jobBranchLimit)"\
-         "exceed the number of available cores"
-    echo
-    usage
+    exit 22
 fi
 
 ## SOURCE ENV FILE AND FUNCTIONS ##

From 073e4a30d4ea92e53453220c9cbff68942650bbe Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 16 Feb 2024 10:24:24 -0700
Subject: [PATCH 300/359] v4.4.11.0 Replace fiona with pyogrio (#1077)

---
 Pipfile                                       |   5 +-
 Pipfile.lock                                  | 455 ++++++++++--------
 data/bathymetry/preprocess_bathymetry.py      |   4 +-
 .../create_flow_forecast_file.py              |   3 +
 data/esri.py                                  |   3 +
 data/nld/levee_download.py                    |   3 +
 data/usgs/acquire_and_preprocess_3dep_dems.py |   3 +
 data/wbd/clip_vectors_to_wbd.py               |   4 +-
 data/wbd/preprocess_wbd.py                    |   3 +
 data/write_parquet_from_calib_pts.py          |   3 +
 docs/CHANGELOG.md                             |  20 +
 src/add_crosswalk.py                          |  10 +-
 src/associate_levelpaths_with_levees.py       |   3 +
 src/bash_variables.env                        |   3 +
 src/bathy_rc_adjust.py                        |   3 +
 src/bathymetric_adjustment.py                 |   8 +-
 src/buffer_stream_branches.py                 |   3 +
 src/build_stream_traversal.py                 |   3 +
 src/crosswalk_nwm_demDerived.py               |   3 +
 src/derive_headwaters.py                      |   3 +
 src/derive_level_paths.py                     |   3 +
 src/edit_points.py                            |   3 +
 src/filter_catchments_and_add_attributes.py   |   3 +
 src/finalize_srcs.py                          |   5 +-
 src/make_stages_and_catchlist.py              |   4 +-
 src/mask_dem.py                               |   3 +
 src/reachID_grid_to_vector_points.py          |   3 +
 src/split_flows.py                            |   3 +
 src/src_adjust_spatial_obs.py                 |   3 +
 src/src_roughness_optimization.py             |   3 +
 src/stream_branches.py                        |   4 +-
 src/subset_catch_list_by_branch_id.py         |   3 +
 src/usgs_gage_crosswalk.py                    |   2 +
 src/usgs_gage_unit_setup.py                   |   2 +
 src/utils/shared_functions.py                 |   3 +
 tools/adjust_rc_with_feedback.py              |   3 +
 tools/check_deep_flooding.py                  |   3 +
 tools/create_flow_forecast_file.py            |   3 +
 tools/eval_plots.py                           |   3 +
 tools/evaluate_continuity.py                  |   3 +
 tools/evaluate_crosswalk.py                   |   3 +
 tools/fimr_to_benchmark.py                    |   3 +
 tools/find_max_catchment_breadth.py           |   3 +
 tools/generate_categorical_fim.py             |   3 +
 tools/generate_categorical_fim_flows.py       |   3 +
 tools/generate_categorical_fim_mapping.py     |   3 +
 tools/generate_nws_lid.py                     |   3 +
 tools/hash_compare.py                         |   3 +
 tools/inundate_events.py                      |   3 +
 tools/inundation.py                           |   3 +
 tools/make_boxes_from_bounds.py               |   4 +-
 tools/mosaic_inundation.py                    |   2 +
 tools/overlapping_inundation.py               |   3 +
 tools/rating_curve_comparison.py              |   3 +
 tools/rating_curve_get_usgs_curves.py         |   3 +
 tools/test_case_by_hydro_id.py                |   3 +
 tools/tools_shared_functions.py               |   3 +
 57 files changed, 428 insertions(+), 230 deletions(-)

diff --git a/Pipfile b/Pipfile
index 97bac2944..5aee10468 100644
--- a/Pipfile
+++ b/Pipfile
@@ -9,7 +9,7 @@ ipython = "==8.12.0"
 [packages]
 certifi = "==2023.7.22"
 fiona = "==1.8.22"
-geopandas = "==0.12.2"
+geopandas = "==0.14.3"
 numba = "==0.56.4"
 numpy = "==1.23.5"
 pandas = "==2.0.2"
@@ -43,8 +43,9 @@ flake8-pyproject = "==1.2.3"
 pre-commit = "==3.3.3"
 isort = "==5.12.0"
 urllib3 = "==1.26.18"
-pyflwdir = "*"
+pyflwdir = "==0.5.8"
 pillow = "==10.2.0"
+pyogrio = "==0.7.2"
 
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index 61248b1d8..0c47c4a04 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "5f29b79268530835205eb8a550b43c847acfcb16ede4de1a6dc1d22c176f5ac1"
+            "sha256": "a3ed6a563de9000738a1df55a5cbce4a7584fcee0725bf71f917916edf6eef3b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -21,7 +21,7 @@
                 "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92",
                 "sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.4.0"
         },
         "aiofiles": {
@@ -29,7 +29,7 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version < '4' and python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
@@ -37,7 +37,7 @@
                 "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d",
                 "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.19.0"
         },
         "anyio": {
@@ -60,7 +60,7 @@
                 "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08",
                 "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==23.1.0"
         },
         "argon2-cffi-bindings": {
@@ -110,7 +110,7 @@
                 "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
                 "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==23.2.0"
         },
         "babel": {
@@ -118,7 +118,7 @@
                 "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363",
                 "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.14.0"
         },
         "beautifulsoup4": {
@@ -189,7 +189,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4'",
+            "markers": "python_version >= '3.8' and python_version < '4.0'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -205,7 +205,7 @@
                 "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df",
                 "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.29.165"
         },
         "brotli": {
@@ -302,7 +302,7 @@
                 "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2",
                 "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==5.3.2"
         },
         "certifi": {
@@ -499,7 +499,7 @@
                 "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
                 "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.3.2"
         },
         "click": {
@@ -507,7 +507,7 @@
                 "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
                 "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==8.1.7"
         },
         "click-plugins": {
@@ -522,7 +522,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -697,27 +697,31 @@
         },
         "debugpy": {
             "hashes": [
-                "sha256:125b9a637e013f9faac0a3d6a82bd17c8b5d2c875fb6b7e2772c5aba6d082332",
-                "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0",
-                "sha256:3c6fb41c98ec51dd010d7ed650accfd07a87fe5e93eca9d5f584d0578f28f35f",
-                "sha256:46ab6780159eeabb43c1495d9c84cf85d62975e48b6ec21ee10c95767c0590aa",
-                "sha256:57161629133113c97b387382045649a2b985a348f0c9366e22217c87b68b73c6",
-                "sha256:5d9de202f5d42e62f932507ee8b21e30d49aae7e46d5b1dd5c908db1d7068637",
-                "sha256:60009b132c91951354f54363f8ebdf7457aeb150e84abba5ae251b8e9f29a8a6",
-                "sha256:61eab4a4c8b6125d41a34bad4e5fe3d2cc145caecd63c3fe953be4cc53e65bf8",
-                "sha256:7fb95ca78f7ac43393cd0e0f2b6deda438ec7c5e47fa5d38553340897d2fbdfb",
-                "sha256:8cd0197141eb9e8a4566794550cfdcdb8b3db0818bdf8c49a8e8f8053e56e38b",
-                "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4",
-                "sha256:a64093656c4c64dc6a438e11d59369875d200bd5abb8f9b26c1f5f723622e153",
-                "sha256:a8b7a2fd27cd9f3553ac112f356ad4ca93338feadd8910277aff71ab24d8775f",
-                "sha256:b05a6b503ed520ad58c8dc682749113d2fd9f41ffd45daec16e558ca884008cd",
-                "sha256:bdc5ef99d14b9c0fcb35351b4fbfc06ac0ee576aeab6b2511702e5a648a2e595",
-                "sha256:e3412f9faa9ade82aa64a50b602544efcba848c91384e9f93497a458767e6926",
-                "sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e",
-                "sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada"
+                "sha256:016a9fcfc2c6b57f939673c874310d8581d51a0fe0858e7fac4e240c5eb743cb",
+                "sha256:0de56aba8249c28a300bdb0672a9b94785074eb82eb672db66c8144fff673146",
+                "sha256:1a9fe0829c2b854757b4fd0a338d93bc17249a3bf69ecf765c61d4c522bb92a8",
+                "sha256:28acbe2241222b87e255260c76741e1fbf04fdc3b6d094fcf57b6c6f75ce1242",
+                "sha256:3a79c6f62adef994b2dbe9fc2cc9cc3864a23575b6e387339ab739873bea53d0",
+                "sha256:3bda0f1e943d386cc7a0e71bfa59f4137909e2ed947fb3946c506e113000f741",
+                "sha256:3ebb70ba1a6524d19fa7bb122f44b74170c447d5746a503e36adc244a20ac539",
+                "sha256:58911e8521ca0c785ac7a0539f1e77e0ce2df753f786188f382229278b4cdf23",
+                "sha256:6df9aa9599eb05ca179fb0b810282255202a66835c6efb1d112d21ecb830ddd3",
+                "sha256:7a3afa222f6fd3d9dfecd52729bc2e12c93e22a7491405a0ecbf9e1d32d45b39",
+                "sha256:7eb7bd2b56ea3bedb009616d9e2f64aab8fc7000d481faec3cd26c98a964bcdd",
+                "sha256:92116039b5500633cc8d44ecc187abe2dfa9b90f7a82bbf81d079fcdd506bae9",
+                "sha256:a2e658a9630f27534e63922ebf655a6ab60c370f4d2fc5c02a5b19baf4410ace",
+                "sha256:bfb20cb57486c8e4793d41996652e5a6a885b4d9175dd369045dad59eaacea42",
+                "sha256:caad2846e21188797a1f17fc09c31b84c7c3c23baf2516fed5b40b378515bbf0",
+                "sha256:d915a18f0597ef685e88bb35e5d7ab968964b7befefe1aaea1eb5b2640b586c7",
+                "sha256:dda73bf69ea479c8577a0448f8c707691152e6c4de7f0c4dec5a4bc11dee516e",
+                "sha256:e38beb7992b5afd9d5244e96ad5fa9135e94993b0c551ceebf3fe1a5d9beb234",
+                "sha256:edcc9f58ec0fd121a25bc950d4578df47428d72e1a0d66c07403b04eb93bcf98",
+                "sha256:efd3fdd3f67a7e576dd869c184c5dd71d9aaa36ded271939da352880c012e703",
+                "sha256:f696d6be15be87aef621917585f9bb94b1dc9e8aced570db1b8a6fc14e8f9b42",
+                "sha256:fd97ed11a4c7f6d042d320ce03d83b20c3fb40da892f994bc041bbc415d7a099"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.8.0"
+            "version": "==1.8.1"
         },
         "decorator": {
             "hashes": [
@@ -831,51 +835,51 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:0255dbc128fee75fb9be364806b940ed450dd6838672a150d501ee86523ac61e",
-                "sha256:0a00bd0e68e88987dcc047ea31c26d40a3c61185153b03457956a87e39d43c37",
-                "sha256:0a1d313a415eaaba2b35d6cd33536560deeebd2ed758b9bfb89ab5d97dc5deac",
-                "sha256:0f750037e02beb8b3569fbff701a572e62a685d2a0e840d75816592280e5feae",
-                "sha256:13819db8445a0cec8c3ff5f243af6418ab19175072a9a92f6cc8ca7d1452754b",
-                "sha256:254d9a6f7be00212bf0c3159e0a420eb19c63793b2c05e049eb337f3023c5ecc",
-                "sha256:29495d6d109cdbabe73cfb6f419ce67080c3ef9ea1e08d5750240fd4b0c4763b",
-                "sha256:32ab2e9702dff0dd4510c7bb958f265a8d3dd5c0e2547e7b5f7a3df4979abb07",
-                "sha256:3480eeb52770ff75140fe7d9a2ec33fb67b07efea0ab5129c7e0c6a639c40c70",
-                "sha256:3a808f3c1d1df1f5bf39be869b6e0c263570cdafb5bdb2df66087733f566ea71",
-                "sha256:3b629108351d25512d4ea1a8393a2dba325b7b7d7308116b605ea3f8e1be88df",
-                "sha256:3d71606c9321f6701642bd4746f99b6089e53d7e9817fc6b964e90d9c5f0ecc6",
-                "sha256:3e2b95dce2ead58fb12524d0ca7d63a63459dd489e7e5838c3cd53557f8933e1",
-                "sha256:4a5a5318ba5365d992666ac4fe35365f93004109d18858a3e18ae46f67907670",
-                "sha256:4c811d3c73b6abac275babb8aa439206288f56fdb2c6f8835e3d7b70de8937a7",
-                "sha256:4e743935139aa485fe3253fc33fe467eab6ea42583fa681223ea3f1a93dd01e6",
-                "sha256:4ec558c543609e71b2275c4894e93493f65d2f41c15fe1d089080c1d0bb4d635",
-                "sha256:5465df494f20a7d01712b072ae3ee9ad2887004701b95cb2cc6dcb9c2c97a899",
-                "sha256:5b60e3afa9635e3dfd3ace2757039593e3bd3cf128be0ddb7a1ff4ac45fa5a50",
-                "sha256:63fbed184979f09a65aa9c88b395ca539c94287ba3a364517698462e13e457c9",
-                "sha256:69731e8bea0578b3c28fdb43dbf95b9386e2d49a399e9a4ad736b8e479b08085",
-                "sha256:6dd58cc03016b281bd2c74c84cdaa6bd3ce54c5a7f47478b7657b930ac3ed8eb",
-                "sha256:740947906590a878a4bde7dd748e85fefa4d470a268b964748403b3ab2aeed6c",
-                "sha256:7df26dd3650e98ca45f1e29883c96a0b9f5bb6af8d632a6a108bc744fa0bd9b3",
-                "sha256:7eb7ad665258fba68fd22228a09f347469d95a97fb88198e133595947a20a184",
-                "sha256:7ee48bd9d6b7e8f66866c9090807e3a4a56cf43ffad48962725a190e0dd774c8",
-                "sha256:86e0427864c6c91cf77f16d1fb9bf1bbf7453e824589e8fb8461b6ee1144f506",
-                "sha256:8f57ecd742545362a0f7186774b2d1c53423ed9ece67689c93a1055b236f638c",
-                "sha256:90f898cdd67f52f18049250a6474185ef6544c91f27a7bee70d87d77a8daf89c",
-                "sha256:94208ea750e3f96e267f394d5588579bb64cc628e321dbb1d4243ffbc291b18b",
-                "sha256:a1c154bb85dc9a4cf145250c88d112d88eb414bad81d4cb524d06258dea1bdc0",
-                "sha256:a5d77479fb885ef38a16a253a2f4096bc3d14e63a56d6246bfdb56365a12b20c",
-                "sha256:a86a5ab2873ed2575d0fcdf1828143cfc6b977ac448e3dc616bb1e3d20efbafa",
-                "sha256:ac71e2e201df041a2891067dc36256755b1229ae167edbdc419b16da78732c2f",
-                "sha256:b3e1304e5f19ca861d86a72218ecce68f391646d85c851742d265787f55457a4",
-                "sha256:b8be28c036b9f186e8c7eaf8a11b42373e7e4949f9e9f370202b9da4c4c3f56c",
-                "sha256:c19044256c44fe299d9a73456aabee4b4d06c6b930287be93b533b4737d70aa1",
-                "sha256:d49ce3ea7b7173faebc5664872243b40cf88814ca3eb135c4a3cdff66af71946",
-                "sha256:e040f905d542362e07e72e03612a6270c33d38281fd573160e1003e43718d68d",
-                "sha256:eabae77a07c41ae0b35184894202305c3ad211a93b2eb53837c2a1143c8bc952",
-                "sha256:f791446ff297fd5f1e2247c188de53c1bfb9dd7f0549eba55b73a3c2087a2703",
-                "sha256:f83a4daef6d2a202acb9bf572958f91cfde5b10c8ee7fb1d09a4c81e5d851fd8"
+                "sha256:0452fcbfbce752ba596737a7c5ec5cf76bc5f83847ce1781f4f90eab14ece252",
+                "sha256:0a2417547462e468edf35b32e3dd06a6215ac26aa6316b41e03b8eeaf9f079ea",
+                "sha256:0d2b01428f7da26f229a5656defc824427b741e454b4e210ad2b25ed6ea2aed4",
+                "sha256:0d533f89819f9b3ee2dbedf0fed3825c425850e32bdda24c558563c71be0064e",
+                "sha256:12ee86abca46193359ea69216b3a724e90c66ab05ab220d39e3fc068c1eb72ac",
+                "sha256:18b35fd1a850ed7233a99bbd6774485271756f717dac8b594958224b54118b61",
+                "sha256:292922dc356d7f11f5063b4111a8b719efb8faea92a2a88ed296408d449d8c2e",
+                "sha256:2eb4167bde04e172a93cf22c875d8b0cff76a2491f67f5eb069566215302d45d",
+                "sha256:3cdb9a92521b81bf717ebccf592bd0292e853244d84115bfb4db0c426de58348",
+                "sha256:4108b1d247953dd7c90ec8f457a2dec5fceb373485973cc852b14200118a51ee",
+                "sha256:4709c5bf123ba10eac210d2d5c9027d3f472591d9f1a04262122710fa3d23199",
+                "sha256:5057ade278e67923000041e2b195c9ea53e87f227690d499b6a4edd3702f7f01",
+                "sha256:56339ec557f0c342bddd7c175f5e41c45fc21282bee58a86bd9aa322bec715f2",
+                "sha256:578c00f93868f64a4102ecc5aa600a03b49162c654676c3fadc33de2ddb88a81",
+                "sha256:594206b31c95fcfa65f484385171fabb4ec69f7d2d7f56d27f17db26b7a31814",
+                "sha256:63c73b9dd56a94a3cbd2f90544b5fca83666948a9e03370888994143b8d7c070",
+                "sha256:63dc592a16cd08388d8c4c7502b59ac74190b23e16dfc863c69fe1ea74605b68",
+                "sha256:6978bade7b6c0335095bdd0bd97f8f3d590d2877b370f17e03e0865241694eb5",
+                "sha256:6f30e605c7565d0da6f0aec75a30ec372072d016957cd8fc4469721a36ea59b7",
+                "sha256:702ae93058c81f46461dc4b2c79f11d3c3d8fd7296eaf8f75b4ba5bbf813cd5f",
+                "sha256:8b8a45254218679c7f1127812761e7854ed5c8e34349aebf581e8c9204e7495a",
+                "sha256:902e9c4e9928301912f34a6638741b8ae0b64824112b42aaf240e06b735774b1",
+                "sha256:97f0a49fa6aa2d6205c6f72f4f98b74ef4b9bfdcb06fd78e6fe6c7af4989b63e",
+                "sha256:9b4ec6d42a7555f5ae35f3b805482f0aad0f1baeeef54859492ea3b782959d4a",
+                "sha256:9b58638d8a85e3a1b32ec0a91d9f8171a877b4b81c408d4cb3257d0dee63e092",
+                "sha256:a8c8b54bd1420c184a995f980f1a8076f87363e2bb24239ef8c171a369d85a31",
+                "sha256:aee76fd81a8571c68841d6ef0da750d5ff08ff2c5f025576473016f16ac3bcf7",
+                "sha256:b10633aafc5932995a391ec07eba5e79f52af0003a1735b2306b3dab8a056d48",
+                "sha256:bcd77f89fc1a6b18428e7a55dde8ef56dae95640293bfb8f4e929929eba5e2a2",
+                "sha256:bff5b38d0e76eb18e0b8abbf35d384e60b3371be92f7be36128ee3e67483b3ec",
+                "sha256:c900508c46274d32d308ae8e82335117f11aaee1f7d369ac16502c9a78930b0a",
+                "sha256:cad5cfd044ea2e306fda44482b3dd32ee47830fa82dfa4679374b41baa294f5f",
+                "sha256:cdfd7557d1bd294a200bd211aa665ca3b02998dcc18f8211a5532da5b8fad5c5",
+                "sha256:cf5a0cd974f85a80b74785db2d5c3c1fd6cc09a2ba3c837359b2b5da629ee1b0",
+                "sha256:d10979ef14a8beaaa32f613bb698743f7241d92f437a3b5e32356dfb9769c65d",
+                "sha256:d20588466367f05025bb1efdf4e5d498ca6d14bde07b6928b79199c588800f0a",
+                "sha256:d3260db55f1843e57115256e91247ad9f68cb02a434b51262fe0019e95a98738",
+                "sha256:df48798f9a4fc4c315ab46e17873436c8746f5df6eddd02fad91299b2af7af95",
+                "sha256:e3e33862fc5261d46d9aae3544acb36203b1a337d00bdb5d3753aae50dac860e",
+                "sha256:e740a7602c2bb71e1091269b5dbe89549749a8817dc294b34628ffd8b2bf7124",
+                "sha256:f40441437b039930428e04fb05ac3a132e77458fb57666c808d74a556779e784",
+                "sha256:f7449493886da6a17472004d3818cc050ba3f4a0aa03fb47972e4fa5578e6703"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.47.2"
+            "version": "==4.48.1"
         },
         "fqdn": {
             "hashes": [
@@ -886,11 +890,11 @@
         },
         "fsspec": {
             "hashes": [
-                "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb",
-                "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"
+                "sha256:817f969556fa5916bc682e02ca2045f96ff7f586d45110fcb76022063ad2c7d8",
+                "sha256:b6ad1a679f760dda52b1168c859d01b7b80648ea6f7f7c7f5a8a91dc3f3ecb84"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.12.2"
+            "version": "==2024.2.0"
         },
         "geocube": {
             "hashes": [
@@ -905,23 +909,23 @@
                 "sha256:6b7225248e45ff7edcee32becc4e0a1504c606ac5ee163a5656d482e0cd38734",
                 "sha256:f7f41c85dc3e1c2d3d935ec86660dc3b2c848c83e17f9a9e51ba9d5146a15859"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.0"
         },
         "geopandas": {
             "hashes": [
-                "sha256:0a470e4bf6f5367e6fd83ab6b40405e0b805c8174665bbcb7c4077ed90202912",
-                "sha256:0acdacddefa176525e4da6d9aeeece225da26055c4becdc6e97cf40fa97c27f4"
+                "sha256:41b31ad39e21bc9e8c4254f78f8dc4ce3d33d144e22e630a00bb336c83160204",
+                "sha256:748af035d4a068a4ae00cab384acb61d387685c833b0022e0729aa45216b23ac"
             ],
             "index": "pypi",
-            "version": "==0.12.2"
+            "version": "==0.14.3"
         },
         "geopy": {
             "hashes": [
                 "sha256:50283d8e7ad07d89be5cb027338c6365a32044df3ae2556ad3f52f4840b3d0d1",
                 "sha256:ae8b4bc5c1131820f4d75fce9d4aaaca0c85189b3aa5d64c3dcaf5e3b7b882a7"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.4.1"
         },
         "gval": {
@@ -934,11 +938,11 @@
         },
         "identify": {
             "hashes": [
-                "sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d",
-                "sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34"
+                "sha256:a4316013779e433d08b96e5eabb7f641e6c7942e4ab5d4c509ebd2e7a8994aed",
+                "sha256:ee17bc9d499899bc9eaec1ac7bf2dc9eedd480db9d88b96d123d3b64a9d34f5d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.33"
+            "version": "==2.5.34"
         },
         "idna": {
             "hashes": [
@@ -1012,7 +1016,7 @@
                 "sha256:f5924499dc8800928c0ee4580fa8eb4ffa880b2cce4431537d0390e503a9c9ee",
                 "sha256:f79542478e49e471e8b23556700e6f688a40dc93e9a746f77a546c13251b59b1"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.0.0"
         },
         "iniconfig": {
@@ -1020,16 +1024,16 @@
                 "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
                 "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.0.0"
         },
         "ipykernel": {
             "hashes": [
-                "sha256:076663ca68492576f051e4af7720d33f34383e655f2be0d544c8b1c9de915b2f",
-                "sha256:b5dd3013cab7b330df712891c96cd1ab868c27a7159e606f762015e9bf8ceb3f"
+                "sha256:3bade28004e3ff624ed57974948116670604ac5f676d12339693f3142176d3f0",
+                "sha256:50384f5c577a260a1d53f1f59a828c7266d321c9b7d00d345693783f66616055"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.29.0"
+            "version": "==6.29.2"
         },
         "ipympl": {
             "hashes": [
@@ -1056,11 +1060,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:2b88d728656aea3bbfd05d32c747cfd0078f9d7e159cf982433b58ad717eed7f",
-                "sha256:40211efb556adec6fa450ccc2a77d59ca44a060f4f9f136833df59c9f538e6e8"
+                "sha256:bbe43850d79fb5e906b14801d6c01402857996864d1e5b6fa62dd2ee35559f60",
+                "sha256:d0b9b41e49bae926a866e613a39b0f0097745d2b9f1f3dd406641b4a57ec42c9"
             ],
-            "markers": "python_full_version >= '3.7.0'",
-            "version": "==8.1.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.2"
         },
         "isoduration": {
             "hashes": [
@@ -1090,7 +1094,7 @@
                 "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
                 "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.1.3"
         },
         "jmespath": {
@@ -1098,7 +1102,7 @@
                 "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
                 "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.0.1"
         },
         "joblib": {
@@ -1106,7 +1110,7 @@
                 "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1",
                 "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.3.2"
         },
         "json5": {
@@ -1156,7 +1160,7 @@
                 "sha256:214668aaea208195f4c13d28eb272ba79f945fc0cf3f11c7092c20b2ca1980e7",
                 "sha256:52be28e04171f07aed8f20e1616a5a552ab9fee9cbbe6c1896ae170c3880d392"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==7.4.9"
         },
         "jupyter-console": {
@@ -1164,7 +1168,7 @@
                 "sha256:309d33409fcc92ffdad25f0bcdf9a4a9daa61b6f341177570fdac03de5352485",
                 "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==6.6.3"
         },
         "jupyter-core": {
@@ -1196,7 +1200,7 @@
                 "sha256:7486bca3acf9bbaab7ce5127f9f64d2df58f5d2de377609fb833291a7217a6a2",
                 "sha256:76dd05a45b78c7ec0cba0be98ece289984c6bcfc1ca2da216d42930e506a4d68"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.9.1"
         },
         "jupyter-server-terminals": {
@@ -1212,7 +1216,7 @@
                 "sha256:969a3a1a77ed4e99487d60a74048dc9fa7d3b0dcd32e60885d835bbf7ba7be11",
                 "sha256:a6fe125091792d16c962cc3720c950c2b87fcc8c3ecf0c54c84e9a20b814526c"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.8.0"
         },
         "jupyter-ydoc": {
@@ -1220,7 +1224,7 @@
                 "sha256:5759170f112c70320a84217dd98d287699076ae65a7f88d458d57940a9f2b882",
                 "sha256:5a02ca7449f0d875f73e8cb8efdf695dddef15a8e71378b1f4eda6b7c90f5382"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.2.5"
         },
         "jupyterlab": {
@@ -1249,11 +1253,11 @@
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:3cf5bdf5b897bf3bccf1c11873aa4afd776d7430200f765e0686bd352487b58d",
-                "sha256:6005a4e974c7beee84060fdfba341a3218495046de8ae3ec64888e5fe19fdb4c"
+                "sha256:04f2ac04976727e4f9d0fa91cdc2f1ab860f965e504c29dbd6a65c882c9d04c0",
+                "sha256:dd61f3ae7a5a7f80299e14585ce6cf3d6925a96c9103c978eda293197730cb64"
             ],
-            "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.0.9"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.0.10"
         },
         "kiwisolver": {
             "hashes": [
@@ -1362,7 +1366,7 @@
                 "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892",
                 "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.4.5"
         },
         "llvmlite": {
@@ -1396,7 +1400,7 @@
                 "sha256:fa9b26939ae553bf30a9f5c4c754db0fb2d2677327f2511e674aa2f5df941789",
                 "sha256:fb62fc7016b592435d3e3a8f680e3ea8897c3c9e62e6e6cc58011e7a4801439e"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.39.1"
         },
         "locket": {
@@ -1409,69 +1413,69 @@
         },
         "markupsafe": {
             "hashes": [
-                "sha256:0042d6a9880b38e1dd9ff83146cc3c9c18a059b9360ceae207805567aacccc69",
-                "sha256:0c26f67b3fe27302d3a412b85ef696792c4a2386293c53ba683a89562f9399b0",
-                "sha256:0fbad3d346df8f9d72622ac71b69565e621ada2ce6572f37c2eae8dacd60385d",
-                "sha256:15866d7f2dc60cfdde12ebb4e75e41be862348b4728300c36cdf405e258415ec",
-                "sha256:1c98c33ffe20e9a489145d97070a435ea0679fddaabcafe19982fe9c971987d5",
-                "sha256:21e7af8091007bf4bebf4521184f4880a6acab8df0df52ef9e513d8e5db23411",
-                "sha256:23984d1bdae01bee794267424af55eef4dfc038dc5d1272860669b2aa025c9e3",
-                "sha256:31f57d64c336b8ccb1966d156932f3daa4fee74176b0fdc48ef580be774aae74",
-                "sha256:3583a3a3ab7958e354dc1d25be74aee6228938312ee875a22330c4dc2e41beb0",
-                "sha256:36d7626a8cca4d34216875aee5a1d3d654bb3dac201c1c003d182283e3205949",
-                "sha256:396549cea79e8ca4ba65525470d534e8a41070e6b3500ce2414921099cb73e8d",
-                "sha256:3a66c36a3864df95e4f62f9167c734b3b1192cb0851b43d7cc08040c074c6279",
-                "sha256:3aae9af4cac263007fd6309c64c6ab4506dd2b79382d9d19a1994f9240b8db4f",
-                "sha256:3ab3a886a237f6e9c9f4f7d272067e712cdb4efa774bef494dccad08f39d8ae6",
-                "sha256:47bb5f0142b8b64ed1399b6b60f700a580335c8e1c57f2f15587bd072012decc",
-                "sha256:49a3b78a5af63ec10d8604180380c13dcd870aba7928c1fe04e881d5c792dc4e",
-                "sha256:4df98d4a9cd6a88d6a585852f56f2155c9cdb6aec78361a19f938810aa020954",
-                "sha256:5045e892cfdaecc5b4c01822f353cf2c8feb88a6ec1c0adef2a2e705eef0f656",
-                "sha256:5244324676254697fe5c181fc762284e2c5fceeb1c4e3e7f6aca2b6f107e60dc",
-                "sha256:54635102ba3cf5da26eb6f96c4b8c53af8a9c0d97b64bdcb592596a6255d8518",
-                "sha256:54a7e1380dfece8847c71bf7e33da5d084e9b889c75eca19100ef98027bd9f56",
-                "sha256:55d03fea4c4e9fd0ad75dc2e7e2b6757b80c152c032ea1d1de487461d8140efc",
-                "sha256:698e84142f3f884114ea8cf83e7a67ca8f4ace8454e78fe960646c6c91c63bfa",
-                "sha256:6aa5e2e7fc9bc042ae82d8b79d795b9a62bd8f15ba1e7594e3db243f158b5565",
-                "sha256:7653fa39578957bc42e5ebc15cf4361d9e0ee4b702d7d5ec96cdac860953c5b4",
-                "sha256:765f036a3d00395a326df2835d8f86b637dbaf9832f90f5d196c3b8a7a5080cb",
-                "sha256:78bc995e004681246e85e28e068111a4c3f35f34e6c62da1471e844ee1446250",
-                "sha256:7a07f40ef8f0fbc5ef1000d0c78771f4d5ca03b4953fc162749772916b298fc4",
-                "sha256:8b570a1537367b52396e53325769608f2a687ec9a4363647af1cded8928af959",
-                "sha256:987d13fe1d23e12a66ca2073b8d2e2a75cec2ecb8eab43ff5624ba0ad42764bc",
-                "sha256:9896fca4a8eb246defc8b2a7ac77ef7553b638e04fbf170bff78a40fa8a91474",
-                "sha256:9e9e3c4020aa2dc62d5dd6743a69e399ce3de58320522948af6140ac959ab863",
-                "sha256:a0b838c37ba596fcbfca71651a104a611543077156cb0a26fe0c475e1f152ee8",
-                "sha256:a4d176cfdfde84f732c4a53109b293d05883e952bbba68b857ae446fa3119b4f",
-                "sha256:a76055d5cb1c23485d7ddae533229039b850db711c554a12ea64a0fd8a0129e2",
-                "sha256:a76cd37d229fc385738bd1ce4cba2a121cf26b53864c1772694ad0ad348e509e",
-                "sha256:a7cc49ef48a3c7a0005a949f3c04f8baa5409d3f663a1b36f0eba9bfe2a0396e",
-                "sha256:abf5ebbec056817057bfafc0445916bb688a255a5146f900445d081db08cbabb",
-                "sha256:b0fe73bac2fed83839dbdbe6da84ae2a31c11cfc1c777a40dbd8ac8a6ed1560f",
-                "sha256:b6f14a9cd50c3cb100eb94b3273131c80d102e19bb20253ac7bd7336118a673a",
-                "sha256:b83041cda633871572f0d3c41dddd5582ad7d22f65a72eacd8d3d6d00291df26",
-                "sha256:b835aba863195269ea358cecc21b400276747cc977492319fd7682b8cd2c253d",
-                "sha256:bf1196dcc239e608605b716e7b166eb5faf4bc192f8a44b81e85251e62584bd2",
-                "sha256:c669391319973e49a7c6230c218a1e3044710bc1ce4c8e6eb71f7e6d43a2c131",
-                "sha256:c7556bafeaa0a50e2fe7dc86e0382dea349ebcad8f010d5a7dc6ba568eaaa789",
-                "sha256:c8f253a84dbd2c63c19590fa86a032ef3d8cc18923b8049d91bcdeeb2581fbf6",
-                "sha256:d18b66fe626ac412d96c2ab536306c736c66cf2a31c243a45025156cc190dc8a",
-                "sha256:d5291d98cd3ad9a562883468c690a2a238c4a6388ab3bd155b0c75dd55ece858",
-                "sha256:d5c31fe855c77cad679b302aabc42d724ed87c043b1432d457f4976add1c2c3e",
-                "sha256:d6e427c7378c7f1b2bef6a344c925b8b63623d3321c09a237b7cc0e77dd98ceb",
-                "sha256:dac1ebf6983148b45b5fa48593950f90ed6d1d26300604f321c74a9ca1609f8e",
-                "sha256:de8153a7aae3835484ac168a9a9bdaa0c5eee4e0bc595503c95d53b942879c84",
-                "sha256:e1a0d1924a5013d4f294087e00024ad25668234569289650929ab871231668e7",
-                "sha256:e7902211afd0af05fbadcc9a312e4cf10f27b779cf1323e78d52377ae4b72bea",
-                "sha256:e888ff76ceb39601c59e219f281466c6d7e66bd375b4ec1ce83bcdc68306796b",
-                "sha256:f06e5a9e99b7df44640767842f414ed5d7bedaaa78cd817ce04bbd6fd86e2dd6",
-                "sha256:f6be2d708a9d0e9b0054856f07ac7070fbe1754be40ca8525d5adccdbda8f475",
-                "sha256:f9917691f410a2e0897d1ef99619fd3f7dd503647c8ff2475bf90c3cf222ad74",
-                "sha256:fc1a75aa8f11b87910ffd98de62b29d6520b6d6e8a3de69a70ca34dea85d2a8a",
-                "sha256:fe8512ed897d5daf089e5bd010c3dc03bb1bdae00b35588c49b98268d4a01e00"
-            ],
-            "markers": "python_full_version >= '3.7.0'",
-            "version": "==2.1.4"
+                "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf",
+                "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff",
+                "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f",
+                "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3",
+                "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532",
+                "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f",
+                "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617",
+                "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df",
+                "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4",
+                "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906",
+                "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f",
+                "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4",
+                "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8",
+                "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371",
+                "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2",
+                "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465",
+                "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52",
+                "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6",
+                "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169",
+                "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad",
+                "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2",
+                "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0",
+                "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029",
+                "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f",
+                "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a",
+                "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced",
+                "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5",
+                "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c",
+                "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf",
+                "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9",
+                "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb",
+                "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad",
+                "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3",
+                "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1",
+                "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46",
+                "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc",
+                "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a",
+                "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee",
+                "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900",
+                "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5",
+                "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea",
+                "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f",
+                "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5",
+                "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e",
+                "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a",
+                "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f",
+                "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50",
+                "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a",
+                "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b",
+                "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4",
+                "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff",
+                "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2",
+                "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46",
+                "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b",
+                "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf",
+                "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5",
+                "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5",
+                "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab",
+                "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd",
+                "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.1.5"
         },
         "matplotlib": {
             "hashes": [
@@ -1548,7 +1552,7 @@
                 "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205",
                 "sha256:fc7f93ded930c92394ef2cb6f04a8aabab4117a91449e72dcc8dfa646a508be8"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.2"
         },
         "morecantile": {
@@ -1673,7 +1677,7 @@
                 "sha256:0ae11eb2319455d805596bf320336cda9554b41d99ab9a3c31bf8180bffa30e3",
                 "sha256:f99e4769b4750076cd4235c044b61232110733322384a94a63791d2e7beacc66"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.0.0"
         },
         "nbclient": {
@@ -1686,11 +1690,11 @@
         },
         "nbconvert": {
             "hashes": [
-                "sha256:a7f8808fd4e082431673ac538400218dd45efd076fbeb07cc6e5aa5a3a4e949e",
-                "sha256:db28590cef90f7faf2ebbc71acd402cbecf13d29176df728c0a9025a49345ea1"
+                "sha256:813e6553796362489ae572e39ba1bff978536192fb518e10826b0e8cadf03ec8",
+                "sha256:ad3dc865ea6e2768d31b7eb6c7ab3be014927216a5ece3ef276748dd809054c7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.14.2"
+            "version": "==7.16.0"
         },
         "nbformat": {
             "hashes": [
@@ -1754,7 +1758,7 @@
                 "sha256:b4625a4b7a597839dd3156b140d5ba2c7123761f98245a3290f67a8b8ee048d9",
                 "sha256:c1e2eb2e3b6079a0552a04974883a48d04c3c05792170d64a4b23d707d453181"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==6.5.6"
         },
         "notebook-shim": {
@@ -1762,7 +1766,7 @@
                 "sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7",
                 "sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.2.3"
         },
         "numba": {
@@ -1897,7 +1901,7 @@
                 "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
                 "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==23.2"
         },
         "pandas": {
@@ -1936,7 +1940,7 @@
                 "sha256:1cc70f5f182f5b4bd2e248f49372fcd88eba8423b8604ea9e1ebd64222730a41",
                 "sha256:1cf13c06068d64edb1630b0e50d6588243b7c202d7ce00d968fe3d6bef7684f8"
             ],
-            "markers": "python_version < '3.12' and python_full_version >= '3.7.0'",
+            "markers": "python_version < '3.12' and python_version >= '3.7'",
             "version": "==0.15.1"
         },
         "pandocfilters": {
@@ -1968,7 +1972,7 @@
                 "sha256:27e766663d36c161e2827aa3e28541c992f0b9527d3cca047e13fb3acdb989e6",
                 "sha256:56c25dd49e6fea5727e731203c466c6e092f308d8f0024e199d02f6aa2167f67"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.4.1"
         },
         "pathspec": {
@@ -2098,7 +2102,7 @@
                 "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
                 "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.43"
         },
         "psutil": {
@@ -2368,7 +2372,7 @@
                 "sha256:a4038a8885059ab8cac6f946ea30e0b5e6bdbe0b92b6723f06737035f9d65e8c",
                 "sha256:dd9f4ac5cbd8e37c352c04036062d3c5f67efec76d404761ef16b0cbf26aa6a0"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.5.0"
         },
         "pydantic": {
@@ -2410,7 +2414,7 @@
                 "sha256:f3d4ee957a727ccb5a36f1b0a6dbd9fad5dedd2a41eada99a8df55c12896e18d",
                 "sha256:f79db3652ed743309f116ba863dae0c974a41b688242482638b892246b7db21d"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.10.10"
         },
         "pyflakes": {
@@ -2434,9 +2438,40 @@
                 "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
                 "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.17.2"
         },
+        "pyogrio": {
+            "hashes": [
+                "sha256:1b7197c72f034ac7187da2a8d50a063a5f1256aab732b154f11f887a7652dc3d",
+                "sha256:234b0d1d22e9680229b0618c25077a0cb2428cbbc2939b4bb9bdd8ee77e0f3e0",
+                "sha256:3001efd5dfee36459d0cfdafbe91ed88fc5ae734353d771cdb75546ef1427735",
+                "sha256:31112bb0b6a4a3f80ec3252d7eeb7be81045860d49fd76e297c073759450652b",
+                "sha256:33ae5aafcf3a557e107a33f5b3e878750d2e467b8cc911dc4bf261c1a602b534",
+                "sha256:33afb7d211c6434613f24174722347a5cb11d22a212f28c817f67c89d30d0c0d",
+                "sha256:429dcff4c36f0e0a15ba4a20f2d4478b9c6d095e70c4bcc007a536ea420a1a93",
+                "sha256:436de39f57e8f8cc41682981518b9490d64d3a1c48bf78d415e5747c296790dc",
+                "sha256:5654e7c33442cbd98e7a56f705e160415d7503b2420d724d4f81b8cc88360b3e",
+                "sha256:5feeb7a0da7ee82580f6aa6508a80602413675b99c60c822929e0e8b925e0517",
+                "sha256:73577fecebeecf0d06e78c1a4bddd460a4d57c6d918affab7594c0bc72f5fa14",
+                "sha256:7e2c856961efdc6cb3809b97b49016cbbcee17c8a1e85fc4000b5fcb3cfcb9b1",
+                "sha256:7e39bb6bfdd74e63ae96acced7297bbe8a157f85c0107f1cbb395d2a937f3a38",
+                "sha256:860b04ddf23b8c253ceb3621e4b0e0dc0f293eab66cb14f799a5c9f9fe0a882c",
+                "sha256:892fdab0e1c44c0125254d92928081c14f93ac553f371addc2c9a1d4bde41cad",
+                "sha256:9cc6db2e5dc50dfe23554d10502920eafa0648c365725e552aaa523432a9bf35",
+                "sha256:a23136d1bffa9d811263807b850c6e9854201710276f09de650131e89f2486aa",
+                "sha256:b9a8a4854c7af2c76683ce5666ee765b207901b362576465219d75deb6159821",
+                "sha256:ba386a02c9b5934c568b40acc95c9863f92075f6990167635e51368976569c66",
+                "sha256:be46be43c4148a3ad09da38670411485ec544a51cbd6b7d004a0eca5035023fc",
+                "sha256:bee556ca305b7e8c68aada259d925c612131205074fb2373badafacbef610b77",
+                "sha256:caaf61d473ac207f170082e602ea57c096e8dd4c4be51de58fba96f1a5944096",
+                "sha256:d5fc2304aeb927564f77caaa4da9a47e2d77a8ceb1c624ea84c505140886b221",
+                "sha256:f219c1edb010d0248891a3d27d15faf17c91cfe69daef84d7471e22e4ed4fcff",
+                "sha256:f2ff58184020da39540a2f5d4a5412005a01b0c4cd03c7b8294bc670d1f3fe50"
+            ],
+            "index": "pypi",
+            "version": "==0.7.2"
+        },
         "pyparsing": {
             "hashes": [
                 "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb",
@@ -2604,10 +2639,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40",
-                "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a"
+                "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812",
+                "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"
             ],
-            "version": "==2023.4"
+            "version": "==2024.1"
         },
         "pyyaml": {
             "hashes": [
@@ -2889,7 +2924,7 @@
                 "sha256:1c1d8c4fa2c884ae742b069151b0abe15b3f70491f3972698c683b8e38de839b",
                 "sha256:a5a15ffd519550a1361bdc56ffc07fda56a6af7292f17c7b395d4083af632987"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.4.1"
         },
         "rasterio": {
@@ -2936,7 +2971,7 @@
                 "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
                 "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.31.0"
         },
         "rfc3339-validator": {
@@ -3151,7 +3186,7 @@
                 "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084",
                 "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.6.2"
         },
         "scipy": {
@@ -3199,11 +3234,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05",
-                "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"
+                "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401",
+                "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==69.0.3"
+            "version": "==69.1.0"
         },
         "shapely": {
             "hashes": [
@@ -3366,7 +3401,7 @@
                 "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101",
                 "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.3.0"
         },
         "snuggs": {
@@ -3434,7 +3469,7 @@
                 "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847",
                 "sha256:8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.2.1"
         },
         "tomli": {
@@ -3450,7 +3485,7 @@
                 "sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85",
                 "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.12.1"
         },
         "tornado": {
@@ -3519,11 +3554,11 @@
         },
         "tzdata": {
             "hashes": [
-                "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3",
-                "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"
+                "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd",
+                "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"
             ],
             "markers": "python_version >= '2'",
-            "version": "==2023.4"
+            "version": "==2024.1"
         },
         "uri-template": {
             "hashes": [
@@ -3545,7 +3580,7 @@
                 "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3",
                 "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==20.25.0"
         },
         "wcwidth": {
@@ -3587,11 +3622,11 @@
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:3c1f5e46dc1166dfd40a42d685e6a51396fd34ff878742a3e47c6f0cc4a2a385",
-                "sha256:91452ca8445beb805792f206e560c1769284267a30ceb1cec9f5bcc887d15175"
+                "sha256:64196c5ff3b9a9183a8e699a4227fb0b7002f252c814098e66c4d1cd0644688f",
+                "sha256:d37c3724ec32d8c48400a435ecfa7d3e259995201fbefa37163124a9fcb393cc"
             ],
-            "markers": "python_full_version >= '3.7.0'",
-            "version": "==4.0.9"
+            "markers": "python_version >= '3.7'",
+            "version": "==4.0.10"
         },
         "wrapt": {
             "hashes": [
@@ -3682,7 +3717,7 @@
                 "sha256:8a356ac66a61ff9522453194db5d184e1eccee8ab890c7ec723bb48c6eafd392",
                 "sha256:fd7e3b447236f6e8f8c5110684a7ca50f97149c806f56ebf4ed1eeef1dd681ce"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.3.5"
         },
         "xyzservices": {
@@ -3777,7 +3812,7 @@
                 "sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff",
                 "sha256:b1ba0dfcc9762f0ca168d2378062d3ca1299d39076b0f145d961359121042be5"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==0.8.4"
         },
         "zipp": {
@@ -3872,7 +3907,7 @@
                 "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
                 "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.43"
         },
         "ptyprocess": {
@@ -3895,7 +3930,7 @@
                 "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
                 "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==2.17.2"
         },
         "six": {
diff --git a/data/bathymetry/preprocess_bathymetry.py b/data/bathymetry/preprocess_bathymetry.py
index f4788c996..aeda133c1 100644
--- a/data/bathymetry/preprocess_bathymetry.py
+++ b/data/bathymetry/preprocess_bathymetry.py
@@ -68,7 +68,7 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
     bathy_gdal = gdal_array.OpenArray(bathy_m)
 
     # Read in shapefiles
-    bathy_bounds = gpd.read_file(survey_gdb, layer=bathy_bounds)
+    bathy_bounds = gpd.read_file(survey_gdb, layer=bathy_bounds, engine="pyogrio", use_arrow=True)
     nwm_streams = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_flows.gpkg", mask=bathy_bounds)
     nwm_catchments = gpd.read_file("/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg", mask=bathy_bounds)
     bathy_bounds = bathy_bounds.to_crs(nwm_streams.crs)
@@ -125,7 +125,7 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
     bathy_nwm_streams = bathy_nwm_streams.to_crs(epsg=5070)
     if os.path.exists(output):
         print(f"{output} already exists. Concatinating now...")
-        existing_bathy_file = gpd.read_file(output)
+        existing_bathy_file = gpd.read_file(output, engine="pyogrio", use_arrow=True)
         bathy_nwm_streams = pd.concat([existing_bathy_file, bathy_nwm_streams])
     bathy_nwm_streams.to_file(output, index=False)
     print(f"Added {num_streams} new NWM features")
diff --git a/data/ble/ble_benchmark/create_flow_forecast_file.py b/data/ble/ble_benchmark/create_flow_forecast_file.py
index 62bdb7b1e..65c22daf0 100755
--- a/data/ble/ble_benchmark/create_flow_forecast_file.py
+++ b/data/ble/ble_benchmark/create_flow_forecast_file.py
@@ -9,6 +9,9 @@
 import pandas as pd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def create_flow_forecast_file(
     huc,
     ble_geodatabase,
diff --git a/data/esri.py b/data/esri.py
index 4b3f9fbd9..9a762fb5f 100644
--- a/data/esri.py
+++ b/data/esri.py
@@ -5,6 +5,9 @@
 from tqdm import tqdm
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 class ESRI_REST(object):
     """
     This class was built for querying ESRI REST endpoints for the purpose of downloading datasets.
diff --git a/data/nld/levee_download.py b/data/nld/levee_download.py
index 995537f84..8893f00bb 100755
--- a/data/nld/levee_download.py
+++ b/data/nld/levee_download.py
@@ -14,6 +14,9 @@
 from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 epsg_code = re.search(r'\d+$', DEFAULT_FIM_PROJECTION_CRS).group()
 today = datetime.now().strftime('%y%m%d')
 nld_vector_output = os.path.join(INPUTS_DIR, 'nld_vectors', f'System_Routes_NLDFS_5070_{today}.gpkg')
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index b6ded4198..c43ce1e44 100755
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -19,6 +19,9 @@
 from utils.shared_functions import FIM_Helpers as fh
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 '''
 TODO:
     - Add input args for resolution size, which means URL and block size also hve to be parameterized.
diff --git a/data/wbd/clip_vectors_to_wbd.py b/data/wbd/clip_vectors_to_wbd.py
index 2bf752bbb..2425bfc45 100755
--- a/data/wbd/clip_vectors_to_wbd.py
+++ b/data/wbd/clip_vectors_to_wbd.py
@@ -41,8 +41,8 @@ def subset_vector_layers(
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
 
-    wbd = gpd.read_file(wbd_filename)
-    dem_domain = gpd.read_file(dem_domain)
+    wbd = gpd.read_file(wbd_filename, engine="pyogrio", use_arrow=True)
+    dem_domain = gpd.read_file(dem_domain, engine="pyogrio", use_arrow=True)
 
     # Get wbd buffer
     print(f"Create wbd buffer for {hucCode}", flush=True)
diff --git a/data/wbd/preprocess_wbd.py b/data/wbd/preprocess_wbd.py
index a61a8010c..0a9d4ef46 100755
--- a/data/wbd/preprocess_wbd.py
+++ b/data/wbd/preprocess_wbd.py
@@ -9,6 +9,9 @@
 from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def clip_wbd_to_dem_domain(dem: str, wbd_in: str, wbd_out: str, huc_level: int):
     """
     Clips Watershed Boundary Dataset (WBD) to DEM domain
diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py
index 5ffb7a50b..181d99bdc 100644
--- a/data/write_parquet_from_calib_pts.py
+++ b/data/write_parquet_from_calib_pts.py
@@ -11,6 +11,9 @@
 from dotenv import load_dotenv
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 ######################################################################################################
 #                                                                                                    #
 #    Overview:                                                                                       #
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d8c074dbb..51488055a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,26 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.4.11.0 - 2024-02-16 - [PR#1077](https://github.com/NOAA-OWP/inundation-mapping/pull/1077)
+
+Replace `fiona` with `pyogrio` to improve I/O speed. `geopandas` will use `pyogrio` by default starting with version 1.0. `pyarrow` was also added as an environment variable to further speedup I/O. As a result of the changes in this PR, `fim_pipeline.sh` runs approximately 10% faster.
+
+### Changes
+
+- `Pipfile`: Upgraded `geopandas` from v0.12.2 to v0.14.3, added `pyogrio`, and fixed version of `pyflwdir`.
+- `src/bash_variables.env`: Added environment variable for `pyogrio` to use `pyarrow`
+- To all of the following files: Added `pyogrio` and `pyarrow`
+    - `data/`
+        - `bathymetry/preprocess_bathymetry.py`, `ble/ble_benchmark/create_flow_forecast_file.py`, `esri.py`, `nld/levee_download.py`, `usgs/acquire_and_preprocess_3dep_dems.py`, `wbd/clip_vectors_to_wbd.py`, `wbd/preprocess_wbd.py`, `write_parquet_from_calib_pts.py`
+    - `src/`
+        - `add_crosswalk.py`, `associate_levelpaths_with_levees.py`, `bathy_rc_adjust.py`, `bathymetric_adjustment.py`, `buffer_stream_branches.py`, `build_stream_traversal.py`, `crosswalk_nwm_demDerived.py`, `derive_headwaters.py`, `derive_level_paths.py`, `edit_points.py`, `filter_catchments_and_add_attributes.py`, `finalize_srcs.py`, `make_stages_and_catchlist.py`, `mask_dem.py`, `reachID_grid_to_vector_points.py`, `split_flows.py`, `src_adjust_spatial_obs.py`, `stream_branches.py`, `subset_catch_list_by_branch_id.py`, `usgs_gage_crosswalk.py`, `usgs_gage_unit_setup.py`, `utils/shared_functions.py`
+    - `tools/`
+        - `adjust_rc_with_feedback.py`, `check_deep_flooding.py`, `create_flow_forecast_file.py`, `eval_plots.py`, `evaluate_continuity.py`, `evaluate_crosswalk.py`, `fimr_to_benchmark.py`, `find_max_catchment_breadth.py`, `generate_categorical_fim.py`, `generate_categorical_fim_flows.py`, `generate_categorical_fim_mapping.py`, `generate_nws_lid.py`, `hash_compare.py`, `inundate_events.py`, `inundation.py`, `make_boxes_from_bounds.py`, `mosaic_inundation.py`, `overlapping_inundation.py`, `rating_curve_comparison.py`, `rating_curve_get_usgs_curves.py`, `test_case_by_hydro_id.py`, `tools_shared_functions.py`
+        
+<br/><br/>
+
+
 ## v4.4.10.1 - 2024-02-16 - [PR#1075](https://github.com/NOAA-OWP/inundation-mapping/pull/1075)
 
 We recently added code to fim_pre_processing.sh that checks the CPU count. Earlier this test was being done in post-processing and was killing a pipeline that had already been running for a while.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index b10453f61..2331e4c65 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -37,10 +37,10 @@ def add_crosswalk(
     min_stream_length,
     calibration_mode=False,
 ):
-    input_catchments = gpd.read_file(input_catchments_fileName)
-    input_flows = gpd.read_file(input_flows_fileName)
-    input_huc = gpd.read_file(input_huc_fileName)
-    input_nwmflows = gpd.read_file(input_nwmflows_fileName)
+    input_catchments = gpd.read_file(input_catchments_fileName, engine="pyogrio", use_arrow=True)
+    input_flows = gpd.read_file(input_flows_fileName, engine="pyogrio", use_arrow=True)
+    input_huc = gpd.read_file(input_huc_fileName, engine="pyogrio", use_arrow=True)
+    input_nwmflows = gpd.read_file(input_nwmflows_fileName, engine="pyogrio", use_arrow=True)
     min_catchment_area = float(min_catchment_area)  # 0.25#
     min_stream_length = float(min_stream_length)  # 0.5#
 
@@ -87,7 +87,7 @@ def add_crosswalk(
 
     elif (extent == 'MS') | (extent == 'GMS'):
         ## crosswalk using stream segment midpoint method
-        input_nwmcat = gpd.read_file(input_nwmcat_fileName, mask=input_huc)
+        input_nwmcat = gpd.read_file(input_nwmcat_fileName, mask=input_huc, engine="fiona")
 
         # only reduce nwm catchments to mainstems if running mainstems
         if extent == 'MS':
diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py
index 181a41d99..29c069ba4 100644
--- a/src/associate_levelpaths_with_levees.py
+++ b/src/associate_levelpaths_with_levees.py
@@ -9,6 +9,9 @@
 import pandas as pd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def associate_levelpaths_with_levees(
     levees_filename: str,
     levee_id_attribute: str,
diff --git a/src/bash_variables.env b/src/bash_variables.env
index a853a78eb..c62f2ef7a 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -39,6 +39,9 @@ export                  fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_da
 # Input file location with HUC, nwm feature_id and manual calibration coefficients
 export                     man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv
 
+# Use pyarrow
+export PYOGRIO_USE_ARROW=1
+
 
 # Styling
 export startDiv="\n-----------------------------------------------------------------\n"
diff --git a/src/bathy_rc_adjust.py b/src/bathy_rc_adjust.py
index c99fc3ebe..b3c408e44 100755
--- a/src/bathy_rc_adjust.py
+++ b/src/bathy_rc_adjust.py
@@ -7,6 +7,9 @@
 import pandas as pd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 sa_ratio_flag = float(environ['surf_area_thalweg_ratio_flag'])  # 10x
 thal_stg_limit = float(environ['thalweg_stg_search_max_limit'])  # 3m
 bankful_xs_ratio_flag = float(environ['bankful_xs_area_ratio_flag'])  # 10x
diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py
index a102d0a80..372f60038 100644
--- a/src/bathymetric_adjustment.py
+++ b/src/bathymetric_adjustment.py
@@ -40,7 +40,7 @@ def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
 
     # Load wbd and use it as a mask to pull the bathymetry data
     fim_huc_dir = join(fim_dir, huc)
-    wbd8_clp = gpd.read_file(join(fim_huc_dir, 'wbd8_clp.gpkg'))
+    wbd8_clp = gpd.read_file(join(fim_huc_dir, 'wbd8_clp.gpkg'), engine="pyogrio", use_arrow=True)
     bathy_data = gpd.read_file(bathy_file, mask=wbd8_clp)
     bathy_data = bathy_data.rename(columns={'ID': 'feature_id'})
 
@@ -171,9 +171,11 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
     # NOTE: This block can be removed if we have estimated bathymetry data for
     # the whole domain later.
     fim_hucs = [h for h in os.listdir(fim_dir) if re.match(r'\d{8}', h)]
-    bathy_gdf = gpd.read_file(bathy_file)
+    bathy_gdf = gpd.read_file(bathy_file, engine="pyogrio", use_arrow=True)
     buffered_bathy = bathy_gdf.geometry.buffer(wbd_buffer)  # We buffer the bathymetric data to get adjacent
-    wbd = gpd.read_file(wbd, mask=buffered_bathy)  # HUCs that could also have bathymetric reaches included
+    wbd = gpd.read_file(
+        wbd, mask=buffered_bathy, engine="fiona"
+    )  # HUCs that could also have bathymetric reaches included
     hucs_with_bathy = wbd.HUC8.to_list()
     hucs = [h for h in fim_hucs if h in hucs_with_bathy]
     log_file.write(f"Identified {len(hucs)} HUCs that have bathymetric data: {hucs}\n")
diff --git a/src/buffer_stream_branches.py b/src/buffer_stream_branches.py
index 6d4e29724..3c877af88 100755
--- a/src/buffer_stream_branches.py
+++ b/src/buffer_stream_branches.py
@@ -8,6 +8,9 @@
 from stream_branches import StreamBranchPolygons, StreamNetwork
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def buffer_stream_branches(
     streams_file: str,
     branch_id_attribute: str,
diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py
index 590398c12..9c287807f 100644
--- a/src/build_stream_traversal.py
+++ b/src/build_stream_traversal.py
@@ -4,6 +4,9 @@
 import geopandas as gpd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 '''
 Description:
         This tool creates unique IDs for each segment and builds the To_Node, From_Node, and NextDownID
diff --git a/src/crosswalk_nwm_demDerived.py b/src/crosswalk_nwm_demDerived.py
index 4ef4469b0..f30d5b3c7 100755
--- a/src/crosswalk_nwm_demDerived.py
+++ b/src/crosswalk_nwm_demDerived.py
@@ -12,6 +12,9 @@
 from utils.shared_variables import FIM_ID
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def Crosswalk_nwm_demDerived(
     nwm_streams,
     demDerived,
diff --git a/src/derive_headwaters.py b/src/derive_headwaters.py
index aabcedc66..19d9dcf76 100644
--- a/src/derive_headwaters.py
+++ b/src/derive_headwaters.py
@@ -8,6 +8,9 @@
 from utils.shared_functions import getDriver
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def findHeadWaterPoints(flows):
     flows = flows.explode(index_parts=True)
     headwater_points = []
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index 8fc1102dc..ca25819dc 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -10,6 +10,9 @@
 from utils.fim_enums import FIM_exit_codes
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def Derive_level_paths(
     in_stream_network,
     wbd,
diff --git a/src/edit_points.py b/src/edit_points.py
index a6f078df5..2a91a3471 100755
--- a/src/edit_points.py
+++ b/src/edit_points.py
@@ -5,6 +5,9 @@
 import geopandas as gpd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def Edit_points(
     stream_reaches,
     branch_id_attribute,
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index bef42fd35..d7e35444c 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -10,6 +10,9 @@
 from utils.shared_variables import FIM_ID
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def filter_catchments_and_add_attributes(
     input_catchments_filename,
     input_flows_filename,
diff --git a/src/finalize_srcs.py b/src/finalize_srcs.py
index 33899aeb5..369a3a77d 100755
--- a/src/finalize_srcs.py
+++ b/src/finalize_srcs.py
@@ -1,13 +1,12 @@
 #!/usr/bin/env python3
 
 import argparse
-import json
 
 import geopandas as gpd
 import pandas as pd
-from numpy import unique
 
-from utils.shared_functions import getDriver
+
+gpd.options.io_engine = "pyogrio"
 
 
 def finalize_srcs(srcbase, srcfull, hydrotable, output_srcfull=None, output_hydrotable=None):
diff --git a/src/make_stages_and_catchlist.py b/src/make_stages_and_catchlist.py
index c8007ed19..39d700c09 100755
--- a/src/make_stages_and_catchlist.py
+++ b/src/make_stages_and_catchlist.py
@@ -1,12 +1,14 @@
 #!/usr/bin/env python3
 
 import argparse
-import sys
 
 import geopandas as gpd
 import numpy as np
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def make_stages_and_catchlist(
     flows_filename,
     catchments_filename,
diff --git a/src/mask_dem.py b/src/mask_dem.py
index bf8897ec7..f551c9df4 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -11,6 +11,9 @@
 from rasterio.mask import mask
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def mask_dem(
     dem_filename: str,
     nld_filename: str,
diff --git a/src/reachID_grid_to_vector_points.py b/src/reachID_grid_to_vector_points.py
index 912bc719f..08e051b61 100755
--- a/src/reachID_grid_to_vector_points.py
+++ b/src/reachID_grid_to_vector_points.py
@@ -11,6 +11,9 @@
 from utils.shared_variables import PREP_PROJECTION
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def convert_grid_cells_to_points(raster, index_option, output_points_filename=False):
     # Input raster
     if isinstance(raster, str):
diff --git a/src/split_flows.py b/src/split_flows.py
index 5c47d4ae2..575b473c0 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -29,6 +29,9 @@
 from utils.shared_variables import FIM_ID
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def split_flows(
     flows_filename,
     dem_filename,
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index 8b39dec45..fba11571f 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -20,6 +20,9 @@
 )
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 # Import variables from .env file
 load_dotenv('/foss_fim/src/bash_variables.env')
 outputsDir = os.getenv("outputsDir")
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index fd8d502aa..bcd860a13 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -16,6 +16,9 @@
 from utils.shared_variables import DOWNSTREAM_THRESHOLD, ROUGHNESS_MAX_THRESH, ROUGHNESS_MIN_THRESH
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def update_rating_curve(
     fim_directory,
     water_edge_median_df,
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 849d6e92f..b9718ea91 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -22,8 +22,10 @@
 from utils.shared_variables import PREP_CRS
 
 
-class StreamNetwork(gpd.GeoDataFrame):
+gpd.options.io_engine = "pyogrio"
+
 
+class StreamNetwork(gpd.GeoDataFrame):
     """
     Notes:
         - Many of the methods support two attributes called branch_id_attribute and values_excluded.
diff --git a/src/subset_catch_list_by_branch_id.py b/src/subset_catch_list_by_branch_id.py
index bf80a9da9..0d7a774a0 100755
--- a/src/subset_catch_list_by_branch_id.py
+++ b/src/subset_catch_list_by_branch_id.py
@@ -9,6 +9,9 @@
 from stream_branches import StreamNetwork
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def Subset_catch_list(
     catch_list, stream_network, branch_id_attribute, branch_id_list=None, out_catch_list=None, verbose=False
 ):
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index b87dfc430..99d3fc29e 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -9,6 +9,8 @@
 import rasterio
 
 
+gpd.options.io_engine = "pyogrio"
+
 warnings.simplefilter("ignore")
 
 
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 61e82c797..a4f014fd5 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -13,6 +13,8 @@
 from utils.shared_variables import PREP_CRS
 
 
+gpd.options.io_engine = "pyogrio"
+
 warnings.simplefilter("ignore")
 
 
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index aacc3b687..8344d5baa 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -18,6 +18,9 @@
 import utils.shared_variables as sv
 
 
+gp.options.io_engine = "pyogrio"
+
+
 def getDriver(fileName):
     driverDictionary = {'.gpkg': 'GPKG', '.geojson': 'GeoJSON', '.shp': 'ESRI Shapefile'}
     driver = driverDictionary[splitext(fileName)[1]]
diff --git a/tools/adjust_rc_with_feedback.py b/tools/adjust_rc_with_feedback.py
index dfd16229e..88a2dab9c 100644
--- a/tools/adjust_rc_with_feedback.py
+++ b/tools/adjust_rc_with_feedback.py
@@ -12,6 +12,9 @@
 from geopandas.tools import sjoin
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 temp_workspace = r''
 HAND_CRS = 'EPSG:3857'
 
diff --git a/tools/check_deep_flooding.py b/tools/check_deep_flooding.py
index daf60aa58..5ed19511f 100644
--- a/tools/check_deep_flooding.py
+++ b/tools/check_deep_flooding.py
@@ -14,6 +14,9 @@
 from shapely.geometry import box
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def check_deep_flooding(args):
     depth_grid_path = args[0]
     shapefile_path = args[1]
diff --git a/tools/create_flow_forecast_file.py b/tools/create_flow_forecast_file.py
index 3512074e7..f716082dd 100755
--- a/tools/create_flow_forecast_file.py
+++ b/tools/create_flow_forecast_file.py
@@ -6,6 +6,9 @@
 import geopandas as gpd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def create_flow_forecast_file(
     ble_geodatabase,
     nwm_geodatabase,
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index 75773deec..d7ff97209 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -20,6 +20,9 @@
 from utils.shared_variables import VIZ_PROJECTION
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 # Get variables from .env file.
 load_dotenv()
 WBD_LAYER = os.getenv("WBD_LAYER")
diff --git a/tools/evaluate_continuity.py b/tools/evaluate_continuity.py
index 85baffeb9..3d86f063e 100644
--- a/tools/evaluate_continuity.py
+++ b/tools/evaluate_continuity.py
@@ -8,6 +8,9 @@
 import pandas as pd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def evaluate_continuity(
     stream_network_file, forecast_file, stream_network_outfile=None, confluences_only=False, plot_file=None
 ):
diff --git a/tools/evaluate_crosswalk.py b/tools/evaluate_crosswalk.py
index 0051c3600..6e0fe3db5 100644
--- a/tools/evaluate_crosswalk.py
+++ b/tools/evaluate_crosswalk.py
@@ -7,6 +7,9 @@
 import pandas as pd
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def evaluate_crosswalk(
     input_flows_fileName: str,
     input_nwmflows_fileName: str,
diff --git a/tools/fimr_to_benchmark.py b/tools/fimr_to_benchmark.py
index f10dac19a..9f3359bdc 100644
--- a/tools/fimr_to_benchmark.py
+++ b/tools/fimr_to_benchmark.py
@@ -13,6 +13,9 @@
 from tools_shared_functions import get_metadata
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 load_dotenv()
 
 
diff --git a/tools/find_max_catchment_breadth.py b/tools/find_max_catchment_breadth.py
index 0f7405399..600e79ec4 100755
--- a/tools/find_max_catchment_breadth.py
+++ b/tools/find_max_catchment_breadth.py
@@ -9,6 +9,9 @@
 from shapely.geometry import Point
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def Find_max_catchment_breadth(hydrofabric_dir):
     catchments_fileNames = glob_file_paths_for_catchments(hydrofabric_dir)
 
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index 3749bb4fe..e3d248be2 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -36,6 +36,9 @@
 from utils.shared_variables import VIZ_PROJECTION
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def process_generate_categorical_fim(
     fim_run_dir,
     job_number_huc,
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index faf539a0c..a89459e06 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -24,6 +24,9 @@
 from utils.shared_variables import VIZ_PROJECTION
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def get_env_paths():
     load_dotenv()
     # import variables from .env file
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index df44a4a52..9ff6cab18 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -19,6 +19,9 @@
 from utils.shared_variables import PREP_PROJECTION, VIZ_PROJECTION
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def generate_categorical_fim(
     fim_run_dir, source_flow_dir, output_catfim_dir, job_number_huc, job_number_inundate, depthtif, log_file
 ):
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
index 88ca96e60..ac769134b 100644
--- a/tools/generate_nws_lid.py
+++ b/tools/generate_nws_lid.py
@@ -14,6 +14,9 @@
 from utils.shared_variables import PREP_PROJECTION
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 load_dotenv()
 # import variables from .env file
 API_BASE_URL = os.getenv("API_BASE_URL")
diff --git a/tools/hash_compare.py b/tools/hash_compare.py
index 8de401be2..33c2dfad1 100755
--- a/tools/hash_compare.py
+++ b/tools/hash_compare.py
@@ -8,6 +8,9 @@
 from geopandas.testing import assert_geodataframe_equal
 
 
+geopandas.options.io_engine = "pyogrio"
+
+
 def main(arg1, arg2, image_only, log_file, gpkg):
     """
     This tool compares either directories or single files. It will create and compare a hashing for
diff --git a/tools/inundate_events.py b/tools/inundate_events.py
index 90d415a85..b8b33b614 100755
--- a/tools/inundate_events.py
+++ b/tools/inundate_events.py
@@ -11,6 +11,9 @@
 from tqdm import tqdm
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def inundate_events(hydrofabric_dir, forecast_file, inundation_file, inundation_polygon=None, jobs=1):
     forecast_df = pd.read_csv(
         forecast_file, infer_datetime_format=True, dtype={'huc': str}, parse_dates=['date_time']
diff --git a/tools/inundation.py b/tools/inundation.py
index b39d9d191..4dc7232c4 100755
--- a/tools/inundation.py
+++ b/tools/inundation.py
@@ -18,6 +18,9 @@
 from shapely.geometry import shape
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 class hydroTableHasOnlyLakes(Exception):
     """Raised when a Hydro-Table only has lakes"""
 
diff --git a/tools/make_boxes_from_bounds.py b/tools/make_boxes_from_bounds.py
index 2810cac37..9f0101d93 100755
--- a/tools/make_boxes_from_bounds.py
+++ b/tools/make_boxes_from_bounds.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-from datetime import datetime
 
 import geopandas as gpd
 import pandas as pd
@@ -10,6 +9,9 @@
 from utils.shared_functions import getDriver
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def find_hucs_of_bounding_boxes(
     bounding_boxes_file,
     wbd=None,
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 7baa7f5dc..4513f067f 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -158,6 +158,8 @@ def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon
     from shapely.geometry.multipolygon import MultiPolygon
     from shapely.geometry.polygon import Polygon
 
+    gpd.options.io_engine = "pyogrio"
+
     with rasterio.open(inundation_raster) as src:
         # Open inundation_raster using rasterio.
         image = src.read(1)
diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py
index d1ee54516..23c6575d8 100644
--- a/tools/overlapping_inundation.py
+++ b/tools/overlapping_inundation.py
@@ -16,6 +16,9 @@
 from scipy.optimize import newton
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 class OverlapWindowMerge:
     def __init__(self, inundation_rsts, num_partitions=None, window_xy_size=None):
         """
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 2df1c179b..58e316068 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -25,6 +25,9 @@
 from shapely.geometry import Polygon
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
 """
diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py
index 7068f4eb4..65553d4f2 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/tools/rating_curve_get_usgs_curves.py
@@ -29,6 +29,9 @@
 from utils.shared_variables import PREP_PROJECTION
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 '''
 This script calls the NOAA Tidal API for datum conversions. Experience shows that
     running script outside of business hours seems to be most consistent way
diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py
index 65ca47e5a..5b24fdd3a 100644
--- a/tools/test_case_by_hydro_id.py
+++ b/tools/test_case_by_hydro_id.py
@@ -12,6 +12,9 @@
 from tools_shared_functions import compute_stats_from_contingency_table
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 #####################################################
 # Perform zonal stats is a funtion stored in pixel_counter.py.
 # The input huc_gpkg is a single huc8 geopackage, the second input argument must be input as a dict.
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 96a7e24bd..4542cf879 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -22,6 +22,9 @@
 from shapely.geometry import MultiPolygon, Polygon, shape
 
 
+gpd.options.io_engine = "pyogrio"
+
+
 def get_env_paths():
     load_dotenv()
     # import variables from .env file

From bfba3792b9df3526251fce3b75086100b783a607 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 8 Mar 2024 13:11:38 -0700
Subject: [PATCH 301/359] v4.4.11.1 pyogrio bug fix (#1080)

---
 docs/CHANGELOG.md             | 11 ++++++++++-
 src/bathymetric_adjustment.py |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 51488055a..c605bc973 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.4.11.1 - 2024-03-08 - [PR#1080](https://github.com/NOAA-OWP/inundation-mapping/pull/1080)
+
+Fixes bug in bathymetric adjustment where `mask` is used with `geopandas.read_file`. The solution is to force `read_file` to use `fiona` instead of `pyogrio`.
+
+### Changes
+
+`src/bathymetric_adjustment.py`: Use `engine=fiona` instead of default `pyogrio` to use `mask=` with `geopandas.read_file`
+
+<br/><br/>
+
 ## v4.4.11.0 - 2024-02-16 - [PR#1077](https://github.com/NOAA-OWP/inundation-mapping/pull/1077)
 
 Replace `fiona` with `pyogrio` to improve I/O speed. `geopandas` will use `pyogrio` by default starting with version 1.0. `pyarrow` was also added as an environment variable to further speedup I/O. As a result of the changes in this PR, `fim_pipeline.sh` runs approximately 10% faster.
@@ -20,7 +30,6 @@ Replace `fiona` with `pyogrio` to improve I/O speed. `geopandas` will use `pyogr
         
 <br/><br/>
 
-
 ## v4.4.10.1 - 2024-02-16 - [PR#1075](https://github.com/NOAA-OWP/inundation-mapping/pull/1075)
 
 We recently added code to fim_pre_processing.sh that checks the CPU count. Earlier this test was being done in post-processing and was killing a pipeline that had already been running for a while.
diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py
index 372f60038..9c114173b 100644
--- a/src/bathymetric_adjustment.py
+++ b/src/bathymetric_adjustment.py
@@ -41,7 +41,7 @@ def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
     # Load wbd and use it as a mask to pull the bathymetry data
     fim_huc_dir = join(fim_dir, huc)
     wbd8_clp = gpd.read_file(join(fim_huc_dir, 'wbd8_clp.gpkg'), engine="pyogrio", use_arrow=True)
-    bathy_data = gpd.read_file(bathy_file, mask=wbd8_clp)
+    bathy_data = gpd.read_file(bathy_file, mask=wbd8_clp, engine="fiona")
     bathy_data = bathy_data.rename(columns={'ID': 'feature_id'})
 
     # Get src_full from each branch

From 23d2d6de0060f83bea5d5eb50e082538ebd07fea Mon Sep 17 00:00:00 2001
From: EmilyDeardorff-NOAA <60829052+EmilyDeardorff@users.noreply.github.com>
Date: Mon, 11 Mar 2024 12:35:12 -0700
Subject: [PATCH 302/359] v4.4.12.0 Add South Alaska to pre-clip vectors system
 (#1078)

---
 data/wbd/clip_vectors_to_wbd.py        |  54 +++------
 data/wbd/generate_pre_clip_fim_huc8.py | 157 +++++++++++++++++--------
 docs/CHANGELOG.md                      |  20 ++++
 src/bash_variables.env                 |  41 ++++---
 4 files changed, 172 insertions(+), 100 deletions(-)

diff --git a/data/wbd/clip_vectors_to_wbd.py b/data/wbd/clip_vectors_to_wbd.py
index 2425bfc45..ede7a6526 100755
--- a/data/wbd/clip_vectors_to_wbd.py
+++ b/data/wbd/clip_vectors_to_wbd.py
@@ -9,7 +9,9 @@
 from shapely.geometry import MultiPolygon, Polygon
 
 from utils.shared_functions import getDriver
-from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
+
+
+# from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
 
 
 def subset_vector_layers(
@@ -36,6 +38,7 @@ def subset_vector_layers(
     wbd_buffer_distance,
     levee_protected_areas,
     subset_levee_protected_areas,
+    huc_CRS,
 ):
     print(f"Getting Cell Size for {hucCode}", flush=True)
     with rio.open(dem_filename) as dem_raster:
@@ -56,26 +59,16 @@ def subset_vector_layers(
 
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
-    wbd_buffer.to_file(
-        wbd_buffer_filename,
-        driver=getDriver(wbd_buffer_filename),
-        index=False,
-        crs=DEFAULT_FIM_PROJECTION_CRS,
-    )
+    wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False, crs=huc_CRS)
     wbd_streams_buffer.to_file(
-        wbd_streams_buffer_filename,
-        driver=getDriver(wbd_streams_buffer_filename),
-        index=False,
-        crs=DEFAULT_FIM_PROJECTION_CRS,
+        wbd_streams_buffer_filename, driver=getDriver(wbd_streams_buffer_filename), index=False, crs=huc_CRS
     )
 
     # Clip ocean water polygon for future masking ocean areas (where applicable)
     landsea = gpd.read_file(landsea, mask=wbd_buffer)
     if not landsea.empty:
         print(f"Create landsea gpkg for {hucCode}", flush=True)
-        landsea.to_file(
-            subset_landsea, driver=getDriver(subset_landsea), index=False, crs=DEFAULT_FIM_PROJECTION_CRS
-        )
+        landsea.to_file(subset_landsea, driver=getDriver(subset_landsea), index=False, crs=huc_CRS)
     del landsea
 
     # Clip levee-protected areas polygons for future masking ocean areas (where applicable)
@@ -86,7 +79,7 @@ def subset_vector_layers(
             subset_levee_protected_areas,
             driver=getDriver(subset_levee_protected_areas),
             index=False,
-            crs=DEFAULT_FIM_PROJECTION_CRS,
+            crs=huc_CRS,
         )
     del levee_protected_areas
 
@@ -104,28 +97,24 @@ def subset_vector_layers(
         # Loop through the filled polygons and insert the new geometry
         for i in range(len(nwm_lakes_fill_holes.geoms)):
             nwm_lakes.loc[i, 'geometry'] = nwm_lakes_fill_holes.geoms[i]
-        nwm_lakes.to_file(
-            subset_nwm_lakes, driver=getDriver(subset_nwm_lakes), index=False, crs=DEFAULT_FIM_PROJECTION_CRS
-        )
+        nwm_lakes.to_file(subset_nwm_lakes, driver=getDriver(subset_nwm_lakes), index=False, crs=huc_CRS)
     del nwm_lakes
 
     # Find intersecting levee lines
     print(f"Subsetting NLD levee lines for {hucCode}", flush=True)
     nld_lines = gpd.read_file(nld_lines, mask=wbd_buffer)
     if not nld_lines.empty:
-        nld_lines.to_file(
-            subset_nld_lines, driver=getDriver(subset_nld_lines), index=False, crs=DEFAULT_FIM_PROJECTION_CRS
-        )
+        nld_lines.to_file(subset_nld_lines, driver=getDriver(subset_nld_lines), index=False, crs=huc_CRS)
     del nld_lines
 
-    # Preprocced levee lines for burning
+    # Preprocessed levee lines for burning
     nld_lines_preprocessed = gpd.read_file(nld_lines_preprocessed, mask=wbd_buffer)
     if not nld_lines_preprocessed.empty:
         nld_lines_preprocessed.to_file(
             subset_nld_lines_preprocessed,
             driver=getDriver(subset_nld_lines_preprocessed),
             index=False,
-            crs=DEFAULT_FIM_PROJECTION_CRS,
+            crs=huc_CRS,
         )
     del nld_lines_preprocessed
 
@@ -135,14 +124,12 @@ def subset_vector_layers(
 
     if len(nwm_headwaters) > 0:
         nwm_headwaters.to_file(
-            subset_nwm_headwaters,
-            driver=getDriver(subset_nwm_headwaters),
-            index=False,
-            crs=DEFAULT_FIM_PROJECTION_CRS,
+            subset_nwm_headwaters, driver=getDriver(subset_nwm_headwaters), index=False, crs=huc_CRS
         )
     else:
         print("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
+
     del nwm_headwaters
 
     # Find intersecting nwm_catchments
@@ -151,14 +138,12 @@ def subset_vector_layers(
 
     if len(nwm_catchments) > 0:
         nwm_catchments.to_file(
-            subset_nwm_catchments,
-            driver=getDriver(subset_nwm_catchments),
-            index=False,
-            crs=DEFAULT_FIM_PROJECTION_CRS,
+            subset_nwm_catchments, driver=getDriver(subset_nwm_catchments), index=False, crs=huc_CRS
         )
     else:
         print("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
+
     del nwm_catchments
 
     # Subset nwm streams
@@ -192,10 +177,7 @@ def subset_vector_layers(
         nwm_streams = pd.concat([nwm_streams_nonoutlets, nwm_streams_outlets])
 
         nwm_streams.to_file(
-            subset_nwm_streams,
-            driver=getDriver(subset_nwm_streams),
-            index=False,
-            crs=DEFAULT_FIM_PROJECTION_CRS,
+            subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False, crs=huc_CRS
         )
     else:
         print("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
@@ -248,6 +230,8 @@ def subset_vector_layers(
         '-lps', '--subset-levee-protected-areas', help='Levee-protected areas subset', required=True
     )
 
+    parser.add_argument('-crs', '--huc-crs', help='HUC crs', required=True)
+
     args = vars(parser.parse_args())
 
     subset_vector_layers(**args)
diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
index 71dbbc9b4..ca6ee44d8 100755
--- a/data/wbd/generate_pre_clip_fim_huc8.py
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -27,7 +27,6 @@
 
     Usage:
         generate_pre_clip_fim_huc8.py
-            -wbd /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
             -n /data/inputs/pre_clip_huc8/24_3_20
             -u /data/inputs/huc_lists/included_huc8.lst
             -j 6
@@ -48,18 +47,37 @@
 
 # Variables from src/bash_variables.env
 DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
+ALASKA_CRS = os.getenv('ALASKA_CRS')  # alaska
+
 inputsDir = os.getenv('inputsDir')
+
 input_WBD_gdb = os.getenv('input_WBD_gdb')
+input_WBD_gdb_Alaska = os.getenv('input_WBD_gdb_Alaska')  # alaska
+
 input_DEM = os.getenv('input_DEM')
+input_DEM_Alaska = os.getenv('input_DEM_Alaska')  # alaska
+
 input_DEM_domain = os.getenv('input_DEM_domain')
+input_DEM_domain_Alaska = os.getenv('input_DEM_domain_Alaska')  # alaska
+
 input_nwm_lakes = os.getenv('input_nwm_lakes')
 input_nwm_catchments = os.getenv('input_nwm_catchments')
+input_nwm_catchments_Alaska = os.getenv('input_nwm_catchments_Alaska')
+
 input_NLD = os.getenv('input_NLD')
+input_NLD_Alaska = os.getenv('input_NLD_Alaska')
+
 input_levees_preprocessed = os.getenv('input_levees_preprocessed')
+input_levees_preprocessed_Alaska = os.getenv('input_levees_preprocessed_Alaska')
+
 input_GL_boundaries = os.getenv('input_GL_boundaries')
 input_nwm_flows = os.getenv('input_nwm_flows')
+input_nwm_flows_Alaska = os.getenv('input_nwm_flows_Alaska')  # alaska
 input_nwm_headwaters = os.getenv('input_nwm_headwaters')
+input_nwm_headwaters_Alaska = os.getenv('input_nwm_headwaters_Alaska')
+
 input_nld_levee_protected_areas = os.getenv('input_nld_levee_protected_areas')
+input_nld_levee_protected_areas_Alaska = os.getenv('input_nld_levee_protected_areas_Alaska')
 
 # Variables from config/params_template.env
 wbd_buffer = os.getenv('wbd_buffer')
@@ -100,13 +118,12 @@ def __setup_logger(outputs_dir):
     logging.info(f"\n \t Started: {start_time_string} \n")
 
 
-def pre_clip_hucs_from_wbd(wbd_file, outputs_dir, huc_list, number_of_jobs, overwrite):
+def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     '''
     The function is the main driver of the program to iterate and parallelize writing
     pre-clipped HUC8 vector files.
 
     Inputs:
-    - wbd_file:                       Default take from src/bash_variables.env, or provided as argument.
     - outputs_dir:                    Output directory to stage pre-clipped vectors.
     - huc_list:                       List of Hucs to generate pre-clipped .gpkg files.
     - number_of_jobs:                 Amount of cpus used for parallelization.
@@ -143,8 +160,8 @@ def pre_clip_hucs_from_wbd(wbd_file, outputs_dir, huc_list, number_of_jobs, over
     if os.path.exists(huc_list):
         hucs_to_pre_clip_list = open(huc_list).read().splitlines()
     else:
-        logging.info("The huclist is not valid. Please check <huc_list> arguemnt.")
-        raise Exception("The huclist is not valid. Please check <huc_list> arguemnt.")
+        logging.info("The huclist is not valid. Please check <huc_list> argument.")
+        raise Exception("The huclist is not valid. Please check <huc_list> argument.")
 
     if os.path.exists(outputs_dir) and not overwrite:
         raise Exception(
@@ -175,7 +192,9 @@ def pre_clip_hucs_from_wbd(wbd_file, outputs_dir, huc_list, number_of_jobs, over
     procs_list = []
     for huc in hucs_to_pre_clip_list:
         print(f"Generating vectors for {huc}. ")
-        procs_list.append([huc, outputs_dir, wbd_file])
+        procs_list.append([huc, outputs_dir])
+
+        # procs_list.append([huc, outputs_dir, wbd_alaska_file])
 
     # Parallelize each huc in hucs_to_parquet_list
     logging.info('Parallelizing HUC level wbd pre-clip vector creation. ')
@@ -210,7 +229,6 @@ def huc_level_clip_vectors_to_wbd(args):
     Inputs:
     - huc:                           Individual HUC to generate vector files for.
     - outputs_dir:                   Output directory to stage pre-clipped vectors.
-    - input_WBD_filename:            Filename of WBD to generate pre-clipped .gpkg files.
 
     Processing:
     - Define (unpack) arguments.
@@ -227,19 +245,32 @@ def huc_level_clip_vectors_to_wbd(args):
     # We have to explicitly unpack the args from pool.map()
     huc = args[0]
     outputs_dir = args[1]
-    input_WBD_filename = args[2]
 
     huc_directory = os.path.join(outputs_dir, huc)
 
     # SET VARIABLES AND FILE INPUTS #
     hucUnitLength = len(huc)
     huc2Identifier = huc[:2]
-    input_NHD_WBHD_layer = f"WBDHU{hucUnitLength}"
+
+    # Check whether the HUC is in Alaska or not and assign the CRS and filenames accordingly
+    if huc2Identifier == '19':
+        huc_CRS = ALASKA_CRS
+        input_NHD_WBHD_layer = 'WBD_National_South_Alaska'
+        input_WBD_filename = input_WBD_gdb_Alaska
+        wbd_gpkg_path = f'{inputsDir}/wbd/WBD_National_South_Alaska.gpkg'
+    else:
+        huc_CRS = DEFAULT_FIM_PROJECTION_CRS
+        input_NHD_WBHD_layer = f"WBDHU{hucUnitLength}"
+        input_WBD_filename = input_WBD_gdb
+        wbd_gpkg_path = f'{inputsDir}/wbd/WBD_National.gpkg'
 
     # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
     if huc2Identifier == "04":
         input_LANDSEA = f"{input_GL_boundaries}"
         print(f'Using {input_LANDSEA} for water body mask (Great Lakes)')
+    elif huc2Identifier == "19":
+        input_LANDSEA = f"{inputsDir}/landsea/water_polygons_alaska.gpkg"
+        print(f'Using {input_LANDSEA} for water body mask (Alaska)')
     else:
         input_LANDSEA = f"{inputsDir}/landsea/water_polygons_us.gpkg"
 
@@ -252,7 +283,7 @@ def huc_level_clip_vectors_to_wbd(args):
             '-f',
             'GPKG',
             '-t_srs',
-            DEFAULT_FIM_PROJECTION_CRS,
+            huc_CRS,
             f'{huc_directory}/wbd.gpkg',
             input_WBD_filename,
             input_NHD_WBHD_layer,
@@ -286,32 +317,64 @@ def huc_level_clip_vectors_to_wbd(args):
     print(msg)
     logging.info(msg)
 
-    # Subset Vector Layers
-    subset_vector_layers(
-        subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
-        subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
-        hucCode=huc,
-        subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
-        wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
-        wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
-        wbd_filename=f"{huc_directory}/wbd.gpkg",
-        dem_filename=input_DEM,
-        dem_domain=input_DEM_domain,
-        nwm_lakes=input_nwm_lakes,
-        nwm_catchments=input_nwm_catchments,
-        subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
-        nld_lines=input_NLD,
-        nld_lines_preprocessed=input_levees_preprocessed,
-        landsea=input_LANDSEA,
-        nwm_streams=input_nwm_flows,
-        subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
-        nwm_headwaters=input_nwm_headwaters,
-        subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
-        subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
-        wbd_buffer_distance=wbd_buffer_int,
-        levee_protected_areas=input_nld_levee_protected_areas,
-        subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
-    )
+    # Subset Vector Layers (after determining whether it's alaska or not)
+    if huc2Identifier == '19':
+        # Yes Alaska
+        subset_vector_layers(
+            subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
+            subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
+            hucCode=huc,
+            subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
+            wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
+            wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
+            wbd_filename=f"{huc_directory}/wbd.gpkg",
+            dem_filename=input_DEM_Alaska,
+            dem_domain=input_DEM_domain_Alaska,
+            nwm_lakes=input_nwm_lakes,
+            nwm_catchments=input_nwm_catchments_Alaska,
+            subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
+            nld_lines=input_NLD_Alaska,
+            nld_lines_preprocessed=input_levees_preprocessed_Alaska,
+            landsea=input_LANDSEA,
+            nwm_streams=input_nwm_flows_Alaska,
+            subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
+            nwm_headwaters=input_nwm_headwaters_Alaska,
+            subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
+            subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
+            wbd_buffer_distance=wbd_buffer_int,
+            levee_protected_areas=input_nld_levee_protected_areas_Alaska,
+            subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
+            huc_CRS=huc_CRS,  # TODO: simplify
+        )
+
+    else:
+        # Not Alaska
+        subset_vector_layers(
+            subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
+            subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
+            hucCode=huc,
+            subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
+            wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
+            wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
+            wbd_filename=f"{huc_directory}/wbd.gpkg",
+            dem_filename=input_DEM,
+            dem_domain=input_DEM_domain,
+            nwm_lakes=input_nwm_lakes,
+            nwm_catchments=input_nwm_catchments,
+            subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
+            nld_lines=input_NLD,
+            nld_lines_preprocessed=input_levees_preprocessed,
+            landsea=input_LANDSEA,
+            nwm_streams=input_nwm_flows,
+            subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
+            nwm_headwaters=input_nwm_headwaters,
+            subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
+            subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
+            wbd_buffer_distance=wbd_buffer_int,
+            levee_protected_areas=input_nld_levee_protected_areas,
+            subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
+            huc_CRS=huc_CRS,  # TODO: simplify
+        )
 
     msg = f"\n\t Completing Get Vector Layers and Subset: {huc} \n"
     print(msg)
@@ -326,12 +389,12 @@ def huc_level_clip_vectors_to_wbd(args):
             '-f',
             'GPKG',
             '-t_srs',
-            DEFAULT_FIM_PROJECTION_CRS,
+            huc_CRS,
             '-clipsrc',
             f'{huc_directory}/wbd_buffered.gpkg',
             f'{huc_directory}/wbd8_clp.gpkg',
-            f'{inputsDir}/wbd/WBD_National.gpkg',
-            'WBDHU8',
+            wbd_gpkg_path,
+            input_NHD_WBHD_layer,
         ],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
@@ -364,30 +427,24 @@ def huc_level_clip_vectors_to_wbd(args):
         'the output directory specified as the <outputs_dir> argument.',
         usage='''
             ./generate_pre_clip_fim_huc8.py
-                -wbd /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
-                -n /data/inputs/pre_clip_huc8/24_3_20
+                -n /data/inputs/pre_clip_huc8/2024_3_20
                 -u /data/inputs/huc_lists/included_huc8.lst
                 -j 6
                 -o
         ''',
     )
-    parser.add_argument(
-        '-wbd',
-        '--wbd_file',
-        help='.wbd file to clip into individual HUC.gpkg files. Default is $input_WBD_gdb from src/bash_variables.env.',
-        default=input_WBD_gdb,
-    )
+
     parser.add_argument(
         '-n',
         '--outputs_dir',
         help='Directory to output all of the HUC level .gpkg files. Use the format: '
-        '<year_month_day> (i.e. September 26, 2023 would be 23_9_26)',
+        '<year_month_day> (i.e. September 26, 2023 would be 2023_9_26)',
     )
     parser.add_argument('-u', '--huc_list', help='List of HUCs to genereate pre-clipped vectors for.')
     parser.add_argument(
         '-j',
         '--number_of_jobs',
-        help='OPTIONAL: number of cores/processes (default=4). This is a memory intensive '
+        help='OPTIONAL: Number of cores/processes (default=4). This is a memory intensive '
         'script, and the multiprocessing will crash if too many CPUs are used. It is recommended to provide '
         'half the amount of available CPUs.',
         type=int,
@@ -397,7 +454,7 @@ def huc_level_clip_vectors_to_wbd(args):
     parser.add_argument(
         '-o',
         '--overwrite',
-        help='Overwrite the file if already existing? (default false)',
+        help='OPTIONAL: Overwrite the file if already existing? (default false)',
         action='store_true',
     )
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c605bc973..ea1f51669 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,26 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.4.12.0 - 2024-03-11 - [PR#1078](https://github.com/NOAA-OWP/inundation-mapping/pull/1078)
+
+Resolves issue #1033 by adding Alaska-specific data to the FIM input folders and updating the pre-clip vector process to use the proper data and CRS when an Alaska HUC is detected. The `-wbd` flag was removed from the optional arguments of `generate_pre_clip_fim_huc8`. The WBD file path will now only be sourced from the `bash_variables.env` file. The `bash_variables.env` file has been updated to include the new Alaska-specific FIM input files.
+
+### Changes
+
+- `/data/wbd/`
+    - `clip_vectors_to_wbd.py`: Replaced all CRS inputs with the `huc_CRS` variable, which is input based on whether the HUC is Alaska or CONUS. Previously, the default FIM projection was automatically assigned as the CRS (which had been retrieved from `utils.shared_variables`).
+
+    - `generate_pre_clip_fim_huc8.py`:
+        - Added Alaska projection and links to the new Alaska data file paths that were added to `bash_variables.env`.
+        - Removed the `wbd` argument from the `pre_clip_hucs_from_wbd` function and made it so that the code gets the WBD path from `bash_variables.env`.
+        - Added logic to check whether the HUC is in Alaska and, if so, use the Alaska-specific HUC and input file paths.
+        - Cleaned up the spelling and formatting of some comments
+- `/src/`
+    - `bash_variables.env`: Added the Alaska-specific projection (EPSG:3338) and file paths for Alaska-specific data (see data changelog for list of new input data)
+
+<br/><br/>
+
+
 ## v4.4.11.1 - 2024-03-08 - [PR#1080](https://github.com/NOAA-OWP/inundation-mapping/pull/1080)
 
 Fixes bug in bathymetric adjustment where `mask` is used with `geopandas.read_file`. The solution is to force `read_file` to use `fiona` instead of `pyogrio`.
diff --git a/src/bash_variables.env b/src/bash_variables.env
index c62f2ef7a..d8d2237b6 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -1,21 +1,32 @@
 ## Define inputs
 
 # NOTE: $inputsDir is defined in Dockerfile
-export        DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
-export                         input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
-export                  input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
-export               input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
-export                         input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
-export         input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg
-export   input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
-export              input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg
-export                   input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
-export              input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
-export                   input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
-export                     input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
-export            input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
-export                  pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
-export                   bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
+export             DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
+export                             ALASKA_CRS=EPSG:3338
+export                              input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
+export                       input_DEM_Alaska=${inputsDir}/3dep_dems/10m_South_Alaska/23_11_07/FIM_3dep_dem_South_Alask_10m.vrt
+export                       input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
+export                input_DEM_domain_Alaska=${inputsDir}/3dep_dems/10m_South_Alaska/23_11_07/DEM_Domain.gpkg
+export                    input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
+export                              input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
+export                       input_NLD_Alaska=${inputsDir}/nld_vectors/System_Routes_NLDFS_3338_230314.gpkg
+export              input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg
+export       input_levees_preprocessed_Alaska=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314_3338.gpkg
+export        input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
+export input_nld_levee_protected_areas_Alaska=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_3338_230314.gpkg
+export                   input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg
+export            input_nwm_catchments_Alaska=${inputsDir}/nwm_hydrofabric/nwm_catchments_alaska.gpkg
+export                        input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
+export                 input_nwm_flows_Alaska=${inputsDir}/nwm_hydrofabric/nwm_flows_alaska_nwmV3_ID.gpkg
+export                   input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
+export            input_nwm_headwaters_Alaska=${inputsDir}/nwm_hydrofabric/nwm_headwaters_alaska.gpkg
+export                        input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
+export                  input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_waterbodies_alaska.gpkg
+export                          input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
+export                   input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg
+export                 input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
+export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
+export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
 export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv

From 4907da22ae80a2f114bab717d338893a03c44a57 Mon Sep 17 00:00:00 2001
From: EmilyDeardorff-NOAA <60829052+EmilyDeardorff@users.noreply.github.com>
Date: Mon, 11 Mar 2024 12:38:45 -0700
Subject: [PATCH 303/359] v4.4.13.0 Remove GMS branch outlet backpools (#1006)

---
 docs/CHANGELOG.md                        |  22 +-
 src/delineate_hydros_and_produce_HAND.sh |  22 +-
 src/mitigate_branch_outlet_backpool.py   | 543 +++++++++++++++++++++++
 src/split_flows.py                       | 185 ++++++--
 4 files changed, 720 insertions(+), 52 deletions(-)
 create mode 100755 src/mitigate_branch_outlet_backpool.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ea1f51669..42df9c18e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,27 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.13.0 - 2024-03-11 - [PR#1006](https://github.com/NOAA-OWP/inundation-mapping/pull/1006)
+
+Adds a new module that mitigates the branch outlet backpool error. In some HUCs, an overly-large catchment appears at the outlet of the branch (as in issue #985) which causes an artificially large amount of water to get routed to the smaller stream instead of the main stem. This issue is mitigated by trimming the levelpath just above the outlet and removing the offending pixel catchment from the pixel catchments and catchment reaches files. 
+
+The branch outlet backpool issue is identified based on two criteria: 
+  1. There is a pixel catchment that is abnormally large (more than two standard deviations above the mean.)
+  2. The abnormally-large pixel catchment occurs at the outlet of the levelpath.
+
+If both criteria are met for a branch, then the issue is mitigated by trimming the flowline to the third-to-last point.
+
+### Additions
+
+- `src/mitigate_branch_outlet_backpool.py`: Detects and mitigates the branch outlet backpool error. If both branch outlet backpool criteria are met, the snapped point is set to be the penultimate vertex and then the flowline is trimmed to that point (instead of the last point). Trims the `gw_catchments_pixels_<id>.tif` and `gw_catchments_reaches_<id>.tif` rasters by using `gdal_polygonize.py` to polygonize the `gw_pixel_catchments_<id>.tif` file, creating a mask that excludes the problematic pixel catchment, and then using that mask to trim the pixel catchment and catchment reaches rasters.
+
+### Changes
+
+- `src/delineate_hydros_and_produce_HAND.sh`: Adds the `mitigate_branch_outlet_backpool.py` module to run after the  `Gage Watershed for Pixels` step. 
+- `src/split_flows.py`: Improves documentation and readability.
+
+<br/><br/>
+
 
 ## v4.4.12.0 - 2024-03-11 - [PR#1078](https://github.com/NOAA-OWP/inundation-mapping/pull/1078)
 
@@ -180,7 +201,6 @@ The alpha test for v4.4.8.1 came back with a large degradation in skill and we n
 
 <br/><br/>
 
-
 ## v4.4.8.1 - 2023-12-08 - [PR#1047](https://github.com/NOAA-OWP/inundation-mapping/pull/1047)
 
 Upgrades JDK to v.17.0.9 in Docker image to address security vulnerabilities.
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 276da3741..527ae54d2 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -97,7 +97,8 @@ $srcDir/split_flows.py -f $tempCurrentBranchDataDir/demDerived_reaches_$current_
     -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
     -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg \
     -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg \
-    -w $tempHucDataDir/wbd8_clp.gpkg -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg \
+    -w $tempHucDataDir/wbd8_clp.gpkg \
+    -l $tempHucDataDir/nwm_lakes_proj_subset.gpkg \
     -n $b_arg \
     -m $max_split_distance_meters \
     -t $slope_min \
@@ -126,7 +127,24 @@ mpiexec -n $ncores_gw $taudemDir/gagewatershed \
     -o $tempCurrentBranchDataDir/flows_points_pixels_$current_branch_id.gpkg \
     -id $tempCurrentBranchDataDir/idFile_$current_branch_id.txt
 
-# D8 REM ##
+## CATCH AND MITIGATE BRANCH OUTLET BACKPOOL ERROR ##
+echo -e $startDiv"Catching and mitigating branch outlet backpool issue $hucNumber $current_branch_id"
+date -u
+Tstart
+$srcDir/mitigate_branch_outlet_backpool.py \
+    -b $tempCurrentBranchDataDir \
+    -cp $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif \
+    -cpp $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.gpkg \
+    -cr $tempCurrentBranchDataDir/gw_catchments_reaches_$current_branch_id.tif \
+    -s $tempCurrentBranchDataDir/demDerived_reaches_split_$current_branch_id.gpkg \
+    -p $tempCurrentBranchDataDir/demDerived_reaches_split_points_$current_branch_id.gpkg \
+    -n $b_arg \
+    -d $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
+    -t $slope_min \
+    --calculate-stats
+Tcount
+
+## D8 REM ##
 echo -e $startDiv"D8 REM $hucNumber $current_branch_id"
 $srcDir/make_rem.py -d $tempCurrentBranchDataDir/dem_thalwegCond_"$current_branch_id".tif \
     -w $tempCurrentBranchDataDir/gw_catchments_pixels_$current_branch_id.tif \
diff --git a/src/mitigate_branch_outlet_backpool.py b/src/mitigate_branch_outlet_backpool.py
new file mode 100755
index 000000000..ecbf52b14
--- /dev/null
+++ b/src/mitigate_branch_outlet_backpool.py
@@ -0,0 +1,543 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import warnings
+from collections import Counter
+from os import remove
+from os.path import isfile
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
+from rasterio.mask import mask
+from shapely import ops
+from shapely.geometry import Point
+
+
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+
+def mitigate_branch_outlet_backpool(
+    branch_dir,
+    catchment_pixels_filename,
+    catchment_pixels_polygonized_filename,
+    catchment_reaches_filename,
+    split_flows_filename,
+    split_points_filename,
+    nwm_streams_filename,
+    dem_filename,
+    slope_min,
+    calculate_stats,
+    dry_run,
+):
+    # --------------------------------------------------------------
+    # Define functions
+
+    # Test whether there are catchment size outliers (backpool error criteria 1)
+    def catch_catchment_size_outliers(catchment_pixels_geom):
+        # Quantify the amount of pixels in each catchment
+        unique_values = np.unique(catchment_pixels_geom)
+        value_counts = Counter(catchment_pixels_geom.ravel())
+
+        vals, counts = [], []
+        for value in unique_values:
+            vals.append(value)
+            counts.append(value_counts[value])
+
+        # Create a structured array from the two lists and convert to pd df
+        catchments_array = np.array(list(zip(vals, counts)), dtype=[('catchment_id', int), ('counts', int)])
+        catchments_df = pd.DataFrame(catchments_array)
+
+        # Remove row for a catchment_id of zero
+        catchments_df = catchments_df[catchments_df['catchment_id'] > 0]
+
+        # Calculate the mean and standard deviation of the 'counts' column
+        mean_counts = catchments_df['counts'].mean()
+        std_dev_counts = catchments_df['counts'].std()
+
+        # Define the threshold for outliers (1 standard deviation from the mean)
+        threshold = 1 * std_dev_counts
+
+        # Create a new column 'outlier' with True for outliers and False for non-outliers
+        catchments_df['outlier'] = abs(catchments_df['counts'] - mean_counts) > threshold
+
+        # Quantify outliers
+        num_outlier = catchments_df['outlier'].value_counts()[True]
+
+        if num_outlier == 0:
+            print('No outliers detected in catchment size.')
+
+            flagged_catchment = False
+        elif num_outlier >= 1:
+            print(f'{num_outlier} outlier catchment(s) found in catchment size.')
+
+            flagged_catchment = True
+        else:
+            print('WARNING: Unable to check outlier count.')
+
+        # Make a list of outlier catchment ID's
+        catchments_df['outlier'] = catchments_df['outlier'].astype('string')
+        outlier_catchment_ids = catchments_df[catchments_df['outlier'] == 'True']['catchment_id'].tolist()
+
+        return flagged_catchment, outlier_catchment_ids
+
+    # Extract raster catchment ID for the last point
+    def get_raster_value(point):
+        row, col = src.index(point.geometry.x, point.geometry.y)
+        value = catchment_pixels_geom[row, col]
+        return value
+
+    # Function to test whether the catchment occurs at the outlet (Error Criteria 2)
+    def check_if_outlet(last_point_geom, outlier_catchment_ids):
+        # Get the catchment ID of the last_point_geom
+        last_point_geom['catchment_id'] = last_point_geom.apply(get_raster_value, axis=1)
+
+        # Check if values in 'catchment_id' column of the snapped point are in outlier_catchments_df
+        outlet_flag = last_point_geom['catchment_id'].isin(outlier_catchment_ids)
+        outlet_flag = any(outlet_flag)
+
+        outlet_catchment_id = last_point_geom['catchment_id']
+
+        return outlet_flag, outlet_catchment_id
+
+    # Function to extract the last point from the line
+    def extract_last_point(line):
+        return line.coords[-1]
+
+    # Function to extract the third-to-last point from the line
+    def extract_pt_3tl(line):
+        return line.coords[-3]
+
+    # Function to count coordinates in a linestring
+    def count_coordinates(line_string):
+        return len(line_string.coords)
+
+    # Function to trim the flow to the specified outlet point
+    # Based on snap_and_trim_flow() from split_flows.py (as of 10/20/23)
+    def snap_and_trim_splitflow(outlet_point, flows):
+        if len(flows.index) == 1:
+            flow = flows
+
+        else:
+            # Get the nearest flowline to the outlet point
+            near_flows = []
+            for index, point in outlet_point.iterrows():
+                nearest_line = flows.loc[flows.distance(point['geometry']).idxmin()]
+                near_flows.append(nearest_line)
+
+            # Create a new GeoDataFrame with the closest flowline(s)
+            near_flows_gdf = gpd.GeoDataFrame(near_flows, crs=flows.crs)
+
+            # Trim the near flows to get the furthest downstream one
+            if len(near_flows_gdf) == 1:
+                flow = near_flows_gdf
+
+            elif len(near_flows_gdf) > 1:
+                # Get the highest node value (i.e. furthest down)
+                last_node = near_flows_gdf['From_Node'].max()
+
+                # Subset flows to get furthest down flow
+                flow = near_flows_gdf[near_flows_gdf['From_Node'] == last_node]
+
+        # Calculate flowline initial length
+        toMetersConversion = 1e-3
+        initial_length_km = flow.geometry.length.iloc[0] * toMetersConversion
+
+        # Reset index if there is an index mismatch
+        if flow.index != outlet_point.index:
+            print('WARNING: Index mismatch detected')
+            print(f'flow.index: {flow.index}; outlet_point.index: {outlet_point.index}')
+            print('Resetting index of flow and outlet_point geometries.')
+
+            flow = flow.reset_index()
+            outlet_point = outlet_point.reset_index()
+
+        # Snap the point to the line
+        outlet_point['geometry'] = flow.interpolate(flow.project(outlet_point))
+
+        # Split the flows at the snapped point
+        # Note: Buffering is to account for python precision issues
+        outlet_point_buffer = outlet_point.iloc[0]['geometry'].buffer(1)
+        split_lines = ops.split(flow.iloc[0]['geometry'], outlet_point_buffer)
+
+        # Get a list of the split line object indices
+        split_lines_indices = list(range(0, len(split_lines.geoms), 1))
+
+        # Produce a table of the geometry length of each splitlines geometries
+        linestring_lengths = []
+        linestring_geoms = []
+
+        for index in split_lines_indices:
+            linestring_geoms.append(split_lines.geoms[index])
+            linestring_lengths.append(split_lines.geoms[index].length)
+
+        split_lines_df = pd.DataFrame(
+            {
+                'split_lines_indices': split_lines_indices,
+                'geometry': linestring_geoms,
+                'len_flow': linestring_lengths,
+            }
+        )
+
+        # Select the longest line segment from the split
+        longest_split_line_df = split_lines_df[split_lines_df.len_flow == split_lines_df.len_flow.max()]
+
+        # Convert the longest split line into a geodataframe
+        # This removes the bits that got cut off
+        longest_split_line_gdf = gpd.GeoDataFrame(
+            longest_split_line_df, geometry=longest_split_line_df['geometry'], crs=flows.crs
+        )
+
+        # Get the new flow geometry
+        flow_geometry = longest_split_line_gdf.iloc[0]['geometry']
+
+        # Replace geometry in merged flowine
+        if len(flows) > 1:
+            flows.loc[flows['NextDownID'] == '-1', 'geometry'] = flow_geometry
+        else:
+            flows['geometry'] = flow_geometry
+
+        return flows, initial_length_km
+
+    def calculate_length_and_slope(flows, dem, slope_min):
+        print('Recalculating length and slope of outlet segment...')
+
+        # Select the last flowline segment (if there are multiple segments)
+        if len(flows) > 1:
+            flow = flows[flows['NextDownID'] == '-1']
+        else:
+            flow = flows
+
+        # Calculate channel slope (adapted from split_flows.py on 11/21/23)
+        start_point = flow.geometry.iloc[0].coords[0]
+        end_point = flow.geometry.iloc[0].coords[-1]
+
+        # Get start and end elevation
+        start_elev, end_elev = [i[0] for i in rasterio.sample.sample_gen(dem, [start_point, end_point])]
+
+        # Calculate the slope by differencing the elevations and dividing it by the length
+        slope = float(abs(start_elev - end_elev) / flow.length)
+
+        if slope < slope_min:
+            slope = slope_min
+
+        # Calculate length
+        toMetersConversion = 1e-3
+        LengthKm = flow.geometry.length * toMetersConversion
+
+        # Update flows with the updated flow object
+        if len(flows) > 1:
+            flows.loc[flows['NextDownID'] == '-1', 'S0'] = slope
+            flows.loc[flows['NextDownID'] == '-1', 'LengthKm'] = LengthKm
+
+        else:
+            flows['S0'] = slope
+            flows['LengthKm'] = LengthKm
+
+        return flows, LengthKm
+
+    # Convert the geodataframe into a json format compatible to rasterio
+    def gdf_to_json(gdf):
+        import json
+
+        return [json.loads(gdf.to_json())['features'][0]['geometry']]
+
+    # Mask a raster to a json boundary and save the new raster
+    def mask_raster_to_boundary(raster_path, boundary_json, save_path):
+        with rasterio.open(raster_path) as raster:
+            # Copy profile
+            raster_profile = raster.profile.copy()
+
+            # Mask catchment reaches to new boundary
+            raster_masked, _ = mask(raster, boundary_json)
+
+            if isfile(save_path):
+                remove(save_path)
+
+            # Save new catchment reaches TODO: update output path
+            with rasterio.open(save_path, "w", **raster_profile, BIGTIFF='YES') as dest:
+                dest.write(raster_masked[0, :, :], indexes=1)
+
+    # --------------------------------------------------------------
+    # Read in nwm lines, explode to ensure linestrings are the only geometry
+    nwm_streams = gpd.read_file(nwm_streams_filename).explode(index_parts=True)
+
+    # Check whether it's branch zero
+    if 'levpa_id' in nwm_streams.columns:
+        # --------------------------------------------------------------
+        # If it's NOT branch zero, check for the two criteria and mitigate issue if needed
+
+        # Read in data and check if the files exist
+        print()
+        print('Non-branch zero, loading data for test ...')
+
+        # Read in the catchment pixels tif
+        if isfile(catchment_pixels_filename):
+            with rasterio.open(catchment_pixels_filename) as src:
+                catchment_pixels_geom = src.read(1)
+        else:
+            catchment_pixels_geom = None
+            print(f'WARNING: No catchment pixels geom at {catchment_pixels_filename}.')
+
+        # # Read in the catchment reaches tif
+        # if isfile(catchment_reaches_filename):
+        #     with rasterio.open(catchment_reaches_filename) as src:
+        #         catchment_reaches_geom = src.read(1)
+        # else:
+        #     catchment_reaches_geom = None
+        #     print(f'No catchment pixels geome at {catchment_reaches_filename}.')
+
+        # Read in split_flows_file and split_points_filename
+        split_flows_geom = gpd.read_file(split_flows_filename)
+        split_points_geom = gpd.read_file(split_points_filename)
+
+        # Subset the split flows to get the last one
+        split_flows_last_geom = split_flows_geom[split_flows_geom['NextDownID'] == '-1']
+
+        # Check whether there are multiple NextDownID's of -1
+        if len(split_flows_last_geom.index) == 1:
+            one_neg1_nextdownid = True
+        elif len(split_flows_last_geom.index) > 1:
+            print('WARNING: Multiple stream segments found with NextDownID of -1.')
+            one_neg1_nextdownid = False
+        elif len(split_flows_last_geom.index) == 0:
+            print('WARNING: Zero stream segments found with NextDownID of -1.')
+            one_neg1_nextdownid = False
+
+        # Check whether catchment_pixels_geom exists
+        if (catchment_pixels_geom is not None) and (one_neg1_nextdownid is True):
+            print(
+                'A catchment geom file and only one NextDownID of -1 were found, testing for backpool criteria...'
+            )  # verbose
+
+            # Check whether any pixel catchment is substantially larger than other catchments
+            # (Backpool Error Criteria 1)
+            flagged_catchment, outlier_catchment_ids = catch_catchment_size_outliers(catchment_pixels_geom)
+
+            # --------------------------------------------------------------
+            # If there are outlier catchments, test whether the catchment occurs at the outlet
+            # (Backpool Error Criteria 2)
+
+            if flagged_catchment is True:
+                # Apply the function to create a new GeoDataFrame
+                last_point = split_flows_last_geom['geometry'].apply(extract_last_point).apply(Point)
+                last_point_geom = gpd.GeoDataFrame(last_point, columns=['geometry'], crs=split_flows_geom.crs)
+
+                # Check whether the last vertex corresponds with any of the outlier catchment ID's
+                print('Flagged catchment(s) detected. Testing for second criteria.')
+                outlet_flag, outlet_catchment_id = check_if_outlet(last_point_geom, outlier_catchment_ids)
+
+            else:
+                # If the catchment flag is False, just set the outlet flag to False automatically
+                outlet_flag = False
+
+            # If there is an outlier catchment at the outlet, set the snapped point as penultimate vertex
+            if outlet_flag is True:
+                # --------------------------------------------------------------
+                # Trim flowline and flow points to penultimate vertex
+
+                print(
+                    'Incorrectly-large outlet pixel catchment detected. \
+                      Snapping line points to penultimate vertex.'
+                )
+
+                # Count coordinates in 'geometry' column
+                split_flows_last_geom['num_coordinates'] = split_flows_last_geom['geometry'].apply(
+                    lambda x: count_coordinates(x) if x.geom_type == 'LineString' else None
+                )
+
+                # Get the last geometry and check the length of the last geometry
+                if split_flows_last_geom['num_coordinates'].iloc[0] < 3:
+                    # If the length is shorter than 3, extract the first point of the 2nd-to-last geometry
+                    if len(split_flows_geom.index) > 1:
+                        print('Extract first point of second-to-last geometry.')
+
+                        # Get "from_node" of last segment
+                        node_2tl = split_flows_last_geom['From_Node'].iloc[0]
+
+                        # Subset second-to-last segment by selecting by the node connection
+                        split_flows_2tl_geom = split_flows_geom[split_flows_geom['To_Node'] == node_2tl]
+
+                        # Get last point of second-to-last segment
+                        pt_3tl = split_flows_2tl_geom['geometry'].apply(extract_last_point).apply(Point)
+                        trim_flowlines_proceed = True
+
+                    else:
+                        print('Geom length is shorter than 3 coords and no second-to-last geom available.')
+                        print('Skipping branch outlet backpool mitigation for this branch.')
+                        trim_flowlines_proceed = False
+
+                else:
+                    # If the length is 3 coords or greater, extract the third-to-last point of the last geom
+                    print('Extract third-to-last point of last geometry.')
+                    pt_3tl = split_flows_last_geom['geometry'].apply(extract_pt_3tl).apply(Point)
+                    trim_flowlines_proceed = True
+
+                if trim_flowlines_proceed is True:
+                    # Apply the function to create a new GeoDataFrame
+                    pt_3tl_geom = gpd.GeoDataFrame(pt_3tl, columns=['geometry'], crs=split_flows_geom.crs)
+
+                    # Get the catchment ID of the new snapped point
+                    pt_3tl_geom['catchment_id'] = pt_3tl_geom.apply(get_raster_value, axis=1)
+
+                    # Snap and trim the flowline to the selected point
+                    trimmed_flows, inital_length_km = snap_and_trim_splitflow(pt_3tl_geom, split_flows_geom)
+
+                    # Create buffer around the updated flows geodataframe (and make sure it's all one shape)
+                    buffer = trimmed_flows.buffer(10).geometry.unary_union
+
+                    # Remove flowpoints that don't intersect with the trimmed flow line
+                    split_points_filtered_geom = split_points_geom[split_points_geom.geometry.within(buffer)]
+
+                    # --------------------------------------------------------------
+                    # Calculate the slope and length of the newly trimmed flows
+                    dem = rasterio.open(dem_filename, 'r')
+                    output_flows, new_length_km = calculate_length_and_slope(trimmed_flows, dem, slope_min)
+
+                    # --------------------------------------------------------------
+                    # Polygonize pixel catchments using subprocess
+
+                    # print('Polygonizing pixel catchments...')  # verbose
+
+                    gdal_args = [
+                        f'gdal_polygonize.py -8 -f GPKG {catchment_pixels_filename} \
+                                 {catchment_pixels_polygonized_filename} catchments HydroID'
+                    ]
+                    return_code = subprocess.call(gdal_args, shell=True)
+
+                    if return_code != 0:
+                        print("gdal_polygonize failed with return code", return_code)
+                    # else:
+                    # print("gdal_polygonize executed successfully.")  # verbose
+
+                    # Read in the polygonized catchment pixels
+                    cp_poly_geom = gpd.read_file(catchment_pixels_polygonized_filename)
+
+                    # --------------------------------------------------------------
+                    # Mask problematic pixel catchment from the catchments rasters
+
+                    # Convert series to number object
+                    outlet_catchment_id = outlet_catchment_id.iloc[0]
+
+                    # Filter out the flagged pixel catchment
+                    cp_poly_filt_geom = cp_poly_geom[cp_poly_geom['HydroID'] != outlet_catchment_id]
+
+                    # Dissolve the filtered pixel catchments into one geometry (the new boundary)
+                    cp_new_boundary_geom = cp_poly_filt_geom.dissolve()
+
+                    # Convert the geodataframe into a format compatible to rasterio
+                    catchment_pixels_new_boundary_json = gdf_to_json(cp_new_boundary_geom)
+
+                    if dry_run is True:
+                        print('Dry run: Skipping raster masking!')
+
+                    elif dry_run is False:
+                        # Mask catchment reaches raster
+                        mask_raster_to_boundary(
+                            catchment_reaches_filename,
+                            catchment_pixels_new_boundary_json,
+                            catchment_reaches_filename,
+                        )
+
+                        # Mask catchment pixels raster
+                        mask_raster_to_boundary(
+                            catchment_pixels_filename,
+                            catchment_pixels_new_boundary_json,
+                            catchment_pixels_filename,
+                        )
+
+                        # print('Finished masking!')  # verbose
+
+                    # --------------------------------------------------------------
+                    if calculate_stats is True:
+                        print('Calculating stats...')  # verbose
+
+                        # Get the area of the old and new catchment boundaries
+                        catchment_pixels_old_boundary_geom = cp_poly_geom.dissolve()
+
+                        old_boundary_area = catchment_pixels_old_boundary_geom.area
+                        new_boundary_area = cp_new_boundary_geom.area
+
+                        # Calculate the km and percent differences of the catchment area
+                        boundary_area_km_diff = float(old_boundary_area - new_boundary_area)
+                        boundary_area_percent_diff = float(
+                            ((boundary_area_km_diff) / old_boundary_area) * 100
+                        )
+
+                        # Calculate the difference (km) of the flowlines
+                        flowlength_km_diff = float(inital_length_km - new_length_km)
+
+                        # Create a dataframe with this data
+                        backpool_stats_df = pd.DataFrame(
+                            {
+                                'flowlength_km_diff': [flowlength_km_diff],
+                                'area_km_diff': [boundary_area_km_diff],
+                                'area_percent_diff': [boundary_area_percent_diff],
+                            }
+                        )
+
+                        backpool_stats_filepath = os.path.join(branch_dir, 'backpool_stats.csv')
+
+                        # Save stats
+                        backpool_stats_df.to_csv(backpool_stats_filepath, index=False)
+                        print(f'Saved backpool stats to {backpool_stats_filepath}')
+
+                    # --------------------------------------------------------------
+                    # Save the outputs
+
+                    if dry_run is True:
+                        print('Test run... not saving outputs!')
+
+                    elif dry_run is False:
+                        if isfile(split_flows_filename):
+                            remove(split_flows_filename)
+                        if isfile(split_points_filename):
+                            remove(split_points_filename)
+
+                        output_flows.to_file(split_flows_filename, driver='GPKG', index=False)
+                        split_points_filtered_geom.to_file(split_points_filename, driver='GPKG', index=False)
+
+            else:
+                print('Incorrectly-large outlet pixel catchment was NOT detected.')
+
+        else:
+            print('Will not test for outlet backpool error.')
+
+    else:
+        print('Will not test for outlet backpool error in branch zero.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Detect and mitigate branch outlet backpools issue.')
+    parser.add_argument('-b', '--branch-dir', help='branch directory', required=True)
+    parser.add_argument('-cp', '--catchment-pixels-filename', help='catchment-pixels-filename', required=True)
+    parser.add_argument(
+        '-cpp',
+        '--catchment-pixels-polygonized-filename',
+        help='catchment-pixels-polygonized-filename',
+        required=True,
+    )
+    parser.add_argument(
+        '-cr', '--catchment-reaches-filename', help='catchment-reaches-filename', required=True
+    )
+    parser.add_argument('-s', '--split-flows-filename', help='split-flows-filename', required=True)
+    parser.add_argument('-p', '--split-points-filename', help='split-points-filename', required=True)
+    parser.add_argument('-n', '--nwm-streams-filename', help='nwm-streams-filename', required=True)
+    parser.add_argument('-d', '--dem-filename', help='dem-filename', required=True)
+    parser.add_argument('-t', '--slope-min', help='Minimum slope', required=True)
+    parser.add_argument(
+        '--calculate-stats', help='Optional flag to calculate stats', required=False, action='store_true'
+    )
+    parser.add_argument(
+        '--dry-run', help='Optional flag to run without changing files.', required=False, action='store_true'
+    )
+
+    # Extract to dictionary and assign to variables
+    args = vars(parser.parse_args())
+    args['slope_min'] = float(args['slope_min'])
+    args['calculate_stats'] = bool(args['calculate_stats'])
+
+    mitigate_branch_outlet_backpool(**args)
diff --git a/src/split_flows.py b/src/split_flows.py
index 575b473c0..0a14943ce 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -1,17 +1,57 @@
 #!/usr/bin/env python3
 
 '''
-Description:
-    1) split stream segments based on lake boundaries and input threshold distance
-    2) calculate channel slope, manning's n, and LengthKm for each segment
-    3) create unique ids using HUC8 boundaries (and unique FIM_ID column)
-    4) create network traversal attribute columns (To_Node, From_Node, NextDownID)
-    5) create points layer with segment verticies encoded with HydroID's (used for catchment delineation in next step)
+Description
+
+    ARGUMENTS
+
+        flows_filename:
+            Filename of existing DEM-derived reaches input file. i.e. <current_branch_folder>/demDerived_reaches_<current_branch_id>.shp
+
+        dem_filename:
+            Filename of existing DEM input file. i.e. <current_branch_folder>/dem_thalwegCond_<current_branch_id>.tif
+
+        catchment_pixels_filename:
+            Filename of existing catchment pixels input file. i.e. <current_branch_folder>/gw_catchments_pixels_<current_branch_id>.tif
+
+        split_flows_filename:
+            Save location for output flowlines. i.e. <current_branch_folder>/demDerived_reaches_split_<current_branch_id>.gpkg
+
+        split_points_filename:
+            Save location for output flowpoints. i.e. <current_branch_folder>/demDerived_reaches_split_points_<current_branch_id>.gpkg
+
+        wbd8_clp_filename:
+            Filename of existing HUC8 geometry file. i.e. <HUC_data_folder>/wbd8_clp.gpkg
+
+        lakes_filename:
+            Filename of existing Lakes geometry file. i.e. <HUC_data_folder>/nwm_lakes_proj_subset.gpkg
+
+        nwm_streams_filename:
+            Filename of existing NWM streams input layer.
+
+        max_length:
+            Maximum acceptable length of stream segments (in meters).
+
+        slope_min:
+            Channel slope minimum value.
+
+        lakes_buffer_input:
+            Buffer size to use with lakes (in meters).
+
+
+    PROCESSING STEPS
+
+        1) Split stream segments based on lake boundaries and input threshold distance
+        2) Calculate channel slope, manning's n, and LengthKm for each segment
+        3) Create unique ids using HUC8 boundaries (and unique FIM_ID column)
+        4) Create network traversal attribute columns (To_Node, From_Node, NextDownID)
+        5) Create points layer with segment verticies encoded with HydroID's (used for catchment delineation in next step)
+
 '''
 
 import argparse
 import sys
-from collections import OrderedDict
+from collections import Counter, OrderedDict
 from os import remove
 from os.path import isfile
 
@@ -44,8 +84,11 @@ def split_flows(
     slope_min,
     lakes_buffer_input,
 ):
+    # --------------------------------------------------------------
+    # Define functions
+
     def snap_and_trim_flow(snapped_point, flows):
-        # Find nearest flow line
+        # Find the flowline nearest to the snapped point (if there's multiple flowlines)
         if len(flows) > 1:
             sjoin_nearest = gpd.sjoin_nearest(snapped_point, flows, max_distance=100)
             if sjoin_nearest.empty:
@@ -66,23 +109,26 @@ def snap_and_trim_flow(snapped_point, flows):
         snapped_point['geometry'] = flow.interpolate(flow.project(snapped_point.geometry))[0]
 
         # Trim flows to snapped point
-        # buffer here because python precision issues, print(demDerived_reaches.distance(snapped_point) < 1e-8)
         trimmed_line = shapely_ops_split(
             flow.iloc[0]['geometry'], snapped_point.iloc[0]['geometry'].buffer(1)
         )
+        # Note: Buffering is to account for python precision issues, print(demDerived_reaches.distance(snapped_point) < 1e-8)
+
         # Edge cases: line string not split?, nothing is returned, split does not preserve linestring order?
-        # Note to dear reader: last here is really the most upstream segment (see crevats above).
+        # Note to dear reader: last here is really the most upstream segment (see caveats above).
         # When we split we should get 3 segments, the most downstream one
         # the tiny 1 meter segment that falls within the snapped point buffer, and the most upstream one.
         # We want that last one which is why we trimmed_line[len(trimmed_line)-1]
+
         last_line_segment = pd.DataFrame(
             {'id': ['first'], 'geometry': [trimmed_line.geoms[len(trimmed_line.geoms) - 1].wkt]}
         )
-        # when we update geopandas verison: last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment)
+
+        # Note: When we update geopandas verison: last_line_segment = gpd.GeoSeries.from_wkt(last_line_segment)
         last_line_segment['geometry'] = last_line_segment['geometry'].apply(wkt.loads)
         last_line_segment_geodataframe = gpd.GeoDataFrame(last_line_segment).set_crs(flow.crs)
 
-        # replace geometry in merged flowine
+        # Replace geometry in merged flowine
         flow_geometry = last_line_segment_geodataframe.iloc[0]['geometry']
 
         if nearest_index is not None:
@@ -93,19 +139,31 @@ def snap_and_trim_flow(snapped_point, flows):
 
         return flows
 
-    toMetersConversion = 1e-3
+    # --------------------------------------------------------------
+    # Read in data and set constants
 
     print('Loading data ...')
-    flows = gpd.read_file(flows_filename)
 
+    toMetersConversion = 1e-3
+
+    # Read in flows data and check for relevant streams within HUC boundary
+    flows = gpd.read_file(flows_filename)
     if len(flows) == 0:
-        # this is not an exception, but a custom exit code that can be trapped
+        # Note: This is not an exception, but a custom exit code that can be trapped
         print("No relevant streams within HUC boundaries.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
+    # Read in and format other data
     wbd8 = gpd.read_file(wbd8_clp_filename)
     dem = rasterio.open(dem_filename, 'r')
 
+    # if isfile(catchment_pixels_filename):
+    #     with rasterio.open(catchment_pixels_filename) as src:
+    #         catchments_geom = src.read(1)
+    # else:
+    #     catchments_geom = None
+    #     print(f'No catchment pixels geometry found at {catchment_pixels_filename}.') ## debug
+
     if isfile(lakes_filename):
         lakes = gpd.read_file(lakes_filename)
     else:
@@ -113,58 +171,64 @@ def snap_and_trim_flow(snapped_point, flows):
 
     wbd8 = wbd8.filter(items=[FIM_ID, 'geometry'])
     wbd8 = wbd8.set_index(FIM_ID)
-    # don't index parts because the new index format causes problems later on
+
+    # Note: We don't index parts because the new index format causes problems later on
     flows = flows.explode(index_parts=False)
 
-    # temp
-    flows = flows.to_crs(wbd8.crs)
+    flows = flows.to_crs(wbd8.crs)  # Note: temporary solution
 
     split_flows = []
     slopes = []
     hydro_id = 'HydroID'
 
+    # --------------------------------------------------------------
+    # Trim DEM streams to NWM branch terminus
     # If loop addressing: https://github.com/NOAA-OWP/inundation-mapping/issues/560
-    print('trimming DEM stream to NWM branch terminus')
-    # read in nwm lines, explode to ensure linestrings are the only geometry
+
+    print('Trimming DEM stream to NWM branch terminus...')
+
+    # Read in nwm lines, explode to ensure linestrings are the only geometry
     nwm_streams = gpd.read_file(nwm_streams_filename).explode(index_parts=True)
 
-    # Dissolve levelpath if not branch 0
+    # If it's NOT branch 0: Dissolve levelpath
     if 'levpa_id' in nwm_streams.columns:
         if len(nwm_streams) > 1:
-            # Dissolve the linestring (how much faith should I hold that these are digitized with flow?)
+            # Dissolve the linestring TODO: How much faith should I hold that these are digitized with flow? (JC)
             linestring_geo = ops.linemerge(nwm_streams.dissolve(by='levpa_id').iloc[0]['geometry'])
         else:
             linestring_geo = nwm_streams.iloc[0]['geometry']
 
-        # Identify the end vertex (most downstream, should be last), transform into geodataframe
-        terminal_nwm_point = []
-
+        # If the linesting is in MultiLineString format, get the last LineString
         if linestring_geo.geom_type == 'MultiLineString':
-            # Get last LineString
             linestring_geo = linestring_geo.geoms[-1]
 
+        # Identify the end vertex (most downstream, should be last), transform into geodataframe
+        terminal_nwm_point = []
         last = Point(linestring_geo.coords[-1])
         terminal_nwm_point.append({'ID': 'terminal', 'geometry': last})
         snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(nwm_streams.crs)
 
+        # Snap and trim the flowline to the snapped point
         flows = snap_and_trim_flow(snapped_point, flows)
 
-    # If branch 0: loop over NWM terminal segments
+    # If it is branch 0: Loop over NWM terminal segments
     else:
         nwm_streams_terminal = nwm_streams[nwm_streams['to'] == 0]
         if not nwm_streams_terminal.empty:
             for i, row in nwm_streams_terminal.iterrows():
                 linestring_geo = row['geometry']
+
                 # Identify the end vertex (most downstream, should be last), transform into geodataframe
                 terminal_nwm_point = []
                 last = Point(linestring_geo.coords[-1])
                 terminal_nwm_point.append({'ID': 'terminal', 'geometry': last})
                 snapped_point = gpd.GeoDataFrame(terminal_nwm_point).set_crs(nwm_streams.crs)
 
+                # Snap and trim the flowline to the snapped point
                 flows = snap_and_trim_flow(snapped_point, flows)
 
-    # split at HUC8 boundaries
-    print('splitting stream segments at HUC8 boundaries')
+    # Split stream segments at HUC8 boundaries
+    print('Splitting stream segments at HUC8 boundaries...')
     flows = (
         gpd.overlay(flows, wbd8, how='union', keep_geom_type=True)
         .explode(index_parts=True)
@@ -172,16 +236,18 @@ def snap_and_trim_flow(snapped_point, flows):
     )
     flows = flows[~flows.is_empty]
 
+    # Make sure flows object doesn't have a length of zero
     if len(flows) == 0:
-        # this is not an exception, but a custom exit code that can be trapped
+        # Note: This is not an exception, but a custom exit code that can be trapped
         print("No relevant streams within HUC boundaries.")
-        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # Note: Will send a 61 back
 
-    # check for lake features
+    # Check for lake features and split flows at lake boundaries, if needed
     if lakes is not None and len(flows) > 0:
         if len(lakes) > 0:
-            print('splitting stream segments at ' + str(len(lakes)) + ' waterbodies')
-            # create splits at lake boundaries
+            print('Splitting stream segments at ' + str(len(lakes)) + ' waterbodies...')
+
+            # Create splits at lake boundaries
             lakes = lakes.filter(items=['newID', 'geometry'])
             lakes = lakes.set_index('newID')
             flows = (
@@ -192,27 +258,36 @@ def snap_and_trim_flow(snapped_point, flows):
             lakes_buffer = lakes.copy()
             lakes_buffer['geometry'] = lakes.buffer(
                 lakes_buffer_input
-            )  # adding X meter buffer for spatial join comparison (currently using 20meters)
-
-    print('splitting ' + str(len(flows)) + ' stream segments based on ' + str(max_length) + ' m max length')
+            )  # Note: adding X meter buffer for spatial join comparison (currently using 20meters)
+
+    print(
+        'Splitting '
+        + str(len(flows))
+        + ' stream segments using on max length of '
+        + str(max_length)
+        + ' meters'
+    )
 
-    # remove empty geometries
+    # Remove empty flow geometries
     flows = flows.loc[~flows.is_empty, :]
 
+    # Exit processing if length of flows is zero
     if len(flows) == 0:
-        # this is not an exception, but a custom exit code that can be trapped
+        # Note: This is not an exception, but a custom exit code that can be trapped
         print("No relevant streams within HUC boundaries.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
 
+    # --- begin copied into branch outlet backpool --- DEBUG -- maybe make this a function so we can call both?
+    # Iterate through flows and calculate channel slope, manning's n, and LengthKm for each segment
     for i, lineString in enumerate(flows.geometry):
         # Reverse geometry order (necessary for BurnLines)
         lineString = LineString(lineString.coords[::-1])
 
-        # skip lines of zero length
+        # Skip lines of zero length
         if lineString.length == 0:
             continue
 
-        # existing reaches of less than max_length
+        # Process existing reaches that are less than the max_length
         if lineString.length < max_length:
             split_flows = split_flows + [lineString]
             line_points = [point for point in zip(*lineString.coords.xy)]
@@ -227,6 +302,7 @@ def snap_and_trim_flow(snapped_point, flows):
             slopes = slopes + [slope]
             continue
 
+        # Calculate the split length
         splitLength = lineString.length / np.ceil(lineString.length / max_length)
 
         cumulative_line = []
@@ -235,6 +311,7 @@ def snap_and_trim_flow(snapped_point, flows):
 
         last_point_in_entire_lineString = list(zip(*lineString.coords.xy))[-1]
 
+        # Calculate cumulative length and channel slope,
         for point in zip(*lineString.coords.xy):
             cumulative_line = cumulative_line + [point]
             line_points = line_points + [point]
@@ -248,6 +325,7 @@ def snap_and_trim_flow(snapped_point, flows):
 
             cumulative_length = LineString(cumulative_line).length
 
+            # If the cumulative line length is greater than or equal to the split length....
             if cumulative_length >= splitLength:
                 splitLineString = LineString(cumulative_line)
                 split_flows = split_flows + [splitLineString]
@@ -283,6 +361,7 @@ def snap_and_trim_flow(snapped_point, flows):
             slope = slope_min
         slopes = slopes + [slope]
 
+    # Assemble the slopes and split flows into a geodataframe
     split_flows_gdf = gpd.GeoDataFrame(
         {'S0': slopes, 'geometry': split_flows}, crs=flows.crs, geometry='geometry'
     )
@@ -290,15 +369,19 @@ def snap_and_trim_flow(snapped_point, flows):
     if lakes is not None:
         split_flows_gdf = gpd.sjoin(
             split_flows_gdf, lakes_buffer, how='left', predicate='within'
-        )  # options: intersects, within, contains, crosses
+        )  # Note: Options include intersects, within, contains, crosses
         split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
     else:
         split_flows_gdf['LakeID'] = -999
 
-    # need to figure out why so many duplicate stream segments for 04010101 FR
-    split_flows_gdf = split_flows_gdf.drop_duplicates()
+    # --- end of copied into branch outlet backpool --- DEBUG -- maybe make this a function so we can call both?
+
+    # Drop duplicate stream segments
+    split_flows_gdf = (
+        split_flows_gdf.drop_duplicates()
+    )  # TODO: Need to figure out why so many duplicate stream segments for 04010101 FR (JC)
 
-    # Create Ids and Network Traversal Columns
+    # Create IDs and Network Traversal Columns
     addattributes = build_stream_traversal.build_stream_traversal_columns()
     tResults = None
     tResults = addattributes.execute(split_flows_gdf, wbd8, hydro_id)
@@ -307,12 +390,14 @@ def snap_and_trim_flow(snapped_point, flows):
     else:
         print('Error: Could not add network attributes to stream segments')
 
-    # remove single node segments
+    # Remove single node segments
     split_flows_gdf = split_flows_gdf.query("From_Node != To_Node")
 
-    # Get all vertices
     split_points = OrderedDict()
+
+    # Iterate through split flows line segments and create the points along each segment
     for index, segment in split_flows_gdf.iterrows():
+        # Get the points of the linestring geometry
         lineString = segment.geometry
 
         for point in zip(*lineString.coords.xy):
@@ -331,6 +416,8 @@ def snap_and_trim_flow(snapped_point, flows):
         {'id': hydroIDs_points, 'geometry': split_points}, crs=flows.crs, geometry='geometry'
     )
 
+    # --------------------------------------------------------------
+    # Save the outputs
     print('Writing outputs ...')
 
     if isfile(split_flows_filename):
@@ -339,9 +426,9 @@ def snap_and_trim_flow(snapped_point, flows):
         remove(split_points_filename)
 
     if len(split_flows_gdf) == 0:
-        # this is not an exception, but a custom exit code that can be trapped
+        # Note: This is not an exception, but a custom exit code that can be trapped
         print("There are no flowlines after stream order filtering.")
-        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # will send a 61 back
+        sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # Note: Will send a 61 back
 
     split_flows_gdf.to_file(split_flows_filename, driver=getDriver(split_flows_filename), index=False)
 

From f2062140b5d7f709710ccdbc34bb9137406396dc Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 11 Mar 2024 13:44:21 -0600
Subject: [PATCH 304/359] v4.4.13.1 Fix bug in levee masking in branch 0
 (#1086)

---
 docs/CHANGELOG.md | 12 ++++++++++++
 src/mask_dem.py   | 26 +++++++++++++-------------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 42df9c18e..a61567601 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.4.13.1 - 2024-03-11 - [PR#1086](https://github.com/NOAA-OWP/inundation-mapping/pull/1086)
+
+Fixes bug where levee-protected areas were not being masked from branch 0 DEMs.
+
+### Changes
+
+`src/mask_dem.py`: Corrects indentation preventing masked branch 0 from overwriting existing DEM.
+
+<br/><br/>
+
 ## v4.4.13.0 - 2024-03-11 - [PR#1006](https://github.com/NOAA-OWP/inundation-mapping/pull/1006)
 
 Adds a new module that mitigates the branch outlet backpool error. In some HUCs, an overly-large catchment appears at the outlet of the branch (as in issue #985) which causes an artificially large amount of water to get routed to the smaller stream instead of the main stem. This issue is mitigated by trimming the levelpath just above the outlet and removing the offending pixel catchment from the pixel catchments and catchment reaches files. 
@@ -53,6 +64,7 @@ Fixes bug in bathymetric adjustment where `mask` is used with `geopandas.read_fi
 
 <br/><br/>
 
+
 ## v4.4.11.0 - 2024-02-16 - [PR#1077](https://github.com/NOAA-OWP/inundation-mapping/pull/1077)
 
 Replace `fiona` with `pyogrio` to improve I/O speed. `geopandas` will use `pyogrio` by default starting with version 1.0. `pyarrow` was also added as an environment variable to further speedup I/O. As a result of the changes in this PR, `fim_pipeline.sh` runs approximately 10% faster.
diff --git a/src/mask_dem.py b/src/mask_dem.py
index f551c9df4..a9dc55dd8 100755
--- a/src/mask_dem.py
+++ b/src/mask_dem.py
@@ -55,6 +55,7 @@ def mask_dem(
     assert os.path.exists(catchments_filename), f"Catchments file {catchments_filename} does not exist"
 
     dem_masked = None
+    levee_catchments_masked = None
 
     with rio.open(dem_filename) as dem:
         dem_profile = dem.profile.copy()
@@ -104,24 +105,23 @@ def mask_dem(
 
             geoms = [feature["geometry"] for i, feature in levee_catchments_to_mask.iterrows()]
 
-            levee_catchments_masked = None
             if len(geoms) > 0:
                 levee_catchments_masked, _ = mask(dem, geoms, invert=True)
 
-            out_masked = None
-            if dem_masked is None:
-                if levee_catchments_masked is not None:
-                    out_masked = levee_catchments_masked
+        out_masked = None
+        if dem_masked is None:
+            if levee_catchments_masked is not None:
+                out_masked = levee_catchments_masked
 
+        else:
+            if levee_catchments_masked is None:
+                out_masked = dem_masked
             else:
-                if levee_catchments_masked is None:
-                    out_masked = dem_masked
-                else:
-                    out_masked = np.where(levee_catchments_masked == nodata, nodata, dem_masked)
-
-            if out_masked is not None:
-                with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
-                    dest.write(out_masked[0, :, :], indexes=1)
+                out_masked = np.where(levee_catchments_masked == nodata, nodata, dem_masked)
+
+        if out_masked is not None:
+            with rio.open(out_dem_filename, "w", **dem_profile, BIGTIFF='YES') as dest:
+                dest.write(out_masked[0, :, :], indexes=1)
 
 
 if __name__ == '__main__':

From 27f74d5cf459a7bf1d7fe7a57563942b325d7e1d Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 4 Apr 2024 21:07:01 +0000
Subject: [PATCH 305/359] v4.4.13.2 Updates for newer openjdk and black
 packages (#1110)

---
 .pre-commit-config.yaml                       |  2 +-
 Dockerfile                                    |  2 +
 Pipfile                                       |  2 +-
 Pipfile.lock                                  | 65 ++++++++++---------
 data/aws/aws_base.py                          |  1 -
 docs/CHANGELOG.md                             | 16 +++++
 src/src_roughness_optimization.py             | 18 ++---
 src/usgs_gage_crosswalk.py                    |  6 +-
 tools/overlapping_inundation.py               |  9 +--
 tools/rating_curve_comparison.py              |  6 +-
 unit_tests/__template.py                      |  1 -
 unit_tests/aggregate_branch_lists_test.py     |  1 -
 unit_tests/clip_vectors_to_wbd_test.py        |  1 -
 unit_tests/derive_level_paths_test.py         |  1 -
 ...lter_catchments_and_add_attributes_test.py |  1 -
 unit_tests/generate_branch_list_csv_test.py   |  1 -
 unit_tests/generate_branch_list_test.py       |  1 -
 unit_tests/inundate_gms_test.py               |  1 -
 unit_tests/inundation_test.py                 |  1 -
 unit_tests/outputs_cleanup_test.py            |  1 -
 unit_tests/rating_curve_comparison_test.py    |  1 -
 unit_tests/shared_functions_test.py           |  1 -
 unit_tests/split_flows_test.py                |  1 -
 unit_tests/usgs_gage_crosswalk_test.py        |  1 -
 24 files changed, 71 insertions(+), 70 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6c2bb6513..ff510b185 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -21,7 +21,7 @@ repos:
             additional_dependencies: [flake8-pyproject]
 
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 24.3.0
     hooks:
         -   id: black
             args: ['--line-length=110', '--skip-magic-trailing-comma', '--skip-string-normalization']
diff --git a/Dockerfile b/Dockerfile
index 70f7d336f..68be823cf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -66,6 +66,8 @@ RUN mkdir -p $workDir
 RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
+RUN apt-get update --fix-missing && apt-get install -y openjdk-19-jdk && rm -rf /var/lib/apt/lists/*
+
 RUN apt update --fix-missing
 RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata
 
diff --git a/Pipfile b/Pipfile
index 5aee10468..31a386c5c 100644
--- a/Pipfile
+++ b/Pipfile
@@ -38,7 +38,7 @@ py7zr = "==0.20.4"
 scipy = "==1.10.1"
 gval = "==0.2.3"
 flake8 = "==6.0.0"
-black = "==23.7.0"
+black = "==24.3.0"
 flake8-pyproject = "==1.2.3"
 pre-commit = "==3.3.3"
 isort = "==5.12.0"
diff --git a/Pipfile.lock b/Pipfile.lock
index 0c47c4a04..76e73a340 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "a3ed6a563de9000738a1df55a5cbce4a7584fcee0725bf71f917916edf6eef3b"
+            "sha256": "6b97aa17b943660fc1d53242b0424db6cdf340811d2ead934e4fececef750ba3"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -131,31 +131,32 @@
         },
         "black": {
             "hashes": [
-                "sha256:01ede61aac8c154b55f35301fac3e730baf0c9cf8120f65a9cd61a81cfb4a0c3",
-                "sha256:022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb",
-                "sha256:25cc308838fe71f7065df53aedd20327969d05671bac95b38fdf37ebe70ac087",
-                "sha256:27eb7a0c71604d5de083757fbdb245b1a4fae60e9596514c6ec497eb63f95320",
-                "sha256:327a8c2550ddc573b51e2c352adb88143464bb9d92c10416feb86b0f5aee5ff6",
-                "sha256:47e56d83aad53ca140da0af87678fb38e44fd6bc0af71eebab2d1f59b1acf1d3",
-                "sha256:501387a9edcb75d7ae8a4412bb8749900386eaef258f1aefab18adddea1936bc",
-                "sha256:552513d5cd5694590d7ef6f46e1767a4df9af168d449ff767b13b084c020e63f",
-                "sha256:5c4bc552ab52f6c1c506ccae05681fab58c3f72d59ae6e6639e8885e94fe2587",
-                "sha256:642496b675095d423f9b8448243336f8ec71c9d4d57ec17bf795b67f08132a91",
-                "sha256:6d1c6022b86f83b632d06f2b02774134def5d4d4f1dac8bef16d90cda18ba28a",
-                "sha256:7f3bf2dec7d541b4619b8ce526bda74a6b0bffc480a163fed32eb8b3c9aed8ad",
-                "sha256:831d8f54c3a8c8cf55f64d0422ee875eecac26f5f649fb6c1df65316b67c8926",
-                "sha256:8417dbd2f57b5701492cd46edcecc4f9208dc75529bcf76c514864e48da867d9",
-                "sha256:86cee259349b4448adb4ef9b204bb4467aae74a386bce85d56ba4f5dc0da27be",
-                "sha256:893695a76b140881531062d48476ebe4a48f5d1e9388177e175d76234ca247cd",
-                "sha256:9fd59d418c60c0348505f2ddf9609c1e1de8e7493eab96198fc89d9f865e7a96",
-                "sha256:ad0014efc7acf0bd745792bd0d8857413652979200ab924fbf239062adc12491",
-                "sha256:b5b0ee6d96b345a8b420100b7d71ebfdd19fab5e8301aff48ec270042cd40ac2",
-                "sha256:c333286dc3ddca6fdff74670b911cccedacb4ef0a60b34e491b8a67c833b343a",
-                "sha256:f9062af71c59c004cd519e2fb8f5d25d39e46d3af011b41ab43b9c74e27e236f",
-                "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995"
+                "sha256:2818cf72dfd5d289e48f37ccfa08b460bf469e67fb7c4abb07edc2e9f16fb63f",
+                "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93",
+                "sha256:4acf672def7eb1725f41f38bf6bf425c8237248bb0804faa3965c036f7672d11",
+                "sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0",
+                "sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9",
+                "sha256:56f52cfbd3dabe2798d76dbdd299faa046a901041faf2cf33288bc4e6dae57b5",
+                "sha256:65b76c275e4c1c5ce6e9870911384bff5ca31ab63d19c76811cb1fb162678213",
+                "sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d",
+                "sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7",
+                "sha256:79dcf34b33e38ed1b17434693763301d7ccbd1c5860674a8f871bd15139e7837",
+                "sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f",
+                "sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395",
+                "sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995",
+                "sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f",
+                "sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597",
+                "sha256:b5991d523eee14756f3c8d5df5231550ae8993e2286b8014e2fdea7156ed0959",
+                "sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5",
+                "sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb",
+                "sha256:c7ed6668cbbfcd231fa0dc1b137d3e40c04c7f786e626b405c62bcd5db5857e4",
+                "sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7",
+                "sha256:e19cb1c6365fd6dc38a6eae2dcb691d7d83935c10215aef8e6c38edee3f77abd",
+                "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"
             ],
             "index": "pypi",
-            "version": "==23.7.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==24.3.0"
         },
         "bleach": {
             "hashes": [
@@ -1898,11 +1899,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
-                "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
+                "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
+                "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.2"
+            "version": "==24.0"
         },
         "pandas": {
             "hashes": [
@@ -3539,11 +3540,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783",
-                "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"
+                "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475",
+                "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==4.9.0"
+            "markers": "python_version < '3.11'",
+            "version": "==4.10.0"
         },
         "typing-inspect": {
             "hashes": [
@@ -3964,4 +3965,4 @@
             "version": "==0.2.13"
         }
     }
-}
+}
\ No newline at end of file
diff --git a/data/aws/aws_base.py b/data/aws/aws_base.py
index b0f6befad..af3688abf 100644
--- a/data/aws/aws_base.py
+++ b/data/aws/aws_base.py
@@ -10,7 +10,6 @@
 
 
 class AWS_Base(object):
-
     '''
     This class implements all common variables related when communicating to AWS
     '''
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a61567601..875709249 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.13.2 - 2024-04-04 - [PR#1110](https://github.com/NOAA-OWP/inundation-mapping/pull/1110)
+
+This PR reflects upgrades for openJDK from 17.0.8 to something higher, minimum of 17.0.9. After some research, we can not upgrade all the way to the latest openJDK but can jump up to 19.0.  This limitation is related to version of our base docker image.  openJDK was identified as requiring an upgrade by a system wide security scan.
+
+The "black" packages is also be upgraded from 23.7.0 to 24.3.
+
+**NOTE: the update of "black" has change the rules slightly for formatting. This is why you see a bunch of files being changed but only for the formatting changes.**
+
+### Files Change
+- `Dockerfile`, `Pipfile`, `Pipefile.lock`
+- `pre-commit-config.yaml` is also has Black upgraded for CI/CD tests for linting during GIT check ins.
+- `many files`:
+     - 19 files have had minor formatting changes related to the upgrade in the "black" package.
+
+<br/><br/>
+
 
 ## v4.4.13.1 - 2024-03-11 - [PR#1086](https://github.com/NOAA-OWP/inundation-mapping/pull/1086)
 
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index bcd860a13..4c0913573 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -585,12 +585,12 @@ def branch_network_tracer(df_input_htable):
         while Q:
             q = Q.popleft()
             if q not in visited:
-                df_input_htable.loc[
-                    df_input_htable.HydroID == q, 'route_count'
-                ] = vert_count  # assign var with flow order ranking
-                df_input_htable.loc[
-                    df_input_htable.HydroID == q, 'branch_id'
-                ] = branch_count  # assign var with current branch id
+                df_input_htable.loc[df_input_htable.HydroID == q, 'route_count'] = (
+                    vert_count  # assign var with flow order ranking
+                )
+                df_input_htable.loc[df_input_htable.HydroID == q, 'branch_id'] = (
+                    branch_count  # assign var with current branch id
+                )
                 vert_count += 1
                 visited.add(q)
                 # find the id for the next downstream hydroid
@@ -676,9 +676,9 @@ def group_manningn_calc(df_nmerge, down_dist_thresh):
                 # only apply the group_calb_coef if there are 2 or more valid hydorids that contributed to the
                 # upstream group_calb_coef
                 if hyid_accum_count > 1:
-                    df_nmerge.loc[
-                        index, 'group_calb_coef'
-                    ] = group_calb_coef  # output the group_calb_coef var
+                    df_nmerge.loc[index, 'group_calb_coef'] = (
+                        group_calb_coef  # output the group_calb_coef var
+                    )
             else:
                 # reset the running average manningn variable (greater than 10km downstream)
                 run_avg_mann = 0
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 99d3fc29e..4a477692c 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -113,9 +113,9 @@ def write(self, output_directory):
 
         # Prep and write out file
         elev_table = self.gages.copy()
-        elev_table.loc[
-            elev_table['location_id'] == elev_table['nws_lid'], 'location_id'
-        ] = None  # set location_id to None where there isn't a gage
+        elev_table.loc[elev_table['location_id'] == elev_table['nws_lid'], 'location_id'] = (
+            None  # set location_id to None where there isn't a gage
+        )
         elev_table = elev_table[elev_table['location_id'].notna()]
         elev_table.source = elev_table.source.apply(str.lower)
 
diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py
index 23c6575d8..34c0e1dad 100644
--- a/tools/overlapping_inundation.py
+++ b/tools/overlapping_inundation.py
@@ -57,12 +57,9 @@ def key_sort_func(x):
         )
 
         # get transform, width, height and bounds
-        (
-            self.proc_unit_transform,
-            self.proc_unit_width,
-            self.proc_unit_height,
-            final_bounds,
-        ) = self.get_final_dims()
+        (self.proc_unit_transform, self.proc_unit_width, self.proc_unit_height, final_bounds) = (
+            self.get_final_dims()
+        )
 
         self.proc_unit_bounds = np.array(
             [[final_bounds["top"], final_bounds["left"]], [final_bounds["bottom"], final_bounds["right"]]]
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 58e316068..5b5369316 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -896,9 +896,9 @@ def generate_rc_and_rem_plots(rc, plot_filename, recurr_data_table, branches_fol
         dem_adj_elevation = rc[rc.location_id == gage].dem_adj_elevation.unique()[0]
         catchment_rem = (catchment_rem + dem_adj_elevation) * 3.28084
         max_elev = rc[(rc.source == 'FIM') & (rc.location_id == gage)].elevation_ft.max()
-        catchment_rem[
-            np.where(catchment_rem > max_elev)
-        ] = np.nan  # <-- Comment out this line to get the full raster that is
+        catchment_rem[np.where(catchment_rem > max_elev)] = (
+            np.nan
+        )  # <-- Comment out this line to get the full raster that is
         # used in rating curve creation
         # Create polygon for perimeter/area stats
         catchment_rem_1s = catchment_rem.copy()
diff --git a/unit_tests/__template.py b/unit_tests/__template.py
index bef8723b1..946e06474 100644
--- a/unit_tests/__template.py
+++ b/unit_tests/__template.py
@@ -10,7 +10,6 @@
 
 
 class test_Your_original_source_python_file_name(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/aggregate_branch_lists_test.py b/unit_tests/aggregate_branch_lists_test.py
index 961db481a..93c4f29d8 100644
--- a/unit_tests/aggregate_branch_lists_test.py
+++ b/unit_tests/aggregate_branch_lists_test.py
@@ -10,7 +10,6 @@
 
 
 class test_aggregate_branch_lists(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
index 9c817a448..b9567a3d3 100644
--- a/unit_tests/clip_vectors_to_wbd_test.py
+++ b/unit_tests/clip_vectors_to_wbd_test.py
@@ -11,7 +11,6 @@
 
 
 class test_clip_vectors_to_wbd(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py
index acb89a538..6ee67973d 100644
--- a/unit_tests/derive_level_paths_test.py
+++ b/unit_tests/derive_level_paths_test.py
@@ -14,7 +14,6 @@
 
 
 class test_Derive_level_paths(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/filter_catchments_and_add_attributes_test.py b/unit_tests/filter_catchments_and_add_attributes_test.py
index 554f4fce4..098d6935f 100644
--- a/unit_tests/filter_catchments_and_add_attributes_test.py
+++ b/unit_tests/filter_catchments_and_add_attributes_test.py
@@ -17,7 +17,6 @@
 # use deny_gms_branches_dev.lst or the word "none" for the deny list arguments
 # (unit and branch deny list parameters). Key files need to exist for this unit test to work.
 class test_filter_catchments_and_add_attributes(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/generate_branch_list_csv_test.py b/unit_tests/generate_branch_list_csv_test.py
index 020556e97..5495767b6 100644
--- a/unit_tests/generate_branch_list_csv_test.py
+++ b/unit_tests/generate_branch_list_csv_test.py
@@ -11,7 +11,6 @@
 
 
 class test_generate_branch_list_csv(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/generate_branch_list_test.py b/unit_tests/generate_branch_list_test.py
index ab654db89..3a35f35ea 100644
--- a/unit_tests/generate_branch_list_test.py
+++ b/unit_tests/generate_branch_list_test.py
@@ -11,7 +11,6 @@
 
 
 class test_Generate_branch_list(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/inundate_gms_test.py b/unit_tests/inundate_gms_test.py
index c8cfdab5e..034830aa2 100644
--- a/unit_tests/inundate_gms_test.py
+++ b/unit_tests/inundate_gms_test.py
@@ -12,7 +12,6 @@
 
 
 class test_inundate_gms(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/inundation_test.py b/unit_tests/inundation_test.py
index c8348ae7e..ef580c15a 100644
--- a/unit_tests/inundation_test.py
+++ b/unit_tests/inundation_test.py
@@ -11,7 +11,6 @@
 
 
 class test_inundate(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/outputs_cleanup_test.py b/unit_tests/outputs_cleanup_test.py
index d339ff183..5d152864f 100644
--- a/unit_tests/outputs_cleanup_test.py
+++ b/unit_tests/outputs_cleanup_test.py
@@ -16,7 +16,6 @@
 
 
 class test_outputs_cleanup(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/rating_curve_comparison_test.py b/unit_tests/rating_curve_comparison_test.py
index 110d58c8a..89352cb26 100644
--- a/unit_tests/rating_curve_comparison_test.py
+++ b/unit_tests/rating_curve_comparison_test.py
@@ -10,7 +10,6 @@
 
 
 class test_rating_curve_comparison(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/shared_functions_test.py b/unit_tests/shared_functions_test.py
index e5046bb2e..62264ec75 100644
--- a/unit_tests/shared_functions_test.py
+++ b/unit_tests/shared_functions_test.py
@@ -10,7 +10,6 @@
 
 
 class test_shared_functions(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py
index b7ee9b0d8..a811fccfc 100644
--- a/unit_tests/split_flows_test.py
+++ b/unit_tests/split_flows_test.py
@@ -10,7 +10,6 @@
 
 
 class test_split_flows(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """
diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py
index 43d9fa2de..863c61590 100644
--- a/unit_tests/usgs_gage_crosswalk_test.py
+++ b/unit_tests/usgs_gage_crosswalk_test.py
@@ -12,7 +12,6 @@
 
 
 class test_usgs_gage_crosswalk(unittest.TestCase):
-
     """
     Allows the params to be loaded one and used for all test methods
     """

From adb6e36b324693faaaf318a8af54ef55ea44164d Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 15 Apr 2024 20:54:32 +0000
Subject: [PATCH 306/359] v4.4.13.3 Updates for pkgs pillow, idna and openpynx
 (#1114)

---
 Pipfile                             |    3 +-
 Pipfile.lock                        | 1381 ++++++++++++++-------------
 README.md                           |   28 +-
 data/ble/ble_benchmark/Dockerfile   |    9 -
 data/ble/ble_benchmark/Pipfile      |   15 -
 data/ble/ble_benchmark/Pipfile.lock |  340 -------
 data/ble/ble_benchmark/README.md    |   12 +-
 docs/CHANGELOG.md                   |   23 +
 8 files changed, 748 insertions(+), 1063 deletions(-)
 delete mode 100644 data/ble/ble_benchmark/Dockerfile
 delete mode 100755 data/ble/ble_benchmark/Pipfile
 delete mode 100644 data/ble/ble_benchmark/Pipfile.lock

diff --git a/Pipfile b/Pipfile
index 31a386c5c..35731f13c 100644
--- a/Pipfile
+++ b/Pipfile
@@ -44,8 +44,9 @@ pre-commit = "==3.3.3"
 isort = "==5.12.0"
 urllib3 = "==1.26.18"
 pyflwdir = "==0.5.8"
-pillow = "==10.2.0"
+pillow = "==10.3.0"
 pyogrio = "==0.7.2"
+openpyxl = "*"
 
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index 76e73a340..a7d7bef21 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "6b97aa17b943660fc1d53242b0424db6cdf340811d2ead934e4fececef750ba3"
+            "sha256": "13d8feb3cb31feea69831b7d547993f4adde0134668a95c9744aa54d45cae748"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -29,24 +29,24 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "markers": "python_version >= '3.7' and python_version < '4'",
             "version": "==22.1.0"
         },
         "aiosqlite": {
             "hashes": [
-                "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d",
-                "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"
+                "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6",
+                "sha256:6d35c8c256637f4672f843c31021464090805bf925385ac39473fb16eaaca3d7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.19.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.20.0"
         },
         "anyio": {
             "hashes": [
-                "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee",
-                "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"
+                "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8",
+                "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.2.0"
+            "version": "==4.3.0"
         },
         "appdirs": {
             "hashes": [
@@ -155,7 +155,6 @@
                 "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
             "version": "==24.3.0"
         },
         "bleach": {
@@ -190,7 +189,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4.0'",
+            "markers": "python_version >= '3.8' and python_version < '4'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -300,11 +299,11 @@
         },
         "cachetools": {
             "hashes": [
-                "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2",
-                "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"
+                "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945",
+                "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.2"
+            "version": "==5.3.3"
         },
         "certifi": {
             "hashes": [
@@ -523,7 +522,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -536,19 +535,19 @@
         },
         "colorcet": {
             "hashes": [
-                "sha256:51455a20353d12fac91f953772d8409f2474e6a0db1af3fa4f7005f405a2480b",
-                "sha256:8daff01824ee9935fdf762d15c444a67d3e361ad4f8b738ad59ac9bf38f30600"
+                "sha256:2921b3cd81a2288aaf2d63dbc0ce3c26dcd882e8c389cc505d6886bf7aa9a4eb",
+                "sha256:2a7d59cc8d0f7938eeedd08aad3152b5319b4ba3bcb7a612398cc17a384cb296"
             ],
-            "markers": "python_version >= '2.7'",
-            "version": "==3.0.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.1.0"
         },
         "comm": {
             "hashes": [
-                "sha256:0bc91edae1344d39d3661dcbc36937181fdaddb304790458f8b044dbc064b89a",
-                "sha256:87928485c0dfc0e7976fd89fc1e187023cf587e7c353e4a9b417555b44adf021"
+                "sha256:3fd7a84065306e07bea1773df6eb8282de51ba82f77c72f9c85716ab11fe980e",
+                "sha256:e6fb86cb70ff661ee8c9c14e7d36d6de3b4066f1441be4063df9c5009f0a64d3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.2.1"
+            "version": "==0.2.2"
         },
         "contextily": {
             "hashes": [
@@ -560,53 +559,53 @@
         },
         "contourpy": {
             "hashes": [
-                "sha256:0274c1cb63625972c0c007ab14dd9ba9e199c36ae1a231ce45d725cbcbfd10a8",
-                "sha256:0d7e03c0f9a4f90dc18d4e77e9ef4ec7b7bbb437f7f675be8e530d65ae6ef956",
-                "sha256:11f8d2554e52f459918f7b8e6aa20ec2a3bce35ce95c1f0ef4ba36fbda306df5",
-                "sha256:139d8d2e1c1dd52d78682f505e980f592ba53c9f73bd6be102233e358b401063",
-                "sha256:16a7380e943a6d52472096cb7ad5264ecee36ed60888e2a3d3814991a0107286",
-                "sha256:171f311cb758de7da13fc53af221ae47a5877be5a0843a9fe150818c51ed276a",
-                "sha256:18fc2b4ed8e4a8fe849d18dce4bd3c7ea637758c6343a1f2bae1e9bd4c9f4686",
-                "sha256:1c203f617abc0dde5792beb586f827021069fb6d403d7f4d5c2b543d87edceb9",
-                "sha256:1c2559d6cffc94890b0529ea7eeecc20d6fadc1539273aa27faf503eb4656d8f",
-                "sha256:1c88dfb9e0c77612febebb6ac69d44a8d81e3dc60f993215425b62c1161353f4",
-                "sha256:1e9dc350fb4c58adc64df3e0703ab076f60aac06e67d48b3848c23647ae4310e",
-                "sha256:247b9d16535acaa766d03037d8e8fb20866d054d3c7fbf6fd1f993f11fc60ca0",
-                "sha256:266270c6f6608340f6c9836a0fb9b367be61dde0c9a9a18d5ece97774105ff3e",
-                "sha256:34b9071c040d6fe45d9826cbbe3727d20d83f1b6110d219b83eb0e2a01d79488",
-                "sha256:3d7d1f8871998cdff5d2ff6a087e5e1780139abe2838e85b0b46b7ae6cc25399",
-                "sha256:461e3ae84cd90b30f8d533f07d87c00379644205b1d33a5ea03381edc4b69431",
-                "sha256:464b423bc2a009088f19bdf1f232299e8b6917963e2b7e1d277da5041f33a779",
-                "sha256:491b1917afdd8638a05b611a56d46587d5a632cabead889a5440f7c638bc6ed9",
-                "sha256:4a1b1208102be6e851f20066bf0e7a96b7d48a07c9b0cfe6d0d4545c2f6cadab",
-                "sha256:575bcaf957a25d1194903a10bc9f316c136c19f24e0985a2b9b5608bdf5dbfe0",
-                "sha256:5c6b28956b7b232ae801406e529ad7b350d3f09a4fde958dfdf3c0520cdde0dd",
-                "sha256:5d16edfc3fc09968e09ddffada434b3bf989bf4911535e04eada58469873e28e",
-                "sha256:5fd1810973a375ca0e097dee059c407913ba35723b111df75671a1976efa04bc",
-                "sha256:67b7f17679fa62ec82b7e3e611c43a016b887bd64fb933b3ae8638583006c6d6",
-                "sha256:68ce4788b7d93e47f84edd3f1f95acdcd142ae60bc0e5493bfd120683d2d4316",
-                "sha256:6d3364b999c62f539cd403f8123ae426da946e142312a514162adb2addd8d808",
-                "sha256:6e739530c662a8d6d42c37c2ed52a6f0932c2d4a3e8c1f90692ad0ce1274abe0",
-                "sha256:6fdd887f17c2f4572ce548461e4f96396681212d858cae7bd52ba3310bc6f00f",
-                "sha256:78e6ad33cf2e2e80c5dfaaa0beec3d61face0fb650557100ee36db808bfa6843",
-                "sha256:884c3f9d42d7218304bc74a8a7693d172685c84bd7ab2bab1ee567b769696df9",
-                "sha256:8d8faf05be5ec8e02a4d86f616fc2a0322ff4a4ce26c0f09d9f7fb5330a35c95",
-                "sha256:999c71939aad2780f003979b25ac5b8f2df651dac7b38fb8ce6c46ba5abe6ae9",
-                "sha256:99ad97258985328b4f207a5e777c1b44a83bfe7cf1f87b99f9c11d4ee477c4de",
-                "sha256:9e6c93b5b2dbcedad20a2f18ec22cae47da0d705d454308063421a3b290d9ea4",
-                "sha256:ab459a1cbbf18e8698399c595a01f6dcc5c138220ca3ea9e7e6126232d102bb4",
-                "sha256:b69303ceb2e4d4f146bf82fda78891ef7bcd80c41bf16bfca3d0d7eb545448aa",
-                "sha256:b7caf9b241464c404613512d5594a6e2ff0cc9cb5615c9475cc1d9b514218ae8",
-                "sha256:b95a225d4948b26a28c08307a60ac00fb8671b14f2047fc5476613252a129776",
-                "sha256:bd2f1ae63998da104f16a8b788f685e55d65760cd1929518fd94cd682bf03e41",
-                "sha256:be16975d94c320432657ad2402f6760990cb640c161ae6da1363051805fa8108",
-                "sha256:ce96dd400486e80ac7d195b2d800b03e3e6a787e2a522bfb83755938465a819e",
-                "sha256:dbd50d0a0539ae2e96e537553aff6d02c10ed165ef40c65b0e27e744a0f10af8",
-                "sha256:dd10c26b4eadae44783c45ad6655220426f971c61d9b239e6f7b16d5cdaaa727",
-                "sha256:ebeac59e9e1eb4b84940d076d9f9a6cec0064e241818bcb6e32124cc5c3e377a"
+                "sha256:00e5388f71c1a0610e6fe56b5c44ab7ba14165cdd6d695429c5cd94021e390b2",
+                "sha256:10a37ae557aabf2509c79715cd20b62e4c7c28b8cd62dd7d99e5ed3ce28c3fd9",
+                "sha256:11959f0ce4a6f7b76ec578576a0b61a28bdc0696194b6347ba3f1c53827178b9",
+                "sha256:187fa1d4c6acc06adb0fae5544c59898ad781409e61a926ac7e84b8f276dcef4",
+                "sha256:1a07fc092a4088ee952ddae19a2b2a85757b923217b7eed584fdf25f53a6e7ce",
+                "sha256:1cac0a8f71a041aa587410424ad46dfa6a11f6149ceb219ce7dd48f6b02b87a7",
+                "sha256:1d59e739ab0e3520e62a26c60707cc3ab0365d2f8fecea74bfe4de72dc56388f",
+                "sha256:2855c8b0b55958265e8b5888d6a615ba02883b225f2227461aa9127c578a4922",
+                "sha256:2e785e0f2ef0d567099b9ff92cbfb958d71c2d5b9259981cd9bee81bd194c9a4",
+                "sha256:309be79c0a354afff9ff7da4aaed7c3257e77edf6c1b448a779329431ee79d7e",
+                "sha256:39f3ecaf76cd98e802f094e0d4fbc6dc9c45a8d0c4d185f0f6c2234e14e5f75b",
+                "sha256:457499c79fa84593f22454bbd27670227874cd2ff5d6c84e60575c8b50a69619",
+                "sha256:49e70d111fee47284d9dd867c9bb9a7058a3c617274900780c43e38d90fe1205",
+                "sha256:4c75507d0a55378240f781599c30e7776674dbaf883a46d1c90f37e563453480",
+                "sha256:4c863140fafc615c14a4bf4efd0f4425c02230eb8ef02784c9a156461e62c965",
+                "sha256:4d8908b3bee1c889e547867ca4cdc54e5ab6be6d3e078556814a22457f49423c",
+                "sha256:5b9eb0ca724a241683c9685a484da9d35c872fd42756574a7cfbf58af26677fd",
+                "sha256:6022cecf8f44e36af10bd9118ca71f371078b4c168b6e0fab43d4a889985dbb5",
+                "sha256:6150ffa5c767bc6332df27157d95442c379b7dce3a38dff89c0f39b63275696f",
+                "sha256:62828cada4a2b850dbef89c81f5a33741898b305db244904de418cc957ff05dc",
+                "sha256:7b4182299f251060996af5249c286bae9361fa8c6a9cda5efc29fe8bfd6062ec",
+                "sha256:94b34f32646ca0414237168d68a9157cb3889f06b096612afdd296003fdd32fd",
+                "sha256:9ce6889abac9a42afd07a562c2d6d4b2b7134f83f18571d859b25624a331c90b",
+                "sha256:9cffe0f850e89d7c0012a1fb8730f75edd4320a0a731ed0c183904fe6ecfc3a9",
+                "sha256:a12a813949e5066148712a0626895c26b2578874e4cc63160bb007e6df3436fe",
+                "sha256:a1eea9aecf761c661d096d39ed9026574de8adb2ae1c5bd7b33558af884fb2ce",
+                "sha256:a31f94983fecbac95e58388210427d68cd30fe8a36927980fab9c20062645609",
+                "sha256:ac58bdee53cbeba2ecad824fa8159493f0bf3b8ea4e93feb06c9a465d6c87da8",
+                "sha256:af3f4485884750dddd9c25cb7e3915d83c2db92488b38ccb77dd594eac84c4a0",
+                "sha256:b33d2bc4f69caedcd0a275329eb2198f560b325605810895627be5d4b876bf7f",
+                "sha256:b59c0ffceff8d4d3996a45f2bb6f4c207f94684a96bf3d9728dbb77428dd8cb8",
+                "sha256:bb6834cbd983b19f06908b45bfc2dad6ac9479ae04abe923a275b5f48f1a186b",
+                "sha256:bd3db01f59fdcbce5b22afad19e390260d6d0222f35a1023d9adc5690a889364",
+                "sha256:bd7c23df857d488f418439686d3b10ae2fbf9bc256cd045b37a8c16575ea1040",
+                "sha256:c2528d60e398c7c4c799d56f907664673a807635b857df18f7ae64d3e6ce2d9f",
+                "sha256:d31a63bc6e6d87f77d71e1abbd7387ab817a66733734883d1fc0021ed9bfa083",
+                "sha256:d4492d82b3bc7fbb7e3610747b159869468079fe149ec5c4d771fa1f614a14df",
+                "sha256:ddcb8581510311e13421b1f544403c16e901c4e8f09083c881fab2be80ee31ba",
+                "sha256:e1d59258c3c67c865435d8fbeb35f8c59b8bef3d6f46c1f29f6123556af28445",
+                "sha256:eb3315a8a236ee19b6df481fc5f997436e8ade24a9f03dfdc6bd490fea20c6da",
+                "sha256:ef2b055471c0eb466033760a521efb9d8a32b99ab907fc8358481a1dd29e3bd3",
+                "sha256:ef5adb9a3b1d0c645ff694f9bca7702ec2c70f4d734f9922ea34de02294fdf72",
+                "sha256:f32c38afb74bd98ce26de7cc74a67b40afb7b05aae7b42924ea990d51e4dac02",
+                "sha256:fe0ccca550bb8e5abc22f530ec0466136379c01321fd94f30a22231e8a48d985"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.2.0"
+            "version": "==1.2.1"
         },
         "cycler": {
             "hashes": [
@@ -618,67 +617,67 @@
         },
         "cython": {
             "hashes": [
-                "sha256:000dc9e135d0eec6ecb2b40a5b02d0868a2f8d2e027a41b0fe16a908a9e6de02",
-                "sha256:05d7eddc668ae7993643f32c7661f25544e791edb745758672ea5b1a82ecffa6",
-                "sha256:0c38c9f0bcce2df0c3347285863621be904ac6b64c5792d871130569d893efd7",
-                "sha256:0cb2dcc565c7851f75d496f724a384a790fab12d1b82461b663e66605bec429a",
-                "sha256:115f0a50f752da6c99941b103b5cb090da63eb206abbc7c2ad33856ffc73f064",
-                "sha256:13c2a5e57a0358da467d97667297bf820b62a1a87ae47c5f87938b9bb593acbd",
-                "sha256:16873d78be63bd38ffb759da7ab82814b36f56c769ee02b1d5859560e4c3ac3c",
-                "sha256:171b27051253d3f9108e9759e504ba59ff06e7f7ba944457f94deaf9c21bf0b6",
-                "sha256:17a642bb01a693e34c914106566f59844b4461665066613913463a719e0dd15d",
-                "sha256:18bfa387d7a7f77d7b2526af69a65dbd0b731b8d941aaff5becff8e21f6d7717",
-                "sha256:1ab75242869ff71e5665fe5c96f3378e79e792fa3c11762641b6c5afbbbbe026",
-                "sha256:1aca1b97e0095b3a9a6c33eada3f661a4ed0d499067d121239b193e5ba3bb4f0",
-                "sha256:289ce7838208211cd166e975865fd73b0649bf118170b6cebaedfbdaf4a37795",
-                "sha256:2cde23c555470db3f149ede78b518e8274853745289c956a0e06ad8d982e4db9",
-                "sha256:2cdfc32252f3b6dc7c94032ab744dcedb45286733443c294d8f909a4854e7f83",
-                "sha256:2f020fa1c0552052e0660790b8153b79e3fc9a15dbd8f1d0b841fe5d204a6ae6",
-                "sha256:314f2355a1f1d06e3c431eaad4708cf10037b5e91e4b231d89c913989d0bdafd",
-                "sha256:3a3d67f079598af49e90ff9655bf85bd358f093d727eb21ca2708f467c489cae",
-                "sha256:45523fdc2b78d79b32834cc1cc12dc2ca8967af87e22a3ee1bff20e77c7f5520",
-                "sha256:4b983c8e6803f016146c26854d9150ddad5662960c804ea7f0c752c9266752f0",
-                "sha256:51d1426263b0e82fb22bda8ea60dc77a428581cc19e97741011b938445d383f1",
-                "sha256:547eb3cdb2f8c6f48e6865d5a741d9dd051c25b3ce076fbca571727977b28ac3",
-                "sha256:5a567d4b9ba70b26db89d75b243529de9e649a2f56384287533cf91512705bee",
-                "sha256:61a237bc9dd23c7faef0fcfce88c11c65d0c9bb73c74ccfa408b3a012073c20e",
-                "sha256:6717c06e9cfc6c1df18543cd31a21f5d8e378a40f70c851fa2d34f0597037abc",
-                "sha256:6c46939c3983217d140999de7c238c3141f56b1ea349e47ca49cae899969aa2c",
-                "sha256:78825a3774211e7d5089730f00cdf7f473042acc9ceb8b9eeebe13ed3a5541de",
-                "sha256:7990ca127e1f1beedaf8fc8bf66541d066ef4723ad7d8d47a7cbf842e0f47580",
-                "sha256:7e8f2454128974905258d86534f4fd4f91d2f1343605657ecab779d80c9d6d5e",
-                "sha256:80fd94c076e1e1b1ee40a309be03080b75f413e8997cddcf401a118879863388",
-                "sha256:8140597a8b5cc4f119a1190f5a2228a84f5ca6d8d9ec386cfce24663f48b2539",
-                "sha256:8333423d8fd5765e7cceea3a9985dd1e0a5dfeb2734629e1a2ed2d6233d39de6",
-                "sha256:85077915a93e359a9b920280d214dc0cf8a62773e1f3d7d30fab8ea4daed670c",
-                "sha256:870d2a0a7e3cbd5efa65aecdb38d715ea337a904ea7bb22324036e78fb7068e7",
-                "sha256:90d3fe31db55685d8cb97d43b0ec39ef614fcf660f83c77ed06aa670cb0e164f",
-                "sha256:96b028f044f5880e3cb18ecdcfc6c8d3ce9d0af28418d5ab464509f26d8adf12",
-                "sha256:97b2a45845b993304f1799664fa88da676ee19442b15fdcaa31f9da7e1acc434",
-                "sha256:9d3f74388db378a3c6fd06e79a809ed98df3f56484d317b81ee762dbf3c263e0",
-                "sha256:9e2be2b340fea46fb849d378f9b80d3c08ff2e81e2bfbcdb656e2e3cd8c6b2dc",
-                "sha256:a1df7a129344b1215c20096d33c00193437df1a8fcca25b71f17c23b1a44f782",
-                "sha256:a846e0a38e2b24e9a5c5dc74b0e54c6e29420d88d1dafabc99e0fc0f3e338636",
-                "sha256:a973268d7ca1a2bdf78575e459a94a78e1a0a9bb62a7db0c50041949a73b02ff",
-                "sha256:aae26f9663e50caf9657148403d9874eea41770ecdd6caf381d177c2b1bb82ba",
-                "sha256:ae7ac561fd8253a9ae96311e91d12af5f701383564edc11d6338a7b60b285a6f",
-                "sha256:baa0b7f3f841fe087410cab66778e2d3fb20ae2d2078a2be3dffe66c6574be39",
-                "sha256:bfabe115deef4ada5d23c87bddb11289123336dcc14347011832c07db616dd93",
-                "sha256:c1949d6aa7bc792554bee2b67a9fe41008acbfe22f4f8df7b6ec7b799613a4b3",
-                "sha256:c26daaeccda072459b48d211415fd1e5507c06bcd976fa0d5b8b9f1063467d7b",
-                "sha256:c8aa05f5e17f8042a3be052c24f2edc013fb8af874b0bf76907d16c51b4e7871",
-                "sha256:c9c0f29246734561c90f36e70ed0506b61aa3d044e4cc4cba559065a2a741fae",
-                "sha256:c9f2c6e1b8f3bcd6cb230bac1843f85114780bb8be8614855b1628b36bb510e0",
-                "sha256:de892422582f5758bd8de187e98ac829330ec1007bc42c661f687792999988a7",
-                "sha256:df8093deabc55f37028190cf5e575c26aad23fc673f34b85d5f45076bc37ce39",
-                "sha256:e24791ddae2324e88e3c902a765595c738f19ae34ee66bfb1a6dac54b1833419",
-                "sha256:e87294e33e40c289c77a135f491cd721bd089f193f956f7b8ed5aa2d0b8c558f",
-                "sha256:f05c0bf9d085c031df8f583f0d506aa3be1692023de18c45d0aaf78685bbb944",
-                "sha256:fa97893d99385386925d00074654aeae3a98867f298d1e12ceaf38a9054a9bae",
-                "sha256:fe81b339cffd87c0069c6049b4d33e28bdd1874625ee515785bf42c9fdff3658"
+                "sha256:051069638abfb076900b0c2bcb6facf545655b3f429e80dd14365192074af5a4",
+                "sha256:076e9fd4e0ca33c5fa00a7479180dbfb62f17fe928e2909f82da814536e96d2b",
+                "sha256:077b61ee789e48700e25d4a16daa4258b8e65167136e457174df400cf9b4feab",
+                "sha256:09f2000041db482cad3bfce94e1fa3a4c82b0e57390a164c02566cbbda8c4f12",
+                "sha256:0bac3ccdd4e03924028220c62ae3529e17efa8ca7e9df9330de95de02f582b26",
+                "sha256:0e9a885ec63d3955a08cefc4eec39fefa9fe14989c6e5e2382bd4aeb6bdb9bc3",
+                "sha256:15b6d397f4ee5ad54e373589522af37935a32863f1b23fa8c6922adf833e28e2",
+                "sha256:206e803598010ecc3813db8748ed685f7beeca6c413f982df9f8a505fce56563",
+                "sha256:269f06e6961e8591d56e30b46e1a51b6ccb42cab04c29fa3b30d3e8723485fb4",
+                "sha256:2c9c1e3e78909488f3b16fabae02308423fa6369ed96ab1e250807d344cfffd7",
+                "sha256:2d29e617fd23cf4b83afe8f93f2966566c9f565918ad1e86a4502fe825cc0a79",
+                "sha256:32fbad02d1189be75eb96456d9c73f5548078e5338d8fa153ecb0115b6ee279f",
+                "sha256:35f6ede7c74024ed1982832ae61c9fad7cf60cc3f5b8c6a63bb34e38bc291936",
+                "sha256:38d40fa1324ac47c04483d151f5e092406a147eac88a18aec789cf01c089c3f2",
+                "sha256:3919a55ec9b6c7db6f68a004c21c05ed540c40dbe459ced5d801d5a1f326a053",
+                "sha256:3cffb666e649dba23810732497442fb339ee67ba4e0be1f0579991e83fcc2436",
+                "sha256:401aba1869a57aba2922ccb656a6320447e55ace42709b504c2f8e8b166f46e1",
+                "sha256:407840c56385b9c085826fe300213e0e76ba15d1d47daf4b58569078ecb94446",
+                "sha256:40fac59c3a7fbcd9c25aea64c342c890a5e2270ce64a1525e840807800167799",
+                "sha256:4f610964ab252a83e573a427e28b103e2f1dd3c23bee54f32319f9e73c3c5499",
+                "sha256:4fadb84193c25641973666e583df8df4e27c52cdc05ddce7c6f6510d690ba34a",
+                "sha256:541fbe725d6534a90b93f8c577eb70924d664b227a4631b90a6e0506d1469591",
+                "sha256:5a036d00caa73550a3a976432ef21c1e3fa12637e1616aab32caded35331ae96",
+                "sha256:5bd49a3a9fdff65446a3e1c2bfc0ec85c6ce4c3cad27cd4ad7ba150a62b7fb59",
+                "sha256:5f465443917d5c0f69825fca3b52b64c74ac3de0143b1fff6db8ba5b48c9fb4a",
+                "sha256:64f1f8bba9d8f37c0cffc934792b4ac7c42d0891077127c11deebe9fa0a0f7e4",
+                "sha256:651a15a8534ebfb9b58cb0b87c269c70984b6f9c88bfe65e4f635f0e3f07dfcd",
+                "sha256:6c5af936940a38c300977b81598d9c0901158f220a58c177820e17e1774f1cf1",
+                "sha256:712760879600907189c7d0d346851525545484e13cd8b787e94bfd293da8ccf0",
+                "sha256:81f356c1c8c0885b8435bfc468025f545c5d764aa9c75ab662616dd1193c331e",
+                "sha256:86998b01f6a6d48398df8467292c7637e57f7e3a2ca68655367f13f66fed7734",
+                "sha256:8adcde00a8a88fab27509b558cd8c2959ab0c70c65d3814cfea8c68b83fa6dcd",
+                "sha256:8c9c4c4f3ab8f8c02817b0e16e8fa7b8cc880f76e9b63fe9c010e60c1a6c2b13",
+                "sha256:8f2864ab5fcd27a346f0b50f901ebeb8f60b25a60a575ccfd982e7f3e9674914",
+                "sha256:90e2f514fc753b55245351305a399463103ec18666150bb1c36779b9862388e9",
+                "sha256:950c0c7b770d2a7cec74fb6f5ccc321d0b51d151f48c075c0d0db635a60ba1b5",
+                "sha256:9cc6a0e7e23a96dec3f3c9d39690d4281beabd5297855140d0d30855f950275e",
+                "sha256:9ea31184c7b3a728ef1f81fccb161d8948c05aa86c79f63b74fb6f3ddec860ec",
+                "sha256:9fa9e7786083b6aa61594c16979d621b62e61fcd9c2edd4761641b95c7fb34b2",
+                "sha256:a181144c2f893ed8e6a994d43d0b96300bc99873f21e3b7334ca26c61c37b680",
+                "sha256:a5e14a8c6a8157d2b0cdc2e8e3444905d20a0e78e19d2a097e89fb8b04b51f6b",
+                "sha256:a9bb402674788a7f4061aeef8057632ec440123e74ed0fb425308a59afdfa10e",
+                "sha256:a9c976e9ec429539a4367cb4b24d15a1e46b925976f4341143f49f5f161171f5",
+                "sha256:acfbe0fff364d54906058fc61f2393f38cd7fa07d344d80923937b87e339adcf",
+                "sha256:adc377aa33c3309191e617bf675fdbb51ca727acb9dc1aa23fc698d8121f7e23",
+                "sha256:b74b700d6a793113d03fb54b63bdbadba6365379424bac7c0470605672769260",
+                "sha256:bcc9795990e525c192bc5c0775e441d7d56d7a7d02210451e9e13c0448dba51b",
+                "sha256:d092c0ddba7e9e530a5c5be4ac06db8360258acc27675d1fc86294a5dc8994c5",
+                "sha256:d10fc9aa82e5e53a0b7fd118f9771199cddac8feb4a6d8350b7d4109085aa775",
+                "sha256:d4e83a8ceff7af60064da4ccfce0ac82372544dd5392f1b350c34f1b04d0fae6",
+                "sha256:dcc96739331fb854dcf503f94607576cfe8488066c61ca50dfd55836f132de99",
+                "sha256:e876272548d73583e90babda94c1299537006cad7a34e515a06c51b41f8657aa",
+                "sha256:e8df79b596633b8295eaa48b1157d796775c2bb078f32267d32f3001b687f2fd",
+                "sha256:f43a58bf2434870d2fc42ac2e9ff8138c9e00c6251468de279d93fa279e9ba3b",
+                "sha256:f4780d0f98ce28191c4d841c4358b5d5e79d96520650910cd59904123821c52d",
+                "sha256:f8a2b8fa0fd8358bccb5f3304be563c4750aae175100463d212d5ea0ec74cbe0",
+                "sha256:fc6e0faf5b57523b073f0cdefadcaef3a51235d519a0594865925cadb3aeadf0",
+                "sha256:fcbb679c0b43514d591577fd0d20021c55c240ca9ccafbdb82d3fb95e5edfee2"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.8"
+            "version": "==3.0.10"
         },
         "dask": {
             "hashes": [
@@ -755,6 +754,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.4"
         },
+        "et-xmlfile": {
+            "hashes": [
+                "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c",
+                "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.1.0"
+        },
         "exceptiongroup": {
             "hashes": [
                 "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14",
@@ -780,11 +787,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e",
-                "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"
+                "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f",
+                "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.13.1"
+            "version": "==3.13.4"
         },
         "fiona": {
             "hashes": [
@@ -836,51 +843,51 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:0452fcbfbce752ba596737a7c5ec5cf76bc5f83847ce1781f4f90eab14ece252",
-                "sha256:0a2417547462e468edf35b32e3dd06a6215ac26aa6316b41e03b8eeaf9f079ea",
-                "sha256:0d2b01428f7da26f229a5656defc824427b741e454b4e210ad2b25ed6ea2aed4",
-                "sha256:0d533f89819f9b3ee2dbedf0fed3825c425850e32bdda24c558563c71be0064e",
-                "sha256:12ee86abca46193359ea69216b3a724e90c66ab05ab220d39e3fc068c1eb72ac",
-                "sha256:18b35fd1a850ed7233a99bbd6774485271756f717dac8b594958224b54118b61",
-                "sha256:292922dc356d7f11f5063b4111a8b719efb8faea92a2a88ed296408d449d8c2e",
-                "sha256:2eb4167bde04e172a93cf22c875d8b0cff76a2491f67f5eb069566215302d45d",
-                "sha256:3cdb9a92521b81bf717ebccf592bd0292e853244d84115bfb4db0c426de58348",
-                "sha256:4108b1d247953dd7c90ec8f457a2dec5fceb373485973cc852b14200118a51ee",
-                "sha256:4709c5bf123ba10eac210d2d5c9027d3f472591d9f1a04262122710fa3d23199",
-                "sha256:5057ade278e67923000041e2b195c9ea53e87f227690d499b6a4edd3702f7f01",
-                "sha256:56339ec557f0c342bddd7c175f5e41c45fc21282bee58a86bd9aa322bec715f2",
-                "sha256:578c00f93868f64a4102ecc5aa600a03b49162c654676c3fadc33de2ddb88a81",
-                "sha256:594206b31c95fcfa65f484385171fabb4ec69f7d2d7f56d27f17db26b7a31814",
-                "sha256:63c73b9dd56a94a3cbd2f90544b5fca83666948a9e03370888994143b8d7c070",
-                "sha256:63dc592a16cd08388d8c4c7502b59ac74190b23e16dfc863c69fe1ea74605b68",
-                "sha256:6978bade7b6c0335095bdd0bd97f8f3d590d2877b370f17e03e0865241694eb5",
-                "sha256:6f30e605c7565d0da6f0aec75a30ec372072d016957cd8fc4469721a36ea59b7",
-                "sha256:702ae93058c81f46461dc4b2c79f11d3c3d8fd7296eaf8f75b4ba5bbf813cd5f",
-                "sha256:8b8a45254218679c7f1127812761e7854ed5c8e34349aebf581e8c9204e7495a",
-                "sha256:902e9c4e9928301912f34a6638741b8ae0b64824112b42aaf240e06b735774b1",
-                "sha256:97f0a49fa6aa2d6205c6f72f4f98b74ef4b9bfdcb06fd78e6fe6c7af4989b63e",
-                "sha256:9b4ec6d42a7555f5ae35f3b805482f0aad0f1baeeef54859492ea3b782959d4a",
-                "sha256:9b58638d8a85e3a1b32ec0a91d9f8171a877b4b81c408d4cb3257d0dee63e092",
-                "sha256:a8c8b54bd1420c184a995f980f1a8076f87363e2bb24239ef8c171a369d85a31",
-                "sha256:aee76fd81a8571c68841d6ef0da750d5ff08ff2c5f025576473016f16ac3bcf7",
-                "sha256:b10633aafc5932995a391ec07eba5e79f52af0003a1735b2306b3dab8a056d48",
-                "sha256:bcd77f89fc1a6b18428e7a55dde8ef56dae95640293bfb8f4e929929eba5e2a2",
-                "sha256:bff5b38d0e76eb18e0b8abbf35d384e60b3371be92f7be36128ee3e67483b3ec",
-                "sha256:c900508c46274d32d308ae8e82335117f11aaee1f7d369ac16502c9a78930b0a",
-                "sha256:cad5cfd044ea2e306fda44482b3dd32ee47830fa82dfa4679374b41baa294f5f",
-                "sha256:cdfd7557d1bd294a200bd211aa665ca3b02998dcc18f8211a5532da5b8fad5c5",
-                "sha256:cf5a0cd974f85a80b74785db2d5c3c1fd6cc09a2ba3c837359b2b5da629ee1b0",
-                "sha256:d10979ef14a8beaaa32f613bb698743f7241d92f437a3b5e32356dfb9769c65d",
-                "sha256:d20588466367f05025bb1efdf4e5d498ca6d14bde07b6928b79199c588800f0a",
-                "sha256:d3260db55f1843e57115256e91247ad9f68cb02a434b51262fe0019e95a98738",
-                "sha256:df48798f9a4fc4c315ab46e17873436c8746f5df6eddd02fad91299b2af7af95",
-                "sha256:e3e33862fc5261d46d9aae3544acb36203b1a337d00bdb5d3753aae50dac860e",
-                "sha256:e740a7602c2bb71e1091269b5dbe89549749a8817dc294b34628ffd8b2bf7124",
-                "sha256:f40441437b039930428e04fb05ac3a132e77458fb57666c808d74a556779e784",
-                "sha256:f7449493886da6a17472004d3818cc050ba3f4a0aa03fb47972e4fa5578e6703"
+                "sha256:0118ef998a0699a96c7b28457f15546815015a2710a1b23a7bf6c1be60c01636",
+                "sha256:0d145976194a5242fdd22df18a1b451481a88071feadf251221af110ca8f00ce",
+                "sha256:0e19bd9e9964a09cd2433a4b100ca7f34e34731e0758e13ba9a1ed6e5468cc0f",
+                "sha256:0f08c901d3866a8905363619e3741c33f0a83a680d92a9f0e575985c2634fcc1",
+                "sha256:1250e818b5f8a679ad79660855528120a8f0288f8f30ec88b83db51515411fcc",
+                "sha256:15c94eeef6b095831067f72c825eb0e2d48bb4cea0647c1b05c981ecba2bf39f",
+                "sha256:1621ee57da887c17312acc4b0e7ac30d3a4fb0fec6174b2e3754a74c26bbed1e",
+                "sha256:180194c7fe60c989bb627d7ed5011f2bef1c4d36ecf3ec64daec8302f1ae0716",
+                "sha256:278e50f6b003c6aed19bae2242b364e575bcb16304b53f2b64f6551b9c000e15",
+                "sha256:32b17504696f605e9e960647c5f64b35704782a502cc26a37b800b4d69ff3c77",
+                "sha256:3bee3f3bd9fa1d5ee616ccfd13b27ca605c2b4270e45715bd2883e9504735034",
+                "sha256:4060acc2bfa2d8e98117828a238889f13b6f69d59f4f2d5857eece5277b829ba",
+                "sha256:54dcf21a2f2d06ded676e3c3f9f74b2bafded3a8ff12f0983160b13e9f2fb4a7",
+                "sha256:56fc244f2585d6c00b9bcc59e6593e646cf095a96fe68d62cd4da53dd1287b55",
+                "sha256:599bdb75e220241cedc6faebfafedd7670335d2e29620d207dd0378a4e9ccc5a",
+                "sha256:5f6bc991d1610f5c3bbe997b0233cbc234b8e82fa99fc0b2932dc1ca5e5afec0",
+                "sha256:60a3409c9112aec02d5fb546f557bca6efa773dcb32ac147c6baf5f742e6258b",
+                "sha256:68b3fb7775a923be73e739f92f7e8a72725fd333eab24834041365d2278c3671",
+                "sha256:76f1777d8b3386479ffb4a282e74318e730014d86ce60f016908d9801af9ca2a",
+                "sha256:806e7912c32a657fa39d2d6eb1d3012d35f841387c8fc6cf349ed70b7c340039",
+                "sha256:84d7751f4468dd8cdd03ddada18b8b0857a5beec80bce9f435742abc9a851a74",
+                "sha256:865a58b6e60b0938874af0968cd0553bcd88e0b2cb6e588727117bd099eef836",
+                "sha256:8ac27f436e8af7779f0bb4d5425aa3535270494d3bc5459ed27de3f03151e4c2",
+                "sha256:8b4850fa2ef2cfbc1d1f689bc159ef0f45d8d83298c1425838095bf53ef46308",
+                "sha256:8b5ad456813d93b9c4b7ee55302208db2b45324315129d85275c01f5cb7e61a2",
+                "sha256:8e2f1a4499e3b5ee82c19b5ee57f0294673125c65b0a1ff3764ea1f9db2f9ef5",
+                "sha256:9696fe9f3f0c32e9a321d5268208a7cc9205a52f99b89479d1b035ed54c923f1",
+                "sha256:96a48e137c36be55e68845fc4284533bda2980f8d6f835e26bca79d7e2006438",
+                "sha256:a8feca65bab31479d795b0d16c9a9852902e3a3c0630678efb0b2b7941ea9c74",
+                "sha256:aefa011207ed36cd280babfaa8510b8176f1a77261833e895a9d96e57e44802f",
+                "sha256:b2b92381f37b39ba2fc98c3a45a9d6383bfc9916a87d66ccb6553f7bdd129097",
+                "sha256:b3c61423f22165541b9403ee39874dcae84cd57a9078b82e1dce8cb06b07fa2e",
+                "sha256:b5b48a1121117047d82695d276c2af2ee3a24ffe0f502ed581acc2673ecf1037",
+                "sha256:c18b49adc721a7d0b8dfe7c3130c89b8704baf599fb396396d07d4aa69b824a1",
+                "sha256:c5b8cab0c137ca229433570151b5c1fc6af212680b58b15abd797dcdd9dd5051",
+                "sha256:c7e91abdfae1b5c9e3a543f48ce96013f9a08c6c9668f1e6be0beabf0a569c1b",
+                "sha256:cadf4e12a608ef1d13e039864f484c8a968840afa0258b0b843a0556497ea9ed",
+                "sha256:dc0673361331566d7a663d7ce0f6fdcbfbdc1f59c6e3ed1165ad7202ca183c68",
+                "sha256:de7c29bdbdd35811f14493ffd2534b88f0ce1b9065316433b22d63ca1cd21f14",
+                "sha256:e9d9298be7a05bb4801f558522adbe2feea1b0b103d5294ebf24a92dd49b78e5",
+                "sha256:ee1af4be1c5afe4c96ca23badd368d8dc75f611887fb0c0dac9f71ee5d6f110e",
+                "sha256:f7e89853d8bea103c8e3514b9f9dc86b5b4120afb4583b57eb10dfa5afbe0936"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.48.1"
+            "version": "==4.51.0"
         },
         "fqdn": {
             "hashes": [
@@ -891,19 +898,19 @@
         },
         "fsspec": {
             "hashes": [
-                "sha256:817f969556fa5916bc682e02ca2045f96ff7f586d45110fcb76022063ad2c7d8",
-                "sha256:b6ad1a679f760dda52b1168c859d01b7b80648ea6f7f7c7f5a8a91dc3f3ecb84"
+                "sha256:918d18d41bf73f0e2b261824baeb1b124bcf771767e3a26425cd7dec3332f512",
+                "sha256:f39780e282d7d117ffb42bb96992f8a90795e4d0fb0f661a70ca39fe9c43ded9"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.2.0"
+            "version": "==2024.3.1"
         },
         "geocube": {
             "hashes": [
-                "sha256:0049913c9ca0e0ff8bd4473ab6a87d185e5c2457ee053a7184abcfe9e9447b57",
-                "sha256:4fbcdcbb187cbf02d9f03bec0c30628f5d5161cd3e9c51aaf0877c5f69539ed7"
+                "sha256:3cd80d0866fec649001529b6d19502937b3f70be4b480fac53e0785ef8ef27d8",
+                "sha256:aa77b078516649d35b3907fcc630fc6a3c22e4d8b2b52add9a09c6b18dd3c462"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.5.0"
+            "version": "==0.5.1"
         },
         "geographiclib": {
             "hashes": [
@@ -939,27 +946,28 @@
         },
         "identify": {
             "hashes": [
-                "sha256:a4316013779e433d08b96e5eabb7f641e6c7942e4ab5d4c509ebd2e7a8994aed",
-                "sha256:ee17bc9d499899bc9eaec1ac7bf2dc9eedd480db9d88b96d123d3b64a9d34f5d"
+                "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791",
+                "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.34"
+            "version": "==2.5.35"
         },
         "idna": {
             "hashes": [
-                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
-                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
+                "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
+                "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
             ],
+            "index": "pypi",
             "markers": "python_version >= '3.5'",
-            "version": "==3.6"
+            "version": "==3.7"
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e",
-                "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"
+                "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570",
+                "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.0.1"
+            "version": "==7.1.0"
         },
         "inflate64": {
             "hashes": [
@@ -1030,11 +1038,11 @@
         },
         "ipykernel": {
             "hashes": [
-                "sha256:3bade28004e3ff624ed57974948116670604ac5f676d12339693f3142176d3f0",
-                "sha256:50384f5c577a260a1d53f1f59a828c7266d321c9b7d00d345693783f66616055"
+                "sha256:1181e653d95c6808039c509ef8e67c4126b3b3af7781496c7cbfb5ed938a27da",
+                "sha256:3d44070060f9475ac2092b760123fadf105d2e2493c24848b6691a7c4f42af5c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.29.2"
+            "version": "==6.29.4"
         },
         "ipympl": {
             "hashes": [
@@ -1046,11 +1054,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:1050a3ab8473488d7eee163796b02e511d0735cf43a04ba2a8348bd0f2eaf8a5",
-                "sha256:48fbc236fbe0e138b88773fa0437751f14c3645fb483f1d4c5dee58b37e5ce73"
+                "sha256:07232af52a5ba146dc3372c7bf52a0f890a23edf38d77caef8d53f9cdc2584c1",
+                "sha256:7468edaf4f6de3e1b912e57f66c241e6fd3c7099f2ec2136e239e142e800274d"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.21.0"
+            "version": "==8.23.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1108,18 +1116,19 @@
         },
         "joblib": {
             "hashes": [
-                "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1",
-                "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"
+                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
+                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.3.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.0"
         },
         "json5": {
             "hashes": [
-                "sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f",
-                "sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02"
+                "sha256:0c638399421da959a20952782800e5c1a78c14e08e1dc9738fa10d8ec14d58c8",
+                "sha256:4ca101fd5c7cb47960c055ef8f4d0e31e15a7c6c48c3b6f1473fc83b6c462a13"
             ],
-            "version": "==0.9.14"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.9.24"
         },
         "jsonpointer": {
             "hashes": [
@@ -1174,27 +1183,27 @@
         },
         "jupyter-core": {
             "hashes": [
-                "sha256:c65c82126453a723a2804aa52409930434598fd9d35091d63dfb919d2b765bb7",
-                "sha256:de61a9d7fc71240f688b2fb5ab659fbb56979458dc66a71decd098e03c79e218"
+                "sha256:4f7315d2f6b4bcf2e3e7cb6e46772eba760ae459cd1f59d29eb57b0a01bd7409",
+                "sha256:aa5f8d32bbf6b431ac830496da7392035d6f61b4f54872f15c4bd2a9c3f536d9"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.7.1"
+            "version": "==5.7.2"
         },
         "jupyter-events": {
             "hashes": [
-                "sha256:81ad2e4bc710881ec274d31c6c50669d71bbaa5dd9d01e600b56faa85700d399",
-                "sha256:d853b3c10273ff9bc8bb8b30076d65e2c9685579db736873de6c2232dde148bf"
+                "sha256:4b72130875e59d57716d327ea70d3ebc3af1944d3717e5a498b8a06c6c159960",
+                "sha256:670b8229d3cc882ec782144ed22e0d29e1c2d639263f92ca8383e66682845e22"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.9.0"
+            "version": "==0.10.0"
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:0edb626c94baa22809be1323f9770cf1c00a952b17097592e40d03e6a3951689",
-                "sha256:184a0f82809a8522777cfb6b760ab6f4b1bb398664c5860a27cec696cb884923"
+                "sha256:77b2b49c3831fbbfbdb5048cef4350d12946191f833a24e5f83e5f8f4803e97b",
+                "sha256:c80bfb049ea20053c3d9641c2add4848b38073bf79f1729cea1faed32fc1c78e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.12.5"
+            "version": "==2.13.0"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1206,11 +1215,11 @@
         },
         "jupyter-server-terminals": {
             "hashes": [
-                "sha256:1b80c12765da979513c42c90215481bbc39bd8ae7c0350b4f85bc3eb58d0fa80",
-                "sha256:396b5ccc0881e550bf0ee7012c6ef1b53edbde69e67cab1d56e89711b46052e8"
+                "sha256:41ee0d7dc0ebf2809c668e0fc726dfaf258fcd3e769568996ca731b6194ae9aa",
+                "sha256:5ae0295167220e9ace0edcfdb212afd2b01ee8d179fe6f23c899590e9b8a5269"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.5.2"
+            "version": "==0.5.3"
         },
         "jupyter-server-ydoc": {
             "hashes": [
@@ -1246,11 +1255,11 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:5b1798c9cc6a44f65c757de9f97fc06fc3d42535afbf47d2ace5e964ab447aaf",
-                "sha256:bd0ec7a99ebcedc8bcff939ef86e52c378e44c2707e053fcd81d046ce979ee63"
+                "sha256:54622cbd330526a385ee0c1fdccdff3a1e7219bf3e864a335284a1270a1973df",
+                "sha256:9b3ba91cf2837f7f124fca36d63f3ca80ace2bed4898a63dd47e6598c1ab006f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.25.2"
+            "version": "==2.26.0"
         },
         "jupyterlab-widgets": {
             "hashes": [
@@ -1566,73 +1575,73 @@
         },
         "msgpack": {
             "hashes": [
-                "sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862",
-                "sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d",
-                "sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3",
-                "sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672",
-                "sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0",
-                "sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9",
-                "sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee",
-                "sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46",
-                "sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524",
-                "sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819",
-                "sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc",
-                "sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc",
-                "sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1",
-                "sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82",
-                "sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81",
-                "sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6",
-                "sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d",
-                "sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2",
-                "sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c",
-                "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87",
-                "sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84",
-                "sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e",
-                "sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95",
-                "sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f",
-                "sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b",
-                "sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93",
-                "sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf",
-                "sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61",
-                "sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c",
-                "sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8",
-                "sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d",
-                "sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c",
-                "sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4",
-                "sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba",
-                "sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415",
-                "sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee",
-                "sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d",
-                "sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9",
-                "sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075",
-                "sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f",
-                "sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7",
-                "sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681",
-                "sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329",
-                "sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1",
-                "sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf",
-                "sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c",
-                "sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5",
-                "sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b",
-                "sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5",
-                "sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e",
-                "sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b",
-                "sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad",
-                "sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd",
-                "sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7",
-                "sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002",
-                "sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc"
+                "sha256:00e073efcba9ea99db5acef3959efa45b52bc67b61b00823d2a1a6944bf45982",
+                "sha256:0726c282d188e204281ebd8de31724b7d749adebc086873a59efb8cf7ae27df3",
+                "sha256:0ceea77719d45c839fd73abcb190b8390412a890df2f83fb8cf49b2a4b5c2f40",
+                "sha256:114be227f5213ef8b215c22dde19532f5da9652e56e8ce969bf0a26d7c419fee",
+                "sha256:13577ec9e247f8741c84d06b9ece5f654920d8365a4b636ce0e44f15e07ec693",
+                "sha256:1876b0b653a808fcd50123b953af170c535027bf1d053b59790eebb0aeb38950",
+                "sha256:1ab0bbcd4d1f7b6991ee7c753655b481c50084294218de69365f8f1970d4c151",
+                "sha256:1cce488457370ffd1f953846f82323cb6b2ad2190987cd4d70b2713e17268d24",
+                "sha256:26ee97a8261e6e35885c2ecd2fd4a6d38252246f94a2aec23665a4e66d066305",
+                "sha256:3528807cbbb7f315bb81959d5961855e7ba52aa60a3097151cb21956fbc7502b",
+                "sha256:374a8e88ddab84b9ada695d255679fb99c53513c0a51778796fcf0944d6c789c",
+                "sha256:376081f471a2ef24828b83a641a02c575d6103a3ad7fd7dade5486cad10ea659",
+                "sha256:3923a1778f7e5ef31865893fdca12a8d7dc03a44b33e2a5f3295416314c09f5d",
+                "sha256:4916727e31c28be8beaf11cf117d6f6f188dcc36daae4e851fee88646f5b6b18",
+                "sha256:493c5c5e44b06d6c9268ce21b302c9ca055c1fd3484c25ba41d34476c76ee746",
+                "sha256:505fe3d03856ac7d215dbe005414bc28505d26f0c128906037e66d98c4e95868",
+                "sha256:5845fdf5e5d5b78a49b826fcdc0eb2e2aa7191980e3d2cfd2a30303a74f212e2",
+                "sha256:5c330eace3dd100bdb54b5653b966de7f51c26ec4a7d4e87132d9b4f738220ba",
+                "sha256:5dbf059fb4b7c240c873c1245ee112505be27497e90f7c6591261c7d3c3a8228",
+                "sha256:5e390971d082dba073c05dbd56322427d3280b7cc8b53484c9377adfbae67dc2",
+                "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273",
+                "sha256:64d0fcd436c5683fdd7c907eeae5e2cbb5eb872fafbc03a43609d7941840995c",
+                "sha256:69284049d07fce531c17404fcba2bb1df472bc2dcdac642ae71a2d079d950653",
+                "sha256:6a0e76621f6e1f908ae52860bdcb58e1ca85231a9b0545e64509c931dd34275a",
+                "sha256:73ee792784d48aa338bba28063e19a27e8d989344f34aad14ea6e1b9bd83f596",
+                "sha256:74398a4cf19de42e1498368c36eed45d9528f5fd0155241e82c4082b7e16cffd",
+                "sha256:7938111ed1358f536daf311be244f34df7bf3cdedb3ed883787aca97778b28d8",
+                "sha256:82d92c773fbc6942a7a8b520d22c11cfc8fd83bba86116bfcf962c2f5c2ecdaa",
+                "sha256:83b5c044f3eff2a6534768ccfd50425939e7a8b5cf9a7261c385de1e20dcfc85",
+                "sha256:8db8e423192303ed77cff4dce3a4b88dbfaf43979d280181558af5e2c3c71afc",
+                "sha256:9517004e21664f2b5a5fd6333b0731b9cf0817403a941b393d89a2f1dc2bd836",
+                "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3",
+                "sha256:99881222f4a8c2f641f25703963a5cefb076adffd959e0558dc9f803a52d6a58",
+                "sha256:9ee32dcb8e531adae1f1ca568822e9b3a738369b3b686d1477cbc643c4a9c128",
+                "sha256:a22e47578b30a3e199ab067a4d43d790249b3c0587d9a771921f86250c8435db",
+                "sha256:b5505774ea2a73a86ea176e8a9a4a7c8bf5d521050f0f6f8426afe798689243f",
+                "sha256:bd739c9251d01e0279ce729e37b39d49a08c0420d3fee7f2a4968c0576678f77",
+                "sha256:d16a786905034e7e34098634b184a7d81f91d4c3d246edc6bd7aefb2fd8ea6ad",
+                "sha256:d3420522057ebab1728b21ad473aa950026d07cb09da41103f8e597dfbfaeb13",
+                "sha256:d56fd9f1f1cdc8227d7b7918f55091349741904d9520c65f0139a9755952c9e8",
+                "sha256:d661dc4785affa9d0edfdd1e59ec056a58b3dbb9f196fa43587f3ddac654ac7b",
+                "sha256:dfe1f0f0ed5785c187144c46a292b8c34c1295c01da12e10ccddfc16def4448a",
+                "sha256:e1dd7839443592d00e96db831eddb4111a2a81a46b028f0facd60a09ebbdd543",
+                "sha256:e2872993e209f7ed04d963e4b4fbae72d034844ec66bc4ca403329db2074377b",
+                "sha256:e2f879ab92ce502a1e65fce390eab619774dda6a6ff719718069ac94084098ce",
+                "sha256:e3aa7e51d738e0ec0afbed661261513b38b3014754c9459508399baf14ae0c9d",
+                "sha256:e532dbd6ddfe13946de050d7474e3f5fb6ec774fbb1a188aaf469b08cf04189a",
+                "sha256:e6b7842518a63a9f17107eb176320960ec095a8ee3b4420b5f688e24bf50c53c",
+                "sha256:e75753aeda0ddc4c28dce4c32ba2f6ec30b1b02f6c0b14e547841ba5b24f753f",
+                "sha256:eadb9f826c138e6cf3c49d6f8de88225a3c0ab181a9b4ba792e006e5292d150e",
+                "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011",
+                "sha256:ef254a06bcea461e65ff0373d8a0dd1ed3aa004af48839f002a0c994a6f72d04",
+                "sha256:f3709997b228685fe53e8c433e2df9f0cdb5f4542bd5114ed17ac3c0129b0480",
+                "sha256:f51bab98d52739c50c56658cc303f190785f9a2cd97b823357e7aeae54c8f68a",
+                "sha256:f9904e24646570539a8950400602d66d2b2c492b9010ea7e965025cb71d0c86d",
+                "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.0.7"
+            "version": "==1.0.8"
         },
         "multimethod": {
             "hashes": [
-                "sha256:e57253f5b6d530b10696843e693c11f0dadbe299a327bd71cdf3edd0019a3853",
-                "sha256:ef40713d84da4015285122a16f57bf5066cc7876b5eaf183262f3c34668c6ad3"
+                "sha256:7f2a4863967142e6db68632fef9cd79053c09670ba0c5f113301e245140bba5c",
+                "sha256:cb338f09395c0ee87d36c7691cdd794d13d8864358082cf1205f812edd5ce05a"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.11"
+            "version": "==1.11.2"
         },
         "multipledispatch": {
             "hashes": [
@@ -1683,27 +1692,27 @@
         },
         "nbclient": {
             "hashes": [
-                "sha256:4b28c207877cf33ef3a9838cdc7a54c5ceff981194a82eac59d558f05487295e",
-                "sha256:a3a1ddfb34d4a9d17fc744d655962714a866639acd30130e9be84191cd97cd15"
+                "sha256:4b3f1b7dba531e498449c4db4f53da339c91d449dc11e9af3a43b4eb5c5abb09",
+                "sha256:f13e3529332a1f1f81d82a53210322476a168bb7090a0289c795fe9cc11c9d3f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.9.0"
+            "version": "==0.10.0"
         },
         "nbconvert": {
             "hashes": [
-                "sha256:813e6553796362489ae572e39ba1bff978536192fb518e10826b0e8cadf03ec8",
-                "sha256:ad3dc865ea6e2768d31b7eb6c7ab3be014927216a5ece3ef276748dd809054c7"
+                "sha256:a6733b78ce3d47c3f85e504998495b07e6ea9cf9bf6ec1c98dda63ec6ad19142",
+                "sha256:ddeff14beeeedf3dd0bc506623e41e4507e551736de59df69a91f86700292b3b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.16.0"
+            "version": "==7.16.3"
         },
         "nbformat": {
             "hashes": [
-                "sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9",
-                "sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192"
+                "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a",
+                "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.9.2"
+            "version": "==5.10.4"
         },
         "nest-asyncio": {
             "hashes": [
@@ -1764,11 +1773,11 @@
         },
         "notebook-shim": {
             "hashes": [
-                "sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7",
-                "sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9"
+                "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef",
+                "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.2.3"
+            "version": "==0.2.4"
         },
         "numba": {
             "hashes": [
@@ -1806,38 +1815,38 @@
         },
         "numexpr": {
             "hashes": [
-                "sha256:00dab81d49239ea5423861ad627097b44d10d802df5f883d1b00f742139c3349",
-                "sha256:0e2574cafb18373774f351cac45ed23b5b360d9ecd1dbf3c12dac6d6eefefc87",
-                "sha256:0f619e91034b346ea85a4e1856ff06011dcb7dce10a60eda75e74db90120f880",
-                "sha256:1a78b937861d13de67d440d54c85a835faed7572be5a6fd10d4f3bd4e66e157f",
-                "sha256:1fae6828042b70c2f52a132bfcb9139da704274ed11b982fbf537f91c075d2ef",
-                "sha256:2749bce1c48706d58894992634a43b8458c4ba9411191471c4565fa41e9979ec",
-                "sha256:374dc6ca54b2af813cb15c2b34e85092dfeac1f73d51ec358dd81876bd9adcec",
-                "sha256:37a7dd36fd79a2b69c3fd2bc2b51ac8270bebc69cc96e6d78f1148e147fcbfa8",
-                "sha256:3b03a6cf37a72f5b52f2b962d7ac7f565bea8eaba83c3c4e5fcf8fbb6a938153",
-                "sha256:50f57bc333f285e8c46b1ce61c6e94ec9bb74e4ea0d674d1c6c6f4a286f64fe4",
-                "sha256:520e55d75bd99c76e376b6326e35ecf44c5ce2635a5caed72799a3885fc49173",
-                "sha256:549afc1622296cca3478a132c6e0fb5e55a19e08d32bc0d5a415434824a9c157",
-                "sha256:5615497c3f34b637fda9b571f7774b6a82f2367cc1364b7a4573068dd1aabcaa",
-                "sha256:6eae6c0c2d5682c02e8ac9c4287c2232c2443c9148b239df22500eaa3c5d73b7",
-                "sha256:7c618a5895e34db0a364dcdb9960084c080f93f9d377c45b1ca9c394c24b4e77",
-                "sha256:7c77392aea53f0700d60eb270ad63174b4ff10b04f8de92861101ca2129fee51",
-                "sha256:8efd879839572bde5a38a1aa3ac23fd4dd9b956fb969bc5e43d1c403419e1e8c",
-                "sha256:943ba141f3884ffafa3fa1a3ebf3cdda9e9688a67a3c91986e6eae13dc073d43",
-                "sha256:972e29b5cecc21466c5b177e38568372ab66aab1f053ae04690a49cea09e747d",
-                "sha256:9761195526a228e05eba400b8c484c94bbabfea853b9ea35ab8fa1bf415331b1",
-                "sha256:977537f2a1cc843f888fb5f0507626f956ada674e4b3847168214a3f3c7446fa",
-                "sha256:aa6298fb46bd7ec69911b5b80927a00663d066e719b29f48eb952d559bdd8371",
-                "sha256:b04f12a6130094a251e3a8fff40130589c1c83be6d4eb223873bea14d8c8b630",
-                "sha256:bffcbc55dea5a5f5255e2586da08f00929998820e6592ee717273a08ad021eb3",
-                "sha256:c52b4ac54514f5d4d8ead66768810cd5f77aa198e6064213d9b5c7b2e1c97c35",
-                "sha256:d655b6eacc4e81006b662cba014e4615a9ddd96881b8b4db4ad0d7f6d38069af",
-                "sha256:e1c31f621a625c7be602f92b027d90f2d3d60dcbc19b106e77fb04a4362152af",
-                "sha256:ee48acd6339748a65c0e32403b802ebfadd9cb0e3b602ba5889896238eafdd61",
-                "sha256:f21d12f6c432ce349089eb95342babf6629aebb3fddf187a4492d3aadaadaaf0"
+                "sha256:03d0ba492e484a5a1aeb24b300c4213ed168f2c246177be5733abb4e18cbb043",
+                "sha256:04e8620e7e676504201d4082e7b3ee2d9b561d1cb9470b47a6104e10c1e2870e",
+                "sha256:05278bad96b5846d712eba58b44e5cec743bdb3e19ca624916c921d049fdbcf6",
+                "sha256:10789450032357afaeda4ac4d06da9542d1535c13151e8d32b49ae1a488d1358",
+                "sha256:1af6dc6b3bd2e11a802337b352bf58f30df0b70be16c4f863b70a3af3a8ef95e",
+                "sha256:1d8eb88b0ae3d3c609d732a17e71096779b2bf47b3a084320ffa93d9f9132786",
+                "sha256:3c66dc0188358cdcc9465b6ee54fd5eef2e83ac64b1d4ba9117c41df59bf6fca",
+                "sha256:416e0e9f0fc4cced67767585e44cb6b301728bdb9edbb7c534a853222ec62cac",
+                "sha256:4f0b045e1831953a47cc9fabae76a6794c69cbb60921751a5cf2d555034c55bf",
+                "sha256:4feafc65ea3044b8bf8f305b757a928e59167a310630c22b97a57dff07a56490",
+                "sha256:56d0d96b130f7cd4d78d0017030d6a0e9d9fc2a717ac51d4cf4860b39637e86a",
+                "sha256:629b66cc1b750671e7fb396506b3f9410612e5bd8bc1dd55b5a0a0041d839f95",
+                "sha256:6b5f8242c075477156d26b3a6b8e0cd0a06d4c8eb68d907bde56dd3c9c683e92",
+                "sha256:745b46a1fb76920a3eebfaf26e50bc94a9c13b5aee34b256ab4b2d792dbaa9ca",
+                "sha256:748e8d4cde22d9a5603165293fb293a4de1a4623513299416c64fdab557118c2",
+                "sha256:78e0a8bc4417c3dedcbae3c473505b69080535246edc977c7dccf3ec8454a685",
+                "sha256:83f1e7a7f7ee741b8dcd20c56c3f862a3a3ec26fa8b9fcadb7dcd819876d2f35",
+                "sha256:937d36c6d3cf15601f26f84f0f706649f976491e9e0892d16cd7c876d77fa7dc",
+                "sha256:96a64d0dd8f8e694da3f8582d73d7da8446ff375f6dd239b546010efea371ac3",
+                "sha256:a602692cd52ce923ce8a0a90fb1d6cf186ebe8706eed83eee0de685e634b9aa9",
+                "sha256:a6cdf9e64c5b3dbb61729edb505ea75ee212fa02b85c5b1d851331381ae3b0e1",
+                "sha256:b276e2ba3e87ace9a30fd49078ad5dcdc6a1674d030b1ec132599c55465c0346",
+                "sha256:c7517b774d309b1f0896c89bdd1ddd33c4418a92ecfbe5e1df3ac698698f6fcf",
+                "sha256:c89e930752639df040539160326d8f99a84159bbea41943ab8e960591edaaef0",
+                "sha256:cb5e12787101f1216f2cdabedc3417748f2e1f472442e16bbfabf0bab2336300",
+                "sha256:d47bb567e330ebe86781864219a36cbccb3a47aec893bd509f0139c6b23e8104",
+                "sha256:dc3506c30c03b082da2cadef43747d474e5170c1f58a6dcdf882b3dc88b1e849",
+                "sha256:e3a973265591b0a875fd1151c4549e468959c7192821aac0bb86937694a08efa",
+                "sha256:efa63ecdc9fcaf582045639ddcf56e9bdc1f4d9a01729be528f62df4db86c9d6"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==2.9.0"
+            "version": "==2.10.0"
         },
         "numpy": {
             "hashes": [
@@ -1889,6 +1898,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.4.1"
         },
+        "openpyxl": {
+            "hashes": [
+                "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184",
+                "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"
+            ],
+            "index": "pypi",
+            "version": "==3.1.2"
+        },
         "overrides": {
             "hashes": [
                 "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a",
@@ -1954,19 +1971,19 @@
         },
         "param": {
             "hashes": [
-                "sha256:785845a727a588eb94c7666d80551c7e2bb97d4309d3507beab66f95e57f7527",
-                "sha256:b269fd7397886ec609e544f81035fa52e1950da0e76d20080bfeca3d7a0317ca"
+                "sha256:a7b30b08b547e2b78b02aeba6ed34e3c6a638f8e4824a76a96ffa2d7cf57e71f",
+                "sha256:f31d3745d227347d29b5868c4e4e3077df07463889b91d3bb28e634fde211e1c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.0.2"
+            "version": "==2.1.0"
         },
         "parso": {
             "hashes": [
-                "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0",
-                "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"
+                "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18",
+                "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.8.3"
+            "version": "==0.8.4"
         },
         "partd": {
             "hashes": [
@@ -1989,82 +2006,83 @@
                 "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523",
                 "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"
             ],
-            "markers": "sys_platform != 'win32'",
+            "markers": "sys_platform != 'win32' and sys_platform != 'emscripten'",
             "version": "==4.9.0"
         },
         "pillow": {
             "hashes": [
-                "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8",
-                "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39",
-                "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac",
-                "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869",
-                "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e",
-                "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04",
-                "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9",
-                "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e",
-                "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe",
-                "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef",
-                "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56",
-                "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa",
-                "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f",
-                "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f",
-                "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e",
-                "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a",
-                "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2",
-                "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2",
-                "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5",
-                "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a",
-                "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2",
-                "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213",
-                "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563",
-                "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591",
-                "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c",
-                "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2",
-                "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb",
-                "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757",
-                "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0",
-                "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452",
-                "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad",
-                "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01",
-                "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f",
-                "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5",
-                "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61",
-                "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e",
-                "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b",
-                "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068",
-                "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9",
-                "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588",
-                "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483",
-                "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f",
-                "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67",
-                "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7",
-                "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311",
-                "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6",
-                "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72",
-                "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6",
-                "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129",
-                "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13",
-                "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67",
-                "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c",
-                "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516",
-                "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e",
-                "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e",
-                "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364",
-                "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023",
-                "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1",
-                "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04",
-                "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d",
-                "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a",
-                "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7",
-                "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb",
-                "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4",
-                "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e",
-                "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1",
-                "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48",
-                "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"
+                "sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c",
+                "sha256:048eeade4c33fdf7e08da40ef402e748df113fd0b4584e32c4af74fe78baaeb2",
+                "sha256:0ba26351b137ca4e0db0342d5d00d2e355eb29372c05afd544ebf47c0956ffeb",
+                "sha256:0ea2a783a2bdf2a561808fe4a7a12e9aa3799b701ba305de596bc48b8bdfce9d",
+                "sha256:1530e8f3a4b965eb6a7785cf17a426c779333eb62c9a7d1bbcf3ffd5bf77a4aa",
+                "sha256:16563993329b79513f59142a6b02055e10514c1a8e86dca8b48a893e33cf91e3",
+                "sha256:19aeb96d43902f0a783946a0a87dbdad5c84c936025b8419da0a0cd7724356b1",
+                "sha256:1a1d1915db1a4fdb2754b9de292642a39a7fb28f1736699527bb649484fb966a",
+                "sha256:1b87bd9d81d179bd8ab871603bd80d8645729939f90b71e62914e816a76fc6bd",
+                "sha256:1dfc94946bc60ea375cc39cff0b8da6c7e5f8fcdc1d946beb8da5c216156ddd8",
+                "sha256:2034f6759a722da3a3dbd91a81148cf884e91d1b747992ca288ab88c1de15999",
+                "sha256:261ddb7ca91fcf71757979534fb4c128448b5b4c55cb6152d280312062f69599",
+                "sha256:2ed854e716a89b1afcedea551cd85f2eb2a807613752ab997b9974aaa0d56936",
+                "sha256:3102045a10945173d38336f6e71a8dc71bcaeed55c3123ad4af82c52807b9375",
+                "sha256:339894035d0ede518b16073bdc2feef4c991ee991a29774b33e515f1d308e08d",
+                "sha256:412444afb8c4c7a6cc11a47dade32982439925537e483be7c0ae0cf96c4f6a0b",
+                "sha256:4203efca580f0dd6f882ca211f923168548f7ba334c189e9eab1178ab840bf60",
+                "sha256:45ebc7b45406febf07fef35d856f0293a92e7417ae7933207e90bf9090b70572",
+                "sha256:4b5ec25d8b17217d635f8935dbc1b9aa5907962fae29dff220f2659487891cd3",
+                "sha256:4c8e73e99da7db1b4cad7f8d682cf6abad7844da39834c288fbfa394a47bbced",
+                "sha256:4e6f7d1c414191c1199f8996d3f2282b9ebea0945693fb67392c75a3a320941f",
+                "sha256:4eaa22f0d22b1a7e93ff0a596d57fdede2e550aecffb5a1ef1106aaece48e96b",
+                "sha256:50b8eae8f7334ec826d6eeffaeeb00e36b5e24aa0b9df322c247539714c6df19",
+                "sha256:50fd3f6b26e3441ae07b7c979309638b72abc1a25da31a81a7fbd9495713ef4f",
+                "sha256:51243f1ed5161b9945011a7360e997729776f6e5d7005ba0c6879267d4c5139d",
+                "sha256:5d512aafa1d32efa014fa041d38868fda85028e3f930a96f85d49c7d8ddc0383",
+                "sha256:5f77cf66e96ae734717d341c145c5949c63180842a545c47a0ce7ae52ca83795",
+                "sha256:6b02471b72526ab8a18c39cb7967b72d194ec53c1fd0a70b050565a0f366d355",
+                "sha256:6fb1b30043271ec92dc65f6d9f0b7a830c210b8a96423074b15c7bc999975f57",
+                "sha256:7161ec49ef0800947dc5570f86568a7bb36fa97dd09e9827dc02b718c5643f09",
+                "sha256:72d622d262e463dfb7595202d229f5f3ab4b852289a1cd09650362db23b9eb0b",
+                "sha256:74d28c17412d9caa1066f7a31df8403ec23d5268ba46cd0ad2c50fb82ae40462",
+                "sha256:78618cdbccaa74d3f88d0ad6cb8ac3007f1a6fa5c6f19af64b55ca170bfa1edf",
+                "sha256:793b4e24db2e8742ca6423d3fde8396db336698c55cd34b660663ee9e45ed37f",
+                "sha256:798232c92e7665fe82ac085f9d8e8ca98826f8e27859d9a96b41d519ecd2e49a",
+                "sha256:81d09caa7b27ef4e61cb7d8fbf1714f5aec1c6b6c5270ee53504981e6e9121ad",
+                "sha256:8ab74c06ffdab957d7670c2a5a6e1a70181cd10b727cd788c4dd9005b6a8acd9",
+                "sha256:8eb0908e954d093b02a543dc963984d6e99ad2b5e36503d8a0aaf040505f747d",
+                "sha256:90b9e29824800e90c84e4022dd5cc16eb2d9605ee13f05d47641eb183cd73d45",
+                "sha256:9797a6c8fe16f25749b371c02e2ade0efb51155e767a971c61734b1bf6293994",
+                "sha256:9d2455fbf44c914840c793e89aa82d0e1763a14253a000743719ae5946814b2d",
+                "sha256:9d3bea1c75f8c53ee4d505c3e67d8c158ad4df0d83170605b50b64025917f338",
+                "sha256:9e2ec1e921fd07c7cda7962bad283acc2f2a9ccc1b971ee4b216b75fad6f0463",
+                "sha256:9e91179a242bbc99be65e139e30690e081fe6cb91a8e77faf4c409653de39451",
+                "sha256:a0eaa93d054751ee9964afa21c06247779b90440ca41d184aeb5d410f20ff591",
+                "sha256:a2c405445c79c3f5a124573a051062300936b0281fee57637e706453e452746c",
+                "sha256:aa7e402ce11f0885305bfb6afb3434b3cd8f53b563ac065452d9d5654c7b86fd",
+                "sha256:aff76a55a8aa8364d25400a210a65ff59d0168e0b4285ba6bf2bd83cf675ba32",
+                "sha256:b09b86b27a064c9624d0a6c54da01c1beaf5b6cadfa609cf63789b1d08a797b9",
+                "sha256:b14f16f94cbc61215115b9b1236f9c18403c15dd3c52cf629072afa9d54c1cbf",
+                "sha256:b50811d664d392f02f7761621303eba9d1b056fb1868c8cdf4231279645c25f5",
+                "sha256:b7bc2176354defba3edc2b9a777744462da2f8e921fbaf61e52acb95bafa9828",
+                "sha256:c78e1b00a87ce43bb37642c0812315b411e856a905d58d597750eb79802aaaa3",
+                "sha256:c83341b89884e2b2e55886e8fbbf37c3fa5efd6c8907124aeb72f285ae5696e5",
+                "sha256:ca2870d5d10d8726a27396d3ca4cf7976cec0f3cb706debe88e3a5bd4610f7d2",
+                "sha256:ccce24b7ad89adb5a1e34a6ba96ac2530046763912806ad4c247356a8f33a67b",
+                "sha256:cd5e14fbf22a87321b24c88669aad3a51ec052eb145315b3da3b7e3cc105b9a2",
+                "sha256:ce49c67f4ea0609933d01c0731b34b8695a7a748d6c8d186f95e7d085d2fe475",
+                "sha256:d33891be6df59d93df4d846640f0e46f1a807339f09e79a8040bc887bdcd7ed3",
+                "sha256:d3b2348a78bc939b4fed6552abfd2e7988e0f81443ef3911a4b8498ca084f6eb",
+                "sha256:d886f5d353333b4771d21267c7ecc75b710f1a73d72d03ca06df49b09015a9ef",
+                "sha256:d93480005693d247f8346bc8ee28c72a2191bdf1f6b5db469c096c0c867ac015",
+                "sha256:dc1a390a82755a8c26c9964d457d4c9cbec5405896cba94cf51f36ea0d855002",
+                "sha256:dd78700f5788ae180b5ee8902c6aea5a5726bac7c364b202b4b3e3ba2d293170",
+                "sha256:e46f38133e5a060d46bd630faa4d9fa0202377495df1f068a8299fd78c84de84",
+                "sha256:e4b878386c4bf293578b48fc570b84ecfe477d3b77ba39a6e87150af77f40c57",
+                "sha256:f0d0591a0aeaefdaf9a5e545e7485f89910c977087e7de2b6c388aec32011e9f",
+                "sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27",
+                "sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a"
             ],
             "index": "pypi",
-            "version": "==10.2.0"
+            "version": "==10.3.0"
         },
         "platformdirs": {
             "hashes": [
@@ -2092,11 +2110,11 @@
         },
         "prometheus-client": {
             "hashes": [
-                "sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1",
-                "sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92"
+                "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89",
+                "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.19.0"
+            "version": "==0.20.0"
         },
         "prompt-toolkit": {
             "hashes": [
@@ -2325,10 +2343,11 @@
         },
         "pycparser": {
             "hashes": [
-                "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
-                "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
+                "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6",
+                "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"
             ],
-            "version": "==2.21"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.22"
         },
         "pycryptodomex": {
             "hashes": [
@@ -2475,11 +2494,11 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb",
-                "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"
+                "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad",
+                "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"
             ],
             "markers": "python_full_version >= '3.6.8'",
-            "version": "==3.1.1"
+            "version": "==3.1.2"
         },
         "pyppmd": {
             "hashes": [
@@ -2616,11 +2635,11 @@
         },
         "python-dateutil": {
             "hashes": [
-                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
-                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+                "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
+                "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.8.2"
+            "version": "==2.9.0.post0"
         },
         "python-dotenv": {
             "hashes": [
@@ -2784,133 +2803,133 @@
         },
         "pyzstd": {
             "hashes": [
-                "sha256:00c188704141c709da96cc4a79f058d51f5318e839d6f904c7cc9badcf78e98e",
-                "sha256:013321ddaff083b24e43a8b06303446771978343b488ed73adf56c70a46e2783",
-                "sha256:02c95d7109052c985b7d90dac6f6010bc0630227f15aec16302162107137bdbc",
-                "sha256:0a4334e972109bdd17fb40dbdd9fcca6137648cab416fca505a2dcd186f50533",
-                "sha256:12668ceb8329aaa908b4d907d3a77bb748ff28b309c3b105c995a8715d535d2b",
-                "sha256:14121a4d95070f54bdc9a80dab1dd8fd9093907a1e687926447ca69b5b40a4d5",
-                "sha256:145ca5ed6240af2cbfc09faa50aada8aacf1e2928ed6dd9da1d6b8ebe39cdc4c",
-                "sha256:1b9cda5314982d64c856f9298be0d9bf69fbff0ca514d1651037616354b473ff",
-                "sha256:1cbf212253abd65e6451acdfb608adafe98ad8f05462fb9a054ddab816545caa",
-                "sha256:1dbe76b6d8fe75f6dbec24793fc07b1d1ae9464de9941138d5b9668f7670e6b0",
-                "sha256:209a92fbe892bd69cde58ffcb4861468e2c3c2d0626763e16e122bb55cb1fb1a",
-                "sha256:20f2dd56d46441cd9277077060c34c0b9ce3469412665ea5ccd506dd2708d994",
-                "sha256:23695dabdfd5081beab25754dc0105b42fbd2085a7c293901bcb45045969c5ec",
-                "sha256:250dad90140a6faea4cef555f339b6ceaad5cf03ed1127b8d06de214ff0db2e7",
-                "sha256:289e25871fe232d2482c0985a75a1faa7c92e10a6c3e3914d165f62d005d0aa6",
-                "sha256:2919afd114fd12309ed2f831ef6e95730ebf13c2a92d258ad055769d00ef4d7a",
-                "sha256:29e452caaf0de9cc17319225921d8c28cdc7a879948e990ff1e7735e7f976517",
-                "sha256:305c232462dbe80d0ee5ec91b1b0ec9153ec6ba6393d5348741af5d30b07ef52",
-                "sha256:31f60f01884350aec24e7a68f3ad089151b7a636490203c41a1a7c8e0cddd9b8",
-                "sha256:3351ad2feb51dcbb936defd47cab00d6f114214f224636503ed08298f30164c9",
-                "sha256:346f835e368e1051f8ea187ad9b49759cf6249c9ebf2f2a3861e435a568104b8",
-                "sha256:370b34a7c2f9c53cee494028daa5a7264690e1756a89c3855fd0be5ad298ec30",
-                "sha256:3a26df749589d898cd3253d2139eb85b867ddffc49286059c8bdb3cb9ce9b545",
-                "sha256:3bc0e7e2cccf78e562ab416daf68448b6552a5b6450a1ff3e15cabfc19254883",
-                "sha256:3f0fe2ef7ebc6e9b347585e414c4fefd32270ba8bdf9eb82496f3030cbdca465",
-                "sha256:3f72f310b10b730cddfb654006ae497e7706c81e6a7642d3da7fd2439df7d88d",
-                "sha256:40bdb468281a5cd525e2e990b97344f0974e0589bd1b395501c25471fcd7edda",
-                "sha256:418e9a676cc7ce00edd2fd044ee063c8639fd8cd6897ffda395a152cdc66ec97",
-                "sha256:4358dd80b315c82d760b44c6df7857c9c898d04e7b0c14abb0eb3692354e9379",
-                "sha256:441078bfd3b508597415338af667c3575980364f1286eedde58291558b9c2832",
-                "sha256:47c2a4c319300c381f194274203f47b12c433e1fd86b90ecdc7fb258c630f93b",
-                "sha256:49c57ae18f138a4b66480b2364fe6a0f2345ada919e93fc729c95c6b17ec73a4",
-                "sha256:4a0dcb32ac4d1d67a77ae6a2d60ea0921af7e682b3427202d8acb8e86642391c",
-                "sha256:4ed01beb31d5177456ec2c4b66591a0df83dbc72df29f05f40502bfefe47bbe4",
-                "sha256:50ccbaafee80b4f1c5c55bbe07f80871b9b8fe3499bf7357dde2c23fb1c2ac0e",
-                "sha256:51607d7d44f94a364ef0e3ccf9a92390def0faf6e7572eef082f15c657b5d03a",
-                "sha256:52dcae42f32f7a25c6b90bd479f3d04902700e3214e8fffe1bfe70053eb35ccb",
-                "sha256:5345c7a697327e2fa7c37534bb2968ea84595d8ec7fc8c4a60216ec1be6e65bd",
-                "sha256:542808d88464d538f5d2c6b48b545a7fe15f0d20c7fa703b469d039a08c9fa10",
-                "sha256:5453ebe42a2c7462fa532fd03cbf64e5c6baf5508b3089736c78444148d3c593",
-                "sha256:5819d502dacd54114c30bc24efcb76e723b93f8f528be70851056a396a792c46",
-                "sha256:5aed5fc86d0bfc5f16e871cbb35ec93df61476d7fde4c1c6081015a075ecfbc1",
-                "sha256:5d9ec8634ab0cbfbcff535ac07555ebdae0282ad66762f0471fad11c16181e33",
-                "sha256:5fb00c706d0b59c53124f982bd84b7d46866a8ea2a7670aaaa1ab4dbe6001b50",
-                "sha256:5fd7cf79949174d1018b896638f88aea1ff2a969f87a6199ea23b25b506e26c5",
-                "sha256:606b2452d78f0f731566d392f8d83cd012c2ffadb2cb2e2903fdd360c1faac8a",
-                "sha256:6128cb653d011f3781554b70ce1f1f388cd516820fbaf8fd03ee245ecaa48349",
-                "sha256:639935b5b3d9ed3911493504581254b76cb578279302f7f340924ac5bfca4090",
-                "sha256:64564f4c175c5bb8e744de5816d69ee0b940e472160a5e665f30adc412b694f3",
-                "sha256:69f12ce4866a3725138e97f22f2c4cb21d3ae18cd422906cd57ed12a9ffd86c5",
-                "sha256:6a60ee6836599363a24367cf780ad45446b07eba49ec72d19bad761d5414aca7",
-                "sha256:6b9af8d62c087354abd071e01d9445ea51b31779c8a4a0d5c14ee12caee3d18f",
-                "sha256:6c456882baab2a48a5bfabe458a557af25d0768ff29acbe200461e84c0f697d5",
-                "sha256:6f281cc2f096530c339f122e0d9866545f5592dd9bffe0fade565c2771130a45",
-                "sha256:73877eebbdcb8259cf0099665f8c8274d4273b361371405a611fb6bd9f4d64f6",
-                "sha256:74455bd918e7bc9883e3178a1a8fe796308670f0ee4488c80a0d9514e13807a1",
-                "sha256:7452ae7e6d80e697d78d3f56d1b4d2a350286eea229afb35f55ab88b934b6acd",
-                "sha256:77294f0f797c97a46ffb3daff1fe097c9d5aa9f96867333978e6791286963e50",
-                "sha256:78c38850af6b990e8ec1bc87b48f73ed5cc633f4baaa7bbc78f9b2f4449cf081",
-                "sha256:7ac886e04f253960ae82e38ded8352085c61d78de99412d178a94ecf475b5e5f",
-                "sha256:7c420878726d677da7484f6021dbe7e1f9345a791b155de632c6ce36678fb621",
-                "sha256:836f1d85a4b5d3689d455aeb1dc6c42acb96aaf8e5282825c00ccf2545ad5630",
-                "sha256:84aa6eecba967bdac167451501dcaceec548d8b8c4ca7fa41ceda4dbfc279297",
-                "sha256:866ba6ce85f337fa1677516217b6f10fc25e19acb6e17a501d5822e66396bdd5",
-                "sha256:86e0e65e205793b337d62d9764700dfd02b5f83b01e26ad345736e7ac0554ebd",
-                "sha256:87a1a4ca93da414f3b6da8131e61aca6d48a4e837fb0b1cbde05ae9d13332317",
-                "sha256:8d3a1b6fa71a0ae7abc320d9db91b5a96a71eef1dbee0d62a6232b71c97af962",
-                "sha256:8f9eb97fb6fd4551ff9d5012b4fcee9abeea9c8af6b9e3ebc3c76cc2bd0a43a7",
-                "sha256:91453ce9476363d777b2ea2e9c6dccecd2073cf35697e048de2e8d47e1f36c7c",
-                "sha256:922f1bb8ef80c42a2fca297ba0b03442c143a9a1f717e83db79f190514888803",
-                "sha256:937f118fdd7a23654886634f650d6502a2dd12c8a8e2bf14beb2fa5fa95058bf",
-                "sha256:9596aeb8c71192f4fba1ca25cec420da195219398d2df811d5082559efd9561f",
-                "sha256:960ab83a977a44284c4ffab2820ccd6c9b332571a3d622fefa4b29b0a5de72b0",
-                "sha256:9638d40ec02a5b194a4c98a5b6e36cdfde4e9d6b721ae6167ef8e57d2e69002f",
-                "sha256:97e05f66c5847e6889594508298d78ddb84a0115e9234d598415dc5a06d3a4a7",
-                "sha256:9ac634753f6d26cba503cea7bb5b350aec7c5366f44fa68c79e9c90be9fd0ebc",
-                "sha256:9e1097d8b57f64878a3f176f4cd6b9a1bbe9fb2d236f1a85a4357722626d8f25",
-                "sha256:a1b81cc86b69ff530d45e735ed479e14704999f534ad28a39f04be4a8fe2b91f",
-                "sha256:a4f786f1b1ab39a0908db04ebe5b2c7cbc6f1ce07a27d3a12eb980bffd7fea7d",
-                "sha256:a594795ef89bd83297c860ff585f2d25580ce9805eb9cc44c831d311e7f1951a",
-                "sha256:a708b9e6ff1826504940beb6b5c2c9dfd4e3b55c16ab88a4572f5b9dbb64cc56",
-                "sha256:a90b901ccfd24b028faea19c927ff03f3cfefe82ba0b931fbb8da4ef0664911b",
-                "sha256:ae3d0575721a372c20130681bfaf873225fd9e1c290b7d56b7e0c14f413318f6",
-                "sha256:afef9eb882cf3b395eef9c85b737a4acd09528975e6a5d9faedf28874ca65f52",
-                "sha256:aff1b469187f6c789cdf17cd95c9b24e87396dc86953b1cf38b9a05cea873c80",
-                "sha256:b2ae8993f3863632d31ca8921c8a5dc9ecc5551c7b88895cefb5a26d17643391",
-                "sha256:b2dd39e12f7467a7422ce50711524759d4d22016714cbae6a7096b954bc2fa32",
-                "sha256:b4de7741d542a477387299bf9450e8be3e768c352d6b3438254eb02af1e59462",
-                "sha256:b5b517fbbc5d223fc36041673e7c2a0d3a82be6a5464a5f0599069330b76f97d",
-                "sha256:bdc09de97b1b3f6c3d87fec04d6fe29dd4fefe6b354ad2d822fc369b8aa0942b",
-                "sha256:c249741b10eb714578d765487b767e0e7fcc2ac84a299209a6073566e730dbea",
-                "sha256:c2b093a74b10232c70b5d29814fcee6544bb6f30e2d922d26db9ab4b4cd00c04",
-                "sha256:c31f6dd5bd60688d51487a3f5e2ae29ed1948926e44d7a2316b193b083f80d5d",
-                "sha256:c36dbbf71480f1fffeaeca901adb31e0c7d59270a239eca63fe26e4647b7aca8",
-                "sha256:c41e5457f4de5d38a270bc44619873589bbe6fe251225deec583ed20199df0f3",
-                "sha256:c46e77c2ad614a0399503dc675d72436cbf6332a20d49a0e5bad03058d6cbfad",
-                "sha256:c8d1966e38c220d5940f8cb6303651af261f0bcfce77218a030b1a24ec986e2f",
-                "sha256:c9589cb79d4e401630481755c92b072aa7ba5505ec81dec865ef43932ec037e4",
-                "sha256:ca19213785f864781848e0216cba07e97f563f60a50bbc7885b54461d8c64873",
-                "sha256:cbfdde6c5768ffa5d2f14127bbc1d7c3c2d03c0ceaeb0736946197e06275ccc7",
-                "sha256:cd6a8d43a0c294918e3afb7e4b1d8c04d2e4c3ea9ddf05475fdaf366c7e5b3a6",
-                "sha256:cfa981cedd54bb8862d9033440a0afac38845db89e7099ceeb4f4d064dffd2f8",
-                "sha256:cffaab46f9e04856dc3daa6097bfb3d3bea0b1771237e869c57b13f3dcc2c238",
-                "sha256:d0929302d187bfeca335b7f710f774f1b2ea3f610b2a80e8a1ac2da216cd9766",
-                "sha256:d44a7d4586f02b630658298c089ff755e74d0677b93c71e09d33dd35bdd4987a",
-                "sha256:d7ddbf234c9adc72189bb552d830e9a0c2c4401b5baf7b003eacd5c552ddcc00",
-                "sha256:da070933d4bcfcbf58472da12ffa77c9fbc90efb39e21a9b74eb04b5af4b412a",
-                "sha256:dca286c6c1ca5febf13f5f2ae7e8aa7536e49bd07f4232796651a43ff741ceca",
-                "sha256:dcb2172ca8b62f82af9d1f8db80c21c64c5ba3991935caefde88bb378f0afb51",
-                "sha256:e4e00c1600022b47ef0e9e1f893cb0c2322209ec6c1581a3e3f63ed78330ddf0",
-                "sha256:e789e19095b818f7126180b4387c0f01700c3ad2378a4e7649b2ddf4bf47ffbc",
-                "sha256:e79babb67b415aa54abb213897ceaa011515a5f3e146a2a97f4e6486b9743af4",
-                "sha256:e8f75e839ee253af60b03d9957182fdd069dfaebb62b4e999bd74016f4e120bb",
-                "sha256:e9934277abdddf9c733267e4dcc4886de8a3302d28f390237d447e215e8ce47d",
-                "sha256:ef3399e0544b46d31c2a8ff14ae1fb3c3571ae1153bbbc5ddf0d242c67bde624",
-                "sha256:f169e166774587227255f6ffe71f5b3303ea73cde0e2c6d52e53b9e12c03d787",
-                "sha256:f1d8b58f00137ccbe8b828a5ede92be3f0115cef75e6bed88d4d0bd1e7a0b1fc",
-                "sha256:f2839c13e486e4a23b19b1d2dc4624565cec6c228bbf803c066be1106515966b",
-                "sha256:f66790e4b2dcfcabc0aa54dd89317ea5671cabf06aa93cbef7cbdd4d2fdb7ee3",
-                "sha256:f6d8a881b50bb2015e9bdba5edb0331e85d41ff44ab33cde551047480b98d748",
-                "sha256:f73821d429bfbb04645b80ec491ab05b35078f031f9fa3273fbf9027d1406233",
-                "sha256:f7cfc683d320402d61205a196ace77f15dcfd16b5771f8b9ffaf406868c98e78",
-                "sha256:f9c5fc29a5b9d61a8f0a3494172107e0e6cf23d0cb800d6285c6722ba7fc3535",
-                "sha256:fc92a718bccb8ce5c9eb63fca743c38f3fa4c4e47f58f0c4ada51b2474668184"
+                "sha256:013b206c22b85512d59dd05f6684c02bf9db45e93f637facb5ce165c2447fbc4",
+                "sha256:015a6794196fcb75ddaaae8186f0dbdcbda4e84451361a808b3e96c8c8a63296",
+                "sha256:0561710b88d4895a7bb202335e802125da6a71c556b1292fd620224b2652f496",
+                "sha256:0691e54fb29a4d1f4f2a0064afaf5d1525824bf139d83f403e5ba5ee630ac4f5",
+                "sha256:0779ccd652113d75cd461fa6581149550b0da55e72b1a6e28e6f49e9978c2d41",
+                "sha256:13515a6abca6751482396b656e5b72279bd405fd6c71b5bc899fcb526d40e564",
+                "sha256:150a7c7004b9df00f7cbb6dad0bdbab5d77c8c54b0857b8b3e50beae2810a041",
+                "sha256:18167c8a16709631eaa2f4fd27e2a7570b4c37ec460b65ff098e948af2d37f33",
+                "sha256:1c3d0e784ea9b66b06c7b76895aa4c1a90975980b1ae136fa8bbd8fb2b93e9b3",
+                "sha256:1e045dc688bb618cf2a2e33cb487c0d8af5e31a4eaec18c9399b34e2aed58777",
+                "sha256:24d061047634971801649ff9aced68bb8403c6220b07c82aa2c0e90d8a61acd0",
+                "sha256:2e6fd189c85f2efe94d3945dfbb85cdbc4e213757be75aa9a45e186625498ddf",
+                "sha256:330062103d25e0d67681fcf5ef8692c71cd979dc17ed98f6cee3b3ba8bd86622",
+                "sha256:330d54a0e5886fa5ca21e3960b9352a7b21f8dd85f98d4b7c4e8a7d48ade6998",
+                "sha256:3443df2dcdfe6e57c2600617e8f377772703b7f907fe065b49948c05ace80fd7",
+                "sha256:357e6545203754e83a0431663aa86cae873abe160923bdf3db07e88f03100b05",
+                "sha256:39a912430c73aa585ede2dc7c9f09c5334862953f26a9a86ac048c3805ed6bb1",
+                "sha256:3aa0ad903da666cf88d8a993e5fe9b1bfa0e5792a53212335af0a89baa268911",
+                "sha256:3b9105a4977247ea8664b113345e66bf4b3ffd321a07797efc6b020ff363b39e",
+                "sha256:3f174c47c370641c5dba74d4fd99374ca0eadd47e43e7f76ee8b76dacdfa2406",
+                "sha256:4094ea7b80081bb024eb887c80eeb6bd8ae24dc2862cc61ae9eadb5bca11dcd6",
+                "sha256:41e0da402bf0d68067be2f16f65b9d1d7d7e6d726f27762226cba7a8badd618a",
+                "sha256:455d2258f247968d18a9eaeb11e214e048acc976a61f72a0282d4c40051458b8",
+                "sha256:468f12eb61cbef394ff412c86d8bc85b90f24dff19f675c864df88a1a398a3a3",
+                "sha256:49c99120f9ea0c512019948e55ba17e60dadb1d3578890d53b2a09858da4e1dd",
+                "sha256:4a5b54ebec1e606821192c86a3b85c05c3f789d33bf44f383fd245f148f8f93b",
+                "sha256:4cd0bd21e0c7721e9f4ea54d533d9196771ab1d6ae0a83a96564dc46147a26eb",
+                "sha256:4dfd0d97ac231c08551543d61d123cf44bdd2fc20ba5ebce1fb0c466036be15a",
+                "sha256:4ea6a794b8a4e0590bea1599e724d4877d34640058caf5c0b3f916ded06fc3a1",
+                "sha256:4ff57d2a67aaff47eb60c37c5476af2a5056319aa2a5cab4b276f78b764be785",
+                "sha256:50079bcfd087c8a8822c9fe940c887244cf2276ca0a4f0b55bec006b564b9541",
+                "sha256:50186fa41c72929d21727dcc6cb87bef0adf09df9e982325a0ae5bfd20a9456b",
+                "sha256:502ec85cb2c0a35350a196e25d5e1350d400102706eba05d7926eadb64ffa976",
+                "sha256:51c644157319e2ca87521bd9694df7f651aac3273e015f35cdfb47cb8597ce37",
+                "sha256:527bfd3e01b8dafa1c24cba1a0455a85bbcdb41a3eefe59e2849634f9206d4f0",
+                "sha256:53172f0781b3db9321a1286bb274121668c3fe560d3bddfa82f585475f7145fa",
+                "sha256:5494515f3d0df1eb54c9f4ce7b9ea0b285bc1ba87eba295604da5f8de711bdea",
+                "sha256:5942e8b7ad869ea5f44d928053d3eda16e877b5fed5326786388f6a377545ab3",
+                "sha256:5ae5f8ab4accc980fa8e6696a402176e20a717ab2b296cd8d62b6a86cafc5f9c",
+                "sha256:6023de5cc800c073cbf6a10a7aad0cbae0c1849c759bb5263a78d9ac774b167b",
+                "sha256:609cc4bed06942e51ebdfa9cc74629e1d19de2339161b65d555d01b9820aec46",
+                "sha256:6217f30c4341619c6066a044ca118f53a3121b5813a56257b1f9818e24450584",
+                "sha256:62a1614558478f7d1014eec94a9ae3bf21c95766ecbe0a4a1ab16d2f1e4a6b67",
+                "sha256:62b8c3801d739a9592c0503dd294fab7f8cd2b1d637429babe13671bedf7d6f7",
+                "sha256:63a90494aeea75d3298941b0b8ddd505cbb54d6b915e579368db21d93454e41d",
+                "sha256:656107ef6ff0966df2a58828f165a17b0b40977358512ade259c52c18bff0a13",
+                "sha256:65e99e57f5fb42ed391fc2f7ffa4b8cce5a32dfd74f52e03bd36bb2ac3b56536",
+                "sha256:6aed97ef51e20ea4fd7e12c4b9e3199cd8e4664054f5d84c3d83e6f4d3144055",
+                "sha256:6bfb81f36fd56664dbbd64a0af241797424c76861faa2682aca89de110fce5d4",
+                "sha256:6d6f7fcfef9166670df4b6710d1105b891efba68ec523f56c64cc8c7aa6c8e88",
+                "sha256:6e742a691f4fa73cadbe7f866b33e33161ede5bc777eee443a62504adcadd65f",
+                "sha256:6ed32af20eeeb7f8307b091fdd0ee8f5f8733dde7e8f560a3115c98a6ac33a61",
+                "sha256:6fa6495ec52ff29b097e6d1b619588ab61e998b198882ec622d947856d4a85b4",
+                "sha256:793077870191d54889cc472eda3bfd84c8cae5ed61c9fa9fb2ce29c31e23f812",
+                "sha256:7b2ac847e8d2ca55be038d29d20d6bfe86c03e66ac2d9741701e4f9eef2abda9",
+                "sha256:7bb791f16c810c98865c373202a44b7b313d01a2a7a6186c9dcc0ac1a0d156dc",
+                "sha256:801ab496766a858cc54b37a66f9d0169d590628e69bde2166f76c35dfc20f64f",
+                "sha256:80ca483ee2084d36dc570004b6793f99c58d2df201c630f60518bca3518faa5d",
+                "sha256:81154db47a9aa901d09a8cebcd1f6559dbbd02a93a16bc203dc3f94c06bea25a",
+                "sha256:8195b3fe2ed91db03f969c3c5cf6a9bcb3a961749caec94479e9e7a96d7af14f",
+                "sha256:83603a97fdbcf2139f475c940789f09e32703f931f29f4a8ddf3551e6700108b",
+                "sha256:846050f1d7520350d26076df12b8a78b540780d7d3478845be79d57e2e6d3612",
+                "sha256:85582502d3df1db8077475d88d0bf5f35b8ddb8abf9badd8625781024da5f346",
+                "sha256:8575d340ffcd89e4558448eec23c252da584f0de4528a73ed45a01f12d6700fd",
+                "sha256:88e8cbd8fdd2af2f94615c32ab113c5cff4dbd188454394c4cac9941d2f72204",
+                "sha256:8e3dff90b6abc56331a886b5067c3b6bc3fad1b6df5766247197b699afd013b7",
+                "sha256:8e4123de14a8dd7bccb1faed374c1f6c87111766279bbba5d6c4c52e39c383d9",
+                "sha256:8f356263a898b2c9961a2661ebb975f2fd20868d2c4fc4a165226fa7d36e1c76",
+                "sha256:91da52897e5219dc440d3b9d290fdee66fb70c65ae594555c8d2c887df77f2de",
+                "sha256:91e8a0852784c758b8de4179c61e80abae997155aa2b324cbee28dcaf9084b6f",
+                "sha256:92abe38cdd7ea3f98a0f1a4521e9e7e70aa001713bff62a5b97fae41b0a39d0d",
+                "sha256:94973e29acf813a572adf3ed2342a0998a41331781d7be82a0a1b02646181e34",
+                "sha256:94d970caaaf3f59d6c7aa0aaf38e755641351f99452c400fc06b7e365ab1620c",
+                "sha256:957751dd7bba26e651c689fc5f41bb269712efcbf19a97ad4f912f0f792e6e95",
+                "sha256:9593a76f516312ec59dbeb73a9f1915d1f67610c054a7b488912b972cbfbcb7e",
+                "sha256:965e04aeb83f594577ea4e8c975765d0953a75c8d6e3cd193ae25ef574bd83ee",
+                "sha256:99902df14861e4f40ed1eb403a17356188428d12bb10d53de10d1ea24996b838",
+                "sha256:9dca0cde0f02e08e11431fff0e5408ac46420912165d828bc148a3c8cedc8852",
+                "sha256:9fa20eb8f710a9d36801451d1cbcfc83d1dd9e400527ad9aba7e7cc635c2729b",
+                "sha256:a07742d42409c3ce98c58f2b155e2d8acdcd8ff49955062f5155dae94b24060a",
+                "sha256:a15a3ef14cb929b07e0a98639f2ebcd69a0038d6e31367157018ec841a54d8fa",
+                "sha256:a22c4b26fb6134ece45e875515961bc9c1777df775f3488800bea55a381df91e",
+                "sha256:a62e98375385ade756d1ea4975f79ec103c515b412884d146c98295c5300fb4a",
+                "sha256:a900a3dcda64b289cf35acfb6c90246b5ae9f2dd4bc05da8dcb8fbd04c91e37b",
+                "sha256:ad30429f499c30e8e19ed389b8e25116d43c63cafdb97d4f5617e94e671209c5",
+                "sha256:adb895a0081a932132aa3a233cfeba682a07ac0dc36b062152f240406eecfba1",
+                "sha256:b1752344c77dbb137244e6ab5af5a7576b8e8b8e5d440c58d935e69f83606e0d",
+                "sha256:b19cdecdc19ad863673709d27ddb0adc104b9ad341f2785ab23c51c9ab39cf3f",
+                "sha256:b62c408ddac3ae113b9dc4867fd2bb32a2ef669c68f256eb5486c02b31603afa",
+                "sha256:b856928e99a1d3080929b0f194c7922c7a7979b0eb21ad9d057f000dcf1a6ac7",
+                "sha256:ba0f1c0201845b769c92b2b3e7666010abe14d6e9dbf4214d04cdf23cbdcd1ae",
+                "sha256:bafef40c8b521ef9df81a118e278d5759c0127395883dcc10bf80ee32a46da39",
+                "sha256:be591f405d82f40acbe2a081f1dee5478a3cf1b694013cc5712e84159311e36a",
+                "sha256:c20a4ee0706db5c9f403e4c459fc81145f48cb20fccc1a64acfb6143f1a7b1fb",
+                "sha256:c2d2a7324fbd118dc951b63b1fc553aec0fc0c430474a6ab7b33608b9606658a",
+                "sha256:c309d50ee6a41053ad4c284295e5c5eedfd084ff94a5ee1148cf0dc64e3223fb",
+                "sha256:c6f67ac9d3ff65ca466821d573b3cb2fb01f0cd9eb082e92f49dfd88fa828a77",
+                "sha256:c7f7a398bcb0f4cfbaf0ddb3e831c43857701c63c66f48eb30011326ed03b0fa",
+                "sha256:c867d446b22a7288229a812da6f211d86934beb47edf8098e2e0dbe9e2529f86",
+                "sha256:ca3df18c269f9c11b40dde85336d3fc4a879867b05c74efdd4d5f8e64ac7e179",
+                "sha256:ca80bb0b8400888e16e8eaa01b506de7d1bbdc8cc78c9ff75272526fd36fcb8a",
+                "sha256:cccfb03f829ecde3ee126befb4fe827e2228a50e8d9eaa0275c4dd21a0e59765",
+                "sha256:ce8e8a4aa8246c615693b9d0504e1df98157ae48907a044baa80f80abecc44ad",
+                "sha256:cfa03e12c0c1bd357a0e1f20adae9c32bc166dcc6ed3be65885134830899b6ba",
+                "sha256:d077f76179de41cdd5ae39b0a6f16f033b077f27fa54a9e038d03e3f5ddeff23",
+                "sha256:d2cc02e94ff4df6a4a871bbd69149ba5fbc5150108a208c6cdd9bc728000995b",
+                "sha256:d55d9e1e22ebfa9cde606a7967eb21fbc65d0d3be600e550c38a1c05094c8015",
+                "sha256:d62a54dd438b51dd9f94bd88dd2f7c9d4093aadc44892143182d51f8488ab5f6",
+                "sha256:d62f357845f90d5327bdd3286ebd79c40b23b8e7c36772fc2949ef54a92bc6de",
+                "sha256:db3d4e991ee727f6c0a54fe975ebe169be3a463ce810cfd8e6edbe8a90ddeb8f",
+                "sha256:dc917e488a9bdc681cb4a9b0ed3d0c8b81816bc1a6b1f85ec4050c827ffa9e76",
+                "sha256:deb41298fb0c916875413373e5eba162e39309d8730be736b3a206d9b73afebf",
+                "sha256:dec319cdbba5ed3dfb2a8798364a8899751e732127c6ae74bc41276899e0311a",
+                "sha256:e0262bba03bb4d38b697c2039a8e48a9e889fb9ae9b727903e8d9c49d32883a4",
+                "sha256:e0f1f590a66769caea6cf47c3c5d06d004f01659449595d4bc737a42d4670bf0",
+                "sha256:e21208605759dc2eeebf607a002e1ddc09548c12f25886d2a2dd8efdf35535a3",
+                "sha256:e2dbb36017e5c7d7499dfc05f8d6ed781fce9fbfa45cc7bd888ec5f871aba3d3",
+                "sha256:e69d3d5257607a3b29f12192ad7bc9fdbdc8aa5f7e11c6f536cb61ee02dcd698",
+                "sha256:e71bb03c754934d7792aa0559b16f9effe1d4196f5834715c115363809f00c37",
+                "sha256:e7c4530ec6bf3a27c3cba229026fa3669e6e1832fdc37cf74c73b113f884cb2d",
+                "sha256:e9d44c5cfe2516a652131ddd91c3a48924e34f4a7b85640cda44694afbeee6cb",
+                "sha256:ea1fe83079c67c3f1549a735f11060350c87453bd86a8442bccb1f9b14c26aef",
+                "sha256:eb4d96fe5d6d8bf5c8c64d8a37edff55e6d9abbf84d9b8fd0fe8e0db1a557625",
+                "sha256:f0822c9f4852f9f6b1458f8fbf011bb887cefba792b0c7c10659e2c9c997d1c9"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.15.9"
+            "version": "==0.15.10"
         },
         "qtconsole": {
             "hashes": [
@@ -2961,11 +2980,11 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:39240f2ecc770258f28b642dd47fd74bc8b02484de54e1882b74b35ebd779bd5",
-                "sha256:c775fedf74bc0f9189c2a3be1c12fd03e8c23f4d371dce795df44e06c5b412f7"
+                "sha256:5773bd84ef41799a5a8ca72dc34590c041eb01bf9aa02632b4a973fb0181a844",
+                "sha256:d53ae300ceddd3169f1ffa9caf2cb7b769e92657e4fafb23d34b93679116dfd4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.33.0"
+            "version": "==0.34.0"
         },
         "requests": {
             "hashes": [
@@ -3020,116 +3039,116 @@
         },
         "rioxarray": {
             "hashes": [
-                "sha256:75c41e5836a9adba2a42a2cdbc62a50b81e01bd4fcc9f55a96f187322182d671",
-                "sha256:c6d30d8b1b12924dffc45c4d55764840039fa0c8b551589d6cc1364e158e6055"
+                "sha256:9dc57a66b3551644322bba6c4d5d4c4647b0e41c481c9f03b287c1d9e16407b6",
+                "sha256:d12441e696dbc8a6113658e75326009ef6fddd3f865f4bcdd659d7fcfe4d8c7f"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.15.1"
+            "version": "==0.15.3"
         },
         "rpds-py": {
             "hashes": [
-                "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147",
-                "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7",
-                "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2",
-                "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68",
-                "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1",
-                "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382",
-                "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d",
-                "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921",
-                "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38",
-                "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4",
-                "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a",
-                "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d",
-                "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518",
-                "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e",
-                "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d",
-                "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf",
-                "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5",
-                "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba",
-                "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6",
-                "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59",
-                "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253",
-                "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6",
-                "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f",
-                "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3",
-                "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea",
-                "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1",
-                "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76",
-                "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93",
-                "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad",
-                "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad",
-                "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc",
-                "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049",
-                "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d",
-                "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90",
-                "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d",
-                "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd",
-                "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25",
-                "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2",
-                "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f",
-                "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6",
-                "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4",
-                "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c",
-                "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8",
-                "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d",
-                "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b",
-                "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19",
-                "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453",
-                "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9",
-                "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde",
-                "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296",
-                "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58",
-                "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec",
-                "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99",
-                "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a",
-                "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb",
-                "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383",
-                "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d",
-                "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896",
-                "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc",
-                "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6",
-                "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b",
-                "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7",
-                "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22",
-                "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf",
-                "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394",
-                "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0",
-                "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57",
-                "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74",
-                "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83",
-                "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29",
-                "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9",
-                "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f",
-                "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745",
-                "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb",
-                "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811",
-                "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55",
-                "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342",
-                "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23",
-                "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82",
-                "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041",
-                "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb",
-                "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066",
-                "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55",
-                "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6",
-                "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a",
-                "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140",
-                "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b",
-                "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9",
-                "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256",
-                "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c",
-                "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772",
-                "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4",
-                "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae",
-                "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920",
-                "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a",
-                "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b",
-                "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361",
-                "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8",
-                "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a"
+                "sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f",
+                "sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c",
+                "sha256:08231ac30a842bd04daabc4d71fddd7e6d26189406d5a69535638e4dcb88fe76",
+                "sha256:08f9ad53c3f31dfb4baa00da22f1e862900f45908383c062c27628754af2e88e",
+                "sha256:0ab39c1ba9023914297dd88ec3b3b3c3f33671baeb6acf82ad7ce883f6e8e157",
+                "sha256:0af039631b6de0397ab2ba16eaf2872e9f8fca391b44d3d8cac317860a700a3f",
+                "sha256:0b8612cd233543a3781bc659c731b9d607de65890085098986dfd573fc2befe5",
+                "sha256:11a8c85ef4a07a7638180bf04fe189d12757c696eb41f310d2426895356dcf05",
+                "sha256:1374f4129f9bcca53a1bba0bb86bf78325a0374577cf7e9e4cd046b1e6f20e24",
+                "sha256:1d4acf42190d449d5e89654d5c1ed3a4f17925eec71f05e2a41414689cda02d1",
+                "sha256:1d9a5be316c15ffb2b3c405c4ff14448c36b4435be062a7f578ccd8b01f0c4d8",
+                "sha256:1df3659d26f539ac74fb3b0c481cdf9d725386e3552c6fa2974f4d33d78e544b",
+                "sha256:22806714311a69fd0af9b35b7be97c18a0fc2826e6827dbb3a8c94eac6cf7eeb",
+                "sha256:2644e47de560eb7bd55c20fc59f6daa04682655c58d08185a9b95c1970fa1e07",
+                "sha256:2e6d75ab12b0bbab7215e5d40f1e5b738aa539598db27ef83b2ec46747df90e1",
+                "sha256:30f43887bbae0d49113cbaab729a112251a940e9b274536613097ab8b4899cf6",
+                "sha256:34b18ba135c687f4dac449aa5157d36e2cbb7c03cbea4ddbd88604e076aa836e",
+                "sha256:36b3ee798c58ace201289024b52788161e1ea133e4ac93fba7d49da5fec0ef9e",
+                "sha256:39514da80f971362f9267c600b6d459bfbbc549cffc2cef8e47474fddc9b45b1",
+                "sha256:39f5441553f1c2aed4de4377178ad8ff8f9d733723d6c66d983d75341de265ab",
+                "sha256:3a96e0c6a41dcdba3a0a581bbf6c44bb863f27c541547fb4b9711fd8cf0ffad4",
+                "sha256:3f26b5bd1079acdb0c7a5645e350fe54d16b17bfc5e71f371c449383d3342e17",
+                "sha256:41ef53e7c58aa4ef281da975f62c258950f54b76ec8e45941e93a3d1d8580594",
+                "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d",
+                "sha256:43fbac5f22e25bee1d482c97474f930a353542855f05c1161fd804c9dc74a09d",
+                "sha256:4457a94da0d5c53dc4b3e4de1158bdab077db23c53232f37a3cb7afdb053a4e3",
+                "sha256:465a3eb5659338cf2a9243e50ad9b2296fa15061736d6e26240e713522b6235c",
+                "sha256:482103aed1dfe2f3b71a58eff35ba105289b8d862551ea576bd15479aba01f66",
+                "sha256:4832d7d380477521a8c1644bbab6588dfedea5e30a7d967b5fb75977c45fd77f",
+                "sha256:4901165d170a5fde6f589acb90a6b33629ad1ec976d4529e769c6f3d885e3e80",
+                "sha256:5307def11a35f5ae4581a0b658b0af8178c65c530e94893345bebf41cc139d33",
+                "sha256:5417558f6887e9b6b65b4527232553c139b57ec42c64570569b155262ac0754f",
+                "sha256:56a737287efecafc16f6d067c2ea0117abadcd078d58721f967952db329a3e5c",
+                "sha256:586f8204935b9ec884500498ccc91aa869fc652c40c093bd9e1471fbcc25c022",
+                "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e",
+                "sha256:5ddcba87675b6d509139d1b521e0c8250e967e63b5909a7e8f8944d0f90ff36f",
+                "sha256:618a3d6cae6ef8ec88bb76dd80b83cfe415ad4f1d942ca2a903bf6b6ff97a2da",
+                "sha256:635dc434ff724b178cb192c70016cc0ad25a275228f749ee0daf0eddbc8183b1",
+                "sha256:661d25cbffaf8cc42e971dd570d87cb29a665f49f4abe1f9e76be9a5182c4688",
+                "sha256:66e6a3af5a75363d2c9a48b07cb27c4ea542938b1a2e93b15a503cdfa8490795",
+                "sha256:67071a6171e92b6da534b8ae326505f7c18022c6f19072a81dcf40db2638767c",
+                "sha256:685537e07897f173abcf67258bee3c05c374fa6fff89d4c7e42fb391b0605e98",
+                "sha256:69e64831e22a6b377772e7fb337533c365085b31619005802a79242fee620bc1",
+                "sha256:6b0817e34942b2ca527b0e9298373e7cc75f429e8da2055607f4931fded23e20",
+                "sha256:6c81e5f372cd0dc5dc4809553d34f832f60a46034a5f187756d9b90586c2c307",
+                "sha256:6d7faa6f14017c0b1e69f5e2c357b998731ea75a442ab3841c0dbbbfe902d2c4",
+                "sha256:6ef0befbb5d79cf32d0266f5cff01545602344eda89480e1dd88aca964260b18",
+                "sha256:6ef687afab047554a2d366e112dd187b62d261d49eb79b77e386f94644363294",
+                "sha256:7223a2a5fe0d217e60a60cdae28d6949140dde9c3bcc714063c5b463065e3d66",
+                "sha256:77f195baa60a54ef9d2de16fbbfd3ff8b04edc0c0140a761b56c267ac11aa467",
+                "sha256:793968759cd0d96cac1e367afd70c235867831983f876a53389ad869b043c948",
+                "sha256:7bd339195d84439cbe5771546fe8a4e8a7a045417d8f9de9a368c434e42a721e",
+                "sha256:7cd863afe7336c62ec78d7d1349a2f34c007a3cc6c2369d667c65aeec412a5b1",
+                "sha256:7f2facbd386dd60cbbf1a794181e6aa0bd429bd78bfdf775436020172e2a23f0",
+                "sha256:84ffab12db93b5f6bad84c712c92060a2d321b35c3c9960b43d08d0f639d60d7",
+                "sha256:8c8370641f1a7f0e0669ddccca22f1da893cef7628396431eb445d46d893e5cd",
+                "sha256:8db715ebe3bb7d86d77ac1826f7d67ec11a70dbd2376b7cc214199360517b641",
+                "sha256:8e8916ae4c720529e18afa0b879473049e95949bf97042e938530e072fde061d",
+                "sha256:8f03bccbd8586e9dd37219bce4d4e0d3ab492e6b3b533e973fa08a112cb2ffc9",
+                "sha256:8f2fc11e8fe034ee3c34d316d0ad8808f45bc3b9ce5857ff29d513f3ff2923a1",
+                "sha256:923d39efa3cfb7279a0327e337a7958bff00cc447fd07a25cddb0a1cc9a6d2da",
+                "sha256:93df1de2f7f7239dc9cc5a4a12408ee1598725036bd2dedadc14d94525192fc3",
+                "sha256:998e33ad22dc7ec7e030b3df701c43630b5bc0d8fbc2267653577e3fec279afa",
+                "sha256:99f70b740dc04d09e6b2699b675874367885217a2e9f782bdf5395632ac663b7",
+                "sha256:9a00312dea9310d4cb7dbd7787e722d2e86a95c2db92fbd7d0155f97127bcb40",
+                "sha256:9d54553c1136b50fd12cc17e5b11ad07374c316df307e4cfd6441bea5fb68496",
+                "sha256:9dbbeb27f4e70bfd9eec1be5477517365afe05a9b2c441a0b21929ee61048124",
+                "sha256:a1ce3ba137ed54f83e56fb983a5859a27d43a40188ba798993812fed73c70836",
+                "sha256:a34d557a42aa28bd5c48a023c570219ba2593bcbbb8dc1b98d8cf5d529ab1434",
+                "sha256:a5f446dd5055667aabaee78487f2b5ab72e244f9bc0b2ffebfeec79051679984",
+                "sha256:ad36cfb355e24f1bd37cac88c112cd7730873f20fb0bdaf8ba59eedf8216079f",
+                "sha256:aec493917dd45e3c69d00a8874e7cbed844efd935595ef78a0f25f14312e33c6",
+                "sha256:b316144e85316da2723f9d8dc75bada12fa58489a527091fa1d5a612643d1a0e",
+                "sha256:b34ae4636dfc4e76a438ab826a0d1eed2589ca7d9a1b2d5bb546978ac6485461",
+                "sha256:b34b7aa8b261c1dbf7720b5d6f01f38243e9b9daf7e6b8bc1fd4657000062f2c",
+                "sha256:bc362ee4e314870a70f4ae88772d72d877246537d9f8cb8f7eacf10884862432",
+                "sha256:bed88b9a458e354014d662d47e7a5baafd7ff81c780fd91584a10d6ec842cb73",
+                "sha256:c0013fe6b46aa496a6749c77e00a3eb07952832ad6166bd481c74bda0dcb6d58",
+                "sha256:c0b5dcf9193625afd8ecc92312d6ed78781c46ecbf39af9ad4681fc9f464af88",
+                "sha256:c4325ff0442a12113a6379af66978c3fe562f846763287ef66bdc1d57925d337",
+                "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7",
+                "sha256:c8362467a0fdeccd47935f22c256bec5e6abe543bf0d66e3d3d57a8fb5731863",
+                "sha256:cd5bf1af8efe569654bbef5a3e0a56eca45f87cfcffab31dd8dde70da5982475",
+                "sha256:cf1ea2e34868f6fbf070e1af291c8180480310173de0b0c43fc38a02929fc0e3",
+                "sha256:d62dec4976954a23d7f91f2f4530852b0c7608116c257833922a896101336c51",
+                "sha256:d68c93e381010662ab873fea609bf6c0f428b6d0bb00f2c6939782e0818d37bf",
+                "sha256:d7c36232a90d4755b720fbd76739d8891732b18cf240a9c645d75f00639a9024",
+                "sha256:dd18772815d5f008fa03d2b9a681ae38d5ae9f0e599f7dda233c439fcaa00d40",
+                "sha256:ddc2f4dfd396c7bfa18e6ce371cba60e4cf9d2e5cdb71376aa2da264605b60b9",
+                "sha256:e003b002ec72c8d5a3e3da2989c7d6065b47d9eaa70cd8808b5384fbb970f4ec",
+                "sha256:e32a92116d4f2a80b629778280103d2a510a5b3f6314ceccd6e38006b5e92dcb",
+                "sha256:e4461d0f003a0aa9be2bdd1b798a041f177189c1a0f7619fe8c95ad08d9a45d7",
+                "sha256:e541ec6f2ec456934fd279a3120f856cd0aedd209fc3852eca563f81738f6861",
+                "sha256:e546e768d08ad55b20b11dbb78a745151acbd938f8f00d0cfbabe8b0199b9880",
+                "sha256:ea7d4a99f3b38c37eac212dbd6ec42b7a5ec51e2c74b5d3223e43c811609e65f",
+                "sha256:ed4eb745efbff0a8e9587d22a84be94a5eb7d2d99c02dacf7bd0911713ed14dd",
+                "sha256:f8a2f084546cc59ea99fda8e070be2fd140c3092dc11524a71aa8f0f3d5a55ca",
+                "sha256:fcb25daa9219b4cf3a0ab24b0eb9a5cc8949ed4dc72acb8fa16b7e1681aa3c58",
+                "sha256:fdea4952db2793c4ad0bdccd27c1d8fdd1423a92f04598bc39425bcc2b8ee46e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.17.1"
+            "version": "==0.18.0"
         },
         "rtree": {
             "hashes": [
@@ -3227,19 +3246,19 @@
         },
         "send2trash": {
             "hashes": [
-                "sha256:a384719d99c07ce1eefd6905d2decb6f8b7ed054025bb0e618919f945de4f679",
-                "sha256:c132d59fa44b9ca2b1699af5c86f57ce9f4c5eb56629d5d55fbb7a35f84e2312"
+                "sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9",
+                "sha256:b18e7a3966d99871aefeb00cfbcfdced55ce4871194810fc71f4aa484b953abf"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.8.2"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==1.8.3"
         },
         "setuptools": {
             "hashes": [
-                "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401",
-                "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"
+                "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e",
+                "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==69.1.0"
+            "version": "==69.2.0"
         },
         "shapely": {
             "hashes": [
@@ -3399,11 +3418,11 @@
         },
         "sniffio": {
             "hashes": [
-                "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101",
-                "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"
+                "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2",
+                "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.3.0"
+            "version": "==1.3.1"
         },
         "snuggs": {
             "hashes": [
@@ -3452,11 +3471,11 @@
         },
         "terminado": {
             "hashes": [
-                "sha256:1ea08a89b835dd1b8c0c900d92848147cef2537243361b2e3f4dc15df9b6fded",
-                "sha256:87b0d96642d0fe5f5abd7783857b9cab167f221a39ff98e3b9619a788a3c0f2e"
+                "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0",
+                "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.18.0"
+            "version": "==0.18.1"
         },
         "texttable": {
             "hashes": [
@@ -3516,35 +3535,35 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74",
-                "sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e"
+                "sha256:8cdd83c040dab7d1dee822678e5f5d100b514f7b72b01615b26fc5718916fdf9",
+                "sha256:fcdf85684a772ddeba87db2f398ce00b40ff550d1528c03c14dbf6a02003cd80"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.14.1"
+            "version": "==5.14.2"
         },
         "typeguard": {
             "hashes": [
-                "sha256:8923e55f8873caec136c892c3bed1f676eae7be57cdb94819281b3d3bc9c0953",
-                "sha256:ea0a113bbc111bcffc90789ebb215625c963411f7096a7e9062d4e4630c155fd"
+                "sha256:7da3bd46e61f03e0852f8d251dcbdc2a336aa495d7daff01e092b55327796eb8",
+                "sha256:c556a1b95948230510070ca53fa0341fb0964611bd05d598d87fb52115d65fee"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.1.5"
+            "version": "==4.2.1"
         },
         "types-python-dateutil": {
             "hashes": [
-                "sha256:1f8db221c3b98e6ca02ea83a58371b22c374f42ae5bbdf186db9c9a76581459f",
-                "sha256:efbbdc54590d0f16152fa103c9879c7d4a00e82078f6e2cf01769042165acaa2"
+                "sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202",
+                "sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.8.19.20240106"
+            "version": "==2.9.0.20240316"
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475",
-                "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"
+                "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
+                "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==4.10.0"
+            "version": "==4.11.0"
         },
         "typing-inspect": {
             "hashes": [
@@ -3578,11 +3597,11 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3",
-                "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"
+                "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a",
+                "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.25.0"
+            "version": "==20.25.1"
         },
         "wcwidth": {
             "hashes": [
@@ -3723,11 +3742,11 @@
         },
         "xyzservices": {
             "hashes": [
-                "sha256:091229269043bc8258042edbedad4fcb44684b0473ede027b5672ad40dc9fa02",
-                "sha256:6a4c38d3a9f89d3e77153eff9414b36a8ee0850c9e8b85796fd1b2a85b8dfd68"
+                "sha256:6a04f11487a6fb77d92a98984cd107fbd9157fd5e65f929add9c3d6e604ee88c",
+                "sha256:b83e48c5b776c9969fffcfff57b03d02b1b1cd6607a9d9c4e7f568b01ef47f4c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.10.1"
+            "version": "==2024.4.0"
         },
         "y-py": {
             "hashes": [
@@ -3818,11 +3837,11 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31",
-                "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"
+                "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b",
+                "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.17.0"
+            "version": "==3.18.1"
         }
     },
     "develop": {
@@ -3858,11 +3877,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:1050a3ab8473488d7eee163796b02e511d0735cf43a04ba2a8348bd0f2eaf8a5",
-                "sha256:48fbc236fbe0e138b88773fa0437751f14c3645fb483f1d4c5dee58b37e5ce73"
+                "sha256:07232af52a5ba146dc3372c7bf52a0f890a23edf38d77caef8d53f9cdc2584c1",
+                "sha256:7468edaf4f6de3e1b912e57f66c241e6fd3c7099f2ec2136e239e142e800274d"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.21.0"
+            "version": "==8.23.0"
         },
         "jedi": {
             "hashes": [
@@ -3882,18 +3901,18 @@
         },
         "parso": {
             "hashes": [
-                "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0",
-                "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"
+                "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18",
+                "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.8.3"
+            "version": "==0.8.4"
         },
         "pexpect": {
             "hashes": [
                 "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523",
                 "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"
             ],
-            "markers": "sys_platform != 'win32'",
+            "markers": "sys_platform != 'win32' and sys_platform != 'emscripten'",
             "version": "==4.9.0"
         },
         "pickleshare": {
@@ -3951,11 +3970,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74",
-                "sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e"
+                "sha256:8cdd83c040dab7d1dee822678e5f5d100b514f7b72b01615b26fc5718916fdf9",
+                "sha256:fcdf85684a772ddeba87db2f398ce00b40ff550d1528c03c14dbf6a02003cd80"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.14.1"
+            "version": "==5.14.2"
         },
         "wcwidth": {
             "hashes": [
@@ -3965,4 +3984,4 @@
             "version": "==0.2.13"
         }
     }
-}
\ No newline at end of file
+}
diff --git a/README.md b/README.md
index 85c071822..7bb7207c6 100644
--- a/README.md
+++ b/README.md
@@ -25,24 +25,36 @@ AWS Resource Name: `arn:aws:s3:::noaa-nws-owp-fim`
 
 ### Accessing Data using the AWS CLI
 
-This S3 Bucket (`s3://noaa-nws-owp-fim`) is set up as a "Requester Pays" bucket. Read more about what that means [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html). If you are using compute resources in the same region as the S3 Bucket, then there is no cost.
+Before attempting to download, you will need ESIP AWS cli credentials (Access key ID and Secret Access Key). You do not have to have your own AWS account. Please contact Carson Pruitt (carson.pruitt@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov). 
+
+Once you get AWS credentials, open your terminal window and type:
+```
+aws configure --profile esip
+```
+It will ask you for the Access key ID, Secret Access Key, Region and default language (just hit tab for that entry).
+
+With the keys in place, you can test your credentials get a list folders prior to download as well as execute other S3 cli commands:
+```
+aws s3 ls s3://noaa-nws-owp-fim --profile esip
+```
+
 ### Examples
 
 **Note:** All examples are based on linux pathing. Also, for each sample below, remove the line breaks [backslash(s) "\"] before running the command.
 
 The available inputs, test cases, and versioned FIM outputs can be found by running:
 ```
-aws s3 ls s3://noaa-nws-owp-fim/hand_fim/ --request-payer
+aws s3 ls s3://noaa-nws-owp-fim/hand_fim/  --profile esip
 ```
 
 Download a directory of sample outputs for a single HUC8:
 ```
-aws s3 sync s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_3_11_0/12090301 \
-    /your_local_folder_name/12090301 --request-payer
+aws s3 sync s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_4_0_0/12090301 \
+    /your_local_folder_name/12090301 --profile esip
 ```
-By adjusting pathing, you can also download entire directories such as the `fim_4_3_11_0` folder. An entire output FIM set (e.g. `fim_4_3_11_0`) is approximately 1.1 TB.
+By adjusting pathing, you can also download entire directories such as the `fim_4_4_0_0` folder. An entire output FIM set (e.g. `fim_4_4_0_0`) is approximately 1.1 TB.
 
-**Note**: There may be newer editions than `fim_4_3_11_0`, and it is recommended to adjust the command above for the latest version.
+**Note**: There may be newer editions than `fim_4_4_0_0`, and it is recommended to adjust the command above for the latest version.
 
 ## Setting up your Environment
 
@@ -83,7 +95,7 @@ Git will auto create a subfolder named `inundation-mapping` where the code will
 Input data can be found on the ESIP S3 Bucket (see "Accessing Data through ESIP S3 Bucket" section above). The FIM inputs directory can be found at `s3://noaa-nws-owp-fim/hand_fim/inputs`. It is appx 400GB and it needs to be in your `data` folder.
 
 ```
-aws s3 sync s3://noaa-nws-owp-fim/hand_fim/inputs /home/projects/fim/data/inputs --request-payer --dryrun
+aws s3 sync s3://noaa-nws-owp-fim/hand_fim/inputs /home/projects/fim/data/inputs --profile esip --dryrun
 ```
 **Note**: When you include the `--dryrun` argument in the command, a large list will be returned showing you exactly which files are to be downloaded and where they will be saved. We recommend including this argument the first time you run the command, then quickly aborting it (CTRL-C) so you don't get the full list. However, you can see that your chosen target path on your machine is correct.  When you are happy with the pathing, run the `aws s3` command again and leave off the `--dryrun` argument.
 
@@ -153,7 +165,7 @@ To test in HUCs other than the provided HUCs, the following processes can be fol
 ```
 /foss_fim/src/acquire_and_preprocess_inputs.py -u <huc8s_to_process>
 ```
-    Sorry, this tool is deprecated, updates will be coming soon.
+    Sorry, this tool is deprecated, replacement scripts are coming soon.
 
 
 ### Evaluating Inundation Map Performance
diff --git a/data/ble/ble_benchmark/Dockerfile b/data/ble/ble_benchmark/Dockerfile
deleted file mode 100644
index bb35c7c88..000000000
--- a/data/ble/ble_benchmark/Dockerfile
+++ /dev/null
@@ -1,9 +0,0 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.0
-
-RUN apt-get update && apt-get install -y \
-    python3-pip
-
-## install python 3 modules ##
-COPY Pipfile .
-COPY Pipfile.lock .
-RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
diff --git a/data/ble/ble_benchmark/Pipfile b/data/ble/ble_benchmark/Pipfile
deleted file mode 100755
index edb4c575e..000000000
--- a/data/ble/ble_benchmark/Pipfile
+++ /dev/null
@@ -1,15 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-pandas = "*"
-rasterio = "*"
-geopandas = "*"
-openpyxl = "*"
-
-[dev-packages]
-
-[requires]
-python_version = "3.10"
diff --git a/data/ble/ble_benchmark/Pipfile.lock b/data/ble/ble_benchmark/Pipfile.lock
deleted file mode 100644
index 5e258cf1b..000000000
--- a/data/ble/ble_benchmark/Pipfile.lock
+++ /dev/null
@@ -1,340 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "43b6d8b600067ac2becfb2d00935884984da473d94e96a01d4943d31d3091e80"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3.10"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "affine": {
-            "hashes": [
-                "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92",
-                "sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.4.0"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
-                "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==23.1.0"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
-                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
-            ],
-            "index": "pypi",
-            "version": "==2023.7.22"
-        },
-        "click": {
-            "hashes": [
-                "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd",
-                "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==8.1.6"
-        },
-        "click-plugins": {
-            "hashes": [
-                "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b",
-                "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"
-            ],
-            "version": "==1.1.1"
-        },
-        "cligj": {
-            "hashes": [
-                "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
-                "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
-            "version": "==0.7.2"
-        },
-        "et-xmlfile": {
-            "hashes": [
-                "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c",
-                "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.1.0"
-        },
-        "fiona": {
-            "hashes": [
-                "sha256:1a585002a6385cc8ab0f66ddf3caf18711f531901906abd011a67a0cc89ab7b0",
-                "sha256:1da8b954f6f222c3c782bc285586ea8dd9d7e55e1bc7861da9cd772bca671660",
-                "sha256:285f3dd4f96aa0a3955ed469f0543375b20989731b2dddc85124453f11ac62bc",
-                "sha256:2c2c7b09eecee3bb074ef8aa518cd6ab30eb663c6fdd0eff3c88d454a9746eaa",
-                "sha256:39baf11ff0e4318397e2b2197de427b4eebdc49d4a9a7c1366f8a7ed682978a4",
-                "sha256:450561d308d3ce7c7e30294822b1de3f4f942033b703ddd4a91a7f7f5f506ca0",
-                "sha256:5679d3f7e0d513035eb72e59527bb90486859af4405755dfc739138633106120",
-                "sha256:71b023ef5248ebfa5524e7a875033f7db3bbfaf634b1b5c1ae36958d1eb82083",
-                "sha256:74511d3755695d75cea0f4ff6f5e0c6c5d5be8e0d46dafff124c6a219e99b1eb",
-                "sha256:7bfb1f49e0e53f6cd7ad64ae809d72646266b37a7b9881205977408b443a8d79",
-                "sha256:a670ea4262cb9140445bcfc97cbfd2f508a058be342f4a97e966b8ce7696601f",
-                "sha256:b0387cae39e27f338fd948b3b50b6e6ce198cc4cec257fc91660849697c69dc3",
-                "sha256:b633a2e550e083805c638d2ab8059c283ca112aaea8241e170c012d2ee0aa905",
-                "sha256:c1faa625d5202b8403471bbc9f9c96b1bf9099cfcb0ee02a80a3641d3d02383e",
-                "sha256:c671d8832287cda397621d79c5a635d52e4631f33a8f0e6fdc732a79a93cb96c",
-                "sha256:d6483a20037db2209c8e9a0c6f1e552f807d03c8f42ed0c865ab500945a37c4d",
-                "sha256:d93c993265f6378b23f47708c83bddb3377ca6814a1f0b5a0ae0bee9c8d72cf8",
-                "sha256:dbe158947099a83ad16f9acd3a21f50ff01114c64e2de67805e382e6b6e0083a",
-                "sha256:ea7c44c15b3a653452b9b3173181490b7afc5f153b0473c145c43c0fbf90448b",
-                "sha256:f5da66b723a876142937e683431bbaa5c3d81bb2ed3ec98941271bc99b7f8cd0"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.9.4.post1"
-        },
-        "geopandas": {
-            "hashes": [
-                "sha256:101cfd0de54bcf9e287a55b5ea17ebe0db53a5e25a28bacf100143d0507cabd9",
-                "sha256:e5b56d9c20800c77bcc0c914db3f27447a37b23b2cd892be543f5001a694a968"
-            ],
-            "index": "pypi",
-            "version": "==0.13.2"
-        },
-        "numpy": {
-            "hashes": [
-                "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
-                "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55",
-                "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf",
-                "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01",
-                "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca",
-                "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901",
-                "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d",
-                "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4",
-                "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf",
-                "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380",
-                "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044",
-                "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545",
-                "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f",
-                "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f",
-                "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3",
-                "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364",
-                "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9",
-                "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418",
-                "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f",
-                "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295",
-                "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3",
-                "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187",
-                "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926",
-                "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357",
-                "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"
-            ],
-            "markers": "python_version >= '3.9'",
-            "version": "==1.25.2"
-        },
-        "openpyxl": {
-            "hashes": [
-                "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184",
-                "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"
-            ],
-            "index": "pypi",
-            "version": "==3.1.2"
-        },
-        "packaging": {
-            "hashes": [
-                "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
-                "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==23.1"
-        },
-        "pandas": {
-            "hashes": [
-                "sha256:02755de164da6827764ceb3bbc5f64b35cb12394b1024fdf88704d0fa06e0e2f",
-                "sha256:0a1e0576611641acde15c2322228d138258f236d14b749ad9af498ab69089e2d",
-                "sha256:1eb09a242184092f424b2edd06eb2b99d06dc07eeddff9929e8667d4ed44e181",
-                "sha256:30a89d0fec4263ccbf96f68592fd668939481854d2ff9da709d32a047689393b",
-                "sha256:50e451932b3011b61d2961b4185382c92cc8c6ee4658dcd4f320687bb2d000ee",
-                "sha256:51a93d422fbb1bd04b67639ba4b5368dffc26923f3ea32a275d2cc450f1d1c86",
-                "sha256:598e9020d85a8cdbaa1815eb325a91cfff2bb2b23c1442549b8a3668e36f0f77",
-                "sha256:66d00300f188fa5de73f92d5725ced162488f6dc6ad4cecfe4144ca29debe3b8",
-                "sha256:69167693cb8f9b3fc060956a5d0a0a8dbfed5f980d9fd2c306fb5b9c855c814c",
-                "sha256:6d6d10c2142d11d40d6e6c0a190b1f89f525bcf85564707e31b0a39e3b398e08",
-                "sha256:713f2f70abcdade1ddd68fc91577cb090b3544b07ceba78a12f799355a13ee44",
-                "sha256:7376e13d28eb16752c398ca1d36ccfe52bf7e887067af9a0474de6331dd948d2",
-                "sha256:77550c8909ebc23e56a89f91b40ad01b50c42cfbfab49b3393694a50549295ea",
-                "sha256:7b21cb72958fc49ad757685db1919021d99650d7aaba676576c9e88d3889d456",
-                "sha256:9ebb9f1c22ddb828e7fd017ea265a59d80461d5a79154b49a4207bd17514d122",
-                "sha256:a18e5c72b989ff0f7197707ceddc99828320d0ca22ab50dd1b9e37db45b010c0",
-                "sha256:a6b5f14cd24a2ed06e14255ff40fe2ea0cfaef79a8dd68069b7ace74bd6acbba",
-                "sha256:b42b120458636a981077cfcfa8568c031b3e8709701315e2bfa866324a83efa8",
-                "sha256:c4af689352c4fe3d75b2834933ee9d0ccdbf5d7a8a7264f0ce9524e877820c08",
-                "sha256:c7319b6e68de14e6209460f72a8d1ef13c09fb3d3ef6c37c1e65b35d50b5c145",
-                "sha256:cf3f0c361a4270185baa89ec7ab92ecaa355fe783791457077473f974f654df5",
-                "sha256:dd46bde7309088481b1cf9c58e3f0e204b9ff9e3244f441accd220dd3365ce7c",
-                "sha256:dd5476b6c3fe410ee95926873f377b856dbc4e81a9c605a0dc05aaccc6a7c6c6",
-                "sha256:e69140bc2d29a8556f55445c15f5794490852af3de0f609a24003ef174528b79",
-                "sha256:f908a77cbeef9bbd646bd4b81214cbef9ac3dda4181d5092a4aa9797d1bc7774"
-            ],
-            "index": "pypi",
-            "version": "==2.0.2"
-        },
-        "pyparsing": {
-            "hashes": [
-                "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb",
-                "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"
-            ],
-            "markers": "python_full_version >= '3.6.8'",
-            "version": "==3.1.1"
-        },
-        "pyproj": {
-            "hashes": [
-                "sha256:00fab048596c17572fa8980014ef117dbb2a445e6f7ba3b9ddfcc683efc598e7",
-                "sha256:08dfc5c9533c78a97afae9d53b99b810a4a8f97c3be9eb2b8f323b726c736403",
-                "sha256:1283d3c1960edbb74828f5f3405b27578a9a27f7766ab6a3956f4bd851f08239",
-                "sha256:137a07404f937f264b11b7130cd4cfa00002dbe4333b222e8056db84849c2ea4",
-                "sha256:18a8bdb87aeb41b60a2e91d32f623227de3569fb83b4c64b174c3a7c5b0ed3ae",
-                "sha256:23787460fab85ba2f857ee60ffb2e8e21fd9bd5db9833c51c1c05b2a6d9f0be5",
-                "sha256:2799499a4045e4fb73e44c31bdacab0593a253a7a4b6baae6fdd27d604cf9bc2",
-                "sha256:3a4d2d438b007cb1f8d5f6f308d53d7ff9a2508cff8f9da6e2a93b76ffd98aaf",
-                "sha256:4d8a9773503085eada59b6892c96ddf686ab8cf64cfdc18ad744d13ee76dfa6f",
-                "sha256:557e6592855111c84eda176ddf6b130f55d5e2b9cb1c017b8c91b69f37f474f5",
-                "sha256:595376e4d3bb72b7dceeccbce0f4c43053d47561f17a1ad0224407e9980ee849",
-                "sha256:78276c6b0c831255c97c56dff7313a3571f327a284d8ac63d6a56437a72ed0e0",
-                "sha256:830e6de7cfe43853967afee5ef908dfd5aa72d1ec12af9b9e3fecc179886e346",
-                "sha256:8fbac2eb9a0e425d7d6b7c6f4ebacd675cf3bdef0c59887057b8b4b0374e7c12",
-                "sha256:95120d65cbc5983dfd877076f28dbc18b9b329cbee38ca6e217bb7a5a043c099",
-                "sha256:9de1aab71234bfd3fd648a1152519b5ee152c43113d7d8ea52590a0140129501",
-                "sha256:a5b111865b3f0f8b77b3983f2fbe4dd6248fc09d3730295949977c8dcd988062",
-                "sha256:ba5e7c8ddd6ed5a3f9fcf95ea80ba44c931913723de2ece841c94bb38b200c4a",
-                "sha256:d7f6cd045df29aae960391dfe06a575c110af598f1dea5add8be6ca42332b0f5",
-                "sha256:de6288b6ceabdeeac01abf627c74414822d322d8f55dc8efe4d29dedd27c5719",
-                "sha256:dfe392dfc0eba2248dc08c976a72f52ff9da2bddfddfd9ff5dcf18e8e88200c7",
-                "sha256:e342b3010b2b20134671564ff9a8c476e5e512bf589477480aded1a5813af7c8",
-                "sha256:e427ccdbb1763872416549bdfa9fa1f5f169054653c4daf674e71480cc39cf11",
-                "sha256:e600f6a2771d3b41aeb2cc1efd96771ae9a01451013da1dd48ff272e7c6e34ef",
-                "sha256:f04f6297c615c3b17f835df2556ac8fb9b4f51f281e960437eaf0cd80e7ae26a"
-            ],
-            "markers": "python_version >= '3.9'",
-            "version": "==3.6.0"
-        },
-        "python-dateutil": {
-            "hashes": [
-                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
-                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.8.2"
-        },
-        "pytz": {
-            "hashes": [
-                "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588",
-                "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"
-            ],
-            "version": "==2023.3"
-        },
-        "rasterio": {
-            "hashes": [
-                "sha256:075731165d26ef33e33e3ab903a0ecabb643893794bea1c78dd6c631713d1820",
-                "sha256:106e6519f5d2a0d35f0e963a8d07dc313ba7f6595a4c89fe9d6dcd8662a86617",
-                "sha256:41f233b7c85a74d66da4e31e37cfabe3c1aa116260063519d49f266e278bd4b3",
-                "sha256:5247affa7122f3c10a82d8cfa56b89fb60bfb981e4a22ebad8889d1caa138215",
-                "sha256:58d3d42cf06645f694911ea2707e6204868006784e0df3540e05e4615209b9c3",
-                "sha256:a8013826cdf540fb4f1507f92bb4ef440b594a4a8895d9038e25498e5b135f52",
-                "sha256:a97610a2997d9dc246ccd3f1361fdd63ae0b23a676e68af4f018f03e8c8a7592",
-                "sha256:abfdcb8f10210b8fad939f40d545d6c47e9e3b5cf4a43773ca8dd11c58204304",
-                "sha256:b4ad780117fefcd0fb81e2459d409a722f5964d1f655252278c7c56eb18f7ae6",
-                "sha256:baab6f3ba468d3b3ada05b282574af0f3110830f50888948d50c1f16aa6130af",
-                "sha256:c04a4932807ae64642f18d7c3f7cf504cb6703ec35496d07f2e5679ec5b9c4d3",
-                "sha256:c2ddc9cb2ccb0d296ca8c9da8af73b4d1c03c887a7dabbb4b4a6fa1e7450c1e7",
-                "sha256:cd21347b58fb60f2f0a8f7eb95ce58680590e205dee80562a67e868eebaaeea5",
-                "sha256:ce334df60b9d6ea66751b03afd1d138a551d27f2588e95ac6d7479856bb575a8",
-                "sha256:d6fa64dfda2ce1fe4c8b4e61d6e476ab3f389ac679e49b5386bb83708939a1b2",
-                "sha256:e3c06f1a62b28437c908851f0fc67abc6b31fce4a535e88e92311e73b8709996",
-                "sha256:feebf547b6785d57330f9ef69af54534a9548b24510c4e2fe87e5b9355112c28"
-            ],
-            "index": "pypi",
-            "version": "==1.3.7"
-        },
-        "setuptools": {
-            "hashes": [
-                "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f",
-                "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==68.0.0"
-        },
-        "shapely": {
-            "hashes": [
-                "sha256:01224899ff692a62929ef1a3f5fe389043e262698a708ab7569f43a99a48ae82",
-                "sha256:05c51a29336e604c084fb43ae5dbbfa2c0ef9bd6fedeae0a0d02c7b57a56ba46",
-                "sha256:09d6c7763b1bee0d0a2b84bb32a4c25c6359ad1ac582a62d8b211e89de986154",
-                "sha256:193a398d81c97a62fc3634a1a33798a58fd1dcf4aead254d080b273efbb7e3ff",
-                "sha256:1a34a23d6266ca162499e4a22b79159dc0052f4973d16f16f990baa4d29e58b6",
-                "sha256:2569a4b91caeef54dd5ae9091ae6f63526d8ca0b376b5bb9fd1a3195d047d7d4",
-                "sha256:33403b8896e1d98aaa3a52110d828b18985d740cc9f34f198922018b1e0f8afe",
-                "sha256:3ad81f292fffbd568ae71828e6c387da7eb5384a79db9b4fde14dd9fdeffca9a",
-                "sha256:3cb256ae0c01b17f7bc68ee2ffdd45aebf42af8992484ea55c29a6151abe4386",
-                "sha256:45b4833235b90bc87ee26c6537438fa77559d994d2d3be5190dd2e54d31b2820",
-                "sha256:4641325e065fd3e07d55677849c9ddfd0cf3ee98f96475126942e746d55b17c8",
-                "sha256:502e0a607f1dcc6dee0125aeee886379be5242c854500ea5fd2e7ac076b9ce6d",
-                "sha256:66a6b1a3e72ece97fc85536a281476f9b7794de2e646ca8a4517e2e3c1446893",
-                "sha256:70a18fc7d6418e5aea76ac55dce33f98e75bd413c6eb39cfed6a1ba36469d7d4",
-                "sha256:7d3bbeefd8a6a1a1017265d2d36f8ff2d79d0162d8c141aa0d37a87063525656",
-                "sha256:83a8ec0ee0192b6e3feee9f6a499d1377e9c295af74d7f81ecba5a42a6b195b7",
-                "sha256:865bc3d7cc0ea63189d11a0b1120d1307ed7a64720a8bfa5be2fde5fc6d0d33f",
-                "sha256:90cfa4144ff189a3c3de62e2f3669283c98fb760cfa2e82ff70df40f11cadb39",
-                "sha256:91575d97fd67391b85686573d758896ed2fc7476321c9d2e2b0c398b628b961c",
-                "sha256:9a6ac34c16f4d5d3c174c76c9d7614ec8fe735f8f82b6cc97a46b54f386a86bf",
-                "sha256:a529218e72a3dbdc83676198e610485fdfa31178f4be5b519a8ae12ea688db14",
-                "sha256:a70a614791ff65f5e283feed747e1cc3d9e6c6ba91556e640636bbb0a1e32a71",
-                "sha256:ac1dfc397475d1de485e76de0c3c91cc9d79bd39012a84bb0f5e8a199fc17bef",
-                "sha256:b06d031bc64149e340448fea25eee01360a58936c89985cf584134171e05863f",
-                "sha256:b4f0711cc83734c6fad94fc8d4ec30f3d52c1787b17d9dca261dc841d4731c64",
-                "sha256:b50c401b64883e61556a90b89948297f1714dbac29243d17ed9284a47e6dd731",
-                "sha256:b519cf3726ddb6c67f6a951d1bb1d29691111eaa67ea19ddca4d454fbe35949c",
-                "sha256:bca57b683e3d94d0919e2f31e4d70fdfbb7059650ef1b431d9f4e045690edcd5",
-                "sha256:c43755d2c46b75a7b74ac6226d2cc9fa2a76c3263c5ae70c195c6fb4e7b08e79",
-                "sha256:c7eed1fb3008a8a4a56425334b7eb82651a51f9e9a9c2f72844a2fb394f38a6c",
-                "sha256:c8b0d834b11be97d5ab2b4dceada20ae8e07bcccbc0f55d71df6729965f406ad",
-                "sha256:ce88ec79df55430e37178a191ad8df45cae90b0f6972d46d867bf6ebbb58cc4d",
-                "sha256:d173d24e85e51510e658fb108513d5bc11e3fd2820db6b1bd0522266ddd11f51",
-                "sha256:d8f55f355be7821dade839df785a49dc9f16d1af363134d07eb11e9207e0b189",
-                "sha256:da71de5bf552d83dcc21b78cc0020e86f8d0feea43e202110973987ffa781c21",
-                "sha256:e55698e0ed95a70fe9ff9a23c763acfe0bf335b02df12142f74e4543095e9a9b",
-                "sha256:f32a748703e7bf6e92dfa3d2936b2fbfe76f8ce5f756e24f49ef72d17d26ad02",
-                "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.0.1"
-        },
-        "six": {
-            "hashes": [
-                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
-                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.16.0"
-        },
-        "snuggs": {
-            "hashes": [
-                "sha256:501cf113fe3892e14e2fee76da5cd0606b7e149c411c271898e6259ebde2617b",
-                "sha256:988dde5d4db88e9d71c99457404773dabcc7a1c45971bfbe81900999942d9f07"
-            ],
-            "version": "==1.4.7"
-        },
-        "tzdata": {
-            "hashes": [
-                "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a",
-                "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"
-            ],
-            "markers": "python_version >= '2'",
-            "version": "==2023.3"
-        }
-    },
-    "develop": {}
-}
diff --git a/data/ble/ble_benchmark/README.md b/data/ble/ble_benchmark/README.md
index b64d484f4..eb9e7d4fb 100644
--- a/data/ble/ble_benchmark/README.md
+++ b/data/ble/ble_benchmark/README.md
@@ -11,16 +11,10 @@ been run for the HUCs being processed.
 
 ## Installation
 
-1. A Dockerfile is provided in the root of the `/foss_fim/data/ble/ble_benchmark` directory.
-The Docker image can be built using the following command from `/foss_fim/data/ble/ble_benchmark`:
+1. The Docker image can be run using the following command:
 ```
-docker build --rm --no-cache --force-rm -t dev:ble_benchmark .
-```
-
-2. The Docker image can be run using the following command:
-```
-docker run --rm -it -v ~/git/inundation-mapping:/foss_fim -v ~/efs/fim_data/fim_4_0_13_1:/data \
--v ~/efs/fim_data/fim_4_0_13_1/outputs:/outputs --name ble_benchmark dev:ble_benchmark
+docker run --rm -it -v ~/git/inundation-mapping:/foss_fim -v ~/efs/fim_data/:/data \
+-v ~/efs/fim_data/fim_4_4_x_x/outputs:/outputs --name ble_benchmark dev:ble_benchmark {your docker image name}
 ```
 
 ## Usage
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 875709249..b56c52809 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,29 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.13.3 - 2024-04-15 - [PR#1114](https://github.com/NOAA-OWP/inundation-mapping/pull/1114)
+
+Two recent dependabot PR's came in, one for upgrading the `pillow` package and the other for upgrading idna. Both have been adjusted in this PR. 
+In this PR, we also moved `openpyxl` package, which was part of an independent dockerfile, Pipfile and Pipefile.lock in the "dev" directory. This is now merged into the parent standard docker image.
+
+Covers [PR 1111](https://github.com/NOAA-OWP/inundation-mapping/pull/1111) and 
+Covers [PR 1119](https://github.com/NOAA-OWP/inundation-mapping/pull/1119)
+
+A small update to the README.md was also updated for an unrelated topic (about AWS S3 credentials).
+
+### Changes
+- `Pipfile / Pipefile.lock`: As described above.
+- `data/ble/ble_benchmark/README.md`: Updated notes to remove talking the specific ble docker image.
+
+### Removals
+- `data/ble/ble_benchmark`
+   - `Dockerfile`: removed in favor the parent root Docker files.
+   - `Pipfile`: removed in favor the parent root Docker files.
+   - `Pipfile.lock` : removed in favor the parent root Docker files.
+
+<br/><br/>
+
+
 ## v4.4.13.2 - 2024-04-04 - [PR#1110](https://github.com/NOAA-OWP/inundation-mapping/pull/1110)
 
 This PR reflects upgrades for openJDK from 17.0.8 to something higher, minimum of 17.0.9. After some research, we can not upgrade all the way to the latest openJDK but can jump up to 19.0.  This limitation is related to version of our base docker image.  openJDK was identified as requiring an upgrade by a system wide security scan.

From 3ca0cebc84ef5fd6f4af61fca2adfd59084933e0 Mon Sep 17 00:00:00 2001
From: EmilyDeardorff-NOAA <60829052+EmilyDeardorff@users.noreply.github.com>
Date: Wed, 17 Apr 2024 10:36:09 -0700
Subject: [PATCH 307/359] v4.4.14.0 Update FIM pipeline to process in Alaska
 (#1106)

---
 data/wbd/generate_pre_clip_fim_huc8.py |  2 +-
 docs/CHANGELOG.md                      | 18 ++++++++++++++-
 src/bash_variables.env                 |  2 +-
 src/check_huc_inputs.py                |  4 +++-
 src/derive_level_paths.py              | 24 ++++++++++++++-----
 src/run_by_branch.sh                   | 15 ++++++++----
 src/run_unit_wb.sh                     | 32 ++++++++++++++++++++++----
 src/utils/shared_variables.py          | 20 ++++++++++++++++
 8 files changed, 98 insertions(+), 19 deletions(-)

diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
index ca6ee44d8..d1391aaf7 100755
--- a/data/wbd/generate_pre_clip_fim_huc8.py
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -427,7 +427,7 @@ def huc_level_clip_vectors_to_wbd(args):
         'the output directory specified as the <outputs_dir> argument.',
         usage='''
             ./generate_pre_clip_fim_huc8.py
-                -n /data/inputs/pre_clip_huc8/2024_3_20
+                -n /data/inputs/pre_clip_huc8/24_3_20
                 -u /data/inputs/huc_lists/included_huc8.lst
                 -j 6
                 -o
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b56c52809..44aeb7e05 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.4.14.0 - 2024-04-17 - [PR#1106](https://github.com/NOAA-OWP/inundation-mapping/pull/10106)
+
+Updates the FIM pipeline so it can process HUCs in southern Alaska. Running FIM in southern Alaska requires that a different CRS and a few different files be used. Additionally, some of the Alaska HUCs displayed an issue where the input stream density was too high, so this update introduces some logic to adjust the threshold of stream orders to exclude based on whether an Alaska HUC is listed as high or medium-high stream density. This update intriduces new Alaska-specific inputs, which are listed in the PR. 
+
+### Changes
+- `data/wbd/generate_pre_clip_fim_huc8.py`: Adjusted comment.
+- `src/bash_variables.env`: Changed pre-clip HUC 8 directory to be a folder with both Alaska and CONUS HUCs.
+- `src/check_huc_inputs.py`: Changed the `included_huc_list` variable to refer to a HUC list that includes Alaska.
+- `src/derive_level_paths.py`: Add in logic to exclude different stream orders based on whether the HUC falls into the high or medium-high density HUC lists.
+- `src/run_by_branch.sh`: Add in logic to check whether the HUC is in Alaska or not and to use the correct CRS accordingly.
+- `src/run_unit_wb.sh`: Add in logic to check whether the HUC is in Alaska or not and to use the correct CRS and DEM domain filename accordingly.
+- `src/utils/shared_variables.py`: Add the Alaska CRS, a list of high stream density HUCs, and a list of medium-high stream density HUCs.
+
+<br/><br/>
+
+
 ## v4.4.13.3 - 2024-04-15 - [PR#1114](https://github.com/NOAA-OWP/inundation-mapping/pull/1114)
 
 Two recent dependabot PR's came in, one for upgrading the `pillow` package and the other for upgrading idna. Both have been adjusted in this PR. 
@@ -40,7 +57,6 @@ The "black" packages is also be upgraded from 23.7.0 to 24.3.
 
 <br/><br/>
 
-
 ## v4.4.13.1 - 2024-03-11 - [PR#1086](https://github.com/NOAA-OWP/inundation-mapping/pull/1086)
 
 Fixes bug where levee-protected areas were not being masked from branch 0 DEMs.
diff --git a/src/bash_variables.env b/src/bash_variables.env
index d8d2237b6..0fc032483 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -25,7 +25,7 @@ export                  input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_
 export                          input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
 export                   input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg
 export                 input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
-export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
+export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/24_4_3 # was 23_10_17
 export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
diff --git a/src/check_huc_inputs.py b/src/check_huc_inputs.py
index 6c6c2fad9..34ceb91f8 100755
--- a/src/check_huc_inputs.py
+++ b/src/check_huc_inputs.py
@@ -14,7 +14,9 @@ def __read_included_files(parent_dir_path):
     # I have just replaced the pattern, but later we might want to clean this up.
 
     # filename_patterns = glob(os.path.join(parent_dir_path, 'included_huc*.lst'))
-    filename_patterns = glob(os.path.join(parent_dir_path, 'included_huc8.lst'))
+
+    included_huc_list = 'included_huc8_withAlaska.lst'  # previous: 'included_huc8.lst'
+    filename_patterns = glob(os.path.join(parent_dir_path, included_huc_list))
 
     accepted_hucs_set = set()
     for filename in filename_patterns:
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index ca25819dc..bb50a4e68 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -8,6 +8,7 @@
 
 from stream_branches import StreamNetwork
 from utils.fim_enums import FIM_exit_codes
+from utils.shared_variables import HIGH_STREAM_DENSITY_HUCS, MEDIUM_HIGH_STREAM_DENSITY_HUCS
 
 
 gpd.options.io_engine = "pyogrio"
@@ -19,8 +20,8 @@ def Derive_level_paths(
     buffer_wbd_streams,
     out_stream_network,
     branch_id_attribute,
+    huc_id,
     out_stream_network_dissolved=None,
-    huc_id=None,
     headwaters_outfile=None,
     catchments=None,
     waterbodies=None,
@@ -57,11 +58,22 @@ def Derive_level_paths(
         print("Sorry, no streams exist and processing can not continue. This could be an empty file.")
         sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
 
-    # values_exluded of 1 and 2 mean where are dropping stream orders 1 and 2. We are leaving those
-    # for branch zero.
-    stream_network = stream_network.exclude_attribute_values(
-        branch_id_attribute="order_", values_excluded=[1, 2]
-    )
+    if huc_id in HIGH_STREAM_DENSITY_HUCS:
+        print('HUC is in high density HUC list... removing additional stream segments.')
+        stream_network = stream_network.exclude_attribute_values(
+            branch_id_attribute="order_", values_excluded=[1, 2, 3, 4]
+        )
+    elif huc_id in MEDIUM_HIGH_STREAM_DENSITY_HUCS:
+        print('HUC is in medium-high density HUC list... removing additional stream segments.')
+        stream_network = stream_network.exclude_attribute_values(
+            branch_id_attribute="order_", values_excluded=[1, 2, 3]
+        )
+    else:
+        # values_exluded of 1 and 2 mean that we are dropping stream orders 1 and 2. We are leaving those
+        # for branch zero.
+        stream_network = stream_network.exclude_attribute_values(
+            branch_id_attribute="order_", values_excluded=[1, 2]
+        )
 
     # if there are no reaches at this point (due to filtering)
     if len(stream_network) == 0:
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index 53db98bee..f052571b3 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -13,6 +13,13 @@ hucUnitLength=${#hucNumber}
 huc4Identifier=${hucNumber:0:4}
 huc2Identifier=${hucNumber:0:2}
 
+## SET CRS
+if [ $huc2Identifier -eq 19 ]; then
+    huc_CRS=$ALASKA_CRS
+else
+    huc_CRS=$DEFAULT_FIM_PROJECTION_CRS
+fi
+
 # Skip branch zero
 if [ $current_branch_id = $branch_zero_id ]; then
     exit 0
@@ -39,19 +46,19 @@ date -u
 ## SUBSET VECTORS
 echo -e $startDiv"Subsetting vectors to branches $hucNumber $current_branch_id"
 echo -e "Querying NWM streams ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+ogr2ogr -f GPKG -t_srs $huc_CRS -where $branch_id_attribute="$current_branch_id" \
     $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg \
     $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg
 echo -e "Querying NWM catchments ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+ogr2ogr -f GPKG -t_srs $huc_CRS -where $branch_id_attribute="$current_branch_id" \
     $tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg \
     $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg
 echo -e "Querying NWM Dissolved Levelpaths headwaters ..."
-ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+ogr2ogr -f GPKG -t_srs $huc_CRS -where $branch_id_attribute="$current_branch_id" \
     $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters_$current_branch_id.gpkg \
     $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
 #echo -e "Querying NWM headwaters ..."
-# ogr2ogr -f GPKG -t_srs $DEFAULT_FIM_PROJECTION_CRS -where $branch_id_attribute="$current_branch_id" \
+# ogr2ogr -f GPKG -t_srs $huc_CRS -where $branch_id_attribute="$current_branch_id" \
 #     $tempCurrentBranchDataDir/nwm_headwaters_$current_branch_id.gpkg \
 #      $tempHucDataDir/nwm_headwaters.gpkg
 
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index ca6c9979a..03129bfa3 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -17,7 +17,25 @@ branch_list_lst_file=$tempHucDataDir/branch_ids.lst
 
 branchSummaryLogFile=$outputDestDir/logs/branch/"$hucNumber"_summary_branch.log
 
-## INITIALIZE TOTAL UNIT AND IT'S BRANCHES TIMER ##
+huc2Identifier=${hucNumber:0:2}
+
+
+## SET CRS and input DEM domain
+if [ $huc2Identifier -eq 19 ]; then
+    huc_CRS=$ALASKA_CRS
+    huc_input_DEM_domain=$input_DEM_domain_Alaska
+    dem_domain_filename=DEM_Domain.gpkg
+
+else
+    huc_CRS=$DEFAULT_FIM_PROJECTION_CRS
+    huc_input_DEM_domain=$input_DEM_domain
+    dem_domain_filename=HUC6_dem_domain.gpkg
+
+fi
+
+echo -e $startDiv"Using CRS: $huc_CRS" ## debug
+
+## INITIALIZE TOTAL TIME TIMER ##
 T_total_start
 huc_start_time=`date +%s`
 date -u
@@ -28,7 +46,7 @@ cp -a $pre_clip_huc_dir/$hucNumber/. $tempHucDataDir
 
 # Copy necessary files from $inputsDir into $tempHucDataDir to avoid File System Collisions
 # For buffer_stream_branches.py
-cp $input_DEM_domain $tempHucDataDir
+cp $huc_input_DEM_domain $tempHucDataDir
 # For usgs_gage_unit_setup.py
 cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir
 cp $ras_rating_curve_points_gpkg $tempHucDataDir
@@ -47,7 +65,9 @@ $srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg \
     -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg \
     -n $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg \
     -w $tempHucDataDir/nwm_lakes_proj_subset.gpkg \
-    -wbd $tempHucDataDir/wbd.gpkg
+    -wbd $tempHucDataDir/wbd.gpkg \
+    -u $hucNumber
+
 
 # test if we received a non-zero code back from derive_level_paths.py
 #subscript_exit_code=$?
@@ -74,7 +94,7 @@ python3 $srcDir/associate_levelpaths_with_levees.py -nld $tempHucDataDir/nld_sub
 
 ## STREAM BRANCH POLYGONS
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
-$srcDir/buffer_stream_branches.py -a $tempHucDataDir/HUC6_dem_domain.gpkg \
+$srcDir/buffer_stream_branches.py -a $tempHucDataDir/$dem_domain_filename \
     -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
     -i $branch_id_attribute \
     -d $branch_buffer_distance_meters \
@@ -99,7 +119,9 @@ echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"
 [ ! -f $tempCurrentBranchDataDir/dem_meters.tif ] && \
 gdalwarp -cutline $tempHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" \
     -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" \
-    -co "BIGTIFF=YES" -t_srs $DEFAULT_FIM_PROJECTION_CRS $input_DEM $tempHucDataDir/dem_meters.tif -q
+    -co "BIGTIFF=YES" -t_srs $huc_CRS $input_DEM $tempHucDataDir/dem_meters.tif
+
+Tcount
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index c9e364730..c47cea1f5 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -30,6 +30,7 @@
     'Wyoming."],BBOX[24.41,-124.79,49.38,-66.91]],ID["EPSG",5070]]'
 )
 DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
+ALASKA_CRS = os.getenv('ALASKA_CRS')
 PREP_CRS = CRS(PREP_PROJECTION)
 VIZ_PROJECTION = (
     'PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",'
@@ -79,6 +80,25 @@
 # -- Field Names -- #
 FIM_ID = 'fimid'
 
+# -- High stream density HUCs that require mitigation (currently just in Alaska) -- #
+HIGH_STREAM_DENSITY_HUCS = {'19020602', '19020503', '19020402', '19020104'}  # stream density 1.5+
+
+MEDIUM_HIGH_STREAM_DENSITY_HUCS = {  # stream density between 0.5 and 1.5
+    '19020504',
+    '19020502',
+    '19020601',
+    '19020505',
+    '19020101',  # 1.0-1.5
+    '19020102',
+    '19020501',
+    '19020301',
+    '19020401',
+    '19020302',
+    '19020103',
+    '19020202',  # 0.5-1.0
+}
+
+
 # -- Other -- #
 CONUS_STATE_LIST = {
     "AL",

From 729cc0198819927d0aad3fa0c23e32b5d191ca46 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 17 Apr 2024 11:43:33 -0600
Subject: [PATCH 308/359] v4.4.14.1 Add checks for intermediate files in AGREE
 (#1103)

---
 docs/CHANGELOG.md | 11 +++++++++++
 src/agreedem.py   | 16 ++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 44aeb7e05..c7f84703f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.4.14.1 - 2024-04-17 - [PR#1103](https://github.com/NOAA-OWP/inundation-mapping/pull/1103)
+
+Adds checks for intermediate files produced by Whitebox in the AGREE process (`src/agreedem.py`). Without these checks, if Whitebox fails to produce an output, no error is generated until much later in the `src/delineate_hydros_and_produce_HAND.sh` processing chain which makes troubleshooting difficult.
+
+### Changes
+
+- `src/agreedem.py`: Added checks to verify existence of intermediate files before continuing
+
+<br/><br/>
+
+
 ## v4.4.14.0 - 2024-04-17 - [PR#1106](https://github.com/NOAA-OWP/inundation-mapping/pull/10106)
 
 Updates the FIM pipeline so it can process HUCs in southern Alaska. Running FIM in southern Alaska requires that a different CRS and a few different files be used. Additionally, some of the Alaska HUCs displayed an issue where the input stream density was too high, so this update introduces some logic to adjust the threshold of stream orders to exclude based on whether an Alaska HUC is listed as high or medium-high stream density. This update intriduces new Alaska-specific inputs, which are listed in the PR. 
diff --git a/src/agreedem.py b/src/agreedem.py
index 437d129ac..b38c7f359 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -105,9 +105,17 @@ def agreedem(
         # Compute allocation and proximity grid using WhiteboxTools
         smo_output_zerod = os.path.join(workspace, 'agree_smogrid_zerod.tif')
         wbt.euclidean_distance(rivers_raster, vectdist_grid)
+
+        assert os.path.exists(vectdist_grid), f'Vector distance grid not created: {vectdist_grid}'
+
         wbt.convert_nodata_to_zero(smo_output, smo_output_zerod)
+
+        assert os.path.exists(smo_output_zerod), f'Vector allocation grid not created: {smo_output_zerod}'
+
         wbt.euclidean_allocation(smo_output_zerod, vectallo_grid)
 
+        assert os.path.exists(vectallo_grid), f'Vector allocation grid not created: {vectallo_grid}'
+
         # ------------------------------------------------------------------
         # 4. From Hellweger documentation: Compute the buffer grid
         # (bufgrid2). The cells in the buffer grid outside the buffer
@@ -177,9 +185,17 @@ def agreedem(
             # Compute allocation and proximity grid using WhiteboxTools
             buf_output_zerod = os.path.join(workspace, 'agree_bufgrid_zerod.tif')
             wbt.euclidean_distance(bin_buf_output, bufdist_grid)
+
+            assert os.path.exists(bufdist_grid), f'Buffer allocation grid not created: {bufdist_grid}'
+
             wbt.convert_nodata_to_zero(buf_output, buf_output_zerod)
+
+            assert os.path.exists(buf_output_zerod), f'Buffer allocation grid not created: {buf_output_zerod}'
+
             wbt.euclidean_allocation(buf_output_zerod, bufallo_grid)
 
+            assert os.path.exists(bufallo_grid), f'Buffer allocation grid not created: {bufallo_grid}'
+
             # Open distance, allocation, elevation grids.
             with rasterio.open(bufdist_grid) as bufdist, rasterio.open(
                 bufallo_grid

From f30a48627e81c4169a1ed53e46811ba42dbabf2b Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Wed, 17 Apr 2024 12:48:28 -0500
Subject: [PATCH 309/359] v4.4.15.0 synthetic rating curve calibration with
 USGS rating and up/down tracing (#1081)

---
 docs/CHANGELOG.md                             |  19 +-
 fim_post_processing.sh                        |   2 +-
 ...ing.py => src_adjust_usgs_rating_trace.py} | 171 ++++++++++++++++--
 src/src_roughness_optimization.py             |   2 +
 src/utils/shared_variables.py                 |   3 +-
 5 files changed, 176 insertions(+), 21 deletions(-)
 rename src/{src_adjust_usgs_rating.py => src_adjust_usgs_rating_trace.py} (74%)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c7f84703f..d658d1c64 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,24 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.4.15.0 - 2024-04-17 - [PR#1081](https://github.com/NOAA-OWP/inundation-mapping/pull/1081)
+
+This enhancement includes changes to the SRC calibration routine that uses the USGS published rating curve database. The modifications attempt to mimic the technique used in the stage-based CatFIM where the USGS WSE/flow is propagated upstream and downstream of the gauge location. This closes #892 
+
+### Additions
+`src/src_adjust_usgs_rating_trace.py`: updated SRC calibration routine to include the a new upstream/downstream tracing routine. The WSE(HAND stage) and flow targets obtained from the USGS rating curve are now applied to all hydroids within 8km (~5 miles) of the gauge location.  
+
+### Changes
+`fim_post_processing.sh`: using the new `src_adjust_usgs_rating_trace.py` in place of the `src_adjust_usgs_rating.py`
+`src/src_roughness_optimization.py`: minor changes to facilitate new calibration input (reset index)
+`src/utils/shared_variables.py`: added `USGS_CALB_TRACE_DIST` as the trace distance variable
+
+### Removals
+`src/src_adjust_usgs_rating.py`: deprecated (replaced with the new `src_adjust_usgs_rating_trace.py`)
+
+<br/><br/>
+
+
 ## v4.4.14.1 - 2024-04-17 - [PR#1103](https://github.com/NOAA-OWP/inundation-mapping/pull/1103)
 
 Adds checks for intermediate files produced by Whitebox in the AGREE process (`src/agreedem.py`). Without these checks, if Whitebox fails to produce an output, no error is generated until much later in the `src/delineate_hydros_and_produce_HAND.sh` processing chain which makes troubleshooting difficult.
@@ -130,7 +148,6 @@ Fixes bug in bathymetric adjustment where `mask` is used with `geopandas.read_fi
 
 <br/><br/>
 
-
 ## v4.4.11.0 - 2024-02-16 - [PR#1077](https://github.com/NOAA-OWP/inundation-mapping/pull/1077)
 
 Replace `fiona` with `pyogrio` to improve I/O speed. `geopandas` will use `pyogrio` by default starting with version 1.0. `pyarrow` was also added as an environment variable to further speedup I/O. As a result of the changes in this PR, `fim_pipeline.sh` runs approximately 10% faster.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 9a176e695..49ac3c29f 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -149,7 +149,7 @@ if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$s
     echo
     echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
     # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow values)
-    python3 $srcDir/src_adjust_usgs_rating.py \
+    python3 $srcDir/src_adjust_usgs_rating_trace.py \
         -run_dir $outputDestDir \
         -usgs_rc $usgs_rating_curve_csv \
         -nwm_recur $nwm_recur_file \
diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating_trace.py
similarity index 74%
rename from src/src_adjust_usgs_rating.py
rename to src/src_adjust_usgs_rating_trace.py
index d93c6848f..739a10af9 100644
--- a/src/src_adjust_usgs_rating.py
+++ b/src/src_adjust_usgs_rating_trace.py
@@ -5,10 +5,12 @@
 import sys
 from multiprocessing import Pool
 
+import geopandas as gpd
 import pandas as pd
 
 from src_roughness_optimization import update_rating_curve
 from utils.shared_functions import check_file_age, concat_huc_csv
+from utils.shared_variables import USGS_CALB_TRACE_DIST
 
 
 '''
@@ -111,10 +113,10 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
     for interval in recurr_intervals:
         log_text += '\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n'
         print('Processing: ' + str(interval) + '-year NWM recurr intervals')
-        ## Calculate the closest SRC discharge value to the NWM flow value
+        # Calculate the closest SRC discharge value to the NWM flow value
         merge_df['Q_find'] = (merge_df['discharge_cms'] - merge_df[interval + "_0_year"]).abs()
 
-        ## Check for any missing/null entries in the input SRC
+        # Check for any missing/null entries in the input SRC
         # There may be null values for lake or coastal flow lines
         # (need to set a value to do groupby idxmin below)
         if merge_df['Q_find'].isnull().values.any():
@@ -125,15 +127,16 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
                 + str(merge_df['feature_id'])
                 + ' --> Null values found in "Q_find" calc. These will be filled with 999999 () \n'
             )
-            ## Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
+            # Fill missing/nan nwm 'Discharge (m3s-1)' values with 999999 to handle later
             merge_df['Q_find'] = merge_df['Q_find'].fillna(999999)
         if merge_df['hydroid'].isnull().values.any():
             log_text += 'HUC: ' + str(merge_df['huc']) + ' --> Null values found in "hydroid"... \n'
 
-        # Create dataframe with crosswalked USGS flow and NWM recurr flow
+        # Create dataframe with crosswalked USGS flow and NWM recurr flow &
+        # find the index of the closest matching flow btw USGS rating and NWM recur
         calc_df = merge_df.loc[merge_df.groupby(['location_id', 'levpa_id'])['Q_find'].idxmin()].reset_index(
             drop=True
-        )  # find the index of the Q_1_5_find (closest matching flow)
+        )
         # Calculate flow difference (variance) to check for large discrepancies between
         # NWM flow and USGS closest flow
         calc_df['check_variance'] = (
@@ -191,6 +194,77 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
     return final_df
 
 
+def trace_network(df, start_id):
+    # This function creates a list of all upstream & downstream hydroids
+    # Input: df --> dataframe of demDerived_reaches with network attribs
+    # Input: start_id --> hydroid value where the trace routine will start
+    current_id = start_id
+    trace_up = []
+    trace_down = []
+    start_order = None  # Variable to store the start_order
+    accumulated_length = 0
+
+    while True:
+        current_row = df[df['HydroID'] == current_id]
+
+        if current_row.empty:
+            break
+
+        next_id = current_row['NextDownID'].values[0]
+        order = current_row['order_'].values[0]
+        length = current_row['LengthKm'].values[0]
+        lake = current_row['LakeID'].values[0]
+
+        # Assign start_order when first encountered
+        if start_order is None:
+            start_order = order
+
+        if order != start_order:
+            break
+
+        accumulated_length += length
+        if accumulated_length >= float(USGS_CALB_TRACE_DIST):
+            break
+
+        if lake > 0:
+            break
+
+        # not dropping the HydroID that has the gauge location (need later)
+        trace_down.append(int(current_id))
+
+        current_id = next_id
+
+    current_id = start_id  # Reset current_id for tracing down
+    accumulated_length = 0
+
+    while True:
+        current_row = df[(df['NextDownID'] == current_id) & (df['order_'] == start_order)]
+        if current_row.empty:
+            break
+
+        next_id = current_row['HydroID'].values[0]
+        order = current_row['order_'].values[0]
+        length = current_row['LengthKm'].values[0]
+        lake = current_row['LakeID'].values[0]
+
+        if order != start_order:
+            break
+
+        accumulated_length += length
+        if accumulated_length >= float(USGS_CALB_TRACE_DIST):
+            break
+
+        if lake > 0:
+            break
+
+        if current_id != start_id:
+            trace_up.append(current_id)
+
+        current_id = next_id
+
+    return trace_up, trace_down
+
+
 def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
     procs_list = []  # Initialize list for mulitprocessing.
 
@@ -217,7 +291,67 @@ def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
                 'gw_catchments_reaches_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg',
             )
             htable_path = os.path.join(branch_dir, 'hydroTable_' + branch_id + '.csv')
-            water_edge_median_ds = usgs_df[(usgs_df['huc'] == huc) & (usgs_df['levpa_id'] == branch_id)]
+            dem_reaches_path = os.path.join(
+                branch_dir,
+                'demDerived_reaches_split_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg',
+            )
+            df = gpd.read_file(dem_reaches_path)
+            usgs_elev = usgs_df[(usgs_df['huc'] == huc) & (usgs_df['levpa_id'] == branch_id)]
+
+            # Calculate updstream/downstream trace ()
+            df = df[['HydroID', 'order_', 'LengthKm', 'NextDownID', 'LakeID']]
+
+            # Change the data type of 'HydroID' and 'NextDownID' to int
+            df['HydroID'] = df['HydroID'].astype(int)
+            df['NextDownID'] = df['NextDownID'].astype(int)
+
+            # Loop through every row in the "usgs_elev" dataframe
+            for index, row in usgs_elev.iterrows():
+                start_id = row['hydroid']
+
+                # Trace the network for each row
+                up, down = trace_network(df, start_id)
+
+                # Append the results to the "usgs_elev" dataframe
+                usgs_elev = usgs_elev.copy()
+                usgs_elev.loc[index, 'up'] = ','.join(map(str, up))
+                usgs_elev.loc[index, 'down'] = ','.join(map(str, down))
+
+            # Handle NaN values and ignore rows where up/down trace list is empty
+            usgs_elev['up'] = (
+                usgs_elev['up']
+                .astype(str)
+                .apply(lambda x: [num.strip() for num in x.split(',')] if pd.notna(x) else [])
+            )
+            usgs_elev['down'] = (
+                usgs_elev['down']
+                .astype(str)
+                .apply(lambda x: [num.strip() for num in x.split(',')] if pd.notna(x) else [])
+            )
+
+            # Combine the up & down hydroid lists into a new column
+            usgs_elev['trace_hydroid'] = [
+                lst1 + lst2 for lst1, lst2 in zip(usgs_elev['up'], usgs_elev['down'])
+            ]
+
+            # Drop up & down columns
+            columns_to_drop = ['up', 'down']
+            usgs_elev.drop(columns=columns_to_drop, inplace=True)
+
+            # Explode the trace column
+            usgs_elev_trace = usgs_elev.explode('trace_hydroid')
+
+            # Check for empty or nan trace lists and convert the column to integers
+            usgs_elev_trace['trace_hydroid'] = usgs_elev_trace['trace_hydroid'].replace('nan', 0)
+            usgs_elev_trace['trace_hydroid'] = usgs_elev_trace['trace_hydroid'].replace('', 0)
+            usgs_elev_trace['trace_hydroid'] = usgs_elev_trace['trace_hydroid'].astype(int)
+
+            # Rename columns
+            usgs_elev_trace.rename(columns={'hydroid': 'hydroid_gauge'}, inplace=True)
+            usgs_elev_trace.rename(columns={'trace_hydroid': 'hydroid'}, inplace=True)
+
+            if debug_outputs_option:
+                usgs_elev_trace.to_csv(os.path.join(branch_dir, 'water_edge_trace.csv'), index=False)
 
             # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
             if not os.path.exists(hand_path):
@@ -263,7 +397,7 @@ def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
                     + '\n'
                 )
             else:
-                ## Additional arguments for src_roughness_optimization
+                # Additional arguments for src_roughness_optimization
                 source_tag = 'usgs_rating'  # tag to use in source attribute field
                 merge_prev_adj = False  # merge in previous SRC adjustment calculations
 
@@ -271,7 +405,7 @@ def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
                 procs_list.append(
                     [
                         branch_dir,
-                        water_edge_median_ds,
+                        usgs_elev_trace,
                         htable_path,
                         huc,
                         branch_id,
@@ -287,6 +421,7 @@ def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
     with Pool(processes=job_number) as pool:
         log_output = pool.starmap(update_rating_curve, procs_list)
         log_file.writelines(["%s\n" % item for item in log_output])
+    # TO-DO update the error handling to properly capture issues in the multiprocessing
     # try statement for debugging
     # try:
     #     with Pool(processes=job_number) as pool:
@@ -300,10 +435,10 @@ def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
 
 
 def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number):
-    ## Check input args are valid
+    # Check input args are valid
     assert os.path.isdir(run_dir), 'ERROR: could not find the input fim_dir location: ' + str(run_dir)
 
-    ## Create an aggregate dataframe with all usgs_elev_table.csv entries for hucs in fim_dir
+    # Create an aggregate dataframe with all usgs_elev_table.csv entries for hucs in fim_dir
     print('Reading USGS gage HAND elevation from usgs_elev_table.csv files...')
     # usgs_elev_file = os.path.join(branch_dir,'usgs_elev_table.csv')
     # usgs_elev_df = pd.read_csv(
@@ -320,13 +455,13 @@ def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_optio
             + " max jobs will be used instead."
         )
 
-    ## Create output dir for log and usgs rc database
+    # Create output dir for log and usgs rc database
     log_dir = os.path.join(run_dir, "logs", "src_optimization")
     print("Log file output here: " + str(log_dir))
     if not os.path.isdir(log_dir):
         os.makedirs(log_dir)
 
-    ## Create a time var to log run time
+    # Create a time var to log run time
     begin_time = dt.datetime.now()
     # Create log file for processing records
     log_file = open(os.path.join(log_dir, 'log_usgs_rc_src_adjust.log'), "w")
@@ -352,11 +487,11 @@ def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_optio
         log_file.write("starting create usgs rating db")
         usgs_df = create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir)
 
-        ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
+        # Create huc proc_list for multiprocessing and execute the update_rating_curve function
         branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file)
 
-    ## Record run time and close log file
-    log_file.write('#########################################################\n\n')
+    # Record run time and close log file
+    log_file.write('########################################################\n\n')
     end_time = dt.datetime.now()
     log_file.write('END TIME: ' + str(end_time) + '\n')
     tot_run_time = end_time - begin_time
@@ -366,7 +501,7 @@ def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_optio
 
 
 if __name__ == '__main__':
-    ## Parse arguments.
+    # Parse arguments.
     parser = argparse.ArgumentParser(
         description='Adjusts rating curve with database of USGS rating curve (calculated WSE/flow).'
     )
@@ -390,7 +525,7 @@ def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_optio
     )
     parser.add_argument('-j', '--job-number', help='Number of jobs to use', required=False, default=1)
 
-    ## Assign variables from arguments.
+    # Assign variables from arguments.
     args = vars(parser.parse_args())
     run_dir = args['run_dir']
     usgs_rc_filepath = args['usgs_ratings']
@@ -398,5 +533,5 @@ def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_optio
     debug_outputs_option = args['extra_outputs']
     job_number = int(args['job_number'])
 
-    ## Prepare/check inputs, create log file, and spin up the proc list
+    # Prepare/check inputs, create log file, and spin up the proc list
     run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number)
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index 4c0913573..8eac9ce64 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -115,6 +115,7 @@ def update_rating_curve(
     log_text += "DOWNSTREAM_THRESHOLD: " + str(down_dist_thresh) + 'km\n'
     log_text += "Merge Previous Adj Values: " + str(merge_prev_adj) + '\n'
     df_nvalues = water_edge_median_df.copy()
+    df_nvalues.reset_index(inplace=True)
     df_nvalues = df_nvalues[
         (df_nvalues.hydroid.notnull()) & (df_nvalues.hydroid > 0)
     ]  # remove null entries that do not have a valid hydroid
@@ -243,6 +244,7 @@ def update_rating_curve(
             find_src_stage = df_htable_hydroid.loc[
                 df_htable_hydroid['stage'].sub(row.hand).abs().idxmin()
             ]  # find closest matching stage to the user provided HAND value
+
             ## copy the corresponding htable values for the matching stage->HAND lookup
             df_nvalues.loc[index, 'feature_id'] = find_src_stage.feature_id
             df_nvalues.loc[index, 'LakeID'] = find_src_stage.LakeID
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index c47cea1f5..99951e1d7 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -159,6 +159,7 @@
 OVERWRITE_ALL = 'OVERWRITE_ALL'
 
 # Rating Curve Adjustment (local calibration) variables
-DOWNSTREAM_THRESHOLD = 10  # distance in km to propogate new roughness values downstream
+USGS_CALB_TRACE_DIST = 8.0  # distance in km to walk the network and perform USGS rating calibation for SRCs
+DOWNSTREAM_THRESHOLD = 8.0  # distance in km to propogate new roughness values downstream (group mean)
 ROUGHNESS_MAX_THRESH = 0.8  # max allowable adjusted roughness value (void values larger than this)
 ROUGHNESS_MIN_THRESH = 0.001  # min allowable adjusted roughness value (void values smaller than this)

From 013353eed85f435eb9aba5c5715ad1b4f31a4597 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Mon, 6 May 2024 15:11:12 -0500
Subject: [PATCH 310/359] v4.4.15.1 Hotfix post-processing src adjust errors
 (#1138)

---
 docs/CHANGELOG.md                   | 11 ++++
 src/src_adjust_usgs_rating_trace.py | 30 +++++++++-
 src/src_roughness_optimization.py   | 89 ++++++++++++++++++++++-------
 3 files changed, 107 insertions(+), 23 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d658d1c64..39dbad90c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.4.15.1 - 2024-05-06 - [PR#1081](https://github.com/NOAA-OWP/inundation-mapping/pull/1038)
+
+This hotfix address a bug within the SRC adjustment routine to filter out USGS gauge locations that were conflated to lakeid reaches. These fatal errors were preventing `fim_post_processing.sh` from completing. There are also new try except blocks to handle potential errors when opening/writing SRC adjustment attributes to the catchment gpkg (unknown issues with collisions or corrupt gpkg files). Closes #1137 
+
+### Changes
+
+- `src/src_adjust_usgs_rating_trace.py`: Added filter for processing valid hydroids that meet criteria (i.e non-lakes) and more robust logging.
+- `src/src_roughness_optimization.py`: Added data checks and logging to ensure input calibration data files contains necessary attributes. Also included a new try/except block to trap and log issues with file collisions or corrupt catchment gpkg read/write.
+
+<br/><br/>
+
 ## v4.4.15.0 - 2024-04-17 - [PR#1081](https://github.com/NOAA-OWP/inundation-mapping/pull/1081)
 
 This enhancement includes changes to the SRC calibration routine that uses the USGS published rating curve database. The modifications attempt to mimic the technique used in the stage-based CatFIM where the USGS WSE/flow is propagated upstream and downstream of the gauge location. This closes #892 
diff --git a/src/src_adjust_usgs_rating_trace.py b/src/src_adjust_usgs_rating_trace.py
index 739a10af9..ac923a5be 100644
--- a/src/src_adjust_usgs_rating_trace.py
+++ b/src/src_adjust_usgs_rating_trace.py
@@ -296,7 +296,7 @@ def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
                 'demDerived_reaches_split_filtered_addedAttributes_crosswalked_' + branch_id + '.gpkg',
             )
             df = gpd.read_file(dem_reaches_path)
-            usgs_elev = usgs_df[(usgs_df['huc'] == huc) & (usgs_df['levpa_id'] == branch_id)]
+            usgs_elev = usgs_df[(usgs_df['huc'] == huc) & (usgs_df['levpa_id'].astype(int) == int(branch_id))]
 
             # Calculate updstream/downstream trace ()
             df = df[['HydroID', 'order_', 'LengthKm', 'NextDownID', 'LakeID']]
@@ -346,12 +346,36 @@ def branch_proc_list(usgs_df, run_dir, debug_outputs_option, log_file):
             usgs_elev_trace['trace_hydroid'] = usgs_elev_trace['trace_hydroid'].replace('', 0)
             usgs_elev_trace['trace_hydroid'] = usgs_elev_trace['trace_hydroid'].astype(int)
 
+            # Drop rows where 'trace_hydroid' column is empty
+            # Addresses backpool removals and lake gauges
+            usgs_elev_trace = usgs_elev_trace[usgs_elev_trace['trace_hydroid'].astype(int) != 0]
+
+            # Check that there are still valid entries in the usgs_elev
+            # May have filtered out all if all locs were lakes
+            if usgs_elev_trace.empty:
+                print(
+                    "ALERT: did not find any valid hydroids to process: "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                )
+                log_file.write(
+                    "ALERT: did not find any valid hydroids to process: "
+                    + str(huc)
+                    + ' - branch-id: '
+                    + str(branch_id)
+                    + '\n'
+                )
+                continue
+
             # Rename columns
             usgs_elev_trace.rename(columns={'hydroid': 'hydroid_gauge'}, inplace=True)
             usgs_elev_trace.rename(columns={'trace_hydroid': 'hydroid'}, inplace=True)
 
             if debug_outputs_option:
-                usgs_elev_trace.to_csv(os.path.join(branch_dir, 'water_edge_trace.csv'), index=False)
+                usgs_elev_trace.to_csv(
+                    os.path.join(branch_dir, 'water_edge_trace_' + str(branch_id) + '.csv'), index=False
+                )
 
             # Check to make sure the fim output files exist. Continue to next iteration if not and warn user.
             if not os.path.exists(hand_path):
@@ -484,7 +508,7 @@ def run_prep(run_dir, usgs_rc_filepath, nwm_recurr_filepath, debug_outputs_optio
 
     else:
         print('This may take a few minutes...')
-        log_file.write("starting create usgs rating db")
+        log_file.write("starting create usgs rating db\n")
         usgs_df = create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filepath, log_dir)
 
         # Create huc proc_list for multiprocessing and execute the update_rating_curve function
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index 8eac9ce64..7f3864875 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -255,6 +255,22 @@ def update_rating_curve(
             df_nvalues.loc[index, 'overbank_n'] = find_src_stage.overbank_n
             df_nvalues.loc[index, 'discharge_cms'] = find_src_stage.discharge_cms
 
+    if 'discharge_cms' not in df_nvalues:
+        print(
+            'ERROR: "discharge_cms" column does not exist in df_nvalues df: '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+        )
+        log_text += (
+            'ERROR: "discharge_cms" column does not exist in df_nvalues df: '
+            + str(huc)
+            + '  branch id: '
+            + str(branch_id)
+            + '\n'
+        )
+        return log_text
+
     ## Calculate calibration coefficient
     df_nvalues = df_nvalues.rename(columns={'hydroid': 'HydroID'})  # rename the previous ManningN column
     df_nvalues['hydroid_calb_coef'] = df_nvalues['discharge_cms'] / df_nvalues['flow']  # Qobs / Qsrc
@@ -430,27 +446,59 @@ def update_rating_curve(
 
                 ## Update the catchments polygon .gpkg with joined attribute - "src_calibrated"
                 if os.path.isfile(catchments_poly_path):
-                    input_catchments = gpd.read_file(catchments_poly_path)
-                    ## Create new "src_calibrated" column for viz query
-                    if (
-                        'src_calibrated' in input_catchments.columns
-                    ):  # check if this attribute already exists and drop if needed
-                        input_catchments = input_catchments.drop(
-                            ['src_calibrated', 'obs_source', 'calb_coef_final'], axis=1, errors='ignore'
+                    try:
+                        input_catchments = gpd.read_file(catchments_poly_path)
+                        ## Create new "src_calibrated" column for viz query
+                        if 'src_calibrated' in input_catchments.columns:
+                            input_catchments = input_catchments.drop(
+                                ['src_calibrated', 'obs_source', 'calb_coef_final'], axis=1, errors='ignore'
+                            )
+                        df_nmerge['src_calibrated'] = np.where(
+                            df_nmerge['calb_coef_final'].notnull(), 'True', 'False'
                         )
-                    df_nmerge['src_calibrated'] = np.where(
-                        df_nmerge['calb_coef_final'].notnull(), 'True', 'False'
-                    )
-                    output_catchments = input_catchments.merge(
-                        df_nmerge[['HydroID', 'src_calibrated', 'obs_source', 'calb_coef_final']],
-                        how='left',
-                        on='HydroID',
-                    )
-                    output_catchments['src_calibrated'].fillna('False', inplace=True)
-                    output_catchments.to_file(
-                        catchments_poly_path, driver="GPKG", index=False
-                    )  # overwrite the previous layer
-                    df_nmerge = df_nmerge.drop(['src_calibrated'], axis=1, errors='ignore')
+                        output_catchments = input_catchments.merge(
+                            df_nmerge[['HydroID', 'src_calibrated', 'obs_source', 'calb_coef_final']],
+                            how='left',
+                            on='HydroID',
+                        )
+                        output_catchments['src_calibrated'].fillna('False', inplace=True)
+
+                        try:
+                            output_catchments.to_file(
+                                catchments_poly_path, driver="GPKG", index=False, overwrite=True
+                            )  # overwrite the previous layer
+
+                        except Exception as e:
+                            error_message = (
+                                "ERROR occurred while writing to catchments gpkg "
+                                f"for huc: {huc} & branch id: {branch_id}"
+                            )
+                            print(error_message)
+                            log_text += f"{error_message}\n"
+                            log_text += f"Error details: {e}\n"
+
+                            # Delete the original GeoPackage file
+                            if os.path.exists(catchments_poly_path):
+                                os.remove(catchments_poly_path)
+                            try:
+                                # Attempt to write to the file again
+                                output_catchments.to_file(
+                                    catchments_poly_path, driver="GPKG", index=False, overwrite=True
+                                )
+                                log_text += 'Successful second attempt to write output_catchments gpkg' + '\n'
+                            except Exception as e:
+                                second_attempt_error_message = "ERROR: Failed to write to catchments gpkg file even after deleting the original"
+                                print(second_attempt_error_message)
+                                log_text += f"{second_attempt_error_message}\n"
+                                log_text += f"Second attempt error details: {e}\n"
+
+                    except Exception as e:
+                        print(f"Error reading GeoPackage file: {e}")
+                        log_text += f"Error reading GeoPackage file: {e}\n"
+                        output_catchments = None
+
+                df_nmerge = df_nmerge.drop(['src_calibrated'], axis=1, errors='ignore')
+
                 ## Optional ouputs:
                 #   1) merge_n_csv csv with all of the calculated n values
                 #   2) a catchments .gpkg with new joined attributes
@@ -468,6 +516,7 @@ def update_rating_curve(
                         )
                         output_catchments = input_catchments.merge(df_nmerge, how='left', on='HydroID')
                         output_catchments.to_file(output_catchments_fileName, driver="GPKG", index=False)
+                        output_catchments = None
 
                 ## Merge the final ManningN dataframe to the original hydroTable
                 df_nmerge = df_nmerge.drop(

From 58b3e36a4badd04e06229a79f3440ecc76fc6f80 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 6 May 2024 14:17:53 -0600
Subject: [PATCH 311/359] v4.4.15.2 Bug fix to prevent `listdir()` from
 returning filenames (#1133)

---
 docs/CHANGELOG.md                  | 15 +++++++++++++++
 src/bathy_src_adjust_topwidth.py   |  3 ++-
 src/identify_src_bankfull.py       |  2 +-
 src/subdiv_chan_obank_src.py       |  2 +-
 src/utils/shared_functions.py      |  3 ++-
 tools/vary_mannings_n_composite.py |  2 +-
 6 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 39dbad90c..0203f29f8 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.4.15.2 - 2024-05-06 - [PR#1133](https://github.com/NOAA-OWP/inundation-mapping/pull/1133)
+
+Bug fix for error when reading the subfolders of a directory using `listdir()` where files exist that start with an 8-digit number that are later interpreted as directories.
+
+### Changes
+
+The following files were modified to use `listdir()` to read only directories instead of both directories and files:
+- `src/`
+    - `bathy_src_adjust_topwidth.py`, `identify_src_bankfull.py`, `subdiv_chan_obank_src.py`, `utils/shared_functions.py`
+- `tools/vary_mannings_n_composite.py`
+
+
+<br/><br/>
+
+
 ## v4.4.15.1 - 2024-05-06 - [PR#1081](https://github.com/NOAA-OWP/inundation-mapping/pull/1038)
 
 This hotfix address a bug within the SRC adjustment routine to filter out USGS gauge locations that were conflated to lakeid reaches. These fatal errors were preventing `fim_post_processing.sh` from completing. There are also new try except blocks to handle potential errors when opening/writing SRC adjustment attributes to the catchment gpkg (unknown issues with collisions or corrupt gpkg files). Closes #1137 
diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py
index c2db145d0..0edeeaf13 100644
--- a/src/bathy_src_adjust_topwidth.py
+++ b/src/bathy_src_adjust_topwidth.py
@@ -3,6 +3,7 @@
 import argparse
 import datetime as dt
 import os
+import re
 import sys
 from multiprocessing import Pool
 from os import environ
@@ -540,7 +541,7 @@ def multi_process(bathy_rc_lookup, procs_list):
             begin_time = dt.datetime.now()
 
             ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
-            huc_list = os.listdir(fim_dir)
+            huc_list = [d for d in os.listdir(fim_dir) if re.match(r'^\d{8}$', d)]
             huc_pass_list = []
             for huc in huc_list:
                 if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
diff --git a/src/identify_src_bankfull.py b/src/identify_src_bankfull.py
index 5fe955ea4..8f38d4459 100755
--- a/src/identify_src_bankfull.py
+++ b/src/identify_src_bankfull.py
@@ -334,7 +334,7 @@ def run_prep(fim_dir, bankfull_flow_filepath, number_of_jobs, verbose, src_plot_
     ]
 
     df_bflows = pd.read_csv(bankfull_flow_filepath, dtype={'feature_id': int})
-    huc_list = os.listdir(fim_dir)
+    huc_list = [d for d in os.listdir(fim_dir) if re.match(r'^\d{8}$', d)]
     huc_list.sort()  # sort huc_list for helping track progress in future print statments
     huc_pass_list = []
     for huc in huc_list:
diff --git a/src/subdiv_chan_obank_src.py b/src/subdiv_chan_obank_src.py
index 1a8fa1435..e7482d1e7 100755
--- a/src/subdiv_chan_obank_src.py
+++ b/src/subdiv_chan_obank_src.py
@@ -408,7 +408,7 @@ def run_prep(fim_dir, mann_n_table, output_suffix, number_of_jobs, verbose, src_
         print('Running the variable_mannings_calc function...')
 
         ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
-        huc_list = os.listdir(fim_dir)
+        huc_list = [d for d in os.listdir(fim_dir) if re.match(r'^\d{8}$', d)]
         huc_list.sort()  # sort huc_list for helping track progress in future print statments
         for huc in huc_list:
             # if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 8344d5baa..45e078b12 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -3,6 +3,7 @@
 import glob
 import inspect
 import os
+import re
 from concurrent.futures import as_completed
 from datetime import datetime, timezone
 from os.path import splitext
@@ -216,7 +217,7 @@ def concat_huc_csv(fim_dir, csv_name):
     '''
 
     merged_csv = []
-    huc_list = os.listdir(fim_dir)
+    huc_list = [d for d in os.listdir(fim_dir) if re.match(r'^\d{8}$', d)]
     for huc in huc_list:
         if huc != 'logs':
             csv_file = os.path.join(fim_dir, huc, str(csv_name))
diff --git a/tools/vary_mannings_n_composite.py b/tools/vary_mannings_n_composite.py
index e451eb5dd..42a70aacd 100755
--- a/tools/vary_mannings_n_composite.py
+++ b/tools/vary_mannings_n_composite.py
@@ -340,7 +340,7 @@ def run_prep(
         print('Running the variable_mannings_calc function...')
 
         ## Loop through hucs in the fim_dir and create list of variables to feed to multiprocessing
-        huc_list = os.listdir(fim_dir)
+        huc_list = [d for d in os.listdir(fim_dir) if re.match(r'^\d{8}$', d)]
         for huc in huc_list:
             # if huc != 'logs' and huc[-3:] != 'log' and huc[-4:] != '.csv':
             if re.match(r'\d{8}', huc):

From 311ab97f75ecb82fe498c9e5a2936ae180423dc3 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 6 May 2024 14:22:09 -0600
Subject: [PATCH 312/359] v4.4.15.3 Fix KeyError in
 `mitigate_branch_outlet_backpool.py` (#1128)

---
 docs/CHANGELOG.md                      | 12 ++++++++++++
 src/mitigate_branch_outlet_backpool.py |  5 ++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0203f29f8..c3d6f97da 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.4.15.3 - 2024-05-06 - [PR#1128](https://github.com/NOAA-OWP/inundation-mapping/pull/1128)
+
+Fixes a KeyError in `src/mitigate_branch_outlet_backpool.py`.
+
+### Changes
+
+`src/mitigate_branch_outlet_backpool.py`: Addresses case where `catchments_df['outlier']` are all False.
+
+
+<br/><br/>
+
+
 ## v4.4.15.2 - 2024-05-06 - [PR#1133](https://github.com/NOAA-OWP/inundation-mapping/pull/1133)
 
 Bug fix for error when reading the subfolders of a directory using `listdir()` where files exist that start with an 8-digit number that are later interpreted as directories.
diff --git a/src/mitigate_branch_outlet_backpool.py b/src/mitigate_branch_outlet_backpool.py
index ecbf52b14..b056b9981 100755
--- a/src/mitigate_branch_outlet_backpool.py
+++ b/src/mitigate_branch_outlet_backpool.py
@@ -65,7 +65,10 @@ def catch_catchment_size_outliers(catchment_pixels_geom):
         catchments_df['outlier'] = abs(catchments_df['counts'] - mean_counts) > threshold
 
         # Quantify outliers
-        num_outlier = catchments_df['outlier'].value_counts()[True]
+        if catchments_df['outlier'].any():
+            num_outlier = catchments_df['outlier'].value_counts()[True]
+        elif ~catchments_df['outlier'].all():
+            num_outlier = 0
 
         if num_outlier == 0:
             print('No outliers detected in catchment size.')

From 2bdeb0da18bbd6ef6a89a37672487f1866e82f96 Mon Sep 17 00:00:00 2001
From: AliForghani-NOAA <124194814+AliForghani-NOAA@users.noreply.github.com>
Date: Mon, 6 May 2024 16:26:11 -0400
Subject: [PATCH 313/359] v4.4.15.4 Update wrds api and get new USGS gages data
 (#1115)

---
 .../usgs}/rating_curve_get_usgs_curves.py     | 29 ++++++++++++---
 docs/CHANGELOG.md                             | 18 ++++++++++
 pyproject.toml                                |  2 +-
 tools/tools_shared_functions.py               | 35 +++++++++++++++----
 4 files changed, 71 insertions(+), 13 deletions(-)
 rename {tools => data/usgs}/rating_curve_get_usgs_curves.py (94%)

diff --git a/tools/rating_curve_get_usgs_curves.py b/data/usgs/rating_curve_get_usgs_curves.py
similarity index 94%
rename from tools/rating_curve_get_usgs_curves.py
rename to data/usgs/rating_curve_get_usgs_curves.py
index 65553d4f2..e77e92857 100644
--- a/tools/rating_curve_get_usgs_curves.py
+++ b/data/usgs/rating_curve_get_usgs_curves.py
@@ -117,7 +117,7 @@ def write_categorical_flow_files(metadata, workspace):
         nws_lid = site.get('identifiers').get('nws_lid')
 
         # thresholds only provided for valid nws_lid.
-        if nws_lid == 'Bogus_ID':
+        if nws_lid == 'Bogus_ID' or nws_lid is None:
             continue
 
         # if invalid feature_id skip to next site
@@ -145,8 +145,11 @@ def write_categorical_flow_files(metadata, workspace):
                 all_data = pd.concat([all_data, data], ignore_index=True)
 
     # Write CatFIM flows to file
-    final_data = all_data[['feature_id', 'discharge_cms', 'recurr_interval']]
-    final_data.to_csv(workspace / 'catfim_flows_cms.csv', index=False)
+    print("writing for CatFIM")
+    if not all_data.empty:
+        final_data = all_data[['feature_id', 'discharge_cms', 'recurr_interval']]
+        final_data.to_csv(workspace / 'catfim_flows_cms.csv', index=False)
+
     return all_data
 
 
@@ -214,14 +217,16 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
         os.mkdir(workspace)
 
     # If 'all' option passed to list of gages sites, it retrieves all sites within CONUS.
-    print('getting metadata for all sites')
     if list_of_gage_sites == ['all']:
+        print('getting metadata for all sites')
         sites_gdf, sites_list, metadata_list = get_all_active_usgs_sites()
     # Otherwise, if a list of sites is passed, retrieve sites from WRDS.
     else:
         # Define arguments to retrieve metadata and then get metadata from WRDS
         select_by = 'usgs_site_code'
         selector = list_of_gage_sites
+        print("Selected sites :", selector)
+
         # Since there is a limit to number characters in url, split up selector if too many sites.
         max_sites = 150
         if len(selector) > max_sites:
@@ -252,6 +257,18 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
                 downstream_trace_distance=None,
             )
 
+        # Get a geospatial layer (gdf) for all acceptable sites
+        print("Aggregating WBD HUCs...")
+        _, sites_gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes=True)
+        if not sites_gdf.empty:
+            # Get a list of all sites in gdf
+            list_of_sites = sites_gdf['identifiers_usgs_site_code'].to_list()
+            # Rename gdf fields
+            sites_gdf.columns = sites_gdf.columns.str.replace('identifiers_', '')
+        else:
+            print("There is no acceptable site.")
+            sys.exit()
+
     # Create DataFrame to store all appended rating curves
     print('processing metadata')
     all_rating_curves = pd.DataFrame()
@@ -265,7 +282,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
 
         # Filter out sites that are not in contiguous US. If this section is removed be sure to test with
         #   datum adjustment section (region will need changed)
-        if usgs['state'] in ['Alaska', 'Puerto Rico', 'Virgin Islands', 'Hawaii']:
+        if usgs['state'] in ['Puerto Rico', 'Virgin Islands', 'Hawaii']:
             continue
         # Get rating curve for site
         location_ids = usgs['usgs_site_code']
@@ -283,6 +300,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
             # To prevent time-out errors
             time.sleep(sleep_time)
             # Get the datum adjustment to convert NGVD to NAVD. Region needs changed if not in CONUS.
+
             datum_adj_ft = ngvd_to_navd_ft(datum_info=usgs, region='contiguous')
 
             # If datum API failed, print message and skip site.
@@ -291,6 +309,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
                 api_failure_messages.append(api_message)
                 print(api_message)
                 continue
+
             # If datum adjustment succeeded, calculate datum in NAVD88
             navd88_datum = round(usgs['datum'] + datum_adj_ft, 2)
             message = f'{location_ids}:succesfully converted NGVD29 to NAVD88'
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c3d6f97da..dc1d3e722 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,24 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.15.4 - 2024-05-06 - [PR#1115](https://github.com/NOAA-OWP/inundation-mapping/pull/1115)
+
+This PR addresses issue #1040 and includes the following updates:
+- Upgraded to WRDS API version 3 and ensured schema compatibility of new USGS gages data.
+- Expanded data retrieval to include Alaska gages alongside CONUS gages. 
+- Enables retrieving SRC data for individual USGS gages, removing the necessity of using 'all' for the '-l' flag in rating_curve_get_usgs_curves.py." 
+
+
+### Changes
+ - `tools/tools_shared_functions.py`   
+    -  Improved the stability of API calls.
+    - Removed the exclusion of Alaska gages from USGS gages metadata (`usgs_gages.gpkg` output), preserving Alaska gages in the metadata.  
+- `rating_curve_get_usgs_curves.py` 
+    - Removed the exclusion of Alaska gages when retrieving SRC values.
+    - Enabled retrieving SRC data for individual USGS gages.
+- Moved the script `rating_curve_get_usgs_curves.py` from `tools` folder into `data/usgs`.
+
+<br/><br/>
 
 ## v4.4.15.3 - 2024-05-06 - [PR#1128](https://github.com/NOAA-OWP/inundation-mapping/pull/1128)
 
diff --git a/pyproject.toml b/pyproject.toml
index 0476fe148..ac0432b03 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -75,10 +75,10 @@ per-file-ignores = """
     src/build_stream_traversal.py: E722
     tools/vary_mannings_n_composite.py: E712
     tools/tools_shared_functions.py: F821, F841, E711
-    tools/rating_curve_get_usgs_curves.py: F841
     tools/rating_curve_comparison.py: F821, F841
     tools/run_test_case.py: E711
     tools/inundation.py: F821
     tools/eval_alt_catfim.py: F841
     tools/check_deep_flooding.py: E712
+    data/usgs/rating_curve_get_usgs_curves.py: F841
     """
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 4542cf879..f59c1be5b 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -19,7 +19,10 @@
 from gval import CatStats
 from rasterio import features
 from rasterio.warp import Resampling, calculate_default_transform, reproject
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
 from shapely.geometry import MultiPolygon, Polygon, shape
+from urllib3.util.retry import Retry
 
 
 gpd.options.io_engine = "pyogrio"
@@ -619,6 +622,9 @@ def get_metadata(
     params['must_include'] = must_include
     params['upstream_trace_distance'] = upstream_trace_distance
     params['downstream_trace_distance'] = downstream_trace_distance
+    # Suppress Insecure Request Warning
+    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
     # Request data from url
     response = requests.get(url, params=params, verify=False)
     #    print(response)
@@ -754,9 +760,6 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes=False):
     )
     joined_gdf = joined_gdf.drop(columns='index_wbd')
 
-    # Remove all Alaska HUCS (Not in NWM v2.0 domain)
-    joined_gdf = joined_gdf[~joined_gdf.states.str.contains('AK')]
-
     # Create a dictionary of huc [key] and nws_lid[value]
     dictionary = joined_gdf.groupby('HUC8')['identifiers_nws_lid'].apply(list).to_dict()
 
@@ -932,8 +935,18 @@ def get_thresholds(threshold_url, select_by, selector, threshold='all'):
     params = {}
     params['threshold'] = threshold
     url = f'{threshold_url}/{select_by}/{selector}'
-    response = requests.get(url, params=params, verify=False)
-    if response.ok:
+
+    # response = requests.get(url, params=params, verify=False)
+
+    # Call the API
+    session = requests.Session()
+    retry = Retry(connect=3, backoff_factor=0.5)
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+
+    response = session.get(url, params=params, verify=False)
+
+    if response.status_code == 200:
         thresholds_json = response.json()
         # Get metadata
         thresholds_info = thresholds_json['value_set']
@@ -1157,11 +1170,19 @@ def ngvd_to_navd_ft(datum_info, region='contiguous'):
     params['t_v_frame'] = 'NAVD88'  # Target vertical datum
     params['tar_vertical_unit'] = 'm'  # Target vertical height
 
+    # Suppress Insecure Request Warning
+    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
     # Call the API
-    response = requests.get(datum_url, params=params, verify=False)
+    session = requests.Session()
+    retry = Retry(connect=3, backoff_factor=0.5)
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+
+    response = session.get(datum_url, params=params, verify=False)
 
     # If successful get the navd adjustment
-    if response:
+    if response.status_code == 200:
         results = response.json()
         # Get adjustment in meters (NGVD29 to NAVD88)
         adjustment = results['t_z']

From ad51dc25a922d21aba8820a6cb08485545d8b083 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 6 May 2024 14:32:43 -0600
Subject: [PATCH 314/359] v4.4.16.0 Clip ocean mask from DEM and extend outlet
 streams (#1121)

---
 data/wbd/clip_vectors_to_wbd.py        | 177 +++++++++--
 data/wbd/generate_pre_clip_fim_huc8.py | 405 ++++++++++++++-----------
 docs/CHANGELOG.md                      |  13 +
 3 files changed, 385 insertions(+), 210 deletions(-)

diff --git a/data/wbd/clip_vectors_to_wbd.py b/data/wbd/clip_vectors_to_wbd.py
index ede7a6526..5d4b1ebbe 100755
--- a/data/wbd/clip_vectors_to_wbd.py
+++ b/data/wbd/clip_vectors_to_wbd.py
@@ -1,17 +1,83 @@
 #!/usr/bin/env python3
 
 import argparse
+import logging
 import sys
 
 import geopandas as gpd
 import pandas as pd
 import rasterio as rio
-from shapely.geometry import MultiPolygon, Polygon
+from shapely.geometry import LineString, MultiPolygon, Point, Polygon
+from shapely.ops import nearest_points
 
 from utils.shared_functions import getDriver
 
 
-# from utils.shared_variables import DEFAULT_FIM_PROJECTION_CRS
+gpd.options.io_engine = "pyogrio"
+
+
+def extend_outlet_streams(streams, wbd_buffered, wbd):
+    """
+    Extend outlet streams to nearest buffered WBD boundary
+    """
+
+    wbd['geometry'] = wbd.geometry.boundary
+    wbd = gpd.GeoDataFrame(data=wbd, geometry='geometry')
+
+    wbd_buffered["linegeom"] = wbd_buffered.geometry
+
+    # Select only the streams that are outlets
+    levelpath_outlets = streams[streams['to'] == 0]
+
+    # Select only the streams that don't intersect the WBD boundary line
+    levelpath_outlets = levelpath_outlets[~levelpath_outlets.intersects(wbd['geometry'].iloc[0])]
+
+    levelpath_outlets['nearest_point'] = None
+    levelpath_outlets['last'] = None
+
+    # print(list(levelpath_outlets.columns))
+
+    levelpath_outlets = levelpath_outlets.explode(index_parts=False)
+
+    for index, row in levelpath_outlets.iterrows():
+        coords = [(coords) for coords in list(row['geometry'].coords)]
+        last_coord = coords[-1]
+        levelpath_outlets.at[index, 'last'] = Point(last_coord)
+
+    wbd_buffered['geometry'] = wbd_buffered.geometry.boundary
+    wbd_buffered = gpd.GeoDataFrame(data=wbd_buffered, geometry='geometry')
+
+    for index, row in levelpath_outlets.iterrows():
+        levelpath_geom = row['last']
+        nearest_point = nearest_points(levelpath_geom, wbd_buffered)
+
+        levelpath_outlets.at[index, 'nearest_point'] = nearest_point[1]['geometry'].iloc[0]
+
+        levelpath_outlets_nearest_points = levelpath_outlets.at[index, 'nearest_point']
+        if isinstance(levelpath_outlets_nearest_points, pd.Series):
+            levelpath_outlets_nearest_points = levelpath_outlets_nearest_points.iloc[-1]
+
+        # tlist = list([levelpath_outlets.at[index, 'nearest_point'].coords[0]])
+        # tlist = list([levelpath_outlets.at[index, 'nearest_point'].row.geometry.get_coordinates().oords[0]])
+
+        levelpath_outlets.at[index, 'geometry'] = LineString(
+            list(row['geometry'].coords) + list([levelpath_outlets_nearest_points.coords[0]])
+        )
+
+        # levelpath_outlets.at[index, 'geometry'] = LineString(
+        #    list(row.geometry.get_coordinates()) + list([levelpath_outlets.at[index, 'nearest_point'].coords[0]])
+        # )
+
+        # geometry.get_coordinates()
+
+    levelpath_outlets = gpd.GeoDataFrame(data=levelpath_outlets, geometry='geometry')
+    levelpath_outlets = levelpath_outlets.drop(columns=['last', 'nearest_point'])
+
+    # Replace the streams in the original file with the extended streams
+    streams = streams[~streams['ID'].isin(levelpath_outlets['ID'])]
+    streams = pd.concat([streams, levelpath_outlets], ignore_index=True)
+
+    return streams
 
 
 def subset_vector_layers(
@@ -40,7 +106,8 @@ def subset_vector_layers(
     subset_levee_protected_areas,
     huc_CRS,
 ):
-    print(f"Getting Cell Size for {hucCode}", flush=True)
+
+    # print(f"Getting Cell Size for {hucCode}", flush=True)
     with rio.open(dem_filename) as dem_raster:
         dem_cellsize = max(dem_raster.res)
 
@@ -48,44 +115,71 @@ def subset_vector_layers(
     dem_domain = gpd.read_file(dem_domain, engine="pyogrio", use_arrow=True)
 
     # Get wbd buffer
-    print(f"Create wbd buffer for {hucCode}", flush=True)
+    logging.info(f"Create wbd buffer for {hucCode}")
     wbd_buffer = wbd.copy()
     wbd_buffer.geometry = wbd_buffer.geometry.buffer(wbd_buffer_distance, resolution=32)
     wbd_buffer = gpd.clip(wbd_buffer, dem_domain)
 
+    # Clip ocean water polygon for future masking ocean areas (where applicable)
+    logging.info(f"Clip ocean water polygon for {hucCode}")
+    landsea = gpd.read_file(landsea, mask=wbd_buffer, engine="fiona")
+    if not landsea.empty:
+        # print(f"Create landsea gpkg for {hucCode}", flush=True)
+        landsea.to_file(
+            subset_landsea, driver=getDriver(subset_landsea), index=False, crs=huc_CRS, engine="fiona"
+        )
+
+        # Exclude landsea area from WBD and wbd_buffer
+        wbd = wbd.overlay(landsea, how='difference')
+        wbd.to_file(
+            wbd_filename,
+            layer='WBDHU8',
+            driver=getDriver(wbd_filename),
+            index=False,
+            crs=huc_CRS,
+            engine="fiona",
+        )
+
+        wbd_buffer = wbd_buffer.overlay(landsea, how='difference')
+
+    del landsea
+
     # Make the streams buffer smaller than the wbd_buffer so streams don't reach the edge of the DEM
+    logging.info(f"Create stream buffer for {hucCode}")
     wbd_streams_buffer = wbd_buffer.copy()
     wbd_streams_buffer.geometry = wbd_streams_buffer.geometry.buffer(-8 * dem_cellsize, resolution=32)
 
     wbd_buffer = wbd_buffer[['geometry']]
     wbd_streams_buffer = wbd_streams_buffer[['geometry']]
-    wbd_buffer.to_file(wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False, crs=huc_CRS)
+    wbd_buffer.to_file(
+        wbd_buffer_filename, driver=getDriver(wbd_buffer_filename), index=False, crs=huc_CRS, engine="fiona"
+    )
     wbd_streams_buffer.to_file(
-        wbd_streams_buffer_filename, driver=getDriver(wbd_streams_buffer_filename), index=False, crs=huc_CRS
+        wbd_streams_buffer_filename,
+        driver=getDriver(wbd_streams_buffer_filename),
+        index=False,
+        crs=huc_CRS,
+        engine="fiona",
     )
 
-    # Clip ocean water polygon for future masking ocean areas (where applicable)
-    landsea = gpd.read_file(landsea, mask=wbd_buffer)
-    if not landsea.empty:
-        print(f"Create landsea gpkg for {hucCode}", flush=True)
-        landsea.to_file(subset_landsea, driver=getDriver(subset_landsea), index=False, crs=huc_CRS)
-    del landsea
-
     # Clip levee-protected areas polygons for future masking ocean areas (where applicable)
-    print(f"Subsetting Levee Protected Areas for {hucCode}", flush=True)
-    levee_protected_areas = gpd.read_file(levee_protected_areas, mask=wbd_buffer)
+    # print(f"Subsetting Levee Protected Areas for {hucCode}", flush=True)
+    logging.info(f"Clip levee-protected areas for {hucCode}")
+    levee_protected_areas = gpd.read_file(levee_protected_areas, mask=wbd_buffer, engine="fiona")
     if not levee_protected_areas.empty:
         levee_protected_areas.to_file(
             subset_levee_protected_areas,
             driver=getDriver(subset_levee_protected_areas),
             index=False,
             crs=huc_CRS,
+            engine="fiona",
         )
     del levee_protected_areas
 
     # Find intersecting lakes and writeout
-    print(f"Subsetting NWM Lakes for {hucCode}", flush=True)
-    nwm_lakes = gpd.read_file(nwm_lakes, mask=wbd_buffer)
+    # print(f"Subsetting NWM Lakes for {hucCode}", flush=True)
+    logging.info(f"Subsetting NWM Lakes for {hucCode}")
+    nwm_lakes = gpd.read_file(nwm_lakes, mask=wbd_buffer, engine="fiona")
     nwm_lakes = nwm_lakes.loc[nwm_lakes.Shape_Area < 18990454000.0]
 
     if not nwm_lakes.empty:
@@ -97,63 +191,79 @@ def subset_vector_layers(
         # Loop through the filled polygons and insert the new geometry
         for i in range(len(nwm_lakes_fill_holes.geoms)):
             nwm_lakes.loc[i, 'geometry'] = nwm_lakes_fill_holes.geoms[i]
-        nwm_lakes.to_file(subset_nwm_lakes, driver=getDriver(subset_nwm_lakes), index=False, crs=huc_CRS)
+        nwm_lakes.to_file(
+            subset_nwm_lakes, driver=getDriver(subset_nwm_lakes), index=False, crs=huc_CRS, engine="fiona"
+        )
     del nwm_lakes
 
     # Find intersecting levee lines
-    print(f"Subsetting NLD levee lines for {hucCode}", flush=True)
-    nld_lines = gpd.read_file(nld_lines, mask=wbd_buffer)
+    logging.info(f"Subsetting NLD levee lines for {hucCode}")
+    nld_lines = gpd.read_file(nld_lines, mask=wbd_buffer, engine="fiona")
     if not nld_lines.empty:
-        nld_lines.to_file(subset_nld_lines, driver=getDriver(subset_nld_lines), index=False, crs=huc_CRS)
+        nld_lines.to_file(
+            subset_nld_lines, driver=getDriver(subset_nld_lines), index=False, crs=huc_CRS, engine="fiona"
+        )
     del nld_lines
 
     # Preprocessed levee lines for burning
-    nld_lines_preprocessed = gpd.read_file(nld_lines_preprocessed, mask=wbd_buffer)
+    nld_lines_preprocessed = gpd.read_file(nld_lines_preprocessed, mask=wbd_buffer, engine="fiona")
     if not nld_lines_preprocessed.empty:
         nld_lines_preprocessed.to_file(
             subset_nld_lines_preprocessed,
             driver=getDriver(subset_nld_lines_preprocessed),
             index=False,
             crs=huc_CRS,
+            engine="fiona",
         )
     del nld_lines_preprocessed
 
     # Subset NWM headwaters
-    print(f"Subsetting NWM Headwater Points for {hucCode}", flush=True)
-    nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer)
+    logging.info(f"Subsetting NWM Headwater Points for {hucCode}")
+    nwm_headwaters = gpd.read_file(nwm_headwaters, mask=wbd_streams_buffer, engine="fiona")
 
     if len(nwm_headwaters) > 0:
         nwm_headwaters.to_file(
-            subset_nwm_headwaters, driver=getDriver(subset_nwm_headwaters), index=False, crs=huc_CRS
+            subset_nwm_headwaters,
+            driver=getDriver(subset_nwm_headwaters),
+            index=False,
+            crs=huc_CRS,
+            engine="fiona",
         )
     else:
         print("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
+        logging.info("No headwater point(s) within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
 
     del nwm_headwaters
 
     # Find intersecting nwm_catchments
-    print(f"Subsetting NWM Catchments for {hucCode}", flush=True)
-    nwm_catchments = gpd.read_file(nwm_catchments, mask=wbd_buffer)
+    # print(f"Subsetting NWM Catchments for {hucCode}", flush=True)
+    nwm_catchments = gpd.read_file(nwm_catchments, mask=wbd_buffer, engine="fiona")
 
     if len(nwm_catchments) > 0:
         nwm_catchments.to_file(
-            subset_nwm_catchments, driver=getDriver(subset_nwm_catchments), index=False, crs=huc_CRS
+            subset_nwm_catchments,
+            driver=getDriver(subset_nwm_catchments),
+            index=False,
+            crs=huc_CRS,
+            engine="fiona",
         )
     else:
         print("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
+        logging.info("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
 
     del nwm_catchments
 
     # Subset nwm streams
-    print(f"Subsetting NWM Streams for {hucCode}", flush=True)
-
-    nwm_streams = gpd.read_file(nwm_streams, mask=wbd_buffer)
+    logging.info(f"Subsetting NWM Streams for {hucCode}")
+    nwm_streams = gpd.read_file(nwm_streams, mask=wbd_buffer, engine="fiona")
 
     # NWM can have duplicate records, but appear to always be identical duplicates
     nwm_streams.drop_duplicates(subset="ID", keep="first", inplace=True)
 
+    nwm_streams = extend_outlet_streams(nwm_streams, wbd_buffer, wbd)
+
     nwm_streams_outlets = nwm_streams[~nwm_streams['to'].isin(nwm_streams['ID'])]
     nwm_streams_nonoutlets = nwm_streams[nwm_streams['to'].isin(nwm_streams['ID'])]
 
@@ -177,10 +287,11 @@ def subset_vector_layers(
         nwm_streams = pd.concat([nwm_streams_nonoutlets, nwm_streams_outlets])
 
         nwm_streams.to_file(
-            subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False, crs=huc_CRS
+            subset_nwm_streams, driver=getDriver(subset_nwm_streams), index=False, crs=huc_CRS, engine="fiona"
         )
     else:
         print("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
+        logging.info("No NWM stream segments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
     del nwm_streams
 
@@ -230,7 +341,7 @@ def subset_vector_layers(
         '-lps', '--subset-levee-protected-areas', help='Levee-protected areas subset', required=True
     )
 
-    parser.add_argument('-crs', '--huc-crs', help='HUC crs', required=True)
+    parser.add_argument('-crs', '--huc-CRS', help='HUC crs', required=True)
 
     args = vars(parser.parse_args())
 
diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
index d1391aaf7..6bf2ea581 100755
--- a/data/wbd/generate_pre_clip_fim_huc8.py
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -6,11 +6,15 @@
 import os
 import shutil
 import subprocess
-from multiprocessing import Pool
+import traceback
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
 
 from clip_vectors_to_wbd import subset_vector_layers
 from dotenv import load_dotenv
 
+from utils.shared_functions import FIM_Helpers as fh
+
 
 '''
     Overview:
@@ -27,8 +31,8 @@
 
     Usage:
         generate_pre_clip_fim_huc8.py
-            -n /data/inputs/pre_clip_huc8/24_3_20
-            -u /data/inputs/huc_lists/included_huc8.lst
+            -n /data/inputs/pre_clip_huc8/24_04_23
+            -u /data/inputs/huc_lists/included_huc8_withAlaska.lst
             -j 6
             -o
 
@@ -36,7 +40,7 @@
       If running this script to generate new data, modify the pre_clip_huc_dir variable in
       src/bash_variables.env to the corresponding outputs_dir argument after running and testing this script.
       The newly generated data should be created in a new folder using the format <year_month_day>
-             (i.e. September 26, 2023 would be 23_9_26)
+             (i.e. September 26, 2023 would be 23_09_26)
 '''
 
 srcDir = os.getenv('srcDir')
@@ -84,7 +88,7 @@
 wbd_buffer_int = int(wbd_buffer)
 
 
-def __setup_logger(outputs_dir):
+def __setup_logger(outputs_dir, huc=None):
     '''
     Set up logging to file. Since log file includes the date, it will be overwritten if this
     script is run more than once on the same day.
@@ -92,7 +96,10 @@ def __setup_logger(outputs_dir):
     datetime_now = dt.datetime.now(dt.timezone.utc)
     curr_date = datetime_now.strftime("%m_%d_%Y")
 
-    log_file_name = f"generate_pre_clip_fim_huc8_{curr_date}.log"
+    if huc is None:
+        log_file_name = f"generate_pre_clip_fim_huc8_{curr_date}.log"
+    else:
+        log_file_name = f"mp_{huc}_generate_pre_clip_fim_huc8_{curr_date}.log"
 
     log_file_path = os.path.join(outputs_dir, log_file_name)
 
@@ -118,6 +125,22 @@ def __setup_logger(outputs_dir):
     logging.info(f"\n \t Started: {start_time_string} \n")
 
 
+def __merge_mp_logs(outputs_dir):
+    log_file_list = list(Path(outputs_dir).rglob("mp_*"))
+    if len(log_file_list) > 0:
+        log_file_list.sort()
+
+    log_mp_rollup_file = os.path.join(outputs_dir, "mp_merged_logs.log")
+
+    with open(log_mp_rollup_file, 'a') as main_log:
+        # Iterate through list
+        for temp_log_file in log_file_list:
+            # Open each file in read mode
+            with open(temp_log_file) as infile:
+                main_log.write(infile.read())
+            os.remove(temp_log_file)
+
+
 def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     '''
     The function is the main driver of the program to iterate and parallelize writing
@@ -185,8 +208,6 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
 
         elif not os.path.isdir(os.path.join(outputs_dir, huc)):
             os.mkdir(os.path.join(outputs_dir, huc))
-            logging.info(f"Created directory: {outputs_dir}/{huc}, huc level files will be written there.")
-            print(f"Created directory: {outputs_dir}/{huc}, huc level files will be written there.")
 
     # Build arguments (procs_list) for each process to execute (huc_level_clip_vectors_to_wbd)
     procs_list = []
@@ -199,8 +220,23 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     # Parallelize each huc in hucs_to_parquet_list
     logging.info('Parallelizing HUC level wbd pre-clip vector creation. ')
     print('Parallelizing HUC level wbd pre-clip vector creation. ')
-    with Pool(processes=number_of_jobs) as pool:
-        pool.map(huc_level_clip_vectors_to_wbd, procs_list)
+    # with Pool(processes=number_of_jobs) as pool:
+    #    pool.map(huc_level_clip_vectors_to_wbd, procs_list)
+
+    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
+        futures = {}
+        for huc in hucs_to_pre_clip_list:
+            args = {"huc": huc, "outputs_dir": outputs_dir}
+            future = executor.submit(huc_level_clip_vectors_to_wbd, **args)
+            futures[future] = future
+
+        for future in as_completed(futures):
+            if future is not None:
+                if future.exception():
+                    raise future.exception()
+
+    print("Merging MP log files")
+    __merge_mp_logs(outputs_dir)
 
     # Get time metrics
     end_time = dt.datetime.now(dt.timezone.utc)
@@ -220,7 +256,7 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     print(f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}")
 
 
-def huc_level_clip_vectors_to_wbd(args):
+def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
     '''
     Create pre-clipped vectors at the huc level. Necessary to have this as an additional
     function for multiprocessing. This is mostly a wrapper for the subset_vector_layers() method in
@@ -242,182 +278,197 @@ def huc_level_clip_vectors_to_wbd(args):
     - .gpkg files* dependant on HUC's WBD (*differing amount based on individual huc)
     '''
 
-    # We have to explicitly unpack the args from pool.map()
-    huc = args[0]
-    outputs_dir = args[1]
-
-    huc_directory = os.path.join(outputs_dir, huc)
+    huc_processing_start = dt.datetime.now(dt.timezone.utc)
+    # with this in Multi-proc, it needs it's own logger and unique logging file.
+    __setup_logger(outputs_dir, huc)
+    logging.info(f"Processing {huc}")
+
+    try:
+
+        huc_directory = os.path.join(outputs_dir, huc)
+
+        # SET VARIABLES AND FILE INPUTS #
+        hucUnitLength = len(huc)
+        huc2Identifier = huc[:2]
+
+        # Check whether the HUC is in Alaska or not and assign the CRS and filenames accordingly
+        if huc2Identifier == '19':
+            huc_CRS = ALASKA_CRS
+            input_NHD_WBHD_layer = 'WBD_National_South_Alaska'
+            input_WBD_filename = input_WBD_gdb_Alaska
+            wbd_gpkg_path = f'{inputsDir}/wbd/WBD_National_South_Alaska.gpkg'
+        else:
+            huc_CRS = DEFAULT_FIM_PROJECTION_CRS
+            input_NHD_WBHD_layer = f"WBDHU{hucUnitLength}"
+            input_WBD_filename = input_WBD_gdb
+            wbd_gpkg_path = f'{inputsDir}/wbd/WBD_National.gpkg'
+
+        # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
+        if huc2Identifier == "04":
+            input_LANDSEA = f"{input_GL_boundaries}"
+            # print(f'Using {input_LANDSEA} for water body mask (Great Lakes)')
+        elif huc2Identifier == "19":
+            input_LANDSEA = f"{inputsDir}/landsea/water_polygons_alaska.gpkg"
+            # print(f'Using {input_LANDSEA} for water body mask (Alaska)')
+        else:
+            input_LANDSEA = f"{inputsDir}/landsea/water_polygons_us.gpkg"
+
+        print(f"\n Get WBD {huc}")
+
+        # TODO: Use Python API (osgeo.ogr) instead of using ogr2ogr executable
+        get_wbd_subprocess = subprocess.run(
+            [
+                'ogr2ogr',
+                '-f',
+                'GPKG',
+                '-t_srs',
+                huc_CRS,
+                f'{huc_directory}/wbd.gpkg',
+                input_WBD_filename,
+                input_NHD_WBHD_layer,
+                '-where',
+                f"HUC{hucUnitLength}='{huc}'",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=True,
+            universal_newlines=True,
+        )
 
-    # SET VARIABLES AND FILE INPUTS #
-    hucUnitLength = len(huc)
-    huc2Identifier = huc[:2]
+        logging.info(f"{huc} : {get_wbd_subprocess.stdout}")
+
+        if get_wbd_subprocess.stderr != "":
+            if "ERROR" in get_wbd_subprocess.stderr.upper():
+                msg = (
+                    f" - Creating -- {huc_directory}/wbd.gpkg"
+                    f"  ERROR -- details: ({get_wbd_subprocess.stderr})"
+                )
+                print(msg)
+                logging.info(msg)
+        else:
+            msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete \n"
+            print(msg)
+            logging.info(msg)
 
-    # Check whether the HUC is in Alaska or not and assign the CRS and filenames accordingly
-    if huc2Identifier == '19':
-        huc_CRS = ALASKA_CRS
-        input_NHD_WBHD_layer = 'WBD_National_South_Alaska'
-        input_WBD_filename = input_WBD_gdb_Alaska
-        wbd_gpkg_path = f'{inputsDir}/wbd/WBD_National_South_Alaska.gpkg'
-    else:
-        huc_CRS = DEFAULT_FIM_PROJECTION_CRS
-        input_NHD_WBHD_layer = f"WBDHU{hucUnitLength}"
-        input_WBD_filename = input_WBD_gdb
-        wbd_gpkg_path = f'{inputsDir}/wbd/WBD_National.gpkg'
-
-    # Define the landsea water body mask using either Great Lakes or Ocean polygon input #
-    if huc2Identifier == "04":
-        input_LANDSEA = f"{input_GL_boundaries}"
-        print(f'Using {input_LANDSEA} for water body mask (Great Lakes)')
-    elif huc2Identifier == "19":
-        input_LANDSEA = f"{inputsDir}/landsea/water_polygons_alaska.gpkg"
-        print(f'Using {input_LANDSEA} for water body mask (Alaska)')
-    else:
-        input_LANDSEA = f"{inputsDir}/landsea/water_polygons_us.gpkg"
-
-    print(f"\n Get WBD {huc}")
-
-    # TODO: Use Python API (osgeo.ogr) instead of using ogr2ogr executable
-    get_wbd_subprocess = subprocess.run(
-        [
-            'ogr2ogr',
-            '-f',
-            'GPKG',
-            '-t_srs',
-            huc_CRS,
-            f'{huc_directory}/wbd.gpkg',
-            input_WBD_filename,
-            input_NHD_WBHD_layer,
-            '-where',
-            f"HUC{hucUnitLength}='{huc}'",
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        check=True,
-        universal_newlines=True,
-    )
+        msg = f"Get Vector Layers and Subset {huc}"
+        # print(msg)
+        logging.info(msg)
 
-    msg = get_wbd_subprocess.stdout
-    print(msg)
-    logging.info(msg)
+        # Subset Vector Layers (after determining whether it's alaska or not)
+        if huc2Identifier == '19':
+            # Yes Alaska
+            subset_vector_layers(
+                subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
+                subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
+                hucCode=huc,
+                subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
+                wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
+                wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
+                wbd_filename=f"{huc_directory}/wbd.gpkg",
+                dem_filename=input_DEM_Alaska,
+                dem_domain=input_DEM_domain_Alaska,
+                nwm_lakes=input_nwm_lakes,
+                nwm_catchments=input_nwm_catchments_Alaska,
+                subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
+                nld_lines=input_NLD_Alaska,
+                nld_lines_preprocessed=input_levees_preprocessed_Alaska,
+                landsea=input_LANDSEA,
+                nwm_streams=input_nwm_flows_Alaska,
+                subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
+                nwm_headwaters=input_nwm_headwaters_Alaska,
+                subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
+                subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
+                wbd_buffer_distance=wbd_buffer_int,
+                levee_protected_areas=input_nld_levee_protected_areas_Alaska,
+                subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
+                huc_CRS=huc_CRS,  # TODO: simplify
+            )
 
-    if get_wbd_subprocess.stderr != "":
-        if "ERROR" in get_wbd_subprocess.stderr.upper():
-            msg = (
-                f" - Creating -- {huc_directory}/wbd.gpkg"
-                f"  ERROR -- details: ({get_wbd_subprocess.stderr})"
+        else:
+            # Not Alaska
+            subset_vector_layers(
+                subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
+                subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
+                hucCode=huc,
+                subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
+                wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
+                wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
+                wbd_filename=f"{huc_directory}/wbd.gpkg",
+                dem_filename=input_DEM,
+                dem_domain=input_DEM_domain,
+                nwm_lakes=input_nwm_lakes,
+                nwm_catchments=input_nwm_catchments,
+                subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
+                nld_lines=input_NLD,
+                nld_lines_preprocessed=input_levees_preprocessed,
+                landsea=input_LANDSEA,
+                nwm_streams=input_nwm_flows,
+                subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
+                nwm_headwaters=input_nwm_headwaters,
+                subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
+                subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
+                wbd_buffer_distance=wbd_buffer_int,
+                levee_protected_areas=input_nld_levee_protected_areas,
+                subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
+                huc_CRS=huc_CRS,  # TODO: simplify
             )
-            print(msg)
-            logging.error(msg)
-    else:
-        msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete \n"
+
+        msg = f" Completing Get Vector Layers and Subset: {huc} \n"
         print(msg)
         logging.info(msg)
 
-    msg = f"Get Vector Layers and Subset {huc}"
-    print(msg)
-    logging.info(msg)
-
-    # Subset Vector Layers (after determining whether it's alaska or not)
-    if huc2Identifier == '19':
-        # Yes Alaska
-        subset_vector_layers(
-            subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
-            subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
-            hucCode=huc,
-            subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
-            wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
-            wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
-            wbd_filename=f"{huc_directory}/wbd.gpkg",
-            dem_filename=input_DEM_Alaska,
-            dem_domain=input_DEM_domain_Alaska,
-            nwm_lakes=input_nwm_lakes,
-            nwm_catchments=input_nwm_catchments_Alaska,
-            subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
-            nld_lines=input_NLD_Alaska,
-            nld_lines_preprocessed=input_levees_preprocessed_Alaska,
-            landsea=input_LANDSEA,
-            nwm_streams=input_nwm_flows_Alaska,
-            subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
-            nwm_headwaters=input_nwm_headwaters_Alaska,
-            subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
-            subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
-            wbd_buffer_distance=wbd_buffer_int,
-            levee_protected_areas=input_nld_levee_protected_areas_Alaska,
-            subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
-            huc_CRS=huc_CRS,  # TODO: simplify
-        )
-
-    else:
-        # Not Alaska
-        subset_vector_layers(
-            subset_nwm_lakes=f"{huc_directory}/nwm_lakes_proj_subset.gpkg",
-            subset_nwm_streams=f"{huc_directory}/nwm_subset_streams.gpkg",
-            hucCode=huc,
-            subset_nwm_headwaters=f"{huc_directory}/nwm_headwater_points_subset.gpkg",
-            wbd_buffer_filename=f"{huc_directory}/wbd_buffered.gpkg",
-            wbd_streams_buffer_filename=f"{huc_directory}/wbd_buffered_streams.gpkg",
-            wbd_filename=f"{huc_directory}/wbd.gpkg",
-            dem_filename=input_DEM,
-            dem_domain=input_DEM_domain,
-            nwm_lakes=input_nwm_lakes,
-            nwm_catchments=input_nwm_catchments,
-            subset_nwm_catchments=f"{huc_directory}/nwm_catchments_proj_subset.gpkg",
-            nld_lines=input_NLD,
-            nld_lines_preprocessed=input_levees_preprocessed,
-            landsea=input_LANDSEA,
-            nwm_streams=input_nwm_flows,
-            subset_landsea=f"{huc_directory}/LandSea_subset.gpkg",
-            nwm_headwaters=input_nwm_headwaters,
-            subset_nld_lines=f"{huc_directory}/nld_subset_levees.gpkg",
-            subset_nld_lines_preprocessed=f"{huc_directory}/3d_nld_subset_levees_burned.gpkg",
-            wbd_buffer_distance=wbd_buffer_int,
-            levee_protected_areas=input_nld_levee_protected_areas,
-            subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
-            huc_CRS=huc_CRS,  # TODO: simplify
+        ## Clip WBD8 ##
+        print(f" Creating WBD buffer and clip version {huc}")
+
+        clip_wbd8_subprocess = subprocess.run(
+            [
+                'ogr2ogr',
+                '-f',
+                'GPKG',
+                '-t_srs',
+                huc_CRS,
+                '-clipsrc',
+                f'{huc_directory}/wbd_buffered.gpkg',
+                f'{huc_directory}/wbd8_clp.gpkg',
+                wbd_gpkg_path,
+                input_NHD_WBHD_layer,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=True,
+            universal_newlines=True,
         )
 
-    msg = f"\n\t Completing Get Vector Layers and Subset: {huc} \n"
-    print(msg)
-    logging.info(msg)
-
-    ## Clip WBD8 ##
-    print(f" Clip WBD {huc}")
-
-    clip_wbd8_subprocess = subprocess.run(
-        [
-            'ogr2ogr',
-            '-f',
-            'GPKG',
-            '-t_srs',
-            huc_CRS,
-            '-clipsrc',
-            f'{huc_directory}/wbd_buffered.gpkg',
-            f'{huc_directory}/wbd8_clp.gpkg',
-            wbd_gpkg_path,
-            input_NHD_WBHD_layer,
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        check=True,
-        universal_newlines=True,
-    )
+        # msg = clip_wbd8_subprocess.stdout
+        # print(f"{huc} : {msg}")
+        # logging.info(f"{huc} : {msg}")
+
+        if clip_wbd8_subprocess.stderr != "":
+            if "ERROR" in clip_wbd8_subprocess.stderr.upper():
+                msg = (
+                    f" - Creating -- {huc_directory}/wbd.gpkg"
+                    f"  ERROR -- details: ({clip_wbd8_subprocess.stderr})"
+                )
+                print(msg)
+                logging.info(msg)
+        else:
+            msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete"
+            print(msg)
+            logging.info(msg)
 
-    msg = clip_wbd8_subprocess.stdout
-    print(msg)
-    logging.info(msg)
+    except Exception:
+        print(f"*** An error occurred while processing {huc}")
+        print(traceback.format_exc())
+        logging.info(f"*** An error occurred while processing {huc}")
+        logging.info(traceback.format_exc())
+        print()
 
-    if clip_wbd8_subprocess.stderr != "":
-        if "ERROR" in clip_wbd8_subprocess.stderr.upper():
-            msg = (
-                f" - Creating -- {huc_directory}/wbd.gpkg"
-                f"  ERROR -- details: ({clip_wbd8_subprocess.stderr})"
-            )
-            print(msg)
-            logging.error(msg)
-    else:
-        msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete"
-        print(msg)
-        logging.info(msg)
+    huc_processing_end = dt.datetime.now(dt.timezone.utc)
+    time_duration = huc_processing_end - huc_processing_start
+    duraton_msg = f"\t \t run time for huc {huc}: is {str(time_duration).split('.')[0]}"
+    print(duraton_msg)
+    logging.info(duraton_msg)
+    return
 
 
 if __name__ == '__main__':
@@ -428,7 +479,7 @@ def huc_level_clip_vectors_to_wbd(args):
         usage='''
             ./generate_pre_clip_fim_huc8.py
                 -n /data/inputs/pre_clip_huc8/24_3_20
-                -u /data/inputs/huc_lists/included_huc8.lst
+                -u /data/inputs/huc_lists/included_huc8_withAlaska.lst
                 -j 6
                 -o
         ''',
@@ -438,7 +489,7 @@ def huc_level_clip_vectors_to_wbd(args):
         '-n',
         '--outputs_dir',
         help='Directory to output all of the HUC level .gpkg files. Use the format: '
-        '<year_month_day> (i.e. September 26, 2023 would be 2023_9_26)',
+        '<year_month_day> (i.e. September 26, 2023 would be 23_09_26)',
     )
     parser.add_argument('-u', '--huc_list', help='List of HUCs to genereate pre-clipped vectors for.')
     parser.add_argument(
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index dc1d3e722..64be87feb 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.16.0 - 2024-05-06 - [PR#1121](https://github.com/NOAA-OWP/inundation-mapping/pull/1121)
+
+Some NWM streams, particularly in coastal areas, fail to reach the edge of the DEM resulting in reverse flow. This issue was resolved by clipping the ocean mask from the buffered WBD and DEM, and any remaining streams that didn't have outlets reaching the edge of the buffered WBD boundary were extended by snapping the end to the nearest point on the buffered WBD.
+
+### Changes
+
+- `data/wbd/clip_vectors_to_wbd.py`: Clips `landsea` ocean mask from the buffered WBD and adds a function to extend outlet streams to the buffered WBD
+- `data/wbd/clip_vectors_to_wbd.py`: Updated multi-processing and added more logging.
+
+<br/><br/>
+
+
 ## v4.4.15.4 - 2024-05-06 - [PR#1115](https://github.com/NOAA-OWP/inundation-mapping/pull/1115)
 
 This PR addresses issue #1040 and includes the following updates:
@@ -142,6 +154,7 @@ The "black" packages is also be upgraded from 23.7.0 to 24.3.
 
 <br/><br/>
 
+
 ## v4.4.13.1 - 2024-03-11 - [PR#1086](https://github.com/NOAA-OWP/inundation-mapping/pull/1086)
 
 Fixes bug where levee-protected areas were not being masked from branch 0 DEMs.

From 4466db046fa01d2fffd0f16aa278d87f6dfc6a31 Mon Sep 17 00:00:00 2001
From: LauraKeys-NOAA <70279180+LauraKeys-NOAA@users.noreply.github.com>
Date: Mon, 6 May 2024 15:56:17 -0500
Subject: [PATCH 315/359] v4.5.0.0 Adding Open Street Map bridge data pulling
 and healing scripts (#1122)

---
 Pipfile                                  |   1 +
 Pipfile.lock                             | 151 ++++++-----
 data/bridges/pull_osm_bridges.py         | 311 +++++++++++++++++++++++
 data/wbd/clip_vectors_to_wbd.py          |  41 ++-
 data/wbd/generate_pre_clip_fim_huc8.py   |  43 +++-
 docs/CHANGELOG.md                        |  24 ++
 src/bash_variables.env                   |   3 +-
 src/delineate_hydros_and_produce_HAND.sh |  19 ++
 src/heal_bridges_osm.py                  | 146 +++++++++++
 src/utils/shared_functions.py            |  31 ---
 src/utils/shared_variables.py            |  59 -----
 11 files changed, 651 insertions(+), 178 deletions(-)
 create mode 100644 data/bridges/pull_osm_bridges.py
 create mode 100644 src/heal_bridges_osm.py

diff --git a/Pipfile b/Pipfile
index 35731f13c..16e8eed68 100644
--- a/Pipfile
+++ b/Pipfile
@@ -47,6 +47,7 @@ pyflwdir = "==0.5.8"
 pillow = "==10.3.0"
 pyogrio = "==0.7.2"
 openpyxl = "*"
+osmnx = "*"
 
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index a7d7bef21..4162d23ab 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "13d8feb3cb31feea69831b7d547993f4adde0134668a95c9744aa54d45cae748"
+            "sha256": "a7f3b5a12b051d29d0dee120d17361cca1ce11c412d15be4c207f6ca43e615b9"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -689,11 +689,11 @@
         },
         "datashader": {
             "hashes": [
-                "sha256:a2cb0f839067bf29cf6cc9c07a1dad35f0e4aed3768387056fcbac8748087bfa",
-                "sha256:ed4c111957578dcb3fcff972d954f77586dafd71a7345fd5cd069d9fb050d0d1"
+                "sha256:317c16a32a7d1c5208444a83fd6d2cb68d9bfb81802c323489bd24f6b7cf5c5f",
+                "sha256:5b2f1dd448defce206ecc16e9960fe9378f32307ea1fcb0c298c22c14f58c881"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.16.0"
+            "version": "==0.16.1"
         },
         "debugpy": {
             "hashes": [
@@ -764,11 +764,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14",
-                "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"
+                "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad",
+                "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.2.0"
+            "version": "==1.2.1"
         },
         "executing": {
             "hashes": [
@@ -906,11 +906,11 @@
         },
         "geocube": {
             "hashes": [
-                "sha256:3cd80d0866fec649001529b6d19502937b3f70be4b480fac53e0785ef8ef27d8",
-                "sha256:aa77b078516649d35b3907fcc630fc6a3c22e4d8b2b52add9a09c6b18dd3c462"
+                "sha256:2fb9e193c67f47f47cdade85748f17cd69bf8ac1cf5194d683846022b8d5c4e4",
+                "sha256:908e13900535a8ba2d6598ed3e69924f8dc60620bde242d760655d5a2a8244f2"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.5.1"
+            "version": "==0.5.2"
         },
         "geographiclib": {
             "hashes": [
@@ -946,18 +946,17 @@
         },
         "identify": {
             "hashes": [
-                "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791",
-                "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"
+                "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa",
+                "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.35"
+            "version": "==2.5.36"
         },
         "idna": {
             "hashes": [
                 "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
                 "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
             ],
-            "index": "pypi",
             "markers": "python_version >= '3.5'",
             "version": "==3.7"
         },
@@ -1124,11 +1123,11 @@
         },
         "json5": {
             "hashes": [
-                "sha256:0c638399421da959a20952782800e5c1a78c14e08e1dc9738fa10d8ec14d58c8",
-                "sha256:4ca101fd5c7cb47960c055ef8f4d0e31e15a7c6c48c3b6f1473fc83b6c462a13"
+                "sha256:34ed7d834b1341a86987ed52f3f76cd8ee184394906b6e22a1e0deb9ab294e8f",
+                "sha256:548e41b9be043f9426776f05df8635a00fe06104ea51ed24b67f908856e151ae"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.9.24"
+            "version": "==0.9.25"
         },
         "jsonpointer": {
             "hashes": [
@@ -1199,19 +1198,19 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:77b2b49c3831fbbfbdb5048cef4350d12946191f833a24e5f83e5f8f4803e97b",
-                "sha256:c80bfb049ea20053c3d9641c2add4848b38073bf79f1729cea1faed32fc1c78e"
+                "sha256:659154cea512083434fd7c93b7fe0897af7a2fd0b9dd4749282b42eaac4ae677",
+                "sha256:fb6be52c713e80e004fac34b35a0990d6d36ba06fd0a2b2ed82b899143a64210"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.13.0"
+            "version": "==2.14.0"
         },
         "jupyter-server-fileid": {
             "hashes": [
-                "sha256:7486bca3acf9bbaab7ce5127f9f64d2df58f5d2de377609fb833291a7217a6a2",
-                "sha256:76dd05a45b78c7ec0cba0be98ece289984c6bcfc1ca2da216d42930e506a4d68"
+                "sha256:76a2fbcea6950968485dcd509c2d6ac417ca11e61ab1ad447a475f0878ca808f",
+                "sha256:ffb11460ca5f8567644f6120b25613fca8e3f3048b38d14c6e3fe1902f314a9b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.9.1"
+            "version": "==0.9.2"
         },
         "jupyter-server-terminals": {
             "hashes": [
@@ -1255,11 +1254,11 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:54622cbd330526a385ee0c1fdccdff3a1e7219bf3e864a335284a1270a1973df",
-                "sha256:9b3ba91cf2837f7f124fca36d63f3ca80ace2bed4898a63dd47e6598c1ab006f"
+                "sha256:097b5ac709b676c7284ac9c5e373f11930a561f52cd5a86e4fc7e5a9c8a8631d",
+                "sha256:f5e26156e5258b24d532c84e7c74cc212e203bff93eb856f81c24c16daeecc75"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.26.0"
+            "version": "==2.27.1"
         },
         "jupyterlab-widgets": {
             "hashes": [
@@ -1536,11 +1535,11 @@
         },
         "matplotlib-inline": {
             "hashes": [
-                "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311",
-                "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"
+                "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90",
+                "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==0.1.6"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.1.7"
         },
         "mccabe": {
             "hashes": [
@@ -1755,6 +1754,14 @@
             "index": "pypi",
             "version": "==1.6.3"
         },
+        "networkx": {
+            "hashes": [
+                "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9",
+                "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"
+            ],
+            "markers": "python_version >= '3.10'",
+            "version": "==3.3"
+        },
         "nodeenv": {
             "hashes": [
                 "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2",
@@ -1884,11 +1891,11 @@
         },
         "numpy-groupies": {
             "hashes": [
-                "sha256:24a574ecdd58c0d669749b67ec31dab971d27e04037b402bbc4f6bedaa2623bc",
-                "sha256:f920c4ded899f5975d94fc63d634e7c89622056bbab8cc98a44d4320a0ae8a12"
+                "sha256:b0e9cf7a4d015beca969f1e90463fa90972df8380a722eb96d679f8838d2f6fe",
+                "sha256:c9f2471b0ee96e202fb0a591a551192c19fa3bdddad33df8665a7accbaf3263e"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.10.2"
+            "version": "==0.11.1"
         },
         "odc-geo": {
             "hashes": [
@@ -1906,6 +1913,14 @@
             "index": "pypi",
             "version": "==3.1.2"
         },
+        "osmnx": {
+            "hashes": [
+                "sha256:59d21756ebdb4a4d211fb1373a3dca170832eec5f25ef94f5500bd79d32c169a",
+                "sha256:9ea17500b45aa8e4740490d649c7f7747fa4f2e3f9747294837fa2c80aabd05f"
+            ],
+            "index": "pypi",
+            "version": "==1.9.2"
+        },
         "overrides": {
             "hashes": [
                 "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a",
@@ -2086,19 +2101,19 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068",
-                "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"
+                "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf",
+                "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.2.0"
+            "version": "==4.2.1"
         },
         "pluggy": {
             "hashes": [
-                "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981",
-                "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"
+                "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1",
+                "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.4.0"
+            "version": "==1.5.0"
         },
         "pre-commit": {
             "hashes": [
@@ -2980,11 +2995,11 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:5773bd84ef41799a5a8ca72dc34590c041eb01bf9aa02632b4a973fb0181a844",
-                "sha256:d53ae300ceddd3169f1ffa9caf2cb7b769e92657e4fafb23d34b93679116dfd4"
+                "sha256:191e936b0c696d0af17ad7430a3dc68e88bc11be6514f4757dc890f04ab05889",
+                "sha256:8080727b30e364e5783152903672df9b6b091c926a146a759080b62ca3126cd6"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.34.0"
+            "version": "==0.35.0"
         },
         "requests": {
             "hashes": [
@@ -3039,11 +3054,11 @@
         },
         "rioxarray": {
             "hashes": [
-                "sha256:9dc57a66b3551644322bba6c4d5d4c4647b0e41c481c9f03b287c1d9e16407b6",
-                "sha256:d12441e696dbc8a6113658e75326009ef6fddd3f865f4bcdd659d7fcfe4d8c7f"
+                "sha256:433b169cd10346ed5fe0123e61fafca4dcaf5ce1b6e2f6cc1a9c0e0bf7d7c1d4",
+                "sha256:849979c3542cefb5ac452af474a0b50ff08b7435c6db1ad615ddb65b1da4bbf3"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.15.3"
+            "version": "==0.15.5"
         },
         "rpds-py": {
             "hashes": [
@@ -3254,11 +3269,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e",
-                "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"
+                "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987",
+                "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==69.2.0"
+            "version": "==69.5.1"
         },
         "shapely": {
             "hashes": [
@@ -3486,11 +3501,11 @@
         },
         "tinycss2": {
             "hashes": [
-                "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847",
-                "sha256:8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627"
+                "sha256:152f9acabd296a8375fbca5b84c961ff95971fcfc32e79550c8df8e29118c54d",
+                "sha256:54a8dbdffb334d536851be0226030e9505965bb2f30f21a4a82c55fb2a80fae7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.2.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.3.0"
         },
         "tomli": {
             "hashes": [
@@ -3535,11 +3550,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:8cdd83c040dab7d1dee822678e5f5d100b514f7b72b01615b26fc5718916fdf9",
-                "sha256:fcdf85684a772ddeba87db2f398ce00b40ff550d1528c03c14dbf6a02003cd80"
+                "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7",
+                "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.14.2"
+            "version": "==5.14.3"
         },
         "typeguard": {
             "hashes": [
@@ -3597,11 +3612,11 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a",
-                "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"
+                "sha256:0846377ea76e818daaa3e00a4365c018bc3ac9760cbb3544de542885aad61fb3",
+                "sha256:ec25a9671a5102c8d2657f62792a27b48f016664c6873f6beed3800008577210"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.25.1"
+            "version": "==20.26.0"
         },
         "wcwidth": {
             "hashes": [
@@ -3626,11 +3641,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6",
-                "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"
+                "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526",
+                "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.7.0"
+            "version": "==1.8.0"
         },
         "whitebox": {
             "hashes": [
@@ -3893,11 +3908,11 @@
         },
         "matplotlib-inline": {
             "hashes": [
-                "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311",
-                "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"
+                "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90",
+                "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==0.1.6"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.1.7"
         },
         "parso": {
             "hashes": [
@@ -3970,11 +3985,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:8cdd83c040dab7d1dee822678e5f5d100b514f7b72b01615b26fc5718916fdf9",
-                "sha256:fcdf85684a772ddeba87db2f398ce00b40ff550d1528c03c14dbf6a02003cd80"
+                "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7",
+                "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.14.2"
+            "version": "==5.14.3"
         },
         "wcwidth": {
             "hashes": [
diff --git a/data/bridges/pull_osm_bridges.py b/data/bridges/pull_osm_bridges.py
new file mode 100644
index 000000000..7c7bf6183
--- /dev/null
+++ b/data/bridges/pull_osm_bridges.py
@@ -0,0 +1,311 @@
+import argparse
+import datetime as dt
+import logging
+import os
+import traceback
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+
+import geopandas as gpd
+import osmnx as ox
+import pandas as pd
+import pyproj
+from shapely.geometry import shape
+
+
+CRS = "epsg:5070"
+
+
+# Save all OSM bridge features by HUC8 to a specified folder location.
+# Bridges will have point geometry converted to linestrings if needed
+#
+def pull_osm_features_by_huc(huc_bridge_file, huc_num, huc_geom):
+    """
+    Returns: The huc number but only if it failed, so we can make a master list of failed HUCs.
+      The errors will be logged as it goes.
+    """
+    tags = {"bridge": True}
+    try:
+
+        if os.path.exists(huc_bridge_file):
+            logging.info(f" **{huc_bridge_file} already exists.. skipped")
+            # return huc_bridge_file
+            return ""
+
+        logging.info(f" ** Creating gkpg for {huc_num}")
+
+        gdf = ox.features_from_polygon(shape(huc_geom), tags)
+
+        if gdf is None or len(gdf) == 0:
+            logging.info(f"osmnx pull for {huc_num} came back with no records")
+            return huc_num
+
+        cols_to_drop = []
+        for col in gdf.columns:
+            if any(isinstance(val, list) for val in gdf[col]):
+                cols_to_drop.append(col)
+
+        # This a common and know duplicate column name (and others)
+        bad_column_names = ["atv", "fixme", "FIXME"]
+        for bad_cn in bad_column_names:
+            if bad_cn in gdf.columns:
+                cols_to_drop.append(bad_cn)
+
+        if len(cols_to_drop) > 0:
+            gdf = gdf.drop(columns=cols_to_drop, axis=1)
+
+        gdf1 = gdf[gdf.geometry.apply(lambda x: x.geom_type == 'LineString')]
+
+        gdf1 = gdf1.to_crs(CRS)
+        gdf1.to_file(huc_bridge_file, driver="GPKG")
+
+        # returns the HUC but only if it failed so we can keep a list of failed HUCs
+        return ""
+
+    except Exception:
+        logging.info(f"\t--- Couldn't write {huc_num}")
+        logging.info(traceback.format_exc())
+
+        try:
+            # rename and we can filter it out later. Even it fails sometimes
+            if os.path.exists(huc_bridge_file):
+                os.remove(huc_bridge_file)
+        except Exception as ex:
+            print(f"Unable to delete {huc_bridge_file} for huc {huc_num} to add 'bad_' in front")
+            print(ex)
+
+        return huc_num
+        # Continue on
+
+
+#
+# Combine all HUC-based OSM bridges from specified folder
+#
+def combine_huc_features(output_dir):
+
+    bridge_file_names = Path(output_dir).glob("huc_*_osm_bridges.gpkg")
+
+    osm_bridge_file = os.path.join(output_dir, "osm_all_bridges.gpkg")
+    osm_bridge_midpoints_file = os.path.join(output_dir, "osm_all_bridges_midpoints.gpkg")
+
+    all_bridges_gdf_raw = pd.concat([gpd.read_file(gpkg) for gpkg in bridge_file_names], ignore_index=True)
+
+    # only save out a subset of columns, because many hucs have different column names
+    # and data, so you could end up with thousands of columns if you keep them all!
+    logging.info(f"Writing bridge lines {osm_bridge_file}")
+    section_time = dt.datetime.now(dt.timezone.utc)
+    logging.info(f"  .. started: {section_time.strftime('%m/%d/%Y %H:%M:%S')}")
+
+    all_bridges_gdf = all_bridges_gdf_raw[['osmid', 'name', 'geometry']]
+    all_bridges_gdf.to_file(osm_bridge_file, driver="GPKG")
+
+    # save out midpoints (centroids) of bridge lines
+    logging.info(f"Writing centroids {osm_bridge_file}")
+    section_time = dt.datetime.now(dt.timezone.utc)
+    logging.info(f"  .. started: {section_time.strftime('%m/%d/%Y %H:%M:%S')}")
+    all_bridges_gdf['geometry'] = all_bridges_gdf.centroid
+    all_bridges_gdf.to_file(osm_bridge_midpoints_file, driver="GPKG")
+
+    return
+
+
+def process_osm_bridges(wbd_file, output_folder, number_of_jobs):
+    start_time = dt.datetime.now(dt.timezone.utc)
+    __setup_logger(output_folder)
+
+    print("==================================")
+    logging.info("Starting load of OSM bridge data")
+    logging.info(f"Start time: {start_time.strftime('%m/%d/%Y %H:%M:%S')}")
+    logging.info("")
+
+    # --------------------------
+    # Validation
+    if os.path.exists(wbd_file) is False:
+        raise Exception(f"The wbd file of {wbd_file} does not exist")
+
+    split_wbd_file_name = os.path.splitext(wbd_file)
+    if len(split_wbd_file_name) != 2:
+        raise Exception(f"The wbd file of {wbd_file} does not appear to valid file name")
+
+    if str(split_wbd_file_name[1]).lower() != ".gpkg":
+        raise Exception(f"The wbd file of {wbd_file} does not appear to valid a gpkg")
+
+    # -------------------
+    # Validation
+    total_cpus_available = os.cpu_count() - 2
+    if number_of_jobs > total_cpus_available:
+        raise ValueError(
+            f'The number of jobs provided: {number_of_jobs} ,'
+            ' exceeds your machine\'s available CPU count minus two.'
+            ' Please lower the number of jobs value accordingly.'
+        )
+
+    # --------------------------
+    if not os.path.exists(output_folder):
+        os.mkdir(output_folder)
+
+    logging.info("*** Reading in and reprojecting the WBD HUC8 file")
+    logging.info("*** Depending on WBD size, this can take a bit. 5 to 40 mins is not uncommon.")
+
+    huc8s = gpd.read_file(wbd_file)
+
+    if len(huc8s) == 0:
+        raise Exception("wbd_file has no records")
+
+    logging.info(f"wbd rec count is {len(huc8s)}")
+    section_time = dt.datetime.now(dt.timezone.utc)
+    logging.info(f"WBD Loaded: {section_time.strftime('%m/%d/%Y %H:%M:%S')}")
+
+    # osm seems to like 4326
+    logging.info("")
+    logging.info("Reprojecting to 4326 (osm seems to like that one)")
+    huc8s = huc8s.to_crs(pyproj.CRS.from_string("epsg:4326"))
+    section_time = dt.datetime.now(dt.timezone.utc)
+    logging.info(f"Reprojection done: {section_time.strftime('%m/%d/%Y %H:%M:%S')}")
+
+    failed_HUCs_list = []
+    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
+        futures = {}
+        for row in huc8s.iterrows():
+            huc = row[1]
+            huc_bridge_file = os.path.join(output_folder, f"huc_{huc['HUC8']}_osm_bridges.gpkg")
+            args = {"huc_num": huc['HUC8'], "huc_bridge_file": huc_bridge_file, "huc_geom": huc['geometry']}
+            future = executor.submit(pull_osm_features_by_huc, **args)
+            futures[future] = future
+
+        for future in as_completed(futures):
+            if future is not None:
+                if not future.exception():
+                    failed_huc = future.result()
+                    if failed_huc != "":
+                        failed_HUCs_list.append(failed_huc)
+                else:
+                    raise future.exception()
+
+    logging.info("")
+    logging.info("Pulling hucs now complete")
+
+    logging.info("")
+
+    section_time = dt.datetime.now(dt.timezone.utc)
+    logging.info(f"Combining feature files started: {section_time.strftime('%m/%d/%Y %H:%M:%S')}")
+
+    # all huc8 processing must be completed before this function call
+    combine_huc_features(output_folder)
+
+    if len(failed_HUCs_list) > 0:
+        logging.info("\n+++++++++++++++++++")
+        logging.info("HUCs that failed to download from OSM correctly are:")
+        huc_error_msg = "  -- "
+        for huc in failed_HUCs_list:
+            huc_error_msg += f", {huc} "
+        logging.info(huc_error_msg)
+        logging.info("  See logs for more details on each HUC fail")
+        logging.info("+++++++++++++++++++")
+
+    # Get time metrics
+    end_time = dt.datetime.now(dt.timezone.utc)
+    logging.info("")
+    logging.info("==================================")
+    logging.info("OSM bridge data load complete")
+    logging.info(f"   End time: {end_time.strftime('%m/%d/%Y %H:%M:%S')}")
+
+    time_delta = end_time - start_time
+    total_seconds = int(time_delta.total_seconds())
+
+    ___, rem_seconds = divmod(total_seconds, 60 * 60 * 24)
+    total_hours, rem_seconds = divmod(rem_seconds, 60 * 60)
+    total_mins, seconds = divmod(rem_seconds, 60)
+
+    time_fmt = f"{total_hours:02d} hours {total_mins:02d} mins {seconds:02d} secs"
+
+    logging.info(f"Duration: {time_fmt}")
+    print()
+    return
+
+
+def __setup_logger(outputs_dir):
+    '''
+    Set up logging to file. Since log file includes the date, it will be overwritten if this
+    script is run more than once on the same day.
+    '''
+    start_time = dt.datetime.now(dt.timezone.utc)
+    file_dt_string = start_time.strftime("%y%m%d-%H%M")
+
+    script_file_name = os.path.basename(__file__).split('.')[0]
+    file_name = f"{script_file_name}-{file_dt_string}.log"
+
+    log_file_path = os.path.join(outputs_dir, file_name)
+
+    if not os.path.exists(outputs_dir):
+        os.mkdir(outputs_dir)
+
+    # set up logging to file
+    logging.basicConfig(
+        filename=log_file_path, level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%H:%M:%S'
+    )
+
+    # set up logging to console
+    console = logging.StreamHandler()
+    console.setLevel(logging.DEBUG)
+    # set a format which is simpler for console use
+    # add the handler to the root logger
+    logging.getLogger('').addHandler(console)
+
+    # logger = logging.getLogger(__name__)
+
+
+if __name__ == "__main__":
+    '''
+    Sample usage (min params):
+        python3 data/bridges/pull_osm_bridges.py
+            -w /data/inputs/wbd/WBD_National_HUC8.gpkg
+            -p /data/inputs/osm/bridges/
+            -j 6
+    Notes:
+        - This tool is meant to pull down all the Open Street Map bridge data for CONUS as a
+        precursor to the bridge healing pre-processing (so, a pre-pre-processing step).
+        It should be run only as often as the user thinks OSM has had any important updates.
+        - As written, the code will skip any HUC that there's already a file for.
+        - Each HUC8's worth of OSM bridge features is saved out individually, then merged together
+        into one. The HUC8 files can be deleted if desired, as an added final cleanup step.
+    '''
+
+    parser = argparse.ArgumentParser(description='Acquires and saves Open Street Map bridge features')
+
+    parser.add_argument(
+        '-w',
+        '--wbd_file',
+        help='REQUIRED: location the gpkg file that will'
+        ' contain all the HUC8 clip regions in one layer. Must contain field \'HUC8\'.',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-p',
+        '--output_folder',
+        help='REQUIRED: folder path location where individual HUC8 geopackages'
+        ' will be saved to after being downloaded from OSM.'
+        ' File names are hardcoded to format hucxxxxxxxx_osm_bridges.gpkg,'
+        ' with xxxxxxxx as the HUC8 value',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-j',
+        '--number_of_jobs',
+        help='OPTIONAL: Number of (jobs) cores/processes to used.',
+        required=False,
+        default=1,
+        type=int,
+    )
+
+    args = vars(parser.parse_args())
+
+    try:
+        process_osm_bridges(**args)
+
+    except Exception:
+        logging.info(traceback.format_exc())
+        end_time = dt.datetime.now(dt.timezone.utc)
+        logging.info(f"   End time: {end_time.strftime('%m/%d/%Y %H:%M:%S')}")
diff --git a/data/wbd/clip_vectors_to_wbd.py b/data/wbd/clip_vectors_to_wbd.py
index 5d4b1ebbe..048539781 100755
--- a/data/wbd/clip_vectors_to_wbd.py
+++ b/data/wbd/clip_vectors_to_wbd.py
@@ -35,8 +35,6 @@ def extend_outlet_streams(streams, wbd_buffered, wbd):
     levelpath_outlets['nearest_point'] = None
     levelpath_outlets['last'] = None
 
-    # print(list(levelpath_outlets.columns))
-
     levelpath_outlets = levelpath_outlets.explode(index_parts=False)
 
     for index, row in levelpath_outlets.iterrows():
@@ -57,19 +55,10 @@ def extend_outlet_streams(streams, wbd_buffered, wbd):
         if isinstance(levelpath_outlets_nearest_points, pd.Series):
             levelpath_outlets_nearest_points = levelpath_outlets_nearest_points.iloc[-1]
 
-        # tlist = list([levelpath_outlets.at[index, 'nearest_point'].coords[0]])
-        # tlist = list([levelpath_outlets.at[index, 'nearest_point'].row.geometry.get_coordinates().oords[0]])
-
         levelpath_outlets.at[index, 'geometry'] = LineString(
             list(row['geometry'].coords) + list([levelpath_outlets_nearest_points.coords[0]])
         )
 
-        # levelpath_outlets.at[index, 'geometry'] = LineString(
-        #    list(row.geometry.get_coordinates()) + list([levelpath_outlets.at[index, 'nearest_point'].coords[0]])
-        # )
-
-        # geometry.get_coordinates()
-
     levelpath_outlets = gpd.GeoDataFrame(data=levelpath_outlets, geometry='geometry')
     levelpath_outlets = levelpath_outlets.drop(columns=['last', 'nearest_point'])
 
@@ -104,6 +93,9 @@ def subset_vector_layers(
     wbd_buffer_distance,
     levee_protected_areas,
     subset_levee_protected_areas,
+    osm_bridges,
+    subset_osm_bridges,
+    is_alaska,
     huc_CRS,
 ):
 
@@ -249,12 +241,35 @@ def subset_vector_layers(
             engine="fiona",
         )
     else:
-        print("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
         logging.info("No NWM catchments within HUC " + str(hucCode) + " boundaries.")
         sys.exit(0)
 
     del nwm_catchments
 
+    # Subset OSM (Open Street Map) bridges
+    if osm_bridges != "":
+        logging.info(f"Subsetting OSM Bridges for {hucCode}")
+
+        subset_osm_bridges_gdb = gpd.read_file(osm_bridges, mask=wbd_buffer, engine="fiona")
+        if subset_osm_bridges_gdb.empty:
+            print("-- No applicable bridges for this HUC")
+            logging.info("-- No applicable bridges for this HUC")
+        else:
+            logging.info(f"Create subset of osm bridges gpkg for {hucCode}")
+            if is_alaska is True:
+                # we need to reproject
+                subset_osm_bridges_gdb = subset_osm_bridges_gdb.to_crs(huc_CRS)
+
+            subset_osm_bridges_gdb.to_file(
+                subset_osm_bridges,
+                driver=getDriver(subset_osm_bridges),
+                index=False,
+                crs=huc_CRS,
+                engine="fiona",
+            )
+
+        del subset_osm_bridges_gdb
+
     # Subset nwm streams
     logging.info(f"Subsetting NWM Streams for {hucCode}")
     nwm_streams = gpd.read_file(nwm_streams, mask=wbd_buffer, engine="fiona")
@@ -340,7 +355,7 @@ def subset_vector_layers(
     parser.add_argument(
         '-lps', '--subset-levee-protected-areas', help='Levee-protected areas subset', required=True
     )
-
+    parser.add_argument('-osm', '--osm-bridges', help='Open Street Maps gkpg', required=True)
     parser.add_argument('-crs', '--huc-CRS', help='HUC crs', required=True)
 
     args = vars(parser.parse_args())
diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
index 6bf2ea581..8a34e6837 100755
--- a/data/wbd/generate_pre_clip_fim_huc8.py
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -83,6 +83,8 @@
 input_nld_levee_protected_areas = os.getenv('input_nld_levee_protected_areas')
 input_nld_levee_protected_areas_Alaska = os.getenv('input_nld_levee_protected_areas_Alaska')
 
+input_osm_bridges = os.getenv('osm_bridges')
+
 # Variables from config/params_template.env
 wbd_buffer = os.getenv('wbd_buffer')
 wbd_buffer_int = int(wbd_buffer)
@@ -94,7 +96,7 @@ def __setup_logger(outputs_dir, huc=None):
     script is run more than once on the same day.
     '''
     datetime_now = dt.datetime.now(dt.timezone.utc)
-    curr_date = datetime_now.strftime("%m_%d_%Y")
+    curr_date = datetime_now.strftime("%y%my%d")
 
     if huc is None:
         log_file_name = f"generate_pre_clip_fim_huc8_{curr_date}.log"
@@ -132,14 +134,32 @@ def __merge_mp_logs(outputs_dir):
 
     log_mp_rollup_file = os.path.join(outputs_dir, "mp_merged_logs.log")
 
+    error_huc_found = False
+
     with open(log_mp_rollup_file, 'a') as main_log:
         # Iterate through list
         for temp_log_file in log_file_list:
             # Open each file in read mode
             with open(temp_log_file) as infile:
-                main_log.write(infile.read())
+                contents = infile.read()
+                temp_upper_contents = contents.upper()
+                if "ERROR" in temp_upper_contents:
+                    print(
+                        f"\nAn error exist in file {temp_log_file}."
+                        " Check the merge logs for that huc number"
+                    )
+                    error_huc_found = True
+                main_log.write(contents)
             os.remove(temp_log_file)
 
+    if error_huc_found:
+        print(
+            "\n\nOften you can just create a new huc list with the fails, re-run to a"
+            " 1temp directory and recheck if errors still exists. Sometimes multi-prod can create"
+            " contention errors.\nFor each HUC that is sucessful, you can just copy it back"
+            " into the original full pre-clip folder.\n"
+        )
+
 
 def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     '''
@@ -175,10 +195,6 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
         )
         number_of_jobs = total_cpus_available - 2
 
-    # Set up logging and set start_time
-    __setup_logger(outputs_dir)
-    start_time = dt.datetime.now(dt.timezone.utc)
-
     # Read in huc_list file and turn into a list data structure
     if os.path.exists(huc_list):
         hucs_to_pre_clip_list = open(huc_list).read().splitlines()
@@ -192,6 +208,10 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
             " is to re-generate all of the data. "
         )
 
+    # Set up logging and set start_time
+    __setup_logger(outputs_dir)
+    start_time = dt.datetime.now(dt.timezone.utc)
+
     # Iterate over the huc_list argument and create a directory for each huc.
     for huc in hucs_to_pre_clip_list:
         if os.path.isdir(os.path.join(outputs_dir, huc)):
@@ -223,6 +243,11 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     # with Pool(processes=number_of_jobs) as pool:
     #    pool.map(huc_level_clip_vectors_to_wbd, procs_list)
 
+    # TODO: Mar 5, 2024: Python native logging does not work well with Multi-proc. We will
+    # likely eventually drop in ras2fim's logging system.
+    # The log files for each multi proc has tons and tons of duplicate lines crossing mp log
+    # processes, but does always log correctly back to the parent log
+
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
         futures = {}
         for huc in hucs_to_pre_clip_list:
@@ -381,6 +406,9 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
                 wbd_buffer_distance=wbd_buffer_int,
                 levee_protected_areas=input_nld_levee_protected_areas_Alaska,
                 subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
+                osm_bridges=input_osm_bridges,
+                subset_osm_bridges=f"{huc_directory}/osm_bridges_subset.gpkg",
+                is_alaska=True,
                 huc_CRS=huc_CRS,  # TODO: simplify
             )
 
@@ -410,6 +438,9 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
                 wbd_buffer_distance=wbd_buffer_int,
                 levee_protected_areas=input_nld_levee_protected_areas,
                 subset_levee_protected_areas=f"{huc_directory}/LeveeProtectedAreas_subset.gpkg",
+                osm_bridges=input_osm_bridges,
+                subset_osm_bridges=f"{huc_directory}/osm_bridges_subset.gpkg",
+                is_alaska=False,
                 huc_CRS=huc_CRS,  # TODO: simplify
             )
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 64be87feb..754885541 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,30 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.0.0 - 2024-05-06 - [PR#1122](https://github.com/NOAA-OWP/inundation-mapping/pull/1122)
+
+This PR includes 2 scripts to add Open Street Map bridge data into the HAND process: a script that pulls data from OSM and a script that heals those bridges in the HAND grids. Both scripts should be run as part of a pre-processing step for FIM runs. They only need to be run if we think OSM data has changed a lot or for any new FIM versions.
+
+A new docker image is also required for `pull_osm_bridges.py` (acquire and preprocess) script.
+
+### Additions
+- `data/bridges/pull_osm_bridges.py`: First pre-processing script that pulls OSM data and saves bridge lines out as separate shapefiles by HUC8 to a specified location
+- `src/heal_bridges_osm.py`: Second pre-processing script that uses the pre-saved OSM bridge lines and heals max HAND values across those bridge lines. Healed HAND grids are saved to a specified location.
+
+### Changes
+- `Pipfile`, `Pipfile.lock`: Adjusted files to add new python package to docker image.
+- `data`
+    - `clip_vectors_to_wdbd.py`: Updated to pre-clip new bridge data. Logging upgraded.
+    - `generate_pre_clip_fim_huc8.py`: Updated to pre-clip new bridge data. Logging added and a system for multi-process logging.
+- `src`
+    - `delineate_hydros_and_produce_HAND.sh`: add python call to run `heal_bridges_osm.py` after hydraulic properties are calculated.
+    - `bash_variables.env`: Added new variable for OSM bridges and adjusted pre-clip output date
+    - `utils`
+        - `shared_functions.py`: removed function no longer in use.
+        - `shared_variables.py`: removed variables no longer in use.
+  
+<br/><br/>
+
 ## v4.4.16.0 - 2024-05-06 - [PR#1121](https://github.com/NOAA-OWP/inundation-mapping/pull/1121)
 
 Some NWM streams, particularly in coastal areas, fail to reach the edge of the DEM resulting in reverse flow. This issue was resolved by clipping the ocean mask from the buffered WBD and DEM, and any remaining streams that didn't have outlets reaching the edge of the buffered WBD boundary were extended by snapping the end to the nearest point on the buffered WBD.
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 0fc032483..ff242a9cc 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -25,8 +25,9 @@ export                  input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_
 export                          input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
 export                   input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg
 export                 input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
-export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/24_4_3 # was 23_10_17
+export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/240502
 export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
+export                            osm_bridges=${inputsDir}/osm/bridges/240426/osm_all_bridges.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
 export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 527ae54d2..05667dd90 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -218,6 +218,7 @@ if  [ -f $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
         --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
 fi
 
+
 ## HYDRAULIC PROPERTIES ##
 echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"
 $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
@@ -227,6 +228,24 @@ $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$curr
     -h $tempCurrentBranchDataDir/stage_$current_branch_id.txt \
     -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv
 
+
+
+## HEAL HAND BRIDGES (note resolution is set to 10m)
+# May or may not be a bridge file, depends if the HUC has an applicble one.
+# Writing back over the rem_zeroed_masked branch tif
+if  [ -f $tempHucDataDir/osm_bridges_subset.gpkg ]; then
+    echo -e $startDiv"Burn in bridges $hucNumber $current_branch_id"
+    python3 $srcDir/heal_bridges_osm.py \
+                -g $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
+                -s $tempHucDataDir/osm_bridges_subset.gpkg \
+                -p $tempCurrentBranchDataDir/bridge_lines_raster_$current_branch_id.tif \
+                -t $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
+                -r 10
+else
+    -e $startDiv"No applicable bridge data for $hucNumber"
+fi
+
+
 ## FINALIZE CATCHMENTS AND MODEL STREAMS ##
 echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"
 python3 $srcDir/add_crosswalk.py \
diff --git a/src/heal_bridges_osm.py b/src/heal_bridges_osm.py
new file mode 100644
index 000000000..bd72a481d
--- /dev/null
+++ b/src/heal_bridges_osm.py
@@ -0,0 +1,146 @@
+import argparse
+import os
+
+import geopandas as gpd
+import numpy as np
+import rasterio
+from rasterio import features, transform
+from rasterstats import zonal_stats
+
+
+def process_bridges_in_huc(
+    source_hand_raster, bridge_vector_file, bridge_raster, output_bridge_lines_raster_file, resolution
+):
+    """
+    Process:
+        - Use the osm_file, likely already pre-clipped to a huc, and create a raster from it.
+        - using the incomeing source_raster, likely a rem at this point, find the max point value for intersecting from the
+          line string. Fundemtally create a z value for the line string based on the touch point of the linestring on both
+          banks. TODO: how does it handle higher elevation on one bank? Might already be in this logic.. not sure.
+        - That value will be assigned to each cell that covers the bridge in the output raster.
+        - We do create a temp raster which we can removed in the "deny" list system. We can use it to show the point values
+          assigned to each cell of the bridge.
+    """
+
+    if not os.path.exists(source_hand_raster):
+        print(f"-- no hand grid, {source_hand_raster}")
+        return
+
+    if os.path.exists(bridge_vector_file):
+        osm_gdf = gpd.read_file(bridge_vector_file)
+    else:
+        # skip this huc because it didn't pull in the initial OSM script
+        # and could have errors in the data or geometry
+        print(f"-- no OSM file, {bridge_vector_file}")
+        return
+
+    hand_grid = rasterio.open(source_hand_raster)
+
+    #######################################################################
+
+    ############# get max hand values for each bridge #########
+    # find max hand value from raster each linestring intersects
+    osm_gdf['max_hand'] = zonal_stats(
+        osm_gdf['geometry'], hand_grid.read(1), affine=hand_grid.transform, stats="max"
+    )
+    # pull the values out of the geopandas columns so we can use them as floats
+    osm_gdf['max_hand'] = [x.get('max') for x in osm_gdf.max_hand]
+    # sort in case of overlaps; display max hand value at any given location
+    osm_gdf = osm_gdf.sort_values(by="max_hand", ascending=False)
+    #######################################################
+
+    ########### setup new raster to save bridge max hand values #############
+    bbox = hand_grid.bounds
+    xmin, ymin, xmax, ymax = bbox
+    w = (xmax - xmin) // resolution
+    h = (ymax - ymin) // resolution
+
+    out_meta = {
+        "driver": "GTiff",
+        "dtype": "float32",
+        "height": h,
+        "width": w,
+        "count": 1,
+        "crs": hand_grid.crs,
+        "nodata": -999999,
+        "transform": transform.from_bounds(xmin, ymin, xmax, ymax, w, h),
+        "compress": 'lzw',
+    }
+
+    ################# rasterize the incoming bridge vectors ####################
+    with rasterio.open(bridge_raster, 'w+', **out_meta) as out:
+        out_arr = out.read(1)
+        # this is where we create a generator of geom, value pairs to use in rasterizing
+        shapes = ((geom, value) for geom, value in zip(osm_gdf.geometry, osm_gdf.max_hand))
+        # burn in values to any pixel that's touched by polygon and add nodata fill value
+        burned = features.rasterize(
+            shapes=shapes, fill=-999999, out=out_arr, transform=out.transform, all_touched=True
+        )
+        out.write_band(1, burned)
+
+    #################### heal / update hand grid ##########################
+    with rasterio.open(bridge_raster) as in_data:
+        new_hand_values = in_data.read(1)
+
+    hand_grid_vals = hand_grid.read(1)
+    # replace values at all locations where there are healed values available
+    combined_hand_values = np.where(new_hand_values == -999999, hand_grid_vals, new_hand_values)
+
+    with rasterio.open(output_bridge_lines_raster_file, 'w+', **out_meta) as out:
+        out.write(combined_hand_values, 1)
+
+    return
+
+
+if __name__ == "__main__":
+    '''
+    Sample usage (min params):
+        python3 src/heal_bridges_osm.py
+            -u 12090301
+            -g /outputs/fim_4_4_15_0/1209301/branches/3763000013/rem_zeroed_masked_3763000013.tif
+            -s /outputs/fim_4_4_15_0/1209301/osm_bridges_subset.gpkg
+            -p /outputs/fim_4_4_15_0/1209301/1209301/branches/3763000013/bridge_lines_raster_3763000013.tif
+            -t /outputs/fim_4_4_15_0/1209301/branches/3763000013/rem_zeroed_masked_3763000013.tif
+            -r 10
+
+    '''
+
+    parser = argparse.ArgumentParser(description='Rasterizes max HAND values under OSM lines and heals HAND')
+
+    parser.add_argument(
+        '-g',
+        '--source_hand_raster',
+        help='REQUIRED: Path and file name of source output raster',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-s', '--bridge_vector_file', help='REQUIRED: A gpkg that contains the bridges vectors', required=True
+    )
+
+    parser.add_argument(
+        '-p',
+        '--bridge_raster',
+        help='REQUIRED: Path and file name of the temp raster created by the bridge vectors',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-t',
+        '--output_bridge_lines_raster_file',
+        help='REQUIRED: Path and file name of the new raster with the bridge burned in',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-r',
+        '--resolution',
+        help='OPTIONAL: Resolution of HAND grid. Default value is 10m',
+        required=False,
+        default=10,
+        type=int,
+    )
+
+    args = vars(parser.parse_args())
+
+    process_bridges_in_huc(**args)
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index 45e078b12..b30f515e3 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -73,37 +73,6 @@ def run_system_command(args):
     os.system(command)
 
 
-def subset_wbd_gpkg(wbd_gpkg, multilayer_wbd_geopackage):
-    print("Subsetting " + wbd_gpkg + "...")
-    # Read geopackage into dataframe.
-    wbd = gp.read_file(wbd_gpkg)
-    gdf = gp.GeoDataFrame(wbd)
-
-    for index, row in gdf.iterrows():
-        state = row["STATES"]
-        if state is not None:  # Some polygons are empty in the STATES field.
-            keep_flag = False  # Default to Fault, i.e. to delete the polygon.
-            if state in sv.CONUS_STATE_LIST:
-                keep_flag = True
-            # Only split if multiple states present. More efficient this way.
-            elif len(state) > 2:
-                for wbd_state in state.split(
-                    ","
-                ):  # Some polygons have multiple states, separated by a comma.
-                    if (
-                        wbd_state in sv.CONUS_STATE_LIST
-                    ):  # Check each polygon to make sure it's state abbrev name is allowed.
-                        keep_flag = True
-                        break
-            if not keep_flag:
-                gdf = gdf.drop(index)  # Delete from dataframe.
-
-    # Overwrite geopackage.
-    layer_name = os.path.split(wbd_gpkg)[1].strip('.gpkg')
-    gdf.crs = sv.PREP_PROJECTION
-    gdf.to_file(multilayer_wbd_geopackage, layer=layer_name, driver='GPKG', index=False)
-
-
 def get_fossid_from_huc8(
     huc8_id,
     foss_id_attribute='fossid',
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 99951e1d7..72a28a66b 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -99,65 +99,6 @@
 }
 
 
-# -- Other -- #
-CONUS_STATE_LIST = {
-    "AL",
-    "AZ",
-    "AR",
-    "CA",
-    "CO",
-    "CT",
-    "DC",
-    "DE",
-    "FL",
-    "GA",
-    "HI",
-    "ID",
-    "IL",
-    "IN",
-    "IA",
-    "KS",
-    "KY",
-    "LA",
-    "ME",
-    "MD",
-    "MA",
-    "MI",
-    "MN",
-    "MS",
-    "MO",
-    "MT",
-    "NE",
-    "NV",
-    "NH",
-    "NJ",
-    "NM",
-    "NY",
-    "NC",
-    "ND",
-    "OH",
-    "OK",
-    "OR",
-    "PA",
-    "PR",
-    "RI",
-    "SC",
-    "SD",
-    "TN",
-    "TX",
-    "UT",
-    "VT",
-    "VA",
-    "WA",
-    "WV",
-    "WI",
-    "WY",
-}
-
-OVERWRITE_WBD = 'OVERWRITE_WBD'
-OVERWRITE_NHD = 'OVERWRITE_NHD'
-OVERWRITE_ALL = 'OVERWRITE_ALL'
-
 # Rating Curve Adjustment (local calibration) variables
 USGS_CALB_TRACE_DIST = 8.0  # distance in km to walk the network and perform USGS rating calibation for SRCs
 DOWNSTREAM_THRESHOLD = 8.0  # distance in km to propogate new roughness values downstream (group mean)

From 0170870e0e8cae1ca28f4ff1afe6eb1e551f3851 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Mon, 6 May 2024 21:00:09 +0000
Subject: [PATCH 316/359] Add pre-commit and deploy notes (#1144)

---
 .github/PULL_REQUEST_TEMPLATE.md | 10 +++++++++
 CONTRIBUTING.md                  | 35 +++++++++++++++++++++++---------
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 4cb221383..f9265baef 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -8,6 +8,16 @@
 
 ### Testing
 
+### Deployment Plan (For developer use)
+
+_How does the changes affect the product?_
+- [ ] Code only?
+- [ ] Require new or adjusted data inputs. Does it have start, end and duration code (in UTC)?
+- [ ] If new or updated data updates, has FIM related code been updated and tested?
+- [ ] Require new pre-clip set?
+- [ ] Has new or updated python packages?
+- [ ] If applicable, is the deployment plan be calculated and work with Deployment person/team?
+
 
 ### Issuer Checklist (For developer use)
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a66e1971f..3e114abed 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -45,7 +45,13 @@ If you would like to contribute, please follow these steps:
     ```
     git checkout -b <dev-your-bugfix-or-feature>
     ```
-4. Pre-commit installation:
+
+4. Build the Docker container (if not already done in the [README.md](./README.md) page.):
+    ```
+    Docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>
+    ```
+
+5. Pre-commit installation:
 
    [pre-commit](https://pre-commit.com/) is used to run auto-formatting and enforce styling rules.
    It is a critical part of development and is enforced at the 'git commit' step. Key tools are included **inside the docker container** if you want to execute the correctly configured linting and formatting command line executables there. If you intend to execute `flake8`, `black` or `isort` from the command line **outside of the docker container**, additional configuration and installation is required, which will not be described here.
@@ -75,7 +81,7 @@ If you would like to contribute, please follow these steps:
    It should respond with the phrase *pre-commit 3.6.0* (version may not be exact).
 
 
-5. pre-commit configuration:
+6. pre-commit configuration:
    
    Now, you need to configure your local clone of the repo to honor the pre-commit hooks.
    The `pre-commit` package is used to pick up the pre-commit hooks which verify your staged changes adhere to the project's style and format requirements (configuration defined in [pyproject.toml](/pyproject.toml)).
@@ -85,24 +91,33 @@ If you would like to contribute, please follow these steps:
     $ pre-commit install
     ```
     
-6. At this point, you should be set up with `pre-commit`. When a commit is made it will run the pre-commit hooks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). For reference, you may run any of the pre-commit hooks manually before issuing the `git commit` command (see below). Some tools used by the pre commit git hook (`isort`, `flake8`, & `black`) are also available as command line executables **within the Docker container***, however, it is recommended to run them through `pre-commit` **outside of the container**, as it picks up the correct configuration.
+7. At this point, you should be set up with `pre-commit`. When a commit is made it will run the pre-commit hooks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). For reference, you may run any of the pre-commit hooks manually before issuing the `git commit` command (see below). Some tools used by the pre commit git hook (`isort`, `flake8`, & `black`) are also available as command line executables **within the Docker container***, however, it is recommended to run them through `pre-commit` **outside of the container**, as it picks up the correct configuration.
 
    ```
+   There are multiple ways to run pre-commit tests. Here are three:
+   
    # Check only the staged changes
    pre-commit run
 
    # Check all files in the repo
    pre-commit run -a
 
-   # Run only the flake8, isort, or black.
-   pre-commit run -a flake8
+   # Run only the isort, black, flake8 (in order).
    pre-commit run -a isort
-   pre-commit run -a black
+   pre-commit run -a black  (If it fails the first time, run it again and now apply a fix if required)
+   pre-commit run -a flake8
    ```
-7. Build the Docker container:
-    ```
-    Docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>
-    ```
+    **Note: The pre-commit system can be a bit tricky. If you have trouble with pre-commit steps above, you have another option.** Some environments may require add the word **sudo** in front commands .
+   ```
+   # Open up a docker container
+   cd /foss_fim
+   isort .
+   black .
+   flake8 .
+
+   # optionally close the container
+   # Back on your terminal console (outside the container), use the typical git add, git commit, git push
+
  
 8.  [Within the container](README.md#startrun-the-docker-container), ensure sure unit tests pass ([instructions here](/unit_tests/README.md)).
     ```

From 45a04303f2bf0098915a6bd05e34acb9f94c4f4b Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Thu, 9 May 2024 10:46:25 -0600
Subject: [PATCH 317/359] v4.5.0.1 Bug fixes for v4.5.0.0 (#1150)

---
 .github/PULL_REQUEST_TEMPLATE.md         |  6 +++---
 docs/CHANGELOG.md                        | 17 ++++++++++++++++
 src/add_crosswalk.py                     | 26 +++++++++++++++++++++---
 src/bash_variables.env                   |  5 ++++-
 src/delineate_hydros_and_produce_HAND.sh |  4 ++--
 src/heal_bridges_osm.py                  |  8 ++++----
 src/run_unit_wb.sh                       |  5 ++---
 7 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index f9265baef..3bcbdf0fa 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -12,11 +12,11 @@
 
 _How does the changes affect the product?_
 - [ ] Code only?
-- [ ] Require new or adjusted data inputs. Does it have start, end and duration code (in UTC)?
-- [ ] If new or updated data updates, has FIM related code been updated and tested?
+- [ ] Require new or adjusted data inputs? Does it have start, end and duration code (in UTC)?
+- [ ] If new or updated data sets, has the FIM code been updated and tested with the new/adjusted data (subset is fine, but must be a subset of the new data)?
 - [ ] Require new pre-clip set?
 - [ ] Has new or updated python packages?
-- [ ] If applicable, is the deployment plan be calculated and work with Deployment person/team?
+- [ ] If applicable, has a deployment plan be created with the deployment person/team?
 
 
 ### Issuer Checklist (For developer use)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 754885541..c904fef02 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.0.1 - 2024-05-09 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150)
+
+Fixes two bugs discovered in v4.5.0.0:
+1. `echo` missing in bash command
+2. raster resolution of `dem_meters.tif` has now been explicitly set in `gdalwarp`.
+
+### Changes
+
+- `src/`
+    - `add_crosswalk.py`: fixed stream order if max > `max_order`
+    - `bash_variables.env`: added `res` environment variable for default raster cell size
+    - `delineate_hydros_and_produce_HAND.sh`: added missing `echo`
+    - `heal_bridges_osm.py`: fixed raster resolution and number of rows/columns
+    - `run_unit_wb.sh`: added `-tr` to gdalwarp when generating `dem_meters.tif`; removed extraneous `Tcount`
+
+<br/><br/>
+
 ## v4.5.0.0 - 2024-05-06 - [PR#1122](https://github.com/NOAA-OWP/inundation-mapping/pull/1122)
 
 This PR includes 2 scripts to add Open Street Map bridge data into the HAND process: a script that pulls data from OSM and a script that heals those bridges in the HAND grids. Both scripts should be run as part of a pre-processing step for FIM runs. They only need to be run if we think OSM data has changed a lot or for any new FIM versions.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 2331e4c65..ddc9444df 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -244,12 +244,32 @@ def add_crosswalk(
                         (output_flows.From_Node == to_node) & (output_flows['order_'] == max_order)
                     ]['HydroID'].item()
 
+                # output_flows has a higher order than the max_order
+                elif output_flows.loc[(output_flows.From_Node == to_node), 'order_'].max() > max_order:
+                    update_id = output_flows.loc[
+                        (output_flows.From_Node == to_node)
+                        & (
+                            output_flows['order_']
+                            == output_flows.loc[(output_flows.From_Node == to_node), 'order_'].max()
+                        )
+                    ]['HydroID'].values[0]
+
                 # Get the first one
                 # Same stream order, without drainage area info it is hard to know which is the main channel.
                 else:
-                    update_id = output_flows.loc[
-                        (output_flows.From_Node == to_node) & (output_flows['order_'] == max_order)
-                    ]['HydroID'].values[0]
+                    if max_order in output_flows.loc[output_flows.From_Node == to_node, 'order_'].values:
+                        update_id = output_flows.loc[
+                            (output_flows.From_Node == to_node) & (output_flows['order_'] == max_order)
+                        ]['HydroID'].values[0]
+
+                    else:
+                        update_id = output_flows.loc[
+                            (output_flows.From_Node == to_node)
+                            & (
+                                output_flows['order_']
+                                == output_flows.loc[output_flows.From_Node == to_node, 'order_'].max()
+                            )
+                        ]['HydroID'].values[0]
 
             # no upstream segments; single downstream segment
             elif len(output_flows.loc[output_flows.From_Node == to_node]['HydroID']) == 1:
diff --git a/src/bash_variables.env b/src/bash_variables.env
index ff242a9cc..42513fad8 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -1,8 +1,11 @@
 ## Define inputs
 
-# NOTE: $inputsDir is defined in Dockerfile
+export res=10 # Default raster cell size in meters
+
 export             DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
 export                             ALASKA_CRS=EPSG:3338
+
+# NOTE: $inputsDir is defined in Dockerfile
 export                              input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
 export                       input_DEM_Alaska=${inputsDir}/3dep_dems/10m_South_Alaska/23_11_07/FIM_3dep_dem_South_Alask_10m.vrt
 export                       input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 05667dd90..379761f1a 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -240,9 +240,9 @@ if  [ -f $tempHucDataDir/osm_bridges_subset.gpkg ]; then
                 -s $tempHucDataDir/osm_bridges_subset.gpkg \
                 -p $tempCurrentBranchDataDir/bridge_lines_raster_$current_branch_id.tif \
                 -t $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
-                -r 10
+                -r $res
 else
-    -e $startDiv"No applicable bridge data for $hucNumber"
+    echo -e $startDiv"No applicable bridge data for $hucNumber"
 fi
 
 
diff --git a/src/heal_bridges_osm.py b/src/heal_bridges_osm.py
index bd72a481d..288572fc5 100644
--- a/src/heal_bridges_osm.py
+++ b/src/heal_bridges_osm.py
@@ -52,8 +52,8 @@ def process_bridges_in_huc(
     ########### setup new raster to save bridge max hand values #############
     bbox = hand_grid.bounds
     xmin, ymin, xmax, ymax = bbox
-    w = (xmax - xmin) // resolution
-    h = (ymax - ymin) // resolution
+    w = hand_grid.shape[1]
+    h = hand_grid.shape[0]
 
     out_meta = {
         "driver": "GTiff",
@@ -74,7 +74,7 @@ def process_bridges_in_huc(
         shapes = ((geom, value) for geom, value in zip(osm_gdf.geometry, osm_gdf.max_hand))
         # burn in values to any pixel that's touched by polygon and add nodata fill value
         burned = features.rasterize(
-            shapes=shapes, fill=-999999, out=out_arr, transform=out.transform, all_touched=True
+            shapes=shapes, fill=-999999, out=out_arr, transform=hand_grid.transform, all_touched=True
         )
         out.write_band(1, burned)
 
@@ -138,7 +138,7 @@ def process_bridges_in_huc(
         help='OPTIONAL: Resolution of HAND grid. Default value is 10m',
         required=False,
         default=10,
-        type=int,
+        type=float,
     )
 
     args = vars(parser.parse_args())
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 03129bfa3..348b5518e 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -119,13 +119,12 @@ echo -e $startDiv"Clipping rasters to branches $hucNumber $branch_zero_id"
 [ ! -f $tempCurrentBranchDataDir/dem_meters.tif ] && \
 gdalwarp -cutline $tempHucDataDir/wbd_buffered.gpkg -crop_to_cutline -ot Float32 -r bilinear -of "GTiff" \
     -overwrite -co "BLOCKXSIZE=512" -co "BLOCKYSIZE=512" -co "TILED=YES" -co "COMPRESS=LZW" \
-    -co "BIGTIFF=YES" -t_srs $huc_CRS $input_DEM $tempHucDataDir/dem_meters.tif
+    -co "BIGTIFF=YES" -t_srs $huc_CRS -tr $res $res $input_DEM $tempHucDataDir/dem_meters.tif
 
-Tcount
 
 ## GET RASTER METADATA
 echo -e $startDiv"Get DEM Metadata $hucNumber $branch_zero_id"
-read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy\
+read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy \
     <<<$($srcDir/getRasterInfoNative.py $tempHucDataDir/dem_meters.tif)
 
 ## RASTERIZE NLD MULTILINES ##

From a488fe2a75107b9b9b3eb99e7560be069839a07f Mon Sep 17 00:00:00 2001
From: AliForghani-NOAA <124194814+AliForghani-NOAA@users.noreply.github.com>
Date: Fri, 17 May 2024 16:04:16 -0400
Subject: [PATCH 318/359] v4.5.0.2 Update nws_lid dataset with Alaska sites
 (#1159)

---
 docs/CHANGELOG.md         | 22 +++++++++++
 tools/generate_nws_lid.py | 83 +++------------------------------------
 2 files changed, 27 insertions(+), 78 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c904fef02..7ee764749 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,28 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.0.2 - 2024-05-17 - [PR#1159](https://github.com/NOAA-OWP/inundation-mapping/pull/1159)
+
+This PR addresses issue #1132 and include the following changes on `tools/generate_nws_lid.py` for updating `nws_lid.gpkg` dataset.
+
+In this revised version, stations only from these two groups are retrieved:
+- lid stations with `rfc_forecast_point= True` 
+- lid stations in `/data/inputs/ahp_sites/evaluated_ahps_sites.csv`
+
+The lid stations in AK (Alaska), HI, and PR, with above two criteria have also been selected, as shown in the map below. In the previous version of the code, **all of lid stations** in PR and HI (regardless of meeting above two criteria), were also being retrieved. I have updated this version to exclude such stations. 
+
+Also, In this revised version, I've eliminated the code sections that previously generated the "is_headwater" and "is_colocated" columns, which are not needed in FIM4. Therefore, in this updated version, these columns are no longer present. 
+
+Similar to 'usgs_gages.gpkg' dataset, all lid stations, including those in Alaska, are stored in a single gpkg file (`nws_lid.gpkg`) with EPSG=5070. The Alaska stations can be identified using their HUC8 numbers (beginning with '19'). 
+
+
+### Changes
+- tools/generate_nws_lid.py
+
+<br/><br/>
+
+
+
 ## v4.5.0.1 - 2024-05-09 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150)
 
 Fixes two bugs discovered in v4.5.0.0:
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
index ac769134b..b91e0fe94 100644
--- a/tools/generate_nws_lid.py
+++ b/tools/generate_nws_lid.py
@@ -22,14 +22,11 @@
 API_BASE_URL = os.getenv("API_BASE_URL")
 EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
 WBD_LAYER = os.getenv("WBD_LAYER")
-# Define path to NWM stream layer
-NWM_FILE = '/data/inputs/nwm_hydrofabric/nwm_flows.gpkg'
 
 
 def generate_nws_lid(workspace):
     '''
-    Generate the nws_lid layer containing all nws_lid points attributed
-    whether site is mainstems and co-located
+    Generate the nws_lid layer containing all nws_lid points
 
     Parameters
     ----------
@@ -79,22 +76,8 @@ def generate_nws_lid(workspace):
         downstream_trace_distance=downstream_trace_distance,
     )
 
-    # Trace downstream from all sites in HI/PR.
-    select_by = 'state'
-    selector = ['HI', 'PR']
-    must_include = None
-    downstream_trace_distance = 'all'
-    islands_list, islands_dataframe = get_metadata(
-        metadata_url=metadata_url,
-        select_by=select_by,
-        selector=selector,
-        must_include=must_include,
-        upstream_trace_distance=None,
-        downstream_trace_distance=downstream_trace_distance,
-    )
-
     # Append all lists
-    all_lists = fcst_list + eval_list + islands_list
+    all_lists = fcst_list + eval_list
 
     # Compile NWM segments from all_lists
 
@@ -129,73 +112,17 @@ def generate_nws_lid(workspace):
             # Dictionary with key of 2nd to last downstream segment and value of site nwm segment
             downstream[int(key)].append(int(value))
 
-    # Walk downstream the network and identify headwater points
-    print('Traversing network..')
-
-    # Import NWM file and create dictionary of network and create the NWM network dictionary.
-    nwm_gdf = gpd.read_file(NWM_FILE)
-    network = nwm_gdf.groupby('ID')['to'].apply(list).to_dict()
-
-    # Walk through network and find headwater points
-    all_dicts = {}
-    for tree, targets in downstream.items():
-        # All targets are assigned headwaters
-        sub_dict = {i: 'is_headwater' for i in targets}
-        # Walk downstream of each target
-        for i in targets:
-            # Check to see element is not a headwater
-            if sub_dict[i] == 'not_headwater':
-                continue
-            # Get from_node and to_node.
-            from_node = i
-            [to_node] = network[from_node]
-            # Walk downstream from target
-            while to_node > 0:
-                # Check if to_node is in targets list
-                if to_node in targets:
-                    sub_dict[to_node] = 'not_headwater'
-                # Assign downstream ID as to_node
-                [to_node] = network[to_node]
-
-        # Append status to master dictionary
-        all_dicts.update(sub_dict)
-
-    # Create dictionaries of nws_lid (key) and headwater status (value) and nws_lid (key) and
-    #   co-located with same feature_id(value)
-    final_dict = {}
-    duplicate_dict = {}
-    for key, status in all_dicts.items():
-        site_list = target[key]
-        for site in site_list:
-            final_dict[site] = status
-            if len(site_list) > 1:
-                duplicate_dict[site] = 'is_colocated'
-            else:
-                duplicate_dict[site] = 'not_colocated'
-
     # Get Spatial data and populate headwater/duplicate attributes
     print('Attributing nws_lid layer..')
 
     # Geodataframe from all_lists, reproject, and reset index.
-    trash, nws_lid_gdf = aggregate_wbd_hucs(all_lists, WBD_LAYER, retain_attributes=False)
+    _, nws_lid_gdf = aggregate_wbd_hucs(all_lists, WBD_LAYER, retain_attributes=False)
     nws_lid_gdf.columns = [name.replace('identifiers_', '') for name in nws_lid_gdf.columns]
     nws_lid_gdf.to_crs(PREP_PROJECTION, inplace=True)
     nws_lid_gdf.reset_index(drop=True)
-
-    # Create DataFrames of headwater and duplicates and join.
-    final_dict_pd = pd.DataFrame(list(final_dict.items()), columns=['nws_lid', 'is_headwater'])
-    duplicate_dict_pd = pd.DataFrame(list(duplicate_dict.items()), columns=['nws_lid', 'is_colocated'])
-    attributes = final_dict_pd.merge(duplicate_dict_pd, on='nws_lid')
-    attributes.replace(
-        {'is_headwater': True, 'is_colocated': True, 'not_headwater': False, 'not_colocated': False},
-        inplace=True,
-    )
-
-    # Join attributes, remove sites with no assigned nwm_feature_id and write to file
-    joined = nws_lid_gdf.merge(attributes, on='nws_lid', how='left')
-    joined.dropna(subset=['nwm_feature_id'], inplace=True)
     Path(workspace).mkdir(parents=True, exist_ok=True)
-    joined.to_file(Path(workspace) / 'nws_lid.gpkg', driver='GPKG')
+    nws_lid_gdf.dropna(subset=['nwm_feature_id'], inplace=True)
+    nws_lid_gdf.to_file(Path(workspace) / 'nws_lid.gpkg', driver='GPKG')
 
 
 if __name__ == '__main__':

From eaeb4c857b698f4439467aa75520bc178f979195 Mon Sep 17 00:00:00 2001
From: HeidiSafa-NOAA <safa.hamideh@gmail.com>
Date: Fri, 17 May 2024 20:10:46 +0000
Subject: [PATCH 319/359] v4.5.1.0 Healed hand removing hydro-conditioning
 artifacts and testing (#1156)

---
 config/params_template.env               |   4 +
 docs/CHANGELOG.md                        |  14 ++
 src/delineate_hydros_and_produce_HAND.sh |  11 ++
 tools/analyze_for_missing_FIM_cells.py   | 187 +++++++++++++++++++++++
 4 files changed, 216 insertions(+)
 create mode 100644 tools/analyze_for_missing_FIM_cells.py

diff --git a/config/params_template.env b/config/params_template.env
index ab761336c..8312bea92 100644
--- a/config/params_template.env
+++ b/config/params_template.env
@@ -31,6 +31,10 @@ export branch_zero_id="0"
 export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from DEM
 export levee_id_attribute=SYSTEM_ID
 
+#### Healed HAND ####
+# Removes Hydro-conditioning Artifacts (true=on; false=off)
+export healed_hand_hydrocondition=true
+
 #### apply bathymetry adjustment to rating curve ####
 export bathymetry_adjust=True
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7ee764749..c4b27d9fb 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.1.0 - 2024-05-17 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150)
+
+This focuses on removing hydro-conditioning artifacts by subtracting the thalweg DEM from HAND REM and adding back the original DEM. Also, a new tool was created to test this feature over multiple HUCs
+
+### Additions
+- `tools/analyze_for_missing_FIM_cells.py`: A new script `analyze_for_missing_FIM_cells.py` was added to test and analyze healed HAND for hydro-conditioning artifacts FIM. 
+
+### Changes
+- `src/delineate_hydros_and_produce_HAND.sh`: Removing hydro-conditioning artifacts from HAND REM.
+- `config/params_template.env`: Creating an option to include/exclude healed HAND from FIM pipeline.
+
+
+<br/><br/>
+
 ## v4.5.0.2 - 2024-05-17 - [PR#1159](https://github.com/NOAA-OWP/inundation-mapping/pull/1159)
 
 This PR addresses issue #1132 and include the following changes on `tools/generate_nws_lid.py` for updating `nws_lid.gpkg` dataset.
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 379761f1a..efe6e5f55 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -229,6 +229,17 @@ $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$curr
     -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv
 
 
+## HEAL HAND -- REMOVES HYDROCONDITIONING ARTIFACTS ##
+if [ "$healed_hand_hydrocondition" = true ]; then
+    echo -e $startDiv"Healed HAND to Remove Hydro-conditioning Artifacts $hucNumber $current_branch_id"
+    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
+        -R $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
+        -D $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
+        -T $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
+        --calc="R+(D-T)" --NoDataValue=$ndv \
+        --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+fi
+
 
 ## HEAL HAND BRIDGES (note resolution is set to 10m)
 # May or may not be a bridge file, depends if the HUC has an applicble one.
diff --git a/tools/analyze_for_missing_FIM_cells.py b/tools/analyze_for_missing_FIM_cells.py
new file mode 100644
index 000000000..bbb23b7cd
--- /dev/null
+++ b/tools/analyze_for_missing_FIM_cells.py
@@ -0,0 +1,187 @@
+"""
+
+This tool analyzes the missing FIM cells after removing hydro-conditioning artifacts (thalweg rem).
+It calculates the number of FIM celles in each HUC that will not be inundated after removing thalweg rem.
+It also calculates the number of catchments that never shows any inundation because of the thalwag notch
+issues, called notched streams. Then it creates a bar chart depicting the percentage of notched streams in
+across all processed HUCs grouped by stream orders.
+
+You may find more detailes in PR #1156 (https://github.com/NOAA-OWP/inundation-mapping/pull/1156)
+
+"""
+
+import argparse
+import os
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import rasterio
+
+
+# from rasterio import plot as rioplot
+
+
+# This will make jupyter display all columns of the hydroTable
+pd.options.display.max_columns = None
+
+
+# ------------------------------------------------------
+def find_missing_fim_cells_for1huc(fim_dir, huc8_num):
+
+    fim_dir = Path(fim_dir)
+    branch0_dir = Path(fim_dir, huc8_num, "branches", "0")
+    hydroTable_csv = Path(branch0_dir, "hydroTable_0.csv")
+    hydroTable = pd.read_csv(hydroTable_csv)
+
+    rem_tif = Path(branch0_dir, "rem_zeroed_masked_0.tif")
+    catchments_tif = Path(branch0_dir, "gw_catchments_reaches_filtered_addedAttributes_0.tif")
+
+    with rasterio.open(catchments_tif) as catchments:
+        catchments = catchments.read(1)
+
+    stream_orders = list(hydroTable["order_"].drop_duplicates(keep='first'))
+    stream_orders = sorted(stream_orders)
+
+    analysis_data_ordr = []
+    for ordr in stream_orders:
+
+        branch0_streams = hydroTable[(hydroTable["order_"] == ordr)]
+        num_streams_order = len(branch0_streams)
+        branch0_hydroids = list(branch0_streams['HydroID'].drop_duplicates(keep='first'))
+
+        with rasterio.open(rem_tif) as rem:
+            rem = rem.read(1)
+
+        # Filter the REM to the catchments of interest
+        rem = np.where(~np.isin(catchments, branch0_hydroids), np.nan, rem)
+
+        # Filter the REM again to only the 0 values
+        rem_zeros = rem.copy()
+        rem_zeros[np.where(rem_zeros != 0)] = np.nan
+
+        cells_remaining = np.count_nonzero(~np.isnan(rem_zeros))
+        print("")
+        print(f"Analyzing Branch0, {ordr}-order streams FIM in HUC8 {huc8_num}")
+        print("")
+        print(f"Actual count of cells remaining: {cells_remaining}")
+
+        non_zero_count = (catchments != 0).sum()
+        percentage_b0_rem0 = round(100 * (cells_remaining / non_zero_count), 4)
+        print("")
+        print(
+            f"Actual percentage of branch0, {ordr}-order stream cells that have not been inundated: {percentage_b0_rem0}%"
+        )
+
+        # Finding branch0 hydroIDs that do not have zero rem (never inundate, notches)
+        target_hydroids = []
+        for hydroid in branch0_hydroids:
+
+            rem_hydroid = rem.copy()
+            hydroid_ls = [hydroid]
+            rem_hydroid = np.where(~np.isin(catchments, hydroid_ls), np.nan, rem_hydroid)
+
+            rem_hydroid_nonnan = rem_hydroid[~np.isnan(rem_hydroid)]
+            cond = min(rem_hydroid_nonnan)
+
+            if cond > 0:
+                target_hydroids.append(hydroid)
+
+        print("")
+        print(
+            f"{len(target_hydroids)} streams between {ordr}-order streams are thalwag notch, including hydroIDs {target_hydroids}"
+        )
+
+        analysis = [ordr, target_hydroids, num_streams_order, cells_remaining, percentage_b0_rem0]
+
+        analysis_data_ordr.append(analysis)
+
+    return analysis_data_ordr
+
+
+# ------------------------------------------------------
+def analysis_missing_fim_cells(huc8_dir):
+
+    list_subdir = os.listdir(huc8_dir)
+    dir_path_ls = [os.path.join(huc8_dir, subdir) for subdir in list_subdir]
+
+    missing_fim_data = []
+    for path1 in dir_path_ls:
+
+        list_subdir = os.listdir(path1)
+        huc8 = [stg for stg in list_subdir if stg.isdigit()][0]
+        missing_fim = find_missing_fim_cells_for1huc(path1, huc8)
+
+        missing_fim_data.append([huc8, missing_fim])
+
+    # Number of thalwag_notch_streams grouped by stream orders
+    thalwag_notch_ord = []
+    for ord1 in range(6):
+        thalwag_notch = []
+        for subls in missing_fim_data:
+            if len(subls[1]) >= (ord1 + 1):
+                if subls[1][ord1][0] == ord1 + 1:
+                    thalwag_notch.append(len(subls[1][ord1][1]))
+        thalwag_notch_ord.append(sum(thalwag_notch))
+
+    # Number of streams grouped by stream orders
+    streams_ord = []
+    for ord1 in range(6):
+        streams = []
+        for subls in missing_fim_data:
+            if len(subls[1]) >= (ord1 + 1):
+                if subls[1][ord1][0] == ord1 + 1:
+                    streams.append(subls[1][ord1][2])
+        streams_ord.append(sum(streams))
+
+    return thalwag_notch_ord, streams_ord
+
+
+if __name__ == "__main__":
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(description="Analysis for missing FIM cells.")
+    parser.add_argument(
+        "-r",
+        dest="huc8s_dir",
+        help="Path to directory storing FIM outputs. Type = string",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        "-o", dest="path2savefig", help="Path to save bar chart. Type = string", required=True, type=str
+    )
+
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+
+    huc8s_dir = args["huc8s_dir"]
+    path2savefig = args["path2savefig"]
+
+    if not os.path.exists(huc8s_dir):
+        print("FIM directory: " + huc8s_dir + " does not exist.")
+        quit
+
+    if not os.path.exists(path2savefig):
+        os.mkdir(path2savefig)
+
+    thalwag_notch_ord, streams_ord = analysis_missing_fim_cells(huc8s_dir)
+
+    stream_orders = ["1st order", "2nd order", "3rd order", "4th order", "5th order", "6th order"]
+    percentage_notches = [
+        round(100 * (thalwag_notch_ord[i] / streams_ord[i]), 4) for i in range(len(streams_ord))
+    ]
+
+    path2savefig1 = os.path.join(path2savefig, "Thalwag_Notch_Streams_perc.png")
+
+    # creating the bar plot
+    fig = plt.figure(figsize=(20, 10))
+    plt.bar(stream_orders, percentage_notches)
+
+    plt.xlabel("Stream Orders", fontsize=20)
+    plt.xticks(fontsize=20)
+    plt.ylabel("Percentage of Thalwag Notch Streams (%)", fontsize=20)
+    plt.yticks(fontsize=20)
+    plt.title("Percentage of catchments that will not been iundated", fontsize=24)
+    plt.savefig(path2savefig1)

From ed703a284c61f79dcfedea9b8b5df3456e3da495 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 17 May 2024 20:14:41 +0000
Subject: [PATCH 320/359] v4.5.1.1 Update benchmark flows in AHPS domain
 (#1094)

---
 data/update_benchmark_flows.py  | 200 ++++++++++++++++++++++++++++++++
 docs/CHANGELOG.md               |  14 +++
 tools/tools_shared_variables.py |  14 +--
 3 files changed, 215 insertions(+), 13 deletions(-)
 create mode 100755 data/update_benchmark_flows.py

diff --git a/data/update_benchmark_flows.py b/data/update_benchmark_flows.py
new file mode 100755
index 000000000..b29b0f957
--- /dev/null
+++ b/data/update_benchmark_flows.py
@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import shutil
+
+import geopandas as gpd
+import pandas as pd
+
+
+gpd.options.io_engine = "pyogrio"
+
+
+def update_benchmark_flows(fim_dir: str, output_dir_base: str, verbose: bool = False):
+    """
+    Update benchmark flows of the levelpath in the domain for stream segments missing from the flow file
+
+    Parameters
+    ----------
+    fim_dir : str
+        Location of FIM files (e.g., "/outputs/dev-4.4.11.0")
+    output_dir_base : str
+        Output directory (e.g., "/outputs/temp"). If None, the output files will be saved in the input directory (the original files will be saved with a ".bak" extension appended to the original filename).
+    """
+
+    def iterate_over_sites(
+        levelpaths: gpd.GeoDataFrame, levelpaths_domain: gpd.GeoDataFrame, flows: pd.DataFrame
+    ) -> pd.DataFrame:
+        """
+        Update benchmark flows of stream segments missing from the flow file
+
+        Parameters
+        ----------
+        levelpaths : gpd.GeoDataFrame
+            Level paths of the HUC
+        levelpaths_domain : gpd.GeoDataFrame
+            Levelpaths in the domain
+        flows : pd.DataFrame
+            Benchmark flows
+
+        Returns
+        -------
+        pd.DataFrame
+            Flows with updated benchmark flows
+        """
+
+        if levelpaths_domain.empty:
+            return flows
+
+        # Find the levelpaths with the highest order (to exclude tributaries)
+        levelpaths_domain = levelpaths_domain[
+            levelpaths_domain['order_'] == levelpaths_domain['order_'].max()
+        ]
+
+        # Find the levelpath that has flows in the flow file
+        if levelpaths_domain['levpa_id'].nunique() > 1:
+            # If there are multiple levelpaths with the highest order, take the longest one (intersect levelpaths_domain with the domain to get the length of the levelpath)
+            levelpaths_domain_intersect = gpd.overlay(levelpaths, domain, how='intersection')
+            levelpaths_domain_intersect['length'] = levelpaths_domain_intersect['geometry'].length
+
+            # Get the total length of all the segments in the levelpath
+            levelpaths_domain_intersect_length = (
+                levelpaths_domain_intersect.groupby('levpa_id').agg({'length': 'sum'}).reset_index()
+            )
+
+            # Get the longest levelpath
+            levelpath = levelpaths_domain_intersect_length.loc[
+                levelpaths_domain_intersect_length['length'].idxmax(), 'levpa_id'
+            ]
+
+        else:
+            levelpath = levelpaths_domain['levpa_id'].iloc[0]
+
+        # Get IDs of all features in the levelpath
+        IDs_to_keep = levelpaths[levelpaths['levpa_id'] == levelpath]['ID']
+
+        # Get IDs of all features in the levelpath in the domain
+        IDs = levelpaths_domain.loc[levelpaths_domain['levpa_id'] == levelpath, 'ID']
+
+        # Keep the flows that are in the levelpath (remove tributaries)
+        flows = flows.iloc[flows.loc[flows['feature_id'].isin(IDs_to_keep)].index]
+
+        if flows.empty:
+            return flows
+
+        # Find IDs not in the flow file
+        add_IDs = ~IDs.isin(flows['feature_id'])
+
+        # Exit if no IDs to add
+        if not any(add_IDs):
+            return flows
+
+        IDs_to_add = levelpaths_domain.loc[add_IDs.index, 'ID']
+
+        # Add the missing IDs with flows to the flow file
+        if not IDs_to_add.empty:
+            flows_out = pd.concat(
+                [flows, pd.DataFrame({'feature_id': IDs_to_add, 'discharge': flows['discharge'].iloc[0]})],
+                ignore_index=True,
+            )
+        else:
+            flows_out = flows
+
+        return flows_out
+
+    for org in ['nws', 'usgs']:
+        if verbose:
+            print('Processing', org)
+
+        count_total = 0
+        count_updated = 0
+
+        base_dir = f'/data/test_cases/{org}_test_cases/validation_data_{org}'
+
+        huc8s = next(os.walk(base_dir))[1]
+
+        for huc8 in huc8s:
+            if verbose:
+                print(f'\t{huc8}')
+
+            lids = next(os.walk(f'{base_dir}/{huc8}'))[1]
+
+            for lid in lids:
+                if verbose:
+                    print(f'\t\t{lid}')
+
+                # Read the input files
+                levelpath_file = f'{fim_dir}/{huc8}/nwm_subset_streams_levelPaths.gpkg'
+                if not os.path.exists(levelpath_file):
+                    continue
+
+                levelpaths = gpd.read_file(levelpath_file)
+
+                validation_path = f'{base_dir}/{huc8}/{lid}'
+
+                domain_file = f'{validation_path}/{lid}_domain.shp'
+                if not os.path.exists(domain_file):
+                    continue
+
+                domain = gpd.read_file(domain_file)
+
+                # Intersect levelpaths with the domain to get feature_ids
+                levelpaths_domain = gpd.sjoin(levelpaths, domain, how="inner", predicate="intersects")
+
+                magnitudes = next(os.walk(validation_path))[1]
+
+                for magnitude in magnitudes:
+                    input_dir = f'{validation_path}/{magnitude}'
+
+                    if output_dir_base is None:
+                        output_dir = input_dir
+                    else:
+                        output_dir = os.path.join(output_dir_base, org)
+
+                        if not os.path.exists(output_dir):
+                            os.makedirs(output_dir)
+
+                    flow_file_in = f'{input_dir}/ahps_{lid}_huc_{huc8}_flows_{magnitude}.csv'
+                    flow_file_out = f'{output_dir}/ahps_{lid}_huc_{huc8}_flows_{magnitude}.csv'
+
+                    # Skip if flow file doesn't exist
+                    if not os.path.exists(flow_file_in):
+                        continue
+
+                    # Backup original flow file
+                    backup_flow_file = flow_file_out + '.bak'
+                    if not os.path.exists(backup_flow_file):
+                        shutil.copy2(flow_file_in, backup_flow_file)
+
+                    flows = pd.read_csv(backup_flow_file)
+
+                    flows_new = iterate_over_sites(levelpaths, levelpaths_domain, flows)
+
+                    if not flows_new.equals(flows):
+                        count_updated += 1
+                        count_total += 1
+                    else:
+                        count_total += 1
+
+                    flows_new.to_csv(flow_file_out, index=False)
+
+        print(f'\tUpdated {count_updated} out of {count_total} flow files for {org}')
+
+
+if __name__ == "__main__":
+    example_text = '''example:
+
+  %(prog)s -f /outputs/dev-4.4.11.0 -o /outputs/temp
+  \n'''
+
+    parser = argparse.ArgumentParser(
+        epilog=example_text, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument('-f', '--fim_dir', help='Location of FIM files', type=str, required=True)
+    parser.add_argument('-o', '--output_dir-base', help='Output directory', type=str, default=None)
+    parser.add_argument('-v', '--verbose', help='Verbose output', action='store_true')
+
+    args = vars(parser.parse_args())
+
+    update_benchmark_flows(**args)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c4b27d9fb..4f66a6088 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.1.1 - 2024-05-17 - [PR#1094](https://github.com/NOAA-OWP/inundation-mapping/pull/1094)
+
+Extends flows (i.e., discharge) to stream segments missing from NWS and USGS validation flow files. The levelpath associated with existing flows in the AHPS domain is identified, and any stream segments of the levelpath in the domain missing from the flow file are added to the flow file by assigning the existing flow (this is a constant value regardless of other tributaries including other levelpaths in the domain). Stream segments not on the levelpath are dropped from the flow file, including tributary flows. The original flow file is saved along with the output with an appended `.bak`.
+
+### Additions
+
+- `data/extend_benchmark_flows.py`: Adds missing flows to NWS or USGS benchmark flow files and removes flows from tributaries. The original flow file is saved with an appended `.bak`.
+
+### Changes
+
+- `tools/tools_shared_variables.py`: Removed corrected flow files from `BAD_SITES` list.
+
+<br/><br/>
+
 ## v4.5.1.0 - 2024-05-17 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150)
 
 This focuses on removing hydro-conditioning artifacts by subtracting the thalweg DEM from HAND REM and adding back the original DEM. Also, a new tool was created to test this feature over multiple HUCs
diff --git a/tools/tools_shared_variables.py b/tools/tools_shared_variables.py
index fac649f70..47b625a8b 100755
--- a/tools/tools_shared_variables.py
+++ b/tools/tools_shared_variables.py
@@ -42,34 +42,22 @@
 # Variables for eval_plots.py
 BAD_SITES = [
     "baki3",  # USGS: ratio of evaluated vs domain is very low
-    "cpei3",  # USGS: mainstems does not extend sufficiently upstream (~35%), significant masking upstream
+    "cart2",  # NWS:  added by request
     "eagi1",  # NWS:  ratio of evaluated vs domain is very low
     "efdn7",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
     "erwn6",  # NWS:  ratio of evaluated vs domain is very low
     "grfi2",  # USGS: incorrect location
-    "hohn4",  # Both: no mainstems in vicinity
     "kcdm7",  # USGS: incorrect location
-    "kilo1",  # Both: mainstems does not extend sufficiently upstream (~20%)
     "ksdm7",  # USGS: masked
     "levk1",  # NWS:  Incorrect feature_id assigned from WRDS, this has been corrected
     "lkcm7",  # NWS:  masked
     "loun7",  # NWS:  benchmark is not consistent between thresholds
     "lrlm7",  # NWS:  masked
     "mcri2",  # USGS: incorrect location
-    "monv1",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
     "mtao1",  # USGS: ratio of evaluated vs domain is very low
-    "nhso1",  # USGS: mainstems does not extend sufficiently upstream (45%)
-    "nmso1",  # Both: mainstems does not extend sufficiently upstream
-    "pori3",  # USGS: mainstems does not extend sufficiently upstream
-    "ptvn6",  # Both: mainstems does not extend sufficiently upstream (50%)
     "roun6",  # USGS: ratio of evaluated vs domain is very low
     "rwdn4",  # Both: no mainstems in vicinity
-    "selt2",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
-    "sweg1",  # Both: mainstems does not extend sufficiently upstream (~30%)
     "vcni3",  # USGS: ratio of evaluated vs domain is very low
-    "watw3",  # NWS:  mainstems does not extend sufficiently upstream (~30%)
-    "weat2",  # NWS:  mainstems does not extend sufficiently upstream (~50%)
-    "wkew3",  # NWS:  mainstems does not extend sufficiently upstream (~45%)
 ]
 DISCARD_AHPS_QUERY = "not flow.isnull() & masked_perc<97 & not nws_lid in @BAD_SITES"
 

From b1a145f2ff7c532cb30553adb69d83928636a430 Mon Sep 17 00:00:00 2001
From: EmilyDeardorff-NOAA <60829052+EmilyDeardorff@users.noreply.github.com>
Date: Fri, 17 May 2024 13:18:31 -0700
Subject: [PATCH 321/359] v4.5.1.2 Add CRS handling to USGS gage processing
 (#1135)

---
 docs/CHANGELOG.md           | 12 ++++++++++++
 src/run_by_branch.sh        |  3 ++-
 src/run_unit_wb.sh          |  7 +++++--
 src/usgs_gage_crosswalk.py  | 28 ++++++++++++++++++++++++----
 src/usgs_gage_unit_setup.py | 22 +++++++++++++++++-----
 5 files changed, 60 insertions(+), 12 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4f66a6088..7c9a2d721 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.1.2 - 2024-05-17 - [PR#1135](https://github.com/NOAA-OWP/inundation-mapping/pull/1135)
+
+Updates USGS gage processing to use the correct projection (determined by whether the HUC is in Alaska or not).
+
+### Changes
+- `src/run_by_branch.sh`: Added `huc_CRS` as an input argument for `usgs_gage_crosswalk.py`
+- `src/run_unit_wb.sh`: Added `huc_CRS` as an input argument for `usgs_gage_unit_setup.py` and `usgs_gage_crosswalk.py`
+- `src/usgs_gage_crosswalk.py`: Added `huc_CRS` as an input argument for the `run_crosswalk()` function and added re-projection steps wherever new data is being read in so that the files are able to be properly merged.
+- `src/usgs_gage_unit_setup.py`: Added `huc_CRS` as an input argument for the `Gage2Branch()` crosswalking class.
+
+<br/><br/>
+
 ## v4.5.1.1 - 2024-05-17 - [PR#1094](https://github.com/NOAA-OWP/inundation-mapping/pull/1094)
 
 Extends flows (i.e., discharge) to stream segments missing from NWS and USGS validation flow files. The levelpath associated with existing flows in the AHPS domain is identified, and any stream segments of the levelpath in the domain missing from the flow file are added to the flow file by assigning the existing flow (this is a constant value regardless of other tributaries including other levelpaths in the domain). Stream segments not on the levelpath are dropped from the flow file, including tributary flows. The original flow file is saved along with the output with an appended `.bak`.
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index f052571b3..69151a74c 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -116,7 +116,8 @@ if [ -f $tempHucDataDir/usgs_subset_gages.gpkg ]; then
         -dem $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
         -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
         -out $tempCurrentBranchDataDir \
-        -b $current_branch_id
+        -b $current_branch_id \
+        -huc_CRS $huc_CRS
 fi
 
 ## REMOVE FILES FROM DENY LIST ##
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 348b5518e..3c7a1b460 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -250,9 +250,11 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
         -o $tempHucDataDir/usgs_subset_gages.gpkg \
         -huc $hucNumber \
         -ahps $tempHucDataDir/nws_lid.gpkg \
-        -bzero_id $branch_zero_id
+        -bzero_id $branch_zero_id \
+        -huc_CRS $huc_CRS
 fi
 
+
 ## USGS CROSSWALK ##
 if [ -f $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
     echo -e $startDiv"USGS Crosswalk $hucNumber $branch_zero_id"
@@ -262,7 +264,8 @@ if [ -f $tempHucDataDir/usgs_subset_gages_$branch_zero_id.gpkg ]; then
         -cat $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$branch_zero_id.gpkg \
         -dem $tempCurrentBranchDataDir/dem_meters_$branch_zero_id.tif \
         -dem_adj $tempCurrentBranchDataDir/dem_thalwegCond_$branch_zero_id.tif \
-        -out $tempCurrentBranchDataDir -b $branch_zero_id
+        -out $tempCurrentBranchDataDir -b $branch_zero_id \
+        -huc_CRS $huc_CRS
 fi
 
 ## CLEANUP BRANCH ZERO OUTPUTS ##
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 4a477692c..98263c602 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -33,6 +33,8 @@
         Directory to create output table. i.e. '/data/path/'
     branch_id: str
         ID of the current branch i.e. '3246000257'
+        huc_CRS: str
+    Projection to be used for the HUC
 '''
 
 
@@ -42,7 +44,13 @@ def __init__(self, usgs_subset_gages_filename, branch_id):
         self.gages = self._load_gages(usgs_subset_gages_filename)
 
     def run_crosswalk(
-        self, input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory
+        self,
+        input_catchment_filename,
+        input_flows_filename,
+        dem_filename,
+        dem_adj_filename,
+        output_directory,
+        huc_CRS,
     ):
         '''Run the gage crosswalk steps: 1) spatial join to branch catchments layer 2) snap sites to
         the dem-derived flows 3) sample both dems at the snapped points 4) write the crosswalked points
@@ -52,16 +60,18 @@ def run_crosswalk(
             print(f'There are no gages for branch {branch_id}')
             os._exit(0)
         # Spatial join to fim catchments
-        self.catchment_sjoin(input_catchment_filename)
+        self.catchment_sjoin(input_catchment_filename, huc_CRS)
         if self.gages.empty:
             print(f'There are no gages for branch {branch_id}')
             os._exit(0)
 
         # Snap to dem derived flow lines
         self.snap_to_dem_derived_flows(input_flows_filename)
+
         # Sample DEM and thalweg adjusted DEM
         self.sample_dem(dem_filename, 'dem_elevation')
         self.sample_dem(dem_adj_filename, 'dem_adj_elevation')
+
         # Write to csv
         num_gages = len(self.gages)
         print(f"{num_gages} gage{'' if num_gages == 1 else 's'} in branch {self.branch_id}")
@@ -71,12 +81,15 @@ def _load_gages(self, gages_filename):
         '''Reads gage geopackage from huc level and filters based on current branch id'''
 
         usgs_gages = gpd.read_file(gages_filename)
+        # TODO: Do we need project this usgs gages to huc_CRS as well?
+        # usgs_gages.to_crs(huc_CRS, inplace=True)
         return usgs_gages[(usgs_gages.levpa_id == self.branch_id)]
 
-    def catchment_sjoin(self, input_catchment_filename):
+    def catchment_sjoin(self, input_catchment_filename, huc_CRS):
         '''Spatial joins gages to FIM catchments'''
 
         input_catchments = gpd.read_file(input_catchment_filename, dtype={'HydroID': int})
+        input_catchments.to_crs(huc_CRS, inplace=True)
         self.gages = gpd.sjoin(self.gages, input_catchments[['HydroID', 'LakeID', 'geometry']], how='inner')
 
     def snap_to_dem_derived_flows(self, input_flows_filename):
@@ -171,6 +184,7 @@ def snap_to_line(row):
     parser.add_argument(
         '-b', '--branch-id', help='Branch ID used to filter the gages', type=str, required=True
     )
+    parser.add_argument('-huc_CRS', help='Projection to be used for the HUC.', type=str, required=True)
 
     args = vars(parser.parse_args())
 
@@ -181,13 +195,19 @@ def snap_to_line(row):
     dem_adj_filename = args['dem_adj_filename']
     output_directory = args['output_directory']
     branch_id = args['branch_id']
+    huc_CRS = args['huc_CRS']
 
     assert os.path.isfile(usgs_gages_filename), f"The input file {usgs_gages_filename} does not exist."
 
     # Instantiate class
     gage_crosswalk = GageCrosswalk(usgs_gages_filename, branch_id)
     gage_crosswalk.run_crosswalk(
-        input_catchment_filename, input_flows_filename, dem_filename, dem_adj_filename, output_directory
+        input_catchment_filename,
+        input_flows_filename,
+        dem_filename,
+        dem_adj_filename,
+        output_directory,
+        huc_CRS,
     )
 
 """
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index a4f014fd5..b2622f4ab 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -19,17 +19,18 @@
 
 
 class Gage2Branch(object):
-    def __init__(self, usgs_gage_filename, ras_locs_filename, ahps_filename, huc8):
+    def __init__(self, usgs_gage_filename, ras_locs_filename, ahps_filename, huc8, huc_CRS):
         self.usgs_gage_filename = usgs_gage_filename
         self.ras_locs_filename = ras_locs_filename
         self.ahps_filename = ahps_filename
         self.huc8 = str(huc8)
-        self.load_gages()
+        self.load_gages(huc_CRS)
 
-    def load_gages(self):
+    def load_gages(self, huc_CRS):
         # Read USGS gage file a
         usgs_gages = gpd.read_file(self.usgs_gage_filename, dtype={'location_id': object})
         usgs_gages['source'] = 'usgs_gage'
+        usgs_gages.to_crs(huc_CRS, inplace=True)
 
         # Read RAS2FIM point locations file
         # !!! Geopandas is not honoring the dtype arg with this read_file below (huc8 being read as int64).
@@ -39,7 +40,7 @@ def load_gages(self):
         ras_locs['location_id'] = ras_locs['fid_xs']
 
         # Convert ras locs crs to match usgs gage crs
-        ras_locs.to_crs(usgs_gages.crs, inplace=True)
+        ras_locs.to_crs(huc_CRS, inplace=True)
         ras_locs = ras_locs.rename(columns={'huc8': 'HUC8'})
 
         # Convert Multipoint geometry to Point geometry
@@ -66,6 +67,7 @@ def load_gages(self):
             ahps_sites = ahps_sites.rename(
                 columns={'nwm_feature_id': 'feature_id', 'usgs_site_code': 'location_id'}
             )
+            ahps_sites.to_crs(huc_CRS, inplace=True)
             ahps_sites = ahps_sites[
                 ahps_sites.location_id.isna()
             ]  # Filter sites that are already in the USGS dataset
@@ -163,6 +165,13 @@ def filter_gage_branches(fim_inputs_filename):
         help='WARNING: only run this parameter if you know exactly what you are doing',
         required=False,
     )
+    parser.add_argument(
+        '-huc_CRS',
+        '--huc_CRS',
+        help='Projection to use (based on whether the HUC is in Alaska or CONUS).',
+        type=str,
+        required=True,
+    )
 
     args = vars(parser.parse_args())
 
@@ -174,9 +183,12 @@ def filter_gage_branches(fim_inputs_filename):
     huc8 = args['huc8_id']
     bzero_id = args['branch_zero_id']
     filter_fim_inputs = args['filter_fim_inputs']
+    huc_CRS = args['huc_CRS']
 
     if not filter_fim_inputs:
-        usgs_gage_subset = Gage2Branch(usgs_gages_filename, ras_locs_filename, nws_lid_filename, huc8)
+        usgs_gage_subset = Gage2Branch(
+            usgs_gages_filename, ras_locs_filename, nws_lid_filename, huc8, huc_CRS
+        )
         if usgs_gage_subset.gages.empty:
             print(f'There are no gages identified for {huc8}')
             os._exit(0)

From 39504f350d4e051e501b1ab4abbfb433d7704715 Mon Sep 17 00:00:00 2001
From: AliForghani-NOAA <124194814+AliForghani-NOAA@users.noreply.github.com>
Date: Fri, 17 May 2024 16:24:22 -0400
Subject: [PATCH 322/359] v4.5.1.3 Hotfix for pyogrio arrow (#1170)

---
 docs/CHANGELOG.md                | 14 +++++++++++++-
 src/usgs_gage_crosswalk.py       |  7 ++++---
 src/usgs_gage_unit_setup.py      |  4 ++--
 tools/rating_curve_comparison.py |  2 +-
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7c9a2d721..a483e4544 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.1.3 - 2024-05-17 - [PR#1170](https://github.com/NOAA-OWP/inundation-mapping/pull/1170)
+
+This hotfix addresses the issue #1162 by explicitly using 'fiona' engine for reading gpkg files with Boolean dtype. This is applicable only for `usgs_gages.gpkg` and `usgs_subset_gages.gpkg` files. 
+
+
+### Changes
+- `src/usgs_gage_unit_setup.py`  ... changed only two lines for fiona engine
+- `src/usgs_gage_crosswalk.py` ...  changed only one line for fiona engine + two small changes to use `self.branch_id` for the correct log report
+- `tools/rating_curve_comparison.py`...  changed only one line for fiona engine
+
+<br/><br/>
+
+
 ## v4.5.1.2 - 2024-05-17 - [PR#1135](https://github.com/NOAA-OWP/inundation-mapping/pull/1135)
 
 Updates USGS gage processing to use the correct projection (determined by whether the HUC is in Alaska or not).
@@ -62,7 +75,6 @@ Similar to 'usgs_gages.gpkg' dataset, all lid stations, including those in Alask
 <br/><br/>
 
 
-
 ## v4.5.0.1 - 2024-05-09 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150)
 
 Fixes two bugs discovered in v4.5.0.0:
diff --git a/src/usgs_gage_crosswalk.py b/src/usgs_gage_crosswalk.py
index 98263c602..5019e09a3 100755
--- a/src/usgs_gage_crosswalk.py
+++ b/src/usgs_gage_crosswalk.py
@@ -57,12 +57,12 @@ def run_crosswalk(
         to usgs_elev_table.csv
         '''
         if self.gages.empty:
-            print(f'There are no gages for branch {branch_id}')
+            print(f'There are no gages for branch {self.branch_id}')
             os._exit(0)
         # Spatial join to fim catchments
         self.catchment_sjoin(input_catchment_filename, huc_CRS)
         if self.gages.empty:
-            print(f'There are no gages for branch {branch_id}')
+            print(f'There are no gages for branch {self.branch_id}')
             os._exit(0)
 
         # Snap to dem derived flow lines
@@ -80,7 +80,8 @@ def run_crosswalk(
     def _load_gages(self, gages_filename):
         '''Reads gage geopackage from huc level and filters based on current branch id'''
 
-        usgs_gages = gpd.read_file(gages_filename)
+        usgs_gages = gpd.read_file(gages_filename, engine='fiona')
+
         # TODO: Do we need project this usgs gages to huc_CRS as well?
         # usgs_gages.to_crs(huc_CRS, inplace=True)
         return usgs_gages[(usgs_gages.levpa_id == self.branch_id)]
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index b2622f4ab..308952b3a 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -28,7 +28,7 @@ def __init__(self, usgs_gage_filename, ras_locs_filename, ahps_filename, huc8, h
 
     def load_gages(self, huc_CRS):
         # Read USGS gage file a
-        usgs_gages = gpd.read_file(self.usgs_gage_filename, dtype={'location_id': object})
+        usgs_gages = gpd.read_file(self.usgs_gage_filename, dtype={'location_id': object}, engine='fiona')
         usgs_gages['source'] = 'usgs_gage'
         usgs_gages.to_crs(huc_CRS, inplace=True)
 
@@ -137,7 +137,7 @@ def filter_gage_branches(fim_inputs_filename):
                 fim_inputs = fim_inputs.drop(fim_inputs.loc[fim_inputs.huc == huc_dir].index)
                 continue
 
-            gages = gpd.read_file(gage_file)
+            gages = gpd.read_file(gage_file, engine='fiona')
             level_paths = gages.levpa_id
             fim_inputs = fim_inputs.drop(
                 fim_inputs.loc[(fim_inputs.huc == huc_dir) & (~fim_inputs.levpa_id.isin(level_paths))].index
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 5b5369316..ef8e2417f 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -1070,7 +1070,7 @@ def create_static_gpkg(output_dir, output_gpkg, agg_recurr_stats_table, gages_gp
     Merges the output dataframe from aggregate_metrics() with the usgs gages GIS data
     '''
     # Load in the usgs_gages geopackage
-    usgs_gages = gpd.read_file(gages_gpkg_filepath)
+    usgs_gages = gpd.read_file(gages_gpkg_filepath, engine='fiona')
     # Merge the stats for all of the recurrance intervals/thresholds
     usgs_gages = usgs_gages.merge(agg_recurr_stats_table, on='location_id')
     # Load in the rating curves file

From 7bdf60e15eebdcab5b6dbfdf8d3039a16b6aa610 Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Mon, 20 May 2024 15:40:59 -0500
Subject: [PATCH 323/359] v4.5.2.0 Bridge inundation points (#1166)

---
 config/deny_branch_zero.lst              |   1 +
 config/deny_branches.lst                 |   1 +
 data/bridges/pull_osm_bridges.py         |   8 --
 docs/CHANGELOG.md                        |  17 +++
 fim_post_processing.sh                   |   1 +
 src/aggregate_by_huc.py                  |  99 ++++++++++++++-
 src/delineate_hydros_and_produce_HAND.sh |  53 ++++----
 src/heal_bridges_osm.py                  | 148 ++++++++++++-----------
 8 files changed, 214 insertions(+), 114 deletions(-)

diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst
index 3d7a8cd9c..f4bf2cba9 100644
--- a/config/deny_branch_zero.lst
+++ b/config/deny_branch_zero.lst
@@ -49,6 +49,7 @@ nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
+#osm_bridge_centroids_{}.gpkg
 rem_{}.tif
 #rem_zeroed_masked_{}.tif
 slopes_d8_dem_meters_{}.tif
diff --git a/config/deny_branches.lst b/config/deny_branches.lst
index 8cce17df4..07ef1f7f4 100644
--- a/config/deny_branches.lst
+++ b/config/deny_branches.lst
@@ -50,6 +50,7 @@ nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
+#osm_bridge_centroids_{}.gpkg
 #ras_elev_table.csv
 rem_{}.tif
 # rem_zeroed_masked_{}.tif
diff --git a/data/bridges/pull_osm_bridges.py b/data/bridges/pull_osm_bridges.py
index 7c7bf6183..1f73ad196 100644
--- a/data/bridges/pull_osm_bridges.py
+++ b/data/bridges/pull_osm_bridges.py
@@ -86,7 +86,6 @@ def combine_huc_features(output_dir):
     bridge_file_names = Path(output_dir).glob("huc_*_osm_bridges.gpkg")
 
     osm_bridge_file = os.path.join(output_dir, "osm_all_bridges.gpkg")
-    osm_bridge_midpoints_file = os.path.join(output_dir, "osm_all_bridges_midpoints.gpkg")
 
     all_bridges_gdf_raw = pd.concat([gpd.read_file(gpkg) for gpkg in bridge_file_names], ignore_index=True)
 
@@ -99,13 +98,6 @@ def combine_huc_features(output_dir):
     all_bridges_gdf = all_bridges_gdf_raw[['osmid', 'name', 'geometry']]
     all_bridges_gdf.to_file(osm_bridge_file, driver="GPKG")
 
-    # save out midpoints (centroids) of bridge lines
-    logging.info(f"Writing centroids {osm_bridge_file}")
-    section_time = dt.datetime.now(dt.timezone.utc)
-    logging.info(f"  .. started: {section_time.strftime('%m/%d/%Y %H:%M:%S')}")
-    all_bridges_gdf['geometry'] = all_bridges_gdf.centroid
-    all_bridges_gdf.to_file(osm_bridge_midpoints_file, driver="GPKG")
-
     return
 
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index a483e4544..3d15f1533 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.0 - 2024-05-20 - [PR#1166](https://github.com/NOAA-OWP/inundation-mapping/pull/1166)
+
+The main goal of this PR is to create bridge point data that be used as a service in HydroVIS. Since every branch processes bridges separately, it's possible to inundate a bridge from more than just the feature_id it crosses. To reflect this, the `osm_bridge_centroids.gpkg` now found in HUC directories will have coincident points - one that is inundated from the reach it crosses and the other a backwater-influenced point indicated by the `is_backwater` field.
+
+### Changes
+
+- ` src/`
+    - `aggregate_by_huc.py`: Added the aggregation steps for bridge centroids; aggregation includes using SRCs to lookup flow values for each bridge, filtering out coincident points that have the same assigned feature_ids and higher overtopping flow, and assigning select points as backwater-influenced.
+    - ` delineate_hydros_and_produce_HAND.sh`: Moved the bridge healing to after the crosswalk so that the centroids can use the crosswalked catchments for feature_id and flow lookups.
+    -  `heal_bridges_osm.py`: Optimized the bridge healing so that it doesn't have to write out an intermediate raster; exports bridge centroids and spatial joins them to catchments; added functions for SRC flow lookups used in `aggregate_by_huc.py`.
+- ` fim_post_processing.sh`: Added a bridge flag input for `aggregate_by_huc.py`.
+- `data/bridges/pull_osm_bridges.py`: Removed the saving of a midpoint geopackage.
+- `config/deny_branch_zero.lst` & `deny_branches.lst`: Added `#osm_bridge_centroids_{}.gpkg` to the deny lists.
+
+<br/><br/>
+
+
 ## v4.5.1.3 - 2024-05-17 - [PR#1170](https://github.com/NOAA-OWP/inundation-mapping/pull/1170)
 
 This hotfix addresses the issue #1162 by explicitly using 'fiona' engine for reading gpkg files with Boolean dtype. This is applicable only for `usgs_gages.gpkg` and `usgs_subset_gages.gpkg` files. 
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 49ac3c29f..fb9c10f82 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -191,6 +191,7 @@ python3 $srcDir/aggregate_by_huc.py \
     -fim $outputDestDir \
     -i $fim_inputs \
     -htable \
+    -bridge \
     -j $jobLimit
 Tcount
 date -u
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index 5a6912543..e3d8db6e9 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -9,8 +9,10 @@
 from datetime import datetime
 from os.path import join
 
+import geopandas as gpd
 import pandas as pd
 
+from heal_bridges_osm import flows_from_hydrotable
 from utils.shared_functions import progress_bar_handler
 
 
@@ -140,6 +142,20 @@ def __init__(self, fim_directory, huc_id, limit_branches=[]):
         }
         self.agg_ras_elev_table = pd.DataFrame(columns=list(self.ras_dtypes.keys()))
 
+        self.bridge_dtypes = {
+            'osmid': int,
+            'name': str,
+            'max_hand': float,
+            'max_hand_75': float,
+            'feature_id': int,
+            'HydroID': int,
+            'order_': str,
+            'branch': str,
+            'mainstem': int,
+            'geometry': object,
+        }
+        self.agg_bridge_pnts = gpd.GeoDataFrame(columns=list(self.bridge_dtypes.keys()))
+
     def iter_branches(self):
         if self.limit_branches:
             for branch in self.limit_branches:
@@ -184,7 +200,23 @@ def ras_elev_table(self, branch_path):
         ras_elev_table = pd.read_csv(ras_elev_filename, dtype=self.ras_dtypes)
         self.agg_ras_elev_table = pd.concat([self.agg_ras_elev_table, ras_elev_table])
 
-    def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id):
+    def aggregate_bridge_pnts(self, branch_path, branch_id):
+        bridge_filename = join(branch_path, f'osm_bridge_centroids_{branch_id}.gpkg')
+        if not os.path.isfile(bridge_filename):
+            return
+
+        bridge_pnts = gpd.read_file(bridge_filename)
+        if bridge_pnts.empty:
+            return
+        hydrotable_filename = join(branch_path, f'hydroTable_{branch_id}.csv')
+        hydrotable = pd.read_csv(hydrotable_filename, dtype=self.hydrotable_dtypes)
+        # Get the flows for each stage
+        bridge_pnts = flows_from_hydrotable(bridge_pnts, hydrotable)
+        self.agg_bridge_pnts = pd.concat([self.agg_bridge_pnts, bridge_pnts])
+
+    def agg_function(
+        self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, bridge_flag, huc_id
+    ):
         try:
             # try catch and its own log file output in error only.
             for branch_id, branch_path in self.iter_branches():
@@ -198,6 +230,8 @@ def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_ele
                     self.aggregate_hydrotables(branch_path, branch_id)
                 if src_cross_flag:
                     self.aggregate_src_full_crosswalk(branch_path, branch_id)
+                if bridge_flag:
+                    self.aggregate_bridge_pnts(branch_path, branch_id)
 
             ## After all of the branches are visited, the code below will write the aggregates
             if usgs_elev_flag:
@@ -232,6 +266,30 @@ def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_ele
                 if not self.agg_ras_elev_table.empty:
                     self.agg_ras_elev_table.to_csv(ras_elev_table_file, index=False)
 
+            if bridge_flag:
+                bridge_pnts_file = join(self.huc_dir_path, 'osm_bridge_centroids.gpkg')
+                if os.path.isfile(bridge_pnts_file):
+                    os.remove(bridge_pnts_file)
+
+                if not self.agg_bridge_pnts.empty:
+                    # Just making things shorter so they are easier to read
+                    bridge_pnts = self.agg_bridge_pnts
+                    # Use branch 0 to get the feature_id each bridge crosses
+                    b0 = bridge_pnts.loc[bridge_pnts.branch == '0', ['osmid', 'feature_id']]
+                    b0 = b0.rename(columns={'feature_id': 'crossing_feature_id'})
+                    bridge_pnts = bridge_pnts.merge(b0, on='osmid', how='left')
+                    # Remove bridge points that have the same osmid and feature_id
+                    g = bridge_pnts.groupby(['osmid', 'feature_id'])['max_discharge'].transform('min')
+                    bridge_pnts = bridge_pnts.copy()[(bridge_pnts['max_discharge'] == g)]
+                    # Set backwater bridge sites
+                    bridge_pnts['is_backwater'] = 0
+                    c = bridge_pnts.groupby(['osmid'])['feature_id'].transform('count')
+                    bridge_pnts.loc[
+                        (c > 1) & (bridge_pnts.feature_id != bridge_pnts.crossing_feature_id), 'is_backwater'
+                    ] = 1
+                    # Write file
+                    bridge_pnts.to_file(bridge_pnts_file, index=False)
+
             # print(f"agg_by_huc for huc id {huc_id} is done")
 
         except Exception:
@@ -247,6 +305,7 @@ def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_ele
                 hydro_table_flag,
                 src_cross_flag,
                 ras_elev_flag,
+                bridge_flag,
                 huc_id,
                 errMsg,
             )
@@ -255,7 +314,16 @@ def agg_function(self, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_ele
 # ==============================
 # This is done independantly in each worker and does not attempt to write to a shared file
 # as those can collide with multi proc
-def log_error(fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg):
+def log_error(
+    fim_directory,
+    usgs_elev_flag,
+    hydro_table_flag,
+    src_cross_flag,
+    ras_elev_flag,
+    bridge_flag,
+    huc_id,
+    errMsg,
+):
     file_name = f"agg_by_huc_{huc_id}"
     if usgs_elev_flag:
         file_name += "_elev"
@@ -265,6 +333,8 @@ def log_error(fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, r
         file_name += "_src_cross"
     if ras_elev_flag:
         file_name += "_ras"
+    if bridge_flag:
+        file_name += "_bridge"
     file_name += "_error.log"
 
     log_path = os.path.join(fim_directory, "logs", "agg_by_huc_errors")
@@ -282,6 +352,7 @@ def aggregate_by_huc(
     hydro_table_flag,
     src_cross_flag,
     ras_elev_flag,
+    bridge_flag,
     num_job_workers,
 ):
     assert os.path.isdir(fim_directory), f'{fim_directory} is not a valid directory'
@@ -315,6 +386,8 @@ def aggregate_by_huc(
             agg_type += "_src_cross"
         if ras_elev_flag:
             agg_type += "_ras"
+        if bridge_flag:
+            agg_type += "_bridge"
         filelist = glob.glob(os.path.join(log_folder, f"*{agg_type}*"))
         for f in filelist:
             os.remove(f)
@@ -345,6 +418,7 @@ def aggregate_by_huc(
                         'hydro_table_flag': hydro_table_flag,
                         'src_cross_flag': src_cross_flag,
                         'ras_elev_flag': ras_elev_flag,
+                        'bridge_flag': bridge_flag,
                         'huc_id': huc_id,
                     }
 
@@ -368,6 +442,7 @@ def aggregate_by_huc(
                         'hydro_table_flag': hydro_table_flag,
                         'src_cross_flag': src_cross_flag,
                         'ras_elev_flag': ras_elev_flag,
+                        'bridge_flag': bridge_flag,
                         'huc_id': huc_id,
                     }
                     future = executor.submit(huc_dir.agg_function, **args_agg)
@@ -381,16 +456,20 @@ def aggregate_by_huc(
             errMsg = errMsg + traceback.format_exc()
             print(errMsg, flush=True)
             log_error(
-                fim_directory, usgs_elev_flag, hydro_table_flag, src_cross_flag, ras_elev_flag, huc_id, errMsg
+                fim_directory,
+                usgs_elev_flag,
+                hydro_table_flag,
+                src_cross_flag,
+                ras_elev_flag,
+                bridge_flag,
+                huc_id,
+                errMsg,
             )
             # sys.exit(1)
 
         # Send the executor to the progress bar and wait for all MS tasks to finish
         progress_bar_handler(executor_dict, f"Running aggregate_by_huc with {num_job_workers} workers")
 
-        # Send the executor to the progress bar and wait for all MS tasks to finish
-        progress_bar_handler(executor_dict, f"Running aggregate_by_huc with {num_job_workers} workers")
-
     end_time = datetime.now()
     dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
     print(f"ended: {dt_string}")
@@ -441,6 +520,14 @@ def aggregate_by_huc(
         default=False,
         action='store_true',
     )
+    parser.add_argument(
+        '-bridge',
+        '--bridge_flag',
+        help='Perform aggregate on branch bridge centroid files',
+        required=False,
+        default=False,
+        action='store_true',
+    )
     parser.add_argument(
         '-j', '--num_job_workers', help='Number of processes to use', required=False, default=1, type=int
     )
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index efe6e5f55..77b64b7f5 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -218,7 +218,6 @@ if  [ -f $tempCurrentBranchDataDir/LandSea_subset_$current_branch_id.tif ]; then
         --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
 fi
 
-
 ## HYDRAULIC PROPERTIES ##
 echo -e $startDiv"Sample reach averaged parameters $hucNumber $current_branch_id"
 $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
@@ -229,34 +228,6 @@ $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$curr
     -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv
 
 
-## HEAL HAND -- REMOVES HYDROCONDITIONING ARTIFACTS ##
-if [ "$healed_hand_hydrocondition" = true ]; then
-    echo -e $startDiv"Healed HAND to Remove Hydro-conditioning Artifacts $hucNumber $current_branch_id"
-    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
-        -R $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
-        -D $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
-        -T $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
-        --calc="R+(D-T)" --NoDataValue=$ndv \
-        --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
-fi
-
-
-## HEAL HAND BRIDGES (note resolution is set to 10m)
-# May or may not be a bridge file, depends if the HUC has an applicble one.
-# Writing back over the rem_zeroed_masked branch tif
-if  [ -f $tempHucDataDir/osm_bridges_subset.gpkg ]; then
-    echo -e $startDiv"Burn in bridges $hucNumber $current_branch_id"
-    python3 $srcDir/heal_bridges_osm.py \
-                -g $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
-                -s $tempHucDataDir/osm_bridges_subset.gpkg \
-                -p $tempCurrentBranchDataDir/bridge_lines_raster_$current_branch_id.tif \
-                -t $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
-                -r $res
-else
-    echo -e $startDiv"No applicable bridge data for $hucNumber"
-fi
-
-
 ## FINALIZE CATCHMENTS AND MODEL STREAMS ##
 echo -e $startDiv"Finalize catchments and model streams $hucNumber $current_branch_id"
 python3 $srcDir/add_crosswalk.py \
@@ -278,6 +249,30 @@ python3 $srcDir/add_crosswalk.py \
     -e $min_catchment_area \
     -g $min_stream_length
 
+## HEAL HAND -- REMOVES HYDROCONDITIONING ARTIFACTS ##
+if [ "$healed_hand_hydrocondition" = true ]; then
+    echo -e $startDiv"Healed HAND to Remove Hydro-conditioning Artifacts $hucNumber $current_branch_id"
+    gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
+        -R $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
+        -D $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \
+        -T $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \
+        --calc="R+(D-T)" --NoDataValue=$ndv \
+        --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif"
+fi
+
+## HEAL HAND BRIDGES ##
+if  [ -f $tempHucDataDir/osm_bridges_subset.gpkg ]; then
+    echo -e $startDiv"Burn in bridges $hucNumber $current_branch_id"
+    python3 $srcDir/heal_bridges_osm.py \
+        -g $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \
+        -s $tempHucDataDir/osm_bridges_subset.gpkg \
+        -p $tempCurrentBranchDataDir/gw_catchments_reaches_filtered_addedAttributes_crosswalked_$current_branch_id.gpkg \
+        -c $tempCurrentBranchDataDir/osm_bridge_centroids_$current_branch_id.gpkg \
+        -b 10 \
+        -r $res
+else
+    echo -e $startDiv"No applicable bridge data for $hucNumber"
+fi
 
 ## EVALUATE CROSSWALK ##
 if [ "$current_branch_id" = "$branch_zero_id" ] && [ "$evaluateCrosswalk" = "1" ] ; then
diff --git a/src/heal_bridges_osm.py b/src/heal_bridges_osm.py
index 288572fc5..19ade89cf 100644
--- a/src/heal_bridges_osm.py
+++ b/src/heal_bridges_osm.py
@@ -1,106 +1,107 @@
 import argparse
 import os
+import re
 
 import geopandas as gpd
 import numpy as np
 import rasterio
-from rasterio import features, transform
+from rasterio import features
 from rasterstats import zonal_stats
 
 
+threatened_percent = 0.75
+
+
 def process_bridges_in_huc(
-    source_hand_raster, bridge_vector_file, bridge_raster, output_bridge_lines_raster_file, resolution
+    source_hand_raster, bridge_vector_file, catchments, bridge_centroids, buffer_width, resolution
 ):
-    """
-    Process:
-        - Use the osm_file, likely already pre-clipped to a huc, and create a raster from it.
-        - using the incomeing source_raster, likely a rem at this point, find the max point value for intersecting from the
-          line string. Fundemtally create a z value for the line string based on the touch point of the linestring on both
-          banks. TODO: how does it handle higher elevation on one bank? Might already be in this logic.. not sure.
-        - That value will be assigned to each cell that covers the bridge in the output raster.
-        - We do create a temp raster which we can removed in the "deny" list system. We can use it to show the point values
-          assigned to each cell of the bridge.
-    """
 
     if not os.path.exists(source_hand_raster):
         print(f"-- no hand grid, {source_hand_raster}")
         return
 
     if os.path.exists(bridge_vector_file):
+        # Read the bridge lines file and buffer it by half of the input width
         osm_gdf = gpd.read_file(bridge_vector_file)
+        osm_gdf['centroid_geometry'] = osm_gdf.centroid
+        osm_gdf['geometry'] = osm_gdf.geometry.buffer(buffer_width, resolution=resolution)
     else:
         # skip this huc because it didn't pull in the initial OSM script
         # and could have errors in the data or geometry
         print(f"-- no OSM file, {bridge_vector_file}")
         return
 
-    hand_grid = rasterio.open(source_hand_raster)
+    with rasterio.open(source_hand_raster, 'r') as hand_grid:
+        profile = hand_grid.profile
+        hand_grid_array = hand_grid.read(1)
 
-    #######################################################################
+        # Get max hand values for each bridge
+        osm_gdf['max_hand'] = zonal_stats(
+            osm_gdf['geometry'], hand_grid_array, affine=hand_grid.transform, stats="max"
+        )
+        # pull the values out of the geopandas columns so we can use them as floats
+        osm_gdf['max_hand'] = [x.get('max') for x in osm_gdf.max_hand]
+        # sort in case of overlaps; display max hand value at any given location
+        osm_gdf = osm_gdf.sort_values(by="max_hand", ascending=False)
 
-    ############# get max hand values for each bridge #########
-    # find max hand value from raster each linestring intersects
-    osm_gdf['max_hand'] = zonal_stats(
-        osm_gdf['geometry'], hand_grid.read(1), affine=hand_grid.transform, stats="max"
-    )
-    # pull the values out of the geopandas columns so we can use them as floats
-    osm_gdf['max_hand'] = [x.get('max') for x in osm_gdf.max_hand]
-    # sort in case of overlaps; display max hand value at any given location
-    osm_gdf = osm_gdf.sort_values(by="max_hand", ascending=False)
-    #######################################################
-
-    ########### setup new raster to save bridge max hand values #############
-    bbox = hand_grid.bounds
-    xmin, ymin, xmax, ymax = bbox
-    w = hand_grid.shape[1]
-    h = hand_grid.shape[0]
-
-    out_meta = {
-        "driver": "GTiff",
-        "dtype": "float32",
-        "height": h,
-        "width": w,
-        "count": 1,
-        "crs": hand_grid.crs,
-        "nodata": -999999,
-        "transform": transform.from_bounds(xmin, ymin, xmax, ymax, w, h),
-        "compress": 'lzw',
-    }
-
-    ################# rasterize the incoming bridge vectors ####################
-    with rasterio.open(bridge_raster, 'w+', **out_meta) as out:
-        out_arr = out.read(1)
-        # this is where we create a generator of geom, value pairs to use in rasterizing
+        # Burn the bridges into the HAND grid
         shapes = ((geom, value) for geom, value in zip(osm_gdf.geometry, osm_gdf.max_hand))
-        # burn in values to any pixel that's touched by polygon and add nodata fill value
-        burned = features.rasterize(
-            shapes=shapes, fill=-999999, out=out_arr, transform=hand_grid.transform, all_touched=True
+        features.rasterize(
+            shapes=shapes, out=hand_grid_array, transform=hand_grid.transform, all_touched=False
         )
-        out.write_band(1, burned)
 
-    #################### heal / update hand grid ##########################
-    with rasterio.open(bridge_raster) as in_data:
-        new_hand_values = in_data.read(1)
+    # Write the new HAND grid
+    with rasterio.open(source_hand_raster, 'w', **profile) as new_hand_grid:
+        new_hand_grid.write(hand_grid_array, 1)
+
+    # Switch the geometry over to the centroid points
+    osm_gdf['geometry'] = osm_gdf['centroid_geometry']
+    osm_gdf = osm_gdf.drop(columns='centroid_geometry')
+
+    # Join the bridge points to the HAND catchments to get the HydroID and feature_id
+    osm_gdf = osm_gdf.loc[osm_gdf.max_hand >= 0]
+    catchments_df = gpd.read_file(catchments)
+    osm_gdf = gpd.sjoin(osm_gdf, catchments_df[['HydroID', 'feature_id', 'order_', 'geometry']], how='inner')
+    osm_gdf = osm_gdf.drop(columns='index_right')
+    # Calculate threatened stage
+    osm_gdf['max_hand_75'] = osm_gdf.max_hand * threatened_percent
+    # Add the branch id to the catchments
+    branch_dir = re.search(r'branches/(\d{10}|0)/', catchments).group()
+    branch_id = re.search(r'(\d{10}|0)', branch_dir).group()
+    osm_gdf['branch'] = branch_id
+    osm_gdf['mainstem'] = 0 if branch_id == '0' else 1
+
+    # Write the bridge points to a geopackage
+    osm_gdf.to_file(bridge_centroids, index=False)
 
-    hand_grid_vals = hand_grid.read(1)
-    # replace values at all locations where there are healed values available
-    combined_hand_values = np.where(new_hand_values == -999999, hand_grid_vals, new_hand_values)
+    return
 
-    with rasterio.open(output_bridge_lines_raster_file, 'w+', **out_meta) as out:
-        out.write(combined_hand_values, 1)
 
-    return
+def flow_lookup(stages, hydroid, hydroTable):
+    single_hydroTable = hydroTable.loc[hydroTable.HydroID == hydroid]
+    return_flows = np.interp(
+        stages, single_hydroTable.loc[:, 'stage'], single_hydroTable.loc[:, 'discharge_cms']
+    )
+    return return_flows
+
+
+def flows_from_hydrotable(bridge_pnts, hydroTable):
+    bridge_pnts[['max_discharge', 'max_discharge75']] = bridge_pnts.apply(
+        lambda row: flow_lookup((row.max_hand, row.max_hand_75), row.HydroID, hydroTable),
+        axis=1,
+        result_type='expand',
+    )
+    return bridge_pnts
 
 
 if __name__ == "__main__":
     '''
     Sample usage (min params):
         python3 src/heal_bridges_osm.py
-            -u 12090301
             -g /outputs/fim_4_4_15_0/1209301/branches/3763000013/rem_zeroed_masked_3763000013.tif
             -s /outputs/fim_4_4_15_0/1209301/osm_bridges_subset.gpkg
-            -p /outputs/fim_4_4_15_0/1209301/1209301/branches/3763000013/bridge_lines_raster_3763000013.tif
-            -t /outputs/fim_4_4_15_0/1209301/branches/3763000013/rem_zeroed_masked_3763000013.tif
+            -c /outputs/fim_4_4_15_0/1209301/1209301/branches/3763000013/osm_bridge_centroids_3763000013.tif
+            -b 10
             -r 10
 
     '''
@@ -119,26 +120,31 @@ def process_bridges_in_huc(
     )
 
     parser.add_argument(
-        '-p',
-        '--bridge_raster',
-        help='REQUIRED: Path and file name of the temp raster created by the bridge vectors',
-        required=True,
+        '-p', '--catchments', help='REQUIRED: Path and file name of the catchments geopackage', required=True
     )
 
     parser.add_argument(
-        '-t',
-        '--output_bridge_lines_raster_file',
-        help='REQUIRED: Path and file name of the new raster with the bridge burned in',
+        '-c',
+        '--bridge_centroids',
+        help='REQUIRED: Path and file name of the output bridge centroid geopackage',
         required=True,
     )
 
+    parser.add_argument(
+        '-b',
+        '--buffer_width',
+        help='OPTIONAL: Buffer to apply to OSM bridges. Default value is 10m (on each side)',
+        required=False,
+        default=10,
+        type=float,
+    )
     parser.add_argument(
         '-r',
         '--resolution',
         help='OPTIONAL: Resolution of HAND grid. Default value is 10m',
         required=False,
         default=10,
-        type=float,
+        type=int,
     )
 
     args = vars(parser.parse_args())

From 3092e979f91c287235b7ebf5fed0d727466c7f7c Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Tue, 21 May 2024 15:23:36 +0000
Subject: [PATCH 324/359] v4.5.2.1 Remove loading of apache-arrow repository
 (#1172)

---
 Dockerfile        |   8 +-
 Pipfile           |  12 +-
 Pipfile.lock      | 946 ++++++++++++++++++++++++++++++++++------------
 docs/CHANGELOG.md |  33 +-
 4 files changed, 726 insertions(+), 273 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 68be823cf..96f6b2d3d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,6 +9,9 @@ ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
+# remove reference to missing repo
+RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+
 RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/*
 
 RUN git clone https://github.com/dtarb/taudem.git
@@ -66,6 +69,9 @@ RUN mkdir -p $workDir
 RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
+# remove reference to missing repo
+RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+
 RUN apt-get update --fix-missing && apt-get install -y openjdk-19-jdk && rm -rf /var/lib/apt/lists/*
 
 RUN apt update --fix-missing
@@ -91,7 +97,7 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/unit_tests:$projectDir/tools
 
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
+RUN pip3 install pipenv==2023.12.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
 
 # ----------------------------------
 # Mar 2023
diff --git a/Pipfile b/Pipfile
index 16e8eed68..b4e5981b2 100644
--- a/Pipfile
+++ b/Pipfile
@@ -4,7 +4,7 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-ipython = "==8.12.0"
+ipython = "==8.24.0"
 
 [packages]
 certifi = "==2023.7.22"
@@ -16,7 +16,7 @@ pandas = "==2.0.2"
 rasterio = "==1.3.6"
 rasterstats = "==0.18.0"
 richdem = "==0.3.4"
-tqdm = "==4.65.0"
+tqdm = "==4.66.3"
 seaborn = "==0.12.2"
 python-dotenv = "==1.0.0"
 natsort = "==8.3.1"
@@ -46,8 +46,12 @@ urllib3 = "==1.26.18"
 pyflwdir = "==0.5.8"
 pillow = "==10.3.0"
 pyogrio = "==0.7.2"
-openpyxl = "*"
-osmnx = "*"
+openpyxl = "==3.1.2"
+osmnx = "==1.9.3"
+lmoments3 = "==1.0.6"
+zarr = "==2.18.0"
+requests = "==2.31.0"
+aiohttp = "==3.9.5"
 
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index 4162d23ab..c3758ef76 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "a7f3b5a12b051d29d0dee120d17361cca1ce11c412d15be4c207f6ca43e615b9"
+            "sha256": "b1097cae3089688432b1d87cf34f1f1877327168066113e534a986fd0a9fa6fa"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -29,9 +29,99 @@
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
                 "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6"
             ],
-            "markers": "python_version >= '3.7' and python_version < '4'",
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
+        "aiohttp": {
+            "hashes": [
+                "sha256:0605cc2c0088fcaae79f01c913a38611ad09ba68ff482402d3410bf59039bfb8",
+                "sha256:0a158704edf0abcac8ac371fbb54044f3270bdbc93e254a82b6c82be1ef08f3c",
+                "sha256:0cbf56238f4bbf49dab8c2dc2e6b1b68502b1e88d335bea59b3f5b9f4c001475",
+                "sha256:1732102949ff6087589408d76cd6dea656b93c896b011ecafff418c9661dc4ed",
+                "sha256:18f634d540dd099c262e9f887c8bbacc959847cfe5da7a0e2e1cf3f14dbf2daf",
+                "sha256:239f975589a944eeb1bad26b8b140a59a3a320067fb3cd10b75c3092405a1372",
+                "sha256:2faa61a904b83142747fc6a6d7ad8fccff898c849123030f8e75d5d967fd4a81",
+                "sha256:320e8618eda64e19d11bdb3bd04ccc0a816c17eaecb7e4945d01deee2a22f95f",
+                "sha256:38d80498e2e169bc61418ff36170e0aad0cd268da8b38a17c4cf29d254a8b3f1",
+                "sha256:3916c8692dbd9d55c523374a3b8213e628424d19116ac4308e434dbf6d95bbdd",
+                "sha256:393c7aba2b55559ef7ab791c94b44f7482a07bf7640d17b341b79081f5e5cd1a",
+                "sha256:3b7b30258348082826d274504fbc7c849959f1989d86c29bc355107accec6cfb",
+                "sha256:3fcb4046d2904378e3aeea1df51f697b0467f2aac55d232c87ba162709478c46",
+                "sha256:4109adee842b90671f1b689901b948f347325045c15f46b39797ae1bf17019de",
+                "sha256:4558e5012ee03d2638c681e156461d37b7a113fe13970d438d95d10173d25f78",
+                "sha256:45731330e754f5811c314901cebdf19dd776a44b31927fa4b4dbecab9e457b0c",
+                "sha256:4715a9b778f4293b9f8ae7a0a7cef9829f02ff8d6277a39d7f40565c737d3771",
+                "sha256:471f0ef53ccedec9995287f02caf0c068732f026455f07db3f01a46e49d76bbb",
+                "sha256:4d3ebb9e1316ec74277d19c5f482f98cc65a73ccd5430540d6d11682cd857430",
+                "sha256:4ff550491f5492ab5ed3533e76b8567f4b37bd2995e780a1f46bca2024223233",
+                "sha256:52c27110f3862a1afbcb2af4281fc9fdc40327fa286c4625dfee247c3ba90156",
+                "sha256:55b39c8684a46e56ef8c8d24faf02de4a2b2ac60d26cee93bc595651ff545de9",
+                "sha256:5a7ee16aab26e76add4afc45e8f8206c95d1d75540f1039b84a03c3b3800dd59",
+                "sha256:5ca51eadbd67045396bc92a4345d1790b7301c14d1848feaac1d6a6c9289e888",
+                "sha256:5d6b3f1fabe465e819aed2c421a6743d8debbde79b6a8600739300630a01bf2c",
+                "sha256:60cdbd56f4cad9f69c35eaac0fbbdf1f77b0ff9456cebd4902f3dd1cf096464c",
+                "sha256:6380c039ec52866c06d69b5c7aad5478b24ed11696f0e72f6b807cfb261453da",
+                "sha256:639d0042b7670222f33b0028de6b4e2fad6451462ce7df2af8aee37dcac55424",
+                "sha256:66331d00fb28dc90aa606d9a54304af76b335ae204d1836f65797d6fe27f1ca2",
+                "sha256:67c3119f5ddc7261d47163ed86d760ddf0e625cd6246b4ed852e82159617b5fb",
+                "sha256:694d828b5c41255e54bc2dddb51a9f5150b4eefa9886e38b52605a05d96566e8",
+                "sha256:6ae79c1bc12c34082d92bf9422764f799aee4746fd7a392db46b7fd357d4a17a",
+                "sha256:702e2c7c187c1a498a4e2b03155d52658fdd6fda882d3d7fbb891a5cf108bb10",
+                "sha256:714d4e5231fed4ba2762ed489b4aec07b2b9953cf4ee31e9871caac895a839c0",
+                "sha256:7b179eea70833c8dee51ec42f3b4097bd6370892fa93f510f76762105568cf09",
+                "sha256:7f64cbd44443e80094309875d4f9c71d0401e966d191c3d469cde4642bc2e031",
+                "sha256:82a6a97d9771cb48ae16979c3a3a9a18b600a8505b1115cfe354dfb2054468b4",
+                "sha256:84dabd95154f43a2ea80deffec9cb44d2e301e38a0c9d331cc4aa0166fe28ae3",
+                "sha256:8676e8fd73141ded15ea586de0b7cda1542960a7b9ad89b2b06428e97125d4fa",
+                "sha256:88e311d98cc0bf45b62fc46c66753a83445f5ab20038bcc1b8a1cc05666f428a",
+                "sha256:8b4f72fbb66279624bfe83fd5eb6aea0022dad8eec62b71e7bf63ee1caadeafe",
+                "sha256:8c64a6dc3fe5db7b1b4d2b5cb84c4f677768bdc340611eca673afb7cf416ef5a",
+                "sha256:8cf142aa6c1a751fcb364158fd710b8a9be874b81889c2bd13aa8893197455e2",
+                "sha256:8d1964eb7617907c792ca00b341b5ec3e01ae8c280825deadbbd678447b127e1",
+                "sha256:93e22add827447d2e26d67c9ac0161756007f152fdc5210277d00a85f6c92323",
+                "sha256:9c69e77370cce2d6df5d12b4e12bdcca60c47ba13d1cbbc8645dd005a20b738b",
+                "sha256:9dbc053ac75ccc63dc3a3cc547b98c7258ec35a215a92bd9f983e0aac95d3d5b",
+                "sha256:9e3a1ae66e3d0c17cf65c08968a5ee3180c5a95920ec2731f53343fac9bad106",
+                "sha256:a6ea1a5b409a85477fd8e5ee6ad8f0e40bf2844c270955e09360418cfd09abac",
+                "sha256:a81b1143d42b66ffc40a441379387076243ef7b51019204fd3ec36b9f69e77d6",
+                "sha256:ad7f2919d7dac062f24d6f5fe95d401597fbb015a25771f85e692d043c9d7832",
+                "sha256:afc52b8d969eff14e069a710057d15ab9ac17cd4b6753042c407dcea0e40bf75",
+                "sha256:b3df71da99c98534be076196791adca8819761f0bf6e08e07fd7da25127150d6",
+                "sha256:c088c4d70d21f8ca5c0b8b5403fe84a7bc8e024161febdd4ef04575ef35d474d",
+                "sha256:c26959ca7b75ff768e2776d8055bf9582a6267e24556bb7f7bd29e677932be72",
+                "sha256:c413016880e03e69d166efb5a1a95d40f83d5a3a648d16486592c49ffb76d0db",
+                "sha256:c6021d296318cb6f9414b48e6a439a7f5d1f665464da507e8ff640848ee2a58a",
+                "sha256:c671dc117c2c21a1ca10c116cfcd6e3e44da7fcde37bf83b2be485ab377b25da",
+                "sha256:c7a4b7a6cf5b6eb11e109a9755fd4fda7d57395f8c575e166d363b9fc3ec4678",
+                "sha256:c8a02fbeca6f63cb1f0475c799679057fc9268b77075ab7cf3f1c600e81dd46b",
+                "sha256:cd2adf5c87ff6d8b277814a28a535b59e20bfea40a101db6b3bdca7e9926bc24",
+                "sha256:d1469f228cd9ffddd396d9948b8c9cd8022b6d1bf1e40c6f25b0fb90b4f893ed",
+                "sha256:d153f652a687a8e95ad367a86a61e8d53d528b0530ef382ec5aaf533140ed00f",
+                "sha256:d5ab8e1f6bee051a4bf6195e38a5c13e5e161cb7bad83d8854524798bd9fcd6e",
+                "sha256:da00da442a0e31f1c69d26d224e1efd3a1ca5bcbf210978a2ca7426dfcae9f58",
+                "sha256:da22dab31d7180f8c3ac7c7635f3bcd53808f374f6aa333fe0b0b9e14b01f91a",
+                "sha256:e0ae53e33ee7476dd3d1132f932eeb39bf6125083820049d06edcdca4381f342",
+                "sha256:e7a6a8354f1b62e15d48e04350f13e726fa08b62c3d7b8401c0a1314f02e3558",
+                "sha256:e9a3d838441bebcf5cf442700e3963f58b5c33f015341f9ea86dcd7d503c07e2",
+                "sha256:edea7d15772ceeb29db4aff55e482d4bcfb6ae160ce144f2682de02f6d693551",
+                "sha256:f22eb3a6c1080d862befa0a89c380b4dafce29dc6cd56083f630073d102eb595",
+                "sha256:f26383adb94da5e7fb388d441bf09c61e5e35f455a3217bfd790c6b6bc64b2ee",
+                "sha256:f3c2890ca8c59ee683fd09adf32321a40fe1cf164e3387799efb2acebf090c11",
+                "sha256:f64fd07515dad67f24b6ea4a66ae2876c01031de91c93075b8093f07c0a2d93d",
+                "sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7",
+                "sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f"
+            ],
+            "index": "pypi",
+            "version": "==3.9.5"
+        },
+        "aiosignal": {
+            "hashes": [
+                "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc",
+                "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.1"
+        },
         "aiosqlite": {
             "hashes": [
                 "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6",
@@ -98,6 +188,12 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.3.0"
         },
+        "asciitree": {
+            "hashes": [
+                "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"
+            ],
+            "version": "==0.3.3"
+        },
         "asttokens": {
             "hashes": [
                 "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24",
@@ -105,6 +201,14 @@
             ],
             "version": "==2.4.1"
         },
+        "async-timeout": {
+            "hashes": [
+                "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f",
+                "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==4.0.3"
+        },
         "attrs": {
             "hashes": [
                 "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
@@ -115,11 +219,11 @@
         },
         "babel": {
             "hashes": [
-                "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363",
-                "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"
+                "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb",
+                "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.14.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.15.0"
         },
         "beautifulsoup4": {
             "hashes": [
@@ -189,7 +293,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4'",
+            "markers": "python_version >= '3.8' and python_version < '4.0'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -522,7 +626,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -778,6 +882,14 @@
             "markers": "python_version >= '3.5'",
             "version": "==2.0.1"
         },
+        "fasteners": {
+            "hashes": [
+                "sha256:758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237",
+                "sha256:b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c"
+            ],
+            "markers": "sys_platform != 'emscripten'",
+            "version": "==0.19"
+        },
         "fastjsonschema": {
             "hashes": [
                 "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0",
@@ -787,11 +899,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f",
-                "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"
+                "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f",
+                "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.13.4"
+            "version": "==3.14.0"
         },
         "fiona": {
             "hashes": [
@@ -896,6 +1008,89 @@
             ],
             "version": "==1.5.1"
         },
+        "frozenlist": {
+            "hashes": [
+                "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7",
+                "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98",
+                "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad",
+                "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5",
+                "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae",
+                "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e",
+                "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a",
+                "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701",
+                "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d",
+                "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6",
+                "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6",
+                "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106",
+                "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75",
+                "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868",
+                "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a",
+                "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0",
+                "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1",
+                "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826",
+                "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec",
+                "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6",
+                "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950",
+                "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19",
+                "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0",
+                "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8",
+                "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a",
+                "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09",
+                "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86",
+                "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c",
+                "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5",
+                "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b",
+                "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b",
+                "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d",
+                "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0",
+                "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea",
+                "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776",
+                "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a",
+                "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897",
+                "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7",
+                "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09",
+                "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9",
+                "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe",
+                "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd",
+                "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742",
+                "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09",
+                "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0",
+                "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932",
+                "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1",
+                "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a",
+                "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49",
+                "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d",
+                "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7",
+                "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480",
+                "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89",
+                "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e",
+                "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b",
+                "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82",
+                "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb",
+                "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068",
+                "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8",
+                "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b",
+                "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb",
+                "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2",
+                "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11",
+                "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b",
+                "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc",
+                "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0",
+                "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497",
+                "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17",
+                "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0",
+                "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2",
+                "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439",
+                "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5",
+                "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac",
+                "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825",
+                "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887",
+                "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced",
+                "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.1"
+        },
         "fsspec": {
             "hashes": [
                 "sha256:918d18d41bf73f0e2b261824baeb1b124bcf771767e3a26425cd7dec3332f512",
@@ -1053,11 +1248,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:07232af52a5ba146dc3372c7bf52a0f890a23edf38d77caef8d53f9cdc2584c1",
-                "sha256:7468edaf4f6de3e1b912e57f66c241e6fd3c7099f2ec2136e239e142e800274d"
+                "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501",
+                "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.23.0"
+            "version": "==8.24.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1099,11 +1294,11 @@
         },
         "jinja2": {
             "hashes": [
-                "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
-                "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
+                "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369",
+                "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.3"
+            "version": "==3.1.4"
         },
         "jmespath": {
             "hashes": [
@@ -1115,11 +1310,11 @@
         },
         "joblib": {
             "hashes": [
-                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
-                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
+                "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6",
+                "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.4.0"
+            "version": "==1.4.2"
         },
         "json5": {
             "hashes": [
@@ -1141,11 +1336,11 @@
                 "format-nongpl"
             ],
             "hashes": [
-                "sha256:7996507afae316306f9e2290407761157c6f78002dcf7419acb99822143d1c6f",
-                "sha256:85727c00279f5fa6bedbe6238d2aa6403bedd8b4864ab11207d07df3cc1b2ee5"
+                "sha256:5b22d434a45935119af990552c862e5d6d564e8f6601206b305a61fdf661a2b7",
+                "sha256:ff4cfd6b1367a40e7bc6411caec72effadd3db0bbe5017de188f2d6108335802"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.21.1"
+            "version": "==4.22.0"
         },
         "jsonschema-specifications": {
             "hashes": [
@@ -1412,6 +1607,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.39.1"
         },
+        "lmoments3": {
+            "hashes": [
+                "sha256:404a589335c16549f5ff519d3fafab247203ef78801296447db939b0c58f824b",
+                "sha256:80e1397f1e3e5d1aa471ba67e49e557e7b3a67c7f7b5086c5fc8febcca6969f7"
+            ],
+            "index": "pypi",
+            "version": "==1.0.6"
+        },
         "locket": {
             "hashes": [
                 "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632",
@@ -1634,6 +1837,102 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.0.8"
         },
+        "multidict": {
+            "hashes": [
+                "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556",
+                "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c",
+                "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29",
+                "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b",
+                "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8",
+                "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7",
+                "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd",
+                "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40",
+                "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6",
+                "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3",
+                "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c",
+                "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9",
+                "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5",
+                "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae",
+                "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442",
+                "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9",
+                "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc",
+                "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c",
+                "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea",
+                "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5",
+                "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50",
+                "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182",
+                "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453",
+                "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e",
+                "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600",
+                "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733",
+                "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda",
+                "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241",
+                "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461",
+                "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e",
+                "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e",
+                "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b",
+                "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e",
+                "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7",
+                "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386",
+                "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd",
+                "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9",
+                "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf",
+                "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee",
+                "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5",
+                "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a",
+                "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271",
+                "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54",
+                "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4",
+                "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496",
+                "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb",
+                "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319",
+                "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3",
+                "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f",
+                "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527",
+                "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed",
+                "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604",
+                "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef",
+                "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8",
+                "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5",
+                "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5",
+                "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626",
+                "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c",
+                "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d",
+                "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c",
+                "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc",
+                "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc",
+                "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b",
+                "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38",
+                "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450",
+                "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1",
+                "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f",
+                "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3",
+                "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755",
+                "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226",
+                "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a",
+                "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046",
+                "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf",
+                "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479",
+                "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e",
+                "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1",
+                "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a",
+                "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83",
+                "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929",
+                "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93",
+                "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a",
+                "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c",
+                "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44",
+                "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89",
+                "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba",
+                "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e",
+                "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da",
+                "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24",
+                "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423",
+                "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.0.5"
+        },
         "multimethod": {
             "hashes": [
                 "sha256:7f2a4863967142e6db68632fef9cd79053c09670ba0c5f113301e245140bba5c",
@@ -1699,11 +1998,11 @@
         },
         "nbconvert": {
             "hashes": [
-                "sha256:a6733b78ce3d47c3f85e504998495b07e6ea9cf9bf6ec1c98dda63ec6ad19142",
-                "sha256:ddeff14beeeedf3dd0bc506623e41e4507e551736de59df69a91f86700292b3b"
+                "sha256:05873c620fe520b6322bf8a5ad562692343fe3452abda5765c7a34b7d1aa3eb3",
+                "sha256:86ca91ba266b0a448dc96fa6c5b9d98affabde2867b363258703536807f9f7f4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.16.3"
+            "version": "==7.16.4"
         },
         "nbformat": {
             "hashes": [
@@ -1772,11 +2071,11 @@
         },
         "notebook": {
             "hashes": [
-                "sha256:b4625a4b7a597839dd3156b140d5ba2c7123761f98245a3290f67a8b8ee048d9",
-                "sha256:c1e2eb2e3b6079a0552a04974883a48d04c3c05792170d64a4b23d707d453181"
+                "sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4",
+                "sha256:a6afa9a4ff4d149a0771ff8b8c881a7a73b3835f9add0606696d6e9d98ac1cd0"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.5.6"
+            "version": "==6.5.7"
         },
         "notebook-shim": {
             "hashes": [
@@ -1820,6 +2119,33 @@
             "index": "pypi",
             "version": "==0.56.4"
         },
+        "numcodecs": {
+            "hashes": [
+                "sha256:05d91a433733e7eef268d7e80ec226a0232da244289614a8f3826901aec1098e",
+                "sha256:0e79bf9d1d37199ac00a60ff3adb64757523291d19d03116832e600cac391c51",
+                "sha256:135b2d47563f7b9dc5ee6ce3d1b81b0f1397f69309e909f1a35bb0f7c553d45e",
+                "sha256:21d8267bd4313f4d16f5b6287731d4c8ebdab236038f29ad1b0e93c9b2ca64ee",
+                "sha256:29dfb195f835a55c4d490fb097aac8c1bcb96c54cf1b037d9218492c95e9d8c5",
+                "sha256:2f1ba2f4af3fd3ba65b1bcffb717fe65efe101a50a91c368f79f3101dbb1e243",
+                "sha256:2f84df6b8693206365a5b37c005bfa9d1be486122bde683a7b6446af4b75d862",
+                "sha256:2fbb12a6a1abe95926f25c65e283762d63a9bf9e43c0de2c6a1a798347dfcb40",
+                "sha256:760627780a8b6afdb7f942f2a0ddaf4e31d3d7eea1d8498cf0fd3204a33c4618",
+                "sha256:82d7107f80f9307235cb7e74719292d101c7ea1e393fe628817f0d635b7384f5",
+                "sha256:941b7446b68cf79f089bcfe92edaa3b154533dcbcd82474f994b28f2eedb1c60",
+                "sha256:a191a8e347ecd016e5c357f2bf41fbcb026f6ffe78fff50c77ab12e96701d155",
+                "sha256:abff3554a6892a89aacf7b642a044e4535499edf07aeae2f2e6e8fc08c9ba07f",
+                "sha256:c17687b1fd1fef68af616bc83f896035d24e40e04e91e7e6dae56379eb59fe33",
+                "sha256:c258bd1d3dfa75a9b708540d23b2da43d63607f9df76dfa0309a7597d1de3b73",
+                "sha256:caf1a1e6678aab9c1e29d2109b299f7a467bd4d4c34235b1f0e082167846b88f",
+                "sha256:d37f628fe92b3699e65831d5733feca74d2e33b50ef29118ffd41c13c677210e",
+                "sha256:e04649ea504aff858dbe294631f098fbfd671baf58bfc04fc48d746554c05d67",
+                "sha256:eeaf42768910f1c6eebf6c1bb00160728e62c9343df9e2e315dc9fe12e3f6071",
+                "sha256:ef964d4860d3e6b38df0633caf3e51dc850a6293fd8e93240473642681d95136",
+                "sha256:f2207871868b2464dc11c513965fd99b958a9d7cde2629be7b2dc84fdaab013b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.12.1"
+        },
         "numexpr": {
             "hashes": [
                 "sha256:03d0ba492e484a5a1aeb24b300c4213ed168f2c246177be5733abb4e18cbb043",
@@ -1915,11 +2241,11 @@
         },
         "osmnx": {
             "hashes": [
-                "sha256:59d21756ebdb4a4d211fb1373a3dca170832eec5f25ef94f5500bd79d32c169a",
-                "sha256:9ea17500b45aa8e4740490d649c7f7747fa4f2e3f9747294837fa2c80aabd05f"
+                "sha256:22548d86d68d36edff3cf9ab76c45745cda86a4ea0b28442e107d6b42992a426",
+                "sha256:ac67bea77b521941af648ef641ae1d006101948d1112475c256ea23ef31b426a"
             ],
             "index": "pypi",
-            "version": "==1.9.2"
+            "version": "==1.9.3"
         },
         "overrides": {
             "hashes": [
@@ -2002,11 +2328,11 @@
         },
         "partd": {
             "hashes": [
-                "sha256:27e766663d36c161e2827aa3e28541c992f0b9527d3cca047e13fb3acdb989e6",
-                "sha256:56c25dd49e6fea5727e731203c466c6e092f308d8f0024e199d02f6aa2167f67"
+                "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f",
+                "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.4.1"
+            "markers": "python_version >= '3.9'",
+            "version": "==1.4.2"
         },
         "pathspec": {
             "hashes": [
@@ -2470,11 +2796,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
-                "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
+                "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199",
+                "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.17.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.18.0"
         },
         "pyogrio": {
             "hashes": [
@@ -2738,83 +3064,97 @@
         },
         "pyzmq": {
             "hashes": [
-                "sha256:0108358dab8c6b27ff6b985c2af4b12665c1bc659648284153ee501000f5c107",
-                "sha256:07bec1a1b22dacf718f2c0e71b49600bb6a31a88f06527dfd0b5aababe3fa3f7",
-                "sha256:0e8f482c44ccb5884bf3f638f29bea0f8dc68c97e38b2061769c4cb697f6140d",
-                "sha256:0ec91f1bad66f3ee8c6deb65fa1fe418e8ad803efedd69c35f3b5502f43bd1dc",
-                "sha256:0f14cffd32e9c4c73da66db97853a6aeceaac34acdc0fae9e5bbc9370281864c",
-                "sha256:15975747462ec49fdc863af906bab87c43b2491403ab37a6d88410635786b0f4",
-                "sha256:1724117bae69e091309ffb8255412c4651d3f6355560d9af312d547f6c5bc8b8",
-                "sha256:1a7c280185c4da99e0cc06c63bdf91f5b0b71deb70d8717f0ab870a43e376db8",
-                "sha256:1b7928bb7580736ffac5baf814097be342ba08d3cfdfb48e52773ec959572287",
-                "sha256:2032d9cb994ce3b4cba2b8dfae08c7e25bc14ba484c770d4d3be33c27de8c45b",
-                "sha256:20e7eeb1166087db636c06cae04a1ef59298627f56fb17da10528ab52a14c87f",
-                "sha256:216f5d7dbb67166759e59b0479bca82b8acf9bed6015b526b8eb10143fb08e77",
-                "sha256:28b119ba97129d3001673a697b7cce47fe6de1f7255d104c2f01108a5179a066",
-                "sha256:3104f4b084ad5d9c0cb87445cc8cfd96bba710bef4a66c2674910127044df209",
-                "sha256:3e6192dbcefaaa52ed81be88525a54a445f4b4fe2fffcae7fe40ebb58bd06bfd",
-                "sha256:42d4f97b9795a7aafa152a36fe2ad44549b83a743fd3e77011136def512e6c2a",
-                "sha256:44e706bac34e9f50779cb8c39f10b53a4d15aebb97235643d3112ac20bd577b4",
-                "sha256:47b11a729d61a47df56346283a4a800fa379ae6a85870d5a2e1e4956c828eedc",
-                "sha256:4854f9edc5208f63f0841c0c667260ae8d6846cfa233c479e29fdc85d42ebd58",
-                "sha256:48f721f070726cd2a6e44f3c33f8ee4b24188e4b816e6dd8ba542c8c3bb5b246",
-                "sha256:52afb0ac962963fff30cf1be775bc51ae083ef4c1e354266ab20e5382057dd62",
-                "sha256:54d8b9c5e288362ec8595c1d98666d36f2070fd0c2f76e2b3c60fbad9bd76227",
-                "sha256:5bd3d7dfd9cd058eb68d9a905dec854f86649f64d4ddf21f3ec289341386c44b",
-                "sha256:613010b5d17906c4367609e6f52e9a2595e35d5cc27d36ff3f1b6fa6e954d944",
-                "sha256:624321120f7e60336be8ec74a172ae7fba5c3ed5bf787cc85f7e9986c9e0ebc2",
-                "sha256:65c94410b5a8355cfcf12fd600a313efee46ce96a09e911ea92cf2acf6708804",
-                "sha256:6640f83df0ae4ae1104d4c62b77e9ef39be85ebe53f636388707d532bee2b7b8",
-                "sha256:687700f8371643916a1d2c61f3fdaa630407dd205c38afff936545d7b7466066",
-                "sha256:77c2713faf25a953c69cf0f723d1b7dd83827b0834e6c41e3fb3bbc6765914a1",
-                "sha256:78068e8678ca023594e4a0ab558905c1033b2d3e806a0ad9e3094e231e115a33",
-                "sha256:7a23ccc1083c260fa9685c93e3b170baba45aeed4b524deb3f426b0c40c11639",
-                "sha256:7abddb2bd5489d30ffeb4b93a428130886c171b4d355ccd226e83254fcb6b9ef",
-                "sha256:80093b595921eed1a2cead546a683b9e2ae7f4a4592bb2ab22f70d30174f003a",
-                "sha256:8242543c522d84d033fe79be04cb559b80d7eb98ad81b137ff7e0a9020f00ace",
-                "sha256:838812c65ed5f7c2bd11f7b098d2e5d01685a3f6d1f82849423b570bae698c00",
-                "sha256:83ea1a398f192957cb986d9206ce229efe0ee75e3c6635baff53ddf39bd718d5",
-                "sha256:8421aa8c9b45ea608c205db9e1c0c855c7e54d0e9c2c2f337ce024f6843cab3b",
-                "sha256:858375573c9225cc8e5b49bfac846a77b696b8d5e815711b8d4ba3141e6e8879",
-                "sha256:86de64468cad9c6d269f32a6390e210ca5ada568c7a55de8e681ca3b897bb340",
-                "sha256:87f7ac99b15270db8d53f28c3c7b968612993a90a5cf359da354efe96f5372b4",
-                "sha256:8bad8210ad4df68c44ff3685cca3cda448ee46e20d13edcff8909eba6ec01ca4",
-                "sha256:8bb4af15f305056e95ca1bd086239b9ebc6ad55e9f49076d27d80027f72752f6",
-                "sha256:8c78bfe20d4c890cb5580a3b9290f700c570e167d4cdcc55feec07030297a5e3",
-                "sha256:8f3f3154fde2b1ff3aa7b4f9326347ebc89c8ef425ca1db8f665175e6d3bd42f",
-                "sha256:94010bd61bc168c103a5b3b0f56ed3b616688192db7cd5b1d626e49f28ff51b3",
-                "sha256:941fab0073f0a54dc33d1a0460cb04e0d85893cb0c5e1476c785000f8b359409",
-                "sha256:9dca7c3956b03b7663fac4d150f5e6d4f6f38b2462c1e9afd83bcf7019f17913",
-                "sha256:a180dbd5ea5d47c2d3b716d5c19cc3fb162d1c8db93b21a1295d69585bfddac1",
-                "sha256:a2712aee7b3834ace51738c15d9ee152cc5a98dc7d57dd93300461b792ab7b43",
-                "sha256:a435ef8a3bd95c8a2d316d6e0ff70d0db524f6037411652803e118871d703333",
-                "sha256:abb756147314430bee5d10919b8493c0ccb109ddb7f5dfd2fcd7441266a25b75",
-                "sha256:abe6eb10122f0d746a0d510c2039ae8edb27bc9af29f6d1b05a66cc2401353ff",
-                "sha256:acbd0a6d61cc954b9f535daaa9ec26b0a60a0d4353c5f7c1438ebc88a359a47e",
-                "sha256:ae08ac90aa8fa14caafc7a6251bd218bf6dac518b7bff09caaa5e781119ba3f2",
-                "sha256:ae61446166983c663cee42c852ed63899e43e484abf080089f771df4b9d272ef",
-                "sha256:afe1f3bc486d0ce40abb0a0c9adb39aed3bbac36ebdc596487b0cceba55c21c1",
-                "sha256:b946da90dc2799bcafa682692c1d2139b2a96ec3c24fa9fc6f5b0da782675330",
-                "sha256:b947e264f0e77d30dcbccbb00f49f900b204b922eb0c3a9f0afd61aaa1cedc3d",
-                "sha256:bb5635c851eef3a7a54becde6da99485eecf7d068bd885ac8e6d173c4ecd68b0",
-                "sha256:bcbebd369493d68162cddb74a9c1fcebd139dfbb7ddb23d8f8e43e6c87bac3a6",
-                "sha256:c31805d2c8ade9b11feca4674eee2b9cce1fec3e8ddb7bbdd961a09dc76a80ea",
-                "sha256:c8840f064b1fb377cffd3efeaad2b190c14d4c8da02316dae07571252d20b31f",
-                "sha256:ccb94342d13e3bf3ffa6e62f95b5e3f0bc6bfa94558cb37f4b3d09d6feb536ff",
-                "sha256:d66689e840e75221b0b290b0befa86f059fb35e1ee6443bce51516d4d61b6b99",
-                "sha256:dabf1a05318d95b1537fd61d9330ef4313ea1216eea128a17615038859da3b3b",
-                "sha256:db03704b3506455d86ec72c3358a779e9b1d07b61220dfb43702b7b668edcd0d",
-                "sha256:de4217b9eb8b541cf2b7fde4401ce9d9a411cc0af85d410f9d6f4333f43640be",
-                "sha256:df0841f94928f8af9c7a1f0aaaffba1fb74607af023a152f59379c01c53aee58",
-                "sha256:dfb992dbcd88d8254471760879d48fb20836d91baa90f181c957122f9592b3dc",
-                "sha256:e7e66b4e403c2836ac74f26c4b65d8ac0ca1eef41dfcac2d013b7482befaad83",
-                "sha256:e8012bce6836d3f20a6c9599f81dfa945f433dab4dbd0c4917a6fb1f998ab33d",
-                "sha256:f01de4ec083daebf210531e2cca3bdb1608dbbbe00a9723e261d92087a1f6ebc",
-                "sha256:f0d945a85b70da97ae86113faf9f1b9294efe66bd4a5d6f82f2676d567338b66",
-                "sha256:fa0ae3275ef706c0309556061185dd0e4c4cd3b7d6f67ae617e4e677c7a41e2e"
+                "sha256:01fbfbeb8249a68d257f601deb50c70c929dc2dfe683b754659569e502fbd3aa",
+                "sha256:0270b49b6847f0d106d64b5086e9ad5dc8a902413b5dbbb15d12b60f9c1747a4",
+                "sha256:03c0ae165e700364b266876d712acb1ac02693acd920afa67da2ebb91a0b3c09",
+                "sha256:068ca17214038ae986d68f4a7021f97e187ed278ab6dccb79f837d765a54d753",
+                "sha256:082a2988364b60bb5de809373098361cf1dbb239623e39e46cb18bc035ed9c0c",
+                "sha256:0aaf982e68a7ac284377d051c742610220fd06d330dcd4c4dbb4cdd77c22a537",
+                "sha256:0c0991f5a96a8e620f7691e61178cd8f457b49e17b7d9cfa2067e2a0a89fc1d5",
+                "sha256:115f8359402fa527cf47708d6f8a0f8234f0e9ca0cab7c18c9c189c194dbf620",
+                "sha256:15c59e780be8f30a60816a9adab900c12a58d79c1ac742b4a8df044ab2a6d920",
+                "sha256:1b7d0e124948daa4d9686d421ef5087c0516bc6179fdcf8828b8444f8e461a77",
+                "sha256:1c8eb19abe87029c18f226d42b8a2c9efdd139d08f8bf6e085dd9075446db450",
+                "sha256:204e0f176fd1d067671157d049466869b3ae1fc51e354708b0dc41cf94e23a3a",
+                "sha256:2136f64fbb86451dbbf70223635a468272dd20075f988a102bf8a3f194a411dc",
+                "sha256:2b291d1230845871c00c8462c50565a9cd6026fe1228e77ca934470bb7d70ea0",
+                "sha256:2c18645ef6294d99b256806e34653e86236eb266278c8ec8112622b61db255de",
+                "sha256:2cc4e280098c1b192c42a849de8de2c8e0f3a84086a76ec5b07bfee29bda7d18",
+                "sha256:2ed8357f4c6e0daa4f3baf31832df8a33334e0fe5b020a61bc8b345a3db7a606",
+                "sha256:3191d312c73e3cfd0f0afdf51df8405aafeb0bad71e7ed8f68b24b63c4f36500",
+                "sha256:3401613148d93ef0fd9aabdbddb212de3db7a4475367f49f590c837355343972",
+                "sha256:34106f68e20e6ff253c9f596ea50397dbd8699828d55e8fa18bd4323d8d966e6",
+                "sha256:3516119f4f9b8671083a70b6afaa0a070f5683e431ab3dc26e9215620d7ca1ad",
+                "sha256:38ece17ec5f20d7d9b442e5174ae9f020365d01ba7c112205a4d59cf19dc38ee",
+                "sha256:3b4032a96410bdc760061b14ed6a33613ffb7f702181ba999df5d16fb96ba16a",
+                "sha256:3bf8b000a4e2967e6dfdd8656cd0757d18c7e5ce3d16339e550bd462f4857e59",
+                "sha256:3e3070e680f79887d60feeda051a58d0ac36622e1759f305a41059eff62c6da7",
+                "sha256:4496b1282c70c442809fc1b151977c3d967bfb33e4e17cedbf226d97de18f709",
+                "sha256:44dd6fc3034f1eaa72ece33588867df9e006a7303725a12d64c3dff92330f625",
+                "sha256:4adfbb5451196842a88fda3612e2c0414134874bffb1c2ce83ab4242ec9e027d",
+                "sha256:4b7c0c0b3244bb2275abe255d4a30c050d541c6cb18b870975553f1fb6f37527",
+                "sha256:4c82a6d952a1d555bf4be42b6532927d2a5686dd3c3e280e5f63225ab47ac1f5",
+                "sha256:5344b896e79800af86ad643408ca9aa303a017f6ebff8cee5a3163c1e9aec987",
+                "sha256:5bde86a2ed3ce587fa2b207424ce15b9a83a9fa14422dcc1c5356a13aed3df9d",
+                "sha256:5bf6c237f8c681dfb91b17f8435b2735951f0d1fad10cc5dfd96db110243370b",
+                "sha256:5dbb9c997932473a27afa93954bb77a9f9b786b4ccf718d903f35da3232317de",
+                "sha256:69ea9d6d9baa25a4dc9cef5e2b77b8537827b122214f210dd925132e34ae9b12",
+                "sha256:6b3146f9ae6af82c47a5282ac8803523d381b3b21caeae0327ed2f7ecb718798",
+                "sha256:6bcb34f869d431799c3ee7d516554797f7760cb2198ecaa89c3f176f72d062be",
+                "sha256:6ca08b840fe95d1c2bd9ab92dac5685f949fc6f9ae820ec16193e5ddf603c3b2",
+                "sha256:6ca7a9a06b52d0e38ccf6bca1aeff7be178917893f3883f37b75589d42c4ac20",
+                "sha256:703c60b9910488d3d0954ca585c34f541e506a091a41930e663a098d3b794c67",
+                "sha256:715bdf952b9533ba13dfcf1f431a8f49e63cecc31d91d007bc1deb914f47d0e4",
+                "sha256:72b67f966b57dbd18dcc7efbc1c7fc9f5f983e572db1877081f075004614fcdd",
+                "sha256:74423631b6be371edfbf7eabb02ab995c2563fee60a80a30829176842e71722a",
+                "sha256:77a85dca4c2430ac04dc2a2185c2deb3858a34fe7f403d0a946fa56970cf60a1",
+                "sha256:7821d44fe07335bea256b9f1f41474a642ca55fa671dfd9f00af8d68a920c2d4",
+                "sha256:788f15721c64109cf720791714dc14afd0f449d63f3a5487724f024345067381",
+                "sha256:7ca684ee649b55fd8f378127ac8462fb6c85f251c2fb027eb3c887e8ee347bcd",
+                "sha256:7daa3e1369355766dea11f1d8ef829905c3b9da886ea3152788dc25ee6079e02",
+                "sha256:7e6bc96ebe49604df3ec2c6389cc3876cabe475e6bfc84ced1bf4e630662cb35",
+                "sha256:80b12f25d805a919d53efc0a5ad7c0c0326f13b4eae981a5d7b7cc343318ebb7",
+                "sha256:871587bdadd1075b112e697173e946a07d722459d20716ceb3d1bd6c64bd08ce",
+                "sha256:88b88282e55fa39dd556d7fc04160bcf39dea015f78e0cecec8ff4f06c1fc2b5",
+                "sha256:8d7a498671ca87e32b54cb47c82a92b40130a26c5197d392720a1bce1b3c77cf",
+                "sha256:926838a535c2c1ea21c903f909a9a54e675c2126728c21381a94ddf37c3cbddf",
+                "sha256:971e8990c5cc4ddcff26e149398fc7b0f6a042306e82500f5e8db3b10ce69f84",
+                "sha256:9b273ecfbc590a1b98f014ae41e5cf723932f3b53ba9367cfb676f838038b32c",
+                "sha256:a42db008d58530efa3b881eeee4991146de0b790e095f7ae43ba5cc612decbc5",
+                "sha256:a72a84570f84c374b4c287183debc776dc319d3e8ce6b6a0041ce2e400de3f32",
+                "sha256:ac97a21de3712afe6a6c071abfad40a6224fd14fa6ff0ff8d0c6e6cd4e2f807a",
+                "sha256:acb704195a71ac5ea5ecf2811c9ee19ecdc62b91878528302dd0be1b9451cc90",
+                "sha256:b32bff85fb02a75ea0b68f21e2412255b5731f3f389ed9aecc13a6752f58ac97",
+                "sha256:b3cd31f859b662ac5d7f4226ec7d8bd60384fa037fc02aee6ff0b53ba29a3ba8",
+                "sha256:b63731993cdddcc8e087c64e9cf003f909262b359110070183d7f3025d1c56b5",
+                "sha256:b6907da3017ef55139cf0e417c5123a84c7332520e73a6902ff1f79046cd3b94",
+                "sha256:ba6e5e6588e49139a0979d03a7deb9c734bde647b9a8808f26acf9c547cab1bf",
+                "sha256:c1c8f2a2ca45292084c75bb6d3a25545cff0ed931ed228d3a1810ae3758f975f",
+                "sha256:ce828058d482ef860746bf532822842e0ff484e27f540ef5c813d516dd8896d2",
+                "sha256:d0a2d1bd63a4ad79483049b26514e70fa618ce6115220da9efdff63688808b17",
+                "sha256:d0cdde3c78d8ab5b46595054e5def32a755fc028685add5ddc7403e9f6de9879",
+                "sha256:d57dfbf9737763b3a60d26e6800e02e04284926329aee8fb01049635e957fe81",
+                "sha256:d8416c23161abd94cc7da80c734ad7c9f5dbebdadfdaa77dad78244457448223",
+                "sha256:dba7d9f2e047dfa2bca3b01f4f84aa5246725203d6284e3790f2ca15fba6b40a",
+                "sha256:dbf012d8fcb9f2cf0643b65df3b355fdd74fc0035d70bb5c845e9e30a3a4654b",
+                "sha256:e1258c639e00bf5e8a522fec6c3eaa3e30cf1c23a2f21a586be7e04d50c9acab",
+                "sha256:e222562dc0f38571c8b1ffdae9d7adb866363134299264a1958d077800b193b7",
+                "sha256:e4946d6bdb7ba972dfda282f9127e5756d4f299028b1566d1245fa0d438847e6",
+                "sha256:e746524418b70f38550f2190eeee834db8850088c834d4c8406fbb9bc1ae10b2",
+                "sha256:e76654e9dbfb835b3518f9938e565c7806976c07b37c33526b574cc1a1050480",
+                "sha256:e8918973fbd34e7814f59143c5f600ecd38b8038161239fd1a3d33d5817a38b8",
+                "sha256:e891ce81edd463b3b4c3b885c5603c00141151dd9c6936d98a680c8c72fe5c67",
+                "sha256:ebbbd0e728af5db9b04e56389e2299a57ea8b9dd15c9759153ee2455b32be6ad",
+                "sha256:eeb438a26d87c123bb318e5f2b3d86a36060b01f22fbdffd8cf247d52f7c9a2b",
+                "sha256:eed56b6a39216d31ff8cd2f1d048b5bf1700e4b32a01b14379c3b6dde9ce3aa3",
+                "sha256:f17cde1db0754c35a91ac00b22b25c11da6eec5746431d6e5092f0cd31a3fea9",
+                "sha256:f1a9b7d00fdf60b4039f4455afd031fe85ee8305b019334b72dcf73c567edc47",
+                "sha256:f4b6cecbbf3b7380f3b61de3a7b93cb721125dc125c854c14ddc91225ba52f83",
+                "sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad",
+                "sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==24.0.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==26.0.3"
         },
         "pyzstd": {
             "hashes": [
@@ -2948,11 +3288,11 @@
         },
         "qtconsole": {
             "hashes": [
-                "sha256:8c75fa3e9b4ed884880ff7cea90a1b67451219279ec33deaee1d59e3df1a5d2b",
-                "sha256:a0e806c6951db9490628e4df80caec9669b65149c7ba40f9bf033c025a5b56bc"
+                "sha256:42d745f3d05d36240244a04e1e1ec2a86d5d9b6edb16dbdef582ccb629e87e0b",
+                "sha256:6b5fb11274b297463706af84dcbbd5c92273b1f619e6d25d08874b0a88516989"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.5.1"
+            "version": "==5.5.2"
         },
         "qtpy": {
             "hashes": [
@@ -2995,18 +3335,18 @@
         },
         "referencing": {
             "hashes": [
-                "sha256:191e936b0c696d0af17ad7430a3dc68e88bc11be6514f4757dc890f04ab05889",
-                "sha256:8080727b30e364e5783152903672df9b6b091c926a146a759080b62ca3126cd6"
+                "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c",
+                "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.35.0"
+            "version": "==0.35.1"
         },
         "requests": {
             "hashes": [
                 "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
                 "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
             ],
-            "markers": "python_version >= '3.7'",
+            "index": "pypi",
             "version": "==2.31.0"
         },
         "rfc3339-validator": {
@@ -3062,108 +3402,108 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f",
-                "sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c",
-                "sha256:08231ac30a842bd04daabc4d71fddd7e6d26189406d5a69535638e4dcb88fe76",
-                "sha256:08f9ad53c3f31dfb4baa00da22f1e862900f45908383c062c27628754af2e88e",
-                "sha256:0ab39c1ba9023914297dd88ec3b3b3c3f33671baeb6acf82ad7ce883f6e8e157",
-                "sha256:0af039631b6de0397ab2ba16eaf2872e9f8fca391b44d3d8cac317860a700a3f",
-                "sha256:0b8612cd233543a3781bc659c731b9d607de65890085098986dfd573fc2befe5",
-                "sha256:11a8c85ef4a07a7638180bf04fe189d12757c696eb41f310d2426895356dcf05",
-                "sha256:1374f4129f9bcca53a1bba0bb86bf78325a0374577cf7e9e4cd046b1e6f20e24",
-                "sha256:1d4acf42190d449d5e89654d5c1ed3a4f17925eec71f05e2a41414689cda02d1",
-                "sha256:1d9a5be316c15ffb2b3c405c4ff14448c36b4435be062a7f578ccd8b01f0c4d8",
-                "sha256:1df3659d26f539ac74fb3b0c481cdf9d725386e3552c6fa2974f4d33d78e544b",
-                "sha256:22806714311a69fd0af9b35b7be97c18a0fc2826e6827dbb3a8c94eac6cf7eeb",
-                "sha256:2644e47de560eb7bd55c20fc59f6daa04682655c58d08185a9b95c1970fa1e07",
-                "sha256:2e6d75ab12b0bbab7215e5d40f1e5b738aa539598db27ef83b2ec46747df90e1",
-                "sha256:30f43887bbae0d49113cbaab729a112251a940e9b274536613097ab8b4899cf6",
-                "sha256:34b18ba135c687f4dac449aa5157d36e2cbb7c03cbea4ddbd88604e076aa836e",
-                "sha256:36b3ee798c58ace201289024b52788161e1ea133e4ac93fba7d49da5fec0ef9e",
-                "sha256:39514da80f971362f9267c600b6d459bfbbc549cffc2cef8e47474fddc9b45b1",
-                "sha256:39f5441553f1c2aed4de4377178ad8ff8f9d733723d6c66d983d75341de265ab",
-                "sha256:3a96e0c6a41dcdba3a0a581bbf6c44bb863f27c541547fb4b9711fd8cf0ffad4",
-                "sha256:3f26b5bd1079acdb0c7a5645e350fe54d16b17bfc5e71f371c449383d3342e17",
-                "sha256:41ef53e7c58aa4ef281da975f62c258950f54b76ec8e45941e93a3d1d8580594",
-                "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d",
-                "sha256:43fbac5f22e25bee1d482c97474f930a353542855f05c1161fd804c9dc74a09d",
-                "sha256:4457a94da0d5c53dc4b3e4de1158bdab077db23c53232f37a3cb7afdb053a4e3",
-                "sha256:465a3eb5659338cf2a9243e50ad9b2296fa15061736d6e26240e713522b6235c",
-                "sha256:482103aed1dfe2f3b71a58eff35ba105289b8d862551ea576bd15479aba01f66",
-                "sha256:4832d7d380477521a8c1644bbab6588dfedea5e30a7d967b5fb75977c45fd77f",
-                "sha256:4901165d170a5fde6f589acb90a6b33629ad1ec976d4529e769c6f3d885e3e80",
-                "sha256:5307def11a35f5ae4581a0b658b0af8178c65c530e94893345bebf41cc139d33",
-                "sha256:5417558f6887e9b6b65b4527232553c139b57ec42c64570569b155262ac0754f",
-                "sha256:56a737287efecafc16f6d067c2ea0117abadcd078d58721f967952db329a3e5c",
-                "sha256:586f8204935b9ec884500498ccc91aa869fc652c40c093bd9e1471fbcc25c022",
-                "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e",
-                "sha256:5ddcba87675b6d509139d1b521e0c8250e967e63b5909a7e8f8944d0f90ff36f",
-                "sha256:618a3d6cae6ef8ec88bb76dd80b83cfe415ad4f1d942ca2a903bf6b6ff97a2da",
-                "sha256:635dc434ff724b178cb192c70016cc0ad25a275228f749ee0daf0eddbc8183b1",
-                "sha256:661d25cbffaf8cc42e971dd570d87cb29a665f49f4abe1f9e76be9a5182c4688",
-                "sha256:66e6a3af5a75363d2c9a48b07cb27c4ea542938b1a2e93b15a503cdfa8490795",
-                "sha256:67071a6171e92b6da534b8ae326505f7c18022c6f19072a81dcf40db2638767c",
-                "sha256:685537e07897f173abcf67258bee3c05c374fa6fff89d4c7e42fb391b0605e98",
-                "sha256:69e64831e22a6b377772e7fb337533c365085b31619005802a79242fee620bc1",
-                "sha256:6b0817e34942b2ca527b0e9298373e7cc75f429e8da2055607f4931fded23e20",
-                "sha256:6c81e5f372cd0dc5dc4809553d34f832f60a46034a5f187756d9b90586c2c307",
-                "sha256:6d7faa6f14017c0b1e69f5e2c357b998731ea75a442ab3841c0dbbbfe902d2c4",
-                "sha256:6ef0befbb5d79cf32d0266f5cff01545602344eda89480e1dd88aca964260b18",
-                "sha256:6ef687afab047554a2d366e112dd187b62d261d49eb79b77e386f94644363294",
-                "sha256:7223a2a5fe0d217e60a60cdae28d6949140dde9c3bcc714063c5b463065e3d66",
-                "sha256:77f195baa60a54ef9d2de16fbbfd3ff8b04edc0c0140a761b56c267ac11aa467",
-                "sha256:793968759cd0d96cac1e367afd70c235867831983f876a53389ad869b043c948",
-                "sha256:7bd339195d84439cbe5771546fe8a4e8a7a045417d8f9de9a368c434e42a721e",
-                "sha256:7cd863afe7336c62ec78d7d1349a2f34c007a3cc6c2369d667c65aeec412a5b1",
-                "sha256:7f2facbd386dd60cbbf1a794181e6aa0bd429bd78bfdf775436020172e2a23f0",
-                "sha256:84ffab12db93b5f6bad84c712c92060a2d321b35c3c9960b43d08d0f639d60d7",
-                "sha256:8c8370641f1a7f0e0669ddccca22f1da893cef7628396431eb445d46d893e5cd",
-                "sha256:8db715ebe3bb7d86d77ac1826f7d67ec11a70dbd2376b7cc214199360517b641",
-                "sha256:8e8916ae4c720529e18afa0b879473049e95949bf97042e938530e072fde061d",
-                "sha256:8f03bccbd8586e9dd37219bce4d4e0d3ab492e6b3b533e973fa08a112cb2ffc9",
-                "sha256:8f2fc11e8fe034ee3c34d316d0ad8808f45bc3b9ce5857ff29d513f3ff2923a1",
-                "sha256:923d39efa3cfb7279a0327e337a7958bff00cc447fd07a25cddb0a1cc9a6d2da",
-                "sha256:93df1de2f7f7239dc9cc5a4a12408ee1598725036bd2dedadc14d94525192fc3",
-                "sha256:998e33ad22dc7ec7e030b3df701c43630b5bc0d8fbc2267653577e3fec279afa",
-                "sha256:99f70b740dc04d09e6b2699b675874367885217a2e9f782bdf5395632ac663b7",
-                "sha256:9a00312dea9310d4cb7dbd7787e722d2e86a95c2db92fbd7d0155f97127bcb40",
-                "sha256:9d54553c1136b50fd12cc17e5b11ad07374c316df307e4cfd6441bea5fb68496",
-                "sha256:9dbbeb27f4e70bfd9eec1be5477517365afe05a9b2c441a0b21929ee61048124",
-                "sha256:a1ce3ba137ed54f83e56fb983a5859a27d43a40188ba798993812fed73c70836",
-                "sha256:a34d557a42aa28bd5c48a023c570219ba2593bcbbb8dc1b98d8cf5d529ab1434",
-                "sha256:a5f446dd5055667aabaee78487f2b5ab72e244f9bc0b2ffebfeec79051679984",
-                "sha256:ad36cfb355e24f1bd37cac88c112cd7730873f20fb0bdaf8ba59eedf8216079f",
-                "sha256:aec493917dd45e3c69d00a8874e7cbed844efd935595ef78a0f25f14312e33c6",
-                "sha256:b316144e85316da2723f9d8dc75bada12fa58489a527091fa1d5a612643d1a0e",
-                "sha256:b34ae4636dfc4e76a438ab826a0d1eed2589ca7d9a1b2d5bb546978ac6485461",
-                "sha256:b34b7aa8b261c1dbf7720b5d6f01f38243e9b9daf7e6b8bc1fd4657000062f2c",
-                "sha256:bc362ee4e314870a70f4ae88772d72d877246537d9f8cb8f7eacf10884862432",
-                "sha256:bed88b9a458e354014d662d47e7a5baafd7ff81c780fd91584a10d6ec842cb73",
-                "sha256:c0013fe6b46aa496a6749c77e00a3eb07952832ad6166bd481c74bda0dcb6d58",
-                "sha256:c0b5dcf9193625afd8ecc92312d6ed78781c46ecbf39af9ad4681fc9f464af88",
-                "sha256:c4325ff0442a12113a6379af66978c3fe562f846763287ef66bdc1d57925d337",
-                "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7",
-                "sha256:c8362467a0fdeccd47935f22c256bec5e6abe543bf0d66e3d3d57a8fb5731863",
-                "sha256:cd5bf1af8efe569654bbef5a3e0a56eca45f87cfcffab31dd8dde70da5982475",
-                "sha256:cf1ea2e34868f6fbf070e1af291c8180480310173de0b0c43fc38a02929fc0e3",
-                "sha256:d62dec4976954a23d7f91f2f4530852b0c7608116c257833922a896101336c51",
-                "sha256:d68c93e381010662ab873fea609bf6c0f428b6d0bb00f2c6939782e0818d37bf",
-                "sha256:d7c36232a90d4755b720fbd76739d8891732b18cf240a9c645d75f00639a9024",
-                "sha256:dd18772815d5f008fa03d2b9a681ae38d5ae9f0e599f7dda233c439fcaa00d40",
-                "sha256:ddc2f4dfd396c7bfa18e6ce371cba60e4cf9d2e5cdb71376aa2da264605b60b9",
-                "sha256:e003b002ec72c8d5a3e3da2989c7d6065b47d9eaa70cd8808b5384fbb970f4ec",
-                "sha256:e32a92116d4f2a80b629778280103d2a510a5b3f6314ceccd6e38006b5e92dcb",
-                "sha256:e4461d0f003a0aa9be2bdd1b798a041f177189c1a0f7619fe8c95ad08d9a45d7",
-                "sha256:e541ec6f2ec456934fd279a3120f856cd0aedd209fc3852eca563f81738f6861",
-                "sha256:e546e768d08ad55b20b11dbb78a745151acbd938f8f00d0cfbabe8b0199b9880",
-                "sha256:ea7d4a99f3b38c37eac212dbd6ec42b7a5ec51e2c74b5d3223e43c811609e65f",
-                "sha256:ed4eb745efbff0a8e9587d22a84be94a5eb7d2d99c02dacf7bd0911713ed14dd",
-                "sha256:f8a2f084546cc59ea99fda8e070be2fd140c3092dc11524a71aa8f0f3d5a55ca",
-                "sha256:fcb25daa9219b4cf3a0ab24b0eb9a5cc8949ed4dc72acb8fa16b7e1681aa3c58",
-                "sha256:fdea4952db2793c4ad0bdccd27c1d8fdd1423a92f04598bc39425bcc2b8ee46e"
+                "sha256:05f3d615099bd9b13ecf2fc9cf2d839ad3f20239c678f461c753e93755d629ee",
+                "sha256:06d218939e1bf2ca50e6b0ec700ffe755e5216a8230ab3e87c059ebb4ea06afc",
+                "sha256:07f2139741e5deb2c5154a7b9629bc5aa48c766b643c1a6750d16f865a82c5fc",
+                "sha256:08d74b184f9ab6289b87b19fe6a6d1a97fbfea84b8a3e745e87a5de3029bf944",
+                "sha256:0abeee75434e2ee2d142d650d1e54ac1f8b01e6e6abdde8ffd6eeac6e9c38e20",
+                "sha256:154bf5c93d79558b44e5b50cc354aa0459e518e83677791e6adb0b039b7aa6a7",
+                "sha256:17c6d2155e2423f7e79e3bb18151c686d40db42d8645e7977442170c360194d4",
+                "sha256:1805d5901779662d599d0e2e4159d8a82c0b05faa86ef9222bf974572286b2b6",
+                "sha256:19ba472b9606c36716062c023afa2484d1e4220548751bda14f725a7de17b4f6",
+                "sha256:19e515b78c3fc1039dd7da0a33c28c3154458f947f4dc198d3c72db2b6b5dc93",
+                "sha256:1d54f74f40b1f7aaa595a02ff42ef38ca654b1469bef7d52867da474243cc633",
+                "sha256:207c82978115baa1fd8d706d720b4a4d2b0913df1c78c85ba73fe6c5804505f0",
+                "sha256:2625f03b105328729f9450c8badda34d5243231eef6535f80064d57035738360",
+                "sha256:27bba383e8c5231cd559affe169ca0b96ec78d39909ffd817f28b166d7ddd4d8",
+                "sha256:2c3caec4ec5cd1d18e5dd6ae5194d24ed12785212a90b37f5f7f06b8bedd7139",
+                "sha256:2cc7c1a47f3a63282ab0f422d90ddac4aa3034e39fc66a559ab93041e6505da7",
+                "sha256:2fc24a329a717f9e2448f8cd1f960f9dac4e45b6224d60734edeb67499bab03a",
+                "sha256:312fe69b4fe1ffbe76520a7676b1e5ac06ddf7826d764cc10265c3b53f96dbe9",
+                "sha256:32b7daaa3e9389db3695964ce8e566e3413b0c43e3394c05e4b243a4cd7bef26",
+                "sha256:338dee44b0cef8b70fd2ef54b4e09bb1b97fc6c3a58fea5db6cc083fd9fc2724",
+                "sha256:352a88dc7892f1da66b6027af06a2e7e5d53fe05924cc2cfc56495b586a10b72",
+                "sha256:35b2b771b13eee8729a5049c976197ff58a27a3829c018a04341bcf1ae409b2b",
+                "sha256:38e14fb4e370885c4ecd734f093a2225ee52dc384b86fa55fe3f74638b2cfb09",
+                "sha256:3c20f05e8e3d4fc76875fc9cb8cf24b90a63f5a1b4c5b9273f0e8225e169b100",
+                "sha256:3dd3cd86e1db5aadd334e011eba4e29d37a104b403e8ca24dcd6703c68ca55b3",
+                "sha256:489bdfe1abd0406eba6b3bb4fdc87c7fa40f1031de073d0cfb744634cc8fa261",
+                "sha256:48c2faaa8adfacefcbfdb5f2e2e7bdad081e5ace8d182e5f4ade971f128e6bb3",
+                "sha256:4a98a1f0552b5f227a3d6422dbd61bc6f30db170939bd87ed14f3c339aa6c7c9",
+                "sha256:4adec039b8e2928983f885c53b7cc4cda8965b62b6596501a0308d2703f8af1b",
+                "sha256:4e0ee01ad8260184db21468a6e1c37afa0529acc12c3a697ee498d3c2c4dcaf3",
+                "sha256:51584acc5916212e1bf45edd17f3a6b05fe0cbb40482d25e619f824dccb679de",
+                "sha256:531796fb842b53f2695e94dc338929e9f9dbf473b64710c28af5a160b2a8927d",
+                "sha256:5463c47c08630007dc0fe99fb480ea4f34a89712410592380425a9b4e1611d8e",
+                "sha256:5c45a639e93a0c5d4b788b2613bd637468edd62f8f95ebc6fcc303d58ab3f0a8",
+                "sha256:6031b25fb1b06327b43d841f33842b383beba399884f8228a6bb3df3088485ff",
+                "sha256:607345bd5912aacc0c5a63d45a1f73fef29e697884f7e861094e443187c02be5",
+                "sha256:618916f5535784960f3ecf8111581f4ad31d347c3de66d02e728de460a46303c",
+                "sha256:636a15acc588f70fda1661234761f9ed9ad79ebed3f2125d44be0862708b666e",
+                "sha256:673fdbbf668dd958eff750e500495ef3f611e2ecc209464f661bc82e9838991e",
+                "sha256:6afd80f6c79893cfc0574956f78a0add8c76e3696f2d6a15bca2c66c415cf2d4",
+                "sha256:6b5ff7e1d63a8281654b5e2896d7f08799378e594f09cf3674e832ecaf396ce8",
+                "sha256:6c4c4c3f878df21faf5fac86eda32671c27889e13570645a9eea0a1abdd50922",
+                "sha256:6cd8098517c64a85e790657e7b1e509b9fe07487fd358e19431cb120f7d96338",
+                "sha256:6d1e42d2735d437e7e80bab4d78eb2e459af48c0a46e686ea35f690b93db792d",
+                "sha256:6e30ac5e329098903262dc5bdd7e2086e0256aa762cc8b744f9e7bf2a427d3f8",
+                "sha256:70a838f7754483bcdc830444952fd89645569e7452e3226de4a613a4c1793fb2",
+                "sha256:720edcb916df872d80f80a1cc5ea9058300b97721efda8651efcd938a9c70a72",
+                "sha256:732672fbc449bab754e0b15356c077cc31566df874964d4801ab14f71951ea80",
+                "sha256:740884bc62a5e2bbb31e584f5d23b32320fd75d79f916f15a788d527a5e83644",
+                "sha256:7700936ef9d006b7ef605dc53aa364da2de5a3aa65516a1f3ce73bf82ecfc7ae",
+                "sha256:7732770412bab81c5a9f6d20aeb60ae943a9b36dcd990d876a773526468e7163",
+                "sha256:7750569d9526199c5b97e5a9f8d96a13300950d910cf04a861d96f4273d5b104",
+                "sha256:7f1944ce16401aad1e3f7d312247b3d5de7981f634dc9dfe90da72b87d37887d",
+                "sha256:81c5196a790032e0fc2464c0b4ab95f8610f96f1f2fa3d4deacce6a79852da60",
+                "sha256:8352f48d511de5f973e4f2f9412736d7dea76c69faa6d36bcf885b50c758ab9a",
+                "sha256:8927638a4d4137a289e41d0fd631551e89fa346d6dbcfc31ad627557d03ceb6d",
+                "sha256:8c7672e9fba7425f79019db9945b16e308ed8bc89348c23d955c8c0540da0a07",
+                "sha256:8d2e182c9ee01135e11e9676e9a62dfad791a7a467738f06726872374a83db49",
+                "sha256:910e71711d1055b2768181efa0a17537b2622afeb0424116619817007f8a2b10",
+                "sha256:942695a206a58d2575033ff1e42b12b2aece98d6003c6bc739fbf33d1773b12f",
+                "sha256:9437ca26784120a279f3137ee080b0e717012c42921eb07861b412340f85bae2",
+                "sha256:967342e045564cef76dfcf1edb700b1e20838d83b1aa02ab313e6a497cf923b8",
+                "sha256:998125738de0158f088aef3cb264a34251908dd2e5d9966774fdab7402edfab7",
+                "sha256:9e6934d70dc50f9f8ea47081ceafdec09245fd9f6032669c3b45705dea096b88",
+                "sha256:a3d456ff2a6a4d2adcdf3c1c960a36f4fd2fec6e3b4902a42a384d17cf4e7a65",
+                "sha256:a7b28c5b066bca9a4eb4e2f2663012debe680f097979d880657f00e1c30875a0",
+                "sha256:a888e8bdb45916234b99da2d859566f1e8a1d2275a801bb8e4a9644e3c7e7909",
+                "sha256:aa3679e751408d75a0b4d8d26d6647b6d9326f5e35c00a7ccd82b78ef64f65f8",
+                "sha256:aaa71ee43a703c321906813bb252f69524f02aa05bf4eec85f0c41d5d62d0f4c",
+                "sha256:b646bf655b135ccf4522ed43d6902af37d3f5dbcf0da66c769a2b3938b9d8184",
+                "sha256:b906b5f58892813e5ba5c6056d6a5ad08f358ba49f046d910ad992196ea61397",
+                "sha256:b9bb1f182a97880f6078283b3505a707057c42bf55d8fca604f70dedfdc0772a",
+                "sha256:bd1105b50ede37461c1d51b9698c4f4be6e13e69a908ab7751e3807985fc0346",
+                "sha256:bf18932d0003c8c4d51a39f244231986ab23ee057d235a12b2684ea26a353590",
+                "sha256:c273e795e7a0f1fddd46e1e3cb8be15634c29ae8ff31c196debb620e1edb9333",
+                "sha256:c69882964516dc143083d3795cb508e806b09fc3800fd0d4cddc1df6c36e76bb",
+                "sha256:c827576e2fa017a081346dce87d532a5310241648eb3700af9a571a6e9fc7e74",
+                "sha256:cbfbea39ba64f5e53ae2915de36f130588bba71245b418060ec3330ebf85678e",
+                "sha256:ce0bb20e3a11bd04461324a6a798af34d503f8d6f1aa3d2aa8901ceaf039176d",
+                "sha256:d0cee71bc618cd93716f3c1bf56653740d2d13ddbd47673efa8bf41435a60daa",
+                "sha256:d21be4770ff4e08698e1e8e0bce06edb6ea0626e7c8f560bc08222880aca6a6f",
+                "sha256:d31dea506d718693b6b2cffc0648a8929bdc51c70a311b2770f09611caa10d53",
+                "sha256:d44607f98caa2961bab4fa3c4309724b185b464cdc3ba6f3d7340bac3ec97cc1",
+                "sha256:d58ad6317d188c43750cb76e9deacf6051d0f884d87dc6518e0280438648a9ac",
+                "sha256:d70129cef4a8d979caa37e7fe957202e7eee8ea02c5e16455bc9808a59c6b2f0",
+                "sha256:d85164315bd68c0806768dc6bb0429c6f95c354f87485ee3593c4f6b14def2bd",
+                "sha256:d960de62227635d2e61068f42a6cb6aae91a7fe00fca0e3aeed17667c8a34611",
+                "sha256:dc48b479d540770c811fbd1eb9ba2bb66951863e448efec2e2c102625328e92f",
+                "sha256:e1735502458621921cee039c47318cb90b51d532c2766593be6207eec53e5c4c",
+                "sha256:e2be6e9dd4111d5b31ba3b74d17da54a8319d8168890fbaea4b9e5c3de630ae5",
+                "sha256:e4c39ad2f512b4041343ea3c7894339e4ca7839ac38ca83d68a832fc8b3748ab",
+                "sha256:ed402d6153c5d519a0faf1bb69898e97fb31613b49da27a84a13935ea9164dfc",
+                "sha256:ee17cd26b97d537af8f33635ef38be873073d516fd425e80559f4585a7b90c43",
+                "sha256:f3027be483868c99b4985fda802a57a67fdf30c5d9a50338d9db646d590198da",
+                "sha256:f5bab211605d91db0e2995a17b5c6ee5edec1270e46223e513eaa20da20076ac",
+                "sha256:f6f8e3fecca256fefc91bb6765a693d96692459d7d4c644660a9fff32e517843",
+                "sha256:f7afbfee1157e0f9376c00bb232e80a60e59ed716e3211a80cb8506550671e6e",
+                "sha256:fa242ac1ff583e4ec7771141606aafc92b361cd90a05c30d93e343a0c2d82a89",
+                "sha256:fab6ce90574645a0d6c58890e9bcaac8d94dff54fb51c69e5522a7358b80ab64"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.18.0"
+            "version": "==0.18.1"
         },
         "rtree": {
             "hashes": [
@@ -3542,11 +3882,11 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5",
-                "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"
+                "sha256:23097a41eba115ba99ecae40d06444c15d1c0c698d527a01c6c8bd1c5d0647e5",
+                "sha256:4f41d54107ff9a223dca80b53efe4fb654c67efaba7f47bada3ee9d50e05bd53"
             ],
             "index": "pypi",
-            "version": "==4.65.0"
+            "version": "==4.66.3"
         },
         "traitlets": {
             "hashes": [
@@ -3612,11 +3952,11 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:0846377ea76e818daaa3e00a4365c018bc3ac9760cbb3544de542885aad61fb3",
-                "sha256:ec25a9671a5102c8d2657f62792a27b48f016664c6873f6beed3800008577210"
+                "sha256:604bfdceaeece392802e6ae48e69cec49168b9c5f4a44e483963f9242eb0e78b",
+                "sha256:7aa9982a728ae5892558bff6a2839c00b9ed145523ece2274fad6f414690ae75"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.26.0"
+            "version": "==20.26.1"
         },
         "wcwidth": {
             "hashes": [
@@ -3842,6 +4182,102 @@
             ],
             "version": "==0.6.2"
         },
+        "yarl": {
+            "hashes": [
+                "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51",
+                "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce",
+                "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559",
+                "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0",
+                "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81",
+                "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc",
+                "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4",
+                "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c",
+                "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130",
+                "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136",
+                "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e",
+                "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec",
+                "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7",
+                "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1",
+                "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455",
+                "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099",
+                "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129",
+                "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10",
+                "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142",
+                "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98",
+                "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa",
+                "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7",
+                "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525",
+                "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c",
+                "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9",
+                "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c",
+                "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8",
+                "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b",
+                "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf",
+                "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23",
+                "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd",
+                "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27",
+                "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f",
+                "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece",
+                "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434",
+                "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec",
+                "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff",
+                "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78",
+                "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d",
+                "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863",
+                "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53",
+                "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31",
+                "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15",
+                "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5",
+                "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b",
+                "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57",
+                "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3",
+                "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1",
+                "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f",
+                "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad",
+                "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c",
+                "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7",
+                "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2",
+                "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b",
+                "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2",
+                "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b",
+                "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9",
+                "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be",
+                "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e",
+                "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984",
+                "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4",
+                "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074",
+                "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2",
+                "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392",
+                "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91",
+                "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541",
+                "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf",
+                "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572",
+                "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66",
+                "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575",
+                "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14",
+                "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5",
+                "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1",
+                "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e",
+                "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551",
+                "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17",
+                "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead",
+                "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0",
+                "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe",
+                "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234",
+                "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0",
+                "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7",
+                "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34",
+                "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42",
+                "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385",
+                "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78",
+                "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be",
+                "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958",
+                "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749",
+                "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.9.4"
+        },
         "ypy-websocket": {
             "hashes": [
                 "sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff",
@@ -3850,6 +4286,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.8.4"
         },
+        "zarr": {
+            "hashes": [
+                "sha256:7f8532b6a3f50f22e809e130e09353637ec8b5bb5e95a5a0bfaae91f63978b5d",
+                "sha256:c3b7d2c85b8a42b0ad0ad268a36fb6886ca852098358c125c6b126a417e0a598"
+            ],
+            "index": "pypi",
+            "version": "==2.18.0"
+        },
         "zipp": {
             "hashes": [
                 "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b",
@@ -3867,13 +4311,6 @@
             ],
             "version": "==2.4.1"
         },
-        "backcall": {
-            "hashes": [
-                "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e",
-                "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"
-            ],
-            "version": "==0.2.0"
-        },
         "decorator": {
             "hashes": [
                 "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330",
@@ -3882,6 +4319,14 @@
             "markers": "python_version >= '3.5'",
             "version": "==5.1.1"
         },
+        "exceptiongroup": {
+            "hashes": [
+                "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad",
+                "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==1.2.1"
+        },
         "executing": {
             "hashes": [
                 "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147",
@@ -3892,11 +4337,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:07232af52a5ba146dc3372c7bf52a0f890a23edf38d77caef8d53f9cdc2584c1",
-                "sha256:7468edaf4f6de3e1b912e57f66c241e6fd3c7099f2ec2136e239e142e800274d"
+                "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501",
+                "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.23.0"
+            "version": "==8.24.0"
         },
         "jedi": {
             "hashes": [
@@ -3930,13 +4375,6 @@
             "markers": "sys_platform != 'win32' and sys_platform != 'emscripten'",
             "version": "==4.9.0"
         },
-        "pickleshare": {
-            "hashes": [
-                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
-                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
-            ],
-            "version": "==0.7.5"
-        },
         "prompt-toolkit": {
             "hashes": [
                 "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
@@ -3962,11 +4400,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
-                "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
+                "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199",
+                "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.17.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.18.0"
         },
         "six": {
             "hashes": [
@@ -3991,6 +4429,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==5.14.3"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
+                "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==4.11.0"
+        },
         "wcwidth": {
             "hashes": [
                 "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859",
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3d15f1533..1b4c56608 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.5.2.1 - 2024-05-21 - [PR#1172](https://github.com/NOAA-OWP/inundation-mapping/pull/1172)
+
+Removes loading of `apache-arrow` repository from the Dockerfile where it was causing a GPG key error during `docker build`.
+
+A number of python packages were updated in this PR. You will need to build a new Docker image for this release.
+
+### Changes
+
+- Dockerfile: Adds a line remove the loading of apache-arrow during `apt-get update`.
+
+<br/><br/>
+
+
 ## v4.5.2.0 - 2024-05-20 - [PR#1166](https://github.com/NOAA-OWP/inundation-mapping/pull/1166)
 
 The main goal of this PR is to create bridge point data that be used as a service in HydroVIS. Since every branch processes bridges separately, it's possible to inundate a bridge from more than just the feature_id it crosses. To reflect this, the `osm_bridge_centroids.gpkg` now found in HUC directories will have coincident points - one that is inundated from the reach it crosses and the other a backwater-influenced point indicated by the `is_backwater` field.
@@ -22,7 +36,6 @@ The main goal of this PR is to create bridge point data that be used as a servic
 
 This hotfix addresses the issue #1162 by explicitly using 'fiona' engine for reading gpkg files with Boolean dtype. This is applicable only for `usgs_gages.gpkg` and `usgs_subset_gages.gpkg` files. 
 
-
 ### Changes
 - `src/usgs_gage_unit_setup.py`  ... changed only two lines for fiona engine
 - `src/usgs_gage_crosswalk.py` ...  changed only one line for fiona engine + two small changes to use `self.branch_id` for the correct log report
@@ -30,7 +43,6 @@ This hotfix addresses the issue #1162 by explicitly using 'fiona' engine for rea
 
 <br/><br/>
 
-
 ## v4.5.1.2 - 2024-05-17 - [PR#1135](https://github.com/NOAA-OWP/inundation-mapping/pull/1135)
 
 Updates USGS gage processing to use the correct projection (determined by whether the HUC is in Alaska or not).
@@ -57,7 +69,7 @@ Extends flows (i.e., discharge) to stream segments missing from NWS and USGS val
 
 <br/><br/>
 
-## v4.5.1.0 - 2024-05-17 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150)
+## v4.5.1.0 - 2024-05-17 - [PR#1156](https://github.com/NOAA-OWP/inundation-mapping/pull/1156)
 
 This focuses on removing hydro-conditioning artifacts by subtracting the thalweg DEM from HAND REM and adding back the original DEM. Also, a new tool was created to test this feature over multiple HUCs
 
@@ -68,7 +80,6 @@ This focuses on removing hydro-conditioning artifacts by subtracting the thalweg
 - `src/delineate_hydros_and_produce_HAND.sh`: Removing hydro-conditioning artifacts from HAND REM.
 - `config/params_template.env`: Creating an option to include/exclude healed HAND from FIM pipeline.
 
-
 <br/><br/>
 
 ## v4.5.0.2 - 2024-05-17 - [PR#1159](https://github.com/NOAA-OWP/inundation-mapping/pull/1159)
@@ -85,7 +96,6 @@ Also, In this revised version, I've eliminated the code sections that previously
 
 Similar to 'usgs_gages.gpkg' dataset, all lid stations, including those in Alaska, are stored in a single gpkg file (`nws_lid.gpkg`) with EPSG=5070. The Alaska stations can be identified using their HUC8 numbers (beginning with '19'). 
 
-
 ### Changes
 - tools/generate_nws_lid.py
 
@@ -144,7 +154,6 @@ Some NWM streams, particularly in coastal areas, fail to reach the edge of the D
 
 <br/><br/>
 
-
 ## v4.4.15.4 - 2024-05-06 - [PR#1115](https://github.com/NOAA-OWP/inundation-mapping/pull/1115)
 
 This PR addresses issue #1040 and includes the following updates:
@@ -172,10 +181,8 @@ Fixes a KeyError in `src/mitigate_branch_outlet_backpool.py`.
 
 `src/mitigate_branch_outlet_backpool.py`: Addresses case where `catchments_df['outlier']` are all False.
 
-
 <br/><br/>
 
-
 ## v4.4.15.2 - 2024-05-06 - [PR#1133](https://github.com/NOAA-OWP/inundation-mapping/pull/1133)
 
 Bug fix for error when reading the subfolders of a directory using `listdir()` where files exist that start with an 8-digit number that are later interpreted as directories.
@@ -230,7 +237,6 @@ Adds checks for intermediate files produced by Whitebox in the AGREE process (`s
 
 <br/><br/>
 
-
 ## v4.4.14.0 - 2024-04-17 - [PR#1106](https://github.com/NOAA-OWP/inundation-mapping/pull/10106)
 
 Updates the FIM pipeline so it can process HUCs in southern Alaska. Running FIM in southern Alaska requires that a different CRS and a few different files be used. Additionally, some of the Alaska HUCs displayed an issue where the input stream density was too high, so this update introduces some logic to adjust the threshold of stream orders to exclude based on whether an Alaska HUC is listed as high or medium-high stream density. This update intriduces new Alaska-specific inputs, which are listed in the PR. 
@@ -269,7 +275,6 @@ A small update to the README.md was also updated for an unrelated topic (about A
 
 <br/><br/>
 
-
 ## v4.4.13.2 - 2024-04-04 - [PR#1110](https://github.com/NOAA-OWP/inundation-mapping/pull/1110)
 
 This PR reflects upgrades for openJDK from 17.0.8 to something higher, minimum of 17.0.9. After some research, we can not upgrade all the way to the latest openJDK but can jump up to 19.0.  This limitation is related to version of our base docker image.  openJDK was identified as requiring an upgrade by a system wide security scan.
@@ -286,7 +291,6 @@ The "black" packages is also be upgraded from 23.7.0 to 24.3.
 
 <br/><br/>
 
-
 ## v4.4.13.1 - 2024-03-11 - [PR#1086](https://github.com/NOAA-OWP/inundation-mapping/pull/1086)
 
 Fixes bug where levee-protected areas were not being masked from branch 0 DEMs.
@@ -318,7 +322,6 @@ If both criteria are met for a branch, then the issue is mitigated by trimming t
 
 <br/><br/>
 
-
 ## v4.4.12.0 - 2024-03-11 - [PR#1078](https://github.com/NOAA-OWP/inundation-mapping/pull/1078)
 
 Resolves issue #1033 by adding Alaska-specific data to the FIM input folders and updating the pre-clip vector process to use the proper data and CRS when an Alaska HUC is detected. The `-wbd` flag was removed from the optional arguments of `generate_pre_clip_fim_huc8`. The WBD file path will now only be sourced from the `bash_variables.env` file. The `bash_variables.env` file has been updated to include the new Alaska-specific FIM input files.
@@ -338,7 +341,6 @@ Resolves issue #1033 by adding Alaska-specific data to the FIM input folders and
 
 <br/><br/>
 
-
 ## v4.4.11.1 - 2024-03-08 - [PR#1080](https://github.com/NOAA-OWP/inundation-mapping/pull/1080)
 
 Fixes bug in bathymetric adjustment where `mask` is used with `geopandas.read_file`. The solution is to force `read_file` to use `fiona` instead of `pyogrio`.
@@ -383,7 +385,6 @@ Tested that the a non zero return exit from pre-processing shuts down the AWS st
 
 <br/><br/>
 
-
 ## v4.4.10.0 - 2024-02-02 - [PR#1054](https://github.com/NOAA-OWP/inundation-mapping/pull/1054)
 
 Recent testing exposed a bug with the `acquire_and_preprocess_3dep_dems.py` script. It lost the ability to be re-run and look for files that were unsuccessful earlier attempts and try them again. It may have been lost due to confusion of the word "retry". Now "retry" means restart the entire run. A new flag called "repair"  has been added meaning fix what failed earlier.  This is a key feature it is common for communication failures when calling USGS to download DEMs.  And with some runs taking many hours, this feature becomes important.
@@ -396,7 +397,6 @@ Also used the opportunity to fix a couple of other minor issues:
 
 ### Changes
 
-
 - `data\usgs\`
     - `acquire_and_preprocess_3dep_dems.py`: Re-add a feature which allowed for restarting and redo missing outputs or partial outputs. System now named as a "repair" system.
 - `fim_pipeline.sh`:  remove the parallel `--eta` flag to reduce logging. It was not needed, also removed "isaws" flag.
@@ -623,7 +623,6 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1
 
 <br/><br/>
 
-
 ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
 
 During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times.
@@ -1172,10 +1171,8 @@ Fixes a bug in CatFIM script where a bracket was missing on a pandas `concat` st
 ### Changes
 - `/tools/generate_categorical_fim.py`: fixes `concat` statement where bracket was missing.
 
-
 <br/><br/>
 
-
 ## v4.3.11.2 - 2023-05-19 - [PR#918](https://github.com/NOAA-OWP/inundation-mapping/pull/918)
 
 This fix addresses a bug that was preventing `burn_in_levees.py` from running. The if statement in run_unit_wb.sh preceeding `burn_in_levees.py` was checking for the existence of a filepath that doesn't exist.

From 5b9a8ce465218099ef46199401de6b8d88639219 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 14 Jun 2024 15:20:36 -0400
Subject: [PATCH 325/359] v4.5.2.2 Upgrade whitebox to v2.3.4 (#1183)

---
 Dockerfile        |  10 +-
 Pipfile           |   4 +-
 Pipfile.lock      | 604 ++++++++++++++++++++++++----------------------
 docs/CHANGELOG.md |  11 +
 4 files changed, 327 insertions(+), 302 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 96f6b2d3d..db1ecf72b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -110,11 +110,11 @@ RUN pip3 install pipenv==2023.12.1 && PIP_NO_CACHE_DIR=off pipenv install --syst
 # We download and unzip it to the same file folder that pip deployed the whitebox library.
 # Whitebox also attempts to always download a folder called testdata regardless of use.
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
-RUN wbox_path=/usr/local/lib/python3.10/dist-packages/whitebox/ && \
-    wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
-    unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
-    cp $wbox_path/WBT/whitebox_tools $wbox_path && \
-    mkdir $wbox_path/testdata
+# RUN wbox_path=/usr/local/lib/python3.10/dist-packages/whitebox/WBT && \
+#     wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
+#     unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
+#     cp $wbox_path/whitebox_tools $wbox_path && \
+#     mkdir $wbox_path/testdata
 # ----------------------------------
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
diff --git a/Pipfile b/Pipfile
index b4e5981b2..6ba0dc993 100644
--- a/Pipfile
+++ b/Pipfile
@@ -30,7 +30,7 @@ jupyter = "==1.0.0"
 jupyterlab = "==3.6.7"
 ipympl = "==0.9.3"
 pytest = "==7.3.0"
-whitebox = "==2.3.1"
+whitebox = "2.3.4"
 shapely = "==2.0.1"
 pyarrow = "==14.0.1"
 rtree = "==1.0.1"
@@ -50,7 +50,7 @@ openpyxl = "==3.1.2"
 osmnx = "==1.9.3"
 lmoments3 = "==1.0.6"
 zarr = "==2.18.0"
-requests = "==2.31.0"
+requests = "==2.32.3"
 aiohttp = "==3.9.5"
 
 [requires]
diff --git a/Pipfile.lock b/Pipfile.lock
index c3758ef76..3c1bc0455 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "b1097cae3089688432b1d87cf34f1f1877327168066113e534a986fd0a9fa6fa"
+            "sha256": "898973145b92f3e3eb9afcaf3cee8bd5da693fbd8ae06046e66d3a73022c27a5"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -112,6 +112,7 @@
                 "sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==3.9.5"
         },
         "aiosignal": {
@@ -132,11 +133,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8",
-                "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"
+                "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94",
+                "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.3.0"
+            "version": "==4.4.0"
         },
         "appdirs": {
             "hashes": [
@@ -230,7 +231,7 @@
                 "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051",
                 "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==4.12.3"
         },
         "black": {
@@ -259,6 +260,7 @@
                 "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==24.3.0"
         },
         "bleach": {
@@ -293,7 +295,7 @@
                 "sha256:f206e6a01a8167b441bf886ff022eb20e0f085b09300f49f3018f566c14d918a",
                 "sha256:f465b8ab54ecde6b8654672a50a4c3699aafd8e2de0dfcd84ed53f8a34c1734a"
             ],
-            "markers": "python_version >= '3.8' and python_version < '4.0'",
+            "markers": "python_version >= '3.8' and python_version < '4'",
             "version": "==2.0.0"
         },
         "boto3": {
@@ -302,6 +304,7 @@
                 "sha256:d388cb7f54f1a3056f91ffcfb5cf18b226454204e5df7a5c10774718c3fbb166"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==1.26.109"
         },
         "botocore": {
@@ -415,6 +418,7 @@
                 "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==2023.7.22"
         },
         "cffi": {
@@ -485,30 +489,39 @@
         },
         "cftime": {
             "hashes": [
-                "sha256:022dabf1610cdd04a693e730fa8f71d307059717f29dba921e7486e553412bb4",
-                "sha256:0a38eb9f5c733a23e1714bd3ef2762ed5acee34f127670f8fb4ad6464946f6b3",
-                "sha256:2d113a01ab924445e61d65c26bbd95bc08e4a22878d3b947064bba056c884c4a",
-                "sha256:3b86be8c2f254147be4ba88f12099466dde457a4a3a21de6c69d52a7224c13ae",
-                "sha256:3cf6e216a4c06f9a628cdf8e9c9d5e8097fb3eb02dd087dd14ab3b18478a7271",
-                "sha256:4d6fbd5f41b322cfa7b0ac3aaadeceb4450100a164b5bccbbb9e7c5048489a88",
-                "sha256:523b9a6bf03f5e36407979e248381d0fcab2d225b915bbde77d00c6dde192b90",
-                "sha256:5f11685663a6af97418908060492a07663c16d42519c139ca03c2ffb1377fd25",
-                "sha256:80eb1170ce1639016f55760847f4aadd04b0312496c5bac2797e930914bba48d",
-                "sha256:8a14d2c7d22fd2a6dfa6ad563283b6d6679f1df95e0ed8d14b8f284dad402887",
-                "sha256:8d2c01456d9d7b46aa710a41d1c711a50d5ea259aff4a987d0e973d1093bc922",
-                "sha256:9eb177a02db7cd84aa6962278e4bd2d3106a545de82e6aacd9404f1e153661db",
-                "sha256:a98abb1d46d118e52b0611ce668a0b714b407be26177ef0581ecf5e95f894725",
-                "sha256:b62d42546fa5c914dfea5b15a9aaed2087ea1211cc36d08c374502ef95892038",
-                "sha256:bbf782ab4ac0605bdec2b941952c897595613203942b7f8c2fccd17efa5147df",
-                "sha256:bedb577bc8b8f3f10f5336c0792e5dae88605781890f50f36b45bb46907968e8",
-                "sha256:d0a6b29f72a13f08e008b9becff247cc75c84acb213332ede18879c5b6aa4dfd",
-                "sha256:d87dadd0824262bdd7493babd2a44447da0a22175ded8ae9e060a3aebec7c5d7",
-                "sha256:d9b00c2844c7a1701d8ede5336b6321dfee256ceab81a34a1aff0483d56891a6",
-                "sha256:eb6dd70b2ccabfe1a14b7fbb0bbdce0418e71697094373c0d573c880790fa291",
-                "sha256:f9878bfd8c1c3f24184ecbd528f739ba46ebaceaf1c8a24d348d7befb117a285"
+                "sha256:03494e7b66a2fbb6b04e364ab67185130dee0ced660abac5c1559070571d143d",
+                "sha256:0b47bf25195fb3889bbae34df0e80957eb69c48f66902f5d538c7a8ec34253f6",
+                "sha256:0edeb1cb019d8155b2513cffb96749c0d7d459370e69bdf03077e0bee214aed8",
+                "sha256:1289e08617be350a6b26c6e4352a0cb088625ac33d25e95110df549c26d6ab8e",
+                "sha256:18b132d9225b4a109929866200846c72302316db9069e2de3ec8d8ec377f567f",
+                "sha256:25f043703e785de0bd7cd8222c0a53317e9aeb3dfc062588b05e6f3ebb007468",
+                "sha256:3c349a91fa7ac9ec50118b04a8746bdea967bd2fc525d87c776003040b8d3392",
+                "sha256:4dcb2a01d4e614437582af33b36db4fb441b7666758482864827a1f037d2b639",
+                "sha256:567574df94d0de1101bb5da76e7fbc6eabfddeeb2eb8cf83286b3599a136bbf7",
+                "sha256:588d073400798adc24ece759cd1cb24ef730f55d1f70e31a898e7686f9d763d8",
+                "sha256:5b5ad7559a16bedadb66af8e417b6805f758acb57aa38d2730844dfc63a1e667",
+                "sha256:6e07b91b488570573bbeb6f815656a8974d13d15b2279c82de2927f4f692bbcd",
+                "sha256:6fc82928cbf477bebf233f41914e64bff7b9e894c7f0c34170784a48250f8da7",
+                "sha256:76b8f1e5d1e424accdf760a43e0a1793a7b640bab83cb067273d5c9dbb336c44",
+                "sha256:90609b3c1a31a756a68ecdbc961a4ce0b22c1620f166c8dabfa3a4c337ac8b9e",
+                "sha256:a8f05d5d6bb4137f9783fa61ad330030fcea8dcc6946dea69a27774edbe480e7",
+                "sha256:b32ac1278a2a111b066d5a1e6e5ce6f38c4c505993a6a3130873b56f99d7b56f",
+                "sha256:bbe11ad73b2a0ddc79995da21459fc2a3fd6b1593ca73f00a60e4d81c3e230f3",
+                "sha256:c05a71389f53d6340cb365b60f028c08268c72401660b9ef76108dee9f1cb5b2",
+                "sha256:c072fe9e09925af66a9473edf5752ca1890ba752e7c1935d1f0245ad48f0977c",
+                "sha256:c1558d9b477bd29626cd8bfc89e736635f72887d1a993e2834ab579bba7abb8c",
+                "sha256:c20f03e12af39534c3450bba376272803bfb850b5ce6433c839bfaa99f8d835a",
+                "sha256:ca1a264570e68fbb611bba251641b8efd0cf88c0ad2dcab5fa784df264232b75",
+                "sha256:d4f2cc0d5c6ffba9c5b0fd1ecd0c7c1c426d0be7b8de1480e2a9fb857c1905e9",
+                "sha256:e325406193758a7ed67308deb52e727782a19e384e183378e7ff62098be0aedc",
+                "sha256:e5456fd58d4cc6b8d7b4932b749617ee142b62a52bc5d8e3c282ce69ce3a20ba",
+                "sha256:e8467b6fbf8dbfe0be8c04d61180765fdd3b9ab0fe51313a0bbf87e63634a3d8",
+                "sha256:ee70074df4bae0d9ee98f201cf5f11fd302791cf1cdeb73c34f685d6b632e17d",
+                "sha256:f92f2e405eeda47b30ab6231d8b7d136a55f21034d394f93ade322d356948654",
+                "sha256:f9acc272df1022f24fe7dbe9de43fa5d8271985161df14549e4d8d28c90dc9ea"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.6.3"
+            "version": "==1.6.4"
         },
         "charset-normalizer": {
             "hashes": [
@@ -603,7 +616,7 @@
                 "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
                 "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_full_version >= '3.7.0'",
             "version": "==3.3.2"
         },
         "click": {
@@ -626,7 +639,7 @@
                 "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27",
                 "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4.0'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'",
             "version": "==0.7.2"
         },
         "cloudpickle": {
@@ -793,11 +806,11 @@
         },
         "datashader": {
             "hashes": [
-                "sha256:317c16a32a7d1c5208444a83fd6d2cb68d9bfb81802c323489bd24f6b7cf5c5f",
-                "sha256:5b2f1dd448defce206ecc16e9960fe9378f32307ea1fcb0c298c22c14f58c881"
+                "sha256:7899979b4c1adba6b4fd2e86caa3f5ef94c4e6ab234cbb7306ca6bfe243fc4df",
+                "sha256:c12ce9c71983a3f74ccba9d9416c270d1ed0a4e6c7bdcc6b63a2121ada0e00cd"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.16.1"
+            "version": "==0.16.2"
         },
         "debugpy": {
             "hashes": [
@@ -899,11 +912,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f",
-                "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"
+                "sha256:58a2549afdf9e02e10720eaa4d4470f56386d7a6f72edd7d0596337af8ed7ad8",
+                "sha256:71b3102950e91dfc1bb4209b64be4dc8854f40e5f534428d8684f953ac847fac"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.14.0"
+            "version": "==3.15.1"
         },
         "fiona": {
             "hashes": [
@@ -936,6 +949,7 @@
                 "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181"
             ],
             "index": "pypi",
+            "markers": "python_full_version >= '3.8.1'",
             "version": "==6.0.0"
         },
         "flake8-pyproject": {
@@ -943,6 +957,7 @@
                 "sha256:6249fe53545205af5e76837644dc80b4c10037e73a0e5db87ff562d75fb5bd4a"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==1.2.3"
         },
         "flox": {
@@ -955,51 +970,51 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:0118ef998a0699a96c7b28457f15546815015a2710a1b23a7bf6c1be60c01636",
-                "sha256:0d145976194a5242fdd22df18a1b451481a88071feadf251221af110ca8f00ce",
-                "sha256:0e19bd9e9964a09cd2433a4b100ca7f34e34731e0758e13ba9a1ed6e5468cc0f",
-                "sha256:0f08c901d3866a8905363619e3741c33f0a83a680d92a9f0e575985c2634fcc1",
-                "sha256:1250e818b5f8a679ad79660855528120a8f0288f8f30ec88b83db51515411fcc",
-                "sha256:15c94eeef6b095831067f72c825eb0e2d48bb4cea0647c1b05c981ecba2bf39f",
-                "sha256:1621ee57da887c17312acc4b0e7ac30d3a4fb0fec6174b2e3754a74c26bbed1e",
-                "sha256:180194c7fe60c989bb627d7ed5011f2bef1c4d36ecf3ec64daec8302f1ae0716",
-                "sha256:278e50f6b003c6aed19bae2242b364e575bcb16304b53f2b64f6551b9c000e15",
-                "sha256:32b17504696f605e9e960647c5f64b35704782a502cc26a37b800b4d69ff3c77",
-                "sha256:3bee3f3bd9fa1d5ee616ccfd13b27ca605c2b4270e45715bd2883e9504735034",
-                "sha256:4060acc2bfa2d8e98117828a238889f13b6f69d59f4f2d5857eece5277b829ba",
-                "sha256:54dcf21a2f2d06ded676e3c3f9f74b2bafded3a8ff12f0983160b13e9f2fb4a7",
-                "sha256:56fc244f2585d6c00b9bcc59e6593e646cf095a96fe68d62cd4da53dd1287b55",
-                "sha256:599bdb75e220241cedc6faebfafedd7670335d2e29620d207dd0378a4e9ccc5a",
-                "sha256:5f6bc991d1610f5c3bbe997b0233cbc234b8e82fa99fc0b2932dc1ca5e5afec0",
-                "sha256:60a3409c9112aec02d5fb546f557bca6efa773dcb32ac147c6baf5f742e6258b",
-                "sha256:68b3fb7775a923be73e739f92f7e8a72725fd333eab24834041365d2278c3671",
-                "sha256:76f1777d8b3386479ffb4a282e74318e730014d86ce60f016908d9801af9ca2a",
-                "sha256:806e7912c32a657fa39d2d6eb1d3012d35f841387c8fc6cf349ed70b7c340039",
-                "sha256:84d7751f4468dd8cdd03ddada18b8b0857a5beec80bce9f435742abc9a851a74",
-                "sha256:865a58b6e60b0938874af0968cd0553bcd88e0b2cb6e588727117bd099eef836",
-                "sha256:8ac27f436e8af7779f0bb4d5425aa3535270494d3bc5459ed27de3f03151e4c2",
-                "sha256:8b4850fa2ef2cfbc1d1f689bc159ef0f45d8d83298c1425838095bf53ef46308",
-                "sha256:8b5ad456813d93b9c4b7ee55302208db2b45324315129d85275c01f5cb7e61a2",
-                "sha256:8e2f1a4499e3b5ee82c19b5ee57f0294673125c65b0a1ff3764ea1f9db2f9ef5",
-                "sha256:9696fe9f3f0c32e9a321d5268208a7cc9205a52f99b89479d1b035ed54c923f1",
-                "sha256:96a48e137c36be55e68845fc4284533bda2980f8d6f835e26bca79d7e2006438",
-                "sha256:a8feca65bab31479d795b0d16c9a9852902e3a3c0630678efb0b2b7941ea9c74",
-                "sha256:aefa011207ed36cd280babfaa8510b8176f1a77261833e895a9d96e57e44802f",
-                "sha256:b2b92381f37b39ba2fc98c3a45a9d6383bfc9916a87d66ccb6553f7bdd129097",
-                "sha256:b3c61423f22165541b9403ee39874dcae84cd57a9078b82e1dce8cb06b07fa2e",
-                "sha256:b5b48a1121117047d82695d276c2af2ee3a24ffe0f502ed581acc2673ecf1037",
-                "sha256:c18b49adc721a7d0b8dfe7c3130c89b8704baf599fb396396d07d4aa69b824a1",
-                "sha256:c5b8cab0c137ca229433570151b5c1fc6af212680b58b15abd797dcdd9dd5051",
-                "sha256:c7e91abdfae1b5c9e3a543f48ce96013f9a08c6c9668f1e6be0beabf0a569c1b",
-                "sha256:cadf4e12a608ef1d13e039864f484c8a968840afa0258b0b843a0556497ea9ed",
-                "sha256:dc0673361331566d7a663d7ce0f6fdcbfbdc1f59c6e3ed1165ad7202ca183c68",
-                "sha256:de7c29bdbdd35811f14493ffd2534b88f0ce1b9065316433b22d63ca1cd21f14",
-                "sha256:e9d9298be7a05bb4801f558522adbe2feea1b0b103d5294ebf24a92dd49b78e5",
-                "sha256:ee1af4be1c5afe4c96ca23badd368d8dc75f611887fb0c0dac9f71ee5d6f110e",
-                "sha256:f7e89853d8bea103c8e3514b9f9dc86b5b4120afb4583b57eb10dfa5afbe0936"
+                "sha256:099634631b9dd271d4a835d2b2a9e042ccc94ecdf7e2dd9f7f34f7daf333358d",
+                "sha256:0c555e039d268445172b909b1b6bdcba42ada1cf4a60e367d68702e3f87e5f64",
+                "sha256:1e677bfb2b4bd0e5e99e0f7283e65e47a9814b0486cb64a41adf9ef110e078f2",
+                "sha256:2367d47816cc9783a28645bc1dac07f8ffc93e0f015e8c9fc674a5b76a6da6e4",
+                "sha256:28d072169fe8275fb1a0d35e3233f6df36a7e8474e56cb790a7258ad822b6fd6",
+                "sha256:31f0e3147375002aae30696dd1dc596636abbd22fca09d2e730ecde0baad1d6b",
+                "sha256:3e0ad3c6ea4bd6a289d958a1eb922767233f00982cf0fe42b177657c86c80a8f",
+                "sha256:45b4afb069039f0366a43a5d454bc54eea942bfb66b3fc3e9a2c07ef4d617380",
+                "sha256:4a2a6ba400d386e904fd05db81f73bee0008af37799a7586deaa4aef8cd5971e",
+                "sha256:4f520d9ac5b938e6494f58a25c77564beca7d0199ecf726e1bd3d56872c59749",
+                "sha256:52a6e0a7a0bf611c19bc8ec8f7592bdae79c8296c70eb05917fd831354699b20",
+                "sha256:5a4788036201c908079e89ae3f5399b33bf45b9ea4514913f4dbbe4fac08efe0",
+                "sha256:6b4f04b1fbc01a3569d63359f2227c89ab294550de277fd09d8fca6185669fa4",
+                "sha256:715b41c3e231f7334cbe79dfc698213dcb7211520ec7a3bc2ba20c8515e8a3b5",
+                "sha256:73121a9b7ff93ada888aaee3985a88495489cc027894458cb1a736660bdfb206",
+                "sha256:74ae2441731a05b44d5988d3ac2cf784d3ee0a535dbed257cbfff4be8bb49eb9",
+                "sha256:7d6166192dcd925c78a91d599b48960e0a46fe565391c79fe6de481ac44d20ac",
+                "sha256:7f193f060391a455920d61684a70017ef5284ccbe6023bb056e15e5ac3de11d1",
+                "sha256:907fa0b662dd8fc1d7c661b90782ce81afb510fc4b7aa6ae7304d6c094b27bce",
+                "sha256:93156dd7f90ae0a1b0e8871032a07ef3178f553f0c70c386025a808f3a63b1f4",
+                "sha256:93bc9e5aaa06ff928d751dc6be889ff3e7d2aa393ab873bc7f6396a99f6fbb12",
+                "sha256:95db0c6581a54b47c30860d013977b8a14febc206c8b5ff562f9fe32738a8aca",
+                "sha256:973d030180eca8255b1bce6ffc09ef38a05dcec0e8320cc9b7bcaa65346f341d",
+                "sha256:9cd7a6beec6495d1dffb1033d50a3f82dfece23e9eb3c20cd3c2444d27514068",
+                "sha256:9fe9096a60113e1d755e9e6bda15ef7e03391ee0554d22829aa506cdf946f796",
+                "sha256:a209d2e624ba492df4f3bfad5996d1f76f03069c6133c60cd04f9a9e715595ec",
+                "sha256:a239afa1126b6a619130909c8404070e2b473dd2b7fc4aacacd2e763f8597fea",
+                "sha256:ba9f09ff17f947392a855e3455a846f9855f6cf6bec33e9a427d3c1d254c712f",
+                "sha256:bb7273789f69b565d88e97e9e1da602b4ee7ba733caf35a6c2affd4334d4f005",
+                "sha256:bd5bc124fae781a4422f61b98d1d7faa47985f663a64770b78f13d2c072410c2",
+                "sha256:bff98816cb144fb7b85e4b5ba3888a33b56ecef075b0e95b95bcd0a5fbf20f06",
+                "sha256:c4ee5a24e281fbd8261c6ab29faa7fd9a87a12e8c0eed485b705236c65999109",
+                "sha256:c93ed66d32de1559b6fc348838c7572d5c0ac1e4a258e76763a5caddd8944002",
+                "sha256:d1a24f51a3305362b94681120c508758a88f207fa0a681c16b5a4172e9e6c7a9",
+                "sha256:d8f191a17369bd53a5557a5ee4bab91d5330ca3aefcdf17fab9a497b0e7cff7a",
+                "sha256:daaef7390e632283051e3cf3e16aff2b68b247e99aea916f64e578c0449c9c68",
+                "sha256:e40013572bfb843d6794a3ce076c29ef4efd15937ab833f520117f8eccc84fd6",
+                "sha256:eceef49f457253000e6a2d0f7bd08ff4e9fe96ec4ffce2dbcb32e34d9c1b8161",
+                "sha256:ee595d7ba9bba130b2bec555a40aafa60c26ce68ed0cf509983e0f12d88674fd",
+                "sha256:ef50ec31649fbc3acf6afd261ed89d09eb909b97cc289d80476166df8438524d",
+                "sha256:fa1f3e34373aa16045484b4d9d352d4c6b5f9f77ac77a178252ccbc851e8b2ee",
+                "sha256:fca66d9ff2ac89b03f5aa17e0b21a97c21f3491c46b583bb131eb32c7bab33af"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.51.0"
+            "version": "==4.53.0"
         },
         "fqdn": {
             "hashes": [
@@ -1093,11 +1108,11 @@
         },
         "fsspec": {
             "hashes": [
-                "sha256:918d18d41bf73f0e2b261824baeb1b124bcf771767e3a26425cd7dec3332f512",
-                "sha256:f39780e282d7d117ffb42bb96992f8a90795e4d0fb0f661a70ca39fe9c43ded9"
+                "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee",
+                "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.3.1"
+            "version": "==2024.6.0"
         },
         "geocube": {
             "hashes": [
@@ -1121,6 +1136,7 @@
                 "sha256:748af035d4a068a4ae00cab384acb61d387685c833b0022e0729aa45216b23ac"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.9'",
             "version": "==0.14.3"
         },
         "geopy": {
@@ -1137,6 +1153,7 @@
                 "sha256:d9f26aa88f164b77a47cb7d7eafc1641e48abe4152501657cb173c44dc84a8fd"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==0.2.3"
         },
         "identify": {
@@ -1248,11 +1265,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501",
-                "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3"
+                "sha256:53eee7ad44df903a06655871cbab66d156a051fd86f3ec6750470ac9604ac1ab",
+                "sha256:c6ed726a140b6e725b911528f80439c534fac915246af3efc39440a6b0f9d716"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.24.0"
+            "version": "==8.25.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1263,11 +1280,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:bbe43850d79fb5e906b14801d6c01402857996864d1e5b6fa62dd2ee35559f60",
-                "sha256:d0b9b41e49bae926a866e613a39b0f0097745d2b9f1f3dd406641b4a57ec42c9"
+                "sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2",
+                "sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.2"
+            "version": "==8.1.3"
         },
         "isoduration": {
             "hashes": [
@@ -1282,6 +1299,7 @@
                 "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"
             ],
             "index": "pypi",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==5.12.0"
         },
         "jedi": {
@@ -1326,10 +1344,10 @@
         },
         "jsonpointer": {
             "hashes": [
-                "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a",
-                "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"
+                "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942",
+                "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"
             ],
-            "version": "==2.4"
+            "version": "==3.0.0"
         },
         "jsonschema": {
             "extras": [
@@ -1393,11 +1411,11 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:659154cea512083434fd7c93b7fe0897af7a2fd0b9dd4749282b42eaac4ae677",
-                "sha256:fb6be52c713e80e004fac34b35a0990d6d36ba06fd0a2b2ed82b899143a64210"
+                "sha256:12558d158ec7a0653bf96cc272bc7ad79e0127d503b982ed144399346694f726",
+                "sha256:16f7177c3a4ea8fe37784e2d31271981a812f0b2874af17339031dc3510cc2a5"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.14.0"
+            "version": "==2.14.1"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1437,6 +1455,7 @@
                 "sha256:d92d57d402f53922bca5090654843aa08e511290dff29fdb0809eafbbeb6df98"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==3.6.7"
         },
         "jupyterlab-pygments": {
@@ -1449,19 +1468,19 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:097b5ac709b676c7284ac9c5e373f11930a561f52cd5a86e4fc7e5a9c8a8631d",
-                "sha256:f5e26156e5258b24d532c84e7c74cc212e203bff93eb856f81c24c16daeecc75"
+                "sha256:15cbb349dc45e954e09bacf81b9f9bcb10815ff660fb2034ecd7417db3a7ea27",
+                "sha256:54aa2d64fd86383b5438d9f0c032f043c4d8c0264b8af9f60bd061157466ea43"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.27.1"
+            "version": "==2.27.2"
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:04f2ac04976727e4f9d0fa91cdc2f1ab860f965e504c29dbd6a65c882c9d04c0",
-                "sha256:dd61f3ae7a5a7f80299e14585ce6cf3d6925a96c9103c978eda293197730cb64"
+                "sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0",
+                "sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.10"
+            "version": "==3.0.11"
         },
         "kiwisolver": {
             "hashes": [
@@ -1613,6 +1632,7 @@
                 "sha256:80e1397f1e3e5d1aa471ba67e49e557e7b3a67c7f7b5086c5fc8febcca6969f7"
             ],
             "index": "pypi",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==1.0.6"
         },
         "locket": {
@@ -1978,22 +1998,23 @@
                 "sha256:d583bc9050dd10538de36297c960b93f873f0cd01671a3c50df5bd86dd391dcb"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==8.3.1"
         },
         "nbclassic": {
             "hashes": [
-                "sha256:0ae11eb2319455d805596bf320336cda9554b41d99ab9a3c31bf8180bffa30e3",
-                "sha256:f99e4769b4750076cd4235c044b61232110733322384a94a63791d2e7beacc66"
+                "sha256:77b77ba85f9e988f9bad85df345b514e9e64c7f0e822992ab1df4a78ac64fc1e",
+                "sha256:8c0fd6e36e320a18657ff44ed96c3a400f17a903a3744fc322303a515778f2ba"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.0.0"
+            "version": "==1.1.0"
         },
         "nbclient": {
             "hashes": [
                 "sha256:4b3f1b7dba531e498449c4db4f53da339c91d449dc11e9af3a43b4eb5c5abb09",
                 "sha256:f13e3529332a1f1f81d82a53210322476a168bb7090a0289c795fe9cc11c9d3f"
             ],
-            "markers": "python_version >= '3.8'",
+            "markers": "python_full_version >= '3.8.0'",
             "version": "==0.10.0"
         },
         "nbconvert": {
@@ -2051,6 +2072,7 @@
                 "sha256:f94a89db78f34fdf68342840efb064fe1474310e8359dffce42e90a9ddf88f2f"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==1.6.3"
         },
         "networkx": {
@@ -2063,11 +2085,11 @@
         },
         "nodeenv": {
             "hashes": [
-                "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2",
-                "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"
+                "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f",
+                "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
-            "version": "==1.8.0"
+            "version": "==1.9.1"
         },
         "notebook": {
             "hashes": [
@@ -2117,6 +2139,7 @@
                 "sha256:fcdf84ba3ed8124eb7234adfbb8792f311991cbf8aed1cad4b1b1a7ee08380c1"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.56.4"
         },
         "numcodecs": {
@@ -2213,6 +2236,7 @@
                 "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.23.5"
         },
         "numpy-groupies": {
@@ -2237,6 +2261,7 @@
                 "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==3.1.2"
         },
         "osmnx": {
@@ -2245,6 +2270,7 @@
                 "sha256:ac67bea77b521941af648ef641ae1d006101948d1112475c256ea23ef31b426a"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.9.3"
         },
         "overrides": {
@@ -2257,11 +2283,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
-                "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
+                "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
+                "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==24.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==24.1"
         },
         "pandas": {
             "hashes": [
@@ -2292,6 +2318,7 @@
                 "sha256:f908a77cbeef9bbd646bd4b81214cbef9ac3dda4181d5092a4aa9797d1bc7774"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==2.0.2"
         },
         "pandera": {
@@ -2423,15 +2450,16 @@
                 "sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==10.3.0"
         },
         "platformdirs": {
             "hashes": [
-                "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf",
-                "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"
+                "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee",
+                "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.2.1"
+            "version": "==4.2.2"
         },
         "pluggy": {
             "hashes": [
@@ -2447,6 +2475,7 @@
                 "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==3.3.3"
         },
         "prometheus-client": {
@@ -2459,11 +2488,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
-                "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
+                "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10",
+                "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.0.43"
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==3.0.47"
         },
         "psutil": {
             "hashes": [
@@ -2553,6 +2582,7 @@
                 "sha256:ffe9dc0a884a8848075e576c1de0290d85a533a9f6e9c4e564f19adf8f6e54a7"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==2.9.6"
         },
         "ptyprocess": {
@@ -2583,6 +2613,7 @@
                 "sha256:94d0c24217f6582741813ee94490a4ca82bd5f9bf35e4f8610cb588cf7445764"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.20.4"
         },
         "pyarrow": {
@@ -2625,6 +2656,7 @@
                 "sha256:fada8396bc739d958d0b81d291cfd201126ed5e7913cb73de6bc606befc30226"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==14.0.1"
         },
         "pybcj": {
@@ -2792,6 +2824,7 @@
                 "sha256:be059dcade8bde51102e7e832903e76bec391b8bb460ca571895f511f4c18e4d"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.9'",
             "version": "==0.5.8"
         },
         "pygments": {
@@ -2831,6 +2864,7 @@
                 "sha256:f2ff58184020da39540a2f5d4a5412005a01b0c4cd03c7b8294bc670d1f3fe50"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==0.7.2"
         },
         "pyparsing": {
@@ -2964,6 +2998,7 @@
                 "sha256:fde5ece4d2436b5a57c8f5f97b49b5de06a856d03959f836c957d3e609f2de7e"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==3.5.0"
         },
         "pytest": {
@@ -2972,6 +3007,7 @@
                 "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==7.3.0"
         },
         "python-dateutil": {
@@ -2988,6 +3024,7 @@
                 "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.0.0"
         },
         "python-json-logger": {
@@ -3158,133 +3195,98 @@
         },
         "pyzstd": {
             "hashes": [
-                "sha256:013b206c22b85512d59dd05f6684c02bf9db45e93f637facb5ce165c2447fbc4",
-                "sha256:015a6794196fcb75ddaaae8186f0dbdcbda4e84451361a808b3e96c8c8a63296",
-                "sha256:0561710b88d4895a7bb202335e802125da6a71c556b1292fd620224b2652f496",
-                "sha256:0691e54fb29a4d1f4f2a0064afaf5d1525824bf139d83f403e5ba5ee630ac4f5",
-                "sha256:0779ccd652113d75cd461fa6581149550b0da55e72b1a6e28e6f49e9978c2d41",
-                "sha256:13515a6abca6751482396b656e5b72279bd405fd6c71b5bc899fcb526d40e564",
-                "sha256:150a7c7004b9df00f7cbb6dad0bdbab5d77c8c54b0857b8b3e50beae2810a041",
-                "sha256:18167c8a16709631eaa2f4fd27e2a7570b4c37ec460b65ff098e948af2d37f33",
-                "sha256:1c3d0e784ea9b66b06c7b76895aa4c1a90975980b1ae136fa8bbd8fb2b93e9b3",
-                "sha256:1e045dc688bb618cf2a2e33cb487c0d8af5e31a4eaec18c9399b34e2aed58777",
-                "sha256:24d061047634971801649ff9aced68bb8403c6220b07c82aa2c0e90d8a61acd0",
-                "sha256:2e6fd189c85f2efe94d3945dfbb85cdbc4e213757be75aa9a45e186625498ddf",
-                "sha256:330062103d25e0d67681fcf5ef8692c71cd979dc17ed98f6cee3b3ba8bd86622",
-                "sha256:330d54a0e5886fa5ca21e3960b9352a7b21f8dd85f98d4b7c4e8a7d48ade6998",
-                "sha256:3443df2dcdfe6e57c2600617e8f377772703b7f907fe065b49948c05ace80fd7",
-                "sha256:357e6545203754e83a0431663aa86cae873abe160923bdf3db07e88f03100b05",
-                "sha256:39a912430c73aa585ede2dc7c9f09c5334862953f26a9a86ac048c3805ed6bb1",
-                "sha256:3aa0ad903da666cf88d8a993e5fe9b1bfa0e5792a53212335af0a89baa268911",
-                "sha256:3b9105a4977247ea8664b113345e66bf4b3ffd321a07797efc6b020ff363b39e",
-                "sha256:3f174c47c370641c5dba74d4fd99374ca0eadd47e43e7f76ee8b76dacdfa2406",
-                "sha256:4094ea7b80081bb024eb887c80eeb6bd8ae24dc2862cc61ae9eadb5bca11dcd6",
-                "sha256:41e0da402bf0d68067be2f16f65b9d1d7d7e6d726f27762226cba7a8badd618a",
-                "sha256:455d2258f247968d18a9eaeb11e214e048acc976a61f72a0282d4c40051458b8",
-                "sha256:468f12eb61cbef394ff412c86d8bc85b90f24dff19f675c864df88a1a398a3a3",
-                "sha256:49c99120f9ea0c512019948e55ba17e60dadb1d3578890d53b2a09858da4e1dd",
-                "sha256:4a5b54ebec1e606821192c86a3b85c05c3f789d33bf44f383fd245f148f8f93b",
-                "sha256:4cd0bd21e0c7721e9f4ea54d533d9196771ab1d6ae0a83a96564dc46147a26eb",
-                "sha256:4dfd0d97ac231c08551543d61d123cf44bdd2fc20ba5ebce1fb0c466036be15a",
-                "sha256:4ea6a794b8a4e0590bea1599e724d4877d34640058caf5c0b3f916ded06fc3a1",
-                "sha256:4ff57d2a67aaff47eb60c37c5476af2a5056319aa2a5cab4b276f78b764be785",
-                "sha256:50079bcfd087c8a8822c9fe940c887244cf2276ca0a4f0b55bec006b564b9541",
-                "sha256:50186fa41c72929d21727dcc6cb87bef0adf09df9e982325a0ae5bfd20a9456b",
-                "sha256:502ec85cb2c0a35350a196e25d5e1350d400102706eba05d7926eadb64ffa976",
-                "sha256:51c644157319e2ca87521bd9694df7f651aac3273e015f35cdfb47cb8597ce37",
-                "sha256:527bfd3e01b8dafa1c24cba1a0455a85bbcdb41a3eefe59e2849634f9206d4f0",
-                "sha256:53172f0781b3db9321a1286bb274121668c3fe560d3bddfa82f585475f7145fa",
-                "sha256:5494515f3d0df1eb54c9f4ce7b9ea0b285bc1ba87eba295604da5f8de711bdea",
-                "sha256:5942e8b7ad869ea5f44d928053d3eda16e877b5fed5326786388f6a377545ab3",
-                "sha256:5ae5f8ab4accc980fa8e6696a402176e20a717ab2b296cd8d62b6a86cafc5f9c",
-                "sha256:6023de5cc800c073cbf6a10a7aad0cbae0c1849c759bb5263a78d9ac774b167b",
-                "sha256:609cc4bed06942e51ebdfa9cc74629e1d19de2339161b65d555d01b9820aec46",
-                "sha256:6217f30c4341619c6066a044ca118f53a3121b5813a56257b1f9818e24450584",
-                "sha256:62a1614558478f7d1014eec94a9ae3bf21c95766ecbe0a4a1ab16d2f1e4a6b67",
-                "sha256:62b8c3801d739a9592c0503dd294fab7f8cd2b1d637429babe13671bedf7d6f7",
-                "sha256:63a90494aeea75d3298941b0b8ddd505cbb54d6b915e579368db21d93454e41d",
-                "sha256:656107ef6ff0966df2a58828f165a17b0b40977358512ade259c52c18bff0a13",
-                "sha256:65e99e57f5fb42ed391fc2f7ffa4b8cce5a32dfd74f52e03bd36bb2ac3b56536",
-                "sha256:6aed97ef51e20ea4fd7e12c4b9e3199cd8e4664054f5d84c3d83e6f4d3144055",
-                "sha256:6bfb81f36fd56664dbbd64a0af241797424c76861faa2682aca89de110fce5d4",
-                "sha256:6d6f7fcfef9166670df4b6710d1105b891efba68ec523f56c64cc8c7aa6c8e88",
-                "sha256:6e742a691f4fa73cadbe7f866b33e33161ede5bc777eee443a62504adcadd65f",
-                "sha256:6ed32af20eeeb7f8307b091fdd0ee8f5f8733dde7e8f560a3115c98a6ac33a61",
-                "sha256:6fa6495ec52ff29b097e6d1b619588ab61e998b198882ec622d947856d4a85b4",
-                "sha256:793077870191d54889cc472eda3bfd84c8cae5ed61c9fa9fb2ce29c31e23f812",
-                "sha256:7b2ac847e8d2ca55be038d29d20d6bfe86c03e66ac2d9741701e4f9eef2abda9",
-                "sha256:7bb791f16c810c98865c373202a44b7b313d01a2a7a6186c9dcc0ac1a0d156dc",
-                "sha256:801ab496766a858cc54b37a66f9d0169d590628e69bde2166f76c35dfc20f64f",
-                "sha256:80ca483ee2084d36dc570004b6793f99c58d2df201c630f60518bca3518faa5d",
-                "sha256:81154db47a9aa901d09a8cebcd1f6559dbbd02a93a16bc203dc3f94c06bea25a",
-                "sha256:8195b3fe2ed91db03f969c3c5cf6a9bcb3a961749caec94479e9e7a96d7af14f",
-                "sha256:83603a97fdbcf2139f475c940789f09e32703f931f29f4a8ddf3551e6700108b",
-                "sha256:846050f1d7520350d26076df12b8a78b540780d7d3478845be79d57e2e6d3612",
-                "sha256:85582502d3df1db8077475d88d0bf5f35b8ddb8abf9badd8625781024da5f346",
-                "sha256:8575d340ffcd89e4558448eec23c252da584f0de4528a73ed45a01f12d6700fd",
-                "sha256:88e8cbd8fdd2af2f94615c32ab113c5cff4dbd188454394c4cac9941d2f72204",
-                "sha256:8e3dff90b6abc56331a886b5067c3b6bc3fad1b6df5766247197b699afd013b7",
-                "sha256:8e4123de14a8dd7bccb1faed374c1f6c87111766279bbba5d6c4c52e39c383d9",
-                "sha256:8f356263a898b2c9961a2661ebb975f2fd20868d2c4fc4a165226fa7d36e1c76",
-                "sha256:91da52897e5219dc440d3b9d290fdee66fb70c65ae594555c8d2c887df77f2de",
-                "sha256:91e8a0852784c758b8de4179c61e80abae997155aa2b324cbee28dcaf9084b6f",
-                "sha256:92abe38cdd7ea3f98a0f1a4521e9e7e70aa001713bff62a5b97fae41b0a39d0d",
-                "sha256:94973e29acf813a572adf3ed2342a0998a41331781d7be82a0a1b02646181e34",
-                "sha256:94d970caaaf3f59d6c7aa0aaf38e755641351f99452c400fc06b7e365ab1620c",
-                "sha256:957751dd7bba26e651c689fc5f41bb269712efcbf19a97ad4f912f0f792e6e95",
-                "sha256:9593a76f516312ec59dbeb73a9f1915d1f67610c054a7b488912b972cbfbcb7e",
-                "sha256:965e04aeb83f594577ea4e8c975765d0953a75c8d6e3cd193ae25ef574bd83ee",
-                "sha256:99902df14861e4f40ed1eb403a17356188428d12bb10d53de10d1ea24996b838",
-                "sha256:9dca0cde0f02e08e11431fff0e5408ac46420912165d828bc148a3c8cedc8852",
-                "sha256:9fa20eb8f710a9d36801451d1cbcfc83d1dd9e400527ad9aba7e7cc635c2729b",
-                "sha256:a07742d42409c3ce98c58f2b155e2d8acdcd8ff49955062f5155dae94b24060a",
-                "sha256:a15a3ef14cb929b07e0a98639f2ebcd69a0038d6e31367157018ec841a54d8fa",
-                "sha256:a22c4b26fb6134ece45e875515961bc9c1777df775f3488800bea55a381df91e",
-                "sha256:a62e98375385ade756d1ea4975f79ec103c515b412884d146c98295c5300fb4a",
-                "sha256:a900a3dcda64b289cf35acfb6c90246b5ae9f2dd4bc05da8dcb8fbd04c91e37b",
-                "sha256:ad30429f499c30e8e19ed389b8e25116d43c63cafdb97d4f5617e94e671209c5",
-                "sha256:adb895a0081a932132aa3a233cfeba682a07ac0dc36b062152f240406eecfba1",
-                "sha256:b1752344c77dbb137244e6ab5af5a7576b8e8b8e5d440c58d935e69f83606e0d",
-                "sha256:b19cdecdc19ad863673709d27ddb0adc104b9ad341f2785ab23c51c9ab39cf3f",
-                "sha256:b62c408ddac3ae113b9dc4867fd2bb32a2ef669c68f256eb5486c02b31603afa",
-                "sha256:b856928e99a1d3080929b0f194c7922c7a7979b0eb21ad9d057f000dcf1a6ac7",
-                "sha256:ba0f1c0201845b769c92b2b3e7666010abe14d6e9dbf4214d04cdf23cbdcd1ae",
-                "sha256:bafef40c8b521ef9df81a118e278d5759c0127395883dcc10bf80ee32a46da39",
-                "sha256:be591f405d82f40acbe2a081f1dee5478a3cf1b694013cc5712e84159311e36a",
-                "sha256:c20a4ee0706db5c9f403e4c459fc81145f48cb20fccc1a64acfb6143f1a7b1fb",
-                "sha256:c2d2a7324fbd118dc951b63b1fc553aec0fc0c430474a6ab7b33608b9606658a",
-                "sha256:c309d50ee6a41053ad4c284295e5c5eedfd084ff94a5ee1148cf0dc64e3223fb",
-                "sha256:c6f67ac9d3ff65ca466821d573b3cb2fb01f0cd9eb082e92f49dfd88fa828a77",
-                "sha256:c7f7a398bcb0f4cfbaf0ddb3e831c43857701c63c66f48eb30011326ed03b0fa",
-                "sha256:c867d446b22a7288229a812da6f211d86934beb47edf8098e2e0dbe9e2529f86",
-                "sha256:ca3df18c269f9c11b40dde85336d3fc4a879867b05c74efdd4d5f8e64ac7e179",
-                "sha256:ca80bb0b8400888e16e8eaa01b506de7d1bbdc8cc78c9ff75272526fd36fcb8a",
-                "sha256:cccfb03f829ecde3ee126befb4fe827e2228a50e8d9eaa0275c4dd21a0e59765",
-                "sha256:ce8e8a4aa8246c615693b9d0504e1df98157ae48907a044baa80f80abecc44ad",
-                "sha256:cfa03e12c0c1bd357a0e1f20adae9c32bc166dcc6ed3be65885134830899b6ba",
-                "sha256:d077f76179de41cdd5ae39b0a6f16f033b077f27fa54a9e038d03e3f5ddeff23",
-                "sha256:d2cc02e94ff4df6a4a871bbd69149ba5fbc5150108a208c6cdd9bc728000995b",
-                "sha256:d55d9e1e22ebfa9cde606a7967eb21fbc65d0d3be600e550c38a1c05094c8015",
-                "sha256:d62a54dd438b51dd9f94bd88dd2f7c9d4093aadc44892143182d51f8488ab5f6",
-                "sha256:d62f357845f90d5327bdd3286ebd79c40b23b8e7c36772fc2949ef54a92bc6de",
-                "sha256:db3d4e991ee727f6c0a54fe975ebe169be3a463ce810cfd8e6edbe8a90ddeb8f",
-                "sha256:dc917e488a9bdc681cb4a9b0ed3d0c8b81816bc1a6b1f85ec4050c827ffa9e76",
-                "sha256:deb41298fb0c916875413373e5eba162e39309d8730be736b3a206d9b73afebf",
-                "sha256:dec319cdbba5ed3dfb2a8798364a8899751e732127c6ae74bc41276899e0311a",
-                "sha256:e0262bba03bb4d38b697c2039a8e48a9e889fb9ae9b727903e8d9c49d32883a4",
-                "sha256:e0f1f590a66769caea6cf47c3c5d06d004f01659449595d4bc737a42d4670bf0",
-                "sha256:e21208605759dc2eeebf607a002e1ddc09548c12f25886d2a2dd8efdf35535a3",
-                "sha256:e2dbb36017e5c7d7499dfc05f8d6ed781fce9fbfa45cc7bd888ec5f871aba3d3",
-                "sha256:e69d3d5257607a3b29f12192ad7bc9fdbdc8aa5f7e11c6f536cb61ee02dcd698",
-                "sha256:e71bb03c754934d7792aa0559b16f9effe1d4196f5834715c115363809f00c37",
-                "sha256:e7c4530ec6bf3a27c3cba229026fa3669e6e1832fdc37cf74c73b113f884cb2d",
-                "sha256:e9d44c5cfe2516a652131ddd91c3a48924e34f4a7b85640cda44694afbeee6cb",
-                "sha256:ea1fe83079c67c3f1549a735f11060350c87453bd86a8442bccb1f9b14c26aef",
-                "sha256:eb4d96fe5d6d8bf5c8c64d8a37edff55e6d9abbf84d9b8fd0fe8e0db1a557625",
-                "sha256:f0822c9f4852f9f6b1458f8fbf011bb887cefba792b0c7c10659e2c9c997d1c9"
+                "sha256:00954290d6d46ab13535becbbc1327c56f0a9c5d7b7cf967e6587c1395cade42",
+                "sha256:06b9dfd615fb5635c05153431e520954a0e81683c5a6e3ed1134f60cc45b80f1",
+                "sha256:094cec5425097ae1f9a40bb02de917d2274bfa872665fe2e5b4101ee94d8b31d",
+                "sha256:0eadba403ec861fa4c600ad43dbd8ac17b7c22a796d3bd9d92918f4e8a15a6e8",
+                "sha256:10143cad228ebeb9eda7793995b2d0b3fef0685258d9b794f6320824302c47d7",
+                "sha256:19deddb2975af861320fd7b68196fbb2a4a8500897354919baf693702786e349",
+                "sha256:1b8db95f23d928ba87297afe6d4fff21bbb1af343147ff50c174674312afc29d",
+                "sha256:1c0bbdb3ae1c300941c1f89219a8d09d142ddb7bfc78e61da80c8bdc03c05be8",
+                "sha256:1c4cdb0e407bec2f3ece10275449822575f6634727ee1a18e87c5e5a7b565bb1",
+                "sha256:20259fa302f1050bd02d78d93db78870bed385c6d3d299990fe806095426869f",
+                "sha256:26e42ccb76a53c1b943021eeb0eb4d78f46093c16e4e658a7204c838d5b36df0",
+                "sha256:2820b607be0346b3e24b097d759393bd4bcccc0620e8e825591061a2c3a0add5",
+                "sha256:29ca6db3fb72d17bcec091b9ba485c715f63ca00bfcd993f92cb20037ae98b25",
+                "sha256:33bc6f6048f7f7fc506e6ad03fb822a78c2b8209e73b2eddc69d3d6767d0385c",
+                "sha256:33efaf2cc4efd2b100699d953cd70b5a54c3ca912297211fda01875f4636f655",
+                "sha256:349d643aeb8d7d9e0a407cef29d6210afbe646cc19b4e237456e585591eda223",
+                "sha256:35962bc10480aebd5b32fa344430bddd19ef384286501c1c8092b6a9a1ee6a99",
+                "sha256:35ba0ee9d6d502da2bc01d78d22f51a1812ff8d55fb444447f7782f5ce8c1e35",
+                "sha256:363c11a4d60fa0e2e7437f7494291c24eaf2752c8d8e3adf8f92cb0168073464",
+                "sha256:3a57f2a0531ad2cd33bb78d8555e85a250877e555a68c0add6308ceeca8d84f1",
+                "sha256:3c3181a462cdb55df5ddeffe3cf5223cda36c81feceeb231688af08d30f11022",
+                "sha256:3c5f28a145677431347772b43a9604b67691b16e233ec7a92fc77fc5fb670608",
+                "sha256:3dd4592c2fca923041c57aa2bfe428de14cc45f3a00ab825b353160994bc15e7",
+                "sha256:3f661848fa1984f3b17da676c88ccd08d8c3fab5501a1d1c8ac5abece48566f2",
+                "sha256:44f818ea8c191285365a0add6fc03f88225f1fdcff570dc78e9f548444042441",
+                "sha256:48037009be790fca505a62705a7997eef0cd384c3ef6c10a769734660245ee73",
+                "sha256:48ff680078aec3b9515f149010981c7feeef6c2706987ac7bdc7cc1ea05f8f7d",
+                "sha256:493edd702bc16dae1f4d76461688714c488af1b33f5b3a77c1a86d5c81240f9e",
+                "sha256:4a765c5fc05fe1c843863cc3723e39e8207c28d9a7152ee6d621fa3908ef4880",
+                "sha256:4d5c98986d774e9321fb1d4fe0362658560e14c1d7afbe2d298b89a24c2f7b4f",
+                "sha256:586538aa2a992a55c10d88c58166e6023968a9825719bce5a09397b73eea658f",
+                "sha256:5c43e2222bbbe660dea8fe335f5c633b3c9ed10628a4b53a160ddd54d15cffc2",
+                "sha256:5d0db66651ed5a866a1452e7a450e41a5ec743abbeea1f1bc85ef7c64f5f6b8f",
+                "sha256:65a55aac43a685b7d2b9e7c4f9f3768ad6e0d5f9ad7698b8bf9124fbeb814d43",
+                "sha256:691cadd48f225097a2588e7db492ac88c669c061208749bc0200ee39e4425e32",
+                "sha256:6aa796663db6d1d01ebdcd80022de840005ae173e01a7b03b3934811b7ae39bc",
+                "sha256:708c442f8f6540ffad24a894bdea3c019250e02dcdbd0fbd27fc977b1a88b4f2",
+                "sha256:74521d819ceea90794aded974cc3024c65c094050e6c4a6f4b7478af3461e3ad",
+                "sha256:75f4363157777cbcbbd14ff823388fddfca597d44c77c27473c4c4000d7a5c99",
+                "sha256:75f5e862e1646f1688e97f4aa69988d6589a1e036f081e98a3f202fa4647e69b",
+                "sha256:76697baa4d9fd621bd5b99719d3b55fadeb665af9a49523debfc9ae5fbefef13",
+                "sha256:784f7f87ae2e25459ef78282fbe9f0d2fec9ced84e4acb5d28621a0db274a13b",
+                "sha256:78f5e65eb15d93f687715be9241c8b55d838fba9b7045d83530f8831544f1413",
+                "sha256:796a29cbb6414b6cb84d8e7448262ba286847b946de9a149dec97469a4789552",
+                "sha256:79f4c9f1d7906eb890dafae4820f69bd24658297e9ebcdd74867330e8e7bf9b0",
+                "sha256:7a82cd4e772e5d1400502d68da7ecd71a6f1ff37243017f284bee3d2106a2496",
+                "sha256:80741b9f18149264acb639287347cfc6eecff109b5c6d95dbf7222756b107b57",
+                "sha256:80962ff81a3389b5579d1206bea1bb48da38991407442d2a9287f6da1ccb2c80",
+                "sha256:84135917c99476c6abeee420ffd005a856d8fde0e5f585b0c484d5923392035b",
+                "sha256:8436ce4fa7e7ddaa8d29717fd73e0699883ef6e78ef4d785c244779a7ad1942b",
+                "sha256:89187af1ca5a9b65c477817e0fe7e411f4edd99e5575aaaef6a9e5ff62028854",
+                "sha256:8a2d5a8b74db3df772bb4f230319241e73629b04cb777b22f9dcd2084d92977a",
+                "sha256:8e4cf6d11427d43734e8cb246ecfb7af169983ef796b415379602ea0605f5116",
+                "sha256:8e51d69446d96f5767e0f1b0676341d5d576c151dfe3dd14aff7a163db1b4d7c",
+                "sha256:8ef9fa7fe28dd6b7d09b8be89aea4e8f2d18b23a89294f51aa48dbc6c306a039",
+                "sha256:94fe8c5f1f11397b5db8b1850168e5bed13b3f3e1bc36e4292819d85be51a63c",
+                "sha256:983ea93ed937d329c88ef15d5e3b09e32372590c1a80586b2013f17aed436cb8",
+                "sha256:994a21a75d7b2602a78c2f88f809427ce1051e43af7aad6cda524ccdc859354e",
+                "sha256:9c68761529a43358151ac507aeb9c6b7c1a990235ce7b7d41f8ea62c62d4679e",
+                "sha256:a4cf0fed2d5c9de3da211dceff3ed9a09b8f998f7df57da847145863a786454b",
+                "sha256:a4e12b6702481ace7071357c1b81b9faf6f660da55ff9ccd6383fed474348cc6",
+                "sha256:acfe529ff44d379ee889f03c2d353f94b1f16c83a92852061f9672982a3ef32d",
+                "sha256:b0fa1ef68839d99b0c0d66fe060303f7f2916f021289a7e04a818ef9461bbbe1",
+                "sha256:b3cc09eecd318310cfd6e7f245248cf16ca014ea5903580d72231d93330952de",
+                "sha256:b5d8a3263b7e23a3593eb4fcc5cc77e053c7d15c874db16ce6ee8b4d94f8d825",
+                "sha256:bbeaa0af865427405a1c0e8c65841a23de66af8ca5d796522f7b105386cd8522",
+                "sha256:bc5e630db572362aef4d8a78f82a40e2b9756de7622feb07031bd400a696ad78",
+                "sha256:bd27ab78269148c65d988a6b26471d621d4cc6eed6b92462b7f8850162e5c4f2",
+                "sha256:c34c06a6496b4aacdab03133671dd5638417bda09a1f186ba1a39c1dbd1add24",
+                "sha256:c48b4368b832233205a74e9f1dfe2647d9bc49ea8357b09963fd5f15062bdd0a",
+                "sha256:c8c675edd26cd2531163e51dcb3c7c73145e2fa3b77a1ff59ce9ed963ff56017",
+                "sha256:c9c1ede5c4e35b059e8734dfa8d23a59b8fcfe3e0ece4f7d226bc5e1816512c9",
+                "sha256:ca9f1f6bd487c9b990e509c17e0a701f554db9e77bd5121c27f1db4594ac4c0a",
+                "sha256:d14862ce066da0494e0f9466afc3b8fcd6c03f7250323cf8ef62c67158c77e57",
+                "sha256:d1e6ae36c717abd32b55a275d7fbf9041b6de3a103639739ec3e8c8283773fb3",
+                "sha256:d7d5888e206190d36fbffed6d7e9cacd79e64fd34e9a07359e16862973d90b33",
+                "sha256:d897ec18822e348f8ef9a17e421716ed224a3726fde806aae04469fec8f0ac9d",
+                "sha256:e0f5a1865a00798a74d50fcc9956a3d7fa7413cbc1c6d6d04833d89f36e35226",
+                "sha256:e31e0d2023b693ca530d95df7cff8d736f66b755018398bc518160f91e80bd0a",
+                "sha256:e8eae552db2aa587c986f460915786bf9058a88d831d562cadba01f3069736a9",
+                "sha256:e9f22fb00bfcca4b2e0b36afd4f3a3194c1bc93b2a76e51932ccfd3b6aa62501",
+                "sha256:ef883837c16c076f11da37323f589779806073eeacaef3912f2da0359cb8c2cf",
+                "sha256:f47aada7fdc6bcad8ec4ee4ff00a8d2d9a0e05b5516df3f304afbf527b026221",
+                "sha256:f4f27e083a63b9463fd2640065af1b924f05831839f23d936a97c4f510a54f6b",
+                "sha256:f560d24557bbc54eb1aa01ee6e587d4d199b785593462567ddf752de3c1c4974",
+                "sha256:fa219d5d6124f1623b39f296a1fcc4cac1d8c82f137516bd362a38c16adcd92b",
+                "sha256:fb70083bf00426194a85d69939c52b1759462873bf6e4d62f481e2bc3e642ea1",
+                "sha256:fd43a0ae38ae15223fb1057729001829c3336e90f4acf04cf12ebdec33346658",
+                "sha256:ff99a11dd76aec5a5234c1158d6b8dacb61b208f3f30a2bf7ae3b23243190581"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.15.10"
+            "version": "==0.16.0"
         },
         "qtconsole": {
             "hashes": [
@@ -3323,6 +3325,7 @@
                 "sha256:eaaeb2e661d1ffc07a7ae4fd997bb326d3561f641178126102842d608a010cc3"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==1.3.6"
         },
         "rasterstats": {
@@ -3331,6 +3334,7 @@
                 "sha256:29389bfcbeac1a4206aba6e1d795058ec8a64d560efad48156c27fad97c2e09a"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.18.0"
         },
         "referencing": {
@@ -3343,11 +3347,12 @@
         },
         "requests": {
             "hashes": [
-                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
-                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
+                "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
+                "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"
             ],
             "index": "pypi",
-            "version": "==2.31.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.32.3"
         },
         "rfc3339-validator": {
             "hashes": [
@@ -3554,6 +3559,7 @@
                 "sha256:f5120da3a1b96f3a7a17dd6af0afdd4e6f3cc9baa87e9ee0a272882f01f980bb"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==1.0.1"
         },
         "s3transfer": {
@@ -3589,6 +3595,7 @@
                 "sha256:fae8a7b898c42dffe3f7361c40d5952b6bf32d10c4569098d276b4c547905ee1"
             ],
             "index": "pypi",
+            "markers": "python_version < '3.12' and python_version >= '3.8'",
             "version": "==1.10.1"
         },
         "seaborn": {
@@ -3597,6 +3604,7 @@
                 "sha256:ebf15355a4dba46037dfd65b7350f014ceb1f13c05e814eda2c9f5fd731afc08"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==0.12.2"
         },
         "send2trash": {
@@ -3609,11 +3617,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987",
-                "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"
+                "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4",
+                "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==69.5.1"
+            "version": "==70.0.0"
         },
         "shapely": {
             "hashes": [
@@ -3657,6 +3665,7 @@
                 "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==2.0.1"
         },
         "simplejson": {
@@ -3822,6 +3831,7 @@
                 "sha256:f0821007048f2af8c1a21eb3d832072046c5df366e39587a7c7e4afad14e73fc"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==3.8.0"
         },
         "terminado": {
@@ -3865,20 +3875,20 @@
         },
         "tornado": {
             "hashes": [
-                "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0",
-                "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63",
-                "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263",
-                "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052",
-                "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f",
-                "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee",
-                "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78",
-                "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579",
-                "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212",
-                "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e",
-                "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"
+                "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8",
+                "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f",
+                "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4",
+                "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3",
+                "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14",
+                "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842",
+                "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9",
+                "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698",
+                "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7",
+                "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d",
+                "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.4"
+            "version": "==6.4.1"
         },
         "tqdm": {
             "hashes": [
@@ -3886,6 +3896,7 @@
                 "sha256:4f41d54107ff9a223dca80b53efe4fb654c67efaba7f47bada3ee9d50e05bd53"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.7'",
             "version": "==4.66.3"
         },
         "traitlets": {
@@ -3898,11 +3909,11 @@
         },
         "typeguard": {
             "hashes": [
-                "sha256:7da3bd46e61f03e0852f8d251dcbdc2a336aa495d7daff01e092b55327796eb8",
-                "sha256:c556a1b95948230510070ca53fa0341fb0964611bd05d598d87fb52115d65fee"
+                "sha256:4d24c5b39a117f8a895b9da7a9b3114f04eb63bade45a4492de49b175b6f7dfa",
+                "sha256:92ee6a0aec9135181eae6067ebd617fd9de8d75d714fb548728a4933b1dea651"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.2.1"
+            "version": "==4.3.0"
         },
         "types-python-dateutil": {
             "hashes": [
@@ -3914,11 +3925,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
-                "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"
+                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==4.11.0"
+            "version": "==4.12.2"
         },
         "typing-inspect": {
             "hashes": [
@@ -3948,15 +3959,16 @@
                 "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
             ],
             "index": "pypi",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==1.26.18"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:604bfdceaeece392802e6ae48e69cec49168b9c5f4a44e483963f9242eb0e78b",
-                "sha256:7aa9982a728ae5892558bff6a2839c00b9ed145523ece2274fad6f414690ae75"
+                "sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c",
+                "sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.26.1"
+            "version": "==20.26.2"
         },
         "wcwidth": {
             "hashes": [
@@ -3967,10 +3979,10 @@
         },
         "webcolors": {
             "hashes": [
-                "sha256:29bc7e8752c0a1bd4a1f03c14d6e6a72e93d82193738fa860cbff59d0fcc11bf",
-                "sha256:c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a"
+                "sha256:1d160d1de46b3e81e58d0a280d0c78b467dc80f47294b91b1ad8029d2cedb55b",
+                "sha256:8cf5bc7e28defd1d48b9e83d5fc30741328305a8195c29a8e668fa45586568a1"
             ],
-            "version": "==1.13"
+            "version": "==24.6.0"
         },
         "webencodings": {
             "hashes": [
@@ -3989,19 +4001,19 @@
         },
         "whitebox": {
             "hashes": [
-                "sha256:38becfa894091c0adcebb162ffe7eb8b735e9a52f91376d36cb1a554bd10544d",
-                "sha256:6b4080bc8181f0876cda6c9093bcb463faa4ab8a8c2c631c06b579d3ec1131ba"
+                "sha256:3479c8127b1928dd0457b6680b9ee4469d0735b2bb220178c678af303d868ef0",
+                "sha256:4322c5a3b42f1f04a3c80640253b672242e0aefb6b6dea70752f00a220f6d7ed"
             ],
             "index": "pypi",
-            "version": "==2.3.1"
+            "version": "==2.3.4"
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:64196c5ff3b9a9183a8e699a4227fb0b7002f252c814098e66c4d1cd0644688f",
-                "sha256:d37c3724ec32d8c48400a435ecfa7d3e259995201fbefa37163124a9fcb393cc"
+                "sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36",
+                "sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.0.10"
+            "version": "==4.0.11"
         },
         "wrapt": {
             "hashes": [
@@ -4085,6 +4097,7 @@
                 "sha256:7e530b1deafdd43e5c2b577d0944e6b528fbe88045fd849e49a8d11871ecd522"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.8'",
             "version": "==2023.1.0"
         },
         "xarray-spatial": {
@@ -4097,11 +4110,11 @@
         },
         "xyzservices": {
             "hashes": [
-                "sha256:6a04f11487a6fb77d92a98984cd107fbd9157fd5e65f929add9c3d6e604ee88c",
-                "sha256:b83e48c5b776c9969fffcfff57b03d02b1b1cd6607a9d9c4e7f568b01ef47f4c"
+                "sha256:58c1bdab4257d2551b9ef91cd48571f77b7c4d2bc45bf5e3c05ac97b3a4d7282",
+                "sha256:fecb2508f0f2b71c819aecf5df2c03cef001c56a4b49302e640f3b34710d25e4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.4.0"
+            "version": "==2024.6.0"
         },
         "y-py": {
             "hashes": [
@@ -4292,15 +4305,16 @@
                 "sha256:c3b7d2c85b8a42b0ad0ad268a36fb6886ca852098358c125c6b126a417e0a598"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.9'",
             "version": "==2.18.0"
         },
         "zipp": {
             "hashes": [
-                "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b",
-                "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"
+                "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19",
+                "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.18.1"
+            "version": "==3.19.2"
         }
     },
     "develop": {
@@ -4337,11 +4351,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501",
-                "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3"
+                "sha256:53eee7ad44df903a06655871cbab66d156a051fd86f3ec6750470ac9604ac1ab",
+                "sha256:c6ed726a140b6e725b911528f80439c534fac915246af3efc39440a6b0f9d716"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.24.0"
+            "version": "==8.25.0"
         },
         "jedi": {
             "hashes": [
@@ -4377,11 +4391,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
-                "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
+                "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10",
+                "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.0.43"
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==3.0.47"
         },
         "ptyprocess": {
             "hashes": [
@@ -4431,11 +4445,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
-                "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"
+                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==4.11.0"
+            "version": "==4.12.2"
         },
         "wcwidth": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1b4c56608..6fe9d3838 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.5.2.2 - 2024-05-29 - [PR#1183](https://github.com/NOAA-OWP/inundation-mapping/pull/1183)
+
+Upgrades whitebox from v2.3.1 to 2.3.4. Whitebox was upgraded by `pip install -U whitebox` and then `pipenv lock` to update the `Pipfile`.
+
+### Changes
+
+- `Dockerfile`: Removed whitebox hack
+- `Pipfile` and `Pipfile.lock`: Upgraded whitebox to v2.3.4.
+
+<br/><br/>
+
 ## v4.5.2.1 - 2024-05-21 - [PR#1172](https://github.com/NOAA-OWP/inundation-mapping/pull/1172)
 
 Removes loading of `apache-arrow` repository from the Dockerfile where it was causing a GPG key error during `docker build`.

From b0d3aaa78fa81164e0b01068a1a1dbfae1286aa3 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 14 Jun 2024 19:47:42 +0000
Subject: [PATCH 326/359] v4.5.2.3 New tool to scan logs for errors (#1169)

---
 docs/CHANGELOG.md                      | 13 ++++++++++++-
 fim_post_processing.sh                 | 17 +++++++++++++++--
 src/mitigate_branch_outlet_backpool.py |  4 ++--
 3 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 6fe9d3838..d4f4011f4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,8 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.3 - 2024-06-14 - [PR#1169](https://github.com/NOAA-OWP/inundation-mapping/pull/1169)
 
-## v4.5.2.2 - 2024-05-29 - [PR#1183](https://github.com/NOAA-OWP/inundation-mapping/pull/1183)
+This tool scans all log directory looking for the word "error" (not case-sensitive). This is primary added to help find errors in the post processing logs such as src_optimization folder (and others).
+
+### Changes
+
+- `fim_post_processing.sh`: as described
+- `src\mitigate_branch_outlet_backpool.py`: Has the word error in text which fills up the error scan logs.
+
+<br/><br/>
+
+
+## v4.5.2.2 - 2024-06-14 - [PR#1183](https://github.com/NOAA-OWP/inundation-mapping/pull/1183)
 
 Upgrades whitebox from v2.3.1 to 2.3.4. Whitebox was upgraded by `pip install -U whitebox` and then `pipenv lock` to update the `Pipfile`.
 
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index fb9c10f82..2a7965efb 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -218,8 +218,21 @@ python3 $toolsDir/combine_crosswalk_tables.py \
 Tcount
 date -u
 
-find $outputDestDir -type d -exec chmod -R 777 {} +
-find $outputDestDir -type f -exec chmod -R 777 {} +
+echo -e $startDiv"Resetting Permissions"
+Tstart
+    find $outputDestDir -type d -exec chmod -R 777 {} +
+    find $outputDestDir -type f -exec chmod 777 {} +  # just root level files
+Tcount
+date -u
+
+echo
+echo -e $startDiv"Scanning logs for errors. Results be saved in root not inside the log folder."
+Tstart
+    # grep -H -r -i -n "error" $outputDestDir/logs/ > $outputDestDir/all_errors_from_logs.log
+    find $outputDestDir -type f | grep -H -r -i -n "error" $outputDestDir/logs/ > $outputDestDir/all_errors_from_logs.log
+Tcount
+date -u
+
 
 echo
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
diff --git a/src/mitigate_branch_outlet_backpool.py b/src/mitigate_branch_outlet_backpool.py
index b056b9981..3b76d8479 100755
--- a/src/mitigate_branch_outlet_backpool.py
+++ b/src/mitigate_branch_outlet_backpool.py
@@ -507,10 +507,10 @@ def mask_raster_to_boundary(raster_path, boundary_json, save_path):
                 print('Incorrectly-large outlet pixel catchment was NOT detected.')
 
         else:
-            print('Will not test for outlet backpool error.')
+            print('Will not test for outlet backpool problem.')
 
     else:
-        print('Will not test for outlet backpool error in branch zero.')
+        print('Will not test for outlet backpool problem in branch zero.')
 
 
 if __name__ == '__main__':

From 87364e140251a1b67a24cc225ae8918c4ef62b69 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 8 Jul 2024 10:52:24 -0700
Subject: [PATCH 327/359] v4.5.2.4  Bug fix for extending outlets (#1204)

---
 data/wbd/clip_vectors_to_wbd.py        |  50 ++++--
 data/wbd/generate_pre_clip_fim_huc8.py | 210 +++++++++++++------------
 docs/CHANGELOG.md                      |  23 ++-
 src/bash_variables.env                 |   4 +-
 4 files changed, 175 insertions(+), 112 deletions(-)

diff --git a/data/wbd/clip_vectors_to_wbd.py b/data/wbd/clip_vectors_to_wbd.py
index 048539781..8cb8603e8 100755
--- a/data/wbd/clip_vectors_to_wbd.py
+++ b/data/wbd/clip_vectors_to_wbd.py
@@ -2,6 +2,7 @@
 
 import argparse
 import logging
+import os
 import sys
 
 import geopandas as gpd
@@ -21,18 +22,26 @@ def extend_outlet_streams(streams, wbd_buffered, wbd):
     Extend outlet streams to nearest buffered WBD boundary
     """
 
-    wbd['geometry'] = wbd.geometry.boundary
-    wbd = gpd.GeoDataFrame(data=wbd, geometry='geometry')
-
-    wbd_buffered["linegeom"] = wbd_buffered.geometry
-
     # Select only the streams that are outlets
     levelpath_outlets = streams[streams['to'] == 0]
 
-    # Select only the streams that don't intersect the WBD boundary line
-    levelpath_outlets = levelpath_outlets[~levelpath_outlets.intersects(wbd['geometry'].iloc[0])]
+    levelpath_outlets_columns = [x for x in levelpath_outlets.columns]
+
+    # Select streams that intersect the WBD but not the WBD buffer
+    levelpath_outlets = levelpath_outlets.sjoin(wbd)[levelpath_outlets_columns]
+
+    wbd_boundary = wbd.copy()
+    wbd_boundary['geometry'] = wbd_boundary.geometry.boundary
+    wbd_boundary = gpd.GeoDataFrame(data=wbd_boundary, geometry='geometry')
+
+    wbd_buffered["linegeom"] = wbd_buffered.geometry
+
+    levelpath_outlets = levelpath_outlets[
+        ~levelpath_outlets.intersects(wbd_buffered["linegeom"].boundary.iloc[0])
+    ]
 
     levelpath_outlets['nearest_point'] = None
+    levelpath_outlets['nearest_point_wbd'] = None
     levelpath_outlets['last'] = None
 
     levelpath_outlets = levelpath_outlets.explode(index_parts=False)
@@ -45,22 +54,36 @@ def extend_outlet_streams(streams, wbd_buffered, wbd):
     wbd_buffered['geometry'] = wbd_buffered.geometry.boundary
     wbd_buffered = gpd.GeoDataFrame(data=wbd_buffered, geometry='geometry')
 
+    errors = 0
     for index, row in levelpath_outlets.iterrows():
         levelpath_geom = row['last']
         nearest_point = nearest_points(levelpath_geom, wbd_buffered)
+        nearest_point_wbd = nearest_points(levelpath_geom, wbd_boundary.geometry)
 
         levelpath_outlets.at[index, 'nearest_point'] = nearest_point[1]['geometry'].iloc[0]
+        levelpath_outlets.at[index, 'nearest_point_wbd'] = nearest_point_wbd[1].iloc[0]
 
         levelpath_outlets_nearest_points = levelpath_outlets.at[index, 'nearest_point']
+        levelpath_outlets_nearest_points_wbd = levelpath_outlets.at[index, 'nearest_point_wbd']
+
         if isinstance(levelpath_outlets_nearest_points, pd.Series):
             levelpath_outlets_nearest_points = levelpath_outlets_nearest_points.iloc[-1]
-
-        levelpath_outlets.at[index, 'geometry'] = LineString(
-            list(row['geometry'].coords) + list([levelpath_outlets_nearest_points.coords[0]])
-        )
+        if isinstance(levelpath_outlets_nearest_points_wbd, pd.Series):
+            levelpath_outlets_nearest_points_wbd = levelpath_outlets_nearest_points_wbd.iloc[-1]
+
+        # Extend outlet stream if outlet point is outside of the WBD or nearest snap point is within 100m of the WBD boundary
+        outlet_point = Point(row['geometry'].coords[-1])
+        if (outlet_point.distance(levelpath_outlets_nearest_points_wbd) < 100) or (
+            ~outlet_point.intersects(wbd.geometry)[0]
+        ):
+            levelpath_outlets.at[index, 'geometry'] = LineString(
+                list(row['geometry'].coords) + list([levelpath_outlets_nearest_points.coords[0]])
+            )
+        else:
+            errors += 1
 
     levelpath_outlets = gpd.GeoDataFrame(data=levelpath_outlets, geometry='geometry')
-    levelpath_outlets = levelpath_outlets.drop(columns=['last', 'nearest_point'])
+    levelpath_outlets = levelpath_outlets.drop(columns=['last', 'nearest_point', 'nearest_point_wbd'])
 
     # Replace the streams in the original file with the extended streams
     streams = streams[~streams['ID'].isin(levelpath_outlets['ID'])]
@@ -116,6 +139,7 @@ def subset_vector_layers(
     logging.info(f"Clip ocean water polygon for {hucCode}")
     landsea = gpd.read_file(landsea, mask=wbd_buffer, engine="fiona")
     if not landsea.empty:
+        os.makedirs(os.path.dirname(subset_landsea), exist_ok=True)
         # print(f"Create landsea gpkg for {hucCode}", flush=True)
         landsea.to_file(
             subset_landsea, driver=getDriver(subset_landsea), index=False, crs=huc_CRS, engine="fiona"
@@ -356,6 +380,8 @@ def subset_vector_layers(
         '-lps', '--subset-levee-protected-areas', help='Levee-protected areas subset', required=True
     )
     parser.add_argument('-osm', '--osm-bridges', help='Open Street Maps gkpg', required=True)
+    parser.add_argument('-osms', '--subset-osm-bridges', help='Open Street Maps subset', required=True)
+    parser.add_argument('-ak', '--is-alaska', help='If in Alaska', action='store_true')
     parser.add_argument('-crs', '--huc-CRS', help='HUC crs', required=True)
 
     args = vars(parser.parse_args())
diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
index 8a34e6837..22e3a6022 100755
--- a/data/wbd/generate_pre_clip_fim_huc8.py
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -41,6 +41,8 @@
       src/bash_variables.env to the corresponding outputs_dir argument after running and testing this script.
       The newly generated data should be created in a new folder using the format <year_month_day>
              (i.e. September 26, 2023 would be 23_09_26)
+
+    Don't worry about error logs saying "Failed to auto identify EPSG: 7"
 '''
 
 srcDir = os.getenv('srcDir')
@@ -89,77 +91,66 @@
 wbd_buffer = os.getenv('wbd_buffer')
 wbd_buffer_int = int(wbd_buffer)
 
+LOG_FILE_PATH = ""
+
 
-def __setup_logger(outputs_dir, huc=None):
+def __setup_logger(outputs_dir, huc=None, is_multi_proc=False):
     '''
     Set up logging to file. Since log file includes the date, it will be overwritten if this
     script is run more than once on the same day.
     '''
+    global LOG_FILE_PATH
+
     datetime_now = dt.datetime.now(dt.timezone.utc)
-    curr_date = datetime_now.strftime("%y%my%d")
+    file_dt_string = datetime_now.strftime("%Y_%m_%d-%H_%M_%S")
 
     if huc is None:
-        log_file_name = f"generate_pre_clip_fim_huc8_{curr_date}.log"
+        log_file_name = f"generate_pre_clip_{file_dt_string}.log"
     else:
-        log_file_name = f"mp_{huc}_generate_pre_clip_fim_huc8_{curr_date}.log"
+        log_file_name = f"mp_{huc}_generate_pre_{file_dt_string}.log"
 
-    log_file_path = os.path.join(outputs_dir, log_file_name)
+    LOG_FILE_PATH = os.path.join(outputs_dir, log_file_name)
 
     if not os.path.exists(outputs_dir):
         os.mkdir(outputs_dir)
 
-    if os.path.exists(log_file_path):
-        os.remove(log_file_path)
+    if os.path.exists(LOG_FILE_PATH):
+        os.remove(LOG_FILE_PATH)
 
-    file_handler = logging.FileHandler(log_file_path)
-    file_handler.setLevel(logging.INFO)
-    console_handler = logging.StreamHandler()
-    console_handler.setLevel(logging.INFO)
+    # set up logging to file
+    logging.basicConfig(
+        filename=LOG_FILE_PATH, level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%H:%M:%S'
+    )
 
-    logger = logging.getLogger()
-    logger.addHandler(file_handler)
-    logger.setLevel(logging.INFO)
+    # If true, console text will be displayed twice, once from the parent and the other from the MP
+    if is_multi_proc is False:
 
-    # Print start time
-    start_time_string = datetime_now.strftime("%m/%d/%Y %H:%M:%S")
-    logging.info('==========================================================================')
-    logging.info("\n generate_pre_clip_fim_huc8.py")
-    logging.info(f"\n \t Started: {start_time_string} \n")
+        # set up logging to console
+        console = logging.StreamHandler()
+        console.setLevel(logging.DEBUG)
+        # set a format which is simpler for console use
+        # add the handler to the root logger
+        logging.getLogger('').addHandler(console)
 
 
-def __merge_mp_logs(outputs_dir):
-    log_file_list = list(Path(outputs_dir).rglob("mp_*"))
+def __merge_mp_logs(log_dir, parent_log_file):
+    # Locks for all logs starting with the phrase mp*
+    log_file_list = list(Path(log_dir).rglob("mp_*"))
+
     if len(log_file_list) > 0:
         log_file_list.sort()
+    else:
+        print("No multi-proc files to merge")
+        return
 
-    log_mp_rollup_file = os.path.join(outputs_dir, "mp_merged_logs.log")
-
-    error_huc_found = False
-
-    with open(log_mp_rollup_file, 'a') as main_log:
+    with open(parent_log_file, 'a') as main_log:
         # Iterate through list
         for temp_log_file in log_file_list:
             # Open each file in read mode
             with open(temp_log_file) as infile:
-                contents = infile.read()
-                temp_upper_contents = contents.upper()
-                if "ERROR" in temp_upper_contents:
-                    print(
-                        f"\nAn error exist in file {temp_log_file}."
-                        " Check the merge logs for that huc number"
-                    )
-                    error_huc_found = True
-                main_log.write(contents)
+                main_log.write(infile.read())
             os.remove(temp_log_file)
 
-    if error_huc_found:
-        print(
-            "\n\nOften you can just create a new huc list with the fails, re-run to a"
-            " 1temp directory and recheck if errors still exists. Sometimes multi-prod can create"
-            " contention errors.\nFor each HUC that is sucessful, you can just copy it back"
-            " into the original full pre-clip folder.\n"
-        )
-
 
 def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     '''
@@ -212,6 +203,11 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     __setup_logger(outputs_dir)
     start_time = dt.datetime.now(dt.timezone.utc)
 
+    logging.info("==================================")
+    logging.info(f"Starting Pre-clip based on huc list of\n    {huc_list}")
+    logging.info(f"Start time: {start_time.strftime('%m/%d/%Y %H:%M:%S')}")
+    logging.info("")
+
     # Iterate over the huc_list argument and create a directory for each huc.
     for huc in hucs_to_pre_clip_list:
         if os.path.isdir(os.path.join(outputs_dir, huc)):
@@ -221,10 +217,6 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
                 f"\n\t Output Directory: {outputs_dir}/{huc} exists.  It will be overwritten, and the "
                 f"newly generated huc level files will be output there. \n"
             )
-            print(
-                f"\n\t Output Directory: {outputs_dir}/{huc} exists.  It will be overwritten, and the "
-                f"newly generated huc level files will be output there. \n"
-            )
 
         elif not os.path.isdir(os.path.join(outputs_dir, huc)):
             os.mkdir(os.path.join(outputs_dir, huc))
@@ -232,14 +224,13 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     # Build arguments (procs_list) for each process to execute (huc_level_clip_vectors_to_wbd)
     procs_list = []
     for huc in hucs_to_pre_clip_list:
-        print(f"Generating vectors for {huc}. ")
+        # logging.info(f"Generating vectors for {huc}. ")
         procs_list.append([huc, outputs_dir])
 
         # procs_list.append([huc, outputs_dir, wbd_alaska_file])
 
     # Parallelize each huc in hucs_to_parquet_list
     logging.info('Parallelizing HUC level wbd pre-clip vector creation. ')
-    print('Parallelizing HUC level wbd pre-clip vector creation. ')
     # with Pool(processes=number_of_jobs) as pool:
     #    pool.map(huc_level_clip_vectors_to_wbd, procs_list)
 
@@ -248,6 +239,7 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     # The log files for each multi proc has tons and tons of duplicate lines crossing mp log
     # processes, but does always log correctly back to the parent log
 
+    failed_HUCs_list = []  # On return from the MP, if it returns a HUC number, that is a failed huc
     with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
         futures = {}
         for huc in hucs_to_pre_clip_list:
@@ -257,11 +249,32 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
 
         for future in as_completed(futures):
             if future is not None:
-                if future.exception():
+                if not future.exception():
+                    failed_huc = future.result()
+                    if failed_huc.lower() != "success" and failed_huc.isnumeric():
+                        failed_HUCs_list.append(failed_huc)
+                else:
                     raise future.exception()
 
     print("Merging MP log files")
-    __merge_mp_logs(outputs_dir)
+    __merge_mp_logs(outputs_dir, LOG_FILE_PATH)
+
+    if len(failed_HUCs_list) > 0:
+        logging.info("\n+++++++++++++++++++")
+        logging.info("HUCs that failed to proccess are: ")
+        huc_error_msg = "  -- "
+        for huc in failed_HUCs_list:
+            huc_error_msg += f"{huc}, "
+        logging.info(huc_error_msg)
+        print("  See logs for more details on each HUC fail")
+        print(
+            "\n\nOften you can just create a new huc list with the fails, re-run to a"
+            " temp directory and recheck if errors still exists. Sometimes multi-prod can create"
+            " contention errors.\nFor each HUC that is sucessful, you can just copy it back"
+            " into the original full pre-clip folder.\n"
+        )
+
+        logging.info("+++++++++++++++++++")
 
     # Get time metrics
     end_time = dt.datetime.now(dt.timezone.utc)
@@ -277,9 +290,6 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     )
     logging.info('==========================================================================')
 
-    print("\n\t Completed writing all huc level files \n")
-    print(f"\t \t TOTAL RUN TIME: {str(time_duration).split('.')[0]}")
-
 
 def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
     '''
@@ -300,13 +310,14 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
         Creation of wbd8_clp.gpkg & wbd_buffered.gpkg using the executable: "ogr2ogr .... -clipsrc ..."
 
     Outputs:
-    - .gpkg files* dependant on HUC's WBD (*differing amount based on individual huc)
+    - .If successful, then it returns the word sudcess. If it fails, return the huc number.
     '''
 
+    in_error = False
     huc_processing_start = dt.datetime.now(dt.timezone.utc)
     # with this in Multi-proc, it needs it's own logger and unique logging file.
-    __setup_logger(outputs_dir, huc)
-    logging.info(f"Processing {huc}")
+    __setup_logger(outputs_dir, huc, True)
+    logging.info(f"Start Processing {huc}")
 
     try:
 
@@ -338,7 +349,7 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
         else:
             input_LANDSEA = f"{inputsDir}/landsea/water_polygons_us.gpkg"
 
-        print(f"\n Get WBD {huc}")
+        logging.info(f"-- {huc} : Get WBD")
 
         # TODO: Use Python API (osgeo.ogr) instead of using ogr2ogr executable
         get_wbd_subprocess = subprocess.run(
@@ -360,24 +371,22 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
             universal_newlines=True,
         )
 
-        logging.info(f"{huc} : {get_wbd_subprocess.stdout}")
-
-        if get_wbd_subprocess.stderr != "":
-            if "ERROR" in get_wbd_subprocess.stderr.upper():
-                msg = (
-                    f" - Creating -- {huc_directory}/wbd.gpkg"
-                    f"  ERROR -- details: ({get_wbd_subprocess.stderr})"
-                )
-                print(msg)
-                logging.info(msg)
+        sub_proc_stdout = get_wbd_subprocess.stdout
+        sub_proc_stderror = get_wbd_subprocess.stderr
+        if sub_proc_stderror.lower().startswith("warning") is True:
+            logging.info(f"Warning from ogr creating wbd file: {sub_proc_stderror}")
+        elif sub_proc_stderror != "":
+            raise Exception(
+                f"*** {huc} : Error creating wbd file from ogr for {huc}:" f" Details: {sub_proc_stderror}\n"
+            )
         else:
-            msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete \n"
-            print(msg)
+            msg = f"-- {huc} : Creating {huc_directory}/wbd.gpkg - Complete"
+            if sub_proc_stdout != "":  # warnings?
+                msg += f": {sub_proc_stdout}"
+
             logging.info(msg)
 
-        msg = f"Get Vector Layers and Subset {huc}"
-        # print(msg)
-        logging.info(msg)
+        logging.info(f"-- {huc} : Get Vector Layers and Subset")
 
         # Subset Vector Layers (after determining whether it's alaska or not)
         if huc2Identifier == '19':
@@ -444,12 +453,10 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
                 huc_CRS=huc_CRS,  # TODO: simplify
             )
 
-        msg = f" Completing Get Vector Layers and Subset: {huc} \n"
-        print(msg)
-        logging.info(msg)
+        logging.info(f"-- {huc} : Completing Get Vector Layers and Subset:\n")
 
         ## Clip WBD8 ##
-        print(f" Creating WBD buffer and clip version {huc}")
+        logging.info(f"-- {huc} : Creating WBD buffer and clip version")
 
         clip_wbd8_subprocess = subprocess.run(
             [
@@ -470,36 +477,38 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
             universal_newlines=True,
         )
 
-        # msg = clip_wbd8_subprocess.stdout
-        # print(f"{huc} : {msg}")
-        # logging.info(f"{huc} : {msg}")
-
-        if clip_wbd8_subprocess.stderr != "":
-            if "ERROR" in clip_wbd8_subprocess.stderr.upper():
-                msg = (
-                    f" - Creating -- {huc_directory}/wbd.gpkg"
-                    f"  ERROR -- details: ({clip_wbd8_subprocess.stderr})"
-                )
-                print(msg)
-                logging.info(msg)
+        sub_proc_stdout = clip_wbd8_subprocess.stdout
+        sub_proc_stderror = clip_wbd8_subprocess.stderr
+        if sub_proc_stderror.lower().startswith("warning") is True:
+            logging.info(f"Warning from ogr clip: {sub_proc_stderror}")
+        elif sub_proc_stderror != "":
+            raise Exception(
+                f"*** {huc} : Error WBD buffer and clip version from ogr for {huc}:"
+                f" Details: {sub_proc_stderror}\n"
+            )
         else:
-            msg = f" - Creating -- {huc_directory}/wbd.gpkg - Complete"
-            print(msg)
+            msg = f"-- {huc} : Creating WBD buffer and clip version - Complete"
+            if sub_proc_stdout != "":  # warnings?
+                msg += f": {sub_proc_stdout}"
+
             logging.info(msg)
 
     except Exception:
-        print(f"*** An error occurred while processing {huc}")
-        print(traceback.format_exc())
         logging.info(f"*** An error occurred while processing {huc}")
         logging.info(traceback.format_exc())
-        print()
+        logging.info("")
+        in_error = True
 
     huc_processing_end = dt.datetime.now(dt.timezone.utc)
     time_duration = huc_processing_end - huc_processing_start
-    duraton_msg = f"\t \t run time for huc {huc}: is {str(time_duration).split('.')[0]}"
-    print(duraton_msg)
+    duraton_msg = f"-- {huc} : run time is {str(time_duration).split('.')[0]}\n"
     logging.info(duraton_msg)
-    return
+
+    # if in error return the failed huc number
+    if in_error is True:
+        return huc
+    else:
+        return "Success"
 
 
 if __name__ == '__main__':
@@ -542,4 +551,9 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
 
     args = vars(parser.parse_args())
 
-    pre_clip_hucs_from_wbd(**args)
+    try:
+        pre_clip_hucs_from_wbd(**args)
+    except Exception:
+        logging.info(traceback.format_exc())
+        end_time = dt.datetime.now(dt.timezone.utc)
+        logging.info(f"   End time: {end_time.strftime('%m/%d/%Y %H:%M:%S')}")
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d4f4011f4..07f1ed850 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,23 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.4 - 2024-07-08 - [PR#1204](https://github.com/NOAA-OWP/inundation-mapping/pull/1204)
+
+Bug fix for extending outlets in order to ensure proper flow direction in depression filling algorithm. This PR adds a distance criteria that in order for the end of an outlet stream to be snapped to the wbd_buffered boundary, the end point must be less than 100 meters from the WBD boundary.
+
+Also adds missing argparse arguments so that the script can be run from the command line.
+
+### Changes
+
+- `data`
+     - `wbd`
+          - `clip_vectors_to_wbd.py`: Adds a 100 meter distance threshold to WBD to snap outlets to buffered WBD.
+          - `generate_pre_clip_fim_huc8.py`: Upgrading logging system.
+- `src`
+     - `bash_variables.env`: Updated pre-clip input path to new pre-clip files.
+
+<br/><br/>
+
 ## v4.5.2.3 - 2024-06-14 - [PR#1169](https://github.com/NOAA-OWP/inundation-mapping/pull/1169)
 
 This tool scans all log directory looking for the word "error" (not case-sensitive). This is primary added to help find errors in the post processing logs such as src_optimization folder (and others).
@@ -12,7 +29,6 @@ This tool scans all log directory looking for the word "error" (not case-sensiti
 
 <br/><br/>
 
-
 ## v4.5.2.2 - 2024-06-14 - [PR#1183](https://github.com/NOAA-OWP/inundation-mapping/pull/1183)
 
 Upgrades whitebox from v2.3.1 to 2.3.4. Whitebox was upgraded by `pip install -U whitebox` and then `pipenv lock` to update the `Pipfile`.
@@ -172,6 +188,10 @@ Some NWM streams, particularly in coastal areas, fail to reach the edge of the D
 ### Changes
 
 - `data/wbd/clip_vectors_to_wbd.py`: Clips `landsea` ocean mask from the buffered WBD and adds a function to extend outlet streams to the buffered WBD
+
+<br/><br/>
+
+
 - `data/wbd/clip_vectors_to_wbd.py`: Updated multi-processing and added more logging.
 
 <br/><br/>
@@ -313,6 +333,7 @@ The "black" packages is also be upgraded from 23.7.0 to 24.3.
 
 <br/><br/>
 
+
 ## v4.4.13.1 - 2024-03-11 - [PR#1086](https://github.com/NOAA-OWP/inundation-mapping/pull/1086)
 
 Fixes bug where levee-protected areas were not being masked from branch 0 DEMs.
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 42513fad8..0416ff087 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -6,6 +6,9 @@ export             DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
 export                             ALASKA_CRS=EPSG:3338
 
 # NOTE: $inputsDir is defined in Dockerfile
+
+export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/20240702
+
 export                              input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
 export                       input_DEM_Alaska=${inputsDir}/3dep_dems/10m_South_Alaska/23_11_07/FIM_3dep_dem_South_Alask_10m.vrt
 export                       input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
@@ -28,7 +31,6 @@ export                  input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_
 export                          input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
 export                   input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg
 export                 input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
-export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/240502
 export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
 export                            osm_bridges=${inputsDir}/osm/bridges/240426/osm_all_bridges.gpkg
 

From 9c1f3af41924a31fa1eddb65177f3e3a96a25064 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 8 Jul 2024 10:56:06 -0700
Subject: [PATCH 328/359] v4.5.2.5 Snap crosswalk to NWM streams (#1205)

---
 docs/CHANGELOG.md    |  11 +++
 src/add_crosswalk.py | 189 +++++++++++++++----------------------------
 2 files changed, 74 insertions(+), 126 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 07f1ed850..35de89aed 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.5 - 2024-07-08 - [PR#1205](https://github.com/NOAA-OWP/inundation-mapping/pull/1205)
+
+Snaps crosswalk from the midpoint of DEM-derived reaches to the nearest point on NWM streams within a threshold of 100 meters. DEM-derived streams that do not locate any NWM streams within 100 meters of their midpoints are removed from the FIM hydrofabric and their catchments are not inundated.
+
+### Changes
+
+- `src/add_crosswalk.py`: Locates nearest NWM stream to midpoint of DEM-derived reaches if within 100 meters. Also fixes a couple of minor bugs. 
+
+<br/><br/>
+
+
 ## v4.5.2.4 - 2024-07-08 - [PR#1204](https://github.com/NOAA-OWP/inundation-mapping/pull/1204)
 
 Bug fix for extending outlets in order to ensure proper flow direction in depression filling algorithm. This PR adds a distance criteria that in order for the end of an outlet stream to be snapped to the wbd_buffered boundary, the end point must be less than 100 meters from the WBD boundary.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index ddc9444df..62789c9b3 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -40,130 +40,70 @@ def add_crosswalk(
     input_catchments = gpd.read_file(input_catchments_fileName, engine="pyogrio", use_arrow=True)
     input_flows = gpd.read_file(input_flows_fileName, engine="pyogrio", use_arrow=True)
     input_huc = gpd.read_file(input_huc_fileName, engine="pyogrio", use_arrow=True)
+    input_nwmcat = gpd.read_file(input_nwmcat_fileName, engine="pyogrio", use_arrow=True)
     input_nwmflows = gpd.read_file(input_nwmflows_fileName, engine="pyogrio", use_arrow=True)
     min_catchment_area = float(min_catchment_area)  # 0.25#
     min_stream_length = float(min_stream_length)  # 0.5#
 
-    if extent == 'FR':
-        ## crosswalk using majority catchment method
-
-        # calculate majority catchments
-        majority_calc = zonal_stats(
-            input_catchments, input_nwmcatras_fileName, stats=['majority'], geojson_out=True
-        )
-        input_majorities = gpd.GeoDataFrame.from_features(majority_calc)
-        input_majorities = input_majorities.rename(columns={'majority': 'feature_id'})
-
-        input_majorities = input_majorities[:][input_majorities['feature_id'].notna()]
-        if input_majorities.feature_id.dtype != 'int':
-            input_majorities.feature_id = input_majorities.feature_id.astype(int)
-        if input_majorities.HydroID.dtype != 'int':
-            input_majorities.HydroID = input_majorities.HydroID.astype(int)
-
-        input_nwmflows = input_nwmflows.rename(columns={'ID': 'feature_id'})
-        if input_nwmflows.feature_id.dtype != 'int':
-            input_nwmflows.feature_id = input_nwmflows.feature_id.astype(int)
-        relevant_input_nwmflows = input_nwmflows[
-            input_nwmflows['feature_id'].isin(input_majorities['feature_id'])
-        ]
-        relevant_input_nwmflows = relevant_input_nwmflows.filter(items=['feature_id', 'order_'])
-
-        if input_catchments.HydroID.dtype != 'int':
-            input_catchments.HydroID = input_catchments.HydroID.astype(int)
-        output_catchments = input_catchments.merge(input_majorities[['HydroID', 'feature_id']], on='HydroID')
-        output_catchments = output_catchments.merge(
-            relevant_input_nwmflows[['order_', 'feature_id']], on='feature_id'
-        )
-
-        if input_flows.HydroID.dtype != 'int':
-            input_flows.HydroID = input_flows.HydroID.astype(int)
-        output_flows = input_flows.merge(input_majorities[['HydroID', 'feature_id']], on='HydroID')
-        if output_flows.HydroID.dtype != 'int':
-            output_flows.HydroID = output_flows.HydroID.astype(int)
-        output_flows = output_flows.merge(relevant_input_nwmflows[['order_', 'feature_id']], on='feature_id')
-        output_flows = output_flows.merge(
-            output_catchments.filter(items=['HydroID', 'areasqkm']), on='HydroID'
-        )
-
-    elif (extent == 'MS') | (extent == 'GMS'):
-        ## crosswalk using stream segment midpoint method
-        input_nwmcat = gpd.read_file(input_nwmcat_fileName, mask=input_huc, engine="fiona")
-
-        # only reduce nwm catchments to mainstems if running mainstems
-        if extent == 'MS':
-            input_nwmcat = input_nwmcat.loc[input_nwmcat.mainstem == 1]
-
-        input_nwmcat = input_nwmcat.rename(columns={'ID': 'feature_id'})
-        if input_nwmcat.feature_id.dtype != 'int':
-            input_nwmcat.feature_id = input_nwmcat.feature_id.astype(int)
-        input_nwmcat = input_nwmcat.set_index('feature_id')
-
-        input_nwmflows = input_nwmflows.rename(columns={'ID': 'feature_id'})
-        if input_nwmflows.feature_id.dtype != 'int':
-            input_nwmflows.feature_id = input_nwmflows.feature_id.astype(int)
-
-        # Get stream midpoint
-        stream_midpoint = []
-        hydroID = []
-        for i, lineString in enumerate(input_flows.geometry):
-            hydroID = hydroID + [input_flows.loc[i, 'HydroID']]
-            stream_midpoint = stream_midpoint + [lineString.interpolate(0.5, normalized=True)]
-
-        input_flows_midpoint = gpd.GeoDataFrame(
-            {'HydroID': hydroID, 'geometry': stream_midpoint}, crs=input_flows.crs, geometry='geometry'
-        )
-        input_flows_midpoint = input_flows_midpoint.set_index('HydroID')
-
-        # Create crosswalk
-        crosswalk = gpd.sjoin(
-            input_flows_midpoint, input_nwmcat, how='left', predicate='within'
-        ).reset_index()
-        crosswalk = crosswalk.rename(columns={"index_right": "feature_id"})
-
-        # fill in missing ms
-        crosswalk_missing = crosswalk.loc[crosswalk.feature_id.isna()]
-        for index, stream in crosswalk_missing.iterrows():
-            # find closest nwm catchment by distance
-            distances = [stream.geometry.distance(poly) for poly in input_nwmcat.geometry]
-            min_dist = min(distances)
-            nwmcat_index = distances.index(min_dist)
-
-            # update crosswalk
-            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'feature_id'] = input_nwmcat.iloc[
-                nwmcat_index
-            ].name
-            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'AreaSqKM'] = input_nwmcat.iloc[
-                nwmcat_index
-            ].AreaSqKM
-            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'Shape_Length'] = input_nwmcat.iloc[
-                nwmcat_index
-            ].Shape_Length
-            crosswalk.loc[crosswalk.HydroID == stream.HydroID, 'Shape_Area'] = input_nwmcat.iloc[
-                nwmcat_index
-            ].Shape_Area
-
-        crosswalk = crosswalk.filter(items=['HydroID', 'feature_id'])
-        crosswalk = crosswalk.merge(input_nwmflows[['feature_id', 'order_']], on='feature_id')
-
-        if len(crosswalk) < 1:
-            print("No relevant streams within HUC boundaries.")
-            sys.exit(0)
-
-        if input_catchments.HydroID.dtype != 'int':
-            input_catchments.HydroID = input_catchments.HydroID.astype(int)
-        output_catchments = input_catchments.merge(crosswalk, on='HydroID')
-
-        if input_flows.HydroID.dtype != 'int':
-            input_flows.HydroID = input_flows.HydroID.astype(int)
-        output_flows = input_flows.merge(crosswalk, on='HydroID')
-
-        # added for GMS. Consider adding filter_catchments_and_add_attributes.py to run_by_branch.sh
-        if 'areasqkm' not in output_catchments.columns:
-            output_catchments['areasqkm'] = output_catchments.geometry.area / (1000**2)
-
-        output_flows = output_flows.merge(
-            output_catchments.filter(items=['HydroID', 'areasqkm']), on='HydroID'
-        )
+    input_catchments = input_catchments.dissolve(by='HydroID').reset_index()
+
+    ## crosswalk using stream segment midpoint method
+    input_nwmcat = gpd.read_file(input_nwmcat_fileName, mask=input_huc, engine="fiona")
+
+    # only reduce nwm catchments to mainstems if running mainstems
+    if extent == 'MS':
+        input_nwmcat = input_nwmcat.loc[input_nwmcat.mainstem == 1]
+
+    input_nwmcat = input_nwmcat.rename(columns={'ID': 'feature_id'})
+    if input_nwmcat.feature_id.dtype != 'int':
+        input_nwmcat.feature_id = input_nwmcat.feature_id.astype(int)
+    input_nwmcat = input_nwmcat.set_index('feature_id')
+
+    input_nwmflows = input_nwmflows.rename(columns={'ID': 'feature_id'})
+    if input_nwmflows.feature_id.dtype != 'int':
+        input_nwmflows.feature_id = input_nwmflows.feature_id.astype(int)
+    input_nwmflows = input_nwmflows.set_index('feature_id')
+
+    # Get stream midpoint
+    stream_midpoint = []
+    hydroID = []
+    for i, lineString in enumerate(input_flows.geometry):
+        hydroID = hydroID + [input_flows.loc[i, 'HydroID']]
+        stream_midpoint = stream_midpoint + [lineString.interpolate(0.5, normalized=True)]
+
+    input_flows_midpoint = gpd.GeoDataFrame(
+        {'HydroID': hydroID, 'geometry': stream_midpoint}, crs=input_flows.crs, geometry='geometry'
+    )
+    input_flows_midpoint = input_flows_midpoint.set_index('HydroID')
+
+    # Create crosswalk
+    crosswalk = gpd.sjoin_nearest(
+        input_flows_midpoint, input_nwmflows, how='left', distance_col='distance'
+    ).reset_index()
+    crosswalk = crosswalk.rename(columns={"index_right": "feature_id"})
+
+    crosswalk.loc[crosswalk['distance'] > 100.0, 'feature_id'] = pd.NA
+
+    crosswalk = crosswalk.filter(items=['HydroID', 'feature_id', 'distance'])
+    crosswalk = crosswalk.merge(input_nwmflows[['order_']], on='feature_id')
+
+    if len(crosswalk) < 1:
+        print("No relevant streams within HUC boundaries.")
+        sys.exit(0)
+
+    if input_catchments.HydroID.dtype != 'int':
+        input_catchments.HydroID = input_catchments.HydroID.astype(int)
+    output_catchments = input_catchments.merge(crosswalk, on='HydroID')
+
+    if input_flows.HydroID.dtype != 'int':
+        input_flows.HydroID = input_flows.HydroID.astype(int)
+    output_flows = input_flows.merge(crosswalk, on='HydroID')
+
+    # added for GMS. Consider adding filter_catchments_and_add_attributes.py to run_by_branch.sh
+    if 'areasqkm' not in output_catchments.columns:
+        output_catchments['areasqkm'] = output_catchments.geometry.area / (1000**2)
+
+    output_flows = output_flows.merge(output_catchments.filter(items=['HydroID', 'areasqkm']), on='HydroID')
 
     output_flows['ManningN'] = mannings_n
 
@@ -223,7 +163,7 @@ def add_crosswalk(
             elif len(output_flows.loc[output_flows.From_Node == to_node]['HydroID']) > 1:
                 try:
                     max_order = max(
-                        output_flows.loc[output_flows.From_Node == to_node]['HydroID']['order_']
+                        output_flows.loc[output_flows.From_Node == to_node]['order_']
                     )  # drainage area would be better than stream order but we would need to calculate
                 except Exception as e:
                     print(
@@ -276,7 +216,7 @@ def add_crosswalk(
                 update_id = output_flows.loc[output_flows.From_Node == to_node]['HydroID'].item()
 
             else:
-                update_id = output_flows.loc[output_flows.HydroID == short_id]['HydroID'].item()
+                update_id = output_flows[output_flows.HydroID == short_id]['HydroID'].iloc[0]
 
             output_order = output_flows.loc[output_flows.HydroID == short_id]['order_']
             if len(output_order) == 1:
@@ -347,10 +287,7 @@ def add_crosswalk(
                     ['Discharge (m3s-1)'],
                 ] = src_stage[1]
 
-    if extent == 'FR':
-        output_src = output_src.merge(input_majorities[['HydroID', 'feature_id']], on='HydroID')
-    elif (extent == 'MS') | (extent == 'GMS'):
-        output_src = output_src.merge(crosswalk[['HydroID', 'feature_id']], on='HydroID')
+    output_src = output_src.merge(crosswalk[['HydroID', 'feature_id']], on='HydroID')
 
     output_crosswalk = output_src[['HydroID', 'feature_id']]
     output_crosswalk = output_crosswalk.drop_duplicates(ignore_index=True)

From 56748d43ea4317a4103f496c74720dbe52d7cb6d Mon Sep 17 00:00:00 2001
From: ZahraGhahremani-NOAA
 <100253864+ZahraGhahremani@users.noreply.github.com>
Date: Fri, 12 Jul 2024 13:26:30 -0600
Subject: [PATCH 329/359] v4.5.2.6 Added Bridge Inundation tool  (#1184)

---
 docs/CHANGELOG.md          |  11 +++
 tools/bridge_inundation.py | 153 +++++++++++++++++++++++++++++++++++++
 2 files changed, 164 insertions(+)
 create mode 100644 tools/bridge_inundation.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 35de89aed..539c53fa2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.6 - 2024-07-12 - [PR#1184](https://github.com/NOAA-OWP/inundation-mapping/pull/1184)
+
+This PR adds a new script to determine which bridges are inundated by a specific flow. It will assign a risk status to each bridge point based on a specific threshold.
+
+### Additions
+
+- `tools/bridge_inundation.py`
+
+<br/><br/>
+
+
 ## v4.5.2.5 - 2024-07-08 - [PR#1205](https://github.com/NOAA-OWP/inundation-mapping/pull/1205)
 
 Snaps crosswalk from the midpoint of DEM-derived reaches to the nearest point on NWM streams within a threshold of 100 meters. DEM-derived streams that do not locate any NWM streams within 100 meters of their midpoints are removed from the FIM hydrofabric and their catchments are not inundated.
diff --git a/tools/bridge_inundation.py b/tools/bridge_inundation.py
new file mode 100644
index 000000000..6f9238196
--- /dev/null
+++ b/tools/bridge_inundation.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+
+import argparse
+import errno
+import os
+import re
+from timeit import default_timer as timer
+
+import geopandas as gpd
+import pandas as pd
+
+
+def bridge_risk_status(
+    hydrofabric_dir: str, flow_file: str, output_dir: str, limit_hucs: list = []
+) -> gpd.GeoDataFrame:
+    """
+    This function detect which bridge points are affected by a specified flow file. The function requires a flow file (expected to follow
+    the schema used by 'inundation_mosaic_wrapper') with data organized by 'feature_id' and 'discharge' in cms. The output includes a geopackage
+    containing bridge points labeled as "threatened", "at risk", or "not at risk" based on forcasted discharge compared to preset discharge
+    ("max_discharge" or "max_discharge75").
+
+    Args:
+        hydrofabric_dir (str):    Path to hydrofabric directory where FIM outputs were written by
+                                    fim_pipeline.
+        flow_file (str):      Path to flow file to be used for inundation.
+                                    feature_ids in flow_file should be present in supplied HUC.
+        output (str):             Path to output geopackage.
+        limit_hucs (list):    Optional. If specified, only the bridges in these HUCs will be processed.
+
+    Example usage:
+    python /foss_fim/tools/bridge_inundation.py \
+        -y /data/previous_fim/fim_4_5_2_0 \
+        -f /data/ble_huc_12090301_flows_100yr.csv \
+        -o /home/user/Documents/bridges/inundated_bridge_pnts.gpkg \
+        -u 12090301 02020005
+    """
+
+    dir_path = hydrofabric_dir
+    # Check that hydrofabric_dir exists
+    if not os.path.exists(dir_path):
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), dir_path)
+
+    # Get the list of all hucs in the directory
+    entries = [d for d in os.listdir(dir_path) if re.match(r'^\d{8}$', d)]
+    hucs = []
+    for entry in entries:
+        # create the full path of the entry
+        full_path = os.path.join(dir_path, entry)
+        # check if the netry is a directory
+        if os.path.isdir(full_path):
+            hucs.append(entry)
+
+    # Check that flow file exists
+    if not os.path.exists(flow_file):
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), flow_file)
+
+    # Read the flow_file
+    dtype_dict = {'feature_id': str}
+    flow_file_data = pd.read_csv(flow_file, dtype=dtype_dict)
+
+    # Initialize an empty list to hold GeoDataFrames
+    gdfs = []
+
+    # Filter HUCs if specified
+    if limit_hucs:
+        hucs = [h for h in limit_hucs if h in hucs]
+
+    # Iterate through hucs
+    for huc in hucs:
+        print(f'Processing HUC: {huc}')
+        # Construct the file path
+        gpkg_path = os.path.join(dir_path, huc, 'osm_bridge_centroids.gpkg')
+        # Check if the file exists
+        if not os.path.exists(gpkg_path):
+            print(f"No GeoPackage file found in {gpkg_path}. Skipping...")
+            continue
+        # Open the bridge point GeoPackage for each huc
+        bri_po = gpd.read_file(gpkg_path)
+
+        # Save the origignal crs in a new column
+        bri_po['original_crs'] = bri_po.crs.to_string()
+
+        # Reproject to EPSG:4326
+        bri_po = bri_po.to_crs('epsg:4326')
+        gdfs.append(bri_po)
+
+    # Concatenate all GeoDataFrame into a single GeoDataFrame
+    bridge_points = gpd.GeoDataFrame(pd.concat(gdfs, ignore_index=True))
+
+    # Find the common feature_id between flow_file and bridge_points
+    merged_bri = bridge_points.merge(flow_file_data, on='feature_id', how='inner')
+
+    # Assign risk status for each point
+    def risk_class(row):
+        if row['discharge'] > row['max_discharge']:
+            return 'threatened'
+        elif row['max_discharge75'] <= row['discharge'] < row['max_discharge']:
+            return 'at_risk'
+        else:
+            return 'not_at_risk'
+
+    # Apply risk_class function to each row
+    merged_bri['risk_status'] = merged_bri.apply(risk_class, axis=1)
+    merged_bri.drop('discharge', axis=1, inplace=True)
+
+    # Drop not_at_risk status from points with the same geometry
+    mapping_dic = {'not_at_risk': 0, 'at_risk': 1, 'threatened': 2}
+    merged_bri['risk'] = merged_bri['risk_status'].map(mapping_dic)
+    merged_bri.reset_index(drop=True, inplace=True)
+    merged_data_max = merged_bri.groupby('geometry')['risk'].idxmax()
+    bridge_out = merged_bri.loc[merged_data_max]
+    bridge_out.reset_index(drop=True, inplace=True)
+    bridge_out.drop('risk', axis=1, inplace=True)
+    bridge_out.to_file(output_dir, driver='GPKG', layer='bridge_risk_status')
+
+    return bridge_out
+
+
+if __name__ == "__main__":
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description="Detect which bridge points are affected by a specified flow file."
+    )
+    parser.add_argument(
+        "-y",
+        "--hydrofabric_dir",
+        help="Directory path to FIM hydrofabric by processing unit.",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        "-f",
+        "--flow_file",
+        help='Discharges in CMS as CSV file. "feature_id" and "discharge" columns MUST be supplied.',
+        required=True,
+        type=str,
+    )
+    parser.add_argument("-o", "--output_dir", help="Path to geopackage output.", required=True, type=str)
+    parser.add_argument(
+        "-u",
+        "--limit_hucs",
+        help="Optional. If specified, only the bridges in these HUCs will be processed.",
+        required=False,
+        type=str,
+        nargs="+",
+    )
+
+    start = timer()
+
+    # Extract to dictionary and run
+    bridge_risk_status(**vars(parser.parse_args()))
+
+    print(f"Completed in {round((timer() - start)/60, 2)} minutes.")

From 7f16ab1b3805c7e399876ba4fc40bdfdcab659e5 Mon Sep 17 00:00:00 2001
From: ZahraGhahremani-NOAA
 <100253864+ZahraGhahremani@users.noreply.github.com>
Date: Fri, 19 Jul 2024 13:08:53 -0600
Subject: [PATCH 330/359] v4.5.2.7 Updating hydroTable and src
 _full_crosswalked (#1220)

---
 config/deny_branch_zero.lst |  2 +-
 config/deny_branches.lst    |  2 +-
 docs/CHANGELOG.md           | 16 +++++++
 fim_post_processing.sh      | 42 +++++++++++++++++++
 src/update_htable_src.py    | 83 +++++++++++++++++++++++++++++++++++++
 5 files changed, 143 insertions(+), 2 deletions(-)
 create mode 100644 src/update_htable_src.py

diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst
index f4bf2cba9..409c306ff 100644
--- a/config/deny_branch_zero.lst
+++ b/config/deny_branch_zero.lst
@@ -56,7 +56,7 @@ slopes_d8_dem_meters_{}.tif
 slopes_d8_dem_meters_masked_{}.tif
 sn_catchments_reaches_{}.tif
 src_{}.json
-src_base_{}.csv
+#src_base_{}.csv
 #src_full_crosswalked_{}.csv
 stage_{}.txt
 streamOrder_{}.tif
diff --git a/config/deny_branches.lst b/config/deny_branches.lst
index 07ef1f7f4..a44a79a49 100644
--- a/config/deny_branches.lst
+++ b/config/deny_branches.lst
@@ -58,7 +58,7 @@ slopes_d8_dem_meters_{}.tif
 slopes_d8_dem_meters_masked_{}.tif
 sn_catchments_reaches_{}.tif
 src_{}.json
-src_base_{}.csv
+# src_base_{}.csv
 # src_full_crosswalked_{}.csv
 stage_{}.txt
 streamOrder_{}.tif
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 539c53fa2..81115c47d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,22 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.7 - 2024-07-19 - [PR#1220](https://github.com/NOAA-OWP/inundation-mapping/pull/1220)
+
+With this PR we can run post_processing.sh multiple times on a processed batch without any concerns that it may change the hydroTable or src_full_crosswalked files.
+
+### Additions
+
+- `src/update_htable_src.py`
+
+### Changes
+
+-  `config/deny_branch_zero.lst`
+-  `config/deny_branches.lst`
+-  `fim_post_processing.sh`
+
+<br/><br/>
+
 ## v4.5.2.6 - 2024-07-12 - [PR#1184](https://github.com/NOAA-OWP/inundation-mapping/pull/1184)
 
 This PR adds a new script to determine which bridges are inundated by a specific flow. It will assign a risk status to each bridge point based on a specific threshold.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 2a7965efb..5d0385c55 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -90,6 +90,47 @@ echo "---- Started: `date -u`"
 T_total_start
 post_proc_start_time=`date +%s`
 
+## RUN UPDATE HYDROTABLE AND SRC ##
+# Define the counter file
+
+COUNTER_FILE="${outputDestDir}/post_processing_attempt.txt"
+# Function to clean up
+cleanup() {
+    if [ "$SUCCESS" = true ]; then
+        if [ -f "$COUNTER_FILE" ]; then
+            COUNTER=$(cat "$COUNTER_FILE")
+            if [ "$COUNTER" -eq 1 ]; then
+                echo "Counter is 1. Removing the counter file."
+                rm "$COUNTER_FILE"
+            fi
+        fi
+    fi
+}
+
+# Set up trap to call cleanup on EXIT, ERR, and INT (interrupt signal)
+trap cleanup EXIT ERR INT
+# Initialize the counter file if it doesn't exist
+if [ ! -f "$COUNTER_FILE" ]; then
+    echo 0 > "$COUNTER_FILE"
+fi
+
+# Read the current counter value
+COUNTER=$(cat "$COUNTER_FILE")
+
+# Increment the counter
+COUNTER=$((COUNTER + 1))
+
+# Save the new counter value
+echo "$COUNTER" > "$COUNTER_FILE"
+
+# Check if the counter is greater than one
+if [ "$COUNTER" -gt 1 ]; then
+    # Execute the Python file
+    echo "Updating hydroTable & scr_full_crosswalked for branches"
+    python3 $srcDir/update_htable_src.py -d $outputDestDir
+else
+    echo "Execution count is $COUNTER, not executing the update_htable_src.py file."
+fi
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
 echo -e $startDiv"Start branch aggregation"
@@ -240,3 +281,4 @@ echo "---- End of fim_post_processing"
 echo "---- Ended: `date -u`"
 Calc_Duration $post_proc_start_time
 echo
+SUCCESS=true
diff --git a/src/update_htable_src.py b/src/update_htable_src.py
new file mode 100644
index 000000000..e2828afd6
--- /dev/null
+++ b/src/update_htable_src.py
@@ -0,0 +1,83 @@
+import argparse
+import os
+import re
+
+import geopandas as gpd
+import pandas as pd
+
+
+def process_branch(sub_branch_path, branch):
+    src_base_file = os.path.join(sub_branch_path, f'src_base_{branch}.csv')
+    hydro_table_file = os.path.join(sub_branch_path, f'hydroTable_{branch}.csv')
+    src_full_file = os.path.join(sub_branch_path, f'src_full_crosswalked_{branch}.csv')
+    input_flows_file = os.path.join(
+        sub_branch_path, f'demDerived_reaches_split_filtered_addedAttributes_crosswalked_{branch}.gpkg'
+    )
+
+    input_src_base = pd.read_csv(src_base_file, dtype=object)
+    input_src_full = pd.read_csv(src_full_file, dtype=object)
+    input_hydro_table = pd.read_csv(hydro_table_file, dtype=object)
+    input_flows = gpd.read_file(input_flows_file, engine="pyogrio", use_arrow=True)
+
+    input_src_base = input_src_base.merge(
+        input_flows[['ManningN', 'HydroID', 'NextDownID', 'order_']], left_on='CatchId', right_on='HydroID'
+    )
+
+    # Update src_full
+    input_src_base = input_src_base.rename(columns=lambda x: x.strip(" "))
+    input_src_base = input_src_base.apply(pd.to_numeric, **{'errors': 'coerce'})
+    input_src_full['Volume (m3)'] = input_src_base['Volume (m3)']
+    input_src_full['BedArea (m2)'] = input_src_base['BedArea (m2)']
+    input_src_full['TopWidth (m)'] = input_src_base['SurfaceArea (m2)'] / input_src_base['LENGTHKM'] / 1000
+    input_src_full['WettedPerimeter (m)'] = input_src_base['BedArea (m2)'] / input_src_base['LENGTHKM'] / 1000
+    input_src_full['WetArea (m2)'] = input_src_base['Volume (m3)'] / input_src_base['LENGTHKM'] / 1000
+    input_src_full['HydraulicRadius (m)'] = (
+        input_src_full['WetArea (m2)'] / input_src_full['WettedPerimeter (m)']
+    )
+    input_src_full['HydraulicRadius (m)'].fillna(0, inplace=True)
+    input_src_full['Discharge (m3s-1)'] = (
+        input_src_full['WetArea (m2)']
+        * pow(input_src_full['HydraulicRadius (m)'], 2.0 / 3)
+        * pow(input_src_base['SLOPE'], 0.5)
+        / input_src_base['ManningN']
+    )
+    input_src_full['Bathymetry_source'] = pd.NA
+    input_src_full = input_src_full.iloc[:, :19]
+
+    # Update hydroTable
+    input_hydro_table['subdiv_discharge_cms'] = pd.NA
+    input_hydro_table['discharge_cms'] = input_hydro_table['default_discharge_cms']
+
+    # Save updated files
+    input_src_full.to_csv(src_full_file, index=False)
+    input_hydro_table.to_csv(hydro_table_file, index=False)
+
+
+def reset_hydro_and_src(fim_dir):
+    hucs = [h for h in os.listdir(fim_dir) if re.match(r'^\d{8}$', h)]
+    for huc_folder in hucs:
+        huc_path = os.path.join(fim_dir, huc_folder)
+        if os.path.isdir(huc_path):
+            for branch_folder in os.listdir(huc_path):
+                branch_path = os.path.join(huc_path, branch_folder)
+                if os.path.isdir(branch_path):
+                    for branch in os.listdir(branch_path):
+                        sub_branch_path = os.path.join(branch_path, branch)
+                        if os.path.isdir(sub_branch_path):
+                            process_branch(sub_branch_path, branch)
+
+
+# Example usage:
+# reset_hydro_and_src('/path/to/fim_dir')
+if __name__ == '__main__':
+    '''
+    Sample usage (min params):
+        python3 src/update_htable_src.py
+            -d /data/previous_fim/fim_4_5_2_0
+    '''
+    parser = argparse.ArgumentParser(description='Update hydrotable and src files.')
+    parser.add_argument('-d', '--fim_dir', help='Directory path for fim_pipeline output.', required=True)
+
+    args = parser.parse_args()
+
+    reset_hydro_and_src(args.fim_dir)

From 5006dbb8a02b929db0f3784f04f89405284415ed Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 19 Jul 2024 13:13:32 -0600
Subject: [PATCH 331/359] v4.5.2.8 Change error messages to warnings (#1219)

---
 docs/CHANGELOG.md                 | 13 ++++++++++++-
 src/bathymetric_adjustment.py     |  2 +-
 src/src_roughness_optimization.py | 14 +++++++-------
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 81115c47d..7e68f479c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.8 - 2024-07-19 - [PR#1219](https://github.com/NOAA-OWP/inundation-mapping/pull/1219)
+
+Changes non-fatal `ERROR` messages to `WARNINGS` to avoid triggering being logged as errors.
+
+### Changes
+
+- `src/`
+    - `bathymetric_adjustment.py`: Changes `WARNING` to `ERROR` in Exception
+    - `src_roughness_optimization.py`: Changes `ERROR` messages to `WARNING`
+
+<br/><br/>
+
 ## v4.5.2.7 - 2024-07-19 - [PR#1220](https://github.com/NOAA-OWP/inundation-mapping/pull/1220)
 
 With this PR we can run post_processing.sh multiple times on a processed batch without any concerns that it may change the hydroTable or src_full_crosswalked files.
@@ -38,7 +50,6 @@ Snaps crosswalk from the midpoint of DEM-derived reaches to the nearest point on
 
 <br/><br/>
 
-
 ## v4.5.2.4 - 2024-07-08 - [PR#1204](https://github.com/NOAA-OWP/inundation-mapping/pull/1204)
 
 Bug fix for extending outlets in order to ensure proper flow direction in depression filling algorithm. This PR adds a distance criteria that in order for the end of an outlet stream to be snapped to the wbd_buffered boundary, the end point must be less than 100 meters from the WBD boundary.
diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py
index 9c114173b..fa43da473 100644
--- a/src/bathymetric_adjustment.py
+++ b/src/bathymetric_adjustment.py
@@ -197,7 +197,7 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
             try:
                 log_file.write(future.result())
             except Exception as ex:
-                print(f"WARNING: {executor_dict[future]} BARC failed for some reason")
+                print(f"ERROR: {executor_dict[future]} BARC failed for some reason")
                 log_file.write(f"ERROR --> {executor_dict[future]} BARC failed (details: *** {ex} )\n")
                 traceback.print_exc(file=log_file)
 
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index 7f3864875..3f33e909c 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -128,7 +128,7 @@ def update_rating_curve(
     elif source_tag == 'ras2fim_rating':
         calb_type = 'calb_coef_ras2fim'
     else:
-        log_text += "ERROR - unknown calibration data source type: " + str(source_tag) + '\n'
+        log_text += "WARNING - unknown calibration data source type: " + str(source_tag) + '\n'
 
     ## Read in the hydroTable.csv and check wether it has previously been updated
     # (rename default columns if needed)
@@ -202,7 +202,7 @@ def update_rating_curve(
     for index, row in df_nvalues.iterrows():
         if row.hydroid not in df_htable['HydroID'].values:
             print(
-                'ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: '
+                'WARNING: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: '
                 + str(huc)
                 + '  branch id: '
                 + str(branch_id)
@@ -210,7 +210,7 @@ def update_rating_curve(
                 + str(row.hydroid)
             )
             log_text += (
-                'ERROR: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: '
+                'WARNING: HydroID for calb point was not found in the hydrotable (check hydrotable) for HUC: '
                 + str(huc)
                 + '  branch id: '
                 + str(branch_id)
@@ -224,7 +224,7 @@ def update_rating_curve(
             df_htable_hydroid = df_htable[(df_htable.HydroID == row.hydroid) & (df_htable.stage > 0)]
             if df_htable_hydroid.empty:
                 print(
-                    'ERROR: df_htable_hydroid is empty but expected data: '
+                    'WARNING: df_htable_hydroid is empty but expected data: '
                     + str(huc)
                     + '  branch id: '
                     + str(branch_id)
@@ -232,7 +232,7 @@ def update_rating_curve(
                     + str(row.hydroid)
                 )
                 log_text += (
-                    'ERROR: df_htable_hydroid is empty but expected data: '
+                    'WARNING: df_htable_hydroid is empty but expected data: '
                     + str(huc)
                     + '  branch id: '
                     + str(branch_id)
@@ -257,13 +257,13 @@ def update_rating_curve(
 
     if 'discharge_cms' not in df_nvalues:
         print(
-            'ERROR: "discharge_cms" column does not exist in df_nvalues df: '
+            'WARNING: "discharge_cms" column does not exist in df_nvalues df: '
             + str(huc)
             + '  branch id: '
             + str(branch_id)
         )
         log_text += (
-            'ERROR: "discharge_cms" column does not exist in df_nvalues df: '
+            'WARNING: "discharge_cms" column does not exist in df_nvalues df: '
             + str(huc)
             + '  branch id: '
             + str(branch_id)

From fd64cc8b1a43a0ee026b84fe5049ab859287bac2 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 19 Jul 2024 13:18:45 -0600
Subject: [PATCH 332/359] v4.5.2.9 Add `NO_VALID_CROSSWALKS` to
 `FIM_exit_codes` (#1216)

---
 docs/CHANGELOG.md      | 12 ++++++++++++
 src/add_crosswalk.py   |  9 +++++++--
 src/process_branch.sh  |  5 +++++
 src/utils/fim_enums.py |  1 +
 4 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7e68f479c..acf02539f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.9 - 2024-07-19 - [PR#1216](https://github.com/NOAA-OWP/inundation-mapping/pull/1216)
+
+Adds `NO_VALID_CROSSWALKS` to `FIM_exit_codes` which is used when the crosswalk table or output_catchments DataFrame is empty. Removes branches that fail with `NO_VALID_CROSSWALKS`.
+
+### Changes
+    - `add_crosswalk.py`: Added `NO_VALID_CROSSWALKS` as exit status when crosswalk or output_catchments is empty
+    - `process_branch.sh`: Removed branches that fail with `NO_VALID_CROSSWALKS`
+    - `utils/fim_enums.py`: Added `NO_VALID_CROSSWALKS` to `FIM_exit_codes`
+
+<br/><br/>
+
+
 ## v4.5.2.8 - 2024-07-19 - [PR#1219](https://github.com/NOAA-OWP/inundation-mapping/pull/1219)
 
 Changes non-fatal `ERROR` messages to `WARNINGS` to avoid triggering being logged as errors.
diff --git a/src/add_crosswalk.py b/src/add_crosswalk.py
index 62789c9b3..61622b2e3 100755
--- a/src/add_crosswalk.py
+++ b/src/add_crosswalk.py
@@ -9,6 +9,7 @@
 from numpy import unique
 from rasterstats import zonal_stats
 
+from utils.fim_enums import FIM_exit_codes
 from utils.shared_functions import getDriver
 from utils.shared_variables import FIM_ID
 
@@ -87,14 +88,18 @@ def add_crosswalk(
     crosswalk = crosswalk.filter(items=['HydroID', 'feature_id', 'distance'])
     crosswalk = crosswalk.merge(input_nwmflows[['order_']], on='feature_id')
 
-    if len(crosswalk) < 1:
+    if crosswalk.empty:
         print("No relevant streams within HUC boundaries.")
-        sys.exit(0)
+        sys.exit(FIM_exit_codes.NO_VALID_CROSSWALKS.value)
 
     if input_catchments.HydroID.dtype != 'int':
         input_catchments.HydroID = input_catchments.HydroID.astype(int)
     output_catchments = input_catchments.merge(crosswalk, on='HydroID')
 
+    if output_catchments.empty:
+        print("No valid catchments remain.")
+        sys.exit(FIM_exit_codes.NO_VALID_CROSSWALKS.value)
+
     if input_flows.HydroID.dtype != 'int':
         input_flows.HydroID = input_flows.HydroID.astype(int)
     output_flows = input_flows.merge(crosswalk, on='HydroID')
diff --git a/src/process_branch.sh b/src/process_branch.sh
index edfa2aa8e..7a85a5627 100755
--- a/src/process_branch.sh
+++ b/src/process_branch.sh
@@ -38,6 +38,11 @@ do
         err_exists=1
         echo "***** Branch has no valid flowlines *****"
         rm -rf $tempHucDataDir/branches/$branchId/
+    elif [ $code -eq 64 ]; then
+        echo
+        err_exists=1
+        echo "***** Branch has no crosswalks *****"
+        rm -rf $tempHucDataDir/branches/$branchId/
     elif [ $code -ne 0 ]; then
         echo
         err_exists=1
diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py
index 7524e0277..cee14f63d 100644
--- a/src/utils/fim_enums.py
+++ b/src/utils/fim_enums.py
@@ -29,3 +29,4 @@ class FIM_exit_codes(Enum):
     NO_FLOWLINES_EXIST = 61
     EXCESS_UNIT_ERRORS = 62
     NO_BRANCH_LEVELPATHS_EXIST = 63
+    NO_VALID_CROSSWALKS = 64

From efd0b2ab68815cb6b76f350eaaaf66e07f018dec Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 19 Jul 2024 13:24:34 -0600
Subject: [PATCH 333/359] v4.5.2.10 Remove warning messages (#1224)

---
 docs/CHANGELOG.md                        | 15 ++++++++++++++-
 src/adjust_thalweg_lateral.py            |  4 ++--
 src/delineate_hydros_and_produce_HAND.sh |  2 +-
 src/getRasterInfoNative.py               |  3 +++
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index acf02539f..ca83bf29e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.10 - 2024-07-19 - [PR#1224](https://github.com/NOAA-OWP/inundation-mapping/pull/1224)
+
+Addresses warnings to reduce output messages.
+
+### Changes
+
+- `src/'
+    - `adjust_thalweg_lateral.py`: fixes number type
+    - `src/delineate_hydros_and_produce_HAND.sh`: removes division by zero warning
+    - `getRasterInfoNative.py`: adds `gdal.UseExceptions()`
+
+<br/><br/>
+
+
 ## v4.5.2.9 - 2024-07-19 - [PR#1216](https://github.com/NOAA-OWP/inundation-mapping/pull/1216)
 
 Adds `NO_VALID_CROSSWALKS` to `FIM_exit_codes` which is used when the crosswalk table or output_catchments DataFrame is empty. Removes branches that fail with `NO_VALID_CROSSWALKS`.
@@ -51,7 +65,6 @@ This PR adds a new script to determine which bridges are inundated by a specific
 
 <br/><br/>
 
-
 ## v4.5.2.5 - 2024-07-08 - [PR#1205](https://github.com/NOAA-OWP/inundation-mapping/pull/1205)
 
 Snaps crosswalk from the midpoint of DEM-derived reaches to the nearest point on NWM streams within a threshold of 100 meters. DEM-derived streams that do not locate any NWM streams within 100 meters of their midpoints are removed from the FIM hydrofabric and their catchments are not inundated.
diff --git a/src/adjust_thalweg_lateral.py b/src/adjust_thalweg_lateral.py
index 5a55b61fc..d14582e22 100755
--- a/src/adjust_thalweg_lateral.py
+++ b/src/adjust_thalweg_lateral.py
@@ -24,7 +24,7 @@ def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window
         for i, elev_m in enumerate(zone_window):
             # If the zone really exists in the dictionary, compare elevation values.
             i = int(i)
-            elev_m = int(elev_m)
+            elev_m = types.int32(elev_m)
 
             if cost_window[i] <= cost_tolerance:
                 if elevation_window[i] > 0:  # Don't allow bad elevation values
@@ -47,7 +47,7 @@ def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_w
 
         for i, elev_m in enumerate(zone_window):
             i = int(i)
-            elev_m = int(elev_m)
+            elev_m = types.int32(elev_m)
             thalweg_cell = thalweg_window[i]  # From flows_grid_boolean.tif (0s and 1s)
             if thalweg_cell == 1:  # Make sure thalweg cells are checked.
                 if elev_m in zone_min_dict:
diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh
index 77b64b7f5..2c827cc57 100755
--- a/src/delineate_hydros_and_produce_HAND.sh
+++ b/src/delineate_hydros_and_produce_HAND.sh
@@ -62,7 +62,7 @@ echo -e $startDiv"Mask Burned DEM for Thalweg Only $hucNumber $current_branch_id
 gdal_calc.py --quiet --type=Int32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \
     -A $tempCurrentBranchDataDir/flowdir_d8_burned_filled_$current_branch_id.tif \
     -B $tempCurrentBranchDataDir/demDerived_streamPixels_$current_branch_id.tif \
-    --calc="A/B" \
+    --calc="A*B" \
     --outfile="$tempCurrentBranchDataDir/flowdir_d8_burned_filled_flows_$current_branch_id.tif" \
     --NoDataValue=0
 
diff --git a/src/getRasterInfoNative.py b/src/getRasterInfoNative.py
index e3b8a69dc..2ebc5f8c8 100755
--- a/src/getRasterInfoNative.py
+++ b/src/getRasterInfoNative.py
@@ -9,6 +9,9 @@
 from osgeo import gdal, osr
 
 
+gdal.UseExceptions()
+
+
 """
 read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy
     <<< $(./getRasterInfoNative.py <raster.tif>)

From 25135e014e0d75f607c230f6f956d3cffd7c3a12 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 19 Jul 2024 19:39:17 +0000
Subject: [PATCH 334/359] v4.5.2.11 log duration times per module for post
 processing (#1222)

---
 docs/CHANGELOG.md      | 14 +++++++
 fim_pipeline.sh        |  5 ++-
 fim_post_processing.sh | 88 +++++++++++++++++++++++-------------------
 src/bash_functions.env | 62 ++++++++++++++++++++++++++++-
 4 files changed, 127 insertions(+), 42 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ca83bf29e..86c63f652 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.2.11 - 2024-07-19 - [PR#1222](https://github.com/NOAA-OWP/inundation-mapping/pull/1222)
+
+We are having problems with post processing overall duration taking a long time. This new system captures duration times for each module/section inside fim_post_processing.sh and records it to a file on the output directory. It records it as it progress and will also help us learn if fim_post_processing.sh stopped along the way.
+
+Note: When used in code, we call `Set_log_file_path` shell variable with a file name and path (no validation done at this time).  The each time a person wants to print to screen and console, use the `l_echo` command instead of the native `echo` command. If the log file has not been set, the output will continue to go to screen, just not the log file.
+
+### Changes
+- `fim_pipeline.sh`: A couple of minor text output changes.
+- `fim_post_processing.sh`:  As described above.
+- `src\bash_functions.env`:  New functions and adjustments to support the new log system.
+
+<br/><br/>
+
+
 ## v4.5.2.10 - 2024-07-19 - [PR#1224](https://github.com/NOAA-OWP/inundation-mapping/pull/1224)
 
 Addresses warnings to reduce output messages.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 8fc4ae9d7..16139fdc7 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -116,6 +116,7 @@ echo
 echo "---- Unit (HUC) processing is complete"
 date -u
 Calc_Duration $pipeline_start_time
+echo "---------------------------------------------------"
 
 ## POST PROCESSING
 
@@ -126,8 +127,10 @@ rm -d $workDir/$runName
 . $projectDir/fim_post_processing.sh -n $runName -j $jobMaxLimit
 
 echo
-echo "======================== End of fim_pipeline.sh =========================="
+
+echo "======================== End of fim_pipeline for $runName =========="
 date -u
+echo "Total Duration is ..."
 Calc_Duration $pipeline_start_time
 echo
 
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 5d0385c55..e17850cb9 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -73,6 +73,7 @@ if [ "$jobLimit" = "" ]; then jobLimit=1; fi
 rm -rdf $outputDestDir/logs/src_optimization
 rm -f $outputDestDir/logs/log_bankfull_indentify.log
 rm -f $outputDestDir/logs/subdiv_src_.log
+rm -f $log_file_name
 
 # load up enviromental information
 args_file=$outputDestDir/runtime_args.env
@@ -83,16 +84,21 @@ source $outputDestDir/params.env
 source $srcDir/bash_functions.env
 source $srcDir/bash_variables.env
 
-echo
+# Tell the system the name and location of the post processing log
+log_file_name=$outputDestDir/post_proc.log
+Set_log_file_path $log_file_name
+
+l_echo ""
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- Start of fim_post_processing"
-echo "---- Started: `date -u`"
+l_echo "---- Start of fim_post_processing"
+l_echo "---- Started: `date -u`"
 T_total_start
 post_proc_start_time=`date +%s`
 
 ## RUN UPDATE HYDROTABLE AND SRC ##
 # Define the counter file
 
+Tstart
 COUNTER_FILE="${outputDestDir}/post_processing_attempt.txt"
 # Function to clean up
 cleanup() {
@@ -100,7 +106,7 @@ cleanup() {
         if [ -f "$COUNTER_FILE" ]; then
             COUNTER=$(cat "$COUNTER_FILE")
             if [ "$COUNTER" -eq 1 ]; then
-                echo "Counter is 1. Removing the counter file."
+                l_echo "Counter is 1. Removing the counter file."
                 rm "$COUNTER_FILE"
             fi
         fi
@@ -121,36 +127,42 @@ COUNTER=$(cat "$COUNTER_FILE")
 COUNTER=$((COUNTER + 1))
 
 # Save the new counter value
-echo "$COUNTER" > "$COUNTER_FILE"
+l_echo "$COUNTER" > "$COUNTER_FILE"
 
 # Check if the counter is greater than one
 if [ "$COUNTER" -gt 1 ]; then
     # Execute the Python file
-    echo "Updating hydroTable & scr_full_crosswalked for branches"
+    l_echo "Updating hydroTable & scr_full_crosswalked for branches"
     python3 $srcDir/update_htable_src.py -d $outputDestDir
 else
-    echo "Execution count is $COUNTER, not executing the update_htable_src.py file."
+    l_echo "Execution count is $COUNTER, not executing the update_htable_src.py file."
 fi
+Tcount
+
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
-echo -e $startDiv"Start branch aggregation"
+l_echo $startDiv"Start branch aggregation"
+Tstart
 python3 $srcDir/aggregate_branch_lists.py -d $outputDestDir -f "branch_ids.csv" -o $fim_inputs
+Tcount
 
 ## GET NON ZERO EXIT CODES FOR BRANCHES ##
-echo -e $startDiv"Start non-zero exit code checking"
+l_echo $startDiv"Start non-zero exit code checking"
 find $outputDestDir/logs/branch -name "*_branch_*.log" -type f | \
     xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > \
     "$outputDestDir/branch_errors/non_zero_exit_codes.log" &
 
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
-echo "Processing usgs & ras2fim elev table aggregation"
+l_echo $startDiv"Processing usgs & ras2fim elev table aggregation"
+Tstart
 python3 $srcDir/aggregate_by_huc.py -fim $outputDestDir -i $fim_inputs -elev -ras -j $jobLimit
+Tcount
 
 ## RUN BATHYMETRY ADJUSTMENT ROUTINE ##
 if [ "$bathymetry_adjust" = "True" ]; then
-    echo -e $startDiv"Performing Bathymetry Adjustment routine"
-    # Run bathymetry adjustment routine
+    l_echo $startDiv"Performing Bathymetry Adjustment routine"
     Tstart
+    # Run bathymetry adjustment routine
     python3 $srcDir/bathymetric_adjustment.py \
         -fim_dir $outputDestDir \
         -bathy $bathymetry_file \
@@ -162,9 +174,9 @@ fi
 
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
-    echo -e $startDiv"Estimating bankfull stage in SRCs"
-    # Run SRC bankfull estimation routine routine
+    l_echo $startDiv"Estimating bankfull stage in SRCs"
     Tstart
+    # Run SRC bankfull estimation routine routine
     python3 $srcDir/identify_src_bankfull.py \
         -fim_dir $outputDestDir \
         -flows $bankfull_flows_file \
@@ -174,7 +186,7 @@ fi
 
 ## RUN SYNTHETIC RATING SUBDIVISION ROUTINE ##
 if [ "$src_subdiv_toggle" = "True" ] && [ "$src_bankfull_toggle" = "True" ]; then
-    echo -e $startDiv"Performing SRC channel/overbank subdivision routine"
+    l_echo $startDiv"Performing SRC channel/overbank subdivision routine"
     # Run SRC Subdivision & Variable Roughness routine
     Tstart
     python3 $srcDir/subdiv_chan_obank_src.py \
@@ -187,8 +199,7 @@ fi
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ USGS GAGE RATING CURVES ##
 if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$skipcal" = "0" ]; then
     Tstart
-    echo
-    echo -e $startDiv"Performing SRC adjustments using USGS rating curve database"
+    l_echo $startDiv"Performing SRC adjustments using USGS rating curve database"
     # Run SRC Optimization routine using USGS rating curve data (WSE and flow @ NWM recur flow values)
     python3 $srcDir/src_adjust_usgs_rating_trace.py \
         -run_dir $outputDestDir \
@@ -202,8 +213,7 @@ fi
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ RAS2FIM CROSS SECTION RATING CURVES ##
 if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$skipcal" = "0" ]; then
     Tstart
-    echo
-    echo -e $startDiv"Performing SRC adjustments using ras2fim rating curve database"
+    l_echo $startDiv"Performing SRC adjustments using ras2fim rating curve database"
     # Run SRC Optimization routine using ras2fim rating curve data (WSE and flow @ NWM recur flow values)
     python3 $srcDir/src_adjust_ras2fim_rating.py \
         -run_dir $outputDestDir \
@@ -217,16 +227,14 @@ fi
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINTS (.parquet files) ##
 if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [ "$skipcal" = "0" ]; then
     Tstart
-    echo
-    echo -e $startDiv"Performing SRC adjustments using benchmark point .parquet files"
+    l_echo $startDiv"Performing SRC adjustments using benchmark point .parquet files"
     python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputDestDir -j $jobLimit
     Tcount
     date -u
 fi
 
 ## AGGREGATE BRANCH TABLES ##
-echo
-echo -e $startDiv"Aggregating branch hydrotables"
+l_echo $startDiv"Aggregating branch hydrotables"
 Tstart
 python3 $srcDir/aggregate_by_huc.py \
     -fim $outputDestDir \
@@ -239,8 +247,7 @@ date -u
 
 ## PERFORM MANUAL CALIBRATION
 if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
-    echo
-    echo -e $startDiv"Performing manual calibration"
+    l_echo $startDiv"Performing manual calibration"
     Tstart
     python3 $srcDir/src_manual_calibration.py \
         -fim_dir $outputDestDir \
@@ -249,9 +256,8 @@ if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
     date -u
 fi
 
-echo
-echo -e $startDiv"Combining crosswalk tables"
-# aggregate outputs
+
+l_echo $startDiv"Combining crosswalk tables"
 Tstart
 python3 $toolsDir/combine_crosswalk_tables.py \
     -d $outputDestDir \
@@ -259,26 +265,30 @@ python3 $toolsDir/combine_crosswalk_tables.py \
 Tcount
 date -u
 
-echo -e $startDiv"Resetting Permissions"
+
+l_echo $startDiv"Resetting Permissions"
 Tstart
     find $outputDestDir -type d -exec chmod -R 777 {} +
     find $outputDestDir -type f -exec chmod 777 {} +  # just root level files
 Tcount
-date -u
 
-echo
-echo -e $startDiv"Scanning logs for errors. Results be saved in root not inside the log folder."
+
+l_echo $startDiv"Scanning logs for errors and warnings. This can take quite a few minutes so stand by."
+echo "Results will be saved in root not inside the log folder."
 Tstart
     # grep -H -r -i -n "error" $outputDestDir/logs/ > $outputDestDir/all_errors_from_logs.log
-    find $outputDestDir -type f | grep -H -r -i -n "error" $outputDestDir/logs/ > $outputDestDir/all_errors_from_logs.log
-Tcount
-date -u
+    find $outputDestDir -type f | grep -H -r -i -n "error" $outputDestDir/logs/ > \
+         $outputDestDir/all_errors_from_logs.log &
+    l_echo "error scan done, now on to warnings scan"
 
+    find $outputDestDir -type f | grep -H -r -i -n "warning" $outputDestDir/logs/ > \
+         $outputDestDir/all_warnings_from_logs.log &
+    l_echo "warning scan done"
+Tcount
 
 echo
-echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-echo "---- End of fim_post_processing"
-echo "---- Ended: `date -u`"
+l_echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+l_echo "---- End of fim_post_processing"
+l_echo "---- Ended: `date -u`"
 Calc_Duration $post_proc_start_time
 echo
-SUCCESS=true
diff --git a/src/bash_functions.env b/src/bash_functions.env
index 72a33f450..e8452003a 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -1,5 +1,44 @@
 #!/bin/bash
 
+
+## Simple logging system
+
+# If this value -nq "", then functions in this file will be printed to
+# console and log file.
+log_file_path=""
+
+# If this is set, all functions will set to a file as well as screen
+# Yes. it needs some validation that the log_file_path is a valid
+# file name and path.
+Set_log_file_path () {
+    log_file_path=$1
+
+    if [ ! -f $log_file_path ]; then
+        touch $log_file_path
+    else
+        rm -f $log_file_path
+    fi
+}
+
+# to stop other calls to make log calls to this all
+Clear_log_file_path () {
+
+    log_file_path=""
+}
+
+
+# if the log_file_path has been set, this will print to screen and add to log file
+l_echo () {
+    local msg=$1
+
+    if [[ $log_file_path != "" ]]; then
+        echo -e $msg ; echo -e $msg >> $log_file_path
+    else
+        echo -e $msg
+    fi
+}
+
+
 ## Timing functions
 
 T_total_start () {
@@ -10,15 +49,25 @@ Tstart () {
     t1=`date +%s`
 }
 
+# This one is based on the value of t1 created earlier, and end is now
 Tcount () {
     t2=`date +%s`
 
     local total_sec=$(( $t2 - $t1))
     local dur_min=$((total_sec / 60))
     local dur_remainder_sec=$((total_sec % 60))
-    echo "Cumulative Time = $dur_min min(s) and $dur_remainder_sec sec"
+
+    local msg="Duration = $dur_min min(s) and $dur_remainder_sec sec"
+    # if an arg comes in and is a file path, we can display and record to a file
+    if [[ $log_file_path != "" ]]; then
+        echo -e $msg ; echo -e $msg >> $log_file_path
+    else
+        echo -e $msg
+    fi
+
 }
 
+# This expects a start time submitted as an arg and assumes the end time as now
 Calc_Duration() {
     local start_time=$1
     local end_time=`date +%s`
@@ -26,9 +75,17 @@ Calc_Duration() {
     local total_sec=$(( $end_time - $start_time ))
     local dur_min=$((total_sec / 60))
     local dur_remainder_sec=$((total_sec % 60))
-    echo "Duration = $dur_min min(s) and $dur_remainder_sec sec"
+
+    local msg="Duration : $dur_min min(s) and $dur_remainder_sec sec"
+    # if an arg comes in and is a file path, we can display and record to a file
+    if [[ $log_file_path != "" ]]; then
+        echo -e $msg ; echo -e $msg >> $log_file_path
+    else
+        echo -e $msg
+    fi
 }
 
+
 Calc_Time() {
     local start_time=$1
     local end_time=`date +%s`
@@ -41,6 +98,7 @@ Calc_Time() {
     else
         echo "$dur_min:$dur_remainder_sec"
     fi
+
 }
 
 Calc_Time_Minutes_in_Percent(){

From 89ba6fe8cc7f773b45a69a716fe4637bdbc87ea4 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 24 Jul 2024 15:38:58 +0000
Subject: [PATCH 335/359] v4.5.3.0 Docker, python packages and jdk updates
 (#1217)

---
 .github/PULL_REQUEST_TEMPLATE.md              |    6 +-
 .gitignore                                    |    1 -
 CONTRIBUTING.md                               |   12 +-
 Dockerfile                                    |   17 +-
 Pipfile                                       |   20 +-
 Pipfile.lock                                  | 1150 +++++++++--------
 config/aws_s3_put_fim3_hydrovis_whitelist.lst |    4 -
 docs/CHANGELOG.md                             |   28 +
 fim_pre_processing.sh                         |    2 +
 pyproject.toml                                |    8 +-
 src/bathymetric_adjustment.py                 |    5 +-
 tools/overlapping_inundation.py               |    6 +-
 unit_tests/README.md                          |  155 ---
 unit_tests/__template.json                    |   12 -
 unit_tests/__template.py                      |   85 --
 unit_tests/aggregate_branch_lists_params.json |    7 -
 unit_tests/aggregate_branch_lists_test.py     |   34 -
 unit_tests/check_unit_errors_params.json      |    8 -
 unit_tests/check_unit_errors_test.py          |  205 ---
 unit_tests/clip_vectors_to_wbd_params.json    |   27 -
 unit_tests/clip_vectors_to_wbd_test.py        |   72 --
 unit_tests/derive_level_paths_params.json     |   17 -
 unit_tests/derive_level_paths_test.py         |  104 --
 ..._catchments_and_add_attributes_params.json |   11 -
 ...lter_catchments_and_add_attributes_test.py |   65 -
 .../generate_branch_list_csv_params.json      |   17 -
 unit_tests/generate_branch_list_csv_test.py   |   65 -
 unit_tests/generate_branch_list_params.json   |    7 -
 unit_tests/generate_branch_list_test.py       |   33 -
 unit_tests/inundate_gms_params.json           |   14 -
 unit_tests/inundate_gms_test.py               |   88 --
 unit_tests/inundation_params.json             |   23 -
 unit_tests/inundation_test.py                 |   65 -
 unit_tests/outputs_cleanup_params.json        |   20 -
 unit_tests/outputs_cleanup_test.py            |  187 ---
 unit_tests/pyproject.toml                     |    8 -
 .../rating_curve_comparison_params.json       |   17 -
 unit_tests/rating_curve_comparison_test.py    |   54 -
 unit_tests/shared_functions_params.json       |   17 -
 unit_tests/shared_functions_test.py           |   73 --
 unit_tests/split_flows_params.json            |   15 -
 unit_tests/split_flows_test.py                |   67 -
 unit_tests/unit_tests_utils.py                |   24 -
 unit_tests/usgs_gage_crosswalk_params.json    |   12 -
 unit_tests/usgs_gage_crosswalk_test.py        |   59 -
 45 files changed, 702 insertions(+), 2224 deletions(-)
 delete mode 100644 config/aws_s3_put_fim3_hydrovis_whitelist.lst
 mode change 100644 => 100755 src/bathymetric_adjustment.py
 delete mode 100644 unit_tests/README.md
 delete mode 100644 unit_tests/__template.json
 delete mode 100644 unit_tests/__template.py
 delete mode 100644 unit_tests/aggregate_branch_lists_params.json
 delete mode 100644 unit_tests/aggregate_branch_lists_test.py
 delete mode 100644 unit_tests/check_unit_errors_params.json
 delete mode 100644 unit_tests/check_unit_errors_test.py
 delete mode 100644 unit_tests/clip_vectors_to_wbd_params.json
 delete mode 100644 unit_tests/clip_vectors_to_wbd_test.py
 delete mode 100644 unit_tests/derive_level_paths_params.json
 delete mode 100644 unit_tests/derive_level_paths_test.py
 delete mode 100644 unit_tests/filter_catchments_and_add_attributes_params.json
 delete mode 100644 unit_tests/filter_catchments_and_add_attributes_test.py
 delete mode 100644 unit_tests/generate_branch_list_csv_params.json
 delete mode 100644 unit_tests/generate_branch_list_csv_test.py
 delete mode 100644 unit_tests/generate_branch_list_params.json
 delete mode 100644 unit_tests/generate_branch_list_test.py
 delete mode 100644 unit_tests/inundate_gms_params.json
 delete mode 100644 unit_tests/inundate_gms_test.py
 delete mode 100644 unit_tests/inundation_params.json
 delete mode 100644 unit_tests/inundation_test.py
 delete mode 100644 unit_tests/outputs_cleanup_params.json
 delete mode 100644 unit_tests/outputs_cleanup_test.py
 delete mode 100644 unit_tests/pyproject.toml
 delete mode 100644 unit_tests/rating_curve_comparison_params.json
 delete mode 100644 unit_tests/rating_curve_comparison_test.py
 delete mode 100644 unit_tests/shared_functions_params.json
 delete mode 100644 unit_tests/shared_functions_test.py
 delete mode 100644 unit_tests/split_flows_params.json
 delete mode 100644 unit_tests/split_flows_test.py
 delete mode 100644 unit_tests/unit_tests_utils.py
 delete mode 100644 unit_tests/usgs_gage_crosswalk_params.json
 delete mode 100644 unit_tests/usgs_gage_crosswalk_test.py

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 3bcbdf0fa..231e86471 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -8,16 +8,16 @@
 
 ### Testing
 
+
 ### Deployment Plan (For developer use)
 
 _How does the changes affect the product?_
 - [ ] Code only?
+- [ ] If applicable, has a deployment plan be created with the deployment person/team?
 - [ ] Require new or adjusted data inputs? Does it have start, end and duration code (in UTC)?
 - [ ] If new or updated data sets, has the FIM code been updated and tested with the new/adjusted data (subset is fine, but must be a subset of the new data)?
 - [ ] Require new pre-clip set?
 - [ ] Has new or updated python packages?
-- [ ] If applicable, has a deployment plan be created with the deployment person/team?
-
 
 ### Issuer Checklist (For developer use)
 
@@ -30,11 +30,9 @@ _You may update this checklist before and/or after creating the PR. If you're un
 - [ ] The feature branch you're submitting as a PR is up to date (merged) with the latest `dev` branch
 - [ ] `pre-commit` hooks were run locally
 - [ ] Any _change_ in functionality is tested
-- [ ] Passes all unit tests locally (inside interactive Docker container, at `/foss_fim/`, run: `pytest unit_tests/`)
 - [ ] New functions are documented (with a description, list of inputs, and expected output)
 - [ ] Placeholder code is flagged / future todos are captured in comments
 - [ ] [CHANGELOG](/docs/CHANGELOG.md) updated with template version number, e.g. `4.x.x.x`
-- [ ] [Reviewers requested](https://help.github.com/articles/requesting-a-pull-request-review/)
 - [ ] Add yourself as an [assignee](https://docs.github.com/en/issues/tracking-your-work-with-issues/assigning-issues-and-pull-requests-to-other-github-users) in the PR  as well as the FIM Technical Lead
 
 ### Merge Checklist (For Technical Lead use only)
diff --git a/.gitignore b/.gitignore
index 7ce22521c..d5643ae0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,5 +13,4 @@ config/**
 config/*.swp
 .vscode/
 **/.DS_Store
-**/*_pytest.py
 .private/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3e114abed..cd75f9d20 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -25,8 +25,7 @@ guidance below.
 ## Changing the code-base
 
 Generally speaking, you should fork this repository, make changes in your
-own fork, and then submit a pull request. All new code should have associated
-unit tests (added to `/unit_tests`) that validate implemented features and the presence or lack of defects.
+own fork, and then submit a pull request. 
 Additionally, the code should follow any stylistic and architectural guidelines
 prescribed by the project. In the absence of such guidelines, mimic the styles
 and patterns in the existing code-base.
@@ -118,19 +117,14 @@ If you would like to contribute, please follow these steps:
    # optionally close the container
    # Back on your terminal console (outside the container), use the typical git add, git commit, git push
 
- 
-8.  [Within the container](README.md#startrun-the-docker-container), ensure sure unit tests pass ([instructions here](/unit_tests/README.md)).
-    ```
-    pytest unit_tests/
-    ```
 
-9. Outside of the Docker container, commit your changes:
+8. Outside of the Docker container, commit your changes:
     ```
     git commit -m "<descriptive sentence or two of changes>"
     ```
     This will invoke pre-commit hooks mentioned in step 6 that will lint & format the code (some others as well). In many cases non-compliant code will be rectified automatically, but in some cases manual changes will be necessary. Make sure all of these checks pass. If not, make necessary changes (`git add <...>`), and re-issue `git commit -m "<...>"`.
    
-10. Push to your forked branch:
+9.  Push to your forked branch:
     ```
     git push -u origin
     ```
diff --git a/Dockerfile b/Dockerfile
index db1ecf72b..6b65469a6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -72,9 +72,17 @@ COPY --from=builder $depDir $depDir
 # remove reference to missing repo
 RUN rm /etc/apt/sources.list.d/apache-arrow.sources
 
-RUN apt-get update --fix-missing && apt-get install -y openjdk-19-jdk && rm -rf /var/lib/apt/lists/*
-
+RUN apt-get update --fix-missing && apt-get install -y openjdk-21-jdk && rm -rf /var/lib/apt/lists/*
 RUN apt update --fix-missing
+
+# An older version of openjdk still exists on the file system but was never cleaned up
+# After research, we realized, it just needs file cleanup. Leaving it there is triggering security warnings
+# RUN apt-get remove -y openjdk-17-jdk  (not installed, just residue left)
+RUN rm -rf ./usr/lib/jvm/*java-1.17* && \
+    rm -rf ./usr/lib/jvm/.java-1.17* && \
+    rm -rdf ./usr/lib/jvm/java-17*
+
+
 RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata
 
 RUN apt auto-remove
@@ -91,7 +99,9 @@ ENV PYTHONUNBUFFERED=TRUE
 
 ## ADD TO PATHS ##
 ENV PATH="$projectDir:${PATH}"
-ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/unit_tests:$projectDir/tools
+#ENV PATH=${PATH}:$projectDir:$projectDir/$srcDir:$projectDir/tools
+# Jul 17, 2024: Even though PYTHONPATH isn't used, it still seems to want it.
+ENV PYTHONPATH=${PATH}:$srcDir:$projectDir/tools
 
 ## install python 3 modules ##
 
@@ -110,6 +120,7 @@ RUN pip3 install pipenv==2023.12.1 && PIP_NO_CACHE_DIR=off pipenv install --syst
 # We download and unzip it to the same file folder that pip deployed the whitebox library.
 # Whitebox also attempts to always download a folder called testdata regardless of use.
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
+
 # RUN wbox_path=/usr/local/lib/python3.10/dist-packages/whitebox/WBT && \
 #     wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
 #     unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
diff --git a/Pipfile b/Pipfile
index 6ba0dc993..9290a1e29 100644
--- a/Pipfile
+++ b/Pipfile
@@ -7,11 +7,11 @@ verify_ssl = true
 ipython = "==8.24.0"
 
 [packages]
-certifi = "==2023.7.22"
+certifi = "==2024.7.4"
 fiona = "==1.8.22"
 geopandas = "==0.14.3"
-numba = "==0.56.4"
-numpy = "==1.23.5"
+numba = "==0.60.0"
+numpy = "==1.26.4"
 pandas = "==2.0.2"
 rasterio = "==1.3.6"
 rasterstats = "==0.18.0"
@@ -20,7 +20,7 @@ tqdm = "==4.66.3"
 seaborn = "==0.12.2"
 python-dotenv = "==1.0.0"
 natsort = "==8.3.1"
-xarray = "==2023.1.0"
+xarray = "==2024.6.0"
 netcdf4 = "==1.6.3"
 tables = "==3.8.0"
 pyproj = "==3.5.0"
@@ -29,20 +29,19 @@ boto3 = "==1.26.109"
 jupyter = "==1.0.0"
 jupyterlab = "==3.6.7"
 ipympl = "==0.9.3"
-pytest = "==7.3.0"
-whitebox = "2.3.4"
+whitebox = "==2.3.4"
 shapely = "==2.0.1"
 pyarrow = "==14.0.1"
 rtree = "==1.0.1"
 py7zr = "==0.20.4"
-scipy = "==1.10.1"
-gval = "==0.2.3"
+scipy = "==1.14.0"
+gval = "==0.2.7"
 flake8 = "==6.0.0"
 black = "==24.3.0"
 flake8-pyproject = "==1.2.3"
 pre-commit = "==3.3.3"
 isort = "==5.12.0"
-urllib3 = "==1.26.18"
+urllib3 = "==1.26.19"
 pyflwdir = "==0.5.8"
 pillow = "==10.3.0"
 pyogrio = "==0.7.2"
@@ -52,6 +51,9 @@ lmoments3 = "==1.0.6"
 zarr = "==2.18.0"
 requests = "==2.32.3"
 aiohttp = "==3.9.5"
+distributed = "==2024.6.2"
+monaco = "==0.13.1"
+dask = "==2024.6.2"
 
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index 3c1bc0455..ed11ccabf 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "898973145b92f3e3eb9afcaf3cee8bd5da693fbd8ae06046e66d3a73022c27a5"
+            "sha256": "75c63212784d57d42fc7b5536a4634408876142a2abc57317537c8e7ef4abab5"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -24,6 +24,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.4.0"
         },
+        "aiobotocore": {
+            "hashes": [
+                "sha256:337429ffd3cc367532572d40be809a84c7b5335f3f8eca2f23e09dfaa9a9ef90",
+                "sha256:e7399f21570db1c287f1c0c814dd3475dfe1c8166722e2c77ce67f172cbcfa89"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.5.2"
+        },
         "aiofiles": {
             "hashes": [
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
@@ -115,6 +123,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.9.5"
         },
+        "aioitertools": {
+            "hashes": [
+                "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394",
+                "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.11.0"
+        },
         "aiosignal": {
             "hashes": [
                 "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc",
@@ -309,11 +325,58 @@
         },
         "botocore": {
             "hashes": [
-                "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df",
-                "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210"
+                "sha256:a50edd715eb510343e27849f36483804aae4b871590db4d4996aa53368dcac40",
+                "sha256:b906999dd53dda2ef0ef6f7f55fcc81a4b06b9f1c8a9f65c546e0b981f959f5f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.29.165"
+            "version": "==1.29.161"
+        },
+        "bottleneck": {
+            "hashes": [
+                "sha256:03c43150f180d86a5633a6da788660d335983f6798fca306ba7f47ff27a1b7e7",
+                "sha256:1490348b3bbc0225523dc2c00c6bb3e66168c537d62797bd29783c0826c09838",
+                "sha256:14b3334a39308fbb05dacd35ac100842aa9e9bc70afbdcebe43e46179d183fd0",
+                "sha256:1b4dac5d2a871b7bd296c2b92426daa27d5b07aa84ef2557db097d29135da4eb",
+                "sha256:2110af22aa8c2779faba8aa021d6b559df04449bdf21d510eacd7910934189fe",
+                "sha256:220b72405f77aebb0137b733b464c2526ded471e4289ac1e840bab8852759a55",
+                "sha256:23834d82177d6997f21fa63156550668cd07a9a6e5a1b66ea80f1a14ac6ffd07",
+                "sha256:28260197ab8a4a6b7adf810523147b1a3e85607f4e26a0f685eb9d155cfc75af",
+                "sha256:2861ff645d236f1a6f5c6d1ddb3db37d19af1d91057bdc4fd7b76299a15b3079",
+                "sha256:2bb79a2ac135567694f13339f0bebcee96aec09c596b324b61cd7fd5e306f49d",
+                "sha256:381cbd1e52338fcdf9ff01c962e6aa187b2d8b3b369d42e779b6d33ac61f8d35",
+                "sha256:44305c70c2a1539b0ae968e033f301ad868a6146b47e3cccd73fdfe3fc07c4ee",
+                "sha256:4a91e40bbb8452e77772614d882be2c34b3b514d9f15460f703293525a6e173d",
+                "sha256:520d7a83cd48b3f58e5df1a258acb547f8a5386a8c21ca9e1058d83a0d622fdf",
+                "sha256:59604949aea476f5075b965129eaa3c2d90891fd43b0dfaf2ad7621bb5db14a5",
+                "sha256:5d6bf45ed58d5e7414c0011ef2da75474fe597a51970df83596b0bcb79c14c5e",
+                "sha256:6136ce7dcf825c432a20b80ab1c460264a437d8430fff32536176147e0b6b832",
+                "sha256:6179791c0119aec3708ef74ddadab8d183e3742adb93a9028718e8696bdf572b",
+                "sha256:67347b0f01f32a232a6269c37afc1c079e08f6455fa12e91f4a1cd12eb0d11a5",
+                "sha256:6a36280ee33d9db799163f04e88b950261e590cc71d089f5e179b21680b5d491",
+                "sha256:817aa43a671ede696ea023d8f35839a391244662340cc95a0f46965dda8b35cf",
+                "sha256:834816c316ad184cae7ecb615b69876a42cd2cafb07ee66c57a9c1ccacb63339",
+                "sha256:8746f0f727997ce4c7457dc1fec4e4e3c0fdd8803514baa3d1c4ea6515ab04b2",
+                "sha256:889e6855b77345622b4ba927335d3118745d590492941f5f78554f157d259e92",
+                "sha256:90d5d188a0cca0b9655ff2904ee61e7f183079e97550be98c2541a2eec358a72",
+                "sha256:9903f017b9d6f2f69ce241b424ddad7265624f64dc6eafbe257d45661febf8bd",
+                "sha256:a704165552496cbcc8bcc5921bb679fd6fa66bb1e758888de091b1223231c9f0",
+                "sha256:b1339b9ad3ee217253f246cde5c3789eb527cf9dd31ff0a1f5a8bf7fc89eadad",
+                "sha256:b5f72b66ccc0272de46b67346cf8490737ba2adc6a302664f5326e7741b6d5ab",
+                "sha256:beb36df519b8709e7d357c0c9639b03b885ca6355bbf5e53752c685de51605b8",
+                "sha256:c2c92545e1bc8e859d8d137aefa3b24843bd374b17c9814dafa3bbcea9fc4ec0",
+                "sha256:c6097bf39723e76ff5bba160daab92ae599df212c859db8d46648548584d04a8",
+                "sha256:d53f1a72b12cfd76b56934c33bc0cb7c1a295f23a2d3ffba8c764514c9b5e0ff",
+                "sha256:d69907d8d679cb5091a3f479c46bf1076f149f6311ff3298bac5089b86a2fab1",
+                "sha256:de17e012694e6a987bb4eb050dd7f0cf939195a8e00cb23aa93ebee5fd5e64a8",
+                "sha256:e720ff24370324c84a82b1a18195274715c23181748b2b9e3dacad24198ca06f",
+                "sha256:ed209f8f3cb9954773764b0fa2510a7a9247ad245593187ac90bd0747771bc5c",
+                "sha256:eea333dbcadb780356c54f5c4fa7754f143573b57508fff43d5daf63298eb26a",
+                "sha256:f2749602200aaa0e12a0f3f936dd6d4035384ad10d3acf7ac4f418c501683397",
+                "sha256:f63e79bfa2f82a7432c8b147ed321d01ca7769bc17cc04644286a4ce58d30549",
+                "sha256:fbcdd01db9e27741fb16a02b720cf02389d4b0b99cefe3c834c7df88c2d7412d",
+                "sha256:ffb4e4edf7997069719b9269926cc00a2a12c6e015422d1ebc2f621c4541396a"
+            ],
+            "version": "==1.4.0"
         },
         "brotli": {
             "hashes": [
@@ -406,20 +469,20 @@
         },
         "cachetools": {
             "hashes": [
-                "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945",
-                "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"
+                "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474",
+                "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.3"
+            "version": "==5.4.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
-                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
+                "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b",
+                "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.6'",
-            "version": "==2023.7.22"
+            "version": "==2024.7.4"
         },
         "cffi": {
             "hashes": [
@@ -644,19 +707,11 @@
         },
         "cloudpickle": {
             "hashes": [
-                "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7",
-                "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"
+                "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f",
+                "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==3.0.0"
-        },
-        "colorcet": {
-            "hashes": [
-                "sha256:2921b3cd81a2288aaf2d63dbc0ce3c26dcd882e8c389cc505d6886bf7aa9a4eb",
-                "sha256:2a7d59cc8d0f7938eeedd08aad3152b5319b4ba3bcb7a612398cc17a384cb296"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==3.1.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.2.1"
         },
         "comm": {
             "hashes": [
@@ -797,48 +852,44 @@
             "version": "==3.0.10"
         },
         "dask": {
-            "hashes": [
-                "sha256:32b34986519b7ddc0947c8ca63c2fc81b964e4c208dfb5cbf9f4f8aec92d152b",
-                "sha256:4f4c28ac406e81b8f21b5be4b31b21308808f3e0e7c7e2f4a914f16476d9941b"
+            "extras": [
+                "array"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==2023.5.0"
-        },
-        "datashader": {
             "hashes": [
-                "sha256:7899979b4c1adba6b4fd2e86caa3f5ef94c4e6ab234cbb7306ca6bfe243fc4df",
-                "sha256:c12ce9c71983a3f74ccba9d9416c270d1ed0a4e6c7bdcc6b63a2121ada0e00cd"
+                "sha256:81b80ee015b2e057b93bb2d1bf13a866136e762e2b24bf54b6b621e8b86b7708",
+                "sha256:d429d6b19e85fd1306ac37c188aaf99d03bbe69a6fe59d2b42882b2ac188686f"
             ],
+            "index": "pypi",
             "markers": "python_version >= '3.9'",
-            "version": "==0.16.2"
+            "version": "==2024.6.2"
         },
         "debugpy": {
             "hashes": [
-                "sha256:016a9fcfc2c6b57f939673c874310d8581d51a0fe0858e7fac4e240c5eb743cb",
-                "sha256:0de56aba8249c28a300bdb0672a9b94785074eb82eb672db66c8144fff673146",
-                "sha256:1a9fe0829c2b854757b4fd0a338d93bc17249a3bf69ecf765c61d4c522bb92a8",
-                "sha256:28acbe2241222b87e255260c76741e1fbf04fdc3b6d094fcf57b6c6f75ce1242",
-                "sha256:3a79c6f62adef994b2dbe9fc2cc9cc3864a23575b6e387339ab739873bea53d0",
-                "sha256:3bda0f1e943d386cc7a0e71bfa59f4137909e2ed947fb3946c506e113000f741",
-                "sha256:3ebb70ba1a6524d19fa7bb122f44b74170c447d5746a503e36adc244a20ac539",
-                "sha256:58911e8521ca0c785ac7a0539f1e77e0ce2df753f786188f382229278b4cdf23",
-                "sha256:6df9aa9599eb05ca179fb0b810282255202a66835c6efb1d112d21ecb830ddd3",
-                "sha256:7a3afa222f6fd3d9dfecd52729bc2e12c93e22a7491405a0ecbf9e1d32d45b39",
-                "sha256:7eb7bd2b56ea3bedb009616d9e2f64aab8fc7000d481faec3cd26c98a964bcdd",
-                "sha256:92116039b5500633cc8d44ecc187abe2dfa9b90f7a82bbf81d079fcdd506bae9",
-                "sha256:a2e658a9630f27534e63922ebf655a6ab60c370f4d2fc5c02a5b19baf4410ace",
-                "sha256:bfb20cb57486c8e4793d41996652e5a6a885b4d9175dd369045dad59eaacea42",
-                "sha256:caad2846e21188797a1f17fc09c31b84c7c3c23baf2516fed5b40b378515bbf0",
-                "sha256:d915a18f0597ef685e88bb35e5d7ab968964b7befefe1aaea1eb5b2640b586c7",
-                "sha256:dda73bf69ea479c8577a0448f8c707691152e6c4de7f0c4dec5a4bc11dee516e",
-                "sha256:e38beb7992b5afd9d5244e96ad5fa9135e94993b0c551ceebf3fe1a5d9beb234",
-                "sha256:edcc9f58ec0fd121a25bc950d4578df47428d72e1a0d66c07403b04eb93bcf98",
-                "sha256:efd3fdd3f67a7e576dd869c184c5dd71d9aaa36ded271939da352880c012e703",
-                "sha256:f696d6be15be87aef621917585f9bb94b1dc9e8aced570db1b8a6fc14e8f9b42",
-                "sha256:fd97ed11a4c7f6d042d320ce03d83b20c3fb40da892f994bc041bbc415d7a099"
+                "sha256:0600faef1d0b8d0e85c816b8bb0cb90ed94fc611f308d5fde28cb8b3d2ff0fe3",
+                "sha256:1523bc551e28e15147815d1397afc150ac99dbd3a8e64641d53425dba57b0ff9",
+                "sha256:15bc2f4b0f5e99bf86c162c91a74c0631dbd9cef3c6a1d1329c946586255e859",
+                "sha256:16c8dcab02617b75697a0a925a62943e26a0330da076e2a10437edd9f0bf3755",
+                "sha256:16e16df3a98a35c63c3ab1e4d19be4cbc7fdda92d9ddc059294f18910928e0ca",
+                "sha256:2cbd4d9a2fc5e7f583ff9bf11f3b7d78dfda8401e8bb6856ad1ed190be4281ad",
+                "sha256:3f8c3f7c53130a070f0fc845a0f2cee8ed88d220d6b04595897b66605df1edd6",
+                "sha256:40f062d6877d2e45b112c0bbade9a17aac507445fd638922b1a5434df34aed02",
+                "sha256:5a019d4574afedc6ead1daa22736c530712465c0c4cd44f820d803d937531b2d",
+                "sha256:5d3ccd39e4021f2eb86b8d748a96c766058b39443c1f18b2dc52c10ac2757835",
+                "sha256:62658aefe289598680193ff655ff3940e2a601765259b123dc7f89c0239b8cd3",
+                "sha256:7ee2e1afbf44b138c005e4380097d92532e1001580853a7cb40ed84e0ef1c3d2",
+                "sha256:7f8d57a98c5a486c5c7824bc0b9f2f11189d08d73635c326abef268f83950326",
+                "sha256:8a13417ccd5978a642e91fb79b871baded925d4fadd4dfafec1928196292aa0a",
+                "sha256:95378ed08ed2089221896b9b3a8d021e642c24edc8fef20e5d4342ca8be65c00",
+                "sha256:acdf39855f65c48ac9667b2801234fc64d46778021efac2de7e50907ab90c634",
+                "sha256:bd11fe35d6fd3431f1546d94121322c0ac572e1bfb1f6be0e9b8655fb4ea941e",
+                "sha256:c78ba1680f1015c0ca7115671fe347b28b446081dada3fedf54138f44e4ba031",
+                "sha256:cf327316ae0c0e7dd81eb92d24ba8b5e88bb4d1b585b5c0d32929274a66a5210",
+                "sha256:d3408fddd76414034c02880e891ea434e9a9cf3a69842098ef92f6e809d09afa",
+                "sha256:e24ccb0cd6f8bfaec68d577cb49e9c680621c336f347479b3fce060ba7c09ec1",
+                "sha256:f179af1e1bd4c88b0b9f0fa153569b24f6b6f3de33f94703336363ae62f4bf47"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.8.1"
+            "version": "==1.8.2"
         },
         "decorator": {
             "hashes": [
@@ -863,6 +914,15 @@
             ],
             "version": "==0.3.8"
         },
+        "distributed": {
+            "hashes": [
+                "sha256:0c1f8ccb1da71273ad8c53c598147dc37e60bef17142fd466cb72618a521880f",
+                "sha256:bb43b766ada860b163956607c80f99871d823c645e326c2b5e35f020351adc55"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==2024.6.2"
+        },
         "entrypoints": {
             "hashes": [
                 "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4",
@@ -881,11 +941,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad",
-                "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"
+                "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b",
+                "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.2.1"
+            "version": "==1.2.2"
         },
         "executing": {
             "hashes": [
@@ -905,18 +965,18 @@
         },
         "fastjsonschema": {
             "hashes": [
-                "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0",
-                "sha256:e3126a94bdc4623d3de4485f8d468a12f02a67921315ddc87836d6e456dc789d"
+                "sha256:3d48fc5300ee96f5d116f10fe6f28d938e6008f59a6a025c2649475b87f76a23",
+                "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a"
             ],
-            "version": "==2.19.1"
+            "version": "==2.20.0"
         },
         "filelock": {
             "hashes": [
-                "sha256:58a2549afdf9e02e10720eaa4d4470f56386d7a6f72edd7d0596337af8ed7ad8",
-                "sha256:71b3102950e91dfc1bb4209b64be4dc8854f40e5f534428d8684f953ac847fac"
+                "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb",
+                "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.15.1"
+            "version": "==3.15.4"
         },
         "fiona": {
             "hashes": [
@@ -970,51 +1030,51 @@
         },
         "fonttools": {
             "hashes": [
-                "sha256:099634631b9dd271d4a835d2b2a9e042ccc94ecdf7e2dd9f7f34f7daf333358d",
-                "sha256:0c555e039d268445172b909b1b6bdcba42ada1cf4a60e367d68702e3f87e5f64",
-                "sha256:1e677bfb2b4bd0e5e99e0f7283e65e47a9814b0486cb64a41adf9ef110e078f2",
-                "sha256:2367d47816cc9783a28645bc1dac07f8ffc93e0f015e8c9fc674a5b76a6da6e4",
-                "sha256:28d072169fe8275fb1a0d35e3233f6df36a7e8474e56cb790a7258ad822b6fd6",
-                "sha256:31f0e3147375002aae30696dd1dc596636abbd22fca09d2e730ecde0baad1d6b",
-                "sha256:3e0ad3c6ea4bd6a289d958a1eb922767233f00982cf0fe42b177657c86c80a8f",
-                "sha256:45b4afb069039f0366a43a5d454bc54eea942bfb66b3fc3e9a2c07ef4d617380",
-                "sha256:4a2a6ba400d386e904fd05db81f73bee0008af37799a7586deaa4aef8cd5971e",
-                "sha256:4f520d9ac5b938e6494f58a25c77564beca7d0199ecf726e1bd3d56872c59749",
-                "sha256:52a6e0a7a0bf611c19bc8ec8f7592bdae79c8296c70eb05917fd831354699b20",
-                "sha256:5a4788036201c908079e89ae3f5399b33bf45b9ea4514913f4dbbe4fac08efe0",
-                "sha256:6b4f04b1fbc01a3569d63359f2227c89ab294550de277fd09d8fca6185669fa4",
-                "sha256:715b41c3e231f7334cbe79dfc698213dcb7211520ec7a3bc2ba20c8515e8a3b5",
-                "sha256:73121a9b7ff93ada888aaee3985a88495489cc027894458cb1a736660bdfb206",
-                "sha256:74ae2441731a05b44d5988d3ac2cf784d3ee0a535dbed257cbfff4be8bb49eb9",
-                "sha256:7d6166192dcd925c78a91d599b48960e0a46fe565391c79fe6de481ac44d20ac",
-                "sha256:7f193f060391a455920d61684a70017ef5284ccbe6023bb056e15e5ac3de11d1",
-                "sha256:907fa0b662dd8fc1d7c661b90782ce81afb510fc4b7aa6ae7304d6c094b27bce",
-                "sha256:93156dd7f90ae0a1b0e8871032a07ef3178f553f0c70c386025a808f3a63b1f4",
-                "sha256:93bc9e5aaa06ff928d751dc6be889ff3e7d2aa393ab873bc7f6396a99f6fbb12",
-                "sha256:95db0c6581a54b47c30860d013977b8a14febc206c8b5ff562f9fe32738a8aca",
-                "sha256:973d030180eca8255b1bce6ffc09ef38a05dcec0e8320cc9b7bcaa65346f341d",
-                "sha256:9cd7a6beec6495d1dffb1033d50a3f82dfece23e9eb3c20cd3c2444d27514068",
-                "sha256:9fe9096a60113e1d755e9e6bda15ef7e03391ee0554d22829aa506cdf946f796",
-                "sha256:a209d2e624ba492df4f3bfad5996d1f76f03069c6133c60cd04f9a9e715595ec",
-                "sha256:a239afa1126b6a619130909c8404070e2b473dd2b7fc4aacacd2e763f8597fea",
-                "sha256:ba9f09ff17f947392a855e3455a846f9855f6cf6bec33e9a427d3c1d254c712f",
-                "sha256:bb7273789f69b565d88e97e9e1da602b4ee7ba733caf35a6c2affd4334d4f005",
-                "sha256:bd5bc124fae781a4422f61b98d1d7faa47985f663a64770b78f13d2c072410c2",
-                "sha256:bff98816cb144fb7b85e4b5ba3888a33b56ecef075b0e95b95bcd0a5fbf20f06",
-                "sha256:c4ee5a24e281fbd8261c6ab29faa7fd9a87a12e8c0eed485b705236c65999109",
-                "sha256:c93ed66d32de1559b6fc348838c7572d5c0ac1e4a258e76763a5caddd8944002",
-                "sha256:d1a24f51a3305362b94681120c508758a88f207fa0a681c16b5a4172e9e6c7a9",
-                "sha256:d8f191a17369bd53a5557a5ee4bab91d5330ca3aefcdf17fab9a497b0e7cff7a",
-                "sha256:daaef7390e632283051e3cf3e16aff2b68b247e99aea916f64e578c0449c9c68",
-                "sha256:e40013572bfb843d6794a3ce076c29ef4efd15937ab833f520117f8eccc84fd6",
-                "sha256:eceef49f457253000e6a2d0f7bd08ff4e9fe96ec4ffce2dbcb32e34d9c1b8161",
-                "sha256:ee595d7ba9bba130b2bec555a40aafa60c26ce68ed0cf509983e0f12d88674fd",
-                "sha256:ef50ec31649fbc3acf6afd261ed89d09eb909b97cc289d80476166df8438524d",
-                "sha256:fa1f3e34373aa16045484b4d9d352d4c6b5f9f77ac77a178252ccbc851e8b2ee",
-                "sha256:fca66d9ff2ac89b03f5aa17e0b21a97c21f3491c46b583bb131eb32c7bab33af"
+                "sha256:02569e9a810f9d11f4ae82c391ebc6fb5730d95a0657d24d754ed7763fb2d122",
+                "sha256:0679a30b59d74b6242909945429dbddb08496935b82f91ea9bf6ad240ec23397",
+                "sha256:10f5e6c3510b79ea27bb1ebfcc67048cde9ec67afa87c7dd7efa5c700491ac7f",
+                "sha256:2af40ae9cdcb204fc1d8f26b190aa16534fcd4f0df756268df674a270eab575d",
+                "sha256:32f029c095ad66c425b0ee85553d0dc326d45d7059dbc227330fc29b43e8ba60",
+                "sha256:35250099b0cfb32d799fb5d6c651220a642fe2e3c7d2560490e6f1d3f9ae9169",
+                "sha256:3b3c8ebafbee8d9002bd8f1195d09ed2bd9ff134ddec37ee8f6a6375e6a4f0e8",
+                "sha256:4824c198f714ab5559c5be10fd1adf876712aa7989882a4ec887bf1ef3e00e31",
+                "sha256:5ff7e5e9bad94e3a70c5cd2fa27f20b9bb9385e10cddab567b85ce5d306ea923",
+                "sha256:651390c3b26b0c7d1f4407cad281ee7a5a85a31a110cbac5269de72a51551ba2",
+                "sha256:6e08f572625a1ee682115223eabebc4c6a2035a6917eac6f60350aba297ccadb",
+                "sha256:6ed170b5e17da0264b9f6fae86073be3db15fa1bd74061c8331022bca6d09bab",
+                "sha256:73379d3ffdeecb376640cd8ed03e9d2d0e568c9d1a4e9b16504a834ebadc2dfb",
+                "sha256:75a157d8d26c06e64ace9df037ee93a4938a4606a38cb7ffaf6635e60e253b7a",
+                "sha256:791b31ebbc05197d7aa096bbc7bd76d591f05905d2fd908bf103af4488e60670",
+                "sha256:7b6b35e52ddc8fb0db562133894e6ef5b4e54e1283dff606fda3eed938c36fc8",
+                "sha256:84ec3fb43befb54be490147b4a922b5314e16372a643004f182babee9f9c3407",
+                "sha256:8959a59de5af6d2bec27489e98ef25a397cfa1774b375d5787509c06659b3671",
+                "sha256:9dfdae43b7996af46ff9da520998a32b105c7f098aeea06b2226b30e74fbba88",
+                "sha256:9e6ceba2a01b448e36754983d376064730690401da1dd104ddb543519470a15f",
+                "sha256:9efd176f874cb6402e607e4cc9b4a9cd584d82fc34a4b0c811970b32ba62501f",
+                "sha256:a1c7c5aa18dd3b17995898b4a9b5929d69ef6ae2af5b96d585ff4005033d82f0",
+                "sha256:aae7bd54187e8bf7fd69f8ab87b2885253d3575163ad4d669a262fe97f0136cb",
+                "sha256:b21952c092ffd827504de7e66b62aba26fdb5f9d1e435c52477e6486e9d128b2",
+                "sha256:b96cd370a61f4d083c9c0053bf634279b094308d52fdc2dd9a22d8372fdd590d",
+                "sha256:becc5d7cb89c7b7afa8321b6bb3dbee0eec2b57855c90b3e9bf5fb816671fa7c",
+                "sha256:bee32ea8765e859670c4447b0817514ca79054463b6b79784b08a8df3a4d78e3",
+                "sha256:c6e7170d675d12eac12ad1a981d90f118c06cf680b42a2d74c6c931e54b50719",
+                "sha256:c818c058404eb2bba05e728d38049438afd649e3c409796723dfc17cd3f08749",
+                "sha256:c8696544c964500aa9439efb6761947393b70b17ef4e82d73277413f291260a4",
+                "sha256:c9cd19cf4fe0595ebdd1d4915882b9440c3a6d30b008f3cc7587c1da7b95be5f",
+                "sha256:d4d0096cb1ac7a77b3b41cd78c9b6bc4a400550e21dc7a92f2b5ab53ed74eb02",
+                "sha256:d92d3c2a1b39631a6131c2fa25b5406855f97969b068e7e08413325bc0afba58",
+                "sha256:da33440b1413bad53a8674393c5d29ce64d8c1a15ef8a77c642ffd900d07bfe1",
+                "sha256:e013aae589c1c12505da64a7d8d023e584987e51e62006e1bb30d72f26522c41",
+                "sha256:e128778a8e9bc11159ce5447f76766cefbd876f44bd79aff030287254e4752c4",
+                "sha256:e54f1bba2f655924c1138bbc7fa91abd61f45c68bd65ab5ed985942712864bbb",
+                "sha256:e5b708073ea3d684235648786f5f6153a48dc8762cdfe5563c57e80787c29fbb",
+                "sha256:e8bf06b94694251861ba7fdeea15c8ec0967f84c3d4143ae9daf42bbc7717fe3",
+                "sha256:f08df60fbd8d289152079a65da4e66a447efc1d5d5a4d3f299cdd39e3b2e4a7d",
+                "sha256:f1f8758a2ad110bd6432203a344269f445a2907dc24ef6bccfd0ac4e14e0d71d",
+                "sha256:f677ce218976496a587ab17140da141557beb91d2a5c1a14212c994093f2eae2"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.53.0"
+            "version": "==4.53.1"
         },
         "fqdn": {
             "hashes": [
@@ -1108,19 +1168,19 @@
         },
         "fsspec": {
             "hashes": [
-                "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee",
-                "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"
+                "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a",
+                "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.6.0"
+            "version": "==2023.6.0"
         },
         "geocube": {
             "hashes": [
-                "sha256:2fb9e193c67f47f47cdade85748f17cd69bf8ac1cf5194d683846022b8d5c4e4",
-                "sha256:908e13900535a8ba2d6598ed3e69924f8dc60620bde242d760655d5a2a8244f2"
+                "sha256:0b06a1401df539d559703053d380f0e10bb7e013653ce9acb5e164e36f0a4ab3",
+                "sha256:bf49d7b19c4d78740c095ab93f1f10f448f131338ccc74de497ec5b437273730"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.5.2"
+            "version": "==0.5.3"
         },
         "geographiclib": {
             "hashes": [
@@ -1149,20 +1209,20 @@
         },
         "gval": {
             "hashes": [
-                "sha256:d1a560f9e8b9b7727c618a56d7a04f3621ceb71477f18410c811a6dc0bd9b05d",
-                "sha256:d9f26aa88f164b77a47cb7d7eafc1641e48abe4152501657cb173c44dc84a8fd"
+                "sha256:5721270a41278888c043a3fc1c91c26998799f31edf151a6383afd8e2bd9de9a",
+                "sha256:9c9e1d9710e237a95cbb8385d7c41f21e0e64051fe526985373a2bc80bf5fa87"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.2.3"
+            "version": "==0.2.7"
         },
         "identify": {
             "hashes": [
-                "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa",
-                "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"
+                "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf",
+                "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.36"
+            "version": "==2.6.0"
         },
         "idna": {
             "hashes": [
@@ -1174,11 +1234,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570",
-                "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"
+                "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f",
+                "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==7.1.0"
+            "markers": "python_version < '3.12'",
+            "version": "==8.0.0"
         },
         "inflate64": {
             "hashes": [
@@ -1239,21 +1299,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.0.0"
         },
-        "iniconfig": {
-            "hashes": [
-                "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
-                "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.0.0"
-        },
         "ipykernel": {
             "hashes": [
-                "sha256:1181e653d95c6808039c509ef8e67c4126b3b3af7781496c7cbfb5ed938a27da",
-                "sha256:3d44070060f9475ac2092b760123fadf105d2e2493c24848b6691a7c4f42af5c"
+                "sha256:afdb66ba5aa354b09b91379bac28ae4afebbb30e8b39510c9690afb7a10421b5",
+                "sha256:f093a22c4a40f8828f8e330a9c297cb93dcab13bd9678ded6de8e5cf81c56215"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.29.4"
+            "version": "==6.29.5"
         },
         "ipympl": {
             "hashes": [
@@ -1265,11 +1317,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:53eee7ad44df903a06655871cbab66d156a051fd86f3ec6750470ac9604ac1ab",
-                "sha256:c6ed726a140b6e725b911528f80439c534fac915246af3efc39440a6b0f9d716"
+                "sha256:1cec0fbba8404af13facebe83d04436a7434c7400e59f47acf467c64abd0956c",
+                "sha256:e6b347c27bdf9c32ee9d31ae85defc525755a1869f14057e900675b9e8d6e6ff"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.25.0"
+            "version": "==8.26.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1354,11 +1406,11 @@
                 "format-nongpl"
             ],
             "hashes": [
-                "sha256:5b22d434a45935119af990552c862e5d6d564e8f6601206b305a61fdf661a2b7",
-                "sha256:ff4cfd6b1367a40e7bc6411caec72effadd3db0bbe5017de188f2d6108335802"
+                "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4",
+                "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.22.0"
+            "version": "==4.23.0"
         },
         "jsonschema-specifications": {
             "hashes": [
@@ -1411,11 +1463,11 @@
         },
         "jupyter-server": {
             "hashes": [
-                "sha256:12558d158ec7a0653bf96cc272bc7ad79e0127d503b982ed144399346694f726",
-                "sha256:16f7177c3a4ea8fe37784e2d31271981a812f0b2874af17339031dc3510cc2a5"
+                "sha256:47ff506127c2f7851a17bf4713434208fc490955d0e8632e95014a9a9afbeefd",
+                "sha256:66095021aa9638ced276c248b1d81862e4c50f292d575920bbe960de1c56b12b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.14.1"
+            "version": "==2.14.2"
         },
         "jupyter-server-fileid": {
             "hashes": [
@@ -1468,11 +1520,11 @@
         },
         "jupyterlab-server": {
             "hashes": [
-                "sha256:15cbb349dc45e954e09bacf81b9f9bcb10815ff660fb2034ecd7417db3a7ea27",
-                "sha256:54aa2d64fd86383b5438d9f0c032f043c4d8c0264b8af9f60bd061157466ea43"
+                "sha256:e697488f66c3db49df675158a77b3b017520d772c6e1548c7d9bcc5df7944ee4",
+                "sha256:eb36caca59e74471988f0ae25c77945610b887f777255aa21f8065def9e51ed4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.27.2"
+            "version": "==2.27.3"
         },
         "jupyterlab-widgets": {
             "hashes": [
@@ -1594,37 +1646,30 @@
         },
         "llvmlite": {
             "hashes": [
-                "sha256:03aee0ccd81735696474dc4f8b6be60774892a2929d6c05d093d17392c237f32",
-                "sha256:1578f5000fdce513712e99543c50e93758a954297575610f48cb1fd71b27c08a",
-                "sha256:16f56eb1eec3cda3a5c526bc3f63594fc24e0c8d219375afeb336f289764c6c7",
-                "sha256:1ec3d70b3e507515936e475d9811305f52d049281eaa6c8273448a61c9b5b7e2",
-                "sha256:22d36591cd5d02038912321d9ab8e4668e53ae2211da5523f454e992b5e13c36",
-                "sha256:3803f11ad5f6f6c3d2b545a303d68d9fabb1d50e06a8d6418e6fcd2d0df00959",
-                "sha256:39dc2160aed36e989610fc403487f11b8764b6650017ff367e45384dff88ffbf",
-                "sha256:3fc14e757bc07a919221f0cbaacb512704ce5774d7fcada793f1996d6bc75f2a",
-                "sha256:4c6ebace910410daf0bebda09c1859504fc2f33d122e9a971c4c349c89cca630",
-                "sha256:50aea09a2b933dab7c9df92361b1844ad3145bfb8dd2deb9cd8b8917d59306fb",
-                "sha256:60f8dd1e76f47b3dbdee4b38d9189f3e020d22a173c00f930b52131001d801f9",
-                "sha256:62c0ea22e0b9dffb020601bb65cb11dd967a095a488be73f07d8867f4e327ca5",
-                "sha256:6546bed4e02a1c3d53a22a0bced254b3b6894693318b16c16c8e43e29d6befb6",
-                "sha256:6717c7a6e93c9d2c3d07c07113ec80ae24af45cde536b34363d4bcd9188091d9",
-                "sha256:7ebf1eb9badc2a397d4f6a6c8717447c81ac011db00064a00408bc83c923c0e4",
-                "sha256:9ffc84ade195abd4abcf0bd3b827b9140ae9ef90999429b9ea84d5df69c9058c",
-                "sha256:a3f331a323d0f0ada6b10d60182ef06c20a2f01be21699999d204c5750ffd0b4",
-                "sha256:b1a0bbdb274fb683f993198775b957d29a6f07b45d184c571ef2a721ce4388cf",
-                "sha256:b43abd7c82e805261c425d50335be9a6c4f84264e34d6d6e475207300005d572",
-                "sha256:c0f158e4708dda6367d21cf15afc58de4ebce979c7a1aa2f6b977aae737e2a54",
-                "sha256:d0bfd18c324549c0fec2c5dc610fd024689de6f27c6cc67e4e24a07541d6e49b",
-                "sha256:ddab526c5a2c4ccb8c9ec4821fcea7606933dc53f510e2a6eebb45a418d3488a",
-                "sha256:e172c73fccf7d6db4bd6f7de963dedded900d1a5c6778733241d878ba613980e",
-                "sha256:e2c00ff204afa721b0bb9835b5bf1ba7fba210eefcec5552a9e05a63219ba0dc",
-                "sha256:e31f4b799d530255aaf0566e3da2df5bfc35d3cd9d6d5a3dcc251663656c27b1",
-                "sha256:e4f212c018db951da3e1dc25c2651abc688221934739721f2dad5ff1dd5f90e7",
-                "sha256:fa9b26939ae553bf30a9f5c4c754db0fb2d2677327f2511e674aa2f5df941789",
-                "sha256:fb62fc7016b592435d3e3a8f680e3ea8897c3c9e62e6e6cc58011e7a4801439e"
+                "sha256:14f0e4bf2fd2d9a75a3534111e8ebeb08eda2f33e9bdd6dfa13282afacdde0ed",
+                "sha256:18e9953c748b105668487b7c81a3e97b046d8abf95c4ddc0cd3c94f4e4651ae8",
+                "sha256:35d80d61d0cda2d767f72de99450766250560399edc309da16937b93d3b676e7",
+                "sha256:3e8d0618cb9bfe40ac38a9633f2493d4d4e9fcc2f438d39a4e854f39cc0f5f98",
+                "sha256:47e147cdda9037f94b399bf03bfd8a6b6b1f2f90be94a454e3386f006455a9b4",
+                "sha256:6912a87782acdff6eb8bf01675ed01d60ca1f2551f8176a300a886f09e836a6a",
+                "sha256:6d4fd101f571a31acb1559ae1af30f30b1dc4b3186669f92ad780e17c81e91bc",
+                "sha256:74937acd22dc11b33946b67dca7680e6d103d6e90eeaaaf932603bec6fe7b03a",
+                "sha256:7a2872ee80dcf6b5dbdc838763d26554c2a18aa833d31a2635bff16aafefb9c9",
+                "sha256:7d434ec7e2ce3cc8f452d1cd9a28591745de022f931d67be688a737320dfcead",
+                "sha256:977525a1e5f4059316b183fb4fd34fa858c9eade31f165427a3977c95e3ee749",
+                "sha256:9cd2a7376f7b3367019b664c21f0c61766219faa3b03731113ead75107f3b66c",
+                "sha256:a289af9a1687c6cf463478f0fa8e8aa3b6fb813317b0d70bf1ed0759eab6f761",
+                "sha256:ae2b5b5c3ef67354824fb75517c8db5fbe93bc02cd9671f3c62271626bc041d5",
+                "sha256:bc9efc739cc6ed760f795806f67889923f7274276f0eb45092a1473e40d9b867",
+                "sha256:c1da416ab53e4f7f3bc8d4eeba36d801cc1894b9fbfbf2022b29b6bad34a7df2",
+                "sha256:d5bd550001d26450bd90777736c69d68c487d17bf371438f975229b2b8241a91",
+                "sha256:df6509e1507ca0760787a199d19439cc887bfd82226f5af746d6977bd9f66844",
+                "sha256:e0a9a1a39d4bf3517f2af9d23d479b4175ead205c592ceeb8b89af48a327ea57",
+                "sha256:eccce86bba940bae0d8d48ed925f21dbb813519169246e2ab292b5092aba121f",
+                "sha256:f99b600aa7f65235a5a05d0b9a9f31150c390f31261f2a0ba678e26823ec38f7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.39.1"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.43.0"
         },
         "lmoments3": {
             "hashes": [
@@ -1787,6 +1832,15 @@
             "markers": "python_version >= '3.7'",
             "version": "==3.0.2"
         },
+        "monaco": {
+            "hashes": [
+                "sha256:23c0d4f1603bbc6f91af26fcb176de6c0cab531f137d789aaaf467eddd8ac2a4",
+                "sha256:913dd70b9565e0c337c46d484785d2047e276e64189ab5ff06c717561ad3b01d"
+            ],
+            "index": "pypi",
+            "markers": "python_version < '3.13' and python_full_version >= '3.9.0'",
+            "version": "==0.13.1"
+        },
         "morecantile": {
             "hashes": [
                 "sha256:610f1dcc3ae0a99f0a0e6c05e18508d0e9e26b53279e06a65150939e2f54963b",
@@ -1955,18 +2009,11 @@
         },
         "multimethod": {
             "hashes": [
-                "sha256:7f2a4863967142e6db68632fef9cd79053c09670ba0c5f113301e245140bba5c",
-                "sha256:cb338f09395c0ee87d36c7691cdd794d13d8864358082cf1205f812edd5ce05a"
+                "sha256:8db8ef2a8d2a247e3570cc23317680892fdf903d84c8c1053667c8e8f7671a67",
+                "sha256:fd0c473c43558908d97cc06e4d68e8f69202f167db46f7b4e4058893e7dbdf60"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.11.2"
-        },
-        "multipledispatch": {
-            "hashes": [
-                "sha256:0c53cd8b077546da4e48869f49b13164bebafd0c2a5afceb6bb6a316e7fb46e4",
-                "sha256:5c839915465c68206c3e9c473357908216c28383b425361e5d144594bf85a7e0"
-            ],
-            "version": "==1.0.0"
+            "version": "==1.12"
         },
         "multivolumefile": {
             "hashes": [
@@ -2109,135 +2156,128 @@
         },
         "numba": {
             "hashes": [
-                "sha256:0240f9026b015e336069329839208ebd70ec34ae5bfbf402e4fcc8e06197528e",
-                "sha256:03634579d10a6129181129de293dd6b5eaabee86881369d24d63f8fe352dd6cb",
-                "sha256:03fe94cd31e96185cce2fae005334a8cc712fc2ba7756e52dff8c9400718173f",
-                "sha256:0611e6d3eebe4cb903f1a836ffdb2bda8d18482bcd0a0dcc56e79e2aa3fefef5",
-                "sha256:0da583c532cd72feefd8e551435747e0e0fbb3c0530357e6845fcc11e38d6aea",
-                "sha256:14dbbabf6ffcd96ee2ac827389afa59a70ffa9f089576500434c34abf9b054a4",
-                "sha256:32d9fef412c81483d7efe0ceb6cf4d3310fde8b624a9cecca00f790573ac96ee",
-                "sha256:3a993349b90569518739009d8f4b523dfedd7e0049e6838c0e17435c3e70dcc4",
-                "sha256:3cb1a07a082a61df80a468f232e452d818f5ae254b40c26390054e4e868556e0",
-                "sha256:42f9e1be942b215df7e6cc9948cf9c15bb8170acc8286c063a9e57994ef82fd1",
-                "sha256:4373da9757049db7c90591e9ec55a2e97b2b36ba7ae3bf9c956a513374077470",
-                "sha256:4e08e203b163ace08bad500b0c16f6092b1eb34fd1fce4feaf31a67a3a5ecf3b",
-                "sha256:553da2ce74e8862e18a72a209ed3b6d2924403bdd0fb341fa891c6455545ba7c",
-                "sha256:720886b852a2d62619ae3900fe71f1852c62db4f287d0c275a60219e1643fc04",
-                "sha256:85dbaed7a05ff96492b69a8900c5ba605551afb9b27774f7f10511095451137c",
-                "sha256:8a95ca9cc77ea4571081f6594e08bd272b66060634b8324e99cd1843020364f9",
-                "sha256:91f021145a8081f881996818474ef737800bcc613ffb1e618a655725a0f9e246",
-                "sha256:9f62672145f8669ec08762895fe85f4cf0ead08ce3164667f2b94b2f62ab23c3",
-                "sha256:a12ef323c0f2101529d455cfde7f4135eaa147bad17afe10b48634f796d96abd",
-                "sha256:c602d015478b7958408d788ba00a50272649c5186ea8baa6cf71d4a1c761bba1",
-                "sha256:c75e8a5f810ce80a0cfad6e74ee94f9fde9b40c81312949bf356b7304ef20740",
-                "sha256:d0ae9270a7a5cc0ede63cd234b4ff1ce166c7a749b91dbbf45e0000c56d3eade",
-                "sha256:d69ad934e13c15684e7887100a8f5f0f61d7a8e57e0fd29d9993210089a5b531",
-                "sha256:dbcc847bac2d225265d054993a7f910fda66e73d6662fe7156452cac0325b073",
-                "sha256:e64d338b504c9394a4a34942df4627e1e6cb07396ee3b49fe7b8d6420aa5104f",
-                "sha256:f4cfc3a19d1e26448032049c79fc60331b104f694cf570a9e94f4e2c9d0932bb",
-                "sha256:fbfb45e7b297749029cb28694abf437a78695a100e7c2033983d69f0ba2698d4",
-                "sha256:fcdf84ba3ed8124eb7234adfbb8792f311991cbf8aed1cad4b1b1a7ee08380c1"
+                "sha256:01ef4cd7d83abe087d644eaa3d95831b777aa21d441a23703d649e06b8e06b74",
+                "sha256:0b983bd6ad82fe868493012487f34eae8bf7dd94654951404114f23c3466d34b",
+                "sha256:0ebaa91538e996f708f1ab30ef4d3ddc344b64b5227b67a57aa74f401bb68b9d",
+                "sha256:1527dc578b95c7c4ff248792ec33d097ba6bef9eda466c948b68dfc995c25781",
+                "sha256:159e618ef213fba758837f9837fb402bbe65326e60ba0633dbe6c7f274d42c1b",
+                "sha256:19407ced081d7e2e4b8d8c36aa57b7452e0283871c296e12d798852bc7d7f198",
+                "sha256:3031547a015710140e8c87226b4cfe927cac199835e5bf7d4fe5cb64e814e3ab",
+                "sha256:38d6ea4c1f56417076ecf8fc327c831ae793282e0ff51080c5094cb726507b1c",
+                "sha256:3fb02b344a2a80efa6f677aa5c40cd5dd452e1b35f8d1c2af0dfd9ada9978e4b",
+                "sha256:4142d7ac0210cc86432b818338a2bc368dc773a2f5cf1e32ff7c5b378bd63ee8",
+                "sha256:5d761de835cd38fb400d2c26bb103a2726f548dc30368853121d66201672e651",
+                "sha256:5df6158e5584eece5fc83294b949fd30b9f1125df7708862205217e068aabf16",
+                "sha256:5f4fde652ea604ea3c86508a3fb31556a6157b2c76c8b51b1d45eb40c8598703",
+                "sha256:62908d29fb6a3229c242e981ca27e32a6e606cc253fc9e8faeb0e48760de241e",
+                "sha256:819a3dfd4630d95fd574036f99e47212a1af41cbcb019bf8afac63ff56834449",
+                "sha256:a17b70fc9e380ee29c42717e8cc0bfaa5556c416d94f9aa96ba13acb41bdece8",
+                "sha256:c151748cd269ddeab66334bd754817ffc0cabd9433acb0f551697e5151917d25",
+                "sha256:cac02c041e9b5bc8cf8f2034ff6f0dbafccd1ae9590dc146b3a02a45e53af4e2",
+                "sha256:d7da4098db31182fc5ffe4bc42c6f24cd7d1cb8a14b59fd755bfee32e34b8404",
+                "sha256:f75262e8fe7fa96db1dca93d53a194a38c46da28b112b8a4aca168f0df860347",
+                "sha256:fe0b28abb8d70f8160798f4de9d486143200f34458d34c4a214114e445d7124e"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==0.56.4"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.60.0"
         },
         "numcodecs": {
             "hashes": [
-                "sha256:05d91a433733e7eef268d7e80ec226a0232da244289614a8f3826901aec1098e",
-                "sha256:0e79bf9d1d37199ac00a60ff3adb64757523291d19d03116832e600cac391c51",
-                "sha256:135b2d47563f7b9dc5ee6ce3d1b81b0f1397f69309e909f1a35bb0f7c553d45e",
-                "sha256:21d8267bd4313f4d16f5b6287731d4c8ebdab236038f29ad1b0e93c9b2ca64ee",
-                "sha256:29dfb195f835a55c4d490fb097aac8c1bcb96c54cf1b037d9218492c95e9d8c5",
-                "sha256:2f1ba2f4af3fd3ba65b1bcffb717fe65efe101a50a91c368f79f3101dbb1e243",
-                "sha256:2f84df6b8693206365a5b37c005bfa9d1be486122bde683a7b6446af4b75d862",
-                "sha256:2fbb12a6a1abe95926f25c65e283762d63a9bf9e43c0de2c6a1a798347dfcb40",
-                "sha256:760627780a8b6afdb7f942f2a0ddaf4e31d3d7eea1d8498cf0fd3204a33c4618",
-                "sha256:82d7107f80f9307235cb7e74719292d101c7ea1e393fe628817f0d635b7384f5",
-                "sha256:941b7446b68cf79f089bcfe92edaa3b154533dcbcd82474f994b28f2eedb1c60",
-                "sha256:a191a8e347ecd016e5c357f2bf41fbcb026f6ffe78fff50c77ab12e96701d155",
-                "sha256:abff3554a6892a89aacf7b642a044e4535499edf07aeae2f2e6e8fc08c9ba07f",
-                "sha256:c17687b1fd1fef68af616bc83f896035d24e40e04e91e7e6dae56379eb59fe33",
-                "sha256:c258bd1d3dfa75a9b708540d23b2da43d63607f9df76dfa0309a7597d1de3b73",
-                "sha256:caf1a1e6678aab9c1e29d2109b299f7a467bd4d4c34235b1f0e082167846b88f",
-                "sha256:d37f628fe92b3699e65831d5733feca74d2e33b50ef29118ffd41c13c677210e",
-                "sha256:e04649ea504aff858dbe294631f098fbfd671baf58bfc04fc48d746554c05d67",
-                "sha256:eeaf42768910f1c6eebf6c1bb00160728e62c9343df9e2e315dc9fe12e3f6071",
-                "sha256:ef964d4860d3e6b38df0633caf3e51dc850a6293fd8e93240473642681d95136",
-                "sha256:f2207871868b2464dc11c513965fd99b958a9d7cde2629be7b2dc84fdaab013b"
+                "sha256:17bc4b568214582f4c623700592f633f3afd920848630049c584fa1e535253ad",
+                "sha256:208cab0f4d9cf4409e9c4a4c935e165833786614822c81dee9d865af372da9df",
+                "sha256:56e49f68ce6aeba29f144992524c8897d94f846d02bbcc820dd29d7c5c2a073e",
+                "sha256:820be89729583c91601a6b35c052008cdd2665b25bfedb91b367cc155fb34ba0",
+                "sha256:a68368d3ce625ec76fcacd84785f6110d30a232909d5c6093a7aa25628880477",
+                "sha256:ac4dd5556fb126271e93bd1a02266e21b01d3617db448d70d00eec8e034506b4",
+                "sha256:ba4fac7036ea5a078c7afe1d4dffeb9685080d42f19c9c16b12dad866703aa2e",
+                "sha256:d67a859dd8a7f026829e91cb1799c26720cc9d29ee4ae0060cc7a581670abc06",
+                "sha256:e7d3b9693df52eeaf978d2a56971d01cf9b4e284ae769ec764807f2087cce51d",
+                "sha256:eed420a9c62d0a569aa94a387f93045f068ad3e7bbd787c6ce70bc5fefbaa7d9",
+                "sha256:f208a1b8b5e66c767ed043812ca74d9045e09b7b2e085d064a585c30b9efc8e7",
+                "sha256:f3cf462d2357998d7f6baaa0427657b0eeda3eb79fba2b146d2d04542912a513",
+                "sha256:f5904216811f2e9d312c23ffaad3b3d4c7442a3583d3a8bf81ca8319e9f5deb5"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==0.12.1"
+            "markers": "python_version >= '3.10'",
+            "version": "==0.13.0"
         },
         "numexpr": {
             "hashes": [
-                "sha256:03d0ba492e484a5a1aeb24b300c4213ed168f2c246177be5733abb4e18cbb043",
-                "sha256:04e8620e7e676504201d4082e7b3ee2d9b561d1cb9470b47a6104e10c1e2870e",
-                "sha256:05278bad96b5846d712eba58b44e5cec743bdb3e19ca624916c921d049fdbcf6",
-                "sha256:10789450032357afaeda4ac4d06da9542d1535c13151e8d32b49ae1a488d1358",
-                "sha256:1af6dc6b3bd2e11a802337b352bf58f30df0b70be16c4f863b70a3af3a8ef95e",
-                "sha256:1d8eb88b0ae3d3c609d732a17e71096779b2bf47b3a084320ffa93d9f9132786",
-                "sha256:3c66dc0188358cdcc9465b6ee54fd5eef2e83ac64b1d4ba9117c41df59bf6fca",
-                "sha256:416e0e9f0fc4cced67767585e44cb6b301728bdb9edbb7c534a853222ec62cac",
-                "sha256:4f0b045e1831953a47cc9fabae76a6794c69cbb60921751a5cf2d555034c55bf",
-                "sha256:4feafc65ea3044b8bf8f305b757a928e59167a310630c22b97a57dff07a56490",
-                "sha256:56d0d96b130f7cd4d78d0017030d6a0e9d9fc2a717ac51d4cf4860b39637e86a",
-                "sha256:629b66cc1b750671e7fb396506b3f9410612e5bd8bc1dd55b5a0a0041d839f95",
-                "sha256:6b5f8242c075477156d26b3a6b8e0cd0a06d4c8eb68d907bde56dd3c9c683e92",
-                "sha256:745b46a1fb76920a3eebfaf26e50bc94a9c13b5aee34b256ab4b2d792dbaa9ca",
-                "sha256:748e8d4cde22d9a5603165293fb293a4de1a4623513299416c64fdab557118c2",
-                "sha256:78e0a8bc4417c3dedcbae3c473505b69080535246edc977c7dccf3ec8454a685",
-                "sha256:83f1e7a7f7ee741b8dcd20c56c3f862a3a3ec26fa8b9fcadb7dcd819876d2f35",
-                "sha256:937d36c6d3cf15601f26f84f0f706649f976491e9e0892d16cd7c876d77fa7dc",
-                "sha256:96a64d0dd8f8e694da3f8582d73d7da8446ff375f6dd239b546010efea371ac3",
-                "sha256:a602692cd52ce923ce8a0a90fb1d6cf186ebe8706eed83eee0de685e634b9aa9",
-                "sha256:a6cdf9e64c5b3dbb61729edb505ea75ee212fa02b85c5b1d851331381ae3b0e1",
-                "sha256:b276e2ba3e87ace9a30fd49078ad5dcdc6a1674d030b1ec132599c55465c0346",
-                "sha256:c7517b774d309b1f0896c89bdd1ddd33c4418a92ecfbe5e1df3ac698698f6fcf",
-                "sha256:c89e930752639df040539160326d8f99a84159bbea41943ab8e960591edaaef0",
-                "sha256:cb5e12787101f1216f2cdabedc3417748f2e1f472442e16bbfabf0bab2336300",
-                "sha256:d47bb567e330ebe86781864219a36cbccb3a47aec893bd509f0139c6b23e8104",
-                "sha256:dc3506c30c03b082da2cadef43747d474e5170c1f58a6dcdf882b3dc88b1e849",
-                "sha256:e3a973265591b0a875fd1151c4549e468959c7192821aac0bb86937694a08efa",
-                "sha256:efa63ecdc9fcaf582045639ddcf56e9bdc1f4d9a01729be528f62df4db86c9d6"
+                "sha256:00204e5853713b5eba5f3d0bc586a5d8d07f76011b597c8b4087592cc2ec2928",
+                "sha256:22cc65e9121aeb3187a2b50827715b2b087ea70e8ab21416ea52662322087b43",
+                "sha256:300e577b3c006dd7a8270f1bb2e8a00ee15bf235b1650fe2a6febec2954bc2c3",
+                "sha256:368a1972c3186355160f6ee330a7eea146d8443da75a38a30083289ae251ef5a",
+                "sha256:37598cca41f8f50dc889b0b72be1616a288758c16ab7d48c9ac8719e1a39d835",
+                "sha256:44f6d12a8c44be90199bbb10d3abf467f88951f48a3d1fbbd3c219d121f39c9d",
+                "sha256:45f598182b4f5c153222e47d5163c3bee8d5ebcaee7e56dd2a5898d4d97e4473",
+                "sha256:4f0985bd1c493b23b5aad7d81fa174798f3812efb78d14844194834c9fee38b8",
+                "sha256:552c8d4b2e3b87cdb2abb40a781b9a61a9090a9f66ac7357fc5a0b93aff76be3",
+                "sha256:56648a04679063175681195670ad53e5c8ca19668166ed13875199b5600089c7",
+                "sha256:5a4db4456e0779d5e024220b7b6a7477ac900679bfa74836b06fa526aaed4e3c",
+                "sha256:6a50370bea77ba94c3734a44781c716751354c6bfda2d369af3aed3d67d42871",
+                "sha256:78b14c19c403df7498954468385768c86b0d2c52ad03dffb74e45d44ae5a9c77",
+                "sha256:82bf04a1495ac475de4ab49fbe0a3a2710ed3fd1a00bc03847316b5d7602402d",
+                "sha256:82fc95c301b15ff4823f98989ee363a2d5555d16a7cfd3710e98ddee726eaaaa",
+                "sha256:926dd426c68f1d927412a2ad843831c1eb9a95871e7bb0bd8b20d547c12238d2",
+                "sha256:9bba99d354a65f1a008ab8b87f07d84404c668e66bab624df5b6b5373403cf81",
+                "sha256:a3c0b0bf165b2d886eb981afa4e77873ca076f5d51c491c4d7b8fc10f17c876f",
+                "sha256:ac23a72eff10f928f23b147bdeb0f1b774e862abe332fc9bf4837e9f1bc0bbf9",
+                "sha256:b28eaf45f1cc1048aad9e90e3a8ada1aef58c5f8155a85267dc781b37998c046",
+                "sha256:b2efa499f460124538a5b4f1bf2e77b28eb443ee244cc5573ed0f6a069ebc635",
+                "sha256:bbd35f17f6efc00ebd4a480192af1ee30996094a0d5343b131b0e90e61e8b554",
+                "sha256:ca8ae46481d0b0689ca0d00a8670bc464ce375e349599fe674a6d4957e7b7eb6",
+                "sha256:cbf79fef834f88607f977ab9867061dcd9b40ccb08bb28547c6dc6c73e560895",
+                "sha256:ce04ae6efe2a9d0be1a0e114115c3ae70c68b8b8fbc615c5c55c15704b01e6a4",
+                "sha256:fa4009d84a8e6e21790e718a80a22d57fe7f215283576ef2adc4183f7247f3c7",
+                "sha256:fb704620657a1c99d64933e8a982148d8bfb2b738a1943e107a2bfdee887ce56",
+                "sha256:fcbf013bb8494e8ef1d11fa3457827c1571c6a3153982d709e5d17594999d4dd",
+                "sha256:fecdf4bf3c1250e56583db0a4a80382a259ba4c2e1efa13e04ed43f0938071f5"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==2.10.0"
+            "version": "==2.10.1"
         },
         "numpy": {
             "hashes": [
-                "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d",
-                "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07",
-                "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df",
-                "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9",
-                "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d",
-                "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a",
-                "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719",
-                "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2",
-                "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280",
-                "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa",
-                "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387",
-                "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1",
-                "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43",
-                "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f",
-                "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398",
-                "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63",
-                "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de",
-                "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8",
-                "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481",
-                "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0",
-                "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d",
-                "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e",
-                "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96",
-                "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb",
-                "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6",
-                "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d",
-                "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a",
-                "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
+                "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b",
+                "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818",
+                "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20",
+                "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0",
+                "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010",
+                "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a",
+                "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea",
+                "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c",
+                "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71",
+                "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110",
+                "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be",
+                "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a",
+                "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a",
+                "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5",
+                "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed",
+                "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd",
+                "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c",
+                "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e",
+                "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0",
+                "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c",
+                "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a",
+                "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b",
+                "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0",
+                "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6",
+                "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2",
+                "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a",
+                "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30",
+                "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218",
+                "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5",
+                "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07",
+                "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2",
+                "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4",
+                "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764",
+                "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef",
+                "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3",
+                "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
-            "version": "==1.23.5"
+            "markers": "python_version >= '3.9'",
+            "version": "==1.26.4"
         },
         "numpy-groupies": {
             "hashes": [
@@ -2337,14 +2377,6 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.5.1"
         },
-        "param": {
-            "hashes": [
-                "sha256:a7b30b08b547e2b78b02aeba6ed34e3c6a638f8e4824a76a96ffa2d7cf57e71f",
-                "sha256:f31d3745d227347d29b5868c4e4e3077df07463889b91d3bb28e634fde211e1c"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==2.1.0"
-        },
         "parso": {
             "hashes": [
                 "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18",
@@ -2461,14 +2493,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==4.2.2"
         },
-        "pluggy": {
-            "hashes": [
-                "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1",
-                "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.5.0"
-        },
         "pre-commit": {
             "hashes": [
                 "sha256:10badb65d6a38caff29703362271d7dca483d01da88f9d7e05d0b97171c136cb",
@@ -2494,6 +2518,13 @@
             "markers": "python_full_version >= '3.7.0'",
             "version": "==3.0.47"
         },
+        "properscoring": {
+            "hashes": [
+                "sha256:b0cc4963cc218b728d6c5f77b3259c8f835ae00e32e82678cdf6936049b93961",
+                "sha256:f84d5b06c13549d0171ce52ad7b45c6f5726ac44b733d24af5c60654cbb821dc"
+            ],
+            "version": "==0.1"
+        },
         "psutil": {
             "hashes": [
                 "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d",
@@ -2513,7 +2544,7 @@
                 "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4",
                 "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"
             ],
-            "markers": "sys_platform != 'cygwin'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==5.9.8"
         },
         "psycopg2-binary": {
@@ -2760,55 +2791,54 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==3.20.0"
         },
-        "pyct": {
-            "hashes": [
-                "sha256:a4038a8885059ab8cac6f946ea30e0b5e6bdbe0b92b6723f06737035f9d65e8c",
-                "sha256:dd9f4ac5cbd8e37c352c04036062d3c5f67efec76d404761ef16b0cbf26aa6a0"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.5.0"
-        },
         "pydantic": {
             "hashes": [
-                "sha256:20a3b30fd255eeeb63caa9483502ba96b7795ce5bf895c6a179b3d909d9f53a6",
-                "sha256:2b71bd504d1573b0b722ae536e8ffb796bedeef978979d076bf206e77dcc55a5",
-                "sha256:3403a090db45d4027d2344859d86eb797484dfda0706cf87af79ace6a35274ef",
-                "sha256:37ebddef68370e6f26243acc94de56d291e01227a67b2ace26ea3543cf53dd5f",
-                "sha256:3b8d5bd97886f9eb59260594207c9f57dce14a6f869c6ceea90188715d29921a",
-                "sha256:409b810f387610cc7405ab2fa6f62bdf7ea485311845a242ebc0bd0496e7e5ac",
-                "sha256:4870f13a4fafd5bc3e93cff3169222534fad867918b188e83ee0496452978437",
-                "sha256:566a04ba755e8f701b074ffb134ddb4d429f75d5dced3fbd829a527aafe74c71",
-                "sha256:67b3714b97ff84b2689654851c2426389bcabfac9080617bcf4306c69db606f6",
-                "sha256:6dab5219659f95e357d98d70577b361383057fb4414cfdb587014a5f5c595f7b",
-                "sha256:748d10ab6089c5d196e1c8be9de48274f71457b01e59736f7a09c9dc34f51887",
-                "sha256:762aa598f79b4cac2f275d13336b2dd8662febee2a9c450a49a2ab3bec4b385f",
-                "sha256:7a26841be620309a9697f5b1ffc47dce74909e350c5315ccdac7a853484d468a",
-                "sha256:7a7db03339893feef2092ff7b1afc9497beed15ebd4af84c3042a74abce02d48",
-                "sha256:7aa75d1bd9cc275cf9782f50f60cddaf74cbaae19b6ada2a28e737edac420312",
-                "sha256:86936c383f7c38fd26d35107eb669c85d8f46dfceae873264d9bab46fe1c7dde",
-                "sha256:88546dc10a40b5b52cae87d64666787aeb2878f9a9b37825aedc2f362e7ae1da",
-                "sha256:8c40964596809eb616d94f9c7944511f620a1103d63d5510440ed2908fc410af",
-                "sha256:990027e77cda6072a566e433b6962ca3b96b4f3ae8bd54748e9d62a58284d9d7",
-                "sha256:9965e49c6905840e526e5429b09e4c154355b6ecc0a2f05492eda2928190311d",
-                "sha256:9f62a727f5c590c78c2d12fda302d1895141b767c6488fe623098f8792255fe5",
-                "sha256:a2d5be50ac4a0976817144c7d653e34df2f9436d15555189f5b6f61161d64183",
-                "sha256:a5939ec826f7faec434e2d406ff5e4eaf1716eb1f247d68cd3d0b3612f7b4c8a",
-                "sha256:aac218feb4af73db8417ca7518fb3bade4534fcca6e3fb00f84966811dd94450",
-                "sha256:adad1ee4ab9888f12dac2529276704e719efcf472e38df7813f5284db699b4ec",
-                "sha256:b69f9138dec566962ec65623c9d57bee44412d2fc71065a5f3ebb3820bdeee96",
-                "sha256:c41bbaae89e32fc582448e71974de738c055aef5ab474fb25692981a08df808a",
-                "sha256:c62376890b819bebe3c717a9ac841a532988372b7e600e76f75c9f7c128219d5",
-                "sha256:ce937a2a2c020bcad1c9fde02892392a1123de6dda906ddba62bfe8f3e5989a2",
-                "sha256:db4c7f7e60ca6f7d6c1785070f3e5771fcb9b2d88546e334d2f2c3934d949028",
-                "sha256:e0014e29637125f4997c174dd6167407162d7af0da73414a9340461ea8573252",
-                "sha256:e088e3865a2270ecbc369924cd7d9fbc565667d9158e7f304e4097ebb9cf98dd",
-                "sha256:ea9eebc2ebcba3717e77cdeee3f6203ffc0e78db5f7482c68b1293e8cc156e5e",
-                "sha256:edfdf0a5abc5c9bf2052ebaec20e67abd52e92d257e4f2d30e02c354ed3e6030",
-                "sha256:f3d4ee957a727ccb5a36f1b0a6dbd9fad5dedd2a41eada99a8df55c12896e18d",
-                "sha256:f79db3652ed743309f116ba863dae0c974a41b688242482638b892246b7db21d"
+                "sha256:098ad8de840c92ea586bf8efd9e2e90c6339d33ab5c1cfbb85be66e4ecf8213f",
+                "sha256:0e2495309b1266e81d259a570dd199916ff34f7f51f1b549a0d37a6d9b17b4dc",
+                "sha256:0fa51175313cc30097660b10eec8ca55ed08bfa07acbfe02f7a42f6c242e9a4b",
+                "sha256:11289fa895bcbc8f18704efa1d8020bb9a86314da435348f59745473eb042e6b",
+                "sha256:2a72d2a5ff86a3075ed81ca031eac86923d44bc5d42e719d585a8eb547bf0c9b",
+                "sha256:371dcf1831f87c9e217e2b6a0c66842879a14873114ebb9d0861ab22e3b5bb1e",
+                "sha256:409b2b36d7d7d19cd8310b97a4ce6b1755ef8bd45b9a2ec5ec2b124db0a0d8f3",
+                "sha256:4866a1579c0c3ca2c40575398a24d805d4db6cb353ee74df75ddeee3c657f9a7",
+                "sha256:48db882e48575ce4b39659558b2f9f37c25b8d348e37a2b4e32971dd5a7d6227",
+                "sha256:525bbef620dac93c430d5d6bdbc91bdb5521698d434adf4434a7ef6ffd5c4b7f",
+                "sha256:543da3c6914795b37785703ffc74ba4d660418620cc273490d42c53949eeeca6",
+                "sha256:62d96b8799ae3d782df7ec9615cb59fc32c32e1ed6afa1b231b0595f6516e8ab",
+                "sha256:6654028d1144df451e1da69a670083c27117d493f16cf83da81e1e50edce72ad",
+                "sha256:7017971ffa7fd7808146880aa41b266e06c1e6e12261768a28b8b41ba55c8076",
+                "sha256:7623b59876f49e61c2e283551cc3647616d2fbdc0b4d36d3d638aae8547ea681",
+                "sha256:7e17c0ee7192e54a10943f245dc79e36d9fe282418ea05b886e1c666063a7b54",
+                "sha256:820ae12a390c9cbb26bb44913c87fa2ff431a029a785642c1ff11fed0a095fcb",
+                "sha256:94833612d6fd18b57c359a127cbfd932d9150c1b72fea7c86ab58c2a77edd7c7",
+                "sha256:95ef534e3c22e5abbdbdd6f66b6ea9dac3ca3e34c5c632894f8625d13d084cbe",
+                "sha256:9c803a5113cfab7bbb912f75faa4fc1e4acff43e452c82560349fff64f852e1b",
+                "sha256:9e53fb834aae96e7b0dadd6e92c66e7dd9cdf08965340ed04c16813102a47fab",
+                "sha256:ab2f976336808fd5d539fdc26eb51f9aafc1f4b638e212ef6b6f05e753c8011d",
+                "sha256:ad1e33dc6b9787a6f0f3fd132859aa75626528b49cc1f9e429cdacb2608ad5f0",
+                "sha256:ae5184e99a060a5c80010a2d53c99aee76a3b0ad683d493e5f0620b5d86eeb75",
+                "sha256:aeb4e741782e236ee7dc1fb11ad94dc56aabaf02d21df0e79e0c21fe07c95741",
+                "sha256:b4ad32aed3bf5eea5ca5decc3d1bbc3d0ec5d4fbcd72a03cdad849458decbc63",
+                "sha256:b8ad363330557beac73159acfbeed220d5f1bfcd6b930302a987a375e02f74fd",
+                "sha256:bfbb18b616abc4df70591b8c1ff1b3eabd234ddcddb86b7cac82657ab9017e33",
+                "sha256:c1e51d1af306641b7d1574d6d3307eaa10a4991542ca324f0feb134fee259815",
+                "sha256:c31d281c7485223caf6474fc2b7cf21456289dbaa31401844069b77160cab9c7",
+                "sha256:c7e8988bb16988890c985bd2093df9dd731bfb9d5e0860db054c23034fab8f7a",
+                "sha256:c87cedb4680d1614f1d59d13fea353faf3afd41ba5c906a266f3f2e8c245d655",
+                "sha256:cafb9c938f61d1b182dfc7d44a7021326547b7b9cf695db5b68ec7b590214773",
+                "sha256:d2f89a719411cb234105735a520b7c077158a81e0fe1cb05a79c01fc5eb59d3c",
+                "sha256:d4b40c9e13a0b61583e5599e7950490c700297b4a375b55b2b592774332798b7",
+                "sha256:d4ecb515fa7cb0e46e163ecd9d52f9147ba57bc3633dca0e586cdb7a232db9e3",
+                "sha256:d8c209af63ccd7b22fba94b9024e8b7fd07feffee0001efae50dd99316b27768",
+                "sha256:db3b48d9283d80a314f7a682f7acae8422386de659fffaba454b77a083c3937d",
+                "sha256:e41b5b973e5c64f674b3b4720286ded184dcc26a691dd55f34391c62c6934688",
+                "sha256:e840e6b2026920fc3f250ea8ebfdedf6ea7a25b77bf04c6576178e681942ae0f",
+                "sha256:ebb249096d873593e014535ab07145498957091aa6ae92759a32d40cb9998e2e",
+                "sha256:f434160fb14b353caf634149baaf847206406471ba70e64657c1e8330277a991",
+                "sha256:fa43f362b46741df8f201bf3e7dff3569fa92069bcc7b4a740dea3602e27ab7a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.10.10"
+            "version": "==1.10.17"
         },
         "pyflakes": {
             "hashes": [
@@ -3001,14 +3031,24 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.5.0"
         },
-        "pytest": {
+        "pystac": {
+            "extras": [
+                "validation"
+            ],
             "hashes": [
-                "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d",
-                "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"
+                "sha256:4617fe5315a79785f79b616b8ac248ba3d4d561457c8300b34573309715808cd",
+                "sha256:a7c31b3dacc44dfc955d9da8c7351c7b5b99100254b36301a1e312709b51bf2f"
             ],
-            "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==7.3.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==1.10.1"
+        },
+        "pystac-client": {
+            "hashes": [
+                "sha256:4b0ed0f7177dfc6e394aeb3ecf1236364f315b1d38c107afbcbbef17c2f7db8b",
+                "sha256:b07c21f0bfbe7ea19cd23e535406ee08ee604b8ff8d9afcee666c0b1fe017dc4"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.7.5"
         },
         "python-dateutil": {
             "hashes": [
@@ -3399,116 +3439,116 @@
         },
         "rioxarray": {
             "hashes": [
-                "sha256:433b169cd10346ed5fe0123e61fafca4dcaf5ce1b6e2f6cc1a9c0e0bf7d7c1d4",
-                "sha256:849979c3542cefb5ac452af474a0b50ff08b7435c6db1ad615ddb65b1da4bbf3"
+                "sha256:a98ea9306739f119b63ffc2245f5d7d23ca1638b99c50ca282d932901f9272e8",
+                "sha256:b0f4a8917bde79c15b1507296bd256de1a33923867f5e2e3078bda2b6ce3673f"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.15.5"
+            "version": "==0.16.0"
         },
         "rpds-py": {
             "hashes": [
-                "sha256:05f3d615099bd9b13ecf2fc9cf2d839ad3f20239c678f461c753e93755d629ee",
-                "sha256:06d218939e1bf2ca50e6b0ec700ffe755e5216a8230ab3e87c059ebb4ea06afc",
-                "sha256:07f2139741e5deb2c5154a7b9629bc5aa48c766b643c1a6750d16f865a82c5fc",
-                "sha256:08d74b184f9ab6289b87b19fe6a6d1a97fbfea84b8a3e745e87a5de3029bf944",
-                "sha256:0abeee75434e2ee2d142d650d1e54ac1f8b01e6e6abdde8ffd6eeac6e9c38e20",
-                "sha256:154bf5c93d79558b44e5b50cc354aa0459e518e83677791e6adb0b039b7aa6a7",
-                "sha256:17c6d2155e2423f7e79e3bb18151c686d40db42d8645e7977442170c360194d4",
-                "sha256:1805d5901779662d599d0e2e4159d8a82c0b05faa86ef9222bf974572286b2b6",
-                "sha256:19ba472b9606c36716062c023afa2484d1e4220548751bda14f725a7de17b4f6",
-                "sha256:19e515b78c3fc1039dd7da0a33c28c3154458f947f4dc198d3c72db2b6b5dc93",
-                "sha256:1d54f74f40b1f7aaa595a02ff42ef38ca654b1469bef7d52867da474243cc633",
-                "sha256:207c82978115baa1fd8d706d720b4a4d2b0913df1c78c85ba73fe6c5804505f0",
-                "sha256:2625f03b105328729f9450c8badda34d5243231eef6535f80064d57035738360",
-                "sha256:27bba383e8c5231cd559affe169ca0b96ec78d39909ffd817f28b166d7ddd4d8",
-                "sha256:2c3caec4ec5cd1d18e5dd6ae5194d24ed12785212a90b37f5f7f06b8bedd7139",
-                "sha256:2cc7c1a47f3a63282ab0f422d90ddac4aa3034e39fc66a559ab93041e6505da7",
-                "sha256:2fc24a329a717f9e2448f8cd1f960f9dac4e45b6224d60734edeb67499bab03a",
-                "sha256:312fe69b4fe1ffbe76520a7676b1e5ac06ddf7826d764cc10265c3b53f96dbe9",
-                "sha256:32b7daaa3e9389db3695964ce8e566e3413b0c43e3394c05e4b243a4cd7bef26",
-                "sha256:338dee44b0cef8b70fd2ef54b4e09bb1b97fc6c3a58fea5db6cc083fd9fc2724",
-                "sha256:352a88dc7892f1da66b6027af06a2e7e5d53fe05924cc2cfc56495b586a10b72",
-                "sha256:35b2b771b13eee8729a5049c976197ff58a27a3829c018a04341bcf1ae409b2b",
-                "sha256:38e14fb4e370885c4ecd734f093a2225ee52dc384b86fa55fe3f74638b2cfb09",
-                "sha256:3c20f05e8e3d4fc76875fc9cb8cf24b90a63f5a1b4c5b9273f0e8225e169b100",
-                "sha256:3dd3cd86e1db5aadd334e011eba4e29d37a104b403e8ca24dcd6703c68ca55b3",
-                "sha256:489bdfe1abd0406eba6b3bb4fdc87c7fa40f1031de073d0cfb744634cc8fa261",
-                "sha256:48c2faaa8adfacefcbfdb5f2e2e7bdad081e5ace8d182e5f4ade971f128e6bb3",
-                "sha256:4a98a1f0552b5f227a3d6422dbd61bc6f30db170939bd87ed14f3c339aa6c7c9",
-                "sha256:4adec039b8e2928983f885c53b7cc4cda8965b62b6596501a0308d2703f8af1b",
-                "sha256:4e0ee01ad8260184db21468a6e1c37afa0529acc12c3a697ee498d3c2c4dcaf3",
-                "sha256:51584acc5916212e1bf45edd17f3a6b05fe0cbb40482d25e619f824dccb679de",
-                "sha256:531796fb842b53f2695e94dc338929e9f9dbf473b64710c28af5a160b2a8927d",
-                "sha256:5463c47c08630007dc0fe99fb480ea4f34a89712410592380425a9b4e1611d8e",
-                "sha256:5c45a639e93a0c5d4b788b2613bd637468edd62f8f95ebc6fcc303d58ab3f0a8",
-                "sha256:6031b25fb1b06327b43d841f33842b383beba399884f8228a6bb3df3088485ff",
-                "sha256:607345bd5912aacc0c5a63d45a1f73fef29e697884f7e861094e443187c02be5",
-                "sha256:618916f5535784960f3ecf8111581f4ad31d347c3de66d02e728de460a46303c",
-                "sha256:636a15acc588f70fda1661234761f9ed9ad79ebed3f2125d44be0862708b666e",
-                "sha256:673fdbbf668dd958eff750e500495ef3f611e2ecc209464f661bc82e9838991e",
-                "sha256:6afd80f6c79893cfc0574956f78a0add8c76e3696f2d6a15bca2c66c415cf2d4",
-                "sha256:6b5ff7e1d63a8281654b5e2896d7f08799378e594f09cf3674e832ecaf396ce8",
-                "sha256:6c4c4c3f878df21faf5fac86eda32671c27889e13570645a9eea0a1abdd50922",
-                "sha256:6cd8098517c64a85e790657e7b1e509b9fe07487fd358e19431cb120f7d96338",
-                "sha256:6d1e42d2735d437e7e80bab4d78eb2e459af48c0a46e686ea35f690b93db792d",
-                "sha256:6e30ac5e329098903262dc5bdd7e2086e0256aa762cc8b744f9e7bf2a427d3f8",
-                "sha256:70a838f7754483bcdc830444952fd89645569e7452e3226de4a613a4c1793fb2",
-                "sha256:720edcb916df872d80f80a1cc5ea9058300b97721efda8651efcd938a9c70a72",
-                "sha256:732672fbc449bab754e0b15356c077cc31566df874964d4801ab14f71951ea80",
-                "sha256:740884bc62a5e2bbb31e584f5d23b32320fd75d79f916f15a788d527a5e83644",
-                "sha256:7700936ef9d006b7ef605dc53aa364da2de5a3aa65516a1f3ce73bf82ecfc7ae",
-                "sha256:7732770412bab81c5a9f6d20aeb60ae943a9b36dcd990d876a773526468e7163",
-                "sha256:7750569d9526199c5b97e5a9f8d96a13300950d910cf04a861d96f4273d5b104",
-                "sha256:7f1944ce16401aad1e3f7d312247b3d5de7981f634dc9dfe90da72b87d37887d",
-                "sha256:81c5196a790032e0fc2464c0b4ab95f8610f96f1f2fa3d4deacce6a79852da60",
-                "sha256:8352f48d511de5f973e4f2f9412736d7dea76c69faa6d36bcf885b50c758ab9a",
-                "sha256:8927638a4d4137a289e41d0fd631551e89fa346d6dbcfc31ad627557d03ceb6d",
-                "sha256:8c7672e9fba7425f79019db9945b16e308ed8bc89348c23d955c8c0540da0a07",
-                "sha256:8d2e182c9ee01135e11e9676e9a62dfad791a7a467738f06726872374a83db49",
-                "sha256:910e71711d1055b2768181efa0a17537b2622afeb0424116619817007f8a2b10",
-                "sha256:942695a206a58d2575033ff1e42b12b2aece98d6003c6bc739fbf33d1773b12f",
-                "sha256:9437ca26784120a279f3137ee080b0e717012c42921eb07861b412340f85bae2",
-                "sha256:967342e045564cef76dfcf1edb700b1e20838d83b1aa02ab313e6a497cf923b8",
-                "sha256:998125738de0158f088aef3cb264a34251908dd2e5d9966774fdab7402edfab7",
-                "sha256:9e6934d70dc50f9f8ea47081ceafdec09245fd9f6032669c3b45705dea096b88",
-                "sha256:a3d456ff2a6a4d2adcdf3c1c960a36f4fd2fec6e3b4902a42a384d17cf4e7a65",
-                "sha256:a7b28c5b066bca9a4eb4e2f2663012debe680f097979d880657f00e1c30875a0",
-                "sha256:a888e8bdb45916234b99da2d859566f1e8a1d2275a801bb8e4a9644e3c7e7909",
-                "sha256:aa3679e751408d75a0b4d8d26d6647b6d9326f5e35c00a7ccd82b78ef64f65f8",
-                "sha256:aaa71ee43a703c321906813bb252f69524f02aa05bf4eec85f0c41d5d62d0f4c",
-                "sha256:b646bf655b135ccf4522ed43d6902af37d3f5dbcf0da66c769a2b3938b9d8184",
-                "sha256:b906b5f58892813e5ba5c6056d6a5ad08f358ba49f046d910ad992196ea61397",
-                "sha256:b9bb1f182a97880f6078283b3505a707057c42bf55d8fca604f70dedfdc0772a",
-                "sha256:bd1105b50ede37461c1d51b9698c4f4be6e13e69a908ab7751e3807985fc0346",
-                "sha256:bf18932d0003c8c4d51a39f244231986ab23ee057d235a12b2684ea26a353590",
-                "sha256:c273e795e7a0f1fddd46e1e3cb8be15634c29ae8ff31c196debb620e1edb9333",
-                "sha256:c69882964516dc143083d3795cb508e806b09fc3800fd0d4cddc1df6c36e76bb",
-                "sha256:c827576e2fa017a081346dce87d532a5310241648eb3700af9a571a6e9fc7e74",
-                "sha256:cbfbea39ba64f5e53ae2915de36f130588bba71245b418060ec3330ebf85678e",
-                "sha256:ce0bb20e3a11bd04461324a6a798af34d503f8d6f1aa3d2aa8901ceaf039176d",
-                "sha256:d0cee71bc618cd93716f3c1bf56653740d2d13ddbd47673efa8bf41435a60daa",
-                "sha256:d21be4770ff4e08698e1e8e0bce06edb6ea0626e7c8f560bc08222880aca6a6f",
-                "sha256:d31dea506d718693b6b2cffc0648a8929bdc51c70a311b2770f09611caa10d53",
-                "sha256:d44607f98caa2961bab4fa3c4309724b185b464cdc3ba6f3d7340bac3ec97cc1",
-                "sha256:d58ad6317d188c43750cb76e9deacf6051d0f884d87dc6518e0280438648a9ac",
-                "sha256:d70129cef4a8d979caa37e7fe957202e7eee8ea02c5e16455bc9808a59c6b2f0",
-                "sha256:d85164315bd68c0806768dc6bb0429c6f95c354f87485ee3593c4f6b14def2bd",
-                "sha256:d960de62227635d2e61068f42a6cb6aae91a7fe00fca0e3aeed17667c8a34611",
-                "sha256:dc48b479d540770c811fbd1eb9ba2bb66951863e448efec2e2c102625328e92f",
-                "sha256:e1735502458621921cee039c47318cb90b51d532c2766593be6207eec53e5c4c",
-                "sha256:e2be6e9dd4111d5b31ba3b74d17da54a8319d8168890fbaea4b9e5c3de630ae5",
-                "sha256:e4c39ad2f512b4041343ea3c7894339e4ca7839ac38ca83d68a832fc8b3748ab",
-                "sha256:ed402d6153c5d519a0faf1bb69898e97fb31613b49da27a84a13935ea9164dfc",
-                "sha256:ee17cd26b97d537af8f33635ef38be873073d516fd425e80559f4585a7b90c43",
-                "sha256:f3027be483868c99b4985fda802a57a67fdf30c5d9a50338d9db646d590198da",
-                "sha256:f5bab211605d91db0e2995a17b5c6ee5edec1270e46223e513eaa20da20076ac",
-                "sha256:f6f8e3fecca256fefc91bb6765a693d96692459d7d4c644660a9fff32e517843",
-                "sha256:f7afbfee1157e0f9376c00bb232e80a60e59ed716e3211a80cb8506550671e6e",
-                "sha256:fa242ac1ff583e4ec7771141606aafc92b361cd90a05c30d93e343a0c2d82a89",
-                "sha256:fab6ce90574645a0d6c58890e9bcaac8d94dff54fb51c69e5522a7358b80ab64"
+                "sha256:0121803b0f424ee2109d6e1f27db45b166ebaa4b32ff47d6aa225642636cd834",
+                "sha256:06925c50f86da0596b9c3c64c3837b2481337b83ef3519e5db2701df695453a4",
+                "sha256:071d4adc734de562bd11d43bd134330fb6249769b2f66b9310dab7460f4bf714",
+                "sha256:1540d807364c84516417115c38f0119dfec5ea5c0dd9a25332dea60b1d26fc4d",
+                "sha256:15e65395a59d2e0e96caf8ee5389ffb4604e980479c32742936ddd7ade914b22",
+                "sha256:19d02c45f2507b489fd4df7b827940f1420480b3e2e471e952af4d44a1ea8e34",
+                "sha256:1c26da90b8d06227d7769f34915913911222d24ce08c0ab2d60b354e2d9c7aff",
+                "sha256:1d16089dfa58719c98a1c06f2daceba6d8e3fb9b5d7931af4a990a3c486241cb",
+                "sha256:1dd46f309e953927dd018567d6a9e2fb84783963650171f6c5fe7e5c41fd5666",
+                "sha256:2575efaa5d949c9f4e2cdbe7d805d02122c16065bfb8d95c129372d65a291a0b",
+                "sha256:3208f9aea18991ac7f2b39721e947bbd752a1abbe79ad90d9b6a84a74d44409b",
+                "sha256:329c719d31362355a96b435f4653e3b4b061fcc9eba9f91dd40804ca637d914e",
+                "sha256:3384d278df99ec2c6acf701d067147320b864ef6727405d6470838476e44d9e8",
+                "sha256:34a01a4490e170376cd79258b7f755fa13b1a6c3667e872c8e35051ae857a92b",
+                "sha256:354f3a91718489912f2e0fc331c24eaaf6a4565c080e00fbedb6015857c00582",
+                "sha256:37f46bb11858717e0efa7893c0f7055c43b44c103e40e69442db5061cb26ed34",
+                "sha256:3b4cf5a9497874822341c2ebe0d5850fed392034caadc0bad134ab6822c0925b",
+                "sha256:3f148c3f47f7f29a79c38cc5d020edcb5ca780020fab94dbc21f9af95c463581",
+                "sha256:443cec402ddd650bb2b885113e1dcedb22b1175c6be223b14246a714b61cd521",
+                "sha256:462b0c18fbb48fdbf980914a02ee38c423a25fcc4cf40f66bacc95a2d2d73bc8",
+                "sha256:474bc83233abdcf2124ed3f66230a1c8435896046caa4b0b5ab6013c640803cc",
+                "sha256:4d438e4c020d8c39961deaf58f6913b1bf8832d9b6f62ec35bd93e97807e9cbc",
+                "sha256:4fdc9afadbeb393b4bbbad75481e0ea78e4469f2e1d713a90811700830b553a9",
+                "sha256:5039e3cef7b3e7a060de468a4a60a60a1f31786da94c6cb054e7a3c75906111c",
+                "sha256:5095a7c838a8647c32aa37c3a460d2c48debff7fc26e1136aee60100a8cd8f68",
+                "sha256:52e466bea6f8f3a44b1234570244b1cff45150f59a4acae3fcc5fd700c2993ca",
+                "sha256:535d4b52524a961d220875688159277f0e9eeeda0ac45e766092bfb54437543f",
+                "sha256:57dbc9167d48e355e2569346b5aa4077f29bf86389c924df25c0a8b9124461fb",
+                "sha256:5a4b07cdf3f84310c08c1de2c12ddadbb7a77568bcb16e95489f9c81074322ed",
+                "sha256:5c872814b77a4e84afa293a1bee08c14daed1068b2bb1cc312edbf020bbbca2b",
+                "sha256:5f83689a38e76969327e9b682be5521d87a0c9e5a2e187d2bc6be4765f0d4600",
+                "sha256:688aa6b8aa724db1596514751ffb767766e02e5c4a87486ab36b8e1ebc1aedac",
+                "sha256:6b130bd4163c93798a6b9bb96be64a7c43e1cec81126ffa7ffaa106e1fc5cef5",
+                "sha256:6b31f059878eb1f5da8b2fd82480cc18bed8dcd7fb8fe68370e2e6285fa86da6",
+                "sha256:6d45080095e585f8c5097897313def60caa2046da202cdb17a01f147fb263b81",
+                "sha256:6f2f78ef14077e08856e788fa482107aa602636c16c25bdf59c22ea525a785e9",
+                "sha256:6fe87efd7f47266dfc42fe76dae89060038f1d9cb911f89ae7e5084148d1cc08",
+                "sha256:75969cf900d7be665ccb1622a9aba225cf386bbc9c3bcfeeab9f62b5048f4a07",
+                "sha256:75a6076289b2df6c8ecb9d13ff79ae0cad1d5fb40af377a5021016d58cd691ec",
+                "sha256:78d57546bad81e0da13263e4c9ce30e96dcbe720dbff5ada08d2600a3502e526",
+                "sha256:79e205c70afddd41f6ee79a8656aec738492a550247a7af697d5bd1aee14f766",
+                "sha256:7c98298a15d6b90c8f6e3caa6457f4f022423caa5fa1a1ca7a5e9e512bdb77a4",
+                "sha256:7ec72df7354e6b7f6eb2a17fa6901350018c3a9ad78e48d7b2b54d0412539a67",
+                "sha256:81ea573aa46d3b6b3d890cd3c0ad82105985e6058a4baed03cf92518081eec8c",
+                "sha256:8344127403dea42f5970adccf6c5957a71a47f522171fafaf4c6ddb41b61703a",
+                "sha256:8445f23f13339da640d1be8e44e5baf4af97e396882ebbf1692aecd67f67c479",
+                "sha256:850720e1b383df199b8433a20e02b25b72f0fded28bc03c5bd79e2ce7ef050be",
+                "sha256:88cb4bac7185a9f0168d38c01d7a00addece9822a52870eee26b8d5b61409213",
+                "sha256:8a790d235b9d39c70a466200d506bb33a98e2ee374a9b4eec7a8ac64c2c261fa",
+                "sha256:8b1a94b8afc154fbe36978a511a1f155f9bd97664e4f1f7a374d72e180ceb0ae",
+                "sha256:8d6ad132b1bc13d05ffe5b85e7a01a3998bf3a6302ba594b28d61b8c2cf13aaf",
+                "sha256:8eb488ef928cdbc05a27245e52de73c0d7c72a34240ef4d9893fdf65a8c1a955",
+                "sha256:90bf55d9d139e5d127193170f38c584ed3c79e16638890d2e36f23aa1630b952",
+                "sha256:9133d75dc119a61d1a0ded38fb9ba40a00ef41697cc07adb6ae098c875195a3f",
+                "sha256:93a91c2640645303e874eada51f4f33351b84b351a689d470f8108d0e0694210",
+                "sha256:959179efb3e4a27610e8d54d667c02a9feaa86bbabaf63efa7faa4dfa780d4f1",
+                "sha256:9625367c8955e4319049113ea4f8fee0c6c1145192d57946c6ffcd8fe8bf48dd",
+                "sha256:9da6f400eeb8c36f72ef6646ea530d6d175a4f77ff2ed8dfd6352842274c1d8b",
+                "sha256:9e65489222b410f79711dc3d2d5003d2757e30874096b2008d50329ea4d0f88c",
+                "sha256:a3e2fd14c5d49ee1da322672375963f19f32b3d5953f0615b175ff7b9d38daed",
+                "sha256:a5a7c1062ef8aea3eda149f08120f10795835fc1c8bc6ad948fb9652a113ca55",
+                "sha256:a5da93debdfe27b2bfc69eefb592e1831d957b9535e0943a0ee8b97996de21b5",
+                "sha256:a6e605bb9edcf010f54f8b6a590dd23a4b40a8cb141255eec2a03db249bc915b",
+                "sha256:a707b158b4410aefb6b054715545bbb21aaa5d5d0080217290131c49c2124a6e",
+                "sha256:a8b6683a37338818646af718c9ca2a07f89787551057fae57c4ec0446dc6224b",
+                "sha256:aa5476c3e3a402c37779e95f7b4048db2cb5b0ed0b9d006983965e93f40fe05a",
+                "sha256:ab1932ca6cb8c7499a4d87cb21ccc0d3326f172cfb6a64021a889b591bb3045c",
+                "sha256:ae8b6068ee374fdfab63689be0963333aa83b0815ead5d8648389a8ded593378",
+                "sha256:b0906357f90784a66e89ae3eadc2654f36c580a7d65cf63e6a616e4aec3a81be",
+                "sha256:b0da31853ab6e58a11db3205729133ce0df26e6804e93079dee095be3d681dc1",
+                "sha256:b1c30841f5040de47a0046c243fc1b44ddc87d1b12435a43b8edff7e7cb1e0d0",
+                "sha256:b228e693a2559888790936e20f5f88b6e9f8162c681830eda303bad7517b4d5a",
+                "sha256:b7cc6cb44f8636fbf4a934ca72f3e786ba3c9f9ba4f4d74611e7da80684e48d2",
+                "sha256:ba0ed0dc6763d8bd6e5de5cf0d746d28e706a10b615ea382ac0ab17bb7388633",
+                "sha256:bc9128e74fe94650367fe23f37074f121b9f796cabbd2f928f13e9661837296d",
+                "sha256:bcf426a8c38eb57f7bf28932e68425ba86def6e756a5b8cb4731d8e62e4e0223",
+                "sha256:bec35eb20792ea64c3c57891bc3ca0bedb2884fbac2c8249d9b731447ecde4fa",
+                "sha256:c3444fe52b82f122d8a99bf66777aed6b858d392b12f4c317da19f8234db4533",
+                "sha256:c5c9581019c96f865483d031691a5ff1cc455feb4d84fc6920a5ffc48a794d8a",
+                "sha256:c6feacd1d178c30e5bc37184526e56740342fd2aa6371a28367bad7908d454fc",
+                "sha256:c8f77e661ffd96ff104bebf7d0f3255b02aa5d5b28326f5408d6284c4a8b3248",
+                "sha256:cb0f6eb3a320f24b94d177e62f4074ff438f2ad9d27e75a46221904ef21a7b05",
+                "sha256:ce84a7efa5af9f54c0aa7692c45861c1667080814286cacb9958c07fc50294fb",
+                "sha256:cf902878b4af334a09de7a45badbff0389e7cf8dc2e4dcf5f07125d0b7c2656d",
+                "sha256:dab8d921b55a28287733263c0e4c7db11b3ee22aee158a4de09f13c93283c62d",
+                "sha256:dc9ac4659456bde7c567107556ab065801622396b435a3ff213daef27b495388",
+                "sha256:dd36b712d35e757e28bf2f40a71e8f8a2d43c8b026d881aa0c617b450d6865c9",
+                "sha256:e19509145275d46bc4d1e16af0b57a12d227c8253655a46bbd5ec317e941279d",
+                "sha256:e21cc693045fda7f745c790cb687958161ce172ffe3c5719ca1764e752237d16",
+                "sha256:e54548e0be3ac117595408fd4ca0ac9278fde89829b0b518be92863b17ff67a2",
+                "sha256:e5b9fc03bf76a94065299d4a2ecd8dfbae4ae8e2e8098bbfa6ab6413ca267709",
+                "sha256:e8481b946792415adc07410420d6fc65a352b45d347b78fec45d8f8f0d7496f0",
+                "sha256:ebcbf356bf5c51afc3290e491d3722b26aaf5b6af3c1c7f6a1b757828a46e336",
+                "sha256:ef9101f3f7b59043a34f1dccbb385ca760467590951952d6701df0da9893ca0c",
+                "sha256:f2afd2164a1e85226fcb6a1da77a5c8896c18bfe08e82e8ceced5181c42d2179",
+                "sha256:f629ecc2db6a4736b5ba95a8347b0089240d69ad14ac364f557d52ad68cf94b0",
+                "sha256:f68eea5df6347d3f1378ce992d86b2af16ad7ff4dcb4a19ccdc23dea901b87fb",
+                "sha256:f757f359f30ec7dcebca662a6bd46d1098f8b9fb1fcd661a9e13f2e8ce343ba1",
+                "sha256:fb37bd599f031f1a6fb9e58ec62864ccf3ad549cf14bac527dbfa97123edcca4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.18.1"
+            "version": "==0.19.0"
         },
         "rtree": {
             "hashes": [
@@ -3562,6 +3602,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.0.1"
         },
+        "s3fs": {
+            "hashes": [
+                "sha256:63fd8ddf05eb722de784b7b503196107f2a518061298cf005a8a4715b4d49117",
+                "sha256:d1a0a423d0d2e17fb2a193d9531935dc3f45ba742693448a461b6b34f6a92a24"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.6.0"
+        },
         "s3transfer": {
             "hashes": [
                 "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084",
@@ -3570,33 +3618,64 @@
             "markers": "python_version >= '3.7'",
             "version": "==0.6.2"
         },
+        "scikit-learn": {
+            "hashes": [
+                "sha256:0828673c5b520e879f2af6a9e99eee0eefea69a2188be1ca68a6121b809055c1",
+                "sha256:0ea5d40c0e3951df445721927448755d3fe1d80833b0b7308ebff5d2a45e6414",
+                "sha256:10e49170691514a94bb2e03787aa921b82dbc507a4ea1f20fd95557862c98dc1",
+                "sha256:154297ee43c0b83af12464adeab378dee2d0a700ccd03979e2b821e7dd7cc1c2",
+                "sha256:161808750c267b77b4a9603cf9c93579c7a74ba8486b1336034c2f1579546d21",
+                "sha256:1bd8d3a19d4bd6dc5a7d4f358c8c3a60934dc058f363c34c0ac1e9e12a31421d",
+                "sha256:1ff4ba34c2abff5ec59c803ed1d97d61b036f659a17f55be102679e88f926fac",
+                "sha256:508907e5f81390e16d754e8815f7497e52139162fd69c4fdbd2dfa5d6cc88915",
+                "sha256:5944ce1faada31c55fb2ba20a5346b88e36811aab504ccafb9f0339e9f780395",
+                "sha256:5f57428de0c900a98389c4a433d4a3cf89de979b3aa24d1c1d251802aa15e44d",
+                "sha256:689b6f74b2c880276e365fe84fe4f1befd6a774f016339c65655eaff12e10cbf",
+                "sha256:781586c414f8cc58e71da4f3d7af311e0505a683e112f2f62919e3019abd3745",
+                "sha256:7b073a27797a283187a4ef4ee149959defc350b46cbf63a84d8514fe16b69855",
+                "sha256:88e0672c7ac21eb149d409c74cc29f1d611d5158175846e7a9c2427bd12b3956",
+                "sha256:909144d50f367a513cee6090873ae582dba019cb3fca063b38054fa42704c3a4",
+                "sha256:97625f217c5c0c5d0505fa2af28ae424bd37949bb2f16ace3ff5f2f81fb4498b",
+                "sha256:9a07f90846313a7639af6a019d849ff72baadfa4c74c778821ae0fad07b7275b",
+                "sha256:b59e3e62d2be870e5c74af4e793293753565c7383ae82943b83383fdcf5cc5c1",
+                "sha256:b5e865e9bd59396220de49cb4a57b17016256637c61b4c5cc81aaf16bc123bbe",
+                "sha256:da3f404e9e284d2b0a157e1b56b6566a34eb2798205cba35a211df3296ab7a74",
+                "sha256:f5b213bc29cc30a89a3130393b0e39c847a15d769d6e59539cd86b75d276b1a7"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.5.1"
+        },
         "scipy": {
             "hashes": [
-                "sha256:049a8bbf0ad95277ffba9b3b7d23e5369cc39e66406d60422c8cfef40ccc8415",
-                "sha256:07c3457ce0b3ad5124f98a86533106b643dd811dd61b548e78cf4c8786652f6f",
-                "sha256:0f1564ea217e82c1bbe75ddf7285ba0709ecd503f048cb1236ae9995f64217bd",
-                "sha256:1553b5dcddd64ba9a0d95355e63fe6c3fc303a8fd77c7bc91e77d61363f7433f",
-                "sha256:15a35c4242ec5f292c3dd364a7c71a61be87a3d4ddcc693372813c0b73c9af1d",
-                "sha256:1b4735d6c28aad3cdcf52117e0e91d6b39acd4272f3f5cd9907c24ee931ad601",
-                "sha256:2cf9dfb80a7b4589ba4c40ce7588986d6d5cebc5457cad2c2880f6bc2d42f3a5",
-                "sha256:39becb03541f9e58243f4197584286e339029e8908c46f7221abeea4b749fa88",
-                "sha256:43b8e0bcb877faf0abfb613d51026cd5cc78918e9530e375727bf0625c82788f",
-                "sha256:4b3f429188c66603a1a5c549fb414e4d3bdc2a24792e061ffbd607d3d75fd84e",
-                "sha256:4c0ff64b06b10e35215abce517252b375e580a6125fd5fdf6421b98efbefb2d2",
-                "sha256:51af417a000d2dbe1ec6c372dfe688e041a7084da4fdd350aeb139bd3fb55353",
-                "sha256:5678f88c68ea866ed9ebe3a989091088553ba12c6090244fdae3e467b1139c35",
-                "sha256:79c8e5a6c6ffaf3a2262ef1be1e108a035cf4f05c14df56057b64acc5bebffb6",
-                "sha256:7ff7f37b1bf4417baca958d254e8e2875d0cc23aaadbe65b3d5b3077b0eb23ea",
-                "sha256:aaea0a6be54462ec027de54fca511540980d1e9eea68b2d5c1dbfe084797be35",
-                "sha256:bce5869c8d68cf383ce240e44c1d9ae7c06078a9396df68ce88a1230f93a30c1",
-                "sha256:cd9f1027ff30d90618914a64ca9b1a77a431159df0e2a195d8a9e8a04c78abf9",
-                "sha256:d925fa1c81b772882aa55bcc10bf88324dadb66ff85d548c71515f6689c6dac5",
-                "sha256:e7354fd7527a4b0377ce55f286805b34e8c54b91be865bac273f527e1b839019",
-                "sha256:fae8a7b898c42dffe3f7361c40d5952b6bf32d10c4569098d276b4c547905ee1"
+                "sha256:076c27284c768b84a45dcf2e914d4000aac537da74236a0d45d82c6fa4b7b3c0",
+                "sha256:07e179dc0205a50721022344fb85074f772eadbda1e1b3eecdc483f8033709b7",
+                "sha256:176c6f0d0470a32f1b2efaf40c3d37a24876cebf447498a4cefb947a79c21e9d",
+                "sha256:42470ea0195336df319741e230626b6225a740fd9dce9642ca13e98f667047c0",
+                "sha256:4c4161597c75043f7154238ef419c29a64ac4a7c889d588ea77690ac4d0d9b20",
+                "sha256:5b083c8940028bb7e0b4172acafda6df762da1927b9091f9611b0bcd8676f2bc",
+                "sha256:64b2ff514a98cf2bb734a9f90d32dc89dc6ad4a4a36a312cd0d6327170339eb0",
+                "sha256:65df4da3c12a2bb9ad52b86b4dcf46813e869afb006e58be0f516bc370165159",
+                "sha256:687af0a35462402dd851726295c1a5ae5f987bd6e9026f52e9505994e2f84ef6",
+                "sha256:6a9c9a9b226d9a21e0a208bdb024c3982932e43811b62d202aaf1bb59af264b1",
+                "sha256:6d056a8709ccda6cf36cdd2eac597d13bc03dba38360f418560a93050c76a16e",
+                "sha256:7d3da42fbbbb860211a811782504f38ae7aaec9de8764a9bef6b262de7a2b50f",
+                "sha256:7e911933d54ead4d557c02402710c2396529540b81dd554fc1ba270eb7308484",
+                "sha256:94c164a9e2498e68308e6e148646e486d979f7fcdb8b4cf34b5441894bdb9caf",
+                "sha256:9e3154691b9f7ed73778d746da2df67a19d046a6c8087c8b385bc4cdb2cfca74",
+                "sha256:9eee2989868e274aae26125345584254d97c56194c072ed96cb433f32f692ed8",
+                "sha256:a01cc03bcdc777c9da3cfdcc74b5a75caffb48a6c39c8450a9a05f82c4250a14",
+                "sha256:a7d46c3e0aea5c064e734c3eac5cf9eb1f8c4ceee756262f2c7327c4c2691c86",
+                "sha256:ad36af9626d27a4326c8e884917b7ec321d8a1841cd6dacc67d2a9e90c2f0359",
+                "sha256:b5923f48cb840380f9854339176ef21763118a7300a88203ccd0bdd26e58527b",
+                "sha256:bbc0471b5f22c11c389075d091d3885693fd3f5e9a54ce051b46308bc787e5d4",
+                "sha256:bff2438ea1330e06e53c424893ec0072640dac00f29c6a43a575cbae4c99b2b9",
+                "sha256:c40003d880f39c11c1edbae8144e3813904b10514cd3d3d00c277ae996488cdb",
+                "sha256:d91db2c41dd6c20646af280355d41dfa1ec7eead235642178bd57635a3f82209",
+                "sha256:f0a50da861a7ec4573b7c716b2ebdcdf142b66b756a0d392c236ae568b3a93fb"
             ],
             "index": "pypi",
-            "markers": "python_version < '3.12' and python_version >= '3.8'",
-            "version": "==1.10.1"
+            "markers": "python_version >= '3.10'",
+            "version": "==1.14.0"
         },
         "seaborn": {
             "hashes": [
@@ -3617,11 +3696,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4",
-                "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"
+                "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5",
+                "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==70.0.0"
+            "version": "==70.3.0"
         },
         "shapely": {
             "hashes": [
@@ -3795,6 +3874,13 @@
             ],
             "version": "==1.4.7"
         },
+        "sortedcontainers": {
+            "hashes": [
+                "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88",
+                "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"
+            ],
+            "version": "==2.4.0"
+        },
         "soupsieve": {
             "hashes": [
                 "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690",
@@ -3834,6 +3920,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.8.0"
         },
+        "tblib": {
+            "hashes": [
+                "sha256:80a6c77e59b55e83911e1e607c649836a69c103963c5f28a46cbeef44acf8129",
+                "sha256:93622790a0a29e04f0346458face1e144dc4d32f493714c6c3dff82a4adb77e6"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.0.0"
+        },
         "terminado": {
             "hashes": [
                 "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0",
@@ -3849,6 +3943,14 @@
             ],
             "version": "==1.7.0"
         },
+        "threadpoolctl": {
+            "hashes": [
+                "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107",
+                "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.5.0"
+        },
         "tinycss2": {
             "hashes": [
                 "sha256:152f9acabd296a8375fbca5b84c961ff95971fcfc32e79550c8df8e29118c54d",
@@ -3955,20 +4057,20 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07",
-                "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
+                "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3",
+                "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"
             ],
             "index": "pypi",
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.18"
+            "version": "==1.26.19"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c",
-                "sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b"
+                "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a",
+                "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.26.2"
+            "version": "==20.26.3"
         },
         "wcwidth": {
             "hashes": [
@@ -4093,20 +4195,28 @@
         },
         "xarray": {
             "hashes": [
-                "sha256:7bee552751ff1b29dab8b7715726e5ecb56691ac54593cf4881dff41978ce0cd",
-                "sha256:7e530b1deafdd43e5c2b577d0944e6b528fbe88045fd849e49a8d11871ecd522"
+                "sha256:0b91e0bc4dc0296947947640fe31ec6e867ce258d2f7cbc10bedf4a6d68340c7",
+                "sha256:721a7394e8ec3d592b2d8ebe21eed074ac077dc1bb1bd777ce00e41700b4866c"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
-            "version": "==2023.1.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==2024.6.0"
+        },
+        "xhistogram": {
+            "hashes": [
+                "sha256:56b0751e1469eaed81710f644c8ba5c574b51883baa2feee26a95f2f708f91a1",
+                "sha256:ad55330d55296d273b3370678223fde0f50085e04cb744c7b3b0bb7702a2c6bf"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.3.2"
         },
-        "xarray-spatial": {
+        "xskillscore": {
             "hashes": [
-                "sha256:8a356ac66a61ff9522453194db5d184e1eccee8ab890c7ec723bb48c6eafd392",
-                "sha256:fd7e3b447236f6e8f8c5110684a7ca50f97149c806f56ebf4ed1eeef1dd681ce"
+                "sha256:39c3ffa75e2cbbde87470ad957d667e729025ac6a6077ca8efce0ca5f69bfafa",
+                "sha256:ce3306c60626eafea722a1522016e272ca516ce6f2447c75f92c52888939f8c2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.3.5"
+            "version": "==0.0.24"
         },
         "xyzservices": {
             "hashes": [
@@ -4308,6 +4418,14 @@
             "markers": "python_version >= '3.9'",
             "version": "==2.18.0"
         },
+        "zict": {
+            "hashes": [
+                "sha256:5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae",
+                "sha256:e321e263b6a97aafc0790c3cfb3c04656b7066e6738c37fffcca95d803c9fba5"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.0.0"
+        },
         "zipp": {
             "hashes": [
                 "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19",
@@ -4335,11 +4453,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad",
-                "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"
+                "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b",
+                "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.2.1"
+            "version": "==1.2.2"
         },
         "executing": {
             "hashes": [
@@ -4351,11 +4469,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:53eee7ad44df903a06655871cbab66d156a051fd86f3ec6750470ac9604ac1ab",
-                "sha256:c6ed726a140b6e725b911528f80439c534fac915246af3efc39440a6b0f9d716"
+                "sha256:1cec0fbba8404af13facebe83d04436a7434c7400e59f47acf467c64abd0956c",
+                "sha256:e6b347c27bdf9c32ee9d31ae85defc525755a1869f14057e900675b9e8d6e6ff"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.25.0"
+            "version": "==8.26.0"
         },
         "jedi": {
             "hashes": [
diff --git a/config/aws_s3_put_fim3_hydrovis_whitelist.lst b/config/aws_s3_put_fim3_hydrovis_whitelist.lst
deleted file mode 100644
index 5d8ed5ca7..000000000
--- a/config/aws_s3_put_fim3_hydrovis_whitelist.lst
+++ /dev/null
@@ -1,4 +0,0 @@
-hydroTable{}.csv
-gw_catchments_reaches_filtered_addedAttributes{}.tif
-rem_zeroed_masked{}.tif
-usgs_elev_table.csv
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 86c63f652..7b062036a 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,34 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.5.3.0 - 2024-07-24 - [PR#1217](https://github.com/NOAA-OWP/inundation-mapping/pull/1217)
+
+This PR rolls up a bunch of other PR's and python packages requests including:
+- Issue [1208](https://github.com/NOAA-OWP/inundation-mapping/issues/1208)  Bump OpenJDK from 17.0.8 to 17.0.10 (via updating to JDK 21.0.3)
+- PR [1207](https://github.com/NOAA-OWP/inundation-mapping/pull/1207) - Dependabot bump certifi from 2023.7.22 to 2024.7.4
+- PR [1192](https://github.com/NOAA-OWP/inundation-mapping/pull/1192) - Dependabot Bump urllib3 from 1.26.18 to 1.26.19
+- Updates required from ongoing PR [1206](https://github.com/NOAA-OWP/inundation-mapping/pull/1206) - Probabilistic Flood Inundation Mapping. These updates make it easier for that branch/task to continue forward and staying in sync with dev. This triggered a few other packages that needed to be updated.
+
+Other tasks included are:
+- Removing the now heavily obsolete `unit_test` system, including the package `pytest`. This included some changes to the `CONTRIBUTING.md` document.
+- Clean of a couple of packages no longer in use: `pluggy` and `iniconfig`
+- Removal of a deprecated file named `config/aws_s3_put_fim3_hydrovis_whitelist.lst`
+- Removed duration stamps around a few parts in `fim_post_processing.sh`
+- Fixes and updates to linting files. e.g. `pyproject.toml`. (line length was not working correctly)
+
+### Changes
+- `Dockerfile`, `Pipfile`, `Pipfile.lock`: as describe above for python package changes
+- `.gitignore`, `CONTRIBUTING.md`: File changes related to removing the `unit_test` system.
+- `fim_post_processing.sh`: noted above.
+- `pyproject.toml`: fixes and updates for linting
+
+### Removals
+- `unit_tests` folder and all related files under it. Appx 25 to 30 files removed.
+
+<br/><br/>
+
+
 ## v4.5.2.11 - 2024-07-19 - [PR#1222](https://github.com/NOAA-OWP/inundation-mapping/pull/1222)
 
 We are having problems with post processing overall duration taking a long time. This new system captures duration times for each module/section inside fim_post_processing.sh and records it to a file on the output directory. It records it as it progress and will also help us learn if fim_post_processing.sh stopped along the way.
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index 0846b522e..d89c554ce 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -122,6 +122,7 @@ fi
 # outputsDir & workDir come from the Dockerfile
 outputDestDir=$outputsDir/$runName
 tempRunDir=$workDir/$runName
+export WBT_PATH=${tempRunDir}/whitebox_temp
 
 # default values
 if [ "$envFile" = "" ]; then envFile=/$projectDir/config/params_template.env; fi
@@ -214,6 +215,7 @@ else
     rm -f $outputDestDir/*.env
 fi
 
+
 mkdir -p $outputDestDir/logs/unit
 mkdir -p $outputDestDir/logs/branch
 mkdir -p $outputDestDir/unit_errors
diff --git a/pyproject.toml b/pyproject.toml
index ac0432b03..ee3862fda 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,17 +15,13 @@ maintainers = [
     {name = "Fernando Salas", email = "fernando.salas@noaa.gov"},
     {name = "Carson Pruitt", email = "carson.pruitt@noaa.gov"},
     {name = "Ali Forghani", email = "ali.forghani@noaa.gov"},
-    {name = "Bradford Bates", email = "bradford.bates@noaa.gov"},
     {name = "Emily Deardorff", email = "emily.deardorff@noaa.gov"},
-    {name = "Fernando Aristizabal", email = "fernando.aristizabal@noaa.gov"},
     {name = "Greg Petrochenkov", email = "greg.petrochenkov@noaa.gov"},
     {name = "Hamideh Safa", email = "hamideh.safa@noaa.gov"},
     {name = "James Coll", email = "james.coll@noaa.gov"},
     {name = "Matt Luck", email = "matt.luck@noaa.gov"},
-    {name = "Laura Keys", email = "laura.keys@noaa.gov"},
     {name = "Nick Chadwick", email = "nick.chadwick@noaa.gov"},
     {name = "Riley McDermott", email = "riley.mcdermott@noaa.gov"},
-    {name = "Rob Gonzalez-Pita", email = "rob.g.pita@noaa.gov"},
     {name = "Robert Hanna", email = "robert.hanna@noaa.gov"},
     {name = "Ryan Spies", email = "ryan.spies@noaa.gov"}
 ]
@@ -51,14 +47,14 @@ skip-magic-trailing-comma = true
 [tool.isort]
 profile = 'black'
 multi_line_output = 3
-line_length = 110 # It should be the same as `tool.black.line-length` above
+line_length = 110
 lines_after_imports = 2
 
 [tool.flake8]
 count = true
 doctests = true
 max-complexity = 56
-max-line-length = 110  # It should be the same as `tool.black.line-length` above
+max-line-length = 110
 extend-ignore = """
     E203,
     E266,
diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py
old mode 100644
new mode 100755
index fa43da473..6e1fe687c
--- a/src/bathymetric_adjustment.py
+++ b/src/bathymetric_adjustment.py
@@ -11,7 +11,8 @@
 
 import geopandas as gpd
 import pandas as pd
-from synthesize_test_cases import progress_bar_handler
+
+from utils.shared_functions import progress_bar_handler
 
 
 def correct_rating_for_bathymetry(fim_dir, huc, bathy_file, verbose):
@@ -191,7 +192,7 @@ def multi_process_hucs(fim_dir, bathy_file, wbd_buffer, wbd, output_suffix, numb
             executor_dict[future] = huc
 
         # Send the executor to the progress bar and wait for all tasks to finish
-        progress_bar_handler(executor_dict, True, f"Running BARC on {len(hucs)} HUCs")
+        progress_bar_handler(executor_dict, f"Running BARC on {len(hucs)} HUCs")
         # Get the returned logs and write to the log file
         for future in executor_dict.keys():
             try:
diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py
index 34c0e1dad..68a13e921 100644
--- a/tools/overlapping_inundation.py
+++ b/tools/overlapping_inundation.py
@@ -141,10 +141,10 @@ def get_window_coords(self):
         # Get window sizes (both normal and edge windows)
         window_bounds1 = np.flip(
             np.array(np.meshgrid(window_width1, window_height1)).T.reshape(-1, 2), axis=1
-        ).astype(np.int)
+        ).astype(np.int64)
         window_bounds2 = np.flip(
             np.array(np.meshgrid(window_width2, window_height2)).T.reshape(-1, 2), axis=1
-        ).astype(np.int)
+        ).astype(np.int64)
 
         window_idx = np.array(np.unravel_index(np.arange(y_res * x_res), (y_res, x_res), order="F"))
 
@@ -224,7 +224,7 @@ def read_rst_data(self, win_idx, datasets, path_points, bbox, meta):
         window = path_points[win_idx]
         window_height, window_width = np.array(
             [np.abs(bbox[win_idx][2] - bbox[win_idx][0]), np.abs(bbox[win_idx][3] - bbox[win_idx][1])]
-        ).astype(np.int)
+        ).astype(np.int64)
 
         bnds = []
         data = []
diff --git a/unit_tests/README.md b/unit_tests/README.md
deleted file mode 100644
index 6612b9761..000000000
--- a/unit_tests/README.md
+++ /dev/null
@@ -1,155 +0,0 @@
-# Inundation Mapping: Flood Inundation Mapping for U.S. National Water Model
-
-Flood inundation mapping software configured to work with the U.S. National Water Model operated and maintained by the National Oceanic and Atmospheric Administration (NOAA) National Water Center (NWC).
-
-#### For more information, see the [Inundation Mapping Wiki](https://github.com/NOAA-OWP/inundation-mapping/wiki).
-
-# The `/unit_tests` folder contains files for unit testing python files
-
-## Creating unit tests
-
-For each python code file that is being tested, unit tests should come in two files: a unit test file (based on the original python code file) and an accompanying json paramerters file.
-
-The files should be named following FIM convention:
-
-{source py file name}_test.py ->     `derive_level_paths_test.py`
-{source py file name}_params.json -> `derive_level_paths_params.json`
-
-
-## Tips to create a new json file for a new python unit test file.
-
-There are multiple way to figure out a set of default json parameters for the new unit test file.
-
-One way is to use the incoming arg parser. Most python files include the code block of ` __name__ == '__main__':`, followed by external arg parsing (`args = vars(parser.parse_args()`).
-* Add a `print(args)` or similar, and get all the values including keys as output.
-* Copy that into an editor being used to create the json file.
-* Add a line break after every comma.
-* Find/replace all single quotes to double quotes then cleanup the left tab formatting.
-
-
-## Setting up unit test data
-
-You can either pull the unit test data from s3, or generate it:
-
-1.) To pull the unit test data from ESIP's S3 bucket.
-```bash
-aws s3 cp --recursive \
-	s3://noaa-nws-owp-fim/hand_fim/unit_test_data/ \
-	<data/outputs/unit_test_data> --no-sign-request
-```
-The directory which the data is copied into must be named: `unit_test_data`
-
-2.) To generate the required unit test data, following these steps:
-
-* Start a docker container as you normally would for any development.
-```bash
-docker run --rm -it --name <a docker container name> \
-	-v /home/<your name>/projects/<folder path>/:/foss_fim \
-	{your docker image name}
-```
-Example:
-```bash
-docker run --rm -it --name mytest \
-	-v /home/abcd/projects/dev/innudation-mapping/:/foss_fim \
-	-v /abcd_share/foss_fim/outputs/:/outputs \
-	-v /abcd_share/foss_fim/:/data \
-	-v /abcd_share/foss_fim/outputs_temp/:/fim_temp \
-	fim_4:dev_20220208_8eba0ee
-```
-* Call the `fim_pipeline.sh` script with the necessary arguments (`-n` must be `unit_test_data`).
-```bash
-fim_pipeline.sh -n unit_test_data -u "02020005 05030104" \
-	-bd /foss_fim/config/deny_branch_unittests.lst \
-	-ud None -j 1 -o
-```
-
-__NOTICE:__ the deny file used for fim_pipeline.sh, has a special one for unittests `deny_branch_unittests.lst`.
-
-* If you need to run inundation tests, fun the following:
-
-```bash
-python3 foss_fim/tools/synthesize_test_cases.py \
-	-c DEV \
-	-v unit_test_data \
-	-jh 1 -jb 1 \
-	-m /data/outputs/unit_test_data/alpha_test_metrics.csv \
-	-o
-```
-
-## Running unit tests
-
-### If you'd like to test the whole unit test suite:
-When inside of the container, ensure you are within the root directory of the repository before running the `pytest` command.
-```
-cd /foss_fim
-pytest /foss_fim/unit_tests
-```
-
-__NOTE:__ This is subject to error, as the downloaded/generated data could potentially have a different path than what is specified in the `/unit_tests/*_params.json` files. The data files are not included in this repository, so are subject to change.
-
-### If you want to test just one unit test (from the root terminal window):
-
-```bash
-cd /foss_fim
-pytest unit_tests/gms/derive_level_paths_test.py
-						or
-pytest unit_tests/inundate_gms_test.py
-```
-
-### If you'd like to run a particular test, you can, for example:
-```
-pytest -v -s -k test_append_id_to_file_name_single_identifier_success
-```
-
-If one test case is choosen, it will scan all of the test files, and scan for the method (test case) specified.
-
-## Key Notes for creating new unit tests
-1) All test functions must start with the phrase `test_`. That is how pytest picks them up. The rest of the function name does not have to match the pattern of `function_name_being_tested` but should. Further, the rest of the function name should say what the test is about, ie) `_failed_input_path`.  ie) `test_{some_function_name_from_the_source_code_file}_failed_input_path`. It is fine that the function names get very long (common in the industry).
-
-2) If you are using this for development purposes, use caution when checking back in files for unit tests files and json file. If you check it in, it still has to work and work for others and not just for a dev test you are doing.
-
-3) As of now, you can not control the order that unit tests are run within a unit test file.
-
-4) There must be at least one associated `{original py file name}_params.json` file per unit test.
-
-5) There must be at least one "happy path (successful)" test inside the unittest file. ie) one function that is expected to fully pass. You can have multiple "happy path" tests if you want to change values that are fundamentally different, but fully expected to pass.
-
-6) Json files can have multiple nodes, so the default "happy path/success" is suggested to be called `valid_data`, if one does not already exist. Generally, the individual unit tests, will call the `valid_data` node and override a local method value to a invalid data. In semi-rare, but possible cases, you can add more nodes if you like, but try not to create new Json nodes for a few small field changes, generally only use a new node if there are major and lots of value changes (ie: major different test conditions).
-
-7) Unit test functions can and should test for all "outputs" from a source function. This includes the functions's return output (if any), but any global variables it might set, and even that saved output files (such as .tif files) have been created and successfully. It is ok to have multiple validation checks (or asserts) in one unit test function.
-
-8) One Python file = one `{original py file name}_test.py` file.
-
-9) Sometimes you may want to run a full successful "happy path" version through `fim_pipeline.sh` (or similar), to get all of the files you need in place to do your testing. However, you will want to ensure that none of the outputs are being deleted during the test. One way to solve this is to put in an invalid value for the `-d` parameter (denylist).
-ie:
-	```bash
-	fim_pipeline.sh -n unit_test_data -u 05030104 \
-		-c /foss_fim/config/params_template.env -j 1 -d /foss_fim/config/deny_unit_default.lst -o
-	```
-
-	 but ours would be:
-
-	```bash
-	fim_pipeline.sh -n unit_test_data -u 05030104 \
-		-c /foss_fim/config/params_template.env -j 1 -d no_list -o
-	```
-
-## [Pytest](https://docs.pytest.org/en/7.2.x/) particulars
-
-The `unit_tests/pyproject.toml` file is used to specify which warnings are disabled for our particular unit tests.
-
-Luckily, `pytest` works well with The Python Standard Library `unittest`. This made the migration of previous unit tests using `unittest` over to `pytest` quite simple. The caveat is that our current unit tests employ elements of both libraries. A full transition to `pytest` will ideally take place at a future date.
-
-## Testing for failing conditions
-- Over time, you want to start adding functions that specifically look for fail conditions. This is a key part of unit test systems. It is not uncommon to have many dozens of tests functions in one unit test file. Each "fail" type test, must check for ONLY one variable value change. A "fail" test function should not fundamentally pass in an invalid huc AND an invalid file path.  Those two failing test conditions and must have two seperate unit test functions.
-
-- It is possible to let a unit test have more than one failed value but only if they are tightly related to trigger just one failure (RARE though). YES.. Over time, we will see TONS of these types of fail unit test functions and they will take a while to run.
-
-- When you create a "fail" test function, you can load up the normal full "params" from the json file, but then you can override (hardcoded) the one (or rarely more than one) variable inside the function. There is a way to "catch" a failure you are expecting, ensure it is the type of failure you expected and make that "failure" to become a true fail, ie) a unit test pass.
-
-An example is in `unit_tests/Derive_level_paths_test.py` -> `test_Derive_level_paths_invalid_input_stream_network` (function). This example gives you the pattern implemented in Pytest.
-
-## Future Enhancements
-1) Full transition to the `pytest` library, removing classes of `unittest.TestCase` and taking full advantage of available code re-use patterns offered through `pytest`.
-
-2) Over time, it is expected that python files will be broken down to many functions inside the file. Currently, we tend to have one very large function in each python file which makes unit testing harder and less specific. Generally, one function will correlated to at least one "happy path" unit test function. Note: The files `derive_level_paths_test.py` and `clip_vectors_to_wbd_test.py` are not complete as they do not yet test all output from a method.
diff --git a/unit_tests/__template.json b/unit_tests/__template.json
deleted file mode 100644
index c7c190673..000000000
--- a/unit_tests/__template.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-    "valid_data": {
-        "some_param_name__suggested_to_match_the_method_param_name": "some value",
-        "another_one_and_as_many_lines_as_you_need": "some value",
-        "a_number": 20,
-        "a_list": [
-            "value 1",
-            "value 2"
-        ],
-        "inline_json_comments_dont_work": "sorry"
-    }
-}
diff --git a/unit_tests/__template.py b/unit_tests/__template.py
deleted file mode 100644
index 946e06474..000000000
--- a/unit_tests/__template.py
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import unittest
-
-import pytest
-import Your_original_source_python_file_name as src
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-
-class test_Your_original_source_python_file_name(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    # MUST start with the name of "test_"
-    # This is the (or one of the) valid test expected to pass
-    def test_method_name_you_will_test(self):
-        """
-        < UPDATE THESE NOTES: to say what you are testing and what you are expecting.
-          If there is no return value from the method, please say so.>
-
-        Dev Notes: (which can be removed after you make this file)
-            Remember... You need to validate the method output if there is any. However, if you have time, it
-            is also recommended that you validate other outputs such as writing or updating file on the file
-            system, aka: Does the expected file exist. Don't worry about its contents.
-        """
-
-        # global params_file
-        params = self.params[
-            # update "valid_data" value if you need to (aka.. more than one node)
-            "valid_data"
-        ].copy()
-
-        # for now we are happy if no exceptions are thrown.
-
-        # See the readme.md, clip_vectors_to_wbd_test.py or gms/derive_level_paths_test.py for examples.
-        # Replace this stub example with your own.
-        # Try to use the same order to make it easier.
-        # Remember, if the method accepts **params, then you can send that in here as well.
-        # ie: my_py_class.my_method(** params)
-
-        src.subset_vector_layers(
-            hucCode=params["hucCode"],
-            nwm_streams_filename=params["nwm_streams"],
-            etc=params["a_number"],
-            etc2=params["a_list"],
-        )
-
-        # The assert statement is what we are acutally testing, one that evaluates as True passes,
-        # and one that evaluates to False fails.
-        # A message (string) can be added after the assert statement to provide detail on
-        # the case being tested, and why it failed.
-
-        assert os.path.exists(params["nwm_streams"]) is True, "The nwm_streams file does not exist"
-
-    # EXAMPLE OF A SUCCESSFUL TEST CASE WHICH CAPTURES AN EXCEPTION (FAILURE)
-    def test_subset_vector_layers_fail_invalid_stream_path(self):
-        """
-        Notes about what the test is and the expected results (or expected exception if applicable)
-        """
-
-        params = self.params[
-            # update "valid_data" value if you need to (aka.. more than one node)
-            "valid_data"
-        ].copy()
-
-        params["nwm_streams"] = "/some/bad/path/"
-
-        with pytest.raises(Exception):
-            src.subset_vector_layers(
-                hucCode=params["hucCode"],
-                nwm_streams_filename=params["nwm_streams"],
-                etc=params["a_number"],
-                etc2=params["a_list"],
-            )
diff --git a/unit_tests/aggregate_branch_lists_params.json b/unit_tests/aggregate_branch_lists_params.json
deleted file mode 100644
index d1090612c..000000000
--- a/unit_tests/aggregate_branch_lists_params.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-    "valid_data": {
-        "output_dir": "/data/outputs/unit_test_data/",
-        "file_name": "branch_ids.csv",
-        "output_file_name": "/data/outputs/unit_test_data/test.csv"
-    }
-}
diff --git a/unit_tests/aggregate_branch_lists_test.py b/unit_tests/aggregate_branch_lists_test.py
deleted file mode 100644
index 93c4f29d8..000000000
--- a/unit_tests/aggregate_branch_lists_test.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import aggregate_branch_lists as src
-
-
-class test_aggregate_branch_lists(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    def test_aggregate_branch_lists_success(self):
-        # global params_file
-
-        params = self.params["valid_data"].copy()
-
-        src.aggregate_branch_lists(
-            output_dir=params["output_dir"],
-            file_name=params["file_name"],
-            output_file_name=params["output_file_name"],
-        )
diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json
deleted file mode 100644
index 33a144fe6..000000000
--- a/unit_tests/check_unit_errors_params.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "valid_data": {
-        "fim_dir": "/data/outputs/unit_test_data"
-    },
-    "invalid_path": {
-        "fim_dir": "/data/outputs/check_errors_example_unit_tests_not_not_valid"
-    }
-}
diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py
deleted file mode 100644
index 2eba1e9d0..000000000
--- a/unit_tests/check_unit_errors_test.py
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import math
-import os
-import shutil
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import check_unit_errors as src
-from utils.shared_variables import UNIT_ERRORS_MIN_NUMBER_THRESHOLD, UNIT_ERRORS_MIN_PERCENT_THRESHOLD
-
-
-class test_check_unit_errors(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        # get_params_filename function in ./unit_test_utils handles errors
-        params_file_path = ut_helpers.get_params_filename(__file__)
-
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    """
-    To make most of this unit test, we have to do the following:
-    - rename the current unit_errors directory (if it exists)
-    - empty the unit_errors directory
-    - create a bunch of dummy files in it
-    - perform the unit test
-    - delete the unit_errors directory
-    - rename the original unit_test folder back to unit_tests
-    """
-
-    # Test Cases:
-
-    def test_check_unit_errors_success_below_min_errors(self):
-        # Expecting no errors.
-        # Test to ensure the number of dummy files is less than the overall min number of error files.
-
-        params = self.params["valid_data"].copy()
-
-        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD - 1
-
-        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-
-        expected_output = 0
-        actual_output = src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-
-        err_msg = "Number of dummy files IS NOT less than the overall min number of error files."
-        assert expected_output == actual_output, err_msg
-
-        self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-    def test_check_unit_errors_fail_above_min_errors(self):
-        # Test to ensure the number of dummy files is more than the overall min number of error files.
-        # Expecting sys.exit of 62
-        # We do expect this to fail and if it fails, it is successful.
-        # Here we expect an exception, and are capturing it using pytest.raises(Exception)
-
-        params = self.params["valid_data"].copy()
-
-        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD + 1
-
-        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-
-        with pytest.raises(Exception):
-            src.check_unit_errors(params["fim_dir"], num_dummy_files_reqd)
-
-        # We have to put the unit_errors folders back to the way it was
-        self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-    def test_check_unit_errors_success_above_percent_errors(self):
-        # Expecting no errors.
-        # Test to ensure the number of dummy files is more than the overall min number of error files.
-        # We do expect this not to to fail as it is greater than 10 errors but below the percent threshhold.
-
-        params = self.params["valid_data"].copy()
-
-        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-
-        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-
-        num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) + 1
-        expected_output = 0
-        actual_output = src.check_unit_errors(params["fim_dir"], num_total_units)
-
-        err_msg = "Number of dummy files IS NOT more than the overall min number of error files"
-        assert expected_output == actual_output, err_msg
-
-        # We have to put the unit_errors folders back to the way is was.
-        self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-    def test_check_unit_errors_fail_below_percent_errors(self):
-        # Expecting sys.exit of 62
-        # Test to ensure the number of dummy files is more than the overall min number of error files.
-
-        # We do expect this to fail as it is greater than 10 errors
-        # AND below the percent threshhold (more percent errors than the threshold)
-        # Here we expect an exception, and are capturing it using pytest.raises(Exception)
-
-        params = self.params["valid_data"].copy()
-
-        num_dummy_files_reqd = UNIT_ERRORS_MIN_NUMBER_THRESHOLD * 2
-
-        self.__create_temp_unit_errors_folder_files(params["fim_dir"], num_dummy_files_reqd)
-
-        num_total_units = math.trunc(num_dummy_files_reqd * (100 / UNIT_ERRORS_MIN_PERCENT_THRESHOLD)) - 10
-
-        with pytest.raises(Exception):
-            src.check_unit_errors(params["fim_dir"], num_total_units)
-
-        # We have to put the unit_errors folders back to the way it was.
-        self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-    # Helper functions:
-
-    def __create_temp_unit_errors_folder_files(self, output_folder, number_of_files):
-        """
-        Process:
-            We want to preserve the original unit_errors folder if it exists,
-            so we wil rename it.
-            Then we will make a new unit_errors folder and fill it with a bunch of
-            dummy files.
-            A dummy file for non_zero_exit_codes.log will also be created.
-        Input:
-            output_folder: the root output folder (ie. /outputs/gms_example_unit_tests/)
-            number_of_files: how many dummy files to create
-        Returns:
-            True if the 'unit_errors' folder did original exist and needs to be renamed back.
-            False if the 'unit_errors' folder never existed in the first place.
-        """
-
-        ue_folder_preexists = False
-
-        if not os.path.isdir(output_folder):
-            raise Exception(f"unit test root folder of {output_folder} does not exist")
-
-        ue_folder = os.path.join(output_folder, "unit_errors")
-        temp_ue_folder = ue_folder + "_temp"
-        if os.path.isdir(ue_folder):
-            ue_folder_preexists = True
-            os.rename(ue_folder, temp_ue_folder)
-
-        os.mkdir(ue_folder)
-
-        for i in range(0, number_of_files):
-            file_name = "sample_" + str(i) + ".txt"
-            file_path = os.path.join(ue_folder, file_name)
-            with open(file_path, "w") as fp:
-                fp.close()
-                pass
-
-        return ue_folder_preexists
-
-    def __remove_temp_unit_errors_folder(self, output_folder):
-        """
-        Process:
-            We want to preserve the original unit_errors folder if it exists,
-            so we will delete our temp unit test version of 'unit_errors', and rename
-            the original back to 'unit_errors'
-            Note.. it is possible the temp folder does not exist,
-            but we don't need to error out on it. Sometimes we got here by a try/catch cleanup
-        Input:
-            output_folder: the root output folder (ie. /data/outputs/unit_test_data/)
-        Returns:
-            nothing
-        """
-
-        ue_folder = os.path.join(output_folder, "unit_errors")
-        if os.path.isdir(ue_folder):
-            shutil.rmtree(ue_folder)
-
-        temp_ue_folder = ue_folder + "_temp"
-        if os.path.isdir(temp_ue_folder):
-            os.rename(temp_ue_folder, ue_folder)
-
-    # Test Cases for Helper funcitons:
-
-    def test_create_temp_unit_errors_folder_files(self):
-        # Here we are testing our helper function to see if it raise exceptions appropriately with a bad path.
-        # In this case, we want the exception to be raised if there is an invalid path.
-
-        params = self.params["invalid_path"]
-        invalid_folder = params["fim_dir"]
-
-        with pytest.raises(Exception):
-            self.__create_temp_unit_errors_folder_files(invalid_folder, 4)
-
-    def test_remove_temp_unit_errors_folder(self):
-        # Test of out helper function to see if the temp folder was removed.
-
-        params = self.params["valid_data"].copy()
-
-        self.__create_temp_unit_errors_folder_files(params["fim_dir"], 2)
-
-        self.__remove_temp_unit_errors_folder(params["fim_dir"])
-
-        temp_folder_created = os.path.join(params["fim_dir"], "unit_errors") + "_temp"
-
-        assert os.path.exists(temp_folder_created) is False
diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json
deleted file mode 100644
index a6e5ea0d6..000000000
--- a/unit_tests/clip_vectors_to_wbd_params.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-    "valid_data": {
-        "subset_nwm_lakes": "/data/outputs/unit_test_data/05030104/nwm_lakes_proj_subset.gpkg",
-        "subset_nwm_streams": "/data/outputs/unit_test_data/05030104/nwm_subset_streams.gpkg",
-        "hucCode": "05030104",
-        "subset_nwm_headwaters": "/data/outputs/unit_test_data/05030104/nhd_headwater_points_subset.gpkg",
-        "wbd_buffer_filename": "/data/outputs/unit_test_data/05030104/wbd_buffered.gpkg",
-        "wbd_streams_buffer_filename": "/data/outputs/unit_test_data/05030104/wbd_buffered_streams.gpkg",
-        "wbd_filename": "/data/outputs/unit_test_data/05030104/wbd.gpkg",
-        "dem_filename": "/data/inputs/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt",
-        "dem_domain": "/data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg",
-        "nwm_lakes": "/data/inputs/nwm_hydrofabric/nwm_lakes.gpkg",
-        "nwm_catchments": "/data/inputs/nwm_hydrofabric/nwm_catchments.gpkg",
-        "subset_nwm_catchments": "/data/outputs/unit_test_data/05030104/nwm_catchments_proj_subset.gpkg",
-        "nld_lines": "/data/inputs/nld_vectors/huc2_levee_lines/nld_preprocessed_05.gpkg",
-        "nld_lines_preprocessed": "/data/inputs/nld_vectors/3d_nld_preprocessed_230314.gpkg",
-        "landsea": "/data/inputs/landsea/water_polygons_us.gpkg",
-        "nwm_streams": "/data/inputs/nwm_hydrofabric/nwm_flows.gpkg",
-        "subset_landsea": "/data/outputs/unit_test_data/05030104/LandSea_subset.gpkg",
-        "nwm_headwaters": "/data/inputs/nwm_hydrofabric/nwm_headwaters.gpkg",
-        "subset_nld_lines": "/data/outputs/unit_test_data/05030104/nld_subset_levees.gpkg",
-        "subset_nld_lines_preprocessed": "/data/outputs/unit_test_data/05030104/3d_nld_subset_levees_burned.gpkg",
-        "wbd_buffer_distance": 5000,
-        "levee_protected_areas": "/data/inputs/nld_vectors/Levee_protected_areas.gpkg",
-        "subset_levee_protected_areas": "/data/outputs/unit_test_data/05030104/LeveeProtectedAreas_subset.gpkg"
-    }
-}
diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py
deleted file mode 100644
index b9567a3d3..000000000
--- a/unit_tests/clip_vectors_to_wbd_test.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import data.wbd.clip_vectors_to_wbd as src
-
-
-class test_clip_vectors_to_wbd(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    # TODO New Test Case to check that files were being created on the file system
-    @pytest.mark.skip(
-        reason="pre clipped vector files are now generated using generate_pre_clip_fim_huc8.py and "
-        "sent to the /inputs directory. Testing the clip_vectors_to_wbd.py is no longer necessary, "
-        "as this script is not run during fim_pipeline.sh execution."
-    )
-    def test_subset_vector_layers_success(self):
-        """
-        This NEEDS be upgraded to check the output, as well as the fact that all of the output files exist.
-        Most of the output test and internal tests with this function will test a wide variety of conditions.
-        Only the basic return output value should be tested to ensure it is as expected.
-        For now, we are adding the very basic "happy path" test.
-        """
-
-        params = self.params["valid_data"].copy()
-
-        # There is no default return value.
-        # For now we are happy if no exceptions are thrown.
-        try:
-            src.subset_vector_layers(
-                subset_nwm_lakes=params["subset_nwm_lakes"],
-                subset_nwm_streams=params["subset_nwm_streams"],
-                hucCode=params["hucCode"],
-                subset_nwm_headwaters=params["subset_nwm_headwaters"],
-                wbd_buffer_filename=params["wbd_buffer_filename"],
-                wbd_streams_buffer_filename=params["wbd_streams_buffer_filename"],
-                wbd_filename=params["wbd_filename"],
-                dem_filename=params["dem_filename"],
-                dem_domain=params["dem_domain"],
-                nwm_lakes=params["nwm_lakes"],
-                nwm_catchments=params["nwm_catchments"],
-                subset_nwm_catchments=params["subset_nwm_catchments"],
-                nld_lines=params["nld_lines"],
-                nld_lines_preprocessed=params["nld_lines_preprocessed"],
-                landsea=params["landsea"],
-                nwm_streams=params["nwm_streams"],
-                subset_landsea=params["subset_landsea"],
-                nwm_headwaters=params["nwm_headwaters"],
-                subset_nld_lines=params["subset_nld_lines"],
-                subset_nld_lines_preprocessed=params["subset_nld_lines_preprocessed"],
-                wbd_buffer_distance=params["wbd_buffer_distance"],
-                levee_protected_areas=params["levee_protected_areas"],
-                subset_levee_protected_areas=params["subset_levee_protected_areas"],
-            )
-
-        except (RuntimeError, TypeError, NameError) as e_info:
-            pytest.fail("Error in subset_vector_layers function", e_info)
diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json
deleted file mode 100644
index df64eee5b..000000000
--- a/unit_tests/derive_level_paths_params.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-	"valid_data": {
-		"outputDestDir": "/data/outputs/unit_test_data/",
-		"in_stream_network": "/data/outputs/unit_test_data/05030104/nwm_subset_streams.gpkg",
-		"wbd": "/data/outputs/unit_test_data/05030104/wbd.gpkg",
-		"buffer_wbd_streams": "/data/outputs/unit_test_data/05030104/wbd_buffered_streams.gpkg",
-		"out_stream_network": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"branch_id_attribute": "levpa_id",
-		"out_stream_network_dissolved": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths.gpkg",
-		"headwaters_outfile": "/data/outputs/unit_test_data/05030104/nwm_headwaters.gpkg",
-		"catchments": "/data/outputs/unit_test_data/05030104/nwm_catchments_proj_subset.gpkg",
-		"catchments_outfile": "/data/outputs/unit_test_data/05030104/nwm_catchments_proj_subset_levelPaths.gpkg",
-		"branch_inlets_outfile": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg",
-		"reach_id_attribute": "ID",
-		"verbose": true
-	}
-}
diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py
deleted file mode 100644
index 6ee67973d..000000000
--- a/unit_tests/derive_level_paths_test.py
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import json
-import os
-import sys
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-from derive_level_paths import Derive_level_paths
-from stream_branches import StreamNetwork
-
-
-class test_Derive_level_paths(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    def test_Derive_level_paths_success_all_params(self):
-        """
-        This test includes all params with many optional parms being set to the default value of the function
-        Notes:
-            Other params such as toNode_attribute and fromNode_attribute are defaulted and not passed into
-            __main__ , so we skip them here.
-            Returns GeoDataframe (the nwm_subset_streams_levelPaths_dissolved.gpkg)
-        """
-
-        params = self.params["valid_data"].copy()
-
-        actual_df = Derive_level_paths(
-            in_stream_network=params["in_stream_network"],
-            buffer_wbd_streams=params["buffer_wbd_streams"],
-            wbd=params["wbd"],
-            out_stream_network=params["out_stream_network"],
-            branch_id_attribute=params["branch_id_attribute"],
-            out_stream_network_dissolved=params["out_stream_network_dissolved"],
-            headwaters_outfile=params["headwaters_outfile"],
-            catchments=params["catchments"],
-            catchments_outfile=params["catchments_outfile"],
-            branch_inlets_outfile=params["branch_inlets_outfile"],
-            reach_id_attribute=params["reach_id_attribute"],
-            verbose=params["verbose"],
-        )
-
-        # test data type being return is as expected. Downstream code might to know the type.
-        assert isinstance(actual_df, StreamNetwork)
-
-        # **** NOTE: Based on 05030104
-        # Test row count for dissolved level path GeoDataframe which is returned.
-        actual_row_count = len(actual_df)
-        expected_row_count = 4
-        assert actual_row_count == expected_row_count
-
-        # Test that output files exist as expected
-        assert (
-            os.path.exists(params["out_stream_network"]) is True
-        ), f"Expected file {params['out_stream_network']} but it does not exist."
-        assert (
-            os.path.exists(params["out_stream_network_dissolved"]) is True
-        ), f"Expected file {params['out_stream_network_dissolved']} but it does not exist."
-        assert (
-            os.path.exists(params["headwaters_outfile"]) is True
-        ), f"Expected file {params['headwaters_outfile']} but it does not exist."
-        assert (
-            os.path.exists(params["catchments_outfile"]) is True
-        ), f"Expected file {params['catchments_outfile']} but it does not exist."
-        assert (
-            os.path.exists(params["catchments_outfile"]) is True
-        ), f"Expected file {params['catchments_outfile']} but it does not exist."
-        assert (
-            os.path.exists(params["branch_inlets_outfile"]) is True
-        ), f"Expected file {params['branch_inlets_outfile']} but it does not exist."
-
-    # Invalid Input stream for demo purposes.
-    def test_Derive_level_paths_invalid_input_stream_network(self):
-        # NOTE: As we are expecting an exception, we use pytest.raises(Exception).
-
-        params = self.params["valid_data"].copy()
-        params["in_stream_network"] = "some bad path"
-
-        with pytest.raises(Exception):
-            Derive_level_paths(
-                in_stream_network=ut_helpers.json_concat(params, "outputDestDir", "in_stream_network"),
-                out_stream_network=params["out_stream_network"],
-                branch_id_attribute=params["branch_id_attribute"],
-                out_stream_network_dissolved=params["out_stream_network_dissolved"],
-                huc_id=params["huc_id"],
-                headwaters_outfile=params["headwaters_outfile"],
-                catchments=params["catchments"],
-                catchments_outfile=params["catchments_outfile"],
-                branch_inlets_outfile=params["branch_inlets_outfile"],
-                reach_id_attribute=params["reach_id_attribute"],
-                verbose=params["verbose"],
-            )
diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json
deleted file mode 100644
index a68a46fd7..000000000
--- a/unit_tests/filter_catchments_and_add_attributes_params.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-    "valid_data": {
-        "outputDestDir": "/outputs/unit_test_data",
-        "input_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/gw_catchments_reaches_2274000033.gpkg",
-        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/demDerived_reaches_split_2274000033.gpkg",
-        "output_catchments_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/gw_catchments_reaches_filtered_addedAttributes_2274000033.gpkg",
-        "output_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000033/demDerived_reaches_split_filtered_2274000033.gpkg",
-        "wbd_filename": "/data/outputs/unit_test_data/02020005/wbd8_clp.gpkg",
-        "huc_code": "02020005"
-    }
-}
diff --git a/unit_tests/filter_catchments_and_add_attributes_test.py b/unit_tests/filter_catchments_and_add_attributes_test.py
deleted file mode 100644
index 098d6935f..000000000
--- a/unit_tests/filter_catchments_and_add_attributes_test.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-
-import inspect
-import json
-import os
-import sys
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import filter_catchments_and_add_attributes as src
-
-
-# *************
-# Important: For this to work, when you run gms_run_branch.sh, you have to
-# use deny_gms_branches_dev.lst or the word "none" for the deny list arguments
-# (unit and branch deny list parameters). Key files need to exist for this unit test to work.
-class test_filter_catchments_and_add_attributes(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    def test_filter_catchments_and_add_attributes_success(self):
-        """
-        The gw_catchments_reaches_filtered_addedAttributes_<branchID>.gpkg and
-        demDerived_reaches_split_filtered_<branchid>.gpkg should not exit prior to this test.
-        If the test is successful, these file will be created.
-        """
-
-        params = self.params["valid_data"].copy()
-
-        # To setup the test, lets start by deleted the two expected output files to ensure
-        # that they are regenerated.
-        if os.path.exists(params["output_flows_filename"]):
-            os.remove(params["output_flows_filename"])
-        if os.path.exists(params["output_catchments_filename"]):
-            os.remove(params["output_catchments_filename"])
-
-        # Test that the files were deleted
-        assert os.path.exists(params["output_flows_filename"]) is False
-
-        assert os.path.exists(params["output_catchments_filename"]) is False
-
-        src.filter_catchments_and_add_attributes(
-            input_catchments_filename=params["input_catchments_filename"],
-            input_flows_filename=params["input_flows_filename"],
-            output_catchments_filename=params["output_catchments_filename"],
-            output_flows_filename=params["output_flows_filename"],
-            wbd_filename=params["wbd_filename"],
-            huc_code=params["huc_code"],
-        )
-
-        # Test that the files were created by filer_catchments_and_add_attributes
-        assert os.path.exists(params["output_flows_filename"]) is True
-
-        assert os.path.exists(params["output_catchments_filename"]) is True
diff --git a/unit_tests/generate_branch_list_csv_params.json b/unit_tests/generate_branch_list_csv_params.json
deleted file mode 100644
index 0e2f2574a..000000000
--- a/unit_tests/generate_branch_list_csv_params.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-    "valid_data_add_branch_zero": {
-        "huc_id": "05030104",
-        "branch_id": "0",
-        "output_branch_csv": "/data/outputs/unit_test_data/05030104/branch_ids.csv"
-    },
-    "valid_data_add_branch": {
-        "huc_id": "05030104",
-        "branch_id": "1946000003",
-        "output_branch_csv": "/data/outputs/unit_test_data/05030104/branch_ids.csv"
-    },
-    "invalid_bad_file_extension": {
-        "huc_id": "05030104",
-        "branch_id": "1946000003",
-        "output_branch_csv": "/data/outputs/unit_test_data/05030104/branch_ids2"
-    }
-}
diff --git a/unit_tests/generate_branch_list_csv_test.py b/unit_tests/generate_branch_list_csv_test.py
deleted file mode 100644
index 5495767b6..000000000
--- a/unit_tests/generate_branch_list_csv_test.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import generate_branch_list_csv as src
-
-
-class test_generate_branch_list_csv(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-        # for these tests to work, we have to check if the .csv exists and remove it
-        # prior to exections of the tests.
-
-        params = self.params["valid_data_add_branch_zero"].copy()
-        if os.path.exists(params["output_branch_csv"]):
-            os.remove(params["output_branch_csv"])
-
-    # Test Cases:
-
-    def test_generate_branch_list_csv_valid_data_add_branch_zero_success(self):
-        # yes.. we know that we can not control the order
-
-        # global params_file
-        params = self.params["valid_data_add_branch_zero"].copy()
-
-        src.generate_branch_list_csv(
-            huc_id=params["huc_id"],
-            branch_id=params["branch_id"],
-            output_branch_csv=params["output_branch_csv"],
-        )
-
-    def test_generate_branch_list_csv_valid_data_add_branch_success(self):
-        # global params_file
-        params = self.params["valid_data_add_branch"].copy()
-
-        src.generate_branch_list_csv(
-            huc_id=params["huc_id"],
-            branch_id=params["branch_id"],
-            output_branch_csv=params["output_branch_csv"],
-        )
-
-    def test_generate_branch_list_csv_invalid_bad_file_extension(self):
-        # global params_file
-        params = self.params["invalid_bad_file_extension"].copy()
-
-        # We expect this to fail. If it does fail with an exception, then this test is sucessful.
-        with pytest.raises(Exception):
-            src.generate_branch_list_csv(
-                huc_id=params["huc_id"],
-                branch_id=params["branch_id"],
-                output_branch_csv=params["output_branch_csv"],
-            )
diff --git a/unit_tests/generate_branch_list_params.json b/unit_tests/generate_branch_list_params.json
deleted file mode 100644
index 88962a708..000000000
--- a/unit_tests/generate_branch_list_params.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-    "valid_data": {
-        "stream_network_dissolved": "/data/outputs/unit_test_data/05030104/nwm_subset_streams_levelPaths_dissolved.gpkg",
-        "branch_id_attribute": "levpa_id",
-        "output_branch_list_file": "/data/outputs/unit_test_data/05030104/branch_ids.lst"
-    }
-}
diff --git a/unit_tests/generate_branch_list_test.py b/unit_tests/generate_branch_list_test.py
deleted file mode 100644
index 3a35f35ea..000000000
--- a/unit_tests/generate_branch_list_test.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import generate_branch_list as src
-
-
-class test_Generate_branch_list(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    def test_Generate_branch_list_success(self):
-        params = self.params["valid_data"].copy()
-
-        src.generate_branch_list(
-            stream_network_dissolved=params["stream_network_dissolved"],
-            branch_id_attribute=params["branch_id_attribute"],
-            output_branch_list_file=params["output_branch_list_file"],
-        )
diff --git a/unit_tests/inundate_gms_params.json b/unit_tests/inundate_gms_params.json
deleted file mode 100644
index 72af7da14..000000000
--- a/unit_tests/inundate_gms_params.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-	"valid_data_inudation_raster_single_huc": {
-		"hydrofabric_dir": "/data/outputs/unit_test_data/",
-		"forecast": "/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_2_0_cms.csv",
-		"num_workers": 4,
-		"hucs": "02020005",
-		"inundation_raster": "/data/outputs/unit_test_data/inundation_extent_02020005.tif",
-		"inundation_polygon": null,
-		"depths_raster": null,
-		"verbose": true,
-		"log_file": "/data/outputs/unit_test_data/logs/inundation_logfile.txt",
-		"output_fileNames": "/data/outputs/unit_test_data/logs/inundation_file_list.csv"
-	}
-}
diff --git a/unit_tests/inundate_gms_test.py b/unit_tests/inundate_gms_test.py
deleted file mode 100644
index 034830aa2..000000000
--- a/unit_tests/inundate_gms_test.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import sys
-import unittest
-
-import inundate_gms as src
-import pandas as pd
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-
-class test_inundate_gms(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-    def test_Inundate_gms_create_inundation_raster_directory_single_huc_success(self):
-        """
-        Test for creating a gms inundation rasters, not a depth raster and no
-        inundation_polygons.
-
-        This test is essentially testing the Inundate_gms function, and the creation of a
-        the "output_fileNames" .csv file.
-        """
-
-        params = self.params["valid_data_inudation_raster_single_huc"].copy()
-
-        # Clear previous outputs, if they exist.
-        for file in os.listdir(params["hydrofabric_dir"]):
-            if file.endswith(".tif"):
-                os.remove(os.path.join(params["hydrofabric_dir"], file))
-
-        if os.path.isfile(params["output_fileNames"]):
-            os.remove(params["output_fileNames"])
-
-        # Test the Inundate_gms function
-        output_fileNames_df = src.Inundate_gms(
-            hydrofabric_dir=params["hydrofabric_dir"],
-            forecast=params["forecast"],
-            num_workers=params["num_workers"],
-            hucs=params["hucs"],
-            inundation_raster=params["inundation_raster"],
-            inundation_polygon=params["inundation_polygon"],
-            depths_raster=params["depths_raster"],
-            verbose=params["verbose"],
-            log_file=None,
-            output_fileNames=params["output_fileNames"],
-        )
-
-        # Check if output files df has records.
-        assert len(output_fileNames_df) > 0, "Expected as least one dataframe record"
-
-    def test_Inundate_gms_create_inundation_rasters(self):
-        """
-        Test for creating a gms inundation rasters, not a depth raster and no
-        inundation_polygons.
-        This test is based on creating a raster based on a single huc and its branches
-        within the output folder.
-        """
-
-        params = self.params["valid_data_inudation_raster_single_huc"].copy()
-
-        # Check all output rasters exist.
-        csv_out = pd.read_csv(params["output_fileNames"], skipinitialspace=True)
-
-        csv_rasters = csv_out.inundation_rasters
-
-        actual_rasters = []
-        for file in os.listdir(params["hydrofabric_dir"]):
-            if file.endswith(".tif"):
-                actual_rasters.append(file)
-
-        for csv_raster, actual_raster in zip(csv_rasters, actual_rasters):
-            assert os.path.exists(
-                os.path.join(params["hydrofabric_dir"], actual_raster)
-            ), f"Inundation Raster {csv_raster} does not exist"
-
-    # Logging removed from tools/inundate_gms.py as of 8/1/23
-    # assert os.path.exists(params["log_file"]), "Log file expected and does not exist"
diff --git a/unit_tests/inundation_params.json b/unit_tests/inundation_params.json
deleted file mode 100644
index b566789f8..000000000
--- a/unit_tests/inundation_params.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
-	"valid_data_inundate_branch": {
-		"rem": "/data/outputs/unit_test_data/02020005/branches/3246000001/rem_zeroed_masked_3246000001.tif",
-		"catchments": "/data/outputs/unit_test_data/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_3246000001.tif",
-		"catchment_poly": "/data/outputs/unit_test_data/02020005/branches/3246000001/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3246000001.gpkg",
-		"hydro_table": "/data/outputs/unit_test_data/02020005/branches/3246000001/hydroTable_3246000001.csv",
-		"forecast": "/data/test_cases/usgs_test_cases/validation_data_usgs/02020005/ptvn6/action/ahps_ptvn6_huc_02020005_flows_action.csv",
-		"mask_type": null,
-		"hucs": null,
-		"hucs_layerName": null,
-		"subset_hucs": null,
-		"num_workers": 1,
-		"aggregate": false,
-		"inundation_raster": "/data/test_cases/usgs_test_cases/02020005_usgs/testing_versions/unit_test_data/action/ptvn6_inundation_extent_02020005_3246000001.tif",
-		"inundation_polygon": null,
-		"depths": null,
-		"out_raster_profile": null,
-		"out_vector_profile": null,
-		"src_table": null,
-		"quiet": true,
-		"expected_inundation_raster": "/data/"
-	}
-}
diff --git a/unit_tests/inundation_test.py b/unit_tests/inundation_test.py
deleted file mode 100644
index ef580c15a..000000000
--- a/unit_tests/inundation_test.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import sys
-import unittest
-
-import inundation as src
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-
-class test_inundate(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    @pytest.mark.skip(reason="Inundate_gms will be rebuilt in the future, so this test will be skipped.")
-    def test_inundate_create_inundation_raster_single_branch_success(self):
-        """
-        Test for creating a inundation branch raster, no depth raster and no
-        inundation_polygons, no subsets, no mask
-        """
-
-        params = self.params["valid_data_inundate_branch"].copy()
-
-        # returns list of rasters and polys
-        in_rasters, depth_rasters, in_polys = src.inundate(
-            rem=params["rem"],
-            catchments=params["catchments"],
-            catchment_poly=params["catchment_poly"],
-            hydro_table=params["hydro_table"],
-            forecast=params["forecast"],
-            mask_type=params["mask_type"],
-            hucs=params["hucs"],
-            hucs_layerName=params["hucs_layerName"],
-            subset_hucs=params["subset_hucs"],
-            num_workers=params["num_workers"],
-            aggregate=params["aggregate"],
-            inundation_raster=params["inundation_raster"],
-            inundation_polygon=params["inundation_polygon"],
-            depths=params["depths"],
-            out_raster_profile=params["out_raster_profile"],
-            out_vector_profile=params["out_vector_profile"],
-            src_table=params["src_table"],
-            quiet=params["quiet"],
-        )
-
-        print("in_rasters")
-        print(in_rasters)
-
-        assert len(in_rasters) == 1, "Expected exactly one inundation raster path records"
-        assert depth_rasters[0] is None, "Expected no depth raster path records"
-        assert in_polys[0] is None, "Expected no inundation_polys records"
-
-        msg = f"Expected file {params['expected_inundation_raster']} but it does not exist."
-        assert os.path.exists(params["expected_inundation_raster"]) is True, msg
diff --git a/unit_tests/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json
deleted file mode 100644
index a3d70bca5..000000000
--- a/unit_tests/outputs_cleanup_params.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "valid_specific_branch_data": {
-        "src_dir": "/data/outputs/unit_test_data/02020005/branches/2274000018",
-        "deny_list": "/foss_fim/config/deny_branches.lst",
-        "branch_id": "2274000018",
-        "verbose": true
-    },
-    "valid_directory_data": {
-        "src_dir": "/data/outputs/unit_test_data",
-        "deny_list": "/foss_fim/config/deny_branches.lst",
-        "branch_id": "0",
-        "verbose": true
-    },
-    "skip_clean": {
-        "src_dir": "/data/outputs/unit_test_data",
-        "deny_list": "None",
-        "branch_id": "",
-        "verbose": true
-    }
-}
diff --git a/unit_tests/outputs_cleanup_test.py b/unit_tests/outputs_cleanup_test.py
deleted file mode 100644
index 5d152864f..000000000
--- a/unit_tests/outputs_cleanup_test.py
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env python3
-
-import io
-import json
-import os
-import sys
-import unittest
-from contextlib import redirect_stdout
-from pathlib import Path
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import outputs_cleanup as src
-from utils.shared_functions import FIM_Helpers as fh
-
-
-class test_outputs_cleanup(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    def test_remove_deny_list_files_specific_branch_success(self):
-        """
-        This validates removal of files for a directory already pointing to a
-        specific branch in a HUC
-        """
-
-        params = self.params["valid_specific_branch_data"].copy()
-
-        # Gather all of the file names into an array from the deny_list
-        deny_files = self.__get_deny_list_filenames(
-            params["src_dir"], params["deny_list"], params["branch_id"]
-        )
-
-        # Test whether we have a list of files to check
-        assert len(deny_files) > 0
-
-        src.remove_deny_list_files(
-            src_dir=params["src_dir"],
-            deny_list=params["deny_list"],
-            branch_id=params["branch_id"],
-            verbose=params["verbose"],
-        )
-
-        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) is True
-
-    def test_remove_deny_list_files_huc_level_success(self):
-        """
-        This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but
-        can be anything
-        """
-
-        params = self.params["valid_directory_data"].copy()
-
-        # Gather all of the file names into an array from the deny_list
-        deny_files = self.__get_deny_list_filenames(
-            params["src_dir"], params["deny_list"], params["branch_id"]
-        )
-
-        # Test whether we have a list of files to check
-        assert len(deny_files) > 0
-
-        src.remove_deny_list_files(
-            src_dir=params["src_dir"],
-            deny_list=params["deny_list"],
-            branch_id=params["branch_id"],
-            verbose=params["verbose"],
-        )
-
-        assert self.__check_no_deny_list_files_exist(params["src_dir"], deny_files) is True
-
-    def test_remove_deny_list_skip_cleaning_success(self):
-        """
-        This validates removal of files for all files and subdirectory files.
-        Normally used for covering all hucs and their branch zeros but
-        can be anything
-        """
-
-        params = self.params["skip_clean"].copy()
-
-        deny_files = self.__get_deny_list_filenames(
-            params["src_dir"], params["deny_list"], params["branch_id"]
-        )
-
-        # Ensure we have a value of "None" for a deny_list value,
-        # __get_deny_list_filenames returns an empty array if "None" is provided
-        assert len(deny_files) == 0
-
-        # This is tricky, as we're capturing the stdout (return statement) from remove_deny_list_files,
-        # to verify the function is returning at the correct place, and not removing files
-        # when we do not provide a deny list file. We set f to the io stream, and redirect it using
-        # redirect_stdout.
-        f = io.StringIO()
-        with redirect_stdout(f):
-            src.remove_deny_list_files(
-                src_dir=params["src_dir"],
-                deny_list=params["deny_list"],
-                branch_id=params["branch_id"],
-                verbose=params["verbose"],
-            )
-
-        # Get the stdout value of remove_deny_list_files and set it to skip_clean_out
-        skip_clean_out = f.getvalue()
-
-        # This string must match the print statement in /src/gms/outputs_cleanup.py, including the \n newline,
-        # which occurs "behind the scenes" with every call to print() in Python
-        assert skip_clean_out == "file clean via the deny list skipped\n"
-
-    def test_remove_deny_list_files_invalid_src_directory(self):
-        """
-        Double check the src directory exists
-        """
-
-        params = self.params["valid_specific_branch_data"].copy()
-        params["src_dir"] = "/data/does_no_exist"
-
-        # We want an exception to be thrown here, if so, the test passes.
-        with pytest.raises(Exception):
-            src.remove_deny_list_files(
-                src_dir=params["src_dir"],
-                deny_list=params["deny_list"],
-                branch_id=params["branch_id"],
-                verbose=params["verbose"],
-            )
-
-    def test_remove_deny_list_files_invalid_deny_list_does_not_exist(self):
-        """
-        Double check the deny list exists
-        """
-
-        params = self.params["valid_specific_branch_data"].copy()
-        params["deny_list"] = "invalid_file_name.txt"
-
-        # We want an exception to be thrown here, if so, the test passes.
-        with pytest.raises(Exception):
-            src.remove_deny_list_files(
-                src_dir=params["src_dir"],
-                deny_list=params["deny_list"],
-                branch_id=params["branch_id"],
-                verbose=params["verbose"],
-            )
-
-    # Helper Functions:
-
-    def __get_deny_list_filenames(self, src_dir, deny_list, branch_id):
-        deny_list_files = []
-
-        if deny_list == "None":
-            return deny_list_files
-
-        # Note: some of the deny_file_names might be a comment line
-        # this will validate file exists
-        deny_file_names = fh.load_list_file(deny_list.strip())
-
-        for deny_file_name in deny_file_names:
-            # Only add files to the list that do not start with a #
-            deny_file_name = deny_file_name.strip()
-            if deny_file_name.startswith("#"):
-                continue
-
-            deny_file_name = deny_file_name.replace("{}", branch_id)
-
-            deny_list_files.append(deny_file_name)
-
-        return deny_list_files
-
-    def __check_no_deny_list_files_exist(self, src_dir, deny_array):
-        found_files = []
-
-        for file_name in deny_array:
-            found_files.append(os.path.join(src_dir, file_name))
-
-        for found_file in found_files:
-            if os.path.exists(found_file):
-                return False
-
-        return True
diff --git a/unit_tests/pyproject.toml b/unit_tests/pyproject.toml
deleted file mode 100644
index 3ed480446..000000000
--- a/unit_tests/pyproject.toml
+++ /dev/null
@@ -1,8 +0,0 @@
-[tool.pytest.ini_options]
-filterwarnings = [
-    "error",
-    "ignore::UserWarning",
-    "ignore::DeprecationWarning",
-    "ignore::PendingDeprecationWarning",
-    "ignore::RuntimeWarning",
-]
diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json
deleted file mode 100644
index 89c254900..000000000
--- a/unit_tests/rating_curve_comparison_params.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-    "valid_data": {
-        "02020005": [
-            "/data/outputs/unit_test_data/02020005/usgs_elev_table.csv",
-            "/data/outputs/unit_test_data/02020005/branches",
-            "/data/inputs/usgs_gages/usgs_rating_curves.csv",
-            "/data/outputs/unit_test_data/rating_curve_comparison/tables/usgs_interpolated_elevation_stats_02020005.csv",
-            "/data/outputs/unit_test_data/rating_curve_comparison/tables/nwm_recurrence_flow_elevations_02020005.csv",
-            "/data/outputs/unit_test_data/rating_curve_comparison/plots/FIM-USGS_rating_curve_comparison_02020005.png",
-            "/data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/",
-            "/data/inputs/usgs_gages/catfim_flows_cms.csv",
-            "02020005",
-            false,
-            false
-        ]
-    }
-}
diff --git a/unit_tests/rating_curve_comparison_test.py b/unit_tests/rating_curve_comparison_test.py
deleted file mode 100644
index 89352cb26..000000000
--- a/unit_tests/rating_curve_comparison_test.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import unittest
-
-import pytest
-from rating_curve_comparison import generate_rating_curve_metrics
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-
-class test_rating_curve_comparison(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases
-
-    @pytest.mark.skip(
-        reason="rating_curve has invalid pathing based on files "
-        "that are normally not in a runtime folder. This test will disabled for now."
-    )
-    def test_generate_rating_curve_metrics_02020005_success(self):
-        """
-        We are testing whether a .png file was created for a FIM-USGS rating curve comparison,
-        for HUC 02020005 using the `generate_rating_curve_metrics` function.
-        The 5th index (parameter) for each HUC in `rating_curve_comparison_params.json` specifies
-        the FIM-USGS rating curve comparison .png filepath.
-        """
-
-        params = self.params[
-            "valid_data"
-        ].copy()  # update "valid_data" value if you need to (aka.. more than one node)
-
-        _indiv_huc_params = params["02020005"]
-
-        # To setup the test, lets start by deleted the expected output file to ensure
-        # that it is regenerated.
-        if os.path.exists(_indiv_huc_params[5]):
-            os.remove(_indiv_huc_params[5])
-
-        # Test that the file was deleted
-        assert os.path.exists(_indiv_huc_params[5]) is False
-
-        generate_rating_curve_metrics(_indiv_huc_params)
-
-        # Test that the file was created by generate_rating_curve_metrics
-        assert os.path.exists(_indiv_huc_params[5]) is True
diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json
deleted file mode 100644
index aff840dd5..000000000
--- a/unit_tests/shared_functions_params.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-    "append_append_id_to_file_name_single_identifier_valid": {
-        "outputDestDir": "/output/unit_test_data/",
-        "file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
-        "identifier": "8888",
-        "expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_8888.tif"
-    },
-    "append_append_id_to_file_name_identifier_list_valid": {
-        "outputDestDir": "/output/unit_test_data/",
-        "file_name": "02020005/branches/3246000003/rem_zeroed_masked_3246000003.tif",
-        "identifier": [
-            "7777",
-            "8888"
-        ],
-        "expected_output": "02020005/branches/3246000003/rem_zeroed_masked_3246000003_7777_8888.tif"
-    }
-}
diff --git a/unit_tests/shared_functions_test.py b/unit_tests/shared_functions_test.py
deleted file mode 100644
index 62264ec75..000000000
--- a/unit_tests/shared_functions_test.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-from utils.shared_functions import FIM_Helpers as src
-
-
-class test_shared_functions(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    def test_append_id_to_file_name_single_identifier_success(self):
-        """
-        Pass in a file name with the single identifier and get the single adjusted file name back
-        """
-
-        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()
-
-        actual_output = src.append_id_to_file_name(
-            file_name=ut_helpers.json_concat(params, "outputDestDir", "file_name"),
-            identifier=params["identifier"],
-        )
-
-        err_msg = "actual output does not match expected output"
-
-        expected_output = ut_helpers.json_concat(params, "outputDestDir", "expected_output")
-
-        assert expected_output == actual_output, err_msg
-
-    def test_append_id_to_file_name_indentifer_list_success(self):
-        """
-        Pass in a file name with the list of identifiers and
-        get a file name back with multiple identifers added.
-        """
-
-        params = self.params["append_append_id_to_file_name_identifier_list_valid"].copy()
-
-        actual_output = src.append_id_to_file_name(
-            file_name=ut_helpers.json_concat(params, "outputDestDir", "file_name"),
-            identifier=params["identifier"],
-        )
-
-        err_msg = "actual output does not match expected output"
-
-        expected_output = ut_helpers.json_concat(params, "outputDestDir", "expected_output")
-
-        assert expected_output == actual_output, err_msg
-
-    def test_append_id_to_file_name_no_file_name_success(self):
-        """
-        Pass in an non existant file name and get None back
-        """
-
-        params = self.params["append_append_id_to_file_name_single_identifier_valid"].copy()
-
-        actual_output = src.append_id_to_file_name(None, identifier=params["identifier"])
-
-        error_msg = "actual output should not have a value"
-
-        assert actual_output is None, error_msg
diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json
deleted file mode 100644
index a4a17c248..000000000
--- a/unit_tests/split_flows_params.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-    "valid_data": {
-        "outputDestDir": "/data/outputs/unit_test_data/",
-        "max_length": 1500,
-        "slope_min": 0.001,
-        "lakes_buffer_input": 20,
-        "flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/demDerived_reaches_2274000031.shp",
-        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/dem_thalwegCond_2274000031.tif",
-        "split_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/demDerived_reaches_split_2274000031.gpkg",
-        "split_points_filename": "/data/outputs/unit_test_data/02020005/branches/2274000031/demDerived_reaches_split_points_2274000031.gpkg",
-        "wbd8_clp_filename": "/data/outputs/unit_test_data/02020005/wbd8_clp.gpkg",
-        "lakes_filename": "/data/outputs/unit_test_data/02020005/nwm_lakes_proj_subset.gpkg",
-        "nwm_streams_filename": "/data/outputs/unit_test_data/02020005/nwm_subset_streams_levelPaths.gpkg"
-    }
-}
diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py
deleted file mode 100644
index a811fccfc..000000000
--- a/unit_tests/split_flows_test.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env python3
-import json
-import os
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-import split_flows as src
-
-
-class test_split_flows(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    # Ensure split_flows_filename & split_points_filename are created by the split_flows function
-    def test_split_flows_success(self):
-        """
-        The /data/outputs/unit_test_data/<huc>/branches/<branchid>/
-            demDerived_reaches_split_<branchid>.gpkg and
-        /data/outputs/unit_test_data/<huc>/branches/<branchid>/
-            demDerived_reaches_split_points_<branchid>.gpkg
-        should not exit prior to this test. If the test is successful, these files will be created.
-        """
-
-        params = self.params["valid_data"].copy()
-
-        # to setup the test, lets start by deleted the two expected output files to ensure
-        # that they are regenerated
-        if os.path.exists(params["split_flows_filename"]):
-            os.remove(params["split_flows_filename"])
-        if os.path.exists(params["split_points_filename"]):
-            os.remove(params["split_points_filename"])
-
-        error_msg = params["split_flows_filename"] + " does exist, when it should not (post os.remove call)"
-        assert os.path.exists(params["split_flows_filename"]) is False, error_msg
-
-        error_msg = params["split_points_filename"] + " does exist, when it should not (post os.remove call)"
-        assert os.path.exists(params["split_points_filename"]) is False, error_msg
-
-        src.split_flows(
-            max_length=params["max_length"],
-            slope_min=params["slope_min"],
-            lakes_buffer_input=params["lakes_buffer_input"],
-            flows_filename=params["flows_filename"],
-            dem_filename=params["dem_filename"],
-            split_flows_filename=params["split_flows_filename"],
-            split_points_filename=params["split_points_filename"],
-            wbd8_clp_filename=params["wbd8_clp_filename"],
-            lakes_filename=params["lakes_filename"],
-            nwm_streams_filename=params["nwm_streams_filename"],
-        )
-
-        error_msg = params["split_flows_filename"] + " does not exist"
-        assert os.path.exists(params["split_flows_filename"]) is True, error_msg
-
-        error_msg = params["split_points_filename"] + " does not exist"
-        assert os.path.exists(params["split_points_filename"]) is True, error_msg
diff --git a/unit_tests/unit_tests_utils.py b/unit_tests/unit_tests_utils.py
deleted file mode 100644
index 847158b0d..000000000
--- a/unit_tests/unit_tests_utils.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-import errno
-import os
-import sys
-
-
-class FIM_unit_test_helpers(object):
-    @staticmethod
-    def get_params_filename(unit_test_file_name):
-        unittest_file_name = os.path.basename(unit_test_file_name)
-        params_file_name = unittest_file_name.replace("_test.py", "_params.json")
-        params_file_path = os.path.join(os.path.dirname(unit_test_file_name), params_file_name)
-
-        if not os.path.exists(params_file_path):
-            raise FileNotFoundError(f"{params_file_path} does not exist")
-
-        return params_file_path
-
-    @staticmethod
-    def json_concat(obj, keyone, keytwo):
-        concat_string = obj[keyone] + obj[keytwo]
-
-        return concat_string
diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json
deleted file mode 100644
index e65627bed..000000000
--- a/unit_tests/usgs_gage_crosswalk_params.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-    "valid_data": {
-        "usgs_gages_filename": "/data/outputs/unit_test_data/02020005/usgs_subset_gages.gpkg",
-        "input_flows_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/demDerived_reaches_split_filtered_2274000028.gpkg",
-        "input_catchment_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/gw_catchments_reaches_filtered_addedAttributes_2274000028.gpkg",
-        "dem_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/dem_meters_2274000028.tif",
-        "dem_adj_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/dem_thalwegCond_2274000028.tif",
-        "output_table_filename": "/data/outputs/unit_test_data/02020005/branches/2274000028/usgs_elev_table.csv",
-        "output_directory": "/data/outputs/unit_test_data/02020005/branches/2274000028",
-        "branch_id": "2274000028"
-    }
-}
diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py
deleted file mode 100644
index 863c61590..000000000
--- a/unit_tests/usgs_gage_crosswalk_test.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import os
-import sys
-import unittest
-
-import pytest
-from unit_tests_utils import FIM_unit_test_helpers as ut_helpers
-
-from usgs_gage_crosswalk import GageCrosswalk
-
-
-class test_usgs_gage_crosswalk(unittest.TestCase):
-    """
-    Allows the params to be loaded one and used for all test methods
-    """
-
-    @classmethod
-    def setUpClass(self):
-        params_file_path = ut_helpers.get_params_filename(__file__)
-        with open(params_file_path) as params_file:
-            self.params = json.load(params_file)
-
-    # Test Cases:
-
-    def test_GageCrosswalk_success(self):
-        """
-        Test whether the GageCrosswalk object can be instantiated, and test that the run_crosswalk method can
-        successfully create the output table (usgs_elev_table.csv).
-        """
-
-        params = self.params[
-            "valid_data"
-        ].copy()  # update "valid_data" value if you need to (aka.. more than one node)
-
-        # Delete the usgs_elev_table.csv if it exists
-        if os.path.exists(params["output_table_filename"]):
-            os.remove(params["output_table_filename"])
-
-        # Verify the usgs_elev_table.csv was deleted
-        msg = f'{params["output_table_filename"]} does exist, when it should have been deleted'
-        assert os.path.exists(params["output_table_filename"]) is False, msg
-
-        # Instantiate and run GageCrosswalk
-        gage_crosswalk = GageCrosswalk(params["usgs_gages_filename"], params["branch_id"])
-
-        # Run crosswalk
-        gage_crosswalk.run_crosswalk(
-            params["input_catchment_filename"],
-            params["input_flows_filename"],
-            params["dem_filename"],
-            params["dem_adj_filename"],
-            params["output_directory"],
-        )
-
-        # Make sure that the usgs_elev_table.csv was written
-        msg = f'{params["output_table_filename"]} does not exist'
-        assert os.path.exists(params["output_table_filename"]) is True, msg

From b6624948fbf558011b12564248fdc321c664d514 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 24 Jul 2024 16:18:07 +0000
Subject: [PATCH 336/359] v4.5.3.1 Remove on screen warning from numpy for
 alpha tests (#1233)

---
 docs/CHANGELOG.md               | 10 ++++++++++
 tools/overlapping_inundation.py |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7b062036a..f4bc51bee 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.5.3.1 - 2024-07-24 - [PR#1233](https://github.com/NOAA-OWP/inundation-mapping/pull/1233)
+
+In a PR [1217](https://github.com/NOAA-OWP/inundation-mapping/pull/1217), which is about to be merged, it updates a bunch of python packages. One is numpy. This has triggered a very large amount of on-screen output from a new numpy warning while running `synthesize_test_cases.py`.
+
+### Changes
+- `tools\overlapping_inundation.py`: As described
+
+ <br/><br/>
+ 
+
 ## v4.5.3.0 - 2024-07-24 - [PR#1217](https://github.com/NOAA-OWP/inundation-mapping/pull/1217)
 
 This PR rolls up a bunch of other PR's and python packages requests including:
diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py
index 68a13e921..fbb1bcc50 100644
--- a/tools/overlapping_inundation.py
+++ b/tools/overlapping_inundation.py
@@ -396,7 +396,7 @@ def merge_data(
     :param rst_dims: dimensions of overlapping rasters
     """
 
-    nan_tile = np.array([np.nan]).astype(dtype)[0]
+    nan_tile = np.array([np.nan])
     window_data = np.tile(float(nan_tile), [int(final_window.height), int(final_window.width)])
 
     for data, bnds, idx in zip(rst_data, window_bnds, datasets):

From 9cb4a636e89a2525c2b313e2581fea362bbccac4 Mon Sep 17 00:00:00 2001
From: ZahraGhahremani-NOAA
 <100253864+ZahraGhahremani@users.noreply.github.com>
Date: Fri, 2 Aug 2024 13:11:23 -0600
Subject: [PATCH 337/359] v4.5.4.0 Updating modules with new AEP (NWM V3)
 (#1198)

---
 docs/CHANGELOG.md                   | 11 +++++++++++
 src/bash_variables.env              |  6 +++---
 src/src_adjust_ras2fim_rating.py    | 10 ++++------
 src/src_adjust_usgs_rating_trace.py | 10 ++++------
 tools/composite_inundation.py       |  2 +-
 tools/inundate_nation.py            | 10 +++++-----
 tools/rating_curve_comparison.py    |  4 ++--
 7 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f4bc51bee..f0ef91ee2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.4.0 - 2024-08-02 - [PR#1198](https://github.com/NOAA-OWP/inundation-mapping/pull/1198)
+
+### Changes
+- `src/bash_variables.env`: high water threshold and recurrence flows CSV files were updated into new NWM v3 flow files. Also, a new Manning numbers file created from the new NWM v3 dataset was used.
+-  `src/src_adjust_ras2fim_rating.py`: 100 year recurrence was removed since it is not included in the new AEP.
+-  `src/src_adjust_usgs_rating_trace.py`: 100 year recurrence was removed since it is not included in the new AEP.
+-  `tools/rating_curve_comparison.py`: 100 year recurrence was removed since it is not included in the new AEP. Also, the name of recurrence flow CSV file was updated.
+-  `tools/composite_inundation.py`
+-  `tools/inundate_nation.py`
+
+<br/><br/>
 
 ## v4.5.3.1 - 2024-07-24 - [PR#1233](https://github.com/NOAA-OWP/inundation-mapping/pull/1233)
 
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 0416ff087..637b24eac 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -35,13 +35,13 @@ export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry
 export                            osm_bridges=${inputsDir}/osm/bridges/240426/osm_all_bridges.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
-export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
+export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv
 
 # input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_06_12.csv
+export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_nwm3.csv
 
 # input file location with nwm feature_id and recurrence flow values
-export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
+export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm3_17C_recurrence_flows_cfs.csv
 
 # input file location with usgs rating curve database
 export             usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
index 22a47b33e..cbeb301c0 100644
--- a/src/src_adjust_ras2fim_rating.py
+++ b/src/src_adjust_ras2fim_rating.py
@@ -89,7 +89,8 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
 
     # read in the NWM recurr csv file
     nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
-    nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
+    if "Unnamed: 0" in nwm_recur_df.columns:
+        nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
     nwm_recur_df.rename(
         columns={
             '2_0_year_recurrence_flow_17C': '2_0_year',
@@ -97,21 +98,18 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
             '10_0_year_recurrence_flow_17C': '10_0_year',
             '25_0_year_recurrence_flow_17C': '25_0_year',
             '50_0_year_recurrence_flow_17C': '50_0_year',
-            '100_0_year_recurrence_flow_17C': '100_0_year',
         },
         inplace=True,
     )
 
     # convert cfs to cms (x 0.028317)
-    nwm_recur_df.loc[
-        :, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year', '100_0_year']
-    ] *= 0.028317
+    nwm_recur_df.loc[:, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year']] *= 0.028317
 
     # merge nwm recurr with ras_rc_df
     merge_df = ras_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
 
     # NWM recurr intervals
-    recurr_intervals = ["2", "5", "10", "25", "50", "100"]  # "2","5","10","25","50","100"
+    recurr_intervals = ["2", "5", "10", "25", "50"]  # "2","5","10","25","50","100"
     final_df = pd.DataFrame()  # create empty dataframe to append flow interval dataframes
     for interval in recurr_intervals:
         log_text += '\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n'
diff --git a/src/src_adjust_usgs_rating_trace.py b/src/src_adjust_usgs_rating_trace.py
index ac923a5be..f3dce13e7 100644
--- a/src/src_adjust_usgs_rating_trace.py
+++ b/src/src_adjust_usgs_rating_trace.py
@@ -87,7 +87,8 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
 
     # read in the NWM recurr csv file
     nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
-    nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
+    if "Unnamed: 0" in nwm_recur_df.columns:
+        nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
     nwm_recur_df = nwm_recur_df.rename(
         columns={
             '2_0_year_recurrence_flow_17C': '2_0_year',
@@ -95,20 +96,17 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
             '10_0_year_recurrence_flow_17C': '10_0_year',
             '25_0_year_recurrence_flow_17C': '25_0_year',
             '50_0_year_recurrence_flow_17C': '50_0_year',
-            '100_0_year_recurrence_flow_17C': '100_0_year',
         }
     )
 
     # convert cfs to cms (x 0.028317)
-    nwm_recur_df.loc[
-        :, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year', '100_0_year']
-    ] *= 0.028317
+    nwm_recur_df.loc[:, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year']] *= 0.028317
 
     # merge nwm recurr with usgs_rc
     merge_df = usgs_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
 
     # NWM recurr intervals
-    recurr_intervals = ("2", "5", "10", "25", "50", "100")
+    recurr_intervals = ("2", "5", "10", "25", "50")
     final_df = pd.DataFrame()  # create empty dataframe to append flow interval dataframes
     for interval in recurr_intervals:
         log_text += '\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n'
diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py
index 695adb075..3651e764f 100644
--- a/tools/composite_inundation.py
+++ b/tools/composite_inundation.py
@@ -433,7 +433,7 @@ def to_bin(x):
     python3 /foss_fim/tools/composite_inundation.py
         -fr /outputs/inundation_test_1_FIM3_fr
         -gms /outputs/inundation_test_1_gms
-        -u 13090001 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv
+        -u 13090001 -f /data/inputs/rating_curve/nwm_recur_flows/nwm3_17C_recurr_25_0_cms.csv
         -o /outputs/inundation_test_1_comp/
         -n test_inundation.tif
     """
diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py
index 050436670..8bfe8476b 100755
--- a/tools/inundate_nation.py
+++ b/tools/inundate_nation.py
@@ -16,7 +16,7 @@
 from utils.shared_functions import FIM_Helpers as fh
 
 
-# INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
+# INUN_REVIEW_DIR = r'/data/inputs/rating_curve/nwm_recur_flows/'
 # INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
 # INPUTS_DIR = r'/data/inputs'
 # OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/
@@ -273,7 +273,7 @@ def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO
     Sample usage:
     python3 /foss_fim/tools/inundate_nation.py
         -r /outputs/fim_4_0_9_2 -m 100_0
-        -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_100_0_cms.csv
+        -f /data/inputs/rating_curve/nwm_recur_flows/nwm3_17C_recurr_10_0_cms.csv
         -s
         -j 10
     outputs become /data/inundation_review/inundate_nation/100_0_fim_4_0_9_2_mosiac.tif (.log, etc)
@@ -281,7 +281,7 @@ def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO
     python3 /foss_fim/tools/inundate_nation.py
         -r /outputs/fim_4_0_9_2
         -m hw
-        -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv
+        -f /data/inputs/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv
         -s
         -j 10
     outputs become /data/inundation_review/inundate_nation/hw_fim_4_0_9_2_mosiac.tif (.log, etc)
@@ -329,8 +329,8 @@ def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO
         '-f',
         '--flow_file',
         help='the path and flow file to be used. '
-        'ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/'
-        'nwm_high_water_threshold_cms.csv',
+        'ie /data/inputs/rating_curve/bankfull_flows/'
+        'nwm3_high_water_threshold_cms.csv',
         required=True,
     )
 
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index ef8e2417f..0a9e58acb 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -200,10 +200,10 @@ def generate_rating_curve_metrics(args):
             rating_curves['order_'] = rating_curves['order_'].astype('int')
 
             # NWM recurr intervals
-            recurr_intervals = ("2", "5", "10", "25", "50", "100")
+            recurr_intervals = ("2", "5", "10", "25", "50")
             recurr_dfs = []
             for interval in recurr_intervals:
-                recurr_file = join(nwm_flow_dir, 'nwm21_17C_recurr_{}_0_cms.csv'.format(interval))
+                recurr_file = join(nwm_flow_dir, 'nwm3_17C_recurr_{}_0_cms.csv'.format(interval))
                 df = pd.read_csv(recurr_file, dtype={'feature_id': str})
                 # Update column names
                 df = df.rename(columns={"discharge": interval})

From fa670d4fb98c721aa1097fc97b801f2b012dcf1d Mon Sep 17 00:00:00 2001
From: AliForghani-NOAA <124194814+AliForghani-NOAA@users.noreply.github.com>
Date: Fri, 2 Aug 2024 15:17:45 -0400
Subject: [PATCH 338/359] v4.5.4.1 Retrieve code to process NWS AHPS benchmarks
 for Alaska (#1185)

---
 data/nws/.gitignore             |   4 -
 data/nws/preprocess_ahps_nws.py | 468 ++++++++++++++++++++++++++++++++
 docs/CHANGELOG.md               |  25 ++
 tools/tools_shared_functions.py |   4 +-
 4 files changed, 495 insertions(+), 6 deletions(-)
 delete mode 100644 data/nws/.gitignore
 create mode 100644 data/nws/preprocess_ahps_nws.py

diff --git a/data/nws/.gitignore b/data/nws/.gitignore
deleted file mode 100644
index 5e7d2734c..000000000
--- a/data/nws/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore
diff --git a/data/nws/preprocess_ahps_nws.py b/data/nws/preprocess_ahps_nws.py
new file mode 100644
index 000000000..8d46b8a9d
--- /dev/null
+++ b/data/nws/preprocess_ahps_nws.py
@@ -0,0 +1,468 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import pathlib
+import sys
+import traceback
+from collections import defaultdict
+from pathlib import Path
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
+from dotenv import load_dotenv
+from tools_shared_functions import (
+    aggregate_wbd_hucs,
+    flow_data,
+    get_datum,
+    get_metadata,
+    get_nwm_segs,
+    get_rating_curve,
+    get_thresholds,
+    mainstem_nwm_segs,
+    ngvd_to_navd_ft,
+    process_extent,
+    process_grid,
+    raster_to_feature,
+    select_grids,
+)
+
+from utils.shared_variables import PREP_PROJECTION, VIZ_PROJECTION
+
+
+def get_env_paths():
+    load_dotenv()
+    # import variables from .env file
+    API_BASE_URL = os.getenv("API_BASE_URL")
+    EVALUATED_SITES_CSV = os.getenv("EVALUATED_SITES_CSV")
+    WBD_LAYER = os.getenv("WBD_LAYER")
+    return API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER
+
+
+########################################################
+# Preprocess AHPS NWS
+# This script will work on NWS AHPS fim data (some assumptions made about the data structure).
+# Provide a source directory path (source_dir) where all NWS AHPS FIM data is located. NWS source data was previously downloaded and extracted. Some data is buried through several layers of subfolders in the source data. In general, the downloaded datasets were unzipped and starting from where the folder name was the AHPS code, this was copied and pasted into a new directory which is the source_dir.
+# Provide a destination directory path (destination) which is where all outputs are located.
+# Provide a reference raster path.
+########################################################
+# source_dir = Path(r'path/to/nws/downloads')
+# destination = Path(r'path/to/preprocessed/nws/data')
+# reference_raster= Path(r'path/to/reference raster')
+
+
+def preprocess_nws(source_dir, destination, reference_raster):
+    source_dir = Path(source_dir)
+    destination = Path(destination)
+    reference_raster = Path(reference_raster)
+    metadata_url = f'{API_BASE_URL}/metadata'
+    threshold_url = f'{API_BASE_URL}/nws_threshold'
+    rating_curve_url = f'{API_BASE_URL}/rating_curve'
+    log_file = destination / 'log.txt'
+
+    # Write a run-time log file
+    destination.mkdir(parents=True, exist_ok=True)
+    log_file = destination / 'log.txt'
+    f = open(log_file, 'a+')
+
+    # Define distance (in miles) to search for nwm segments
+    nwm_ds_search = 10
+    nwm_us_search = 10
+    # The NWS data was downloaded and unzipped. The ahps folder (with 5 digit code as folder name) was cut and pasted into a separate directory. So the ahps_codes iterates through that parent directory to get all of the AHPS codes that have data.
+    ahps_codes = [i.name for i in source_dir.glob('*') if i.is_dir() and len(i.name) == 5]
+    # Get mainstems NWM segments
+    # Workaround for sites in 02030103 and 02030104, many are not rfc_forecast_point = True
+    list_of_sites = pd.read_csv(EVALUATED_SITES_CSV)['Total_List'].to_list()
+    print("getting mainstem nwm segments")
+    ms_segs = mainstem_nwm_segs(metadata_url, list_of_sites)
+
+    # Find depth grid subfolder
+    for code in ahps_codes:
+        f.write(f'{code} : Processing\n')
+        print(f'processing {code}')
+        # 'mnda2' is in Alaska outside of NWM domain.
+        if code in ['mnda2']:
+            f.write(f'{code} : skipping because outside of NWM domain\n')
+            continue
+
+        # Get metadata of site and search for NWM segments x miles upstream/x miles downstream
+        select_by = 'nws_lid'
+        selector = [code]
+        metadata_list, metadata_df = get_metadata(
+            metadata_url,
+            select_by,
+            selector,
+            must_include=None,
+            upstream_trace_distance=nwm_us_search,
+            downstream_trace_distance=nwm_ds_search,
+        )
+        metadata = metadata_list[0]
+
+        # Assign huc to site using FIM huc layer.
+        dictionary, out_gdf = aggregate_wbd_hucs(metadata_list, Path(WBD_LAYER), retain_attributes=False)
+        [huc] = list(dictionary.keys())
+
+        # Get thresholds for action, minor, moderate, major. If no threshold data present, exit.
+        # The threshold flows source will dictate what rating curve (and datum) to use as it uses a decision tree (USGS priority then NRLDB)
+        # In multiple instances a USGS ID is given but then no USGS rating curve or in some cases no USGS datum is supplied.
+        select_by = 'nws_lid'
+        selector = code
+        stages, flows = get_thresholds(threshold_url, select_by, selector, threshold='all')
+
+        # Make sure at least one valid threshold is supplied from WRDS.
+        threshold_categories = ['action', 'minor', 'moderate', 'major']
+        if not any([stages[threshold] for threshold in threshold_categories]):
+            f.write(f'{code} : skipping because no threshold stages available\n')
+            continue
+
+        # determine source of interpolated threshold flows, this will be the rating curve that will be used.
+        rating_curve_source = flows.get('source')
+        if rating_curve_source is None:
+            f.write(f'{code} : skipping because no rating curve source\n')
+            continue
+
+        # Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null). Modifying rating curve source will influence the rating curve and datum retrieved for benchmark determinations.
+        if code == 'bmbp1':
+            rating_curve_source = 'NRLDB'
+        # Get the datum and adjust to NAVD if necessary.
+        nws, usgs = get_datum(metadata)
+        datum_data = {}
+        if rating_curve_source == 'USGS Rating Depot':
+            datum_data = usgs
+        elif rating_curve_source == 'NRLDB':
+            datum_data = nws
+
+        # If datum not supplied, skip to new site
+        datum = datum_data.get('datum', None)
+        if datum is None:
+            f.write(f'{code} : skipping because site is missing datum\n')
+            continue
+
+        # Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29 conversion to NAVD88
+        # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7, stkn4, wlln7
+        # Assumed to be NAD83 (no info from USGS or NWS data): dlrt2, eagi1, eppt2, jffw3, ldot2, rgdt2
+        if code in [
+            'bgwn7',
+            'dlrt2',
+            'eagi1',
+            'eppt2',
+            'fatw3',
+            'jffw3',
+            'ldot2',
+            'mnvn4',
+            'nhpp1',
+            'pinn4',
+            'rgdt2',
+            'rgln4',
+            'rssk1',
+            'sign4',
+            'smfn7',
+            'stkn4',
+            'wlln7',
+        ]:
+            datum_data.update(crs='NAD83')
+
+        # Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and is actually NAD27. This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and it matches NRLDB coordinates.
+        if code == 'bmbp1':
+            datum_data.update(crs='NAD27')
+        # Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure datum reported in NAVD88. If NGVD29 it is converted to NAVD88.
+        # bgwn7, eagi1 vertical datum unknown, assume navd88
+        # fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
+        # wlln7 USGS data indicates vcs is NGVD29 (USGS and NWS info agree on datum value).
+        if code in ['bgwn7', 'eagi1', 'fatw3']:
+            datum_data.update(vcs='NAVD88')
+        elif code == 'wlln7':
+            datum_data.update(vcs='NGVD29')
+
+        # Adjust datum to NAVD88 if needed
+        if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929', 'NGVD,1929']:
+            # Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously removed otherwise the region needs changed.
+            datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
+            datum88 = round(datum + datum_adj_ft, 2)
+        else:
+            datum88 = datum
+
+        # get entire rating curve, same source as interpolated threshold flows (USGS Rating Depot first then NRLDB rating curve).
+        if rating_curve_source == 'NRLDB':
+            site = [code]
+        elif rating_curve_source == 'USGS Rating Depot':
+            site = [metadata.get('identifiers').get('usgs_site_code')]
+
+        rating_curve = get_rating_curve(rating_curve_url, site)
+
+        # Add elevation fields to rating curve
+        # Add field with vertical coordinate system
+        vcs = datum_data['vcs']
+        if not vcs:
+            vcs = 'Unspecified, Assumed NAVD88'
+        rating_curve['vcs'] = vcs
+
+        # Add field with original datum
+        rating_curve['datum'] = datum
+
+        # If VCS is NGVD29 add rating curve elevation (in NGVD) as well as the NAVD88 datum
+        if vcs in ['NGVD29', 'NGVD 1929']:
+            # Add field with raw elevation conversion (datum + stage)
+            rating_curve['elevation_ngvd29'] = rating_curve['stage'] + datum
+            # Add field with adjusted NAVD88 datum
+            rating_curve['datum_navd88'] = datum88
+
+        # Add field with NAVD88 elevation
+        rating_curve['elevation_navd88'] = rating_curve['stage'] + datum88
+
+        # Search through ahps directory find depth grid folder
+        parent_path = source_dir / code
+
+        # Work around for bgwn7 and smit2 where grids were custom created from polygons (bgwn7-no grids, smit2 - no projection and applying projection from polygons had errors)
+        if code in ['bgwn7', 'smit2']:
+            [grids_dir] = [directory for directory in parent_path.glob('*custom*') if directory.is_dir()]
+        else:
+            # Find the directory containing depth grids. Assumes only one directory will be returned.
+            [grids_dir] = [directory for directory in parent_path.glob('*depth_grid*') if directory.is_dir()]
+
+        # Get grids (all NWS ESRI grids were converted to Geotiff)
+        grid_paths = [grids for grids in grids_dir.glob('*.tif*') if grids.suffix in ['.tif', '.tiff']]
+        grid_names = [name.stem for name in grid_paths]
+        # If grids are present, interpolate a flow for the grid.
+        if grid_paths:
+            # Construct Dataframe containing grid paths, names, datum, code
+            df = pd.DataFrame({'code': code, 'path': grid_paths, 'name': grid_names, 'datum88': datum88})
+            # Determine elevation from the grid name. All elevations are assumed to be in NAVD88 based on random inspection of AHPS inundation website layers.
+            df['elevation'] = (
+                df['name'].str.replace('elev_', '', case=False).str.replace('_', '.').astype(float)
+            )
+            # Add a stage column using the datum (in NAVD88). Stage is rounded to the nearest 0.1 ft.
+            df['stage'] = round(df['elevation'] - df['datum88'], 1)
+            # Sort stage in ascending order
+            df.sort_values(by='elevation', ascending=True, inplace=True)
+            # Interpolate flow from the rating curve using the elevation_navd88 values, if value is above or below the rating curve assign nan.
+            df['flow'] = np.interp(
+                df['elevation'],
+                rating_curve['elevation_navd88'],
+                rating_curve['flow'],
+                left=np.nan,
+                right=np.nan,
+            )
+            # Assign flow source to reflect interpolation from rc
+            df['flow_source'] = f'interpolated from {rating_curve_source} rating curve'
+
+        else:
+            f.write(f'{code} : Site has no benchmark grids\n')
+
+        # Select the appropriate threshold grid for evaluation. Using the supplied threshold stages and the calculated map stages.
+        grids, grid_flows = select_grids(df, stages, datum88, 1.1)
+
+        # workaroud for bigi1 and eag1 which have gridnames based on flows (not elevations)
+        if code in ['eagi1', 'bigi1']:
+            # Elevation is really flows (due to file names), assign this to stage
+            df['flow'] = df['elevation']
+            df['stage'] = df['elevation']
+            # Select grids using flows
+            grids, grid_flows = select_grids(df, flows, datum88, 500)
+            f.write(f'{code} : Site workaround grids names based on flows not elevation\n')
+
+        # Obtain NWM segments that are on ms to apply flows
+        segments = get_nwm_segs(metadata)
+        site_ms_segs = set(segments).intersection(ms_segs)
+        segments = list(site_ms_segs)
+
+        # Write out boolean benchmark raster and flow file
+        try:
+            # for each threshold
+            for i in ['action', 'minor', 'moderate', 'major']:
+                # Obtain the flow and grid associated with threshold.
+                flow = grid_flows[i]
+                grid = grids[i]
+                extent = grids['extent']
+                # Make sure that flow and flow grid are valid
+                if grid not in ['No Map', 'No Threshold', 'No Flow']:
+                    # define output directory (to be created later)
+                    outputdir = destination / huc / code / i
+
+                    # Create Binary Grids, first create domain of analysis, then create binary grid
+
+                    # Domain extent is largest floodmap in the static library WITH holes filled
+                    filled_domain_raster = outputdir.parent / f'{code}_filled_orig_domain.tif'
+
+                    # Open benchmark data as a rasterio object.
+                    benchmark = rasterio.open(grid)
+                    benchmark_profile = benchmark.profile
+
+                    # Open extent data as rasterio object
+                    domain = rasterio.open(extent)
+                    domain_profile = domain.profile
+
+                    # if grid doesn't have CRS, then assign CRS using a polygon from the ahps inundation library
+                    if not benchmark.crs:
+                        # Obtain crs of the first polygon inundation layer associated with ahps code. Assumes only one polygon* subdirectory and assumes the polygon directory has at least 1 inundation shapefile.
+                        [ahps_polygons_directory] = [
+                            directory for directory in parent_path.glob('*polygon*') if directory.is_dir()
+                        ]
+                        shapefile_path = list(ahps_polygons_directory.glob('*.shp'))[0]
+                        shapefile = gpd.read_file(shapefile_path)
+                        # Update benchmark and domain profiles with crs from shapefile. Assumed that benchmark/extent have same crs.
+                        benchmark_profile.update(crs=shapefile.crs)
+                        domain_profile.update(crs=shapefile.crs)
+
+                    # Create a domain raster if it does not exist.
+                    if not filled_domain_raster.exists():
+                        # Domain should have donut holes removed
+                        process_extent(domain, domain_profile, output_raster=filled_domain_raster)
+
+                    # Open domain raster as rasterio object
+                    filled_domain = rasterio.open(filled_domain_raster)
+                    filled_domain_profile = filled_domain.profile
+
+                    # Create the binary benchmark raster
+                    boolean_benchmark, boolean_profile = process_grid(
+                        benchmark, benchmark_profile, filled_domain, filled_domain_profile, reference_raster
+                    )
+
+                    # Output binary benchmark grid and flow file to destination
+                    outputdir.mkdir(parents=True, exist_ok=True)
+                    output_raster = outputdir / (f'ahps_{code}_huc_{huc}_extent_{i}.tif')
+
+                    with rasterio.Env():
+                        with rasterio.open(output_raster, 'w', **boolean_profile) as dst:
+                            dst.write(boolean_benchmark, 1)
+
+                    # Close datasets
+                    domain.close()
+                    filled_domain.close()
+                    benchmark.close()
+
+                    # Create the guts of the flow file.
+                    flow_info = flow_data(segments, flow)
+                    # Write out the flow file to csv
+                    output_flow_file = outputdir / (f'ahps_{code}_huc_{huc}_flows_{i}.csv')
+                    flow_info.to_csv(output_flow_file, index=False)
+
+        except Exception as e:
+            f.write(f'{code} : Error preprocessing benchmark\n{repr(e)}\n')
+            f.write(traceback.format_exc())
+            f.write('\n')
+            print(traceback.format_exc())
+        # Process extents, only create extent if ahps code subfolder is present in destination directory.
+        ahps_directory = destination / huc / code
+        if ahps_directory.exists():
+            # Delete original filled domain raster (it is an intermediate file to create benchmark data)
+            orig_domain_grid = ahps_directory / f'{code}_filled_orig_domain.tif'
+            orig_domain_grid.unlink()
+            # Create domain shapefile from any benchmark grid for site (each benchmark has domain footprint, value = 0).
+            filled_extent = list(ahps_directory.rglob('*_extent_*.tif'))[0]
+            domain_gpd = raster_to_feature(grid=filled_extent, profile_override=False, footprint_only=True)
+            domain_gpd['nws_lid'] = code
+            domain_gpd.to_file(ahps_directory / f'{code}_domain.shp')
+
+            # Populate attribute information for site
+            grids_attributes = pd.DataFrame(data=grids.items(), columns=['magnitude', 'path'])
+            flows_attributes = pd.DataFrame(data=grid_flows.items(), columns=['magnitude', 'grid_flow_cfs'])
+            threshold_attributes = pd.DataFrame(data=stages.items(), columns=['magnitude', 'magnitude_stage'])
+            # merge dataframes
+            attributes = grids_attributes.merge(flows_attributes, on='magnitude')
+            attributes = attributes.merge(threshold_attributes, on='magnitude')
+            attributes = attributes.merge(df[['path', 'stage', 'elevation', 'flow_source']], on='path')
+            # Strip out sensitive paths and convert magnitude stage to elevation
+            attributes['path'] = attributes['path'].apply(lambda x: Path(x).name)
+            attributes['magnitude_elev_navd88'] = (
+                (datum88 + attributes['magnitude_stage']).astype(float).round(1)
+            )
+            # Add general site information
+            attributes['nws_lid'] = code
+            attributes['wfo'] = metadata['nws_data']['wfo']
+            attributes['rfc'] = metadata['nws_data']['rfc']
+            attributes['state'] = metadata['nws_data']['state']
+            attributes['huc'] = huc
+            # Rename and Reorder columns
+            attributes.rename(
+                columns={
+                    'path': 'grid_name',
+                    'flow_source': 'grid_flow_source',
+                    'stage': 'grid_stage',
+                    'elevation': 'grid_elev_navd88',
+                },
+                inplace=True,
+            )
+            attributes = attributes[
+                [
+                    'nws_lid',
+                    'wfo',
+                    'rfc',
+                    'state',
+                    'huc',
+                    'magnitude',
+                    'magnitude_stage',
+                    'magnitude_elev_navd88',
+                    'grid_name',
+                    'grid_stage',
+                    'grid_elev_navd88',
+                    'grid_flow_cfs',
+                    'grid_flow_source',
+                ]
+            ]
+            # Save attributes to csv
+            attributes.to_csv(ahps_directory / f'{code}_attributes.csv', index=False)
+
+            # Write the rating curve to a file
+            rating_curve_output = ahps_directory / (f'{code}_rating_curve.csv')
+            rating_curve['lat'] = datum_data['lat']
+            rating_curve['lon'] = datum_data['lon']
+            rating_curve.to_csv(rating_curve_output, index=False)
+
+            # Write the interpolated flows to file
+            df_output = ahps_directory / (f'{code}_interpolated_flows.csv')
+            df.to_csv(df_output, index=False)
+
+        else:
+            f.write(f'{code} : Unable to evaluate site, missing all flows\n')
+
+    # Close log file.
+    f.close()
+
+    # Combine all attribute files
+    attribute_files = list(destination.rglob('*_attributes.csv'))
+    all_attributes_list = []
+    for i in attribute_files:
+        attribute_df = pd.read_csv(i, dtype={'huc': str})
+        all_attributes_list.append(attribute_df)
+    all_attributes_df = pd.concat(all_attributes_list)
+
+    if not all_attributes_df.empty:
+        all_attributes_df.to_csv(destination / 'attributes.csv', index=False)
+    return
+
+
+if __name__ == '__main__':
+
+    # sample:
+    # python  foss_fim/data/nws/preprocess_ahps_nws.py -s inputPath    -d outputPath    -r  referenceRaster
+
+    # inputPath: path to a directory containing a folder for each gage (gage/folder names must be 5 character). Each gage folder must
+    # have a sub-folder named "depth_grid" with depth grids as TIFF files prepared as below:
+    #     - Collect/Download the grid depth dataset, typically available as ESRI gdb.
+    #     - Use arcpy (or ArcGIS pro ) to convert the grid depths (in ESRI gdb) into TIFF file. Make sure the TIFF files have crs
+
+    # referenceRaster: path to an arbitrary output TIFF file from a FIM run. Note that for a site in CONUS, this referenceRaster
+    # must be from a FIM run for a CONUS HUC, and for a site in Alaska, this referenceRaster must be from a Alaska HUC FIM run.
+
+    # Two notes:
+    # 1- Sites in CONUS and Alaska cannot be mixed in a single run. Separate runs should be done for Alaska sites and CONUS sites.
+    # 2- Before running this script, add the name of the new site(s) to the  '/data/inputs/ahps_sites/evaluated_ahps_sites.csv' file
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description='Create preprocessed USGS benchmark datasets at AHPS locations.'
+    )
+    parser.add_argument(
+        '-s', '--source_dir', help='Workspace where all source data is located.', required=True
+    )
+    parser.add_argument('-d', '--destination', help='Directory where outputs are to be stored', required=True)
+    parser.add_argument(
+        '-r', '--reference_raster', help='reference raster used for benchmark raster creation', required=True
+    )
+    args = vars(parser.parse_args())
+
+    # Run get_env_paths and static_flow_lids
+    API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER = get_env_paths()
+    preprocess_nws(**args)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f0ef91ee2..31ecd3ce7 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,31 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.4.1 - 2024-08-02 - [PR#1185](https://github.com/NOAA-OWP/inundation-mapping/pull/1185)
+
+This PR brings back the `preprocess_ahps_nws.py` code to FIM4 and generates new AHPS benchmark datasets for sites SXRA2 and SKLA2 in Alaska.  The new AHPS benchmark datasets are available on dev1 here: "/dev_fim_share/foss_fim/outputs/ali_ahps_alaska/AHPS_Results_Alaska/19020302/"
+
+This PR closes issue #1130. 
+
+To process a new station, follow these steps:
+
+1. Add the name of the new site (s) to the`/data/inputs/ahps_sites/evaluated_ahps_sites.csv` file. 
+2. Collect/Download the grid depth dataset, typically available as ESRI gdb.
+3. Use arcpy (or ArcGIS pro ) to convert the grid depths (in ESRI gdb) into TIFF files
+    - Make sure the TIFF files have crs
+    - Store all the TIFF files in a directory called "depth_grid," which should be a sub-folder inside a folder named after the gage code (must be a 5-character code)
+4. Run the script as described below. **Note that sites in CONUS and Alaska cannot be mixed in a single run. Separate runs should be done for Alaska sites and CONUS sites.**
+
+Note that for the "SKLA2" site, the downloaded ESRI-GDB grid files had a datum issue. This issue was manually corrected during the conversion of GDB files into TIFF files.
+
+### Additions
+- `data/nws/preprocess_ahps_nws.py`  ... retrieved from previous versions of FIM and updated for shapely v2
+
+### Changes
+- `tools/tools_shared_functions.py`  ... updated for shapely v2
+
+<br/><br/>
+
 ## v4.5.4.0 - 2024-08-02 - [PR#1198](https://github.com/NOAA-OWP/inundation-mapping/pull/1198)
 
 ### Changes
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index f59c1be5b..66c0a969c 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -1390,8 +1390,8 @@ def process_extent(extent, profile, output_raster=False):
     # Fill holes in extent
     poly_extent_fill_holes = MultiPolygon(Polygon(p.exterior) for p in poly_extent['geometry'])
     # loop through the filled polygons and insert the new geometry
-    for i in range(len(poly_extent_fill_holes)):
-        poly_extent.loc[i, 'geometry'] = poly_extent_fill_holes[i]
+    for i, part in enumerate(poly_extent_fill_holes.geoms):
+        poly_extent.loc[i, 'geometry'] = part
 
     # Dissolve filled holes with main map and explode
     poly_extent['dissolve_field'] = 1

From 34237579383a2dd1f1000c600bc6f875242bc2d3 Mon Sep 17 00:00:00 2001
From: HeidiSafa-NOAA <safa.hamideh@gmail.com>
Date: Fri, 2 Aug 2024 19:28:50 +0000
Subject: [PATCH 339/359] v4.5.4.2 Updating bathymetry-preprocessing script
 (#1125)

---
 data/bathymetry/preprocess_bathymetry.py | 15 ++++++++++++++-
 docs/CHANGELOG.md                        | 11 ++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/data/bathymetry/preprocess_bathymetry.py b/data/bathymetry/preprocess_bathymetry.py
index aeda133c1..bad573ad8 100644
--- a/data/bathymetry/preprocess_bathymetry.py
+++ b/data/bathymetry/preprocess_bathymetry.py
@@ -52,6 +52,7 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
         bathy_affine = bathy_ft.transform
         bathy_ft = bathy_ft.read(1)
         bathy_ft[np.where(bathy_ft == -9999.0)] = np.nan
+        bathy_ft[np.where(bathy_ft <= 0.0)] = 0.000001
     survey_min_depth = np.nanmin(bathy_ft)
 
     assert survey_min_depth < min_depth_threshold, (
@@ -81,6 +82,10 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
     zs_area = zs_area.set_crs(nwm_streams.crs)
     zs_area = zs_area.rename(columns={"sum": "missing_volume_m3"})
 
+    # print("------------------------------")
+    # print(zs_area.ID)
+    # print("------------------------------")
+
     # Derive slope tif
     output_slope_tif = os.path.join(os.path.dirname(tif), 'bathy_slope.tif')
     slope_tif = gdal.DEMProcessing(output_slope_tif, bathy_gdal, 'slope', format='GTiff')
@@ -115,7 +120,10 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
     )
 
     # Add survey meta data
-    bathy_nwm_streams['SurveyDateStamp'] = bathy_bounds.loc[0, 'SurveyDateStamp']
+    time_stamp = bathy_bounds.loc[0, 'SurveyDateStamp']
+    time_stamp_obj = str(time_stamp)
+
+    bathy_nwm_streams['SurveyDateStamp'] = time_stamp_obj  # bathy_bounds.loc[0, 'SurveyDateStamp']
     bathy_nwm_streams['SurveyId'] = bathy_bounds.loc[0, 'SurveyId']
     bathy_nwm_streams['Sheet_Name'] = bathy_bounds.loc[0, 'Sheet_Name']
     bathy_nwm_streams["Bathymetry_source"] = 'USACE eHydro'
@@ -123,6 +131,11 @@ def preprocessing_ehydro(tif, bathy_bounds, survey_gdb, output, min_depth_thresh
     # Export geopackage with bathymetry
     num_streams = len(bathy_nwm_streams)
     bathy_nwm_streams = bathy_nwm_streams.to_crs(epsg=5070)
+
+    # schema = gpd.io.file.infer_schema(bathy_nwm_streams)
+    # print(schema)
+    # print("---------------------------")
+
     if os.path.exists(output):
         print(f"{output} already exists. Concatinating now...")
         existing_bathy_file = gpd.read_file(output, engine="pyogrio", use_arrow=True)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 31ecd3ce7..30a351c33 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,11 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.4.2 - 2024-08-02 - [PR#1125](https://github.com/NOAA-OWP/inundation-mapping/pull/1125)
+
+This PR focuses on updating the preprocess_bathymetry.py for 3 issues: 1) the capability of preprocessing SurveyJobs that have negative depth values, 2) changing the SurveyDateStamp format, and 3) the capability of including multiple SurveyJobs for one NWM feature-id if needed.
+
+### Changes
+`data/bathymetry/preprocess_bathymetry.py`: Addressing 3 issues including, the capability of preprocessing SurveyJobs that have negative depth values, changing the SurveyDateStamp format, and the capability of including multiple SurveyJobs for one NWM feature-id.
+
+
+<br/><br/>
+
 ## v4.5.4.1 - 2024-08-02 - [PR#1185](https://github.com/NOAA-OWP/inundation-mapping/pull/1185)
 
 This PR brings back the `preprocess_ahps_nws.py` code to FIM4 and generates new AHPS benchmark datasets for sites SXRA2 and SKLA2 in Alaska.  The new AHPS benchmark datasets are available on dev1 here: "/dev_fim_share/foss_fim/outputs/ali_ahps_alaska/AHPS_Results_Alaska/19020302/"
 
-This PR closes issue #1130. 
 
 To process a new station, follow these steps:
 

From b47e4fafb94b6e6d9f66ff5256cdf68f34faf613 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 2 Aug 2024 19:33:19 +0000
Subject: [PATCH 340/359] v4.5.4.3 Unmask leveed areas update (#1136)

---
 docs/CHANGELOG.md                       | 12 ++++++++++++
 src/associate_levelpaths_with_levees.py | 17 +++++++++++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 30a351c33..2b86bedf0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.5.4.3 - 2024-08-02 - [PR#1136](https://github.com/NOAA-OWP/inundation-mapping/pull/1136)
+
+Levee-protected areas are associated with levelpaths based on a 1000 m buffer on each side of the levee line. However, not all levees are designed to protect against all associated levelpaths, especially where the levelpath flows through the levee-protected area. Levee-protected areas are unmasked by removing levelpaths from association that don't intersect levees but instead flow around them which allows inundation by these branches.
+
+### Changes
+
+- `src/associate_levelpaths_with_levees.py`: Finds levelpaths that don't intersect levees and removes them from their association with their levee-protected area.
+
+<br/><br/>
+
+
 ## v4.5.4.2 - 2024-08-02 - [PR#1125](https://github.com/NOAA-OWP/inundation-mapping/pull/1125)
 
 This PR focuses on updating the preprocess_bathymetry.py for 3 issues: 1) the capability of preprocessing SurveyJobs that have negative depth values, 2) changing the SurveyDateStamp format, and 3) the capability of including multiple SurveyJobs for one NWM feature-id if needed.
diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py
index 29c069ba4..9f07aa37a 100644
--- a/src/associate_levelpaths_with_levees.py
+++ b/src/associate_levelpaths_with_levees.py
@@ -22,7 +22,7 @@ def associate_levelpaths_with_levees(
     out_filename: str,
 ):
     """
-    Finds the level path associated with each levee. Ignores level paths that cross a levee exactly once.
+    Finds the levelpath(s) associated with each levee. Ignores levelpaths that cross a levee exactly once.
 
     Parameters
     ----------
@@ -177,7 +177,6 @@ def associate_levelpaths_with_levees(
                 .reset_index(drop=True)
             )
 
-        # Remove levelpaths that cross the levee exactly once
         for j, row in out_df.iterrows():
             # Intersect levees and levelpaths
             row_intersections = gpd.overlay(
@@ -193,9 +192,23 @@ def associate_levelpaths_with_levees(
             # Select Point geometry type
             row_intersections = row_intersections[row_intersections.geom_type == 'Point']
 
+            # Remove levelpaths that cross the levee exactly once
             if len(row_intersections) == 1:
                 out_df = out_df.drop(j)
 
+            # Find associated levelpaths that don't intersect levees
+            elif row_intersections.empty:
+                # Get levelpaths that intersect leveed areas
+                leveed_area_levelpaths = gpd.overlay(
+                    levelpaths[levelpaths[branch_id_attribute] == row[branch_id_attribute]],
+                    leveed_areas[leveed_areas[levee_id_attribute] == row[levee_id_attribute]],
+                    how='intersection',
+                    keep_geom_type=False,
+                )
+
+                if not leveed_area_levelpaths.empty:
+                    out_df = out_df.drop(j)
+
         out_df.to_csv(out_filename, columns=[levee_id_attribute, branch_id_attribute], index=False)
 
 

From d99e7a97ed513f4ea8592d497301f1e485699bfd Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 2 Aug 2024 19:38:38 +0000
Subject: [PATCH 341/359] v4.5.4.4 Change Post proc perms updates (#1238)

---
 docs/CHANGELOG.md                | 19 +++++++++++++++++++
 fim_pipeline.sh                  |  5 ++---
 fim_post_processing.sh           | 17 ++++++++---------
 src/bash_functions.env           | 21 ++++++++++++++++-----
 src/run_by_branch.sh             |  2 +-
 src/run_unit_wb.sh               |  6 +++---
 src/src_adjust_ras2fim_rating.py |  3 ++-
 tools/hash_compare.py            |  4 ++++
 8 files changed, 55 insertions(+), 22 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2b86bedf0..537a7d2ee 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.5.4.4 - 2024-08-02 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
+
+Prior to this fix, fim_post_processing.sh took just under 4 hours to reset permissions on all files and folder under the entire run. On closer inspection, it was updating permissions for all HUC folders where were already correct. A few other folders needed to have permission updates added. This will speed that up significantly.
+
+Also, use this opportunity to added a new note to hash_compare.py and fix an annoying duration time showing milliseconds.
+
+### Changes
+- `fim_pipeline.sh`: fix duration msgs.
+- `fim_post_processing.sh`:  permissions reset fix, a bit of output cleanup and fix duration msgs.
+- `src`
+    - `bash_functions.env`: update the Calc duration to allow for a msg prefix to be added to the duration calcs. Also adjusted the duration message to show hours as well, previously only min and seconds.
+    - `run_by_branch.sh`: fix duration msgs.
+    - `run_unit_wb.sh`: fix duration msgs.
+    - `src\src_adjust_ras2fim_rating.py`: minor duration display msg change.
+- `tools\hash_compare.py`: Added note
+
+ <br/><br/>
+
+
 ## v4.5.4.3 - 2024-08-02 - [PR#1136](https://github.com/NOAA-OWP/inundation-mapping/pull/1136)
 
 Levee-protected areas are associated with levelpaths based on a 1000 m buffer on each side of the levee line. However, not all levees are designed to protect against all associated levelpaths, especially where the levelpath flows through the levee-protected area. Levee-protected areas are unmasked by removing levelpaths from association that don't intersect levees but instead flow around them which allows inundation by these branches.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 16139fdc7..3934c27a7 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -115,7 +115,7 @@ fi
 echo
 echo "---- Unit (HUC) processing is complete"
 date -u
-Calc_Duration $pipeline_start_time
+Calc_Duration "Duration : " $pipeline_start_time
 echo "---------------------------------------------------"
 
 ## POST PROCESSING
@@ -130,8 +130,7 @@ echo
 
 echo "======================== End of fim_pipeline for $runName =========="
 date -u
-echo "Total Duration is ..."
-Calc_Duration $pipeline_start_time
+Calc_Duration "Total Duration is ... " $pipeline_start_time
 echo
 
 # Exit the script
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index e17850cb9..6a58c075c 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -92,6 +92,7 @@ l_echo ""
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 l_echo "---- Start of fim_post_processing"
 l_echo "---- Started: `date -u`"
+echo ""
 T_total_start
 post_proc_start_time=`date +%s`
 
@@ -134,10 +135,10 @@ if [ "$COUNTER" -gt 1 ]; then
     # Execute the Python file
     l_echo "Updating hydroTable & scr_full_crosswalked for branches"
     python3 $srcDir/update_htable_src.py -d $outputDestDir
+    Tcount
 else
     l_echo "Execution count is $COUNTER, not executing the update_htable_src.py file."
 fi
-Tcount
 
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
@@ -207,7 +208,6 @@ if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$s
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ RAS2FIM CROSS SECTION RATING CURVES ##
@@ -221,7 +221,6 @@ if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINTS (.parquet files) ##
@@ -230,11 +229,11 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [
     l_echo $startDiv"Performing SRC adjustments using benchmark point .parquet files"
     python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputDestDir -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## AGGREGATE BRANCH TABLES ##
 l_echo $startDiv"Aggregating branch hydrotables"
+
 Tstart
 python3 $srcDir/aggregate_by_huc.py \
     -fim $outputDestDir \
@@ -243,7 +242,7 @@ python3 $srcDir/aggregate_by_huc.py \
     -bridge \
     -j $jobLimit
 Tcount
-date -u
+
 
 ## PERFORM MANUAL CALIBRATION
 if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
@@ -253,7 +252,6 @@ if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
         -fim_dir $outputDestDir \
         -calb_file $man_calb_file
     Tcount
-    date -u
 fi
 
 
@@ -263,12 +261,13 @@ python3 $toolsDir/combine_crosswalk_tables.py \
     -d $outputDestDir \
     -o $outputDestDir/crosswalk_table.csv
 Tcount
-date -u
 
 
 l_echo $startDiv"Resetting Permissions"
 Tstart
-    find $outputDestDir -type d -exec chmod -R 777 {} +
+    find $outputDestDir/logs/ -type d -exec chmod -R 777 {} +
+    find $outputDestDir/branch_errors/ -type d -exec chmod -R 777 {} +
+    find $outputDestDir/unit_errors/ -type d -exec chmod -R 777 {} +
     find $outputDestDir -type f -exec chmod 777 {} +  # just root level files
 Tcount
 
@@ -290,5 +289,5 @@ echo
 l_echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 l_echo "---- End of fim_post_processing"
 l_echo "---- Ended: `date -u`"
-Calc_Duration $post_proc_start_time
+Calc_Duration "Post Processing Duration:" $post_proc_start_time
 echo
diff --git a/src/bash_functions.env b/src/bash_functions.env
index e8452003a..961a46285 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -68,15 +68,26 @@ Tcount () {
 }
 
 # This expects a start time submitted as an arg and assumes the end time as now
-Calc_Duration() {
-    local start_time=$1
+Calc_Duration( ) {
+    local duration_msg_prefix=$1
+    local start_time=$2
     local end_time=`date +%s`
 
     local total_sec=$(( $end_time - $start_time ))
-    local dur_min=$((total_sec / 60))
-    local dur_remainder_sec=$((total_sec % 60))
+    local dur_hrs=$((total_sec / 3600 ))
+    local dur_min=$((total_sec %3600 / 60))
+    local dur_remainder_sec=$((total_sec %60))
+
+    if [[ "$dur_hrs" = "0" ]]; then
+        dur_hrs=""
+    else
+        dur_hrs="${dur_hrs} hrs"
+    fi
+
+#    local msg="${duration_msg_prefix} $dur_min min(s) and $dur_remainder_sec sec"
+    local msg="${duration_msg_prefix} ${dur_hrs} $dur_min min(s) and $dur_remainder_sec seconds"
+
 
-    local msg="Duration : $dur_min min(s) and $dur_remainder_sec sec"
     # if an arg comes in and is a file path, we can display and record to a file
     if [[ $log_file_path != "" ]]; then
         echo -e $msg ; echo -e $msg >> $log_file_path
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index 69151a74c..28977c2e0 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -128,5 +128,5 @@ fi
 
 echo -e $startDiv"End Branch Processing $hucNumber $current_branch_id ..."
 date -u
-Calc_Duration $branch_start_time
+Calc_Duration "Duration : " $branch_start_time
 echo
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 3c7a1b460..60c7a3812 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -305,8 +305,8 @@ if [ -f $deny_unit_list ]; then
 fi
 
 echo "---- HUC $hucNumber - branches have now been processed"
-Calc_Duration $branch_processing_start_time
-echo
+Calc_Duration "Duration for processing branches : " $branch_processing_start_time
+#echo
 
 # WRITE TO LOG FILE CONTAINING ALL HUC PROCESSING TIMES
 total_duration_display="$hucNumber,$(Calc_Time $huc_start_time),$(Calc_Time_Minutes_in_Percent $huc_start_time)"
@@ -314,5 +314,5 @@ echo "$total_duration_display" >> "$outputDestDir/logs/unit/total_duration_run_b
 
 date -u
 echo "---- HUC processing for $hucNumber is complete"
-Calc_Duration $huc_start_time
+Calc_Duration "Duration for huc processing: " $huc_start_time
 echo
diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
index cbeb301c0..7f1c20d8a 100644
--- a/src/src_adjust_ras2fim_rating.py
+++ b/src/src_adjust_ras2fim_rating.py
@@ -57,7 +57,8 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
     )  # , nrows=30000)
     ras_rc_df.rename(columns={'fid_xs': 'location_id'}, inplace=True)
     # ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
-    print('Duration (read ras_rc_csv): {}'.format(dt.datetime.now() - start_time))
+    run_time = dt.datetime.now() - start_time
+    print(f"Duration (read ras_rc_csv): {str(run_time).split('.')[0]}")
 
     # convert WSE navd88 values to meters
     ras_rc_df.rename(
diff --git a/tools/hash_compare.py b/tools/hash_compare.py
index 33c2dfad1..be23ade07 100755
--- a/tools/hash_compare.py
+++ b/tools/hash_compare.py
@@ -20,6 +20,10 @@ def main(arg1, arg2, image_only, log_file, gpkg):
     Please be advised that this feature has not been thoroughly tested to ensure robustness.
 
     When arg1 and arg2 are single files, the files names need not match but the extensions must match.
+
+    Note: If any file has a date/time field, it will likely never be successful on a hash compare. This
+    is seen often in gpkgs.
+
     """
 
     if log_file is not None:

From c20e8974fc45ce443b7acb638c002b3ad2902e88 Mon Sep 17 00:00:00 2001
From: RyanSpies-NOAA <ryan.spies@noaa.gov>
Date: Fri, 16 Aug 2024 14:06:33 -0500
Subject: [PATCH 342/359] v4.5.5.0 Update SRC optimization w/ ras2fim v2 inputs
 (#1247)

---
 docs/CHANGELOG.md                | 12 ++++
 fim_post_processing.sh           |  3 +-
 src/bash_variables.env           |  6 +-
 src/run_unit_wb.sh               | 20 ++++++-
 src/src_adjust_ras2fim_rating.py | 98 +++++++++++++++++++++-----------
 src/usgs_gage_unit_setup.py      | 39 +++++++------
 src/utils/shared_functions.py    | 24 ++++++++
 7 files changed, 144 insertions(+), 58 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 537a7d2ee..e385208b5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.5.0 - 2024-08-16 - [PR#1247](https://github.com/NOAA-OWP/inundation-mapping/pull/1247)
+
+Updated the gauge crosswalk and SRC adjustment routine to use the ras2fim v2 files. The v2 ras2fim file structure was changed to organize the data by huc8 - one gpkg and csv per huc8. Addresses #1091 
+
+### Changes
+- `fim_post_processing.sh`: added new input variables for running the `src_adjust_ras2fim_rating.py`
+- `src/bash_variables.env`: renamed and reassigned the ras2fim input variables: `ras2fim_input_dir`, `ras_rating_curve_csv_filename`, `ras_rating_curve_gpkg_filename`
+- `src/run_unit_wb.sh`: Added logic to check if huc in process has ras2fim input data to process. If yes - copy the ras2fim cross section point gpkg to the huc run directory.
+- `src/src_adjust_ras2fim_rating.py`: Updated code logic to use the huc-specific input files containing the ras2fim rating curve data (previous ras2fim input file contained all hucs in one csv)
+- `src/utils/shared_functions.py`: Added function to find huc subdirectories with the same name btw two parent folders
+
+ <br/><br/>
 
 ## v4.5.4.4 - 2024-08-02 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
 
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 6a58c075c..7dd2a7608 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -217,7 +217,8 @@ if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [
     # Run SRC Optimization routine using ras2fim rating curve data (WSE and flow @ NWM recur flow values)
     python3 $srcDir/src_adjust_ras2fim_rating.py \
         -run_dir $outputDestDir \
-        -ras_rc $ras_rating_curve_csv \
+        -ras_input $ras2fim_input_dir \
+        -ras_rc $ras_rating_curve_csv_filename \
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 637b24eac..b1318f4e5 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -47,9 +47,9 @@ export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flo
 export             usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
 
 # input file locations for ras2fim locations and rating curve data
-export              ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
-export      ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
-export    ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points_rel_101.gpkg
+export                  ras2fim_input_dir=${inputsDir}/rating_curve/ras2fim_exports/v2_0
+export      ras_rating_curve_csv_filename=reformat_ras_rating_curve_table.csv
+export     ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points.gpkg
 
 export                  fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
 
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 60c7a3812..c8a8766a1 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -48,9 +48,25 @@ cp -a $pre_clip_huc_dir/$hucNumber/. $tempHucDataDir
 # For buffer_stream_branches.py
 cp $huc_input_DEM_domain $tempHucDataDir
 # For usgs_gage_unit_setup.py
-cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir
-cp $ras_rating_curve_points_gpkg $tempHucDataDir
 cp $inputsDir/ahps_sites/nws_lid.gpkg $tempHucDataDir
+cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir
+# Check if the $hucNumber directory exists in the ras2fim $inputsDir
+if [ -d "$ras2fim_input_dir/$hucNumber" ]; then
+    ras_rating_gpkg="$ras2fim_input_dir/$hucNumber/$ras_rating_curve_gpkg_filename"
+    ras_rating_csv="$ras2fim_input_dir/$hucNumber/$ras_rating_curve_csv_filename"
+    if [ -f "$ras_rating_gpkg" ]; then
+        cp "$ras_rating_gpkg" $tempHucDataDir
+        echo "Copied $ras_rating_gpkg to $tempHucDataDir"
+    else
+        echo "File $ras_rating_gpkg does not exist. Skipping copy."
+    fi
+    if [ -f "$ras_rating_csv" ]; then
+        cp "$ras_rating_csv" $tempHucDataDir
+        echo "Copied $ras_rating_csv to $tempHucDataDir"
+    else
+        echo "File $ras_rating_csv does not exist. Skipping copy."
+    fi
+fi
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"
diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
index 7f1c20d8a..ec35c8618 100644
--- a/src/src_adjust_ras2fim_rating.py
+++ b/src/src_adjust_ras2fim_rating.py
@@ -13,7 +13,7 @@
 import pandas as pd
 
 from src_roughness_optimization import update_rating_curve
-from utils.shared_functions import check_file_age, concat_huc_csv
+from utils.shared_functions import check_file_age, concat_huc_csv, find_matching_subdirectories
 
 
 '''
@@ -47,23 +47,26 @@
 '''
 
 
-def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir):
+def create_ras2fim_rating_database(huc_ras_input_file, ras_elev_df, nwm_recurr_filepath, log_dir):
     start_time = dt.datetime.now()
     print('Reading RAS2FIM rating curves from csv...')
     log_text = 'Processing database for RAS2FIM flow/WSE at NWM flow recur intervals...\n'
-    col_filter = ["fid_xs", "flow", "wse"]
+    # Note that we are using flow_cfs not flow_cms (error in raw data)
+    col_filter = ["fid_xs", "flow_cfs", "wse_m"]
     ras_rc_df = pd.read_csv(
-        ras_rc_filepath, dtype={'fid_xs': object}, usecols=col_filter, encoding="unicode_escape"
+        huc_ras_input_file, dtype={'fid_xs': object}, usecols=col_filter, encoding="unicode_escape"
     )  # , nrows=30000)
     ras_rc_df.rename(columns={'fid_xs': 'location_id'}, inplace=True)
     # ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
     run_time = dt.datetime.now() - start_time
     print(f"Duration (read ras_rc_csv): {str(run_time).split('.')[0]}")
 
-    # convert WSE navd88 values to meters
-    ras_rc_df.rename(
-        columns={'wse': 'wse_navd88_m', 'flow': 'discharge_cms'}, inplace=True
-    )  # assume ras2fim elevation data in feet
+    # rename WSE column
+    ras_rc_df.rename(columns={'wse_m': 'wse_navd88_m'}, inplace=True)
+
+    # Need to use the flow_cfs because there is an error in the raw flow_cms
+    ras_rc_df['discharge_cms'] = ras_rc_df['flow_cfs'] * 0.0283168
+    ras_rc_df = ras_rc_df.drop(columns=["flow_cfs"])
 
     # read in the aggregate RAS elev table csv
     start_time = dt.datetime.now()
@@ -196,10 +199,10 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
     return final_df
 
 
-def branch_proc_list(ras_df, run_dir, debug_outputs_option, log_file):
+def branch_proc_list(ras_df, huc_run_dir, debug_outputs_option, log_file):
     procs_list = []  # Initialize list for mulitprocessing.
 
-    # loop through all unique level paths that have a USGS gage
+    # loop through all unique level paths that have a ras2fim data points
     huc_branch_dict = ras_df.groupby('huc')['levpa_id'].apply(set).to_dict()
 
     for huc in sorted(
@@ -210,7 +213,7 @@ def branch_proc_list(ras_df, run_dir, debug_outputs_option, log_file):
             # Define paths to branch HAND data.
             # Define paths to HAND raster, catchments raster, and synthetic rating curve JSON.
             # Assumes outputs are for HUC8 (not HUC6)
-            branch_dir = os.path.join(run_dir, huc, 'branches', branch_id)
+            branch_dir = os.path.join(huc_run_dir, 'branches', branch_id)
             hand_path = os.path.join(branch_dir, 'rem_zeroed_masked_' + branch_id + '.tif')
             catchments_path = os.path.join(
                 branch_dir, 'gw_catchments_reaches_filtered_addedAttributes_' + branch_id + '.tif'
@@ -302,14 +305,10 @@ def branch_proc_list(ras_df, run_dir, debug_outputs_option, log_file):
     #     )
 
 
-def run_prep(run_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number):
+def run_prep(run_dir, ras_input_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number):
     ## Check input args are valid
     assert os.path.isdir(run_dir), 'ERROR: could not find the input fim_dir location: ' + str(run_dir)
 
-    ## Create an aggregate dataframe with all ras_elev_table.csv entries for hucs in fim_dir
-    print('Reading RAS2FIM point loc HAND elevation from ras_elev_table csv files...')
-    csv_name = 'ras_elev_table.csv'  # file name to search for ras location data (in the huc/branch dirs)
-
     available_cores = multiprocessing.cpu_count()
     if job_number > available_cores:
         job_number = available_cores - 1
@@ -332,25 +331,52 @@ def run_prep(run_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option
     log_file.write('START TIME: ' + str(begin_time) + '\n')
     log_file.write('#########################################################\n\n')
 
-    ras_elev_df = concat_huc_csv(run_dir, csv_name)
-
-    if ras_elev_df is None:
-        warn_err = 'WARNING: ras_elev_df not created - check that ' + csv_name + ' files exist in fim_dir!'
-        print(warn_err)
-        log_file.write(warn_err)
+    hucs_with_data = find_matching_subdirectories(run_dir, ras_input_dir)
+    if len(hucs_with_data) == 0:
+        print('ALERT: Did not find any HUCs with ras2fim data to perform adjustments')
+        log_file.write('ALERT: Did not find any HUCs with ras2fim data to perform adjustments\n')
+        return
+
+    log_file.write('RAS2FIM data available and will perform SRC adjustments for hucs:\n')
+    log_file.write(str(hucs_with_data))
+    log_file.write('\n#########################################################\n\n')
+    for huc in hucs_with_data:
+        huc_run_dir = os.path.join(run_dir, huc)
+        huc_ras_input_file = os.path.join(huc_run_dir, ras_rc_filepath)
+        ## Create an aggregate dataframe with all ras_elev_table.csv entries for hucs in fim_dir
+        print('Reading RAS2FIM point loc HAND elevation from ras_elev_table csv files...')
+        csv_elev = 'ras_elev_table.csv'  # file name to search for ras location data (in the huc/branch dirs)
+        # ras_elev_df = concat_huc_csv(huc_run_dir, csv_elev)
+        ras_elev_df = pd.read_csv(
+            os.path.join(huc_run_dir, 'ras_elev_table.csv'),
+            dtype={'HUC8': object, 'location_id': object, 'feature_id': int, 'levpa_id': object},
+        )
 
-    elif ras_elev_df.empty:
-        warn_err = 'WARNING: ras_elev_df is empty - check that ' + csv_name + ' files exist in fim_dir!'
-        print(warn_err)
-        log_file.write(warn_err)
+        ## Create an aggregate dataframe with all ras2fim rating curve csv files
+        # print('Reading RAS2FIM rating curves csv files from the input directory...')
+        # ras_rating_df = concat_huc_csv(ras_input_dir, ras_rc_filepath)
 
-    else:
-        print('This may take a few minutes...')
-        log_file.write("starting create RAS2FIM rating db")
-        ras_df = create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir)
+        if ras_elev_df is None:
+            warn_err = (
+                'WARNING: ras_elev_df not created - check that ' + csv_elev + ' files exist in fim_dir!'
+            )
+            print(warn_err)
+            log_file.write(warn_err)
+
+        elif ras_elev_df.empty:
+            warn_err = 'WARNING: ras_elev_df is empty - check that ' + csv_elev + ' files exist in fim_dir!'
+            print(warn_err)
+            log_file.write(warn_err)
+
+        else:
+            print('This may take a few minutes...')
+            log_file.write("starting create RAS2FIM rating db")
+            ras_df = create_ras2fim_rating_database(
+                huc_ras_input_file, ras_elev_df, nwm_recurr_filepath, log_dir
+            )
 
-        ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
-        branch_proc_list(ras_df, run_dir, debug_outputs_option, log_file)
+            ## Create huc proc_list for multiprocessing and execute the update_rating_curve function
+            branch_proc_list(ras_df, huc_run_dir, debug_outputs_option, log_file)
 
     ## Record run time and close log file
     log_file.write('#########################################################\n\n')
@@ -369,10 +395,13 @@ def run_prep(run_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option
         '(calculated WSE/flow).'
     )
     parser.add_argument('-run_dir', '--run-dir', help='Parent directory of FIM run.', required=True)
+    parser.add_argument(
+        '-ras_input', '--ras2fim-dir', help='Path to RAS2FIM rating curve input directory', required=True
+    )
     parser.add_argument(
         '-ras_rc',
         '--ras2fim-ratings',
-        help='Path to RAS2FIM rating curve (reach avg) csv file',
+        help='CSV file name for RAS2FIM rating curve (reach avg)',
         required=True,
     )
     parser.add_argument(
@@ -394,10 +423,11 @@ def run_prep(run_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option
     ## Assign variables from arguments.
     args = vars(parser.parse_args())
     run_dir = args['run_dir']
+    ras_input_dir = args['ras2fim_dir']
     ras_rc_filepath = args['ras2fim_ratings']
     nwm_recurr_filepath = args['nwm_recur']
     debug_outputs_option = args['extra_outputs']
     job_number = int(args['job_number'])
 
     ## Prepare/check inputs, create log file, and spin up the proc list
-    run_prep(run_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number)
+    run_prep(run_dir, ras_input_dir, ras_rc_filepath, nwm_recurr_filepath, debug_outputs_option, job_number)
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 308952b3a..464fee471 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -35,24 +35,27 @@ def load_gages(self, huc_CRS):
         # Read RAS2FIM point locations file
         # !!! Geopandas is not honoring the dtype arg with this read_file below (huc8 being read as int64).
         # Need the raw data to store the 'huc8' attribute as an object to avoid issues with integers truncating the leading zero from some hucs
-        ras_locs = gpd.read_file(self.ras_locs_filename, dtype={'huc8': 'object'})
-        ras_locs = ras_locs[['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'geometry']]
-        ras_locs['location_id'] = ras_locs['fid_xs']
-
-        # Convert ras locs crs to match usgs gage crs
-        ras_locs.to_crs(huc_CRS, inplace=True)
-        ras_locs = ras_locs.rename(columns={'huc8': 'HUC8'})
-
-        # Convert Multipoint geometry to Point geometry
-        ras_locs['geometry'] = ras_locs.representative_point()
-
-        # if ras_locs.huc8.dtype == 'int64':
-        #     ras_locs = ras_locs[ras_locs.huc8 == int(self.huc8)]
-        #     ras_locs['HUC8'] = str(self.huc8)
-        #     ras_locs = ras_locs.drop('huc8', axis=1)
-        # elif ras_locs.huc8.dtype == 'int64':
-        #     ras_locs = ras_locs.rename(columns={'huc8':'HUC8'})
-
+        if os.path.exists(self.ras_locs_filename):
+            ras_columns = ['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'geometry']
+            ras_locs = gpd.read_file(self.ras_locs_filename, dtype={'huc8': 'object'}, usecols=ras_columns)
+            ras_locs = ras_locs[ras_columns]
+            ras_locs['location_id'] = ras_locs['fid_xs']
+
+            # Convert ras locs crs to match usgs gage crs
+            ras_locs.to_crs(huc_CRS, inplace=True)
+            ras_locs = ras_locs.rename(columns={'huc8': 'HUC8'})
+
+            # Convert Multipoint geometry to Point geometry
+            ras_locs['geometry'] = ras_locs.representative_point()
+
+            # if ras_locs.huc8.dtype == 'int64':
+            #     ras_locs = ras_locs[ras_locs.huc8 == int(self.huc8)]
+            #     ras_locs['HUC8'] = str(self.huc8)
+            #     ras_locs = ras_locs.drop('huc8', axis=1)
+            # elif ras_locs.huc8.dtype == 'int64':
+            #     ras_locs = ras_locs.rename(columns={'huc8':'HUC8'})
+        else:
+            ras_locs = pd.DataFrame(columns=['feature_id', 'stream_stn', 'fid_xs', 'source', 'geometry'])
         # Concat USGS points and RAS2FIM points
         gages_locs = pd.concat([usgs_gages, ras_locs], axis=0, ignore_index=True)
         # gages_locs.to_crs(PREP_CRS, inplace=True)
diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py
index b30f515e3..f9f51a539 100644
--- a/src/utils/shared_functions.py
+++ b/src/utils/shared_functions.py
@@ -174,6 +174,30 @@ def check_file_age(file):
         return modified_date
 
 
+########################################################################
+# Function to find huc subdirectories with the same name btw two parent folders
+########################################################################
+def find_matching_subdirectories(parent_folder1, parent_folder2):
+    # List all subdirectories in the first parent folder
+    subdirs1 = {
+        d
+        for d in os.listdir(parent_folder1)
+        if os.path.isdir(os.path.join(parent_folder1, d)) and len(d) == 8
+    }
+
+    # List all subdirectories in the second parent folder
+    subdirs2 = {
+        d
+        for d in os.listdir(parent_folder2)
+        if os.path.isdir(os.path.join(parent_folder2, d)) and len(d) == 8
+    }
+
+    # Find common subdirectories with exactly 8 characters
+    matching_subdirs = list(subdirs1 & subdirs2)
+
+    return matching_subdirs
+
+
 ########################################################################
 # Function to concatenate huc csv files to a single dataframe/csv
 ########################################################################

From b282955b20ffeb426a1613bb33cddd1f566ee475 Mon Sep 17 00:00:00 2001
From: ZahraGhahremani-NOAA
 <100253864+ZahraGhahremani@users.noreply.github.com>
Date: Fri, 16 Aug 2024 13:13:16 -0600
Subject: [PATCH 343/359] v4.5.5.1 Fix `heal_bridges_osm.py` warning (#1225)

---
 docs/CHANGELOG.md       | 11 +++++++++++
 src/heal_bridges_osm.py | 12 ++++++++----
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e385208b5..0dcd9742f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.5.1 - 2024-08-16 - [PR#1225](https://github.com/NOAA-OWP/inundation-mapping/pull/1225)
+
+Removes warning when running `heal_bridges_osm.py` by not saving the empty DataFrame.
+
+### Changes
+
+- `src/heal_bridges_osm.py`
+
+<br/><br/>
+
+
 ## v4.5.5.0 - 2024-08-16 - [PR#1247](https://github.com/NOAA-OWP/inundation-mapping/pull/1247)
 
 Updated the gauge crosswalk and SRC adjustment routine to use the ras2fim v2 files. The v2 ras2fim file structure was changed to organize the data by huc8 - one gpkg and csv per huc8. Addresses #1091 
diff --git a/src/heal_bridges_osm.py b/src/heal_bridges_osm.py
index 19ade89cf..4478ad58f 100644
--- a/src/heal_bridges_osm.py
+++ b/src/heal_bridges_osm.py
@@ -37,7 +37,7 @@ def process_bridges_in_huc(
 
         # Get max hand values for each bridge
         osm_gdf['max_hand'] = zonal_stats(
-            osm_gdf['geometry'], hand_grid_array, affine=hand_grid.transform, stats="max"
+            osm_gdf['geometry'], hand_grid_array, affine=hand_grid.transform, stats="max", nodata=-999
         )
         # pull the values out of the geopandas columns so we can use them as floats
         osm_gdf['max_hand'] = [x.get('max') for x in osm_gdf.max_hand]
@@ -71,9 +71,12 @@ def process_bridges_in_huc(
     osm_gdf['branch'] = branch_id
     osm_gdf['mainstem'] = 0 if branch_id == '0' else 1
 
-    # Write the bridge points to a geopackage
-    osm_gdf.to_file(bridge_centroids, index=False)
-
+    # Check if the GeoDataFrame is empty
+    if not osm_gdf.empty:
+        # Write the bridge points to a geopackage
+        osm_gdf.to_file(bridge_centroids, index=False)
+    else:
+        print('The geoDataFrame is empty.File not saved.')
     return
 
 
@@ -100,6 +103,7 @@ def flows_from_hydrotable(bridge_pnts, hydroTable):
         python3 src/heal_bridges_osm.py
             -g /outputs/fim_4_4_15_0/1209301/branches/3763000013/rem_zeroed_masked_3763000013.tif
             -s /outputs/fim_4_4_15_0/1209301/osm_bridges_subset.gpkg
+            -p /outputs/fim_4_4_15_0/1209301/branches/3763000013/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3763000013.gpkg
             -c /outputs/fim_4_4_15_0/1209301/1209301/branches/3763000013/osm_bridge_centroids_3763000013.tif
             -b 10
             -r 10

From 6321468aa5b79bbf5c762967872bd52c5801f1f5 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 23 Aug 2024 13:25:48 -0600
Subject: [PATCH 344/359] v4.5.6.0 Upgrade Python packages (#1253)

---
 Dockerfile                             |    6 +-
 Pipfile                                |   76 +-
 Pipfile.lock                           | 2019 ++++++++++--------------
 docs/CHANGELOG.md                      |   15 +
 src/build_stream_traversal.py          |    4 +-
 src/derive_level_paths.py              |    5 +-
 src/heal_bridges_osm.py                |    4 +-
 src/mitigate_branch_outlet_backpool.py |    2 +-
 src/split_flows.py                     |    2 +-
 src/stream_branches.py                 |   26 +-
 src/usgs_gage_unit_setup.py            |    4 +-
 11 files changed, 939 insertions(+), 1224 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6b65469a6..a15575710 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 ## Temporary image to build the libraries and only save the needed artifacts
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3 AS builder
 WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
@@ -45,7 +45,7 @@ RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d
 
 ###############################################################################################
 # Base Image that has GDAL, PROJ, etc
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3
 ARG dataDir=/data
 ENV projectDir=/foss_fim
 ARG depDir=/dependencies
@@ -107,7 +107,7 @@ ENV PYTHONPATH=${PATH}:$srcDir:$projectDir/tools
 
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv==2023.12.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
+RUN pip3 install pipenv==2024.0.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
 
 # ----------------------------------
 # Mar 2023
diff --git a/Pipfile b/Pipfile
index 9290a1e29..dd0495222 100644
--- a/Pipfile
+++ b/Pipfile
@@ -7,53 +7,53 @@ verify_ssl = true
 ipython = "==8.24.0"
 
 [packages]
+black = "==24.3.0"
+boto3 = "1.35.1"
 certifi = "==2024.7.4"
-fiona = "==1.8.22"
-geopandas = "==0.14.3"
+dask = "==2024.6.2"
+dask-expr = "==1.1.6"
+distributed = "==2024.6.2"
+fiona = "==1.9.6"
+flake8 = "==6.0.0"
+flake8-pyproject = "==1.2.3"
+geopandas = "==1.0.1"
+gval = "==0.2.7.post1"
+ipympl = "==0.9.3"
+isort = "==5.12.0"
+jupyter = "==1.0.0"
+jupyterlab = "==3.6.7"
+lmoments3 = "==1.0.6"
+monaco = "==0.13.1"
+natsort = "==8.3.1"
+netcdf4 = "==1.6.3"
 numba = "==0.60.0"
 numpy = "==1.26.4"
+openpyxl = "==3.1.2"
+osmnx = "==1.9.3"
 pandas = "==2.0.2"
-rasterio = "==1.3.6"
-rasterstats = "==0.18.0"
-richdem = "==0.3.4"
-tqdm = "==4.66.3"
-seaborn = "==0.12.2"
-python-dotenv = "==1.0.0"
-natsort = "==8.3.1"
-xarray = "==2024.6.0"
-netcdf4 = "==1.6.3"
-tables = "==3.8.0"
-pyproj = "==3.5.0"
+pillow = "==10.3.0"
+pre-commit = "==3.3.3"
 psycopg2-binary = "==2.9.6"
-boto3 = "==1.26.109"
-jupyter = "==1.0.0"
-jupyterlab = "==3.6.7"
-ipympl = "==0.9.3"
-whitebox = "==2.3.4"
-shapely = "==2.0.1"
-pyarrow = "==14.0.1"
-rtree = "==1.0.1"
+pyarrow = "==17.0.0"
+pyflwdir = "==0.5.8"
+pyogrio = "==0.8.0"
+pyproj = "==3.6.1"
+python-dotenv = "==1.0.0"
 py7zr = "==0.20.4"
+rasterio = "==1.3.10"
+rasterstats = "==0.19.0"
+requests = "==2.32.3"
+richdem = "==0.3.4"
+rtree = "==1.0.1"
 scipy = "==1.14.0"
-gval = "==0.2.7"
-flake8 = "==6.0.0"
-black = "==24.3.0"
-flake8-pyproject = "==1.2.3"
-pre-commit = "==3.3.3"
-isort = "==5.12.0"
+seaborn = "==0.12.2"
+tables = "==3.8.0"
+tqdm = "==4.66.3"
 urllib3 = "==1.26.19"
-pyflwdir = "==0.5.8"
-pillow = "==10.3.0"
-pyogrio = "==0.7.2"
-openpyxl = "==3.1.2"
-osmnx = "==1.9.3"
-lmoments3 = "==1.0.6"
+whitebox = "==2.3.4"
+xarray = "==2024.6.0"
 zarr = "==2.18.0"
-requests = "==2.32.3"
-aiohttp = "==3.9.5"
-distributed = "==2024.6.2"
-monaco = "==0.13.1"
-dask = "==2024.6.2"
+
 
 [requires]
 python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
index ed11ccabf..23f15d493 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "75c63212784d57d42fc7b5536a4634408876142a2abc57317537c8e7ef4abab5"
+            "sha256": "69e8fc55d7092cbd650b00f89e5644b3ef68a4f2ef1a00ec62d5f8645222dae0"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -24,14 +24,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.4.0"
         },
-        "aiobotocore": {
-            "hashes": [
-                "sha256:337429ffd3cc367532572d40be809a84c7b5335f3f8eca2f23e09dfaa9a9ef90",
-                "sha256:e7399f21570db1c287f1c0c814dd3475dfe1c8166722e2c77ce67f172cbcfa89"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.5.2"
-        },
         "aiofiles": {
             "hashes": [
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
@@ -40,105 +32,6 @@
             "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
-        "aiohttp": {
-            "hashes": [
-                "sha256:0605cc2c0088fcaae79f01c913a38611ad09ba68ff482402d3410bf59039bfb8",
-                "sha256:0a158704edf0abcac8ac371fbb54044f3270bdbc93e254a82b6c82be1ef08f3c",
-                "sha256:0cbf56238f4bbf49dab8c2dc2e6b1b68502b1e88d335bea59b3f5b9f4c001475",
-                "sha256:1732102949ff6087589408d76cd6dea656b93c896b011ecafff418c9661dc4ed",
-                "sha256:18f634d540dd099c262e9f887c8bbacc959847cfe5da7a0e2e1cf3f14dbf2daf",
-                "sha256:239f975589a944eeb1bad26b8b140a59a3a320067fb3cd10b75c3092405a1372",
-                "sha256:2faa61a904b83142747fc6a6d7ad8fccff898c849123030f8e75d5d967fd4a81",
-                "sha256:320e8618eda64e19d11bdb3bd04ccc0a816c17eaecb7e4945d01deee2a22f95f",
-                "sha256:38d80498e2e169bc61418ff36170e0aad0cd268da8b38a17c4cf29d254a8b3f1",
-                "sha256:3916c8692dbd9d55c523374a3b8213e628424d19116ac4308e434dbf6d95bbdd",
-                "sha256:393c7aba2b55559ef7ab791c94b44f7482a07bf7640d17b341b79081f5e5cd1a",
-                "sha256:3b7b30258348082826d274504fbc7c849959f1989d86c29bc355107accec6cfb",
-                "sha256:3fcb4046d2904378e3aeea1df51f697b0467f2aac55d232c87ba162709478c46",
-                "sha256:4109adee842b90671f1b689901b948f347325045c15f46b39797ae1bf17019de",
-                "sha256:4558e5012ee03d2638c681e156461d37b7a113fe13970d438d95d10173d25f78",
-                "sha256:45731330e754f5811c314901cebdf19dd776a44b31927fa4b4dbecab9e457b0c",
-                "sha256:4715a9b778f4293b9f8ae7a0a7cef9829f02ff8d6277a39d7f40565c737d3771",
-                "sha256:471f0ef53ccedec9995287f02caf0c068732f026455f07db3f01a46e49d76bbb",
-                "sha256:4d3ebb9e1316ec74277d19c5f482f98cc65a73ccd5430540d6d11682cd857430",
-                "sha256:4ff550491f5492ab5ed3533e76b8567f4b37bd2995e780a1f46bca2024223233",
-                "sha256:52c27110f3862a1afbcb2af4281fc9fdc40327fa286c4625dfee247c3ba90156",
-                "sha256:55b39c8684a46e56ef8c8d24faf02de4a2b2ac60d26cee93bc595651ff545de9",
-                "sha256:5a7ee16aab26e76add4afc45e8f8206c95d1d75540f1039b84a03c3b3800dd59",
-                "sha256:5ca51eadbd67045396bc92a4345d1790b7301c14d1848feaac1d6a6c9289e888",
-                "sha256:5d6b3f1fabe465e819aed2c421a6743d8debbde79b6a8600739300630a01bf2c",
-                "sha256:60cdbd56f4cad9f69c35eaac0fbbdf1f77b0ff9456cebd4902f3dd1cf096464c",
-                "sha256:6380c039ec52866c06d69b5c7aad5478b24ed11696f0e72f6b807cfb261453da",
-                "sha256:639d0042b7670222f33b0028de6b4e2fad6451462ce7df2af8aee37dcac55424",
-                "sha256:66331d00fb28dc90aa606d9a54304af76b335ae204d1836f65797d6fe27f1ca2",
-                "sha256:67c3119f5ddc7261d47163ed86d760ddf0e625cd6246b4ed852e82159617b5fb",
-                "sha256:694d828b5c41255e54bc2dddb51a9f5150b4eefa9886e38b52605a05d96566e8",
-                "sha256:6ae79c1bc12c34082d92bf9422764f799aee4746fd7a392db46b7fd357d4a17a",
-                "sha256:702e2c7c187c1a498a4e2b03155d52658fdd6fda882d3d7fbb891a5cf108bb10",
-                "sha256:714d4e5231fed4ba2762ed489b4aec07b2b9953cf4ee31e9871caac895a839c0",
-                "sha256:7b179eea70833c8dee51ec42f3b4097bd6370892fa93f510f76762105568cf09",
-                "sha256:7f64cbd44443e80094309875d4f9c71d0401e966d191c3d469cde4642bc2e031",
-                "sha256:82a6a97d9771cb48ae16979c3a3a9a18b600a8505b1115cfe354dfb2054468b4",
-                "sha256:84dabd95154f43a2ea80deffec9cb44d2e301e38a0c9d331cc4aa0166fe28ae3",
-                "sha256:8676e8fd73141ded15ea586de0b7cda1542960a7b9ad89b2b06428e97125d4fa",
-                "sha256:88e311d98cc0bf45b62fc46c66753a83445f5ab20038bcc1b8a1cc05666f428a",
-                "sha256:8b4f72fbb66279624bfe83fd5eb6aea0022dad8eec62b71e7bf63ee1caadeafe",
-                "sha256:8c64a6dc3fe5db7b1b4d2b5cb84c4f677768bdc340611eca673afb7cf416ef5a",
-                "sha256:8cf142aa6c1a751fcb364158fd710b8a9be874b81889c2bd13aa8893197455e2",
-                "sha256:8d1964eb7617907c792ca00b341b5ec3e01ae8c280825deadbbd678447b127e1",
-                "sha256:93e22add827447d2e26d67c9ac0161756007f152fdc5210277d00a85f6c92323",
-                "sha256:9c69e77370cce2d6df5d12b4e12bdcca60c47ba13d1cbbc8645dd005a20b738b",
-                "sha256:9dbc053ac75ccc63dc3a3cc547b98c7258ec35a215a92bd9f983e0aac95d3d5b",
-                "sha256:9e3a1ae66e3d0c17cf65c08968a5ee3180c5a95920ec2731f53343fac9bad106",
-                "sha256:a6ea1a5b409a85477fd8e5ee6ad8f0e40bf2844c270955e09360418cfd09abac",
-                "sha256:a81b1143d42b66ffc40a441379387076243ef7b51019204fd3ec36b9f69e77d6",
-                "sha256:ad7f2919d7dac062f24d6f5fe95d401597fbb015a25771f85e692d043c9d7832",
-                "sha256:afc52b8d969eff14e069a710057d15ab9ac17cd4b6753042c407dcea0e40bf75",
-                "sha256:b3df71da99c98534be076196791adca8819761f0bf6e08e07fd7da25127150d6",
-                "sha256:c088c4d70d21f8ca5c0b8b5403fe84a7bc8e024161febdd4ef04575ef35d474d",
-                "sha256:c26959ca7b75ff768e2776d8055bf9582a6267e24556bb7f7bd29e677932be72",
-                "sha256:c413016880e03e69d166efb5a1a95d40f83d5a3a648d16486592c49ffb76d0db",
-                "sha256:c6021d296318cb6f9414b48e6a439a7f5d1f665464da507e8ff640848ee2a58a",
-                "sha256:c671dc117c2c21a1ca10c116cfcd6e3e44da7fcde37bf83b2be485ab377b25da",
-                "sha256:c7a4b7a6cf5b6eb11e109a9755fd4fda7d57395f8c575e166d363b9fc3ec4678",
-                "sha256:c8a02fbeca6f63cb1f0475c799679057fc9268b77075ab7cf3f1c600e81dd46b",
-                "sha256:cd2adf5c87ff6d8b277814a28a535b59e20bfea40a101db6b3bdca7e9926bc24",
-                "sha256:d1469f228cd9ffddd396d9948b8c9cd8022b6d1bf1e40c6f25b0fb90b4f893ed",
-                "sha256:d153f652a687a8e95ad367a86a61e8d53d528b0530ef382ec5aaf533140ed00f",
-                "sha256:d5ab8e1f6bee051a4bf6195e38a5c13e5e161cb7bad83d8854524798bd9fcd6e",
-                "sha256:da00da442a0e31f1c69d26d224e1efd3a1ca5bcbf210978a2ca7426dfcae9f58",
-                "sha256:da22dab31d7180f8c3ac7c7635f3bcd53808f374f6aa333fe0b0b9e14b01f91a",
-                "sha256:e0ae53e33ee7476dd3d1132f932eeb39bf6125083820049d06edcdca4381f342",
-                "sha256:e7a6a8354f1b62e15d48e04350f13e726fa08b62c3d7b8401c0a1314f02e3558",
-                "sha256:e9a3d838441bebcf5cf442700e3963f58b5c33f015341f9ea86dcd7d503c07e2",
-                "sha256:edea7d15772ceeb29db4aff55e482d4bcfb6ae160ce144f2682de02f6d693551",
-                "sha256:f22eb3a6c1080d862befa0a89c380b4dafce29dc6cd56083f630073d102eb595",
-                "sha256:f26383adb94da5e7fb388d441bf09c61e5e35f455a3217bfd790c6b6bc64b2ee",
-                "sha256:f3c2890ca8c59ee683fd09adf32321a40fe1cf164e3387799efb2acebf090c11",
-                "sha256:f64fd07515dad67f24b6ea4a66ae2876c01031de91c93075b8093f07c0a2d93d",
-                "sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7",
-                "sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f"
-            ],
-            "index": "pypi",
-            "markers": "python_version >= '3.8'",
-            "version": "==3.9.5"
-        },
-        "aioitertools": {
-            "hashes": [
-                "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394",
-                "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.11.0"
-        },
-        "aiosignal": {
-            "hashes": [
-                "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc",
-                "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.3.1"
-        },
         "aiosqlite": {
             "hashes": [
                 "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6",
@@ -218,29 +111,21 @@
             ],
             "version": "==2.4.1"
         },
-        "async-timeout": {
-            "hashes": [
-                "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f",
-                "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"
-            ],
-            "markers": "python_version < '3.11'",
-            "version": "==4.0.3"
-        },
         "attrs": {
             "hashes": [
-                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
-                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
+                "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346",
+                "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.2.0"
+            "version": "==24.2.0"
         },
         "babel": {
             "hashes": [
-                "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb",
-                "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"
+                "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b",
+                "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.15.0"
+            "version": "==2.16.0"
         },
         "beautifulsoup4": {
             "hashes": [
@@ -316,20 +201,20 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:a1c48674c2ff25ddced599c51d14ba7a6dbe9fd51641e8d63a6887bebd9712d7",
-                "sha256:d388cb7f54f1a3056f91ffcfb5cf18b226454204e5df7a5c10774718c3fbb166"
+                "sha256:9028dd814c7f06bc92822d1fabdd8b68fe39b5993da5ca7acd7f7c6f7c7c332f",
+                "sha256:9fe1a97de136a89d2957e41e3591a7648a6e08d63165255b4cc69c5ba1e2a177"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==1.26.109"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.35.1"
         },
         "botocore": {
             "hashes": [
-                "sha256:a50edd715eb510343e27849f36483804aae4b871590db4d4996aa53368dcac40",
-                "sha256:b906999dd53dda2ef0ef6f7f55fcc81a4b06b9f1c8a9f65c546e0b981f959f5f"
+                "sha256:bce42967d0f03b79cf25b2b6a36221fb2fb15f98e6fa4155b66b672ab192013b",
+                "sha256:e599cef6305e950a212f85adb86c1f4713d4b2678b3c2ec54df6b8e2dbe9ef2f"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.29.161"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.35.1"
         },
         "bottleneck": {
             "hashes": [
@@ -469,11 +354,11 @@
         },
         "cachetools": {
             "hashes": [
-                "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474",
-                "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"
+                "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292",
+                "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.0"
+            "version": "==5.5.0"
         },
         "certifi": {
             "hashes": [
@@ -486,61 +371,76 @@
         },
         "cffi": {
             "hashes": [
-                "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc",
-                "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a",
-                "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417",
-                "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab",
-                "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520",
-                "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36",
-                "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743",
-                "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8",
-                "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed",
-                "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684",
-                "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56",
-                "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324",
-                "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d",
-                "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235",
-                "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e",
-                "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088",
-                "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000",
-                "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7",
-                "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e",
-                "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673",
-                "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c",
-                "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe",
-                "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2",
-                "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098",
-                "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8",
-                "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a",
-                "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0",
-                "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b",
-                "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896",
-                "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e",
-                "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9",
-                "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2",
-                "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b",
-                "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6",
-                "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404",
-                "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f",
-                "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0",
-                "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4",
-                "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc",
-                "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936",
-                "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba",
-                "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872",
-                "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb",
-                "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614",
-                "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1",
-                "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d",
-                "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969",
-                "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b",
-                "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4",
-                "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627",
-                "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956",
-                "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"
+                "sha256:011aff3524d578a9412c8b3cfaa50f2c0bd78e03eb7af7aa5e0df59b158efb2f",
+                "sha256:0a048d4f6630113e54bb4b77e315e1ba32a5a31512c31a273807d0027a7e69ab",
+                "sha256:0bb15e7acf8ab35ca8b24b90af52c8b391690ef5c4aec3d31f38f0d37d2cc499",
+                "sha256:0d46ee4764b88b91f16661a8befc6bfb24806d885e27436fdc292ed7e6f6d058",
+                "sha256:0e60821d312f99d3e1569202518dddf10ae547e799d75aef3bca3a2d9e8ee693",
+                "sha256:0fdacad9e0d9fc23e519efd5ea24a70348305e8d7d85ecbb1a5fa66dc834e7fb",
+                "sha256:14b9cbc8f7ac98a739558eb86fabc283d4d564dafed50216e7f7ee62d0d25377",
+                "sha256:17c6d6d3260c7f2d94f657e6872591fe8733872a86ed1345bda872cfc8c74885",
+                "sha256:1a2ddbac59dc3716bc79f27906c010406155031a1c801410f1bafff17ea304d2",
+                "sha256:2404f3de742f47cb62d023f0ba7c5a916c9c653d5b368cc966382ae4e57da401",
+                "sha256:24658baf6224d8f280e827f0a50c46ad819ec8ba380a42448e24459daf809cf4",
+                "sha256:24aa705a5f5bd3a8bcfa4d123f03413de5d86e497435693b638cbffb7d5d8a1b",
+                "sha256:2770bb0d5e3cc0e31e7318db06efcbcdb7b31bcb1a70086d3177692a02256f59",
+                "sha256:331ad15c39c9fe9186ceaf87203a9ecf5ae0ba2538c9e898e3a6967e8ad3db6f",
+                "sha256:3aa9d43b02a0c681f0bfbc12d476d47b2b2b6a3f9287f11ee42989a268a1833c",
+                "sha256:41f4915e09218744d8bae14759f983e466ab69b178de38066f7579892ff2a555",
+                "sha256:4304d4416ff032ed50ad6bb87416d802e67139e31c0bde4628f36a47a3164bfa",
+                "sha256:435a22d00ec7d7ea533db494da8581b05977f9c37338c80bc86314bec2619424",
+                "sha256:45f7cd36186db767d803b1473b3c659d57a23b5fa491ad83c6d40f2af58e4dbb",
+                "sha256:48b389b1fd5144603d61d752afd7167dfd205973a43151ae5045b35793232aa2",
+                "sha256:4e67d26532bfd8b7f7c05d5a766d6f437b362c1bf203a3a5ce3593a645e870b8",
+                "sha256:516a405f174fd3b88829eabfe4bb296ac602d6a0f68e0d64d5ac9456194a5b7e",
+                "sha256:5ba5c243f4004c750836f81606a9fcb7841f8874ad8f3bf204ff5e56332b72b9",
+                "sha256:5bdc0f1f610d067c70aa3737ed06e2726fd9d6f7bfee4a351f4c40b6831f4e82",
+                "sha256:6107e445faf057c118d5050560695e46d272e5301feffda3c41849641222a828",
+                "sha256:6327b572f5770293fc062a7ec04160e89741e8552bf1c358d1a23eba68166759",
+                "sha256:669b29a9eca6146465cc574659058ed949748f0809a2582d1f1a324eb91054dc",
+                "sha256:6ce01337d23884b21c03869d2f68c5523d43174d4fc405490eb0091057943118",
+                "sha256:6d872186c1617d143969defeadac5a904e6e374183e07977eedef9c07c8953bf",
+                "sha256:6f76a90c345796c01d85e6332e81cab6d70de83b829cf1d9762d0a3da59c7932",
+                "sha256:70d2aa9fb00cf52034feac4b913181a6e10356019b18ef89bc7c12a283bf5f5a",
+                "sha256:7cbc78dc018596315d4e7841c8c3a7ae31cc4d638c9b627f87d52e8abaaf2d29",
+                "sha256:856bf0924d24e7f93b8aee12a3a1095c34085600aa805693fb7f5d1962393206",
+                "sha256:8a98748ed1a1df4ee1d6f927e151ed6c1a09d5ec21684de879c7ea6aa96f58f2",
+                "sha256:93a7350f6706b31f457c1457d3a3259ff9071a66f312ae64dc024f049055f72c",
+                "sha256:964823b2fc77b55355999ade496c54dde161c621cb1f6eac61dc30ed1b63cd4c",
+                "sha256:a003ac9edc22d99ae1286b0875c460351f4e101f8c9d9d2576e78d7e048f64e0",
+                "sha256:a0ce71725cacc9ebf839630772b07eeec220cbb5f03be1399e0457a1464f8e1a",
+                "sha256:a47eef975d2b8b721775a0fa286f50eab535b9d56c70a6e62842134cf7841195",
+                "sha256:a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6",
+                "sha256:a9015f5b8af1bb6837a3fcb0cdf3b874fe3385ff6274e8b7925d81ccaec3c5c9",
+                "sha256:aec510255ce690d240f7cb23d7114f6b351c733a74c279a84def763660a2c3bc",
+                "sha256:b00e7bcd71caa0282cbe3c90966f738e2db91e64092a877c3ff7f19a1628fdcb",
+                "sha256:b50aaac7d05c2c26dfd50c3321199f019ba76bb650e346a6ef3616306eed67b0",
+                "sha256:b7b6ea9e36d32582cda3465f54c4b454f62f23cb083ebc7a94e2ca6ef011c3a7",
+                "sha256:bb9333f58fc3a2296fb1d54576138d4cf5d496a2cc118422bd77835e6ae0b9cb",
+                "sha256:c1c13185b90bbd3f8b5963cd8ce7ad4ff441924c31e23c975cb150e27c2bf67a",
+                "sha256:c3b8bd3133cd50f6b637bb4322822c94c5ce4bf0d724ed5ae70afce62187c492",
+                "sha256:c5d97162c196ce54af6700949ddf9409e9833ef1003b4741c2b39ef46f1d9720",
+                "sha256:c815270206f983309915a6844fe994b2fa47e5d05c4c4cef267c3b30e34dbe42",
+                "sha256:cab2eba3830bf4f6d91e2d6718e0e1c14a2f5ad1af68a89d24ace0c6b17cced7",
+                "sha256:d1df34588123fcc88c872f5acb6f74ae59e9d182a2707097f9e28275ec26a12d",
+                "sha256:d6bdcd415ba87846fd317bee0774e412e8792832e7805938987e4ede1d13046d",
+                "sha256:db9a30ec064129d605d0f1aedc93e00894b9334ec74ba9c6bdd08147434b33eb",
+                "sha256:dbc183e7bef690c9abe5ea67b7b60fdbca81aa8da43468287dae7b5c046107d4",
+                "sha256:dca802c8db0720ce1c49cce1149ff7b06e91ba15fa84b1d59144fef1a1bc7ac2",
+                "sha256:dec6b307ce928e8e112a6bb9921a1cb00a0e14979bf28b98e084a4b8a742bd9b",
+                "sha256:df8bb0010fdd0a743b7542589223a2816bdde4d94bb5ad67884348fa2c1c67e8",
+                "sha256:e4094c7b464cf0a858e75cd14b03509e84789abf7b79f8537e6a72152109c76e",
+                "sha256:e4760a68cab57bfaa628938e9c2971137e05ce48e762a9cb53b76c9b569f1204",
+                "sha256:eb09b82377233b902d4c3fbeeb7ad731cdab579c6c6fda1f763cd779139e47c3",
+                "sha256:eb862356ee9391dc5a0b3cbc00f416b48c1b9a52d252d898e5b7696a5f9fe150",
+                "sha256:ef9528915df81b8f4c7612b19b8628214c65c9b7f74db2e34a646a0a2a0da2d4",
+                "sha256:f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76",
+                "sha256:f3e0992f23bbb0be00a921eae5363329253c3b86287db27092461c887b791e5e",
+                "sha256:f9338cc05451f1942d0d8203ec2c346c830f8e86469903d5126c1f0a13a2bcbb",
+                "sha256:ffef8fd58a36fb5f1196919638f73dd3ae0db1a878982b27a9a5a176ede4ba91"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.16.0"
+            "version": "==1.17.0"
         },
         "cfgv": {
             "hashes": [
@@ -789,67 +689,75 @@
         },
         "cython": {
             "hashes": [
-                "sha256:051069638abfb076900b0c2bcb6facf545655b3f429e80dd14365192074af5a4",
-                "sha256:076e9fd4e0ca33c5fa00a7479180dbfb62f17fe928e2909f82da814536e96d2b",
-                "sha256:077b61ee789e48700e25d4a16daa4258b8e65167136e457174df400cf9b4feab",
-                "sha256:09f2000041db482cad3bfce94e1fa3a4c82b0e57390a164c02566cbbda8c4f12",
-                "sha256:0bac3ccdd4e03924028220c62ae3529e17efa8ca7e9df9330de95de02f582b26",
-                "sha256:0e9a885ec63d3955a08cefc4eec39fefa9fe14989c6e5e2382bd4aeb6bdb9bc3",
-                "sha256:15b6d397f4ee5ad54e373589522af37935a32863f1b23fa8c6922adf833e28e2",
-                "sha256:206e803598010ecc3813db8748ed685f7beeca6c413f982df9f8a505fce56563",
-                "sha256:269f06e6961e8591d56e30b46e1a51b6ccb42cab04c29fa3b30d3e8723485fb4",
-                "sha256:2c9c1e3e78909488f3b16fabae02308423fa6369ed96ab1e250807d344cfffd7",
-                "sha256:2d29e617fd23cf4b83afe8f93f2966566c9f565918ad1e86a4502fe825cc0a79",
-                "sha256:32fbad02d1189be75eb96456d9c73f5548078e5338d8fa153ecb0115b6ee279f",
-                "sha256:35f6ede7c74024ed1982832ae61c9fad7cf60cc3f5b8c6a63bb34e38bc291936",
-                "sha256:38d40fa1324ac47c04483d151f5e092406a147eac88a18aec789cf01c089c3f2",
-                "sha256:3919a55ec9b6c7db6f68a004c21c05ed540c40dbe459ced5d801d5a1f326a053",
-                "sha256:3cffb666e649dba23810732497442fb339ee67ba4e0be1f0579991e83fcc2436",
-                "sha256:401aba1869a57aba2922ccb656a6320447e55ace42709b504c2f8e8b166f46e1",
-                "sha256:407840c56385b9c085826fe300213e0e76ba15d1d47daf4b58569078ecb94446",
-                "sha256:40fac59c3a7fbcd9c25aea64c342c890a5e2270ce64a1525e840807800167799",
-                "sha256:4f610964ab252a83e573a427e28b103e2f1dd3c23bee54f32319f9e73c3c5499",
-                "sha256:4fadb84193c25641973666e583df8df4e27c52cdc05ddce7c6f6510d690ba34a",
-                "sha256:541fbe725d6534a90b93f8c577eb70924d664b227a4631b90a6e0506d1469591",
-                "sha256:5a036d00caa73550a3a976432ef21c1e3fa12637e1616aab32caded35331ae96",
-                "sha256:5bd49a3a9fdff65446a3e1c2bfc0ec85c6ce4c3cad27cd4ad7ba150a62b7fb59",
-                "sha256:5f465443917d5c0f69825fca3b52b64c74ac3de0143b1fff6db8ba5b48c9fb4a",
-                "sha256:64f1f8bba9d8f37c0cffc934792b4ac7c42d0891077127c11deebe9fa0a0f7e4",
-                "sha256:651a15a8534ebfb9b58cb0b87c269c70984b6f9c88bfe65e4f635f0e3f07dfcd",
-                "sha256:6c5af936940a38c300977b81598d9c0901158f220a58c177820e17e1774f1cf1",
-                "sha256:712760879600907189c7d0d346851525545484e13cd8b787e94bfd293da8ccf0",
-                "sha256:81f356c1c8c0885b8435bfc468025f545c5d764aa9c75ab662616dd1193c331e",
-                "sha256:86998b01f6a6d48398df8467292c7637e57f7e3a2ca68655367f13f66fed7734",
-                "sha256:8adcde00a8a88fab27509b558cd8c2959ab0c70c65d3814cfea8c68b83fa6dcd",
-                "sha256:8c9c4c4f3ab8f8c02817b0e16e8fa7b8cc880f76e9b63fe9c010e60c1a6c2b13",
-                "sha256:8f2864ab5fcd27a346f0b50f901ebeb8f60b25a60a575ccfd982e7f3e9674914",
-                "sha256:90e2f514fc753b55245351305a399463103ec18666150bb1c36779b9862388e9",
-                "sha256:950c0c7b770d2a7cec74fb6f5ccc321d0b51d151f48c075c0d0db635a60ba1b5",
-                "sha256:9cc6a0e7e23a96dec3f3c9d39690d4281beabd5297855140d0d30855f950275e",
-                "sha256:9ea31184c7b3a728ef1f81fccb161d8948c05aa86c79f63b74fb6f3ddec860ec",
-                "sha256:9fa9e7786083b6aa61594c16979d621b62e61fcd9c2edd4761641b95c7fb34b2",
-                "sha256:a181144c2f893ed8e6a994d43d0b96300bc99873f21e3b7334ca26c61c37b680",
-                "sha256:a5e14a8c6a8157d2b0cdc2e8e3444905d20a0e78e19d2a097e89fb8b04b51f6b",
-                "sha256:a9bb402674788a7f4061aeef8057632ec440123e74ed0fb425308a59afdfa10e",
-                "sha256:a9c976e9ec429539a4367cb4b24d15a1e46b925976f4341143f49f5f161171f5",
-                "sha256:acfbe0fff364d54906058fc61f2393f38cd7fa07d344d80923937b87e339adcf",
-                "sha256:adc377aa33c3309191e617bf675fdbb51ca727acb9dc1aa23fc698d8121f7e23",
-                "sha256:b74b700d6a793113d03fb54b63bdbadba6365379424bac7c0470605672769260",
-                "sha256:bcc9795990e525c192bc5c0775e441d7d56d7a7d02210451e9e13c0448dba51b",
-                "sha256:d092c0ddba7e9e530a5c5be4ac06db8360258acc27675d1fc86294a5dc8994c5",
-                "sha256:d10fc9aa82e5e53a0b7fd118f9771199cddac8feb4a6d8350b7d4109085aa775",
-                "sha256:d4e83a8ceff7af60064da4ccfce0ac82372544dd5392f1b350c34f1b04d0fae6",
-                "sha256:dcc96739331fb854dcf503f94607576cfe8488066c61ca50dfd55836f132de99",
-                "sha256:e876272548d73583e90babda94c1299537006cad7a34e515a06c51b41f8657aa",
-                "sha256:e8df79b596633b8295eaa48b1157d796775c2bb078f32267d32f3001b687f2fd",
-                "sha256:f43a58bf2434870d2fc42ac2e9ff8138c9e00c6251468de279d93fa279e9ba3b",
-                "sha256:f4780d0f98ce28191c4d841c4358b5d5e79d96520650910cd59904123821c52d",
-                "sha256:f8a2b8fa0fd8358bccb5f3304be563c4750aae175100463d212d5ea0ec74cbe0",
-                "sha256:fc6e0faf5b57523b073f0cdefadcaef3a51235d519a0594865925cadb3aeadf0",
-                "sha256:fcbb679c0b43514d591577fd0d20021c55c240ca9ccafbdb82d3fb95e5edfee2"
+                "sha256:0b1d1f6f94cc5d42a4591f6d60d616786b9cd15576b112bc92a23131fcf38020",
+                "sha256:0e25f6425ad4a700d7f77cd468da9161e63658837d1bc34861a9861a4ef6346d",
+                "sha256:0fc6fdd6fa493be7bdda22355689d5446ac944cd71286f6f44a14b0d67ee3ff5",
+                "sha256:104d6f2f2c827ccc5e9e42c80ef6773a6aa94752fe6bc5b24a4eab4306fb7f07",
+                "sha256:11996c40c32abf843ba652a6d53cb15944c88d91f91fc4e6f0028f5df8a8f8a1",
+                "sha256:13062ce556a1e98d2821f7a0253b50569fdc98c36efd6653a65b21e3f8bbbf5f",
+                "sha256:14701edb3107a5d9305a82d9d646c4f28bfecbba74b26cc1ee2f4be08f602057",
+                "sha256:187685e25e037320cae513b8cc4bf9dbc4465c037051aede509cbbf207524de2",
+                "sha256:1dd47865f4c0a224da73acf83d113f93488d17624e2457dce1753acdfb1cc40c",
+                "sha256:221de0b48bf387f209003508e602ce839a80463522fc6f583ad3c8d5c890d2c1",
+                "sha256:2252b5aa57621848e310fe7fa6f7dce5f73aa452884a183d201a8bcebfa05a00",
+                "sha256:2a8ea2e7e2d3bc0d8630dafe6c4a5a89485598ff8a61885b74f8ed882597efd5",
+                "sha256:301bde949b4f312a1c70e214b0c3bc51a3f955d466010d2f68eb042df36447b0",
+                "sha256:3379c6521e25aa6cd7703bb7d635eaca75c0f9c7f1b0fdd6dd15a03bfac5f68d",
+                "sha256:351955559b37e6c98b48aecb178894c311be9d731b297782f2b78d111f0c9015",
+                "sha256:3699391125ab344d8d25438074d1097d9ba0fb674d0320599316cfe7cf5f002a",
+                "sha256:3999fb52d3328a6a5e8c63122b0a8bd110dfcdb98dda585a3def1426b991cba7",
+                "sha256:3f2b062f6df67e8a56c75e500ca330cf62c85ac26dd7fd006f07ef0f83aebfa3",
+                "sha256:3ff8ac1f0ecd4f505db4ab051e58e4531f5d098b6ac03b91c3b902e8d10c67b3",
+                "sha256:421017466e9260aca86823974e26e158e6358622f27c0f4da9c682f3b6d2e624",
+                "sha256:4341d6a64d47112884e0bcf31e6c075268220ee4cd02223047182d4dda94d637",
+                "sha256:44292aae17524abb4b70a25111fe7dec1a0ad718711d47e3786a211d5408fdaa",
+                "sha256:46aec30f217bdf096175a1a639203d44ac73a36fe7fa3dd06bd012e8f39eca0f",
+                "sha256:473d35681d9f93ce380e6a7c8feb2d65fc6333bd7117fbc62989e404e241dbb0",
+                "sha256:4e9a8d92978b15a0c7ca7f98447c6c578dc8923a0941d9d172d0b077cb69c576",
+                "sha256:52186101d51497519e99b60d955fd5cb3bf747c67f00d742e70ab913f1e42d31",
+                "sha256:52205347e916dd65d2400b977df4c697390c3aae0e96275a438cc4ae85dadc08",
+                "sha256:525d09b3405534763fa73bd78c8e51ac8264036ce4c16d37dfd1555a7da6d3a7",
+                "sha256:53b6072a89049a991d07f42060f65398448365c59c9cb515c5925b9bdc9d71f8",
+                "sha256:598699165cfa7c6d69513ee1bffc9e1fdd63b00b624409174c388538aa217975",
+                "sha256:63f2c892e9f9c1698ecfee78205541623eb31cd3a1b682668be7ac12de94aa8e",
+                "sha256:6823aef13669a32caf18bbb036de56065c485d9f558551a9b55061acf9c4c27f",
+                "sha256:6fb68cef33684f8cc97987bee6ae919eee7e18ee6a3ad7ed9516b8386ef95ae6",
+                "sha256:7146dd2af8682b4ca61331851e6aebce9fe5158e75300343f80c07ca80b1faff",
+                "sha256:75ba1c70b6deeaffbac123856b8d35f253da13552207aa969078611c197377e4",
+                "sha256:780f89c95b8aec1e403005b3bf2f0a2afa060b3eba168c86830f079339adad89",
+                "sha256:790263b74432cb997740d73665f4d8d00b9cd1cecbdd981d93591ddf993d4f12",
+                "sha256:8948802e1f5677a673ea5d22a1e7e273ca5f83e7a452786ca286eebf97cee67c",
+                "sha256:8acdc87e9009110adbceb7569765eb0980129055cc954c62f99fe9f094c9505e",
+                "sha256:8b14c24f1dc4c4c9d997cca8d1b7fb01187a218aab932328247dcf5694a10102",
+                "sha256:989899a85f0d9a57cebb508bd1f194cb52f0e3f7e22ac259f33d148d6422375c",
+                "sha256:9c02361af9bfa10ff1ccf967fc75159e56b1c8093caf565739ed77a559c1f29f",
+                "sha256:a0583076c4152b417a3a8a5d81ec02f58c09b67d3f22d5857e64c8734ceada8c",
+                "sha256:a1f4cbc70f6b7f0c939522118820e708e0d490edca42d852fa8004ec16780be2",
+                "sha256:a690f2ff460682ea985e8d38ec541be97e0977fa0544aadc21efc116ff8d7579",
+                "sha256:a75d45fbc20651c1b72e4111149fed3b33d270b0a4fb78328c54d965f28d55e1",
+                "sha256:aedceb6090a60854b31bf9571dc55f642a3fa5b91f11b62bcef167c52cac93d8",
+                "sha256:af91497dc098718e634d6ec8f91b182aea6bb3690f333fc9a7777bc70abe8810",
+                "sha256:b4ab2b92a3e6ed552adbe9350fd2ef3aa0cc7853cf91569f9dbed0c0699bbeab",
+                "sha256:b8c7e514075696ca0f60c337f9e416e61d7ccbc1aa879a56c39181ed90ec3059",
+                "sha256:bcd29945fafd12484cf37b1d84f12f0e7a33ba3eac5836531c6bd5283a6b3a0c",
+                "sha256:bfa550d9ae39e827a6e7198076df763571cb53397084974a6948af558355e028",
+                "sha256:c3d68751668c66c7a140b6023dba5d5d507f72063407bb609d3a5b0f3b8dfbe4",
+                "sha256:c69d5cad51388522b98a99b4be1b77316de85b0c0523fa865e0ea58bbb622e0a",
+                "sha256:c8eed5c015685106db15dd103fd040948ddca9197b1dd02222711815ea782a27",
+                "sha256:cee29846471ce60226b18e931d8c1c66a158db94853e3e79bc2da9bd22345008",
+                "sha256:d02f4ebe15aac7cdacce1a628e556c1983f26d140fd2e0ac5e0a090e605a2d38",
+                "sha256:d566a4e09b8979be8ab9f843bac0dd216c81f5e5f45661a9b25cd162ed80508c",
+                "sha256:d80a7232938d523c1a12f6b1794ab5efb1ae77ad3fde79de4bb558d8ab261619",
+                "sha256:d89a82937ce4037f092e9848a7bbcc65bc8e9fc9aef2bb74f5c15e7d21a73080",
+                "sha256:da394654c6da15c1d37f0b7ec5afd325c69a15ceafee2afba14b67a5df8a82c8",
+                "sha256:ddd1fe25af330f4e003421636746a546474e4ccd8f239f55d2898d80983d20ed",
+                "sha256:e6dd395d1a704e34a9fac00b25f0036dce6654c6b898be6f872ac2bb4f2eda48",
+                "sha256:eeb6860b0f4bfa402de8929833fe5370fa34069c7ebacb2d543cb017f21fb891",
+                "sha256:f3953d2f504176f929862e5579cfc421860c33e9707f585d70d24e1096accdf7",
+                "sha256:f988f7f8164a6079c705c39e2d75dbe9967e3dacafe041420d9af7b9ee424162"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.0.10"
+            "version": "==3.0.11"
         },
         "dask": {
             "extras": [
@@ -863,33 +771,42 @@
             "markers": "python_version >= '3.9'",
             "version": "==2024.6.2"
         },
+        "dask-expr": {
+            "hashes": [
+                "sha256:04e31cb941b7cbdce7b1384f2bcf17fd17e828e45e9c74491e28473095ee6891",
+                "sha256:ace366c6d9c248a7fa7b48f7a34140afae3b6a0ea14ee57743babe5a9d1ef43f"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==1.1.6"
+        },
         "debugpy": {
             "hashes": [
-                "sha256:0600faef1d0b8d0e85c816b8bb0cb90ed94fc611f308d5fde28cb8b3d2ff0fe3",
-                "sha256:1523bc551e28e15147815d1397afc150ac99dbd3a8e64641d53425dba57b0ff9",
-                "sha256:15bc2f4b0f5e99bf86c162c91a74c0631dbd9cef3c6a1d1329c946586255e859",
-                "sha256:16c8dcab02617b75697a0a925a62943e26a0330da076e2a10437edd9f0bf3755",
-                "sha256:16e16df3a98a35c63c3ab1e4d19be4cbc7fdda92d9ddc059294f18910928e0ca",
-                "sha256:2cbd4d9a2fc5e7f583ff9bf11f3b7d78dfda8401e8bb6856ad1ed190be4281ad",
-                "sha256:3f8c3f7c53130a070f0fc845a0f2cee8ed88d220d6b04595897b66605df1edd6",
-                "sha256:40f062d6877d2e45b112c0bbade9a17aac507445fd638922b1a5434df34aed02",
-                "sha256:5a019d4574afedc6ead1daa22736c530712465c0c4cd44f820d803d937531b2d",
-                "sha256:5d3ccd39e4021f2eb86b8d748a96c766058b39443c1f18b2dc52c10ac2757835",
-                "sha256:62658aefe289598680193ff655ff3940e2a601765259b123dc7f89c0239b8cd3",
-                "sha256:7ee2e1afbf44b138c005e4380097d92532e1001580853a7cb40ed84e0ef1c3d2",
-                "sha256:7f8d57a98c5a486c5c7824bc0b9f2f11189d08d73635c326abef268f83950326",
-                "sha256:8a13417ccd5978a642e91fb79b871baded925d4fadd4dfafec1928196292aa0a",
-                "sha256:95378ed08ed2089221896b9b3a8d021e642c24edc8fef20e5d4342ca8be65c00",
-                "sha256:acdf39855f65c48ac9667b2801234fc64d46778021efac2de7e50907ab90c634",
-                "sha256:bd11fe35d6fd3431f1546d94121322c0ac572e1bfb1f6be0e9b8655fb4ea941e",
-                "sha256:c78ba1680f1015c0ca7115671fe347b28b446081dada3fedf54138f44e4ba031",
-                "sha256:cf327316ae0c0e7dd81eb92d24ba8b5e88bb4d1b585b5c0d32929274a66a5210",
-                "sha256:d3408fddd76414034c02880e891ea434e9a9cf3a69842098ef92f6e809d09afa",
-                "sha256:e24ccb0cd6f8bfaec68d577cb49e9c680621c336f347479b3fce060ba7c09ec1",
-                "sha256:f179af1e1bd4c88b0b9f0fa153569b24f6b6f3de33f94703336363ae62f4bf47"
+                "sha256:0a1029a2869d01cb777216af8c53cda0476875ef02a2b6ff8b2f2c9a4b04176c",
+                "sha256:1cd04a73eb2769eb0bfe43f5bfde1215c5923d6924b9b90f94d15f207a402226",
+                "sha256:28ced650c974aaf179231668a293ecd5c63c0a671ae6d56b8795ecc5d2f48d3c",
+                "sha256:345d6a0206e81eb68b1493ce2fbffd57c3088e2ce4b46592077a943d2b968ca3",
+                "sha256:3df6692351172a42af7558daa5019651f898fc67450bf091335aa8a18fbf6f3a",
+                "sha256:4413b7a3ede757dc33a273a17d685ea2b0c09dbd312cc03f5534a0fd4d40750a",
+                "sha256:4fbb3b39ae1aa3e5ad578f37a48a7a303dad9a3d018d369bc9ec629c1cfa7408",
+                "sha256:55919dce65b471eff25901acf82d328bbd5b833526b6c1364bd5133754777a44",
+                "sha256:5b5c770977c8ec6c40c60d6f58cacc7f7fe5a45960363d6974ddb9b62dbee156",
+                "sha256:606bccba19f7188b6ea9579c8a4f5a5364ecd0bf5a0659c8a5d0e10dcee3032a",
+                "sha256:7b0fe36ed9d26cb6836b0a51453653f8f2e347ba7348f2bbfe76bfeb670bfb1c",
+                "sha256:7e4d594367d6407a120b76bdaa03886e9eb652c05ba7f87e37418426ad2079f7",
+                "sha256:8f913ee8e9fcf9d38a751f56e6de12a297ae7832749d35de26d960f14280750a",
+                "sha256:a697beca97dad3780b89a7fb525d5e79f33821a8bc0c06faf1f1289e549743cf",
+                "sha256:ad84b7cde7fd96cf6eea34ff6c4a1b7887e0fe2ea46e099e53234856f9d99a34",
+                "sha256:b2112cfeb34b4507399d298fe7023a16656fc553ed5246536060ca7bd0e668d0",
+                "sha256:b78c1250441ce893cb5035dd6f5fc12db968cc07f91cc06996b2087f7cefdd8e",
+                "sha256:c0a65b00b7cdd2ee0c2cf4c7335fef31e15f1b7056c7fdbce9e90193e1a8c8cb",
+                "sha256:c9f7c15ea1da18d2fcc2709e9f3d6de98b69a5b0fff1807fb80bc55f906691f7",
+                "sha256:db9fb642938a7a609a6c865c32ecd0d795d56c1aaa7a7a5722d77855d5e77f2b",
+                "sha256:dd3811bd63632bb25eda6bd73bea8e0521794cda02be41fa3160eb26fc29e7ed",
+                "sha256:e84c276489e141ed0b93b0af648eef891546143d6a48f610945416453a8ad406"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.8.2"
+            "version": "==1.8.5"
         },
         "decorator": {
             "hashes": [
@@ -980,28 +897,34 @@
         },
         "fiona": {
             "hashes": [
-                "sha256:18649326a7724611b16b648e14fd094089d517413b95ac91d0cdb0adc5fcb8de",
-                "sha256:3f26c8b6ea9bc92cbd52a4dd83ffd44472450bf92f4e3d4ef2341adc2f35a54d",
-                "sha256:59a3800bc09ebee3516d64d02a8a6818d07ab1573c6096f3ef3468bf9f8f95f8",
-                "sha256:6ba2294bc6adcbc36229862667aac6b98e6c306e1958caf53b8bfcf9a3b8c77a",
-                "sha256:75924f69c51db6e258c91308780546278028c509db12aa33a47692a0266c9667",
-                "sha256:89cfcc3bdb4aba7bba1eb552b3866b851334693ab694529803122b21f5927960",
-                "sha256:904793b17aee70ca9c3d582dbf01623eccfdeacd00c5e1a8e421be41f2e43d67",
-                "sha256:a82a99ce9b3e7825740157c45c9fb2259d4e92f0a886aaac25f0db40ffe1eea3",
-                "sha256:b5cad3424b7473eb0e19f17ee45abec92133a694a4b452a278f02e3b8d0f810f",
-                "sha256:b88e2e6548a41c1dfa3f96c8275ff472a3edca729e14a641c0fa5b2e146a8ab5",
-                "sha256:c28d9ffa5d230a1d9eaf571529fa9eb7573d39613354c090ad077ad153a37ee1",
-                "sha256:c4aafdd565b3a30bdd78cafae35d4945f6741eef31401c1bb1e166b6262d7539",
-                "sha256:ce9a22c9883cc5d11c05ba3fb9db5082044a07c6b299753ea5bb8e178b8ba53b",
-                "sha256:d0df3e105ad7f0cca5f16b441c232fd693ef6c4adf2c1b6271aaaa1cdc06164d",
-                "sha256:d47777890aa1d715025abc7a6d6b2a6bb8d2a37cc94c44ce95940b80eda21444",
-                "sha256:df34c980cd7396adfbc89bbb363bdd6e358c76f91969fc98c9dfc076dd11638d",
-                "sha256:e33860aaf70bbd2726cff12fd3857bd832b6dc2ad3ce4b27e7563bd68abdc26f",
-                "sha256:e3ed1c0c1c60f710a612aaeb294de54214d228c4ef40e0c1dc159e46f86a9446",
-                "sha256:ed75dd29c89e0e455e3a322f28cd92f192bcb8fced16e2bfb6422a7f95ffe5e9"
+                "sha256:0eaffbf3bfae9960484c0c08ea461b0c40e111497f04e9475ebf15ac7a22d9dc",
+                "sha256:11af1afc1255642a7787fe112c29d01f968f1053e4d4700fc6f3bb879c1622e0",
+                "sha256:1dcd6eca7524535baf2a39d7981b4a46d33ae28c313934a7c3eae62eecf9dfa5",
+                "sha256:39819fb8f5ec6d9971cb01b912b4431615a3d3f50c83798565d8ce41917930db",
+                "sha256:42d8a0e5570948d3821c493b6141866d9a4d7a64edad2be4ecbb89f81904baac",
+                "sha256:48b6218224e96de5e36b5eb259f37160092260e5de0dcd82ca200b1887aa9884",
+                "sha256:50037c3b7a5f6f434b562b5b1a5b664f1caa7a4383b00af23cdb59bfc6ba852c",
+                "sha256:52e8fec650b72fc5253d8f86b63859acc687182281c29bfacd3930496cf982d1",
+                "sha256:53bedd2989e255df1bf3378ae9c06d6d241ec273c280c544bb44ffffebb97fb0",
+                "sha256:63e528b5ea3d8b1038d788e7c65117835c787ba7fdc94b1b42f09c2cbc0aaff2",
+                "sha256:77653a08564a44e634c44cd74a068d2f55d1d4029edd16d1c8aadcc4d8cc1d2c",
+                "sha256:791b3494f8b218c06ea56f892bd6ba893dfa23525347761d066fb7738acda3b1",
+                "sha256:89095c2d542325ee45894b8837e8048cdbb2f22274934e1be3b673ca628010d7",
+                "sha256:918bd27d8625416672e834593970f96dff63215108f81efb876fe5c0bc58a3b4",
+                "sha256:98cea6f435843b2119731c6b0470e5b7386aa16b6aa7edabbf1ed93aefe029c3",
+                "sha256:9b53034efdf93ada9295b081e6a8280af7c75496a20df82d4c2ca46d65b85905",
+                "sha256:bf51846ad602757bf27876f458c5c9f14b09421fac612f64273cc4e3fcabc441",
+                "sha256:c1dd5fbc29b7303bb87eb683455e8451e1a53bb8faf20ef97fdcd843c9e4a7f6",
+                "sha256:c9b92aa1badb2773e7cac19bef3064d73e9d80c67c42f0928db2520a04be6f2f",
+                "sha256:e313210b30d09ed8f829bf625599e248dadd78622728030221f6526580ff26c5",
+                "sha256:e5404ed08c711489abcb3a50a184816825b8af06eb73ad2a99e18b8e7b47c96a",
+                "sha256:e7617563b36d2be99f048f0d0054b4d765f4aae454398f88f19de9c2c324b7f8",
+                "sha256:f1b49d51a744874608b689f029766aa1e078dd72e94b44cf8eeef6d7bd2e9051",
+                "sha256:f4230eccbd896a79d1ebfa551d84bf90f512f7bcbe1ca61e3f82231321f1a532"
             ],
             "index": "pypi",
-            "version": "==1.8.22"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.9.6"
         },
         "flake8": {
             "hashes": [
@@ -1083,104 +1006,21 @@
             ],
             "version": "==1.5.1"
         },
-        "frozenlist": {
-            "hashes": [
-                "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7",
-                "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98",
-                "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad",
-                "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5",
-                "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae",
-                "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e",
-                "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a",
-                "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701",
-                "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d",
-                "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6",
-                "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6",
-                "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106",
-                "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75",
-                "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868",
-                "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a",
-                "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0",
-                "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1",
-                "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826",
-                "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec",
-                "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6",
-                "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950",
-                "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19",
-                "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0",
-                "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8",
-                "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a",
-                "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09",
-                "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86",
-                "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c",
-                "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5",
-                "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b",
-                "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b",
-                "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d",
-                "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0",
-                "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea",
-                "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776",
-                "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a",
-                "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897",
-                "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7",
-                "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09",
-                "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9",
-                "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe",
-                "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd",
-                "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742",
-                "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09",
-                "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0",
-                "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932",
-                "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1",
-                "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a",
-                "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49",
-                "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d",
-                "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7",
-                "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480",
-                "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89",
-                "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e",
-                "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b",
-                "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82",
-                "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb",
-                "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068",
-                "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8",
-                "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b",
-                "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb",
-                "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2",
-                "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11",
-                "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b",
-                "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc",
-                "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0",
-                "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497",
-                "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17",
-                "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0",
-                "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2",
-                "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439",
-                "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5",
-                "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac",
-                "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825",
-                "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887",
-                "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced",
-                "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==1.4.1"
-        },
         "fsspec": {
             "hashes": [
-                "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a",
-                "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"
+                "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e",
+                "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.6.0"
+            "version": "==2024.6.1"
         },
         "geocube": {
             "hashes": [
-                "sha256:0b06a1401df539d559703053d380f0e10bb7e013653ce9acb5e164e36f0a4ab3",
-                "sha256:bf49d7b19c4d78740c095ab93f1f10f448f131338ccc74de497ec5b437273730"
+                "sha256:37c0d15cfb739883ce1564b37382d1286cf7b5ceab4972743d28c06822f325e5",
+                "sha256:c72f8f926b30a257a782dd61f6f334c91b7d3009487e4e2aa8e154c4a9812d8c"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.5.3"
+            "version": "==0.6.0"
         },
         "geographiclib": {
             "hashes": [
@@ -1192,12 +1032,12 @@
         },
         "geopandas": {
             "hashes": [
-                "sha256:41b31ad39e21bc9e8c4254f78f8dc4ce3d33d144e22e630a00bb336c83160204",
-                "sha256:748af035d4a068a4ae00cab384acb61d387685c833b0022e0729aa45216b23ac"
+                "sha256:01e147d9420cc374d26f51fc23716ac307f32b49406e4bd8462c07e82ed1d3d6",
+                "sha256:b8bf70a5534588205b7a56646e2082fb1de9a03599651b3d80c99ea4c2ca08ab"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.9'",
-            "version": "==0.14.3"
+            "version": "==1.0.1"
         },
         "geopy": {
             "hashes": [
@@ -1209,12 +1049,12 @@
         },
         "gval": {
             "hashes": [
-                "sha256:5721270a41278888c043a3fc1c91c26998799f31edf151a6383afd8e2bd9de9a",
-                "sha256:9c9e1d9710e237a95cbb8385d7c41f21e0e64051fe526985373a2bc80bf5fa87"
+                "sha256:3f8808d56fe4d153dfcdc613079595e9b1e3a2028f80f9a9daa2af618f56c5a7",
+                "sha256:ee690055953f8a2c65b8996fd0e45fc9f945a197d09f7df887c94ac19929bb44"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.2.7"
+            "version": "==0.2.7.post1"
         },
         "identify": {
             "hashes": [
@@ -1234,11 +1074,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f",
-                "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"
+                "sha256:42817a4a0be5845d22c6e212db66a94ad261e2318d80b3e0d363894a79df2b67",
+                "sha256:9c8fa6e8ea0f9516ad5c8db9246a731c948193c7754d3babb0114a05b27dd364"
             ],
             "markers": "python_version < '3.12'",
-            "version": "==8.0.0"
+            "version": "==8.3.0"
         },
         "inflate64": {
             "hashes": [
@@ -1911,102 +1751,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.0.8"
         },
-        "multidict": {
-            "hashes": [
-                "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556",
-                "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c",
-                "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29",
-                "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b",
-                "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8",
-                "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7",
-                "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd",
-                "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40",
-                "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6",
-                "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3",
-                "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c",
-                "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9",
-                "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5",
-                "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae",
-                "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442",
-                "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9",
-                "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc",
-                "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c",
-                "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea",
-                "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5",
-                "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50",
-                "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182",
-                "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453",
-                "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e",
-                "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600",
-                "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733",
-                "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda",
-                "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241",
-                "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461",
-                "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e",
-                "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e",
-                "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b",
-                "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e",
-                "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7",
-                "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386",
-                "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd",
-                "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9",
-                "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf",
-                "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee",
-                "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5",
-                "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a",
-                "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271",
-                "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54",
-                "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4",
-                "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496",
-                "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb",
-                "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319",
-                "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3",
-                "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f",
-                "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527",
-                "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed",
-                "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604",
-                "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef",
-                "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8",
-                "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5",
-                "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5",
-                "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626",
-                "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c",
-                "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d",
-                "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c",
-                "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc",
-                "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc",
-                "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b",
-                "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38",
-                "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450",
-                "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1",
-                "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f",
-                "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3",
-                "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755",
-                "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226",
-                "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a",
-                "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046",
-                "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf",
-                "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479",
-                "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e",
-                "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1",
-                "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a",
-                "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83",
-                "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929",
-                "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93",
-                "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a",
-                "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c",
-                "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44",
-                "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89",
-                "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba",
-                "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e",
-                "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da",
-                "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24",
-                "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423",
-                "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==6.0.5"
-        },
         "multimethod": {
             "hashes": [
                 "sha256:8db8ef2a8d2a247e3570cc23317680892fdf903d84c8c1053667c8e8f7671a67",
@@ -2023,14 +1767,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.2.3"
         },
-        "munch": {
-            "hashes": [
-                "sha256:542cb151461263216a4e37c3fd9afc425feeaf38aaa3025cd2a981fadb422235",
-                "sha256:71033c45db9fb677a0b7eb517a4ce70ae09258490e419b0e7f00d1e386ecb1b4"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==4.0.0"
-        },
         "mypy-extensions": {
             "hashes": [
                 "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d",
@@ -2281,11 +2017,11 @@
         },
         "numpy-groupies": {
             "hashes": [
-                "sha256:b0e9cf7a4d015beca969f1e90463fa90972df8380a722eb96d679f8838d2f6fe",
-                "sha256:c9f2471b0ee96e202fb0a591a551192c19fa3bdddad33df8665a7accbaf3263e"
+                "sha256:2fda978c4d28d2f1633a63972f425d0a7f2f12a75505d215b41b6de712e2ec4b",
+                "sha256:8d0d686160f860c12d97f12f457116f8b8bebfa33a68b8efcd24dcfedc2837f1"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.11.1"
+            "version": "==0.11.2"
         },
         "odc-geo": {
             "hashes": [
@@ -2544,7 +2280,7 @@
                 "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4",
                 "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "markers": "sys_platform != 'cygwin'",
             "version": "==5.9.8"
         },
         "psycopg2-binary": {
@@ -2626,10 +2362,10 @@
         },
         "pure-eval": {
             "hashes": [
-                "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350",
-                "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"
+                "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0",
+                "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"
             ],
-            "version": "==0.2.2"
+            "version": "==0.2.3"
         },
         "py-cpuinfo": {
             "hashes": [
@@ -2649,46 +2385,46 @@
         },
         "pyarrow": {
             "hashes": [
-                "sha256:0140c7e2b740e08c5a459439d87acd26b747fc408bde0a8806096ee0baaa0c15",
-                "sha256:01e44de9749cddc486169cb632f3c99962318e9dacac7778315a110f4bf8a450",
-                "sha256:05fe7994745b634c5fb16ce5717e39a1ac1fac3e2b0795232841660aa76647cd",
-                "sha256:06ca79080ef89d6529bb8e5074d4b4f6086143b2520494fcb7cf8a99079cde93",
-                "sha256:097828b55321897db0e1dbfc606e3ff8101ae5725673498cbfa7754ee0da80e4",
-                "sha256:0f6f053cb66dc24091f5511e5920e45c83107f954a21032feadc7b9e3a8e7851",
-                "sha256:11e045dfa09855b6d3e7705a37c42e2dc2c71d608fab34d3c23df2e02df9aec3",
-                "sha256:1a8ae88c0038d1bc362a682320112ee6774f006134cd5afc291591ee4bc06505",
-                "sha256:1daab52050a1c48506c029e6fa0944a7b2436334d7e44221c16f6f1b2cc9c510",
-                "sha256:2a145dab9ed7849fc1101bf03bcdc69913547f10513fdf70fc3ab6c0a50c7eee",
-                "sha256:30d8494870d9916bb53b2a4384948491444741cb9a38253c590e21f836b01222",
-                "sha256:323cbe60210173ffd7db78bfd50b80bdd792c4c9daca8843ef3cd70b186649db",
-                "sha256:32542164d905002c42dff896efdac79b3bdd7291b1b74aa292fac8450d0e4dcd",
-                "sha256:33c1f6110c386464fd2e5e4ea3624466055bbe681ff185fd6c9daa98f30a3f9a",
-                "sha256:3c76807540989fe8fcd02285dd15e4f2a3da0b09d27781abec3adc265ddbeba1",
-                "sha256:3f6d5faf4f1b0d5a7f97be987cf9e9f8cd39902611e818fe134588ee99bf0283",
-                "sha256:450e4605e3c20e558485f9161a79280a61c55efe585d51513c014de9ae8d393f",
-                "sha256:470ae0194fbfdfbf4a6b65b4f9e0f6e1fa0ea5b90c1ee6b65b38aecee53508c8",
-                "sha256:4756a2b373a28f6166c42711240643fb8bd6322467e9aacabd26b488fa41ec23",
-                "sha256:58c889851ca33f992ea916b48b8540735055201b177cb0dcf0596a495a667b00",
-                "sha256:6263cffd0c3721c1e348062997babdf0151301f7353010c9c9a8ed47448f82ab",
-                "sha256:78d4a77a46a7de9388b653af1c4ce539350726cd9af62e0831e4f2bd0c95a2f4",
-                "sha256:7a8089d7e77d1455d529dbd7cff08898bbb2666ee48bc4085203af1d826a33cc",
-                "sha256:906b0dc25f2be12e95975722f1e60e162437023f490dbd80d0deb7375baf3171",
-                "sha256:922e8b49b88da8633d6cac0e1b5a690311b6758d6f5d7c2be71acb0f1e14cd61",
-                "sha256:96d64e5ba7dceb519a955e5eeb5c9adcfd63f73a56aea4722e2cc81364fc567a",
-                "sha256:981670b4ce0110d8dcb3246410a4aabf5714db5d8ea63b15686bce1c914b1f83",
-                "sha256:a8eeef015ae69d104c4c3117a6011e7e3ecd1abec79dc87fd2fac6e442f666ee",
-                "sha256:b8b3f4fe8d4ec15e1ef9b599b94683c5216adaed78d5cb4c606180546d1e2ee1",
-                "sha256:be28e1a07f20391bb0b15ea03dcac3aade29fc773c5eb4bee2838e9b2cdde0cb",
-                "sha256:c7331b4ed3401b7ee56f22c980608cf273f0380f77d0f73dd3c185f78f5a6220",
-                "sha256:cf87e2cec65dd5cf1aa4aba918d523ef56ef95597b545bbaad01e6433851aa10",
-                "sha256:d0351fecf0e26e152542bc164c22ea2a8e8c682726fce160ce4d459ea802d69c",
-                "sha256:d264ad13605b61959f2ae7c1d25b1a5b8505b112715c961418c8396433f213ad",
-                "sha256:e592e482edd9f1ab32f18cd6a716c45b2c0f2403dc2af782f4e9674952e6dd27",
-                "sha256:fada8396bc739d958d0b81d291cfd201126ed5e7913cb73de6bc606befc30226"
+                "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a",
+                "sha256:02dae06ce212d8b3244dd3e7d12d9c4d3046945a5933d28026598e9dbbda1fca",
+                "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597",
+                "sha256:0cdb0e627c86c373205a2f94a510ac4376fdc523f8bb36beab2e7f204416163c",
+                "sha256:13d7a460b412f31e4c0efa1148e1d29bdf18ad1411eb6757d38f8fbdcc8645fb",
+                "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977",
+                "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3",
+                "sha256:32503827abbc5aadedfa235f5ece8c4f8f8b0a3cf01066bc8d29de7539532687",
+                "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7",
+                "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204",
+                "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28",
+                "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087",
+                "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15",
+                "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc",
+                "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2",
+                "sha256:7c7916bff914ac5d4a8fe25b7a25e432ff921e72f6f2b7547d1e325c1ad9d155",
+                "sha256:9b564a51fbccfab5a04a80453e5ac6c9954a9c5ef2890d1bcf63741909c3f8df",
+                "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22",
+                "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a",
+                "sha256:a155acc7f154b9ffcc85497509bcd0d43efb80d6f733b0dc3bb14e281f131c8b",
+                "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03",
+                "sha256:a48ddf5c3c6a6c505904545c25a4ae13646ae1f8ba703c4df4a1bfe4f4006bda",
+                "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07",
+                "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204",
+                "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b",
+                "sha256:d7d192305d9d8bc9082d10f361fc70a73590a4c65cf31c3e6926cd72b76bc35c",
+                "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545",
+                "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655",
+                "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420",
+                "sha256:dec8d129254d0188a49f8a1fc99e0560dc1b85f60af729f47de4046015f9b0a5",
+                "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4",
+                "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8",
+                "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053",
+                "sha256:f553ca691b9e94b202ff741bdd40f6ccb70cdd5fbf65c187af132f1317de6145",
+                "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047",
+                "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==14.0.1"
+            "version": "==17.0.0"
         },
         "pybcj": {
             "hashes": [
@@ -2867,35 +2603,36 @@
         },
         "pyogrio": {
             "hashes": [
-                "sha256:1b7197c72f034ac7187da2a8d50a063a5f1256aab732b154f11f887a7652dc3d",
-                "sha256:234b0d1d22e9680229b0618c25077a0cb2428cbbc2939b4bb9bdd8ee77e0f3e0",
-                "sha256:3001efd5dfee36459d0cfdafbe91ed88fc5ae734353d771cdb75546ef1427735",
-                "sha256:31112bb0b6a4a3f80ec3252d7eeb7be81045860d49fd76e297c073759450652b",
-                "sha256:33ae5aafcf3a557e107a33f5b3e878750d2e467b8cc911dc4bf261c1a602b534",
-                "sha256:33afb7d211c6434613f24174722347a5cb11d22a212f28c817f67c89d30d0c0d",
-                "sha256:429dcff4c36f0e0a15ba4a20f2d4478b9c6d095e70c4bcc007a536ea420a1a93",
-                "sha256:436de39f57e8f8cc41682981518b9490d64d3a1c48bf78d415e5747c296790dc",
-                "sha256:5654e7c33442cbd98e7a56f705e160415d7503b2420d724d4f81b8cc88360b3e",
-                "sha256:5feeb7a0da7ee82580f6aa6508a80602413675b99c60c822929e0e8b925e0517",
-                "sha256:73577fecebeecf0d06e78c1a4bddd460a4d57c6d918affab7594c0bc72f5fa14",
-                "sha256:7e2c856961efdc6cb3809b97b49016cbbcee17c8a1e85fc4000b5fcb3cfcb9b1",
-                "sha256:7e39bb6bfdd74e63ae96acced7297bbe8a157f85c0107f1cbb395d2a937f3a38",
-                "sha256:860b04ddf23b8c253ceb3621e4b0e0dc0f293eab66cb14f799a5c9f9fe0a882c",
-                "sha256:892fdab0e1c44c0125254d92928081c14f93ac553f371addc2c9a1d4bde41cad",
-                "sha256:9cc6db2e5dc50dfe23554d10502920eafa0648c365725e552aaa523432a9bf35",
-                "sha256:a23136d1bffa9d811263807b850c6e9854201710276f09de650131e89f2486aa",
-                "sha256:b9a8a4854c7af2c76683ce5666ee765b207901b362576465219d75deb6159821",
-                "sha256:ba386a02c9b5934c568b40acc95c9863f92075f6990167635e51368976569c66",
-                "sha256:be46be43c4148a3ad09da38670411485ec544a51cbd6b7d004a0eca5035023fc",
-                "sha256:bee556ca305b7e8c68aada259d925c612131205074fb2373badafacbef610b77",
-                "sha256:caaf61d473ac207f170082e602ea57c096e8dd4c4be51de58fba96f1a5944096",
-                "sha256:d5fc2304aeb927564f77caaa4da9a47e2d77a8ceb1c624ea84c505140886b221",
-                "sha256:f219c1edb010d0248891a3d27d15faf17c91cfe69daef84d7471e22e4ed4fcff",
-                "sha256:f2ff58184020da39540a2f5d4a5412005a01b0c4cd03c7b8294bc670d1f3fe50"
+                "sha256:00347b7187eb289819295333cf363161eb4cf8d8abb469af7f5bc84a00916ff5",
+                "sha256:04e673fe5d712711eed4877aed069683f9734948a66e58785f045166343c5f3f",
+                "sha256:12eaa1dd13d4632f12e334f7dccbbf7cea6ae3ed967d8db2b0ade44787f44cfa",
+                "sha256:1bcca7902eb7c3c5d2d7528f6fa86cd9f0409a993e63c661dfec2415ad5a3a55",
+                "sha256:2779f0c04573321b879570e07de3525cadb4e47efa4bc245ff655a419d84a3e2",
+                "sha256:3a574acccec2a2d8af097a01975688bc059d8d03b49c56b4ee029eb3d9b384e2",
+                "sha256:3b05184adb265d8b1d17b8da4a0d08c311ba09d38d843286184d399997028344",
+                "sha256:3c2550195df759486337aea76353c467bb7f4b5287df6bc789756d4b7f84aab8",
+                "sha256:459ec590c3c1cda451f3f73c88a678c32b127de783bf54c41ea6ad708969f020",
+                "sha256:4fc58e9e9201605bd5f9056734e67314a296a696f28a71c9a0fb0152108965b0",
+                "sha256:56997a41ef9cf1e36996526e9f9c1f620d18cfc67cebe94afcd82d44315b490f",
+                "sha256:63dc3ef1272c4b01b8dd6a5fd3c696ab81cc6f7030d2dcb266d01fb8909e27cf",
+                "sha256:7434b5222e6c373c9fc01937fdea7e1cfa88030770efe650f296cdd7656f7c35",
+                "sha256:7a830a0a89f5846c4b7fbb9faf34d3ca5cdb56f427f0eb12dfa46bc2664affda",
+                "sha256:83bdf508a1903328731d3843955a951b717092317e6192ff6d882ceece559da7",
+                "sha256:a06cfd08d9abcc28a1ae07f3c3def02c6698767a8200c0248f3cd06e50e27433",
+                "sha256:b8fef111416d149d098e999cf2df2dd3cbe3d8dac2b7c63017b8b931dcbde51b",
+                "sha256:bd4c7b294c62d48f4263db78d8998d27db29298cfbbbe244ba7687a6280853c7",
+                "sha256:d78585d100cb86b98582702416e56cd6bc10540a6cbedfd2b5d3d75b9a44e3d3",
+                "sha256:dca63e63a50ca69e0334d6d4f8904e622974007ed072f179f5c26f81277a6dbe",
+                "sha256:e09b72756700537456007ee7c2734cc76cb2239e20617735b019a2c402b93a91",
+                "sha256:e83ddf7e821ec3c1376ab9cf57ba3def18f687a117e43f508c4be1174045ffc7",
+                "sha256:e8cc26bb7685371da471e8b6107bed9fe4a56a4fe649c38dea8aaee99b26bf2e",
+                "sha256:ed10c44e7b2fe899ecd25e9bdd82a1b4ba5d2792850924858f7aea04cad3d801",
+                "sha256:f67c0e68896fd49218e6451a95d2dee75816a715fb80048250c21e3b39babc48",
+                "sha256:fc77e8b07cd971601868e1c8ff1fa231773df1c889ca6e2c99a0ad22380aefb2"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.7.2"
+            "version": "==0.8.0"
         },
         "pyparsing": {
             "hashes": [
@@ -2991,45 +2728,37 @@
         },
         "pyproj": {
             "hashes": [
-                "sha256:052c49fce8b5d55943a35c36ccecb87350c68b48ba95bc02a789770c374ef819",
-                "sha256:1507138ea28bf2134d31797675380791cc1a7156a3aeda484e65a78a4aba9b62",
-                "sha256:1798ff7d65d9057ebb2d017ffe8403268b8452f24d0428b2140018c25c7fa1bc",
-                "sha256:1b7c2113c4d11184a238077ec85e31eda1dcc58ffeb9a4429830e0a7036e787d",
-                "sha256:2b708fd43453b985642b737d4a6e7f1d6a0ab1677ffa4e14cc258537b49224b0",
-                "sha256:385b0341861d3ebc8cad98337a738821dcb548d465576527399f4955ca24b6ed",
-                "sha256:38862fe07316ae12b79d82d298e390973a4f00b684f3c2d037238e20e00610ba",
-                "sha256:5674923351e76222e2c10c58b5e1ac119d7a46b270d822c463035971b06f724b",
-                "sha256:5c4b85ac10d733c42d73a2e6261c8d6745bf52433a31848dd1b6561c9a382da3",
-                "sha256:61e4ad57d89b03a7b173793b31bca8ee110112cde1937ef0f42a70b9120c827d",
-                "sha256:621d78a9d8bf4d06e08bef2471021fbcb1a65aa629ad4a20c22e521ce729cc20",
-                "sha256:6475ce653880938468a1a1b7321267243909e34b972ba9e53d5982c41d555918",
-                "sha256:6a87b419a2a352413fbf759ecb66da9da50bd19861c8f26db6a25439125b27b9",
-                "sha256:6f316a66031a14e9c5a88c91f8b77aa97f5454895674541ed6ab630b682be35d",
-                "sha256:71b65f2a38cd9e16883dbb0f8ae82bdf8f6b79b1b02975c78483ab8428dbbf2f",
-                "sha256:73f7960a97225812f9b1d7aeda5fb83812f38de9441e3476fcc8abb3e2b2f4de",
-                "sha256:7572983134e310e0ca809c63f1722557a040fe9443df5f247bf11ba887eb1229",
-                "sha256:788a5dadb532644a64efe0f5f01bf508c821eb7e984f13a677d56002f1e8a67a",
-                "sha256:7bdd2021bb6f7f346bfe1d2a358aa109da017d22c4704af2d994e7c7ee0a7a53",
-                "sha256:8fe6bb1b68a35d07378d38be77b5b2f8dd2bea5910c957bfcc7bee55988d3910",
-                "sha256:97ed199033c2c770e7eea2ef80ff5e6413426ec2d7ec985b869792f04ab95d05",
-                "sha256:9859d1591c1863414d875ae0759e72c2cffc01ab989dc64137fbac572cc81bf6",
-                "sha256:a730f5b4c98c8a0f312437873e6e34dbd4cc6dc23d5afd91a6691c62724b1f68",
-                "sha256:b60d93a200639e8367c6542a964fd0aa2dbd152f256c1831dc18cd5aa470fb8a",
-                "sha256:b752b7d9c4b08181c7e8c0d9c7f277cbefff42227f34d3310696a87c863d9dd3",
-                "sha256:b937215bfbaf404ec8f03ca741fc3f9f2c4c2c5590a02ccddddd820ae3c71331",
-                "sha256:c02742ef3d846401861a878a61ef7ad911ea7539d6cc4619ddb52dbdf7b45aee",
-                "sha256:cd5e2b6aa255023c4acd0b977590f1f7cc801ba21b4d806fcf6dfac3474ebb83",
-                "sha256:d711517a8487ef3245b08dc82f781a906df9abb3b6cb0ce0486f0eeb823ca570",
-                "sha256:d9a024370e917c899bff9171f03ea6079deecdc7482a146a2c565f3b9df134ea",
-                "sha256:e08db25b61cf024648d55973cc3d1c3f1d0818fabf594d5f5a8e2318103d2aa0",
-                "sha256:e97573de0ab3bbbcb4c7748bc41f4ceb6da10b45d35b1a294b5820701e7c25f0",
-                "sha256:eccb417b91d0be27805dfc97550bfb8b7db94e9fe1db5ebedb98f5b88d601323",
-                "sha256:f7c2f4d9681e810cf40239caaca00079930a6d9ee6591139b88d592d36051d82",
-                "sha256:fde5ece4d2436b5a57c8f5f97b49b5de06a856d03959f836c957d3e609f2de7e"
+                "sha256:18faa54a3ca475bfe6255156f2f2874e9a1c8917b0004eee9f664b86ccc513d3",
+                "sha256:1e9fbaf920f0f9b4ee62aab832be3ae3968f33f24e2e3f7fbb8c6728ef1d9746",
+                "sha256:2d6ff73cc6dbbce3766b6c0bce70ce070193105d8de17aa2470009463682a8eb",
+                "sha256:36b64c2cb6ea1cc091f329c5bd34f9c01bb5da8c8e4492c709bda6a09f96808f",
+                "sha256:38a3361941eb72b82bd9a18f60c78b0df8408416f9340521df442cebfc4306e2",
+                "sha256:447db19c7efad70ff161e5e46a54ab9cc2399acebb656b6ccf63e4bc4a04b97a",
+                "sha256:44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf",
+                "sha256:4ba1f9b03d04d8cab24d6375609070580a26ce76eaed54631f03bab00a9c737b",
+                "sha256:4bc0472302919e59114aa140fd7213c2370d848a7249d09704f10f5b062031fe",
+                "sha256:50100b2726a3ca946906cbaa789dd0749f213abf0cbb877e6de72ca7aa50e1ae",
+                "sha256:5279586013b8d6582e22b6f9e30c49796966770389a9d5b85e25a4223286cd3f",
+                "sha256:6420ea8e7d2a88cb148b124429fba8cd2e0fae700a2d96eab7083c0928a85110",
+                "sha256:65ad699e0c830e2b8565afe42bd58cc972b47d829b2e0e48ad9638386d994915",
+                "sha256:6d227a865356f225591b6732430b1d1781e946893789a609bb34f59d09b8b0f8",
+                "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0",
+                "sha256:80fafd1f3eb421694857f254a9bdbacd1eb22fc6c24ca74b136679f376f97d35",
+                "sha256:83039e5ae04e5afc974f7d25ee0870a80a6bd6b7957c3aca5613ccbe0d3e72bf",
+                "sha256:8b8acc31fb8702c54625f4d5a2a6543557bec3c28a0ef638778b7ab1d1772132",
+                "sha256:9274880263256f6292ff644ca92c46d96aa7e57a75c6df3f11d636ce845a1877",
+                "sha256:ab7aa4d9ff3c3acf60d4b285ccec134167a948df02347585fdd934ebad8811b4",
+                "sha256:c41e80ddee130450dcb8829af7118f1ab69eaf8169c4bf0ee8d52b72f098dc2f",
+                "sha256:db3aedd458e7f7f21d8176f0a1d924f1ae06d725228302b872885a1c34f3119e",
+                "sha256:e7e13c40183884ec7f94eb8e0f622f08f1d5716150b8d7a134de48c6110fee85",
+                "sha256:ebfbdbd0936e178091309f6cd4fcb4decd9eab12aa513cdd9add89efa3ec2882",
+                "sha256:fd43bd9a9b9239805f406fd82ba6b106bf4838d9ef37c167d3ed70383943ade1",
+                "sha256:fd93c1a0c6c4aedc77c0fe275a9f2aba4d59b8acf88cebfc19fe3c430cfabf4f",
+                "sha256:fffb059ba3bced6f6725961ba758649261d85ed6ce670d3e3b0a26e81cf1aa8d"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
-            "version": "==3.5.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==3.6.1"
         },
         "pystac": {
             "extras": [
@@ -3084,249 +2813,272 @@
         },
         "pyyaml": {
             "hashes": [
-                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
-                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
-                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
-                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
-                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
-                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
-                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
-                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
-                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
-                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
-                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
-                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
-                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
-                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
-                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
-                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
-                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
-                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
-                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
-                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
-                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
-                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
-                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
-                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
-                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
-                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
-                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
-                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
-                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
-                "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
-                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
-                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
-                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
-                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
-                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
-                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
-                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
-                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
-                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
-                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
-                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
-                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
-                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
-                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
-                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
-                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
-                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
-                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
-                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
-                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
-                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
+                "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff",
+                "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48",
+                "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086",
+                "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e",
+                "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133",
+                "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5",
+                "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484",
+                "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee",
+                "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5",
+                "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68",
+                "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a",
+                "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf",
+                "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99",
+                "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8",
+                "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85",
+                "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19",
+                "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc",
+                "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a",
+                "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1",
+                "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317",
+                "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c",
+                "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631",
+                "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d",
+                "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652",
+                "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5",
+                "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e",
+                "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b",
+                "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8",
+                "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476",
+                "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706",
+                "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563",
+                "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237",
+                "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b",
+                "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083",
+                "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180",
+                "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425",
+                "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e",
+                "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f",
+                "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725",
+                "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183",
+                "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab",
+                "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774",
+                "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725",
+                "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e",
+                "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5",
+                "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d",
+                "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290",
+                "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44",
+                "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed",
+                "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4",
+                "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba",
+                "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12",
+                "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==6.0.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==6.0.2"
         },
         "pyzmq": {
             "hashes": [
-                "sha256:01fbfbeb8249a68d257f601deb50c70c929dc2dfe683b754659569e502fbd3aa",
-                "sha256:0270b49b6847f0d106d64b5086e9ad5dc8a902413b5dbbb15d12b60f9c1747a4",
-                "sha256:03c0ae165e700364b266876d712acb1ac02693acd920afa67da2ebb91a0b3c09",
-                "sha256:068ca17214038ae986d68f4a7021f97e187ed278ab6dccb79f837d765a54d753",
-                "sha256:082a2988364b60bb5de809373098361cf1dbb239623e39e46cb18bc035ed9c0c",
-                "sha256:0aaf982e68a7ac284377d051c742610220fd06d330dcd4c4dbb4cdd77c22a537",
-                "sha256:0c0991f5a96a8e620f7691e61178cd8f457b49e17b7d9cfa2067e2a0a89fc1d5",
-                "sha256:115f8359402fa527cf47708d6f8a0f8234f0e9ca0cab7c18c9c189c194dbf620",
-                "sha256:15c59e780be8f30a60816a9adab900c12a58d79c1ac742b4a8df044ab2a6d920",
-                "sha256:1b7d0e124948daa4d9686d421ef5087c0516bc6179fdcf8828b8444f8e461a77",
-                "sha256:1c8eb19abe87029c18f226d42b8a2c9efdd139d08f8bf6e085dd9075446db450",
-                "sha256:204e0f176fd1d067671157d049466869b3ae1fc51e354708b0dc41cf94e23a3a",
-                "sha256:2136f64fbb86451dbbf70223635a468272dd20075f988a102bf8a3f194a411dc",
-                "sha256:2b291d1230845871c00c8462c50565a9cd6026fe1228e77ca934470bb7d70ea0",
-                "sha256:2c18645ef6294d99b256806e34653e86236eb266278c8ec8112622b61db255de",
-                "sha256:2cc4e280098c1b192c42a849de8de2c8e0f3a84086a76ec5b07bfee29bda7d18",
-                "sha256:2ed8357f4c6e0daa4f3baf31832df8a33334e0fe5b020a61bc8b345a3db7a606",
-                "sha256:3191d312c73e3cfd0f0afdf51df8405aafeb0bad71e7ed8f68b24b63c4f36500",
-                "sha256:3401613148d93ef0fd9aabdbddb212de3db7a4475367f49f590c837355343972",
-                "sha256:34106f68e20e6ff253c9f596ea50397dbd8699828d55e8fa18bd4323d8d966e6",
-                "sha256:3516119f4f9b8671083a70b6afaa0a070f5683e431ab3dc26e9215620d7ca1ad",
-                "sha256:38ece17ec5f20d7d9b442e5174ae9f020365d01ba7c112205a4d59cf19dc38ee",
-                "sha256:3b4032a96410bdc760061b14ed6a33613ffb7f702181ba999df5d16fb96ba16a",
-                "sha256:3bf8b000a4e2967e6dfdd8656cd0757d18c7e5ce3d16339e550bd462f4857e59",
-                "sha256:3e3070e680f79887d60feeda051a58d0ac36622e1759f305a41059eff62c6da7",
-                "sha256:4496b1282c70c442809fc1b151977c3d967bfb33e4e17cedbf226d97de18f709",
-                "sha256:44dd6fc3034f1eaa72ece33588867df9e006a7303725a12d64c3dff92330f625",
-                "sha256:4adfbb5451196842a88fda3612e2c0414134874bffb1c2ce83ab4242ec9e027d",
-                "sha256:4b7c0c0b3244bb2275abe255d4a30c050d541c6cb18b870975553f1fb6f37527",
-                "sha256:4c82a6d952a1d555bf4be42b6532927d2a5686dd3c3e280e5f63225ab47ac1f5",
-                "sha256:5344b896e79800af86ad643408ca9aa303a017f6ebff8cee5a3163c1e9aec987",
-                "sha256:5bde86a2ed3ce587fa2b207424ce15b9a83a9fa14422dcc1c5356a13aed3df9d",
-                "sha256:5bf6c237f8c681dfb91b17f8435b2735951f0d1fad10cc5dfd96db110243370b",
-                "sha256:5dbb9c997932473a27afa93954bb77a9f9b786b4ccf718d903f35da3232317de",
-                "sha256:69ea9d6d9baa25a4dc9cef5e2b77b8537827b122214f210dd925132e34ae9b12",
-                "sha256:6b3146f9ae6af82c47a5282ac8803523d381b3b21caeae0327ed2f7ecb718798",
-                "sha256:6bcb34f869d431799c3ee7d516554797f7760cb2198ecaa89c3f176f72d062be",
-                "sha256:6ca08b840fe95d1c2bd9ab92dac5685f949fc6f9ae820ec16193e5ddf603c3b2",
-                "sha256:6ca7a9a06b52d0e38ccf6bca1aeff7be178917893f3883f37b75589d42c4ac20",
-                "sha256:703c60b9910488d3d0954ca585c34f541e506a091a41930e663a098d3b794c67",
-                "sha256:715bdf952b9533ba13dfcf1f431a8f49e63cecc31d91d007bc1deb914f47d0e4",
-                "sha256:72b67f966b57dbd18dcc7efbc1c7fc9f5f983e572db1877081f075004614fcdd",
-                "sha256:74423631b6be371edfbf7eabb02ab995c2563fee60a80a30829176842e71722a",
-                "sha256:77a85dca4c2430ac04dc2a2185c2deb3858a34fe7f403d0a946fa56970cf60a1",
-                "sha256:7821d44fe07335bea256b9f1f41474a642ca55fa671dfd9f00af8d68a920c2d4",
-                "sha256:788f15721c64109cf720791714dc14afd0f449d63f3a5487724f024345067381",
-                "sha256:7ca684ee649b55fd8f378127ac8462fb6c85f251c2fb027eb3c887e8ee347bcd",
-                "sha256:7daa3e1369355766dea11f1d8ef829905c3b9da886ea3152788dc25ee6079e02",
-                "sha256:7e6bc96ebe49604df3ec2c6389cc3876cabe475e6bfc84ced1bf4e630662cb35",
-                "sha256:80b12f25d805a919d53efc0a5ad7c0c0326f13b4eae981a5d7b7cc343318ebb7",
-                "sha256:871587bdadd1075b112e697173e946a07d722459d20716ceb3d1bd6c64bd08ce",
-                "sha256:88b88282e55fa39dd556d7fc04160bcf39dea015f78e0cecec8ff4f06c1fc2b5",
-                "sha256:8d7a498671ca87e32b54cb47c82a92b40130a26c5197d392720a1bce1b3c77cf",
-                "sha256:926838a535c2c1ea21c903f909a9a54e675c2126728c21381a94ddf37c3cbddf",
-                "sha256:971e8990c5cc4ddcff26e149398fc7b0f6a042306e82500f5e8db3b10ce69f84",
-                "sha256:9b273ecfbc590a1b98f014ae41e5cf723932f3b53ba9367cfb676f838038b32c",
-                "sha256:a42db008d58530efa3b881eeee4991146de0b790e095f7ae43ba5cc612decbc5",
-                "sha256:a72a84570f84c374b4c287183debc776dc319d3e8ce6b6a0041ce2e400de3f32",
-                "sha256:ac97a21de3712afe6a6c071abfad40a6224fd14fa6ff0ff8d0c6e6cd4e2f807a",
-                "sha256:acb704195a71ac5ea5ecf2811c9ee19ecdc62b91878528302dd0be1b9451cc90",
-                "sha256:b32bff85fb02a75ea0b68f21e2412255b5731f3f389ed9aecc13a6752f58ac97",
-                "sha256:b3cd31f859b662ac5d7f4226ec7d8bd60384fa037fc02aee6ff0b53ba29a3ba8",
-                "sha256:b63731993cdddcc8e087c64e9cf003f909262b359110070183d7f3025d1c56b5",
-                "sha256:b6907da3017ef55139cf0e417c5123a84c7332520e73a6902ff1f79046cd3b94",
-                "sha256:ba6e5e6588e49139a0979d03a7deb9c734bde647b9a8808f26acf9c547cab1bf",
-                "sha256:c1c8f2a2ca45292084c75bb6d3a25545cff0ed931ed228d3a1810ae3758f975f",
-                "sha256:ce828058d482ef860746bf532822842e0ff484e27f540ef5c813d516dd8896d2",
-                "sha256:d0a2d1bd63a4ad79483049b26514e70fa618ce6115220da9efdff63688808b17",
-                "sha256:d0cdde3c78d8ab5b46595054e5def32a755fc028685add5ddc7403e9f6de9879",
-                "sha256:d57dfbf9737763b3a60d26e6800e02e04284926329aee8fb01049635e957fe81",
-                "sha256:d8416c23161abd94cc7da80c734ad7c9f5dbebdadfdaa77dad78244457448223",
-                "sha256:dba7d9f2e047dfa2bca3b01f4f84aa5246725203d6284e3790f2ca15fba6b40a",
-                "sha256:dbf012d8fcb9f2cf0643b65df3b355fdd74fc0035d70bb5c845e9e30a3a4654b",
-                "sha256:e1258c639e00bf5e8a522fec6c3eaa3e30cf1c23a2f21a586be7e04d50c9acab",
-                "sha256:e222562dc0f38571c8b1ffdae9d7adb866363134299264a1958d077800b193b7",
-                "sha256:e4946d6bdb7ba972dfda282f9127e5756d4f299028b1566d1245fa0d438847e6",
-                "sha256:e746524418b70f38550f2190eeee834db8850088c834d4c8406fbb9bc1ae10b2",
-                "sha256:e76654e9dbfb835b3518f9938e565c7806976c07b37c33526b574cc1a1050480",
-                "sha256:e8918973fbd34e7814f59143c5f600ecd38b8038161239fd1a3d33d5817a38b8",
-                "sha256:e891ce81edd463b3b4c3b885c5603c00141151dd9c6936d98a680c8c72fe5c67",
-                "sha256:ebbbd0e728af5db9b04e56389e2299a57ea8b9dd15c9759153ee2455b32be6ad",
-                "sha256:eeb438a26d87c123bb318e5f2b3d86a36060b01f22fbdffd8cf247d52f7c9a2b",
-                "sha256:eed56b6a39216d31ff8cd2f1d048b5bf1700e4b32a01b14379c3b6dde9ce3aa3",
-                "sha256:f17cde1db0754c35a91ac00b22b25c11da6eec5746431d6e5092f0cd31a3fea9",
-                "sha256:f1a9b7d00fdf60b4039f4455afd031fe85ee8305b019334b72dcf73c567edc47",
-                "sha256:f4b6cecbbf3b7380f3b61de3a7b93cb721125dc125c854c14ddc91225ba52f83",
-                "sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad",
-                "sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc"
+                "sha256:0065026e624052a51033857e5cd45a94b52946b44533f965f0bdf182460e965d",
+                "sha256:00f39c367bbd6aa8e4bc36af6510561944c619b58eb36199fa334b594a18f615",
+                "sha256:0841633446cb1539a832a19bb24c03a20c00887d0cedd1d891b495b07e5c5cb5",
+                "sha256:08956c26dbcd4fd8835cb777a16e21958ed2412317630e19f0018d49dbeeb470",
+                "sha256:0cf1d980c969fb9e538f52abd2227f09e015096bc5c3ef7aa26e0d64051c1db8",
+                "sha256:146956aec7d947c5afc5e7da0841423d7a53f84fd160fff25e682361dcfb32cb",
+                "sha256:14bdbae02f72f4716b0ffe7500e9da303d719ddde1f3dcfb4c4f6cc1cf73bb02",
+                "sha256:18da8e84dbc30688fd2baefd41df7190607511f916be34f9a24b0e007551822e",
+                "sha256:2067e63fd9d5c13cfe12624dab0366053e523b37a7a01678ce4321f839398939",
+                "sha256:23f2fe4fb567e8098ebaa7204819658195b10ddd86958a97a6058eed2901eed3",
+                "sha256:2508bdc8ab246e5ed7c92023d4352aaad63020ca3b098a4e3f1822db202f703d",
+                "sha256:25d128524207f53f7aae7c5abdc2b63f8957a060b00521af5ffcd20986b5d8f4",
+                "sha256:26131b1cec02f941ed2d2b4b8cc051662b1c248b044eff5069df1f500bbced56",
+                "sha256:29de77ba1b1877fe7defc1b9140e65cbd35f72a63bc501e56c2eae55bde5fff4",
+                "sha256:2ae7aa1408778dc74582a1226052b930f9083b54b64d7e6ef6ec0466cfdcdec2",
+                "sha256:2b045647caf620ce0ed6c8fd9fb6a73116f99aceed966b152a5ba1b416d25311",
+                "sha256:2c0fdb7b758e0e1605157e480b00b3a599073068a37091a1c75ec65bf7498645",
+                "sha256:2f6071ec95af145d7b659dae6786871cd85f0acc599286b6f8ba0c74592d83dd",
+                "sha256:32123ff0a6db521aadf2b95201e967a4e0d11fb89f73663a99d2f54881c07214",
+                "sha256:32de51744820857a6f7c3077e620ab3f607d0e4388dfead885d5124ab9bcdc5e",
+                "sha256:362cac2423e36966d336d79d3ec3eafeabc153ee3e7a5cf580d7e74a34b3d912",
+                "sha256:3abb15df0c763339edb27a644c19381b2425ddd1aea3dbd77c1601a3b31867b8",
+                "sha256:3ddbd851a3a2651fdc5065a2804d50cf2f4b13b1bcd66de8e9e855d0217d4fcd",
+                "sha256:3e3b66fe6131b4f33d239f7d4c3bfb2f8532d8644bae3b3da4f3987073edac55",
+                "sha256:3ee5cbf2625b94de21c68d0cefd35327c8dfdbd6a98fcc41682b4e8bb00d841f",
+                "sha256:40908ec2dd3b29bbadc0916a0d3c87f8dbeebbd8fead8e618539f09e0506dec4",
+                "sha256:4350233569b4bbef88595c5e77ee38995a6f1f1790fae148b578941bfffd1c24",
+                "sha256:4437af9fee7a58302dbd511cc49f0cc2b35c112a33a1111fb123cf0be45205ca",
+                "sha256:443ebf5e261a95ee9725693f2a5a71401f89b89df0e0ea58844b074067aac2f1",
+                "sha256:472cacd16f627c06d3c8b2d374345ab74446bae913584a6245e2aa935336d929",
+                "sha256:48dee75c2a9fa4f4a583d4028d564a0453447ee1277a29b07acc3743c092e259",
+                "sha256:4d4c7fe5e50e269f9c63a260638488fec194a73993008618a59b54c47ef6ae72",
+                "sha256:4e1fcdc333afbf9918d0a614a6e10858aede7da49a60f6705a77e343fe86a317",
+                "sha256:50f0669324e27cc2091ef6ab76ca7112f364b6249691790b4cffce31e73fda28",
+                "sha256:583f73b113b8165713b6ce028d221402b1b69483055b5aa3f991937e34dd1ead",
+                "sha256:59b57e912feef6951aec8bb03fe0faa5ad5f36962883c72a30a9c965e6d988fd",
+                "sha256:5ccfcf13e80719f6a2d9c0a021d9e47d4550907a29253554be2c09582f6d7963",
+                "sha256:5e6f39ecb8eb7bfcb976c49262e8cf83ff76e082b77ca23ba90c9b6691a345be",
+                "sha256:62b5180e23e6f581600459cd983473cd723fdc64350f606d21407c99832aaf5f",
+                "sha256:63351392f948b5d50b9f55161994bc4feedbfb3f3cfe393d2f503dea2c3ec445",
+                "sha256:65e2a18e845c6ea7ab849c70db932eaeadee5edede9e379eb21c0a44cf523b2e",
+                "sha256:6c8087a3281c20b1d11042d372ed5a47734af05975d78e4d1d6e7bd1018535f3",
+                "sha256:6f0512fc87629ad968889176bf2165d721cd817401a281504329e2a2ed0ca6a3",
+                "sha256:6f34453ef3496ca3462f30435bf85f535f9550392987341f9ccc92c102825a79",
+                "sha256:6ff14c2fae6c0c2c1c02590c5c5d75aa1db35b859971b3ca2fcd28f983d9f2b6",
+                "sha256:70af4c9c991714ef1c65957605a8de42ef0d0620dd5f125953c8e682281bdb80",
+                "sha256:717960855f2d6fdc2dba9df49dff31c414187bb11c76af36343a57d1f7083d9a",
+                "sha256:732f957441e5b1c65a7509395e6b6cafee9e12df9aa5f4bf92ed266fe0ba70ee",
+                "sha256:741bdb4d96efe8192616abdc3671931d51a8bcd38c71da2d53fb3127149265d1",
+                "sha256:75bd448a28b1001b6928679015bc95dd5f172703ed30135bb9e34fc9cda0a3e7",
+                "sha256:76154943e4c4054b2591792eb3484ef1dd23d59805759f9cebd2f010aa30ee8c",
+                "sha256:76390d3d66406cb01b9681c382874400e9dfd77f30ecdea4bd1bf5226dd4aff0",
+                "sha256:7a5342110510045a47de1e87f5f1dcc1d9d90109522316dc9830cfc6157c800f",
+                "sha256:7c506a51cb01bb997a3f6440db0d121e5e7a32396e9948b1fdb6a7bfa67243f4",
+                "sha256:7dfabc180a4da422a4b349c63077347392463a75fa07aa3be96712ed6d42c547",
+                "sha256:809673947e95752e407aaaaf03f205ee86ebfff9ca51db6d4003dfd87b8428d1",
+                "sha256:8285b25aa20fcc46f1ca4afbc39fd3d5f2fe4c4bbf7f2c7f907a214e87a70024",
+                "sha256:85f2d2ee5ea9a8f1de86a300e1062fbab044f45b5ce34d20580c0198a8196db0",
+                "sha256:8d042d6446cab3a1388b38596f5acabb9926b0b95c3894c519356b577a549458",
+                "sha256:8d4234f335b0d0842f7d661d8cd50cbad0729be58f1c4deb85cd96b38fe95025",
+                "sha256:8eaffcd6bf6a9d00b66a2052a33fa7e6a6575427e9644395f13c3d070f2918dc",
+                "sha256:92eca4f80e8a748d880e55d3cf57ef487692e439f12d5c5a2e1cce84aaa7f6cb",
+                "sha256:9498ac427d20d0e0ef0e4bbd6200841e91640dfdf619f544ceec7f464cfb6070",
+                "sha256:9521b874fd489495865172f344e46e0159095d1f161858e3fc6e28e43ca15160",
+                "sha256:9763a8d3f5f74ef679989b373c37cc22e8d07e56d26439205cb83edb7722357f",
+                "sha256:9f380d5333fc7cd17423f486125dcc073918676e33db70a6a8172b19fc78d23d",
+                "sha256:a4cbecda4ddbfc1e309c3be04d333f9be3fc6178b8b6592b309676f929767a15",
+                "sha256:a7db05d8b7cd1a8c6610e9e9aa55d525baae7a44a43e18bc3260eb3f92de96c6",
+                "sha256:a8234571df7816f99dde89c3403cb396d70c6554120b795853a8ea56fcc26cd3",
+                "sha256:a83653c6bbe5887caea55e49fbd2909c14b73acf43bcc051eb60b2d514bbd46e",
+                "sha256:a880240597010914ffb1d6edd04d3deb7ce6a2abf79a0012751438d13630a671",
+                "sha256:aa79c528706561306938b275f89bb2c6985ce08469c27e5de05bc680df5e826f",
+                "sha256:abad7b897e960d577eb4a0f3f789c1780bc3ffe2e7c27cf317e7c90ad26acf12",
+                "sha256:af690ea4be6ca92a67c2b44a779a023bf0838e92d48497a2268175dc4a505691",
+                "sha256:b1bb952d1e407463c9333ea7e0c0600001e54e08ce836d4f0aff1fb3f902cf63",
+                "sha256:b8e153f5dffb0310af71fc6fc9cd8174f4c8ea312c415adcb815d786fee78179",
+                "sha256:bc5df31e36e4fddd4c8b5c42daee8d54d7b529e898ac984be97bf5517de166a7",
+                "sha256:bc904e86de98f8fc5bd41597da5d61232d2d6d60c4397f26efffabb961b2b245",
+                "sha256:bc994e220c1403ae087d7f0fa45129d583e46668a019e389060da811a5a9320e",
+                "sha256:be3fc2b11c0c384949cf1f01f9a48555039408b0f3e877863b1754225635953e",
+                "sha256:c11a95d3f6fc7e714ccd1066f68f9c1abd764a8b3596158be92f46dd49f41e03",
+                "sha256:c513d829a548c2d5c88983167be2b3aa537f6d1191edcdc6fcd8999e18bdd994",
+                "sha256:c5248e6e0fcbbbc912982e99cdd51c342601f495b0fa5bd667f3bdbdbf3e170f",
+                "sha256:c70dab93d98b2bf3f0ac1265edbf6e7f83acbf71dabcc4611889bb0dea45bed7",
+                "sha256:cc09b1de8b985ca5a0ca343dd7fb007267c6b329347a74e200f4654268084239",
+                "sha256:cc109be2ee3638035d276e18eaf66a1e1f44201c0c4bea4ee0c692766bbd3570",
+                "sha256:cc8d655627d775475eafdcf0e49e74bcc1e5e90afd9ab813b4da98f092ed7b93",
+                "sha256:ce05841322b58510607f9508a573138d995a46c7928887bc433de9cb760fd2ad",
+                "sha256:cf4be7460a0c1bc71e9b0e64ecdd75a86386ca6afaa36641686f5542d0314e9d",
+                "sha256:cf4e283f97688d993cb7a8acbc22889effbbb7cbaa19ee9709751f44be928f5d",
+                "sha256:d0da97e65ee73261dba70469cc8f63d8da3a8a825337a2e3d246b9e95141cdd0",
+                "sha256:d3df226ab7464684ae6706e20a5cbab717c3735a7e409b3fa598b754d49f1946",
+                "sha256:d74b925d997e4f92b042bdd7085cd0a309ee0fd7cb4dc376059bbff6b32ff34f",
+                "sha256:db1b7e2b50ef21f398036786da4c153db63203a402396d9f21e08ea61f3f8dba",
+                "sha256:de6f384864a959866b782e6a3896538d1424d183f2d3c7ef079f71dcecde7284",
+                "sha256:def7ae3006924b8a0c146a89ab4008310913fa903beedb95e25dea749642528e",
+                "sha256:e03be7ed17836c9434cce0668ac1e2cc9143d7169f90f46a0167f6155e176e32",
+                "sha256:e790602d7ea1d6c7d8713d571226d67de7ffe47b1e22ae2c043ebd537de1bccb",
+                "sha256:e80345900ae241c2c51bead7c9fa247bba6d4b2a83423e9791bae8b0a7f12c52",
+                "sha256:ebef7d3fe11fe4c688f08bc0211a976c3318c097057f258428200737b9fff4da",
+                "sha256:ec8fe214fcc45dfb0c32e4a7ad1db20244ba2d2fecbf0cbf9d5242d81ca0a375",
+                "sha256:f0a45102ad7ed9f9ddf2bd699cc5df37742cf7301111cba06001b927efecb120",
+                "sha256:f1483d4975ae1b387b39bb8e23d1ff32fe5621aa9e4ed3055d05e9c5613fea53",
+                "sha256:f218179c90a12d660906e04b25a340dd63e9743000ba16232ddaf46888f269da",
+                "sha256:f66dcb6625c002f209cdc12cae1a1fec926493cd2262efe37dc6b25a30cea863",
+                "sha256:fc657577f057d60dd3642c9f95f28b432889b73143140061f7c1331d02f03df6",
+                "sha256:fcb90592c5d5c562e1b1a1ceccf6f00036d73c51db0271bf4d352b8d6b31d468",
+                "sha256:fe73d7c89d6f803bed122135ff5783364e8cdb479cf6fe2d764a44b6349e7e0f",
+                "sha256:ffecc43b3c18e36b62fcec995761829b6ac325d8dd74a4f2c5c1653afbb4495a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==26.0.3"
+            "version": "==26.1.1"
         },
         "pyzstd": {
             "hashes": [
-                "sha256:00954290d6d46ab13535becbbc1327c56f0a9c5d7b7cf967e6587c1395cade42",
-                "sha256:06b9dfd615fb5635c05153431e520954a0e81683c5a6e3ed1134f60cc45b80f1",
-                "sha256:094cec5425097ae1f9a40bb02de917d2274bfa872665fe2e5b4101ee94d8b31d",
-                "sha256:0eadba403ec861fa4c600ad43dbd8ac17b7c22a796d3bd9d92918f4e8a15a6e8",
-                "sha256:10143cad228ebeb9eda7793995b2d0b3fef0685258d9b794f6320824302c47d7",
-                "sha256:19deddb2975af861320fd7b68196fbb2a4a8500897354919baf693702786e349",
-                "sha256:1b8db95f23d928ba87297afe6d4fff21bbb1af343147ff50c174674312afc29d",
-                "sha256:1c0bbdb3ae1c300941c1f89219a8d09d142ddb7bfc78e61da80c8bdc03c05be8",
-                "sha256:1c4cdb0e407bec2f3ece10275449822575f6634727ee1a18e87c5e5a7b565bb1",
-                "sha256:20259fa302f1050bd02d78d93db78870bed385c6d3d299990fe806095426869f",
-                "sha256:26e42ccb76a53c1b943021eeb0eb4d78f46093c16e4e658a7204c838d5b36df0",
-                "sha256:2820b607be0346b3e24b097d759393bd4bcccc0620e8e825591061a2c3a0add5",
-                "sha256:29ca6db3fb72d17bcec091b9ba485c715f63ca00bfcd993f92cb20037ae98b25",
-                "sha256:33bc6f6048f7f7fc506e6ad03fb822a78c2b8209e73b2eddc69d3d6767d0385c",
-                "sha256:33efaf2cc4efd2b100699d953cd70b5a54c3ca912297211fda01875f4636f655",
-                "sha256:349d643aeb8d7d9e0a407cef29d6210afbe646cc19b4e237456e585591eda223",
-                "sha256:35962bc10480aebd5b32fa344430bddd19ef384286501c1c8092b6a9a1ee6a99",
-                "sha256:35ba0ee9d6d502da2bc01d78d22f51a1812ff8d55fb444447f7782f5ce8c1e35",
-                "sha256:363c11a4d60fa0e2e7437f7494291c24eaf2752c8d8e3adf8f92cb0168073464",
-                "sha256:3a57f2a0531ad2cd33bb78d8555e85a250877e555a68c0add6308ceeca8d84f1",
-                "sha256:3c3181a462cdb55df5ddeffe3cf5223cda36c81feceeb231688af08d30f11022",
-                "sha256:3c5f28a145677431347772b43a9604b67691b16e233ec7a92fc77fc5fb670608",
-                "sha256:3dd4592c2fca923041c57aa2bfe428de14cc45f3a00ab825b353160994bc15e7",
-                "sha256:3f661848fa1984f3b17da676c88ccd08d8c3fab5501a1d1c8ac5abece48566f2",
-                "sha256:44f818ea8c191285365a0add6fc03f88225f1fdcff570dc78e9f548444042441",
-                "sha256:48037009be790fca505a62705a7997eef0cd384c3ef6c10a769734660245ee73",
-                "sha256:48ff680078aec3b9515f149010981c7feeef6c2706987ac7bdc7cc1ea05f8f7d",
-                "sha256:493edd702bc16dae1f4d76461688714c488af1b33f5b3a77c1a86d5c81240f9e",
-                "sha256:4a765c5fc05fe1c843863cc3723e39e8207c28d9a7152ee6d621fa3908ef4880",
-                "sha256:4d5c98986d774e9321fb1d4fe0362658560e14c1d7afbe2d298b89a24c2f7b4f",
-                "sha256:586538aa2a992a55c10d88c58166e6023968a9825719bce5a09397b73eea658f",
-                "sha256:5c43e2222bbbe660dea8fe335f5c633b3c9ed10628a4b53a160ddd54d15cffc2",
-                "sha256:5d0db66651ed5a866a1452e7a450e41a5ec743abbeea1f1bc85ef7c64f5f6b8f",
-                "sha256:65a55aac43a685b7d2b9e7c4f9f3768ad6e0d5f9ad7698b8bf9124fbeb814d43",
-                "sha256:691cadd48f225097a2588e7db492ac88c669c061208749bc0200ee39e4425e32",
-                "sha256:6aa796663db6d1d01ebdcd80022de840005ae173e01a7b03b3934811b7ae39bc",
-                "sha256:708c442f8f6540ffad24a894bdea3c019250e02dcdbd0fbd27fc977b1a88b4f2",
-                "sha256:74521d819ceea90794aded974cc3024c65c094050e6c4a6f4b7478af3461e3ad",
-                "sha256:75f4363157777cbcbbd14ff823388fddfca597d44c77c27473c4c4000d7a5c99",
-                "sha256:75f5e862e1646f1688e97f4aa69988d6589a1e036f081e98a3f202fa4647e69b",
-                "sha256:76697baa4d9fd621bd5b99719d3b55fadeb665af9a49523debfc9ae5fbefef13",
-                "sha256:784f7f87ae2e25459ef78282fbe9f0d2fec9ced84e4acb5d28621a0db274a13b",
-                "sha256:78f5e65eb15d93f687715be9241c8b55d838fba9b7045d83530f8831544f1413",
-                "sha256:796a29cbb6414b6cb84d8e7448262ba286847b946de9a149dec97469a4789552",
-                "sha256:79f4c9f1d7906eb890dafae4820f69bd24658297e9ebcdd74867330e8e7bf9b0",
-                "sha256:7a82cd4e772e5d1400502d68da7ecd71a6f1ff37243017f284bee3d2106a2496",
-                "sha256:80741b9f18149264acb639287347cfc6eecff109b5c6d95dbf7222756b107b57",
-                "sha256:80962ff81a3389b5579d1206bea1bb48da38991407442d2a9287f6da1ccb2c80",
-                "sha256:84135917c99476c6abeee420ffd005a856d8fde0e5f585b0c484d5923392035b",
-                "sha256:8436ce4fa7e7ddaa8d29717fd73e0699883ef6e78ef4d785c244779a7ad1942b",
-                "sha256:89187af1ca5a9b65c477817e0fe7e411f4edd99e5575aaaef6a9e5ff62028854",
-                "sha256:8a2d5a8b74db3df772bb4f230319241e73629b04cb777b22f9dcd2084d92977a",
-                "sha256:8e4cf6d11427d43734e8cb246ecfb7af169983ef796b415379602ea0605f5116",
-                "sha256:8e51d69446d96f5767e0f1b0676341d5d576c151dfe3dd14aff7a163db1b4d7c",
-                "sha256:8ef9fa7fe28dd6b7d09b8be89aea4e8f2d18b23a89294f51aa48dbc6c306a039",
-                "sha256:94fe8c5f1f11397b5db8b1850168e5bed13b3f3e1bc36e4292819d85be51a63c",
-                "sha256:983ea93ed937d329c88ef15d5e3b09e32372590c1a80586b2013f17aed436cb8",
-                "sha256:994a21a75d7b2602a78c2f88f809427ce1051e43af7aad6cda524ccdc859354e",
-                "sha256:9c68761529a43358151ac507aeb9c6b7c1a990235ce7b7d41f8ea62c62d4679e",
-                "sha256:a4cf0fed2d5c9de3da211dceff3ed9a09b8f998f7df57da847145863a786454b",
-                "sha256:a4e12b6702481ace7071357c1b81b9faf6f660da55ff9ccd6383fed474348cc6",
-                "sha256:acfe529ff44d379ee889f03c2d353f94b1f16c83a92852061f9672982a3ef32d",
-                "sha256:b0fa1ef68839d99b0c0d66fe060303f7f2916f021289a7e04a818ef9461bbbe1",
-                "sha256:b3cc09eecd318310cfd6e7f245248cf16ca014ea5903580d72231d93330952de",
-                "sha256:b5d8a3263b7e23a3593eb4fcc5cc77e053c7d15c874db16ce6ee8b4d94f8d825",
-                "sha256:bbeaa0af865427405a1c0e8c65841a23de66af8ca5d796522f7b105386cd8522",
-                "sha256:bc5e630db572362aef4d8a78f82a40e2b9756de7622feb07031bd400a696ad78",
-                "sha256:bd27ab78269148c65d988a6b26471d621d4cc6eed6b92462b7f8850162e5c4f2",
-                "sha256:c34c06a6496b4aacdab03133671dd5638417bda09a1f186ba1a39c1dbd1add24",
-                "sha256:c48b4368b832233205a74e9f1dfe2647d9bc49ea8357b09963fd5f15062bdd0a",
-                "sha256:c8c675edd26cd2531163e51dcb3c7c73145e2fa3b77a1ff59ce9ed963ff56017",
-                "sha256:c9c1ede5c4e35b059e8734dfa8d23a59b8fcfe3e0ece4f7d226bc5e1816512c9",
-                "sha256:ca9f1f6bd487c9b990e509c17e0a701f554db9e77bd5121c27f1db4594ac4c0a",
-                "sha256:d14862ce066da0494e0f9466afc3b8fcd6c03f7250323cf8ef62c67158c77e57",
-                "sha256:d1e6ae36c717abd32b55a275d7fbf9041b6de3a103639739ec3e8c8283773fb3",
-                "sha256:d7d5888e206190d36fbffed6d7e9cacd79e64fd34e9a07359e16862973d90b33",
-                "sha256:d897ec18822e348f8ef9a17e421716ed224a3726fde806aae04469fec8f0ac9d",
-                "sha256:e0f5a1865a00798a74d50fcc9956a3d7fa7413cbc1c6d6d04833d89f36e35226",
-                "sha256:e31e0d2023b693ca530d95df7cff8d736f66b755018398bc518160f91e80bd0a",
-                "sha256:e8eae552db2aa587c986f460915786bf9058a88d831d562cadba01f3069736a9",
-                "sha256:e9f22fb00bfcca4b2e0b36afd4f3a3194c1bc93b2a76e51932ccfd3b6aa62501",
-                "sha256:ef883837c16c076f11da37323f589779806073eeacaef3912f2da0359cb8c2cf",
-                "sha256:f47aada7fdc6bcad8ec4ee4ff00a8d2d9a0e05b5516df3f304afbf527b026221",
-                "sha256:f4f27e083a63b9463fd2640065af1b924f05831839f23d936a97c4f510a54f6b",
-                "sha256:f560d24557bbc54eb1aa01ee6e587d4d199b785593462567ddf752de3c1c4974",
-                "sha256:fa219d5d6124f1623b39f296a1fcc4cac1d8c82f137516bd362a38c16adcd92b",
-                "sha256:fb70083bf00426194a85d69939c52b1759462873bf6e4d62f481e2bc3e642ea1",
-                "sha256:fd43a0ae38ae15223fb1057729001829c3336e90f4acf04cf12ebdec33346658",
-                "sha256:ff99a11dd76aec5a5234c1158d6b8dacb61b208f3f30a2bf7ae3b23243190581"
+                "sha256:036d76e26300bc03cf05108a019fb0dd0a40ee6ed40128ead1c953fc603fba68",
+                "sha256:070434349fdd2fd56207a82a146c89a50811c5e0f767ac00d09f513919335f6f",
+                "sha256:089f3d04430b1044fccedbd4e88bd5429cd1220cf523b8841ead0127d8eedd9f",
+                "sha256:0a456ca431e4968a31c350004eca7957490f51245be8f3b44e49a9f143251312",
+                "sha256:0cff110d121598f9eb638ce15393fece65bb5fac9a9d38c60fc5cb1ac8631465",
+                "sha256:123aba9d2bfdc1840b1fadd386c0095130948c10cd5a4f0acc48368d61448c9e",
+                "sha256:1274d766f8a2655f99bd8f2ebc8f109ccf640734e941ca484ef03e275441e220",
+                "sha256:13ab3b66c660438cf9031543a1cb9a4c7adde6b58b65e05783d32044178e871c",
+                "sha256:15a242d03c1516e1325d41a43b05c95abce0306d6f9782408b44f6225fadea9b",
+                "sha256:1751fabc263654b3b4fbfb2729f63d6b3a51bf498bfbb1851ed332cd1b9a02e8",
+                "sha256:1f00c7d40304329fbebbe9891cd2b144b09844876fe65a8bcfef71d80d417214",
+                "sha256:22f7b5d93b5e7d3b3bd4a0f665b2bfab61a9cc78cb19b4f9d2faa454ae19133e",
+                "sha256:2325ff41ff4bea19065894244c4dade5ae6b40df6e9def9dd4bc6e4c81edabf1",
+                "sha256:234423432d2e66328bdb06121aad3477bb97e200141a863aba0d1a14ff30b0cb",
+                "sha256:2595819277b46d195565861f2966f58908444c7787da1ec45ea56390650013a6",
+                "sha256:26ddb50c2767ebf411f2b28e698d61d1671c87e943dac81b2a6e89529052c8ad",
+                "sha256:28b33701e0a5bdb7aa96229ef7f680442894a4be3dfb39daf2fbae805778ade7",
+                "sha256:2df7e255b4aef73d7f8b11301bb6e39cf43e46cf80aa885ff7c1570565cf2398",
+                "sha256:3630a47b7d418e65521a45afbea5d77a825e4fb675fdf884eff42e6ce3230f91",
+                "sha256:397ca9ea892fece84fbbc5847ce46d16ee03501de3bbc6fb1f9b69bb14fe47a3",
+                "sha256:3b1ce3eae59fd7093a05b8f073c7dce4795cccbf5987371fda5931b38fa9a567",
+                "sha256:3b23295a6aedc71e5318b7e490f2ed1ea3fda6b31f2b5957c8da49a5aac7aa81",
+                "sha256:3b8af1f24361728cb0abeb447204015b2af016bfaf61d55b7c7bc44edc50348b",
+                "sha256:3cba92b21b12bff45c0393e022ca4e6029aa5d4d3f11d1d9f05ca9a13245d325",
+                "sha256:3fcf498488cf2a866142a35d0c14c021a58c7d96b25bafd13c72676458912011",
+                "sha256:46feda6257df4cde7dda55811851c2096dea7b38dcd601099acb95d7acdc795f",
+                "sha256:487efbe3da2b879c5835e0d762bc8ea69e6bd765d31d6de32b20146bc7f5b2cc",
+                "sha256:4ae871967fc080a24118135dd8465339cf69c990fdea8755aef8806c5ebfb0e3",
+                "sha256:4d9f8aa524f99f593ebf38639e6d066984b0f9ed084d45ee8877761d1ee6aa48",
+                "sha256:53ae4ac03c286896b2a6741c9069afd80e432526d267f900420d8083f8ab1f78",
+                "sha256:55e6dd2667a58ca92924f7ced5ac2c53ed26e07c453cfbde50693bf58c4c7b5b",
+                "sha256:593a4ec2f639a80523c6d8cb6a3f97899a4b3db4eadb768039dbd61fed4fe675",
+                "sha256:59d016a105efd11db4305d43712ca2aab5e4f7dc73f42cc6324bc8f1b0ce2402",
+                "sha256:61450162fb86504d16c00558976a4864ae12537e362f7346a0a79594ec2eb491",
+                "sha256:6467ba4ccbc1e09793e763c602079bb5b95813dcb2b0d2afffb40130b5927e69",
+                "sha256:64ae91c0c19160cc0b95d33a5802e708ab15f11213f8043906d484b6062a80b3",
+                "sha256:65683cb63d631b159e02738376987c26106b37a1345105c52067441e6259cf87",
+                "sha256:65fc3e12ad4d3ddc1f408e31ad2b70e110bbb7f835e4737f0f7b99ed1ff110cd",
+                "sha256:68ea4cbeb5fa722222e8607ed22eab7723dfe8f502cbdaaab0989fc47f2fe4e6",
+                "sha256:748ea21376016b77f93eb6e5d3fdf158620a27d36d2a05cb319f3e7b8b1943a5",
+                "sha256:763e084e0a7273d81d4bd68c4c89d642e3a447e30d1108d3dc0d0ec07a3ad01c",
+                "sha256:7bb570705a39e2a78619e6134a68be00ccd04398d782827180c0d1df79fc88c1",
+                "sha256:7cdc3c293ab30ea141789a4454a4fd7b7858e005f6d2f61113d239a20d9bafd4",
+                "sha256:81567ffe7f5ba6d6612399a82191448ba4f7780c96f2643bea36403a49462e0b",
+                "sha256:83e770056823f8add3be33599219aa962c36f60eff24fa815579bc65bb053499",
+                "sha256:84059dfa917a2704e04776f26d5105bebc5019fc4f13379b44e71e57b575fc28",
+                "sha256:8b54ea942847b6e2f842f8b524f0c4dcc199f99b39420e06262cbcf25cb24363",
+                "sha256:8c7507175f8d3f48358e28001a19242d3d4df819b6cd4cbc4f0fbe6f9dee9427",
+                "sha256:8f3b74f42ac91dfcd5b3e8dfa691714e23c4bb3931070fdc134dbbaa2c92c51e",
+                "sha256:96c04f3ef21f8c84672468358001b1f78b18f62a1b6af202e9fe0c71d0cd85f8",
+                "sha256:9865ffbff114ad4411c9794deb1cbe57a03902f82a2671c23929a2628fd70bbc",
+                "sha256:a426a253413ede9dad34fffde2d533950aa6aab82d0e9c7c7660168e323c43dc",
+                "sha256:a4f2f1bd58361e4994e0fed4223038554bdb61644b2449f50f8c2960a8aeffc4",
+                "sha256:a5633a0e9ac780a5577fc5dee3d6d05b8edf2f3d646ffe2c71e065d62a1b538c",
+                "sha256:aadbab6d6b79bd37697c3de28d4c2cbac3545ae9622be2f86ae5e426c6e1b192",
+                "sha256:acbf3d01f79be0bd284ab316e33d6a3fceab478a932ce93de7275d7d9547b9be",
+                "sha256:ad8686ae57a59432860907e4c62d4b08b98d2330a129928145d797eda118da7b",
+                "sha256:b73e9d8ae8eca8dd600d54408584b625503761ad6b0e481e47e270a19e968141",
+                "sha256:b867f620b7402e0befa4b5e7eaa79693be099a52304f31bfc1006cdc915d21c7",
+                "sha256:bb26734a5cda4b5e58b33c5fe20aee697fb9ad8dd72999bc71d7df09783f44db",
+                "sha256:bb49c7854c6c56d9d41abdcd970b5fec2681a6a74f390b6f8f8fe9d1ca1f8530",
+                "sha256:bc09abfd5e281dba33a1cfdc653ece69fc239ad2c6cebd99506facbcb2669c91",
+                "sha256:bc6326d017c618e7897c2f529dc71100403c0dfdbc523cd6c62f6ba1ed9f23f1",
+                "sha256:c088b57288a8e1818c032ed7e3e3e573b3fe8fad698d02740a1583f55458a73f",
+                "sha256:c78ca31b0d83f77ab6ff041808304f51672f925683ffc3a1a866469f1678fc10",
+                "sha256:c870947722ae4c7de8e2d259690041f8b3332b1d75b4c3ca2caf17b170d10be3",
+                "sha256:ca952ce3388b5f7ee78931733ec41c8939482b466882e41d79a9a8c1387dd398",
+                "sha256:cb00ce5e9a88e27f27db3ff4f4c6080c4158ad848d620b68d48bbc413d99f0ef",
+                "sha256:cd3d79a74f863ec277ee3297b43f30178aa1a014eba54c286ea48f21248e525e",
+                "sha256:cd62933e3a11f7dd6c892fa38c67e7ba45de17cae08f1355bf07b31e631a36f3",
+                "sha256:cdedcddd851139605b0dbc9b9ed5767052f67c02fa98c66b0a0bd4c1bce0ba49",
+                "sha256:cf08a0fa9af8d690a41b9b7db6b8ae174ba2ac42b5463993c2cd3d144a094644",
+                "sha256:cf0dec2978f9bc622c4daa48dd286f3f7e6ab196b1e17c46437abb6d4a968201",
+                "sha256:d9175bf699ec234189dd5549b4ededc676b66010e2eef5b3170501a17d765cf5",
+                "sha256:dc0e4d4c832adcd3c25a5d5b5bf0aa05bc25a279b8e8356eb2b95975b2a67fa0",
+                "sha256:deea37b1618f31fd2618be0aad42bb5bafcdddc24df9fc18c71071314239e3a2",
+                "sha256:eb1d26304c41cc07a87b1b85f4bf61a0f853368e0c00bb700dc7245971dedd53",
+                "sha256:eb2e62ed3d04fed425e009e9948c5e1478665475c5a6ca52d9f02295db7cffb1",
+                "sha256:ed50c08233878c155c73ab2622e115cd9e46c0f1c2e2ddd76f2e7ca24933f195",
+                "sha256:ef5943a471b0d51cdb4eb05187b4be81cd6c95349e73818c4b959f60a05dfccd",
+                "sha256:efeac4bf8a12cc0a1284164e77cca85727f8a5ec20328cef2e5c72f8eabf7630",
+                "sha256:f0a685bea6ba4e965d0de77cda3e380efeb144bb4fa0eff362626b4cdec71814",
+                "sha256:f42bb898d5335e91d4575758cb11f68308756061d1eff042c7c4daa09cc560ba",
+                "sha256:f4725fb00bf06bd674f73f37cb168dd73ca67e68287207fece340e7425f0754d",
+                "sha256:f5faf5894b58f38491ecb458e6f4032ae0bbebea64dfeff86abc6c6176829ac3",
+                "sha256:f6a7996f56abc23ad96bb73aea363720a1fca91a99822f8267bb5d3c4b7af7dc",
+                "sha256:f949a5375ca8a546059193400b2e7c70f1a10de58bd87d35bdc31c6230e47ab0",
+                "sha256:fd568900f5ce7e2ced7928342b7cbc234c2b5648cff6a84bbf5e713377fce4f5",
+                "sha256:ffa579210ae03a0aeeff86d492ff26acd358ec1daea8553beaac5f1ba774991d"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.16.0"
+            "version": "==0.16.1"
         },
         "qtconsole": {
             "hashes": [
@@ -3346,36 +3098,40 @@
         },
         "rasterio": {
             "hashes": [
-                "sha256:0883a38bd32e6a3d8d85bac67e3b75a2f04f7de265803585516883223ddbb8d1",
-                "sha256:1321372c653a36928b4e5e11cbe7f851903fb76608b8e48a860168b248d5f8e6",
-                "sha256:23a8d10ba17301029962a5667915381a8b4711ed80b712eb71cf68834cb5f946",
-                "sha256:50785004d7adf66cf96c9c3498cf530ec91292e9349e66e8d1f1183085ee93b1",
-                "sha256:69fdc712e9c79e82d00d783d23034bb16ca8faa18856e83e297bb7e4d7e3e277",
-                "sha256:76b6bd4b566cd733f0ddd05ba88bea3f96705ff74e2e5fab73ead2a26cbc5979",
-                "sha256:83f764c2b30e3d07bea5626392f1ce5481e61d5583256ab66f3a610a2f40dec7",
-                "sha256:8a584fedd92953a0580e8de3f41ce9f33a3205ba79ea58fff8f90ba5d14a0c04",
-                "sha256:92f0f92254fcce57d25d5f60ef2cf649297f8a1e1fa279b32795bde20f11ff41",
-                "sha256:9f3f901097c3f306f1143d6fdc503440596c66a2c39054e25604bdf3f4eaaff3",
-                "sha256:a732f8d314b7d9cb532b1969e968d08bf208886f04309662a5d16884af39bb4a",
-                "sha256:b72fc032ddca55d73de87ef3872530b7384989378a1bc66d77c69cedafe7feaf",
-                "sha256:c8b90eb10e16102d1ab0334a7436185f295de1c07f0d197e206d1c005fc33905",
-                "sha256:cb3288add5d55248f5d48815f9d509819ba8985cd0302d2e8dd743f83c5ec96d",
-                "sha256:d03e2fcd8f3aafb0ea1fa27a021fecc385655630a46c70d6ba693675c6cc3830",
-                "sha256:e73339e8fb9b9091a4a0ffd9f84725b2d1f118cf51c35fb0d03b94e82e1736a3",
-                "sha256:eaaeb2e661d1ffc07a7ae4fd997bb326d3561f641178126102842d608a010cc3"
+                "sha256:0bbd62b45a35cab53cb7fe72419e823e47ab31ee2d055af8e21dc7f37fe5ed6c",
+                "sha256:287e8d0d0472c778aa0b6392e9c00894a80f2bace28fa6eddb76c0a895097947",
+                "sha256:2ef27c3eff6f44f8b5d5de228003367c1843593edf648d85c0dc1319c00dc57d",
+                "sha256:30f27e309a14a70c821d10a0ea18b110968dc2e2186b06a900aebd92094f4e00",
+                "sha256:3a9c4fb63e050e11bcd23e53f084ca186b445f976df1f70e7abd851c4072837f",
+                "sha256:450f2bd45335308829da90566fbcbdb8e8aa0251a9d1f6ebb60667855dfb7554",
+                "sha256:505b3e659eb3b137192c25233bf7954bc4997b1a474bae9e129fbd5ac2619404",
+                "sha256:73ea4d0e584f696ef115601bbb97ba8d2b68a67c2bb3b40999414d31b6c7cf89",
+                "sha256:7a22c0e0cf07dbed6576faf9a49bc4afa1afedd5a14441b64a3d3dd6d10dc274",
+                "sha256:7c7ddca79444fd3b933f4cd1a1773e9f7839d0ce5d76e600bdf92ee9a79b95f8",
+                "sha256:7e653968f64840654d277e0f86f8666ed8f3030ba36fa865f420f9bc38d619ee",
+                "sha256:97d867cada29f16cb83f1743217f775f8b982676fcdda77671d25abb26698159",
+                "sha256:a420e5f25108b1c92c5d071cfd6518b3766f20a6eddb1b322d06c3d46a89fab6",
+                "sha256:c711b497e9ef0c4f5e1c01e34ba910708e066e1c4a69c25df18d1bcc04481287",
+                "sha256:cbb2eea127328302f9e3158a000363a7d9eea22537378dee4f824a7fa2d78c05",
+                "sha256:ce182c735b4f9e8735d90600607ecab15ef895eb8aa660bf665751529477e326",
+                "sha256:d1ac85857144cb8075e332e9d908b65426d30ddc1f59f7a04bcf6ed6fd3c0d47",
+                "sha256:d29d30c2271fa265913bd3db93fa213d3a0894362ec704e7273cf30443098a90",
+                "sha256:e6eece6420d7d6ef9b9830633b8fcd15e86b8702cb13419abe251c16ca502cf3",
+                "sha256:ef8a496740df1e68f7a3d3449aa3be9c3210c22f4bb78a4a9e1c290183abd9b1",
+                "sha256:f9cd757e11cfb07ef39b1cc79a32497bf22aff7fec41fe330b868cb3043b4db5"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==1.3.6"
+            "version": "==1.3.10"
         },
         "rasterstats": {
             "hashes": [
-                "sha256:00014f422160f6047d3431c9132ebe3d356021ca8e22c6f40d04764f98a60795",
-                "sha256:29389bfcbeac1a4206aba6e1d795058ec8a64d560efad48156c27fad97c2e09a"
+                "sha256:066c44feb6f3936804a0c79d112271fa5bf5de0d5058823ab5c1e0047ab7bbbc",
+                "sha256:d9196f8672fc56e23aa606f34a4e5449fdbcf8e0fb5bbb00f6c05b454e020994"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==0.18.0"
+            "version": "==0.19.0"
         },
         "referencing": {
             "hashes": [
@@ -3439,116 +3195,120 @@
         },
         "rioxarray": {
             "hashes": [
-                "sha256:a98ea9306739f119b63ffc2245f5d7d23ca1638b99c50ca282d932901f9272e8",
-                "sha256:b0f4a8917bde79c15b1507296bd256de1a33923867f5e2e3078bda2b6ce3673f"
+                "sha256:46c29938827fff268d497f7ae277077066fcfbac4e53132ed3d4e2b96455be62",
+                "sha256:cf31a0674b9e6abd0cc2f0b4d847869eb23065835827d0018d17bde8047d448c"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==0.16.0"
+            "version": "==0.17.0"
         },
         "rpds-py": {
             "hashes": [
-                "sha256:0121803b0f424ee2109d6e1f27db45b166ebaa4b32ff47d6aa225642636cd834",
-                "sha256:06925c50f86da0596b9c3c64c3837b2481337b83ef3519e5db2701df695453a4",
-                "sha256:071d4adc734de562bd11d43bd134330fb6249769b2f66b9310dab7460f4bf714",
-                "sha256:1540d807364c84516417115c38f0119dfec5ea5c0dd9a25332dea60b1d26fc4d",
-                "sha256:15e65395a59d2e0e96caf8ee5389ffb4604e980479c32742936ddd7ade914b22",
-                "sha256:19d02c45f2507b489fd4df7b827940f1420480b3e2e471e952af4d44a1ea8e34",
-                "sha256:1c26da90b8d06227d7769f34915913911222d24ce08c0ab2d60b354e2d9c7aff",
-                "sha256:1d16089dfa58719c98a1c06f2daceba6d8e3fb9b5d7931af4a990a3c486241cb",
-                "sha256:1dd46f309e953927dd018567d6a9e2fb84783963650171f6c5fe7e5c41fd5666",
-                "sha256:2575efaa5d949c9f4e2cdbe7d805d02122c16065bfb8d95c129372d65a291a0b",
-                "sha256:3208f9aea18991ac7f2b39721e947bbd752a1abbe79ad90d9b6a84a74d44409b",
-                "sha256:329c719d31362355a96b435f4653e3b4b061fcc9eba9f91dd40804ca637d914e",
-                "sha256:3384d278df99ec2c6acf701d067147320b864ef6727405d6470838476e44d9e8",
-                "sha256:34a01a4490e170376cd79258b7f755fa13b1a6c3667e872c8e35051ae857a92b",
-                "sha256:354f3a91718489912f2e0fc331c24eaaf6a4565c080e00fbedb6015857c00582",
-                "sha256:37f46bb11858717e0efa7893c0f7055c43b44c103e40e69442db5061cb26ed34",
-                "sha256:3b4cf5a9497874822341c2ebe0d5850fed392034caadc0bad134ab6822c0925b",
-                "sha256:3f148c3f47f7f29a79c38cc5d020edcb5ca780020fab94dbc21f9af95c463581",
-                "sha256:443cec402ddd650bb2b885113e1dcedb22b1175c6be223b14246a714b61cd521",
-                "sha256:462b0c18fbb48fdbf980914a02ee38c423a25fcc4cf40f66bacc95a2d2d73bc8",
-                "sha256:474bc83233abdcf2124ed3f66230a1c8435896046caa4b0b5ab6013c640803cc",
-                "sha256:4d438e4c020d8c39961deaf58f6913b1bf8832d9b6f62ec35bd93e97807e9cbc",
-                "sha256:4fdc9afadbeb393b4bbbad75481e0ea78e4469f2e1d713a90811700830b553a9",
-                "sha256:5039e3cef7b3e7a060de468a4a60a60a1f31786da94c6cb054e7a3c75906111c",
-                "sha256:5095a7c838a8647c32aa37c3a460d2c48debff7fc26e1136aee60100a8cd8f68",
-                "sha256:52e466bea6f8f3a44b1234570244b1cff45150f59a4acae3fcc5fd700c2993ca",
-                "sha256:535d4b52524a961d220875688159277f0e9eeeda0ac45e766092bfb54437543f",
-                "sha256:57dbc9167d48e355e2569346b5aa4077f29bf86389c924df25c0a8b9124461fb",
-                "sha256:5a4b07cdf3f84310c08c1de2c12ddadbb7a77568bcb16e95489f9c81074322ed",
-                "sha256:5c872814b77a4e84afa293a1bee08c14daed1068b2bb1cc312edbf020bbbca2b",
-                "sha256:5f83689a38e76969327e9b682be5521d87a0c9e5a2e187d2bc6be4765f0d4600",
-                "sha256:688aa6b8aa724db1596514751ffb767766e02e5c4a87486ab36b8e1ebc1aedac",
-                "sha256:6b130bd4163c93798a6b9bb96be64a7c43e1cec81126ffa7ffaa106e1fc5cef5",
-                "sha256:6b31f059878eb1f5da8b2fd82480cc18bed8dcd7fb8fe68370e2e6285fa86da6",
-                "sha256:6d45080095e585f8c5097897313def60caa2046da202cdb17a01f147fb263b81",
-                "sha256:6f2f78ef14077e08856e788fa482107aa602636c16c25bdf59c22ea525a785e9",
-                "sha256:6fe87efd7f47266dfc42fe76dae89060038f1d9cb911f89ae7e5084148d1cc08",
-                "sha256:75969cf900d7be665ccb1622a9aba225cf386bbc9c3bcfeeab9f62b5048f4a07",
-                "sha256:75a6076289b2df6c8ecb9d13ff79ae0cad1d5fb40af377a5021016d58cd691ec",
-                "sha256:78d57546bad81e0da13263e4c9ce30e96dcbe720dbff5ada08d2600a3502e526",
-                "sha256:79e205c70afddd41f6ee79a8656aec738492a550247a7af697d5bd1aee14f766",
-                "sha256:7c98298a15d6b90c8f6e3caa6457f4f022423caa5fa1a1ca7a5e9e512bdb77a4",
-                "sha256:7ec72df7354e6b7f6eb2a17fa6901350018c3a9ad78e48d7b2b54d0412539a67",
-                "sha256:81ea573aa46d3b6b3d890cd3c0ad82105985e6058a4baed03cf92518081eec8c",
-                "sha256:8344127403dea42f5970adccf6c5957a71a47f522171fafaf4c6ddb41b61703a",
-                "sha256:8445f23f13339da640d1be8e44e5baf4af97e396882ebbf1692aecd67f67c479",
-                "sha256:850720e1b383df199b8433a20e02b25b72f0fded28bc03c5bd79e2ce7ef050be",
-                "sha256:88cb4bac7185a9f0168d38c01d7a00addece9822a52870eee26b8d5b61409213",
-                "sha256:8a790d235b9d39c70a466200d506bb33a98e2ee374a9b4eec7a8ac64c2c261fa",
-                "sha256:8b1a94b8afc154fbe36978a511a1f155f9bd97664e4f1f7a374d72e180ceb0ae",
-                "sha256:8d6ad132b1bc13d05ffe5b85e7a01a3998bf3a6302ba594b28d61b8c2cf13aaf",
-                "sha256:8eb488ef928cdbc05a27245e52de73c0d7c72a34240ef4d9893fdf65a8c1a955",
-                "sha256:90bf55d9d139e5d127193170f38c584ed3c79e16638890d2e36f23aa1630b952",
-                "sha256:9133d75dc119a61d1a0ded38fb9ba40a00ef41697cc07adb6ae098c875195a3f",
-                "sha256:93a91c2640645303e874eada51f4f33351b84b351a689d470f8108d0e0694210",
-                "sha256:959179efb3e4a27610e8d54d667c02a9feaa86bbabaf63efa7faa4dfa780d4f1",
-                "sha256:9625367c8955e4319049113ea4f8fee0c6c1145192d57946c6ffcd8fe8bf48dd",
-                "sha256:9da6f400eeb8c36f72ef6646ea530d6d175a4f77ff2ed8dfd6352842274c1d8b",
-                "sha256:9e65489222b410f79711dc3d2d5003d2757e30874096b2008d50329ea4d0f88c",
-                "sha256:a3e2fd14c5d49ee1da322672375963f19f32b3d5953f0615b175ff7b9d38daed",
-                "sha256:a5a7c1062ef8aea3eda149f08120f10795835fc1c8bc6ad948fb9652a113ca55",
-                "sha256:a5da93debdfe27b2bfc69eefb592e1831d957b9535e0943a0ee8b97996de21b5",
-                "sha256:a6e605bb9edcf010f54f8b6a590dd23a4b40a8cb141255eec2a03db249bc915b",
-                "sha256:a707b158b4410aefb6b054715545bbb21aaa5d5d0080217290131c49c2124a6e",
-                "sha256:a8b6683a37338818646af718c9ca2a07f89787551057fae57c4ec0446dc6224b",
-                "sha256:aa5476c3e3a402c37779e95f7b4048db2cb5b0ed0b9d006983965e93f40fe05a",
-                "sha256:ab1932ca6cb8c7499a4d87cb21ccc0d3326f172cfb6a64021a889b591bb3045c",
-                "sha256:ae8b6068ee374fdfab63689be0963333aa83b0815ead5d8648389a8ded593378",
-                "sha256:b0906357f90784a66e89ae3eadc2654f36c580a7d65cf63e6a616e4aec3a81be",
-                "sha256:b0da31853ab6e58a11db3205729133ce0df26e6804e93079dee095be3d681dc1",
-                "sha256:b1c30841f5040de47a0046c243fc1b44ddc87d1b12435a43b8edff7e7cb1e0d0",
-                "sha256:b228e693a2559888790936e20f5f88b6e9f8162c681830eda303bad7517b4d5a",
-                "sha256:b7cc6cb44f8636fbf4a934ca72f3e786ba3c9f9ba4f4d74611e7da80684e48d2",
-                "sha256:ba0ed0dc6763d8bd6e5de5cf0d746d28e706a10b615ea382ac0ab17bb7388633",
-                "sha256:bc9128e74fe94650367fe23f37074f121b9f796cabbd2f928f13e9661837296d",
-                "sha256:bcf426a8c38eb57f7bf28932e68425ba86def6e756a5b8cb4731d8e62e4e0223",
-                "sha256:bec35eb20792ea64c3c57891bc3ca0bedb2884fbac2c8249d9b731447ecde4fa",
-                "sha256:c3444fe52b82f122d8a99bf66777aed6b858d392b12f4c317da19f8234db4533",
-                "sha256:c5c9581019c96f865483d031691a5ff1cc455feb4d84fc6920a5ffc48a794d8a",
-                "sha256:c6feacd1d178c30e5bc37184526e56740342fd2aa6371a28367bad7908d454fc",
-                "sha256:c8f77e661ffd96ff104bebf7d0f3255b02aa5d5b28326f5408d6284c4a8b3248",
-                "sha256:cb0f6eb3a320f24b94d177e62f4074ff438f2ad9d27e75a46221904ef21a7b05",
-                "sha256:ce84a7efa5af9f54c0aa7692c45861c1667080814286cacb9958c07fc50294fb",
-                "sha256:cf902878b4af334a09de7a45badbff0389e7cf8dc2e4dcf5f07125d0b7c2656d",
-                "sha256:dab8d921b55a28287733263c0e4c7db11b3ee22aee158a4de09f13c93283c62d",
-                "sha256:dc9ac4659456bde7c567107556ab065801622396b435a3ff213daef27b495388",
-                "sha256:dd36b712d35e757e28bf2f40a71e8f8a2d43c8b026d881aa0c617b450d6865c9",
-                "sha256:e19509145275d46bc4d1e16af0b57a12d227c8253655a46bbd5ec317e941279d",
-                "sha256:e21cc693045fda7f745c790cb687958161ce172ffe3c5719ca1764e752237d16",
-                "sha256:e54548e0be3ac117595408fd4ca0ac9278fde89829b0b518be92863b17ff67a2",
-                "sha256:e5b9fc03bf76a94065299d4a2ecd8dfbae4ae8e2e8098bbfa6ab6413ca267709",
-                "sha256:e8481b946792415adc07410420d6fc65a352b45d347b78fec45d8f8f0d7496f0",
-                "sha256:ebcbf356bf5c51afc3290e491d3722b26aaf5b6af3c1c7f6a1b757828a46e336",
-                "sha256:ef9101f3f7b59043a34f1dccbb385ca760467590951952d6701df0da9893ca0c",
-                "sha256:f2afd2164a1e85226fcb6a1da77a5c8896c18bfe08e82e8ceced5181c42d2179",
-                "sha256:f629ecc2db6a4736b5ba95a8347b0089240d69ad14ac364f557d52ad68cf94b0",
-                "sha256:f68eea5df6347d3f1378ce992d86b2af16ad7ff4dcb4a19ccdc23dea901b87fb",
-                "sha256:f757f359f30ec7dcebca662a6bd46d1098f8b9fb1fcd661a9e13f2e8ce343ba1",
-                "sha256:fb37bd599f031f1a6fb9e58ec62864ccf3ad549cf14bac527dbfa97123edcca4"
+                "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c",
+                "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585",
+                "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5",
+                "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6",
+                "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef",
+                "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2",
+                "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29",
+                "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318",
+                "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b",
+                "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399",
+                "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739",
+                "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee",
+                "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174",
+                "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a",
+                "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344",
+                "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2",
+                "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03",
+                "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5",
+                "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22",
+                "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e",
+                "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96",
+                "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91",
+                "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752",
+                "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075",
+                "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253",
+                "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee",
+                "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad",
+                "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5",
+                "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce",
+                "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7",
+                "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b",
+                "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8",
+                "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57",
+                "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3",
+                "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec",
+                "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209",
+                "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921",
+                "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045",
+                "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074",
+                "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580",
+                "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7",
+                "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5",
+                "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3",
+                "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0",
+                "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24",
+                "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139",
+                "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db",
+                "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc",
+                "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789",
+                "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f",
+                "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2",
+                "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c",
+                "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232",
+                "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6",
+                "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c",
+                "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29",
+                "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489",
+                "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94",
+                "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751",
+                "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2",
+                "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda",
+                "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9",
+                "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51",
+                "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c",
+                "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8",
+                "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989",
+                "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511",
+                "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1",
+                "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2",
+                "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150",
+                "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c",
+                "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965",
+                "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f",
+                "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58",
+                "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b",
+                "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f",
+                "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d",
+                "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821",
+                "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de",
+                "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121",
+                "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855",
+                "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272",
+                "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60",
+                "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02",
+                "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1",
+                "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140",
+                "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879",
+                "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940",
+                "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364",
+                "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4",
+                "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e",
+                "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420",
+                "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5",
+                "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24",
+                "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c",
+                "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf",
+                "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f",
+                "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e",
+                "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab",
+                "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08",
+                "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92",
+                "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a",
+                "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.19.0"
+            "version": "==0.20.0"
         },
         "rtree": {
             "hashes": [
@@ -3604,19 +3364,19 @@
         },
         "s3fs": {
             "hashes": [
-                "sha256:63fd8ddf05eb722de784b7b503196107f2a518061298cf005a8a4715b4d49117",
-                "sha256:d1a0a423d0d2e17fb2a193d9531935dc3f45ba742693448a461b6b34f6a92a24"
+                "sha256:2ca5de8dc18ad7ad350c0bd01aef0406aa5d0fff78a561f0f710f9d9858abdd0",
+                "sha256:91c1dfb45e5217bd441a7a560946fe865ced6225ff7eb0fb459fe6e601a95ed3"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==2023.6.0"
+            "markers": "python_version >= '3.5'",
+            "version": "==0.4.2"
         },
         "s3transfer": {
             "hashes": [
-                "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084",
-                "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861"
+                "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6",
+                "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.6.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.10.2"
         },
         "scikit-learn": {
             "hashes": [
@@ -3696,11 +3456,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5",
-                "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"
+                "sha256:3c08705fadfc8c7c445cf4d98078f0fafb9225775b2b4e8447e40348f82597c0",
+                "sha256:f2bfcce7ae1784d90b04c57c2802e8649e1976530bb25dc72c2b078d3ecf4864"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==70.3.0"
+            "version": "==73.0.0"
         },
         "shapely": {
             "hashes": [
@@ -3743,113 +3503,124 @@
                 "sha256:f32a748703e7bf6e92dfa3d2936b2fbfe76f8ce5f756e24f49ef72d17d26ad02",
                 "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"
             ],
-            "index": "pypi",
             "markers": "python_version >= '3.7'",
             "version": "==2.0.1"
         },
         "simplejson": {
             "hashes": [
-                "sha256:0405984f3ec1d3f8777c4adc33eac7ab7a3e629f3b1c05fdded63acc7cf01137",
-                "sha256:0436a70d8eb42bea4fe1a1c32d371d9bb3b62c637969cb33970ad624d5a3336a",
-                "sha256:061e81ea2d62671fa9dea2c2bfbc1eec2617ae7651e366c7b4a2baf0a8c72cae",
-                "sha256:064300a4ea17d1cd9ea1706aa0590dcb3be81112aac30233823ee494f02cb78a",
-                "sha256:08889f2f597ae965284d7b52a5c3928653a9406d88c93e3161180f0abc2433ba",
-                "sha256:0a48679310e1dd5c9f03481799311a65d343748fe86850b7fb41df4e2c00c087",
-                "sha256:0b0a3eb6dd39cce23801a50c01a0976971498da49bc8a0590ce311492b82c44b",
-                "sha256:0d2d5119b1d7a1ed286b8af37357116072fc96700bce3bec5bb81b2e7057ab41",
-                "sha256:0d551dc931638e2102b8549836a1632e6e7cf620af3d093a7456aa642bff601d",
-                "sha256:1018bd0d70ce85f165185d2227c71e3b1e446186f9fa9f971b69eee223e1e3cd",
-                "sha256:11c39fbc4280d7420684494373b7c5904fa72a2b48ef543a56c2d412999c9e5d",
-                "sha256:11cc3afd8160d44582543838b7e4f9aa5e97865322844b75d51bf4e0e413bb3e",
-                "sha256:1537b3dd62d8aae644f3518c407aa8469e3fd0f179cdf86c5992792713ed717a",
-                "sha256:16ca9c90da4b1f50f089e14485db8c20cbfff2d55424062791a7392b5a9b3ff9",
-                "sha256:176a1b524a3bd3314ed47029a86d02d5a95cc0bee15bd3063a1e1ec62b947de6",
-                "sha256:18955c1da6fc39d957adfa346f75226246b6569e096ac9e40f67d102278c3bcb",
-                "sha256:1bb5b50dc6dd671eb46a605a3e2eb98deb4a9af787a08fcdddabe5d824bb9664",
-                "sha256:1c768e7584c45094dca4b334af361e43b0aaa4844c04945ac7d43379eeda9bc2",
-                "sha256:1dd4f692304854352c3e396e9b5f0a9c9e666868dd0bdc784e2ac4c93092d87b",
-                "sha256:25785d038281cd106c0d91a68b9930049b6464288cea59ba95b35ee37c2d23a5",
-                "sha256:287e39ba24e141b046812c880f4619d0ca9e617235d74abc27267194fc0c7835",
-                "sha256:2c1467d939932901a97ba4f979e8f2642415fcf02ea12f53a4e3206c9c03bc17",
-                "sha256:2c433a412e96afb9a3ce36fa96c8e61a757af53e9c9192c97392f72871e18e69",
-                "sha256:2d022b14d7758bfb98405672953fe5c202ea8a9ccf9f6713c5bd0718eba286fd",
-                "sha256:2f98d918f7f3aaf4b91f2b08c0c92b1774aea113334f7cde4fe40e777114dbe6",
-                "sha256:2fc697be37585eded0c8581c4788fcfac0e3f84ca635b73a5bf360e28c8ea1a2",
-                "sha256:3194cd0d2c959062b94094c0a9f8780ffd38417a5322450a0db0ca1a23e7fbd2",
-                "sha256:332c848f02d71a649272b3f1feccacb7e4f7e6de4a2e6dc70a32645326f3d428",
-                "sha256:346820ae96aa90c7d52653539a57766f10f33dd4be609206c001432b59ddf89f",
-                "sha256:3471e95110dcaf901db16063b2e40fb394f8a9e99b3fe9ee3acc6f6ef72183a2",
-                "sha256:3848427b65e31bea2c11f521b6fc7a3145d6e501a1038529da2391aff5970f2f",
-                "sha256:39b6d79f5cbfa3eb63a869639cfacf7c41d753c64f7801efc72692c1b2637ac7",
-                "sha256:3e74355cb47e0cd399ead3477e29e2f50e1540952c22fb3504dda0184fc9819f",
-                "sha256:3f39bb1f6e620f3e158c8b2eaf1b3e3e54408baca96a02fe891794705e788637",
-                "sha256:40847f617287a38623507d08cbcb75d51cf9d4f9551dd6321df40215128325a3",
-                "sha256:4280e460e51f86ad76dc456acdbfa9513bdf329556ffc8c49e0200878ca57816",
-                "sha256:445a96543948c011a3a47c8e0f9d61e9785df2544ea5be5ab3bc2be4bd8a2565",
-                "sha256:4969d974d9db826a2c07671273e6b27bc48e940738d768fa8f33b577f0978378",
-                "sha256:49aaf4546f6023c44d7e7136be84a03a4237f0b2b5fb2b17c3e3770a758fc1a0",
-                "sha256:49e0e3faf3070abdf71a5c80a97c1afc059b4f45a5aa62de0c2ca0444b51669b",
-                "sha256:49f9da0d6cd17b600a178439d7d2d57c5ef01f816b1e0e875e8e8b3b42db2693",
-                "sha256:4a8c3cc4f9dfc33220246760358c8265dad6e1104f25f0077bbca692d616d358",
-                "sha256:4d36081c0b1c12ea0ed62c202046dca11438bee48dd5240b7c8de8da62c620e9",
-                "sha256:4edcd0bf70087b244ba77038db23cd98a1ace2f91b4a3ecef22036314d77ac23",
-                "sha256:554313db34d63eac3b3f42986aa9efddd1a481169c12b7be1e7512edebff8eaf",
-                "sha256:5675e9d8eeef0aa06093c1ff898413ade042d73dc920a03e8cea2fb68f62445a",
-                "sha256:60848ab779195b72382841fc3fa4f71698a98d9589b0a081a9399904487b5832",
-                "sha256:66e5dc13bfb17cd6ee764fc96ccafd6e405daa846a42baab81f4c60e15650414",
-                "sha256:6779105d2fcb7fcf794a6a2a233787f6bbd4731227333a072d8513b252ed374f",
-                "sha256:6ad331349b0b9ca6da86064a3599c425c7a21cd41616e175ddba0866da32df48",
-                "sha256:6f0a0b41dd05eefab547576bed0cf066595f3b20b083956b1405a6f17d1be6ad",
-                "sha256:73a8a4653f2e809049999d63530180d7b5a344b23a793502413ad1ecea9a0290",
-                "sha256:778331444917108fa8441f59af45886270d33ce8a23bfc4f9b192c0b2ecef1b3",
-                "sha256:7cb98be113911cb0ad09e5523d0e2a926c09a465c9abb0784c9269efe4f95917",
-                "sha256:7d74beca677623481810c7052926365d5f07393c72cbf62d6cce29991b676402",
-                "sha256:7f2398361508c560d0bf1773af19e9fe644e218f2a814a02210ac2c97ad70db0",
-                "sha256:8434dcdd347459f9fd9c526117c01fe7ca7b016b6008dddc3c13471098f4f0dc",
-                "sha256:8a390e56a7963e3946ff2049ee1eb218380e87c8a0e7608f7f8790ba19390867",
-                "sha256:92c4a4a2b1f4846cd4364855cbac83efc48ff5a7d7c06ba014c792dd96483f6f",
-                "sha256:9300aee2a8b5992d0f4293d88deb59c218989833e3396c824b69ba330d04a589",
-                "sha256:9453419ea2ab9b21d925d0fd7e3a132a178a191881fab4169b6f96e118cc25bb",
-                "sha256:9652e59c022e62a5b58a6f9948b104e5bb96d3b06940c6482588176f40f4914b",
-                "sha256:972a7833d4a1fcf7a711c939e315721a88b988553fc770a5b6a5a64bd6ebeba3",
-                "sha256:9c1a4393242e321e344213a90a1e3bf35d2f624aa8b8f6174d43e3c6b0e8f6eb",
-                "sha256:9e038c615b3906df4c3be8db16b3e24821d26c55177638ea47b3f8f73615111c",
-                "sha256:9e4c166f743bb42c5fcc60760fb1c3623e8fda94f6619534217b083e08644b46",
-                "sha256:9eb117db8d7ed733a7317c4215c35993b815bf6aeab67523f1f11e108c040672",
-                "sha256:9eb442a2442ce417801c912df68e1f6ccfcd41577ae7274953ab3ad24ef7d82c",
-                "sha256:a3cd18e03b0ee54ea4319cdcce48357719ea487b53f92a469ba8ca8e39df285e",
-                "sha256:a8617625369d2d03766413bff9e64310feafc9fc4f0ad2b902136f1a5cd8c6b0",
-                "sha256:a970a2e6d5281d56cacf3dc82081c95c1f4da5a559e52469287457811db6a79b",
-                "sha256:aad7405c033d32c751d98d3a65801e2797ae77fac284a539f6c3a3e13005edc4",
-                "sha256:adcb3332979cbc941b8fff07181f06d2b608625edc0a4d8bc3ffc0be414ad0c4",
-                "sha256:af9c7e6669c4d0ad7362f79cb2ab6784d71147503e62b57e3d95c4a0f222c01c",
-                "sha256:b01fda3e95d07a6148702a641e5e293b6da7863f8bc9b967f62db9461330562c",
-                "sha256:b8d940fd28eb34a7084877747a60873956893e377f15a32ad445fe66c972c3b8",
-                "sha256:bccb3e88ec26ffa90f72229f983d3a5d1155e41a1171190fa723d4135523585b",
-                "sha256:bcedf4cae0d47839fee7de344f96b5694ca53c786f28b5f773d4f0b265a159eb",
-                "sha256:be893258d5b68dd3a8cba8deb35dc6411db844a9d35268a8d3793b9d9a256f80",
-                "sha256:c0521e0f07cb56415fdb3aae0bbd8701eb31a9dfef47bb57206075a0584ab2a2",
-                "sha256:c594642d6b13d225e10df5c16ee15b3398e21a35ecd6aee824f107a625690374",
-                "sha256:c87c22bd6a987aca976e3d3e23806d17f65426191db36d40da4ae16a6a494cbc",
-                "sha256:c9ac1c2678abf9270e7228133e5b77c6c3c930ad33a3c1dfbdd76ff2c33b7b50",
-                "sha256:d0e5ffc763678d48ecc8da836f2ae2dd1b6eb2d27a48671066f91694e575173c",
-                "sha256:d0f402e787e6e7ee7876c8b05e2fe6464820d9f35ba3f172e95b5f8b699f6c7f",
-                "sha256:d222a9ed082cd9f38b58923775152003765016342a12f08f8c123bf893461f28",
-                "sha256:d94245caa3c61f760c4ce4953cfa76e7739b6f2cbfc94cc46fff6c050c2390c5",
-                "sha256:de9a2792612ec6def556d1dc621fd6b2073aff015d64fba9f3e53349ad292734",
-                "sha256:e2f5a398b5e77bb01b23d92872255e1bcb3c0c719a3be40b8df146570fe7781a",
-                "sha256:e8dd53a8706b15bc0e34f00e6150fbefb35d2fd9235d095b4f83b3c5ed4fa11d",
-                "sha256:e9eb3cff1b7d71aa50c89a0536f469cb8d6dcdd585d8f14fb8500d822f3bdee4",
-                "sha256:ed628c1431100b0b65387419551e822987396bee3c088a15d68446d92f554e0c",
-                "sha256:ef7938a78447174e2616be223f496ddccdbf7854f7bf2ce716dbccd958cc7d13",
-                "sha256:f1c70249b15e4ce1a7d5340c97670a95f305ca79f376887759b43bb33288c973",
-                "sha256:f3c7363a8cb8c5238878ec96c5eb0fc5ca2cb11fc0c7d2379863d342c6ee367a",
-                "sha256:fbbcc6b0639aa09b9649f36f1bcb347b19403fe44109948392fbb5ea69e48c3e",
-                "sha256:febffa5b1eda6622d44b245b0685aff6fb555ce0ed734e2d7b1c3acd018a2cff",
-                "sha256:ff836cd4041e16003549449cc0a5e372f6b6f871eb89007ab0ee18fb2800fded"
+                "sha256:01c6657485393f2e9b8177c77a7634f13ebe70d5e6de150aae1677d91516ce6b",
+                "sha256:0552eb06e7234da892e1d02365cd2b7b2b1f8233aa5aabdb2981587b7cc92ea0",
+                "sha256:06662392e4913dc8846d6a71a6d5de86db5fba244831abe1dd741d62a4136764",
+                "sha256:0733ecd95ae03ae718ec74aad818f5af5f3155d596f7b242acbc1621e765e5fb",
+                "sha256:0766ca6222b410e08e0053a0dda3606cafb3973d5d00538307f631bb59743396",
+                "sha256:0791f64fed7d4abad639491f8a6b1ba56d3c604eb94b50f8697359b92d983f36",
+                "sha256:08f9b443a94e72dd02c87098c96886d35790e79e46b24e67accafbf13b73d43b",
+                "sha256:0959e6cb62e3994b5a40e31047ff97ef5c4138875fae31659bead691bed55896",
+                "sha256:0a32859d45d7b85fb803bb68f6bee14526991a1190269116c33399fa0daf9bbf",
+                "sha256:0b5ddd2c7d1d3f4d23224bc8a04bbf1430ae9a8149c05b90f8fc610f7f857a23",
+                "sha256:0bc5544e3128891bf613b9f71813ee2ec9c11574806f74dd8bb84e5e95bf64a2",
+                "sha256:101a3c8392028cd704a93c7cba8926594e775ca3c91e0bee82144e34190903f1",
+                "sha256:1069143a8fb3905e1bc0696c62be7e3adf812e9f1976ac9ae15b05112ff57cc9",
+                "sha256:1773cabfba66a6337b547e45dafbd471b09487370bcab75bd28f626520410d29",
+                "sha256:1a53a07320c5ff574d8b1a89c937ce33608832f166f39dff0581ac43dc979abd",
+                "sha256:1bd41f2cb1a2c57656ceff67b12d005cb255c728265e222027ad73193a04005a",
+                "sha256:1c49eeb94b8f09dc8a5843c156a22b8bde6aa1ddc65ca8ddc62dddcc001e6a2d",
+                "sha256:1df0aaf1cb787fdf34484ed4a1f0c545efd8811f6028623290fef1a53694e597",
+                "sha256:1e557712fc79f251673aeb3fad3501d7d4da3a27eff0857af2e1d1afbbcf6685",
+                "sha256:1e662336db50ad665777e6548b5076329a94a0c3d4a0472971c588b3ef27de3a",
+                "sha256:212fce86a22188b0c7f53533b0f693ea9605c1a0f02c84c475a30616f55a744d",
+                "sha256:23228037dc5d41c36666384062904d74409a62f52283d9858fa12f4c22cffad1",
+                "sha256:23833ee7e791ec968b744dfee2a2d39df7152050051096caf4296506d75608d8",
+                "sha256:256e09d0f94d9c3d177d9e95fd27a68c875a4baa2046633df387b86b652f5747",
+                "sha256:2876027ebdd599d730d36464debe84619b0368e9a642ca6e7c601be55aed439e",
+                "sha256:2a6a750d3c7461b1c47cfc6bba8d9e57a455e7c5f80057d2a82f738040dd1129",
+                "sha256:2a954b30810988feeabde843e3263bf187697e0eb5037396276db3612434049b",
+                "sha256:2b737a5fefedb8333fa50b8db3dcc9b1d18fd6c598f89fa7debff8b46bf4e511",
+                "sha256:2c78293470313aefa9cfc5e3f75ca0635721fb016fb1121c1c5b0cb8cc74712a",
+                "sha256:2f56eb03bc9e432bb81adc8ecff2486d39feb371abb442964ffb44f6db23b332",
+                "sha256:32a3ada8f3ea41db35e6d37b86dade03760f804628ec22e4fe775b703d567426",
+                "sha256:37105d1d708365b91165e1a6e505bdecc88637091348cf4b6adcdcb4f5a5fb8b",
+                "sha256:3bbcdc438dc1683b35f7a8dc100960c721f922f9ede8127f63bed7dfded4c64c",
+                "sha256:3dc5c1a85ff388e98ea877042daec3d157b6db0d85bac6ba5498034689793e7e",
+                "sha256:42e5acf80d4d971238d4df97811286a044d720693092b20a56d5e56b7dcc5d09",
+                "sha256:49549e3d81ab4a58424405aa545602674d8c35c20e986b42bb8668e782a94bac",
+                "sha256:49cc4c7b940d43bd12bf87ec63f28cbc4964fc4e12c031cc8cd01650f43eb94e",
+                "sha256:4a0710d1a5e41c4f829caa1572793dd3130c8d65c2b194c24ff29c4c305c26e0",
+                "sha256:4dfa420bb9225dd33b6efdabde7c6a671b51150b9b1d9c4e5cd74d3b420b3fe1",
+                "sha256:50d8b742d74c449c4dcac570d08ce0f21f6a149d2d9cf7652dbf2ba9a1bc729a",
+                "sha256:56134bbafe458a7b21f6fddbf889d36bec6d903718f4430768e3af822f8e27c2",
+                "sha256:5bf6a3b9a7d7191471b464fe38f684df10eb491ec9ea454003edb45a011ab187",
+                "sha256:5d9e8f836688a8fabe6a6b41b334aa550a6823f7b4ac3d3712fc0ad8655be9a8",
+                "sha256:619756f1dd634b5bdf57d9a3914300526c3b348188a765e45b8b08eabef0c94e",
+                "sha256:6300680d83a399be2b8f3b0ef7ef90b35d2a29fe6e9c21438097e0938bbc1564",
+                "sha256:637c4d4b81825c1f4d651e56210bd35b5604034b192b02d2d8f17f7ce8c18f42",
+                "sha256:66a0399e21c2112acacfebf3d832ebe2884f823b1c7e6d1363f2944f1db31a99",
+                "sha256:67a20641afebf4cfbcff50061f07daad1eace6e7b31d7622b6fa2c40d43900ba",
+                "sha256:6890ff9cf0bd2e1d487e2a8869ebd620a44684c0a9667fa5ee751d099d5d84c8",
+                "sha256:6d43e24b88c80f997081503f693be832fc90854f278df277dd54f8a4c847ab61",
+                "sha256:6ef9383c5e05f445be60f1735c1816163c874c0b1ede8bb4390aff2ced34f333",
+                "sha256:6f455672f4738b0f47183c5896e3606cd65c9ddee3805a4d18e8c96aa3f47c84",
+                "sha256:6fea0716c593dabb4392c4996d4e902a83b2428e6da82938cf28a523a11eb277",
+                "sha256:7017329ca8d4dca94ad5e59f496e5fc77630aecfc39df381ffc1d37fb6b25832",
+                "sha256:7137e69c6781ecf23afab064be94a277236c9cba31aa48ff1a0ec3995c69171e",
+                "sha256:72e8abbc86fcac83629a030888b45fed3a404d54161118be52cb491cd6975d3e",
+                "sha256:7355c7203353c36d46c4e7b6055293b3d2be097bbc5e2874a2b8a7259f0325dd",
+                "sha256:76f8c28fe2d426182405b18ddf3001fce47835a557dc15c3d8bdea01c03361da",
+                "sha256:7923878b7a0142d39763ec2dbecff3053c1bedd3653585a8474666e420fe83f5",
+                "sha256:7a7bfad839c624e139a4863007233a3f194e7c51551081f9789cba52e4da5167",
+                "sha256:7b5c472099b39b274dcde27f1113db8d818c9aa3ba8f78cbb8ad04a4c1ac2118",
+                "sha256:7c0104b4b7d2c75ccedbf1d9d5a3bd2daa75e51053935a44ba012e2fd4c43752",
+                "sha256:7e062767ac165df9a46963f5735aa4eee0089ec1e48b3f2ec46182754b96f55e",
+                "sha256:7e2a098c21ad8924076a12b6c178965d88a0ad75d1de67e1afa0a66878f277a5",
+                "sha256:817abad79241ed4a507b3caf4d3f2be5079f39d35d4c550a061988986bffd2ec",
+                "sha256:83c87706265ae3028e8460d08b05f30254c569772e859e5ba61fe8af2c883468",
+                "sha256:89b35433186e977fa86ff1fd179c1fadff39cfa3afa1648dab0b6ca53153acd9",
+                "sha256:8e086896c36210ab6050f2f9f095a5f1e03c83fa0e7f296d6cba425411364680",
+                "sha256:8f41bb5370b34f63171e65fdb00e12be1d83675cecb23e627df26f4c88dfc021",
+                "sha256:934a50a614fb831614db5dbfba35127ee277624dda4d15895c957d2f5d48610c",
+                "sha256:93be280fc69a952c76e261036312c20b910e7fa9e234f1d89bdfe3fa34f8a023",
+                "sha256:951095be8d4451a7182403354c22ec2de3e513e0cc40408b689af08d02611588",
+                "sha256:a0782cb9bf827f0c488b6aa0f2819f618308a3caf2973cfd792e45d631bec4db",
+                "sha256:ab69f811a660c362651ae395eba8ce84f84c944cea0df5718ea0ba9d1e4e7252",
+                "sha256:ad0e0b1ce9bd3edb5cf64b5b5b76eacbfdac8c5367153aeeec8a8b1407f68342",
+                "sha256:add8850db04b98507a8b62d248a326ecc8561e6d24336d1ca5c605bbfaab4cad",
+                "sha256:afab2f7f2486a866ff04d6d905e9386ca6a231379181a3838abce1f32fbdcc37",
+                "sha256:b5587feda2b65a79da985ae6d116daf6428bf7489992badc29fc96d16cd27b05",
+                "sha256:b9198c1f1f8910a3b86b60f4fe2556d9d28d3fefe35bffe6be509a27402e694d",
+                "sha256:bc164f32dd9691e7082ce5df24b4cf8c6c394bbf9bdeeb5d843127cd07ab8ad2",
+                "sha256:bcde83a553a96dc7533736c547bddaa35414a2566ab0ecf7d3964fc4bdb84c11",
+                "sha256:c40df31a75de98db2cdfead6074d4449cd009e79f54c1ebe5e5f1f153c68ad20",
+                "sha256:c4f614581b61a26fbbba232a1391f6cee82bc26f2abbb6a0b44a9bba25c56a1c",
+                "sha256:c9bedebdc5fdad48af8783022bae307746d54006b783007d1d3c38e10872a2c6",
+                "sha256:cb324bb903330cbb35d87cce367a12631cd5720afa06e5b9c906483970946da6",
+                "sha256:d00313681015ac498e1736b304446ee6d1c72c5b287cd196996dad84369998f7",
+                "sha256:d0b0efc7279d768db7c74d3d07f0b5c81280d16ae3fb14e9081dc903e8360771",
+                "sha256:d0d5a63f1768fed7e78cf55712dee81f5a345e34d34224f3507ebf71df2b754d",
+                "sha256:d1b8b4d6379fe55f471914345fe6171d81a18649dacf3248abfc9c349b4442eb",
+                "sha256:d36608557b4dcd7a62c29ad4cd7c5a1720bbf7dc942eff9dc42d2c542a5f042d",
+                "sha256:d43c2d7504eda566c50203cdc9dc043aff6f55f1b7dae0dcd79dfefef9159d1c",
+                "sha256:d73efb03c5b39249c82488a994f0998f9e4399e3d085209d2120503305ba77a8",
+                "sha256:d936ae682d5b878af9d9eb4d8bb1fdd5e41275c8eb59ceddb0aeed857bb264a2",
+                "sha256:dd011fc3c1d88b779645495fdb8189fb318a26981eebcce14109460e062f209b",
+                "sha256:dd5b9b1783e14803e362a558680d88939e830db2466f3fa22df5c9319f8eea94",
+                "sha256:dd6a7dabcc4c32daf601bc45e01b79175dde4b52548becea4f9545b0a4428169",
+                "sha256:dd7230d061e755d60a4d5445bae854afe33444cdb182f3815cff26ac9fb29a15",
+                "sha256:e0d2b00ecbcd1a3c5ea1abc8bb99a26508f758c1759fd01c3be482a3655a176f",
+                "sha256:e1a1452ad5723ff129b081e3c8aa4ba56b8734fee4223355ed7b815a7ece69bc",
+                "sha256:e88abff510dcff903a18d11c2a75f9964e768d99c8d147839913886144b2065e",
+                "sha256:ea7a4a998c87c5674a27089e022110a1a08a7753f21af3baf09efe9915c23c3c",
+                "sha256:eb47ee773ce67476a960e2db4a0a906680c54f662521550828c0cc57d0099426",
+                "sha256:eed8cd98a7b24861da9d3d937f5fbfb6657350c547528a117297fe49e3960667",
+                "sha256:ef28c3b328d29b5e2756903aed888960bc5df39b4c2eab157ae212f70ed5bf74",
+                "sha256:ef59a53be400c1fad2c914b8d74c9d42384fed5174f9321dd021b7017fd40270",
+                "sha256:f39caec26007a2d0efab6b8b1d74873ede9351962707afab622cc2285dd26ed0",
+                "sha256:f8efb03ca77bd7725dfacc9254df00d73e6f43013cf39bd37ef1a8ed0ebb5165",
+                "sha256:fa97278ae6614346b5ca41a45a911f37a3261b57dbe4a00602048652c862c28b",
+                "sha256:fc3dc9fb413fc34c396f52f4c87de18d0bd5023804afa8ab5cc224deeb6a9900",
+                "sha256:ff7bc1bbdaa3e487c9469128bf39408e91f5573901cb852e03af378d3582c52d"
             ],
             "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==3.19.2"
+            "version": "==3.19.3"
         },
         "six": {
             "hashes": [
@@ -3883,11 +3654,11 @@
         },
         "soupsieve": {
             "hashes": [
-                "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690",
-                "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"
+                "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb",
+                "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5"
+            "version": "==2.6"
         },
         "stack-data": {
             "hashes": [
@@ -4081,10 +3852,10 @@
         },
         "webcolors": {
             "hashes": [
-                "sha256:1d160d1de46b3e81e58d0a280d0c78b467dc80f47294b91b1ad8029d2cedb55b",
-                "sha256:8cf5bc7e28defd1d48b9e83d5fc30741328305a8195c29a8e668fa45586568a1"
+                "sha256:08b07af286a01bcd30d583a7acadf629583d1f79bfef27dd2c2c5c263817277d",
+                "sha256:fc4c3b59358ada164552084a8ebee637c221e4059267d0f8325b3b560f6c7f0a"
             ],
-            "version": "==24.6.0"
+            "version": "==24.8.0"
         },
         "webencodings": {
             "hashes": [
@@ -4305,102 +4076,6 @@
             ],
             "version": "==0.6.2"
         },
-        "yarl": {
-            "hashes": [
-                "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51",
-                "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce",
-                "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559",
-                "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0",
-                "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81",
-                "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc",
-                "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4",
-                "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c",
-                "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130",
-                "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136",
-                "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e",
-                "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec",
-                "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7",
-                "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1",
-                "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455",
-                "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099",
-                "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129",
-                "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10",
-                "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142",
-                "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98",
-                "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa",
-                "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7",
-                "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525",
-                "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c",
-                "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9",
-                "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c",
-                "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8",
-                "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b",
-                "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf",
-                "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23",
-                "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd",
-                "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27",
-                "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f",
-                "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece",
-                "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434",
-                "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec",
-                "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff",
-                "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78",
-                "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d",
-                "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863",
-                "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53",
-                "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31",
-                "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15",
-                "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5",
-                "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b",
-                "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57",
-                "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3",
-                "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1",
-                "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f",
-                "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad",
-                "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c",
-                "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7",
-                "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2",
-                "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b",
-                "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2",
-                "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b",
-                "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9",
-                "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be",
-                "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e",
-                "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984",
-                "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4",
-                "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074",
-                "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2",
-                "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392",
-                "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91",
-                "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541",
-                "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf",
-                "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572",
-                "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66",
-                "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575",
-                "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14",
-                "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5",
-                "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1",
-                "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e",
-                "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551",
-                "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17",
-                "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead",
-                "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0",
-                "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe",
-                "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234",
-                "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0",
-                "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7",
-                "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34",
-                "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42",
-                "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385",
-                "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78",
-                "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be",
-                "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958",
-                "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749",
-                "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.9.4"
-        },
         "ypy-websocket": {
             "hashes": [
                 "sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff",
@@ -4428,11 +4103,11 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19",
-                "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"
+                "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31",
+                "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.19.2"
+            "version": "==3.20.0"
         }
     },
     "develop": {
@@ -4525,10 +4200,10 @@
         },
         "pure-eval": {
             "hashes": [
-                "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350",
-                "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"
+                "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0",
+                "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"
             ],
-            "version": "==0.2.2"
+            "version": "==0.2.3"
         },
         "pygments": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0dcd9742f..170272d2c 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.6.0 - 2024-08-23 - [PR#1253](https://github.com/NOAA-OWP/inundation-mapping/pull/1253)
+
+Upgrades Python packages and dependencies and fixes backwards incompatibilities with new version of `geopandas`. Major changes include:
+- Upgrading `boto3`, `fiona`, `geopandas`, `gval`, `pyarrow`, `pyogrio`, `pyproj`, and `rasterio`
+- Removing `aiobotocore` and `aiohttp`
+
+### Changes
+
+- `Dockerfile`: Upgrade GDAL (v3.8.3) and pipenv
+- `Pipfile` and `Pipfile.lock`: Upgrade Python packages, add dask-expr, and remove aiohttp
+- `src/`
+    - `build_stream_traversal.py`, `derive_level_paths.py`, `heal_bridges_osm.py`, `mitigate_branch_outlet_backpool.py`, `split_flows.py`, `stream_branches.py`, `usgs_gage_unit_setup.py`: Fix backwards incompatibilities with new version of `geopandas`.
+
+<br/><br/>
+
 ## v4.5.5.1 - 2024-08-16 - [PR#1225](https://github.com/NOAA-OWP/inundation-mapping/pull/1225)
 
 Removes warning when running `heal_bridges_osm.py` by not saving the empty DataFrame.
diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py
index 9c287807f..08ec986c3 100644
--- a/src/build_stream_traversal.py
+++ b/src/build_stream_traversal.py
@@ -64,9 +64,7 @@ def execute(self, streams, wbd8, hydro_id):
                     {'geometry': stream_midpoint}, crs=streams.crs, geometry='geometry'
                 )
                 stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', predicate='within')
-                stream_wbdjoin = stream_wbdjoin.rename(
-                    columns={"geometry": "midpoint", "index_right": "HUC8id"}
-                )
+                stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "midpoint", "fimid": "HUC8id"})
                 streams = streams.join(stream_wbdjoin).drop(columns=['midpoint'])
 
                 streams['seqID'] = (
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index bb50a4e68..36e3fd437 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -192,7 +192,10 @@ def Derive_level_paths(
         )
 
         stream_network = stream_network.remove_branches_in_waterbodies(
-            waterbodies=waterbodies, out_vector_files=out_stream_network_dissolved, verbose=False
+            waterbodies=waterbodies,
+            out_vector_files=out_stream_network_dissolved,
+            branch_id_attribute=branch_id_attribute,
+            verbose=False,
         )
         stream_network = stream_network.select_branches_intersecting_huc(
             wbd=wbd,
diff --git a/src/heal_bridges_osm.py b/src/heal_bridges_osm.py
index 4478ad58f..13c3b1c65 100644
--- a/src/heal_bridges_osm.py
+++ b/src/heal_bridges_osm.py
@@ -74,9 +74,9 @@ def process_bridges_in_huc(
     # Check if the GeoDataFrame is empty
     if not osm_gdf.empty:
         # Write the bridge points to a geopackage
-        osm_gdf.to_file(bridge_centroids, index=False)
+        osm_gdf.to_file(bridge_centroids, index=False, engine='fiona')
     else:
-        print('The geoDataFrame is empty.File not saved.')
+        print('The geoDataFrame is empty. File not saved.')
     return
 
 
diff --git a/src/mitigate_branch_outlet_backpool.py b/src/mitigate_branch_outlet_backpool.py
index 3b76d8479..c35a832cf 100755
--- a/src/mitigate_branch_outlet_backpool.py
+++ b/src/mitigate_branch_outlet_backpool.py
@@ -390,7 +390,7 @@ def mask_raster_to_boundary(raster_path, boundary_json, save_path):
                     trimmed_flows, inital_length_km = snap_and_trim_splitflow(pt_3tl_geom, split_flows_geom)
 
                     # Create buffer around the updated flows geodataframe (and make sure it's all one shape)
-                    buffer = trimmed_flows.buffer(10).geometry.unary_union
+                    buffer = trimmed_flows.buffer(10).geometry.union_all()
 
                     # Remove flowpoints that don't intersect with the trimmed flow line
                     split_points_filtered_geom = split_points_geom[split_points_geom.geometry.within(buffer)]
diff --git a/src/split_flows.py b/src/split_flows.py
index 0a14943ce..6a7b53e16 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -370,7 +370,7 @@ def snap_and_trim_flow(snapped_point, flows):
         split_flows_gdf = gpd.sjoin(
             split_flows_gdf, lakes_buffer, how='left', predicate='within'
         )  # Note: Options include intersects, within, contains, crosses
-        split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
+        split_flows_gdf = split_flows_gdf.rename(columns={"newID": "LakeID"}).fillna(-999)
     else:
         split_flows_gdf['LakeID'] = -999
 
diff --git a/src/stream_branches.py b/src/stream_branches.py
index b9718ea91..d0f918500 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -188,6 +188,22 @@ def drop(self, labels=None, axis=0):
         )
         return self
 
+    def rename(self, columns):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+
+        self = super(gpd.GeoDataFrame, self)
+        self = self.rename(columns=columns)
+
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+        return self
+
     def dissolve(self, by=None):
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
@@ -645,7 +661,9 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
 
         return self
 
-    def remove_branches_in_waterbodies(self, waterbodies, out_vector_files=None, verbose=False):
+    def remove_branches_in_waterbodies(
+        self, waterbodies, branch_id_attribute, out_vector_files=None, verbose=False
+    ):
         """
         Removes branches completely in waterbodies
         """
@@ -658,9 +676,13 @@ def remove_branches_in_waterbodies(self, waterbodies, out_vector_files=None, ver
             waterbodies = gpd.read_file(waterbodies)
 
         if isinstance(waterbodies, gpd.GeoDataFrame):
+            waterbodies = waterbodies.drop('OBJECTID', axis=1)
+
             # Find branches in waterbodies
+            self = self.rename(columns={branch_id_attribute: "bids"})
             sjoined = gpd.sjoin(self, waterbodies, predicate="within")
             self = self.drop(sjoined.index)
+            self = self.rename(columns={"bids": branch_id_attribute})
 
             if out_vector_files is not None:
                 if verbose:
@@ -693,7 +715,9 @@ def select_branches_intersecting_huc(self, wbd, buffer_wbd_streams, out_vector_f
 
         if isinstance(wbd, gpd.GeoDataFrame):
             # Find branches intersecting HUC
+            self = self.rename(columns={branch_id_attribute: "bids"})
             sjoined = gpd.sjoin(self, wbd, predicate="intersects")
+            self = self.rename(columns={"bids": branch_id_attribute})
 
             self = self[self.index.isin(sjoined.index.values)]
 
diff --git a/src/usgs_gage_unit_setup.py b/src/usgs_gage_unit_setup.py
index 464fee471..b1bfa7bf2 100755
--- a/src/usgs_gage_unit_setup.py
+++ b/src/usgs_gage_unit_setup.py
@@ -94,7 +94,7 @@ def sort_into_branch(self, nwm_subset_streams_levelPaths):
 
         if not self.gages[self.gages.feature_id.isnull()].empty:
             missing_feature_id = self.gages.loc[self.gages.feature_id.isnull()].copy()
-            nwm_reaches_union = nwm_reaches.geometry.unary_union
+            nwm_reaches_union = nwm_reaches.geometry.union_all()
             missing_feature_id['feature_id'] = missing_feature_id.apply(
                 lambda row: self.sjoin_nearest_to_nwm(row.geometry, nwm_reaches, nwm_reaches_union), axis=1
             )
@@ -118,7 +118,7 @@ def branch_zero(self, bzero_id):
         return self.gages
 
     def write(self, out_name):
-        self.gages.to_file(out_name, driver='GPKG', index=False)
+        self.gages.to_file(out_name, driver='GPKG', index=False, engine='fiona')
 
     @staticmethod
     def sjoin_nearest_to_nwm(pnt, lines, union):

From 484145941e1b304c8863db2c846ff0e3452c9d9e Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Fri, 13 Sep 2024 14:19:53 -0500
Subject: [PATCH 345/359] v4.5.6.1 FIM Performance for Alaska and logging
 (#1271)

---
 docs/CHANGELOG.md              |  13 +-
 tools/test_case_by_hydro_id.py | 219 +++++++++++++++++++++------------
 2 files changed, 151 insertions(+), 81 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 170272d2c..436b10c45 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.6.1 - 2024-09-13 - [PR#1271](https://github.com/NOAA-OWP/inundation-mapping/pull/1271)
+
+Upgrade for `test_case_by_hydro_id.py` that enables the ability to run on HUCs with differing projections (e.g. Alaska) and adds a logging system.
+
+### Changes
+
+- `tools/test_case_by_hydro_id.py`: Moved the reprojection step to accommodate  multiple input projections and fixed a lot of unnecessary logic. Also added an optional logging system that is activated by the new `-l` flag.
+
+<br/><br/>
+
+
 ## v4.5.6.0 - 2024-08-23 - [PR#1253](https://github.com/NOAA-OWP/inundation-mapping/pull/1253)
 
 Upgrades Python packages and dependencies and fixes backwards incompatibilities with new version of `geopandas`. Major changes include:
@@ -38,7 +49,7 @@ Updated the gauge crosswalk and SRC adjustment routine to use the ras2fim v2 fil
 - `src/src_adjust_ras2fim_rating.py`: Updated code logic to use the huc-specific input files containing the ras2fim rating curve data (previous ras2fim input file contained all hucs in one csv)
 - `src/utils/shared_functions.py`: Added function to find huc subdirectories with the same name btw two parent folders
 
- <br/><br/>
+<br/><br/>
 
 ## v4.5.4.4 - 2024-08-02 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
 
diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py
index 5b24fdd3a..73186f33e 100644
--- a/tools/test_case_by_hydro_id.py
+++ b/tools/test_case_by_hydro_id.py
@@ -2,6 +2,7 @@
 
 import argparse
 import os
+import traceback
 from datetime import datetime
 
 import geopandas as gpd
@@ -10,10 +11,25 @@
 from run_test_case import Test_Case
 from shapely.validation import make_valid
 from tools_shared_functions import compute_stats_from_contingency_table
+from tqdm import tqdm
 
 
 gpd.options.io_engine = "pyogrio"
 
+"""
+This module uses zonal stats to subdivide alpha metrics by each HAND catchment.
+The output is a vector geopackage and is also known as the "FIM Performance" layer
+when loaded into HydroVIS. At the time of this commit, it takes approximately
+32 hours to complete.
+
+Example usage:
+python /foss_fim/tools/test_case_by_hydro_id.py \
+    -b all \
+    -v fim_4_5_2_11 \
+    -g /outputs/fim_performance_v4_5_2_11.gpkg \
+    -l
+"""
+
 
 #####################################################
 # Perform zonal stats is a funtion stored in pixel_counter.py.
@@ -180,39 +196,7 @@ def assemble_hydro_alpha_for_single_huc(stats, huc8, mag, bench):
     return in_mem_df
 
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Produces alpha metrics by hyrdoid.')
-
-    parser.add_argument(
-        '-b',
-        '--benchmark_category',
-        help='Choice of truth data. Options are: all, ble, ifc, nws, usgs, ras2fim',
-        required=True,
-    )
-    parser.add_argument(
-        '-v', '--version', help='The fim version to use. Should be similar to fim_3_0_24_14_ms', required=True
-    )
-    parser.add_argument(
-        '-g',
-        '--gpkg',
-        help='filepath and filename to hold exported gpkg (and csv) file. '
-        'Similar to /data/path/fim_performance_catchments.gpkg Need to use gpkg as output.',
-        required=True,
-    )
-
-    # Assign variables from arguments.
-    args = vars(parser.parse_args())
-    benchmark_category = args['benchmark_category']
-    version = args['version']
-    csv = args['gpkg']
-
-    print("================================")
-    print("Start test_case_by_hydroid.py")
-    start_time = datetime.now()
-    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print(f"started: {dt_string}")
-    print()
-
+def catchment_zonal_stats(benchmark_category, version, csv, log):
     # Execution code
     csv_output = gpd.GeoDataFrame(
         columns=[
@@ -237,92 +221,167 @@ def assemble_hydro_alpha_for_single_huc(stats, huc8, mag, bench):
             'geometry',
         ],
         geometry='geometry',
-    )
+    ).set_crs('EPSG:3857')
 
     # This funtion, relies on the Test_Case class defined in run_test_case.py to list all available test cases
-    print('listing_test_cases_with_updates')
     all_test_cases = Test_Case.list_all_test_cases(
         version=version,
         archive=True,
         benchmark_categories=[] if benchmark_category == "all" else [benchmark_category],
     )
+    print(f'Found {len(all_test_cases)} test cases')
+    if log:
+        log.write(f'Found {len(all_test_cases)} test cases...\n')
+    missing_hucs = []
 
-    for test_case_class in all_test_cases:
+    for test_case_class in tqdm(all_test_cases, desc=f'Running {len(all_test_cases)} test cases'):
         if not os.path.exists(test_case_class.fim_dir):
             print(f'{test_case_class.fim_dir} does not exist')
+            missing_hucs.append(test_case_class)
+            if log:
+                log.write(f'{test_case_class.fim_dir} does not exist\n')
             continue
 
-        print(test_case_class.fim_dir, end='\r')
+        if log:
+            log.write(test_case_class.test_id + '\n')
 
         agreement_dict = test_case_class.get_current_agreements()
 
         for agree_rast in agreement_dict:
-            print(f'performing_zonal_stats for {agree_rast}')
 
-            branches_dir = os.path.join(test_case_class.fim_dir, 'branches')
-            for branches in os.listdir(branches_dir):
-                if branches != "0":
-                    continue
-                huc_gpkg = os.path.join(branches_dir, branches)
+            # We are only using branch 0 catchments to define boundaries for zonal stats
+            catchment_gpkg = os.path.join(
+                test_case_class.fim_dir,
+                'branches',
+                "gw_catchments_reaches_filtered_addedAttributes_crosswalked_0.gpkg",
+            )
 
-                string_manip = (
-                    "gw_catchments_reaches_filtered_addedAttributes_crosswalked_" + branches + ".gpkg"
-                )
+            define_mag = agree_rast.split(version)
+            define_mag_1 = define_mag[1].split('/')
+            mag = define_mag_1[1]
 
-                huc_gpkg = os.path.join(huc_gpkg, string_manip)
+            if log:
+                log.write(f'  {define_mag[1]}\n')
 
-                define_mag = agree_rast.split(version)
+            stats = perform_zonal_stats(catchment_gpkg, agree_rast)
+            if stats == []:
+                continue
 
-                define_mag_1 = define_mag[1].split('/')
+            get_geom = gpd.read_file(catchment_gpkg)
 
-                mag = define_mag_1[1]
+            get_geom['geometry'] = get_geom.apply(lambda row: make_valid(row.geometry), axis=1)
 
-                stats = perform_zonal_stats(huc_gpkg, agree_rast)
-                if stats == []:
-                    continue
+            in_mem_df = assemble_hydro_alpha_for_single_huc(
+                stats, test_case_class.huc, mag, test_case_class.benchmark_cat
+            )
 
-                print('assembling_hydroalpha_for_single_huc')
-                get_geom = gpd.read_file(huc_gpkg)
+            hydro_geom_df = get_geom[["HydroID", "geometry"]]
 
-                get_geom['geometry'] = get_geom.apply(lambda row: make_valid(row.geometry), axis=1)
+            geom_output = hydro_geom_df.merge(in_mem_df, on='HydroID', how='inner').to_crs('EPSG:3857')
 
-                in_mem_df = assemble_hydro_alpha_for_single_huc(
-                    stats, test_case_class.huc, mag, test_case_class.benchmark_cat
-                )
+            concat_df_list = [geom_output, csv_output]
 
-                hydro_geom_df = get_geom[["HydroID", "geometry"]]
+            csv_output = pd.concat(concat_df_list, sort=False)
 
-                geom_output = hydro_geom_df.merge(in_mem_df, on='HydroID', how='inner')
+    if missing_hucs:
+        log.write(
+            f"There were {len(missing_hucs)} HUCs missing from the input FIM version:\n"
+            + "\n".join([h.fim_dir for h in missing_hucs])
+        )
 
-                concat_df_list = [geom_output, csv_output]
+    print()
+    print(csv_output.groupby('BENCH').size())
+    print(f'total     {len(csv_output)}')
+    log.write("\n------------------------------------\n")
+    csv_output.groupby('BENCH').size().to_string(log)
+    log.write(f'\ntotal     {len(csv_output)}\n')
+
+    print('Writing to GPKG')
+    log.write(f'Writing geopackage {csv}\n')
+    csv_output.to_file(csv, driver="GPKG")
 
-                csv_output = pd.concat(concat_df_list, sort=False)
+    # Add version information to csv_output dataframe
+    csv_output['version'] = version
 
-    print('projecting to 3857')
-    csv_output = csv_output.to_crs('EPSG:3857')
+    print('Writing to CSV')
+    csv_path = csv.replace(".gpkg", ".csv")
+    log.write(f'Writing CSV {csv_path}\n')
+    csv_output.to_csv(csv_path)  # Save to CSV
 
-    print('manipulating the input string to exclude gpkg and include csv')
-    csv_path_list = csv.split(".")
-    csv_path = csv_path_list[0]
-    csv_path_dot = csv_path + ".csv"
 
-    print('writing_to_gpkg')
-    csv_output.to_file(csv, driver="GPKG")
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Produces alpha metrics by hyrdoid.')
 
-    # Add version information to csv_output dataframe
-    csv_output['version'] = version
+    parser.add_argument(
+        '-b',
+        '--benchmark_category',
+        help='Choice of truth data. Options are: all, ble, ifc, nws, usgs, ras2fim',
+        required=True,
+    )
+    parser.add_argument(
+        '-v', '--version', help='The fim version to use. Should be similar to fim_3_0_24_14_ms', required=True
+    )
+    parser.add_argument(
+        '-g',
+        '--gpkg',
+        help='Filepath and filename to hold exported gpkg file. '
+        'Similar to /data/path/fim_performance_catchments.gpkg. A CSV with the same name will also be written.',
+        required=True,
+    )
+    parser.add_argument(
+        '-l',
+        '--log',
+        help='Optional flag to write a log file with the same name as the --GPKG.',
+        required=False,
+        default=None,
+        action='store_true',
+    )
 
-    print('writing_to_csv')
-    csv_output.to_csv(csv_path_dot)  # Save to CSV
+    # Assign variables from arguments.
+    args = vars(parser.parse_args())
+    benchmark_category = args['benchmark_category']
+    version = args['version']
+    csv = args['gpkg']
+    log = args['log']
 
     print("================================")
-    print("End test_case_by_hydroid.py")
+    print("Start test_case_by_hydroid.py")
+    start_time = datetime.now()
+    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print(f"started: {dt_string}")
+    print()
+
+    ## Initiate log file
+    if log:
+        log = open(csv.replace('.gpkg', '.log'), "w")
+        log.write('START TIME: ' + str(start_time) + '\n')
+        log.write('#########################################################\n\n')
+        log.write('')
+        log.write(f'Runtime args:\n {args}\n\n')
+
+    # This is the main execution -- try block is to catch and log errors
+    try:
+        catchment_zonal_stats(benchmark_category, version, csv, log)
+    except Exception as ex:
+        print(f"ERROR: Execution failed. Please check the log file for details. \n {log.name if log else ''}")
+        if log:
+            log.write(f"ERROR -->\n{ex}")
+        traceback.print_exc(file=log)
+        if log:
+            log.write(f'Errored at: {str(datetime.now().strftime("%m/%d/%Y %H:%M:%S"))} \n')
 
     end_time = datetime.now()
-    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    dt_string = end_time.strftime("%m/%d/%Y %H:%M:%S")
+    tot_run_time = end_time - start_time
+    if log:
+        log.write(f'END TIME: {str(end_time)} \n')
+        log.write(f'TOTAL RUN TIME: {str(tot_run_time)} \n')
+        log.close()
+
+    print("================================")
+    print("End test_case_by_hydroid.py")
+
     print(f"ended: {dt_string}")
 
-    # calculate duration
-    time_duration = end_time - start_time
-    print(f"Duration: {str(time_duration).split('.')[0]}")
+    print(f"Duration: {str(tot_run_time).split('.')[0]}")
     print()

From 5b86929f2bbf58c7eaaf3f17fa544c4f3b2a7eda Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 13 Sep 2024 15:36:21 -0400
Subject: [PATCH 346/359] v4.5.7.0 Use fiona for writing files (#1267)

---
 data/esri.py                                  |  2 +-
 data/nld/levee_download.py                    |  6 +++---
 data/usgs/acquire_and_preprocess_3dep_dems.py |  2 +-
 data/usgs/rating_curve_get_usgs_curves.py     |  8 +++++---
 data/wbd/preprocess_wbd.py                    |  2 +-
 docs/CHANGELOG.md                             | 13 +++++++++++++
 src/derive_headwaters.py                      |  2 +-
 src/derive_level_paths.py                     | 13 +++++++++----
 src/edit_points.py                            |  4 ++--
 src/filter_catchments_and_add_attributes.py   |  6 ++++--
 src/reachID_grid_to_vector_points.py          |  4 +++-
 src/split_flows.py                            |  8 ++++++--
 src/src_adjust_spatial_obs.py                 |  4 ++--
 src/src_roughness_optimization.py             | 16 +++++++++++++---
 src/stream_branches.py                        | 10 +++++++++-
 tools/eval_plots.py                           |  4 ++--
 tools/evaluate_continuity.py                  |  2 +-
 tools/generate_nws_lid.py                     |  2 +-
 tools/make_boxes_from_bounds.py               |  4 +++-
 tools/mosaic_inundation.py                    |  2 +-
 tools/rating_curve_comparison.py              |  2 +-
 tools/test_case_by_hydro_id.py                |  2 +-
 22 files changed, 83 insertions(+), 35 deletions(-)

diff --git a/data/esri.py b/data/esri.py
index 9a762fb5f..fdf3c443e 100644
--- a/data/esri.py
+++ b/data/esri.py
@@ -50,7 +50,7 @@ def query(cls, query_url: str, save_file: str = None, **kwargs):
         gdf_complete = rest_call._query_rest()
         # Save geodataframe as geopackage
         if save_file:
-            gdf_complete.to_file(save_file, driver="GPKG", index=False)
+            gdf_complete.to_file(save_file, driver="GPKG", index=False, engine='fiona')
         else:
             return gdf_complete
 
diff --git a/data/nld/levee_download.py b/data/nld/levee_download.py
index 8893f00bb..765ce23f6 100755
--- a/data/nld/levee_download.py
+++ b/data/nld/levee_download.py
@@ -42,7 +42,7 @@ def download_nld_lines():
     )
 
     # Write levees to a single geopackage
-    levees.to_file(nld_vector_output, index=False, driver='GPKG')
+    levees.to_file(nld_vector_output, index=False, driver='GPKG', engine='fiona')
     print(f"Levees written to file:\n{nld_vector_output}")
 
     # Spatial join to huc2
@@ -75,7 +75,7 @@ def process_levee_lines(levee_gdf: gpd.GeoDataFrame, out_levees: str):
     levee_gdf['geometry'] = levee_gdf.progress_apply(lambda row: remove_nulls(row.geometry, row.HUC2), axis=1)
     # Remove levees that have empty geometries resulting from the previous filter
     levee_gdf = levee_gdf[~levee_gdf.is_empty]
-    levee_gdf.to_file(out_levees, index=False, driver='GPKG')
+    levee_gdf.to_file(out_levees, index=False, driver='GPKG', engine='fiona')
     print(f"Preprocessed levees written to \n{out_levees}")
 
 
@@ -160,7 +160,7 @@ def download_nld_poly():
     )
 
     # Write levees to a single geopackage
-    leveed_areas.to_file(nld_protected_areas, index=False, driver='GPKG')
+    leveed_areas.to_file(nld_protected_areas, index=False, driver='GPKG', engine='fiona')
     print(f"Levees written to file:\n{nld_protected_areas}")
 
 
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index c43ce1e44..56f21aee8 100755
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -413,7 +413,7 @@ def polygonize(target_output_folder_path):
 
     dem_gpkgs['DN'] = 1
     dem_dissolved = dem_gpkgs.dissolve(by='DN')
-    dem_dissolved.to_file(dem_domain_file, driver='GPKG')
+    dem_dissolved.to_file(dem_domain_file, driver='GPKG', engine='fiona')
 
     if not os.path.exists(dem_domain_file):
         msg = f" - Polygonizing -- {dem_domain_file} - Failed"
diff --git a/data/usgs/rating_curve_get_usgs_curves.py b/data/usgs/rating_curve_get_usgs_curves.py
index e77e92857..5741f69c5 100644
--- a/data/usgs/rating_curve_get_usgs_curves.py
+++ b/data/usgs/rating_curve_get_usgs_curves.py
@@ -370,7 +370,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
         sites_gdf['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False
     )
 
-    sites_gdf.to_file(os.path.join(workspace, 'sites_bool_flags.gpkg'), driver='GPKG')
+    sites_gdf.to_file(os.path.join(workspace, 'sites_bool_flags.gpkg'), driver='GPKG', engine='fiona')
 
     # Filter and save filtered file for viewing
     acceptable_sites_gdf = sites_gdf[
@@ -379,7 +379,7 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
     acceptable_sites_gdf = acceptable_sites_gdf[acceptable_sites_gdf['curve'] == 'yes']
     acceptable_sites_gdf.to_csv(os.path.join(workspace, 'acceptable_sites_for_rating_curves.csv'))
     acceptable_sites_gdf.to_file(
-        os.path.join(workspace, 'acceptable_sites_for_rating_curves.gpkg'), driver='GPKG'
+        os.path.join(workspace, 'acceptable_sites_for_rating_curves.gpkg'), driver='GPKG', engine='fiona'
     )
 
     # Make list of acceptable sites
@@ -404,7 +404,9 @@ def usgs_rating_to_elev(list_of_gage_sites, workspace=False, sleep_time=1.0):
         # If 'all' option specified, reproject then write out shapefile of acceptable sites.
         if list_of_gage_sites == ['all']:
             sites_gdf = sites_gdf.to_crs(PREP_PROJECTION)
-            sites_gdf.to_file(Path(workspace) / 'usgs_gages.gpkg', layer='usgs_gages', driver='GPKG')
+            sites_gdf.to_file(
+                Path(workspace) / 'usgs_gages.gpkg', layer='usgs_gages', driver='GPKG', engine='fiona'
+            )
 
         # Write out flow files for each threshold across all sites
         write_categorical_flow_files(metadata_list, workspace)
diff --git a/data/wbd/preprocess_wbd.py b/data/wbd/preprocess_wbd.py
index 0a9d4ef46..76d46d829 100755
--- a/data/wbd/preprocess_wbd.py
+++ b/data/wbd/preprocess_wbd.py
@@ -39,7 +39,7 @@ def clip_wbd_to_dem_domain(dem: str, wbd_in: str, wbd_out: str, huc_level: int):
         wbd = gpd.clip(wbd, dem_domain)
 
         # Write output file
-        wbd.to_file(wbd_out, layer=layer, crs=DEFAULT_FIM_PROJECTION_CRS, driver='GPKG')
+        wbd.to_file(wbd_out, layer=layer, crs=DEFAULT_FIM_PROJECTION_CRS, driver='GPKG', engine='fiona')
 
 
 if __name__ == '__main__':
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 436b10c45..e80b57d35 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.7.0 - 2024-09-13 - [PR#1267](https://github.com/NOAA-OWP/inundation-mapping/pull/1267)
+
+`pyogrio` seems to have a difficulty writing files when all values in a column are null (None or nan). The workaround here is to use `fiona` for writing files where `pyogrio` is explicitly set in geopandas (gpd) by `gpd.options.io_engine = "pyogrio"`.
+
+### Changes
+Adds `engine='fiona'` to `.to_file()` in all of the following files
+- `data/`: `esri.py`, `nld/levee_download.py`, `usgs/acquire_and_preprocess_3dep_dems.py`, `usgs/rating_curve_get_usgs_curves.py`, `wbd/preprocess_wbd.py`
+- `src/`: `derive_headwaters.py`, `derive_level_paths.py`, `edit_points.py`, `filter_catchments_and_add_attributes.py`, `reachID_grid_to_vector_points.py`, `reachID_grid_to_vector_points.py`, `split_flows.py`, `src_adjust_spatial_obs.py`, `src_roughness_optimization.py`, `stream_branches.py`
+- `tools/`: `eval_plots.py`, `evaluate_continuity.py`, `generate_nws_lid.py`, `make_boxes_from_bounds.py`, `mosaic_inundation.py`, `rating_curve_comparison.py`, `test_case_by_hydro_id.py`
+
+<br/><br/>
+
+
 ## v4.5.6.1 - 2024-09-13 - [PR#1271](https://github.com/NOAA-OWP/inundation-mapping/pull/1271)
 
 Upgrade for `test_case_by_hydro_id.py` that enables the ability to run on HUCs with differing projections (e.g. Alaska) and adds a logging system.
diff --git a/src/derive_headwaters.py b/src/derive_headwaters.py
index 19d9dcf76..9015bb68c 100644
--- a/src/derive_headwaters.py
+++ b/src/derive_headwaters.py
@@ -66,4 +66,4 @@ def findHeadWaterPoints(flows):
     output_headwaters = args['output_headwaters']
 
     if output_headwaters is not None:
-        hw_gdf.to_file(args['output_headwaters'], driver=getDriver(args['output_headwaters']))
+        hw_gdf.to_file(args['output_headwaters'], driver=getDriver(args['output_headwaters']), engine='fiona')
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index 36e3fd437..97429f865 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -162,7 +162,7 @@ def Derive_level_paths(
 
         catchments = catchments.reset_index(drop=True)
 
-        catchments.to_file(catchments_outfile, index=False, driver="GPKG")
+        catchments.to_file(catchments_outfile, index=False, driver="GPKG", engine='fiona')
 
     # derive headwaters
     if headwaters_outfile is not None:
@@ -170,7 +170,7 @@ def Derive_level_paths(
             inlets_attribute=inlets_attribute, outlet_linestring_index=outlet_linestring_index
         )
         # headwaters write
-        headwaters.to_file(headwaters_outfile, index=False, driver="GPKG")
+        headwaters.to_file(headwaters_outfile, index=False, driver="GPKG", engine='fiona')
 
     if out_stream_network is not None:
         if verbose:
@@ -209,9 +209,14 @@ def Derive_level_paths(
             feature_attribute=branch_id_attribute, outlet_linestring_index=outlet_linestring_index
         )
 
-        branch_inlets.to_file(branch_inlets_outfile, index=False, driver="GPKG")
+        if not branch_inlets.empty:
+            branch_inlets.to_file(branch_inlets_outfile, index=False, driver="GPKG", engine='fiona')
 
-    return stream_network
+    if stream_network.empty:
+        print("Sorry, no streams exist and processing can not continue. This could be an empty file.")
+        sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
+    else:
+        return stream_network
 
 
 if __name__ == "__main__":
diff --git a/src/edit_points.py b/src/edit_points.py
index 2a91a3471..8aa148a94 100755
--- a/src/edit_points.py
+++ b/src/edit_points.py
@@ -36,7 +36,7 @@ def Edit_points(
 
     # join on HydroID to add branch_id
     if out_reach_points is not None:
-        reach_points.to_file(out_reach_points, driver='GPKG', index=False)
+        reach_points.to_file(out_reach_points, driver='GPKG', index=False, engine='fiona')
 
     # make pixel points
     if verbose:
@@ -46,7 +46,7 @@ def Edit_points(
     pixel_points['id'] = list(range(1, len(pixel_points) + 1))
 
     if out_pixel_points is not None:
-        pixel_points.to_file(out_pixel_points, driver='GPKG', index=False)
+        pixel_points.to_file(out_pixel_points, driver='GPKG', index=False, engine='fiona')
 
     return (reach_points, pixel_points)
 
diff --git a/src/filter_catchments_and_add_attributes.py b/src/filter_catchments_and_add_attributes.py
index d7e35444c..a9f3b4649 100755
--- a/src/filter_catchments_and_add_attributes.py
+++ b/src/filter_catchments_and_add_attributes.py
@@ -55,8 +55,10 @@ def filter_catchments_and_add_attributes(
 
         if not output_catchments.empty:
             try:
-                output_catchments.to_file(output_catchments_filename, driver="GPKG", index=False)
-                output_flows.to_file(output_flows_filename, driver="GPKG", index=False)
+                output_catchments.to_file(
+                    output_catchments_filename, driver="GPKG", index=False, engine='fiona'
+                )
+                output_flows.to_file(output_flows_filename, driver="GPKG", index=False, engine='fiona')
             except ValueError:
                 # this is not an exception, but a custom exit code that can be trapped
                 print("There are no flowlines in the HUC after stream order filtering.")
diff --git a/src/reachID_grid_to_vector_points.py b/src/reachID_grid_to_vector_points.py
index 08e051b61..022fe1bc2 100755
--- a/src/reachID_grid_to_vector_points.py
+++ b/src/reachID_grid_to_vector_points.py
@@ -51,7 +51,9 @@ def convert_grid_cells_to_points(raster, index_option, output_points_filename=Fa
     if output_points_filename is False:
         return pointGDF
     else:
-        pointGDF.to_file(output_points_filename, driver=getDriver(output_points_filename), index=False)
+        pointGDF.to_file(
+            output_points_filename, driver=getDriver(output_points_filename), index=False, engine='fiona'
+        )
 
 
 if __name__ == '__main__':
diff --git a/src/split_flows.py b/src/split_flows.py
index 6a7b53e16..f44daeb43 100755
--- a/src/split_flows.py
+++ b/src/split_flows.py
@@ -430,11 +430,15 @@ def snap_and_trim_flow(snapped_point, flows):
         print("There are no flowlines after stream order filtering.")
         sys.exit(FIM_exit_codes.NO_FLOWLINES_EXIST.value)  # Note: Will send a 61 back
 
-    split_flows_gdf.to_file(split_flows_filename, driver=getDriver(split_flows_filename), index=False)
+    split_flows_gdf.to_file(
+        split_flows_filename, driver=getDriver(split_flows_filename), index=False, engine='fiona'
+    )
 
     if len(split_points_gdf) == 0:
         raise Exception("No points exist.")
-    split_points_gdf.to_file(split_points_filename, driver=getDriver(split_points_filename), index=False)
+    split_points_gdf.to_file(
+        split_points_filename, driver=getDriver(split_points_filename), index=False, engine='fiona'
+    )
 
 
 if __name__ == '__main__':
diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py
index fba11571f..4c298ff53 100644
--- a/src/src_adjust_spatial_obs.py
+++ b/src/src_adjust_spatial_obs.py
@@ -108,7 +108,7 @@ def process_points(args):
             branch_debug_pts_out_gpkg = os.path.join(
                 branch_dir, 'export_water_edge_df_' + branch_id + '.gpkg'
             )
-            water_edge_df.to_file(branch_debug_pts_out_gpkg, driver='GPKG', index=False)
+            water_edge_df.to_file(branch_debug_pts_out_gpkg, driver='GPKG', index=False, engine='fiona')
 
         # print('Processing points for HUC: ' + str(huc) + '  Branch: ' + str(branch_id))
         ## Get median HAND value for appropriate groups.
@@ -281,7 +281,7 @@ def ingest_points_layer(fim_directory, job_number, debug_outputs_option, log_fil
             huc_debug_pts_out = os.path.join(fim_directory, huc, 'debug_water_edge_df_' + huc + '.csv')
             water_edge_df.to_csv(huc_debug_pts_out)
             huc_debug_pts_out_gpkg = os.path.join(fim_directory, huc, 'export_water_edge_df_' + huc + '.gpkg')
-            water_edge_df.to_file(huc_debug_pts_out_gpkg, driver='GPKG', index=False)
+            water_edge_df.to_file(huc_debug_pts_out_gpkg, driver='GPKG', index=False, engine='fiona')
             # write parquet file using ".to_parquet() method"
             parquet_filepath = os.path.join(fim_directory, huc, 'debug_water_edge_df_' + huc + '.parquet')
             water_edge_df.to_parquet(parquet_filepath, index=False)
diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py
index 3f33e909c..2e913d920 100644
--- a/src/src_roughness_optimization.py
+++ b/src/src_roughness_optimization.py
@@ -465,7 +465,11 @@ def update_rating_curve(
 
                         try:
                             output_catchments.to_file(
-                                catchments_poly_path, driver="GPKG", index=False, overwrite=True
+                                catchments_poly_path,
+                                driver="GPKG",
+                                index=False,
+                                overwrite=True,
+                                engine='fiona',
                             )  # overwrite the previous layer
 
                         except Exception as e:
@@ -483,7 +487,11 @@ def update_rating_curve(
                             try:
                                 # Attempt to write to the file again
                                 output_catchments.to_file(
-                                    catchments_poly_path, driver="GPKG", index=False, overwrite=True
+                                    catchments_poly_path,
+                                    driver="GPKG",
+                                    index=False,
+                                    overwrite=True,
+                                    engine='fiona',
                                 )
                                 log_text += 'Successful second attempt to write output_catchments gpkg' + '\n'
                             except Exception as e:
@@ -515,7 +523,9 @@ def update_rating_curve(
                             "gw_catchments_src_adjust_" + str(branch_id) + ".gpkg",
                         )
                         output_catchments = input_catchments.merge(df_nmerge, how='left', on='HydroID')
-                        output_catchments.to_file(output_catchments_fileName, driver="GPKG", index=False)
+                        output_catchments.to_file(
+                            output_catchments_fileName, driver="GPKG", index=False, engine='fiona'
+                        )
                         output_catchments = None
 
                 ## Merge the final ManningN dataframe to the original hydroTable
diff --git a/src/stream_branches.py b/src/stream_branches.py
index d0f918500..591281477 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import sys
 from collections import deque
 from os.path import isfile, splitext
 from random import sample
@@ -19,6 +20,7 @@
 from shapely.strtree import STRtree
 from tqdm import tqdm
 
+from utils.fim_enums import FIM_exit_codes
 from utils.shared_variables import PREP_CRS
 
 
@@ -134,7 +136,7 @@ def write(self, fileName, layer=None, index=True, verbose=False):
         driverDictionary = {".gpkg": "GPKG", ".geojson": "GeoJSON", ".shp": "ESRI Shapefile"}
         driver = driverDictionary[splitext(fileName)[1]]
 
-        self.to_file(fileName, driver=driver, layer=layer, index=index)
+        self.to_file(fileName, driver=driver, layer=layer, index=index, engine='fiona')
 
     def set_index(self, reach_id_attribute, drop=True):
         branch_id_attribute = self.branch_id_attribute
@@ -732,6 +734,12 @@ def select_branches_intersecting_huc(self, wbd, buffer_wbd_streams, out_vector_f
                 if verbose:
                     print("Writing selected branches ...")
 
+                if self.empty:
+                    print(
+                        "Sorry, no streams exist and processing can not continue. This could be an empty file."
+                    )
+                    sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
+
                 self.write(out_vector_files, index=False)
 
         return self
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index d7ff97209..e534582a5 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -891,7 +891,7 @@ def eval_plots(
             joined = gdf.merge(all_ahps_datasets, on='nws_lid')
             # Project to VIZ projection and write to file
             joined = joined.to_crs(VIZ_PROJECTION)
-            joined.to_file(Path(workspace) / 'fim_performance_points.shp')
+            joined.to_file(Path(workspace) / 'fim_performance_points.shp', engine='fiona')
         else:
             print(
                 'NWS/USGS MS datasets not analyzed, no spatial data created.\n'
@@ -949,7 +949,7 @@ def eval_plots(
             # Project to VIZ projection
             wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION)
             # Write out to file
-            wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp')
+            wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp', engine='fiona')
         else:
             print(
                 'BLE/IFC/RAS2FIM FR datasets not analyzed, no spatial data created.\n'
diff --git a/tools/evaluate_continuity.py b/tools/evaluate_continuity.py
index 3d86f063e..48cfe3530 100644
--- a/tools/evaluate_continuity.py
+++ b/tools/evaluate_continuity.py
@@ -113,7 +113,7 @@ def evaluate_continuity(
         fig.savefig(plot_file)
 
     if stream_network_outfile is not None:
-        stream_network.to_file(stream_network_outfile, index=False, driver='GPKG')
+        stream_network.to_file(stream_network_outfile, index=False, driver='GPKG', engine='fiona')
 
     return stream_network
 
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
index b91e0fe94..f6a8c77f2 100644
--- a/tools/generate_nws_lid.py
+++ b/tools/generate_nws_lid.py
@@ -122,7 +122,7 @@ def generate_nws_lid(workspace):
     nws_lid_gdf.reset_index(drop=True)
     Path(workspace).mkdir(parents=True, exist_ok=True)
     nws_lid_gdf.dropna(subset=['nwm_feature_id'], inplace=True)
-    nws_lid_gdf.to_file(Path(workspace) / 'nws_lid.gpkg', driver='GPKG')
+    nws_lid_gdf.to_file(Path(workspace) / 'nws_lid.gpkg', driver='GPKG', engine='fiona')
 
 
 if __name__ == '__main__':
diff --git a/tools/make_boxes_from_bounds.py b/tools/make_boxes_from_bounds.py
index 9f0101d93..8c29f5918 100755
--- a/tools/make_boxes_from_bounds.py
+++ b/tools/make_boxes_from_bounds.py
@@ -37,7 +37,9 @@ def find_hucs_of_bounding_boxes(
     bounding_boxes = bounding_boxes.to_crs(wbd_proj)
 
     if bounding_boxes_outfile is not None:
-        bounding_boxes.to_file(bounding_boxes_outfile, driver=getDriver(bounding_boxes_outfile), index=False)
+        bounding_boxes.to_file(
+            bounding_boxes_outfile, driver=getDriver(bounding_boxes_outfile), index=False, engine='fiona'
+        )
 
     wbdcol_name = 'HUC' + wbd_layer[-1]
 
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 4513f067f..5bded6538 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -185,7 +185,7 @@ def mosaic_final_inundation_extent_to_poly(inundation_raster, inundation_polygon
         ]
 
         # Write polygon
-        extent_poly_diss.to_file(inundation_polygon, driver=driver)
+        extent_poly_diss.to_file(inundation_polygon, driver=driver, engine='fiona')
 
 
 if __name__ == "__main__":
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index 0a9e58acb..a57b26fc1 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -1082,7 +1082,7 @@ def create_static_gpkg(output_dir, output_gpkg, agg_recurr_stats_table, gages_gp
     usgs_gages = usgs_gages.round(decimals=2)
 
     # Write to file
-    usgs_gages.to_file(join(output_dir, output_gpkg), driver='GPKG', index=False)
+    usgs_gages.to_file(join(output_dir, output_gpkg), driver='GPKG', index=False, engine='fiona')
 
     # Create figure
     usgs_gages.replace(np.inf, np.nan, inplace=True)  # replace inf with nan for plotting
diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py
index 73186f33e..2f4f5a539 100644
--- a/tools/test_case_by_hydro_id.py
+++ b/tools/test_case_by_hydro_id.py
@@ -298,7 +298,7 @@ def catchment_zonal_stats(benchmark_category, version, csv, log):
 
     print('Writing to GPKG')
     log.write(f'Writing geopackage {csv}\n')
-    csv_output.to_file(csv, driver="GPKG")
+    csv_output.to_file(csv, driver="GPKG", engine='fiona')
 
     # Add version information to csv_output dataframe
     csv_output['version'] = version

From 20640c9a3f3155a90f92df7b792ddf7f7e37597d Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 13 Sep 2024 15:44:37 -0400
Subject: [PATCH 347/359] v4.5.7.1 Mosaic depths and extents (#1246)

---
 docs/CHANGELOG.md                | 12 ++++++++++++
 tools/inundate_mosaic_wrapper.py | 28 ++++++++++++++--------------
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index e80b57d35..898cb56f4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.5.7.1 - 2024-09-13 - [PR#1246](https://github.com/NOAA-OWP/inundation-mapping/pull/1246)
+
+Indents the mosaicking block so that `inundate_mosaic_wrapper.py` mosaics both inundation extents and depths.
+
+### Changes
+
+- `tools/inundate_mosaic_wrapper.py`: Moves mosaicking inside `for` loop.
+
+ <br/><br/>
+
+ 
 ## v4.5.7.0 - 2024-09-13 - [PR#1267](https://github.com/NOAA-OWP/inundation-mapping/pull/1267)
 
 `pyogrio` seems to have a difficulty writing files when all values in a column are null (None or nan). The workaround here is to use `fiona` for writing files where `pyogrio` is explicitly set in geopandas (gpd) by `gpd.options.io_engine = "pyogrio"`.
diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py
index 5b274122a..2e68e9a6b 100644
--- a/tools/inundate_mosaic_wrapper.py
+++ b/tools/inundate_mosaic_wrapper.py
@@ -122,20 +122,20 @@ def produce_mosaicked_inundation(
             if depths_raster is not None:
                 mosaic_output = depths_raster
 
-    if mosaic_output is not None:
-        # Call Mosaic_inundation
-        mosaic_file_path = Mosaic_inundation(
-            map_file.copy(),
-            mosaic_attribute=mosaic_attribute,
-            mosaic_output=mosaic_output,
-            mask=mask,
-            unit_attribute_name=unit_attribute_name,
-            nodata=elev_raster_ndv,
-            remove_inputs=remove_intermediate,
-            verbose=verbose,
-            is_mosaic_for_branches=is_mosaic_for_branches,
-            inundation_polygon=inundation_polygon,
-        )
+        if mosaic_output is not None:
+            # Call Mosaic_inundation
+            mosaic_file_path = Mosaic_inundation(
+                map_file.copy(),
+                mosaic_attribute=mosaic_attribute,
+                mosaic_output=mosaic_output,
+                mask=mask,
+                unit_attribute_name=unit_attribute_name,
+                nodata=elev_raster_ndv,
+                remove_inputs=remove_intermediate,
+                verbose=verbose,
+                is_mosaic_for_branches=is_mosaic_for_branches,
+                inundation_polygon=inundation_polygon,
+            )
 
     fh.vprint("Mosaicking complete.", verbose)
 

From 4e4ba68b54528364594bf05897b537c3a6dc9e6b Mon Sep 17 00:00:00 2001
From: Riley McDermott <139250904+RileyMcDermott-NOAA@users.noreply.github.com>
Date: Fri, 13 Sep 2024 14:48:12 -0500
Subject: [PATCH 348/359] v4.5.7.2 Catchment Boundary Tool Addition (#1149)

---
 docs/CHANGELOG.md                    |  10 ++
 tools/identify_catchment_boundary.py | 157 +++++++++++++++++++++++++++
 2 files changed, 167 insertions(+)
 mode change 100644 => 100755 docs/CHANGELOG.md
 create mode 100755 tools/identify_catchment_boundary.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
old mode 100644
new mode 100755
index 898cb56f4..18d83ad44
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,16 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.7.2 - 2024-09-13 - [PR#1149](https://github.com/NOAA-OWP/inundation-mapping/pull/1149)
+
+This PR adds scripts that can identify areas within produced inundation rasters where glasswalling of inundation occurs due to catchment boundaries, know as catchment boundary issues.
+
+### Additions
+- `tools/identify_catchment_boundary.py`: Identifies where catchment boundaries are glasswalling inundation extent.
+
+- `tools/inundate_catchment_boundary.py`: Produces inundation for given HUC and identifies catchment boundary issues in produced FIM. 
+
+ <br/><br/>
 
 ## v4.5.7.1 - 2024-09-13 - [PR#1246](https://github.com/NOAA-OWP/inundation-mapping/pull/1246)
 
diff --git a/tools/identify_catchment_boundary.py b/tools/identify_catchment_boundary.py
new file mode 100755
index 000000000..259d525dd
--- /dev/null
+++ b/tools/identify_catchment_boundary.py
@@ -0,0 +1,157 @@
+import argparse
+import os
+from timeit import default_timer as timer
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
+from rasterio import features
+from shapely import geometry, ops
+from shapely.geometry import shape
+
+
+def catchment_boundary_errors(hydrofabric_dir, huc, inundation_raster, output, min_error_length):
+    """
+    This function compares output inundation raster extent to catchment extents to identify catchment boundary
+    issues. The output of this function is a geopackage of lines that identifys sections of inundation with catchment
+    boundary issues present.
+
+    Args:
+        hydrofabric_dir (str):    Path to hydrofabric directory where FIM outputs were written by
+                                    fim_pipeline.
+        huc (str):                The HUC for which to check for catchment boundary issues.
+        inundation_raster (str):  Full path to inundation raster
+                                    (encoded by positive and negative HydroIDs).
+        output (str):             Path to output location for catchment boundary geopackage.
+        min_error_length (int):   Minimum length for output error lines. Default 100 meters.
+    """
+
+    huc = huc[0]
+    print(f'Processing HUC:{huc} now.')
+    # Get branch names for input HUC
+    dirs = [x[1] for x in os.walk(f"{hydrofabric_dir}/{huc}/branches")][0]
+
+    # Merge all catchment geopackages into one file
+    catchments = gpd.GeoDataFrame()
+    for d in dirs:
+        catch = gpd.read_file(
+            f"{hydrofabric_dir}/{huc}/branches/{d}/gw_catchments_reaches_filtered_addedAttributes_crosswalked_{d}.gpkg"
+        )
+        catch = catch.assign(branch_id=d)
+        catchments = pd.concat([catchments, catch], ignore_index=True)
+
+    # Vectorize inundation
+    with rasterio.open(inundation_raster) as src:
+        affine = src.transform
+        band = src.read(1)
+        band[np.where(band <= 0)] = 0
+        band[np.where(band > 0)] = 1
+        results = (
+            {'properties': {'raster_val': v}, 'geometry': s}
+            for i, (s, v) in enumerate(features.shapes(band, mask=None, transform=affine))
+        )
+
+    # Save features to geodataframe and select inundated pixels
+    geoms = list(results)
+    inund_poly = gpd.GeoDataFrame.from_features(geoms)
+    inund_poly = inund_poly.loc[inund_poly['raster_val'] == 1.0]
+
+    # Get boundary lines for inundation
+    inundation_boundary = inund_poly.boundary
+    inundation_boundary = inundation_boundary.set_crs(catchments.boundary.crs)
+
+    # Save boundary lines to geodataframe
+    inundation_boundary_df = gpd.GeoDataFrame(geometry=inundation_boundary)
+    catchment_boundary_df = catchments.assign(geometry=catchments.boundary)
+
+    # Explode geomtries into many individual linestrings
+    catchment_boundary_explode = catchment_boundary_df.explode(ignore_index=True)
+    inundation_boundary_explode = inundation_boundary_df.explode(ignore_index=True)
+
+    # Find where catchment boundary and inundation boundary intersect (catchment boundary errors)
+    intersect = inundation_boundary_explode.overlay(
+        catchment_boundary_explode, how='intersection', keep_geom_type=True
+    )
+    error_lines = gpd.GeoDataFrame()
+    for i in intersect['branch_id'].unique():
+        branch_df = intersect.loc[intersect['branch_id'] == f'{i}']
+        branch_df = branch_df.explode(index_parts=True)
+        branch_df = branch_df.dissolve(by=['HydroID', 'feature_id'], as_index=False)
+
+        for index, row in branch_df.iterrows():
+            dissolved_lines = branch_df.iloc[index, 2]
+            if isinstance(dissolved_lines, geometry.multilinestring.MultiLineString):
+                merged_lines = ops.linemerge(dissolved_lines)
+                branch_df.loc[branch_df['geometry'] == dissolved_lines, 'geometry'] = merged_lines
+        error_lines = pd.concat([error_lines, branch_df])
+    error_lines_explode = error_lines.explode(index_parts=False)
+
+    # Link HydroID and feature_id to error lines
+    hydroid_join = gpd.GeoDataFrame()
+    for i in catchments['branch_id'].unique():
+        catchments1 = catchments.loc[catchments['branch_id'] == f'{i}']
+        ip = inund_poly.set_crs(catchments1.crs)
+        poly_intersect = ip.overlay(catchments1, how='intersection', keep_geom_type=True)
+
+        branch_explode = error_lines_explode.loc[error_lines_explode['branch_id'] == f'{i}']
+
+        poly_join = poly_intersect[['HydroID', 'feature_id', 'branch_id', 'geometry']]
+        branch_join = branch_explode[['HydroID', 'feature_id', 'branch_id', 'geometry']]
+
+        feature_join = branch_join.overlay(poly_join, how='intersection', keep_geom_type=False)
+        lines_joined = feature_join.loc[feature_join.geom_type.isin(['LineString', 'MultiLineString'])]
+
+        lines_drop = lines_joined.drop(columns=['HydroID_1', 'feature_id_1', 'branch_id_1'])
+        rename_attributes = lines_drop.drop_duplicates().rename(
+            columns={'HydroID_2': 'HydroID', 'feature_id_2': 'feature_id', 'branch_id_2': 'branch_id'}
+        )
+        hydroid_join = pd.concat([hydroid_join, rename_attributes])
+
+    # Filter remaining lines by length
+    hydroid_join_len = hydroid_join.assign(Length=hydroid_join.length)
+    error_lines_final = hydroid_join_len.loc[hydroid_join_len['Length'] >= min_error_length].copy()
+    num_catchment_boundary_lines = len(error_lines_final)
+
+    if os.path.exists(output):
+        print(f"{output} already exists. Concatinating now...")
+        existing_error_lines = gpd.read_file(output, engine="pyogrio", use_arrow=True)
+        error_lines_final = pd.concat([existing_error_lines, error_lines_final])
+    error_lines_final.to_file(output, driver="GPKG", index=False)
+
+    print(
+        f'Finished processing huc: {huc}. Number of boundary line issues identified: {num_catchment_boundary_lines}.'
+    )
+
+
+if __name__ == "__main__":
+    # Parse arguments
+    parser = argparse.ArgumentParser(description="Helpful utility to identify catchment boundary errors.")
+    parser.add_argument(
+        "-y",
+        "--hydrofabric_dir",
+        help="Directory path to FIM hydrofabric by processing unit.",
+        required=True,
+        type=str,
+    )
+    parser.add_argument("-u", "--huc", help="HUC to run", required=True, default="", type=str, nargs="+")
+    parser.add_argument(
+        "-i", "--inundation-raster", help="Inundation raster output.", required=True, default=None, type=str
+    )
+    parser.add_argument(
+        "-o", "--output", help="Output geopackage location.", required=True, default=None, type=str
+    )
+    parser.add_argument(
+        '-min',
+        '--min_error_length',
+        help='Minimum length for output error lines. Default is 100 meters.',
+        required=False,
+        type=int,
+        default=100,
+    )
+
+    start = timer()
+
+    catchment_boundary_errors(**vars(parser.parse_args()))
+
+    print(f"Completed in {round((timer() - start)/60, 2)} minutes.")

From a19ca57fef037560030ac9fad2171756f81106f3 Mon Sep 17 00:00:00 2001
From: EmilyDeardorff-NOAA <60829052+EmilyDeardorff@users.noreply.github.com>
Date: Fri, 13 Sep 2024 12:56:24 -0700
Subject: [PATCH 349/359] v4.5.8.0 Fix CatFIM to re-run stage based (Catfim
 V2.0) (#1165)

---
 .pre-commit-config.yaml                   |    1 +
 config/catfim_template.env                |    6 +
 docs/CHANGELOG.md                         |   38 +
 pyproject.toml                            |   16 +-
 src/utils/fim_logger.py                   |  414 +++++
 src/utils/shared_variables.py             |    2 +-
 tools/generate_categorical_fim.py         | 1784 ++++++++++++---------
 tools/generate_categorical_fim_flows.py   |  730 ++++++---
 tools/generate_categorical_fim_mapping.py | 1005 +++++++++---
 tools/generate_nws_lid.py                 |    0
 tools/mosaic_inundation.py                |    2 +-
 tools/tools_shared_functions.py           |   37 +-
 12 files changed, 2813 insertions(+), 1222 deletions(-)
 create mode 100644 config/catfim_template.env
 create mode 100644 src/utils/fim_logger.py
 mode change 100644 => 100755 tools/generate_nws_lid.py

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ff510b185..01acf9e1b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -31,3 +31,4 @@ repos:
     hooks:
       - id: isort
         name: isort (python)
+        args: ["--profile", "black"]
diff --git a/config/catfim_template.env b/config/catfim_template.env
new file mode 100644
index 000000000..f6e2cdd04
--- /dev/null
+++ b/config/catfim_template.env
@@ -0,0 +1,6 @@
+API_BASE_URL=  Enter Words API path here
+EVALUATED_SITES_CSV=/data/inputs/ahp_sites/evaluated_ahps_sites.csv
+WBD_LAYER=/data/inputs/wbd/WBD_National.gpkg
+NWM_FLOWS_MS=/data/inputs/nwm_hydrofabric/nwm_flows_ms_wrds.gpkg
+USGS_METADATA_URL=https://fimnew.wim.usgs.gov
+USGS_DOWNLOAD_URL=https://s3.amazonaws.com/wimcloud.usgs.gov
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 18d83ad44..900119b10 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,44 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.8.0 - 2024-09-13 - [PR#1165](https://github.com/NOAA-OWP/inundation-mapping/pull/1165)
+
+This PR was originally intended to get Alaska HUCs incorporated into CatFIM, but there were a very, very large array of problems and the tool was unable to run. We have made some major modifications and many more will come in the near future. There are partial hooks and commented code for Alaska integration, but temporarily disabled are included and will be handled by a separate branch / PR.
+
+One of the biggest improvement was to add a logging system to track what is breaking and where.  Earlier, there were a very large number of places were errors occurred but they were suppressed and never recorded anywhere. A few put the errors on screen but this is a very long running process tool, which can take 2 days, and any messages to the screen are lost. Now all  errors and warning are caught and at least logged in the master log but also the "warning" or "error" log to help them stand out better. Many of the warnings are truly and fairly rejected but at least we know when and why. When we started working with CatFIM again a few months back, there were show stopping errors and we did not know where or why but now we can find and diagnose them.
+
+All three of the core "generate_catfim...py" files include major amounts of changes to improve variable and function names, improve flow and readability, move functions for better understanding of the product, lots of new inline commenting. However, there is a lot to do but it is on a better footing, is pretty stable and hopefully easier to continue upgrades in the near future.
+
+CatFIM is still considered a WIP but it is fully functional again and more adjustments hopefully will go much quicker and smoother.
+
+Also added a system where a config file can be passed into the CatFIM tools instead of assuming a file name and path of simply ".env" in the tools directory. 
+
+This update also relaxes the coordinate accuracy requirements for stage-based CatFIM, which will result in stage-based CatFIM being generated for more sites. 
+
+#### Informally, this is now known as CatFIM 2.0
+
+
+### Additions
+- `config/catfim_template.env`:  Template version of the required catfim env file. The template keeps all values that are non sensitive but removes one that is. The true catfim.env for OWP can be found in our .. data/config/catfim.env. Example pathing here based on docker mounts.
+
+- `src/utils/fim_logger.py`:  A new multi proc custom logging system, modelled directly off of the proven ras2fim logging system. The reason for this custom system is that the native Python logging is not stable in multi-proc environments and tends to loose data. This new logger can relatively easily be bolted into almost any of our py scripts if required.
+
+### Changes
+- `.pre-commit-config.yaml`: A linting config adjustment.
+- `pyproject.toml`: linting config adjustments
+- `src/utils/shared_variables.py`:  Added a comment
+- `tools`
+    - `generate_categorical_fim.py`: As mentioned above
+    - `generate_categorical_fim_flows.py`: As mentioned above
+    - `generate_categorical_fim_mapping.py`: As mentioned above
+    - `generate_nws_lid.py`:  No real changes but Git thinks something did. It is the same as in current Dev.
+    - `mosaic_inundation.py`: Added a comment
+    - `tools_shared_functions.py`
+         - added some better error handing in a few places, plus some commenting and cleanup.
+         - Added a feature to the `aggregate_wbd_hucs` function to optionally submit a list of HUCs for filtering results.
+
+<br/><br/>
+
 ## v4.5.7.2 - 2024-09-13 - [PR#1149](https://github.com/NOAA-OWP/inundation-mapping/pull/1149)
 
 This PR adds scripts that can identify areas within produced inundation rasters where glasswalling of inundation occurs due to catchment boundaries, know as catchment boundary issues.
diff --git a/pyproject.toml b/pyproject.toml
index ee3862fda..c91682ff3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,8 +46,8 @@ skip-magic-trailing-comma = true
 
 [tool.isort]
 profile = 'black'
-multi_line_output = 3
 line_length = 110
+multi_line_output = 3
 lines_after_imports = 2
 
 [tool.flake8]
@@ -69,12 +69,16 @@ per-file-ignores = """
     src/src_roughness_optimization.py: E712, F841
     src/agreedem.py: E712
     src/build_stream_traversal.py: E722
-    tools/vary_mannings_n_composite.py: E712
-    tools/tools_shared_functions.py: F821, F841, E711
+
+    tools/check_deep_flooding.py: E712
+    tools/eval_alt_catfim.py: F841
+    tools/generate_categorical_fim.py: C901, E712
+    tools/generate_categorical_fim_mapping.py: E712
+    tools/inundation.py: F821
     tools/rating_curve_comparison.py: F821, F841
     tools/run_test_case.py: E711
-    tools/inundation.py: F821
-    tools/eval_alt_catfim.py: F841
-    tools/check_deep_flooding.py: E712
+    tools/tools_shared_functions.py: F821, F841, E711
+    tools/vary_mannings_n_composite.py: E712
+
     data/usgs/rating_curve_get_usgs_curves.py: F841
     """
diff --git a/src/utils/fim_logger.py b/src/utils/fim_logger.py
new file mode 100644
index 000000000..1b9a77ec8
--- /dev/null
+++ b/src/utils/fim_logger.py
@@ -0,0 +1,414 @@
+#!/usr/bin/env python3
+
+import datetime as dt
+import os
+import random
+import traceback
+from pathlib import Path
+
+
+# Careful... You might not put shared_functions here as it can create circular references,
+#  so shared_functions is put inside functions only/as is need
+
+# Why does this class exist instead of useing standard python logging tools or other python packes?
+# Some of those loggers cache the log file output until the end which can be a problem for crashes.
+# Also, none of them are very good at handling multi-processing including file collisions and mixed
+# multi-proc output data into the log file.
+
+# Our answer.. A bit weird, but highly effective. Write to the log file as it goes along.
+# For multi-processing, create a temp log for each process, then when the total process is done, merge
+# those outputs in the master log file.
+
+
+# This is not a perfect system. When rolling multi-proc logs into a rollup log, there can still be
+# collisions but it is rare.
+
+
+class FIM_logger:
+    CUSTOM_LOG_FILES_PATHS = {}
+    LOG_SYSTEM_IS_SETUP = False
+
+    LOG_DEFAULT_FOLDER = ""
+    LOG_FILE_PATH = ""  # full path and file name
+    LOG_WARNING_FILE_PATH = ""
+    LOG_ERROR_FILE_PATH = ""
+
+    LOG_SYS_NOT_SETUP_MSG = "******  Logging to the file system not yet setup.\n"
+    "******  Sometimes this is not setup until after initial validation.\n"
+
+    """
+    Levels available for use
+      trace - does not show in console but goes to the default log file
+      lprint - goes to console and default log file
+        (use use "print" if you want console only)
+      notice - goes to console and default log file (Adds word "NOTICE" to front of output)
+      success - goes to console and default log file (Adds word "SUCCESS" to front of output)
+      warning - goes to console, log file and warning log file (Adds word "WARNING" to front of output)
+      error - goes to console, log file and error log file. Normally used when the error
+          does not kill the application. (Adds word "ERROR" to front of output)
+      critical - goes to console, log file and error log file. Normally used when the
+          program is aborted. (Adds word "CRITICAL" to front of output)
+
+
+    NOTE: While unconfirmed in the Linux world, special backslashs
+    combo's like \r, \t, \n can get create problems.
+
+    """
+
+    # -------------------------------------------------
+    def __get_dt(self):
+        cur_dt = dt.datetime.now()
+        ret_dt = f"{cur_dt.strftime('%Y-%m-%d')} {cur_dt.strftime('%H:%M:%S')}"
+        return ret_dt
+
+    # -------------------------------------------------
+    def calc_log_name_and_path(self, output_log_dir, file_prefix):
+        # setup general logger
+        os.makedirs(output_log_dir, exist_ok=True)
+        start_time = dt.datetime.now(dt.timezone.utc)
+        file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
+        log_file_name = f"{file_prefix}_{file_dt_string}.log"
+        log_output_file = os.path.join(output_log_dir, log_file_name)
+
+        return log_output_file
+
+    # -------------------------------------------------
+    def setup(self, log_file_path: str):
+        """
+        During this process, a second log file will be created as an error file which will
+        duplicate all log message with the levels of ERROR, and CRITICAL.
+
+        input:
+            - log_file_path : ie) /data/catfim/test/logs/gen_catfim.log
+        """
+
+        # -----------
+        # Validation
+        if log_file_path is None:
+            raise ValueError("Error: log_file_path not defined")
+        log_file_path = log_file_path.strip()
+        if log_file_path == "":
+            raise ValueError("Error: log_file_path can not be empty")
+
+        folder_path = os.path.dirname(log_file_path)
+        log_file_name = os.path.basename(log_file_path)
+
+        os.makedirs(folder_path, exist_ok=True)
+
+        self.LOG_DEFAULT_FOLDER = folder_path
+
+        # pull out the file name without extension
+        file_name_parts = os.path.splitext(log_file_name)
+        if len(file_name_parts) != 2:
+            raise ValueError("The submitted log_file_name appears to be an invalid file name")
+
+        self.__calc_warning_error_file_names(log_file_path)
+        self.LOG_FILE_PATH = log_file_path
+
+        # We need to remove the older ones if they already exist. Why? one attempt of running an script
+        # might trigger and log file and an error file. So, it is correct, run again and now we have an
+        # old invalid error file
+        # if os.path.isfile(log_file_path):
+        #     os.remove(log_file_path)
+
+        # if os.path.isfile(self.LOG_ERROR_FILE_PATH):
+        #     os.remove(self.LOG_ERROR_FILE_PATH)
+
+        # if os.path.isfile(self.LOG_WARNING_FILE_PATH):
+        #     os.remove(self.LOG_WARNING_FILE_PATH)
+
+        self.LOG_SYSTEM_IS_SETUP = True
+        return
+
+    # -------------------------------------------------
+    def MP_calc_prefix_name(self, parent_log_output_file, file_prefix, huc: str = ""):
+        """_summary_
+        Uses the file name of the parent log to create the new prefix name (without MP date)
+
+        You don't need to use this method if you don't want to prepend the parent file name prefix
+        """
+
+        parent_log_file_name = os.path.basename(parent_log_output_file).replace(".log", "")
+
+        if huc != "":
+            prefix = f"{parent_log_file_name}--{huc}--{file_prefix}"
+        else:
+            prefix = f"{parent_log_file_name}--{file_prefix}"
+        return prefix
+
+    # -------------------------------------------------
+    def MP_Log_setup(self, parent_log_output_file, file_prefix):
+        """
+        Overview:
+            This is for logs used inside code that is multi-processing, aka. inside the actual functions
+            of the call from Pool
+
+            This method is sort of a wrapper in that it just manually creates a file name
+            using a defined file path.
+
+            As this is an MP file, the parent_log_output_file may have a date in it
+            The file name is calculated as such {file_prefix}-{random 12 digit key}.log()
+            ie) catfim_2024_07_09-16_30_02__012345678901.log
+
+            The extra file portion is added as in MultiProc, you can have dozens of processes
+            and each are loggign to their own file. At then end of an MP, you call a function called merge_log_files
+            which will merge them into a parent log file if requested.
+
+        Inputs:
+            file_prefix (str): a value to prepend to the file names. Often is the name of the function
+               that called this method. Note: Later, when a person has these MP_Logs cleaned up
+               they will use this file_prefix again to search and remove the temp MP_log files as they
+               get rolled back up to the master log file.
+            parent_log_output_file (str): folder location for the files to be created. Note: it has to be in
+               the same folder as the master log file.
+        """
+        # -----------------
+        log_folder = os.path.dirname(parent_log_output_file)
+
+        random_id = random.randrange(1000000000, 99999999999)
+        log_file_name = f"{file_prefix}___{random_id}.log"
+        log_file_path = os.path.join(log_folder, log_file_name)
+
+        self.setup(log_file_path)
+        return
+
+    # -------------------------------------------------
+    def __calc_warning_error_file_names(self, log_file_and_path):
+        """
+        Process:
+            Parses the log_file_and_path to add either the name of _warnings or _errors
+            into the file name.
+            Why not update LOG_WARNING_FILE_PATH and LOG_ERROR_FILE_PATH
+        Input:
+            log_file_and_path: ie) /data/outputs/rob_test/logs/catfim.log
+        Output:
+            Updates LOG_WARNING_FILE_PATH and LOG_ERROR_FILE_PATH variables
+        """
+
+        folder_path = os.path.dirname(log_file_and_path)
+        log_file_name = os.path.basename(log_file_and_path)
+
+        # pull out the file name without extension
+        file_name_parts = os.path.splitext(log_file_name)
+        if len(file_name_parts) != 2:
+            raise ValueError("The submitted log_file_name appears to be an invalid file name")
+
+        # now calc the warning log file
+        self.LOG_WARNING_FILE_PATH = os.path.join(
+            folder_path, file_name_parts[0] + "_warnings" + file_name_parts[1]
+        )
+
+        # now calc the error log file
+        self.LOG_ERROR_FILE_PATH = os.path.join(
+            folder_path, file_name_parts[0] + "_errors" + file_name_parts[1]
+        )
+        return
+
+    # -------------------------------------------------
+    def merge_log_files(self, parent_log_output_file, file_prefix, remove_old_files=True):
+        """
+        Overview:
+            This tool is mostly for merging log files during multi processing which each had their own file.
+
+            This will search all of the files in directory in the same folder as the
+            incoming log_file_and_path. It then looks for all files starting with the
+            file_prefix and adds them to the log file (via log_file_and_path)
+        Inputs:
+            - log_file_and_path: ie) /data/outputs/rob_test/logs/catfim.log
+            - file_prefix: This value must be the start of file names. ie) mp_create_gdf_of_points
+                as in /data/outputs/rob_test/logs/mp_generate_categorical_fim(_231122_1407444333_12345).log
+        """
+
+        # -----------
+        # Validation
+        if parent_log_output_file is None:
+            raise ValueError("Error: parent_log_file_and_path not defined")
+
+        parent_log_output_file = parent_log_output_file.strip()
+
+        if parent_log_output_file == "":
+            raise ValueError("Error: parent log_file_and_path can not be empty")
+
+        folder_path = os.path.dirname(parent_log_output_file)
+        os.makedirs(folder_path, exist_ok=True)
+
+        log_file_list_paths = list(Path(folder_path).rglob(f"{file_prefix}*"))
+        log_file_list = [str(x) for x in log_file_list_paths]
+
+        if len(log_file_list) > 0:
+            log_file_list.sort()
+
+            # we are merging them in order (reg files, then warnings, then errors)
+
+            # open and write to the parent log
+            # This will write all logs including errors and warning
+            with open(parent_log_output_file, 'a') as main_log:
+                # Iterate through list
+                for temp_log_file in log_file_list:
+                    # Open each file in read mode
+                    with open(temp_log_file) as infile:
+                        main_log.write(infile.read())
+                    if "warning" not in temp_log_file and "error" not in temp_log_file:
+                        if remove_old_files:
+                            os.remove(temp_log_file)
+
+            # now the warning files if there are any
+            log_warning_file_list = list(Path(folder_path).rglob(f"{file_prefix}*_warnings*"))
+            if len(log_warning_file_list) > 0:
+                log_warning_file_list.sort()
+                parent_warning_file = parent_log_output_file.replace(".log", "_warnings.log")
+                with open(parent_warning_file, 'a') as warning_log:
+                    # Iterate through list
+                    for temp_log_file in log_warning_file_list:
+                        # Open each file in read mode
+                        with open(temp_log_file) as infile:
+                            warning_log.write(infile.read())
+
+                        if remove_old_files:
+                            os.remove(temp_log_file)
+
+            # now the warning files if there are any
+            log_error_file_list = list(Path(folder_path).rglob(f"{file_prefix}*_errors*"))
+            if len(log_error_file_list) > 0:
+                log_error_file_list.sort()
+                parent_error_file = parent_log_output_file.replace(".log", "_errors.log")
+                # doesn't yet exist, then create a blank one
+                with open(parent_error_file, 'a') as error_log:
+                    # Iterate through list
+                    for temp_log_file in log_error_file_list:
+                        # Open each file in read mode
+                        with open(temp_log_file) as infile:
+                            error_log.write(infile.read())
+
+                        if remove_old_files:
+                            os.remove(temp_log_file)
+
+        return
+
+    # -------------------------------------------------
+    def trace(self, msg):
+        # goes to file only, not console
+        level = "TRACE   "  # keeps spacing the same
+        if self.LOG_FILE_PATH == "":
+            print(self.LOG_SYS_NOT_SETUP_MSG)
+            return
+
+        with open(self.LOG_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        return
+
+    # -------------------------------------------------
+    def lprint(self, msg):
+        # goes to console and log file
+        level = "LPRINT  "  # keeps spacing the same
+        print(f"{msg} ")
+
+        if self.LOG_FILE_PATH == "":
+            print(self.LOG_SYS_NOT_SETUP_MSG)
+            return
+
+        with open(self.LOG_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        return
+
+    # -------------------------------------------------
+    def notice(self, msg):
+        # goes to console and log file
+        level = "NOTICE  "  # keeps spacing the same
+        # print(f"{cl.fore.TURQUOISE_2}{msg}{cl.style.RESET}")
+        print(f"{level}: {msg}")
+
+        if self.LOG_FILE_PATH == "":
+            print(self.LOG_SYS_NOT_SETUP_MSG)
+            return
+
+        with open(self.LOG_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        return
+
+    # -------------------------------------------------
+    def success(self, msg):
+        # goes to console and log file
+        level = "SUCCESS "  # keeps spacing the same
+
+        # c_msg_type = f"{cl.fore.SPRING_GREEN_2B}<{level}>{cl.style.RESET}"
+        # print(f"{self.__get_clog_dt()} {c_msg_type} : {msg}")
+        print(f"{level}: {msg}")
+
+        if self.LOG_FILE_PATH == "":
+            print(self.LOG_SYS_NOT_SETUP_MSG)
+            return
+
+        with open(self.LOG_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        return
+
+    # -------------------------------------------------
+    def warning(self, msg):
+        # goes to console and log file and warning log file
+        level = "WARNING "  # keeps spacing the same
+
+        # c_msg_type = f"{cl.fore.LIGHT_YELLOW}<{level}>{cl.style.RESET}"
+        # print(f"{self.__get_clog_dt()} {c_msg_type} : {msg}")
+        print(f"{level}: {msg}")
+
+        if self.LOG_FILE_PATH == "":
+            print(self.LOG_SYS_NOT_SETUP_MSG)
+            return
+
+        with open(self.LOG_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        # and also write to warning logs
+        with open(self.LOG_WARNING_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        return
+
+    # -------------------------------------------------
+    def error(self, msg):
+        # goes to console and log file and error log file
+        level = "ERROR   "  # keeps spacing the same
+
+        # c_msg_type = f"{cl.fore.RED_1}<{level}>{cl.style.RESET}"
+        # print(f"{self.__get_clog_dt()} {c_msg_type} : {msg}")
+        print(f"{level}{msg}")
+
+        if self.LOG_FILE_PATH == "":
+            print(self.LOG_SYS_NOT_SETUP_MSG)
+            return
+
+        with open(self.LOG_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        # and also write to error logs
+        with open(self.LOG_ERROR_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        return
+
+    # -------------------------------------------------
+    def critical(self, msg):
+        level = "CRITICAL"  # keeps spacing the same
+
+        # c_msg_type = f"{cl.style.BOLD}{cl.fore.RED_3A}{cl.back.WHITE}{self.__get_dt()}"
+        # c_msg_type += f" <{level}>"
+        # print(f" {c_msg_type} : {msg} {cl.style.RESET}")
+        print(f"{level}{msg}")
+
+        if self.LOG_FILE_PATH == "":
+            print(self.LOG_SYS_NOT_SETUP_MSG)
+            return
+
+        with open(self.LOG_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        # and also write to error logs
+        with open(self.LOG_ERROR_FILE_PATH, "a") as f_log:
+            f_log.write(f"{self.__get_dt()} | {level} || {msg}\n")
+
+        return
diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
index 72a28a66b..8ea520e76 100644
--- a/src/utils/shared_variables.py
+++ b/src/utils/shared_variables.py
@@ -38,7 +38,7 @@
     '0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],'
     'PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],'
     'PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
-)
+)  # 3857
 
 # -- Data URLs-- #
 NHD_URL_PARENT = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/'
diff --git a/tools/generate_categorical_fim.py b/tools/generate_categorical_fim.py
index e3d248be2..5d2a3bcde 100755
--- a/tools/generate_categorical_fim.py
+++ b/tools/generate_categorical_fim.py
@@ -4,20 +4,28 @@
 import csv
 import glob
 import os
+import shutil
 import sys
 import time
 import traceback
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait
-from datetime import datetime
+
+# from logging.handlers import QueueHandler, QueueListener
+from datetime import datetime, timezone
+from itertools import repeat
 from pathlib import Path
 
 import geopandas as gpd
 import numpy as np
 import pandas as pd
 import rasterio
-from generate_categorical_fim_flows import generate_catfim_flows
-from generate_categorical_fim_mapping import manage_catfim_mapping, post_process_cat_fim_for_viz
-from rasterio.warp import Resampling, calculate_default_transform, reproject
+from dotenv import load_dotenv
+from generate_categorical_fim_flows import generate_flows
+from generate_categorical_fim_mapping import (
+    manage_catfim_mapping,
+    post_process_cat_fim_for_viz,
+    produce_stage_based_catfim_tifs,
+)
 from tools_shared_functions import (
     filter_nwm_segments_by_stream_order,
     get_datum,
@@ -33,198 +41,293 @@
     acceptable_site_type_list,
 )
 
+import utils.fim_logger as fl
 from utils.shared_variables import VIZ_PROJECTION
 
 
+# global RLOG
+FLOG = fl.FIM_logger()  # the non mp version
+MP_LOG = fl.FIM_logger()  # the Multi Proc version
+
 gpd.options.io_engine = "pyogrio"
 
 
+# TODO: Aug 2024: This script was upgraded significantly with lots of misc TODO's embedded.
+# Lots of inline documenation needs updating as well
+
+
+"""
+Jun 17, 2024
+This system is continuing to mature over time. It has a number of optimizations that can still
+be applied in areas such as logic, performance and error handling.
+
+In the interium there is still a consider amount of debug lines and tools embedded in that can
+be commented on/off as required.
+
+
+NOTE: For now.. all logs roll up to the parent log file. ie) catfim_2024_07_09-22-20-12.log
+This creates a VERY large final log file, but the warnings and errors file should be manageable.
+Later: Let's split this to seperate log files per huc. Easy to do that for Stage Based it has
+"iterate_through_stage_based" function. Flow based? we have to think that one out a bit
+
+"""
+
+
 def process_generate_categorical_fim(
     fim_run_dir,
+    env_file,
     job_number_huc,
     job_number_inundate,
-    stage_based,
+    is_stage_based,
     output_folder,
     overwrite,
     search,
     lid_to_run,
     job_number_intervals,
     past_major_interval_cap,
+    nwm_metafile,
 ):
-    print("================================")
-    print("Start generate categorical fim")
-    overall_start_time = datetime.now()
-    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print(f"started: {dt_string}")
-    print()
 
+    # ================================
+    # Validation and setup
+
+    # Append option configuration (flow_based or stage_based) to output folder name.
+    if is_stage_based:
+        catfim_method = "stage_based"
+    else:
+        catfim_method = "flow_based"
+
+    # Define output directories
+    if output_folder.endswith("/"):
+        output_folder = output_folder[:-1]
+    output_catfim_dir = output_folder + "_" + catfim_method
+
+    output_flows_dir = os.path.join(output_catfim_dir, 'flows')
+    output_mapping_dir = os.path.join(output_catfim_dir, 'mapping')
+    attributes_dir = os.path.join(output_catfim_dir, 'attributes')
+
+    # The override is not for the parent folder as we want to keep logs around with or without override
+    if os.path.exists(output_catfim_dir) is False:
+        os.mkdir(output_catfim_dir)
+
+    # Create output directories (check against maping only as a proxy for all three)
+    if os.path.exists(output_mapping_dir) is True:
+        if overwrite is False:
+            raise Exception(
+                f"The output mapping folder of {output_catfim_dir} already exists."
+                " If you want to overwrite it, please add the -o flag. Note: When overwritten, "
+                " the three folders of mapping, flows and attributes wil be deleted and rebuilt"
+            )
+
+        gpkg_dir = os.path.join(output_mapping_dir, 'gpkg')
+        shutil.rmtree(gpkg_dir, ignore_errors=True)
+
+        shutil.rmtree(output_mapping_dir, ignore_errors=True)
+        shutil.rmtree(output_flows_dir, ignore_errors=True)
+        shutil.rmtree(attributes_dir, ignore_errors=True)
+
+        # Keeps the logs folder
+
+    if nwm_metafile != "":
+        if os.path.exists(nwm_metafile) is False:
+            raise Exception("The nwm_metadata (-me) file can not be found. Please remove or fix pathing.")
+        file_ext = os.path.splitext(nwm_metafile)
+        if file_ext.count == 0:
+            raise Exception("The nwm_metadata (-me) file appears to be invalid. It is missing an extension.")
+        if file_ext[1].lower() != ".pkl":
+            raise Exception("The nwm_metadata (-me) file appears to be invalid. The extention is not pkl.")
+
+    # Define default arguments. Modify these if necessary
+    fim_version = os.path.split(fim_run_dir)[1]
+
+    # TODO: Aug 2024: Job values are not well used. There are some times where not
+    # all three job values are not being used. This needs to be cleaned up.
     # Check job numbers and raise error if necessary
-    total_cpus_requested = job_number_huc * job_number_inundate * job_number_intervals
-    total_cpus_available = os.cpu_count() - 1
-    if total_cpus_requested > total_cpus_available:
+    # Considering how we are using each CPU very well at all, we could experiment
+    # with either overclocking or chagnign to threading. Of course, if we change
+    # to threading we ahve to be super careful about file and thread collisions (locking)
+
+    # commented out for now for some small overclocking tests (carefully of course)
+    # total_cpus_requested = job_number_huc * job_number_inundate * job_number_intervals
+    # total_cpus_available = os.cpu_count() - 2
+    # if total_cpus_requested > total_cpus_available:
+    #     raise ValueError(
+    #         f"The HUC job number (jh) [{job_number_huc}]"
+    #         f" multiplied by the inundate job number (jn) [{job_number_inundate}]"
+    #         f" multiplied by the job number intervals (ji) [{job_number_intervals}]"
+    #         " exceeds your machine\'s available CPU count minus one."
+    #         " Please lower one or more of those values accordingly."
+    #     )
+
+    # we are getting too many folders and files. We want just huc folders.
+    # output_flow_dir_list = os.listdir(fim_run_dir)
+    # looking for folders only starting with 0, 1, or 2
+    # Code variation for dropping all Alaska HUCS:
+    valid_ahps_hucs = [
+        x
+        for x in os.listdir(fim_run_dir)
+        if os.path.isdir(os.path.join(fim_run_dir, x)) and x[0] in ['0', '1', '2'] and x[:2] != "19"
+    ]
+
+    # # Code variation for KEEPING Alaska HUCS:
+    # valid_ahps_hucs = [
+    #     x
+    #     for x in os.listdir(fim_run_dir)
+    #     if os.path.isdir(os.path.join(fim_run_dir, x)) and x[0] in ['0', '1', '2']
+    # ]
+
+    valid_ahps_hucs.sort()
+
+    num_hucs = len(valid_ahps_hucs)
+    if num_hucs == 0:
         raise ValueError(
-            'The HUC job number, {}, multiplied by the inundate job number, {}, '
-            'exceeds your machine\'s available CPU count minus one. '
-            'Please lower the job_number_huc or job_number_inundate '
-            'values accordingly.'.format(job_number_huc, job_number_inundate)
+            f'Output directory {fim_run_dir} is empty. Verify that you have the correct input folder.'
         )
+    # End of Validation and setup
+    # ================================
 
-    # Define default arguments. Modify these if necessary
-    fim_version = os.path.split(fim_run_dir)[1]
+    log_dir = os.path.join(output_catfim_dir, "logs")
+    log_output_file = FLOG.calc_log_name_and_path(log_dir, "catfim")
+    FLOG.setup(log_output_file)
 
-    # Append option configuration (flow_based or stage_based) to output folder name.
-    if stage_based:
-        file_handle_appendage, catfim_method = "_stage_based", "STAGE-BASED"
-    else:
-        file_handle_appendage, catfim_method = "_flow_based", "FLOW-BASED"
+    overall_start_time = datetime.now(timezone.utc)
+    dt_string = overall_start_time.strftime("%m/%d/%Y %H:%M:%S")
 
-    # Define output directories
-    output_catfim_dir_parent = output_folder + file_handle_appendage
-    output_flows_dir = os.path.join(output_catfim_dir_parent, 'flows')
-    output_mapping_dir = os.path.join(output_catfim_dir_parent, 'mapping')
-    attributes_dir = os.path.join(output_catfim_dir_parent, 'attributes')
-
-    # Create output directories
-    if not os.path.exists(output_catfim_dir_parent):
-        os.mkdir(output_catfim_dir_parent)
-    if not os.path.exists(output_flows_dir):
-        os.mkdir(output_flows_dir)
-    if not os.path.exists(output_mapping_dir):
-        os.mkdir(output_mapping_dir)
-    if not os.path.exists(attributes_dir):
-        os.mkdir(attributes_dir)
+    # os.makedirs(output_flows_dir, exist_ok=True) # Stage doesn't use it
+    os.makedirs(output_mapping_dir, exist_ok=True)
+    os.makedirs(attributes_dir, exist_ok=True)
+
+    FLOG.lprint("================================")
+    FLOG.lprint(f"Start generate categorical fim for {catfim_method} - (UTC): {dt_string}")
+    FLOG.lprint("")
+
+    FLOG.lprint(
+        f"Processing {num_hucs} huc(s) with Alaska temporarily removed"
+    )  # Code variation for DROPPING Alaska HUCs
+    # FLOG.lprint(f"Processing {num_hucs} huc(s)") # Code variation for KEEPING Alaska HUCs
+
+    load_dotenv(env_file)
+    API_BASE_URL = os.getenv('API_BASE_URL')
+    if API_BASE_URL is None:
+        raise ValueError(
+            'API base url not found. '
+            'Ensure inundation_mapping/tools/ has an .env file with the following info: '
+            'API_BASE_URL, EVALUATED_SITES_CSV, WBD_LAYER, NWM_FLOWS_MS, '
+            'USGS_METADATA_URL, USGS_DOWNLOAD_URL'
+        )
+
+    # TODO: Add check for if lid_to_run and lst_hucs parameters conflict
+
+    # Check that fim_inputs.csv exists and raise error if necessary
+    fim_inputs_csv_path = os.path.join(fim_run_dir, 'fim_inputs.csv')
+    if not os.path.exists(fim_inputs_csv_path):
+        raise ValueError(f'{fim_inputs_csv_path} not found. Verify that you have the correct input files.')
+
+    # print()
+
+    # FLOG.lprint("Filtering out HUCs that do not have related ahps site in them.")
+    # valid_ahps_hucs = __filter_hucs_to_ahps(lst_hucs)
+
+    # num_valid_hucs = len(valid_ahps_hucs)
+    # if num_valid_hucs == 0:
+    #     raise Exception("None of the HUCs supplied have ahps sites in them. Check your fim output folder")
+    # else:
+    #     FLOG.lprint(f"Processing {num_valid_hucs} huc(s) with AHPS sites")
 
     # Define upstream and downstream search in miles
     nwm_us_search, nwm_ds_search = search, search
-    fim_dir = fim_version
-
-    # Set up logging
-    log_dir = os.path.join(output_catfim_dir_parent, 'logs')
-    log_file = os.path.join(log_dir, 'errors.log')
+    nws_lid_gpkg_file_path = ""
 
     # STAGE-BASED
-    if stage_based:
+    if is_stage_based:
         # Generate Stage-Based CatFIM mapping
-        nws_sites_layer = generate_stage_based_categorical_fim(
-            output_mapping_dir,
-            fim_version,
+        # does flows and inundation  (mapping)
+        nws_lid_gpkg_file_path = generate_stage_based_categorical_fim(
+            output_catfim_dir,
             fim_run_dir,
             nwm_us_search,
             nwm_ds_search,
-            job_number_inundate,
             lid_to_run,
-            attributes_dir,
+            env_file,
+            job_number_inundate,
+            job_number_huc,
+            valid_ahps_hucs,
             job_number_intervals,
             past_major_interval_cap,
-            job_number_huc,
+            nwm_metafile,
         )
 
-        job_number_tif = job_number_inundate * job_number_intervals
-        print("Post-processing TIFs...")
+        # creates the gkpgs (tif's created above)
+        # TODO: Aug 2024, so we need to clean it up
+        # This step does not need a job_number_inundate as it can't really use use it.
+        # It processes primarily hucs and ahps in multiproc
+        # for now, we will manuall multiple the huc * 5 (max number of ahps types)
+        ahps_jobs = job_number_huc * 5
         post_process_cat_fim_for_viz(
-            job_number_huc,
-            job_number_tif,
-            output_mapping_dir,
-            attributes_dir,
-            log_file=log_file,
-            fim_version=fim_version,
+            catfim_method, output_catfim_dir, ahps_jobs, fim_version, log_output_file
         )
 
-        # Updating mapping status
-        print('Updating mapping status...')
-        update_mapping_status(str(output_mapping_dir), str(output_flows_dir), nws_sites_layer, stage_based)
-
     # FLOW-BASED
     else:
-        fim_dir = ""
-        print('Creating flow files using the ' + catfim_method + ' technique...')
+        FLOG.lprint('Creating flow files using the ' + catfim_method + ' technique...')
         start = time.time()
-        nws_sites_layer = generate_catfim_flows(
-            output_flows_dir,
+
+        # generate flows is only using one of the incoming job number params
+        # so let's multiply -jh (huc) and -jn (inundate)
+        job_flows = job_number_huc * job_number_inundate
+        nws_lid_gpkg_file_path = generate_flows(
+            output_catfim_dir,
             nwm_us_search,
             nwm_ds_search,
-            stage_based,
-            fim_dir,
             lid_to_run,
-            attributes_dir,
-            job_number_huc,
+            env_file,
+            job_flows,
+            is_stage_based,
+            valid_ahps_hucs,
+            nwm_metafile,
+            log_output_file,
         )
         end = time.time()
         elapsed_time = (end - start) / 60
+        FLOG.lprint(f"Finished creating flow files in {str(elapsed_time).split('.')[0]} minutes")
 
-        print(f'Finished creating flow files in {elapsed_time} minutes')
-        # Generate CatFIM mapping
-        print('Begin mapping')
-        start = time.time()
+        # Generate CatFIM mapping (both flow and stage based need it, but if different orders
         manage_catfim_mapping(
             fim_run_dir,
             output_flows_dir,
-            output_mapping_dir,
-            attributes_dir,
+            output_catfim_dir,
+            catfim_method,
             job_number_huc,
             job_number_inundate,
-            overwrite,
-            depthtif=False,
+            False,
+            log_output_file,
         )
-        end = time.time()
-        elapsed_time = (end - start) / 60
-        print(f'Finished mapping in {elapsed_time} minutes')
 
-        # Updating mapping status
-        print('Updating mapping status')
-        update_mapping_status(str(output_mapping_dir), str(output_flows_dir), nws_sites_layer, stage_based)
+    # end if else
+
+    # Updating mapping status
+    FLOG.lprint('Updating mapping status...')
+    update_flow_mapping_status(output_mapping_dir, nws_lid_gpkg_file_path)
 
     # Create CSV versions of the final geopackages.
-    print('Creating CSVs. This may take several minutes.')
-    reformatted_catfim_method = catfim_method.lower().replace('-', '_')
-    create_csvs(output_mapping_dir, reformatted_catfim_method)
+    # FLOG.lprint('Creating CSVs. This may take several minutes.')
+    # create_csvs(output_mapping_dir, is_stage_based)
 
-    print("================================")
-    print("End generate categorical fim")
+    FLOG.lprint("================================")
+    FLOG.lprint("End generate categorical fim")
 
-    end_time = datetime.now()
-    dt_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-    print(f"ended: {dt_string}")
+    overall_end_time = datetime.now(timezone.utc)
+    dt_string = overall_end_time.strftime("%m/%d/%Y %H:%M:%S")
+    FLOG.lprint(f"Ended (UTC): {dt_string}")
 
     # calculate duration
-    time_duration = end_time - overall_start_time
-    print(f"Duration: {str(time_duration).split('.')[0]}")
-    print()
-
-
-def create_csvs(output_mapping_dir, reformatted_catfim_method):
-    '''
-    Produces CSV versions of desired geopackage in the output_mapping_dir.
-
-    Parameters
-    ----------
-    output_mapping_dir : STR
-        Path to the output directory of all inundation maps.
-    reformatted_catfim_method : STR
-        Text to append to CSV to communicate the type of CatFIM.
-
-    Returns
-    -------
-    None.
-
-    '''
+    time_duration = overall_end_time - overall_start_time
+    FLOG.lprint(f"Duration: {str(time_duration).split('.')[0]}")
 
-    # Convert any geopackage in the root level of output_mapping_dir to CSV and rename.
-    gpkg_list = glob.glob(os.path.join(output_mapping_dir, '*.gpkg'))
-    for gpkg in gpkg_list:
-        print(f"Creating CSV for {gpkg}")
-        gdf = gpd.read_file(gpkg)
-        parent_directory = os.path.split(gpkg)[0]
-        if 'catfim_library' in gpkg:
-            file_name = reformatted_catfim_method + '_catfim.csv'
-        if 'nws_lid_sites' in gpkg:
-            file_name = reformatted_catfim_method + '_catfim_sites.csv'
-
-        csv_output_path = os.path.join(parent_directory, file_name)
-        gdf.to_csv(csv_output_path)
+    return
 
 
-def update_mapping_status(output_mapping_dir, output_flows_dir, nws_sites_layer, stage_based):
+def update_flow_mapping_status(output_mapping_dir, nws_lid_gpkg_file_path):
     '''
     Updates the status for nws_lids from the flows subdirectory. Status
     is updated for sites where the inundation.py routine was not able to
@@ -232,12 +335,14 @@ def update_mapping_status(output_mapping_dir, output_flows_dir, nws_sites_layer,
     an error occured in inundation.py that all flow files for a given site
     experienced the error as they all would have the same nwm segments.
 
+    If there is no valid mapping files, update the nws_lids record
+
     Parameters
     ----------
     output_mapping_dir : STR
         Path to the output directory of all inundation maps.
-    output_flows_dir : STR
-        Path to the directory containing all flows.
+    nws_lid_gpkg_file_path : STR
+
 
     Returns
     -------
@@ -246,422 +351,291 @@ def update_mapping_status(output_mapping_dir, output_flows_dir, nws_sites_layer,
     '''
     # Find all LIDs with empty mapping output folders
     subdirs = [str(i) for i in Path(output_mapping_dir).rglob('**/*') if i.is_dir()]
+
+    print("")
+
     empty_nws_lids = [Path(directory).name for directory in subdirs if not list(Path(directory).iterdir())]
+    if len(empty_nws_lids) > 0:
+        FLOG.warning(f"Empty_nws_lids are.. {empty_nws_lids}")
 
     # Write list of empty nws_lids to DataFrame, these are sites that failed in inundation.py
     mapping_df = pd.DataFrame({'nws_lid': empty_nws_lids})
+
     mapping_df['did_it_map'] = 'no'
     mapping_df['map_status'] = ' and all categories failed to map'
 
     # Import geopackage output from flows creation
-    flows_df = gpd.read_file(nws_sites_layer)
+    flows_gdf = gpd.read_file(nws_lid_gpkg_file_path, engine='fiona')
+
+    if len(flows_gdf) == 0:
+        FLOG.critical(f"flows_gdf is empty. Path is {nws_lid_gpkg_file_path}. Program aborted.")
+        sys.exit(1)
 
     try:
         # Join failed sites to flows df
-        flows_df = flows_df.merge(mapping_df, how='left', on='nws_lid')
+        flows_gdf = flows_gdf.merge(mapping_df, how='left', on='nws_lid')
 
         # Switch mapped column to no for failed sites and update status
-        flows_df.loc[flows_df['did_it_map'] == 'no', 'mapped'] = 'no'
-        flows_df.loc[flows_df['did_it_map'] == 'no', 'status'] = flows_df['status'] + flows_df['map_status']
-
-        #    # Perform pass for HUCs where mapping was skipped due to missing data  #TODO check with Brian
-        #    if stage_based:
-        #        missing_mapping_hucs =
-        #    else:
-        #        flows_hucs = [i.stem for i in Path(output_flows_dir).iterdir() if i.is_dir()]
-        #        mapping_hucs = [i.stem for i in Path(output_mapping_dir).iterdir() if i.is_dir()]
-        #        missing_mapping_hucs = list(set(flows_hucs) - set(mapping_hucs))
-        #
-        #    # Update status for nws_lid in missing hucs and change mapped attribute to 'no'
-        #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'status'] =
-        #           flows_df['status'] + ' and all categories failed to map because missing HUC information'
-        #    flows_df.loc[flows_df.eval('HUC8 in @missing_mapping_hucs & mapped == "yes"'), 'mapped'] = 'no'
+        flows_gdf.loc[flows_gdf['did_it_map'] == 'no', 'mapped'] = 'no'
+        flows_gdf.loc[flows_gdf['did_it_map'] == 'no', 'status'] = (
+            flows_gdf['status'] + flows_gdf['map_status']
+        )
 
         # Clean up GeoDataFrame and rename columns for consistency
-        flows_df = flows_df.drop(columns=['did_it_map', 'map_status'])
-        flows_df = flows_df.rename(columns={'nws_lid': 'ahps_lid'})
+        flows_gdf = flows_gdf.drop(columns=['did_it_map', 'map_status'])
+        flows_gdf = flows_gdf.rename(columns={'nws_lid': 'ahps_lid'})
 
         # Write out to file
-        flows_df.to_file(nws_sites_layer)
-    except Exception as e:
-        print(f"No LIDs, \n Exception: \n {repr(e)} \n")
-
-
-def produce_inundation_map_with_stage_and_feature_ids(
-    rem_path, catchments_path, hydroid_list, hand_stage, lid_directory, category, huc, lid, branch
-):
-    # Open rem_path and catchment_path using rasterio.
-    rem_src = rasterio.open(rem_path)
-    catchments_src = rasterio.open(catchments_path)
-    rem_array = rem_src.read(1)
-    catchments_array = catchments_src.read(1)
-
-    # Use numpy.where operation to reclassify rem_path on the condition that the pixel values
-    #   are <= to hand_stage and the catchments value is in the hydroid_list.
-    reclass_rem_array = np.where((rem_array <= hand_stage) & (rem_array != rem_src.nodata), 1, 0).astype(
-        'uint8'
-    )
-    hydroid_mask = np.isin(catchments_array, hydroid_list)
-    target_catchments_array = np.where(
-        (hydroid_mask is True) & (catchments_array != catchments_src.nodata), 1, 0
-    ).astype('uint8')
-    masked_reclass_rem_array = np.where(
-        (reclass_rem_array == 1) & (target_catchments_array == 1), 1, 0
-    ).astype('uint8')
-
-    # Save resulting array to new tif with appropriate name. brdc1_record_extent_18060005.tif
-    is_all_zero = np.all((masked_reclass_rem_array == 0))
-
-    if not is_all_zero:
-        output_tif = os.path.join(
-            lid_directory, lid + '_' + category + '_extent_' + huc + '_' + branch + '.tif'
-        )
-        with rasterio.Env():
-            profile = rem_src.profile
-            profile.update(dtype=rasterio.uint8)
-            profile.update(nodata=10)
+        # TODO: Aug 29, 204: Not 100% sure why, but the gpkg errors out... likely missing a projection
+        flows_gdf.to_file(nws_lid_gpkg_file_path, index=False, driver='GPKG', engine="fiona")
 
-            with rasterio.open(output_tif, 'w', **profile) as dst:
-                dst.write(masked_reclass_rem_array, 1)
+        # csv flow file name
+        nws_lid_csv_file_path = nws_lid_gpkg_file_path.replace(".gkpg", ".csv")
 
+        # and we write a csv version at this time as well.
+        # and this csv is good
+        flows_gdf.to_csv(nws_lid_csv_file_path)
 
-def mark_complete(site_directory):
-    marker_file = Path(site_directory) / 'complete.txt'
-    marker_file.touch()
+    except Exception as e:
+        FLOG.critical(f"{output_mapping_dir} : No LIDs, \n Exception: \n {repr(e)} \n")
+        FLOG.critical(traceback.format_exc())
     return
 
 
+# This is always called as part of Multi-processing so uses MP_LOG variable and
+# creates it's own logging object.
+# This does flow files and mapping in the same function by HUC
 def iterate_through_huc_stage_based(
-    workspace,
+    output_catfim_dir,
     huc,
     fim_dir,
     huc_dictionary,
     threshold_url,
-    flood_categories,
+    magnitudes,
     all_lists,
     past_major_interval_cap,
-    number_of_jobs,
-    number_of_interval_jobs,
-    attributes_dir,
-    nwm_flows_df,
+    job_number_inundate,
+    job_number_intervals,
+    nwm_flows_region_df,
+    parent_log_output_file,
+    child_log_file_prefix,
+    progress_stmt,
 ):
-    missing_huc_files = []
-    all_messages = []
-    stage_based_att_dict = {}
-
-    print(f'Iterating through {huc}...')
-    # Make output directory for huc.
-    huc_directory = os.path.join(workspace, huc)
-    if not os.path.exists(huc_directory):
-        os.mkdir(huc_directory)
-
-    # Define paths to necessary HAND and HAND-related files.
-    usgs_elev_table = os.path.join(fim_dir, huc, 'usgs_elev_table.csv')
-    branch_dir = os.path.join(fim_dir, huc, 'branches')
-
-    # Loop through each lid in nws_lids list
-    nws_lids = huc_dictionary[huc]
-    for lid in nws_lids:
-        lid = lid.lower()  # Convert lid to lower case
-        # -- If necessary files exist, continue -- #
-        if not os.path.exists(usgs_elev_table):
-            all_messages.append(
-                [
-                    f'{lid}:usgs_elev_table missing, likely unacceptable gage datum error--'
-                    'more details to come in future release'
-                ]
+    """_summary_
+    This and its children will create stage based tifs and catfim data based on a huc
+    """
+
+    try:
+        # This is setting up logging for this function to go up to the parent
+        # child_log_file_prefix is likely MP_iter_hucs
+        MP_LOG.MP_Log_setup(parent_log_output_file, child_log_file_prefix)
+        MP_LOG.lprint("\n**********************")
+        MP_LOG.lprint(f'Processing {huc} ...')
+        MP_LOG.lprint(f'... {progress_stmt} ...')
+        MP_LOG.lprint("")
+
+        missing_huc_files = []
+        all_messages = []
+        stage_based_att_dict = {}
+
+        mapping_dir = os.path.join(output_catfim_dir, "mapping")
+        attributes_dir = os.path.join(output_catfim_dir, 'attributes')
+        # output_flows_dir = os.path.join(output_catfim_dir, "flows")
+        huc_messages_dir = os.path.join(mapping_dir, 'huc_messages')
+
+        # Make output directory for the particular huc in the mapping folder
+        mapping_huc_directory = os.path.join(mapping_dir, huc)
+        if not os.path.exists(mapping_huc_directory):
+            os.mkdir(mapping_huc_directory)
+
+        # Define paths to necessary HAND and HAND-related files.
+        usgs_elev_table = os.path.join(fim_dir, huc, 'usgs_elev_table.csv')
+        branch_dir = os.path.join(fim_dir, huc, 'branches')
+
+        # Loop through each lid in nws_lids list
+        nws_lids = huc_dictionary[huc]
+
+        MP_LOG.lprint(f"Lids to process for {huc} are {nws_lids}")
+
+        for lid in nws_lids:
+            MP_LOG.lprint("-----------------------------------")
+            huc_lid_id = f"{huc} : {lid}"
+            MP_LOG.lprint(huc_lid_id)
+
+            lid = lid.lower()  # Convert lid to lower case
+            # -- If necessary files exist, continue -- #
+            # Yes, each lid gets a record no matter what, so we need some of these messages duplicated
+            # per lid record
+            if not os.path.exists(usgs_elev_table):
+                msg = ":usgs_elev_table missing, likely unacceptable gage datum error -- more details to come in future release"
+                all_messages.append(lid + msg)
+                MP_LOG.warning(huc_lid_id + msg)
+                continue
+            if not os.path.exists(branch_dir):
+                msg = ":branch directory missing"
+                all_messages.append(lid + msg)
+                MP_LOG.warning(huc_lid_id + msg)
+                continue
+            usgs_elev_df = pd.read_csv(usgs_elev_table)
+
+            # Make mapping lid_directory.
+            mapping_lid_directory = os.path.join(mapping_huc_directory, lid)
+            if not os.path.exists(mapping_lid_directory):
+                os.mkdir(mapping_lid_directory)
+
+            # Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
+            stages, flows = get_thresholds(
+                threshold_url=threshold_url, select_by='nws_lid', selector=lid, threshold='all'
             )
-            continue
-        if not os.path.exists(branch_dir):
-            all_messages.append([f'{lid}:branch directory missing'])
-            continue
-        usgs_elev_df = pd.read_csv(usgs_elev_table)
-        # Make lid_directory.
-
-        lid_directory = os.path.join(huc_directory, lid)
-        if not os.path.exists(lid_directory):
-            os.mkdir(lid_directory)
-        else:
-            complete_marker = os.path.join(lid_directory, 'complete.txt')
-            if os.path.exists(complete_marker):
-                all_messages.append([f"{lid}: already completed in previous run."])
+
+            if stages is None:
+                msg = ':error getting thresholds from WRDS API'
+                all_messages.append(lid + msg)
+                MP_LOG.warning(huc_lid_id + msg)
                 continue
-        # Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
-        stages, flows = get_thresholds(
-            threshold_url=threshold_url, select_by='nws_lid', selector=lid, threshold='all'
-        )
 
-        if stages is None:
-            all_messages.append([f'{lid}:error getting thresholds from WRDS API'])
-            continue
-        # Check if stages are supplied, if not write message and exit.
-        if all(stages.get(category, None) is None for category in flood_categories):
-            all_messages.append([f'{lid}:missing threshold stages'])
-            continue
+            # Check if stages are supplied, if not write message and exit.
+            if all(stages.get(category, None) is None for category in magnitudes):
+                msg = ':missing threshold stages'
+                all_messages.append(lid + msg)
+                MP_LOG.warning(huc_lid_id + msg)
+                continue
 
-        try:
-            # Drop columns that offend acceptance criteria
-            usgs_elev_df['acceptable_codes'] = (
-                usgs_elev_df['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
-                & usgs_elev_df['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
-                & usgs_elev_df['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
-                & usgs_elev_df['usgs_data_site_type'].isin(acceptable_site_type_list)
+            acceptable_usgs_elev_df = __create_acceptable_usgs_elev_df(usgs_elev_df, huc_lid_id)
+            if acceptable_usgs_elev_df is None:
+                # This should only happen in a catastrophic code error.
+                # Exceptions inside the function, normally return usgs_elev_df or a variant of it
+                raise Exception("acceptable_usgs_elev_df failed to be created")
+
+            # Get the dem_adj_elevation value from usgs_elev_table.csv.
+            # Prioritize the value that is not from branch 0.
+            lid_usgs_elev, dem_eval_messages = __adj_dem_evalation_val(
+                acceptable_usgs_elev_df, lid, huc_lid_id
             )
+            all_messages += dem_eval_messages
+            if lid_usgs_elev is None:
+                continue
 
-            usgs_elev_df = usgs_elev_df.astype({'usgs_data_alt_accuracy_code': float})
-            usgs_elev_df['acceptable_alt_error'] = np.where(
-                usgs_elev_df['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False
+            # Initialize nested dict for lid attributes
+            stage_based_att_dict.update({lid: {}})
+
+            # Find lid metadata from master list of metadata dictionaries.
+            metadata = next(
+                (item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False
             )
+            lid_altitude = metadata['usgs_data']['altitude']
 
-            acceptable_usgs_elev_df = usgs_elev_df[
-                (usgs_elev_df['acceptable_codes'] is True) & (usgs_elev_df['acceptable_alt_error'] is True)
-            ]
-        except Exception as e:
-            # Not sure any of the sites actually have those USGS-related
-            # columns in this particular file, so just assume it's fine to use
+            # Filter out sites that don't have "good" data
+            try:
+                ## Removed this part to relax coordinate accuracy requirements
+                # if not metadata['usgs_data']['coord_accuracy_code'] in acceptable_coord_acc_code_list:
+                #     MP_LOG.warning(
+                #         f"\t{huc_lid_id}: {metadata['usgs_data']['coord_accuracy_code']} "
+                #         "Not in acceptable coord acc codes"
+                #     )
+                #     continue
+                # if not metadata['usgs_data']['coord_method_code'] in acceptable_coord_method_code_list:
+                #     MP_LOG.warning(f"\t{huc_lid_id}: Not in acceptable coord method codes")
+                #     continue
+                if not metadata['usgs_data']['alt_method_code'] in acceptable_alt_meth_code_list:
+                    MP_LOG.warning(f"{huc_lid_id}: Not in acceptable alt method codes")
+                    continue
+                if not metadata['usgs_data']['site_type'] in acceptable_site_type_list:
+                    MP_LOG.warning(f"{huc_lid_id}: Not in acceptable site type codes")
+                    continue
+                if not float(metadata['usgs_data']['alt_accuracy_code']) <= acceptable_alt_acc_thresh:
+                    MP_LOG.warning(f"{huc_lid_id}: Not in acceptable threshold range")
+                    continue
+            except Exception:
+                MP_LOG.error(f"{huc_lid_id}: filtering out 'bad' data in the usgs_data")
+                MP_LOG.error(traceback.format_exc())
+                continue
 
-            # print("(Various columns related to USGS probably not in this csv)")
-            print(f"Exception: \n {repr(e)} \n")
-            acceptable_usgs_elev_df = usgs_elev_df
+            datum_adj_ft, datum_messages = __adjust_datum_ft(flows, metadata, lid, huc_lid_id)
 
-        # Get the dem_adj_elevation value from usgs_elev_table.csv.
-        # Prioritize the value that is not from branch 0.
-        try:
-            matching_rows = acceptable_usgs_elev_df.loc[
-                acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'
-            ]
+            all_messages = all_messages + datum_messages
+            if datum_adj_ft is None:
+                continue
 
-            if len(matching_rows) == 2:  # It means there are two level paths, use the one that is not 0
-                lid_usgs_elev = acceptable_usgs_elev_df.loc[
-                    (acceptable_usgs_elev_df['nws_lid'] == lid.upper())
-                    & (acceptable_usgs_elev_df['levpa_id'] != 0),
-                    'dem_adj_elevation',
-                ].values[0]
-            else:
-                lid_usgs_elev = acceptable_usgs_elev_df.loc[
-                    acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'
-                ].values[0]
-        except IndexError:  # Occurs when LID is missing from table
-            all_messages.append(
-                [
-                    f'{lid}:likely unacceptable gage datum error or accuracy code(s); '
-                    'please see acceptance criteria'
-                ]
+            ### -- Concluded Datum Offset --- ###
+            # Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
+            unfiltered_segments = list(set(get_nwm_segs(metadata)))
+
+            # Filter segments to be of like stream order.
+            desired_order = metadata['nwm_feature_data']['stream_order']
+            segments = filter_nwm_segments_by_stream_order(
+                unfiltered_segments, desired_order, nwm_flows_region_df
             )
-            continue
-        # Initialize nested dict for lid attributes
-        stage_based_att_dict.update({lid: {}})
+            action_stage = stages['action']
+            minor_stage = stages['minor']
+            moderate_stage = stages['moderate']
+            major_stage = stages['major']
 
-        # Find lid metadata from master list of metadata dictionaries.
-        metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
-        lid_altitude = metadata['usgs_data']['altitude']
+            stage_list = [
+                i for i in [action_stage, minor_stage, moderate_stage, major_stage] if i is not None
+            ]
 
-        # Filter out sites that don't have "good" data
-        try:
-            if not metadata['usgs_data']['coord_accuracy_code'] in acceptable_coord_acc_code_list:
-                print(
-                    f"\t{lid}: {metadata['usgs_data']['coord_accuracy_code']} "
-                    "Not in acceptable coord acc codes"
-                )
-                continue
-            if not metadata['usgs_data']['coord_method_code'] in acceptable_coord_method_code_list:
-                print(f"\t{lid}: Not in acceptable coord method codes")
-                continue
-            if not metadata['usgs_data']['alt_method_code'] in acceptable_alt_meth_code_list:
-                print(f"\t{lid}: Not in acceptable alt method codes")
+            if stage_list == []:
+                msg = ':no stage values available'
+                all_messages.append(lid + msg)
+                MP_LOG.warning(huc_lid_id + msg)
                 continue
-            if not metadata['usgs_data']['site_type'] in acceptable_site_type_list:
-                print(f"\t{lid}: Not in acceptable site type codes")
-                continue
-            if not float(metadata['usgs_data']['alt_accuracy_code']) <= acceptable_alt_acc_thresh:
-                print(f"\t{lid}: Not in acceptable threshold range")
+
+            # TODO: Aug 2024, Is it really ok that record is missing? hummm
+            # Earlier code lower was doing comparisons to see if the interval
+            # value was been each of these 4 but sometimes one or more was None
+            missing_stages = ""
+            for stage_type in ["action", "minor", "moderate", "major"]:
+                stage_val = stages[stage_type]
+                if stage_val is None:
+                    if missing_stages != "":
+                        missing_stages += ", "
+                    missing_stages += stage_type
+
+            if missing_stages != "":
+                msg = f':Missing Stages of {missing_stages}'
+                all_messages.append(lid + msg)
+                MP_LOG.warning(huc_lid_id + msg)
                 continue
-        except Exception as e:
-            print(e)
-            continue
-
-        ### --- Do Datum Offset --- ###
-        # determine source of interpolated threshold flows, this will be the rating curve that will be used.
-        rating_curve_source = flows.get('source')
-        if rating_curve_source is None:
-            all_messages.append([f'{lid}:No source for rating curve'])
-            continue
-        # Get the datum and adjust to NAVD if necessary.
-        nws_datum_info, usgs_datum_info = get_datum(metadata)
-        if rating_curve_source == 'USGS Rating Depot':
-            datum_data = usgs_datum_info
-        elif rating_curve_source == 'NRLDB':
-            datum_data = nws_datum_info
-
-        # If datum not supplied, skip to new site
-        datum = datum_data.get('datum', None)
-        if datum is None:
-            all_messages.append([f'{lid}:datum info unavailable'])
-            continue
-        # ___________________________________________________________________________________________________#
-        # SPECIAL CASE: Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null).
-        # Modifying rating curve source will influence the rating curve and
-        #   datum retrieved for benchmark determinations.
-        if lid == 'bmbp1':
-            rating_curve_source = 'NRLDB'
-        # ___________________________________________________________________________________________________#
-
-        # SPECIAL CASE: Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29
-        #   conversion to NAVD88
-        # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7,
-        #   stkn4, wlln7
-        # Assumed to be NAD83 (no info from USGS or NWS data): dlrt2, eagi1, eppt2, jffw3, ldot2, rgdt2
-        if lid in [
-            'bgwn7',
-            'dlrt2',
-            'eagi1',
-            'eppt2',
-            'fatw3',
-            'jffw3',
-            'ldot2',
-            'mnvn4',
-            'nhpp1',
-            'pinn4',
-            'rgdt2',
-            'rgln4',
-            'rssk1',
-            'sign4',
-            'smfn7',
-            'stkn4',
-            'wlln7',
-        ]:
-            datum_data.update(crs='NAD83')
-        # ___________________________________________________________________________________________________#
-
-        # SPECIAL CASE: Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and
-        #   is actually NAD27.
-        # This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and
-        #   it matches NRLDB coordinates.
-        if lid == 'bmbp1':
-            datum_data.update(crs='NAD27')
-        # ___________________________________________________________________________________________________#
-
-        # SPECIAL CASE: Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure
-        #   datum reported in NAVD88.
-        # If NGVD29 it is converted to NAVD88.
-        # bgwn7, eagi1 vertical datum unknown, assume navd88
-        # fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
-        # wlln7 USGS data indicates vcs is NGVD29 (USGS and NWS info agree on datum value).
-        if lid in ['bgwn7', 'eagi1', 'fatw3']:
-            datum_data.update(vcs='NAVD88')
-        elif lid == 'wlln7':
-            datum_data.update(vcs='NGVD29')
-        # ___________________________________________________________________________________________________#
-
-        # Adjust datum to NAVD88 if needed
-        # Default datum_adj_ft to 0.0
-        datum_adj_ft = 0.0
-        crs = datum_data.get('crs')
-        if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929', 'NGVD,1929', 'NGVD OF 1929', 'NGVD']:
-            # Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously
-            #   removed otherwise the region needs changed.
-            try:
-                datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
-            except Exception as e:
-                e = str(e)
-                if crs is None:
-                    all_messages.append([f'{lid}:NOAA VDatum adjustment error, CRS is missing'])
-                if 'HTTPSConnectionPool' in e:
-                    time.sleep(10)  # Maybe the API needs a break, so wait 10 seconds
-                    try:
-                        datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
-                    except Exception:
-                        all_messages.append([f'{lid}:NOAA VDatum adjustment error, possible API issue'])
-                if 'Invalid projection' in e:
-                    all_messages.append(
-                        [f'{lid}:NOAA VDatum adjustment error, invalid projection: crs={crs}']
-                    )
+
+            interval_list = np.arange(
+                min(stage_list), max(stage_list) + past_major_interval_cap, 1.0
+            )  # Go an extra 10 ft beyond the max stage, arbitrary
+
+            # Check for large discrepancies between the elevation values from WRDS and HAND.
+            #   Otherwise this causes bad mapping.
+            elevation_diff = lid_usgs_elev - (lid_altitude * 0.3048)
+            if abs(elevation_diff) > 10:
+                msg = ':large discrepancy in elevation estimates from gage and HAND'
+                all_messages.append(lid + msg)
+                MP_LOG.warning(huc_lid_id + msg)
                 continue
 
-        ### -- Concluded Datum Offset --- ###
-        # Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
-        unfiltered_segments = list(set(get_nwm_segs(metadata)))
-
-        # Filter segments to be of like stream order.
-        desired_order = metadata['nwm_feature_data']['stream_order']
-        segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df)
-        action_stage = stages['action']
-        minor_stage = stages['minor']
-        moderate_stage = stages['moderate']
-        major_stage = stages['major']
-        stage_list = [i for i in [action_stage, minor_stage, moderate_stage, major_stage] if i is not None]
-        # Create a list of stages, incrementing by 1 ft.
-        if stage_list == []:
-            all_messages.append([f'{lid}:no stage values available'])
-            continue
-        interval_list = np.arange(
-            min(stage_list), max(stage_list) + past_major_interval_cap, 1.0
-        )  # Go an extra 10 ft beyond the max stage, arbitrary
-
-        # Check for large discrepancies between the elevation values from WRDS and HAND.
-        #   Otherwise this causes bad mapping.
-        elevation_diff = lid_usgs_elev - (lid_altitude * 0.3048)
-        if abs(elevation_diff) > 10:
-            all_messages.append([f'{lid}:large discrepancy in elevation estimates from gage and HAND'])
-            continue
-
-            # For each flood category
-        for category in flood_categories:
-            # Pull stage value and confirm it's valid, then process
-            stage = stages[category]
-
-            if stage is not None and datum_adj_ft is not None and lid_altitude is not None:
-                # Call function to execute mapping of the TIFs.
-                (messages, hand_stage, datum_adj_wse, datum_adj_wse_m) = produce_stage_based_catfim_tifs(
-                    stage,
-                    datum_adj_ft,
-                    branch_dir,
-                    lid_usgs_elev,
-                    lid_altitude,
-                    fim_dir,
-                    segments,
-                    lid,
-                    huc,
-                    lid_directory,
-                    category,
-                    number_of_jobs,
-                )
-                all_messages += messages
-
-                # Extra metadata for alternative CatFIM technique.
-                # TODO Revisit because branches complicate things
-                stage_based_att_dict[lid].update(
-                    {
-                        category: {
-                            'datum_adj_wse_ft': datum_adj_wse,
-                            'datum_adj_wse_m': datum_adj_wse_m,
-                            'hand_stage': hand_stage,
-                            'datum_adj_ft': datum_adj_ft,
-                            'lid_alt_ft': lid_altitude,
-                            'lid_alt_m': lid_altitude * 0.3048,
-                        }
-                    }
-                )
-            # If missing HUC file data, write message
-            if huc in missing_huc_files:
-                all_messages.append([f'{lid}:missing some HUC data'])
+            # This function sometimes is called within a MP but sometimes not.
+            # So, we might have an MP inside an MP
+            # and we will need a new prefix for it.
 
-        # Now that the "official" category maps are made, produce the incremental maps.
-        with ProcessPoolExecutor(max_workers=number_of_interval_jobs) as executor:
-            try:
-                for interval_stage in interval_list:
-                    # Determine category the stage value belongs with.
-                    if action_stage <= interval_stage < minor_stage:
-                        category = 'action_' + str(interval_stage).replace('.', 'p') + 'ft'
-                    if minor_stage <= interval_stage < moderate_stage:
-                        category = 'minor_' + str(interval_stage).replace('.', 'p') + 'ft'
-                    if moderate_stage <= interval_stage < major_stage:
-                        category = 'moderate_' + str(interval_stage).replace('.', 'p') + 'ft'
-                    if interval_stage >= major_stage:
-                        category = 'major_' + str(interval_stage).replace('.', 'p') + 'ft'
-                    executor.submit(
-                        produce_stage_based_catfim_tifs,
-                        interval_stage,
+            # Becuase we already are in an MP, lets merge up what we have at this point
+            # Before creating child MP files
+            MP_LOG.merge_log_files(parent_log_output_file, child_log_file_prefix)
+
+            # For each flood category / magnitude
+            MP_LOG.lprint(f"{huc_lid_id}: About to process flood categories")
+            child_log_file_prefix = MP_LOG.MP_calc_prefix_name(
+                parent_log_output_file, "MP_produce_catfim_tifs"
+            )
+            # print(f"child log name is {child_log_file_prefix}")
+            for category in magnitudes:
+                # MP_LOG.lprint(f"{huc_lid_id}: Magnitude is {category}")
+                # Pull stage value and confirm it's valid, then process
+                stage = stages[category]
+
+                if stage is not None and datum_adj_ft is not None and lid_altitude is not None:
+                    # Call function to execute mapping of the TIFs.
+
+                    # These are the 5 magnitudes being inundated at their stage value
+                    (messages, hand_stage, datum_adj_wse, datum_adj_wse_m) = produce_stage_based_catfim_tifs(
+                        stage,
                         datum_adj_ft,
                         branch_dir,
                         lid_usgs_elev,
@@ -670,116 +644,466 @@ def iterate_through_huc_stage_based(
                         segments,
                         lid,
                         huc,
-                        lid_directory,
+                        mapping_lid_directory,
                         category,
-                        number_of_jobs,
+                        job_number_inundate,
+                        parent_log_output_file,
+                        child_log_file_prefix,
+                    )
+                    all_messages += messages
+
+                    # Extra metadata for alternative CatFIM technique.
+                    # TODO Revisit because branches complicate things
+                    stage_based_att_dict[lid].update(
+                        {
+                            category: {
+                                'datum_adj_wse_ft': datum_adj_wse,
+                                'datum_adj_wse_m': datum_adj_wse_m,
+                                'hand_stage': hand_stage,
+                                'datum_adj_ft': datum_adj_ft,
+                                'lid_alt_ft': lid_altitude,
+                                'lid_alt_m': lid_altitude * 0.3048,
+                            }
+                        }
                     )
-            except TypeError:  # sometimes the thresholds are Nonetypes
-                pass
 
-        # Create a csv with same information as geopackage but with each threshold as new record.
-        # Probably a less verbose way.
-        csv_df = pd.DataFrame()
-        for threshold in flood_categories:
-            try:
-                line_df = pd.DataFrame(
-                    {
-                        'nws_lid': [lid],
-                        'name': metadata['nws_data']['name'],
-                        'WFO': metadata['nws_data']['wfo'],
-                        'rfc': metadata['nws_data']['rfc'],
-                        'huc': [huc],
-                        'state': metadata['nws_data']['state'],
-                        'county': metadata['nws_data']['county'],
-                        'magnitude': threshold,
-                        'q': flows[threshold],
-                        'q_uni': flows['units'],
-                        'q_src': flows['source'],
-                        'stage': stages[threshold],
-                        'stage_uni': stages['units'],
-                        's_src': stages['source'],
-                        'wrds_time': stages['wrds_timestamp'],
-                        'nrldb_time': metadata['nrldb_timestamp'],
-                        'nwis_time': metadata['nwis_timestamp'],
-                        'lat': [float(metadata['nws_preferred']['latitude'])],
-                        'lon': [float(metadata['nws_preferred']['longitude'])],
-                        'dtm_adj_ft': stage_based_att_dict[lid][threshold]['datum_adj_ft'],
-                        'dadj_w_ft': stage_based_att_dict[lid][threshold]['datum_adj_wse_ft'],
-                        'dadj_w_m': stage_based_att_dict[lid][threshold]['datum_adj_wse_m'],
-                        'lid_alt_ft': stage_based_att_dict[lid][threshold]['lid_alt_ft'],
-                        'lid_alt_m': stage_based_att_dict[lid][threshold]['lid_alt_m'],
-                    }
-                )
-                csv_df = pd.concat([csv_df, line_df])
-
-            except Exception as e:
-                print(e)
-
-        # Round flow and stage columns to 2 decimal places.
-        csv_df = csv_df.round({'q': 2, 'stage': 2})
-        # If a site folder exists (ie a flow file was written) save files containing site attributes.
-        output_dir = os.path.join(workspace, huc, lid)
-        if os.path.exists(output_dir):
-            # Export DataFrame to csv containing attributes
-            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index=False)
+                # If missing HUC file data, write message
+                if huc in missing_huc_files:
+                    msg = ':missing some HUC data'
+                    all_messages.append(lid + msg)
+                    MP_LOG.error(huc_lid_id + msg)
+
+            # So, we might have an MP inside an MP
+            # let's merge what we have at this point, before we go into another MP
+            MP_LOG.merge_log_files(parent_log_output_file, child_log_file_prefix, True)
+
+            # Now we will do another set of inundations, but this one is based on
+            # not the stage flow but flow based on each interval
+            tif_child_log_file_prefix = MP_LOG.MP_calc_prefix_name(parent_log_output_file, "MP_prod_sb_tifs")
+            with ProcessPoolExecutor(max_workers=job_number_intervals) as executor:
+                try:
+                    # MP_LOG.lprint(f"{huc_lid_id} : action_stage is {action_stage}")
+                    # MP_LOG.lprint(f"{huc_lid_id} : minor_stage is {minor_stage}")
+                    # MP_LOG.lprint(f"{huc_lid_id} : moderate_stage is {moderate_stage}")
+                    # MP_LOG.lprint(f"{huc_lid_id} : major_stage is {major_stage}")
+
+                    for interval_stage in interval_list:
+                        # MP_LOG.lprint(f"{huc_lid_id} : interval_stage is {interval_stage}")
+
+                        # Determine category the stage value belongs with.
+                        if action_stage <= interval_stage < minor_stage:
+                            category = 'action_' + str(interval_stage).replace('.', 'p') + 'ft'
+                        if minor_stage <= interval_stage < moderate_stage:
+                            category = 'minor_' + str(interval_stage).replace('.', 'p') + 'ft'
+                        if moderate_stage <= interval_stage < major_stage:
+                            category = 'moderate_' + str(interval_stage).replace('.', 'p') + 'ft'
+                        if interval_stage >= major_stage:
+                            category = 'major_' + str(interval_stage).replace('.', 'p') + 'ft'
+                        executor.submit(
+                            produce_stage_based_catfim_tifs,
+                            interval_stage,
+                            datum_adj_ft,
+                            branch_dir,
+                            lid_usgs_elev,
+                            lid_altitude,
+                            fim_dir,
+                            segments,
+                            lid,
+                            huc,
+                            mapping_lid_directory,
+                            category,
+                            job_number_inundate,
+                            parent_log_output_file,
+                            tif_child_log_file_prefix,
+                        )
+                except TypeError:  # sometimes the thresholds are Nonetypes
+                    MP_LOG.error("ERROR: type error in ProcessPool, likely in the interval tests")
+                    MP_LOG.error(traceback.format_exc())
+                    continue
+
+                except Exception:
+                    MP_LOG.critical("ERROR: ProcessPool has an error")
+                    MP_LOG.critical(traceback.format_exc())
+                    # merge MP Logs (Yes)
+                    MP_LOG.merge_log_files(parent_log_output_file, tif_child_log_file_prefix, True)
+                    sys.exit(1)
+
+            # merge MP Logs (merging MP into an MP (proc_pool in a proc_pool))
+            MP_LOG.merge_log_files(parent_log_output_file, tif_child_log_file_prefix, True)
+
+            # Create a csv with same information as geopackage but with each threshold as new record.
+            # Probably a less verbose way.
+            csv_df = pd.DataFrame()
+            for threshold in magnitudes:
+                try:
+                    line_df = pd.DataFrame(
+                        {
+                            'nws_lid': [lid],
+                            'name': metadata['nws_data']['name'],
+                            'WFO': metadata['nws_data']['wfo'],
+                            'rfc': metadata['nws_data']['rfc'],
+                            'huc': [huc],
+                            'state': metadata['nws_data']['state'],
+                            'county': metadata['nws_data']['county'],
+                            'magnitude': threshold,
+                            'q': flows[threshold],
+                            'q_uni': flows['units'],
+                            'q_src': flows['source'],
+                            'stage': stages[threshold],
+                            'stage_uni': stages['units'],
+                            's_src': stages['source'],
+                            'wrds_time': stages['wrds_timestamp'],
+                            'nrldb_time': metadata['nrldb_timestamp'],
+                            'nwis_time': metadata['nwis_timestamp'],
+                            'lat': [float(metadata['nws_preferred']['latitude'])],
+                            'lon': [float(metadata['nws_preferred']['longitude'])],
+                            'dtm_adj_ft': stage_based_att_dict[lid][threshold]['datum_adj_ft'],
+                            'dadj_w_ft': stage_based_att_dict[lid][threshold]['datum_adj_wse_ft'],
+                            'dadj_w_m': stage_based_att_dict[lid][threshold]['datum_adj_wse_m'],
+                            'lid_alt_ft': stage_based_att_dict[lid][threshold]['lid_alt_ft'],
+                            'lid_alt_m': stage_based_att_dict[lid][threshold]['lid_alt_m'],
+                        }
+                    )
+                    csv_df = pd.concat([csv_df, line_df])
+
+                except Exception:
+                    MP_LOG.error("ERROR: threshold has an error")
+                    MP_LOG.error(traceback.format_exc())
+                    return
+                    # sys.exit(1)
+
+            # Round flow and stage columns to 2 decimal places.
+            csv_df = csv_df.round({'q': 2, 'stage': 2})
+
+            # If a site folder exists (ie a flow file was written) save files containing site attributes.
+            if os.path.exists(mapping_lid_directory):
+                # Export DataFrame to csv containing attributes
+                attributes_filepath = os.path.join(attributes_dir, f'{lid}_attributes.csv')
+
+                csv_df.to_csv(attributes_filepath, index=False)
+            else:
+                msg = ':missing all calculated flows'
+                all_messages.append(lid + msg)
+                MP_LOG.error(huc_lid_id + msg)
+
+            # If it made it to this point (i.e. no continues), there were no major preventers of mapping
+            all_messages.append(lid + ':OK')
+            MP_LOG.success(f'{huc_lid_id}: Complete')
+            # mark_complete(mapping_lid_directory)
+
+        # Write all_messages by HUC to be scraped later.
+        if len(all_messages) > 0:
+
+            # TODO: Aug 2024: This is now identical to the way flow handles messages
+            # but the system should probably be changed to somethign more elegant but good enough
+            # for now. At least is is MP safe.
+            huc_messages_txt_file = os.path.join(huc_messages_dir, str(huc) + '_messages.txt')
+            with open(huc_messages_txt_file, 'w') as f:
+                for item in all_messages:
+                    item = item.strip()
+                    # f.write("%s\n" % item)
+                    f.write(f"{item}\n")
+        else:
+            # something has likely gone very poorly.
+            MP_LOG.error(f"No messages found for {huc}")
+            sys.exit(1)
+
+    except Exception:
+        MP_LOG.error(f"{huc} : {lid} Error iterating through huc stage based")
+        MP_LOG.error(traceback.format_exc())
+
+    return
+
+
+def __adjust_datum_ft(flows, metadata, lid, huc_lid_id):
+
+    # TODO: Aug 2024: This whole parts needs revisiting. Lots of lid data has changed and this
+    # is all likely very old.
+
+    # Jul 2024: For now, we will duplicate messages via all_messsages and via the logging system.
+    all_messages = []
+
+    datum_adj_ft = None
+    ### --- Do Datum Offset --- ###
+    # determine source of interpolated threshold flows, this will be the rating curve that will be used.
+    rating_curve_source = flows.get('source')
+    if rating_curve_source is None:
+        msg = f'{huc_lid_id}:No source for rating curve'
+        all_messages.append(msg)
+        MP_LOG.warning(msg)
+        return None, all_messages
+
+    # Get the datum and adjust to NAVD if necessary.
+    nws_datum_info, usgs_datum_info = get_datum(metadata)
+    if rating_curve_source == 'USGS Rating Depot':
+        datum_data = usgs_datum_info
+    elif rating_curve_source == 'NRLDB':
+        datum_data = nws_datum_info
+
+    # If datum not supplied, skip to new site
+    datum = datum_data.get('datum', None)
+    if datum is None:
+        msg = f'{huc_lid_id}:datum info unavailable'
+        all_messages.append(msg)
+        MP_LOG.warning(msg)
+        return None, all_messages
+
+    # ___________________________________________________________________________________________________#
+    # SPECIAL CASE: Workaround for "bmbp1" where the only valid datum is from NRLDB (USGS datum is null).
+    # Modifying rating curve source will influence the rating curve and
+    #   datum retrieved for benchmark determinations.
+    if lid == 'bmbp1':
+        rating_curve_source = 'NRLDB'
+    # ___________________________________________________________________________________________________#
+
+    # SPECIAL CASE: Custom workaround these sites have faulty crs from WRDS. CRS needed for NGVD29
+    #   conversion to NAVD88
+    # USGS info indicates NAD83 for site: bgwn7, fatw3, mnvn4, nhpp1, pinn4, rgln4, rssk1, sign4, smfn7,
+    #   stkn4, wlln7
+    # Assumed to be NAD83 (no info from USGS or NWS data): dlrt2, eagi1, eppt2, jffw3, ldot2, rgdt2
+    if lid in [
+        'bgwn7',
+        'dlrt2',
+        'eagi1',
+        'eppt2',
+        'fatw3',
+        'jffw3',
+        'ldot2',
+        'mnvn4',
+        'nhpp1',
+        'pinn4',
+        'rgdt2',
+        'rgln4',
+        'rssk1',
+        'sign4',
+        'smfn7',
+        'stkn4',
+        'wlln7',
+    ]:
+        datum_data.update(crs='NAD83')
+    # ___________________________________________________________________________________________________#
+
+    # SPECIAL CASE: Workaround for bmbp1; CRS supplied by NRLDB is mis-assigned (NAD29) and
+    #   is actually NAD27.
+    # This was verified by converting USGS coordinates (in NAD83) for bmbp1 to NAD27 and
+    #   it matches NRLDB coordinates.
+    if lid == 'bmbp1':
+        datum_data.update(crs='NAD27')
+    # ___________________________________________________________________________________________________#
+
+    # SPECIAL CASE: Custom workaround these sites have poorly defined vcs from WRDS. VCS needed to ensure
+    #   datum reported in NAVD88.
+    # If NGVD29 it is converted to NAVD88.
+    # bgwn7, eagi1 vertical datum unknown, assume navd88
+    # fatw3 USGS data indicates vcs is NAVD88 (USGS and NWS info agree on datum value).
+    # wlln7 USGS data indicates vcs is NGVD29 (USGS and NWS info agree on datum value).
+    if lid in ['bgwn7', 'eagi1', 'fatw3']:
+        datum_data.update(vcs='NAVD88')
+    elif lid == 'wlln7':
+        datum_data.update(vcs='NGVD29')
+    # ___________________________________________________________________________________________________#
+
+    # Adjust datum to NAVD88 if needed
+    # Default datum_adj_ft to 0.0
+    datum_adj_ft = 0.0
+    crs = datum_data.get('crs')
+    if datum_data.get('vcs') in ['NGVD29', 'NGVD 1929', 'NGVD,1929', 'NGVD OF 1929', 'NGVD']:
+        # Get the datum adjustment to convert NGVD to NAVD. Sites not in contiguous US are previously
+        #   removed otherwise the region needs changed.
+        try:
+            datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
+        except Exception as ex:
+            MP_LOG.error(f"ERROR: {huc_lid_id}: ngvd_to_navd_ft")
+            MP_LOG.error(traceback.format_exc())
+            ex = str(ex)
+            if crs is None:
+                msg = f'{huc_lid_id}:NOAA VDatum adjustment error, CRS is missing'
+                all_messages.append(msg)
+                MP_LOG.error(msg)
+            if 'HTTPSConnectionPool' in ex:
+                time.sleep(10)  # Maybe the API needs a break, so wait 10 seconds
+                try:
+                    datum_adj_ft = ngvd_to_navd_ft(datum_info=datum_data, region='contiguous')
+                except Exception:
+                    msg = f'{huc_lid_id}:NOAA VDatum adjustment error, possible API issue'
+                    all_messages.append(msg)
+                    MP_LOG.error(msg)
+            if 'Invalid projection' in ex:
+                msg = f'{huc_lid_id}:NOAA VDatum adjustment error, invalid projection: crs={crs}'
+                all_messages.append(msg)
+                MP_LOG.error(msg)
+            return None, all_messages
+
+    return datum_adj_ft, all_messages
+
+
+def __create_acceptable_usgs_elev_df(usgs_elev_df, huc_lid_id):
+    acceptable_usgs_elev_df = None
+    try:
+        # Drop columns that offend acceptance criteria
+        usgs_elev_df['acceptable_codes'] = (
+            # usgs_elev_df['usgs_data_coord_accuracy_code'].isin(acceptable_coord_acc_code_list)
+            # & usgs_elev_df['usgs_data_coord_method_code'].isin(acceptable_coord_method_code_list)
+            usgs_elev_df['usgs_data_alt_method_code'].isin(acceptable_alt_meth_code_list)
+            & usgs_elev_df['usgs_data_site_type'].isin(acceptable_site_type_list)
+        )
+
+        usgs_elev_df = usgs_elev_df.astype({'usgs_data_alt_accuracy_code': float})
+        usgs_elev_df['acceptable_alt_error'] = np.where(
+            usgs_elev_df['usgs_data_alt_accuracy_code'] <= acceptable_alt_acc_thresh, True, False
+        )
+
+        acceptable_usgs_elev_df = usgs_elev_df[
+            (usgs_elev_df['acceptable_codes'] == True) & (usgs_elev_df['acceptable_alt_error'] == True)
+        ]
+
+        # # TEMP DEBUG Record row difference and write it to a CSV or something
+        # label = 'Old code' ## TEMP DEBUG
+        # num_potential_rows = usgs_elev_df.shape[0]
+        # num_acceptable_rows = acceptable_usgs_elev_df.shape[0]
+        # out_message = f'{label}: kept {num_acceptable_rows} rows out of {num_potential_rows} available rows.'
+
+    except Exception:
+        # Not sure any of the sites actually have those USGS-related
+        # columns in this particular file, so just assume it's fine to use
+
+        # print("(Various columns related to USGS probably not in this csv)")
+        # print(f"Exception: \n {repr(e)} \n")
+        MP_LOG.error(f"{huc_lid_id}: An error has occurred while working with the usgs_elev table")
+        MP_LOG.error(traceback.format_exc())
+        acceptable_usgs_elev_df = usgs_elev_df
+
+    return acceptable_usgs_elev_df
+
+
+def __adj_dem_evalation_val(acceptable_usgs_elev_df, lid, huc_lid_id):
+
+    lid_usgs_elev = None
+    all_messages = []
+    try:
+        matching_rows = acceptable_usgs_elev_df.loc[
+            acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'
+        ]
+
+        if len(matching_rows) == 2:  # It means there are two level paths, use the one that is not 0
+            lid_usgs_elev = acceptable_usgs_elev_df.loc[
+                (acceptable_usgs_elev_df['nws_lid'] == lid.upper())
+                & (acceptable_usgs_elev_df['levpa_id'] != 0),
+                'dem_adj_elevation',
+            ].values[0]
         else:
-            all_messages.append([f'{lid}:missing all calculated flows'])
+            lid_usgs_elev = acceptable_usgs_elev_df.loc[
+                acceptable_usgs_elev_df['nws_lid'] == lid.upper(), 'dem_adj_elevation'
+            ].values[0]
 
-        # If it made it to this point (i.e. no continues), there were no major preventers of mapping
-        all_messages.append([f'{lid}:OK'])
-        mark_complete(output_dir)
-    # Write all_messages by HUC to be scraped later.
-    messages_dir = os.path.join(workspace, 'messages')
-    if not os.path.exists(messages_dir):
-        os.mkdir(messages_dir)
-    huc_messages_csv = os.path.join(messages_dir, huc + '_messages.csv')
-    with open(huc_messages_csv, 'w') as output_csv:
-        writer = csv.writer(output_csv)
-        writer.writerows(all_messages)
+    except IndexError:  # Occurs when LID is missing from table (yes. warning)
+        MP_LOG.warning(f"{huc_lid_id}: adjusting dem_adj_elevation")
+        MP_LOG.warning(traceback.format_exc())
+        msg = ':likely unacceptable gage datum error or accuracy code(s); please see acceptance criteria'
+        all_messages.append(lid + msg)
+        MP_LOG.warning(huc_lid_id + msg)
 
+    return lid_usgs_elev, all_messages
 
+
+# This creates a HUC iterator with each HUC creating its flow files and tifs
 def generate_stage_based_categorical_fim(
-    workspace,
-    fim_version,
-    fim_dir,
+    output_catfim_dir,
+    fim_run_dir,
     nwm_us_search,
     nwm_ds_search,
-    number_of_jobs,
     lid_to_run,
-    attributes_dir,
-    number_of_interval_jobs,
-    past_major_interval_cap,
+    env_file,
+    job_number_inundate,
     job_number_huc,
+    lst_hucs,
+    job_number_intervals,
+    past_major_interval_cap,
+    nwm_metafile,
 ):
-    flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
-
-    (huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists, nwm_flows_df) = generate_catfim_flows(
-        workspace, nwm_us_search, nwm_ds_search, stage_based=True, fim_dir=fim_dir, lid_to_run=lid_to_run
+    magnitudes = ['action', 'minor', 'moderate', 'major', 'record']
+
+    output_mapping_dir = os.path.join(output_catfim_dir, 'mapping')
+    attributes_dir = os.path.join(output_catfim_dir, 'attributes')
+
+    # Create HUC message directory to store messages that will be read and joined after multiprocessing
+    huc_messages_dir = os.path.join(output_mapping_dir, 'huc_messages')
+    os.makedirs(huc_messages_dir, exist_ok=True)
+
+    FLOG.lprint("Starting generate_flows (Stage Based)")
+
+    # If it is stage based, generate flows returns all of these objects.
+    # If flow based, generate flows returns only
+    # (huc_dictionary, out_gdf, ___, threshold_url, all_lists, nwm_flows_df, nwm_flows_alaska_df) = generate_flows( # With Alaska
+
+    # Generate flows is only using one of the incoming job number params
+    # so let's multiply -jh (huc) and -jn (inundate)
+    job_flows = job_number_huc * job_number_inundate
+    (huc_dictionary, out_gdf, ___, threshold_url, all_lists, all_nwm_flows_df) = generate_flows(  # No Alaska
+        output_catfim_dir,
+        nwm_us_search,
+        nwm_ds_search,
+        lid_to_run,
+        env_file,
+        job_flows,
+        True,
+        lst_hucs,
+        nwm_metafile,
+        str(FLOG.LOG_FILE_PATH),
     )
+    FLOG.lprint("End generate_flows (Stage Based)")
+
+    child_log_file_prefix = FLOG.MP_calc_prefix_name(FLOG.LOG_FILE_PATH, "MP_iter_hucs")
+
+    FLOG.lprint(">>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+    FLOG.lprint("Start processing HUCs for Stage-Based CatFIM")
+    num_hucs = len(lst_hucs)
+    huc_index = 0
+    FLOG.lprint(f"Number of hucs to process is {num_hucs}")
 
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
         for huc in huc_dictionary:
-            executor.submit(
-                iterate_through_huc_stage_based,
-                workspace,
-                huc,
-                fim_dir,
-                huc_dictionary,
-                threshold_url,
-                flood_categories,
-                all_lists,
-                past_major_interval_cap,
-                number_of_jobs,
-                number_of_interval_jobs,
-                attributes_dir,
-                nwm_flows_df,
-            )
+            if huc in lst_hucs:
+                # FLOG.lprint(f'Generating stage based catfim for : {huc}')
+
+                # Code variation for DROPPING Alaska HUCs
+                nwm_flows_region_df = all_nwm_flows_df
+
+                # # Code variation for keeping alaska HUCs
+                # nwm_flows_region_df = nwm_flows_alaska_df if str(huc[:2]) == '19' else nwm_flows_df
+
+                progress_stmt = f"index {huc_index + 1} of {num_hucs}"
+                executor.submit(
+                    iterate_through_huc_stage_based,
+                    output_catfim_dir,
+                    huc,
+                    fim_run_dir,
+                    huc_dictionary,
+                    threshold_url,
+                    magnitudes,
+                    all_lists,
+                    past_major_interval_cap,
+                    job_number_inundate,
+                    job_number_intervals,
+                    nwm_flows_region_df,
+                    str(FLOG.LOG_FILE_PATH),
+                    child_log_file_prefix,
+                    progress_stmt,
+                )
+                huc_index += 1
+    # Need to merge MP logs here, merged into the "master log file"
+
+    FLOG.merge_log_files(FLOG.LOG_FILE_PATH, child_log_file_prefix, True)
+
+    FLOG.lprint('Wrapping up processing HUCs for Stage-Based CatFIM...')
+    FLOG.lprint(">>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+
+    csv_files = [x for x in os.listdir(attributes_dir) if x.endswith('.csv')]
 
-    print('Wrapping up Stage-Based CatFIM...')
-    csv_files = os.listdir(attributes_dir)
     all_csv_df = pd.DataFrame()
     refined_csv_files_list = []
     for csv_file in csv_files:
+
         full_csv_path = os.path.join(attributes_dir, csv_file)
         # HUC has to be read in as string to preserve leading zeros.
         try:
@@ -787,15 +1111,21 @@ def generate_stage_based_categorical_fim(
             all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index=True)
             refined_csv_files_list.append(csv_file)
         except Exception:  # Happens if a file is empty (i.e. no mapping)
+            FLOG.error(f"ERROR: loading csv {full_csv_path}")
+            FLOG.error(traceback.format_exc())
             pass
+
     # Write to file
-    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index=False)
+    all_csv_df.to_csv(os.path.join(attributes_dir, 'nws_lid_attributes.csv'), index=False)
 
     # This section populates a geopackage of all potential sites and details
     # whether it was mapped or not (mapped field) and if not, why (status field).
 
     # Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
-    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)
+
+    # TODO: Accomodate AK projection?   Yes.. and Alaska and CONUS should all end up as the same projection output
+    # epsg:5070, we really want 3857 out for all outputs
+    viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)  # TODO: Accomodate AK projection?
     viz_out_gdf.rename(
         columns={
             'identifiers_nwm_feature_id': 'nwm_seg',
@@ -819,39 +1149,48 @@ def generate_stage_based_categorical_fim(
     viz_out_gdf = viz_out_gdf.merge(lids_df, how='left', on='nws_lid')
     viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
 
-    # Create list from all messages in messages dir.
-    messages_dir = os.path.join(workspace, 'messages')
-    all_messages = []
-    all_message_csvs = os.listdir(messages_dir)
-    for message_csv in all_message_csvs:
-        full_message_csv_path = os.path.join(messages_dir, message_csv)
-        with open(full_message_csv_path, newline='') as message_file:
-            reader = csv.reader(message_file)
-            for row in reader:
-                all_messages.append(row)
+    # Read all messages for all HUCs
+    # This is basically identical to a chunk in flow based. At a min, consolidate
+    # or better yet, find a more elegant, yet still MP safe, system than .txt files
+    # but it works.. so maybe someday.
+    huc_message_list = []
+    huc_messages_dir_list = os.listdir(huc_messages_dir)
+    for huc_message_file in huc_messages_dir_list:
+        full_path_file = os.path.join(huc_messages_dir, huc_message_file)
+        with open(full_path_file, 'r') as f:
+            if full_path_file.endswith('.txt'):
+                lines = f.readlines()
+                for line in lines:
+                    line = line.strip()
+                    huc_message_list.append(line)
 
     # Filter out columns and write out to file
-    nws_sites_layer = os.path.join(workspace, 'nws_lid_sites.gpkg')
+    # flow based doesn't make it here only stage
+    nws_lid_gpkg_file_path = os.path.join(output_mapping_dir, 'stage_based_catfim_sites.gpkg')
 
-    # Only write to sites geopackage if it didn't exist yet
-    # (and this line shouldn't have been reached if we had an interrupted
-    # run previously and are picking back up with a restart)
-    if not os.path.exists(nws_sites_layer):
-        # Write messages to DataFrame, split into columns, aggregate messages.
-        messages_df = pd.DataFrame(all_messages, columns=['message'])
+    # Write messages to DataFrame, split into columns, aggregate messages.
+    if len(huc_message_list) > 0:
+
+        FLOG.lprint(f"nws_sites_layer ({nws_lid_gpkg_file_path}) : adding messages")
+        messages_df = pd.DataFrame(huc_message_list, columns=['message'])
 
         messages_df = (
             messages_df['message']
             .str.split(':', n=1, expand=True)
             .rename(columns={0: 'nws_lid', 1: 'status'})
         )
-        status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
+
+        # We want one viz_out_gdf record per ahps and if there are more than one, contact the messages
+        # status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
+        status_df = messages_df.groupby(['nws_lid'])['status'].agg(lambda x: ',\n'.join(x)).reset_index()
 
         # Join messages to populate status field to candidate sites. Assign
         # status for null fields.
         viz_out_gdf = viz_out_gdf.merge(status_df, how='left', on='nws_lid')
 
-        #    viz_out_gdf['status'] = viz_out_gdf['status'].fillna('OK')
+        # viz_out_gdf.reset_index(inplace=True)
+
+        viz_out_gdf['status'] = viz_out_gdf['status'].fillna('undetermined')
 
         # Add acceptance criteria to viz_out_gdf before writing
         viz_out_gdf['acceptable_coord_acc_code_list'] = str(acceptable_coord_acc_code_list)
@@ -860,148 +1199,14 @@ def generate_stage_based_categorical_fim(
         viz_out_gdf['acceptable_alt_meth_code_list'] = str(acceptable_alt_meth_code_list)
         viz_out_gdf['acceptable_site_type_list'] = str(acceptable_site_type_list)
 
-        viz_out_gdf.to_file(nws_sites_layer, driver='GPKG')
-
-    return nws_sites_layer
-
-
-def produce_stage_based_catfim_tifs(
-    stage,
-    datum_adj_ft,
-    branch_dir,
-    lid_usgs_elev,
-    lid_altitude,
-    fim_dir,
-    segments,
-    lid,
-    huc,
-    lid_directory,
-    category,
-    number_of_jobs,
-):
-    messages = []
-
-    # Determine datum-offset water surface elevation (from above).
-    datum_adj_wse = stage + datum_adj_ft + lid_altitude
-    datum_adj_wse_m = datum_adj_wse * 0.3048  # Convert ft to m
-
-    # Subtract HAND gage elevation from HAND WSE to get HAND stage.
-    hand_stage = datum_adj_wse_m - lid_usgs_elev
-
-    # Produce extent tif hand_stage. Multiprocess across branches.
-    branches = os.listdir(branch_dir)
-    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
-        for branch in branches:
-            # Define paths to necessary files to produce inundation grids.
-            full_branch_path = os.path.join(branch_dir, branch)
-            rem_path = os.path.join(fim_dir, huc, full_branch_path, 'rem_zeroed_masked_' + branch + '.tif')
-            catchments_path = os.path.join(
-                fim_dir,
-                huc,
-                full_branch_path,
-                'gw_catchments_reaches_filtered_addedAttributes_' + branch + '.tif',
-            )
-            hydrotable_path = os.path.join(fim_dir, huc, full_branch_path, 'hydroTable_' + branch + '.csv')
-
-            if not os.path.exists(rem_path):
-                messages.append([f"{lid}:rem doesn't exist"])
-                continue
-            if not os.path.exists(catchments_path):
-                messages.append([f"{lid}:catchments files don't exist"])
-                continue
-            if not os.path.exists(hydrotable_path):
-                messages.append([f"{lid}:hydrotable doesn't exist"])
-                continue
-
-            # Use hydroTable to determine hydroid_list from site_ms_segments.
-            hydrotable_df = pd.read_csv(hydrotable_path)
-            hydroid_list = []
-
-            # Determine hydroids at which to perform inundation
-            for feature_id in segments:
-                try:
-                    subset_hydrotable_df = hydrotable_df[hydrotable_df['feature_id'] == int(feature_id)]
-                    hydroid_list += list(subset_hydrotable_df.HydroID.unique())
-                except IndexError:
-                    pass
-
-            # Some branches don't have matching hydroids
-            # if len(hydroid_list) == 0:
-            #     messages.append(f"{lid}:no matching hydroids")
-            #     continue
-
-            # If no segments, write message and exit out
-            if not segments:
-                messages.append([f'{lid}:missing nwm segments'])
-                continue
-
-            # Create inundation maps with branch and stage data
-            try:
-                print("Generating stage-based FIM for " + huc + " and branch " + branch)
-                executor.submit(
-                    produce_inundation_map_with_stage_and_feature_ids,
-                    rem_path,
-                    catchments_path,
-                    hydroid_list,
-                    hand_stage,
-                    lid_directory,
-                    category,
-                    huc,
-                    lid,
-                    branch,
-                )
-            except Exception:
-                messages.append([f'{lid}:inundation failed at {category}'])
-
-    # -- MOSAIC -- #
-    # Merge all rasters in lid_directory that have the same magnitude/category.
-    path_list = []
-    lid_dir_list = os.listdir(lid_directory)
-    print("Merging " + category)
-    for f in lid_dir_list:
-        if category in f:
-            path_list.append(os.path.join(lid_directory, f))
-    path_list.sort()  # To force branch 0 first in list, sort
-
-    if len(path_list) > 0:
-        zero_branch_grid = path_list[0]
-        zero_branch_src = rasterio.open(zero_branch_grid)
-        zero_branch_array = zero_branch_src.read(1)
-        summed_array = zero_branch_array  # Initialize it as the branch zero array
-
-        # Loop through remaining items in list and sum them with summed_array
-        for remaining_raster in path_list[1:]:
-            remaining_raster_src = rasterio.open(remaining_raster)
-            remaining_raster_array_original = remaining_raster_src.read(1)
-
-            # Reproject non-branch-zero grids so I can sum them with the branch zero grid
-            remaining_raster_array = np.empty(zero_branch_array.shape, dtype=np.int8)
-            reproject(
-                remaining_raster_array_original,
-                destination=remaining_raster_array,
-                src_transform=remaining_raster_src.transform,
-                src_crs=remaining_raster_src.crs,
-                src_nodata=remaining_raster_src.nodata,
-                dst_transform=zero_branch_src.transform,
-                dst_crs=zero_branch_src.crs,
-                dst_nodata=-1,
-                dst_resolution=zero_branch_src.res,
-                resampling=Resampling.nearest,
-            )
-            # Sum rasters
-            summed_array = summed_array + remaining_raster_array
-
-        del zero_branch_array  # Clean up
+        viz_out_gdf.to_file(nws_lid_gpkg_file_path, driver='GPKG', index=True, engine='fiona')
 
-        # Define path to merged file, in same format as expected by post_process_cat_fim_for_viz function
-        output_tif = os.path.join(lid_directory, lid + '_' + category + '_extent.tif')
-        profile = zero_branch_src.profile
-        summed_array = summed_array.astype('uint8')
-        with rasterio.open(output_tif, 'w', **profile) as dst:
-            dst.write(summed_array, 1)
-        del summed_array
+        csv_file_path = nws_lid_gpkg_file_path.replace(".gpkg", ".csv")
+        viz_out_gdf.to_csv(csv_file_path)
+    else:
+        FLOG.lprint(f"nws_sites_layer ({nws_lid_gpkg_file_path}) : has no messages")
 
-    return messages, hand_stage, datum_adj_wse, datum_adj_wse_m
+    return nws_lid_gpkg_file_path
 
 
 if __name__ == '__main__':
@@ -1013,13 +1218,19 @@ def produce_stage_based_catfim_tifs(
         help='Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_0_9_2',
         required=True,
     )
+    parser.add_argument(
+        '-e',
+        '--env_file',
+        help='Docker mount path to the catfim environment file. ie) data/config/catfim.env',
+        required=True,
+    )
     parser.add_argument(
         '-jh',
         '--job_number_huc',
-        help='Number of processes to use for HUC scale operations.'
+        help='OPTIONAL: Number of processes to use for HUC scale operations.'
         ' HUC and inundation job numbers should multiply to no more than one less than the CPU count of the'
         ' machine. CatFIM sites generally only have 2-3 branches overlapping a site, so this number can be '
-        'kept low (2-4)',
+        'kept low (2-4). Defaults to 1.',
         required=False,
         default=1,
         type=int,
@@ -1027,17 +1238,30 @@ def produce_stage_based_catfim_tifs(
     parser.add_argument(
         '-jn',
         '--job_number_inundate',
-        help='Number of processes to use for inundating'
+        help='OPTIONAL: Number of processes to use for inundating'
         ' HUC and inundation job numbers should multiply to no more than one less than the CPU count'
-        ' of the machine.',
+        ' of the machine. Defaults to 1.',
         required=False,
         default=1,
         type=int,
     )
+
+    parser.add_argument(
+        '-ji',
+        '--job_number_intervals',
+        help='OPTIONAL: Number of processes to use for inundating multiple intervals in stage-based'
+        ' inundation and interval job numbers should multiply to no more than one less than the CPU count '
+        'of the machine. Defaults to 1.',
+        required=False,
+        default=1,
+        type=int,
+    )
+
     parser.add_argument(
         '-a',
-        '--stage_based',
-        help='Run stage-based CatFIM instead of flow-based?' ' NOTE: flow-based CatFIM is the default.',
+        '--is_stage_based',
+        help='Run stage-based CatFIM instead of flow-based? Add this -a param to make it stage based,'
+        ' leave it off for flow based',
         required=False,
         default=False,
         action='store_true',
@@ -1045,44 +1269,64 @@ def produce_stage_based_catfim_tifs(
     parser.add_argument(
         '-t',
         '--output_folder',
-        help='Target: Where the output folder will be',
+        help='OPTIONAL: Target location, Where the output folder will be. Defaults to /data/catfim/',
         required=False,
         default='/data/catfim/',
     )
-    parser.add_argument('-o', '--overwrite', help='Overwrite files', required=False, action="store_true")
+    parser.add_argument(
+        '-o', '--overwrite', help='OPTIONAL: Overwrite files', required=False, action="store_true"
+    )
     parser.add_argument(
         '-s',
         '--search',
-        help='Upstream and downstream search in miles. How far up and downstream do you want to go?',
+        help='OPTIONAL: Upstream and downstream search in miles. How far up and downstream do you want to go? Defaults to 5.',
         required=False,
         default='5',
     )
+
     parser.add_argument(
         '-l',
         '--lid_to_run',
-        help='NWS LID, lowercase, to produce CatFIM for. Currently only accepts one. Default is all sites',
+        help='OPTIONAL: NWS LID, lowercase, to produce CatFIM for. Currently only accepts one. Defaults to all sites',
         required=False,
         default='all',
     )
-    parser.add_argument(
-        '-ji',
-        '--job_number_intervals',
-        help='Number of processes to use for inundating multiple intervals in stage-based'
-        ' inundation and interval job numbers should multiply to no more than one less than the CPU count '
-        'of the machine.',
-        required=False,
-        default=1,
-        type=int,
-    )
+
+    # lst_hucs temp disabled. All hucs in fim outputs in a directory will used
+    # parser.add_argument(
+    #     '-lh',
+    #     '--lst_hucs',
+    #     help='OPTIONAL: Space-delimited list of HUCs to produce CatFIM for. Defaults to all HUCs',
+    #     required=False,
+    #     default='all',
+    # )
+
     parser.add_argument(
         '-mc',
         '--past_major_interval_cap',
-        help='Stage-Based Only. How many feet past major do you want to go for the interval FIMs?'
-        ' of the machine.',
+        help='OPTIONAL: Stage-Based Only. How many feet past major do you want to go for the interval FIMs?'
+        ' of the machine. Defaults to 5.',
         required=False,
         default=5.0,
         type=float,
     )
 
+    # NOTE: This params is for quick debugging only and should not be used in a production mode
+    parser.add_argument(
+        '-me',
+        '--nwm_metafile',
+        help='OPTIONAL: If you have a pre-existing nwm metadata pickle file, you can path to it here.'
+        ' e.g.: /data/catfim/nwm_metafile.pkl',
+        required=False,
+        default="",
+    )
+
     args = vars(parser.parse_args())
-    process_generate_categorical_fim(**args)
+
+    try:
+
+        # call main program
+        process_generate_categorical_fim(**args)
+
+    except Exception:
+        FLOG.critical(traceback.format_exc())
diff --git a/tools/generate_categorical_fim_flows.py b/tools/generate_categorical_fim_flows.py
index a89459e06..7acfb4631 100755
--- a/tools/generate_categorical_fim_flows.py
+++ b/tools/generate_categorical_fim_flows.py
@@ -1,11 +1,17 @@
 #!/usr/bin/env python3
 
+# import csv
+
 import argparse
+import copy
 import os
+import pickle
+import random
 import sys
 import time
+import traceback
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
 import geopandas as gpd
@@ -21,164 +27,257 @@
     get_thresholds,
 )
 
+import utils.fim_logger as fl
 from utils.shared_variables import VIZ_PROJECTION
 
 
+# TODO: Aug 2024: This script was upgraded significantly with lots of misc TODO's embedded.
+# Lots of inline documenation needs updating as well
+
+
+# will become global once initiallized
+FLOG = fl.FIM_logger()
+MP_LOG = fl.FIM_logger()
+
 gpd.options.io_engine = "pyogrio"
 
 
-def get_env_paths():
-    load_dotenv()
+def get_env_paths(env_file):
+
+    if os.path.exists(env_file) is False:
+        raise Exception(f"The environment file of {env_file} does not seem to exist")
+
+    load_dotenv(env_file)
     # import variables from .env file
     API_BASE_URL = os.getenv("API_BASE_URL")
     WBD_LAYER = os.getenv("WBD_LAYER")
     return API_BASE_URL, WBD_LAYER
 
 
-def process_generate_flows(
-    huc, huc_dictionary, threshold_url, all_lists, workspace, attributes_dir, huc_messages_dir, nwm_flows_df
+# This one is for lid magnitudes only and is part of an MP pool
+def generate_flows_for_huc(
+    huc,
+    huc_dictionary,
+    threshold_url,
+    all_meta_lists,
+    output_flows_dir,
+    attributes_dir,
+    huc_messages_dir,
+    nwm_flows_df,
+    parent_log_output_file,
+    child_log_file_prefix,
 ):
-    # Process each huc unit, first define message variable and flood categories.
-    all_messages = []
-    flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
-
-    print(f'Iterating through {huc}')
-    # Get list of nws_lids
-    nws_lids = huc_dictionary[huc]
-    # Loop through each lid in list to create flow file
-    for lid in nws_lids:
-        # Convert lid to lower case
-        lid = lid.lower()
-        # Get stages and flows for each threshold from the WRDS API. Priority given to USGS calculated flows.
-        print("getting thresholds")
-        stages, flows = get_thresholds(
-            threshold_url=threshold_url, select_by='nws_lid', selector=lid, threshold='all'
-        )
-        if stages is None or flows is None:
-            print("Likely WRDS error")
-            continue
-        # Check if stages are supplied, if not write message and exit.
-        if all(stages.get(category, None) is None for category in flood_categories):
-            message = f'{lid}:missing threshold stages'
-            all_messages.append(message)
-            continue
-        # Check if calculated flows are supplied, if not write message and exit.
-        if all(flows.get(category, None) is None for category in flood_categories):
-            message = f'{lid}:missing calculated flows'
-            all_messages.append(message)
-            continue
-        # find lid metadata from master list of metadata dictionaries (line 66).
-        metadata = next((item for item in all_lists if item['identifiers']['nws_lid'] == lid.upper()), False)
-
-        # Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
-        unfiltered_segments = list(set(get_nwm_segs(metadata)))
-
-        desired_order = metadata['nwm_feature_data']['stream_order']
-        # Filter segments to be of like stream order.
-        print("filtering segments")
-        start = time.time()
-        segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df)
-        end = time.time()
-        elapsed_time = round(((end - start) / 60), 6)
-        print(f'Finished filtering segments in {elapsed_time} minutes')
-        # if no segments, write message and exit out
-        if not segments:
-            message = f'{lid}:missing nwm segments'
-            all_messages.append(message)
-            continue
-        # For each flood category
-        for category in flood_categories:
-            # G et the flow
-            flow = flows[category]
-            # If there is a valid flow value, write a flow file.
-            if flow:
-                # round flow to nearest hundredth
-                flow = round(flow, 2)
-                # Create the guts of the flow file.
-                flow_info = flow_data(segments, flow)
-                # Define destination path and create folders
-                output_file = (
-                    workspace / huc / lid / category / (f'ahps_{lid}_huc_{huc}_flows_{category}.csv')
-                )
-                output_file.parent.mkdir(parents=True, exist_ok=True)
-                # Write flow file to file
-                flow_info.to_csv(output_file, index=False)
-            else:
-                message = f'{lid}:{category} is missing calculated flow'
+
+    try:
+        # Note: child_log_file_prefix is "MP_process_gen_flows", meaning all logs created by this function start
+        #  with the phrase "MP_process_gen_flows". This will roll up to the master catfim log.
+        # This is setting up logging for this function to go up to the parent
+        MP_LOG.MP_Log_setup(parent_log_output_file, child_log_file_prefix)
+
+        start_time = datetime.now(timezone.utc)
+        dt_string = start_time.strftime("%m/%d/%Y %H:%M:%S")
+
+        # A bit of start staggering to help not overload the MP (20 sec)
+        time_delay = random.randrange(0, 20)
+        # MP_LOG.lprint(f" ... {huc} start time is {dt_string} and delay is {time_delay}")
+        MP_LOG.lprint(f" ... {huc} flow generation start time is {dt_string}")
+
+        time.sleep(time_delay)
+
+        # Process each huc unit, first define message variable and flood categories.
+        all_messages = []
+        flood_categories = ['action', 'minor', 'moderate', 'major', 'record']
+
+        nws_lids = huc_dictionary[huc]
+
+        if len(nws_lids) == 0:
+            MP_LOG.lprint(f"huc {huc} has no applicable nws_lids")
+            return
+
+        # Loop through each lid in list to create flow file
+        for lid in nws_lids:
+            # Convert lid to lower case
+            lid = lid.lower()
+
+            # TODO:  Jun 17, 2024 - This gets recalled for every huc but only uses the nws_list.
+            # Move this somewhere outside the huc list so it doesn't need to be called over and over again
+
+            # Careful, for "all_message.append" the syntax into it must be f'{lid}: (whever messages)
+            # this is gets parsed and logic used against it.
+
+            MP_LOG.trace(f'Getting thresholds for {lid}')
+            stages, flows = get_thresholds(
+                threshold_url=threshold_url, select_by='nws_lid', selector=lid, threshold='all'
+            )
+
+            if len(stages) == 0 or len(flows) == 0:
+                message = f'{lid}:no stages or flows exist, likely WRDS error'
                 all_messages.append(message)
-        # Get various attributes of the site.
-        lat = float(metadata['nws_preferred']['latitude'])
-        lon = float(metadata['nws_preferred']['longitude'])
-        wfo = metadata['nws_data']['wfo']
-        rfc = metadata['nws_data']['rfc']
-        state = metadata['nws_data']['state']
-        county = metadata['nws_data']['county']
-        name = metadata['nws_data']['name']
-        flow_source = flows['source']
-        stage_source = stages['source']
-        wrds_timestamp = stages['wrds_timestamp']
-        nrldb_timestamp = metadata['nrldb_timestamp']
-        nwis_timestamp = metadata['nwis_timestamp']
-
-        # Create a csv with same information as shapefile but with each threshold as new record.
-        csv_df = pd.DataFrame()
-        for threshold in flood_categories:
-            line_df = pd.DataFrame(
-                {
-                    'nws_lid': [lid],
-                    'name': name,
-                    'WFO': wfo,
-                    'rfc': rfc,
-                    'huc': [huc],
-                    'state': state,
-                    'county': county,
-                    'magnitude': threshold,
-                    'q': flows[threshold],
-                    'q_uni': flows['units'],
-                    'q_src': flow_source,
-                    'stage': stages[threshold],
-                    'stage_uni': stages['units'],
-                    's_src': stage_source,
-                    'wrds_time': wrds_timestamp,
-                    'nrldb_time': nrldb_timestamp,
-                    'nwis_time': nwis_timestamp,
-                    'lat': [lat],
-                    'lon': [lon],
-                }
+                MP_LOG.warning(f"{huc} - {message}")
+                continue
+
+            # Check if stages are supplied, if not write message and exit.
+            if all(stages.get(category, None) is None for category in flood_categories):
+                message = f'{lid}:missing threshold stages'
+                all_messages.append(message)
+                MP_LOG.warning(f"{huc} - {message}")
+                continue
+
+            # Check if calculated flows are supplied, if not write message and exit.
+            if all(flows.get(category, None) is None for category in flood_categories):
+                message = f'{lid}:missing calculated flows'
+                all_messages.append(message)
+                MP_LOG.warning(f"{huc} - {message}")
+                continue
+
+            # Find lid metadata from master list of metadata dictionaries (line 66).
+            metadata = next(
+                (item for item in all_meta_lists if item['identifiers']['nws_lid'] == lid.upper()), False
             )
-            csv_df = pd.concat([csv_df, line_df])
-        # Round flow and stage columns to 2 decimal places.
-        csv_df = csv_df.round({'q': 2, 'stage': 2})
-
-        # If a site folder exists (ie a flow file was written) save files containing site attributes.
-        output_dir = workspace / huc / lid
-        if output_dir.exists():
-            # Export DataFrame to csv containing attributes
-            csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index=False)
-            message = f'{lid}:flows available'
-            all_messages.append(message)
-        else:
-            message = f'{lid}:missing all calculated flows'
-            all_messages.append(message)
 
-    # Write all_messages to huc-specific file.
-    print("Writing message file for huc")
-    huc_messages_txt_file = os.path.join(huc_messages_dir, str(huc) + '_messages.txt')
-    with open(huc_messages_txt_file, 'w') as f:
-        for item in all_messages:
-            f.write("%s\n" % item)
+            # Get mainstem segments of LID by intersecting LID segments with known mainstem segments.
+            unfiltered_segments = list(set(get_nwm_segs(metadata)))
+            desired_order = metadata['nwm_feature_data']['stream_order']
 
+            # Filter segments to be of like stream order.
+            segments = filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_flows_df)
 
-def generate_catfim_flows(
-    workspace,
+            # If there are no segments, write message and exit out
+            if not segments or len(segments) == 0:
+                message = f'{lid}:missing nwm segments'
+                all_messages.append(message)
+                MP_LOG.warning(f"{huc} - {message}")
+                continue
+
+            # For each flood category
+            for category in flood_categories:
+                # Get the flow
+                flow = flows[category]
+
+                if flow is not None and flow != 0:
+
+                    # If there is a valid flow value, write a flow file.
+                    # if flow:
+                    # round flow to nearest hundredth
+                    flow = round(flow, 2)
+
+                    # Create the guts of the flow file.
+                    flow_info = flow_data(segments, flow)
+
+                    # Define destination path and create folders
+                    csv_output_folder = os.path.join(output_flows_dir, huc, lid, category)
+                    os.makedirs(csv_output_folder, exist_ok=True)
+                    output_file = os.path.join(
+                        csv_output_folder, f'ahps_{lid}_huc_{huc}_flows_{category}.csv'
+                    )
+
+                    # Write flow file to file
+                    flow_info.to_csv(output_file, index=False)
+
+                else:
+                    message = f'{lid}:{category} is missing calculated flow'
+                    all_messages.append(message)
+                    MP_LOG.warning(f"{huc} - {message}")
+
+            # Get various attributes of the site.
+            lat = float(metadata['nws_preferred']['latitude'])
+            lon = float(metadata['nws_preferred']['longitude'])
+            wfo = metadata['nws_data']['wfo']
+            rfc = metadata['nws_data']['rfc']
+            state = metadata['nws_data']['state']
+            county = metadata['nws_data']['county']
+            name = metadata['nws_data']['name']
+            flow_source = flows['source']
+            stage_source = stages['source']
+            wrds_timestamp = stages['wrds_timestamp']
+            nrldb_timestamp = metadata['nrldb_timestamp']
+            nwis_timestamp = metadata['nwis_timestamp']
+
+            # Create a csv with same information as shapefile but with each threshold as new record.
+            csv_df = pd.DataFrame()
+            for threshold in flood_categories:
+                line_df = pd.DataFrame(
+                    {
+                        'nws_lid': [lid],
+                        'name': name,
+                        'WFO': wfo,
+                        'rfc': rfc,
+                        'huc': [huc],
+                        'state': state,
+                        'county': county,
+                        'magnitude': threshold,
+                        'q': flows[threshold],
+                        'q_uni': flows['units'],
+                        'q_src': flow_source,
+                        'stage': stages[threshold],
+                        'stage_uni': stages['units'],
+                        's_src': stage_source,
+                        'wrds_time': wrds_timestamp,
+                        'nrldb_time': nrldb_timestamp,
+                        'nwis_time': nwis_timestamp,
+                        'lat': [lat],
+                        'lon': [lon],
+                    }
+                )
+                csv_df = pd.concat([csv_df, line_df])
+
+            # Round flow and stage columns to 2 decimal places.
+            csv_df = csv_df.round({'q': 2, 'stage': 2})
+
+            # If a site folder exists (ie a flow file was written) save files containing site attributes.
+            huc_lid_flow_dir = os.path.join(output_flows_dir, huc, lid)
+
+            if os.path.exists(huc_lid_flow_dir):
+                # Export DataFrame to csv containing attributes
+                csv_df.to_csv(os.path.join(attributes_dir, f'{lid}_attributes.csv'), index=False)
+                message = f'{lid}:flows available'
+                all_messages.append(message)
+            else:
+                message = f'{lid}:missing all calculated flows'
+                all_messages.append(message)
+                MP_LOG.warning(f"Missing all calculated flows for {huc} - {lid}")
+
+        # Write all_messages to huc-specific file.
+        # MP_LOG.lprint(f'Writing message file for {huc}')
+        huc_messages_txt_file = os.path.join(huc_messages_dir, str(huc) + '_messages.txt')
+        with open(huc_messages_txt_file, 'w') as f:
+            for item in all_messages:
+                item = item.strip()
+                # f.write("%s\n" % item)
+                f.write(f"{item}\n")
+        # MP_LOG.lprint(f'--- generate_flow_for_huc done for {huc}')
+
+        end_time = datetime.now(timezone.utc)
+        dt_string = end_time.strftime("%m/%d/%Y %H:%M:%S")
+        time_duration = end_time - start_time
+        MP_LOG.lprint(f" ... {huc} end time is {dt_string} :  Duration: {str(time_duration).split('.')[0]}")
+        print("")
+
+    except Exception as ex:
+        MP_LOG.error(f"An error occured while generating flows for huc {huc}")
+        MP_LOG.error(f"Details: {ex}")
+        MP_LOG.error(traceback.format_exc())
+
+    print("")
+    return
+
+
+# This is called from within this script and is not MP, so it can use FLOG directly
+# lid_to_run is temp disabled
+def generate_flows(
+    output_catfim_dir,
     nwm_us_search,
     nwm_ds_search,
-    stage_based,
-    fim_dir,
     lid_to_run,
-    attributes_dir="",
-    job_number_huc=1,
+    env_file,
+    job_number_huc,
+    is_stage_based,
+    lst_hucs,
+    nwm_metafile,
+    log_output_file,
 ):
+
+    # TODO; Most docstrings like this are now very outdated and need updating
     '''
     This will create static flow files for all nws_lids and save to the
     workspace directory with the following format:
@@ -192,8 +291,8 @@ def generate_catfim_flows(
     flow file to use for inundation mapping.
     Parameters
     ----------
-    workspace : STR
-        Location where output flow files will exist.
+    output_catfim_dir : STR
+        root catfim dir for the particular run. ie) fim_4_3_3_4_stage_based
     nwm_us_search : STR
         Upstream distance (in miles) for walking up NWM network.
     nwm_ds_search : STR
@@ -203,133 +302,152 @@ def generate_catfim_flows(
 
     Returns
     -------
-    None.
+    nws_lid_gpkg_file_path. - Name and path of the nws_lid file
     '''
 
-    all_start = datetime.now()
-    API_BASE_URL, WBD_LAYER = get_env_paths()
-    # Define workspace and wbd_path as a pathlib Path. Convert search distances to integer.
-    workspace = Path(workspace)
+    FLOG.setup(log_output_file)  # reusing the parent logs
+
+    # FLOG.trace("args coming into generate flows")
+    # FLOG.trace(locals()) # see all args coming in to the function
+
+    attributes_dir = os.path.join(output_catfim_dir, 'attributes')
+    mapping_dir = os.path.join(output_catfim_dir, "mapping")  # create var but don't make folder yet
+
+    all_start = datetime.now(timezone.utc)
+    API_BASE_URL, WBD_LAYER = get_env_paths(env_file)
     nwm_us_search = int(nwm_us_search)
     nwm_ds_search = int(nwm_ds_search)
     metadata_url = f'{API_BASE_URL}/metadata'
     threshold_url = f'{API_BASE_URL}/nws_threshold'
     ###################################################################
 
-    # Create workspace
-    workspace.mkdir(parents=True, exist_ok=True)
-
     # Create HUC message directory to store messages that will be read and joined after multiprocessing
-    huc_messages_dir = os.path.join(workspace, 'huc_messages')
-    if not os.path.exists(huc_messages_dir):
-        os.mkdir(huc_messages_dir)
+    huc_messages_dir = os.path.join(mapping_dir, 'huc_messages')
+    os.makedirs(huc_messages_dir, exist_ok=True)
+
+    FLOG.lprint("Loading nwm flow metadata")
+    start_dt = datetime.now(timezone.utc)
 
-    # Open NWM flows geopackage
+    # Open NWM flows geopackages
     nwm_flows_gpkg = r'/data/inputs/nwm_hydrofabric/nwm_flows.gpkg'
     nwm_flows_df = gpd.read_file(nwm_flows_gpkg)
 
-    print(f'Retrieving metadata for site(s): {lid_to_run}...')
-    start_dt = datetime.now()
+    # nwm_flows_alaska_gpkg = r'/data/inputs/nwm_hydrofabric/nwm_flows_alaska_nwmV3_ID.gpkg' # Uncomment to include Alaska
+    # nwm_flows_alaska_df = gpd.read_file(nwm_flows_alaska_gpkg) # Uncomment to include Alaska
 
-    # Get metadata for 'CONUS'
-    print(metadata_url)
-    if lid_to_run != 'all':
-        all_lists, conus_dataframe = get_metadata(
-            metadata_url,
-            select_by='nws_lid',
-            selector=[lid_to_run],
-            must_include='nws_data.rfc_forecast_point',
-            upstream_trace_distance=nwm_us_search,
-            downstream_trace_distance=nwm_ds_search,
-        )
-    else:
-        # Get CONUS metadata
-        conus_list, conus_dataframe = get_metadata(
-            metadata_url,
-            select_by='nws_lid',
-            selector=['all'],
-            must_include='nws_data.rfc_forecast_point',
-            upstream_trace_distance=nwm_us_search,
-            downstream_trace_distance=nwm_ds_search,
-        )
-        # Get metadata for Islands
-        islands_list, islands_dataframe = get_metadata(
-            metadata_url,
-            select_by='state',
-            selector=['HI', 'PR'],
-            must_include=None,
-            upstream_trace_distance=nwm_us_search,
-            downstream_trace_distance=nwm_ds_search,
-        )
-        # Append the dataframes and lists
-        all_lists = conus_list + islands_list
-    print(len(all_lists))
+    # nwm_metafile might be an empty string
+    # maybe ensure all projections are changed to one standard output of 3857 (see shared_variables) as the come out
 
-    end_dt = datetime.now()
+    # TODO: Aug 2024:
+    # Filter the meta list to just HUCs in the fim run output or huc if sent in as a param
+    all_meta_lists = __load_nwm_metadata(
+        output_catfim_dir, metadata_url, nwm_us_search, nwm_ds_search, lid_to_run, nwm_metafile
+    )
+
+    end_dt = datetime.now(timezone.utc)
     time_duration = end_dt - start_dt
-    print(f"Retrieving metadata Duration: {str(time_duration).split('.')[0]}")
-    print()
+    FLOG.lprint(f"Retrieving metadata - Duration: {str(time_duration).split('.')[0]}")
 
-    print('Determining HUC using WBD layer...')
-    start_dt = datetime.now()
+    print("")
 
-    # Assign HUCs to all sites using a spatial join of the FIM 3 HUC layer.
+    # Assign HUCs to all sites using a spatial join of the FIM 4 HUC layer.
     # Get a dictionary of hucs (key) and sites (values) as well as a GeoDataFrame
     # of all sites used later in script.
-    huc_dictionary, out_gdf = aggregate_wbd_hucs(
-        metadata_list=all_lists, wbd_huc8_path=WBD_LAYER, retain_attributes=True
-    )
+
+    FLOG.lprint("Start aggregate_wbd_hucs")
+    start_dt = datetime.now(timezone.utc)
+
+    huc_dictionary, out_gdf = aggregate_wbd_hucs(all_meta_lists, WBD_LAYER, True, lst_hucs)
+
     # Drop list fields if invalid
     out_gdf = out_gdf.drop(['downstream_nwm_features'], axis=1, errors='ignore')
     out_gdf = out_gdf.drop(['upstream_nwm_features'], axis=1, errors='ignore')
     out_gdf = out_gdf.astype({'metadata_sources': str})
 
-    end_dt = datetime.now()
+    end_dt = datetime.now(timezone.utc)
     time_duration = end_dt - start_dt
-    print(f"Determining HUC using WBD layer Duration: {str(time_duration).split('.')[0]}")
-    print()
+    FLOG.lprint(f"End aggregate_wbd_hucs - Duration: {str(time_duration).split('.')[0]}")
+
+    FLOG.lprint("Start Flow Generation")
 
-    if stage_based:
-        return huc_dictionary, out_gdf, metadata_url, threshold_url, all_lists, nwm_flows_df
+    # It this is stage-based, it returns all of these objects here, but if it continues
+    # (aka. Flow based), then it returns only nws_lid_layer (created later in this function)
+    if is_stage_based:  # If it's stage-based, the function stops running here
+        return (
+            huc_dictionary,
+            out_gdf,
+            metadata_url,
+            threshold_url,
+            all_meta_lists,
+            nwm_flows_df,
+        )  # No Alaska
+        # return (huc_dictionary, out_gdf, metadata_url, threshold_url, all_meta_lists, nwm_flows_df, nwm_flows_alaska_df) # Alaska
 
-    print("Generating flows for hucs using " + str(job_number_huc) + " jobs...")
-    start_dt = datetime.now()
+    # only flow based needs the "flow" dir
+    output_flows_dir = os.path.join(output_catfim_dir, "flows")
+    if not os.path.exists(output_flows_dir):
+        os.mkdir(output_flows_dir)
 
+    start_dt = datetime.now(timezone.utc)
+
+    # pulls out the parent log file and replaces it with the child prefix
+    # catfim if coming from generate_categorical_fim.py
+
+    child_log_file_prefix = FLOG.MP_calc_prefix_name(log_output_file, "MP_process_gen_flows")
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
         for huc in huc_dictionary:
+
+            nwm_flows_region_df = nwm_flows_df  # To exclude Alaska
+            # nwm_flows_region_df = nwm_flows_alaska_df if huc[:2] == '19' else nwm_flows_df # To include Alaska
+
+            # Deep copy that speed up Multi-Proc a little as all_meta_lists
+            # is a huge object. Need to figure out how to filter that down somehow
+            # later. Can not just filter by huc per loop, tried it and there are other factors
+            copy_all_meta_lists = copy.copy(all_meta_lists)
             executor.submit(
-                process_generate_flows,
+                generate_flows_for_huc,
                 huc,
                 huc_dictionary,
                 threshold_url,
-                all_lists,
-                workspace,
+                copy_all_meta_lists,
+                output_flows_dir,
                 attributes_dir,
                 huc_messages_dir,
-                nwm_flows_df,
+                nwm_flows_region_df,
+                log_output_file,
+                child_log_file_prefix,
             )
+    # end ProcessPoolExecutor
 
-    end_dt = datetime.now()
+    # rolls up logs from child MP processes into this parent_log_output_file
+    FLOG.merge_log_files(log_output_file, child_log_file_prefix, True)
+
+    end_dt = datetime.now(timezone.utc)
     time_duration = end_dt - start_dt
-    print(f"Generating flows for hucs Duration: {str(time_duration).split('.')[0]}")
+    FLOG.lprint(f"End flow generation - Duration: {str(time_duration).split('.')[0]}")
     print()
 
-    print('Wrapping up flows generation...')
+    FLOG.lprint('Start merging and finalizing flows generation data')
     # Recursively find all *_attributes csv files and append
-    csv_files = os.listdir(attributes_dir)
+    csv_files = [x for x in os.listdir(attributes_dir) if x.endswith('_attributes.csv')]
+
+    if len(csv_files) == 0:
+        MP_LOG.critical(f"No new flow files exist in the {attributes_dir} folder (errors in creating them?)")
+        sys.exit(1)
+
     all_csv_df = pd.DataFrame()
-    for csv in csv_files:
-        full_csv_path = os.path.join(attributes_dir, csv)
+    for csv_file in csv_files:
+        full_csv_path = os.path.join(attributes_dir, csv_file)
         # Huc has to be read in as string to preserve leading zeros.
         temp_df = pd.read_csv(full_csv_path, dtype={'huc': str})
         all_csv_df = pd.concat([all_csv_df, temp_df], ignore_index=True)
     # Write to file
-    all_csv_df.to_csv(os.path.join(workspace, 'nws_lid_attributes.csv'), index=False)
+    all_csv_df.to_csv(os.path.join(attributes_dir, 'nws_lid_attributes.csv'), index=False)
 
     # This section populates a shapefile of all potential sites and details
     # whether it was mapped or not (mapped field) and if not, why (status field).
-
     # Preprocess the out_gdf GeoDataFrame. Reproject and reformat fields.
+
     viz_out_gdf = out_gdf.to_crs(VIZ_PROJECTION)
     viz_out_gdf.rename(
         columns={
@@ -354,7 +472,8 @@ def generate_catfim_flows(
     viz_out_gdf = viz_out_gdf.merge(lids_df, how='left', on='nws_lid')
     viz_out_gdf['mapped'] = viz_out_gdf['mapped'].fillna('no')
 
-    # Read all messages for all HUCs TODO
+    # Read all messages for all HUCs
+    # this is basically identical to a stage based set. Seach for huc_message_list and see my notes
     huc_message_list = []
     huc_messages_dir_list = os.listdir(huc_messages_dir)
     for huc_message_file in huc_messages_dir_list:
@@ -363,63 +482,198 @@ def generate_catfim_flows(
             if full_path_file.endswith('.txt'):
                 lines = f.readlines()
                 for line in lines:
+                    line = line.strip()
                     huc_message_list.append(line)
 
     # Write messages to DataFrame, split into columns, aggregate messages.
-    messages_df = pd.DataFrame(huc_message_list, columns=['message'])
-    messages_df = (
-        messages_df['message'].str.split(':', n=1, expand=True).rename(columns={0: 'nws_lid', 1: 'status'})
-    )
-    status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
+    if len(huc_message_list) > 0:
+
+        messages_df = pd.DataFrame(huc_message_list, columns=['message'])
+        messages_df = (
+            messages_df['message']
+            .str.split(':', n=1, expand=True)
+            .rename(columns={0: 'nws_lid', 1: 'status'})
+        )
+
+        # There could be duplicate message for one ahps (ie. missing nwm segments), so drop dups
+        messages_df.drop_duplicates(subset=["nws_lid", "status"], keep="first", inplace=True)
+
+        # We want one viz_out_gdf record per ahps and if there are more than one, contact the messages
 
-    # Join messages to populate status field to candidate sites. Assign
-    # status for null fields.
-    viz_out_gdf = viz_out_gdf.merge(status_df, how='left', on='nws_lid')
-    viz_out_gdf['status'] = viz_out_gdf['status'].fillna('all calculated flows available')
+        # status_df = messages_df.groupby(['nws_lid'])['status'].apply(', '.join).reset_index()
+        # df1 = df.groupby(['ID1','ID2'])['Status'].agg(lambda x: ','.join(x.dropna())).reset_index()
+        status_df = messages_df.groupby(['nws_lid'])['status'].agg(lambda x: ',\n'.join(x)).reset_index()
+
+        # some messages status values start with a space as the first character. Remove it
+        # status_df["status"] = status_df["status"].apply(lambda x: x.strip())
+
+        # Join messages to populate status field to candidate sites. Assign
+        # status for null fields.
+        viz_out_gdf = viz_out_gdf.merge(status_df, how='left', on='nws_lid')
+
+        viz_out_gdf['status'] = viz_out_gdf['status'].fillna('all calculated flows available')
 
     # Filter out columns and write out to file
     # viz_out_gdf = viz_out_gdf.filter(
     #     ['nws_lid', 'usgs_gage', 'nwm_seg', 'HUC8', 'mapped', 'status', 'geometry']
     # )
-    nws_lid_layer = os.path.join(workspace, 'nws_lid_sites.gpkg').replace('flows', 'mapping')
 
-    viz_out_gdf.to_file(nws_lid_layer, driver='GPKG')
+    # stage based doesn't get here
+    # crs is 3857 - web mercator at this point
+    nws_lid_csv_file_path = os.path.join(mapping_dir, 'flow_based_catfim_sites.csv')
+    viz_out_gdf.to_csv(nws_lid_csv_file_path)
+
+    nws_lid_gpkg_file_path = os.path.join(mapping_dir, 'flow_based_catfim_sites.gpkg')
+    viz_out_gdf.to_file(nws_lid_gpkg_file_path, driver='GPKG', index=False, engine='fiona')
 
     # time operation
-    all_end = datetime.now()
+    all_end = datetime.now(timezone.utc)
     all_time_duration = all_end - all_start
-    print(f"Duration: {str(all_time_duration).split('.')[0]}")
+    FLOG.lprint(f"End Wrapping up flows generation Duration: {str(all_time_duration).split('.')[0]}")
     print()
 
-    return nws_lid_layer
+    return nws_lid_gpkg_file_path
+
+
+# local script calls __load_nwm_metadata so FLOG is already setup
+def __load_nwm_metadata(
+    output_catfim_dir, metadata_url, nwm_us_search, nwm_ds_search, lid_to_run, nwm_metafile
+):
+    FLOG.trace(metadata_url)
+
+    all_meta_lists = []
+    # Check to see if meta file already exists
+    # This feature means we can copy the pickle file to another enviro (AWS?) as it won't need to call
+    # WRDS unless we need a smaller or modified version. This one likely has all nws_lid data.
+
+    if os.path.isfile(nwm_metafile) is True:
+        FLOG.lprint(f"Meta file already downloaded and exists at {nwm_metafile}")
+
+        with open(nwm_metafile, "rb") as p_handle:
+            all_meta_lists = pickle.load(p_handle)
+
+    else:
+        meta_file = os.path.join(output_catfim_dir, "nwm_metafile.pkl")
+
+        FLOG.lprint(f"Meta file will be downloaded and saved at {meta_file}")
+
+        # lid_to_run coudl be a single lid or the word "all"
+
+        if lid_to_run != "all":
+            all_meta_lists, ___ = get_metadata(
+                metadata_url,
+                select_by='nws_lid',
+                selector=[lid_to_run],
+                must_include='nws_data.rfc_forecast_point',
+                upstream_trace_distance=nwm_us_search,
+                downstream_trace_distance=nwm_ds_search,
+            )
+        else:
+            conus_list, ___ = get_metadata(
+                metadata_url,
+                select_by='nws_lid',
+                selector=['all'],
+                must_include='nws_data.rfc_forecast_point',
+                upstream_trace_distance=nwm_us_search,
+                downstream_trace_distance=nwm_ds_search,
+            )
+            # Get metadata for Islands and Alaska
+            islands_list, ___ = get_metadata(
+                metadata_url,
+                select_by='state',
+                selector=['HI', 'PR', 'AK'],
+                must_include=None,
+                upstream_trace_distance=nwm_us_search,
+                downstream_trace_distance=nwm_ds_search,
+            )
+            # Append the lists
+            all_meta_lists = conus_list + islands_list
+
+        with open(meta_file, "wb") as p_handle:
+            pickle.dump(all_meta_lists, p_handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+    return all_meta_lists
 
 
 if __name__ == '__main__':
     # Parse arguments
     parser = argparse.ArgumentParser(description='Create forecast files for all nws_lid sites')
-    parser.add_argument('-w', '--workspace', help='Workspace where all data will be stored.', required=True)
     parser.add_argument(
-        '-u', '--nwm_us_search', help='Walk upstream on NWM network this many miles', required=True
+        '-w', '--output_catfim_dir', help='Workspace where all data will be stored.', required=True
     )
+
     parser.add_argument(
-        '-d', '--nwm_ds_search', help='Walk downstream on NWM network this many miles', required=True
+        '-log',
+        '--log_output_file',
+        help='REQUIRED: Path to where the output log file will be.'
+        r'ie) /data/catfim/rob_test/logs/catfim_2024_07_07-22_26_18.log',
+        required=True,
+        type=str,
     )
+
+    parser.add_argument(
+        '-e',
+        '--env_file',
+        help='Docker mount path to the catfim environment file. ie) data/config/catfim.env',
+        required=True,
+    )
+
+    parser.add_argument(
+        '-hucs',
+        '--lst_hucs',
+        help='list of hucs that you want to process. ie) -hucs 12090301 01100006 12040101',
+        required=True,
+        type=str,
+        nargs='+',
+    )
+
+    parser.add_argument(
+        '-u',
+        '--nwm_us_search',
+        help='Walk upstream on NWM network this many miles',
+        required=False,
+        default=5,
+    )
+
+    parser.add_argument(
+        '-d',
+        '--nwm_ds_search',
+        help='Walk downstream on NWM network this many miles',
+        required=False,
+        default=5,
+    )
+
+    parser.add_argument(
+        '-jh',
+        '--job_number_huc',
+        help='OPTIONAL: Number of processes to use for HUC scale operations.'
+        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count of the'
+        ' machine. CatFIM sites generally only have 2-3 branches overlapping a site, so this number can be '
+        'kept low (2-4). Defaults to 1.',
+        required=False,
+        default=1,
+        type=int,
+    )
+
     parser.add_argument(
         '-a',
-        '--stage_based',
-        help='Run stage-based CatFIM instead of flow-based? NOTE: flow-based CatFIM is the default.',
+        '--is_stage_based',
+        help='Is this a stage based or flow based run? Add the -a to mean is_stage_based is True ',
         required=False,
         default=False,
         action='store_true',
     )
+
     parser.add_argument(
-        '-f',
-        '--fim-dir',
-        help='Path to FIM outputs directory. Only use this option if you are running in alt-catfim mode.',
+        '-n',
+        '--nwm_metafile',
+        help='OPTIONAL: Path to the pre-made pickle file that already holds the nwm metadata',
         required=False,
+        type=str,
         default="",
     )
+
     args = vars(parser.parse_args())
 
     # Run get_env_paths and static_flow_lids
-    generate_catfim_flows(**args)
+    generate_flows(**args)
diff --git a/tools/generate_categorical_fim_mapping.py b/tools/generate_categorical_fim_mapping.py
index 9ff6cab18..a09a5c65b 100755
--- a/tools/generate_categorical_fim_mapping.py
+++ b/tools/generate_categorical_fim_mapping.py
@@ -1,68 +1,385 @@
 #!/usr/bin/env python3
 
 import argparse
+
+# import glob
 import os
+
+# import shutil
 import sys
+import time
 import traceback
-from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+from concurrent.futures import ProcessPoolExecutor
 
 import geopandas as gpd
+import numpy as np
 import pandas as pd
 import rasterio
 from inundate_gms import Inundate_gms
 from mosaic_inundation import Mosaic_inundation
 from rasterio.features import shapes
+from rasterio.warp import Resampling, calculate_default_transform, reproject
 from shapely.geometry.multipolygon import MultiPolygon
 from shapely.geometry.polygon import Polygon
+from tqdm import tqdm
 
+import utils.fim_logger as fl
 from utils.shared_functions import getDriver
-from utils.shared_variables import PREP_PROJECTION, VIZ_PROJECTION
+from utils.shared_variables import ALASKA_CRS, PREP_PROJECTION, VIZ_PROJECTION
+
 
+# TODO: Aug 2024: This script was upgraded significantly with lots of misc TODO's embedded.
+# Lots of inline documenation needs updating as well
+
+
+# will become global once initiallized
+FLOG = fl.FIM_logger()
+MP_LOG = fl.FIM_logger()
 
 gpd.options.io_engine = "pyogrio"
 
 
-def generate_categorical_fim(
-    fim_run_dir, source_flow_dir, output_catfim_dir, job_number_huc, job_number_inundate, depthtif, log_file
+# Technically, this is once called as a non MP, but also called in an MP pool
+# we will use an MP object either way
+def produce_stage_based_catfim_tifs(
+    stage,
+    datum_adj_ft,
+    branch_dir,
+    lid_usgs_elev,
+    lid_altitude,
+    fim_dir,
+    segments,
+    lid,
+    huc,
+    lid_directory,
+    category,
+    number_of_jobs,
+    parent_log_output_file,
+    child_log_file_prefix,
+):
+
+    MP_LOG.MP_Log_setup(parent_log_output_file, child_log_file_prefix)
+
+    messages = []
+
+    MP_LOG.lprint("-----------------")
+    huc_lid_cat_id = f"{huc} : {lid} : {category}"
+    MP_LOG.lprint(f"{huc_lid_cat_id}: Starting to create tifs")
+
+    # Determine datum-offset water surface elevation (from above).
+    datum_adj_wse = stage + datum_adj_ft + lid_altitude
+    datum_adj_wse_m = datum_adj_wse * 0.3048  # Convert ft to m
+
+    # Subtract HAND gage elevation from HAND WSE to get HAND stage.
+    hand_stage = datum_adj_wse_m - lid_usgs_elev
+
+    # TODO: see what happens if this is returned with tj
+
+    # If no segments, write message and exit out
+    if not segments or len(segments) == 0:
+        msg = ': missing nwm segments'
+        messages.append(lid + msg)
+        MP_LOG.warning(huc_lid_cat_id + msg)
+        return messages, hand_stage, datum_adj_wse, datum_adj_wse_m
+
+    # Produce extent tif hand_stage. Multiprocess across branches.
+    # branches = os.listdir(branch_dir)
+    MP_LOG.lprint(f"{huc_lid_cat_id} branch_dir is {branch_dir}")
+
+    branches = [x for x in os.listdir(branch_dir) if os.path.isdir(os.path.join(branch_dir, x))]
+    branches.sort()
+
+    # We need to merge what we have up to this point.
+    # MP_LOG.merge_log_files(parent_log_output_file, child_log_file_prefix)
+    # child_log_file_prefix = MP_LOG.MP_calc_prefix_name(parent_log_output_file, "MP_prod_huc_mag_stage", huc)
+    child_log_file_prefix = MP_LOG.MP_calc_prefix_name(parent_log_output_file, "MP_prod_huc_mag_stage")
+    with ProcessPoolExecutor(max_workers=number_of_jobs) as executor:
+        for branch in branches:
+            msg_id_w_branch = f"{huc} - {branch} - {lid} - {category}"
+            MP_LOG.trace(f"{msg_id_w_branch} : inundating branches")
+            # Define paths to necessary files to produce inundation grids.
+            full_branch_path = os.path.join(branch_dir, branch)
+            rem_path = os.path.join(fim_dir, huc, full_branch_path, 'rem_zeroed_masked_' + branch + '.tif')
+            catchments_path = os.path.join(
+                fim_dir,
+                huc,
+                full_branch_path,
+                'gw_catchments_reaches_filtered_addedAttributes_' + branch + '.tif',
+            )
+            hydrotable_path = os.path.join(fim_dir, huc, full_branch_path, 'hydroTable_' + branch + '.csv')
+
+            if not os.path.exists(rem_path):
+                msg = ": rem doesn't exist"
+                messages.append(lid + msg)
+                MP_LOG.warning(msg_id_w_branch + msg)
+                continue
+            if not os.path.exists(catchments_path):
+                msg = ": catchments files don't exist"
+                messages.append(lid + msg)
+                MP_LOG.warning(msg_id_w_branch + msg)
+                continue
+            if not os.path.exists(hydrotable_path):
+                msg = ": hydrotable doesn't exist"
+                messages.append(lid + msg)
+                MP_LOG.warning(msg_id_w_branch + msg)
+                continue
+
+            # Use hydroTable to determine hydroid_list from site_ms_segments.
+            hydrotable_df = pd.read_csv(
+                hydrotable_path, low_memory=False, dtype={'HUC': str, 'LakeID': float, 'subdiv_applied': int}
+            )
+            hydroid_list = []
+
+            # Determine hydroids at which to perform inundation
+            for feature_id in segments:
+                # print(f"... feature id is {feature_id}")
+                try:
+                    subset_hydrotable_df = hydrotable_df[hydrotable_df['feature_id'] == int(feature_id)]
+                    hydroid_list += list(subset_hydrotable_df.HydroID.unique())
+                except IndexError:
+                    MP_LOG.trace(
+                        f"Index Error for {huc} -- {branch} -- {category}. FeatureId is {feature_id} : Continuing on."
+                    )
+                    pass
+
+            # Create inundation maps with branch and stage data
+            try:
+                # print("Generating stage-based FIM for " + huc + " and branch " + branch)
+                #
+                # MP_LOG.lprint(f"{huc_lid_cat_id} : Generating stage-based FIM")
+
+                executor.submit(
+                    produce_tif_per_huc_per_mag_for_stage,
+                    rem_path,
+                    catchments_path,
+                    hydroid_list,
+                    hand_stage,
+                    lid_directory,
+                    category,
+                    huc,
+                    lid,
+                    branch,
+                    parent_log_output_file,
+                    child_log_file_prefix,
+                )
+
+            except Exception:
+                msg = f': inundation failed at {category}'
+                messages.append(lid + msg)
+                MP_LOG.warning(msg_id_w_branch + msg)
+                MP_LOG.error(traceback.format_exc())
+
+    MP_LOG.merge_log_files(parent_log_output_file, child_log_file_prefix, True)
+
+    # -- MOSAIC -- #
+    # Merge all rasters in lid_directory that have the same magnitude/category.
+    path_list = []
+
+    MP_LOG.trace(f"Merging files from {lid_directory}")
+    lid_dir_list = os.listdir(lid_directory)
+
+    MP_LOG.lprint(f"{huc}: Merging {category}")
+    # MP_LOG.trace("lid_dir_list is ")
+    # MP_LOG.trace(lid_dir_list)
+    # MP_LOG.lprint("")
+
+    for f in lid_dir_list:
+        if category in f:
+            path_list.append(os.path.join(lid_directory, f))
+
+    # MP_LOG.error("???")
+    # MP_LOG.trace(f"path_list is (pre sort) is {path_list}")
+    # path_list.sort()  # To force branch 0 first in list, sort  it isn't branchs and we don't care the order for mosaiking
+    # MP_LOG.trace(f"path_list is (post sort) is {path_list}")
+
+    path_list.sort()  # To force branch 0 first in list, sort
+
+    # MP_LOG.trace(f"len of path_list is {len(path_list)}")
+
+    if len(path_list) > 0:
+        zero_branch_grid = path_list[0]
+        zero_branch_src = rasterio.open(zero_branch_grid)
+        zero_branch_array = zero_branch_src.read(1)
+        summed_array = zero_branch_array  # Initialize it as the branch zero array
+
+        # Loop through remaining items in list and sum them with summed_array
+        for remaining_raster in path_list[1:]:
+            remaining_raster_src = rasterio.open(remaining_raster)
+            remaining_raster_array_original = remaining_raster_src.read(1)
+
+            # Reproject non-branch-zero grids so I can sum them with the branch zero grid
+            remaining_raster_array = np.empty(zero_branch_array.shape, dtype=np.int8)
+            reproject(
+                remaining_raster_array_original,
+                destination=remaining_raster_array,
+                src_transform=remaining_raster_src.transform,
+                src_crs=remaining_raster_src.crs,  # TODO: Accomodate AK projection?
+                src_nodata=remaining_raster_src.nodata,
+                dst_transform=zero_branch_src.transform,
+                dst_crs=zero_branch_src.crs,  # TODO: Accomodate AK projection?
+                dst_nodata=-1,
+                dst_resolution=zero_branch_src.res,
+                resampling=Resampling.nearest,
+            )
+            # Sum rasters
+            summed_array = summed_array + remaining_raster_array
+
+        del zero_branch_array  # Clean up
+
+        # Define path to merged file, in same format as expected by post_process_cat_fim_for_viz function
+        output_tif = os.path.join(lid_directory, lid + '_' + category + '_extent.tif')
+        profile = zero_branch_src.profile
+        summed_array = summed_array.astype('uint8')
+        with rasterio.open(output_tif, 'w', **profile) as dst:
+            dst.write(summed_array, 1)
+            MP_LOG.lprint(f"output_tif is {output_tif}")
+        del summed_array
+
+    return messages, hand_stage, datum_adj_wse, datum_adj_wse_m
+
+
+# This is part of an MP call and needs MP_LOG
+# This does not actually inundate, it just uses the stage and the catchment to create a tif
+def produce_tif_per_huc_per_mag_for_stage(
+    rem_path,
+    catchments_path,
+    hydroid_list,
+    hand_stage,
+    lid_directory,
+    category,
+    huc,
+    lid,
+    branch,
+    parent_log_output_file,
+    child_log_file_prefix,
+):
+    """
+    # Open rem_path and catchment_path using rasterio.
+    """
+
+    try:
+        # This is setting up logging for this function to go up to the parent
+        MP_LOG.MP_Log_setup(parent_log_output_file, child_log_file_prefix)
+
+        # MP_LOG.lprint("+++++++++++++++++++++++")
+        # MP_LOG.lprint(f"At the start of producing a tif for {huc}")
+        # MP_LOG.trace(locals())
+        # MP_LOG.trace("+++++++++++++++++++++++")
+
+        rem_src = rasterio.open(rem_path)
+        catchments_src = rasterio.open(catchments_path)
+        rem_array = rem_src.read(1)
+        catchments_array = catchments_src.read(1)
+
+        # TEMP: look at a catchment and rem from the same branch.
+        # Then look at a stage based 4.4.0.0 for this huc and see if we can figure out the
+        # intended results. Are we trying for a image that makes all values below the hand stage
+        # value to be a value (kinda like a 1 and 0 ?)
+
+        # Use numpy.where operation to reclassify rem_path on the condition that the pixel values
+        #   are <= to hand_stage and the catchments value is in the hydroid_list.
+        reclass_rem_array = np.where((rem_array <= hand_stage) & (rem_array != rem_src.nodata), 1, 0).astype(
+            'uint8'
+        )
+        hydroid_mask = np.isin(catchments_array, hydroid_list)
+        target_catchments_array = np.where(
+            ((hydroid_mask == True) & (catchments_array != catchments_src.nodata)), 1, 0
+        ).astype('uint8')
+        masked_reclass_rem_array = np.where(
+            ((reclass_rem_array == 1) & (target_catchments_array == 1)), 1, 0
+        ).astype('uint8')
+
+        # Save resulting array to new tif with appropriate name. ie) brdc1_record_extent_18060005.tif
+        # to our mapping/huc/lid site
+        is_all_zero = np.all((masked_reclass_rem_array == 0))
+
+        # MP_LOG.lprint(f"{huc}: masked_reclass_rem_array, is_all_zero is {is_all_zero} for {rem_path}")
+
+        # if not is_all_zero:
+        # if is_all_zero is False: # this logic didn't let ANY files get saved
+        #    'is False' means that the object does not exist and not that it really equals the value of False
+        if is_all_zero == False:  # corrected logic
+            output_tif = os.path.join(
+                lid_directory, lid + '_' + category + '_extent_' + huc + '_' + branch + '.tif'
+            )
+            # MP_LOG.lprint(f" +++ Output_Tif is {output_tif}")
+            with rasterio.Env():
+                profile = rem_src.profile
+                profile.update(dtype=rasterio.uint8)
+                profile.update(nodata=10)
+
+                with rasterio.open(output_tif, 'w', **profile) as dst:
+                    dst.write(masked_reclass_rem_array, 1)
+
+    except Exception:
+        MP_LOG.error(f"{huc} : {lid} Error producing inundation maps with stage")
+        MP_LOG.error(traceback.format_exc())
+
+    return
+
+
+# This is not part of an MP process, but needs to have FLOG carried over so this file can see it
+def run_catfim_inundation(
+    fim_run_dir, output_flows_dir, output_mapping_dir, job_number_huc, job_number_inundate, log_output_file
 ):
-    source_flow_dir_list = os.listdir(source_flow_dir)
-    output_flow_dir_list = os.listdir(fim_run_dir)
+    # Adding a pointer in this file coming from generate_categorial_fim so they can share the same log file
+    FLOG.setup(log_output_file)
 
+    print()
+    FLOG.lprint(">>> Start Inundating and Mosaicking")
+
+    source_flow_huc_dir_list = [
+        x
+        for x in os.listdir(output_flows_dir)
+        if os.path.isdir(os.path.join(output_flows_dir, x)) and x[0] in ['0', '1', '2']
+    ]
+    fim_source_huc_dir_list = [
+        x
+        for x in os.listdir(fim_run_dir)
+        if os.path.isdir(os.path.join(fim_run_dir, x)) and x[0] in ['0', '1', '2']
+    ]
     # Log missing hucs
-    missing_hucs = list(set(source_flow_dir_list) - set(output_flow_dir_list))
+    missing_hucs = list(set(source_flow_huc_dir_list) - set(fim_source_huc_dir_list))
     missing_hucs = [huc for huc in missing_hucs if "." not in huc]
-    if len(missing_hucs) > 0:
-        f = open(log_file, 'a+')
-        f.write(f"Missing hucs from output directory: {', '.join(missing_hucs)}\n")
-        f.close()
-
     # Loop through matching huc directories in the source_flow directory
-    matching_hucs = list(set(output_flow_dir_list) & set(source_flow_dir_list))
+    matching_hucs = list(set(fim_source_huc_dir_list) & set(source_flow_huc_dir_list))
+    matching_hucs.sort()
 
+    child_log_file_prefix = FLOG.MP_calc_prefix_name(log_output_file, "MP_run_ind")
     with ProcessPoolExecutor(max_workers=job_number_huc) as executor:
         for huc in matching_hucs:
             if "." in huc:
                 continue
 
             # Get list of AHPS site directories
-            ahps_site_dir = os.path.join(source_flow_dir, huc)
-            ahps_site_dir_list = os.listdir(ahps_site_dir)
+            huc_flows_dir = os.path.join(output_flows_dir, huc)
+
+            # ahps_site_dir_list = os.listdir(ahps_site_dir)
+            ahps_site_dir_list = [
+                x for x in os.listdir(huc_flows_dir) if os.path.isdir(os.path.join(huc_flows_dir, x))
+            ]
 
-            # Map path to huc directory inside out output_catfim_dir
-            cat_fim_huc_dir = os.path.join(output_catfim_dir, huc)
-            if not os.path.exists(cat_fim_huc_dir):
-                os.mkdir(cat_fim_huc_dir)
+            # Map path to huc directory inside the mapping directory
+            huc_mapping_dir = os.path.join(output_mapping_dir, huc)
+            if not os.path.exists(huc_mapping_dir):
+                os.mkdir(huc_mapping_dir)
 
             # Loop through AHPS sites
             for ahps_site in ahps_site_dir_list:
                 # map parent directory for AHPS source data dir and list AHPS thresholds (act, min, mod, maj)
-                ahps_site_parent = os.path.join(ahps_site_dir, ahps_site)
-                thresholds_dir_list = os.listdir(ahps_site_parent)
+                ahps_site_parent = os.path.join(huc_flows_dir, ahps_site)
 
-                # Map parent directory for all inundation output filesoutput files.
-                cat_fim_huc_ahps_dir = os.path.join(cat_fim_huc_dir, ahps_site)
-                if not os.path.exists(cat_fim_huc_ahps_dir):
-                    os.mkdir(cat_fim_huc_ahps_dir)
+                # thresholds_dir_list = os.listdir(ahps_site_parent)
+                thresholds_dir_list = [
+                    x
+                    for x in os.listdir(ahps_site_parent)
+                    if os.path.isdir(os.path.join(ahps_site_parent, x))
+                ]
+
+                # Map parent directory for all inundation output files output files.
+                huc_site_mapping_dir = os.path.join(huc_mapping_dir, ahps_site)
+                if not os.path.exists(huc_site_mapping_dir):
+                    os.mkdir(huc_site_mapping_dir)
 
                 # Loop through thresholds/magnitudes and define inundation output files paths
                 for magnitude in thresholds_dir_list:
@@ -73,57 +390,91 @@ def generate_categorical_fim(
                         magnitude,
                         'ahps_' + ahps_site + '_huc_' + huc + '_flows_' + magnitude + '.csv',
                     )
+                    # print(f"magnitude_flows_csv is {magnitude_flows_csv}")
                     if os.path.exists(magnitude_flows_csv):
-                        output_extent_grid = os.path.join(
-                            cat_fim_huc_ahps_dir, ahps_site + '_' + magnitude + '_extent.tif'
-                        )
+                        tif_name = ahps_site + '_' + magnitude + '_extent.tif'
+                        output_extent_tif = os.path.join(huc_site_mapping_dir, tif_name)
+                        FLOG.trace(f"Begin inundation for {tif_name}")
                         try:
                             executor.submit(
                                 run_inundation,
                                 magnitude_flows_csv,
                                 huc,
-                                output_extent_grid,
+                                huc_site_mapping_dir,
+                                output_extent_tif,
                                 ahps_site,
                                 magnitude,
-                                log_file,
                                 fim_run_dir,
                                 job_number_inundate,
+                                log_output_file,
+                                child_log_file_prefix,
                             )
+
                         except Exception:
-                            traceback.print_exc()
+                            FLOG.critical(
+                                "An critical error occured while attempting inundation"
+                                f" for {huc} -- {ahps_site} -- {magnitude}"
+                            )
+                            FLOG.critical(traceback.format_exc())
+                            FLOG.merge_log_files(log_output_file, child_log_file_prefix)
                             sys.exit(1)
 
+    # end of ProcessPoolExecutor
+
+    # rolls up logs from child MP processes into this parent_log_output_file
+
+    # hold on merging it up for now, to keep the overall log size down a little
+    FLOG.merge_log_files(log_output_file, child_log_file_prefix, True)
+
+    print()
+    FLOG.lprint(">>> End Inundating and Mosaicking")
 
+    return
+
+
+# This is part of an MP Pool
 def run_inundation(
     magnitude_flows_csv,
     huc,
-    output_extent_grid,
+    output_huc_site_mapping_dir,
+    output_extent_tif,
     ahps_site,
     magnitude,
-    log_file,
     fim_run_dir,
     job_number_inundate,
+    parent_log_output_file,
+    child_log_file_prefix,
 ):
+    # Note: child_log_file_prefix is "MP_run_ind", meaning all logs created by this function start
+    #  with the phrase "MP_run_ind"
+    #  They will be rolled up into the parent_log_output_file
+    # This is setting up logging for this function to go up to the parent\
+    MP_LOG.MP_Log_setup(parent_log_output_file, child_log_file_prefix)
+    # MP_LOG.trace(locals())
+
     huc_dir = os.path.join(fim_run_dir, huc)
+    # Why all high number for job_number_inundate? Inundate_gms has to create inundation for each
+    # branch and merge them.
     try:
-        print("Running Inundate_gms for " + huc)
+        MP_LOG.lprint(f"... Running Inundate_gms and mosiacking for {huc} : {ahps_site} : {magnitude}")
         map_file = Inundate_gms(
             hydrofabric_dir=fim_run_dir,
             forecast=magnitude_flows_csv,
             num_workers=job_number_inundate,
             hucs=huc,
-            inundation_raster=output_extent_grid,
+            inundation_raster=output_extent_tif,
             inundation_polygon=None,
             depths_raster=None,
             verbose=False,
             log_file=None,
             output_fileNames=None,
         )
-        print("Mosaicking for " + huc)
+
+        MP_LOG.trace(f"Mosaicking for {huc} : {ahps_site} : {magnitude}")
         Mosaic_inundation(
             map_file,
             mosaic_attribute='inundation_rasters',
-            mosaic_output=output_extent_grid,
+            mosaic_output=output_extent_tif,
             mask=os.path.join(huc_dir, 'wbd.gpkg'),
             unit_attribute_name='huc8',
             nodata=-9999,
@@ -132,59 +483,152 @@ def run_inundation(
             subset=None,
             verbose=False,
         )
-
+        MP_LOG.trace(f"Mosaicking complete for {huc} : {ahps_site} : {magnitude}")
     except Exception:
         # Log errors and their tracebacks
-        f = open(log_file, 'a+')
-        f.write(f"{output_extent_grid} - inundation error: {traceback.format_exc()}\n")
-        f.close()
+        MP_LOG.error(f"Exception: running inundation for {huc}")
+        MP_LOG.error(traceback.format_exc())
+        return
 
-    # Inundation.py appends the huc code to the supplied output_extent_grid.
+    # Inundation.py appends the huc code to the supplied output_extent_grid for stage-based.
     # Modify output_extent_grid to match inundation.py saved filename.
     # Search for this file, if it didn't create, send message to log file.
-    base_file_path, extension = os.path.splitext(output_extent_grid)
-    saved_extent_grid_filename = "{}_{}{}".format(base_file_path, huc, extension)
-    if not os.path.exists(saved_extent_grid_filename):
-        with open(log_file, 'a+') as f:
-            f.write('FAILURE_huc_{}:{}:{} map failed to create\n'.format(huc, ahps_site, magnitude))
 
+    # base_file_path, extension = os.path.splitext(output_extent_tif)
+    # saved_extent_grid_filename = "{}_{}{}".format(base_file_path, huc, extension)
+
+    # MP_LOG.trace(f"saved_extent_grid_filename is {saved_extent_grid_filename}")
 
-def post_process_huc_level(
-    job_number_tif, ahps_dir_list, huc_dir, attributes_dir, gpkg_dir, fim_version, huc
+    if not os.path.exists(output_extent_tif):
+        MP_LOG.error(f"FAILURE_huc_{huc} - {ahps_site} - {magnitude} map failed to create")
+        return
+
+    # For space reasons, we need to delete all of the intermediary files such as:
+    #    Stage: grmn3_action_extent_0.tif, grmn3_action_extent_1933000003.tif. The give aways are a number before
+    #        the .tif
+    #    Flows: allm1_action_12p0ft_extent_01010002_0.tif, allm1_action_12p0ft_extent_01010002_7170000001.tif
+    #       your give away is to just delete any file that has the HUC number in teh file name
+    # The intermediatary are all inundated branch tifs.
+
+    # The ones we want to keep stop at _extent.tif
+    # branch_tifs = glob.glob(os.path.join(output_huc_site_mapping_dir, '*_extent_*.tif'))
+    # for tif_file in branch_tifs:
+    #    os.remove(tif_file)
+
+    return
+
+
+# This is part of an MP Pool
+# TODO: Aug 2024: job_number_inundate is not used well at all and is partially
+# with more cleanup to do later. Partially removed now.
+def post_process_huc(
+    output_catfim_dir,
+    ahps_dir_list,
+    huc_dir,
+    gpkg_dir,
+    fim_version,
+    huc,
+    parent_log_output_file,
+    child_log_file_prefix,
+    progress_stmt,
 ):
-    # Loop through ahps sites
-    for ahps_lid in ahps_dir_list:
-        tifs_to_reformat_list = []
-        ahps_lid_dir = os.path.join(huc_dir, ahps_lid)
-
-        # Append desired filenames to list.
-        tif_list = os.listdir(ahps_lid_dir)
-        for tif in tif_list:
-            if 'extent.tif' in tif:
-                tifs_to_reformat_list.append(os.path.join(ahps_lid_dir, tif))
-
-        # Stage-Based CatFIM uses attributes from individual CSVs instead of the master CSV.
-        nws_lid_attributes_filename = os.path.join(attributes_dir, ahps_lid + '_attributes.csv')
-
-        print(f"Reformatting TIFs {ahps_lid} for {huc_dir}")
-        with ProcessPoolExecutor(max_workers=job_number_tif) as executor:
+
+    # Note: child_log_file_prefix is "MP_post_process_{huc}", meaning all logs created by this function start
+    #  with the phrase "MP_post_process_{huc}". This one rollups up to the master catfim log
+    # This is setting up logging for this function to go up to the parent
+    try:
+        MP_LOG.MP_Log_setup(parent_log_output_file, child_log_file_prefix)
+        MP_LOG.lprint(f'Post Processing {huc} ...')
+        MP_LOG.lprint(f'... {progress_stmt} ...')
+
+        # Loop through ahps sites
+        attributes_dir = os.path.join(output_catfim_dir, 'attributes')
+
+        for ahps_lid in ahps_dir_list:
+            tifs_to_reformat_list = []
+            mapping_huc_lid_dir = os.path.join(huc_dir, ahps_lid)
+            MP_LOG.trace(f"mapping_huc_lid_dir is {mapping_huc_lid_dir}")
+
+            # Append desired filenames to list. (notice.. no value after the word extent)
+            # tif_list = [x for x in os.listdir(mapping_huc_lid_dir) if x.endswith("extent.tif")]  # doesn't match the old filenames
+
+            # new logic actually finds the extent tifs
+            # It will find only the mag rollups and not its branches
+
+            # if stage based, the file names looks like this: masm1_major_20p0ft_extent.tif
+            #    but there also is masm1_major_extent.tif, so we want both
+            # if flow based, the file name looks like this: masm1_action_extent.tif
+
+            tif_list = [x for x in os.listdir(mapping_huc_lid_dir) if ('extent.tif') in x]
+
+            if len(tif_list) == 0:
+                MP_LOG.warning(f">> no tifs found for {huc} {ahps_lid} at {mapping_huc_lid_dir}")
+                continue
+
+            for tif in tif_list:
+                if 'extent.tif' in tif:
+                    # as we processing tifs for just one ahps at a time, we can further files that have that
+                    # ahps_lid in it.
+                    if ahps_lid in tif:
+                        tifs_to_reformat_list.append(os.path.join(mapping_huc_lid_dir, tif))
+
+            if len(tifs_to_reformat_list) == 0:
+                MP_LOG.warning(f">> no tifs found for {huc} {ahps_lid} at {mapping_huc_lid_dir}")
+                continue
+
+            # Stage-Based CatFIM uses attributes from individual CSVs instead of the master CSV.
+            nws_lid_attributes_filename = os.path.join(attributes_dir, ahps_lid + '_attributes.csv')
+
+            # There may not necessarily be an attributes.csv for this lid, depending on how flow processing went
+            # lots of lids fall out in the attributes or flow steps.
+            if os.path.exists(nws_lid_attributes_filename) is False:
+                MP_LOG.warning(f"{ahps_lid} has no attributes file which may perfectly fine.")
+                continue
+
+            # We are going to do an MP in MP.
+            # child_log_file_prefix = MP_LOG.MP_calc_prefix_name(
+            #    parent_log_output_file, "MP_reformat_tifs", huc
+            # )
+            # Weird case, we ahve to delete any of these files that might already exist (MP in MP)
+            # Get parent log dir
+            # log_dir = os.path.dirname(parent_log_output_file)
+            # old_refomat_log_files = glob.glob(os.path.join(log_dir, 'MP_reformat_tifs_*'))
+            # for log_file in old_refomat_log_files:
+            #     os.remove(log_file)
+
+            # with ProcessPoolExecutor(max_workers=job_number_inundate) as executor:
+            # TODO:
+            # Aug 2024, Using MP (job number inundate has very little value, drop it)
+            # Clean up that ji MP
+            #            with ProcessPoolExecutor(max_workers=job_number_inundate) as executor:
             for tif_to_process in tifs_to_reformat_list:
-                if not os.path.exists(tif_to_process):
-                    continue
+                # If not os.path.exists(tif_to_process):
+                #    continue
+
+                # If stage based, the file names looks like this: masm1_major_20p0ft_extent.tif
+                #    but there also is masm1_major_extent.tif, so we want both
+                # If flow based, the file name looks like this: masm1_action_extent.tif
+                MP_LOG.trace(f".. Tif to Process = {tif_to_process}")
                 try:
-                    magnitude = os.path.split(tif_to_process)[1].split('_')[1]
-                    try:
-                        interval_stage = float(
-                            (os.path.split(tif_to_process)[1].split('_')[2])
-                            .replace('p', '.')
-                            .replace("ft", "")
-                        )
-                        if interval_stage == 'extent':
+
+                    tif_file_name = os.path.basename(tif_to_process)
+                    file_name_parts = tif_file_name.split("_")
+                    magnitude = file_name_parts[1]
+
+                    if "ft" in tif_file_name:  # stage based, ie grnm1_action_11p0ft_extent.tif
+                        try:
+                            interval_stage = float(file_name_parts[2].replace('p', '.').replace("ft", ""))
+                        except ValueError:
                             interval_stage = None
-                    except ValueError:
+                            MP_LOG.error(
+                                f"Value Error for {huc} - {ahps_lid} - magnitude {magnitude}"
+                                " at {mapping_huc_lid_dir}"
+                            )
+                            MP_LOG.error(traceback.format_exc())
+                    else:  # flow based. ie) cfkt2_action_extent.tif
                         interval_stage = None
-                    executor.submit(
-                        reformat_inundation_maps,
+
+                    reformat_inundation_maps(
                         ahps_lid,
                         tif_to_process,
                         gpkg_dir,
@@ -193,89 +637,195 @@ def post_process_huc_level(
                         magnitude,
                         nws_lid_attributes_filename,
                         interval_stage,
+                        parent_log_output_file,
+                        child_log_file_prefix,
+                    )
+                except Exception:
+                    MP_LOG.error(
+                        f"An ind reformat map error occured for {huc} - {ahps_lid} - magnitude {magnitude}"
                     )
-                except Exception as ex:
-                    print(f"*** {ex}")
-                    traceback.print_exc()
+                    MP_LOG.error(traceback.format_exc())
 
+            # rolls up logs from child MP processes into this parent_log_output_file
+            # MP_LOG.merge_log_files(parent_log_output_file, child_log_file_prefix, True)
 
+    except Exception:
+        MP_LOG.error(f"An error has occurred in post processing for {huc}")
+        MP_LOG.error(traceback.format_exc())
+
+    return
+
+
+# This is not part of an MP process, but does need FLOG carried into it so it can use FLOG directly
 def post_process_cat_fim_for_viz(
-    job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file="", fim_version=""
+    catfim_method, output_catfim_dir, job_huc_ahps, fim_version, log_output_file
 ):
-    print("In post processing...")
-    # Create workspace
-    gpkg_dir = os.path.join(output_catfim_dir, 'gpkg')
-    if not os.path.exists(gpkg_dir):
-        os.mkdir(gpkg_dir)
-
-    # Find the FIM version
-    merged_layer = os.path.join(output_catfim_dir, 'catfim_library.gpkg')
-    if not os.path.exists(merged_layer):  # prevents appending to existing output
-        huc_ahps_dir_list = os.listdir(output_catfim_dir)
-        skip_list = ['errors', 'logs', 'gpkg', 'missing_files.txt', 'messages', merged_layer]
-
-        # Loop through all categories
-        print("Building list of TIFs to reformat...")
-        with ProcessPoolExecutor(max_workers=job_number_huc) as huc_exector:
-            for huc in huc_ahps_dir_list:
-                if huc in skip_list:
-                    continue
-                huc_dir = os.path.join(output_catfim_dir, huc)
-                try:
-                    ahps_dir_list = os.listdir(huc_dir)
-                except NotADirectoryError:
-                    continue
-                # If there's no mapping for a HUC, delete the HUC directory.
-                if ahps_dir_list == []:
-                    os.rmdir(huc_dir)
-                    continue
-
-                huc_exector.submit(
-                    post_process_huc_level,
-                    job_number_tif,
-                    ahps_dir_list,
-                    huc_dir,
-                    attributes_dir,
-                    gpkg_dir,
-                    fim_version,
-                    huc,
-                )
 
-        # Merge all layers
-        print(f"Merging {len(os.listdir(gpkg_dir))} layers...")
-        for layer in os.listdir(gpkg_dir):
-            # Open dissolved extent layers
-            diss_extent_filename = os.path.join(gpkg_dir, layer)
-            diss_extent = gpd.read_file(diss_extent_filename)
-            diss_extent['viz'] = 'yes'
+    # Adding a pointer in this file coming from generate_categorial_fim so they can share the same log file
+    FLOG.setup(log_output_file)
 
-            # Write/append aggregate diss_extent
-            print(f"Merging layer: {layer}")
-            if os.path.isfile(merged_layer):
-                diss_extent.to_file(merged_layer, driver=getDriver(merged_layer), index=False, mode='a')
-            else:
-                diss_extent.to_file(merged_layer, driver=getDriver(merged_layer), index=False)
-            del diss_extent
+    FLOG.lprint("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+    FLOG.lprint("Start post processing TIFs (TIF extents into poly into gpkg)...")
+    output_mapping_dir = os.path.join(output_catfim_dir, 'mapping')
+    gpkg_dir = os.path.join(output_mapping_dir, 'gpkg')
+    os.mkdir(gpkg_dir)
 
-            # shutil.rmtree(gpkg_dir)  # TODO
+    huc_ahps_dir_list = [
+        x
+        for x in os.listdir(output_mapping_dir)
+        if os.path.isdir(os.path.join(output_mapping_dir, x)) and x[0] in ['0', '1', '2']
+    ]
 
-    else:
-        print(f"{merged_layer} already exists.")
+    skip_list = ['errors', 'logs', 'gpkg', 'missing_files.txt', 'messages']
+
+    num_hucs = len(huc_ahps_dir_list)
+    huc_index = 0
+    FLOG.lprint(f"Number of hucs to post process is {num_hucs}")
+
+    child_log_file_prefix = MP_LOG.MP_calc_prefix_name(log_output_file, "MP_post_process")
+    with ProcessPoolExecutor(max_workers=job_huc_ahps) as huc_exector:
+        for huc in huc_ahps_dir_list:
+            FLOG.lprint(f"TIF post processing for {huc}")
+            if huc in skip_list:
+                continue
+
+            huc_dir = os.path.join(output_mapping_dir, huc)
+            progress_stmt = f"index {huc_index + 1} of {num_hucs}"
+            huc_index += 1
+
+            try:
+                ahps_dir_list = [x for x in os.listdir(huc_dir) if os.path.isdir(os.path.join(huc_dir, x))]
+                # ahps_dir_list = os.listdir(huc_dir)
+            except NotADirectoryError:
+                FLOG.warning(f"{huc_dir} directory missing. Continuing on")
+                continue
+
+            # If there's no mapping for a HUC, delete the HUC directory.
+            if len(ahps_dir_list) == 0:
+                os.rmdir(huc_dir)
+                FLOG.warning(f"no mapping for {huc}")
+                continue
+
+            huc_exector.submit(
+                post_process_huc,
+                output_catfim_dir,
+                ahps_dir_list,
+                huc_dir,
+                gpkg_dir,
+                fim_version,
+                huc,
+                log_output_file,
+                child_log_file_prefix,
+                progress_stmt,
+            )
+
+    # end of ProcessPoolExecutor
+
+    # rolls up logs from child MP processes into this parent_log_output_file
+    FLOG.merge_log_files(FLOG.LOG_FILE_PATH, child_log_file_prefix, True)
+
+    # Merge all layers
+    gpkg_files = [x for x in os.listdir(gpkg_dir) if x.endswith('.gpkg')]
+    FLOG.lprint(f"Merging {len(gpkg_files)} from layers in {gpkg_dir}")
+
+    # TODO: put a tqdm in here for visual only.
+
+    gpkg_files.sort()
+
+    merged_layers_gdf = None
+    ctr = 0
+    for layer in tqdm(
+        gpkg_files,
+        total=len(gpkg_files),
+        desc="Merging gpkg layers",
+        bar_format="{desc}:({n_fmt}/{total_fmt})|{bar}| {percentage:.1f}% ",
+        ncols=80,
+    ):
 
+        # for ctr, layer in enumerate(gpkg_files):
+        # FLOG.lprint(f"Merging gpkg ({ctr+1} of {len(gpkg_files)} - {}")
+        FLOG.trace(f"Merging gpkg ({ctr+1} of {len(gpkg_files)} : {layer}")
 
+        # Concatenate each /gpkg/{aphs}_{magnitude}_extent_{huc}_dissolved.gkpg
+        diss_extent_filename = os.path.join(gpkg_dir, layer)
+        diss_extent_gdf = gpd.read_file(diss_extent_filename, engine='fiona')
+        diss_extent_gdf['viz'] = 'yes'
+
+        if ctr == 0:
+            merged_layers_gdf = diss_extent_gdf
+        else:
+            merged_layers_gdf = pd.concat([merged_layers_gdf, diss_extent_gdf])
+
+        del diss_extent_gdf
+        ctr += 1
+
+    if merged_layers_gdf is None or len(merged_layers_gdf) == 0:
+        raise Exception(f"No gpkgs found in {gpkg_dir}")
+
+    # TODO: July 9, 2024: Consider deleting all of the interium .gkpg files in the gkpg folder.
+    # It will get very big quick. But not yet.
+    # shutil.rmtree(gpkg_dir)
+
+    # Now dissolve based on ahps and magnitude (we no longer saved non dissolved versrons)
+    # Aug 2024: We guessed on what might need to be dissolved from 4.4.0.0. In 4.4.0.0 there
+    # are "_dissolved" versions of catfim files but no notes on why or how, but this script
+    # did not do it. We are going to guess on what the dissolving rules are.
+    if catfim_method == "flow_based":
+        FLOG.lprint("Dissolving flow based catfim_libary by ahps and magnitudes")
+        merged_layers_gdf = merged_layers_gdf.dissolve(by=['ahps_lid', 'magnitude'], as_index=False)
+
+    merged_layers_gdf.reset_index(inplace=True)
+
+    output_file_name = f"{catfim_method}_catfim_library"
+
+    # TODO: Aug 2024: gpkg are not opening in qgis now? project, wkt, non defined geometry columns?
+    # gkpg_file_path = os.path.join(output_mapping_dir, f'{output_file_name}.gpkg')
+    # FLOG.lprint(f"Saving catfim library gpkg version to {gkpg_file_path}")
+    # merged_layers_gdf.to_file(gkpg_file_path, driver='GPKG', index=True, engine="fiona", crs=PREP_PROJECTION)
+
+    csv_file_path = os.path.join(output_mapping_dir, f'{output_file_name}.csv')
+    FLOG.lprint(f"Saving catfim library csv version to {csv_file_path}")
+    merged_layers_gdf.to_csv(csv_file_path)
+
+    FLOG.lprint("End post processing TIFs...")
+
+    return
+
+
+# This is part of an MP pool
 def reformat_inundation_maps(
     ahps_lid,
-    extent_grid,
+    tif_to_process,
     gpkg_dir,
     fim_version,
     huc,
     magnitude,
     nws_lid_attributes_filename,
-    interval_stage=None,
+    interval_stage,
+    parent_log_output_file,
+    child_log_file_prefix,
 ):
+    """_summary_
+    Turns inundated tifs into dissolved polys gpkg with more attributes
+
+    """
+    # interval stage might come in as null and that is ok
+
+    # Note: child_log_file_prefix is "MP_reformat_tifs_{huc}", meaning all logs created by this
+    # function start with the phrase "MP_reformat_tifs_{huc}". This will rollup to the master
+    # catfim logs
+
+    # This is setting up logging for this function to go up to the parent
+    MP_LOG.MP_Log_setup(parent_log_output_file, child_log_file_prefix)
+
     try:
-        # Convert raster to to shapes
-        with rasterio.open(extent_grid) as src:
+        MP_LOG.trace(
+            f"{huc} : {ahps_lid} : {magnitude} -- Start reformat_inundation_maps" " (tif extent to gpkg poly)"
+        )
+        MP_LOG.trace(F"Tif to process is {tif_to_process}")
+
+        # Convert raster to shapes
+        with rasterio.open(tif_to_process) as src:
             image = src.read(1)
             mask = image > 0
 
@@ -286,7 +836,11 @@ def reformat_inundation_maps(
         )
 
         # Convert list of shapes to polygon
-        extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)
+        # lots of polys
+        extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=PREP_PROJECTION)  # Previous code
+        # extent_poly = gpd.GeoDataFrame.from_features(list(results))  # Updated to accomodate AK projection
+        # extent_poly = extent_poly.set_crs(src.crs)  # Update to accomodate AK projection
+
         # Dissolve polygons
         extent_poly_diss = extent_poly.dissolve(by='extent')
 
@@ -297,6 +851,7 @@ def reformat_inundation_maps(
         extent_poly_diss['version'] = fim_version
         extent_poly_diss['huc'] = huc
         extent_poly_diss['interval_stage'] = interval_stage
+
         # Project to Web Mercator
         extent_poly_diss = extent_poly_diss.to_crs(VIZ_PROJECTION)
 
@@ -311,9 +866,10 @@ def reformat_inundation_maps(
             right_on=['nws_lid', 'magnitude', 'huc'],
         )
         extent_poly_diss = extent_poly_diss.drop(columns='nws_lid')
+
         # Save dissolved multipolygon
-        handle = os.path.split(extent_grid)[1].replace('.tif', '')
-        diss_extent_filename = os.path.join(gpkg_dir, f"{handle}_{huc}_dissolved.gpkg")
+        handle = os.path.split(tif_to_process)[1].replace('.tif', '')
+        diss_extent_filename = os.path.join(gpkg_dir, f"{huc}_{handle}_dissolved.gpkg")
         extent_poly_diss["geometry"] = [
             MultiPolygon([feature]) if type(feature) is Polygon else feature
             for feature in extent_poly_diss["geometry"]
@@ -323,70 +879,99 @@ def reformat_inundation_maps(
             extent_poly_diss.to_file(
                 diss_extent_filename, driver=getDriver(diss_extent_filename), index=False
             )
+            # MP_LOG.trace(
+            #    f"{huc} : {ahps_lid} : {magnitude} - Reformatted inundation map saved"
+            #    f" as {diss_extent_filename}"
+            # )
+        else:
+            MP_LOG.error(f"{huc} : {ahps_lid} : {magnitude} tif to gpkg, geodataframe is empty")
+
+    except ValueError as ve:
+        msg = f"{huc} : {ahps_lid} : {magnitude} - Reformatted inundation map"
+        if "Assigning CRS to a GeoDataFrame without a geometry column is not supported" in ve:
+            MP_LOG.warning(f"{msg} - Warning: details: {ve}")
+        else:
+            MP_LOG.error(f"{msg} - Exception")
+            MP_LOG.error(traceback.format_exc())
 
     except Exception:
-        pass
-        # Log and clean out the gdb so it's not merged in later
+        MP_LOG.error(f"{huc} : {ahps_lid} : {magnitude} - Reformatted inundation map - Exception")
+        MP_LOG.error(traceback.format_exc())
 
+    return
 
-#        try:
-#            print(e)
-##            f = open(log_file, 'a+')
-##            f.write(str(diss_extent_filename) + " - dissolve error: " + str(e))
-##            f.close()
-#        except:
-#            pass
 
+# This is not part of an MP progress and simply needs the
+# pointer of FLOG carried over here so it can use it directly.
 
+
+# TODO: Aug, 2024. We need re-evaluate job numbers, see usage of job numbers below
+# Used for Flow only
 def manage_catfim_mapping(
     fim_run_dir,
-    source_flow_dir,
+    output_flows_dir,
     output_catfim_dir,
-    attributes_dir,
+    catfim_method,
     job_number_huc,
     job_number_inundate,
-    overwrite,
     depthtif,
+    log_output_file,
+    step_number=1,
 ):
-    # Create output directory
-    if not os.path.exists(output_catfim_dir):
-        os.mkdir(output_catfim_dir)
-
-    # Create log directory
-    log_dir = os.path.join(output_catfim_dir, 'logs')
-    if not os.path.exists(log_dir):
-        os.mkdir(log_dir)
-
-    # Create error log path
-    log_file = os.path.join(log_dir, 'errors.log')
-
-    job_number_tif = job_number_inundate
-
-    print("Generating Categorical FIM")
-    generate_categorical_fim(
-        fim_run_dir,
-        source_flow_dir,
-        output_catfim_dir,
-        job_number_huc,
-        job_number_inundate,
-        depthtif,
-        log_file,
-    )
 
-    print("Aggregating Categorical FIM")
+    # Adding a pointer in this file coming from generate_categorial_fim so they can share the same log file
+    FLOG.setup(log_output_file)
+
+    FLOG.lprint('Begin mapping')
+    start = time.time()
+
+    output_mapping_dir = os.path.join(output_catfim_dir, 'mapping')
+    if not os.path.exists(output_mapping_dir):
+        os.mkdir(output_mapping_dir)
+
+    if step_number <= 1:
+        run_catfim_inundation(
+            fim_run_dir,
+            output_flows_dir,
+            output_mapping_dir,
+            job_number_huc,
+            job_number_inundate,
+            FLOG.LOG_FILE_PATH,
+        )
+    else:
+        FLOG.lprint("Skip running Inundation as Step > 1")
+
+    # FLOG.lprint("Aggregating Categorical FIM")
     # Get fim_version.
-    fim_version = (
-        os.path.basename(os.path.normpath(fim_run_dir))
-        .replace('fim_', '')
-        .replace('_ms_c', '')
-        .replace('_', '.')
-    )
+    fim_version = os.path.basename(os.path.normpath(fim_run_dir)).replace('fim_', '').replace('_', '.')
+
+    # Step 2
+    # TODO: Aug 2024, so we need to clean it up
+    # This step does not need a job_number_inundate as it can't really use it.
+    # It processes primarily hucs and ahps in multiproc
+    # for now, we will manually multiple the huc * 5 (max number of ahps types)
+    ahps_jobs = job_number_huc * 5
     post_process_cat_fim_for_viz(
-        job_number_huc, job_number_tif, output_catfim_dir, attributes_dir, log_file, fim_version
+        catfim_method, output_catfim_dir, ahps_jobs, fim_version, str(FLOG.LOG_FILE_PATH)
     )
 
+    end = time.time()
+    elapsed_time = (end - start) / 60
+    FLOG.lprint(f"Finished mapping in {str(elapsed_time).split('.')[0]} minutes")
+
+    return
+
 
 if __name__ == '__main__':
+
+    """
+    Sample Usage:
+    python3 /foss_fim/tools/generate_categorical_fim_mapping.py -r "/outputs/rob_test_catfim_huc"
+     -s "/data/catfim/rob_test/test_5_flow_based/flows" -o "/data/catfim/rob_test/test_5_flow_based"
+     -jh 1 -jn 40
+
+    """
+
     # Parse arguments
     parser = argparse.ArgumentParser(description='Categorical inundation mapping for FOSS FIM.')
     parser.add_argument(
@@ -407,19 +992,38 @@ def manage_catfim_mapping(
         default="",
     )
     parser.add_argument(
-        '-j',
-        '--number-of-jobs',
-        help='Number of processes to use. Default is 1.',
+        '-jh',
+        '--job-number-huc',
+        help='Number of processes to use for huc processing. Default is 1.',
         required=False,
         default="1",
         type=int,
     )
     parser.add_argument(
-        '-depthtif',
-        '--write-depth-tiff',
+        '-jn',
+        '--job-number-inundate',
+        help='OPTIONAL: Number of processes to use for inundating'
+        ' HUC and inundation job numbers should multiply to no more than one less than the CPU count'
+        ' of the machine. Defaults to 1.',
+        required=False,
+        default=1,
+        type=int,
+    )
+    # parser.add_argument(
+    #     '-depthtif',
+    #     '--write-depth-tiff',
+    #     help='Using this option will write depth TIFFs.',
+    #     required=False,
+    #     action='store_true',
+    # )
+
+    parser.add_argument(
+        '-step',
+        '--step_number',
         help='Using this option will write depth TIFFs.',
         required=False,
-        action='store_true',
+        default=1,
+        type=int,
     )
 
     args = vars(parser.parse_args())
@@ -427,7 +1031,14 @@ def manage_catfim_mapping(
     fim_run_dir = args['fim_run_dir']
     source_flow_dir = args['source_flow_dir']
     output_catfim_dir = args['output_catfim_dir']
-    number_of_jobs = int(args['number_of_jobs'])
-    depthtif = args['write_depth_tiff']
+    job_number_huc = int(args['job_number_huc'])
+    job_number_inundate = int(args['job_number_inundate'])
+    # depthtif = args['write_depth_tiff']
+    step_num = args['step_number']
 
-    manage_catfim_mapping(fim_run_dir, source_flow_dir, output_catfim_dir, number_of_jobs, depthtif)
+    log_dir = os.path.join(output_catfim_dir, "logs")
+    log_output_file = FLOG.calc_log_name_and_path(log_dir, "gen_cat_mapping")
+
+    manage_catfim_mapping(
+        source_flow_dir, output_catfim_dir, job_number_huc, job_number_inundate, log_output_file, step_num
+    )
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
old mode 100644
new mode 100755
diff --git a/tools/mosaic_inundation.py b/tools/mosaic_inundation.py
index 5bded6538..bc20265c7 100755
--- a/tools/mosaic_inundation.py
+++ b/tools/mosaic_inundation.py
@@ -55,7 +55,7 @@ def Mosaic_inundation(
 
     inundation_maps_df = inundation_maps_df.set_index(unit_attribute_name, drop=True)
 
-    # decide upon whether to display
+    # decide upon whether to display the progress bar
     if verbose & len(aggregation_units) == 1:
         tqdm_disable = False
     elif verbose:
diff --git a/tools/tools_shared_functions.py b/tools/tools_shared_functions.py
index 66c0a969c..2c9da7369 100755
--- a/tools/tools_shared_functions.py
+++ b/tools/tools_shared_functions.py
@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 
+import datetime as dt
 import json
+import logging
 import os
 import pathlib
+import traceback
 from pathlib import Path
 
 import geopandas as gpd
@@ -13,6 +16,7 @@
 import rasterio.shutil
 import requests
 import rioxarray as rxr
+import urllib3
 import xarray as xr
 from dotenv import load_dotenv
 from geocube.api.core import make_geocube
@@ -63,9 +67,16 @@ def filter_nwm_segments_by_stream_order(unfiltered_segments, desired_order, nwm_
     filtered_segments = []
 
     for feature_id in unfiltered_segments:
-        stream_order = nwm_flows_df.loc[nwm_flows_df['ID'] == int(feature_id), 'order_'].values[0]
+
+        try:
+            stream_order = nwm_flows_df.loc[nwm_flows_df['ID'] == int(feature_id), 'order_'].values[0]
+        except Exception as e:
+            print(f'WARNING: Exception occurred during filter_nwm_segments_by_stream_order():{e}')
+
         if stream_order == desired_order:
             filtered_segments.append(feature_id)
+        # else:
+        #     print(f'Stream order for {feature_id} did not match desired stream order...')
 
     return filtered_segments
 
@@ -477,9 +488,7 @@ def get_stats_table_from_binary_rasters(
         # Write legend text file
         legend_txt = os.path.join(os.path.split(agreement_raster)[0], 'read_me.txt')
 
-        from datetime import datetime
-
-        now = datetime.now()
+        now = dt.datetime.now()
         current_time = now.strftime("%m/%d/%Y %H:%M:%S")
 
         with open(legend_txt, 'w') as f:
@@ -594,7 +603,7 @@ def get_metadata(
     metadata_url : STR
         metadata base URL.
     select_by : STR
-        Location search option.
+        Location search option. Options include: 'state', TODO: add options
     selector : LIST
         Value to match location data against. Supplied as a LIST.
     must_include : STR, optional
@@ -660,7 +669,7 @@ def get_metadata(
             metadata.update(crosswalk_info)
         metadata_dataframe = pd.json_normalize(metadata_list)
         # Replace all periods with underscores in column names
-        metadata_dataframe.columns = metadata_dataframe.columns.str.replace('.', '_')
+        metadata_dataframe.columns = metadata_dataframe.columns.astype(str).str.replace('.', '_')
     else:
         # if request was not succesful, print error message.
         print(f'Code: {response.status_code}\nMessage: {response.reason}\nURL: {response.url}')
@@ -673,7 +682,7 @@ def get_metadata(
 ########################################################################
 # Function to assign HUC code using the WBD spatial layer using a spatial join
 ########################################################################
-def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes=False):
+def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes=False, huc_list=list()):
     '''
     Assigns the proper FIM HUC 08 code to each site in the input DataFrame.
     Converts input DataFrame to a GeoDataFrame using lat/lon attributes
@@ -703,9 +712,16 @@ def aggregate_wbd_hucs(metadata_list, wbd_huc8_path, retain_attributes=False):
     '''
     # Import huc8 layer as geodataframe and retain necessary columns
     print("Reading WBD...")
-    huc8 = gpd.read_file(wbd_huc8_path, layer='WBDHU8')
+    huc8_all = gpd.read_file(wbd_huc8_path, layer='WBDHU8')
     print("WBD read.")
-    huc8 = huc8[['HUC8', 'name', 'states', 'geometry']]
+    huc8 = huc8_all[['HUC8', 'name', 'states', 'geometry']]
+
+    if len(huc_list) > 0:
+        # filter by hucs we are using
+        huc8 = huc8[huc8['HUC8'].isin(huc_list)]
+
+    huc8.sort_values(by='HUC8', ascending=True, inplace=True)
+
     # Define EPSG codes for possible latlon datum names (default of NAD83 if unassigned)
     crs_lookup = {'NAD27': 'EPSG:4267', 'NAD83': 'EPSG:4269', 'WGS84': 'EPSG:4326'}
     # Create empty geodataframe and define CRS for potential horizontal datums
@@ -940,6 +956,9 @@ def get_thresholds(threshold_url, select_by, selector, threshold='all'):
 
     # Call the API
     session = requests.Session()
+
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
     retry = Retry(connect=3, backoff_factor=0.5)
     adapter = HTTPAdapter(max_retries=retry)
     session.mount('http://', adapter)

From be2f8b88569d8ea7767f95d0db5fb95673dee38e Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Wed, 25 Sep 2024 16:12:07 -0400
Subject: [PATCH 350/359] v4.8.9.0 Change Docker base image and stop Whitebox
 downloads (#1291)

---
 Dockerfile                              |   37 +-
 Pipfile                                 |    6 +-
 Pipfile.lock                            | 1584 +++++++++++++++--------
 docs/CHANGELOG.md                       |   23 +
 fim_pre_processing.sh                   |    2 +-
 src/agreedem.py                         |    1 +
 src/unique_pixel_and_allocation.py      |    1 +
 tools/aggregate_mannings_calibration.py |    3 +-
 tools/eval_plots.py                     |    1 -
 tools/generate_nws_lid.py               |    1 -
 tools/synthesize_test_cases.py          |    2 -
 11 files changed, 1063 insertions(+), 598 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a15575710..8eb84d07a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 ## Temporary image to build the libraries and only save the needed artifacts
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.4 AS builder
 WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
@@ -10,7 +10,7 @@ ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
 # remove reference to missing repo
-RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+# RUN rm /etc/apt/sources.list.d/apache-arrow.sources
 
 RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/*
 
@@ -45,7 +45,7 @@ RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d
 
 ###############################################################################################
 # Base Image that has GDAL, PROJ, etc
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.4
 ARG dataDir=/data
 ENV projectDir=/foss_fim
 ARG depDir=/dependencies
@@ -70,20 +70,12 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 # remove reference to missing repo
-RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+# RUN rm /etc/apt/sources.list.d/apache-arrow.sources
 
-RUN apt-get update --fix-missing && apt-get install -y openjdk-21-jdk && rm -rf /var/lib/apt/lists/*
+RUN apt-get update --fix-missing && rm -rf /var/lib/apt/lists/*
 RUN apt update --fix-missing
 
-# An older version of openjdk still exists on the file system but was never cleaned up
-# After research, we realized, it just needs file cleanup. Leaving it there is triggering security warnings
-# RUN apt-get remove -y openjdk-17-jdk  (not installed, just residue left)
-RUN rm -rf ./usr/lib/jvm/*java-1.17* && \
-    rm -rf ./usr/lib/jvm/.java-1.17* && \
-    rm -rdf ./usr/lib/jvm/java-17*
-
-
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata wget
 
 RUN apt auto-remove
 
@@ -99,18 +91,15 @@ ENV PYTHONUNBUFFERED=TRUE
 
 ## ADD TO PATHS ##
 ENV PATH="$projectDir:${PATH}"
-#ENV PATH=${PATH}:$projectDir:$projectDir/$srcDir:$projectDir/tools
-# Jul 17, 2024: Even though PYTHONPATH isn't used, it still seems to want it.
-ENV PYTHONPATH=${PATH}:$srcDir:$projectDir/tools
+ENV PYTHONPATH="${PYTHONPATH}:${projectDir}:${srcDir}:${srcDir}/utils:${projectDir}/data:${toolsDir}"
 
 ## install python 3 modules ##
-
 COPY Pipfile .
 COPY Pipfile.lock .
 RUN pip3 install pipenv==2024.0.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
 
 # ----------------------------------
-# Mar 2023
+# Mar 2023 / Sep 2024
 # There are some nuances in the whitebox python downloads in that the first time it loads
 # it goes to the internet and downloads the latest/greatest WBT (whiteboxtools) engine which is
 # required for the whitebox python library to work. We don't want to have FIM attempting a download
@@ -120,12 +109,10 @@ RUN pip3 install pipenv==2024.0.1 && PIP_NO_CACHE_DIR=off pipenv install --syste
 # We download and unzip it to the same file folder that pip deployed the whitebox library.
 # Whitebox also attempts to always download a folder called testdata regardless of use.
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
-
-# RUN wbox_path=/usr/local/lib/python3.10/dist-packages/whitebox/WBT && \
-#     wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
-#     unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
-#     cp $wbox_path/whitebox_tools $wbox_path && \
-#     mkdir $wbox_path/testdata
+ENV WBT_PATH=/usr/local/lib/python3.10/dist-packages/whitebox/WBT
+RUN wget -P $WBT_PATH https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
+    unzip -o $WBT_PATH/WhiteboxTools_linux_musl.zip -d $WBT_PATH && \
+    cp $WBT_PATH/WhiteboxTools_linux_amd64/WBT/whitebox_tools $WBT_PATH
 # ----------------------------------
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
diff --git a/Pipfile b/Pipfile
index dd0495222..fa068694d 100644
--- a/Pipfile
+++ b/Pipfile
@@ -13,7 +13,7 @@ certifi = "==2024.7.4"
 dask = "==2024.6.2"
 dask-expr = "==1.1.6"
 distributed = "==2024.6.2"
-fiona = "==1.9.6"
+fiona = "==1.10.0"
 flake8 = "==6.0.0"
 flake8-pyproject = "==1.2.3"
 geopandas = "==1.0.1"
@@ -21,7 +21,7 @@ gval = "==0.2.7.post1"
 ipympl = "==0.9.3"
 isort = "==5.12.0"
 jupyter = "==1.0.0"
-jupyterlab = "==3.6.7"
+jupyterlab = "==3.6.8"
 lmoments3 = "==1.0.6"
 monaco = "==0.13.1"
 natsort = "==8.3.1"
@@ -50,7 +50,7 @@ seaborn = "==0.12.2"
 tables = "==3.8.0"
 tqdm = "==4.66.3"
 urllib3 = "==1.26.19"
-whitebox = "==2.3.4"
+whitebox = "==2.3.5"
 xarray = "==2024.6.0"
 zarr = "==2.18.0"
 
diff --git a/Pipfile.lock b/Pipfile.lock
index 23f15d493..8a570af5e 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "69e8fc55d7092cbd650b00f89e5644b3ef68a4f2ef1a00ec62d5f8645222dae0"
+            "sha256": "5b0ad67893736f5ae7afa8fe3b9f7740f0c53fbda0b0369e4d614609e77b2528"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -24,6 +24,14 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.4.0"
         },
+        "aiobotocore": {
+            "hashes": [
+                "sha256:6d0b4a51d70bc33b1b4eba411076b0cc979aecbdad8e084bab202202423c0725",
+                "sha256:988eef33fd9dd4b070959cfec922278e84166950695b2160bd581623cb6a420c"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.15.0"
+        },
         "aiofiles": {
             "hashes": [
                 "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad",
@@ -32,6 +40,127 @@
             "markers": "python_version >= '3.7' and python_version < '4.0'",
             "version": "==22.1.0"
         },
+        "aiohappyeyeballs": {
+            "hashes": [
+                "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2",
+                "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.4.0"
+        },
+        "aiohttp": {
+            "hashes": [
+                "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277",
+                "sha256:03f2645adbe17f274444953bdea69f8327e9d278d961d85657cb0d06864814c1",
+                "sha256:074d1bff0163e107e97bd48cad9f928fa5a3eb4b9d33366137ffce08a63e37fe",
+                "sha256:0912b8a8fadeb32ff67a3ed44249448c20148397c1ed905d5dac185b4ca547bb",
+                "sha256:0d277cfb304118079e7044aad0b76685d30ecb86f83a0711fc5fb257ffe832ca",
+                "sha256:0d93400c18596b7dc4794d48a63fb361b01a0d8eb39f28800dc900c8fbdaca91",
+                "sha256:123dd5b16b75b2962d0fff566effb7a065e33cd4538c1692fb31c3bda2bfb972",
+                "sha256:17e997105bd1a260850272bfb50e2a328e029c941c2708170d9d978d5a30ad9a",
+                "sha256:18a01eba2574fb9edd5f6e5fb25f66e6ce061da5dab5db75e13fe1558142e0a3",
+                "sha256:1923a5c44061bffd5eebeef58cecf68096e35003907d8201a4d0d6f6e387ccaa",
+                "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77",
+                "sha256:1b2c16a919d936ca87a3c5f0e43af12a89a3ce7ccbce59a2d6784caba945b68b",
+                "sha256:1c19de68896747a2aa6257ae4cf6ef59d73917a36a35ee9d0a6f48cff0f94db8",
+                "sha256:1e72589da4c90337837fdfe2026ae1952c0f4a6e793adbbfbdd40efed7c63599",
+                "sha256:22c0a23a3b3138a6bf76fc553789cb1a703836da86b0f306b6f0dc1617398abc",
+                "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf",
+                "sha256:2d21ac12dc943c68135ff858c3a989f2194a709e6e10b4c8977d7fcd67dfd511",
+                "sha256:2f1f1c75c395991ce9c94d3e4aa96e5c59c8356a15b1c9231e783865e2772699",
+                "sha256:305be5ff2081fa1d283a76113b8df7a14c10d75602a38d9f012935df20731487",
+                "sha256:33e6bc4bab477c772a541f76cd91e11ccb6d2efa2b8d7d7883591dfb523e5987",
+                "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff",
+                "sha256:380f926b51b92d02a34119d072f178d80bbda334d1a7e10fa22d467a66e494db",
+                "sha256:38172a70005252b6893088c0f5e8a47d173df7cc2b2bd88650957eb84fcf5022",
+                "sha256:391cc3a9c1527e424c6865e087897e766a917f15dddb360174a70467572ac6ce",
+                "sha256:3a1c32a19ee6bbde02f1cb189e13a71b321256cc1d431196a9f824050b160d5a",
+                "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5",
+                "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7",
+                "sha256:44b324a6b8376a23e6ba25d368726ee3bc281e6ab306db80b5819999c737d820",
+                "sha256:4790f0e15f00058f7599dab2b206d3049d7ac464dc2e5eae0e93fa18aee9e7bf",
+                "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e",
+                "sha256:4b38b1570242fbab8d86a84128fb5b5234a2f70c2e32f3070143a6d94bc854cf",
+                "sha256:4d46c7b4173415d8e583045fbc4daa48b40e31b19ce595b8d92cf639396c15d5",
+                "sha256:4f1c9866ccf48a6df2b06823e6ae80573529f2af3a0992ec4fe75b1a510df8a6",
+                "sha256:4f7acae3cf1a2a2361ec4c8e787eaaa86a94171d2417aae53c0cca6ca3118ff6",
+                "sha256:54d9ddea424cd19d3ff6128601a4a4d23d54a421f9b4c0fff740505813739a91",
+                "sha256:58718e181c56a3c02d25b09d4115eb02aafe1a732ce5714ab70326d9776457c3",
+                "sha256:5ede29d91a40ba22ac1b922ef510aab871652f6c88ef60b9dcdf773c6d32ad7a",
+                "sha256:61645818edd40cc6f455b851277a21bf420ce347baa0b86eaa41d51ef58ba23d",
+                "sha256:66bf9234e08fe561dccd62083bf67400bdbf1c67ba9efdc3dac03650e97c6088",
+                "sha256:673f988370f5954df96cc31fd99c7312a3af0a97f09e407399f61583f30da9bc",
+                "sha256:676f94c5480d8eefd97c0c7e3953315e4d8c2b71f3b49539beb2aa676c58272f",
+                "sha256:6c225286f2b13bab5987425558baa5cbdb2bc925b2998038fa028245ef421e75",
+                "sha256:7384d0b87d4635ec38db9263e6a3f1eb609e2e06087f0aa7f63b76833737b471",
+                "sha256:7e2fe37ac654032db1f3499fe56e77190282534810e2a8e833141a021faaab0e",
+                "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697",
+                "sha256:7f6b639c36734eaa80a6c152a238242bedcee9b953f23bb887e9102976343092",
+                "sha256:814375093edae5f1cb31e3407997cf3eacefb9010f96df10d64829362ae2df69",
+                "sha256:8224f98be68a84b19f48e0bdc14224b5a71339aff3a27df69989fa47d01296f3",
+                "sha256:898715cf566ec2869d5cb4d5fb4be408964704c46c96b4be267442d265390f32",
+                "sha256:8989f46f3d7ef79585e98fa991e6ded55d2f48ae56d2c9fa5e491a6e4effb589",
+                "sha256:8ba01ebc6175e1e6b7275c907a3a36be48a2d487549b656aa90c8a910d9f3178",
+                "sha256:8c5c6fa16412b35999320f5c9690c0f554392dc222c04e559217e0f9ae244b92",
+                "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2",
+                "sha256:8eaf44ccbc4e35762683078b72bf293f476561d8b68ec8a64f98cf32811c323e",
+                "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058",
+                "sha256:9093a81e18c45227eebe4c16124ebf3e0d893830c6aca7cc310bfca8fe59d857",
+                "sha256:94c4381ffba9cc508b37d2e536b418d5ea9cfdc2848b9a7fea6aebad4ec6aac1",
+                "sha256:94fac7c6e77ccb1ca91e9eb4cb0ac0270b9fb9b289738654120ba8cebb1189c6",
+                "sha256:95c4dc6f61d610bc0ee1edc6f29d993f10febfe5b76bb470b486d90bbece6b22",
+                "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0",
+                "sha256:ad146dae5977c4dd435eb31373b3fe9b0b1bf26858c6fc452bf6af394067e10b",
+                "sha256:afe16a84498441d05e9189a15900640a2d2b5e76cf4efe8cbb088ab4f112ee57",
+                "sha256:b1c43eb1ab7cbf411b8e387dc169acb31f0ca0d8c09ba63f9eac67829585b44f",
+                "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e",
+                "sha256:b98e698dc34966e5976e10bbca6d26d6724e6bdea853c7c10162a3235aba6e16",
+                "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1",
+                "sha256:c31ad0c0c507894e3eaa843415841995bf8de4d6b2d24c6e33099f4bc9fc0d4f",
+                "sha256:c3b9162bab7e42f21243effc822652dc5bb5e8ff42a4eb62fe7782bcbcdfacf6",
+                "sha256:c58c6837a2c2a7cf3133983e64173aec11f9c2cd8e87ec2fdc16ce727bcf1a04",
+                "sha256:c83f7a107abb89a227d6c454c613e7606c12a42b9a4ca9c5d7dad25d47c776ae",
+                "sha256:cde98f323d6bf161041e7627a5fd763f9fd829bcfcd089804a5fdce7bb6e1b7d",
+                "sha256:ce91db90dbf37bb6fa0997f26574107e1b9d5ff939315247b7e615baa8ec313b",
+                "sha256:d00f3c5e0d764a5c9aa5a62d99728c56d455310bcc288a79cab10157b3af426f",
+                "sha256:d17920f18e6ee090bdd3d0bfffd769d9f2cb4c8ffde3eb203777a3895c128862",
+                "sha256:d55f011da0a843c3d3df2c2cf4e537b8070a419f891c930245f05d329c4b0689",
+                "sha256:d742c36ed44f2798c8d3f4bc511f479b9ceef2b93f348671184139e7d708042c",
+                "sha256:d9a487ef090aea982d748b1b0d74fe7c3950b109df967630a20584f9a99c0683",
+                "sha256:d9ef084e3dc690ad50137cc05831c52b6ca428096e6deb3c43e95827f531d5ef",
+                "sha256:da452c2c322e9ce0cfef392e469a26d63d42860f829026a63374fde6b5c5876f",
+                "sha256:dc4826823121783dccc0871e3f405417ac116055bf184ac04c36f98b75aacd12",
+                "sha256:de7a5299827253023c55ea549444e058c0eb496931fa05d693b95140a947cb73",
+                "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061",
+                "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072",
+                "sha256:ee40b40aa753d844162dcc80d0fe256b87cba48ca0054f64e68000453caead11",
+                "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691",
+                "sha256:f29930bc2921cef955ba39a3ff87d2c4398a0394ae217f41cb02d5c26c8b1b77",
+                "sha256:f489a2c9e6455d87eabf907ac0b7d230a9786be43fbe884ad184ddf9e9c1e385",
+                "sha256:f5bf3ead3cb66ab990ee2561373b009db5bc0e857549b6c9ba84b20bc462e172",
+                "sha256:f6f18898ace4bcd2d41a122916475344a87f1dfdec626ecde9ee802a711bc569",
+                "sha256:f8112fb501b1e0567a1251a2fd0747baae60a4ab325a871e975b7bb67e59221f",
+                "sha256:fd31f176429cecbc1ba499d4aba31aaccfea488f418d60376b911269d3b883c5"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.10.5"
+        },
+        "aioitertools": {
+            "hashes": [
+                "sha256:c2a9055b4fbb7705f561b9d86053e8af5d10cc845d22c32008c43490b2d8dd6b",
+                "sha256:fc1f5fac3d737354de8831cbba3eb04f79dd649d8f3afb4c5b114925e662a796"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.12.0"
+        },
+        "aiosignal": {
+            "hashes": [
+                "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc",
+                "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.1"
+        },
         "aiosqlite": {
             "hashes": [
                 "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6",
@@ -42,11 +171,11 @@
         },
         "anyio": {
             "hashes": [
-                "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94",
-                "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"
+                "sha256:c5a275fe5ca0afd788001f58fca1e69e29ce706d746e317d660e21f70c530ef9",
+                "sha256:fdeb095b7cc5a5563175eedd926ec4ae55413bb4be5770c424af0ba46ccb4a78"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.4.0"
+            "version": "==4.5.0"
         },
         "appdirs": {
             "hashes": [
@@ -111,6 +240,14 @@
             ],
             "version": "==2.4.1"
         },
+        "async-timeout": {
+            "hashes": [
+                "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f",
+                "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==4.0.3"
+        },
         "attrs": {
             "hashes": [
                 "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346",
@@ -210,11 +347,11 @@
         },
         "botocore": {
             "hashes": [
-                "sha256:bce42967d0f03b79cf25b2b6a36221fb2fb15f98e6fa4155b66b672ab192013b",
-                "sha256:e599cef6305e950a212f85adb86c1f4713d4b2678b3c2ec54df6b8e2dbe9ef2f"
+                "sha256:1b48c94e8a4bbe23143f3d1c21a32b9ffc7476b651ef42371ab45d678f6dbfbc",
+                "sha256:3564a980d95ff2861a6ca74313173d8778aa659125c63cf49c93ad23896c63b1"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.35.1"
+            "version": "==1.35.16"
         },
         "bottleneck": {
             "hashes": [
@@ -371,76 +508,76 @@
         },
         "cffi": {
             "hashes": [
-                "sha256:011aff3524d578a9412c8b3cfaa50f2c0bd78e03eb7af7aa5e0df59b158efb2f",
-                "sha256:0a048d4f6630113e54bb4b77e315e1ba32a5a31512c31a273807d0027a7e69ab",
-                "sha256:0bb15e7acf8ab35ca8b24b90af52c8b391690ef5c4aec3d31f38f0d37d2cc499",
-                "sha256:0d46ee4764b88b91f16661a8befc6bfb24806d885e27436fdc292ed7e6f6d058",
-                "sha256:0e60821d312f99d3e1569202518dddf10ae547e799d75aef3bca3a2d9e8ee693",
-                "sha256:0fdacad9e0d9fc23e519efd5ea24a70348305e8d7d85ecbb1a5fa66dc834e7fb",
-                "sha256:14b9cbc8f7ac98a739558eb86fabc283d4d564dafed50216e7f7ee62d0d25377",
-                "sha256:17c6d6d3260c7f2d94f657e6872591fe8733872a86ed1345bda872cfc8c74885",
-                "sha256:1a2ddbac59dc3716bc79f27906c010406155031a1c801410f1bafff17ea304d2",
-                "sha256:2404f3de742f47cb62d023f0ba7c5a916c9c653d5b368cc966382ae4e57da401",
-                "sha256:24658baf6224d8f280e827f0a50c46ad819ec8ba380a42448e24459daf809cf4",
-                "sha256:24aa705a5f5bd3a8bcfa4d123f03413de5d86e497435693b638cbffb7d5d8a1b",
-                "sha256:2770bb0d5e3cc0e31e7318db06efcbcdb7b31bcb1a70086d3177692a02256f59",
-                "sha256:331ad15c39c9fe9186ceaf87203a9ecf5ae0ba2538c9e898e3a6967e8ad3db6f",
-                "sha256:3aa9d43b02a0c681f0bfbc12d476d47b2b2b6a3f9287f11ee42989a268a1833c",
-                "sha256:41f4915e09218744d8bae14759f983e466ab69b178de38066f7579892ff2a555",
-                "sha256:4304d4416ff032ed50ad6bb87416d802e67139e31c0bde4628f36a47a3164bfa",
-                "sha256:435a22d00ec7d7ea533db494da8581b05977f9c37338c80bc86314bec2619424",
-                "sha256:45f7cd36186db767d803b1473b3c659d57a23b5fa491ad83c6d40f2af58e4dbb",
-                "sha256:48b389b1fd5144603d61d752afd7167dfd205973a43151ae5045b35793232aa2",
-                "sha256:4e67d26532bfd8b7f7c05d5a766d6f437b362c1bf203a3a5ce3593a645e870b8",
-                "sha256:516a405f174fd3b88829eabfe4bb296ac602d6a0f68e0d64d5ac9456194a5b7e",
-                "sha256:5ba5c243f4004c750836f81606a9fcb7841f8874ad8f3bf204ff5e56332b72b9",
-                "sha256:5bdc0f1f610d067c70aa3737ed06e2726fd9d6f7bfee4a351f4c40b6831f4e82",
-                "sha256:6107e445faf057c118d5050560695e46d272e5301feffda3c41849641222a828",
-                "sha256:6327b572f5770293fc062a7ec04160e89741e8552bf1c358d1a23eba68166759",
-                "sha256:669b29a9eca6146465cc574659058ed949748f0809a2582d1f1a324eb91054dc",
-                "sha256:6ce01337d23884b21c03869d2f68c5523d43174d4fc405490eb0091057943118",
-                "sha256:6d872186c1617d143969defeadac5a904e6e374183e07977eedef9c07c8953bf",
-                "sha256:6f76a90c345796c01d85e6332e81cab6d70de83b829cf1d9762d0a3da59c7932",
-                "sha256:70d2aa9fb00cf52034feac4b913181a6e10356019b18ef89bc7c12a283bf5f5a",
-                "sha256:7cbc78dc018596315d4e7841c8c3a7ae31cc4d638c9b627f87d52e8abaaf2d29",
-                "sha256:856bf0924d24e7f93b8aee12a3a1095c34085600aa805693fb7f5d1962393206",
-                "sha256:8a98748ed1a1df4ee1d6f927e151ed6c1a09d5ec21684de879c7ea6aa96f58f2",
-                "sha256:93a7350f6706b31f457c1457d3a3259ff9071a66f312ae64dc024f049055f72c",
-                "sha256:964823b2fc77b55355999ade496c54dde161c621cb1f6eac61dc30ed1b63cd4c",
-                "sha256:a003ac9edc22d99ae1286b0875c460351f4e101f8c9d9d2576e78d7e048f64e0",
-                "sha256:a0ce71725cacc9ebf839630772b07eeec220cbb5f03be1399e0457a1464f8e1a",
-                "sha256:a47eef975d2b8b721775a0fa286f50eab535b9d56c70a6e62842134cf7841195",
-                "sha256:a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6",
-                "sha256:a9015f5b8af1bb6837a3fcb0cdf3b874fe3385ff6274e8b7925d81ccaec3c5c9",
-                "sha256:aec510255ce690d240f7cb23d7114f6b351c733a74c279a84def763660a2c3bc",
-                "sha256:b00e7bcd71caa0282cbe3c90966f738e2db91e64092a877c3ff7f19a1628fdcb",
-                "sha256:b50aaac7d05c2c26dfd50c3321199f019ba76bb650e346a6ef3616306eed67b0",
-                "sha256:b7b6ea9e36d32582cda3465f54c4b454f62f23cb083ebc7a94e2ca6ef011c3a7",
-                "sha256:bb9333f58fc3a2296fb1d54576138d4cf5d496a2cc118422bd77835e6ae0b9cb",
-                "sha256:c1c13185b90bbd3f8b5963cd8ce7ad4ff441924c31e23c975cb150e27c2bf67a",
-                "sha256:c3b8bd3133cd50f6b637bb4322822c94c5ce4bf0d724ed5ae70afce62187c492",
-                "sha256:c5d97162c196ce54af6700949ddf9409e9833ef1003b4741c2b39ef46f1d9720",
-                "sha256:c815270206f983309915a6844fe994b2fa47e5d05c4c4cef267c3b30e34dbe42",
-                "sha256:cab2eba3830bf4f6d91e2d6718e0e1c14a2f5ad1af68a89d24ace0c6b17cced7",
-                "sha256:d1df34588123fcc88c872f5acb6f74ae59e9d182a2707097f9e28275ec26a12d",
-                "sha256:d6bdcd415ba87846fd317bee0774e412e8792832e7805938987e4ede1d13046d",
-                "sha256:db9a30ec064129d605d0f1aedc93e00894b9334ec74ba9c6bdd08147434b33eb",
-                "sha256:dbc183e7bef690c9abe5ea67b7b60fdbca81aa8da43468287dae7b5c046107d4",
-                "sha256:dca802c8db0720ce1c49cce1149ff7b06e91ba15fa84b1d59144fef1a1bc7ac2",
-                "sha256:dec6b307ce928e8e112a6bb9921a1cb00a0e14979bf28b98e084a4b8a742bd9b",
-                "sha256:df8bb0010fdd0a743b7542589223a2816bdde4d94bb5ad67884348fa2c1c67e8",
-                "sha256:e4094c7b464cf0a858e75cd14b03509e84789abf7b79f8537e6a72152109c76e",
-                "sha256:e4760a68cab57bfaa628938e9c2971137e05ce48e762a9cb53b76c9b569f1204",
-                "sha256:eb09b82377233b902d4c3fbeeb7ad731cdab579c6c6fda1f763cd779139e47c3",
-                "sha256:eb862356ee9391dc5a0b3cbc00f416b48c1b9a52d252d898e5b7696a5f9fe150",
-                "sha256:ef9528915df81b8f4c7612b19b8628214c65c9b7f74db2e34a646a0a2a0da2d4",
-                "sha256:f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76",
-                "sha256:f3e0992f23bbb0be00a921eae5363329253c3b86287db27092461c887b791e5e",
-                "sha256:f9338cc05451f1942d0d8203ec2c346c830f8e86469903d5126c1f0a13a2bcbb",
-                "sha256:ffef8fd58a36fb5f1196919638f73dd3ae0db1a878982b27a9a5a176ede4ba91"
+                "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8",
+                "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2",
+                "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1",
+                "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15",
+                "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36",
+                "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824",
+                "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8",
+                "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36",
+                "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17",
+                "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf",
+                "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc",
+                "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3",
+                "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed",
+                "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702",
+                "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1",
+                "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8",
+                "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903",
+                "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6",
+                "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d",
+                "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b",
+                "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e",
+                "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be",
+                "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c",
+                "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683",
+                "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9",
+                "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c",
+                "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8",
+                "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1",
+                "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4",
+                "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655",
+                "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67",
+                "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595",
+                "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0",
+                "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65",
+                "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41",
+                "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6",
+                "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401",
+                "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6",
+                "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3",
+                "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16",
+                "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93",
+                "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e",
+                "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4",
+                "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964",
+                "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c",
+                "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576",
+                "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0",
+                "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3",
+                "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662",
+                "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3",
+                "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff",
+                "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5",
+                "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd",
+                "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f",
+                "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5",
+                "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14",
+                "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d",
+                "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9",
+                "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7",
+                "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382",
+                "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a",
+                "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e",
+                "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a",
+                "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4",
+                "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99",
+                "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87",
+                "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.17.0"
+            "version": "==1.17.1"
         },
         "cfgv": {
             "hashes": [
@@ -631,53 +768,74 @@
         },
         "contourpy": {
             "hashes": [
-                "sha256:00e5388f71c1a0610e6fe56b5c44ab7ba14165cdd6d695429c5cd94021e390b2",
-                "sha256:10a37ae557aabf2509c79715cd20b62e4c7c28b8cd62dd7d99e5ed3ce28c3fd9",
-                "sha256:11959f0ce4a6f7b76ec578576a0b61a28bdc0696194b6347ba3f1c53827178b9",
-                "sha256:187fa1d4c6acc06adb0fae5544c59898ad781409e61a926ac7e84b8f276dcef4",
-                "sha256:1a07fc092a4088ee952ddae19a2b2a85757b923217b7eed584fdf25f53a6e7ce",
-                "sha256:1cac0a8f71a041aa587410424ad46dfa6a11f6149ceb219ce7dd48f6b02b87a7",
-                "sha256:1d59e739ab0e3520e62a26c60707cc3ab0365d2f8fecea74bfe4de72dc56388f",
-                "sha256:2855c8b0b55958265e8b5888d6a615ba02883b225f2227461aa9127c578a4922",
-                "sha256:2e785e0f2ef0d567099b9ff92cbfb958d71c2d5b9259981cd9bee81bd194c9a4",
-                "sha256:309be79c0a354afff9ff7da4aaed7c3257e77edf6c1b448a779329431ee79d7e",
-                "sha256:39f3ecaf76cd98e802f094e0d4fbc6dc9c45a8d0c4d185f0f6c2234e14e5f75b",
-                "sha256:457499c79fa84593f22454bbd27670227874cd2ff5d6c84e60575c8b50a69619",
-                "sha256:49e70d111fee47284d9dd867c9bb9a7058a3c617274900780c43e38d90fe1205",
-                "sha256:4c75507d0a55378240f781599c30e7776674dbaf883a46d1c90f37e563453480",
-                "sha256:4c863140fafc615c14a4bf4efd0f4425c02230eb8ef02784c9a156461e62c965",
-                "sha256:4d8908b3bee1c889e547867ca4cdc54e5ab6be6d3e078556814a22457f49423c",
-                "sha256:5b9eb0ca724a241683c9685a484da9d35c872fd42756574a7cfbf58af26677fd",
-                "sha256:6022cecf8f44e36af10bd9118ca71f371078b4c168b6e0fab43d4a889985dbb5",
-                "sha256:6150ffa5c767bc6332df27157d95442c379b7dce3a38dff89c0f39b63275696f",
-                "sha256:62828cada4a2b850dbef89c81f5a33741898b305db244904de418cc957ff05dc",
-                "sha256:7b4182299f251060996af5249c286bae9361fa8c6a9cda5efc29fe8bfd6062ec",
-                "sha256:94b34f32646ca0414237168d68a9157cb3889f06b096612afdd296003fdd32fd",
-                "sha256:9ce6889abac9a42afd07a562c2d6d4b2b7134f83f18571d859b25624a331c90b",
-                "sha256:9cffe0f850e89d7c0012a1fb8730f75edd4320a0a731ed0c183904fe6ecfc3a9",
-                "sha256:a12a813949e5066148712a0626895c26b2578874e4cc63160bb007e6df3436fe",
-                "sha256:a1eea9aecf761c661d096d39ed9026574de8adb2ae1c5bd7b33558af884fb2ce",
-                "sha256:a31f94983fecbac95e58388210427d68cd30fe8a36927980fab9c20062645609",
-                "sha256:ac58bdee53cbeba2ecad824fa8159493f0bf3b8ea4e93feb06c9a465d6c87da8",
-                "sha256:af3f4485884750dddd9c25cb7e3915d83c2db92488b38ccb77dd594eac84c4a0",
-                "sha256:b33d2bc4f69caedcd0a275329eb2198f560b325605810895627be5d4b876bf7f",
-                "sha256:b59c0ffceff8d4d3996a45f2bb6f4c207f94684a96bf3d9728dbb77428dd8cb8",
-                "sha256:bb6834cbd983b19f06908b45bfc2dad6ac9479ae04abe923a275b5f48f1a186b",
-                "sha256:bd3db01f59fdcbce5b22afad19e390260d6d0222f35a1023d9adc5690a889364",
-                "sha256:bd7c23df857d488f418439686d3b10ae2fbf9bc256cd045b37a8c16575ea1040",
-                "sha256:c2528d60e398c7c4c799d56f907664673a807635b857df18f7ae64d3e6ce2d9f",
-                "sha256:d31a63bc6e6d87f77d71e1abbd7387ab817a66733734883d1fc0021ed9bfa083",
-                "sha256:d4492d82b3bc7fbb7e3610747b159869468079fe149ec5c4d771fa1f614a14df",
-                "sha256:ddcb8581510311e13421b1f544403c16e901c4e8f09083c881fab2be80ee31ba",
-                "sha256:e1d59258c3c67c865435d8fbeb35f8c59b8bef3d6f46c1f29f6123556af28445",
-                "sha256:eb3315a8a236ee19b6df481fc5f997436e8ade24a9f03dfdc6bd490fea20c6da",
-                "sha256:ef2b055471c0eb466033760a521efb9d8a32b99ab907fc8358481a1dd29e3bd3",
-                "sha256:ef5adb9a3b1d0c645ff694f9bca7702ec2c70f4d734f9922ea34de02294fdf72",
-                "sha256:f32c38afb74bd98ce26de7cc74a67b40afb7b05aae7b42924ea990d51e4dac02",
-                "sha256:fe0ccca550bb8e5abc22f530ec0466136379c01321fd94f30a22231e8a48d985"
+                "sha256:00ccd0dbaad6d804ab259820fa7cb0b8036bda0686ef844d24125d8287178ce0",
+                "sha256:0be4d8425bfa755e0fd76ee1e019636ccc7c29f77a7c86b4328a9eb6a26d0639",
+                "sha256:0dce35502151b6bd35027ac39ba6e5a44be13a68f55735c3612c568cac3805fd",
+                "sha256:0fa4c02abe6c446ba70d96ece336e621efa4aecae43eaa9b030ae5fb92b309ad",
+                "sha256:14e262f67bd7e6eb6880bc564dcda30b15e351a594657e55b7eec94b6ef72843",
+                "sha256:167d6c890815e1dac9536dca00828b445d5d0df4d6a8c6adb4a7ec3166812fa8",
+                "sha256:1ec4dc6bf570f5b22ed0d7efba0dfa9c5b9e0431aeea7581aa217542d9e809a4",
+                "sha256:303c252947ab4b14c08afeb52375b26781ccd6a5ccd81abcdfc1fafd14cf93c1",
+                "sha256:31cd3a85dbdf1fc002280c65caa7e2b5f65e4a973fcdf70dd2fdcb9868069294",
+                "sha256:32b238b3b3b649e09ce9aaf51f0c261d38644bdfa35cbaf7b263457850957a84",
+                "sha256:33c92cdae89ec5135d036e7218e69b0bb2851206077251f04a6c4e0e21f03927",
+                "sha256:345af746d7766821d05d72cb8f3845dfd08dd137101a2cb9b24de277d716def8",
+                "sha256:3634b5385c6716c258d0419c46d05c8aa7dc8cb70326c9a4fb66b69ad2b52e09",
+                "sha256:364174c2a76057feef647c802652f00953b575723062560498dc7930fc9b1cb7",
+                "sha256:36e0cff201bcb17a0a8ecc7f454fe078437fa6bda730e695a92f2d9932bd507f",
+                "sha256:36f965570cff02b874773c49bfe85562b47030805d7d8360748f3eca570f4cab",
+                "sha256:3bb3808858a9dc68f6f03d319acd5f1b8a337e6cdda197f02f4b8ff67ad2057b",
+                "sha256:3e1c7fa44aaae40a2247e2e8e0627f4bea3dd257014764aa644f319a5f8600e3",
+                "sha256:3faeb2998e4fcb256542e8a926d08da08977f7f5e62cf733f3c211c2a5586223",
+                "sha256:420d39daa61aab1221567b42eecb01112908b2cab7f1b4106a52caaec8d36973",
+                "sha256:4553c421929ec95fb07b3aaca0fae668b2eb5a5203d1217ca7c34c063c53d087",
+                "sha256:4865cd1d419e0c7a7bf6de1777b185eebdc51470800a9f42b9e9decf17762081",
+                "sha256:4cfb5c62ce023dfc410d6059c936dcf96442ba40814aefbfa575425a3a7f19dc",
+                "sha256:4d63ee447261e963af02642ffcb864e5a2ee4cbfd78080657a9880b8b1868e18",
+                "sha256:570ef7cf892f0afbe5b2ee410c507ce12e15a5fa91017a0009f79f7d93a1268f",
+                "sha256:637f674226be46f6ba372fd29d9523dd977a291f66ab2a74fbeb5530bb3f445d",
+                "sha256:68a32389b06b82c2fdd68276148d7b9275b5f5cf13e5417e4252f6d1a34f72a2",
+                "sha256:69375194457ad0fad3a839b9e29aa0b0ed53bb54db1bfb6c3ae43d111c31ce41",
+                "sha256:6cb6cc968059db9c62cb35fbf70248f40994dfcd7aa10444bbf8b3faeb7c2d67",
+                "sha256:710a26b3dc80c0e4febf04555de66f5fd17e9cf7170a7b08000601a10570bda6",
+                "sha256:732896af21716b29ab3e988d4ce14bc5133733b85956316fb0c56355f398099b",
+                "sha256:75ee7cb1a14c617f34a51d11fa7524173e56551646828353c4af859c56b766e2",
+                "sha256:76a896b2f195b57db25d6b44e7e03f221d32fe318d03ede41f8b4d9ba1bff53c",
+                "sha256:76c905ef940a4474a6289c71d53122a4f77766eef23c03cd57016ce19d0f7b42",
+                "sha256:7a52040312b1a858b5e31ef28c2e865376a386c60c0e248370bbea2d3f3b760d",
+                "sha256:7ffa0db17717a8ffb127efd0c95a4362d996b892c2904db72428d5b52e1938a4",
+                "sha256:81cb5ed4952aae6014bc9d0421dec7c5835c9c8c31cdf51910b708f548cf58e5",
+                "sha256:834e0cfe17ba12f79963861e0f908556b2cedd52e1f75e6578801febcc6a9f49",
+                "sha256:87ddffef1dbe5e669b5c2440b643d3fdd8622a348fe1983fad7a0f0ccb1cd67b",
+                "sha256:880ea32e5c774634f9fcd46504bf9f080a41ad855f4fef54f5380f5133d343c7",
+                "sha256:8ca947601224119117f7c19c9cdf6b3ab54c5726ef1d906aa4a69dfb6dd58102",
+                "sha256:90f73a5116ad1ba7174341ef3ea5c3150ddf20b024b98fb0c3b29034752c8aeb",
+                "sha256:92f8557cbb07415a4d6fa191f20fd9d2d9eb9c0b61d1b2f52a8926e43c6e9af7",
+                "sha256:94e848a6b83da10898cbf1311a815f770acc9b6a3f2d646f330d57eb4e87592e",
+                "sha256:9c0da700bf58f6e0b65312d0a5e695179a71d0163957fa381bb3c1f72972537c",
+                "sha256:a11077e395f67ffc2c44ec2418cfebed032cd6da3022a94fc227b6faf8e2acb8",
+                "sha256:aea348f053c645100612b333adc5983d87be69acdc6d77d3169c090d3b01dc35",
+                "sha256:b11b39aea6be6764f84360fce6c82211a9db32a7c7de8fa6dd5397cf1d079c3b",
+                "sha256:c6c7c2408b7048082932cf4e641fa3b8ca848259212f51c8c59c45aa7ac18f14",
+                "sha256:c6ec93afeb848a0845a18989da3beca3eec2c0f852322efe21af1931147d12cb",
+                "sha256:cacd81e2d4b6f89c9f8a5b69b86490152ff39afc58a95af002a398273e5ce589",
+                "sha256:d402880b84df3bec6eab53cd0cf802cae6a2ef9537e70cf75e91618a3801c20c",
+                "sha256:d51fca85f9f7ad0b65b4b9fe800406d0d77017d7270d31ec3fb1cc07358fdea0",
+                "sha256:d73f659398a0904e125280836ae6f88ba9b178b2fed6884f3b1f95b989d2c8da",
+                "sha256:d78ab28a03c854a873787a0a42254a0ccb3cb133c672f645c9f9c8f3ae9d0800",
+                "sha256:da84c537cb8b97d153e9fb208c221c45605f73147bd4cadd23bdae915042aad6",
+                "sha256:dbc4c3217eee163fa3984fd1567632b48d6dfd29216da3ded3d7b844a8014a66",
+                "sha256:e12968fdfd5bb45ffdf6192a590bd8ddd3ba9e58360b29683c6bb71a7b41edca",
+                "sha256:e1fd23e9d01591bab45546c089ae89d926917a66dceb3abcf01f6105d927e2cb",
+                "sha256:e8134301d7e204c88ed7ab50028ba06c683000040ede1d617298611f9dc6240c",
+                "sha256:eb8b141bb00fa977d9122636b16aa67d37fd40a3d8b52dd837e536d64b9a4d06",
+                "sha256:eca7e17a65f72a5133bdbec9ecf22401c62bcf4821361ef7811faee695799779",
+                "sha256:f317576606de89da6b7e0861cf6061f6146ead3528acabff9236458a6ba467f8",
+                "sha256:fd2a0fc506eccaaa7595b7e1418951f213cf8255be2600f1ea1b61e46a60c55f",
+                "sha256:fe41b41505a5a33aeaed2a613dccaeaa74e0e3ead6dd6fd3a118fb471644fd6c"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.2.1"
+            "version": "==1.3.0"
         },
         "cycler": {
             "hashes": [
@@ -866,11 +1024,11 @@
         },
         "executing": {
             "hashes": [
-                "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147",
-                "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"
+                "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf",
+                "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.0.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.1.0"
         },
         "fasteners": {
             "hashes": [
@@ -889,42 +1047,43 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb",
-                "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"
+                "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0",
+                "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.15.4"
+            "version": "==3.16.1"
         },
         "fiona": {
             "hashes": [
-                "sha256:0eaffbf3bfae9960484c0c08ea461b0c40e111497f04e9475ebf15ac7a22d9dc",
-                "sha256:11af1afc1255642a7787fe112c29d01f968f1053e4d4700fc6f3bb879c1622e0",
-                "sha256:1dcd6eca7524535baf2a39d7981b4a46d33ae28c313934a7c3eae62eecf9dfa5",
-                "sha256:39819fb8f5ec6d9971cb01b912b4431615a3d3f50c83798565d8ce41917930db",
-                "sha256:42d8a0e5570948d3821c493b6141866d9a4d7a64edad2be4ecbb89f81904baac",
-                "sha256:48b6218224e96de5e36b5eb259f37160092260e5de0dcd82ca200b1887aa9884",
-                "sha256:50037c3b7a5f6f434b562b5b1a5b664f1caa7a4383b00af23cdb59bfc6ba852c",
-                "sha256:52e8fec650b72fc5253d8f86b63859acc687182281c29bfacd3930496cf982d1",
-                "sha256:53bedd2989e255df1bf3378ae9c06d6d241ec273c280c544bb44ffffebb97fb0",
-                "sha256:63e528b5ea3d8b1038d788e7c65117835c787ba7fdc94b1b42f09c2cbc0aaff2",
-                "sha256:77653a08564a44e634c44cd74a068d2f55d1d4029edd16d1c8aadcc4d8cc1d2c",
-                "sha256:791b3494f8b218c06ea56f892bd6ba893dfa23525347761d066fb7738acda3b1",
-                "sha256:89095c2d542325ee45894b8837e8048cdbb2f22274934e1be3b673ca628010d7",
-                "sha256:918bd27d8625416672e834593970f96dff63215108f81efb876fe5c0bc58a3b4",
-                "sha256:98cea6f435843b2119731c6b0470e5b7386aa16b6aa7edabbf1ed93aefe029c3",
-                "sha256:9b53034efdf93ada9295b081e6a8280af7c75496a20df82d4c2ca46d65b85905",
-                "sha256:bf51846ad602757bf27876f458c5c9f14b09421fac612f64273cc4e3fcabc441",
-                "sha256:c1dd5fbc29b7303bb87eb683455e8451e1a53bb8faf20ef97fdcd843c9e4a7f6",
-                "sha256:c9b92aa1badb2773e7cac19bef3064d73e9d80c67c42f0928db2520a04be6f2f",
-                "sha256:e313210b30d09ed8f829bf625599e248dadd78622728030221f6526580ff26c5",
-                "sha256:e5404ed08c711489abcb3a50a184816825b8af06eb73ad2a99e18b8e7b47c96a",
-                "sha256:e7617563b36d2be99f048f0d0054b4d765f4aae454398f88f19de9c2c324b7f8",
-                "sha256:f1b49d51a744874608b689f029766aa1e078dd72e94b44cf8eeef6d7bd2e9051",
-                "sha256:f4230eccbd896a79d1ebfa551d84bf90f512f7bcbe1ca61e3f82231321f1a532"
+                "sha256:065328fcc61caeca7c5d4b0b1b741213e422997ea66eaa04c2e207e1609eb0e7",
+                "sha256:16072f9313f60220310951561c5df29122d078d26316455dc58516c330881d35",
+                "sha256:1a7f28c5667d5cb05891fd00ed8ca2d3067bb58c90ca928a2741c0da74daf10d",
+                "sha256:2854fca6478ca3c1dc3f06531025550046f405488909a5be11ea97e4c55894db",
+                "sha256:2f8c6e7d2e5e2dc3b72b520cd160ca6d3689748b20d35b518eec9073f5a3c5bf",
+                "sha256:34575acb68b41009b28237121c728015aebb2ab3969cf46e57d9b9e5510ece61",
+                "sha256:3529fd46d269ff3f70aeb9316a93ae95cf2f87d7e148a8ff0d68532bf81ff7ae",
+                "sha256:382cffc905d44a933951857ae07f79cf59567db08f4201ed2a28f9f9bcc86932",
+                "sha256:5f1db6fc3f9212fdc817628981731e0adcdce88b0afd30531986c732a75f6a98",
+                "sha256:80929d220a92b50592b5ce96233d631c61a3c9b9bebd26ea136e7e48ffb50616",
+                "sha256:82514e8cc113245f2c7deda1c66f0aba67b183e20404365bbb7347e12b35c80f",
+                "sha256:8ad1ea80555bc56047d4b4e60c9424cf66875f161b3e56ebd60fa0574dc6693f",
+                "sha256:8c1a555b2546d3c69d0826a526a5c3e50a1cd330c95e2e0ebf6e82fa81fc2d96",
+                "sha256:a273cc957e86289609b47155f75d60c2cc37974a92618e016f75bc5ab9cc2dec",
+                "sha256:b2a42fb0a7168003015ab9648712f7d7eb971237af3c9be6000df69cd4b97ba7",
+                "sha256:bf11de868ab601b8d917dc38719b39bf8bdeaadc4fda9964d96b4b354e528fff",
+                "sha256:bfc9b76345a58ddcf9aae8697eb444728ff7a2cde7d5925f3031ff25e9eaa339",
+                "sha256:c0e72c4ed6ce7e2b2531a62844f3216528c149b55b89ff2881bcfe34d3b37300",
+                "sha256:cc68efb479a04be9a8e9c381940493c22a47f6aec5bbec22d0bb57a9cb240296",
+                "sha256:d099b1b5ab99db1d144ac47f657e14471178537d878ebd016c37afdf37303399",
+                "sha256:d95a93d6d37e04232b141d9f08190b30e7dc368126a34342d764b9d0a857d770",
+                "sha256:d99d2faa8cb3590dc0018a2832af39e1930b39cfff53017f3681d505bf813fee",
+                "sha256:ef27d75046e57b3e55c4b5d4e1bf7e43ef490e73ada5e86f9b148aa23a2e7fd4",
+                "sha256:ef44b3b0704244503a5f12ae92ca88b2cf4fedc9fd881b00c7c58b6899ca5133",
+                "sha256:f335beb927c5e9d9997a9cf665d6eee2e24191a02582f01786afa76119b71110"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==1.9.6"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.10.0"
         },
         "flake8": {
             "hashes": [
@@ -1006,13 +1165,96 @@
             ],
             "version": "==1.5.1"
         },
+        "frozenlist": {
+            "hashes": [
+                "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7",
+                "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98",
+                "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad",
+                "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5",
+                "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae",
+                "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e",
+                "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a",
+                "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701",
+                "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d",
+                "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6",
+                "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6",
+                "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106",
+                "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75",
+                "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868",
+                "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a",
+                "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0",
+                "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1",
+                "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826",
+                "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec",
+                "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6",
+                "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950",
+                "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19",
+                "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0",
+                "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8",
+                "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a",
+                "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09",
+                "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86",
+                "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c",
+                "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5",
+                "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b",
+                "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b",
+                "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d",
+                "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0",
+                "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea",
+                "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776",
+                "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a",
+                "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897",
+                "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7",
+                "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09",
+                "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9",
+                "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe",
+                "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd",
+                "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742",
+                "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09",
+                "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0",
+                "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932",
+                "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1",
+                "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a",
+                "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49",
+                "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d",
+                "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7",
+                "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480",
+                "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89",
+                "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e",
+                "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b",
+                "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82",
+                "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb",
+                "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068",
+                "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8",
+                "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b",
+                "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb",
+                "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2",
+                "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11",
+                "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b",
+                "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc",
+                "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0",
+                "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497",
+                "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17",
+                "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0",
+                "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2",
+                "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439",
+                "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5",
+                "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac",
+                "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825",
+                "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887",
+                "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced",
+                "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.1"
+        },
         "fsspec": {
             "hashes": [
-                "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e",
-                "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"
+                "sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0",
+                "sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.6.1"
+            "version": "==2023.12.1"
         },
         "geocube": {
             "hashes": [
@@ -1058,27 +1300,27 @@
         },
         "identify": {
             "hashes": [
-                "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf",
-                "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"
+                "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0",
+                "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.6.0"
+            "version": "==2.6.1"
         },
         "idna": {
             "hashes": [
-                "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
-                "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
+                "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9",
+                "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==3.7"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.10"
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:42817a4a0be5845d22c6e212db66a94ad261e2318d80b3e0d363894a79df2b67",
-                "sha256:9c8fa6e8ea0f9516ad5c8db9246a731c948193c7754d3babb0114a05b27dd364"
+                "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b",
+                "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"
             ],
             "markers": "python_version < '3.12'",
-            "version": "==8.3.0"
+            "version": "==8.5.0"
         },
         "inflate64": {
             "hashes": [
@@ -1157,11 +1399,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:1cec0fbba8404af13facebe83d04436a7434c7400e59f47acf467c64abd0956c",
-                "sha256:e6b347c27bdf9c32ee9d31ae85defc525755a1869f14057e900675b9e8d6e6ff"
+                "sha256:0b99a2dc9f15fd68692e898e5568725c6d49c527d36a9fb5960ffbdeaa82ff7e",
+                "sha256:f68b3cb8bde357a5d7adc9598d57e22a45dfbea19eb6b98286fa3b288c9cd55c"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.26.0"
+            "version": "==8.27.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -1172,11 +1414,11 @@
         },
         "ipywidgets": {
             "hashes": [
-                "sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2",
-                "sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c"
+                "sha256:3290f526f87ae6e77655555baba4f36681c555b8bdbbff430b70e52c34c86245",
+                "sha256:870e43b1a35656a80c18c9503bbf2d16802db1cb487eec6fab27d683381dde17"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.3"
+            "version": "==8.1.5"
         },
         "isoduration": {
             "hashes": [
@@ -1311,11 +1553,11 @@
         },
         "jupyter-server-fileid": {
             "hashes": [
-                "sha256:76a2fbcea6950968485dcd509c2d6ac417ca11e61ab1ad447a475f0878ca808f",
-                "sha256:ffb11460ca5f8567644f6120b25613fca8e3f3048b38d14c6e3fe1902f314a9b"
+                "sha256:521608bb87f606a8637fcbdce2f3d24a8b3cc89d2eef61751cb40e468d4e54be",
+                "sha256:f73c01c19f90005d3fff93607b91b4955ba4e1dccdde9bfe8026646f94053791"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.9.2"
+            "version": "==0.9.3"
         },
         "jupyter-server-terminals": {
             "hashes": [
@@ -1343,12 +1585,12 @@
         },
         "jupyterlab": {
             "hashes": [
-                "sha256:2fadeaec161b0d1aec19f17721d8b803aef1d267f89c8b636b703be14f435c8f",
-                "sha256:d92d57d402f53922bca5090654843aa08e511290dff29fdb0809eafbbeb6df98"
+                "sha256:891284e75158998e23eb7a23ecc4caaf27b365e41adca374109b1305b9f769db",
+                "sha256:a2477383e23f20009188bd9dac7e6e38dbc54307bc36d716bea6ced450647c97"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==3.6.7"
+            "version": "==3.6.8"
         },
         "jupyterlab-pygments": {
             "hashes": [
@@ -1368,121 +1610,131 @@
         },
         "jupyterlab-widgets": {
             "hashes": [
-                "sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0",
-                "sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27"
+                "sha256:a2966d385328c1942b683a8cd96b89b8dd82c8b8f81dda902bb2bc06d46f5bed",
+                "sha256:e3cda2c233ce144192f1e29914ad522b2f4c40e77214b0cc97377ca3d323db54"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.0.11"
+            "version": "==3.0.13"
         },
         "kiwisolver": {
             "hashes": [
-                "sha256:00bd361b903dc4bbf4eb165f24d1acbee754fce22ded24c3d56eec268658a5cf",
-                "sha256:040c1aebeda72197ef477a906782b5ab0d387642e93bda547336b8957c61022e",
-                "sha256:05703cf211d585109fcd72207a31bb170a0f22144d68298dc5e61b3c946518af",
-                "sha256:06f54715b7737c2fecdbf140d1afb11a33d59508a47bf11bb38ecf21dc9ab79f",
-                "sha256:0dc9db8e79f0036e8173c466d21ef18e1befc02de8bf8aa8dc0813a6dc8a7046",
-                "sha256:0f114aa76dc1b8f636d077979c0ac22e7cd8f3493abbab152f20eb8d3cda71f3",
-                "sha256:11863aa14a51fd6ec28688d76f1735f8f69ab1fabf388851a595d0721af042f5",
-                "sha256:11c7de8f692fc99816e8ac50d1d1aef4f75126eefc33ac79aac02c099fd3db71",
-                "sha256:11d011a7574eb3b82bcc9c1a1d35c1d7075677fdd15de527d91b46bd35e935ee",
-                "sha256:146d14bebb7f1dc4d5fbf74f8a6cb15ac42baadee8912eb84ac0b3b2a3dc6ac3",
-                "sha256:15568384086b6df3c65353820a4473575dbad192e35010f622c6ce3eebd57af9",
-                "sha256:19df6e621f6d8b4b9c4d45f40a66839294ff2bb235e64d2178f7522d9170ac5b",
-                "sha256:1b04139c4236a0f3aff534479b58f6f849a8b351e1314826c2d230849ed48985",
-                "sha256:210ef2c3a1f03272649aff1ef992df2e724748918c4bc2d5a90352849eb40bea",
-                "sha256:2270953c0d8cdab5d422bee7d2007f043473f9d2999631c86a223c9db56cbd16",
-                "sha256:2400873bccc260b6ae184b2b8a4fec0e4082d30648eadb7c3d9a13405d861e89",
-                "sha256:2a40773c71d7ccdd3798f6489aaac9eee213d566850a9533f8d26332d626b82c",
-                "sha256:2c5674c4e74d939b9d91dda0fae10597ac7521768fec9e399c70a1f27e2ea2d9",
-                "sha256:3195782b26fc03aa9c6913d5bad5aeb864bdc372924c093b0f1cebad603dd712",
-                "sha256:31a82d498054cac9f6d0b53d02bb85811185bcb477d4b60144f915f3b3126342",
-                "sha256:32d5cf40c4f7c7b3ca500f8985eb3fb3a7dfc023215e876f207956b5ea26632a",
-                "sha256:346f5343b9e3f00b8db8ba359350eb124b98c99efd0b408728ac6ebf38173958",
-                "sha256:378a214a1e3bbf5ac4a8708304318b4f890da88c9e6a07699c4ae7174c09a68d",
-                "sha256:39b42c68602539407884cf70d6a480a469b93b81b7701378ba5e2328660c847a",
-                "sha256:3a2b053a0ab7a3960c98725cfb0bf5b48ba82f64ec95fe06f1d06c99b552e130",
-                "sha256:3aba7311af82e335dd1e36ffff68aaca609ca6290c2cb6d821a39aa075d8e3ff",
-                "sha256:3cd32d6c13807e5c66a7cbb79f90b553642f296ae4518a60d8d76243b0ad2898",
-                "sha256:3edd2fa14e68c9be82c5b16689e8d63d89fe927e56debd6e1dbce7a26a17f81b",
-                "sha256:4c380469bd3f970ef677bf2bcba2b6b0b4d5c75e7a020fb863ef75084efad66f",
-                "sha256:4e66e81a5779b65ac21764c295087de82235597a2293d18d943f8e9e32746265",
-                "sha256:53abb58632235cd154176ced1ae8f0d29a6657aa1aa9decf50b899b755bc2b93",
-                "sha256:5794cf59533bc3f1b1c821f7206a3617999db9fbefc345360aafe2e067514929",
-                "sha256:59415f46a37f7f2efeec758353dd2eae1b07640d8ca0f0c42548ec4125492635",
-                "sha256:59ec7b7c7e1a61061850d53aaf8e93db63dce0c936db1fda2658b70e4a1be709",
-                "sha256:59edc41b24031bc25108e210c0def6f6c2191210492a972d585a06ff246bb79b",
-                "sha256:5a580c91d686376f0f7c295357595c5a026e6cbc3d77b7c36e290201e7c11ecb",
-                "sha256:5b94529f9b2591b7af5f3e0e730a4e0a41ea174af35a4fd067775f9bdfeee01a",
-                "sha256:5c7b3b3a728dc6faf3fc372ef24f21d1e3cee2ac3e9596691d746e5a536de920",
-                "sha256:5c90ae8c8d32e472be041e76f9d2f2dbff4d0b0be8bd4041770eddb18cf49a4e",
-                "sha256:5e7139af55d1688f8b960ee9ad5adafc4ac17c1c473fe07133ac092310d76544",
-                "sha256:5ff5cf3571589b6d13bfbfd6bcd7a3f659e42f96b5fd1c4830c4cf21d4f5ef45",
-                "sha256:620ced262a86244e2be10a676b646f29c34537d0d9cc8eb26c08f53d98013390",
-                "sha256:6512cb89e334e4700febbffaaa52761b65b4f5a3cf33f960213d5656cea36a77",
-                "sha256:6c08e1312a9cf1074d17b17728d3dfce2a5125b2d791527f33ffbe805200a355",
-                "sha256:6c3bd3cde54cafb87d74d8db50b909705c62b17c2099b8f2e25b461882e544ff",
-                "sha256:6ef7afcd2d281494c0a9101d5c571970708ad911d028137cd558f02b851c08b4",
-                "sha256:7269d9e5f1084a653d575c7ec012ff57f0c042258bf5db0954bf551c158466e7",
-                "sha256:72d40b33e834371fd330fb1472ca19d9b8327acb79a5821d4008391db8e29f20",
-                "sha256:74d1b44c6cfc897df648cc9fdaa09bc3e7679926e6f96df05775d4fb3946571c",
-                "sha256:74db36e14a7d1ce0986fa104f7d5637aea5c82ca6326ed0ec5694280942d1162",
-                "sha256:763773d53f07244148ccac5b084da5adb90bfaee39c197554f01b286cf869228",
-                "sha256:76c6a5964640638cdeaa0c359382e5703e9293030fe730018ca06bc2010c4437",
-                "sha256:76d9289ed3f7501012e05abb8358bbb129149dbd173f1f57a1bf1c22d19ab7cc",
-                "sha256:7931d8f1f67c4be9ba1dd9c451fb0eeca1a25b89e4d3f89e828fe12a519b782a",
-                "sha256:7b8b454bac16428b22560d0a1cf0a09875339cab69df61d7805bf48919415901",
-                "sha256:7e5bab140c309cb3a6ce373a9e71eb7e4873c70c2dda01df6820474f9889d6d4",
-                "sha256:83d78376d0d4fd884e2c114d0621624b73d2aba4e2788182d286309ebdeed770",
-                "sha256:852542f9481f4a62dbb5dd99e8ab7aedfeb8fb6342349a181d4036877410f525",
-                "sha256:85267bd1aa8880a9c88a8cb71e18d3d64d2751a790e6ca6c27b8ccc724bcd5ad",
-                "sha256:88a2df29d4724b9237fc0c6eaf2a1adae0cdc0b3e9f4d8e7dc54b16812d2d81a",
-                "sha256:88b9f257ca61b838b6f8094a62418421f87ac2a1069f7e896c36a7d86b5d4c29",
-                "sha256:8ab3919a9997ab7ef2fbbed0cc99bb28d3c13e6d4b1ad36e97e482558a91be90",
-                "sha256:92dea1ffe3714fa8eb6a314d2b3c773208d865a0e0d35e713ec54eea08a66250",
-                "sha256:9407b6a5f0d675e8a827ad8742e1d6b49d9c1a1da5d952a67d50ef5f4170b18d",
-                "sha256:9408acf3270c4b6baad483865191e3e582b638b1654a007c62e3efe96f09a9a3",
-                "sha256:955e8513d07a283056b1396e9a57ceddbd272d9252c14f154d450d227606eb54",
-                "sha256:9db8ea4c388fdb0f780fe91346fd438657ea602d58348753d9fb265ce1bca67f",
-                "sha256:9eaa8b117dc8337728e834b9c6e2611f10c79e38f65157c4c38e9400286f5cb1",
-                "sha256:a51a263952b1429e429ff236d2f5a21c5125437861baeed77f5e1cc2d2c7c6da",
-                "sha256:a6aa6315319a052b4ee378aa171959c898a6183f15c1e541821c5c59beaa0238",
-                "sha256:aa12042de0171fad672b6c59df69106d20d5596e4f87b5e8f76df757a7c399aa",
-                "sha256:aaf7be1207676ac608a50cd08f102f6742dbfc70e8d60c4db1c6897f62f71523",
-                "sha256:b0157420efcb803e71d1b28e2c287518b8808b7cf1ab8af36718fd0a2c453eb0",
-                "sha256:b3f7e75f3015df442238cca659f8baa5f42ce2a8582727981cbfa15fee0ee205",
-                "sha256:b9098e0049e88c6a24ff64545cdfc50807818ba6c1b739cae221bbbcbc58aad3",
-                "sha256:ba55dce0a9b8ff59495ddd050a0225d58bd0983d09f87cfe2b6aec4f2c1234e4",
-                "sha256:bb86433b1cfe686da83ce32a9d3a8dd308e85c76b60896d58f082136f10bffac",
-                "sha256:bbea0db94288e29afcc4c28afbf3a7ccaf2d7e027489c449cf7e8f83c6346eb9",
-                "sha256:bbf1d63eef84b2e8c89011b7f2235b1e0bf7dacc11cac9431fc6468e99ac77fb",
-                "sha256:c7940c1dc63eb37a67721b10d703247552416f719c4188c54e04334321351ced",
-                "sha256:c9bf3325c47b11b2e51bca0824ea217c7cd84491d8ac4eefd1e409705ef092bd",
-                "sha256:cdc8a402aaee9a798b50d8b827d7ecf75edc5fb35ea0f91f213ff927c15f4ff0",
-                "sha256:ceec1a6bc6cab1d6ff5d06592a91a692f90ec7505d6463a88a52cc0eb58545da",
-                "sha256:cfe6ab8da05c01ba6fbea630377b5da2cd9bcbc6338510116b01c1bc939a2c18",
-                "sha256:d099e745a512f7e3bbe7249ca835f4d357c586d78d79ae8f1dcd4d8adeb9bda9",
-                "sha256:d0ef46024e6a3d79c01ff13801cb19d0cad7fd859b15037aec74315540acc276",
-                "sha256:d2e5a98f0ec99beb3c10e13b387f8db39106d53993f498b295f0c914328b1333",
-                "sha256:da4cfb373035def307905d05041c1d06d8936452fe89d464743ae7fb8371078b",
-                "sha256:da802a19d6e15dffe4b0c24b38b3af68e6c1a68e6e1d8f30148c83864f3881db",
-                "sha256:dced8146011d2bc2e883f9bd68618b8247387f4bbec46d7392b3c3b032640126",
-                "sha256:dfdd7c0b105af050eb3d64997809dc21da247cf44e63dc73ff0fd20b96be55a9",
-                "sha256:e368f200bbc2e4f905b8e71eb38b3c04333bddaa6a2464a6355487b02bb7fb09",
-                "sha256:e391b1f0a8a5a10ab3b9bb6afcfd74f2175f24f8975fb87ecae700d1503cdee0",
-                "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec",
-                "sha256:e5d706eba36b4c4d5bc6c6377bb6568098765e990cfc21ee16d13963fab7b3e7",
-                "sha256:ec20916e7b4cbfb1f12380e46486ec4bcbaa91a9c448b97023fde0d5bbf9e4ff",
-                "sha256:f1d072c2eb0ad60d4c183f3fb44ac6f73fb7a8f16a2694a91f988275cbf352f9",
-                "sha256:f846c260f483d1fd217fe5ed7c173fb109efa6b1fc8381c8b7552c5781756192",
-                "sha256:f91de7223d4c7b793867797bacd1ee53bfe7359bd70d27b7b58a04efbb9436c8",
-                "sha256:faae4860798c31530dd184046a900e652c95513796ef51a12bc086710c2eec4d",
-                "sha256:fc579bf0f502e54926519451b920e875f433aceb4624a3646b3252b5caa9e0b6",
-                "sha256:fcc700eadbbccbf6bc1bcb9dbe0786b4b1cb91ca0dcda336eef5c2beed37b797",
-                "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892",
-                "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f"
+                "sha256:073a36c8273647592ea332e816e75ef8da5c303236ec0167196793eb1e34657a",
+                "sha256:08471d4d86cbaec61f86b217dd938a83d85e03785f51121e791a6e6689a3be95",
+                "sha256:0c18ec74c0472de033e1bebb2911c3c310eef5649133dd0bedf2a169a1b269e5",
+                "sha256:0c6c43471bc764fad4bc99c5c2d6d16a676b1abf844ca7c8702bdae92df01ee0",
+                "sha256:10849fb2c1ecbfae45a693c070e0320a91b35dd4bcf58172c023b994283a124d",
+                "sha256:18077b53dc3bb490e330669a99920c5e6a496889ae8c63b58fbc57c3d7f33a18",
+                "sha256:18e0cca3e008e17fe9b164b55735a325140a5a35faad8de92dd80265cd5eb80b",
+                "sha256:22f499f6157236c19f4bbbd472fa55b063db77a16cd74d49afe28992dff8c258",
+                "sha256:2a8781ac3edc42ea4b90bc23e7d37b665d89423818e26eb6df90698aa2287c95",
+                "sha256:2e6039dcbe79a8e0f044f1c39db1986a1b8071051efba3ee4d74f5b365f5226e",
+                "sha256:34ea1de54beef1c104422d210c47c7d2a4999bdecf42c7b5718fbe59a4cac383",
+                "sha256:3ab58c12a2cd0fc769089e6d38466c46d7f76aced0a1f54c77652446733d2d02",
+                "sha256:3abc5b19d24af4b77d1598a585b8a719beb8569a71568b66f4ebe1fb0449460b",
+                "sha256:3bf1ed55088f214ba6427484c59553123fdd9b218a42bbc8c6496d6754b1e523",
+                "sha256:3ce6b2b0231bda412463e152fc18335ba32faf4e8c23a754ad50ffa70e4091ee",
+                "sha256:3da53da805b71e41053dc670f9a820d1157aae77b6b944e08024d17bcd51ef88",
+                "sha256:3f9362ecfca44c863569d3d3c033dbe8ba452ff8eed6f6b5806382741a1334bd",
+                "sha256:409afdfe1e2e90e6ee7fc896f3df9a7fec8e793e58bfa0d052c8a82f99c37abb",
+                "sha256:40fa14dbd66b8b8f470d5fc79c089a66185619d31645f9b0773b88b19f7223c4",
+                "sha256:4322872d5772cae7369f8351da1edf255a604ea7087fe295411397d0cfd9655e",
+                "sha256:44756f9fd339de0fb6ee4f8c1696cfd19b2422e0d70b4cefc1cc7f1f64045a8c",
+                "sha256:46707a10836894b559e04b0fd143e343945c97fd170d69a2d26d640b4e297935",
+                "sha256:48b571ecd8bae15702e4f22d3ff6a0f13e54d3d00cd25216d5e7f658242065ee",
+                "sha256:48be928f59a1f5c8207154f935334d374e79f2b5d212826307d072595ad76a2e",
+                "sha256:4bfa75a048c056a411f9705856abfc872558e33c055d80af6a380e3658766038",
+                "sha256:4c00336b9dd5ad96d0a558fd18a8b6f711b7449acce4c157e7343ba92dd0cf3d",
+                "sha256:4c26ed10c4f6fa6ddb329a5120ba3b6db349ca192ae211e882970bfc9d91420b",
+                "sha256:4d05d81ecb47d11e7f8932bd8b61b720bf0b41199358f3f5e36d38e28f0532c5",
+                "sha256:4e77f2126c3e0b0d055f44513ed349038ac180371ed9b52fe96a32aa071a5107",
+                "sha256:5337ec7809bcd0f424c6b705ecf97941c46279cf5ed92311782c7c9c2026f07f",
+                "sha256:5360cc32706dab3931f738d3079652d20982511f7c0ac5711483e6eab08efff2",
+                "sha256:58370b1ffbd35407444d57057b57da5d6549d2d854fa30249771775c63b5fe17",
+                "sha256:58cb20602b18f86f83a5c87d3ee1c766a79c0d452f8def86d925e6c60fbf7bfb",
+                "sha256:599b5c873c63a1f6ed7eead644a8a380cfbdf5db91dcb6f85707aaab213b1674",
+                "sha256:5b7dfa3b546da08a9f622bb6becdb14b3e24aaa30adba66749d38f3cc7ea9706",
+                "sha256:5b9c3f4ee0b9a439d2415012bd1b1cc2df59e4d6a9939f4d669241d30b414327",
+                "sha256:5d34eb8494bea691a1a450141ebb5385e4b69d38bb8403b5146ad279f4b30fa3",
+                "sha256:5d5abf8f8ec1f4e22882273c423e16cae834c36856cac348cfbfa68e01c40f3a",
+                "sha256:5e3bc157fed2a4c02ec468de4ecd12a6e22818d4f09cde2c31ee3226ffbefab2",
+                "sha256:612a10bdae23404a72941a0fc8fa2660c6ea1217c4ce0dbcab8a8f6543ea9e7f",
+                "sha256:657a05857bda581c3656bfc3b20e353c232e9193eb167766ad2dc58b56504948",
+                "sha256:65e720d2ab2b53f1f72fb5da5fb477455905ce2c88aaa671ff0a447c2c80e8e3",
+                "sha256:693902d433cf585133699972b6d7c42a8b9f8f826ebcaf0132ff55200afc599e",
+                "sha256:6af936f79086a89b3680a280c47ea90b4df7047b5bdf3aa5c524bbedddb9e545",
+                "sha256:71bb308552200fb2c195e35ef05de12f0c878c07fc91c270eb3d6e41698c3bcc",
+                "sha256:764202cc7e70f767dab49e8df52c7455e8de0df5d858fa801a11aa0d882ccf3f",
+                "sha256:76c8094ac20ec259471ac53e774623eb62e6e1f56cd8690c67ce6ce4fcb05650",
+                "sha256:78a42513018c41c2ffd262eb676442315cbfe3c44eed82385c2ed043bc63210a",
+                "sha256:79849239c39b5e1fd906556c474d9b0439ea6792b637511f3fe3a41158d89ca8",
+                "sha256:7ab9ccab2b5bd5702ab0803676a580fffa2aa178c2badc5557a84cc943fcf750",
+                "sha256:7bbfcb7165ce3d54a3dfbe731e470f65739c4c1f85bb1018ee912bae139e263b",
+                "sha256:7c06a4c7cf15ec739ce0e5971b26c93638730090add60e183530d70848ebdd34",
+                "sha256:801fa7802e5cfabe3ab0c81a34c323a319b097dfb5004be950482d882f3d7225",
+                "sha256:803b8e1459341c1bb56d1c5c010406d5edec8a0713a0945851290a7930679b51",
+                "sha256:82a5c2f4b87c26bb1a0ef3d16b5c4753434633b83d365cc0ddf2770c93829e3c",
+                "sha256:84ec80df401cfee1457063732d90022f93951944b5b58975d34ab56bb150dfb3",
+                "sha256:8705f17dfeb43139a692298cb6637ee2e59c0194538153e83e9ee0c75c2eddde",
+                "sha256:88a9ca9c710d598fd75ee5de59d5bda2684d9db36a9f50b6125eaea3969c2599",
+                "sha256:88f17c5ffa8e9462fb79f62746428dd57b46eb931698e42e990ad63103f35e6c",
+                "sha256:8a3ec5aa8e38fc4c8af308917ce12c536f1c88452ce554027e55b22cbbfbff76",
+                "sha256:8a9c83f75223d5e48b0bc9cb1bf2776cf01563e00ade8775ffe13b0b6e1af3a6",
+                "sha256:8b01aac285f91ca889c800042c35ad3b239e704b150cfd3382adfc9dcc780e39",
+                "sha256:8d53103597a252fb3ab8b5845af04c7a26d5e7ea8122303dd7a021176a87e8b9",
+                "sha256:8e045731a5416357638d1700927529e2b8ab304811671f665b225f8bf8d8f933",
+                "sha256:8f0ea6da6d393d8b2e187e6a5e3fb81f5862010a40c3945e2c6d12ae45cfb2ad",
+                "sha256:90da3b5f694b85231cf93586dad5e90e2d71b9428f9aad96952c99055582f520",
+                "sha256:913983ad2deb14e66d83c28b632fd35ba2b825031f2fa4ca29675e665dfecbe1",
+                "sha256:9242795d174daa40105c1d86aba618e8eab7bf96ba8c3ee614da8302a9f95503",
+                "sha256:929e294c1ac1e9f615c62a4e4313ca1823ba37326c164ec720a803287c4c499b",
+                "sha256:933d4de052939d90afbe6e9d5273ae05fb836cc86c15b686edd4b3560cc0ee36",
+                "sha256:942216596dc64ddb25adb215c3c783215b23626f8d84e8eff8d6d45c3f29f75a",
+                "sha256:94252291e3fe68001b1dd747b4c0b3be12582839b95ad4d1b641924d68fd4643",
+                "sha256:9893ff81bd7107f7b685d3017cc6583daadb4fc26e4a888350df530e41980a60",
+                "sha256:9e838bba3a3bac0fe06d849d29772eb1afb9745a59710762e4ba3f4cb8424483",
+                "sha256:a0f64a48bb81af7450e641e3fe0b0394d7381e342805479178b3d335d60ca7cf",
+                "sha256:a17f6a29cf8935e587cc8a4dbfc8368c55edc645283db0ce9801016f83526c2d",
+                "sha256:a1ecf0ac1c518487d9d23b1cd7139a6a65bc460cd101ab01f1be82ecf09794b6",
+                "sha256:a79ae34384df2b615eefca647a2873842ac3b596418032bef9a7283675962644",
+                "sha256:a91b5f9f1205845d488c928e8570dcb62b893372f63b8b6e98b863ebd2368ff2",
+                "sha256:aa0abdf853e09aff551db11fce173e2177d00786c688203f52c87ad7fcd91ef9",
+                "sha256:ac542bf38a8a4be2dc6b15248d36315ccc65f0743f7b1a76688ffb6b5129a5c2",
+                "sha256:ad42ba922c67c5f219097b28fae965e10045ddf145d2928bfac2eb2e17673640",
+                "sha256:aeb3531b196ef6f11776c21674dba836aeea9d5bd1cf630f869e3d90b16cfade",
+                "sha256:b38ac83d5f04b15e515fd86f312479d950d05ce2368d5413d46c088dda7de90a",
+                "sha256:b7d755065e4e866a8086c9bdada157133ff466476a2ad7861828e17b6026e22c",
+                "sha256:bd3de6481f4ed8b734da5df134cd5a6a64fe32124fe83dde1e5b5f29fe30b1e6",
+                "sha256:bfa1acfa0c54932d5607e19a2c24646fb4c1ae2694437789129cf099789a3b00",
+                "sha256:c619b101e6de2222c1fcb0531e1b17bbffbe54294bfba43ea0d411d428618c27",
+                "sha256:ce8be0466f4c0d585cdb6c1e2ed07232221df101a4c6f28821d2aa754ca2d9e2",
+                "sha256:cf0438b42121a66a3a667de17e779330fc0f20b0d97d59d2f2121e182b0505e4",
+                "sha256:cf8bcc23ceb5a1b624572a1623b9f79d2c3b337c8c455405ef231933a10da379",
+                "sha256:d2b0e12a42fb4e72d509fc994713d099cbb15ebf1103545e8a45f14da2dfca54",
+                "sha256:d83db7cde68459fc803052a55ace60bea2bae361fc3b7a6d5da07e11954e4b09",
+                "sha256:dda56c24d869b1193fcc763f1284b9126550eaf84b88bbc7256e15028f19188a",
+                "sha256:dea0bf229319828467d7fca8c7c189780aa9ff679c94539eed7532ebe33ed37c",
+                "sha256:e1631290ee9271dffe3062d2634c3ecac02c83890ada077d225e081aca8aab89",
+                "sha256:e28c7fea2196bf4c2f8d46a0415c77a1c480cc0724722f23d7410ffe9842c407",
+                "sha256:e2e6c39bd7b9372b0be21456caab138e8e69cc0fc1190a9dfa92bd45a1e6e904",
+                "sha256:e33e8fbd440c917106b237ef1a2f1449dfbb9b6f6e1ce17c94cd6a1e0d438376",
+                "sha256:e8df2eb9b2bac43ef8b082e06f750350fbbaf2887534a5be97f6cf07b19d9583",
+                "sha256:e968b84db54f9d42046cf154e02911e39c0435c9801681e3fc9ce8a3c4130278",
+                "sha256:eb542fe7933aa09d8d8f9d9097ef37532a7df6497819d16efe4359890a2f417a",
+                "sha256:edcfc407e4eb17e037bca59be0e85a2031a2ac87e4fed26d3e9df88b4165f92d",
+                "sha256:eee3ea935c3d227d49b4eb85660ff631556841f6e567f0f7bda972df6c2c9935",
+                "sha256:ef97b8df011141c9b0f6caf23b29379f87dd13183c978a30a3c546d2c47314cb",
+                "sha256:f106407dda69ae456dd1227966bf445b157ccc80ba0dff3802bb63f30b74e895",
+                "sha256:f3160309af4396e0ed04db259c3ccbfdc3621b5559b5453075e5de555e1f3a1b",
+                "sha256:f32d6edbc638cde7652bd690c3e728b25332acbadd7cad670cc4a02558d9c417",
+                "sha256:f37cfe618a117e50d8c240555331160d73d0411422b59b5ee217843d7b693608",
+                "sha256:f4c9aee212bc89d4e13f58be11a56cc8036cabad119259d12ace14b34476fd07",
+                "sha256:f4d742cb7af1c28303a51b7a27aaee540e71bb8e24f68c736f6f2ffc82f2bf05",
+                "sha256:f5a8b53bdc0b3961f8b6125e198617c40aeed638b387913bf1ce78afb1b0be2a",
+                "sha256:f816dd2277f8d63d79f9c8473a79fe54047bc0467754962840782c575522224d",
+                "sha256:f9a9e8a507420fe35992ee9ecb302dab68550dedc0da9e2880dd88071c5fb052"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.4.5"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.7"
         },
         "llvmlite": {
             "hashes": [
@@ -1691,65 +1943,171 @@
         },
         "msgpack": {
             "hashes": [
-                "sha256:00e073efcba9ea99db5acef3959efa45b52bc67b61b00823d2a1a6944bf45982",
-                "sha256:0726c282d188e204281ebd8de31724b7d749adebc086873a59efb8cf7ae27df3",
-                "sha256:0ceea77719d45c839fd73abcb190b8390412a890df2f83fb8cf49b2a4b5c2f40",
-                "sha256:114be227f5213ef8b215c22dde19532f5da9652e56e8ce969bf0a26d7c419fee",
-                "sha256:13577ec9e247f8741c84d06b9ece5f654920d8365a4b636ce0e44f15e07ec693",
-                "sha256:1876b0b653a808fcd50123b953af170c535027bf1d053b59790eebb0aeb38950",
-                "sha256:1ab0bbcd4d1f7b6991ee7c753655b481c50084294218de69365f8f1970d4c151",
-                "sha256:1cce488457370ffd1f953846f82323cb6b2ad2190987cd4d70b2713e17268d24",
-                "sha256:26ee97a8261e6e35885c2ecd2fd4a6d38252246f94a2aec23665a4e66d066305",
-                "sha256:3528807cbbb7f315bb81959d5961855e7ba52aa60a3097151cb21956fbc7502b",
-                "sha256:374a8e88ddab84b9ada695d255679fb99c53513c0a51778796fcf0944d6c789c",
-                "sha256:376081f471a2ef24828b83a641a02c575d6103a3ad7fd7dade5486cad10ea659",
-                "sha256:3923a1778f7e5ef31865893fdca12a8d7dc03a44b33e2a5f3295416314c09f5d",
-                "sha256:4916727e31c28be8beaf11cf117d6f6f188dcc36daae4e851fee88646f5b6b18",
-                "sha256:493c5c5e44b06d6c9268ce21b302c9ca055c1fd3484c25ba41d34476c76ee746",
-                "sha256:505fe3d03856ac7d215dbe005414bc28505d26f0c128906037e66d98c4e95868",
-                "sha256:5845fdf5e5d5b78a49b826fcdc0eb2e2aa7191980e3d2cfd2a30303a74f212e2",
-                "sha256:5c330eace3dd100bdb54b5653b966de7f51c26ec4a7d4e87132d9b4f738220ba",
-                "sha256:5dbf059fb4b7c240c873c1245ee112505be27497e90f7c6591261c7d3c3a8228",
-                "sha256:5e390971d082dba073c05dbd56322427d3280b7cc8b53484c9377adfbae67dc2",
-                "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273",
-                "sha256:64d0fcd436c5683fdd7c907eeae5e2cbb5eb872fafbc03a43609d7941840995c",
-                "sha256:69284049d07fce531c17404fcba2bb1df472bc2dcdac642ae71a2d079d950653",
-                "sha256:6a0e76621f6e1f908ae52860bdcb58e1ca85231a9b0545e64509c931dd34275a",
-                "sha256:73ee792784d48aa338bba28063e19a27e8d989344f34aad14ea6e1b9bd83f596",
-                "sha256:74398a4cf19de42e1498368c36eed45d9528f5fd0155241e82c4082b7e16cffd",
-                "sha256:7938111ed1358f536daf311be244f34df7bf3cdedb3ed883787aca97778b28d8",
-                "sha256:82d92c773fbc6942a7a8b520d22c11cfc8fd83bba86116bfcf962c2f5c2ecdaa",
-                "sha256:83b5c044f3eff2a6534768ccfd50425939e7a8b5cf9a7261c385de1e20dcfc85",
-                "sha256:8db8e423192303ed77cff4dce3a4b88dbfaf43979d280181558af5e2c3c71afc",
-                "sha256:9517004e21664f2b5a5fd6333b0731b9cf0817403a941b393d89a2f1dc2bd836",
-                "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3",
-                "sha256:99881222f4a8c2f641f25703963a5cefb076adffd959e0558dc9f803a52d6a58",
-                "sha256:9ee32dcb8e531adae1f1ca568822e9b3a738369b3b686d1477cbc643c4a9c128",
-                "sha256:a22e47578b30a3e199ab067a4d43d790249b3c0587d9a771921f86250c8435db",
-                "sha256:b5505774ea2a73a86ea176e8a9a4a7c8bf5d521050f0f6f8426afe798689243f",
-                "sha256:bd739c9251d01e0279ce729e37b39d49a08c0420d3fee7f2a4968c0576678f77",
-                "sha256:d16a786905034e7e34098634b184a7d81f91d4c3d246edc6bd7aefb2fd8ea6ad",
-                "sha256:d3420522057ebab1728b21ad473aa950026d07cb09da41103f8e597dfbfaeb13",
-                "sha256:d56fd9f1f1cdc8227d7b7918f55091349741904d9520c65f0139a9755952c9e8",
-                "sha256:d661dc4785affa9d0edfdd1e59ec056a58b3dbb9f196fa43587f3ddac654ac7b",
-                "sha256:dfe1f0f0ed5785c187144c46a292b8c34c1295c01da12e10ccddfc16def4448a",
-                "sha256:e1dd7839443592d00e96db831eddb4111a2a81a46b028f0facd60a09ebbdd543",
-                "sha256:e2872993e209f7ed04d963e4b4fbae72d034844ec66bc4ca403329db2074377b",
-                "sha256:e2f879ab92ce502a1e65fce390eab619774dda6a6ff719718069ac94084098ce",
-                "sha256:e3aa7e51d738e0ec0afbed661261513b38b3014754c9459508399baf14ae0c9d",
-                "sha256:e532dbd6ddfe13946de050d7474e3f5fb6ec774fbb1a188aaf469b08cf04189a",
-                "sha256:e6b7842518a63a9f17107eb176320960ec095a8ee3b4420b5f688e24bf50c53c",
-                "sha256:e75753aeda0ddc4c28dce4c32ba2f6ec30b1b02f6c0b14e547841ba5b24f753f",
-                "sha256:eadb9f826c138e6cf3c49d6f8de88225a3c0ab181a9b4ba792e006e5292d150e",
-                "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011",
-                "sha256:ef254a06bcea461e65ff0373d8a0dd1ed3aa004af48839f002a0c994a6f72d04",
-                "sha256:f3709997b228685fe53e8c433e2df9f0cdb5f4542bd5114ed17ac3c0129b0480",
-                "sha256:f51bab98d52739c50c56658cc303f190785f9a2cd97b823357e7aeae54c8f68a",
-                "sha256:f9904e24646570539a8950400602d66d2b2c492b9010ea7e965025cb71d0c86d",
-                "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"
+                "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b",
+                "sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf",
+                "sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca",
+                "sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330",
+                "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f",
+                "sha256:13599f8829cfbe0158f6456374e9eea9f44eee08076291771d8ae93eda56607f",
+                "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39",
+                "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247",
+                "sha256:3180065ec2abbe13a4ad37688b61b99d7f9e012a535b930e0e683ad6bc30155b",
+                "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c",
+                "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7",
+                "sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044",
+                "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6",
+                "sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b",
+                "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0",
+                "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2",
+                "sha256:46c34e99110762a76e3911fc923222472c9d681f1094096ac4102c18319e6468",
+                "sha256:471e27a5787a2e3f974ba023f9e265a8c7cfd373632247deb225617e3100a3c7",
+                "sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734",
+                "sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434",
+                "sha256:4d1b7ff2d6146e16e8bd665ac726a89c74163ef8cd39fa8c1087d4e52d3a2325",
+                "sha256:53258eeb7a80fc46f62fd59c876957a2d0e15e6449a9e71842b6d24419d88ca1",
+                "sha256:534480ee5690ab3cbed89d4c8971a5c631b69a8c0883ecfea96c19118510c846",
+                "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88",
+                "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420",
+                "sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e",
+                "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2",
+                "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59",
+                "sha256:646afc8102935a388ffc3914b336d22d1c2d6209c773f3eb5dd4d6d3b6f8c1cb",
+                "sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68",
+                "sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915",
+                "sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f",
+                "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701",
+                "sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b",
+                "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d",
+                "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa",
+                "sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d",
+                "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd",
+                "sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc",
+                "sha256:7e7b853bbc44fb03fbdba34feb4bd414322180135e2cb5164f20ce1c9795ee48",
+                "sha256:879a7b7b0ad82481c52d3c7eb99bf6f0645dbdec5134a4bddbd16f3506947feb",
+                "sha256:8a706d1e74dd3dea05cb54580d9bd8b2880e9264856ce5068027eed09680aa74",
+                "sha256:8a84efb768fb968381e525eeeb3d92857e4985aacc39f3c47ffd00eb4509315b",
+                "sha256:8cf9e8c3a2153934a23ac160cc4cba0ec035f6867c8013cc6077a79823370346",
+                "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e",
+                "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6",
+                "sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5",
+                "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f",
+                "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5",
+                "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b",
+                "sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c",
+                "sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f",
+                "sha256:c40ffa9a15d74e05ba1fe2681ea33b9caffd886675412612d93ab17b58ea2fec",
+                "sha256:c5a91481a3cc573ac8c0d9aace09345d989dc4a0202b7fcb312c88c26d4e71a8",
+                "sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5",
+                "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d",
+                "sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e",
+                "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e",
+                "sha256:e0856a2b7e8dcb874be44fea031d22e5b3a19121be92a1e098f46068a11b0870",
+                "sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f",
+                "sha256:f1ba6136e650898082d9d5a5217d5906d1e138024f836ff48691784bbe1adf96",
+                "sha256:f3e9b4936df53b970513eac1758f3882c88658a220b58dcc1e39606dccaaf01c",
+                "sha256:f80bc7d47f76089633763f952e67f8214cb7b3ee6bfa489b3cb6a84cfac114cd",
+                "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.1.0"
+        },
+        "multidict": {
+            "hashes": [
+                "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f",
+                "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056",
+                "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761",
+                "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3",
+                "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b",
+                "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6",
+                "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748",
+                "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966",
+                "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f",
+                "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1",
+                "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6",
+                "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada",
+                "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305",
+                "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2",
+                "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d",
+                "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a",
+                "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef",
+                "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c",
+                "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb",
+                "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60",
+                "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6",
+                "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4",
+                "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478",
+                "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81",
+                "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7",
+                "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56",
+                "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3",
+                "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6",
+                "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30",
+                "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb",
+                "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506",
+                "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0",
+                "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925",
+                "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c",
+                "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6",
+                "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e",
+                "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95",
+                "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2",
+                "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133",
+                "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2",
+                "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa",
+                "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3",
+                "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3",
+                "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436",
+                "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657",
+                "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581",
+                "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492",
+                "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43",
+                "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2",
+                "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2",
+                "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926",
+                "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057",
+                "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc",
+                "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80",
+                "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255",
+                "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1",
+                "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972",
+                "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53",
+                "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1",
+                "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423",
+                "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a",
+                "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160",
+                "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c",
+                "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd",
+                "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa",
+                "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5",
+                "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b",
+                "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa",
+                "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef",
+                "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44",
+                "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4",
+                "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156",
+                "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753",
+                "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28",
+                "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d",
+                "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a",
+                "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304",
+                "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008",
+                "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429",
+                "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72",
+                "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399",
+                "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3",
+                "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392",
+                "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167",
+                "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c",
+                "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774",
+                "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351",
+                "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76",
+                "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875",
+                "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd",
+                "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28",
+                "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.0.8"
+            "version": "==6.1.0"
         },
         "multimethod": {
             "hashes": [
@@ -2223,11 +2581,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee",
-                "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"
+                "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907",
+                "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.2.2"
+            "version": "==4.3.6"
         },
         "pre-commit": {
             "hashes": [
@@ -2529,52 +2887,52 @@
         },
         "pydantic": {
             "hashes": [
-                "sha256:098ad8de840c92ea586bf8efd9e2e90c6339d33ab5c1cfbb85be66e4ecf8213f",
-                "sha256:0e2495309b1266e81d259a570dd199916ff34f7f51f1b549a0d37a6d9b17b4dc",
-                "sha256:0fa51175313cc30097660b10eec8ca55ed08bfa07acbfe02f7a42f6c242e9a4b",
-                "sha256:11289fa895bcbc8f18704efa1d8020bb9a86314da435348f59745473eb042e6b",
-                "sha256:2a72d2a5ff86a3075ed81ca031eac86923d44bc5d42e719d585a8eb547bf0c9b",
-                "sha256:371dcf1831f87c9e217e2b6a0c66842879a14873114ebb9d0861ab22e3b5bb1e",
-                "sha256:409b2b36d7d7d19cd8310b97a4ce6b1755ef8bd45b9a2ec5ec2b124db0a0d8f3",
-                "sha256:4866a1579c0c3ca2c40575398a24d805d4db6cb353ee74df75ddeee3c657f9a7",
-                "sha256:48db882e48575ce4b39659558b2f9f37c25b8d348e37a2b4e32971dd5a7d6227",
-                "sha256:525bbef620dac93c430d5d6bdbc91bdb5521698d434adf4434a7ef6ffd5c4b7f",
-                "sha256:543da3c6914795b37785703ffc74ba4d660418620cc273490d42c53949eeeca6",
-                "sha256:62d96b8799ae3d782df7ec9615cb59fc32c32e1ed6afa1b231b0595f6516e8ab",
-                "sha256:6654028d1144df451e1da69a670083c27117d493f16cf83da81e1e50edce72ad",
-                "sha256:7017971ffa7fd7808146880aa41b266e06c1e6e12261768a28b8b41ba55c8076",
-                "sha256:7623b59876f49e61c2e283551cc3647616d2fbdc0b4d36d3d638aae8547ea681",
-                "sha256:7e17c0ee7192e54a10943f245dc79e36d9fe282418ea05b886e1c666063a7b54",
-                "sha256:820ae12a390c9cbb26bb44913c87fa2ff431a029a785642c1ff11fed0a095fcb",
-                "sha256:94833612d6fd18b57c359a127cbfd932d9150c1b72fea7c86ab58c2a77edd7c7",
-                "sha256:95ef534e3c22e5abbdbdd6f66b6ea9dac3ca3e34c5c632894f8625d13d084cbe",
-                "sha256:9c803a5113cfab7bbb912f75faa4fc1e4acff43e452c82560349fff64f852e1b",
-                "sha256:9e53fb834aae96e7b0dadd6e92c66e7dd9cdf08965340ed04c16813102a47fab",
-                "sha256:ab2f976336808fd5d539fdc26eb51f9aafc1f4b638e212ef6b6f05e753c8011d",
-                "sha256:ad1e33dc6b9787a6f0f3fd132859aa75626528b49cc1f9e429cdacb2608ad5f0",
-                "sha256:ae5184e99a060a5c80010a2d53c99aee76a3b0ad683d493e5f0620b5d86eeb75",
-                "sha256:aeb4e741782e236ee7dc1fb11ad94dc56aabaf02d21df0e79e0c21fe07c95741",
-                "sha256:b4ad32aed3bf5eea5ca5decc3d1bbc3d0ec5d4fbcd72a03cdad849458decbc63",
-                "sha256:b8ad363330557beac73159acfbeed220d5f1bfcd6b930302a987a375e02f74fd",
-                "sha256:bfbb18b616abc4df70591b8c1ff1b3eabd234ddcddb86b7cac82657ab9017e33",
-                "sha256:c1e51d1af306641b7d1574d6d3307eaa10a4991542ca324f0feb134fee259815",
-                "sha256:c31d281c7485223caf6474fc2b7cf21456289dbaa31401844069b77160cab9c7",
-                "sha256:c7e8988bb16988890c985bd2093df9dd731bfb9d5e0860db054c23034fab8f7a",
-                "sha256:c87cedb4680d1614f1d59d13fea353faf3afd41ba5c906a266f3f2e8c245d655",
-                "sha256:cafb9c938f61d1b182dfc7d44a7021326547b7b9cf695db5b68ec7b590214773",
-                "sha256:d2f89a719411cb234105735a520b7c077158a81e0fe1cb05a79c01fc5eb59d3c",
-                "sha256:d4b40c9e13a0b61583e5599e7950490c700297b4a375b55b2b592774332798b7",
-                "sha256:d4ecb515fa7cb0e46e163ecd9d52f9147ba57bc3633dca0e586cdb7a232db9e3",
-                "sha256:d8c209af63ccd7b22fba94b9024e8b7fd07feffee0001efae50dd99316b27768",
-                "sha256:db3b48d9283d80a314f7a682f7acae8422386de659fffaba454b77a083c3937d",
-                "sha256:e41b5b973e5c64f674b3b4720286ded184dcc26a691dd55f34391c62c6934688",
-                "sha256:e840e6b2026920fc3f250ea8ebfdedf6ea7a25b77bf04c6576178e681942ae0f",
-                "sha256:ebb249096d873593e014535ab07145498957091aa6ae92759a32d40cb9998e2e",
-                "sha256:f434160fb14b353caf634149baaf847206406471ba70e64657c1e8330277a991",
-                "sha256:fa43f362b46741df8f201bf3e7dff3569fa92069bcc7b4a740dea3602e27ab7a"
+                "sha256:069b9c9fc645474d5ea3653788b544a9e0ccd3dca3ad8c900c4c6eac844b4620",
+                "sha256:06a189b81ffc52746ec9c8c007f16e5167c8b0a696e1a726369327e3db7b2a82",
+                "sha256:11d9d9b87b50338b1b7de4ebf34fd29fdb0d219dc07ade29effc74d3d2609c62",
+                "sha256:15fdbe568beaca9aacfccd5ceadfb5f1a235087a127e8af5e48df9d8a45ae85c",
+                "sha256:19a3bd00b9dafc2cd7250d94d5b578edf7a0bd7daf102617153ff9a8fa37871c",
+                "sha256:23e8ec1ce4e57b4f441fc91e3c12adba023fedd06868445a5b5f1d48f0ab3682",
+                "sha256:24a4a159d0f7a8e26bf6463b0d3d60871d6a52eac5bb6a07a7df85c806f4c048",
+                "sha256:2ce3fcf75b2bae99aa31bd4968de0474ebe8c8258a0110903478bd83dfee4e3b",
+                "sha256:335a32d72c51a313b33fa3a9b0fe283503272ef6467910338e123f90925f0f03",
+                "sha256:3445426da503c7e40baccefb2b2989a0c5ce6b163679dd75f55493b460f05a8f",
+                "sha256:34a3613c7edb8c6fa578e58e9abe3c0f5e7430e0fc34a65a415a1683b9c32d9a",
+                "sha256:3d5492dbf953d7d849751917e3b2433fb26010d977aa7a0765c37425a4026ff1",
+                "sha256:44ae8a3e35a54d2e8fa88ed65e1b08967a9ef8c320819a969bfa09ce5528fafe",
+                "sha256:467a14ee2183bc9c902579bb2f04c3d3dac00eff52e252850509a562255b2a33",
+                "sha256:46f379b8cb8a3585e3f61bf9ae7d606c70d133943f339d38b76e041ec234953f",
+                "sha256:49e26c51ca854286bffc22b69787a8d4063a62bf7d83dc21d44d2ff426108518",
+                "sha256:65f7361a09b07915a98efd17fdec23103307a54db2000bb92095457ca758d485",
+                "sha256:6951f3f47cb5ca4da536ab161ac0163cab31417d20c54c6de5ddcab8bc813c3f",
+                "sha256:72fa46abace0a7743cc697dbb830a41ee84c9db8456e8d77a46d79b537efd7ec",
+                "sha256:74fe19dda960b193b0eb82c1f4d2c8e5e26918d9cda858cbf3f41dd28549cb70",
+                "sha256:7a4c5eec138a9b52c67f664c7d51d4c7234c5ad65dd8aacd919fb47445a62c86",
+                "sha256:80b982d42515632eb51f60fa1d217dfe0729f008e81a82d1544cc392e0a50ddf",
+                "sha256:941a2eb0a1509bd7f31e355912eb33b698eb0051730b2eaf9e70e2e1589cae1d",
+                "sha256:9f463abafdc92635da4b38807f5b9972276be7c8c5121989768549fceb8d2588",
+                "sha256:a00e63104346145389b8e8f500bc6a241e729feaf0559b88b8aa513dd2065481",
+                "sha256:aad8771ec8dbf9139b01b56f66386537c6fe4e76c8f7a47c10261b69ad25c2c9",
+                "sha256:ae6fa2008e1443c46b7b3a5eb03800121868d5ab6bc7cda20b5df3e133cde8b3",
+                "sha256:b661ce52c7b5e5f600c0c3c5839e71918346af2ef20062705ae76b5c16914cab",
+                "sha256:b74be007703547dc52e3c37344d130a7bfacca7df112a9e5ceeb840a9ce195c7",
+                "sha256:baebdff1907d1d96a139c25136a9bb7d17e118f133a76a2ef3b845e831e3403a",
+                "sha256:c20f682defc9ef81cd7eaa485879ab29a86a0ba58acf669a78ed868e72bb89e0",
+                "sha256:c3e742f62198c9eb9201781fbebe64533a3bbf6a76a91b8d438d62b813079dbc",
+                "sha256:c5ae6b7c8483b1e0bf59e5f1843e4fd8fd405e11df7de217ee65b98eb5462861",
+                "sha256:c6d0a9f9eccaf7f438671a64acf654ef0d045466e63f9f68a579e2383b63f357",
+                "sha256:cbfbca662ed3729204090c4d09ee4beeecc1a7ecba5a159a94b5a4eb24e3759a",
+                "sha256:d5389eb3b48a72da28c6e061a247ab224381435256eb541e175798483368fdd3",
+                "sha256:e306e280ebebc65040034bff1a0a81fd86b2f4f05daac0131f29541cafd80b80",
+                "sha256:e405ffcc1254d76bb0e760db101ee8916b620893e6edfbfee563b3c6f7a67c02",
+                "sha256:e9ee4e6ca1d9616797fa2e9c0bfb8815912c7d67aca96f77428e316741082a1b",
+                "sha256:ef0fe7ad7cbdb5f372463d42e6ed4ca9c443a52ce544472d8842a0576d830da5",
+                "sha256:efbc8a7f9cb5fe26122acba1852d8dcd1e125e723727c59dcd244da7bdaa54f2",
+                "sha256:fcb20d4cb355195c75000a49bb4a31d75e4295200df620f454bbc6bdf60ca890",
+                "sha256:fe734914977eed33033b70bfc097e1baaffb589517863955430bf2e0846ac30f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.10.17"
+            "version": "==1.10.18"
         },
         "pyflakes": {
             "hashes": [
@@ -2636,11 +2994,11 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad",
-                "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"
+                "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c",
+                "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"
             ],
             "markers": "python_full_version >= '3.6.8'",
-            "version": "==3.1.2"
+            "version": "==3.1.4"
         },
         "pyppmd": {
             "hashes": [
@@ -2806,10 +3164,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812",
-                "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"
+                "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a",
+                "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"
             ],
-            "version": "==2024.1"
+            "version": "==2024.2"
         },
         "pyyaml": {
             "hashes": [
@@ -2872,118 +3230,118 @@
         },
         "pyzmq": {
             "hashes": [
-                "sha256:0065026e624052a51033857e5cd45a94b52946b44533f965f0bdf182460e965d",
-                "sha256:00f39c367bbd6aa8e4bc36af6510561944c619b58eb36199fa334b594a18f615",
-                "sha256:0841633446cb1539a832a19bb24c03a20c00887d0cedd1d891b495b07e5c5cb5",
-                "sha256:08956c26dbcd4fd8835cb777a16e21958ed2412317630e19f0018d49dbeeb470",
-                "sha256:0cf1d980c969fb9e538f52abd2227f09e015096bc5c3ef7aa26e0d64051c1db8",
-                "sha256:146956aec7d947c5afc5e7da0841423d7a53f84fd160fff25e682361dcfb32cb",
-                "sha256:14bdbae02f72f4716b0ffe7500e9da303d719ddde1f3dcfb4c4f6cc1cf73bb02",
-                "sha256:18da8e84dbc30688fd2baefd41df7190607511f916be34f9a24b0e007551822e",
-                "sha256:2067e63fd9d5c13cfe12624dab0366053e523b37a7a01678ce4321f839398939",
-                "sha256:23f2fe4fb567e8098ebaa7204819658195b10ddd86958a97a6058eed2901eed3",
-                "sha256:2508bdc8ab246e5ed7c92023d4352aaad63020ca3b098a4e3f1822db202f703d",
-                "sha256:25d128524207f53f7aae7c5abdc2b63f8957a060b00521af5ffcd20986b5d8f4",
-                "sha256:26131b1cec02f941ed2d2b4b8cc051662b1c248b044eff5069df1f500bbced56",
-                "sha256:29de77ba1b1877fe7defc1b9140e65cbd35f72a63bc501e56c2eae55bde5fff4",
-                "sha256:2ae7aa1408778dc74582a1226052b930f9083b54b64d7e6ef6ec0466cfdcdec2",
-                "sha256:2b045647caf620ce0ed6c8fd9fb6a73116f99aceed966b152a5ba1b416d25311",
-                "sha256:2c0fdb7b758e0e1605157e480b00b3a599073068a37091a1c75ec65bf7498645",
-                "sha256:2f6071ec95af145d7b659dae6786871cd85f0acc599286b6f8ba0c74592d83dd",
-                "sha256:32123ff0a6db521aadf2b95201e967a4e0d11fb89f73663a99d2f54881c07214",
-                "sha256:32de51744820857a6f7c3077e620ab3f607d0e4388dfead885d5124ab9bcdc5e",
-                "sha256:362cac2423e36966d336d79d3ec3eafeabc153ee3e7a5cf580d7e74a34b3d912",
-                "sha256:3abb15df0c763339edb27a644c19381b2425ddd1aea3dbd77c1601a3b31867b8",
-                "sha256:3ddbd851a3a2651fdc5065a2804d50cf2f4b13b1bcd66de8e9e855d0217d4fcd",
-                "sha256:3e3b66fe6131b4f33d239f7d4c3bfb2f8532d8644bae3b3da4f3987073edac55",
-                "sha256:3ee5cbf2625b94de21c68d0cefd35327c8dfdbd6a98fcc41682b4e8bb00d841f",
-                "sha256:40908ec2dd3b29bbadc0916a0d3c87f8dbeebbd8fead8e618539f09e0506dec4",
-                "sha256:4350233569b4bbef88595c5e77ee38995a6f1f1790fae148b578941bfffd1c24",
-                "sha256:4437af9fee7a58302dbd511cc49f0cc2b35c112a33a1111fb123cf0be45205ca",
-                "sha256:443ebf5e261a95ee9725693f2a5a71401f89b89df0e0ea58844b074067aac2f1",
-                "sha256:472cacd16f627c06d3c8b2d374345ab74446bae913584a6245e2aa935336d929",
-                "sha256:48dee75c2a9fa4f4a583d4028d564a0453447ee1277a29b07acc3743c092e259",
-                "sha256:4d4c7fe5e50e269f9c63a260638488fec194a73993008618a59b54c47ef6ae72",
-                "sha256:4e1fcdc333afbf9918d0a614a6e10858aede7da49a60f6705a77e343fe86a317",
-                "sha256:50f0669324e27cc2091ef6ab76ca7112f364b6249691790b4cffce31e73fda28",
-                "sha256:583f73b113b8165713b6ce028d221402b1b69483055b5aa3f991937e34dd1ead",
-                "sha256:59b57e912feef6951aec8bb03fe0faa5ad5f36962883c72a30a9c965e6d988fd",
-                "sha256:5ccfcf13e80719f6a2d9c0a021d9e47d4550907a29253554be2c09582f6d7963",
-                "sha256:5e6f39ecb8eb7bfcb976c49262e8cf83ff76e082b77ca23ba90c9b6691a345be",
-                "sha256:62b5180e23e6f581600459cd983473cd723fdc64350f606d21407c99832aaf5f",
-                "sha256:63351392f948b5d50b9f55161994bc4feedbfb3f3cfe393d2f503dea2c3ec445",
-                "sha256:65e2a18e845c6ea7ab849c70db932eaeadee5edede9e379eb21c0a44cf523b2e",
-                "sha256:6c8087a3281c20b1d11042d372ed5a47734af05975d78e4d1d6e7bd1018535f3",
-                "sha256:6f0512fc87629ad968889176bf2165d721cd817401a281504329e2a2ed0ca6a3",
-                "sha256:6f34453ef3496ca3462f30435bf85f535f9550392987341f9ccc92c102825a79",
-                "sha256:6ff14c2fae6c0c2c1c02590c5c5d75aa1db35b859971b3ca2fcd28f983d9f2b6",
-                "sha256:70af4c9c991714ef1c65957605a8de42ef0d0620dd5f125953c8e682281bdb80",
-                "sha256:717960855f2d6fdc2dba9df49dff31c414187bb11c76af36343a57d1f7083d9a",
-                "sha256:732f957441e5b1c65a7509395e6b6cafee9e12df9aa5f4bf92ed266fe0ba70ee",
-                "sha256:741bdb4d96efe8192616abdc3671931d51a8bcd38c71da2d53fb3127149265d1",
-                "sha256:75bd448a28b1001b6928679015bc95dd5f172703ed30135bb9e34fc9cda0a3e7",
-                "sha256:76154943e4c4054b2591792eb3484ef1dd23d59805759f9cebd2f010aa30ee8c",
-                "sha256:76390d3d66406cb01b9681c382874400e9dfd77f30ecdea4bd1bf5226dd4aff0",
-                "sha256:7a5342110510045a47de1e87f5f1dcc1d9d90109522316dc9830cfc6157c800f",
-                "sha256:7c506a51cb01bb997a3f6440db0d121e5e7a32396e9948b1fdb6a7bfa67243f4",
-                "sha256:7dfabc180a4da422a4b349c63077347392463a75fa07aa3be96712ed6d42c547",
-                "sha256:809673947e95752e407aaaaf03f205ee86ebfff9ca51db6d4003dfd87b8428d1",
-                "sha256:8285b25aa20fcc46f1ca4afbc39fd3d5f2fe4c4bbf7f2c7f907a214e87a70024",
-                "sha256:85f2d2ee5ea9a8f1de86a300e1062fbab044f45b5ce34d20580c0198a8196db0",
-                "sha256:8d042d6446cab3a1388b38596f5acabb9926b0b95c3894c519356b577a549458",
-                "sha256:8d4234f335b0d0842f7d661d8cd50cbad0729be58f1c4deb85cd96b38fe95025",
-                "sha256:8eaffcd6bf6a9d00b66a2052a33fa7e6a6575427e9644395f13c3d070f2918dc",
-                "sha256:92eca4f80e8a748d880e55d3cf57ef487692e439f12d5c5a2e1cce84aaa7f6cb",
-                "sha256:9498ac427d20d0e0ef0e4bbd6200841e91640dfdf619f544ceec7f464cfb6070",
-                "sha256:9521b874fd489495865172f344e46e0159095d1f161858e3fc6e28e43ca15160",
-                "sha256:9763a8d3f5f74ef679989b373c37cc22e8d07e56d26439205cb83edb7722357f",
-                "sha256:9f380d5333fc7cd17423f486125dcc073918676e33db70a6a8172b19fc78d23d",
-                "sha256:a4cbecda4ddbfc1e309c3be04d333f9be3fc6178b8b6592b309676f929767a15",
-                "sha256:a7db05d8b7cd1a8c6610e9e9aa55d525baae7a44a43e18bc3260eb3f92de96c6",
-                "sha256:a8234571df7816f99dde89c3403cb396d70c6554120b795853a8ea56fcc26cd3",
-                "sha256:a83653c6bbe5887caea55e49fbd2909c14b73acf43bcc051eb60b2d514bbd46e",
-                "sha256:a880240597010914ffb1d6edd04d3deb7ce6a2abf79a0012751438d13630a671",
-                "sha256:aa79c528706561306938b275f89bb2c6985ce08469c27e5de05bc680df5e826f",
-                "sha256:abad7b897e960d577eb4a0f3f789c1780bc3ffe2e7c27cf317e7c90ad26acf12",
-                "sha256:af690ea4be6ca92a67c2b44a779a023bf0838e92d48497a2268175dc4a505691",
-                "sha256:b1bb952d1e407463c9333ea7e0c0600001e54e08ce836d4f0aff1fb3f902cf63",
-                "sha256:b8e153f5dffb0310af71fc6fc9cd8174f4c8ea312c415adcb815d786fee78179",
-                "sha256:bc5df31e36e4fddd4c8b5c42daee8d54d7b529e898ac984be97bf5517de166a7",
-                "sha256:bc904e86de98f8fc5bd41597da5d61232d2d6d60c4397f26efffabb961b2b245",
-                "sha256:bc994e220c1403ae087d7f0fa45129d583e46668a019e389060da811a5a9320e",
-                "sha256:be3fc2b11c0c384949cf1f01f9a48555039408b0f3e877863b1754225635953e",
-                "sha256:c11a95d3f6fc7e714ccd1066f68f9c1abd764a8b3596158be92f46dd49f41e03",
-                "sha256:c513d829a548c2d5c88983167be2b3aa537f6d1191edcdc6fcd8999e18bdd994",
-                "sha256:c5248e6e0fcbbbc912982e99cdd51c342601f495b0fa5bd667f3bdbdbf3e170f",
-                "sha256:c70dab93d98b2bf3f0ac1265edbf6e7f83acbf71dabcc4611889bb0dea45bed7",
-                "sha256:cc09b1de8b985ca5a0ca343dd7fb007267c6b329347a74e200f4654268084239",
-                "sha256:cc109be2ee3638035d276e18eaf66a1e1f44201c0c4bea4ee0c692766bbd3570",
-                "sha256:cc8d655627d775475eafdcf0e49e74bcc1e5e90afd9ab813b4da98f092ed7b93",
-                "sha256:ce05841322b58510607f9508a573138d995a46c7928887bc433de9cb760fd2ad",
-                "sha256:cf4be7460a0c1bc71e9b0e64ecdd75a86386ca6afaa36641686f5542d0314e9d",
-                "sha256:cf4e283f97688d993cb7a8acbc22889effbbb7cbaa19ee9709751f44be928f5d",
-                "sha256:d0da97e65ee73261dba70469cc8f63d8da3a8a825337a2e3d246b9e95141cdd0",
-                "sha256:d3df226ab7464684ae6706e20a5cbab717c3735a7e409b3fa598b754d49f1946",
-                "sha256:d74b925d997e4f92b042bdd7085cd0a309ee0fd7cb4dc376059bbff6b32ff34f",
-                "sha256:db1b7e2b50ef21f398036786da4c153db63203a402396d9f21e08ea61f3f8dba",
-                "sha256:de6f384864a959866b782e6a3896538d1424d183f2d3c7ef079f71dcecde7284",
-                "sha256:def7ae3006924b8a0c146a89ab4008310913fa903beedb95e25dea749642528e",
-                "sha256:e03be7ed17836c9434cce0668ac1e2cc9143d7169f90f46a0167f6155e176e32",
-                "sha256:e790602d7ea1d6c7d8713d571226d67de7ffe47b1e22ae2c043ebd537de1bccb",
-                "sha256:e80345900ae241c2c51bead7c9fa247bba6d4b2a83423e9791bae8b0a7f12c52",
-                "sha256:ebef7d3fe11fe4c688f08bc0211a976c3318c097057f258428200737b9fff4da",
-                "sha256:ec8fe214fcc45dfb0c32e4a7ad1db20244ba2d2fecbf0cbf9d5242d81ca0a375",
-                "sha256:f0a45102ad7ed9f9ddf2bd699cc5df37742cf7301111cba06001b927efecb120",
-                "sha256:f1483d4975ae1b387b39bb8e23d1ff32fe5621aa9e4ed3055d05e9c5613fea53",
-                "sha256:f218179c90a12d660906e04b25a340dd63e9743000ba16232ddaf46888f269da",
-                "sha256:f66dcb6625c002f209cdc12cae1a1fec926493cd2262efe37dc6b25a30cea863",
-                "sha256:fc657577f057d60dd3642c9f95f28b432889b73143140061f7c1331d02f03df6",
-                "sha256:fcb90592c5d5c562e1b1a1ceccf6f00036d73c51db0271bf4d352b8d6b31d468",
-                "sha256:fe73d7c89d6f803bed122135ff5783364e8cdb479cf6fe2d764a44b6349e7e0f",
-                "sha256:ffecc43b3c18e36b62fcec995761829b6ac325d8dd74a4f2c5c1653afbb4495a"
+                "sha256:007137c9ac9ad5ea21e6ad97d3489af654381324d5d3ba614c323f60dab8fae6",
+                "sha256:034da5fc55d9f8da09015d368f519478a52675e558c989bfcb5cf6d4e16a7d2a",
+                "sha256:05590cdbc6b902101d0e65d6a4780af14dc22914cc6ab995d99b85af45362cc9",
+                "sha256:070672c258581c8e4f640b5159297580a9974b026043bd4ab0470be9ed324f1f",
+                "sha256:0aca98bc423eb7d153214b2df397c6421ba6373d3397b26c057af3c904452e37",
+                "sha256:0bed0e799e6120b9c32756203fb9dfe8ca2fb8467fed830c34c877e25638c3fc",
+                "sha256:0d987a3ae5a71c6226b203cfd298720e0086c7fe7c74f35fa8edddfbd6597eed",
+                "sha256:0eaa83fc4c1e271c24eaf8fb083cbccef8fde77ec8cd45f3c35a9a123e6da097",
+                "sha256:160c7e0a5eb178011e72892f99f918c04a131f36056d10d9c1afb223fc952c2d",
+                "sha256:17bf5a931c7f6618023cdacc7081f3f266aecb68ca692adac015c383a134ca52",
+                "sha256:17c412bad2eb9468e876f556eb4ee910e62d721d2c7a53c7fa31e643d35352e6",
+                "sha256:18c8dc3b7468d8b4bdf60ce9d7141897da103c7a4690157b32b60acb45e333e6",
+                "sha256:1a534f43bc738181aa7cbbaf48e3eca62c76453a40a746ab95d4b27b1111a7d2",
+                "sha256:1c17211bc037c7d88e85ed8b7d8f7e52db6dc8eca5590d162717c654550f7282",
+                "sha256:1f3496d76b89d9429a656293744ceca4d2ac2a10ae59b84c1da9b5165f429ad3",
+                "sha256:1fcc03fa4997c447dce58264e93b5aa2d57714fbe0f06c07b7785ae131512732",
+                "sha256:226af7dcb51fdb0109f0016449b357e182ea0ceb6b47dfb5999d569e5db161d5",
+                "sha256:23f4aad749d13698f3f7b64aad34f5fc02d6f20f05999eebc96b89b01262fb18",
+                "sha256:25bf2374a2a8433633c65ccb9553350d5e17e60c8eb4de4d92cc6bd60f01d306",
+                "sha256:28ad5233e9c3b52d76196c696e362508959741e1a005fb8fa03b51aea156088f",
+                "sha256:28c812d9757fe8acecc910c9ac9dafd2ce968c00f9e619db09e9f8f54c3a68a3",
+                "sha256:29c6a4635eef69d68a00321e12a7d2559fe2dfccfa8efae3ffb8e91cd0b36a8b",
+                "sha256:29c7947c594e105cb9e6c466bace8532dc1ca02d498684128b339799f5248277",
+                "sha256:2a50625acdc7801bc6f74698c5c583a491c61d73c6b7ea4dee3901bb99adb27a",
+                "sha256:2ae90ff9dad33a1cfe947d2c40cb9cb5e600d759ac4f0fd22616ce6540f72797",
+                "sha256:2c4a71d5d6e7b28a47a394c0471b7e77a0661e2d651e7ae91e0cab0a587859ca",
+                "sha256:2ea4ad4e6a12e454de05f2949d4beddb52460f3de7c8b9d5c46fbb7d7222e02c",
+                "sha256:2eb7735ee73ca1b0d71e0e67c3739c689067f055c764f73aac4cc8ecf958ee3f",
+                "sha256:31507f7b47cc1ead1f6e86927f8ebb196a0bab043f6345ce070f412a59bf87b5",
+                "sha256:35cffef589bcdc587d06f9149f8d5e9e8859920a071df5a2671de2213bef592a",
+                "sha256:367b4f689786fca726ef7a6c5ba606958b145b9340a5e4808132cc65759abd44",
+                "sha256:39887ac397ff35b7b775db7201095fc6310a35fdbae85bac4523f7eb3b840e20",
+                "sha256:3a495b30fc91db2db25120df5847d9833af237546fd59170701acd816ccc01c4",
+                "sha256:3b55a4229ce5da9497dd0452b914556ae58e96a4381bb6f59f1305dfd7e53fc8",
+                "sha256:402b190912935d3db15b03e8f7485812db350d271b284ded2b80d2e5704be780",
+                "sha256:43a47408ac52647dfabbc66a25b05b6a61700b5165807e3fbd40063fcaf46386",
+                "sha256:4661c88db4a9e0f958c8abc2b97472e23061f0bc737f6f6179d7a27024e1faa5",
+                "sha256:46a446c212e58456b23af260f3d9fb785054f3e3653dbf7279d8f2b5546b21c2",
+                "sha256:470d4a4f6d48fb34e92d768b4e8a5cc3780db0d69107abf1cd7ff734b9766eb0",
+                "sha256:49d34ab71db5a9c292a7644ce74190b1dd5a3475612eefb1f8be1d6961441971",
+                "sha256:4d29ab8592b6ad12ebbf92ac2ed2bedcfd1cec192d8e559e2e099f648570e19b",
+                "sha256:4d80b1dd99c1942f74ed608ddb38b181b87476c6a966a88a950c7dee118fdf50",
+                "sha256:4da04c48873a6abdd71811c5e163bd656ee1b957971db7f35140a2d573f6949c",
+                "sha256:4f78c88905461a9203eac9faac157a2a0dbba84a0fd09fd29315db27be40af9f",
+                "sha256:4ff9dc6bc1664bb9eec25cd17506ef6672d506115095411e237d571e92a58231",
+                "sha256:5506f06d7dc6ecf1efacb4a013b1f05071bb24b76350832c96449f4a2d95091c",
+                "sha256:55cf66647e49d4621a7e20c8d13511ef1fe1efbbccf670811864452487007e08",
+                "sha256:5a509df7d0a83a4b178d0f937ef14286659225ef4e8812e05580776c70e155d5",
+                "sha256:5c2b3bfd4b9689919db068ac6c9911f3fcb231c39f7dd30e3138be94896d18e6",
+                "sha256:6835dd60355593de10350394242b5757fbbd88b25287314316f266e24c61d073",
+                "sha256:689c5d781014956a4a6de61d74ba97b23547e431e9e7d64f27d4922ba96e9d6e",
+                "sha256:6a96179a24b14fa6428cbfc08641c779a53f8fcec43644030328f44034c7f1f4",
+                "sha256:6ace4f71f1900a548f48407fc9be59c6ba9d9aaf658c2eea6cf2779e72f9f317",
+                "sha256:6b274e0762c33c7471f1a7471d1a2085b1a35eba5cdc48d2ae319f28b6fc4de3",
+                "sha256:706e794564bec25819d21a41c31d4df2d48e1cc4b061e8d345d7fb4dd3e94072",
+                "sha256:70fc7fcf0410d16ebdda9b26cbd8bf8d803d220a7f3522e060a69a9c87bf7bad",
+                "sha256:7133d0a1677aec369d67dd78520d3fa96dd7f3dcec99d66c1762870e5ea1a50a",
+                "sha256:7445be39143a8aa4faec43b076e06944b8f9d0701b669df4af200531b21e40bb",
+                "sha256:76589c020680778f06b7e0b193f4b6dd66d470234a16e1df90329f5e14a171cd",
+                "sha256:76589f2cd6b77b5bdea4fca5992dc1c23389d68b18ccc26a53680ba2dc80ff2f",
+                "sha256:77eb0968da535cba0470a5165468b2cac7772cfb569977cff92e240f57e31bef",
+                "sha256:794a4562dcb374f7dbbfb3f51d28fb40123b5a2abadee7b4091f93054909add5",
+                "sha256:7ad1bc8d1b7a18497dda9600b12dc193c577beb391beae5cd2349184db40f187",
+                "sha256:7f98f6dfa8b8ccaf39163ce872bddacca38f6a67289116c8937a02e30bbe9711",
+                "sha256:8423c1877d72c041f2c263b1ec6e34360448decfb323fa8b94e85883043ef988",
+                "sha256:8685fa9c25ff00f550c1fec650430c4b71e4e48e8d852f7ddcf2e48308038640",
+                "sha256:878206a45202247781472a2d99df12a176fef806ca175799e1c6ad263510d57c",
+                "sha256:89289a5ee32ef6c439086184529ae060c741334b8970a6855ec0b6ad3ff28764",
+                "sha256:8ab5cad923cc95c87bffee098a27856c859bd5d0af31bd346035aa816b081fe1",
+                "sha256:8b435f2753621cd36e7c1762156815e21c985c72b19135dac43a7f4f31d28dd1",
+                "sha256:8be4700cd8bb02cc454f630dcdf7cfa99de96788b80c51b60fe2fe1dac480289",
+                "sha256:8c997098cc65e3208eca09303630e84d42718620e83b733d0fd69543a9cab9cb",
+                "sha256:8ea039387c10202ce304af74def5021e9adc6297067f3441d348d2b633e8166a",
+                "sha256:8f7e66c7113c684c2b3f1c83cdd3376103ee0ce4c49ff80a648643e57fb22218",
+                "sha256:90412f2db8c02a3864cbfc67db0e3dcdbda336acf1c469526d3e869394fe001c",
+                "sha256:92a78853d7280bffb93df0a4a6a2498cba10ee793cc8076ef797ef2f74d107cf",
+                "sha256:989d842dc06dc59feea09e58c74ca3e1678c812a4a8a2a419046d711031f69c7",
+                "sha256:9cb3a6460cdea8fe8194a76de8895707e61ded10ad0be97188cc8463ffa7e3a8",
+                "sha256:9dd8cd1aeb00775f527ec60022004d030ddc51d783d056e3e23e74e623e33726",
+                "sha256:9ed69074a610fad1c2fda66180e7b2edd4d31c53f2d1872bc2d1211563904cd9",
+                "sha256:9edda2df81daa129b25a39b86cb57dfdfe16f7ec15b42b19bfac503360d27a93",
+                "sha256:a2224fa4a4c2ee872886ed00a571f5e967c85e078e8e8c2530a2fb01b3309b88",
+                "sha256:a4f96f0d88accc3dbe4a9025f785ba830f968e21e3e2c6321ccdfc9aef755115",
+                "sha256:aedd5dd8692635813368e558a05266b995d3d020b23e49581ddd5bbe197a8ab6",
+                "sha256:aee22939bb6075e7afededabad1a56a905da0b3c4e3e0c45e75810ebe3a52672",
+                "sha256:b1d464cb8d72bfc1a3adc53305a63a8e0cac6bc8c5a07e8ca190ab8d3faa43c2",
+                "sha256:b8f86dd868d41bea9a5f873ee13bf5551c94cf6bc51baebc6f85075971fe6eea",
+                "sha256:bc6bee759a6bddea5db78d7dcd609397449cb2d2d6587f48f3ca613b19410cfc",
+                "sha256:bea2acdd8ea4275e1278350ced63da0b166421928276c7c8e3f9729d7402a57b",
+                "sha256:bfa832bfa540e5b5c27dcf5de5d82ebc431b82c453a43d141afb1e5d2de025fa",
+                "sha256:c0e6091b157d48cbe37bd67233318dbb53e1e6327d6fc3bb284afd585d141003",
+                "sha256:c3789bd5768ab5618ebf09cef6ec2b35fed88709b104351748a63045f0ff9797",
+                "sha256:c530e1eecd036ecc83c3407f77bb86feb79916d4a33d11394b8234f3bd35b940",
+                "sha256:c811cfcd6a9bf680236c40c6f617187515269ab2912f3d7e8c0174898e2519db",
+                "sha256:c92d73464b886931308ccc45b2744e5968cbaade0b1d6aeb40d8ab537765f5bc",
+                "sha256:cccba051221b916a4f5e538997c45d7d136a5646442b1231b916d0164067ea27",
+                "sha256:cdeabcff45d1c219636ee2e54d852262e5c2e085d6cb476d938aee8d921356b3",
+                "sha256:ced65e5a985398827cc9276b93ef6dfabe0273c23de8c7931339d7e141c2818e",
+                "sha256:d049df610ac811dcffdc147153b414147428567fbbc8be43bb8885f04db39d98",
+                "sha256:dacd995031a01d16eec825bf30802fceb2c3791ef24bcce48fa98ce40918c27b",
+                "sha256:ddf33d97d2f52d89f6e6e7ae66ee35a4d9ca6f36eda89c24591b0c40205a3629",
+                "sha256:ded0fc7d90fe93ae0b18059930086c51e640cdd3baebdc783a695c77f123dcd9",
+                "sha256:e3e0210287329272539eea617830a6a28161fbbd8a3271bf4150ae3e58c5d0e6",
+                "sha256:e6fa2e3e683f34aea77de8112f6483803c96a44fd726d7358b9888ae5bb394ec",
+                "sha256:ea0eb6af8a17fa272f7b98d7bebfab7836a0d62738e16ba380f440fceca2d951",
+                "sha256:ea7f69de383cb47522c9c208aec6dd17697db7875a4674c4af3f8cfdac0bdeae",
+                "sha256:eac5174677da084abf378739dbf4ad245661635f1600edd1221f150b165343f4",
+                "sha256:fc4f7a173a5609631bb0c42c23d12c49df3966f89f496a51d3eb0ec81f4519d6",
+                "sha256:fdb5b3e311d4d4b0eb8b3e8b4d1b0a512713ad7e6a68791d0923d1aec433d919"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==26.1.1"
+            "version": "==26.2.0"
         },
         "pyzstd": {
             "hashes": [
@@ -3082,11 +3440,11 @@
         },
         "qtconsole": {
             "hashes": [
-                "sha256:42d745f3d05d36240244a04e1e1ec2a86d5d9b6edb16dbdef582ccb629e87e0b",
-                "sha256:6b5fb11274b297463706af84dcbbd5c92273b1f619e6d25d08874b0a88516989"
+                "sha256:4c82120a3b53a3d36e3f76e6a1a26ffddf4e1ce2359d56a19889c55e1d73a436",
+                "sha256:c36e0d497a473b67898b96dd38666e645e4594019244263da7b409b84fa2ebb5"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.5.2"
+            "version": "==5.6.0"
         },
         "qtpy": {
             "hashes": [
@@ -3364,11 +3722,11 @@
         },
         "s3fs": {
             "hashes": [
-                "sha256:2ca5de8dc18ad7ad350c0bd01aef0406aa5d0fff78a561f0f710f9d9858abdd0",
-                "sha256:91c1dfb45e5217bd441a7a560946fe865ced6225ff7eb0fb459fe6e601a95ed3"
+                "sha256:63e429bb6b5e814568cacd3f2a8551fc35493e8c418ddfcb44e6f86aa8696ccd",
+                "sha256:ed0b7df8cc20a2b5cefe607b1cf4e860d37c5ca4ac2d68f55464805d75d18710"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==0.4.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.12.1"
         },
         "s3transfer": {
             "hashes": [
@@ -3380,30 +3738,30 @@
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:0828673c5b520e879f2af6a9e99eee0eefea69a2188be1ca68a6121b809055c1",
-                "sha256:0ea5d40c0e3951df445721927448755d3fe1d80833b0b7308ebff5d2a45e6414",
-                "sha256:10e49170691514a94bb2e03787aa921b82dbc507a4ea1f20fd95557862c98dc1",
-                "sha256:154297ee43c0b83af12464adeab378dee2d0a700ccd03979e2b821e7dd7cc1c2",
-                "sha256:161808750c267b77b4a9603cf9c93579c7a74ba8486b1336034c2f1579546d21",
-                "sha256:1bd8d3a19d4bd6dc5a7d4f358c8c3a60934dc058f363c34c0ac1e9e12a31421d",
-                "sha256:1ff4ba34c2abff5ec59c803ed1d97d61b036f659a17f55be102679e88f926fac",
-                "sha256:508907e5f81390e16d754e8815f7497e52139162fd69c4fdbd2dfa5d6cc88915",
-                "sha256:5944ce1faada31c55fb2ba20a5346b88e36811aab504ccafb9f0339e9f780395",
-                "sha256:5f57428de0c900a98389c4a433d4a3cf89de979b3aa24d1c1d251802aa15e44d",
-                "sha256:689b6f74b2c880276e365fe84fe4f1befd6a774f016339c65655eaff12e10cbf",
-                "sha256:781586c414f8cc58e71da4f3d7af311e0505a683e112f2f62919e3019abd3745",
-                "sha256:7b073a27797a283187a4ef4ee149959defc350b46cbf63a84d8514fe16b69855",
-                "sha256:88e0672c7ac21eb149d409c74cc29f1d611d5158175846e7a9c2427bd12b3956",
-                "sha256:909144d50f367a513cee6090873ae582dba019cb3fca063b38054fa42704c3a4",
-                "sha256:97625f217c5c0c5d0505fa2af28ae424bd37949bb2f16ace3ff5f2f81fb4498b",
-                "sha256:9a07f90846313a7639af6a019d849ff72baadfa4c74c778821ae0fad07b7275b",
-                "sha256:b59e3e62d2be870e5c74af4e793293753565c7383ae82943b83383fdcf5cc5c1",
-                "sha256:b5e865e9bd59396220de49cb4a57b17016256637c61b4c5cc81aaf16bc123bbe",
-                "sha256:da3f404e9e284d2b0a157e1b56b6566a34eb2798205cba35a211df3296ab7a74",
-                "sha256:f5b213bc29cc30a89a3130393b0e39c847a15d769d6e59539cd86b75d276b1a7"
+                "sha256:03b6158efa3faaf1feea3faa884c840ebd61b6484167c711548fce208ea09445",
+                "sha256:1ff45e26928d3b4eb767a8f14a9a6efbf1cbff7c05d1fb0f95f211a89fd4f5de",
+                "sha256:299406827fb9a4f862626d0fe6c122f5f87f8910b86fe5daa4c32dcd742139b6",
+                "sha256:2d4cad1119c77930b235579ad0dc25e65c917e756fe80cab96aa3b9428bd3fb0",
+                "sha256:394397841449853c2290a32050382edaec3da89e35b3e03d6cc966aebc6a8ae6",
+                "sha256:3a686885a4b3818d9e62904d91b57fa757fc2bed3e465c8b177be652f4dd37c8",
+                "sha256:3b923d119d65b7bd555c73be5423bf06c0105678ce7e1f558cb4b40b0a5502b1",
+                "sha256:3bed4909ba187aca80580fe2ef370d9180dcf18e621a27c4cf2ef10d279a7efe",
+                "sha256:52788f48b5d8bca5c0736c175fa6bdaab2ef00a8f536cda698db61bd89c551c1",
+                "sha256:57cc1786cfd6bd118220a92ede80270132aa353647684efa385a74244a41e3b1",
+                "sha256:643964678f4b5fbdc95cbf8aec638acc7aa70f5f79ee2cdad1eec3df4ba6ead8",
+                "sha256:6c16d84a0d45e4894832b3c4d0bf73050939e21b99b01b6fd59cbb0cf39163b6",
+                "sha256:757c7d514ddb00ae249832fe87100d9c73c6ea91423802872d9e74970a0e40b9",
+                "sha256:8c412ccc2ad9bf3755915e3908e677b367ebc8d010acbb3f182814524f2e5540",
+                "sha256:b4237ed7b3fdd0a4882792e68ef2545d5baa50aca3bb45aa7df468138ad8f94d",
+                "sha256:c15b1ca23d7c5f33cc2cb0a0d6aaacf893792271cddff0edbd6a40e8319bc113",
+                "sha256:ca64b3089a6d9b9363cd3546f8978229dcbb737aceb2c12144ee3f70f95684b7",
+                "sha256:f60021ec1574e56632be2a36b946f8143bf4e5e6af4a06d85281adc22938e0dd",
+                "sha256:f763897fe92d0e903aa4847b0aec0e68cadfff77e8a0687cabd946c89d17e675",
+                "sha256:f8b0ccd4a902836493e026c03256e8b206656f91fbcc4fde28c57a5b752561f1",
+                "sha256:f932a02c3f4956dfb981391ab24bda1dbd90fe3d628e4b42caef3e041c67707a"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.5.1"
+            "version": "==1.5.2"
         },
         "scipy": {
             "hashes": [
@@ -3456,11 +3814,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:3c08705fadfc8c7c445cf4d98078f0fafb9225775b2b4e8447e40348f82597c0",
-                "sha256:f2bfcce7ae1784d90b04c57c2802e8649e1976530bb25dc72c2b078d3ecf4864"
+                "sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2",
+                "sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==73.0.0"
+            "version": "==75.1.0"
         },
         "shapely": {
             "hashes": [
@@ -3790,11 +4148,11 @@
         },
         "types-python-dateutil": {
             "hashes": [
-                "sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202",
-                "sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b"
+                "sha256:27c8cc2d058ccb14946eebcaaa503088f4f6dbc4fb6093d3d456a49aef2753f6",
+                "sha256:9706c3b68284c25adffc47319ecc7947e5bb86b3773f843c73906fd598bc176e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.9.0.20240316"
+            "version": "==2.9.0.20240906"
         },
         "typing-extensions": {
             "hashes": [
@@ -3837,11 +4195,11 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a",
-                "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"
+                "sha256:4f3ac17b81fba3ce3bd6f4ead2749a72da5929c01774948e243db9ba41df4ff6",
+                "sha256:ce489cac131aa58f4b25e321d6d186171f78e6cb13fafbf32a840cee67733ff4"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.26.3"
+            "version": "==20.26.5"
         },
         "wcwidth": {
             "hashes": [
@@ -3874,19 +4232,19 @@
         },
         "whitebox": {
             "hashes": [
-                "sha256:3479c8127b1928dd0457b6680b9ee4469d0735b2bb220178c678af303d868ef0",
-                "sha256:4322c5a3b42f1f04a3c80640253b672242e0aefb6b6dea70752f00a220f6d7ed"
+                "sha256:4a9fb6da799dc0f99daf139d6dbe573ca12af3785c48708e56728b8372c6a2a7",
+                "sha256:dd9db85bbb687356741afe935dd25286a2b31e7bf87b74c60b655cdf84a9d922"
             ],
             "index": "pypi",
-            "version": "==2.3.4"
+            "version": "==2.3.5"
         },
         "widgetsnbextension": {
             "hashes": [
-                "sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36",
-                "sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474"
+                "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71",
+                "sha256:ffcb67bc9febd10234a362795f643927f4e0c05d9342c727b65d2384f8feacb6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.0.11"
+            "version": "==4.0.13"
         },
         "wrapt": {
             "hashes": [
@@ -3991,11 +4349,11 @@
         },
         "xyzservices": {
             "hashes": [
-                "sha256:58c1bdab4257d2551b9ef91cd48571f77b7c4d2bc45bf5e3c05ac97b3a4d7282",
-                "sha256:fecb2508f0f2b71c819aecf5df2c03cef001c56a4b49302e640f3b34710d25e4"
+                "sha256:68fb8353c9dbba4f1ff6c0f2e5e4e596bb9e1db7f94f4f7dfbcb26e25aa66fde",
+                "sha256:776ae82b78d6e5ca63dd6a94abb054df8130887a4a308473b54a6bd364de8644"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.6.0"
+            "version": "==2024.9.0"
         },
         "y-py": {
             "hashes": [
@@ -4076,6 +4434,104 @@
             ],
             "version": "==0.6.2"
         },
+        "yarl": {
+            "hashes": [
+                "sha256:01a8697ec24f17c349c4f655763c4db70eebc56a5f82995e5e26e837c6eb0e49",
+                "sha256:02da8759b47d964f9173c8675710720b468aa1c1693be0c9c64abb9d8d9a4867",
+                "sha256:04293941646647b3bfb1719d1d11ff1028e9c30199509a844da3c0f5919dc520",
+                "sha256:067b961853c8e62725ff2893226fef3d0da060656a9827f3f520fb1d19b2b68a",
+                "sha256:077da604852be488c9a05a524068cdae1e972b7dc02438161c32420fb4ec5e14",
+                "sha256:09696438cb43ea6f9492ef237761b043f9179f455f405279e609f2bc9100212a",
+                "sha256:0b8486f322d8f6a38539136a22c55f94d269addb24db5cb6f61adc61eabc9d93",
+                "sha256:0ea9682124fc062e3d931c6911934a678cb28453f957ddccf51f568c2f2b5e05",
+                "sha256:0f351fa31234699d6084ff98283cb1e852270fe9e250a3b3bf7804eb493bd937",
+                "sha256:14438dfc5015661f75f85bc5adad0743678eefee266ff0c9a8e32969d5d69f74",
+                "sha256:15061ce6584ece023457fb8b7a7a69ec40bf7114d781a8c4f5dcd68e28b5c53b",
+                "sha256:15439f3c5c72686b6c3ff235279630d08936ace67d0fe5c8d5bbc3ef06f5a420",
+                "sha256:17b5a386d0d36fb828e2fb3ef08c8829c1ebf977eef88e5367d1c8c94b454639",
+                "sha256:18ac56c9dd70941ecad42b5a906820824ca72ff84ad6fa18db33c2537ae2e089",
+                "sha256:1bb2d9e212fb7449b8fb73bc461b51eaa17cc8430b4a87d87be7b25052d92f53",
+                "sha256:1e969fa4c1e0b1a391f3fcbcb9ec31e84440253325b534519be0d28f4b6b533e",
+                "sha256:1fa2e7a406fbd45b61b4433e3aa254a2c3e14c4b3186f6e952d08a730807fa0c",
+                "sha256:2164cd9725092761fed26f299e3f276bb4b537ca58e6ff6b252eae9631b5c96e",
+                "sha256:21a7c12321436b066c11ec19c7e3cb9aec18884fe0d5b25d03d756a9e654edfe",
+                "sha256:238a21849dd7554cb4d25a14ffbfa0ef380bb7ba201f45b144a14454a72ffa5a",
+                "sha256:250e888fa62d73e721f3041e3a9abf427788a1934b426b45e1b92f62c1f68366",
+                "sha256:25861303e0be76b60fddc1250ec5986c42f0a5c0c50ff57cc30b1be199c00e63",
+                "sha256:267b24f891e74eccbdff42241c5fb4f974de2d6271dcc7d7e0c9ae1079a560d9",
+                "sha256:27fcb271a41b746bd0e2a92182df507e1c204759f460ff784ca614e12dd85145",
+                "sha256:2909fa3a7d249ef64eeb2faa04b7957e34fefb6ec9966506312349ed8a7e77bf",
+                "sha256:3257978c870728a52dcce8c2902bf01f6c53b65094b457bf87b2644ee6238ddc",
+                "sha256:327c724b01b8641a1bf1ab3b232fb638706e50f76c0b5bf16051ab65c868fac5",
+                "sha256:3de5292f9f0ee285e6bd168b2a77b2a00d74cbcfa420ed078456d3023d2f6dff",
+                "sha256:3fce4da3703ee6048ad4138fe74619c50874afe98b1ad87b2698ef95bf92c96d",
+                "sha256:3ff6b1617aa39279fe18a76c8d165469c48b159931d9b48239065767ee455b2b",
+                "sha256:400cd42185f92de559d29eeb529e71d80dfbd2f45c36844914a4a34297ca6f00",
+                "sha256:4179522dc0305c3fc9782549175c8e8849252fefeb077c92a73889ccbcd508ad",
+                "sha256:4307d9a3417eea87715c9736d050c83e8c1904e9b7aada6ce61b46361b733d92",
+                "sha256:476e20c433b356e16e9a141449f25161e6b69984fb4cdbd7cd4bd54c17844998",
+                "sha256:489fa8bde4f1244ad6c5f6d11bb33e09cf0d1d0367edb197619c3e3fc06f3d91",
+                "sha256:48a28bed68ab8fb7e380775f0029a079f08a17799cb3387a65d14ace16c12e2b",
+                "sha256:48dfd117ab93f0129084577a07287376cc69c08138694396f305636e229caa1a",
+                "sha256:4973eac1e2ff63cf187073cd4e1f1148dcd119314ab79b88e1b3fad74a18c9d5",
+                "sha256:498442e3af2a860a663baa14fbf23fb04b0dd758039c0e7c8f91cb9279799bff",
+                "sha256:501c503eed2bb306638ccb60c174f856cc3246c861829ff40eaa80e2f0330367",
+                "sha256:504cf0d4c5e4579a51261d6091267f9fd997ef58558c4ffa7a3e1460bd2336fa",
+                "sha256:61a5f2c14d0a1adfdd82258f756b23a550c13ba4c86c84106be4c111a3a4e413",
+                "sha256:637c7ddb585a62d4469f843dac221f23eec3cbad31693b23abbc2c366ad41ff4",
+                "sha256:66b63c504d2ca43bf7221a1f72fbe981ff56ecb39004c70a94485d13e37ebf45",
+                "sha256:67459cf8cf31da0e2cbdb4b040507e535d25cfbb1604ca76396a3a66b8ba37a6",
+                "sha256:688654f8507464745ab563b041d1fb7dab5d9912ca6b06e61d1c4708366832f5",
+                "sha256:6907daa4b9d7a688063ed098c472f96e8181733c525e03e866fb5db480a424df",
+                "sha256:69721b8effdb588cb055cc22f7c5105ca6fdaa5aeb3ea09021d517882c4a904c",
+                "sha256:6d23754b9939cbab02c63434776df1170e43b09c6a517585c7ce2b3d449b7318",
+                "sha256:7175a87ab8f7fbde37160a15e58e138ba3b2b0e05492d7351314a250d61b1591",
+                "sha256:72bf26f66456baa0584eff63e44545c9f0eaed9b73cb6601b647c91f14c11f38",
+                "sha256:74db2ef03b442276d25951749a803ddb6e270d02dda1d1c556f6ae595a0d76a8",
+                "sha256:750f656832d7d3cb0c76be137ee79405cc17e792f31e0a01eee390e383b2936e",
+                "sha256:75e0ae31fb5ccab6eda09ba1494e87eb226dcbd2372dae96b87800e1dcc98804",
+                "sha256:768ecc550096b028754ea28bf90fde071c379c62c43afa574edc6f33ee5daaec",
+                "sha256:7d51324a04fc4b0e097ff8a153e9276c2593106a811704025bbc1d6916f45ca6",
+                "sha256:7e975a2211952a8a083d1b9d9ba26472981ae338e720b419eb50535de3c02870",
+                "sha256:8215f6f21394d1f46e222abeb06316e77ef328d628f593502d8fc2a9117bde83",
+                "sha256:8258c86f47e080a258993eed877d579c71da7bda26af86ce6c2d2d072c11320d",
+                "sha256:8418c053aeb236b20b0ab8fa6bacfc2feaaf7d4683dd96528610989c99723d5f",
+                "sha256:87f020d010ba80a247c4abc335fc13421037800ca20b42af5ae40e5fd75e7909",
+                "sha256:884eab2ce97cbaf89f264372eae58388862c33c4f551c15680dd80f53c89a269",
+                "sha256:8a336eaa7ee7e87cdece3cedb395c9657d227bfceb6781295cf56abcd3386a26",
+                "sha256:8aef1b64da41d18026632d99a06b3fefe1d08e85dd81d849fa7c96301ed22f1b",
+                "sha256:8aef97ba1dd2138112890ef848e17d8526fe80b21f743b4ee65947ea184f07a2",
+                "sha256:8ed653638ef669e0efc6fe2acb792275cb419bf9cb5c5049399f3556995f23c7",
+                "sha256:9361628f28f48dcf8b2f528420d4d68102f593f9c2e592bfc842f5fb337e44fd",
+                "sha256:946eedc12895873891aaceb39bceb484b4977f70373e0122da483f6c38faaa68",
+                "sha256:94d0caaa912bfcdc702a4204cd5e2bb01eb917fc4f5ea2315aa23962549561b0",
+                "sha256:964a428132227edff96d6f3cf261573cb0f1a60c9a764ce28cda9525f18f7786",
+                "sha256:999bfee0a5b7385a0af5ffb606393509cfde70ecca4f01c36985be6d33e336da",
+                "sha256:a08ea567c16f140af8ddc7cb58e27e9138a1386e3e6e53982abaa6f2377b38cc",
+                "sha256:a28b70c9e2213de425d9cba5ab2e7f7a1c8ca23a99c4b5159bf77b9c31251447",
+                "sha256:a34e1e30f1774fa35d37202bbeae62423e9a79d78d0874e5556a593479fdf239",
+                "sha256:a4264515f9117be204935cd230fb2a052dd3792789cc94c101c535d349b3dab0",
+                "sha256:a7915ea49b0c113641dc4d9338efa9bd66b6a9a485ffe75b9907e8573ca94b84",
+                "sha256:aac44097d838dda26526cffb63bdd8737a2dbdf5f2c68efb72ad83aec6673c7e",
+                "sha256:b91044952da03b6f95fdba398d7993dd983b64d3c31c358a4c89e3c19b6f7aef",
+                "sha256:ba444bdd4caa2a94456ef67a2f383710928820dd0117aae6650a4d17029fa25e",
+                "sha256:c2dc4250fe94d8cd864d66018f8344d4af50e3758e9d725e94fecfa27588ff82",
+                "sha256:c35f493b867912f6fda721a59cc7c4766d382040bdf1ddaeeaa7fa4d072f4675",
+                "sha256:c92261eb2ad367629dc437536463dc934030c9e7caca861cc51990fe6c565f26",
+                "sha256:ce928c9c6409c79e10f39604a7e214b3cb69552952fbda8d836c052832e6a979",
+                "sha256:d95b52fbef190ca87d8c42f49e314eace4fc52070f3dfa5f87a6594b0c1c6e46",
+                "sha256:dae7bd0daeb33aa3e79e72877d3d51052e8b19c9025ecf0374f542ea8ec120e4",
+                "sha256:e286580b6511aac7c3268a78cdb861ec739d3e5a2a53b4809faef6b49778eaff",
+                "sha256:e4b53f73077e839b3f89c992223f15b1d2ab314bdbdf502afdc7bb18e95eae27",
+                "sha256:e8f63904df26d1a66aabc141bfd258bf738b9bc7bc6bdef22713b4f5ef789a4c",
+                "sha256:f3a6d90cab0bdf07df8f176eae3a07127daafcf7457b997b2bf46776da2c7eb7",
+                "sha256:f41fa79114a1d2eddb5eea7b912d6160508f57440bd302ce96eaa384914cd265",
+                "sha256:f46f81501160c28d0c0b7333b4f7be8983dbbc161983b6fb814024d1b4952f79",
+                "sha256:f61db3b7e870914dbd9434b560075e0366771eecbe6d2b5561f5bc7485f39efd"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.11.1"
+        },
         "ypy-websocket": {
             "hashes": [
                 "sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff",
@@ -4103,11 +4559,11 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31",
-                "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"
+                "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350",
+                "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.20.0"
+            "version": "==3.20.2"
         }
     },
     "develop": {
@@ -4136,19 +4592,19 @@
         },
         "executing": {
             "hashes": [
-                "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147",
-                "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"
+                "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf",
+                "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.0.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.1.0"
         },
         "ipython": {
             "hashes": [
-                "sha256:1cec0fbba8404af13facebe83d04436a7434c7400e59f47acf467c64abd0956c",
-                "sha256:e6b347c27bdf9c32ee9d31ae85defc525755a1869f14057e900675b9e8d6e6ff"
+                "sha256:0b99a2dc9f15fd68692e898e5568725c6d49c527d36a9fb5960ffbdeaa82ff7e",
+                "sha256:f68b3cb8bde357a5d7adc9598d57e22a45dfbea19eb6b98286fa3b288c9cd55c"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==8.26.0"
+            "version": "==8.27.0"
         },
         "jedi": {
             "hashes": [
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 900119b10..f9ba717e3 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,29 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.9.0 - 2024-09-25 - [PR#1291](https://github.com/NOAA-OWP/inundation-mapping/pull/1291)
+
+Changes Docker base image to `gdal:ubuntu-small` in order to avoid JDK from being carried over in the base image and triggering security vulnerabilities.
+
+This PR incorporates a number of changes to the Docker environment:
+- Changes Docker base image to `gdal:ubuntu-small` in order to avoid JDK from being carried over in the base image and triggering security vulnerabilities. Resolves #1278.
+- Upgrades `fiona` and `jupterlab`. Closes #1270 and closes #1290.
+- Eliminates `whitebox` downloading during `fim_pipeline`. Resolves #1209 and closes #1293.
+
+During testing, it was discovered that many files which are not in the `src` directory, can no longer see the `src\utils` files. Adjusting the dockerfile to add extra values to the PYTHONPATH variable fixed it.
+
+Note: This triggers new docker images to be made.
+
+### Changes
+
+- `Dockerfile`: Changes base image to `gdal:ubuntu-small-3.8.4` and removes code related to JDK
+- `Pipfile` and `Pipfile.lock`: Upgrades `fiona`, `jupyterlab`, and `whitebox`
+- `fim_pre_processing`: Removes `WBT_PATH` assignment
+- `src/`
+    - `agreedem.py` and `unique_pixel_and_allocation.py`: sets `whitebox_dir` to `WBT_PATH`
+
+<br/><br/>
+
 ## v4.5.8.0 - 2024-09-13 - [PR#1165](https://github.com/NOAA-OWP/inundation-mapping/pull/1165)
 
 This PR was originally intended to get Alaska HUCs incorporated into CatFIM, but there were a very, very large array of problems and the tool was unable to run. We have made some major modifications and many more will come in the near future. There are partial hooks and commented code for Alaska integration, but temporarily disabled are included and will be handled by a separate branch / PR.
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index d89c554ce..bd556940c 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -122,7 +122,7 @@ fi
 # outputsDir & workDir come from the Dockerfile
 outputDestDir=$outputsDir/$runName
 tempRunDir=$workDir/$runName
-export WBT_PATH=${tempRunDir}/whitebox_temp
+# export WBT_PATH=${tempRunDir}/whitebox_temp
 
 # default values
 if [ "$envFile" = "" ]; then envFile=/$projectDir/config/params_template.env; fi
diff --git a/src/agreedem.py b/src/agreedem.py
index b38c7f359..7384c1813 100755
--- a/src/agreedem.py
+++ b/src/agreedem.py
@@ -50,6 +50,7 @@ def agreedem(
     # Set wbt envs
     wbt = whitebox.WhiteboxTools()
     wbt.set_verbose_mode(False)
+    wbt.set_whitebox_dir(os.environ.get("WBT_PATH"))
 
     # ------------------------------------------------------------------
     # 1. From Hellweger documentation: Compute the vector grid
diff --git a/src/unique_pixel_and_allocation.py b/src/unique_pixel_and_allocation.py
index e5c71e61d..14c3f68fa 100755
--- a/src/unique_pixel_and_allocation.py
+++ b/src/unique_pixel_and_allocation.py
@@ -34,6 +34,7 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels):
     # Set wbt envs
     wbt = whitebox.WhiteboxTools()
     wbt.set_verbose_mode(False)
+    wbt.set_whitebox_dir(os.environ.get("WBT_PATH"))
 
     workspace = os.path.dirname(unique_stream_pixels)
     base = os.path.basename(unique_stream_pixels)
diff --git a/tools/aggregate_mannings_calibration.py b/tools/aggregate_mannings_calibration.py
index d447c696e..067cd4de2 100755
--- a/tools/aggregate_mannings_calibration.py
+++ b/tools/aggregate_mannings_calibration.py
@@ -7,6 +7,8 @@
 
 import pandas as pd
 
+from utils.shared_functions import compute_stats_from_contingency_table
+
 
 def aggregate_parameter_sets(huc_list_path, calibration_stat_folder, summary_file, mannings_json):
     outfolder = os.path.dirname(summary_file)
@@ -70,7 +72,6 @@ def aggregate_parameter_sets(huc_list_path, calibration_stat_folder, summary_fil
     mannings_summary_table.to_csv(summary_file, index=False)
 
     ## calculate optimal parameter set
-    from utils.shared_functions import compute_stats_from_contingency_table
 
     true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = (
         0,
diff --git a/tools/eval_plots.py b/tools/eval_plots.py
index e534582a5..2af666d2b 100644
--- a/tools/eval_plots.py
+++ b/tools/eval_plots.py
@@ -4,7 +4,6 @@
 import glob
 import os
 import re
-import sys
 from pathlib import Path
 
 import geopandas as gpd
diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py
index f6a8c77f2..f95adfa29 100755
--- a/tools/generate_nws_lid.py
+++ b/tools/generate_nws_lid.py
@@ -2,7 +2,6 @@
 
 import argparse
 import os
-import sys
 from collections import defaultdict
 from pathlib import Path
 
diff --git a/tools/synthesize_test_cases.py b/tools/synthesize_test_cases.py
index dfe0dac7e..ab734bf7f 100755
--- a/tools/synthesize_test_cases.py
+++ b/tools/synthesize_test_cases.py
@@ -1,12 +1,10 @@
 #!/usr/bin/env python3
 
 import argparse
-import ast
 import csv
 import json
 import os
 import re
-import signal
 import sys
 import traceback
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait

From 7e2009181ef6e23a93235f67786c3c4b1165970d Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 25 Sep 2024 20:26:55 +0000
Subject: [PATCH 351/359] v4.5.10.0 PR: Reload 3dep DEMs and pre-clips (#1301)

---
 CITATION.cff                                  |   6 +-
 data/usgs/acquire_and_preprocess_3dep_dems.py | 113 ++++++----
 data/wbd/generate_pre_clip_fim_huc8.py        |  20 +-
 data/wbd/preprocess_wbd.py                    |  16 +-
 docs/CHANGELOG.md                             |  40 ++++
 fim_post_processing.sh                        |   6 +-
 src/bash_variables.env                        |  10 +-
 tools/rating_curve_comparison.py              | 213 +++++++++++-------
 8 files changed, 275 insertions(+), 149 deletions(-)

diff --git a/CITATION.cff b/CITATION.cff
index cfa1f10b7..e122fc9cf 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,8 +1,8 @@
 cff-version: 1.2.0
-message: "If you use this software, please cite it as below."
+message: "If you use this software, please cite it as below. To ensure you have latest and correct version, please look at the changelog.md at 'https://github.com/NOAA-OWP/inundation-mapping/blob/dev/docs/CHANGELOG.md'"
 authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.4.2.1
-date-released: 2023
+version: 4.5.10.0
+date-released: 2024
diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py
index 56f21aee8..5a1fac2d5 100755
--- a/data/usgs/acquire_and_preprocess_3dep_dems.py
+++ b/data/usgs/acquire_and_preprocess_3dep_dems.py
@@ -4,12 +4,11 @@
 import glob
 import logging
 import os
-import shutil
 import subprocess
 import sys
 import traceback
 from concurrent.futures import ProcessPoolExecutor, as_completed, wait
-from datetime import datetime
+from datetime import datetime, timezone
 
 import geopandas as gpd
 import pandas as pd
@@ -27,6 +26,14 @@
     - Add input args for resolution size, which means URL and block size also hve to be parameterized.
 '''
 
+"""
+*****************************
+
+### IMPORTANT: Sep 13, 2024: FIM uses vrt's primariy for DEMs but this tool only downloads and preps the DEMs but does
+not create the vrt. That is done using the create_vrt_file.py tool.
+
+"""
+
 
 # local constants (until changed to input param)
 # This URL is part of a series of vrt data available from USGS via an S3 Bucket.
@@ -59,13 +66,20 @@ def acquire_and_preprocess_3dep_dems(
 
     Notes:
         - It really can be used for any huc size or any extent poly as long as it is 10m.
+
         - As this is a very low use tool, most values such as the USGS vrt path, output
           file names, etc are all hardcoded
+
         - Currently there is no tools to extract the WBD HUC8's that are applicable. You will want
           a gkpg for each download extent (ie. HUC8, HUC6, whatever. Make a folder of each extent
           file you want. ie) a folder of WBD HUC8's. One gkpg per HUC8 (or whatever size you like)
+
         - When we originally used this tool for CONUS+, we loaded them in HUC6's, but now
           with selected HUC8's for South Alaska, we will use input extent files as WBD HUC8.
+
+        - We run conus at HUC6 as there is often a number of download fails requiring our tool to run in -r (repair)
+          mode to fix it. HUC8's are more files and allow for more possible errors. (usually just communication fails)
+
         - We have a separate tool to create a VRT of any folder of rasters.
 
     Parameters
@@ -76,8 +90,7 @@ def acquire_and_preprocess_3dep_dems(
             ie) /data/inputs/wbd/HUC6
 
         - target_output_folder_path (str):
-            The output location of the new 3dep dem files. When the param is not submitted,
-            it will be sent to /data/input/usgs/3dep_dems/10m/.
+            The output location of the new 3dep dem files.
 
         - number_of_jobs (int):
             This program supports multiple procs if multiple procs/cores are available.
@@ -95,8 +108,6 @@ def acquire_and_preprocess_3dep_dems(
 
         - target_projection (String)
             Projection of the output DEMS and polygons (if included)
-
-
     '''
     # -------------------
     # Validation
@@ -109,12 +120,31 @@ def acquire_and_preprocess_3dep_dems(
             ' value accordingly.'
         )
 
+    if number_of_jobs > 15:
+        print("")
+        print(f"You have asked for {number_of_jobs} jobs\n")
+        print("For each core, it opens up another extenal connection.")
+        print(" But if you try to download more files simultaneously, many files an be partially downloaded")
+        print(
+            " with no notification or warning. It is recommended to slow down the job numbers to ensure stability."
+        )
+        print("")
+        print(" Type 'CONTINUE' if you want to keep your original job numbers")
+        print("      'MAX' if you want change your job count to 15")
+        print("      any other value to abort.")
+        resp = input(">> ").lower()
+
+        if resp == "max":
+            number_of_jobs = 15
+        elif resp != "continue":
+            print("Program aborted")
+            sys.exit(1)
+        print(f".. Continuing with {number_of_jobs} jobs")
+    print("")
+
     if not os.path.exists(extent_file_path):
         raise ValueError(f'extent_file_path value of {extent_file_path}' ' not set to a valid path')
 
-    if (target_output_folder_path is None) or (target_output_folder_path == ""):
-        target_output_folder_path = os.environ['usgs_3dep_dems_10m']
-
     if not os.path.exists(target_output_folder_path):
         # It is ok if the child diretory does not exist, but the parent folder must
         # parent directory
@@ -125,28 +155,7 @@ def acquire_and_preprocess_3dep_dems(
                 f"For the output path of {target_output_folder_path}, the child directory"
                 " need not exist but the parent folder must."
             )
-        os.makedir(target_output_folder_path)
-
-    else:  # path exists
-        if repair is False:
-            file_list = os.listdir(target_output_folder_path)
-            if len(file_list) > 0:
-                print()
-                msg = (
-                    f"The target output folder of {target_output_folder_path} appears to not be empty.\n\n"
-                    "Do you want to empty the folder first?\n"
-                    "  -- Type 'overwrite' if you want to empty the folder and continue.\n"
-                    "  -- Type any other value to abort and stop the program.\n"
-                )
-                print(msg)
-                resp = input(" ?=").lower()
-                if resp == "overwrite":
-                    shutil.rmtree(target_output_folder_path)
-                    os.mkdir(target_output_folder_path)
-                else:
-                    print("Program stopped\n")
-                    sys.exit(0)
-        # no else:
+        os.makedirs(target_output_folder_path, exist_ok=True)
 
     # I don't need the crs_number for now
     crs_is_valid, err_msg, crs_number = val.is_valid_crs(target_projection)
@@ -155,7 +164,7 @@ def acquire_and_preprocess_3dep_dems(
 
     # -------------------
     # setup logs
-    start_time = datetime.utcnow()
+    start_time = datetime.now(timezone.utc)
     fh.print_start_header('Loading 3dep dems', start_time)
 
     # print(f"Downloading to {target_output_folder_path}")
@@ -179,7 +188,7 @@ def acquire_and_preprocess_3dep_dems(
     if skip_polygons is False:
         polygonize(target_output_folder_path)
 
-    end_time = datetime.utcnow()
+    end_time = datetime.now(timezone.utc)
     fh.print_end_header('Loading 3dep dems complete', start_time, end_time)
 
     print()
@@ -426,7 +435,7 @@ def polygonize(target_output_folder_path):
 
 
 def __setup_logger(output_folder_path):
-    start_time = datetime.utcnow()
+    start_time = datetime.now(timezone.utc)
     file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
     log_file_name = f"3Dep_downloaded-{file_dt_string}.log"
 
@@ -453,16 +462,34 @@ def __setup_logger(output_folder_path):
             -t /data/inputs/3dep_dems/10m_South_Alaska/
             -j 20
 
+    or
+        python3 /foss_fim/data/usgs/acquire_and_preprocess_3dep_dems.py
+            -e /data/inputs/wbd/HUC6_ESPG_5070/
+            -t /data/inputs/usgs/3dep_dems/10m_5070/20240916 -r -j 15
+
     Notes:
+      - There is alot to know, so read the notes in the functions above.
+
+      - It is very common for not all DEMs to not all download correctly on each pass.
+        Review the output files and the logs so you know which are missing. Delete the ones in the outputs
+        that are in error. Then run the tool again wihth the -r flag (repair) which will fill in the wholes
+
+        This is also why we run it at HUC6 as it is easier to trace for failed files. We get alot of
+        communication error during downloads.
+
       - This is a very low use tool. So for now, this only can load 10m (1/3 arc second) and is using
         hardcoded paths for the wbd gpkg to be used for clipping (no buffer for now).
         Also hardcoded usgs 3dep urls, etc.  Minor
         upgrades can easily be made for different urls, output folder paths, huc units, etc
         as/if needed (command line params)
+
       - The output path can be adjusted in case of a test reload of newer data for 3dep.
         The default is /data/input/usgs/3dep_dems/10m/
+
       - Each output file will be the name of the input poly plus "_dem.tif". ie) if the wbd gpkg
         is named named "HUC8_12090301", then the output file name will be "HUC8_12090301_dem.tif"
+        Or depends what file name you sent in for the boundary: ie) HUC6_120903
+
       - While you can (and should use more than one job number (if manageable by your server)),
         this tool is memory intensive and needs more RAM then it needs cores / cpus. Go ahead and
         anyways and increase the job number so you are getting the most out of your RAM. Or
@@ -491,6 +518,14 @@ def __setup_logger(output_folder_path):
         required=True,
     )
 
+    parser.add_argument(
+        '-t',
+        '--target_output_folder_path',
+        help='REQUIRED: location of where the 3dep files will be saved.',
+        required=True,
+        default='',
+    )
+
     parser.add_argument(
         '-j',
         '--number_of_jobs',
@@ -504,20 +539,12 @@ def __setup_logger(output_folder_path):
         '-rp',
         '--repair',
         help='OPTIONAL: If included, it process only HUCs missing output DEMs or if the output DEM'
-        ' is too small (under 10 MB), which does happen.',
+        ' is too small (under 10 MB), which does happen. Read all inline notes about this feature',
         required=False,
         action='store_true',
         default=False,
     )
 
-    parser.add_argument(
-        '-t',
-        '--target_output_folder_path',
-        help='OPTIONAL: location of where the 3dep files will be saved.',
-        required=False,
-        default='',
-    )
-
     parser.add_argument(
         '-sp',
         '--skip_polygons',
diff --git a/data/wbd/generate_pre_clip_fim_huc8.py b/data/wbd/generate_pre_clip_fim_huc8.py
index 22e3a6022..564addfb2 100755
--- a/data/wbd/generate_pre_clip_fim_huc8.py
+++ b/data/wbd/generate_pre_clip_fim_huc8.py
@@ -178,13 +178,12 @@ def pre_clip_hucs_from_wbd(outputs_dir, huc_list, number_of_jobs, overwrite):
     '''
 
     # Validation
-    total_cpus_available = os.cpu_count()
+    total_cpus_available = os.cpu_count() - 2
     if number_of_jobs > total_cpus_available:
         print(
             f'Provided: -j {number_of_jobs}, which is greater than than amount of available cpus -2: '
             f'{total_cpus_available - 2} will be used instead.'
         )
-        number_of_jobs = total_cpus_available - 2
 
     # Read in huc_list file and turn into a list data structure
     if os.path.exists(huc_list):
@@ -512,13 +511,25 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
 
 
 if __name__ == '__main__':
+
+    # NOTE: Super important: Make sure the bash_variables are correct before doing pre-clip.
+    # It pulls a wide number of values from there.
+    # Especially if you can a diretory for a new data load.
+    #     ie) DEMS at data/inputs/3dep_dems/10m_5070/20240916/
+    #
+    # You have to run this twice, once for Alaska and once for CONUS
+    # but make sure put both results the same folder
+    # and you will need to submit the two HUC lists
+    # SouthernAlaska_HUC8.lst
+    # included_huc8.lst
+
     parser = argparse.ArgumentParser(
         description='This script gets WBD layer, calls the clip_vectors_to_wbd.py script, and clips the wbd. '
         'A plethora gpkg files per huc are generated (see args to subset_vector_layers), and placed within '
         'the output directory specified as the <outputs_dir> argument.',
         usage='''
             ./generate_pre_clip_fim_huc8.py
-                -n /data/inputs/pre_clip_huc8/24_3_20
+                -n /data/inputs/pre_clip_huc8/20240927
                 -u /data/inputs/huc_lists/included_huc8_withAlaska.lst
                 -j 6
                 -o
@@ -529,9 +540,10 @@ def huc_level_clip_vectors_to_wbd(huc, outputs_dir):
         '-n',
         '--outputs_dir',
         help='Directory to output all of the HUC level .gpkg files. Use the format: '
-        '<year_month_day> (i.e. September 26, 2023 would be 23_09_26)',
+        '<year_month_day> (i.e. September 26, 2024 would be 20240926)',
     )
     parser.add_argument('-u', '--huc_list', help='List of HUCs to genereate pre-clipped vectors for.')
+
     parser.add_argument(
         '-j',
         '--number_of_jobs',
diff --git a/data/wbd/preprocess_wbd.py b/data/wbd/preprocess_wbd.py
index 76d46d829..6461bb646 100755
--- a/data/wbd/preprocess_wbd.py
+++ b/data/wbd/preprocess_wbd.py
@@ -11,6 +11,9 @@
 
 gpd.options.io_engine = "pyogrio"
 
+# NOTE:
+# Sep 2024. This file might be deprecated as no code calls it
+
 
 def clip_wbd_to_dem_domain(dem: str, wbd_in: str, wbd_out: str, huc_level: int):
     """
@@ -43,6 +46,16 @@ def clip_wbd_to_dem_domain(dem: str, wbd_in: str, wbd_out: str, huc_level: int):
 
 
 if __name__ == '__main__':
+
+    # Example:
+    # preprocess_wbd.py -d /data/inputs/3dep_dems/10m_5070/20240916//HUC6_dem_domain.gpkg
+    #  -w /data/inputs/wbd/WBD_National_EPSG_5070.gpkg
+    #  -o /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
+    #  -l 8
+
+    # WATCH FOR Alaska as well.  During the 3dep download of 20240916, it did not include
+    # Alaska. That one is in data/inputs/3dep_dems/10m_South_Alaska/20240912/
+
     parser = argparse.ArgumentParser(description='Clip WBD to DEM domain')
     parser.add_argument('-d', '--dem', help='Path to DEM', type=str, required=True)
     parser.add_argument('-w', '--wbd-in', help='Input WBD filename', type=str, required=True)
@@ -52,6 +65,3 @@ def clip_wbd_to_dem_domain(dem: str, wbd_in: str, wbd_out: str, huc_level: int):
     args = vars(parser.parse_args())
 
     clip_wbd_to_dem_domain(**args)
-
-    # Example:
-    # preprocess_wbd.py -d /data/inputs/3dep_dems/10m_5070/HUC6_dem_domain.gpkg -w /data/inputs/wbd/WBD_National_EPSG_5070.gpkg -o /data/inputs/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -l 8
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f9ba717e3..7fbf8a591 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,46 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.10.0 - 2024-09-25 - [PR#1301](https://github.com/NOAA-OWP/inundation-mapping/pull/1301)
+
+A reload of all 3Dep DEMs from USGS was performed to refresh our data.
+
+`acquire_and_preprocess_3dep_dems.py` had to be run twice, one for Alaska and once for the rest to two different folder. This is due to different CRS's. Eventually, we could merge these into one run. This also meant two separate vrt runs / files. 
+
+This also triggered a new set of pre-clips for both AK and CONUS+ but the outputs can/were put into the same folder, so fim_pipeline looks in one common pre-clip folder.
+
+Other minor adjustment include:
+- A change to chmod (permissions) files / folder for the logging folders. After careful re-analysis, it was discovered there was some duplication. 
+- Added a simple duration system to the sierra test system, `rating_curve_comparions.py`. This was added as it is expected to be used soon for a  full BED/Production.  The fix focuses purely on duration, but a test did detect a possible pre-existing logic problem. A separate card will be created for that.
+
+Note:
+The root folder for DEM is being changed from:
+    /inputs/3dep_dems/....   to  
+    /inputs/dems/3dep_dems/....
+    This recognizes other DEMs that may be coming in the near future.
+    The same sub-folder patterns have not be changed.
+    No attempts will be made at this time to move older files, only new incoming from this PR.
+
+### Changes
+- `CITATION.cff`: has not be updated for a very long time.
+- `fim_post_processing.sh`: Update to file/folder permissions.
+- `data`
+    - `usgs\acquire_and_preprocesss_3dep_dem.pys
+        - Minor text updates and updated datatime.now patterns as the old ones are not deprecated
+        - An adjustment to how number of jobs are handled. The system dis-likes too many multi-procs due to open network connections to the source.
+        - Change the target output folder from optional to required.
+    - `wbd`
+        - `generate_pre_clip_from_huc8.py`: 
+            - Minor text updates
+        - `preprocess_wbd.py`
+            - Minor text updates
+- `src\base_variables.env`: Changes to variables to reflect new dems and pre-clip paths.
+- `tools\rating_curve_comparisons.py`
+    - Added duration system as mentioned above.
+
+<br/><br/>
+
+
 ## v4.5.9.0 - 2024-09-25 - [PR#1291](https://github.com/NOAA-OWP/inundation-mapping/pull/1291)
 
 Changes Docker base image to `gdal:ubuntu-small` in order to avoid JDK from being carried over in the base image and triggering security vulnerabilities.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index 7dd2a7608..e866f3c9d 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -266,10 +266,8 @@ Tcount
 
 l_echo $startDiv"Resetting Permissions"
 Tstart
-    find $outputDestDir/logs/ -type d -exec chmod -R 777 {} +
-    find $outputDestDir/branch_errors/ -type d -exec chmod -R 777 {} +
-    find $outputDestDir/unit_errors/ -type d -exec chmod -R 777 {} +
-    find $outputDestDir -type f -exec chmod 777 {} +  # just root level files
+    # super slow to change chmod on the log folder. Not really manditory anyways
+    find $outputDestDir -maxdepth 1 -type f -exec chmod 666 {} +  # just root level files
 Tcount
 
 
diff --git a/src/bash_variables.env b/src/bash_variables.env
index b1318f4e5..02ec3e663 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -7,12 +7,12 @@ export                             ALASKA_CRS=EPSG:3338
 
 # NOTE: $inputsDir is defined in Dockerfile
 
-export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/20240702
+export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/20240917
 
-export                              input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
-export                       input_DEM_Alaska=${inputsDir}/3dep_dems/10m_South_Alaska/23_11_07/FIM_3dep_dem_South_Alask_10m.vrt
-export                       input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
-export                input_DEM_domain_Alaska=${inputsDir}/3dep_dems/10m_South_Alaska/23_11_07/DEM_Domain.gpkg
+export                              input_DEM=${inputsDir}/dems/3dep_dems/10m_5070/20240916/hand_seamless_3dep_dem_10m_5070.vrt
+export                       input_DEM_Alaska=${inputsDir}/dems/3dep_dems/10m_South_Alaska/20240912/FIM_3dep_dem_South_Alaska_10m.vrt
+export                       input_DEM_domain=${inputsDir}/dems/3dep_dems/10m_5070/20240916/DEM_Domain.gpkg
+export                input_DEM_domain_Alaska=${inputsDir}/dems/3dep_dems/10m_South_Alaska/20240912/DEM_Domain.gpkg
 export                    input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
 export                              input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
 export                       input_NLD_Alaska=${inputsDir}/nld_vectors/System_Routes_NLDFS_3338_230314.gpkg
diff --git a/tools/rating_curve_comparison.py b/tools/rating_curve_comparison.py
index a57b26fc1..277375fbf 100755
--- a/tools/rating_curve_comparison.py
+++ b/tools/rating_curve_comparison.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import datetime as dt
 import logging
 import os
 import re
@@ -1205,6 +1206,18 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
 
 
 if __name__ == '__main__':
+
+    """
+    Sample Usage:
+    python3 /foss_fim/tools/rating_curve_comparison.py
+        -fim_dir data/previous_fim/hand_4_5_8_0/
+        -output_dir data/fim_performance/hand_4_5_8_0/rating_curve_comparison/
+        -gages /data/inputs/usgs_gages/usgs_rating_curves.csv
+        -flows /data/inputs/rating_curve/nwm_recur_flows/
+        -catfim /data/inputs/usgs_gages/catfim_flows_cms.csv
+        -j 40
+    """
+
     parser = argparse.ArgumentParser(
         description='generate rating curve plots and tables for FIM and USGS gages'
     )
@@ -1214,6 +1227,9 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     )
     parser.add_argument('-gages', '--usgs-gages-filename', help='USGS rating curves', required=True, type=str)
     parser.add_argument('-flows', '--nwm-flow-dir', help='NWM recurrence flows dir', required=True, type=str)
+
+    # TODO Sep 2024: catfim_flows_cms.csv should be renamed as it which made no sense.
+    #    It has nothing to do with catfim
     parser.add_argument(
         '-catfim', '--catfim-flows-filename', help='Categorical FIM flows file', required=True, type=str
     )
@@ -1272,104 +1288,127 @@ def evaluate_results(sierra_results=[], labels=[], save_location=''):
     else:
         stat_gages = None
 
-    # Make sure that location_id is the only -group when using -pnts
-    assert not stat_gages or (
-        stat_gages and (not stat_groups or stat_groups == ['location_id'])
-    ), "location_id is the only acceptable stat_groups argument when producting an output GPKG"
-    # Make sure that the -pnts flag is used with the -eval flag
-    assert not eval or (eval and stat_gages), "You must use the -pnts flag with the -eval flag"
-    procs_list = []
-
-    plots_dir = join(output_dir, 'plots')
-    os.makedirs(plots_dir, exist_ok=True)
-    tables_dir = join(output_dir, 'tables')
-    os.makedirs(tables_dir, exist_ok=True)
-
-    # Check age of gages csv and recommend updating if older than 30 days.
-    print(check_file_age(usgs_gages_filename))
-
+    start_time = dt.datetime.now()
+    dt_string = dt.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    log_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
+    print("==========================================================================")
     # Open log file
     # Create log output
     level = (
         logging.INFO
     )  # using WARNING level to avoid benign? info messages ("Failed to auto identify EPSG: 7")
     format = '  %(message)s'
+    log_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S")
     handlers = [
-        logging.FileHandler(os.path.join(output_dir, 'rating_curve_comparison.log')),
+        logging.FileHandler(os.path.join(output_dir, f'rating_curve_comparison_{log_dt_string}.log')),
         logging.StreamHandler(),
     ]
     logging.basicConfig(level=level, format=format, handlers=handlers)
 
-    merged_elev_table = []
-    huc_list = [huc for huc in os.listdir(fim_dir) if re.search(r"^\d{6,8}$", huc)]
-    for huc in huc_list:
-        elev_table_filename = join(fim_dir, huc, 'usgs_elev_table.csv')
-        branches_folder = join(fim_dir, huc, 'branches')
-        usgs_recurr_stats_filename = join(tables_dir, f"usgs_interpolated_elevation_stats_{huc}.csv")
-        nwm_recurr_data_filename = join(tables_dir, f"nwm_recurrence_flow_elevations_{huc}.csv")
-        rc_comparison_plot_filename = join(plots_dir, f"FIM-USGS_rating_curve_comparison_{huc}.png")
-
-        if isfile(elev_table_filename):
-            procs_list.append(
-                [
+    logging.info(".. (Sierra Test) / rating curve comparison")
+    logging.info(f".. Started: {dt_string} \n")
+
+    try:
+        # Make sure that location_id is the only -group when using -pnts
+        assert not stat_gages or (
+            stat_gages and (not stat_groups or stat_groups == ['location_id'])
+        ), "location_id is the only acceptable stat_groups argument when producting an output GPKG"
+        # Make sure that the -pnts flag is used with the -eval flag
+        assert not eval or (eval and stat_gages), "You must use the -pnts flag with the -eval flag"
+        procs_list = []
+
+        plots_dir = join(output_dir, 'plots')
+        os.makedirs(plots_dir, exist_ok=True)
+        tables_dir = join(output_dir, 'tables')
+        os.makedirs(tables_dir, exist_ok=True)
+
+        # Check age of gages csv and recommend updating if older than 30 days.
+        print(check_file_age(usgs_gages_filename))
+
+        merged_elev_table = []
+        huc_list = [huc for huc in os.listdir(fim_dir) if re.search(r"^\d{6,8}$", huc)]
+        for huc in huc_list:
+            elev_table_filename = join(fim_dir, huc, 'usgs_elev_table.csv')
+            branches_folder = join(fim_dir, huc, 'branches')
+            usgs_recurr_stats_filename = join(tables_dir, f"usgs_interpolated_elevation_stats_{huc}.csv")
+            nwm_recurr_data_filename = join(tables_dir, f"nwm_recurrence_flow_elevations_{huc}.csv")
+            rc_comparison_plot_filename = join(plots_dir, f"FIM-USGS_rating_curve_comparison_{huc}.png")
+
+            if isfile(elev_table_filename):
+                procs_list.append(
+                    [
+                        elev_table_filename,
+                        branches_folder,
+                        usgs_gages_filename,
+                        usgs_recurr_stats_filename,
+                        nwm_recurr_data_filename,
+                        rc_comparison_plot_filename,
+                        nwm_flow_dir,
+                        catfim_flows_filename,
+                        huc,
+                        alt_plot,
+                        single_plot,
+                    ]
+                )
+                # Aggregate all of the individual huc elev_tables into one aggregate
+                #      for accessing all data in one csv
+                read_elev_table = pd.read_csv(
                     elev_table_filename,
-                    branches_folder,
-                    usgs_gages_filename,
-                    usgs_recurr_stats_filename,
-                    nwm_recurr_data_filename,
-                    rc_comparison_plot_filename,
-                    nwm_flow_dir,
-                    catfim_flows_filename,
-                    huc,
-                    alt_plot,
-                    single_plot,
-                ]
-            )
-            # Aggregate all of the individual huc elev_tables into one aggregate
-            #      for accessing all data in one csv
-            read_elev_table = pd.read_csv(
-                elev_table_filename, dtype={'location_id': str, 'HydroID': str, 'huc': str, 'feature_id': int}
+                    dtype={'location_id': str, 'HydroID': str, 'huc': str, 'feature_id': int},
+                )
+                read_elev_table['huc'] = huc
+                merged_elev_table.append(read_elev_table)
+
+        # Output a concatenated elev_table to_csv
+        if merged_elev_table:
+            logging.info("Creating aggregate elev table csv")
+            concat_elev_table = pd.concat(merged_elev_table)
+            concat_elev_table['thal_burn_depth_meters'] = (
+                concat_elev_table['dem_elevation'] - concat_elev_table['dem_adj_elevation']
             )
-            read_elev_table['huc'] = huc
-            merged_elev_table.append(read_elev_table)
-
-    # Output a concatenated elev_table to_csv
-    if merged_elev_table:
-        logging.info("Creating aggregate elev table csv")
-        concat_elev_table = pd.concat(merged_elev_table)
-        concat_elev_table['thal_burn_depth_meters'] = (
-            concat_elev_table['dem_elevation'] - concat_elev_table['dem_adj_elevation']
-        )
-        concat_elev_table.to_csv(join(output_dir, 'agg_usgs_elev_table.csv'), index=False)
-
-    # Initiate multiprocessing
-    logging.info(f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs")
-    with Pool(processes=number_of_jobs) as pool:
-        pool.map(generate_rating_curve_metrics, procs_list)
-
-    # Create point layer of usgs gages with joined stats attributes
-    if stat_gages:
-        logging.info("Creating usgs gages GPKG with joined rating curve summary stats")
-        agg_recurr_stats_table = aggregate_metrics(output_dir, procs_list, ['location_id'])
-        create_static_gpkg(output_dir, stat_gages, agg_recurr_stats_table, gages_gpkg_filepath)
-        del agg_recurr_stats_table  # memory cleanup
-    else:  # if not producing GIS layer, just aggregate metrics
-        logging.info(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
-        aggregate_metrics(output_dir, procs_list, stat_groups)
-
-    logging.info('Delete intermediate tables')
-    shutil.rmtree(tables_dir, ignore_errors=True)
-
-    # Compare current sierra test results to previous tests
-    if eval:
-        # Transpose comparison sierra results
-        sierra_test_paths, sierra_test_labels = np.array(eval).T.tolist()
-        # Add current sierra test to lists
-        sierra_test_paths = sierra_test_paths + [join(output_dir, stat_gages)]
-        sierra_test_labels = sierra_test_labels + [stat_gages.replace('.gpkg', '')]
-        # Read in all sierra test results
-        sierra_test_dfs = [gpd.read_file(i) for i in sierra_test_paths]
-        # Feed results into evaluation function
-        evaluate_results(
-            sierra_test_dfs, sierra_test_labels, join(output_dir, 'Sierra_Test_Eval_boxplot.png')
+            concat_elev_table.to_csv(join(output_dir, 'agg_usgs_elev_table.csv'), index=False)
+
+        # Initiate multiprocessing
+        logging.info(
+            f"Generating rating curve metrics for {len(procs_list)} hucs using {number_of_jobs} jobs"
         )
+        with Pool(processes=number_of_jobs) as pool:
+            pool.map(generate_rating_curve_metrics, procs_list)
+
+        # Create point layer of usgs gages with joined stats attributes
+        if stat_gages:
+            logging.info("Creating usgs gages GPKG with joined rating curve summary stats")
+            agg_recurr_stats_table = aggregate_metrics(output_dir, procs_list, ['location_id'])
+            create_static_gpkg(output_dir, stat_gages, agg_recurr_stats_table, gages_gpkg_filepath)
+            del agg_recurr_stats_table  # memory cleanup
+        else:  # if not producing GIS layer, just aggregate metrics
+            logging.info(f"Aggregating rating curve metrics for {len(procs_list)} hucs")
+            aggregate_metrics(output_dir, procs_list, stat_groups)
+
+        logging.info('Delete intermediate tables')
+        shutil.rmtree(tables_dir, ignore_errors=True)
+
+        # Compare current sierra test results to previous tests
+        if eval:
+            # Transpose comparison sierra results
+            sierra_test_paths, sierra_test_labels = np.array(eval).T.tolist()
+            # Add current sierra test to lists
+            sierra_test_paths = sierra_test_paths + [join(output_dir, stat_gages)]
+            sierra_test_labels = sierra_test_labels + [stat_gages.replace('.gpkg', '')]
+            # Read in all sierra test results
+            sierra_test_dfs = [gpd.read_file(i) for i in sierra_test_paths]
+            # Feed results into evaluation function
+            evaluate_results(
+                sierra_test_dfs, sierra_test_labels, join(output_dir, 'Sierra_Test_Eval_boxplot.png')
+            )
+    except Exception:
+        logging.info("-- Exception")
+        print("")
+        logging.info(traceback.format_exc())
+
+    end_time = dt.datetime.now()
+    time_duration = end_time - start_time
+    dt_string = dt.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+    print("==========================================================================")
+    logging.info(f".. Ended: {dt_string} \n")
+    logging.info(f".. Duration: {str(time_duration).split('.')[0]}")

From 86100144eb84ae48a059a09cdca7a5218851869d Mon Sep 17 00:00:00 2001
From: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com>
Date: Wed, 9 Oct 2024 13:10:38 -0500
Subject: [PATCH 352/359] Update LICENSE (#1313)

---
 LICENSE | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/LICENSE b/LICENSE
index a02303323..f9ef4ed8e 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,3 +1,17 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-----------------
+
 “Software code created by U.S. Government employees is not subject to copyright
 in the United States (17 U.S.C. §105). The United States/Department of Commerce
 reserve all rights to seek and obtain copyright protection in countries other
@@ -5,3 +19,5 @@ than the United States for Software authored in its entirety by the Department
 of Commerce. To this end, the Department of Commerce hereby grants to Recipient
 a royalty-free, nonexclusive license to use, copy, and create derivative works
 of the Software outside of the United States.”
+
+-----------------

From 08ff0e716dc865d2b5d6fe9d1f421b93018de1aa Mon Sep 17 00:00:00 2001
From: ZahraGhahremani-NOAA
 <100253864+ZahraGhahremani@users.noreply.github.com>
Date: Fri, 11 Oct 2024 18:56:36 +0000
Subject: [PATCH 353/359] v4.5.10.1 Fix bugs from hand_4_5_10_0 (#1314)

---
 docs/CHANGELOG.md         | 13 +++++++++++++
 src/derive_level_paths.py | 12 ++++++------
 src/run_unit_wb.sh        |  2 ++
 src/stream_branches.py    | 32 +++++++++++++++++---------------
 4 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7fbf8a591..8114e3d13 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.10.1 - 2024-10-11 - [PR#1314](https://github.com/NOAA-OWP/inundation-mapping/pull/1314)
+
+This PR fixes bugs from hand_4_5_10_0, which failed to run for Alaska HUCs and HUC 02030201. It modifies scripts to use two different DEM paths: one for Alaska and one for the CONUS.
+
+### Changes
+
+- `src/derive_level_paths.py`
+- `src/stream_branches.py`
+- `src/run_unit_wb.sh`
+
+<br/><br/>
+
+
 ## v4.5.10.0 - 2024-09-25 - [PR#1301](https://github.com/NOAA-OWP/inundation-mapping/pull/1301)
 
 A reload of all 3Dep DEMs from USGS was performed to refresh our data.
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index 97429f865..f29249882 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -203,6 +203,12 @@ def Derive_level_paths(
             out_vector_files=out_stream_network_dissolved,
             verbose=False,
         )
+    if stream_network.empty:
+        print("Sorry, no streams exist and processing can not continue. This could be an empty file.")
+        # sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
+        return
+    # else:
+    #     return stream_network
 
     if branch_inlets_outfile is not None:
         branch_inlets = stream_network.derive_inlet_points_by_feature(
@@ -212,12 +218,6 @@ def Derive_level_paths(
         if not branch_inlets.empty:
             branch_inlets.to_file(branch_inlets_outfile, index=False, driver="GPKG", engine='fiona')
 
-    if stream_network.empty:
-        print("Sorry, no streams exist and processing can not continue. This could be an empty file.")
-        sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
-    else:
-        return stream_network
-
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Create stream network level paths")
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index c8a8766a1..f0499a58b 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -24,11 +24,13 @@ huc2Identifier=${hucNumber:0:2}
 if [ $huc2Identifier -eq 19 ]; then
     huc_CRS=$ALASKA_CRS
     huc_input_DEM_domain=$input_DEM_domain_Alaska
+    input_DEM=$input_DEM_Alaska
     dem_domain_filename=DEM_Domain.gpkg
 
 else
     huc_CRS=$DEFAULT_FIM_PROJECTION_CRS
     huc_input_DEM_domain=$input_DEM_domain
+    input_DEM=$input_DEM
     dem_domain_filename=HUC6_dem_domain.gpkg
 
 fi
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 591281477..95366a3b7 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -686,11 +686,11 @@ def remove_branches_in_waterbodies(
             self = self.drop(sjoined.index)
             self = self.rename(columns={"bids": branch_id_attribute})
 
-            if out_vector_files is not None:
-                if verbose:
-                    print("Writing pruned branches ...")
+            # if out_vector_files is not None:
+            #     if verbose:
+            #         print("Writing pruned branches ...")
 
-                self.write(out_vector_files, index=False)
+            # self.write(out_vector_files, index=False)
 
         return self
 
@@ -738,7 +738,9 @@ def select_branches_intersecting_huc(self, wbd, buffer_wbd_streams, out_vector_f
                     print(
                         "Sorry, no streams exist and processing can not continue. This could be an empty file."
                     )
-                    sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
+                    # sys.exit(FIM_exit_codes.UNIT_NO_BRANCHES.value)  # will send a 60 back
+                    return self
+                    # sys.exit(FIM_exit_codes.NO_BRANCH_LEVELPATHS_EXIST.value)  # will send a 63 back
 
                 self.write(out_vector_files, index=False)
 
@@ -1029,20 +1031,20 @@ def dissolve_by_branch(
             values_excluded=values_excluded,
         )
 
-        if out_vector_files is not None:
-            # base_file_path,extension = splitext(out_vector_files)
+        # if out_vector_files is not None:
+        #     # base_file_path,extension = splitext(out_vector_files)
 
-            if verbose:
-                print("Writing dissolved branches ...")
+        #     if verbose:
+        #         print("Writing dissolved branches ...")
 
-            # for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
-            # out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
+        # for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
+        # out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
 
-            # bid_indices = self.loc[:,branch_id_attribute] == bid
-            # current_stream_network = StreamNetwork(self.loc[bid_indices,:])
+        # bid_indices = self.loc[:,branch_id_attribute] == bid
+        # current_stream_network = StreamNetwork(self.loc[bid_indices,:])
 
-            # current_stream_network.write(out_vector_file,index=False)
-            self.write(out_vector_files, index=False)
+        # current_stream_network.write(out_vector_file,index=False)
+        # self.write(out_vector_files, index=False)
 
         return self
 

From 751b8bf2b730b21f2342227ff1f94b40786e0da5 Mon Sep 17 00:00:00 2001
From: Emma Rachel Kaufman <143554481+erkaufman@users.noreply.github.com>
Date: Fri, 11 Oct 2024 15:04:48 -0400
Subject: [PATCH 354/359] v4.5.10.2 New Flood Impact Assessment Tool (#1244)

---
 docs/CHANGELOG.md             |  13 ++
 tools/analyze_flood_impact.py | 247 ++++++++++++++++++++++++++++++++++
 2 files changed, 260 insertions(+)
 create mode 100644 tools/analyze_flood_impact.py

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8114e3d13..8b284abb7 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,17 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.5.10.2 - 2024-10-11 - [PR#1244](https://github.com/NOAA-OWP/inundation-mapping/pull/1244)
+
+New tool that can assess the impact of a flood on road and/or building vector files. Closes #1226.
+
+### Additions
+- `tools/analyze_flood_impact.py` : added a tool that assesses the impact of a flood on roads and buildings by calculating how many roads and structures the test flood extent intersects, comparing the test impacted roads and structures to a benchmark, and calculating CSI.
+
+ <br/><br/>
+
+
 ## v4.5.10.1 - 2024-10-11 - [PR#1314](https://github.com/NOAA-OWP/inundation-mapping/pull/1314)
 
 This PR fixes bugs from hand_4_5_10_0, which failed to run for Alaska HUCs and HUC 02030201. It modifies scripts to use two different DEM paths: one for Alaska and one for the CONUS.
@@ -200,6 +211,8 @@ Updated the gauge crosswalk and SRC adjustment routine to use the ras2fim v2 fil
 
 <br/><br/>
 
+
+
 ## v4.5.4.4 - 2024-08-02 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
 
 Prior to this fix, fim_post_processing.sh took just under 4 hours to reset permissions on all files and folder under the entire run. On closer inspection, it was updating permissions for all HUC folders where were already correct. A few other folders needed to have permission updates added. This will speed that up significantly.
diff --git a/tools/analyze_flood_impact.py b/tools/analyze_flood_impact.py
new file mode 100644
index 000000000..95092b5cf
--- /dev/null
+++ b/tools/analyze_flood_impact.py
@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+
+import argparse
+from timeit import default_timer as timer
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rasterio
+from rasterio import features as riofeatures
+
+
+def analyze_flood_impact(
+    benchmark_inundation_tif, test_inundation_tif, model_domain_shp, structures_gpkg, roads_gpkg, output_gpkg
+):
+    """
+    Assesses the impact of a flood on road and building vector files. Counts how many roads and structures a benchmark and test
+    flood extent intersect and calculates CSI.
+
+    Data information:
+    - Building vector files can be accessed from the FEMA Geospatial Rescource Center USA Structures webpage here:
+        https://gis-fema.hub.arcgis.com/pages/usa-structures.
+    - Road vector files are provided by state DOT GIS services. Ex. Texas DOT data can be found here:
+        https://gis-txdot.opendata.arcgis.com/
+
+    Parameters
+    ----------
+    benchmark_inundation_tif : str
+        Input path for benchmark inundation raster.
+    test_inundation_tif : str
+        Input path for test inundation raster.
+    model_domain_shp : str
+        Input path for the model domain vector file. Domain file should match the extent and CRS of the benchmark inundation file.
+    structures_gpkg : str
+        Input path for the structures vector file. File must have an OBJECTID field identifying individual structures.
+    roads_gpkg : str
+        Input path for the roads vector file. File must have an OBJECTID field identifying individual roads.
+
+
+    Outputs
+    -------
+    output_gpkg: str
+        Output path for the geopackage vector file that contains the following:
+            - test inundation: test inundation extent vector file, clipped to the model domain.
+            - test impacted structures: structures that intersect the test inundation extent.
+            - test impacted roads: roads that intersect the test inundation extent.
+            - benchmark inundation: benchmark inundation extent.
+            - benchmark impacted structures: structures that intersect the benchmark inundation extent.
+            - benchmark impacted roads: roads that intersect the test inundation extent.
+
+    This function will also print CSI and print the number of impacted roads and structures for each inundation extent.
+    """
+    # Load vector files
+    flood_extent_bench = vectorize(benchmark_inundation_tif)
+    flood_extent_test_whole = vectorize(test_inundation_tif)
+    domain = gpd.read_file(model_domain_shp)
+    structures = gpd.read_file(structures_gpkg)
+    roads = gpd.read_file(roads_gpkg)
+
+    # Clip the test extent to the model domain
+    flood_extent_test = gpd.clip(flood_extent_test_whole, domain)
+
+    # Ensure all data are in the same CRS
+    structures = structures.to_crs(flood_extent_test.crs)
+    roads = roads.to_crs(flood_extent_test.crs)
+    flood_extent_bench = flood_extent_bench.to_crs(flood_extent_test.crs)
+
+    # Find intersecting structures/roads and create gdf for benchmark and test
+    impacted_structures_bench = impacted(structures, flood_extent_bench)
+    impacted_structures_test = impacted(structures, flood_extent_test)
+    impacted_roads_bench = impacted(roads, flood_extent_bench)
+    impacted_roads_test = impacted(roads, flood_extent_test)
+
+    # Calculate CSI
+
+    # TP: features in both benchmark and test
+    true_positives_structures = impacted_structures_bench.merge(
+        impacted_structures_test, how='left', on='OBJECTID', suffixes=('_benchmark', '_test')
+    )
+    true_positives_roads = impacted_roads_bench.merge(
+        impacted_roads_test, how='left', on='OBJECTID', suffixes=('_benchmark', '_test')
+    )
+
+    # FN: features in benchmark but not in test
+    false_neg_structures = impacted_structures_bench[
+        ~impacted_structures_bench['OBJECTID'].isin(impacted_structures_test['OBJECTID'])
+    ]
+    false_neg_roads = impacted_roads_bench[
+        ~impacted_roads_bench['OBJECTID'].isin(impacted_roads_test['OBJECTID'])
+    ]
+
+    # FP: features in test but not in benchmark
+    false_pos_structures = impacted_structures_test[
+        ~impacted_structures_test['OBJECTID'].isin(impacted_structures_bench['OBJECTID'])
+    ]
+    false_pos_roads = impacted_roads_test[
+        ~impacted_roads_test['OBJECTID'].isin(impacted_roads_bench['OBJECTID'])
+    ]
+
+    # Calculation
+    TP = len(true_positives_structures) + len(true_positives_roads)
+    FN = len(false_neg_structures) + len(false_neg_roads)
+    FP = len(false_pos_structures) + len(false_pos_roads)
+    CSI = TP / (TP + FN + FP)
+
+    # Save the combined data to new layers in a GeoPackage file
+    impacted_structures_bench.to_file(output_gpkg, layer='benchmark impacted structures', driver="GPKG")
+    impacted_roads_bench.to_file(output_gpkg, layer='benchmark impacted roads', driver="GPKG")
+    impacted_structures_test.to_file(output_gpkg, layer='test impacted structures', driver="GPKG")
+    impacted_roads_test.to_file(output_gpkg, layer='test impacted roads', driver="GPKG")
+    flood_extent_bench.to_file(output_gpkg, layer='benchmark inundation', index=False)
+    flood_extent_test.to_file(output_gpkg, layer='test inundation', index=False)
+
+    print(f"Structures and roads with impact attribute for benchmark and test data saved to {output_gpkg}.")
+
+    # Total impacted infrastructure
+    total_structures_test_impact = len(impacted_structures_test)
+    total_road_test_impact = len(impacted_roads_test)
+    total_structures_bench_impact = len(impacted_structures_bench)
+    total_road_bench_impact = len(impacted_roads_bench)
+
+    print(
+        f" Benchmark: {total_structures_bench_impact} structures impacted by this flood and {total_road_bench_impact} roads impacted by this flood."
+    )
+    print(
+        f" Test: {total_structures_test_impact} structures impacted by this flood and {total_road_test_impact} roads impacted by this flood."
+    )
+    print(f'Critical Success Index: {CSI}')
+
+
+def vectorize(inundation_tif):
+    """
+    Converts inundation raster into a vector file and returns it.
+
+    Parameters
+    ----------
+    inundation_tif : str
+        Input path for inundation raster.
+
+    Returns
+    ----------
+    extent_poly_diss :
+        Inundation vector
+    """
+    with rasterio.open(inundation_tif) as fim_rast:
+        fim_nodata = fim_rast.profile['nodata']
+        fim_transform = fim_rast.transform
+        fim_crs = fim_rast.crs
+        fim = fim_rast.read(1).astype(np.float32)
+
+    # Create binary raster
+    fim[np.where(fim == fim_nodata)] = np.nan
+    fim[np.where(fim <= 0)] = np.nan
+    fim[np.where(fim > 0)] = 1
+
+    # Vectorize
+    results = (
+        {"properties": {"extent": 1}, "geometry": s}
+        for i, (s, v) in enumerate(
+            riofeatures.shapes(fim, mask=fim > 0, transform=fim_transform, connectivity=8)
+        )
+    )
+
+    # Convert list of shapes to polygon, then dissolve
+    extent_poly = gpd.GeoDataFrame.from_features(list(results), crs=fim_crs)
+    extent_poly_diss = extent_poly.dissolve()
+
+    return extent_poly_diss
+
+
+def impacted(features_gpkg, inundation_tif):
+    """
+    Finds features that intersect with an inundation raster (impacted features) and returns the impacted features.
+
+    Parameters
+    ----------
+    features_gpkg : str
+        Input path for feature vector (ex. roads or structures).
+    inundation_tif : str
+        Input path for inundation raster.
+
+    Returns
+    --------
+    impacted_features :
+        Vector features that intersect with the inundation extent.
+    """
+    # Find intersecting features with inundation
+    impacted_features = gpd.GeoDataFrame(
+        gpd.sjoin(features_gpkg, inundation_tif, how='inner', predicate='intersects'), crs=inundation_tif.crs
+    )
+
+    impacted_features['isImpacted'] = True
+    return impacted_features
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="""Assesses the impact of a flood on road and building vector files. Counts how many roads and structures a
+benchmark and test flood extent intersect and calculates CSI.
+
+Sample usage:
+python3  analyze_flood_impact.py -b home/user/benchmark_inundation.tif -t home/user/test_inundation.tif \
+    -d home/user/model_domain.shp -s home/user/structures_vector.gpkg -rd home/user/roads_vector.gpkg \
+    -o home/user/impacted_roads_and_structures_output.gpkg
+"""
+    )
+    # Parse arguments
+    parser.add_argument(
+        '-b', '--benchmark_inundation', required=True, help="Path to the benchmark inundation TIF file."
+    )
+    parser.add_argument(
+        '-t', '--test_inundation', required=True, help="Path to the test inundation TIF file."
+    )
+    parser.add_argument(
+        '-d',
+        '--domain',
+        required=True,
+        help="Path to the model domain vector file. Domain file should match the extent and CRS of the benchmark inundation file.",
+    )
+    parser.add_argument(
+        '-s',
+        '--structures',
+        required=True,
+        help="Path to the structures vector file. File must have an OBJECTID field identifying individual structures.",
+    )
+    parser.add_argument(
+        '-rd',
+        '--roads',
+        required=True,
+        help="Path to the roads vector file. File must have an OBJECTID field identifying individual roads.",
+    )
+    parser.add_argument('-o', '--output', required=True, help="Path to the output vector file (GeoPackage).")
+
+    args = vars(parser.parse_args())
+
+    start = timer()
+
+    analyze_flood_impact(
+        args['benchmark_inundation'],
+        args['test_inundation'],
+        args['domain'],
+        args['structures'],
+        args['roads'],
+        args['output'],
+    )
+
+    print(f"Completed in {round((timer() - start)/60, 2)} minutes.")

From 48808425f8505d1972af5d7f96b4a2e64fbb8e7b Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Fri, 11 Oct 2024 15:23:26 -0400
Subject: [PATCH 355/359] v4.5.10.3 Extend outlet levelpath(s) outside HUC
 (#1306)

---
 config/deny_branch_zero.lst |   1 +
 config/deny_branches.lst    |   1 +
 config/deny_unit.lst        |   1 +
 docs/CHANGELOG.md           |  21 +++++
 src/derive_level_paths.py   |  14 ++-
 src/run_by_branch.sh        |   5 +-
 src/run_unit_wb.sh          |   3 +-
 src/stream_branches.py      | 175 ++++++++++++++++++++++++++++++++----
 8 files changed, 199 insertions(+), 22 deletions(-)

diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst
index 409c306ff..cef6f218b 100644
--- a/config/deny_branch_zero.lst
+++ b/config/deny_branch_zero.lst
@@ -48,6 +48,7 @@ LandSea_subset_{}.tif
 nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_dissolved_extended_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
 #osm_bridge_centroids_{}.gpkg
 rem_{}.tif
diff --git a/config/deny_branches.lst b/config/deny_branches.lst
index a44a79a49..72bd9e4c7 100644
--- a/config/deny_branches.lst
+++ b/config/deny_branches.lst
@@ -49,6 +49,7 @@ levee_levelpaths.csv
 nld_rasterized_elev_{}.tif
 nwm_catchments_proj_subset_levelPaths_{}.gpkg
 nwm_subset_streams_levelPaths_{}.gpkg
+nwm_subset_streams_levelPaths_dissolved_extended_{}.gpkg
 nwm_subset_streams_levelPaths_dissolved_headwaters_{}.gpkg
 #osm_bridge_centroids_{}.gpkg
 #ras_elev_table.csv
diff --git a/config/deny_unit.lst b/config/deny_unit.lst
index f46240f89..031019ea2 100644
--- a/config/deny_unit.lst
+++ b/config/deny_unit.lst
@@ -27,6 +27,7 @@ nwm_headwaters.gpkg
 # nwm_subset_streams.gpkg
 # nwm_subset_streams_levelPaths.gpkg
 # nwm_subset_streams_levelPaths_dissolved.gpkg
+# nwm_subset_streams_levelPaths_dissolved_extended.gpkg
 # nwm_subset_streams_levelPaths_dissolved_headwaters.gpkg
 # ras_elev_table.csv
 # usgs_elev_table.csv
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8b284abb7..293609513 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,27 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.10.3 - 2024-10-11 - [PR#1306](https://github.com/NOAA-OWP/inundation-mapping/pull/1306)
+
+Extends outlet levelpath(s) outside HUC.
+
+Previously, levelpaths at the outlet of a HUC may not extend to the buffered WBD that is used to clip the DEM, and during pit-filling this results in reverse flow which can cause DEM-derived reaches to deviate from the channel in the DEM and may result in dropped catchments where the midpoint of the reaches exceeds the snap distance from the NWM stream lines.
+
+This PR extends outlet levelpaths in two ways:
+- Segments of levelpaths that terminate in waterbodies are removed from the levelpath. If there is a waterbody downstream of the HUC then the outlet reaches may be trimmed such that the outlet no longer reaches the edge of the DEM, which causes a number of cascading issues originating in the pit-filling such that reverse flow in the DEM-derived reaches can result in erroneous flowlines and inundation. This PR stops trimming levelpaths outside of the HUC.
+- Dissolved outlet levelpaths may terminate downstream outside of the HUC (e.g., at a confluence with a larger river) at a point that is within the buffered WBD. These levelpaths are extended by adding on the downstream segment(s) of the HUC's `nwm_subset_streams` layer. The extended levelpath(s) are saved in a new file that is used to create the boolean raster stream network.
+
+### Changes
+
+- `config/`
+    - `deny_unit.lst`, `deny_branch_zero.lst`, and `deny_branches.lst`: Adds new file to deny lists
+- `src/`
+    - `derive_level_paths.py`:  Adds WBD as an input to `stream_network.trim_branches_in_waterbodies()` and adds new argument for new filename.
+    - `run_unit_wb.sh`: Adds new argument for new filename.
+    - `stream_branches.py`: Selects only segments intersecting the WBD as candidates for removal if they end in waterbodies and adds downstream segment(s) to outlet levelpath(s).
+    
+<br/><br/>
+
 
 ## v4.5.10.2 - 2024-10-11 - [PR#1244](https://github.com/NOAA-OWP/inundation-mapping/pull/1244)
 
diff --git a/src/derive_level_paths.py b/src/derive_level_paths.py
index f29249882..2dc631e2a 100755
--- a/src/derive_level_paths.py
+++ b/src/derive_level_paths.py
@@ -22,6 +22,7 @@ def Derive_level_paths(
     branch_id_attribute,
     huc_id,
     out_stream_network_dissolved=None,
+    out_stream_network_dissolved_extended=None,
     headwaters_outfile=None,
     catchments=None,
     waterbodies=None,
@@ -179,15 +180,17 @@ def Derive_level_paths(
 
     if out_stream_network_dissolved is not None:
         stream_network = stream_network.trim_branches_in_waterbodies(
-            branch_id_attribute=branch_id_attribute, verbose=verbose
+            wbd=wbd, branch_id_attribute=branch_id_attribute, verbose=verbose
         )
 
         # dissolve by levelpath
         stream_network = stream_network.dissolve_by_branch(
+            wbd=wbd,
             branch_id_attribute=branch_id_attribute,
             attribute_excluded=None,  # 'order_',
             values_excluded=None,  # [1,2],
             out_vector_files=out_stream_network_dissolved,
+            out_extended_vector_files=out_stream_network_dissolved_extended,
             verbose=verbose,
         )
 
@@ -212,7 +215,7 @@ def Derive_level_paths(
 
     if branch_inlets_outfile is not None:
         branch_inlets = stream_network.derive_inlet_points_by_feature(
-            feature_attribute=branch_id_attribute, outlet_linestring_index=outlet_linestring_index
+            branch_id_attribute=branch_id_attribute, outlet_linestring_index=outlet_linestring_index
         )
 
         if not branch_inlets.empty:
@@ -272,6 +275,13 @@ def Derive_level_paths(
         required=False,
         default=None,
     )
+    parser.add_argument(
+        "-de",
+        "--out-stream-network-dissolved-extended",
+        help="Dissolved output stream network extended",
+        required=False,
+        default=None,
+    )
     parser.add_argument(
         "-v", "--verbose", help="Verbose output", required=False, default=False, action="store_true"
     )
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index 28977c2e0..b2f3c74b9 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -49,6 +49,9 @@ echo -e "Querying NWM streams ..."
 ogr2ogr -f GPKG -t_srs $huc_CRS -where $branch_id_attribute="$current_branch_id" \
     $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg \
     $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg
+ogr2ogr -f GPKG -t_srs $huc_CRS -where $branch_id_attribute="$current_branch_id" \
+    $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_extended_$current_branch_id.gpkg \
+    $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_extended.gpkg
 echo -e "Querying NWM catchments ..."
 ogr2ogr -f GPKG -t_srs $huc_CRS -where $branch_id_attribute="$current_branch_id" \
     $tempCurrentBranchDataDir/nwm_catchments_proj_subset_levelPaths_$current_branch_id.gpkg \
@@ -81,7 +84,7 @@ read fsize ncols nrows ndv xmin ymin xmax ymax cellsize_resx cellsize_resy\
 echo -e $startDiv"Rasterize Reach Boolean $hucNumber $current_branch_id"
 gdal_rasterize -q -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
     -te $xmin $ymin $xmax $ymax \
-    -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_$current_branch_id.gpkg \
+    -ts $ncols $nrows $tempCurrentBranchDataDir/nwm_subset_streams_levelPaths_dissolved_extended_$current_branch_id.gpkg \
     $tempCurrentBranchDataDir/flows_grid_boolean_$current_branch_id.tif
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index f0499a58b..28794ede1 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -78,6 +78,7 @@ $srcDir/derive_level_paths.py -i $tempHucDataDir/nwm_subset_streams.gpkg \
     -r "ID" \
     -o $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
     -d $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
+    -de $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_extended.gpkg \
     -e $tempHucDataDir/nwm_headwaters.gpkg \
     -c $tempHucDataDir/nwm_catchments_proj_subset.gpkg \
     -t $tempHucDataDir/nwm_catchments_proj_subset_levelPaths.gpkg \
@@ -176,7 +177,7 @@ if [ "$levelpaths_exist" = "1" ]; then
     echo -e $startDiv"Rasterize Reach Boolean $hucNumber (Branches)"
     gdal_rasterize -q -ot Int32 -burn 1 -init 0 -co "COMPRESS=LZW" -co "BIGTIFF=YES" -co "TILED=YES" \
         -te $xmin $ymin $xmax $ymax -ts $ncols $nrows \
-        $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg $tempHucDataDir/flows_grid_boolean.tif
+        $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved_extended.gpkg $tempHucDataDir/flows_grid_boolean.tif
 fi
 
 ## RASTERIZE NWM Levelpath HEADWATERS (1 & 0) ##
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 95366a3b7..6f9c02641 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -16,7 +16,7 @@
 from rasterio.mask import mask
 from scipy.stats import mode
 from shapely.geometry import LineString, MultiLineString, MultiPoint, Point
-from shapely.ops import linemerge
+from shapely.ops import linemerge, unary_union
 from shapely.strtree import STRtree
 from tqdm import tqdm
 
@@ -465,7 +465,7 @@ def derive_inlets(
 
         return self
 
-    def derive_inlet_points_by_feature(self, feature_attribute, outlet_linestring_index):
+    def derive_inlet_points_by_feature(self, branch_id_attribute, outlet_linestring_index):
         """Finds the upstream point of every feature in the stream network"""
 
         inlet_linestring_index = StreamNetwork.flip_inlet_outlet_linestring_index(outlet_linestring_index)
@@ -479,8 +479,8 @@ def derive_inlet_points_by_feature(self, feature_attribute, outlet_linestring_in
             if row.geom_type[idx] == "MultiLineString":
                 # Convert MultiLineString to LineString
                 row = row.explode(index_parts=False)
-                row.loc[row["levpa_id"].duplicated(), "levpa_id"] = np.nan
-                row = row.dropna(subset=["levpa_id"])
+                row.loc[row[branch_id_attribute].duplicated(), branch_id_attribute] = np.nan
+                row = row.dropna(subset=[branch_id_attribute])
 
             feature_inlet_point = Point(row.geometry[0].coords[inlet_linestring_index])
 
@@ -586,7 +586,7 @@ def remove_branches_without_catchments(
 
         return self
 
-    def trim_branches_in_waterbodies(self, branch_id_attribute, verbose=False):
+    def trim_branches_in_waterbodies(self, wbd, branch_id_attribute, verbose=False):
         """
         Recursively trims the reaches from the ends of the branches if they are in a
         waterbody (determined by the Lake attribute).
@@ -647,6 +647,14 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
         for branch in self[branch_id_attribute].astype(int).unique():
             tmp_self = self[self[branch_id_attribute].astype(int) == branch]
 
+            # load waterbodies
+            if isinstance(wbd, str):
+                wbd = gpd.read_file(wbd)
+
+            # trim only branches in WBD (to prevent outlet from being trimmed)
+            if isinstance(wbd, gpd.GeoDataFrame):
+                tmp_self = gpd.sjoin(tmp_self, wbd)
+
             # If entire branch is in waterbody
             if all(tmp_self.Lake.values != -9999):
                 tmp_IDs = tmp_self.From_Node.astype(int)
@@ -982,12 +990,75 @@ def get_arbolate_sum(
 
     def dissolve_by_branch(
         self,
+        wbd,
         branch_id_attribute="LevelPathI",
         attribute_excluded="StreamOrde",
         values_excluded=[1, 2],
         out_vector_files=None,
+        out_extended_vector_files=None,
         verbose=False,
     ):
+
+        def add_outlet_segments(
+            self_extended: gpd.GeoDataFrame,
+            self_copy: gpd.GeoDataFrame,
+            outlet_id: int,
+            ds_outlet_tuple: tuple,
+            extended_id: int = None,
+        ) -> gpd.GeoDataFrame:
+            """
+            Recursively adds outlet segments to the stream network
+
+            Parameters
+            ----------
+            self_ : GeoDataFrame
+                Dissolved tream network GeoDataFrame
+            self_copy : GeoDataFrame
+                Stream network GeoDataFrame
+            outlet_id : int
+                Outlet segment ID
+            ds_outlet_tuple : tuple
+                Outlet segment tuple
+
+            Returns
+            -------
+            GeoDataFrame
+                Stream network with outlet segments
+            """
+
+            if not extended_id:
+                extended_id = outlet_id
+
+            # add outlet segment to stream network
+            idx = self_extended['bids_temp'] == outlet.levpa_id
+            if self_extended.loc[idx].empty:
+                idx = self_extended['ID'] == extended_id
+
+            extended_gs = gpd.GeoSeries(
+                [self_extended.loc[idx, 'geometry'].item(), ds_outlet_tuple.geometry]
+            ).line_merge()
+
+            if isinstance(extended_gs[0], MultiLineString):
+                self_extended.loc[idx, 'geometry'] = MultiLineString(
+                    [linestring for linestring in extended_gs[0].geoms] + [extended_gs[1]]
+                )
+            else:
+                self_extended.loc[idx, 'geometry'] = MultiLineString(
+                    [extended_gs[0].coords, extended_gs[1].coords]
+                )
+
+            if ds_outlet_tuple.to in self_copy.ID.values:
+                # find the next downstream segment
+                outlet_id = ds_outlet_tuple.to
+                for ds_outlet_tuple in self_copy[self_copy.ID == ds_outlet_tuple.to].itertuples():
+
+                    # recursively add outlet segments
+                    self_extended = add_outlet_segments(
+                        self_extended, self_copy, outlet_id, ds_outlet_tuple, extended_id
+                    )
+
+            return self_extended
+
         if verbose:
             print("Dissolving by branch ...")
 
@@ -1000,22 +1071,81 @@ def dissolve_by_branch(
         # dissolve lines
         self["bids_temp"] = self.loc[:, branch_id_attribute].copy()
 
+        wbd = gpd.read_file(wbd)
+        wbd = wbd.drop('shape_Length', axis=1)
+
+        # Filter segments that are in the HUC
+        self_in_wbd = gpd.sjoin(self, wbd)
+        self_in_wbd = self_in_wbd.drop('index_right', axis=1)
+
         # ensure the new stream order has the order from it's highest child
         max_stream_order = (
-            self[[branch_id_attribute, "order_"]].groupby(branch_id_attribute).max()["order_"].copy()
+            self_in_wbd[[branch_id_attribute, "order_"]].groupby(branch_id_attribute).max()["order_"].copy()
         )
 
-        self = self.dissolve(by=branch_id_attribute)
-        self = self.rename(columns={"bids_temp": branch_id_attribute})
+        # Find the HUC outlet(s) -- downstream segments that intersect WBD boundary
+        sjoin = gpd.sjoin(self, wbd, predicate='crosses')
+
+        # Get ID of segments downstream of WBD boundary
+        s = self[self['ID'].isin(sjoin['to'])]
+
+        # Find downstream segments outside of WBD
+        s_in_wbd = gpd.sjoin(s, wbd)
+        s_not_in_wbd = s[~s['ID'].isin(s_in_wbd['ID'])]
+
+        self_copy = self.copy(deep=True)
+
+        self = self_in_wbd.dissolve(by=branch_id_attribute)
 
         self["order_"] = max_stream_order.values
 
+        if not s_not_in_wbd.empty:
+            outlets_extended = self.copy(deep=True)
+
+            # For each outlet
+            for outlet in s_not_in_wbd.itertuples():
+                # Select segments in levelpath
+                temp_df = self_copy[self_copy[branch_id_attribute] == outlet.bids_temp]
+
+                # Check if the levelpath outlet is external
+                if not len(temp_df.merge(self_in_wbd, left_on='to', right_on='ID')) == len(temp_df):
+                    outlet_id = self_in_wbd.loc[self_in_wbd['to'] == outlet.ID, 'ID'].values[0]
+                    outlets_extended = add_outlet_segments(outlets_extended, self_copy, outlet_id, outlet)
+
+            # merges each multi-line string to a singular linestring
+            for lpid, row in tqdm(
+                outlets_extended.iterrows(),
+                total=len(outlets_extended),
+                disable=(not verbose),
+                desc="Merging mult-part geoms",
+            ):
+                if isinstance(row.geometry, MultiLineString):
+                    merged_line = linemerge(row.geometry)
+
+                    # outlets_extended.loc[lpid,'geometry'] = merged_line
+                    try:
+                        outlets_extended.loc[lpid, "geometry"] = merged_line
+                    except ValueError:
+                        merged_line = list(merged_line.geoms)[0]
+                        outlets_extended.loc[lpid, "geometry"] = merged_line
+
+            # self[branch_id_attribute] = bids
+            outlets_extended = StreamNetwork(
+                outlets_extended,
+                branch_id_attribute=branch_id_attribute,
+                attribute_excluded=attribute_excluded,
+                values_excluded=values_excluded,
+            )
+
+            outlets_extended = outlets_extended.rename(columns={'bids_temp': branch_id_attribute})
+
         # merges each multi-line string to a singular linestring
         for lpid, row in tqdm(
             self.iterrows(), total=len(self), disable=(not verbose), desc="Merging mult-part geoms"
         ):
             if isinstance(row.geometry, MultiLineString):
                 merged_line = linemerge(row.geometry)
+
                 # self.loc[lpid,'geometry'] = merged_line
                 try:
                     self.loc[lpid, "geometry"] = merged_line
@@ -1031,20 +1161,29 @@ def dissolve_by_branch(
             values_excluded=values_excluded,
         )
 
-        # if out_vector_files is not None:
-        #     # base_file_path,extension = splitext(out_vector_files)
+        if out_vector_files is not None:
+            if verbose:
+                print("Writing dissolved branches ...")
 
-        #     if verbose:
-        #         print("Writing dissolved branches ...")
+            # if out_vector_files is not None:
+            #     # base_file_path,extension = splitext(out_vector_files)
 
-        # for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
-        # out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
+            #     if verbose:
+            #         print("Writing dissolved branches ...")
 
-        # bid_indices = self.loc[:,branch_id_attribute] == bid
-        # current_stream_network = StreamNetwork(self.loc[bid_indices,:])
+            # for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
+            # out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
 
-        # current_stream_network.write(out_vector_file,index=False)
-        # self.write(out_vector_files, index=False)
+            # bid_indices = self.loc[:,branch_id_attribute] == bid
+            # current_stream_network = StreamNetwork(self.loc[bid_indices,:])
+
+            # current_stream_network.write(out_vector_file,index=False)
+            self.write(out_vector_files, index=False)
+
+        if out_extended_vector_files is not None and not s_not_in_wbd.empty:
+            outlets_extended.write(out_extended_vector_files, index=False)
+        else:
+            self.write(out_extended_vector_files, index=False)
 
         return self
 

From 42b75d26f09c2f3ec1373eca9c8a1fed95a40acc Mon Sep 17 00:00:00 2001
From: ZahraGhahremani-NOAA
 <100253864+ZahraGhahremani@users.noreply.github.com>
Date: Fri, 11 Oct 2024 19:29:23 +0000
Subject: [PATCH 356/359] v4.5.11.0 Modifying OSM bridge scripts (#1298)

---
 data/bridges/pull_osm_bridges.py | 112 +++++++++++++++++++++++++++++--
 docs/CHANGELOG.md                |  12 ++++
 src/bash_variables.env           |   4 +-
 src/heal_bridges_osm.py          |   6 ++
 4 files changed, 127 insertions(+), 7 deletions(-)

diff --git a/data/bridges/pull_osm_bridges.py b/data/bridges/pull_osm_bridges.py
index 1f73ad196..ed6bb79ff 100644
--- a/data/bridges/pull_osm_bridges.py
+++ b/data/bridges/pull_osm_bridges.py
@@ -3,6 +3,7 @@
 import logging
 import os
 import traceback
+import warnings
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from pathlib import Path
 
@@ -10,7 +11,8 @@
 import osmnx as ox
 import pandas as pd
 import pyproj
-from shapely.geometry import shape
+from networkx import Graph, connected_components
+from shapely.geometry import LineString, shape
 
 
 CRS = "epsg:5070"
@@ -19,6 +21,32 @@
 # Save all OSM bridge features by HUC8 to a specified folder location.
 # Bridges will have point geometry converted to linestrings if needed
 #
+# Dissolve touching lines
+def find_touching_groups(gdf):
+    # Create a graph
+    graph = Graph()
+
+    # Add nodes for each geometry
+    graph.add_nodes_from(gdf.index)
+
+    # Create spatial index for efficient querying
+    spatial_index = gdf.sindex
+
+    # For each geometry, find touching geometries and add edges to the graph
+    for idx, geometry in gdf.iterrows():
+        possible_matches_index = list(spatial_index.intersection(geometry['geometry'].bounds))
+        possible_matches = gdf.iloc[possible_matches_index]
+        precise_matches = possible_matches[possible_matches.intersects(geometry['geometry'])]
+
+        for match_idx in precise_matches.index:
+            if match_idx != idx:
+                graph.add_edge(idx, match_idx)
+
+    # Find connected components
+    groups = list(connected_components(graph))
+    return groups
+
+
 def pull_osm_features_by_huc(huc_bridge_file, huc_num, huc_geom):
     """
     Returns: The huc number but only if it failed, so we can make a master list of failed HUCs.
@@ -40,13 +68,47 @@ def pull_osm_features_by_huc(huc_bridge_file, huc_num, huc_geom):
             logging.info(f"osmnx pull for {huc_num} came back with no records")
             return huc_num
 
+        # Create bridge_type column
+        # Check if 'highway' column exists
+        if 'highway' not in gdf.columns:
+            gdf['highway'] = None
+
+        # Check if 'railway' column exists
+        if 'railway' not in gdf.columns:
+            gdf['railway'] = None
+
+        # Create the bridge_type column by combining above information
+        gdf['bridge_type'] = gdf.apply(
+            lambda row: (
+                f"highway-{row['highway']}" if pd.notna(row['highway']) else f"railway-{row['railway']}"
+            ),
+            axis=1,
+        )
+        gdf.reset_index(inplace=True)
+        # Remove abandoned bridges
+        gdf = gdf[gdf['bridge'] != 'abandoned']
+
         cols_to_drop = []
         for col in gdf.columns:
             if any(isinstance(val, list) for val in gdf[col]):
                 cols_to_drop.append(col)
 
         # This a common and know duplicate column name (and others)
-        bad_column_names = ["atv", "fixme", "FIXME"]
+        bad_column_names = [
+            "atv",
+            "fixme",
+            "FIXME",
+            "NYSDOT_ref",
+            "REF",
+            "fid",
+            "fixme:maxspeed",
+            "LAYER",
+            "unsigned_ref",
+            "Fut_Ref",
+            "Ref",
+            "FIXME:ref",
+            "id",
+        ]
         for bad_cn in bad_column_names:
             if bad_cn in gdf.columns:
                 cols_to_drop.append(bad_cn)
@@ -57,7 +119,37 @@ def pull_osm_features_by_huc(huc_bridge_file, huc_num, huc_geom):
         gdf1 = gdf[gdf.geometry.apply(lambda x: x.geom_type == 'LineString')]
 
         gdf1 = gdf1.to_crs(CRS)
-        gdf1.to_file(huc_bridge_file, driver="GPKG")
+
+        # Perform dissolve touching lines
+        buffered = gdf1.copy()
+        buffered['geometry'] = buffered['geometry'].buffer(0.0001)
+        # Find groups of touching geometries
+        touching_groups = find_touching_groups(buffered)
+
+        # Dissolve each group separately
+        warnings.filterwarnings('ignore')
+        dissolved_groups = []
+        for group in touching_groups:
+            group_gdf = buffered.loc[list(group)]
+            if not group_gdf.empty:
+                dissolved_group = group_gdf.dissolve()
+                single_part_group = dissolved_group.explode(index_parts=False)
+                dissolved_groups.append(single_part_group)
+
+        # Combine dissolved groups and reconstruct GeoDataFrame
+        if dissolved_groups:
+            dissolved_gdf = pd.concat(dissolved_groups, ignore_index=True)
+            final_gdf = gpd.GeoDataFrame(dissolved_gdf, crs=buffered.crs)
+        else:
+            final_gdf = buffered.copy()
+
+        # Polygon to linestring
+        final_gdf['geometry'] = final_gdf['geometry'].apply(
+            lambda geom: LineString(geom.exterior.coords) if geom.geom_type == 'Polygon' else geom
+        )
+        # Reconstruct the GeoDataFrame to remove fragmentation
+        final_gdf = final_gdf.copy()
+        final_gdf.to_file(huc_bridge_file, driver="GPKG")
 
         # returns the HUC but only if it failed so we can keep a list of failed HUCs
         return ""
@@ -95,7 +187,7 @@ def combine_huc_features(output_dir):
     section_time = dt.datetime.now(dt.timezone.utc)
     logging.info(f"  .. started: {section_time.strftime('%m/%d/%Y %H:%M:%S')}")
 
-    all_bridges_gdf = all_bridges_gdf_raw[['osmid', 'name', 'geometry']]
+    all_bridges_gdf = all_bridges_gdf_raw[['osmid', 'name', 'bridge_type', 'geometry']]
     all_bridges_gdf.to_file(osm_bridge_file, driver="GPKG")
 
     return
@@ -185,6 +277,15 @@ def process_osm_bridges(wbd_file, output_folder, number_of_jobs):
     # all huc8 processing must be completed before this function call
     combine_huc_features(output_folder)
 
+    # Clean up individual HUC8 files
+    logging.info('Deleting individual HUC8 files as a final cleanup step')
+    huc_files = Path(output_folder).glob('huc_*_osm_bridges.gpkg')
+    for huc_file in huc_files:
+        try:
+            os.remove(huc_file)
+        except Exception as e:
+            logging.info(f"Error deleting {huc_file}: {str(e)}")
+
     if len(failed_HUCs_list) > 0:
         logging.info("\n+++++++++++++++++++")
         logging.info("HUCs that failed to download from OSM correctly are:")
@@ -236,6 +337,7 @@ def __setup_logger(outputs_dir):
     logging.basicConfig(
         filename=log_file_path, level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%H:%M:%S'
     )
+    logging.captureWarnings(True)
 
     # set up logging to console
     console = logging.StreamHandler()
@@ -260,7 +362,7 @@ def __setup_logger(outputs_dir):
         It should be run only as often as the user thinks OSM has had any important updates.
         - As written, the code will skip any HUC that there's already a file for.
         - Each HUC8's worth of OSM bridge features is saved out individually, then merged together
-        into one. The HUC8 files can be deleted if desired, as an added final cleanup step.
+        into one.
     '''
 
     parser = argparse.ArgumentParser(description='Acquires and saves Open Street Map bridge features')
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 293609513..64f33c963 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.11.0 - 2024-10-11 - [PR#1298](https://github.com/NOAA-OWP/inundation-mapping/pull/1298)
+
+This PR addresses four issues regarding OSM bridges. It dissolves touching bridge lines so each bridge has a single linestring. It also removes abandoned bridge from the dataset and it adds bridge type field to bridge centroids. As part of this PR, `max_hand_ft` and `max_discharge_cfs` columns are added to `osm_bridge_centroids.gkpg`.
+
+### Changes
+
+- `data/bridges/pull_osm_bridges.py`
+- `src/heal_bridges_osm.py`
+
+<br/><br/>
+
+
 ## v4.5.10.3 - 2024-10-11 - [PR#1306](https://github.com/NOAA-OWP/inundation-mapping/pull/1306)
 
 Extends outlet levelpath(s) outside HUC.
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 02ec3e663..1d538e625 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -7,7 +7,7 @@ export                             ALASKA_CRS=EPSG:3338
 
 # NOTE: $inputsDir is defined in Dockerfile
 
-export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/20240917
+export                       pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/20241002
 
 export                              input_DEM=${inputsDir}/dems/3dep_dems/10m_5070/20240916/hand_seamless_3dep_dem_10m_5070.vrt
 export                       input_DEM_Alaska=${inputsDir}/dems/3dep_dems/10m_South_Alaska/20240912/FIM_3dep_dem_South_Alaska_10m.vrt
@@ -32,7 +32,7 @@ export                          input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG
 export                   input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg
 export                 input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
 export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
-export                            osm_bridges=${inputsDir}/osm/bridges/240426/osm_all_bridges.gpkg
+export                            osm_bridges=${inputsDir}/osm/bridges/241002/osm_all_bridges.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
 export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv
diff --git a/src/heal_bridges_osm.py b/src/heal_bridges_osm.py
index 13c3b1c65..5b060b915 100644
--- a/src/heal_bridges_osm.py
+++ b/src/heal_bridges_osm.py
@@ -94,6 +94,12 @@ def flows_from_hydrotable(bridge_pnts, hydroTable):
         axis=1,
         result_type='expand',
     )
+    # Convert stages and dischrages to ft and cfs respectively
+    bridge_pnts['max_hand_ft'] = bridge_pnts['max_hand'] * 3.28084
+    bridge_pnts['max_hand_75_ft'] = bridge_pnts['max_hand_75'] * 3.28084
+    bridge_pnts['max_discharge_cfs'] = bridge_pnts['max_discharge'] * 35.3147
+    bridge_pnts['max_discharge_75_cfs'] = bridge_pnts['max_discharge75'] * 35.3147
+
     return bridge_pnts
 
 

From 853b08176d140546fcd1ebf029b9dc07d4676509 Mon Sep 17 00:00:00 2001
From: MattLuck-NOAA <matt.luck@noaa.gov>
Date: Mon, 21 Oct 2024 11:18:20 -0400
Subject: [PATCH 357/359] v4.5.11.1 Fix merge conflict bugs (#1318)

---
 docs/CHANGELOG.md       | 12 ++++++++++++
 src/aggregate_by_huc.py |  2 +-
 src/bash_variables.env  |  2 +-
 src/stream_branches.py  | 40 +++++++++++++++++++---------------------
 4 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 64f33c963..2498181f6 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,18 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.11.1 - 2024-10-16 - [PR#1318](https://github.com/NOAA-OWP/inundation-mapping/pull/1318)
+
+Bug fixes to address issues during `fim_pipeline.sh`.
+
+### Changes
+
+- `src/`
+    - `aggregate_by_huc.py`: Fix `pyogrio` field error.
+    - `stream_branches.py`: Remove `bids_temp` and fix index.
+
+<br/><br/>
+
 ## v4.5.11.0 - 2024-10-11 - [PR#1298](https://github.com/NOAA-OWP/inundation-mapping/pull/1298)
 
 This PR addresses four issues regarding OSM bridges. It dissolves touching bridge lines so each bridge has a single linestring. It also removes abandoned bridge from the dataset and it adds bridge type field to bridge centroids. As part of this PR, `max_hand_ft` and `max_discharge_cfs` columns are added to `osm_bridge_centroids.gkpg`.
diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py
index e3d8db6e9..93c0242b7 100644
--- a/src/aggregate_by_huc.py
+++ b/src/aggregate_by_huc.py
@@ -288,7 +288,7 @@ def agg_function(
                         (c > 1) & (bridge_pnts.feature_id != bridge_pnts.crossing_feature_id), 'is_backwater'
                     ] = 1
                     # Write file
-                    bridge_pnts.to_file(bridge_pnts_file, index=False)
+                    bridge_pnts.to_file(bridge_pnts_file, index=False, engine='fiona')
 
             # print(f"agg_by_huc for huc id {huc_id} is done")
 
diff --git a/src/bash_variables.env b/src/bash_variables.env
index 1d538e625..fff5d7857 100644
--- a/src/bash_variables.env
+++ b/src/bash_variables.env
@@ -31,7 +31,7 @@ export                  input_nwm_lakes_Alaska=${inputsDir}/nwm_hydrofabric/nwm_
 export                          input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
 export                   input_WBD_gdb_Alaska=${inputsDir}/wbd/WBD_National_South_Alaska.gpkg
 export                 input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
-export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
+export                        bathymetry_file=${inputsDir}/bathymetry/bathymetric_adjustment_data.gpkg
 export                            osm_bridges=${inputsDir}/osm/bridges/241002/osm_all_bridges.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
diff --git a/src/stream_branches.py b/src/stream_branches.py
index 6f9c02641..adc3b75e7 100755
--- a/src/stream_branches.py
+++ b/src/stream_branches.py
@@ -482,7 +482,7 @@ def derive_inlet_points_by_feature(self, branch_id_attribute, outlet_linestring_
                 row.loc[row[branch_id_attribute].duplicated(), branch_id_attribute] = np.nan
                 row = row.dropna(subset=[branch_id_attribute])
 
-            feature_inlet_point = Point(row.geometry[0].coords[inlet_linestring_index])
+            feature_inlet_point = Point(row.geometry[idx].coords[inlet_linestring_index])
 
             feature_inlet_points_gdf.loc[idx, "geometry"] = feature_inlet_point
 
@@ -1030,7 +1030,7 @@ def add_outlet_segments(
                 extended_id = outlet_id
 
             # add outlet segment to stream network
-            idx = self_extended['bids_temp'] == outlet.levpa_id
+            idx = self_extended[branch_id_attribute] == outlet.levpa_id
             if self_extended.loc[idx].empty:
                 idx = self_extended['ID'] == extended_id
 
@@ -1068,9 +1068,6 @@ def add_outlet_segments(
             exclude_indices = [False if i in values_excluded else True for i in self[attribute_excluded]]
             self = self.loc[exclude_indices, :]
 
-        # dissolve lines
-        self["bids_temp"] = self.loc[:, branch_id_attribute].copy()
-
         wbd = gpd.read_file(wbd)
         wbd = wbd.drop('shape_Length', axis=1)
 
@@ -1093,9 +1090,23 @@ def add_outlet_segments(
         s_in_wbd = gpd.sjoin(s, wbd)
         s_not_in_wbd = s[~s['ID'].isin(s_in_wbd['ID'])]
 
+        # Make a copy of the stream network
         self_copy = self.copy(deep=True)
 
-        self = self_in_wbd.dissolve(by=branch_id_attribute)
+        # Dissolve levelpath(s)
+        self = self_in_wbd.dissolve(by=branch_id_attribute, as_index=False)
+
+        # Fix ID and to attributes to downstream segment (these may be incorrect after dissolve)
+        for idx, row in self.iterrows():
+            # Get all segments of the levelpath
+            self_copy_levpa = self_copy.loc[self_copy[branch_id_attribute] == row[branch_id_attribute]]
+
+            # Get downstream segment of self_copy_levpa (self_copy_levpa.to not in self_copy_levpa.ID)
+            ds_outlet = self_copy_levpa.loc[~self_copy_levpa['to'].isin(self_copy_levpa['ID'])]
+
+            # Update self.ID and self.to to the downstream segment
+            self.loc[idx, 'ID'] = ds_outlet['ID'].values[0]
+            self.loc[idx, 'to'] = ds_outlet['to'].values[0]
 
         self["order_"] = max_stream_order.values
 
@@ -1105,7 +1116,7 @@ def add_outlet_segments(
             # For each outlet
             for outlet in s_not_in_wbd.itertuples():
                 # Select segments in levelpath
-                temp_df = self_copy[self_copy[branch_id_attribute] == outlet.bids_temp]
+                temp_df = self_copy[self_copy[branch_id_attribute] == outlet.levpa_id]
 
                 # Check if the levelpath outlet is external
                 if not len(temp_df.merge(self_in_wbd, left_on='to', right_on='ID')) == len(temp_df):
@@ -1137,7 +1148,7 @@ def add_outlet_segments(
                 values_excluded=values_excluded,
             )
 
-            outlets_extended = outlets_extended.rename(columns={'bids_temp': branch_id_attribute})
+            # outlets_extended = outlets_extended.rename(columns={'bids_temp': branch_id_attribute})
 
         # merges each multi-line string to a singular linestring
         for lpid, row in tqdm(
@@ -1165,19 +1176,6 @@ def add_outlet_segments(
             if verbose:
                 print("Writing dissolved branches ...")
 
-            # if out_vector_files is not None:
-            #     # base_file_path,extension = splitext(out_vector_files)
-
-            #     if verbose:
-            #         print("Writing dissolved branches ...")
-
-            # for bid in tqdm(self.loc[:,branch_id_attribute],total=len(self),disable=(not verbose)):
-            # out_vector_file = "{}_{}{}".format(base_file_path,bid,extension)
-
-            # bid_indices = self.loc[:,branch_id_attribute] == bid
-            # current_stream_network = StreamNetwork(self.loc[bid_indices,:])
-
-            # current_stream_network.write(out_vector_file,index=False)
             self.write(out_vector_files, index=False)
 
         if out_extended_vector_files is not None and not s_not_in_wbd.empty:

From 789618dedb4588a25c06f46159ec1e00d5d7e6b5 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 25 Oct 2024 19:03:50 +0000
Subject: [PATCH 358/359] v4.5.11.2 docker file for OWP, non root user (#1322)

---
 Dockerfile => Dockerfile.dev |   0
 Dockerfile.prod              | 129 +++++++++++++++++++++++++++++++++++
 README.md                    |  12 ++--
 docs/CHANGELOG.md            |  24 +++++++
 fim_pipeline.sh              |   2 +-
 fim_post_processing.sh       |   2 +-
 fim_pre_processing.sh        |   4 ++
 fim_process_unit_wb.sh       |   2 +
 8 files changed, 168 insertions(+), 7 deletions(-)
 rename Dockerfile => Dockerfile.dev (100%)
 create mode 100644 Dockerfile.prod

diff --git a/Dockerfile b/Dockerfile.dev
similarity index 100%
rename from Dockerfile
rename to Dockerfile.dev
diff --git a/Dockerfile.prod b/Dockerfile.prod
new file mode 100644
index 000000000..5b702e75e
--- /dev/null
+++ b/Dockerfile.prod
@@ -0,0 +1,129 @@
+## Temporary image to build the libraries and only save the needed artifacts
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.4 AS builder
+WORKDIR /opt/builder
+ARG dataDir=/data
+ARG projectDir=/foss_fim
+ARG depDir=/dependencies
+ARG taudemVersion=98137bb6541a0d0077a9c95becfed4e56d0aa0ac
+ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
+ENV taudemDir=$depDir/taudem/bin
+ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
+
+# remove reference to missing repo
+# RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+
+RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/dtarb/taudem.git
+RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
+
+RUN apt-get update --fix-missing && apt-get install -y cmake mpich \
+    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/*
+
+## Compile Main taudem repo ##
+RUN mkdir -p taudem/bin
+RUN cd taudem \
+    && git checkout $taudemVersion \
+    && cd src \
+    && make
+
+## Compile taudem repo with accelerated flow directions ##
+RUN cd taudem_accelerated_flowDirections/taudem \
+    && git checkout $taudemVersion2 \
+    && mkdir build \
+    && cd build \
+    && cmake .. \
+    && make
+
+RUN mkdir -p $taudemDir
+RUN mkdir -p $taudemDir2
+
+## Move needed binaries to the next stage of the image
+RUN cd taudem/bin && mv -t $taudemDir flowdircond streamnet gagewatershed catchhydrogeo dinfdistdown
+RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d8flowdir dinfflowdir
+
+
+###############################################################################################
+# Base Image that has GDAL, PROJ, etc
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.4
+ARG dataDir=/data
+ENV projectDir=/foss_fim
+ARG depDir=/dependencies
+ENV inputsDir=$dataDir/inputs
+ENV outputsDir=/outputs
+ENV srcDir=$projectDir/src
+ENV toolsDir=$projectDir/tools
+ENV workDir=/fim_temp
+ENV taudemDir=$depDir/taudem/bin
+ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
+
+## ADDING FIM GROUP ##
+ARG GroupID=1370800235
+ARG GroupName=fim
+RUN addgroup --gid $GroupID $GroupName
+ENV GID=$GroupID
+ENV GN=$GroupName
+
+RUN mkdir -p $workDir
+RUN mkdir -p $depDir
+COPY --from=builder $depDir $depDir
+
+# remove reference to missing repo
+# RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+
+RUN apt-get update --fix-missing && rm -rf /var/lib/apt/lists/*
+RUN apt update --fix-missing
+
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata wget
+
+RUN apt auto-remove
+
+## adding AWS CLI (for bash) ##
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+   unzip awscliv2.zip && \
+   ./aws/install
+
+## adding environment variables for numba and python ##
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+ENV PYTHONUNBUFFERED=TRUE
+
+## ADD TO PATHS ##
+ENV PATH="$projectDir:${PATH}"
+ENV PYTHONPATH="${PYTHONPATH}:${projectDir}:${srcDir}:${srcDir}/utils:${projectDir}/data:${toolsDir}"
+
+## install python 3 modules ##
+COPY Pipfile .
+COPY Pipfile.lock .
+RUN pip3 install pipenv==2024.0.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
+
+# ----------------------------------
+# Mar 2023 / Sep 2024
+# There are some nuances in the whitebox python downloads in that the first time it loads
+# it goes to the internet and downloads the latest/greatest WBT (whiteboxtools) engine which is
+# required for the whitebox python library to work. We don't want to have FIM attempting a download
+# each time a container is opened and the whitebox engine is called.
+# Instead we will setup the WBT engine at time of docker build (same as Taudem and AWS).
+# Whitebox code detects that the engine it there and makes no attempt to update it.
+# We download and unzip it to the same file folder that pip deployed the whitebox library.
+# Whitebox also attempts to always download a folder called testdata regardless of use.
+# We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
+ENV WBT_PATH=/usr/local/lib/python3.10/dist-packages/whitebox/WBT
+RUN wget -P $WBT_PATH https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
+    unzip -o $WBT_PATH/WhiteboxTools_linux_musl.zip -d $WBT_PATH && \
+    cp $WBT_PATH/WhiteboxTools_linux_amd64/WBT/whitebox_tools $WBT_PATH
+# ----------------------------------
+
+# The containiner will auto use this account to run
+ARG RuntimeUser=svc_user
+RUN useradd -u 8877 -g $GroupName -s /bin/bash $RuntimeUser
+RUN chmod 777 $workDir
+RUN mkdir -p "/home/${RuntimeUser}"
+RUN chmod 777 /home/$RuntimeUser
+
+## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
+ADD ./src/entrypoint.sh /
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
+
+## This results in the default user being the svc_user user
+USER $RuntimeUser
diff --git a/README.md b/README.md
index 7bb7207c6..8e3d786e9 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,8 @@ This software uses the Height Above Nearest Drainage (HAND) method to generate R
 
 # FIM Version 4
 
+#### Note: While we use the phrase "FIM" regularily, the phrase "HAND" is also used and is generally interchangeable. Most output folders now follow the convenction of "hand_4_x_x_x".
+
 ## Accessing Data through ESIP S3 Bucket
 The latest national generated HAND data and a subset of the inputs can be found in an Amazon S3 Bucket hosted by [Earth Science Information Partners (ESIP)](https://www.esipfed.org/). These data can be accessed using the AWS CLI tools. Please contact Carson Pruitt (carson.pruitt@noaa.gov) or Fernando Salas (fernando.salas@noaa.gov) if you experience issues with permissions.
 
@@ -49,12 +51,12 @@ aws s3 ls s3://noaa-nws-owp-fim/hand_fim/  --profile esip
 
 Download a directory of sample outputs for a single HUC8:
 ```
-aws s3 sync s3://noaa-nws-owp-fim/hand_fim/outputs/fim_4_4_0_0/12090301 \
+aws s3 sync s3://noaa-nws-owp-fim/hand_fim/outputs/hand_4_5_2_11/12090301 \
     /your_local_folder_name/12090301 --profile esip
 ```
-By adjusting pathing, you can also download entire directories such as the `fim_4_4_0_0` folder. An entire output FIM set (e.g. `fim_4_4_0_0`) is approximately 1.1 TB.
+By adjusting pathing, you can also download entire directories such as the `hand_4_5_2_11` folder. An entire output HAND set is approximately 1.7 TB.
 
-**Note**: There may be newer editions than `fim_4_4_0_0`, and it is recommended to adjust the command above for the latest version.
+**Note**: There may be newer editions than `hand_4_5_11_1`, and it is recommended to adjust the command above for the latest version.
 
 ## Setting up your Environment
 
@@ -85,7 +87,7 @@ Git will auto create a subfolder named `inundation-mapping` where the code will
 
 ### Installation
 1. Install Docker : [Docker](https://docs.docker.com/get-docker/)
-2. Build Docker Image : `docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>`
+2. Build Docker Image : `docker build -f Dockerfile.dev -t <image_name>:<tag> <path/to/repository>`
 3. Create FIM group on host machine:
     - Linux: `groupadd -g 1370800178 fim`
 4. Change group ownership of repo (needs to be redone when a new file occurs in the repo):
@@ -128,7 +130,7 @@ docker run --rm -it --name <your_container_name> \
 ```
 For example:
 ```bash
-docker run --rm -it --name robs_container \
+docker run --rm -it --name Robs_container \
     -v /home/projects/fim/code/inundation-mapping/:/foss_fim \
     -v /home/projects/fim/data/outputs/:/outputs \
     -v /home/projects/fim/data/outputs_temp/:/fim_temp \
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2498181f6..4618b15f7 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,30 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.11.2 - 2024-10-25 - [PR#1322](https://github.com/NOAA-OWP/inundation-mapping/pull/1322)
+
+For security reasons, we needed to create a docker image that does not use the root user in anyway. The new `Dockerfile.prod` file is to be used when we want to use a non-root user. The  original `Dockerfile` has been renamed to `Dockerfile.dev` and will continue to use it's root users which has no problems with interacting with external mounts.
+
+Note: Re: using pip or pipenv installs.
+In the Dockerfile.prod, you can not do installs or update using either pipenv or pip.  Those types of tests and adjustments need to be done in the `Dockerfile.dev`. `Dockerfile.dev` will also allow change to the `Pipfile` and `Pipfile.lock` . Both docker files share the Pipfiles so it should be just fine.
+
+### File Renames
+- Was: `Dockerfile`,  now `Dockerfile.dev`
+
+### Additions
+
+- Dockerfile.prod: as described
+
+### Changes
+- `README.md`: change notes from phrase `Dockerfile` to `Dockerfile.dev`. Also added some notes about the new convention of outputs no longer starting with `fim_` but now `hand_`
+- `fim_pipeline.sh`: Change for the new `Dockerfile.prod` for permissions.
+- `fim_post_processing.sh`: Change for the new `Dockerfile.prod` for permissions.
+- `fim_pre_processing.sh`: Change for the new `Dockerfile.prod` for permissions.
+- `fim_process_unit_wb.sh`: Change for the new `Dockerfile.prod` for permissions.
+
+<br/><br/>
+
+
 ## v4.5.11.1 - 2024-10-16 - [PR#1318](https://github.com/NOAA-OWP/inundation-mapping/pull/1318)
 
 Bug fixes to address issues during `fim_pipeline.sh`.
diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 3934c27a7..35987e0f1 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -121,7 +121,7 @@ echo "---------------------------------------------------"
 ## POST PROCESSING
 
 # Remove run from the fim_temp directory
-rm -d $workDir/$runName
+rm -df $workDir/$runName
 
 # Pipe into post processing
 . $projectDir/fim_post_processing.sh -n $runName -j $jobMaxLimit
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index e866f3c9d..ccf2e10df 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -267,7 +267,7 @@ Tcount
 l_echo $startDiv"Resetting Permissions"
 Tstart
     # super slow to change chmod on the log folder. Not really manditory anyways
-    find $outputDestDir -maxdepth 1 -type f -exec chmod 666 {} +  # just root level files
+    find $outputDestDir -maxdepth 1 -type f -exec chmod 777 {} +  # just root level files
 Tcount
 
 
diff --git a/fim_pre_processing.sh b/fim_pre_processing.sh
index bd556940c..658377db8 100755
--- a/fim_pre_processing.sh
+++ b/fim_pre_processing.sh
@@ -204,6 +204,7 @@ if [ ! -d $outputDestDir ]; then
     mkdir -p $outputDestDir
     chmod 777 $outputDestDir
     mkdir -p $tempRunDir
+	chmod 777 $tempRunDir
 else
     # remove these directories and files on a new or overwrite run
     rm -rdf $outputDestDir/logs
@@ -231,6 +232,9 @@ cp $envFile $outputDestDir/params.env
 
 args_file=$outputDestDir/runtime_args.env
 
+# reset it again (this time recursive for the new incoming folders
+chmod 777 -R $outputDestDir
+
 # the jobHucLimit is not from the args files, only jobBranchLimit
 echo "export runName=$runName" >> $args_file
 echo "export jobHucLimit=$jobHucLimit" >> $args_file
diff --git a/fim_process_unit_wb.sh b/fim_process_unit_wb.sh
index 1c013f777..962b1df15 100755
--- a/fim_process_unit_wb.sh
+++ b/fim_process_unit_wb.sh
@@ -78,6 +78,8 @@ fi
 # make outputs directory
 mkdir -p $tempHucDataDir
 mkdir -p $tempBranchDataDir
+chmod 777 $tempHucDataDir
+chmod 777 $tempBranchDataDir
 
 # Clean out previous unit logs and branch logs starting with this huc
 rm -f $outputDestDir/logs/unit/"$hucNumber"_unit.log

From 889a7854613e314ae0e57471332a795bc357541f Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Fri, 25 Oct 2024 19:07:58 +0000
Subject: [PATCH 359/359] 4.5.11.3 Fix error and warning scan bug (#1320)

---
 docs/CHANGELOG.md      | 13 +++++++++++++
 fim_post_processing.sh |  4 ++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4618b15f7..f818d53da 100755
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.5.11.3 - 2024-10-25 - [PR#1320](https://github.com/NOAA-OWP/inundation-mapping/pull/1320)
+
+The fix: During the post processing scan for the word "error" or "warning", it was only finding records which had either of those two words as stand alone words and not part of bigger phrases.  ie); "error" was found, but not "fielderror". Added wildcards and it is now fixed.
+
+Note: it is finding a good handful more errors and warnings that were being missed in earlier code versions.
+
+### Changes
+`fim_post_processing.sh`: fix as described.
+
+<br/><br/>
+
+
 ## v4.5.11.2 - 2024-10-25 - [PR#1322](https://github.com/NOAA-OWP/inundation-mapping/pull/1322)
 
 For security reasons, we needed to create a docker image that does not use the root user in anyway. The new `Dockerfile.prod` file is to be used when we want to use a non-root user. The  original `Dockerfile` has been renamed to `Dockerfile.dev` and will continue to use it's root users which has no problems with interacting with external mounts.
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index ccf2e10df..91db004d6 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -275,11 +275,11 @@ l_echo $startDiv"Scanning logs for errors and warnings. This can take quite a fe
 echo "Results will be saved in root not inside the log folder."
 Tstart
     # grep -H -r -i -n "error" $outputDestDir/logs/ > $outputDestDir/all_errors_from_logs.log
-    find $outputDestDir -type f | grep -H -r -i -n "error" $outputDestDir/logs/ > \
+    find $outputDestDir -type f | grep -H -R -i -n ".*error.*" $outputDestDir/logs/ > \
          $outputDestDir/all_errors_from_logs.log &
     l_echo "error scan done, now on to warnings scan"
 
-    find $outputDestDir -type f | grep -H -r -i -n "warning" $outputDestDir/logs/ > \
+    find $outputDestDir -type f | grep -H -R -i -n ".*warning.*" $outputDestDir/logs/ > \
          $outputDestDir/all_warnings_from_logs.log &
     l_echo "warning scan done"
 Tcount